familia 2.0.0.pre18 → 2.0.0.pre21

data/try/features/relationships/indexing_rebuild_try.rb

@@ -0,0 +1,600 @@
+# try/features/relationships/indexing_rebuild_try.rb
+#
+# Comprehensive tests for index rebuild functionality
+# Tests both unique_index (class-level and instance-scoped) and multi_index rebuild
+
+require_relative '../../support/helpers/test_helpers'
+
+Familia.enable_database_logging = true
+
+# Test classes for class-level unique index rebuild
+class ::RebuildTestUser < Familia::Horreum
+  feature :relationships
+
+  identifier_field :user_id
+  field :user_id
+  field :email
+  field :username
+  field :department
+
+  unique_index :email, :email_lookup
+  unique_index :username, :username_lookup
+
+  class_sorted_set :instances, reference: true
+end
+
+# Test classes for instance-scoped unique index rebuild
+# Define Company first so Employee can reference it
+class ::RebuildTestCompany < Familia::Horreum
+  feature :relationships
+
+  identifier_field :company_id
+  field :company_id
+  field :name
+
+  sorted_set :employees
+
+  class_sorted_set :instances, reference: true
+end
+
+class ::RebuildTestEmployee < Familia::Horreum
+  feature :relationships
+
+  identifier_field :emp_id
+  field :emp_id
+  field :email
+  field :badge_number
+  field :department
+
+  unique_index :badge_number, :badge_index, within: RebuildTestCompany
+  multi_index :department, :dept_index, within: RebuildTestCompany
+
+  participates_in RebuildTestCompany, :employees, score: :emp_id
+
+  class_sorted_set :instances, reference: true
+end
+
+# Setup: Create test data for class-level unique index
+@user1 = RebuildTestUser.new(user_id: "user_1", email: "user1@test.com", username: "user1")
+@user1.save
+@user2 = RebuildTestUser.new(user_id: "user_2", email: "user2@test.com", username: "user2")
+@user2.save
+@user3 = RebuildTestUser.new(user_id: "user_3", email: "user3@test.com", username: "user3")
+@user3.save
+
+# Setup: Create test data for instance-scoped indexes
+@company = RebuildTestCompany.new(company_id: "company_1", name: "Acme Corp")
+@company.save
+
+@emp1 = RebuildTestEmployee.new(emp_id: "emp_1", email: "emp1@acme.com", badge_number: "BADGE001", department: "engineering")
+@emp1.save
+@emp1.add_to_rebuild_test_company_employees(@company)
+
+@emp2 = RebuildTestEmployee.new(emp_id: "emp_2", email: "emp2@acme.com", badge_number: "BADGE002", department: "sales")
+@emp2.save
+@emp2.add_to_rebuild_test_company_employees(@company)
+
+@emp3 = RebuildTestEmployee.new(emp_id: "emp_3", email: "emp3@acme.com", badge_number: "BADGE003", department: "engineering")
+@emp3.save
+@emp3.add_to_rebuild_test_company_employees(@company)
+
+# =============================================
+# 1. Class-Level Unique Index Rebuild Tests
+# =============================================
+
+## Class-level rebuild method exists
+RebuildTestUser.respond_to?(:rebuild_email_lookup)
+#=> true
+
+## Index starts empty before rebuild
+RebuildTestUser.email_lookup.clear
+RebuildTestUser.email_lookup.size
+#=> 0
+
+## Rebuild returns count of indexed objects
+count = RebuildTestUser.rebuild_email_lookup
+count
+#=> 3
+
+## Index size matches object count after rebuild
+RebuildTestUser.email_lookup.size
+#=> 3
+
+## Find by email works after rebuild
+found = RebuildTestUser.find_by_email("user1@test.com")
+found&.user_id
+#=> "user_1"
+
+## All users are findable after rebuild
+found = RebuildTestUser.find_by_email("user2@test.com")
+found&.user_id
+#=> "user_2"
+
+## Third user is findable after rebuild
+found = RebuildTestUser.find_by_email("user3@test.com")
+found&.user_id
+#=> "user_3"
+
+## Bulk query works after rebuild
+emails = ["user1@test.com", "user3@test.com"]
+found_users = RebuildTestUser.find_all_by_email(emails)
+found_users.map(&:user_id).sort
+#=> ["user_1", "user_3"]
+
+## Rebuild can be called multiple times
+count = RebuildTestUser.rebuild_email_lookup
+count
+#=> 3
+
+## Index remains consistent after multiple rebuilds
+RebuildTestUser.email_lookup.size
+#=> 3
+
+## Rebuild for second unique index works independently
+RebuildTestUser.username_lookup.clear
+count = RebuildTestUser.rebuild_username_lookup
+count
+#=> 3
+
+## Second index is populated correctly
+found = RebuildTestUser.find_by_username("user2")
+found&.user_id
+#=> "user_2"
+
+# =============================================
+# 2. Class-Level Index Rebuild Edge Cases
+# =============================================
+
+## Rebuild with nil field values skips those objects
+@user_nil = RebuildTestUser.new(user_id: "user_nil", email: nil, username: "user_nil")
+@user_nil.save
+RebuildTestUser.email_lookup.clear
+count = RebuildTestUser.rebuild_email_lookup
+count
+#=> 3
+
+## Nil field values are not in index
+RebuildTestUser.find_by_email("")
+#=> nil
+
+## Non-nil fields are still indexed for object with nil field
+found = RebuildTestUser.find_by_username("user_nil")
+found&.user_id
+#=> "user_nil"
+
+## Rebuild with empty string field values skips those objects
+@user_empty = RebuildTestUser.new(user_id: "user_empty", email: "", username: "user_empty")
+@user_empty.save
+RebuildTestUser.email_lookup.clear
+count = RebuildTestUser.rebuild_email_lookup
+count
+#=> 3
+
+## Empty string field values are not in index
+RebuildTestUser.find_by_email("")
+#=> nil
+
+## Rebuild with whitespace-only field values skips those objects
+@user_whitespace = RebuildTestUser.new(user_id: "user_ws", email: "  ", username: "user_ws")
+@user_whitespace.save
+RebuildTestUser.email_lookup.clear
+count = RebuildTestUser.rebuild_email_lookup
+count
+#=> 3
+
+## Whitespace-only field values are not in index
+RebuildTestUser.find_by_email("  ")
+#=> nil
+
+## Rebuild handles stale object IDs gracefully
+RebuildTestUser.instances.add("stale_user_id")
+RebuildTestUser.email_lookup.clear
+count = RebuildTestUser.rebuild_email_lookup
+count
+#=> 3
+
+## Index size is correct despite stale ID
+RebuildTestUser.email_lookup.size
+#=> 3
+
+## Rebuild with no instances returns zero
+RebuildTestUser.instances.clear
+RebuildTestUser.email_lookup.clear
+count = RebuildTestUser.rebuild_email_lookup
+count
+#=> 0
+
+## Index is empty after rebuilding with no instances
+RebuildTestUser.email_lookup.size
+#=> 0
+
+# Restore instances for remaining tests
+## Check instances key
+RebuildTestUser.instances.dbkey
+#=> "rebuild_test_user:instances"
+
+## Manually populate instances for testing
+result1 = RebuildTestUser.instances.add("user_1")
+result2 = RebuildTestUser.instances.add("user_2")
+result3 = RebuildTestUser.instances.add("user_3")
+[result1, result2, result3]
+#=> [true, true, true]
+
+## Instances restored successfully
+RebuildTestUser.instances.size
+#=> 3
+
+# =============================================
+# 3. Instance-Scoped Unique Index Rebuild
+# =============================================
+
+## Instance-scoped rebuild method exists
+@company.respond_to?(:rebuild_badge_index)
+#=> true
+
+## Instance-scoped index starts empty before rebuild
+@company.badge_index.clear
+@company.badge_index.size
+#=> 0
+
+## Instance-scoped rebuild returns count
+count = @company.rebuild_badge_index
+count
+#=> 3
+
+## Instance-scoped index size matches after rebuild
+@company.badge_index.size
+#=> 3
+
+## Find by badge works after instance-scoped rebuild
+found = @company.find_by_badge_number("BADGE001")
+found&.emp_id
+#=> "emp_1"
+
+## All employees findable after instance-scoped rebuild
+found = @company.find_by_badge_number("BADGE002")
+found&.emp_id
+#=> "emp_2"
+
+## Third employee findable after instance-scoped rebuild
+found = @company.find_by_badge_number("BADGE003")
+found&.emp_id
+#=> "emp_3"
+
+## Bulk query works after instance-scoped rebuild
+badges = ["BADGE001", "BADGE003"]
+found_emps = @company.find_all_by_badge_number(badges)
+found_emps.map(&:emp_id).sort
+#=> ["emp_1", "emp_3"]
+
+## Instance-scoped rebuild only indexes employees in that company
+@company2 = RebuildTestCompany.new(company_id: "company_2", name: "TechCo")
+@company2.save
+@company2.badge_index.clear
+count = @company2.rebuild_badge_index
+count
+#=> 0
+
+## Second company has empty index after rebuild
+@company2.badge_index.size
+#=> 0
+
+## First company still has correct index
+@company.badge_index.size
+#=> 3
+
+# =============================================
+# 4. Multi-Value Index Rebuild
+# =============================================
+
+## Multi-index rebuild method exists
+@company.respond_to?(:rebuild_dept_index)
+#=> true
+
+## Multi-index starts empty before rebuild
+engineering_set = @company.dept_index_for("engineering")
+engineering_set.clear
+sales_set = @company.dept_index_for("sales")
+sales_set.clear
+engineering_set.size
+#=> 0
+
+## Multi-index rebuild returns processed count
+count = @company.rebuild_dept_index
+count
+#=> 3
+
+## Manually populate departments to test structure
+@emp1.add_to_rebuild_test_company_dept_index(@company)
+@emp2.add_to_rebuild_test_company_dept_index(@company)
+@emp3.add_to_rebuild_test_company_dept_index(@company)
+engineering_count = @company.dept_index_for("engineering").size
+engineering_count
+#=> 2
+
+## Sales department has correct count
+sales_count = @company.dept_index_for("sales").size
+sales_count
+#=> 1
+
+## Find all by department works for engineering
+eng_emps = @company.find_all_by_department("engineering")
+eng_emps.map(&:emp_id).sort
+#=> ["emp_1", "emp_3"]
+
+## Find all by department works for sales
+sales_emps = @company.find_all_by_department("sales")
+sales_emps.map(&:emp_id)
+#=> ["emp_2"]
+
+## Sample from department returns employees
+sample = @company.sample_from_department("engineering", 1)
+["emp_1", "emp_3"].include?(sample.first&.emp_id)
+#=> true
+
+# =============================================
+# 5. Rebuild Progress Callbacks
+# =============================================
+
+## Instances collection has users before rebuild
+RebuildTestUser.instances.size
+#=> 3
+
+## Rebuild accepts batch_size parameter
+RebuildTestUser.email_lookup.clear
+count = RebuildTestUser.rebuild_email_lookup(batch_size: 1)
+count
+#=> 3
+
+## Index works correctly with small batch size
+found = RebuildTestUser.find_by_email("user1@test.com")
+found&.user_id
+#=> "user_1"
+
+## Rebuild accepts large batch_size parameter
+RebuildTestUser.email_lookup.clear
+count = RebuildTestUser.rebuild_email_lookup(batch_size: 1000)
+count
+#=> 3
+
+## Index works correctly with large batch size
+found = RebuildTestUser.find_by_email("user2@test.com")
+found&.user_id
+#=> "user_2"
+
+## Rebuild accepts progress callback block
+@progress_updates = []
+RebuildTestUser.email_lookup.clear
+count = RebuildTestUser.rebuild_email_lookup { |progress| @progress_updates << progress }
+count
+#=> 3
+
+## Progress callback receives updates
+@progress_updates.size > 0
+#=> true
+
+## Progress callback includes completed count
+@progress_updates.last[:completed]
+#=> 3
+
+## Progress callback includes total count
+@progress_updates.last[:total]
+#=> 3
+
+## Progress callback includes rate information
+@progress_updates.last.key?(:rate)
+#=> true
+
+## Progress updates are incremental
+completed_values = @progress_updates.map { |p| p[:completed] }
+completed_values.sort == completed_values
+#=> true
+
+# =============================================
+# 6. Rebuild with Modified Data
+# =============================================
+
+## Rebuild reflects updated field values
+@user1.email = "updated1@test.com"
+@user1.save
+RebuildTestUser.email_lookup.clear
+RebuildTestUser.rebuild_email_lookup
+found = RebuildTestUser.find_by_email("updated1@test.com")
+found&.user_id
+#=> "user_1"
+
+## Old email is not in index after rebuild
+RebuildTestUser.find_by_email("user1@test.com")
+#=> nil
+
+## Rebuild after deleting object removes from index
+@user3.destroy if @user3.respond_to?(:destroy)
+RebuildTestUser.instances.remove(@user3.identifier)
+RebuildTestUser.email_lookup.clear
+count = RebuildTestUser.rebuild_email_lookup
+count
+#=> 2
+
+## Deleted object not findable after rebuild
+RebuildTestUser.find_by_email("user3@test.com")
+#=> nil
+
+## Remaining objects still findable
+found = RebuildTestUser.find_by_email("user2@test.com")
+found&.user_id
+#=> "user_2"
+
+# =============================================
+# 7. Instance-Scoped Rebuild with Batch Sizes
+# =============================================
+
+## Instance-scoped rebuild accepts batch_size
+@company.badge_index.clear
+count = @company.rebuild_badge_index(batch_size: 1)
+count
+#=> 3
+
+## Instance-scoped index works with small batch
+found = @company.find_by_badge_number("BADGE001")
+found&.emp_id
+#=> "emp_1"
+
+## Instance-scoped rebuild with large batch_size
+@company.badge_index.clear
+count = @company.rebuild_badge_index(batch_size: 100)
+count
+#=> 3
+
+## Instance-scoped index works with large batch
+found = @company.find_by_badge_number("BADGE002")
+found&.emp_id
+#=> "emp_2"
+
+# =============================================
+# 8. Concurrent Rebuilds (Thread Safety)
+# =============================================
+
+## Multiple rebuilds don't corrupt index
+RebuildTestUser.email_lookup.clear
+counts = 3.times.map { RebuildTestUser.rebuild_email_lookup }
+counts
+#=> [2, 2, 2]
+
+## Index remains consistent after concurrent rebuilds
+RebuildTestUser.email_lookup.size
+#=> 2
+
+## All expected objects findable after concurrent rebuilds
+found1 = RebuildTestUser.find_by_email("updated1@test.com")
+found2 = RebuildTestUser.find_by_email("user2@test.com")
+[found1&.user_id, found2&.user_id].sort
+#=> ["user_1", "user_2"]
+
+# =============================================
+# 9. Orphaned Data Cleanup (SCAN-based)
+# =============================================
+
+## Clear all dept indexes from earlier tests
+["engineering", "sales", "marketing", "finance"].each do |dept|
+  @company.dept_index_for(dept).clear rescue nil
+end
+
+## Manually create orphaned stale data in finance dept
+@company.dept_index_for("finance").add("emp_1")
+@company.dept_index_for("finance").add("emp_2")
+@company.dept_index_for("finance").size
+#=> 2
+
+## Also add some marketing entries (will be orphaned)
+@company.dept_index_for("marketing").add("emp_1")
+@company.dept_index_for("marketing").add("emp_3")
+@company.dept_index_for("marketing").size
+#=> 2
+
+## Rebuild via participation collection processes 3 employees
+@company.rebuild_dept_index
+#=> 3
+
+## After rebuild: Current engineering dept correctly has both emp1 and emp3
+@company.dept_index_for("engineering").size
+#=> 2
+
+## After rebuild: Current sales dept correctly has emp2
+@company.dept_index_for("sales").size
+#=> 1
+
+## SCAN cleanup removes orphaned finance keys
+@company.dept_index_for("finance").size
+#=> 0
+
+## SCAN cleanup removes orphaned marketing keys
+@company.dept_index_for("marketing").size
+#=> 0
+
+# =============================================
+# 10. Scope Filtering (SCAN Strategy)
+# =============================================
+
+## Company 1 has 3 employees
+@company.employees.size
+#=> 3
+
+## Clear company 1 badge index to force SCAN strategy
+@company.badge_index.clear
+@company.badge_index.size
+#=> 0
+
+## Company has 3 employees participating
+@company.employees.size
+#=> 3
+
+## Rebuild company 1 index via SCAN - should filter to only company 1's employees
+count = @company.rebuild_badge_index
+count
+#=> 3
+
+## Company 1 index only has its own employees (scope filtering verified)
+@company.badge_index.size
+#=> 3
+
+## All expected employees found via index
+found1 = @company.find_by_badge_number("BADGE001")
+found2 = @company.find_by_badge_number("BADGE002")
+found3 = @company.find_by_badge_number("BADGE003")
+[found1&.emp_id, found2&.emp_id, found3&.emp_id]
+#=> ["emp_1", "emp_2", "emp_3"]
+
+# =============================================
+# 11. Cardinality Guard Protection
+# =============================================
+
+## Cardinality guard prevents multi-index corruption
+# Note: This would require manually calling the private method with wrong cardinality
+# The architecture prevents this via factory pattern, but guard provides explicit protection
+begin
+  # Simulate calling rebuild_via_participation with multi-index cardinality
+  Familia::Features::Relationships::Indexing::RebuildStrategies.rebuild_via_participation(
+    @company,
+    RebuildTestEmployee,
+    :department,
+    :add_to_rebuild_test_company_dept_index,
+    @company.employees,
+    :multi,  # Wrong cardinality!
+    batch_size: 100
+  )
+  "should have raised"
+rescue ArgumentError => e
+  e.message.include?("only supports unique indexes")
+end
+#=> true
+
+## Guard accepts correct cardinality (:unique)
+begin
+  index_config = RebuildTestEmployee.indexing_relationships.find { |r| r.index_name == :badge_index }
+  Familia::Features::Relationships::Indexing::RebuildStrategies.rebuild_via_participation(
+    @company,
+    RebuildTestEmployee,
+    :badge_number,
+    :add_to_rebuild_test_company_badge_index,
+    @company.employees,
+    :unique,  # Correct cardinality
+    batch_size: 100
+  )
+  "no error"
+rescue ArgumentError
+  "should not raise"
+end
+#=> "no error"
+
+# Teardown
+RebuildTestUser.email_lookup.clear
+RebuildTestUser.username_lookup.clear
+RebuildTestUser.instances.clear
+@company.badge_index.clear
+@company.employees.clear
+# Clear all department index keys
+["engineering", "sales", "marketing", "finance"].each do |dept|
+  @company.dept_index_for(dept).clear rescue nil
+end
+RebuildTestCompany.instances.clear
+RebuildTestEmployee.instances.clear

data/try/features/relationships/indexing_try.rb

@@ -1,5 +1,7 @@
 # try/features/relationships/indexing_try.rb
 #
+# frozen_string_literal: true
+
 # Comprehensive tests for Familia indexing relationships functionality
 # Tests both multi_index (parent-context) and unique_index (class-level) indexing
 #
@@ -62,7 +64,7 @@ end
 @user3.save
 
 @company_id = "comp_#{rand(10000000)}"
-@company = TestCompany.create(company_id: @company_id, name: 'Acme Corp')
+@company = TestCompany.create!(company_id: @company_id, name: 'Acme Corp')
 @emp1 = TestEmployee.new(emp_id: 'emp_001', email: 'alice@acme.com', department: 'engineering', manager_id: 'mgr_001', badge_number: 'BADGE001')
 @emp1.save
 @emp2 = TestEmployee.new(emp_id: 'emp_002', email: 'bob@acme.com', department: 'sales', manager_id: 'mgr_002', badge_number: 'BADGE002')
@@ -86,7 +88,7 @@ sample = @company.sample_from_department(@emp2.department)
 
 ## First indexing relationship has correct configuration
 config = @user1.class.indexing_relationships.first
-[config.field, config.index_name, config.target_class == TestUser, config.query]
+[config.field, config.index_name, config.scope_class == TestUser, config.query]
 #=> [:email, :email_lookup, true, true]
 
 ## Second indexing relationship has query disabled
@@ -189,7 +191,7 @@ TestUser.respond_to?(:find_by_username)
 
 ## Instance-scoped unique index has correct configuration
 config = @emp1.class.indexing_relationships.find { |r| r.field == :badge_number }
-[config.index_name, config.target_class, config.cardinality]
+[config.index_name, config.scope_class, config.cardinality]
 #=> [:badge_index, TestCompany, :unique]
 
 ## Target class gets finder method for unique index
@@ -251,6 +253,16 @@ found_emps = @company.find_all_by_badge_number('BADGE002')
 found_emps.map(&:emp_id)
 #=> ["emp_002"]
 
+## Instance-scoped bulk query filters nil inputs
+badges_with_nil = [nil, 'BADGE001', nil]
+found_emps = @company.find_all_by_badge_number(badges_with_nil)
+found_emps.map(&:emp_id)
+#=> ["emp_001"]
+
+## Instance-scoped bulk query with only nil returns empty
+@company.find_all_by_badge_number([nil, nil]).length
+#=> 0
+
 ## Update badge index entry
 old_badge = @emp1.badge_number
 @emp1.badge_number = 'BADGE001_NEW'
@@ -277,7 +289,7 @@ old_badge = @emp1.badge_number
 
 ## Context-scoped multi_index relationship has correct configuration
 config = @emp1.class.indexing_relationships.find { |r| r.field == :department }
-[config.field, config.index_name, config.target_class]
+[config.field, config.index_name, config.scope_class]
 #=> [:department, :dept_index, TestCompany]
 
 ## Context-scoped methods are generated with collision-free naming
@@ -420,6 +432,20 @@ found = TestUser.find_all_by_email(emails)
 found.map(&:user_id)
 #=> ["user_002"]
 
+## Bulk query filters nil inputs before querying
+emails_with_nil = [nil, 'bob@example.com', nil]
+found = TestUser.find_all_by_email(emails_with_nil)
+found.map(&:user_id)
+#=> ["user_002"]
+
+## Bulk query with only nil inputs returns empty array
+TestUser.find_all_by_email([nil, nil]).length
+#=> 0
+
+## Bulk query with nil as single value returns empty array
+TestUser.find_all_by_email(nil).length
+#=> 0
+
 ## Adding to index with nil field value does nothing
 @user_nil = TestUser.new(user_id: 'user_nil', email: nil)
 @user_nil.add_to_class_email_lookup