familia 2.0.0.pre18 → 2.0.0.pre21

data/lib/familia/features/external_identifier.rb

@@ -1,4 +1,6 @@
 # lib/familia/features/external_identifier.rb
+#
+# frozen_string_literal: true
 
 module Familia
   module Features
@@ -12,8 +14,8 @@ module Familia
         base.extend ModelClassMethods
         base.include ModelInstanceMethods
 
-        # Ensure default prefix is set in feature options
-        base.add_feature_options(:external_identifier, prefix: 'ext')
+        # Ensure default format is set in feature options
+        base.add_feature_options(:external_identifier, format: 'ext_%{id}')
 
         # Add class-level mapping for extid -> id lookups
         base.class_hashkey :extid_lookup
@@ -35,10 +37,10 @@ module Familia
       # - Deterministic generation from objid ensures consistency
       # - Shorter than objid (128-bit vs 256-bit) for external use
       # - Base-36 encoding for URL-safe identifiers
-      # - 'ext_' prefix for clear identification as external IDs
+      # - Customizable format template (default: 'ext_' prefix)
       # - Lazy generation preserves values from initialization
       #
-      # @example Using external identifier fields
+      # @example Using external identifier fields with default format
       #   class User < Familia::Horreum
       #     feature :object_identifier
       #     feature :external_identifier
@@ -51,6 +53,30 @@ module Familia
       #   user2 = User.new(objid: user.objid, email: 'user@example.com')
       #   user2.extid  # => "ext_abc123def456ghi789" (identical to user.extid)
       #
+      # @example Using custom format template with hyphen separator
+      #   class APIKey < Familia::Horreum
+      #     feature :object_identifier
+      #     feature :external_identifier, format: 'api-%{id}'
+      #   end
+      #   key = APIKey.new
+      #   key.extid  # => "api-abc123def456ghi789"
+      #
+      # @example Using custom format template with custom prefix
+      #   class Customer < Familia::Horreum
+      #     feature :object_identifier
+      #     feature :external_identifier, format: 'cust_%{id}'
+      #   end
+      #   customer = Customer.new
+      #   customer.extid  # => "cust_abc123def456ghi789"
+      #
+      # @example Using format template without prefix
+      #   class Resource < Familia::Horreum
+      #     feature :object_identifier
+      #     feature :external_identifier, format: 'v2/%{id}'
+      #   end
+      #   resource = Resource.new
+      #   resource.extid  # => "v2/abc123def456ghi789"
+      #
       class ExternalIdentifierFieldType < Familia::FieldType
         # Override getter to provide lazy generation from objid
         #
@@ -252,11 +278,11 @@ module Familia
         # 128 bits is approximately 25 characters in base36.
         external_part = random_bytes.unpack1('H*').to_i(16).to_s(36).rjust(25, '0')
 
-        # Get prefix from feature options, default to "ext"
+        # Get format from feature options and interpolate the ID
         options = self.class.feature_options(:external_identifier)
-        prefix = options[:prefix] || 'ext'
+        format = options[:format] || 'ext_%{id}'
 
-        "#{prefix}_#{external_part}"
+        format % { id: external_part }
       end
 
       # Full-length alias for extid for clarity when needed

data/lib/familia/features/object_identifier.rb

@@ -1,4 +1,6 @@
 # lib/familia/features/object_identifier.rb
+#
+# frozen_string_literal: true
 
 module Familia
   module Features

data/lib/familia/features/quantization.rb

@@ -1,4 +1,6 @@
 # lib/familia/features/quantization.rb
+#
+# frozen_string_literal: true
 
 module Familia
   module Features
@@ -389,7 +391,7 @@ module Familia
       #   user.quantized_identifier(1.hour)  # => "123:1672531200"
       #   user.quantized_identifier(1.hour, pattern: '%Y%m%d%H')  # => "123:2023010114"
       #
-      def quantized_identifier(quantum, pattern: nil, separator: ':')
+      def quantized_identifier(quantum, pattern: nil, separator: Familia.delim)
         timestamp = qstamp(quantum, pattern: pattern)
         base_id = respond_to?(:identifier) ? identifier : object_id
         "#{base_id}#{separator}#{timestamp}"

data/lib/familia/features/relationships/README.md

@@ -18,6 +18,7 @@ participates_in Organization, :members, score: :joined_at, bidirectional: true
 
 **unique_index** - Fast unique lookups ("find object by unique field value")
 ```ruby
+unique_index :email, :email_index                        # Class-level: User.find_by_email()
 unique_index :email, :email_index, within: Organization  # Scoped: org.find_by_email()
 ```
 
@@ -89,7 +90,8 @@ class Customer < Familia::Horreum
   feature :relationships
 
   participates_in Organization, :members    # Customer belongs to org
-  unique_index :email, :email_index, within: Organization  # Find by unique email
+  unique_index :email, :email_index        # Class-level: Customer.find_by_email()
+  multi_index :status, :status_index, within: Organization  # Scoped: org.find_all_by_status()
 end
 ```
 

data/lib/familia/features/relationships/collection_operations.rb

@@ -1,4 +1,6 @@
 # lib/familia/features/relationships/collection_operations.rb
+#
+# frozen_string_literal: true
 
 module Familia
   module Features

data/lib/familia/features/relationships/indexing/multi_index_generators.rb

@@ -1,3 +1,5 @@
+# lib/familia/features/relationships/indexing/multi_index_generators.rb
+#
 # frozen_string_literal: true
 
 module Familia
@@ -32,29 +34,30 @@ module Familia
           # @param indexed_class [Class] The class being indexed (e.g., Employee)
           # @param field [Symbol] The field to index
           # @param index_name [Symbol] Name of the index
-          # @param within [Class, Symbol] Parent class for instance-scoped index (required)
+          # @param within [Class, Symbol] Scope class for instance-scoped index (required)
           # @param query [Boolean] Whether to generate query methods
           def setup(indexed_class:, field:, index_name:, within:, query:)
-            # Multi-index always requires a parent context
-            target_class = within
-            resolved_class = Familia.resolve_class(target_class)
+            # Multi-index always requires a scope context
+            scope_class = within
+            resolved_class = Familia.resolve_class(scope_class)
 
             # Store metadata for this indexing relationship
             indexed_class.indexing_relationships << IndexingRelationship.new(
               field:             field,
-              target_class:      target_class,
+              scope_class:       scope_class,
+              within:            within,
               index_name:        index_name,
               query:            query,
               cardinality:       :multi,
             )
 
             # Always generate the factory method - required by mutation methods
-            if target_class.is_a?(Class)
+            if scope_class.is_a?(Class)
               generate_factory_method(resolved_class, index_name)
             end
 
-            # Generate query methods on the parent class (optional)
-            if query && target_class.is_a?(Class)
+            # Generate query methods on the scope class (optional)
+            if query && scope_class.is_a?(Class)
               generate_query_methods_destination(indexed_class, field, resolved_class, index_name)
             end
 
@@ -62,42 +65,45 @@ module Familia
             generate_mutation_methods_self(indexed_class, field, resolved_class, index_name)
           end
 
-          # Generates the factory method ON THE PARENT CLASS (Company when within: Company):
+          # Generates the factory method ON THE SCOPE CLASS (Company when within: Company):
           # - company.index_name_for(field_value) - DataType factory (always needed)
           #
           # This method is required by mutation methods even when query: false
           #
-          # @param target_class [Class] The parent class (e.g., Company)
+          # @param scope_class [Class] The scope class providing uniqueness context (e.g., Company)
           # @param index_name [Symbol] Name of the index (e.g., :dept_index)
-          def generate_factory_method(target_class, index_name)
-            actual_target_class = Familia.resolve_class(target_class)
+          def generate_factory_method(scope_class, index_name)
+            actual_scope_class = Familia.resolve_class(scope_class)
 
-            actual_target_class.class_eval do
+            actual_scope_class.class_eval do
               # Helper method to get index set for a specific field value
               # This acts as a factory for field-value-specific DataTypes
               define_method(:"#{index_name}_for") do |field_value|
                 # Return properly managed DataType instance with parameterized key
-                index_key = "#{index_name}:#{field_value}"
+                index_key = Familia.join(index_name, field_value)
                 Familia::UnsortedSet.new(index_key, parent: self)
               end
             end
           end
 
-          # Generates query methods ON THE PARENT CLASS (Company when within: Company):
+          # Generates query methods ON THE SCOPE CLASS (Company when within: Company):
           # - company.sample_from_department(dept, count=1) - random sampling
           # - company.find_all_by_department(dept) - all objects
           # - company.rebuild_dept_index - rebuild index
           #
           # @param indexed_class [Class] The class being indexed (e.g., Employee)
           # @param field [Symbol] The field to index (e.g., :department)
-          # @param target_class [Class] The parent class (e.g., Company)
+          # @param scope_class [Class] The scope class providing uniqueness context (e.g., Company)
           # @param index_name [Symbol] Name of the index (e.g., :dept_index)
-          def generate_query_methods_destination(indexed_class, field, target_class, index_name)
-            # Resolve target class using Familia pattern
-            actual_target_class = Familia.resolve_class(target_class)
+          def generate_query_methods_destination(indexed_class, field, scope_class, index_name)
+            # Resolve scope class using Familia pattern
+            actual_scope_class = Familia.resolve_class(scope_class)
+
+            # Get scope_class_config for method naming (needed for rebuild methods)
+            scope_class_config = actual_scope_class.config_name
 
             # Generate instance sampling method (e.g., company.sample_from_department)
-            actual_target_class.class_eval do
+            actual_scope_class.class_eval do
 
               define_method(:"sample_from_#{field}") do |field_value, count = 1|
                 index_set = send("#{index_name}_for", field_value) # i.e. UnsortedSet
@@ -117,11 +123,135 @@ module Familia
                 index_set.members.map { |id| indexed_class.find_by_identifier(id) }
               end
 
-              # Generate method to rebuild the index for this parent instance
-              define_method(:"rebuild_#{index_name}") do
-                # This would need to be implemented based on how you track which
-                # objects belong to this parent instance
-                # For now, just a placeholder
+              # Generate method to rebuild the multi-value index for this parent instance
+              #
+              # Multi-indexes create separate sets for each field value, requiring a three-phase approach:
+              # 1. Loading: Load all objects once and cache them (discovers field values simultaneously)
+              # 2. Clearing: Remove all existing index sets using SCAN
+              # 3. Rebuilding: Rebuild index from cached objects (no reload needed)
+              #
+              # @param batch_size [Integer] Number of identifiers to process per batch
+              # @yield [progress] Optional block called with progress updates
+              # @yieldparam progress [Hash] Progress information with keys:
+              #   - :phase [Symbol] Current phase (:loading, :clearing, :rebuilding)
+              #   - :current [Integer] Current item count
+              #   - :total [Integer] Total items (when known)
+              #   - :field_value [String] Current field value being processed
+              #
+              # @example Basic rebuild
+              #   company.rebuild_dept_index
+              #
+              # @example With progress monitoring
+              #   company.rebuild_dept_index do |progress|
+              #     puts "#{progress[:phase]}: #{progress[:current]}/#{progress[:total]}"
+              #   end
+              #
+              # @example Memory-conscious rebuild for large collections
+              #   # Process in smaller batches to reduce memory footprint
+              #   company.rebuild_dept_index(batch_size: 50)
+              #
+              # @note Memory Considerations:
+              #   This method caches all objects in memory during rebuild to avoid duplicate
+              #   database loads. For very large collections (>100k objects), monitor memory usage
+              #   and consider processing in chunks or using a streaming approach if memory
+              #   constraints are encountered. The batch_size parameter controls Redis I/O
+              #   batching but does not affect memory usage since all objects are cached.
+              #
+              define_method(:"rebuild_#{index_name}") do |batch_size: 100, &progress_block|
+                # PHASE 1: Find the collection containing the indexed objects
+                # Look for a participation relationship where indexed_class participates in this scope_class
+                collection_name = nil
+
+                # Check if indexed_class has participation to this scope_class
+                if indexed_class.respond_to?(:participation_relationships)
+                  participation = indexed_class.participation_relationships.find do |rel|
+                    rel.target_class == self.class
+                  end
+                  collection_name = participation&.collection_name if participation
+                end
+
+                # Get the collection DataType if we found a participation relationship
+                collection = collection_name ? send(collection_name) : nil
+
+                if collection
+                  # PHASE 2: Load objects once and cache them for both discovery and rebuilding
+                  # This avoids duplicate load_multi calls (previous approach loaded twice)
+                  progress_block&.call(phase: :loading, current: 0, total: collection.size)
+
+                  field_values = Set.new
+                  cached_objects = []
+                  processed = 0
+
+                  collection.members.each_slice(batch_size) do |identifiers|
+                    # Load objects in batches - SINGLE LOAD for both phases
+                    objects = indexed_class.load_multi(identifiers).compact
+                    cached_objects.concat(objects)
+
+                    objects.each do |obj|
+                      value = obj.send(field)
+                      # Only track non-nil, non-empty field values
+                      field_values << value.to_s if value && !value.to_s.strip.empty?
+                    end
+
+                    processed += identifiers.size
+                    progress_block&.call(phase: :loading, current: processed, total: collection.size)
+                  end
+
+                  # PHASE 3: Clear all existing field-value-specific index sets
+                  # Use SCAN to find all existing index keys (including orphaned ones from deleted field values)
+                  progress_block&.call(phase: :clearing, current: 0, total: field_values.size)
+
+                  # Get the base pattern for this index by creating a sample index set
+                  # The "*" creates a wildcard pattern like "company:123:dept_index:*" for SCAN
+                  sample_index = send(:"#{index_name}_for", "*")
+                  index_pattern = sample_index.dbkey
+
+                  # Find all existing index keys using SCAN
+                  cleared_count = 0
+                  dbclient.scan_each(match: index_pattern) do |key|
+                    dbclient.del(key)
+                    cleared_count += 1
+                    progress_block&.call(phase: :clearing, current: cleared_count, total: field_values.size, key: key)
+                  end
+
+                  # PHASE 4: Rebuild index from cached objects (no reload needed)
+                  progress_block&.call(phase: :rebuilding, current: 0, total: cached_objects.size)
+
+                  processed = 0
+                  cached_objects.each_slice(batch_size) do |objects|
+                    transaction do |_tx|
+                      objects.each do |obj|
+                        # Use the generated add_to method to maintain consistency
+                        # This ensures the same logic is used as during normal operation
+                        obj.send(:"add_to_#{scope_class_config}_#{index_name}", self)
+                      end
+                    end
+
+                    processed += objects.size
+                    progress_block&.call(phase: :rebuilding, current: processed, total: cached_objects.size)
+                  end
+
+                  Familia.info "[Rebuild] Multi-index #{index_name} rebuilt: #{field_values.size} field values, #{processed} objects"
+
+                  processed  # Return count of processed objects
+
+                else
+                  # No participation relationship found - warn and suggest alternative
+                  Familia.warn <<~WARNING
+                    [Rebuild] Cannot rebuild multi-index #{index_name}: no participation relationship found
+
+                    Multi-index rebuild requires a participation relationship to find objects.
+                    Add a participation relationship to #{indexed_class.name}:
+
+                      class #{indexed_class.name} < Familia::Horreum
+                        participates_in #{self.class.name}, :collection_name, score: :field
+                      end
+
+                    Then access the collection via: #{self.class.config_name}.collection_name
+                  WARNING
+
+                  nil
+                end
               end
             end
           end
@@ -133,62 +263,62 @@ module Familia
           #
           # @param indexed_class [Class] The class being indexed (e.g., Employee)
           # @param field [Symbol] The field to index (e.g., :department)
-          # @param target_class [Class] The parent class (e.g., Company)
+          # @param scope_class [Class] The scope class providing uniqueness context (e.g., Company)
           # @param index_name [Symbol] Name of the index (e.g., :dept_index)
-          def generate_mutation_methods_self(indexed_class, field, target_class, index_name)
-            target_class_config = target_class.config_name
+          def generate_mutation_methods_self(indexed_class, field, scope_class, index_name)
+            scope_class_config = scope_class.config_name
             indexed_class.class_eval do
-              method_name = :"add_to_#{target_class_config}_#{index_name}"
-              Familia.ld("[MultiIndexGenerators] #{name} method #{method_name}")
+              method_name = :"add_to_#{scope_class_config}_#{index_name}"
+              Familia.debug("[MultiIndexGenerators] #{name} method #{method_name}")
 
-              define_method(method_name) do |target_instance|
-                return unless target_instance
+              define_method(method_name) do |scope_instance|
+                return unless scope_instance
 
                 field_value = send(field)
                 return unless field_value
 
-                # Use helper method on target instance instead of manual instantiation
-                index_set = target_instance.send("#{index_name}_for", field_value)
+                # Use helper method on scope instance instead of manual instantiation
+                index_set = scope_instance.send("#{index_name}_for", field_value)
 
                 # Use UnsortedSet DataType method (no scoring)
                 index_set.add(identifier)
               end
 
-              method_name = :"remove_from_#{target_class_config}_#{index_name}"
-              Familia.ld("[MultiIndexGenerators] #{name} method #{method_name}")
+              method_name = :"remove_from_#{scope_class_config}_#{index_name}"
+              Familia.debug("[MultiIndexGenerators] #{name} method #{method_name}")
 
-              define_method(method_name) do |target_instance|
-                return unless target_instance
+              define_method(method_name) do |scope_instance|
+                return unless scope_instance
 
                 field_value = send(field)
                 return unless field_value
 
-                # Use helper method on target instance instead of manual instantiation
-                index_set = target_instance.send("#{index_name}_for", field_value)
+                # Use helper method on scope instance instead of manual instantiation
+                index_set = scope_instance.send("#{index_name}_for", field_value)
 
                 # Remove using UnsortedSet DataType method
                 index_set.remove(identifier)
               end
 
-              method_name = :"update_in_#{target_class_config}_#{index_name}"
-              Familia.ld("[MultiIndexGenerators] #{name} method #{method_name}")
+              method_name = :"update_in_#{scope_class_config}_#{index_name}"
+              Familia.debug("[MultiIndexGenerators] #{name} method #{method_name}")
 
-              define_method(method_name) do |target_instance, old_field_value = nil|
-                return unless target_instance
+              define_method(method_name) do |scope_instance, old_field_value = nil|
+                return unless scope_instance
 
                 new_field_value = send(field)
 
                 # Use Familia's transaction method for atomicity with DataType abstraction
-                target_instance.transaction do |_tx|
+                scope_instance.transaction do |_tx|
                   # Remove from old index if provided - use helper method
                   if old_field_value
-                    old_index_set = target_instance.send("#{index_name}_for", old_field_value)
+                    old_index_set = scope_instance.send("#{index_name}_for", old_field_value)
                     old_index_set.remove(identifier)
                   end
 
                   # Add to new index if present - use helper method
                   if new_field_value
-                    new_index_set = target_instance.send("#{index_name}_for", new_field_value)
+                    new_index_set = scope_instance.send("#{index_name}_for", new_field_value)
                     new_index_set.add(identifier)
                   end
                 end