familia 2.0.0.pre19 → 2.0.0.pre22

data/lib/familia/features/external_identifier.rb

@@ -1,4 +1,6 @@
 # lib/familia/features/external_identifier.rb
+#
+# frozen_string_literal: true
 
 module Familia
   module Features
@@ -12,8 +14,8 @@ module Familia
         base.extend ModelClassMethods
         base.include ModelInstanceMethods
 
-        # Ensure default prefix is set in feature options
-        base.add_feature_options(:external_identifier, prefix: 'ext')
+        # Ensure default format is set in feature options
+        base.add_feature_options(:external_identifier, format: 'ext_%{id}')
 
         # Add class-level mapping for extid -> id lookups
         base.class_hashkey :extid_lookup
@@ -35,10 +37,10 @@ module Familia
       # - Deterministic generation from objid ensures consistency
       # - Shorter than objid (128-bit vs 256-bit) for external use
       # - Base-36 encoding for URL-safe identifiers
-      # - 'ext_' prefix for clear identification as external IDs
+      # - Customizable format template (default: 'ext_' prefix)
       # - Lazy generation preserves values from initialization
       #
-      # @example Using external identifier fields
+      # @example Using external identifier fields with default format
       #   class User < Familia::Horreum
       #     feature :object_identifier
       #     feature :external_identifier
@@ -51,6 +53,30 @@ module Familia
       #   user2 = User.new(objid: user.objid, email: 'user@example.com')
       #   user2.extid  # => "ext_abc123def456ghi789" (identical to user.extid)
       #
+      # @example Using custom format template with hyphen separator
+      #   class APIKey < Familia::Horreum
+      #     feature :object_identifier
+      #     feature :external_identifier, format: 'api-%{id}'
+      #   end
+      #   key = APIKey.new
+      #   key.extid  # => "api-abc123def456ghi789"
+      #
+      # @example Using custom format template with custom prefix
+      #   class Customer < Familia::Horreum
+      #     feature :object_identifier
+      #     feature :external_identifier, format: 'cust_%{id}'
+      #   end
+      #   customer = Customer.new
+      #   customer.extid  # => "cust_abc123def456ghi789"
+      #
+      # @example Using format template without prefix
+      #   class Resource < Familia::Horreum
+      #     feature :object_identifier
+      #     feature :external_identifier, format: 'v2/%{id}'
+      #   end
+      #   resource = Resource.new
+      #   resource.extid  # => "v2/abc123def456ghi789"
+      #
       class ExternalIdentifierFieldType < Familia::FieldType
         # Override getter to provide lazy generation from objid
         #
@@ -150,6 +176,35 @@ module Familia
           # extid_lookup.remove_field(extid)
           nil
         end
+
+        # Check if a string matches the extid format for the Horreum class. The specific
+        # class is important b/c each one can have its own custom prefix, like `ext_`.
+        #
+        # The validator accepts a reasonable range of ID lengths (20-32 characters) to
+        # accommodate potential changes to the entropy or encoding while maintaining
+        # security. The current implementation generates exactly 25 base36 characters
+        # from 16 bytes (128 bits), but this flexibility allows for future adjustments
+        # without breaking validation.
+        #
+        # @param guess [String] The string to check
+        # @return [Boolean] true if the guess matches the extid format, false otherwise
+        def extid?(guess)
+          return false if guess.to_s.empty?
+
+          options = feature_options(:external_identifier)
+          format = options[:format] || 'ext_%{id}'
+
+          # Extract prefix and suffix from format
+          return false unless format.include?('%{id}')
+          prefix, suffix = format.split('%{id}', 2)
+
+          # Build regex pattern to match the extid format
+          # Accept 20-32 base36 characters to allow for entropy/encoding variations
+          # Current generation: 16 bytes -> base36 -> 25 chars (rjust with '0')
+          pattern = /\A#{Regexp.escape(prefix)}[0-9a-z]{20,32}#{Regexp.escape(suffix)}\z/i
+
+          !!(guess =~ pattern)
+        end
       end
 
       # Instance methods for external identifier management
@@ -252,11 +307,11 @@ module Familia
         # 128 bits is approximately 25 characters in base36.
         external_part = random_bytes.unpack1('H*').to_i(16).to_s(36).rjust(25, '0')
 
-        # Get prefix from feature options, default to "ext"
+        # Get format from feature options and interpolate the ID
         options = self.class.feature_options(:external_identifier)
-        prefix = options[:prefix] || 'ext'
+        format = options[:format] || 'ext_%{id}'
 
-        "#{prefix}_#{external_part}"
+        format % { id: external_part }
       end
 
       # Full-length alias for extid for clarity when needed

data/lib/familia/features/object_identifier.rb

@@ -1,4 +1,6 @@
 # lib/familia/features/object_identifier.rb
+#
+# frozen_string_literal: true
 
 module Familia
   module Features
@@ -277,6 +279,53 @@ module Familia
           # objid_lookup.remove_field(objid)
           nil
         end
+
+        # Check if a string matches the objid format for the Horreum class. The specific
+        # class is important b/c each one can have its own type of objid generator.
+        #
+        # @param guess [String] The string to check
+        # @return [Boolean] true if the guess matches the objid format, false otherwise
+        def objid?(guess)
+          return false if guess.to_s.empty?
+
+          options = feature_options(:object_identifier)
+          generator = options[:generator] || DEFAULT_GENERATOR
+
+          case generator
+          when :uuid_v7, :uuid_v4
+            # UUID format: xxxxxxxx-xxxx-Vxxx-xxxx-xxxxxxxxxxxx (36 chars with hyphens)
+            # Validate structure and that all characters are valid hex digits
+            guess_str = guess.to_s
+            return false unless guess_str.length == 36
+            return false unless guess_str[8] == '-' && guess_str[13] == '-' && guess_str[18] == '-' && guess_str[23] == '-'
+
+            # Extract segments and validate each is valid hex
+            segments = guess_str.split('-')
+            return false unless segments.length == 5
+            return false unless segments[0] =~ /\A[0-9a-fA-F]{8}\z/  # 8 hex chars
+            return false unless segments[1] =~ /\A[0-9a-fA-F]{4}\z/  # 4 hex chars
+            return false unless segments[2] =~ /\A[0-9a-fA-F]{4}\z/  # 4 hex chars (includes version)
+            return false unless segments[3] =~ /\A[0-9a-fA-F]{4}\z/  # 4 hex chars
+            return false unless segments[4] =~ /\A[0-9a-fA-F]{12}\z/ # 12 hex chars
+
+            # Validate version character
+            version_char = guess_str[14]
+            if generator == :uuid_v7
+              version_char == '7'
+            else # generator == :uuid_v4
+              version_char == '4'
+            end
+          when :hex
+            # Hex format: pure hexadecimal without hyphens
+            !!(guess =~ /\A[0-9a-fA-F]+\z/)
+          when Proc
+            # Cannot determine format for custom Proc generators
+            Familia.warn "[objid?] Validation not supported for custom Proc generators on #{name}" if Familia.debug?
+            false
+          else
+            false
+          end
+        end
       end
 
       # Instance methods for object identifier management

data/lib/familia/features/quantization.rb

@@ -1,4 +1,6 @@
 # lib/familia/features/quantization.rb
+#
+# frozen_string_literal: true
 
 module Familia
   module Features
@@ -389,7 +391,7 @@ module Familia
       #   user.quantized_identifier(1.hour)  # => "123:1672531200"
       #   user.quantized_identifier(1.hour, pattern: '%Y%m%d%H')  # => "123:2023010114"
       #
-      def quantized_identifier(quantum, pattern: nil, separator: ':')
+      def quantized_identifier(quantum, pattern: nil, separator: Familia.delim)
         timestamp = qstamp(quantum, pattern: pattern)
         base_id = respond_to?(:identifier) ? identifier : object_id
         "#{base_id}#{separator}#{timestamp}"

data/lib/familia/features/relationships/README.md

@@ -18,6 +18,7 @@ participates_in Organization, :members, score: :joined_at, bidirectional: true
 
 **unique_index** - Fast unique lookups ("find object by unique field value")
 ```ruby
+unique_index :email, :email_index                        # Class-level: User.find_by_email()
 unique_index :email, :email_index, within: Organization  # Scoped: org.find_by_email()
 ```
 
@@ -89,7 +90,8 @@ class Customer < Familia::Horreum
   feature :relationships
 
   participates_in Organization, :members    # Customer belongs to org
-  unique_index :email, :email_index, within: Organization  # Find by unique email
+  unique_index :email, :email_index        # Class-level: Customer.find_by_email()
+  multi_index :status, :status_index, within: Organization  # Scoped: org.find_all_by_status()
 end
 ```
 

data/lib/familia/features/relationships/collection_operations.rb

@@ -1,4 +1,6 @@
 # lib/familia/features/relationships/collection_operations.rb
+#
+# frozen_string_literal: true
 
 module Familia
   module Features

data/lib/familia/features/relationships/indexing/multi_index_generators.rb

@@ -1,3 +1,5 @@
+# lib/familia/features/relationships/indexing/multi_index_generators.rb
+#
 # frozen_string_literal: true
 
 module Familia
@@ -78,7 +80,7 @@ module Familia
               # This acts as a factory for field-value-specific DataTypes
               define_method(:"#{index_name}_for") do |field_value|
                 # Return properly managed DataType instance with parameterized key
-                index_key = "#{index_name}:#{field_value}"
+                index_key = Familia.join(index_name, field_value)
                 Familia::UnsortedSet.new(index_key, parent: self)
               end
             end
@@ -97,6 +99,9 @@ module Familia
             # Resolve scope class using Familia pattern
             actual_scope_class = Familia.resolve_class(scope_class)
 
+            # Get scope_class_config for method naming (needed for rebuild methods)
+            scope_class_config = actual_scope_class.config_name
+
             # Generate instance sampling method (e.g., company.sample_from_department)
             actual_scope_class.class_eval do
 
@@ -118,11 +123,135 @@ module Familia
                 index_set.members.map { |id| indexed_class.find_by_identifier(id) }
               end
 
-              # Generate method to rebuild the index for this parent instance
-              define_method(:"rebuild_#{index_name}") do
-                # This would need to be implemented based on how you track which
-                # objects belong to this parent instance
-                # For now, just a placeholder
+              # Generate method to rebuild the multi-value index for this parent instance
+              #
+              # Multi-indexes create separate sets for each field value, requiring a three-phase approach:
+              # 1. Loading: Load all objects once and cache them (discovers field values simultaneously)
+              # 2. Clearing: Remove all existing index sets using SCAN
+              # 3. Rebuilding: Rebuild index from cached objects (no reload needed)
+              #
+              # @param batch_size [Integer] Number of identifiers to process per batch
+              # @yield [progress] Optional block called with progress updates
+              # @yieldparam progress [Hash] Progress information with keys:
+              #   - :phase [Symbol] Current phase (:loading, :clearing, :rebuilding)
+              #   - :current [Integer] Current item count
+              #   - :total [Integer] Total items (when known)
+              #   - :field_value [String] Current field value being processed
+              #
+              # @example Basic rebuild
+              #   company.rebuild_dept_index
+              #
+              # @example With progress monitoring
+              #   company.rebuild_dept_index do |progress|
+              #     puts "#{progress[:phase]}: #{progress[:current]}/#{progress[:total]}"
+              #   end
+              #
+              # @example Memory-conscious rebuild for large collections
+              #   # Process in smaller batches to reduce memory footprint
+              #   company.rebuild_dept_index(batch_size: 50)
+              #
+              # @note Memory Considerations:
+              #   This method caches all objects in memory during rebuild to avoid duplicate
+              #   database loads. For very large collections (>100k objects), monitor memory usage
+              #   and consider processing in chunks or using a streaming approach if memory
+              #   constraints are encountered. The batch_size parameter controls Redis I/O
+              #   batching but does not affect memory usage since all objects are cached.
+              #
+              define_method(:"rebuild_#{index_name}") do |batch_size: 100, &progress_block|
+                # PHASE 1: Find the collection containing the indexed objects
+                # Look for a participation relationship where indexed_class participates in this scope_class
+                collection_name = nil
+
+                # Check if indexed_class has participation to this scope_class
+                if indexed_class.respond_to?(:participation_relationships)
+                  participation = indexed_class.participation_relationships.find do |rel|
+                    rel.target_class == self.class
+                  end
+                  collection_name = participation&.collection_name if participation
+                end
+
+                # Get the collection DataType if we found a participation relationship
+                collection = collection_name ? send(collection_name) : nil
+
+                if collection
+                  # PHASE 2: Load objects once and cache them for both discovery and rebuilding
+                  # This avoids duplicate load_multi calls (previous approach loaded twice)
+                  progress_block&.call(phase: :loading, current: 0, total: collection.size)
+
+                  field_values = Set.new
+                  cached_objects = []
+                  processed = 0
+
+                  collection.members.each_slice(batch_size) do |identifiers|
+                    # Load objects in batches - SINGLE LOAD for both phases
+                    objects = indexed_class.load_multi(identifiers).compact
+                    cached_objects.concat(objects)
+
+                    objects.each do |obj|
+                      value = obj.send(field)
+                      # Only track non-nil, non-empty field values
+                      field_values << value.to_s if value && !value.to_s.strip.empty?
+                    end
+
+                    processed += identifiers.size
+                    progress_block&.call(phase: :loading, current: processed, total: collection.size)
+                  end
+
+                  # PHASE 3: Clear all existing field-value-specific index sets
+                  # Use SCAN to find all existing index keys (including orphaned ones from deleted field values)
+                  progress_block&.call(phase: :clearing, current: 0, total: field_values.size)
+
+                  # Get the base pattern for this index by creating a sample index set
+                  # The "*" creates a wildcard pattern like "company:123:dept_index:*" for SCAN
+                  sample_index = send(:"#{index_name}_for", "*")
+                  index_pattern = sample_index.dbkey
+
+                  # Find all existing index keys using SCAN
+                  cleared_count = 0
+                  dbclient.scan_each(match: index_pattern) do |key|
+                    dbclient.del(key)
+                    cleared_count += 1
+                    progress_block&.call(phase: :clearing, current: cleared_count, total: field_values.size, key: key)
+                  end
+
+                  # PHASE 4: Rebuild index from cached objects (no reload needed)
+                  progress_block&.call(phase: :rebuilding, current: 0, total: cached_objects.size)
+
+                  processed = 0
+                  cached_objects.each_slice(batch_size) do |objects|
+                    transaction do |_tx|
+                      objects.each do |obj|
+                        # Use the generated add_to method to maintain consistency
+                        # This ensures the same logic is used as during normal operation
+                        obj.send(:"add_to_#{scope_class_config}_#{index_name}", self)
+                      end
+                    end
+
+                    processed += objects.size
+                    progress_block&.call(phase: :rebuilding, current: processed, total: cached_objects.size)
+                  end
+
+                  Familia.info "[Rebuild] Multi-index #{index_name} rebuilt: #{field_values.size} field values, #{processed} objects"
+
+                  processed  # Return count of processed objects
+
+                else
+                  # No participation relationship found - warn and suggest alternative
+                  Familia.warn <<~WARNING
+                    [Rebuild] Cannot rebuild multi-index #{index_name}: no participation relationship found
+
+                    Multi-index rebuild requires a participation relationship to find objects.
+                    Add a participation relationship to #{indexed_class.name}:
+
+                      class #{indexed_class.name} < Familia::Horreum
+                        participates_in #{self.class.name}, :collection_name, score: :field
+                      end
+
+                    Then access the collection via: #{self.class.config_name}.collection_name
+                  WARNING
+
+                  nil
+                end
               end
             end
           end
@@ -140,7 +269,7 @@ module Familia
             scope_class_config = scope_class.config_name
             indexed_class.class_eval do
               method_name = :"add_to_#{scope_class_config}_#{index_name}"
-              Familia.ld("[MultiIndexGenerators] #{name} method #{method_name}")
+              Familia.debug("[MultiIndexGenerators] #{name} method #{method_name}")
 
               define_method(method_name) do |scope_instance|
                 return unless scope_instance
@@ -156,7 +285,7 @@ module Familia
               end
 
               method_name = :"remove_from_#{scope_class_config}_#{index_name}"
-              Familia.ld("[MultiIndexGenerators] #{name} method #{method_name}")
+              Familia.debug("[MultiIndexGenerators] #{name} method #{method_name}")
 
               define_method(method_name) do |scope_instance|
                 return unless scope_instance
@@ -172,7 +301,7 @@ module Familia
               end
 
               method_name = :"update_in_#{scope_class_config}_#{index_name}"
-              Familia.ld("[MultiIndexGenerators] #{name} method #{method_name}")
+              Familia.debug("[MultiIndexGenerators] #{name} method #{method_name}")
 
               define_method(method_name) do |scope_instance, old_field_value = nil|
                 return unless scope_instance