familia 2.0.0.pre19 → 2.0.0.pre21

data/lib/familia/encryption/encrypted_data.rb

@@ -1,4 +1,6 @@
 # lib/familia/encryption/encrypted_data.rb
+#
+# frozen_string_literal: true
 
 module Familia
   module Encryption
@@ -15,10 +17,10 @@ module Familia
           # Check for required fields
           required_fields = %i[algorithm nonce ciphertext auth_tag key_version]
           result = required_fields.all? { |field| parsed.key?(field) }
-          Familia.ld "[valid?] result: #{result}, parsed: #{parsed}, required: #{required_fields}"
+          Familia.debug "[valid?] result: #{result}, parsed: #{parsed}, required: #{required_fields}"
           result
         rescue Familia::SerializerError => e
-          Familia.ld "[valid?] JSON error: #{e.message}"
+          Familia.debug "[valid?] JSON error: #{e.message}"
           false
         end
       end

data/lib/familia/encryption/manager.rb

@@ -1,4 +1,6 @@
 # lib/familia/encryption/manager.rb
+#
+# frozen_string_literal: true
 
 module Familia
   module Encryption

data/lib/familia/encryption/provider.rb

@@ -1,4 +1,6 @@
 # lib/familia/encryption/provider.rb
+#
+# frozen_string_literal: true
 
 module Familia
   module Encryption

data/lib/familia/encryption/providers/aes_gcm_provider.rb

@@ -1,4 +1,6 @@
 # lib/familia/encryption/providers/aes_gcm_provider.rb
+#
+# frozen_string_literal: true
 
 # ⚠️  RUBY MEMORY SAFETY WARNING ⚠️
 #

data/lib/familia/encryption/providers/secure_xchacha20_poly1305_provider.rb

@@ -1,4 +1,6 @@
 # lib/familia/encryption/providers/secure_xchacha20_poly1305_provider.rb
+#
+# frozen_string_literal: true
 
 # ⚠️  PROTOTYPE IMPLEMENTATION - NOT FOR PRODUCTION USE ⚠️
 #

data/lib/familia/encryption/providers/xchacha20_poly1305_provider.rb

@@ -1,4 +1,6 @@
 # lib/familia/encryption/providers/xchacha20_poly1305_provider.rb
+#
+# frozen_string_literal: true
 
 # ⚠️  RUBY MEMORY SAFETY WARNING ⚠️
 #

data/lib/familia/encryption/registry.rb

@@ -1,4 +1,6 @@
 # lib/familia/encryption/registry.rb
+#
+# frozen_string_literal: true
 
 module Familia
   module Encryption

data/lib/familia/encryption/request_cache.rb

@@ -1,5 +1,7 @@
 # lib/familia/encryption/request_cache.rb
 #
+# frozen_string_literal: true
+
 # Request-scoped caching for encryption keys (if needed for performance)
 # This should ONLY be enabled if performance testing shows it's necessary
 #

data/lib/familia/encryption.rb

@@ -1,4 +1,6 @@
 # lib/familia/encryption.rb
+#
+# frozen_string_literal: true
 
 require 'base64'
 require 'oj'
@@ -60,9 +62,14 @@ module Familia
     #   end
     class << self
       # Get or create a manager with specific algorithm
+      #
+      # Thread-safe lazy initialization using Concurrent::Map to ensure
+      # only a single Manager instance is created per algorithm even under
+      # concurrent encryption/decryption requests.
+      #
       def manager(algorithm: nil)
-        @managers ||= {}
-        @managers[algorithm] ||= Manager.new(algorithm: algorithm)
+        @managers ||= Concurrent::Map.new
+        @managers.fetch_or_store(algorithm) { Manager.new(algorithm: algorithm) }
       end
 
       # Quick encryption with auto-selected best provider

data/lib/familia/errors.rb

@@ -1,5 +1,7 @@
 # lib/familia/errors.rb
 #
+# frozen_string_literal: true
+
 module Familia
   # Base exception class for all Familia errors
   class Problem < RuntimeError; end

data/lib/familia/features/autoloader.rb

@@ -1,4 +1,6 @@
 # lib/familia/features/autoloader.rb
+#
+# frozen_string_literal: true
 
 # rubocop:disable Style/ClassAndModuleChildren
 module Familia::Features

data/lib/familia/features/encrypted_fields/concealed_string.rb

@@ -1,4 +1,6 @@
 # lib/familia/features/encrypted_fields/concealed_string.rb
+#
+# frozen_string_literal: true
 
 # ConcealedString
 #

data/lib/familia/features/encrypted_fields/encrypted_field_type.rb

@@ -1,3 +1,7 @@
+# lib/familia/features/encrypted_fields/encrypted_field_type.rb
+#
+# frozen_string_literal: true
+
 # lib/familia/field_types/encrypted_field_type.rb
 
 require_relative '../../field_type'

data/lib/familia/features/encrypted_fields.rb

@@ -1,4 +1,6 @@
 # lib/familia/features/encrypted_fields.rb
+#
+# frozen_string_literal: true
 
 require_relative 'encrypted_fields/encrypted_field_type'
 
@@ -341,8 +343,6 @@ module Familia
       # Check if this instance has any encrypted fields with values
       #
       # @return [Boolean] true if any encrypted fields have values
-      #
-      # TODO: Missing test coverage
       def encrypted_data?
         self.class.encrypted_fields.any? do |field_name|
           field_value = instance_variable_get("@#{field_name}")

data/lib/familia/features/expiration/extensions.rb

@@ -1,4 +1,6 @@
 # lib/familia/features/expiration/extensions.rb
+#
+# frozen_string_literal: true
 
 module Familia
   # Add a default update_expiration method for all classes that include
@@ -24,7 +26,7 @@ module Familia
     # @example MyModel.new.update_expiration(expiration: 3600) # => nothing happens
     #
     def update_expiration(expiration: nil)
-      Familia.ld <<~LOG
+      Familia.debug <<~LOG
         [update_expiration] Expiration feature not enabled for #{self.class}.
         Key: #{dbkey} Arg: #{expiration} (caller: #{caller(1..1)})
       LOG

data/lib/familia/features/expiration.rb

@@ -1,4 +1,6 @@
 # lib/familia/features/expiration.rb
+#
+# frozen_string_literal: true
 
 require_relative 'expiration/extensions'
 
@@ -252,13 +254,13 @@ module Familia
 
         # Handle cascading expiration to related data structures
         if self.class.relations?
-          Familia.ld "[update_expiration] #{self.class} has relations: #{self.class.related_fields.keys}"
+          Familia.debug "[update_expiration] #{self.class} has relations: #{self.class.related_fields.keys}"
           self.class.related_fields.each do |name, definition|
             # Skip relations that don't have their own expiration settings
             next if definition.opts[:default_expiration].nil?
 
             obj = send(name)
-            Familia.ld "[update_expiration] Updating expiration for #{name} (#{obj.dbkey}) to #{expiration}"
+            Familia.debug "[update_expiration] Updating expiration for #{name} (#{obj.dbkey}) to #{expiration}"
             obj.update_expiration(expiration: expiration)
           end
         end
@@ -280,11 +282,17 @@ module Familia
         # If zero, simply skip setting an expiry for this key. If we were to set
         # 0, Valkey/Redis would drop the key immediately.
         if expiration.zero?
-          Familia.ld "[update_expiration] No expiration for #{self.class} (#{dbkey})"
+          Familia.debug "[update_expiration] No expiration for #{self.class} (#{dbkey})"
           return true
         end
 
-        Familia.ld "[update_expiration] Expires #{dbkey} in #{expiration} seconds"
+        # Structured TTL operation logging
+        Familia.debug "TTL updated",
+          operation: :expire,
+          key: dbkey,
+          ttl_seconds: expiration,
+          class: self.class.name,
+          identifier: (identifier rescue nil)
 
         # The Valkey/Redis' EXPIRE command returns 1 if the timeout was set, 0
         # if key does not exist or the timeout could not be set. Via redis-rb,

data/lib/familia/features/external_identifier.rb

@@ -1,4 +1,6 @@
 # lib/familia/features/external_identifier.rb
+#
+# frozen_string_literal: true
 
 module Familia
   module Features
@@ -12,8 +14,8 @@ module Familia
         base.extend ModelClassMethods
         base.include ModelInstanceMethods
 
-        # Ensure default prefix is set in feature options
-        base.add_feature_options(:external_identifier, prefix: 'ext')
+        # Ensure default format is set in feature options
+        base.add_feature_options(:external_identifier, format: 'ext_%{id}')
 
         # Add class-level mapping for extid -> id lookups
         base.class_hashkey :extid_lookup
@@ -35,10 +37,10 @@ module Familia
       # - Deterministic generation from objid ensures consistency
       # - Shorter than objid (128-bit vs 256-bit) for external use
       # - Base-36 encoding for URL-safe identifiers
-      # - 'ext_' prefix for clear identification as external IDs
+      # - Customizable format template (default: 'ext_' prefix)
       # - Lazy generation preserves values from initialization
       #
-      # @example Using external identifier fields
+      # @example Using external identifier fields with default format
       #   class User < Familia::Horreum
       #     feature :object_identifier
       #     feature :external_identifier
@@ -51,6 +53,30 @@ module Familia
       #   user2 = User.new(objid: user.objid, email: 'user@example.com')
       #   user2.extid  # => "ext_abc123def456ghi789" (identical to user.extid)
       #
+      # @example Using custom format template with hyphen separator
+      #   class APIKey < Familia::Horreum
+      #     feature :object_identifier
+      #     feature :external_identifier, format: 'api-%{id}'
+      #   end
+      #   key = APIKey.new
+      #   key.extid  # => "api-abc123def456ghi789"
+      #
+      # @example Using custom format template with custom prefix
+      #   class Customer < Familia::Horreum
+      #     feature :object_identifier
+      #     feature :external_identifier, format: 'cust_%{id}'
+      #   end
+      #   customer = Customer.new
+      #   customer.extid  # => "cust_abc123def456ghi789"
+      #
+      # @example Using format template without prefix
+      #   class Resource < Familia::Horreum
+      #     feature :object_identifier
+      #     feature :external_identifier, format: 'v2/%{id}'
+      #   end
+      #   resource = Resource.new
+      #   resource.extid  # => "v2/abc123def456ghi789"
+      #
       class ExternalIdentifierFieldType < Familia::FieldType
         # Override getter to provide lazy generation from objid
         #
@@ -252,11 +278,11 @@ module Familia
         # 128 bits is approximately 25 characters in base36.
         external_part = random_bytes.unpack1('H*').to_i(16).to_s(36).rjust(25, '0')
 
-        # Get prefix from feature options, default to "ext"
+        # Get format from feature options and interpolate the ID
         options = self.class.feature_options(:external_identifier)
-        prefix = options[:prefix] || 'ext'
+        format = options[:format] || 'ext_%{id}'
 
-        "#{prefix}_#{external_part}"
+        format % { id: external_part }
       end
 
       # Full-length alias for extid for clarity when needed

data/lib/familia/features/object_identifier.rb

@@ -1,4 +1,6 @@
 # lib/familia/features/object_identifier.rb
+#
+# frozen_string_literal: true
 
 module Familia
   module Features

data/lib/familia/features/quantization.rb

@@ -1,4 +1,6 @@
 # lib/familia/features/quantization.rb
+#
+# frozen_string_literal: true
 
 module Familia
   module Features
@@ -389,7 +391,7 @@ module Familia
       #   user.quantized_identifier(1.hour)  # => "123:1672531200"
       #   user.quantized_identifier(1.hour, pattern: '%Y%m%d%H')  # => "123:2023010114"
       #
-      def quantized_identifier(quantum, pattern: nil, separator: ':')
+      def quantized_identifier(quantum, pattern: nil, separator: Familia.delim)
         timestamp = qstamp(quantum, pattern: pattern)
         base_id = respond_to?(:identifier) ? identifier : object_id
         "#{base_id}#{separator}#{timestamp}"

data/lib/familia/features/relationships/README.md

@@ -18,6 +18,7 @@ participates_in Organization, :members, score: :joined_at, bidirectional: true
 
 **unique_index** - Fast unique lookups ("find object by unique field value")
 ```ruby
+unique_index :email, :email_index                        # Class-level: User.find_by_email()
 unique_index :email, :email_index, within: Organization  # Scoped: org.find_by_email()
 ```
 
@@ -89,7 +90,8 @@ class Customer < Familia::Horreum
   feature :relationships
 
   participates_in Organization, :members    # Customer belongs to org
-  unique_index :email, :email_index, within: Organization  # Find by unique email
+  unique_index :email, :email_index        # Class-level: Customer.find_by_email()
+  multi_index :status, :status_index, within: Organization  # Scoped: org.find_all_by_status()
 end
 ```
 

data/lib/familia/features/relationships/collection_operations.rb

@@ -1,4 +1,6 @@
 # lib/familia/features/relationships/collection_operations.rb
+#
+# frozen_string_literal: true
 
 module Familia
   module Features

data/lib/familia/features/relationships/indexing/multi_index_generators.rb

@@ -1,3 +1,5 @@
+# lib/familia/features/relationships/indexing/multi_index_generators.rb
+#
 # frozen_string_literal: true
 
 module Familia
@@ -78,7 +80,7 @@ module Familia
               # This acts as a factory for field-value-specific DataTypes
               define_method(:"#{index_name}_for") do |field_value|
                 # Return properly managed DataType instance with parameterized key
-                index_key = "#{index_name}:#{field_value}"
+                index_key = Familia.join(index_name, field_value)
                 Familia::UnsortedSet.new(index_key, parent: self)
               end
             end
@@ -97,6 +99,9 @@ module Familia
             # Resolve scope class using Familia pattern
             actual_scope_class = Familia.resolve_class(scope_class)
 
+            # Get scope_class_config for method naming (needed for rebuild methods)
+            scope_class_config = actual_scope_class.config_name
+
             # Generate instance sampling method (e.g., company.sample_from_department)
             actual_scope_class.class_eval do
 
@@ -118,11 +123,135 @@ module Familia
                 index_set.members.map { |id| indexed_class.find_by_identifier(id) }
               end
 
-              # Generate method to rebuild the index for this parent instance
-              define_method(:"rebuild_#{index_name}") do
-                # This would need to be implemented based on how you track which
-                # objects belong to this parent instance
-                # For now, just a placeholder
+              # Generate method to rebuild the multi-value index for this parent instance
+              #
+              # Multi-indexes create separate sets for each field value, requiring a three-phase approach:
+              # 1. Loading: Load all objects once and cache them (discovers field values simultaneously)
+              # 2. Clearing: Remove all existing index sets using SCAN
+              # 3. Rebuilding: Rebuild index from cached objects (no reload needed)
+              #
+              # @param batch_size [Integer] Number of identifiers to process per batch
+              # @yield [progress] Optional block called with progress updates
+              # @yieldparam progress [Hash] Progress information with keys:
+              #   - :phase [Symbol] Current phase (:loading, :clearing, :rebuilding)
+              #   - :current [Integer] Current item count
+              #   - :total [Integer] Total items (when known)
+              #   - :field_value [String] Current field value being processed
+              #
+              # @example Basic rebuild
+              #   company.rebuild_dept_index
+              #
+              # @example With progress monitoring
+              #   company.rebuild_dept_index do |progress|
+              #     puts "#{progress[:phase]}: #{progress[:current]}/#{progress[:total]}"
+              #   end
+              #
+              # @example Memory-conscious rebuild for large collections
+              #   # Process in smaller batches to reduce memory footprint
+              #   company.rebuild_dept_index(batch_size: 50)
+              #
+              # @note Memory Considerations:
+              #   This method caches all objects in memory during rebuild to avoid duplicate
+              #   database loads. For very large collections (>100k objects), monitor memory usage
+              #   and consider processing in chunks or using a streaming approach if memory
+              #   constraints are encountered. The batch_size parameter controls Redis I/O
+              #   batching but does not affect memory usage since all objects are cached.
+              #
+              define_method(:"rebuild_#{index_name}") do |batch_size: 100, &progress_block|
+                # PHASE 1: Find the collection containing the indexed objects
+                # Look for a participation relationship where indexed_class participates in this scope_class
+                collection_name = nil
+
+                # Check if indexed_class has participation to this scope_class
+                if indexed_class.respond_to?(:participation_relationships)
+                  participation = indexed_class.participation_relationships.find do |rel|
+                    rel.target_class == self.class
+                  end
+                  collection_name = participation&.collection_name if participation
+                end
+
+                # Get the collection DataType if we found a participation relationship
+                collection = collection_name ? send(collection_name) : nil
+
+                if collection
+                  # PHASE 2: Load objects once and cache them for both discovery and rebuilding
+                  # This avoids duplicate load_multi calls (previous approach loaded twice)
+                  progress_block&.call(phase: :loading, current: 0, total: collection.size)
+
+                  field_values = Set.new
+                  cached_objects = []
+                  processed = 0
+
+                  collection.members.each_slice(batch_size) do |identifiers|
+                    # Load objects in batches - SINGLE LOAD for both phases
+                    objects = indexed_class.load_multi(identifiers).compact
+                    cached_objects.concat(objects)
+
+                    objects.each do |obj|
+                      value = obj.send(field)
+                      # Only track non-nil, non-empty field values
+                      field_values << value.to_s if value && !value.to_s.strip.empty?
+                    end
+
+                    processed += identifiers.size
+                    progress_block&.call(phase: :loading, current: processed, total: collection.size)
+                  end
+
+                  # PHASE 3: Clear all existing field-value-specific index sets
+                  # Use SCAN to find all existing index keys (including orphaned ones from deleted field values)
+                  progress_block&.call(phase: :clearing, current: 0, total: field_values.size)
+
+                  # Get the base pattern for this index by creating a sample index set
+                  # The "*" creates a wildcard pattern like "company:123:dept_index:*" for SCAN
+                  sample_index = send(:"#{index_name}_for", "*")
+                  index_pattern = sample_index.dbkey
+
+                  # Find all existing index keys using SCAN
+                  cleared_count = 0
+                  dbclient.scan_each(match: index_pattern) do |key|
+                    dbclient.del(key)
+                    cleared_count += 1
+                    progress_block&.call(phase: :clearing, current: cleared_count, total: field_values.size, key: key)
+                  end
+
+                  # PHASE 4: Rebuild index from cached objects (no reload needed)
+                  progress_block&.call(phase: :rebuilding, current: 0, total: cached_objects.size)
+
+                  processed = 0
+                  cached_objects.each_slice(batch_size) do |objects|
+                    transaction do |_tx|
+                      objects.each do |obj|
+                        # Use the generated add_to method to maintain consistency
+                        # This ensures the same logic is used as during normal operation
+                        obj.send(:"add_to_#{scope_class_config}_#{index_name}", self)
+                      end
+                    end
+
+                    processed += objects.size
+                    progress_block&.call(phase: :rebuilding, current: processed, total: cached_objects.size)
+                  end
+
+                  Familia.info "[Rebuild] Multi-index #{index_name} rebuilt: #{field_values.size} field values, #{processed} objects"
+
+                  processed  # Return count of processed objects
+
+                else
+                  # No participation relationship found - warn and suggest alternative
+                  Familia.warn <<~WARNING
+                    [Rebuild] Cannot rebuild multi-index #{index_name}: no participation relationship found
+
+                    Multi-index rebuild requires a participation relationship to find objects.
+                    Add a participation relationship to #{indexed_class.name}:
+
+                      class #{indexed_class.name} < Familia::Horreum
+                        participates_in #{self.class.name}, :collection_name, score: :field
+                      end
+
+                    Then access the collection via: #{self.class.config_name}.collection_name
+                  WARNING
+
+                  nil
+                end
               end
             end
           end
@@ -140,7 +269,7 @@ module Familia
             scope_class_config = scope_class.config_name
             indexed_class.class_eval do
               method_name = :"add_to_#{scope_class_config}_#{index_name}"
-              Familia.ld("[MultiIndexGenerators] #{name} method #{method_name}")
+              Familia.debug("[MultiIndexGenerators] #{name} method #{method_name}")
 
               define_method(method_name) do |scope_instance|
                 return unless scope_instance
@@ -156,7 +285,7 @@ module Familia
               end
 
               method_name = :"remove_from_#{scope_class_config}_#{index_name}"
-              Familia.ld("[MultiIndexGenerators] #{name} method #{method_name}")
+              Familia.debug("[MultiIndexGenerators] #{name} method #{method_name}")
 
               define_method(method_name) do |scope_instance|
                 return unless scope_instance
@@ -172,7 +301,7 @@ module Familia
               end
 
               method_name = :"update_in_#{scope_class_config}_#{index_name}"
-              Familia.ld("[MultiIndexGenerators] #{name} method #{method_name}")
+              Familia.debug("[MultiIndexGenerators] #{name} method #{method_name}")
 
               define_method(method_name) do |scope_instance, old_field_value = nil|
                 return unless scope_instance