familia 2.0.0.pre23 → 2.0.0.pre25

data/lib/familia/features/relationships/indexing/multi_index_generators.rb

@@ -29,6 +29,48 @@ module Familia
 
           using Familia::Refinements::StylizeWords
 
+          # Maximum recommended length for field values used in index keys.
+          # Longer values are allowed but will trigger a warning.
+          MAX_FIELD_VALUE_LENGTH = 256
+
+          # Validates a field value for use in index key construction.
+          # This is primarily for data quality and debugging clarity, not security.
+          #
+          # Security note: Redis SCAN patterns use the namespace prefix (e.g.,
+          # "customer:role_index:") which is derived from class/index metadata,
+          # not user input. Glob characters in stored field values are treated
+          # as literal characters in key names, not as pattern wildcards.
+          #
+          # @param field_value [Object] The field value to validate
+          # @param context [String] Description for warning messages
+          # @return [String, nil] The validated string value, or nil if invalid
+          def validate_field_value(field_value, context: 'index')
+            return nil if field_value.nil?
+
+            str_value = field_value.to_s
+            return nil if str_value.strip.empty?
+
+            # Warn on values containing Redis glob pattern characters
+            # These are legal but can be confusing when debugging key patterns
+            if str_value.match?(/[*?\[\]]/)
+              Familia.warn "[#{context}] Field value contains glob pattern characters: #{str_value.inspect}. " \
+                           'These are stored as literal characters but may be confusing during debugging.'
+            end
+
+            # Warn on control characters (except common whitespace)
+            if str_value.match?(/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/)
+              Familia.warn "[#{context}] Field value contains control characters: #{str_value.inspect}"
+            end
+
+            # Warn on excessively long values
+            if str_value.length > MAX_FIELD_VALUE_LENGTH
+              Familia.warn "[#{context}] Field value exceeds #{MAX_FIELD_VALUE_LENGTH} characters " \
+                           "(#{str_value.length} chars): #{str_value[0..50]}..."
+            end
+
+            str_value
+          end
+
           # Main setup method that orchestrates multi-value index creation
           #
           # @param indexed_class [Class] The class being indexed (e.g., Employee)
@@ -37,32 +79,36 @@ module Familia
           # @param within [Class, Symbol] Scope class for instance-scoped index (required)
           # @param query [Boolean] Whether to generate query methods
           def setup(indexed_class:, field:, index_name:, within:, query:)
-            # Multi-index always requires a scope context
-            scope_class = within
-            resolved_class = Familia.resolve_class(scope_class)
+            # Determine scope type: class-level or instance-scoped
+            scope_class, scope_type = if within == :class
+              [indexed_class, :class]
+            else
+              k = Familia.resolve_class(within)
+              [k, :instance]
+            end
 
             # Store metadata for this indexing relationship
             indexed_class.indexing_relationships << IndexingRelationship.new(
               field:             field,
               scope_class:       scope_class,
-              within:            within,
+              within:            within,  # Preserve original (:class or actual class)
               index_name:        index_name,
               query:            query,
               cardinality:       :multi,
             )
 
-            # Always generate the factory method - required by mutation methods
-            if scope_class.is_a?(Class)
-              generate_factory_method(resolved_class, index_name)
-            end
-
-            # Generate query methods on the scope class (optional)
-            if query && scope_class.is_a?(Class)
-              generate_query_methods_destination(indexed_class, field, resolved_class, index_name)
+            case scope_type
+            when :instance
+              # Instance-scoped multi-index (existing behavior)
+              generate_factory_method(scope_class, index_name)
+              generate_query_methods_destination(indexed_class, field, scope_class, index_name) if query
+              generate_mutation_methods_self(indexed_class, field, scope_class, index_name)
+            when :class
+              # Class-level multi-index (new behavior)
+              generate_factory_method_class(indexed_class, index_name)
+              generate_query_methods_class(indexed_class, field, index_name) if query
+              generate_mutation_methods_class(indexed_class, field, index_name)
             end
-
-            # Generate mutation methods on the indexed class
-            generate_mutation_methods_self(indexed_class, field, resolved_class, index_name)
           end
 
           # Generates the factory method ON THE SCOPE CLASS (Company when within: Company):
@@ -325,6 +371,226 @@ module Familia
               end
             end
           end
+
+          # =========================================================================
+          # CLASS-LEVEL MULTI-INDEX GENERATORS
+          # =========================================================================
+          #
+          # When within: :class is used, these generators create class-level methods
+          # instead of instance-scoped methods.
+          #
+          # Example:
+          #   multi_index :role, :role_index  # within: :class is default
+          #
+          # Generates on Customer (class methods):
+          #   - Customer.role_index_for('admin')   -> UnsortedSet factory
+          #   - Customer.find_all_by_role('admin') -> [Customer, ...]
+          #   - Customer.sample_from_role('admin', 3) -> random sample
+          #   - Customer.rebuild_role_index          -> rebuild index
+          #
+          # Generates on Customer (instance methods, auto-called on save):
+          #   - customer.add_to_class_role_index
+          #   - customer.remove_from_class_role_index
+          #   - customer.update_in_class_role_index(old_value)
+
+          # Generates class-level factory method:
+          # - Customer.role_index_for(field_value) -> UnsortedSet
+          #
+          # The factory validates field values for data quality. Glob pattern
+          # characters (*, ?, [, ]) in field values are allowed but trigger
+          # warnings since they can be confusing during debugging.
+          #
+          # @param indexed_class [Class] The class being indexed (e.g., Customer)
+          # @param index_name [Symbol] Name of the index (e.g., :role_index)
+          def generate_factory_method_class(indexed_class, index_name)
+            # Capture index_name for use in validation context
+            idx_name = index_name
+            indexed_class.define_singleton_method(:"#{index_name}_for") do |field_value|
+              # Validate field value and use the validated string for consistent key format.
+              # Validation returns nil for nil/empty values, string otherwise.
+              # We allow nil through (creates a "null" index key) but use the validated
+              # string to ensure consistent type handling in key construction.
+              validated = MultiIndexGenerators.validate_field_value(field_value, context: "#{name}.#{idx_name}")
+              index_key = Familia.join(index_name, validated)
+              Familia::UnsortedSet.new(index_key, parent: self)
+            end
+          end
+
+          # Generates class-level query methods:
+          # - Customer.find_all_by_role(value) -> [Customer, ...]
+          # - Customer.sample_from_role(value, count) -> random sample
+          # - Customer.rebuild_role_index -> rebuild index
+          #
+          # @param indexed_class [Class] The class being indexed (e.g., Customer)
+          # @param field [Symbol] The field to index (e.g., :role)
+          # @param index_name [Symbol] Name of the index (e.g., :role_index)
+          def generate_query_methods_class(indexed_class, field, index_name)
+            # find_all_by_role(value)
+            # Uses load_multi for efficient batch loading (avoids N+1 queries)
+            indexed_class.define_singleton_method(:"find_all_by_#{field}") do |field_value|
+              index_set = send("#{index_name}_for", field_value)
+              identifiers = index_set.members
+              load_multi(identifiers).compact
+            end
+
+            # sample_from_role(value, count)
+            # Uses load_multi for efficient batch loading (avoids N+1 queries)
+            indexed_class.define_singleton_method(:"sample_from_#{field}") do |field_value, count = 1|
+              return [] if field_value.nil? || field_value.to_s.strip.empty?
+
+              index_set = send("#{index_name}_for", field_value)
+              identifiers = index_set.sample(count)
+              load_multi(identifiers).compact
+            end
+
+            # rebuild_role_index(batch_size:, &progress)
+            # For class-level indexes, we iterate all instances of the class
+            indexed_class.define_singleton_method(:"rebuild_#{index_name}") do |batch_size: 100, &progress_block|
+              # PHASE 1: Discover all field values and collect objects
+              progress_block&.call(phase: :discovering, current: 0, total: 0)
+
+              # Use class-level instances collection if available
+              unless respond_to?(:instances) && instances.respond_to?(:members)
+                Familia.warn "[Rebuild] Cannot rebuild class-level multi-index #{index_name}: " \
+                             "no instances collection found. " \
+                             "Ensure #{name} has class_sorted_set :instances or similar."
+                return 0  # Return 0 for consistency - always return integer count
+              end
+
+              field_values = Set.new
+              cached_objects = []
+              processed = 0
+              total_count = instances.size
+
+              progress_block&.call(phase: :loading, current: 0, total: total_count)
+
+              instances.members.each_slice(batch_size) do |identifiers|
+                objects = load_multi(identifiers).compact
+                cached_objects.concat(objects)
+
+                objects.each do |obj|
+                  value = obj.send(field)
+                  field_values << value.to_s if value && !value.to_s.strip.empty?
+                end
+
+                processed += identifiers.size
+                progress_block&.call(phase: :loading, current: processed, total: total_count)
+              end
+
+              # PHASE 2: Clear existing index sets using SCAN
+              progress_block&.call(phase: :clearing, current: 0, total: field_values.size)
+
+              # Get pattern for all index keys: "customer:role_index:*"
+              #
+              # Security note: The SCAN pattern is safe because:
+              # 1. The namespace prefix (e.g., "customer:role_index:") is derived from
+              #    class metadata and index name, not user input
+              # 2. The "*" wildcard only matches keys within this namespace
+              # 3. Glob characters stored IN field values (e.g., a role named "admin*")
+              #    are literal characters in the key name, not SCAN wildcards
+              # 4. SCAN cannot match keys outside the namespace prefix
+              sample_index = send(:"#{index_name}_for", "*")
+              index_pattern = sample_index.dbkey
+
+              cleared_count = 0
+              dbclient.scan_each(match: index_pattern) do |key|
+                dbclient.del(key)
+                cleared_count += 1
+                progress_block&.call(phase: :clearing, current: cleared_count, total: field_values.size, key: key)
+              end
+
+              # PHASE 3: Rebuild from cached objects
+              progress_block&.call(phase: :rebuilding, current: 0, total: cached_objects.size)
+
+              processed = 0
+              cached_objects.each_slice(batch_size) do |objects|
+                dbclient.multi do |conn|
+                  objects.each do |obj|
+                    field_value = obj.send(field)
+                    next unless field_value && !field_value.to_s.strip.empty?
+
+                    index_set = send("#{index_name}_for", field_value)
+                    # Use JsonSerializer for consistent serialization with update method
+                    serialized_id = Familia::JsonSerializer.dump(obj.identifier)
+                    conn.sadd(index_set.dbkey, serialized_id)
+                  end
+                end
+
+                processed += objects.size
+                progress_block&.call(phase: :rebuilding, current: processed, total: cached_objects.size)
+              end
+
+              Familia.info "[Rebuild] Class-level multi-index #{index_name} rebuilt: " \
+                           "#{field_values.size} field values, #{processed} objects"
+
+              processed
+            end
+          end
+
+          # Generates instance mutation methods for class-level indexes:
+          # - customer.add_to_class_role_index
+          # - customer.remove_from_class_role_index
+          # - customer.update_in_class_role_index(old_value)
+          #
+          # These are auto-called on save/destroy when auto-indexing is enabled.
+          #
+          # @param indexed_class [Class] The class being indexed (e.g., Customer)
+          # @param field [Symbol] The field to index (e.g., :role)
+          # @param index_name [Symbol] Name of the index (e.g., :role_index)
+          def generate_mutation_methods_class(indexed_class, field, index_name)
+            indexed_class.class_eval do
+              method_name = :"add_to_class_#{index_name}"
+              Familia.debug("[MultiIndexGenerators] #{name} class method #{method_name}")
+
+              define_method(method_name) do
+                field_value = send(field)
+                return unless field_value && !field_value.to_s.strip.empty?
+
+                index_set = self.class.send("#{index_name}_for", field_value)
+                index_set.add(identifier)
+              end
+
+              method_name = :"remove_from_class_#{index_name}"
+              Familia.debug("[MultiIndexGenerators] #{name} class method #{method_name}")
+
+              define_method(method_name) do
+                field_value = send(field)
+                return unless field_value && !field_value.to_s.strip.empty?
+
+                index_set = self.class.send("#{index_name}_for", field_value)
+                index_set.remove(identifier)
+              end
+
+              method_name = :"update_in_class_#{index_name}"
+              Familia.debug("[MultiIndexGenerators] #{name} class method #{method_name}")
+
+              define_method(method_name) do |old_field_value|
+                return unless old_field_value
+
+                new_field_value = send(field)
+                return if old_field_value == new_field_value
+
+                # Get the index sets for old and new values
+                old_set = self.class.send("#{index_name}_for", old_field_value)
+
+                # Use DataType's serialize_value for consistency with add/remove methods.
+                # This ensures the same serialization path is used across all index operations.
+                serialized_id = old_set.serialize_value(identifier)
+
+                # Use transaction for atomic remove + add to prevent data inconsistency
+                transaction do |conn|
+                  # Remove from old index
+                  conn.srem(old_set.dbkey, serialized_id)
+
+                  # Add to new index if present
+                  if new_field_value && !new_field_value.to_s.strip.empty?
+                    new_set = self.class.send("#{index_name}_for", new_field_value)
+                    conn.sadd(new_set.dbkey, serialized_id)
+                  end
+                end
+              end
+            end
+          end
         end
       end
     end

data/lib/familia/features/relationships/indexing.rb

@@ -98,7 +98,7 @@ module Familia
           # @example Instance-scoped multi-value indexing
           #   multi_index :department, :dept_index, within: Company
           #
-          def multi_index(field, index_name, within:, query: true)
+          def multi_index(field, index_name, within: :class, query: true)
             MultiIndexGenerators.setup(
               indexed_class: self,
               field: field,
@@ -159,12 +159,14 @@ module Familia
               index_name = config.index_name
               old_field_value = old_values[field]
 
-              # Determine which update method to call
-              if config.within.nil?
-                # Class-level index (unique_index without within:)
+              # Determine which update method to call based on scope type
+              # Class-level: within is nil (unique_index default) or :class (multi_index default)
+              # Instance-scoped: within is a specific class
+              if config.within.nil? || config.within == :class
+                # Class-level index (auto-indexed on save)
                 send("update_in_class_#{index_name}", old_field_value)
               else
-                # Instance-scoped index (unique_index or multi_index with within:) - requires scope context
+                # Instance-scoped index - requires explicit scope context
                 next unless scope_context
 
                 # Use config_name for method naming
@@ -183,12 +185,14 @@ module Familia
             self.class.indexing_relationships.each do |config|
               index_name = config.index_name
 
-              # Determine which remove method to call
-              if config.within.nil?
-                # Class-level index (unique_index without within:)
+              # Determine which remove method to call based on scope type
+              # Class-level: within is nil (unique_index default) or :class (multi_index default)
+              # Instance-scoped: within is a specific class
+              if config.within.nil? || config.within == :class
+                # Class-level index (auto-indexed on save)
                 send("remove_from_class_#{index_name}")
               else
-                # Instance-scoped index (unique_index or multi_index with within:) - requires scope context
+                # Instance-scoped index - requires explicit scope context
                 next unless scope_context
 
                 # Use config_name for method naming
@@ -216,20 +220,38 @@ module Familia
 
               next unless field_value
 
-              if config.within.nil?
-                # Class-level index (unique_index without within:) - check hash key using DataType
-                index_hash = self.class.send(index_name)
-                next unless index_hash.key?(field_value.to_s)
+              # Class-level indexes have within: nil (unique_index) or within: :class (multi_index)
+              # Instance-scoped indexes have within: SomeClass (a specific class)
+              if config.within.nil? || config.within == :class
+                if cardinality == :unique
+                  # Class-level unique index - check hash key using DataType
+                  index_hash = self.class.send(index_name)
+                  next unless index_hash.key?(field_value.to_s)
 
-                memberships << {
-                  scope_class: 'class',
-                  index_name: index_name,
-                  field: field,
-                  field_value: field_value,
-                  index_key: index_hash.dbkey,
-                  cardinality: cardinality,
-                  type: 'unique_index',
-                }
+                  memberships << {
+                    scope_class: 'class',
+                    index_name: index_name,
+                    field: field,
+                    field_value: field_value,
+                    index_key: index_hash.dbkey,
+                    cardinality: cardinality,
+                    type: 'unique_index',
+                  }
+                else
+                  # Class-level multi index - check set membership using factory method
+                  index_set = self.class.send("#{index_name}_for", field_value)
+                  next unless index_set.member?(identifier)
+
+                  memberships << {
+                    scope_class: 'class',
+                    index_name: index_name,
+                    field: field,
+                    field_value: field_value,
+                    index_key: index_set.dbkey,
+                    cardinality: cardinality,
+                    type: 'multi_index',
+                  }
+                end
               else
                 # Instance-scoped index (unique_index or multi_index with within:) - cannot check without scope instance
                 # This would require scanning all possible scope instances
@@ -250,7 +272,7 @@ module Familia
           end
 
           # Check if this object is indexed in a specific scope
-          # For class-level indexes, checks the hash key
+          # For class-level indexes, checks the hash key (unique) or set membership (multi)
           # For instance-scoped indexes, returns false (requires scope instance)
           def indexed_in?(index_name)
             return false unless self.class.respond_to?(:indexing_relationships)
@@ -262,10 +284,18 @@ module Familia
             field_value = send(field)
             return false unless field_value
 
-            if config.within.nil?
-              # Class-level index (class_indexed_by) - check hash key using DataType
-              index_hash = self.class.send(index_name)
-              index_hash.key?(field_value.to_s)
+            # Class-level indexes have within: nil (unique_index) or within: :class (multi_index)
+            # Instance-scoped indexes have within: SomeClass (a specific class)
+            if config.within.nil? || config.within == :class
+              if config.cardinality == :unique
+                # Class-level unique index - check hash key using DataType
+                index_hash = self.class.send(index_name)
+                index_hash.key?(field_value.to_s)
+              else
+                # Class-level multi index - check set membership using factory method
+                index_set = self.class.send("#{index_name}_for", field_value)
+                index_set.member?(identifier)
+              end
             else
               # Instance-scoped index (with within:) - cannot verify without scope instance
               false

data/lib/familia/features/safe_dump.rb

@@ -46,9 +46,6 @@ module Familia
 
       using Familia::Refinements::StylizeWords
 
-      @dump_method = :to_json
-      @load_method = :from_json
-
       def self.included(base)
         Familia.trace(:LOADED, self, base) if Familia.debug?
         base.extend ModelClassMethods

data/lib/familia/horreum/management.rb

@@ -156,7 +156,10 @@ module Familia
           # doesn't, we return nil. If it does, we proceed to load the object.
           # Otherwise, hgetall will return an empty hash, which will be passed to
           # the constructor, which will then be annoying to debug.
-          return unless does_exist
+          unless does_exist
+            cleanup_stale_instance_entry(objkey)
+            return nil
+          end
         else
           # Optimized mode: Skip existence check
           Familia.debug "[find_by_key] #{self} from key #{objkey} (check_exists: false)"
@@ -166,12 +169,42 @@ module Familia
         obj = dbclient.hgetall(objkey) # horreum objects are persisted as database hashes
         Familia.trace :FIND_BY_DBKEY_INSPECT, nil, "#{objkey}: #{obj.inspect}"
 
-        # If we skipped existence check and got empty hash, key doesn't exist
-        return nil if !check_exists && obj.empty?
+        # Always check for empty hash to handle race conditions where the key
+        # expires between EXISTS check and HGETALL (when check_exists: true),
+        # or simply doesn't exist (when check_exists: false).
+        if obj.empty?
+          cleanup_stale_instance_entry(objkey)
+          return nil
+        end
 
         # Create instance and deserialize fields using shared helper method
         instantiate_from_hash(obj)
       end
+
+      # Removes a stale entry from the instances sorted set.
+      # Called when find_by_dbkey detects that an object no longer exists
+      # (either EXISTS returned false, or HGETALL returned empty hash).
+      #
+      # This provides lazy cleanup of phantom instance entries that can
+      # accumulate when objects expire via TTL without explicit destroy!
+      #
+      # @param objkey [String] The full database key (prefix:identifier:suffix)
+      # @return [void]
+      # @api private
+      def cleanup_stale_instance_entry(objkey)
+        return unless respond_to?(:instances)
+
+        # Key format is prefix:identifier:suffix, so identifier is at index 1
+        parts = Familia.split(objkey)
+        return unless parts.length >= 2
+
+        identifier = parts[1]
+        return if identifier.nil? || identifier.empty?
+
+        instances.remove(identifier)
+        Familia.debug "[find_by_dbkey] Removed stale instance entry: #{identifier}"
+      end
+      private :cleanup_stale_instance_entry
       alias find_by_key find_by_dbkey
 
       # Retrieves and instantiates an object from Database using its identifier.

data/lib/familia/horreum/persistence.rb

@@ -589,7 +589,10 @@ module Familia
           # Skip instance-scoped indexes (require scope context)
           # Instance-scoped indexes must be manually populated because they need
           # the scope instance reference (e.g., employee.add_to_company_badge_index(company))
-          if rel.within
+          #
+          # Class-level indexes have within: nil (unique_index) or within: :class (multi_index)
+          # Instance-scoped indexes have within: SomeClass (a specific class)
+          if rel.within && rel.within != :class
             Familia.debug <<~LOG_MESSAGE
               [auto_update_class_indexes] Skipping #{rel.index_name} (requires scope context)
             LOG_MESSAGE

data/lib/familia/horreum/settings.rb

@@ -10,10 +10,10 @@ module Familia
   #
   class Horreum
     # Settings - Instance-level configuration methods for Horreum models
-    # Provides per-instance settings like logical_database, dump_method, load_method, suffix
+    # Provides per-instance settings like logical_database, suffix
     #
     module Settings
-      attr_writer :dump_method, :load_method, :suffix
+      attr_writer :suffix
 
       def opts
         @opts ||= {}
@@ -40,14 +40,6 @@ module Familia
       def suffix
         @suffix || self.class.suffix
       end
-
-      def dump_method
-        @dump_method || self.class.dump_method
-      end
-
-      def load_method
-        @load_method || self.class.load_method
-      end
     end
   end
 end

data/lib/familia/horreum.rb

@@ -80,13 +80,12 @@ module Familia
     # Class-Level Attributes:
     # * @parent - Parent object reference for nested relationships
     # * @dbclient - Database connection override for this class
-    # * @dump_method/@load_method - Serialization method configuration
     # * @has_related_fields - Flag indicating if DataType relationships are defined
     #
     class << self
       attr_accessor :parent
       # TODO: Where are we calling dbclient= from now with connection pool?
-      attr_writer :dbclient, :dump_method, :load_method
+      attr_writer :dbclient
       attr_reader :has_related_fields
 
       # Extends ClassMethods to subclasses and tracks Familia members

data/lib/familia/version.rb

@@ -4,5 +4,5 @@
 
 module Familia
   # Version information for the Familia
-  VERSION = '2.0.0.pre23'.freeze unless defined?(Familia::VERSION)
+  VERSION = '2.0.0.pre25' unless defined?(Familia::VERSION)
 end