familia 2.0.0.pre15 → 2.0.0.pre17

data/lib/familia/horreum/{core/serialization.rb → persistence.rb}

@@ -1,5 +1,5 @@
-# lib/familia/horreum/serialization.rb
-#
+# lib/familia/horreum/persistence.rb
+
 module Familia
   # Familia::Horreum
   #
@@ -21,7 +21,7 @@ module Familia
     #   - nil - Valid response for certain operations
     #
     # @example Validating a command response
-    #   response = redis.set("key", "value")
+    #   response = dbclient.set("key", "value")
     #   valid = @valid_command_return_values.include?(response)
     #   # => true if response is "OK"
     #
@@ -31,10 +31,10 @@ module Familia
       attr_reader :valid_command_return_values
     end
 
-    # Serialization: Object persistence and retrieval from the DB
+    # Serialization - Instance-level methods for object persistence and retrieval
     # Handles conversion between Ruby objects and Valkey hash storage
     #
-    module Serialization
+    module Persistence
       # Persists the object to Valkey storage with automatic timestamping.
       #
       # Saves the current object state to Valkey storage, automatically setting
@@ -61,16 +61,26 @@ module Familia
       # @see #commit_fields The underlying method that performs the field persistence
       #
       def save(update_expiration: true)
-        Familia.trace :SAVE, dbclient, uri, caller(1..1) if Familia.debug?
+        Familia.trace :SAVE, nil, uri if Familia.debug?
 
         # No longer need to sync computed identifier with a cache field
         self.created ||= Familia.now.to_i if respond_to?(:created)
         self.updated = Familia.now.to_i if respond_to?(:updated)
 
         # Commit our tale to the Database chronicles
-        #
+        # Wrap in transaction for atomicity between save and indexing
         ret = commit_fields(update_expiration: update_expiration)
 
+        # Auto-index for class-level indexes after successful save
+        # Use transaction to ensure atomicity with the save operation
+        if ret
+          transaction do |conn|
+            auto_update_class_indexes
+            # Add to class-level instances collection after successful save
+            self.class.instances.add(identifier, Familia.now) if self.class.respond_to?(:instances)
+          end
+        end
+
         Familia.ld "[save] #{self.class} #{dbkey} #{ret} (update_expiration: #{update_expiration})"
 
         # Did Database accept our offering?
@@ -123,9 +133,9 @@ module Familia
         identifier_field = self.class.identifier_field
 
         Familia.ld "[save_if_not_exists]: #{self.class} #{identifier_field}=#{identifier}"
-        Familia.trace :SAVE_IF_NOT_EXISTS, dbclient, uri, caller(1..1) if Familia.debug?
+        Familia.trace :SAVE_IF_NOT_EXISTS, nil, uri if Familia.debug?
 
-        dbclient.watch(dbkey) do
+        success = dbclient.watch(dbkey) do
           if dbclient.exists(dbkey).positive?
             dbclient.unwatch
             raise Familia::RecordExistsError, dbkey
@@ -137,6 +147,16 @@ module Familia
 
           result.is_a?(Array) # transaction succeeded
         end
+
+        # Auto-index for class-level indexes after successful save
+        # Use transaction to ensure atomicity with the save operation
+        if success
+          transaction do |conn|
+            auto_update_class_indexes
+          end
+        end
+
+        success
       end
 
       # Commits object fields to the DB storage.
@@ -180,7 +200,7 @@ module Familia
 
       # Updates multiple fields atomically in a Database transaction.
       #
-      # @param fields [Hash] Field names and values to update. Special key :update_expiration
+      # @param kwargs [Hash] Field names and values to update. Special key :update_expiration
       #   controls whether to update key expiration (default: true)
       # @return [MultiResult] Transaction result
       #
@@ -194,9 +214,9 @@ module Familia
         update_expiration = kwargs.delete(:update_expiration) { true }
         fields = kwargs
 
-        Familia.trace :BATCH_UPDATE, dbclient, fields.keys, caller(1..1) if Familia.debug?
+        Familia.trace :BATCH_UPDATE, nil, fields.keys if Familia.debug?
 
-        command_return_values = transaction do |conn|
+        transaction_result = transaction do |conn|
           fields.each do |field, value|
             prepared_value = serialize_value(value)
             conn.hset dbkey, field, prepared_value
@@ -208,9 +228,8 @@ module Familia
         # Update expiration if requested and supported
         self.update_expiration(default_expiration: nil) if update_expiration && respond_to?(:update_expiration)
 
-        # Return same MultiResult format as other methods
-        summary_boolean = command_return_values.all? { |ret| %w[OK 0 1].include?(ret.to_s) }
-        MultiResult.new(summary_boolean, command_return_values)
+        # Return the MultiResult directly (transaction already returns MultiResult)
+        transaction_result
       end
 
       # Updates the object by applying multiple field values.
@@ -235,11 +254,12 @@ module Familia
         self
       end
 
-      # Permanently removes this object from the DB storage.
+      # Permanently removes this object and its related fields from the DB.
       #
-      # Deletes the object's Valkey key and all associated data. This operation
+      # Deletes the object's database key and all associated data. This operation
       # is irreversible and will permanently destroy all stored information
-      # for this object instance.
+      # for this object instance and the additional list, set, hash, string
+      # etc fields defined for this class.
       #
       # @return [void]
       #
@@ -259,8 +279,25 @@ module Familia
       # @see #delete! The underlying method that performs the key deletion
       #
       def destroy!
-        Familia.trace :DESTROY, dbclient, uri, caller(1..1) if Familia.debug?
-        delete!
+        Familia.trace :DESTROY, dbkey, uri
+
+        # Execute all deletion operations within a transaction
+        transaction do |conn|
+          # Delete the main object key
+          conn.del(dbkey)
+
+          # Delete all related fields if present
+          if self.class.relations?
+            Familia.trace :DELETE_RELATED_FIELDS!, nil,
+                          "#{self.class} has relations: #{self.class.related_fields.keys}"
+
+            self.class.related_fields.each do |name, _definition|
+              obj = send(name)
+              Familia.trace :DELETE_RELATED_FIELD, name, "Deleting related field #{name} (#{obj.dbkey})"
+              conn.del(obj.dbkey)
+            end
+          end
+        end
       end
 
       # Clears all fields by setting them to nil.
@@ -306,7 +343,7 @@ module Familia
       #   no authoritative source in Valkey storage.
       #
       def refresh!
-        Familia.trace :REFRESH, dbclient, uri, caller(1..1) if Familia.debug?
+        Familia.trace :REFRESH, nil, uri if Familia.debug?
         raise Familia::KeyNotFoundError, dbkey unless dbclient.exists(dbkey)
 
         fields = hgetall
@@ -341,169 +378,6 @@ module Familia
         self
       end
 
-      # Converts the object's persistent fields to a hash for external use.
-      #
-      # Serializes persistent field values for external consumption (APIs, logs),
-      # excluding non-loggable fields like encrypted fields for security.
-      # Only non-nil values are included in the resulting hash.
-      #
-      # @return [Hash] Hash with field names as keys and serialized values
-      #   safe for external exposure
-      #
-      # @example Converting an object to hash format for API response
-      #   user = User.new(name: "John", email: "john@example.com", age: 30)
-      #   user.to_h
-      #   # => {"name"=>"John", "email"=>"john@example.com", "age"=>"30"}
-      #   # encrypted fields are excluded for security
-      #
-      # @note Only loggable fields are included for security
-      # @note Only fields with non-nil values are included
-      #
-      def to_h
-        self.class.persistent_fields.each_with_object({}) do |field, hsh|
-          field_type = self.class.field_types[field]
-
-          # Security: Skip non-loggable fields (e.g., encrypted fields)
-          next unless field_type.loggable
-
-          method_name = field_type.method_name
-          val = send(method_name)
-          prepared = serialize_value(val)
-          Familia.ld " [to_h] field: #{field} val: #{val.class} prepared: #{prepared&.class || '[nil]'}"
-
-          # Only include non-nil values in the hash for Valkey
-          # Use string key for database compatibility
-          hsh[field.to_s] = prepared unless prepared.nil?
-        end
-      end
-
-      # Converts the object's persistent fields to a hash for database storage.
-      #
-      # Serializes ALL persistent field values for database storage, including
-      # encrypted fields. This is used internally by commit_fields and other
-      # persistence operations.
-      #
-      # @return [Hash] Hash with field names as keys and serialized values
-      #   ready for database storage
-      #
-      # @note Includes ALL persistent fields, including encrypted fields
-      # @note Only fields with non-nil values are included for storage efficiency
-      #
-      def to_h_for_storage
-        self.class.persistent_fields.each_with_object({}) do |field, hsh|
-          field_type = self.class.field_types[field]
-          method_name = field_type.method_name
-          val = send(method_name)
-          prepared = serialize_value(val)
-          Familia.ld " [to_h_for_storage] field: #{field} val: #{val.class} prepared: #{prepared&.class || '[nil]'}"
-
-          # Only include non-nil values in the hash for Valkey
-          # Use string key for database compatibility
-          hsh[field.to_s] = prepared unless prepared.nil?
-        end
-      end
-
-      # Converts the object's persistent fields to an array.
-      #
-      # Serializes all persistent field values in field definition order,
-      # preparing them for Valkey storage. Each value is processed through
-      # the serialization pipeline to ensure Valkey compatibility.
-      #
-      # @return [Array] Array of serialized field values in field order
-      #
-      # @example Converting an object to array format
-      #   user = User.new(name: "John", email: "john@example.com", age: 30)
-      #   user.to_a
-      #   # => ["John", "john@example.com", "30"]
-      #
-      # @note Values are serialized using the same process as other persistence
-      #   methods to maintain data consistency across operations.
-      #
-      def to_a
-        self.class.persistent_fields.filter_map do |field|
-          field_type = self.class.field_types[field]
-
-          # Security: Skip non-loggable fields (e.g., encrypted fields)
-          next unless field_type.loggable
-
-          method_name = field_type.method_name
-          val = send(method_name)
-          prepared = serialize_value(val)
-          Familia.ld " [to_a] field: #{field} method: #{method_name} val: #{val.class} prepared: #{prepared.class}"
-          prepared
-        end
-      end
-
-      # Serializes a Ruby object for Valkey storage.
-      #
-      # Converts Ruby objects into the DB-compatible string representations using
-      # the Familia distinguisher for type coercion. Falls back to JSON serialization
-      # for complex types (Hash, Array) when the primary distinguisher returns nil.
-      #
-      # The serialization process:
-      # 1. Attempts conversion using Familia.distinguisher with relaxed type checking
-      # 2. For Hash/Array types that return nil, tries custom dump_method or JSON.dump
-      # 3. Logs warnings when serialization fails completely
-      #
-      # @param val [Object] The Ruby object to serialize for Valkey storage
-      #
-      # @return [String, nil] The serialized value ready for Valkey storage, or nil
-      #   if serialization failed
-      #
-      # @example Serializing different data types
-      #   serialize_value("hello")        # => "hello"
-      #   serialize_value(42)             # => "42"
-      #   serialize_value({name: "John"}) # => '{"name":"John"}'
-      #   serialize_value([1, 2, 3])      # => "[1,2,3]"
-      #
-      # @note This method integrates with Familia's type system and supports
-      #   custom serialization methods when available on the object
-      #
-      # @see Familia.distinguisher The primary serialization mechanism
-      #
-      def serialize_value(val)
-        # Security: Handle ConcealedString safely - extract encrypted data for storage
-        return val.encrypted_value if val.respond_to?(:encrypted_value)
-
-        prepared = Familia.distinguisher(val, strict_values: false)
-
-        # If the distinguisher returns nil, try using the dump_method but only
-        # use JSON serialization for complex types that need it.
-        if prepared.nil? && (val.is_a?(Hash) || val.is_a?(Array))
-          prepared = val.respond_to?(dump_method) ? val.send(dump_method) : Familia::JsonSerializer.dump(val)
-        end
-
-        # If both the distinguisher and dump_method return nil, log an error
-        Familia.ld "[#{self.class}#serialize_value] nil returned for #{self.class}" if prepared.nil?
-
-        prepared
-      end
-
-      # Converts a Database string value back to its original Ruby type
-      #
-      # This method attempts to deserialize JSON strings back to their original
-      # Hash or Array types. Simple string values are returned as-is.
-      #
-      # @param val [String] The string value from Database to deserialize
-      # @param symbolize_keys [Boolean] Whether to symbolize hash keys (default: true for compatibility)
-      # @return [Object] The deserialized value (Hash, Array, or original string)
-      #
-      def deserialize_value(val, symbolize: true)
-        return val if val.nil? || val == ''
-
-        # Try to parse as JSON first for complex types
-        begin
-          parsed = Familia::JsonSerializer.parse(val, symbolize_names: symbolize)
-          # Only return parsed value if it's a complex type (Hash/Array)
-          # Simple values should remain as strings
-          return parsed if parsed.is_a?(Hash) || parsed.is_a?(Array)
-        rescue Familia::SerializerError
-          # Not valid JSON, return as-is
-        end
-
-        val
-      end
-
       private
 
       # Reset all transient fields to nil
@@ -522,11 +396,60 @@ module Familia
           field_type = self.class.field_types[field_name]
           next unless field_type&.method_name
 
-          # Set the transient field back to nil
+          # UnsortedSet the transient field back to nil
           send("#{field_type.method_name}=", nil)
           Familia.ld "[reset_transient_fields!] Reset #{field_name} to nil"
         end
       end
+
+      # Automatically update class-level indexes after save
+      #
+      # Iterates through class-level indexing relationships and calls their
+      # corresponding add_to_class_* methods to populate indexes. Only processes
+      # class-level indexes (where target_class == self.class), skipping
+      # instance-scoped indexes which require parent context.
+      #
+      # Uses idempotent Redis commands (HSET for unique_index) so repeated calls
+      # are safe and have negligible performance overhead. Note that multi_index
+      # always requires within: parameter, so only unique_index benefits from this.
+      #
+      # @return [void]
+      #
+      # @example Automatic indexing on save
+      #   class Customer < Familia::Horreum
+      #     feature :relationships
+      #     unique_index :email, :email_lookup
+      #   end
+      #
+      #   customer = Customer.new(email: 'test@example.com')
+      #   customer.save  # Automatically calls add_to_class_email_lookup
+      #
+      # @note Only class-level unique_index declarations auto-populate.
+      #   Instance-scoped indexes (with within:) require manual population:
+      #   employee.add_to_company_badge_index(company)
+      #
+      # @see Familia::Features::Relationships::Indexing For index declaration details
+      #
+      def auto_update_class_indexes
+        return unless self.class.respond_to?(:indexing_relationships)
+
+        self.class.indexing_relationships.each do |rel|
+          # Skip instance-scoped indexes (require parent context)
+          # Instance-scoped indexes must be manually populated because they need
+          # the parent object reference (e.g., employee.add_to_company_badge_index(company))
+          unless rel.target_class == self.class
+            Familia.ld <<~LOG_MESSAGE
+              [auto_update_class_indexes] Skipping #{rel.index_name} (requires parent context)
+            LOG_MESSAGE
+            next
+          end
+
+          # Call the existing add_to_class_* methods
+          add_method = :"add_to_class_#{rel.index_name}"
+          send(add_method) if respond_to?(add_method)
+        end
+      end
+
     end
   end
 end

data/lib/familia/horreum/{subclass/related_fields_management.rb → related_fields.rb}

@@ -1,32 +1,99 @@
-# lib/familia/horreum/related_fields_management.rb
+# lib/familia/horreum/related_fields.rb
 
 module Familia
+
+  RelatedFieldDefinition = Data.define(:name, :klass, :opts)
+
   class Horreum
+
+    # Each related field needs some details from the parent (Horreum model)
+    # in order to generate its dbkey. We use a parent proxy pattern to store
+    # only essential parent information instead of full object reference. We
+    # need only the model class and an optional unique identifier to generate
+    # the dbkey; when the identifier is nil, we treat this as a class-level
+    # relation (e.g. model_name:related_field_name); when the identifier
+    # is not nil, we treat this as an instance-level relation
+    # (model_name:identifier:related_field_name).
+    #
+    ParentDefinition = Data.define(:model_klass, :identifier) do
+      # Factory method to create ParentDefinition from a parent instance
+      def self.from_parent(parent_instance)
+        case parent_instance
+        when Class
+          # Handle class-level relationships
+          new(parent_instance, nil)
+        else
+          # Handle instance-level relationships
+          identifier = parent_instance.respond_to?(:identifier) ? parent_instance.identifier : nil
+          new(parent_instance.class, identifier)
+        end
+      end
+
+      # Delegation methods for common operations needed by DataTypes
+      def dbclient(uri = nil)
+        model_klass.dbclient(uri)
+      end
+
+      def logical_database
+        model_klass.logical_database
+      end
+
+      def dbkey(keystring = nil)
+        if identifier
+          # Instance-level relation: model_name:identifier:keystring
+          model_klass.dbkey(identifier, keystring)
+        else
+          # Class-level relation: model_name:keystring
+          model_klass.dbkey(keystring, nil)
+        end
+      end
+
+      # Allow comparison with the original parent instance
+      def ==(other)
+        case other
+        when ParentDefinition
+          model_klass == other.model_klass && identifier == other.identifier
+        when Class
+          model_klass == other && identifier.nil?
+        else
+          # Compare with instance: check class and identifier match
+          other.is_a?(model_klass) && other.respond_to?(:identifier) && identifier == other.identifier
+        end
+      end
+      alias eql? ==
+    end
+
+    # RelatedFieldsManagement - Class-level methods for defining DataType relationships
     #
-    # RelatedFieldsManagement: Manages DataType fields and relations
+    # This module uses metaprogramming to dynamically create field definition methods
+    # that generate both class-level and instance-level accessor methods for DataTypes
+    # (e.g., list, set, zset, hashkey, string).
     #
-    # This module uses metaprogramming to dynamically create methods
-    # for managing different types of Database objects (e.g., sets, lists, hashes).
+    # When included in a class via ManagementMethods, it provides class methods like:
+    # * Customer.list :recent_orders    # defines class method for class-level list
+    # * customer.recent_orders          # creates instance method returning list instance
     #
     # Key metaprogramming features:
-    # * Dynamically defines methods for each Database type (e.g., set, list, hashkey)
-    # * Creates both instance-level and class-level relation methods
+    # * Dynamically defines DSL methods for each Database type (e.g., set, list, hashkey)
+    # * Each DSL method creates corresponding instance/class accessor methods
     # * Provides query methods for checking relation types
     #
     # Usage:
     #   Include this module in classes that need DataType management
-    #   Call setup_related_fields_accessors to initialize the feature
+    #   Call setup_related_fields_definition_methods to initialize the feature
     #
     module RelatedFieldsManagement
       # A practical flag to indicate that a Horreum member has relations,
       # not just theoretically but actually at least one list/haskey/etc.
-      @has_relations = nil
+      @has_related_fields = nil
 
       def self.included(base)
         base.extend(RelatedFieldsAccessors)
-        base.setup_related_fields_accessors
+        base.setup_related_fields_definition_methods
       end
 
+      # RelatedFieldsManagement::RelatedFieldsAccessors
+      #
       module RelatedFieldsAccessors
         # Sets up all DataType related methods
         # This method generates the following for each registered DataType:
@@ -36,9 +103,9 @@ module Familia
         # Collection methods: sets(), lists(), hashkeys(), sorted_sets(), etc.
         # Class methods: class_set(), class_list(), etc.
         #
-        def setup_related_fields_accessors
+        def setup_related_fields_definition_methods
           Familia::DataType.registered_types.each_pair do |kind, klass|
-            Familia.trace :registered_types, kind, klass, caller(1..1) if Familia.debug?
+            Familia.trace :registered_types, kind, klass if Familia.debug?
 
             # Dynamically define instance-level relation methods
             #
@@ -50,7 +117,7 @@ module Familia
               name, opts = *args
 
               # As log as we have at least one relation, we can set this flag.
-              @has_relations = true
+              @has_related_fields = true
 
               attach_instance_related_field name, klass, opts
             end
@@ -95,18 +162,35 @@ module Familia
 
       # Creates an instance-level relation
       def attach_instance_related_field(name, klass, opts)
-        Familia.trace :attach_instance, "#{name} #{klass}", opts, caller(1..1) if Familia.debug?
+        Familia.trace :attach_instance_related_field, name, klass, opts if Familia.debug?
         raise ArgumentError, "Name is blank (#{klass})" if name.to_s.empty?
 
         name = name.to_s.to_sym
         opts ||= {}
 
-        related_fields[name] = Struct.new(:name, :klass, :opts).new
-        related_fields[name].name = name
-        related_fields[name].klass = klass
-        related_fields[name].opts = opts
+        related_fields[name] = RelatedFieldDefinition.new(name, klass, opts)
 
-        attr_reader name
+        # Create lazy-initializing accessor that calls initialize_relatives if needed
+        define_method name do
+          ivar = :"@#{name}"
+          value = instance_variable_get(ivar)
+
+          # If nil and we haven't initialized relatives, do it now
+          # Check singleton class to avoid polluting instance variables
+          if value.nil? && !singleton_class.instance_variable_defined?(:"@relatives_initialized")
+            initialize_relatives
+            value = instance_variable_get(ivar)
+          end
+
+          # If still nil after lazy initialization attempt, raise helpful error
+          # Only raise if we tried to initialize but it's still nil
+          if value.nil? && singleton_class.instance_variable_defined?(:"@relatives_initialized")
+            raise "#{self.class}##{name} is nil. Did you override initialize without calling super? " \
+                  "(Field is nil after initialization attempt)"
+          end
+
+          value
+        end
 
         define_method :"#{name}=" do |val|
           send(name).replace val
@@ -120,17 +204,14 @@ module Familia
 
       # Creates a class-level relation
       def attach_class_related_field(name, klass, opts)
-        Familia.trace :attach_class_related_field, "#{name} #{klass}", opts, caller(1..1) if Familia.debug?
+        Familia.trace :attach_class_related_field, "#{name} #{klass}", opts if Familia.debug?
         raise ArgumentError, 'Name is blank (klass)' if name.to_s.empty?
 
         name = name.to_s.to_sym
         opts = opts.nil? ? {} : opts.clone
         opts[:parent] = self unless opts.key?(:parent)
 
-        class_related_fields[name] = Struct.new(:name, :klass, :opts).new
-        class_related_fields[name].name = name
-        class_related_fields[name].klass = klass
-        class_related_fields[name].opts = opts
+        class_related_fields[name] = RelatedFieldDefinition.new(name, klass, opts)
 
         # An accessor method created in the metaclass will
         # access the instance variables for this class.