familia 2.0.0.pre16 → 2.0.0.pre18

data/lib/familia/horreum/{core/serialization.rb → persistence.rb}

@@ -1,4 +1,4 @@
-# lib/familia/horreum/serialization.rb
+# lib/familia/horreum/persistence.rb
 
 module Familia
   # Familia::Horreum
@@ -34,7 +34,7 @@ module Familia
     # Serialization - Instance-level methods for object persistence and retrieval
     # Handles conversion between Ruby objects and Valkey hash storage
     #
-    module Serialization
+    module Persistence
       # Persists the object to Valkey storage with automatic timestamping.
       #
       # Saves the current object state to Valkey storage, automatically setting
@@ -68,11 +68,18 @@ module Familia
         self.updated = Familia.now.to_i if respond_to?(:updated)
 
         # Commit our tale to the Database chronicles
-        #
+        # Wrap in transaction for atomicity between save and indexing
         ret = commit_fields(update_expiration: update_expiration)
 
-        # Add to class-level instances collection after successful save
-        self.class.instances.add(identifier, Familia.now) if ret && self.class.respond_to?(:instances)
+        # Auto-index for class-level indexes after successful save
+        # Use transaction to ensure atomicity with the save operation
+        if ret
+          transaction do |conn|
+            auto_update_class_indexes
+            # Add to class-level instances collection after successful save
+            self.class.instances.add(identifier, Familia.now) if self.class.respond_to?(:instances)
+          end
+        end
 
         Familia.ld "[save] #{self.class} #{dbkey} #{ret} (update_expiration: #{update_expiration})"
 
@@ -128,7 +135,7 @@ module Familia
         Familia.ld "[save_if_not_exists]: #{self.class} #{identifier_field}=#{identifier}"
         Familia.trace :SAVE_IF_NOT_EXISTS, nil, uri if Familia.debug?
 
-        dbclient.watch(dbkey) do
+        success = dbclient.watch(dbkey) do
           if dbclient.exists(dbkey).positive?
             dbclient.unwatch
             raise Familia::RecordExistsError, dbkey
@@ -140,6 +147,16 @@ module Familia
 
           result.is_a?(Array) # transaction succeeded
         end
+
+        # Auto-index for class-level indexes after successful save
+        # Use transaction to ensure atomicity with the save operation
+        if success
+          transaction do |conn|
+            auto_update_class_indexes
+          end
+        end
+
+        success
       end
 
       # Commits object fields to the DB storage.
@@ -337,7 +354,7 @@ module Familia
         # their uninitialized state during refresh operations
         reset_transient_fields!
 
-        optimistic_refresh(**fields)
+        naive_refresh(**fields)
       end
 
       # Refreshes object state from the DB and returns self for method chaining.
@@ -361,169 +378,6 @@ module Familia
         self
       end
 
-      # Converts the object's persistent fields to a hash for external use.
-      #
-      # Serializes persistent field values for external consumption (APIs, logs),
-      # excluding non-loggable fields like encrypted fields for security.
-      # Only non-nil values are included in the resulting hash.
-      #
-      # @return [Hash] Hash with field names as keys and serialized values
-      #   safe for external exposure
-      #
-      # @example Converting an object to hash format for API response
-      #   user = User.new(name: "John", email: "john@example.com", age: 30)
-      #   user.to_h
-      #   # => {"name"=>"John", "email"=>"john@example.com", "age"=>"30"}
-      #   # encrypted fields are excluded for security
-      #
-      # @note Only loggable fields are included for security
-      # @note Only fields with non-nil values are included
-      #
-      def to_h
-        self.class.persistent_fields.each_with_object({}) do |field, hsh|
-          field_type = self.class.field_types[field]
-
-          # Security: Skip non-loggable fields (e.g., encrypted fields)
-          next unless field_type.loggable
-
-          method_name = field_type.method_name
-          val = send(method_name)
-          prepared = serialize_value(val)
-          Familia.ld " [to_h] field: #{field} val: #{val.class} prepared: #{prepared&.class || '[nil]'}"
-
-          # Only include non-nil values in the hash for Valkey
-          # Use string key for database compatibility
-          hsh[field.to_s] = prepared unless prepared.nil?
-        end
-      end
-
-      # Converts the object's persistent fields to a hash for database storage.
-      #
-      # Serializes ALL persistent field values for database storage, including
-      # encrypted fields. This is used internally by commit_fields and other
-      # persistence operations.
-      #
-      # @return [Hash] Hash with field names as keys and serialized values
-      #   ready for database storage
-      #
-      # @note Includes ALL persistent fields, including encrypted fields
-      # @note Only fields with non-nil values are included for storage efficiency
-      #
-      def to_h_for_storage
-        self.class.persistent_fields.each_with_object({}) do |field, hsh|
-          field_type = self.class.field_types[field]
-          method_name = field_type.method_name
-          val = send(method_name)
-          prepared = serialize_value(val)
-          Familia.ld " [to_h_for_storage] field: #{field} val: #{val.class} prepared: #{prepared&.class || '[nil]'}"
-
-          # Only include non-nil values in the hash for Valkey
-          # Use string key for database compatibility
-          hsh[field.to_s] = prepared unless prepared.nil?
-        end
-      end
-
-      # Converts the object's persistent fields to an array.
-      #
-      # Serializes all persistent field values in field definition order,
-      # preparing them for Valkey storage. Each value is processed through
-      # the serialization pipeline to ensure Valkey compatibility.
-      #
-      # @return [Array] Array of serialized field values in field order
-      #
-      # @example Converting an object to array format
-      #   user = User.new(name: "John", email: "john@example.com", age: 30)
-      #   user.to_a
-      #   # => ["John", "john@example.com", "30"]
-      #
-      # @note Values are serialized using the same process as other persistence
-      #   methods to maintain data consistency across operations.
-      #
-      def to_a
-        self.class.persistent_fields.filter_map do |field|
-          field_type = self.class.field_types[field]
-
-          # Security: Skip non-loggable fields (e.g., encrypted fields)
-          next unless field_type.loggable
-
-          method_name = field_type.method_name
-          val = send(method_name)
-          prepared = serialize_value(val)
-          Familia.ld " [to_a] field: #{field} method: #{method_name} val: #{val.class} prepared: #{prepared.class}"
-          prepared
-        end
-      end
-
-      # Serializes a Ruby object for Valkey storage.
-      #
-      # Converts Ruby objects into the DB-compatible string representations using
-      # the Familia distinguisher for type coercion. Falls back to JSON serialization
-      # for complex types (Hash, Array) when the primary distinguisher returns nil.
-      #
-      # The serialization process:
-      # 1. Attempts conversion using Familia.distinguisher with relaxed type checking
-      # 2. For Hash/Array types that return nil, tries custom dump_method or Familia::JsonSerializer.dump
-      # 3. Logs warnings when serialization fails completely
-      #
-      # @param val [Object] The Ruby object to serialize for Valkey storage
-      #
-      # @return [String, nil] The serialized value ready for Valkey storage, or nil
-      #   if serialization failed
-      #
-      # @example Serializing different data types
-      #   serialize_value("hello")        # => "hello"
-      #   serialize_value(42)             # => "42"
-      #   serialize_value({name: "John"}) # => '{"name":"John"}'
-      #   serialize_value([1, 2, 3])      # => "[1,2,3]"
-      #
-      # @note This method integrates with Familia's type system and supports
-      #   custom serialization methods when available on the object
-      #
-      # @see Familia.distinguisher The primary serialization mechanism
-      #
-      def serialize_value(val)
-        # Security: Handle ConcealedString safely - extract encrypted data for storage
-        return val.encrypted_value if val.respond_to?(:encrypted_value)
-
-        prepared = Familia.distinguisher(val, strict_values: false)
-
-        # If the distinguisher returns nil, try using the dump_method but only
-        # use JSON serialization for complex types that need it.
-        if prepared.nil? && (val.is_a?(Hash) || val.is_a?(Array))
-          prepared = val.respond_to?(dump_method) ? val.send(dump_method) : Familia::JsonSerializer.dump(val)
-        end
-
-        # If both the distinguisher and dump_method return nil, log an error
-        Familia.ld "[#{self.class}#serialize_value] nil returned for #{self.class}" if prepared.nil?
-
-        prepared
-      end
-
-      # Converts a Database string value back to its original Ruby type
-      #
-      # This method attempts to deserialize JSON strings back to their original
-      # Hash or Array types. Simple string values are returned as-is.
-      #
-      # @param val [String] The string value from Database to deserialize
-      # @param symbolize [Boolean] Whether to symbolize hash keys (default: true for compatibility)
-      # @return [Object] The deserialized value (Hash, Array, or original string)
-      #
-      def deserialize_value(val, symbolize: true)
-        return val if val.nil? || val == ''
-
-        # Try to parse as JSON first for complex types
-        begin
-          parsed = Familia::JsonSerializer.parse(val, symbolize_names: symbolize)
-          # Only return parsed value if it's a complex type (Hash/Array)
-          # Simple values should remain as strings
-          return parsed if parsed.is_a?(Hash) || parsed.is_a?(Array)
-        rescue Familia::SerializerError
-          # Not valid JSON, return as-is
-        end
-
-        val
-      end
-
       private
 
       # Reset all transient fields to nil
@@ -547,6 +401,55 @@ module Familia
           Familia.ld "[reset_transient_fields!] Reset #{field_name} to nil"
         end
       end
+
+      # Automatically update class-level indexes after save
+      #
+      # Iterates through class-level indexing relationships and calls their
+      # corresponding add_to_class_* methods to populate indexes. Only processes
+      # class-level indexes (where target_class == self.class), skipping
+      # instance-scoped indexes which require parent context.
+      #
+      # Uses idempotent Redis commands (HSET for unique_index) so repeated calls
+      # are safe and have negligible performance overhead. Note that multi_index
+      # always requires within: parameter, so only unique_index benefits from this.
+      #
+      # @return [void]
+      #
+      # @example Automatic indexing on save
+      #   class Customer < Familia::Horreum
+      #     feature :relationships
+      #     unique_index :email, :email_lookup
+      #   end
+      #
+      #   customer = Customer.new(email: 'test@example.com')
+      #   customer.save  # Automatically calls add_to_class_email_lookup
+      #
+      # @note Only class-level unique_index declarations auto-populate.
+      #   Instance-scoped indexes (with within:) require manual population:
+      #   employee.add_to_company_badge_index(company)
+      #
+      # @see Familia::Features::Relationships::Indexing For index declaration details
+      #
+      def auto_update_class_indexes
+        return unless self.class.respond_to?(:indexing_relationships)
+
+        self.class.indexing_relationships.each do |rel|
+          # Skip instance-scoped indexes (require parent context)
+          # Instance-scoped indexes must be manually populated because they need
+          # the parent object reference (e.g., employee.add_to_company_badge_index(company))
+          unless rel.target_class == self.class
+            Familia.ld <<~LOG_MESSAGE
+              [auto_update_class_indexes] Skipping #{rel.index_name} (requires parent context)
+            LOG_MESSAGE
+            next
+          end
+
+          # Call the existing add_to_class_* methods
+          add_method = :"add_to_class_#{rel.index_name}"
+          send(add_method) if respond_to?(add_method)
+        end
+      end
+
     end
   end
 end

data/lib/familia/horreum/{subclass/related_fields_management.rb → related_fields.rb}

@@ -1,4 +1,4 @@
-# lib/familia/horreum/related_fields_management.rb
+# lib/familia/horreum/related_fields.rb
 
 module Familia
 
@@ -170,7 +170,27 @@ module Familia
 
         related_fields[name] = RelatedFieldDefinition.new(name, klass, opts)
 
-        attr_reader name
+        # Create lazy-initializing accessor that calls initialize_relatives if needed
+        define_method name do
+          ivar = :"@#{name}"
+          value = instance_variable_get(ivar)
+
+          # If nil and we haven't initialized relatives, do it now
+          # Check singleton class to avoid polluting instance variables
+          if value.nil? && !singleton_class.instance_variable_defined?(:"@relatives_initialized")
+            initialize_relatives
+            value = instance_variable_get(ivar)
+          end
+
+          # If still nil after lazy initialization attempt, raise helpful error
+          # Only raise if we tried to initialize but it's still nil
+          if value.nil? && singleton_class.instance_variable_defined?(:"@relatives_initialized")
+            raise "#{self.class}##{name} is nil. Did you override initialize without calling super? " \
+                  "(Field is nil after initialization attempt)"
+          end
+
+          value
+        end
 
         define_method :"#{name}=" do |val|
           send(name).replace val

data/lib/familia/horreum/serialization.rb

@@ -0,0 +1,190 @@
+# lib/familia/horreum/serialization.rb
+
+module Familia
+  class Horreum
+    # Serialization - Instance-level methods for object serialization
+    # Handles conversion between Ruby objects and Valkey hash storage
+    module Serialization
+      # Converts the object's persistent fields to a hash for external use.
+      #
+      # Returns actual Ruby values (String, Integer, Hash, etc.) for API consumption,
+      # NOT JSON-encoded strings. Excludes non-loggable fields like encrypted fields
+      # for security.
+      #
+      # @return [Hash] Hash with field names as string keys and Ruby values
+      #
+      # @example Converting an object to hash format for API response
+      #   user = User.new(name: "John", email: "john@example.com", age: 30)
+      #   user.to_h
+      #   # => {"name"=>"John", "email"=>"john@example.com", "age"=>30}
+      #   # Note: Returns actual Ruby types, not JSON strings
+      #
+      # @note Only loggable fields are included. Encrypted fields are excluded.
+      # @note Nil values are excluded from the returned hash (storage optimization)
+      #
+      def to_h
+        self.class.persistent_fields.each_with_object({}) do |field, hsh|
+          field_type = self.class.field_types[field]
+
+          # Security: Skip non-loggable fields (e.g., encrypted fields)
+          next unless field_type.loggable
+
+          val = send(field_type.method_name)
+          Familia.ld " [to_h] field: #{field} val: #{val.class}"
+
+          # Use string key for external API compatibility
+          # Return Ruby values, not JSON-encoded strings
+          hsh[field.to_s] = val
+        end
+      end
+
+      # Converts the object's persistent fields to a hash for database storage.
+      #
+      # Returns JSON-encoded strings for ALL persistent field values, ready for
+      # Redis storage. Unlike to_h, this includes encrypted fields and serializes
+      # values using serialize_value (JSON encoding).
+      #
+      # @return [Hash] Hash with field names as string keys and JSON-encoded values
+      #
+      # @example Internal storage preparation
+      #   user = User.new(name: "John", age: 30)
+      #   user.to_h_for_storage
+      #   # => {"name"=>"\"John\"", "age"=>"30"}
+      #   # Note: Strings are JSON-encoded with quotes
+      #
+      # @note This is an internal method used by commit_fields and hmset
+      # @note Nil values are excluded to optimize Redis storage
+      #
+      def to_h_for_storage
+        self.class.persistent_fields.each_with_object({}) do |field, hsh|
+          field_type = self.class.field_types[field]
+
+          val = send(field_type.method_name)
+          prepared = serialize_value(val)
+
+          if Familia.debug?
+            Familia.ld " [to_h_for_storage] field: #{field} val: #{val.class} prepared: #{prepared&.class || '[nil]'}"
+          end
+
+          # Use string key for database compatibility
+          hsh[field.to_s] = prepared
+        end
+      end
+
+      # Converts the object's persistent fields to an array.
+      #
+      # Serializes all persistent field values in field definition order,
+      # preparing them for Valkey storage. Each value is processed through
+      # the serialization pipeline to ensure Valkey compatibility.
+      #
+      # @return [Array] Array of serialized field values in field order
+      #
+      # @example Converting an object to array format
+      #   user = User.new(name: "John", email: "john@example.com", age: 30)
+      #   user.to_a
+      #   # => ["John", "john@example.com", "30"]
+      #
+      # @note Values are serialized using the same process as other persistence
+      #   methods to maintain data consistency across operations.
+      #
+      def to_a
+        self.class.persistent_fields.map do |field|
+          field_type = self.class.field_types[field]
+
+          # Security: Skip non-loggable fields (e.g., encrypted fields)
+          next unless field_type.loggable
+
+          method_name = field_type.method_name
+          val = send(method_name)
+          Familia.ld " [to_a] field: #{field} method: #{method_name} val: #{val.class}"
+
+          # Return actual Ruby values, including nil to maintain array positions
+          val
+        end
+      end
+
+      # Serializes a Ruby object for Valkey storage.
+      #
+      # Converts ALL Ruby values (including strings) to JSON-encoded strings for
+      # type-safe storage. This ensures round-trip type preservation: the type you
+      # save is the type you get back.
+      #
+      # The serialization process:
+      # 1. ConcealedStrings (encrypted values) → extract encrypted_value
+      # 2. ALL other types → JSON serialization (String, Integer, Boolean, Float, nil, Hash, Array)
+      #
+      # @param val [Object] The Ruby object to serialize for Valkey storage
+      #
+      # @return [String] JSON-encoded string representation
+      #
+      # @example Type preservation through JSON encoding
+      #   serialize_value("007")           # => "\"007\"" (JSON string)
+      #   serialize_value(123)             # => "123" (JSON number)
+      #   serialize_value(true)            # => "true" (JSON boolean)
+      #   serialize_value({a: 1})          # => "{\"a\":1}" (JSON object)
+      #
+      # @note Strings are JSON-encoded to prevent type coercion bugs where
+      #   string "123" would be indistinguishable from integer 123 in storage
+      #
+      # @note This method integrates with Familia's type system and supports
+      #   custom serialization methods when available on the object
+      #
+      # @see Familia.identifier_extractor For extracting identifiers from Familia objects
+      #
+      def serialize_value(val)
+        # Security: Handle ConcealedString safely - extract encrypted data for storage
+        return val.encrypted_value if val.respond_to?(:encrypted_value)
+
+        # ALWAYS write valid JSON for type preservation
+        # This includes strings, which get JSON-encoded with wrapping quotes
+        Familia::JsonSerializer.dump(val)
+      end
+
+      # Converts a Redis string value back to its original Ruby type
+      #
+      # This method deserializes JSON strings back to their original Ruby types
+      # (Integer, Boolean, Float, nil, Hash, Array). Plain strings that cannot
+      # be parsed as JSON are returned as-is.
+      #
+      # This pairs with serialize_value which JSON-encodes all non-string values.
+      # The contract ensures type preservation across Redis storage:
+      # - Strings stored as-is → returned as-is
+      # - All other types JSON-encoded → JSON-decoded back to original type
+      #
+      # @param val [String] The string value from Redis to deserialize
+      # @param symbolize [Boolean] Whether to symbolize hash keys (default: false)
+      # @param field_name [Symbol, nil] Optional field name for better error context
+      # @return [Object] The deserialized value with original Ruby type, or the original string if not JSON
+      #
+      def deserialize_value(val, symbolize: false, field_name: nil)
+        return nil if val.nil? || val == ''
+
+        begin
+          Familia::JsonSerializer.parse(val, symbolize_names: symbolize)
+        rescue Familia::SerializerError
+          log_deserialization_issue(val, field_name)
+          val
+        end
+      end
+
+      private
+
+      def log_deserialization_issue(val, field_name)
+        context = field_name ? "#{self.class}##{field_name}" : self.class.to_s
+        dbkey_info = respond_to?(:dbkey) ? dbkey : 'no dbkey'
+
+        msg = if looks_like_json?(val)
+          "Corrupted JSON in #{context}: #{val.inspect} (#{dbkey_info})"
+        else
+          "Legacy plain string in #{context}: #{val.inspect} (#{dbkey_info})"
+        end
+
+        Familia.le(msg)
+      end
+
+      def looks_like_json?(val)
+        val.start_with?('{', '[', '"') || %w[true false null].include?(val)
+      end
+    end
+  end
+end

data/lib/familia/horreum.rb

@@ -1,9 +1,14 @@
 # lib/familia/horreum.rb
 
-require_relative 'horreum/subclass/definition'
-require_relative 'horreum/subclass/management'
-require_relative 'horreum/shared/settings'
-require_relative 'horreum/core'
+require_relative 'horreum/settings'
+require_relative 'horreum/connection'
+require_relative 'horreum/database_commands'
+require_relative 'horreum/related_fields'
+require_relative 'horreum/definition'
+require_relative 'horreum/management'
+require_relative 'horreum/persistence'
+require_relative 'horreum/serialization'
+require_relative 'horreum/utils'
 
 module Familia
   #
@@ -44,8 +49,12 @@ module Familia
   #
   class Horreum
     include Familia::Base
-    include Familia::Horreum::Core
+    include Familia::Horreum::Persistence
+    include Familia::Horreum::Serialization
+    include Familia::Horreum::Connection
+    include Familia::Horreum::DatabaseCommands
     include Familia::Horreum::Settings
+    include Familia::Horreum::Utils
 
     using Familia::Refinements::TimeLiterals
 
@@ -223,6 +232,15 @@ module Familia
       init
     end
 
+    # Override this method in subclasses for custom initialization logic.
+    # This is called AFTER fields are set and relatives are initialized.
+    #
+    # DO NOT override initialize() - use this init() hook instead.
+    #
+    # Example:
+    #   def init(name = nil)
+    #     @name = name || SecureRandom.hex(4)
+    #   end
     def init(*args, **kwargs)
       # Default no-op
     end
@@ -233,6 +251,8 @@ module Familia
     # This needs to be called in the initialize method.
     #
     def initialize_relatives
+      # Store initialization flag on singleton class to avoid polluting instance variables
+      return if singleton_class.instance_variable_defined?(:"@relatives_initialized")
       # Generate instances of each DataType. These need to be
       # unique for each instance of this class so they can piggyback
       # on the specifc index of this instance.
@@ -272,11 +292,16 @@ module Familia
         # e.g. customer.name  #=> `#<Familia::HashKey:0x0000...>`
         instance_variable_set :"@#{name}", related_object
       end
+
+      # Mark relatives as initialized on singleton class to avoid polluting instance variables
+      singleton_class.instance_variable_set(:"@relatives_initialized", true)
     end
 
     def initialize_with_keyword_args_deserialize_value(**fields)
       # Deserialize Database string values back to their original types
-      deserialized_fields = fields.transform_values { |value| deserialize_value(value) }
+      deserialized_fields = fields.each_with_object({}) do |(field_name, value), hsh|
+        hsh[field_name] = deserialize_value(value, field_name: field_name)
+      end
       initialize_with_keyword_args(**deserialized_fields)
     end
 
@@ -285,7 +310,7 @@ module Familia
     #
     # This method is part of horreum.rb rather than serialization.rb because it
     # operates solely on the provided values and doesn't query Database or other
-    # external sources. That's why it's called "optimistic" refresh: it assumes
+    # external sources. That's why it's called "naive" refresh: it assumes
     # the provided values are correct and updates the object accordingly.
     #
     # @see #refresh!
@@ -293,8 +318,8 @@ module Familia
     # @param fields [Hash] A hash of field names and their new values to update
     #   the object with.
     # @return [Array] The list of field names that were updated.
-    def optimistic_refresh(**fields)
-      Familia.ld "[optimistic_refresh] #{self.class} #{dbkey} #{fields.keys}"
+    def naive_refresh(**fields)
+      Familia.ld "[naive_refresh] #{self.class} #{dbkey} #{fields.keys}"
       initialize_with_keyword_args_deserialize_value(**fields)
     end
 
@@ -311,7 +336,7 @@ module Familia
                     send(definition)
                   when Proc
                     definition.call(self)
-                  end
+      end
 
       # Return nil for unpopulated identifiers (like unsaved ActiveRecord objects)
       # Only raise errors when the identifier is actually needed for db operations
@@ -331,7 +356,6 @@ module Familia
     # @return [Redis] the Database connection instance.
     #
 
-
     def generate_id
       @objid ||= Familia.generate_id
     end
@@ -379,8 +403,10 @@ module Familia
       self.class.fields.filter_map do |field|
         # Database will give us field names as strings back, but internally
         # we use symbols. So we check for both.
-        value = fields[field.to_sym] || fields[field.to_s]
-        if value
+        # Use fetch with default to avoid || operator which skips false values
+        value = fields.fetch(field.to_sym) { fields[field.to_s] }
+        # Check for nil explicitly to allow false and 0 values
+        unless value.nil?
           # Use the mapped method name, not the field name
           method_name = self.class.field_method_map[field] || field
           send(:"#{method_name}=", value)
@@ -390,6 +416,5 @@ module Familia
     end
 
     # Builds the instance-level connection chain with handlers in priority order
-
   end
 end