familia 2.0.0.pre16 → 2.0.0.pre18

data/lib/familia/data_type.rb

@@ -1,6 +1,9 @@
 # lib/familia/data_type.rb
 
-require_relative 'data_type/commands'
+require_relative 'data_type/class_methods'
+require_relative 'data_type/settings'
+require_relative 'data_type/connection'
+require_relative 'data_type/database_commands'
 require_relative 'data_type/serialization'
 
 # Familia
@@ -14,6 +17,7 @@ module Familia
   # @abstract Subclass and implement Database data type specific methods
   class DataType
     include Familia::Base
+    extend ClassMethods
     extend Familia::Features
 
     using Familia::Refinements::TimeLiterals
@@ -29,60 +33,6 @@ module Familia
       attr_reader :registered_types, :valid_options, :has_related_fields
     end
 
-    # DataType::ClassMethods
-    #
-    module ClassMethods
-      attr_accessor :parent, :suffix, :prefix, :uri
-      attr_writer :logical_database
-
-      # To be called inside every class that inherits DataType
-      # +methname+ is the term used for the class and instance methods
-      # that are created for the given +klass+ (e.g. set, list, etc)
-      def register(klass, methname)
-        Familia.trace :REGISTER, nil, "[#{self}] Registering #{klass} as #{methname.inspect}" if Familia.debug?
-
-        @registered_types[methname] = klass
-      end
-
-      # Get the registered type class from a given method name
-      # +methname+ is the method name used to register the class (e.g. :set, :list, etc)
-      # Returns the registered class or nil if not found
-      def registered_type(methname)
-        @registered_types[methname]
-      end
-
-      def logical_database(val = nil)
-        @logical_database = val unless val.nil?
-        @logical_database || parent&.logical_database
-      end
-
-      def uri(val = nil)
-        @uri = val unless val.nil?
-        @uri || (parent ? parent.uri : Familia.uri)
-      end
-
-      def inherited(obj)
-        Familia.trace :DATATYPE, nil, "#{obj} is my kinda type" if Familia.debug?
-        obj.logical_database = logical_database
-        obj.default_expiration = default_expiration # method added via Features::Expiration
-        obj.uri = uri
-        super
-      end
-
-      def valid_keys_only(opts)
-        opts.slice(*DataType.valid_options)
-      end
-
-      def relations?
-        @has_related_fields ||= false
-      end
-    end
-    extend ClassMethods
-
-    attr_reader :keystring, :opts, :uri, :logical_database
-
-    alias url uri
-
     # +keystring+: If parent is set, this will be used as the suffix
     # for dbkey. Otherwise this becomes the value of the key.
     # If this is an Array, the elements will be joined.
@@ -125,146 +75,9 @@ module Familia
       init if respond_to? :init
     end
 
-    # TODO: Replace with Chain of Responsibility pattern
-    def dbclient
-      return Fiber[:familia_transaction] if Fiber[:familia_transaction]
-      return @dbclient if @dbclient
-
-      # Delegate to parent if present, otherwise fall back to Familia
-      parent ? parent.dbclient : Familia.dbclient(opts[:logical_database])
-    end
-
-    # Produces the full dbkey for this object.
-    #
-    # @return [String] The full dbkey.
-    #
-    # This method determines the appropriate dbkey based on the context of the DataType object:
-    #
-    # 1. If a hardcoded key is set in the options, it returns that key.
-    # 2. For instance-level DataType objects, it uses the parent instance's dbkey method.
-    # 3. For class-level DataType objects, it uses the parent class's dbkey method.
-    # 4. For standalone DataType objects, it uses the keystring as the full dbkey.
-    #
-    # For class-level DataType objects (parent_class? == true):
-    # - The suffix is optional and used to differentiate between different types of objects.
-    # - If no suffix is provided, the class's default suffix is used (via the self.suffix method).
-    # - If a nil suffix is explicitly passed, it won't appear in the resulting dbkey.
-    # - Passing nil as the suffix is how class-level DataType objects are created without
-    #   the global default 'object' suffix.
-    #
-    # @example Instance-level DataType
-    #   user_instance.some_datatype.dbkey  # => "user:123:some_datatype"
-    #
-    # @example Class-level DataType
-    #   User.some_datatype.dbkey  # => "user:some_datatype"
-    #
-    # @example Standalone DataType
-    #   DataType.new("mykey").dbkey  # => "mykey"
-    #
-    # @example Class-level DataType with explicit nil suffix
-    #   User.dbkey("123", nil)  # => "user:123"
-    #
-    def dbkey
-      # Return the hardcoded key if it's set. This is useful for
-      # support legacy keys that aren't derived in the same way.
-      return opts[:dbkey] if opts[:dbkey]
-
-      if parent_instance?
-        # This is an instance-level datatype object so the parent instance's
-        # dbkey method is defined in Familia::Horreum::InstanceMethods.
-        parent.dbkey(keystring)
-      elsif parent_class?
-        # This is a class-level datatype object so the parent class' dbkey
-        # method is defined in Familia::Horreum::DefinitionMethods.
-        parent.dbkey(keystring, nil)
-      else
-        # This is a standalone DataType object where it's keystring
-        # is the full database key (dbkey).
-        keystring
-      end
-    end
-
-    def class?
-      !@opts[:class].to_s.empty? && @opts[:class].is_a?(Familia)
-    end
-
-    # Provides a structured way to "gear down" to run db commands that are
-    # not implemented in our DataType classes since we intentionally don't
-    # have a method_missing method.
-    def direct_access
-      yield(dbclient, dbkey)
-    end
-
-    def parent_instance?
-      parent&.is_a?(Horreum::ParentDefinition)
-    end
-
-    def parent_class?
-      parent.is_a?(Class) && parent.ancestors.include?(Familia::Horreum)
-    end
-
-    def parent?
-      parent_class? || parent_instance?
-    end
-
-    def parent
-      # Return cached ParentDefinition if available
-      return @parent if @parent
-
-      # Return class-level parent if no instance parent
-      return self.class.parent unless @parent_ref
-
-      # Create ParentDefinition dynamically from stored reference.
-      # This ensures we get the current identifier value (available after initialization)
-      # rather than a stale nil value from initialization time. Cannot cache due to frozen object.
-      Horreum::ParentDefinition.from_parent(@parent_ref)
-    end
-
-    def parent=(value)
-      case value
-      when Horreum::ParentDefinition
-        @parent = value
-      when nil
-        @parent = nil
-        @parent_ref = nil
-      else
-        # Store parent instance reference for lazy ParentDefinition creation.
-        # During initialization, the parent's identifier may not be available yet,
-        # so we defer ParentDefinition creation until first access for memory efficiency.
-        # Note: @parent_ref is not cleared after use because DataType objects are frozen.
-        @parent_ref = value
-        @parent = nil  # Will be created dynamically in parent method
-      end
-    end
-
-    def uri
-      # Return explicit instance URI if set
-      return @uri if @uri
-
-      # If we have a parent with logical_database, build URI with that database
-      if parent && parent.respond_to?(:logical_database) && parent.logical_database
-        new_uri = (self.class.uri || Familia.uri).dup
-        new_uri.db = parent.logical_database
-        new_uri
-      else
-        # Fall back to class-level URI or global Familia.uri
-        self.class.uri || Familia.uri
-      end
-    end
-
-    def uri=(value)
-      @uri = value
-    end
-
-    def dump_method
-      self.class.dump_method
-    end
-
-    def load_method
-      self.class.load_method
-    end
-
-    include Commands
+    include Settings
+    include Connection
+    include DatabaseCommands
     include Serialization
   end
 

data/lib/familia/encryption/encrypted_data.rb

@@ -44,8 +44,18 @@ module Familia
         new(**parsed)
       end
 
-      def self.from_json(json_string)
-        validate!(json_string)
+      def self.from_json(json_string_or_hash)
+        # Support both JSON strings (legacy) and already-parsed Hashes (v2.0 deserialization)
+        if json_string_or_hash.is_a?(Hash)
+          # Already parsed - use directly
+          parsed = json_string_or_hash
+          # Symbolize keys if they're strings
+          parsed = parsed.transform_keys(&:to_sym) if parsed.keys.first.is_a?(String)
+          new(**parsed)
+        else
+          # JSON string - validate and parse
+          validate!(json_string_or_hash)
+        end
       end
 
       # Instance methods for decryptability validation

data/lib/familia/encryption/manager.rb

@@ -32,15 +32,22 @@ module Familia
         Familia::Encryption.secure_wipe(key) if key
       end
 
-      def decrypt(encrypted_json, context:, additional_data: nil)
-        return nil if encrypted_json.nil? || encrypted_json.empty?
+      def decrypt(encrypted_json_or_hash, context:, additional_data: nil)
+        return nil if encrypted_json_or_hash.nil? || (encrypted_json_or_hash.respond_to?(:empty?) && encrypted_json_or_hash.empty?)
 
         # Increment counter immediately to track all decryption attempts, even failed ones
         Familia::Encryption.derivation_count.increment
 
         begin
-          data = Familia::Encryption::EncryptedData.new(**Familia::JsonSerializer.parse(encrypted_json,
-                                                                                        symbolize_names: true))
+          # Delegate parsing and instantiation to EncryptedData.from_json
+          # Wrap validation errors for security (don't expose internal structure details)
+          begin
+            data = Familia::Encryption::EncryptedData.from_json(encrypted_json_or_hash)
+            raise EncryptionError, 'Failed to parse encrypted data' unless data
+          rescue EncryptionError => e
+            # Re-wrap validation errors with generic message for security
+            raise EncryptionError, "Decryption failed: #{e.message}"
+          end
 
           # Validate algorithm support
           provider = Registry.get(data.algorithm)

data/lib/familia/features/autoloader.rb

@@ -28,7 +28,9 @@ module Familia::Features
       ]
 
       # Ensure the Features module exists within the base module
-      base.const_set(:Features, Module.new) unless base.const_defined?(:Features) || config_name.eql?('features')
+      unless base.const_defined?(:Features) || config_name.eql?('features')
+        base.const_set(:Features, Module.new)
+      end
 
       # Use the shared autoload_files method
       autoload_files(dir_patterns, log_prefix: "Autoloader[#{config_name}]")

data/lib/familia/features/encrypted_fields/encrypted_field_type.rb

@@ -29,8 +29,10 @@ module Familia
             # Already concealed, store as-is
             instance_variable_set(:"@#{field_name}", value)
           elsif field_type.encrypted_json?(value)
-            # Already encrypted JSON from database - wrap in ConcealedString without re-encrypting
-            concealed = ConcealedString.new(value, self, field_type)
+            # Already encrypted (JSON string or Hash from database) - wrap in ConcealedString without re-encrypting
+            # Convert Hash back to JSON string if needed (v2.0 deserialization returns Hash)
+            encrypted_string = value.is_a?(Hash) ? Familia::JsonSerializer.dump(value) : value
+            concealed = ConcealedString.new(encrypted_string, self, field_type)
             instance_variable_set(:"@#{field_name}", concealed)
           else
             # Encrypt plaintext and wrap in ConcealedString
@@ -138,7 +140,13 @@ module Familia
 
     # Check if a string looks like encrypted JSON data
     def encrypted_json?(data)
-      Familia::Encryption::EncryptedData.valid?(data)
+      # Support both JSON strings (legacy) and Hashes (v2.0 deserialization)
+      if data.is_a?(Hash)
+        required_keys = %w[algorithm nonce ciphertext auth_tag key_version]
+        required_keys.all? { |key| data.key?(key) || data.key?(key.to_sym) }
+      else
+        Familia::Encryption::EncryptedData.valid?(data)
+      end
     end
 
     private

data/lib/familia/features/encrypted_fields.rb

@@ -259,7 +259,7 @@ module Familia
     # - Insider threats with application access
     #
     module EncryptedFields
-      Familia::Base.add_feature self, :encrypted_fields
+      Familia::Base.add_feature self, :encrypted_fields, depends_on: nil, field_group: :encrypted_fields
 
       def self.included(base)
         Familia.trace :LOADED, self, base if Familia.debug?
@@ -297,7 +297,10 @@ module Familia
           @encrypted_fields ||= []
           @encrypted_fields << name unless @encrypted_fields.include?(name)
 
-          require_relative 'encrypted_fields/encrypted_field_type'
+          # Add to field_groups if the group exists
+          if field_groups&.key?(:encrypted_fields)
+            field_groups[:encrypted_fields] << name
+          end
 
           field_type = EncryptedFieldType.new(name, aad_fields: aad_fields, **)
           register_field_type(field_type)

data/lib/familia/features/external_identifier.rb

@@ -10,6 +10,7 @@ module Familia
       def self.included(base)
         Familia.trace :LOADED, self, base if Familia.debug?
         base.extend ModelClassMethods
+        base.include ModelInstanceMethods
 
         # Ensure default prefix is set in feature options
         base.add_feature_options(:external_identifier, prefix: 'ext')
@@ -75,9 +76,6 @@ module Familia
 
               instance_variable_set(:"@#{field_name}", derived_extid)
 
-              # Update mapping if we have an identifier (objid)
-              self.class.extid_lookup[derived_extid] = identifier if respond_to?(:identifier) && identifier
-
               derived_extid
             end
           end
@@ -103,11 +101,6 @@ module Familia
 
               # Set the new value
               instance_variable_set(:"@#{field_name}", value)
-
-              # Update mapping if we have both extid and identifier
-              return unless value && respond_to?(:identifier) && identifier
-
-              self.class.extid_lookup[value] = identifier
             end
           end
         end
@@ -159,6 +152,54 @@ module Familia
         end
       end
 
+      # Instance methods for external identifier management
+      module ModelInstanceMethods
+        # Override save to update extid_lookup mapping
+        #
+        # This ensures the extid_lookup index is populated during save operations
+        # rather than during object initialization, preventing unwanted database
+        # writes when calling .new()
+        #
+        # @param update_expiration [Boolean] Whether to update key expiration
+        # @return [Boolean] True if save was successful
+        #
+        def save(update_expiration: true)
+          result = super
+
+          # Update extid_lookup mapping after successful save
+          if result && respond_to?(:extid) && respond_to?(:identifier)
+            current_extid = extid  # Triggers lazy generation if needed
+            if current_extid && identifier
+              self.class.extid_lookup[current_extid] = identifier
+            end
+          end
+
+          result
+        end
+
+        # Override save_if_not_exists to update extid_lookup mapping
+        #
+        # This ensures the extid_lookup index is populated during create operations
+        # which use save_if_not_exists instead of save.
+        #
+        # @param update_expiration [Boolean] Whether to update key expiration
+        # @return [Boolean] True if save was successful
+        #
+        def save_if_not_exists(update_expiration: true)
+          result = super
+
+          # Update extid_lookup mapping after successful save
+          if result && respond_to?(:extid) && respond_to?(:identifier)
+            current_extid = extid  # Triggers lazy generation if needed
+            if current_extid && identifier
+              self.class.extid_lookup[current_extid] = identifier
+            end
+          end
+
+          result
+        end
+      end
+
       # Derives a deterministic, public-facing external identifier from the object's
       # internal `objid`.
       #

data/lib/familia/features/object_identifier.rb

@@ -88,6 +88,7 @@ module Familia
       def self.included(base)
         Familia.trace :LOADED, self, base if Familia.debug?
         base.extend ModelClassMethods
+        base.include ModelInstanceMethods
 
         # Ensure default generator is set in feature options
         base.add_feature_options(:object_identifier, generator: DEFAULT_GENERATOR)
@@ -160,9 +161,6 @@ module Familia
               generator = options[:generator] || DEFAULT_GENERATOR
               instance_variable_set(:"@#{field_name}_generator_used", generator)
 
-              # Update mapping from objid to model primary key
-              self.class.objid_lookup[generated_id] = identifier if respond_to?(:identifier) && identifier
-
               generated_id
             end
           end
@@ -198,14 +196,11 @@ module Familia
 
               instance_variable_set(:"@#{field_name}", value)
 
-              # Update mapping from objid to this new identifier
-              self.class.objid_lookup[value] = identifier unless value.nil? || identifier.nil?
-
               # When setting objid from external source (e.g., loading from Valkey/Redis),
-              # we cannot determine the original generator, so we clear the provenance
-              # tracking to indicate unknown origin. This prevents false assumptions
-              # about the security properties of externally-provided identifiers.
-              instance_variable_set(:"@#{field_name}_generator_used", nil)
+              # infer the generator type from the format to restore provenance tracking.
+              # This allows features like ExternalIdentifier to work correctly on loaded objects.
+              inferred_generator = infer_objid_generator(value)
+              instance_variable_set(:"@#{field_name}_generator_used", inferred_generator)
             end
           end
         end
@@ -284,6 +279,54 @@ module Familia
         end
       end
 
+      # Instance methods for object identifier management
+      module ModelInstanceMethods
+        # Override save to update objid_lookup mapping
+        #
+        # This ensures the objid_lookup index is populated during save operations
+        # rather than during object initialization, preventing unwanted database
+        # writes when calling .new()
+        #
+        # @param update_expiration [Boolean] Whether to update key expiration
+        # @return [Boolean] True if save was successful
+        #
+        def save(update_expiration: true)
+          result = super
+
+          # Update objid_lookup mapping after successful save
+          if result && respond_to?(:objid) && respond_to?(:identifier)
+            current_objid = objid  # Triggers lazy generation if needed
+            if current_objid && identifier
+              self.class.objid_lookup[current_objid] = identifier
+            end
+          end
+
+          result
+        end
+
+        # Override save_if_not_exists to update objid_lookup mapping
+        #
+        # This ensures the objid_lookup index is populated during create operations
+        # which use save_if_not_exists instead of save.
+        #
+        # @param update_expiration [Boolean] Whether to update key expiration
+        # @return [Boolean] True if save was successful
+        #
+        def save_if_not_exists(update_expiration: true)
+          result = super
+
+          # Update objid_lookup mapping after successful save
+          if result && respond_to?(:objid) && respond_to?(:identifier)
+            current_objid = objid  # Triggers lazy generation if needed
+            if current_objid && identifier
+              self.class.objid_lookup[current_objid] = identifier
+            end
+          end
+
+          result
+        end
+      end
+
       # Instance method for generating object identifier using configured strategy
       #
       # This method is called by the ObjectIdentifierFieldType when lazy generation
@@ -304,10 +347,39 @@ module Familia
         objid
       end
 
-      # Full-length alias setter for objid
+      # Infers the generator type (:uuid_v7, :uuid_v4, :hex) from the format of an objid string.
       #
-      # @param value [String] The object identifier to set
+      # This method analyzes the objid format to restore provenance tracking when loading
+      # objects from Redis, allowing dependent features like ExternalIdentifier to work correctly.
       #
+      # @param objid_value [String] The objid string to analyze
+      # @return [Symbol, nil] The inferred generator type or nil if unknown
+      def infer_objid_generator(objid_value)
+        return nil if objid_value.nil? || objid_value.to_s.empty?
+
+        objid_str = objid_value.to_s
+
+        # UUID format: xxxxxxxx-xxxx-Vxxx-xxxx-xxxxxxxxxxxx (36 chars with hyphens)
+        # where V is the version nibble at position 14
+        if objid_str.length == 36 && objid_str[8] == '-' && objid_str[13] == '-' && objid_str[18] == '-' && objid_str[23] == '-'
+          version_char = objid_str[14]
+          case version_char
+          when '7'
+            :uuid_v7
+          when '4'
+            :uuid_v4
+          else
+            nil # Unknown UUID version
+          end
+        # Hex format: pure hexadecimal without hyphens (32 or 64 chars typically)
+        elsif objid_str.match?(/\A[0-9a-fA-F]+\z/)
+          :hex
+        else
+          nil # Unknown format
+        end
+      end
+      private :infer_objid_generator
+
       def object_identifier=(value)
         self.objid = value
       end

data/lib/familia/features/relationships/indexing/multi_index_generators.rb

@@ -75,7 +75,7 @@ module Familia
             actual_target_class.class_eval do
               # Helper method to get index set for a specific field value
               # This acts as a factory for field-value-specific DataTypes
-              define_method("#{index_name}_for") do |field_value|
+              define_method(:"#{index_name}_for") do |field_value|
                 # Return properly managed DataType instance with parameterized key
                 index_key = "#{index_name}:#{field_value}"
                 Familia::UnsortedSet.new(index_key, parent: self)
@@ -99,26 +99,26 @@ module Familia
             # Generate instance sampling method (e.g., company.sample_from_department)
             actual_target_class.class_eval do
 
-              define_method("sample_from_#{field}") do |field_value, count = 1|
+              define_method(:"sample_from_#{field}") do |field_value, count = 1|
                 index_set = send("#{index_name}_for", field_value) # i.e. UnsortedSet
 
                 # Get random members efficiently (O(1) via SRANDMEMBER with count)
                 # Returns array even for count=1 for consistent API
                 index_set.sample(count).map do |id|
-                  indexed_class.new(index_set.deserialize_value(id))
+                  indexed_class.find_by_identifier(id)
                 end
               end
 
               # Generate bulk query method (e.g., company.find_all_by_department)
-              define_method("find_all_by_#{field}") do |field_value|
+              define_method(:"find_all_by_#{field}") do |field_value|
                 index_set = send("#{index_name}_for", field_value) # i.e. UnsortedSet
 
                 # Get all members from set
-                index_set.members.map { |id| indexed_class.new(id) }
+                index_set.members.map { |id| indexed_class.find_by_identifier(id) }
               end
 
               # Generate method to rebuild the index for this parent instance
-              define_method("rebuild_#{index_name}") do
+              define_method(:"rebuild_#{index_name}") do
                 # This would need to be implemented based on how you track which
                 # objects belong to this parent instance
                 # For now, just a placeholder
@@ -138,7 +138,7 @@ module Familia
           def generate_mutation_methods_self(indexed_class, field, target_class, index_name)
             target_class_config = target_class.config_name
             indexed_class.class_eval do
-              method_name = "add_to_#{target_class_config}_#{index_name}"
+              method_name = :"add_to_#{target_class_config}_#{index_name}"
               Familia.ld("[MultiIndexGenerators] #{name} method #{method_name}")
 
               define_method(method_name) do |target_instance|
@@ -154,7 +154,7 @@ module Familia
                 index_set.add(identifier)
               end
 
-              method_name = "remove_from_#{target_class_config}_#{index_name}"
+              method_name = :"remove_from_#{target_class_config}_#{index_name}"
               Familia.ld("[MultiIndexGenerators] #{name} method #{method_name}")
 
               define_method(method_name) do |target_instance|
@@ -170,7 +170,7 @@ module Familia
                 index_set.remove(identifier)
               end
 
-              method_name = "update_in_#{target_class_config}_#{index_name}"
+              method_name = :"update_in_#{target_class_config}_#{index_name}"
               Familia.ld("[MultiIndexGenerators] #{name} method #{method_name}")
 
               define_method(method_name) do |target_instance, old_field_value = nil|