familia 2.0.0.pre16 → 2.0.0.pre17

data/lib/familia/data_type/types/sorted_set.rb

@@ -46,17 +46,81 @@ module Familia
       add val, score
     end
 
-    def add(val, score = nil)
-      # TODO: Support some or all of the ZADD options.
-      # XX: Only update existing elements. Don't add new ones.
-      # NX: Only add new elements. Don't update existing ones.
-      # LT: Only update if new score < current score. Doesn't prevent adding.
-      # GT: Only update if new score > current score. Doesn't prevent adding.
-      # CH: Return total changed elements (new + updated) instead of just new.
-      # INCR: Acts like ZINCRBY. Only one score-element pair allowed.
-      # Note: GT, LT and NX options are mutually exclusive.
+    # Adds an element to the sorted set with an optional score and ZADD options.
+    #
+    # This method supports Redis ZADD options for conditional adds and updates:
+    # - **NX**: Only add new elements (don't update existing)
+    # - **XX**: Only update existing elements (don't add new)
+    # - **GT**: Only update if new score > current score
+    # - **LT**: Only update if new score < current score
+    # - **CH**: Return changed count (new + updated) instead of just new count
+    #
+    # @param val [Object] The value to add to the sorted set
+    # @param score [Numeric, nil] The score for ranking (defaults to current timestamp)
+    # @param nx [Boolean] Only add new elements, don't update existing (default: false)
+    # @param xx [Boolean] Only update existing elements, don't add new (default: false)
+    # @param gt [Boolean] Only update if new score > current score (default: false)
+    # @param lt [Boolean] Only update if new score < current score (default: false)
+    # @param ch [Boolean] Return changed count instead of added count (default: false)
+    #
+    # @return [Boolean] Returns the return value from the redis gem's ZADD
+    #   command. Returns true if element was added or changed (with CH option),
+    #   false if element score was updated without change tracking or no
+    #   operation occurred due to option constraints (NX, XX, GT, LT).
+    #
+    # @raise [ArgumentError] If mutually exclusive options are specified together
+    #   (NX+XX, GT+LT, NX+GT, NX+LT)
+    #
+    # @example Add new element with timestamp
+    #   metrics.add('pageview', Time.now.to_f)  #=> true
+    #
+    # @example Preserve original timestamp on subsequent saves
+    #   index.add(email, Time.now.to_f, nx: true)  #=> true
+    #   index.add(email, Time.now.to_f, nx: true)  #=> false (unchanged)
+    #
+    # @example Update timestamp only for existing entries
+    #   index.add(email, Time.now.to_f, xx: true)  #=> false (if doesn't exist)
+    #
+    # @example Only update if new score is higher (leaderboard)
+    #   scores.add(player, 1000, gt: true)  #=> true (new entry)
+    #   scores.add(player, 1500, gt: true)  #=> false (updated)
+    #   scores.add(player, 1200, gt: true)  #=> false (not updated, score lower)
+    #
+    # @example Track total changes for analytics
+    #   changed = metrics.add(user, score, ch: true)  #=> true (new or updated)
+    #
+    # @example Combined options: only update existing, only if score increases
+    #   index.add(key, new_score, xx: true, gt: true)
+    #
+    # @note GT and LT options do NOT prevent adding new elements, they only
+    #   affect update behavior for existing elements.
+    #
+    # @note Default behavior (no options) adds new elements and updates existing
+    #   ones unconditionally, matching standard Redis ZADD semantics.
+    #
+    # @note INCR option is not supported. Use the increment method for ZINCRBY operations.
+    #
+    def add(val, score = nil, nx: false, xx: false, gt: false, lt: false, ch: false)
       score ||= Familia.now
-      ret = dbclient.zadd dbkey, score, serialize_value(val)
+
+      # Validate mutual exclusivity
+      validate_zadd_options!(nx: nx, xx: xx, gt: gt, lt: lt)
+
+      # Build options hash for redis gem
+      opts = {}
+      opts[:nx] = true if nx
+      opts[:xx] = true if xx
+      opts[:gt] = true if gt
+      opts[:lt] = true if lt
+      opts[:ch] = true if ch
+
+      # Pass options to ZADD
+      ret = if opts.empty?
+        dbclient.zadd(dbkey, score, serialize_value(val))
+      else
+        dbclient.zadd(dbkey, score, serialize_value(val), **opts)
+      end
+
       update_expiration
       ret
     end
@@ -242,6 +306,45 @@ module Familia
       at(-1)
     end
 
+
+    private
+
+    # Validates that mutually exclusive ZADD options are not specified together.
+    #
+    # @param nx [Boolean] NX option flag
+    # @param xx [Boolean] XX option flag
+    # @param gt [Boolean] GT option flag
+    # @param lt [Boolean] LT option flag
+    #
+    # @raise [ArgumentError] If mutually exclusive options are specified
+    #
+    # @note Valid combinations: XX+GT, XX+LT
+    # @note Invalid combinations: NX+XX, GT+LT, NX+GT, NX+LT
+    #
+    def validate_zadd_options!(nx:, xx:, gt:, lt:)
+      # NX and XX are mutually exclusive
+      if nx && xx
+        raise ArgumentError, "ZADD options NX and XX are mutually exclusive"
+      end
+
+      # GT and LT are mutually exclusive
+      if gt && lt
+        raise ArgumentError, "ZADD options GT and LT are mutually exclusive"
+      end
+
+      # NX is mutually exclusive with GT
+      if nx && gt
+        raise ArgumentError, "ZADD options NX and GT are mutually exclusive"
+      end
+
+      # NX is mutually exclusive with LT
+      if nx && lt
+        raise ArgumentError, "ZADD options NX and LT are mutually exclusive"
+      end
+
+      # Note: XX + GT and XX + LT are valid combinations
+    end
+
     Familia::DataType.register self, :sorted_set
     Familia::DataType.register self, :zset
   end

data/lib/familia/data_type/types/stringkey.rb

@@ -114,10 +114,6 @@ module Familia
       ret.positive?
     end
 
-    def nil?
-      value.nil?
-    end
-
     Familia::DataType.register self, :string
     Familia::DataType.register self, :stringkey
   end

data/lib/familia/data_type.rb

@@ -1,5 +1,8 @@
 # lib/familia/data_type.rb
 
+require_relative 'data_type/class_methods'
+require_relative 'data_type/settings'
+require_relative 'data_type/connection'
 require_relative 'data_type/commands'
 require_relative 'data_type/serialization'
 
@@ -14,6 +17,7 @@ module Familia
   # @abstract Subclass and implement Database data type specific methods
   class DataType
     include Familia::Base
+    extend ClassMethods
     extend Familia::Features
 
     using Familia::Refinements::TimeLiterals
@@ -29,60 +33,6 @@ module Familia
       attr_reader :registered_types, :valid_options, :has_related_fields
     end
 
-    # DataType::ClassMethods
-    #
-    module ClassMethods
-      attr_accessor :parent, :suffix, :prefix, :uri
-      attr_writer :logical_database
-
-      # To be called inside every class that inherits DataType
-      # +methname+ is the term used for the class and instance methods
-      # that are created for the given +klass+ (e.g. set, list, etc)
-      def register(klass, methname)
-        Familia.trace :REGISTER, nil, "[#{self}] Registering #{klass} as #{methname.inspect}" if Familia.debug?
-
-        @registered_types[methname] = klass
-      end
-
-      # Get the registered type class from a given method name
-      # +methname+ is the method name used to register the class (e.g. :set, :list, etc)
-      # Returns the registered class or nil if not found
-      def registered_type(methname)
-        @registered_types[methname]
-      end
-
-      def logical_database(val = nil)
-        @logical_database = val unless val.nil?
-        @logical_database || parent&.logical_database
-      end
-
-      def uri(val = nil)
-        @uri = val unless val.nil?
-        @uri || (parent ? parent.uri : Familia.uri)
-      end
-
-      def inherited(obj)
-        Familia.trace :DATATYPE, nil, "#{obj} is my kinda type" if Familia.debug?
-        obj.logical_database = logical_database
-        obj.default_expiration = default_expiration # method added via Features::Expiration
-        obj.uri = uri
-        super
-      end
-
-      def valid_keys_only(opts)
-        opts.slice(*DataType.valid_options)
-      end
-
-      def relations?
-        @has_related_fields ||= false
-      end
-    end
-    extend ClassMethods
-
-    attr_reader :keystring, :opts, :uri, :logical_database
-
-    alias url uri
-
     # +keystring+: If parent is set, this will be used as the suffix
     # for dbkey. Otherwise this becomes the value of the key.
     # If this is an Array, the elements will be joined.
@@ -125,145 +75,8 @@ module Familia
       init if respond_to? :init
     end
 
-    # TODO: Replace with Chain of Responsibility pattern
-    def dbclient
-      return Fiber[:familia_transaction] if Fiber[:familia_transaction]
-      return @dbclient if @dbclient
-
-      # Delegate to parent if present, otherwise fall back to Familia
-      parent ? parent.dbclient : Familia.dbclient(opts[:logical_database])
-    end
-
-    # Produces the full dbkey for this object.
-    #
-    # @return [String] The full dbkey.
-    #
-    # This method determines the appropriate dbkey based on the context of the DataType object:
-    #
-    # 1. If a hardcoded key is set in the options, it returns that key.
-    # 2. For instance-level DataType objects, it uses the parent instance's dbkey method.
-    # 3. For class-level DataType objects, it uses the parent class's dbkey method.
-    # 4. For standalone DataType objects, it uses the keystring as the full dbkey.
-    #
-    # For class-level DataType objects (parent_class? == true):
-    # - The suffix is optional and used to differentiate between different types of objects.
-    # - If no suffix is provided, the class's default suffix is used (via the self.suffix method).
-    # - If a nil suffix is explicitly passed, it won't appear in the resulting dbkey.
-    # - Passing nil as the suffix is how class-level DataType objects are created without
-    #   the global default 'object' suffix.
-    #
-    # @example Instance-level DataType
-    #   user_instance.some_datatype.dbkey  # => "user:123:some_datatype"
-    #
-    # @example Class-level DataType
-    #   User.some_datatype.dbkey  # => "user:some_datatype"
-    #
-    # @example Standalone DataType
-    #   DataType.new("mykey").dbkey  # => "mykey"
-    #
-    # @example Class-level DataType with explicit nil suffix
-    #   User.dbkey("123", nil)  # => "user:123"
-    #
-    def dbkey
-      # Return the hardcoded key if it's set. This is useful for
-      # support legacy keys that aren't derived in the same way.
-      return opts[:dbkey] if opts[:dbkey]
-
-      if parent_instance?
-        # This is an instance-level datatype object so the parent instance's
-        # dbkey method is defined in Familia::Horreum::InstanceMethods.
-        parent.dbkey(keystring)
-      elsif parent_class?
-        # This is a class-level datatype object so the parent class' dbkey
-        # method is defined in Familia::Horreum::DefinitionMethods.
-        parent.dbkey(keystring, nil)
-      else
-        # This is a standalone DataType object where it's keystring
-        # is the full database key (dbkey).
-        keystring
-      end
-    end
-
-    def class?
-      !@opts[:class].to_s.empty? && @opts[:class].is_a?(Familia)
-    end
-
-    # Provides a structured way to "gear down" to run db commands that are
-    # not implemented in our DataType classes since we intentionally don't
-    # have a method_missing method.
-    def direct_access
-      yield(dbclient, dbkey)
-    end
-
-    def parent_instance?
-      parent&.is_a?(Horreum::ParentDefinition)
-    end
-
-    def parent_class?
-      parent.is_a?(Class) && parent.ancestors.include?(Familia::Horreum)
-    end
-
-    def parent?
-      parent_class? || parent_instance?
-    end
-
-    def parent
-      # Return cached ParentDefinition if available
-      return @parent if @parent
-
-      # Return class-level parent if no instance parent
-      return self.class.parent unless @parent_ref
-
-      # Create ParentDefinition dynamically from stored reference.
-      # This ensures we get the current identifier value (available after initialization)
-      # rather than a stale nil value from initialization time. Cannot cache due to frozen object.
-      Horreum::ParentDefinition.from_parent(@parent_ref)
-    end
-
-    def parent=(value)
-      case value
-      when Horreum::ParentDefinition
-        @parent = value
-      when nil
-        @parent = nil
-        @parent_ref = nil
-      else
-        # Store parent instance reference for lazy ParentDefinition creation.
-        # During initialization, the parent's identifier may not be available yet,
-        # so we defer ParentDefinition creation until first access for memory efficiency.
-        # Note: @parent_ref is not cleared after use because DataType objects are frozen.
-        @parent_ref = value
-        @parent = nil  # Will be created dynamically in parent method
-      end
-    end
-
-    def uri
-      # Return explicit instance URI if set
-      return @uri if @uri
-
-      # If we have a parent with logical_database, build URI with that database
-      if parent && parent.respond_to?(:logical_database) && parent.logical_database
-        new_uri = (self.class.uri || Familia.uri).dup
-        new_uri.db = parent.logical_database
-        new_uri
-      else
-        # Fall back to class-level URI or global Familia.uri
-        self.class.uri || Familia.uri
-      end
-    end
-
-    def uri=(value)
-      @uri = value
-    end
-
-    def dump_method
-      self.class.dump_method
-    end
-
-    def load_method
-      self.class.load_method
-    end
-
+    include Settings
+    include Connection
     include Commands
     include Serialization
   end

data/lib/familia/features/encrypted_fields.rb

@@ -259,7 +259,7 @@ module Familia
     # - Insider threats with application access
     #
     module EncryptedFields
-      Familia::Base.add_feature self, :encrypted_fields
+      Familia::Base.add_feature self, :encrypted_fields, depends_on: nil, field_group: :encrypted_fields
 
       def self.included(base)
         Familia.trace :LOADED, self, base if Familia.debug?
@@ -297,7 +297,10 @@ module Familia
           @encrypted_fields ||= []
           @encrypted_fields << name unless @encrypted_fields.include?(name)
 
-          require_relative 'encrypted_fields/encrypted_field_type'
+          # Add to field_groups if the group exists
+          if field_groups&.key?(:encrypted_fields)
+            field_groups[:encrypted_fields] << name
+          end
 
           field_type = EncryptedFieldType.new(name, aad_fields: aad_fields, **)
           register_field_type(field_type)

data/lib/familia/features/external_identifier.rb

@@ -10,6 +10,7 @@ module Familia
       def self.included(base)
         Familia.trace :LOADED, self, base if Familia.debug?
         base.extend ModelClassMethods
+        base.include ModelInstanceMethods
 
         # Ensure default prefix is set in feature options
         base.add_feature_options(:external_identifier, prefix: 'ext')
@@ -75,9 +76,6 @@ module Familia
 
               instance_variable_set(:"@#{field_name}", derived_extid)
 
-              # Update mapping if we have an identifier (objid)
-              self.class.extid_lookup[derived_extid] = identifier if respond_to?(:identifier) && identifier
-
               derived_extid
             end
           end
@@ -103,11 +101,6 @@ module Familia
 
               # Set the new value
               instance_variable_set(:"@#{field_name}", value)
-
-              # Update mapping if we have both extid and identifier
-              return unless value && respond_to?(:identifier) && identifier
-
-              self.class.extid_lookup[value] = identifier
             end
           end
         end
@@ -159,6 +152,54 @@ module Familia
         end
       end
 
+      # Instance methods for external identifier management
+      module ModelInstanceMethods
+        # Override save to update extid_lookup mapping
+        #
+        # This ensures the extid_lookup index is populated during save operations
+        # rather than during object initialization, preventing unwanted database
+        # writes when calling .new()
+        #
+        # @param update_expiration [Boolean] Whether to update key expiration
+        # @return [Boolean] True if save was successful
+        #
+        def save(update_expiration: true)
+          result = super
+
+          # Update extid_lookup mapping after successful save
+          if result && respond_to?(:extid) && respond_to?(:identifier)
+            current_extid = extid  # Triggers lazy generation if needed
+            if current_extid && identifier
+              self.class.extid_lookup[current_extid] = identifier
+            end
+          end
+
+          result
+        end
+
+        # Override save_if_not_exists to update extid_lookup mapping
+        #
+        # This ensures the extid_lookup index is populated during create operations
+        # which use save_if_not_exists instead of save.
+        #
+        # @param update_expiration [Boolean] Whether to update key expiration
+        # @return [Boolean] True if save was successful
+        #
+        def save_if_not_exists(update_expiration: true)
+          result = super
+
+          # Update extid_lookup mapping after successful save
+          if result && respond_to?(:extid) && respond_to?(:identifier)
+            current_extid = extid  # Triggers lazy generation if needed
+            if current_extid && identifier
+              self.class.extid_lookup[current_extid] = identifier
+            end
+          end
+
+          result
+        end
+      end
+
       # Derives a deterministic, public-facing external identifier from the object's
       # internal `objid`.
       #

data/lib/familia/features/object_identifier.rb

@@ -88,6 +88,7 @@ module Familia
       def self.included(base)
         Familia.trace :LOADED, self, base if Familia.debug?
         base.extend ModelClassMethods
+        base.include ModelInstanceMethods
 
         # Ensure default generator is set in feature options
         base.add_feature_options(:object_identifier, generator: DEFAULT_GENERATOR)
@@ -160,9 +161,6 @@ module Familia
               generator = options[:generator] || DEFAULT_GENERATOR
               instance_variable_set(:"@#{field_name}_generator_used", generator)
 
-              # Update mapping from objid to model primary key
-              self.class.objid_lookup[generated_id] = identifier if respond_to?(:identifier) && identifier
-
               generated_id
             end
           end
@@ -198,14 +196,11 @@ module Familia
 
               instance_variable_set(:"@#{field_name}", value)
 
-              # Update mapping from objid to this new identifier
-              self.class.objid_lookup[value] = identifier unless value.nil? || identifier.nil?
-
               # When setting objid from external source (e.g., loading from Valkey/Redis),
-              # we cannot determine the original generator, so we clear the provenance
-              # tracking to indicate unknown origin. This prevents false assumptions
-              # about the security properties of externally-provided identifiers.
-              instance_variable_set(:"@#{field_name}_generator_used", nil)
+              # infer the generator type from the format to restore provenance tracking.
+              # This allows features like ExternalIdentifier to work correctly on loaded objects.
+              inferred_generator = infer_objid_generator(value)
+              instance_variable_set(:"@#{field_name}_generator_used", inferred_generator)
             end
           end
         end
@@ -284,6 +279,54 @@ module Familia
         end
       end
 
+      # Instance methods for object identifier management
+      module ModelInstanceMethods
+        # Override save to update objid_lookup mapping
+        #
+        # This ensures the objid_lookup index is populated during save operations
+        # rather than during object initialization, preventing unwanted database
+        # writes when calling .new()
+        #
+        # @param update_expiration [Boolean] Whether to update key expiration
+        # @return [Boolean] True if save was successful
+        #
+        def save(update_expiration: true)
+          result = super
+
+          # Update objid_lookup mapping after successful save
+          if result && respond_to?(:objid) && respond_to?(:identifier)
+            current_objid = objid  # Triggers lazy generation if needed
+            if current_objid && identifier
+              self.class.objid_lookup[current_objid] = identifier
+            end
+          end
+
+          result
+        end
+
+        # Override save_if_not_exists to update objid_lookup mapping
+        #
+        # This ensures the objid_lookup index is populated during create operations
+        # which use save_if_not_exists instead of save.
+        #
+        # @param update_expiration [Boolean] Whether to update key expiration
+        # @return [Boolean] True if save was successful
+        #
+        def save_if_not_exists(update_expiration: true)
+          result = super
+
+          # Update objid_lookup mapping after successful save
+          if result && respond_to?(:objid) && respond_to?(:identifier)
+            current_objid = objid  # Triggers lazy generation if needed
+            if current_objid && identifier
+              self.class.objid_lookup[current_objid] = identifier
+            end
+          end
+
+          result
+        end
+      end
+
       # Instance method for generating object identifier using configured strategy
       #
       # This method is called by the ObjectIdentifierFieldType when lazy generation
@@ -304,10 +347,39 @@ module Familia
         objid
       end
 
-      # Full-length alias setter for objid
+      # Infers the generator type (:uuid_v7, :uuid_v4, :hex) from the format of an objid string.
       #
-      # @param value [String] The object identifier to set
+      # This method analyzes the objid format to restore provenance tracking when loading
+      # objects from Redis, allowing dependent features like ExternalIdentifier to work correctly.
       #
+      # @param objid_value [String] The objid string to analyze
+      # @return [Symbol, nil] The inferred generator type or nil if unknown
+      def infer_objid_generator(objid_value)
+        return nil if objid_value.nil? || objid_value.to_s.empty?
+
+        objid_str = objid_value.to_s
+
+        # UUID format: xxxxxxxx-xxxx-Vxxx-xxxx-xxxxxxxxxxxx (36 chars with hyphens)
+        # where V is the version nibble at position 14
+        if objid_str.length == 36 && objid_str[8] == '-' && objid_str[13] == '-' && objid_str[18] == '-' && objid_str[23] == '-'
+          version_char = objid_str[14]
+          case version_char
+          when '7'
+            :uuid_v7
+          when '4'
+            :uuid_v4
+          else
+            nil # Unknown UUID version
+          end
+        # Hex format: pure hexadecimal without hyphens (32 or 64 chars typically)
+        elsif objid_str.match?(/\A[0-9a-fA-F]+\z/)
+          :hex
+        else
+          nil # Unknown format
+        end
+      end
+      private :infer_objid_generator
+
       def object_identifier=(value)
         self.objid = value
       end

data/lib/familia/features/relationships/indexing/unique_index_generators.rb

@@ -33,9 +33,12 @@ module Familia
         #   - Employee.rebuild_email_index
         #
         # Generates on Employee (self):
-        #   - employee.add_to_class_email_index
+        #   - employee.add_to_class_email_index (called automatically on save)
         #   - employee.remove_from_class_email_index
         #   - employee.update_in_class_email_index(old_email)
+        #
+        # Note: Class-level indexes auto-populate on save(). Instance-scoped indexes
+        # (with within:) remain manual as they require parent context.
         module UniqueIndexGenerators
           module_function
 
@@ -114,6 +117,7 @@ module Familia
 
               # Generate bulk query method (e.g., company.find_all_by_badge_number)
               define_method("find_all_by_#{field}") do |field_values|
+                field_values = Array(field_values)
                 return [] if field_values.empty?
 
                 # Use declared field accessor instead of manual instantiation
@@ -229,6 +233,7 @@ module Familia
 
             # Generate class-level bulk query method
             indexed_class.define_singleton_method("find_all_by_#{field}") do |field_values|
+              field_values = Array(field_values)
               return [] if field_values.empty?
 
               index_hash = send(index_name) # Access the class-level hashkey DataType

data/lib/familia/features/relationships/indexing.rb

@@ -18,7 +18,7 @@ module Familia
       #   end
       #
       #   user = User.new(user_id: 'u1', email: 'alice@example.com')
-      #   user.add_to_class_email_lookup
+      #   user.save  # Automatically populates email_lookup index
       #   User.find_by_email('alice@example.com')  # → user
       #
       # @example Instance-scoped unique index (within parent, 1:1 via HashKey)
@@ -58,6 +58,12 @@ module Familia
       # - Instance unique: "company:c1:badge_index" → HashKey
       # - Instance multi: "company:c1:dept_index:engineering" → UnsortedSet
       #
+      # Auto-Indexing:
+      # Class-level unique_index declarations automatically populate on save():
+      #   user = User.new(email: 'test@example.com')
+      #   user.save  # Auto-indexes email → user_id
+      # Instance-scoped indexes (with within:) remain manual (require parent context).
+      #
       # Design Philosophy:
       # Indexing is for finding objects by attribute, not ordering them.
       # Use multi_index with UnsortedSet (no temporal scores), then sort in Ruby: