familia 2.1.1 → 2.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 923e4c76b9ee95ca39160a66ec9534462b21b16e273fc27c6e67e707647da1a4
-  data.tar.gz: c8dd58304412dd4d9ad57727a4044d8680c16f701869e3f8dd7d0e2a57ed221a
+  metadata.gz: b5608f472e264e8cbf6228e398d4a25517173e55ad0d7d10a2b74de31121b18f
+  data.tar.gz: 665996a35ca541c9884453c4c664d197b865bdba888e424cdd8a5a555df96a37
 SHA512:
-  metadata.gz: 0716f0b0e47a4758933547b884da5265ce186ce3eba55def7672e2540823b8f15671791ed7b2906f6862d863faa273436fb9e52434cec0e10a91a9bd7614ae9e
-  data.tar.gz: 6e018ee75bb16fa2e1be7a7e812dbcd8d8ca9e8a3c82fe078ef19958e3736f01f8c5f34b925a9fc6a0fd3d7ca709604571a5915ee2b828f2a67539e61ac401a2
+  metadata.gz: 22c1c274df0683053f768b418e7bc239fbc31fce4decf405af2e87c0b60f6f0c365c7dd88ba7a897746fffd785bdf15cfba71549ea8f32a0cdbf054eeb2467fc
+  data.tar.gz: b8bbe2ec2153c5de43b98bf9e03e80e94d494d056bbb0b65711f20b76fb77b99f5847be4b5420ccce5497ef18fb29dd38fd7f8329b67ce4928dff6d6b43546d4

data/CHANGELOG.rst

@@ -7,6 +7,54 @@ The format is based on `Keep a Changelog <https://keepachangelog.com/en/1.1.0/>`
 
    <!--scriv-insert-here-->
 
+.. _changelog-2.2.0:
+
+2.2.0 — 2026-02-23
+==================
+
+Added
+-----
+
+- Introduced ``reference: true`` option for DataType collection declarations.
+  Collections with this option store member identifiers raw instead of
+  JSON-encoding them, resolving the semantic mismatch between field storage
+  (type-preserving JSON) and collection member storage (identity references).
+
+Fixed
+-----
+
+- Fixed serialization mismatch in ``instances`` sorted set where
+  ``persist_to_storage`` passed a string identifier (JSON-encoded as
+  ``"\"abc-123\""``), while direct calls passed Familia objects (stored raw as
+  ``abc-123``). Now passes ``self`` to ``instances.add`` and declares
+  ``reference: true`` on the collection, ensuring consistent storage.
+  (`#215 <https://github.com/delano/familia/issues/215>`_)
+
+- Fixed ``UnsortedSet#pop`` returning raw Redis strings instead of deserialized
+  values.
+
+- Fixed ``UnsortedSet#move`` passing raw values to Redis instead of serializing
+  them.
+
+- Fixed ``SortedSet#increment`` truncating scores to integer (``.to_i``) instead
+  of preserving float precision (``.to_f``).
+
+Documentation
+-------------
+
+- Added collection member serialization guide to ``docs/guides/field-system.md``
+  explaining the distinction between field serialization (JSON for type
+  preservation) and collection member serialization (raw identifiers for
+  reference collections).
+
+AI Assistance
+-------------
+
+- Claude assisted with systematic audit of all ``.add()`` call sites and
+  collection declarations across the codebase, identifying the root cause of the
+  serialization mismatch and the three additional DataType method bugs discovered
+  during the audit.
+
 .. _changelog-2.1.1:
 
 2.1.1 — 2026-02-02

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    familia (2.1.1)
+    familia (2.2.0)
       concurrent-ruby (~> 1.3)
       connection_pool (>= 2.4, < 4.0)
       csv (~> 3.3)

data/docs/guides/feature-object-identifiers.md

@@ -193,6 +193,10 @@ custom.objid_generator_used     # => nil (unknown provenance)
 - Debugging and auditing benefit from generator tracking
 - Format validation can be performed based on expected generator
 
+## Identifiers in Collections
+
+When object identifiers are stored in DataType collections (sorted sets, sets, lists), they are stored as **raw strings** — not JSON-encoded. This is critical for consistent membership checks and lookups. Collections that hold object references must be declared with `class:` and `reference: true` options so that `serialize_value` treats both Familia objects and plain identifier strings identically. See [Collection Member Serialization](field-system.md#collection-member-serialization) for the full explanation of why this distinction exists.
+
 ## Lookup Management
 
 ### Automatic Mapping

data/docs/guides/feature-relationships-indexing.md

@@ -366,6 +366,10 @@ company.badge_index.to_h.each do |badge, emp_id|
 end
 ```
 
+## Index Storage Format
+
+Index values (the object identifiers stored in hash keys and sets) are raw strings, not JSON-encoded. This is a deliberate design choice shared across all Familia collections that store object references — it ensures that lookups, membership checks, and key construction all operate on the same byte representation. See [Collection Member Serialization](field-system.md#collection-member-serialization) for the underlying serialization rules.
+
 ## Redis Key Patterns
 
 | Type | Pattern | Example |

data/docs/guides/feature-relationships.md

@@ -167,6 +167,10 @@ team.members.to_a                         # Just IDs
 team.member_instances                     # Load objects
 ```
 
+## Serialization of Collection Members
+
+Relationship collections (participation sorted sets, index hash keys, instance-scoped sets) store object identifiers as raw strings. When adding objects to these collections, `serialize_value` extracts the `.identifier` from Familia objects and stores it without JSON encoding. This ensures consistent membership checks regardless of whether code passes an object reference or a string identifier. See [Collection Member Serialization](field-system.md#collection-member-serialization) for the authoritative explanation of why reference collections use raw identifiers while value fields use JSON.
+
 ## Best Practices
 
 1. **Use bulk methods** for multiple additions: `add_domains([d1, d2, d3])`

data/docs/guides/field-system.md

@@ -146,6 +146,43 @@ user.preferences["theme"] = "dark"
 user.login_count.increment
 ```
 
+## Collection Member Serialization
+
+DataType collections (lists, sets, sorted sets, hash keys) and hashkey fields use different serialization strategies based on what they store.
+
+### Fields: JSON Serialization for Type Preservation
+
+Hashkey fields store arbitrary Ruby values — integers, booleans, hashes, nil. All values are JSON-encoded so types survive the Redis round-trip. An Integer `35` stores as `"35"` and loads back as Integer `35`, not String `"35"`.
+
+### Collections: Raw Identifiers for Object References
+
+When a collection's members represent references to Familia objects, those members must be stored as **raw identifier strings** — not JSON-encoded. Identifiers are lookup keys: they're matched against, compared with, and used to construct Redis keys (e.g. `customer:abc-def-123:object`). JSON-encoding an identifier produces a different byte sequence (`"\"abc-def-123\""` vs `abc-def-123`), which causes silent duplicates and broken membership checks.
+
+The `class:` and `reference: true` options on a collection declaration tell `serialize_value` that members are object references, not arbitrary values:
+
+```ruby
+class Customer < Familia::Horreum
+  # Members are object references — stored as raw identifiers
+  class_sorted_set :instances, class: self, reference: true
+
+  # Members are arbitrary values — stored as JSON
+  list :activity_log
+end
+```
+
+With these options set, `serialize_value` normalizes both code paths:
+- Passing a Familia object extracts `.identifier` and stores it raw
+- Passing a String identifier for a Familia class stores it raw (same result)
+- Passing any other value JSON-encodes it for type preservation
+
+Without `class:` metadata, a collection has no way to distinguish "this string is an identifier" from "this string is an arbitrary value" — and the two paths silently diverge.
+
+### The `instances` Sorted Set
+
+Every Horreum subclass automatically gets a `class_sorted_set :instances` — a class-level registry of persisted objects. Members are raw identifier strings; scores are timestamps of when each object was last saved. This is the index used to enumerate all known instances of a class, check persistence, or clean up stale entries.
+
+Because `instances` stores object references, it is declared with `class:` and `reference: true` to ensure consistent serialization regardless of whether callers pass an object or a string identifier.
+
 ## Advanced Field Types
 
 ### Creating Custom Field Types

data/lib/familia/data_type/serialization.rb

@@ -38,6 +38,15 @@ module Familia
           return prepared
         end
 
+        # Priority 1b: If this collection stores object references (reference: true)
+        # and the value is a String, treat it as a raw identifier. This prevents
+        # mismatches when callers pass identifier strings directly instead of
+        # Familia objects.
+        if val.is_a?(String) && opts[:reference]
+          Familia.debug "  String identifier (reference): #{val}"
+          return val
+        end
+
         # Priority 2: Everything else gets JSON serialized for type preservation
         # This unifies behavior with Horreum fields (Issue #190)
         prepared = Familia::JsonSerializer.dump(val)
@@ -80,6 +89,11 @@ module Familia
         Familia.debug "deserialize_values: (#{@opts}) #{values}"
         return [] if values.empty?
 
+        # Reference collections store raw identifiers — return as-is
+        if @opts[:reference]
+          return values.flatten
+        end
+
         # If a class option is specified, use class-based deserialization
         if @opts[:class]
           unless @opts[:class].respond_to?(:from_json)
@@ -94,9 +108,9 @@ module Familia
 
             val
           rescue StandardError => e
-            Familia.info obj
-            Familia.info "Parse error for #{dbkey} (from_json): #{e.message}"
-            Familia.info e.backtrace
+            Familia.debug "[deserialize] from_json error in #{dbkey}: #{e.message}"
+            Familia.debug "  raw value: #{obj.inspect[0..80]}"
+            Familia.trace :DESERIALIZE_ERROR, dbkey, e.message if Familia.debug?
             nil
           end
 
@@ -110,7 +124,7 @@ module Familia
           begin
             Familia::JsonSerializer.parse(obj)
           rescue Familia::SerializerError
-            # Fallback for legacy data stored without JSON encoding
+            Familia.debug "[deserialize] Raw fallback in #{dbkey}: #{obj.inspect[0..80]}"
             obj
           end
         end
@@ -138,6 +152,9 @@ module Familia
 
         return @opts[:default] if val.nil?
 
+        # Reference collections store raw identifiers — return as-is
+        return val if @opts[:reference]
+
         # If a class option is specified, use the existing class-based deserialization
         if @opts[:class]
           ret = deserialize_values val
@@ -149,7 +166,7 @@ module Familia
         begin
           Familia::JsonSerializer.parse(val)
         rescue Familia::SerializerError
-          # Fallback for legacy data stored without JSON encoding
+          Familia.debug "[deserialize] Raw fallback in #{dbkey}: #{val.inspect[0..80]}"
           val
         end
       end

data/lib/familia/data_type/types/sorted_set.rb

@@ -269,7 +269,7 @@ module Familia
     end
 
     def increment(val, by = 1)
-      dbclient.zincrby(dbkey, by, serialize_value(val)).to_i
+      dbclient.zincrby(dbkey, by, serialize_value(val)).to_f
     end
     alias incr increment
     alias incrby increment

data/lib/familia/data_type/types/unsorted_set.rb

@@ -92,11 +92,11 @@ module Familia
     end
 
     def pop
-      dbclient.spop dbkey
+      deserialize_value(dbclient.spop(dbkey))
     end
 
     def move(dstkey, val)
-      dbclient.smove dbkey, dstkey, val
+      dbclient.smove dbkey, dstkey, serialize_value(val)
     end
 
     # Get one or more random members from the set

data/lib/familia/data_type.rb

@@ -25,7 +25,7 @@ module Familia
     using Familia::Refinements::TimeLiterals
 
     @registered_types = {}
-    @valid_options = %i[class parent default_expiration default logical_database dbkey dbclient suffix prefix].freeze
+    @valid_options = %i[class parent default_expiration default logical_database dbkey dbclient suffix prefix reference].freeze
     @logical_database = nil
 
     feature :expiration

data/lib/familia/horreum/persistence.rb

@@ -643,7 +643,7 @@ module Familia
         auto_update_class_indexes
 
         # 4. Add to instances collection if available
-        self.class.instances.add(identifier, Familia.now) if self.class.respond_to?(:instances)
+        self.class.instances.add(self, Familia.now) if self.class.respond_to?(:instances)
 
         hmset_result
       end

data/lib/familia/horreum.rb

@@ -169,7 +169,7 @@ module Familia
         Familia.members << member
 
         # Set up automatic instance tracking using built-in class_sorted_set
-        member.class_sorted_set :instances
+        member.class_sorted_set :instances, class: member, reference: true
 
         super
       end

data/lib/familia/version.rb

@@ -4,5 +4,5 @@
 
 module Familia
   # Version information for the Familia
-  VERSION = '2.1.1' unless defined?(Familia::VERSION)
+  VERSION = '2.2.0' unless defined?(Familia::VERSION)
 end

data/try/integration/models/customer_try.rb

@@ -98,7 +98,7 @@ multi_result = @customer.destroy!
 cust = Customer.find_by_id('test@example.com')
 exists = Customer.exists?('test@example.com')
 [multi_result.results, cust.nil?, exists]
-#=> [[1, 0, 1, 1, 1, 1, 1, true], true, false]
+#=> [[1, 0, 1, 1, 1, 1, 1, false], true, false]
 
 ## Customer.destroy! can be called on an already destroyed object
 @customer.destroy!

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: familia
 version: !ruby/object:Gem::Version
-  version: 2.1.1
+  version: 2.2.0
 platform: ruby
 authors:
 - Delano Mandelbaum