lutaml-store 0.3.2 → 0.3.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 3666640ef12a937789e1fa2357edbb7bb882468a78fac8053e8bcc0efbe94ccd
-  data.tar.gz: da2a66ff3067ead5082ff16a6abea51232e49359a63b84e4cde45a3e42c768ac
+  metadata.gz: 1e80cbd1af20bbb11719de55936da8916731e8984b6db23a37ff87938339da5e
+  data.tar.gz: cb8491705513e1a6bff912033a97f9b9bde3f4a7fc32923691f47bcf5f81a94f
 SHA512:
-  metadata.gz: 3d8eba2c1775fc8ac1fd170c4134be61046954212319ba63e5d6ca8ffeca845a5e3a808e49fded3668f6dba387d7f75b326a6b7821e17375f658a4ad76c92b75
-  data.tar.gz: 75f6f23cbfd0f30cc58a7d21f45906878d260f07bf4b2201b84ca75493233d79406c71df1692353a7e263df962f5a34e498007ecc11c7a0ed46c2ffa89db4c97
+  metadata.gz: 8a7c4cb9de5eac681e24d45ad3df485e8dbe0edc7f3c43cd9f3e613177f3f7233426603a0452752d9d42d1ef54b9540e0f93e1bd956e765aed45c6762f36d367
+  data.tar.gz: a8b684093cedacacd4a8bea265eef3b4d11f5de0ce0b8f9dc256b881212f1a6f843f85dbae27fbc6da283858c3a8634641536e4cf383c51be40de848862d312a

data/docs/src/pages/stores.md

@@ -76,6 +76,28 @@ cache.fetch("key") { expensive_call }     # block runs only on miss
 
 Entries past their TTL are evicted lazily on access; when `max_size` is exceeded the least-recently-used entries are evicted first.
 
+### TTL guarantees (pinned)
+
+The TTL semantics are pinned by specs and safe to build expiry policy on:
+
+- **Persistence** — entries serialize `created_at` as ISO-8601 with
+  microsecond precision, so a FileSystem (or SQLite) cache reopened in a new
+  process keeps every entry's original clock; expiry survives restarts.
+  The memory adapter keeps no state across processes by nature.
+- **Clock** — wall clock, not monotonic. This is the deliberate trade:
+  wall-clock timestamps survive restarts, but a system clock jump changes
+  apparent freshness (a backward jump un-expires nothing an entry's absolute
+  `created_at + ttl` already expired). Consumers needing jump-immunity
+  re-derive freshness from the source.
+- **Expired reads** — `get` on an expired entry returns `nil` **and deletes
+  the entry**: the key disappears from `keys`/`size`/`exists?`.
+- **Sweep** — expiry is lazy-on-read plus an opportunistic
+  `cleanup_expired` (interval `cleanup_interval`, default 300 s) invoked
+  inside mutating reads. There is **no background reaper**; a consumer like
+  the relaton shard cache needs none.
+- **TTL values** — `ttl: nil` (and no `default_ttl`) never expires;
+  `ttl: 0` is immediately expired.
+
 ## PackageStore — multi-model packages
 
 ```ruby

data/lib/lutaml/store/adapter/filesystem.rb

@@ -115,11 +115,32 @@ module Lutaml
           end
         end
 
+        # Reads every entry under one shared lock instead of taking a
+        # fresh flock per key; corrupt entries are repaired after the
+        # shared lock releases, preserving #get's contract for them
         def all
-          keys.each_with_object({}) do |key, result|
-            value = get(key)
-            result[key] = value unless value.nil?
+          result = {}
+          corrupt = []
+
+          with_lock(:sh) do
+            data_files.each do |file_path|
+              key = key_from_path(file_path)
+              next unless key
+
+              status, value = read_verified_at(file_path, key)
+              if status == :corrupt
+                corrupt << [key, value]
+              else
+                result[key] = value
+              end
+            end
           end
+
+          corrupt.each do |key, error|
+            result[key] = repair_corruption(key) || raise(error)
+          end
+
+          result
         end
 
         def clear
@@ -257,6 +278,10 @@ module Lutaml
           file_path = existing_path_for_key(key)
           return [:missing, nil] unless file_path
 
+          read_verified_at(file_path, key)
+        end
+
+        def read_verified_at(file_path, _key)
           data = file_safe_read(file_path)
           metadata = read_metadata(metadata_path_for(file_path))
 

data/lib/lutaml/store/adapter/memory.rb

@@ -19,6 +19,9 @@ module Lutaml
           @ttl_data = @ttl_enabled ? {} : nil
           @default_ttl = @config[:default_ttl] || 3600
           @max_entries = @config[:max_entries]
+          # Earliest live expiry; TTL cleanup skips its scan while now
+          # is before this, so N reads don't cost N full-table scans
+          @next_expiry = nil
         end
 
         # ── Read operations (lock-free via snapshot) ──
@@ -71,7 +74,11 @@ module Lutaml
 
             if @ttl_enabled
               ttl_value = metadata[:ttl] || @default_ttl
-              @ttl_data[key] = Time.now + ttl_value if ttl_value
+              if ttl_value
+                expires_at = Time.now + ttl_value
+                @ttl_data[key] = expires_at
+                track_expiry(expires_at)
+              end
             end
           end
           value
@@ -104,6 +111,7 @@ module Lutaml
             count = @data.size
             @data.clear
             @ttl_data&.clear
+            @next_expiry = nil
             invalidate_snapshot
             count
           end
@@ -113,6 +121,7 @@ module Lutaml
           @write_mutex.synchronize do
             @data.clear
             @ttl_data&.clear
+            @next_expiry = nil
             invalidate_snapshot
           end
         end
@@ -123,10 +132,14 @@ module Lutaml
               evict_oldest_if_needed(key)
               @data[key] = value
 
-              if @ttl_enabled
-                ttl_value = ttl || @default_ttl
-                @ttl_data[key] = Time.now + ttl_value if ttl_value
-              end
+              next unless @ttl_enabled
+
+              ttl_value = ttl || @default_ttl
+              next unless ttl_value
+
+              expires_at = Time.now + ttl_value
+              @ttl_data[key] = expires_at
+              track_expiry(expires_at)
             end
             invalidate_snapshot
           end
@@ -159,6 +172,7 @@ module Lutaml
               @data.delete(key)
               @ttl_data.delete(key)
             end
+            recompute_next_expiry unless expired_keys.empty?
             invalidate_snapshot unless expired_keys.empty?
             expired_keys.size
           end
@@ -171,7 +185,9 @@ module Lutaml
             return false unless @data.key?(key)
 
             if ttl
-              @ttl_data[key] = Time.now + ttl
+              expires_at = Time.now + ttl
+              @ttl_data[key] = expires_at
+              track_expiry(expires_at)
             else
               @ttl_data.delete(key)
             end
@@ -283,13 +299,22 @@ module Lutaml
           @snapshot_stale = true
         end
 
+        def track_expiry(expires_at)
+          @next_expiry = expires_at if @next_expiry.nil? || expires_at < @next_expiry
+        end
+
+        # Full-table scan; recomputes the watermark afterwards
+        def recompute_next_expiry
+          @next_expiry = @ttl_data.values.min
+        end
+
         # Capacity bound (max_entries): evict the oldest-INSERTED entry to
         # make room. Insertion order, not access order — refreshing on read
         # would cost the lock-free snapshot reads. Updating an existing key
         # never evicts.
         def evict_oldest_if_needed(incoming_key)
           return if @max_entries.nil? || @data.key?(incoming_key) ||
-            @data.size < @max_entries
+                    @data.size < @max_entries
 
           oldest = @data.each_key.first
           @data.delete(oldest)
@@ -298,9 +323,8 @@ module Lutaml
 
         def cleanup_expired_if_needed
           return unless @ttl_enabled
-
-          now = Time.now
-          return unless @ttl_data.any? { |_k, expiry| now > expiry }
+          return if @ttl_data.empty?
+          return if @next_expiry && Time.now < @next_expiry
 
           cleanup_expired
         end

data/lib/lutaml/store/adapter/sqlite.rb

@@ -26,26 +26,26 @@ module Lutaml
         end
 
         # ── Key-value operations ──
+        #
+        # Non-string values are serialized as JSON and marked in the
+        # is_json column; strings are stored verbatim and returned
+        # verbatim. Without the marker, a stored string that happens to
+        # look like JSON (e.g. '{"x": 5}') decodes as an object.
 
         def get(key)
-          result = nil
-          execute_query_raw("SELECT value FROM #{@table_name} WHERE key = ?", [key]) do |row|
-            value = row[0]
-            begin
-              result = JSON.parse(value)
-            rescue JSON::ParserError
-              result = value
-            end
-          end
-          result
+          row = query_row("SELECT value, is_json FROM #{@table_name} WHERE key = ?", [key])
+          return nil unless row
+
+          decode_value(row[0], row[1])
         end
 
         def set(key, value)
-          serialized_value = value.is_a?(String) ? value : JSON.generate(value)
+          is_string = value.is_a?(String)
+          serialized_value = is_string ? value : JSON.generate(value)
 
           execute_statement(
-            "INSERT OR REPLACE INTO #{@table_name} (key, value, updated_at) VALUES (?, ?, ?)",
-            [key, serialized_value, Time.now.to_f]
+            "INSERT OR REPLACE INTO #{@table_name} (key, value, is_json, updated_at) VALUES (?, ?, ?, ?)",
+            [key, serialized_value, is_string ? 0 : 1, Time.now.to_f]
           )
           value
         end
@@ -67,8 +67,8 @@ module Lutaml
 
         def all
           result = {}
-          execute_query_raw("SELECT key, value FROM #{@table_name}") do |row|
-            result[row[0]] = row[1]
+          execute_query_raw("SELECT key, value, is_json FROM #{@table_name}") do |row|
+            result[row[0]] = decode_value(row[1], row[2])
           end
           result
         end
@@ -142,9 +142,7 @@ module Lutaml
           sql, params = build_select_sql(query)
           results = []
           execute_query_raw(sql, params) do |row|
-            key = row[0]
-            value = parse_json_value(row[1])
-            results << [key, value]
+            results << [row[0], decode_value(row[1], row[2])]
           end
           results
         end
@@ -165,7 +163,7 @@ module Lutaml
             params << after
           end
 
-          sql = "SELECT key, value FROM #{@table_name}"
+          sql = "SELECT key, value, is_json FROM #{@table_name}"
           sql += " WHERE #{conditions.join(" AND ")}" unless conditions.empty?
           sql += " ORDER BY key ASC"
           sql += " LIMIT ?"
@@ -173,9 +171,7 @@ module Lutaml
 
           results = []
           execute_query_raw(sql, params) do |row|
-            key = row[0]
-            value = parse_json_value(row[1])
-            results << [key, value]
+            results << [row[0].to_s, decode_value(row[1], row[2])]
           end
           results
         end
@@ -221,6 +217,7 @@ module Lutaml
           @db.execute("PRAGMA temp_store=memory")
 
           create_table_if_not_exists
+          migrate_schema
           create_indexes
         end
 
@@ -229,11 +226,24 @@ module Lutaml
             CREATE TABLE IF NOT EXISTS #{@table_name} (
               key TEXT PRIMARY KEY,
               value BLOB NOT NULL,
+              is_json INTEGER NOT NULL DEFAULT 1,
               updated_at REAL NOT NULL DEFAULT (julianday('now'))
             )
           SQL
         end
 
+        # Databases created before the is_json column existed predate
+        # format tracking: legacy rows (default is_json = 1) decode as
+        # JSON when they parse, staying a string otherwise
+        def migrate_schema
+          columns = @db.execute("PRAGMA table_info(#{@table_name})").map do |row|
+            row[1]
+          end
+          return if columns.include?("is_json")
+
+          @db.execute("ALTER TABLE #{@table_name} ADD COLUMN is_json INTEGER NOT NULL DEFAULT 1")
+        end
+
         def create_indexes
           @db.execute("CREATE INDEX IF NOT EXISTS idx_#{@table_name}_updated_at ON #{@table_name} (updated_at)")
         end
@@ -243,7 +253,7 @@ module Lutaml
         def build_select_sql(query)
           conditions, params = build_conditions(query)
 
-          sql = "SELECT key, value FROM #{@table_name}"
+          sql = "SELECT key, value, is_json FROM #{@table_name}"
           sql += " WHERE #{conditions.join(" AND ")}" unless conditions.empty?
           sql += build_order_clause(query.orders)
           sql += build_limit_offset(query.limit_value, query.offset_value, params)
@@ -352,14 +362,27 @@ module Lutaml
           "%#{pattern}%"
         end
 
-        def parse_json_value(raw)
+        # ── Raw query helpers ──
+
+        def query_row(sql, params = [])
+          execute_query_raw(sql, params) do |row|
+            return row
+          end
+          nil
+        end
+
+        # is_json marks rows written from non-string values (the
+        # default, including every pre-format-tracking row): they
+        # decode as JSON when they parse. Rows written from strings
+        # come back verbatim.
+        def decode_value(raw, is_json)
+          return raw unless is_json.to_i == 1
+
           JSON.parse(raw)
         rescue JSON::ParserError
           raw
         end
 
-        # ── Raw query helpers ──
-
         def execute_query_raw(sql, params = [])
           @db.execute(sql, params) do |row|
             yield row if block_given?

data/lib/lutaml/store/cache_store.rb

@@ -36,10 +36,12 @@ module Lutaml
 
         def to_h
           {
+            # sub-second precision: a second-truncated created_at shortens
+            # every restored TTL by up to a second (matters at small TTLs)
             value: @value,
-            created_at: @created_at.iso8601,
+            created_at: @created_at.iso8601(6),
             ttl: @ttl,
-            expires_at: expires_at&.iso8601,
+            expires_at: expires_at&.iso8601(6),
             metadata: @metadata
           }
         end

data/lib/lutaml/store/version.rb

@@ -2,6 +2,6 @@
 
 module Lutaml
   module Store
-    VERSION = "0.3.2"
+    VERSION = "0.3.4"
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: lutaml-store
 version: !ruby/object:Gem::Version
-  version: 0.3.2
+  version: 0.3.4
 platform: ruby
 authors:
 - Ronald Tse
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2026-09-29 00:00:00.000000000 Z
+date: 2026-11-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: lutaml-model
@@ -16,14 +16,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.8.15
+        version: 0.8.156
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.8.15
+        version: 0.8.156
 - !ruby/object:Gem::Dependency
   name: rubyzip
   requirement: !ruby/object:Gem::Requirement