lutaml-store 0.2.4 → 0.3.1

data/lib/lutaml/store/cache_store.rb

@@ -1,11 +1,16 @@
 # frozen_string_literal: true
 
 require "json"
+require "monitor"
 require "time"
 
 module Lutaml
   module Store
     # TTL-aware cache store with LRU eviction. Wraps a storage adapter directly.
+    #
+    # All public operations run under one re-entrant lock, so the LRU and
+    # cleanup bookkeeping is safe across threads. Across processes only the
+    # adapter's own guarantees apply.
     class CacheStore
       class CacheEntry
         attr_reader :value, :created_at, :ttl, :metadata
@@ -58,66 +63,75 @@ module Lutaml
         @cleanup_interval = config[:cleanup_interval] || 300
         @last_cleanup = Time.now
         @access_times = {}
+        @lock = ::Monitor.new
       end
 
       def get(key)
-        cleanup_expired if should_cleanup?
+        @lock.synchronize do
+          cleanup_if_due
 
-        entry_data = @adapter.get(key)
-        return nil unless entry_data
+          entry_data = @adapter.get(key)
+          next nil unless entry_data
 
-        begin
-          entry = deserialize_entry(entry_data)
+          begin
+            entry = deserialize_entry(entry_data)
+
+            if entry.expired?
+              delete(key)
+              next nil
+            end
 
-          if entry.expired?
+            @access_times[key] = Time.now
+            entry.value
+          rescue StandardError
             delete(key)
-            return nil
+            nil
           end
-
-          @access_times[key] = Time.now
-          entry.value
-        rescue StandardError
-          delete(key)
-          nil
         end
       end
 
       def set(key, value, ttl: :default, metadata: {})
-        cleanup_expired if should_cleanup?
-        evict_if_needed
+        @lock.synchronize do
+          cleanup_if_due
+          evict_if_needed
 
-        effective_ttl = ttl == :default ? @default_ttl : ttl
-        entry = CacheEntry.new(value, ttl: effective_ttl, metadata: metadata)
+          effective_ttl = ttl == :default ? @default_ttl : ttl
+          entry = CacheEntry.new(value, ttl: effective_ttl, metadata: metadata)
 
-        serialized_entry = serialize_entry(entry)
-        @adapter.set(key, serialized_entry)
+          serialized_entry = serialize_entry(entry)
+          @adapter.set(key, serialized_entry)
 
-        @access_times[key] = Time.now
-        value
+          @access_times[key] = Time.now
+          value
+        end
       end
 
       def delete(key)
-        value = nil
-        entry_data = @adapter.get(key)
-        if entry_data
-          begin
-            entry = deserialize_entry(entry_data)
-            value = entry.value unless entry.expired?
-          rescue StandardError
-            # If we can't deserialize, treat as nil
+        @lock.synchronize do
+          value = nil
+          entry_data = @adapter.get(key)
+          if entry_data
+            begin
+              entry = deserialize_entry(entry_data)
+              value = entry.value unless entry.expired?
+            rescue StandardError
+              # If we can't deserialize, treat as nil
+            end
           end
-        end
 
-        @access_times.delete(key)
+          @access_times.delete(key)
 
-        deleted = @adapter.delete(key)
+          deleted = @adapter.delete(key)
 
-        deleted ? value : nil
+          deleted ? value : nil
+        end
       end
 
       def clear
-        @access_times.clear
-        @adapter.clear
+        @lock.synchronize do
+          @access_times.clear
+          @adapter.clear
+        end
       end
 
       def exists?(key)
@@ -135,12 +149,13 @@ module Lutaml
       end
 
       def keys
-        cleanup_expired if should_cleanup?
-        @adapter.keys.select { |key| exists?(key) }
+        @lock.synchronize do
+          cleanup_if_due
+          @adapter.keys.select { |key| exists?(key) }
+        end
       end
 
       def size
-        cleanup_expired if should_cleanup?
         keys.size
       end
 
@@ -169,22 +184,7 @@ module Lutaml
       end
 
       def cleanup_expired
-        expired_keys = []
-
-        @adapter.each_key do |key|
-          entry_data = @adapter.get(key)
-          next unless entry_data
-
-          entry = deserialize_entry(entry_data)
-          expired_keys << key if entry.expired?
-        rescue StandardError
-          expired_keys << key
-        end
-
-        expired_keys.each { |key| delete(key) }
-        @last_cleanup = Time.now
-
-        expired_keys.size
+        @lock.synchronize { cleanup_expired_entries }
       end
 
       def cache_info
@@ -203,23 +203,25 @@ module Lutaml
       end
 
       def touch(key, ttl: nil)
-        entry_data = @adapter.get(key)
-        return false unless entry_data
+        @lock.synchronize do
+          entry_data = @adapter.get(key)
+          next false unless entry_data
 
-        begin
-          entry = deserialize_entry(entry_data)
-          return false if entry.expired?
+          begin
+            entry = deserialize_entry(entry_data)
+            next false if entry.expired?
 
-          new_ttl = ttl || entry.ttl
-          new_entry = CacheEntry.new(entry.value, ttl: new_ttl, metadata: entry.metadata)
+            new_ttl = ttl || entry.ttl
+            new_entry = CacheEntry.new(entry.value, ttl: new_ttl, metadata: entry.metadata)
 
-          serialized_entry = serialize_entry(new_entry)
-          @adapter.set(key, serialized_entry)
+            serialized_entry = serialize_entry(new_entry)
+            @adapter.set(key, serialized_entry)
 
-          @access_times[key] = Time.now
-          true
-        rescue StandardError
-          false
+            @access_times[key] = Time.now
+            true
+          rescue StandardError
+            false
+          end
         end
       end
 
@@ -263,14 +265,45 @@ module Lutaml
         Time.now - @last_cleanup > @cleanup_interval
       end
 
+      # Caller holds @lock.
+      def cleanup_if_due
+        cleanup_expired_entries if should_cleanup?
+      end
+
+      # Caller holds @lock. @last_cleanup is set first, so a nested call
+      # (through #delete) does not start a second scan.
+      def cleanup_expired_entries
+        @last_cleanup = Time.now
+        expired_keys = []
+
+        @adapter.each_key do |key|
+          entry_data = @adapter.get(key)
+          next unless entry_data
+
+          entry = deserialize_entry(entry_data)
+          expired_keys << key if entry.expired?
+        rescue StandardError
+          expired_keys << key
+        end
+
+        expired_keys.each { |key| delete(key) }
+
+        expired_keys.size
+      end
+
+      # Caller holds @lock. Keys this process never touched count as the
+      # least recently used.
       def evict_if_needed
         return unless @max_size
-        return if size < @max_size
 
-        keys_by_access = @access_times.sort_by { |_, time| time }.map(&:first)
-        keys_to_evict = keys_by_access.first(size - @max_size + 1)
+        stored_keys = @adapter.keys
+        overflow = stored_keys.size - @max_size + 1
+        return unless overflow.positive?
+
+        untouched = stored_keys.reject { |key| @access_times.key?(key) }
+        by_access = @access_times.sort_by { |_, time| time }.map(&:first)
 
-        keys_to_evict.each { |key| delete(key) }
+        (untouched + by_access).first(overflow).each { |key| delete(key) }
       end
     end
   end

data/lib/lutaml/store/config.rb

@@ -14,7 +14,7 @@ module Lutaml
                      cache: {}, monitoring: {}, events: {},
                      compression: {}, serialization: {}, **)
         @adapter_type = normalize_adapter_type(adapter_type)
-        @adapter_options = symbolize_keys(adapter_options)
+        @adapter_options = inline_adapter_options(adapter_type).merge(symbolize_keys(adapter_options))
 
         cache_config = symbolize_keys(cache)
         @cache_enabled = cache_config.fetch(:enabled, true)
@@ -134,6 +134,18 @@ module Lutaml
         end
       end
 
+      # Options given inline with the type, as in
+      # `adapter: { type: :filesystem, path: "./data" }` or
+      # `adapter: { type: :sqlite, options: { path: "a.db" } }`.
+      def inline_adapter_options(adapter_type)
+        return {} unless adapter_type.is_a?(Hash)
+
+        inline = symbolize_keys(adapter_type)
+        nested = inline.delete(:options) || {}
+        inline.delete(:type)
+        inline.merge(nested)
+      end
+
       def validate_adapter_config
         valid_adapters = %i[memory filesystem sqlite]
         unless valid_adapters.include?(@adapter_type)

data/lib/lutaml/store/format.rb

@@ -31,6 +31,19 @@ module Lutaml
         const_get(entry).new
       end
 
+      # Self-describing content: XML documents start with "<?xml" or "<",
+      # JSON with "{" or "["; everything else is YAML, the ecosystem's
+      # default. Extensionless records (object-storage keys carry no
+      # extension) declare their format this way instead of by guesswork
+      # out of band.
+      def self.guess(data)
+        stripped = data.to_s.lstrip
+        return :xml if stripped.start_with?("<?xml", "<")
+        return :json if stripped.start_with?("{", "[")
+
+        :yaml
+      end
+
       def self.for_extension(ext)
         extension_map[ext] || extension_map[".#{ext.to_s.sub(/\A\./, "")}"]
       end

data/lib/lutaml/store/manifest.rb

@@ -0,0 +1,85 @@
+# frozen_string_literal: true
+
+require "lutaml/model"
+require "digest"
+
+module Lutaml
+  module Store
+    # The enumeration document of a LutaML data repository: which keys
+    # exist, where each one lives relative to the source root, and the
+    # sha256 digest to verify it against.
+    #
+    # Sharding metadata (`shards`, `Entry#shard`) is carried but never
+    # interpreted: computing "which shard holds this key" is domain logic
+    # (e.g. relaton's crc32 over the pubid root number). The store only
+    # enumerates.
+    class Manifest
+      include Lutaml::Model::Serialize
+
+      VERSION = 1
+
+      class Entry
+        include Lutaml::Model::Serialize
+
+        attribute :key, :string
+        attribute :location, :string
+        attribute :digest, :string
+        attribute :shard, :integer
+        attribute :metadata, :hash, default: {}
+
+        def digest_for(body)
+          "sha256:#{Digest::SHA256.hexdigest(body)}"
+        end
+
+        def matches?(body)
+          return true unless digest
+
+          digest == digest_for(body)
+        end
+      end
+
+      attribute :version, :integer, default: VERSION
+      attribute :generated, :string
+      attribute :count, :integer, default: 0
+      attribute :shards, :integer, default: 0
+      attribute :entries, Manifest::Entry, collection: true, initialize_empty: true
+
+      # @param text [String]
+      # @param format [Symbol] :json or :yaml
+      # @return [Manifest]
+      def self.parse(text, format: :json)
+        case format.to_sym
+        when :json then from_json(text)
+        when :yaml then from_yaml(text)
+        else raise ConfigurationError, "unsupported manifest format: #{format}"
+        end
+      end
+
+      def self.build(entries, generated: nil, shards: 0, version: VERSION)
+        new(
+          version: version,
+          generated: (generated || Time.now.utc).iso8601,
+          count: entries.size,
+          shards: shards,
+          entries: entries
+        )
+      end
+
+      def keys
+        entries.map(&:key)
+      end
+
+      def entry_for(key)
+        entries.find { |e| e.key == key }
+      end
+
+      def key?(key)
+        !entry_for(key).nil?
+      end
+
+      def shard_of_declared?
+        shards.positive?
+      end
+    end
+  end
+end

data/lib/lutaml/store/mirror.rb

@@ -0,0 +1,94 @@
+# frozen_string_literal: true
+
+require "cgi"
+require "fileutils"
+require "json"
+
+module Lutaml
+  module Store
+    # Pulls a source into a local package directory — the GCR-style shape:
+    #
+    #   <into>[/<collection>]/
+    #     manifest.json
+    #     entries/<percent-encoded-key>
+    #
+    # The result is readable by Source::Directory, is the local-cache layout
+    # Repository uses, and is byte-compatible with the TypeScript
+    # implementation's LocalStore (conformance fixtures pin this).
+    #
+    # Pulls are incremental and digest-verified: entries whose local sha256
+    # already matches are not rewritten; a mismatched local entry is
+    # re-fetched. Digest mismatches against the source manifest are hard
+    # errors — never silently accepted.
+    module Mirror
+      class IntegrityError < Error; end
+
+      class << self
+        # @param source [Source::Base]
+        # @param into [String] directory to place the package in
+        # @param collection [String, nil] optional subdirectory of +into+
+        # @param force [Boolean] re-write entries even when digests match
+        # @return [Source::Directory] a source over the written package
+        def pull(source, into:, collection: nil, force: false)
+          manifest = source.manifest
+          root = collection ? ::File.join(into, collection) : into
+          entries_dir = ::File.join(root, "entries")
+          FileUtils.mkdir_p(entries_dir)
+
+          pulled = 0
+          entries = manifest.entries.map do |entry|
+            body = source.read(entry.key)
+            unless entry.matches?(body)
+              raise IntegrityError,
+                    "digest mismatch for #{entry.key.inspect}: manifest declares " \
+                    "#{entry.digest}, source returned #{entry.digest_for(body)}"
+            end
+
+            location = entry.location || "entries/#{Source.encode_key(entry.key)}"
+            path = ::File.join(root, location)
+            write_if_changed(path, body, force)
+            pulled += 1
+            Manifest::Entry.new(
+              key: entry.key, location: location,
+              digest: entry.digest_for(body), shard: entry.shard,
+              metadata: entry.metadata
+            )
+          end
+
+          local = Manifest.build(entries, shards: manifest.shards)
+          ::File.write(::File.join(root, "manifest.json"), JSON.pretty_generate(local.to_hash))
+
+          Source.for(:directory, path: root)
+        end
+
+        # Packs an existing package directory (as written by .pull) into a
+        # distributable .zip — the "downloaded official package" artifact.
+        # Source::Zip reads it back with no unpacking step.
+        #
+        # @param package_dir [String] a directory containing manifest.json + entries/
+        # @param to [String] the .zip path to write
+        # @return [String] the path written
+        def pack(package_dir, to:)
+          require "zip"
+          FileUtils.mkdir_p(::File.dirname(to))
+          ::Zip::File.open(to, create: true) do |zip|
+            Dir["#{package_dir}/**/*"].sort.each do |path|
+              next if ::File.directory?(path)
+
+              zip.add(path.delete_prefix("#{package_dir}/"), path)
+            end
+          end
+          to
+        end
+
+        private
+
+        def write_if_changed(path, body, force)
+          return if !force && ::File.file?(path) && ::File.binread(path) == body
+
+          ::File.binwrite(path, body)
+        end
+      end
+    end
+  end
+end

data/lib/lutaml/store/repository.rb

@@ -0,0 +1,178 @@
+# frozen_string_literal: true
+
+require "cgi"
+require "digest"
+require "fileutils"
+require "json"
+
+module Lutaml
+  module Store
+    # The uniform consumer facade: one read API whether the repository is a
+    # local package, a downloaded distribution, a temp-folder cache, or the
+    # cloud API. Backends are chosen by explicit composition — the
+    # repository never guesses, never falls back, and never hides which
+    # layer answered.
+    #
+    #   # cloud, with an explicit local package cache
+    #   repo = Repository.new(
+    #     source:   Source.for(:rest, base_url: "https://api.relaton.org",
+    #                                collection: "ietf"),
+    #     cache:    Source.for(:directory, path: "~/.cache/relaton/ietf"),
+    #   )
+    #
+    #   # a downloaded official package (zip) — identical read API
+    #   repo = Repository.new(source: Source.for(:zip, path: "ietf.zip"))
+    #
+    #   # a local GCR-style checkout
+    #   repo = Repository.new(source: Source.for(:directory, path: "~/gcr/ietf"))
+    #
+    # Modes are explicit:
+    #   :online  — read cache first, then the source (default)
+    #   :offline — never touch the source; absent keys raise NotFoundError
+    class Repository
+      MODES = %i[online offline].freeze
+
+      # @param source [Source::Base] the authoritative repository
+      # @param cache [Source::Directory, nil] explicit read-through local
+      #   package (same layout as Mirror#pull writes); nil disables caching
+      # @param mode [Symbol] :online or :offline
+      def initialize(source:, cache: nil, mode: :online)
+        raise ConfigurationError, "mode must be one of #{MODES.inspect}" unless MODES.include?(mode)
+        raise ConfigurationError, "offline mode requires a cache source" if mode == :offline && cache.nil?
+
+        @source = source
+        @cache = cache
+        @mode = mode
+      end
+
+      # Read-through convenience constructors — still explicit about every
+      # backend, just shorter.
+      def self.for_cloud(base_url:, collection:, cache: nil, **source_options)
+        new(source: Source.for(:rest, base_url: base_url, collection: collection,
+                                      **source_options), cache: cache)
+      end
+
+      def self.for_package(path, type: :directory, **source_options)
+        new(source: Source.for(type, path: path, **source_options))
+      end
+
+      attr_reader :source, :cache, :mode
+
+      # @return [String] the record's bytes
+      def read(key)
+        if @cache
+          begin
+            return @cache.read(key)
+          rescue NotFoundError
+            raise if @mode == :offline
+          end
+          body = @source.read(key)
+          write_cache_entry(key, body)
+          return body
+        end
+
+        @source.read(key)
+      end
+
+      # @return the model instance built by the consumer's model class
+      def get(key, model_class)
+        format_for(key).deserialize(read(key), model_class)
+      end
+
+      def exist?(key)
+        read(key)
+        true
+      rescue NotFoundError
+        false
+      end
+
+      def keys
+        @source.keys
+      end
+
+      def each_key(&block)
+        keys.each(&block)
+      end
+
+      def manifest
+        @source.manifest
+      end
+
+      # Filters the collection's manifest entries by metadata key/values.
+      # The store is field-agnostic: it matches the literal metadata hash —
+      # the domain (relaton pubid, Glossarist concepts) narrows semantics.
+      #
+      #   repo.search(doctype: "rfc", stream: "IETF")
+      #   repo.search(docid: "RFC 7231")
+      #
+      # @param filter [Hash<String=>String>] ALL key/values must match
+      # @return [Array<Manifest::Entry>] matching entries (empty if none)
+      def search(**filter)
+        return [] if filter.empty?
+
+        manifest.entries.select do |entry|
+          filter.all? do |k, v|
+            mv = entry.metadata[k.to_s]
+            mv.is_a?(String) ? mv.casecmp?(v.to_s) : mv == v
+          end
+        end
+      end
+
+      # Mirror the whole source into a local package (the cache layout).
+      # Returns a Repository over the written package.
+      def pull!(into:, collection: nil, force: false)
+        pulled = Mirror.pull(@source, into: into, collection: collection, force: force)
+        Repository.new(source: pulled, mode: @mode)
+      end
+
+      private
+
+      # The write-through target is a Directory package (the Mirror layout);
+      # anything else is a configuration error, not a duck-type guess. The
+      # package manifest is maintained on every write so later reads —
+      # including reference resolution and offline mode — see the entry.
+      def write_cache_entry(key, body)
+        unless @cache.is_a?(Source::Directory)
+          raise ConfigurationError,
+                "cache must be a Source::Directory package, got #{@cache.class}"
+        end
+
+        location = "entries/#{Source.encode_key(key)}"
+        FileUtils.mkdir_p(::File.join(@cache.package_root, "entries"))
+        ::File.binwrite(::File.join(@cache.package_root, location), body)
+
+        manifest = begin
+          @cache.manifest
+        rescue NotFoundError
+          Lutaml::Store::Manifest.build([])
+        end
+        # The source manifest's metadata (e.g. the domain docid) rides
+        # along — offline reference resolution depends on it.
+        declared = begin
+          @source.manifest.entry_for(key)
+        rescue StandardError
+          nil
+        end
+        entry = Manifest::Entry.new(
+          key: key, location: location,
+          digest: "sha256:#{Digest::SHA256.hexdigest(body)}",
+          metadata: declared&.metadata || {}
+        )
+        entries = manifest.entries.reject { |e| e.key == key } + [entry]
+        updated = Manifest.build(entries, shards: manifest.shards)
+        ::File.write(
+          ::File.join(@cache.package_root, "manifest.json"),
+          JSON.pretty_generate(updated.to_hash)
+        )
+      end
+
+      def format_for(key)
+        fmt = Format.for_extension(::File.extname(key.to_s))
+        fmt ||= @source.options[:default_format] &&
+                Format.resolve(@source.options[:default_format])
+        fmt || raise(ConfigurationError,
+                     "cannot determine the format of #{key.inspect}: pass default_format:")
+      end
+    end
+  end
+end