lutaml-store 0.2.4 → 0.3.0

data/lib/lutaml/store/mirror.rb

@@ -0,0 +1,94 @@
+# frozen_string_literal: true
+
+require "cgi"
+require "fileutils"
+require "json"
+
+module Lutaml
+  module Store
+    # Pulls a source into a local package directory — the GCR-style shape:
+    #
+    #   <into>[/<collection>]/
+    #     manifest.json
+    #     entries/<percent-encoded-key>
+    #
+    # The result is readable by Source::Directory, is the local-cache layout
+    # Repository uses, and is byte-compatible with the TypeScript
+    # implementation's LocalStore (conformance fixtures pin this).
+    #
+    # Pulls are incremental and digest-verified: entries whose local sha256
+    # already matches are not rewritten; a mismatched local entry is
+    # re-fetched. Digest mismatches against the source manifest are hard
+    # errors — never silently accepted.
+    module Mirror
+      class IntegrityError < Error; end
+
+      class << self
+        # @param source [Source::Base]
+        # @param into [String] directory to place the package in
+        # @param collection [String, nil] optional subdirectory of +into+
+        # @param force [Boolean] re-write entries even when digests match
+        # @return [Source::Directory] a source over the written package
+        def pull(source, into:, collection: nil, force: false)
+          manifest = source.manifest
+          root = collection ? ::File.join(into, collection) : into
+          entries_dir = ::File.join(root, "entries")
+          FileUtils.mkdir_p(entries_dir)
+
+          pulled = 0
+          entries = manifest.entries.map do |entry|
+            body = source.read(entry.key)
+            unless entry.matches?(body)
+              raise IntegrityError,
+                    "digest mismatch for #{entry.key.inspect}: manifest declares " \
+                    "#{entry.digest}, source returned #{entry.digest_for(body)}"
+            end
+
+            location = entry.location || "entries/#{Source.encode_key(entry.key)}"
+            path = ::File.join(root, location)
+            write_if_changed(path, body, force)
+            pulled += 1
+            Manifest::Entry.new(
+              key: entry.key, location: location,
+              digest: entry.digest_for(body), shard: entry.shard,
+              metadata: entry.metadata
+            )
+          end
+
+          local = Manifest.build(entries, shards: manifest.shards)
+          ::File.write(::File.join(root, "manifest.json"), JSON.pretty_generate(local.to_hash))
+
+          Source.for(:directory, path: root)
+        end
+
+        # Packs an existing package directory (as written by .pull) into a
+        # distributable .zip — the "downloaded official package" artifact.
+        # Source::Zip reads it back with no unpacking step.
+        #
+        # @param package_dir [String] a directory containing manifest.json + entries/
+        # @param to [String] the .zip path to write
+        # @return [String] the path written
+        def pack(package_dir, to:)
+          require "zip"
+          FileUtils.mkdir_p(::File.dirname(to))
+          ::Zip::File.open(to, create: true) do |zip|
+            Dir["#{package_dir}/**/*"].sort.each do |path|
+              next if ::File.directory?(path)
+
+              zip.add(path.delete_prefix("#{package_dir}/"), path)
+            end
+          end
+          to
+        end
+
+        private
+
+        def write_if_changed(path, body, force)
+          return if !force && ::File.file?(path) && ::File.binread(path) == body
+
+          ::File.binwrite(path, body)
+        end
+      end
+    end
+  end
+end

data/lib/lutaml/store/repository.rb

@@ -0,0 +1,178 @@
+# frozen_string_literal: true
+
+require "cgi"
+require "digest"
+require "fileutils"
+require "json"
+
+module Lutaml
+  module Store
+    # The uniform consumer facade: one read API whether the repository is a
+    # local package, a downloaded distribution, a temp-folder cache, or the
+    # cloud API. Backends are chosen by explicit composition — the
+    # repository never guesses, never falls back, and never hides which
+    # layer answered.
+    #
+    #   # cloud, with an explicit local package cache
+    #   repo = Repository.new(
+    #     source:   Source.for(:rest, base_url: "https://api.relaton.org",
+    #                                collection: "ietf"),
+    #     cache:    Source.for(:directory, path: "~/.cache/relaton/ietf"),
+    #   )
+    #
+    #   # a downloaded official package (zip) — identical read API
+    #   repo = Repository.new(source: Source.for(:zip, path: "ietf.zip"))
+    #
+    #   # a local GCR-style checkout
+    #   repo = Repository.new(source: Source.for(:directory, path: "~/gcr/ietf"))
+    #
+    # Modes are explicit:
+    #   :online  — read cache first, then the source (default)
+    #   :offline — never touch the source; absent keys raise NotFoundError
+    class Repository
+      MODES = %i[online offline].freeze
+
+      # @param source [Source::Base] the authoritative repository
+      # @param cache [Source::Directory, nil] explicit read-through local
+      #   package (same layout as Mirror#pull writes); nil disables caching
+      # @param mode [Symbol] :online or :offline
+      def initialize(source:, cache: nil, mode: :online)
+        raise ConfigurationError, "mode must be one of #{MODES.inspect}" unless MODES.include?(mode)
+        raise ConfigurationError, "offline mode requires a cache source" if mode == :offline && cache.nil?
+
+        @source = source
+        @cache = cache
+        @mode = mode
+      end
+
+      # Read-through convenience constructors — still explicit about every
+      # backend, just shorter.
+      def self.for_cloud(base_url:, collection:, cache: nil, **source_options)
+        new(source: Source.for(:rest, base_url: base_url, collection: collection,
+                                      **source_options), cache: cache)
+      end
+
+      def self.for_package(path, type: :directory, **source_options)
+        new(source: Source.for(type, path: path, **source_options))
+      end
+
+      attr_reader :source, :cache, :mode
+
+      # @return [String] the record's bytes
+      def read(key)
+        if @cache
+          begin
+            return @cache.read(key)
+          rescue NotFoundError
+            raise if @mode == :offline
+          end
+          body = @source.read(key)
+          write_cache_entry(key, body)
+          return body
+        end
+
+        @source.read(key)
+      end
+
+      # @return the model instance built by the consumer's model class
+      def get(key, model_class)
+        format_for(key).deserialize(read(key), model_class)
+      end
+
+      def exist?(key)
+        read(key)
+        true
+      rescue NotFoundError
+        false
+      end
+
+      def keys
+        @source.keys
+      end
+
+      def each_key(&block)
+        keys.each(&block)
+      end
+
+      def manifest
+        @source.manifest
+      end
+
+      # Filters the collection's manifest entries by metadata key/values.
+      # The store is field-agnostic: it matches the literal metadata hash —
+      # the domain (relaton pubid, Glossarist concepts) narrows semantics.
+      #
+      #   repo.search(doctype: "rfc", stream: "IETF")
+      #   repo.search(docid: "RFC 7231")
+      #
+      # @param filter [Hash<String=>String>] ALL key/values must match
+      # @return [Array<Manifest::Entry>] matching entries (empty if none)
+      def search(**filter)
+        return [] if filter.empty?
+
+        manifest.entries.select do |entry|
+          filter.all? do |k, v|
+            mv = entry.metadata[k.to_s]
+            mv.is_a?(String) ? mv.casecmp?(v.to_s) : mv == v
+          end
+        end
+      end
+
+      # Mirror the whole source into a local package (the cache layout).
+      # Returns a Repository over the written package.
+      def pull!(into:, collection: nil, force: false)
+        pulled = Mirror.pull(@source, into: into, collection: collection, force: force)
+        Repository.new(source: pulled, mode: @mode)
+      end
+
+      private
+
+      # The write-through target is a Directory package (the Mirror layout);
+      # anything else is a configuration error, not a duck-type guess. The
+      # package manifest is maintained on every write so later reads —
+      # including reference resolution and offline mode — see the entry.
+      def write_cache_entry(key, body)
+        unless @cache.is_a?(Source::Directory)
+          raise ConfigurationError,
+                "cache must be a Source::Directory package, got #{@cache.class}"
+        end
+
+        location = "entries/#{Source.encode_key(key)}"
+        FileUtils.mkdir_p(::File.join(@cache.package_root, "entries"))
+        ::File.binwrite(::File.join(@cache.package_root, location), body)
+
+        manifest = begin
+          @cache.manifest
+        rescue NotFoundError
+          Lutaml::Store::Manifest.build([])
+        end
+        # The source manifest's metadata (e.g. the domain docid) rides
+        # along — offline reference resolution depends on it.
+        declared = begin
+          @source.manifest.entry_for(key)
+        rescue StandardError
+          nil
+        end
+        entry = Manifest::Entry.new(
+          key: key, location: location,
+          digest: "sha256:#{Digest::SHA256.hexdigest(body)}",
+          metadata: declared&.metadata || {}
+        )
+        entries = manifest.entries.reject { |e| e.key == key } + [entry]
+        updated = Manifest.build(entries, shards: manifest.shards)
+        ::File.write(
+          ::File.join(@cache.package_root, "manifest.json"),
+          JSON.pretty_generate(updated.to_hash)
+        )
+      end
+
+      def format_for(key)
+        fmt = Format.for_extension(::File.extname(key.to_s))
+        fmt ||= @source.options[:default_format] &&
+                Format.resolve(@source.options[:default_format])
+        fmt || raise(ConfigurationError,
+                     "cannot determine the format of #{key.inspect}: pass default_format:")
+      end
+    end
+  end
+end

data/lib/lutaml/store/source/base.rb

@@ -0,0 +1,137 @@
+# frozen_string_literal: true
+
+module Lutaml
+  module Store
+    module Source
+      # Contract shared by every source.
+      #
+      # Subclasses implement {#read} (bytes for one key, raising
+      # NotFoundError for a definitive miss and BackendError for transport
+      # trouble) and {#manifest_raw} (the manifest document). Everything
+      # else — keys, existence, typed reads — composes those two.
+      class Base
+        OPTIONS = [].freeze
+
+        attr_reader :options
+
+        def initialize(**options)
+          @options = options
+          missing = (self.class::OPTIONS - options.keys)
+          unless missing.empty?
+            raise ConfigurationError,
+                  "#{self.class.name.split("::").last.downcase} source requires: " \
+                  "#{missing.map(&:inspect).join(", ")}"
+          end
+          configure
+        end
+
+        # Bytes of one record. Raises NotFoundError (definitive) or
+        # BackendError (transport). Never returns nil.
+        #
+        # @param key [String]
+        # @return [String]
+        def read(_key)
+          raise NotImplementedError, "#{self.class}#read"
+        end
+
+        # The manifest document as a string.
+        #
+        # @return [String]
+        def manifest_raw
+          raise NotImplementedError, "#{self.class}#manifest_raw"
+        end
+
+        # The parsed manifest, memoized.
+        #
+        # @return [Manifest]
+        def manifest
+          @manifest ||= Manifest.parse(manifest_raw, format: manifest_format)
+        end
+
+        # All keys declared by the manifest. There is deliberately no
+        # fallback listing: an unenumerable source raises ConfigurationError
+        # instead of guessing.
+        #
+        # @return [Array<String>]
+        def keys
+          manifest.keys
+        end
+
+        # Manifest entry for one key.
+        #
+        # @param key [String]
+        # @return [Manifest::Entry, nil]
+        def entry_for(key)
+          manifest.entry_for(key)
+        end
+
+        def each_key(&block)
+          return to_enum(:each_key) unless block
+
+          keys.each(&block)
+        end
+
+        # Existence via a full read. A HEAD-based optimization may be
+        # provided per source; the default is honest and uniform.
+        #
+        # @param key [String]
+        # @return [Boolean]
+        def exist?(key)
+          read(key)
+          true
+        rescue NotFoundError
+          false
+        end
+
+        # Deserialized record using the consumer's model class — the model
+        # is declared by the caller, never inferred.
+        #
+        # @param key [String]
+        # @param model_class [Class] a Lutaml::Model::Serializable subclass
+        # @return the model instance
+        def get(key, model_class)
+          format_for(key).deserialize(read(key), model_class)
+        end
+
+        # Location of one key relative to the source. Manifest-declared
+        # locations win; otherwise the single-segment convention
+        # "entries/<percent-encoded key>" applies.
+        #
+        # @param key [String]
+        # @return [String]
+        def path_for(key)
+          entry = manifest.entry_for(key) if manifestable?
+          return entry.location if entry&.location
+
+          "entries/#{Source.encode_key(key)}"
+        end
+
+        private
+
+        def manifestable?
+          manifest
+          true
+        rescue NotImplementedError, BackendError, NotFoundError
+          false
+        end
+
+        def manifest_format
+          @options[:manifest_format] || :json
+        end
+
+        def format_for(key)
+          fmt = Format.for_extension(::File.extname(path_for(key).to_s))
+          fmt ||= Format.for_extension(::File.extname(key.to_s))
+          fmt ||= @options[:default_format] &&
+                  Format.resolve(@options[:default_format])
+          fmt || raise(ConfigurationError,
+                       "cannot determine the format of #{key.inspect}: pass default_format:")
+        end
+
+        def require_option(name)
+          @options[name] || raise(ConfigurationError, "#{name} is required")
+        end
+      end
+    end
+  end
+end

data/lib/lutaml/store/source/directory.rb

@@ -0,0 +1,49 @@
+# frozen_string_literal: true
+
+require "cgi"
+require "fileutils"
+
+module Lutaml
+  module Store
+    module Source
+      # A local package directory: `manifest.json` + `entries/`. This is the
+      # layout {Mirror}#pull writes and what a downloaded official package
+      # (GCR-style) ships — one reader for caches, distributions and
+      # checkouts.
+      class Directory < Base
+        OPTIONS = %i[path].freeze
+
+        attr_reader :package_root
+
+        private
+
+        def configure
+          @path = require_option(:path)
+          @package_root = @path
+          # A read-only source never writes - not even directory creation.
+          # A missing path simply holds no entries and no manifest.
+        end
+
+        def entry_path(key)
+          File.join(@path, path_for(key))
+        end
+
+        public
+
+        def read(key)
+          File.binread(entry_path(key))
+        rescue Errno::ENOENT
+          raise NotFoundError, "no entry #{key.inspect} in #{@path}"
+        rescue SystemCallError => e
+          raise BackendError, "cannot read #{key.inspect} in #{@path}: #{e.message}"
+        end
+
+        def manifest_raw
+          File.binread(File.join(@path, "manifest.json"))
+        rescue Errno::ENOENT
+          raise NotFoundError, "no manifest.json in #{@path}"
+        end
+      end
+    end
+  end
+end

data/lib/lutaml/store/source/https.rb

@@ -0,0 +1,109 @@
+# frozen_string_literal: true
+
+require "json"
+require "net/http"
+require "openssl"
+require "uri"
+
+module Lutaml
+  module Store
+    module Source
+      # Any static HTTP host — GitHub Pages, raw.githubusercontent, an
+      # object-storage bucket. The consumer states the base URL; the source
+      # appends one percent-encoded path segment per key.
+      #
+      # Every read may be wrapped in an explicit `cache:` (an
+      # HttpCacheConfig hash) so ETag/304 revalidation is handled by the
+      # store's HttpCache. With no cache configured, reads go straight to
+      # the network — nothing is implied.
+      #
+      # `transport:` accepts a callable for tests and alternative HTTP
+      # stacks: `->(uri, headers) { {status_code:, headers:, body:} }`.
+      class Https < Base
+        OPTIONS = %i[base_url].freeze
+
+        RETRIED_ERRORS = [
+          SocketError, Timeout::Error, IOError, SystemCallError,
+          OpenSSL::SSL::SSLError, Net::HTTPBadResponse, Net::HTTPHeaderSyntaxError,
+          Net::ProtocolError
+        ].freeze
+
+        private
+
+        def configure
+          base = require_option(:base_url)
+          @base = URI.parse(base.to_s)
+          raise ConfigurationError, "base_url must be http(s)" unless @base.is_a?(URI::HTTP) ||
+                                                                      @base.is_a?(URI::HTTPS)
+
+          @headers = @options.fetch(:headers, {})
+          @timeout = @options.fetch(:timeout, 60)
+          @open_timeout = @options.fetch(:open_timeout, 60)
+          @max_redirects = @options.fetch(:max_redirects, 3)
+          @transport = @options[:transport]
+          @cache = @options[:cache] && HttpCache.new(@options[:cache])
+        end
+
+        def fetch(uri, headers = {}, redirects = @max_redirects)
+          response = raw_fetch(uri, headers)
+          case response[:status_code]
+          when 200..299 then response
+          when 301, 302, 307, 308
+            raise BackendError, "too many redirects fetching #{uri}" if redirects.zero?
+
+            fetch(URI.parse(response[:headers]["location"]), headers, redirects - 1)
+          when 404 then raise NotFoundError, "no entry at #{uri}"
+          else raise BackendError, "HTTP #{response[:status_code]} fetching #{uri}"
+          end
+        end
+
+        def raw_fetch(uri, headers)
+          if @transport
+            begin
+              return @transport.call(uri, headers)
+            rescue *RETRIED_ERRORS => e
+              raise BackendError, "cannot fetch #{uri}: #{e.class}: #{e.message}"
+            end
+          end
+
+          if @cache
+            return @cache.fetch(:get, uri.to_s, headers) do |h|
+              http_get(uri, h)
+            end
+          end
+
+          http_get(uri, headers)
+        end
+
+        def http_get(uri, headers)
+          http = Net::HTTP.new(uri.host, uri.port)
+          http.use_ssl = uri.is_a?(URI::HTTPS)
+          http.open_timeout = @open_timeout
+          http.read_timeout = @timeout
+          begin
+            resp = http.get(uri, @headers.merge(headers))
+            { status_code: resp.code.to_i, headers: resp.each_header.to_h, body: resp.body }
+          rescue *RETRIED_ERRORS => e
+            raise BackendError, "cannot fetch #{uri}: #{e.class}: #{e.message}"
+          end
+        end
+
+        def url_for(relative_path)
+          base = @base.to_s.sub(%r{/+\z}, "")
+          path = relative_path.sub(%r{\A/+}, "")
+          URI.parse("#{base}/#{path}")
+        end
+
+        public
+
+        def read(key)
+          fetch(url_for(Source.encode_key(key)))[:body]
+        end
+
+        def manifest_raw
+          fetch(url_for(@options.fetch(:manifest_path, "manifest.json")))[:body]
+        end
+      end
+    end
+  end
+end

data/lib/lutaml/store/source/rest.rb

@@ -0,0 +1,61 @@
+# frozen_string_literal: true
+
+require "json"
+
+module Lutaml
+  module Store
+    module Source
+      # A repository served by the lutaml cloud store API (the contract
+      # distilled from api.relaton.org):
+      #
+      #   GET {base}/collections                          → collection list
+      #   GET {base}/collections/{c}/manifest             → Manifest
+      #   GET {base}/collections/{c}/entries/{key}        → record
+      #
+      # 404 is a definitive "no such key/collection" (NotFoundError); every
+      # other non-2xx is a BackendError. Path shapes are fixed here so
+      # server implementers have one contract — not per-client guesswork.
+      class Rest < Https
+        OPTIONS = %i[base_url collection].freeze
+
+        private
+
+        def configure
+          super
+          @collection = require_option(:collection)
+        end
+
+        def collection_path(suffix)
+          "collections/#{Source.encode_key(@collection)}#{suffix}"
+        end
+
+        public
+
+        def read(key)
+          fetch(url_for(collection_path("/entries/#{Source.encode_key(key)}")))[:body]
+        end
+
+        def manifest_raw
+          fetch(url_for(collection_path("/manifest")))[:body]
+        end
+
+        # All collections the API publishes.
+        #
+        # @return [Array<Hash>]
+        def collections
+          JSON.parse(fetch(url_for("collections"))[:body])["collections"]
+        end
+
+        # The keys of one declared shard, when the collection's manifest
+        # advertises sharding. Shard-number semantics belong to the domain;
+        # this only fetches the named part.
+        #
+        # @param number [Integer]
+        # @return [Array<String>]
+        def shard(number)
+          JSON.parse(fetch(url_for(collection_path("/shards/#{Integer(number)}")))[:body])["keys"]
+        end
+      end
+    end
+  end
+end

data/lib/lutaml/store/source/zip.rb

@@ -0,0 +1,52 @@
+# frozen_string_literal: true
+
+require "zip"
+
+module Lutaml
+  module Store
+    module Source
+      # A packaged repository read in place from a .zip — the shape of a
+      # downloaded official package. Same internal layout as
+      # {Directory}: manifest.json + entries/.
+      class Zip < Base
+        OPTIONS = %i[path].freeze
+
+        private
+
+        def configure
+          @path = require_option(:path)
+          return if File.file?(@path)
+
+          raise ConfigurationError, "no such package file: #{@path}"
+        end
+
+        def with_zip(&block)
+          ::Zip::File.open(@path, &block)
+        rescue ::Zip::Error => e
+          raise BackendError, "cannot open package #{@path}: #{e.message}"
+        end
+
+        public
+
+        def read(key)
+          inner = path_for(key)
+          with_zip do |zip|
+            entry = zip.find_entry(inner)
+            raise NotFoundError, "no entry #{key.inspect} in #{::File.basename(@path)}" unless entry
+
+            entry.get_input_stream.read
+          end
+        end
+
+        def manifest_raw
+          with_zip do |zip|
+            entry = zip.find_entry("manifest.json")
+            raise NotFoundError, "no manifest.json in #{::File.basename(@path)}" unless entry
+
+            entry.get_input_stream.read
+          end
+        end
+      end
+    end
+  end
+end