lutaml-store 0.2.2 → 0.3.0

data/lib/lutaml/store/database_store.rb

@@ -2,9 +2,10 @@
 
 module Lutaml
   module Store
-    # Store-centric API with model registry and database-style operations
+    # Store-centric API with model registry and database-style operations.
+    # Provides CRUD, querying, file I/O, transactions, and events.
     class DatabaseStore
-      attr_reader :store, :registry, :composite_handler, :attribute_updater
+      attr_reader :store, :registry, :composite_handler, :attribute_updater, :serializer
 
       def initialize(adapter:, models: [], **options)
         @store = BasicStore.new(adapter_type: adapter, **options)
@@ -12,11 +13,13 @@ module Lutaml
         @serializer = ModelSerializer.new
         @composite_handler = CompositeModelHandler.new(@registry, @store, self, serializer: @serializer)
         @attribute_updater = AttributeUpdater.new(@registry, @composite_handler)
+        @scopes = {}
 
         validate_configuration!
       end
 
-      # Save single model or array of models
+      # ── CRUD ──
+
       def save(models)
         models_array = Array(models)
         saved_models = models_array.map { |model| save_single_model(model) }
@@ -25,7 +28,6 @@ module Lutaml
         models.is_a?(Array) ? saved_models : saved_models.first
       end
 
-      # Fetch model by class and key field
       def fetch(model:, **key_params)
         registration = @registry.registration_for(model)
         key_field = registration.key_field
@@ -35,20 +37,12 @@ module Lutaml
         stored_data = find_stored_data(registration, model, key_value)
         return nil unless stored_data
 
-        model_instance = @serializer.deserialize(stored_data, model, registration)
-
-        composite_references = stored_data["_composite_models"]
-        if composite_references
-          model_instance = @composite_handler.restore_composite_models(
-            model_instance, composite_references
-          )
-        end
+        model_instance = deserialize_record(stored_data, model, registration)
 
         @store.emit_event(:model_fetch, model: model_instance, key: key_value, source: :backend)
         model_instance
       end
 
-      # Update model with attributes array or block
       def update(model:, attributes: nil, **key_params, &block)
         current_model = fetch(model: model, **key_params)
         raise ModelNotRegisteredError, "Model not found" unless current_model
@@ -75,7 +69,6 @@ module Lutaml
         updated_model
       end
 
-      # Destroy model by class and key field
       def destroy(model:, **key_params)
         registration = @registry.registration_for(model)
         key_field = registration.key_field
@@ -94,54 +87,96 @@ module Lutaml
         deleted
       end
 
-      # Query operations
+      # ── Query API ──
+
+      def query(model_class)
+        Query.new(self, model_class)
+      end
+
       def where(model:, **conditions)
-        all(model: model).select do |model_instance|
-          conditions.all? { |field, value| model_instance.public_send(field) == value }
-        end
+        query(model).where(**conditions)
       end
 
-      # Get all models of a specific type
       def all(model:)
-        registration = @registry.registration_for(model)
+        query(model)
+      end
 
-        models = []
-        @store.each_key do |storage_key|
-          parsed = StorageKey.parse(storage_key.to_s)
-          next unless parsed.class_name == model.name
+      def count(model:)
+        query(model).count
+      end
 
-          stored_data = @store.get(storage_key)
-          next unless stored_data
+      def exists?(model:, **key_params)
+        !fetch(model: model, **key_params).nil?
+      end
 
-          begin
-            model_instance = @serializer.deserialize(stored_data, model, registration)
+      def find_by(model:, **conditions)
+        query(model).find_by(**conditions)
+      end
 
-            composite_references = stored_data["_composite_models"]
-            if composite_references
-              model_instance = @composite_handler.restore_composite_models(
-                model_instance, composite_references
-              )
-            end
+      def find_by!(model:, **conditions)
+        query(model).find_by!(**conditions)
+      end
 
-            models << model_instance
-          rescue StandardError => e
-            @store.emit_event(:deserialization_error, key: storage_key, error: e)
-          end
+      # ── Transactions ──
+
+      def transaction(&block)
+        adapter.transaction(&block)
+      end
+
+      # ── Scopes ──
+
+      def scope(name, body)
+        @scopes[name.to_sym] = body
+      end
+
+      def scope_for(name)
+        @scopes[name.to_sym] || raise(ArgumentError, "Unknown scope: #{name}")
+      end
+
+      # ── Query execution (called by Query) ──
+
+      def execute_query(query)
+        registration = @registry.registration_for(query.model_class)
+
+        if custom_serializer?(registration)
+          execute_with_post_filter(query, registration)
+        else
+          execute_with_pre_filter(query, registration)
         end
+      end
 
-        models
+      def count_query(query)
+        registration = @registry.registration_for(query.model_class)
+
+        if custom_serializer?(registration)
+          execute_with_post_filter(query, registration).size
+        else
+          count_with_pre_filter(query, registration)
+        end
       end
 
-      def exists?(model:, **key_params)
-        !fetch(model: model, **key_params).nil?
+      def fetch_batch(query, after: nil, limit: 1000)
+        scan_query = Query.new(self, query.model_class)
+        raw_batch = adapter.batch_query(scan_query, after: after, limit: limit)
+        if raw_batch
+          models = deserialize_results(raw_batch, query.model_class)
+          query.predicates.empty? ? models : models.select { |m| query.predicates.all? { |p| p.match?(m) } }
+        else
+          fallback_batch(query, after: after, limit: limit)
+        end
       end
 
-      def count(model:)
-        all(model: model).size
+      def last_storage_key_from(models, model_class)
+        return nil if models.empty?
+
+        registration = @registry.registration_for(model_class)
+        last_model = models.last
+        key_value = registration.extract_key(last_model)
+        registration.generate_storage_key_from_value(key_value).to_s
       end
 
-      # Load all models of a type from a directory using format-specific serialization.
-      # Bypasses the key-value layer and reads files directly using the format handler.
+      # ── File I/O ──
+
       def load_all(model_class, path: nil, format: :yaml, layout: :separate)
         fmt = Format.resolve(format)
         dir = resolve_model_dir(model_class, path)
@@ -161,8 +196,6 @@ module Lutaml
         end
       end
 
-      # Load from directory and store into the key-value backend.
-      # Returns the loaded models and makes them available via fetch/where/all.
       def import_all(model_class, path: nil, format: :yaml, layout: :separate)
         models = load_all(model_class, path: path, format: format, layout: layout)
         models.each { |model| save(model) }
@@ -170,7 +203,6 @@ module Lutaml
         models
       end
 
-      # Save all models to a directory using format-specific serialization.
       def save_all(models, path: nil, format: :yaml, layout: :separate)
         fmt = Format.resolve(format)
         models_array = Array(models)
@@ -198,7 +230,6 @@ module Lutaml
         saved
       end
 
-      # Export models to a single file or directory.
       def export(models, path:, format: :yaml)
         fmt = Format.resolve(format)
         models_array = Array(models)
@@ -212,6 +243,8 @@ module Lutaml
         path
       end
 
+      # ── Events ──
+
       def on(event, &block)
         @store.on(event, &block)
       end
@@ -235,6 +268,226 @@ module Lutaml
 
       private
 
+      def adapter
+        @store.adapter
+      end
+
+      def custom_serializer?(registration)
+        registration&.serializer
+      end
+
+      # Pre-deserialization filtering: predicates evaluated on raw hash data.
+      # Used when the serializer stores model fields as direct hash keys.
+      def execute_with_pre_filter(query, registration)
+        adapter_results = adapter.execute_query(query)
+
+        if adapter_results
+          deserialize_results(adapter_results, query.model_class)
+        else
+          fallback_execute(query, registration)
+        end
+      end
+
+      def count_with_pre_filter(query, registration)
+        adapter_count = adapter.count_query(query)
+        return adapter_count unless adapter_count.nil?
+
+        adapter_results = adapter.execute_query(query)
+        if adapter_results
+          adapter_results.size
+        else
+          fallback_count(query, registration)
+        end
+      end
+
+      # Post-deserialization filtering: scan by class name only, deserialize
+      # everything, then filter predicates on model instances.
+      # Used when custom serializers store data in non-standard hash format.
+      def execute_with_post_filter(query, _registration)
+        scan_query = Query.new(self, query.model_class)
+        adapter_results = adapter.execute_query(scan_query)
+
+        raw_results = adapter_results || scan_all_by_class(query.model_class)
+
+        models = deserialize_results(raw_results, query.model_class)
+        results = apply_model_predicates(models, query.predicates)
+        results = apply_model_sort(results, query.orders)
+        apply_model_pagination(results, query.limit_value, query.offset_value)
+      end
+
+      def scan_all_by_class(model_class)
+        results = []
+        @store.each_key do |storage_key|
+          parsed = StorageKey.parse(storage_key.to_s)
+          next unless parsed.class_name == model_class.name
+
+          stored_data = @store.get(storage_key)
+          results << [storage_key, stored_data] if stored_data
+        rescue StandardError
+          next
+        end
+        results
+      end
+
+      def apply_model_predicates(models, predicates)
+        return models if predicates.empty?
+
+        models.select { |m| predicates.all? { |p| p.match?(m) } }
+      end
+
+      def apply_model_sort(models, orders)
+        return models if orders.empty?
+
+        models.sort do |a, b|
+          orders.reduce(0) do |cmp, o|
+            break cmp unless cmp.zero?
+
+            va = a.public_send(o.field)
+            vb = b.public_send(o.field)
+
+            cmp_val = if va.nil? && vb.nil?
+                        0
+                      elsif va.nil?
+                        1
+                      elsif vb.nil?
+                        -1
+                      else
+                        va <=> vb || 0
+                      end
+
+            o.direction == :desc ? -cmp_val : cmp_val
+          end
+        end
+      end
+
+      def apply_model_pagination(models, limit, offset)
+        start = offset || 0
+        models = models[start..] || []
+        models = models.first(limit) if limit
+        models
+      end
+
+      def deserialize_record(hash_data, model_class, registration)
+        model_instance = @serializer.deserialize(hash_data, model_class, registration)
+
+        composite_references = hash_data["_composite_models"]
+        if composite_references
+          model_instance = @composite_handler.restore_composite_models(
+            model_instance, composite_references
+          )
+        end
+
+        model_instance
+      end
+
+      def deserialize_results(raw_results, model_class)
+        registration = @registry.registration_for(model_class)
+
+        raw_results.filter_map do |(storage_key, hash_data)|
+          deserialize_record(hash_data, model_class, registration)
+        rescue StandardError => e
+          @store.emit_event(:deserialization_error, key: storage_key, error: e)
+          nil
+        end
+      end
+
+      # ── Fallback query execution (no adapter query support) ──
+
+      def fallback_execute(query, _registration)
+        results = []
+
+        @store.each_key do |storage_key|
+          parsed = StorageKey.parse(storage_key.to_s)
+          next unless parsed.class_name == query.model_class.name
+
+          stored_data = @store.get(storage_key)
+          next unless stored_data
+          next unless query.predicates.all? { |p| p.match?(stored_data) }
+
+          results << [storage_key, stored_data]
+        rescue StandardError
+          next
+        end
+
+        results = apply_sort(results, query.orders)
+        results = apply_pagination(results, query.limit_value, query.offset_value)
+        deserialize_results(results, query.model_class)
+      end
+
+      def fallback_count(query, _registration)
+        count = 0
+
+        @store.each_key do |storage_key|
+          parsed = StorageKey.parse(storage_key.to_s)
+          next unless parsed.class_name == query.model_class.name
+
+          stored_data = @store.get(storage_key)
+          next unless stored_data
+          next unless query.predicates.all? { |p| p.match?(stored_data) }
+
+          count += 1
+        rescue StandardError
+          next
+        end
+
+        count
+      end
+
+      def fallback_batch(query, after: nil, limit: 1000)
+        results = []
+
+        @store.each_key do |storage_key|
+          parsed = StorageKey.parse(storage_key.to_s)
+          next unless parsed.class_name == query.model_class.name
+          next if after && storage_key.to_s <= after
+
+          stored_data = @store.get(storage_key)
+          next unless stored_data
+          next unless query.predicates.all? { |p| p.match?(stored_data) }
+
+          results << [storage_key, stored_data]
+          break if results.size >= limit
+        rescue StandardError
+          next
+        end
+
+        deserialize_results(results, query.model_class)
+      end
+
+      def apply_sort(results, orders)
+        return results if orders.empty?
+
+        results.sort do |a, b|
+          orders.reduce(0) do |cmp, o|
+            break cmp unless cmp.zero?
+
+            va = a.last.is_a?(Hash) ? a.last[o.field.to_s] : nil
+            vb = b.last.is_a?(Hash) ? b.last[o.field.to_s] : nil
+
+            cmp_val = if va.nil? && vb.nil?
+                        0
+                      elsif va.nil?
+                        1
+                      elsif vb.nil?
+                        -1
+                      else
+                        va <=> vb || 0
+                      end
+
+            o.direction == :desc ? -cmp_val : cmp_val
+          end
+        end
+      end
+
+      def apply_pagination(results, limit, offset)
+        start = offset || 0
+        results = results[start..] || []
+        results = results.first(limit) if limit
+        results
+      end
+
+      # ── CRUD internals ──
+
       def find_stored_data(registration, model, key_value)
         if registration.polymorphic?
           find_polymorphic_data(model, key_value)

data/lib/lutaml/store/format.rb

@@ -31,6 +31,19 @@ module Lutaml
         const_get(entry).new
       end
 
+      # Self-describing content: XML documents start with "<?xml" or "<",
+      # JSON with "{" or "["; everything else is YAML, the ecosystem's
+      # default. Extensionless records (object-storage keys carry no
+      # extension) declare their format this way instead of by guesswork
+      # out of band.
+      def self.guess(data)
+        stripped = data.to_s.lstrip
+        return :xml if stripped.start_with?("<?xml", "<")
+        return :json if stripped.start_with?("{", "[")
+
+        :yaml
+      end
+
       def self.for_extension(ext)
         extension_map[ext] || extension_map[".#{ext.to_s.sub(/\A\./, "")}"]
       end

data/lib/lutaml/store/manifest.rb

@@ -0,0 +1,85 @@
+# frozen_string_literal: true
+
+require "lutaml/model"
+require "digest"
+
+module Lutaml
+  module Store
+    # The enumeration document of a LutaML data repository: which keys
+    # exist, where each one lives relative to the source root, and the
+    # sha256 digest to verify it against.
+    #
+    # Sharding metadata (`shards`, `Entry#shard`) is carried but never
+    # interpreted: computing "which shard holds this key" is domain logic
+    # (e.g. relaton's crc32 over the pubid root number). The store only
+    # enumerates.
+    class Manifest
+      include Lutaml::Model::Serialize
+
+      VERSION = 1
+
+      class Entry
+        include Lutaml::Model::Serialize
+
+        attribute :key, :string
+        attribute :location, :string
+        attribute :digest, :string
+        attribute :shard, :integer
+        attribute :metadata, :hash, default: {}
+
+        def digest_for(body)
+          "sha256:#{Digest::SHA256.hexdigest(body)}"
+        end
+
+        def matches?(body)
+          return true unless digest
+
+          digest == digest_for(body)
+        end
+      end
+
+      attribute :version, :integer, default: VERSION
+      attribute :generated, :string
+      attribute :count, :integer, default: 0
+      attribute :shards, :integer, default: 0
+      attribute :entries, Manifest::Entry, collection: true, initialize_empty: true
+
+      # @param text [String]
+      # @param format [Symbol] :json or :yaml
+      # @return [Manifest]
+      def self.parse(text, format: :json)
+        case format.to_sym
+        when :json then from_json(text)
+        when :yaml then from_yaml(text)
+        else raise ConfigurationError, "unsupported manifest format: #{format}"
+        end
+      end
+
+      def self.build(entries, generated: nil, shards: 0, version: VERSION)
+        new(
+          version: version,
+          generated: (generated || Time.now.utc).iso8601,
+          count: entries.size,
+          shards: shards,
+          entries: entries
+        )
+      end
+
+      def keys
+        entries.map(&:key)
+      end
+
+      def entry_for(key)
+        entries.find { |e| e.key == key }
+      end
+
+      def key?(key)
+        !entry_for(key).nil?
+      end
+
+      def shard_of_declared?
+        shards.positive?
+      end
+    end
+  end
+end

data/lib/lutaml/store/mirror.rb

@@ -0,0 +1,94 @@
+# frozen_string_literal: true
+
+require "cgi"
+require "fileutils"
+require "json"
+
+module Lutaml
+  module Store
+    # Pulls a source into a local package directory — the GCR-style shape:
+    #
+    #   <into>[/<collection>]/
+    #     manifest.json
+    #     entries/<percent-encoded-key>
+    #
+    # The result is readable by Source::Directory, is the local-cache layout
+    # Repository uses, and is byte-compatible with the TypeScript
+    # implementation's LocalStore (conformance fixtures pin this).
+    #
+    # Pulls are incremental and digest-verified: entries whose local sha256
+    # already matches are not rewritten; a mismatched local entry is
+    # re-fetched. Digest mismatches against the source manifest are hard
+    # errors — never silently accepted.
+    module Mirror
+      class IntegrityError < Error; end
+
+      class << self
+        # @param source [Source::Base]
+        # @param into [String] directory to place the package in
+        # @param collection [String, nil] optional subdirectory of +into+
+        # @param force [Boolean] re-write entries even when digests match
+        # @return [Source::Directory] a source over the written package
+        def pull(source, into:, collection: nil, force: false)
+          manifest = source.manifest
+          root = collection ? ::File.join(into, collection) : into
+          entries_dir = ::File.join(root, "entries")
+          FileUtils.mkdir_p(entries_dir)
+
+          pulled = 0
+          entries = manifest.entries.map do |entry|
+            body = source.read(entry.key)
+            unless entry.matches?(body)
+              raise IntegrityError,
+                    "digest mismatch for #{entry.key.inspect}: manifest declares " \
+                    "#{entry.digest}, source returned #{entry.digest_for(body)}"
+            end
+
+            location = entry.location || "entries/#{Source.encode_key(entry.key)}"
+            path = ::File.join(root, location)
+            write_if_changed(path, body, force)
+            pulled += 1
+            Manifest::Entry.new(
+              key: entry.key, location: location,
+              digest: entry.digest_for(body), shard: entry.shard,
+              metadata: entry.metadata
+            )
+          end
+
+          local = Manifest.build(entries, shards: manifest.shards)
+          ::File.write(::File.join(root, "manifest.json"), JSON.pretty_generate(local.to_hash))
+
+          Source.for(:directory, path: root)
+        end
+
+        # Packs an existing package directory (as written by .pull) into a
+        # distributable .zip — the "downloaded official package" artifact.
+        # Source::Zip reads it back with no unpacking step.
+        #
+        # @param package_dir [String] a directory containing manifest.json + entries/
+        # @param to [String] the .zip path to write
+        # @return [String] the path written
+        def pack(package_dir, to:)
+          require "zip"
+          FileUtils.mkdir_p(::File.dirname(to))
+          ::Zip::File.open(to, create: true) do |zip|
+            Dir["#{package_dir}/**/*"].sort.each do |path|
+              next if ::File.directory?(path)
+
+              zip.add(path.delete_prefix("#{package_dir}/"), path)
+            end
+          end
+          to
+        end
+
+        private
+
+        def write_if_changed(path, body, force)
+          return if !force && ::File.file?(path) && ::File.binread(path) == body
+
+          ::File.binwrite(path, body)
+        end
+      end
+    end
+  end
+end