lutaml-store 0.2.1 → 0.2.4

data/lib/lutaml/store/adapter.rb

@@ -7,6 +7,23 @@ module Lutaml
       autoload :Memory, "lutaml/store/adapter/memory"
       autoload :FileSystem, "lutaml/store/adapter/filesystem"
       autoload :Sqlite, "lutaml/store/adapter/sqlite"
+
+      @registry = {
+        memory: "Memory",
+        filesystem: "FileSystem",
+        sqlite: "Sqlite"
+      }
+
+      def self.resolve(type, options = {})
+        entry = @registry[type.to_sym]
+        raise ConfigurationError, "Unknown adapter type: #{type}" unless entry
+
+        const_get(entry).new(options)
+      end
+
+      def self.register(type, adapter_class)
+        @registry[type.to_sym] = adapter_class.name
+      end
     end
   end
 end

data/lib/lutaml/store/basic_store.rb

@@ -167,16 +167,7 @@ module Lutaml
       end
 
       def create_adapter
-        case @config.adapter_type
-        when :memory
-          Adapter::Memory.new(@config.adapter_options)
-        when :filesystem
-          Adapter::FileSystem.new(@config.adapter_options)
-        when :sqlite
-          Adapter::Sqlite.new(@config.adapter_options)
-        else
-          raise ConfigurationError, "Unknown adapter type: #{@config.adapter_type}"
-        end
+        Adapter.resolve(@config.adapter_type, @config.adapter_options)
       end
 
       def create_cache

data/lib/lutaml/store/cache_store.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 
 require "json"
+require "time"
 
 module Lutaml
   module Store
@@ -246,16 +247,7 @@ module Lutaml
         adapter_type = config[:adapter]&.dig(:type) || config[:adapter_type] || :memory
         adapter_options = config[:adapter]&.dig(:options) || config[:adapter_options] || {}
 
-        case adapter_type.to_sym
-        when :memory
-          Adapter::Memory.new(adapter_options)
-        when :filesystem
-          Adapter::FileSystem.new(adapter_options)
-        when :sqlite
-          Adapter::Sqlite.new(adapter_options)
-        else
-          raise ConfigurationError, "Unknown adapter type: #{adapter_type}"
-        end
+        Adapter.resolve(adapter_type, adapter_options)
       end
 
       def serialize_entry(entry)

data/lib/lutaml/store/database_store.rb

@@ -2,9 +2,10 @@
 
 module Lutaml
   module Store
-    # Store-centric API with model registry and database-style operations
+    # Store-centric API with model registry and database-style operations.
+    # Provides CRUD, querying, file I/O, transactions, and events.
     class DatabaseStore
-      attr_reader :store, :registry, :composite_handler, :attribute_updater
+      attr_reader :store, :registry, :composite_handler, :attribute_updater, :serializer
 
       def initialize(adapter:, models: [], **options)
         @store = BasicStore.new(adapter_type: adapter, **options)
@@ -12,11 +13,13 @@ module Lutaml
         @serializer = ModelSerializer.new
         @composite_handler = CompositeModelHandler.new(@registry, @store, self, serializer: @serializer)
         @attribute_updater = AttributeUpdater.new(@registry, @composite_handler)
+        @scopes = {}
 
         validate_configuration!
       end
 
-      # Save single model or array of models
+      # ── CRUD ──
+
       def save(models)
         models_array = Array(models)
         saved_models = models_array.map { |model| save_single_model(model) }
@@ -25,7 +28,6 @@ module Lutaml
         models.is_a?(Array) ? saved_models : saved_models.first
       end
 
-      # Fetch model by class and key field
       def fetch(model:, **key_params)
         registration = @registry.registration_for(model)
         key_field = registration.key_field
@@ -35,20 +37,12 @@ module Lutaml
         stored_data = find_stored_data(registration, model, key_value)
         return nil unless stored_data
 
-        model_instance = @serializer.deserialize(stored_data, model, registration)
-
-        composite_references = stored_data["_composite_models"]
-        if composite_references
-          model_instance = @composite_handler.restore_composite_models(
-            model_instance, composite_references
-          )
-        end
+        model_instance = deserialize_record(stored_data, model, registration)
 
         @store.emit_event(:model_fetch, model: model_instance, key: key_value, source: :backend)
         model_instance
       end
 
-      # Update model with attributes array or block
       def update(model:, attributes: nil, **key_params, &block)
         current_model = fetch(model: model, **key_params)
         raise ModelNotRegisteredError, "Model not found" unless current_model
@@ -75,7 +69,6 @@ module Lutaml
         updated_model
       end
 
-      # Destroy model by class and key field
       def destroy(model:, **key_params)
         registration = @registry.registration_for(model)
         key_field = registration.key_field
@@ -94,54 +87,96 @@ module Lutaml
         deleted
       end
 
-      # Query operations
+      # ── Query API ──
+
+      def query(model_class)
+        Query.new(self, model_class)
+      end
+
       def where(model:, **conditions)
-        all(model: model).select do |model_instance|
-          conditions.all? { |field, value| model_instance.public_send(field) == value }
-        end
+        query(model).where(**conditions)
       end
 
-      # Get all models of a specific type
       def all(model:)
-        registration = @registry.registration_for(model)
+        query(model)
+      end
 
-        models = []
-        @store.each_key do |storage_key|
-          parsed = StorageKey.parse(storage_key.to_s)
-          next unless parsed.class_name == model.name
+      def count(model:)
+        query(model).count
+      end
 
-          stored_data = @store.get(storage_key)
-          next unless stored_data
+      def exists?(model:, **key_params)
+        !fetch(model: model, **key_params).nil?
+      end
 
-          begin
-            model_instance = @serializer.deserialize(stored_data, model, registration)
+      def find_by(model:, **conditions)
+        query(model).find_by(**conditions)
+      end
 
-            composite_references = stored_data["_composite_models"]
-            if composite_references
-              model_instance = @composite_handler.restore_composite_models(
-                model_instance, composite_references
-              )
-            end
+      def find_by!(model:, **conditions)
+        query(model).find_by!(**conditions)
+      end
 
-            models << model_instance
-          rescue StandardError => e
-            @store.emit_event(:deserialization_error, key: storage_key, error: e)
-          end
+      # ── Transactions ──
+
+      def transaction(&block)
+        adapter.transaction(&block)
+      end
+
+      # ── Scopes ──
+
+      def scope(name, body)
+        @scopes[name.to_sym] = body
+      end
+
+      def scope_for(name)
+        @scopes[name.to_sym] || raise(ArgumentError, "Unknown scope: #{name}")
+      end
+
+      # ── Query execution (called by Query) ──
+
+      def execute_query(query)
+        registration = @registry.registration_for(query.model_class)
+
+        if custom_serializer?(registration)
+          execute_with_post_filter(query, registration)
+        else
+          execute_with_pre_filter(query, registration)
         end
+      end
 
-        models
+      def count_query(query)
+        registration = @registry.registration_for(query.model_class)
+
+        if custom_serializer?(registration)
+          execute_with_post_filter(query, registration).size
+        else
+          count_with_pre_filter(query, registration)
+        end
       end
 
-      def exists?(model:, **key_params)
-        !fetch(model: model, **key_params).nil?
+      def fetch_batch(query, after: nil, limit: 1000)
+        scan_query = Query.new(self, query.model_class)
+        raw_batch = adapter.batch_query(scan_query, after: after, limit: limit)
+        if raw_batch
+          models = deserialize_results(raw_batch, query.model_class)
+          query.predicates.empty? ? models : models.select { |m| query.predicates.all? { |p| p.match?(m) } }
+        else
+          fallback_batch(query, after: after, limit: limit)
+        end
       end
 
-      def count(model:)
-        all(model: model).size
+      def last_storage_key_from(models, model_class)
+        return nil if models.empty?
+
+        registration = @registry.registration_for(model_class)
+        last_model = models.last
+        key_value = registration.extract_key(last_model)
+        registration.generate_storage_key_from_value(key_value).to_s
       end
 
-      # Load all models of a type from a directory using format-specific serialization.
-      # Bypasses the key-value layer and reads files directly using the format handler.
+      # ── File I/O ──
+
       def load_all(model_class, path: nil, format: :yaml, layout: :separate)
         fmt = Format.resolve(format)
         dir = resolve_model_dir(model_class, path)
@@ -161,8 +196,6 @@ module Lutaml
         end
       end
 
-      # Load from directory and store into the key-value backend.
-      # Returns the loaded models and makes them available via fetch/where/all.
       def import_all(model_class, path: nil, format: :yaml, layout: :separate)
         models = load_all(model_class, path: path, format: format, layout: layout)
         models.each { |model| save(model) }
@@ -170,7 +203,6 @@ module Lutaml
         models
       end
 
-      # Save all models to a directory using format-specific serialization.
       def save_all(models, path: nil, format: :yaml, layout: :separate)
         fmt = Format.resolve(format)
         models_array = Array(models)
@@ -198,7 +230,6 @@ module Lutaml
         saved
       end
 
-      # Export models to a single file or directory.
       def export(models, path:, format: :yaml)
         fmt = Format.resolve(format)
         models_array = Array(models)
@@ -207,11 +238,13 @@ module Lutaml
 
         content = fmt.serialize_many(models_array)
 
-        File.write(path, content, encoding: "utf-8")
+        write_file(path, content, fmt)
         @store.emit_event(:model_export, count: models_array.size, path: path)
         path
       end
 
+      # ── Events ──
+
       def on(event, &block)
         @store.on(event, &block)
       end
@@ -235,6 +268,226 @@ module Lutaml
 
       private
 
+      def adapter
+        @store.adapter
+      end
+
+      def custom_serializer?(registration)
+        registration&.serializer
+      end
+
+      # Pre-deserialization filtering: predicates evaluated on raw hash data.
+      # Used when the serializer stores model fields as direct hash keys.
+      def execute_with_pre_filter(query, registration)
+        adapter_results = adapter.execute_query(query)
+
+        if adapter_results
+          deserialize_results(adapter_results, query.model_class)
+        else
+          fallback_execute(query, registration)
+        end
+      end
+
+      def count_with_pre_filter(query, registration)
+        adapter_count = adapter.count_query(query)
+        return adapter_count unless adapter_count.nil?
+
+        adapter_results = adapter.execute_query(query)
+        if adapter_results
+          adapter_results.size
+        else
+          fallback_count(query, registration)
+        end
+      end
+
+      # Post-deserialization filtering: scan by class name only, deserialize
+      # everything, then filter predicates on model instances.
+      # Used when custom serializers store data in non-standard hash format.
+      def execute_with_post_filter(query, _registration)
+        scan_query = Query.new(self, query.model_class)
+        adapter_results = adapter.execute_query(scan_query)
+
+        raw_results = adapter_results || scan_all_by_class(query.model_class)
+
+        models = deserialize_results(raw_results, query.model_class)
+        results = apply_model_predicates(models, query.predicates)
+        results = apply_model_sort(results, query.orders)
+        apply_model_pagination(results, query.limit_value, query.offset_value)
+      end
+
+      def scan_all_by_class(model_class)
+        results = []
+        @store.each_key do |storage_key|
+          parsed = StorageKey.parse(storage_key.to_s)
+          next unless parsed.class_name == model_class.name
+
+          stored_data = @store.get(storage_key)
+          results << [storage_key, stored_data] if stored_data
+        rescue StandardError
+          next
+        end
+        results
+      end
+
+      def apply_model_predicates(models, predicates)
+        return models if predicates.empty?
+
+        models.select { |m| predicates.all? { |p| p.match?(m) } }
+      end
+
+      def apply_model_sort(models, orders)
+        return models if orders.empty?
+
+        models.sort do |a, b|
+          orders.reduce(0) do |cmp, o|
+            break cmp unless cmp.zero?
+
+            va = a.public_send(o.field)
+            vb = b.public_send(o.field)
+
+            cmp_val = if va.nil? && vb.nil?
+                        0
+                      elsif va.nil?
+                        1
+                      elsif vb.nil?
+                        -1
+                      else
+                        va <=> vb || 0
+                      end
+
+            o.direction == :desc ? -cmp_val : cmp_val
+          end
+        end
+      end
+
+      def apply_model_pagination(models, limit, offset)
+        start = offset || 0
+        models = models[start..] || []
+        models = models.first(limit) if limit
+        models
+      end
+
+      def deserialize_record(hash_data, model_class, registration)
+        model_instance = @serializer.deserialize(hash_data, model_class, registration)
+
+        composite_references = hash_data["_composite_models"]
+        if composite_references
+          model_instance = @composite_handler.restore_composite_models(
+            model_instance, composite_references
+          )
+        end
+
+        model_instance
+      end
+
+      def deserialize_results(raw_results, model_class)
+        registration = @registry.registration_for(model_class)
+
+        raw_results.filter_map do |(storage_key, hash_data)|
+          deserialize_record(hash_data, model_class, registration)
+        rescue StandardError => e
+          @store.emit_event(:deserialization_error, key: storage_key, error: e)
+          nil
+        end
+      end
+
+      # ── Fallback query execution (no adapter query support) ──
+
+      def fallback_execute(query, _registration)
+        results = []
+
+        @store.each_key do |storage_key|
+          parsed = StorageKey.parse(storage_key.to_s)
+          next unless parsed.class_name == query.model_class.name
+
+          stored_data = @store.get(storage_key)
+          next unless stored_data
+          next unless query.predicates.all? { |p| p.match?(stored_data) }
+
+          results << [storage_key, stored_data]
+        rescue StandardError
+          next
+        end
+
+        results = apply_sort(results, query.orders)
+        results = apply_pagination(results, query.limit_value, query.offset_value)
+        deserialize_results(results, query.model_class)
+      end
+
+      def fallback_count(query, _registration)
+        count = 0
+
+        @store.each_key do |storage_key|
+          parsed = StorageKey.parse(storage_key.to_s)
+          next unless parsed.class_name == query.model_class.name
+
+          stored_data = @store.get(storage_key)
+          next unless stored_data
+          next unless query.predicates.all? { |p| p.match?(stored_data) }
+
+          count += 1
+        rescue StandardError
+          next
+        end
+
+        count
+      end
+
+      def fallback_batch(query, after: nil, limit: 1000)
+        results = []
+
+        @store.each_key do |storage_key|
+          parsed = StorageKey.parse(storage_key.to_s)
+          next unless parsed.class_name == query.model_class.name
+          next if after && storage_key.to_s <= after
+
+          stored_data = @store.get(storage_key)
+          next unless stored_data
+          next unless query.predicates.all? { |p| p.match?(stored_data) }
+
+          results << [storage_key, stored_data]
+          break if results.size >= limit
+        rescue StandardError
+          next
+        end
+
+        deserialize_results(results, query.model_class)
+      end
+
+      def apply_sort(results, orders)
+        return results if orders.empty?
+
+        results.sort do |a, b|
+          orders.reduce(0) do |cmp, o|
+            break cmp unless cmp.zero?
+
+            va = a.last.is_a?(Hash) ? a.last[o.field.to_s] : nil
+            vb = b.last.is_a?(Hash) ? b.last[o.field.to_s] : nil
+
+            cmp_val = if va.nil? && vb.nil?
+                        0
+                      elsif va.nil?
+                        1
+                      elsif vb.nil?
+                        -1
+                      else
+                        va <=> vb || 0
+                      end
+
+            o.direction == :desc ? -cmp_val : cmp_val
+          end
+        end
+      end
+
+      def apply_pagination(results, limit, offset)
+        start = offset || 0
+        results = results[start..] || []
+        results = results.first(limit) if limit
+        results
+      end
+
+      # ── CRUD internals ──
+
       def find_stored_data(registration, model, key_value)
         if registration.polymorphic?
           find_polymorphic_data(model, key_value)
@@ -331,8 +584,8 @@ module Lutaml
         Dir.glob(glob).sort.each do |file_path|
           next unless File.file?(file_path)
 
-          raw = File.read(file_path, encoding: "utf-8")
-          next if raw.strip.empty?
+          raw = read_file(file_path, fmt)
+          next if !fmt.binary? && raw.strip.empty?
 
           begin
             model = fmt.deserialize(raw, model_class)
@@ -351,8 +604,8 @@ module Lutaml
         Dir.glob(glob).sort.each do |file_path|
           next unless File.file?(file_path)
 
-          raw = File.read(file_path, encoding: "utf-8")
-          next if raw.strip.empty?
+          raw = read_file(file_path, fmt)
+          next if !fmt.binary? && raw.strip.empty?
 
           begin
             loaded = fmt.deserialize_many(raw, model_class)
@@ -375,8 +628,7 @@ module Lutaml
           key = extract_model_key(model)
           filename = key || model.class.name.to_s.gsub("::", "_")
           file_path = File.join(dir, "#{filename}#{fmt.extension}")
-          content = fmt.serialize(model)
-          File.write(file_path, content, encoding: "utf-8")
+          write_file(file_path, fmt.serialize(model), fmt)
           model
         end
       end
@@ -391,8 +643,7 @@ module Lutaml
 
         grouped.map do |key, group|
           file_path = File.join(dir, "#{key}#{fmt.extension}")
-          content = fmt.serialize_many(group)
-          File.write(file_path, content, encoding: "utf-8")
+          write_file(file_path, fmt.serialize_many(group), fmt)
           group
         end.flatten
       end
@@ -420,6 +671,18 @@ module Lutaml
         basename = File.basename(file_path, ".*")
         model.public_send(:"#{registration.key_field}=", basename)
       end
+
+      def read_file(file_path, fmt)
+        fmt.binary? ? File.binread(file_path) : File.read(file_path, encoding: "utf-8")
+      end
+
+      def write_file(file_path, content, fmt)
+        if fmt.binary?
+          File.binwrite(file_path, content)
+        else
+          File.write(file_path, content, encoding: "utf-8")
+        end
+      end
     end
   end
 end

data/lib/lutaml/store/format/base.rb

@@ -27,6 +27,10 @@ module Lutaml
         def deserialize_many(_data, _model_class)
           raise NotImplementedError, "#{self.class} does not support multi-document deserialization"
         end
+
+        def binary?
+          false
+        end
       end
     end
   end

data/lib/lutaml/store/format/marshal_format.rb

@@ -31,6 +31,10 @@ module Lutaml
           hash_array = ::Marshal.load(data)
           hash_array.map { |h| model_class.from_hash(h) }
         end
+
+        def binary?
+          true
+        end
       end
     end
   end

data/lib/lutaml/store/format/xml.rb

@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+module Lutaml
+  module Store
+    module Format
+      class Xml < Base
+        def extension
+          ".xml"
+        end
+
+        def glob_pattern
+          "*.xml"
+        end
+
+        def serialize(model)
+          model.to_xml
+        end
+
+        def deserialize(data, model_class)
+          model_class.from_xml(data)
+        end
+
+        def serialize_many(models)
+          inner = models.map(&:to_xml).join("\n")
+          "<items>\n#{inner}\n</items>"
+        end
+
+        def deserialize_many(data, model_class)
+          doc = Moxml.parse(data)
+          doc.root.children.select(&:element?).map do |child|
+            model_class.from_xml(child.to_xml)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/lutaml/store/format/yamls.rb

@@ -17,7 +17,7 @@ module Lutaml
         end
 
         def serialize_many(models)
-          models.map(&:to_yamls).join
+          models.map(&:to_yamls).join("\n")
         end
 
         def deserialize(data, model_class)

data/lib/lutaml/store/format.rb

@@ -13,13 +13,15 @@ module Lutaml
       autoload :Json, "lutaml/store/format/json"
       autoload :Jsonl, "lutaml/store/format/jsonl"
       autoload :MarshalFormat, "lutaml/store/format/marshal_format"
+      autoload :Xml, "lutaml/store/format/xml"
 
       FORMATS = {
         yaml: "Yaml",
         yamls: "Yamls",
         json: "Json",
         jsonl: "Jsonl",
-        marshal: "MarshalFormat"
+        marshal: "MarshalFormat",
+        xml: "Xml"
       }.freeze
 
       def self.resolve(format)