lutaml-store 0.2.2 → 0.2.4

data/lib/lutaml/store/adapter/sqlite.rb

@@ -23,9 +23,11 @@ module Lutaml
           setup_database
         end
 
+        # ── Key-value operations ──
+
         def get(key)
           result = nil
-          execute_query("SELECT value FROM #{@table_name} WHERE key = ?", [key]) do |row|
+          execute_query_raw("SELECT value FROM #{@table_name} WHERE key = ?", [key]) do |row|
             value = row[0]
             begin
               result = JSON.parse(value)
@@ -55,7 +57,7 @@ module Lutaml
         end
 
         def exists?(key)
-          execute_query("SELECT 1 FROM #{@table_name} WHERE key = ? LIMIT 1", [key]) do |_row|
+          execute_query_raw("SELECT 1 FROM #{@table_name} WHERE key = ? LIMIT 1", [key]) do |_row|
             return true
           end
           false
@@ -63,7 +65,7 @@ module Lutaml
 
         def all
           result = {}
-          execute_query("SELECT key, value FROM #{@table_name}") do |row|
+          execute_query_raw("SELECT key, value FROM #{@table_name}") do |row|
             result[row[0]] = row[1]
           end
           result
@@ -76,7 +78,7 @@ module Lutaml
         end
 
         def size
-          execute_query("SELECT COUNT(*) FROM #{@table_name}") do |row|
+          execute_query_raw("SELECT COUNT(*) FROM #{@table_name}") do |row|
             return row[0]
           end
           0
@@ -84,7 +86,7 @@ module Lutaml
 
         def keys
           result = []
-          execute_query("SELECT key FROM #{@table_name}") do |row|
+          execute_query_raw("SELECT key FROM #{@table_name}") do |row|
             result << row[0]
           end
           result
@@ -132,6 +134,56 @@ module Lutaml
           result
         end
 
+        # ── Query operations ──
+
+        def execute_query(query)
+          sql, params = build_select_sql(query)
+          results = []
+          execute_query_raw(sql, params) do |row|
+            key = row[0]
+            value = parse_json_value(row[1])
+            results << [key, value]
+          end
+          results
+        end
+
+        def count_query(query)
+          sql, params = build_count_sql(query)
+          execute_query_raw(sql, params) do |row|
+            return row[0]
+          end
+          0
+        end
+
+        def batch_query(query, after: nil, limit: 1000)
+          conditions, params = build_conditions(query)
+
+          if after
+            conditions << "key > ?"
+            params << after
+          end
+
+          sql = "SELECT key, value FROM #{@table_name}"
+          sql += " WHERE #{conditions.join(" AND ")}" unless conditions.empty?
+          sql += " ORDER BY key ASC"
+          sql += " LIMIT ?"
+          params << limit
+
+          results = []
+          execute_query_raw(sql, params) do |row|
+            key = row[0]
+            value = parse_json_value(row[1])
+            results << [key, value]
+          end
+          results
+        end
+
+        def transaction(&block)
+          @db.transaction(&block)
+        rescue SQLite3::Exception => e
+          raise BackendError, "Transaction failed: #{e.message}"
+        end
+
         private
 
         def setup_database
@@ -161,7 +213,129 @@ module Lutaml
           @db.execute("CREATE INDEX IF NOT EXISTS idx_#{@table_name}_updated_at ON #{@table_name} (updated_at)")
         end
 
-        def execute_query(sql, params = [])
+        # ── SQL generation ──
+
+        def build_select_sql(query)
+          conditions, params = build_conditions(query)
+
+          sql = "SELECT key, value FROM #{@table_name}"
+          sql += " WHERE #{conditions.join(" AND ")}" unless conditions.empty?
+          sql += build_order_clause(query.orders)
+          sql += build_limit_offset(query.limit_value, query.offset_value, params)
+
+          [sql, params]
+        end
+
+        def build_count_sql(query)
+          conditions, params = build_conditions(query)
+
+          sql = "SELECT COUNT(*) FROM #{@table_name}"
+          sql += " WHERE #{conditions.join(" AND ")}" unless conditions.empty?
+
+          [sql, params]
+        end
+
+        def build_conditions(query)
+          conditions = ["key LIKE ?"]
+          params = ["#{query.model_class.name}:%"]
+
+          query.predicates.each do |pred|
+            clause, bind = translate_predicate(pred)
+            next unless clause
+
+            conditions << clause
+            params.concat(bind)
+          end
+
+          [conditions, params]
+        end
+
+        SIMPLE_PREDICATES = {
+          Predicate::Equal => "=",
+          Predicate::NotEqual => "!=",
+          Predicate::GreaterThan => ">",
+          Predicate::LessThan => "<",
+          Predicate::GreaterThanOrEqual => ">=",
+          Predicate::LessThanOrEqual => "<="
+        }.freeze
+
+        def translate_predicate(pred)
+          field_json = "json_extract(value, '$.#{pred.field}')"
+
+          op = SIMPLE_PREDICATES[pred.class]
+          return ["#{field_json} #{op} ?", [pred.value]] if op
+
+          case pred
+          when Predicate::Between
+            ["#{field_json} BETWEEN ? AND ?", [pred.value.first, pred.value.last]]
+          when Predicate::NotBetween
+            ["#{field_json} NOT BETWEEN ? AND ?", [pred.value.first, pred.value.last]]
+          when Predicate::In
+            ["#{field_json} IN (#{in_placeholders(pred)})", pred.value]
+          when Predicate::NotIn
+            ["#{field_json} NOT IN (#{in_placeholders(pred)})", pred.value]
+          when Predicate::Matches
+            translate_matches(field_json, pred, "LIKE")
+          when Predicate::NotMatches
+            translate_matches(field_json, pred, "NOT LIKE")
+          when Predicate::Nil
+            ["#{field_json} IS NULL", []]
+          when Predicate::NotNil
+            ["#{field_json} IS NOT NULL", []]
+          end
+        end
+
+        def in_placeholders(pred)
+          pred.value.map { "?" }.join(", ")
+        end
+
+        def translate_matches(field_json, pred, op)
+          pattern = pred.value.is_a?(Regexp) ? regex_to_like(pred.value) : "%#{pred.value}%"
+          ["#{field_json} #{op} ?", [pattern]]
+        end
+
+        def pred_value(pred) # :nodoc:
+          pred.value
+        end
+
+        def build_order_clause(orders)
+          return "" if orders.empty?
+
+          clauses = orders.map do |o|
+            dir = o.direction == :desc ? "DESC" : "ASC"
+            "json_extract(value, '$.#{o.field}') #{dir} NULLS LAST"
+          end
+          " ORDER BY #{clauses.join(", ")}"
+        end
+
+        def build_limit_offset(limit, offset, params)
+          sql = ""
+          if limit
+            sql += " LIMIT ?"
+            params << limit.to_i
+          end
+          if offset
+            sql += " OFFSET ?"
+            params << offset.to_i
+          end
+          sql
+        end
+
+        def regex_to_like(regex)
+          source = regex.source
+          pattern = source.gsub(".+", "%").gsub(".*", "%").gsub(".", "_").gsub("^", "").gsub("$", "")
+          "%#{pattern}%"
+        end
+
+        def parse_json_value(raw)
+          JSON.parse(raw)
+        rescue JSON::ParserError
+          raw
+        end
+
+        # ── Raw query helpers ──
+
+        def execute_query_raw(sql, params = [])
           @db.execute(sql, params) do |row|
             yield row if block_given?
           end
@@ -182,7 +356,7 @@ module Lutaml
         end
 
         def get_schema_version
-          execute_query("PRAGMA user_version") do |row|
+          execute_query_raw("PRAGMA user_version") do |row|
             return row[0]
           end
           0

data/lib/lutaml/store/database_store.rb

@@ -2,9 +2,10 @@
 
 module Lutaml
   module Store
-    # Store-centric API with model registry and database-style operations
+    # Store-centric API with model registry and database-style operations.
+    # Provides CRUD, querying, file I/O, transactions, and events.
     class DatabaseStore
-      attr_reader :store, :registry, :composite_handler, :attribute_updater
+      attr_reader :store, :registry, :composite_handler, :attribute_updater, :serializer
 
       def initialize(adapter:, models: [], **options)
         @store = BasicStore.new(adapter_type: adapter, **options)
@@ -12,11 +13,13 @@ module Lutaml
         @serializer = ModelSerializer.new
         @composite_handler = CompositeModelHandler.new(@registry, @store, self, serializer: @serializer)
         @attribute_updater = AttributeUpdater.new(@registry, @composite_handler)
+        @scopes = {}
 
         validate_configuration!
       end
 
-      # Save single model or array of models
+      # ── CRUD ──
+
       def save(models)
         models_array = Array(models)
         saved_models = models_array.map { |model| save_single_model(model) }
@@ -25,7 +28,6 @@ module Lutaml
         models.is_a?(Array) ? saved_models : saved_models.first
       end
 
-      # Fetch model by class and key field
       def fetch(model:, **key_params)
         registration = @registry.registration_for(model)
         key_field = registration.key_field
@@ -35,20 +37,12 @@ module Lutaml
         stored_data = find_stored_data(registration, model, key_value)
         return nil unless stored_data
 
-        model_instance = @serializer.deserialize(stored_data, model, registration)
-
-        composite_references = stored_data["_composite_models"]
-        if composite_references
-          model_instance = @composite_handler.restore_composite_models(
-            model_instance, composite_references
-          )
-        end
+        model_instance = deserialize_record(stored_data, model, registration)
 
         @store.emit_event(:model_fetch, model: model_instance, key: key_value, source: :backend)
         model_instance
       end
 
-      # Update model with attributes array or block
       def update(model:, attributes: nil, **key_params, &block)
         current_model = fetch(model: model, **key_params)
         raise ModelNotRegisteredError, "Model not found" unless current_model
@@ -75,7 +69,6 @@ module Lutaml
         updated_model
       end
 
-      # Destroy model by class and key field
       def destroy(model:, **key_params)
         registration = @registry.registration_for(model)
         key_field = registration.key_field
@@ -94,54 +87,96 @@ module Lutaml
         deleted
       end
 
-      # Query operations
+      # ── Query API ──
+
+      def query(model_class)
+        Query.new(self, model_class)
+      end
+
       def where(model:, **conditions)
-        all(model: model).select do |model_instance|
-          conditions.all? { |field, value| model_instance.public_send(field) == value }
-        end
+        query(model).where(**conditions)
       end
 
-      # Get all models of a specific type
       def all(model:)
-        registration = @registry.registration_for(model)
+        query(model)
+      end
 
-        models = []
-        @store.each_key do |storage_key|
-          parsed = StorageKey.parse(storage_key.to_s)
-          next unless parsed.class_name == model.name
+      def count(model:)
+        query(model).count
+      end
 
-          stored_data = @store.get(storage_key)
-          next unless stored_data
+      def exists?(model:, **key_params)
+        !fetch(model: model, **key_params).nil?
+      end
 
-          begin
-            model_instance = @serializer.deserialize(stored_data, model, registration)
+      def find_by(model:, **conditions)
+        query(model).find_by(**conditions)
+      end
 
-            composite_references = stored_data["_composite_models"]
-            if composite_references
-              model_instance = @composite_handler.restore_composite_models(
-                model_instance, composite_references
-              )
-            end
+      def find_by!(model:, **conditions)
+        query(model).find_by!(**conditions)
+      end
 
-            models << model_instance
-          rescue StandardError => e
-            @store.emit_event(:deserialization_error, key: storage_key, error: e)
-          end
+      # ── Transactions ──
+
+      def transaction(&block)
+        adapter.transaction(&block)
+      end
+
+      # ── Scopes ──
+
+      def scope(name, body)
+        @scopes[name.to_sym] = body
+      end
+
+      def scope_for(name)
+        @scopes[name.to_sym] || raise(ArgumentError, "Unknown scope: #{name}")
+      end
+
+      # ── Query execution (called by Query) ──
+
+      def execute_query(query)
+        registration = @registry.registration_for(query.model_class)
+
+        if custom_serializer?(registration)
+          execute_with_post_filter(query, registration)
+        else
+          execute_with_pre_filter(query, registration)
         end
+      end
 
-        models
+      def count_query(query)
+        registration = @registry.registration_for(query.model_class)
+
+        if custom_serializer?(registration)
+          execute_with_post_filter(query, registration).size
+        else
+          count_with_pre_filter(query, registration)
+        end
       end
 
-      def exists?(model:, **key_params)
-        !fetch(model: model, **key_params).nil?
+      def fetch_batch(query, after: nil, limit: 1000)
+        scan_query = Query.new(self, query.model_class)
+        raw_batch = adapter.batch_query(scan_query, after: after, limit: limit)
+        if raw_batch
+          models = deserialize_results(raw_batch, query.model_class)
+          query.predicates.empty? ? models : models.select { |m| query.predicates.all? { |p| p.match?(m) } }
+        else
+          fallback_batch(query, after: after, limit: limit)
+        end
       end
 
-      def count(model:)
-        all(model: model).size
+      def last_storage_key_from(models, model_class)
+        return nil if models.empty?
+
+        registration = @registry.registration_for(model_class)
+        last_model = models.last
+        key_value = registration.extract_key(last_model)
+        registration.generate_storage_key_from_value(key_value).to_s
       end
 
-      # Load all models of a type from a directory using format-specific serialization.
-      # Bypasses the key-value layer and reads files directly using the format handler.
+      # ── File I/O ──
+
       def load_all(model_class, path: nil, format: :yaml, layout: :separate)
         fmt = Format.resolve(format)
         dir = resolve_model_dir(model_class, path)
@@ -161,8 +196,6 @@ module Lutaml
         end
       end
 
-      # Load from directory and store into the key-value backend.
-      # Returns the loaded models and makes them available via fetch/where/all.
       def import_all(model_class, path: nil, format: :yaml, layout: :separate)
         models = load_all(model_class, path: path, format: format, layout: layout)
         models.each { |model| save(model) }
@@ -170,7 +203,6 @@ module Lutaml
         models
       end
 
-      # Save all models to a directory using format-specific serialization.
       def save_all(models, path: nil, format: :yaml, layout: :separate)
         fmt = Format.resolve(format)
         models_array = Array(models)
@@ -198,7 +230,6 @@ module Lutaml
         saved
       end
 
-      # Export models to a single file or directory.
       def export(models, path:, format: :yaml)
         fmt = Format.resolve(format)
         models_array = Array(models)
@@ -212,6 +243,8 @@ module Lutaml
         path
       end
 
+      # ── Events ──
+
       def on(event, &block)
         @store.on(event, &block)
       end
@@ -235,6 +268,226 @@ module Lutaml
 
       private
 
+      def adapter
+        @store.adapter
+      end
+
+      def custom_serializer?(registration)
+        registration&.serializer
+      end
+
+      # Pre-deserialization filtering: predicates evaluated on raw hash data.
+      # Used when the serializer stores model fields as direct hash keys.
+      def execute_with_pre_filter(query, registration)
+        adapter_results = adapter.execute_query(query)
+
+        if adapter_results
+          deserialize_results(adapter_results, query.model_class)
+        else
+          fallback_execute(query, registration)
+        end
+      end
+
+      def count_with_pre_filter(query, registration)
+        adapter_count = adapter.count_query(query)
+        return adapter_count unless adapter_count.nil?
+
+        adapter_results = adapter.execute_query(query)
+        if adapter_results
+          adapter_results.size
+        else
+          fallback_count(query, registration)
+        end
+      end
+
+      # Post-deserialization filtering: scan by class name only, deserialize
+      # everything, then filter predicates on model instances.
+      # Used when custom serializers store data in non-standard hash format.
+      def execute_with_post_filter(query, _registration)
+        scan_query = Query.new(self, query.model_class)
+        adapter_results = adapter.execute_query(scan_query)
+
+        raw_results = adapter_results || scan_all_by_class(query.model_class)
+
+        models = deserialize_results(raw_results, query.model_class)
+        results = apply_model_predicates(models, query.predicates)
+        results = apply_model_sort(results, query.orders)
+        apply_model_pagination(results, query.limit_value, query.offset_value)
+      end
+
+      def scan_all_by_class(model_class)
+        results = []
+        @store.each_key do |storage_key|
+          parsed = StorageKey.parse(storage_key.to_s)
+          next unless parsed.class_name == model_class.name
+
+          stored_data = @store.get(storage_key)
+          results << [storage_key, stored_data] if stored_data
+        rescue StandardError
+          next
+        end
+        results
+      end
+
+      def apply_model_predicates(models, predicates)
+        return models if predicates.empty?
+
+        models.select { |m| predicates.all? { |p| p.match?(m) } }
+      end
+
+      def apply_model_sort(models, orders)
+        return models if orders.empty?
+
+        models.sort do |a, b|
+          orders.reduce(0) do |cmp, o|
+            break cmp unless cmp.zero?
+
+            va = a.public_send(o.field)
+            vb = b.public_send(o.field)
+
+            cmp_val = if va.nil? && vb.nil?
+                        0
+                      elsif va.nil?
+                        1
+                      elsif vb.nil?
+                        -1
+                      else
+                        va <=> vb || 0
+                      end
+
+            o.direction == :desc ? -cmp_val : cmp_val
+          end
+        end
+      end
+
+      def apply_model_pagination(models, limit, offset)
+        start = offset || 0
+        models = models[start..] || []
+        models = models.first(limit) if limit
+        models
+      end
+
+      def deserialize_record(hash_data, model_class, registration)
+        model_instance = @serializer.deserialize(hash_data, model_class, registration)
+
+        composite_references = hash_data["_composite_models"]
+        if composite_references
+          model_instance = @composite_handler.restore_composite_models(
+            model_instance, composite_references
+          )
+        end
+
+        model_instance
+      end
+
+      def deserialize_results(raw_results, model_class)
+        registration = @registry.registration_for(model_class)
+
+        raw_results.filter_map do |(storage_key, hash_data)|
+          deserialize_record(hash_data, model_class, registration)
+        rescue StandardError => e
+          @store.emit_event(:deserialization_error, key: storage_key, error: e)
+          nil
+        end
+      end
+
+      # ── Fallback query execution (no adapter query support) ──
+
+      def fallback_execute(query, _registration)
+        results = []
+
+        @store.each_key do |storage_key|
+          parsed = StorageKey.parse(storage_key.to_s)
+          next unless parsed.class_name == query.model_class.name
+
+          stored_data = @store.get(storage_key)
+          next unless stored_data
+          next unless query.predicates.all? { |p| p.match?(stored_data) }
+
+          results << [storage_key, stored_data]
+        rescue StandardError
+          next
+        end
+
+        results = apply_sort(results, query.orders)
+        results = apply_pagination(results, query.limit_value, query.offset_value)
+        deserialize_results(results, query.model_class)
+      end
+
+      def fallback_count(query, _registration)
+        count = 0
+
+        @store.each_key do |storage_key|
+          parsed = StorageKey.parse(storage_key.to_s)
+          next unless parsed.class_name == query.model_class.name
+
+          stored_data = @store.get(storage_key)
+          next unless stored_data
+          next unless query.predicates.all? { |p| p.match?(stored_data) }
+
+          count += 1
+        rescue StandardError
+          next
+        end
+
+        count
+      end
+
+      def fallback_batch(query, after: nil, limit: 1000)
+        results = []
+
+        @store.each_key do |storage_key|
+          parsed = StorageKey.parse(storage_key.to_s)
+          next unless parsed.class_name == query.model_class.name
+          next if after && storage_key.to_s <= after
+
+          stored_data = @store.get(storage_key)
+          next unless stored_data
+          next unless query.predicates.all? { |p| p.match?(stored_data) }
+
+          results << [storage_key, stored_data]
+          break if results.size >= limit
+        rescue StandardError
+          next
+        end
+
+        deserialize_results(results, query.model_class)
+      end
+
+      def apply_sort(results, orders)
+        return results if orders.empty?
+
+        results.sort do |a, b|
+          orders.reduce(0) do |cmp, o|
+            break cmp unless cmp.zero?
+
+            va = a.last.is_a?(Hash) ? a.last[o.field.to_s] : nil
+            vb = b.last.is_a?(Hash) ? b.last[o.field.to_s] : nil
+
+            cmp_val = if va.nil? && vb.nil?
+                        0
+                      elsif va.nil?
+                        1
+                      elsif vb.nil?
+                        -1
+                      else
+                        va <=> vb || 0
+                      end
+
+            o.direction == :desc ? -cmp_val : cmp_val
+          end
+        end
+      end
+
+      def apply_pagination(results, limit, offset)
+        start = offset || 0
+        results = results[start..] || []
+        results = results.first(limit) if limit
+        results
+      end
+
+      # ── CRUD internals ──
+
       def find_stored_data(registration, model, key_value)
         if registration.polymorphic?
           find_polymorphic_data(model, key_value)