maglev-rb 0.1.1 → 0.2.1

data/lib/maglev/structured_result.rb

@@ -0,0 +1,97 @@
+# frozen_string_literal: true
+
+module Maglev
+  class StructuredEvidence
+    State = Struct.new(:loaded, :records, :count, :truncated)
+
+    attr_reader :scalar, :filters, :date_ranges
+
+    def initialize(records: [], scalar: nil, filters: [], date_ranges: [], count: 0, truncated: false, loader: nil)
+      materialized = deep_freeze(records)
+      @state = State.new(loader.nil?, materialized, count, truncated)
+      @loader = loader
+      @materialization_lock = Mutex.new
+      @scalar = deep_freeze(scalar)
+      @filters = filters.map { |filter| deep_freeze(filter) }.freeze
+      @date_ranges = date_ranges.map { |range| deep_freeze(range) }.freeze
+      freeze
+    end
+
+    def records
+      materialize.records
+    end
+
+    def count
+      materialize.count
+    end
+
+    def truncated
+      materialize.truncated
+    end
+
+    def truncated? = truncated
+
+    private
+
+    def materialize
+      return @state if @state.loaded
+
+      @materialization_lock.synchronize do
+        return @state if @state.loaded
+
+        records, count, truncated = @loader.call
+        @state.records = deep_freeze(records)
+        @state.count = count
+        @state.truncated = truncated
+        @state.loaded = true
+      end
+      @state
+    end
+
+    def deep_freeze(value)
+      case value
+      when Hash then value.to_h { |key, item| [deep_freeze(key), deep_freeze(item)] }.freeze
+      when Array then value.map { |item| deep_freeze(item) }.freeze
+      else value.frozen? ? value : value.freeze
+      end
+    end
+  end
+
+  class StructuredResult
+    STATUSES = %i[succeeded clarification_required unsupported failed].freeze
+    KINDS = %i[relation scalar aggregate none].freeze
+
+    attr_reader :status, :route, :kind, :value, :evidence, :interpretation, :warnings, :plan, :trace_id
+
+    def initialize(status:, kind:, plan:, trace_id:, value: nil, evidence: StructuredEvidence.new,
+      interpretation: nil, warnings: [])
+      raise ArgumentError, "invalid structured result status" unless STATUSES.include?(status)
+      raise ArgumentError, "invalid structured result kind" unless KINDS.include?(kind)
+      raise ArgumentError, "only successful results may carry a value" if status != :succeeded && !value.nil?
+
+      @status = status
+      @route = :structured
+      @kind = kind
+      @value = value
+      @evidence = evidence
+      @interpretation = interpretation&.to_s&.freeze
+      @warnings = Array(warnings).map { |warning| warning.to_s.freeze }.freeze
+      @plan = plan
+      @trace_id = trace_id.to_s.freeze
+      freeze
+    end
+
+    def render
+      return warnings.first || "The request is unsupported." if status == :unsupported
+      return plan.clarification.fetch(:message) if status == :clarification_required
+      return "The structured request failed." if status == :failed
+      return "Count: #{value}" if kind == :aggregate && plan.ir.aggregate.function == :count
+      return "#{plan.ir.aggregate.function.to_s.capitalize}: #{value}" if kind == :aggregate
+
+      return "No matching records." if evidence.records.empty?
+
+      headings = evidence.records.first.keys
+      ([headings.join(" | ")] + evidence.records.map { |record| headings.map { |heading| record[heading] }.join(" | ") }).join("\n")
+    end
+  end
+end

data/lib/maglev/trace.rb

@@ -0,0 +1,56 @@
+# frozen_string_literal: true
+
+require "time"
+
+module Maglev
+  module Trace
+    module_function
+
+    def instrument(step, trace_id:, resource: nil, operation: nil)
+      payload = compact_payload(trace_id: trace_id, route: :structured, resource: resource,
+        operation: operation, status: :succeeded, timestamp: Time.now.utc.iso8601)
+      error = nil
+      started = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+      value = ActiveSupport::Notifications.instrument("maglev.structured.#{step}", payload) do
+        yield(payload)
+      rescue => caught
+        error = caught
+        payload[:status] = :failed
+        payload[:error_code] = safe_error_code(caught)
+        nil
+      end
+      payload[:duration_bucket] = duration_bucket(Process.clock_gettime(Process::CLOCK_MONOTONIC) - started)
+      emit_audit(step, payload.freeze)
+      raise error if error
+
+      value
+    end
+
+    def emit_audit(step, payload)
+      sink = Maglev.configuration.audit_sink
+      sink&.call(payload.merge(event: "maglev.structured.#{step}").freeze)
+    rescue => error
+      Maglev.configuration.logger&.warn("Maglev audit sink failed: #{error.class.name}")
+    end
+
+    def compact_payload(**values)
+      values.compact
+    end
+
+    def safe_error_code(error)
+      case error.class.name
+      when "Maglev::QueryCompilationError" then :compilation_failed
+      when "Maglev::StructuredExecutionError" then :execution_failed
+      else :internal_error
+      end
+    end
+
+    def duration_bucket(seconds)
+      return :under_10ms if seconds < 0.01
+      return :under_100ms if seconds < 0.1
+      return :under_1s if seconds < 1
+
+      :one_second_or_more
+    end
+  end
+end

data/lib/maglev/vector_stores/base.rb

@@ -3,10 +3,22 @@
 module Maglev
   module VectorStores
     class Base
+      CONTRACT_VERSION = 2
+
+      def contract_version = CONTRACT_VERSION
+
+      def fetch(ids:)
+        raise NotImplementedError, "#{self.class.name} must implement #fetch"
+      end
+
       def upsert(documents:)
         raise NotImplementedError, "#{self.class.name} must implement #upsert"
       end
 
+      def replace_owner(owner_type:, owner_id:, documents:)
+        raise NotImplementedError, "#{self.class.name} must implement #replace_owner"
+      end
+
       def search(vector:, filters:, limit:)
         raise NotImplementedError, "#{self.class.name} must implement #search"
       end

data/lib/maglev/vector_stores/document.rb

@@ -3,25 +3,30 @@
 module Maglev
   module VectorStores
     class Document
-      attr_reader :id, :owner_type, :owner_id, :owner_model_name, :source,
+      attr_reader :id, :owner_type, :owner_id, :owner_model_name, :source, :source_identity, :source_type, :tenant_id,
         :chunk_index, :content, :content_checksum, :embedding_model,
-        :embedding, :owner, :distance
+        :index_version, :embedding, :owner, :distance, :score
 
       def initialize(owner_type:, owner_id:, owner_model_name:, source:, chunk_index:,
-        content:, content_checksum:, embedding_model:, embedding:, id: nil,
-        owner: nil, distance: nil)
+        content:, content_checksum:, embedding_model:, index_version:, embedding:, id: nil,
+        owner: nil, distance: nil, score: nil, source_identity: nil, source_type: nil, tenant_id: nil)
         @owner_type = owner_type
         @owner_id = owner_id
         @owner_model_name = owner_model_name
         @source = source
+        @source_identity = source_identity || source
+        @source_type = (source_type || :snapshot).to_sym
+        @tenant_id = tenant_id
         @chunk_index = chunk_index
         @content = content
         @content_checksum = content_checksum
         @embedding_model = embedding_model
+        @index_version = index_version
         @embedding = embedding
         @id = id || "#{owner_type}:#{owner_id}:#{source}:#{chunk_index}"
         @owner = owner
         @distance = distance
+        @score = (score.nil? && !distance.nil?) ? (1.0 - distance.to_f).clamp(0.0, 1.0) : score
         freeze
       end
 
@@ -31,7 +36,11 @@ module Maglev
           owner_id: owner_id,
           owner_model_name: owner_model_name,
           source: source,
-          chunk_index: chunk_index
+          source_identity: source_identity,
+          source_type: source_type,
+          tenant_id: tenant_id,
+          chunk_index: chunk_index,
+          index_version: index_version
         }
       end
     end

data/lib/maglev/vector_stores/document_id.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+require "base64"
+
+module Maglev
+  module VectorStores
+    module DocumentId
+      module_function
+
+      def build(owner_type:, owner_id:, source_identity:, chunk_index:)
+        source = (source_identity == "snapshot") ? "snapshot" : "b64-#{Base64.urlsafe_encode64(source_identity, padding: false)}"
+        "#{owner_type}:#{owner_id}:#{source}:#{chunk_index}"
+      end
+
+      def parse(id)
+        parts = id.split(":")
+        chunk_index = parts.pop
+        source = parts.pop
+        owner_id = parts.pop
+        source_identity = source.start_with?("b64-") ? Base64.urlsafe_decode64(source.delete_prefix("b64-")) : source
+        [parts.join(":"), owner_id, source_identity, chunk_index]
+      rescue ArgumentError
+        raise ArgumentError, "invalid Maglev document id"
+      end
+    end
+  end
+end

data/lib/maglev/vector_stores/memory.rb

@@ -2,20 +2,58 @@
 
 require_relative "base"
 require_relative "document"
+require_relative "metadata_filter"
 
 module Maglev
   module VectorStores
     class Memory < Base
+      Entry = Struct.new(:owner_type, :owner_id, :document)
+
       def initialize
         @documents = {}
+        @mutex = Mutex.new
+      end
+
+      def fetch(ids:)
+        requested_ids = ids.to_a
+        documents = @mutex.synchronize { @documents }
+        requested_ids.filter_map { |id| documents[id]&.document }
       end
 
       def upsert(documents:)
-        documents.each { |document| @documents[document.id] = document }
+        staged = stage(documents)
+        @mutex.synchronize do
+          replacement = @documents.dup
+          staged.each { |id, entry| replacement[id] = entry }
+          @documents = replacement
+        end
+      end
+
+      def replace_owner(owner_type:, owner_id:, documents:)
+        staged = stage(documents)
+        unless staged.all? { |_id, entry| entry.owner_type == owner_type && entry.owner_id == owner_id }
+          raise ArgumentError, "replacement documents must match the requested owner"
+        end
+
+        @mutex.synchronize do
+          conflict = staged.any? do |id, _entry|
+            existing = @documents[id]
+            existing && (existing.owner_type != owner_type || existing.owner_id != owner_id)
+          end
+          raise ArgumentError, "replacement document id belongs to another owner" if conflict
+
+          replacement = @documents.reject do |_id, entry|
+            entry.owner_type == owner_type && entry.owner_id == owner_id
+          end
+          staged.each { |id, entry| replacement[id] = entry }
+          @documents = replacement
+        end
       end
 
       def search(vector:, filters:, limit:)
-        @documents.values
+        filters = MetadataFilter.coerce(filters)
+        documents = @mutex.synchronize { @documents.values.map(&:document) }
+        documents
           .select { |document| matches_filters?(document, filters) }
           .map { |document| with_distance(document, cosine_distance(vector, document.embedding)) }
           .sort_by(&:distance)
@@ -23,12 +61,19 @@ module Maglev
       end
 
       def delete(ids:)
-        ids.each { |id| @documents.delete(id) }
+        requested_ids = ids.to_a
+        @mutex.synchronize do
+          replacement = @documents.dup
+          requested_ids.each { |id| replacement.delete(id) }
+          @documents = replacement
+        end
       end
 
       def delete_by_owner(owner_type:, owner_id:)
-        @documents.delete_if do |_id, document|
-          document.owner_type == owner_type && document.owner_id == owner_id
+        @mutex.synchronize do
+          @documents = @documents.reject do |_id, document|
+            document.owner_type == owner_type && document.owner_id == owner_id
+          end
         end
       end
 
@@ -42,8 +87,21 @@ module Maglev
 
       private
 
+      def stage(documents)
+        documents.map do |document|
+          entry = Entry.new(document.owner_type, document.owner_id, document).freeze
+          [document.id, entry]
+        end
+      end
+
       def matches_filters?(document, filters)
-        filters.all? { |key, value| document.metadata.fetch(key) == value }
+        filters.all? do |key, value|
+          case key
+          when :owner_ids then value.include?(document.owner_id)
+          when :source_types then value.map(&:to_sym).include?(document.source_type)
+          else document.metadata.fetch(key) == value
+          end
+        end
       end
 
       def cosine_distance(left, right)
@@ -62,10 +120,14 @@ module Maglev
           owner_id: document.owner_id,
           owner_model_name: document.owner_model_name,
           source: document.source,
+          source_identity: document.source_identity,
+          source_type: document.source_type,
+          tenant_id: document.tenant_id,
           chunk_index: document.chunk_index,
           content: document.content,
           content_checksum: document.content_checksum,
           embedding_model: document.embedding_model,
+          index_version: document.index_version,
           embedding: document.embedding,
           owner: document.owner,
           distance: distance

data/lib/maglev/vector_stores/metadata_filter.rb

@@ -0,0 +1,55 @@
+# frozen_string_literal: true
+
+module Maglev
+  module VectorStores
+    class MetadataFilter
+      include Enumerable
+
+      FIELDS = %i[owner_type owner_id owner_ids owner_model_name tenant_id source_type source_types index_version].freeze
+      SOURCE_TYPES = %i[snapshot attribute related_record rich_text attachment tag].freeze
+      MAX_VALUES = 1_000
+
+      def self.coerce(value)
+        value.is_a?(self) ? value : new(**value.to_h)
+      end
+
+      def initialize(**values)
+        unknown = values.keys.map(&:to_sym) - FIELDS
+        raise ArgumentError, "Unsupported metadata filter: #{unknown.join(", ")}" if unknown.any?
+
+        normalized = values.transform_keys(&:to_sym)
+        validate!(normalized)
+        @values = normalized.transform_values { |value| value.is_a?(Array) ? value.dup.freeze : value }.freeze
+        freeze
+      end
+
+      def each(&block) = @values.each(&block)
+      def to_h = @values
+      def fetch(...) = @values.fetch(...)
+      def key?(key) = @values.key?(key)
+
+      private
+
+      def validate!(values)
+        %i[owner_ids source_types].each do |field|
+          next unless values.key?(field)
+          value = values[field]
+          raise ArgumentError, "#{field} must be a non-empty Array" unless value.is_a?(Array) && value.any?
+          raise ArgumentError, "#{field} exceeds #{MAX_VALUES} values" if value.size > MAX_VALUES
+        end
+        Array(values[:source_type]).each { |value| validate_source_type!(value) } if values.key?(:source_type)
+        Array(values[:source_types]).each { |value| validate_source_type!(value) } if values.key?(:source_types)
+        %i[owner_type owner_model_name tenant_id index_version].each do |field|
+          next unless values.key?(field)
+          raise ArgumentError, "#{field} must be a non-empty String" unless values[field].is_a?(String) && !values[field].empty?
+        end
+      end
+
+      def validate_source_type!(value)
+        return if SOURCE_TYPES.include?(value.to_sym)
+
+        raise ArgumentError, "Unsupported source type: #{value.inspect}"
+      end
+    end
+  end
+end

data/lib/maglev/vector_stores/pgvector.rb

@@ -1,8 +1,12 @@
 # frozen_string_literal: true
 
+require "digest"
+
 require_relative "../chunk"
 require_relative "base"
 require_relative "document"
+require_relative "metadata_filter"
+require_relative "document_id"
 
 module Maglev
   module VectorStores
@@ -19,7 +23,7 @@ module Maglev
             next if existing
 
             scope.where(chunk_index: document.chunk_index).delete_all
-            @chunk_model.create!(
+            attributes = {
               owner_type: document.owner_type,
               owner_id: document.owner_id,
               owner_model_name: document.owner_model_name,
@@ -29,15 +33,44 @@ module Maglev
               content: document.content,
               content_checksum: document.content_checksum,
               embedding_model: document.embedding_model,
+              index_version: document.index_version,
               embedding: document.embedding
-            )
+            }
+            if source_metadata_columns?
+              attributes[:source_identity] = document.source_identity
+              attributes[:source_type] = document.source_type
+              attributes[:tenant_id] = document.tenant_id
+            end
+            @chunk_model.create!(attributes)
           end
         end
       end
 
+      def fetch(ids:)
+        ids.to_a.filter_map do |id|
+          owner_type, owner_id, source, chunk_index = parse_id(id)
+          row = @chunk_model.find_by(owner_type: owner_type, owner_id: owner_id, source: source, chunk_index: chunk_index)
+          document_for(row) if row
+        end
+      end
+
+      def replace_owner(owner_type:, owner_id:, documents:)
+        staged = stage(documents)
+        unless staged.all? { |attributes| attributes[:owner_type] == owner_type && attributes[:owner_id] == owner_id }
+          raise ArgumentError, "replacement documents must match the requested owner"
+        end
+
+        @chunk_model.transaction do
+          lock_owner(owner_type, owner_id)
+          @chunk_model.where(owner_type: owner_type, owner_id: owner_id).delete_all
+          staged.each { |attributes| @chunk_model.create!(attributes) }
+        end
+      end
+
       def search(vector:, filters:, limit:)
-        scope = filters.reduce(@chunk_model.all) do |current_scope, (key, value)|
-          current_scope.where(key => value)
+        scope = MetadataFilter.coerce(filters).reduce(@chunk_model.all) do |current_scope, (key, value)|
+          column = {owner_ids: :owner_id, source_types: :source_type}.fetch(key, key)
+          current_scope.where(column => value)
         end
         scope.nearest_neighbors(:embedding, vector, distance: "cosine")
           .first(limit)
@@ -46,13 +79,16 @@ module Maglev
 
       def delete(ids:)
         ids.each do |id|
-          owner_type, owner_id, source, chunk_index = id.split(":", 4)
+          owner_type, owner_id, source, chunk_index = parse_id(id)
           @chunk_model.where(owner_type: owner_type, owner_id: owner_id, source: source, chunk_index: chunk_index).delete_all
         end
       end
 
       def delete_by_owner(owner_type:, owner_id:)
-        @chunk_model.where(owner_type: owner_type, owner_id: owner_id).delete_all
+        @chunk_model.transaction do
+          lock_owner(owner_type, owner_id)
+          @chunk_model.where(owner_type: owner_type, owner_id: owner_id).delete_all
+        end
       end
 
       def healthcheck
@@ -65,6 +101,44 @@ module Maglev
 
       private
 
+      def parse_id(id)
+        DocumentId.parse(id)
+      end
+
+      def stage(documents)
+        documents.map do |document|
+          attributes = {
+            owner_type: document.owner_type,
+            owner_id: document.owner_id,
+            owner_model_name: document.owner_model_name,
+            owner: document.owner,
+            source: document.source,
+            chunk_index: document.chunk_index,
+            content: document.content,
+            content_checksum: document.content_checksum,
+            embedding_model: document.embedding_model,
+            index_version: document.index_version,
+            embedding: document.embedding
+          }
+          if source_metadata_columns?
+            attributes[:source_identity] = document.source_identity
+            attributes[:source_type] = document.source_type
+            attributes[:tenant_id] = document.tenant_id
+          end
+          attributes
+        end
+      end
+
+      def source_metadata_columns?
+        !@chunk_model.respond_to?(:columns_hash) || @chunk_model.columns_hash.key?("source_identity")
+      end
+
+      def lock_owner(owner_type, owner_id)
+        key = Digest::SHA256.digest("#{owner_type}\0#{owner_id}").unpack1("q>")
+        quoted_key = @chunk_model.connection.quote(key)
+        @chunk_model.connection.execute("SELECT pg_advisory_xact_lock(#{quoted_key})")
+      end
+
       def identity_scope(document)
         @chunk_model.where(
           owner_type: document.owner_type,
@@ -76,19 +150,32 @@ module Maglev
 
       def document_for(row)
         Document.new(
+          id: DocumentId.build(owner_type: row.owner_type, owner_id: row.owner_id,
+            source_identity: ((row.respond_to?(:source_identity) && row.source_identity) ? row.source_identity : row.source),
+            chunk_index: row.chunk_index),
           owner_type: row.owner_type,
           owner_id: row.owner_id,
           owner_model_name: row.owner_model_name,
           source: row.source,
+          source_identity: row.respond_to?(:source_identity) ? row.source_identity : row.source,
+          source_type: row.respond_to?(:source_type) ? row.source_type : :snapshot,
+          tenant_id: row.respond_to?(:tenant_id) ? row.tenant_id : nil,
           chunk_index: row.chunk_index,
           content: row.content,
           content_checksum: row.content_checksum,
           embedding_model: row.embedding_model,
+          index_version: row.index_version,
           embedding: row.embedding,
           owner: row.owner,
-          distance: row.respond_to?(:neighbor_distance) ? row.neighbor_distance : row.distance
+          distance: distance_for(row)
         )
       end
+
+      def distance_for(row)
+        return row.neighbor_distance if row.respond_to?(:neighbor_distance)
+
+        row.distance if row.respond_to?(:distance)
+      end
     end
   end
 end

data/lib/maglev/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Maglev
-  VERSION = "0.1.1"
+  VERSION = "0.2.1"
 end

data/lib/maglev-rb.rb

@@ -0,0 +1,3 @@
+# frozen_string_literal: true
+
+require_relative "maglev"

data/lib/maglev.rb

@@ -7,19 +7,50 @@ require_relative "maglev/errors"
 require_relative "maglev/authorization"
 require_relative "maglev/provider_call"
 require_relative "maglev/knowledge_config"
+require_relative "maglev/knowledge_registry"
+require_relative "maglev/resource_config"
+require_relative "maglev/schema_snapshot"
+require_relative "maglev/registry"
+require_relative "maglev/query_ir"
+require_relative "maglev/query_validator"
+require_relative "maglev/planner_adapter"
+require_relative "maglev/planner"
+require_relative "maglev/planner_evaluation"
+require_relative "maglev/adapters/faraday_planner"
+require_relative "maglev/request"
+require_relative "maglev/routing_adapter"
+require_relative "maglev/router"
+require_relative "maglev/result"
+require_relative "maglev/hybrid_candidate_set"
+require_relative "maglev/hybrid_result"
+require_relative "maglev/hybrid_coordinator"
+require_relative "maglev/request_executor"
+require_relative "maglev/query_compiler"
+require_relative "maglev/structured_executor"
+require_relative "maglev/structured_result"
+require_relative "maglev/trace"
+require_relative "maglev/structured_evidence_builder"
+require_relative "maglev/structured_answer_composer"
 require_relative "maglev/snapshot"
+require_relative "maglev/snapshot_budget"
 require_relative "maglev/snapshot_builder"
+require_relative "maglev/source_fragment"
+require_relative "maglev/source_extractor"
 require_relative "maglev/chunker"
 require_relative "maglev/embedding_adapter"
-require_relative "maglev/adapters/ruby_llm_embedding"
+require_relative "maglev/index_identity"
+require_relative "maglev/adapters/faraday_client"
+require_relative "maglev/adapters/faraday_embedding"
 require_relative "maglev/generation_adapter"
-require_relative "maglev/adapters/ruby_llm_generation"
+require_relative "maglev/adapters/faraday_generation"
 require_relative "maglev/extracted_document"
 require_relative "maglev/attachment_extractor"
-require_relative "maglev/adapters/ruby_llm_attachment_extractor"
 require_relative "maglev/search_result"
+require_relative "maglev/retrieval_result"
 require_relative "maglev/vector_stores/base"
 require_relative "maglev/vector_stores/document"
+require_relative "maglev/vector_stores/metadata_filter"
+require_relative "maglev/vector_stores/document_id"
 require_relative "maglev/vector_stores/memory"
 require_relative "maglev/vector_stores/pgvector"
 require_relative "maglev/response"
@@ -27,6 +58,8 @@ require_relative "maglev/prompt_builder"
 require_relative "maglev/context_assembler"
 require_relative "maglev/context_preview"
 require_relative "maglev/indexer"
+require_relative "maglev/index_diagnostics"
+require_relative "maglev/index_state"
 require_relative "maglev/reindex_job"
 require_relative "maglev/retriever"
 require_relative "maglev/schema_compiler"