maglev-rb 0.1.1 → 0.2.1

data/lib/maglev/answerer.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 require_relative "configuration"
-require_relative "adapters/ruby_llm_generation"
+require_relative "adapters/faraday_generation"
 require_relative "authorization"
 require_relative "context_assembler"
 require_relative "prompt_builder"
@@ -16,26 +16,31 @@ module Maglev
       authorization: Authorization.new)
       @model_class = model_class
       @retriever = retriever
-      @generation_adapter = generation_adapter || Adapters::RubyLLMGeneration.new
+      @generation_adapter = generation_adapter || Adapters::FaradayGeneration.new
       @authorization = authorization
     end
 
-    def ask(question, limit:, owner: nil, user: nil)
+    def ask(question, limit:, owner: nil, user: nil, minimum_similarity: nil, chunks_per_owner: nil)
+      chunks_per_owner = validate_chunks_per_owner(chunks_per_owner)
       @authorization.authorize(record: owner, user: user) if owner
 
-      results = if user
-        @retriever.search(question, limit: limit, owner: owner, user: user)
-      else
-        @retriever.search(question, limit: limit, owner: owner)
-      end
-      results = results.select { |result| @authorization.authorized?(record: result.owner, user: user) }
-      return Response.insufficient_context(question: question) if results.empty?
+      outcome = @retriever.retrieval_outcome(
+        question,
+        limit: limit,
+        owner: owner,
+        user: user,
+        minimum_similarity: minimum_similarity,
+        chunks_per_owner: chunks_per_owner
+      )
+
+      results = outcome.results.select { |result| @authorization.authorized?(record: result.owner, user: user) }
+      return Response.insufficient_context(question: question, retrieval_metadata: rejection_metadata(outcome)) if results.empty?
 
       context = nil
       ActiveSupport::Notifications.instrument("maglev.query.retrieval", model: @model_class.name, result_count: results.size) do
         context = ContextAssembler.new.assemble(results)
       end
-      return Response.insufficient_context(question: question) if context.sources.empty?
+      return Response.insufficient_context(question: question, retrieval_metadata: rejection_metadata(outcome)) if context.sources.empty?
 
       prompt = PromptBuilder.new.build(question: question, context: context.text)
       text = nil
@@ -45,12 +50,25 @@ module Maglev
       Response.new(
         text: text,
         sources: context.sources,
-        metadata: context.metadata.merge(question: question, owner_scope: owner && owner_scope(owner))
+        metadata: context.metadata.merge(outcome.metadata).merge(question: question, owner_scope: owner && owner_scope(owner))
       )
     end
 
     private
 
+    def validate_chunks_per_owner(value)
+      return 1 if value.nil?
+
+      unless value.is_a?(Integer) && value.positive?
+        raise ArgumentError, "chunks_per_owner must be a positive Integer, got: #{value.inspect}"
+      end
+      value
+    end
+
+    def rejection_metadata(outcome)
+      outcome.metadata
+    end
+
     def owner_scope(owner)
       {owner_type: owner.class.name, owner_id: owner.respond_to?(:id) ? owner.id : nil}
     end

data/lib/maglev/chunker.rb

@@ -2,12 +2,26 @@
 
 module Maglev
   class Chunker
-    def initialize(max_characters: Maglev.configuration.chunk_size)
+    ALGORITHM_VERSION = "1"
+
+    def initialize(max_characters: Maglev.configuration.chunk_size, max_chunks: Maglev.configuration.snapshot_max_chunks)
+      unless max_characters.is_a?(Integer) && max_characters.positive?
+        raise ArgumentError, "max_characters must be positive"
+      end
+
       @max_characters = max_characters
+      if max_chunks && !max_chunks.is_a?(Integer)
+        raise ArgumentError, "max_chunks must be a positive Integer or nil"
+      end
+      if max_chunks && !max_chunks.positive?
+        raise ArgumentError, "max_chunks must be a positive Integer or nil"
+      end
+      @max_chunks = max_chunks
     end
 
     def call(text)
-      text.to_s.split(/\n{2,}/).flat_map { |part| split_part(part.strip) }.reject(&:empty?)
+      chunks = text.to_s.split(/\n{2,}/).flat_map { |part| split_part(part.strip) }.reject(&:empty?)
+      @max_chunks ? chunks.first(@max_chunks) : chunks
     end
 
     private
@@ -17,7 +31,7 @@ module Maglev
       return [part] if part.length <= @max_characters
 
       line_chunks = split_lines(part)
-      return line_chunks if line_chunks.length > 1
+      return line_chunks.flat_map { |chunk| hard_split(chunk) } if line_chunks.length > 1
 
       chunks = []
       current = +""
@@ -33,7 +47,7 @@ module Maglev
       end
 
       chunks << current unless current.empty?
-      chunks
+      chunks.flat_map { |chunk| hard_split(chunk) }
     end
 
     def split_lines(part)
@@ -54,5 +68,26 @@ module Maglev
       chunks << current unless current.empty?
       chunks
     end
+
+    def hard_split(text)
+      chunks = []
+      current = +""
+
+      text.scan(/\X/).each do |grapheme|
+        if grapheme.length > @max_characters
+          chunks << current unless current.empty?
+          chunks.concat(grapheme.chars.each_slice(@max_characters).map(&:join))
+          current = +""
+        elsif current.length + grapheme.length > @max_characters
+          chunks << current
+          current = +grapheme
+        else
+          current << grapheme
+        end
+      end
+
+      chunks << current unless current.empty?
+      chunks
+    end
   end
 end

data/lib/maglev/configuration.rb

@@ -20,7 +20,12 @@ module Maglev
       :attachment_extractor, :attachment_allowed_content_types,
       :attachment_max_bytes, :attachment_max_characters, :authorization_adapter,
       :provider_max_attempts, :provider_timeout, :source_redactor, :logger,
-      :vector_store
+      :vector_store, :embedding_adapter_id, :embedding_adapter_version,
+      :application_index_version, :minimum_similarity,
+      :snapshot_attribute_max_characters, :snapshot_related_record_max_characters,
+      :snapshot_max_characters, :snapshot_max_chunks,
+      :structured_query_role, :structured_query_executor_wrapper, :planner_adapter, :routing_adapter, :audit_sink,
+      :tenant_id_resolver
 
     def initialize
       @embedding_provider = ProviderConfiguration.new(
@@ -43,6 +48,18 @@ module Maglev
       @provider_max_attempts = 2
       @provider_timeout = 30
       @source_redactor = nil
+      @application_index_version = "1"
+      @minimum_similarity = nil
+      @snapshot_attribute_max_characters = 20_000
+      @snapshot_related_record_max_characters = 50_000
+      @snapshot_max_characters = 100_000
+      @snapshot_max_chunks = 100
+      @structured_query_timeout = 5
+      @structured_query_role = nil
+      @structured_query_executor_wrapper = nil
+      @structured_evidence_max_rows = 100
+      @structured_evidence_max_bytes = 32_768
+      @retrieval_max_candidates = 1_000
     end
 
     def embedding_provider
@@ -72,5 +89,53 @@ module Maglev
     def generation_model=(model)
       @generation_provider.model = model
     end
+
+    def tenant_id(record: nil, user: nil)
+      return unless @tenant_id_resolver
+
+      value = @tenant_id_resolver.call(record: record, user: user)
+      return if value.nil?
+      raise ArgumentError, "tenant id must be a non-empty String" unless value.is_a?(String) && !value.empty?
+
+      value
+    end
+
+    attr_reader :structured_query_timeout, :structured_evidence_max_rows, :structured_evidence_max_bytes,
+      :retrieval_max_candidates
+
+    def structured_query_timeout=(value)
+      raise ArgumentError, "structured_query_timeout must be positive" unless value.is_a?(Numeric) && value.positive?
+
+      @structured_query_timeout = value
+    end
+
+    %i[structured_evidence_max_rows structured_evidence_max_bytes].each do |name|
+      define_method(:"#{name}=") do |value|
+        raise ArgumentError, "#{name} must be a positive Integer" unless value.is_a?(Integer) && value.positive?
+
+        instance_variable_set(:"@#{name}", value)
+      end
+    end
+
+    def retrieval_max_candidates=(value)
+      raise ArgumentError, "retrieval_max_candidates must be a positive Integer" unless value.is_a?(Integer) && value.positive?
+
+      @retrieval_max_candidates = value
+    end
+
+    %i[
+      snapshot_attribute_max_characters
+      snapshot_related_record_max_characters
+      snapshot_max_characters
+      snapshot_max_chunks
+    ].each do |name|
+      define_method(:"#{name}=") do |value|
+        unless value.is_a?(Integer) && value.positive?
+          raise ArgumentError, "#{name} must be a positive Integer"
+        end
+
+        instance_variable_set(:"@#{name}", value)
+      end
+    end
   end
 end

data/lib/maglev/content_source_graph.rb

@@ -1,6 +1,6 @@
 # frozen_string_literal: true
 
-require_relative "reindex_job"
+require_relative "dependency_graph"
 
 module Maglev
   class ContentSourceGraph
@@ -14,41 +14,47 @@ module Maglev
         record = attachment.record
         return unless record && declared_attached?(record.class, attachment.name)
 
-        ReindexJob.perform_later(record.class.name, record.id) if record.id
+        DependencyGraph.reindex_record_and_dependents_for(record) if record.id
       end
 
       def reindex_rich_text_owner(rich_text)
         record = rich_text.record
         return unless record && declared_rich_text?(record.class, rich_text.name)
 
-        ReindexJob.perform_later(record.class.name, record.id) if record.id
+        DependencyGraph.reindex_record_and_dependents_for(record) if record.id
       end
 
       private
 
       def register_active_storage
         return unless defined?(ActiveStorage::Attachment)
-        return if ActiveStorage::Attachment.method_defined?(:maglev_reindex_attachment_owner)
 
         ActiveStorage::Attachment.class_eval do
-          def maglev_reindex_attachment_owner
-            Maglev::ContentSourceGraph.reindex_attachment_owner(self)
+          unless method_defined?(:maglev_reindex_attachment_owner)
+            def maglev_reindex_attachment_owner
+              Maglev::ContentSourceGraph.reindex_attachment_owner(self)
+            end
           end
 
-          after_commit :maglev_reindex_attachment_owner, on: %i[create destroy]
+          unless _commit_callbacks.any? { |callback| callback.filter == :maglev_reindex_attachment_owner }
+            after_commit :maglev_reindex_attachment_owner, on: %i[create destroy]
+          end
         end
       end
 
       def register_action_text
         return unless defined?(ActionText::RichText)
-        return if ActionText::RichText.method_defined?(:maglev_reindex_rich_text_owner)
 
         ActionText::RichText.class_eval do
-          def maglev_reindex_rich_text_owner
-            Maglev::ContentSourceGraph.reindex_rich_text_owner(self)
+          unless method_defined?(:maglev_reindex_rich_text_owner)
+            def maglev_reindex_rich_text_owner
+              Maglev::ContentSourceGraph.reindex_rich_text_owner(self)
+            end
           end
 
-          after_commit :maglev_reindex_rich_text_owner, on: %i[create update destroy]
+          unless _commit_callbacks.any? { |callback| callback.filter == :maglev_reindex_rich_text_owner }
+            after_commit :maglev_reindex_rich_text_owner, on: %i[create update destroy]
+          end
         end
       end
 

data/lib/maglev/dependency_graph.rb

@@ -4,35 +4,49 @@ require_relative "reindex_job"
 
 module Maglev
   class DependencyGraph
-    Edge = Struct.new(:owner_class, :related_class, :relation_name, :inverse)
+    Edge = Struct.new(:owner_class, :related_class, :relation_name, :inverse, :depth)
 
     class << self
+      def reset!
+        @edges = Hash.new { |hash, klass| hash[klass] = [] }
+      end
+
       def register(schema)
         schema.relations.each do |relation|
-          edge = Edge.new(schema.model_class, relation.related_class, relation.name, relation.inverse)
-          next if edges_for(relation.related_class).any? { |existing| same_edge?(existing, edge) }
+          edge = Edge.new(schema.model_class, relation.related_class, relation.name, relation.inverse, relation.depth)
+          existing = edges_for(relation.related_class).find { |candidate| same_edge?(candidate, edge) }
+          if existing
+            existing.depth = edge.depth
+            next
+          end
 
           edges_for(relation.related_class) << edge
           install_callbacks(relation.related_class)
         end
       end
 
-      def reindex_dependents_for(record)
-        dependent_owners = previous_dependent_owners(record)
-        edges_for(record.class).each do |edge|
-          dependent_owners.concat(owners_for(record, edge))
+      def unregister(owner_class)
+        edges.each_value do |registered_edges|
+          registered_edges.reject! { |edge| edge.owner_class == owner_class }
         end
+      end
 
-        dependent_owners.uniq { |owner| [owner.class.name, owner.id] }.each do |owner|
-          next unless owner.respond_to?(:id) && owner.id
+      def reindex_dependents_for(record)
+        enqueue(transitive_dependents_for(record))
+        clear_previous_dependent_owners(record)
+      end
 
-          ReindexJob.perform_later(owner.class.name, owner.id)
-        end
-        record.remove_instance_variable(:@maglev_previous_dependent_owners) if record.instance_variable_defined?(:@maglev_previous_dependent_owners)
+      def reindex_record_and_dependents_for(record)
+        enqueue([record, *transitive_dependents_for(record)])
+        clear_previous_dependent_owners(record)
       end
 
       def capture_previous_dependents_for(record)
-        owners = edges_for(record.class).flat_map { |edge| previous_owners_for(record, edge) }
+        owners = edges_for(record.class).flat_map do |edge|
+          next [] if edge.depth < 1
+
+          previous_owners_for(record, edge)
+        end
         record.instance_variable_set(:@maglev_previous_dependent_owners, owners)
       end
 
@@ -98,6 +112,51 @@ module Maglev
         end
       end
 
+      def transitive_dependents_for(record)
+        visited = {owner_key(record) => true}
+        queue = previous_dependent_owners(record).map { |owner| [owner, 1] }
+        queue.concat(direct_dependents_for(record, 0))
+        dependents = []
+
+        until queue.empty?
+          owner, distance = queue.shift
+          key = owner_key(owner)
+          next if visited[key]
+
+          visited[key] = true
+          dependents << owner
+          queue.concat(direct_dependents_for(owner, distance))
+        end
+
+        dependents
+      end
+
+      def direct_dependents_for(record, distance)
+        next_distance = distance + 1
+
+        edges_for(record.class).flat_map do |edge|
+          next [] if edge.depth < next_distance
+
+          owners_for(record, edge).map { |owner| [owner, next_distance] }
+        end
+      end
+
+      def enqueue(owners)
+        owners.uniq { |owner| owner_key(owner) }.each do |owner|
+          next unless owner.respond_to?(:id) && owner.id
+
+          ReindexJob.perform_later(owner.class.name, owner.id)
+        end
+      end
+
+      def owner_key(owner)
+        [owner.class.name, owner.respond_to?(:id) ? owner.id : owner.object_id]
+      end
+
+      def clear_previous_dependent_owners(record)
+        record.remove_instance_variable(:@maglev_previous_dependent_owners) if record.instance_variable_defined?(:@maglev_previous_dependent_owners)
+      end
+
       def previous_owners_for(record, edge)
         inverse_reflection = record.class.reflect_on_association(edge.inverse.to_sym)
         return owners_for(record, edge) unless inverse_reflection&.belongs_to?

data/lib/maglev/embedding_adapter.rb

@@ -1,7 +1,17 @@
 # frozen_string_literal: true
 
+require_relative "errors"
+
 module Maglev
   class EmbeddingAdapter
+    def maglev_adapter_id
+      raise ConfigurationError, "embedding adapter ID must be implemented by concrete adapters"
+    end
+
+    def maglev_adapter_version
+      raise ConfigurationError, "embedding adapter version must be implemented by concrete adapters"
+    end
+
     def embed(_text)
       raise NotImplementedError, "#{self.class.name} must implement #embed"
     end

data/lib/maglev/hybrid_candidate_set.rb

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+module Maglev
+  class HybridCandidateSet
+    DEFAULT_LIMIT = 100
+
+    attr_reader :model_class, :ids, :tenant_id, :trace_id
+
+    def initialize(model_class:, ids:, tenant_id:, trace_id:, limit: DEFAULT_LIMIT)
+      raise ArgumentError, "candidate limit must be a positive Integer" unless limit.is_a?(Integer) && limit.positive?
+      raise ConfigurationError, "hybrid candidate set exceeds #{limit} ids" if ids.size > limit
+
+      primary_key = model_class.primary_key
+      type = model_class.type_for_attribute(primary_key)
+      cast_ids = ids.map { |id| type.cast(id) }
+      raise ConfigurationError, "hybrid candidate set contains an invalid primary key" if cast_ids.any?(&:nil?)
+
+      @model_class = model_class
+      @ids = cast_ids.uniq.freeze
+      @tenant_id = tenant_id&.to_s&.freeze
+      @trace_id = trace_id.to_s.freeze
+      freeze
+    end
+  end
+end

data/lib/maglev/hybrid_coordinator.rb

@@ -0,0 +1,112 @@
+# frozen_string_literal: true
+
+require "securerandom"
+
+module Maglev
+  class HybridCoordinator
+    SHAPES = %i[structured_first rag_first].freeze
+
+    def initialize(retriever_factory:)
+      @retriever_factory = retriever_factory
+    end
+
+    def call(request, decision)
+      shape = request.options[:hybrid_plan]&.to_sym
+      raise ConfigurationError, "a fixed hybrid plan (structured_first or rag_first) is required" unless SHAPES.include?(shape)
+
+      entry = hybrid_entry(request)
+      relation = authorized_relation(request, entry)
+      trace_id = SecureRandom.uuid
+      send(shape, request, decision, entry, relation, trace_id)
+    end
+
+    private
+
+    def hybrid_entry(request)
+      entries = request.resources.filter_map { |identifier| Registry.fetch(identifier) }
+        .select { |entry| entry.queryable && entry.knowledge }
+      raise ConfigurationError, "hybrid routing requires exactly one registered resource with structured and RAG capabilities" unless entries.one?
+
+      entries.first
+    end
+
+    def authorized_relation(request, entry)
+      relation = request.base_relation
+      relation ||= entry.model_class.all if entry.queryable.allow_unscoped_model_queries
+      unless relation && relation.klass == entry.model_class
+        raise ConfigurationError, "hybrid requests require a matching authorized base relation"
+      end
+      relation
+    end
+
+    def structured_first(request, decision, entry, relation, trace_id)
+      structured = structured_result(request, entry, relation)
+      return wrap_non_success(structured, decision, :structured_first, trace_id) unless structured.status == :succeeded
+      raise ConfigurationError, "hybrid structured stages require record plans" unless structured.kind == :relation
+
+      ids = structured.value.pluck(entry.model_class.primary_key)
+      candidates = candidate_set(entry, ids, request, trace_id)
+      retrieval = retrieve(request, entry, candidates: candidates, trace_id: trace_id)
+      evidence = [HybridEvidence.new(provenance: :structured, value: structured.evidence),
+        HybridEvidence.new(provenance: :rag, value: retrieval)].freeze
+      result(decision, trace_id, :structured_first, candidates.ids, evidence,
+        ["structured filter", "RAG within typed candidates"])
+    end
+
+    def rag_first(request, decision, entry, relation, trace_id)
+      retrieval = retrieve(request, entry, trace_id: trace_id)
+      owners = retrieval.selected.map(&:owner)
+      if owners.any? { |owner| !owner.is_a?(entry.model_class) }
+        raise ConfigurationError, "hybrid retrieval returned mixed model candidates"
+      end
+      candidates = candidate_set(entry, owners.map(&:id), request, trace_id)
+      candidate_relation = relation.where(entry.model_class.primary_key => candidates.ids)
+      structured = structured_result(request, entry, candidate_relation)
+      return wrap_non_success(structured, decision, :rag_first, trace_id) unless structured.status == :succeeded
+      raise ConfigurationError, "hybrid structured stages require record plans" unless structured.kind == :relation
+
+      records = structured.value.to_a
+      missing = candidates.ids.size - records.size
+      warnings = missing.positive? ? ["#{missing} stale, deleted, or unauthorized candidates were excluded."] : []
+      evidence = [HybridEvidence.new(provenance: :rag, value: retrieval),
+        HybridEvidence.new(provenance: :structured, value: structured.evidence)].freeze
+      result(decision, trace_id, :rag_first, records, evidence,
+        ["RAG candidate retrieval", "structured verification"], warnings: warnings)
+    end
+
+    def structured_result(request, entry, relation)
+      plan = Maglev.plan(request.question, resource: entry.identifier, resources: request.resources,
+        base_relation: relation, user: request.user, authorizer: request.options[:authorizer],
+        constraints: request.options.fetch(:constraints, {}),
+        adapter: request.options[:planner_adapter] || Maglev.configuration.planner_adapter)
+      Maglev.execute(plan)
+    end
+
+    def retrieve(request, entry, trace_id:, candidates: nil)
+      options = {limit: request.options.fetch(:limit, 10), user: request.user,
+                 minimum_similarity: request.options[:minimum_similarity],
+                 chunks_per_owner: request.options.fetch(:chunks_per_owner, 1), trace_id: trace_id}
+      options[:candidates] = candidates if candidates
+      @retriever_factory.call(entry.model_class).retrieve(request.question, **options)
+    end
+
+    def candidate_set(entry, ids, request, trace_id)
+      HybridCandidateSet.new(model_class: entry.model_class, ids: ids,
+        tenant_id: request.user && Maglev.configuration.tenant_id(user: request.user), trace_id: trace_id,
+        limit: request.options.fetch(:candidate_limit, HybridCandidateSet::DEFAULT_LIMIT))
+    end
+
+    def result(decision, trace_id, shape, records, evidence, operations, warnings: [])
+      Result.new(status: :succeeded, route: :hybrid, kind: :hybrid_answer,
+        value: HybridAnswer.new(records: records), evidence: evidence, warnings: warnings,
+        trace_id: trace_id, confidence: decision.confidence, reasons: decision.reasons,
+        metadata: {plan_shape: shape, operations: operations.freeze}.freeze)
+    end
+
+    def wrap_non_success(structured, decision, shape, trace_id)
+      Result.new(status: structured.status, route: :hybrid, kind: :none,
+        warnings: structured.warnings, trace_id: trace_id, confidence: decision.confidence,
+        reasons: decision.reasons, metadata: {plan_shape: shape}.freeze)
+    end
+  end
+end

data/lib/maglev/hybrid_result.rb

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+module Maglev
+  class HybridEvidence
+    attr_reader :provenance, :value
+
+    def initialize(provenance:, value:)
+      raise ArgumentError, "invalid hybrid evidence provenance" unless %i[structured rag].include?(provenance)
+
+      @provenance = provenance
+      @value = value
+      freeze
+    end
+  end
+
+  class HybridAnswer
+    attr_reader :records, :claims
+
+    def initialize(records:, claims: [])
+      @records = records.freeze
+      @claims = claims.freeze
+      freeze
+    end
+  end
+end