maglev-rb 0.1.1

data/lib/maglev/adapters/ruby_llm_provider.rb

@@ -0,0 +1,64 @@
+# frozen_string_literal: true
+
+require_relative "../errors"
+
+module Maglev
+  module Adapters
+    class RubyLLMProvider
+      def initialize(provider)
+        @provider = provider
+      end
+
+      def embed(text)
+        result = translate_errors do
+          context.embed(
+            text,
+            model: @provider.model,
+            provider: :openai,
+            assume_model_exists: true,
+            dimensions: @provider.dimensions
+          )
+        end
+        result.vectors.first.is_a?(Array) ? result.vectors.first : result.vectors
+      end
+
+      def generate(prompt)
+        response = translate_errors do
+          context.chat(
+            model: @provider.model,
+            provider: :openai,
+            assume_model_exists: true
+          ).ask(prompt)
+        end
+        response.respond_to?(:content) ? response.content : response.to_s
+      end
+
+      private
+
+      def context
+        @context ||= begin
+          require "ruby_llm"
+
+          RubyLLM.context do |configuration|
+            configuration.openai_api_base = @provider.url
+            configuration.openai_api_key = @provider.api_key
+            configuration.request_timeout = Maglev.configuration.provider_timeout
+            configuration.max_retries = 0
+            configuration.logger = Maglev.configuration.logger
+          end
+        end
+      end
+
+      def translate_errors
+        yield
+      rescue RubyLLM::RateLimitError, RubyLLM::OverloadedError,
+        RubyLLM::ServerError, RubyLLM::ServiceUnavailableError,
+        Faraday::TimeoutError, Faraday::ConnectionFailed => error
+        raise RetryableProviderError, error.message
+      rescue RubyLLM::Error, RubyLLM::ConfigurationError,
+        RubyLLM::ModelNotFoundError => error
+        raise PermanentProviderError, error.message
+      end
+    end
+  end
+end

data/lib/maglev/answerer.rb

@@ -0,0 +1,58 @@
+# frozen_string_literal: true
+
+require_relative "configuration"
+require_relative "adapters/ruby_llm_generation"
+require_relative "authorization"
+require_relative "context_assembler"
+require_relative "prompt_builder"
+require_relative "provider_call"
+require_relative "response"
+require_relative "retriever"
+
+module Maglev
+  class Answerer
+    def initialize(model_class, retriever: Retriever.new(model_class),
+      generation_adapter: Maglev.configuration.generation_adapter,
+      authorization: Authorization.new)
+      @model_class = model_class
+      @retriever = retriever
+      @generation_adapter = generation_adapter || Adapters::RubyLLMGeneration.new
+      @authorization = authorization
+    end
+
+    def ask(question, limit:, owner: nil, user: nil)
+      @authorization.authorize(record: owner, user: user) if owner
+
+      results = if user
+        @retriever.search(question, limit: limit, owner: owner, user: user)
+      else
+        @retriever.search(question, limit: limit, owner: owner)
+      end
+      results = results.select { |result| @authorization.authorized?(record: result.owner, user: user) }
+      return Response.insufficient_context(question: question) if results.empty?
+
+      context = nil
+      ActiveSupport::Notifications.instrument("maglev.query.retrieval", model: @model_class.name, result_count: results.size) do
+        context = ContextAssembler.new.assemble(results)
+      end
+      return Response.insufficient_context(question: question) if context.sources.empty?
+
+      prompt = PromptBuilder.new.build(question: question, context: context.text)
+      text = nil
+      ActiveSupport::Notifications.instrument("maglev.query.generation", model: @model_class.name) do
+        text = ProviderCall.new.call(operation: "generate") { @generation_adapter.generate(prompt) }
+      end
+      Response.new(
+        text: text,
+        sources: context.sources,
+        metadata: context.metadata.merge(question: question, owner_scope: owner && owner_scope(owner))
+      )
+    end
+
+    private
+
+    def owner_scope(owner)
+      {owner_type: owner.class.name, owner_id: owner.respond_to?(:id) ? owner.id : nil}
+    end
+  end
+end

data/lib/maglev/attachment_extractor.rb

@@ -0,0 +1,106 @@
+# frozen_string_literal: true
+
+require "cgi"
+
+require_relative "configuration"
+require_relative "extracted_document"
+
+module Maglev
+  class AttachmentExtractor
+    def initialize(allowed_content_types: Maglev.configuration.attachment_allowed_content_types,
+      max_bytes: Maglev.configuration.attachment_max_bytes,
+      max_characters: Maglev.configuration.attachment_max_characters)
+      @allowed_content_types = allowed_content_types
+      @max_bytes = max_bytes
+      @max_characters = max_characters
+    end
+
+    def extract(blob, source_name:)
+      source_identifier = source_identifier(source_name, blob)
+      metadata = metadata_for(blob)
+      return skipped(source_identifier, "unsupported_content_type", metadata) unless allowed?(blob)
+      return skipped(source_identifier, "size_limit", metadata) if too_large?(blob)
+
+      text = extract_text(blob)
+      text, truncated = truncate(text)
+      ExtractedDocument.extracted(
+        source_identifier: source_identifier,
+        text: text,
+        metadata: metadata.merge(strategy: "deterministic", truncated: truncated)
+      )
+    end
+
+    private
+
+    def source_identifier(source_name, blob)
+      "#{source_name}[blob:#{blob_identifier(blob)}]"
+    end
+
+    def blob_identifier(blob)
+      if blob.respond_to?(:id) && blob.id
+        blob.id
+      elsif blob.respond_to?(:key) && blob.key
+        blob.key
+      else
+        filename(blob)
+      end
+    end
+
+    def metadata_for(blob)
+      {
+        filename: filename(blob),
+        content_type: content_type(blob),
+        byte_size: byte_size(blob)
+      }
+    end
+
+    def filename(blob)
+      blob.respond_to?(:filename) ? blob.filename.to_s : "unknown"
+    end
+
+    def content_type(blob)
+      blob.respond_to?(:content_type) ? blob.content_type.to_s : ""
+    end
+
+    def byte_size(blob)
+      blob.respond_to?(:byte_size) ? blob.byte_size.to_i : 0
+    end
+
+    def allowed?(blob)
+      @allowed_content_types.include?(content_type(blob))
+    end
+
+    def too_large?(blob)
+      byte_size(blob) > @max_bytes
+    end
+
+    def extract_text(blob)
+      text = blob.download.to_s
+      html?(blob) ? sanitize_html(text) : text
+    end
+
+    def html?(blob)
+      %w[text/html application/xhtml+xml].include?(content_type(blob))
+    end
+
+    def sanitize_html(html)
+      without_scripts = html.gsub(%r{<script\b[^>]*>.*?</script>}im, " ")
+        .gsub(%r{<style\b[^>]*>.*?</style>}im, " ")
+      CGI.unescapeHTML(without_scripts.gsub(/<[^>]+>/, " ").squeeze(" ").strip)
+    end
+
+    def truncate(text)
+      return [text, false] if text.length <= @max_characters
+
+      [text[0, @max_characters], true]
+    end
+
+    def skipped(source_identifier, reason, metadata)
+      ExtractedDocument.skipped(
+        source_identifier: source_identifier,
+        reason: reason,
+        metadata: metadata.merge(strategy: "deterministic")
+      )
+    end
+  end
+end

data/lib/maglev/authorization.rb

@@ -0,0 +1,43 @@
+# frozen_string_literal: true
+
+require_relative "configuration"
+require_relative "errors"
+
+module Maglev
+  class Authorization
+    def initialize(adapter: Maglev.configuration.authorization_adapter)
+      @adapter = adapter
+    end
+
+    def configured?
+      !@adapter.nil?
+    end
+
+    def scope(model:, user:)
+      return model.all unless configured? && user
+
+      @adapter.scope(model: model, user: user)
+    end
+
+    def authorize(record:, user:)
+      return true unless configured? && user
+
+      result = @adapter.authorize(record: record, user: user)
+      raise AuthorizationError, "Maglev authorization denied #{record.class.name}##{record_id(record)}" if result == false
+
+      true
+    end
+
+    def authorized?(record:, user:)
+      authorize(record: record, user: user)
+    rescue AuthorizationError
+      false
+    end
+
+    private
+
+    def record_id(record)
+      record.respond_to?(:id) ? record.id : "unknown"
+    end
+  end
+end

data/lib/maglev/chunk.rb

@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+require "active_record"
+require "neighbor"
+
+module Maglev
+  class Chunk < ActiveRecord::Base
+    self.table_name = "maglev_chunks"
+
+    belongs_to :owner, polymorphic: true
+
+    has_neighbors :embedding
+  end
+end

data/lib/maglev/chunker.rb

@@ -0,0 +1,58 @@
+# frozen_string_literal: true
+
+module Maglev
+  class Chunker
+    def initialize(max_characters: Maglev.configuration.chunk_size)
+      @max_characters = max_characters
+    end
+
+    def call(text)
+      text.to_s.split(/\n{2,}/).flat_map { |part| split_part(part.strip) }.reject(&:empty?)
+    end
+
+    private
+
+    def split_part(part)
+      return [] if part.empty?
+      return [part] if part.length <= @max_characters
+
+      line_chunks = split_lines(part)
+      return line_chunks if line_chunks.length > 1
+
+      chunks = []
+      current = +""
+
+      part.split(/\s+/).each do |word|
+        candidate = current.empty? ? word : "#{current} #{word}"
+        if candidate.length > @max_characters && !current.empty?
+          chunks << current
+          current = word
+        else
+          current = candidate
+        end
+      end
+
+      chunks << current unless current.empty?
+      chunks
+    end
+
+    def split_lines(part)
+      chunks = []
+      current = +""
+
+      part.lines(chomp: true).each do |line|
+        candidate = current.empty? ? line : "#{current}\n#{line}"
+        if candidate.length > @max_characters && !current.empty?
+          chunks << current
+          chunks.concat(split_part(line))
+          current = +""
+        else
+          current = candidate
+        end
+      end
+
+      chunks << current unless current.empty?
+      chunks
+    end
+  end
+end

data/lib/maglev/configuration.rb

@@ -0,0 +1,76 @@
+# frozen_string_literal: true
+
+require_relative "provider_configuration"
+
+module Maglev
+  class << self
+    def configuration
+      @configuration ||= Configuration.new
+    end
+
+    def configure
+      yield configuration
+    end
+  end
+
+  class Configuration
+    attr_accessor :embedding_adapter, :chunk_size,
+      :generation_adapter, :context_max_characters,
+      :context_per_owner_characters, :explain_question, :max_relation_depth,
+      :attachment_extractor, :attachment_allowed_content_types,
+      :attachment_max_bytes, :attachment_max_characters, :authorization_adapter,
+      :provider_max_attempts, :provider_timeout, :source_redactor, :logger,
+      :vector_store
+
+    def initialize
+      @embedding_provider = ProviderConfiguration.new(
+        url: "https://api.openai.com/v1",
+        model: "text-embedding-3-small",
+        dimensions: 1536
+      )
+      @generation_provider = ProviderConfiguration.new(
+        url: "https://api.openai.com/v1",
+        model: "gpt-4.1-mini"
+      )
+      @chunk_size = 1000
+      @context_max_characters = 4000
+      @context_per_owner_characters = 1200
+      @explain_question = "Explain what the available knowledge says about this record."
+      @max_relation_depth = 3
+      @attachment_allowed_content_types = ["text/plain", "text/markdown", "text/html", "application/xhtml+xml"]
+      @attachment_max_bytes = 5 * 1024 * 1024
+      @attachment_max_characters = 20_000
+      @provider_max_attempts = 2
+      @provider_timeout = 30
+      @source_redactor = nil
+    end
+
+    def embedding_provider
+      yield @embedding_provider if block_given?
+      @embedding_provider
+    end
+
+    def generation_provider
+      yield @generation_provider if block_given?
+      @generation_provider
+    end
+
+    def embedding_model = @embedding_provider.model
+
+    def embedding_model=(model)
+      @embedding_provider.model = model
+    end
+
+    def embedding_dimensions = @embedding_provider.dimensions
+
+    def embedding_dimensions=(dimensions)
+      @embedding_provider.dimensions = dimensions
+    end
+
+    def generation_model = @generation_provider.model
+
+    def generation_model=(model)
+      @generation_provider.model = model
+    end
+  end
+end

data/lib/maglev/content_source_graph.rb

@@ -0,0 +1,66 @@
+# frozen_string_literal: true
+
+require_relative "reindex_job"
+
+module Maglev
+  class ContentSourceGraph
+    class << self
+      def register(config)
+        register_active_storage if config.attached_sources.any?
+        register_action_text if config.rich_text_sources.any?
+      end
+
+      def reindex_attachment_owner(attachment)
+        record = attachment.record
+        return unless record && declared_attached?(record.class, attachment.name)
+
+        ReindexJob.perform_later(record.class.name, record.id) if record.id
+      end
+
+      def reindex_rich_text_owner(rich_text)
+        record = rich_text.record
+        return unless record && declared_rich_text?(record.class, rich_text.name)
+
+        ReindexJob.perform_later(record.class.name, record.id) if record.id
+      end
+
+      private
+
+      def register_active_storage
+        return unless defined?(ActiveStorage::Attachment)
+        return if ActiveStorage::Attachment.method_defined?(:maglev_reindex_attachment_owner)
+
+        ActiveStorage::Attachment.class_eval do
+          def maglev_reindex_attachment_owner
+            Maglev::ContentSourceGraph.reindex_attachment_owner(self)
+          end
+
+          after_commit :maglev_reindex_attachment_owner, on: %i[create destroy]
+        end
+      end
+
+      def register_action_text
+        return unless defined?(ActionText::RichText)
+        return if ActionText::RichText.method_defined?(:maglev_reindex_rich_text_owner)
+
+        ActionText::RichText.class_eval do
+          def maglev_reindex_rich_text_owner
+            Maglev::ContentSourceGraph.reindex_rich_text_owner(self)
+          end
+
+          after_commit :maglev_reindex_rich_text_owner, on: %i[create update destroy]
+        end
+      end
+
+      def declared_attached?(klass, name)
+        config = klass.respond_to?(:maglev_config) ? klass.maglev_config : nil
+        config&.attached_sources&.any? { |source| source.name == name.to_s }
+      end
+
+      def declared_rich_text?(klass, name)
+        config = klass.respond_to?(:maglev_config) ? klass.maglev_config : nil
+        config&.rich_text_sources&.any? { |source| source.name == name.to_s }
+      end
+    end
+  end
+end

data/lib/maglev/context_assembler.rb

@@ -0,0 +1,94 @@
+# frozen_string_literal: true
+
+module Maglev
+  class ContextAssembler
+    Context = Struct.new(:text, :sources, :metadata)
+
+    def initialize(max_characters: nil, per_owner_characters: nil)
+      max_characters ||= Maglev.configuration.context_max_characters
+      per_owner_characters ||= Maglev.configuration.context_per_owner_characters
+      @max_characters = max_characters
+      @per_owner_characters = per_owner_characters
+    end
+
+    def assemble(results)
+      entries = []
+      sources = []
+      owner_characters = Hash.new(0)
+      text_length = 0
+
+      sorted_results(results).each do |result|
+        marker = "[S#{sources.length + 1}]"
+        content = redacted_content(result)
+        entry = entry_for(result, marker, content)
+        owner_key = owner_key(result.owner)
+
+        next if owner_characters[owner_key] + content.length > @per_owner_characters
+        next if text_length + separator_length(entries) + entry.length > @max_characters
+
+        separator_length = separator_length(entries)
+        entries << entry
+        text_length += separator_length + entry.length
+        owner_characters[owner_key] += content.length
+        sources << source_for(result, marker, content)
+      end
+
+      text = entries.join("\n")
+      Context.new(
+        text: text,
+        sources: sources,
+        metadata: {context_characters: text.length, source_count: sources.length}
+      )
+    end
+
+    private
+
+    def sorted_results(results)
+      results.sort_by { |result| result.distance || Float::INFINITY }
+    end
+
+    def entry_for(result, marker, content)
+      "#{marker} #{owner_label(result.owner)} chunk #{result.chunk_index} source: #{result.source}\n#{content}"
+    end
+
+    def source_for(result, marker, content)
+      {
+        marker: marker,
+        owner_type: owner_type(result.owner),
+        owner_id: owner_id(result.owner),
+        source: result.source,
+        chunk_index: result.chunk_index,
+        content: content,
+        distance: result.distance,
+        similarity: result.similarity
+      }
+    end
+
+    def redacted_content(result)
+      redactor = Maglev.configuration.source_redactor
+      return result.content unless redactor
+
+      redactor.call(result.content, result)
+    end
+
+    def separator_length(entries)
+      entries.empty? ? 0 : 1
+    end
+
+    def owner_key(owner)
+      [owner_type(owner), owner_id(owner), owner.object_id]
+    end
+
+    def owner_label(owner)
+      "#{owner_type(owner)}##{owner_id(owner)}"
+    end
+
+    def owner_type(owner)
+      owner.class.name
+    end
+
+    def owner_id(owner)
+      owner.respond_to?(:id) ? owner.id : owner
+    end
+  end
+end

data/lib/maglev/context_preview.rb

@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+module Maglev
+  class ContextPreview
+    attr_reader :text, :metadata
+
+    def initialize(text:, metadata:)
+      @text = text
+      @metadata = metadata.freeze
+      freeze
+    end
+  end
+end