omni_agent 0.1.11 → 0.2.0

data/lib/generators/omni_agent/vector_store.rb

@@ -0,0 +1,113 @@
+module OmniAgent
+  module Generators
+    # Picks the vector store and vector size for the embeddings and chunks
+    # generators, and writes what each store needs. Expects `backend` and
+    # `dimensions` class options.
+    module VectorStore
+      BACKENDS = %w[pgvector sqlite mariadb ruby].freeze
+      PGVECTOR_INDEX_MAX_DIMENSIONS = 2000
+
+      private
+
+      def backend
+        @backend ||= options[:backend] || detect_backend
+      end
+
+      def detect_backend
+        case ActiveRecord::Base.connection_db_config.adapter
+        when /postg/ then "pgvector"
+        when /sqlite/ then "sqlite"
+        when /mysql|trilogy/
+          return "mariadb" if mariadb?
+
+          raise Thor::Error, "MySQL can't search vectors without HeatWave. Use Postgres, SQLite or MariaDB 11.8+, " \
+                             "or pass --backend=ruby, which loads every row on each search and only suits small tables."
+        else
+          raise Thor::Error, "Couldn't detect a vector backend for this database. Pass --backend=#{BACKENDS.join('|')}."
+        end
+      end
+
+      def mariadb?
+        ActiveRecord::Base.with_connection(&:mariadb?)
+      rescue ActiveRecord::ActiveRecordError => e
+        raise Thor::Error, "Couldn't connect to the database to tell MySQL from MariaDB (#{e.message}). " \
+                           "Create it first, or pass --backend=mariadb|ruby."
+      end
+
+      def dimensions
+        return if backend == "ruby"
+
+        @dimensions ||= options[:dimensions] || detect_dimensions
+      end
+
+      def detect_dimensions
+        OmniAgent.embed("dimension check").length
+      rescue StandardError => e
+        raise Thor::Error, "Couldn't call the embedding model to detect its dimensions (#{e.message}). Pass --dimensions=N."
+      end
+
+      # Migration lines that add a vector column (and its index) to a table.
+      def vector_column_lines(table, column)
+        case backend
+        when "pgvector"
+          lines = [ 'enable_extension "vector"', "add_column :#{table}, :#{column}, :vector, limit: #{dimensions}" ]
+          if dimensions <= PGVECTOR_INDEX_MAX_DIMENSIONS
+            lines << "add_index :#{table}, :#{column}, using: :hnsw, opclass: :vector_cosine_ops"
+          else
+            lines << "# pgvector can't index more than #{PGVECTOR_INDEX_MAX_DIMENSIONS} dimensions; searches scan the table."
+          end
+        when "sqlite"
+          [ "add_column :#{table}, :#{column}, :binary" ]
+        when "mariadb"
+          [
+            "add_column :#{table}, :#{column}, :vector, limit: #{dimensions}",
+            "# For a vector index, backfill first, then in a new migration:",
+            "#   change_column_null :#{table}, :#{column}, false",
+            "#   add_index :#{table}, :#{column}, type: :vector"
+          ]
+        when "ruby"
+          [ "add_column :#{table}, :#{column}, :json" ]
+        end
+      end
+
+      def migration_file(name, body_lines)
+        # Like Rails' generators: after the newest migration, even one created this same second.
+        newest = Dir[File.join(destination_root, "db", "migrate", "*.rb")].map { |path| File.basename(path).to_i }.max.to_i
+        number = [ Time.now.utc.strftime("%Y%m%d%H%M%S").to_i, newest + 1 ].max
+        create_file File.join("db", "migrate", "#{number}_#{name.underscore}.rb"), <<~RUBY
+          class #{name} < ActiveRecord::Migration[#{ActiveRecord::Migration.current_version}]
+            def change
+          #{body_lines.map { |line| line.empty? ? "" : "    #{line}" }.join("\n")}
+            end
+          end
+        RUBY
+      end
+
+      def create_sqlite_initializer
+        return unless backend == "sqlite"
+        return if File.exist?(File.join(destination_root, "config", "initializers", "neighbor.rb"))
+
+        create_file File.join("config", "initializers", "neighbor.rb"), "# Load the sqlite-vec extension\nNeighbor::SQLite.initialize!\n"
+      end
+
+      # has_embedding arguments the store needs.
+      def vector_store_options
+        if backend == "sqlite" then [ "dimensions: #{dimensions}" ]
+        elsif backend == "ruby" then [ "using: :ruby" ]
+        else []
+        end
+      end
+
+      def say_vector_store_setup
+        if backend == "ruby"
+          say "Warning: the Ruby backend loads every row on each search. Use it for small tables only " \
+              "(under ~10k rows), and move to pgvector, sqlite-vec or MariaDB as the table grows.", :yellow
+          say ""
+        else
+          say "Add the gems if you haven't:"
+          say "  bundle add neighbor#{backend == 'sqlite' ? ' sqlite-vec' : ''}"
+        end
+      end
+    end
+  end
+end

data/lib/omni_agent/agent.rb

@@ -1,5 +1,9 @@
+require "json"
+
 module OmniAgent
   class Agent
+    DEFAULT_SYSTEM_PROMPT = "You are a helpful assistant with access to local tools.".freeze
+
     attr_reader :provider
 
     module ImplicitRunEntrypoints
@@ -39,7 +43,7 @@ module OmniAgent
 
     class << self
       def provider(name, **options)
-        if configured_with_use_model?
+        if @configured_with_use_model == true
           raise OmniAgent::Error, "Cannot combine `provider` and `use_model` in the same agent. Use either `provider ..., model: ...` or `use_model ...`."
         end
 
@@ -52,7 +56,7 @@ module OmniAgent
       end
 
       def use_model(name)
-        if configured_provider_name || configured_provider_options.any?
+        if @provider_name || (@provider_options || {}).any?
           raise OmniAgent::Error, "Cannot combine `provider` and `use_model` in the same agent. Use either `provider ..., model: ...` or `use_model ...`."
         end
 
@@ -69,11 +73,22 @@ module OmniAgent
       end
 
       def tags(*tag_names)
-        return @configured_tags || [] if tag_names.empty?
+        return @configured_tags || inherited_setting(:tags) || [] if tag_names.empty?
 
         @configured_tags = (tags + normalize_tags(tag_names)).uniq
       end
 
+      # Declares the structured output the agent must produce, using the same
+      # DSL as Tool.input. The final response is parsed and validated against
+      # it and exposed as response.structured_output.
+      def output(&block)
+        @output_schema = OmniAgent::Schema.build(&block) if block_given?
+      end
+
+      def output_schema
+        @output_schema || inherited_setting(:output_schema)
+      end
+
       def run_aliases(*method_names)
         aliases = normalize_callbacks(:run_aliases, method_names)
 
@@ -90,13 +105,45 @@ module OmniAgent
         end
 
         tool_class = build_delegated_tool_class(agent_class, description: description, run_alias: run_alias, forward: forward)
-        delegated_tools_module.const_set(delegated_tool_const_name(as), tool_class)
+        add_built_tool(as, tool_class)
+      end
+
+      # Gives the agent a Search<Models> tool backed by `target`: a model with
+      # `has_embedding`, a meilisearch-rails model, or anything responding to
+      # `semantic_search(query, limit:)`. `scope` runs on the target with the
+      # run context, e.g. ->(context) { where(account_id: context[:account_id]) }.
+      def searches(target, as: nil, description: nil, scope: nil, limit: 5, embedder: "default")
+        unless OmniAgent::SemanticSearch.searchable?(target)
+          raise ArgumentError,
+                "#{target.name || target.inspect} can't be searched: add `has_embedding` to it, " \
+                "include Meilisearch::Rails, or define `self.semantic_search(query, limit:)`."
+        end
 
-        @delegated_tool_classes = configured_delegated_tool_classes + [ tool_class ]
+        plural = target.name.to_s.split("::").last.underscore.pluralize
+        tool_description = description || "Search #{plural.humanize(capitalize: false)} by meaning. Returns the closest matches with their ids."
+
+        tool_class = Class.new(OmniAgent::Tool) do
+          description tool_description
+          annotations read_only: true
+
+          input do
+            string :query, description: "What to look for, in natural language."
+          end
+
+          define_method(:execute) do |query:|
+            searched = scope ? target.instance_exec(context, &scope) : target
+            results = OmniAgent::SemanticSearch.call(searched, query, limit: limit, embedder: embedder)
+            next "No results." if results.empty?
+
+            results.map { |result| OmniAgent::Agent.__send__(:format_search_result, result, target) }.join("\n\n---\n\n")
+          end
+        end
+
+        add_built_tool(as || "search_#{plural}", tool_class)
       end
 
       def configured_delegated_tool_classes
-        @delegated_tool_classes || []
+        @delegated_tool_classes || inherited_setting(:configured_delegated_tool_classes) || []
       end
 
       def tool_classes
@@ -111,7 +158,10 @@ module OmniAgent
           []
         end
 
-        namespace_tools + configured_delegated_tool_classes
+        # The model calls tools by simple name, so a subclass tool or a later
+        # delegate_to with the same name replaces the inherited one.
+        delegated_tools = configured_delegated_tool_classes.reverse.uniq { |tool| tool_call_name(tool) }.reverse
+        (namespace_tools + delegated_tools + (inherited_setting(:tool_classes) || [])).uniq { |tool| tool_call_name(tool) }
       end
 
       def build_delegated_tool_class(agent_class, description:, run_alias:, forward:)
@@ -144,12 +194,22 @@ module OmniAgent
         )
       end
 
-      def configured_provider_name; @provider_name; end
-      def configured_provider_options; @provider_options || {}; end
-      def configured_model_options; @model_options || {}; end
-      def configured_with_use_model?; @configured_with_use_model == true; end
-      def configured_before_generation_callbacks; @before_generation_callbacks || []; end
-      def configured_after_generation_callbacks; @after_generation_callbacks || []; end
+      # Class-level settings fall back to the parent agent, so subclasses
+      # (e.g. of a stock agent) keep its provider, options, callbacks and tools.
+      # A class's own `use_model` means "default provider + this model", so it
+      # does not inherit the parent's provider name.
+      def configured_provider_name
+        return @provider_name if @provider_name
+        return nil if @configured_with_use_model == true
+
+        inherited_setting(:configured_provider_name)
+      end
+
+      def configured_provider_options; @provider_options || inherited_setting(:configured_provider_options) || {}; end
+      def configured_model_options; @model_options || inherited_setting(:configured_model_options) || {}; end
+      def configured_with_use_model?; @configured_with_use_model == true || inherited_setting(:configured_with_use_model?) == true; end
+      def configured_before_generation_callbacks; @before_generation_callbacks || inherited_setting(:configured_before_generation_callbacks) || []; end
+      def configured_after_generation_callbacks; @after_generation_callbacks || inherited_setting(:configured_after_generation_callbacks) || []; end
       def configured_tags; tags; end
 
       def inherited(subclass)
@@ -159,6 +219,34 @@ module OmniAgent
 
       private
 
+      def inherited_setting(reader)
+        return nil unless superclass < OmniAgent::Agent
+
+        superclass.public_send(reader)
+      end
+
+      def tool_call_name(tool)
+        tool.name ? tool.name.split("::").last : tool
+      end
+
+      def add_built_tool(as, tool_class)
+        delegated_tools_module.const_set(delegated_tool_const_name(as), tool_class)
+        @delegated_tool_classes = configured_delegated_tool_classes + [ tool_class ]
+      end
+
+      def format_search_result(result, target)
+        return target.format_search_chunk(result) if target.respond_to?(:format_search_chunk)
+        return result.to_s unless result.respond_to?(:attributes)
+
+        text = if result.respond_to?(:embedding_text)
+          result.embedding_text
+        else
+          result.attributes.select { |_, value| value.is_a?(String) }.map { |key, value| "#{key}: #{value}" }.join("\n")
+        end
+
+        "[#{result.class.name}##{result.id}] #{text}"
+      end
+
       def delegated_tools_module
         @delegated_tools_module ||= const_set(:DelegatedTools, Module.new)
       end
@@ -241,7 +329,16 @@ module OmniAgent
       sync_context_from_instance_variables(context)
 
       messages.replace(build_messages(input: input, history: context[:history]))
+      output_schema = self.class.output_schema
+      native_output = output_schema && provider_supports_structured_output?
+      chat_options = native_output ? @chat_options.merge(output_schema: output_schema) : @chat_options
+
       messages[0][:content] = system_prompt(context: context, prompt_method: prompt_method)
+      # Providers without native structured output (and non-strict schemas,
+      # which the model only treats as guidance) also get the schema in the prompt.
+      if output_schema && !(native_output && output_schema.strict_compatible?)
+        messages[0][:content] = "#{messages[0][:content]}\n\n#{structured_output_instructions(output_schema)}"
+      end
       initial_messages_count = messages.length - 1
 
       filtered_tools = tool_filter(tools: available_tools, agent_tags: self.class.tags)
@@ -256,10 +353,11 @@ module OmniAgent
                 "Increase OmniAgent.configuration.max_tool_iterations if more tool calls are expected."
         end
 
-        response = provider.chat(messages: messages, tools: filtered_tools, stream: stream, **@chat_options)
+        response = provider.chat(messages: messages, tools: filtered_tools, stream: stream, **chat_options)
 
         if response.content && !response.tool_calls?
           messages << { role: "assistant", content: response.content }
+          response.with_structured_output(parse_structured_output(response, output_schema)) if output_schema
           set_after_generation_state(response: response, messages: messages, initial_messages_count: initial_messages_count)
           run_after_generation_callbacks(input: input, context: context, messages: messages, response: response)
           sync_context_from_instance_variables(context)
@@ -340,6 +438,41 @@ module OmniAgent
       tools
     end
 
+    def provider_supports_structured_output?
+      provider.respond_to?(:supports_structured_output?) && provider.supports_structured_output?
+    end
+
+    def structured_output_instructions(schema)
+      "Respond only with a JSON object that matches this JSON schema, without any other text:\n" \
+        "#{JSON.pretty_generate(schema.json_schema)}"
+    end
+
+    def parse_structured_output(response, schema)
+      payload = JSON.parse(strip_json_code_fence(response.content.to_s), symbolize_names: true)
+      schema.parse_typed(payload)
+    rescue JSON::ParserError => e
+      raise_structured_output_error("returned invalid JSON for its output schema: #{e.message}", response)
+    rescue ArgumentError => e
+      raise_structured_output_error("output does not match its output schema: #{e.message}", response)
+    rescue StandardError => e
+      # e.g. a polymorphic `resolve: true` field whose record can't be found
+      raise_structured_output_error("output could not be resolved: #{e.class}: #{e.message}", response)
+    end
+
+    def raise_structured_output_error(reason, response)
+      raise OmniAgent::StructuredOutputError.new(
+        "#{self.class.name || 'Agent'} #{reason}",
+        content: response.content,
+        response: response
+      )
+    end
+
+    # Models without native structured output sometimes wrap JSON in a fence.
+    def strip_json_code_fence(content)
+      match = content.strip.match(/\A```(?:json)?\s*(.*?)\s*```\z/m)
+      match ? match[1] : content
+    end
+
     def build_messages(input:, history:)
       normalized_history = normalize_history_messages(history)
 
@@ -406,15 +539,7 @@ module OmniAgent
     end
 
     def resolve_provider(name, model)
-      provider_class = OmniAgent::Providers.registry[name.to_sym]
-
-      unless provider_class
-        known = OmniAgent::Providers.registry.keys.join(", ")
-        raise OmniAgent::UnknownProviderError,
-              "Unknown provider #{name.inspect}. Known providers: #{known}"
-      end
-
-      provider_class.new(model: model)
+      OmniAgent::Providers.fetch(name).new(model: model)
     end
 
     def run_before_generation_callbacks(input:, context:, messages:)
@@ -495,17 +620,8 @@ module OmniAgent
     end
 
     def system_prompt(context:, prompt_method: nil)
-      return "You are a helpful assistant with access to local tools." unless defined?(Rails)
-
-      class_name = self.class.name
-      return "You are a helpful assistant with access to local tools." if class_name.nil?
-
-      agent_dir = inflector_underscore(class_name)
-      base_file_path = Rails.root.join("app", "agents", agent_dir, "prompt.md.erb")
-      method_file_path = if prompt_method
-        prompt_method_name = inflector_underscore(prompt_method.to_s)
-        Rails.root.join("app", "agents", agent_dir, "#{prompt_method_name}.md.erb")
-      end
+      template_paths = prompt_template_paths(prompt_method)
+      return DEFAULT_SYSTEM_PROMPT if template_paths.empty?
 
       isolated_scope = Object.new
 
@@ -519,13 +635,26 @@ module OmniAgent
         isolated_scope.instance_variable_set(ivar, instance_variable_get(ivar))
       end
 
-      base_prompt = render_prompt_template(base_file_path, isolated_scope)
-      method_prompt = render_prompt_template(method_file_path, isolated_scope)
-
-      prompts = [ base_prompt, method_prompt ].compact.reject(&:empty?)
+      prompts = template_paths.map { |path| render_prompt_template(path, isolated_scope) }.compact.reject(&:empty?)
       return prompts.join("\n\n") if prompts.any?
 
-      "You are a helpful assistant with access to local tools."
+      DEFAULT_SYSTEM_PROMPT
+    end
+
+    # Ordered list of prompt templates to render; missing files are skipped.
+    def prompt_template_paths(prompt_method)
+      return [] unless defined?(Rails)
+
+      class_name = self.class.name
+      return [] if class_name.nil?
+
+      prompt_templates_in(Rails.root.join("app", "agents", inflector_underscore(class_name)), prompt_method)
+    end
+
+    def prompt_templates_in(directory, prompt_method)
+      paths = [ File.join(directory.to_s, "prompt.md.erb") ]
+      paths << File.join(directory.to_s, "#{inflector_underscore(prompt_method.to_s)}.md.erb") if prompt_method
+      paths
     end
 
     def render_prompt_template(file_path, isolated_scope)

data/lib/omni_agent/chunkable.rb

@@ -0,0 +1,181 @@
+module OmniAgent
+  # ActiveRecord macro that turns a document into searchable chunks. Extended
+  # into every model by the engine.
+  #
+  #   class Document < ApplicationRecord
+  #     has_one_attached :file
+  #     has_chunks :file
+  #   end
+  #
+  #   Document.where(account_id: 1).semantic_search("refund policy") # => DocumentChunk records
+  #
+  # The source is an ActiveStorage attachment, a text column, rich text or
+  # any method returning a String or IO. When it changes, OmniAgent::IngestJob
+  # loads it into text (Loaders), splits it (Chunkers), embeds the chunks and
+  # replaces the record's chunks. The chunk model (DocumentChunk by default)
+  # belongs to the document and uses `has_embedding :content`.
+  module Chunkable
+    EMBED_BATCH_SIZE = 100
+
+    def has_chunks(source, as: :auto, chunker: :recursive, size: 1000, overlap: 200, class_name: "#{name}Chunk", later: true)
+      raise ArgumentError, "has_chunks size: must be a positive Integer" unless size.is_a?(Integer) && size.positive?
+      raise ArgumentError, "has_chunks overlap: must be between 0 and size:" unless overlap.is_a?(Integer) && overlap >= 0 && overlap < size
+      raise ArgumentError, "has_chunks later: must be true, false or a Hash of job options" unless [ true, false ].include?(later) || later.is_a?(Hash)
+      raise ArgumentError, "#{name} already uses has_embedding. A model uses has_embedding or has_chunks, not both." if respond_to?(:embedding_options)
+
+      Chunkers.fetch(chunker)
+      Loaders.fetch(as) unless as.to_s == "auto"
+
+      class_attribute :chunk_options, instance_writer: false
+      self.chunk_options = {
+        source: source.to_sym,
+        loader: as,
+        chunker: chunker,
+        size: size,
+        overlap: overlap,
+        later: later != false,
+        job_options: later.is_a?(Hash) ? Embeddable.validate_job_options!(later, "later:") : {}
+      }
+
+      has_many :chunks, -> { order(:position) }, class_name: class_name, dependent: :delete_all
+
+      extend ClassMethods
+      include InstanceMethods
+
+      validate :validate_chunk_source_type, unless: -> { as.to_s == "auto" }
+      before_save :remember_chunk_source_change
+      after_commit :ingest_changed_chunks, on: %i[create update], if: -> { @chunk_source_changed }
+    end
+
+    module ClassMethods
+      # The closest chunks of the records in the current scope. Chunks that
+      # share a parent section (hierarchical) count once.
+      def semantic_search(query, limit: 5)
+        reflection = reflect_on_association(:chunks)
+        # ponytail: fetches 3x the limit so merged siblings still fill it; raise if hierarchical results come up short.
+        hits = reflection.klass.where(reflection.foreign_key => all.select(primary_key)).semantic_search(query, limit: limit * 3)
+
+        hits.uniq { |chunk| chunk.try(:context) ? [ chunk[reflection.foreign_key], chunk.context ] : chunk.id }.first(limit)
+      end
+
+      # How a `searches` tool shows a chunk: its document, heading path and
+      # text, or its parent section for hierarchical chunks.
+      def format_search_chunk(chunk)
+        label = [ "#{name}##{chunk[reflect_on_association(:chunks).foreign_key]}", *chunk.try(:metadata)&.dig("headings") ]
+        "[#{label.join(' › ')}] #{chunk.try(:context) || chunk.content}"
+      end
+
+      # Ingests records that have no chunks yet, or every record with
+      # force: true. Records no loader can read are skipped and returned as
+      # { id => error message }.
+      def ingest_all(force: false)
+        skipped = {}
+        (force ? all : all.where.missing(:chunks)).find_each do |record|
+          record.ingest!
+        rescue OmniAgent::UnsupportedDocumentError => e
+          skipped[record.id] = e.message
+        end
+        skipped
+      end
+    end
+
+    module InstanceMethods
+      # Loads, splits and embeds the source, then replaces this record's
+      # chunks. Chunks whose text didn't change keep their vector, so
+      # re-ingesting an edited document only embeds what's new.
+      def ingest!
+        chunk_class = chunks.klass
+        column = chunk_class.try(:embedding_options)&.dig(:column)
+        raise OmniAgent::Error, "#{chunk_class.name} needs `has_embedding :content` to store chunk vectors." unless column
+
+        pieces = chunk_pieces
+        vectors = chunks.where.not(column => nil).pluck(:content, column).to_h
+        (pieces.map { |piece| piece[:content] }.uniq - vectors.keys).each_slice(EMBED_BATCH_SIZE) do |batch|
+          vectors.merge!(batch.zip(OmniAgent.embed(batch)).to_h)
+        end
+
+        attributes = %i[content context metadata] & chunk_class.column_names.map(&:to_sym)
+        foreign_key = self.class.reflect_on_association(:chunks).foreign_key
+        rows = pieces.each_with_index.map do |piece, position|
+          piece.slice(*attributes).merge(foreign_key => id, position: position, column => vectors[piece[:content]])
+        end
+
+        transaction do
+          # Serializes concurrent ingests of this record, and raises
+          # RecordNotFound if it was deleted while embedding.
+          self.class.unscoped.lock.find(id)
+          chunks.delete_all
+          chunk_class.insert_all!(rows) if rows.any?
+        end
+        chunks.reset
+        rows.size
+      end
+
+      # The source as clean text, or nil when there's nothing to read.
+      def chunk_source_text
+        value = public_send(chunk_options[:source])
+        return if value.blank?
+
+        if value.respond_to?(:attached?)
+          value.blob.open { |file| Loaders.load(file, as: chunk_options[:loader], filename: value.filename.to_s, content_type: value.content_type) }
+        elsif value.respond_to?(:to_plain_text)
+          value.to_plain_text
+        else
+          Loaders.load(value.respond_to?(:read) ? value : StringIO.new(value.to_s), as: chunk_options[:loader])
+        end
+      end
+
+      private
+
+      def chunk_pieces
+        text = chunk_source_text
+        return [] if text.blank?
+
+        Chunkers.chunk(text, chunker: chunk_options[:chunker], size: chunk_options[:size], overlap: chunk_options[:overlap])
+      end
+
+      def chunk_attachment_changed?
+        respond_to?(:attachment_changes) && attachment_changes.key?(chunk_options[:source].to_s)
+      end
+
+      def remember_chunk_source_change
+        source = chunk_options[:source].to_s
+        # Other sources (rich text, methods) can't tell; ingest! re-embeds only changed chunks.
+        @chunk_source_changed = if self.class.try(:reflect_on_attachment, source) then chunk_attachment_changed?
+        elsif has_attribute?(source) then will_save_change_to_attribute?(source)
+        else true
+        end
+      end
+
+      def ingest_changed_chunks
+        @chunk_source_changed = false
+        upload_pending_attachment
+        return ingest! unless chunk_options[:later]
+
+        job_options = OmniAgent.configuration.embedding_job_options.merge(chunk_options[:job_options])
+        OmniAgent::IngestJob.set(**job_options).perform_later(self)
+      rescue OmniAgent::UnsupportedDocumentError => e
+        logger&.warn("[OmniAgent] Skipped chunking #{self.class.name}##{id}: #{e.message}")
+      end
+
+      # ActiveStorage uploads in its own after_commit, which runs after this one
+      # when has_chunks is declared first. Upload now, the way it does; it then
+      # finds nothing left to upload.
+      def upload_pending_attachment
+        source = chunk_options[:source].to_s
+        attachment_changes.delete(source).upload if chunk_attachment_changed? && attachment_changes[source].respond_to?(:upload)
+      end
+
+      # An enforced loader (as: :pdf) rejects other files at save, instead of
+      # failing later in the job.
+      def validate_chunk_source_type
+        loader = Loaders.fetch(chunk_options[:loader])
+        attachment = public_send(chunk_options[:source])
+        return unless loader.respond_to?(:handles?) && chunk_attachment_changed? && attachment.attached?
+        return if loader.handles?(filename: attachment.filename.to_s, content_type: attachment.content_type)
+
+        errors.add(chunk_options[:source], "must be a #{chunk_options[:loader].to_s.upcase} file")
+      end
+    end
+  end
+end

data/lib/omni_agent/chunkers/hierarchical.rb

@@ -0,0 +1,20 @@
+module OmniAgent
+  module Chunkers
+    # Small child chunks for precise matching, each carrying its parent
+    # section as context, so a search matches the child and the agent reads
+    # the parent. Parents are Markdown sections of up to `size * 4` characters.
+    module Hierarchical
+      PARENT_SIZE_FACTOR = 4
+
+      # ponytail: each child stores its parent's text, so parents are duplicated
+      # per child; store parents once with a parent_id if storage matters.
+      def self.call(text, size: 1000, overlap: 200)
+        Markdown.call(text, size: size * PARENT_SIZE_FACTOR, overlap: overlap).flat_map do |parent|
+          Recursive.call(parent[:content], size: size, overlap: overlap).map do |child|
+            { content: child, context: parent[:content], metadata: parent[:metadata] }
+          end
+        end
+      end
+    end
+  end
+end

data/lib/omni_agent/chunkers/markdown.rb

@@ -0,0 +1,41 @@
+module OmniAgent
+  module Chunkers
+    # One chunk per Markdown section, with its heading path as metadata
+    # ("headings" => ["Billing", "Refunds"]). Sections longer than `size`
+    # are split further with Recursive. Text without headings is one section.
+    module Markdown
+      HEADING = /\A(\#{1,6})\s+(.+?)\s*#*\s*\z/
+      FENCE = /\A\s*(```|~~~)/
+
+      def self.call(text, size: 1000, overlap: 200)
+        sections(text.to_s).flat_map do |headings, body|
+          Recursive.call(body, size: size, overlap: overlap).map do |content|
+            { content: content, metadata: headings.any? ? { "headings" => headings } : {} }
+          end
+        end
+      end
+
+      # [[headings, body], ...]: each section's body includes its own heading line.
+      def self.sections(text)
+        sections = [ [ [], +"" ] ]
+        path = []
+        in_fence = false
+
+        text.each_line do |line|
+          in_fence = !in_fence if line.match?(FENCE)
+          heading = !in_fence && line.chomp.match(HEADING)
+
+          if heading
+            level = heading[1].length
+            path = path.first(level - 1) + [ heading[2] ]
+            sections << [ path, +"" ]
+          end
+          sections.last[1] << line
+        end
+
+        # A heading with nothing under it lives on in its subsections' paths.
+        sections.reject { |headings, body| body.lines.drop(headings.any? ? 1 : 0).join.strip.empty? }
+      end
+    end
+  end
+end