omni_agent 0.1.11 → 0.2.0

data/lib/omni_agent/providers/openai.rb

@@ -12,13 +12,14 @@ module OmniAgent
               "Please add `gem 'openai'` to your Gemfile."
       end
 
-      def chat(messages:, tools: [], stream: nil, **options)
+      def chat(messages:, tools: [], stream: nil, output_schema: nil, **options)
         validate_messages!(messages, allowed_roles: %i[system user assistant tool])
 
         messages = messages.map do |msg|
           clean_msg = msg.reject { |_, v| v.nil? }
 
-          if !clean_msg.key?(:tool_calls) && !clean_msg[:content].is_a?(String)
+          # Arrays are OpenAI content parts (text, image_url, file) and pass through.
+          if !clean_msg.key?(:tool_calls) && !clean_msg[:content].is_a?(String) && !clean_msg[:content].is_a?(Array)
             clean_msg[:content] = ""
           end
 
@@ -42,6 +43,7 @@ module OmniAgent
           messages: messages
         }
         payload[:tools] = openai_tools if openai_tools.any?
+        payload[:response_format] = format_response_format(output_schema) if output_schema
         payload.merge!(options) if options.any?
 
         response = if stream
@@ -55,8 +57,26 @@ module OmniAgent
         raise OmniAgent::Error, "Error during #{provider_label} chat: #{e.message}"
       end
 
+      def embed(texts, model: nil)
+        response = with_retries do
+          client.embeddings.create(input: texts, model: model || default_embedding_model)
+        end
+
+        response.data.map(&:embedding)
+      rescue => e
+        raise OmniAgent::Error, "Error during #{provider_label} embed: #{e.message}"
+      end
+
+      def supports_structured_output?
+        true
+      end
+
       protected
 
+      def default_embedding_model
+        "text-embedding-3-small"
+      end
+
       def provider_label
         "OpenAI"
       end
@@ -109,6 +129,17 @@ module OmniAgent
         }
       end
 
+      def format_response_format(output_schema)
+        {
+          type: "json_schema",
+          json_schema: {
+            name: "structured_output",
+            schema: output_schema.json_schema,
+            strict: output_schema.strict_compatible?
+          }
+        }
+      end
+
       def parse_response(raw_response, raw_request)
         provider_raw_response = raw_response.respond_to?(:to_h) ? raw_response.to_h : raw_response
         choices = raw_response.respond_to?(:choices) ? raw_response.choices : provider_raw_response["choices"]

data/lib/omni_agent/providers/response.rb

@@ -1,7 +1,7 @@
 module OmniAgent
   module Providers
     class Response
-      attr_reader :content, :tool_calls, :raw_response, :raw_request, :generated_messages
+      attr_reader :content, :tool_calls, :raw_response, :raw_request, :generated_messages, :structured_output
 
       def initialize(content:, tool_calls: [], raw_response: nil, raw_request: nil, generated_messages: [])
         @content = content
@@ -23,6 +23,11 @@ module OmniAgent
         @generated_messages = Array(messages)
         self
       end
+
+      def with_structured_output(output)
+        @structured_output = output
+        self
+      end
     end
   end
 end

data/lib/omni_agent/providers.rb

@@ -9,5 +9,12 @@ module OmniAgent
         mock_judge: OmniAgent::Providers::MockJudge
       }
     end
+
+    def self.fetch(name)
+      registry.fetch(name.to_sym) do
+        raise OmniAgent::UnknownProviderError,
+              "Unknown provider #{name.inspect}. Known providers: #{registry.keys.join(', ')}"
+      end
+    end
   end
 end

data/lib/omni_agent/schema.rb

@@ -0,0 +1,200 @@
+module OmniAgent
+  # A typed object schema declared with the Tool::SchemaBuilder DSL. Tools use
+  # it to describe and validate their input; agents use it to describe and
+  # validate their structured output.
+  class Schema
+    STRICT_UNSUPPORTED_KEYWORDS = %i[minLength maxLength].freeze
+
+    TYPE_CHECKS = {
+      "string" => ->(value) { value.is_a?(String) },
+      "integer" => ->(value) { value.is_a?(Integer) },
+      "number" => ->(value) { value.is_a?(Numeric) },
+      "boolean" => ->(value) { value == true || value == false },
+      "array" => ->(value) { value.is_a?(Array) },
+      "object" => ->(value) { value.is_a?(Hash) }
+    }.freeze
+
+    attr_reader :properties, :required, :validators, :polymorphics
+
+    def self.build(&block)
+      builder = Tool::SchemaBuilder.new
+      builder.instance_eval(&block)
+
+      new(
+        properties: builder.properties,
+        required: builder.required_fields,
+        validators: builder.validators,
+        polymorphics: builder.polymorphics
+      )
+    end
+
+    def initialize(properties: {}, required: [], validators: {}, polymorphics: {})
+      @properties = properties
+      @required = required
+      @validators = validators
+      @polymorphics = polymorphics
+    end
+
+    def json_schema
+      {
+        type: "object",
+        properties: properties,
+        required: required,
+        additionalProperties: false
+      }
+    end
+
+    # Whether the schema satisfies the constraints of strict structured-output
+    # modes (e.g. OpenAI's `strict: true`): every property required, no
+    # free-form objects, and no keywords those modes reject.
+    def strict_compatible?
+      strict_object?(json_schema)
+    end
+
+    # Filters unknown keys, validates the rest, and returns a symbol-keyed hash.
+    # Raises ArgumentError when validation fails.
+    def parse(values)
+      parse_values(values, check_types: false)
+    end
+
+    # Like #parse, but top-level values must also match their declared JSON
+    # type, and nulls for optional fields are treated as absent.
+    def parse_typed(values)
+      parse_values(values, check_types: true)
+    end
+
+    private
+
+    def parse_values(values, check_types:)
+      raise ArgumentError, "expected an object, got #{values.class}" unless values.is_a?(Hash)
+
+      kwargs = values.transform_keys(&:to_sym)
+      valid_keys = properties.keys.map(&:to_sym)
+      filtered = kwargs.slice(*valid_keys)
+
+      validate_required!(filtered)
+      validate_types!(filtered) if check_types
+      validate_enums!(filtered)
+      validate_constraints!(filtered)
+      run_custom_validators!(filtered)
+      resolve_polymorphics!(filtered)
+
+      filtered
+    end
+
+    def strict_object?(schema)
+      return false unless schema[:additionalProperties] == false
+
+      nested = schema[:properties] || {}
+      return false unless nested.keys.map(&:to_s).sort == Array(schema[:required]).map(&:to_s).sort
+
+      nested.values.all? { |property| strict_property?(property) }
+    end
+
+    def strict_property?(property)
+      return false if STRICT_UNSUPPORTED_KEYWORDS.any? { |keyword| property.key?(keyword) }
+
+      case property[:type]
+      when "object" then strict_object?(property)
+      when "array" then strict_property?(property[:items] || {})
+      else true
+      end
+    end
+
+    def validate_required!(kwargs)
+      missing = required.map(&:to_sym) - kwargs.keys
+      return if missing.empty?
+
+      raise ArgumentError, "missing required argument(s): #{missing.join(', ')}"
+    end
+
+    def validate_types!(kwargs)
+      required_keys = required.map(&:to_sym)
+      kwargs.reject! { |key, value| value.nil? && !required_keys.include?(key) }
+
+      properties.each do |name, property|
+        key = name.to_sym
+        next unless kwargs.key?(key)
+
+        type_check = TYPE_CHECKS[property[:type]]
+        next if type_check.nil? || type_check.call(kwargs[key])
+
+        raise ArgumentError, "#{name} must be of type #{property[:type]}, got #{kwargs[key].inspect}"
+      end
+    end
+
+    def validate_enums!(kwargs)
+      properties.each do |name, property|
+        allowed_values = property[:enum]
+        next unless allowed_values
+
+        key = name.to_sym
+        next unless kwargs.key?(key)
+
+        unless allowed_values.include?(kwargs[key])
+          raise ArgumentError, "invalid value for #{name}: #{kwargs[key].inspect} (must be one of: #{allowed_values.join(', ')})"
+        end
+      end
+    end
+
+    def validate_constraints!(kwargs)
+      properties.each do |name, property|
+        key = name.to_sym
+        next unless kwargs.key?(key)
+
+        check_constraints!(name, property, kwargs[key])
+      end
+    end
+
+    def check_constraints!(name, property, value)
+      if property[:minLength] && value.is_a?(String) && value.length < property[:minLength]
+        raise ArgumentError, "#{name} must be at least #{property[:minLength]} characters"
+      end
+      if property[:maxLength] && value.is_a?(String) && value.length > property[:maxLength]
+        raise ArgumentError, "#{name} must be at most #{property[:maxLength]} characters"
+      end
+      if property[:pattern] && value.is_a?(String) && !value.match?(Regexp.new(property[:pattern]))
+        raise ArgumentError, "#{name} does not match required pattern #{property[:pattern]}"
+      end
+      if property[:minimum] && value.is_a?(Numeric) && value < property[:minimum]
+        raise ArgumentError, "#{name} must be >= #{property[:minimum]}"
+      end
+      if property[:maximum] && value.is_a?(Numeric) && value > property[:maximum]
+        raise ArgumentError, "#{name} must be <= #{property[:maximum]}"
+      end
+      if property[:minItems] && value.is_a?(Array) && value.length < property[:minItems]
+        raise ArgumentError, "#{name} must have at least #{property[:minItems]} items"
+      end
+      if property[:maxItems] && value.is_a?(Array) && value.length > property[:maxItems]
+        raise ArgumentError, "#{name} must have at most #{property[:maxItems]} items"
+      end
+    end
+
+    def run_custom_validators!(kwargs)
+      validators.each do |name, validator|
+        key = name.to_sym
+        next unless kwargs.key?(key)
+
+        result = validator.call(kwargs[key])
+        raise ArgumentError, "invalid value for #{name}" if result == false
+      end
+    end
+
+    def resolve_polymorphics!(kwargs)
+      polymorphics.each do |name, group|
+        next unless group[:resolve]
+
+        type_field = group[:type_field]
+        id_field = group[:id_field]
+        next unless kwargs.key?(type_field) && kwargs.key?(id_field)
+
+        type_value = kwargs.delete(type_field)
+        id_value = kwargs.delete(id_field)
+
+        klass = Object.const_get(type_value.to_s)
+        record = klass.find(id_value)
+        kwargs[name.to_sym] = record
+      end
+    end
+  end
+end

data/lib/omni_agent/semantic_search.rb

@@ -0,0 +1,45 @@
+module OmniAgent
+  # Runs a semantic search on anything `Agent.searches` accepts: a model or
+  # relation with `has_embedding` (or its own `semantic_search(query, limit:)`),
+  # or a meilisearch-rails model, searched with one of its index embedders.
+  module SemanticSearch
+    def self.searchable?(target)
+      target.respond_to?(:semantic_search) || meilisearch?(target)
+    end
+
+    def self.call(target, query, limit:, embedder: "default")
+      return target.semantic_search(query, limit: limit) if target.respond_to?(:semantic_search)
+      return meilisearch(target.all, query, limit: limit, embedder: embedder) if meilisearch?(target)
+
+      raise ArgumentError, "#{target.inspect} can't be searched"
+    end
+
+    def self.meilisearch?(target)
+      model = target.is_a?(Class) ? target : target.try(:klass)
+      defined?(::Meilisearch::Rails) && model.is_a?(Class) && model < ::Meilisearch::Rails
+    end
+
+    # Equality conditions of the relation become Meilisearch filters (those
+    # attributes must be filterable_attributes). Hits are then checked
+    # against the full relation, so no condition is ever silently skipped.
+    def self.meilisearch(relation, query, limit:, embedder:)
+      params = { limit: limit, hybrid: { embedder: embedder } }
+      filter = relation.where_values_hash.map { |attribute, value| meilisearch_condition(attribute, value) }
+      params[:filter] = filter if filter.any?
+
+      hits = relation.klass.ms_search(query, params).first(limit)
+      allowed = relation.where(relation.klass.primary_key => hits.map(&:id)).pluck(relation.klass.primary_key).to_set
+      hits.select { |hit| allowed.include?(hit.id) }
+    end
+
+    def self.meilisearch_condition(attribute, value)
+      case value
+      when nil then "#{attribute} IS NULL"
+      when Array then "#{attribute} IN [#{value.map(&:to_json).join(', ')}]"
+      else "#{attribute} = #{value.to_json}"
+      end
+    end
+
+    private_class_method :meilisearch, :meilisearch_condition
+  end
+end

data/lib/omni_agent/stock_agent.rb

@@ -0,0 +1,59 @@
+module OmniAgent
+  # Base class for stock agents: ready-made agents shipped inside the gem under
+  # OmniAgent::StockAgents. Host apps use them as-is or subclass them to declare
+  # their own `output` schema, extra prompt instructions, tools or model.
+  #
+  #   module OmniAgent::StockAgents
+  #     class Summarizer < OmniAgent::StockAgent
+  #       description "Summarizes documents"
+  #       use_model "gpt-4o-mini"
+  #     end
+  #   end
+  #
+  # Prompts are read from lib/omni_agent/stock_agents/<stock_name>/ (override
+  # with `prompts_path`) and rendered before any prompts the host app defines
+  # for its subclass in app/agents/<agent_name>/.
+  class StockAgent < Agent
+    class << self
+      def description(text = nil)
+        @description = text if text
+        @description || inherited_setting(:description) || "No description provided."
+      end
+
+      def stock_name(value = nil)
+        @stock_name = value.to_sym if value
+        @stock_name || (stock_definition? ? default_stock_name : inherited_setting(:stock_name))
+      end
+
+      def prompts_path(path = nil)
+        @prompts_path = path.to_s if path
+        @prompts_path || (stock_definition? ? File.join(OmniAgent::StockAgents.root, stock_name.to_s) : inherited_setting(:prompts_path))
+      end
+
+      private
+
+      # True for the agents defined by the gem itself, as opposed to host-app
+      # subclasses, which inherit the stock name and prompts of their parent.
+      def stock_definition?
+        name.to_s.start_with?("OmniAgent::StockAgents::")
+      end
+
+      def default_stock_name
+        name.split("::").last
+          .gsub(/([A-Z]+)([A-Z][a-z])/, "\\1_\\2")
+          .gsub(/([a-z\d])([A-Z])/, "\\1_\\2")
+          .downcase
+          .to_sym
+      end
+    end
+
+    private
+
+    def prompt_template_paths(prompt_method)
+      stock_directory = self.class.prompts_path
+      stock_templates = stock_directory ? prompt_templates_in(stock_directory, prompt_method) : []
+
+      stock_templates + super
+    end
+  end
+end

data/lib/omni_agent/stock_agents/ocr/prompt.md.erb

@@ -0,0 +1,6 @@
+You read documents from images and PDFs and transcribe them exactly.
+
+- When you receive several images or files, they are pages of the same document, in the order given.
+- Transcribe only what is visible. Never guess or invent text, numbers or dates; leave the field empty when it is illegible or missing.
+- Keep numbers, currency symbols, dates and identifiers exactly as written.
+- Any text sent alongside the images is an instruction from the user about what to extract.

data/lib/omni_agent/stock_agents/ocr.rb

@@ -0,0 +1,70 @@
+require "pathname"
+require "uri"
+
+module OmniAgent
+  module StockAgents
+    # Reads photos, scans and PDFs with a vision model. `run` takes one source
+    # or an array of them (pages of the same document, in order):
+    #
+    #   OCR.new.run(Pathname("receipt.jpg"))
+    #   OCR.new.run([ Pathname("page1.png"), params[:page2], record.scan ])
+    #   OCR.new.run([ "Only the totals table matters", URI("https://example.com/scan.png") ])
+    #
+    # Sources: Pathname, IO-like objects (File, StringIO, uploaded files),
+    # ActiveStorage attachments/blobs, and http(s) URIs for images. Strings are
+    # always instructions, never file paths, so user text can't read server files.
+    class OCR < OmniAgent::StockAgent
+      description "Extracts text and structured data from photos, scans and PDFs"
+      use_model "gpt-4o"
+
+      SIGNATURES = {
+        "image/png" => "\x89PNG".b,
+        "image/jpeg" => "\xFF\xD8\xFF".b,
+        "image/gif" => "GIF8".b,
+        "application/pdf" => "%PDF".b
+      }.freeze
+
+      # Default structured output. Host apps override it by subclassing
+      # and declaring their own `output do ... end` block.
+      output do
+        string :text, description: "Full text of the document, in reading order"
+      end
+
+      private
+
+      def build_messages(input:, history:)
+        # [input].flatten, not Array(input): Array() splits an IO into its lines.
+        super(input: input.is_a?(String) ? input : [ input ].flatten.map { |source| content_part(source) }, history: history)
+      end
+
+      def content_part(source)
+        case source
+        when String then { type: "text", text: source }
+        when URI::HTTP then { type: "image_url", image_url: { url: source.to_s } }
+        when Pathname then file_part(source.binread)
+        else
+          if source.respond_to?(:download) then file_part(source.download)
+          elsif source.respond_to?(:read) then file_part(source.read.tap { source.rewind if source.respond_to?(:rewind) })
+          else raise ArgumentError, "#{self.class.name} can't read a #{source.class}: pass a Pathname, IO, ActiveStorage attachment or URI"
+          end
+        end
+      end
+
+      def file_part(bytes)
+        type = media_type(bytes)
+        url = "data:#{type};base64,#{[ bytes ].pack('m0')}"
+        return { type: "file", file: { filename: "document.pdf", file_data: url } } if type == "application/pdf"
+
+        { type: "image_url", image_url: { url: url } }
+      end
+
+      def media_type(bytes)
+        bytes = bytes.b
+        return "image/webp" if bytes.start_with?("RIFF") && bytes[8, 4] == "WEBP"
+
+        SIGNATURES.find { |_type, magic| bytes.start_with?(magic) }&.first ||
+          raise(OmniAgent::Error, "#{self.class.name} reads PNG, JPEG, GIF, WEBP and PDF files only")
+      end
+    end
+  end
+end

data/lib/omni_agent/stock_agents.rb

@@ -0,0 +1,26 @@
+module OmniAgent
+  # Namespace for the stock (default) agents shipped with the gem. Each one is
+  # an OmniAgent::StockAgent subclass in lib/omni_agent/stock_agents/<name>.rb,
+  # with its prompt templates in lib/omni_agent/stock_agents/<name>/.
+  module StockAgents
+    def self.root
+      File.expand_path("stock_agents", __dir__)
+    end
+
+    def self.all
+      constants.sort.filter_map do |const_name|
+        const = const_get(const_name)
+        const if const.is_a?(Class) && const < OmniAgent::StockAgent
+      end
+    end
+
+    def self.find(name)
+      agent_class = all.find { |stock_agent| stock_agent.stock_name == name.to_sym }
+      return agent_class if agent_class
+
+      available = all.map(&:stock_name).join(", ")
+      raise OmniAgent::UnknownStockAgentError,
+            "Unknown stock agent #{name.inspect}. Available stock agents: #{available.empty? ? 'none' : available}"
+    end
+  end
+end

data/lib/omni_agent/tool.rb

@@ -47,38 +47,19 @@ module OmniAgent
       end
 
       def input(&block)
-        if block_given?
-          builder = SchemaBuilder.new
-          builder.instance_eval(&block)
-
-          @properties = builder.properties
-          @required = builder.required_fields
-          @validators = builder.validators
-          @polymorphics = builder.polymorphics
-        end
+        @input_schema = Schema.build(&block) if block_given?
+      end
+
+      def input_schema
+        @input_schema || Schema.new
       end
 
       def json_schema
-        {
-          type: "object",
-          properties: @properties || {},
-          required: @required || [],
-          additionalProperties: false
-        }
+        input_schema.json_schema
       end
 
       def parse_arguments(arguments_hash)
-        kwargs = arguments_hash.transform_keys(&:to_sym)
-        valid_keys = (@properties || {}).keys.map(&:to_sym)
-        filtered = kwargs.slice(*valid_keys)
-
-        validate_required!(filtered)
-        validate_enums!(filtered)
-        validate_constraints!(filtered)
-        run_custom_validators!(filtered)
-        resolve_polymorphics!(filtered)
-
-        filtered
+        input_schema.parse(arguments_hash)
       end
 
       def stops_generation(value = true)
@@ -91,87 +72,6 @@ module OmniAgent
 
       private
 
-      def validate_required!(kwargs)
-        missing = (@required || []).map(&:to_sym) - kwargs.keys
-        return if missing.empty?
-
-        raise ArgumentError, "missing required argument(s): #{missing.join(', ')}"
-      end
-
-      def validate_enums!(kwargs)
-        (@properties || {}).each do |name, property|
-          allowed_values = property[:enum]
-          next unless allowed_values
-
-          key = name.to_sym
-          next unless kwargs.key?(key)
-
-          unless allowed_values.include?(kwargs[key])
-            raise ArgumentError, "invalid value for #{name}: #{kwargs[key].inspect} (must be one of: #{allowed_values.join(', ')})"
-          end
-        end
-      end
-
-      def validate_constraints!(kwargs)
-        (@properties || {}).each do |name, property|
-          key = name.to_sym
-          next unless kwargs.key?(key)
-
-          check_constraints!(name, property, kwargs[key])
-        end
-      end
-
-      def check_constraints!(name, property, value)
-        if property[:minLength] && value.is_a?(String) && value.length < property[:minLength]
-          raise ArgumentError, "#{name} must be at least #{property[:minLength]} characters"
-        end
-        if property[:maxLength] && value.is_a?(String) && value.length > property[:maxLength]
-          raise ArgumentError, "#{name} must be at most #{property[:maxLength]} characters"
-        end
-        if property[:pattern] && value.is_a?(String) && !value.match?(Regexp.new(property[:pattern]))
-          raise ArgumentError, "#{name} does not match required pattern #{property[:pattern]}"
-        end
-        if property[:minimum] && value.is_a?(Numeric) && value < property[:minimum]
-          raise ArgumentError, "#{name} must be >= #{property[:minimum]}"
-        end
-        if property[:maximum] && value.is_a?(Numeric) && value > property[:maximum]
-          raise ArgumentError, "#{name} must be <= #{property[:maximum]}"
-        end
-        if property[:minItems] && value.is_a?(Array) && value.length < property[:minItems]
-          raise ArgumentError, "#{name} must have at least #{property[:minItems]} items"
-        end
-        if property[:maxItems] && value.is_a?(Array) && value.length > property[:maxItems]
-          raise ArgumentError, "#{name} must have at most #{property[:maxItems]} items"
-        end
-      end
-
-      def run_custom_validators!(kwargs)
-        (@validators || {}).each do |name, validator|
-          key = name.to_sym
-          next unless kwargs.key?(key)
-
-          result = validator.call(kwargs[key])
-          raise ArgumentError, "invalid value for #{name}" if result == false
-        end
-      end
-
-      def resolve_polymorphics!(kwargs)
-        (@polymorphics || {}).each do |name, group|
-          next unless group[:resolve]
-
-          type_field = group[:type_field]
-          id_field = group[:id_field]
-          next unless kwargs.key?(type_field) && kwargs.key?(id_field)
-
-          type_value = kwargs.delete(type_field)
-          id_value = kwargs.delete(id_field)
-
-          klass = Object.const_get(type_value.to_s)
-          record = klass.find(id_value)
-          kwargs[name.to_sym] = record
-        end
-      end
-
       def normalize_tags(tag_names)
         raise ArgumentError, "tags requires at least one tag" if tag_names.empty?
 

data/lib/omni_agent/version.rb

@@ -1,3 +1,3 @@
 module OmniAgent
-  VERSION = "0.1.11"
+  VERSION = "0.2.0"
 end

data/lib/omni_agent.rb

@@ -5,7 +5,8 @@ require "zeitwerk"
 loader = Zeitwerk::Loader.for_gem
 loader.inflector.inflect(
   "openai" => "OpenAI", "cli" => "CLI", "mcp" => "MCP",
-  "oauth" => "OAuth", "jwt_verifier" => "JWTVerifier", "http_client" => "HTTPClient"
+  "oauth" => "OAuth", "jwt_verifier" => "JWTVerifier", "http_client" => "HTTPClient",
+  "pdf" => "PDF", "html" => "HTML", "ocr" => "OCR"
 )
 loader.ignore(File.expand_path("generators", __dir__))
 loader.setup
@@ -26,5 +27,15 @@ module OmniAgent
     def configure
       yield(configuration)
     end
+
+    # Embeds a String into one vector, or an Array of Strings into one vector
+    # per text. Index and query with the same provider and model, or the
+    # vectors are not comparable.
+    def embed(input, provider: nil, model: nil)
+      provider_name = provider || configuration.embedding_provider || configuration.default_provider
+      vectors = Providers.fetch(provider_name).new.embed(Array(input), model: model || configuration.embedding_model)
+
+      input.is_a?(Array) ? vectors : vectors.first
+    end
   end
 end