activeagent 1.3.1 → 1.5.0

data/lib/active_agent/evals/diagnosis.rb

@@ -0,0 +1,258 @@
+# frozen_string_literal: true
+
+module ActiveAgent
+  module Evals
+    # Explains why a scenario did not pass and what would fix it.
+    #
+    # The fault is assigned from the replay's evidence, the most mechanical
+    # cause first, so a run that crashed is a `run_error` even if its empty
+    # answer would also have failed a content check:
+    #
+    #   run_error                — the run raised, or the agent returned nothing
+    #   tool_error               — a tool the agent called returned an error
+    #   missing_capability       — the agent said no tool covers the task
+    #   expected_tool_not_called — the scenario expects a tool the agent did not call
+    #   forbidden_content        — the answer contains a pattern the scenario forbids
+    #   missing_content          — the answer lacks a pattern the scenario expects
+    #   low_quality              — the answer scored below the threshold
+    #
+    # Each fault carries a recommendation written from the evidence; a Judge
+    # can replace it with one that names the tool to add (Runner does this).
+    # Returns nil for a passing result.
+    class Diagnosis
+      FAULTS = %w[
+        run_error tool_error missing_capability expected_tool_not_called
+        forbidden_content missing_content low_quality judge_unavailable
+      ].freeze
+
+      # Phrasings an agent uses when nothing in its toolset covers the task.
+      # "find" and "see" are deliberately absent: "I can't find any…" and
+      # "I don't see…" report a negative result, not a missing capability.
+      REFUSED_VERBS = "have|access|retrieve|look up|query|check|view|search"
+      CAPABILITY_REFUSALS = [
+        /\bI(?:'m| am)? (?:do not |don't |cannot |can't |unable to |not able to )(?:currently )?(?:#{REFUSED_VERBS})\b/i,
+        /\b(?:no|don't have (?:a|any)) tools? (?:is |are )?(?:available|that can|to)\b/i,
+        /\bnot (?:something|able|possible) (?:I|to) (?:can|am able to )?(?:do|access|retrieve|look up)\b/i,
+        /\bI (?:don't|do not) have (?:the ability|a way|access|visibility|the tools?)\b/i,
+        /\boutside (?:of )?(?:my|the) (?:capabilities|available tools|scope)\b/i,
+        /\bcan(?:'|no)t (?:be )?(?:done|determined|answered) with (?:the|my) (?:current|available) tools\b/i
+      ].freeze
+
+      Result = Struct.new(:fault, :summary, :recommendation, :evidence, keyword_init: true) do
+        def to_h
+          {
+            "fault" => fault,
+            "summary" => summary,
+            "recommendation" => recommendation,
+            "evidence" => evidence
+          }
+        end
+      end
+
+      # @param scenario [Scenario]
+      # @param replay [Replay]
+      # @param scores [Hash] criterion key => 0.0..1.0 (nil when unscorable)
+      # @param score [Float, nil] the mean score
+      # @param available_tools [Array<String>] tool names the agent could call
+      # @param threshold [Float] the pass threshold for `score`
+      # @param agent_name [String] how the recommendations refer to the agent
+      # @param judge_keys [Array] keys in `scores` a judge graded the answer on
+      #   (llm_judge criteria); `task_completion` always counts as one
+      def self.call(scenario:, replay:, scores:, score:, available_tools:, threshold: PASS_THRESHOLD, agent_name: "The agent",
+                    judge_keys: [])
+        new(scenario:, replay:, scores:, score:, available_tools:, threshold:, agent_name:, judge_keys:).call
+      end
+
+      def initialize(scenario:, replay:, scores:, score:, available_tools:, threshold: PASS_THRESHOLD, agent_name: "The agent",
+                     judge_keys: [])
+        @scenario = scenario
+        @replay = replay
+        @scores = scores || {}
+        @score = score
+        @available_tools = Array(available_tools).map(&:to_s)
+        @threshold = threshold
+        @agent_name = agent_name
+        @judge_keys = Array(judge_keys) | [ "task_completion" ]
+      end
+
+      def call
+        run_error || tool_error || missing_capability || expected_tool_not_called ||
+          forbidden_content || missing_content || low_quality
+      end
+
+      private
+
+      def answer
+        @replay.answer.to_s
+      end
+
+      def called_tools
+        @replay.tool_names
+      end
+
+      def agent
+        @agent_name
+      end
+
+      def run_error
+        if @replay.errored?
+          message = @replay.error.to_s
+          return result("run_error", "The run failed before #{agent.downcase} answered: #{message.truncate(200)}",
+                        run_error_recommendation(message), "error" => message.truncate(1_000))
+        end
+        return nil if answer.present?
+
+        result("run_error", "#{agent} returned an empty answer.",
+               "The provider returned no content. Check the model name is one the provider serves and that the " \
+               "output budget leaves room for an answer after the tool calls.",
+               "error" => "empty answer")
+      end
+
+      def run_error_recommendation(message)
+        case message
+        when /credentials|api.?key|access_token|unauthori[sz]ed|401/i
+          "Add credentials for the provider this model runs on before comparing it."
+        when /model.*(not found|does not exist|unknown|unsupported)|404/i
+          "The provider rejected the model name. Check the spelling against the provider's catalog, or prefix " \
+          "it with the provider (`ollama/qwen3:8b`) so it runs where it exists."
+        when /rate limit|429|overloaded|529/i
+          "The provider throttled the run. Re-run the failed scenarios; if it recurs, run fewer scenarios per " \
+          "batch or compare fewer models at once."
+        else
+          "Inspect the run's error. A failure here is infrastructure — it says nothing about the answer yet."
+        end
+      end
+
+      def tool_error
+        failed = @replay.failed_tool_calls
+        return nil if failed.empty?
+
+        names = failed.map { |call| call["name"] }.uniq
+        detail = failed.first["detail"].to_s.truncate(300)
+        result("tool_error", "Tool #{names.join(', ')} returned an error while answering.",
+               "Fix the failing tool before judging the answer: #{names.join(', ')} errored with \"#{detail}\". " \
+               "If the arguments look wrong, tighten the tool's parameter descriptions so the model calls it " \
+               "correctly; if the tool itself broke, fix its implementation.",
+               "tools" => names, "detail" => detail, "arguments" => failed.first["arguments"])
+      end
+
+      def missing_capability
+        return nil unless CAPABILITY_REFUSALS.any? { |pattern| answer.match?(pattern) }
+        return nil if called_tools.any? && @score.to_f >= @threshold
+
+        missing = @scenario.expected_tools - @available_tools
+        recommendation =
+          if missing.any?
+            "#{agent} said it cannot do this, and the expected tool(s) #{missing.join(', ')} are not in its " \
+            "toolset. Add or enable them."
+          elsif @available_tools.empty?
+            "#{agent} has no tools, so it can only answer from its instructions. Give it a tool that reads the " \
+            "data this task needs."
+          else
+            "None of the available tools (#{@available_tools.join(', ')}) covers this task. Add a tool that does, " \
+            "or, if one of them should, rewrite its description so the model recognises when to use it."
+          end
+
+        result("missing_capability", "#{agent} said it lacks the ability to perform this task.", recommendation,
+               "refusal" => refusal_excerpt, "tools_available" => @available_tools, "tools_called" => called_tools)
+      end
+
+      def refusal_excerpt
+        pattern = CAPABILITY_REFUSALS.find { |candidate| answer.match?(candidate) }
+        match = answer.match(pattern)
+        return nil unless match
+
+        answer[[ match.begin(0) - 80, 0 ].max, 260].to_s.strip
+      end
+
+      def expected_tool_not_called
+        expected = @scenario.expected_tools
+        return nil if expected.empty? || (expected & called_tools).any?
+
+        unavailable = expected - @available_tools
+        recommendation =
+          if unavailable.any?
+            "The scenario expects #{unavailable.join(', ')}, which #{agent.downcase} does not have. Enable the " \
+            "tool (or add the server that provides it) and re-run."
+          elsif called_tools.any?
+            "#{agent} answered with #{called_tools.uniq.join(', ')} instead of #{expected.join(', ')}. Sharpen " \
+            "both tools' descriptions so the model can tell them apart, or say in the instructions which tool " \
+            "answers this kind of task."
+          else
+            "#{expected.join(', ')} is available but #{agent.downcase} answered without calling any tool. Tell " \
+            "it in the instructions to prefer tool-backed answers for this kind of task, and check the tool's " \
+            "description says what it returns."
+          end
+
+        result("expected_tool_not_called",
+               "Expected #{expected.join(' or ')} to be called; #{agent.downcase} called " \
+               "#{called_tools.uniq.presence&.join(', ') || 'nothing'}.",
+               recommendation, "expected" => expected, "called" => called_tools, "unavailable" => unavailable)
+      end
+
+      def forbidden_content
+        matched = @scenario.forbidden_patterns.select { |pattern| Scorer.matches_pattern?(answer, pattern) }
+        return nil if matched.empty?
+
+        result("forbidden_content", "The answer contains content the scenario forbids: #{matched.join(', ')}.",
+               "Add an explicit instruction against \"#{matched.first}\" and, if the phrase comes from a tool " \
+               "result, filter it in the tool rather than relying on the model to omit it.",
+               "matched" => matched)
+      end
+
+      def missing_content
+        missing = @scenario.expected_patterns.reject { |pattern| Scorer.matches_pattern?(answer, pattern) }
+        return nil if missing.empty?
+
+        recommendation =
+          if called_tools.empty? && @available_tools.any?
+            "#{agent} answered without calling a tool, so it could not have found \"#{missing.first}\". " \
+            "Instruct it to use its tools for this kind of task."
+          else
+            "The answer never mentions \"#{missing.first}\". Check whether the tool result contained it — if it " \
+            "did, the instructions should ask for it explicitly; if not, the tool needs to return it."
+          end
+
+        result("missing_content", "The answer is missing expected content: #{missing.join(', ')}.", recommendation,
+               "missing" => missing)
+      end
+
+      # A judge grade — the implicit task_completion score, or the llm_judge
+      # criteria the evaluation configured — measures the answer itself, so
+      # its mean has to reach the threshold on its own. Rule checks (a tool
+      # was called, a phrase is present) cannot carry a badly graded answer.
+      def low_quality
+        grades = @scores.slice(*@judge_keys).compact
+        grade = grades.any? ? (grades.values.sum / grades.size).round(3) : nil
+        failed_grade = grade && grade < @threshold
+        return nil unless failed_grade || (@score && @score < @threshold)
+
+        weakest = (failed_grade ? grades : @scores.compact).min_by { |_, value| value }
+        summary = if failed_grade
+          "#{graded_label(grades)} scored #{grade.round(2)} against a pass threshold of #{@threshold}"
+        else
+          "Scored #{@score.round(2)} against a pass threshold of #{@threshold}"
+        end
+        summary += ", weakest on #{weakest.first} (#{weakest.last.round(2)})" if weakest
+        recommendation =
+          if weakest
+            "Read the answer against the #{weakest.first.to_s.humanize.downcase} criterion and adjust the " \
+            "instructions where it falls short. A criterion that keeps scoring low across scenarios points at " \
+            "the instructions; one that fails on one scenario points at that task's tooling."
+          else
+            "Compare this answer with a passing one for a similar scenario and adjust the instructions."
+          end
+
+        result("low_quality", "#{summary}.", recommendation, "scores" => @scores)
+      end
+
+      def graded_label(grades)
+        grades.keys == [ "task_completion" ] ? "Task completion" : "Judged quality"
+      end
+
+      def result(fault, summary, recommendation, evidence = {})
+        Result.new(fault: fault, summary: summary, recommendation: recommendation, evidence: evidence.compact)
+      end
+    end
+  end
+end

data/lib/active_agent/evals/judge.rb

@@ -0,0 +1,212 @@
+# frozen_string_literal: true
+
+module ActiveAgent
+  module Evals
+    # A second model that scores answers, refines recommendations, and picks a
+    # winner. The gem owns the prompts and the parsing; the caller supplies the
+    # one thing that differs per stack — how to get a completion:
+    #
+    #   judge = ActiveAgent::Evals::Judge.new(label: "claude-opus-5") do |instructions:, prompt:|
+    #     RubyLLM.chat(model: "claude-opus-5").with_instructions(instructions).ask(prompt).content
+    #   end
+    #
+    # Every method returns nil when the judge fails or answers unusably, so an
+    # evaluation degrades to rule scoring rather than aborting.
+    class Judge
+      SCORE_INSTRUCTIONS = "You are an impartial evaluation judge. Respond ONLY with JSON: " \
+                           '{"score": <float between 0.0 and 1.0>}'
+      RECOMMEND_INSTRUCTIONS = "You diagnose why an AI agent failed a task and recommend the fix. Respond ONLY with JSON."
+      VERDICT_INSTRUCTIONS = "You are an impartial evaluation judge comparing model cohorts. Respond ONLY with JSON."
+
+      attr_reader :label
+
+      # @param label [String] how reports name the judge (usually its model)
+      # @yieldparam instructions [String] the system prompt
+      # @yieldparam prompt [String] the user prompt
+      # @yieldreturn [String] the completion text
+      def initialize(label:, &generate)
+        raise ArgumentError, "Judge.new needs a block that returns the model's completion" unless generate
+
+        @label = label
+        @generate = generate
+      end
+
+      # Scores `answer` against one llm_judge criterion, 0.0..1.0.
+      def score_criterion(criterion:, prompt:, answer:)
+        return nil if answer.blank?
+
+        guidance = criterion.dig("config", "prompt").presence || criterion["key"].to_s.humanize
+        parse_score(ask(SCORE_INSTRUCTIONS, <<~PROMPT))
+          Criterion: #{guidance}
+
+          The user asked:
+          ---
+          #{prompt.to_s.truncate(1_500)}
+          ---
+
+          The agent answered:
+          ---
+          #{answer.to_s.truncate(4_000)}
+          ---
+
+          Score the answer against the criterion from 0.0 (fails completely) to 1.0 (fully satisfies).
+          Respond only with JSON: {"score": <float>}
+        PROMPT
+      end
+
+      # Scores how well `answer` accomplishes the scenario's task, 0.0..1.0 —
+      # the single-criterion judgement for a scenario with no criteria of its own.
+      def score_task(scenario:, answer:)
+        return nil if answer.blank?
+
+        parse_score(ask(SCORE_INSTRUCTIONS, <<~PROMPT))
+          A user asked an assistant:
+          ---
+          #{scenario.prompt}
+          ---
+          #{"Context for the evaluator: #{scenario.notes.truncate(300)}\n" if scenario.notes.present?}
+          The assistant answered:
+          ---
+          #{answer.to_s.truncate(4_000)}
+          ---
+
+          Score from 0.0 (the task was not done — a refusal, a guess, or an unrelated answer) to 1.0 (the task
+          was done with specific, tool-backed data and a clear next step for the user).
+          Respond only with JSON: {"score": <float>}
+        PROMPT
+      end
+
+      # Asks what to change so the scenario passes. Returns a hash with any of
+      # `recommendation`, `suggested_tool` (`{ "name", "description" }`) and
+      # `instruction_change`, or nil.
+      def recommend(scenario:, replay:, diagnosis:, available_tools: {}, instructions: nil)
+        roster = available_tools.to_h.map { |name, description| "- #{name}: #{description.to_s.truncate(160)}" }.join("\n")
+        calls = replay.tool_calls.map do |call|
+          "- #{call['name']}#{' (errored)' if call['error']}: #{call['arguments'].to_json.truncate(200)}"
+        end.join("\n")
+
+        parsed = parse_object(ask(RECOMMEND_INSTRUCTIONS, <<~PROMPT))
+          An AI agent failed one evaluation scenario. Recommend the fix.
+
+          Agent instructions:
+          ---
+          #{instructions.to_s.truncate(2_000).presence || '(no instructions configured)'}
+          ---
+
+          Tools available to the agent:
+          #{roster.presence || '(none)'}
+
+          Scenario (the user's message):
+          #{scenario.prompt}
+          #{"Expected tools: #{scenario.expected_tools.join(', ')}" if scenario.expected_tools.any?}
+          #{"Notes: #{scenario.notes.truncate(300)}" if scenario.notes.present?}
+
+          Tools the agent called:
+          #{calls.presence || '(none)'}
+
+          The agent's answer:
+          ---
+          #{replay.answer.to_s.truncate(2_500).presence || '(empty)'}
+          ---
+
+          Detected fault: #{diagnosis.fault} — #{diagnosis.summary}
+
+          Say what to change so this scenario passes. If the agent lacks a tool for the task, describe the
+          tool to add. If the tools suffice, say what to change in the instructions.
+          Respond ONLY with JSON:
+          {"recommendation": "<two sentences at most>",
+           "suggested_tool": {"name": "snake_case_name", "description": "what it returns"} or null,
+           "instruction_change": "<the sentence to add or change>" or null}
+        PROMPT
+
+        parsed = parsed&.slice("recommendation", "suggested_tool", "instruction_change")&.compact
+        return nil if parsed.blank?
+
+        %w[recommendation instruction_change].each do |key|
+          parsed.delete(key) unless parsed[key].is_a?(String) && parsed[key].present?
+        end
+        parsed["suggested_tool"] = suggested_tool(parsed["suggested_tool"]) if parsed.key?("suggested_tool")
+        parsed.compact.presence
+      end
+
+      # Picks the model that best accomplishes the agent's goals from the
+      # per-model summaries (Report#summary_by_model). Returns
+      # `{ "winner", "rationale" }` or nil; a winner that is not one of the
+      # compared models is discarded.
+      def verdict(summaries, instructions: nil)
+        lines = summaries.map do |label, stats|
+          faults = (stats["faults"] || {}).map { |fault, count| "#{fault}×#{count}" }.join(", ")
+          "#{label}: pass rate #{stats['pass_rate']}%, mean score #{stats['avg_score'] || 'n/a'}, " \
+            "avg latency #{stats['avg_duration_ms'] || 'n/a'}ms, cost $#{stats['cost'] || 'n/a'}" \
+            "#{", faults: #{faults}" if faults.present?}"
+        end
+
+        parsed = parse_object(ask(VERDICT_INSTRUCTIONS, <<~PROMPT))
+          An AI agent ran the same scenarios under several models. Its goals:
+          ---
+          #{instructions.to_s.truncate(1_000).presence || '(no instructions configured)'}
+          ---
+
+          Results per model:
+          #{lines.join("\n")}
+
+          Which model best accomplishes the agent's goals, weighing task completion first and cost and
+          latency second? Respond ONLY with JSON: {"winner": "<model>", "rationale": "<at most two sentences>"}
+        PROMPT
+
+        return nil unless parsed && summaries.key?(parsed["winner"])
+
+        { "winner" => parsed["winner"], "rationale" => parsed["rationale"].to_s }
+      end
+
+      private
+
+      # The judge is asked for `{ "name", "description" }`; a bare string is
+      # taken as the name, and anything else is dropped rather than rendered
+      # as an empty tool.
+      def suggested_tool(tool)
+        case tool
+        when Hash
+          if tool["name"].is_a?(String) && tool["name"].present?
+            { "name" => tool["name"], "description" => tool["description"].is_a?(String) ? tool["description"] : "" }
+          end
+        when String
+          { "name" => tool, "description" => "" } if tool.present?
+        end
+      end
+
+      def ask(instructions, prompt)
+        @generate.call(instructions: instructions, prompt: prompt).to_s
+      rescue StandardError => e
+        warn_failure(e)
+        nil
+      end
+
+      def warn_failure(error)
+        message = "[ActiveAgent::Evals] judge #{label} failed: #{error.class}: #{error.message}"
+        if defined?(Rails) && Rails.respond_to?(:logger) && Rails.logger
+          Rails.logger.warn(message)
+        else
+          warn(message)
+        end
+      end
+
+      def parse_score(content)
+        value = parse_object(content)&.dig("score")
+        return nil unless value.is_a?(Numeric) && value.finite?
+
+        value.to_f.clamp(0.0, 1.0)
+      end
+
+      def parse_object(content)
+        json = content.to_s[/\{.*\}/m]
+        return nil unless json
+
+        parsed = JSON.parse(json)
+        parsed.is_a?(Hash) ? parsed : nil
+      rescue JSON::ParserError
+        nil
+      end
+    end
+  end
+end

data/lib/active_agent/evals/model_spec.rb

@@ -0,0 +1,80 @@
+# frozen_string_literal: true
+
+module ActiveAgent
+  module Evals
+    # A candidate model for a comparison run, resolved from the string a user
+    # types. `label` is that string verbatim and keys the model's cohort in a
+    # report; `provider` and `model` are what the run executes under:
+    #
+    #   "anthropic/claude-sonnet-5"       → anthropic, claude-sonnet-5
+    #   "claude-haiku-4-5"                → anthropic (inferred), claude-haiku-4-5
+    #   "gpt-5-mini"                      → openai (inferred), gpt-5-mini
+    #   "qwen3:8b"                        → ollama (inferred from the tag), qwen3:8b
+    #   "meta-llama/llama-3.3-70b"        → openrouter, meta-llama/llama-3.3-70b
+    #   "openrouter/anthropic/claude-3"   → openrouter, anthropic/claude-3
+    #
+    # `providers` lists the names a leading path segment may name; a vendor
+    # prefix that is not one of them routes through openrouter when that is
+    # available and otherwise stays part of the model name. A bare name no
+    # inference rule recognises runs under `default_provider`.
+    class ModelSpec
+      DEFAULT_PROVIDERS = %w[openai anthropic ollama openrouter].freeze
+
+      DEFAULT_INFERENCE_RULES = [
+        [ /\Aclaude/i, "anthropic" ],
+        [ /\A(gpt-|o\d|chatgpt|text-embedding)/i, "openai" ],
+        [ /:/, "ollama" ]
+      ].freeze
+
+      attr_reader :label, :provider, :model
+
+      def self.parse(value, default_provider:, providers: DEFAULT_PROVIDERS, inference_rules: DEFAULT_INFERENCE_RULES)
+        raw = value.to_s.strip
+        raise ArgumentError, "model name is blank" if raw.blank?
+
+        head, rest = raw.split("/", 2)
+        if rest.present? && providers.include?(head)
+          new(label: raw, provider: head, model: rest)
+        elsif rest.present? && providers.include?("openrouter")
+          new(label: raw, provider: "openrouter", model: raw)
+        else
+          new(label: raw, provider: infer_provider(raw, default_provider, inference_rules, providers), model: raw)
+        end
+      end
+
+      # Parses a comma-separated string or an array, dropping blanks and
+      # duplicates by label.
+      def self.parse_all(values, **options)
+        values = values.to_s.split(",") unless values.is_a?(Array)
+        values.map { |value| value.to_s.strip }.reject(&:blank?).uniq.map { |value| parse(value, **options) }
+      end
+
+      # The provider a bare model name runs under. A rule whose provider the
+      # caller does not offer is skipped, so an app without Ollama does not
+      # route `name:tag` there.
+      def self.infer_provider(model, default_provider, inference_rules, providers)
+        rule = inference_rules.find { |pattern, provider| model.match?(pattern) && providers.include?(provider) }
+        (rule ? rule.last : default_provider).to_s
+      end
+
+      def initialize(label:, provider:, model:)
+        @label = label
+        @provider = provider.to_s
+        @model = model
+      end
+
+      def to_h
+        { "label" => label, "provider" => provider, "model" => model }
+      end
+
+      def ==(other)
+        other.is_a?(ModelSpec) && to_h == other.to_h
+      end
+      alias eql? ==
+
+      def hash
+        to_h.hash
+      end
+    end
+  end
+end

data/lib/active_agent/evals/publisher.rb

@@ -0,0 +1,76 @@
+# frozen_string_literal: true
+
+require "json"
+require "net/http"
+require "openssl"
+require "uri"
+
+module ActiveAgent
+  module Evals
+    # Publishes a completed report without replaying the agent. The caller must
+    # retain run_id when retrying: compatible collectors treat that identity as
+    # immutable within the authenticated account. Delivery is blocking and does
+    # not follow redirects with the account's bearer credential.
+    class Publisher
+      DEFAULT_ENDPOINT = "https://api.activeagents.ai/v1/evaluations"
+      MAX_BYTES = 2 * 1024 * 1024
+      class Error < StandardError; end
+
+      def initialize(api_key:, endpoint: DEFAULT_ENDPOINT, timeout: 10, open_timeout: 10)
+        @uri = URI.parse(endpoint.to_s)
+        unless @uri.is_a?(URI::HTTP) && @uri.host && !@uri.userinfo && !@uri.query && !@uri.fragment
+          raise ArgumentError, "Evaluation endpoint must be an HTTP(S) URL without credentials, query or fragment"
+        end
+        unless @uri.scheme == "https" || %w[localhost 127.0.0.1 ::1].include?(@uri.hostname)
+          raise ArgumentError, "Evaluation endpoint requires HTTPS except on loopback hosts"
+        end
+        raise ArgumentError, "Evaluation API key is required" if api_key.to_s.strip.empty?
+
+        @api_key = api_key.to_s
+        @timeout = Float(timeout)
+        @open_timeout = Float(open_timeout)
+        unless [ @timeout, @open_timeout ].all? { |value| value.finite? && value.positive? }
+          raise ArgumentError, "Evaluation delivery timeouts must be positive and finite"
+        end
+      rescue URI::InvalidURIError
+        raise ArgumentError, "Evaluation endpoint is not a valid URL"
+      end
+
+      # report may be a Report or its saved JSON hash. Full prompts, answers and
+      # tool results are included; applications should make publication opt-in.
+      def call(report:, run_id:, source:, agent_name:, suite:)
+        identities = { "run_id" => run_id, "source" => source, "agent_name" => agent_name, "suite" => suite }
+        identities.each do |key, value|
+          raise ArgumentError, "#{key} must be a nonempty string" unless value.is_a?(String) && !value.strip.empty?
+        end
+        body = JSON.generate(identities.merge("version" => 1, "report" => report.to_h))
+        raise Error, "Evaluation report exceeds the 2 MiB delivery limit; publish a smaller selection" if body.bytesize > MAX_BYTES
+
+        http = Net::HTTP.new(@uri.hostname, @uri.port)
+        http.use_ssl = @uri.scheme == "https"
+        http.open_timeout = @open_timeout
+        http.read_timeout = @timeout
+        http.write_timeout = @timeout
+        request = Net::HTTP::Post.new(@uri.request_uri)
+        request["Authorization"] = "Bearer #{@api_key}"
+        request["Content-Type"] = "application/json"
+        request["Accept"] = "application/json"
+        request.body = body
+        response = http.request(request)
+        unless %w[200 201].include?(response.code)
+          raise Error, "Evaluation delivery rejected (HTTP #{response.code}); retain the report and run_id for retry"
+        end
+
+        receipt = JSON.parse(response.body)
+        unless receipt.is_a?(Hash) && receipt["run_id"] == run_id && receipt["status"] == "complete" && receipt["id"] && receipt["evaluation_id"]
+          raise Error, "Evaluation collector returned an invalid completion receipt; retain the report and run_id for retry"
+        end
+        receipt
+      rescue JSON::ParserError
+        raise Error, "Evaluation collector returned invalid JSON; retain the report and run_id for retry"
+      rescue IOError, SocketError, SystemCallError, Timeout::Error, OpenSSL::SSL::SSLError => e
+        raise Error, "Evaluation delivery failed (#{e.class}); retain the report and run_id for retry"
+      end
+    end
+  end
+end