activeagent 1.9.1 → 1.9.3

data/lib/active_agent/evals/comparison.rb

@@ -0,0 +1,169 @@
+# frozen_string_literal: true
+
+module ActiveAgent
+  module Evals
+    # Whether the gap between the models of a run is larger than the noise of
+    # its scenario sample: each model's pass rate with a confidence interval,
+    # and an interval for the difference between every pair of models.
+    #
+    # The scenario is the unit. A model's pass rate is the mean over its
+    # scenarios of the share of its results there that passed (1.0 or 0.0
+    # for one result per scenario), and the intervals come from resampling
+    # whole scenarios with replacement, +resamples+ times: the percentile
+    # bootstrap. A pair is compared on the scenarios both models ran, each
+    # resample drawing the same scenarios for both (a paired bootstrap of
+    # the per-scenario differences). A pair is decisive when its interval
+    # excludes zero.
+    #
+    # The resampling is seeded, so the same results always give the same
+    # intervals and a report renders the same page each time.
+    #
+    #   comparison = Comparison.new(results: report.results, models: report.models)
+    #   comparison.to_h["pairs"].first
+    #   # => { "models" => ["gpt-5-mini", "qwen3:8b"], "scenarios" => 42, "difference" => 7.1,
+    #   #      "interval" => [0.0, 16.7], "decisive" => false }
+    #   comparison.conclusion
+    #   # => "The pairwise difference interval includes zero: the ranking is not decisive evidence of superiority."
+    class Comparison
+      RESAMPLES = 5_000
+      CONFIDENCE = 0.95
+      SEED = 0
+      # Fewer scenarios than this give no interval: resampling one scenario
+      # only ever draws that scenario.
+      MINIMUM_SCENARIOS = 2
+
+      attr_reader :resamples, :confidence, :seed
+
+      # @param results [Array<Result>]
+      # @param models [Array<ModelSpec>] in the run's order, which breaks ties in the ranking
+      # @param resamples [Integer]
+      # @param confidence [Float] 0..1
+      # @param seed [Integer]
+      def initialize(results:, models:, resamples: RESAMPLES, confidence: CONFIDENCE, seed: SEED)
+        @labels = models.map(&:label)
+        @resamples = resamples
+        @confidence = confidence
+        @seed = seed
+        @outcomes = @labels.to_h do |label|
+          by_scenario = results.select { |result| result.label == label }.group_by { |result| result.scenario.key }
+          [ label, by_scenario.transform_values { |cohort| cohort.count(&:passed?).fdiv(cohort.size) } ]
+        end
+      end
+
+      # Per model label, in the run's order: `{ "scenarios", "pass_rate",
+      # "interval" }`, the pass rate and its interval in percent. The interval
+      # is nil below MINIMUM_SCENARIOS, and the pass rate nil for none.
+      def models
+        @models ||= @labels.to_h do |label|
+          values = @outcomes[label].values
+          [ label, {
+            "scenarios" => values.size,
+            "pass_rate" => percent(mean(values)),
+            "interval" => interval(values)
+          } ]
+        end
+      end
+
+      # The labels by pass rate, best first; ties keep the run's order.
+      def ranking
+        @ranking ||= @labels.sort_by.with_index { |label, index| [ -(models[label]["pass_rate"] || -1), index ] }
+      end
+
+      # One entry per pair of models, the better ranked first: `{ "models" =>
+      # [a, b], "scenarios", "difference", "interval", "decisive" }`, where
+      # "difference" is a's pass rate minus b's over the scenarios both ran,
+      # in percentage points.
+      def pairs
+        @pairs ||= ranking.combination(2).map do |a, b|
+          shared = @outcomes[a].keys & @outcomes[b].keys
+          differences = shared.map { |key| @outcomes[a][key] - @outcomes[b][key] }
+          range = interval(differences)
+
+          {
+            "models" => [ a, b ],
+            "scenarios" => shared.size,
+            "difference" => percent(mean(differences)),
+            "interval" => range,
+            "decisive" => !range.nil? && (range.first.positive? || range.last.negative?)
+          }
+        end
+      end
+
+      # The honest read of the ranking, in a sentence: which pairs differ by
+      # more than the noise, or that none does. nil for a single model.
+      def conclusion
+        return nil if pairs.empty?
+
+        compared, too_few = pairs.partition { |pair| pair["interval"] }
+        decisive, inconclusive = compared.partition { |pair| pair["decisive"] }
+        sentences =
+          if compared.empty?
+            [ "The models share too few scenarios to compare." ]
+          elsif decisive.empty?
+            [ "#{all_include_zero(compared.size)}: the ranking is not decisive evidence of superiority." ]
+          else
+            decisive.map { |pair| decisive_sentence(pair) } + (inconclusive.any? ? [ others_include_zero(inconclusive.size) ] : [])
+          end
+        sentences << "#{plural(too_few.size, 'pair')} share#{'s' if too_few.one?} too few scenarios to compare." if compared.any? && too_few.any?
+        sentences.join(" ")
+      end
+
+      # How the intervals were computed, for a caption.
+      def method_summary
+        { "statistic" => "pass rate", "interval" => "percentile bootstrap", "unit" => "scenario",
+          "resamples" => @resamples, "confidence" => @confidence, "seed" => @seed }
+      end
+
+      def to_h
+        { "method" => method_summary, "models" => models, "ranking" => ranking, "pairs" => pairs, "conclusion" => conclusion }
+      end
+
+      private
+
+      # The bootstrap interval of the mean of +values+, in percent, or nil
+      # below MINIMUM_SCENARIOS.
+      def interval(values)
+        return nil if values.size < MINIMUM_SCENARIOS
+
+        random = Random.new(@seed)
+        size = values.size
+        means = Array.new(@resamples) do
+          sum = 0.0
+          size.times { sum += values[random.rand(size)] }
+          sum / size
+        end
+        # A rational tail, so 95% leaves exactly 2.5: in floating point it
+        # is 2.5000000000000022, whose nearest rank is one too high.
+        tail = (1 - @confidence.rationalize) / 2 * 100
+        [ percent(Statistics.percentile(means, tail)), percent(Statistics.percentile(means, 100 - tail)) ]
+      end
+
+      def mean(values)
+        values.sum.fdiv(values.size) if values.any?
+      end
+
+      def percent(fraction)
+        (fraction * 100).round(1) if fraction
+      end
+
+      def all_include_zero(count)
+        count == 1 ? "The pairwise difference interval includes zero" : "All #{count} pairwise difference intervals include zero"
+      end
+
+      def others_include_zero(count)
+        count == 1 ? "The other pairwise interval includes zero." : "The other #{count} pairwise intervals include zero."
+      end
+
+      def decisive_sentence(pair)
+        better, worse = pair["difference"].positive? ? pair["models"] : pair["models"].reverse
+        low, high = pair["interval"].map(&:abs).sort
+        "#{better} passes more scenarios than #{worse}, by #{pair['difference'].abs} points " \
+          "(#{(@confidence * 100).round}% CI #{low} to #{high})."
+      end
+
+      def plural(count, word)
+        "#{count} #{word}#{'s' unless count == 1}"
+      end
+    end
+  end
+end

data/lib/active_agent/evals/publisher.rb

@@ -94,11 +94,7 @@ module ActiveAgent
       # report may be a Report or its saved JSON hash. Full prompts, answers and
       # tool results are included; applications should make publication opt-in.
       def call(report:, run_id:, source:, agent_name:, suite:)
-        identities = { "run_id" => run_id, "source" => source, "agent_name" => agent_name, "suite" => suite }
-        identities.each do |key, value|
-          raise ArgumentError, "#{key} must be a nonempty string" unless value.is_a?(String) && !value.strip.empty?
-        end
-        body = encode(identities.merge("version" => 1, "report" => report_hash(report)))
+        body = encode(self.class.envelope(report: report, run_id: run_id, source: source, agent_name: agent_name, suite: suite))
         raise Error, "Evaluation report exceeds the 2 MiB delivery limit; publish a smaller selection" if body.bytesize > MAX_BYTES
 
         deliver("retain the report and run_id for retry") do
@@ -143,6 +139,24 @@ module ActiveAgent
         end
       end
 
+      # The version-1 envelope a collector stores: the four identities, the
+      # version and the report's hash. What #call sends, and what a
+      # collector's export of a stored run returns, so the export can be
+      # posted again. report may be a Report or its saved JSON hash.
+      #
+      # Raises ArgumentError for an identity that is not a nonempty string,
+      # or a report that is neither.
+      def self.envelope(report:, run_id:, source:, agent_name:, suite:)
+        identities = { "run_id" => run_id, "source" => source, "agent_name" => agent_name, "suite" => suite }
+        identities.each do |key, value|
+          raise ArgumentError, "#{key} must be a nonempty string" unless value.is_a?(String) && !value.strip.empty?
+        end
+        hash = report.to_h if report.respond_to?(:to_h) && !report.nil? && !report.is_a?(Array)
+        raise ArgumentError, "report must be a Report or its saved JSON hash" unless hash.is_a?(Hash)
+
+        identities.merge("version" => 1, "report" => hash)
+      end
+
       private
 
       # Posts +body+ to the endpoint with the bearer key and the delivery timeouts.
@@ -176,13 +190,6 @@ module ActiveAgent
         raise Error.new("Evaluation delivery failed (#{e.class}); #{guidance}", retryable: true)
       end
 
-      def report_hash(report)
-        hash = report.to_h if report.respond_to?(:to_h) && !report.nil? && !report.is_a?(Array)
-        raise ArgumentError, "report must be a Report or its saved JSON hash" unless hash.is_a?(Hash)
-
-        hash
-      end
-
       # Returns the envelope as JSON. A report holding invalid UTF-8, NaN,
       # Infinity or nesting over JSON's depth limit raises a non-retryable
       # Error, without the generator's message, which can quote the report.

data/lib/active_agent/evals/report.rb

@@ -8,6 +8,7 @@ module ActiveAgent
     # Markdown, or a self-contained HTML page (ReportHtml).
     class Report
       include ReportHtml
+      include ReportMarkdown
 
       # The judge a verdict names when the framework ranked the models by
       # pass rate itself, no judge having been available to rule on them.
@@ -37,9 +38,12 @@ module ActiveAgent
       # @param release [Hash, nil] the release of the agent under evaluation,
       #   `{ "digest", "revision", "label" }`, so the report names the code
       #   it scored: in `to_h["release"]` and as a header chip.
+      # @param comparison [Hash, nil] a comparison already recorded for these
+      #   results (Comparison#to_h). A report rebuilt from a persisted run
+      #   renders the intervals that run recorded instead of resampling them.
       def initialize(results:, models:, judge: nil, instructions: nil, threshold: PASS_THRESHOLD, metadata: {},
                      tool_resolver: nil, agent_name: nil, links: {}, verdict: nil, judge_label: nil,
-                     judge_usage: nil, release: nil)
+                     judge_usage: nil, release: nil, comparison: nil)
         @results = results
         @models = models
         @judge = judge
@@ -53,6 +57,7 @@ module ActiveAgent
         @recorded_verdict = verdict.is_a?(Hash) ? verdict.to_h.stringify_keys.presence : nil
         @run_judge_usage = judge_usage.is_a?(Hash) ? judge_usage.to_h.stringify_keys.presence : nil
         @release = release.is_a?(Hash) ? release.to_h.stringify_keys.compact.presence : nil
+        @recorded_comparison = comparison.is_a?(Hash) ? comparison.to_h.deep_stringify_keys.presence : nil
       end
 
       # The release of the agent this report scored, `{ "digest",
@@ -64,7 +69,8 @@ module ActiveAgent
       end
 
       # Per model, keyed by label: scenario count, passes, errors, pass rate,
-      # mean score, mean latency, tokens, cost and fault counts. "cost" sums
+      # mean score, latency (mean, and the P50 and P95 by nearest rank,
+      # Statistics.percentile), tokens, cost and fault counts. "cost" sums
       # the replays that carried a cost — "priced" of the "scenarios", of
       # which "reported" came from the caller and "estimated" from tokens ×
       # a model rate (Result#cost_source) — and is nil when none did.
@@ -90,6 +96,8 @@ module ActiveAgent
             "avg_score" => scored.any? ? (scored.sum / scored.size).round(3) : nil,
             "avg_task_completion" => task_scores.any? ? (task_scores.sum / task_scores.size).round(3) : nil,
             "avg_duration_ms" => durations.any? ? (durations.sum.to_f / durations.size).round : nil,
+            "p50_duration_ms" => Statistics.percentile(durations, 50)&.round,
+            "p95_duration_ms" => Statistics.percentile(durations, 95)&.round,
             "input_tokens" => cohort.sum { |result| result.replay.input_tokens.to_i },
             "output_tokens" => cohort.sum { |result| result.replay.output_tokens.to_i },
             "cost" => costs.any? ? costs.sum.to_f.round(6) : nil,
@@ -267,14 +275,27 @@ module ActiveAgent
         verdict&.dig("winner")
       end
 
-      # Adds "judge_usage" and "release" only when there is one to add, so
-      # a report built the way it always was serializes exactly as before.
+      # Whether the gap between the models is real (Comparison#to_h): each
+      # model's pass rate with its interval, every pair's difference with
+      # its interval and whether it is decisive, and the conclusion. The
+      # comparison recorded for these results when one was handed in. nil
+      # for a single model.
+      def comparison
+        return nil unless comparing?
+
+        @comparison ||= @recorded_comparison || Comparison.new(results: @results, models: @models).to_h
+      end
+
+      # Adds "judge_usage" and "release" only when there is one to add, and
+      # "comparison" only when comparing models, so a single-model report
+      # built the way it always was serializes exactly as before.
       def to_h
         {
           "models" => summary_by_model,
           "criteria" => criterion_scores,
           "recommendations" => recommendations,
           "verdict" => verdict,
+          "comparison" => comparison,
           "judge" => @judge_label || @judge&.label,
           "judge_usage" => judge_usage,
           "release" => release,
@@ -287,23 +308,6 @@ module ActiveAgent
         JSON.pretty_generate(to_h, *args)
       end
 
-      def to_markdown
-        scenario_count = @results.map { |result| result.scenario.key }.uniq.size
-        lines = [ "# Evaluation — #{scenario_count} scenario#{'s' unless scenario_count == 1} × #{@models.size} model#{'s' unless @models.size == 1}", "" ]
-        label = @judge_label || @judge&.label
-        lines << (label ? "Judged by `#{label}`." : "No judge; scored on rules and expectations alone.")
-        lines << ""
-        lines.concat(summary_table)
-        lines << ""
-        lines.concat(total_lines)
-        lines << "**Best model: #{winner}**" if winner
-        lines << ""
-        lines.concat(matrix_table)
-        lines.concat(detail_lines)
-        lines.concat(recommendation_lines)
-        lines.join("\n")
-      end
-
       private
 
       # Whether a model summary's cost is an estimate — any replay priced
@@ -525,30 +529,43 @@ module ActiveAgent
 
       # --- Markdown ----------------------------------------------------------
 
-      # The per-model table. A Judge column joins it when a judge spent
-      # anything, so the agent's cost and the evaluation's stay two numbers.
+      # The per-model table. Cost / 1,000 puts every model on one
+      # denominator however many of its scenarios ran. A Judge column joins
+      # it when a judge spent anything, so the agent's cost and the
+      # evaluation's stay two numbers.
       def summary_table
         judged = judge_usage.present?
-        columns = [ "Model", "Passed", "Mean score", "Mean latency", "Tokens in/out", "Cost", ("Judge" if judged), "Faults" ].compact
+        columns = [ "Model", "Passed", "Mean score", "Mean latency", "Median / P95", "Tokens in/out", "Cost", "Cost / 1,000", ("Judge" if judged), "Faults" ].compact
         header = [ "| #{columns.join(' | ')} |", "|#{columns.map { '---' }.join('|')}|" ]
         rows = summary_by_model.map do |label, stats|
           faults = stats["faults"].map { |fault, count| "#{fault.tr('_', ' ')} ×#{count}" }.join(", ")
           latency = stats["avg_duration_ms"] ? "#{stats['avg_duration_ms']} ms" : "—"
+          percentiles = stats["p50_duration_ms"] ? "#{stats['p50_duration_ms']} / #{stats['p95_duration_ms']} ms" : "—"
           cells = [
-            "`#{label}`",
+            cell("`#{label}`"),
             Format.passes(stats["passed"], stats["scenarios"], style: :markdown),
             Format.score(stats["avg_score"]),
             latency,
+            percentiles,
             "#{stats['input_tokens']}/#{stats['output_tokens']}",
             Format.money(stats["cost"], estimated: estimated_cost?(stats)),
+            per_thousand_cell(stats),
             (judge_cell(stats) if judged),
-            faults.presence || "—"
+            cell(faults.presence || "—")
           ].compact
           "| #{cells.join(' | ')} |"
         end
         header + rows
       end
 
+      # "$3.6300": what 1,000 scenario runs cost at the model's mean cost
+      # per priced replay, marked "~" when that mean is estimated or covers
+      # only some of the replays. "—" when none was priced.
+      def per_thousand_cell(stats)
+        per = cost_per_priced(stats)
+        per ? Format.money(per * 1_000, estimated: estimated_cost?(stats)) : "—"
+      end
+
       # "~$0.0030 (3 calls)" for a model's judge spend, "—" when the judge
       # was not asked about its results.
       def judge_cell(stats)
@@ -572,20 +589,52 @@ module ActiveAgent
         lines << ""
       end
 
+      # Each model's pass rate with its interval, best first, then every
+      # pair's difference with its interval, the conclusion, and how the
+      # intervals were computed. Nothing for a single model.
+      def comparison_lines
+        return [] unless comparison
+
+        method = comparison["method"]
+        confidence = "#{(method['confidence'].to_f * 100).round}% CI"
+        lines = [ "## Comparison", "", "| Model | Pass rate | #{confidence} | Scenarios |", "|---|---|---|---|" ]
+        comparison["ranking"].each do |label|
+          stats = comparison["models"][label]
+          lines << "| #{cell("`#{label}`")} | #{rate_text(stats['pass_rate'])} | #{interval_text(stats['interval'])} | #{stats['scenarios']} |"
+        end
+        lines += [ "", "| Pair | Difference | #{confidence} | Shared scenarios | Decisive |", "|---|---|---|---|---|" ]
+        comparison["pairs"].each do |pair|
+          a, b = pair["models"]
+          difference = pair["difference"] ? format("%+.1f points", pair["difference"]) : "—"
+          lines << "| #{cell("`#{a}` vs `#{b}`")} | #{difference} | #{interval_text(pair['interval'])} | #{pair['scenarios']} | #{pair['decisive'] ? 'yes' : 'no'} |"
+        end
+        lines += [ "", comparison["conclusion"].to_s, "",
+                   "_#{confidence} from #{Format.delimited(method['resamples'])} resamples of whole scenarios (seed #{method['seed']})._", "" ]
+      end
+
+      def rate_text(rate)
+        rate ? "#{rate}%" : "—"
+      end
+
+      # "73.8 to 95.2", or "—" when there were too few scenarios.
+      def interval_text(interval)
+        interval ? "#{interval.first} to #{interval.last}" : "—"
+      end
+
       # The scenario matrix, with a trailing Cost column: what the scenario
       # cost across every model, and in brackets what the judge spent on it.
       def matrix_table
         labels = @models.map(&:label)
-        header = [ "| Scenario | #{labels.map { |label| "`#{label}`" }.join(' | ')} | Cost |", "|---|#{labels.map { '---' }.join('|')}|---|" ]
+        header = [ "| Scenario | #{labels.map { |label| cell("`#{label}`") }.join(' | ')} | Cost |", "|---|#{labels.map { '---' }.join('|')}|---|" ]
         rows = @results.group_by { |result| result.scenario.key }.map do |key, cohort|
           cells = labels.map do |label|
             result = cohort.find { |candidate| candidate.label == label }
             next "—" unless result
 
             mark = result.passed? ? "✅" : (result.errored? ? "⚠️" : "❌")
-            [ mark, (Format.score(result.score) if result.score), result.fault&.tr("_", " ") ].compact.join(" ")
+            cell([ mark, (Format.score(result.score) if result.score), result.fault&.tr("_", " ") ].compact.join(" "))
           end
-          "| `#{key}` #{cell(cohort.first.scenario.prompt.truncate(70))} | #{cells.join(' | ')} | #{scenario_cost_cell(key)} |"
+          "| #{cell("`#{key}` #{cohort.first.scenario.prompt.truncate(70)}")} | #{cells.join(' | ')} | #{scenario_cost_cell(key)} |"
         end
         header + rows
       end
@@ -598,30 +647,11 @@ module ActiveAgent
         text
       end
 
-      # A prompt may contain " | " (ScenarioParser keeps it), which would
-      # otherwise split the table cell.
+      # A prompt may contain " | " (ScenarioParser keeps it), and an answer
+      # or a label a line break, either of which would otherwise break the
+      # table row.
       def cell(text)
-        text.to_s.gsub("|") { "\\|" }
-      end
-
-      def detail_lines
-        lines = [ "", "## Answers", "" ]
-        @results.each do |result|
-          lines << "### `#{result.scenario.key}` · `#{result.label}` · #{result.status}" \
-                   "#{" · score #{Format.score(result.score)}" if result.score}#{" · #{answer_cost(result)}" if result.replay.cost}"
-          lines << ""
-          lines << "> #{result.scenario.prompt}"
-          lines << ""
-          if result.replay.tool_calls.any?
-            lines << "Tools: #{result.replay.tool_calls.map { |call| "#{call['name']}#{' ✗' if call['error']}" }.join(', ')}"
-          end
-          lines << "Fault: #{result.fault.tr('_', ' ')} — #{result.summary} #{result.recommendation}" if result.fault
-          lines << "Error: #{result.replay.error}" if result.replay.error
-          lines << ""
-          lines << (result.replay.answer.presence || "_(no answer)_").to_s.truncate(1_500)
-          lines << ""
-        end
-        lines
+        text.to_s.gsub(/\s*\r?\n\s*/, " ").gsub("|") { "\\|" }
       end
 
       # "~$0.0243 · judge ~$0.0015": what one answer cost, and what judging
@@ -632,20 +662,6 @@ module ActiveAgent
         text += " · judge #{Format.money(judge_cost, estimated: estimated_usage?(result.judge_usage))}" if judge_cost
         text
       end
-
-      # Renders after `detail_lines`, whose trailing blank line separates the two sections.
-      def recommendation_lines
-        return [] if recommendations.empty?
-
-        lines = [ "## Recommendations", "" ]
-        recommendations.each do |entry|
-          lines << "- **#{entry['fault'].tr('_', ' ')}** ×#{entry['count']} (#{entry['scenario_keys'].join(', ')}): #{entry['recommendation']}"
-          entry["suggested_tools"].each do |tool|
-            lines << "  - suggested tool `#{tool['name']}`: #{tool['description']}"
-          end
-        end
-        lines << ""
-      end
     end
   end
 end