activeagent 1.9.6 → 1.9.8

data/lib/active_agent/evals/publisher.rb

@@ -20,12 +20,21 @@ module ActiveAgent
     # ArgumentError before anything is sent.
     class Publisher
       DEFAULT_ENDPOINT = "https://api.activeagents.ai/v1/evaluations"
+      # The hosted dashboard people sign in to; the hosted collector answers
+      # on the API host, where a dashboard session does not reach.
+      HOSTED_DASHBOARD_URL = "https://activeagents.ai"
       MAX_BYTES = 2 * 1024 * 1024
       DETAIL_LIMIT = 200
 
       # @return [String] the collector URL reports go to
       attr_reader :endpoint
 
+      # @return [String] the address a receipt's "url" path resolves against: the
+      #   dashboard_url given, or by default the endpoint's origin (its scheme,
+      #   host and any non-default port, with no path), except that the hosted
+      #   API host resolves to HOSTED_DASHBOARD_URL
+      attr_reader :dashboard_url
+
       # Raised for every failed delivery. Only a network failure keeps the
       # underlying error as its +cause+, so neither the response nor the
       # report reaches a log through the exception chain.
@@ -68,7 +77,14 @@ module ActiveAgent
       # The key is sent as a bearer token and filtered from the collector's
       # explanation, so it is limited to visible ASCII: the sanitizer in
       # +collector_detail+ never alters it, and an echo of it always matches.
-      def initialize(api_key:, endpoint: DEFAULT_ENDPOINT, timeout: 10, open_timeout: 10)
+      #
+      # @param dashboard_url [String, nil] the address people open the dashboard at,
+      #   where a receipt's "url" path resolves; by default the endpoint's origin,
+      #   or HOSTED_DASHBOARD_URL for the hosted collector.
+      #   The receipt's path, which carries the engine's mount, replaces any path on it
+      # @raise [ArgumentError] "Evaluation dashboard_url is not an http(s) URL" when it is anything else, or
+      #   "Evaluation dashboard_url must be an http(s) URL without credentials, query or fragment" when it carries one of those
+      def initialize(api_key:, endpoint: DEFAULT_ENDPOINT, dashboard_url: nil, timeout: 10, open_timeout: 10)
         @uri = URI.parse(endpoint.to_s)
         unless @uri.is_a?(URI::HTTP) && @uri.host && !@uri.userinfo && !@uri.query && !@uri.fragment
           raise ArgumentError, "Evaluation endpoint must be an HTTP(S) URL without credentials, query or fragment"
@@ -78,6 +94,8 @@ module ActiveAgent
         end
         @endpoint = @uri.to_s
 
+        @dashboard_url = dashboard_url.nil? ? default_dashboard_url(@uri) : checked_dashboard_url(dashboard_url)
+
         @api_key = api_key.to_s.strip
         raise ArgumentError, "Evaluation API key is required" if @api_key.empty?
         unless @api_key.match?(/\A[\x21-\x7E]+\z/)
@@ -93,9 +111,42 @@ module ActiveAgent
         raise ArgumentError, "Evaluation endpoint is not a valid URL"
       end
 
-      # report may be a Report or its saved JSON hash. Full prompts, answers and
-      # tool results are included; applications should make publication opt-in.
-      def call(report:, run_id:, source:, agent_name:, suite:)
+      # Returns a publisher built from +env+: ACTIVEAGENTS_API_KEY is the key,
+      # ACTIVEAGENTS_EVALUATIONS_ENDPOINT the collector (DEFAULT_ENDPOINT when
+      # blank) and ACTIVEAGENTS_DASHBOARD_URL the dashboard a receipt's "url"
+      # path resolves against (the endpoint's origin when blank), so
+      # publishing is one line:
+      #
+      #   Publisher.from_env.call(report: report, source: "support-app", agent_name: "SupportBot", suite: "support_desk")
+      #
+      # @param env [#[]] the environment to read, ENV by default
+      # @raise [ArgumentError] "ACTIVEAGENTS_API_KEY is not set" for a blank key, and what +new+ raises for the other two
+      # @return [Publisher]
+      def self.from_env(env = ENV)
+        api_key = env["ACTIVEAGENTS_API_KEY"].to_s.strip
+        raise ArgumentError, "ACTIVEAGENTS_API_KEY is not set" if api_key.empty?
+
+        endpoint = env["ACTIVEAGENTS_EVALUATIONS_ENDPOINT"].to_s.strip
+        dashboard_url = env["ACTIVEAGENTS_DASHBOARD_URL"].to_s.strip
+        new(api_key: api_key, endpoint: endpoint.empty? ? DEFAULT_ENDPOINT : endpoint, dashboard_url: dashboard_url.empty? ? nil : dashboard_url)
+      end
+
+      # Returns the collector's receipt for the completed +report+, a Report
+      # or its saved JSON hash. Full prompts, answers and tool results are
+      # included; applications should make publication opt-in.
+      #
+      # The receipt's "url" is the run's dashboard page, made absolute: a path
+      # beginning with "/" is joined onto +dashboard_url+. An absolute URL, a
+      # url that is neither (a relative reference without a leading slash, a
+      # network-path reference, or a path that is not a valid URI) and a
+      # receipt without one are left as the collector sent them.
+      #
+      # @param run_id [String, nil] the run's identity, kept across retries; by default
+      #   the one the report's metadata records under "run_id" or :run_id, which
+      #   Correlation#with_run puts there
+      # @raise [ArgumentError] when no run_id is given and the report's metadata records none as a nonempty string
+      def call(report:, run_id: nil, source:, agent_name:, suite:)
+        run_id = resolve_run_id(report, run_id)
         envelope = self.class.envelope(report: report, run_id: run_id, source: source, agent_name: agent_name, suite: suite)
         publish(envelope, accepted: %w[complete], receipt: "completion receipt")
       end
@@ -104,13 +155,18 @@ module ActiveAgent
       # which the collector shows as a running run under +run_id+. Each call
       # replaces the results the previous one sent; +call+ with the finished
       # report under the same run_id completes the run. The receipt's status
-      # is "running", or "complete" when the run already finished.
+      # is "running", or "complete" when the run already finished, and its
+      # "url" is made absolute as +call+ makes it.
       #
       # Raises Error, without sending the report, when the collector does not
       # list "progress" among its capabilities (see +supports_progress?+): a
       # collector that predates progress would store the partial report as
       # complete.
-      def progress(report:, run_id:, source:, agent_name:, suite:)
+      #
+      # @param run_id [String, nil] as for +call+: by default the one the report's metadata records
+      # @raise [ArgumentError] when no run_id is given and the report's metadata records none as a nonempty string
+      def progress(report:, run_id: nil, source:, agent_name:, suite:)
+        run_id = resolve_run_id(report, run_id)
         unless supports_progress?
           raise Error, "Evaluation collector does not accept progress; publish the finished report with call"
         end
@@ -186,8 +242,54 @@ module ActiveAgent
 
       private
 
+      # The run_id a delivery goes under: +run_id+ when given, otherwise the
+      # one the report's metadata records under a String or Symbol key, which
+      # Correlation#with_run puts there. Raises ArgumentError for a report
+      # that is neither a Report nor a hash, as +envelope+ does, and when
+      # neither +run_id+ nor the metadata names a nonempty string.
+      def resolve_run_id(report, run_id)
+        return run_id unless run_id.nil?
+
+        hash = report.to_h if report.respond_to?(:to_h) && !report.nil? && !report.is_a?(Array)
+        raise ArgumentError, "report must be a Report or its saved JSON hash" unless hash.is_a?(Hash)
+
+        metadata = hash["metadata"]
+        recorded = metadata["run_id"] || metadata[:run_id] if metadata.is_a?(Hash)
+        return recorded if recorded.is_a?(String) && !recorded.strip.empty?
+
+        raise ArgumentError, "run_id must be given, or the report's metadata must carry one (Correlation#with_run records it)"
+      end
+
+      # The hosted collector's API host serves no dashboard session, so its
+      # receipts resolve against the hosted dashboard; any other collector is
+      # its own dashboard.
+      def default_dashboard_url(uri)
+        uri.hostname == URI.parse(DEFAULT_ENDPOINT).hostname ? HOSTED_DASHBOARD_URL : origin(uri)
+      end
+
+      # Returns the dashboard_url given as a String, raising ArgumentError
+      # unless it is an http(s) URL with a host and without credentials, a
+      # query or a fragment.
+      def checked_dashboard_url(value)
+        uri = URI.parse(value.to_s)
+        raise ArgumentError, "Evaluation dashboard_url is not an http(s) URL" unless uri.is_a?(URI::HTTP) && !uri.host.to_s.empty?
+        if uri.userinfo || uri.query || uri.fragment
+          raise ArgumentError, "Evaluation dashboard_url must be an http(s) URL without credentials, query or fragment"
+        end
+
+        value.to_s
+      rescue URI::InvalidURIError
+        raise ArgumentError, "Evaluation dashboard_url is not an http(s) URL"
+      end
+
+      # The origin of +uri+: its scheme, host and any non-default port, with no path.
+      def origin(uri)
+        uri.dup.tap { |without_path| without_path.path = "" }.to_s
+      end
+
       # Delivers +envelope+ and returns the collector's receipt, which must name
-      # the envelope's run_id, a stored run, and one of the +accepted+ statuses.
+      # the envelope's run_id, a stored run, and one of the +accepted+ statuses,
+      # with a "url" path made absolute on the dashboard.
       def publish(envelope, accepted:, receipt:)
         body = encode(envelope)
         raise Error, "Evaluation report exceeds the 2 MiB delivery limit; publish a smaller selection" if body.bytesize > MAX_BYTES
@@ -201,10 +303,24 @@ module ActiveAgent
                  answer["id"] && answer["evaluation_id"]
             raise Error.new("Evaluation collector returned an invalid #{receipt}; retain the report and run_id for retry", retryable: true)
           end
-          answer
+          resolve_url(answer)
         end
       end
 
+      # Returns +receipt+ with a "url" that is a path beginning with "/" joined
+      # onto +dashboard_url+. Any other url (an absolute one, a relative
+      # reference without a leading slash, a network-path reference, a path
+      # that is not a valid URI) and a receipt without one are left as the
+      # collector sent them.
+      def resolve_url(receipt)
+        url = receipt["url"]
+        return receipt unless url.is_a?(String) && url.start_with?("/") && !url.start_with?("//")
+
+        receipt.merge("url" => URI.join(@dashboard_url, url).to_s)
+      rescue URI::InvalidURIError
+        receipt
+      end
+
       # The string entries of a JSON object body's +capabilities+ array, or none.
       def collector_capabilities(body)
         parsed = JSON.parse(body.to_s)

data/lib/active_agent/evals/report.rb

@@ -14,7 +14,33 @@ module ActiveAgent
       # pass rate itself, no judge having been available to rule on them.
       PASS_RATE_JUDGE = "pass rate"
 
-      attr_reader :results, :models, :judge, :judge_label, :instructions, :metadata, :agent_name, :links, :tool_resolver
+      attr_reader :results, :models, :judge, :judge_label, :instructions, :metadata, :agent_name, :links, :tool_resolver,
+                  :pass_thresholds
+
+      # One group's (or the whole report's, with group nil and name "All")
+      # pass rate under one model against the bar the suite set for it.
+      # `pass_rate` and `min_pass_rate` are 0..1 fractions, so they compare
+      # directly and render with Format.percent; an ungated row is always met.
+      Gate = Struct.new(:group, :name, :label, :passed, :total, :pass_rate, :min_pass_rate, :gated, keyword_init: true) do
+        def met?
+          !gated || pass_rate >= min_pass_rate
+        end
+
+        def to_h
+          super.transform_keys(&:to_s).merge("met" => met?)
+        end
+
+        # "Order support · gpt-5-mini: 1/2 (50%) passed, below 100%" — one
+        # sentence a failing assertion can print.
+        def to_s
+          bar =
+            if !gated then "not gated"
+            elsif met? then "at or above #{Format.percent(min_pass_rate)}"
+            else "below #{Format.percent(min_pass_rate)}"
+            end
+          "#{name} · #{label}: #{Format.passes(passed, total, style: :markdown)} passed, #{bar}"
+        end
+      end
 
       # @param tool_resolver [#call, nil] maps a tool name to the MCP server
       #   that provides it — `{ "key", "name", "status" }` with status
@@ -41,15 +67,22 @@ module ActiveAgent
       # @param comparison [Hash, nil] a comparison already recorded for these
       #   results (Comparison#to_h). A report rebuilt from a persisted run
       #   renders the intervals that run recorded instead of resampling them.
-      def initialize(results:, models:, judge: nil, instructions: nil, threshold: PASS_THRESHOLD, metadata: {},
+      # @param threshold [Float, nil] the score an answer must reach to pass
+      #   a criterion; an explicit value wins over the `pass_thresholds` bar,
+      #   and without either it is PASS_THRESHOLD
+      # @param pass_thresholds [PassThresholds, nil] the bars the suite
+      #   declared: its threshold applies when none is given here, #gates
+      #   and #passed? read them, and `to_h` carries them with the per-group
+      #   summary and the gates.
+      def initialize(results:, models:, judge: nil, instructions: nil, threshold: nil, metadata: {},
                      tool_resolver: nil, agent_name: nil, links: {}, verdict: nil, judge_label: nil,
-                     judge_usage: nil, release: nil, comparison: nil)
+                     judge_usage: nil, release: nil, comparison: nil, pass_thresholds: nil)
         @results = results
         @models = models
         @judge = judge
         @judge_label = judge_label.presence
         @instructions = instructions
-        @threshold = threshold
+        @threshold = threshold || pass_thresholds&.threshold || PASS_THRESHOLD
         @metadata = metadata
         @tool_resolver = tool_resolver
         @agent_name = agent_name.presence || "the agent"
@@ -58,6 +91,7 @@ module ActiveAgent
         @run_judge_usage = judge_usage.is_a?(Hash) ? judge_usage.to_h.stringify_keys.presence : nil
         @release = release.is_a?(Hash) ? release.to_h.stringify_keys.compact.presence : nil
         @recorded_comparison = comparison.is_a?(Hash) ? comparison.to_h.deep_stringify_keys.presence : nil
+        @pass_thresholds = pass_thresholds
       end
 
       # The release of the agent this report scored, `{ "digest",
@@ -68,12 +102,20 @@ module ActiveAgent
         @models.size > 1
       end
 
-      # Per model, keyed by label: scenario count, passes, errors, pass rate,
-      # mean score, latency (mean, and the P50 and P95 by nearest rank,
-      # Statistics.percentile), tokens, cost and fault counts. "cost" sums
-      # the replays that carried a cost — "priced" of the "scenarios", of
-      # which "reported" came from the caller and "estimated" from tokens ×
-      # a model rate (Result#cost_source) — and is nil when none did.
+      # How many times each scenario was replayed under each model: the
+      # highest trial number any result carries, 1 for a run with none.
+      def trials
+        @results.map(&:trial).max || 1
+      end
+
+      # Per model, keyed by label: replay count ("scenarios", every trial of
+      # every scenario), distinct scenarios ("unique_scenarios"), passes,
+      # errors, pass rate over every replay, mean score, latency (mean, and
+      # the P50 and P95 by nearest rank, Statistics.percentile), tokens,
+      # cost and fault counts. "cost" sums the replays that carried a cost
+      # — "priced" of the "scenarios", of which "reported" came from the
+      # caller and "estimated" from tokens × a model rate
+      # (Result#cost_source) — and is nil when none did.
       # "judge_cost" and "judge_calls" sum what the judge spent on the
       # cohort's results (Result#judge_usage); the judge's cost never joins
       # the agent's.
@@ -90,6 +132,7 @@ module ActiveAgent
             "provider" => spec.provider,
             "model" => spec.model,
             "scenarios" => cohort.size,
+            "unique_scenarios" => cohort.map { |result| result.scenario.key }.uniq.size,
             "passed" => cohort.count(&:passed?),
             "errored" => cohort.count(&:errored?),
             "pass_rate" => cohort.any? ? (cohort.count(&:passed?) * 100.0 / cohort.size).round(1) : 0.0,
@@ -111,6 +154,58 @@ module ActiveAgent
         end
       end
 
+      # Per group key, in run order, then per model under "models" (keyed by
+      # label, only the models with a result in the group): scenario count,
+      # passes, errors, pass rate as a percent, mean score and fault counts,
+      # counted as summary_by_model counts them. "name" is the group's
+      # display name, or its key. Scenarios with no group are left out.
+      def summary_by_group
+        @summary_by_group ||= begin
+          grouped = @results.select { |result| result.scenario.group.present? }.group_by { |result| result.scenario.group }
+          grouped.to_h do |group, cohort|
+            models = @models.filter_map do |spec|
+              rows = cohort.select { |result| result.label == spec.label }
+              [ spec.label, cohort_summary(rows) ] if rows.any?
+            end
+            [ group, { "name" => group_name(cohort.first.scenario), "models" => models.to_h } ]
+          end
+        end
+      end
+
+      # One Gate per group and model with results there, in summary_by_group
+      # order, then one per model for the whole report (group nil, name
+      # "All") over every gated result of that model, ungrouped results
+      # included; a model with no gated result has no "All" row. Each row
+      # carries the bar `thresholds` sets for its group.
+      def gates(thresholds = pass_thresholds || PassThresholds.default)
+        rows = summary_by_group.flat_map do |group, summary|
+          summary["models"].map do |label, stats|
+            gate(group, summary["name"], label, stats["passed"], stats["scenarios"],
+                 thresholds.min_pass_rate(group), thresholds.gated?(group))
+          end
+        end
+        gated = @results.select { |result| thresholds.gated?(result.scenario.group) }
+        rows + @models.filter_map do |spec|
+          cohort = gated.select { |result| result.label == spec.label }
+          gate(nil, "All", spec.label, cohort.count(&:passed?), cohort.size, thresholds.min_pass_rate(nil), true) if cohort.any?
+        end
+      end
+
+      # Whether every gate is met. A report with no gate, because no result
+      # reached it (a selection that matched nothing, every scenario left
+      # out as production_only), has shown nothing and does not pass.
+      def passed?(thresholds = pass_thresholds || PassThresholds.default)
+        rows = gates(thresholds)
+        rows.any? && rows.all?(&:met?)
+      end
+
+      # The results that did not pass — failed and errored — in run order,
+      # narrowed to one group when given.
+      def failures(group: nil)
+        failed = @results.reject(&:passed?)
+        group.nil? ? failed : failed.select { |result| result.scenario.group.to_s == group.to_s }
+      end
+
       # Per scenario key, in run order: what every model's answer cost
       # together — the agent's cost summed over the models, the judge's, and
       # the two as "total" — plus the same per model under "models", keyed
@@ -258,7 +353,7 @@ module ActiveAgent
             cost = " at #{Format.money(stats['cost'], estimated: estimated_cost?(stats))}"
             cost += " (estimated)" if estimated_cost?(stats)
           end
-          rationale = "Passed #{stats['passed']} of #{stats['scenarios']} scenarios" \
+          rationale = "Passed #{stats['passed']} of #{counted_scenarios(stats)}" \
             " (#{Format.percent(stats['scenarios'].to_i.positive? ? stats['passed'].to_f / stats['scenarios'] : nil)})" \
             "#{" with a mean score of #{Format.score(stats['avg_score'])}" if stats['avg_score']}#{cost}."
           judged = @judge&.verdict(summary_by_model, instructions: @instructions)
@@ -286,21 +381,28 @@ module ActiveAgent
         @comparison ||= @recorded_comparison || Comparison.new(results: @results, models: @models).to_h
       end
 
-      # Adds "judge_usage" and "release" only when there is one to add, and
-      # "comparison" only when comparing models, so a single-model report
-      # built the way it always was serializes exactly as before.
+      # Adds "judge_usage" and "release" only when there is one to add,
+      # "comparison" only when comparing models, "trials" only when a scenario
+      # was replayed more than once, and "pass_thresholds" with the per-group
+      # summary ("groups") and the "gates" only when the suite declared its
+      # bars, so a single-model, single-trial report built the way it always
+      # was serializes exactly as before.
       def to_h
         {
           "models" => summary_by_model,
+          "trials" => (trials if trials > 1),
           "criteria" => criterion_scores,
           "recommendations" => recommendations,
           "verdict" => verdict,
           "comparison" => comparison,
+          "pass_thresholds" => pass_thresholds&.to_h&.merge("threshold" => @threshold),
+          "groups" => (summary_by_group if pass_thresholds),
+          "gates" => (gates.map(&:to_h) if pass_thresholds),
           "judge" => @judge_label || @judge&.label,
           "judge_usage" => judge_usage,
           "release" => release,
           "metadata" => @metadata.presence,
-          "results" => @results.map(&:to_h)
+          "results" => serialized_results
         }.compact
       end
 
@@ -310,6 +412,14 @@ module ActiveAgent
 
       private
 
+      # Each result's hash; a single-trial report leaves out every "trial",
+      # so it is the JSON an earlier gem wrote and a re-delivery of it under
+      # the same run_id matches the stored digest.
+      def serialized_results
+        hashes = @results.map(&:to_h)
+        trials > 1 ? hashes : hashes.map { |hash| hash.except("trial") }
+      end
+
       # Whether a model summary's cost is an estimate — any replay priced
       # from tokens, or some replay unpriced, which leaves the sum a lower
       # bound — and so reads with a "~".
@@ -339,10 +449,37 @@ module ActiveAgent
         stats["cost"].to_f / priced if stats["cost"] && priced.positive?
       end
 
+      # The denominator a verdict names: "6 scenarios", or "18 replays of 6
+      # scenarios" when the model replayed each more than once.
+      def counted_scenarios(stats)
+        replays = stats["scenarios"].to_i
+        unique = stats["unique_scenarios"].to_i
+        replays > unique ? "#{replays} replays of #{unique} scenarios" : "#{replays} scenarios"
+      end
+
       def criterion_keys
         @results.flat_map { |result| result.scores.keys }.uniq
       end
 
+      # One model's counts over the results of a group, the way
+      # summary_by_model counts a model's whole cohort.
+      def cohort_summary(cohort)
+        scored = cohort.filter_map(&:score)
+        {
+          "scenarios" => cohort.size,
+          "passed" => cohort.count(&:passed?),
+          "errored" => cohort.count(&:errored?),
+          "pass_rate" => cohort.any? ? (cohort.count(&:passed?) * 100.0 / cohort.size).round(1) : 0.0,
+          "avg_score" => scored.any? ? (scored.sum / scored.size).round(3) : nil,
+          "faults" => cohort.filter_map(&:fault).tally
+        }
+      end
+
+      def gate(group, name, label, passed, total, min_pass_rate, gated)
+        Gate.new(group: group, name: name, label: label, passed: passed, total: total,
+                 pass_rate: passed.to_f / total, min_pass_rate: min_pass_rate, gated: gated)
+      end
+
       def stats_for(values)
         scored = values.compact
         return { "skipped" => true, "reason" => "No scorable answers" } if scored.empty?
@@ -628,17 +765,25 @@ module ActiveAgent
         header = [ "| Scenario | #{labels.map { |label| cell("`#{label}`") }.join(' | ')} | Cost |", "|---|#{labels.map { '---' }.join('|')}|---|" ]
         rows = @results.group_by { |result| result.scenario.key }.map do |key, cohort|
           cells = labels.map do |label|
-            result = cohort.find { |candidate| candidate.label == label }
-            next "—" unless result
-
-            mark = result.passed? ? "✅" : (result.errored? ? "⚠️" : "❌")
-            cell([ mark, (Format.score(result.score) if result.score), result.fault&.tr("_", " ") ].compact.join(" "))
+            mine = cohort.select { |candidate| candidate.label == label }.sort_by(&:trial)
+            mine.empty? ? "—" : cell(matrix_cell_text(mine))
           end
           "| #{cell("`#{key}` #{cohort.first.scenario.prompt.truncate(70)}")} | #{cells.join(' | ')} | #{scenario_cost_cell(key)} |"
         end
         header + rows
       end
 
+      # "✅ 100%" or "❌ 50% missing capability" for one result; for a
+      # scenario replayed more than once, every trial's mark run together
+      # ("✅✅❌"), the mean of the trials' scores, and the fault seen most
+      # often among them (a tie goes to the first seen).
+      def matrix_cell_text(results)
+        marks = results.map { |result| status_mark(result) }.join
+        scored = results.filter_map(&:score)
+        fault = results.filter_map(&:fault).tally.max_by(&:last)&.first
+        [ marks, (Format.score(scored.sum.fdiv(scored.size)) if scored.any?), fault&.tr("_", " ") ].compact.join(" ")
+      end
+
       # "~$0.0243 (judge ~$0.0015)" for a scenario's total across models.
       def scenario_cost_cell(key)
         costs = scenario_costs[key] || {}