actionagent 1.8.0 → 1.8.1

data/app/serializers/action_agent/evaluation_serializer.rb

@@ -12,19 +12,29 @@ module ActionAgent
 
     # Returns the evaluation with its latest run in full and just enough of
     # the run before it to show movement ("+3 passed vs #2") without a
-    # request per evaluation.
+    # request per evaluation. The headline run — the newest complete one —
+    # rides along in full as `headline_run` when a newer run is pending or
+    # failed, so a page can describe the finished run while showing the
+    # newer one beside it.
     def evaluation(evaluation)
       summary = summary(evaluation)
       latest, previous = recent_runs(evaluation, 2)
+      headline = evaluation.standing_info.headline_run
+      headline = nil if headline.nil? || headline.id == latest&.id
 
       summary.merge(
         latest_run: latest ? run(latest, number: summary[:run_count]) : nil,
-        previous_run: previous ? run_summary(previous, number: summary[:run_count] - 1) : nil
+        previous_run: previous ? run_summary(previous, number: summary[:run_count] - 1) : nil,
+        headline_run: headline ? run(headline, number: run_number(evaluation, headline)) : nil
       )
     end
 
-    # Returns the evaluation's configuration and run count, without its runs.
+    # Returns the evaluation's configuration and run count, without its runs,
+    # and where it stands: `headline_run_id` (its newest complete run),
+    # `standing` against the agent's current version (EvaluationStanding),
+    # `archived_at`, and the headline run's passes `per_model`.
     def summary(evaluation)
+      standing = evaluation.standing_info
       {
         id: evaluation.id,
         name: evaluation.name,
@@ -40,7 +50,11 @@ module ActionAgent
         scenario_groups: evaluation.scenario_suite? ? evaluation.scenario_groups : [],
         created_at: evaluation.created_at.iso8601,
         # size reads a preloaded association and COUNTs otherwise.
-        run_count: evaluation.evaluation_runs.size
+        run_count: evaluation.evaluation_runs.size,
+        headline_run_id: standing.headline_run&.id,
+        standing: standing.standing,
+        archived_at: evaluation.archived_at&.iso8601,
+        per_model: standing.per_model
       }
     end
 
@@ -48,7 +62,7 @@ module ActionAgent
     # error.
     def run(run, number: nil)
       run_summary(run, number: number).merge(
-        scores: run.scores,
+        scores: scores(run),
         selection: run.selection,
         models: run.models,
         usage: run.usage,
@@ -56,9 +70,18 @@ module ActionAgent
       )
     end
 
+    # Returns the run's scores with each scenario model summary carrying its
+    # "priced" count (EvaluationRun#model_summaries).
+    def scores(run)
+      summaries = run.model_summaries
+      summaries.empty? ? run.scores : run.scores.merge("_models" => summaries)
+    end
+
     # Returns the run's status and headline numbers. `sandbox` is the
     # checkout sandbox the run replayed against, if any: its session id and
-    # checkout, never its token.
+    # checkout, never its token. `agent_version` is the version of the
+    # agent the run scored, and `version_state` whether that is the agent's
+    # current version ("current", "earlier" or "unrecorded").
     def run_summary(run, number: nil)
       {
         id: run.id,
@@ -69,7 +92,9 @@ module ActionAgent
         samples_passed: run.samples_passed,
         completed_at: run.completed_at&.iso8601,
         created_at: run.created_at.iso8601,
-        sandbox: run.sandbox
+        sandbox: run.sandbox,
+        agent_version: run.agent_version_summary,
+        version_state: run.evaluation.standing_info.version_state(run)
       }
     end
 
@@ -81,13 +106,19 @@ module ActionAgent
       if runs.loaded?
         runs.sort_by { |run| [ run.created_at, run.id ] }.reverse.first(limit)
       else
-        runs.recent.limit(limit).to_a
+        runs.order(created_at: :desc, id: :desc).limit(limit).to_a
       end
     end
 
-    # Returns the run's position in its evaluation's history, oldest = 1.
+    # Returns the run's position in its evaluation's history, oldest = 1 —
+    # counted in the preloaded association when one was loaded.
     def run_number(evaluation, run)
-      evaluation.evaluation_runs.where("created_at < ? OR (created_at = ? AND id <= ?)", run.created_at, run.created_at, run.id).count
+      runs = evaluation.evaluation_runs
+      if runs.loaded?
+        runs.count { |other| other.created_at < run.created_at || (other.created_at == run.created_at && other.id <= run.id) }
+      else
+        runs.where("created_at < ? OR (created_at = ? AND id <= ?)", run.created_at, run.created_at, run.id).count
+      end
     end
 
     # Returns the run's fix items, or [] when they cannot be built. They are

data/app/services/action_agent/agent_scorecard.rb

@@ -32,7 +32,7 @@ module ActionAgent
       avg_durations = windowed.where.not(duration_ms: nil).group(:agent_id).average(:duration_ms)
       token_sums = windowed.group(:agent_id).sum("COALESCE(total_tokens, 0)")
       last_runs = AgentRun.where(agent_id: ids).group(:agent_id).maximum(:created_at)
-      eval_runs = latest_evaluation_runs(ids)
+      evals = evaluation_tiles(ids)
 
       trace_stats = telemetry_stats(ids, window_start)
       trace_last = unclaimed_traces(ids, nil).group(:agent_id).maximum(:timestamp)
@@ -44,7 +44,7 @@ module ActionAgent
         traced = stats[:count].to_i
         total = runs + traced
         succeeded = completed_counts[id].to_i + stats[:ok].to_i
-        eval_run = eval_runs[id]
+        eval_tile = evals[id] || {}
 
         {
           window_days: (WINDOW / 1.day).to_i,
@@ -55,9 +55,13 @@ module ActionAgent
           avg_duration_ms: blended_duration(avg_durations[id], runs, stats[:avg_duration], traced),
           tokens: token_sums[id].to_i + stats[:tokens].to_i,
           cost: costs[id],
-          eval_score: eval_run&.average_score,
-          eval_samples_passed: eval_run&.samples_passed,
-          eval_samples_evaluated: eval_run&.samples_evaluated,
+          # The evaluation tile: the pass rate pooled over the headline runs
+          # of the agent's evaluations that stand against its current version.
+          eval_score: eval_tile[:score],
+          eval_samples_passed: eval_tile[:passed],
+          eval_samples_evaluated: eval_tile[:evaluated],
+          eval_runs: eval_tile[:runs],
+          eval_not_counted: eval_tile[:not_counted],
           last_run_at: [ last_runs[id], trace_last[id] ].compact.max&.iso8601
         }
       end
@@ -170,22 +174,47 @@ module ActionAgent
     end
     private_class_method :blended_duration
 
-    # Latest complete evaluation run per agent. DISTINCT ON would do this in
-    # one pass on PostgreSQL, but it has no portable equivalent, so the
-    # highest id per agent (runs are only ever appended) is selected first
-    # and those rows fetched by id.
-    def self.latest_evaluation_runs(agent_ids)
-      evaluations = Evaluation.table_name
-      runs = EvaluationRun.table_name
-
-      newest = EvaluationRun.complete.joins(:evaluation)
-        .where(evaluations => { agent_id: agent_ids })
-        .group("#{evaluations}.agent_id")
-        .pluck(Arel.sql("#{evaluations}.agent_id"), Arel.sql("MAX(#{runs}.id)"))
-
-      by_run_id = newest.to_h { |agent_id, run_id| [ run_id, agent_id ] }
-      EvaluationRun.where(id: by_run_id.keys).index_by { |run| by_run_id[run.id] }
+    # The evaluation tile per agent: passes pooled over the headline runs
+    # (newest complete run) of its evaluations that are current against the
+    # agent's version or unrecorded, as the Evaluations page pools them —
+    # never a stale suite last run against older code, nor an archived one
+    # — so the tile is the pass rate of the agent as it is now, not a mean
+    # of criterion scores over whatever ran last.
+    #
+    #   { agent_id => { score:, passed:, evaluated:, runs:, not_counted: } }
+    #
+    # `score` is the pooled pass rate 0..1 (nil with nothing to count),
+    # `runs` the headline runs pooled and `not_counted` the evaluations
+    # left out as stale or archived. The newest complete run per evaluation
+    # is selected by id first (runs are only ever appended) and those rows
+    # fetched by id, which runs the same on every database.
+    def self.evaluation_tiles(agent_ids)
+      evaluations = Evaluation.where(agent_id: agent_ids).includes(:agent).to_a
+      return {} if evaluations.empty?
+
+      newest = EvaluationRun.complete.where(evaluation_id: evaluations.map(&:id)).group(:evaluation_id).maximum(:id)
+      runs = EvaluationRun.where(id: newest.values).includes(:agent_version).index_by(&:evaluation_id)
+      EvaluationStanding.preload(evaluations)
+
+      evaluations.group_by(&:agent_id).to_h do |agent_id, mine|
+        tile = { score: nil, passed: 0, evaluated: 0, runs: 0, not_counted: 0 }
+        mine.each do |evaluation|
+          run = runs[evaluation.id]
+          next unless run
+
+          unless evaluation.standing_info.with_headline(run).counted?
+            tile[:not_counted] += 1
+            next
+          end
+
+          tile[:runs] += 1
+          tile[:passed] += run.samples_passed.to_i
+          tile[:evaluated] += run.samples_evaluated.to_i
+        end
+        tile[:score] = (tile[:passed].to_f / tile[:evaluated]).round(3) if tile[:evaluated].positive?
+        [ agent_id, tile ]
+      end
     end
-    private_class_method :latest_evaluation_runs
+    private_class_method :evaluation_tiles
   end
 end

data/app/services/action_agent/codex_session_events.rb

@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+module ActionAgent
+  # Keep Codex's native JSONL transcript while adapting its terminal event
+  # to CodeSession's shared result contract. A process exit alone is never
+  # enough to mark a coding session successful.
+  class CodexSessionEvents
+    def consume(event)
+      return unless event.is_a?(Hash)
+
+      case event["type"]
+      when "thread.started"
+        @thread_id = event["thread_id"]
+      when "item.completed"
+        item = event["item"]
+        @answer = item["text"] if item.is_a?(Hash) && item["type"] == "agent_message"
+      when "turn.completed"
+        return result(false, @answer, event["usage"])
+      when "turn.failed", "error"
+        error = event["error"]
+        message = error.is_a?(Hash) ? error["message"] : event["message"]
+        return result(true, message.presence || "Codex reported a failed turn", {})
+      end
+      nil
+    end
+
+    private
+
+    def result(failed, answer, usage)
+      {
+        "type" => "result", "subtype" => failed ? "error_during_execution" : "success",
+        "is_error" => failed, "result" => answer, "session_id" => @thread_id,
+        "usage" => usage.is_a?(Hash) ? usage.slice("input_tokens", "output_tokens") : {}
+      }
+    end
+  end
+end

data/app/services/action_agent/evaluation_report_import.rb

@@ -22,7 +22,12 @@ module ActionAgent
   #
   # The run's per-model summary, criterion scores and recommendations are
   # computed from the stored results, not taken from the report. The judge's
-  # verdict and label are taken from it.
+  # verdict and label are taken from it, and so is what the judge spent —
+  # per result (`result.judge_usage`, kept beside the result's diagnosis) and
+  # for the run (`report.judge_usage.run`) — and the release the report says
+  # it evaluated (`report.release`), which pins the run to that version of
+  # the agent (Agent#find_or_record_release!). A report naming no release
+  # leaves the run unrecorded rather than claiming the dashboard's latest.
   #
   # An identical retry returns the stored run, before anything is asked of the
   # `admit` callable. Different content under a run_id already stored raises
@@ -70,6 +75,11 @@ module ActionAgent
     IDENTIFIER_PATTERN = /\A[^[:cntrl:]]{1,200}\z/
     AGENT_NAME_PATTERN = /\A[^[:cntrl:]]{2,100}\z/
     TRACE_PATTERN = /\A[a-zA-Z0-9_-]{1,128}\z/
+    DIGEST_PATTERN = /\A[a-zA-Z0-9._-]{1,64}\z/
+    # The judge usage keys a report may send, per result and for the run.
+    JUDGE_USAGE_KEYS = %w[calls input_tokens output_tokens cost model by_kind source].freeze
+    JUDGE_USAGE_SOURCES = %w[reported traces meter].freeze
+    MAX_JUDGE_KINDS = 10
     SCOPE_PATTERN = %r{\A[\w .:/@-]{1,100}\z}
     STATUSES = %w[passed failed errored].freeze
     # Report metadata that tells one evaluation of a suite from another, in the
@@ -253,6 +263,7 @@ module ActionAgent
         external_tenant: tenant_key,
         external_run_id: @payload["run_id"],
         external_report_digest: digest,
+        agent_version: reported_release_version(agent),
         status: :complete,
         selection: selection,
         scores: recorded_scores,
@@ -267,6 +278,15 @@ module ActionAgent
       run
     end
 
+    # The agent version for the release the report names, or nil for a
+    # report that names none.
+    def reported_release_version(agent)
+      release = report["release"]
+      return nil unless release.is_a?(Hash) && release["digest"].present?
+
+      agent.find_or_record_release!(digest: release["digest"], revision: release["revision"].presence, label: release["label"].presence)
+    end
+
     # --- ownership -------------------------------------------------------------
 
     # Who a newly observed agent belongs to. Whatever the host's
@@ -492,11 +512,28 @@ module ActionAgent
             "key" => result["scenario_key"], "group" => result["group"], "prompt" => result["prompt"],
             "position" => scenario.position, "expectations" => {}
           }
-        ),
+        ).merge(bounded_judge_usage(result["judge_usage"]).then { |usage| usage ? { "_judge_usage" => usage } : {} }),
         error_message: truncated(result["error"], TEXT_BYTES)
       )
     end
 
+    # A judge usage as the engine stores it: the keys the dashboard reads,
+    # each cut to its type, or nil for none.
+    def bounded_judge_usage(usage)
+      return nil unless usage.is_a?(Hash)
+
+      by_kind = usage["by_kind"].is_a?(Hash) ? usage["by_kind"].first(MAX_JUDGE_KINDS).to_h { |kind, count| [ kind.to_s.first(40), count.to_i ] } : {}
+      {
+        "calls" => usage["calls"].to_i,
+        "input_tokens" => usage["input_tokens"].to_i,
+        "output_tokens" => usage["output_tokens"].to_i,
+        "cost" => usage["cost"].is_a?(Numeric) ? usage["cost"].to_f : nil,
+        "model" => usage["model"].is_a?(String) ? usage["model"].first(200).presence : nil,
+        "by_kind" => by_kind,
+        "source" => JUDGE_USAGE_SOURCES.include?(usage["source"]) ? usage["source"] : "reported"
+      }
+    end
+
     # The first +bytes+ bytes of +text+, dropping a character the cut splits,
     # or nil for blank text.
     def truncated(text, bytes)
@@ -521,14 +558,18 @@ module ActionAgent
       {
         "_verdict" => report["verdict"],
         "_selection" => selection,
-        "_metadata" => metadata
+        "_metadata" => metadata,
+        "_judge_usage_run" => bounded_judge_usage(report.dig("judge_usage", "run"))
       }.compact.merge("_judge_label" => judge_label)
     end
 
     # The run's scores in the shape the Evaluations view renders
-    # (ScenarioEvaluationRunner#scores_for), summarized from the stored results.
+    # (ScenarioEvaluationRunner#scores_for), summarized from the stored
+    # results as the application reported them — no cost is estimated here,
+    # so the summaries stored stay the application's own figures; the API
+    # adds the estimates when it serves the run (EvaluationRun#model_summaries).
     def summarized_scores(run)
-      rebuilt = run.to_report
+      rebuilt = run.to_report(estimate: false)
       rebuilt.criterion_scores.merge(
         "_models" => rebuilt.summary_by_model,
         "_recommendations" => rebuilt.recommendations
@@ -559,6 +600,9 @@ module ActionAgent
 
       judge_trace_ids!(metadata["judge_trace_ids"])
       string!(report["judge"], "report.judge", 200)
+      validate_release!
+      validate_judge_usage!(report.dig("judge_usage", "run"), "report.judge_usage.run") if report["judge_usage"].is_a?(Hash)
+      optional_object!(report["judge_usage"], "report.judge_usage")
       object!(report["models"], "report.models")
       raise Invalid, "report.models must contain 1-#{MAX_MODELS} models" unless report["models"].size.between?(1, MAX_MODELS)
       unless results.is_a?(Array) && results.size.between?(1, MAX_RESULTS)
@@ -605,6 +649,7 @@ module ActionAgent
         end
         trace_id!(result_metadata["trace_id"]) if result_metadata["trace_id"]
         judge_trace_ids!(result_metadata["judge_trace_ids"])
+        validate_judge_usage!(result["judge_usage"], "result.judge_usage")
       end
       distinct_specs = label_specs.values.uniq
       raise Invalid, "two model labels name the same provider/model" if distinct_specs.size < label_specs.size
@@ -655,6 +700,40 @@ module ActionAgent
       string!(judge.dig("suggested_tool", "name"), "result.diagnosis.judge.suggested_tool.name", 200)
     end
 
+    # The release the report says it evaluated: a digest the dashboard can
+    # match to a version cut on deploy, and the deploy and label as text.
+    def validate_release!
+      release = report["release"]
+      return if release.nil?
+
+      object!(release, "report.release")
+      unless release["digest"].nil? || (release["digest"].is_a?(String) && DIGEST_PATTERN.match?(release["digest"]))
+        raise Invalid, "report.release.digest must be 1-64 letters, digits or . _ -"
+      end
+
+      string!(release["revision"], "report.release.revision", 200)
+      string!(release["label"], "report.release.label", 200)
+    end
+
+    # What the judge spent, as ActiveAgent::Evals::Report writes it per
+    # result and for the run.
+    def validate_judge_usage!(usage, name)
+      return if usage.nil?
+
+      object!(usage, name)
+      %w[calls input_tokens output_tokens].each do |key|
+        next if usage[key].nil?
+        raise Invalid, "#{name}.#{key} must be a non-negative integer" unless usage[key].is_a?(Integer) && usage[key] >= 0 && usage[key] <= NUMERIC_LIMITS["input_tokens"]
+      end
+      numeric!(usage["cost"], "#{name}.cost", max: NUMERIC_LIMITS["cost"])
+      string!(usage["model"], "#{name}.model", 200)
+      string!(usage["source"], "#{name}.source", 20)
+      optional_object!(usage["by_kind"], "#{name}.by_kind")
+      (usage["by_kind"] || {}).each_value do |count|
+        raise Invalid, "#{name}.by_kind counts must be non-negative integers" unless count.is_a?(Integer) && count >= 0
+      end
+    end
+
     def validate_verdict!
       verdict = report["verdict"]
       return if verdict.nil?