actionagent 1.8.0 → 1.9.0

data/app/services/action_agent/evaluation_fix.rb

@@ -0,0 +1,101 @@
+# frozen_string_literal: true
+
+module ActionAgent
+  # A bounded, server-built brief for one persisted fix card. Request bodies
+  # choose an existing card and a subset of its scope; they cannot inject a
+  # different prompt, agent, scenario or model into the automatic fix loop.
+  class EvaluationFix
+    class Invalid < StandardError; end
+    attr_reader :run, :item
+
+    def initialize(run, requested)
+      @run = run
+      raise Invalid, "Only completed scenario runs have implementable fixes" unless run.complete? && run.evaluation.scenario_suite?
+      requested = requested.to_h.deep_stringify_keys
+      original = EvaluationSerializer.fix_items(run).find do |candidate|
+        candidate = candidate.deep_stringify_keys
+        candidate["kind"] == requested["kind"] &&
+          (requested["kind"] == "instruction" ? candidate["quote"] == requested["quote"] : candidate["fault"] == requested["fault"])
+      end
+      raise Invalid, "This fix card no longer exists on the selected run" unless original && %w[fault instruction].include?(requested["kind"])
+
+      @item = original.deep_stringify_keys.slice("kind", "fault", "quote", "recommendation", "scenario_keys", "models")
+      keys = Array(requested["scenario_keys"])
+      allowed_models = Array(@item["models"]).presence || model_labels
+      models = Array(requested["models"]).presence || allowed_models
+      unless keys.any? && (keys - Array(@item["scenario_keys"])).empty? && models.any? && (models - allowed_models).empty?
+        raise Invalid, "Choose scenarios and models from this fix card"
+      end
+      @item.merge!("scenario_keys" => keys.uniq, "models" => models.uniq)
+      @item["scenarios"] = run.scenario_results.includes(:scenario).filter_map do |result|
+        snapshot = result.evaluated_scenario.slice("key", "prompt", "expectations", "notes", "enabled")
+        snapshot if keys.include?(snapshot["key"])
+      end.uniq
+      self.class.check_scenarios!(run.evaluation, @item)
+    end
+
+    def self.check_scenarios!(evaluation, item)
+      Array(item["scenarios"]).each do |snapshot|
+        current = evaluation.scenarios.find_by(key: snapshot.fetch("key"))
+        unless current&.enabled? && current.as_json_summary.stringify_keys.slice(*snapshot.keys) == snapshot
+          raise Invalid, "The selected scenarios changed since this run. Run the evaluation again before implementing a fix."
+        end
+      end
+    end
+
+    def model_labels
+      run.scenario_results.map { |result| [ result.provider, result.model ].compact_blank.join("/") }.uniq
+    end
+
+    # How much of each piece of evidence the brief quotes, at full size. A
+    # card with many scenarios and models, or a retry, shrinks them all
+    # (see #prompt) rather than refusing the card.
+    EXCERPTS = { prompt: 1000, expectations: 1000, diagnosis: 1200, answer: 1000,
+      previous_result: 1500, previous_diff: 5000, previous_results: 3000 }.freeze
+    SCALES = [ 1, 0.5, 0.25, 0.1 ].freeze
+
+    def prompt(previous: nil)
+      SCALES.each do |scale|
+        text = brief(previous, EXCERPTS.transform_values { |size| (size * scale).to_i })
+        return text if text.length <= CodeSession::MAX_PROMPT_CHARACTERS
+      end
+      raise Invalid, "This fix has too much context for one session; run the evaluation on fewer scenarios or models"
+    end
+
+    private
+
+    def brief(previous, limit)
+      agent = run.evaluation.agent
+      name = agent.agent_class_name.presence || agent.telemetry_agent_class
+      path = name.to_s.underscore
+      lines = [ "# Implement one evaluation fix", "", "Agent: #{name}", "Evaluation: #{run.evaluation.name} · run #{run.id}",
+        "", "## What to fix", item["fault"], item["recommendation"], item["quote"],
+        "", "## Where to look", "- app/agents/#{path}.rb and its tools.",
+        "- app/views/agents/#{path.delete_suffix('_agent')}/ and app/views/#{path}/ for instructions and prompts.",
+        "- Follow the checkout's own instructions and locate the source of a synced agent before editing.",
+        "", "## Failing scenarios" ]
+      run.scenario_results.includes(:scenario).each do |result|
+        scenario = result.evaluated_scenario
+        next unless item["scenario_keys"].include?(scenario["key"])
+        label = [ result.provider, result.model ].compact_blank.join("/")
+        next unless item["models"].include?(label) || item["models"].include?(result.model)
+        lines += [ "", "### #{scenario['key']} · #{label}",
+          "Prompt: #{scenario['prompt'].to_s.truncate(limit[:prompt])}", "Expectations: #{scenario['expectations'].to_json.truncate(limit[:expectations])}",
+          "Before: #{result.status} · score #{result.score} · fault #{result.fault}",
+          "Diagnosis: #{result.diagnosis.to_json.truncate(limit[:diagnosis])}", "Answer: #{result.output.to_s.truncate(limit[:answer])}" ]
+      end
+      if previous
+        lines += [ "", "## Previous attempt", previous.result.to_s.truncate(limit[:previous_result]),
+          "Previous diff:", previous.diff.to_s.truncate(limit[:previous_diff]),
+          "Verification results: #{EvaluationFixComparison.new(previous).rows.to_json.truncate(limit[:previous_results])}" ]
+      end
+      lines += [ "", "## Verify", "Change the agent, not the scenarios or their expectations.",
+        "Run the relevant checkout tests. Do not commit or push.",
+        "The dashboard will re-run these exact scenarios and models against this sandbox when you finish.",
+        "Do not call dashboard URLs or dashboard MCP tools from this sandbox.",
+        "Never read or copy the sandbox's Claude configuration or credentials.",
+        "Summarize the change and any remaining limitations." ]
+      lines.compact.join("\n")
+    end
+  end
+end

data/app/services/action_agent/evaluation_fix_comparison.rb

@@ -0,0 +1,66 @@
+# frozen_string_literal: true
+
+module ActionAgent
+  class EvaluationFixComparison
+    def initialize(session)
+      @session = session
+    end
+
+    def self.change(before, after)
+      return "pending" unless after
+      return after.passed? ? "added_pass" : "added_fail" unless before
+      return after.passed? ? "unchanged_pass" : "regressed" if before.passed?
+      after.passed? ? "fixed" : "still_failing"
+    end
+
+    def rows
+      before = indexed(@session.evaluation_run)
+      after = indexed(@session.verification_run)
+      (before.keys | after.keys).map do |key|
+        first, last = before[key], after[key]
+        { scenario_key: key[0], model: key[1], change: self.class.change(first, last),
+          before: first&.status, after: last&.status, before_score: first&.score&.to_f, after_score: last&.score&.to_f,
+          score_change: first&.score && last&.score ? (last.score - first.score).to_f.round(3) : nil,
+          before_fault: first&.fault, fault: last&.fault }
+      end
+    end
+
+    def summary
+      { run_id: @session.verification_run_id, status: @session.verification_run&.status,
+        error: @session.verification_error || @session.verification_run&.error_message,
+        counts: rows.group_by { |row| row[:change] }.transform_values(&:size), rows: rows }
+    end
+
+    # The before/after section a fix's draft pull request carries, cut to
+    # +limit+ characters from the end (the session summary goes first), so
+    # appending it never pushes a body over DraftPullRequest's maximum.
+    def pull_request_body(mount:, limit: DraftPullRequest::MAX_BODY_CHARACTERS)
+      base = @session.evaluation_run
+      title = @session.fix_item["quote"].presence || @session.fix_item["fault"].to_s.humanize
+      lines = [ "## Evaluation fix", title, "",
+        "[Original run](#{mount}/evaluations/#{base.evaluation_id}/runs/#{base.id})",
+        "[Verification run](#{mount}/evaluations/#{base.evaluation_id}/runs/#{@session.verification_run_id})", "",
+        "| Scenario / model | Before | After | Score change | Fault |", "| --- | --- | --- | --- | --- |" ]
+      rows.each do |row|
+        cells = [ "#{row[:scenario_key]} / #{row[:model]}", row[:before], row[:after] || "pending", row[:score_change], row[:fault] || "—" ]
+        lines << "| #{cells.map { |cell| cell.to_s.gsub('|', '\\|').gsub(/[\r\n]/, ' ') }.join(' | ')} |"
+      end
+      lines += [ "", "## Session summary", @session.result.to_s, "", "Code session ##{@session.id}. Review the diff and full-suite regressions before merging." ]
+      SecretScrubber.scrub(lines.join("\n"), @session.secrets).truncate([ limit, 0 ].max, omission: "\n…")
+    end
+
+    private
+
+    def indexed(run)
+      return {} unless run
+      item = @session.fix_item || {}
+      run.scenario_results.includes(:scenario).each_with_object({}) do |result, index|
+        key = result.evaluated_scenario["key"]
+        model = [ result.provider, result.model ].compact_blank.join("/")
+        next unless Array(item["scenario_keys"]).include?(key) &&
+          (Array(item["models"]).include?(model) || Array(item["models"]).include?(result.model))
+        index[[ key, model ]] = result
+      end
+    end
+  end
+end

data/app/services/action_agent/evaluation_report_import.rb

@@ -22,7 +22,12 @@ module ActionAgent
   #
   # The run's per-model summary, criterion scores and recommendations are
   # computed from the stored results, not taken from the report. The judge's
-  # verdict and label are taken from it.
+  # verdict and label are taken from it, and so is what the judge spent —
+  # per result (`result.judge_usage`, kept beside the result's diagnosis) and
+  # for the run (`report.judge_usage.run`) — and the release the report says
+  # it evaluated (`report.release`), which pins the run to that version of
+  # the agent (Agent#find_or_record_release!). A report naming no release
+  # leaves the run unrecorded rather than claiming the dashboard's latest.
   #
   # An identical retry returns the stored run, before anything is asked of the
   # `admit` callable. Different content under a run_id already stored raises
@@ -70,6 +75,11 @@ module ActionAgent
     IDENTIFIER_PATTERN = /\A[^[:cntrl:]]{1,200}\z/
     AGENT_NAME_PATTERN = /\A[^[:cntrl:]]{2,100}\z/
     TRACE_PATTERN = /\A[a-zA-Z0-9_-]{1,128}\z/
+    DIGEST_PATTERN = /\A[a-zA-Z0-9._-]{1,64}\z/
+    # The judge usage keys a report may send, per result and for the run.
+    JUDGE_USAGE_KEYS = %w[calls input_tokens output_tokens cost model by_kind source].freeze
+    JUDGE_USAGE_SOURCES = %w[reported traces meter].freeze
+    MAX_JUDGE_KINDS = 10
     SCOPE_PATTERN = %r{\A[\w .:/@-]{1,100}\z}
     STATUSES = %w[passed failed errored].freeze
     # Report metadata that tells one evaluation of a suite from another, in the
@@ -253,6 +263,7 @@ module ActionAgent
         external_tenant: tenant_key,
         external_run_id: @payload["run_id"],
         external_report_digest: digest,
+        agent_version: reported_release_version(agent),
         status: :complete,
         selection: selection,
         scores: recorded_scores,
@@ -267,6 +278,15 @@ module ActionAgent
       run
     end
 
+    # The agent version for the release the report names, or nil for a
+    # report that names none.
+    def reported_release_version(agent)
+      release = report["release"]
+      return nil unless release.is_a?(Hash) && release["digest"].present?
+
+      agent.find_or_record_release!(digest: release["digest"], revision: release["revision"].presence, label: release["label"].presence)
+    end
+
     # --- ownership -------------------------------------------------------------
 
     # Who a newly observed agent belongs to. Whatever the host's
@@ -492,11 +512,28 @@ module ActionAgent
             "key" => result["scenario_key"], "group" => result["group"], "prompt" => result["prompt"],
             "position" => scenario.position, "expectations" => {}
           }
-        ),
+        ).merge(bounded_judge_usage(result["judge_usage"]).then { |usage| usage ? { "_judge_usage" => usage } : {} }),
         error_message: truncated(result["error"], TEXT_BYTES)
       )
     end
 
+    # A judge usage as the engine stores it: the keys the dashboard reads,
+    # each cut to its type, or nil for none.
+    def bounded_judge_usage(usage)
+      return nil unless usage.is_a?(Hash)
+
+      by_kind = usage["by_kind"].is_a?(Hash) ? usage["by_kind"].first(MAX_JUDGE_KINDS).to_h { |kind, count| [ kind.to_s.first(40), count.to_i ] } : {}
+      {
+        "calls" => usage["calls"].to_i,
+        "input_tokens" => usage["input_tokens"].to_i,
+        "output_tokens" => usage["output_tokens"].to_i,
+        "cost" => usage["cost"].is_a?(Numeric) ? usage["cost"].to_f : nil,
+        "model" => usage["model"].is_a?(String) ? usage["model"].first(200).presence : nil,
+        "by_kind" => by_kind,
+        "source" => JUDGE_USAGE_SOURCES.include?(usage["source"]) ? usage["source"] : "reported"
+      }
+    end
+
     # The first +bytes+ bytes of +text+, dropping a character the cut splits,
     # or nil for blank text.
     def truncated(text, bytes)
@@ -521,14 +558,18 @@ module ActionAgent
       {
         "_verdict" => report["verdict"],
         "_selection" => selection,
-        "_metadata" => metadata
+        "_metadata" => metadata,
+        "_judge_usage_run" => bounded_judge_usage(report.dig("judge_usage", "run"))
       }.compact.merge("_judge_label" => judge_label)
     end
 
     # The run's scores in the shape the Evaluations view renders
-    # (ScenarioEvaluationRunner#scores_for), summarized from the stored results.
+    # (ScenarioEvaluationRunner#scores_for), summarized from the stored
+    # results as the application reported them — no cost is estimated here,
+    # so the summaries stored stay the application's own figures; the API
+    # adds the estimates when it serves the run (EvaluationRun#model_summaries).
     def summarized_scores(run)
-      rebuilt = run.to_report
+      rebuilt = run.to_report(estimate: false)
       rebuilt.criterion_scores.merge(
         "_models" => rebuilt.summary_by_model,
         "_recommendations" => rebuilt.recommendations
@@ -559,6 +600,9 @@ module ActionAgent
 
       judge_trace_ids!(metadata["judge_trace_ids"])
       string!(report["judge"], "report.judge", 200)
+      validate_release!
+      validate_judge_usage!(report.dig("judge_usage", "run"), "report.judge_usage.run") if report["judge_usage"].is_a?(Hash)
+      optional_object!(report["judge_usage"], "report.judge_usage")
       object!(report["models"], "report.models")
       raise Invalid, "report.models must contain 1-#{MAX_MODELS} models" unless report["models"].size.between?(1, MAX_MODELS)
       unless results.is_a?(Array) && results.size.between?(1, MAX_RESULTS)
@@ -605,6 +649,7 @@ module ActionAgent
         end
         trace_id!(result_metadata["trace_id"]) if result_metadata["trace_id"]
         judge_trace_ids!(result_metadata["judge_trace_ids"])
+        validate_judge_usage!(result["judge_usage"], "result.judge_usage")
       end
       distinct_specs = label_specs.values.uniq
       raise Invalid, "two model labels name the same provider/model" if distinct_specs.size < label_specs.size
@@ -655,6 +700,40 @@ module ActionAgent
       string!(judge.dig("suggested_tool", "name"), "result.diagnosis.judge.suggested_tool.name", 200)
     end
 
+    # The release the report says it evaluated: a digest the dashboard can
+    # match to a version cut on deploy, and the deploy and label as text.
+    def validate_release!
+      release = report["release"]
+      return if release.nil?
+
+      object!(release, "report.release")
+      unless release["digest"].nil? || (release["digest"].is_a?(String) && DIGEST_PATTERN.match?(release["digest"]))
+        raise Invalid, "report.release.digest must be 1-64 letters, digits or . _ -"
+      end
+
+      string!(release["revision"], "report.release.revision", 200)
+      string!(release["label"], "report.release.label", 200)
+    end
+
+    # What the judge spent, as ActiveAgent::Evals::Report writes it per
+    # result and for the run.
+    def validate_judge_usage!(usage, name)
+      return if usage.nil?
+
+      object!(usage, name)
+      %w[calls input_tokens output_tokens].each do |key|
+        next if usage[key].nil?
+        raise Invalid, "#{name}.#{key} must be a non-negative integer" unless usage[key].is_a?(Integer) && usage[key] >= 0 && usage[key] <= NUMERIC_LIMITS["input_tokens"]
+      end
+      numeric!(usage["cost"], "#{name}.cost", max: NUMERIC_LIMITS["cost"])
+      string!(usage["model"], "#{name}.model", 200)
+      string!(usage["source"], "#{name}.source", 20)
+      optional_object!(usage["by_kind"], "#{name}.by_kind")
+      (usage["by_kind"] || {}).each_value do |count|
+        raise Invalid, "#{name}.by_kind counts must be non-negative integers" unless count.is_a?(Integer) && count >= 0
+      end
+    end
+
     def validate_verdict!
       verdict = report["verdict"]
       return if verdict.nil?