actionagent 1.9.1 → 1.9.2

data/app/controllers/action_agent/api/evaluations_controller.rb

@@ -82,18 +82,30 @@ module ActionAgent
       # `evaluation: { archived: true | false }` archives the evaluation —
       # it keeps its runs but leaves the index, the pooled pass rate and the
       # agent's scorecard — or brings it back. A new run brings it back too.
+      # `evaluation: { judge_model: "…" }` pins the model the judge runs on
+      # from the next run; blank unpins it, back to the provider's default.
       def update
         evaluation = evaluations_scope.find(params[:id])
-        archived = params.require(:evaluation)[:archived]
-        if archived.nil?
+        attributes = params.require(:evaluation)
+        archived = attributes[:archived]
+        pin_judge = attributes.key?(:judge_model)
+        if archived.nil? && !pin_judge
           return render json: { errors: [ "evaluation.archived must be true or false" ] }, status: :unprocessable_entity
         end
 
-        ActiveModel::Type::Boolean.new.cast(archived) ? evaluation.archive! : evaluation.unarchive!
+        judge_model = attributes[:judge_model].to_s.strip.presence
+        if judge_model && judge_model.length > MAX_JUDGE_MODEL
+          return render json: { errors: [ "evaluation.judge_model is longer than #{MAX_JUDGE_MODEL} characters" ] }, status: :unprocessable_entity
+        end
+
+        evaluation.update!(judge_model: judge_model) if pin_judge
+        ActiveModel::Type::Boolean.new.cast(archived) ? evaluation.archive! : evaluation.unarchive! unless archived.nil?
 
         render json: { evaluation: serialize(evaluation.reload) }
       end
 
+      MAX_JUDGE_MODEL = 200
+
       # Runs listed per evaluation on GET /api/evaluations/:id. The rest of
       # the history stays reachable by run id; `run_count` says how long it is.
       RUN_HISTORY_LIMIT = 20

data/app/controllers/concerns/action_agent/api/mcp_dashboard_tools.rb

@@ -270,7 +270,7 @@ module ActionAgent
             properties: {
               evaluation_id: { type: "integer", description: "The evaluation's id" },
               run_id: { type: "integer", description: "The run's id; the latest run when omitted" },
-              failed_only: { type: "boolean", description: "Only results that did not pass" },
+              failed_only: { type: "boolean", description: "Only results that failed or errored (a running run's pending results are neither)" },
               limit: { type: "integer", description: "At most this many results (default #{RESULT_LIMIT}, max #{MAX_RESULT_LIMIT})" }
             },
             required: [ "evaluation_id" ]
@@ -737,7 +737,7 @@ module ActionAgent
 
         results = run.scenario_results.includes(:scenario, :agent_run).joins(:scenario)
           .order(EvaluationScenario.arel_table[:position], EvaluationScenario.arel_table[:id], :model)
-        results = results.where.not(status: :passed) if boolean_argument(:failed_only)
+        results = results.where(status: %i[failed errored]) if boolean_argument(:failed_only)
         total = results.count
         page = results.limit(limit).to_a
         recorded = recorded_trace_ids(page.filter_map { |result| result.agent_run&.trace_id })

data/app/jobs/action_agent/evaluation_judge_job.rb

@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+module ActionAgent
+  # Scores, diagnoses and stores one pending result of a fanned-out
+  # evaluation run (see EvaluationReplayJob). The job that scores the run's
+  # last pending result completes the run.
+  #
+  # A failure while scoring is stored on the result as an error rather than
+  # raised, so the run still completes.
+  class EvaluationJudgeJob < ApplicationJob
+    queue_as :agents
+
+    discard_on ActiveRecord::RecordNotFound
+
+    def perform(result_id, selection = {}, run_id = nil)
+      row = EvaluationScenarioResult.find_by(id: result_id)
+      return finish_without_result(run_id, selection) unless row
+
+      run = row.evaluation_run
+      begin
+        ScenarioEvaluationRunner.new(run.evaluation, selection: selection, run: run).judge_result!(row)
+      rescue StandardError => e
+        Rails.logger.warn("[ActionAgent] evaluation run #{run.id}: scoring result #{result_id} failed: #{e.class}: #{e.message}")
+        ScenarioEvaluationRunner.new(run.evaluation, selection: selection, run: run).fail_judging!(row.reload, e)
+      end
+    end
+
+    private
+
+    # A result destroyed before it was scored went with its scenario, removed
+    # from the suite during the run (EvaluationScenario destroys its results).
+    # It may have been the last result the run waited for, so the run's
+    # completion is checked without it.
+    def finish_without_result(run_id, selection)
+      run = run_id && EvaluationRun.find_by(id: run_id)
+      ScenarioEvaluationRunner.new(run.evaluation, selection: selection, run: run).finish_fan_out!(run) if run
+    end
+  end
+end

data/app/jobs/action_agent/evaluation_replay_job.rb

@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+module ActionAgent
+  # Replays one scenario under one model for a fanned-out evaluation run
+  # (ActionAgent.scenario_evaluation_fan_out) and stores the answer as a
+  # pending result; EvaluationJudgeJob scores it. One is enqueued per
+  # scenario x model, so the run is as parallel as the :agents queue.
+  #
+  # A failure outside the replay is stored as the pair's errored result
+  # rather than raised, so the run still completes; a scenario removed from
+  # the suite during the run is left out of it.
+  class EvaluationReplayJob < ApplicationJob
+    queue_as :agents
+
+    discard_on ActiveRecord::RecordNotFound
+
+    def perform(run_id, scenario_id, spec, selection = {})
+      run = EvaluationRun.find(run_id)
+      runner = ScenarioEvaluationRunner.new(run.evaluation, selection: selection, run: run)
+      begin
+        runner.replay_scenario!(run, scenario_id, spec)
+      rescue StandardError => e
+        Rails.logger.warn("[ActionAgent] evaluation run #{run_id}: replay of scenario #{scenario_id} failed: #{e.class}: #{e.message}")
+        ScenarioEvaluationRunner.new(run.evaluation, selection: selection, run: run).fail_result!(run.reload, scenario_id, spec, e)
+      end
+    end
+  end
+end

data/app/models/action_agent/evaluation_run.rb

@@ -78,6 +78,15 @@ module ActionAgent
       verdict.to_h.stringify_keys.presence if verdict.is_a?(Hash)
     end
 
+    # The model comparison a comparison run recorded (Report#comparison):
+    # each model's pass rate with its interval, every pair's difference and
+    # whether it is decisive, and the conclusion. nil for a single-model
+    # run, or one recorded before runs kept it.
+    def recorded_comparison
+      comparison = scores&.dig("_comparison")
+      comparison if comparison.is_a?(Hash) && comparison.present?
+    end
+
     # How the report names the judge, the way the suite panel does: the
     # judge the recorded verdict names — unless that is only the pass-rate
     # ranking — else the evaluation's judge model. nil when neither is set,
@@ -220,7 +229,7 @@ module ActionAgent
     # writes with Report#to_html. The run's recorded verdict and judge go
     # with it: the report is not to re-rank the rebuilt results by pass
     # rate and show a different judge's pick, verdict or `judged by` than
-    # the suite panel does. Raises ActiveRecord::RecordNotFound via the
+    # the suite panel does, nor resample the comparison the run recorded. Raises ActiveRecord::RecordNotFound via the
     # caller for a run of a generation-sampling evaluation, which has no
     # scenario results to report on.
     #
@@ -234,7 +243,10 @@ module ActionAgent
     # The engine runs against every activeagent since 1.4, so the report
     # kwargs that arrived later are passed only when Report.new takes them.
     def to_report(links: report_links, estimate: true)
-      rows = scenario_results.includes(:scenario).sort_by do |row|
+      # A replay not yet scored (a fanned-out run in progress) is not a
+      # result the report can grade, so it is left out rather than read as
+      # a failure.
+      rows = scenario_results.where.not(status: :pending).includes(:scenario).sort_by do |row|
         [ row.evaluated_scenario["position"].to_i, row.evaluation_scenario_id, row.model ]
       end
       breakdown = cost_breakdown if estimate
@@ -276,6 +288,7 @@ module ActionAgent
       }
       kwargs[:judge_usage] = breakdown.judge_usage_run if breakdown && self.class.report_accepts?(:judge_usage)
       kwargs[:release] = release_summary if release_summary && self.class.report_accepts?(:release)
+      kwargs[:comparison] = recorded_comparison if recorded_comparison && self.class.report_accepts?(:comparison)
       self.class.report_class.new(**kwargs)
     end
 
@@ -319,7 +332,11 @@ module ActionAgent
     # for it, and — when estimating — how its cost was priced.
     def replay_metadata_for(row, breakdown)
       metadata = row.replay_metadata.dup
-      reported_judge = row.diagnosis.is_a?(Hash) ? row.diagnosis["_judge_usage"] : nil
+      # A fanned-out run's results each keep what their scoring job spent on
+      # the judge, already folded into the run's meter. The meter is the
+      # run's figure (EvaluationRunCost#result_judge reads it the same way),
+      # so the parts are not counted a second time.
+      reported_judge = row.diagnosis.is_a?(Hash) && !judge_usage_meter ? row.diagnosis["_judge_usage"] : nil
       metadata["judge_usage"] = reported_judge if reported_judge.is_a?(Hash)
       return metadata unless breakdown
 

data/app/services/action_agent/evaluation_fix_comparison.rb

@@ -6,8 +6,11 @@ module ActionAgent
       @session = session
     end
 
+    # +after+ is pending while the verification run is in progress and, with
+    # a fanned-out run, while its stored replay waits to be scored: not a
+    # result yet, so not a regression.
     def self.change(before, after)
-      return "pending" unless after
+      return "pending" if after.nil? || after.pending?
       return after.passed? ? "added_pass" : "added_fail" unless before
       return after.passed? ? "unchanged_pass" : "regressed" if before.passed?
       after.passed? ? "fixed" : "still_failing"

data/app/services/action_agent/evaluation_report_import.rb

@@ -568,12 +568,17 @@ module ActionAgent
     # results as the application reported them — no cost is estimated here,
     # so the summaries stored stay the application's own figures; the API
     # adds the estimates when it serves the run (EvaluationRun#model_summaries).
+    #
+    # The comparison is worked out here from the stored results too, rather
+    # than taken from the payload, like the summaries.
     def summarized_scores(run)
       rebuilt = run.to_report(estimate: false)
+      comparison = rebuilt.comparison if rebuilt.respond_to?(:comparison)
       rebuilt.criterion_scores.merge(
         "_models" => rebuilt.summary_by_model,
-        "_recommendations" => rebuilt.recommendations
-      ).merge(recorded_scores)
+        "_recommendations" => rebuilt.recommendations,
+        "_comparison" => comparison
+      ).compact.merge(recorded_scores)
     end
 
     # --- validation ------------------------------------------------------------

data/app/services/action_agent/evaluation_runner_service.rb

@@ -10,6 +10,8 @@ module ActionAgent
   # credentials and is skipped — never faked — when none are available.
   class EvaluationRunnerService
     PASS_THRESHOLD = 0.7
+    # The class name the judge runs as, which its traces and the run record.
+    JUDGE_AGENT_NAME = "EvaluationJudgeAgent"
 
     def self.call(evaluation)
       new(evaluation).call
@@ -66,6 +68,7 @@ module ActionAgent
 
       scores["_cohorts"] = cohort_summaries(samples, per_sample_scores) if samples.any?
       scores["_judge_usage"] = judge_usage if judge_usage
+      scores["_judge"] = judge_descriptor if judge_usage
 
       run.update!(
         status: :complete,
@@ -136,6 +139,7 @@ module ActionAgent
         [ model, cohort_summary(samples, per_model_sample_scores[model] || {}) ]
       end
       scores["_judge_usage"] = judge_usage if judge_usage
+      scores["_judge"] = judge_descriptor if judge_usage
 
       run.update!(
         status: :complete,
@@ -236,7 +240,8 @@ module ActionAgent
     # first. Persisted as scores["_judge_usage"]:
     #
     #   { "calls", "input_tokens", "output_tokens", "cost", "model",
-    #     "by_kind" => { "score" => n, "recommend" => n, "verdict" => n, "define" => n } }
+    #     "by_kind" => { "score" => n, "recommend" => n, "verdict" => n, "define" => n },
+    #     "models" => { "<model that answered>" => n } }
     #
     # nil until the judge has been asked something, so a rules-only run
     # records no judge at all rather than a judge that cost nothing.
@@ -263,13 +268,42 @@ module ActionAgent
       model = (response.respond_to?(:model) && response.model.presence) || @evaluation.judge_model.presence
       cost = ModelPricing.estimate(model: model, provider: judge_provider, input_tokens: input_tokens, output_tokens: output_tokens)
 
-      @judge_usage ||= { "calls" => 0, "input_tokens" => 0, "output_tokens" => 0, "cost" => nil, "model" => nil, "by_kind" => {} }
+      @judge_usage ||= { "calls" => 0, "input_tokens" => 0, "output_tokens" => 0, "cost" => nil, "model" => nil, "by_kind" => {}, "models" => {} }
       @judge_usage["calls"] += 1
       @judge_usage["input_tokens"] += input_tokens
       @judge_usage["output_tokens"] += output_tokens
       @judge_usage["cost"] = (@judge_usage["cost"] || 0.0) + cost if cost
       @judge_usage["model"] ||= model
       @judge_usage["by_kind"][kind.to_s] = @judge_usage["by_kind"].fetch(kind.to_s, 0) + 1
+      (@judge_usage["models"] ||= {})[model] = @judge_usage["models"].fetch(model, 0) + 1 if model
+    end
+
+    # The judge as a run records it (scores["_judge"]): the agent class it
+    # runs as, its provider, the model it asks for and where that setting
+    # came from, and the models that answered. A judge with no model pinned
+    # runs on its provider's default, and OpenRouter's (openrouter/auto) is a
+    # router that can answer each call with a different model, so "answered"
+    # is what actually judged. nil when no provider has credentials.
+    def judge_descriptor
+      return nil unless judge_provider
+
+      {
+        "agent" => JUDGE_AGENT_NAME,
+        "provider" => judge_provider.to_s,
+        "model" => judge_requested_model,
+        "model_source" => judge_model_source,
+        "answered" => judge_usage&.dig("models").to_h
+      }
+    end
+
+    # How a run names its judge: the model it asked for, else the one model
+    # that answered, else the provider's default.
+    def judge_display_label
+      return nil unless judge_provider
+      return judge_requested_model if judge_requested_model
+
+      answered = judge_usage&.dig("models").to_h.keys
+      answered.one? ? answered.first : "#{judge_provider} default"
     end
 
     # Every token the judge was billed for reading. Anthropic reports the
@@ -602,6 +636,28 @@ module ActionAgent
       @owner_provider_options[name.to_s] ||= ProviderCredentials.resolve(owner: owner, actor: nil, provider: name).options
     end
 
+    # The model the judge asks for, in the order judge_class applies them:
+    # the evaluation's judge_model, the owner's provider key, the host's
+    # config/active_agent.yml. nil leaves the choice to the provider.
+    def judge_requested_model
+      judge_model_setting&.last
+    end
+
+    def judge_model_source
+      judge_model_setting&.first || "provider_default"
+    end
+
+    def judge_model_setting
+      return @judge_model_setting if defined?(@judge_model_setting)
+
+      config = ActiveAgent.configuration[judge_provider]
+      @judge_model_setting = {
+        "evaluation" => @evaluation.judge_model,
+        "provider_key" => owner_provider_options(judge_provider)[:model],
+        "config" => (config[:model] if config.respond_to?(:[]))
+      }.find { |_source, model| model.present? }&.then { |source, model| [ source, model.to_s ] }
+    end
+
     def owner
       @owner ||= @evaluation&.agent&.owner
     end
@@ -615,7 +671,7 @@ module ActionAgent
         options.merge!(owner_provider_options(provider))
 
         Class.new(ActiveAgent::Base) do
-          define_singleton_method(:name) { "EvaluationJudgeAgent" }
+          define_singleton_method(:name) { JUDGE_AGENT_NAME }
           generate_with provider, **options
         end
       end