actionagent 1.9.1 → 1.9.2

data/app/services/action_agent/scenario_evaluation_runner.rb

@@ -17,6 +17,9 @@ module ActionAgent
   #   "_models"          — per model: pass rate, mean score, latency, tokens, cost, fault counts
   #   "_recommendations" — faults grouped across scenarios with the fix each calls for
   #   "_verdict"         — the best model and why (judge-written when a judge is available)
+  #   "_comparison"      — when comparing models: each model's pass rate with its
+  #                        interval, every pair's difference and whether it is
+  #                        decisive (ActiveAgent::Evals::Comparison)
   #   "_selection"       — the scenarios and models this run covered, and
   #                        the checkout sandbox it replayed against, if any
   #
@@ -87,6 +90,9 @@ module ActionAgent
 
       ensure_browser_live!
 
+      adapter = ActionAgent.scenario_evaluation_adapter_resolver&.call(@evaluation)
+      return fan_out!(run, scenarios, specs) if fan_out?(adapter)
+
       records = scenarios.index_by(&:key)
       tasks = scenarios.map { |scenario| Evals::Scenario.from_hash(scenario.as_json_summary) }
       expected = tasks.product(specs).map { |task, spec| [ task.key, spec.label ] }
@@ -102,7 +108,6 @@ module ActionAgent
         persist(run, records.fetch(result.scenario.key), result)
         persisted << pair
       end
-      adapter = ActionAgent.scenario_evaluation_adapter_resolver&.call(@evaluation)
       report = if adapter
         raise ArgumentError, "scenario evaluation adapter must be callable" unless adapter.respond_to?(:call)
 
@@ -142,8 +147,361 @@ module ActionAgent
       stop_started_browser
     end
 
+    # --- fan-out (EvaluationReplayJob, EvaluationJudgeJob) -----------------
+    #
+    # With ActionAgent.scenario_evaluation_fan_out on, #call enqueues one
+    # EvaluationReplayJob per scenario x model and returns the run still
+    # running. Each replay is stored as a pending result and scored by its
+    # own EvaluationJudgeJob, so the agent's next replay need not wait for
+    # the judge. The job that scores the last pending result completes the
+    # run with the same scores the sequential path writes.
+    #
+    # The agent's tool roster is discovered once, before the replays, and
+    # kept with the run's bookkeeping (scores["_fan_out"]) for every job to
+    # read, as the sequential path resolves it once per run: a diagnosis
+    # names the same tools whichever job wrote it, and the run's discovery
+    # warning does not depend on which job completed the run.
+
+    # Replays one scenario under one model and stores the answer as a pending
+    # result, then enqueues its scoring. A result already stored for the
+    # pair (a retried job) is not replayed again.
+    def replay_scenario!(run, scenario_id, spec_value)
+      return unless run.running?
+
+      scenario = @evaluation.scenarios.find(scenario_id)
+      spec = model_spec_from(spec_value)
+      row = stored_result(run, scenario, spec)
+      unless row
+        task = Evals::Scenario.from_hash(scenario.as_json_summary)
+        replay = fan_out_replay(task, spec)
+        row = run.with_lock { stored_result(run, scenario, spec) || store_replay(run, scenario, task, spec, replay) }
+      end
+      EvaluationJudgeJob.perform_later(row.id, @selection.to_h, run.id) if row.pending?
+      row
+    end
+
+    # Scores, diagnoses and stores one pending result, then completes the run
+    # when it was the last.
+    #
+    # A job delivered again for a result that is already scored, or that a
+    # concurrent delivery has claimed, scores nothing. It still checks
+    # whether the run is complete: the earlier delivery may have stored the
+    # last result and stopped before completing the run.
+    def judge_result!(row)
+      run = row.evaluation_run
+      return unless run.running?
+      return finish_fan_out!(run) unless row.pending? && claim_result!(row)
+
+      spec = fan_out_specs(run).fetch([ row.provider.to_s, row.model ]) do
+        Evals::ModelSpec.new(label: row.model, provider: row.provider.to_s, model: row.model)
+      end
+      result = fan_out_scorer(run).evaluate(Evals::Scenario.from_hash(row.evaluated_scenario), spec, replay_from(row))
+      row.update!(
+        status: result.status,
+        score: result.score,
+        scores: result.scores,
+        fault: result.fault,
+        recommendation: result.recommendation,
+        diagnosis: (result.diagnosis || {}).merge(row.diagnosis.to_h.slice("_replay_metadata", "_scenario_snapshot"))
+          .merge("_judge_usage" => judge_usage).compact
+      )
+      # The result now holds what its scoring cost; whatever this runner asks
+      # the judge from here on (the verdict, when it completes the run) is the
+      # run's own.
+      @judge_usage = nil
+      finish_fan_out!(run)
+    end
+
+    # Stores +error+ as the result of a pair whose job failed outside the
+    # replay (which records its own errors), so the run still completes. A
+    # scenario removed from the suite during the run has no result to
+    # store; the run completes without it (see #finish_fan_out!).
+    def fail_result!(run, scenario_id, spec_value, error)
+      return unless run.running?
+
+      scenario = @evaluation.scenarios.find_by(id: scenario_id)
+      if scenario
+        spec = model_spec_from(spec_value)
+        run.with_lock do
+          row = stored_result(run, scenario, spec)
+          task = Evals::Scenario.from_hash(scenario.as_json_summary)
+          row ||= store_replay(run, scenario, task, spec, Evals::Replay.failed(error))
+          row.update!(status: :errored, error_message: row.error_message.presence || error_text(error)) if row.pending?
+        end
+      end
+      finish_fan_out!(run)
+    end
+
+    # Marks a pending result errored when its scoring job failed, so the run
+    # still completes.
+    def fail_judging!(row, error)
+      run = row.evaluation_run
+      row.update!(status: :errored, error_message: [ row.error_message.presence, "Scoring failed: #{error_text(error)}" ].compact.join(" ")) if row.pending?
+      finish_fan_out!(run)
+    end
+
+    # Completes the run once every selected scenario x model has a scored
+    # result, with the scores, verdict and judge usage a sequential run
+    # writes. A scenario removed from the suite during the run is no longer
+    # waited for (its results went with it), so the run completes without it.
+    #
+    # Nothing is computed under a lock. The verdict is a judge call, and a
+    # lock on the run is, on SQLite, the write lock on the whole database:
+    # held for the call, it would stall every other writer. The lock covers
+    # only the check that the run is still running and the one write that
+    # completes it. Of two jobs that both find the run finished, one
+    # completes it and the other's work is dropped, at the cost of a second
+    # verdict call in that rare tie.
+    def finish_fan_out!(run)
+      run.reload
+      return unless run.running?
+
+      specs = fan_out_specs(run)
+      rows = run.scenario_results.includes(:agent_run).to_a
+      return if rows.size < expected_results(run, specs) || rows.any?(&:pending?)
+
+      attributes = begin
+        completion_attributes(run, rows, specs)
+      rescue StandardError => e
+        { status: :failed, error_message: e.message, completed_at: Time.current }
+      end
+      run.with_lock do
+        run.update!(attributes) if run.running?
+      end
+    end
+
     private
 
+    def fan_out?(adapter)
+      ActionAgent.scenario_evaluation_fan_out == true && adapter.nil? && !browser_requested? &&
+        Evals::Runner.instance_method(:initialize).parameters.any? { |_type, name| name == :refine_gate }
+    end
+
+    def fan_out!(run, scenarios, specs)
+      refuse_duplicate_specs!(specs)
+      ensure_judge_defined_kpis! if @evaluation.judge_defined?
+      # Discovered here, before any lock is taken and before a sandbox the
+      # run replays against can expire; the jobs read it from the run.
+      roster = { "tools" => tool_roster, "discovery_warning" => discovery_warning }
+      update_fan_out_state!(run) do |state|
+        state.merge(roster).merge(judge_usage ? { "judge_usage" => judge_usage } : {})
+      end
+
+      selection = @selection.to_h
+      scenarios.product(specs).each do |scenario, spec|
+        EvaluationReplayJob.perform_later(run.id, scenario.id, spec.to_h, selection)
+      end
+      run
+    end
+
+    # A fanned-out run keys its results by provider and model, where the
+    # sequential path keys them by label, so one model listed under two
+    # labels ("gpt-5-mini" and "openai/gpt-5-mini") would be replayed once
+    # and reported under one of them; the run refuses the selection instead.
+    def refuse_duplicate_specs!(specs)
+      duplicates = specs.group_by { |spec| [ spec.provider, spec.model ] }.select { |_pair, group| group.size > 1 }
+      return if duplicates.empty?
+
+      names = duplicates.map { |(provider, model), group| "#{provider}/#{model} as #{group.map(&:label).join(' and ')}" }
+      raise ArgumentError, "the same model is listed more than once (#{names.to_sentence}); list each provider/model once"
+    end
+
+    def model_spec_from(value)
+      value = value.to_h.stringify_keys
+      Evals::ModelSpec.new(label: value["label"].presence || value["model"], provider: value["provider"].to_s, model: value["model"])
+    end
+
+    # The run's candidate models, in the order it selected them, keyed by
+    # provider and model as a result row records them.
+    def fan_out_specs(run)
+      Array(run.selection&.dig("models")).filter_map do |entry|
+        model_spec_from(entry) if entry.is_a?(Hash) && entry["model"].present?
+      end.index_by { |spec| [ spec.provider, spec.model ] }
+    end
+
+    def stored_result(run, scenario, spec)
+      run.scenario_results.find_by(evaluation_scenario_id: scenario.id, model: spec.model, provider: spec.provider)
+    end
+
+    # How many results the run waits for: one per selected scenario still in
+    # the suite, per model.
+    def expected_results(run, specs)
+      ids = Array(run.selection&.dig("scenario_ids"))
+      @evaluation.scenarios.where(id: ids).count * specs.size
+    end
+
+    # Claims a pending result for this job before it is scored. Of two judge
+    # jobs delivered for one result (a replay job delivered again after it
+    # enqueued the first), only the one whose claim lands scores it, so the
+    # judge is asked once and the run's meter counts each call once. The
+    # claim is a conditional write on the row's timestamp: it holds no lock
+    # while the judge is asked, and a row whose claimant then stops is still
+    # pending for the next delivery.
+    def claim_result!(row)
+      EvaluationScenarioResult.where(id: row.id, status: :pending, updated_at: row.updated_at).update_all(updated_at: Time.current) == 1
+    end
+
+    # What Evals::Runner does with a replay that raises: an errored replay,
+    # scored like any other.
+    def fan_out_replay(task, spec)
+      replay(task, spec)
+    rescue StandardError => e
+      Evals::Replay.failed(e)
+    end
+
+    def store_replay(run, scenario, task, spec, replay)
+      run.scenario_results.create!(
+        scenario: scenario,
+        agent_run_id: replay.metadata["agent_run_id"],
+        model: spec.model,
+        provider: spec.provider,
+        status: :pending,
+        output: replay.answer.to_s.byteslice(0, 20_000).to_s.scrub.presence,
+        tool_calls: replay.tool_calls,
+        duration_ms: replay.duration_ms,
+        input_tokens: replay.input_tokens,
+        output_tokens: replay.output_tokens,
+        cost: replay.cost,
+        diagnosis: {
+          "_replay_metadata" => replay.metadata,
+          "_scenario_snapshot" => task.to_h.merge(expectations: task.expectations)
+        },
+        error_message: replay.error
+      )
+    end
+
+    # The stored replay, with the answer as the agent run recorded it. The
+    # result keeps the first 20,000 bytes of the answer to show; a rule or
+    # an expectation (contains, not_contains, min_length) reads all of it, as
+    # it does on the sequential path, so a long answer scores the same
+    # fanned out.
+    def replay_from(row)
+      Evals::Replay.new(
+        answer: row.agent_run&.output.presence || row.output, tool_calls: row.tool_calls, duration_ms: row.duration_ms,
+        input_tokens: row.input_tokens, output_tokens: row.output_tokens, error: row.error_message,
+        cost: row.cost&.to_f, metadata: row.replay_metadata
+      )
+    end
+
+    # A Runner that scores one replay. Its refine_gate shares the run-wide
+    # judge_limit with every other result's scorer, and its roster is the one
+    # discovered when the run fanned out (a run fanned out before the roster
+    # was kept discovers its own).
+    def fan_out_scorer(run)
+      Evals::Runner.new(
+        scenarios: [],
+        models: [],
+        criteria: sample_criteria,
+        judge: evals_judge,
+        available_tools: fan_out_state(run).fetch("tools") { tool_roster },
+        instructions: @evaluation.agent.instructions,
+        agent_name: @evaluation.agent.name,
+        threshold: PASS_THRESHOLD,
+        replay: ->(*) { raise ArgumentError, "a fanned-out result is scored from its stored replay" },
+        refine_gate: -> { reserve_refinement!(run) }
+      )
+    end
+
+    def reserve_refinement!(run)
+      reserved = false
+      update_fan_out_state!(run) do |state|
+        count = state["refinements"].to_i
+        next state if count >= Evals::Runner::DEFAULT_JUDGE_LIMIT
+
+        reserved = true
+        state.merge("refinements" => count + 1)
+      end
+      reserved
+    end
+
+    # The run's fan-out bookkeeping, kept in scores["_fan_out"] until the run
+    # completes and its scores are written: the tool roster and discovery
+    # warning found before the replays, the judge usage spent before them
+    # (judge-defined KPIs), and the recommendations reserved so far.
+    def fan_out_state(run)
+      state = run.scores.is_a?(Hash) ? run.scores["_fan_out"] : nil
+      state.is_a?(Hash) ? state : {}
+    end
+
+    # Changes the bookkeeping under the run's lock. A completed run's scores
+    # are left as written: a reservation that lands after the last result
+    # has nothing to reserve.
+    def update_fan_out_state!(run)
+      run.with_lock do
+        next unless run.running?
+
+        scores = run.scores.is_a?(Hash) ? run.scores : {}
+        run.update!(scores: scores.merge("_fan_out" => yield(fan_out_state(run))))
+      end
+    end
+
+    # What completes the run: the attributes the sequential path writes at
+    # the end of #call, from the stored results.
+    def completion_attributes(run, rows, specs)
+      if rows.empty?
+        return { status: :failed, error_message: "Every selected scenario was removed during the run", completed_at: Time.current }
+      end
+
+      order = specs.keys
+      rows = rows.sort_by do |row|
+        [ row.evaluated_scenario["position"].to_i, row.evaluation_scenario_id, order.index([ row.provider.to_s, row.model ]) || order.size ]
+      end
+      results = rows.map do |row|
+        Evals::Result.new(
+          scenario: Evals::Scenario.from_hash(row.evaluated_scenario),
+          spec: specs.fetch([ row.provider.to_s, row.model ]) { model_spec_from(row.attributes.slice("provider", "model")) },
+          replay: replay_from(row),
+          scores: row.scores.to_h, score: row.score, status: row.status,
+          diagnosis: row.evaluation_diagnosis.presence
+        )
+      end
+      report = Evals::Report.new(results: results, models: specs.values, judge: evals_judge,
+                                 instructions: @evaluation.agent.instructions, threshold: PASS_THRESHOLD)
+      state = fan_out_state(run)
+
+      # The other jobs' judge calls go into this runner's meter before the
+      # verdict is asked for, so the run's usage, and anything scores_for
+      # reads from the meter, covers every call once.
+      @judge_usage = combined_judge_usage([ state["judge_usage"], *rows.map { |row| row.diagnosis.to_h["_judge_usage"] }, judge_usage ])
+
+      {
+        status: :complete,
+        scores: scores_for(report, run),
+        samples_evaluated: results.size,
+        samples_passed: results.count(&:passed?),
+        error_message: state.key?("discovery_warning") ? state["discovery_warning"] : discovery_warning,
+        completed_at: Time.current
+      }
+    end
+
+    # Sums the judge usage of the run's parts into the one meter
+    # scores["_judge_usage"] holds for a sequential run.
+    def combined_judge_usage(parts)
+      parts = parts.select { |part| part.is_a?(Hash) }
+      return nil if parts.empty?
+
+      costs = parts.filter_map { |part| part["cost"] }
+      {
+        "calls" => parts.sum { |part| part["calls"].to_i },
+        "input_tokens" => parts.sum { |part| part["input_tokens"].to_i },
+        "output_tokens" => parts.sum { |part| part["output_tokens"].to_i },
+        "cost" => costs.any? ? costs.sum.round(6) : nil,
+        "model" => parts.filter_map { |part| part["model"] }.first,
+        "by_kind" => tally(parts, "by_kind"),
+        "models" => tally(parts, "models")
+      }
+    end
+
+    def tally(parts, key)
+      parts.each_with_object(Hash.new(0)) do |part, counts|
+        part[key].to_h.each { |name, count| counts[name] += count.to_i }
+      end.to_h
+    end
+
+    def error_text(error)
+      error.is_a?(Exception) ? "#{error.class}: #{error.message}" : error.to_s
+    end
+
     def default_report(tasks, specs, on_result)
       Evals::Runner.new(
         scenarios: tasks,
@@ -407,16 +765,35 @@ module ActionAgent
       scores["_models"] = report.summary_by_model
       scores["_recommendations"] = report.recommendations
       scores["_verdict"] = report.verdict if report.comparing?
+      # The framework gained Report#comparison after the oldest activeagent
+      # this engine runs against.
+      comparison = report.comparison if report.respond_to?(:comparison)
+      scores["_comparison"] = comparison if comparison
       scores["_selection"] = run.selection
       scores["_metadata"] = report.metadata
       scores["_judge_label"] = report.judge_label || report.judge&.label
       # The judge's own spend, apart from the replays' (which the results
       # carry and EvaluationRun#usage sums). A host adapter that runs its own
       # judge is out of reach of this meter, so its runs record none.
-      scores["_judge_usage"] = judge_usage if judge_usage
+      if judge_usage
+        scores["_judge_usage"] = judge_usage
+        name_judge!(scores, report.judge&.label)
+      end
       scores
     end
 
+    # Records who judged once the calls are in. A judge with no model pinned
+    # started out labelled by its provider; it is renamed after the model
+    # that answered, here and on the verdict it wrote.
+    def name_judge!(scores, provisional)
+      scores["_judge"] = judge_descriptor
+      label = judge_display_label
+      return unless label && provisional && scores["_judge_label"] == provisional
+
+      scores["_judge_label"] = label
+      scores["_verdict"]["judge"] = label if scores["_verdict"].is_a?(Hash) && scores["_verdict"]["judge"] == provisional
+    end
+
     # --- judge ------------------------------------------------------------
 
     def sample_criteria
@@ -481,7 +858,7 @@ module ActionAgent
     def evals_judge
       return nil unless judge_available?
 
-      @evals_judge ||= Evals::Judge.new(label: @evaluation.judge_model.presence || judge_provider.to_s) do |instructions:, prompt:, kind:|
+      @evals_judge ||= Evals::Judge.new(label: judge_display_label) do |instructions:, prompt:, kind:|
         judge_generate(kind, message: prompt, instructions: instructions).message&.content
       end
     end

data/lib/action_agent/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module ActionAgent
-  VERSION = "1.9.1"
+  VERSION = "1.9.2"
 end

data/lib/action_agent.rb

@@ -685,6 +685,20 @@ module ActionAgent
     # @return [Proc, nil]
     attr_accessor :scenario_evaluation_adapter_resolver
 
+    # Whether a scenario evaluation runs as background jobs, one per
+    # scenario x model (EvaluationReplayJob) followed by one to score it
+    # (EvaluationJudgeJob), instead of as one job that replays the suite in
+    # order. The jobs run on the :agents queue, so a run is as parallel as
+    # the host's workers for that queue: a single-worker box stays
+    # sequential. The last result to be scored completes the run.
+    #
+    # A run that drives a sandbox's browser, and a run a host adapter
+    # (scenario_evaluation_adapter_resolver) replays, stay sequential: the
+    # browser is one page, and the adapter owns its own scheduling.
+    # Off unless set.
+    # @return [Boolean]
+    attr_accessor :scenario_evaluation_fan_out
+
     # How many answerable candidates an exploration's review pre-selects for
     # acceptance: an Integer, or a Proc called with (owner) that returns one.
     # A platform can cap a free plan's suite this way. Unset, or nil from the
@@ -1243,6 +1257,7 @@ module ActionAgent
       @capture_dashboard_sessions = true
 
       @scenario_evaluation_adapter_resolver = nil
+      @scenario_evaluation_fan_out = false
       @table_name_prefix = "active_agent_"
       @agent_polymorphic_name = nil
       @encrypt_credentials = true

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: actionagent
 version: !ruby/object:Gem::Version
-  version: 1.9.1
+  version: 1.9.2
 platform: ruby
 authors:
 - Justin Bowen
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2026-10-17 00:00:00.000000000 Z
+date: 2026-10-20 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activeagent
@@ -190,6 +190,8 @@ files:
 - app/jobs/action_agent/application_job.rb
 - app/jobs/action_agent/code_session_job.rb
 - app/jobs/action_agent/draft_pull_request_job.rb
+- app/jobs/action_agent/evaluation_judge_job.rb
+- app/jobs/action_agent/evaluation_replay_job.rb
 - app/jobs/action_agent/evaluation_run_job.rb
 - app/jobs/action_agent/exploration_job.rb
 - app/jobs/action_agent/input_request_expiry_job.rb