actionagent 1.9.5 → 1.9.7

data/app/controllers/action_agent/api/agents_controller.rb

@@ -279,10 +279,10 @@ module ActionAgent
       # ones the caller may see as well. Intersected by id rather than with
       # merge: both relations constrain the same owner column, and merge
       # lets the second condition replace the first instead of adding to it.
+      # The servers the owner added on the MCP Services page are offered the
+      # same way.
       def tool_roster
-        runtimes = SandboxSession.runtime_server_listings(
-          SandboxSession.for_owner(@agent.owner).where(id: owned(SandboxSession).select(:id))
-        )
+        runtimes = owner_server_listings(@agent.owner)
 
         render json: AgentToolRoster.new(
           agent: @agent,
@@ -303,9 +303,7 @@ module ActionAgent
       # an agent's history: which services its agents already enable.
       def new_tool_roster
         agent = owner_agents.build
-        runtimes = SandboxSession.runtime_server_listings(
-          SandboxSession.for_owner(agent.owner).where(id: owned(SandboxSession).select(:id))
-        )
+        runtimes = owner_server_listings(agent.owner)
 
         render json: AgentToolRoster.new(
           agent: agent,
@@ -444,6 +442,17 @@ module ActionAgent
 
       private
 
+      # The checkout runtimes and added MCP servers an agent of +owner+ can be
+      # given, as MCP server listings: +owner+'s, and of those only the ones
+      # the caller may see as well.
+      def owner_server_listings(owner)
+        SandboxSession.runtime_server_listings(
+          SandboxSession.for_owner(owner).where(id: owned(SandboxSession).select(:id))
+        ) + MCPServerConfig.listings(
+          MCPServerConfig.for_owner(owner).where(id: owned(MCPServerConfig).select(:id))
+        )
+      end
+
       # Observed agents were discovered from reported telemetry; the platform
       # has no configuration to run them with (a placeholder model, no
       # instructions), so executing one only manufactured a failed run that

data/app/controllers/action_agent/api/base_controller.rb

@@ -114,8 +114,8 @@ module ActionAgent
 
       # Returns the credentials the caller owns, for masking with
       # SecretScrubber: provider keys, GitHub tokens, checkout sandboxes'
-      # runtime tokens, dashboard API keys (newest first) and the owner's
-      # telemetry key. Raises what a lookup raises, such as a credential that
+      # runtime tokens, dashboard API keys (newest first), the header values
+      # of the owner's own MCP servers and the owner's telemetry key. Raises what a lookup raises, such as a credential that
       # cannot be decrypted, so each caller decides whether to go on without
       # the list.
       # @return [Array<String>]
@@ -125,6 +125,7 @@ module ActionAgent
           *owned(GithubConnection).limit(OWNER_CREDENTIAL_LIMIT).pluck(:access_token),
           *owned(SandboxSession).where.not(runtime_mcp_token: nil).order(id: :desc).limit(OWNER_CREDENTIAL_LIMIT).pluck(:runtime_mcp_token),
           *owned(ApiKey).order(id: :desc).limit(OWNER_CREDENTIAL_LIMIT).pluck(:token),
+          *owned(MCPServerConfig).order(id: :desc).limit(OWNER_CREDENTIAL_LIMIT).flat_map(&:secret_values),
           current_owner.try(:telemetry_api_key)
         ].compact
       end

data/app/controllers/action_agent/api/mcp_server_configs_controller.rb

@@ -0,0 +1,114 @@
+# frozen_string_literal: true
+
+module ActionAgent
+  module Api
+    # The caller's own remote MCP servers (MCPServerConfig), managed from the
+    # MCP Services page and enabled on agents as "custom:<id>".
+    #
+    # Every query goes through `owned`, so a caller reads, changes, tests and
+    # deletes only their owner's configurations; another owner's id is a 404.
+    # Header values are write-only: responses carry each header's name and a
+    # masked value. On update, a header sent with a blank value keeps the
+    # value stored under that name, and a header left out is removed; leaving
+    # +headers+ out of the request keeps them all.
+    #
+    # Writes and tests ask permission_checker for :manage_credentials, since a
+    # configuration holds credentials and decides what the owner's agents
+    # call. Reading the list asks nothing.
+    class MCPServerConfigsController < BaseController
+      before_action :require_owner!
+      before_action :set_config, only: %i[update destroy test]
+
+      # GET /api/mcp_server_configs
+      def index
+        configs = owned(MCPServerConfig).order(:name, :id)
+
+        render json: {
+          mcp_server_configs: configs.map(&:as_api_json),
+          allow_private_hosts: MCPHostGuard.allow_private?,
+          can_manage: can_manage?
+        }
+      end
+
+      # POST /api/mcp_server_configs
+      def create
+        config = owned(MCPServerConfig).new
+        assign_attributes(config)
+        return unless authorize_action!(:manage_credentials, config)
+
+        config.save!
+        render json: { mcp_server_config: config.as_api_json }, status: :created
+      end
+
+      # PATCH /api/mcp_server_configs/:id
+      def update
+        return unless authorize_action!(:manage_credentials, @config)
+
+        assign_attributes(@config)
+        # A new URL has not been tested; what an earlier test found about the
+        # old one no longer describes it.
+        reset_test_result(@config) if @config.url_changed?
+        @config.save!
+        render json: { mcp_server_config: @config.as_api_json }
+      end
+
+      # DELETE /api/mcp_server_configs/:id
+      #
+      # Agents that enable the configuration keep naming its key, which then
+      # resolves to nothing: the Tools tab lists it as unknown so it can be
+      # switched off.
+      def destroy
+        return unless authorize_action!(:manage_credentials, @config)
+
+        @config.destroy!
+        head :no_content
+      end
+
+      # POST /api/mcp_server_configs/:id/test
+      #
+      # Runs tools/list against the saved configuration and records the
+      # outcome, the discovered tool names included. Answers 200 either way:
+      # a server that does not answer is a result to show, not a failed
+      # request.
+      def test
+        return unless authorize_action!(:manage_credentials, @config)
+
+        result = @config.test_connection!
+        render json: result.merge(mcp_server_config: @config.as_api_json)
+      end
+
+      private
+
+      def set_config
+        @config = owned(MCPServerConfig).find(params[:id])
+      end
+
+      def can_manage?
+        ActionAgent.permitted?(current_user, :manage_credentials, owned(MCPServerConfig).new)
+      end
+
+      def assign_attributes(config)
+        attributes = params.require(:mcp_server_config).permit(:name, :description, :url, :enabled)
+        config.assign_attributes(attributes)
+        config.assign_headers(header_params) if params[:mcp_server_config].key?(:headers)
+      end
+
+      # The submitted headers as { name, value } pairs. Accepts a list, or a
+      # Hash of name => value.
+      def header_params
+        submitted = params[:mcp_server_config][:headers]
+        return [] if submitted.blank?
+
+        if submitted.is_a?(Array)
+          submitted.map { |header| header.respond_to?(:permit) ? header.permit(:name, :value).to_h : {} }
+        else
+          submitted.permit!.to_h.map { |name, value| { "name" => name, "value" => value } }
+        end
+      end
+
+      def reset_test_result(config)
+        config.assign_attributes(last_tested_at: nil, last_test_status: nil, last_test_error: nil, tool_names: [])
+      end
+    end
+  end
+end

data/app/controllers/action_agent/api/mcp_servers_controller.rb

@@ -10,7 +10,8 @@ module ActionAgent
     # its configuration, and the default catalog (MCPCatalog). An install
     # therefore sees both what it is already using and what it could turn on.
     # The caller's live checkout sandbox runtimes are listed with the catalog,
-    # so the one started from Settings -> Integrations shows up here too.
+    # so the one started from Settings -> Integrations shows up here too, and
+    # so are the servers the caller added (MCPServerConfigsController).
     class MCPServersController < BaseController
       before_action :require_owner!
       # Launching provisions a sandbox and runs a server in it, so it answers
@@ -110,14 +111,16 @@ module ActionAgent
 
       private
 
-      # Runtimes are the caller's own: a listing carries no token, but which
-      # repositories another owner has running is still theirs to know.
+      # Runtimes and added servers are the caller's own: a listing carries no
+      # token or header, but which repositories another owner has running and
+      # which servers they call are still theirs to know.
       def discovery
         ToolDiscovery.new(
           traces: owned_traces,
           agents: owner_agents,
           hours: window_hours,
-          runtimes: SandboxSession.runtime_server_listings(owned(SandboxSession))
+          runtimes: SandboxSession.runtime_server_listings(owned(SandboxSession)) +
+            MCPServerConfig.listings(owned(MCPServerConfig))
         )
       end
 

data/app/controllers/action_agent/api/metrics_controller.rb

@@ -181,17 +181,18 @@ module ActionAgent
       end
 
       # Estimated spend for the window, total and per agent class. The model
-      # lives inside the spans JSON (first llm span).
+      # and the provider it ran on live inside the spans JSON (first llm
+      # span); the rate is the model's under that provider.
       def cost_statistics(scope)
-        rows = ActionAgent.trace_model.pluck_with_llm_model(
+        rows = ActionAgent.trace_model.pluck_with_llm_model_and_provider(
           scope, :agent_class, :total_input_tokens, :total_output_tokens
         )
 
         by_agent = Hash.new(0.0)
         total = 0.0
 
-        rows.each do |model, agent_class, input_tokens, output_tokens|
-          cost = ModelPricing.estimate(model: model, input_tokens: input_tokens, output_tokens: output_tokens)
+        rows.each do |model, provider, agent_class, input_tokens, output_tokens|
+          cost = ModelPricing.estimate(model: model, provider: provider, input_tokens: input_tokens, output_tokens: output_tokens)
           next unless cost
 
           total += cost

data/app/controllers/action_agent/api/projects_controller.rb

@@ -17,6 +17,8 @@ module ActionAgent
       LOG_TAIL_BYTES = 8 * 1024
       LOG_PAGE_BYTES = 64 * 1024
       LOG_MAX_PAGE_BYTES = 1024 * 1024
+      # The most of a backend's error the boot status carries (see #boot_report).
+      BACKEND_ERROR_BYTES = 1_000
       UPDATABLE = %i[name default_ref start_url].freeze
 
       before_action :require_owner!
@@ -214,18 +216,20 @@ module ActionAgent
       # The current sandbox's boot, step by step (see
       # SandboxOrchestrator#boot_status), with the scrubbed tail of the step
       # that failed or is running, and the modes its browser can start in
-      # (none when the backend runs no browsers).
+      # (none when the backend runs no browsers). When the backend cannot
+      # report the boot, `boot` is null and `backend_error` names the backend
+      # and the reason (see #boot_report).
       def boot_status
         sandbox = @project.current_sandbox_session
         orchestrator = SandboxOrchestrator.new
-        status = sandbox && orchestrator.supports?(:boot_status) ? orchestrator.boot_status(sandbox) : nil
-        status = SecretScrubber.scrub(status.deep_symbolize_keys, @project.scrub_values) if status.is_a?(Hash)
+        status, log_tail, backend_error = boot_report(orchestrator, sandbox)
 
         render json: {
           project: @project.summary,
           boot: status,
-          log_tail: status && log_tail(orchestrator, sandbox, status),
+          log_tail: log_tail,
           error: sandbox&.failed? ? SecretScrubber.scrub(sandbox.error_summary, @project.scrub_values) : nil,
+          backend_error: backend_error,
           logs: sandbox.present? && orchestrator.supports?(:boot_log),
           confirmation: @project.local_boot_confirmation(orchestrator),
           browser_modes: browser_modes(orchestrator)
@@ -535,6 +539,27 @@ module ActionAgent
         []
       end
 
+      # The backend's account of the sandbox's boot, scrubbed, with the tail
+      # of the step's log that goes with it: [status, log_tail, nil]. When the
+      # backend cannot give one (no host configured, the host unreachable, a
+      # container that has not written its state yet), [nil, nil, error],
+      # where error names the backend and the scrubbed reason. The page then
+      # shows the boot as unknown for that reason and keeps following it,
+      # instead of failing to load the project (#650). The sandbox's own
+      # failure, if it failed, is reported beside it as `error`.
+      def boot_report(orchestrator, sandbox)
+        return [ nil, nil, nil ] unless sandbox && orchestrator.supports?(:boot_status)
+
+        status = orchestrator.boot_status(sandbox)
+        status = SecretScrubber.scrub(status.deep_symbolize_keys, @project.scrub_values) if status.is_a?(Hash)
+        [ status, status && log_tail(orchestrator, sandbox, status), nil ]
+      rescue StandardError => e
+        Rails.logger.warn("[ActionAgent] the #{orchestrator.backend_name} sandbox backend could not report the boot of " \
+          "sandbox #{sandbox.session_id}: #{e.class}: #{e.message}")
+        [ nil, nil, { backend: orchestrator.backend_name,
+                      message: SecretScrubber.scrub(e.message.to_s, @project.scrub_values).truncate(BACKEND_ERROR_BYTES) } ]
+      end
+
       # The last LOG_TAIL_BYTES of the step that failed, else of the one
       # running, else of the last that ran: { step:, text:, truncated: }.
       def log_tail(orchestrator, sandbox, status)

data/app/controllers/action_agent/api/provider_keys_controller.rb

@@ -94,7 +94,7 @@ module ActionAgent
 
         host = params[:credential].presence || stored&.credential || platform_host(provider)
         if host.blank?
-          return render json: { ok: false, host: nil, models: [], latency_ms: nil, error: "No host configured" }
+          return render json: { ok: false, host: nil, models: [], latency_ms: nil, error: "No host configured", cloud: nil, rates: {} }
         end
 
         render json: OllamaHostProbe.call(host: host, api_key: probe_api_key(stored, host)).to_h

data/app/controllers/action_agent/api/provider_models_controller.rb

@@ -9,10 +9,12 @@ module ActionAgent
     # side, so the UI can't drift stale). Ollama is queried live from the
     # caller's effective host (see ProviderCredentials: their personal key,
     # the host resolver's, the account's, falling back to the platform
-    # config) so locally pulled models appear; OpenRouter is queried from its
-    # public catalog, and Anthropic from its Models API with the caller's
-    # effective key. Live lookups fall back to the curated list on any
-    # failure, and when the caller's credentials cannot be resolved.
+    # config) so locally pulled models appear — or, when the host is Ollama
+    # Cloud, the cloud's models, which the response marks (`cloud`) and
+    # prices (`rates`, per model a price list names); OpenRouter is queried
+    # from its public catalog, and Anthropic from its Models API with the
+    # caller's effective key. Live lookups fall back to the curated list on
+    # any failure, and when the caller's credentials cannot be resolved.
     #
     # When the host app loads RubyLLM, the chat models its registry lists for
     # the provider that take and return text follow: RubyLLM's bundled
@@ -51,7 +53,9 @@ module ActionAgent
           source = "curated"
         end
 
-        render json: { provider: provider, models: (models + registry_models(provider)).uniq, source: source }
+        payload = { provider: provider, models: (models + registry_models(provider)).uniq, source: source }
+        payload.merge!(ollama_detail) if provider == "ollama"
+        render json: payload
       end
 
       private
@@ -101,9 +105,17 @@ module ActionAgent
           return nil
         end
 
+        @ollama_probe = result
         [ result.models, "live" ] if result.models.any?
       end
 
+      # What the live lookup learned about the caller's Ollama host: whether
+      # it is Ollama Cloud (nil when no host answered), and the rate of
+      # every listed model a price list names, in $ per million tokens.
+      def ollama_detail
+        { cloud: @ollama_probe&.cloud, rates: @ollama_probe&.rates || {} }
+      end
+
       # Queries the Anthropic Models API with the caller's effective key
       # (newest first, as returned by the API) so new model releases appear
       # without a deploy. The platform's own key is never used here.

data/app/controllers/action_agent/api/sandboxes_controller.rb

@@ -211,18 +211,19 @@ module ActionAgent
       # GET /api/sandboxes/:session_id/boot
       # How a checkout's boot went, step by step (see
       # SandboxOrchestrator#boot_status). `boot` is null when the backend
-      # reports no steps, or holds nothing for the sandbox.
+      # reports no steps, or holds nothing for the sandbox; when the backend
+      # cannot report it at all, `backend_error` names the backend and the
+      # reason (see #boot_report).
       def boot
         orchestrator = SandboxOrchestrator.new
-        status = @sandbox.app_runtime? && orchestrator.supports?(:boot_status) ? orchestrator.boot_status(@sandbox) : nil
-        # A project's checkout booted with its secrets, which the backend may
-        # not know to mask.
-        status = SecretScrubber.scrub(status, @sandbox.project_scrub_values) if status
-        render json: {
+        status, backend_error = boot_report(orchestrator)
+        payload = {
           boot: status,
           resumable: !!(status&.dig(:kept) && @sandbox.failed? && !@sandbox.past_expiry? && orchestrator.supports?(:resume_boot)),
           logs: @sandbox.app_runtime? && orchestrator.supports?(:boot_log)
         }
+        payload[:backend_error] = backend_error if backend_error
+        render json: payload
       end
 
       # GET /api/sandboxes/:session_id/boot_log?step=NAME&offset=N&limit=N
@@ -298,6 +299,30 @@ module ActionAgent
 
       private
 
+      # The most of a backend's error the boot status carries (see #boot_report).
+      BACKEND_ERROR_BYTES = 1_000
+
+      # The backend's account of the sandbox's boot, scrubbed of its
+      # project's secrets (a project's checkout booted with them, which the
+      # backend may not know to mask): [status, nil]. When the backend cannot
+      # give one (no host configured, the host unreachable, a container that
+      # has not written its state yet), [nil, error], where error names the
+      # backend and the scrubbed reason, so the page reports it instead of
+      # failing on a 500 (#650; Api::ProjectsController#boot_report is the
+      # same for a project).
+      def boot_report(orchestrator)
+        return [ nil, nil ] unless @sandbox.app_runtime? && orchestrator.supports?(:boot_status)
+
+        status = orchestrator.boot_status(@sandbox)
+        status = SecretScrubber.scrub(status, @sandbox.project_scrub_values) if status
+        [ status, nil ]
+      rescue StandardError => e
+        Rails.logger.warn("[ActionAgent] the #{orchestrator.backend_name} sandbox backend could not report the boot of " \
+          "sandbox #{@sandbox.session_id}: #{e.class}: #{e.message}")
+        [ nil, { backend: orchestrator.backend_name,
+                 message: SecretScrubber.scrub(e.message.to_s, @sandbox.project_scrub_values).truncate(BACKEND_ERROR_BYTES) } ]
+      end
+
       def set_sandbox
         @sandbox = account_scoped(owned(SandboxSession)).find_by!(session_id: params[:id])
       end

data/app/controllers/action_agent/traces_controller.rb

@@ -4,7 +4,7 @@ module ActionAgent
   # Controller for viewing telemetry traces.
   #
   # Provides:
-  # - List view with filtering and pagination
+  # - Bounded list of recent traces with filtering (no pagination)
   # - Detail view with span timeline
   # - Metrics overview
   # - Live updates via Turbo Streams
@@ -76,7 +76,8 @@ module ActionAgent
         )
       end
 
-      traces.limit(params[:limit] || 50)
+      limit = Integer(params[:limit], 10, exception: false) || 50
+      traces.limit(limit.clamp(1, 1000))
     end
 
     def calculate_metrics

data/app/controllers/concerns/action_agent/api/mcp_dashboard_tools.rb

@@ -842,7 +842,7 @@ module ActionAgent
           change = comparison_change(before, after)
           next if change == "unchanged_pass"
 
-          { scenario_key: key[0], model: key[1], change: change, before: before&.status, after: after&.status,
+          { scenario_key: key[0], model: key[1], trial: key[2], change: change, before: before&.status, after: after&.status,
             fault: after&.fault, result_id: after&.id }
         end
 
@@ -1012,8 +1012,10 @@ module ActionAgent
       end
 
       # The run's results keyed by [scenario key, model].
+      # Results by scenario key, model and trial: a run that replayed each
+      # scenario several times compares trial by trial.
       def comparison_rows(run)
-        run.scenario_results.includes(:scenario).index_by { |result| [ result.evaluated_scenario["key"], result.model ] }
+        run.scenario_results.includes(:scenario).index_by { |result| [ result.evaluated_scenario["key"], result.model, result.trial ] }
       end
 
       # One of: removed, added_pass, added_fail, unchanged_pass, regressed,

data/app/jobs/action_agent/agent_execution_job.rb

@@ -6,13 +6,12 @@ module ActionAgent
 
     def perform(run_id)
       run = AgentRun.find(run_id)
-      # Anything already finished — complete, failed or cancelled — is never
-      # executed again, and neither is a run waiting for input, which only
-      # AgentResumeJob continues. A retry of a failed run would re-run the
-      # generation.
-      return if run.finished? || run.awaiting_input?
+      # Only a pending run is executed. A finished run (complete, failed or
+      # cancelled) is never executed again, a run waiting for input only
+      # AgentResumeJob continues, and a running run already has a worker. A
+      # retry of a failed run would re-run the generation.
+      return unless claim_pending(run)
 
-      run.update!(status: :running, started_at: Time.current)
       run.add_log("Starting execution", level: :info)
 
       begin
@@ -38,6 +37,27 @@ module ActionAgent
 
     private
 
+    # Moves the run from pending to running in one conditional update, as
+    # CodeSessionJob#start claims a session, and returns whether this job is
+    # the one that did. The write checks the status itself, so of two
+    # workers that loaded the run as pending only the first to write
+    # executes it, and the other starts no generation, on every database. A
+    # read under the row lock followed by a write would be refused on SQLite
+    # with Rails 7.2, whose deferred transactions cannot write once another
+    # writer has committed. Nothing is held over the generation, so a cancel
+    # never waits for it. The update skips callbacks, so the status change is
+    # broadcast here.
+    def claim_pending(run)
+      now = Time.current
+      claimed = AgentRun.where(id: run.id, status: AgentRun.statuses[:pending])
+        .update_all(status: AgentRun.statuses[:running], started_at: now, updated_at: now)
+      return false if claimed.zero?
+
+      run.reload
+      run.broadcast_update
+      true
+    end
+
     def execute_agent(agent_record, run)
       AgentExecutionService.call(agent_record, run)
     end

data/app/jobs/action_agent/evaluation_judge_job.rb

@@ -17,11 +17,12 @@ module ActionAgent
       return finish_without_result(run_id, selection) unless row
 
       run = row.evaluation_run
+      runner = ScenarioEvaluationRunner.new(run.evaluation, selection: selection, run: run)
       begin
-        ScenarioEvaluationRunner.new(run.evaluation, selection: selection, run: run).judge_result!(row)
+        runner.judge_result!(row)
       rescue StandardError => e
         Rails.logger.warn("[ActionAgent] evaluation run #{run.id}: scoring result #{result_id} failed: #{e.class}: #{e.message}")
-        ScenarioEvaluationRunner.new(run.evaluation, selection: selection, run: run).fail_judging!(row.reload, e)
+        runner.fail_judging!(row.reload, e)
       end
     end
 

data/app/models/action_agent/catalog_scenario.rb

@@ -27,9 +27,15 @@ module ActionAgent
       value.is_a?(Hash) ? value : {}
     end
 
+    def metadata
+      value = self[:metadata]
+      value.is_a?(Hash) ? value : {}
+    end
+
     # @return [ActiveAgent::Evals::Scenario]
     def to_scenario
-      ActiveAgent::Evals::Scenario.from_hash(to_document.merge("position" => position), group: set.key, group_name: set.name)
+      ActiveAgent::Evals::Scenario.from_hash(to_document.merge("position" => position), group: set.key, group_name: set.name,
+        metadata: metadata)
     end
 
     def to_document
@@ -41,7 +47,7 @@ module ActionAgent
         "tags" => tags.presence,
         "params" => params.presence,
         "production_only" => (true if production_only?)
-      }.compact
+      }.compact.merge(metadata.deep_stringify_keys.except(*ActiveAgent::Evals::Catalog::SCENARIO_KEYS))
     end
 
     # The attributes Evaluation#replace_scenarios! takes.

data/app/models/action_agent/evaluation_run.rb

@@ -255,17 +255,19 @@ module ActionAgent
     # for one the engine judged.
     #
     # The engine runs against every activeagent since 1.4, so the report
-    # kwargs that arrived later are passed only when Report.new takes them.
+    # kwargs that arrived later are passed only when Report.new takes them,
+    # and a result's trial only when Result carries one.
     def to_report(links: report_links, estimate: true)
       # A replay not yet scored (a fanned-out run in progress) is not a
       # result the report can grade, so it is left out rather than read as
       # a failure.
       rows = scenario_results.where.not(status: :pending).includes(:scenario).sort_by do |row|
-        [ row.evaluated_scenario["position"].to_i, row.evaluation_scenario_id, row.model ]
+        [ row.evaluated_scenario["position"].to_i, row.evaluation_scenario_id, row.model, row.trial, row.id ]
       end
       breakdown = cost_breakdown if estimate
       selected = selected_specs
       specs = {}
+      trials = self.class.result_accepts?(:trial)
       results = rows.map do |row|
         spec = specs[[ row.provider.to_s, row.model ]] ||= selected[[ row.provider.to_s, row.model ]] || ModelSpec.new(
           label: [ row.provider.presence, row.model ].compact.join("/"), provider: row.provider.to_s, model: row.model
@@ -280,7 +282,8 @@ module ActionAgent
             metadata: replay_metadata_for(row, breakdown)
           ),
           scores: row.scores.to_h, score: row.score, status: row.status,
-          diagnosis: row.evaluation_diagnosis.presence
+          diagnosis: row.evaluation_diagnosis.presence,
+          **(trials ? { trial: row.trial } : {})
         )
       end
 
@@ -317,6 +320,11 @@ module ActionAgent
       report_class.instance_method(:initialize).parameters.any? { |type, name| name == keyword && %i[key keyreq].include?(type) }
     end
 
+    # Whether the installed framework's Result carries +member+.
+    def self.result_accepts?(member)
+      ActiveAgent::Evals::Result.members.include?(member)
+    end
+
     # The release this run scored, as the report names it — `{ "digest",
     # "revision", "label" }` — or nil for a run pinned to a dashboard edit
     # or to no version.

data/app/models/action_agent/evaluation_scenario_result.rb

@@ -45,10 +45,19 @@ module ActionAgent
     end
 
     # The diagnosis as the dashboard shows it: without the storage keys —
-    # the replay metadata, the scenario snapshot and the judge usage the
-    # publishing application reported (served as `judge_usage` instead).
+    # the replay metadata, the scenario snapshot, the judge usage the
+    # publishing application reported (served as `judge_usage` instead) and
+    # the trial number.
     def evaluation_diagnosis
-      (diagnosis || {}).except("_replay_metadata", "_scenario_snapshot", "_judge_usage")
+      (diagnosis || {}).except("_replay_metadata", "_scenario_snapshot", "_judge_usage", "_trial")
+    end
+
+    # Which replay of the scenario under the model this is, from 1: a
+    # published run keeps every replay of a scenario the application ran
+    # more than once (EvaluationReportImport). A result with no record is
+    # the first.
+    def trial
+      [ diagnosis&.dig("_trial").to_i, 1 ].max
     end
 
     # A catalog can be refreshed without changing what an earlier run asked