actionagent 1.9.1 → 1.9.3

data/app/controllers/action_agent/api/agents_controller.rb

@@ -292,6 +292,29 @@ module ActionAgent
         ).as_json
       end
 
+      # GET /api/agents/tool_roster
+      #
+      # The roster an agent not created yet can be given — what the builder's
+      # Configure step offers. The same rows as #tool_roster for a saved
+      # agent with nothing enabled: every tool the toolbox or the host's
+      # schema tools implement, and every MCP service the catalog, the host's
+      # registrations and the caller's live checkout runtimes describe, each
+      # saying whether a run can call it. Usage reads the workspace instead of
+      # an agent's history: which services its agents already enable.
+      def new_tool_roster
+        agent = owner_agents.build
+        runtimes = SandboxSession.runtime_server_listings(
+          SandboxSession.for_owner(agent.owner).where(id: owned(SandboxSession).select(:id))
+        )
+
+        render json: AgentToolRoster.new(
+          agent: agent,
+          traces: owned_traces,
+          runtimes: runtimes,
+          workspace_agents: owner_agents
+        ).as_json
+      end
+
       # GET /api/agents/:id/recorded_models
       #
       # The model names this agent's generations were recorded under, most

data/app/controllers/action_agent/api/code_sessions_controller.rb

@@ -4,7 +4,8 @@ module ActionAgent
   module Api
     # Headless Claude Code sessions inside one of the caller's checkout
     # sandboxes (Settings -> Integrations): start one with a prompt, poll its
-    # transcript, cancel it.
+    # transcript, cancel it, and carry its conversation on with more
+    # messages (#messages, CodeConversation).
     #
     # A session runs on the owner's Anthropic API key (or, with
     # ActionAgent.claude_code_auth = :local_login, this machine's own Claude
@@ -18,9 +19,9 @@ module ActionAgent
       VERIFICATION_HOLD = 1.hour
 
       before_action :require_owner!
-      before_action :require_execution_enabled!, only: [ :create ]
+      before_action :require_execution_enabled!, only: [ :create, :messages ]
       before_action :set_sandbox
-      before_action :set_code_session, only: [ :show, :cancel ]
+      before_action :set_code_session, only: [ :show, :cancel, :messages ]
 
       # A model name as Claude Code's --model takes it ("sonnet",
       # "claude-sonnet-4-5", "claude-sonnet-4-5[1m]"). It becomes an argument
@@ -48,14 +49,17 @@ module ActionAgent
 
         code_session = CodeSession.new(
           sandbox_session: @sandbox,
-          prompt: string_param(:prompt),
+          prompt: CodeSession.strip_note_tags(string_param(:prompt)).presence,
           model: string_param(:model).presence,
           runner: runner,
+          # A conversation started to chat (the project's Sessions tab, or
+          # Interactions) rather than from Settings.
+          origin: params[:origin] == "chat" ? "chat" : "settings",
           # Subscription use always belongs to the person who clicked Run.
           user_id: current_user.respond_to?(:id) ? current_user.id : @sandbox.try(:user_id),
           account_id: @sandbox.try(:account_id)
         )
-        code_session.credential_mode = runner == "claude_code" ? ClaudeCodeAuth.credential_mode(@sandbox, user_id: current_user&.id) : "api_key"
+        code_session.credential_mode = runner == "claude_code" ? ClaudeCodeAuth.credential_mode(@sandbox, user_id: current_user&.id) : CodexAuth.mode
         attach_fix!(code_session) if params[:evaluation_run_id].present?
         if (error = invalid_request(code_session))
           return render json: { error: error }, status: :unprocessable_entity
@@ -110,6 +114,43 @@ module ActionAgent
         render json: { code_session: @code_session.summary }
       end
 
+      # POST /api/sandboxes/:sandbox_id/code_sessions/:id/messages
+      #   { prompt, model?, handoff?: { by, taken_at, released_at, page_url } }
+      # Adds a turn to the conversation +:id+ belongs to. 201 when it starts
+      # now, 202 when it waits behind the turn that runs.
+      def messages
+        root = @code_session.root
+        return unless authorize_action!(:chat_code_session, root)
+
+        if (refusal = refusal_for(@sandbox, runner: root.runner))
+          return render json: { error: refusal }, status: :unprocessable_entity
+        end
+        if root.credential_mode == "sandbox_login" && @sandbox.claude_login_user_id != current_user&.id
+          return render json: { error: "Only the person signed in to Claude Code in this sandbox can continue its conversations" },
+            status: :forbidden
+        end
+        prompt = CodeSession.strip_note_tags(string_param(:prompt)).strip
+        return render json: { error: "Write a message" }, status: :unprocessable_entity if prompt.empty?
+        model = string_param(:model).presence
+        return render json: { error: "model is not a Claude Code model name" }, status: :unprocessable_entity if model && !model.match?(MODEL_NAME)
+
+        enforce_execution_quota!
+        return if performed?
+
+        handoff = params[:handoff].respond_to?(:permit) ? params[:handoff].permit(:by, :taken_at, :released_at, :page_url).to_h : nil
+        mode = root.runner == "claude_code" ? ClaudeCodeAuth.credential_mode(@sandbox, user_id: current_user&.id) : nil
+        turn = CodeConversation.post!(root, prompt: prompt, user_id: current_user.respond_to?(:id) ? current_user.id : nil,
+          credential_mode: mode, handoff: handoff, model: model)
+        record_execution_usage
+
+        behind = root.reload.turns.any? { |other| other.id != turn.id && (other.running? || other.queued?) && other.id < turn.id }
+        render json: { code_session: turn.reload.summary, waiting: behind }, status: behind ? :accepted : :created
+      rescue CodeConversation::Refused => e
+        render json: { error: e.message }, status: :conflict
+      rescue ActiveRecord::RecordInvalid => e
+        render json: { error: e.record.errors.full_messages.to_sentence }, status: :unprocessable_entity
+      end
+
       private
 
       # A checkout runs on its account's GitHub token and Claude Code
@@ -141,10 +182,8 @@ module ActionAgent
         if (refusal = ClaudeCodeAuth.login_owner_refusal(sandbox, user_id: current_user&.id))
           return refusal
         end
-        if runner == "codex"
-          return "Codex is not connected: connect an OpenAI API key in Settings -> Integrations first" if sandbox.runtime_environment(runner: runner).blank?
-          return
-        end
+        return CodexAuth.backend_refusal(orchestrator) || CodexAuth.credential_refusal(sandbox) if runner == "codex"
+
         ClaudeCodeAuth.backend_refusal(orchestrator) || ClaudeCodeAuth.credential_refusal(sandbox, user_id: current_user&.id)
       end
 
@@ -185,7 +224,7 @@ module ActionAgent
           raise EvaluationFix::Invalid, "The previous session must implement this same fix"
         end
         session.assign_attributes(evaluation_run: run, fix_item: fix.item, previous_code_session: previous,
-          prompt: fix.prompt(previous: previous))
+          prompt: fix.prompt(previous: previous), origin: "evaluation_fix")
       end
 
       rescue_from EvaluationFix::Invalid do |error|

data/app/controllers/action_agent/api/evaluation_reports_controller.rb

@@ -25,15 +25,28 @@ module ActionAgent
     #   401 — a missing or unknown key
     #   501 — an install with no evaluation tables, or not yet migrated
     #   503 — another import for the same agent held the lock too long
+    #
+    # GET <mount>/api/evaluation_reports/:run_id returns a stored run as the
+    # same version-1 envelope (EvaluationReportExport), so it can be posted to
+    # another install, or back to this one as an identical retry. Responds
+    # 200 with the envelope (or, with `format=html` or `format=md`, the
+    # report page or its Markdown), 404 for a run_id the key's tenant holds
+    # no run under, 422 for a run that is not complete or an unknown format,
+    # 401 as above and also on a single-tenant install that has no
+    # ingest_api_key, and 501 as above.
     class EvaluationReportsController < ActionController::API
       include IngestAuthentication
 
+      # What #show serves for each `format`.
+      EXPORT_FORMATS = { "json" => "application/json", "html" => "text/html", "md" => "text/markdown" }.freeze
+
       # New reports one key may store per minute.
       RATE_LIMIT = 30
 
       wrap_parameters false
 
-      before_action :require_json!
+      before_action :require_json!, only: :create
+      before_action :require_export_key!, only: :show
       before_action :require_report_store!
 
       # POST <mount>/api/evaluation_reports
@@ -64,6 +77,41 @@ module ActionAgent
         render json: { error: e.message }, status: :service_unavailable
       end
 
+      # GET <mount>/api/evaluation_reports/:run_id
+      #
+      # :run_id is an imported run's run_id as it was published, or the
+      # "dashboard-run-<id>-<stamp>" a run the dashboard executed exports
+      # under (EvaluationReportExport.run_id_for). Found within the key's
+      # tenant exactly as #create finds a stored run. `format=html` returns
+      # the run's self-contained report page (text/html) as run_report renders
+      # it, and `format=md` its Markdown (text/markdown), instead of the
+      # envelope; any other format is a 422. `download=1` sends any of them
+      # as an attachment named like the dashboard's downloads.
+      def show
+        run = EvaluationReportExport.find(params[:run_id], account: @account)
+        return render json: { error: "No evaluation report with that run_id" }, status: :not_found unless run
+
+        format = params[:format].presence || "json"
+        unless EXPORT_FORMATS.key?(format)
+          return render json: { error: "format is one of #{EXPORT_FORMATS.keys.join(', ')}" }, status: :unprocessable_entity
+        end
+
+        export = EvaluationReportExport.new(run)
+        export.exportable!
+        body =
+          case format
+          when "json" then export.to_json
+          when "md" then export.markdown
+          # The page run_report serves, its fix actions linking into this
+          # dashboard by absolute path; no theme, so it follows the viewer.
+          when "html" then run.to_report(links: run.report_links(mount: request.script_name)).to_html
+          end
+        send_data body, type: EXPORT_FORMATS.fetch(format), disposition: download? ? "attachment" : "inline",
+                        filename: export.filename(format)
+      rescue EvaluationReportExport::NotExportable => e
+        render json: { error: e.message }, status: :unprocessable_entity
+      end
+
       private
 
       # Rails reads and parses a JSON body into params before any callback
@@ -74,6 +122,18 @@ module ActionAgent
         super
       end
 
+      # A single-tenant install with no ingest key takes reports from anyone
+      # who reaches the mount, which a host keeps off the network. Reading a
+      # run back is different: every stored run, prompts and answers
+      # included, would be open under a guessable run_id. So the export
+      # needs the key to exist; a multi-tenant install authenticates the
+      # tenant's key already.
+      def require_export_key!
+        return if ActionAgent.multi_tenant? || ActionAgent.ingest_api_key.present?
+
+        render json: { error: "Set ActionAgent.ingest_api_key to export evaluation reports" }, status: :unauthorized
+      end
+
       # A cross-site page can send a text/plain or form POST without a CORS
       # preflight; it cannot send application/json.
       def require_json!
@@ -82,6 +142,10 @@ module ActionAgent
         render json: { error: "Content-Type must be application/json" }, status: :unsupported_media_type
       end
 
+      def download?
+        ActiveModel::Type::Boolean.new.cast(params[:download]) == true
+      end
+
       def require_report_store!
         reason = EvaluationReportImport.unavailable_reason
         render json: { error: reason }, status: :not_implemented if reason
@@ -110,7 +174,7 @@ module ActionAgent
         {
           id: run.id,
           evaluation_id: run.evaluation_id,
-          run_id: run.external_run_id,
+          run_id: EvaluationReportExport.run_id_for(run),
           status: run.status,
           duplicate: duplicate,
           url: run_url(run)

data/app/controllers/action_agent/api/evaluations_controller.rb

@@ -48,7 +48,7 @@ module ActionAgent
       #
       # Three fields feed the dashboard's model pickers. They describe the
       # credentials of #picker_credentials_owner:
-      #   - judge_provider:        the provider a judge model runs on, null
+      #   - judge_provider:        the default provider of an unpinned judge, null
       #                            when none has credentials or the lookup
       #                            raised
       #   - judge_provider_error:  true when that lookup raised, as it does
@@ -82,18 +82,30 @@ module ActionAgent
       # `evaluation: { archived: true | false }` archives the evaluation —
       # it keeps its runs but leaves the index, the pooled pass rate and the
       # agent's scorecard — or brings it back. A new run brings it back too.
+      # `evaluation: { judge_model: "…" }` pins the model the judge runs on
+      # from the next run; blank unpins it, back to the provider's default.
       def update
         evaluation = evaluations_scope.find(params[:id])
-        archived = params.require(:evaluation)[:archived]
-        if archived.nil?
+        attributes = params.require(:evaluation)
+        archived = attributes[:archived]
+        pin_judge = attributes.key?(:judge_model)
+        if archived.nil? && !pin_judge
           return render json: { errors: [ "evaluation.archived must be true or false" ] }, status: :unprocessable_entity
         end
 
-        ActiveModel::Type::Boolean.new.cast(archived) ? evaluation.archive! : evaluation.unarchive!
+        judge_model = attributes[:judge_model].to_s.strip.presence
+        if judge_model && judge_model.length > MAX_JUDGE_MODEL
+          return render json: { errors: [ "evaluation.judge_model is longer than #{MAX_JUDGE_MODEL} characters" ] }, status: :unprocessable_entity
+        end
+
+        evaluation.update!(judge_model: judge_model) if pin_judge
+        ActiveModel::Type::Boolean.new.cast(archived) ? evaluation.archive! : evaluation.unarchive! unless archived.nil?
 
         render json: { evaluation: serialize(evaluation.reload) }
       end
 
+      MAX_JUDGE_MODEL = 200
+
       # Runs listed per evaluation on GET /api/evaluations/:id. The rest of
       # the history stays reachable by run id; `run_count` says how long it is.
       RUN_HISTORY_LIMIT = 20
@@ -243,14 +255,44 @@ module ActionAgent
       # the page follows the viewer's own preference. The page is served
       # outside the React app, so its fix item actions link at the absolute
       # mount path (request.script_name) rather than relative to it.
+      #
+      # `report.json` (or `?format=json`) returns the run instead as the
+      # version-1 envelope the evaluation report collector takes
+      # (EvaluationReportExport), the same the API-key export at
+      # GET /api/evaluation_reports/:run_id serves; 422 for a run that is not
+      # complete. `report.md` (or `?format=md`) returns it as GitHub-flavored
+      # Markdown, the whole report or one `part` — `verdict`, `fix_items`
+      # (narrowed by `model`, a model label) or `scenario` (named by
+      # `scenario`, its key; 404 for one the run did not replay) — which the
+      # dashboard's "Copy as Markdown" buttons read. `download=1` sends any of
+      # them as an attachment named for the evaluation and the run's number:
+      # `orders-eu-run-3.html`, `orders-eu-run-3-fix-items.md`.
       def run_report
         evaluation = evaluations_scope.find(params[:id])
         run = evaluation.evaluation_runs.find(params[:run_id])
         raise ActiveRecord::RecordNotFound unless evaluation.scenario_suite?
 
+        export = EvaluationReportExport.new(run)
+        if request.format.symbol == :md
+          body = export.markdown(part: params[:part], model: params[:model], scenario: params[:scenario])
+          return send_report(body, export.filename("md", part: params[:part]), "text/markdown") if download?
+
+          return render plain: body, content_type: "text/markdown"
+        end
+        if request.format.json?
+          body = export.to_json
+          return send_report(body, export.filename("json"), "application/json") if download?
+
+          return render json: body
+        end
+
         report = run.to_report(links: run.report_links(mount: request.script_name))
+        html = report.to_html(theme: params[:theme])
+        return send_report(html, export.filename("html"), "text/html") if download?
 
-        render html: report.to_html(theme: params[:theme]).html_safe, layout: false
+        render html: html.html_safe, layout: false
+      rescue EvaluationReportExport::NotExportable, EvaluationReportExport::UnknownPart => e
+        render json: { errors: [ e.message ] }, status: :unprocessable_entity
       end
 
       # DELETE /api/evaluations/:id
@@ -261,6 +303,14 @@ module ActionAgent
 
       private
 
+      def download?
+        ActiveModel::Type::Boolean.new.cast(params[:download]) == true
+      end
+
+      def send_report(body, filename, type)
+        send_data body, filename: filename, type: type, disposition: "attachment"
+      end
+
       def include_archived?
         ActiveModel::Type::Boolean.new.cast(params[:archived]) == true
       end
@@ -411,7 +461,7 @@ module ActionAgent
 
       def run_number(evaluation, run) = EvaluationSerializer.run_number(evaluation, run)
 
-      def safe_fix_items(run) = EvaluationSerializer.fix_items(run)
+      def safe_fix_items(run) = EvaluationSerializer.fix_items(run, markdown: true)
     end
   end
 end

data/app/controllers/action_agent/api/interactions_controller.rb

@@ -8,15 +8,32 @@ module ActionAgent
     # A context is one agent's interaction stream (AgentContext +
     # AgentMessages + AgentGenerations, recorded by SolidAgent::HasContext
     # during execution), scoped to the current user's agents.
+    #
+    # Claude Code and Codex conversations in the caller's checkout sandboxes
+    # are interactions too ("code-<id>", CodeConversationSerializer).
+    #
+    # GET /api/interactions takes:
+    #   source      all (the default), agent, claude_code or codex
+    #   project_id  only that project's code conversations
+    #   sandbox_id  only that sandbox's (its session id)
+    #   agent_id    only that agent's contexts and traces; no code conversations
     class InteractionsController < BaseController
       before_action :require_owner!
 
       DEFAULT_LIMIT = 50
       MAX_WINDOW_MINUTES = 60 * 24 * 90
+      SOURCES = %w[all agent claude_code codex].freeze
 
       # GET /api/interactions
       def index
         limit = clamped_param(:limit, default: DEFAULT_LIMIT, min: 1, max: 200)
+        source = SOURCES.include?(params[:source]) ? params[:source] : "all"
+        coded = source == "all" ? code_conversations(limit) : (source == "agent" ? [] : code_conversations(limit, runner: source))
+        # A project or sandbox filter, or a code source, names code
+        # conversations only.
+        if source != "all" && source != "agent" || params[:project_id].present? || params[:sandbox_id].present?
+          return render json: { interactions: coded.first(limit) }
+        end
 
         contexts = interactions_scope
           .includes(:contextable)
@@ -53,12 +70,20 @@ module ActionAgent
         reported = reported_traces(limit).map { |trace| TraceInteractionSerializer.summary(trace) }
 
         render json: {
-          interactions: (persisted + reported).sort_by { |row| row[:last_activity_at].to_s }.reverse.first(limit)
+          interactions: (persisted + reported + coded).sort_by { |row| row[:last_activity_at].to_s }.reverse.first(limit)
         }
       end
 
       # GET /api/interactions/:id
+      # For "code-<id>", ?after[<turn id>]=N sends each turn's transcript from
+      # event N onward.
       def show
+        if (root_id = params[:id].to_s[/\Acode-(\d+)\z/, 1])
+          root = code_session_scope.conversation_roots.find(root_id)
+          after = params[:after].respond_to?(:permit) ? params[:after].permit!.to_h : {}
+          return render json: { interaction: CodeConversationSerializer.detail(root, after: after) }
+        end
+
         if (trace_id = params[:id].to_s[/\Atrace-(\d+)\z/, 1])
           trace = owned_traces.find(trace_id)
           return render json: { interaction: TraceInteractionSerializer.detail(trace) }
@@ -78,6 +103,35 @@ module ActionAgent
 
       private
 
+      # Code sessions in the caller's checkout sandboxes, of their current
+      # account in a multi-tenant install, as CodeSessionsController reaches
+      # them. Per-agent views list none.
+      def code_session_scope
+        sandboxes = owned(SandboxSession)
+        sandboxes = sandboxes.where(account_id: current_account.id) if current_account
+        CodeSession.where(sandbox_session_id: sandboxes.select(:id))
+      end
+
+      # The newest code conversations, as interaction rows.
+      def code_conversations(limit, runner: nil)
+        return [] if params[:agent_id].present?
+
+        roots = code_session_scope.conversation_roots
+        roots = roots.where(runner: runner) if runner
+        if params[:sandbox_id].present?
+          roots = roots.where(sandbox_session_id: SandboxSession.where(session_id: params[:sandbox_id].to_s).select(:id))
+        end
+        if params[:project_id].present?
+          roots = roots.where(sandbox_session_id: SandboxSession.where(project_id: params[:project_id].to_i).select(:id))
+        end
+        roots = roots.where(updated_at: window_minutes.minutes.ago..) if window_minutes
+        roots = roots.includes(:sandbox_session).order(created_at: :desc).limit(limit).to_a
+        follow_ups = CodeSession.where(conversation_root_id: roots.map(&:id)).order(:turn_index, :id).group_by(&:conversation_root_id)
+
+        roots.map { |root| CodeConversationSerializer.summary(root, [ root, *follow_ups.fetch(root.id, []) ]) }
+          .sort_by { |row| row[:last_activity_at].to_s }.reverse
+      end
+
       # What each stream opened with and what it finally answered, so a
       # collapsed interaction row says what it was about — the same two lines a
       # collapsed trace row shows. Two grouped queries rather than loading

data/app/controllers/action_agent/api/sandboxes_controller.rb

@@ -110,7 +110,7 @@ module ActionAgent
           code_sessions_supported: code_sessions_supported?,
           codex_sessions_supported: codex_sessions_supported?,
           pull_requests_supported: pull_requests_supported?,
-          codex_connected: owned(ProviderKey).where(provider: "codex").any? { |key| key.runtime_environment.present? },
+          **codex_status,
           browser_modes: browser_modes,
           **claude_code_status
         }
@@ -359,12 +359,23 @@ module ActionAgent
         []
       end
 
+      # Whether the configured backend runs Codex sessions, and runs them
+      # with the configured authentication (CodexAuth.backend_refusal).
       def codex_sessions_supported?
-        SandboxOrchestrator.new.supports_code_runner?("codex")
+        orchestrator = SandboxOrchestrator.new
+        orchestrator.supports_code_runner?("codex") && CodexAuth.backend_refusal(orchestrator).nil?
       rescue StandardError, LoadError
         false
       end
 
+      # How Codex authenticates and whether the caller's can run: the mode,
+      # a boolean, and in :local_login mode this machine's login status.
+      # Never a credential.
+      def codex_status
+        status = CodexAuth.status(owned(ProviderKey))
+        { codex_auth: status[:mode], codex_connected: status[:connected], codex_login: status[:login] }.compact
+      end
+
       # Whether the configured backend can read a checkout's changes, which
       # opening a pull request and downloading a patch both need.
       def pull_requests_supported?

data/app/controllers/action_agent/api/trace_reports_controller.rb

@@ -27,16 +27,19 @@ module ActionAgent
       #   service   a service_name, narrowing `traces` only
       #   status    "error" for failed traces only
       def index
-        window = params.fetch(:minutes, DEFAULT_WINDOW_MINUTES).to_i.clamp(1, MAX_WINDOW_MINUTES)
-        window_scope = agent_scope(traces_scope).for_date_range(window.minutes.ago, Time.current)
+        window = clamped_param(:minutes, default: DEFAULT_WINDOW_MINUTES, min: 1, max: MAX_WINDOW_MINUTES)
+        to = Time.current
+        from = to - window.minutes
+        window_scope = agent_scope(traces_scope).where(timestamp: from...to)
 
         scope = window_scope
         scope = scope.for_agent(params[:agent]) if params[:agent].present?
         scope = scope.for_service(params[:service]) if params[:service].present?
         scope = scope.with_errors if params[:status] == "error"
 
-        limit = params.fetch(:limit, DEFAULT_LIMIT).to_i.clamp(1, 1000)
-        traces = scope.recent.limit(limit)
+        limit = clamped_param(:limit, default: DEFAULT_LIMIT, min: 1, max: 1000)
+        traces = scope.recent.limit(limit).to_a
+        report = TraceWindowReport.new(scope, from: from, to: to, bucket_seconds: integer_param(:bucket_seconds, default: 60)).as_json
 
         render json: {
           traces: traces.map { |trace| TelemetryTraceSerializer.summary(trace) },
@@ -45,8 +48,10 @@ module ActionAgent
           # trace back to the agent it belongs to (AgentRegistrar attributes
           # SDK-reported traces to an Agent record on ingest).
           agent_ids: agent_ids_by_class(window_scope),
-          window_minutes: window
-        }
+          window_minutes: window,
+          returned_count: traces.size,
+          truncated: report[:total_count] > traces.size
+        }.merge(report)
       end
 
       # GET /api/traces/:id — accepts the record id or the OTel trace_id

data/app/controllers/concerns/action_agent/api/mcp_dashboard_tools.rb

@@ -270,7 +270,7 @@ module ActionAgent
             properties: {
               evaluation_id: { type: "integer", description: "The evaluation's id" },
               run_id: { type: "integer", description: "The run's id; the latest run when omitted" },
-              failed_only: { type: "boolean", description: "Only results that did not pass" },
+              failed_only: { type: "boolean", description: "Only results that failed or errored (a running run's pending results are neither)" },
               limit: { type: "integer", description: "At most this many results (default #{RESULT_LIMIT}, max #{MAX_RESULT_LIMIT})" }
             },
             required: [ "evaluation_id" ]
@@ -737,7 +737,7 @@ module ActionAgent
 
         results = run.scenario_results.includes(:scenario, :agent_run).joins(:scenario)
           .order(EvaluationScenario.arel_table[:position], EvaluationScenario.arel_table[:id], :model)
-        results = results.where.not(status: :passed) if boolean_argument(:failed_only)
+        results = results.where(status: %i[failed errored]) if boolean_argument(:failed_only)
         total = results.count
         page = results.limit(limit).to_a
         recorded = recorded_trace_ids(page.filter_map { |result| result.agent_run&.trace_id })

data/app/jobs/action_agent/code_session_job.rb

@@ -60,15 +60,30 @@ module ActionAgent
       finish(code_session, outcome.to_h, result_event, secrets)
       verify(code_session)
       resume_boot(code_session)
+      advance(code_session)
     rescue StandardError => e
       message = SecretScrubber.scrub(e.message.to_s, secrets || safe_secrets(code_session))
       Rails.logger.error("Code session #{code_session_id} failed: #{message}")
       fail!(code_session, message) if code_session
       resume_boot(code_session) if code_session
+      advance(code_session) if code_session
     end
 
+    # What a turn after the first says when the CLI no longer has the
+    # session it resumes.
+    RESUME_LOST = "The conversation could not be resumed: Claude Code no longer has its session " \
+                  "(the sandbox restarted, or its Claude Code login changed). Start a new conversation."
+
     private
 
+    # Starts the next queued turn of the conversation, now this one settled
+    # (CodeConversation).
+    def advance(code_session)
+      CodeConversation.advance!(code_session.reload)
+    rescue StandardError => e
+      Rails.logger.warn("Code session #{code_session.id}: the conversation's next turn was not started: #{e.class}: #{e.message}")
+    end
+
     # A project's boot fix resumes the boot it fixed (ProjectBootFix). Its
     # failure is the boot's, not the session's, so it is logged and left on
     # the project rather than raised.
@@ -112,7 +127,8 @@ module ActionAgent
         else
           code_session.assign_attributes(
             status: :failed,
-            error_message: failure_message(result_event, outcome, secrets, label: code_session.runner == "codex" ? "Codex" : "Claude Code"),
+            error_message: failure_message(result_event, outcome, secrets, label: code_session.runner == "codex" ? "Codex" : "Claude Code",
+              resumed: code_session.turn_index.to_i.positive?),
             finished_at: Time.current
           )
         end
@@ -134,8 +150,9 @@ module ActionAgent
 
     # What went wrong, in Claude Code's own words when it reported a failure
     # and from the process otherwise.
-    def failure_message(result_event, outcome, secrets, label: "Claude Code")
+    def failure_message(result_event, outcome, secrets, label: "Claude Code", resumed: false)
       message = reported_failure(result_event) || process_failure(result_event, outcome, label: label)
+      message = RESUME_LOST if resumed && message.match?(/no conversation found|session.{0,40}not found|thread.{0,40}not found/i)
       SecretScrubber.scrub(message, secrets).truncate(MAX_ERROR_MESSAGE)
     end
 

data/app/jobs/action_agent/evaluation_judge_job.rb

@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+module ActionAgent
+  # Scores, diagnoses and stores one pending result of a fanned-out
+  # evaluation run (see EvaluationReplayJob). The job that scores the run's
+  # last pending result completes the run.
+  #
+  # A failure while scoring is stored on the result as an error rather than
+  # raised, so the run still completes.
+  class EvaluationJudgeJob < ApplicationJob
+    queue_as :agents
+
+    discard_on ActiveRecord::RecordNotFound
+
+    def perform(result_id, selection = {}, run_id = nil)
+      row = EvaluationScenarioResult.find_by(id: result_id)
+      return finish_without_result(run_id, selection) unless row
+
+      run = row.evaluation_run
+      begin
+        ScenarioEvaluationRunner.new(run.evaluation, selection: selection, run: run).judge_result!(row)
+      rescue StandardError => e
+        Rails.logger.warn("[ActionAgent] evaluation run #{run.id}: scoring result #{result_id} failed: #{e.class}: #{e.message}")
+        ScenarioEvaluationRunner.new(run.evaluation, selection: selection, run: run).fail_judging!(row.reload, e)
+      end
+    end
+
+    private
+
+    # A result destroyed before it was scored went with its scenario, removed
+    # from the suite during the run (EvaluationScenario destroys its results).
+    # It may have been the last result the run waited for, so the run's
+    # completion is checked without it.
+    def finish_without_result(run_id, selection)
+      run = run_id && EvaluationRun.find_by(id: run_id)
+      ScenarioEvaluationRunner.new(run.evaluation, selection: selection, run: run).finish_fan_out!(run) if run
+    end
+  end
+end

data/app/jobs/action_agent/evaluation_replay_job.rb

@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+module ActionAgent
+  # Replays one scenario under one model for a fanned-out evaluation run
+  # (ActionAgent.scenario_evaluation_fan_out) and stores the answer as a
+  # pending result; EvaluationJudgeJob scores it. One is enqueued per
+  # scenario x model, so the run is as parallel as the :agents queue.
+  #
+  # A failure outside the replay is stored as the pair's errored result
+  # rather than raised, so the run still completes; a scenario removed from
+  # the suite during the run is left out of it.
+  class EvaluationReplayJob < ApplicationJob
+    queue_as :agents
+
+    discard_on ActiveRecord::RecordNotFound
+
+    def perform(run_id, scenario_id, spec, selection = {})
+      run = EvaluationRun.find(run_id)
+      runner = ScenarioEvaluationRunner.new(run.evaluation, selection: selection, run: run)
+      begin
+        runner.replay_scenario!(run, scenario_id, spec)
+      rescue StandardError => e
+        Rails.logger.warn("[ActionAgent] evaluation run #{run_id}: replay of scenario #{scenario_id} failed: #{e.class}: #{e.message}")
+        ScenarioEvaluationRunner.new(run.evaluation, selection: selection, run: run).fail_result!(run.reload, scenario_id, spec, e)
+      end
+    end
+  end
+end