actionagent 1.8.0 → 1.9.0

data/app/controllers/concerns/action_agent/api/project_browser.rb

@@ -0,0 +1,46 @@
+# frozen_string_literal: true
+
+module ActionAgent
+  module Api
+    # The browser of a project's sandbox, for the endpoints that drive it
+    # from the engine: an exploration's start and the test account step.
+    module ProjectBrowser
+      # The tool groups the project's browser starts with: the explorer's
+      # locator and verify tools come from "testing".
+      CAPABILITIES = %w[testing].freeze
+
+      private
+
+      # The project's ready sandbox, or nil after rendering 409 when it has
+      # none.
+      def ready_project_sandbox!(project)
+        sandbox = project.current_sandbox_session
+        return sandbox if sandbox && project.sandbox_state == "ready"
+
+        render json: { error: "Boot the project's sandbox first", code: "sandbox_not_ready", sandbox_state: project.sandbox_state },
+          status: :conflict
+        nil
+      end
+
+      # [sandbox, whether its browser was started now] with the browser
+      # running: the one already running, or one started headless with the
+      # project's saved sign-in after the host's quota allows
+      # :browser_minutes. Nil after rendering why it cannot run.
+      def ensure_project_browser!(project, sandbox)
+        return [ sandbox, false ] if sandbox.browser_running?
+
+        enforce_quota!(:browser_minutes)
+        return nil if performed?
+
+        sandbox, started = SandboxBrowser.ensure_running!(sandbox, capabilities: CAPABILITIES,
+          recording_url: ->(recording) { events_api_session_recording_url(recording) },
+          storage_state: project.saved_storage_state)
+        [ sandbox, started ]
+      rescue SandboxBrowser::Error => e
+        render json: { error: SecretScrubber.scrub(e.message, project.scrub_values), code: "browser_unavailable" },
+          status: :unprocessable_entity
+        nil
+      end
+    end
+  end
+end

data/app/controllers/concerns/action_agent/api/project_secret_authorization.rb

@@ -0,0 +1,57 @@
+# frozen_string_literal: true
+
+module ActionAgent
+  module Api
+    # Who may decide a project's secrets and the code that reads them. Each
+    # method answers whether the signed-in user may, and renders the 403
+    # first when not, so a caller returns unless it is true.
+    #
+    # :manage_project_secrets is always asked about a ProjectSecret, and
+    # :manage_credentials about the ProviderKey a secret hands over.
+    module ProjectSecretAuthorization
+      # The sources a person may give a secret. A value that is not secret
+      # (ProjectSecret source "setup_assistant") is set only by the setup
+      # assistant.
+      PERSON_SECRET_SOURCES = %w[entered organization_key].freeze
+
+      private
+
+      # Whether every one of +entries+ names a source a person may give, or
+      # none. Renders 422 first when not.
+      def secret_sources_allowed!(entries)
+        refused = entries.find do |entry|
+          source = entry.respond_to?(:permit) || entry.is_a?(Hash) ? entry[:source] || entry["source"] : nil
+          source.present? && !PERSON_SECRET_SOURCES.include?(source.to_s)
+        end
+        return true if refused.nil?
+
+        render json: { error: "A secret's source must be one of #{PERSON_SECRET_SOURCES.join(", ")}", code: "invalid_source" },
+          status: :unprocessable_entity
+        false
+      end
+
+      # Setting +secret+ needs :manage_project_secrets. A secret that takes
+      # the organization's stored provider key also needs
+      # :manage_credentials, because the repository's code can then read a
+      # key the provider keys API never reveals.
+      def authorize_secret!(secret)
+        return false unless authorize_action!(:manage_project_secrets, secret)
+
+        key = secret.organization_key? ? secret.organization_provider_key : nil
+        key.nil? || authorize_action!(:manage_credentials, key)
+      end
+
+      # Handing +project+'s secrets to other code, such as the code at a new
+      # ref, needs what setting each of them needs.
+      def authorize_secret_handover!(project)
+        project.secrets.to_a.all? { |secret| authorize_secret!(secret) }
+      end
+
+      # Deleting +project+ deletes its secrets, which needs
+      # :manage_project_secrets for each.
+      def authorize_secrets_removal!(project)
+        project.secrets.to_a.all? { |secret| authorize_action!(:manage_project_secrets, secret) }
+      end
+    end
+  end
+end

data/app/jobs/action_agent/agent_execution_job.rb

@@ -7,8 +7,10 @@ module ActionAgent
     def perform(run_id)
       run = AgentRun.find(run_id)
       # Anything already finished — complete, failed or cancelled — is never
-      # executed again. A retry of a failed run would re-run the generation.
-      return if run.finished?
+      # executed again, and neither is a run waiting for input, which only
+      # AgentResumeJob continues. A retry of a failed run would re-run the
+      # generation.
+      return if run.finished? || run.awaiting_input?
 
       run.update!(status: :running, started_at: Time.current)
       run.add_log("Starting execution", level: :info)
@@ -19,30 +21,13 @@ module ActionAgent
         # Build the agent class dynamically based on configuration
         result = execute_agent(agent_record, run)
 
-        finish_unless_cancelled(run) do
-          run.update!(
-            output: result[:output],
-            output_metadata: result[:metadata],
-            status: :complete,
-            completed_at: Time.current,
-            duration_ms: ((Time.current - run.started_at) * 1000).to_i,
-            input_tokens: result.dig(:usage, :input_tokens),
-            output_tokens: result.dig(:usage, :output_tokens),
-            total_tokens: result.dig(:usage, :total_tokens)
-          )
-          run.add_log("Execution completed successfully", level: :info)
+        run.unless_cancelled do
+          run.record_result!(result, segment_started_at: run.started_at)
+          run.add_log(result[:input_requests].present? ? "Execution paused for input" : "Execution completed successfully", level: :info)
         end
       rescue => e
-        finish_unless_cancelled(run) do
-          run.update!(
-            status: :failed,
-            completed_at: Time.current,
-            error_message: e.message,
-            error_backtrace: e.backtrace&.first(10)&.join("\n"),
-            # Same as the synchronous path: the class is what distinguishes
-            # a refusal from a crash.
-            output_metadata: run.output_metadata.to_h.merge("error_class" => e.class.name)
-          )
+        run.unless_cancelled do
+          run.record_failure!(e)
           run.add_log("Execution failed: #{e.message}", level: :error)
         end
         raise
@@ -56,20 +41,5 @@ module ActionAgent
     def execute_agent(agent_record, run)
       AgentExecutionService.call(agent_record, run)
     end
-
-    # A cancel that arrived while the service was running must survive it:
-    # the run's terminal state is written under a row lock after re-reading
-    # the status, so a cancelled run is never flipped back to complete or
-    # failed by the job that was still executing it.
-    def finish_unless_cancelled(run)
-      run.with_lock do
-        if run.cancelled?
-          run.add_log("Execution finished after cancellation; result discarded", level: :info)
-          return
-        end
-
-        yield
-      end
-    end
   end
 end

data/app/jobs/action_agent/agent_resume_job.rb

@@ -0,0 +1,84 @@
+# frozen_string_literal: true
+
+module ActionAgent
+  # Continues a run that paused for input, once every request of the pause
+  # is answered or declined.
+  #
+  # Its only argument is the id of one of the pause's requests. The answers
+  # are read and decrypted here, so none of them, `:secret` answers
+  # included, is ever a job argument or reaches the queue's storage. Two
+  # answers that settle a pause together each enqueue this job, and only the
+  # first to take the run's lock resumes it.
+  class AgentResumeJob < ApplicationJob
+    queue_as :agents
+
+    # Raised, and recorded as the run's failure, when agent execution was
+    # turned off after the pause was settled.
+    class ExecutionDisabled < StandardError; end
+
+    def perform(input_request_id)
+      request = InputRequest.find_by(id: input_request_id)
+      run = request&.subject
+      return unless run.is_a?(AgentRun)
+
+      pause = claim_pause(run, request)
+      return unless pause
+
+      segment_started_at = Time.current
+      secrets = pause.select { |paused| paused.kind == "secret" }.filter_map(&:answer)
+
+      begin
+        raise ExecutionDisabled, "Agent execution is disabled on this dashboard" unless ActionAgent.execution_enabled?
+
+        result = AgentExecutionService.call(
+          run.agent, run,
+          resume: {
+            checkpoint: request.checkpoint_data,
+            answers: pause.to_h { |paused| [ paused.tool_call_id, paused.resume_answer ] },
+            secrets: secrets
+          }
+        )
+
+        run.unless_cancelled do
+          run.record_result!(result, segment_started_at: segment_started_at)
+          run.add_log(result[:input_requests].present? ? "Execution paused for input" : "Execution completed successfully", level: :info)
+        end
+      rescue => e
+        message = ActiveAgent::InputRequest.scrub(e.message, secrets)
+        run.unless_cancelled do
+          run.record_failure!(e, message: message)
+          run.add_log("Execution failed: #{message}", level: :error)
+        end
+      ensure
+        forget_secret_answers(pause)
+        run.broadcast_update
+      end
+    end
+
+    private
+
+    # Moves the run from awaiting_input to running when every request of the
+    # pause is settled, under the run's row lock, and returns the pause's
+    # requests in call order; nil when this job is not the one to resume it,
+    # or when the request's pause is not the one the run waits on.
+    def claim_pause(run, request)
+      run.with_lock do
+        next nil unless run.awaiting_input? && request.current_pause?
+
+        pause = request.pause_requests.order(:id).to_a
+        next nil unless pause.all? { |paused| paused.answered? || paused.declined? }
+
+        run.update!(status: :running)
+        pause
+      end
+    end
+
+    # A secret answer has reached its tool once the resume ran, so it is not
+    # kept any longer than that.
+    def forget_secret_answers(pause)
+      pause.select { |paused| paused.kind == "secret" && paused.answer.present? }.each do |paused|
+        paused.update_columns(answer: nil)
+      end
+    end
+  end
+end

data/app/jobs/action_agent/code_session_job.rb

@@ -24,13 +24,15 @@ module ActionAgent
       # connection and the Claude Code key.
       secrets = code_session.secrets
       result_event = nil
+      codex_events = CodexSessionEvents.new if code_session.runner == "codex"
       stop_sent = false
       orchestrator = SandboxOrchestrator.new
       sandbox = code_session.sandbox_session
 
       # Queued behind a Stop (the cleanup job shares this queue): never start
-      # Claude Code, with the owner's credential, in a stopped sandbox.
-      unless sandbox.reload.ready? && sandbox.active?
+      # Claude Code, with the owner's credential, in a stopped sandbox. A
+      # project's boot fix runs in the checkout its failed boot kept.
+      unless (sandbox.reload.ready? && sandbox.active?) || ProjectBootFix.runs_in_failed_boot?(code_session)
         return fail!(code_session, "The sandbox was stopped before the session started")
       end
 
@@ -41,6 +43,10 @@ module ActionAgent
         # has no process to stop; an event means the process exists now, so
         # the stop is sent again, once.
         code_session.append_event!(event, secrets: secrets)
+        if codex_events
+          event = codex_events.consume(event)
+          code_session.append_event!(event, secrets: secrets) if event
+        end
         if code_session.cancelled? && !stop_sent
           stop_sent = true
           stop(orchestrator, sandbox, code_session)
@@ -52,14 +58,37 @@ module ActionAgent
       end
 
       finish(code_session, outcome.to_h, result_event, secrets)
+      verify(code_session)
+      resume_boot(code_session)
     rescue StandardError => e
       message = SecretScrubber.scrub(e.message.to_s, secrets || safe_secrets(code_session))
-      Rails.logger.error("Claude Code session #{code_session_id} failed: #{message}")
+      Rails.logger.error("Code session #{code_session_id} failed: #{message}")
       fail!(code_session, message) if code_session
+      resume_boot(code_session) if code_session
     end
 
     private
 
+    # A project's boot fix resumes the boot it fixed (ProjectBootFix). Its
+    # failure is the boot's, not the session's, so it is logged and left on
+    # the project rather than raised.
+    def resume_boot(code_session)
+      ProjectBootFix.after_session(code_session.reload)
+    rescue StandardError => e
+      Rails.logger.warn("Code session #{code_session.id}: the project's boot was not resumed: #{e.class}: #{e.message}")
+    end
+
+    # A fix session that changed the checkout is verified next. If that
+    # cannot even be queued, the error is the session's, so the sandbox is
+    # not left waiting for a verification that will never come.
+    def verify(code_session)
+      return unless code_session.succeeded? && code_session.evaluation_run_id && code_session.diff.present?
+
+      VerifyEvaluationFixJob.perform_later(code_session.id)
+    rescue StandardError => e
+      code_session.update_columns(verification_error: "Verification could not be queued: #{e.class}")
+    end
+
     def start(code_session)
       now = Time.current
       claimed = CodeSession.where(id: code_session.id, status: CodeSession.statuses[:queued])
@@ -83,7 +112,7 @@ module ActionAgent
         else
           code_session.assign_attributes(
             status: :failed,
-            error_message: failure_message(result_event, outcome, secrets),
+            error_message: failure_message(result_event, outcome, secrets, label: code_session.runner == "codex" ? "Codex" : "Claude Code"),
             finished_at: Time.current
           )
         end
@@ -105,8 +134,8 @@ module ActionAgent
 
     # What went wrong, in Claude Code's own words when it reported a failure
     # and from the process otherwise.
-    def failure_message(result_event, outcome, secrets)
-      message = reported_failure(result_event) || process_failure(result_event, outcome)
+    def failure_message(result_event, outcome, secrets, label: "Claude Code")
+      message = reported_failure(result_event) || process_failure(result_event, outcome, label: label)
       SecretScrubber.scrub(message, secrets).truncate(MAX_ERROR_MESSAGE)
     end
 
@@ -119,12 +148,12 @@ module ActionAgent
         "Claude Code stopped: #{result_event['subtype']}"
     end
 
-    def process_failure(result_event, outcome)
+    def process_failure(result_event, outcome, label: "Claude Code")
       status = outcome[:exit_status]
       headline =
-        if !status.nil? && status != 0 then "Claude Code exited with status #{status}"
-        elsif result_event.nil? then "Claude Code ended without reporting a result"
-        else "Claude Code did not finish"
+        if !status.nil? && status != 0 then "#{label} exited with status #{status}"
+        elsif result_event.nil? then "#{label} ended without reporting a result"
+        else "#{label} did not finish"
         end
 
       [ headline, outcome[:stderr_tail].to_s.strip.presence ].compact.join(": ")

data/app/jobs/action_agent/draft_pull_request_job.rb

@@ -0,0 +1,43 @@
+# frozen_string_literal: true
+
+module ActionAgent
+  # Runs one queued publish of a DraftPullRequest through
+  # DraftPullRequestPublisher, after asking ActionAgent.permission_checker
+  # again whether the user who asked for it may publish pull requests. A
+  # project's install pull request is published with the files the project
+  # generates (ProjectInstallPullRequest).
+  #
+  # Takes the record's id and nothing else: the token is minted while the
+  # job runs, and never passes through the queue. Not retried, since a
+  # publish writes to GitHub.
+  class DraftPullRequestJob < ApplicationJob
+    queue_as :sandboxes
+
+    def perform(draft_pull_request_id)
+      # Claimed atomically, so a second job for the same record finds it no
+      # longer queued.
+      claimed = DraftPullRequest.where(id: draft_pull_request_id, status: "queued")
+        .update_all(status: "publishing", updated_at: Time.current)
+      return unless claimed == 1
+
+      record = DraftPullRequest.find(draft_pull_request_id)
+      user = record.publisher
+      if ActionAgent.permitted?(user, :publish_pull_request, record)
+        ProjectInstallPullRequest.publisher_for(record, user: user).publish!(record)
+      else
+        record.update!(status: "failed", error_code: "forbidden", error_message: "You do not have permission to publish pull requests")
+      end
+      broadcast(record)
+    rescue StandardError => e
+      Rails.logger.error("[ActionAgent] publishing draft pull request #{draft_pull_request_id} failed: #{e.class}: #{e.message}")
+      DraftPullRequest.where(id: draft_pull_request_id, status: %w[queued publishing])
+        .update_all(status: "failed", error_code: "github_error", error_message: "The publish failed unexpectedly", updated_at: Time.current)
+    end
+
+    private
+
+    def broadcast(record)
+      LiveUpdates.broadcast("sandbox_#{record.sandbox_session.session_id}", type: "pull_request", id: record.id, status: record.status)
+    end
+  end
+end

data/app/jobs/action_agent/exploration_job.rb

@@ -0,0 +1,80 @@
+# frozen_string_literal: true
+
+module ActionAgent
+  # Walks a project's app with the explorer agent for one pending explorer
+  # Exploration (see Api::ProjectExplorationsController and
+  # ExplorerExecutionService), then records how it ended:
+  #
+  #   - the explorer finished, ran out of budget or was stopped: the run is
+  #     complete and the exploration moves to review with its candidates
+  #   - anything else went wrong: the run and the exploration fail, and the
+  #     exploration keeps the candidates it found
+  #
+  # A browser the start launched for the walk is stopped afterwards, which
+  # completes its recording and stops its minutes. That includes a walk that
+  # never began, because it was stopped before the job ran or its run is
+  # gone.
+  class ExplorationJob < ApplicationJob
+    queue_as :agents
+
+    def perform(exploration_id, stop_browser = false)
+      exploration = Exploration.find_by(id: exploration_id)
+      return unless exploration&.source == "explorer"
+
+      begin
+        begin_walk(exploration)
+      ensure
+        stop(exploration) if stop_browser
+      end
+    end
+
+    private
+
+    def begin_walk(exploration)
+      return unless exploration.status == "pending"
+
+      run = exploration.agent_run
+      return exploration.fail_walk!("The explorer's run is missing") if run.nil? || run.finished?
+
+      exploration.start_walk!
+      run.update!(status: :running, started_at: Time.current)
+      walk(exploration, run)
+    end
+
+    def walk(exploration, run)
+      result = ExplorerExecutionService.new(exploration, run).walk!
+      complete(run, result[:output], result)
+      exploration.finish_walk!(reason: result[:stop_reason])
+    rescue ExplorerExecutionService::WalkCutShort => e
+      complete(run, e.message, {})
+      exploration.finish_walk!(reason: e.reason)
+    rescue StandardError => e
+      message = SecretScrubber.scrub(e.message, exploration.scrub_secrets)
+      run.update!(status: :failed, completed_at: Time.current, error_message: message,
+        output_metadata: run.output_metadata.to_h.merge("error_class" => e.class.name))
+      exploration.fail_walk!(message)
+    ensure
+      run.broadcast_update
+    end
+
+    def complete(run, output, result)
+      run.update!(
+        output: output,
+        output_metadata: result[:metadata].to_h.merge("stop_reason" => result[:stop_reason], "summary" => result[:summary]).compact,
+        status: :complete,
+        completed_at: Time.current,
+        duration_ms: ((Time.current - run.started_at) * 1000).to_i,
+        input_tokens: result.dig(:usage, :input_tokens),
+        output_tokens: result.dig(:usage, :output_tokens),
+        total_tokens: result.dig(:usage, :total_tokens)
+      )
+    end
+
+    def stop(exploration)
+      sandbox = exploration.sandbox_session&.reload
+      SandboxBrowser.stop(sandbox) if sandbox && SandboxBrowser::STARTED.include?(sandbox.browser_status)
+    rescue SandboxBrowser::Error => e
+      Rails.logger.warn("[ActionAgent] exploration #{exploration.id}: could not stop the browser it started: #{e.message}")
+    end
+  end
+end

data/app/jobs/action_agent/input_request_expiry_job.rb

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+module ActionAgent
+  # Expires the requests for input left unanswered past their `expires_at`,
+  # and fails their runs (InputRequest.expire_overdue!). An answer, the
+  # request list and a run's own page expire the requests they reach; this
+  # job reaches the rest.
+  #
+  # Not scheduled by default. To schedule it:
+  #
+  #   # config/recurring.yml
+  #   input_request_expiry:
+  #     class: ActionAgent::InputRequestExpiryJob
+  #     schedule: every 15 minutes
+  class InputRequestExpiryJob < ApplicationJob
+    queue_as :default
+
+    def perform
+      InputRequest.expire_overdue!
+    end
+  end
+end

data/app/jobs/action_agent/project_evaluation_job.rb

@@ -0,0 +1,59 @@
+# frozen_string_literal: true
+
+module ActionAgent
+  # Runs a project's evaluation (Api::ProjectsController#run_evaluation)
+  # against the project's sandbox once it serves. The run was created
+  # pending; while the sandbox boots, the job checks again every
+  # POLL_INTERVAL rather than holding a worker, and fails the run when the
+  # boot fails, stops, or takes longer than BOOT_WAIT.
+  #
+  # Every replay also reaches the sandbox's browser (see
+  # ScenarioEvaluationRunner), whose recording posts to the dashboard
+  # mounted at +mount_url+ when the run starts it.
+  class ProjectEvaluationJob < ApplicationJob
+    queue_as :agents
+
+    POLL_INTERVAL = 10.seconds
+    BOOT_WAIT = 1.hour
+
+    def perform(project_id, run_id, mount_url = nil)
+      run = EvaluationRun.find_by(id: run_id)
+      return unless run&.pending?
+
+      project = Project.find_by(id: project_id)
+      return fail_run(run, "The project was deleted before the run started") if project.nil?
+
+      sandbox = project.current_sandbox_session
+      case project.sandbox_state
+      when "ready"
+        start(run, sandbox, mount_url)
+      when "booting"
+        if run.created_at <= BOOT_WAIT.ago
+          fail_run(run, "The project's sandbox did not finish booting within #{BOOT_WAIT.inspect}")
+        else
+          self.class.set(wait: POLL_INTERVAL).perform_later(project_id, run_id, mount_url)
+        end
+      when "failed"
+        reason = SecretScrubber.scrub(sandbox.error_message.to_s.lines.first.to_s.strip, project.scrub_values)
+        fail_run(run, "The project's sandbox failed to boot#{": #{reason}" if reason.present?}")
+      else
+        fail_run(run, "The project's sandbox stopped before the run started: run the evaluation again")
+      end
+    end
+
+    private
+
+    def start(run, sandbox, mount_url)
+      evaluation = run.evaluation
+      if ActionAgent.scenario_evaluation_adapter_resolver&.call(evaluation).respond_to?(:call)
+        return fail_run(run, "This install replays scenarios through its own adapter, which cannot reach a project's sandbox")
+      end
+
+      evaluation.run!(run: run, sandbox_id: sandbox.session_id, browser: true, mount_url: mount_url)
+    end
+
+    def fail_run(run, message)
+      run.update!(status: :failed, error_message: message, completed_at: Time.current)
+    end
+  end
+end

data/app/jobs/action_agent/project_setup_job.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+module ActionAgent
+  # Starts help for a project once a boot of the project failed: a Claude
+  # Code session in the failed checkout when the owner's Claude Code is
+  # connected (see ProjectBootFix.after_boot_failed), and the setup assistant
+  # otherwise (see ProjectSetup.after_boot_failed). Takes ids only; a project or
+  # sandbox deleted since, or a sandbox that is no longer the project's
+  # current one, starts nothing.
+  class ProjectSetupJob < ApplicationJob
+    queue_as :agents
+
+    def perform(project_id, sandbox_session_id)
+      project = Project.find_by(id: project_id)
+      sandbox = SandboxSession.find_by(id: sandbox_session_id)
+      return if project.nil? || sandbox.nil? || !sandbox.failed?
+
+      ProjectBootFix.after_boot_failed(project, sandbox) || ProjectSetup.after_boot_failed(project, sandbox)
+    end
+  end
+end

data/app/jobs/action_agent/sandbox_cleanup_job.rb

@@ -15,6 +15,7 @@ module ActionAgent
       # after a failed terminate, nothing would ever know the process (or
       # container) was still there. cleanup_expired! retries these.
       handle = sandbox.cloud_run_job_id.presence || derived_handle(sandbox)
+      stop_browser(sandbox)
       return if handle.present? && !released?(sandbox, handle)
 
       # Optionally delete old sandbox records
@@ -52,15 +53,20 @@ module ActionAgent
     # indistinguishable from an unreleased one, and a terminate of a
     # sandbox that is gone is a cheap no-op, so the window is what bounds
     # the retries.
+    #
+    # A failed checkout is among them for a day once its expiry has passed:
+    # a boot with keep_on_failure keeps its workspace and databases for
+    # SandboxSession#resume_boot! until then. Its status stays failed.
     RETRY_UNRECORDED_FOR = 1.day
     RETRY_UNRECORDED_LIMIT = 100
 
     def self.unrecorded_checkouts
       return [] unless SandboxOrchestrator.new.derives_handles?
 
-      SandboxSession.expired.by_type("app_runtime").where(cloud_run_job_id: [ nil, "" ])
-        .where(updated_at: RETRY_UNRECORDED_FOR.ago..)
-        .order(updated_at: :desc).limit(RETRY_UNRECORDED_LIMIT).pluck(:id)
+      checkouts = SandboxSession.by_type("app_runtime").where(cloud_run_job_id: [ nil, "" ])
+      expired = checkouts.expired.where(updated_at: RETRY_UNRECORDED_FOR.ago..)
+      kept = checkouts.failed.where(expires_at: RETRY_UNRECORDED_FOR.ago..Time.current)
+      expired.or(kept).order(updated_at: :desc).limit(RETRY_UNRECORDED_LIMIT).pluck(:id)
     rescue StandardError, LoadError => e
       Rails.logger.warn("[ActionAgent] sandbox backend unavailable to the reaper: #{e.message}")
       []
@@ -81,6 +87,17 @@ module ActionAgent
       nil
     end
 
+    # A checkout's browser, which a backend may run apart from the sandbox
+    # itself. Best effort: the :local backend's terminate stops it as well.
+    def stop_browser(sandbox)
+      return unless sandbox.app_runtime?
+
+      orchestrator = SandboxOrchestrator.new
+      orchestrator.stop_browser(sandbox) if orchestrator.supports?(:stop_browser)
+    rescue StandardError, LoadError => e
+      Rails.logger.warn("[ActionAgent] could not stop the browser of sandbox #{sandbox.session_id}: #{e.message}")
+    end
+
     # Whether the backend behind +handle+ let go of it.
     #
     # In development a sandbox other than a checkout was only simulated (see