actionagent 1.7.2 → 1.8.1

data/app/controllers/action_agent/api/agents_controller.rb

@@ -4,6 +4,7 @@ module ActionAgent
   module Api
     class AgentsController < BaseController
       include AgentSerialization
+      include RunSandbox
 
       # Ranking for the agent cards. Every dimension except "recent" reads the
       # scorecard, which is computed in Ruby over both execution sources, so
@@ -25,7 +26,7 @@ module ActionAgent
       # same reason as the rest and one more: a keyword splat wins over the
       # arguments before it, so a client sending params[params][actor] would
       # otherwise name the caller its own run is authorized as.
-      RESERVED_EXECUTION_KEYS = [ :attachments, :action, :actor, :current_user ].freeze
+      RESERVED_EXECUTION_KEYS = [ :attachments, :action, :actor, :current_user, :runtime_sandbox ].freeze
 
       before_action :set_agent, only: [
         :show, :update, :destroy, :versions, :runs, :execute, :test, :restore, :duplicate, :export, :analytics,
@@ -169,13 +170,16 @@ module ActionAgent
       #
       # JSON as before, or multipart from the runner's composer: the new
       # user message, its files (attachments[]) and the conversation to
-      # continue (params[context_id], or a top-level context_id).
+      # continue (params[context_id], or a top-level context_id). A
+      # `sandbox_id` runs it against that checkout sandbox's app runtime
+      # too, without editing the agent (see RunSandbox); so does /test.
       def execute
         run = @agent.execute(
           execution_prompt,
           action: params[:action_name],
           attachments: uploaded_attachments,
           actor: agent_actor,
+          runtime_sandbox: run_sandbox_for(@agent, params[:sandbox_id])&.runtime_server_key,
           **execution_params
         )
         record_execution_usage
@@ -190,6 +194,7 @@ module ActionAgent
           action: params[:action_name],
           attachments: uploaded_attachments,
           actor: agent_actor,
+          runtime_sandbox: run_sandbox_for(@agent, params[:sandbox_id])&.runtime_server_key,
           **execution_params
         )
         record_execution_usage
@@ -268,11 +273,22 @@ module ActionAgent
       # What the Tools tab edits: the MCP services this agent can be given
       # and the tools it can be offered, each with the calls, errors and
       # latency recorded for it in the window.
+      #
+      # The checkout runtimes offered are the agent owner's — the only ones
+      # MCPToolDispatcher reaches for this agent — and of those, only the
+      # ones the caller may see as well. Intersected by id rather than with
+      # merge: both relations constrain the same owner column, and merge
+      # lets the second condition replace the first instead of adding to it.
       def tool_roster
+        runtimes = SandboxSession.runtime_server_listings(
+          SandboxSession.for_owner(@agent.owner).where(id: owned(SandboxSession).select(:id))
+        )
+
         render json: AgentToolRoster.new(
           agent: @agent,
           traces: owned_traces,
-          hours: params.fetch(:hours, ToolDiscovery::DEFAULT_WINDOW_HOURS).to_i
+          hours: params.fetch(:hours, ToolDiscovery::DEFAULT_WINDOW_HOURS).to_i,
+          runtimes: runtimes
         ).as_json
       end
 

data/app/controllers/action_agent/api/code_sessions_controller.rb

@@ -0,0 +1,168 @@
+# frozen_string_literal: true
+
+module ActionAgent
+  module Api
+    # Headless Claude Code sessions inside one of the caller's checkout
+    # sandboxes (Settings -> Integrations): start one with a prompt, poll its
+    # transcript, cancel it.
+    #
+    # A session runs on the owner's Anthropic API key (or, with
+    # ActionAgent.claude_code_auth = :local_login, this machine's own Claude
+    # Code login; see ClaudeCodeAuth) and edits the checkout, so starting one
+    # is execution — gated, and counted against the host app's quota, like
+    # running an agent.
+    class CodeSessionsController < BaseController
+      before_action :require_owner!
+      before_action :require_execution_enabled!, only: [ :create ]
+      before_action :set_sandbox
+      before_action :set_code_session, only: [ :show, :cancel ]
+
+      # A model name as Claude Code's --model takes it ("sonnet",
+      # "claude-sonnet-4-5", "claude-sonnet-4-5[1m]"). It becomes an argument
+      # to the CLI, so nothing that could read as another option gets there.
+      MODEL_NAME = /\A[A-Za-z0-9][A-Za-z0-9._:\[\]-]{0,99}\z/
+
+      # GET /api/sandboxes/:sandbox_id/code_sessions
+      def index
+        render json: { code_sessions: @sandbox.code_sessions.recent.limit(20).map(&:summary) }
+      end
+
+      # POST /api/sandboxes/:sandbox_id/code_sessions
+      def create
+        runner = params[:runner].nil? ? "claude_code" : params[:runner]
+        unless CodeSession::RUNNERS.include?(runner)
+          return render json: { error: "runner must be claude_code or codex" }, status: :unprocessable_entity
+        end
+        if (refusal = refusal_for(@sandbox, runner: runner))
+          return render json: { error: refusal }, status: :unprocessable_entity
+        end
+
+        if (current = busy_session)
+          return render json: busy_body(current), status: :conflict
+        end
+
+        code_session = CodeSession.new(
+          sandbox_session: @sandbox,
+          prompt: string_param(:prompt),
+          model: string_param(:model).presence,
+          runner: runner,
+          # Owned like the sandbox it runs in.
+          user_id: @sandbox.try(:user_id),
+          account_id: @sandbox.try(:account_id)
+        )
+        if (error = invalid_request(code_session))
+          return render json: { error: error }, status: :unprocessable_entity
+        end
+
+        enforce_execution_quota!
+        return if performed?
+
+        # Checked again under the sandbox's row lock, so two requests racing
+        # past the check above cannot both start a session in one checkout.
+        current = nil
+        @sandbox.with_lock do
+          current = busy_session
+          code_session.save! unless current
+        end
+        return render json: busy_body(current), status: :conflict if current
+
+        record_execution_usage
+        CodeSessionJob.perform_later(code_session.id)
+
+        render json: { code_session: code_session.summary }, status: :created
+      end
+
+      # GET /api/sandboxes/:sandbox_id/code_sessions/:id?after=N
+      # The transcript from event N onward, for incremental polling.
+      def show
+        render json: { code_session: @code_session.details(after: integer_param(:after, default: 0)) }
+      end
+
+      # POST /api/sandboxes/:sandbox_id/code_sessions/:id/cancel
+      def cancel
+        was_running = false
+        @code_session.with_lock do
+          next unless @code_session.queued? || @code_session.running?
+
+          was_running = @code_session.running?
+          # A queued session is settled at once: nothing will ever run it. A
+          # running one is settled by CodeSessionJob once its Claude Code has
+          # stopped and its diff was taken (see CodeSession#diff_pending?).
+          @code_session.update!(status: :cancelled, finished_at: was_running ? nil : Time.current)
+        end
+
+        # A queued session never started: CodeSessionJob skips it. A running
+        # one is stopped by its backend; CodeSessionJob then finds it
+        # cancelled and leaves it so.
+        stop(@code_session) if was_running
+
+        render json: { code_session: @code_session.summary }
+      end
+
+      private
+
+      # A checkout runs on its account's GitHub token and Claude Code
+      # credential, so in a multi-tenant install it must belong to the
+      # caller's current account too — not only to the caller, who may have
+      # left that account or switched to another.
+      def set_sandbox
+        scope = owned(SandboxSession)
+        scope = scope.where(account_id: current_account.id) if current_account
+        @sandbox = scope.find_by!(session_id: params[:sandbox_id])
+      end
+
+      def set_code_session
+        @code_session = @sandbox.code_sessions.find(params[:id])
+      end
+
+      # Why +sandbox+ cannot take a Claude Code session now, or nil.
+      def refusal_for(sandbox, runner: "claude_code")
+        label = runner == "codex" ? "Codex" : "Claude Code"
+        return "#{label} sessions run only in a checkout (app_runtime) sandbox" unless sandbox.app_runtime?
+        return "The sandbox has expired; start a new one" if sandbox.ready? && !sandbox.active?
+        return "The sandbox is #{sandbox.status}; wait until it is ready" unless sandbox.ready?
+
+        orchestrator = SandboxOrchestrator.new
+        unless orchestrator.supports_code_runner?(runner)
+          return "The #{orchestrator.backend_name} sandbox backend cannot run #{label} sessions"
+        end
+
+        if runner == "codex"
+          return "Codex is not connected: connect an OpenAI API key in Settings -> Integrations first" if sandbox.runtime_environment(runner: runner).blank?
+          return
+        end
+        ClaudeCodeAuth.backend_refusal(orchestrator) || ClaudeCodeAuth.credential_refusal(sandbox)
+      end
+
+      # The session of this sandbox that is queued or running, if any: one
+      # checkout, one Claude Code at a time.
+      def busy_session
+        @sandbox.code_sessions.where(status: [ :queued, :running ]).first
+      end
+
+      def busy_body(current)
+        label = current.runner == "codex" ? "Codex" : "Claude Code"
+        { error: "A #{label} session is already #{current.status} in this sandbox", code_session: current.summary }
+      end
+
+      def invalid_request(code_session)
+        label = code_session.runner == "codex" ? "Codex" : "Claude Code"
+        return "model is not a #{label} model name" if code_session.model && !code_session.model.match?(MODEL_NAME)
+
+        code_session.errors.full_messages.to_sentence unless code_session.valid?
+      end
+
+      # A prompt or model sent as an array or object is not one.
+      def string_param(name)
+        value = params[name]
+        value.is_a?(String) ? value : nil
+      end
+
+      def stop(code_session)
+        SandboxOrchestrator.new.cancel_code_session(@sandbox, code_session)
+      rescue StandardError => e
+        Rails.logger.warn("Failed to stop Claude Code session #{code_session.id}: #{e.message}")
+      end
+    end
+  end
+end

data/app/controllers/action_agent/api/evaluations_controller.rb

@@ -10,6 +10,8 @@ module ActionAgent
     # than sampling recorded generations, and can be narrowed to a group, to
     # specific scenarios, or to specific models.
     class EvaluationsController < BaseController
+      include EvaluationRunStarting
+
       rescue_from ActiveAgent::Evals::ScenarioParser::ParseError do |error|
         render json: { errors: [ error.message ] }, status: :unprocessable_entity
       end
@@ -39,6 +41,11 @@ module ActionAgent
       # 50 most recent. The scope is already restricted to the current user's
       # agents, so an id outside it simply returns nothing.
       #
+      # Archived evaluations are left out the same way, before the limit,
+      # unless `archived=1` asks for them; `archived_count` says how many
+      # the page left out. Each evaluation carries its standing against the
+      # agent's current version and its headline run (EvaluationSerializer).
+      #
       # Three fields feed the dashboard's model pickers. They describe the
       # credentials of #picker_credentials_owner:
       #   - judge_provider:        the provider a judge model runs on, null
@@ -52,16 +59,36 @@ module ActionAgent
       def index
         scope = evaluations_scope
         scope = scope.where(agent_id: params[:agent_id]) if params[:agent_id].present?
-        evaluations = scope.includes(:agent, :evaluation_runs, :scenarios).recent.limit(50)
+        archived_count = scope.archived.count
+        scope = scope.unarchived unless include_archived?
+        evaluations = scope.includes(:agent, :scenarios, evaluation_runs: :agent_version).recent.limit(50).to_a
+        preload_standing_and_costs(evaluations)
         owner = picker_credentials_owner
 
         render json: {
           evaluations: evaluations.map { |evaluation| serialize(evaluation) },
+          archived_count: archived_count,
           **judge_provider_fields(owner),
           model_providers: AgentExecutionService.available_providers(owner)
         }
       end
 
+      # PATCH /api/evaluations/:id
+      # `evaluation: { archived: true | false }` archives the evaluation —
+      # it keeps its runs but leaves the index, the pooled pass rate and the
+      # agent's scorecard — or brings it back. A new run brings it back too.
+      def update
+        evaluation = evaluations_scope.find(params[:id])
+        archived = params.require(:evaluation)[:archived]
+        if archived.nil?
+          return render json: { errors: [ "evaluation.archived must be true or false" ] }, status: :unprocessable_entity
+        end
+
+        ActiveModel::Type::Boolean.new.cast(archived) ? evaluation.archive! : evaluation.unarchive!
+
+        render json: { evaluation: serialize(evaluation.reload) }
+      end
+
       # Runs listed per evaluation on GET /api/evaluations/:id. The rest of
       # the history stays reachable by run id; `run_count` says how long it is.
       RUN_HISTORY_LIMIT = 20
@@ -70,7 +97,8 @@ module ActionAgent
       def show
         evaluation = evaluations_scope.find(params[:id])
         run_count = evaluation.evaluation_runs.count
-        runs = evaluation.evaluation_runs.recent.limit(RUN_HISTORY_LIMIT).to_a
+        runs = evaluation.evaluation_runs.includes(:agent_version).recent.limit(RUN_HISTORY_LIMIT).to_a
+        EvaluationRunCost.preload(runs).each { |id, breakdown| runs.find { |run| run.id == id }&.cost_breakdown = breakdown }
 
         render json: {
           evaluation: serialize(evaluation).merge(
@@ -107,7 +135,7 @@ module ActionAgent
         end
 
         if evaluation.save
-          start_run(evaluation, selection_params) if run_requested?
+          start_evaluation_run(evaluation, selection_params) if run_requested?
           render json: { evaluation: serialize(evaluation.reload) }, status: :created
         else
           render json: { errors: evaluation.errors.full_messages }, status: :unprocessable_entity
@@ -116,10 +144,17 @@ module ActionAgent
 
       # POST /api/evaluations/:id/run
       # A scenario suite accepts a selection: scenario_ids[], keys[], group,
-      # models[] (or a comma-separated `models` string).
+      # models[] (or a comma-separated `models` string), and `sandbox_id`: a
+      # checkout sandbox of the caller's whose app runtime every replay of
+      # this run reaches as well, without the agent being edited (see
+      # RunSandbox). The run records which one it used.
       def run
         evaluation = current_evaluation
-        run = start_run(evaluation, selection_params)
+        selection = selection_params
+        if (sandbox = evaluation_run_sandbox(evaluation, requested_sandbox_id))
+          selection[:sandbox_id] = sandbox.session_id
+        end
+        run = start_evaluation_run(evaluation, selection)
         evaluation.reload
 
         render json: {
@@ -134,16 +169,21 @@ module ActionAgent
       # fix items — the faults grouped with the tools, MCP server and
       # dashboard action that address each. Fix item paths are relative to
       # the mount: the React app resolves them itself (dashboardPath).
+      # `costs` is what each scenario cost across the models, judge apart,
+      # and the run's total (EvaluationRunCost#costs); every result carries
+      # its effective cost and how it was priced.
       def show_run
         evaluation = evaluations_scope.find(params[:id])
         run = evaluation.evaluation_runs.find(params[:run_id])
         results = run.scenario_results.includes(:scenario).joins(:scenario)
           .order(EvaluationScenario.arel_table[:position], EvaluationScenario.arel_table[:id], :model)
+        breakdown = run.cost_breakdown
 
         render json: {
           evaluation: serialize(evaluation),
           run: serialize_run(run, number: run_number(evaluation, run))
-            .merge(results: results.map(&:as_json_summary), fix_items: safe_fix_items(run))
+            .merge(results: results.map { |result| result.as_json_summary(costs: breakdown.result(result)) },
+                   fix_items: safe_fix_items(run), costs: breakdown.costs)
         }
       end
 
@@ -216,28 +256,19 @@ module ActionAgent
 
       private
 
-      # A scenario suite replays through the provider once per scenario and
-      # model, so it runs in the background; a generation-sampling evaluation
-      # scores recorded data and finishes inline.
-      def start_run(evaluation, selection)
-        return evaluation.run_later!(**selection) if evaluation.scenario_suite?
-
-        # EvaluationRunnerService marks the run failed with the error message
-        # and then re-raises. Letting that escape returned an HTML 500 for a
-        # request that had already persisted the evaluation and its failed
-        # run: the client saw a JSON parse error, the form stayed open, and a
-        # resubmit failed on the now-taken name. The failure is on the run
-        # record, which is what the response carries.
-        evaluation.run!
-      rescue StandardError => e
-        Rails.logger.warn(
-          "[ActionAgent] evaluation #{evaluation.id} run failed: #{e.class}: #{e.message}"
-        )
-        # The service records the failure before re-raising; a failure that
-        # predates the run record (creating it, say) is recorded here so the
-        # response always carries one.
-        evaluation.evaluation_runs.recent.first ||
-          evaluation.evaluation_runs.create!(status: :failed, error_message: e.message, completed_at: Time.current)
+      def include_archived?
+        ActiveModel::Type::Boolean.new.cast(params[:archived]) == true
+      end
+
+      # One query per table for the page's standings and for the costs of
+      # the runs it serializes in full (each evaluation's latest and headline
+      # run), rather than one per evaluation.
+      def preload_standing_and_costs(evaluations)
+        EvaluationStanding.preload(evaluations)
+        runs = evaluations.flat_map do |evaluation|
+          [ EvaluationSerializer.recent_runs(evaluation, 1).first, evaluation.standing_info.headline_run ]
+        end.compact.uniq(&:id)
+        EvaluationRunCost.preload(runs).each { |id, breakdown| runs.find { |run| run.id == id }&.cost_breakdown = breakdown }
       end
 
       # Returns whose credentials the index's model picker fields describe.
@@ -292,10 +323,7 @@ module ActionAgent
       def require_executable_scenario_agent!
         agent = action_name == "create" ? requested_agent : current_evaluation.agent
         return unless agent.observed?
-        if action_name == "run"
-          adapter = ActionAgent.scenario_evaluation_adapter_resolver&.call(current_evaluation)
-          return if adapter.respond_to?(:call)
-        end
+        return if action_name == "run" && !unexecutable_scenario_run?(current_evaluation)
 
         render json: {
           error: "Observed agents are read-only — duplicate this agent to create an executable copy"
@@ -310,19 +338,18 @@ module ActionAgent
         params.require(:scenario).permit(:prompt, :group, :notes, :enabled, :key, expectations: {})
       end
 
-      # scenario_ids, keys, group and models narrow a scenario run. `models`
-      # may arrive as an array or as the comma-separated field the form posts.
+      # The sandbox id a run was asked to use, from the selection or the top
+      # level of the request.
+      def requested_sandbox_id
+        selection_source[:sandbox_id].presence || params[:sandbox_id].presence
+      end
+
       def selection_params
-        source = params[:evaluation].is_a?(ActionController::Parameters) && params[:evaluation].key?(:selection) ? params[:evaluation][:selection] : params
-        models = source[:models]
-        models = models.to_s.split(",") unless models.is_a?(Array)
+        evaluation_run_selection(selection_source)
+      end
 
-        {
-          scenario_ids: Array(source[:scenario_ids]).map(&:to_s).reject(&:blank?),
-          keys: Array(source[:keys]).map(&:to_s).reject(&:blank?),
-          group: source[:group].to_s.presence,
-          models: models.map(&:to_s).map(&:strip).reject(&:blank?)
-        }.compact_blank
+      def selection_source
+        params[:evaluation].is_a?(ActionController::Parameters) && params[:evaluation].key?(:selection) ? params[:evaluation][:selection] : params
       end
 
       # Scenarios from text, a YAML/JSON suite, or a list of objects. Production
@@ -373,97 +400,13 @@ module ActionAgent
         models.map(&:to_s).map(&:strip).reject(&:blank?)
       end
 
-      def serialize(evaluation)
-        # size reads the preloaded association on index and COUNTs elsewhere.
-        run_count = evaluation.evaluation_runs.size
-        latest, previous = recent_runs(evaluation, 2)
-
-        {
-          id: evaluation.id,
-          name: evaluation.name,
-          agent: { id: evaluation.agent.id, name: evaluation.agent.name, slug: evaluation.agent.slug },
-          judge_kind: evaluation.judge_kind,
-          judge_model: evaluation.judge_model,
-          criteria: evaluation.criteria,
-          compare_models: evaluation.compare_models,
-          config: evaluation.config,
-          sample_size: evaluation.sample_size,
-          scenario_suite: evaluation.scenario_suite?,
-          scenario_count: evaluation.scenarios.size,
-          scenario_groups: evaluation.scenario_suite? ? evaluation.scenario_groups : [],
-          created_at: evaluation.created_at.iso8601,
-          run_count: run_count,
-          latest_run: latest ? serialize_run(latest, number: run_count) : nil,
-          # Just enough of the run before it for the list to show movement
-          # ("+3 passed vs #2") without a request per evaluation.
-          previous_run: previous ? serialize_run_summary(previous, number: run_count - 1) : nil
-        }
-      end
-
-      # Newest first. Sorts the preloaded association when index loaded it
-      # rather than issuing one ORDER BY query per evaluation.
-      def recent_runs(evaluation, limit)
-        runs = evaluation.evaluation_runs
-        if runs.loaded?
-          runs.sort_by { |run| [ run.created_at, run.id ] }.reverse.first(limit)
-        else
-          runs.recent.limit(limit).to_a
-        end
-      end
-
-      # A run's position in its evaluation's history, oldest = 1.
-      def run_number(evaluation, run)
-        evaluation.evaluation_runs.where("created_at < ? OR (created_at = ? AND id <= ?)", run.created_at, run.created_at, run.id).count
-      end
-
-      # `number` is the run's position in its evaluation's history, oldest =
-      # 1, so the dashboard can say "Run #3" and "vs #2".
-      def serialize_run(run, number: nil)
-        serialize_run_summary(run, number: number).merge(
-          scores: run.scores,
-          selection: run.selection,
-          models: run.models,
-          usage: run.usage,
-          error_message: run.error_message
-        )
-      end
+      def serialize(evaluation) = EvaluationSerializer.evaluation(evaluation)
 
-      def serialize_run_summary(run, number: nil)
-        {
-          id: run.id,
-          number: number,
-          status: run.status,
-          average_score: safe_average_score(run),
-          samples_evaluated: run.samples_evaluated,
-          samples_passed: run.samples_passed,
-          completed_at: run.completed_at&.iso8601,
-          created_at: run.created_at.iso8601
-        }
-      end
+      def serialize_run(run, number: nil) = EvaluationSerializer.run(run, number: number)
 
-      # The fix items are derived from every persisted result's diagnosis,
-      # which older runs recorded in earlier shapes; a run they cannot be
-      # built for still serves its results rather than 500-ing the panel.
-      def safe_fix_items(run)
-        run.fix_items
-      rescue StandardError => e
-        Rails.logger.warn(
-          "[ActionAgent] evaluation run #{run.id} fix_items failed: #{e.class}: #{e.message}"
-        )
-        []
-      end
+      def run_number(evaluation, run) = EvaluationSerializer.run_number(evaluation, run)
 
-      # index serializes the latest run of every listed evaluation, so an
-      # unaverageable scores payload used to 500 the entire Evaluations page
-      # instead of degrading that one run's headline number.
-      def safe_average_score(run)
-        run.average_score
-      rescue StandardError => e
-        Rails.logger.warn(
-          "[ActionAgent] evaluation run #{run.id} average_score failed: #{e.class}: #{e.message}"
-        )
-        nil
-      end
+      def safe_fix_items(run) = EvaluationSerializer.fix_items(run)
     end
   end
 end