actionagent 1.8.0 → 1.8.1

data/app/controllers/action_agent/api/code_sessions_controller.rb

@@ -29,7 +29,11 @@ module ActionAgent
 
       # POST /api/sandboxes/:sandbox_id/code_sessions
       def create
-        if (refusal = refusal_for(@sandbox))
+        runner = params[:runner].nil? ? "claude_code" : params[:runner]
+        unless CodeSession::RUNNERS.include?(runner)
+          return render json: { error: "runner must be claude_code or codex" }, status: :unprocessable_entity
+        end
+        if (refusal = refusal_for(@sandbox, runner: runner))
           return render json: { error: refusal }, status: :unprocessable_entity
         end
 
@@ -41,6 +45,7 @@ module ActionAgent
           sandbox_session: @sandbox,
           prompt: string_param(:prompt),
           model: string_param(:model).presence,
+          runner: runner,
           # Owned like the sandbox it runs in.
           user_id: @sandbox.try(:user_id),
           account_id: @sandbox.try(:account_id)
@@ -111,16 +116,21 @@ module ActionAgent
       end
 
       # Why +sandbox+ cannot take a Claude Code session now, or nil.
-      def refusal_for(sandbox)
-        return "Claude Code sessions run only in a checkout (app_runtime) sandbox" unless sandbox.app_runtime?
+      def refusal_for(sandbox, runner: "claude_code")
+        label = runner == "codex" ? "Codex" : "Claude Code"
+        return "#{label} sessions run only in a checkout (app_runtime) sandbox" unless sandbox.app_runtime?
         return "The sandbox has expired; start a new one" if sandbox.ready? && !sandbox.active?
         return "The sandbox is #{sandbox.status}; wait until it is ready" unless sandbox.ready?
 
         orchestrator = SandboxOrchestrator.new
-        unless orchestrator.supports?(:code_session)
-          return "The #{orchestrator.backend_name} sandbox backend cannot run Claude Code sessions"
+        unless orchestrator.supports_code_runner?(runner)
+          return "The #{orchestrator.backend_name} sandbox backend cannot run #{label} sessions"
         end
 
+        if runner == "codex"
+          return "Codex is not connected: connect an OpenAI API key in Settings -> Integrations first" if sandbox.runtime_environment(runner: runner).blank?
+          return
+        end
         ClaudeCodeAuth.backend_refusal(orchestrator) || ClaudeCodeAuth.credential_refusal(sandbox)
       end
 
@@ -131,11 +141,13 @@ module ActionAgent
       end
 
       def busy_body(current)
-        { error: "A Claude Code session is already #{current.status} in this sandbox", code_session: current.summary }
+        label = current.runner == "codex" ? "Codex" : "Claude Code"
+        { error: "A #{label} session is already #{current.status} in this sandbox", code_session: current.summary }
       end
 
       def invalid_request(code_session)
-        return "model is not a Claude Code model name" if code_session.model && !code_session.model.match?(MODEL_NAME)
+        label = code_session.runner == "codex" ? "Codex" : "Claude Code"
+        return "model is not a #{label} model name" if code_session.model && !code_session.model.match?(MODEL_NAME)
 
         code_session.errors.full_messages.to_sentence unless code_session.valid?
       end

data/app/controllers/action_agent/api/evaluations_controller.rb

@@ -41,6 +41,11 @@ module ActionAgent
       # 50 most recent. The scope is already restricted to the current user's
       # agents, so an id outside it simply returns nothing.
       #
+      # Archived evaluations are left out the same way, before the limit,
+      # unless `archived=1` asks for them; `archived_count` says how many
+      # the page left out. Each evaluation carries its standing against the
+      # agent's current version and its headline run (EvaluationSerializer).
+      #
       # Three fields feed the dashboard's model pickers. They describe the
       # credentials of #picker_credentials_owner:
       #   - judge_provider:        the provider a judge model runs on, null
@@ -54,16 +59,36 @@ module ActionAgent
       def index
         scope = evaluations_scope
         scope = scope.where(agent_id: params[:agent_id]) if params[:agent_id].present?
-        evaluations = scope.includes(:agent, :evaluation_runs, :scenarios).recent.limit(50)
+        archived_count = scope.archived.count
+        scope = scope.unarchived unless include_archived?
+        evaluations = scope.includes(:agent, :scenarios, evaluation_runs: :agent_version).recent.limit(50).to_a
+        preload_standing_and_costs(evaluations)
         owner = picker_credentials_owner
 
         render json: {
           evaluations: evaluations.map { |evaluation| serialize(evaluation) },
+          archived_count: archived_count,
           **judge_provider_fields(owner),
           model_providers: AgentExecutionService.available_providers(owner)
         }
       end
 
+      # PATCH /api/evaluations/:id
+      # `evaluation: { archived: true | false }` archives the evaluation —
+      # it keeps its runs but leaves the index, the pooled pass rate and the
+      # agent's scorecard — or brings it back. A new run brings it back too.
+      def update
+        evaluation = evaluations_scope.find(params[:id])
+        archived = params.require(:evaluation)[:archived]
+        if archived.nil?
+          return render json: { errors: [ "evaluation.archived must be true or false" ] }, status: :unprocessable_entity
+        end
+
+        ActiveModel::Type::Boolean.new.cast(archived) ? evaluation.archive! : evaluation.unarchive!
+
+        render json: { evaluation: serialize(evaluation.reload) }
+      end
+
       # Runs listed per evaluation on GET /api/evaluations/:id. The rest of
       # the history stays reachable by run id; `run_count` says how long it is.
       RUN_HISTORY_LIMIT = 20
@@ -72,7 +97,8 @@ module ActionAgent
       def show
         evaluation = evaluations_scope.find(params[:id])
         run_count = evaluation.evaluation_runs.count
-        runs = evaluation.evaluation_runs.recent.limit(RUN_HISTORY_LIMIT).to_a
+        runs = evaluation.evaluation_runs.includes(:agent_version).recent.limit(RUN_HISTORY_LIMIT).to_a
+        EvaluationRunCost.preload(runs).each { |id, breakdown| runs.find { |run| run.id == id }&.cost_breakdown = breakdown }
 
         render json: {
           evaluation: serialize(evaluation).merge(
@@ -143,16 +169,21 @@ module ActionAgent
       # fix items — the faults grouped with the tools, MCP server and
       # dashboard action that address each. Fix item paths are relative to
       # the mount: the React app resolves them itself (dashboardPath).
+      # `costs` is what each scenario cost across the models, judge apart,
+      # and the run's total (EvaluationRunCost#costs); every result carries
+      # its effective cost and how it was priced.
       def show_run
         evaluation = evaluations_scope.find(params[:id])
         run = evaluation.evaluation_runs.find(params[:run_id])
         results = run.scenario_results.includes(:scenario).joins(:scenario)
           .order(EvaluationScenario.arel_table[:position], EvaluationScenario.arel_table[:id], :model)
+        breakdown = run.cost_breakdown
 
         render json: {
           evaluation: serialize(evaluation),
           run: serialize_run(run, number: run_number(evaluation, run))
-            .merge(results: results.map(&:as_json_summary), fix_items: safe_fix_items(run))
+            .merge(results: results.map { |result| result.as_json_summary(costs: breakdown.result(result)) },
+                   fix_items: safe_fix_items(run), costs: breakdown.costs)
         }
       end
 
@@ -225,6 +256,21 @@ module ActionAgent
 
       private
 
+      def include_archived?
+        ActiveModel::Type::Boolean.new.cast(params[:archived]) == true
+      end
+
+      # One query per table for the page's standings and for the costs of
+      # the runs it serializes in full (each evaluation's latest and headline
+      # run), rather than one per evaluation.
+      def preload_standing_and_costs(evaluations)
+        EvaluationStanding.preload(evaluations)
+        runs = evaluations.flat_map do |evaluation|
+          [ EvaluationSerializer.recent_runs(evaluation, 1).first, evaluation.standing_info.headline_run ]
+        end.compact.uniq(&:id)
+        EvaluationRunCost.preload(runs).each { |id, breakdown| runs.find { |run| run.id == id }&.cost_breakdown = breakdown }
+      end
+
       # Returns whose credentials the index's model picker fields describe.
       # Agent runs and their judge use the evaluated agent's owner's
       # credentials, so a list scoped to one agent reads that agent's owner,

data/app/controllers/action_agent/api/sandboxes_controller.rb

@@ -97,6 +97,8 @@ module ActionAgent
           sample_tasks: sample_tasks,
           sandboxes: listed_sandboxes.map(&:summary),
           code_sessions_supported: code_sessions_supported?,
+          codex_sessions_supported: codex_sessions_supported?,
+          codex_connected: owned(ProviderKey).where(provider: "codex").any? { |key| key.runtime_environment.present? },
           **claude_code_status
         }
       end
@@ -234,6 +236,12 @@ module ActionAgent
         false
       end
 
+      def codex_sessions_supported?
+        SandboxOrchestrator.new.supports_code_runner?("codex")
+      rescue StandardError, LoadError
+        false
+      end
+
       # Whether the caller's Claude Code can run sessions, by
       # ClaudeCodeAuth's rule: an API key they connected, or this machine's
       # own login. Never a credential.

data/app/controllers/concerns/action_agent/api/mcp_dashboard_tools.rb

@@ -202,13 +202,15 @@ module ActionAgent
           scope = scope.where(agent: resolve_tool_agent!(agent))
         end
         limit = integer_argument(:limit, default: LIST_LIMIT, min: 1, max: MAX_LIST_LIMIT)
-        evaluations = scope.includes(:agent, :evaluation_runs, :scenarios).recent.limit(limit)
+        evaluations = scope.includes(:agent, :scenarios, evaluation_runs: :agent_version).recent.limit(limit).to_a
+        EvaluationStanding.preload(evaluations)
 
         {
           evaluations: evaluations.map do |evaluation|
             summary = EvaluationSerializer.summary(evaluation)
             latest = EvaluationSerializer.recent_runs(evaluation, 1).first
-            summary.slice(:id, :name, :agent, :judge_kind, :scenario_suite, :scenario_count, :scenario_groups, :created_at, :run_count)
+            summary.slice(:id, :name, :agent, :judge_kind, :scenario_suite, :scenario_count, :scenario_groups, :created_at, :run_count,
+                          :headline_run_id, :standing, :archived_at)
               .merge(latest_run: latest && EvaluationSerializer.run_summary(latest, number: summary[:run_count]))
           end
         }

data/app/jobs/action_agent/code_session_job.rb

@@ -24,6 +24,7 @@ module ActionAgent
       # connection and the Claude Code key.
       secrets = code_session.secrets
       result_event = nil
+      codex_events = CodexSessionEvents.new if code_session.runner == "codex"
       stop_sent = false
       orchestrator = SandboxOrchestrator.new
       sandbox = code_session.sandbox_session
@@ -41,6 +42,10 @@ module ActionAgent
         # has no process to stop; an event means the process exists now, so
         # the stop is sent again, once.
         code_session.append_event!(event, secrets: secrets)
+        if codex_events
+          event = codex_events.consume(event)
+          code_session.append_event!(event, secrets: secrets) if event
+        end
         if code_session.cancelled? && !stop_sent
           stop_sent = true
           stop(orchestrator, sandbox, code_session)
@@ -54,7 +59,7 @@ module ActionAgent
       finish(code_session, outcome.to_h, result_event, secrets)
     rescue StandardError => e
       message = SecretScrubber.scrub(e.message.to_s, secrets || safe_secrets(code_session))
-      Rails.logger.error("Claude Code session #{code_session_id} failed: #{message}")
+      Rails.logger.error("Code session #{code_session_id} failed: #{message}")
       fail!(code_session, message) if code_session
     end
 
@@ -83,7 +88,7 @@ module ActionAgent
         else
           code_session.assign_attributes(
             status: :failed,
-            error_message: failure_message(result_event, outcome, secrets),
+            error_message: failure_message(result_event, outcome, secrets, label: code_session.runner == "codex" ? "Codex" : "Claude Code"),
             finished_at: Time.current
           )
         end
@@ -105,8 +110,8 @@ module ActionAgent
 
     # What went wrong, in Claude Code's own words when it reported a failure
     # and from the process otherwise.
-    def failure_message(result_event, outcome, secrets)
-      message = reported_failure(result_event) || process_failure(result_event, outcome)
+    def failure_message(result_event, outcome, secrets, label: "Claude Code")
+      message = reported_failure(result_event) || process_failure(result_event, outcome, label: label)
       SecretScrubber.scrub(message, secrets).truncate(MAX_ERROR_MESSAGE)
     end
 
@@ -119,12 +124,12 @@ module ActionAgent
         "Claude Code stopped: #{result_event['subtype']}"
     end
 
-    def process_failure(result_event, outcome)
+    def process_failure(result_event, outcome, label: "Claude Code")
       status = outcome[:exit_status]
       headline =
-        if !status.nil? && status != 0 then "Claude Code exited with status #{status}"
-        elsif result_event.nil? then "Claude Code ended without reporting a result"
-        else "Claude Code did not finish"
+        if !status.nil? && status != 0 then "#{label} exited with status #{status}"
+        elsif result_event.nil? then "#{label} ended without reporting a result"
+        else "#{label} did not finish"
         end
 
       [ headline, outcome[:stderr_tail].to_s.strip.presence ].compact.join(": ")

data/app/models/action_agent/agent.rb

@@ -313,6 +313,39 @@ module ActionAgent
       version
     end
 
+    # The version carrying a release an application reports having
+    # evaluated (EvaluationReportImport reads it from `report.release`),
+    # found among every version of this agent — a report may describe a
+    # deploy older than the latest — or recorded now when none does. A
+    # version recorded this way carries no manifest: the report names the
+    # digest, not what went into it.
+    #
+    # The agent's own `release_digest` is what its deploys last recorded
+    # (#record_release!); a report never moves it, except to set it for an
+    # agent that never recorded a deploy at all.
+    #
+    # @param digest [String] the ActiveAgent::Release digest the report names
+    # @param revision [String, nil] the deploy the report names
+    # @param label [String, nil] how the report labels the release
+    # @return [AgentVersion]
+    def find_or_record_release!(digest:, revision: nil, label: nil)
+      existing = agent_versions.find_by(release_digest: digest)
+      return existing if existing
+
+      version = agent_versions.create!(
+        version_number: (latest_version&.version_number || 0) + 1,
+        change_summary: [ "Release #{digest}", revision.presence, label.presence ].compact.join(" · ") + ": reported by an evaluation",
+        configuration_snapshot: configuration_snapshot.merge("release" => {}),
+        release_digest: digest,
+        revision: revision,
+        created_by: "evaluation-report"
+      )
+      update_columns(release_digest: digest) if release_digest.blank?
+      version
+    rescue ActiveRecord::RecordNotUnique
+      agent_versions.find_by!(release_digest: digest)
+    end
+
     # Maps each historical instructions digest to the first version that
     # introduced it ("v3"), so run cohorts can label instruction changes with
     # real agent versions instead of raw hashes.

data/app/models/action_agent/code_session.rb

@@ -22,8 +22,10 @@ module ActionAgent
     # Per string inside an event: tool results can be whole files.
     MAX_EVENT_STRING = 4_000
     MAX_DIFF_BYTES = 500_000
+    RUNNERS = %w[claude_code codex].freeze
 
     validates :prompt, presence: true, length: { maximum: MAX_PROMPT_CHARACTERS }
+    validates :runner, inclusion: { in: RUNNERS }
 
     scope :recent, -> { order(created_at: :desc) }
 
@@ -51,7 +53,7 @@ module ActionAgent
     # Claude Code credential this session ran with.
     def secrets
       spec = sandbox_session.checkout_spec rescue nil
-      [ spec&.dig(:token), *sandbox_session.runtime_environment.values ].compact
+      [ spec&.dig(:token), *sandbox_session.runtime_environment(runner: runner).values ].compact
     end
 
     # Appends one stream-json event, scrubbed and bounded. Past MAX_EVENTS
@@ -77,7 +79,8 @@ module ActionAgent
 
       update!(
         result: event["result"].to_s.presence && SecretScrubber.scrub(event["result"].to_s, secrets).truncate(MAX_EVENT_STRING * 4),
-        claude_session_id: event["session_id"],
+        claude_session_id: runner == "claude_code" ? event["session_id"] : nil,
+        runner_session_id: event["session_id"],
         num_turns: event["num_turns"],
         duration_ms: event["duration_ms"],
         total_cost_usd: event["total_cost_usd"],
@@ -98,6 +101,7 @@ module ActionAgent
         status: status,
         prompt: prompt,
         model: model,
+        runner: runner,
         result: result,
         error_message: error_message,
         num_turns: num_turns,

data/app/models/action_agent/evaluation.rb

@@ -34,6 +34,20 @@ module ActionAgent
     validate :validate_criteria
 
     scope :recent, -> { order(updated_at: :desc) }
+    # Archived evaluations keep their runs but leave the index and its
+    # tiles unless asked for (see #archive!). The mark lives in the config
+    # JSON, so the filter reads it with each database's own JSON path.
+    scope :archived, -> { where("#{archived_at_sql} IS NOT NULL") }
+    scope :unarchived, -> { where("#{archived_at_sql} IS NULL") }
+
+    # SQL reading config["archived_at"] on the connected database.
+    def self.archived_at_sql
+      case connection.adapter_name.to_s.downcase
+      when /postgres/ then "#{quoted_table_name}.config ->> 'archived_at'"
+      when /mysql|trilogy/ then "JSON_UNQUOTE(JSON_EXTRACT(#{quoted_table_name}.config, '$.archived_at'))"
+      else "json_extract(#{quoted_table_name}.config, '$.archived_at')"
+      end
+    end
 
     # MySQL cannot give a JSON column a default, so a row inserted there
     # without `criteria` or `config` reads back nil. Both readers answer with
@@ -50,6 +64,56 @@ module ActionAgent
       evaluation_runs.order(created_at: :desc).first
     end
 
+    # The newest run that finished: the one the evaluation's pass rate
+    # describes. A newer run still pending, or one that failed, shows
+    # beside it and never in its place. Read from the loaded association
+    # when the index preloaded it.
+    def headline_run
+      if evaluation_runs.loaded?
+        evaluation_runs.select(&:complete?).max_by { |run| [ run.created_at, run.id ] }
+      else
+        evaluation_runs.complete.order(created_at: :desc, id: :desc).first
+      end
+    end
+
+    # --- archiving ---------------------------------------------------------
+    #
+    # An evaluation nobody maintains — its suite superseded, its agent
+    # retired — keeps its history but stops counting: the index leaves it
+    # out, and so do the pooled pass rate and the agent's scorecard. The
+    # mark is config["archived_at"]; a new run or a published report clears
+    # it, since either says the evaluation is alive after all.
+
+    def archived_at
+      value = config["archived_at"]
+      value.present? ? Time.zone.parse(value.to_s) : nil
+    rescue ArgumentError
+      nil
+    end
+
+    def archived?
+      config["archived_at"].present?
+    end
+
+    def archive!
+      update!(config: config.merge("archived_at" => Time.current.iso8601))
+    end
+
+    def unarchive!
+      return unless archived?
+
+      update!(config: config.except("archived_at"))
+    end
+
+    # Where this evaluation stands against the agent as it is now: whether
+    # its headline run scored the current version (EvaluationStanding).
+    # Memoized per instance; the index preloads what it needs.
+    def standing_info
+      @standing_info ||= EvaluationStanding.new(self)
+    end
+
+    attr_writer :standing_info
+
     def judge_defined?
       judge_kind == "judge_defined"
     end