actionagent 1.8.0 → 1.8.1

data/app/services/action_agent/evaluation_standing.rb

@@ -0,0 +1,145 @@
+# frozen_string_literal: true
+
+module ActionAgent
+  # Where an evaluation stands against the agent as it is now, so a pass
+  # rate on the Evaluations page describes the current agent rather than
+  # pooling every suite's last run, however old the code it scored.
+  #
+  # The evaluation's headline run is its newest complete run; a newer run
+  # still pending or failed shows beside it, never in its place. Its
+  # standing is one of:
+  #
+  #   current    — the headline run scored the agent's current version
+  #   stale      — it scored an earlier version, or a release the dashboard
+  #                has not matched to the agent's current release
+  #   unrecorded — the run recorded no version and the agent has no release
+  #                versions at all, so nothing says which code it scored
+  #   archived   — the evaluation is archived (Evaluation#archive!)
+  #   none       — no complete run
+  #
+  # Only edits the model can see count as a new version: the instructions,
+  # the action prompts, the tools, the MCP servers, the model config and
+  # the response format. A dashboard edit to the agent's appearance cuts an
+  # AgentVersion like any other edit, but the run before it scored the same
+  # agent and stays current.
+  class EvaluationStanding
+    STANDINGS = %w[current stale unrecorded archived none].freeze
+    VERSION_STATES = %w[current earlier unrecorded].freeze
+    # The configuration snapshot keys the model is given.
+    MODEL_FACING = %w[instructions action_prompts tools mcp_servers model_config response_format].freeze
+
+    # Gives every evaluation its standing with one query per table rather
+    # than one per evaluation: the agents' latest versions and whether each
+    # has a release are read for the page at once.
+    def self.preload(evaluations)
+      evaluations = Array(evaluations)
+      agent_ids = evaluations.filter_map(&:agent_id).uniq
+      return if agent_ids.empty?
+
+      newest = AgentVersion.where(agent_id: agent_ids).group(:agent_id).maximum(:version_number)
+      latest = AgentVersion.where(agent_id: agent_ids).where(version_number: newest.values.uniq)
+        .select { |version| newest[version.agent_id] == version.version_number }.index_by(&:agent_id)
+      releases = AgentVersion.releases.where(agent_id: agent_ids).distinct.pluck(:agent_id).to_set
+
+      evaluations.each do |evaluation|
+        evaluation.standing_info = new(evaluation, latest_version: latest[evaluation.agent_id], has_releases: releases.include?(evaluation.agent_id))
+      end
+    end
+
+    attr_reader :evaluation
+
+    # @param headline_run [EvaluationRun, nil] the headline run when the
+    #   caller already holds it (AgentScorecard selects one per evaluation)
+    def initialize(evaluation, latest_version: :unknown, has_releases: :unknown, headline_run: :unknown)
+      @evaluation = evaluation
+      @latest_version = latest_version
+      @has_releases = has_releases
+      @headline_run = headline_run
+    end
+
+    def headline_run
+      return @headline_run unless @headline_run == :unknown
+
+      @headline_run = evaluation.headline_run
+    end
+
+    # The same standing read against +run+ as the headline run.
+    def with_headline(run)
+      self.class.new(evaluation, latest_version: latest_version, has_releases: has_releases?, headline_run: run)
+    end
+
+    def standing
+      return "archived" if evaluation.archived?
+      return "none" unless headline_run
+
+      case version_state(headline_run)
+      when "current" then "current"
+      when "unrecorded" then has_releases? ? "stale" : "unrecorded"
+      else "stale"
+      end
+    end
+
+    # Whether +run+ scored the agent's current version: "current",
+    # "earlier", or "unrecorded" when it recorded no version.
+    def version_state(run)
+      version = run&.agent_version
+      return "unrecorded" unless version
+
+      # The agent's last recorded deploy is the truth about the code that
+      # runs: a release version matching it is current whatever the
+      # dashboard edited since, and one that does not is earlier even when
+      # it was recorded last (a report of an older deploy published late).
+      if version.release? && (deployed = evaluation.agent&.release_digest.presence)
+        return deployed == version.release_digest ? "current" : "earlier"
+      end
+
+      latest = latest_version
+      return "current" if latest.nil? || latest.id == version.id
+      return "current" if model_facing(version) == model_facing(latest)
+
+      "earlier"
+    end
+
+    # Passes per model label of the headline run, `{ label => { passed, total } }`,
+    # from its recorded summaries; empty without them.
+    def per_model
+      run = headline_run
+      return {} unless run
+
+      summaries = run.scores.is_a?(Hash) ? (run.scores["_models"].presence || run.scores["_cohorts"]) : nil
+      return {} unless summaries.is_a?(Hash)
+
+      summaries.filter_map do |label, stats|
+        next unless stats.is_a?(Hash)
+
+        total = (stats["scenarios"] || stats["samples"]).to_i
+        [ label, { passed: stats["passed"].to_i, total: total } ]
+      end.to_h
+    end
+
+    # What the index and the scorecards pool: a current or unrecorded
+    # headline run; a stale or archived evaluation is left out and said so.
+    def counted?
+      %w[current unrecorded].include?(standing)
+    end
+
+    private
+
+    def latest_version
+      return @latest_version unless @latest_version == :unknown
+
+      @latest_version = evaluation.agent&.latest_version
+    end
+
+    def has_releases?
+      return @has_releases unless @has_releases == :unknown
+
+      @has_releases = evaluation.agent&.agent_versions&.releases&.exists? || false
+    end
+
+    def model_facing(version)
+      snapshot = version.configuration_snapshot.to_h.stringify_keys
+      MODEL_FACING.to_h { |key| [ key, snapshot[key] ] }
+    end
+  end
+end

data/app/services/action_agent/local_sandbox_backend.rb

@@ -141,7 +141,7 @@ module ActionAgent
       # ANTHROPIC_BASE_URL; a session inheriting those joins the developer's
       # session, and a base URL redirects the owner's credential. A session
       # gets exactly the Claude Code variables the backend sets.
-      \A(?:ANTHROPIC|CLAUDE|OPENAI|OPEN_AI|OPENROUTER|OPEN_ROUTER|OLLAMA)(?:_|\z) | \ACLAUDECODE\z
+      \A(?:ANTHROPIC|CLAUDE|CODEX|OPENAI|OPEN_AI|OPENROUTER|OPEN_ROUTER|OLLAMA)(?:_|\z) | \ACLAUDECODE\z
     /x
     SECRET_VARIABLE = /
       SECRET | TOKEN | PASSWORD | PASSWD | PASSPHRASE | API_KEY | APIKEY | PRIVATE_KEY | CREDENTIAL | ACCESS_KEY |
@@ -436,7 +436,11 @@ module ActionAgent
       0
     end
 
-    # Runs Claude Code headless in the sandbox's checkout, yielding each
+    def code_runners
+      %w[claude_code codex]
+    end
+
+    # Runs a coding agent headless in the sandbox's checkout, yielding each
     # stream-json event (a Hash, already scrubbed of the sandbox's secrets)
     # as it arrives.
     #
@@ -448,9 +452,12 @@ module ActionAgent
       app = workspace.join("app")
       raise Error, "Sandbox #{session_id} has no local checkout: start the sandbox again" unless app.directory?
 
-      credentials = session_credentials(sandbox)
+      runner = code_session.try(:runner) || "claude_code"
+      raise Error, "Unsupported code runner: #{runner}" unless code_runners.include?(runner)
+
+      credentials = session_credentials(sandbox, runner: runner)
       secrets = sandbox_secrets(sandbox, credentials)
-      argv = claude_argv(code_session)
+      argv = runner == "codex" ? codex_argv(code_session) : claude_argv(code_session)
       database_env = read_state(workspace)["database_env"]
       env = self.class.sanitized_environment
         .merge(database_env.is_a?(Hash) ? database_env.transform_values(&:to_s) : {})
@@ -466,12 +473,15 @@ module ActionAgent
       # own, in the workspace. With the machine's own login it must use the
       # user's: that is where `claude /login` left the credentials (HOME,
       # which the sanitized environment keeps, or the keychain).
-      unless ClaudeCodeAuth.local_login?
+      if runner == "codex"
+        env["CODEX_HOME"] = workspace.join("codex").to_s
+        FileUtils.mkdir_p(workspace.join("codex"), mode: 0o700)
+      elsif !ClaudeCodeAuth.local_login?
         env["CLAUDE_CONFIG_DIR"] = workspace.join("claude").to_s
         FileUtils.mkdir_p(workspace.join("claude"), mode: 0o700)
       end
 
-      run_claude(workspace, code_session, argv, env, secrets, &on_event)
+      run_claude(workspace, code_session, argv, env, secrets, runner: runner, &on_event)
     end
 
     # Stops a running Claude Code session: SIGTERM to its process group. The
@@ -969,6 +979,19 @@ module ActionAgent
 
     # --- Claude Code --------------------------------------------------------
 
+    def codex_argv(code_session)
+      argv = [
+        ActionAgent.codex_command.to_s, "exec", "--json", "--ephemeral",
+        "--sandbox", "workspace-write", "--config", 'approval_policy="never"', "--color", "never"
+      ]
+      if (model = code_session.model.presence)
+        raise Error, "#{model.inspect} is not a model name" unless MODEL_NAME.match?(model.to_s)
+
+        argv += [ "--model", model.to_s ]
+      end
+      argv + [ "-" ]
+    end
+
     def claude_argv(code_session)
       command = ActionAgent.claude_code_command.to_s
       argv = [
@@ -1009,18 +1032,20 @@ module ActionAgent
       LOGGED_OUT
     end
 
-    def run_claude(workspace, code_session, argv, env, secrets, &on_event)
+    def run_claude(workspace, code_session, argv, env, secrets, runner: "claude_code", &on_event)
+      label = runner == "codex" ? "Codex" : "Claude Code"
+      timeout = runner == "codex" ? ActionAgent.codex_timeout : ActionAgent.claude_code_timeout
       key = code_session.id.to_s
-      log = log_path(workspace, "claude-#{key}")
-      deadline = deadline_after(ActionAgent.claude_code_timeout)
+      log = log_path(workspace, "#{runner == 'codex' ? 'codex' : 'claude'}-#{key}")
+      deadline = deadline_after(timeout)
       # Checked again once Claude Code is recorded; this saves starting it.
       state = read_state(workspace)
-      refuse_stopped_session!(state, key)
+      refuse_stopped_session!(state, key, label: label)
       # A cancelled session frees its slot as soon as it is marked cancelled,
       # while its Claude Code may still be exiting (and diffing). Two in one
       # checkout would edit the same files.
       if other_session_running?(state, key, workspace.basename.to_s)
-        raise Error, "The previous Claude Code session in this sandbox is still stopping; try again in a moment"
+        raise Error, "The previous code session in this sandbox is still stopping; try again in a moment"
       end
       stdin_read, stdin_write = IO.pipe
       stdout_read, stdout_write = IO.pipe
@@ -1029,7 +1054,7 @@ module ActionAgent
       begin
         pid = spawn_group(env, *argv, chdir: workspace.join("app"), in: stdin_read, out: stdout_write, err: stderr_write)
       rescue SystemCallError => e
-        raise Error, "Could not start Claude Code (#{argv.first}): #{e.message}"
+        raise Error, "Could not start #{label} (#{argv.first}): #{e.message}"
       ensure
         [ stdin_read, stdout_write, stderr_write ].each(&:close)
       end
@@ -1040,7 +1065,7 @@ module ActionAgent
       case record_code_session(workspace, key, pid)
       when :terminating
         # Stopped by the ensure below, before it had the prompt.
-        raise Error, "The sandbox is being stopped, so Claude Code did not run"
+        raise Error, "The sandbox is being stopped, so #{label} did not run"
       when :cancelled
         # Runs its course like any cancelled session: it ends on SIGTERM.
         signal_group(pid, "TERM")
@@ -1067,7 +1092,7 @@ module ActionAgent
       unless finished
         # Cancelled, and SIGTERM did not end it within the grace: SIGKILL,
         # and the session finishes like any cancelled one.
-        raise Error, "Claude Code did not finish within #{ActionAgent.claude_code_timeout}s and was stopped" unless cancel_seen
+        raise Error, "#{label} did not finish within #{timeout}s and was stopped" unless cancel_seen
 
         stop_groups([ pid ], grace: 0)
       end
@@ -1091,11 +1116,11 @@ module ActionAgent
 
     # Why a session must not start: a terminate under way, or a cancel that
     # came before there was a process to stop.
-    def refuse_stopped_session!(state, key)
-      raise Error, "The sandbox is being stopped, so Claude Code did not run" if state["terminating"]
+    def refuse_stopped_session!(state, key, label: "Claude Code")
+      raise Error, "The sandbox is being stopped, so #{label} did not run" if state["terminating"]
 
       cancels = state["cancelled_code_sessions"]
-      raise Error, "Claude Code session #{key} was cancelled before it started" if cancels.is_a?(Hash) && cancels.key?(key)
+      raise Error, "#{label} session #{key} was cancelled before it started" if cancels.is_a?(Hash) && cancels.key?(key)
     end
 
     # Records Claude Code's pid (and start time) under the state.json lock,
@@ -1263,12 +1288,13 @@ module ActionAgent
     # The Claude Code variables a session runs with. An API key comes from
     # the owner's connection. The machine's own login needs none: `claude`
     # finds it itself, and the dashboard neither reads nor passes it on.
-    def session_credentials(sandbox)
-      return {} if ClaudeCodeAuth.local_login?
+    def session_credentials(sandbox, runner: "claude_code")
+      return {} if runner == "claude_code" && ClaudeCodeAuth.local_login?
 
-      credentials = sandbox.runtime_environment.to_h
+      credentials = (runner == "codex" ? sandbox.runtime_environment(runner: runner) : sandbox.runtime_environment).to_h
       if credentials.empty?
-        raise Error, "Claude Code is not connected: connect an Anthropic API key in Settings → Integrations"
+        label = runner == "codex" ? "Codex is not connected: connect an OpenAI API key" : "Claude Code is not connected: connect an Anthropic API key"
+        raise Error, "#{label} in Settings → Integrations"
       end
 
       credentials

data/app/services/action_agent/sandbox_orchestrator.rb

@@ -191,13 +191,26 @@ module ActionAgent
       ADAPTER_METHODS.fetch(verb).any? { |m| @backend.respond_to?(m) }
     end
 
+    # Existing backends predate runner selection and support Claude only.
+    # A backend must explicitly advertise Codex before accepting its keys.
+    def supports_code_runner?(runner)
+      return false unless supports?(:code_session)
+
+      runners = @backend.respond_to?(:code_runners) ? @backend.code_runners : [ "claude_code" ]
+      Array(runners).include?(runner)
+    end
+
     # Runs a Claude Code session in +sandbox_session+'s checkout, yielding
     # each stream-json event (a Hash) as it arrives. Returns the backend's
     # outcome: { exit_status:, diff: }.
     def run_code_session(sandbox_session, code_session, &on_event)
       # Checked when a session is requested too; this covers one queued
       # before the configuration changed.
-      refusal = ClaudeCodeAuth.backend_refusal(self)
+      runner = code_session.try(:runner) || "claude_code"
+      unless supports_code_runner?(runner)
+        raise UnsupportedBackendError, "The #{backend_name} sandbox backend cannot run #{runner} sessions"
+      end
+      refusal = ClaudeCodeAuth.backend_refusal(self) unless runner == "codex"
       raise UnsupportedBackendError, refusal if refusal
 
       @backend.public_send(adapter_method(:code_session), sandbox_session, code_session, &on_event)

data/app/services/action_agent/scenario_evaluation_runner.rb

@@ -223,7 +223,8 @@ module ActionAgent
         input_tokens: agent_run.input_tokens,
         output_tokens: agent_run.output_tokens,
         error: agent_run.failed? ? agent_run.error_message.presence || "run failed" : nil,
-        cost: ModelPricing.estimate(model: spec.model, input_tokens: agent_run.input_tokens, output_tokens: agent_run.output_tokens),
+        cost: ModelPricing.estimate(model: spec.model, provider: spec.provider, input_tokens: agent_run.input_tokens,
+                                    output_tokens: agent_run.output_tokens),
         metadata: { "agent_run_id" => agent_run.id }
       )
     end

data/config/routes.rb

@@ -142,7 +142,7 @@ ActionAgent::Engine.routes.draw do
 
     # Agent output evaluations. A scenario suite also manages its scenarios
     # here, and exposes each run's per-scenario, per-model results.
-    resources :evaluations, only: [ :index, :show, :create, :destroy ] do
+    resources :evaluations, only: [ :index, :show, :create, :update, :destroy ] do
       member do
         post :run
         get "runs/:run_id", action: :show_run, as: :run_result

data/lib/action_agent/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module ActionAgent
-  VERSION = "1.8.0"
+  VERSION = "1.8.1"
 end

data/lib/action_agent.rb

@@ -349,6 +349,10 @@ module ActionAgent
     # @return [Integer]
     attr_accessor :claude_code_timeout
 
+    # Codex CLI sessions use the owner's OpenAI API key and workspace-write
+    # sandboxing. They share checkout lifecycle and cancellation with Claude.
+    attr_accessor :codex_command, :codex_timeout
+
     # How Claude Code sessions authenticate.
     #
     # :api_key (the default) runs them on the Anthropic API key the owner
@@ -772,6 +776,8 @@ module ActionAgent
       @claude_code_permission_mode = "acceptEdits"
       @claude_code_max_turns = nil
       @claude_code_timeout = 1800
+      @codex_command = "codex"
+      @codex_timeout = 1800
       @claude_code_auth = :api_key
       @execution_enabled = true
       @run_host_agent_classes = false

data/lib/generators/action_agent/install_generator.rb

@@ -129,12 +129,15 @@ module ActionAgent
       end
 
       # Claude Code sessions inside those checkouts.
-      return if existing_migration?("create_active_agent_code_sessions")
-
-      migration_template(
-        "create_active_agent_code_sessions.rb.erb",
-        "db/migrate/create_active_agent_code_sessions.rb"
-      )
+      unless existing_migration?("create_active_agent_code_sessions")
+        migration_template(
+          "create_active_agent_code_sessions.rb.erb",
+          "db/migrate/create_active_agent_code_sessions.rb"
+        )
+      end
+      unless existing_migration?("add_code_session_runner")
+        migration_template("add_code_session_runner.rb.erb", "db/migrate/add_code_session_runner.rb")
+      end
     end
 
     def add_route

data/lib/generators/action_agent/templates/action_agent.rb.erb

@@ -136,6 +136,9 @@ ActionAgent.configure do |config|
   # config.claude_code_permission_mode = "acceptEdits"
   # config.claude_code_max_turns = nil        # Claude Code's own default
   # config.claude_code_timeout = 1800         # seconds
+  # Codex uses a separate API-key connection in Settings -> Integrations.
+  # config.codex_command = "codex"
+  # config.codex_timeout = 1800               # seconds
   #
   # How sessions authenticate. :api_key (the default) uses the Anthropic API
   # key connected in Settings -> Integrations (from the Claude Console,

data/lib/generators/action_agent/templates/add_code_session_runner.rb.erb

@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+class AddCodeSessionRunner < ActiveRecord::Migration<%= migration_version %>
+  def change
+    table = "#{ActionAgent.table_name_prefix}code_sessions"
+    add_column table, :runner, :string, null: false, default: "claude_code" unless column_exists?(table, :runner)
+    add_column table, :runner_session_id, :string unless column_exists?(table, :runner_session_id)
+  end
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: actionagent
 version: !ruby/object:Gem::Version
-  version: 1.8.0
+  version: 1.8.1
 platform: ruby
 authors:
 - Justin Bowen
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2026-09-30 00:00:00.000000000 Z
+date: 2026-10-01 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activeagent
@@ -210,10 +210,13 @@ files:
 - app/services/action_agent/agent_tool_roster.rb
 - app/services/action_agent/agent_toolbox.rb
 - app/services/action_agent/claude_code_auth.rb
+- app/services/action_agent/codex_session_events.rb
 - app/services/action_agent/dashboard_assistant_service.rb
 - app/services/action_agent/evaluation_evidence.rb
 - app/services/action_agent/evaluation_report_import.rb
+- app/services/action_agent/evaluation_run_cost.rb
 - app/services/action_agent/evaluation_runner_service.rb
+- app/services/action_agent/evaluation_standing.rb
 - app/services/action_agent/evaluation_tool_resolver.rb
 - app/services/action_agent/github_client.rb
 - app/services/action_agent/local_sandbox_backend.rb
@@ -250,6 +253,7 @@ files:
 - lib/generators/action_agent/templates/action_agent.rb.erb
 - lib/generators/action_agent/templates/add_agent_id_to_active_agent_telemetry_traces.rb.erb
 - lib/generators/action_agent/templates/add_agent_releases.rb.erb
+- lib/generators/action_agent/templates/add_code_session_runner.rb.erb
 - lib/generators/action_agent/templates/add_evaluation_report_identity.rb.erb
 - lib/generators/action_agent/templates/add_provider_key_api_key.rb.erb
 - lib/generators/action_agent/templates/create_active_agent_code_sessions.rb.erb