actionagent 1.8.0 → 1.8.1

data/app/models/action_agent/evaluation_run.rb

@@ -9,15 +9,29 @@ module ActionAgent
   class EvaluationRun < ApplicationRecord
     belongs_to :evaluation
     # The version of the evaluated agent this run scored, so a pass rate is
-    # a statement about a release rather than about "the agent".
+    # a statement about a release rather than about "the agent". A run the
+    # engine executes scored the agent as it is now; an imported run scored
+    # the publishing application's code, which only its report can name
+    # (EvaluationReportImport sets the version from `report.release`), so
+    # a release-less import stays unrecorded rather than claiming the
+    # dashboard's latest version.
     belongs_to :agent_version, optional: true
-    before_create { self.agent_version_id ||= evaluation&.agent&.latest_version&.id }
+    before_create { self.agent_version_id ||= evaluation&.agent&.latest_version&.id unless imported? }
+    # A run started on an archived evaluation brings it back: archiving says
+    # "no longer maintained", which a new run contradicts.
+    after_create { evaluation.unarchive! if evaluation&.archived? }
     has_many :scenario_results, class_name: "EvaluationScenarioResult", dependent: :destroy
 
     enum :status, { pending: 0, running: 1, complete: 2, failed: 3 }
 
     scope :recent, -> { order(created_at: :desc) }
 
+    # Whether this run was published by an application that ran it itself
+    # (EvaluationReportImport) rather than executed here.
+    def imported?
+      external_run_id.present?
+    end
+
     # Which scenarios and models a scenario run covered; empty for a
     # generation-sampling run.
     def selection
@@ -92,14 +106,37 @@ module ActionAgent
       (values.sum.to_f / values.size).round(3)
     end
 
-    # The judge's own spend on this run — calls, tokens, estimated cost and
-    # how many calls served each purpose — as the runner recorded it; nil
-    # for a run that never asked a judge.
-    def judge_usage
+    # The judge's own spend on this run as the engine's meter recorded it —
+    # calls, tokens, estimated cost and how many calls served each purpose
+    # — or nil for a run the engine did not judge (a rules-only run, or an
+    # imported one, whose judge the application ran).
+    def judge_usage_meter
       value = scores&.dig("_judge_usage")
       value.is_a?(Hash) ? value : nil
     end
 
+    # The judge's spend on this run from whatever recorded it: the meter,
+    # the publishing application's figures, or the judge traces priced
+    # (EvaluationRunCost). nil for a run no judge was asked about.
+    def judge_usage
+      cost_breakdown.judge_usage
+    end
+
+    # Every cost figure of this run — per result, per scenario, per model
+    # and in total — worked out once (EvaluationRunCost). A controller that
+    # lists runs preloads it (EvaluationRunCost.preload) and hands it in.
+    def cost_breakdown
+      @cost_breakdown ||= EvaluationRunCost.for(self)
+    end
+
+    attr_writer :cost_breakdown
+
+    # Forgets the breakdown, so a run whose results just changed is priced again.
+    def reload(*)
+      @cost_breakdown = nil
+      super
+    end
+
     # Per-model summaries of a generation-sampling run's cohorts, keyed by
     # model; empty for a scenario run or a run recorded before they were.
     def cohorts
@@ -107,59 +144,51 @@ module ActionAgent
       value.is_a?(Hash) ? value : {}
     end
 
+    # Per-model summaries of a scenario run, keyed by label, as the runner
+    # recorded them under "_models", each with its cost as it stands now:
+    # the effective "cost" over its results (reported, else estimated), the
+    # "priced", "reported" and "estimated" counts behind it, and the
+    # judge's "judge_cost" and "judge_calls" on them (EvaluationRunCost). A
+    # recorded summary no result maps to is served as recorded. Empty for a
+    # generation-sampling run.
+    def model_summaries
+      summaries = scores&.dig("_models")
+      return {} unless summaries.is_a?(Hash)
+
+      costed = cost_breakdown.by_label(summaries.keys)
+      summaries.to_h do |label, stats|
+        next [ label, stats ] unless stats.is_a?(Hash) && costed.key?(label)
+
+        [ label, stats.merge(costed[label]) ]
+      end
+    end
+
     # What the run spent, for display after it: the agent's side and the
     # judge's, kept apart because they answer different questions.
     #
     # The agent's side is the operating figure — what the interactions cost
-    # to serve. For a scenario run that is the replays' estimated cost,
-    # tokens and summed model time (`replays` of them); for a
-    # generation-sampling run it is the sampled generations' (`samples`),
-    # which were served before the run and cost it nothing. `per_interaction`
-    # is that cost spread over the interactions, the number a per-conversation
-    # budget is set against.
+    # to serve. For a scenario run that is its replays' cost, tokens and
+    # summed model time (`replays` of them); for a generation-sampling run
+    # it is the sampled generations' (`samples`), which were served before
+    # the run and cost it nothing.
+    #
+    # Every interaction with tokens is priced (EvaluationRunCost): `cost`
+    # sums the reported costs and, where none was reported, the estimates
+    # from tokens × model rates. `priced` and `unpriced` count the
+    # interactions either way, `reported` and `estimated` say how the priced
+    # ones were priced, and `cost_basis` sums that up as "reported",
+    # "estimated" or "mixed". `per_interaction` is the cost per priced
+    # interaction, the number a per-conversation budget is set against.
     #
     # `judge` is the evaluation's own overhead: the judge model's calls
     # (scoring, recommending, the verdict, authoring KPIs), which run
-    # agent-to-agent and offline. It is present only when a judge was asked.
+    # agent-to-agent and offline — from the engine's meter, the publishing
+    # application's figures or the judge traces, with `source` naming which
+    # and `run` the calls no result owns. `total` is the two sides together.
     #
     # Returns nil for a run that recorded nothing on either side.
     def usage
-      totals = scenario_results.pick(
-        Arel.sql("COUNT(*)"), Arel.sql("SUM(cost)"), Arel.sql("SUM(input_tokens)"),
-        Arel.sql("SUM(output_tokens)"), Arel.sql("SUM(duration_ms)")
-      )
-      replays = totals&.first.to_i
-      judge = judge_usage
-      runtime_ms = completed_at.present? ? ((completed_at - created_at) * 1000).round : nil
-
-      if replays.positive?
-        cost = totals[1]&.to_f
-        {
-          replays: replays,
-          cost: cost,
-          per_interaction: cost && (cost / replays).round(6),
-          input_tokens: totals[2].to_i,
-          output_tokens: totals[3].to_i,
-          model_time_ms: totals[4].to_i,
-          runtime_ms: runtime_ms,
-          judge: judge
-        }.compact
-      elsif cohorts.any?
-        samples = cohorts.values.sum { |cohort| cohort["samples"].to_i }
-        costs = cohorts.values.filter_map { |cohort| cohort["cost"] }
-        cost = costs.any? ? costs.sum.to_f.round(6) : nil
-        {
-          samples: samples,
-          cost: cost,
-          per_interaction: cost && samples.positive? ? (cost / samples).round(6) : nil,
-          input_tokens: cohorts.values.sum { |cohort| cohort["input_tokens"].to_i },
-          output_tokens: cohorts.values.sum { |cohort| cohort["output_tokens"].to_i },
-          runtime_ms: runtime_ms,
-          judge: judge
-        }.compact
-      elsif judge
-        { runtime_ms: runtime_ms, judge: judge }.compact
-      end
+      cost_breakdown.usage
     end
 
     # Route templates for the report's fix item actions, relative to the
@@ -194,10 +223,21 @@ module ActionAgent
     # the suite panel does. Raises ActiveRecord::RecordNotFound via the
     # caller for a run of a generation-sampling evaluation, which has no
     # scenario results to report on.
-    def to_report(links: report_links)
+    #
+    # With `estimate` (the default) each replay carries its effective cost
+    # and how it was priced (EvaluationRunCost), and the report is told the
+    # judge's run-level spend, so the page shows every cost the dashboard
+    # does. `estimate: false` rebuilds the report from what was recorded
+    # alone — the import summarizes a published run that way, so the
+    # summaries it stores stay the application's own figures.
+    #
+    # The engine runs against every activeagent since 1.4, so the report
+    # kwargs that arrived later are passed only when Report.new takes them.
+    def to_report(links: report_links, estimate: true)
       rows = scenario_results.includes(:scenario).sort_by do |row|
         [ row.evaluated_scenario["position"].to_i, row.evaluation_scenario_id, row.model ]
       end
+      breakdown = cost_breakdown if estimate
       selected = selected_specs
       specs = {}
       results = rows.map do |row|
@@ -210,14 +250,15 @@ module ActionAgent
           replay: ActiveAgent::Evals::Replay.new(
             answer: row.output, tool_calls: Array(row.tool_calls), duration_ms: row.duration_ms,
             input_tokens: row.input_tokens, output_tokens: row.output_tokens,
-            cost: row.cost&.to_f, error: row.error_message, metadata: row.replay_metadata
+            cost: breakdown ? breakdown.result(row)["cost"] : row.cost&.to_f, error: row.error_message,
+            metadata: replay_metadata_for(row, breakdown)
           ),
           scores: row.scores.to_h, score: row.score, status: row.status,
           diagnosis: row.evaluation_diagnosis.presence
         )
       end
 
-      ActiveAgent::Evals::Report.new(
+      kwargs = {
         results: results,
         models: (selected.values & specs.values) + (specs.values - selected.values),
         metadata: {
@@ -232,7 +273,40 @@ module ActionAgent
         tool_resolver: EvaluationToolResolver.new(evaluation.agent),
         agent_name: evaluation.agent&.name,
         links: links
-      )
+      }
+      kwargs[:judge_usage] = breakdown.judge_usage_run if breakdown && self.class.report_accepts?(:judge_usage)
+      kwargs[:release] = release_summary if release_summary && self.class.report_accepts?(:release)
+      self.class.report_class.new(**kwargs)
+    end
+
+    # The framework's Report, as installed.
+    def self.report_class
+      ActiveAgent::Evals::Report
+    end
+
+    # Whether the installed framework's Report.new declares +keyword+.
+    def self.report_accepts?(keyword)
+      report_class.instance_method(:initialize).parameters.any? { |type, name| name == keyword && %i[key keyreq].include?(type) }
+    end
+
+    # The release this run scored, as the report names it — `{ "digest",
+    # "revision", "label" }` — or nil for a run pinned to a dashboard edit
+    # or to no version.
+    def release_summary
+      version = agent_version
+      return nil unless version&.release?
+
+      { "digest" => version.release_digest, "revision" => version.revision, "label" => "v#{version.version_number}" }.compact
+    end
+
+    # The agent version this run scored, for the run's JSON: `{ id, number,
+    # release_digest, revision, release }`, or nil when none was recorded.
+    def agent_version_summary
+      version = agent_version
+      return nil unless version
+
+      { id: version.id, number: version.version_number, release_digest: version.release_digest,
+        revision: version.revision, release: version.release? }
     end
 
     private
@@ -240,6 +314,22 @@ module ActionAgent
     ModelSpec = ActiveAgent::Evals::ModelSpec
     private_constant :ModelSpec
 
+    # The replay metadata the rebuilt report reads a result's costs from:
+    # what the result recorded, the judge usage the application reported
+    # for it, and — when estimating — how its cost was priced.
+    def replay_metadata_for(row, breakdown)
+      metadata = row.replay_metadata.dup
+      reported_judge = row.diagnosis.is_a?(Hash) ? row.diagnosis["_judge_usage"] : nil
+      metadata["judge_usage"] = reported_judge if reported_judge.is_a?(Hash)
+      return metadata unless breakdown
+
+      entry = breakdown.result(row)
+      metadata["cost_source"] = entry["cost_source"]
+      metadata["cost_rate"] = entry["cost_rate"] if entry["cost_rate"]
+      metadata["judge_usage"] = entry["judge_usage"] if entry["judge_usage"]
+      metadata
+    end
+
     # How the report's header names the sandbox: its checkout and session.
     def sandbox_label
       return nil unless sandbox

data/app/models/action_agent/evaluation_scenario_result.rb

@@ -44,8 +44,11 @@ module ActionAgent
       value.is_a?(Hash) ? value : {}
     end
 
+    # The diagnosis as the dashboard shows it: without the storage keys —
+    # the replay metadata, the scenario snapshot and the judge usage the
+    # publishing application reported (served as `judge_usage` instead).
     def evaluation_diagnosis
-      (diagnosis || {}).except("_replay_metadata", "_scenario_snapshot")
+      (diagnosis || {}).except("_replay_metadata", "_scenario_snapshot", "_judge_usage")
     end
 
     # A catalog can be refreshed without changing what an earlier run asked
@@ -55,7 +58,12 @@ module ActionAgent
       snapshot.is_a?(Hash) ? snapshot : scenario.as_json_summary.stringify_keys
     end
 
-    def as_json_summary
+    # The result for the JSON API. Its cost is the effective one — reported,
+    # else estimated — with `reported_cost`, `cost_source`, `cost_rate` and
+    # the judge's `judge_usage` on it (EvaluationRunCost). A caller listing
+    # a run's results hands in the run's breakdown so nothing is priced per row.
+    def as_json_summary(costs: nil)
+      costs ||= evaluation_run.cost_breakdown.result(self)
       {
         id: id,
         scenario_id: evaluation_scenario_id,
@@ -73,7 +81,11 @@ module ActionAgent
         duration_ms: duration_ms,
         input_tokens: input_tokens,
         output_tokens: output_tokens,
-        cost: cost&.to_f,
+        cost: costs["cost"],
+        reported_cost: costs["reported_cost"],
+        cost_source: costs["cost_source"],
+        cost_rate: costs["cost_rate"],
+        judge_usage: costs["judge_usage"],
         fault: fault,
         recommendation: recommendation,
         diagnosis: evaluation_diagnosis,

data/app/models/action_agent/model_pricing.rb

@@ -1,16 +1,40 @@
 # frozen_string_literal: true
 
+require "digest"
+
 module ActionAgent
   # Estimates LLM spend from token counts. The activeagent gem's telemetry
   # records tokens only; the platform layers pricing on top for the cost
-  # figures shown in Traces and Metrics.
+  # figures shown in Traces, Metrics and Evaluations.
   #
   # Rates come from RubyLLM's model registry (USD per million tokens,
-  # maintained upstream per model) when the model is known there; the static
-  # pattern table below is the fallback for aliases/self-hosted models, and
-  # a conservative blended rate covers everything else so totals stay
-  # meaningful. Costs are always presented as estimates.
+  # maintained upstream per model) when the model is known there — looked
+  # up under the provider it ran on when the caller knows it, since
+  # `gpt-5.5` through OpenRouter and `gpt-5.5` at OpenAI are two entries.
+  # The static tables below are the fallback for aliases, self-hosted
+  # models and an install without RubyLLM: exact rows (STATIC_RATES) for
+  # the current frontier models, then name patterns (PRICES), then a
+  # conservative blended rate so totals stay meaningful. Costs are always
+  # presented as estimates, and every rate says where it came from
+  # (`source`), so a figure worked out at a fallback rate can say so.
   class ModelPricing
+    # Exact rates for models the pattern table would otherwise misprice —
+    # claude-sonnet-5 is not a $3/$15 Sonnet, and the gpt-5 family is not
+    # one price — keyed by the name with its vendor prefix and date suffix
+    # removed and its dots and dashes normalised (see .normalize).
+    STATIC_RATES = {
+      "claude-sonnet-5" => [ 2.00, 10.00 ],
+      "gpt-5-5" => [ 5.00, 30.00 ],
+      "gpt-5-5-mini" => [ 0.25, 2.00 ],
+      "gpt-5-5-nano" => [ 0.05, 0.40 ],
+      "gpt-5-1" => [ 1.25, 10.00 ],
+      "gpt-5-1-mini" => [ 0.25, 2.00 ],
+      "gpt-5-1-nano" => [ 0.05, 0.40 ],
+      "gpt-5" => [ 1.25, 10.00 ],
+      "gpt-5-mini" => [ 0.25, 2.00 ],
+      "gpt-5-nano" => [ 0.05, 0.40 ]
+    }.freeze
+
     PRICES = [
       # [pattern, input $/1M, output $/1M]
       [ /gpt-4o-mini/i, 0.15, 0.60 ],
@@ -18,10 +42,15 @@ module ActionAgent
       [ /gpt-4\.1-nano/i, 0.10, 0.40 ],
       [ /gpt-4\.1-mini/i, 0.40, 1.60 ],
       [ /gpt-4\.1/i, 2.00, 8.00 ],
+      [ /gpt-5\.?5/i, 5.00, 30.00 ],
+      [ /gpt-5(\.\d+)?-nano/i, 0.05, 0.40 ],
+      [ /gpt-5(\.\d+)?-mini/i, 0.25, 2.00 ],
+      [ /gpt-5/i, 1.25, 10.00 ],
       [ /o3-mini|o4-mini/i, 1.10, 4.40 ],
       [ /claude.*(fable|mythos)/i, 10.00, 50.00 ],
       [ /claude.*haiku-?4/i, 1.00, 5.00 ],
       [ /claude.*(haiku)/i, 0.80, 4.00 ],
+      [ /claude.*sonnet-?5/i, 2.00, 10.00 ],
       [ /claude.*(sonnet)/i, 3.00, 15.00 ],
       [ /claude.*opus-(5|4-[5-9])/i, 5.00, 25.00 ],
       [ /claude.*(opus)/i, 15.00, 75.00 ],
@@ -36,45 +65,196 @@ module ActionAgent
     # Fallback blended rate for unknown models ($/1M input, $/1M output)
     DEFAULT_RATE = [ 1.00, 4.00 ].freeze
 
-    # @return [Float, nil] estimated USD cost, nil when there is nothing to price
-    def self.estimate(model:, input_tokens:, output_tokens:)
-      input = input_tokens.to_i
-      output = output_tokens.to_i
-      return nil if input.zero? && output.zero?
+    # Where a rate came from: RubyLLM's bundled catalog, a registry the host
+    # keeps itself (RubyLLM's model table or a refreshed listing), a row of
+    # STATIC_RATES or PRICES, or DEFAULT_RATE. The last two are a guess at
+    # the model's price rather than its listing.
+    SOURCES = %w[catalog remote pattern default].freeze
+    FALLBACK_SOURCES = %w[pattern default].freeze
 
-      input_rate, output_rate = rate_for(model)
-      ((input * input_rate) + (output * output_rate)) / 1_000_000.0
-    end
+    # A gateway's prefix on a model name, which names the provider rather
+    # than the model.
+    GATEWAY_PREFIXES = %w[openrouter requesty].freeze
+    # Vendor prefixes a gateway (or a host) puts before a model name, and
+    # the provider the bare name is listed under.
+    VENDOR_PROVIDERS = {
+      "openai" => "openai", "anthropic" => "anthropic", "google" => "gemini", "gemini" => "gemini",
+      "meta-llama" => nil, "mistralai" => "mistral", "mistral" => "mistral", "deepseek" => "deepseek",
+      "qwen" => nil, "x-ai" => "xai", "xai" => "xai", "cohere" => nil, "perplexity" => "perplexity"
+    }.freeze
 
-    def self.rate_for(model)
-      return DEFAULT_RATE if model.blank?
+    class << self
+      # @return [Float, nil] estimated USD cost, nil when there is nothing to price
+      def estimate(model:, input_tokens:, output_tokens:, provider: nil)
+        estimate_detailed(model: model, input_tokens: input_tokens, output_tokens: output_tokens, provider: provider)&.fetch(:cost)
+      end
 
-      registry_rate(model) || static_rate(model)
-    end
+      # The estimate with its working: `{ cost:, input_rate:, output_rate:,
+      # source: }`, rates in $ per million tokens. nil when both token
+      # counts are zero — a $0.00 is the caller's to decide on, since
+      # nothing generated and nothing recorded look alike here.
+      def estimate_detailed(model:, input_tokens:, output_tokens:, provider: nil)
+        input = input_tokens.to_i
+        output = output_tokens.to_i
+        return nil if input.zero? && output.zero?
+
+        rate = rate_detail(model, provider: provider)
+        {
+          cost: ((input * rate[:input]) + (output * rate[:output])) / 1_000_000.0,
+          input_rate: rate[:input],
+          output_rate: rate[:output],
+          source: rate[:source]
+        }
+      end
+
+      # @return [Array(Float, Float)] input and output rates in $ per million tokens
+      def rate_for(model, provider = nil)
+        rate = rate_detail(model, provider: provider)
+        [ rate[:input], rate[:output] ]
+      end
+
+      # `{ input:, output:, source: }` for a model under a provider, memoized
+      # per [provider, model]: the registry scan is not free and trace
+      # serialization asks per row.
+      def rate_detail(model, provider: nil)
+        return { input: DEFAULT_RATE[0], output: DEFAULT_RATE[1], source: "default" } if model.blank?
+
+        key = [ provider.to_s, model.to_s ]
+        @rates ||= {}
+        return @rates[key] if @rates.key?(key)
+
+        @rates[key] = registry_rate(model.to_s, provider.to_s.presence) || static_rate(model.to_s)
+      end
 
-    # Exact per-model rates from RubyLLM's registry. Lookups are memoized —
-    # the registry scan is not free and trace serialization calls this per
-    # row.
-    def self.registry_rate(model)
-      @registry_rates ||= {}
-      return @registry_rates[model] if @registry_rates.key?(model)
-
-      @registry_rates[model] = begin
-        info = RubyLLM.models.find(model.to_s)
-        tokens = info&.pricing&.text_tokens
-        if tokens&.input && tokens&.output
-          [ tokens.input, tokens.output ]
+      # Forgets memoized rates — after RubyLLM is loaded, or in tests.
+      def reset!
+        @rates = {}
+      end
+
+      # Names the rate tables in force, so a cache of figures worked out at
+      # them is invalidated when they change: the static rows, the default,
+      # and the version of the registry the install has (or none).
+      def fingerprint
+        registry = defined?(::RubyLLM) && ::RubyLLM.const_defined?(:VERSION) ? ::RubyLLM::VERSION.to_s : "none"
+        tables = [ STATIC_RATES, PRICES.map { |pattern, input, output| [ pattern.source, input, output ] }, DEFAULT_RATE, registry ]
+        Digest::SHA256.hexdigest(JSON.generate(tables))[0, 12]
+      end
+
+      # "openrouter/anthropic/claude-sonnet-4-5-20250929" → "claude-sonnet-4-5":
+      # the name without a gateway or vendor prefix, a date suffix or a
+      # colon-tagged size, with dots as dashes, so the same model under two
+      # spellings keys one row.
+      def normalize(model)
+        name = model.to_s.strip.downcase
+        name = name.split("/").last.to_s
+        name = name.sub(/-\d{8}\z/, "").sub(/-\d{4}-\d{2}-\d{2}\z/, "")
+        name.tr(".", "-")
+      end
+
+      private
+
+      # The registry's rate for the first spelling it knows: the name as
+      # given under the provider it ran on, then bare, then with a gateway
+      # or vendor prefix stripped (under the vendor's own provider), each
+      # with its dots and dashes swapped and its date suffix dropped.
+      def registry_rate(model, provider)
+        return nil unless registry_available?
+
+        lookup_candidates(model, provider).each do |id, candidate_provider|
+          info = registry_find(id, candidate_provider)
+          tokens = info&.pricing&.text_tokens
+          next unless tokens.respond_to?(:input) && tokens.input && tokens.output
+
+          return { input: tokens.input.to_f, output: tokens.output.to_f, source: registry_source }
         end
+        nil
       rescue StandardError
         nil
       end
-    end
 
-    def self.static_rate(model)
-      PRICES.each do |pattern, input_rate, output_rate|
-        return [ input_rate, output_rate ] if model.to_s.match?(pattern)
+      def registry_available?
+        defined?(::RubyLLM) && ::RubyLLM.respond_to?(:models)
+      end
+
+      # A host that keeps its own model registry (RubyLLM's model table or
+      # a refreshed listing) prices from it rather than from the gem's
+      # bundled catalog.
+      def registry_source
+        config = ::RubyLLM.respond_to?(:config) ? ::RubyLLM.config : nil
+        return "remote" if config.respond_to?(:model_registry_class) && config.model_registry_class.present?
+
+        "catalog"
+      end
+
+      # RubyLLM 2 takes the provider as a keyword; 1.x took it positionally.
+      def registry_find(id, provider)
+        models = ::RubyLLM.models
+        return models.find(id) if provider.blank?
+
+        if registry_find_keyword?(models)
+          models.find(id, provider: provider)
+        else
+          models.find(id, provider)
+        end
+      rescue StandardError
+        nil
+      end
+
+      def registry_find_keyword?(models)
+        models.method(:find).parameters.any? { |type, name| name == :provider && %i[key keyreq].include?(type) }
+      end
+
+      # [[id, provider], ...] to try in order, without repeats.
+      def lookup_candidates(model, provider)
+        name = model.strip
+        provider = provider&.downcase
+        candidates = []
+
+        head, rest = name.split("/", 2)
+        if rest.present? && GATEWAY_PREFIXES.include?(head.downcase)
+          provider ||= head.downcase
+          name = rest
+          head, rest = name.split("/", 2)
+        end
+
+        names = [ name ]
+        if rest.present? && VENDOR_PROVIDERS.key?(head.downcase)
+          names << rest
+          vendor_provider = VENDOR_PROVIDERS[head.downcase]
+        end
+
+        names.each do |candidate|
+          spellings(candidate).each do |spelling|
+            candidates << [ spelling, provider ] if provider
+            candidates << [ spelling, vendor_provider ] if vendor_provider && vendor_provider != provider && candidate == rest
+            candidates << [ spelling, nil ]
+          end
+        end
+        candidates.uniq
+      end
+
+      # The name as given, then with its date suffix dropped, each with its
+      # dots and dashes swapped: "claude-sonnet-4.5" and "claude-sonnet-4-5"
+      # are one model.
+      def spellings(name)
+        undated = name.sub(/-\d{8}\z/, "").sub(/-\d{4}-\d{2}-\d{2}\z/, "")
+        [ name, undated ].uniq.flat_map do |spelling|
+          [ spelling, spelling.gsub(/(\d)\.(\d)/, '\1-\2'), spelling.gsub(/(\d)-(\d)/, '\1.\2') ]
+        end.uniq
+      end
+
+      # An exact row for the normalised name, else the first pattern the
+      # name as given matches, else the default.
+      def static_rate(model)
+        if (exact = STATIC_RATES[normalize(model)])
+          return { input: exact[0], output: exact[1], source: "pattern" }
+        end
+
+        PRICES.each do |pattern, input_rate, output_rate|
+          return { input: input_rate, output: output_rate, source: "pattern" } if model.match?(pattern)
+        end
+        { input: DEFAULT_RATE[0], output: DEFAULT_RATE[1], source: "default" }
       end
-      DEFAULT_RATE
     end
   end
 end

data/app/models/action_agent/provider_key.rb

@@ -26,7 +26,7 @@ module ActionAgent
     HOST_PROVIDERS = %w[ollama].freeze
     # Tools connected with a credential, configured beside the providers
     # (Settings -> Integrations) but never offered to the agent builder.
-    CONNECTION_PROVIDERS = %w[claude_code].freeze
+    CONNECTION_PROVIDERS = %w[claude_code codex].freeze
     PROVIDERS = (KEY_PROVIDERS + HOST_PROVIDERS + CONNECTION_PROVIDERS).freeze
 
     # Only an Anthropic API key (sk-ant-api03-…, from the Claude Console or
@@ -38,6 +38,7 @@ module ActionAgent
     # ActionAgent.claude_code_auth = :local_login instead, where the
     # dashboard never touches the credential.
     CLAUDE_CODE_CREDENTIAL = /\Ask-ant-api\d{2}-[A-Za-z0-9_-]+\z/
+    CODEX_CREDENTIAL = /\Ask-(?!ant-|or-)[A-Za-z0-9_-]+\z/
     # A Claude subscription token, as earlier versions stored. Recognized so
     # a stored one is never handed out (see #needs_replacing?).
     SUBSCRIPTION_TOKEN_PREFIX = "sk-ant-oat"
@@ -66,6 +67,8 @@ module ActionAgent
         "third-party apps to hold Claude.ai credentials. To use your own Claude login on this machine, set " \
         "ActionAgent.claude_code_auth = :local_login with the :local sandbox backend instead"
     }, if: -> { provider == "claude_code" }
+    validates :credential, format: { with: CODEX_CREDENTIAL, message: "must be an OpenAI API key (sk-…)" },
+      if: -> { provider == "codex" }
     validates :api_key, length: { maximum: 500 }, allow_nil: true
 
     # Deletes every Claude Code connection that still holds a Claude
@@ -139,6 +142,9 @@ module ActionAgent
     #
     # @return [Hash{String => String}]
     def runtime_environment
+      if provider == "codex"
+        return CODEX_CREDENTIAL.match?(credential.to_s) ? { "CODEX_API_KEY" => credential } : {}
+      end
       return {} unless provider == "claude_code"
       return {} if needs_replacing? || !CLAUDE_CODE_CREDENTIAL.match?(credential.to_s)
 

data/app/models/action_agent/sandbox_session.rb

@@ -158,10 +158,11 @@ module ActionAgent
     # connected. Secret — for the backend, never a response.
     #
     # @return [Hash{String => String}]
-    def runtime_environment
+    def runtime_environment(runner: "claude_code")
       return {} unless app_runtime?
+      return {} unless ProviderKey::CONNECTION_PROVIDERS.include?(runner)
 
-      owners_record(ProviderKey.where(provider: "claude_code"))&.runtime_environment || {}
+      owners_record(ProviderKey.where(provider: runner))&.runtime_environment || {}
     end
 
     # Check if session is still valid