actionagent 1.9.1 → 1.9.3

data/app/services/action_agent/dashboard_assistant_service.rb

@@ -206,11 +206,13 @@ module ActionAgent
     # listing reports it: an API key the owner connected (auth "api_key"),
     # this machine's own login (auth "local_login", with its status), or a
     # login inside each checkout sandbox (auth "sandbox_login", with the
-    # actor's per-sandbox state under sandboxes).
+    # actor's per-sandbox state under sandboxes). Codex by CodexAuth's: an
+    # API key, or this machine's own login.
     def connections
       claude_code = ClaudeCodeAuth.status(ProviderKey.for_owner(@owner),
         sandboxes: SandboxSession.for_owner(@owner).where(sandbox_type: "app_runtime").recent.limit(20),
         user_id: @actor.respond_to?(:id) ? @actor.id : nil)
+      codex = CodexAuth.status(ProviderKey.for_owner(@owner))
       {
         github: { supported: true, connected: GithubConnection.for_owner(@owner).exists? },
         claude_code: {
@@ -220,6 +222,12 @@ module ActionAgent
           login: claude_code[:login],
           sandboxes: claude_code[:sandboxes]
         }.compact,
+        codex: {
+          supported: codex_sessions_supported?,
+          connected: codex[:connected],
+          auth: codex[:mode],
+          login: codex[:login]
+        }.compact,
         coi: { supported: false }
       }
     end
@@ -241,6 +249,16 @@ module ActionAgent
       false
     end
 
+    # Codex needs a backend that advertises the runner (code_runners), and
+    # one its authentication works on (CodexAuth.backend_refusal).
+    def codex_sessions_supported?
+      orchestrator = SandboxOrchestrator.new
+      orchestrator.supports_code_runner?("codex") && CodexAuth.backend_refusal(orchestrator).nil?
+    rescue StandardError, LoadError => e
+      Rails.logger.warn("[ActionAgent] sandbox backend unavailable: #{e.message}")
+      false
+    end
+
     def require_processing_consent!
       return if @allow_provider_processing == true
 

data/app/services/action_agent/diff_stat.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+module ActionAgent
+  # Counts what a unified diff changes: { files:, additions:, deletions: }.
+  # A code session's diff runs from the checkout's base commit, so each turn
+  # of a conversation reports everything changed since the checkout, up to
+  # and including that turn.
+  module DiffStat
+    module_function
+
+    # @param diff [String, nil]
+    # @return [Hash, nil] nil when there is no diff
+    def call(diff)
+      return nil if diff.blank?
+
+      files = additions = deletions = 0
+      diff.each_line do |line|
+        if line.start_with?("diff --git ") then files += 1
+        elsif line.start_with?("+++ ", "--- ") then next
+        elsif line.start_with?("+") then additions += 1
+        elsif line.start_with?("-") then deletions += 1
+        end
+      end
+      { files: files, additions: additions, deletions: deletions }
+    end
+  end
+end

data/app/services/action_agent/evaluation_fix_comparison.rb

@@ -6,8 +6,11 @@ module ActionAgent
       @session = session
     end
 
+    # +after+ is pending while the verification run is in progress and, with
+    # a fanned-out run, while its stored replay waits to be scored: not a
+    # result yet, so not a regression.
     def self.change(before, after)
-      return "pending" unless after
+      return "pending" if after.nil? || after.pending?
       return after.passed? ? "added_pass" : "added_fail" unless before
       return after.passed? ? "unchanged_pass" : "regressed" if before.passed?
       after.passed? ? "fixed" : "still_failing"

data/app/services/action_agent/evaluation_report_export.rb

@@ -0,0 +1,175 @@
+# frozen_string_literal: true
+
+require "digest"
+require "json"
+
+module ActionAgent
+  # A complete scenario run as the version-1 envelope
+  # ActiveAgent::Evals::Publisher sends, built with Publisher.envelope so the
+  # two share one shape. What the export endpoints and
+  # `action_agent:evaluations:export` serve: POSTed to another install's
+  # <mount>/api/evaluation_reports it is stored there, and POSTed back to this
+  # one it is an identical retry (EvaluationReportImport accepts a run's own
+  # export as identical to the run).
+  #
+  # The identities come from the run:
+  #   run_id     — an imported run's own run_id, exactly as published. A run
+  #                the dashboard executed has none, so it is given
+  #                "dashboard-run-<id>-<created_at as YYYYMMDDhhmmss UTC>":
+  #                stable for the run, and unlikely to meet another
+  #                install's run of the same id where both are imported.
+  #   source     — an imported run's source; for a dashboard run the agent's
+  #                service, else the host application's name ("support-hub").
+  #   agent_name — the agent's class name, else its name.
+  #   suite      — an imported run's suite; for a dashboard run the
+  #                evaluation's name.
+  #
+  # The report is EvaluationRun#to_report(estimate: false): what the run
+  # recorded, with no cost estimated, so the same run always exports the
+  # same envelope. Its metadata leaves out the run's database id.
+  class EvaluationReportExport
+    # The run cannot be exported: it is not complete, or its evaluation is
+    # not a scenario suite.
+    class NotExportable < StandardError; end
+    # #markdown was asked for a part the report does not have.
+    class UnknownPart < ArgumentError; end
+
+    DASHBOARD_RUN_ID = /\Adashboard-run-(\d+)-(\d{14})\z/
+    STAMP_FORMAT = "%Y%m%d%H%M%S"
+
+    # The run_id a run exports under.
+    def self.run_id_for(run)
+      run.external_run_id.presence || "dashboard-run-#{run.id}-#{run.created_at.utc.strftime(STAMP_FORMAT)}"
+    end
+
+    # The run +run_id+ names for the tenant +account+ (nil on a single-tenant
+    # install), or nil. An imported run is found by the run_id it was
+    # published under, within the tenant that published it, as
+    # EvaluationReportImport finds it. A dashboard run is found by its
+    # exported run_id, when its agent is one of the tenant's
+    # (ActionAgent.agents_for); a single-tenant key reaches every agent, as
+    # the import's does.
+    def self.find(run_id, account: nil)
+      return nil unless run_id.is_a?(String) && run_id.present?
+
+      imported = EvaluationRun.find_by(external_tenant: account ? account.id.to_s : "", external_run_id: run_id)
+      return imported if imported
+
+      match = DASHBOARD_RUN_ID.match(run_id)
+      return nil unless match
+
+      run = EvaluationRun.where(external_run_id: nil).find_by(id: match[1].to_i)
+      return nil unless run && run_id_for(run) == run_id
+      return run if account.nil? && !ActionAgent.multi_tenant?
+
+      run if account && ActionAgent.agents_for(account).exists?(id: run.evaluation.agent_id)
+    end
+
+    attr_reader :run
+
+    def initialize(run)
+      @run = run
+    end
+
+    def run_id
+      self.class.run_id_for(run)
+    end
+
+    # Raises NotExportable for a run that is not a complete scenario run:
+    # what every format of the export, not only the envelope, needs.
+    def exportable!
+      raise NotExportable, "Only a complete run can be exported" unless run.complete?
+      raise NotExportable, "Only a scenario suite's run can be exported" unless evaluation.scenario_suite?
+    end
+
+    # Raises NotExportable for a run that is not a complete scenario run.
+    def envelope
+      @envelope ||= begin
+        exportable!
+
+        built = ActiveAgent::Evals::Publisher.envelope(
+          report: run.to_report(links: {}, estimate: false),
+          run_id: run_id, source: source, agent_name: agent_name, suite: suite
+        )
+        # As JSON reads it back, so the digest is the one a POST of it gets.
+        JSON.parse(JSON.generate(built)).tap do |parsed|
+          # The run's database id means nothing on another install; run_id
+          # is the run's identity there.
+          metadata = parsed["report"]["metadata"]
+          metadata.delete("run") if metadata.is_a?(Hash) && metadata["run"] == run.id
+        end
+      end
+    end
+
+    def to_json(*)
+      JSON.pretty_generate(envelope)
+    end
+
+    # Parts of the report #markdown serves on its own.
+    MARKDOWN_PARTS = %w[report verdict fix_items scenario].freeze
+
+    # The run's report as GitHub-flavored Markdown
+    # (ActiveAgent::Evals::ReportMarkdown), with the costs the dashboard shows:
+    # the whole report, or one +part+ — "verdict" (with the model
+    # comparison), "fix_items" (for +model+, a model label, when given) or
+    # "scenario" (the one +scenario+ names). Raises UnknownPart for an
+    # unknown part and ActiveRecord::RecordNotFound for a scenario the run
+    # has no result for.
+    def markdown(part: nil, model: nil, scenario: nil)
+      report = run.to_report(links: {})
+      case part.presence || "report"
+      when "report" then report.to_markdown
+      when "verdict" then report.verdict_markdown
+      when "fix_items" then report.fix_items_markdown(model: model.presence)
+      when "scenario" then scenario_markdown(report, scenario.to_s)
+      else raise UnknownPart, "part is one of #{MARKDOWN_PARTS.join(', ')}"
+      end
+    end
+
+    # The digest EvaluationReportImport records for this envelope.
+    def digest
+      EvaluationReportImport.digest(envelope)
+    end
+
+    # "orders-eu-run-3.json" for the third run of "orders (eu)"; a part of
+    # the Markdown report adds its name: "orders-eu-run-3-fix-items.md".
+    def filename(extension, number: EvaluationSerializer.run_number(evaluation, run), part: nil)
+      base = "#{evaluation.name.to_s.parameterize.presence || "evaluation-#{evaluation.id}"}-run-#{number}"
+      base += "-#{part.to_s.dasherize.parameterize}" if part.present? && part.to_s != "report"
+      "#{base}.#{extension}"
+    end
+
+    private
+
+    def scenario_markdown(report, key)
+      raise ActiveRecord::RecordNotFound, "The run has no result for scenario #{key}" unless report.results.any? { |result| result.scenario.key == key }
+
+      report.scenario_markdown(key)
+    end
+
+    def evaluation
+      run.evaluation
+    end
+
+    def agent
+      evaluation.agent
+    end
+
+    def external
+      value = evaluation.config.to_h["external"]
+      run.imported? && value.is_a?(Hash) ? value : {}
+    end
+
+    def source
+      (external["source"].presence || agent.service_name.presence || Rails.application.class.module_parent_name.underscore.dasherize).first(200)
+    end
+
+    def agent_name
+      (agent.agent_class_name.presence || agent.name).to_s.first(100)
+    end
+
+    def suite
+      (external["suite"].presence || evaluation.name).to_s.first(200)
+    end
+  end
+end

data/app/services/action_agent/evaluation_report_import.rb

@@ -30,11 +30,14 @@ module ActionAgent
   # leaves the run unrecorded rather than claiming the dashboard's latest.
   #
   # An identical retry returns the stored run, before anything is asked of the
-  # `admit` callable. Different content under a run_id already stored raises
-  # Conflict. Imports for one owner run one at a time under an advisory lock
-  # (OwnerLock), so concurrent first deliveries find or create one agent,
-  # evaluation and scenario set, and the unique index on the run's tenant and
-  # run_id settles any delivery the lock does not serialize.
+  # `admit` callable. So does a run's own export (EvaluationReportExport)
+  # posted back to the install and tenant that hold the run, whether the run
+  # was imported or executed by the dashboard. Different content under a
+  # run_id already stored raises Conflict. Imports for one owner run one at a
+  # time under an advisory lock (OwnerLock), so concurrent first deliveries
+  # find or create one agent, evaluation and scenario set, and the unique
+  # index on the run's tenant and run_id settles any delivery the lock does
+  # not serialize.
   class EvaluationReportImport
     class Error < StandardError; end
     # The payload is not a report the engine can store as sent; 422.
@@ -119,6 +122,21 @@ module ActionAgent
       new(payload, account, admit).call
     end
 
+    # The digest a run records for the payload it was stored from: the
+    # SHA-256 of the payload's JSON with every object's keys sorted.
+    def self.digest(payload)
+      Digest::SHA256.hexdigest(JSON.generate(canonical(payload)))
+    end
+
+    def self.canonical(value)
+      case value
+      when Hash then value.keys.sort.to_h { |key| [ key, canonical(value[key]) ] }
+      when Array then value.map { |child| canonical(child) }
+      else value
+      end
+    end
+    private_class_method :canonical
+
     # Why this install cannot store reports, or nil when it can: an install
     # with no evaluation tables (generated with --traces-only), or one whose
     # evaluation runs lack a column a later migration adds.
@@ -142,7 +160,7 @@ module ActionAgent
 
     def call
       validate!
-      digest = Digest::SHA256.hexdigest(JSON.generate(canonical(@payload)))
+      digest = self.class.digest(@payload)
       existing = stored_run
       return [ identical!(existing, digest), true ] if existing
 
@@ -240,14 +258,27 @@ module ActionAgent
       @account ? @account.id.to_s : ""
     end
 
+    # The run this run_id already names for the tenant: one stored from an
+    # earlier delivery, or one this install exported (a dashboard run's
+    # exported run_id), found the way the export endpoint finds it.
     def stored_run
-      EvaluationRun.find_by(external_tenant: tenant_key, external_run_id: @payload["run_id"])
+      EvaluationReportExport.find(@payload["run_id"], account: @account)
     end
 
+    # The stored run, when the payload is the report it was stored from or
+    # the run's own export (EvaluationReportExport), which is how an
+    # exported run posted back to its install is told from a different
+    # report under the same run_id.
     def identical!(run, digest)
-      raise Conflict, "run_id already exists with a different report" unless run.external_report_digest == digest
+      return run if run.external_report_digest == digest || exported_digest(run) == digest
 
-      run
+      raise Conflict, "run_id already exists with a different report"
+    end
+
+    def exported_digest(run)
+      EvaluationReportExport.new(run).digest
+    rescue EvaluationReportExport::NotExportable
+      nil
     end
 
     def admit!
@@ -568,12 +599,17 @@ module ActionAgent
     # results as the application reported them — no cost is estimated here,
     # so the summaries stored stay the application's own figures; the API
     # adds the estimates when it serves the run (EvaluationRun#model_summaries).
+    #
+    # The comparison is worked out here from the stored results too, rather
+    # than taken from the payload, like the summaries.
     def summarized_scores(run)
       rebuilt = run.to_report(estimate: false)
+      comparison = rebuilt.comparison if rebuilt.respond_to?(:comparison)
       rebuilt.criterion_scores.merge(
         "_models" => rebuilt.summary_by_model,
-        "_recommendations" => rebuilt.recommendations
-      ).merge(recorded_scores)
+        "_recommendations" => rebuilt.recommendations,
+        "_comparison" => comparison
+      ).compact.merge(recorded_scores)
     end
 
     # --- validation ------------------------------------------------------------
@@ -810,13 +846,5 @@ module ActionAgent
       else value
       end
     end
-
-    def canonical(value)
-      case value
-      when Hash then value.keys.sort.to_h { |key| [ key, canonical(value[key]) ] }
-      when Array then value.map { |child| canonical(child) }
-      else value
-      end
-    end
   end
 end

data/app/services/action_agent/evaluation_runner_service.rb

@@ -10,6 +10,8 @@ module ActionAgent
   # credentials and is skipped — never faked — when none are available.
   class EvaluationRunnerService
     PASS_THRESHOLD = 0.7
+    # The class name the judge runs as, which its traces and the run record.
+    JUDGE_AGENT_NAME = "EvaluationJudgeAgent"
 
     def self.call(evaluation)
       new(evaluation).call
@@ -31,6 +33,10 @@ module ActionAgent
     def call
       run = @evaluation.evaluation_runs.create!(status: :running)
 
+      # A pin whose provider has no credentials fails here, before any work.
+      # A rules-only run never asks the judge, so a stale pin does not stop it.
+      judge_provider if @evaluation.judge_model.present? && judge_needed?
+
       # judge_defined evaluations author their KPI criteria on first run.
       ensure_judge_defined_kpis! if @evaluation.judge_defined?
 
@@ -66,6 +72,7 @@ module ActionAgent
 
       scores["_cohorts"] = cohort_summaries(samples, per_sample_scores) if samples.any?
       scores["_judge_usage"] = judge_usage if judge_usage
+      scores["_judge"] = judge_descriptor if judge_usage
 
       run.update!(
         status: :complete,
@@ -80,19 +87,52 @@ module ActionAgent
       raise
     end
 
-    # Returns the provider the judge runs on: the first of Anthropic, OpenAI
-    # and OpenRouter the owner or the host's config has a key for, else Ollama
-    # when the owner configured a host; nil when none. The judge runs
-    # `judge_model` as that provider's own model id.
+    # Unpinned judges keep the configured default. A pinned model uses the
+    # existing ModelSpec syntax, so adding an unrelated key cannot reroute it.
     def judge_provider
-      @judge_provider ||=
-        %i[anthropic openai openrouter].find do |name|
-          owner_provider_options(name).any? || global_provider_token?(name)
-        end || (:ollama if owner_provider_options(:ollama).any?)
+      @judge_provider ||= begin
+        raw = @evaluation&.judge_model.presence
+        spec = raw && ActiveAgent::Evals::ModelSpec.parse(raw, default_provider: nil)
+        if spec&.provider.present?
+          provider = spec.provider.to_sym
+          if judge_provider_configured?(provider)
+            provider
+          elsif default_judge_provider
+            raise ArgumentError,
+              "No credentials for judge provider #{provider} (#{raw}); configure that provider or pin a provider-qualified gateway model ID"
+          end
+        else
+          default_judge_provider
+        end
+      end
     end
 
     private
 
+    # Whether a sample-based run asks the judge: to author KPIs, to score an
+    # llm_judge criterion, or for a comparison's verdict.
+    def judge_needed?
+      @evaluation.judge_defined? || @evaluation.llm_criteria.any? || @evaluation.compare_models.any?
+    end
+
+    def default_judge_provider
+      %i[anthropic openai openrouter ollama].find { |name| judge_provider_configured?(name) }
+    end
+
+    def judge_provider_configured?(name)
+      owner_provider_options(name).any? || (name != :ollama && global_provider_token?(name))
+    end
+
+    # A provider qualifier selects the route, not a prefix to send to the
+    # provider. Vendor prefixes inside an explicit gateway ID are retained.
+    def judge_pinned_model(provider = judge_provider)
+      raw = @evaluation&.judge_model.presence
+      return unless raw
+
+      spec = ActiveAgent::Evals::ModelSpec.parse(raw, default_provider: provider)
+      spec.provider.to_sym == provider ? spec.model : raw
+    end
+
     # Scores each sample-based criterion once per candidate model cohort and
     # asks the judge for a comparative verdict — "haiku vs qwen, judged".
     def score_comparison(run, sample_criteria, telemetry_criteria)
@@ -136,6 +176,7 @@ module ActionAgent
         [ model, cohort_summary(samples, per_model_sample_scores[model] || {}) ]
       end
       scores["_judge_usage"] = judge_usage if judge_usage
+      scores["_judge"] = judge_descriptor if judge_usage
 
       run.update!(
         status: :complete,
@@ -236,7 +277,8 @@ module ActionAgent
     # first. Persisted as scores["_judge_usage"]:
     #
     #   { "calls", "input_tokens", "output_tokens", "cost", "model",
-    #     "by_kind" => { "score" => n, "recommend" => n, "verdict" => n, "define" => n } }
+    #     "by_kind" => { "score" => n, "recommend" => n, "verdict" => n, "define" => n },
+    #     "models" => { "<model that answered>" => n } }
     #
     # nil until the judge has been asked something, so a rules-only run
     # records no judge at all rather than a judge that cost nothing.
@@ -263,13 +305,42 @@ module ActionAgent
       model = (response.respond_to?(:model) && response.model.presence) || @evaluation.judge_model.presence
       cost = ModelPricing.estimate(model: model, provider: judge_provider, input_tokens: input_tokens, output_tokens: output_tokens)
 
-      @judge_usage ||= { "calls" => 0, "input_tokens" => 0, "output_tokens" => 0, "cost" => nil, "model" => nil, "by_kind" => {} }
+      @judge_usage ||= { "calls" => 0, "input_tokens" => 0, "output_tokens" => 0, "cost" => nil, "model" => nil, "by_kind" => {}, "models" => {} }
       @judge_usage["calls"] += 1
       @judge_usage["input_tokens"] += input_tokens
       @judge_usage["output_tokens"] += output_tokens
       @judge_usage["cost"] = (@judge_usage["cost"] || 0.0) + cost if cost
       @judge_usage["model"] ||= model
       @judge_usage["by_kind"][kind.to_s] = @judge_usage["by_kind"].fetch(kind.to_s, 0) + 1
+      (@judge_usage["models"] ||= {})[model] = @judge_usage["models"].fetch(model, 0) + 1 if model
+    end
+
+    # The judge as a run records it (scores["_judge"]): the agent class it
+    # runs as, its provider, the model it asks for and where that setting
+    # came from, and the models that answered. A judge with no model pinned
+    # runs on its provider's default, and OpenRouter's (openrouter/auto) is a
+    # router that can answer each call with a different model, so "answered"
+    # is what actually judged. nil when no provider has credentials.
+    def judge_descriptor
+      return nil unless judge_provider
+
+      {
+        "agent" => JUDGE_AGENT_NAME,
+        "provider" => judge_provider.to_s,
+        "model" => judge_requested_model,
+        "model_source" => judge_model_source,
+        "answered" => judge_usage&.dig("models").to_h
+      }
+    end
+
+    # How a run names its judge: the model it asked for, else the one model
+    # that answered, else the provider's default.
+    def judge_display_label
+      return nil unless judge_provider
+      return judge_requested_model if judge_requested_model
+
+      answered = judge_usage&.dig("models").to_h.keys
+      answered.one? ? answered.first : "#{judge_provider} default"
     end
 
     # Every token the judge was billed for reading. Anthropic reports the
@@ -602,6 +673,28 @@ module ActionAgent
       @owner_provider_options[name.to_s] ||= ProviderCredentials.resolve(owner: owner, actor: nil, provider: name).options
     end
 
+    # The model the judge asks for, in the order judge_class applies them:
+    # the evaluation's judge_model, the owner's provider key, the host's
+    # config/active_agent.yml. nil leaves the choice to the provider.
+    def judge_requested_model
+      judge_model_setting&.last
+    end
+
+    def judge_model_source
+      judge_model_setting&.first || "provider_default"
+    end
+
+    def judge_model_setting
+      return @judge_model_setting if defined?(@judge_model_setting)
+
+      config = ActiveAgent.configuration[judge_provider]
+      @judge_model_setting = {
+        "evaluation" => judge_pinned_model,
+        "provider_key" => owner_provider_options(judge_provider)[:model],
+        "config" => (config[:model] if config.respond_to?(:[]))
+      }.find { |_source, model| model.present? }&.then { |source, model| [ source, model.to_s ] }
+    end
+
     def owner
       @owner ||= @evaluation&.agent&.owner
     end
@@ -609,13 +702,12 @@ module ActionAgent
     def judge_class
       @judge_class ||= begin
         provider = judge_provider
-        model = @evaluation.judge_model.presence
-        options = {}
+        model = judge_pinned_model(provider)
+        options = owner_provider_options(provider).dup
         options[:model] = model if model
-        options.merge!(owner_provider_options(provider))
 
         Class.new(ActiveAgent::Base) do
-          define_singleton_method(:name) { "EvaluationJudgeAgent" }
+          define_singleton_method(:name) { JUDGE_AGENT_NAME }
           generate_with provider, **options
         end
       end