actionagent 1.7.2 → 1.8.1

data/app/services/action_agent/claude_code_auth.rb

@@ -0,0 +1,86 @@
+# frozen_string_literal: true
+
+module ActionAgent
+  # How Claude Code sessions authenticate (ActionAgent.claude_code_auth), and
+  # the one rule for whether an owner's Claude Code is "connected", shared by
+  # the sandbox listing, the assistant's configuration and the session API so
+  # they never disagree.
+  #
+  # :api_key     the owner's Anthropic API key, stored as a ProviderKey. A
+  #              Claude subscription token stored by an earlier version does
+  #              not count (ProviderKey#needs_replacing?).
+  # :local_login the login of the machine the dashboard runs on, as
+  #              `claude auth status` reports it (see
+  #              LocalSandboxBackend.claude_login_status). The dashboard never
+  #              sees that credential.
+  #
+  # Anthropic does not let third-party products store or route requests
+  # through Claude.ai subscription credentials, so there is no third mode:
+  # https://code.claude.com/docs/en/legal-and-compliance.md
+  module ClaudeCodeAuth
+    module_function
+
+    # @return [String] "api_key" or "local_login"
+    def mode
+      ActionAgent.claude_code_auth.to_s
+    end
+
+    def local_login?
+      mode == "local_login"
+    end
+
+    # Why +orchestrator+'s backend cannot run Claude Code sessions with the
+    # configured authentication, or nil. A machine's own login is the
+    # dashboard user's, so only a backend running sessions as that user, on
+    # that machine, may use it: a container or a remote host would either
+    # find no login or need it copied there, which is what this mode exists
+    # to never do.
+    def backend_refusal(orchestrator)
+      return unless local_login?
+      return if orchestrator.local?
+
+      "ActionAgent.claude_code_auth = :local_login uses this machine's own Claude Code login, so it works only " \
+        "with the :local sandbox backend, not #{orchestrator.backend_name}"
+    end
+
+    # Whether the owner of +provider_keys+ (a ProviderKey scope already
+    # narrowed to them) has an API key Claude Code can run on.
+    def api_key_connected?(provider_keys)
+      provider_keys.where(provider: "claude_code").any? { |key| key.runtime_environment.present? }
+    end
+
+    # What the dashboard reports about Claude Code for the owner of
+    # +provider_keys+: booleans and the login's method, never a credential.
+    #
+    # @return [Hash] { mode:, connected:, login: } (login for :local_login only)
+    def status(provider_keys)
+      if local_login?
+        # Only the :local backend uses the login, and no other one needs
+        # this machine's CLI asked about it.
+        login = local_backend? ? LocalSandboxBackend.claude_login_status : LocalSandboxBackend::LOGGED_OUT
+        { mode: mode, connected: login[:logged_in], login: login.slice(:logged_in, :auth_method) }
+      else
+        { mode: mode, connected: api_key_connected?(provider_keys) }
+      end
+    end
+
+    # A backend that cannot even be loaded is not the :local one.
+    def local_backend?
+      SandboxOrchestrator.new.local?
+    rescue StandardError, LoadError
+      false
+    end
+
+    # Why a Claude Code session cannot start in +sandbox+ for want of
+    # credentials, or nil.
+    def credential_refusal(sandbox)
+      if local_login?
+        return if LocalSandboxBackend.claude_login_status[:logged_in]
+
+        "Claude Code is not logged in on this machine: run `claude /login` as the user the dashboard runs as"
+      elsif sandbox.runtime_environment.blank?
+        "Claude Code is not connected: connect an Anthropic API key in Settings -> Integrations first"
+      end
+    end
+  end
+end

data/app/services/action_agent/codex_session_events.rb

@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+module ActionAgent
+  # Keep Codex's native JSONL transcript while adapting its terminal event
+  # to CodeSession's shared result contract. A process exit alone is never
+  # enough to mark a coding session successful.
+  class CodexSessionEvents
+    def consume(event)
+      return unless event.is_a?(Hash)
+
+      case event["type"]
+      when "thread.started"
+        @thread_id = event["thread_id"]
+      when "item.completed"
+        item = event["item"]
+        @answer = item["text"] if item.is_a?(Hash) && item["type"] == "agent_message"
+      when "turn.completed"
+        return result(false, @answer, event["usage"])
+      when "turn.failed", "error"
+        error = event["error"]
+        message = error.is_a?(Hash) ? error["message"] : event["message"]
+        return result(true, message.presence || "Codex reported a failed turn", {})
+      end
+      nil
+    end
+
+    private
+
+    def result(failed, answer, usage)
+      {
+        "type" => "result", "subtype" => failed ? "error_during_execution" : "success",
+        "is_error" => failed, "result" => answer, "session_id" => @thread_id,
+        "usage" => usage.is_a?(Hash) ? usage.slice("input_tokens", "output_tokens") : {}
+      }
+    end
+  end
+end

data/app/services/action_agent/dashboard_assistant_service.rb

@@ -31,7 +31,7 @@ module ActionAgent
     PROCESSING_DISCLOSURE = "The selected provider receives your current message, bounded conversation history, and authorized report excerpts requested through assistant tools."
     LIMITATIONS = [
       "Reports describe recorded behavior; no repository branch or current main has been verified.",
-      "GitHub connections, COI execution, and native Claude Code session connections are not implemented here.",
+      "This assistant cannot connect GitHub, start checkout sandboxes, or run Claude Code sessions: do that in Settings → Integrations. COI execution is not available.",
       "Agent proposals are drafts only. Review them in the builder before saving or running."
     ].freeze
     INSTRUCTIONS = <<~TEXT.freeze
@@ -55,9 +55,12 @@ module ActionAgent
       agents. These groups do not provide repository editing, arbitrary browser
       automation, or private database access. Instructions alone do not add tools
       or data access. Explain missing capabilities when proposing an agent.
-      GitHub auth, repo checkout, COI, native Claude Code sessions, running
-      evaluations, and publishing PRs are not available in this version. Explain
-      those limitations without suggesting fake authentication links.
+      You cannot connect GitHub, start checkout sandboxes, run Claude Code
+      sessions, run evaluations, or publish PRs. The developer connects GitHub
+      and Claude Code, starts checkout sandboxes, and runs Claude Code sessions
+      in Settings → Integrations; point them there instead of claiming a sandbox
+      or session started. COI execution is not available in this version.
+      Explain those limitations without suggesting fake authentication links.
     TEXT
     TOOL_DEFINITIONS = [
       {
@@ -107,7 +110,7 @@ module ActionAgent
         providers: DEFAULT_MODELS.map { |id, model| { id: id, configured: provider_configured?(id), default_model: model } },
         defaults: { provider: nil, model: nil },
         processing: { consent_required: true, disclosure: PROCESSING_DISCLOSURE },
-        connections: %i[github coi claude_code].index_with { { supported: false } },
+        connections: connections,
         limits: { message_characters: MAX_MESSAGE_CHARACTERS, history_messages: MAX_HISTORY_MESSAGES, history_characters: MAX_HISTORY_CHARACTERS },
         limitations: LIMITATIONS
       }
@@ -191,6 +194,45 @@ module ActionAgent
 
     private
 
+    # What the owner has set up in Settings → Integrations, reported by the
+    # assistant's configuration endpoint (booleans only, never a token). The
+    # model is not told: it starts nothing, and sandboxes and Claude Code
+    # sessions are started from that view (see LIMITATIONS).
+    #
+    # Claude Code is connected by ClaudeCodeAuth's rule, as the sandbox
+    # listing reports it: an API key the owner connected (auth "api_key"), or
+    # this machine's own login (auth "local_login", with its status).
+    def connections
+      claude_code = ClaudeCodeAuth.status(ProviderKey.for_owner(@owner))
+      {
+        github: { supported: true, connected: GithubConnection.for_owner(@owner).exists? },
+        claude_code: {
+          supported: code_sessions_supported?,
+          connected: claude_code[:connected],
+          auth: claude_code[:mode],
+          login: claude_code[:login]
+        }.compact,
+        coi: { supported: false }
+      }
+    end
+
+    # Claude Code needs a sandbox backend that runs sessions, which :mock and
+    # :local do and a host's may not. A backend that cannot even be loaded (a
+    # misspelled class in ActionAgent.sandbox_backends, or a class file that
+    # requires an SDK the host doesn't bundle) runs none, and must not take
+    # the assistant's configuration down with it. The SDK case raises
+    # LoadError, a ScriptError rather than a StandardError.
+    #
+    # Nor can one whose Claude Code authentication does not work there
+    # (ActionAgent.claude_code_auth = :local_login needs :local).
+    def code_sessions_supported?
+      orchestrator = SandboxOrchestrator.new
+      orchestrator.supports?(:code_session) && ClaudeCodeAuth.backend_refusal(orchestrator).nil?
+    rescue StandardError, LoadError => e
+      Rails.logger.warn("[ActionAgent] sandbox backend unavailable: #{e.message}")
+      false
+    end
+
     def require_processing_consent!
       return if @allow_provider_processing == true
 

data/app/services/action_agent/evaluation_report_import.rb

@@ -22,7 +22,12 @@ module ActionAgent
   #
   # The run's per-model summary, criterion scores and recommendations are
   # computed from the stored results, not taken from the report. The judge's
-  # verdict and label are taken from it.
+  # verdict and label are taken from it, and so is what the judge spent —
+  # per result (`result.judge_usage`, kept beside the result's diagnosis) and
+  # for the run (`report.judge_usage.run`) — and the release the report says
+  # it evaluated (`report.release`), which pins the run to that version of
+  # the agent (Agent#find_or_record_release!). A report naming no release
+  # leaves the run unrecorded rather than claiming the dashboard's latest.
   #
   # An identical retry returns the stored run, before anything is asked of the
   # `admit` callable. Different content under a run_id already stored raises
@@ -70,6 +75,11 @@ module ActionAgent
     IDENTIFIER_PATTERN = /\A[^[:cntrl:]]{1,200}\z/
     AGENT_NAME_PATTERN = /\A[^[:cntrl:]]{2,100}\z/
     TRACE_PATTERN = /\A[a-zA-Z0-9_-]{1,128}\z/
+    DIGEST_PATTERN = /\A[a-zA-Z0-9._-]{1,64}\z/
+    # The judge usage keys a report may send, per result and for the run.
+    JUDGE_USAGE_KEYS = %w[calls input_tokens output_tokens cost model by_kind source].freeze
+    JUDGE_USAGE_SOURCES = %w[reported traces meter].freeze
+    MAX_JUDGE_KINDS = 10
     SCOPE_PATTERN = %r{\A[\w .:/@-]{1,100}\z}
     STATUSES = %w[passed failed errored].freeze
     # Report metadata that tells one evaluation of a suite from another, in the
@@ -253,6 +263,7 @@ module ActionAgent
         external_tenant: tenant_key,
         external_run_id: @payload["run_id"],
         external_report_digest: digest,
+        agent_version: reported_release_version(agent),
         status: :complete,
         selection: selection,
         scores: recorded_scores,
@@ -267,6 +278,15 @@ module ActionAgent
       run
     end
 
+    # The agent version for the release the report names, or nil for a
+    # report that names none.
+    def reported_release_version(agent)
+      release = report["release"]
+      return nil unless release.is_a?(Hash) && release["digest"].present?
+
+      agent.find_or_record_release!(digest: release["digest"], revision: release["revision"].presence, label: release["label"].presence)
+    end
+
     # --- ownership -------------------------------------------------------------
 
     # Who a newly observed agent belongs to. Whatever the host's
@@ -492,11 +512,28 @@ module ActionAgent
             "key" => result["scenario_key"], "group" => result["group"], "prompt" => result["prompt"],
             "position" => scenario.position, "expectations" => {}
           }
-        ),
+        ).merge(bounded_judge_usage(result["judge_usage"]).then { |usage| usage ? { "_judge_usage" => usage } : {} }),
         error_message: truncated(result["error"], TEXT_BYTES)
       )
     end
 
+    # A judge usage as the engine stores it: the keys the dashboard reads,
+    # each cut to its type, or nil for none.
+    def bounded_judge_usage(usage)
+      return nil unless usage.is_a?(Hash)
+
+      by_kind = usage["by_kind"].is_a?(Hash) ? usage["by_kind"].first(MAX_JUDGE_KINDS).to_h { |kind, count| [ kind.to_s.first(40), count.to_i ] } : {}
+      {
+        "calls" => usage["calls"].to_i,
+        "input_tokens" => usage["input_tokens"].to_i,
+        "output_tokens" => usage["output_tokens"].to_i,
+        "cost" => usage["cost"].is_a?(Numeric) ? usage["cost"].to_f : nil,
+        "model" => usage["model"].is_a?(String) ? usage["model"].first(200).presence : nil,
+        "by_kind" => by_kind,
+        "source" => JUDGE_USAGE_SOURCES.include?(usage["source"]) ? usage["source"] : "reported"
+      }
+    end
+
     # The first +bytes+ bytes of +text+, dropping a character the cut splits,
     # or nil for blank text.
     def truncated(text, bytes)
@@ -521,14 +558,18 @@ module ActionAgent
       {
         "_verdict" => report["verdict"],
         "_selection" => selection,
-        "_metadata" => metadata
+        "_metadata" => metadata,
+        "_judge_usage_run" => bounded_judge_usage(report.dig("judge_usage", "run"))
       }.compact.merge("_judge_label" => judge_label)
     end
 
     # The run's scores in the shape the Evaluations view renders
-    # (ScenarioEvaluationRunner#scores_for), summarized from the stored results.
+    # (ScenarioEvaluationRunner#scores_for), summarized from the stored
+    # results as the application reported them — no cost is estimated here,
+    # so the summaries stored stay the application's own figures; the API
+    # adds the estimates when it serves the run (EvaluationRun#model_summaries).
     def summarized_scores(run)
-      rebuilt = run.to_report
+      rebuilt = run.to_report(estimate: false)
       rebuilt.criterion_scores.merge(
         "_models" => rebuilt.summary_by_model,
         "_recommendations" => rebuilt.recommendations
@@ -559,6 +600,9 @@ module ActionAgent
 
       judge_trace_ids!(metadata["judge_trace_ids"])
       string!(report["judge"], "report.judge", 200)
+      validate_release!
+      validate_judge_usage!(report.dig("judge_usage", "run"), "report.judge_usage.run") if report["judge_usage"].is_a?(Hash)
+      optional_object!(report["judge_usage"], "report.judge_usage")
       object!(report["models"], "report.models")
       raise Invalid, "report.models must contain 1-#{MAX_MODELS} models" unless report["models"].size.between?(1, MAX_MODELS)
       unless results.is_a?(Array) && results.size.between?(1, MAX_RESULTS)
@@ -605,6 +649,7 @@ module ActionAgent
         end
         trace_id!(result_metadata["trace_id"]) if result_metadata["trace_id"]
         judge_trace_ids!(result_metadata["judge_trace_ids"])
+        validate_judge_usage!(result["judge_usage"], "result.judge_usage")
       end
       distinct_specs = label_specs.values.uniq
       raise Invalid, "two model labels name the same provider/model" if distinct_specs.size < label_specs.size
@@ -655,6 +700,40 @@ module ActionAgent
       string!(judge.dig("suggested_tool", "name"), "result.diagnosis.judge.suggested_tool.name", 200)
     end
 
+    # The release the report says it evaluated: a digest the dashboard can
+    # match to a version cut on deploy, and the deploy and label as text.
+    def validate_release!
+      release = report["release"]
+      return if release.nil?
+
+      object!(release, "report.release")
+      unless release["digest"].nil? || (release["digest"].is_a?(String) && DIGEST_PATTERN.match?(release["digest"]))
+        raise Invalid, "report.release.digest must be 1-64 letters, digits or . _ -"
+      end
+
+      string!(release["revision"], "report.release.revision", 200)
+      string!(release["label"], "report.release.label", 200)
+    end
+
+    # What the judge spent, as ActiveAgent::Evals::Report writes it per
+    # result and for the run.
+    def validate_judge_usage!(usage, name)
+      return if usage.nil?
+
+      object!(usage, name)
+      %w[calls input_tokens output_tokens].each do |key|
+        next if usage[key].nil?
+        raise Invalid, "#{name}.#{key} must be a non-negative integer" unless usage[key].is_a?(Integer) && usage[key] >= 0 && usage[key] <= NUMERIC_LIMITS["input_tokens"]
+      end
+      numeric!(usage["cost"], "#{name}.cost", max: NUMERIC_LIMITS["cost"])
+      string!(usage["model"], "#{name}.model", 200)
+      string!(usage["source"], "#{name}.source", 20)
+      optional_object!(usage["by_kind"], "#{name}.by_kind")
+      (usage["by_kind"] || {}).each_value do |count|
+        raise Invalid, "#{name}.by_kind counts must be non-negative integers" unless count.is_a?(Integer) && count >= 0
+      end
+    end
+
     def validate_verdict!
       verdict = report["verdict"]
       return if verdict.nil?