actionagent 1.7.2 → 1.8.1

data/app/services/action_agent/sandbox_manifest.rb

@@ -0,0 +1,67 @@
+# frozen_string_literal: true
+
+module ActionAgent
+  # The runtime manifest: how a booted checkout tells the sandbox backend
+  # where its MCP facade answers and which bearer token opens it.
+  #
+  #   { "mcp_path": "/activeagents/mcp", "mcp_token": "aa_..." }
+  #
+  # The checked-out app writes it with `bin/rails action_agent:sandbox:manifest`
+  # (it mounts this engine, so the task ships with it), and a backend reads it
+  # back with .parse. Both halves live here so they cannot drift.
+  module SandboxManifest
+    # Where the manifest task writes, when set; stdout otherwise.
+    PATH_ENV = "ACTION_AGENT_SANDBOX_MANIFEST"
+    # The dashboard API key the manifest hands out, created once per checkout.
+    KEY_NAME = "Checkout sandbox runtime"
+
+    class Error < StandardError; end
+
+    module_function
+
+    # Builds the manifest inside the booted app: the engine's MCP path under
+    # wherever the app mounts it, and an API key for the MCP facade.
+    #
+    # @param routes [ActionDispatch::Routing::RouteSet] the app's routes
+    # @return [Hash{String => String}]
+    def generate(routes: Rails.application.routes)
+      # Rails 8 loads routes lazily in development and test; the mount is
+      # invisible until they are.
+      Rails.application.reload_routes_unless_loaded if Rails.application.respond_to?(:reload_routes_unless_loaded)
+
+      mount = ActiveAgent::Telemetry::Configuration.new.mount_path_in(routes)
+      raise Error, "ActionAgent::Engine is not mounted in this app's routes" if mount.nil?
+
+      { "mcp_path" => "#{mount}/mcp", "mcp_token" => api_key.token }
+    end
+
+    # The checkout's own dashboard API key for the facade. Reused across
+    # runs so a re-run manifest does not mint a key per boot. A host that
+    # owns keys by account or user gets a key with no owner, which reaches
+    # no agents over MCP — the manifest says so rather than failing.
+    def api_key
+      ActionAgent::ApiKey.find_or_create_by!(name: KEY_NAME)
+    end
+
+    # Reads a manifest a checkout wrote.
+    #
+    # @param json [String]
+    # @return [Hash{String => String}] with "mcp_path" and "mcp_token"
+    def parse(json)
+      data = JSON.parse(json.to_s)
+      raise Error, "the manifest is not a JSON object" unless data.is_a?(Hash)
+
+      path = data["mcp_path"]
+      unless path.is_a?(String) && path.start_with?("/")
+        raise Error, "the manifest names no mcp_path (expected a path such as /activeagents/mcp)"
+      end
+
+      token = data["mcp_token"]
+      raise Error, "the manifest's mcp_token is not a string" unless token.nil? || token.is_a?(String)
+
+      { "mcp_path" => path, "mcp_token" => token }
+    rescue JSON::ParserError => e
+      raise Error, "the manifest is not JSON (#{e.message.truncate(120)})"
+    end
+  end
+end

data/app/services/action_agent/sandbox_orchestrator.rb

@@ -3,16 +3,14 @@
 module ActionAgent
   # SandboxOrchestrator
   #
-  # Unified interface for managing agent sandbox sessions.
-  # Supports multiple backends for cloud-agnostic deployment:
-  #
-  #   - incus:     Self-hosted Incus containers (any Linux host)
-  #   - cloud_run: Google Cloud Run Jobs (serverless)
-  #   - kubernetes: Kubernetes pods (GKE, EKS, self-hosted k8s)
+  # Unified interface for managing agent sandbox sessions. The engine ships
+  # two backends — :mock (in-memory, runs nothing) and :local (checkouts as
+  # child processes of the dashboard) — and a host registers the rest
+  # (Incus, Cloud Run, Kubernetes) in ActionAgent.sandbox_backends.
   #
   # Configuration:
-  #   Set SANDBOX_BACKEND environment variable to choose backend.
-  #   Default: "incus" for simplicity
+  #   ActionAgent.sandbox_service, or the SANDBOX_BACKEND environment
+  #   variable, which wins. Default: :mock.
   #
   # Usage:
   #   orchestrator = SandboxOrchestrator.new
@@ -21,14 +19,21 @@ module ActionAgent
   #   orchestrator.terminate(container_id)
   #
   class SandboxOrchestrator
-    # The engine ships only the in-memory backend. Anything that talks to
-    # real infrastructure (Incus, Kubernetes, Cloud Run) is registered by
-    # the app that operates it, so the engine carries none of those SDKs:
+    # Anything that talks to real infrastructure (Incus, Kubernetes, Cloud
+    # Run) is registered by the app that operates it, so the engine carries
+    # none of those SDKs:
     #
     #   ActionAgent.sandbox_backends = {
     #     "cloud_run" => "CloudRunService"
     #   }
-    BUILT_IN_BACKENDS = { "mock" => "ActionAgent::MockSandboxBackend" }.freeze
+    #
+    # The engine also ships :local, which boots app_runtime checkouts as
+    # child processes of the dashboard itself (see LocalSandboxBackend and
+    # ActionAgent.local_sandboxes_enabled?).
+    BUILT_IN_BACKENDS = {
+      "mock" => "ActionAgent::MockSandboxBackend",
+      "local" => "ActionAgent::LocalSandboxBackend"
+    }.freeze
 
     # Backends disagree on what to call each verb. Candidates are tried in
     # order and the first the backend responds to wins, so a host-registered
@@ -38,7 +43,11 @@ module ActionAgent
       status: %i[status container_status pod_status job_status],
       terminate: %i[terminate terminate_pod cancel_job],
       list: %i[list_sandboxes list_sandbox_pods list_jobs],
-      cleanup: %i[cleanup_expired cleanup_expired_pods cleanup_expired_jobs]
+      cleanup: %i[cleanup_expired cleanup_expired_pods cleanup_expired_jobs],
+      # Claude Code sessions inside an app_runtime checkout. Optional: a
+      # backend without them simply cannot run sessions (see #supports?).
+      code_session: %i[run_code_session],
+      cancel_code_session: %i[cancel_code_session]
     }.freeze
 
     class UnsupportedBackendError < StandardError; end
@@ -101,7 +110,12 @@ module ActionAgent
         instance_tier: result[:instance_tier] || tier&.id,
         resources: result[:resources],
         hourly_cost: result[:hourly_cost] || tier&.hourly_cost&.to_f,
-        created_at: result[:created_at] || Time.current
+        created_at: result[:created_at] || Time.current,
+        # An app_runtime sandbox's backend clones sandbox_session.checkout_spec,
+        # boots the app, and reports where its MCP facade answers (and the
+        # bearer token it expects) so agents can use the checkout's tools.
+        mcp_url: result[:mcp_url],
+        mcp_token: result[:mcp_token]
       }
     end
 
@@ -153,6 +167,60 @@ module ActionAgent
       @backend.public_send(adapter_method(:cleanup))
     end
 
+    # The handle the backend would give +sandbox_session+'s sandbox, for a
+    # backend that derives it from the session (nil otherwise).
+    def handle_for(sandbox_session)
+      @backend.respond_to?(:handle_for) ? @backend.handle_for(sandbox_session) : nil
+    end
+
+    # Whether the backend can name a session's sandbox without a recorded
+    # handle (see #handle_for).
+    def derives_handles?
+      @backend.respond_to?(:handle_for)
+    end
+
+    # Whether the backend runs sandboxes as processes of the dashboard, on
+    # its own machine and as its own user (LocalSandboxBackend): the only
+    # place ActionAgent.claude_code_auth = :local_login can work.
+    def local?
+      @backend.is_a?(LocalSandboxBackend)
+    end
+
+    # Whether the backend implements +verb+ (an ADAPTER_METHODS key).
+    def supports?(verb)
+      ADAPTER_METHODS.fetch(verb).any? { |m| @backend.respond_to?(m) }
+    end
+
+    # Existing backends predate runner selection and support Claude only.
+    # A backend must explicitly advertise Codex before accepting its keys.
+    def supports_code_runner?(runner)
+      return false unless supports?(:code_session)
+
+      runners = @backend.respond_to?(:code_runners) ? @backend.code_runners : [ "claude_code" ]
+      Array(runners).include?(runner)
+    end
+
+    # Runs a Claude Code session in +sandbox_session+'s checkout, yielding
+    # each stream-json event (a Hash) as it arrives. Returns the backend's
+    # outcome: { exit_status:, diff: }.
+    def run_code_session(sandbox_session, code_session, &on_event)
+      # Checked when a session is requested too; this covers one queued
+      # before the configuration changed.
+      runner = code_session.try(:runner) || "claude_code"
+      unless supports_code_runner?(runner)
+        raise UnsupportedBackendError, "The #{backend_name} sandbox backend cannot run #{runner} sessions"
+      end
+      refusal = ClaudeCodeAuth.backend_refusal(self) unless runner == "codex"
+      raise UnsupportedBackendError, refusal if refusal
+
+      @backend.public_send(adapter_method(:code_session), sandbox_session, code_session, &on_event)
+    end
+
+    # Stops a running Claude Code session.
+    def cancel_code_session(sandbox_session, code_session)
+      @backend.public_send(adapter_method(:cancel_code_session), sandbox_session, code_session)
+    end
+
     # Check if the backend is healthy
     #
     # @return [Boolean] true if backend is reachable

data/app/services/action_agent/scenario_evaluation_runner.rb

@@ -17,7 +17,16 @@ module ActionAgent
   #   "_models"          — per model: pass rate, mean score, latency, tokens, cost, fault counts
   #   "_recommendations" — faults grouped across scenarios with the fix each calls for
   #   "_verdict"         — the best model and why (judge-written when a judge is available)
-  #   "_selection"       — the scenarios and models this run covered
+  #   "_selection"       — the scenarios and models this run covered, and
+  #                        the checkout sandbox it replayed against, if any
+  #
+  # A selection's `sandbox_id` names a checkout sandbox whose app runtime
+  # every replay reaches beside the agent's own MCP servers, as if the agent
+  # listed it (see Api::RunSandbox, which checked the caller owns it). The
+  # agent is not changed. The runtime is resolved again here, among the
+  # agent's owner's sessions, since the run may start well after it was
+  # asked for; one that is no longer live fails the run rather than
+  # replaying without the tools it was meant to test.
   class ScenarioEvaluationRunner < EvaluationRunnerService
     Evals = ActiveAgent::Evals
 
@@ -43,6 +52,7 @@ module ActionAgent
       specs = model_specs
       run = @run || @evaluation.evaluation_runs.create!(status: :pending)
       run.update!(status: :running, selection: selection_summary(scenarios, specs))
+      ensure_sandbox_live!
 
       if scenarios.empty?
         run.update!(status: :failed, error_message: "No scenarios selected — add scenarios to the evaluation or widen the selection",
@@ -151,10 +161,45 @@ module ActionAgent
         "scenario_ids" => scenarios.map(&:id),
         "scenario_keys" => scenarios.map(&:key),
         "group" => @selection[:group].presence,
-        "models" => specs.map(&:to_h)
+        "models" => specs.map(&:to_h),
+        "sandbox" => sandbox_summary
       }.compact
     end
 
+    # --- sandbox ----------------------------------------------------------
+
+    # "sandbox:<session_id>" for a run against a checkout sandbox, or nil.
+    def sandbox_server_key
+      id = @selection[:sandbox_id]
+      "#{SandboxSession::RUNTIME_SERVER_PREFIX}#{id}" if id.is_a?(String) && id.present?
+    end
+
+    def sandbox_session
+      return @sandbox_session if defined?(@sandbox_session)
+
+      @sandbox_session = sandbox_server_key && SandboxSession.for_owner(owner).find_by(session_id: @selection[:sandbox_id])
+    end
+
+    # What the run records about its sandbox: which checkout, never its
+    # token.
+    def sandbox_summary
+      return nil unless sandbox_server_key
+
+      {
+        "session_id" => @selection[:sandbox_id],
+        "server_key" => sandbox_server_key,
+        "repository" => sandbox_session&.repository,
+        "repository_ref" => sandbox_session&.repository_ref
+      }.compact
+    end
+
+    def ensure_sandbox_live!
+      return unless sandbox_server_key
+      return if SandboxSession.runtime_server_entry(sandbox_server_key, owner: owner)
+
+      raise ArgumentError, "Sandbox #{@selection[:sandbox_id]} is no longer running; start it again, or run without it"
+    end
+
     # --- replay -----------------------------------------------------------
 
     def replay(scenario, spec)
@@ -167,7 +212,8 @@ module ActionAgent
         scenario.prompt,
         model_override: spec.model,
         provider_override: spec.provider,
-        actor: replay_actor
+        actor: replay_actor,
+        runtime_sandbox: sandbox_server_key
       )
 
       Evals::Replay.new(
@@ -177,7 +223,8 @@ module ActionAgent
         input_tokens: agent_run.input_tokens,
         output_tokens: agent_run.output_tokens,
         error: agent_run.failed? ? agent_run.error_message.presence || "run failed" : nil,
-        cost: ModelPricing.estimate(model: spec.model, input_tokens: agent_run.input_tokens, output_tokens: agent_run.output_tokens),
+        cost: ModelPricing.estimate(model: spec.model, provider: spec.provider, input_tokens: agent_run.input_tokens,
+                                    output_tokens: agent_run.output_tokens),
         metadata: { "agent_run_id" => agent_run.id }
       )
     end
@@ -303,7 +350,7 @@ module ActionAgent
     end
 
     def mcp_dispatcher
-      @mcp_dispatcher ||= MCPToolDispatcher.new(@evaluation.agent)
+      @mcp_dispatcher ||= MCPToolDispatcher.new(@evaluation.agent, extra_server_keys: [ sandbox_server_key ].compact)
     end
 
     # A run whose every declared MCP server failed discovery scored an agent

data/app/services/action_agent/secret_scrubber.rb

@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+module ActionAgent
+  # Replaces known secrets in text or nested JSON-like data before it is
+  # stored or shown: a sandbox's GitHub token and Claude Code credential can
+  # surface in a process's output (an error echoing a URL, a session that
+  # prints its environment), and none of that may reach a transcript, a log
+  # tail or an API response.
+  module SecretScrubber
+    MASK = "[REDACTED]"
+    # Shorter values are not credentials, and masking them would mangle text.
+    MIN_SECRET_LENGTH = 8
+
+    module_function
+
+    # @param value [String, Hash, Array, Object] what to scrub
+    # @param secrets [Array<String>] the values to mask
+    # @return a copy of +value+ with every secret masked
+    def scrub(value, secrets)
+      secrets = Array(secrets).compact.map(&:to_s).select { |secret| secret.length >= MIN_SECRET_LENGTH }.uniq
+      return value if secrets.empty?
+
+      # Longest first, so a secret that contains another is masked whole.
+      pattern = Regexp.union(secrets.sort_by { |secret| -secret.length })
+      deep_scrub(value, pattern)
+    end
+
+    def deep_scrub(value, pattern)
+      case value
+      when String then value.gsub(pattern, MASK)
+      when Hash then value.to_h { |key, item| [ key, deep_scrub(item, pattern) ] }
+      when Array then value.map { |item| deep_scrub(item, pattern) }
+      else value
+      end
+    end
+  end
+end

data/app/services/action_agent/tool_discovery.rb

@@ -35,6 +35,9 @@ module ActionAgent
   #
   # Scopes are passed in rather than derived, so the caller's ownership
   # rules (single-user, per-user, or multi-tenant) decide what is visible.
+  # The same goes for checkout sandbox runtimes: the caller hands in the live
+  # ones it may see (SandboxSession.runtime_server_listings), and they are
+  # listed beside the catalog under their "sandbox:<session_id>" keys.
   class ToolDiscovery
     DEFAULT_WINDOW_HOURS = 24 * 7
     MAX_WINDOW_HOURS = 24 * 90
@@ -53,16 +56,20 @@ module ActionAgent
     ORIGIN_BUILTIN = "builtin"
     ORIGIN_AGENT = "agent"
 
-    attr_reader :traces, :agents, :window_hours, :since
+    attr_reader :traces, :agents, :window_hours, :since, :runtimes
 
     # @param traces [ActiveRecord::Relation] the traces the caller may read
     # @param agents [ActiveRecord::Relation] the agents the caller may reach
     # @param hours [Integer] how far back to look
-    def initialize(traces:, agents:, hours: DEFAULT_WINDOW_HOURS)
+    # @param runtimes [Array<Hash>] the live checkout sandbox runtimes the
+    #   caller may see, as SandboxSession.runtime_server_listings returns
+    #   them — token-free catalog-shaped entries
+    def initialize(traces:, agents:, hours: DEFAULT_WINDOW_HOURS, runtimes: [])
       @traces = traces
       @agents = agents
       @window_hours = hours.to_i.clamp(1, MAX_WINDOW_HOURS)
       @since = @window_hours.hours.ago
+      @runtimes = Array(runtimes).index_by { |runtime| runtime[:key] }
     end
 
     # The full inventory: every tool seen in the window, plus the MCP servers
@@ -500,7 +507,7 @@ module ActionAgent
         end
       end
 
-      keys = (MCPCatalog.keys + detected.keys + configured_servers.keys).uniq
+      keys = (MCPCatalog.keys + runtimes.keys + detected.keys + configured_servers.keys).uniq
 
       # detected has a default block that would materialize a bucket on
       # lookup, so unseen servers are passed through as an explicit nil.
@@ -509,7 +516,11 @@ module ActionAgent
     end
 
     def server_row(key, bucket)
-      catalog = MCPCatalog.find(key)
+      # A live checkout runtime reads like a catalog entry: known, named after
+      # its repository and ref, with the endpoint it serves on. Its listing
+      # carries no bearer token, so nothing below can render one.
+      catalog = MCPCatalog.find(key) || runtimes[key]
+      runtime = SandboxSession.runtime_server_key?(key)
       configured = configured_servers[key].to_a.sort
       calls = bucket ? bucket[:calls] : 0
 
@@ -524,11 +535,14 @@ module ActionAgent
         docs_url: catalog&.fetch(:docs_url, nil),
         first_party: catalog ? catalog[:first_party] : false,
         requires_credentials: catalog ? catalog[:requires_credentials] : [],
-        launchable: catalog ? catalog[:sandbox] : false,
+        # A runtime is already running — it was started from Settings ->
+        # Integrations — so there is nothing here to launch.
+        launchable: catalog && !runtime ? catalog[:sandbox] : false,
         sandbox_type: catalog&.fetch(:sandbox_type, nil),
         # Catalog membership is what "known" means — a server detected purely
         # from traffic is real but undocumented here, and the view says so.
         known: !catalog.nil?,
+        runtime: runtime,
         status: server_status(calls, configured, catalog),
         calls: calls,
         errors: bucket ? bucket[:errors] : 0,

data/config/routes.rb

@@ -71,8 +71,8 @@ ActionAgent::Engine.routes.draw do
       end
     end
 
-    # Sandboxes. The engine ships the in-memory backend; an operator registers
-    # real ones (see ActionAgent.sandbox_backends).
+    # Sandboxes. The engine ships the in-memory and local backends; an
+    # operator registers the rest (see ActionAgent.sandbox_backends).
     resources :sandboxes, param: :id, only: [ :index, :create, :show, :destroy ] do
       collection do
         post :compare
@@ -80,6 +80,12 @@ ActionAgent::Engine.routes.draw do
       member do
         post :run
       end
+      # Claude Code sessions in an app_runtime sandbox's checkout.
+      resources :code_sessions, only: [ :index, :create, :show ] do
+        member do
+          post :cancel
+        end
+      end
     end
 
     # Tool inventory — auto-detected from the tool roster each generation
@@ -136,7 +142,7 @@ ActionAgent::Engine.routes.draw do
 
     # Agent output evaluations. A scenario suite also manages its scenarios
     # here, and exposes each run's per-scenario, per-model results.
-    resources :evaluations, only: [ :index, :show, :create, :destroy ] do
+    resources :evaluations, only: [ :index, :show, :create, :update, :destroy ] do
       member do
         post :run
         get "runs/:run_id", action: :show_run, as: :run_result
@@ -151,7 +157,20 @@ ActionAgent::Engine.routes.draw do
     # Credentials: dashboard API keys (token shown once on create) and the
     # owner's own LLM provider credentials, both encrypted at rest.
     resources :api_keys, only: [ :index, :create, :destroy ]
-    resources :provider_keys, only: [ :index, :create, :destroy ], param: :provider
+    resources :provider_keys, only: [ :index, :create, :destroy ], param: :provider do
+      # Reachability + model list for host-based providers (Ollama), for a
+      # submitted or the stored host. Read-only.
+      post :test, on: :collection
+    end
+
+    # The owner's GitHub connection: the OAuth web flow (connect redirects to
+    # GitHub, which returns to callback) and the repositories it makes
+    # available to checkout sandboxes.
+    resource :github_connection, only: [ :show, :update, :destroy ], controller: "github_connections" do
+      get :repositories
+      get :connect
+      get :callback
+    end
 
     # Model catalogs for the agent builder (Ollama queried live from the
     # configured host; hosted providers curated).

data/lib/action_agent/engine.rb

@@ -24,6 +24,7 @@ module ActionAgent
       "mcp_client" => "MCPClient",
       "mcp_tool_dispatcher" => "MCPToolDispatcher",
       "mcp_controller" => "MCPController",
+      "mcp_dashboard_tools" => "MCPDashboardTools",
       "mcp_recording_middleware" => "MCPRecordingMiddleware",
       "mcp_servers_controller" => "MCPServersController",
       "playwright_mcp_client" => "PlaywrightMCPClient"

data/lib/action_agent/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module ActionAgent
-  VERSION = "1.7.2"
+  VERSION = "1.8.1"
 end