actionagent 1.8.1 → 1.9.0

data/app/services/action_agent/scenario_evaluation_runner.rb

@@ -27,6 +27,15 @@ module ActionAgent
   # agent's owner's sessions, since the run may start well after it was
   # asked for; one that is no longer live fails the run rather than
   # replaying without the tools it was meant to test.
+  #
+  # A selection with `browser` true (a run of a project's evaluation, see
+  # ProjectEvaluationJob) also gives every replay that sandbox's browser:
+  # one already running, or one started headless with the project's saved
+  # sign-in before the first replay, on a backend that runs browsers.
+  # `mount_url`, the dashboard's absolute mount URL, is where a browser
+  # started here posts its recording. Each replay's browser opens at the
+  # project's start URL. A browser that cannot start fails the run before
+  # any replay, and one started here is stopped once the run ends.
   class ScenarioEvaluationRunner < EvaluationRunnerService
     Evals = ActiveAgent::Evals
 
@@ -41,6 +50,22 @@ module ActionAgent
       new(evaluation, selection: selection, run: run).call
     end
 
+    # The caller a replay of +owner+'s agent runs on behalf of, which is also
+    # who its provider credentials are resolved for: +owner+, when the
+    # install owns agents per user. A run with no caller reads, through any
+    # host scope, as "no access" — every tool answers empty and the suite
+    # grades an agent that never saw a row — so the person the evaluation
+    # belongs to is the right default, as the key's owner is over MCP. An
+    # account is who is billed, not who is allowed (see Api::BaseController
+    # #agent_actor), so a multi-tenant install replays unattributed unless a
+    # host adapter (ActionAgent.scenario_evaluation_adapter_resolver) runs
+    # the suite itself.
+    def self.replay_actor_for(owner)
+      return nil if ActionAgent.multi_tenant? || ActionAgent.user_class.blank?
+
+      owner
+    end
+
     def initialize(evaluation, selection: {}, run: nil)
       super(evaluation)
       @selection = (selection || {}).to_h.with_indifferent_access
@@ -60,6 +85,8 @@ module ActionAgent
         return run
       end
 
+      ensure_browser_live!
+
       records = scenarios.index_by(&:key)
       tasks = scenarios.map { |scenario| Evals::Scenario.from_hash(scenario.as_json_summary) }
       expected = tasks.product(specs).map { |task, spec| [ task.key, spec.label ] }
@@ -111,6 +138,8 @@ module ActionAgent
     rescue StandardError => e
       run&.update!(status: :failed, error_message: e.message, completed_at: Time.current)
       raise
+    ensure
+      stop_started_browser
     end
 
     private
@@ -162,7 +191,8 @@ module ActionAgent
         "scenario_keys" => scenarios.map(&:key),
         "group" => @selection[:group].presence,
         "models" => specs.map(&:to_h),
-        "sandbox" => sandbox_summary
+        "sandbox" => sandbox_summary,
+        "browser" => browser_requested? ? { "server_key" => "#{SandboxSession::BROWSER_SERVER_PREFIX}#{@selection[:sandbox_id]}" } : nil
       }.compact
     end
 
@@ -200,6 +230,66 @@ module ActionAgent
       raise ArgumentError, "Sandbox #{@selection[:sandbox_id]} is no longer running; start it again, or run without it"
     end
 
+    # --- browser ----------------------------------------------------------
+
+    # Whether the replays reach the sandbox's browser: asked for, and
+    # running or startable on the sandbox's backend.
+    def browser_requested?
+      return @browser_requested if defined?(@browser_requested)
+
+      @browser_requested = sandbox_server_key.present? && ActiveModel::Type::Boolean.new.cast(@selection[:browser]) == true &&
+        sandbox_session.present? && (sandbox_session.browser_running? || SandboxBrowser.orchestrator!.supports?(:start_browser))
+    rescue SandboxBrowser::Error
+      @browser_requested = false
+    end
+
+    # The project the selected sandbox was booted for, or nil.
+    def sandbox_project
+      return @sandbox_project if defined?(@sandbox_project)
+
+      @sandbox_project = sandbox_session&.project_id && Project.find_by(id: sandbox_session.project_id)
+    end
+
+    def ensure_browser_live!
+      return unless browser_requested?
+
+      sandbox = sandbox_session
+      unless sandbox.browser_running?
+        denial = ActionAgent.quota_denial(sandbox.metering_owner, :browser_minutes)
+        raise ArgumentError, "The sandbox's browser could not start: the plan allows no more browser minutes" if denial.present?
+
+        _, @started_browser = SandboxBrowser.ensure_running!(sandbox,
+          recording_url: SandboxBrowser.recording_url_for(@selection[:mount_url]), storage_state: sandbox_project&.saved_storage_state)
+      end
+      @sandbox_session = sandbox.reload
+    rescue SandboxBrowser::Error => e
+      raise ArgumentError, "The sandbox's browser could not start: #{e.message}"
+    end
+
+    # Stops the browser ensure_browser_live! started, which completes its
+    # recording and stops its minutes. One that was already running is left
+    # running.
+    def stop_started_browser
+      return unless @started_browser
+
+      @started_browser = false
+      sandbox = sandbox_session.reload
+      SandboxBrowser.stop(sandbox) if SandboxBrowser::STARTED.include?(sandbox.browser_status)
+    rescue SandboxBrowser::Error, ActiveRecord::RecordNotFound => e
+      Rails.logger.warn("[ActionAgent] evaluation #{@evaluation.id}: could not stop the browser it started: #{e.message}")
+    end
+
+    # Opens the project's start URL in the sandbox's browser, so each replay
+    # starts on the same page. A browser that does not answer is left to the
+    # replay, which fails when it cannot reach it.
+    def open_start_url
+      return unless browser_requested?
+
+      SandboxBrowserDriver.new(sandbox_session).open(sandbox_project&.start_url.presence || "/")
+    rescue SandboxBrowserDriver::Error => e
+      Rails.logger.warn("[ActionAgent] evaluation #{@evaluation.id}: could not open the start URL: #{e.message}")
+    end
+
     # --- replay -----------------------------------------------------------
 
     def replay(scenario, spec)
@@ -207,6 +297,7 @@ module ActionAgent
       # (the order SandboxesController#compare uses), so it is counted even
       # when the run fails.
       ActionAgent.record_usage(owner, :execution)
+      open_start_url
 
       agent_run = @evaluation.agent.test_execute(
         scenario.prompt,
@@ -215,6 +306,9 @@ module ActionAgent
         actor: replay_actor,
         runtime_sandbox: sandbox_server_key
       )
+      # A replay has nobody to answer, so a paused run ends here as an error.
+      paused = agent_run.awaiting_input?
+      agent_run.cancel!("Paused for input during an evaluation replay") if paused
 
       Evals::Replay.new(
         answer: agent_run.output,
@@ -222,13 +316,19 @@ module ActionAgent
         duration_ms: agent_run.calculated_duration_ms,
         input_tokens: agent_run.input_tokens,
         output_tokens: agent_run.output_tokens,
-        error: agent_run.failed? ? agent_run.error_message.presence || "run failed" : nil,
+        error: replay_error(agent_run, paused),
         cost: ModelPricing.estimate(model: spec.model, provider: spec.provider, input_tokens: agent_run.input_tokens,
                                     output_tokens: agent_run.output_tokens),
         metadata: { "agent_run_id" => agent_run.id }
       )
     end
 
+    def replay_error(agent_run, paused)
+      return "paused for input" if paused
+
+      agent_run.failed? ? agent_run.error_message.presence || "run failed" : nil
+    end
+
     # The caller a replay runs on behalf of: the evaluation's owner, when the
     # install owns agents per user. A run with no caller reads, through any
     # host scope, as "no access" — every tool answers empty and the suite
@@ -239,9 +339,7 @@ module ActionAgent
     # host adapter (ActionAgent.scenario_evaluation_adapter_resolver) runs
     # the suite itself.
     def replay_actor
-      return nil if ActionAgent.multi_tenant? || ActionAgent.user_class.blank?
-
-      owner
+      self.class.replay_actor_for(owner)
     end
 
     # Each tool call the run made, rebuilt from the run's progress events
@@ -334,23 +432,28 @@ module ActionAgent
     # report "none of the available tools covers this task" while naming a list
     # the agent's MCP tools were missing from.
     def tool_roster
-      @tool_roster ||= begin
-        definitions = mcp_dispatcher.tool_definitions +
-          AgentToolbox.definitions_for(@evaluation.agent.tools)
-
-        definitions.to_h { |definition| [ definition[:name].to_s, definition[:description].to_s ] }
-      end
+      runtime_tools.tools
     end
 
-    # Discovery failures from the roster above, keyed by server. Reading them
-    # requires tool_definitions to have run, which tool_roster does.
+    # Discovery failures from the roster above, keyed by server.
     def mcp_discovery_errors
-      tool_roster
-      mcp_dispatcher.discovery_errors
+      runtime_tools.discovery_errors
     end
 
+    # The sandbox's browser is on the roster while it runs, as each replay
+    # (Agent#test_execute) reaches it then.
     def mcp_dispatcher
-      @mcp_dispatcher ||= MCPToolDispatcher.new(@evaluation.agent, extra_server_keys: [ sandbox_server_key ].compact)
+      runtime_tools.dispatcher
+    end
+
+    def runtime_tools
+      @runtime_tools ||= RuntimeToolRoster.new(
+        @evaluation.agent, extra_server_keys: [ sandbox_server_key, sandbox_browser_key ].compact
+      )
+    end
+
+    def sandbox_browser_key
+      sandbox_session.browser_server_key if sandbox_session&.browser_running?
     end
 
     # A run whose every declared MCP server failed discovery scored an agent

data/app/services/action_agent/secret_scrubber.rb

@@ -6,28 +6,57 @@ module ActionAgent
   # surface in a process's output (an error echoing a URL, a session that
   # prints its environment), and none of that may reach a transcript, a log
   # tail or an API response.
+  #
+  # Besides the values it is given, it masks anything shaped like a GitHub
+  # token, given or not. A GitHub App checkout's token is never stored, so a
+  # later code session in that checkout cannot name it by value.
   module SecretScrubber
     MASK = "[REDACTED]"
     # Shorter values are not credentials, and masking them would mangle text.
     MIN_SECRET_LENGTH = 8
+    # GitHub's token formats: personal (ghp_), OAuth (gho_), App user (ghu_),
+    # installation (ghs_) and refresh (ghr_) tokens, and fine-grained
+    # personal tokens (github_pat_). Matched wherever they start, so a token
+    # straight after a percent-escape (`%3Aghs_…` in an encoded URL) or a
+    # name (`GITHUB_TOKEN_ghs_…`) is masked too.
+    GITHUB_TOKEN = /gh[pousr]_[A-Za-z0-9]{30,}|github_pat_[A-Za-z0-9_]{30,}/
+    CLAUDE_TOKEN = /sk-ant-(?:oat|ort|api)[A-Za-z0-9_-]+/
 
     module_function
 
     # @param value [String, Hash, Array, Object] what to scrub
     # @param secrets [Array<String>] the values to mask
-    # @return a copy of +value+ with every secret masked
+    # @return a copy of +value+ with every secret and every GitHub token masked
     def scrub(value, secrets)
       secrets = Array(secrets).compact.map(&:to_s).select { |secret| secret.length >= MIN_SECRET_LENGTH }.uniq
-      return value if secrets.empty?
 
       # Longest first, so a secret that contains another is masked whole.
-      pattern = Regexp.union(secrets.sort_by { |secret| -secret.length })
+      pattern = Regexp.union(*secrets.sort_by { |secret| -secret.length }, GITHUB_TOKEN, CLAUDE_TOKEN)
       deep_scrub(value, pattern)
     end
 
+    # +values+ with the other forms each can take in output: URL-encoded (as
+    # a form and as a path encode it), Base64 (standard and URL-safe, with
+    # and without padding) and escaped inside a JSON string (as JSON.stringify
+    # and as Rails' to_json escape it). Blank values are dropped.
+    #
+    # @param values [Array<String>]
+    # @return [Array<String>]
+    def with_encodings(values)
+      Array(values).compact.map(&:to_s).reject(&:empty?).flat_map do |value|
+        base64 = [ value ].pack("m0")
+        urlsafe = base64.tr("+/", "-_")
+        [ value, URI.encode_www_form_component(value), ERB::Util.url_encode(value),
+          base64, base64.delete("="), urlsafe, urlsafe.delete("="),
+          JSON.generate(value)[1...-1], value.to_json[1...-1] ]
+      end.uniq
+    end
+
     def deep_scrub(value, pattern)
       case value
-      when String then value.gsub(pattern, MASK)
+      # A regexp cannot scan bytes that are invalid in the string's encoding,
+      # so those are replaced first.
+      when String then (value.valid_encoding? ? value : value.scrub).gsub(pattern, MASK)
       when Hash then value.to_h { |key, item| [ key, deep_scrub(item, pattern) ] }
       when Array then value.map { |item| deep_scrub(item, pattern) }
       else value

data/app/services/action_agent/session_index.rb

@@ -0,0 +1,270 @@
+# frozen_string_literal: true
+
+module ActionAgent
+  # Used to list the sessions a caller can replay, newest first, a page at a
+  # time.
+  #
+  # Kinds, and the source each one is filtered by:
+  #   context          dashboard   a conversation (AgentContext) of one of the
+  #                                caller's agents
+  #   scenario_result  evaluation  the run that replayed an evaluation scenario
+  #   recording        agent       a browser recording linked to no
+  #                                conversation, made by no run that wrote to
+  #                                one, and made by no evaluation replay
+  #
+  # An evaluation replay writes to its agent's conversation as well, so a
+  # conversation whose every generation came from evaluation replays is left
+  # out: each replay is listed once, as its scenario result.
+  #
+  # A conversation's runs are the runs that wrote a generation to it and the
+  # runs pinned to it (input_params "context_id"). A session is placed by its
+  # last activity: a conversation's updated_at, a result's or a recording's
+  # created_at. Pages are read with a cursor, so a deep page costs what the
+  # first one does.
+  class SessionIndex
+    KINDS = { "context" => "dashboard", "scenario_result" => "evaluation", "recording" => "agent" }.freeze
+    SOURCES = KINDS.values.uniq.freeze
+    OUTCOMES = %w[failed passed].freeze
+    TIME_COLUMNS = { "context" => :updated_at, "scenario_result" => :created_at, "recording" => :created_at }.freeze
+    DEFAULT_PER_PAGE = 25
+    MAX_PER_PAGE = 100
+
+    # An actor that matches no session, for a "mine" filter with nobody
+    # signed in.
+    NO_ACTOR = Object.new.freeze
+
+    # Raised for a cursor the index cannot read.
+    class InvalidCursor < ArgumentError; end
+
+    # What narrows the index. Every member is optional.
+    #
+    # @!attribute agent_id
+    #   @return [Integer] one of the caller's agents
+    # @!attribute actor
+    #   @return [Object] who the sessions' runs were executed on behalf of
+    #     (AgentRun.on_behalf_of)
+    # @!attribute user
+    #   @return [Object] the signed-in user, whose own browser recordings
+    #     match along with the actor's runs. Read only with +actor+.
+    # @!attribute source
+    #   @return [String] one of SOURCES
+    # @!attribute outcome
+    #   @return [String] "failed" (a failed or errored replay, a conversation
+    #     with a failed run, a failed recording) or "passed" (a passed replay)
+    # @!attribute from
+    #   @return [Time] the earliest last activity, inclusive
+    # @!attribute to
+    #   @return [Time] the latest last activity, exclusive
+    Filters = Struct.new(:agent_id, :actor, :user, :source, :outcome, :from, :to, keyword_init: true)
+
+    # One position in the index: sessions after it sort older.
+    Cursor = Struct.new(:time, :kind, :id) do
+      # Reads a cursor written by #to_s: `<ISO 8601 time>|<kind>|<id>`.
+      def self.parse(value)
+        time, kind, id = value.to_s.split("|", 3)
+        raise InvalidCursor unless KINDS.key?(kind) && id.to_s.match?(/\A\d+\z/)
+
+        new(Time.iso8601(time.to_s), kind, id.to_i)
+      rescue ArgumentError
+        raise InvalidCursor, "Unknown cursor"
+      end
+
+      def to_s
+        "#{time.utc.iso8601(6)}|#{kind}|#{id}"
+      end
+    end
+
+    # +agents+ and +recordings+ are what the caller may see
+    # (Api::BaseController#owner_agents and #reachable_recordings). +before+
+    # is the `next_before` of the previous page.
+    def initialize(agents:, recordings:, filters: Filters.new, per_page: DEFAULT_PER_PAGE, before: nil)
+      @all_agents = agents
+      @agents = filters.agent_id ? agents.where(id: filters.agent_id) : agents
+      @recordings = recordings
+      @filters = filters
+      @per_page = per_page.to_i.clamp(1, MAX_PER_PAGE)
+      @before = before.present? ? Cursor.parse(before) : nil
+      @relations = {}
+    end
+
+    # The page as the JSON API returns it: `sessions`, `has_more`,
+    # `next_before` (the cursor for the next page, nil on the last) and
+    # `total` (every session the filters match).
+    # @return [Hash]
+    def as_json(*)
+      rows = kinds.flat_map { |kind| page_rows(kind) }.sort_by { |row| sort_key(row) }
+      page = rows.first(@per_page)
+      more = rows.size > @per_page
+
+      {
+        sessions: SessionSummarySerializer.new(page, agents: @all_agents, recordings: @recordings,
+          failed_context_ids: failed_context_ids(page)).as_json,
+        has_more: more,
+        next_before: more ? Cursor.new(page.last[:time], page.last[:kind], page.last[:record].id).to_s : nil,
+        total: kinds.sum { |kind| relation(kind).count }
+      }
+    end
+
+    private
+
+    def kinds
+      KINDS.keys.select { |kind| @filters.source.nil? || KINDS[kind] == @filters.source }
+    end
+
+    # Newest first; at the same time, in KINDS order, then the newest id.
+    def sort_key(row)
+      [ -row[:time].to_r, KINDS.keys.index(row[:kind]), -row[:record].id ]
+    end
+
+    # The first per_page + 1 sessions of +kind+ after the cursor: enough to
+    # fill the page and to tell whether another follows.
+    def page_rows(kind)
+      column = TIME_COLUMNS.fetch(kind)
+      scope = relation(kind)
+      table = scope.klass.arel_table
+      after_cursor(scope, kind, table, column)
+        .order(table[column].desc, table[:id].desc)
+        .limit(@per_page + 1)
+        .map { |record| { kind: kind, record: record, time: record.public_send(column) } }
+    end
+
+    def after_cursor(scope, kind, table, column)
+      return scope unless @before
+
+      time = table[column]
+      condition =
+        case KINDS.keys.index(kind) <=> KINDS.keys.index(@before.kind)
+        when 1 then time.lteq(@before.time)
+        when 0 then time.lt(@before.time).or(time.eq(@before.time).and(table[:id].lt(@before.id)))
+        else time.lt(@before.time)
+        end
+      scope.where(condition)
+    end
+
+    def relation(kind)
+      @relations[kind] ||= within_dates(
+        case kind
+        when "context" then contexts
+        when "scenario_result" then scenario_results
+        else recordings
+        end,
+        TIME_COLUMNS.fetch(kind)
+      )
+    end
+
+    def within_dates(scope, column)
+      table = scope.klass.arel_table
+      scope = scope.where(table[column].gteq(@filters.from)) if @filters.from
+      scope = scope.where(table[column].lt(@filters.to)) if @filters.to
+      scope
+    end
+
+    # --- conversations --------------------------------------------------------
+
+    def contexts
+      base = AgentContext.for_agents(@agents)
+      generations = AgentGeneration.where(agent_context_id: base.select(:id))
+      dashboard_generations = generations.where(trace_id: nil).or(generations.where.not(trace_id: replay_traces))
+      scope = base.where(id: dashboard_generations.select(:agent_context_id))
+        .or(base.where.not(id: generations.select(:agent_context_id)))
+
+      scope = with_runs(scope, AgentRun.where(agent: @all_agents).on_behalf_of(@filters.actor)) if @filters.actor
+      case @filters.outcome
+      when "failed" then with_runs(scope, failed_runs)
+      when "passed" then scope.none
+      else scope
+      end
+    end
+
+    # The conversations in +scope+ that +runs+ wrote a generation to or were
+    # pinned to.
+    def with_runs(scope, runs)
+      by_generation = AgentGeneration.where(trace_id: runs.select(:trace_id)).select(:agent_context_id)
+      scope.where(id: by_generation).or(scope.where(id: pinned_context_ids(runs)))
+    end
+
+    def pinned_context_ids(runs)
+      sql = AgentRun.input_param_sql("context_id")
+      runs.where("#{sql} IS NOT NULL").distinct.pluck(Arel.sql(sql))
+        .filter_map { |value| Integer(value.to_s, exception: false) }
+    end
+
+    def failed_runs
+      AgentRun.where(agent: @all_agents, status: :failed)
+    end
+
+    # Which conversations on +page+ had a failed run.
+    def failed_context_ids(page)
+      ids = page.select { |row| row[:kind] == "context" }.map { |row| row[:record].id }
+      return [] if ids.empty?
+
+      with_runs(AgentContext.where(id: ids), failed_runs).pluck(:id)
+    end
+
+    # --- evaluation replays ---------------------------------------------------
+
+    def scenario_results
+      evaluations = Evaluation.where(agent: @agents).select(:id)
+      scope = EvaluationScenarioResult
+        .where(evaluation_run_id: EvaluationRun.where(evaluation_id: evaluations).select(:id))
+        .where.not(agent_run_id: nil)
+
+      scope = scope.where(agent_run_id: AgentRun.on_behalf_of(@filters.actor).select(:id)) if @filters.actor
+      case @filters.outcome
+      when "failed" then scope.where(status: %i[failed errored])
+      when "passed" then scope.where(status: :passed)
+      else scope
+      end
+    end
+
+    def replay_run_ids
+      evaluations = Evaluation.where(agent: @all_agents).select(:id)
+      EvaluationScenarioResult
+        .where(evaluation_run_id: EvaluationRun.where(evaluation_id: evaluations).select(:id))
+        .where.not(agent_run_id: nil)
+        .select(:agent_run_id)
+    end
+
+    def replay_traces
+      AgentRun.where(id: replay_run_ids).where.not(trace_id: nil).select(:trace_id)
+    end
+
+    # --- browser recordings ---------------------------------------------------
+
+    def recordings
+      base = @recordings.where(agent_context_id: nil)
+      scope = base.where(agent_run_id: nil)
+        .or(base.where.not(agent_run_id: replay_run_ids).where.not(agent_run_id: conversation_run_ids))
+      scope = scope.where(agent_run_id: AgentRun.where(agent: @agents).select(:id)) if @filters.agent_id
+      scope = recordings_by_actor(scope) if @filters.actor
+
+      case @filters.outcome
+      when "failed" then scope.where(status: :failed).or(scope.where(agent_run_id: failed_runs.select(:id)))
+      when "passed" then scope.none
+      else scope
+      end
+    end
+
+    # The runs that wrote a generation to one of the caller's conversations.
+    # SessionTimeline.for_context replays their recordings with the
+    # conversation.
+    def conversation_run_ids
+      generations = AgentGeneration.where(agent_context_id: AgentContext.for_agents(@all_agents).select(:id))
+      AgentRun.where(agent: @all_agents, trace_id: generations.select(:trace_id)).select(:id)
+    end
+
+    # A recording's runs were executed on behalf of the actor, or the user
+    # made it.
+    def recordings_by_actor(scope)
+      by_run = scope.where(agent_run_id: AgentRun.on_behalf_of(@filters.actor).select(:id))
+      return by_run unless own_recording_column?
+
+      by_run.or(scope.where(user_id: @filters.user.id))
+    end
+
+    def own_recording_column?
+      user_class = ActionAgent.user_class.to_s
+      user_class.present? && @filters.user.class.name == user_class && SessionRecording.column_names.include?("user_id")
+    end
+  end
+end

data/app/services/action_agent/session_recording_service.rb

@@ -154,6 +154,17 @@ module ActionAgent
       )
     end
 
+    # Record the moment the agent stops for a person: what the page asked
+    # for, where it stopped, and what the person should do. Pairs with
+    # #capture_handoff_state, which stores the page to resume on.
+    def handoff(reason:, url:, instructions: nil)
+      @recording.record_action!(
+        action_type: "handoff",
+        value: reason,
+        metadata: { url: url, instructions: instructions }.compact
+      )
+    end
+
     # Capture browser state for handoff
     def capture_handoff_state(url:, cookies: nil, local_storage: nil, session_storage: nil, form_values: nil)
       @recording.update!(