actionagent 1.2.1 → 1.3.0

data/app/services/action_agent/agent_execution_service.rb

@@ -354,8 +354,14 @@ module ActionAgent
         define_singleton_method(:name) { klass_name }
 
         # Persist the conversation (agent_contexts / agent_messages /
-        # agent_generations) via solid_agent. contextable: false — the context
-        # is loaded explicitly in the action below.
+        # agent_generations) via solid_agent. Auto-context is switched off —
+        # the context is loaded explicitly in the action below.
+        #
+        # The keyword that switches it off was renamed (contextable: ->
+        # contextual:) between solid_agent 0.1 and 0.2, and the gemspec floor
+        # admits both, so it is resolved from the installed method rather than
+        # hard-coded: passing the wrong one is an ArgumentError that only
+        # surfaces when a run executes.
         #
         # The model classes are named explicitly because solid_agent infers
         # bare "AgentContext"/"AgentMessage"/"AgentGeneration" and resolves
@@ -363,10 +369,12 @@ module ActionAgent
         # inferred names only resolve in a host app that happens to have
         # top-level models of its own.
         include SolidAgent::HasContext
-        has_context contextable: false,
+        has_context(
+          ActionAgent.solid_agent_auto_context_keyword => false,
           class_name: "ActionAgent::AgentContext",
           message_class: "ActionAgent::AgentMessage",
           generation_class: "ActionAgent::AgentGeneration"
+        )
 
         if effective_provider == :mock
           # Test environment only (see #provider_available?).

data/app/services/action_agent/agent_toolbox.rb

@@ -46,7 +46,7 @@ module ActionAgent
           }
         }
       ],
-      # A real browser via a Playwright MCP server (PlaywrightMcpClient).
+      # A real browser via a Playwright MCP server (PlaywrightMCPClient).
       # Stateful: navigate changes what snapshot/click see, so these bypass
       # the toolbox result cache.
       "playwright_mcp" => [
@@ -288,17 +288,17 @@ module ActionAgent
       SNAPSHOT_LINK = /\[Snapshot\]\(([^)]+)\)/
 
       def playwright_mcp(tool, arguments, retried: false)
-        result = PlaywrightMcpClient.instance.call_tool(tool, arguments)
+        result = PlaywrightMCPClient.instance.call_tool(tool, arguments)
         text = inline_snapshot(result[:text].to_s)
         if text.length > PLAYWRIGHT_RESULT_LIMIT
           text = "#{text[0, PLAYWRIGHT_RESULT_LIMIT]}\n…(truncated, #{text.length} chars total)"
         end
         result[:is_error] ? { error: text.presence || "browser tool failed" } : { text: text }
-      rescue PlaywrightMcpClient::Error => e
+      rescue PlaywrightMCPClient::Error => e
         # One fresh-session retry: the first call after a server (re)start can
         # race the browser launch.
         unless retried
-          PlaywrightMcpClient.reset!
+          PlaywrightMCPClient.reset!
           return playwright_mcp(tool, arguments, retried: true)
         end
         { error: e.message }
@@ -399,7 +399,7 @@ module ActionAgent
         if defined?(SolidAgent::ToolCache)
           SolidAgent::ToolCache.fetch(tool: name.to_s, args: kwargs, ttl: CACHE_TTL, &block)
         else
-          key = "solid_agent:tool_cache:#{name}:#{Digest::SHA256.hexdigest(kwargs.sort.to_h.to_json)}"
+          key = fallback_cache_key(name, kwargs)
           cached = Rails.cache.read(key)
           return cached.merge(cached: true) unless cached.nil?
 
@@ -411,6 +411,29 @@ module ActionAgent
         end
       end
 
+      # Byte-for-byte the key SolidAgent::ToolCache would compute, so an app
+      # that upgrades solid_agent mid-TTL keeps reading what it already
+      # cached instead of silently starting over. Nested hashes and
+      # symbol/string keys have to normalize the same way, which a plain
+      # `kwargs.sort.to_h.to_json` does not do.
+      #
+      # test/integration/solid_agent/tool_cache_test.rb asserts the two
+      # schemes still agree.
+      def fallback_cache_key(name, kwargs)
+        "solid_agent:tool_cache:#{name}:#{Digest::SHA256.hexdigest(normalize_cache_args(kwargs).to_json)}"
+      end
+
+      def normalize_cache_args(args)
+        case args
+        when Hash
+          args.map { |key, value| [ key.to_s, normalize_cache_args(value) ] }.sort_by(&:first)
+        when Array
+          args.map { |value| normalize_cache_args(value) }
+        else
+          args
+        end
+      end
+
       # SSRF guard for fetch_url: reject hosts that resolve to loopback,
       # private, or link-local addresses.
       def public_host?(host)

data/app/services/action_agent/evaluation_tool_resolver.rb

@@ -0,0 +1,154 @@
+# frozen_string_literal: true
+
+module ActionAgent
+  # Names the MCP server behind a tool an evaluation run needs, for the
+  # report's fix items (ActiveAgent::Evals::Report#fix_items): a scenario
+  # that expected +search_slots+ and never got it is fixed by enabling the
+  # server that serves it, and the item can only say so — and deep-link to
+  # MCP Services — when something here can name that server.
+  #
+  # Resolution follows the order ToolDiscovery attributes traffic in: an
+  # explicit +mcp__server__tool+ namespace wins because the name said so,
+  # then MCPCatalog's hints for the bare names of well-known servers, then
+  # the servers the agent itself declares in +mcp_servers+ when one of them
+  # lists the tool. Nothing here reads telemetry: the resolver runs inside
+  # the request that serializes a run, and every lookup is a constant or a
+  # single agent attribute.
+  #
+  # The status is what the fix item's action turns on:
+  #
+  #   "enabled"   — the agent's mcp_servers configuration names the server
+  #   "available" — the catalog (built-in or ActionAgent.mcp_catalog) knows
+  #                 the server and the agent has not enabled it
+  #   nil         — the namespace named a server nothing here knows; the
+  #                 report renders that as "unknown"
+  #
+  # @example
+  #   resolver = EvaluationToolResolver.new(agent)
+  #   resolver.call("browser_navigate")
+  #   # => { "key" => "playwright", "name" => "Playwright", "status" => "available" }
+  class EvaluationToolResolver
+    ENABLED = "enabled"
+    AVAILABLE = "available"
+
+    attr_reader :agent
+
+    # @param agent [ActionAgent::Agent, nil] the agent the run evaluated; nil
+    #   resolves against the catalog alone
+    def initialize(agent)
+      @agent = agent
+    end
+
+    # The Report's +tool_resolver+ contract.
+    #
+    # @param tool_name [String, Symbol] a tool name as a scenario expected it
+    #   or the model called it
+    # @return [Hash, nil] +{ "key", "name", "status" }+, or nil when no
+    #   server can be named for the tool
+    def call(tool_name)
+      key = server_key_for(tool_name)
+      return nil if key.nil?
+
+      { "key" => key, "name" => display_name_for(key), "status" => status_for(key) }
+    end
+
+    # @param tool_name [String, Symbol]
+    # @return [String, nil] the server key the tool belongs to
+    def server_key_for(tool_name)
+      name = tool_name.to_s.strip
+      return nil if name.blank?
+
+      ActiveAgent::Telemetry::ToolOrigin.server_for(name).presence ||
+        MCPCatalog.server_for_tool(name).presence ||
+        configured_tools[name]
+    end
+
+    # @param key [String] a server key
+    # @return [String, nil] ENABLED, AVAILABLE, or nil when unknown
+    def status_for(key)
+      return ENABLED if configured_keys.include?(normalize(key))
+      return AVAILABLE if MCPCatalog.find(key)
+
+      nil
+    end
+
+    private
+
+    # The catalog's name when it has one; otherwise the name the agent's
+    # own configuration gives the server, and the key as a last resort
+    # (which is what MCPCatalog.display_name falls back to as well).
+    def display_name_for(key)
+      return MCPCatalog.display_name(key) if MCPCatalog.find(key)
+
+      configured_names[normalize(key)] || MCPCatalog.display_name(key)
+    end
+
+    # Server keys the agent declares, normalized for comparison.
+    def configured_keys
+      @configured_keys ||= configured_entries.filter_map { |entry| normalize(entry_key(entry)) }.to_set
+    end
+
+    # normalized key => the display name a configured hash entry carries
+    # alongside its key ({"key" => "sparkle", "name" => "Sparkle Match"}).
+    def configured_names
+      @configured_names ||= configured_entries.each_with_object({}) do |entry, map|
+        next unless entry.respond_to?(:key?)
+
+        key = normalize(entry_key(entry))
+        name = (entry["name"] || entry[:name]).to_s.strip
+        next if key.nil? || name.blank? || name.downcase == key
+
+        map[key] ||= name
+      end
+    end
+
+    # bare tool name => server key, from configured entries that list the
+    # tools they serve ({"name" => "sparkle", "tools" => ["search_slots"]}),
+    # in the catalog's own +tool_hints+ spelling or as tool hashes.
+    def configured_tools
+      @configured_tools ||= configured_entries.each_with_object({}) do |entry, map|
+        next unless entry.respond_to?(:key?)
+
+        key = entry_key(entry)
+        next if key.nil?
+
+        Array(entry["tools"] || entry[:tools] || entry["tool_hints"] || entry[:tool_hints]).each do |tool|
+          name = (tool.respond_to?(:key?) ? tool["name"] || tool[:name] : tool).to_s.strip
+          map[name] ||= key unless name.blank?
+        end
+      end
+    end
+
+    # The agent's mcp_servers as a list of entries. Agents store an Array of
+    # bare names or builder hashes, but an agent seeded from an older
+    # template carries a top-level Hash keyed by server name
+    # ({"playwright" => {"command" => ...}}) — the same shape ToolDiscovery
+    # tolerates — whose values become entries carrying that key.
+    def configured_entries
+      @configured_entries ||= begin
+        servers = agent&.mcp_servers
+
+        if servers.is_a?(Hash)
+          servers.map do |key, value|
+            value.respond_to?(:key?) ? value.to_h.stringify_keys.merge("key" => key.to_s) : key.to_s
+          end
+        else
+          Array(servers)
+        end
+      end
+    end
+
+    # An entry names its server as a bare string, or under +key+ or +name+
+    # in a builder hash. Anything else (a stray Array, a number) is skipped.
+    def entry_key(entry)
+      return entry.to_s.strip.presence if entry.is_a?(String) || entry.is_a?(Symbol)
+      return nil unless entry.respond_to?(:key?)
+
+      (entry["key"] || entry[:key] || entry["name"] || entry[:name]).to_s.strip.presence
+    end
+
+    def normalize(key)
+      key.to_s.strip.downcase.presence
+    end
+  end
+end

data/app/services/action_agent/mcp_catalog.rb

@@ -19,7 +19,7 @@ module ActionAgent
   # shows up in the MCP Services view (as +known: false+) the moment a
   # namespaced tool call from it is ingested. The catalog only adds names,
   # descriptions, and the ability to launch.
-  class McpCatalog
+  class MCPCatalog
     # Whether a server can be started inside a sandbox session. Servers that
     # need workspace-specific credentials (github, slack, postgres) are
     # listable and attributable but not launchable from the dashboard — there
@@ -156,7 +156,12 @@ module ActionAgent
         sandbox: false,
         first_party: true,
         requires_credentials: [ "Platform API key" ],
-        tool_hints: %w[call_agent list_agents]
+        # No tool_hints: the facade's tools are run_<slug> (one per agent, see
+        # Api::MCPController#tools_list), which no static list can name. The
+        # old hints advertised call_agent and list_agents, which the facade
+        # never exposed, and would have misattributed an agent-defined tool
+        # of either name to this server.
+        tool_hints: []
       }
     ].freeze
 
@@ -172,11 +177,12 @@ module ActionAgent
     end.freeze
 
     class << self
-      # Every catalog entry, as API-shaped hashes.
+      # Every catalog entry — built-ins plus the host's
+      # +ActionAgent.mcp_catalog+ registrations — as API-shaped hashes.
       #
       # @return [Array<Hash>]
       def all
-        SERVERS.map { |server| present(server) }
+        entries.map { |server| present(server) }
       end
 
       # Entries that can be started inside a sandbox session.
@@ -189,14 +195,21 @@ module ActionAgent
       # @param key [String, Symbol]
       # @return [Hash, nil] the catalog entry, or nil when unknown
       def find(key)
-        entry = BY_KEY[key.to_s]
+        entry = index[key.to_s]
         present(entry) if entry
       end
 
+      # Every catalog key, built-ins first.
+      #
+      # @return [Array<String>]
+      def keys
+        entries.map { |server| server[:key] }
+      end
+
       # @param key [String, Symbol]
       # @return [Boolean] whether the server can be started in a sandbox
       def launchable?(key)
-        BY_KEY[key.to_s]&.fetch(:sandbox, false) || false
+        index[key.to_s]&.fetch(:sandbox, false) || false
       end
 
       # The catalog server a bare (non-namespaced) tool name belongs to.
@@ -208,7 +221,7 @@ module ActionAgent
       # @param tool_name [String, Symbol]
       # @return [String, nil] the server key
       def server_for_tool(tool_name)
-        TOOL_HINTS[tool_name.to_s]
+        tool_hints[tool_name.to_s]
       end
 
       # The display name for a server key, falling back to the key itself for
@@ -217,11 +230,36 @@ module ActionAgent
       # @param key [String, Symbol]
       # @return [String]
       def display_name(key)
-        BY_KEY[key.to_s]&.fetch(:name) || key.to_s
+        index[key.to_s]&.[](:name) || key.to_s
       end
 
       private
 
+      # Built-ins plus host registrations, deduplicated by key (first
+      # declaration wins, so a built-in keeps its key). Derived per call
+      # rather than memoized: the host list is tiny and tests reconfigure it.
+      def entries
+        (SERVERS + host_entries).uniq { |server| server[:key] }
+      end
+
+      def host_entries
+        Array(ActionAgent.mcp_catalog).filter_map do |entry|
+          normalized = entry.to_h.symbolize_keys
+          normalized[:key] = normalized[:key].to_s
+          normalized unless normalized[:key].blank?
+        end
+      end
+
+      def index
+        entries.index_by { |server| server[:key] }
+      end
+
+      def tool_hints
+        entries.each_with_object({}) do |server, map|
+          Array(server[:tool_hints]).each { |tool| map[tool.to_s] ||= server[:key] }
+        end
+      end
+
       # Drops the internal-only tool_hints and normalizes optional keys so
       # every entry serializes with the same shape.
       def present(server)

data/app/services/action_agent/mcp_recording_middleware.rb

@@ -7,7 +7,7 @@ module ActionAgent
   # to a SessionRecording for playback and debugging.
   #
   # Usage:
-  #   middleware = McpRecordingMiddleware.new(session_recording: recording)
+  #   middleware = MCPRecordingMiddleware.new(session_recording: recording)
   #
   #   # Process a tool call
   #   result = middleware.intercept(tool_call) do
@@ -15,7 +15,7 @@ module ActionAgent
   #     mcp_client.call_tool(tool_call)
   #   end
   #
-  class McpRecordingMiddleware
+  class MCPRecordingMiddleware
     # Map of Playwright MCP tool names to our action types
     PLAYWRIGHT_TOOLS = {
       "browser_navigate" => "navigate",

data/app/services/action_agent/playwright_mcp_client.rb

@@ -9,7 +9,7 @@ module ActionAgent
   # app in development, or a sandbox-provisioned browser container in
   # production. Speaks just enough JSON-RPC for tools/call: initialize once
   # per process, then call tools under the session id the server hands back.
-  class PlaywrightMcpClient
+  class PlaywrightMCPClient
     DEFAULT_URL = ENV.fetch("PLAYWRIGHT_MCP_URL", "http://host.orb.internal:8931/mcp")
     OPEN_TIMEOUT_SECONDS = 5
     READ_TIMEOUT_SECONDS = 60
@@ -31,7 +31,7 @@ module ActionAgent
 
     # Returns { text:, is_error: } — the tool result's text content.
     def call_tool(name, arguments = {})
-      Rails.logger.debug("[PlaywrightMcpClient] call #{name} args=#{arguments.inspect[0, 200]}")
+      Rails.logger.debug("[PlaywrightMCPClient] call #{name} args=#{arguments.inspect[0, 200]}")
       ensure_session!
       response = post(
         { jsonrpc: "2.0", id: next_id, method: "tools/call",
@@ -40,7 +40,7 @@ module ActionAgent
       )
       result = response["result"]
       unless result
-        Rails.logger.warn("[PlaywrightMcpClient] #{name} unexpected response: #{response.inspect[0, 500]}")
+        Rails.logger.warn("[PlaywrightMCPClient] #{name} unexpected response: #{response.inspect[0, 500]}")
         raise Error, (response.dig("error", "message") || "empty MCP response")
       end
 
@@ -100,17 +100,17 @@ module ActionAgent
 
       response = http.request(request)
       Rails.logger.debug(
-        "[PlaywrightMcpClient] #{payload[:method]} -> #{response.code} " \
+        "[PlaywrightMCPClient] #{payload[:method]} -> #{response.code} " \
         "ct=#{response['Content-Type']} bytes=#{response.body.to_s.bytesize} session=#{session ? 'yes' : 'no'}"
       )
       unless response.code.to_i.between?(200, 299)
-        Rails.logger.warn("[PlaywrightMcpClient] HTTP #{response.code}: #{response.body.to_s[0, 300]}")
+        Rails.logger.warn("[PlaywrightMCPClient] HTTP #{response.code}: #{response.body.to_s[0, 300]}")
         raise Error, "MCP server returned HTTP #{response.code}"
       end
 
       parsed = parse_body(response)
       if parsed.empty? && payload[:id]
-        Rails.logger.warn("[PlaywrightMcpClient] unparsed body (#{response['Content-Type']}): #{response.body.to_s[0, 500]}")
+        Rails.logger.warn("[PlaywrightMCPClient] unparsed body (#{response['Content-Type']}): #{response.body.to_s[0, 500]}")
       end
       [ parsed, response ]
     end

data/app/services/action_agent/sandbox_orchestrator.rb

@@ -52,7 +52,18 @@ module ActionAgent
     # as sandbox_service, falling back to the in-memory one.
     def self.default_backend
       name = ENV["SANDBOX_BACKEND"].presence || ActionAgent.sandbox_service.to_s
-      backends.key?(name) ? name : "mock"
+      return name if backends.key?(name)
+
+      # Substituting the mock silently made a misconfigured operator's
+      # sandbox "runs" succeed against nothing real.
+      if name.present? && name != "mock"
+        Rails.logger.warn(
+          "[ActionAgent] sandbox backend #{name.inspect} is not registered " \
+          "(ActionAgent.sandbox_backends knows #{backends.keys.inspect}); " \
+          "using the in-memory mock backend, which runs nothing."
+        )
+      end
+      "mock"
     end
 
     def initialize(backend: nil)

data/app/services/action_agent/scenario_evaluation_runner.rb

@@ -0,0 +1,226 @@
+# frozen_string_literal: true
+
+module ActionAgent
+  # Runs a scenario evaluation: replays every selected scenario through the
+  # agent once per candidate model and writes one EvaluationScenarioResult per
+  # scenario × model plus a per-model summary and verdict on the run.
+  #
+  # The scoring, fault diagnosis and roll-up are ActiveAgent::Evals'; this
+  # class supplies what only the dashboard knows — how to run the agent
+  # (Agent#test_execute with a model override), where to persist each result,
+  # how to price tokens, and which judge model the owner has credentials for.
+  #
+  # The run's `scores` keep the shape the Evaluations UI renders — criterion
+  # => stats, or criterion => { model => stats } when comparing — and add
+  # underscore-prefixed summaries:
+  #
+  #   "_models"          — per model: pass rate, mean score, latency, tokens, cost, fault counts
+  #   "_recommendations" — faults grouped across scenarios with the fix each calls for
+  #   "_verdict"         — the best model and why (judge-written when a judge is available)
+  #   "_selection"       — the scenarios and models this run covered
+  class ScenarioEvaluationRunner < EvaluationRunnerService
+    Evals = ActiveAgent::Evals
+
+    # `run` is an EvaluationRun created ahead of time (by run_later!, so the
+    # UI can show it pending while the job waits); absent, one is created here.
+    def self.call(evaluation, selection: {}, run: nil)
+      new(evaluation, selection: selection, run: run).call
+    end
+
+    def initialize(evaluation, selection: {}, run: nil)
+      super(evaluation)
+      @selection = (selection || {}).to_h.with_indifferent_access
+      @run = run
+    end
+
+    def call
+      scenarios = selected_scenarios
+      specs = model_specs
+      run = @run || @evaluation.evaluation_runs.create!(status: :pending)
+      run.update!(status: :running, selection: selection_summary(scenarios, specs))
+
+      if scenarios.empty?
+        run.update!(status: :failed, error_message: "No scenarios selected — add scenarios to the evaluation or widen the selection",
+          completed_at: Time.current)
+        return run
+      end
+
+      ensure_judge_defined_kpis! if @evaluation.judge_defined?
+
+      records = scenarios.index_by(&:key)
+      report = Evals::Runner.new(
+        scenarios: scenarios.map { |scenario| Evals::Scenario.from_hash(scenario.as_json_summary) },
+        models: specs,
+        criteria: sample_criteria,
+        judge: evals_judge,
+        available_tools: tool_roster,
+        instructions: @evaluation.agent.instructions,
+        agent_name: @evaluation.agent.name,
+        threshold: PASS_THRESHOLD,
+        replay: ->(scenario, spec) { replay(scenario, spec) },
+        on_result: ->(result) { persist(run, records.fetch(result.scenario.key), result) }
+      ).call
+
+      run.update!(
+        status: :complete,
+        scores: scores_for(report, run),
+        samples_evaluated: report.results.size,
+        samples_passed: report.results.count(&:passed?),
+        completed_at: Time.current
+      )
+      run
+    rescue StandardError => e
+      run&.update!(status: :failed, error_message: e.message, completed_at: Time.current)
+      raise
+    end
+
+    private
+
+    # --- selection --------------------------------------------------------
+
+    def selected_scenarios
+      scope = @evaluation.scenarios.enabled.ordered
+      scope = scope.where(id: Array(@selection[:scenario_ids])) if @selection[:scenario_ids].present?
+      scope = scope.where(key: Array(@selection[:keys])) if @selection[:keys].present?
+      scope = scope.in_group(@selection[:group]) if @selection[:group].present?
+      scope.to_a
+    end
+
+    # The models to compare: an explicit selection, else the evaluation's
+    # compare_models, else the agent as configured. `mock` is the framework's
+    # test double, accepted so the test suite can compare cohorts offline.
+    def model_specs
+      names = Array(@selection[:models]).presence || @evaluation.compare_models
+      specs = Evals::ModelSpec.parse_all(names, default_provider: @evaluation.agent.provider, providers: Agent::PROVIDERS + %w[mock])
+      return specs if specs.any?
+
+      [ Evals::ModelSpec.new(label: @evaluation.agent.model, provider: @evaluation.agent.provider, model: @evaluation.agent.model) ]
+    end
+
+    def selection_summary(scenarios, specs)
+      {
+        "scenario_ids" => scenarios.map(&:id),
+        "scenario_keys" => scenarios.map(&:key),
+        "group" => @selection[:group].presence,
+        "models" => specs.map(&:to_h)
+      }.compact
+    end
+
+    # --- replay -----------------------------------------------------------
+
+    def replay(scenario, spec)
+      # One execution per replay, reported to the host before the run starts
+      # (the order SandboxesController#compare uses), so it is counted even
+      # when the run fails.
+      ActionAgent.record_usage(owner, :execution)
+
+      agent_run = @evaluation.agent.test_execute(
+        scenario.prompt,
+        model_override: spec.model,
+        provider_override: spec.provider
+      )
+
+      Evals::Replay.new(
+        answer: agent_run.output,
+        tool_calls: tool_calls_for(agent_run),
+        duration_ms: agent_run.calculated_duration_ms,
+        input_tokens: agent_run.input_tokens,
+        output_tokens: agent_run.output_tokens,
+        error: agent_run.failed? ? agent_run.error_message.presence || "run failed" : nil,
+        cost: ModelPricing.estimate(model: spec.model, input_tokens: agent_run.input_tokens, output_tokens: agent_run.output_tokens),
+        metadata: { "agent_run_id" => agent_run.id }
+      )
+    end
+
+    # Each tool call the run made, rebuilt from the run's progress events
+    # (AgentRun#append_event pairs a "started" event with its "done"/"error"
+    # by eid). Falls back to the bare names in the run's metadata for a run
+    # recorded without events.
+    def tool_calls_for(agent_run)
+      events = Array(agent_run.logs).select { |event| event.is_a?(Hash) && %w[tool agent].include?(event["kind"]) }
+      if events.empty?
+        return Array(agent_run.output_metadata&.dig("tool_calls")).map { |name| { "name" => name.to_s } }
+      end
+
+      events.group_by { |event| event["eid"] }.values.map do |group|
+        started = group.find { |event| event["status"] == "started" }
+        finished = group.find { |event| %w[done error].include?(event["status"]) }
+        label = (started || finished)["label"].to_s
+
+        {
+          "name" => label.sub(/\s*→.*\z/, ""),
+          "arguments" => parse_json(started&.dig("detail")),
+          "error" => finished&.dig("status") == "error",
+          "detail" => finished&.dig("detail"),
+          "duration_ms" => finished&.dig("duration_ms")
+        }.compact
+      end
+    end
+
+    def parse_json(text)
+      return nil if text.blank?
+
+      JSON.parse(text)
+    rescue JSON::ParserError
+      text
+    end
+
+    # --- persistence ------------------------------------------------------
+
+    def persist(run, scenario, result)
+      run.scenario_results.create!(
+        scenario: scenario,
+        agent_run_id: result.replay.metadata["agent_run_id"],
+        model: result.model,
+        provider: result.provider,
+        status: result.status,
+        score: result.score,
+        scores: result.scores,
+        output: result.replay.answer.to_s.byteslice(0, 20_000).to_s.scrub.presence,
+        tool_calls: result.replay.tool_calls,
+        duration_ms: result.replay.duration_ms,
+        input_tokens: result.replay.input_tokens,
+        output_tokens: result.replay.output_tokens,
+        cost: result.replay.cost,
+        fault: result.fault,
+        recommendation: result.recommendation,
+        diagnosis: result.diagnosis || {},
+        error_message: result.replay.error
+      )
+    end
+
+    def scores_for(report, run)
+      scores = report.criterion_scores
+      scores["_models"] = report.summary_by_model
+      scores["_recommendations"] = report.recommendations
+      scores["_verdict"] = report.verdict if report.comparing?
+      scores["_selection"] = run.selection
+      scores
+    end
+
+    # --- judge ------------------------------------------------------------
+
+    def sample_criteria
+      @sample_criteria ||= @evaluation.criteria.reject do |criterion|
+        Evaluation::TELEMETRY_CRITERION_TYPES.include?(criterion["type"])
+      end
+    end
+
+    def tool_roster
+      @tool_roster ||= AgentToolbox.definitions_for(@evaluation.agent.tools).to_h do |definition|
+        [ definition[:name].to_s, definition[:description].to_s ]
+      end
+    end
+
+    # The judge the evaluation's owner has credentials for, wrapped for the
+    # evaluation core; nil when none is configured, in which case scoring
+    # stays on rules and expectations.
+    def evals_judge
+      return nil unless judge_available?
+
+      @evals_judge ||= Evals::Judge.new(label: @evaluation.judge_model.presence || judge_provider.to_s) do |instructions:, prompt:|
+        judge_class.prompt(message: prompt, instructions: instructions).generate_now.message&.content
+      end
+    end
+  end
+end