actionagent 1.8.1 → 1.9.1

data/app/controllers/concerns/action_agent/api/mcp_dashboard_tools.rb

@@ -14,6 +14,14 @@ module ActionAgent
     # (Api::TraceReportsController). A record outside that scope reads as
     # nonexistent.
     #
+    # Two tools write: `evaluations_create` and `scenarios_merge`, so a
+    # harness that has read or browsed the app can seed the suite it then
+    # runs. Both ask ActionAgent.permitted? for :replace_scenarios as the
+    # key's user, and neither removes, disables or reorders a scenario it
+    # was not given. `explorations_submit` stores candidate scenarios for a
+    # person to review on the dashboard instead (Exploration), which needs
+    # only the key: accepting them there is what asks :replace_scenarios.
+    #
     # Names are a noun family followed by a verb (`evaluations_list`,
     # `traces_get`). Host schema tools are always `find_`, `count_` or `get_`
     # plus a model name, and agent tools are `run_<slug>`, so no host model or
@@ -28,10 +36,16 @@ module ActionAgent
       include EvaluationRunStarting
 
       NAMES = %w[
-        evaluations_list evaluations_get evaluations_run evaluation_runs_get evaluation_runs_compare
-        traces_search traces_get
+        evaluations_list evaluations_get evaluations_create scenarios_merge explorations_submit evaluations_run
+        evaluation_runs_get evaluation_runs_compare traces_search traces_get input_requests_list input_requests_answer
+        catalogs_list catalogs_import catalog_set_run
       ].freeze
 
+      # The kinds input_requests_answer answers. An approval and a secret are
+      # answered in the dashboard only: a secret must not pass through the
+      # client's model, and an approval is a person's decision.
+      MCP_ANSWERABLE_KINDS = %w[text choice].freeze
+
       LIST_LIMIT = 20
       MAX_LIST_LIMIT = 50
       SCENARIO_LIMIT = 200
@@ -47,6 +61,10 @@ module ActionAgent
       MAX_STRING = 1_000
       # Owned credentials read for scrubbing, per kind.
       SECRET_LOOKUP_LIMIT = 100
+      # The longest scenario key, scenario group or judge model a write tool
+      # accepts, as the report collector does. Each is a string column,
+      # VARCHAR(255) on MySQL.
+      MAX_LABEL = 200
 
       SELECTION_PROPERTIES = {
         scenario_ids: { type: "array", items: { type: "integer" }, description: "Replay only these scenarios, by id" },
@@ -60,6 +78,53 @@ module ActionAgent
         }
       }.freeze
 
+      # An expectation list of an explorations_submit candidate.
+      EXPLORATION_EXPECTATION_LIST = {
+        type: "array", maxItems: Exploration::MAX_ITEMS, items: { type: "string", maxLength: Exploration::MAX_LABEL }
+      }.freeze
+
+      # The scenarios evaluations_create and scenarios_merge accept, in the
+      # forms POST /api/evaluations accepts them.
+      SCENARIO_PROPERTIES = {
+        scenarios: {
+          type: "array",
+          description: "Scenarios as objects. Give `scenarios` or `scenarios_text`.",
+          items: {
+            type: "object",
+            properties: {
+              prompt: { type: "string", description: "The message a user would send the agent" },
+              key: {
+                type: "string",
+                description: "A stable key. A scenario whose key the evaluation holds is replaced by this one, " \
+                             "keeping its results, position and enabled flag; one without a key gets the next " \
+                             "unused <group>_<n>"
+              },
+              group: { type: "string", description: "The group of related scenarios this one belongs to" },
+              notes: { type: "string", description: "What a good answer does: the judge grades the answer against it" },
+              tools: { type: "array", items: { type: "string" }, description: "A passing answer calls at least one of these tools" },
+              contains: { type: "array", items: { type: "string" }, description: "Patterns the answer must contain" },
+              not_contains: { type: "array", items: { type: "string" }, description: "Patterns the answer must not contain" }
+            },
+            required: [ "prompt" ]
+          }
+        },
+        scenarios_text: {
+          type: "string",
+          description: "Scenarios as text: one message per line, `# Heading` lines starting a group, and " \
+                       "`| tools: a, b | contains: x | not_contains: y | notes: n | key: k` options after a line; or a " \
+                       "JSON array, or a YAML or JSON suite document with groups"
+        },
+        include_production_only: {
+          type: "boolean",
+          description: "Keep a suite document's production_only scenarios, which are left out by default"
+        },
+        key_prefix: {
+          type: "string",
+          description: "A namespace for generated keys: `batch2` makes the first generated Orders key batch2_orders_1. " \
+                       "Keys you give are kept as they are"
+        }
+      }.freeze
+
       DEFINITIONS = [
         {
           name: "evaluations_list",
@@ -83,6 +148,107 @@ module ActionAgent
             required: [ "evaluation_id" ]
           }
         },
+        {
+          name: "evaluations_create",
+          description: "Create an evaluation of one of this key's agents: a scenario suite, which replays its scenarios " \
+                       "through the agent, or, without scenarios, criteria scored against the agent's recorded " \
+                       "generations. Runs nothing; start a run with evaluations_run. Add scenarios later with " \
+                       "scenarios_merge.",
+          inputSchema: {
+            type: "object",
+            properties: {
+              agent: { type: "string", description: "The agent's slug or id" },
+              name: { type: "string", description: "The evaluation's name, unique among the agent's evaluations" },
+              judge_kind: {
+                type: "string", enum: Evaluation::JUDGE_KINDS,
+                description: "rules (the default) scores with rule criteria alone; llm adds a judge model; " \
+                             "judge_defined lets the judge write the criteria on the first run"
+              },
+              judge_model: { type: "string", description: "The model the judge runs on" },
+              criteria: {
+                type: "array",
+                description: "Criteria every result is scored on. Rule-based defaults (an answer is present, long " \
+                             "enough, fast enough and within a token budget) when omitted",
+                items: {
+                  type: "object",
+                  properties: {
+                    key: { type: "string", description: "The criterion's name; its type when omitted" },
+                    type: { type: "string", enum: Evaluation::CRITERION_TYPES },
+                    config: { type: "object", description: "The type's settings, such as { \"chars\": 40 } for min_length" }
+                  },
+                  required: [ "type" ]
+                }
+              },
+              compare_models: { type: "array", items: { type: "string" }, description: "Candidate models a run compares" }
+            }.merge(SCENARIO_PROPERTIES),
+            required: %w[agent name]
+          }
+        },
+        {
+          name: "scenarios_merge",
+          description: "Add scenarios to an evaluation, and update the ones whose keys it already holds. A scenario " \
+                       "the call does not name is left exactly as it is, results included; nothing is removed or " \
+                       "disabled. A scenario without a key gets the next <group>_<n> the evaluation does not use. " \
+                       "Returns the added, updated and unchanged keys.",
+          inputSchema: {
+            type: "object",
+            properties: { evaluation_id: { type: "integer", description: "The evaluation's id" } }.merge(SCENARIO_PROPERTIES),
+            required: [ "evaluation_id" ]
+          }
+        },
+        {
+          name: "explorations_submit",
+          description: "Submit candidate scenarios you found by reading or browsing a project's app, for a person to " \
+                       "review on the dashboard. Each is checked against the tools of the agent the project evaluates: " \
+                       "the result gives each candidate's verdict (answerable, needs_tool with the missing tools, or " \
+                       "unverified when the tools could not be read) so you can revise, and the review link. Nothing " \
+                       "reaches the evaluation until a person accepts it. Pass exploration_id to add to an exploration " \
+                       "you submitted earlier.",
+          inputSchema: {
+            type: "object",
+            properties: {
+              project_id: { type: "integer", description: "The project whose app you explored" },
+              evaluation_id: { type: "integer", description: "An evaluation to propose scenarios for, without a project" },
+              exploration_id: {
+                type: "integer",
+                description: "An exploration submitted from outside the dashboard for this key's owner, to add these candidates to"
+              },
+              candidates: {
+                type: "array",
+                description: "At most #{Exploration::MAX_CANDIDATES} per exploration, and " \
+                             "#{Exploration::MAX_BYTES / 1.megabyte} MiB of JSON per call and per exploration",
+                items: {
+                  type: "object",
+                  properties: {
+                    prompt: {
+                      type: "string", maxLength: Exploration::MAX_STRING,
+                      description: "A question a user of the app would ask its agent"
+                    },
+                    group: { type: "string", description: "The scenario group it belongs to" },
+                    rubric: {
+                      type: "string", maxLength: Exploration::MAX_STRING,
+                      description: "What a good answer does; the judge grades the answer against it. Not click steps"
+                    },
+                    tools: { **EXPLORATION_EXPECTATION_LIST, description: "Tools a good answer calls" },
+                    contains: { **EXPLORATION_EXPECTATION_LIST, description: "Patterns the answer must contain" },
+                    not_contains: { **EXPLORATION_EXPECTATION_LIST, description: "Patterns the answer must not contain" },
+                    provenance: {
+                      type: "object",
+                      description: "How you found it, shown to the reviewer and never to the judge",
+                      properties: {
+                        urls: { type: "array", items: { type: "string" } },
+                        steps: { type: "array", items: { type: "string" } },
+                        screenshots: { type: "array", items: { type: "string" } }
+                      }
+                    }
+                  },
+                  required: [ "prompt" ]
+                }
+              }
+            },
+            required: [ "candidates" ]
+          }
+        },
         {
           name: "evaluations_run",
           description: "Start a run of an evaluation. A scenario suite replays each scenario through the agent in the " \
@@ -153,9 +319,80 @@ module ActionAgent
             properties: { trace_id: { type: "string", description: "The trace's id, its OpenTelemetry trace id, or that id's first 8 characters" } },
             required: [ "trace_id" ]
           }
+        },
+        {
+          name: "input_requests_list",
+          description: "List the requests for input this key's paused agent runs are waiting on, newest first: each " \
+                       "request's id, kind (text, choice, confirm or secret), prompt, options, agent, run and expiry.",
+          inputSchema: {
+            type: "object",
+            properties: {
+              run_id: { type: "integer", description: "Only this run's requests" },
+              limit: { type: "integer", description: "At most this many (default #{LIST_LIMIT}, max #{MAX_LIST_LIMIT})" }
+            }
+          }
+        },
+        {
+          name: "input_requests_answer",
+          description: "Answer a text or choice request for input, which resumes its run once every request of the " \
+                       "pause is answered. A choice answer must be one of the request's options. Confirm and secret " \
+                       "requests are answered in the dashboard only.",
+          inputSchema: {
+            type: "object",
+            properties: {
+              input_request_id: { type: "integer", description: "The request's id" },
+              answer: { type: "string", description: "The answer" }
+            },
+            required: [ "input_request_id", "answer" ]
+          }
+        },
+        {
+          name: "catalogs_list",
+          description: "The owner's scenario catalogs: YAML catalogs of evaluation scenarios, each a set of products " \
+                       "(an agent or a project under test) with named sets of scenarios, with their keys, sources, " \
+                       "digests and counts.",
+          inputSchema: { type: "object", properties: {} }
+        },
+        {
+          name: "catalogs_import",
+          description: "Import a scenario catalog from its YAML document (`catalog:`, `products:` with `sets:` of " \
+                       "`scenarios:`; a suite document with `groups:` is one product). Products, sets and scenarios are " \
+                       "matched by key and replaced, so importing the same document twice changes nothing. A product's " \
+                       "`agent:` names one of the owner's agents; `repository:` or `project:` names a project. Needs the " \
+                       "replace_scenarios permission.",
+          inputSchema: {
+            type: "object",
+            properties: {
+              document: { type: "string", description: "The catalog as YAML" },
+              name: { type: "string", description: "The catalog key when the document sets no `catalog:`" }
+            },
+            required: [ "document" ]
+          }
+        },
+        {
+          name: "catalog_set_run",
+          description: "Run one set of a catalog as an evaluation of the agent under test: the product's agent, the " \
+                       "agent given, or a project's agent, against that project's running sandbox and browser when " \
+                       "project_id is given (boot the project on the dashboard first). The run comes back pending; poll " \
+                       "evaluation_runs_get with the evaluation's id.",
+          inputSchema: {
+            type: "object",
+            properties: {
+              catalog: { type: "string", description: "The catalog's key" },
+              product: { type: "string", description: "The product's key" },
+              set: { type: "string", description: "The set's key" },
+              agent: { type: "string", description: "The agent to run against, by id, slug or name; the product's own when omitted" },
+              project_id: { type: "integer", description: "A project whose booted app and browser every replay reaches" },
+              keys: { type: "array", items: { type: "string" }, description: "Replay only these scenarios, by key" },
+              models: { type: "array", items: { type: "string" }, description: "Candidate models to compare; the agent's own model when omitted" }
+            },
+            required: %w[catalog product set]
+          }
         }
       ].freeze
 
+      OBSERVED_AGENT_REFUSAL = "Observed agents are read-only — duplicate this agent to create an executable copy"
+
       # Raised by a tool for a call the client can correct. Answered as a tool
       # result with isError.
       class ToolError < StandardError; end
@@ -181,11 +418,19 @@ module ActionAgent
         case name
         when "evaluations_list" then evaluations_list_tool
         when "evaluations_get" then evaluations_get_tool
+        when "evaluations_create" then evaluations_create_tool
+        when "scenarios_merge" then scenarios_merge_tool
+        when "explorations_submit" then explorations_submit_tool
         when "evaluations_run" then evaluations_run_tool
         when "evaluation_runs_get" then evaluation_runs_get_tool
         when "evaluation_runs_compare" then evaluation_runs_compare_tool
         when "traces_search" then traces_search_tool
         when "traces_get" then traces_get_tool
+        when "input_requests_list" then input_requests_list_tool
+        when "input_requests_answer" then input_requests_answer_tool
+        when "catalogs_list" then catalogs_list_tool
+        when "catalogs_import" then catalogs_import_tool
+        when "catalog_set_run" then catalog_set_run_tool
         end
       end
 
@@ -230,6 +475,232 @@ module ActionAgent
         }
       end
 
+      # Builds the evaluation as POST /api/evaluations does, scenarios
+      # before the save, because an evaluation with neither criteria nor
+      # scenarios is invalid. Runs nothing, so neither the execution switch
+      # nor the execution quota applies.
+      def evaluations_create_tool
+        agent = resolve_tool_agent!(required_tool_argument!(:agent))
+        name = required_tool_argument!(:name).to_s.strip
+        if name.length > EvaluationReportImport::MAX_EVALUATION_NAME
+          raise ToolError, "name is longer than #{EvaluationReportImport::MAX_EVALUATION_NAME} characters"
+        end
+        raise ToolError, "judge_model is longer than #{MAX_LABEL} characters" if tool_argument(:judge_model).to_s.length > MAX_LABEL
+
+        attributes = tool_scenario_attributes
+        if attributes.size > EvaluationReportImport::MAX_SCENARIOS_PER_EVALUATION
+          raise ToolError, "Scenario limit reached (#{EvaluationReportImport::MAX_SCENARIOS_PER_EVALUATION} per " \
+                           "evaluation): this call gives #{attributes.size}"
+        end
+
+        evaluation = build_tool_evaluation(agent, name, attributes)
+        authorize_scenario_write!(evaluation)
+
+        # Locked, so two creates for one agent cannot both pass the limit.
+        agent.with_lock do
+          if agent.evaluations.count >= EvaluationReportImport::MAX_EVALUATIONS_PER_AGENT
+            raise ToolError, "Evaluation limit reached (#{EvaluationReportImport::MAX_EVALUATIONS_PER_AGENT} for this " \
+                             "agent); remove evaluations on the dashboard first"
+          end
+          raise ToolError, evaluation.errors.full_messages.to_sentence unless evaluation.save
+
+          # Checked once saved, because a host adapter resolver is handed a
+          # persisted evaluation; the refusal rolls the save back.
+          refuse_unexecutable_suite!(evaluation) if attributes.any?
+        end
+
+        {
+          evaluation: EvaluationSerializer.summary(evaluation),
+          added: evaluation.scenarios.ordered.pluck(:key)
+        }
+      end
+
+      # A scenario given without a key gets the next <group>_<n> the suite
+      # does not hold, so a second batch never lands on the first one's keys.
+      def scenarios_merge_tool
+        evaluation = find_tool_evaluation!
+        authorize_scenario_write!(evaluation)
+        refuse_unexecutable_suite!(evaluation)
+
+        merged = evaluation.with_lock do
+          attributes = tool_scenario_attributes(reserved_keys: evaluation.scenarios.pluck(:key))
+          raise ToolError, "Give the scenarios to merge as scenarios or scenarios_text" if attributes.empty?
+
+          evaluation.merge_scenarios!(attributes.map { |attrs| attrs.except("position") },
+                                      limit: EvaluationReportImport::MAX_SCENARIOS_PER_EVALUATION)
+        end
+
+        {
+          evaluation: { id: evaluation.id, name: evaluation.name },
+          added: merged[:added],
+          updated: merged[:updated],
+          unchanged: merged[:unchanged],
+          scenario_count: evaluation.scenarios.count
+        }
+      rescue Evaluation::ScenarioLimitExceeded, ActiveRecord::RecordInvalid => e
+        raise ToolError, e.message
+      end
+
+      # Candidates are stored as Exploration#add_candidates! stores them:
+      # scrubbed, bounded and checked against the target agent's tools.
+      def explorations_submit_tool
+        list = tool_argument(:candidates)
+        raise ToolError, "candidates must be an array of candidate objects" unless list.is_a?(Array)
+
+        list = list.map { |entry| entry.respond_to?(:to_unsafe_h) ? entry.to_unsafe_h : entry }
+        exploration = tool_exploration
+        stored = exploration.persisted? ? exploration.add_candidates!(list) : exploration.save_with_candidates!(list)
+
+        {
+          exploration: exploration.summary,
+          candidates: stored.map { |candidate| candidate.slice("id", "prompt", "verdict", "missing_tools", "state") },
+          review_url: "#{request.base_url}#{request.script_name}/explorations/#{exploration.id}"
+        }
+      rescue Exploration::InvalidCandidate, Exploration::CandidateLimitExceeded => e
+        raise ToolError, e.message
+      end
+
+      # The external exploration exploration_id names, or a new one for the
+      # project or evaluation the call names.
+      def tool_exploration
+        if (id = tool_argument(:exploration_id)).present?
+          return owned(Exploration).find_by(id: id.to_s, source: "external") ||
+              raise(ToolError, "No exploration #{id.to_s.truncate(32)} submitted from outside the dashboard was found")
+        end
+
+        attributes = { source: "external", status: "review", started_at: Time.current, finished_at: Time.current }
+        if (project_id = tool_argument(:project_id)).present?
+          project = owned(Project).find_by(id: project_id.to_s) or
+            raise ToolError, "No project #{project_id.to_s.truncate(32)} was found"
+          Exploration.build_for(project: project, **attributes)
+        elsif tool_argument(:evaluation_id).present?
+          evaluation = find_tool_evaluation!
+          refuse_unexecutable_suite!(evaluation)
+          Exploration.build_for(evaluation: evaluation, user: current_user, account: current_account, **attributes)
+        else
+          raise MCPController::McpError.new("Give project_id, evaluation_id or exploration_id", MCPController::JSONRPC_INVALID_PARAMS)
+        end
+      end
+
+      def build_tool_evaluation(agent, name, attributes)
+        judge_kind = tool_argument(:judge_kind).to_s.presence || "rules"
+        criteria = tool_criteria
+        criteria = EvaluationsController::DEFAULT_CRITERIA.deep_dup if criteria.empty? && judge_kind != "judge_defined"
+        compare_models = tool_compare_models
+
+        evaluation = Evaluation.new(
+          agent: agent,
+          name: name,
+          judge_kind: judge_kind,
+          judge_model: tool_argument(:judge_model).to_s.presence,
+          criteria: criteria,
+          config: compare_models.any? ? { "compare_models" => compare_models } : {}
+        )
+        attributes.each_with_index do |attrs, index|
+          evaluation.scenarios.build(
+            key: attrs["key"], prompt: attrs["prompt"], group: attrs["group"], notes: attrs["notes"],
+            expectations: attrs["expectations"] || {}, position: index
+          )
+        end
+        evaluation
+      end
+
+      # The scenarios the call gives, parsed as POST /api/evaluations parses
+      # them; empty when it gives none. Generated keys skip +reserved_keys+.
+      def tool_scenario_attributes(reserved_keys: [])
+        list = tool_argument(:scenarios)
+        text = tool_argument(:scenarios_text).to_s
+        return [] if list.blank? && text.blank?
+
+        source =
+          if list.present?
+            raise ToolError, "scenarios must be an array of scenario objects" unless list.is_a?(Array)
+
+            list.map { |entry| entry.respond_to?(:to_unsafe_h) ? entry.to_unsafe_h : entry }.to_json
+          else
+            text
+          end
+        parsed = ActiveAgent::Evals::ScenarioParser.parse(
+          source,
+          include_production_only: boolean_argument(:include_production_only),
+          reserved_keys: reserved_keys,
+          key_prefix: tool_argument(:key_prefix).to_s
+        )
+        raise ToolError, "No scenarios matched the import. Check the text or the include_production_only selection." if parsed.empty?
+
+        parsed.each { |attrs| refuse_oversized_scenario!(attrs) }
+        refuse_oversized_batch!(parsed)
+        parsed
+      rescue ActiveAgent::Evals::ScenarioParser::ParseError => e
+        raise ToolError, e.message
+      end
+
+      # Raises for a scenario whose key or group is longer than MAX_LABEL
+      # characters, or whose prompt or notes are larger than a MySQL TEXT
+      # column holds.
+      def refuse_oversized_scenario!(attrs)
+        label = attrs["key"].to_s.truncate(64)
+        %w[key group].each do |field|
+          next if attrs[field].to_s.length <= MAX_LABEL
+
+          raise ToolError, "Scenario #{label}: #{field} is longer than #{MAX_LABEL} characters"
+        end
+        %w[prompt notes].each do |field|
+          next if attrs[field].to_s.bytesize <= EvaluationReportImport::TEXT_BYTES
+
+          raise ToolError, "Scenario #{label}: #{field} is larger than #{EvaluationReportImport::TEXT_BYTES} bytes"
+        end
+      end
+
+      # Raises when the scenarios one call gives total more than the report
+      # collector accepts in one report, measured as the JSON of the fields
+      # a scenario stores.
+      def refuse_oversized_batch!(parsed)
+        bytes = parsed.sum { |attrs| attrs.slice("key", "group", "prompt", "notes", "expectations").to_json.bytesize }
+        return if bytes <= EvaluationReportImport::MAX_BYTES
+
+        raise ToolError, "The scenarios in one call may total #{EvaluationReportImport::MAX_BYTES / 1.megabyte} MiB and " \
+                         "these total #{(bytes.to_f / 1.megabyte).round(1)} MiB; give them over several scenarios_merge calls"
+      end
+
+      def tool_criteria
+        raw = tool_argument(:criteria)
+        return [] if raw.blank?
+        unless raw.is_a?(Array) && raw.all? { |criterion| criterion.respond_to?(:permit) }
+          raise ToolError, "criteria must be an array of { key, type, config } objects"
+        end
+
+        raw.map do |criterion|
+          criterion.permit(:key, :type, config: {}).to_h.tap do |c|
+            c["key"] = c["key"].presence || c["type"]
+            c["config"] ||= {}
+          end
+        end
+      end
+
+      def tool_compare_models
+        models = tool_argument(:compare_models)
+        models = models.to_s.split(",") unless models.is_a?(Array)
+        models.map(&:to_s).map(&:strip).reject(&:blank?)
+      end
+
+      # Raises unless the key's user may change +evaluation+'s scenarios. With
+      # a permission checker set, a multi-tenant key that records no user is
+      # refused; with none, every key may (ActionAgent.permitted?).
+      def authorize_scenario_write!(evaluation)
+        return if ActionAgent.permitted?(current_user, :replace_scenarios, evaluation)
+
+        raise MCPController::McpError.new(
+          "This key's user may not change evaluation scenarios (replace_scenarios)", MCPController::JSONRPC_FORBIDDEN
+        )
+      end
+
+      # Raises for a suite that could never run: its agent is observed and
+      # no host adapter replays it, the case evaluations_run refuses.
+      def refuse_unexecutable_suite!(evaluation)
+        raise ToolError, OBSERVED_AGENT_REFUSAL if unexecutable_scenario_run?(evaluation)
+      end
+
       # Checked in the order Api::EvaluationsController#run checks a request:
       # the execution switch, an executable agent, the execution quota, then
       # the sandbox. Only a scenario suite executes the agent, so a sampling
@@ -239,9 +710,7 @@ module ActionAgent
         evaluation = find_tool_evaluation!
         if evaluation.scenario_suite?
           raise MCPController::McpError.new("Agent execution is disabled on this dashboard") unless ActionAgent.execution_enabled?
-          if unexecutable_scenario_run?(evaluation)
-            raise ToolError, "Observed agents are read-only — duplicate this agent to create an executable copy"
-          end
+          raise ToolError, OBSERVED_AGENT_REFUSAL if unexecutable_scenario_run?(evaluation)
           if (denial = ActionAgent.quota_denial(current_owner, :execution)).present?
             raise MCPController::McpError.new(denial.is_a?(Hash) ? denial[:message] || denial["message"] : denial)
           end
@@ -380,6 +849,40 @@ module ActionAgent
         PayloadBounds.bound(payload, max_string: MAX_STRING, max_items: MAX_TRACE_SPANS)
       end
 
+      def input_requests_list_tool
+        InputRequest.expire_overdue!(owner_input_requests)
+        scope = owner_input_requests.pending.for_listing.includes(:subject).recent
+        if (run_id = tool_argument(:run_id)).present?
+          scope = scope.where(subject_type: AgentRun.polymorphic_name, subject_id: run_id.to_s)
+        end
+        limit = integer_argument(:limit, default: LIST_LIMIT, min: 1, max: MAX_LIST_LIMIT)
+
+        { input_requests: scope.limit(limit).map { |request| InputRequestSerializer.call(request) } }
+      end
+
+      # Answered as the key's user, under the permission check the dashboard
+      # applies.
+      def input_requests_answer_tool
+        id = tool_argument(:input_request_id)
+        raise MCPController::McpError.new("Missing required argument: input_request_id", MCPController::JSONRPC_INVALID_PARAMS) if id.blank?
+        raise MCPController::McpError.new("Agent execution is disabled on this dashboard") unless ActionAgent.execution_enabled?
+
+        request = owner_input_requests.find_by(id: id.to_s) or raise ToolError, "No input request #{id.to_s.truncate(32)} was found"
+        unless MCP_ANSWERABLE_KINDS.include?(request.kind)
+          raise ToolError, "Input request #{request.id} is a #{request.kind} request, which is answered in the dashboard, " \
+                           "not over MCP: open the run #{request.subject_id} there to answer it"
+        end
+        raise ToolError, "You do not have permission to answer input request #{request.id}" unless request.answerable_by?(current_user)
+
+        begin
+          request.answer!(tool_argument(:answer), user: current_user)
+        rescue InputRequest::Conflict, InputRequest::InvalidAnswer => e
+          raise ToolError, e.message
+        end
+
+        { input_request: InputRequestSerializer.call(request) }
+      end
+
       # `trace_id` names the result's telemetry trace for traces_get, and is
       # nil when telemetry recorded none for the replay.
       def result_row(result, recorded_trace_ids)
@@ -422,10 +925,7 @@ module ActionAgent
       # fixed, still_failing.
       def comparison_change(before, after)
         return "removed" if after.nil?
-        return after.passed? ? "added_pass" : "added_fail" if before.nil?
-        return after.passed? ? "unchanged_pass" : "regressed" if before.passed?
-
-        after.passed? ? "fixed" : "still_failing"
+        EvaluationFixComparison.change(before, after)
       end
 
       # Fix item links at the facade's absolute mount, since a client has no
@@ -438,9 +938,88 @@ module ActionAgent
         Evaluation.joins(:agent).where(agent: owner_agents)
       end
 
+      # The owner's scenario catalogs, as GET /api/scenario_catalogs lists them.
+      def catalogs_list_tool
+        catalogs = owned(ScenarioCatalog).ordered.includes(products: { sets: :scenarios }).limit(MAX_LIST_LIMIT)
+        {
+          catalogs: catalogs.map { |catalog| ScenarioCatalogSerializer.summary(catalog) },
+          storage_available: ScenarioCatalog.attachments_available?
+        }
+      end
+
+      # Imports a catalog document as POST /api/scenario_catalogs does: the
+      # owner's agents and projects resolve the products' targets.
+      def catalogs_import_tool
+        document = required_tool_argument!(:document).to_s
+        raise ToolError, "document is larger than #{ScenarioCatalog::MAX_BYTES} bytes" if document.bytesize > ScenarioCatalog::MAX_BYTES
+
+        authorize_catalog_write!(ScenarioCatalog.new)
+        catalog = ScenarioCatalogImport.new(
+          owner: current_owner, document: document, name: tool_argument(:name).to_s.presence,
+          source_kind: "api", agents: owner_agents, projects: owned(Project)
+        ).call
+
+        {
+          catalog: ScenarioCatalogSerializer.summary(catalog),
+          products: catalog.products.map do |product|
+            { key: product.key, agent: product.agent&.name, project_id: product.project_id,
+              sets: product.sets.map { |set| { key: set.key, scenario_count: set.scenarios.size } } }
+          end
+        }
+      rescue ScenarioCatalogImport::Invalid => e
+        raise ToolError, e.message
+      end
+
+      # Runs one set as POST /api/scenario_catalogs/:id/sets/:set_id/run
+      # does, gated as evaluations_run is. A project's sandbox must already
+      # be running: booting one is a dashboard action, not a tool's.
+      def catalog_set_run_tool
+        catalog = owned(ScenarioCatalog).find_by(key: required_tool_argument!(:catalog).to_s) ||
+          raise(ToolError, "No catalog #{tool_argument(:catalog).inspect}; catalogs_list names them")
+        product = catalog.products.find_by(key: required_tool_argument!(:product).to_s) ||
+          raise(ToolError, "No product #{tool_argument(:product).inspect} in catalog #{catalog.key}")
+        set = product.sets.find_by(key: required_tool_argument!(:set).to_s) ||
+          raise(ToolError, "No set #{tool_argument(:set).inspect} in product #{product.key}")
+        authorize_catalog_write!(catalog)
+
+        project = tool_argument(:project_id).present? ? owned(Project).find_by(id: tool_argument(:project_id)) : nil
+        raise ToolError, "No project #{tool_argument(:project_id)}" if tool_argument(:project_id).present? && project.nil?
+        agent = tool_argument(:agent).present? ? resolve_tool_agent!(tool_argument(:agent)) : nil
+        agent ||= project&.target_agent || product.target_agent(owner_agents)
+        raise ToolError, "Choose the agent to run against (agent) or a project (project_id)" if agent.nil?
+        raise ToolError, OBSERVED_AGENT_REFUSAL if agent.observed?
+        raise MCPController::McpError.new("Agent execution is disabled on this dashboard") unless ActionAgent.execution_enabled?
+        if (denial = ActionAgent.quota_denial(current_owner, :execution)).present?
+          raise MCPController::McpError.new(denial.is_a?(Hash) ? denial[:message] || denial["message"] : denial)
+        end
+        if project && project.current_sandbox_session.nil?
+          raise ToolError, "Project #{project.name} has no running sandbox: boot it on the dashboard first"
+        end
+
+        selection = evaluation_run_selection(tool_arguments).except(:scenario_ids, :group)
+        run = set.run!(agent: agent, project: project, mount_url: "#{request.base_url}#{request.script_name}", **selection)
+
+        {
+          set: set.reload.summary,
+          evaluation: { id: run.evaluation_id, name: run.evaluation.name },
+          run: EvaluationSerializer.run_summary(run, number: EvaluationSerializer.run_number(run.evaluation, run))
+            .merge(selection: run.selection),
+          background: true
+        }
+      rescue ScenarioSet::NoAgent => e
+        raise ToolError, e.message
+      end
+
+      def authorize_catalog_write!(catalog)
+        return if ActionAgent.permitted?(current_user, :replace_scenarios, catalog)
+
+        raise MCPController::McpError.new(
+          "This key's user may not change scenario catalogs (replace_scenarios)", MCPController::JSONRPC_FORBIDDEN
+        )
+      end
+
       def find_tool_evaluation!
-        id = tool_argument(:evaluation_id)
-        raise MCPController::McpError.new("Missing required argument: evaluation_id", MCPController::JSONRPC_INVALID_PARAMS) if id.blank?
+        id = required_tool_argument!(:evaluation_id)
 
         dashboard_evaluations.find_by(id: id.to_s) or raise ToolError, "No evaluation #{id.to_s.truncate(32)} was found"
       end
@@ -472,6 +1051,14 @@ module ActionAgent
         tool_arguments[name]
       end
 
+      # The argument, or a JSON-RPC invalid-params error when it is blank.
+      def required_tool_argument!(name)
+        value = tool_argument(name)
+        raise MCPController::McpError.new("Missing required argument: #{name}", MCPController::JSONRPC_INVALID_PARAMS) if value.blank?
+
+        value
+      end
+
       # An integer argument clamped into [min, max], or +default+ when absent
       # or not a number.
       def integer_argument(name, default:, min: nil, max: nil)
@@ -489,21 +1076,39 @@ module ActionAgent
       end
 
       # Every credential the owner holds, masked out of each tool's output:
-      # this key's token, provider keys, the GitHub token, and each checkout
-      # sandbox's runtime token. None of them belongs in these payloads; this
-      # keeps one that leaked into a recorded output or a trace from being
-      # handed on. A failed lookup masks nothing rather than failing the call.
+      # this key's token and the #owner_credentials (provider keys, the GitHub
+      # token, runtime tokens, the owner's other API keys and ingest key),
+      # plus every member's personal provider key, each checkout sandbox's
+      # browser token, and the values of projects' secrets with their
+      # encodings, which an app tool can echo into a recorded output. None of
+      # them belongs in these payloads; this keeps one that leaked into a
+      # recorded output or a trace from being handed on. A failed lookup
+      # masks nothing rather than failing the call.
       def dashboard_tool_secrets
         @dashboard_tool_secrets ||= [
           @api_key&.token,
-          *owned(ProviderKey).limit(SECRET_LOOKUP_LIMIT).pluck(:credential, :api_key).flatten,
-          *owned(GithubConnection).limit(SECRET_LOOKUP_LIMIT).pluck(:access_token),
-          *owned(SandboxSession).where.not(runtime_mcp_token: nil).order(id: :desc).limit(SECRET_LOOKUP_LIMIT).pluck(:runtime_mcp_token)
+          *owner_credentials,
+          *ProviderCredentials.secrets_for(current_owner, limit: SECRET_LOOKUP_LIMIT),
+          *owned(SandboxSession).where.not(browser_token: nil).order(id: :desc).limit(SECRET_LOOKUP_LIMIT).pluck(:browser_token),
+          *project_secret_values
         ].compact
       rescue StandardError => e
         Rails.logger.warn("[ActionAgent] MCP secret lookup failed: #{e.class}: #{e.message}")
         @dashboard_tool_secrets = [ @api_key&.token ].compact
       end
+
+      # The owner's entered project secrets, with their encodings and a
+      # sign-in secret's parts (ProjectSecret#scrub_parts); a secret that
+      # uses the organization's key is masked as that provider key.
+      # Empty on an install that has not run the projects migration, so the
+      # other credentials are still masked there.
+      def project_secret_values
+        secrets = owned(ProjectSecret).where(source: "entered").order(id: :desc).limit(SECRET_LOOKUP_LIMIT)
+        SecretScrubber.with_encodings(secrets.flat_map(&:scrub_parts))
+      rescue StandardError => e
+        Rails.logger.warn("[ActionAgent] MCP project secret lookup failed: #{e.class}")
+        []
+      end
     end
   end
 end