actionagent 1.2.2 → 1.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (67) hide show
  1. checksums.yaml +4 -4
  2. data/README.md +14 -3
  3. data/app/assets/builds/action_agent.css +1 -1
  4. data/app/assets/builds/action_agent.js +69 -43
  5. data/app/controllers/action_agent/api/agent_runs_controller.rb +28 -8
  6. data/app/controllers/action_agent/api/agents_controller.rb +191 -56
  7. data/app/controllers/action_agent/api/analytics_controller.rb +31 -9
  8. data/app/controllers/action_agent/api/base_controller.rb +16 -0
  9. data/app/controllers/action_agent/api/dashboard_assistant_controller.rb +83 -0
  10. data/app/controllers/action_agent/api/evaluations_controller.rb +252 -7
  11. data/app/controllers/action_agent/api/interaction_messages_controller.rb +98 -0
  12. data/app/controllers/action_agent/api/mcp_controller.rb +13 -3
  13. data/app/controllers/action_agent/api/mcp_servers_controller.rb +28 -8
  14. data/app/controllers/action_agent/api/metrics_controller.rb +44 -11
  15. data/app/controllers/action_agent/api/provider_models_controller.rb +1 -1
  16. data/app/controllers/action_agent/api/sandboxes_controller.rb +6 -0
  17. data/app/controllers/action_agent/api/session_recordings_controller.rb +34 -12
  18. data/app/controllers/action_agent/api/templates_controller.rb +25 -21
  19. data/app/controllers/action_agent/api/traces_controller.rb +25 -5
  20. data/app/controllers/action_agent/api/usage_controller.rb +20 -0
  21. data/app/controllers/action_agent/application_controller.rb +25 -2
  22. data/app/controllers/action_agent/dashboard_controller.rb +3 -1
  23. data/app/controllers/concerns/action_agent/api/agent_serialization.rb +53 -0
  24. data/app/jobs/action_agent/agent_execution_job.rb +40 -20
  25. data/app/jobs/action_agent/application_job.rb +7 -3
  26. data/app/jobs/action_agent/evaluation_run_job.rb +18 -0
  27. data/app/jobs/action_agent/sandbox_cleanup_job.rb +13 -10
  28. data/app/models/action_agent/agent.rb +74 -23
  29. data/app/models/action_agent/agent_run.rb +99 -0
  30. data/app/models/action_agent/agent_template.rb +22 -7
  31. data/app/models/action_agent/evaluation.rb +64 -4
  32. data/app/models/action_agent/evaluation_run.rb +190 -2
  33. data/app/models/action_agent/evaluation_scenario.rb +59 -0
  34. data/app/models/action_agent/evaluation_scenario_result.rb +86 -0
  35. data/app/models/action_agent/recording_action.rb +11 -7
  36. data/app/models/action_agent/sandbox_session.rb +1 -1
  37. data/app/models/action_agent/session_recording.rb +31 -8
  38. data/app/models/action_agent/telemetry_trace.rb +126 -3
  39. data/app/models/concerns/action_agent/adapter_aware.rb +19 -0
  40. data/app/models/concerns/action_agent/ownable.rb +15 -2
  41. data/app/queries/action_agent/metrics_report.rb +498 -0
  42. data/app/serializers/action_agent/agent_message_serializer.rb +1 -0
  43. data/app/services/action_agent/agent_execution_service.rb +294 -16
  44. data/app/services/action_agent/agent_registrar.rb +7 -6
  45. data/app/services/action_agent/agent_toolbox.rb +49 -7
  46. data/app/services/action_agent/dashboard_assistant_service.rb +342 -0
  47. data/app/services/action_agent/evaluation_evidence.rb +234 -0
  48. data/app/services/action_agent/evaluation_runner_service.rb +13 -3
  49. data/app/services/action_agent/evaluation_tool_resolver.rb +162 -0
  50. data/app/services/action_agent/mcp_catalog.rb +46 -8
  51. data/app/services/action_agent/mcp_client.rb +167 -0
  52. data/app/services/action_agent/mcp_recording_middleware.rb +2 -2
  53. data/app/services/action_agent/mcp_tool_dispatcher.rb +116 -0
  54. data/app/services/action_agent/playwright_mcp_client.rb +11 -126
  55. data/app/services/action_agent/sandbox_orchestrator.rb +12 -1
  56. data/app/services/action_agent/scenario_evaluation_runner.rb +260 -0
  57. data/app/services/action_agent/tool_discovery.rb +22 -8
  58. data/config/routes.rb +36 -3
  59. data/lib/action_agent/assistant_request_filter.rb +22 -0
  60. data/lib/action_agent/engine.rb +106 -19
  61. data/lib/action_agent/version.rb +1 -1
  62. data/lib/action_agent.rb +104 -6
  63. data/lib/generators/action_agent/install_generator.rb +20 -7
  64. data/lib/generators/action_agent/templates/action_agent.rb.erb +12 -0
  65. data/lib/generators/action_agent/templates/create_active_agent_evaluation_scenarios.rb.erb +79 -0
  66. data/lib/tasks/action_agent.rake +9 -0
  67. metadata +22 -5
@@ -1,20 +1,10 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  module ActionAgent
4
- require "net/http"
5
- require "resolv"
6
-
7
- # Minimal MCP client (streamable HTTP transport) for a Playwright MCP
8
- # server — typically `npx @playwright/mcp --port 8931` running beside the
9
- # app in development, or a sandbox-provisioned browser container in
10
- # production. Speaks just enough JSON-RPC for tools/call: initialize once
11
- # per process, then call tools under the session id the server hands back.
12
- class PlaywrightMcpClient
4
+ # The Playwright MCP server as one configured MCPClient: a process-wide
5
+ # instance pointed at the url PLAYWRIGHT_MCP_URL names.
6
+ class PlaywrightMCPClient < MCPClient
13
7
  DEFAULT_URL = ENV.fetch("PLAYWRIGHT_MCP_URL", "http://host.orb.internal:8931/mcp")
14
- OPEN_TIMEOUT_SECONDS = 5
15
- READ_TIMEOUT_SECONDS = 60
16
-
17
- class Error < StandardError; end
18
8
 
19
9
  def self.instance
20
10
  @instance ||= new
@@ -25,124 +15,19 @@ module ActionAgent
25
15
  end
26
16
 
27
17
  def initialize(url: DEFAULT_URL)
28
- @uri = URI(url)
29
- @mutex = Mutex.new
18
+ super(url: url, label: "Playwright")
30
19
  end
31
20
 
32
- # Returns { text:, is_error: } the tool result's text content.
21
+ # Restarting the shared instance on an unreachable server is this
22
+ # subclass's concern: the next call re-initializes rather than reusing a
23
+ # session the server has forgotten. The message names the local fix.
33
24
  def call_tool(name, arguments = {})
34
- Rails.logger.debug("[PlaywrightMcpClient] call #{name} args=#{arguments.inspect[0, 200]}")
35
- ensure_session!
36
- response = post(
37
- { jsonrpc: "2.0", id: next_id, method: "tools/call",
38
- params: { name: name, arguments: arguments } },
39
- session: @session_id
40
- )
41
- result = response["result"]
42
- unless result
43
- Rails.logger.warn("[PlaywrightMcpClient] #{name} unexpected response: #{response.inspect[0, 500]}")
44
- raise Error, (response.dig("error", "message") || "empty MCP response")
45
- end
46
-
47
- text = Array(result["content"]).filter_map { |block| block["text"] }.join("\n")
48
- { text: text, is_error: result["isError"] ? true : false }
49
- rescue Errno::ECONNREFUSED, Errno::EHOSTUNREACH, Net::OpenTimeout, SocketError => e
25
+ super
26
+ rescue MCPClient::Error => e
50
27
  self.class.reset!
51
- raise Error, "Playwright MCP server unreachable at #{@uri} (#{e.class}): start it with `npx @playwright/mcp --port 8931`"
52
- end
53
-
54
- private
55
-
56
- def ensure_session!
57
- @mutex.synchronize do
58
- next if @session_id
59
-
60
- _body, response = post_raw(
61
- { jsonrpc: "2.0", id: next_id, method: "initialize",
62
- params: { protocolVersion: "2025-03-26", capabilities: {},
63
- clientInfo: { name: "activeagents", version: "1.0" } } }
64
- )
65
- @session_id = response["mcp-session-id"]
66
- raise Error, "MCP server did not return a session id" unless @session_id
67
-
68
- post({ jsonrpc: "2.0", method: "notifications/initialized" }, session: @session_id)
69
- end
70
- end
71
-
72
- def post(payload, session: nil)
73
- body, _response = post_raw(payload, session: session)
74
- body
75
- end
76
-
77
- def post_raw(payload, session: nil)
78
- # Tool calls run inside the provider SDK's streaming enumerator — a
79
- # fiber, where Net::HTTP reads of SSE bodies misbehave (headers arrive,
80
- # body comes back empty). A dedicated thread always does real blocking
81
- # IO outside any fiber/scheduler context.
82
- Thread.new { blocking_post_raw(payload, session: session) }.value
83
- end
84
-
85
- def blocking_post_raw(payload, session: nil)
86
- http = Net::HTTP.new(@uri.host, @uri.port)
87
- # Container->host bridge hostnames (host.orb.internal) publish an IPv6
88
- # address whose path doesn't reach the server; dual-stack connects then
89
- # fail intermittently. Pin to IPv4 while keeping the Host header.
90
- if (ipv4 = ipv4_address)
91
- http.ipaddr = ipv4
92
- end
93
- http.open_timeout = OPEN_TIMEOUT_SECONDS
94
- http.read_timeout = READ_TIMEOUT_SECONDS
95
- request = Net::HTTP::Post.new(@uri.request_uri)
96
- request["Content-Type"] = "application/json"
97
- request["Accept"] = "application/json, text/event-stream"
98
- request["Mcp-Session-Id"] = session if session
99
- request.body = payload.to_json
100
-
101
- response = http.request(request)
102
- Rails.logger.debug(
103
- "[PlaywrightMcpClient] #{payload[:method]} -> #{response.code} " \
104
- "ct=#{response['Content-Type']} bytes=#{response.body.to_s.bytesize} session=#{session ? 'yes' : 'no'}"
105
- )
106
- unless response.code.to_i.between?(200, 299)
107
- Rails.logger.warn("[PlaywrightMcpClient] HTTP #{response.code}: #{response.body.to_s[0, 300]}")
108
- raise Error, "MCP server returned HTTP #{response.code}"
109
- end
110
-
111
- parsed = parse_body(response)
112
- if parsed.empty? && payload[:id]
113
- Rails.logger.warn("[PlaywrightMcpClient] unparsed body (#{response['Content-Type']}): #{response.body.to_s[0, 500]}")
114
- end
115
- [ parsed, response ]
116
- end
117
-
118
- # Streamable HTTP answers as plain JSON or as an SSE stream whose data:
119
- # lines carry the JSON-RPC response — accept both.
120
- def parse_body(response)
121
- body = response.body.to_s
122
- return {} if body.empty?
123
-
124
- if response["Content-Type"].to_s.include?("text/event-stream")
125
- body.lines
126
- .select { |line| line.start_with?("data:") }
127
- .filter_map { |line| JSON.parse(line.delete_prefix("data:").strip) rescue nil }
128
- .find { |json| json["result"] || json["error"] } || {}
129
- else
130
- JSON.parse(body)
131
- end
132
- rescue JSON::ParserError
133
- {}
134
- end
135
-
136
- def ipv4_address
137
- return @ipv4_address if defined?(@ipv4_address)
138
-
139
- @ipv4_address = Resolv.getaddresses(@uri.host).find { |address| address =~ Resolv::IPv4::Regex }
140
- rescue Resolv::ResolvError
141
- @ipv4_address = nil
142
- end
28
+ raise Error, "#{e.message}: start it with `npx @playwright/mcp --port 8931`" if e.message.include?("unreachable")
143
29
 
144
- def next_id
145
- @id = (@id || 0) + 1
30
+ raise
146
31
  end
147
32
  end
148
33
  end
@@ -52,7 +52,18 @@ module ActionAgent
52
52
  # as sandbox_service, falling back to the in-memory one.
53
53
  def self.default_backend
54
54
  name = ENV["SANDBOX_BACKEND"].presence || ActionAgent.sandbox_service.to_s
55
- backends.key?(name) ? name : "mock"
55
+ return name if backends.key?(name)
56
+
57
+ # Substituting the mock silently made a misconfigured operator's
58
+ # sandbox "runs" succeed against nothing real.
59
+ if name.present? && name != "mock"
60
+ Rails.logger.warn(
61
+ "[ActionAgent] sandbox backend #{name.inspect} is not registered " \
62
+ "(ActionAgent.sandbox_backends knows #{backends.keys.inspect}); " \
63
+ "using the in-memory mock backend, which runs nothing."
64
+ )
65
+ end
66
+ "mock"
56
67
  end
57
68
 
58
69
  def initialize(backend: nil)
@@ -0,0 +1,260 @@
1
+ # frozen_string_literal: true
2
+
3
+ module ActionAgent
4
+ # Runs a scenario evaluation: replays every selected scenario through the
5
+ # agent once per candidate model and writes one EvaluationScenarioResult per
6
+ # scenario × model plus a per-model summary and verdict on the run.
7
+ #
8
+ # The scoring, fault diagnosis and roll-up are ActiveAgent::Evals'; this
9
+ # class supplies what only the dashboard knows — how to run the agent
10
+ # (Agent#test_execute with a model override), where to persist each result,
11
+ # how to price tokens, and which judge model the owner has credentials for.
12
+ #
13
+ # The run's `scores` keep the shape the Evaluations UI renders — criterion
14
+ # => stats, or criterion => { model => stats } when comparing — and add
15
+ # underscore-prefixed summaries:
16
+ #
17
+ # "_models" — per model: pass rate, mean score, latency, tokens, cost, fault counts
18
+ # "_recommendations" — faults grouped across scenarios with the fix each calls for
19
+ # "_verdict" — the best model and why (judge-written when a judge is available)
20
+ # "_selection" — the scenarios and models this run covered
21
+ class ScenarioEvaluationRunner < EvaluationRunnerService
22
+ Evals = ActiveAgent::Evals
23
+
24
+ # `run` is an EvaluationRun created ahead of time (by run_later!, so the
25
+ # UI can show it pending while the job waits); absent, one is created here.
26
+ def self.call(evaluation, selection: {}, run: nil)
27
+ new(evaluation, selection: selection, run: run).call
28
+ end
29
+
30
+ def initialize(evaluation, selection: {}, run: nil)
31
+ super(evaluation)
32
+ @selection = (selection || {}).to_h.with_indifferent_access
33
+ @run = run
34
+ end
35
+
36
+ def call
37
+ scenarios = selected_scenarios
38
+ specs = model_specs
39
+ run = @run || @evaluation.evaluation_runs.create!(status: :pending)
40
+ run.update!(status: :running, selection: selection_summary(scenarios, specs))
41
+
42
+ if scenarios.empty?
43
+ run.update!(status: :failed, error_message: "No scenarios selected — add scenarios to the evaluation or widen the selection",
44
+ completed_at: Time.current)
45
+ return run
46
+ end
47
+
48
+ records = scenarios.index_by(&:key)
49
+ tasks = scenarios.map { |scenario| Evals::Scenario.from_hash(scenario.as_json_summary) }
50
+ expected = tasks.product(specs).map { |task, spec| [ task.key, spec.label ] }
51
+ # Every result tests membership twice, so look the pairs up in a set;
52
+ # `expected` stays an array for the completeness comparison below, and
53
+ # `persisted` sorts the same way either way.
54
+ allowed = expected.to_set
55
+ persisted = Set.new
56
+ on_result = lambda do |result|
57
+ pair = [ result.scenario.key, result.label ]
58
+ raise ArgumentError, "unexpected or duplicate scenario evaluation result" unless allowed.include?(pair) && !persisted.include?(pair)
59
+
60
+ persist(run, records.fetch(result.scenario.key), result)
61
+ persisted << pair
62
+ end
63
+ adapter = ActionAgent.scenario_evaluation_adapter_resolver&.call(@evaluation)
64
+ report = if adapter
65
+ raise ArgumentError, "scenario evaluation adapter must be callable" unless adapter.respond_to?(:call)
66
+
67
+ adapter.call(evaluation: @evaluation, owner: owner, scenarios: tasks, models: specs, on_result: on_result)
68
+ else
69
+ ensure_judge_defined_kpis! if @evaluation.judge_defined?
70
+ default_report(tasks, specs, on_result)
71
+ end
72
+ raise ArgumentError, "scenario evaluation adapter must return an ActiveAgent::Evals::Report" unless report.is_a?(Evals::Report)
73
+ reported = report.results.map { |result| [ result.scenario.key, result.label ] }
74
+ unless reported.sort == expected.sort && persisted.sort == expected.sort
75
+ raise ArgumentError, "scenario evaluation adapter must report and persist every selected scenario and model"
76
+ end
77
+
78
+ run.update!(
79
+ status: :complete,
80
+ scores: scores_for(report, run),
81
+ samples_evaluated: report.results.size,
82
+ samples_passed: report.results.count(&:passed?),
83
+ completed_at: Time.current
84
+ )
85
+ run
86
+ rescue StandardError => e
87
+ run&.update!(status: :failed, error_message: e.message, completed_at: Time.current)
88
+ raise
89
+ end
90
+
91
+ private
92
+
93
+ def default_report(tasks, specs, on_result)
94
+ Evals::Runner.new(
95
+ scenarios: tasks,
96
+ models: specs,
97
+ criteria: sample_criteria,
98
+ judge: evals_judge,
99
+ available_tools: tool_roster,
100
+ instructions: @evaluation.agent.instructions,
101
+ agent_name: @evaluation.agent.name,
102
+ threshold: PASS_THRESHOLD,
103
+ replay: ->(scenario, spec) { replay(scenario, spec) },
104
+ on_result: on_result
105
+ ).call
106
+ end
107
+
108
+ # --- selection --------------------------------------------------------
109
+
110
+ def selected_scenarios
111
+ scope = @evaluation.scenarios.enabled.ordered
112
+ scope = scope.where(id: Array(@selection[:scenario_ids])) if @selection[:scenario_ids].present?
113
+ scope = scope.where(key: Array(@selection[:keys])) if @selection[:keys].present?
114
+ scope = scope.in_group(@selection[:group]) if @selection[:group].present?
115
+ scope.to_a
116
+ end
117
+
118
+ # The models to compare: an explicit selection, else the evaluation's
119
+ # compare_models, else the agent as configured. `mock` is the framework's
120
+ # test double, accepted so the test suite can compare cohorts offline.
121
+ def model_specs
122
+ names = Array(@selection[:models]).presence || @evaluation.compare_models
123
+ specs = Evals::ModelSpec.parse_all(names, default_provider: @evaluation.agent.provider, providers: Agent::PROVIDERS + %w[mock])
124
+ return specs if specs.any?
125
+
126
+ [ Evals::ModelSpec.new(label: @evaluation.agent.model, provider: @evaluation.agent.provider, model: @evaluation.agent.model) ]
127
+ end
128
+
129
+ def selection_summary(scenarios, specs)
130
+ {
131
+ "scenario_ids" => scenarios.map(&:id),
132
+ "scenario_keys" => scenarios.map(&:key),
133
+ "group" => @selection[:group].presence,
134
+ "models" => specs.map(&:to_h)
135
+ }.compact
136
+ end
137
+
138
+ # --- replay -----------------------------------------------------------
139
+
140
+ def replay(scenario, spec)
141
+ # One execution per replay, reported to the host before the run starts
142
+ # (the order SandboxesController#compare uses), so it is counted even
143
+ # when the run fails.
144
+ ActionAgent.record_usage(owner, :execution)
145
+
146
+ agent_run = @evaluation.agent.test_execute(
147
+ scenario.prompt,
148
+ model_override: spec.model,
149
+ provider_override: spec.provider
150
+ )
151
+
152
+ Evals::Replay.new(
153
+ answer: agent_run.output,
154
+ tool_calls: tool_calls_for(agent_run),
155
+ duration_ms: agent_run.calculated_duration_ms,
156
+ input_tokens: agent_run.input_tokens,
157
+ output_tokens: agent_run.output_tokens,
158
+ error: agent_run.failed? ? agent_run.error_message.presence || "run failed" : nil,
159
+ cost: ModelPricing.estimate(model: spec.model, input_tokens: agent_run.input_tokens, output_tokens: agent_run.output_tokens),
160
+ metadata: { "agent_run_id" => agent_run.id }
161
+ )
162
+ end
163
+
164
+ # Each tool call the run made, rebuilt from the run's progress events
165
+ # (AgentRun#append_event pairs a "started" event with its "done"/"error"
166
+ # by eid). Falls back to the bare names in the run's metadata for a run
167
+ # recorded without events.
168
+ def tool_calls_for(agent_run)
169
+ events = Array(agent_run.logs).select { |event| event.is_a?(Hash) && %w[tool agent].include?(event["kind"]) }
170
+ if events.empty?
171
+ return Array(agent_run.output_metadata&.dig("tool_calls")).map { |name| { "name" => name.to_s } }
172
+ end
173
+
174
+ events.group_by { |event| event["eid"] }.values.map do |group|
175
+ started = group.find { |event| event["status"] == "started" }
176
+ finished = group.find { |event| %w[done error].include?(event["status"]) }
177
+ label = (started || finished)["label"].to_s
178
+
179
+ {
180
+ "name" => label.sub(/\s*→.*\z/, ""),
181
+ "arguments" => parse_json(started&.dig("detail")),
182
+ "error" => finished&.dig("status") == "error",
183
+ "detail" => finished&.dig("detail"),
184
+ "duration_ms" => finished&.dig("duration_ms")
185
+ }.compact
186
+ end
187
+ end
188
+
189
+ def parse_json(text)
190
+ return nil if text.blank?
191
+
192
+ JSON.parse(text)
193
+ rescue JSON::ParserError
194
+ text
195
+ end
196
+
197
+ # --- persistence ------------------------------------------------------
198
+
199
+ def persist(run, scenario, result)
200
+ run.scenario_results.create!(
201
+ scenario: scenario,
202
+ agent_run_id: result.replay.metadata["agent_run_id"],
203
+ model: result.model,
204
+ provider: result.provider,
205
+ status: result.status,
206
+ score: result.score,
207
+ scores: result.scores,
208
+ output: result.replay.answer.to_s.byteslice(0, 20_000).to_s.scrub.presence,
209
+ tool_calls: result.replay.tool_calls,
210
+ duration_ms: result.replay.duration_ms,
211
+ input_tokens: result.replay.input_tokens,
212
+ output_tokens: result.replay.output_tokens,
213
+ cost: result.replay.cost,
214
+ fault: result.fault,
215
+ recommendation: result.recommendation,
216
+ diagnosis: (result.diagnosis || {}).merge(
217
+ "_replay_metadata" => result.replay.metadata,
218
+ "_scenario_snapshot" => result.scenario.to_h.merge(expectations: result.scenario.expectations)
219
+ ),
220
+ error_message: result.replay.error
221
+ )
222
+ end
223
+
224
+ def scores_for(report, run)
225
+ scores = report.criterion_scores
226
+ scores["_models"] = report.summary_by_model
227
+ scores["_recommendations"] = report.recommendations
228
+ scores["_verdict"] = report.verdict if report.comparing?
229
+ scores["_selection"] = run.selection
230
+ scores["_metadata"] = report.metadata
231
+ scores["_judge_label"] = report.judge_label || report.judge&.label
232
+ scores
233
+ end
234
+
235
+ # --- judge ------------------------------------------------------------
236
+
237
+ def sample_criteria
238
+ @sample_criteria ||= @evaluation.criteria.reject do |criterion|
239
+ Evaluation::TELEMETRY_CRITERION_TYPES.include?(criterion["type"])
240
+ end
241
+ end
242
+
243
+ def tool_roster
244
+ @tool_roster ||= AgentToolbox.definitions_for(@evaluation.agent.tools).to_h do |definition|
245
+ [ definition[:name].to_s, definition[:description].to_s ]
246
+ end
247
+ end
248
+
249
+ # The judge the evaluation's owner has credentials for, wrapped for the
250
+ # evaluation core; nil when none is configured, in which case scoring
251
+ # stays on rules and expectations.
252
+ def evals_judge
253
+ return nil unless judge_available?
254
+
255
+ @evals_judge ||= Evals::Judge.new(label: @evaluation.judge_model.presence || judge_provider.to_s) do |instructions:, prompt:|
256
+ judge_class.prompt(message: prompt, instructions: instructions).generate_now.message&.content
257
+ end
258
+ end
259
+ end
260
+ end
@@ -30,7 +30,7 @@ module ActionAgent
30
30
  #
31
31
  # MCP attribution comes from ActiveAgent::Telemetry::ToolOrigin (the
32
32
  # +mcp__server__tool+ convention, tagged onto spans at instrumentation
33
- # time), then McpCatalog's hints for bare tool names, then the tool is
33
+ # time), then MCPCatalog's hints for bare tool names, then the tool is
34
34
  # treated as a method the agent class defines.
35
35
  #
36
36
  # Scopes are passed in rather than derived, so the caller's ownership
@@ -259,7 +259,7 @@ module ActionAgent
259
259
  end
260
260
  end
261
261
 
262
- Array(agent.mcp_servers).each do |server|
262
+ configured_mcp_servers(agent).each do |server|
263
263
  key = mcp_server_key(server)
264
264
  next if key.blank?
265
265
 
@@ -404,7 +404,7 @@ module ActionAgent
404
404
  classification = ActiveAgent::Telemetry::ToolOrigin.classify(name)
405
405
  return { origin: ORIGIN_MCP, server: classification[:server] } if classification[:server].present?
406
406
 
407
- if (hinted = McpCatalog.server_for_tool(name))
407
+ if (hinted = MCPCatalog.server_for_tool(name))
408
408
  # A catalog hint is weaker evidence than a namespaced name: the tool
409
409
  # is *probably* this server's, but a builtin of the same name is the
410
410
  # dashboard's own implementation, so builtins win the tie.
@@ -474,7 +474,7 @@ module ActionAgent
474
474
 
475
475
  def source_label(origin, server)
476
476
  case origin
477
- when ORIGIN_MCP then server.present? ? "MCP · #{McpCatalog.display_name(server)}" : "MCP"
477
+ when ORIGIN_MCP then server.present? ? "MCP · #{MCPCatalog.display_name(server)}" : "MCP"
478
478
  when ORIGIN_BUILTIN then "Dashboard toolbox"
479
479
  else "Agent-defined"
480
480
  end
@@ -500,7 +500,7 @@ module ActionAgent
500
500
  end
501
501
  end
502
502
 
503
- keys = (McpCatalog::BY_KEY.keys + detected.keys + configured_servers.keys).uniq
503
+ keys = (MCPCatalog.keys + detected.keys + configured_servers.keys).uniq
504
504
 
505
505
  # detected has a default block that would materialize a bucket on
506
506
  # lookup, so unseen servers are passed through as an explicit nil.
@@ -509,7 +509,7 @@ module ActionAgent
509
509
  end
510
510
 
511
511
  def server_row(key, bucket)
512
- catalog = McpCatalog.find(key)
512
+ catalog = MCPCatalog.find(key)
513
513
  configured = configured_servers[key].to_a.sort
514
514
  calls = bucket ? bucket[:calls] : 0
515
515
 
@@ -559,11 +559,25 @@ module ActionAgent
559
559
  @configured_servers ||= Hash.new { |hash, key| hash[key] = Set.new }
560
560
  end
561
561
 
562
+ # The servers an agent declares, as a list. Agents store an Array, but
563
+ # an agent created from an older template seed carried a top-level Hash
564
+ # keyed by server name ({"playwright" => {"command" => ...}}); Array()
565
+ # turned that into [key, value] pairs and the key lookup below raised
566
+ # TypeError on the Array, taking down /api/tools and /api/mcp_servers
567
+ # for the whole workspace.
568
+ def configured_mcp_servers(agent)
569
+ servers = agent.mcp_servers
570
+ return servers.keys if servers.is_a?(Hash)
571
+
572
+ Array(servers)
573
+ end
574
+
562
575
  # An agent's mcp_servers entries are free-form: a bare string name, or a
563
576
  # hash from the builder ({"name" => "playwright", "url" => ...}).
577
+ # Anything else (a stray Array, a number) is skipped rather than raised on.
564
578
  def mcp_server_key(server)
565
- return server.to_s.strip if server.is_a?(String)
566
- return nil unless server.respond_to?(:[])
579
+ return server.to_s.strip.presence if server.is_a?(String) || server.is_a?(Symbol)
580
+ return nil unless server.respond_to?(:key?)
567
581
 
568
582
  (server["key"] || server[:key] || server["name"] || server[:name]).to_s.strip.presence
569
583
  end
data/config/routes.rb CHANGED
@@ -20,6 +20,8 @@ ActionAgent::Engine.routes.draw do
20
20
 
21
21
  # The dashboard's own JSON API, read and written by the React app.
22
22
  namespace :api do
23
+ resource :dashboard_assistant, only: [ :show, :create ], controller: "dashboard_assistant"
24
+
23
25
  # Telemetry ingestion, relative to wherever the engine is mounted:
24
26
  # <mount>/api/traces (e.g. /activeagents/api/traces at the default mount).
25
27
  # Authenticated with a bearer token, not a session.
@@ -36,6 +38,10 @@ ActionAgent::Engine.routes.draw do
36
38
  post :duplicate
37
39
  get :export
38
40
  get :analytics
41
+ # The runner's conversation picker: this agent's persisted contexts,
42
+ # and a fresh one to pin a first message to.
43
+ get :conversations
44
+ post :conversations, action: :create_conversation
39
45
  end
40
46
  collection do
41
47
  get :presets
@@ -70,7 +76,7 @@ ActionAgent::Engine.routes.draw do
70
76
  resources :tools, only: [ :index ]
71
77
 
72
78
  # MCP services — detected servers unioned with the default catalog
73
- # (McpCatalog), plus on-demand sandbox provisioning. Keys are catalog
79
+ # (MCPCatalog), plus on-demand sandbox provisioning. Keys are catalog
74
80
  # slugs like "sequential-thinking", so the id segment allows dashes.
75
81
  resources :mcp_servers, only: [ :index, :show ], id: /[^\/]+/ do
76
82
  member do
@@ -111,12 +117,23 @@ ActionAgent::Engine.routes.draw do
111
117
  resource :metrics, only: [ :show ], controller: "metrics"
112
118
 
113
119
  # Conversations (contexts, messages, generations) behind Interactions.
114
- resources :interactions, only: [ :index, :show ]
120
+ # The runner edits a conversation in place — seeds, fixes or drops a
121
+ # turn — so the next run sees exactly the history it should.
122
+ resources :interactions, only: [ :index, :show ] do
123
+ resources :messages, only: [ :create, :update, :destroy ], controller: "interaction_messages"
124
+ end
115
125
 
116
- # Agent output evaluations.
126
+ # Agent output evaluations. A scenario suite also manages its scenarios
127
+ # here, and exposes each run's per-scenario, per-model results.
117
128
  resources :evaluations, only: [ :index, :show, :create, :destroy ] do
118
129
  member do
119
130
  post :run
131
+ get "runs/:run_id", action: :show_run, as: :run_result
132
+ get "runs/:run_id/report", action: :run_report, as: :run_report
133
+ get :scenarios
134
+ put :scenarios, action: :replace_scenarios
135
+ patch "scenarios/:scenario_id", action: :update_scenario, as: :scenario
136
+ delete "scenarios/:scenario_id", action: :destroy_scenario
120
137
  end
121
138
  end
122
139
 
@@ -128,6 +145,12 @@ ActionAgent::Engine.routes.draw do
128
145
  # Model catalogs for the agent builder (Ollama queried live from the
129
146
  # configured host; hosted providers curated).
130
147
  resources :provider_models, only: [ :index ]
148
+
149
+ # The plan meter the Organization view and the Run Agents quota banner
150
+ # read. The engine meters nothing itself: a host that tracks usage
151
+ # against a plan answers through ActionAgent.usage_resolver, and a bare
152
+ # mount reports unlimited rather than 404.
153
+ resource :usage, only: [ :show ], controller: "usage"
131
154
  end
132
155
 
133
156
  # The account's agents presented as an authenticated MCP server (tools +
@@ -136,6 +159,16 @@ ActionAgent::Engine.routes.draw do
136
159
  # namespace's session-authenticated controllers.
137
160
  post "mcp", to: "api/mcp#create"
138
161
 
162
+ # MCP Streamable HTTP (2025-03-26): a client MAY open the server-to-client
163
+ # SSE stream with GET, and ends a session with DELETE. This facade offers
164
+ # no stream and keeps no sessions, so both answer 405 with Allow: POST —
165
+ # the clean "not offered" signal SDK clients expect, instead of the
166
+ # dashboard's HTML page parsed as an event stream. A browser's GET (Accept
167
+ # prefers HTML) is the MCP Services view's deep link, and falls through to
168
+ # the catch-all below like any other client-side route.
169
+ match "mcp", to: "api/mcp#unsupported", via: [ :get, :delete ],
170
+ constraints: ->(request) { request.delete? || !ActionAgent::Engine.html_request?(request) }
171
+
139
172
  # Everything else under the mount is a client-side route: render the
140
173
  # dashboard and let the browser resolve it. Anchored last so it can only
141
174
  # ever catch what the routes above did not, and refuses /api paths so a
@@ -0,0 +1,22 @@
1
+ # frozen_string_literal: true
2
+
3
+ module ActionAgent
4
+ # Runs before Rails' request logger, including for requests later rejected by
5
+ # authentication, consent or CSRF. Match the endpoint under any engine mount;
6
+ # leave the host application's unrelated message/history parameters alone.
7
+ class AssistantRequestFilter
8
+ PATH = %r{/api/dashboard_assistant(?:\.[^/]+)?/?\z}
9
+ PARAMETERS = [ /\Amessage\z/i, /\Ahistory\z/i ].freeze
10
+
11
+ def initialize(app)
12
+ @app = app
13
+ end
14
+
15
+ def call(env)
16
+ if PATH.match?(env["PATH_INFO"].to_s)
17
+ env["action_dispatch.parameter_filter"] = Array(env["action_dispatch.parameter_filter"]) + PARAMETERS
18
+ end
19
+ @app.call(env)
20
+ end
21
+ end
22
+ end