actionagent 1.2.2 → 1.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +14 -3
- data/app/assets/builds/action_agent.css +1 -1
- data/app/assets/builds/action_agent.js +69 -43
- data/app/controllers/action_agent/api/agent_runs_controller.rb +28 -8
- data/app/controllers/action_agent/api/agents_controller.rb +191 -56
- data/app/controllers/action_agent/api/analytics_controller.rb +31 -9
- data/app/controllers/action_agent/api/base_controller.rb +16 -0
- data/app/controllers/action_agent/api/dashboard_assistant_controller.rb +83 -0
- data/app/controllers/action_agent/api/evaluations_controller.rb +252 -7
- data/app/controllers/action_agent/api/interaction_messages_controller.rb +98 -0
- data/app/controllers/action_agent/api/mcp_controller.rb +13 -3
- data/app/controllers/action_agent/api/mcp_servers_controller.rb +28 -8
- data/app/controllers/action_agent/api/metrics_controller.rb +44 -11
- data/app/controllers/action_agent/api/provider_models_controller.rb +1 -1
- data/app/controllers/action_agent/api/sandboxes_controller.rb +6 -0
- data/app/controllers/action_agent/api/session_recordings_controller.rb +34 -12
- data/app/controllers/action_agent/api/templates_controller.rb +25 -21
- data/app/controllers/action_agent/api/traces_controller.rb +25 -5
- data/app/controllers/action_agent/api/usage_controller.rb +20 -0
- data/app/controllers/action_agent/application_controller.rb +25 -2
- data/app/controllers/action_agent/dashboard_controller.rb +3 -1
- data/app/controllers/concerns/action_agent/api/agent_serialization.rb +53 -0
- data/app/jobs/action_agent/agent_execution_job.rb +40 -20
- data/app/jobs/action_agent/application_job.rb +7 -3
- data/app/jobs/action_agent/evaluation_run_job.rb +18 -0
- data/app/jobs/action_agent/sandbox_cleanup_job.rb +13 -10
- data/app/models/action_agent/agent.rb +74 -23
- data/app/models/action_agent/agent_run.rb +99 -0
- data/app/models/action_agent/agent_template.rb +22 -7
- data/app/models/action_agent/evaluation.rb +64 -4
- data/app/models/action_agent/evaluation_run.rb +190 -2
- data/app/models/action_agent/evaluation_scenario.rb +59 -0
- data/app/models/action_agent/evaluation_scenario_result.rb +86 -0
- data/app/models/action_agent/recording_action.rb +11 -7
- data/app/models/action_agent/sandbox_session.rb +1 -1
- data/app/models/action_agent/session_recording.rb +31 -8
- data/app/models/action_agent/telemetry_trace.rb +126 -3
- data/app/models/concerns/action_agent/adapter_aware.rb +19 -0
- data/app/models/concerns/action_agent/ownable.rb +15 -2
- data/app/queries/action_agent/metrics_report.rb +498 -0
- data/app/serializers/action_agent/agent_message_serializer.rb +1 -0
- data/app/services/action_agent/agent_execution_service.rb +294 -16
- data/app/services/action_agent/agent_registrar.rb +7 -6
- data/app/services/action_agent/agent_toolbox.rb +49 -7
- data/app/services/action_agent/dashboard_assistant_service.rb +342 -0
- data/app/services/action_agent/evaluation_evidence.rb +234 -0
- data/app/services/action_agent/evaluation_runner_service.rb +13 -3
- data/app/services/action_agent/evaluation_tool_resolver.rb +162 -0
- data/app/services/action_agent/mcp_catalog.rb +46 -8
- data/app/services/action_agent/mcp_client.rb +167 -0
- data/app/services/action_agent/mcp_recording_middleware.rb +2 -2
- data/app/services/action_agent/mcp_tool_dispatcher.rb +116 -0
- data/app/services/action_agent/playwright_mcp_client.rb +11 -126
- data/app/services/action_agent/sandbox_orchestrator.rb +12 -1
- data/app/services/action_agent/scenario_evaluation_runner.rb +260 -0
- data/app/services/action_agent/tool_discovery.rb +22 -8
- data/config/routes.rb +36 -3
- data/lib/action_agent/assistant_request_filter.rb +22 -0
- data/lib/action_agent/engine.rb +106 -19
- data/lib/action_agent/version.rb +1 -1
- data/lib/action_agent.rb +104 -6
- data/lib/generators/action_agent/install_generator.rb +20 -7
- data/lib/generators/action_agent/templates/action_agent.rb.erb +12 -0
- data/lib/generators/action_agent/templates/create_active_agent_evaluation_scenarios.rb.erb +79 -0
- data/lib/tasks/action_agent.rake +9 -0
- metadata +22 -5
|
@@ -1,20 +1,10 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
3
|
module ActionAgent
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
# Minimal MCP client (streamable HTTP transport) for a Playwright MCP
|
|
8
|
-
# server — typically `npx @playwright/mcp --port 8931` running beside the
|
|
9
|
-
# app in development, or a sandbox-provisioned browser container in
|
|
10
|
-
# production. Speaks just enough JSON-RPC for tools/call: initialize once
|
|
11
|
-
# per process, then call tools under the session id the server hands back.
|
|
12
|
-
class PlaywrightMcpClient
|
|
4
|
+
# The Playwright MCP server as one configured MCPClient: a process-wide
|
|
5
|
+
# instance pointed at the url PLAYWRIGHT_MCP_URL names.
|
|
6
|
+
class PlaywrightMCPClient < MCPClient
|
|
13
7
|
DEFAULT_URL = ENV.fetch("PLAYWRIGHT_MCP_URL", "http://host.orb.internal:8931/mcp")
|
|
14
|
-
OPEN_TIMEOUT_SECONDS = 5
|
|
15
|
-
READ_TIMEOUT_SECONDS = 60
|
|
16
|
-
|
|
17
|
-
class Error < StandardError; end
|
|
18
8
|
|
|
19
9
|
def self.instance
|
|
20
10
|
@instance ||= new
|
|
@@ -25,124 +15,19 @@ module ActionAgent
|
|
|
25
15
|
end
|
|
26
16
|
|
|
27
17
|
def initialize(url: DEFAULT_URL)
|
|
28
|
-
|
|
29
|
-
@mutex = Mutex.new
|
|
18
|
+
super(url: url, label: "Playwright")
|
|
30
19
|
end
|
|
31
20
|
|
|
32
|
-
#
|
|
21
|
+
# Restarting the shared instance on an unreachable server is this
|
|
22
|
+
# subclass's concern: the next call re-initializes rather than reusing a
|
|
23
|
+
# session the server has forgotten. The message names the local fix.
|
|
33
24
|
def call_tool(name, arguments = {})
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
response = post(
|
|
37
|
-
{ jsonrpc: "2.0", id: next_id, method: "tools/call",
|
|
38
|
-
params: { name: name, arguments: arguments } },
|
|
39
|
-
session: @session_id
|
|
40
|
-
)
|
|
41
|
-
result = response["result"]
|
|
42
|
-
unless result
|
|
43
|
-
Rails.logger.warn("[PlaywrightMcpClient] #{name} unexpected response: #{response.inspect[0, 500]}")
|
|
44
|
-
raise Error, (response.dig("error", "message") || "empty MCP response")
|
|
45
|
-
end
|
|
46
|
-
|
|
47
|
-
text = Array(result["content"]).filter_map { |block| block["text"] }.join("\n")
|
|
48
|
-
{ text: text, is_error: result["isError"] ? true : false }
|
|
49
|
-
rescue Errno::ECONNREFUSED, Errno::EHOSTUNREACH, Net::OpenTimeout, SocketError => e
|
|
25
|
+
super
|
|
26
|
+
rescue MCPClient::Error => e
|
|
50
27
|
self.class.reset!
|
|
51
|
-
raise Error, "
|
|
52
|
-
end
|
|
53
|
-
|
|
54
|
-
private
|
|
55
|
-
|
|
56
|
-
def ensure_session!
|
|
57
|
-
@mutex.synchronize do
|
|
58
|
-
next if @session_id
|
|
59
|
-
|
|
60
|
-
_body, response = post_raw(
|
|
61
|
-
{ jsonrpc: "2.0", id: next_id, method: "initialize",
|
|
62
|
-
params: { protocolVersion: "2025-03-26", capabilities: {},
|
|
63
|
-
clientInfo: { name: "activeagents", version: "1.0" } } }
|
|
64
|
-
)
|
|
65
|
-
@session_id = response["mcp-session-id"]
|
|
66
|
-
raise Error, "MCP server did not return a session id" unless @session_id
|
|
67
|
-
|
|
68
|
-
post({ jsonrpc: "2.0", method: "notifications/initialized" }, session: @session_id)
|
|
69
|
-
end
|
|
70
|
-
end
|
|
71
|
-
|
|
72
|
-
def post(payload, session: nil)
|
|
73
|
-
body, _response = post_raw(payload, session: session)
|
|
74
|
-
body
|
|
75
|
-
end
|
|
76
|
-
|
|
77
|
-
def post_raw(payload, session: nil)
|
|
78
|
-
# Tool calls run inside the provider SDK's streaming enumerator — a
|
|
79
|
-
# fiber, where Net::HTTP reads of SSE bodies misbehave (headers arrive,
|
|
80
|
-
# body comes back empty). A dedicated thread always does real blocking
|
|
81
|
-
# IO outside any fiber/scheduler context.
|
|
82
|
-
Thread.new { blocking_post_raw(payload, session: session) }.value
|
|
83
|
-
end
|
|
84
|
-
|
|
85
|
-
def blocking_post_raw(payload, session: nil)
|
|
86
|
-
http = Net::HTTP.new(@uri.host, @uri.port)
|
|
87
|
-
# Container->host bridge hostnames (host.orb.internal) publish an IPv6
|
|
88
|
-
# address whose path doesn't reach the server; dual-stack connects then
|
|
89
|
-
# fail intermittently. Pin to IPv4 while keeping the Host header.
|
|
90
|
-
if (ipv4 = ipv4_address)
|
|
91
|
-
http.ipaddr = ipv4
|
|
92
|
-
end
|
|
93
|
-
http.open_timeout = OPEN_TIMEOUT_SECONDS
|
|
94
|
-
http.read_timeout = READ_TIMEOUT_SECONDS
|
|
95
|
-
request = Net::HTTP::Post.new(@uri.request_uri)
|
|
96
|
-
request["Content-Type"] = "application/json"
|
|
97
|
-
request["Accept"] = "application/json, text/event-stream"
|
|
98
|
-
request["Mcp-Session-Id"] = session if session
|
|
99
|
-
request.body = payload.to_json
|
|
100
|
-
|
|
101
|
-
response = http.request(request)
|
|
102
|
-
Rails.logger.debug(
|
|
103
|
-
"[PlaywrightMcpClient] #{payload[:method]} -> #{response.code} " \
|
|
104
|
-
"ct=#{response['Content-Type']} bytes=#{response.body.to_s.bytesize} session=#{session ? 'yes' : 'no'}"
|
|
105
|
-
)
|
|
106
|
-
unless response.code.to_i.between?(200, 299)
|
|
107
|
-
Rails.logger.warn("[PlaywrightMcpClient] HTTP #{response.code}: #{response.body.to_s[0, 300]}")
|
|
108
|
-
raise Error, "MCP server returned HTTP #{response.code}"
|
|
109
|
-
end
|
|
110
|
-
|
|
111
|
-
parsed = parse_body(response)
|
|
112
|
-
if parsed.empty? && payload[:id]
|
|
113
|
-
Rails.logger.warn("[PlaywrightMcpClient] unparsed body (#{response['Content-Type']}): #{response.body.to_s[0, 500]}")
|
|
114
|
-
end
|
|
115
|
-
[ parsed, response ]
|
|
116
|
-
end
|
|
117
|
-
|
|
118
|
-
# Streamable HTTP answers as plain JSON or as an SSE stream whose data:
|
|
119
|
-
# lines carry the JSON-RPC response — accept both.
|
|
120
|
-
def parse_body(response)
|
|
121
|
-
body = response.body.to_s
|
|
122
|
-
return {} if body.empty?
|
|
123
|
-
|
|
124
|
-
if response["Content-Type"].to_s.include?("text/event-stream")
|
|
125
|
-
body.lines
|
|
126
|
-
.select { |line| line.start_with?("data:") }
|
|
127
|
-
.filter_map { |line| JSON.parse(line.delete_prefix("data:").strip) rescue nil }
|
|
128
|
-
.find { |json| json["result"] || json["error"] } || {}
|
|
129
|
-
else
|
|
130
|
-
JSON.parse(body)
|
|
131
|
-
end
|
|
132
|
-
rescue JSON::ParserError
|
|
133
|
-
{}
|
|
134
|
-
end
|
|
135
|
-
|
|
136
|
-
def ipv4_address
|
|
137
|
-
return @ipv4_address if defined?(@ipv4_address)
|
|
138
|
-
|
|
139
|
-
@ipv4_address = Resolv.getaddresses(@uri.host).find { |address| address =~ Resolv::IPv4::Regex }
|
|
140
|
-
rescue Resolv::ResolvError
|
|
141
|
-
@ipv4_address = nil
|
|
142
|
-
end
|
|
28
|
+
raise Error, "#{e.message}: start it with `npx @playwright/mcp --port 8931`" if e.message.include?("unreachable")
|
|
143
29
|
|
|
144
|
-
|
|
145
|
-
@id = (@id || 0) + 1
|
|
30
|
+
raise
|
|
146
31
|
end
|
|
147
32
|
end
|
|
148
33
|
end
|
|
@@ -52,7 +52,18 @@ module ActionAgent
|
|
|
52
52
|
# as sandbox_service, falling back to the in-memory one.
|
|
53
53
|
def self.default_backend
|
|
54
54
|
name = ENV["SANDBOX_BACKEND"].presence || ActionAgent.sandbox_service.to_s
|
|
55
|
-
backends.key?(name)
|
|
55
|
+
return name if backends.key?(name)
|
|
56
|
+
|
|
57
|
+
# Substituting the mock silently made a misconfigured operator's
|
|
58
|
+
# sandbox "runs" succeed against nothing real.
|
|
59
|
+
if name.present? && name != "mock"
|
|
60
|
+
Rails.logger.warn(
|
|
61
|
+
"[ActionAgent] sandbox backend #{name.inspect} is not registered " \
|
|
62
|
+
"(ActionAgent.sandbox_backends knows #{backends.keys.inspect}); " \
|
|
63
|
+
"using the in-memory mock backend, which runs nothing."
|
|
64
|
+
)
|
|
65
|
+
end
|
|
66
|
+
"mock"
|
|
56
67
|
end
|
|
57
68
|
|
|
58
69
|
def initialize(backend: nil)
|
|
@@ -0,0 +1,260 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module ActionAgent
|
|
4
|
+
# Runs a scenario evaluation: replays every selected scenario through the
|
|
5
|
+
# agent once per candidate model and writes one EvaluationScenarioResult per
|
|
6
|
+
# scenario × model plus a per-model summary and verdict on the run.
|
|
7
|
+
#
|
|
8
|
+
# The scoring, fault diagnosis and roll-up are ActiveAgent::Evals'; this
|
|
9
|
+
# class supplies what only the dashboard knows — how to run the agent
|
|
10
|
+
# (Agent#test_execute with a model override), where to persist each result,
|
|
11
|
+
# how to price tokens, and which judge model the owner has credentials for.
|
|
12
|
+
#
|
|
13
|
+
# The run's `scores` keep the shape the Evaluations UI renders — criterion
|
|
14
|
+
# => stats, or criterion => { model => stats } when comparing — and add
|
|
15
|
+
# underscore-prefixed summaries:
|
|
16
|
+
#
|
|
17
|
+
# "_models" — per model: pass rate, mean score, latency, tokens, cost, fault counts
|
|
18
|
+
# "_recommendations" — faults grouped across scenarios with the fix each calls for
|
|
19
|
+
# "_verdict" — the best model and why (judge-written when a judge is available)
|
|
20
|
+
# "_selection" — the scenarios and models this run covered
|
|
21
|
+
class ScenarioEvaluationRunner < EvaluationRunnerService
|
|
22
|
+
Evals = ActiveAgent::Evals
|
|
23
|
+
|
|
24
|
+
# `run` is an EvaluationRun created ahead of time (by run_later!, so the
|
|
25
|
+
# UI can show it pending while the job waits); absent, one is created here.
|
|
26
|
+
def self.call(evaluation, selection: {}, run: nil)
|
|
27
|
+
new(evaluation, selection: selection, run: run).call
|
|
28
|
+
end
|
|
29
|
+
|
|
30
|
+
def initialize(evaluation, selection: {}, run: nil)
|
|
31
|
+
super(evaluation)
|
|
32
|
+
@selection = (selection || {}).to_h.with_indifferent_access
|
|
33
|
+
@run = run
|
|
34
|
+
end
|
|
35
|
+
|
|
36
|
+
def call
|
|
37
|
+
scenarios = selected_scenarios
|
|
38
|
+
specs = model_specs
|
|
39
|
+
run = @run || @evaluation.evaluation_runs.create!(status: :pending)
|
|
40
|
+
run.update!(status: :running, selection: selection_summary(scenarios, specs))
|
|
41
|
+
|
|
42
|
+
if scenarios.empty?
|
|
43
|
+
run.update!(status: :failed, error_message: "No scenarios selected — add scenarios to the evaluation or widen the selection",
|
|
44
|
+
completed_at: Time.current)
|
|
45
|
+
return run
|
|
46
|
+
end
|
|
47
|
+
|
|
48
|
+
records = scenarios.index_by(&:key)
|
|
49
|
+
tasks = scenarios.map { |scenario| Evals::Scenario.from_hash(scenario.as_json_summary) }
|
|
50
|
+
expected = tasks.product(specs).map { |task, spec| [ task.key, spec.label ] }
|
|
51
|
+
# Every result tests membership twice, so look the pairs up in a set;
|
|
52
|
+
# `expected` stays an array for the completeness comparison below, and
|
|
53
|
+
# `persisted` sorts the same way either way.
|
|
54
|
+
allowed = expected.to_set
|
|
55
|
+
persisted = Set.new
|
|
56
|
+
on_result = lambda do |result|
|
|
57
|
+
pair = [ result.scenario.key, result.label ]
|
|
58
|
+
raise ArgumentError, "unexpected or duplicate scenario evaluation result" unless allowed.include?(pair) && !persisted.include?(pair)
|
|
59
|
+
|
|
60
|
+
persist(run, records.fetch(result.scenario.key), result)
|
|
61
|
+
persisted << pair
|
|
62
|
+
end
|
|
63
|
+
adapter = ActionAgent.scenario_evaluation_adapter_resolver&.call(@evaluation)
|
|
64
|
+
report = if adapter
|
|
65
|
+
raise ArgumentError, "scenario evaluation adapter must be callable" unless adapter.respond_to?(:call)
|
|
66
|
+
|
|
67
|
+
adapter.call(evaluation: @evaluation, owner: owner, scenarios: tasks, models: specs, on_result: on_result)
|
|
68
|
+
else
|
|
69
|
+
ensure_judge_defined_kpis! if @evaluation.judge_defined?
|
|
70
|
+
default_report(tasks, specs, on_result)
|
|
71
|
+
end
|
|
72
|
+
raise ArgumentError, "scenario evaluation adapter must return an ActiveAgent::Evals::Report" unless report.is_a?(Evals::Report)
|
|
73
|
+
reported = report.results.map { |result| [ result.scenario.key, result.label ] }
|
|
74
|
+
unless reported.sort == expected.sort && persisted.sort == expected.sort
|
|
75
|
+
raise ArgumentError, "scenario evaluation adapter must report and persist every selected scenario and model"
|
|
76
|
+
end
|
|
77
|
+
|
|
78
|
+
run.update!(
|
|
79
|
+
status: :complete,
|
|
80
|
+
scores: scores_for(report, run),
|
|
81
|
+
samples_evaluated: report.results.size,
|
|
82
|
+
samples_passed: report.results.count(&:passed?),
|
|
83
|
+
completed_at: Time.current
|
|
84
|
+
)
|
|
85
|
+
run
|
|
86
|
+
rescue StandardError => e
|
|
87
|
+
run&.update!(status: :failed, error_message: e.message, completed_at: Time.current)
|
|
88
|
+
raise
|
|
89
|
+
end
|
|
90
|
+
|
|
91
|
+
private
|
|
92
|
+
|
|
93
|
+
def default_report(tasks, specs, on_result)
|
|
94
|
+
Evals::Runner.new(
|
|
95
|
+
scenarios: tasks,
|
|
96
|
+
models: specs,
|
|
97
|
+
criteria: sample_criteria,
|
|
98
|
+
judge: evals_judge,
|
|
99
|
+
available_tools: tool_roster,
|
|
100
|
+
instructions: @evaluation.agent.instructions,
|
|
101
|
+
agent_name: @evaluation.agent.name,
|
|
102
|
+
threshold: PASS_THRESHOLD,
|
|
103
|
+
replay: ->(scenario, spec) { replay(scenario, spec) },
|
|
104
|
+
on_result: on_result
|
|
105
|
+
).call
|
|
106
|
+
end
|
|
107
|
+
|
|
108
|
+
# --- selection --------------------------------------------------------
|
|
109
|
+
|
|
110
|
+
def selected_scenarios
|
|
111
|
+
scope = @evaluation.scenarios.enabled.ordered
|
|
112
|
+
scope = scope.where(id: Array(@selection[:scenario_ids])) if @selection[:scenario_ids].present?
|
|
113
|
+
scope = scope.where(key: Array(@selection[:keys])) if @selection[:keys].present?
|
|
114
|
+
scope = scope.in_group(@selection[:group]) if @selection[:group].present?
|
|
115
|
+
scope.to_a
|
|
116
|
+
end
|
|
117
|
+
|
|
118
|
+
# The models to compare: an explicit selection, else the evaluation's
|
|
119
|
+
# compare_models, else the agent as configured. `mock` is the framework's
|
|
120
|
+
# test double, accepted so the test suite can compare cohorts offline.
|
|
121
|
+
def model_specs
|
|
122
|
+
names = Array(@selection[:models]).presence || @evaluation.compare_models
|
|
123
|
+
specs = Evals::ModelSpec.parse_all(names, default_provider: @evaluation.agent.provider, providers: Agent::PROVIDERS + %w[mock])
|
|
124
|
+
return specs if specs.any?
|
|
125
|
+
|
|
126
|
+
[ Evals::ModelSpec.new(label: @evaluation.agent.model, provider: @evaluation.agent.provider, model: @evaluation.agent.model) ]
|
|
127
|
+
end
|
|
128
|
+
|
|
129
|
+
def selection_summary(scenarios, specs)
|
|
130
|
+
{
|
|
131
|
+
"scenario_ids" => scenarios.map(&:id),
|
|
132
|
+
"scenario_keys" => scenarios.map(&:key),
|
|
133
|
+
"group" => @selection[:group].presence,
|
|
134
|
+
"models" => specs.map(&:to_h)
|
|
135
|
+
}.compact
|
|
136
|
+
end
|
|
137
|
+
|
|
138
|
+
# --- replay -----------------------------------------------------------
|
|
139
|
+
|
|
140
|
+
def replay(scenario, spec)
|
|
141
|
+
# One execution per replay, reported to the host before the run starts
|
|
142
|
+
# (the order SandboxesController#compare uses), so it is counted even
|
|
143
|
+
# when the run fails.
|
|
144
|
+
ActionAgent.record_usage(owner, :execution)
|
|
145
|
+
|
|
146
|
+
agent_run = @evaluation.agent.test_execute(
|
|
147
|
+
scenario.prompt,
|
|
148
|
+
model_override: spec.model,
|
|
149
|
+
provider_override: spec.provider
|
|
150
|
+
)
|
|
151
|
+
|
|
152
|
+
Evals::Replay.new(
|
|
153
|
+
answer: agent_run.output,
|
|
154
|
+
tool_calls: tool_calls_for(agent_run),
|
|
155
|
+
duration_ms: agent_run.calculated_duration_ms,
|
|
156
|
+
input_tokens: agent_run.input_tokens,
|
|
157
|
+
output_tokens: agent_run.output_tokens,
|
|
158
|
+
error: agent_run.failed? ? agent_run.error_message.presence || "run failed" : nil,
|
|
159
|
+
cost: ModelPricing.estimate(model: spec.model, input_tokens: agent_run.input_tokens, output_tokens: agent_run.output_tokens),
|
|
160
|
+
metadata: { "agent_run_id" => agent_run.id }
|
|
161
|
+
)
|
|
162
|
+
end
|
|
163
|
+
|
|
164
|
+
# Each tool call the run made, rebuilt from the run's progress events
|
|
165
|
+
# (AgentRun#append_event pairs a "started" event with its "done"/"error"
|
|
166
|
+
# by eid). Falls back to the bare names in the run's metadata for a run
|
|
167
|
+
# recorded without events.
|
|
168
|
+
def tool_calls_for(agent_run)
|
|
169
|
+
events = Array(agent_run.logs).select { |event| event.is_a?(Hash) && %w[tool agent].include?(event["kind"]) }
|
|
170
|
+
if events.empty?
|
|
171
|
+
return Array(agent_run.output_metadata&.dig("tool_calls")).map { |name| { "name" => name.to_s } }
|
|
172
|
+
end
|
|
173
|
+
|
|
174
|
+
events.group_by { |event| event["eid"] }.values.map do |group|
|
|
175
|
+
started = group.find { |event| event["status"] == "started" }
|
|
176
|
+
finished = group.find { |event| %w[done error].include?(event["status"]) }
|
|
177
|
+
label = (started || finished)["label"].to_s
|
|
178
|
+
|
|
179
|
+
{
|
|
180
|
+
"name" => label.sub(/\s*→.*\z/, ""),
|
|
181
|
+
"arguments" => parse_json(started&.dig("detail")),
|
|
182
|
+
"error" => finished&.dig("status") == "error",
|
|
183
|
+
"detail" => finished&.dig("detail"),
|
|
184
|
+
"duration_ms" => finished&.dig("duration_ms")
|
|
185
|
+
}.compact
|
|
186
|
+
end
|
|
187
|
+
end
|
|
188
|
+
|
|
189
|
+
def parse_json(text)
|
|
190
|
+
return nil if text.blank?
|
|
191
|
+
|
|
192
|
+
JSON.parse(text)
|
|
193
|
+
rescue JSON::ParserError
|
|
194
|
+
text
|
|
195
|
+
end
|
|
196
|
+
|
|
197
|
+
# --- persistence ------------------------------------------------------
|
|
198
|
+
|
|
199
|
+
def persist(run, scenario, result)
|
|
200
|
+
run.scenario_results.create!(
|
|
201
|
+
scenario: scenario,
|
|
202
|
+
agent_run_id: result.replay.metadata["agent_run_id"],
|
|
203
|
+
model: result.model,
|
|
204
|
+
provider: result.provider,
|
|
205
|
+
status: result.status,
|
|
206
|
+
score: result.score,
|
|
207
|
+
scores: result.scores,
|
|
208
|
+
output: result.replay.answer.to_s.byteslice(0, 20_000).to_s.scrub.presence,
|
|
209
|
+
tool_calls: result.replay.tool_calls,
|
|
210
|
+
duration_ms: result.replay.duration_ms,
|
|
211
|
+
input_tokens: result.replay.input_tokens,
|
|
212
|
+
output_tokens: result.replay.output_tokens,
|
|
213
|
+
cost: result.replay.cost,
|
|
214
|
+
fault: result.fault,
|
|
215
|
+
recommendation: result.recommendation,
|
|
216
|
+
diagnosis: (result.diagnosis || {}).merge(
|
|
217
|
+
"_replay_metadata" => result.replay.metadata,
|
|
218
|
+
"_scenario_snapshot" => result.scenario.to_h.merge(expectations: result.scenario.expectations)
|
|
219
|
+
),
|
|
220
|
+
error_message: result.replay.error
|
|
221
|
+
)
|
|
222
|
+
end
|
|
223
|
+
|
|
224
|
+
def scores_for(report, run)
|
|
225
|
+
scores = report.criterion_scores
|
|
226
|
+
scores["_models"] = report.summary_by_model
|
|
227
|
+
scores["_recommendations"] = report.recommendations
|
|
228
|
+
scores["_verdict"] = report.verdict if report.comparing?
|
|
229
|
+
scores["_selection"] = run.selection
|
|
230
|
+
scores["_metadata"] = report.metadata
|
|
231
|
+
scores["_judge_label"] = report.judge_label || report.judge&.label
|
|
232
|
+
scores
|
|
233
|
+
end
|
|
234
|
+
|
|
235
|
+
# --- judge ------------------------------------------------------------
|
|
236
|
+
|
|
237
|
+
def sample_criteria
|
|
238
|
+
@sample_criteria ||= @evaluation.criteria.reject do |criterion|
|
|
239
|
+
Evaluation::TELEMETRY_CRITERION_TYPES.include?(criterion["type"])
|
|
240
|
+
end
|
|
241
|
+
end
|
|
242
|
+
|
|
243
|
+
def tool_roster
|
|
244
|
+
@tool_roster ||= AgentToolbox.definitions_for(@evaluation.agent.tools).to_h do |definition|
|
|
245
|
+
[ definition[:name].to_s, definition[:description].to_s ]
|
|
246
|
+
end
|
|
247
|
+
end
|
|
248
|
+
|
|
249
|
+
# The judge the evaluation's owner has credentials for, wrapped for the
|
|
250
|
+
# evaluation core; nil when none is configured, in which case scoring
|
|
251
|
+
# stays on rules and expectations.
|
|
252
|
+
def evals_judge
|
|
253
|
+
return nil unless judge_available?
|
|
254
|
+
|
|
255
|
+
@evals_judge ||= Evals::Judge.new(label: @evaluation.judge_model.presence || judge_provider.to_s) do |instructions:, prompt:|
|
|
256
|
+
judge_class.prompt(message: prompt, instructions: instructions).generate_now.message&.content
|
|
257
|
+
end
|
|
258
|
+
end
|
|
259
|
+
end
|
|
260
|
+
end
|
|
@@ -30,7 +30,7 @@ module ActionAgent
|
|
|
30
30
|
#
|
|
31
31
|
# MCP attribution comes from ActiveAgent::Telemetry::ToolOrigin (the
|
|
32
32
|
# +mcp__server__tool+ convention, tagged onto spans at instrumentation
|
|
33
|
-
# time), then
|
|
33
|
+
# time), then MCPCatalog's hints for bare tool names, then the tool is
|
|
34
34
|
# treated as a method the agent class defines.
|
|
35
35
|
#
|
|
36
36
|
# Scopes are passed in rather than derived, so the caller's ownership
|
|
@@ -259,7 +259,7 @@ module ActionAgent
|
|
|
259
259
|
end
|
|
260
260
|
end
|
|
261
261
|
|
|
262
|
-
|
|
262
|
+
configured_mcp_servers(agent).each do |server|
|
|
263
263
|
key = mcp_server_key(server)
|
|
264
264
|
next if key.blank?
|
|
265
265
|
|
|
@@ -404,7 +404,7 @@ module ActionAgent
|
|
|
404
404
|
classification = ActiveAgent::Telemetry::ToolOrigin.classify(name)
|
|
405
405
|
return { origin: ORIGIN_MCP, server: classification[:server] } if classification[:server].present?
|
|
406
406
|
|
|
407
|
-
if (hinted =
|
|
407
|
+
if (hinted = MCPCatalog.server_for_tool(name))
|
|
408
408
|
# A catalog hint is weaker evidence than a namespaced name: the tool
|
|
409
409
|
# is *probably* this server's, but a builtin of the same name is the
|
|
410
410
|
# dashboard's own implementation, so builtins win the tie.
|
|
@@ -474,7 +474,7 @@ module ActionAgent
|
|
|
474
474
|
|
|
475
475
|
def source_label(origin, server)
|
|
476
476
|
case origin
|
|
477
|
-
when ORIGIN_MCP then server.present? ? "MCP · #{
|
|
477
|
+
when ORIGIN_MCP then server.present? ? "MCP · #{MCPCatalog.display_name(server)}" : "MCP"
|
|
478
478
|
when ORIGIN_BUILTIN then "Dashboard toolbox"
|
|
479
479
|
else "Agent-defined"
|
|
480
480
|
end
|
|
@@ -500,7 +500,7 @@ module ActionAgent
|
|
|
500
500
|
end
|
|
501
501
|
end
|
|
502
502
|
|
|
503
|
-
keys = (
|
|
503
|
+
keys = (MCPCatalog.keys + detected.keys + configured_servers.keys).uniq
|
|
504
504
|
|
|
505
505
|
# detected has a default block that would materialize a bucket on
|
|
506
506
|
# lookup, so unseen servers are passed through as an explicit nil.
|
|
@@ -509,7 +509,7 @@ module ActionAgent
|
|
|
509
509
|
end
|
|
510
510
|
|
|
511
511
|
def server_row(key, bucket)
|
|
512
|
-
catalog =
|
|
512
|
+
catalog = MCPCatalog.find(key)
|
|
513
513
|
configured = configured_servers[key].to_a.sort
|
|
514
514
|
calls = bucket ? bucket[:calls] : 0
|
|
515
515
|
|
|
@@ -559,11 +559,25 @@ module ActionAgent
|
|
|
559
559
|
@configured_servers ||= Hash.new { |hash, key| hash[key] = Set.new }
|
|
560
560
|
end
|
|
561
561
|
|
|
562
|
+
# The servers an agent declares, as a list. Agents store an Array, but
|
|
563
|
+
# an agent created from an older template seed carried a top-level Hash
|
|
564
|
+
# keyed by server name ({"playwright" => {"command" => ...}}); Array()
|
|
565
|
+
# turned that into [key, value] pairs and the key lookup below raised
|
|
566
|
+
# TypeError on the Array, taking down /api/tools and /api/mcp_servers
|
|
567
|
+
# for the whole workspace.
|
|
568
|
+
def configured_mcp_servers(agent)
|
|
569
|
+
servers = agent.mcp_servers
|
|
570
|
+
return servers.keys if servers.is_a?(Hash)
|
|
571
|
+
|
|
572
|
+
Array(servers)
|
|
573
|
+
end
|
|
574
|
+
|
|
562
575
|
# An agent's mcp_servers entries are free-form: a bare string name, or a
|
|
563
576
|
# hash from the builder ({"name" => "playwright", "url" => ...}).
|
|
577
|
+
# Anything else (a stray Array, a number) is skipped rather than raised on.
|
|
564
578
|
def mcp_server_key(server)
|
|
565
|
-
return server.to_s.strip if server.is_a?(String)
|
|
566
|
-
return nil unless server.respond_to?(:
|
|
579
|
+
return server.to_s.strip.presence if server.is_a?(String) || server.is_a?(Symbol)
|
|
580
|
+
return nil unless server.respond_to?(:key?)
|
|
567
581
|
|
|
568
582
|
(server["key"] || server[:key] || server["name"] || server[:name]).to_s.strip.presence
|
|
569
583
|
end
|
data/config/routes.rb
CHANGED
|
@@ -20,6 +20,8 @@ ActionAgent::Engine.routes.draw do
|
|
|
20
20
|
|
|
21
21
|
# The dashboard's own JSON API, read and written by the React app.
|
|
22
22
|
namespace :api do
|
|
23
|
+
resource :dashboard_assistant, only: [ :show, :create ], controller: "dashboard_assistant"
|
|
24
|
+
|
|
23
25
|
# Telemetry ingestion, relative to wherever the engine is mounted:
|
|
24
26
|
# <mount>/api/traces (e.g. /activeagents/api/traces at the default mount).
|
|
25
27
|
# Authenticated with a bearer token, not a session.
|
|
@@ -36,6 +38,10 @@ ActionAgent::Engine.routes.draw do
|
|
|
36
38
|
post :duplicate
|
|
37
39
|
get :export
|
|
38
40
|
get :analytics
|
|
41
|
+
# The runner's conversation picker: this agent's persisted contexts,
|
|
42
|
+
# and a fresh one to pin a first message to.
|
|
43
|
+
get :conversations
|
|
44
|
+
post :conversations, action: :create_conversation
|
|
39
45
|
end
|
|
40
46
|
collection do
|
|
41
47
|
get :presets
|
|
@@ -70,7 +76,7 @@ ActionAgent::Engine.routes.draw do
|
|
|
70
76
|
resources :tools, only: [ :index ]
|
|
71
77
|
|
|
72
78
|
# MCP services — detected servers unioned with the default catalog
|
|
73
|
-
# (
|
|
79
|
+
# (MCPCatalog), plus on-demand sandbox provisioning. Keys are catalog
|
|
74
80
|
# slugs like "sequential-thinking", so the id segment allows dashes.
|
|
75
81
|
resources :mcp_servers, only: [ :index, :show ], id: /[^\/]+/ do
|
|
76
82
|
member do
|
|
@@ -111,12 +117,23 @@ ActionAgent::Engine.routes.draw do
|
|
|
111
117
|
resource :metrics, only: [ :show ], controller: "metrics"
|
|
112
118
|
|
|
113
119
|
# Conversations (contexts, messages, generations) behind Interactions.
|
|
114
|
-
|
|
120
|
+
# The runner edits a conversation in place — seeds, fixes or drops a
|
|
121
|
+
# turn — so the next run sees exactly the history it should.
|
|
122
|
+
resources :interactions, only: [ :index, :show ] do
|
|
123
|
+
resources :messages, only: [ :create, :update, :destroy ], controller: "interaction_messages"
|
|
124
|
+
end
|
|
115
125
|
|
|
116
|
-
# Agent output evaluations.
|
|
126
|
+
# Agent output evaluations. A scenario suite also manages its scenarios
|
|
127
|
+
# here, and exposes each run's per-scenario, per-model results.
|
|
117
128
|
resources :evaluations, only: [ :index, :show, :create, :destroy ] do
|
|
118
129
|
member do
|
|
119
130
|
post :run
|
|
131
|
+
get "runs/:run_id", action: :show_run, as: :run_result
|
|
132
|
+
get "runs/:run_id/report", action: :run_report, as: :run_report
|
|
133
|
+
get :scenarios
|
|
134
|
+
put :scenarios, action: :replace_scenarios
|
|
135
|
+
patch "scenarios/:scenario_id", action: :update_scenario, as: :scenario
|
|
136
|
+
delete "scenarios/:scenario_id", action: :destroy_scenario
|
|
120
137
|
end
|
|
121
138
|
end
|
|
122
139
|
|
|
@@ -128,6 +145,12 @@ ActionAgent::Engine.routes.draw do
|
|
|
128
145
|
# Model catalogs for the agent builder (Ollama queried live from the
|
|
129
146
|
# configured host; hosted providers curated).
|
|
130
147
|
resources :provider_models, only: [ :index ]
|
|
148
|
+
|
|
149
|
+
# The plan meter the Organization view and the Run Agents quota banner
|
|
150
|
+
# read. The engine meters nothing itself: a host that tracks usage
|
|
151
|
+
# against a plan answers through ActionAgent.usage_resolver, and a bare
|
|
152
|
+
# mount reports unlimited rather than 404.
|
|
153
|
+
resource :usage, only: [ :show ], controller: "usage"
|
|
131
154
|
end
|
|
132
155
|
|
|
133
156
|
# The account's agents presented as an authenticated MCP server (tools +
|
|
@@ -136,6 +159,16 @@ ActionAgent::Engine.routes.draw do
|
|
|
136
159
|
# namespace's session-authenticated controllers.
|
|
137
160
|
post "mcp", to: "api/mcp#create"
|
|
138
161
|
|
|
162
|
+
# MCP Streamable HTTP (2025-03-26): a client MAY open the server-to-client
|
|
163
|
+
# SSE stream with GET, and ends a session with DELETE. This facade offers
|
|
164
|
+
# no stream and keeps no sessions, so both answer 405 with Allow: POST —
|
|
165
|
+
# the clean "not offered" signal SDK clients expect, instead of the
|
|
166
|
+
# dashboard's HTML page parsed as an event stream. A browser's GET (Accept
|
|
167
|
+
# prefers HTML) is the MCP Services view's deep link, and falls through to
|
|
168
|
+
# the catch-all below like any other client-side route.
|
|
169
|
+
match "mcp", to: "api/mcp#unsupported", via: [ :get, :delete ],
|
|
170
|
+
constraints: ->(request) { request.delete? || !ActionAgent::Engine.html_request?(request) }
|
|
171
|
+
|
|
139
172
|
# Everything else under the mount is a client-side route: render the
|
|
140
173
|
# dashboard and let the browser resolve it. Anchored last so it can only
|
|
141
174
|
# ever catch what the routes above did not, and refuses /api paths so a
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module ActionAgent
|
|
4
|
+
# Runs before Rails' request logger, including for requests later rejected by
|
|
5
|
+
# authentication, consent or CSRF. Match the endpoint under any engine mount;
|
|
6
|
+
# leave the host application's unrelated message/history parameters alone.
|
|
7
|
+
class AssistantRequestFilter
|
|
8
|
+
PATH = %r{/api/dashboard_assistant(?:\.[^/]+)?/?\z}
|
|
9
|
+
PARAMETERS = [ /\Amessage\z/i, /\Ahistory\z/i ].freeze
|
|
10
|
+
|
|
11
|
+
def initialize(app)
|
|
12
|
+
@app = app
|
|
13
|
+
end
|
|
14
|
+
|
|
15
|
+
def call(env)
|
|
16
|
+
if PATH.match?(env["PATH_INFO"].to_s)
|
|
17
|
+
env["action_dispatch.parameter_filter"] = Array(env["action_dispatch.parameter_filter"]) + PARAMETERS
|
|
18
|
+
end
|
|
19
|
+
@app.call(env)
|
|
20
|
+
end
|
|
21
|
+
end
|
|
22
|
+
end
|