actionagent 1.3.0 → 1.5.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/app/assets/builds/action_agent.css +1 -1
- data/app/assets/builds/action_agent.js +70 -48
- data/app/controllers/action_agent/api/agent_runs_controller.rb +3 -1
- data/app/controllers/action_agent/api/agents_controller.rb +117 -10
- data/app/controllers/action_agent/api/dashboard_assistant_controller.rb +83 -0
- data/app/controllers/action_agent/api/evaluations_controller.rb +22 -7
- data/app/controllers/action_agent/api/interaction_messages_controller.rb +98 -0
- data/app/controllers/action_agent/dashboard_controller.rb +2 -1
- data/app/models/action_agent/agent.rb +107 -19
- data/app/models/action_agent/agent_run.rb +99 -0
- data/app/models/action_agent/evaluation_run.rb +14 -6
- data/app/models/action_agent/evaluation_scenario_result.rb +24 -4
- data/app/models/action_agent/telemetry_trace.rb +16 -1
- data/app/serializers/action_agent/agent_message_serializer.rb +1 -0
- data/app/services/action_agent/agent_execution_service.rb +294 -16
- data/app/services/action_agent/agent_registrar.rb +7 -6
- data/app/services/action_agent/agent_toolbox.rb +69 -5
- data/app/services/action_agent/dashboard_assistant_service.rb +342 -0
- data/app/services/action_agent/evaluation_evidence.rb +234 -0
- data/app/services/action_agent/evaluation_runner_service.rb +13 -3
- data/app/services/action_agent/evaluation_tool_resolver.rb +10 -2
- data/app/services/action_agent/mcp_client.rb +167 -0
- data/app/services/action_agent/mcp_tool_dispatcher.rb +148 -0
- data/app/services/action_agent/playwright_mcp_client.rb +11 -126
- data/app/services/action_agent/scenario_evaluation_runner.rb +90 -17
- data/config/routes.rb +11 -1
- data/lib/action_agent/assistant_request_filter.rb +22 -0
- data/lib/action_agent/engine.rb +5 -0
- data/lib/action_agent/version.rb +1 -1
- data/lib/action_agent.rb +113 -0
- data/lib/generators/action_agent/templates/action_agent.rb.erb +12 -0
- metadata +9 -2
|
@@ -1,20 +1,10 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
3
|
module ActionAgent
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
# Minimal MCP client (streamable HTTP transport) for a Playwright MCP
|
|
8
|
-
# server — typically `npx @playwright/mcp --port 8931` running beside the
|
|
9
|
-
# app in development, or a sandbox-provisioned browser container in
|
|
10
|
-
# production. Speaks just enough JSON-RPC for tools/call: initialize once
|
|
11
|
-
# per process, then call tools under the session id the server hands back.
|
|
12
|
-
class PlaywrightMCPClient
|
|
4
|
+
# The Playwright MCP server as one configured MCPClient: a process-wide
|
|
5
|
+
# instance pointed at the url PLAYWRIGHT_MCP_URL names.
|
|
6
|
+
class PlaywrightMCPClient < MCPClient
|
|
13
7
|
DEFAULT_URL = ENV.fetch("PLAYWRIGHT_MCP_URL", "http://host.orb.internal:8931/mcp")
|
|
14
|
-
OPEN_TIMEOUT_SECONDS = 5
|
|
15
|
-
READ_TIMEOUT_SECONDS = 60
|
|
16
|
-
|
|
17
|
-
class Error < StandardError; end
|
|
18
8
|
|
|
19
9
|
def self.instance
|
|
20
10
|
@instance ||= new
|
|
@@ -25,124 +15,19 @@ module ActionAgent
|
|
|
25
15
|
end
|
|
26
16
|
|
|
27
17
|
def initialize(url: DEFAULT_URL)
|
|
28
|
-
|
|
29
|
-
@mutex = Mutex.new
|
|
18
|
+
super(url: url, label: "Playwright")
|
|
30
19
|
end
|
|
31
20
|
|
|
32
|
-
#
|
|
21
|
+
# Restarting the shared instance on an unreachable server is this
|
|
22
|
+
# subclass's concern: the next call re-initializes rather than reusing a
|
|
23
|
+
# session the server has forgotten. The message names the local fix.
|
|
33
24
|
def call_tool(name, arguments = {})
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
response = post(
|
|
37
|
-
{ jsonrpc: "2.0", id: next_id, method: "tools/call",
|
|
38
|
-
params: { name: name, arguments: arguments } },
|
|
39
|
-
session: @session_id
|
|
40
|
-
)
|
|
41
|
-
result = response["result"]
|
|
42
|
-
unless result
|
|
43
|
-
Rails.logger.warn("[PlaywrightMCPClient] #{name} unexpected response: #{response.inspect[0, 500]}")
|
|
44
|
-
raise Error, (response.dig("error", "message") || "empty MCP response")
|
|
45
|
-
end
|
|
46
|
-
|
|
47
|
-
text = Array(result["content"]).filter_map { |block| block["text"] }.join("\n")
|
|
48
|
-
{ text: text, is_error: result["isError"] ? true : false }
|
|
49
|
-
rescue Errno::ECONNREFUSED, Errno::EHOSTUNREACH, Net::OpenTimeout, SocketError => e
|
|
25
|
+
super
|
|
26
|
+
rescue MCPClient::Error => e
|
|
50
27
|
self.class.reset!
|
|
51
|
-
raise Error, "
|
|
52
|
-
end
|
|
53
|
-
|
|
54
|
-
private
|
|
55
|
-
|
|
56
|
-
def ensure_session!
|
|
57
|
-
@mutex.synchronize do
|
|
58
|
-
next if @session_id
|
|
59
|
-
|
|
60
|
-
_body, response = post_raw(
|
|
61
|
-
{ jsonrpc: "2.0", id: next_id, method: "initialize",
|
|
62
|
-
params: { protocolVersion: "2025-03-26", capabilities: {},
|
|
63
|
-
clientInfo: { name: "activeagents", version: "1.0" } } }
|
|
64
|
-
)
|
|
65
|
-
@session_id = response["mcp-session-id"]
|
|
66
|
-
raise Error, "MCP server did not return a session id" unless @session_id
|
|
67
|
-
|
|
68
|
-
post({ jsonrpc: "2.0", method: "notifications/initialized" }, session: @session_id)
|
|
69
|
-
end
|
|
70
|
-
end
|
|
71
|
-
|
|
72
|
-
def post(payload, session: nil)
|
|
73
|
-
body, _response = post_raw(payload, session: session)
|
|
74
|
-
body
|
|
75
|
-
end
|
|
76
|
-
|
|
77
|
-
def post_raw(payload, session: nil)
|
|
78
|
-
# Tool calls run inside the provider SDK's streaming enumerator — a
|
|
79
|
-
# fiber, where Net::HTTP reads of SSE bodies misbehave (headers arrive,
|
|
80
|
-
# body comes back empty). A dedicated thread always does real blocking
|
|
81
|
-
# IO outside any fiber/scheduler context.
|
|
82
|
-
Thread.new { blocking_post_raw(payload, session: session) }.value
|
|
83
|
-
end
|
|
84
|
-
|
|
85
|
-
def blocking_post_raw(payload, session: nil)
|
|
86
|
-
http = Net::HTTP.new(@uri.host, @uri.port)
|
|
87
|
-
# Container->host bridge hostnames (host.orb.internal) publish an IPv6
|
|
88
|
-
# address whose path doesn't reach the server; dual-stack connects then
|
|
89
|
-
# fail intermittently. Pin to IPv4 while keeping the Host header.
|
|
90
|
-
if (ipv4 = ipv4_address)
|
|
91
|
-
http.ipaddr = ipv4
|
|
92
|
-
end
|
|
93
|
-
http.open_timeout = OPEN_TIMEOUT_SECONDS
|
|
94
|
-
http.read_timeout = READ_TIMEOUT_SECONDS
|
|
95
|
-
request = Net::HTTP::Post.new(@uri.request_uri)
|
|
96
|
-
request["Content-Type"] = "application/json"
|
|
97
|
-
request["Accept"] = "application/json, text/event-stream"
|
|
98
|
-
request["Mcp-Session-Id"] = session if session
|
|
99
|
-
request.body = payload.to_json
|
|
100
|
-
|
|
101
|
-
response = http.request(request)
|
|
102
|
-
Rails.logger.debug(
|
|
103
|
-
"[PlaywrightMCPClient] #{payload[:method]} -> #{response.code} " \
|
|
104
|
-
"ct=#{response['Content-Type']} bytes=#{response.body.to_s.bytesize} session=#{session ? 'yes' : 'no'}"
|
|
105
|
-
)
|
|
106
|
-
unless response.code.to_i.between?(200, 299)
|
|
107
|
-
Rails.logger.warn("[PlaywrightMCPClient] HTTP #{response.code}: #{response.body.to_s[0, 300]}")
|
|
108
|
-
raise Error, "MCP server returned HTTP #{response.code}"
|
|
109
|
-
end
|
|
110
|
-
|
|
111
|
-
parsed = parse_body(response)
|
|
112
|
-
if parsed.empty? && payload[:id]
|
|
113
|
-
Rails.logger.warn("[PlaywrightMCPClient] unparsed body (#{response['Content-Type']}): #{response.body.to_s[0, 500]}")
|
|
114
|
-
end
|
|
115
|
-
[ parsed, response ]
|
|
116
|
-
end
|
|
117
|
-
|
|
118
|
-
# Streamable HTTP answers as plain JSON or as an SSE stream whose data:
|
|
119
|
-
# lines carry the JSON-RPC response — accept both.
|
|
120
|
-
def parse_body(response)
|
|
121
|
-
body = response.body.to_s
|
|
122
|
-
return {} if body.empty?
|
|
123
|
-
|
|
124
|
-
if response["Content-Type"].to_s.include?("text/event-stream")
|
|
125
|
-
body.lines
|
|
126
|
-
.select { |line| line.start_with?("data:") }
|
|
127
|
-
.filter_map { |line| JSON.parse(line.delete_prefix("data:").strip) rescue nil }
|
|
128
|
-
.find { |json| json["result"] || json["error"] } || {}
|
|
129
|
-
else
|
|
130
|
-
JSON.parse(body)
|
|
131
|
-
end
|
|
132
|
-
rescue JSON::ParserError
|
|
133
|
-
{}
|
|
134
|
-
end
|
|
135
|
-
|
|
136
|
-
def ipv4_address
|
|
137
|
-
return @ipv4_address if defined?(@ipv4_address)
|
|
138
|
-
|
|
139
|
-
@ipv4_address = Resolv.getaddresses(@uri.host).find { |address| address =~ Resolv::IPv4::Regex }
|
|
140
|
-
rescue Resolv::ResolvError
|
|
141
|
-
@ipv4_address = nil
|
|
142
|
-
end
|
|
28
|
+
raise Error, "#{e.message}: start it with `npx @playwright/mcp --port 8931`" if e.message.include?("unreachable")
|
|
143
29
|
|
|
144
|
-
|
|
145
|
-
@id = (@id || 0) + 1
|
|
30
|
+
raise
|
|
146
31
|
end
|
|
147
32
|
end
|
|
148
33
|
end
|
|
@@ -45,27 +45,42 @@ module ActionAgent
|
|
|
45
45
|
return run
|
|
46
46
|
end
|
|
47
47
|
|
|
48
|
-
ensure_judge_defined_kpis! if @evaluation.judge_defined?
|
|
49
|
-
|
|
50
48
|
records = scenarios.index_by(&:key)
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
49
|
+
tasks = scenarios.map { |scenario| Evals::Scenario.from_hash(scenario.as_json_summary) }
|
|
50
|
+
expected = tasks.product(specs).map { |task, spec| [ task.key, spec.label ] }
|
|
51
|
+
# Every result tests membership twice, so look the pairs up in a set;
|
|
52
|
+
# `expected` stays an array for the completeness comparison below, and
|
|
53
|
+
# `persisted` sorts the same way either way.
|
|
54
|
+
allowed = expected.to_set
|
|
55
|
+
persisted = Set.new
|
|
56
|
+
on_result = lambda do |result|
|
|
57
|
+
pair = [ result.scenario.key, result.label ]
|
|
58
|
+
raise ArgumentError, "unexpected or duplicate scenario evaluation result" unless allowed.include?(pair) && !persisted.include?(pair)
|
|
59
|
+
|
|
60
|
+
persist(run, records.fetch(result.scenario.key), result)
|
|
61
|
+
persisted << pair
|
|
62
|
+
end
|
|
63
|
+
adapter = ActionAgent.scenario_evaluation_adapter_resolver&.call(@evaluation)
|
|
64
|
+
report = if adapter
|
|
65
|
+
raise ArgumentError, "scenario evaluation adapter must be callable" unless adapter.respond_to?(:call)
|
|
66
|
+
|
|
67
|
+
adapter.call(evaluation: @evaluation, owner: owner, scenarios: tasks, models: specs, on_result: on_result)
|
|
68
|
+
else
|
|
69
|
+
ensure_judge_defined_kpis! if @evaluation.judge_defined?
|
|
70
|
+
default_report(tasks, specs, on_result)
|
|
71
|
+
end
|
|
72
|
+
raise ArgumentError, "scenario evaluation adapter must return an ActiveAgent::Evals::Report" unless report.is_a?(Evals::Report)
|
|
73
|
+
reported = report.results.map { |result| [ result.scenario.key, result.label ] }
|
|
74
|
+
unless reported.sort == expected.sort && persisted.sort == expected.sort
|
|
75
|
+
raise ArgumentError, "scenario evaluation adapter must report and persist every selected scenario and model"
|
|
76
|
+
end
|
|
63
77
|
|
|
64
78
|
run.update!(
|
|
65
79
|
status: :complete,
|
|
66
80
|
scores: scores_for(report, run),
|
|
67
81
|
samples_evaluated: report.results.size,
|
|
68
82
|
samples_passed: report.results.count(&:passed?),
|
|
83
|
+
error_message: discovery_warning,
|
|
69
84
|
completed_at: Time.current
|
|
70
85
|
)
|
|
71
86
|
run
|
|
@@ -76,6 +91,21 @@ module ActionAgent
|
|
|
76
91
|
|
|
77
92
|
private
|
|
78
93
|
|
|
94
|
+
def default_report(tasks, specs, on_result)
|
|
95
|
+
Evals::Runner.new(
|
|
96
|
+
scenarios: tasks,
|
|
97
|
+
models: specs,
|
|
98
|
+
criteria: sample_criteria,
|
|
99
|
+
judge: evals_judge,
|
|
100
|
+
available_tools: tool_roster,
|
|
101
|
+
instructions: @evaluation.agent.instructions,
|
|
102
|
+
agent_name: @evaluation.agent.name,
|
|
103
|
+
threshold: PASS_THRESHOLD,
|
|
104
|
+
replay: ->(scenario, spec) { replay(scenario, spec) },
|
|
105
|
+
on_result: on_result
|
|
106
|
+
).call
|
|
107
|
+
end
|
|
108
|
+
|
|
79
109
|
# --- selection --------------------------------------------------------
|
|
80
110
|
|
|
81
111
|
def selected_scenarios
|
|
@@ -184,7 +214,10 @@ module ActionAgent
|
|
|
184
214
|
cost: result.replay.cost,
|
|
185
215
|
fault: result.fault,
|
|
186
216
|
recommendation: result.recommendation,
|
|
187
|
-
diagnosis: result.diagnosis || {}
|
|
217
|
+
diagnosis: (result.diagnosis || {}).merge(
|
|
218
|
+
"_replay_metadata" => result.replay.metadata,
|
|
219
|
+
"_scenario_snapshot" => result.scenario.to_h.merge(expectations: result.scenario.expectations)
|
|
220
|
+
),
|
|
188
221
|
error_message: result.replay.error
|
|
189
222
|
)
|
|
190
223
|
end
|
|
@@ -195,6 +228,8 @@ module ActionAgent
|
|
|
195
228
|
scores["_recommendations"] = report.recommendations
|
|
196
229
|
scores["_verdict"] = report.verdict if report.comparing?
|
|
197
230
|
scores["_selection"] = run.selection
|
|
231
|
+
scores["_metadata"] = report.metadata
|
|
232
|
+
scores["_judge_label"] = report.judge_label || report.judge&.label
|
|
198
233
|
scores
|
|
199
234
|
end
|
|
200
235
|
|
|
@@ -206,12 +241,50 @@ module ActionAgent
|
|
|
206
241
|
end
|
|
207
242
|
end
|
|
208
243
|
|
|
244
|
+
# Every tool the agent could actually call, for the diagnosis roster: the
|
|
245
|
+
# engine's own toolbox plus whatever its MCP servers serve.
|
|
246
|
+
#
|
|
247
|
+
# Listing only the toolbox understated the roster, so a diagnosis could
|
|
248
|
+
# report "none of the available tools covers this task" while naming a list
|
|
249
|
+
# the agent's MCP tools were missing from.
|
|
209
250
|
def tool_roster
|
|
210
|
-
@tool_roster ||=
|
|
211
|
-
|
|
251
|
+
@tool_roster ||= begin
|
|
252
|
+
definitions = mcp_dispatcher.tool_definitions +
|
|
253
|
+
AgentToolbox.definitions_for(@evaluation.agent.tools)
|
|
254
|
+
|
|
255
|
+
definitions.to_h { |definition| [ definition[:name].to_s, definition[:description].to_s ] }
|
|
212
256
|
end
|
|
213
257
|
end
|
|
214
258
|
|
|
259
|
+
# Discovery failures from the roster above, keyed by server. Reading them
|
|
260
|
+
# requires tool_definitions to have run, which tool_roster does.
|
|
261
|
+
def mcp_discovery_errors
|
|
262
|
+
tool_roster
|
|
263
|
+
mcp_dispatcher.discovery_errors
|
|
264
|
+
end
|
|
265
|
+
|
|
266
|
+
def mcp_dispatcher
|
|
267
|
+
@mcp_dispatcher ||= MCPToolDispatcher.new(@evaluation.agent)
|
|
268
|
+
end
|
|
269
|
+
|
|
270
|
+
# A run whose every declared MCP server failed discovery scored an agent
|
|
271
|
+
# that had no tools to call. The scores are real but meaningless — the
|
|
272
|
+
# model answered from its own weights — so the run says so rather than
|
|
273
|
+
# leaving the cause in a log line (#425).
|
|
274
|
+
def discovery_warning
|
|
275
|
+
errors = mcp_discovery_errors
|
|
276
|
+
return nil if errors.empty?
|
|
277
|
+
|
|
278
|
+
prefix = if mcp_dispatcher.all_servers_failed?
|
|
279
|
+
"Every MCP server this agent declares failed tool discovery, so it ran with no MCP tools " \
|
|
280
|
+
"and any specifics in its answers are unverified."
|
|
281
|
+
else
|
|
282
|
+
"Some of this agent's MCP servers failed tool discovery, so part of its toolset was unavailable."
|
|
283
|
+
end
|
|
284
|
+
|
|
285
|
+
"#{prefix} #{errors.values.join(' ')}"
|
|
286
|
+
end
|
|
287
|
+
|
|
215
288
|
# The judge the evaluation's owner has credentials for, wrapped for the
|
|
216
289
|
# evaluation core; nil when none is configured, in which case scoring
|
|
217
290
|
# stays on rules and expectations.
|
data/config/routes.rb
CHANGED
|
@@ -20,6 +20,8 @@ ActionAgent::Engine.routes.draw do
|
|
|
20
20
|
|
|
21
21
|
# The dashboard's own JSON API, read and written by the React app.
|
|
22
22
|
namespace :api do
|
|
23
|
+
resource :dashboard_assistant, only: [ :show, :create ], controller: "dashboard_assistant"
|
|
24
|
+
|
|
23
25
|
# Telemetry ingestion, relative to wherever the engine is mounted:
|
|
24
26
|
# <mount>/api/traces (e.g. /activeagents/api/traces at the default mount).
|
|
25
27
|
# Authenticated with a bearer token, not a session.
|
|
@@ -36,6 +38,10 @@ ActionAgent::Engine.routes.draw do
|
|
|
36
38
|
post :duplicate
|
|
37
39
|
get :export
|
|
38
40
|
get :analytics
|
|
41
|
+
# The runner's conversation picker: this agent's persisted contexts,
|
|
42
|
+
# and a fresh one to pin a first message to.
|
|
43
|
+
get :conversations
|
|
44
|
+
post :conversations, action: :create_conversation
|
|
39
45
|
end
|
|
40
46
|
collection do
|
|
41
47
|
get :presets
|
|
@@ -111,7 +117,11 @@ ActionAgent::Engine.routes.draw do
|
|
|
111
117
|
resource :metrics, only: [ :show ], controller: "metrics"
|
|
112
118
|
|
|
113
119
|
# Conversations (contexts, messages, generations) behind Interactions.
|
|
114
|
-
|
|
120
|
+
# The runner edits a conversation in place — seeds, fixes or drops a
|
|
121
|
+
# turn — so the next run sees exactly the history it should.
|
|
122
|
+
resources :interactions, only: [ :index, :show ] do
|
|
123
|
+
resources :messages, only: [ :create, :update, :destroy ], controller: "interaction_messages"
|
|
124
|
+
end
|
|
115
125
|
|
|
116
126
|
# Agent output evaluations. A scenario suite also manages its scenarios
|
|
117
127
|
# here, and exposes each run's per-scenario, per-model results.
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module ActionAgent
|
|
4
|
+
# Runs before Rails' request logger, including for requests later rejected by
|
|
5
|
+
# authentication, consent or CSRF. Match the endpoint under any engine mount;
|
|
6
|
+
# leave the host application's unrelated message/history parameters alone.
|
|
7
|
+
class AssistantRequestFilter
|
|
8
|
+
PATH = %r{/api/dashboard_assistant(?:\.[^/]+)?/?\z}
|
|
9
|
+
PARAMETERS = [ /\Amessage\z/i, /\Ahistory\z/i ].freeze
|
|
10
|
+
|
|
11
|
+
def initialize(app)
|
|
12
|
+
@app = app
|
|
13
|
+
end
|
|
14
|
+
|
|
15
|
+
def call(env)
|
|
16
|
+
if PATH.match?(env["PATH_INFO"].to_s)
|
|
17
|
+
env["action_dispatch.parameter_filter"] = Array(env["action_dispatch.parameter_filter"]) + PARAMETERS
|
|
18
|
+
end
|
|
19
|
+
@app.call(env)
|
|
20
|
+
end
|
|
21
|
+
end
|
|
22
|
+
end
|
data/lib/action_agent/engine.rb
CHANGED
|
@@ -1,5 +1,7 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
+
require_relative "assistant_request_filter"
|
|
4
|
+
|
|
3
5
|
module ActionAgent
|
|
4
6
|
# Rails engine for the Active Agent dashboard: the agent builder, runs,
|
|
5
7
|
# conversations, evaluations, traces, metrics, sandboxes and session
|
|
@@ -19,6 +21,8 @@ module ActionAgent
|
|
|
19
21
|
# rather than relying on the host to register one.
|
|
20
22
|
INFLECTION_OVERRIDES = {
|
|
21
23
|
"mcp_catalog" => "MCPCatalog",
|
|
24
|
+
"mcp_client" => "MCPClient",
|
|
25
|
+
"mcp_tool_dispatcher" => "MCPToolDispatcher",
|
|
22
26
|
"mcp_controller" => "MCPController",
|
|
23
27
|
"mcp_recording_middleware" => "MCPRecordingMiddleware",
|
|
24
28
|
"mcp_servers_controller" => "MCPServersController",
|
|
@@ -35,6 +39,7 @@ module ActionAgent
|
|
|
35
39
|
}.freeze
|
|
36
40
|
|
|
37
41
|
config.action_agent = ActiveSupport::OrderedOptions.new
|
|
42
|
+
config.app_middleware.insert_before Rails::Rack::Logger, ActionAgent::AssistantRequestFilter
|
|
38
43
|
|
|
39
44
|
# Whether a request is a browser asking for a page, as opposed to an API
|
|
40
45
|
# or MCP client: the routes use it to tell the dashboard's client-side
|
data/lib/action_agent/version.rb
CHANGED
data/lib/action_agent.rb
CHANGED
|
@@ -236,6 +236,25 @@ module ActionAgent
|
|
|
236
236
|
# @return [Boolean]
|
|
237
237
|
attr_accessor :execution_enabled
|
|
238
238
|
|
|
239
|
+
# Whether the "Ask ActiveAgents" assistant is available.
|
|
240
|
+
#
|
|
241
|
+
# The assistant is a tool for developing and CI-ing agents: it sends
|
|
242
|
+
# recorded prompts, outputs and evaluation report excerpts to a model
|
|
243
|
+
# provider, which is the right trade in a development or CI workspace
|
|
244
|
+
# and a decision nobody should inherit by default in production. Left
|
|
245
|
+
# unset it is on in development and test only. Set it to true to run it
|
|
246
|
+
# somewhere else deliberately, or false to remove it everywhere.
|
|
247
|
+
# @return [Boolean, nil]
|
|
248
|
+
attr_accessor :assistant_enabled
|
|
249
|
+
|
|
250
|
+
# Resolves a host application's runner for one scenario evaluation.
|
|
251
|
+
# Return nil for the engine's normal Agent#test_execute path, or a callable
|
|
252
|
+
# accepting evaluation:, owner:, scenarios:, models:, on_result: and
|
|
253
|
+
# returning an ActiveAgent::Evals::Report. The host runs its own agent and
|
|
254
|
+
# judge and yields every result to on_result for dashboard persistence.
|
|
255
|
+
# @return [Proc, nil]
|
|
256
|
+
attr_accessor :scenario_evaluation_adapter_resolver
|
|
257
|
+
|
|
239
258
|
# Where the dashboard's upgrade CTAs should send people. Unset in a
|
|
240
259
|
# self-hosted install, where there is nothing to upgrade, and the CTAs
|
|
241
260
|
# say so instead of linking nowhere.
|
|
@@ -311,6 +330,30 @@ module ActionAgent
|
|
|
311
330
|
# @return [Array<Hash>]
|
|
312
331
|
attr_accessor :mcp_catalog
|
|
313
332
|
|
|
333
|
+
# Host-declared ActiveAgent::SchemaTools subclasses whose generated tools
|
|
334
|
+
# are offered alongside AgentToolbox's built-ins: each generated tool is
|
|
335
|
+
# individually selectable in the agent editor, dispatched by name at
|
|
336
|
+
# execution, and named in an evaluation's +tools:+ expectation.
|
|
337
|
+
#
|
|
338
|
+
# ActionAgent.configure do |config|
|
|
339
|
+
# config.schema_tools = [TicketTools, TaskTools, MilestoneTools]
|
|
340
|
+
# end
|
|
341
|
+
#
|
|
342
|
+
# Accepts classes or class-name strings; strings are resolved lazily so a
|
|
343
|
+
# host can declare them from an initializer before autoloading has run.
|
|
344
|
+
#
|
|
345
|
+
# Leave it unset and every ActiveAgent::SchemaTools subclass found in
|
|
346
|
+
# {#schema_tools_path} is offered instead — a host adds a tool by adding a
|
|
347
|
+
# file, without naming it twice.
|
|
348
|
+
# @return [Array<Class, String>, nil]
|
|
349
|
+
attr_accessor :schema_tools
|
|
350
|
+
|
|
351
|
+
# Directory scanned for SchemaTools subclasses when {#schema_tools} is
|
|
352
|
+
# unset. Relative to the host's root. Set to nil to disable discovery and
|
|
353
|
+
# require an explicit declaration.
|
|
354
|
+
# @return [String, nil]
|
|
355
|
+
attr_accessor :schema_tools_path
|
|
356
|
+
|
|
314
357
|
# Value stored in polymorphic *_type columns for dashboard agents
|
|
315
358
|
# (agent_memories.memorable_type, agent_contexts.contextable_type).
|
|
316
359
|
# Unset means the class name. A host app whose existing rows were
|
|
@@ -332,6 +375,16 @@ module ActionAgent
|
|
|
332
375
|
@execution_enabled != false
|
|
333
376
|
end
|
|
334
377
|
|
|
378
|
+
# Returns whether the dashboard assistant is available. Unconfigured, it
|
|
379
|
+
# follows the environment: development and test yes, everywhere else no.
|
|
380
|
+
#
|
|
381
|
+
# @return [Boolean]
|
|
382
|
+
def assistant_enabled?
|
|
383
|
+
return @assistant_enabled == true unless @assistant_enabled.nil?
|
|
384
|
+
|
|
385
|
+
Rails.env.local?
|
|
386
|
+
end
|
|
387
|
+
|
|
335
388
|
# Tells the host app that +owner+ performed +kind+. Never raises: a
|
|
336
389
|
# bookkeeping failure must not fail the action that was already taken.
|
|
337
390
|
def record_usage(owner, kind)
|
|
@@ -452,6 +505,9 @@ module ActionAgent
|
|
|
452
505
|
@provider_credentials_resolver = nil
|
|
453
506
|
@sandbox_backends = {}
|
|
454
507
|
@execution_enabled = true
|
|
508
|
+
@assistant_enabled = nil
|
|
509
|
+
|
|
510
|
+
@scenario_evaluation_adapter_resolver = nil
|
|
455
511
|
@table_name_prefix = "active_agent_"
|
|
456
512
|
@agent_polymorphic_name = nil
|
|
457
513
|
@encrypt_credentials = true
|
|
@@ -463,6 +519,63 @@ module ActionAgent
|
|
|
463
519
|
@sign_out_path = nil
|
|
464
520
|
@sign_in_path = nil
|
|
465
521
|
@mcp_catalog = []
|
|
522
|
+
@schema_tools = nil
|
|
523
|
+
@schema_tools_path = "app/agent_tools"
|
|
524
|
+
end
|
|
525
|
+
|
|
526
|
+
# Host-declared schema tool classes, resolved from names and filtered to
|
|
527
|
+
# those that are usable (declared a model and generated a roster).
|
|
528
|
+
#
|
|
529
|
+
# Resolution happens per call rather than at configure time: a host
|
|
530
|
+
# declares these in an initializer, before its own classes are autoloaded.
|
|
531
|
+
# @return [Array<Class>]
|
|
532
|
+
def schema_tool_classes
|
|
533
|
+
declared = @schema_tools.nil? ? discovered_schema_tools : Array(@schema_tools)
|
|
534
|
+
|
|
535
|
+
declared.filter_map do |entry|
|
|
536
|
+
klass = entry.is_a?(String) ? entry.safe_constantize : entry
|
|
537
|
+
next unless klass.respond_to?(:tool_definitions) && klass.respond_to?(:model)
|
|
538
|
+
next if klass.model.blank?
|
|
539
|
+
# An anonymous class built at runtime is usable but not discoverable —
|
|
540
|
+
# it would accumulate across reloads with no way to supersede itself.
|
|
541
|
+
next if entry.is_a?(Class) && klass.name.blank? && @schema_tools.nil?
|
|
542
|
+
|
|
543
|
+
klass
|
|
544
|
+
end
|
|
545
|
+
end
|
|
546
|
+
|
|
547
|
+
# Every tool name generated by the declared schema tool classes.
|
|
548
|
+
# @return [Array<String>]
|
|
549
|
+
def schema_tool_names
|
|
550
|
+
schema_tool_classes.flat_map(&:tool_names).map(&:to_s)
|
|
551
|
+
end
|
|
552
|
+
|
|
553
|
+
# SchemaTools subclasses defined under {#schema_tools_path}.
|
|
554
|
+
#
|
|
555
|
+
# The files are loaded before reading +descendants+: in development nothing
|
|
556
|
+
# has referenced those constants yet, so the list would otherwise be empty
|
|
557
|
+
# at boot and fill in only once something happened to touch them.
|
|
558
|
+
# @return [Array<Class>]
|
|
559
|
+
def discovered_schema_tools
|
|
560
|
+
return [] if @schema_tools_path.blank? || !defined?(ActiveAgent::SchemaTools)
|
|
561
|
+
return [] unless defined?(Rails) && Rails.respond_to?(:root) && Rails.root
|
|
562
|
+
|
|
563
|
+
root = Rails.root.join(@schema_tools_path)
|
|
564
|
+
return [] unless Dir.exist?(root)
|
|
565
|
+
|
|
566
|
+
Dir[root.join("**/*.rb")].sort.each do |path|
|
|
567
|
+
require_dependency path
|
|
568
|
+
rescue StandardError, ScriptError => e
|
|
569
|
+
warn "[ActionAgent] could not load #{path}: #{e.class} - #{e.message}"
|
|
570
|
+
end
|
|
571
|
+
|
|
572
|
+
ActiveAgent::SchemaTools.descendants.select { |klass| klass.name.present? }
|
|
573
|
+
end
|
|
574
|
+
|
|
575
|
+
# The schema tool class that generated +name+, or nil.
|
|
576
|
+
# @return [Class, nil]
|
|
577
|
+
def schema_tool_class_for(name)
|
|
578
|
+
schema_tool_classes.find { |klass| klass.tool?(name) }
|
|
466
579
|
end
|
|
467
580
|
end
|
|
468
581
|
|
|
@@ -70,6 +70,18 @@ ActionAgent.configure do |config|
|
|
|
70
70
|
# nothing rather than everything. An empty dashboard for a signed-in user
|
|
71
71
|
# means the resolver above returned nil.
|
|
72
72
|
|
|
73
|
+
# ==========================================================================
|
|
74
|
+
# Ask ActiveAgents assistant
|
|
75
|
+
# ==========================================================================
|
|
76
|
+
#
|
|
77
|
+
# A tool for developing and CI-ing agents. Answering a question sends
|
|
78
|
+
# recorded prompts, outputs and evaluation report excerpts to a model
|
|
79
|
+
# provider, so the page and its API are on in development and test only.
|
|
80
|
+
# Set it to true to run it in another environment deliberately, or false
|
|
81
|
+
# to remove it everywhere.
|
|
82
|
+
#
|
|
83
|
+
# config.assistant_enabled = true
|
|
84
|
+
|
|
73
85
|
# ==========================================================================
|
|
74
86
|
# UI
|
|
75
87
|
# ==========================================================================
|
metadata
CHANGED
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: actionagent
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 1.
|
|
4
|
+
version: 1.5.2
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Justin Bowen
|
|
8
8
|
autorequire:
|
|
9
9
|
bindir: bin
|
|
10
10
|
cert_chain: []
|
|
11
|
-
date: 2026-09-
|
|
11
|
+
date: 2026-09-11 00:00:00.000000000 Z
|
|
12
12
|
dependencies:
|
|
13
13
|
- !ruby/object:Gem::Dependency
|
|
14
14
|
name: activeagent
|
|
@@ -129,8 +129,10 @@ files:
|
|
|
129
129
|
- app/controllers/action_agent/api/analytics_controller.rb
|
|
130
130
|
- app/controllers/action_agent/api/api_keys_controller.rb
|
|
131
131
|
- app/controllers/action_agent/api/base_controller.rb
|
|
132
|
+
- app/controllers/action_agent/api/dashboard_assistant_controller.rb
|
|
132
133
|
- app/controllers/action_agent/api/evaluations_controller.rb
|
|
133
134
|
- app/controllers/action_agent/api/instance_tiers_controller.rb
|
|
135
|
+
- app/controllers/action_agent/api/interaction_messages_controller.rb
|
|
134
136
|
- app/controllers/action_agent/api/interactions_controller.rb
|
|
135
137
|
- app/controllers/action_agent/api/mcp_controller.rb
|
|
136
138
|
- app/controllers/action_agent/api/mcp_servers_controller.rb
|
|
@@ -193,10 +195,14 @@ files:
|
|
|
193
195
|
- app/services/action_agent/agent_registrar.rb
|
|
194
196
|
- app/services/action_agent/agent_scorecard.rb
|
|
195
197
|
- app/services/action_agent/agent_toolbox.rb
|
|
198
|
+
- app/services/action_agent/dashboard_assistant_service.rb
|
|
199
|
+
- app/services/action_agent/evaluation_evidence.rb
|
|
196
200
|
- app/services/action_agent/evaluation_runner_service.rb
|
|
197
201
|
- app/services/action_agent/evaluation_tool_resolver.rb
|
|
198
202
|
- app/services/action_agent/mcp_catalog.rb
|
|
203
|
+
- app/services/action_agent/mcp_client.rb
|
|
199
204
|
- app/services/action_agent/mcp_recording_middleware.rb
|
|
205
|
+
- app/services/action_agent/mcp_tool_dispatcher.rb
|
|
200
206
|
- app/services/action_agent/mock_sandbox_backend.rb
|
|
201
207
|
- app/services/action_agent/playwright_mcp_client.rb
|
|
202
208
|
- app/services/action_agent/sandbox_orchestrator.rb
|
|
@@ -212,6 +218,7 @@ files:
|
|
|
212
218
|
- app/views/layouts/action_agent/react.html.erb
|
|
213
219
|
- config/routes.rb
|
|
214
220
|
- lib/action_agent.rb
|
|
221
|
+
- lib/action_agent/assistant_request_filter.rb
|
|
215
222
|
- lib/action_agent/compatibility.rb
|
|
216
223
|
- lib/action_agent/engine.rb
|
|
217
224
|
- lib/action_agent/version.rb
|