actionagent 1.3.0 → 1.5.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (33) hide show
  1. checksums.yaml +4 -4
  2. data/app/assets/builds/action_agent.css +1 -1
  3. data/app/assets/builds/action_agent.js +70 -48
  4. data/app/controllers/action_agent/api/agent_runs_controller.rb +3 -1
  5. data/app/controllers/action_agent/api/agents_controller.rb +117 -10
  6. data/app/controllers/action_agent/api/dashboard_assistant_controller.rb +83 -0
  7. data/app/controllers/action_agent/api/evaluations_controller.rb +22 -7
  8. data/app/controllers/action_agent/api/interaction_messages_controller.rb +98 -0
  9. data/app/controllers/action_agent/dashboard_controller.rb +2 -1
  10. data/app/models/action_agent/agent.rb +107 -19
  11. data/app/models/action_agent/agent_run.rb +99 -0
  12. data/app/models/action_agent/evaluation_run.rb +14 -6
  13. data/app/models/action_agent/evaluation_scenario_result.rb +24 -4
  14. data/app/models/action_agent/telemetry_trace.rb +16 -1
  15. data/app/serializers/action_agent/agent_message_serializer.rb +1 -0
  16. data/app/services/action_agent/agent_execution_service.rb +294 -16
  17. data/app/services/action_agent/agent_registrar.rb +7 -6
  18. data/app/services/action_agent/agent_toolbox.rb +69 -5
  19. data/app/services/action_agent/dashboard_assistant_service.rb +342 -0
  20. data/app/services/action_agent/evaluation_evidence.rb +234 -0
  21. data/app/services/action_agent/evaluation_runner_service.rb +13 -3
  22. data/app/services/action_agent/evaluation_tool_resolver.rb +10 -2
  23. data/app/services/action_agent/mcp_client.rb +167 -0
  24. data/app/services/action_agent/mcp_tool_dispatcher.rb +148 -0
  25. data/app/services/action_agent/playwright_mcp_client.rb +11 -126
  26. data/app/services/action_agent/scenario_evaluation_runner.rb +90 -17
  27. data/config/routes.rb +11 -1
  28. data/lib/action_agent/assistant_request_filter.rb +22 -0
  29. data/lib/action_agent/engine.rb +5 -0
  30. data/lib/action_agent/version.rb +1 -1
  31. data/lib/action_agent.rb +113 -0
  32. data/lib/generators/action_agent/templates/action_agent.rb.erb +12 -0
  33. metadata +9 -2
@@ -1,20 +1,10 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  module ActionAgent
4
- require "net/http"
5
- require "resolv"
6
-
7
- # Minimal MCP client (streamable HTTP transport) for a Playwright MCP
8
- # server — typically `npx @playwright/mcp --port 8931` running beside the
9
- # app in development, or a sandbox-provisioned browser container in
10
- # production. Speaks just enough JSON-RPC for tools/call: initialize once
11
- # per process, then call tools under the session id the server hands back.
12
- class PlaywrightMCPClient
4
+ # The Playwright MCP server as one configured MCPClient: a process-wide
5
+ # instance pointed at the url PLAYWRIGHT_MCP_URL names.
6
+ class PlaywrightMCPClient < MCPClient
13
7
  DEFAULT_URL = ENV.fetch("PLAYWRIGHT_MCP_URL", "http://host.orb.internal:8931/mcp")
14
- OPEN_TIMEOUT_SECONDS = 5
15
- READ_TIMEOUT_SECONDS = 60
16
-
17
- class Error < StandardError; end
18
8
 
19
9
  def self.instance
20
10
  @instance ||= new
@@ -25,124 +15,19 @@ module ActionAgent
25
15
  end
26
16
 
27
17
  def initialize(url: DEFAULT_URL)
28
- @uri = URI(url)
29
- @mutex = Mutex.new
18
+ super(url: url, label: "Playwright")
30
19
  end
31
20
 
32
- # Returns { text:, is_error: } — the tool result's text content.
21
+ # Restarting the shared instance on an unreachable server is this
22
+ # subclass's concern: the next call re-initializes rather than reusing a
23
+ # session the server has forgotten. The message names the local fix.
33
24
  def call_tool(name, arguments = {})
34
- Rails.logger.debug("[PlaywrightMCPClient] call #{name} args=#{arguments.inspect[0, 200]}")
35
- ensure_session!
36
- response = post(
37
- { jsonrpc: "2.0", id: next_id, method: "tools/call",
38
- params: { name: name, arguments: arguments } },
39
- session: @session_id
40
- )
41
- result = response["result"]
42
- unless result
43
- Rails.logger.warn("[PlaywrightMCPClient] #{name} unexpected response: #{response.inspect[0, 500]}")
44
- raise Error, (response.dig("error", "message") || "empty MCP response")
45
- end
46
-
47
- text = Array(result["content"]).filter_map { |block| block["text"] }.join("\n")
48
- { text: text, is_error: result["isError"] ? true : false }
49
- rescue Errno::ECONNREFUSED, Errno::EHOSTUNREACH, Net::OpenTimeout, SocketError => e
25
+ super
26
+ rescue MCPClient::Error => e
50
27
  self.class.reset!
51
- raise Error, "Playwright MCP server unreachable at #{@uri} (#{e.class}): start it with `npx @playwright/mcp --port 8931`"
52
- end
53
-
54
- private
55
-
56
- def ensure_session!
57
- @mutex.synchronize do
58
- next if @session_id
59
-
60
- _body, response = post_raw(
61
- { jsonrpc: "2.0", id: next_id, method: "initialize",
62
- params: { protocolVersion: "2025-03-26", capabilities: {},
63
- clientInfo: { name: "activeagents", version: "1.0" } } }
64
- )
65
- @session_id = response["mcp-session-id"]
66
- raise Error, "MCP server did not return a session id" unless @session_id
67
-
68
- post({ jsonrpc: "2.0", method: "notifications/initialized" }, session: @session_id)
69
- end
70
- end
71
-
72
- def post(payload, session: nil)
73
- body, _response = post_raw(payload, session: session)
74
- body
75
- end
76
-
77
- def post_raw(payload, session: nil)
78
- # Tool calls run inside the provider SDK's streaming enumerator — a
79
- # fiber, where Net::HTTP reads of SSE bodies misbehave (headers arrive,
80
- # body comes back empty). A dedicated thread always does real blocking
81
- # IO outside any fiber/scheduler context.
82
- Thread.new { blocking_post_raw(payload, session: session) }.value
83
- end
84
-
85
- def blocking_post_raw(payload, session: nil)
86
- http = Net::HTTP.new(@uri.host, @uri.port)
87
- # Container->host bridge hostnames (host.orb.internal) publish an IPv6
88
- # address whose path doesn't reach the server; dual-stack connects then
89
- # fail intermittently. Pin to IPv4 while keeping the Host header.
90
- if (ipv4 = ipv4_address)
91
- http.ipaddr = ipv4
92
- end
93
- http.open_timeout = OPEN_TIMEOUT_SECONDS
94
- http.read_timeout = READ_TIMEOUT_SECONDS
95
- request = Net::HTTP::Post.new(@uri.request_uri)
96
- request["Content-Type"] = "application/json"
97
- request["Accept"] = "application/json, text/event-stream"
98
- request["Mcp-Session-Id"] = session if session
99
- request.body = payload.to_json
100
-
101
- response = http.request(request)
102
- Rails.logger.debug(
103
- "[PlaywrightMCPClient] #{payload[:method]} -> #{response.code} " \
104
- "ct=#{response['Content-Type']} bytes=#{response.body.to_s.bytesize} session=#{session ? 'yes' : 'no'}"
105
- )
106
- unless response.code.to_i.between?(200, 299)
107
- Rails.logger.warn("[PlaywrightMCPClient] HTTP #{response.code}: #{response.body.to_s[0, 300]}")
108
- raise Error, "MCP server returned HTTP #{response.code}"
109
- end
110
-
111
- parsed = parse_body(response)
112
- if parsed.empty? && payload[:id]
113
- Rails.logger.warn("[PlaywrightMCPClient] unparsed body (#{response['Content-Type']}): #{response.body.to_s[0, 500]}")
114
- end
115
- [ parsed, response ]
116
- end
117
-
118
- # Streamable HTTP answers as plain JSON or as an SSE stream whose data:
119
- # lines carry the JSON-RPC response — accept both.
120
- def parse_body(response)
121
- body = response.body.to_s
122
- return {} if body.empty?
123
-
124
- if response["Content-Type"].to_s.include?("text/event-stream")
125
- body.lines
126
- .select { |line| line.start_with?("data:") }
127
- .filter_map { |line| JSON.parse(line.delete_prefix("data:").strip) rescue nil }
128
- .find { |json| json["result"] || json["error"] } || {}
129
- else
130
- JSON.parse(body)
131
- end
132
- rescue JSON::ParserError
133
- {}
134
- end
135
-
136
- def ipv4_address
137
- return @ipv4_address if defined?(@ipv4_address)
138
-
139
- @ipv4_address = Resolv.getaddresses(@uri.host).find { |address| address =~ Resolv::IPv4::Regex }
140
- rescue Resolv::ResolvError
141
- @ipv4_address = nil
142
- end
28
+ raise Error, "#{e.message}: start it with `npx @playwright/mcp --port 8931`" if e.message.include?("unreachable")
143
29
 
144
- def next_id
145
- @id = (@id || 0) + 1
30
+ raise
146
31
  end
147
32
  end
148
33
  end
@@ -45,27 +45,42 @@ module ActionAgent
45
45
  return run
46
46
  end
47
47
 
48
- ensure_judge_defined_kpis! if @evaluation.judge_defined?
49
-
50
48
  records = scenarios.index_by(&:key)
51
- report = Evals::Runner.new(
52
- scenarios: scenarios.map { |scenario| Evals::Scenario.from_hash(scenario.as_json_summary) },
53
- models: specs,
54
- criteria: sample_criteria,
55
- judge: evals_judge,
56
- available_tools: tool_roster,
57
- instructions: @evaluation.agent.instructions,
58
- agent_name: @evaluation.agent.name,
59
- threshold: PASS_THRESHOLD,
60
- replay: ->(scenario, spec) { replay(scenario, spec) },
61
- on_result: ->(result) { persist(run, records.fetch(result.scenario.key), result) }
62
- ).call
49
+ tasks = scenarios.map { |scenario| Evals::Scenario.from_hash(scenario.as_json_summary) }
50
+ expected = tasks.product(specs).map { |task, spec| [ task.key, spec.label ] }
51
+ # Every result tests membership twice, so look the pairs up in a set;
52
+ # `expected` stays an array for the completeness comparison below, and
53
+ # `persisted` sorts the same way either way.
54
+ allowed = expected.to_set
55
+ persisted = Set.new
56
+ on_result = lambda do |result|
57
+ pair = [ result.scenario.key, result.label ]
58
+ raise ArgumentError, "unexpected or duplicate scenario evaluation result" unless allowed.include?(pair) && !persisted.include?(pair)
59
+
60
+ persist(run, records.fetch(result.scenario.key), result)
61
+ persisted << pair
62
+ end
63
+ adapter = ActionAgent.scenario_evaluation_adapter_resolver&.call(@evaluation)
64
+ report = if adapter
65
+ raise ArgumentError, "scenario evaluation adapter must be callable" unless adapter.respond_to?(:call)
66
+
67
+ adapter.call(evaluation: @evaluation, owner: owner, scenarios: tasks, models: specs, on_result: on_result)
68
+ else
69
+ ensure_judge_defined_kpis! if @evaluation.judge_defined?
70
+ default_report(tasks, specs, on_result)
71
+ end
72
+ raise ArgumentError, "scenario evaluation adapter must return an ActiveAgent::Evals::Report" unless report.is_a?(Evals::Report)
73
+ reported = report.results.map { |result| [ result.scenario.key, result.label ] }
74
+ unless reported.sort == expected.sort && persisted.sort == expected.sort
75
+ raise ArgumentError, "scenario evaluation adapter must report and persist every selected scenario and model"
76
+ end
63
77
 
64
78
  run.update!(
65
79
  status: :complete,
66
80
  scores: scores_for(report, run),
67
81
  samples_evaluated: report.results.size,
68
82
  samples_passed: report.results.count(&:passed?),
83
+ error_message: discovery_warning,
69
84
  completed_at: Time.current
70
85
  )
71
86
  run
@@ -76,6 +91,21 @@ module ActionAgent
76
91
 
77
92
  private
78
93
 
94
+ def default_report(tasks, specs, on_result)
95
+ Evals::Runner.new(
96
+ scenarios: tasks,
97
+ models: specs,
98
+ criteria: sample_criteria,
99
+ judge: evals_judge,
100
+ available_tools: tool_roster,
101
+ instructions: @evaluation.agent.instructions,
102
+ agent_name: @evaluation.agent.name,
103
+ threshold: PASS_THRESHOLD,
104
+ replay: ->(scenario, spec) { replay(scenario, spec) },
105
+ on_result: on_result
106
+ ).call
107
+ end
108
+
79
109
  # --- selection --------------------------------------------------------
80
110
 
81
111
  def selected_scenarios
@@ -184,7 +214,10 @@ module ActionAgent
184
214
  cost: result.replay.cost,
185
215
  fault: result.fault,
186
216
  recommendation: result.recommendation,
187
- diagnosis: result.diagnosis || {},
217
+ diagnosis: (result.diagnosis || {}).merge(
218
+ "_replay_metadata" => result.replay.metadata,
219
+ "_scenario_snapshot" => result.scenario.to_h.merge(expectations: result.scenario.expectations)
220
+ ),
188
221
  error_message: result.replay.error
189
222
  )
190
223
  end
@@ -195,6 +228,8 @@ module ActionAgent
195
228
  scores["_recommendations"] = report.recommendations
196
229
  scores["_verdict"] = report.verdict if report.comparing?
197
230
  scores["_selection"] = run.selection
231
+ scores["_metadata"] = report.metadata
232
+ scores["_judge_label"] = report.judge_label || report.judge&.label
198
233
  scores
199
234
  end
200
235
 
@@ -206,12 +241,50 @@ module ActionAgent
206
241
  end
207
242
  end
208
243
 
244
+ # Every tool the agent could actually call, for the diagnosis roster: the
245
+ # engine's own toolbox plus whatever its MCP servers serve.
246
+ #
247
+ # Listing only the toolbox understated the roster, so a diagnosis could
248
+ # report "none of the available tools covers this task" while naming a list
249
+ # the agent's MCP tools were missing from.
209
250
  def tool_roster
210
- @tool_roster ||= AgentToolbox.definitions_for(@evaluation.agent.tools).to_h do |definition|
211
- [ definition[:name].to_s, definition[:description].to_s ]
251
+ @tool_roster ||= begin
252
+ definitions = mcp_dispatcher.tool_definitions +
253
+ AgentToolbox.definitions_for(@evaluation.agent.tools)
254
+
255
+ definitions.to_h { |definition| [ definition[:name].to_s, definition[:description].to_s ] }
212
256
  end
213
257
  end
214
258
 
259
+ # Discovery failures from the roster above, keyed by server. Reading them
260
+ # requires tool_definitions to have run, which tool_roster does.
261
+ def mcp_discovery_errors
262
+ tool_roster
263
+ mcp_dispatcher.discovery_errors
264
+ end
265
+
266
+ def mcp_dispatcher
267
+ @mcp_dispatcher ||= MCPToolDispatcher.new(@evaluation.agent)
268
+ end
269
+
270
+ # A run whose every declared MCP server failed discovery scored an agent
271
+ # that had no tools to call. The scores are real but meaningless — the
272
+ # model answered from its own weights — so the run says so rather than
273
+ # leaving the cause in a log line (#425).
274
+ def discovery_warning
275
+ errors = mcp_discovery_errors
276
+ return nil if errors.empty?
277
+
278
+ prefix = if mcp_dispatcher.all_servers_failed?
279
+ "Every MCP server this agent declares failed tool discovery, so it ran with no MCP tools " \
280
+ "and any specifics in its answers are unverified."
281
+ else
282
+ "Some of this agent's MCP servers failed tool discovery, so part of its toolset was unavailable."
283
+ end
284
+
285
+ "#{prefix} #{errors.values.join(' ')}"
286
+ end
287
+
215
288
  # The judge the evaluation's owner has credentials for, wrapped for the
216
289
  # evaluation core; nil when none is configured, in which case scoring
217
290
  # stays on rules and expectations.
data/config/routes.rb CHANGED
@@ -20,6 +20,8 @@ ActionAgent::Engine.routes.draw do
20
20
 
21
21
  # The dashboard's own JSON API, read and written by the React app.
22
22
  namespace :api do
23
+ resource :dashboard_assistant, only: [ :show, :create ], controller: "dashboard_assistant"
24
+
23
25
  # Telemetry ingestion, relative to wherever the engine is mounted:
24
26
  # <mount>/api/traces (e.g. /activeagents/api/traces at the default mount).
25
27
  # Authenticated with a bearer token, not a session.
@@ -36,6 +38,10 @@ ActionAgent::Engine.routes.draw do
36
38
  post :duplicate
37
39
  get :export
38
40
  get :analytics
41
+ # The runner's conversation picker: this agent's persisted contexts,
42
+ # and a fresh one to pin a first message to.
43
+ get :conversations
44
+ post :conversations, action: :create_conversation
39
45
  end
40
46
  collection do
41
47
  get :presets
@@ -111,7 +117,11 @@ ActionAgent::Engine.routes.draw do
111
117
  resource :metrics, only: [ :show ], controller: "metrics"
112
118
 
113
119
  # Conversations (contexts, messages, generations) behind Interactions.
114
- resources :interactions, only: [ :index, :show ]
120
+ # The runner edits a conversation in place — seeds, fixes or drops a
121
+ # turn — so the next run sees exactly the history it should.
122
+ resources :interactions, only: [ :index, :show ] do
123
+ resources :messages, only: [ :create, :update, :destroy ], controller: "interaction_messages"
124
+ end
115
125
 
116
126
  # Agent output evaluations. A scenario suite also manages its scenarios
117
127
  # here, and exposes each run's per-scenario, per-model results.
@@ -0,0 +1,22 @@
1
+ # frozen_string_literal: true
2
+
3
+ module ActionAgent
4
+ # Runs before Rails' request logger, including for requests later rejected by
5
+ # authentication, consent or CSRF. Match the endpoint under any engine mount;
6
+ # leave the host application's unrelated message/history parameters alone.
7
+ class AssistantRequestFilter
8
+ PATH = %r{/api/dashboard_assistant(?:\.[^/]+)?/?\z}
9
+ PARAMETERS = [ /\Amessage\z/i, /\Ahistory\z/i ].freeze
10
+
11
+ def initialize(app)
12
+ @app = app
13
+ end
14
+
15
+ def call(env)
16
+ if PATH.match?(env["PATH_INFO"].to_s)
17
+ env["action_dispatch.parameter_filter"] = Array(env["action_dispatch.parameter_filter"]) + PARAMETERS
18
+ end
19
+ @app.call(env)
20
+ end
21
+ end
22
+ end
@@ -1,5 +1,7 @@
1
1
  # frozen_string_literal: true
2
2
 
3
+ require_relative "assistant_request_filter"
4
+
3
5
  module ActionAgent
4
6
  # Rails engine for the Active Agent dashboard: the agent builder, runs,
5
7
  # conversations, evaluations, traces, metrics, sandboxes and session
@@ -19,6 +21,8 @@ module ActionAgent
19
21
  # rather than relying on the host to register one.
20
22
  INFLECTION_OVERRIDES = {
21
23
  "mcp_catalog" => "MCPCatalog",
24
+ "mcp_client" => "MCPClient",
25
+ "mcp_tool_dispatcher" => "MCPToolDispatcher",
22
26
  "mcp_controller" => "MCPController",
23
27
  "mcp_recording_middleware" => "MCPRecordingMiddleware",
24
28
  "mcp_servers_controller" => "MCPServersController",
@@ -35,6 +39,7 @@ module ActionAgent
35
39
  }.freeze
36
40
 
37
41
  config.action_agent = ActiveSupport::OrderedOptions.new
42
+ config.app_middleware.insert_before Rails::Rack::Logger, ActionAgent::AssistantRequestFilter
38
43
 
39
44
  # Whether a request is a browser asking for a page, as opposed to an API
40
45
  # or MCP client: the routes use it to tell the dashboard's client-side
@@ -1,5 +1,5 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  module ActionAgent
4
- VERSION = "1.3.0"
4
+ VERSION = "1.5.2"
5
5
  end
data/lib/action_agent.rb CHANGED
@@ -236,6 +236,25 @@ module ActionAgent
236
236
  # @return [Boolean]
237
237
  attr_accessor :execution_enabled
238
238
 
239
+ # Whether the "Ask ActiveAgents" assistant is available.
240
+ #
241
+ # The assistant is a tool for developing and CI-ing agents: it sends
242
+ # recorded prompts, outputs and evaluation report excerpts to a model
243
+ # provider, which is the right trade in a development or CI workspace
244
+ # and a decision nobody should inherit by default in production. Left
245
+ # unset it is on in development and test only. Set it to true to run it
246
+ # somewhere else deliberately, or false to remove it everywhere.
247
+ # @return [Boolean, nil]
248
+ attr_accessor :assistant_enabled
249
+
250
+ # Resolves a host application's runner for one scenario evaluation.
251
+ # Return nil for the engine's normal Agent#test_execute path, or a callable
252
+ # accepting evaluation:, owner:, scenarios:, models:, on_result: and
253
+ # returning an ActiveAgent::Evals::Report. The host runs its own agent and
254
+ # judge and yields every result to on_result for dashboard persistence.
255
+ # @return [Proc, nil]
256
+ attr_accessor :scenario_evaluation_adapter_resolver
257
+
239
258
  # Where the dashboard's upgrade CTAs should send people. Unset in a
240
259
  # self-hosted install, where there is nothing to upgrade, and the CTAs
241
260
  # say so instead of linking nowhere.
@@ -311,6 +330,30 @@ module ActionAgent
311
330
  # @return [Array<Hash>]
312
331
  attr_accessor :mcp_catalog
313
332
 
333
+ # Host-declared ActiveAgent::SchemaTools subclasses whose generated tools
334
+ # are offered alongside AgentToolbox's built-ins: each generated tool is
335
+ # individually selectable in the agent editor, dispatched by name at
336
+ # execution, and named in an evaluation's +tools:+ expectation.
337
+ #
338
+ # ActionAgent.configure do |config|
339
+ # config.schema_tools = [TicketTools, TaskTools, MilestoneTools]
340
+ # end
341
+ #
342
+ # Accepts classes or class-name strings; strings are resolved lazily so a
343
+ # host can declare them from an initializer before autoloading has run.
344
+ #
345
+ # Leave it unset and every ActiveAgent::SchemaTools subclass found in
346
+ # {#schema_tools_path} is offered instead — a host adds a tool by adding a
347
+ # file, without naming it twice.
348
+ # @return [Array<Class, String>, nil]
349
+ attr_accessor :schema_tools
350
+
351
+ # Directory scanned for SchemaTools subclasses when {#schema_tools} is
352
+ # unset. Relative to the host's root. Set to nil to disable discovery and
353
+ # require an explicit declaration.
354
+ # @return [String, nil]
355
+ attr_accessor :schema_tools_path
356
+
314
357
  # Value stored in polymorphic *_type columns for dashboard agents
315
358
  # (agent_memories.memorable_type, agent_contexts.contextable_type).
316
359
  # Unset means the class name. A host app whose existing rows were
@@ -332,6 +375,16 @@ module ActionAgent
332
375
  @execution_enabled != false
333
376
  end
334
377
 
378
+ # Returns whether the dashboard assistant is available. Unconfigured, it
379
+ # follows the environment: development and test yes, everywhere else no.
380
+ #
381
+ # @return [Boolean]
382
+ def assistant_enabled?
383
+ return @assistant_enabled == true unless @assistant_enabled.nil?
384
+
385
+ Rails.env.local?
386
+ end
387
+
335
388
  # Tells the host app that +owner+ performed +kind+. Never raises: a
336
389
  # bookkeeping failure must not fail the action that was already taken.
337
390
  def record_usage(owner, kind)
@@ -452,6 +505,9 @@ module ActionAgent
452
505
  @provider_credentials_resolver = nil
453
506
  @sandbox_backends = {}
454
507
  @execution_enabled = true
508
+ @assistant_enabled = nil
509
+
510
+ @scenario_evaluation_adapter_resolver = nil
455
511
  @table_name_prefix = "active_agent_"
456
512
  @agent_polymorphic_name = nil
457
513
  @encrypt_credentials = true
@@ -463,6 +519,63 @@ module ActionAgent
463
519
  @sign_out_path = nil
464
520
  @sign_in_path = nil
465
521
  @mcp_catalog = []
522
+ @schema_tools = nil
523
+ @schema_tools_path = "app/agent_tools"
524
+ end
525
+
526
+ # Host-declared schema tool classes, resolved from names and filtered to
527
+ # those that are usable (declared a model and generated a roster).
528
+ #
529
+ # Resolution happens per call rather than at configure time: a host
530
+ # declares these in an initializer, before its own classes are autoloaded.
531
+ # @return [Array<Class>]
532
+ def schema_tool_classes
533
+ declared = @schema_tools.nil? ? discovered_schema_tools : Array(@schema_tools)
534
+
535
+ declared.filter_map do |entry|
536
+ klass = entry.is_a?(String) ? entry.safe_constantize : entry
537
+ next unless klass.respond_to?(:tool_definitions) && klass.respond_to?(:model)
538
+ next if klass.model.blank?
539
+ # An anonymous class built at runtime is usable but not discoverable —
540
+ # it would accumulate across reloads with no way to supersede itself.
541
+ next if entry.is_a?(Class) && klass.name.blank? && @schema_tools.nil?
542
+
543
+ klass
544
+ end
545
+ end
546
+
547
+ # Every tool name generated by the declared schema tool classes.
548
+ # @return [Array<String>]
549
+ def schema_tool_names
550
+ schema_tool_classes.flat_map(&:tool_names).map(&:to_s)
551
+ end
552
+
553
+ # SchemaTools subclasses defined under {#schema_tools_path}.
554
+ #
555
+ # The files are loaded before reading +descendants+: in development nothing
556
+ # has referenced those constants yet, so the list would otherwise be empty
557
+ # at boot and fill in only once something happened to touch them.
558
+ # @return [Array<Class>]
559
+ def discovered_schema_tools
560
+ return [] if @schema_tools_path.blank? || !defined?(ActiveAgent::SchemaTools)
561
+ return [] unless defined?(Rails) && Rails.respond_to?(:root) && Rails.root
562
+
563
+ root = Rails.root.join(@schema_tools_path)
564
+ return [] unless Dir.exist?(root)
565
+
566
+ Dir[root.join("**/*.rb")].sort.each do |path|
567
+ require_dependency path
568
+ rescue StandardError, ScriptError => e
569
+ warn "[ActionAgent] could not load #{path}: #{e.class} - #{e.message}"
570
+ end
571
+
572
+ ActiveAgent::SchemaTools.descendants.select { |klass| klass.name.present? }
573
+ end
574
+
575
+ # The schema tool class that generated +name+, or nil.
576
+ # @return [Class, nil]
577
+ def schema_tool_class_for(name)
578
+ schema_tool_classes.find { |klass| klass.tool?(name) }
466
579
  end
467
580
  end
468
581
 
@@ -70,6 +70,18 @@ ActionAgent.configure do |config|
70
70
  # nothing rather than everything. An empty dashboard for a signed-in user
71
71
  # means the resolver above returned nil.
72
72
 
73
+ # ==========================================================================
74
+ # Ask ActiveAgents assistant
75
+ # ==========================================================================
76
+ #
77
+ # A tool for developing and CI-ing agents. Answering a question sends
78
+ # recorded prompts, outputs and evaluation report excerpts to a model
79
+ # provider, so the page and its API are on in development and test only.
80
+ # Set it to true to run it in another environment deliberately, or false
81
+ # to remove it everywhere.
82
+ #
83
+ # config.assistant_enabled = true
84
+
73
85
  # ==========================================================================
74
86
  # UI
75
87
  # ==========================================================================
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: actionagent
3
3
  version: !ruby/object:Gem::Version
4
- version: 1.3.0
4
+ version: 1.5.2
5
5
  platform: ruby
6
6
  authors:
7
7
  - Justin Bowen
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2026-09-09 00:00:00.000000000 Z
11
+ date: 2026-09-11 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: activeagent
@@ -129,8 +129,10 @@ files:
129
129
  - app/controllers/action_agent/api/analytics_controller.rb
130
130
  - app/controllers/action_agent/api/api_keys_controller.rb
131
131
  - app/controllers/action_agent/api/base_controller.rb
132
+ - app/controllers/action_agent/api/dashboard_assistant_controller.rb
132
133
  - app/controllers/action_agent/api/evaluations_controller.rb
133
134
  - app/controllers/action_agent/api/instance_tiers_controller.rb
135
+ - app/controllers/action_agent/api/interaction_messages_controller.rb
134
136
  - app/controllers/action_agent/api/interactions_controller.rb
135
137
  - app/controllers/action_agent/api/mcp_controller.rb
136
138
  - app/controllers/action_agent/api/mcp_servers_controller.rb
@@ -193,10 +195,14 @@ files:
193
195
  - app/services/action_agent/agent_registrar.rb
194
196
  - app/services/action_agent/agent_scorecard.rb
195
197
  - app/services/action_agent/agent_toolbox.rb
198
+ - app/services/action_agent/dashboard_assistant_service.rb
199
+ - app/services/action_agent/evaluation_evidence.rb
196
200
  - app/services/action_agent/evaluation_runner_service.rb
197
201
  - app/services/action_agent/evaluation_tool_resolver.rb
198
202
  - app/services/action_agent/mcp_catalog.rb
203
+ - app/services/action_agent/mcp_client.rb
199
204
  - app/services/action_agent/mcp_recording_middleware.rb
205
+ - app/services/action_agent/mcp_tool_dispatcher.rb
200
206
  - app/services/action_agent/mock_sandbox_backend.rb
201
207
  - app/services/action_agent/playwright_mcp_client.rb
202
208
  - app/services/action_agent/sandbox_orchestrator.rb
@@ -212,6 +218,7 @@ files:
212
218
  - app/views/layouts/action_agent/react.html.erb
213
219
  - config/routes.rb
214
220
  - lib/action_agent.rb
221
+ - lib/action_agent/assistant_request_filter.rb
215
222
  - lib/action_agent/compatibility.rb
216
223
  - lib/action_agent/engine.rb
217
224
  - lib/action_agent/version.rb