actionagent 1.5.2 → 1.6.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/app/assets/builds/action_agent.css +1 -1
- data/app/assets/builds/action_agent.js +54 -54
- data/app/controllers/action_agent/api/agent_runs_controller.rb +2 -2
- data/app/controllers/action_agent/api/agents_controller.rb +47 -10
- data/app/controllers/action_agent/api/analytics_controller.rb +1 -1
- data/app/controllers/action_agent/api/base_controller.rb +37 -0
- data/app/controllers/action_agent/api/interactions_controller.rb +3 -3
- data/app/controllers/action_agent/api/mcp_controller.rb +115 -4
- data/app/controllers/action_agent/api/sandboxes_controller.rb +6 -1
- data/app/controllers/action_agent/api/session_recordings_controller.rb +24 -7
- data/app/jobs/action_agent/agent_execution_job.rb +4 -1
- data/app/models/action_agent/agent.rb +88 -7
- data/app/models/action_agent/agent_run.rb +63 -0
- data/app/models/action_agent/agent_version.rb +10 -0
- data/app/models/action_agent/evaluation.rb +11 -0
- data/app/models/action_agent/evaluation_run.rb +4 -0
- data/app/models/action_agent/telemetry_trace.rb +28 -1
- data/app/services/action_agent/agent_execution_service.rb +28 -2
- data/app/services/action_agent/agent_registrar.rb +23 -1
- data/app/services/action_agent/agent_release.rb +61 -0
- data/app/services/action_agent/agent_tool_roster.rb +295 -0
- data/app/services/action_agent/agent_toolbox.rb +12 -3
- data/app/services/action_agent/evaluation_runner_service.rb +11 -9
- data/app/services/action_agent/scenario_evaluation_runner.rb +17 -1
- data/config/routes.rb +6 -0
- data/lib/action_agent/version.rb +1 -1
- data/lib/action_agent.rb +66 -4
- data/lib/generators/action_agent/install_generator.rb +6 -0
- data/lib/generators/action_agent/templates/action_agent.rb.erb +8 -0
- data/lib/generators/action_agent/templates/add_agent_releases.rb.erb +50 -0
- data/lib/tasks/action_agent.rake +33 -0
- metadata +5 -2
|
@@ -3,6 +3,10 @@
|
|
|
3
3
|
module ActionAgent
|
|
4
4
|
class AgentRun < ApplicationRecord
|
|
5
5
|
belongs_to :agent
|
|
6
|
+
# The version of the agent this run executed under — the latest at the
|
|
7
|
+
# time, since a run is against the agent as it is.
|
|
8
|
+
belongs_to :agent_version, optional: true
|
|
9
|
+
before_create { self.agent_version_id ||= agent&.latest_version&.id }
|
|
6
10
|
|
|
7
11
|
# Raised when a caller hands a run files to attach in a host app that
|
|
8
12
|
# has nowhere to keep them.
|
|
@@ -20,6 +24,55 @@ module ActionAgent
|
|
|
20
24
|
# --skip-active-storage has no has_many_attached to call.
|
|
21
25
|
has_many_attached :attachments if defined?(ActiveStorage)
|
|
22
26
|
|
|
27
|
+
# The key the caller's identity is recorded under in +input_params+.
|
|
28
|
+
# Underscored so it cannot collide with a provider override, and
|
|
29
|
+
# stripped from anything a client sends (see Api::AgentsController).
|
|
30
|
+
ACTOR_PARAM = "_actor_gid"
|
|
31
|
+
|
|
32
|
+
# +input_params+ with the caller recorded alongside them.
|
|
33
|
+
#
|
|
34
|
+
# The actor is stored as a Global ID rather than as the record, so the
|
|
35
|
+
# worker that picks the run up — on another machine, minutes later —
|
|
36
|
+
# authorizes as the same person who asked for the run. A caller the host
|
|
37
|
+
# cannot address that way (a plain object, a service account) is simply
|
|
38
|
+
# not recorded: the run then executes unattributed, which a host scope
|
|
39
|
+
# reads as "no access", rather than executing as somebody else.
|
|
40
|
+
#
|
|
41
|
+
# @param params [Hash] the run's own parameters
|
|
42
|
+
# @param actor [Object, nil] the caller
|
|
43
|
+
# @return [Hash]
|
|
44
|
+
def self.params_with_actor(params, actor)
|
|
45
|
+
params = (params || {}).to_h.except(ACTOR_PARAM, ACTOR_PARAM.to_sym)
|
|
46
|
+
gid = actor.respond_to?(:to_global_id) ? actor.to_global_id.to_s : nil
|
|
47
|
+
gid ? params.merge(ACTOR_PARAM => gid) : params
|
|
48
|
+
rescue StandardError => e
|
|
49
|
+
Rails.logger.warn("[AgentRun] could not record the run's actor: #{e.class} - #{e.message}")
|
|
50
|
+
params
|
|
51
|
+
end
|
|
52
|
+
|
|
53
|
+
# The caller this run executes on behalf of.
|
|
54
|
+
#
|
|
55
|
+
# Set in memory for a synchronous run; rehydrated from the stored Global
|
|
56
|
+
# ID for one picked up by a worker. A Global ID that no longer resolves
|
|
57
|
+
# (the user was deleted) yields nil, so the run loses access rather than
|
|
58
|
+
# inheriting someone else's.
|
|
59
|
+
# @return [Object, nil]
|
|
60
|
+
def actor
|
|
61
|
+
return @actor if defined?(@actor)
|
|
62
|
+
|
|
63
|
+
@actor = locate_actor
|
|
64
|
+
end
|
|
65
|
+
|
|
66
|
+
attr_writer :actor
|
|
67
|
+
|
|
68
|
+
# Whether this run knows who it is for. A run with a recorded actor that
|
|
69
|
+
# no longer resolves is *not* unattributed — it is broken, and callers
|
|
70
|
+
# that care can tell the two apart.
|
|
71
|
+
# @return [Boolean]
|
|
72
|
+
def actor_recorded?
|
|
73
|
+
input_params.is_a?(Hash) && input_params[ACTOR_PARAM].present?
|
|
74
|
+
end
|
|
75
|
+
|
|
23
76
|
# Whether runs can carry files in this host app: Active Storage loaded,
|
|
24
77
|
# the macro applied, and its tables migrated. Never raises — a host
|
|
25
78
|
# that skipped `rails active_storage:install` still runs agents, it
|
|
@@ -220,6 +273,16 @@ module ActionAgent
|
|
|
220
273
|
|
|
221
274
|
private
|
|
222
275
|
|
|
276
|
+
def locate_actor
|
|
277
|
+
return nil unless actor_recorded?
|
|
278
|
+
return nil unless defined?(GlobalID::Locator)
|
|
279
|
+
|
|
280
|
+
GlobalID::Locator.locate(input_params[ACTOR_PARAM])
|
|
281
|
+
rescue StandardError => e
|
|
282
|
+
Rails.logger.warn("[AgentRun] could not resolve the run's actor: #{e.class} - #{e.message}")
|
|
283
|
+
nil
|
|
284
|
+
end
|
|
285
|
+
|
|
223
286
|
def set_trace_id
|
|
224
287
|
self.trace_id ||= SecureRandom.uuid
|
|
225
288
|
end
|
|
@@ -9,6 +9,9 @@ module ActionAgent
|
|
|
9
9
|
|
|
10
10
|
# Scopes
|
|
11
11
|
scope :recent, -> { order(version_number: :desc) }
|
|
12
|
+
# Versions cut from the agent's code on deploy, as opposed to edits made
|
|
13
|
+
# in the dashboard.
|
|
14
|
+
scope :releases, -> { where.not(release_digest: [ nil, "" ]) }
|
|
12
15
|
scope :by_version, ->(num) { where(version_number: num) }
|
|
13
16
|
|
|
14
17
|
# Compare two versions
|
|
@@ -36,6 +39,13 @@ module ActionAgent
|
|
|
36
39
|
end
|
|
37
40
|
|
|
38
41
|
# Check if this is the latest version
|
|
42
|
+
# Whether this version was cut from the agent's code (it carries the
|
|
43
|
+
# release digest) rather than from a dashboard edit.
|
|
44
|
+
# @return [Boolean]
|
|
45
|
+
def release?
|
|
46
|
+
release_digest.present?
|
|
47
|
+
end
|
|
48
|
+
|
|
39
49
|
def latest?
|
|
40
50
|
agent.latest_version&.id == id
|
|
41
51
|
end
|
|
@@ -35,6 +35,17 @@ module ActionAgent
|
|
|
35
35
|
|
|
36
36
|
scope :recent, -> { order(updated_at: :desc) }
|
|
37
37
|
|
|
38
|
+
# MySQL cannot give a JSON column a default, so a row inserted there
|
|
39
|
+
# without `criteria` or `config` reads back nil. Both readers answer with
|
|
40
|
+
# the empty value the column default supplies on other databases.
|
|
41
|
+
def criteria
|
|
42
|
+
super || []
|
|
43
|
+
end
|
|
44
|
+
|
|
45
|
+
def config
|
|
46
|
+
super || {}
|
|
47
|
+
end
|
|
48
|
+
|
|
38
49
|
def latest_run
|
|
39
50
|
evaluation_runs.order(created_at: :desc).first
|
|
40
51
|
end
|
|
@@ -8,6 +8,10 @@ module ActionAgent
|
|
|
8
8
|
# See #average_score, which is what has to tolerate both shapes.
|
|
9
9
|
class EvaluationRun < ApplicationRecord
|
|
10
10
|
belongs_to :evaluation
|
|
11
|
+
# The version of the evaluated agent this run scored, so a pass rate is
|
|
12
|
+
# a statement about a release rather than about "the agent".
|
|
13
|
+
belongs_to :agent_version, optional: true
|
|
14
|
+
before_create { self.agent_version_id ||= evaluation&.agent&.latest_version&.id }
|
|
11
15
|
has_many :scenario_results, class_name: "EvaluationScenarioResult", dependent: :destroy
|
|
12
16
|
|
|
13
17
|
enum :status, { pending: 0, running: 1, complete: 2, failed: 3 }
|
|
@@ -43,6 +43,8 @@ module ActionAgent
|
|
|
43
43
|
scope :for_date_range, ->(start_date, end_date) { where(timestamp: start_date..end_date) }
|
|
44
44
|
# The dashboard agent this trace was attributed to on ingest, if any.
|
|
45
45
|
belongs_to :agent, class_name: "ActionAgent::Agent", optional: true
|
|
46
|
+
# The release of the agent this trace came from (see #attach_agent_version!).
|
|
47
|
+
belongs_to :agent_version, class_name: "ActionAgent::AgentVersion", optional: true
|
|
46
48
|
|
|
47
49
|
scope :for_account, ->(account) { where(account: account) if ActionAgent.multi_tenant? }
|
|
48
50
|
|
|
@@ -252,7 +254,10 @@ module ActionAgent
|
|
|
252
254
|
# dashboard-authored agent by guessing a primary key.
|
|
253
255
|
attrs[:agent_id] = agent&.id
|
|
254
256
|
|
|
255
|
-
create!(attrs).tap
|
|
257
|
+
create!(attrs).tap do |record|
|
|
258
|
+
AgentRegistrar.call(record)
|
|
259
|
+
record.attach_agent_version!
|
|
260
|
+
end
|
|
256
261
|
end
|
|
257
262
|
|
|
258
263
|
|
|
@@ -265,6 +270,28 @@ module ActionAgent
|
|
|
265
270
|
tokens.fetch("input", 0).to_i + tokens.fetch("output", 0).to_i + tokens.fetch("thinking", 0).to_i
|
|
266
271
|
end
|
|
267
272
|
|
|
273
|
+
# Pins this trace to the version of its agent that produced it. The
|
|
274
|
+
# instrumentation stamps the root span with `agent.version` — the digest
|
|
275
|
+
# ActiveAgent::Release computes from the class — and a release cut on
|
|
276
|
+
# deploy carries the same digest, so the two meet here. A trace from a
|
|
277
|
+
# dashboard run carries no digest and takes the agent's latest version.
|
|
278
|
+
#
|
|
279
|
+
# @return [AgentVersion, nil]
|
|
280
|
+
def attach_agent_version!
|
|
281
|
+
return if agent_version_id.present? || agent_id.blank?
|
|
282
|
+
|
|
283
|
+
digest = root_span&.dig("attributes", "agent.version").presence
|
|
284
|
+
version = if digest
|
|
285
|
+
AgentVersion.find_by(agent_id: agent_id, release_digest: digest)
|
|
286
|
+
else
|
|
287
|
+
AgentVersion.where(agent_id: agent_id).order(version_number: :desc).first
|
|
288
|
+
end
|
|
289
|
+
return unless version
|
|
290
|
+
|
|
291
|
+
update_columns(agent_version_id: version.id)
|
|
292
|
+
version
|
|
293
|
+
end
|
|
294
|
+
|
|
268
295
|
# Returns the root span of this trace.
|
|
269
296
|
#
|
|
270
297
|
# @return [Hash, nil] The root span or nil
|
|
@@ -45,6 +45,13 @@ module ActionAgent
|
|
|
45
45
|
new(agent_record, run).call
|
|
46
46
|
end
|
|
47
47
|
|
|
48
|
+
# Tool-call keywords that name the caller. The model's arguments and the
|
|
49
|
+
# run's actor share one keyword namespace by the time they reach a tool,
|
|
50
|
+
# so anything a model emits under these names is dropped before the call:
|
|
51
|
+
# an actor a model can name is not an authorization boundary, and the
|
|
52
|
+
# documents a model reads are attacker-reachable.
|
|
53
|
+
ACTOR_KEYWORDS = %i[actor current_user].freeze
|
|
54
|
+
|
|
48
55
|
def initialize(agent_record, run)
|
|
49
56
|
@agent_record = agent_record
|
|
50
57
|
@run = run
|
|
@@ -52,6 +59,14 @@ module ActionAgent
|
|
|
52
59
|
@event_sequence = 0
|
|
53
60
|
end
|
|
54
61
|
|
|
62
|
+
# The caller this run executes on behalf of, or nil when it runs
|
|
63
|
+
# unattributed. Passed to every tool as +actor:+ — a host's SchemaTools
|
|
64
|
+
# scope block, Pundit policy or agent callback decides what that means.
|
|
65
|
+
# @return [Object, nil]
|
|
66
|
+
def actor
|
|
67
|
+
@run.actor
|
|
68
|
+
end
|
|
69
|
+
|
|
55
70
|
# Emits a progress event on the run (streamed to the UI by pollers).
|
|
56
71
|
# Never lets telemetry break execution.
|
|
57
72
|
def emit_event(**kwargs)
|
|
@@ -266,6 +281,12 @@ module ActionAgent
|
|
|
266
281
|
# execution) and recorded in @tool_invocations so tool names, arguments
|
|
267
282
|
# and durations reach Traces and the persisted conversation.
|
|
268
283
|
def execute_tool(name, **kwargs)
|
|
284
|
+
forged = kwargs.slice(*ACTOR_KEYWORDS)
|
|
285
|
+
if forged.any?
|
|
286
|
+
Rails.logger.warn("[AgentExecutionService] dropped caller-named arguments from #{name}: #{forged.keys.join(', ')}")
|
|
287
|
+
kwargs = kwargs.except(*ACTOR_KEYWORDS)
|
|
288
|
+
end
|
|
289
|
+
|
|
269
290
|
# Record the absolute URL browse_page will actually fetch, not the bare
|
|
270
291
|
# path the model passed — spans/events/persisted args stay unambiguous.
|
|
271
292
|
kwargs[:url] = AgentToolbox.resolve_browse_url(kwargs[:url]) if name.to_s == "browse_page" && kwargs[:url]
|
|
@@ -309,7 +330,9 @@ module ActionAgent
|
|
|
309
330
|
else
|
|
310
331
|
# A tool one of the agent's own MCP servers serves is called there;
|
|
311
332
|
# AgentToolbox answers the rest.
|
|
312
|
-
|
|
333
|
+
# `actor:` comes from the run, never from kwargs (see
|
|
334
|
+
# ACTOR_KEYWORDS): it is who the run is for, not what it is about.
|
|
335
|
+
mcp_dispatcher.call(name, kwargs) || AgentToolbox.call(name, actor: actor, **kwargs)
|
|
313
336
|
end
|
|
314
337
|
rescue StandardError => e
|
|
315
338
|
Rails.logger.warn("[AgentExecutionService] Tool #{name} failed: #{e.class} - #{e.message}")
|
|
@@ -529,7 +552,10 @@ module ActionAgent
|
|
|
529
552
|
private :persist_tool_messages_to_context
|
|
530
553
|
end
|
|
531
554
|
|
|
532
|
-
|
|
555
|
+
# `as` carries the caller onto the agent instance, so an agent's own
|
|
556
|
+
# before_action callbacks (ActiveAgent::Authorization) authorize
|
|
557
|
+
# against the same person the tools are scoped to.
|
|
558
|
+
agent_class.as(actor).public_send(action).generate_now
|
|
533
559
|
end
|
|
534
560
|
|
|
535
561
|
# Function-calling schemas for the agent's enabled tools that have
|
|
@@ -77,8 +77,22 @@ module ActionAgent
|
|
|
77
77
|
ActionAgent.multi_tenant? ? @trace.try(:account) : nil
|
|
78
78
|
end
|
|
79
79
|
|
|
80
|
+
# The set an observed agent is deduplicated within.
|
|
81
|
+
#
|
|
82
|
+
# `for_owner(nil)` is `none` — correct for a multi-tenant read, where an
|
|
83
|
+
# unresolved tenant must see nothing. But a single-tenant dashboard has no
|
|
84
|
+
# owner to resolve and legitimately registers with `owner` nil (see
|
|
85
|
+
# #owner_for_trace), and `none` makes every dedupe lookup miss: each trace
|
|
86
|
+
# created another copy of the same agent, and the MAX_OBSERVED_PER_OWNER
|
|
87
|
+
# cap never engaged because the count it reads was always zero. Scoping to
|
|
88
|
+
# the whole table when there is no owner concept is what `for_owner`
|
|
89
|
+
# already does for a model with no owner association.
|
|
90
|
+
def agents_for_owner(owner)
|
|
91
|
+
owner.nil? && !ActionAgent.multi_tenant? ? Agent.all : Agent.for_owner(owner)
|
|
92
|
+
end
|
|
93
|
+
|
|
80
94
|
def find_or_create_agent(owner)
|
|
81
|
-
agents =
|
|
95
|
+
agents = agents_for_owner(owner)
|
|
82
96
|
existing = agents.find_by(
|
|
83
97
|
service_name: @trace.service_name,
|
|
84
98
|
agent_class_name: agent_class,
|
|
@@ -86,6 +100,14 @@ module ActionAgent
|
|
|
86
100
|
)
|
|
87
101
|
return existing if existing
|
|
88
102
|
|
|
103
|
+
# A host that mirrors its agent classes into the dashboard (a sync, a
|
|
104
|
+
# release) names the class on the record and nothing else: that record
|
|
105
|
+
# stands for every action of the class, so a trace from the class is
|
|
106
|
+
# its trace — not an observed twin's. Observed records are per action
|
|
107
|
+
# and are only ever matched on all three keys above.
|
|
108
|
+
mirrored = agents.where.not(status: "observed").find_by(agent_class_name: agent_class)
|
|
109
|
+
return mirrored if mirrored
|
|
110
|
+
|
|
89
111
|
return if agents.observed_agents.count >= MAX_OBSERVED_PER_OWNER
|
|
90
112
|
|
|
91
113
|
create_observed_agent(owner)
|
|
@@ -0,0 +1,61 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module ActionAgent
|
|
4
|
+
# Cuts a version of every dashboard agent that mirrors a host class, from
|
|
5
|
+
# that class's release digest — what a deploy runs so the dashboard's
|
|
6
|
+
# versions line up with the code that shipped.
|
|
7
|
+
#
|
|
8
|
+
# The host owns the mirror: whatever syncs its ActiveAgent classes into
|
|
9
|
+
# Agent records sets `agent_class_name`, and this reads it back. A record
|
|
10
|
+
# whose class no longer resolves, or whose class predates
|
|
11
|
+
# ActiveAgent::Release, is reported and skipped rather than failed.
|
|
12
|
+
#
|
|
13
|
+
# Idempotent: a redeploy of an unchanged agent cuts nothing, so running it
|
|
14
|
+
# on every deploy is the intended use (`rake action_agent:agents:release`).
|
|
15
|
+
class AgentRelease
|
|
16
|
+
Row = Struct.new(:agent, :version, :cut, :skipped, keyword_init: true)
|
|
17
|
+
Result = Struct.new(:rows, :revision, keyword_init: true) do
|
|
18
|
+
def cut = rows.select(&:cut)
|
|
19
|
+
def skipped = rows.select(&:skipped)
|
|
20
|
+
end
|
|
21
|
+
|
|
22
|
+
# @param revision [String, nil] the deploy (git SHA, release label);
|
|
23
|
+
# ActiveAgent::Release.revision when nil
|
|
24
|
+
# @param agents [ActiveRecord::Relation] the records to release; every
|
|
25
|
+
# record naming a host class by default
|
|
26
|
+
# @param released_by [String, nil] recorded on each version cut
|
|
27
|
+
def self.call(revision: nil, agents: nil, released_by: nil)
|
|
28
|
+
new(revision: revision, agents: agents, released_by: released_by).call
|
|
29
|
+
end
|
|
30
|
+
|
|
31
|
+
def initialize(revision: nil, agents: nil, released_by: nil)
|
|
32
|
+
@revision = revision.presence || ActiveAgent::Release.revision
|
|
33
|
+
@agents = agents || Agent.where.not(agent_class_name: [ nil, "" ])
|
|
34
|
+
@released_by = released_by
|
|
35
|
+
end
|
|
36
|
+
|
|
37
|
+
# @return [Result]
|
|
38
|
+
def call
|
|
39
|
+
rows = @agents.order(:name).map { |agent| release(agent) }
|
|
40
|
+
Result.new(rows: rows, revision: @revision)
|
|
41
|
+
end
|
|
42
|
+
|
|
43
|
+
private
|
|
44
|
+
|
|
45
|
+
def release(agent)
|
|
46
|
+
klass = agent.agent_class_name.to_s.safe_constantize
|
|
47
|
+
unless klass.respond_to?(:release_digest)
|
|
48
|
+
return Row.new(agent: agent, version: nil, cut: false, skipped: "#{agent.agent_class_name} does not resolve to a releasable class")
|
|
49
|
+
end
|
|
50
|
+
|
|
51
|
+
before = agent.latest_version&.id
|
|
52
|
+
version = agent.record_release!(
|
|
53
|
+
digest: klass.release_digest,
|
|
54
|
+
manifest: klass.release_manifest,
|
|
55
|
+
revision: @revision,
|
|
56
|
+
released_by: @released_by
|
|
57
|
+
)
|
|
58
|
+
Row.new(agent: agent, version: version, cut: version.id != before, skipped: nil)
|
|
59
|
+
end
|
|
60
|
+
end
|
|
61
|
+
end
|
|
@@ -0,0 +1,295 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module ActionAgent
|
|
4
|
+
# One agent's tool roster, as the agent editor's Tools tab reads it: every
|
|
5
|
+
# tool the agent can be offered and every MCP service it can be given, each
|
|
6
|
+
# carrying what the window actually recorded for it.
|
|
7
|
+
#
|
|
8
|
+
# Three groups, one row vocabulary — the same one the Tools and MCP
|
|
9
|
+
# Services pages use, so a roster reads the way the observability views do:
|
|
10
|
+
#
|
|
11
|
+
# * **Agent-defined** — the tools this agent's own generations offered
|
|
12
|
+
# (ToolDiscovery's +agent+ origin), plus every schema tool the host
|
|
13
|
+
# declares, on or off. Two kinds share the group. A *schema tool* — one
|
|
14
|
+
# a host ActiveAgent::SchemaTools class generates — is offered only while
|
|
15
|
+
# +agent.tools+ names it: that is the reading AgentToolbox takes at
|
|
16
|
+
# generation time, and the evaluation runner, dashboard runs and the MCP
|
|
17
|
+
# facade all follow it. So it is selected here, and switchable, and every
|
|
18
|
+
# one has a row because any agent may enable any of them
|
|
19
|
+
# (Agent.available_tools) and a tool switched off has to keep its row to
|
|
20
|
+
# be switched back on. A tool the agent class declares *in code* is
|
|
21
|
+
# offered by the class itself; the dashboard reports it and cannot switch
|
|
22
|
+
# it, so its row is read-only — a checkbox that cannot add or remove the
|
|
23
|
+
# tool is a control that changes nothing.
|
|
24
|
+
# * **Dashboard** — Agent::AVAILABLE_TOOLS, the capabilities the builder
|
|
25
|
+
# offers every agent. These are the roster: +agent.tools+ is what
|
|
26
|
+
# AgentToolbox turns into function schemas at generation time.
|
|
27
|
+
# * **MCP** — never stored on the roster. Computed from the services the
|
|
28
|
+
# agent enables, which is where they are edited.
|
|
29
|
+
#
|
|
30
|
+
# Enablement reads the agent's own configuration: +tools+ for the dashboard
|
|
31
|
+
# capabilities and the schema tools, +mcp_servers+ for services and their
|
|
32
|
+
# per-server allow-lists (an entry with no +tools+ key offers everything
|
|
33
|
+
# the server serves).
|
|
34
|
+
class AgentToolRoster
|
|
35
|
+
AGENT_DEFINED = "agent_defined"
|
|
36
|
+
DASHBOARD = "dashboard"
|
|
37
|
+
MCP = "mcp"
|
|
38
|
+
|
|
39
|
+
# Ordering for the services list: what this agent uses, then what the
|
|
40
|
+
# workspace already talks to, then the rest of the catalog.
|
|
41
|
+
STATUS_RANK = { "active" => 0, "configured" => 1, "available" => 2, "idle" => 3 }.freeze
|
|
42
|
+
|
|
43
|
+
attr_reader :agent, :discovery
|
|
44
|
+
|
|
45
|
+
# @param agent [ActionAgent::Agent] the agent being edited
|
|
46
|
+
# @param traces [ActiveRecord::Relation] the traces the caller may read
|
|
47
|
+
# @param hours [Integer] the window the usage columns are scoped to
|
|
48
|
+
def initialize(agent:, traces:, hours: ToolDiscovery::DEFAULT_WINDOW_HOURS)
|
|
49
|
+
@agent = agent
|
|
50
|
+
@discovery = ToolDiscovery.new(
|
|
51
|
+
traces: traces.for_agent(agent.telemetry_agent_class),
|
|
52
|
+
agents: Agent.where(id: agent.id),
|
|
53
|
+
hours: hours
|
|
54
|
+
)
|
|
55
|
+
end
|
|
56
|
+
|
|
57
|
+
def as_json(*)
|
|
58
|
+
{
|
|
59
|
+
window_hours: discovery.window_hours,
|
|
60
|
+
# Whether any record source had rows in this window. False means the
|
|
61
|
+
# usage columns have nothing behind them, and the view renders them
|
|
62
|
+
# as "—" rather than as a row of honest-looking zeroes.
|
|
63
|
+
usage_available: inventory[:sources].values.any?,
|
|
64
|
+
services: services,
|
|
65
|
+
tools: tools
|
|
66
|
+
}
|
|
67
|
+
end
|
|
68
|
+
|
|
69
|
+
private
|
|
70
|
+
|
|
71
|
+
def inventory
|
|
72
|
+
@inventory ||= discovery.inventory
|
|
73
|
+
end
|
|
74
|
+
|
|
75
|
+
def detected
|
|
76
|
+
inventory[:tools]
|
|
77
|
+
end
|
|
78
|
+
|
|
79
|
+
def saved_tools
|
|
80
|
+
@saved_tools ||= Array(agent.tools).map(&:to_s)
|
|
81
|
+
end
|
|
82
|
+
|
|
83
|
+
# --- services ------------------------------------------------------
|
|
84
|
+
|
|
85
|
+
def services
|
|
86
|
+
rows = service_keys.map { |key| service_row(key) }
|
|
87
|
+
rows.sort_by do |row|
|
|
88
|
+
[ row[:enabled] ? 0 : 1, STATUS_RANK.fetch(row[:status], 9), -row[:calls], row[:name].to_s.downcase ]
|
|
89
|
+
end
|
|
90
|
+
end
|
|
91
|
+
|
|
92
|
+
# The catalog, plus anything this agent's traffic or configuration names
|
|
93
|
+
# that the catalog doesn't describe.
|
|
94
|
+
def service_keys
|
|
95
|
+
(MCPCatalog.keys + detected_by_server.keys + configured_servers.keys).uniq
|
|
96
|
+
end
|
|
97
|
+
|
|
98
|
+
def detected_by_server
|
|
99
|
+
@detected_by_server ||= detected.reject { |tool| tool[:mcp_server].blank? }.group_by { |tool| tool[:mcp_server] }
|
|
100
|
+
end
|
|
101
|
+
|
|
102
|
+
def service_row(key)
|
|
103
|
+
catalog = MCPCatalog.find(key)
|
|
104
|
+
used = detected_by_server[key] || []
|
|
105
|
+
calls = used.sum { |tool| tool[:calls] }
|
|
106
|
+
enabled = configured_servers.key?(key)
|
|
107
|
+
|
|
108
|
+
{
|
|
109
|
+
key: key,
|
|
110
|
+
name: catalog ? catalog[:name] : key,
|
|
111
|
+
description: catalog&.dig(:description),
|
|
112
|
+
docs_url: catalog&.dig(:docs_url),
|
|
113
|
+
first_party: catalog ? catalog[:first_party] : false,
|
|
114
|
+
known: !catalog.nil?,
|
|
115
|
+
transport: transport_label(catalog),
|
|
116
|
+
status: service_status(calls, enabled, catalog),
|
|
117
|
+
enabled: enabled,
|
|
118
|
+
calls: calls,
|
|
119
|
+
errors: used.sum { |tool| tool[:errors] },
|
|
120
|
+
last_seen: used.filter_map { |tool| tool[:last_seen] }.max,
|
|
121
|
+
tools: service_tools(key, catalog, used)
|
|
122
|
+
}
|
|
123
|
+
end
|
|
124
|
+
|
|
125
|
+
# How to reach the server, in the one line the expanded panel shows:
|
|
126
|
+
# "Streamable HTTP · <url>" for the ones the dashboard can call,
|
|
127
|
+
# "sandbox · <command>" for the ones it can start, "stdio · <command>"
|
|
128
|
+
# for the rest.
|
|
129
|
+
def transport_label(catalog)
|
|
130
|
+
return nil if catalog.nil?
|
|
131
|
+
|
|
132
|
+
transport = catalog[:transport].to_s
|
|
133
|
+
return [ "Streamable HTTP", catalog[:url] ].compact.join(" · ") if transport == "http"
|
|
134
|
+
|
|
135
|
+
prefix = catalog[:sandbox] ? "sandbox" : transport.presence
|
|
136
|
+
[ prefix, catalog[:command] ].compact.join(" · ").presence
|
|
137
|
+
end
|
|
138
|
+
|
|
139
|
+
def service_status(calls, enabled, catalog)
|
|
140
|
+
return "active" if calls.positive?
|
|
141
|
+
return "configured" if enabled
|
|
142
|
+
return "available" if catalog
|
|
143
|
+
|
|
144
|
+
"idle"
|
|
145
|
+
end
|
|
146
|
+
|
|
147
|
+
# What the service offers: the catalog's tool hints unioned with the
|
|
148
|
+
# tools this agent was actually seen calling on it, so a server whose
|
|
149
|
+
# roster has drifted from the catalog still lists what it really serves.
|
|
150
|
+
def service_tools(key, catalog, used)
|
|
151
|
+
by_name = used.index_by { |tool| tool[:base_name] }
|
|
152
|
+
allowed = allowed_tools(key)
|
|
153
|
+
names = (Array(catalog&.dig(:tools)) + by_name.keys).uniq
|
|
154
|
+
|
|
155
|
+
names.map do |name|
|
|
156
|
+
usage_row(by_name[name]).merge(
|
|
157
|
+
name: name,
|
|
158
|
+
description: by_name[name]&.dig(:description),
|
|
159
|
+
enabled: allowed.nil? || allowed.include?(name)
|
|
160
|
+
)
|
|
161
|
+
end
|
|
162
|
+
end
|
|
163
|
+
|
|
164
|
+
# --- tools ---------------------------------------------------------
|
|
165
|
+
|
|
166
|
+
def tools
|
|
167
|
+
agent_defined_rows + dashboard_rows
|
|
168
|
+
end
|
|
169
|
+
|
|
170
|
+
# Agent-defined tools: whatever this agent's generations offered that is
|
|
171
|
+
# neither an MCP tool nor one of the dashboard's own, plus every schema
|
|
172
|
+
# tool the host declares, whether or not the window saw it.
|
|
173
|
+
def agent_defined_rows
|
|
174
|
+
observed = detected
|
|
175
|
+
.select { |tool| tool[:origin] == ToolDiscovery::ORIGIN_AGENT }
|
|
176
|
+
.reject { |tool| dashboard_function_names.include?(tool[:name]) }
|
|
177
|
+
.index_by { |tool| tool[:name] }
|
|
178
|
+
|
|
179
|
+
(observed.keys + ActionAgent.schema_tool_names).uniq.map do |name|
|
|
180
|
+
tool = observed[name]
|
|
181
|
+
schema = schema_tool?(name)
|
|
182
|
+
|
|
183
|
+
usage_row(tool).merge(
|
|
184
|
+
key: name,
|
|
185
|
+
name: name,
|
|
186
|
+
source: AGENT_DEFINED,
|
|
187
|
+
description: tool&.dig(:description) || schema_tool_description(name),
|
|
188
|
+
# A schema tool is offered only while the roster names it — the
|
|
189
|
+
# window may still show it being called, but that is history. A
|
|
190
|
+
# tool the agent class declares in code is offered by the class.
|
|
191
|
+
enabled: schema ? saved_tools.include?(name) : true,
|
|
192
|
+
editable: schema
|
|
193
|
+
)
|
|
194
|
+
end
|
|
195
|
+
end
|
|
196
|
+
|
|
197
|
+
def schema_tool?(name)
|
|
198
|
+
@schema_tool ||= Hash.new { |cache, key| cache[key] = ActionAgent.schema_tool_class_for(key).present? }
|
|
199
|
+
@schema_tool[name]
|
|
200
|
+
end
|
|
201
|
+
|
|
202
|
+
def schema_tool_description(name)
|
|
203
|
+
AgentToolbox.schema_tool_definitions(name).first&.dig(:description)
|
|
204
|
+
end
|
|
205
|
+
|
|
206
|
+
def dashboard_rows
|
|
207
|
+
Agent::AVAILABLE_TOOLS.map do |capability|
|
|
208
|
+
usage_row(capability_usage(capability)).merge(
|
|
209
|
+
key: capability,
|
|
210
|
+
name: capability,
|
|
211
|
+
source: DASHBOARD,
|
|
212
|
+
description: Agent::TOOL_DESCRIPTIONS[capability],
|
|
213
|
+
enabled: saved_tools.include?(capability),
|
|
214
|
+
editable: true
|
|
215
|
+
)
|
|
216
|
+
end
|
|
217
|
+
end
|
|
218
|
+
|
|
219
|
+
# A capability is one checkbox over the several functions it exposes
|
|
220
|
+
# ("memory" is save_memory + recall_memory), so its usage is their sum.
|
|
221
|
+
def capability_usage(capability)
|
|
222
|
+
names = (AgentToolbox::DEFINITIONS[capability]&.map { |definition| definition[:name].to_s } || []) + [ capability ]
|
|
223
|
+
rows = detected.select { |tool| names.include?(tool[:name]) }
|
|
224
|
+
return nil if rows.empty?
|
|
225
|
+
|
|
226
|
+
timed = rows.filter_map { |row| [ row[:avg_duration_ms], row[:calls] ] if row[:avg_duration_ms] }
|
|
227
|
+
weighted = timed.sum { |average, calls| average * [ calls, 1 ].max }
|
|
228
|
+
samples = timed.sum { |_average, calls| [ calls, 1 ].max }
|
|
229
|
+
|
|
230
|
+
{
|
|
231
|
+
calls: rows.sum { |row| row[:calls] },
|
|
232
|
+
errors: rows.sum { |row| row[:errors] },
|
|
233
|
+
avg_duration_ms: samples.positive? ? (weighted / samples).round : nil,
|
|
234
|
+
last_seen: rows.filter_map { |row| row[:last_seen] }.max
|
|
235
|
+
}
|
|
236
|
+
end
|
|
237
|
+
|
|
238
|
+
def usage_row(tool)
|
|
239
|
+
{
|
|
240
|
+
calls: tool ? tool[:calls] : 0,
|
|
241
|
+
errors: tool ? tool[:errors] : 0,
|
|
242
|
+
avg_duration_ms: tool ? tool[:avg_duration_ms] : nil,
|
|
243
|
+
last_seen: tool ? tool[:last_seen] : nil
|
|
244
|
+
}
|
|
245
|
+
end
|
|
246
|
+
|
|
247
|
+
# Every function name the dashboard's own toolbox implements, so a
|
|
248
|
+
# builtin never lands in the agent-defined group under its bare name.
|
|
249
|
+
def dashboard_function_names
|
|
250
|
+
@dashboard_function_names ||= ToolDiscovery.builtin_tools | Agent::AVAILABLE_TOOLS.to_set
|
|
251
|
+
end
|
|
252
|
+
|
|
253
|
+
# --- the agent's MCP configuration ---------------------------------
|
|
254
|
+
|
|
255
|
+
# server key => the entry the agent stores for it. Entries are bare
|
|
256
|
+
# strings or builder hashes, and an agent seeded from an older template
|
|
257
|
+
# carries a top-level Hash keyed by server name — the same three shapes
|
|
258
|
+
# EvaluationToolResolver tolerates.
|
|
259
|
+
def configured_servers
|
|
260
|
+
@configured_servers ||= configured_entries.each_with_object({}) do |entry, map|
|
|
261
|
+
key = entry_key(entry)
|
|
262
|
+
map[key] = entry if key.present?
|
|
263
|
+
end
|
|
264
|
+
end
|
|
265
|
+
|
|
266
|
+
def configured_entries
|
|
267
|
+
servers = agent.mcp_servers
|
|
268
|
+
|
|
269
|
+
if servers.is_a?(Hash)
|
|
270
|
+
servers.map { |key, value| value.respond_to?(:key?) ? value.to_h.stringify_keys.merge("key" => key.to_s) : key.to_s }
|
|
271
|
+
else
|
|
272
|
+
Array(servers)
|
|
273
|
+
end
|
|
274
|
+
end
|
|
275
|
+
|
|
276
|
+
def entry_key(entry)
|
|
277
|
+
return entry.to_s.strip.presence if entry.is_a?(String) || entry.is_a?(Symbol)
|
|
278
|
+
return nil unless entry.respond_to?(:key?)
|
|
279
|
+
|
|
280
|
+
(entry["key"] || entry[:key] || entry["name"] || entry[:name]).to_s.strip.presence
|
|
281
|
+
end
|
|
282
|
+
|
|
283
|
+
# The per-server allow-list, or nil when the entry names none — which
|
|
284
|
+
# means the agent is offered every tool the server serves.
|
|
285
|
+
def allowed_tools(key)
|
|
286
|
+
entry = configured_servers[key]
|
|
287
|
+
return nil unless entry.respond_to?(:key?)
|
|
288
|
+
|
|
289
|
+
names = entry["tools"] || entry[:tools]
|
|
290
|
+
return nil if names.nil?
|
|
291
|
+
|
|
292
|
+
Array(names).filter_map { |tool| (tool.respond_to?(:key?) ? tool["name"] || tool[:name] : tool).to_s.presence }
|
|
293
|
+
end
|
|
294
|
+
end
|
|
295
|
+
end
|
|
@@ -231,13 +231,22 @@ module ActionAgent
|
|
|
231
231
|
def call(name, **kwargs)
|
|
232
232
|
return { error: "Unknown tool: #{name}" } unless function?(name)
|
|
233
233
|
|
|
234
|
-
|
|
235
|
-
|
|
234
|
+
# Who the call is for is never one of the call's arguments: it comes
|
|
235
|
+
# off here, before a tool sees them. That keeps a built-in from
|
|
236
|
+
# meeting an unexpected keyword, and keeps the cache key below over
|
|
237
|
+
# the arguments alone.
|
|
238
|
+
actor = kwargs.delete(:actor)
|
|
239
|
+
|
|
240
|
+
if (schema_tool = ActionAgent.schema_tool_class_for(name.to_s))
|
|
236
241
|
# `actor:` is the host's authorization seam — the scope block runs
|
|
237
242
|
# inside the call. It is passed through untouched, including nil,
|
|
238
243
|
# so a host scope decides what an unattributed run may read rather
|
|
239
244
|
# than the engine widening it.
|
|
240
|
-
|
|
245
|
+
#
|
|
246
|
+
# Deliberately not cached: a scoped read is one caller's answer,
|
|
247
|
+
# and replaying it for the next caller would hand them rows their
|
|
248
|
+
# own scope would have refused.
|
|
249
|
+
return schema_tool.call(name.to_s, actor: actor, **kwargs)
|
|
241
250
|
end
|
|
242
251
|
|
|
243
252
|
return public_send(FUNCTIONS.fetch(name.to_s), **kwargs) if UNCACHED_FUNCTIONS.include?(name.to_s)
|