actionagent 1.6.0 → 1.6.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/app/assets/builds/action_agent.js +45 -45
- data/app/controllers/action_agent/api/agent_runs_controller.rb +2 -2
- data/app/controllers/action_agent/api/agents_controller.rb +8 -5
- data/app/controllers/action_agent/api/analytics_controller.rb +1 -1
- data/app/controllers/action_agent/api/base_controller.rb +20 -0
- data/app/controllers/action_agent/api/interactions_controller.rb +3 -3
- data/app/controllers/action_agent/api/sandboxes_controller.rb +6 -1
- data/app/controllers/action_agent/api/session_recordings_controller.rb +24 -7
- data/app/models/action_agent/agent.rb +60 -0
- data/app/models/action_agent/agent_run.rb +4 -0
- data/app/models/action_agent/agent_version.rb +10 -0
- data/app/models/action_agent/evaluation.rb +19 -2
- data/app/models/action_agent/evaluation_run.rb +4 -0
- data/app/models/action_agent/telemetry_trace.rb +28 -1
- data/app/services/action_agent/agent_execution_service.rb +63 -12
- data/app/services/action_agent/agent_registrar.rb +23 -1
- data/app/services/action_agent/agent_release.rb +61 -0
- data/app/services/action_agent/agent_tool_roster.rb +47 -21
- data/app/services/action_agent/evaluation_runner_service.rb +12 -12
- data/app/services/action_agent/scenario_evaluation_runner.rb +16 -1
- data/config/routes.rb +3 -0
- data/lib/action_agent/version.rb +1 -1
- data/lib/generators/action_agent/install_generator.rb +6 -0
- data/lib/generators/action_agent/templates/action_agent.rb.erb +8 -0
- data/lib/generators/action_agent/templates/add_agent_releases.rb.erb +50 -0
- data/lib/tasks/action_agent.rake +33 -0
- metadata +8 -3
|
@@ -33,8 +33,8 @@ module ActionAgent
|
|
|
33
33
|
scope = scope.where(agent_id: params[:agent_id]) if params[:agent_id].present?
|
|
34
34
|
scope = scope.where(status: params[:status]) if params[:status].present?
|
|
35
35
|
|
|
36
|
-
page = (
|
|
37
|
-
per_page = (
|
|
36
|
+
page = integer_param(:page, default: 1)
|
|
37
|
+
per_page = integer_param(:per_page, default: 20)
|
|
38
38
|
total = scope.count
|
|
39
39
|
runs = scope.offset((page - 1) * per_page).limit(per_page)
|
|
40
40
|
|
|
@@ -132,9 +132,9 @@ module ActionAgent
|
|
|
132
132
|
# observed from telemetry have no AgentRun rows at all, so a runs-only
|
|
133
133
|
# list showed them as empty while their scorecard reported real traffic.
|
|
134
134
|
def runs
|
|
135
|
-
minutes =
|
|
136
|
-
page = (
|
|
137
|
-
per_page = (
|
|
135
|
+
minutes = integer_param(:minutes)&.clamp(1, 60 * 24 * 90)
|
|
136
|
+
page = integer_param(:page, default: 1)
|
|
137
|
+
per_page = integer_param(:per_page, default: 20)
|
|
138
138
|
|
|
139
139
|
executions = AgentExecutions.new(
|
|
140
140
|
agents: [ @agent ],
|
|
@@ -282,7 +282,7 @@ module ActionAgent
|
|
|
282
282
|
# with all-zero metrics beside a card and a runs list reporting real
|
|
283
283
|
# traffic.
|
|
284
284
|
def analytics
|
|
285
|
-
days = (
|
|
285
|
+
days = integer_param(:days, default: 30)
|
|
286
286
|
start_date = days.days.ago.beginning_of_day
|
|
287
287
|
|
|
288
288
|
runs = @agent.agent_runs.where("created_at >= ?", start_date)
|
|
@@ -532,7 +532,10 @@ module ActionAgent
|
|
|
532
532
|
change_summary: version.change_summary,
|
|
533
533
|
created_by: version.created_by,
|
|
534
534
|
created_at: version.created_at,
|
|
535
|
-
is_latest: version.latest
|
|
535
|
+
is_latest: version.latest?,
|
|
536
|
+
release: version.release?,
|
|
537
|
+
release_digest: version.release_digest,
|
|
538
|
+
revision: version.revision
|
|
536
539
|
}
|
|
537
540
|
|
|
538
541
|
if include_diff && version.previous
|
|
@@ -5,7 +5,7 @@ module ActionAgent
|
|
|
5
5
|
class AnalyticsController < BaseController
|
|
6
6
|
# GET /api/analytics
|
|
7
7
|
def index
|
|
8
|
-
days = (
|
|
8
|
+
days = integer_param(:days, default: 30)
|
|
9
9
|
start_date = days.days.ago.beginning_of_day
|
|
10
10
|
|
|
11
11
|
# Table names are interpolated rather than written literally: the
|
|
@@ -129,6 +129,26 @@ module ActionAgent
|
|
|
129
129
|
render json: { error: "Agent execution is disabled on this dashboard" }, status: :forbidden
|
|
130
130
|
end
|
|
131
131
|
|
|
132
|
+
# An integer query param. A value can arrive as a container
|
|
133
|
+
# (`minutes[]=1&minutes[]=2`, or `page[x]=1`), and neither Array nor
|
|
134
|
+
# ActionController::Parameters responds to `to_i`: reading them
|
|
135
|
+
# directly raised NoMethodError and turned a malformed query into a
|
|
136
|
+
# 500. A multi-valued param means its first value; anything else that
|
|
137
|
+
# is not a scalar falls back to the default.
|
|
138
|
+
def integer_param(name, default: nil)
|
|
139
|
+
raw = params[name]
|
|
140
|
+
raw = raw.first if raw.is_a?(Array)
|
|
141
|
+
return default if raw.blank? || !(raw.is_a?(String) || raw.is_a?(Numeric))
|
|
142
|
+
|
|
143
|
+
raw.to_s.to_i
|
|
144
|
+
end
|
|
145
|
+
|
|
146
|
+
# integer_param, then clamped into [min, max]. Non-numeric input becomes
|
|
147
|
+
# 0 and is then clamped up to `min`.
|
|
148
|
+
def clamped_param(name, default:, min:, max:)
|
|
149
|
+
integer_param(name, default: default).clamp(min, max)
|
|
150
|
+
end
|
|
151
|
+
|
|
132
152
|
def not_found
|
|
133
153
|
render json: { error: "Record not found" }, status: :not_found
|
|
134
154
|
end
|
|
@@ -16,7 +16,7 @@ module ActionAgent
|
|
|
16
16
|
|
|
17
17
|
# GET /api/interactions
|
|
18
18
|
def index
|
|
19
|
-
limit =
|
|
19
|
+
limit = clamped_param(:limit, default: DEFAULT_LIMIT, min: 1, max: 200)
|
|
20
20
|
|
|
21
21
|
contexts = interactions_scope
|
|
22
22
|
.includes(:contextable)
|
|
@@ -140,8 +140,8 @@ module ActionAgent
|
|
|
140
140
|
def window_minutes
|
|
141
141
|
return @window_minutes if defined?(@window_minutes)
|
|
142
142
|
|
|
143
|
-
raw =
|
|
144
|
-
@window_minutes = raw ? raw.
|
|
143
|
+
raw = integer_param(:minutes)
|
|
144
|
+
@window_minutes = raw ? raw.clamp(1, MAX_WINDOW_MINUTES) : nil
|
|
145
145
|
end
|
|
146
146
|
|
|
147
147
|
def interactions_scope
|
|
@@ -18,11 +18,16 @@ module ActionAgent
|
|
|
18
18
|
# POST /api/sandboxes/compare
|
|
19
19
|
# Run multiple providers in a single sandbox using parallel generation jobs
|
|
20
20
|
def compare
|
|
21
|
-
providers = params[:providers]
|
|
21
|
+
providers = params[:providers].nil? ? %w[anthropic openai ollama] : params[:providers]
|
|
22
22
|
task = params[:task]
|
|
23
23
|
sandbox_id = params[:sandbox_id]
|
|
24
24
|
|
|
25
25
|
return render json: { error: "Task required" }, status: :bad_request unless task.present?
|
|
26
|
+
# A bare string or a nested object is a malformed request, not a list
|
|
27
|
+
# of one provider: reading it as a list raised NoMethodError.
|
|
28
|
+
unless providers.is_a?(Array) && providers.all? { |name| name.is_a?(String) }
|
|
29
|
+
return render json: { error: "providers must be a list of provider names" }, status: :bad_request
|
|
30
|
+
end
|
|
26
31
|
return render json: { error: "At least 2 providers required" }, status: :bad_request if providers.size < 2
|
|
27
32
|
|
|
28
33
|
# Validate providers
|
|
@@ -13,6 +13,12 @@ module ActionAgent
|
|
|
13
13
|
|
|
14
14
|
before_action :set_recording, only: [ :show, :actions, :snapshot, :export, :handoff ]
|
|
15
15
|
|
|
16
|
+
# Browser state that must never leave the server in a read response:
|
|
17
|
+
# the handoff state a recording carries is a copy of the visitor's
|
|
18
|
+
# cookies and web storage. Only #handoff returns it, to the owner, when
|
|
19
|
+
# they continue the session.
|
|
20
|
+
SENSITIVE_STATE_KEYS = %w[cookies session_storage local_storage].freeze
|
|
21
|
+
|
|
16
22
|
# GET /api/session_recordings
|
|
17
23
|
# List recordings with optional filters
|
|
18
24
|
def index
|
|
@@ -36,8 +42,8 @@ module ActionAgent
|
|
|
36
42
|
end
|
|
37
43
|
|
|
38
44
|
# Pagination
|
|
39
|
-
page = (
|
|
40
|
-
per_page = [ (
|
|
45
|
+
page = integer_param(:page, default: 1)
|
|
46
|
+
per_page = [ integer_param(:per_page, default: 20), 100 ].min
|
|
41
47
|
offset = (page - 1) * per_page
|
|
42
48
|
|
|
43
49
|
total = recordings.count
|
|
@@ -79,10 +85,10 @@ module ActionAgent
|
|
|
79
85
|
|
|
80
86
|
# Support pagination for large recordings
|
|
81
87
|
if params[:after_sequence].present?
|
|
82
|
-
actions = actions.where("sequence > ?",
|
|
88
|
+
actions = actions.where("sequence > ?", integer_param(:after_sequence, default: 0))
|
|
83
89
|
end
|
|
84
90
|
|
|
85
|
-
limit = [
|
|
91
|
+
limit = [ integer_param(:limit, default: 100), 500 ].min
|
|
86
92
|
actions = actions.limit(limit)
|
|
87
93
|
|
|
88
94
|
render json: {
|
|
@@ -332,7 +338,7 @@ module ActionAgent
|
|
|
332
338
|
created_at: recording.created_at.iso8601,
|
|
333
339
|
updated_at: recording.updated_at.iso8601,
|
|
334
340
|
timeline: recording.timeline,
|
|
335
|
-
handoff_state: recording.metadata["handoff_state"],
|
|
341
|
+
handoff_state: safe_handoff_state(recording.metadata["handoff_state"]),
|
|
336
342
|
agent: recording.agent_run&.agent&.slice(:id, :name),
|
|
337
343
|
sandbox_session: recording.sandbox_session&.summary
|
|
338
344
|
}
|
|
@@ -343,9 +349,20 @@ module ActionAgent
|
|
|
343
349
|
action&.screenshot_url(expires_in: 1.hour)
|
|
344
350
|
end
|
|
345
351
|
|
|
352
|
+
# Strips the browser state at the top level and inside handoff_state,
|
|
353
|
+
# which the model stores nested (a recording's metadata carries the
|
|
354
|
+
# handoff as one key), so a show response never ships a session cookie.
|
|
346
355
|
def safe_metadata(metadata)
|
|
347
|
-
|
|
348
|
-
|
|
356
|
+
safe = (metadata || {}).except(*SENSITIVE_STATE_KEYS)
|
|
357
|
+
return safe unless safe["handoff_state"].is_a?(Hash)
|
|
358
|
+
|
|
359
|
+
safe.merge("handoff_state" => safe_handoff_state(safe["handoff_state"]))
|
|
360
|
+
end
|
|
361
|
+
|
|
362
|
+
def safe_handoff_state(handoff_state)
|
|
363
|
+
return handoff_state unless handoff_state.is_a?(Hash)
|
|
364
|
+
|
|
365
|
+
handoff_state.except(*SENSITIVE_STATE_KEYS)
|
|
349
366
|
end
|
|
350
367
|
|
|
351
368
|
def generate_visitor_id
|
|
@@ -11,6 +11,14 @@ module ActionAgent
|
|
|
11
11
|
has_many :agent_runs, dependent: :destroy
|
|
12
12
|
has_many :evaluations, dependent: :destroy
|
|
13
13
|
has_many :agent_memories, as: :memorable, dependent: :destroy
|
|
14
|
+
# Generations hang off AgentContext polymorphically, which is an
|
|
15
|
+
# implementation detail of how contexts are modelled — so without these a
|
|
16
|
+
# host that wants an agent's recorded history writes that join itself and is
|
|
17
|
+
# coupled to the shape. Deliberately no `dependent:` on the contexts: the
|
|
18
|
+
# association is added to read them, and destroying an agent has never taken
|
|
19
|
+
# its conversations with it. Making it do so is a separate call.
|
|
20
|
+
has_many :agent_contexts, as: :contextable
|
|
21
|
+
has_many :generations, through: :agent_contexts
|
|
14
22
|
|
|
15
23
|
# Polymorphic rows (agent_memories, agent_contexts) store this string.
|
|
16
24
|
# A host app that grew these tables under its own Agent constant keeps
|
|
@@ -219,6 +227,46 @@ module ActionAgent
|
|
|
219
227
|
agent_versions.order(version_number: :desc).first
|
|
220
228
|
end
|
|
221
229
|
|
|
230
|
+
# The most recent version cut from the agent's code, if any.
|
|
231
|
+
# @return [AgentVersion, nil]
|
|
232
|
+
def latest_release
|
|
233
|
+
agent_versions.releases.order(version_number: :desc).first
|
|
234
|
+
end
|
|
235
|
+
|
|
236
|
+
# Cuts a version for a release of the agent's code, identified by the
|
|
237
|
+
# digest ActiveAgent::Release computes from what the model is given.
|
|
238
|
+
# Returns the existing version when the latest release already carries
|
|
239
|
+
# this digest — a redeploy of an unchanged agent is not a new version —
|
|
240
|
+
# so it is safe to call on every deploy.
|
|
241
|
+
#
|
|
242
|
+
# The version's snapshot is the dashboard configuration plus the release
|
|
243
|
+
# manifest under "release", so the Versions tab can diff two releases the
|
|
244
|
+
# same way it diffs two dashboard edits.
|
|
245
|
+
#
|
|
246
|
+
# @param digest [String] ActiveAgent::Release digest of the host class
|
|
247
|
+
# @param manifest [Hash, nil] the class's release manifest
|
|
248
|
+
# @param revision [String, nil] the deploy (git SHA, release label)
|
|
249
|
+
# @param released_by [String, nil]
|
|
250
|
+
# @return [AgentVersion]
|
|
251
|
+
def record_release!(digest:, manifest: nil, revision: nil, released_by: nil)
|
|
252
|
+
current = latest_release
|
|
253
|
+
if current && current.release_digest == digest
|
|
254
|
+
update_columns(release_digest: digest) if release_digest != digest
|
|
255
|
+
return current
|
|
256
|
+
end
|
|
257
|
+
|
|
258
|
+
version = agent_versions.create!(
|
|
259
|
+
version_number: (latest_version&.version_number || 0) + 1,
|
|
260
|
+
change_summary: release_summary(digest, revision, current&.configuration_snapshot&.dig("release"), manifest),
|
|
261
|
+
configuration_snapshot: configuration_snapshot.merge("release" => manifest || {}),
|
|
262
|
+
release_digest: digest,
|
|
263
|
+
revision: revision,
|
|
264
|
+
created_by: released_by || "release"
|
|
265
|
+
)
|
|
266
|
+
update_columns(release_digest: digest)
|
|
267
|
+
version
|
|
268
|
+
end
|
|
269
|
+
|
|
222
270
|
# Maps each historical instructions digest to the first version that
|
|
223
271
|
# introduced it ("v3"), so run cohorts can label instruction changes with
|
|
224
272
|
# real agent versions instead of raw hashes.
|
|
@@ -396,6 +444,18 @@ module ActionAgent
|
|
|
396
444
|
end
|
|
397
445
|
end
|
|
398
446
|
|
|
447
|
+
# "Release 1a2b3c4d5e6f · abc1234: templates, tools" — the digest, the
|
|
448
|
+
# deploy, and which parts of the manifest moved since the last release.
|
|
449
|
+
def release_summary(digest, revision, previous_manifest, manifest)
|
|
450
|
+
label = [ "Release #{digest}", revision.presence ].compact.join(" · ")
|
|
451
|
+
return "#{label}: first release" if previous_manifest.blank? || manifest.blank?
|
|
452
|
+
|
|
453
|
+
changed = (previous_manifest.keys | manifest.stringify_keys.keys).select do |key|
|
|
454
|
+
previous_manifest[key] != manifest.stringify_keys[key]
|
|
455
|
+
end
|
|
456
|
+
changed.any? ? "#{label}: #{changed.sort.join(', ')}" : label
|
|
457
|
+
end
|
|
458
|
+
|
|
399
459
|
def create_initial_version
|
|
400
460
|
agent_versions.create!(
|
|
401
461
|
version_number: 1,
|
|
@@ -3,6 +3,10 @@
|
|
|
3
3
|
module ActionAgent
|
|
4
4
|
class AgentRun < ApplicationRecord
|
|
5
5
|
belongs_to :agent
|
|
6
|
+
# The version of the agent this run executed under — the latest at the
|
|
7
|
+
# time, since a run is against the agent as it is.
|
|
8
|
+
belongs_to :agent_version, optional: true
|
|
9
|
+
before_create { self.agent_version_id ||= agent&.latest_version&.id }
|
|
6
10
|
|
|
7
11
|
# Raised when a caller hands a run files to attach in a host app that
|
|
8
12
|
# has nowhere to keep them.
|
|
@@ -9,6 +9,9 @@ module ActionAgent
|
|
|
9
9
|
|
|
10
10
|
# Scopes
|
|
11
11
|
scope :recent, -> { order(version_number: :desc) }
|
|
12
|
+
# Versions cut from the agent's code on deploy, as opposed to edits made
|
|
13
|
+
# in the dashboard.
|
|
14
|
+
scope :releases, -> { where.not(release_digest: [ nil, "" ]) }
|
|
12
15
|
scope :by_version, ->(num) { where(version_number: num) }
|
|
13
16
|
|
|
14
17
|
# Compare two versions
|
|
@@ -36,6 +39,13 @@ module ActionAgent
|
|
|
36
39
|
end
|
|
37
40
|
|
|
38
41
|
# Check if this is the latest version
|
|
42
|
+
# Whether this version was cut from the agent's code (it carries the
|
|
43
|
+
# release digest) rather than from a dashboard edit.
|
|
44
|
+
# @return [Boolean]
|
|
45
|
+
def release?
|
|
46
|
+
release_digest.present?
|
|
47
|
+
end
|
|
48
|
+
|
|
39
49
|
def latest?
|
|
40
50
|
agent.latest_version&.id == id
|
|
41
51
|
end
|
|
@@ -35,6 +35,17 @@ module ActionAgent
|
|
|
35
35
|
|
|
36
36
|
scope :recent, -> { order(updated_at: :desc) }
|
|
37
37
|
|
|
38
|
+
# MySQL cannot give a JSON column a default, so a row inserted there
|
|
39
|
+
# without `criteria` or `config` reads back nil. Both readers answer with
|
|
40
|
+
# the empty value the column default supplies on other databases.
|
|
41
|
+
def criteria
|
|
42
|
+
super || []
|
|
43
|
+
end
|
|
44
|
+
|
|
45
|
+
def config
|
|
46
|
+
super || {}
|
|
47
|
+
end
|
|
48
|
+
|
|
38
49
|
def latest_run
|
|
39
50
|
evaluation_runs.order(created_at: :desc).first
|
|
40
51
|
end
|
|
@@ -89,10 +100,16 @@ module ActionAgent
|
|
|
89
100
|
# ActiveAgent::Evals::ScenarioParser output). Keys already in the suite keep their records, so
|
|
90
101
|
# earlier runs' results still resolve to their scenario, and keep their
|
|
91
102
|
# enabled flag unless the attributes set it (a paste cannot).
|
|
92
|
-
|
|
103
|
+
# `on_removed:` decides what happens to a scenario the new attributes no
|
|
104
|
+
# longer name: `:destroy` drops it, `:disable` keeps the row with
|
|
105
|
+
# `enabled: false` so runs that scored it still resolve their results.
|
|
106
|
+
def replace_scenarios!(attributes, on_removed: :destroy)
|
|
107
|
+
raise ArgumentError, "on_removed must be :destroy or :disable" unless %i[destroy disable].include?(on_removed)
|
|
108
|
+
|
|
93
109
|
transaction do
|
|
94
110
|
keep = attributes.map { |attrs| attrs["key"] }
|
|
95
|
-
scenarios.where.not(key: keep)
|
|
111
|
+
removed = scenarios.where.not(key: keep)
|
|
112
|
+
on_removed == :disable ? removed.update_all(enabled: false) : removed.destroy_all
|
|
96
113
|
|
|
97
114
|
attributes.each_with_index do |attrs, index|
|
|
98
115
|
scenario = scenarios.find_or_initialize_by(key: attrs["key"])
|
|
@@ -8,6 +8,10 @@ module ActionAgent
|
|
|
8
8
|
# See #average_score, which is what has to tolerate both shapes.
|
|
9
9
|
class EvaluationRun < ApplicationRecord
|
|
10
10
|
belongs_to :evaluation
|
|
11
|
+
# The version of the evaluated agent this run scored, so a pass rate is
|
|
12
|
+
# a statement about a release rather than about "the agent".
|
|
13
|
+
belongs_to :agent_version, optional: true
|
|
14
|
+
before_create { self.agent_version_id ||= evaluation&.agent&.latest_version&.id }
|
|
11
15
|
has_many :scenario_results, class_name: "EvaluationScenarioResult", dependent: :destroy
|
|
12
16
|
|
|
13
17
|
enum :status, { pending: 0, running: 1, complete: 2, failed: 3 }
|
|
@@ -43,6 +43,8 @@ module ActionAgent
|
|
|
43
43
|
scope :for_date_range, ->(start_date, end_date) { where(timestamp: start_date..end_date) }
|
|
44
44
|
# The dashboard agent this trace was attributed to on ingest, if any.
|
|
45
45
|
belongs_to :agent, class_name: "ActionAgent::Agent", optional: true
|
|
46
|
+
# The release of the agent this trace came from (see #attach_agent_version!).
|
|
47
|
+
belongs_to :agent_version, class_name: "ActionAgent::AgentVersion", optional: true
|
|
46
48
|
|
|
47
49
|
scope :for_account, ->(account) { where(account: account) if ActionAgent.multi_tenant? }
|
|
48
50
|
|
|
@@ -252,7 +254,10 @@ module ActionAgent
|
|
|
252
254
|
# dashboard-authored agent by guessing a primary key.
|
|
253
255
|
attrs[:agent_id] = agent&.id
|
|
254
256
|
|
|
255
|
-
create!(attrs).tap
|
|
257
|
+
create!(attrs).tap do |record|
|
|
258
|
+
AgentRegistrar.call(record)
|
|
259
|
+
record.attach_agent_version!
|
|
260
|
+
end
|
|
256
261
|
end
|
|
257
262
|
|
|
258
263
|
|
|
@@ -265,6 +270,28 @@ module ActionAgent
|
|
|
265
270
|
tokens.fetch("input", 0).to_i + tokens.fetch("output", 0).to_i + tokens.fetch("thinking", 0).to_i
|
|
266
271
|
end
|
|
267
272
|
|
|
273
|
+
# Pins this trace to the version of its agent that produced it. The
|
|
274
|
+
# instrumentation stamps the root span with `agent.version` — the digest
|
|
275
|
+
# ActiveAgent::Release computes from the class — and a release cut on
|
|
276
|
+
# deploy carries the same digest, so the two meet here. A trace from a
|
|
277
|
+
# dashboard run carries no digest and takes the agent's latest version.
|
|
278
|
+
#
|
|
279
|
+
# @return [AgentVersion, nil]
|
|
280
|
+
def attach_agent_version!
|
|
281
|
+
return if agent_version_id.present? || agent_id.blank?
|
|
282
|
+
|
|
283
|
+
digest = root_span&.dig("attributes", "agent.version").presence
|
|
284
|
+
version = if digest
|
|
285
|
+
AgentVersion.find_by(agent_id: agent_id, release_digest: digest)
|
|
286
|
+
else
|
|
287
|
+
AgentVersion.where(agent_id: agent_id).order(version_number: :desc).first
|
|
288
|
+
end
|
|
289
|
+
return unless version
|
|
290
|
+
|
|
291
|
+
update_columns(agent_version_id: version.id)
|
|
292
|
+
version
|
|
293
|
+
end
|
|
294
|
+
|
|
268
295
|
# Returns the root span of this trace.
|
|
269
296
|
#
|
|
270
297
|
# @return [Hash, nil] The root span or nil
|
|
@@ -33,9 +33,13 @@ module ActionAgent
|
|
|
33
33
|
# so a multi-gigabyte log named .csv costs a fixed slice of memory rather
|
|
34
34
|
# than its whole size. Four bytes per character is UTF-8's worst case.
|
|
35
35
|
ATTACHMENT_TEXT_BYTE_LIMIT = ATTACHMENT_TEXT_LIMIT * 4
|
|
36
|
+
# What one prompt-span attribute stores. The value is a preview for reading,
|
|
37
|
+
# so it is clipped; a size that has to stay exact travels as its own
|
|
38
|
+
# `*.tokens` attribute instead.
|
|
39
|
+
PROMPT_SPAN_ATTRIBUTE_LIMIT = 6000
|
|
36
40
|
# The prompt span records the transcript, not the data URIs; keep the
|
|
37
41
|
# whole serialized list within the same budget as the other attributes.
|
|
38
|
-
PROMPT_SPAN_MESSAGE_LIMIT =
|
|
42
|
+
PROMPT_SPAN_MESSAGE_LIMIT = PROMPT_SPAN_ATTRIBUTE_LIMIT
|
|
39
43
|
# Prior turns sent with a pinned conversation: the most recent ones,
|
|
40
44
|
# trimmed oldest-first to a character budget.
|
|
41
45
|
HISTORY_TURN_LIMIT = 40
|
|
@@ -164,11 +168,19 @@ module ActionAgent
|
|
|
164
168
|
def record_prompt_span(root_span)
|
|
165
169
|
span = root_span.add_span("agent.prompt", span_type: :prompt)
|
|
166
170
|
if composed_instructions.present?
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
span.set_attribute("prompt.input.tools", tool_schemas.to_json.byteslice(0, 6000).to_s.scrub)
|
|
171
|
+
instructions = composed_instructions.to_s
|
|
172
|
+
span.set_attribute("prompt.input.instructions", instructions.byteslice(0, PROMPT_SPAN_ATTRIBUTE_LIMIT).to_s.scrub)
|
|
173
|
+
span.set_attribute("prompt.input.instructions.tokens", estimated_tokens(instructions))
|
|
171
174
|
end
|
|
175
|
+
# MCP and toolbox schemas are attributed separately so the meter can name
|
|
176
|
+
# which half fills the window, and each carries its size. The content
|
|
177
|
+
# attributes are truncated previews for reading: sizing the context from
|
|
178
|
+
# one understates it by whatever the clip dropped, which for a twelve-tool
|
|
179
|
+
# agent is most of the schema.
|
|
180
|
+
record_tool_schema_attributes(span)
|
|
181
|
+
full_transcript_json = prompt_turn[:transcript].map do |message|
|
|
182
|
+
{ role: message[:role], content: message[:content].to_s }
|
|
183
|
+
end.to_json
|
|
172
184
|
transcript = prompt_turn[:transcript].map do |message|
|
|
173
185
|
{ role: message[:role], content: message[:content].to_s.byteslice(0, 4000).to_s.scrub }
|
|
174
186
|
end
|
|
@@ -181,6 +193,13 @@ module ActionAgent
|
|
|
181
193
|
end
|
|
182
194
|
span.set_attribute("prompt.input.messages", serialized)
|
|
183
195
|
span.set_attribute("messages.count", transcript.size)
|
|
196
|
+
# The stored attribute is the tail of the history that fit, so its size is
|
|
197
|
+
# not the transcript's. The meter apportions the provider's prompt_tokens
|
|
198
|
+
# across the segments it can size, and a transcript missing from that set
|
|
199
|
+
# is not merely imprecise: the segments that remain are scaled up to cover
|
|
200
|
+
# it, so a long conversation reads as an enormous system prompt. Measured
|
|
201
|
+
# over the full turn, before either the per-message clip or the trim.
|
|
202
|
+
span.set_attribute("prompt.input.messages.tokens", estimated_tokens(full_transcript_json))
|
|
184
203
|
span.finish
|
|
185
204
|
rescue StandardError => e
|
|
186
205
|
Rails.logger.warn("[AgentExecutionService] prompt span failed: #{e.message}")
|
|
@@ -385,6 +404,30 @@ module ActionAgent
|
|
|
385
404
|
@mcp_dispatcher ||= MCPToolDispatcher.new(@agent_record)
|
|
386
405
|
end
|
|
387
406
|
|
|
407
|
+
# Splits the offered schemas the way `tool_schemas` assembles them, so the
|
|
408
|
+
# span reports what the model was actually sent: nothing for a mock run, and
|
|
409
|
+
# one MCP round trip rather than a second one for telemetry.
|
|
410
|
+
def record_tool_schema_attributes(span)
|
|
411
|
+
mcp_definitions, toolbox_definitions = tool_schema_halves
|
|
412
|
+
{
|
|
413
|
+
"prompt.input.tools" => toolbox_definitions,
|
|
414
|
+
"prompt.input.mcp_tools" => mcp_definitions
|
|
415
|
+
}.each do |key, definitions|
|
|
416
|
+
next if definitions.blank?
|
|
417
|
+
|
|
418
|
+
json = definitions.to_json
|
|
419
|
+
span.set_attribute(key, json.byteslice(0, PROMPT_SPAN_ATTRIBUTE_LIMIT).to_s.scrub)
|
|
420
|
+
span.set_attribute("#{key}.tokens", estimated_tokens(json))
|
|
421
|
+
end
|
|
422
|
+
end
|
|
423
|
+
|
|
424
|
+
# ~4 chars/token, the same approximation the context meter applies to content
|
|
425
|
+
# it sizes itself. Taken before truncation, so the meter reads the whole
|
|
426
|
+
# schema rather than the preview the attribute stores.
|
|
427
|
+
def estimated_tokens(text)
|
|
428
|
+
(text.length / 4.0).round
|
|
429
|
+
end
|
|
430
|
+
|
|
388
431
|
def call_agent(slug:, message:)
|
|
389
432
|
depth = Thread.current[:agent_call_depth].to_i
|
|
390
433
|
return { error: "call_agent depth limit (#{MAX_CALL_DEPTH}) reached" } if depth >= MAX_CALL_DEPTH
|
|
@@ -562,13 +605,21 @@ module ActionAgent
|
|
|
562
605
|
# server-side implementations (none for mock runs — the mock provider
|
|
563
606
|
# doesn't do tool calling).
|
|
564
607
|
def tool_schemas
|
|
565
|
-
|
|
566
|
-
|
|
567
|
-
|
|
568
|
-
|
|
569
|
-
|
|
570
|
-
|
|
571
|
-
|
|
608
|
+
mcp_definitions, toolbox_definitions = tool_schema_halves
|
|
609
|
+
mcp_definitions + toolbox_definitions
|
|
610
|
+
end
|
|
611
|
+
|
|
612
|
+
# The agent's own MCP servers describe their tools; the toolbox describes the
|
|
613
|
+
# rest. Without the first half a tool the agent declares is never offered to
|
|
614
|
+
# the model, which then answers from memory instead of calling it. Memoized
|
|
615
|
+
# because listing a server's tools is a request to that server.
|
|
616
|
+
def tool_schema_halves
|
|
617
|
+
@tool_schema_halves ||=
|
|
618
|
+
if provider == :mock
|
|
619
|
+
[ [], [] ]
|
|
620
|
+
else
|
|
621
|
+
[ mcp_dispatcher.tool_definitions, AgentToolbox.definitions_for(@agent_record.tools) ]
|
|
622
|
+
end
|
|
572
623
|
end
|
|
573
624
|
|
|
574
625
|
# Persists the tool interaction stream to the solid_agent conversation
|
|
@@ -77,8 +77,22 @@ module ActionAgent
|
|
|
77
77
|
ActionAgent.multi_tenant? ? @trace.try(:account) : nil
|
|
78
78
|
end
|
|
79
79
|
|
|
80
|
+
# The set an observed agent is deduplicated within.
|
|
81
|
+
#
|
|
82
|
+
# `for_owner(nil)` is `none` — correct for a multi-tenant read, where an
|
|
83
|
+
# unresolved tenant must see nothing. But a single-tenant dashboard has no
|
|
84
|
+
# owner to resolve and legitimately registers with `owner` nil (see
|
|
85
|
+
# #owner_for_trace), and `none` makes every dedupe lookup miss: each trace
|
|
86
|
+
# created another copy of the same agent, and the MAX_OBSERVED_PER_OWNER
|
|
87
|
+
# cap never engaged because the count it reads was always zero. Scoping to
|
|
88
|
+
# the whole table when there is no owner concept is what `for_owner`
|
|
89
|
+
# already does for a model with no owner association.
|
|
90
|
+
def agents_for_owner(owner)
|
|
91
|
+
owner.nil? && !ActionAgent.multi_tenant? ? Agent.all : Agent.for_owner(owner)
|
|
92
|
+
end
|
|
93
|
+
|
|
80
94
|
def find_or_create_agent(owner)
|
|
81
|
-
agents =
|
|
95
|
+
agents = agents_for_owner(owner)
|
|
82
96
|
existing = agents.find_by(
|
|
83
97
|
service_name: @trace.service_name,
|
|
84
98
|
agent_class_name: agent_class,
|
|
@@ -86,6 +100,14 @@ module ActionAgent
|
|
|
86
100
|
)
|
|
87
101
|
return existing if existing
|
|
88
102
|
|
|
103
|
+
# A host that mirrors its agent classes into the dashboard (a sync, a
|
|
104
|
+
# release) names the class on the record and nothing else: that record
|
|
105
|
+
# stands for every action of the class, so a trace from the class is
|
|
106
|
+
# its trace — not an observed twin's. Observed records are per action
|
|
107
|
+
# and are only ever matched on all three keys above.
|
|
108
|
+
mirrored = agents.where.not(status: "observed").find_by(agent_class_name: agent_class)
|
|
109
|
+
return mirrored if mirrored
|
|
110
|
+
|
|
89
111
|
return if agents.observed_agents.count >= MAX_OBSERVED_PER_OWNER
|
|
90
112
|
|
|
91
113
|
create_observed_agent(owner)
|
|
@@ -0,0 +1,61 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module ActionAgent
|
|
4
|
+
# Cuts a version of every dashboard agent that mirrors a host class, from
|
|
5
|
+
# that class's release digest — what a deploy runs so the dashboard's
|
|
6
|
+
# versions line up with the code that shipped.
|
|
7
|
+
#
|
|
8
|
+
# The host owns the mirror: whatever syncs its ActiveAgent classes into
|
|
9
|
+
# Agent records sets `agent_class_name`, and this reads it back. A record
|
|
10
|
+
# whose class no longer resolves, or whose class predates
|
|
11
|
+
# ActiveAgent::Release, is reported and skipped rather than failed.
|
|
12
|
+
#
|
|
13
|
+
# Idempotent: a redeploy of an unchanged agent cuts nothing, so running it
|
|
14
|
+
# on every deploy is the intended use (`rake action_agent:agents:release`).
|
|
15
|
+
class AgentRelease
|
|
16
|
+
Row = Struct.new(:agent, :version, :cut, :skipped, keyword_init: true)
|
|
17
|
+
Result = Struct.new(:rows, :revision, keyword_init: true) do
|
|
18
|
+
def cut = rows.select(&:cut)
|
|
19
|
+
def skipped = rows.select(&:skipped)
|
|
20
|
+
end
|
|
21
|
+
|
|
22
|
+
# @param revision [String, nil] the deploy (git SHA, release label);
|
|
23
|
+
# ActiveAgent::Release.revision when nil
|
|
24
|
+
# @param agents [ActiveRecord::Relation] the records to release; every
|
|
25
|
+
# record naming a host class by default
|
|
26
|
+
# @param released_by [String, nil] recorded on each version cut
|
|
27
|
+
def self.call(revision: nil, agents: nil, released_by: nil)
|
|
28
|
+
new(revision: revision, agents: agents, released_by: released_by).call
|
|
29
|
+
end
|
|
30
|
+
|
|
31
|
+
def initialize(revision: nil, agents: nil, released_by: nil)
|
|
32
|
+
@revision = revision.presence || ActiveAgent::Release.revision
|
|
33
|
+
@agents = agents || Agent.where.not(agent_class_name: [ nil, "" ])
|
|
34
|
+
@released_by = released_by
|
|
35
|
+
end
|
|
36
|
+
|
|
37
|
+
# @return [Result]
|
|
38
|
+
def call
|
|
39
|
+
rows = @agents.order(:name).map { |agent| release(agent) }
|
|
40
|
+
Result.new(rows: rows, revision: @revision)
|
|
41
|
+
end
|
|
42
|
+
|
|
43
|
+
private
|
|
44
|
+
|
|
45
|
+
def release(agent)
|
|
46
|
+
klass = agent.agent_class_name.to_s.safe_constantize
|
|
47
|
+
unless klass.respond_to?(:release_digest)
|
|
48
|
+
return Row.new(agent: agent, version: nil, cut: false, skipped: "#{agent.agent_class_name} does not resolve to a releasable class")
|
|
49
|
+
end
|
|
50
|
+
|
|
51
|
+
before = agent.latest_version&.id
|
|
52
|
+
version = agent.record_release!(
|
|
53
|
+
digest: klass.release_digest,
|
|
54
|
+
manifest: klass.release_manifest,
|
|
55
|
+
revision: @revision,
|
|
56
|
+
released_by: @released_by
|
|
57
|
+
)
|
|
58
|
+
Row.new(agent: agent, version: version, cut: version.id != before, skipped: nil)
|
|
59
|
+
end
|
|
60
|
+
end
|
|
61
|
+
end
|