actionagent 1.3.0 → 1.5.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/app/assets/builds/action_agent.css +1 -1
- data/app/assets/builds/action_agent.js +70 -48
- data/app/controllers/action_agent/api/agent_runs_controller.rb +3 -1
- data/app/controllers/action_agent/api/agents_controller.rb +117 -10
- data/app/controllers/action_agent/api/dashboard_assistant_controller.rb +83 -0
- data/app/controllers/action_agent/api/evaluations_controller.rb +22 -7
- data/app/controllers/action_agent/api/interaction_messages_controller.rb +98 -0
- data/app/controllers/action_agent/dashboard_controller.rb +2 -1
- data/app/models/action_agent/agent.rb +107 -19
- data/app/models/action_agent/agent_run.rb +99 -0
- data/app/models/action_agent/evaluation_run.rb +14 -6
- data/app/models/action_agent/evaluation_scenario_result.rb +24 -4
- data/app/models/action_agent/telemetry_trace.rb +16 -1
- data/app/serializers/action_agent/agent_message_serializer.rb +1 -0
- data/app/services/action_agent/agent_execution_service.rb +294 -16
- data/app/services/action_agent/agent_registrar.rb +7 -6
- data/app/services/action_agent/agent_toolbox.rb +69 -5
- data/app/services/action_agent/dashboard_assistant_service.rb +342 -0
- data/app/services/action_agent/evaluation_evidence.rb +234 -0
- data/app/services/action_agent/evaluation_runner_service.rb +13 -3
- data/app/services/action_agent/evaluation_tool_resolver.rb +10 -2
- data/app/services/action_agent/mcp_client.rb +167 -0
- data/app/services/action_agent/mcp_tool_dispatcher.rb +148 -0
- data/app/services/action_agent/playwright_mcp_client.rb +11 -126
- data/app/services/action_agent/scenario_evaluation_runner.rb +90 -17
- data/config/routes.rb +11 -1
- data/lib/action_agent/assistant_request_filter.rb +22 -0
- data/lib/action_agent/engine.rb +5 -0
- data/lib/action_agent/version.rb +1 -1
- data/lib/action_agent.rb +113 -0
- data/lib/generators/action_agent/templates/action_agent.rb.erb +12 -0
- metadata +9 -2
|
@@ -4,6 +4,49 @@ module ActionAgent
|
|
|
4
4
|
class AgentRun < ApplicationRecord
|
|
5
5
|
belongs_to :agent
|
|
6
6
|
|
|
7
|
+
# Raised when a caller hands a run files to attach in a host app that
|
|
8
|
+
# has nowhere to keep them.
|
|
9
|
+
class AttachmentsUnavailable < StandardError
|
|
10
|
+
def initialize(message = "Attachments need Active Storage in the host app (run `rails active_storage:install`)")
|
|
11
|
+
super
|
|
12
|
+
end
|
|
13
|
+
end
|
|
14
|
+
|
|
15
|
+
# Files uploaded with the run. The execution service delivers them to
|
|
16
|
+
# the model (images and PDFs as data URIs, text inlined) and the
|
|
17
|
+
# persisted user message keeps a manifest of them.
|
|
18
|
+
#
|
|
19
|
+
# Guarded like RecordingSnapshot: a host app created with
|
|
20
|
+
# --skip-active-storage has no has_many_attached to call.
|
|
21
|
+
has_many_attached :attachments if defined?(ActiveStorage)
|
|
22
|
+
|
|
23
|
+
# Whether runs can carry files in this host app: Active Storage loaded,
|
|
24
|
+
# the macro applied, and its tables migrated. Never raises — a host
|
|
25
|
+
# that skipped `rails active_storage:install` still runs agents, it
|
|
26
|
+
# just can't attach files to them.
|
|
27
|
+
def self.attachments_available?
|
|
28
|
+
defined?(ActiveStorage) && method_defined?(:attachments) && ActiveStorage::Blob.table_exists?
|
|
29
|
+
rescue StandardError
|
|
30
|
+
false
|
|
31
|
+
end
|
|
32
|
+
|
|
33
|
+
# How an attachment reaches the model, by MIME type with a filename
|
|
34
|
+
# fallback for the text formats browsers upload as octet-stream:
|
|
35
|
+
# images and documents ride along as data URIs, text is inlined into
|
|
36
|
+
# the prompt, anything else is only described.
|
|
37
|
+
TEXT_CONTENT_TYPES = %w[application/json application/xml application/x-yaml application/csv].freeze
|
|
38
|
+
TEXT_EXTENSIONS = %w[.csv .md .txt .json .yml .yaml].freeze
|
|
39
|
+
|
|
40
|
+
def self.attachment_kind(content_type, filename = nil)
|
|
41
|
+
type = content_type.to_s.downcase
|
|
42
|
+
return "image" if type.start_with?("image/")
|
|
43
|
+
return "document" if type == "application/pdf"
|
|
44
|
+
return "text" if type.start_with?("text/") || TEXT_CONTENT_TYPES.include?(type)
|
|
45
|
+
return "text" if TEXT_EXTENSIONS.include?(File.extname(filename.to_s).downcase)
|
|
46
|
+
|
|
47
|
+
"file"
|
|
48
|
+
end
|
|
49
|
+
|
|
7
50
|
# Status enum
|
|
8
51
|
enum :status, { pending: 0, running: 1, complete: 2, failed: 3, cancelled: 4 }
|
|
9
52
|
|
|
@@ -103,6 +146,37 @@ module ActionAgent
|
|
|
103
146
|
complete? || failed? || cancelled?
|
|
104
147
|
end
|
|
105
148
|
|
|
149
|
+
# The conversation this run belongs to: the one it actually wrote to
|
|
150
|
+
# once it has executed (output_metadata), else the one the caller asked
|
|
151
|
+
# to continue. A pinned id the run declined — another agent's context,
|
|
152
|
+
# or another action's — must not be the id the API reports, or the
|
|
153
|
+
# runner would open a conversation the turn is not in.
|
|
154
|
+
def context_id
|
|
155
|
+
output_metadata&.dig("context_id") || input_params&.dig("context_id")
|
|
156
|
+
end
|
|
157
|
+
|
|
158
|
+
# The run's files as the runner and the persisted user message show
|
|
159
|
+
# them: one hash per attachment, with the kind the execution service
|
|
160
|
+
# sorted it into and a blob URL for thumbnails (nil when the host app
|
|
161
|
+
# didn't draw Active Storage's routes). Empty without Active Storage.
|
|
162
|
+
def attachment_manifest
|
|
163
|
+
return [] unless self.class.attachments_available?
|
|
164
|
+
|
|
165
|
+
attachment_records.map do |attachment|
|
|
166
|
+
blob = attachment.blob
|
|
167
|
+
{
|
|
168
|
+
"id" => attachment.id,
|
|
169
|
+
"blob_id" => blob.id,
|
|
170
|
+
"signed_id" => blob.signed_id,
|
|
171
|
+
"filename" => blob.filename.to_s,
|
|
172
|
+
"content_type" => blob.content_type,
|
|
173
|
+
"byte_size" => blob.byte_size,
|
|
174
|
+
"kind" => self.class.attachment_kind(blob.content_type, blob.filename.to_s),
|
|
175
|
+
"url" => blob_path(blob)
|
|
176
|
+
}
|
|
177
|
+
end
|
|
178
|
+
end
|
|
179
|
+
|
|
106
180
|
# Get a summary for display
|
|
107
181
|
def summary
|
|
108
182
|
{
|
|
@@ -118,6 +192,8 @@ module ActionAgent
|
|
|
118
192
|
instructions_digest: instructions_digest,
|
|
119
193
|
instructions_codename: instructions_codename,
|
|
120
194
|
instructions_preview: output_metadata&.dig("instructions")&.truncate(120),
|
|
195
|
+
attachments: attachment_manifest,
|
|
196
|
+
context_id: context_id,
|
|
121
197
|
created_at: created_at,
|
|
122
198
|
error: error_message
|
|
123
199
|
}
|
|
@@ -147,5 +223,28 @@ module ActionAgent
|
|
|
147
223
|
def set_trace_id
|
|
148
224
|
self.trace_id ||= SecureRandom.uuid
|
|
149
225
|
end
|
|
226
|
+
|
|
227
|
+
# Reads the association as loaded when a list preloaded it
|
|
228
|
+
# (with_attachments below), so serializing a page of runs costs two
|
|
229
|
+
# queries rather than two per run; a single run fetches its own.
|
|
230
|
+
def attachment_records
|
|
231
|
+
if attachments_attachments.loaded?
|
|
232
|
+
attachments_attachments.sort_by(&:id)
|
|
233
|
+
else
|
|
234
|
+
attachments_attachments.includes(:blob).order(:id).to_a
|
|
235
|
+
end
|
|
236
|
+
end
|
|
237
|
+
|
|
238
|
+
# Preloads attachments and blobs for a list of runs — a no-op scope in
|
|
239
|
+
# a host without Active Storage, so callers need no guard of their own.
|
|
240
|
+
def self.with_attachments
|
|
241
|
+
attachments_available? ? with_attached_attachments : all
|
|
242
|
+
end
|
|
243
|
+
|
|
244
|
+
def blob_path(blob)
|
|
245
|
+
Rails.application.routes.url_helpers.rails_blob_path(blob, only_path: true)
|
|
246
|
+
rescue StandardError
|
|
247
|
+
nil
|
|
248
|
+
end
|
|
150
249
|
end
|
|
151
250
|
end
|
|
@@ -27,6 +27,11 @@ module ActionAgent
|
|
|
27
27
|
Array(scores&.dig("_models")&.keys)
|
|
28
28
|
end
|
|
29
29
|
|
|
30
|
+
def report_metadata
|
|
31
|
+
value = scores&.dig("_metadata")
|
|
32
|
+
value.is_a?(Hash) ? value : {}
|
|
33
|
+
end
|
|
34
|
+
|
|
30
35
|
# The label ActiveAgent::Evals::Report gives a verdict it ranked by pass
|
|
31
36
|
# rate itself, for a comparison no judge was available to rule on. Read
|
|
32
37
|
# from the framework rather than restated: the report reads it back when
|
|
@@ -47,6 +52,8 @@ module ActionAgent
|
|
|
47
52
|
# ranking — else the evaluation's judge model. nil when neither is set,
|
|
48
53
|
# which the report reads as "rules".
|
|
49
54
|
def judge_label
|
|
55
|
+
return scores["_judge_label"] if scores&.key?("_judge_label")
|
|
56
|
+
|
|
50
57
|
recorded = recorded_verdict&.dig("judge").to_s
|
|
51
58
|
return recorded if recorded.present? && recorded != PASS_RATE_JUDGE
|
|
52
59
|
|
|
@@ -123,8 +130,9 @@ module ActionAgent
|
|
|
123
130
|
# caller for a run of a generation-sampling evaluation, which has no
|
|
124
131
|
# scenario results to report on.
|
|
125
132
|
def to_report(links: report_links)
|
|
126
|
-
rows = scenario_results.includes(:scenario).
|
|
127
|
-
.
|
|
133
|
+
rows = scenario_results.includes(:scenario).sort_by do |row|
|
|
134
|
+
[ row.evaluated_scenario["position"].to_i, row.evaluation_scenario_id, row.model ]
|
|
135
|
+
end
|
|
128
136
|
selected = selected_specs
|
|
129
137
|
specs = {}
|
|
130
138
|
results = rows.map do |row|
|
|
@@ -132,15 +140,15 @@ module ActionAgent
|
|
|
132
140
|
label: [ row.provider.presence, row.model ].compact.join("/"), provider: row.provider.to_s, model: row.model
|
|
133
141
|
)
|
|
134
142
|
ActiveAgent::Evals::Result.new(
|
|
135
|
-
scenario: ActiveAgent::Evals::Scenario.from_hash(row.
|
|
143
|
+
scenario: ActiveAgent::Evals::Scenario.from_hash(row.evaluated_scenario),
|
|
136
144
|
spec: spec,
|
|
137
145
|
replay: ActiveAgent::Evals::Replay.new(
|
|
138
146
|
answer: row.output, tool_calls: Array(row.tool_calls), duration_ms: row.duration_ms,
|
|
139
147
|
input_tokens: row.input_tokens, output_tokens: row.output_tokens,
|
|
140
|
-
cost: row.cost&.to_f, error: row.error_message
|
|
148
|
+
cost: row.cost&.to_f, error: row.error_message, metadata: row.replay_metadata
|
|
141
149
|
),
|
|
142
150
|
scores: row.scores.to_h, score: row.score, status: row.status,
|
|
143
|
-
diagnosis: row.
|
|
151
|
+
diagnosis: row.evaluation_diagnosis.presence
|
|
144
152
|
)
|
|
145
153
|
end
|
|
146
154
|
|
|
@@ -152,7 +160,7 @@ module ActionAgent
|
|
|
152
160
|
"agent" => evaluation.agent&.name,
|
|
153
161
|
"run" => id,
|
|
154
162
|
"finished" => completed_at&.iso8601
|
|
155
|
-
}.compact,
|
|
163
|
+
}.compact.merge(report_metadata),
|
|
156
164
|
verdict: recorded_verdict,
|
|
157
165
|
judge_label: judge_label,
|
|
158
166
|
tool_resolver: EvaluationToolResolver.new(evaluation.agent),
|
|
@@ -37,13 +37,32 @@ module ActionAgent
|
|
|
37
37
|
tool_calls.select { |call| call.is_a?(Hash) && (call["error"] || call[:error]) }
|
|
38
38
|
end
|
|
39
39
|
|
|
40
|
+
# Adapter-provided correlation and context are stored alongside diagnosis
|
|
41
|
+
# in a reserved JSON key, leaving the public diagnosis contract unchanged.
|
|
42
|
+
def replay_metadata
|
|
43
|
+
value = diagnosis&.dig("_replay_metadata")
|
|
44
|
+
value.is_a?(Hash) ? value : {}
|
|
45
|
+
end
|
|
46
|
+
|
|
47
|
+
def evaluation_diagnosis
|
|
48
|
+
(diagnosis || {}).except("_replay_metadata", "_scenario_snapshot")
|
|
49
|
+
end
|
|
50
|
+
|
|
51
|
+
# A catalog can be refreshed without changing what an earlier run asked
|
|
52
|
+
# or expected. Older results did not record this snapshot.
|
|
53
|
+
def evaluated_scenario
|
|
54
|
+
snapshot = diagnosis&.dig("_scenario_snapshot")
|
|
55
|
+
snapshot.is_a?(Hash) ? snapshot : scenario.as_json_summary.stringify_keys
|
|
56
|
+
end
|
|
57
|
+
|
|
40
58
|
def as_json_summary
|
|
41
59
|
{
|
|
42
60
|
id: id,
|
|
43
61
|
scenario_id: evaluation_scenario_id,
|
|
44
|
-
scenario_key:
|
|
45
|
-
group:
|
|
46
|
-
prompt:
|
|
62
|
+
scenario_key: evaluated_scenario["key"],
|
|
63
|
+
group: evaluated_scenario["group"],
|
|
64
|
+
prompt: evaluated_scenario["prompt"],
|
|
65
|
+
scenario: evaluated_scenario,
|
|
47
66
|
model: model,
|
|
48
67
|
provider: provider,
|
|
49
68
|
status: status,
|
|
@@ -57,7 +76,8 @@ module ActionAgent
|
|
|
57
76
|
cost: cost&.to_f,
|
|
58
77
|
fault: fault,
|
|
59
78
|
recommendation: recommendation,
|
|
60
|
-
diagnosis:
|
|
79
|
+
diagnosis: evaluation_diagnosis,
|
|
80
|
+
metadata: replay_metadata,
|
|
61
81
|
error_message: error_message,
|
|
62
82
|
agent_run_id: agent_run_id
|
|
63
83
|
}
|
|
@@ -184,7 +184,7 @@ module ActionAgent
|
|
|
184
184
|
end
|
|
185
185
|
end
|
|
186
186
|
|
|
187
|
-
def self.create_from_payload(trace, sdk_info = {}, account: nil)
|
|
187
|
+
def self.create_from_payload(trace, sdk_info = {}, account: nil, agent: nil)
|
|
188
188
|
spans = trace["spans"] || []
|
|
189
189
|
root_span = spans.find { |s| s["parent_span_id"].nil? } || spans.first || {}
|
|
190
190
|
|
|
@@ -238,9 +238,24 @@ module ActionAgent
|
|
|
238
238
|
# Add account if in multi-tenant mode
|
|
239
239
|
attrs[:account] = account if ActionAgent.multi_tenant? && account
|
|
240
240
|
|
|
241
|
+
# A trace the dashboard recorded for one of its own runs belongs to the
|
|
242
|
+
# agent it ran, which only the caller that ran it may name (+agent+,
|
|
243
|
+
# passed by AgentExecutionService). Attribute it up front: left to the
|
|
244
|
+
# registrar, the run's class and action match no authored record —
|
|
245
|
+
# those carry no service_name or agent_class_name — and every dashboard
|
|
246
|
+
# run would register an "observed" twin of the agent that produced it.
|
|
247
|
+
#
|
|
248
|
+
# Never taken from the trace itself. A payload's resource attributes are
|
|
249
|
+
# whatever the reporter sent, and ingest is unauthenticated on a
|
|
250
|
+
# single-tenant install with no ActionAgent.ingest_api_key, so trusting
|
|
251
|
+
# an id from there would let any reporter bind its traces to any
|
|
252
|
+
# dashboard-authored agent by guessing a primary key.
|
|
253
|
+
attrs[:agent_id] = agent&.id
|
|
254
|
+
|
|
241
255
|
create!(attrs).tap { |record| AgentRegistrar.call(record) }
|
|
242
256
|
end
|
|
243
257
|
|
|
258
|
+
|
|
244
259
|
# Sums a span's token counts (used to decide which spans carry the
|
|
245
260
|
# authoritative token data during ingestion).
|
|
246
261
|
#
|
|
@@ -15,6 +15,7 @@ module ActionAgent
|
|
|
15
15
|
tool_arguments: message.tool_arguments.presence,
|
|
16
16
|
tool_result: message.tool_result,
|
|
17
17
|
duration_ms: message.metadata&.dig("duration_ms"),
|
|
18
|
+
attachments: message.attachments || [],
|
|
18
19
|
content_checksum: message.content_checksum,
|
|
19
20
|
created_at: message.created_at.iso8601(3)
|
|
20
21
|
}
|
|
@@ -23,6 +23,24 @@ module ActionAgent
|
|
|
23
23
|
|
|
24
24
|
SERVICE_NAME = "activeagents-platform"
|
|
25
25
|
|
|
26
|
+
# Images and PDFs above this size are described rather than sent —
|
|
27
|
+
# a data URI of that size is most of a context window by itself.
|
|
28
|
+
ATTACHMENT_DATA_LIMIT = 8.megabytes
|
|
29
|
+
# Inlined text attachments are cut here: enough for a CSV or a config
|
|
30
|
+
# file, not enough for a log dump to crowd out the conversation.
|
|
31
|
+
ATTACHMENT_TEXT_LIMIT = 20_000
|
|
32
|
+
# ...and only this many bytes are ever read to produce those characters,
|
|
33
|
+
# so a multi-gigabyte log named .csv costs a fixed slice of memory rather
|
|
34
|
+
# than its whole size. Four bytes per character is UTF-8's worst case.
|
|
35
|
+
ATTACHMENT_TEXT_BYTE_LIMIT = ATTACHMENT_TEXT_LIMIT * 4
|
|
36
|
+
# The prompt span records the transcript, not the data URIs; keep the
|
|
37
|
+
# whole serialized list within the same budget as the other attributes.
|
|
38
|
+
PROMPT_SPAN_MESSAGE_LIMIT = 6000
|
|
39
|
+
# Prior turns sent with a pinned conversation: the most recent ones,
|
|
40
|
+
# trimmed oldest-first to a character budget.
|
|
41
|
+
HISTORY_TURN_LIMIT = 40
|
|
42
|
+
HISTORY_CHAR_BUDGET = 60_000
|
|
43
|
+
|
|
26
44
|
def self.call(agent_record, run)
|
|
27
45
|
new(agent_record, run).call
|
|
28
46
|
end
|
|
@@ -59,6 +77,7 @@ module ActionAgent
|
|
|
59
77
|
end
|
|
60
78
|
|
|
61
79
|
def call
|
|
80
|
+
@agent_record.ensure_executable!
|
|
62
81
|
root_span = @root_span = build_root_span
|
|
63
82
|
record_prompt_span(root_span)
|
|
64
83
|
llm_span = root_span.add_span(
|
|
@@ -124,7 +143,9 @@ module ActionAgent
|
|
|
124
143
|
# The outbound prompt as a span, in the SDK's attribute shape — gives the
|
|
125
144
|
# Traces UI its System/User conversation rows and lets the context-pressure
|
|
126
145
|
# meter attribute instructions and tool schemas instead of lumping the
|
|
127
|
-
# whole input into "messages".
|
|
146
|
+
# whole input into "messages". Messages are the text-only transcript:
|
|
147
|
+
# the same list the provider gets, with data URIs replaced by
|
|
148
|
+
# "[image: sales_chart.png]" placeholders.
|
|
128
149
|
def record_prompt_span(root_span)
|
|
129
150
|
span = root_span.add_span("agent.prompt", span_type: :prompt)
|
|
130
151
|
if composed_instructions.present?
|
|
@@ -133,16 +154,69 @@ module ActionAgent
|
|
|
133
154
|
if tool_schemas.present?
|
|
134
155
|
span.set_attribute("prompt.input.tools", tool_schemas.to_json.byteslice(0, 6000).to_s.scrub)
|
|
135
156
|
end
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
157
|
+
transcript = prompt_turn[:transcript].map do |message|
|
|
158
|
+
{ role: message[:role], content: message[:content].to_s.byteslice(0, 4000).to_s.scrub }
|
|
159
|
+
end
|
|
160
|
+
# A replayed conversation is re-sent every turn, so the span keeps the
|
|
161
|
+
# most recent messages that fit rather than the whole history again.
|
|
162
|
+
serialized = transcript.to_json
|
|
163
|
+
while serialized.bytesize > PROMPT_SPAN_MESSAGE_LIMIT && transcript.size > 1
|
|
164
|
+
transcript.shift
|
|
165
|
+
serialized = transcript.to_json
|
|
166
|
+
end
|
|
167
|
+
span.set_attribute("prompt.input.messages", serialized)
|
|
168
|
+
span.set_attribute("messages.count", transcript.size)
|
|
141
169
|
span.finish
|
|
142
170
|
rescue StandardError => e
|
|
143
171
|
Rails.logger.warn("[AgentExecutionService] prompt span failed: #{e.message}")
|
|
144
172
|
end
|
|
145
173
|
|
|
174
|
+
# The list handed to prompt(messages:): the pinned conversation's prior
|
|
175
|
+
# turns, then the new user turn carrying the run's attachments — images
|
|
176
|
+
# and PDFs as data URIs in the provider-neutral {text:, image:} /
|
|
177
|
+
# {document:} shorthand, text files inlined, anything else described.
|
|
178
|
+
# Memoized, so the provider and the prompt span see one list.
|
|
179
|
+
def prompt_messages
|
|
180
|
+
prompt_turn[:messages]
|
|
181
|
+
end
|
|
182
|
+
|
|
183
|
+
# Persists the tool calls this service executed when the provider's
|
|
184
|
+
# response carries no tool-role messages to persist them from (the
|
|
185
|
+
# OpenAI Responses API). With such messages present the usual path
|
|
186
|
+
# — solid_agent's, then #persist_tool_messages — already writes the
|
|
187
|
+
# rows, keyed by tool_call_id, and this is a no-op.
|
|
188
|
+
def persist_tool_invocations(context, response)
|
|
189
|
+
return unless context.respond_to?(:add_tool_message)
|
|
190
|
+
return if @tool_invocations.empty?
|
|
191
|
+
return if Array(response.respond_to?(:messages) ? response.messages : nil).any? do |message|
|
|
192
|
+
message.respond_to?(:role) && message.role.to_s == "tool"
|
|
193
|
+
end
|
|
194
|
+
|
|
195
|
+
@tool_invocations.each do |invocation|
|
|
196
|
+
context.add_tool_message(
|
|
197
|
+
tool_call_id: nil,
|
|
198
|
+
tool_name: invocation[:name],
|
|
199
|
+
result: invocation[:result],
|
|
200
|
+
arguments: invocation[:arguments],
|
|
201
|
+
duration_ms: invocation[:duration_ms]
|
|
202
|
+
)
|
|
203
|
+
end
|
|
204
|
+
rescue StandardError => e
|
|
205
|
+
Rails.logger.error("[AgentExecutionService] Failed to persist tool invocations: #{e.message}")
|
|
206
|
+
end
|
|
207
|
+
|
|
208
|
+
# The person's own words for this turn — what the persisted user
|
|
209
|
+
# message says, without the file bodies inlined for the model.
|
|
210
|
+
def user_text
|
|
211
|
+
@run.input_prompt.to_s.presence || (attachment_records.any? ? "(see attached files)" : "")
|
|
212
|
+
end
|
|
213
|
+
|
|
214
|
+
# What was attached, as stored on the persisted user message so the
|
|
215
|
+
# conversation shows the files afterwards.
|
|
216
|
+
def attachment_manifest
|
|
217
|
+
@attachment_manifest ||= @run.attachment_manifest
|
|
218
|
+
end
|
|
219
|
+
|
|
146
220
|
# Per-run provider/model overrides (input_params) let callers replay the
|
|
147
221
|
# same agent under a different model — the basis of evaluation comparison
|
|
148
222
|
# runs. Absent overrides, the agent's own configuration applies.
|
|
@@ -233,7 +307,9 @@ module ActionAgent
|
|
|
233
307
|
when "call_agent"
|
|
234
308
|
call_agent(slug: kwargs[:slug], message: kwargs[:message])
|
|
235
309
|
else
|
|
236
|
-
|
|
310
|
+
# A tool one of the agent's own MCP servers serves is called there;
|
|
311
|
+
# AgentToolbox answers the rest.
|
|
312
|
+
mcp_dispatcher.call(name, kwargs) || AgentToolbox.call(name, **kwargs)
|
|
237
313
|
end
|
|
238
314
|
rescue StandardError => e
|
|
239
315
|
Rails.logger.warn("[AgentExecutionService] Tool #{name} failed: #{e.class} - #{e.message}")
|
|
@@ -262,6 +338,7 @@ module ActionAgent
|
|
|
262
338
|
@tool_invocations << {
|
|
263
339
|
name: name.to_s,
|
|
264
340
|
arguments: kwargs,
|
|
341
|
+
result: result,
|
|
265
342
|
duration_ms: duration_ms,
|
|
266
343
|
error: errored
|
|
267
344
|
}
|
|
@@ -279,6 +356,12 @@ module ActionAgent
|
|
|
279
356
|
# Executes another agent of the same account synchronously and returns
|
|
280
357
|
# its reply, so agents can delegate to each other as a tool call. The
|
|
281
358
|
# sub-run is a real AgentRun with its own trace.
|
|
359
|
+
# One dispatcher per run, so every tool call shares the MCP sessions the
|
|
360
|
+
# first call opens.
|
|
361
|
+
def mcp_dispatcher
|
|
362
|
+
@mcp_dispatcher ||= MCPToolDispatcher.new(@agent_record)
|
|
363
|
+
end
|
|
364
|
+
|
|
282
365
|
def call_agent(slug:, message:)
|
|
283
366
|
depth = Thread.current[:agent_call_depth].to_i
|
|
284
367
|
return { error: "call_agent depth limit (#{MAX_CALL_DEPTH}) reached" } if depth >= MAX_CALL_DEPTH
|
|
@@ -341,7 +424,10 @@ module ActionAgent
|
|
|
341
424
|
model_options.merge!(owner_provider_options(effective_provider))
|
|
342
425
|
klass_name = agent_class_name
|
|
343
426
|
agent_record = @agent_record
|
|
344
|
-
|
|
427
|
+
# Pinned, or the default stream resolved here rather than left to
|
|
428
|
+
# solid_agent's unordered find_or_create_by!, which cannot tell the
|
|
429
|
+
# agent's original conversation from a later one on the same triple.
|
|
430
|
+
pinned = conversation_context
|
|
345
431
|
instructions = composed_instructions
|
|
346
432
|
action = action_name
|
|
347
433
|
run_trace_id = trace_id
|
|
@@ -395,19 +481,52 @@ module ActionAgent
|
|
|
395
481
|
|
|
396
482
|
# One method per invokable action (the default plus each named action
|
|
397
483
|
# prompt) — solid_agent keys the persisted context by action_name, so
|
|
398
|
-
# each action gets its own interaction stream.
|
|
484
|
+
# each action gets its own interaction stream. A run pinned to a
|
|
485
|
+
# conversation continues that context instead.
|
|
399
486
|
define_method action do
|
|
400
487
|
# Thread the run's telemetry trace_id through prompt_options so
|
|
401
488
|
# SolidAgent's provenance (and AgentContext#record_generation_with_
|
|
402
489
|
# provenance!) can correlate the persisted generation with its trace.
|
|
403
490
|
prompt_options[:trace_id] = run_trace_id
|
|
404
|
-
|
|
491
|
+
if pinned
|
|
492
|
+
load_context(context_id: pinned.id)
|
|
493
|
+
else
|
|
494
|
+
load_context(contextable: agent_record)
|
|
495
|
+
end
|
|
405
496
|
|
|
406
|
-
options = {
|
|
497
|
+
options = { messages: service.prompt_messages }
|
|
407
498
|
options[:instructions] = instructions if instructions.present?
|
|
408
499
|
options[:tools] = tool_definitions if tool_definitions.present?
|
|
409
500
|
prompt(**options)
|
|
410
501
|
end
|
|
502
|
+
|
|
503
|
+
# solid_agent's after_prompt callback persists the last prompt
|
|
504
|
+
# message's content: string — so a turn that carries files (a
|
|
505
|
+
# {text:, image:} hash) would never be written, and the history
|
|
506
|
+
# replayed from the pinned context is not this run's to persist.
|
|
507
|
+
# Instead: exactly one user message per run, through the agent-level
|
|
508
|
+
# add_user_message that stamps provenance (its trace_id is how the
|
|
509
|
+
# run detail API finds the run's slice of the conversation), with
|
|
510
|
+
# the attachment manifest alongside.
|
|
511
|
+
define_method(:persist_prompt_to_context) do
|
|
512
|
+
text = service.user_text
|
|
513
|
+
return unless context && text.present?
|
|
514
|
+
|
|
515
|
+
add_user_message(text, attachments: service.attachment_manifest)
|
|
516
|
+
end
|
|
517
|
+
private :persist_prompt_to_context
|
|
518
|
+
|
|
519
|
+
# solid_agent persists the tool exchange from the response's
|
|
520
|
+
# tool-role messages. The Responses API carries function calls as
|
|
521
|
+
# items rather than messages, so that list is empty and the
|
|
522
|
+
# exchange — a render_ui call is the reply — would vanish from the
|
|
523
|
+
# conversation. The service saw every call go by; fall back to its
|
|
524
|
+
# own records, here so the rows land before the assistant turn.
|
|
525
|
+
define_method(:persist_tool_messages_to_context) do
|
|
526
|
+
super()
|
|
527
|
+
service.persist_tool_invocations(context, generation_response)
|
|
528
|
+
end
|
|
529
|
+
private :persist_tool_messages_to_context
|
|
411
530
|
end
|
|
412
531
|
|
|
413
532
|
agent_class.public_send(action).generate_now
|
|
@@ -419,7 +538,11 @@ module ActionAgent
|
|
|
419
538
|
def tool_schemas
|
|
420
539
|
return [] if provider == :mock
|
|
421
540
|
|
|
422
|
-
|
|
541
|
+
# The agent's own MCP servers describe their tools; the toolbox describes
|
|
542
|
+
# the rest. Without the first half a tool the agent declares is never
|
|
543
|
+
# offered to the model, which then answers from memory instead of calling
|
|
544
|
+
# it.
|
|
545
|
+
mcp_dispatcher.tool_definitions + AgentToolbox.definitions_for(@agent_record.tools)
|
|
423
546
|
end
|
|
424
547
|
|
|
425
548
|
# Persists the tool interaction stream to the solid_agent conversation
|
|
@@ -530,10 +653,165 @@ module ActionAgent
|
|
|
530
653
|
@trace_id ||= @run.trace_id.presence || SecureRandom.hex(16)
|
|
531
654
|
end
|
|
532
655
|
|
|
533
|
-
# The solid_agent conversation context this execution persisted into
|
|
534
|
-
#
|
|
656
|
+
# The solid_agent conversation context this execution persisted into:
|
|
657
|
+
# the pinned one when the run continues a conversation, else the
|
|
658
|
+
# agent + action's default stream.
|
|
535
659
|
def conversation_context
|
|
536
|
-
|
|
660
|
+
pinned_context || default_stream_context
|
|
661
|
+
end
|
|
662
|
+
|
|
663
|
+
# The agent + action's default stream: the oldest context on the triple
|
|
664
|
+
# solid_agent keys by, so a second conversation the dashboard started for
|
|
665
|
+
# the same action cannot become the row an unpinned run appends to.
|
|
666
|
+
def default_stream_context
|
|
667
|
+
AgentContext.where(contextable: @agent_record, agent_name: agent_class_name, action_name: action_name)
|
|
668
|
+
.order(:id).first
|
|
669
|
+
end
|
|
670
|
+
|
|
671
|
+
# The conversation the run was pinned to (input_params context_id). A
|
|
672
|
+
# context belonging to another agent — or recorded under another action,
|
|
673
|
+
# whose instructions and stream are not this run's — is ignored rather
|
|
674
|
+
# than continued: the run falls back to the default stream as if nothing
|
|
675
|
+
# had been pinned, and reports the conversation it actually wrote to.
|
|
676
|
+
def pinned_context
|
|
677
|
+
return @pinned_context if defined?(@pinned_context)
|
|
678
|
+
|
|
679
|
+
id = run_params[:context_id]
|
|
680
|
+
@pinned_context =
|
|
681
|
+
if id.present?
|
|
682
|
+
# Matched on the action too, not just ownership: a run for another
|
|
683
|
+
# action would append to this conversation and rewrite the recorded
|
|
684
|
+
# instructions with its own. The agent_name is deliberately not part
|
|
685
|
+
# of it — renaming an agent changes that string, and the
|
|
686
|
+
# conversations it already has must stay pinnable.
|
|
687
|
+
AgentContext.find_by(id: id, contextable: @agent_record, action_name: action_name)
|
|
688
|
+
end
|
|
689
|
+
end
|
|
690
|
+
|
|
691
|
+
# The pinned conversation's prior turns as plain {role:, content:}
|
|
692
|
+
# messages — the conversation as the person saw it: tool rows and empty
|
|
693
|
+
# assistant rows (a turn that only carried a tool call) are skipped.
|
|
694
|
+
# The most recent turns, dropped oldest-first once the budget is spent.
|
|
695
|
+
def history_messages
|
|
696
|
+
context = pinned_context
|
|
697
|
+
return [] unless context
|
|
698
|
+
|
|
699
|
+
turns = context.messages.chronological
|
|
700
|
+
.where(role: %w[user assistant])
|
|
701
|
+
.where.not(content: [ nil, "" ])
|
|
702
|
+
.last(HISTORY_TURN_LIMIT)
|
|
703
|
+
|
|
704
|
+
budget = HISTORY_CHAR_BUDGET
|
|
705
|
+
kept = turns.reverse_each.with_object([]) do |message, collected|
|
|
706
|
+
content = message.content.to_s
|
|
707
|
+
break collected if content.length > budget
|
|
708
|
+
|
|
709
|
+
budget -= content.length
|
|
710
|
+
collected.unshift(role: message.role, content: content)
|
|
711
|
+
end
|
|
712
|
+
|
|
713
|
+
# Neither cut lands on a turn boundary, so the oldest survivor can be an
|
|
714
|
+
# assistant reply whose question was dropped. Anthropic rejects a
|
|
715
|
+
# conversation that opens on one, and every provider reads it as an
|
|
716
|
+
# answer to nothing.
|
|
717
|
+
kept.shift while kept.first && kept.first[:role] != "user"
|
|
718
|
+
kept
|
|
719
|
+
end
|
|
720
|
+
|
|
721
|
+
# Builds the message list and, in the same pass, its text-only
|
|
722
|
+
# transcript for the prompt span (data URIs are too big to trace).
|
|
723
|
+
#
|
|
724
|
+
# The first image or document rides on the user's text as a
|
|
725
|
+
# {text:, image:} / {text:, document:} message; each further one is a
|
|
726
|
+
# message of its own, since the shorthand carries one part per key.
|
|
727
|
+
def prompt_turn
|
|
728
|
+
@prompt_turn ||= begin
|
|
729
|
+
# dup: the inlined file bodies must not land on the run's own
|
|
730
|
+
# input_prompt through in-place mutation.
|
|
731
|
+
text = user_text.dup
|
|
732
|
+
media = []
|
|
733
|
+
|
|
734
|
+
attachment_records.each do |attachment|
|
|
735
|
+
blob = attachment.blob
|
|
736
|
+
filename = blob.filename.to_s
|
|
737
|
+
descriptor = "#{filename} (#{blob.content_type}, #{human_size(blob.byte_size)})"
|
|
738
|
+
|
|
739
|
+
# A file the storage service can no longer produce costs the file,
|
|
740
|
+
# not the run: every branch below degrades to the same descriptor
|
|
741
|
+
# the unsupported kinds get.
|
|
742
|
+
begin
|
|
743
|
+
case AgentRun.attachment_kind(blob.content_type, filename)
|
|
744
|
+
when "text"
|
|
745
|
+
body = text_prefix(blob)
|
|
746
|
+
suffix = body.bytesize < blob.byte_size ? "\n… (truncated)" : ""
|
|
747
|
+
text << "\n\n[Attached file: #{descriptor}]\n```\n#{body}#{suffix}\n```"
|
|
748
|
+
when "image", "document"
|
|
749
|
+
if blob.byte_size > ATTACHMENT_DATA_LIMIT
|
|
750
|
+
text << "\n\n[Attached file: #{descriptor} — not sent to the model]"
|
|
751
|
+
else
|
|
752
|
+
key = blob.content_type.to_s.start_with?("image/") ? :image : :document
|
|
753
|
+
media << { key => data_uri(blob), label: "[#{key}: #{filename}]" }
|
|
754
|
+
end
|
|
755
|
+
else
|
|
756
|
+
text << "\n\n[Attached file: #{descriptor} — not sent to the model]"
|
|
757
|
+
end
|
|
758
|
+
rescue StandardError => e
|
|
759
|
+
Rails.logger.warn("[AgentExecutionService] attachment #{filename} unreadable: #{e.message}")
|
|
760
|
+
text << "\n\n[Attached file: #{descriptor} — not sent to the model]"
|
|
761
|
+
end
|
|
762
|
+
end
|
|
763
|
+
|
|
764
|
+
first, *rest = media
|
|
765
|
+
history = history_messages
|
|
766
|
+
# No prompt and no files sends no turn at all, which leaves the
|
|
767
|
+
# gem's template fallback in charge exactly as before.
|
|
768
|
+
turn =
|
|
769
|
+
if first
|
|
770
|
+
{ role: "user", text: text }.merge(first.except(:label))
|
|
771
|
+
elsif text.present?
|
|
772
|
+
{ role: "user", content: text }
|
|
773
|
+
end
|
|
774
|
+
{
|
|
775
|
+
messages: history + [ turn ].compact + rest.map { |item| { role: "user" }.merge(item.except(:label)) },
|
|
776
|
+
transcript: history +
|
|
777
|
+
[ turn && { role: "user", content: [ text, first&.dig(:label) ].compact.join("\n") } ].compact +
|
|
778
|
+
rest.map { |item| { role: "user", content: item[:label] } }
|
|
779
|
+
}
|
|
780
|
+
end
|
|
781
|
+
end
|
|
782
|
+
|
|
783
|
+
def attachment_records
|
|
784
|
+
@attachment_records ||= AgentRun.attachments_available? ? @run.attachments_attachments.includes(:blob).order(:id).to_a : []
|
|
785
|
+
end
|
|
786
|
+
|
|
787
|
+
def data_uri(blob)
|
|
788
|
+
"data:#{blob.content_type};base64,#{Base64.strict_encode64(blob.download)}"
|
|
789
|
+
end
|
|
790
|
+
|
|
791
|
+
# The head of a text attachment, reading a bounded number of bytes: only
|
|
792
|
+
# ATTACHMENT_TEXT_LIMIT characters are ever sent, so a huge file must not
|
|
793
|
+
# be materialised whole to produce them. The byte prefix can split a
|
|
794
|
+
# multibyte character, which scrub removes.
|
|
795
|
+
def text_prefix(blob)
|
|
796
|
+
bytes =
|
|
797
|
+
if blob.byte_size <= ATTACHMENT_TEXT_BYTE_LIMIT
|
|
798
|
+
blob.download
|
|
799
|
+
elsif blob.service.respond_to?(:download_chunk)
|
|
800
|
+
blob.service.download_chunk(blob.key, 0...ATTACHMENT_TEXT_BYTE_LIMIT)
|
|
801
|
+
else
|
|
802
|
+
buffer = +""
|
|
803
|
+
blob.download do |chunk|
|
|
804
|
+
buffer << chunk
|
|
805
|
+
break if buffer.bytesize >= ATTACHMENT_TEXT_BYTE_LIMIT
|
|
806
|
+
end
|
|
807
|
+
buffer
|
|
808
|
+
end
|
|
809
|
+
|
|
810
|
+
bytes.to_s.dup.force_encoding(Encoding::UTF_8).scrub[0, ATTACHMENT_TEXT_LIMIT]
|
|
811
|
+
end
|
|
812
|
+
|
|
813
|
+
def human_size(bytes)
|
|
814
|
+
ActiveSupport::NumberHelper.number_to_human_size(bytes, precision: 2)
|
|
537
815
|
end
|
|
538
816
|
|
|
539
817
|
# The agent's owner under the configured mode; nil when the install
|
|
@@ -563,7 +841,7 @@ module ActionAgent
|
|
|
563
841
|
tenant = ActionAgent.tenant_for(owner)
|
|
564
842
|
return if trace_model.for_account(tenant).exists?(trace_id: root_span.trace_id)
|
|
565
843
|
|
|
566
|
-
trace_model.create_from_payload(payload, sdk_info, account: tenant)
|
|
844
|
+
trace_model.create_from_payload(payload, sdk_info, account: tenant, agent: @agent_record)
|
|
567
845
|
rescue StandardError => e
|
|
568
846
|
Rails.logger.error("[AgentExecutionService] Failed to record trace #{root_span.trace_id}: #{e.class} - #{e.message}")
|
|
569
847
|
nil
|