actionagent 1.9.2 → 1.9.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +23 -0
- data/app/assets/builds/action_agent.css +1 -1
- data/app/assets/builds/action_agent.js +61 -61
- data/app/controllers/action_agent/api/agents_controller.rb +23 -0
- data/app/controllers/action_agent/api/code_sessions_controller.rb +49 -10
- data/app/controllers/action_agent/api/evaluation_reports_controller.rb +66 -2
- data/app/controllers/action_agent/api/evaluations_controller.rb +41 -3
- data/app/controllers/action_agent/api/interactions_controller.rb +55 -1
- data/app/controllers/action_agent/api/sandboxes_controller.rb +13 -2
- data/app/controllers/action_agent/api/trace_reports_controller.rb +11 -6
- data/app/jobs/action_agent/code_session_job.rb +19 -2
- data/app/models/action_agent/agent.rb +54 -1
- data/app/models/action_agent/code_session.rb +80 -0
- data/app/models/action_agent/evaluation_run.rb +17 -2
- data/app/models/action_agent/provider_key.rb +5 -1
- data/app/queries/action_agent/trace_window_report.rb +60 -0
- data/app/serializers/action_agent/code_conversation_serializer.rb +86 -0
- data/app/serializers/action_agent/evaluation_serializer.rb +13 -2
- data/app/services/action_agent/agent_execution_service.rb +37 -0
- data/app/services/action_agent/agent_tool_roster.rb +53 -18
- data/app/services/action_agent/agent_toolbox.rb +156 -3
- data/app/services/action_agent/browser_client_token.rb +86 -0
- data/app/services/action_agent/checkout_tools.rb +154 -0
- data/app/services/action_agent/code_conversation.rb +133 -0
- data/app/services/action_agent/codex_auth.rb +81 -0
- data/app/services/action_agent/dashboard_assistant_service.rb +19 -1
- data/app/services/action_agent/diff_stat.rb +27 -0
- data/app/services/action_agent/evaluation_report_export.rb +175 -0
- data/app/services/action_agent/evaluation_report_import.rb +40 -17
- data/app/services/action_agent/evaluation_runner_service.rb +48 -12
- data/app/services/action_agent/local_sandbox_backend.rb +295 -30
- data/app/services/action_agent/mcp_catalog.rb +25 -0
- data/app/services/action_agent/mcp_client.rb +1 -0
- data/app/services/action_agent/mcp_tool_dispatcher.rb +36 -1
- data/app/services/action_agent/project_boot_fix.rb +10 -3
- data/app/services/action_agent/sandbox_orchestrator.rb +41 -1
- data/app/services/action_agent/scenario_evaluation_runner.rb +1 -0
- data/app/services/action_agent/slack_tools.rb +101 -0
- data/config/routes.rb +10 -1
- data/lib/action_agent/engine.rb +6 -0
- data/lib/action_agent/version.rb +1 -1
- data/lib/action_agent.rb +35 -2
- data/lib/generators/action_agent/templates/migrations/028_add_conversations_to_code_sessions.rb.erb +18 -0
- data/lib/tasks/evaluations.rake +78 -0
- metadata +13 -2
|
@@ -6,6 +6,12 @@ module ActionAgent
|
|
|
6
6
|
# outcome Claude Code reported, and the diff the session left in the
|
|
7
7
|
# working tree.
|
|
8
8
|
#
|
|
9
|
+
# A session is one turn of a conversation (CodeConversation). The first
|
|
10
|
+
# turn is the conversation's root; each later one names it
|
|
11
|
+
# (conversation_root_id), runs after the one before it, and resumes the same
|
|
12
|
+
# CLI session (resume_session_id), so Claude Code or Codex remembers what
|
|
13
|
+
# came before. Every existing session is a conversation of one turn.
|
|
14
|
+
#
|
|
9
15
|
# The transcript is scrubbed of the sandbox's secrets (its GitHub token and
|
|
10
16
|
# Claude Code credential) before it is stored, and bounded: long strings
|
|
11
17
|
# are truncated and events past MAX_EVENTS are counted rather than kept.
|
|
@@ -17,6 +23,9 @@ module ActionAgent
|
|
|
17
23
|
belongs_to :evaluation_run, optional: true
|
|
18
24
|
belongs_to :verification_run, class_name: "EvaluationRun", optional: true
|
|
19
25
|
belongs_to :previous_code_session, class_name: "CodeSession", optional: true
|
|
26
|
+
belongs_to :conversation_root, class_name: "CodeSession", optional: true
|
|
27
|
+
has_many :follow_up_turns, -> { order(:turn_index, :id) }, class_name: "CodeSession",
|
|
28
|
+
foreign_key: :conversation_root_id, inverse_of: :conversation_root
|
|
20
29
|
|
|
21
30
|
enum :status, { queued: 0, running: 1, succeeded: 2, failed: 3, cancelled: 4 }
|
|
22
31
|
|
|
@@ -26,11 +35,32 @@ module ActionAgent
|
|
|
26
35
|
MAX_EVENT_STRING = 4_000
|
|
27
36
|
MAX_DIFF_BYTES = 500_000
|
|
28
37
|
RUNNERS = %w[claude_code codex].freeze
|
|
38
|
+
# Where a conversation was started. Older sessions recorded none; see
|
|
39
|
+
# #origin.
|
|
40
|
+
ORIGINS = %w[settings boot_fix evaluation_fix chat].freeze
|
|
41
|
+
# A Claude Code session id, as --session-id and --resume take it.
|
|
42
|
+
SESSION_UUID = /\A\h{8}-\h{4}-\h{4}-\h{4}-\h{12}\z/
|
|
43
|
+
# A note from the dashboard is fenced in this tag ahead of the person's
|
|
44
|
+
# message, and the tag is taken out of anything the person typed.
|
|
45
|
+
NOTE_TAG = "dashboard-note"
|
|
29
46
|
|
|
30
47
|
validates :prompt, presence: true, length: { maximum: MAX_PROMPT_CHARACTERS }
|
|
31
48
|
validates :runner, inclusion: { in: RUNNERS }
|
|
32
49
|
|
|
50
|
+
validates :origin, inclusion: { in: ORIGINS }, allow_nil: true
|
|
51
|
+
# A Codex conversation resumes the thread id its first turn reported.
|
|
52
|
+
validates :resume_session_id, format: { with: SESSION_UUID }, allow_nil: true, if: -> { runner == "claude_code" }
|
|
53
|
+
|
|
54
|
+
# A conversation's first Claude Code turn names the CLI session it
|
|
55
|
+
# starts (--session-id), so every later turn can resume it even when the
|
|
56
|
+
# first ended before reporting its own id.
|
|
57
|
+
before_validation(on: :create) do
|
|
58
|
+
self.resume_session_id ||= SecureRandom.uuid if root? && runner == "claude_code"
|
|
59
|
+
end
|
|
60
|
+
|
|
33
61
|
scope :recent, -> { order(created_at: :desc) }
|
|
62
|
+
# The first turn of every conversation.
|
|
63
|
+
scope :conversation_roots, -> { where(conversation_root_id: nil) }
|
|
34
64
|
|
|
35
65
|
# JSON columns carry no default on MySQL or SQLite (see the migration).
|
|
36
66
|
def events
|
|
@@ -41,6 +71,51 @@ module ActionAgent
|
|
|
41
71
|
succeeded? || failed? || cancelled?
|
|
42
72
|
end
|
|
43
73
|
|
|
74
|
+
def root?
|
|
75
|
+
conversation_root_id.nil?
|
|
76
|
+
end
|
|
77
|
+
|
|
78
|
+
# The conversation's first turn: this one, or the one it names.
|
|
79
|
+
def root
|
|
80
|
+
root? ? self : conversation_root
|
|
81
|
+
end
|
|
82
|
+
|
|
83
|
+
# Every turn of this session's conversation, in order.
|
|
84
|
+
#
|
|
85
|
+
# @return [Array<CodeSession>]
|
|
86
|
+
def turns
|
|
87
|
+
# Read afresh: turns are added under the sandbox's lock while this
|
|
88
|
+
# session may hold an older list.
|
|
89
|
+
[ root, *CodeSession.where(conversation_root_id: root.id).order(:turn_index, :id).to_a ]
|
|
90
|
+
end
|
|
91
|
+
|
|
92
|
+
# Where the conversation was started. Sessions from before origins were
|
|
93
|
+
# recorded are told apart by what they ran for: an evaluation fix names
|
|
94
|
+
# its run, and a boot fix is one its project recorded
|
|
95
|
+
# (ProjectBootFix.project_for).
|
|
96
|
+
def origin
|
|
97
|
+
recorded = super.presence
|
|
98
|
+
return recorded if recorded
|
|
99
|
+
return "evaluation_fix" if evaluation_run_id
|
|
100
|
+
return "boot_fix" if persisted? && root? && ProjectBootFix.project_for(self)
|
|
101
|
+
|
|
102
|
+
"settings"
|
|
103
|
+
end
|
|
104
|
+
|
|
105
|
+
# What the CLI is given on stdin: the dashboard's note, fenced, ahead of
|
|
106
|
+
# the person's message. The prompt itself stays what they typed.
|
|
107
|
+
def stdin_text
|
|
108
|
+
return prompt if context_note.blank?
|
|
109
|
+
|
|
110
|
+
"<#{NOTE_TAG}>\n#{context_note}\n</#{NOTE_TAG}>\n\n#{prompt}"
|
|
111
|
+
end
|
|
112
|
+
|
|
113
|
+
# +text+ without the dashboard's note tags, so a person cannot make their
|
|
114
|
+
# own words read as the dashboard's.
|
|
115
|
+
def self.strip_note_tags(text)
|
|
116
|
+
text.to_s.gsub(%r{</?\s*#{NOTE_TAG}[^>]*>}i, "")
|
|
117
|
+
end
|
|
118
|
+
|
|
44
119
|
# Whether an outcome and a diff may still be recorded for this session.
|
|
45
120
|
# finished_at is set once nothing more will be: by CodeSessionJob when it
|
|
46
121
|
# settles a session it ran (whatever the backend returned, raised or
|
|
@@ -111,6 +186,10 @@ module ActionAgent
|
|
|
111
186
|
evaluation_run_id: evaluation_run_id,
|
|
112
187
|
evaluation_id: evaluation_run&.evaluation_id,
|
|
113
188
|
fix_item: fix_item,
|
|
189
|
+
conversation_id: root? ? id : conversation_root_id,
|
|
190
|
+
turn_index: turn_index,
|
|
191
|
+
origin: origin,
|
|
192
|
+
context_note: context_note,
|
|
114
193
|
verification_run_id: verification_run_id,
|
|
115
194
|
verification_error: verification_error,
|
|
116
195
|
result: result,
|
|
@@ -121,6 +200,7 @@ module ActionAgent
|
|
|
121
200
|
input_tokens: input_tokens,
|
|
122
201
|
output_tokens: output_tokens,
|
|
123
202
|
event_count: events.size + dropped_events_count.to_i,
|
|
203
|
+
diff_stat: finished? ? DiffStat.call(diff) : nil,
|
|
124
204
|
# What a client polls until: false once nothing about the session
|
|
125
205
|
# changes any more.
|
|
126
206
|
diff_pending: diff_pending?,
|
|
@@ -200,6 +200,16 @@ module ActionAgent
|
|
|
200
200
|
cost_breakdown.usage
|
|
201
201
|
end
|
|
202
202
|
|
|
203
|
+
# The judge's run-level spend as recorded, without pricing anything:
|
|
204
|
+
# what the publishing application reported for the calls no result owns,
|
|
205
|
+
# else the engine's meter, else nil.
|
|
206
|
+
def recorded_judge_usage_run
|
|
207
|
+
reported = scores&.dig("_judge_usage_run")
|
|
208
|
+
return reported if reported.is_a?(Hash)
|
|
209
|
+
|
|
210
|
+
judge_usage_meter&.merge("source" => "meter")
|
|
211
|
+
end
|
|
212
|
+
|
|
203
213
|
# Route templates for the report's fix item actions, relative to the
|
|
204
214
|
# dashboard mount: `%{key}` is filled in per MCP server by the report.
|
|
205
215
|
# The JSON API leaves `mount` empty — the React app resolves paths
|
|
@@ -238,7 +248,11 @@ module ActionAgent
|
|
|
238
248
|
# judge's run-level spend, so the page shows every cost the dashboard
|
|
239
249
|
# does. `estimate: false` rebuilds the report from what was recorded
|
|
240
250
|
# alone — the import summarizes a published run that way, so the
|
|
241
|
-
# summaries it stores stay the application's own figures
|
|
251
|
+
# summaries it stores stay the application's own figures, and the
|
|
252
|
+
# export (EvaluationReportExport) serves it that way, so the same run
|
|
253
|
+
# always exports the same report. The judge's run-level spend is then
|
|
254
|
+
# the one recorded: the application's for an imported run, the meter's
|
|
255
|
+
# for one the engine judged.
|
|
242
256
|
#
|
|
243
257
|
# The engine runs against every activeagent since 1.4, so the report
|
|
244
258
|
# kwargs that arrived later are passed only when Report.new takes them.
|
|
@@ -286,7 +300,8 @@ module ActionAgent
|
|
|
286
300
|
agent_name: evaluation.agent&.name,
|
|
287
301
|
links: links
|
|
288
302
|
}
|
|
289
|
-
|
|
303
|
+
judge_usage_run = breakdown ? breakdown.judge_usage_run : recorded_judge_usage_run
|
|
304
|
+
kwargs[:judge_usage] = judge_usage_run if judge_usage_run && self.class.report_accepts?(:judge_usage)
|
|
290
305
|
kwargs[:release] = release_summary if release_summary && self.class.report_accepts?(:release)
|
|
291
306
|
kwargs[:comparison] = recorded_comparison if recorded_comparison && self.class.report_accepts?(:comparison)
|
|
292
307
|
self.class.report_class.new(**kwargs)
|
|
@@ -39,7 +39,8 @@ module ActionAgent
|
|
|
39
39
|
HOST_PROVIDERS = %w[ollama].freeze
|
|
40
40
|
# Tools connected with a credential, configured beside the providers
|
|
41
41
|
# (Settings -> Integrations) but never offered to the agent builder.
|
|
42
|
-
|
|
42
|
+
# Slack's is the bot token the agents' slack tools post with (SlackTools).
|
|
43
|
+
CONNECTION_PROVIDERS = %w[claude_code codex slack].freeze
|
|
43
44
|
PROVIDERS = (KEY_PROVIDERS + HOST_PROVIDERS + CONNECTION_PROVIDERS).freeze
|
|
44
45
|
|
|
45
46
|
# Only an Anthropic API key (sk-ant-api03-…, from the Claude Console or
|
|
@@ -52,6 +53,7 @@ module ActionAgent
|
|
|
52
53
|
# dashboard never touches the credential.
|
|
53
54
|
CLAUDE_CODE_CREDENTIAL = /\Ask-ant-api\d{2}-[A-Za-z0-9_-]+\z/
|
|
54
55
|
CODEX_CREDENTIAL = /\Ask-(?!ant-|or-)[A-Za-z0-9_-]+\z/
|
|
56
|
+
SLACK_CREDENTIAL = /\Axoxb-[A-Za-z0-9-]+\z/
|
|
55
57
|
# A Claude subscription token, as earlier versions stored. Recognized so
|
|
56
58
|
# a stored one is never handed out (see #needs_replacing?).
|
|
57
59
|
SUBSCRIPTION_TOKEN_PREFIX = "sk-ant-oat"
|
|
@@ -95,6 +97,8 @@ module ActionAgent
|
|
|
95
97
|
}, if: -> { provider == "claude_code" }
|
|
96
98
|
validates :credential, format: { with: CODEX_CREDENTIAL, message: "must be an OpenAI API key (sk-…)" },
|
|
97
99
|
if: -> { provider == "codex" }
|
|
100
|
+
validates :credential, format: { with: SLACK_CREDENTIAL, message: "must be a Slack bot token (xoxb-…)" },
|
|
101
|
+
if: -> { provider == "slack" }
|
|
98
102
|
validates :api_key, length: { maximum: 500 }, allow_nil: true
|
|
99
103
|
|
|
100
104
|
# Deletes every Claude Code connection that still holds a Claude
|
|
@@ -0,0 +1,60 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module ActionAgent
|
|
4
|
+
# Full-window counts and chart buckets, independent of the capped trace
|
|
5
|
+
# details. Read only scalar columns, never the potentially large spans.
|
|
6
|
+
class TraceWindowReport
|
|
7
|
+
MAX_BUCKETS = 180
|
|
8
|
+
|
|
9
|
+
def initialize(scope, from:, to:, bucket_seconds:)
|
|
10
|
+
@scope, @from, @to = scope, from, to
|
|
11
|
+
duration = (to - from).ceil
|
|
12
|
+
@bucket_seconds = bucket_seconds.to_i.clamp([ (duration.to_f / MAX_BUCKETS).ceil, 1 ].max, duration)
|
|
13
|
+
end
|
|
14
|
+
|
|
15
|
+
def as_json
|
|
16
|
+
width = @bucket_seconds
|
|
17
|
+
buckets = Array.new(((@to - @from) / width).ceil) do |index|
|
|
18
|
+
tally.merge(start_ms: milliseconds(@from + index * width), end_ms: milliseconds([ @from + (index + 1) * width, @to ].min), agents: {})
|
|
19
|
+
end
|
|
20
|
+
total = tally
|
|
21
|
+
@scope.in_batches(of: 1000) do |batch|
|
|
22
|
+
batch.pluck(:timestamp, :agent_class, :status, :total_duration_ms).each do |timestamp, agent, status, duration|
|
|
23
|
+
index = ((timestamp - @from) / width).floor
|
|
24
|
+
next unless timestamp < @to && index >= 0 && index < buckets.size
|
|
25
|
+
|
|
26
|
+
[ total, buckets[index] ].each do |counts|
|
|
27
|
+
counts[:requests] += 1
|
|
28
|
+
counts[:errors] += 1 if status == TelemetryTrace::STATUS_ERROR
|
|
29
|
+
if duration && duration.finite?
|
|
30
|
+
counts[:duration_sum] += duration
|
|
31
|
+
counts[:duration_count] += 1
|
|
32
|
+
end
|
|
33
|
+
end
|
|
34
|
+
agents = buckets[index][:agents]
|
|
35
|
+
agents[agent] = agents.fetch(agent, 0) + 1 if agent
|
|
36
|
+
end
|
|
37
|
+
end
|
|
38
|
+
{
|
|
39
|
+
total_count: total[:requests],
|
|
40
|
+
summary: summary(total).merge(throughput_per_minute: (total[:requests] * 60.0 / (@to - @from)).round(1)),
|
|
41
|
+
window_start_ms: milliseconds(@from), window_end_ms: milliseconds(@to), bucket_seconds: width,
|
|
42
|
+
buckets: buckets.map { |bucket| summary(bucket).merge(bucket.slice(:start_ms, :end_ms, :agents)) }
|
|
43
|
+
}
|
|
44
|
+
end
|
|
45
|
+
|
|
46
|
+
private
|
|
47
|
+
|
|
48
|
+
def tally
|
|
49
|
+
{ requests: 0, errors: 0, duration_sum: 0.0, duration_count: 0 }
|
|
50
|
+
end
|
|
51
|
+
|
|
52
|
+
def summary(counts)
|
|
53
|
+
counts.slice(:requests, :errors).merge(avg_latency_ms: counts[:duration_count].positive? ? (counts[:duration_sum] / counts[:duration_count]).round : 0)
|
|
54
|
+
end
|
|
55
|
+
|
|
56
|
+
def milliseconds(time)
|
|
57
|
+
(time.to_f * 1000).to_i
|
|
58
|
+
end
|
|
59
|
+
end
|
|
60
|
+
end
|
|
@@ -0,0 +1,86 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module ActionAgent
|
|
4
|
+
# A Claude Code or Codex conversation (CodeConversation) in the shape the
|
|
5
|
+
# Interactions view lists: one row per conversation, keyed "code-<root id>"
|
|
6
|
+
# beside contexts ("<id>") and reported traces ("trace-<id>"), and, in
|
|
7
|
+
# detail, its turns with their transcripts.
|
|
8
|
+
#
|
|
9
|
+
# Each turn is one run of the CLI, so it carries what a trace would: its
|
|
10
|
+
# tokens, cost and duration, from the CLI's result event.
|
|
11
|
+
class CodeConversationSerializer
|
|
12
|
+
LABELS = { "claude_code" => "Claude Code", "codex" => "Codex" }.freeze
|
|
13
|
+
|
|
14
|
+
def self.summary(root, turns = root.turns)
|
|
15
|
+
new(root, turns).summary
|
|
16
|
+
end
|
|
17
|
+
|
|
18
|
+
def self.detail(root, turns = root.turns, after: {})
|
|
19
|
+
new(root, turns).detail(after: after)
|
|
20
|
+
end
|
|
21
|
+
|
|
22
|
+
def initialize(root, turns)
|
|
23
|
+
@root = root
|
|
24
|
+
@turns = turns
|
|
25
|
+
@sandbox = root.sandbox_session
|
|
26
|
+
end
|
|
27
|
+
|
|
28
|
+
def summary
|
|
29
|
+
last = @turns.last
|
|
30
|
+
project = @sandbox.try(:project)
|
|
31
|
+
{
|
|
32
|
+
id: "code-#{@root.id}",
|
|
33
|
+
source: "code_session",
|
|
34
|
+
runner: @root.runner,
|
|
35
|
+
origin: @root.origin,
|
|
36
|
+
agent_name: LABELS.fetch(@root.runner, @root.runner),
|
|
37
|
+
action_name: nil,
|
|
38
|
+
display_name: [ LABELS.fetch(@root.runner, @root.runner), @sandbox.try(:repository) ].compact.join(" · "),
|
|
39
|
+
agent: nil,
|
|
40
|
+
model: @turns.filter_map(&:model).last,
|
|
41
|
+
status: active&.status || last.status,
|
|
42
|
+
active: !active.nil?,
|
|
43
|
+
tokens: {
|
|
44
|
+
input: @turns.sum { |turn| turn.input_tokens.to_i },
|
|
45
|
+
output: @turns.sum { |turn| turn.output_tokens.to_i },
|
|
46
|
+
total: @turns.sum { |turn| turn.input_tokens.to_i + turn.output_tokens.to_i }
|
|
47
|
+
},
|
|
48
|
+
total_cost_usd: @turns.sum { |turn| turn.total_cost_usd.to_f }.round(6),
|
|
49
|
+
duration_ms: @turns.sum { |turn| turn.duration_ms.to_i },
|
|
50
|
+
message_count: @turns.size,
|
|
51
|
+
turn_count: @turns.size,
|
|
52
|
+
preview: {
|
|
53
|
+
input: InteractionPreview.line(@root.prompt),
|
|
54
|
+
output: InteractionPreview.line(@turns.reverse.find { |turn| turn.result.present? }&.result)
|
|
55
|
+
},
|
|
56
|
+
sandbox_session_id: @sandbox.session_id,
|
|
57
|
+
sandbox_status: @sandbox.status,
|
|
58
|
+
project: project ? { id: project.id, name: project.name } : nil,
|
|
59
|
+
credential_mode: @root.credential_mode,
|
|
60
|
+
created_at: @root.created_at.iso8601,
|
|
61
|
+
last_activity_at: (@turns.filter_map { |turn| turn.finished_at || turn.started_at }.max || last.created_at).iso8601
|
|
62
|
+
}
|
|
63
|
+
end
|
|
64
|
+
|
|
65
|
+
# The summary plus every turn: its summary, and its transcript from
|
|
66
|
+
# +after+[turn id] onward, so a client polling the running turn asks only
|
|
67
|
+
# for what it has not seen. A finished turn's diff comes from the session
|
|
68
|
+
# endpoint, on demand: it is the whole checkout's.
|
|
69
|
+
def detail(after: {})
|
|
70
|
+
summary.merge(
|
|
71
|
+
turns: @turns.map do |turn|
|
|
72
|
+
from = after[turn.id.to_s].to_i.clamp(0, turn.events.size)
|
|
73
|
+
turn.summary.merge(events: turn.events.drop(from), events_offset: from,
|
|
74
|
+
dropped_events_count: turn.dropped_events_count.to_i)
|
|
75
|
+
end,
|
|
76
|
+
browser: @sandbox.respond_to?(:browser_summary) ? @sandbox.browser_summary : nil
|
|
77
|
+
)
|
|
78
|
+
end
|
|
79
|
+
|
|
80
|
+
private
|
|
81
|
+
|
|
82
|
+
def active
|
|
83
|
+
@turns.find { |turn| turn.running? } || @turns.find { |turn| turn.queued? }
|
|
84
|
+
end
|
|
85
|
+
end
|
|
86
|
+
end
|
|
@@ -125,8 +125,19 @@ module ActionAgent
|
|
|
125
125
|
# derived from every persisted result's diagnosis, which older runs
|
|
126
126
|
# recorded in earlier shapes, so a run they fail for still serves its
|
|
127
127
|
# results. Without +links+ the item paths are relative to the mount.
|
|
128
|
-
|
|
129
|
-
|
|
128
|
+
# With +markdown+ each item carries its `markdown`
|
|
129
|
+
# (ActiveAgent::Evals::ReportMarkdown#fix_item_markdown), which the
|
|
130
|
+
# dashboard's fix cards copy.
|
|
131
|
+
def fix_items(run, links: nil, markdown: false)
|
|
132
|
+
unless markdown
|
|
133
|
+
return links ? run.fix_items(links: links) : run.fix_items
|
|
134
|
+
end
|
|
135
|
+
|
|
136
|
+
report = links ? run.to_report(links: links) : run.to_report
|
|
137
|
+
items = report.fix_items
|
|
138
|
+
return items unless report.respond_to?(:fix_item_markdown)
|
|
139
|
+
|
|
140
|
+
items.map { |item| item.merge("markdown" => report.fix_item_markdown(item)) }
|
|
130
141
|
rescue StandardError => e
|
|
131
142
|
Rails.logger.warn(
|
|
132
143
|
"[ActionAgent] evaluation run #{run.id} fix_items failed: #{e.class}: #{e.message}"
|
|
@@ -537,6 +537,12 @@ module ActionAgent
|
|
|
537
537
|
}
|
|
538
538
|
when "call_agent"
|
|
539
539
|
call_agent(slug: kwargs[:slug], message: kwargs[:message])
|
|
540
|
+
when "translate_text"
|
|
541
|
+
translate_text(**kwargs.to_h.symbolize_keys.slice(:text, :target_language, :source_language))
|
|
542
|
+
when *CheckoutTools::FUNCTIONS
|
|
543
|
+
checkout_tools.call(name.to_s, kwargs)
|
|
544
|
+
when *SlackTools::FUNCTIONS
|
|
545
|
+
SlackTools.new(SlackTools.token_for(owner)).call(name.to_s, kwargs)
|
|
540
546
|
when "request_handoff"
|
|
541
547
|
request_handoff(**kwargs.slice(:reason, :url, :form_values, :instructions))
|
|
542
548
|
else
|
|
@@ -727,6 +733,37 @@ module ActionAgent
|
|
|
727
733
|
(text.length / 4.0).round
|
|
728
734
|
end
|
|
729
735
|
|
|
736
|
+
# The checkout tools act on the sandbox the run was given, found among
|
|
737
|
+
# the agent's owner's sessions as the dispatcher finds its runtime.
|
|
738
|
+
def checkout_tools
|
|
739
|
+
@checkout_tools ||= begin
|
|
740
|
+
sandbox_id = @run.try(:sandbox_id)
|
|
741
|
+
CheckoutTools.new(sandbox_id && SandboxSession.for_owner(owner).find_by(session_id: sandbox_id))
|
|
742
|
+
end
|
|
743
|
+
end
|
|
744
|
+
|
|
745
|
+
# translate_text: one generation on this run's provider, model and the
|
|
746
|
+
# owner's credentials, with no tools and no conversation.
|
|
747
|
+
def translate_text(text:, target_language:, source_language: nil)
|
|
748
|
+
return { error: "No text to translate" } if text.to_s.strip.empty?
|
|
749
|
+
return { error: "target_language is required" } if target_language.to_s.strip.empty?
|
|
750
|
+
|
|
751
|
+
name = provider
|
|
752
|
+
options = { model: requested_model }.merge(owner_provider_options(name))
|
|
753
|
+
translator = Class.new(ActiveAgent::Base) do
|
|
754
|
+
define_singleton_method(:name) { "ActionAgent::Translator" }
|
|
755
|
+
generate_with name, **options
|
|
756
|
+
end
|
|
757
|
+
from = source_language.present? ? " from #{source_language}" : ""
|
|
758
|
+
instructions = "Translate the user's text#{from} into #{target_language}. Reply with the translation only — " \
|
|
759
|
+
"no notes, quotes or preamble. Keep formatting, code, URLs and placeholders exactly as they are."
|
|
760
|
+
response = translator.prompt(message: text.to_s, instructions: instructions).generate_now
|
|
761
|
+
|
|
762
|
+
{ translation: response&.message&.content.to_s, target_language: target_language }
|
|
763
|
+
rescue StandardError => e
|
|
764
|
+
{ error: "Translation failed: #{e.message}" }
|
|
765
|
+
end
|
|
766
|
+
|
|
730
767
|
def call_agent(slug:, message:)
|
|
731
768
|
depth = Thread.current[:agent_call_depth].to_i
|
|
732
769
|
return { error: "call_agent depth limit (#{MAX_CALL_DEPTH}) reached" } if depth >= MAX_CALL_DEPTH
|
|
@@ -50,18 +50,25 @@ module ActionAgent
|
|
|
50
50
|
# workspace already talks to, then the rest of the catalog.
|
|
51
51
|
STATUS_RANK = { "active" => 0, "configured" => 1, "available" => 2, "idle" => 3 }.freeze
|
|
52
52
|
|
|
53
|
-
attr_reader :agent, :discovery, :runtimes
|
|
53
|
+
attr_reader :agent, :discovery, :runtimes, :workspace_agents
|
|
54
54
|
|
|
55
55
|
# @param agent [ActionAgent::Agent] the agent being edited
|
|
56
56
|
# @param traces [ActiveRecord::Relation] the traces the caller may read
|
|
57
57
|
# @param hours [Integer] the window the usage columns are scoped to
|
|
58
58
|
# @param runtimes [Array<Hash>] live checkout sandbox runtimes this agent
|
|
59
59
|
# can be given, as SandboxSession.runtime_server_listings returns them
|
|
60
|
-
|
|
60
|
+
# @param workspace_agents [ActiveRecord::Relation, nil] the workspace's
|
|
61
|
+
# other agents, whose MCP servers each service row counts as
|
|
62
|
+
# +workspace_agents+ — what a new agent's builder shows as already in
|
|
63
|
+
# use, since an unsaved agent has no usage of its own to show
|
|
64
|
+
def initialize(agent:, traces:, hours: ToolDiscovery::DEFAULT_WINDOW_HOURS, runtimes: [], workspace_agents: nil)
|
|
61
65
|
@agent = agent
|
|
62
66
|
@runtimes = Array(runtimes).index_by { |runtime| runtime[:key] }
|
|
67
|
+
@workspace_agents = workspace_agents
|
|
68
|
+
# An agent not saved yet has no traces of its own, and no name to
|
|
69
|
+
# match unattributed ones by.
|
|
63
70
|
@discovery = ToolDiscovery.new(
|
|
64
|
-
traces: agent.telemetry_traces(traces),
|
|
71
|
+
traces: agent.persisted? ? agent.telemetry_traces(traces) : traces.none,
|
|
65
72
|
agents: Agent.where(id: agent.id),
|
|
66
73
|
hours: hours
|
|
67
74
|
)
|
|
@@ -118,6 +125,7 @@ module ActionAgent
|
|
|
118
125
|
used = detected_by_server[key] || []
|
|
119
126
|
calls = used.sum { |tool| tool[:calls] }
|
|
120
127
|
enabled = configured_servers.key?(key)
|
|
128
|
+
unreachable = MCPToolDispatcher.unreachable_reason(MCPCatalog.connection(key) || runtimes[key])
|
|
121
129
|
|
|
122
130
|
{
|
|
123
131
|
key: key,
|
|
@@ -133,10 +141,25 @@ module ActionAgent
|
|
|
133
141
|
calls: calls,
|
|
134
142
|
errors: used.sum { |tool| tool[:errors] },
|
|
135
143
|
last_seen: used.filter_map { |tool| tool[:last_seen] }.max,
|
|
144
|
+
# Whether a dashboard run reaches it (MCPToolDispatcher), and when
|
|
145
|
+
# not, why: a stdio server, or one needing a credential it can't send.
|
|
146
|
+
callable: unreachable.nil?,
|
|
147
|
+
unavailable_reason: unreachable,
|
|
148
|
+
workspace_agents: workspace_usage[key].to_i,
|
|
136
149
|
tools: service_tools(key, catalog, used)
|
|
137
150
|
}
|
|
138
151
|
end
|
|
139
152
|
|
|
153
|
+
# server key => how many of the workspace's agents enable it. Empty
|
|
154
|
+
# unless the caller handed in +workspace_agents+.
|
|
155
|
+
def workspace_usage
|
|
156
|
+
@workspace_usage ||= Array(workspace_agents).each_with_object(Hash.new(0)) do |other, counts|
|
|
157
|
+
next if agent.persisted? && other.id == agent.id
|
|
158
|
+
|
|
159
|
+
self.class.configured_servers(other).each_key { |key| counts[key] += 1 }
|
|
160
|
+
end
|
|
161
|
+
end
|
|
162
|
+
|
|
140
163
|
# How to reach the server, in the one line the expanded panel shows:
|
|
141
164
|
# "Streamable HTTP · <url>" for the ones the dashboard can call,
|
|
142
165
|
# "sandbox · <command>" for the ones it can start, "stdio · <command>"
|
|
@@ -233,6 +256,10 @@ module ActionAgent
|
|
|
233
256
|
description: Agent::TOOL_DESCRIPTIONS[capability],
|
|
234
257
|
enabled: saved_tools.include?(capability),
|
|
235
258
|
editable: true,
|
|
259
|
+
# Whether AgentToolbox gives the model anything for it. A capability
|
|
260
|
+
# without definitions is a name on the roster and nothing more, so
|
|
261
|
+
# the builder offers its MCP service (or nothing) in its place.
|
|
262
|
+
implemented: AgentToolbox::DEFINITIONS.key?(capability),
|
|
236
263
|
approval_names: capability_functions(capability) - AgentToolbox::INPUT_FUNCTIONS
|
|
237
264
|
)
|
|
238
265
|
end
|
|
@@ -283,27 +310,35 @@ module ActionAgent
|
|
|
283
310
|
# carries a top-level Hash keyed by server name — the same three shapes
|
|
284
311
|
# EvaluationToolResolver tolerates.
|
|
285
312
|
def configured_servers
|
|
286
|
-
@configured_servers ||=
|
|
287
|
-
key = entry_key(entry)
|
|
288
|
-
map[key] = entry if key.present?
|
|
289
|
-
end
|
|
313
|
+
@configured_servers ||= self.class.configured_servers(agent)
|
|
290
314
|
end
|
|
291
315
|
|
|
292
|
-
|
|
293
|
-
|
|
316
|
+
class << self
|
|
317
|
+
# server key => the entry +agent+ stores for it, in any of the three
|
|
318
|
+
# shapes above.
|
|
319
|
+
def configured_servers(agent)
|
|
320
|
+
configured_entries(agent.mcp_servers).each_with_object({}) do |entry, map|
|
|
321
|
+
key = entry_key(entry)
|
|
322
|
+
map[key] = entry if key.present?
|
|
323
|
+
end
|
|
324
|
+
end
|
|
325
|
+
|
|
326
|
+
private
|
|
294
327
|
|
|
295
|
-
|
|
296
|
-
servers.
|
|
297
|
-
|
|
298
|
-
|
|
328
|
+
def configured_entries(servers)
|
|
329
|
+
if servers.is_a?(Hash)
|
|
330
|
+
servers.map { |key, value| value.respond_to?(:key?) ? value.to_h.stringify_keys.merge("key" => key.to_s) : key.to_s }
|
|
331
|
+
else
|
|
332
|
+
Array(servers)
|
|
333
|
+
end
|
|
299
334
|
end
|
|
300
|
-
end
|
|
301
335
|
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
|
|
336
|
+
def entry_key(entry)
|
|
337
|
+
return entry.to_s.strip.presence if entry.is_a?(String) || entry.is_a?(Symbol)
|
|
338
|
+
return nil unless entry.respond_to?(:key?)
|
|
305
339
|
|
|
306
|
-
|
|
340
|
+
(entry["key"] || entry[:key] || entry["name"] || entry[:name]).to_s.strip.presence
|
|
341
|
+
end
|
|
307
342
|
end
|
|
308
343
|
|
|
309
344
|
# The per-server allow-list, or nil when the entry names none — which
|