actionagent 1.9.1 → 1.9.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +23 -0
- data/app/assets/builds/action_agent.css +1 -1
- data/app/assets/builds/action_agent.js +61 -61
- data/app/controllers/action_agent/api/agents_controller.rb +23 -0
- data/app/controllers/action_agent/api/code_sessions_controller.rb +49 -10
- data/app/controllers/action_agent/api/evaluation_reports_controller.rb +66 -2
- data/app/controllers/action_agent/api/evaluations_controller.rb +56 -6
- data/app/controllers/action_agent/api/interactions_controller.rb +55 -1
- data/app/controllers/action_agent/api/sandboxes_controller.rb +13 -2
- data/app/controllers/action_agent/api/trace_reports_controller.rb +11 -6
- data/app/controllers/concerns/action_agent/api/mcp_dashboard_tools.rb +2 -2
- data/app/jobs/action_agent/code_session_job.rb +19 -2
- data/app/jobs/action_agent/evaluation_judge_job.rb +39 -0
- data/app/jobs/action_agent/evaluation_replay_job.rb +28 -0
- data/app/models/action_agent/agent.rb +54 -1
- data/app/models/action_agent/code_session.rb +80 -0
- data/app/models/action_agent/evaluation_run.rb +37 -5
- data/app/models/action_agent/provider_key.rb +5 -1
- data/app/queries/action_agent/trace_window_report.rb +60 -0
- data/app/serializers/action_agent/code_conversation_serializer.rb +86 -0
- data/app/serializers/action_agent/evaluation_serializer.rb +13 -2
- data/app/services/action_agent/agent_execution_service.rb +37 -0
- data/app/services/action_agent/agent_tool_roster.rb +53 -18
- data/app/services/action_agent/agent_toolbox.rb +156 -3
- data/app/services/action_agent/browser_client_token.rb +86 -0
- data/app/services/action_agent/checkout_tools.rb +154 -0
- data/app/services/action_agent/code_conversation.rb +133 -0
- data/app/services/action_agent/codex_auth.rb +81 -0
- data/app/services/action_agent/dashboard_assistant_service.rb +19 -1
- data/app/services/action_agent/diff_stat.rb +27 -0
- data/app/services/action_agent/evaluation_fix_comparison.rb +4 -1
- data/app/services/action_agent/evaluation_report_export.rb +175 -0
- data/app/services/action_agent/evaluation_report_import.rb +47 -19
- data/app/services/action_agent/evaluation_runner_service.rb +106 -14
- data/app/services/action_agent/local_sandbox_backend.rb +295 -30
- data/app/services/action_agent/mcp_catalog.rb +25 -0
- data/app/services/action_agent/mcp_client.rb +1 -0
- data/app/services/action_agent/mcp_tool_dispatcher.rb +36 -1
- data/app/services/action_agent/project_boot_fix.rb +10 -3
- data/app/services/action_agent/sandbox_orchestrator.rb +41 -1
- data/app/services/action_agent/scenario_evaluation_runner.rb +381 -3
- data/app/services/action_agent/slack_tools.rb +101 -0
- data/config/routes.rb +10 -1
- data/lib/action_agent/engine.rb +6 -0
- data/lib/action_agent/version.rb +1 -1
- data/lib/action_agent.rb +50 -2
- data/lib/generators/action_agent/templates/migrations/028_add_conversations_to_code_sessions.rb.erb +18 -0
- data/lib/tasks/evaluations.rake +78 -0
- metadata +15 -2
|
@@ -51,6 +51,7 @@ module ActionAgent
|
|
|
51
51
|
# Callbacks
|
|
52
52
|
before_validation :generate_slug, on: :create
|
|
53
53
|
before_validation :apply_conventional_schema_tools, on: :create
|
|
54
|
+
before_validation :require_approval_for_gained_side_effects, unless: :observed?
|
|
54
55
|
after_create :create_initial_version
|
|
55
56
|
after_update :create_version_on_config_change, if: :configuration_changed?
|
|
56
57
|
after_destroy :release_telemetry_traces, if: :observed?
|
|
@@ -83,6 +84,30 @@ module ActionAgent
|
|
|
83
84
|
github ruby rails aws gcp python typescript docker kubernetes
|
|
84
85
|
].freeze
|
|
85
86
|
|
|
87
|
+
# What each instruction set adds to the instructions a run executes
|
|
88
|
+
# under (#composed_instructions_for): the conventions of that stack, as a
|
|
89
|
+
# developer message the model can act on.
|
|
90
|
+
INSTRUCTION_SET_GUIDANCE = {
|
|
91
|
+
"github" => "Work the GitHub way: small focused commits with messages that say why, branches named for the " \
|
|
92
|
+
"change, pull requests that describe what changed and how it was verified, and issues referenced by number.",
|
|
93
|
+
"ruby" => "Write idiomatic Ruby: follow the project's RuboCop configuration, prefer small methods and plain " \
|
|
94
|
+
"objects, use keyword arguments for options, and add or update tests (Minitest or RSpec, as the project uses).",
|
|
95
|
+
"rails" => "Follow Rails conventions: fat models and thin controllers, strong parameters, migrations for every " \
|
|
96
|
+
"schema change, scopes and validations on the model, and bin/rails commands for generators, tests and tasks.",
|
|
97
|
+
"aws" => "For AWS, prefer least-privilege IAM, keep infrastructure in code (CloudFormation, CDK or Terraform), " \
|
|
98
|
+
"name the region explicitly, and never print or commit credentials.",
|
|
99
|
+
"gcp" => "For Google Cloud, prefer least-privilege service accounts, keep infrastructure in code (Terraform), " \
|
|
100
|
+
"name the project and region explicitly, and use Secret Manager rather than plain environment values.",
|
|
101
|
+
"python" => "Write idiomatic Python: PEP 8, type hints on public functions, the project's own environment and " \
|
|
102
|
+
"dependency tool, and pytest tests for new behaviour.",
|
|
103
|
+
"typescript" => "Write strict TypeScript: no implicit any, explicit types on exported APIs, the project's lint and " \
|
|
104
|
+
"format settings, and tests in the framework the project already uses.",
|
|
105
|
+
"docker" => "For Docker, keep images small and reproducible: pinned base image versions, multi-stage builds, a " \
|
|
106
|
+
"non-root user, and no secrets baked into layers.",
|
|
107
|
+
"kubernetes" => "For Kubernetes, write declarative manifests with resource requests and limits, readiness and " \
|
|
108
|
+
"liveness probes, and Secrets for credentials; apply changes namespace by namespace."
|
|
109
|
+
}.freeze
|
|
110
|
+
|
|
86
111
|
# Built-in tools/MCPs. Host-declared schema tools are offered alongside
|
|
87
112
|
# these — see .available_tools, which is what the editor and the APIs
|
|
88
113
|
# serialize. This constant stays the built-in set so existing references
|
|
@@ -156,6 +181,24 @@ module ActionAgent
|
|
|
156
181
|
self.tools = defaults if defaults.any?
|
|
157
182
|
end
|
|
158
183
|
|
|
184
|
+
# run_command, write_file, edit_file and slack_post_message wait for a
|
|
185
|
+
# person's approval unless the agent's approval list says otherwise
|
|
186
|
+
# (AgentToolbox::APPROVAL_BY_DEFAULT). They join the list when the agent
|
|
187
|
+
# gains their capability and this save brought no approval list of its
|
|
188
|
+
# own: an agent built from a template, a POST to the agents API with
|
|
189
|
+
# tools alone, a PATCH that enables a capability. The builder and the
|
|
190
|
+
# Tools tab send the list, so an Approval switch turned off holds.
|
|
191
|
+
def require_approval_for_gained_side_effects
|
|
192
|
+
return unless has_attribute?(:approval_required_tools)
|
|
193
|
+
return if attribute_came_from_user?("approval_required_tools")
|
|
194
|
+
|
|
195
|
+
gained = Array(tools).map(&:to_s) - Array(tools_in_database).map(&:to_s)
|
|
196
|
+
names = AgentToolbox.default_approvals(gained)
|
|
197
|
+
return if names.empty?
|
|
198
|
+
|
|
199
|
+
self[:approval_required_tools] = approval_required_tools | names
|
|
200
|
+
end
|
|
201
|
+
|
|
159
202
|
# The ActiveAgent class name this agent's runs are recorded under — the
|
|
160
203
|
# correlation key between platform Agent records and telemetry traces
|
|
161
204
|
# (TelemetryTrace#agent_class) and solid_agent contexts. An observed
|
|
@@ -229,9 +272,19 @@ module ActionAgent
|
|
|
229
272
|
# The system instructions an action executes under: named actions stack
|
|
230
273
|
# their prompt below the agent's base instructions; the default action
|
|
231
274
|
# uses the base instructions alone.
|
|
275
|
+
#
|
|
276
|
+
# The agent's instruction sets come last, as the conventions to work by.
|
|
232
277
|
def composed_instructions_for(action_name)
|
|
233
278
|
action = action_prompt_for(action_name)
|
|
234
|
-
[ instructions, action&.dig("prompt") ].map(&:presence).compact.join("\n\n").presence
|
|
279
|
+
[ instructions, action&.dig("prompt"), instruction_set_guidance ].map(&:presence).compact.join("\n\n").presence
|
|
280
|
+
end
|
|
281
|
+
|
|
282
|
+
# The guidance of the instruction sets this agent selected, or nil.
|
|
283
|
+
def instruction_set_guidance
|
|
284
|
+
lines = Array(instruction_sets).filter_map { |set| INSTRUCTION_SET_GUIDANCE[set.to_s] }
|
|
285
|
+
return nil if lines.empty?
|
|
286
|
+
|
|
287
|
+
"Conventions to follow:\n#{lines.map { |line| "- #{line}" }.join("\n")}"
|
|
235
288
|
end
|
|
236
289
|
|
|
237
290
|
# Returns the configuration as a hash for versioning
|
|
@@ -6,6 +6,12 @@ module ActionAgent
|
|
|
6
6
|
# outcome Claude Code reported, and the diff the session left in the
|
|
7
7
|
# working tree.
|
|
8
8
|
#
|
|
9
|
+
# A session is one turn of a conversation (CodeConversation). The first
|
|
10
|
+
# turn is the conversation's root; each later one names it
|
|
11
|
+
# (conversation_root_id), runs after the one before it, and resumes the same
|
|
12
|
+
# CLI session (resume_session_id), so Claude Code or Codex remembers what
|
|
13
|
+
# came before. Every existing session is a conversation of one turn.
|
|
14
|
+
#
|
|
9
15
|
# The transcript is scrubbed of the sandbox's secrets (its GitHub token and
|
|
10
16
|
# Claude Code credential) before it is stored, and bounded: long strings
|
|
11
17
|
# are truncated and events past MAX_EVENTS are counted rather than kept.
|
|
@@ -17,6 +23,9 @@ module ActionAgent
|
|
|
17
23
|
belongs_to :evaluation_run, optional: true
|
|
18
24
|
belongs_to :verification_run, class_name: "EvaluationRun", optional: true
|
|
19
25
|
belongs_to :previous_code_session, class_name: "CodeSession", optional: true
|
|
26
|
+
belongs_to :conversation_root, class_name: "CodeSession", optional: true
|
|
27
|
+
has_many :follow_up_turns, -> { order(:turn_index, :id) }, class_name: "CodeSession",
|
|
28
|
+
foreign_key: :conversation_root_id, inverse_of: :conversation_root
|
|
20
29
|
|
|
21
30
|
enum :status, { queued: 0, running: 1, succeeded: 2, failed: 3, cancelled: 4 }
|
|
22
31
|
|
|
@@ -26,11 +35,32 @@ module ActionAgent
|
|
|
26
35
|
MAX_EVENT_STRING = 4_000
|
|
27
36
|
MAX_DIFF_BYTES = 500_000
|
|
28
37
|
RUNNERS = %w[claude_code codex].freeze
|
|
38
|
+
# Where a conversation was started. Older sessions recorded none; see
|
|
39
|
+
# #origin.
|
|
40
|
+
ORIGINS = %w[settings boot_fix evaluation_fix chat].freeze
|
|
41
|
+
# A Claude Code session id, as --session-id and --resume take it.
|
|
42
|
+
SESSION_UUID = /\A\h{8}-\h{4}-\h{4}-\h{4}-\h{12}\z/
|
|
43
|
+
# A note from the dashboard is fenced in this tag ahead of the person's
|
|
44
|
+
# message, and the tag is taken out of anything the person typed.
|
|
45
|
+
NOTE_TAG = "dashboard-note"
|
|
29
46
|
|
|
30
47
|
validates :prompt, presence: true, length: { maximum: MAX_PROMPT_CHARACTERS }
|
|
31
48
|
validates :runner, inclusion: { in: RUNNERS }
|
|
32
49
|
|
|
50
|
+
validates :origin, inclusion: { in: ORIGINS }, allow_nil: true
|
|
51
|
+
# A Codex conversation resumes the thread id its first turn reported.
|
|
52
|
+
validates :resume_session_id, format: { with: SESSION_UUID }, allow_nil: true, if: -> { runner == "claude_code" }
|
|
53
|
+
|
|
54
|
+
# A conversation's first Claude Code turn names the CLI session it
|
|
55
|
+
# starts (--session-id), so every later turn can resume it even when the
|
|
56
|
+
# first ended before reporting its own id.
|
|
57
|
+
before_validation(on: :create) do
|
|
58
|
+
self.resume_session_id ||= SecureRandom.uuid if root? && runner == "claude_code"
|
|
59
|
+
end
|
|
60
|
+
|
|
33
61
|
scope :recent, -> { order(created_at: :desc) }
|
|
62
|
+
# The first turn of every conversation.
|
|
63
|
+
scope :conversation_roots, -> { where(conversation_root_id: nil) }
|
|
34
64
|
|
|
35
65
|
# JSON columns carry no default on MySQL or SQLite (see the migration).
|
|
36
66
|
def events
|
|
@@ -41,6 +71,51 @@ module ActionAgent
|
|
|
41
71
|
succeeded? || failed? || cancelled?
|
|
42
72
|
end
|
|
43
73
|
|
|
74
|
+
def root?
|
|
75
|
+
conversation_root_id.nil?
|
|
76
|
+
end
|
|
77
|
+
|
|
78
|
+
# The conversation's first turn: this one, or the one it names.
|
|
79
|
+
def root
|
|
80
|
+
root? ? self : conversation_root
|
|
81
|
+
end
|
|
82
|
+
|
|
83
|
+
# Every turn of this session's conversation, in order.
|
|
84
|
+
#
|
|
85
|
+
# @return [Array<CodeSession>]
|
|
86
|
+
def turns
|
|
87
|
+
# Read afresh: turns are added under the sandbox's lock while this
|
|
88
|
+
# session may hold an older list.
|
|
89
|
+
[ root, *CodeSession.where(conversation_root_id: root.id).order(:turn_index, :id).to_a ]
|
|
90
|
+
end
|
|
91
|
+
|
|
92
|
+
# Where the conversation was started. Sessions from before origins were
|
|
93
|
+
# recorded are told apart by what they ran for: an evaluation fix names
|
|
94
|
+
# its run, and a boot fix is one its project recorded
|
|
95
|
+
# (ProjectBootFix.project_for).
|
|
96
|
+
def origin
|
|
97
|
+
recorded = super.presence
|
|
98
|
+
return recorded if recorded
|
|
99
|
+
return "evaluation_fix" if evaluation_run_id
|
|
100
|
+
return "boot_fix" if persisted? && root? && ProjectBootFix.project_for(self)
|
|
101
|
+
|
|
102
|
+
"settings"
|
|
103
|
+
end
|
|
104
|
+
|
|
105
|
+
# What the CLI is given on stdin: the dashboard's note, fenced, ahead of
|
|
106
|
+
# the person's message. The prompt itself stays what they typed.
|
|
107
|
+
def stdin_text
|
|
108
|
+
return prompt if context_note.blank?
|
|
109
|
+
|
|
110
|
+
"<#{NOTE_TAG}>\n#{context_note}\n</#{NOTE_TAG}>\n\n#{prompt}"
|
|
111
|
+
end
|
|
112
|
+
|
|
113
|
+
# +text+ without the dashboard's note tags, so a person cannot make their
|
|
114
|
+
# own words read as the dashboard's.
|
|
115
|
+
def self.strip_note_tags(text)
|
|
116
|
+
text.to_s.gsub(%r{</?\s*#{NOTE_TAG}[^>]*>}i, "")
|
|
117
|
+
end
|
|
118
|
+
|
|
44
119
|
# Whether an outcome and a diff may still be recorded for this session.
|
|
45
120
|
# finished_at is set once nothing more will be: by CodeSessionJob when it
|
|
46
121
|
# settles a session it ran (whatever the backend returned, raised or
|
|
@@ -111,6 +186,10 @@ module ActionAgent
|
|
|
111
186
|
evaluation_run_id: evaluation_run_id,
|
|
112
187
|
evaluation_id: evaluation_run&.evaluation_id,
|
|
113
188
|
fix_item: fix_item,
|
|
189
|
+
conversation_id: root? ? id : conversation_root_id,
|
|
190
|
+
turn_index: turn_index,
|
|
191
|
+
origin: origin,
|
|
192
|
+
context_note: context_note,
|
|
114
193
|
verification_run_id: verification_run_id,
|
|
115
194
|
verification_error: verification_error,
|
|
116
195
|
result: result,
|
|
@@ -121,6 +200,7 @@ module ActionAgent
|
|
|
121
200
|
input_tokens: input_tokens,
|
|
122
201
|
output_tokens: output_tokens,
|
|
123
202
|
event_count: events.size + dropped_events_count.to_i,
|
|
203
|
+
diff_stat: finished? ? DiffStat.call(diff) : nil,
|
|
124
204
|
# What a client polls until: false once nothing about the session
|
|
125
205
|
# changes any more.
|
|
126
206
|
diff_pending: diff_pending?,
|
|
@@ -78,6 +78,15 @@ module ActionAgent
|
|
|
78
78
|
verdict.to_h.stringify_keys.presence if verdict.is_a?(Hash)
|
|
79
79
|
end
|
|
80
80
|
|
|
81
|
+
# The model comparison a comparison run recorded (Report#comparison):
|
|
82
|
+
# each model's pass rate with its interval, every pair's difference and
|
|
83
|
+
# whether it is decisive, and the conclusion. nil for a single-model
|
|
84
|
+
# run, or one recorded before runs kept it.
|
|
85
|
+
def recorded_comparison
|
|
86
|
+
comparison = scores&.dig("_comparison")
|
|
87
|
+
comparison if comparison.is_a?(Hash) && comparison.present?
|
|
88
|
+
end
|
|
89
|
+
|
|
81
90
|
# How the report names the judge, the way the suite panel does: the
|
|
82
91
|
# judge the recorded verdict names — unless that is only the pass-rate
|
|
83
92
|
# ranking — else the evaluation's judge model. nil when neither is set,
|
|
@@ -191,6 +200,16 @@ module ActionAgent
|
|
|
191
200
|
cost_breakdown.usage
|
|
192
201
|
end
|
|
193
202
|
|
|
203
|
+
# The judge's run-level spend as recorded, without pricing anything:
|
|
204
|
+
# what the publishing application reported for the calls no result owns,
|
|
205
|
+
# else the engine's meter, else nil.
|
|
206
|
+
def recorded_judge_usage_run
|
|
207
|
+
reported = scores&.dig("_judge_usage_run")
|
|
208
|
+
return reported if reported.is_a?(Hash)
|
|
209
|
+
|
|
210
|
+
judge_usage_meter&.merge("source" => "meter")
|
|
211
|
+
end
|
|
212
|
+
|
|
194
213
|
# Route templates for the report's fix item actions, relative to the
|
|
195
214
|
# dashboard mount: `%{key}` is filled in per MCP server by the report.
|
|
196
215
|
# The JSON API leaves `mount` empty — the React app resolves paths
|
|
@@ -220,7 +239,7 @@ module ActionAgent
|
|
|
220
239
|
# writes with Report#to_html. The run's recorded verdict and judge go
|
|
221
240
|
# with it: the report is not to re-rank the rebuilt results by pass
|
|
222
241
|
# rate and show a different judge's pick, verdict or `judged by` than
|
|
223
|
-
# the suite panel does. Raises ActiveRecord::RecordNotFound via the
|
|
242
|
+
# the suite panel does, nor resample the comparison the run recorded. Raises ActiveRecord::RecordNotFound via the
|
|
224
243
|
# caller for a run of a generation-sampling evaluation, which has no
|
|
225
244
|
# scenario results to report on.
|
|
226
245
|
#
|
|
@@ -229,12 +248,19 @@ module ActionAgent
|
|
|
229
248
|
# judge's run-level spend, so the page shows every cost the dashboard
|
|
230
249
|
# does. `estimate: false` rebuilds the report from what was recorded
|
|
231
250
|
# alone — the import summarizes a published run that way, so the
|
|
232
|
-
# summaries it stores stay the application's own figures
|
|
251
|
+
# summaries it stores stay the application's own figures, and the
|
|
252
|
+
# export (EvaluationReportExport) serves it that way, so the same run
|
|
253
|
+
# always exports the same report. The judge's run-level spend is then
|
|
254
|
+
# the one recorded: the application's for an imported run, the meter's
|
|
255
|
+
# for one the engine judged.
|
|
233
256
|
#
|
|
234
257
|
# The engine runs against every activeagent since 1.4, so the report
|
|
235
258
|
# kwargs that arrived later are passed only when Report.new takes them.
|
|
236
259
|
def to_report(links: report_links, estimate: true)
|
|
237
|
-
|
|
260
|
+
# A replay not yet scored (a fanned-out run in progress) is not a
|
|
261
|
+
# result the report can grade, so it is left out rather than read as
|
|
262
|
+
# a failure.
|
|
263
|
+
rows = scenario_results.where.not(status: :pending).includes(:scenario).sort_by do |row|
|
|
238
264
|
[ row.evaluated_scenario["position"].to_i, row.evaluation_scenario_id, row.model ]
|
|
239
265
|
end
|
|
240
266
|
breakdown = cost_breakdown if estimate
|
|
@@ -274,8 +300,10 @@ module ActionAgent
|
|
|
274
300
|
agent_name: evaluation.agent&.name,
|
|
275
301
|
links: links
|
|
276
302
|
}
|
|
277
|
-
|
|
303
|
+
judge_usage_run = breakdown ? breakdown.judge_usage_run : recorded_judge_usage_run
|
|
304
|
+
kwargs[:judge_usage] = judge_usage_run if judge_usage_run && self.class.report_accepts?(:judge_usage)
|
|
278
305
|
kwargs[:release] = release_summary if release_summary && self.class.report_accepts?(:release)
|
|
306
|
+
kwargs[:comparison] = recorded_comparison if recorded_comparison && self.class.report_accepts?(:comparison)
|
|
279
307
|
self.class.report_class.new(**kwargs)
|
|
280
308
|
end
|
|
281
309
|
|
|
@@ -319,7 +347,11 @@ module ActionAgent
|
|
|
319
347
|
# for it, and — when estimating — how its cost was priced.
|
|
320
348
|
def replay_metadata_for(row, breakdown)
|
|
321
349
|
metadata = row.replay_metadata.dup
|
|
322
|
-
|
|
350
|
+
# A fanned-out run's results each keep what their scoring job spent on
|
|
351
|
+
# the judge, already folded into the run's meter. The meter is the
|
|
352
|
+
# run's figure (EvaluationRunCost#result_judge reads it the same way),
|
|
353
|
+
# so the parts are not counted a second time.
|
|
354
|
+
reported_judge = row.diagnosis.is_a?(Hash) && !judge_usage_meter ? row.diagnosis["_judge_usage"] : nil
|
|
323
355
|
metadata["judge_usage"] = reported_judge if reported_judge.is_a?(Hash)
|
|
324
356
|
return metadata unless breakdown
|
|
325
357
|
|
|
@@ -39,7 +39,8 @@ module ActionAgent
|
|
|
39
39
|
HOST_PROVIDERS = %w[ollama].freeze
|
|
40
40
|
# Tools connected with a credential, configured beside the providers
|
|
41
41
|
# (Settings -> Integrations) but never offered to the agent builder.
|
|
42
|
-
|
|
42
|
+
# Slack's is the bot token the agents' slack tools post with (SlackTools).
|
|
43
|
+
CONNECTION_PROVIDERS = %w[claude_code codex slack].freeze
|
|
43
44
|
PROVIDERS = (KEY_PROVIDERS + HOST_PROVIDERS + CONNECTION_PROVIDERS).freeze
|
|
44
45
|
|
|
45
46
|
# Only an Anthropic API key (sk-ant-api03-…, from the Claude Console or
|
|
@@ -52,6 +53,7 @@ module ActionAgent
|
|
|
52
53
|
# dashboard never touches the credential.
|
|
53
54
|
CLAUDE_CODE_CREDENTIAL = /\Ask-ant-api\d{2}-[A-Za-z0-9_-]+\z/
|
|
54
55
|
CODEX_CREDENTIAL = /\Ask-(?!ant-|or-)[A-Za-z0-9_-]+\z/
|
|
56
|
+
SLACK_CREDENTIAL = /\Axoxb-[A-Za-z0-9-]+\z/
|
|
55
57
|
# A Claude subscription token, as earlier versions stored. Recognized so
|
|
56
58
|
# a stored one is never handed out (see #needs_replacing?).
|
|
57
59
|
SUBSCRIPTION_TOKEN_PREFIX = "sk-ant-oat"
|
|
@@ -95,6 +97,8 @@ module ActionAgent
|
|
|
95
97
|
}, if: -> { provider == "claude_code" }
|
|
96
98
|
validates :credential, format: { with: CODEX_CREDENTIAL, message: "must be an OpenAI API key (sk-…)" },
|
|
97
99
|
if: -> { provider == "codex" }
|
|
100
|
+
validates :credential, format: { with: SLACK_CREDENTIAL, message: "must be a Slack bot token (xoxb-…)" },
|
|
101
|
+
if: -> { provider == "slack" }
|
|
98
102
|
validates :api_key, length: { maximum: 500 }, allow_nil: true
|
|
99
103
|
|
|
100
104
|
# Deletes every Claude Code connection that still holds a Claude
|
|
@@ -0,0 +1,60 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module ActionAgent
|
|
4
|
+
# Full-window counts and chart buckets, independent of the capped trace
|
|
5
|
+
# details. Read only scalar columns, never the potentially large spans.
|
|
6
|
+
class TraceWindowReport
|
|
7
|
+
MAX_BUCKETS = 180
|
|
8
|
+
|
|
9
|
+
def initialize(scope, from:, to:, bucket_seconds:)
|
|
10
|
+
@scope, @from, @to = scope, from, to
|
|
11
|
+
duration = (to - from).ceil
|
|
12
|
+
@bucket_seconds = bucket_seconds.to_i.clamp([ (duration.to_f / MAX_BUCKETS).ceil, 1 ].max, duration)
|
|
13
|
+
end
|
|
14
|
+
|
|
15
|
+
def as_json
|
|
16
|
+
width = @bucket_seconds
|
|
17
|
+
buckets = Array.new(((@to - @from) / width).ceil) do |index|
|
|
18
|
+
tally.merge(start_ms: milliseconds(@from + index * width), end_ms: milliseconds([ @from + (index + 1) * width, @to ].min), agents: {})
|
|
19
|
+
end
|
|
20
|
+
total = tally
|
|
21
|
+
@scope.in_batches(of: 1000) do |batch|
|
|
22
|
+
batch.pluck(:timestamp, :agent_class, :status, :total_duration_ms).each do |timestamp, agent, status, duration|
|
|
23
|
+
index = ((timestamp - @from) / width).floor
|
|
24
|
+
next unless timestamp < @to && index >= 0 && index < buckets.size
|
|
25
|
+
|
|
26
|
+
[ total, buckets[index] ].each do |counts|
|
|
27
|
+
counts[:requests] += 1
|
|
28
|
+
counts[:errors] += 1 if status == TelemetryTrace::STATUS_ERROR
|
|
29
|
+
if duration && duration.finite?
|
|
30
|
+
counts[:duration_sum] += duration
|
|
31
|
+
counts[:duration_count] += 1
|
|
32
|
+
end
|
|
33
|
+
end
|
|
34
|
+
agents = buckets[index][:agents]
|
|
35
|
+
agents[agent] = agents.fetch(agent, 0) + 1 if agent
|
|
36
|
+
end
|
|
37
|
+
end
|
|
38
|
+
{
|
|
39
|
+
total_count: total[:requests],
|
|
40
|
+
summary: summary(total).merge(throughput_per_minute: (total[:requests] * 60.0 / (@to - @from)).round(1)),
|
|
41
|
+
window_start_ms: milliseconds(@from), window_end_ms: milliseconds(@to), bucket_seconds: width,
|
|
42
|
+
buckets: buckets.map { |bucket| summary(bucket).merge(bucket.slice(:start_ms, :end_ms, :agents)) }
|
|
43
|
+
}
|
|
44
|
+
end
|
|
45
|
+
|
|
46
|
+
private
|
|
47
|
+
|
|
48
|
+
def tally
|
|
49
|
+
{ requests: 0, errors: 0, duration_sum: 0.0, duration_count: 0 }
|
|
50
|
+
end
|
|
51
|
+
|
|
52
|
+
def summary(counts)
|
|
53
|
+
counts.slice(:requests, :errors).merge(avg_latency_ms: counts[:duration_count].positive? ? (counts[:duration_sum] / counts[:duration_count]).round : 0)
|
|
54
|
+
end
|
|
55
|
+
|
|
56
|
+
def milliseconds(time)
|
|
57
|
+
(time.to_f * 1000).to_i
|
|
58
|
+
end
|
|
59
|
+
end
|
|
60
|
+
end
|
|
@@ -0,0 +1,86 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module ActionAgent
|
|
4
|
+
# A Claude Code or Codex conversation (CodeConversation) in the shape the
|
|
5
|
+
# Interactions view lists: one row per conversation, keyed "code-<root id>"
|
|
6
|
+
# beside contexts ("<id>") and reported traces ("trace-<id>"), and, in
|
|
7
|
+
# detail, its turns with their transcripts.
|
|
8
|
+
#
|
|
9
|
+
# Each turn is one run of the CLI, so it carries what a trace would: its
|
|
10
|
+
# tokens, cost and duration, from the CLI's result event.
|
|
11
|
+
class CodeConversationSerializer
|
|
12
|
+
LABELS = { "claude_code" => "Claude Code", "codex" => "Codex" }.freeze
|
|
13
|
+
|
|
14
|
+
def self.summary(root, turns = root.turns)
|
|
15
|
+
new(root, turns).summary
|
|
16
|
+
end
|
|
17
|
+
|
|
18
|
+
def self.detail(root, turns = root.turns, after: {})
|
|
19
|
+
new(root, turns).detail(after: after)
|
|
20
|
+
end
|
|
21
|
+
|
|
22
|
+
def initialize(root, turns)
|
|
23
|
+
@root = root
|
|
24
|
+
@turns = turns
|
|
25
|
+
@sandbox = root.sandbox_session
|
|
26
|
+
end
|
|
27
|
+
|
|
28
|
+
def summary
|
|
29
|
+
last = @turns.last
|
|
30
|
+
project = @sandbox.try(:project)
|
|
31
|
+
{
|
|
32
|
+
id: "code-#{@root.id}",
|
|
33
|
+
source: "code_session",
|
|
34
|
+
runner: @root.runner,
|
|
35
|
+
origin: @root.origin,
|
|
36
|
+
agent_name: LABELS.fetch(@root.runner, @root.runner),
|
|
37
|
+
action_name: nil,
|
|
38
|
+
display_name: [ LABELS.fetch(@root.runner, @root.runner), @sandbox.try(:repository) ].compact.join(" · "),
|
|
39
|
+
agent: nil,
|
|
40
|
+
model: @turns.filter_map(&:model).last,
|
|
41
|
+
status: active&.status || last.status,
|
|
42
|
+
active: !active.nil?,
|
|
43
|
+
tokens: {
|
|
44
|
+
input: @turns.sum { |turn| turn.input_tokens.to_i },
|
|
45
|
+
output: @turns.sum { |turn| turn.output_tokens.to_i },
|
|
46
|
+
total: @turns.sum { |turn| turn.input_tokens.to_i + turn.output_tokens.to_i }
|
|
47
|
+
},
|
|
48
|
+
total_cost_usd: @turns.sum { |turn| turn.total_cost_usd.to_f }.round(6),
|
|
49
|
+
duration_ms: @turns.sum { |turn| turn.duration_ms.to_i },
|
|
50
|
+
message_count: @turns.size,
|
|
51
|
+
turn_count: @turns.size,
|
|
52
|
+
preview: {
|
|
53
|
+
input: InteractionPreview.line(@root.prompt),
|
|
54
|
+
output: InteractionPreview.line(@turns.reverse.find { |turn| turn.result.present? }&.result)
|
|
55
|
+
},
|
|
56
|
+
sandbox_session_id: @sandbox.session_id,
|
|
57
|
+
sandbox_status: @sandbox.status,
|
|
58
|
+
project: project ? { id: project.id, name: project.name } : nil,
|
|
59
|
+
credential_mode: @root.credential_mode,
|
|
60
|
+
created_at: @root.created_at.iso8601,
|
|
61
|
+
last_activity_at: (@turns.filter_map { |turn| turn.finished_at || turn.started_at }.max || last.created_at).iso8601
|
|
62
|
+
}
|
|
63
|
+
end
|
|
64
|
+
|
|
65
|
+
# The summary plus every turn: its summary, and its transcript from
|
|
66
|
+
# +after+[turn id] onward, so a client polling the running turn asks only
|
|
67
|
+
# for what it has not seen. A finished turn's diff comes from the session
|
|
68
|
+
# endpoint, on demand: it is the whole checkout's.
|
|
69
|
+
def detail(after: {})
|
|
70
|
+
summary.merge(
|
|
71
|
+
turns: @turns.map do |turn|
|
|
72
|
+
from = after[turn.id.to_s].to_i.clamp(0, turn.events.size)
|
|
73
|
+
turn.summary.merge(events: turn.events.drop(from), events_offset: from,
|
|
74
|
+
dropped_events_count: turn.dropped_events_count.to_i)
|
|
75
|
+
end,
|
|
76
|
+
browser: @sandbox.respond_to?(:browser_summary) ? @sandbox.browser_summary : nil
|
|
77
|
+
)
|
|
78
|
+
end
|
|
79
|
+
|
|
80
|
+
private
|
|
81
|
+
|
|
82
|
+
def active
|
|
83
|
+
@turns.find { |turn| turn.running? } || @turns.find { |turn| turn.queued? }
|
|
84
|
+
end
|
|
85
|
+
end
|
|
86
|
+
end
|
|
@@ -125,8 +125,19 @@ module ActionAgent
|
|
|
125
125
|
# derived from every persisted result's diagnosis, which older runs
|
|
126
126
|
# recorded in earlier shapes, so a run they fail for still serves its
|
|
127
127
|
# results. Without +links+ the item paths are relative to the mount.
|
|
128
|
-
|
|
129
|
-
|
|
128
|
+
# With +markdown+ each item carries its `markdown`
|
|
129
|
+
# (ActiveAgent::Evals::ReportMarkdown#fix_item_markdown), which the
|
|
130
|
+
# dashboard's fix cards copy.
|
|
131
|
+
def fix_items(run, links: nil, markdown: false)
|
|
132
|
+
unless markdown
|
|
133
|
+
return links ? run.fix_items(links: links) : run.fix_items
|
|
134
|
+
end
|
|
135
|
+
|
|
136
|
+
report = links ? run.to_report(links: links) : run.to_report
|
|
137
|
+
items = report.fix_items
|
|
138
|
+
return items unless report.respond_to?(:fix_item_markdown)
|
|
139
|
+
|
|
140
|
+
items.map { |item| item.merge("markdown" => report.fix_item_markdown(item)) }
|
|
130
141
|
rescue StandardError => e
|
|
131
142
|
Rails.logger.warn(
|
|
132
143
|
"[ActionAgent] evaluation run #{run.id} fix_items failed: #{e.class}: #{e.message}"
|
|
@@ -537,6 +537,12 @@ module ActionAgent
|
|
|
537
537
|
}
|
|
538
538
|
when "call_agent"
|
|
539
539
|
call_agent(slug: kwargs[:slug], message: kwargs[:message])
|
|
540
|
+
when "translate_text"
|
|
541
|
+
translate_text(**kwargs.to_h.symbolize_keys.slice(:text, :target_language, :source_language))
|
|
542
|
+
when *CheckoutTools::FUNCTIONS
|
|
543
|
+
checkout_tools.call(name.to_s, kwargs)
|
|
544
|
+
when *SlackTools::FUNCTIONS
|
|
545
|
+
SlackTools.new(SlackTools.token_for(owner)).call(name.to_s, kwargs)
|
|
540
546
|
when "request_handoff"
|
|
541
547
|
request_handoff(**kwargs.slice(:reason, :url, :form_values, :instructions))
|
|
542
548
|
else
|
|
@@ -727,6 +733,37 @@ module ActionAgent
|
|
|
727
733
|
(text.length / 4.0).round
|
|
728
734
|
end
|
|
729
735
|
|
|
736
|
+
# The checkout tools act on the sandbox the run was given, found among
|
|
737
|
+
# the agent's owner's sessions as the dispatcher finds its runtime.
|
|
738
|
+
def checkout_tools
|
|
739
|
+
@checkout_tools ||= begin
|
|
740
|
+
sandbox_id = @run.try(:sandbox_id)
|
|
741
|
+
CheckoutTools.new(sandbox_id && SandboxSession.for_owner(owner).find_by(session_id: sandbox_id))
|
|
742
|
+
end
|
|
743
|
+
end
|
|
744
|
+
|
|
745
|
+
# translate_text: one generation on this run's provider, model and the
|
|
746
|
+
# owner's credentials, with no tools and no conversation.
|
|
747
|
+
def translate_text(text:, target_language:, source_language: nil)
|
|
748
|
+
return { error: "No text to translate" } if text.to_s.strip.empty?
|
|
749
|
+
return { error: "target_language is required" } if target_language.to_s.strip.empty?
|
|
750
|
+
|
|
751
|
+
name = provider
|
|
752
|
+
options = { model: requested_model }.merge(owner_provider_options(name))
|
|
753
|
+
translator = Class.new(ActiveAgent::Base) do
|
|
754
|
+
define_singleton_method(:name) { "ActionAgent::Translator" }
|
|
755
|
+
generate_with name, **options
|
|
756
|
+
end
|
|
757
|
+
from = source_language.present? ? " from #{source_language}" : ""
|
|
758
|
+
instructions = "Translate the user's text#{from} into #{target_language}. Reply with the translation only — " \
|
|
759
|
+
"no notes, quotes or preamble. Keep formatting, code, URLs and placeholders exactly as they are."
|
|
760
|
+
response = translator.prompt(message: text.to_s, instructions: instructions).generate_now
|
|
761
|
+
|
|
762
|
+
{ translation: response&.message&.content.to_s, target_language: target_language }
|
|
763
|
+
rescue StandardError => e
|
|
764
|
+
{ error: "Translation failed: #{e.message}" }
|
|
765
|
+
end
|
|
766
|
+
|
|
730
767
|
def call_agent(slug:, message:)
|
|
731
768
|
depth = Thread.current[:agent_call_depth].to_i
|
|
732
769
|
return { error: "call_agent depth limit (#{MAX_CALL_DEPTH}) reached" } if depth >= MAX_CALL_DEPTH
|
|
@@ -50,18 +50,25 @@ module ActionAgent
|
|
|
50
50
|
# workspace already talks to, then the rest of the catalog.
|
|
51
51
|
STATUS_RANK = { "active" => 0, "configured" => 1, "available" => 2, "idle" => 3 }.freeze
|
|
52
52
|
|
|
53
|
-
attr_reader :agent, :discovery, :runtimes
|
|
53
|
+
attr_reader :agent, :discovery, :runtimes, :workspace_agents
|
|
54
54
|
|
|
55
55
|
# @param agent [ActionAgent::Agent] the agent being edited
|
|
56
56
|
# @param traces [ActiveRecord::Relation] the traces the caller may read
|
|
57
57
|
# @param hours [Integer] the window the usage columns are scoped to
|
|
58
58
|
# @param runtimes [Array<Hash>] live checkout sandbox runtimes this agent
|
|
59
59
|
# can be given, as SandboxSession.runtime_server_listings returns them
|
|
60
|
-
|
|
60
|
+
# @param workspace_agents [ActiveRecord::Relation, nil] the workspace's
|
|
61
|
+
# other agents, whose MCP servers each service row counts as
|
|
62
|
+
# +workspace_agents+ — what a new agent's builder shows as already in
|
|
63
|
+
# use, since an unsaved agent has no usage of its own to show
|
|
64
|
+
def initialize(agent:, traces:, hours: ToolDiscovery::DEFAULT_WINDOW_HOURS, runtimes: [], workspace_agents: nil)
|
|
61
65
|
@agent = agent
|
|
62
66
|
@runtimes = Array(runtimes).index_by { |runtime| runtime[:key] }
|
|
67
|
+
@workspace_agents = workspace_agents
|
|
68
|
+
# An agent not saved yet has no traces of its own, and no name to
|
|
69
|
+
# match unattributed ones by.
|
|
63
70
|
@discovery = ToolDiscovery.new(
|
|
64
|
-
traces: agent.telemetry_traces(traces),
|
|
71
|
+
traces: agent.persisted? ? agent.telemetry_traces(traces) : traces.none,
|
|
65
72
|
agents: Agent.where(id: agent.id),
|
|
66
73
|
hours: hours
|
|
67
74
|
)
|
|
@@ -118,6 +125,7 @@ module ActionAgent
|
|
|
118
125
|
used = detected_by_server[key] || []
|
|
119
126
|
calls = used.sum { |tool| tool[:calls] }
|
|
120
127
|
enabled = configured_servers.key?(key)
|
|
128
|
+
unreachable = MCPToolDispatcher.unreachable_reason(MCPCatalog.connection(key) || runtimes[key])
|
|
121
129
|
|
|
122
130
|
{
|
|
123
131
|
key: key,
|
|
@@ -133,10 +141,25 @@ module ActionAgent
|
|
|
133
141
|
calls: calls,
|
|
134
142
|
errors: used.sum { |tool| tool[:errors] },
|
|
135
143
|
last_seen: used.filter_map { |tool| tool[:last_seen] }.max,
|
|
144
|
+
# Whether a dashboard run reaches it (MCPToolDispatcher), and when
|
|
145
|
+
# not, why: a stdio server, or one needing a credential it can't send.
|
|
146
|
+
callable: unreachable.nil?,
|
|
147
|
+
unavailable_reason: unreachable,
|
|
148
|
+
workspace_agents: workspace_usage[key].to_i,
|
|
136
149
|
tools: service_tools(key, catalog, used)
|
|
137
150
|
}
|
|
138
151
|
end
|
|
139
152
|
|
|
153
|
+
# server key => how many of the workspace's agents enable it. Empty
|
|
154
|
+
# unless the caller handed in +workspace_agents+.
|
|
155
|
+
def workspace_usage
|
|
156
|
+
@workspace_usage ||= Array(workspace_agents).each_with_object(Hash.new(0)) do |other, counts|
|
|
157
|
+
next if agent.persisted? && other.id == agent.id
|
|
158
|
+
|
|
159
|
+
self.class.configured_servers(other).each_key { |key| counts[key] += 1 }
|
|
160
|
+
end
|
|
161
|
+
end
|
|
162
|
+
|
|
140
163
|
# How to reach the server, in the one line the expanded panel shows:
|
|
141
164
|
# "Streamable HTTP · <url>" for the ones the dashboard can call,
|
|
142
165
|
# "sandbox · <command>" for the ones it can start, "stdio · <command>"
|
|
@@ -233,6 +256,10 @@ module ActionAgent
|
|
|
233
256
|
description: Agent::TOOL_DESCRIPTIONS[capability],
|
|
234
257
|
enabled: saved_tools.include?(capability),
|
|
235
258
|
editable: true,
|
|
259
|
+
# Whether AgentToolbox gives the model anything for it. A capability
|
|
260
|
+
# without definitions is a name on the roster and nothing more, so
|
|
261
|
+
# the builder offers its MCP service (or nothing) in its place.
|
|
262
|
+
implemented: AgentToolbox::DEFINITIONS.key?(capability),
|
|
236
263
|
approval_names: capability_functions(capability) - AgentToolbox::INPUT_FUNCTIONS
|
|
237
264
|
)
|
|
238
265
|
end
|
|
@@ -283,27 +310,35 @@ module ActionAgent
|
|
|
283
310
|
# carries a top-level Hash keyed by server name — the same three shapes
|
|
284
311
|
# EvaluationToolResolver tolerates.
|
|
285
312
|
def configured_servers
|
|
286
|
-
@configured_servers ||=
|
|
287
|
-
key = entry_key(entry)
|
|
288
|
-
map[key] = entry if key.present?
|
|
289
|
-
end
|
|
313
|
+
@configured_servers ||= self.class.configured_servers(agent)
|
|
290
314
|
end
|
|
291
315
|
|
|
292
|
-
|
|
293
|
-
|
|
316
|
+
class << self
|
|
317
|
+
# server key => the entry +agent+ stores for it, in any of the three
|
|
318
|
+
# shapes above.
|
|
319
|
+
def configured_servers(agent)
|
|
320
|
+
configured_entries(agent.mcp_servers).each_with_object({}) do |entry, map|
|
|
321
|
+
key = entry_key(entry)
|
|
322
|
+
map[key] = entry if key.present?
|
|
323
|
+
end
|
|
324
|
+
end
|
|
325
|
+
|
|
326
|
+
private
|
|
294
327
|
|
|
295
|
-
|
|
296
|
-
servers.
|
|
297
|
-
|
|
298
|
-
|
|
328
|
+
def configured_entries(servers)
|
|
329
|
+
if servers.is_a?(Hash)
|
|
330
|
+
servers.map { |key, value| value.respond_to?(:key?) ? value.to_h.stringify_keys.merge("key" => key.to_s) : key.to_s }
|
|
331
|
+
else
|
|
332
|
+
Array(servers)
|
|
333
|
+
end
|
|
299
334
|
end
|
|
300
|
-
end
|
|
301
335
|
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
|
|
336
|
+
def entry_key(entry)
|
|
337
|
+
return entry.to_s.strip.presence if entry.is_a?(String) || entry.is_a?(Symbol)
|
|
338
|
+
return nil unless entry.respond_to?(:key?)
|
|
305
339
|
|
|
306
|
-
|
|
340
|
+
(entry["key"] || entry[:key] || entry["name"] || entry[:name]).to_s.strip.presence
|
|
341
|
+
end
|
|
307
342
|
end
|
|
308
343
|
|
|
309
344
|
# The per-server allow-list, or nil when the entry names none — which
|