actionagent 1.9.1 → 1.9.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (50) hide show
  1. checksums.yaml +4 -4
  2. data/README.md +23 -0
  3. data/app/assets/builds/action_agent.css +1 -1
  4. data/app/assets/builds/action_agent.js +61 -61
  5. data/app/controllers/action_agent/api/agents_controller.rb +23 -0
  6. data/app/controllers/action_agent/api/code_sessions_controller.rb +49 -10
  7. data/app/controllers/action_agent/api/evaluation_reports_controller.rb +66 -2
  8. data/app/controllers/action_agent/api/evaluations_controller.rb +56 -6
  9. data/app/controllers/action_agent/api/interactions_controller.rb +55 -1
  10. data/app/controllers/action_agent/api/sandboxes_controller.rb +13 -2
  11. data/app/controllers/action_agent/api/trace_reports_controller.rb +11 -6
  12. data/app/controllers/concerns/action_agent/api/mcp_dashboard_tools.rb +2 -2
  13. data/app/jobs/action_agent/code_session_job.rb +19 -2
  14. data/app/jobs/action_agent/evaluation_judge_job.rb +39 -0
  15. data/app/jobs/action_agent/evaluation_replay_job.rb +28 -0
  16. data/app/models/action_agent/agent.rb +54 -1
  17. data/app/models/action_agent/code_session.rb +80 -0
  18. data/app/models/action_agent/evaluation_run.rb +37 -5
  19. data/app/models/action_agent/provider_key.rb +5 -1
  20. data/app/queries/action_agent/trace_window_report.rb +60 -0
  21. data/app/serializers/action_agent/code_conversation_serializer.rb +86 -0
  22. data/app/serializers/action_agent/evaluation_serializer.rb +13 -2
  23. data/app/services/action_agent/agent_execution_service.rb +37 -0
  24. data/app/services/action_agent/agent_tool_roster.rb +53 -18
  25. data/app/services/action_agent/agent_toolbox.rb +156 -3
  26. data/app/services/action_agent/browser_client_token.rb +86 -0
  27. data/app/services/action_agent/checkout_tools.rb +154 -0
  28. data/app/services/action_agent/code_conversation.rb +133 -0
  29. data/app/services/action_agent/codex_auth.rb +81 -0
  30. data/app/services/action_agent/dashboard_assistant_service.rb +19 -1
  31. data/app/services/action_agent/diff_stat.rb +27 -0
  32. data/app/services/action_agent/evaluation_fix_comparison.rb +4 -1
  33. data/app/services/action_agent/evaluation_report_export.rb +175 -0
  34. data/app/services/action_agent/evaluation_report_import.rb +47 -19
  35. data/app/services/action_agent/evaluation_runner_service.rb +106 -14
  36. data/app/services/action_agent/local_sandbox_backend.rb +295 -30
  37. data/app/services/action_agent/mcp_catalog.rb +25 -0
  38. data/app/services/action_agent/mcp_client.rb +1 -0
  39. data/app/services/action_agent/mcp_tool_dispatcher.rb +36 -1
  40. data/app/services/action_agent/project_boot_fix.rb +10 -3
  41. data/app/services/action_agent/sandbox_orchestrator.rb +41 -1
  42. data/app/services/action_agent/scenario_evaluation_runner.rb +381 -3
  43. data/app/services/action_agent/slack_tools.rb +101 -0
  44. data/config/routes.rb +10 -1
  45. data/lib/action_agent/engine.rb +6 -0
  46. data/lib/action_agent/version.rb +1 -1
  47. data/lib/action_agent.rb +50 -2
  48. data/lib/generators/action_agent/templates/migrations/028_add_conversations_to_code_sessions.rb.erb +18 -0
  49. data/lib/tasks/evaluations.rake +78 -0
  50. metadata +15 -2
@@ -51,6 +51,7 @@ module ActionAgent
51
51
  # Callbacks
52
52
  before_validation :generate_slug, on: :create
53
53
  before_validation :apply_conventional_schema_tools, on: :create
54
+ before_validation :require_approval_for_gained_side_effects, unless: :observed?
54
55
  after_create :create_initial_version
55
56
  after_update :create_version_on_config_change, if: :configuration_changed?
56
57
  after_destroy :release_telemetry_traces, if: :observed?
@@ -83,6 +84,30 @@ module ActionAgent
83
84
  github ruby rails aws gcp python typescript docker kubernetes
84
85
  ].freeze
85
86
 
87
+ # What each instruction set adds to the instructions a run executes
88
+ # under (#composed_instructions_for): the conventions of that stack, as a
89
+ # developer message the model can act on.
90
+ INSTRUCTION_SET_GUIDANCE = {
91
+ "github" => "Work the GitHub way: small focused commits with messages that say why, branches named for the " \
92
+ "change, pull requests that describe what changed and how it was verified, and issues referenced by number.",
93
+ "ruby" => "Write idiomatic Ruby: follow the project's RuboCop configuration, prefer small methods and plain " \
94
+ "objects, use keyword arguments for options, and add or update tests (Minitest or RSpec, as the project uses).",
95
+ "rails" => "Follow Rails conventions: fat models and thin controllers, strong parameters, migrations for every " \
96
+ "schema change, scopes and validations on the model, and bin/rails commands for generators, tests and tasks.",
97
+ "aws" => "For AWS, prefer least-privilege IAM, keep infrastructure in code (CloudFormation, CDK or Terraform), " \
98
+ "name the region explicitly, and never print or commit credentials.",
99
+ "gcp" => "For Google Cloud, prefer least-privilege service accounts, keep infrastructure in code (Terraform), " \
100
+ "name the project and region explicitly, and use Secret Manager rather than plain environment values.",
101
+ "python" => "Write idiomatic Python: PEP 8, type hints on public functions, the project's own environment and " \
102
+ "dependency tool, and pytest tests for new behaviour.",
103
+ "typescript" => "Write strict TypeScript: no implicit any, explicit types on exported APIs, the project's lint and " \
104
+ "format settings, and tests in the framework the project already uses.",
105
+ "docker" => "For Docker, keep images small and reproducible: pinned base image versions, multi-stage builds, a " \
106
+ "non-root user, and no secrets baked into layers.",
107
+ "kubernetes" => "For Kubernetes, write declarative manifests with resource requests and limits, readiness and " \
108
+ "liveness probes, and Secrets for credentials; apply changes namespace by namespace."
109
+ }.freeze
110
+
86
111
  # Built-in tools/MCPs. Host-declared schema tools are offered alongside
87
112
  # these — see .available_tools, which is what the editor and the APIs
88
113
  # serialize. This constant stays the built-in set so existing references
@@ -156,6 +181,24 @@ module ActionAgent
156
181
  self.tools = defaults if defaults.any?
157
182
  end
158
183
 
184
+ # run_command, write_file, edit_file and slack_post_message wait for a
185
+ # person's approval unless the agent's approval list says otherwise
186
+ # (AgentToolbox::APPROVAL_BY_DEFAULT). They join the list when the agent
187
+ # gains their capability and this save brought no approval list of its
188
+ # own: an agent built from a template, a POST to the agents API with
189
+ # tools alone, a PATCH that enables a capability. The builder and the
190
+ # Tools tab send the list, so an Approval switch turned off holds.
191
+ def require_approval_for_gained_side_effects
192
+ return unless has_attribute?(:approval_required_tools)
193
+ return if attribute_came_from_user?("approval_required_tools")
194
+
195
+ gained = Array(tools).map(&:to_s) - Array(tools_in_database).map(&:to_s)
196
+ names = AgentToolbox.default_approvals(gained)
197
+ return if names.empty?
198
+
199
+ self[:approval_required_tools] = approval_required_tools | names
200
+ end
201
+
159
202
  # The ActiveAgent class name this agent's runs are recorded under — the
160
203
  # correlation key between platform Agent records and telemetry traces
161
204
  # (TelemetryTrace#agent_class) and solid_agent contexts. An observed
@@ -229,9 +272,19 @@ module ActionAgent
229
272
  # The system instructions an action executes under: named actions stack
230
273
  # their prompt below the agent's base instructions; the default action
231
274
  # uses the base instructions alone.
275
+ #
276
+ # The agent's instruction sets come last, as the conventions to work by.
232
277
  def composed_instructions_for(action_name)
233
278
  action = action_prompt_for(action_name)
234
- [ instructions, action&.dig("prompt") ].map(&:presence).compact.join("\n\n").presence
279
+ [ instructions, action&.dig("prompt"), instruction_set_guidance ].map(&:presence).compact.join("\n\n").presence
280
+ end
281
+
282
+ # The guidance of the instruction sets this agent selected, or nil.
283
+ def instruction_set_guidance
284
+ lines = Array(instruction_sets).filter_map { |set| INSTRUCTION_SET_GUIDANCE[set.to_s] }
285
+ return nil if lines.empty?
286
+
287
+ "Conventions to follow:\n#{lines.map { |line| "- #{line}" }.join("\n")}"
235
288
  end
236
289
 
237
290
  # Returns the configuration as a hash for versioning
@@ -6,6 +6,12 @@ module ActionAgent
6
6
  # outcome Claude Code reported, and the diff the session left in the
7
7
  # working tree.
8
8
  #
9
+ # A session is one turn of a conversation (CodeConversation). The first
10
+ # turn is the conversation's root; each later one names it
11
+ # (conversation_root_id), runs after the one before it, and resumes the same
12
+ # CLI session (resume_session_id), so Claude Code or Codex remembers what
13
+ # came before. Every existing session is a conversation of one turn.
14
+ #
9
15
  # The transcript is scrubbed of the sandbox's secrets (its GitHub token and
10
16
  # Claude Code credential) before it is stored, and bounded: long strings
11
17
  # are truncated and events past MAX_EVENTS are counted rather than kept.
@@ -17,6 +23,9 @@ module ActionAgent
17
23
  belongs_to :evaluation_run, optional: true
18
24
  belongs_to :verification_run, class_name: "EvaluationRun", optional: true
19
25
  belongs_to :previous_code_session, class_name: "CodeSession", optional: true
26
+ belongs_to :conversation_root, class_name: "CodeSession", optional: true
27
+ has_many :follow_up_turns, -> { order(:turn_index, :id) }, class_name: "CodeSession",
28
+ foreign_key: :conversation_root_id, inverse_of: :conversation_root
20
29
 
21
30
  enum :status, { queued: 0, running: 1, succeeded: 2, failed: 3, cancelled: 4 }
22
31
 
@@ -26,11 +35,32 @@ module ActionAgent
26
35
  MAX_EVENT_STRING = 4_000
27
36
  MAX_DIFF_BYTES = 500_000
28
37
  RUNNERS = %w[claude_code codex].freeze
38
+ # Where a conversation was started. Older sessions recorded none; see
39
+ # #origin.
40
+ ORIGINS = %w[settings boot_fix evaluation_fix chat].freeze
41
+ # A Claude Code session id, as --session-id and --resume take it.
42
+ SESSION_UUID = /\A\h{8}-\h{4}-\h{4}-\h{4}-\h{12}\z/
43
+ # A note from the dashboard is fenced in this tag ahead of the person's
44
+ # message, and the tag is taken out of anything the person typed.
45
+ NOTE_TAG = "dashboard-note"
29
46
 
30
47
  validates :prompt, presence: true, length: { maximum: MAX_PROMPT_CHARACTERS }
31
48
  validates :runner, inclusion: { in: RUNNERS }
32
49
 
50
+ validates :origin, inclusion: { in: ORIGINS }, allow_nil: true
51
+ # A Codex conversation resumes the thread id its first turn reported.
52
+ validates :resume_session_id, format: { with: SESSION_UUID }, allow_nil: true, if: -> { runner == "claude_code" }
53
+
54
+ # A conversation's first Claude Code turn names the CLI session it
55
+ # starts (--session-id), so every later turn can resume it even when the
56
+ # first ended before reporting its own id.
57
+ before_validation(on: :create) do
58
+ self.resume_session_id ||= SecureRandom.uuid if root? && runner == "claude_code"
59
+ end
60
+
33
61
  scope :recent, -> { order(created_at: :desc) }
62
+ # The first turn of every conversation.
63
+ scope :conversation_roots, -> { where(conversation_root_id: nil) }
34
64
 
35
65
  # JSON columns carry no default on MySQL or SQLite (see the migration).
36
66
  def events
@@ -41,6 +71,51 @@ module ActionAgent
41
71
  succeeded? || failed? || cancelled?
42
72
  end
43
73
 
74
+ def root?
75
+ conversation_root_id.nil?
76
+ end
77
+
78
+ # The conversation's first turn: this one, or the one it names.
79
+ def root
80
+ root? ? self : conversation_root
81
+ end
82
+
83
+ # Every turn of this session's conversation, in order.
84
+ #
85
+ # @return [Array<CodeSession>]
86
+ def turns
87
+ # Read afresh: turns are added under the sandbox's lock while this
88
+ # session may hold an older list.
89
+ [ root, *CodeSession.where(conversation_root_id: root.id).order(:turn_index, :id).to_a ]
90
+ end
91
+
92
+ # Where the conversation was started. Sessions from before origins were
93
+ # recorded are told apart by what they ran for: an evaluation fix names
94
+ # its run, and a boot fix is one its project recorded
95
+ # (ProjectBootFix.project_for).
96
+ def origin
97
+ recorded = super.presence
98
+ return recorded if recorded
99
+ return "evaluation_fix" if evaluation_run_id
100
+ return "boot_fix" if persisted? && root? && ProjectBootFix.project_for(self)
101
+
102
+ "settings"
103
+ end
104
+
105
+ # What the CLI is given on stdin: the dashboard's note, fenced, ahead of
106
+ # the person's message. The prompt itself stays what they typed.
107
+ def stdin_text
108
+ return prompt if context_note.blank?
109
+
110
+ "<#{NOTE_TAG}>\n#{context_note}\n</#{NOTE_TAG}>\n\n#{prompt}"
111
+ end
112
+
113
+ # +text+ without the dashboard's note tags, so a person cannot make their
114
+ # own words read as the dashboard's.
115
+ def self.strip_note_tags(text)
116
+ text.to_s.gsub(%r{</?\s*#{NOTE_TAG}[^>]*>}i, "")
117
+ end
118
+
44
119
  # Whether an outcome and a diff may still be recorded for this session.
45
120
  # finished_at is set once nothing more will be: by CodeSessionJob when it
46
121
  # settles a session it ran (whatever the backend returned, raised or
@@ -111,6 +186,10 @@ module ActionAgent
111
186
  evaluation_run_id: evaluation_run_id,
112
187
  evaluation_id: evaluation_run&.evaluation_id,
113
188
  fix_item: fix_item,
189
+ conversation_id: root? ? id : conversation_root_id,
190
+ turn_index: turn_index,
191
+ origin: origin,
192
+ context_note: context_note,
114
193
  verification_run_id: verification_run_id,
115
194
  verification_error: verification_error,
116
195
  result: result,
@@ -121,6 +200,7 @@ module ActionAgent
121
200
  input_tokens: input_tokens,
122
201
  output_tokens: output_tokens,
123
202
  event_count: events.size + dropped_events_count.to_i,
203
+ diff_stat: finished? ? DiffStat.call(diff) : nil,
124
204
  # What a client polls until: false once nothing about the session
125
205
  # changes any more.
126
206
  diff_pending: diff_pending?,
@@ -78,6 +78,15 @@ module ActionAgent
78
78
  verdict.to_h.stringify_keys.presence if verdict.is_a?(Hash)
79
79
  end
80
80
 
81
+ # The model comparison a comparison run recorded (Report#comparison):
82
+ # each model's pass rate with its interval, every pair's difference and
83
+ # whether it is decisive, and the conclusion. nil for a single-model
84
+ # run, or one recorded before runs kept it.
85
+ def recorded_comparison
86
+ comparison = scores&.dig("_comparison")
87
+ comparison if comparison.is_a?(Hash) && comparison.present?
88
+ end
89
+
81
90
  # How the report names the judge, the way the suite panel does: the
82
91
  # judge the recorded verdict names — unless that is only the pass-rate
83
92
  # ranking — else the evaluation's judge model. nil when neither is set,
@@ -191,6 +200,16 @@ module ActionAgent
191
200
  cost_breakdown.usage
192
201
  end
193
202
 
203
+ # The judge's run-level spend as recorded, without pricing anything:
204
+ # what the publishing application reported for the calls no result owns,
205
+ # else the engine's meter, else nil.
206
+ def recorded_judge_usage_run
207
+ reported = scores&.dig("_judge_usage_run")
208
+ return reported if reported.is_a?(Hash)
209
+
210
+ judge_usage_meter&.merge("source" => "meter")
211
+ end
212
+
194
213
  # Route templates for the report's fix item actions, relative to the
195
214
  # dashboard mount: `%{key}` is filled in per MCP server by the report.
196
215
  # The JSON API leaves `mount` empty — the React app resolves paths
@@ -220,7 +239,7 @@ module ActionAgent
220
239
  # writes with Report#to_html. The run's recorded verdict and judge go
221
240
  # with it: the report is not to re-rank the rebuilt results by pass
222
241
  # rate and show a different judge's pick, verdict or `judged by` than
223
- # the suite panel does. Raises ActiveRecord::RecordNotFound via the
242
+ # the suite panel does, nor resample the comparison the run recorded. Raises ActiveRecord::RecordNotFound via the
224
243
  # caller for a run of a generation-sampling evaluation, which has no
225
244
  # scenario results to report on.
226
245
  #
@@ -229,12 +248,19 @@ module ActionAgent
229
248
  # judge's run-level spend, so the page shows every cost the dashboard
230
249
  # does. `estimate: false` rebuilds the report from what was recorded
231
250
  # alone — the import summarizes a published run that way, so the
232
- # summaries it stores stay the application's own figures.
251
+ # summaries it stores stay the application's own figures, and the
252
+ # export (EvaluationReportExport) serves it that way, so the same run
253
+ # always exports the same report. The judge's run-level spend is then
254
+ # the one recorded: the application's for an imported run, the meter's
255
+ # for one the engine judged.
233
256
  #
234
257
  # The engine runs against every activeagent since 1.4, so the report
235
258
  # kwargs that arrived later are passed only when Report.new takes them.
236
259
  def to_report(links: report_links, estimate: true)
237
- rows = scenario_results.includes(:scenario).sort_by do |row|
260
+ # A replay not yet scored (a fanned-out run in progress) is not a
261
+ # result the report can grade, so it is left out rather than read as
262
+ # a failure.
263
+ rows = scenario_results.where.not(status: :pending).includes(:scenario).sort_by do |row|
238
264
  [ row.evaluated_scenario["position"].to_i, row.evaluation_scenario_id, row.model ]
239
265
  end
240
266
  breakdown = cost_breakdown if estimate
@@ -274,8 +300,10 @@ module ActionAgent
274
300
  agent_name: evaluation.agent&.name,
275
301
  links: links
276
302
  }
277
- kwargs[:judge_usage] = breakdown.judge_usage_run if breakdown && self.class.report_accepts?(:judge_usage)
303
+ judge_usage_run = breakdown ? breakdown.judge_usage_run : recorded_judge_usage_run
304
+ kwargs[:judge_usage] = judge_usage_run if judge_usage_run && self.class.report_accepts?(:judge_usage)
278
305
  kwargs[:release] = release_summary if release_summary && self.class.report_accepts?(:release)
306
+ kwargs[:comparison] = recorded_comparison if recorded_comparison && self.class.report_accepts?(:comparison)
279
307
  self.class.report_class.new(**kwargs)
280
308
  end
281
309
 
@@ -319,7 +347,11 @@ module ActionAgent
319
347
  # for it, and — when estimating — how its cost was priced.
320
348
  def replay_metadata_for(row, breakdown)
321
349
  metadata = row.replay_metadata.dup
322
- reported_judge = row.diagnosis.is_a?(Hash) ? row.diagnosis["_judge_usage"] : nil
350
+ # A fanned-out run's results each keep what their scoring job spent on
351
+ # the judge, already folded into the run's meter. The meter is the
352
+ # run's figure (EvaluationRunCost#result_judge reads it the same way),
353
+ # so the parts are not counted a second time.
354
+ reported_judge = row.diagnosis.is_a?(Hash) && !judge_usage_meter ? row.diagnosis["_judge_usage"] : nil
323
355
  metadata["judge_usage"] = reported_judge if reported_judge.is_a?(Hash)
324
356
  return metadata unless breakdown
325
357
 
@@ -39,7 +39,8 @@ module ActionAgent
39
39
  HOST_PROVIDERS = %w[ollama].freeze
40
40
  # Tools connected with a credential, configured beside the providers
41
41
  # (Settings -> Integrations) but never offered to the agent builder.
42
- CONNECTION_PROVIDERS = %w[claude_code codex].freeze
42
+ # Slack's is the bot token the agents' slack tools post with (SlackTools).
43
+ CONNECTION_PROVIDERS = %w[claude_code codex slack].freeze
43
44
  PROVIDERS = (KEY_PROVIDERS + HOST_PROVIDERS + CONNECTION_PROVIDERS).freeze
44
45
 
45
46
  # Only an Anthropic API key (sk-ant-api03-…, from the Claude Console or
@@ -52,6 +53,7 @@ module ActionAgent
52
53
  # dashboard never touches the credential.
53
54
  CLAUDE_CODE_CREDENTIAL = /\Ask-ant-api\d{2}-[A-Za-z0-9_-]+\z/
54
55
  CODEX_CREDENTIAL = /\Ask-(?!ant-|or-)[A-Za-z0-9_-]+\z/
56
+ SLACK_CREDENTIAL = /\Axoxb-[A-Za-z0-9-]+\z/
55
57
  # A Claude subscription token, as earlier versions stored. Recognized so
56
58
  # a stored one is never handed out (see #needs_replacing?).
57
59
  SUBSCRIPTION_TOKEN_PREFIX = "sk-ant-oat"
@@ -95,6 +97,8 @@ module ActionAgent
95
97
  }, if: -> { provider == "claude_code" }
96
98
  validates :credential, format: { with: CODEX_CREDENTIAL, message: "must be an OpenAI API key (sk-…)" },
97
99
  if: -> { provider == "codex" }
100
+ validates :credential, format: { with: SLACK_CREDENTIAL, message: "must be a Slack bot token (xoxb-…)" },
101
+ if: -> { provider == "slack" }
98
102
  validates :api_key, length: { maximum: 500 }, allow_nil: true
99
103
 
100
104
  # Deletes every Claude Code connection that still holds a Claude
@@ -0,0 +1,60 @@
1
+ # frozen_string_literal: true
2
+
3
+ module ActionAgent
4
+ # Full-window counts and chart buckets, independent of the capped trace
5
+ # details. Read only scalar columns, never the potentially large spans.
6
+ class TraceWindowReport
7
+ MAX_BUCKETS = 180
8
+
9
+ def initialize(scope, from:, to:, bucket_seconds:)
10
+ @scope, @from, @to = scope, from, to
11
+ duration = (to - from).ceil
12
+ @bucket_seconds = bucket_seconds.to_i.clamp([ (duration.to_f / MAX_BUCKETS).ceil, 1 ].max, duration)
13
+ end
14
+
15
+ def as_json
16
+ width = @bucket_seconds
17
+ buckets = Array.new(((@to - @from) / width).ceil) do |index|
18
+ tally.merge(start_ms: milliseconds(@from + index * width), end_ms: milliseconds([ @from + (index + 1) * width, @to ].min), agents: {})
19
+ end
20
+ total = tally
21
+ @scope.in_batches(of: 1000) do |batch|
22
+ batch.pluck(:timestamp, :agent_class, :status, :total_duration_ms).each do |timestamp, agent, status, duration|
23
+ index = ((timestamp - @from) / width).floor
24
+ next unless timestamp < @to && index >= 0 && index < buckets.size
25
+
26
+ [ total, buckets[index] ].each do |counts|
27
+ counts[:requests] += 1
28
+ counts[:errors] += 1 if status == TelemetryTrace::STATUS_ERROR
29
+ if duration && duration.finite?
30
+ counts[:duration_sum] += duration
31
+ counts[:duration_count] += 1
32
+ end
33
+ end
34
+ agents = buckets[index][:agents]
35
+ agents[agent] = agents.fetch(agent, 0) + 1 if agent
36
+ end
37
+ end
38
+ {
39
+ total_count: total[:requests],
40
+ summary: summary(total).merge(throughput_per_minute: (total[:requests] * 60.0 / (@to - @from)).round(1)),
41
+ window_start_ms: milliseconds(@from), window_end_ms: milliseconds(@to), bucket_seconds: width,
42
+ buckets: buckets.map { |bucket| summary(bucket).merge(bucket.slice(:start_ms, :end_ms, :agents)) }
43
+ }
44
+ end
45
+
46
+ private
47
+
48
+ def tally
49
+ { requests: 0, errors: 0, duration_sum: 0.0, duration_count: 0 }
50
+ end
51
+
52
+ def summary(counts)
53
+ counts.slice(:requests, :errors).merge(avg_latency_ms: counts[:duration_count].positive? ? (counts[:duration_sum] / counts[:duration_count]).round : 0)
54
+ end
55
+
56
+ def milliseconds(time)
57
+ (time.to_f * 1000).to_i
58
+ end
59
+ end
60
+ end
@@ -0,0 +1,86 @@
1
+ # frozen_string_literal: true
2
+
3
+ module ActionAgent
4
+ # A Claude Code or Codex conversation (CodeConversation) in the shape the
5
+ # Interactions view lists: one row per conversation, keyed "code-<root id>"
6
+ # beside contexts ("<id>") and reported traces ("trace-<id>"), and, in
7
+ # detail, its turns with their transcripts.
8
+ #
9
+ # Each turn is one run of the CLI, so it carries what a trace would: its
10
+ # tokens, cost and duration, from the CLI's result event.
11
+ class CodeConversationSerializer
12
+ LABELS = { "claude_code" => "Claude Code", "codex" => "Codex" }.freeze
13
+
14
+ def self.summary(root, turns = root.turns)
15
+ new(root, turns).summary
16
+ end
17
+
18
+ def self.detail(root, turns = root.turns, after: {})
19
+ new(root, turns).detail(after: after)
20
+ end
21
+
22
+ def initialize(root, turns)
23
+ @root = root
24
+ @turns = turns
25
+ @sandbox = root.sandbox_session
26
+ end
27
+
28
+ def summary
29
+ last = @turns.last
30
+ project = @sandbox.try(:project)
31
+ {
32
+ id: "code-#{@root.id}",
33
+ source: "code_session",
34
+ runner: @root.runner,
35
+ origin: @root.origin,
36
+ agent_name: LABELS.fetch(@root.runner, @root.runner),
37
+ action_name: nil,
38
+ display_name: [ LABELS.fetch(@root.runner, @root.runner), @sandbox.try(:repository) ].compact.join(" · "),
39
+ agent: nil,
40
+ model: @turns.filter_map(&:model).last,
41
+ status: active&.status || last.status,
42
+ active: !active.nil?,
43
+ tokens: {
44
+ input: @turns.sum { |turn| turn.input_tokens.to_i },
45
+ output: @turns.sum { |turn| turn.output_tokens.to_i },
46
+ total: @turns.sum { |turn| turn.input_tokens.to_i + turn.output_tokens.to_i }
47
+ },
48
+ total_cost_usd: @turns.sum { |turn| turn.total_cost_usd.to_f }.round(6),
49
+ duration_ms: @turns.sum { |turn| turn.duration_ms.to_i },
50
+ message_count: @turns.size,
51
+ turn_count: @turns.size,
52
+ preview: {
53
+ input: InteractionPreview.line(@root.prompt),
54
+ output: InteractionPreview.line(@turns.reverse.find { |turn| turn.result.present? }&.result)
55
+ },
56
+ sandbox_session_id: @sandbox.session_id,
57
+ sandbox_status: @sandbox.status,
58
+ project: project ? { id: project.id, name: project.name } : nil,
59
+ credential_mode: @root.credential_mode,
60
+ created_at: @root.created_at.iso8601,
61
+ last_activity_at: (@turns.filter_map { |turn| turn.finished_at || turn.started_at }.max || last.created_at).iso8601
62
+ }
63
+ end
64
+
65
+ # The summary plus every turn: its summary, and its transcript from
66
+ # +after+[turn id] onward, so a client polling the running turn asks only
67
+ # for what it has not seen. A finished turn's diff comes from the session
68
+ # endpoint, on demand: it is the whole checkout's.
69
+ def detail(after: {})
70
+ summary.merge(
71
+ turns: @turns.map do |turn|
72
+ from = after[turn.id.to_s].to_i.clamp(0, turn.events.size)
73
+ turn.summary.merge(events: turn.events.drop(from), events_offset: from,
74
+ dropped_events_count: turn.dropped_events_count.to_i)
75
+ end,
76
+ browser: @sandbox.respond_to?(:browser_summary) ? @sandbox.browser_summary : nil
77
+ )
78
+ end
79
+
80
+ private
81
+
82
+ def active
83
+ @turns.find { |turn| turn.running? } || @turns.find { |turn| turn.queued? }
84
+ end
85
+ end
86
+ end
@@ -125,8 +125,19 @@ module ActionAgent
125
125
  # derived from every persisted result's diagnosis, which older runs
126
126
  # recorded in earlier shapes, so a run they fail for still serves its
127
127
  # results. Without +links+ the item paths are relative to the mount.
128
- def fix_items(run, links: nil)
129
- links ? run.fix_items(links: links) : run.fix_items
128
+ # With +markdown+ each item carries its `markdown`
129
+ # (ActiveAgent::Evals::ReportMarkdown#fix_item_markdown), which the
130
+ # dashboard's fix cards copy.
131
+ def fix_items(run, links: nil, markdown: false)
132
+ unless markdown
133
+ return links ? run.fix_items(links: links) : run.fix_items
134
+ end
135
+
136
+ report = links ? run.to_report(links: links) : run.to_report
137
+ items = report.fix_items
138
+ return items unless report.respond_to?(:fix_item_markdown)
139
+
140
+ items.map { |item| item.merge("markdown" => report.fix_item_markdown(item)) }
130
141
  rescue StandardError => e
131
142
  Rails.logger.warn(
132
143
  "[ActionAgent] evaluation run #{run.id} fix_items failed: #{e.class}: #{e.message}"
@@ -537,6 +537,12 @@ module ActionAgent
537
537
  }
538
538
  when "call_agent"
539
539
  call_agent(slug: kwargs[:slug], message: kwargs[:message])
540
+ when "translate_text"
541
+ translate_text(**kwargs.to_h.symbolize_keys.slice(:text, :target_language, :source_language))
542
+ when *CheckoutTools::FUNCTIONS
543
+ checkout_tools.call(name.to_s, kwargs)
544
+ when *SlackTools::FUNCTIONS
545
+ SlackTools.new(SlackTools.token_for(owner)).call(name.to_s, kwargs)
540
546
  when "request_handoff"
541
547
  request_handoff(**kwargs.slice(:reason, :url, :form_values, :instructions))
542
548
  else
@@ -727,6 +733,37 @@ module ActionAgent
727
733
  (text.length / 4.0).round
728
734
  end
729
735
 
736
+ # The checkout tools act on the sandbox the run was given, found among
737
+ # the agent's owner's sessions as the dispatcher finds its runtime.
738
+ def checkout_tools
739
+ @checkout_tools ||= begin
740
+ sandbox_id = @run.try(:sandbox_id)
741
+ CheckoutTools.new(sandbox_id && SandboxSession.for_owner(owner).find_by(session_id: sandbox_id))
742
+ end
743
+ end
744
+
745
+ # translate_text: one generation on this run's provider, model and the
746
+ # owner's credentials, with no tools and no conversation.
747
+ def translate_text(text:, target_language:, source_language: nil)
748
+ return { error: "No text to translate" } if text.to_s.strip.empty?
749
+ return { error: "target_language is required" } if target_language.to_s.strip.empty?
750
+
751
+ name = provider
752
+ options = { model: requested_model }.merge(owner_provider_options(name))
753
+ translator = Class.new(ActiveAgent::Base) do
754
+ define_singleton_method(:name) { "ActionAgent::Translator" }
755
+ generate_with name, **options
756
+ end
757
+ from = source_language.present? ? " from #{source_language}" : ""
758
+ instructions = "Translate the user's text#{from} into #{target_language}. Reply with the translation only — " \
759
+ "no notes, quotes or preamble. Keep formatting, code, URLs and placeholders exactly as they are."
760
+ response = translator.prompt(message: text.to_s, instructions: instructions).generate_now
761
+
762
+ { translation: response&.message&.content.to_s, target_language: target_language }
763
+ rescue StandardError => e
764
+ { error: "Translation failed: #{e.message}" }
765
+ end
766
+
730
767
  def call_agent(slug:, message:)
731
768
  depth = Thread.current[:agent_call_depth].to_i
732
769
  return { error: "call_agent depth limit (#{MAX_CALL_DEPTH}) reached" } if depth >= MAX_CALL_DEPTH
@@ -50,18 +50,25 @@ module ActionAgent
50
50
  # workspace already talks to, then the rest of the catalog.
51
51
  STATUS_RANK = { "active" => 0, "configured" => 1, "available" => 2, "idle" => 3 }.freeze
52
52
 
53
- attr_reader :agent, :discovery, :runtimes
53
+ attr_reader :agent, :discovery, :runtimes, :workspace_agents
54
54
 
55
55
  # @param agent [ActionAgent::Agent] the agent being edited
56
56
  # @param traces [ActiveRecord::Relation] the traces the caller may read
57
57
  # @param hours [Integer] the window the usage columns are scoped to
58
58
  # @param runtimes [Array<Hash>] live checkout sandbox runtimes this agent
59
59
  # can be given, as SandboxSession.runtime_server_listings returns them
60
- def initialize(agent:, traces:, hours: ToolDiscovery::DEFAULT_WINDOW_HOURS, runtimes: [])
60
+ # @param workspace_agents [ActiveRecord::Relation, nil] the workspace's
61
+ # other agents, whose MCP servers each service row counts as
62
+ # +workspace_agents+ — what a new agent's builder shows as already in
63
+ # use, since an unsaved agent has no usage of its own to show
64
+ def initialize(agent:, traces:, hours: ToolDiscovery::DEFAULT_WINDOW_HOURS, runtimes: [], workspace_agents: nil)
61
65
  @agent = agent
62
66
  @runtimes = Array(runtimes).index_by { |runtime| runtime[:key] }
67
+ @workspace_agents = workspace_agents
68
+ # An agent not saved yet has no traces of its own, and no name to
69
+ # match unattributed ones by.
63
70
  @discovery = ToolDiscovery.new(
64
- traces: agent.telemetry_traces(traces),
71
+ traces: agent.persisted? ? agent.telemetry_traces(traces) : traces.none,
65
72
  agents: Agent.where(id: agent.id),
66
73
  hours: hours
67
74
  )
@@ -118,6 +125,7 @@ module ActionAgent
118
125
  used = detected_by_server[key] || []
119
126
  calls = used.sum { |tool| tool[:calls] }
120
127
  enabled = configured_servers.key?(key)
128
+ unreachable = MCPToolDispatcher.unreachable_reason(MCPCatalog.connection(key) || runtimes[key])
121
129
 
122
130
  {
123
131
  key: key,
@@ -133,10 +141,25 @@ module ActionAgent
133
141
  calls: calls,
134
142
  errors: used.sum { |tool| tool[:errors] },
135
143
  last_seen: used.filter_map { |tool| tool[:last_seen] }.max,
144
+ # Whether a dashboard run reaches it (MCPToolDispatcher), and when
145
+ # not, why: a stdio server, or one needing a credential it can't send.
146
+ callable: unreachable.nil?,
147
+ unavailable_reason: unreachable,
148
+ workspace_agents: workspace_usage[key].to_i,
136
149
  tools: service_tools(key, catalog, used)
137
150
  }
138
151
  end
139
152
 
153
+ # server key => how many of the workspace's agents enable it. Empty
154
+ # unless the caller handed in +workspace_agents+.
155
+ def workspace_usage
156
+ @workspace_usage ||= Array(workspace_agents).each_with_object(Hash.new(0)) do |other, counts|
157
+ next if agent.persisted? && other.id == agent.id
158
+
159
+ self.class.configured_servers(other).each_key { |key| counts[key] += 1 }
160
+ end
161
+ end
162
+
140
163
  # How to reach the server, in the one line the expanded panel shows:
141
164
  # "Streamable HTTP · <url>" for the ones the dashboard can call,
142
165
  # "sandbox · <command>" for the ones it can start, "stdio · <command>"
@@ -233,6 +256,10 @@ module ActionAgent
233
256
  description: Agent::TOOL_DESCRIPTIONS[capability],
234
257
  enabled: saved_tools.include?(capability),
235
258
  editable: true,
259
+ # Whether AgentToolbox gives the model anything for it. A capability
260
+ # without definitions is a name on the roster and nothing more, so
261
+ # the builder offers its MCP service (or nothing) in its place.
262
+ implemented: AgentToolbox::DEFINITIONS.key?(capability),
236
263
  approval_names: capability_functions(capability) - AgentToolbox::INPUT_FUNCTIONS
237
264
  )
238
265
  end
@@ -283,27 +310,35 @@ module ActionAgent
283
310
  # carries a top-level Hash keyed by server name — the same three shapes
284
311
  # EvaluationToolResolver tolerates.
285
312
  def configured_servers
286
- @configured_servers ||= configured_entries.each_with_object({}) do |entry, map|
287
- key = entry_key(entry)
288
- map[key] = entry if key.present?
289
- end
313
+ @configured_servers ||= self.class.configured_servers(agent)
290
314
  end
291
315
 
292
- def configured_entries
293
- servers = agent.mcp_servers
316
+ class << self
317
+ # server key => the entry +agent+ stores for it, in any of the three
318
+ # shapes above.
319
+ def configured_servers(agent)
320
+ configured_entries(agent.mcp_servers).each_with_object({}) do |entry, map|
321
+ key = entry_key(entry)
322
+ map[key] = entry if key.present?
323
+ end
324
+ end
325
+
326
+ private
294
327
 
295
- if servers.is_a?(Hash)
296
- servers.map { |key, value| value.respond_to?(:key?) ? value.to_h.stringify_keys.merge("key" => key.to_s) : key.to_s }
297
- else
298
- Array(servers)
328
+ def configured_entries(servers)
329
+ if servers.is_a?(Hash)
330
+ servers.map { |key, value| value.respond_to?(:key?) ? value.to_h.stringify_keys.merge("key" => key.to_s) : key.to_s }
331
+ else
332
+ Array(servers)
333
+ end
299
334
  end
300
- end
301
335
 
302
- def entry_key(entry)
303
- return entry.to_s.strip.presence if entry.is_a?(String) || entry.is_a?(Symbol)
304
- return nil unless entry.respond_to?(:key?)
336
+ def entry_key(entry)
337
+ return entry.to_s.strip.presence if entry.is_a?(String) || entry.is_a?(Symbol)
338
+ return nil unless entry.respond_to?(:key?)
305
339
 
306
- (entry["key"] || entry[:key] || entry["name"] || entry[:name]).to_s.strip.presence
340
+ (entry["key"] || entry[:key] || entry["name"] || entry[:name]).to_s.strip.presence
341
+ end
307
342
  end
308
343
 
309
344
  # The per-server allow-list, or nil when the entry names none — which