actionagent 1.7.2 → 1.8.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (69) hide show
  1. checksums.yaml +4 -4
  2. data/app/assets/builds/action_agent.css +1 -1
  3. data/app/assets/builds/action_agent.js +59 -54
  4. data/app/controllers/action_agent/api/agents_controller.rb +19 -3
  5. data/app/controllers/action_agent/api/code_sessions_controller.rb +168 -0
  6. data/app/controllers/action_agent/api/evaluations_controller.rb +74 -131
  7. data/app/controllers/action_agent/api/github_connections_controller.rb +147 -0
  8. data/app/controllers/action_agent/api/mcp_controller.rb +38 -5
  9. data/app/controllers/action_agent/api/mcp_servers_controller.rb +10 -1
  10. data/app/controllers/action_agent/api/provider_keys_controller.rb +54 -3
  11. data/app/controllers/action_agent/api/provider_models_controller.rb +10 -6
  12. data/app/controllers/action_agent/api/sandboxes_controller.rb +105 -6
  13. data/app/controllers/concerns/action_agent/api/evaluation_run_starting.rb +93 -0
  14. data/app/controllers/concerns/action_agent/api/mcp_dashboard_tools.rb +509 -0
  15. data/app/controllers/concerns/action_agent/api/run_sandbox.rb +65 -0
  16. data/app/jobs/action_agent/code_session_job.rb +171 -0
  17. data/app/jobs/action_agent/sandbox_cleanup_job.rb +80 -11
  18. data/app/jobs/action_agent/sandbox_provision_job.rb +122 -14
  19. data/app/jobs/action_agent/sandbox_run_job.rb +10 -3
  20. data/app/models/action_agent/agent.rb +49 -6
  21. data/app/models/action_agent/agent_run.rb +20 -1
  22. data/app/models/action_agent/code_session.rb +145 -0
  23. data/app/models/action_agent/evaluation.rb +64 -0
  24. data/app/models/action_agent/evaluation_run.rb +166 -54
  25. data/app/models/action_agent/evaluation_scenario_result.rb +15 -3
  26. data/app/models/action_agent/github_connection.rb +75 -0
  27. data/app/models/action_agent/model_pricing.rb +214 -34
  28. data/app/models/action_agent/provider_key.rb +148 -10
  29. data/app/models/action_agent/sandbox_session.rb +194 -17
  30. data/app/serializers/action_agent/evaluation_serializer.rb +149 -0
  31. data/app/serializers/action_agent/telemetry_trace_serializer.rb +10 -2
  32. data/app/services/action_agent/agent_execution_service.rb +4 -2
  33. data/app/services/action_agent/agent_scorecard.rb +50 -21
  34. data/app/services/action_agent/agent_tool_roster.rb +20 -9
  35. data/app/services/action_agent/claude_code_auth.rb +86 -0
  36. data/app/services/action_agent/codex_session_events.rb +37 -0
  37. data/app/services/action_agent/dashboard_assistant_service.rb +47 -5
  38. data/app/services/action_agent/evaluation_report_import.rb +84 -5
  39. data/app/services/action_agent/evaluation_run_cost.rb +609 -0
  40. data/app/services/action_agent/evaluation_runner_service.rb +25 -5
  41. data/app/services/action_agent/evaluation_standing.rb +145 -0
  42. data/app/services/action_agent/evaluation_tool_resolver.rb +18 -0
  43. data/app/services/action_agent/github_client.rb +111 -0
  44. data/app/services/action_agent/local_sandbox_backend.rb +1715 -0
  45. data/app/services/action_agent/local_sandbox_databases.rb +257 -0
  46. data/app/services/action_agent/mcp_client.rb +5 -1
  47. data/app/services/action_agent/mcp_tool_dispatcher.rb +137 -17
  48. data/app/services/action_agent/mock_sandbox_backend.rb +39 -0
  49. data/app/services/action_agent/ollama_host_probe.rb +75 -0
  50. data/app/services/action_agent/payload_bounds.rb +36 -0
  51. data/app/services/action_agent/sandbox_manifest.rb +67 -0
  52. data/app/services/action_agent/sandbox_orchestrator.rb +82 -14
  53. data/app/services/action_agent/scenario_evaluation_runner.rb +52 -5
  54. data/app/services/action_agent/secret_scrubber.rb +37 -0
  55. data/app/services/action_agent/tool_discovery.rb +19 -5
  56. data/config/routes.rb +23 -4
  57. data/lib/action_agent/engine.rb +1 -0
  58. data/lib/action_agent/version.rb +1 -1
  59. data/lib/action_agent.rb +153 -3
  60. data/lib/generators/action_agent/install_generator.rb +35 -5
  61. data/lib/generators/action_agent/templates/action_agent.rb.erb +47 -0
  62. data/lib/generators/action_agent/templates/add_code_session_runner.rb.erb +9 -0
  63. data/lib/generators/action_agent/templates/add_provider_key_api_key.rb.erb +25 -0
  64. data/lib/generators/action_agent/templates/create_active_agent_code_sessions.rb.erb +59 -0
  65. data/lib/generators/action_agent/templates/create_active_agent_dashboard_tables.rb.erb +2 -0
  66. data/lib/generators/action_agent/templates/create_active_agent_github_connections.rb.erb +61 -0
  67. data/lib/tasks/claude_code.rake +16 -0
  68. data/lib/tasks/sandbox.rake +26 -0
  69. metadata +28 -2
@@ -0,0 +1,145 @@
1
+ # frozen_string_literal: true
2
+
3
+ module ActionAgent
4
+ # A headless Claude Code session run inside an app_runtime sandbox's
5
+ # checkout: the prompt, the stream-json transcript as it arrived, the
6
+ # outcome Claude Code reported, and the diff the session left in the
7
+ # working tree.
8
+ #
9
+ # The transcript is scrubbed of the sandbox's secrets (its GitHub token and
10
+ # Claude Code credential) before it is stored, and bounded: long strings
11
+ # are truncated and events past MAX_EVENTS are counted rather than kept.
12
+ class CodeSession < ApplicationRecord
13
+ include Ownable
14
+ owned_by :user, :account
15
+
16
+ belongs_to :sandbox_session
17
+
18
+ enum :status, { queued: 0, running: 1, succeeded: 2, failed: 3, cancelled: 4 }
19
+
20
+ MAX_PROMPT_CHARACTERS = 20_000
21
+ MAX_EVENTS = 1_000
22
+ # Per string inside an event: tool results can be whole files.
23
+ MAX_EVENT_STRING = 4_000
24
+ MAX_DIFF_BYTES = 500_000
25
+ RUNNERS = %w[claude_code codex].freeze
26
+
27
+ validates :prompt, presence: true, length: { maximum: MAX_PROMPT_CHARACTERS }
28
+ validates :runner, inclusion: { in: RUNNERS }
29
+
30
+ scope :recent, -> { order(created_at: :desc) }
31
+
32
+ # JSON columns carry no default on MySQL or SQLite (see the migration).
33
+ def events
34
+ Array(super)
35
+ end
36
+
37
+ def finished?
38
+ succeeded? || failed? || cancelled?
39
+ end
40
+
41
+ # Whether an outcome and a diff may still be recorded for this session.
42
+ # finished_at is set once nothing more will be: by CodeSessionJob when it
43
+ # settles a session it ran (whatever the backend returned, raised or
44
+ # refused), or by the cancel of a session that never left the queue. A
45
+ # session cancelled while running is finished at once but settled only
46
+ # when its Claude Code has stopped, so its diff may still come, and its
47
+ # transcript still grow, until then.
48
+ def diff_pending?
49
+ finished_at.nil?
50
+ end
51
+
52
+ # The values that must never be stored: the checkout token and the
53
+ # Claude Code credential this session ran with.
54
+ def secrets
55
+ spec = sandbox_session.checkout_spec rescue nil
56
+ [ spec&.dig(:token), *sandbox_session.runtime_environment(runner: runner).values ].compact
57
+ end
58
+
59
+ # Appends one stream-json event, scrubbed and bounded. Past MAX_EVENTS
60
+ # only the count grows, so a runaway session cannot grow the row without
61
+ # limit.
62
+ def append_event!(event, secrets: self.secrets)
63
+ stored = truncate_strings(SecretScrubber.scrub(event.to_h, secrets))
64
+
65
+ with_lock do
66
+ list = events
67
+ if list.size < MAX_EVENTS
68
+ self.events = list + [ stored ]
69
+ else
70
+ self.dropped_events_count = dropped_events_count.to_i + 1
71
+ end
72
+ save!
73
+ end
74
+ end
75
+
76
+ # Records Claude Code's final "result" event.
77
+ def record_result!(event)
78
+ usage = event["usage"].is_a?(Hash) ? event["usage"] : {}
79
+
80
+ update!(
81
+ result: event["result"].to_s.presence && SecretScrubber.scrub(event["result"].to_s, secrets).truncate(MAX_EVENT_STRING * 4),
82
+ claude_session_id: runner == "claude_code" ? event["session_id"] : nil,
83
+ runner_session_id: event["session_id"],
84
+ num_turns: event["num_turns"],
85
+ duration_ms: event["duration_ms"],
86
+ total_cost_usd: event["total_cost_usd"],
87
+ input_tokens: usage["input_tokens"],
88
+ output_tokens: usage["output_tokens"]
89
+ )
90
+ end
91
+
92
+ def diff=(value)
93
+ text = SecretScrubber.scrub(value.to_s, secrets)
94
+ super(text.bytesize > MAX_DIFF_BYTES ? "#{text.byteslice(0, MAX_DIFF_BYTES).scrub}\n… diff truncated" : text)
95
+ end
96
+
97
+ def summary
98
+ {
99
+ id: id,
100
+ sandbox_session_id: sandbox_session.session_id,
101
+ status: status,
102
+ prompt: prompt,
103
+ model: model,
104
+ runner: runner,
105
+ result: result,
106
+ error_message: error_message,
107
+ num_turns: num_turns,
108
+ duration_ms: duration_ms,
109
+ total_cost_usd: total_cost_usd&.to_f,
110
+ input_tokens: input_tokens,
111
+ output_tokens: output_tokens,
112
+ event_count: events.size + dropped_events_count.to_i,
113
+ # What a client polls until: false once nothing about the session
114
+ # changes any more.
115
+ diff_pending: diff_pending?,
116
+ started_at: started_at&.iso8601,
117
+ finished_at: finished_at&.iso8601,
118
+ created_at: created_at&.iso8601
119
+ }
120
+ end
121
+
122
+ # The summary plus the transcript from event index +after+ onward (for
123
+ # incremental polling) and the diff once finished.
124
+ def details(after: 0)
125
+ after = after.to_i.clamp(0, events.size)
126
+ summary.merge(
127
+ events: events.drop(after),
128
+ events_offset: after,
129
+ dropped_events_count: dropped_events_count.to_i,
130
+ diff: finished? ? diff : nil
131
+ )
132
+ end
133
+
134
+ private
135
+
136
+ def truncate_strings(value)
137
+ case value
138
+ when String then value.length > MAX_EVENT_STRING ? "#{value[0, MAX_EVENT_STRING]}… (truncated)" : value
139
+ when Hash then value.to_h { |key, item| [ key, truncate_strings(item) ] }
140
+ when Array then value.map { |item| truncate_strings(item) }
141
+ else value
142
+ end
143
+ end
144
+ end
145
+ end
@@ -34,6 +34,20 @@ module ActionAgent
34
34
  validate :validate_criteria
35
35
 
36
36
  scope :recent, -> { order(updated_at: :desc) }
37
+ # Archived evaluations keep their runs but leave the index and its
38
+ # tiles unless asked for (see #archive!). The mark lives in the config
39
+ # JSON, so the filter reads it with each database's own JSON path.
40
+ scope :archived, -> { where("#{archived_at_sql} IS NOT NULL") }
41
+ scope :unarchived, -> { where("#{archived_at_sql} IS NULL") }
42
+
43
+ # SQL reading config["archived_at"] on the connected database.
44
+ def self.archived_at_sql
45
+ case connection.adapter_name.to_s.downcase
46
+ when /postgres/ then "#{quoted_table_name}.config ->> 'archived_at'"
47
+ when /mysql|trilogy/ then "JSON_UNQUOTE(JSON_EXTRACT(#{quoted_table_name}.config, '$.archived_at'))"
48
+ else "json_extract(#{quoted_table_name}.config, '$.archived_at')"
49
+ end
50
+ end
37
51
 
38
52
  # MySQL cannot give a JSON column a default, so a row inserted there
39
53
  # without `criteria` or `config` reads back nil. Both readers answer with
@@ -50,6 +64,56 @@ module ActionAgent
50
64
  evaluation_runs.order(created_at: :desc).first
51
65
  end
52
66
 
67
+ # The newest run that finished: the one the evaluation's pass rate
68
+ # describes. A newer run still pending, or one that failed, shows
69
+ # beside it and never in its place. Read from the loaded association
70
+ # when the index preloaded it.
71
+ def headline_run
72
+ if evaluation_runs.loaded?
73
+ evaluation_runs.select(&:complete?).max_by { |run| [ run.created_at, run.id ] }
74
+ else
75
+ evaluation_runs.complete.order(created_at: :desc, id: :desc).first
76
+ end
77
+ end
78
+
79
+ # --- archiving ---------------------------------------------------------
80
+ #
81
+ # An evaluation nobody maintains — its suite superseded, its agent
82
+ # retired — keeps its history but stops counting: the index leaves it
83
+ # out, and so do the pooled pass rate and the agent's scorecard. The
84
+ # mark is config["archived_at"]; a new run or a published report clears
85
+ # it, since either says the evaluation is alive after all.
86
+
87
+ def archived_at
88
+ value = config["archived_at"]
89
+ value.present? ? Time.zone.parse(value.to_s) : nil
90
+ rescue ArgumentError
91
+ nil
92
+ end
93
+
94
+ def archived?
95
+ config["archived_at"].present?
96
+ end
97
+
98
+ def archive!
99
+ update!(config: config.merge("archived_at" => Time.current.iso8601))
100
+ end
101
+
102
+ def unarchive!
103
+ return unless archived?
104
+
105
+ update!(config: config.except("archived_at"))
106
+ end
107
+
108
+ # Where this evaluation stands against the agent as it is now: whether
109
+ # its headline run scored the current version (EvaluationStanding).
110
+ # Memoized per instance; the index preloads what it needs.
111
+ def standing_info
112
+ @standing_info ||= EvaluationStanding.new(self)
113
+ end
114
+
115
+ attr_writer :standing_info
116
+
53
117
  def judge_defined?
54
118
  judge_kind == "judge_defined"
55
119
  end
@@ -9,15 +9,29 @@ module ActionAgent
9
9
  class EvaluationRun < ApplicationRecord
10
10
  belongs_to :evaluation
11
11
  # The version of the evaluated agent this run scored, so a pass rate is
12
- # a statement about a release rather than about "the agent".
12
+ # a statement about a release rather than about "the agent". A run the
13
+ # engine executes scored the agent as it is now; an imported run scored
14
+ # the publishing application's code, which only its report can name
15
+ # (EvaluationReportImport sets the version from `report.release`), so
16
+ # a release-less import stays unrecorded rather than claiming the
17
+ # dashboard's latest version.
13
18
  belongs_to :agent_version, optional: true
14
- before_create { self.agent_version_id ||= evaluation&.agent&.latest_version&.id }
19
+ before_create { self.agent_version_id ||= evaluation&.agent&.latest_version&.id unless imported? }
20
+ # A run started on an archived evaluation brings it back: archiving says
21
+ # "no longer maintained", which a new run contradicts.
22
+ after_create { evaluation.unarchive! if evaluation&.archived? }
15
23
  has_many :scenario_results, class_name: "EvaluationScenarioResult", dependent: :destroy
16
24
 
17
25
  enum :status, { pending: 0, running: 1, complete: 2, failed: 3 }
18
26
 
19
27
  scope :recent, -> { order(created_at: :desc) }
20
28
 
29
+ # Whether this run was published by an application that ran it itself
30
+ # (EvaluationReportImport) rather than executed here.
31
+ def imported?
32
+ external_run_id.present?
33
+ end
34
+
21
35
  # Which scenarios and models a scenario run covered; empty for a
22
36
  # generation-sampling run.
23
37
  def selection
@@ -25,6 +39,19 @@ module ActionAgent
25
39
  value.is_a?(Hash) ? value : {}
26
40
  end
27
41
 
42
+ # The checkout sandbox a scenario run replayed against
43
+ # (ScenarioEvaluationRunner records it), as { "session_id", "server_key",
44
+ # "repository", "repository_ref" }; nil for a run against the agent's own
45
+ # servers only.
46
+ def sandbox
47
+ value = selection["sandbox"]
48
+ return value if value.is_a?(Hash)
49
+
50
+ # Still pending: run_later! recorded only what was asked for.
51
+ id = selection["sandbox_id"]
52
+ { "session_id" => id } if id.is_a?(String) && id.present?
53
+ end
54
+
28
55
  # The candidate models a scenario run compared, in the order they were
29
56
  # requested; empty for a generation-sampling run.
30
57
  def models
@@ -79,14 +106,37 @@ module ActionAgent
79
106
  (values.sum.to_f / values.size).round(3)
80
107
  end
81
108
 
82
- # The judge's own spend on this run — calls, tokens, estimated cost and
83
- # how many calls served each purpose — as the runner recorded it; nil
84
- # for a run that never asked a judge.
85
- def judge_usage
109
+ # The judge's own spend on this run as the engine's meter recorded it —
110
+ # calls, tokens, estimated cost and how many calls served each purpose
111
+ # — or nil for a run the engine did not judge (a rules-only run, or an
112
+ # imported one, whose judge the application ran).
113
+ def judge_usage_meter
86
114
  value = scores&.dig("_judge_usage")
87
115
  value.is_a?(Hash) ? value : nil
88
116
  end
89
117
 
118
+ # The judge's spend on this run from whatever recorded it: the meter,
119
+ # the publishing application's figures, or the judge traces priced
120
+ # (EvaluationRunCost). nil for a run no judge was asked about.
121
+ def judge_usage
122
+ cost_breakdown.judge_usage
123
+ end
124
+
125
+ # Every cost figure of this run — per result, per scenario, per model
126
+ # and in total — worked out once (EvaluationRunCost). A controller that
127
+ # lists runs preloads it (EvaluationRunCost.preload) and hands it in.
128
+ def cost_breakdown
129
+ @cost_breakdown ||= EvaluationRunCost.for(self)
130
+ end
131
+
132
+ attr_writer :cost_breakdown
133
+
134
+ # Forgets the breakdown, so a run whose results just changed is priced again.
135
+ def reload(*)
136
+ @cost_breakdown = nil
137
+ super
138
+ end
139
+
90
140
  # Per-model summaries of a generation-sampling run's cohorts, keyed by
91
141
  # model; empty for a scenario run or a run recorded before they were.
92
142
  def cohorts
@@ -94,59 +144,51 @@ module ActionAgent
94
144
  value.is_a?(Hash) ? value : {}
95
145
  end
96
146
 
147
+ # Per-model summaries of a scenario run, keyed by label, as the runner
148
+ # recorded them under "_models", each with its cost as it stands now:
149
+ # the effective "cost" over its results (reported, else estimated), the
150
+ # "priced", "reported" and "estimated" counts behind it, and the
151
+ # judge's "judge_cost" and "judge_calls" on them (EvaluationRunCost). A
152
+ # recorded summary no result maps to is served as recorded. Empty for a
153
+ # generation-sampling run.
154
+ def model_summaries
155
+ summaries = scores&.dig("_models")
156
+ return {} unless summaries.is_a?(Hash)
157
+
158
+ costed = cost_breakdown.by_label(summaries.keys)
159
+ summaries.to_h do |label, stats|
160
+ next [ label, stats ] unless stats.is_a?(Hash) && costed.key?(label)
161
+
162
+ [ label, stats.merge(costed[label]) ]
163
+ end
164
+ end
165
+
97
166
  # What the run spent, for display after it: the agent's side and the
98
167
  # judge's, kept apart because they answer different questions.
99
168
  #
100
169
  # The agent's side is the operating figure — what the interactions cost
101
- # to serve. For a scenario run that is the replays' estimated cost,
102
- # tokens and summed model time (`replays` of them); for a
103
- # generation-sampling run it is the sampled generations' (`samples`),
104
- # which were served before the run and cost it nothing. `per_interaction`
105
- # is that cost spread over the interactions, the number a per-conversation
106
- # budget is set against.
170
+ # to serve. For a scenario run that is its replays' cost, tokens and
171
+ # summed model time (`replays` of them); for a generation-sampling run
172
+ # it is the sampled generations' (`samples`), which were served before
173
+ # the run and cost it nothing.
174
+ #
175
+ # Every interaction with tokens is priced (EvaluationRunCost): `cost`
176
+ # sums the reported costs and, where none was reported, the estimates
177
+ # from tokens × model rates. `priced` and `unpriced` count the
178
+ # interactions either way, `reported` and `estimated` say how the priced
179
+ # ones were priced, and `cost_basis` sums that up as "reported",
180
+ # "estimated" or "mixed". `per_interaction` is the cost per priced
181
+ # interaction, the number a per-conversation budget is set against.
107
182
  #
108
183
  # `judge` is the evaluation's own overhead: the judge model's calls
109
184
  # (scoring, recommending, the verdict, authoring KPIs), which run
110
- # agent-to-agent and offline. It is present only when a judge was asked.
185
+ # agent-to-agent and offline — from the engine's meter, the publishing
186
+ # application's figures or the judge traces, with `source` naming which
187
+ # and `run` the calls no result owns. `total` is the two sides together.
111
188
  #
112
189
  # Returns nil for a run that recorded nothing on either side.
113
190
  def usage
114
- totals = scenario_results.pick(
115
- Arel.sql("COUNT(*)"), Arel.sql("SUM(cost)"), Arel.sql("SUM(input_tokens)"),
116
- Arel.sql("SUM(output_tokens)"), Arel.sql("SUM(duration_ms)")
117
- )
118
- replays = totals&.first.to_i
119
- judge = judge_usage
120
- runtime_ms = completed_at.present? ? ((completed_at - created_at) * 1000).round : nil
121
-
122
- if replays.positive?
123
- cost = totals[1]&.to_f
124
- {
125
- replays: replays,
126
- cost: cost,
127
- per_interaction: cost && (cost / replays).round(6),
128
- input_tokens: totals[2].to_i,
129
- output_tokens: totals[3].to_i,
130
- model_time_ms: totals[4].to_i,
131
- runtime_ms: runtime_ms,
132
- judge: judge
133
- }.compact
134
- elsif cohorts.any?
135
- samples = cohorts.values.sum { |cohort| cohort["samples"].to_i }
136
- costs = cohorts.values.filter_map { |cohort| cohort["cost"] }
137
- cost = costs.any? ? costs.sum.to_f.round(6) : nil
138
- {
139
- samples: samples,
140
- cost: cost,
141
- per_interaction: cost && samples.positive? ? (cost / samples).round(6) : nil,
142
- input_tokens: cohorts.values.sum { |cohort| cohort["input_tokens"].to_i },
143
- output_tokens: cohorts.values.sum { |cohort| cohort["output_tokens"].to_i },
144
- runtime_ms: runtime_ms,
145
- judge: judge
146
- }.compact
147
- elsif judge
148
- { runtime_ms: runtime_ms, judge: judge }.compact
149
- end
191
+ cost_breakdown.usage
150
192
  end
151
193
 
152
194
  # Route templates for the report's fix item actions, relative to the
@@ -181,10 +223,21 @@ module ActionAgent
181
223
  # the suite panel does. Raises ActiveRecord::RecordNotFound via the
182
224
  # caller for a run of a generation-sampling evaluation, which has no
183
225
  # scenario results to report on.
184
- def to_report(links: report_links)
226
+ #
227
+ # With `estimate` (the default) each replay carries its effective cost
228
+ # and how it was priced (EvaluationRunCost), and the report is told the
229
+ # judge's run-level spend, so the page shows every cost the dashboard
230
+ # does. `estimate: false` rebuilds the report from what was recorded
231
+ # alone — the import summarizes a published run that way, so the
232
+ # summaries it stores stay the application's own figures.
233
+ #
234
+ # The engine runs against every activeagent since 1.4, so the report
235
+ # kwargs that arrived later are passed only when Report.new takes them.
236
+ def to_report(links: report_links, estimate: true)
185
237
  rows = scenario_results.includes(:scenario).sort_by do |row|
186
238
  [ row.evaluated_scenario["position"].to_i, row.evaluation_scenario_id, row.model ]
187
239
  end
240
+ breakdown = cost_breakdown if estimate
188
241
  selected = selected_specs
189
242
  specs = {}
190
243
  results = rows.map do |row|
@@ -197,28 +250,63 @@ module ActionAgent
197
250
  replay: ActiveAgent::Evals::Replay.new(
198
251
  answer: row.output, tool_calls: Array(row.tool_calls), duration_ms: row.duration_ms,
199
252
  input_tokens: row.input_tokens, output_tokens: row.output_tokens,
200
- cost: row.cost&.to_f, error: row.error_message, metadata: row.replay_metadata
253
+ cost: breakdown ? breakdown.result(row)["cost"] : row.cost&.to_f, error: row.error_message,
254
+ metadata: replay_metadata_for(row, breakdown)
201
255
  ),
202
256
  scores: row.scores.to_h, score: row.score, status: row.status,
203
257
  diagnosis: row.evaluation_diagnosis.presence
204
258
  )
205
259
  end
206
260
 
207
- ActiveAgent::Evals::Report.new(
261
+ kwargs = {
208
262
  results: results,
209
263
  models: (selected.values & specs.values) + (specs.values - selected.values),
210
264
  metadata: {
211
265
  "evaluation" => evaluation.name,
212
266
  "agent" => evaluation.agent&.name,
213
267
  "run" => id,
214
- "finished" => completed_at&.iso8601
268
+ "finished" => completed_at&.iso8601,
269
+ "sandbox" => sandbox_label
215
270
  }.compact.merge(report_metadata),
216
271
  verdict: recorded_verdict,
217
272
  judge_label: judge_label,
218
273
  tool_resolver: EvaluationToolResolver.new(evaluation.agent),
219
274
  agent_name: evaluation.agent&.name,
220
275
  links: links
221
- )
276
+ }
277
+ kwargs[:judge_usage] = breakdown.judge_usage_run if breakdown && self.class.report_accepts?(:judge_usage)
278
+ kwargs[:release] = release_summary if release_summary && self.class.report_accepts?(:release)
279
+ self.class.report_class.new(**kwargs)
280
+ end
281
+
282
+ # The framework's Report, as installed.
283
+ def self.report_class
284
+ ActiveAgent::Evals::Report
285
+ end
286
+
287
+ # Whether the installed framework's Report.new declares +keyword+.
288
+ def self.report_accepts?(keyword)
289
+ report_class.instance_method(:initialize).parameters.any? { |type, name| name == keyword && %i[key keyreq].include?(type) }
290
+ end
291
+
292
+ # The release this run scored, as the report names it — `{ "digest",
293
+ # "revision", "label" }` — or nil for a run pinned to a dashboard edit
294
+ # or to no version.
295
+ def release_summary
296
+ version = agent_version
297
+ return nil unless version&.release?
298
+
299
+ { "digest" => version.release_digest, "revision" => version.revision, "label" => "v#{version.version_number}" }.compact
300
+ end
301
+
302
+ # The agent version this run scored, for the run's JSON: `{ id, number,
303
+ # release_digest, revision, release }`, or nil when none was recorded.
304
+ def agent_version_summary
305
+ version = agent_version
306
+ return nil unless version
307
+
308
+ { id: version.id, number: version.version_number, release_digest: version.release_digest,
309
+ revision: version.revision, release: version.release? }
222
310
  end
223
311
 
224
312
  private
@@ -226,6 +314,30 @@ module ActionAgent
226
314
  ModelSpec = ActiveAgent::Evals::ModelSpec
227
315
  private_constant :ModelSpec
228
316
 
317
+ # The replay metadata the rebuilt report reads a result's costs from:
318
+ # what the result recorded, the judge usage the application reported
319
+ # for it, and — when estimating — how its cost was priced.
320
+ def replay_metadata_for(row, breakdown)
321
+ metadata = row.replay_metadata.dup
322
+ reported_judge = row.diagnosis.is_a?(Hash) ? row.diagnosis["_judge_usage"] : nil
323
+ metadata["judge_usage"] = reported_judge if reported_judge.is_a?(Hash)
324
+ return metadata unless breakdown
325
+
326
+ entry = breakdown.result(row)
327
+ metadata["cost_source"] = entry["cost_source"]
328
+ metadata["cost_rate"] = entry["cost_rate"] if entry["cost_rate"]
329
+ metadata["judge_usage"] = entry["judge_usage"] if entry["judge_usage"]
330
+ metadata
331
+ end
332
+
333
+ # How the report's header names the sandbox: its checkout and session.
334
+ def sandbox_label
335
+ return nil unless sandbox
336
+
337
+ checkout = [ sandbox["repository"], sandbox["repository_ref"] ].compact_blank.join("@")
338
+ [ checkout.presence, "sandbox #{sandbox["session_id"].to_s.first(8)}" ].compact.join(" · ")
339
+ end
340
+
229
341
  # The candidate specs the run was asked to compare, keyed by
230
342
  # [provider, model] in the order requested. ScenarioEvaluationRunner
231
343
  # persists each ModelSpec#to_h in `selection`, and it is that label —
@@ -44,8 +44,11 @@ module ActionAgent
44
44
  value.is_a?(Hash) ? value : {}
45
45
  end
46
46
 
47
+ # The diagnosis as the dashboard shows it: without the storage keys —
48
+ # the replay metadata, the scenario snapshot and the judge usage the
49
+ # publishing application reported (served as `judge_usage` instead).
47
50
  def evaluation_diagnosis
48
- (diagnosis || {}).except("_replay_metadata", "_scenario_snapshot")
51
+ (diagnosis || {}).except("_replay_metadata", "_scenario_snapshot", "_judge_usage")
49
52
  end
50
53
 
51
54
  # A catalog can be refreshed without changing what an earlier run asked
@@ -55,7 +58,12 @@ module ActionAgent
55
58
  snapshot.is_a?(Hash) ? snapshot : scenario.as_json_summary.stringify_keys
56
59
  end
57
60
 
58
- def as_json_summary
61
+ # The result for the JSON API. Its cost is the effective one — reported,
62
+ # else estimated — with `reported_cost`, `cost_source`, `cost_rate` and
63
+ # the judge's `judge_usage` on it (EvaluationRunCost). A caller listing
64
+ # a run's results hands in the run's breakdown so nothing is priced per row.
65
+ def as_json_summary(costs: nil)
66
+ costs ||= evaluation_run.cost_breakdown.result(self)
59
67
  {
60
68
  id: id,
61
69
  scenario_id: evaluation_scenario_id,
@@ -73,7 +81,11 @@ module ActionAgent
73
81
  duration_ms: duration_ms,
74
82
  input_tokens: input_tokens,
75
83
  output_tokens: output_tokens,
76
- cost: cost&.to_f,
84
+ cost: costs["cost"],
85
+ reported_cost: costs["reported_cost"],
86
+ cost_source: costs["cost_source"],
87
+ cost_rate: costs["cost_rate"],
88
+ judge_usage: costs["judge_usage"],
77
89
  fault: fault,
78
90
  recommendation: recommendation,
79
91
  diagnosis: evaluation_diagnosis,
@@ -0,0 +1,75 @@
1
+ # frozen_string_literal: true
2
+
3
+ module ActionAgent
4
+ # An owner's GitHub OAuth grant (Settings -> Integrations) and the
5
+ # repositories they made available to the workspace.
6
+ #
7
+ # The access token is encrypted at rest like a provider key and never
8
+ # rendered back to the client. +repositories+ holds only what GitHub
9
+ # itself listed for this token when the owner chose them (see
10
+ # Api::GithubConnectionsController#update), so a checkout sandbox can
11
+ # trust a name found here without asking GitHub again.
12
+ class GithubConnection < ApplicationRecord
13
+ include Ownable
14
+ owned_by :account, :user
15
+
16
+ encrypts :access_token if ActionAgent.encrypt_credentials
17
+
18
+ validates :access_token, :github_user_id, :login, presence: true
19
+ # One connection per owner; which column that means depends on the
20
+ # configured mode, so it is checked at validation time.
21
+ validate :unique_within_owner
22
+
23
+ # JSON columns carry no default on MySQL or SQLite (see the migration), so
24
+ # an unset column reads as nil.
25
+ def repositories
26
+ Array(super)
27
+ end
28
+
29
+ def repository_names
30
+ repositories.map { |repo| repo["full_name"] }
31
+ end
32
+
33
+ def repository(full_name)
34
+ repositories.find { |repo| repo["full_name"].casecmp?(full_name.to_s) }
35
+ end
36
+
37
+ def client
38
+ GithubClient.new(access_token)
39
+ end
40
+
41
+ # The checkout a sandbox backend clones: repository, ref, and an
42
+ # authenticated HTTPS clone URL. Carries the token — hand it to a backend,
43
+ # never to a response.
44
+ def checkout_spec(full_name, ref: nil)
45
+ repo = repository(full_name) or raise ArgumentError, "#{full_name} is not an available repository"
46
+
47
+ {
48
+ repository: repo["full_name"],
49
+ ref: ref.presence || repo["default_branch"],
50
+ clone_url: "https://github.com/#{repo["full_name"]}.git",
51
+ username: "x-access-token",
52
+ token: access_token
53
+ }
54
+ end
55
+
56
+ def as_summary
57
+ {
58
+ login: login,
59
+ avatar_url: avatar_url,
60
+ scopes: scopes.to_s.split(/[\s,]+/).reject(&:blank?),
61
+ repositories: repositories,
62
+ connected_at: created_at&.iso8601,
63
+ updated_at: updated_at&.iso8601
64
+ }
65
+ end
66
+
67
+ private
68
+
69
+ def unique_within_owner
70
+ siblings = self.class.for_owner(owner)
71
+ siblings = siblings.where.not(id: id) if persisted?
72
+ errors.add(:base, "GitHub is already connected") if siblings.exists?
73
+ end
74
+ end
75
+ end