actionagent 1.7.2 → 1.8.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/app/assets/builds/action_agent.css +1 -1
- data/app/assets/builds/action_agent.js +59 -54
- data/app/controllers/action_agent/api/agents_controller.rb +19 -3
- data/app/controllers/action_agent/api/code_sessions_controller.rb +168 -0
- data/app/controllers/action_agent/api/evaluations_controller.rb +74 -131
- data/app/controllers/action_agent/api/github_connections_controller.rb +147 -0
- data/app/controllers/action_agent/api/mcp_controller.rb +38 -5
- data/app/controllers/action_agent/api/mcp_servers_controller.rb +10 -1
- data/app/controllers/action_agent/api/provider_keys_controller.rb +54 -3
- data/app/controllers/action_agent/api/provider_models_controller.rb +10 -6
- data/app/controllers/action_agent/api/sandboxes_controller.rb +105 -6
- data/app/controllers/concerns/action_agent/api/evaluation_run_starting.rb +93 -0
- data/app/controllers/concerns/action_agent/api/mcp_dashboard_tools.rb +509 -0
- data/app/controllers/concerns/action_agent/api/run_sandbox.rb +65 -0
- data/app/jobs/action_agent/code_session_job.rb +171 -0
- data/app/jobs/action_agent/sandbox_cleanup_job.rb +80 -11
- data/app/jobs/action_agent/sandbox_provision_job.rb +122 -14
- data/app/jobs/action_agent/sandbox_run_job.rb +10 -3
- data/app/models/action_agent/agent.rb +49 -6
- data/app/models/action_agent/agent_run.rb +20 -1
- data/app/models/action_agent/code_session.rb +145 -0
- data/app/models/action_agent/evaluation.rb +64 -0
- data/app/models/action_agent/evaluation_run.rb +166 -54
- data/app/models/action_agent/evaluation_scenario_result.rb +15 -3
- data/app/models/action_agent/github_connection.rb +75 -0
- data/app/models/action_agent/model_pricing.rb +214 -34
- data/app/models/action_agent/provider_key.rb +148 -10
- data/app/models/action_agent/sandbox_session.rb +194 -17
- data/app/serializers/action_agent/evaluation_serializer.rb +149 -0
- data/app/serializers/action_agent/telemetry_trace_serializer.rb +10 -2
- data/app/services/action_agent/agent_execution_service.rb +4 -2
- data/app/services/action_agent/agent_scorecard.rb +50 -21
- data/app/services/action_agent/agent_tool_roster.rb +20 -9
- data/app/services/action_agent/claude_code_auth.rb +86 -0
- data/app/services/action_agent/codex_session_events.rb +37 -0
- data/app/services/action_agent/dashboard_assistant_service.rb +47 -5
- data/app/services/action_agent/evaluation_report_import.rb +84 -5
- data/app/services/action_agent/evaluation_run_cost.rb +609 -0
- data/app/services/action_agent/evaluation_runner_service.rb +25 -5
- data/app/services/action_agent/evaluation_standing.rb +145 -0
- data/app/services/action_agent/evaluation_tool_resolver.rb +18 -0
- data/app/services/action_agent/github_client.rb +111 -0
- data/app/services/action_agent/local_sandbox_backend.rb +1715 -0
- data/app/services/action_agent/local_sandbox_databases.rb +257 -0
- data/app/services/action_agent/mcp_client.rb +5 -1
- data/app/services/action_agent/mcp_tool_dispatcher.rb +137 -17
- data/app/services/action_agent/mock_sandbox_backend.rb +39 -0
- data/app/services/action_agent/ollama_host_probe.rb +75 -0
- data/app/services/action_agent/payload_bounds.rb +36 -0
- data/app/services/action_agent/sandbox_manifest.rb +67 -0
- data/app/services/action_agent/sandbox_orchestrator.rb +82 -14
- data/app/services/action_agent/scenario_evaluation_runner.rb +52 -5
- data/app/services/action_agent/secret_scrubber.rb +37 -0
- data/app/services/action_agent/tool_discovery.rb +19 -5
- data/config/routes.rb +23 -4
- data/lib/action_agent/engine.rb +1 -0
- data/lib/action_agent/version.rb +1 -1
- data/lib/action_agent.rb +153 -3
- data/lib/generators/action_agent/install_generator.rb +35 -5
- data/lib/generators/action_agent/templates/action_agent.rb.erb +47 -0
- data/lib/generators/action_agent/templates/add_code_session_runner.rb.erb +9 -0
- data/lib/generators/action_agent/templates/add_provider_key_api_key.rb.erb +25 -0
- data/lib/generators/action_agent/templates/create_active_agent_code_sessions.rb.erb +59 -0
- data/lib/generators/action_agent/templates/create_active_agent_dashboard_tables.rb.erb +2 -0
- data/lib/generators/action_agent/templates/create_active_agent_github_connections.rb.erb +61 -0
- data/lib/tasks/claude_code.rake +16 -0
- data/lib/tasks/sandbox.rake +26 -0
- metadata +28 -2
|
@@ -0,0 +1,145 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module ActionAgent
|
|
4
|
+
# A headless Claude Code session run inside an app_runtime sandbox's
|
|
5
|
+
# checkout: the prompt, the stream-json transcript as it arrived, the
|
|
6
|
+
# outcome Claude Code reported, and the diff the session left in the
|
|
7
|
+
# working tree.
|
|
8
|
+
#
|
|
9
|
+
# The transcript is scrubbed of the sandbox's secrets (its GitHub token and
|
|
10
|
+
# Claude Code credential) before it is stored, and bounded: long strings
|
|
11
|
+
# are truncated and events past MAX_EVENTS are counted rather than kept.
|
|
12
|
+
class CodeSession < ApplicationRecord
|
|
13
|
+
include Ownable
|
|
14
|
+
owned_by :user, :account
|
|
15
|
+
|
|
16
|
+
belongs_to :sandbox_session
|
|
17
|
+
|
|
18
|
+
enum :status, { queued: 0, running: 1, succeeded: 2, failed: 3, cancelled: 4 }
|
|
19
|
+
|
|
20
|
+
MAX_PROMPT_CHARACTERS = 20_000
|
|
21
|
+
MAX_EVENTS = 1_000
|
|
22
|
+
# Per string inside an event: tool results can be whole files.
|
|
23
|
+
MAX_EVENT_STRING = 4_000
|
|
24
|
+
MAX_DIFF_BYTES = 500_000
|
|
25
|
+
RUNNERS = %w[claude_code codex].freeze
|
|
26
|
+
|
|
27
|
+
validates :prompt, presence: true, length: { maximum: MAX_PROMPT_CHARACTERS }
|
|
28
|
+
validates :runner, inclusion: { in: RUNNERS }
|
|
29
|
+
|
|
30
|
+
scope :recent, -> { order(created_at: :desc) }
|
|
31
|
+
|
|
32
|
+
# JSON columns carry no default on MySQL or SQLite (see the migration).
|
|
33
|
+
def events
|
|
34
|
+
Array(super)
|
|
35
|
+
end
|
|
36
|
+
|
|
37
|
+
def finished?
|
|
38
|
+
succeeded? || failed? || cancelled?
|
|
39
|
+
end
|
|
40
|
+
|
|
41
|
+
# Whether an outcome and a diff may still be recorded for this session.
|
|
42
|
+
# finished_at is set once nothing more will be: by CodeSessionJob when it
|
|
43
|
+
# settles a session it ran (whatever the backend returned, raised or
|
|
44
|
+
# refused), or by the cancel of a session that never left the queue. A
|
|
45
|
+
# session cancelled while running is finished at once but settled only
|
|
46
|
+
# when its Claude Code has stopped, so its diff may still come, and its
|
|
47
|
+
# transcript still grow, until then.
|
|
48
|
+
def diff_pending?
|
|
49
|
+
finished_at.nil?
|
|
50
|
+
end
|
|
51
|
+
|
|
52
|
+
# The values that must never be stored: the checkout token and the
|
|
53
|
+
# Claude Code credential this session ran with.
|
|
54
|
+
def secrets
|
|
55
|
+
spec = sandbox_session.checkout_spec rescue nil
|
|
56
|
+
[ spec&.dig(:token), *sandbox_session.runtime_environment(runner: runner).values ].compact
|
|
57
|
+
end
|
|
58
|
+
|
|
59
|
+
# Appends one stream-json event, scrubbed and bounded. Past MAX_EVENTS
|
|
60
|
+
# only the count grows, so a runaway session cannot grow the row without
|
|
61
|
+
# limit.
|
|
62
|
+
def append_event!(event, secrets: self.secrets)
|
|
63
|
+
stored = truncate_strings(SecretScrubber.scrub(event.to_h, secrets))
|
|
64
|
+
|
|
65
|
+
with_lock do
|
|
66
|
+
list = events
|
|
67
|
+
if list.size < MAX_EVENTS
|
|
68
|
+
self.events = list + [ stored ]
|
|
69
|
+
else
|
|
70
|
+
self.dropped_events_count = dropped_events_count.to_i + 1
|
|
71
|
+
end
|
|
72
|
+
save!
|
|
73
|
+
end
|
|
74
|
+
end
|
|
75
|
+
|
|
76
|
+
# Records Claude Code's final "result" event.
|
|
77
|
+
def record_result!(event)
|
|
78
|
+
usage = event["usage"].is_a?(Hash) ? event["usage"] : {}
|
|
79
|
+
|
|
80
|
+
update!(
|
|
81
|
+
result: event["result"].to_s.presence && SecretScrubber.scrub(event["result"].to_s, secrets).truncate(MAX_EVENT_STRING * 4),
|
|
82
|
+
claude_session_id: runner == "claude_code" ? event["session_id"] : nil,
|
|
83
|
+
runner_session_id: event["session_id"],
|
|
84
|
+
num_turns: event["num_turns"],
|
|
85
|
+
duration_ms: event["duration_ms"],
|
|
86
|
+
total_cost_usd: event["total_cost_usd"],
|
|
87
|
+
input_tokens: usage["input_tokens"],
|
|
88
|
+
output_tokens: usage["output_tokens"]
|
|
89
|
+
)
|
|
90
|
+
end
|
|
91
|
+
|
|
92
|
+
def diff=(value)
|
|
93
|
+
text = SecretScrubber.scrub(value.to_s, secrets)
|
|
94
|
+
super(text.bytesize > MAX_DIFF_BYTES ? "#{text.byteslice(0, MAX_DIFF_BYTES).scrub}\n… diff truncated" : text)
|
|
95
|
+
end
|
|
96
|
+
|
|
97
|
+
def summary
|
|
98
|
+
{
|
|
99
|
+
id: id,
|
|
100
|
+
sandbox_session_id: sandbox_session.session_id,
|
|
101
|
+
status: status,
|
|
102
|
+
prompt: prompt,
|
|
103
|
+
model: model,
|
|
104
|
+
runner: runner,
|
|
105
|
+
result: result,
|
|
106
|
+
error_message: error_message,
|
|
107
|
+
num_turns: num_turns,
|
|
108
|
+
duration_ms: duration_ms,
|
|
109
|
+
total_cost_usd: total_cost_usd&.to_f,
|
|
110
|
+
input_tokens: input_tokens,
|
|
111
|
+
output_tokens: output_tokens,
|
|
112
|
+
event_count: events.size + dropped_events_count.to_i,
|
|
113
|
+
# What a client polls until: false once nothing about the session
|
|
114
|
+
# changes any more.
|
|
115
|
+
diff_pending: diff_pending?,
|
|
116
|
+
started_at: started_at&.iso8601,
|
|
117
|
+
finished_at: finished_at&.iso8601,
|
|
118
|
+
created_at: created_at&.iso8601
|
|
119
|
+
}
|
|
120
|
+
end
|
|
121
|
+
|
|
122
|
+
# The summary plus the transcript from event index +after+ onward (for
|
|
123
|
+
# incremental polling) and the diff once finished.
|
|
124
|
+
def details(after: 0)
|
|
125
|
+
after = after.to_i.clamp(0, events.size)
|
|
126
|
+
summary.merge(
|
|
127
|
+
events: events.drop(after),
|
|
128
|
+
events_offset: after,
|
|
129
|
+
dropped_events_count: dropped_events_count.to_i,
|
|
130
|
+
diff: finished? ? diff : nil
|
|
131
|
+
)
|
|
132
|
+
end
|
|
133
|
+
|
|
134
|
+
private
|
|
135
|
+
|
|
136
|
+
def truncate_strings(value)
|
|
137
|
+
case value
|
|
138
|
+
when String then value.length > MAX_EVENT_STRING ? "#{value[0, MAX_EVENT_STRING]}… (truncated)" : value
|
|
139
|
+
when Hash then value.to_h { |key, item| [ key, truncate_strings(item) ] }
|
|
140
|
+
when Array then value.map { |item| truncate_strings(item) }
|
|
141
|
+
else value
|
|
142
|
+
end
|
|
143
|
+
end
|
|
144
|
+
end
|
|
145
|
+
end
|
|
@@ -34,6 +34,20 @@ module ActionAgent
|
|
|
34
34
|
validate :validate_criteria
|
|
35
35
|
|
|
36
36
|
scope :recent, -> { order(updated_at: :desc) }
|
|
37
|
+
# Archived evaluations keep their runs but leave the index and its
|
|
38
|
+
# tiles unless asked for (see #archive!). The mark lives in the config
|
|
39
|
+
# JSON, so the filter reads it with each database's own JSON path.
|
|
40
|
+
scope :archived, -> { where("#{archived_at_sql} IS NOT NULL") }
|
|
41
|
+
scope :unarchived, -> { where("#{archived_at_sql} IS NULL") }
|
|
42
|
+
|
|
43
|
+
# SQL reading config["archived_at"] on the connected database.
|
|
44
|
+
def self.archived_at_sql
|
|
45
|
+
case connection.adapter_name.to_s.downcase
|
|
46
|
+
when /postgres/ then "#{quoted_table_name}.config ->> 'archived_at'"
|
|
47
|
+
when /mysql|trilogy/ then "JSON_UNQUOTE(JSON_EXTRACT(#{quoted_table_name}.config, '$.archived_at'))"
|
|
48
|
+
else "json_extract(#{quoted_table_name}.config, '$.archived_at')"
|
|
49
|
+
end
|
|
50
|
+
end
|
|
37
51
|
|
|
38
52
|
# MySQL cannot give a JSON column a default, so a row inserted there
|
|
39
53
|
# without `criteria` or `config` reads back nil. Both readers answer with
|
|
@@ -50,6 +64,56 @@ module ActionAgent
|
|
|
50
64
|
evaluation_runs.order(created_at: :desc).first
|
|
51
65
|
end
|
|
52
66
|
|
|
67
|
+
# The newest run that finished: the one the evaluation's pass rate
|
|
68
|
+
# describes. A newer run still pending, or one that failed, shows
|
|
69
|
+
# beside it and never in its place. Read from the loaded association
|
|
70
|
+
# when the index preloaded it.
|
|
71
|
+
def headline_run
|
|
72
|
+
if evaluation_runs.loaded?
|
|
73
|
+
evaluation_runs.select(&:complete?).max_by { |run| [ run.created_at, run.id ] }
|
|
74
|
+
else
|
|
75
|
+
evaluation_runs.complete.order(created_at: :desc, id: :desc).first
|
|
76
|
+
end
|
|
77
|
+
end
|
|
78
|
+
|
|
79
|
+
# --- archiving ---------------------------------------------------------
|
|
80
|
+
#
|
|
81
|
+
# An evaluation nobody maintains — its suite superseded, its agent
|
|
82
|
+
# retired — keeps its history but stops counting: the index leaves it
|
|
83
|
+
# out, and so do the pooled pass rate and the agent's scorecard. The
|
|
84
|
+
# mark is config["archived_at"]; a new run or a published report clears
|
|
85
|
+
# it, since either says the evaluation is alive after all.
|
|
86
|
+
|
|
87
|
+
def archived_at
|
|
88
|
+
value = config["archived_at"]
|
|
89
|
+
value.present? ? Time.zone.parse(value.to_s) : nil
|
|
90
|
+
rescue ArgumentError
|
|
91
|
+
nil
|
|
92
|
+
end
|
|
93
|
+
|
|
94
|
+
def archived?
|
|
95
|
+
config["archived_at"].present?
|
|
96
|
+
end
|
|
97
|
+
|
|
98
|
+
def archive!
|
|
99
|
+
update!(config: config.merge("archived_at" => Time.current.iso8601))
|
|
100
|
+
end
|
|
101
|
+
|
|
102
|
+
def unarchive!
|
|
103
|
+
return unless archived?
|
|
104
|
+
|
|
105
|
+
update!(config: config.except("archived_at"))
|
|
106
|
+
end
|
|
107
|
+
|
|
108
|
+
# Where this evaluation stands against the agent as it is now: whether
|
|
109
|
+
# its headline run scored the current version (EvaluationStanding).
|
|
110
|
+
# Memoized per instance; the index preloads what it needs.
|
|
111
|
+
def standing_info
|
|
112
|
+
@standing_info ||= EvaluationStanding.new(self)
|
|
113
|
+
end
|
|
114
|
+
|
|
115
|
+
attr_writer :standing_info
|
|
116
|
+
|
|
53
117
|
def judge_defined?
|
|
54
118
|
judge_kind == "judge_defined"
|
|
55
119
|
end
|
|
@@ -9,15 +9,29 @@ module ActionAgent
|
|
|
9
9
|
class EvaluationRun < ApplicationRecord
|
|
10
10
|
belongs_to :evaluation
|
|
11
11
|
# The version of the evaluated agent this run scored, so a pass rate is
|
|
12
|
-
# a statement about a release rather than about "the agent".
|
|
12
|
+
# a statement about a release rather than about "the agent". A run the
|
|
13
|
+
# engine executes scored the agent as it is now; an imported run scored
|
|
14
|
+
# the publishing application's code, which only its report can name
|
|
15
|
+
# (EvaluationReportImport sets the version from `report.release`), so
|
|
16
|
+
# a release-less import stays unrecorded rather than claiming the
|
|
17
|
+
# dashboard's latest version.
|
|
13
18
|
belongs_to :agent_version, optional: true
|
|
14
|
-
before_create { self.agent_version_id ||= evaluation&.agent&.latest_version&.id }
|
|
19
|
+
before_create { self.agent_version_id ||= evaluation&.agent&.latest_version&.id unless imported? }
|
|
20
|
+
# A run started on an archived evaluation brings it back: archiving says
|
|
21
|
+
# "no longer maintained", which a new run contradicts.
|
|
22
|
+
after_create { evaluation.unarchive! if evaluation&.archived? }
|
|
15
23
|
has_many :scenario_results, class_name: "EvaluationScenarioResult", dependent: :destroy
|
|
16
24
|
|
|
17
25
|
enum :status, { pending: 0, running: 1, complete: 2, failed: 3 }
|
|
18
26
|
|
|
19
27
|
scope :recent, -> { order(created_at: :desc) }
|
|
20
28
|
|
|
29
|
+
# Whether this run was published by an application that ran it itself
|
|
30
|
+
# (EvaluationReportImport) rather than executed here.
|
|
31
|
+
def imported?
|
|
32
|
+
external_run_id.present?
|
|
33
|
+
end
|
|
34
|
+
|
|
21
35
|
# Which scenarios and models a scenario run covered; empty for a
|
|
22
36
|
# generation-sampling run.
|
|
23
37
|
def selection
|
|
@@ -25,6 +39,19 @@ module ActionAgent
|
|
|
25
39
|
value.is_a?(Hash) ? value : {}
|
|
26
40
|
end
|
|
27
41
|
|
|
42
|
+
# The checkout sandbox a scenario run replayed against
|
|
43
|
+
# (ScenarioEvaluationRunner records it), as { "session_id", "server_key",
|
|
44
|
+
# "repository", "repository_ref" }; nil for a run against the agent's own
|
|
45
|
+
# servers only.
|
|
46
|
+
def sandbox
|
|
47
|
+
value = selection["sandbox"]
|
|
48
|
+
return value if value.is_a?(Hash)
|
|
49
|
+
|
|
50
|
+
# Still pending: run_later! recorded only what was asked for.
|
|
51
|
+
id = selection["sandbox_id"]
|
|
52
|
+
{ "session_id" => id } if id.is_a?(String) && id.present?
|
|
53
|
+
end
|
|
54
|
+
|
|
28
55
|
# The candidate models a scenario run compared, in the order they were
|
|
29
56
|
# requested; empty for a generation-sampling run.
|
|
30
57
|
def models
|
|
@@ -79,14 +106,37 @@ module ActionAgent
|
|
|
79
106
|
(values.sum.to_f / values.size).round(3)
|
|
80
107
|
end
|
|
81
108
|
|
|
82
|
-
# The judge's own spend on this run
|
|
83
|
-
# how many calls served each purpose
|
|
84
|
-
# for a run
|
|
85
|
-
|
|
109
|
+
# The judge's own spend on this run as the engine's meter recorded it —
|
|
110
|
+
# calls, tokens, estimated cost and how many calls served each purpose
|
|
111
|
+
# — or nil for a run the engine did not judge (a rules-only run, or an
|
|
112
|
+
# imported one, whose judge the application ran).
|
|
113
|
+
def judge_usage_meter
|
|
86
114
|
value = scores&.dig("_judge_usage")
|
|
87
115
|
value.is_a?(Hash) ? value : nil
|
|
88
116
|
end
|
|
89
117
|
|
|
118
|
+
# The judge's spend on this run from whatever recorded it: the meter,
|
|
119
|
+
# the publishing application's figures, or the judge traces priced
|
|
120
|
+
# (EvaluationRunCost). nil for a run no judge was asked about.
|
|
121
|
+
def judge_usage
|
|
122
|
+
cost_breakdown.judge_usage
|
|
123
|
+
end
|
|
124
|
+
|
|
125
|
+
# Every cost figure of this run — per result, per scenario, per model
|
|
126
|
+
# and in total — worked out once (EvaluationRunCost). A controller that
|
|
127
|
+
# lists runs preloads it (EvaluationRunCost.preload) and hands it in.
|
|
128
|
+
def cost_breakdown
|
|
129
|
+
@cost_breakdown ||= EvaluationRunCost.for(self)
|
|
130
|
+
end
|
|
131
|
+
|
|
132
|
+
attr_writer :cost_breakdown
|
|
133
|
+
|
|
134
|
+
# Forgets the breakdown, so a run whose results just changed is priced again.
|
|
135
|
+
def reload(*)
|
|
136
|
+
@cost_breakdown = nil
|
|
137
|
+
super
|
|
138
|
+
end
|
|
139
|
+
|
|
90
140
|
# Per-model summaries of a generation-sampling run's cohorts, keyed by
|
|
91
141
|
# model; empty for a scenario run or a run recorded before they were.
|
|
92
142
|
def cohorts
|
|
@@ -94,59 +144,51 @@ module ActionAgent
|
|
|
94
144
|
value.is_a?(Hash) ? value : {}
|
|
95
145
|
end
|
|
96
146
|
|
|
147
|
+
# Per-model summaries of a scenario run, keyed by label, as the runner
|
|
148
|
+
# recorded them under "_models", each with its cost as it stands now:
|
|
149
|
+
# the effective "cost" over its results (reported, else estimated), the
|
|
150
|
+
# "priced", "reported" and "estimated" counts behind it, and the
|
|
151
|
+
# judge's "judge_cost" and "judge_calls" on them (EvaluationRunCost). A
|
|
152
|
+
# recorded summary no result maps to is served as recorded. Empty for a
|
|
153
|
+
# generation-sampling run.
|
|
154
|
+
def model_summaries
|
|
155
|
+
summaries = scores&.dig("_models")
|
|
156
|
+
return {} unless summaries.is_a?(Hash)
|
|
157
|
+
|
|
158
|
+
costed = cost_breakdown.by_label(summaries.keys)
|
|
159
|
+
summaries.to_h do |label, stats|
|
|
160
|
+
next [ label, stats ] unless stats.is_a?(Hash) && costed.key?(label)
|
|
161
|
+
|
|
162
|
+
[ label, stats.merge(costed[label]) ]
|
|
163
|
+
end
|
|
164
|
+
end
|
|
165
|
+
|
|
97
166
|
# What the run spent, for display after it: the agent's side and the
|
|
98
167
|
# judge's, kept apart because they answer different questions.
|
|
99
168
|
#
|
|
100
169
|
# The agent's side is the operating figure — what the interactions cost
|
|
101
|
-
# to serve. For a scenario run that is
|
|
102
|
-
#
|
|
103
|
-
#
|
|
104
|
-
#
|
|
105
|
-
#
|
|
106
|
-
#
|
|
170
|
+
# to serve. For a scenario run that is its replays' cost, tokens and
|
|
171
|
+
# summed model time (`replays` of them); for a generation-sampling run
|
|
172
|
+
# it is the sampled generations' (`samples`), which were served before
|
|
173
|
+
# the run and cost it nothing.
|
|
174
|
+
#
|
|
175
|
+
# Every interaction with tokens is priced (EvaluationRunCost): `cost`
|
|
176
|
+
# sums the reported costs and, where none was reported, the estimates
|
|
177
|
+
# from tokens × model rates. `priced` and `unpriced` count the
|
|
178
|
+
# interactions either way, `reported` and `estimated` say how the priced
|
|
179
|
+
# ones were priced, and `cost_basis` sums that up as "reported",
|
|
180
|
+
# "estimated" or "mixed". `per_interaction` is the cost per priced
|
|
181
|
+
# interaction, the number a per-conversation budget is set against.
|
|
107
182
|
#
|
|
108
183
|
# `judge` is the evaluation's own overhead: the judge model's calls
|
|
109
184
|
# (scoring, recommending, the verdict, authoring KPIs), which run
|
|
110
|
-
# agent-to-agent and offline
|
|
185
|
+
# agent-to-agent and offline — from the engine's meter, the publishing
|
|
186
|
+
# application's figures or the judge traces, with `source` naming which
|
|
187
|
+
# and `run` the calls no result owns. `total` is the two sides together.
|
|
111
188
|
#
|
|
112
189
|
# Returns nil for a run that recorded nothing on either side.
|
|
113
190
|
def usage
|
|
114
|
-
|
|
115
|
-
Arel.sql("COUNT(*)"), Arel.sql("SUM(cost)"), Arel.sql("SUM(input_tokens)"),
|
|
116
|
-
Arel.sql("SUM(output_tokens)"), Arel.sql("SUM(duration_ms)")
|
|
117
|
-
)
|
|
118
|
-
replays = totals&.first.to_i
|
|
119
|
-
judge = judge_usage
|
|
120
|
-
runtime_ms = completed_at.present? ? ((completed_at - created_at) * 1000).round : nil
|
|
121
|
-
|
|
122
|
-
if replays.positive?
|
|
123
|
-
cost = totals[1]&.to_f
|
|
124
|
-
{
|
|
125
|
-
replays: replays,
|
|
126
|
-
cost: cost,
|
|
127
|
-
per_interaction: cost && (cost / replays).round(6),
|
|
128
|
-
input_tokens: totals[2].to_i,
|
|
129
|
-
output_tokens: totals[3].to_i,
|
|
130
|
-
model_time_ms: totals[4].to_i,
|
|
131
|
-
runtime_ms: runtime_ms,
|
|
132
|
-
judge: judge
|
|
133
|
-
}.compact
|
|
134
|
-
elsif cohorts.any?
|
|
135
|
-
samples = cohorts.values.sum { |cohort| cohort["samples"].to_i }
|
|
136
|
-
costs = cohorts.values.filter_map { |cohort| cohort["cost"] }
|
|
137
|
-
cost = costs.any? ? costs.sum.to_f.round(6) : nil
|
|
138
|
-
{
|
|
139
|
-
samples: samples,
|
|
140
|
-
cost: cost,
|
|
141
|
-
per_interaction: cost && samples.positive? ? (cost / samples).round(6) : nil,
|
|
142
|
-
input_tokens: cohorts.values.sum { |cohort| cohort["input_tokens"].to_i },
|
|
143
|
-
output_tokens: cohorts.values.sum { |cohort| cohort["output_tokens"].to_i },
|
|
144
|
-
runtime_ms: runtime_ms,
|
|
145
|
-
judge: judge
|
|
146
|
-
}.compact
|
|
147
|
-
elsif judge
|
|
148
|
-
{ runtime_ms: runtime_ms, judge: judge }.compact
|
|
149
|
-
end
|
|
191
|
+
cost_breakdown.usage
|
|
150
192
|
end
|
|
151
193
|
|
|
152
194
|
# Route templates for the report's fix item actions, relative to the
|
|
@@ -181,10 +223,21 @@ module ActionAgent
|
|
|
181
223
|
# the suite panel does. Raises ActiveRecord::RecordNotFound via the
|
|
182
224
|
# caller for a run of a generation-sampling evaluation, which has no
|
|
183
225
|
# scenario results to report on.
|
|
184
|
-
|
|
226
|
+
#
|
|
227
|
+
# With `estimate` (the default) each replay carries its effective cost
|
|
228
|
+
# and how it was priced (EvaluationRunCost), and the report is told the
|
|
229
|
+
# judge's run-level spend, so the page shows every cost the dashboard
|
|
230
|
+
# does. `estimate: false` rebuilds the report from what was recorded
|
|
231
|
+
# alone — the import summarizes a published run that way, so the
|
|
232
|
+
# summaries it stores stay the application's own figures.
|
|
233
|
+
#
|
|
234
|
+
# The engine runs against every activeagent since 1.4, so the report
|
|
235
|
+
# kwargs that arrived later are passed only when Report.new takes them.
|
|
236
|
+
def to_report(links: report_links, estimate: true)
|
|
185
237
|
rows = scenario_results.includes(:scenario).sort_by do |row|
|
|
186
238
|
[ row.evaluated_scenario["position"].to_i, row.evaluation_scenario_id, row.model ]
|
|
187
239
|
end
|
|
240
|
+
breakdown = cost_breakdown if estimate
|
|
188
241
|
selected = selected_specs
|
|
189
242
|
specs = {}
|
|
190
243
|
results = rows.map do |row|
|
|
@@ -197,28 +250,63 @@ module ActionAgent
|
|
|
197
250
|
replay: ActiveAgent::Evals::Replay.new(
|
|
198
251
|
answer: row.output, tool_calls: Array(row.tool_calls), duration_ms: row.duration_ms,
|
|
199
252
|
input_tokens: row.input_tokens, output_tokens: row.output_tokens,
|
|
200
|
-
cost: row.cost&.to_f, error: row.error_message,
|
|
253
|
+
cost: breakdown ? breakdown.result(row)["cost"] : row.cost&.to_f, error: row.error_message,
|
|
254
|
+
metadata: replay_metadata_for(row, breakdown)
|
|
201
255
|
),
|
|
202
256
|
scores: row.scores.to_h, score: row.score, status: row.status,
|
|
203
257
|
diagnosis: row.evaluation_diagnosis.presence
|
|
204
258
|
)
|
|
205
259
|
end
|
|
206
260
|
|
|
207
|
-
|
|
261
|
+
kwargs = {
|
|
208
262
|
results: results,
|
|
209
263
|
models: (selected.values & specs.values) + (specs.values - selected.values),
|
|
210
264
|
metadata: {
|
|
211
265
|
"evaluation" => evaluation.name,
|
|
212
266
|
"agent" => evaluation.agent&.name,
|
|
213
267
|
"run" => id,
|
|
214
|
-
"finished" => completed_at&.iso8601
|
|
268
|
+
"finished" => completed_at&.iso8601,
|
|
269
|
+
"sandbox" => sandbox_label
|
|
215
270
|
}.compact.merge(report_metadata),
|
|
216
271
|
verdict: recorded_verdict,
|
|
217
272
|
judge_label: judge_label,
|
|
218
273
|
tool_resolver: EvaluationToolResolver.new(evaluation.agent),
|
|
219
274
|
agent_name: evaluation.agent&.name,
|
|
220
275
|
links: links
|
|
221
|
-
|
|
276
|
+
}
|
|
277
|
+
kwargs[:judge_usage] = breakdown.judge_usage_run if breakdown && self.class.report_accepts?(:judge_usage)
|
|
278
|
+
kwargs[:release] = release_summary if release_summary && self.class.report_accepts?(:release)
|
|
279
|
+
self.class.report_class.new(**kwargs)
|
|
280
|
+
end
|
|
281
|
+
|
|
282
|
+
# The framework's Report, as installed.
|
|
283
|
+
def self.report_class
|
|
284
|
+
ActiveAgent::Evals::Report
|
|
285
|
+
end
|
|
286
|
+
|
|
287
|
+
# Whether the installed framework's Report.new declares +keyword+.
|
|
288
|
+
def self.report_accepts?(keyword)
|
|
289
|
+
report_class.instance_method(:initialize).parameters.any? { |type, name| name == keyword && %i[key keyreq].include?(type) }
|
|
290
|
+
end
|
|
291
|
+
|
|
292
|
+
# The release this run scored, as the report names it — `{ "digest",
|
|
293
|
+
# "revision", "label" }` — or nil for a run pinned to a dashboard edit
|
|
294
|
+
# or to no version.
|
|
295
|
+
def release_summary
|
|
296
|
+
version = agent_version
|
|
297
|
+
return nil unless version&.release?
|
|
298
|
+
|
|
299
|
+
{ "digest" => version.release_digest, "revision" => version.revision, "label" => "v#{version.version_number}" }.compact
|
|
300
|
+
end
|
|
301
|
+
|
|
302
|
+
# The agent version this run scored, for the run's JSON: `{ id, number,
|
|
303
|
+
# release_digest, revision, release }`, or nil when none was recorded.
|
|
304
|
+
def agent_version_summary
|
|
305
|
+
version = agent_version
|
|
306
|
+
return nil unless version
|
|
307
|
+
|
|
308
|
+
{ id: version.id, number: version.version_number, release_digest: version.release_digest,
|
|
309
|
+
revision: version.revision, release: version.release? }
|
|
222
310
|
end
|
|
223
311
|
|
|
224
312
|
private
|
|
@@ -226,6 +314,30 @@ module ActionAgent
|
|
|
226
314
|
ModelSpec = ActiveAgent::Evals::ModelSpec
|
|
227
315
|
private_constant :ModelSpec
|
|
228
316
|
|
|
317
|
+
# The replay metadata the rebuilt report reads a result's costs from:
|
|
318
|
+
# what the result recorded, the judge usage the application reported
|
|
319
|
+
# for it, and — when estimating — how its cost was priced.
|
|
320
|
+
def replay_metadata_for(row, breakdown)
|
|
321
|
+
metadata = row.replay_metadata.dup
|
|
322
|
+
reported_judge = row.diagnosis.is_a?(Hash) ? row.diagnosis["_judge_usage"] : nil
|
|
323
|
+
metadata["judge_usage"] = reported_judge if reported_judge.is_a?(Hash)
|
|
324
|
+
return metadata unless breakdown
|
|
325
|
+
|
|
326
|
+
entry = breakdown.result(row)
|
|
327
|
+
metadata["cost_source"] = entry["cost_source"]
|
|
328
|
+
metadata["cost_rate"] = entry["cost_rate"] if entry["cost_rate"]
|
|
329
|
+
metadata["judge_usage"] = entry["judge_usage"] if entry["judge_usage"]
|
|
330
|
+
metadata
|
|
331
|
+
end
|
|
332
|
+
|
|
333
|
+
# How the report's header names the sandbox: its checkout and session.
|
|
334
|
+
def sandbox_label
|
|
335
|
+
return nil unless sandbox
|
|
336
|
+
|
|
337
|
+
checkout = [ sandbox["repository"], sandbox["repository_ref"] ].compact_blank.join("@")
|
|
338
|
+
[ checkout.presence, "sandbox #{sandbox["session_id"].to_s.first(8)}" ].compact.join(" · ")
|
|
339
|
+
end
|
|
340
|
+
|
|
229
341
|
# The candidate specs the run was asked to compare, keyed by
|
|
230
342
|
# [provider, model] in the order requested. ScenarioEvaluationRunner
|
|
231
343
|
# persists each ModelSpec#to_h in `selection`, and it is that label —
|
|
@@ -44,8 +44,11 @@ module ActionAgent
|
|
|
44
44
|
value.is_a?(Hash) ? value : {}
|
|
45
45
|
end
|
|
46
46
|
|
|
47
|
+
# The diagnosis as the dashboard shows it: without the storage keys —
|
|
48
|
+
# the replay metadata, the scenario snapshot and the judge usage the
|
|
49
|
+
# publishing application reported (served as `judge_usage` instead).
|
|
47
50
|
def evaluation_diagnosis
|
|
48
|
-
(diagnosis || {}).except("_replay_metadata", "_scenario_snapshot")
|
|
51
|
+
(diagnosis || {}).except("_replay_metadata", "_scenario_snapshot", "_judge_usage")
|
|
49
52
|
end
|
|
50
53
|
|
|
51
54
|
# A catalog can be refreshed without changing what an earlier run asked
|
|
@@ -55,7 +58,12 @@ module ActionAgent
|
|
|
55
58
|
snapshot.is_a?(Hash) ? snapshot : scenario.as_json_summary.stringify_keys
|
|
56
59
|
end
|
|
57
60
|
|
|
58
|
-
|
|
61
|
+
# The result for the JSON API. Its cost is the effective one — reported,
|
|
62
|
+
# else estimated — with `reported_cost`, `cost_source`, `cost_rate` and
|
|
63
|
+
# the judge's `judge_usage` on it (EvaluationRunCost). A caller listing
|
|
64
|
+
# a run's results hands in the run's breakdown so nothing is priced per row.
|
|
65
|
+
def as_json_summary(costs: nil)
|
|
66
|
+
costs ||= evaluation_run.cost_breakdown.result(self)
|
|
59
67
|
{
|
|
60
68
|
id: id,
|
|
61
69
|
scenario_id: evaluation_scenario_id,
|
|
@@ -73,7 +81,11 @@ module ActionAgent
|
|
|
73
81
|
duration_ms: duration_ms,
|
|
74
82
|
input_tokens: input_tokens,
|
|
75
83
|
output_tokens: output_tokens,
|
|
76
|
-
cost: cost
|
|
84
|
+
cost: costs["cost"],
|
|
85
|
+
reported_cost: costs["reported_cost"],
|
|
86
|
+
cost_source: costs["cost_source"],
|
|
87
|
+
cost_rate: costs["cost_rate"],
|
|
88
|
+
judge_usage: costs["judge_usage"],
|
|
77
89
|
fault: fault,
|
|
78
90
|
recommendation: recommendation,
|
|
79
91
|
diagnosis: evaluation_diagnosis,
|
|
@@ -0,0 +1,75 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module ActionAgent
|
|
4
|
+
# An owner's GitHub OAuth grant (Settings -> Integrations) and the
|
|
5
|
+
# repositories they made available to the workspace.
|
|
6
|
+
#
|
|
7
|
+
# The access token is encrypted at rest like a provider key and never
|
|
8
|
+
# rendered back to the client. +repositories+ holds only what GitHub
|
|
9
|
+
# itself listed for this token when the owner chose them (see
|
|
10
|
+
# Api::GithubConnectionsController#update), so a checkout sandbox can
|
|
11
|
+
# trust a name found here without asking GitHub again.
|
|
12
|
+
class GithubConnection < ApplicationRecord
|
|
13
|
+
include Ownable
|
|
14
|
+
owned_by :account, :user
|
|
15
|
+
|
|
16
|
+
encrypts :access_token if ActionAgent.encrypt_credentials
|
|
17
|
+
|
|
18
|
+
validates :access_token, :github_user_id, :login, presence: true
|
|
19
|
+
# One connection per owner; which column that means depends on the
|
|
20
|
+
# configured mode, so it is checked at validation time.
|
|
21
|
+
validate :unique_within_owner
|
|
22
|
+
|
|
23
|
+
# JSON columns carry no default on MySQL or SQLite (see the migration), so
|
|
24
|
+
# an unset column reads as nil.
|
|
25
|
+
def repositories
|
|
26
|
+
Array(super)
|
|
27
|
+
end
|
|
28
|
+
|
|
29
|
+
def repository_names
|
|
30
|
+
repositories.map { |repo| repo["full_name"] }
|
|
31
|
+
end
|
|
32
|
+
|
|
33
|
+
def repository(full_name)
|
|
34
|
+
repositories.find { |repo| repo["full_name"].casecmp?(full_name.to_s) }
|
|
35
|
+
end
|
|
36
|
+
|
|
37
|
+
def client
|
|
38
|
+
GithubClient.new(access_token)
|
|
39
|
+
end
|
|
40
|
+
|
|
41
|
+
# The checkout a sandbox backend clones: repository, ref, and an
|
|
42
|
+
# authenticated HTTPS clone URL. Carries the token — hand it to a backend,
|
|
43
|
+
# never to a response.
|
|
44
|
+
def checkout_spec(full_name, ref: nil)
|
|
45
|
+
repo = repository(full_name) or raise ArgumentError, "#{full_name} is not an available repository"
|
|
46
|
+
|
|
47
|
+
{
|
|
48
|
+
repository: repo["full_name"],
|
|
49
|
+
ref: ref.presence || repo["default_branch"],
|
|
50
|
+
clone_url: "https://github.com/#{repo["full_name"]}.git",
|
|
51
|
+
username: "x-access-token",
|
|
52
|
+
token: access_token
|
|
53
|
+
}
|
|
54
|
+
end
|
|
55
|
+
|
|
56
|
+
def as_summary
|
|
57
|
+
{
|
|
58
|
+
login: login,
|
|
59
|
+
avatar_url: avatar_url,
|
|
60
|
+
scopes: scopes.to_s.split(/[\s,]+/).reject(&:blank?),
|
|
61
|
+
repositories: repositories,
|
|
62
|
+
connected_at: created_at&.iso8601,
|
|
63
|
+
updated_at: updated_at&.iso8601
|
|
64
|
+
}
|
|
65
|
+
end
|
|
66
|
+
|
|
67
|
+
private
|
|
68
|
+
|
|
69
|
+
def unique_within_owner
|
|
70
|
+
siblings = self.class.for_owner(owner)
|
|
71
|
+
siblings = siblings.where.not(id: id) if persisted?
|
|
72
|
+
errors.add(:base, "GitHub is already connected") if siblings.exists?
|
|
73
|
+
end
|
|
74
|
+
end
|
|
75
|
+
end
|