actionagent 1.8.1 → 1.9.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/app/assets/builds/action_agent.css +1 -1
- data/app/assets/builds/action_agent.js +76 -73
- data/app/assets/builds/action_agent_recorder.js +78 -0
- data/app/assets/builds/action_agent_replay.js +226 -0
- data/app/controllers/action_agent/api/agent_runs_controller.rb +3 -1
- data/app/controllers/action_agent/api/agents_controller.rb +1 -0
- data/app/controllers/action_agent/api/api_keys_controller.rb +12 -2
- data/app/controllers/action_agent/api/base_controller.rb +63 -2
- data/app/controllers/action_agent/api/claude_logins_controller.rb +93 -0
- data/app/controllers/action_agent/api/code_sessions_controller.rb +53 -6
- data/app/controllers/action_agent/api/dashboard_assistant_controller.rb +2 -2
- data/app/controllers/action_agent/api/draft_pull_requests_controller.rb +248 -0
- data/app/controllers/action_agent/api/evaluation_fixes_controller.rb +27 -0
- data/app/controllers/action_agent/api/evaluations_controller.rb +9 -4
- data/app/controllers/action_agent/api/explorations_controller.rb +213 -0
- data/app/controllers/action_agent/api/feedback_ingest_controller.rb +34 -0
- data/app/controllers/action_agent/api/feedbacks_controller.rb +40 -0
- data/app/controllers/action_agent/api/github_app_manifests_controller.rb +125 -0
- data/app/controllers/action_agent/api/github_connections_controller.rb +40 -2
- data/app/controllers/action_agent/api/github_installations_controller.rb +280 -0
- data/app/controllers/action_agent/api/input_requests_controller.rb +77 -0
- data/app/controllers/action_agent/api/mcp_controller.rb +46 -12
- data/app/controllers/action_agent/api/members_controller.rb +36 -0
- data/app/controllers/action_agent/api/project_explorations_controller.rb +80 -0
- data/app/controllers/action_agent/api/project_install_pull_requests_controller.rb +305 -0
- data/app/controllers/action_agent/api/project_secrets_controller.rb +96 -0
- data/app/controllers/action_agent/api/project_setup_controller.rb +151 -0
- data/app/controllers/action_agent/api/project_sign_ins_controller.rb +112 -0
- data/app/controllers/action_agent/api/projects_controller.rb +512 -0
- data/app/controllers/action_agent/api/provider_keys_controller.rb +176 -24
- data/app/controllers/action_agent/api/provider_models_controller.rb +37 -15
- data/app/controllers/action_agent/api/recording_event_ingest_controller.rb +67 -0
- data/app/controllers/action_agent/api/sandbox_browser_tickets_controller.rb +50 -0
- data/app/controllers/action_agent/api/sandbox_browsers_controller.rb +65 -0
- data/app/controllers/action_agent/api/sandboxes_controller.rb +140 -8
- data/app/controllers/action_agent/api/scenario_catalog_entries_controller.rb +188 -0
- data/app/controllers/action_agent/api/scenario_catalogs_controller.rb +302 -0
- data/app/controllers/action_agent/api/session_recordings_controller.rb +165 -34
- data/app/controllers/action_agent/api/sessions_controller.rb +111 -0
- data/app/controllers/action_agent/api/telemetry_keys_controller.rb +21 -0
- data/app/controllers/action_agent/dashboard_controller.rb +17 -6
- data/app/controllers/action_agent/session_player_controller.rb +58 -0
- data/app/controllers/concerns/action_agent/api/agent_serialization.rb +1 -0
- data/app/controllers/concerns/action_agent/api/mcp_dashboard_tools.rb +623 -18
- data/app/controllers/concerns/action_agent/api/project_browser.rb +46 -0
- data/app/controllers/concerns/action_agent/api/project_secret_authorization.rb +57 -0
- data/app/jobs/action_agent/agent_execution_job.rb +9 -39
- data/app/jobs/action_agent/agent_resume_job.rb +84 -0
- data/app/jobs/action_agent/code_session_job.rb +26 -2
- data/app/jobs/action_agent/draft_pull_request_job.rb +43 -0
- data/app/jobs/action_agent/exploration_job.rb +80 -0
- data/app/jobs/action_agent/input_request_expiry_job.rb +22 -0
- data/app/jobs/action_agent/project_evaluation_job.rb +59 -0
- data/app/jobs/action_agent/project_setup_job.rb +21 -0
- data/app/jobs/action_agent/sandbox_cleanup_job.rb +20 -3
- data/app/jobs/action_agent/sandbox_provision_job.rb +77 -10
- data/app/jobs/action_agent/sandbox_run_job.rb +9 -37
- data/app/jobs/action_agent/verify_evaluation_fix_job.rb +37 -0
- data/app/models/action_agent/agent.rb +39 -26
- data/app/models/action_agent/agent_run.rb +134 -23
- data/app/models/action_agent/agent_template.rb +35 -1
- data/app/models/action_agent/api_key.rb +14 -0
- data/app/models/action_agent/catalog_scenario.rb +75 -0
- data/app/models/action_agent/code_session.rb +16 -4
- data/app/models/action_agent/draft_pull_request.rb +131 -0
- data/app/models/action_agent/evaluation.rb +84 -0
- data/app/models/action_agent/exploration.rb +847 -0
- data/app/models/action_agent/github_installation.rb +159 -0
- data/app/models/action_agent/input_request.rb +291 -0
- data/app/models/action_agent/interaction_feedback.rb +24 -0
- data/app/models/action_agent/project.rb +789 -0
- data/app/models/action_agent/project_secret.rb +339 -0
- data/app/models/action_agent/provider_key.rb +200 -3
- data/app/models/action_agent/recording_event.rb +145 -0
- data/app/models/action_agent/recording_snapshot.rb +7 -7
- data/app/models/action_agent/sandbox_session.rb +297 -14
- data/app/models/action_agent/scenario_catalog.rb +172 -0
- data/app/models/action_agent/scenario_product.rb +57 -0
- data/app/models/action_agent/scenario_set.rb +138 -0
- data/app/models/action_agent/session_recording.rb +121 -14
- data/app/models/action_agent/telemetry_trace.rb +74 -19
- data/app/models/concerns/action_agent/ownable.rb +84 -13
- data/app/serializers/action_agent/input_request_serializer.rb +47 -0
- data/app/serializers/action_agent/scenario_catalog_serializer.rb +23 -0
- data/app/serializers/action_agent/session_summary_serializer.rb +146 -0
- data/app/services/action_agent/agent_execution_service.rb +450 -72
- data/app/services/action_agent/agent_sync.rb +5 -2
- data/app/services/action_agent/agent_tool_roster.rb +19 -4
- data/app/services/action_agent/agent_toolbox.rb +215 -9
- data/app/services/action_agent/browser_live_ticket.rb +78 -0
- data/app/services/action_agent/browser_sidecar.rb +213 -0
- data/app/services/action_agent/browser_sign_in.rb +163 -0
- data/app/services/action_agent/browser_tool_redaction.rb +92 -0
- data/app/services/action_agent/claude_code_auth.rb +70 -4
- data/app/services/action_agent/dashboard_assistant_service.rb +24 -10
- data/app/services/action_agent/draft_pull_request_publisher.rb +719 -0
- data/app/services/action_agent/evaluation_fix.rb +101 -0
- data/app/services/action_agent/evaluation_fix_comparison.rb +66 -0
- data/app/services/action_agent/evaluation_runner_service.rb +4 -5
- data/app/services/action_agent/evaluation_standing.rb +15 -8
- data/app/services/action_agent/explorer_execution_service.rb +517 -0
- data/app/services/action_agent/github_client.rb +405 -64
- data/app/services/action_agent/live_updates.rb +31 -0
- data/app/services/action_agent/local_sandbox_backend.rb +1218 -83
- data/app/services/action_agent/local_sandbox_claude_login.rb +151 -0
- data/app/services/action_agent/mcp_client.rb +6 -2
- data/app/services/action_agent/mcp_recording_middleware.rb +104 -145
- data/app/services/action_agent/mcp_tool_dispatcher.rb +36 -13
- data/app/services/action_agent/mock_sandbox_backend.rb +3 -1
- data/app/services/action_agent/playwright_mcp_client.rb +7 -0
- data/app/services/action_agent/project_boot_fix.rb +309 -0
- data/app/services/action_agent/project_capabilities.rb +200 -0
- data/app/services/action_agent/project_install_pull_request.rb +225 -0
- data/app/services/action_agent/project_preflight.rb +178 -0
- data/app/services/action_agent/project_secret_discovery.rb +142 -0
- data/app/services/action_agent/project_setup.rb +429 -0
- data/app/services/action_agent/provider_credentials.rb +102 -0
- data/app/services/action_agent/recording_event_ingest.rb +181 -0
- data/app/services/action_agent/runtime_tool_roster.rb +52 -0
- data/app/services/action_agent/sample_data/conference_ticket.rb +516 -0
- data/app/services/action_agent/sandbox_boot_spec.rb +558 -0
- data/app/services/action_agent/sandbox_browser.rb +299 -0
- data/app/services/action_agent/sandbox_browser_driver.rb +85 -0
- data/app/services/action_agent/sandbox_credential_paths.rb +26 -0
- data/app/services/action_agent/sandbox_mail.rb +103 -0
- data/app/services/action_agent/sandbox_manifest.rb +166 -13
- data/app/services/action_agent/sandbox_orchestrator.rb +276 -22
- data/app/services/action_agent/sandbox_patch.rb +253 -0
- data/app/services/action_agent/scenario_catalog_import.rb +255 -0
- data/app/services/action_agent/scenario_catalog_repository_import.rb +81 -0
- data/app/services/action_agent/scenario_evaluation_runner.rb +119 -16
- data/app/services/action_agent/secret_scrubber.rb +33 -4
- data/app/services/action_agent/session_index.rb +270 -0
- data/app/services/action_agent/session_recording_service.rb +11 -0
- data/app/services/action_agent/session_timeline.rb +505 -0
- data/app/views/action_agent/github_app_manifests/created.html.erb +55 -0
- data/app/views/action_agent/session_player/show.html.erb +19 -0
- data/config/routes.rb +155 -1
- data/lib/action_agent/claude_login_process.rb +107 -0
- data/lib/action_agent/engine.rb +55 -5
- data/lib/action_agent/secret_requests.rb +80 -0
- data/lib/action_agent/version.rb +1 -1
- data/lib/action_agent.rb +507 -22
- data/lib/generators/action_agent/install_generator.rb +37 -0
- data/lib/generators/action_agent/templates/action_agent.rb.erb +158 -0
- data/lib/generators/action_agent/templates/migrations/015_add_provider_key_scope.rb.erb +72 -0
- data/lib/generators/action_agent/templates/migrations/016_create_active_agent_input_requests.rb.erb +85 -0
- data/lib/generators/action_agent/templates/migrations/017_create_active_agent_recording_events.rb.erb +57 -0
- data/lib/generators/action_agent/templates/migrations/018_add_browser_to_active_agent_sandbox_sessions.rb.erb +25 -0
- data/lib/generators/action_agent/templates/migrations/019_create_active_agent_projects.rb.erb +79 -0
- data/lib/generators/action_agent/templates/migrations/020_add_github_app_installations.rb.erb +62 -0
- data/lib/generators/action_agent/templates/migrations/021_create_active_agent_explorations.rb.erb +63 -0
- data/lib/generators/action_agent/templates/migrations/023_create_active_agent_draft_pull_requests.rb.erb +75 -0
- data/lib/generators/action_agent/templates/migrations/024_add_kind_to_active_agent_project_secrets.rb.erb +14 -0
- data/lib/generators/action_agent/templates/migrations/025_create_active_agent_scenario_catalogs.rb.erb +110 -0
- data/lib/generators/action_agent/templates/migrations/026_add_evaluation_fixes_to_code_sessions.rb.erb +21 -0
- data/lib/generators/action_agent/templates/migrations/027_create_active_agent_interaction_feedbacks.rb.erb +17 -0
- data/lib/tasks/action_agent.rake +33 -0
- data/lib/tasks/browser.rake +22 -0
- data/lib/tasks/sandbox.rake +1 -1
- data/lib/tasks/scenario_catalogs.rake +71 -0
- metadata +100 -2
|
@@ -14,8 +14,14 @@ module ActionAgent
|
|
|
14
14
|
# So an app with users scopes agents per user, an app with only accounts
|
|
15
15
|
# scopes them per account, and a single-user self-hosted install declares
|
|
16
16
|
# neither association and owns everything implicitly. That ordering is
|
|
17
|
-
# per-model on purpose: an app can
|
|
18
|
-
#
|
|
17
|
+
# per-model on purpose: an app can keep agents per user while keeping API
|
|
18
|
+
# keys per account. A host that configures both classes and wants the
|
|
19
|
+
# account to own everything re-declares `owned_by :account, :user` on the
|
|
20
|
+
# user-first models from `to_prepare`, as the hosted platform does.
|
|
21
|
+
#
|
|
22
|
+
# An owner is matched by its class, never by its id alone (see .for_owner):
|
|
23
|
+
# the two owner tables share an id space, so `where(account_id: user.id)`
|
|
24
|
+
# reads whichever account happens to share that user's id.
|
|
19
25
|
#
|
|
20
26
|
# Both foreign keys exist in the engine's schema either way, so moving a
|
|
21
27
|
# deployment between shapes is a configuration change, not a migration.
|
|
@@ -30,7 +36,7 @@ module ActionAgent
|
|
|
30
36
|
@owner_candidates = candidates.map(&:to_sym)
|
|
31
37
|
|
|
32
38
|
@owner_candidates.each do |candidate|
|
|
33
|
-
class_name =
|
|
39
|
+
class_name = owner_class_name(candidate)
|
|
34
40
|
next if class_name.blank?
|
|
35
41
|
|
|
36
42
|
belongs_to candidate, class_name: class_name, optional: true
|
|
@@ -44,7 +50,20 @@ module ActionAgent
|
|
|
44
50
|
# The association this install owns the model through, or nil when the
|
|
45
51
|
# host app configured no owner model at all.
|
|
46
52
|
def owner_association
|
|
47
|
-
owner_candidates.find { |
|
|
53
|
+
owner_candidates.find { |candidate| owner_class_name(candidate).present? }
|
|
54
|
+
end
|
|
55
|
+
|
|
56
|
+
# The class the host configured for +association+, or nil when it is
|
|
57
|
+
# unset or names nothing loaded.
|
|
58
|
+
def owner_class_for(association = owner_association)
|
|
59
|
+
return nil if association.nil?
|
|
60
|
+
|
|
61
|
+
owner_class_name(association)&.safe_constantize
|
|
62
|
+
end
|
|
63
|
+
|
|
64
|
+
# The class name the host configured for +association+, or nil.
|
|
65
|
+
def owner_class_name(association)
|
|
66
|
+
ActionAgent.public_send(CLASS_FOR.fetch(association)).presence
|
|
48
67
|
end
|
|
49
68
|
|
|
50
69
|
# Scopes to records owned by +owner+.
|
|
@@ -58,15 +77,50 @@ module ActionAgent
|
|
|
58
77
|
# request, or a resolver that returned nil. Returning `all` here
|
|
59
78
|
# would hand one tenant every other tenant's records, so it
|
|
60
79
|
# returns nothing instead.
|
|
80
|
+
#
|
|
81
|
+
# An owner of another class than the configured one also scopes to
|
|
82
|
+
# nothing. A user handed to an account-owned model is first mapped to
|
|
83
|
+
# its tenant through `ActionAgent.tenant_for`, so a host that resolves
|
|
84
|
+
# tenants reads that tenant's rows; without a tenant resolver the user
|
|
85
|
+
# is nobody's account and reads nothing.
|
|
86
|
+
#
|
|
87
|
+
# Only the rows .owned_rows admits are returned.
|
|
61
88
|
def for_owner(owner)
|
|
62
|
-
return
|
|
89
|
+
return owned_rows if owner_association.nil?
|
|
90
|
+
|
|
91
|
+
owner = resolve_owner(owner)
|
|
63
92
|
return none if owner.nil?
|
|
64
93
|
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
94
|
+
owned_rows.where("#{owner_association}_id": owner.id)
|
|
95
|
+
end
|
|
96
|
+
|
|
97
|
+
# The rows an owner's scope reads: every row, unless the model holds
|
|
98
|
+
# rows its owner does not share, which it leaves out here. .for_owner
|
|
99
|
+
# and the dashboard API's `owned` both apply it.
|
|
100
|
+
def owned_rows
|
|
101
|
+
all
|
|
102
|
+
end
|
|
103
|
+
|
|
104
|
+
# +owner+ as an instance of the configured owner class: itself when it
|
|
105
|
+
# is one, its tenant when it is a user and this model is owned by
|
|
106
|
+
# account, nil otherwise.
|
|
107
|
+
def resolve_owner(owner)
|
|
108
|
+
return nil if owner.nil?
|
|
109
|
+
|
|
110
|
+
owner_class = owner_class_for
|
|
111
|
+
return nil if owner_class.nil?
|
|
112
|
+
return owner if owner.is_a?(owner_class)
|
|
113
|
+
return nil unless owner_association == :account && user?(owner)
|
|
114
|
+
|
|
115
|
+
tenant = ActionAgent.tenant_for(owner)
|
|
116
|
+
tenant if tenant.is_a?(owner_class)
|
|
117
|
+
end
|
|
118
|
+
|
|
119
|
+
private
|
|
120
|
+
|
|
121
|
+
def user?(record)
|
|
122
|
+
user_class = owner_class_for(:user)
|
|
123
|
+
user_class.present? && record.is_a?(user_class)
|
|
70
124
|
end
|
|
71
125
|
end
|
|
72
126
|
|
|
@@ -88,12 +142,17 @@ module ActionAgent
|
|
|
88
142
|
end
|
|
89
143
|
|
|
90
144
|
# Assigns +owner+ to whichever association this install uses. A no-op
|
|
91
|
-
# when the host app configured no owner model.
|
|
92
|
-
#
|
|
93
|
-
#
|
|
145
|
+
# when the host app configured no owner model. The record is resolved as
|
|
146
|
+
# .for_owner resolves it, so a user handed to an account-owned model
|
|
147
|
+
# assigns the user's tenant, and an owner that resolves to nothing raises
|
|
148
|
+
# ArgumentError rather than writing its id into another class's column.
|
|
149
|
+
# Writes the foreign key directly when the association is missing or was
|
|
150
|
+
# declared for another class, as #owner reads it.
|
|
94
151
|
def owner=(record)
|
|
95
152
|
association = self.class.owner_association
|
|
96
153
|
return unless association
|
|
154
|
+
|
|
155
|
+
record = assignable_owner(record, association)
|
|
97
156
|
return public_send(:"#{association}=", record) if owner_association_current?(association)
|
|
98
157
|
|
|
99
158
|
self[:"#{association}_id"] = record&.id
|
|
@@ -101,6 +160,18 @@ module ActionAgent
|
|
|
101
160
|
|
|
102
161
|
private
|
|
103
162
|
|
|
163
|
+
def assignable_owner(record, association)
|
|
164
|
+
return nil if record.nil?
|
|
165
|
+
|
|
166
|
+
resolved = self.class.resolve_owner(record)
|
|
167
|
+
return resolved if resolved
|
|
168
|
+
|
|
169
|
+
raise ArgumentError,
|
|
170
|
+
"#{self.class.name} is owned by #{association} (#{self.class.owner_class_name(association)}), " \
|
|
171
|
+
"so a #{record.class.name} cannot own it. Assign the #{association} itself, or configure " \
|
|
172
|
+
"ActionAgent.tenant_resolver to map a user to its account."
|
|
173
|
+
end
|
|
174
|
+
|
|
104
175
|
# Whether +association+ was declared for the class the configuration
|
|
105
176
|
# names now.
|
|
106
177
|
def owner_association_current?(association)
|
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module ActionAgent
|
|
4
|
+
# The JSON shape of an InputRequest, shared by the JSON API and the MCP
|
|
5
|
+
# facade. It never includes the request's answer or checkpoint.
|
|
6
|
+
#
|
|
7
|
+
# `actor` is the person the paused run executes on behalf of, and
|
|
8
|
+
# `arguments` the paused call's arguments, for a `confirm` request only.
|
|
9
|
+
module InputRequestSerializer
|
|
10
|
+
module_function
|
|
11
|
+
|
|
12
|
+
# @param request [InputRequest]
|
|
13
|
+
# @return [Hash]
|
|
14
|
+
def call(request)
|
|
15
|
+
run = request.subject
|
|
16
|
+
agent = run.try(:agent)
|
|
17
|
+
|
|
18
|
+
{
|
|
19
|
+
id: request.id,
|
|
20
|
+
kind: request.kind,
|
|
21
|
+
status: request.status,
|
|
22
|
+
prompt: request.prompt,
|
|
23
|
+
options: request.options,
|
|
24
|
+
answer_schema: request.answer_schema,
|
|
25
|
+
tool_name: request.tool_name,
|
|
26
|
+
tool_call_id: request.tool_call_id,
|
|
27
|
+
arguments: request.kind == "confirm" ? request.arguments : nil,
|
|
28
|
+
agent: agent && { id: agent.id, name: agent.name, slug: agent.slug },
|
|
29
|
+
run_id: run.is_a?(AgentRun) ? run.id : nil,
|
|
30
|
+
actor: actor_json(run.try(:actor)),
|
|
31
|
+
created_at: request.created_at,
|
|
32
|
+
expires_at: request.expires_at,
|
|
33
|
+
answered_at: request.answered_at
|
|
34
|
+
}
|
|
35
|
+
end
|
|
36
|
+
|
|
37
|
+
def actor_json(actor)
|
|
38
|
+
return nil if actor.nil?
|
|
39
|
+
|
|
40
|
+
{
|
|
41
|
+
type: actor.class.name,
|
|
42
|
+
id: actor.try(:id),
|
|
43
|
+
name: actor.try(:name).presence || actor.try(:email).presence
|
|
44
|
+
}
|
|
45
|
+
end
|
|
46
|
+
end
|
|
47
|
+
end
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module ActionAgent
|
|
4
|
+
# JSON for the catalogs API: a catalog's summary for the index, and the
|
|
5
|
+
# whole tree (products, sets, scenarios) for one catalog.
|
|
6
|
+
module ScenarioCatalogSerializer
|
|
7
|
+
module_function
|
|
8
|
+
|
|
9
|
+
def summary(catalog)
|
|
10
|
+
catalog.summary
|
|
11
|
+
end
|
|
12
|
+
|
|
13
|
+
def full(catalog)
|
|
14
|
+
products = catalog.products.includes(:agent, :project, sets: [ :scenarios, { evaluation: :evaluation_runs } ])
|
|
15
|
+
catalog.summary.merge(
|
|
16
|
+
metadata: catalog.metadata,
|
|
17
|
+
products: products.map do |product|
|
|
18
|
+
product.summary.merge(sets: product.sets.map { |set| set.summary.merge(scenarios: set.scenarios.map(&:summary)) })
|
|
19
|
+
end
|
|
20
|
+
)
|
|
21
|
+
end
|
|
22
|
+
end
|
|
23
|
+
end
|
|
@@ -0,0 +1,146 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module ActionAgent
|
|
4
|
+
# Serializes one page of SessionIndex rows for the Sessions list. What each
|
|
5
|
+
# row needs from other tables is read once for the page, never per row.
|
|
6
|
+
#
|
|
7
|
+
# Every row has `kind` and `id` (which address its timeline), `source`,
|
|
8
|
+
# `title`, `preview`, `agent`, `outcome` ("failed", "passed" or nil),
|
|
9
|
+
# `status`, `started_at` and `last_activity_at`, plus what its kind adds.
|
|
10
|
+
class SessionSummarySerializer
|
|
11
|
+
OUTCOMES = { "passed" => "passed", "failed" => "failed", "errored" => "failed" }.freeze
|
|
12
|
+
|
|
13
|
+
# +rows+ are `{ kind:, record:, time: }` hashes in display order.
|
|
14
|
+
# +agents+ and +recordings+ are what the caller may see.
|
|
15
|
+
# +failed_context_ids+ are the conversations among them with a failed run.
|
|
16
|
+
def initialize(rows, agents:, recordings:, failed_context_ids: [])
|
|
17
|
+
@rows = rows
|
|
18
|
+
@agents = agents
|
|
19
|
+
@recordings = recordings
|
|
20
|
+
@failed_context_ids = failed_context_ids.to_set
|
|
21
|
+
end
|
|
22
|
+
|
|
23
|
+
# @return [Array<Hash>]
|
|
24
|
+
def as_json(*)
|
|
25
|
+
preload
|
|
26
|
+
@rows.map do |row|
|
|
27
|
+
case row[:kind]
|
|
28
|
+
when "context" then context_row(row[:record], row[:time])
|
|
29
|
+
when "scenario_result" then scenario_result_row(row[:record], row[:time])
|
|
30
|
+
else recording_row(row[:record], row[:time])
|
|
31
|
+
end
|
|
32
|
+
end
|
|
33
|
+
end
|
|
34
|
+
|
|
35
|
+
private
|
|
36
|
+
|
|
37
|
+
def records(kind)
|
|
38
|
+
@rows.select { |row| row[:kind] == kind }.map { |row| row[:record] }
|
|
39
|
+
end
|
|
40
|
+
|
|
41
|
+
def preload
|
|
42
|
+
contexts = records("context")
|
|
43
|
+
preloader(contexts, :contextable)
|
|
44
|
+
preloader(records("scenario_result"), [ :scenario, :agent_run, { evaluation_run: { evaluation: :agent } } ])
|
|
45
|
+
preloader(records("recording"), agent_run: :agent)
|
|
46
|
+
|
|
47
|
+
ids = contexts.map(&:id)
|
|
48
|
+
@message_counts = ids.empty? ? {} : AgentMessage.where(agent_context_id: ids).group(:agent_context_id).count
|
|
49
|
+
@recording_counts = recording_counts(ids)
|
|
50
|
+
@first_inputs = first_user_messages(ids)
|
|
51
|
+
end
|
|
52
|
+
|
|
53
|
+
def preloader(records, associations)
|
|
54
|
+
ActiveRecord::Associations::Preloader.new(records: records, associations: associations).call if records.any?
|
|
55
|
+
end
|
|
56
|
+
|
|
57
|
+
# How many recordings each conversation replays, found as
|
|
58
|
+
# SessionTimeline.for_context finds them: linked to the conversation, or
|
|
59
|
+
# made by a run that wrote a generation to it.
|
|
60
|
+
def recording_counts(ids)
|
|
61
|
+
return {} if ids.empty?
|
|
62
|
+
|
|
63
|
+
traces = AgentGeneration.where(agent_context_id: ids).where.not(trace_id: [ nil, "" ])
|
|
64
|
+
.distinct.pluck(:trace_id, :agent_context_id)
|
|
65
|
+
contexts_by_trace = traces.group_by(&:first).transform_values { |pairs| pairs.map(&:last) }
|
|
66
|
+
contexts_by_run = AgentRun.where(agent: @agents, trace_id: contexts_by_trace.keys).pluck(:id, :trace_id)
|
|
67
|
+
.to_h { |run_id, trace_id| [ run_id, contexts_by_trace.fetch(trace_id) ] }
|
|
68
|
+
|
|
69
|
+
found = @recordings.where(agent_context_id: ids).or(@recordings.where(agent_run_id: contexts_by_run.keys))
|
|
70
|
+
.pluck(:id, :agent_context_id, :agent_run_id)
|
|
71
|
+
recordings = Hash.new { |sets, context_id| sets[context_id] = Set.new }
|
|
72
|
+
found.each do |id, context_id, run_id|
|
|
73
|
+
([ context_id, *contexts_by_run.fetch(run_id, []) ] & ids).each { |owner| recordings[owner] << id }
|
|
74
|
+
end
|
|
75
|
+
recordings.transform_values(&:size)
|
|
76
|
+
end
|
|
77
|
+
|
|
78
|
+
# What each conversation was first asked. Messages are only appended, so
|
|
79
|
+
# the smallest id is the first.
|
|
80
|
+
def first_user_messages(ids)
|
|
81
|
+
return {} if ids.empty?
|
|
82
|
+
|
|
83
|
+
first_ids = AgentMessage.where(agent_context_id: ids, role: "user").where.not(content: [ nil, "" ])
|
|
84
|
+
.group(:agent_context_id).minimum(:id)
|
|
85
|
+
AgentMessage.where(id: first_ids.values).pluck(:agent_context_id, :content).to_h
|
|
86
|
+
end
|
|
87
|
+
|
|
88
|
+
def context_row(context, time)
|
|
89
|
+
agent = context.contextable.is_a?(Agent) ? context.contextable : nil
|
|
90
|
+
common("context", context, time).merge(
|
|
91
|
+
title: "#{context.agent_name}##{context.action_name}",
|
|
92
|
+
preview: InteractionPreview.line(@first_inputs[context.id]),
|
|
93
|
+
agent: agent_json(agent),
|
|
94
|
+
outcome: @failed_context_ids.include?(context.id) ? "failed" : nil,
|
|
95
|
+
status: nil,
|
|
96
|
+
message_count: @message_counts[context.id] || 0,
|
|
97
|
+
recording_count: @recording_counts[context.id] || 0,
|
|
98
|
+
started_at: timestamp(context.created_at)
|
|
99
|
+
)
|
|
100
|
+
end
|
|
101
|
+
|
|
102
|
+
def scenario_result_row(result, time)
|
|
103
|
+
scenario = result.evaluated_scenario
|
|
104
|
+
evaluation = result.evaluation_run.evaluation
|
|
105
|
+
common("scenario_result", result, time).merge(
|
|
106
|
+
title: scenario["key"].presence || "Scenario #{result.evaluation_scenario_id}",
|
|
107
|
+
preview: InteractionPreview.line(scenario["prompt"]),
|
|
108
|
+
agent: agent_json(evaluation.agent),
|
|
109
|
+
outcome: OUTCOMES[result.status],
|
|
110
|
+
status: result.status,
|
|
111
|
+
model: result.model,
|
|
112
|
+
evaluation: { id: evaluation.id, name: evaluation.name, run_id: result.evaluation_run_id },
|
|
113
|
+
agent_run_id: result.agent_run_id,
|
|
114
|
+
started_at: timestamp(result.agent_run&.started_at || result.created_at)
|
|
115
|
+
)
|
|
116
|
+
end
|
|
117
|
+
|
|
118
|
+
def recording_row(recording, time)
|
|
119
|
+
run = recording.agent_run
|
|
120
|
+
common("recording", recording, time).merge(
|
|
121
|
+
title: recording.name.presence || "Recording #{recording.id}",
|
|
122
|
+
preview: nil,
|
|
123
|
+
agent: agent_json(run&.agent),
|
|
124
|
+
outcome: recording.failed? || run&.failed? ? "failed" : nil,
|
|
125
|
+
status: recording.status,
|
|
126
|
+
recording_source: recording.source,
|
|
127
|
+
event_count: recording.event_count,
|
|
128
|
+
action_count: recording.action_count,
|
|
129
|
+
agent_run_id: recording.agent_run_id,
|
|
130
|
+
started_at: timestamp(recording.created_at)
|
|
131
|
+
)
|
|
132
|
+
end
|
|
133
|
+
|
|
134
|
+
def common(kind, record, time)
|
|
135
|
+
{ kind: kind, id: record.id, source: SessionIndex::KINDS.fetch(kind), last_activity_at: timestamp(time) }
|
|
136
|
+
end
|
|
137
|
+
|
|
138
|
+
def agent_json(agent)
|
|
139
|
+
agent && { id: agent.id, name: agent.name, slug: agent.slug }
|
|
140
|
+
end
|
|
141
|
+
|
|
142
|
+
def timestamp(time)
|
|
143
|
+
time&.utc&.iso8601(3)
|
|
144
|
+
end
|
|
145
|
+
end
|
|
146
|
+
end
|