actionagent 1.2.2 → 1.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +14 -3
- data/app/assets/builds/action_agent.css +1 -1
- data/app/assets/builds/action_agent.js +69 -43
- data/app/controllers/action_agent/api/agent_runs_controller.rb +28 -8
- data/app/controllers/action_agent/api/agents_controller.rb +191 -56
- data/app/controllers/action_agent/api/analytics_controller.rb +31 -9
- data/app/controllers/action_agent/api/base_controller.rb +16 -0
- data/app/controllers/action_agent/api/dashboard_assistant_controller.rb +83 -0
- data/app/controllers/action_agent/api/evaluations_controller.rb +252 -7
- data/app/controllers/action_agent/api/interaction_messages_controller.rb +98 -0
- data/app/controllers/action_agent/api/mcp_controller.rb +13 -3
- data/app/controllers/action_agent/api/mcp_servers_controller.rb +28 -8
- data/app/controllers/action_agent/api/metrics_controller.rb +44 -11
- data/app/controllers/action_agent/api/provider_models_controller.rb +1 -1
- data/app/controllers/action_agent/api/sandboxes_controller.rb +6 -0
- data/app/controllers/action_agent/api/session_recordings_controller.rb +34 -12
- data/app/controllers/action_agent/api/templates_controller.rb +25 -21
- data/app/controllers/action_agent/api/traces_controller.rb +25 -5
- data/app/controllers/action_agent/api/usage_controller.rb +20 -0
- data/app/controllers/action_agent/application_controller.rb +25 -2
- data/app/controllers/action_agent/dashboard_controller.rb +3 -1
- data/app/controllers/concerns/action_agent/api/agent_serialization.rb +53 -0
- data/app/jobs/action_agent/agent_execution_job.rb +40 -20
- data/app/jobs/action_agent/application_job.rb +7 -3
- data/app/jobs/action_agent/evaluation_run_job.rb +18 -0
- data/app/jobs/action_agent/sandbox_cleanup_job.rb +13 -10
- data/app/models/action_agent/agent.rb +74 -23
- data/app/models/action_agent/agent_run.rb +99 -0
- data/app/models/action_agent/agent_template.rb +22 -7
- data/app/models/action_agent/evaluation.rb +64 -4
- data/app/models/action_agent/evaluation_run.rb +190 -2
- data/app/models/action_agent/evaluation_scenario.rb +59 -0
- data/app/models/action_agent/evaluation_scenario_result.rb +86 -0
- data/app/models/action_agent/recording_action.rb +11 -7
- data/app/models/action_agent/sandbox_session.rb +1 -1
- data/app/models/action_agent/session_recording.rb +31 -8
- data/app/models/action_agent/telemetry_trace.rb +126 -3
- data/app/models/concerns/action_agent/adapter_aware.rb +19 -0
- data/app/models/concerns/action_agent/ownable.rb +15 -2
- data/app/queries/action_agent/metrics_report.rb +498 -0
- data/app/serializers/action_agent/agent_message_serializer.rb +1 -0
- data/app/services/action_agent/agent_execution_service.rb +294 -16
- data/app/services/action_agent/agent_registrar.rb +7 -6
- data/app/services/action_agent/agent_toolbox.rb +49 -7
- data/app/services/action_agent/dashboard_assistant_service.rb +342 -0
- data/app/services/action_agent/evaluation_evidence.rb +234 -0
- data/app/services/action_agent/evaluation_runner_service.rb +13 -3
- data/app/services/action_agent/evaluation_tool_resolver.rb +162 -0
- data/app/services/action_agent/mcp_catalog.rb +46 -8
- data/app/services/action_agent/mcp_client.rb +167 -0
- data/app/services/action_agent/mcp_recording_middleware.rb +2 -2
- data/app/services/action_agent/mcp_tool_dispatcher.rb +116 -0
- data/app/services/action_agent/playwright_mcp_client.rb +11 -126
- data/app/services/action_agent/sandbox_orchestrator.rb +12 -1
- data/app/services/action_agent/scenario_evaluation_runner.rb +260 -0
- data/app/services/action_agent/tool_discovery.rb +22 -8
- data/config/routes.rb +36 -3
- data/lib/action_agent/assistant_request_filter.rb +22 -0
- data/lib/action_agent/engine.rb +106 -19
- data/lib/action_agent/version.rb +1 -1
- data/lib/action_agent.rb +104 -6
- data/lib/generators/action_agent/install_generator.rb +20 -7
- data/lib/generators/action_agent/templates/action_agent.rb.erb +12 -0
- data/lib/generators/action_agent/templates/create_active_agent_evaluation_scenarios.rb.erb +79 -0
- data/lib/tasks/action_agent.rake +9 -0
- metadata +22 -5
|
@@ -51,6 +51,12 @@ module ActionAgent
|
|
|
51
51
|
sandbox.update!(status: :running)
|
|
52
52
|
comparison_id = SecureRandom.uuid
|
|
53
53
|
|
|
54
|
+
# One execution per provider. Recorded before the jobs are enqueued,
|
|
55
|
+
# mirroring #run's record-before-enqueue order, so usage is counted
|
|
56
|
+
# even if a later enqueue raises. Without this the quota gate on
|
|
57
|
+
# compare was checked but never advanced: comparisons were free.
|
|
58
|
+
providers.size.times { record_execution_usage }
|
|
59
|
+
|
|
54
60
|
# Spawn a separate generation job for each provider (all in same sandbox)
|
|
55
61
|
runs = providers.map do |provider|
|
|
56
62
|
run_id = SecureRandom.uuid
|
|
@@ -16,10 +16,10 @@ module ActionAgent
|
|
|
16
16
|
# GET /api/session_recordings
|
|
17
17
|
# List recordings with optional filters
|
|
18
18
|
def index
|
|
19
|
-
# Recordings the caller
|
|
20
|
-
# real column rather than a JSON metadata key, so this works on
|
|
19
|
+
# Recordings the caller can reach, plus the shared demo. Ownership is
|
|
20
|
+
# a real column rather than a JSON metadata key, so this works on
|
|
21
21
|
# every adapter.
|
|
22
|
-
recordings =
|
|
22
|
+
recordings = reachable_recordings.or(SessionRecording.where(name: "lander_demo")).recent
|
|
23
23
|
|
|
24
24
|
# Filter by status
|
|
25
25
|
recordings = recordings.where(status: params[:status]) if params[:status].present?
|
|
@@ -57,7 +57,7 @@ module ActionAgent
|
|
|
57
57
|
# GET /api/session_recordings/recent
|
|
58
58
|
# Get recent recordings for the current user
|
|
59
59
|
def recent
|
|
60
|
-
recordings =
|
|
60
|
+
recordings = reachable_recordings.recent.limit(10)
|
|
61
61
|
|
|
62
62
|
render json: {
|
|
63
63
|
recordings: recordings.map { |r| recording_summary(r) }
|
|
@@ -132,14 +132,17 @@ module ActionAgent
|
|
|
132
132
|
recording = SessionRecording.start_user_session!(
|
|
133
133
|
visitor_id: params[:visitor_id] || generate_visitor_id,
|
|
134
134
|
parent_demo_id: params[:parent_demo_id],
|
|
135
|
-
page_url: params[:page_url]
|
|
135
|
+
page_url: params[:page_url],
|
|
136
|
+
owner: current_owner
|
|
136
137
|
)
|
|
137
138
|
|
|
138
|
-
# Set user agent from request
|
|
139
|
+
# Set user agent from request. String keys: the stored metadata is
|
|
140
|
+
# string-keyed, and merging symbols wrote a second "user_agent" that
|
|
141
|
+
# json 3.0 refuses to encode.
|
|
139
142
|
recording.update!(
|
|
140
143
|
metadata: recording.metadata.merge(
|
|
141
|
-
user_agent
|
|
142
|
-
ip_hash
|
|
144
|
+
"user_agent" => request.user_agent,
|
|
145
|
+
"ip_hash" => Digest::SHA256.hexdigest(request.remote_ip.to_s)[0..16]
|
|
143
146
|
)
|
|
144
147
|
)
|
|
145
148
|
|
|
@@ -226,8 +229,10 @@ module ActionAgent
|
|
|
226
229
|
return
|
|
227
230
|
end
|
|
228
231
|
|
|
229
|
-
# Create a new recording for the user's continuation
|
|
230
|
-
|
|
232
|
+
# Create a new recording for the user's continuation. It records the
|
|
233
|
+
# caller's own session, so it is owned by the caller — through the
|
|
234
|
+
# owner column the list reads, not a metadata key it never consults.
|
|
235
|
+
continuation = SessionRecording.new(
|
|
231
236
|
sandbox_session: @recording.sandbox_session,
|
|
232
237
|
agent_run: @recording.agent_run,
|
|
233
238
|
name: "#{@recording.name}_continuation",
|
|
@@ -239,6 +244,8 @@ module ActionAgent
|
|
|
239
244
|
user_id: current_user&.id
|
|
240
245
|
}
|
|
241
246
|
)
|
|
247
|
+
continuation.owner = current_owner || @recording.owner
|
|
248
|
+
continuation.save!
|
|
242
249
|
|
|
243
250
|
render json: {
|
|
244
251
|
handoff_state: handoff_state,
|
|
@@ -275,6 +282,19 @@ module ActionAgent
|
|
|
275
282
|
not_found
|
|
276
283
|
end
|
|
277
284
|
|
|
285
|
+
# What the list shows: recordings the caller owns, plus recordings made
|
|
286
|
+
# inside a sandbox the caller owns — the same reachability
|
|
287
|
+
# can_manage_recording? grants to a direct read, so a recording never
|
|
288
|
+
# opens by id while missing from the list. The second clause is what
|
|
289
|
+
# keeps recordings created before the owner column was written
|
|
290
|
+
# reachable.
|
|
291
|
+
def reachable_recordings
|
|
292
|
+
scope = owned(SessionRecording)
|
|
293
|
+
return scope if SessionRecording.owner_association.nil?
|
|
294
|
+
|
|
295
|
+
scope.or(SessionRecording.where(sandbox_session_id: owned(SandboxSession).select(:id)))
|
|
296
|
+
end
|
|
297
|
+
|
|
278
298
|
def can_manage_recording?(recording)
|
|
279
299
|
# An install with no owner model owns everything it can see.
|
|
280
300
|
return true if SessionRecording.owner_association.nil?
|
|
@@ -346,8 +366,10 @@ module ActionAgent
|
|
|
346
366
|
sequence: action.sequence,
|
|
347
367
|
timestamp_ms: action.timestamp_ms,
|
|
348
368
|
selector: action.selector,
|
|
349
|
-
|
|
350
|
-
|
|
369
|
+
# Redacted like /actions: the export used to ship the cleartext
|
|
370
|
+
# the action list had masked.
|
|
371
|
+
value: action.redacted_value,
|
|
372
|
+
metadata: action.safe_metadata
|
|
351
373
|
}
|
|
352
374
|
end
|
|
353
375
|
}
|
|
@@ -3,13 +3,24 @@
|
|
|
3
3
|
module ActionAgent
|
|
4
4
|
module Api
|
|
5
5
|
class TemplatesController < BaseController
|
|
6
|
+
include AgentSerialization
|
|
7
|
+
|
|
6
8
|
# No anonymous exemption: #show looks a template up by bare id, so the
|
|
7
9
|
# exemption served unpublished drafts and private prompt libraries to
|
|
8
10
|
# anyone walking the id space. #index was already limited to public
|
|
9
11
|
# templates; #show was not.
|
|
10
12
|
|
|
13
|
+
before_action :require_owner!, only: [ :use ]
|
|
14
|
+
|
|
11
15
|
# GET /api/templates
|
|
12
16
|
def index
|
|
17
|
+
# The library ships with the engine but nothing ever seeded it, so a
|
|
18
|
+
# fresh install showed "No templates found" behind every Browse
|
|
19
|
+
# Templates button. seed_defaults! is idempotent (find_or_create_by!
|
|
20
|
+
# on slug), so an empty table is seeded on first read; the
|
|
21
|
+
# action_agent:seed_templates task does the same on demand.
|
|
22
|
+
AgentTemplate.seed_defaults! if AgentTemplate.none?
|
|
23
|
+
|
|
13
24
|
@templates = AgentTemplate.public_templates.order(usage_count: :desc)
|
|
14
25
|
|
|
15
26
|
# Filter by category
|
|
@@ -31,16 +42,23 @@ module ActionAgent
|
|
|
31
42
|
end
|
|
32
43
|
|
|
33
44
|
# POST /api/templates/:id/use
|
|
45
|
+
#
|
|
46
|
+
# Built through the engine's ownership layer (owner_agents, as
|
|
47
|
+
# AgentsController#create does) rather than the host user's `agents`
|
|
48
|
+
# association: a single-user install has no user, and the old
|
|
49
|
+
# `current_user.agents.build` raised NoMethodError on nil for every
|
|
50
|
+
# click of "Use This Template".
|
|
34
51
|
def use
|
|
35
52
|
@template = AgentTemplate.find(params[:id])
|
|
53
|
+
agent = @template.build_agent_in(owner_agents, name: params[:name])
|
|
36
54
|
|
|
37
|
-
agent
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
render json: { agent: agent_json(agent) }, status: :created
|
|
55
|
+
if agent.save
|
|
56
|
+
@template.increment!(:usage_count)
|
|
57
|
+
# The detail shape, not a summary: the dashboard opens the new agent
|
|
58
|
+
# in the editor straight from this response, and an editor seeded
|
|
59
|
+
# from a summary saved empty instructions/tools/model_config over
|
|
60
|
+
# the template's real ones.
|
|
61
|
+
render json: { agent: agent_json(agent, include_details: true) }, status: :created
|
|
44
62
|
else
|
|
45
63
|
render json: { errors: agent.errors.full_messages }, status: :unprocessable_entity
|
|
46
64
|
end
|
|
@@ -75,20 +93,6 @@ module ActionAgent
|
|
|
75
93
|
|
|
76
94
|
json
|
|
77
95
|
end
|
|
78
|
-
|
|
79
|
-
def agent_json(agent)
|
|
80
|
-
{
|
|
81
|
-
id: agent.id,
|
|
82
|
-
name: agent.name,
|
|
83
|
-
slug: agent.slug,
|
|
84
|
-
description: agent.description,
|
|
85
|
-
provider: agent.provider,
|
|
86
|
-
model: agent.model,
|
|
87
|
-
status: agent.status,
|
|
88
|
-
preset_type: agent.preset_type,
|
|
89
|
-
appearance: agent.appearance
|
|
90
|
-
}
|
|
91
|
-
end
|
|
92
96
|
end
|
|
93
97
|
end
|
|
94
98
|
end
|
|
@@ -36,14 +36,18 @@ module ActionAgent
|
|
|
36
36
|
class TracesController < ActionController::API
|
|
37
37
|
before_action :authenticate_api_key!, if: -> { ActionAgent.multi_tenant? }
|
|
38
38
|
before_action :authenticate_ingest_key!, unless: -> { ActionAgent.multi_tenant? }
|
|
39
|
-
|
|
40
|
-
# Maximum traces accepted per request (mirrors
|
|
41
|
-
# ProcessTelemetryTracesJob::MAX_TRACES_PER_JOB).
|
|
42
|
-
MAX_TRACES_PER_REQUEST = 100
|
|
39
|
+
before_action :enforce_ingest_quota!
|
|
43
40
|
|
|
44
41
|
# POST <mount>/api/traces (e.g. /activeagents/api/traces)
|
|
42
|
+
#
|
|
43
|
+
# Every trace in the request is accepted. The reporter flushes its
|
|
44
|
+
# whole buffer once it reaches batch_size (which is configurable), so
|
|
45
|
+
# a single POST legitimately carries more than a hundred traces; this
|
|
46
|
+
# used to keep the first hundred and answer 202 for the rest, which
|
|
47
|
+
# were silently gone. ProcessTelemetryTracesJob bounds its own work by
|
|
48
|
+
# slicing and re-enqueueing the remainder.
|
|
45
49
|
def create
|
|
46
|
-
traces = Array(params[:traces])
|
|
50
|
+
traces = Array(params[:traces])
|
|
47
51
|
sdk_info = params[:sdk] || {}
|
|
48
52
|
|
|
49
53
|
return head :accepted if traces.empty?
|
|
@@ -106,6 +110,22 @@ module ActionAgent
|
|
|
106
110
|
render json: { error: "Invalid API key" }, status: :unauthorized
|
|
107
111
|
end
|
|
108
112
|
|
|
113
|
+
# The host app's quota checker, asked with kind :trace_ingest — the
|
|
114
|
+
# counterpart to Api::BaseController#enforce_quota!, which asks with
|
|
115
|
+
# :execution. Denials are 429 here rather than 402: a reporter that is
|
|
116
|
+
# over its ingest allowance should back off, not upgrade mid-flush.
|
|
117
|
+
# Same body shape, so a checker's message or Hash payload reads the
|
|
118
|
+
# same on both.
|
|
119
|
+
def enforce_ingest_quota!
|
|
120
|
+
denial = ActionAgent.quota_denial(@account, :trace_ingest)
|
|
121
|
+
return if denial.blank?
|
|
122
|
+
|
|
123
|
+
body = { error: "Trace ingest limit reached" }
|
|
124
|
+
body = denial.is_a?(Hash) ? body.merge(denial) : body.merge(message: denial)
|
|
125
|
+
|
|
126
|
+
render json: body, status: :too_many_requests
|
|
127
|
+
end
|
|
128
|
+
|
|
109
129
|
# Extracts Bearer token from Authorization header.
|
|
110
130
|
def extract_bearer_token
|
|
111
131
|
auth_header = request.headers["Authorization"]
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module ActionAgent
|
|
4
|
+
module Api
|
|
5
|
+
# The plan meter the Organization view and the Run Agents quota banner
|
|
6
|
+
# read. Both were extracted from the platform along with the route they
|
|
7
|
+
# fetch, but the engine never gained the route, so every visit to either
|
|
8
|
+
# view logged a 404.
|
|
9
|
+
#
|
|
10
|
+
# The engine meters nothing itself. A host that tracks usage against a
|
|
11
|
+
# plan answers through ActionAgent.usage_resolver; a bare mount reports
|
|
12
|
+
# unlimited, in the same shape, so the views can hide the meter.
|
|
13
|
+
class UsageController < BaseController
|
|
14
|
+
# GET /api/usage
|
|
15
|
+
def show
|
|
16
|
+
render json: { usage: ActionAgent.usage_for(current_owner) }
|
|
17
|
+
end
|
|
18
|
+
end
|
|
19
|
+
end
|
|
20
|
+
end
|
|
@@ -39,10 +39,33 @@ module ActionAgent
|
|
|
39
39
|
end
|
|
40
40
|
|
|
41
41
|
result = ActionAgent.authentication_method.call(self)
|
|
42
|
-
|
|
42
|
+
deny_access unless result
|
|
43
43
|
rescue StandardError => e
|
|
44
44
|
Rails.logger.error("[ActionAgent] Authentication error: #{e.message}")
|
|
45
|
-
|
|
45
|
+
deny_access
|
|
46
|
+
end
|
|
47
|
+
|
|
48
|
+
# A browser asking for a page is sent to the host's sign-in page when one
|
|
49
|
+
# is configured, and otherwise shown a minimal session-expired page — a
|
|
50
|
+
# bare 401 with no body renders as a browser error screen. API and MCP
|
|
51
|
+
# clients get the bare 401 they expect.
|
|
52
|
+
def deny_access
|
|
53
|
+
return head :unauthorized unless Engine.html_request?(request)
|
|
54
|
+
|
|
55
|
+
if ActionAgent.sign_in_path.present?
|
|
56
|
+
redirect_to ActionAgent.sign_in_path, allow_other_host: false
|
|
57
|
+
else
|
|
58
|
+
render html: <<~HTML.html_safe, status: :unauthorized, layout: false
|
|
59
|
+
<!doctype html>
|
|
60
|
+
<html><head><meta charset="utf-8"><title>Sign in required</title></head>
|
|
61
|
+
<body style="font-family: -apple-system, 'Segoe UI', Roboto, sans-serif; display: grid; place-items: center; min-height: 100vh; margin: 0; color: #0f172a; background: #f8fafc;">
|
|
62
|
+
<div style="text-align: center;">
|
|
63
|
+
<h1 style="font-size: 20px;">Sign in required</h1>
|
|
64
|
+
<p style="color: #475569;">Your session has expired or you are not signed in.<br>Sign in to the host application, then reload this page.</p>
|
|
65
|
+
</div>
|
|
66
|
+
</body></html>
|
|
67
|
+
HTML
|
|
68
|
+
end
|
|
46
69
|
end
|
|
47
70
|
|
|
48
71
|
# Returns the current user from the host application.
|
|
@@ -43,8 +43,10 @@ module ActionAgent
|
|
|
43
43
|
instructionSets: Agent::INSTRUCTION_SETS,
|
|
44
44
|
availableTools: Agent::AVAILABLE_TOOLS,
|
|
45
45
|
executionEnabled: ActionAgent.execution_enabled?,
|
|
46
|
+
assistantEnabled: ActionAgent.assistant_enabled?,
|
|
46
47
|
multiTenant: ActionAgent.multi_tenant?,
|
|
47
|
-
upgradeUrl: ActionAgent.upgrade_url
|
|
48
|
+
upgradeUrl: ActionAgent.upgrade_url,
|
|
49
|
+
signOutPath: ActionAgent.sign_out_path
|
|
48
50
|
}
|
|
49
51
|
end
|
|
50
52
|
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module ActionAgent
|
|
4
|
+
module Api
|
|
5
|
+
# The agent JSON the dashboard reads, shared by every endpoint that hands
|
|
6
|
+
# an agent to the React app.
|
|
7
|
+
#
|
|
8
|
+
# Two shapes: the summary the list renders, and the detail (instructions,
|
|
9
|
+
# tools, mcp_servers, model_config, ...) the editor initializes from. Any
|
|
10
|
+
# endpoint that returns an agent the client may go on to *edit* must
|
|
11
|
+
# render the detail shape: AgentEditor seeds its form from whatever it is
|
|
12
|
+
# given, and a summary seeds it with empty instructions/tools/config,
|
|
13
|
+
# which the first Save then persists over the real record.
|
|
14
|
+
module AgentSerialization
|
|
15
|
+
extend ActiveSupport::Concern
|
|
16
|
+
|
|
17
|
+
private
|
|
18
|
+
|
|
19
|
+
def agent_json(agent, include_details: false)
|
|
20
|
+
json = {
|
|
21
|
+
id: agent.id,
|
|
22
|
+
name: agent.name,
|
|
23
|
+
slug: agent.slug,
|
|
24
|
+
description: agent.description,
|
|
25
|
+
provider: agent.provider,
|
|
26
|
+
model: agent.model,
|
|
27
|
+
status: agent.status,
|
|
28
|
+
preset_type: agent.preset_type,
|
|
29
|
+
appearance: agent.appearance,
|
|
30
|
+
version_count: agent.version_count,
|
|
31
|
+
created_at: agent.created_at,
|
|
32
|
+
updated_at: agent.updated_at
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
if include_details
|
|
36
|
+
json.merge!(
|
|
37
|
+
instructions: agent.instructions,
|
|
38
|
+
action_prompts: agent.action_prompts,
|
|
39
|
+
instruction_sets: agent.instruction_sets,
|
|
40
|
+
tools: agent.tools,
|
|
41
|
+
mcp_servers: agent.mcp_servers,
|
|
42
|
+
model_config: agent.model_config,
|
|
43
|
+
response_format: agent.response_format,
|
|
44
|
+
agent_class_name: agent.agent_class_name,
|
|
45
|
+
telemetry_agent_class: agent.telemetry_agent_class
|
|
46
|
+
)
|
|
47
|
+
end
|
|
48
|
+
|
|
49
|
+
json
|
|
50
|
+
end
|
|
51
|
+
end
|
|
52
|
+
end
|
|
53
|
+
end
|
|
@@ -6,7 +6,9 @@ module ActionAgent
|
|
|
6
6
|
|
|
7
7
|
def perform(run_id)
|
|
8
8
|
run = AgentRun.find(run_id)
|
|
9
|
-
|
|
9
|
+
# Anything already finished — complete, failed or cancelled — is never
|
|
10
|
+
# executed again. A retry of a failed run would re-run the generation.
|
|
11
|
+
return if run.finished?
|
|
10
12
|
|
|
11
13
|
run.update!(status: :running, started_at: Time.current)
|
|
12
14
|
run.add_log("Starting execution", level: :info)
|
|
@@ -17,26 +19,29 @@ module ActionAgent
|
|
|
17
19
|
# Build the agent class dynamically based on configuration
|
|
18
20
|
result = execute_agent(agent_record, run)
|
|
19
21
|
|
|
20
|
-
run
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
22
|
+
finish_unless_cancelled(run) do
|
|
23
|
+
run.update!(
|
|
24
|
+
output: result[:output],
|
|
25
|
+
output_metadata: result[:metadata],
|
|
26
|
+
status: :complete,
|
|
27
|
+
completed_at: Time.current,
|
|
28
|
+
duration_ms: ((Time.current - run.started_at) * 1000).to_i,
|
|
29
|
+
input_tokens: result.dig(:usage, :input_tokens),
|
|
30
|
+
output_tokens: result.dig(:usage, :output_tokens),
|
|
31
|
+
total_tokens: result.dig(:usage, :total_tokens)
|
|
32
|
+
)
|
|
33
|
+
run.add_log("Execution completed successfully", level: :info)
|
|
34
|
+
end
|
|
32
35
|
rescue => e
|
|
33
|
-
run
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
36
|
+
finish_unless_cancelled(run) do
|
|
37
|
+
run.update!(
|
|
38
|
+
status: :failed,
|
|
39
|
+
completed_at: Time.current,
|
|
40
|
+
error_message: e.message,
|
|
41
|
+
error_backtrace: e.backtrace&.first(10)&.join("\n")
|
|
42
|
+
)
|
|
43
|
+
run.add_log("Execution failed: #{e.message}", level: :error)
|
|
44
|
+
end
|
|
40
45
|
raise
|
|
41
46
|
ensure
|
|
42
47
|
run.broadcast_update
|
|
@@ -48,5 +53,20 @@ module ActionAgent
|
|
|
48
53
|
def execute_agent(agent_record, run)
|
|
49
54
|
AgentExecutionService.call(agent_record, run)
|
|
50
55
|
end
|
|
56
|
+
|
|
57
|
+
# A cancel that arrived while the service was running must survive it:
|
|
58
|
+
# the run's terminal state is written under a row lock after re-reading
|
|
59
|
+
# the status, so a cancelled run is never flipped back to complete or
|
|
60
|
+
# failed by the job that was still executing it.
|
|
61
|
+
def finish_unless_cancelled(run)
|
|
62
|
+
run.with_lock do
|
|
63
|
+
if run.cancelled?
|
|
64
|
+
run.add_log("Execution finished after cancellation; result discarded", level: :info)
|
|
65
|
+
return
|
|
66
|
+
end
|
|
67
|
+
|
|
68
|
+
yield
|
|
69
|
+
end
|
|
70
|
+
end
|
|
51
71
|
end
|
|
52
72
|
end
|
|
@@ -2,10 +2,14 @@
|
|
|
2
2
|
|
|
3
3
|
module ActionAgent
|
|
4
4
|
# Base class for all Dashboard engine jobs.
|
|
5
|
+
#
|
|
6
|
+
# No blanket retry_on: the engine's jobs are not idempotent. An agent run
|
|
7
|
+
# that fails after real provider work would be re-executed (and re-billed)
|
|
8
|
+
# on every retry, flipping the run's status failed -> running -> failed
|
|
9
|
+
# and overwriting its timings, after the UI had already stopped polling on
|
|
10
|
+
# the first failure. Jobs that can safely retry declare it themselves, for
|
|
11
|
+
# the specific transient errors they can tolerate.
|
|
5
12
|
class ApplicationJob < ActiveJob::Base
|
|
6
|
-
# Retry failed jobs
|
|
7
|
-
retry_on StandardError, wait: :polynomially_longer, attempts: 3
|
|
8
|
-
|
|
9
13
|
# Discard jobs for records that no longer exist
|
|
10
14
|
discard_on ActiveRecord::RecordNotFound
|
|
11
15
|
end
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module ActionAgent
|
|
4
|
+
# Executes an evaluation run created by Evaluation#run_later!. The run
|
|
5
|
+
# record already exists (pending), so the dashboard can show it while the
|
|
6
|
+
# job is queued; the runner marks it running, then complete or failed.
|
|
7
|
+
class EvaluationRunJob < ApplicationJob
|
|
8
|
+
queue_as :agents
|
|
9
|
+
|
|
10
|
+
def perform(evaluation_id, run_id, selection = {})
|
|
11
|
+
evaluation = Evaluation.find(evaluation_id)
|
|
12
|
+
run = evaluation.evaluation_runs.find(run_id)
|
|
13
|
+
return unless run.pending?
|
|
14
|
+
|
|
15
|
+
evaluation.run!(run: run, **selection.to_h.symbolize_keys)
|
|
16
|
+
end
|
|
17
|
+
end
|
|
18
|
+
end
|
|
@@ -4,16 +4,16 @@ module ActionAgent
|
|
|
4
4
|
class SandboxCleanupJob < ApplicationJob
|
|
5
5
|
queue_as :sandboxes
|
|
6
6
|
|
|
7
|
-
#
|
|
7
|
+
# Release the infrastructure behind an expired sandbox.
|
|
8
8
|
def perform(sandbox_session_id)
|
|
9
9
|
sandbox = SandboxSession.find_by(id: sandbox_session_id)
|
|
10
10
|
return unless sandbox
|
|
11
11
|
|
|
12
12
|
Rails.logger.info("Cleaning up sandbox: #{sandbox.session_id}")
|
|
13
13
|
|
|
14
|
-
#
|
|
14
|
+
# Terminate the backend resource if one was provisioned
|
|
15
15
|
if sandbox.cloud_run_job_id.present? && !Rails.env.development?
|
|
16
|
-
|
|
16
|
+
terminate_backend_sandbox(sandbox.cloud_run_job_id)
|
|
17
17
|
end
|
|
18
18
|
|
|
19
19
|
# Optionally delete old sandbox records
|
|
@@ -30,13 +30,16 @@ module ActionAgent
|
|
|
30
30
|
|
|
31
31
|
private
|
|
32
32
|
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
33
|
+
# Through the orchestrator, like provisioning: whichever backend the
|
|
34
|
+
# host registered (Incus, Kubernetes, Cloud Run, or the built-in mock)
|
|
35
|
+
# reclaims its own resource. This used to require google/cloud/run/v2
|
|
36
|
+
# directly, which the engine does not depend on — a LoadError is a
|
|
37
|
+
# ScriptError, not a StandardError, so it escaped the rescue and the job
|
|
38
|
+
# failed on every host but the one that happened to bundle the SDK.
|
|
39
|
+
def terminate_backend_sandbox(sandbox_id)
|
|
40
|
+
SandboxOrchestrator.new.terminate(sandbox_id)
|
|
41
|
+
rescue StandardError => e
|
|
42
|
+
Rails.logger.warn("Failed to terminate sandbox #{sandbox_id}: #{e.message}")
|
|
40
43
|
end
|
|
41
44
|
end
|
|
42
45
|
end
|