actionagent 1.6.4 → 1.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +5 -0
- data/app/assets/builds/action_agent.css +1 -1
- data/app/assets/builds/action_agent.js +55 -55
- data/app/controllers/action_agent/api/agents_controller.rb +8 -2
- data/app/controllers/action_agent/api/base_controller.rb +20 -4
- data/app/controllers/action_agent/api/dashboard_assistant_controller.rb +0 -11
- data/app/controllers/action_agent/api/evaluation_reports_controller.rb +134 -0
- data/app/controllers/action_agent/api/evaluations_controller.rb +52 -11
- data/app/controllers/action_agent/api/interactions_controller.rb +2 -3
- data/app/controllers/action_agent/api/mcp_controller.rb +3 -1
- data/app/controllers/action_agent/api/provider_models_controller.rb +4 -1
- data/app/controllers/action_agent/api/trace_reports_controller.rb +20 -1
- data/app/controllers/action_agent/api/traces_controller.rb +8 -57
- data/app/controllers/action_agent/application_controller.rb +4 -0
- data/app/controllers/concerns/action_agent/api/ingest_authentication.rb +94 -0
- data/app/models/action_agent/agent.rb +71 -3
- data/app/models/action_agent/application_record.rb +4 -0
- data/app/models/action_agent/evaluation_run.rb +61 -13
- data/app/models/action_agent/telemetry_trace.rb +4 -3
- data/app/models/concerns/action_agent/ownable.rb +18 -6
- data/app/queries/action_agent/metrics_report.rb +1 -1
- data/app/services/action_agent/agent_execution_service.rb +49 -0
- data/app/services/action_agent/agent_sync.rb +136 -0
- data/app/services/action_agent/agent_tool_roster.rb +1 -1
- data/app/services/action_agent/evaluation_report_import.rb +743 -0
- data/app/services/action_agent/evaluation_runner_service.rb +119 -10
- data/app/services/action_agent/scenario_evaluation_runner.rb +8 -3
- data/config/routes.rb +5 -0
- data/lib/action_agent/version.rb +1 -1
- data/lib/action_agent.rb +110 -13
- data/lib/generators/action_agent/install_generator.rb +26 -3
- data/lib/generators/action_agent/templates/action_agent.rb.erb +19 -3
- data/lib/generators/action_agent/templates/add_agent_releases.rb.erb +18 -13
- data/lib/generators/action_agent/templates/add_evaluation_report_identity.rb.erb +54 -0
- data/lib/generators/action_agent/templates/create_active_agent_dashboard_tables.rb.erb +34 -0
- data/lib/generators/action_agent/templates/ensure_agent_release_columns.rb.erb +51 -0
- metadata +8 -2
|
@@ -90,7 +90,7 @@ module ActionAgent
|
|
|
90
90
|
if @agent.save
|
|
91
91
|
render json: { agent: agent_json(@agent, include_details: true) }, status: :created
|
|
92
92
|
else
|
|
93
|
-
render json:
|
|
93
|
+
render json: agent_errors_json(@agent), status: :unprocessable_entity
|
|
94
94
|
end
|
|
95
95
|
end
|
|
96
96
|
|
|
@@ -99,7 +99,7 @@ module ActionAgent
|
|
|
99
99
|
if @agent.update(agent_params)
|
|
100
100
|
render json: { agent: agent_json(@agent, include_details: true) }
|
|
101
101
|
else
|
|
102
|
-
render json:
|
|
102
|
+
render json: agent_errors_json(@agent), status: :unprocessable_entity
|
|
103
103
|
end
|
|
104
104
|
end
|
|
105
105
|
|
|
@@ -495,6 +495,12 @@ module ActionAgent
|
|
|
495
495
|
@agent = owner_agents.find(params[:id])
|
|
496
496
|
end
|
|
497
497
|
|
|
498
|
+
# +errors+ for a form-level summary; +field_errors+ (attribute => full
|
|
499
|
+
# messages) so the builder and editor can put each under its field.
|
|
500
|
+
def agent_errors_json(agent)
|
|
501
|
+
{ errors: agent.errors.full_messages, field_errors: agent.errors.to_hash(true) }
|
|
502
|
+
end
|
|
503
|
+
|
|
498
504
|
def agent_params
|
|
499
505
|
permitted = params.require(:agent).permit(
|
|
500
506
|
:name, :description, :provider, :model, :instructions,
|
|
@@ -12,11 +12,23 @@ module ActionAgent
|
|
|
12
12
|
# install, a per-user install and a multi-tenant platform all read the
|
|
13
13
|
# same controllers.
|
|
14
14
|
#
|
|
15
|
-
#
|
|
16
|
-
#
|
|
15
|
+
# Because it authenticates with the host's session cookie, it keeps the
|
|
16
|
+
# forgery protection ApplicationController turns on: the dashboard sends
|
|
17
|
+
# the page's CSRF token with every mutating request (frontend
|
|
18
|
+
# utils/apiFetch.mjs). Endpoints that authenticate with a bearer token
|
|
19
|
+
# instead — the telemetry ingest endpoint (Api::TracesController), the
|
|
20
|
+
# evaluation report collector (Api::EvaluationReportsController) and the
|
|
21
|
+
# MCP facade (Api::MCPController) — are exempt.
|
|
17
22
|
class BaseController < ActionAgent::ApplicationController
|
|
18
|
-
|
|
19
|
-
|
|
23
|
+
# Rails 8.2 verifies forgery protection from the browser's Sec-Fetch-Site
|
|
24
|
+
# header, renamed the failure to InvalidCrossOriginRequest, and deprecated
|
|
25
|
+
# the old name. Rescue whichever names the running Rails defines, so a
|
|
26
|
+
# rejected request answers with the dashboard's JSON either way.
|
|
27
|
+
# const_defined? does not fire the deprecation the bare constant would.
|
|
28
|
+
rescue_from ActionController::InvalidCrossOriginRequest, with: :invalid_authenticity_token
|
|
29
|
+
if ActionController.const_defined?(:InvalidAuthenticityToken, false)
|
|
30
|
+
rescue_from ActionController::InvalidAuthenticityToken, with: :invalid_authenticity_token
|
|
31
|
+
end
|
|
20
32
|
rescue_from ActiveRecord::RecordNotFound, with: :not_found
|
|
21
33
|
rescue_from ActiveRecord::RecordInvalid, with: :unprocessable_entity
|
|
22
34
|
rescue_from ActionController::ParameterMissing, with: :bad_request
|
|
@@ -24,6 +36,10 @@ module ActionAgent
|
|
|
24
36
|
|
|
25
37
|
private
|
|
26
38
|
|
|
39
|
+
def invalid_authenticity_token
|
|
40
|
+
render json: { error: "Refresh the dashboard and try again", code: "invalid_csrf_token" }, status: :unprocessable_entity
|
|
41
|
+
end
|
|
42
|
+
|
|
27
43
|
# API keys and provider credentials are encrypted at rest, which needs
|
|
28
44
|
# Active Record Encryption keys. The engine derives fallback keys when
|
|
29
45
|
# the host set none (see Engine's action_agent.active_record_encryption
|
|
@@ -3,8 +3,6 @@
|
|
|
3
3
|
module ActionAgent
|
|
4
4
|
module Api
|
|
5
5
|
class DashboardAssistantController < BaseController
|
|
6
|
-
protect_from_forgery with: :exception
|
|
7
|
-
|
|
8
6
|
before_action :require_assistant_enabled!
|
|
9
7
|
before_action :require_owner!
|
|
10
8
|
before_action :require_execution_enabled!, only: :create
|
|
@@ -13,15 +11,6 @@ module ActionAgent
|
|
|
13
11
|
rescue_from DashboardAssistantService::InvalidInput, with: :invalid_input
|
|
14
12
|
rescue_from DashboardAssistantService::ProcessingConsentRequired, with: :processing_consent_required
|
|
15
13
|
rescue_from DashboardAssistantService::SetupRequired, with: :setup_required
|
|
16
|
-
# Rails 8.2 verifies forgery protection from the browser's Sec-Fetch-Site
|
|
17
|
-
# header, renamed the failure to InvalidCrossOriginRequest, and deprecated
|
|
18
|
-
# the old name. Rescue whichever names the running Rails defines, so a
|
|
19
|
-
# rejected request answers with the dashboard's JSON either way.
|
|
20
|
-
# const_defined? does not fire the deprecation the bare constant would.
|
|
21
|
-
rescue_from ActionController::InvalidCrossOriginRequest, with: :invalid_authenticity_token
|
|
22
|
-
if ActionController.const_defined?(:InvalidAuthenticityToken, false)
|
|
23
|
-
rescue_from ActionController::InvalidAuthenticityToken, with: :invalid_authenticity_token
|
|
24
|
-
end
|
|
25
14
|
|
|
26
15
|
def show
|
|
27
16
|
render json: DashboardAssistantService.new(owner: current_owner).configuration
|
|
@@ -0,0 +1,134 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module ActionAgent
|
|
4
|
+
module Api
|
|
5
|
+
# Collector for evaluation reports an application ran itself and
|
|
6
|
+
# published with ActiveAgent::Evals::Publisher:
|
|
7
|
+
# POST <mount>/api/evaluation_reports (e.g. /activeagents/api/evaluation_reports).
|
|
8
|
+
#
|
|
9
|
+
# Authenticated exactly as trace ingest is (IngestAuthentication), and
|
|
10
|
+
# stored by EvaluationReportImport. Responds:
|
|
11
|
+
#
|
|
12
|
+
# 201 — stored; the receipt the publisher checks
|
|
13
|
+
# 200 — an identical retry; the receipt names the stored run. Never
|
|
14
|
+
# refused by the quota or the rate limit.
|
|
15
|
+
# 409 — this run_id already holds a different report
|
|
16
|
+
# 422 — not a valid version-1 report, or an evaluation name the agent
|
|
17
|
+
# already holds for another suite or scope
|
|
18
|
+
# 403 — storing it needs an operator first: a cap on observed agents,
|
|
19
|
+
# evaluations or scenarios, or no owner for the tenant
|
|
20
|
+
# 429 — a new report over the host's quota (kind :evaluation_report) or
|
|
21
|
+
# over RATE_LIMIT new reports a minute from one key
|
|
22
|
+
# 413 — a body over EvaluationReportImport::MAX_BYTES
|
|
23
|
+
# 415 — a body that is not declared application/json
|
|
24
|
+
# 400 — a body that is not JSON
|
|
25
|
+
# 401 — a missing or unknown key
|
|
26
|
+
# 501 — an install with no evaluation tables, or not yet migrated
|
|
27
|
+
# 503 — another import for the same agent held the lock too long
|
|
28
|
+
class EvaluationReportsController < ActionController::API
|
|
29
|
+
include IngestAuthentication
|
|
30
|
+
|
|
31
|
+
# New reports one key may store per minute.
|
|
32
|
+
RATE_LIMIT = 30
|
|
33
|
+
|
|
34
|
+
wrap_parameters false
|
|
35
|
+
|
|
36
|
+
before_action :require_json!
|
|
37
|
+
before_action :require_report_store!
|
|
38
|
+
|
|
39
|
+
# POST <mount>/api/evaluation_reports
|
|
40
|
+
def create
|
|
41
|
+
# The header, not request.content_length, which reads a chunked body
|
|
42
|
+
# in full to measure it.
|
|
43
|
+
return report_too_large if request.get_header("CONTENT_LENGTH").to_i > EvaluationReportImport::MAX_BYTES
|
|
44
|
+
|
|
45
|
+
body = request.body.read(EvaluationReportImport::MAX_BYTES + 1).to_s
|
|
46
|
+
return report_too_large if body.bytesize > EvaluationReportImport::MAX_BYTES
|
|
47
|
+
|
|
48
|
+
run, duplicate = EvaluationReportImport.call(account: @account, payload: JSON.parse(body), admit: -> { admission_denial })
|
|
49
|
+
ActionAgent.record_usage(@account, :evaluation_report) unless duplicate
|
|
50
|
+
|
|
51
|
+
render json: receipt(run, duplicate), status: duplicate ? :ok : :created
|
|
52
|
+
rescue JSON::ParserError
|
|
53
|
+
render json: { error: "Invalid JSON" }, status: :bad_request
|
|
54
|
+
rescue EvaluationReportImport::Invalid, ActiveRecord::RecordInvalid => e
|
|
55
|
+
render json: { error: e.message }, status: :unprocessable_entity
|
|
56
|
+
rescue EvaluationReportImport::Conflict => e
|
|
57
|
+
render json: { error: e.message }, status: :conflict
|
|
58
|
+
rescue EvaluationReportImport::Refused => e
|
|
59
|
+
render json: { error: e.message }, status: :forbidden
|
|
60
|
+
rescue EvaluationReportImport::Denied => e
|
|
61
|
+
render json: e.denial, status: :too_many_requests
|
|
62
|
+
rescue EvaluationReportImport::Busy => e
|
|
63
|
+
response.headers["Retry-After"] = "5"
|
|
64
|
+
render json: { error: e.message }, status: :service_unavailable
|
|
65
|
+
end
|
|
66
|
+
|
|
67
|
+
private
|
|
68
|
+
|
|
69
|
+
# Rails reads and parses a JSON body into params before any callback
|
|
70
|
+
# runs, for the request log among others. With none to parse, the body
|
|
71
|
+
# is read only by #create, and only up to the size limit.
|
|
72
|
+
def process_action(*)
|
|
73
|
+
request.request_parameters = {}
|
|
74
|
+
super
|
|
75
|
+
end
|
|
76
|
+
|
|
77
|
+
# A cross-site page can send a text/plain or form POST without a CORS
|
|
78
|
+
# preflight; it cannot send application/json.
|
|
79
|
+
def require_json!
|
|
80
|
+
return if request.media_type == "application/json"
|
|
81
|
+
|
|
82
|
+
render json: { error: "Content-Type must be application/json" }, status: :unsupported_media_type
|
|
83
|
+
end
|
|
84
|
+
|
|
85
|
+
def require_report_store!
|
|
86
|
+
reason = EvaluationReportImport.unavailable_reason
|
|
87
|
+
render json: { error: reason }, status: :not_implemented if reason
|
|
88
|
+
end
|
|
89
|
+
|
|
90
|
+
# What refuses a report that would be stored, or nil: the rate limit,
|
|
91
|
+
# then the host app's quota checker, asked with kind :evaluation_report.
|
|
92
|
+
# Never asked for an identical retry.
|
|
93
|
+
def admission_denial
|
|
94
|
+
return { error: "Too many evaluation reports; retry in a minute" } if rate_limited?
|
|
95
|
+
|
|
96
|
+
denial = ActionAgent.quota_denial(@account, :evaluation_report)
|
|
97
|
+
quota_denial_body(denial, "Evaluation report limit reached") if denial.present?
|
|
98
|
+
end
|
|
99
|
+
|
|
100
|
+
# Counts a new report against its key's bucket, in the store Rails'
|
|
101
|
+
# own rate_limit uses. One bucket per key: the tenant's on a
|
|
102
|
+
# multi-tenant install, the install's own on a single-tenant one.
|
|
103
|
+
def rate_limited?
|
|
104
|
+
bucket = @account ? "account:#{@account.id}" : "install"
|
|
105
|
+
count = self.class.cache_store.increment("rate-limit:#{controller_path}:#{bucket}", 1, expires_in: 1.minute)
|
|
106
|
+
count.present? && count > RATE_LIMIT
|
|
107
|
+
end
|
|
108
|
+
|
|
109
|
+
def receipt(run, duplicate)
|
|
110
|
+
{
|
|
111
|
+
id: run.id,
|
|
112
|
+
evaluation_id: run.evaluation_id,
|
|
113
|
+
run_id: run.external_run_id,
|
|
114
|
+
status: run.status,
|
|
115
|
+
duplicate: duplicate,
|
|
116
|
+
url: run_url(run)
|
|
117
|
+
}
|
|
118
|
+
end
|
|
119
|
+
|
|
120
|
+
# The dashboard page that shows the run, as a path on this host. A host
|
|
121
|
+
# that routes to this controller from outside the engine's mount
|
|
122
|
+
# overrides it.
|
|
123
|
+
def run_url(run)
|
|
124
|
+
"#{request.script_name}/evaluations/#{run.evaluation_id}/runs/#{run.id}"
|
|
125
|
+
end
|
|
126
|
+
|
|
127
|
+
# 413 by number: Rack named it :payload_too_large before 3.1 and
|
|
128
|
+
# :content_too_large since, and the engine supports both.
|
|
129
|
+
def report_too_large
|
|
130
|
+
render json: { error: "Report exceeds #{EvaluationReportImport::MAX_BYTES / 1.megabyte} MiB" }, status: 413
|
|
131
|
+
end
|
|
132
|
+
end
|
|
133
|
+
end
|
|
134
|
+
end
|
|
@@ -46,14 +46,20 @@ module ActionAgent
|
|
|
46
46
|
render json: { evaluations: evaluations.map { |evaluation| serialize(evaluation) } }
|
|
47
47
|
end
|
|
48
48
|
|
|
49
|
+
# Runs listed per evaluation on GET /api/evaluations/:id. The rest of
|
|
50
|
+
# the history stays reachable by run id; `run_count` says how long it is.
|
|
51
|
+
RUN_HISTORY_LIMIT = 20
|
|
52
|
+
|
|
49
53
|
# GET /api/evaluations/:id
|
|
50
54
|
def show
|
|
51
55
|
evaluation = evaluations_scope.find(params[:id])
|
|
56
|
+
run_count = evaluation.evaluation_runs.count
|
|
57
|
+
runs = evaluation.evaluation_runs.recent.limit(RUN_HISTORY_LIMIT).to_a
|
|
52
58
|
|
|
53
59
|
render json: {
|
|
54
60
|
evaluation: serialize(evaluation).merge(
|
|
55
61
|
scenarios: evaluation.scenarios.ordered.map(&:as_json_summary),
|
|
56
|
-
runs:
|
|
62
|
+
runs: runs.each_with_index.map { |run, index| serialize_run(run, number: run_count - index) }
|
|
57
63
|
)
|
|
58
64
|
}
|
|
59
65
|
end
|
|
@@ -98,8 +104,12 @@ module ActionAgent
|
|
|
98
104
|
def run
|
|
99
105
|
evaluation = current_evaluation
|
|
100
106
|
run = start_run(evaluation, selection_params)
|
|
107
|
+
evaluation.reload
|
|
101
108
|
|
|
102
|
-
render json: {
|
|
109
|
+
render json: {
|
|
110
|
+
evaluation: serialize(evaluation),
|
|
111
|
+
run: serialize_run(run, number: evaluation.evaluation_runs.count)
|
|
112
|
+
}
|
|
103
113
|
end
|
|
104
114
|
|
|
105
115
|
# GET /api/evaluations/:id/runs/:run_id
|
|
@@ -116,7 +126,8 @@ module ActionAgent
|
|
|
116
126
|
|
|
117
127
|
render json: {
|
|
118
128
|
evaluation: serialize(evaluation),
|
|
119
|
-
run: serialize_run(run
|
|
129
|
+
run: serialize_run(run, number: run_number(evaluation, run))
|
|
130
|
+
.merge(results: results.map(&:as_json_summary), fix_items: safe_fix_items(run))
|
|
120
131
|
}
|
|
121
132
|
end
|
|
122
133
|
|
|
@@ -328,7 +339,9 @@ module ActionAgent
|
|
|
328
339
|
end
|
|
329
340
|
|
|
330
341
|
def serialize(evaluation)
|
|
331
|
-
|
|
342
|
+
# size reads the preloaded association on index and COUNTs elsewhere.
|
|
343
|
+
run_count = evaluation.evaluation_runs.size
|
|
344
|
+
latest, previous = recent_runs(evaluation, 2)
|
|
332
345
|
|
|
333
346
|
{
|
|
334
347
|
id: evaluation.id,
|
|
@@ -344,22 +357,50 @@ module ActionAgent
|
|
|
344
357
|
scenario_count: evaluation.scenarios.size,
|
|
345
358
|
scenario_groups: evaluation.scenario_suite? ? evaluation.scenario_groups : [],
|
|
346
359
|
created_at: evaluation.created_at.iso8601,
|
|
347
|
-
|
|
360
|
+
run_count: run_count,
|
|
361
|
+
latest_run: latest ? serialize_run(latest, number: run_count) : nil,
|
|
362
|
+
# Just enough of the run before it for the list to show movement
|
|
363
|
+
# ("+3 passed vs #2") without a request per evaluation.
|
|
364
|
+
previous_run: previous ? serialize_run_summary(previous, number: run_count - 1) : nil
|
|
348
365
|
}
|
|
349
366
|
end
|
|
350
367
|
|
|
351
|
-
|
|
352
|
-
|
|
353
|
-
|
|
354
|
-
|
|
368
|
+
# Newest first. Sorts the preloaded association when index loaded it
|
|
369
|
+
# rather than issuing one ORDER BY query per evaluation.
|
|
370
|
+
def recent_runs(evaluation, limit)
|
|
371
|
+
runs = evaluation.evaluation_runs
|
|
372
|
+
if runs.loaded?
|
|
373
|
+
runs.sort_by { |run| [ run.created_at, run.id ] }.reverse.first(limit)
|
|
374
|
+
else
|
|
375
|
+
runs.recent.limit(limit).to_a
|
|
376
|
+
end
|
|
377
|
+
end
|
|
378
|
+
|
|
379
|
+
# A run's position in its evaluation's history, oldest = 1.
|
|
380
|
+
def run_number(evaluation, run)
|
|
381
|
+
evaluation.evaluation_runs.where("created_at < ? OR (created_at = ? AND id <= ?)", run.created_at, run.created_at, run.id).count
|
|
382
|
+
end
|
|
383
|
+
|
|
384
|
+
# `number` is the run's position in its evaluation's history, oldest =
|
|
385
|
+
# 1, so the dashboard can say "Run #3" and "vs #2".
|
|
386
|
+
def serialize_run(run, number: nil)
|
|
387
|
+
serialize_run_summary(run, number: number).merge(
|
|
355
388
|
scores: run.scores,
|
|
356
389
|
selection: run.selection,
|
|
357
390
|
models: run.models,
|
|
391
|
+
usage: run.usage,
|
|
392
|
+
error_message: run.error_message
|
|
393
|
+
)
|
|
394
|
+
end
|
|
395
|
+
|
|
396
|
+
def serialize_run_summary(run, number: nil)
|
|
397
|
+
{
|
|
398
|
+
id: run.id,
|
|
399
|
+
number: number,
|
|
400
|
+
status: run.status,
|
|
358
401
|
average_score: safe_average_score(run),
|
|
359
402
|
samples_evaluated: run.samples_evaluated,
|
|
360
403
|
samples_passed: run.samples_passed,
|
|
361
|
-
usage: run.usage,
|
|
362
|
-
error_message: run.error_message,
|
|
363
404
|
completed_at: run.completed_at&.iso8601,
|
|
364
405
|
created_at: run.created_at.iso8601
|
|
365
406
|
}
|
|
@@ -127,13 +127,12 @@ module ActionAgent
|
|
|
127
127
|
end
|
|
128
128
|
|
|
129
129
|
# Traces belong to an agent by foreign key once AgentRegistrar attributes
|
|
130
|
-
# them; older rows predate that, so fall back to the
|
|
130
|
+
# them; older rows predate that, so fall back to the agent's identity.
|
|
131
131
|
def traces_for_agent(agent_id)
|
|
132
132
|
agent = owner_agents.find_by(id: agent_id)
|
|
133
133
|
return ActionAgent.trace_model.none unless agent
|
|
134
134
|
|
|
135
|
-
|
|
136
|
-
.or(ActionAgent.trace_model.where(agent_id: nil, agent_class: agent.telemetry_agent_class))
|
|
135
|
+
owned_traces.where(agent_id: agent.id).or(agent.unattributed_telemetry_traces(owned_traces))
|
|
137
136
|
end
|
|
138
137
|
|
|
139
138
|
# The dashboard-wide time window, shared with Traces. Absent means "all".
|
|
@@ -24,8 +24,10 @@ module ActionAgent
|
|
|
24
24
|
# { "type": "http", "url": "https://activeagents.ai/mcp",
|
|
25
25
|
# "headers": { "Authorization": "Bearer aa_..." } }
|
|
26
26
|
class MCPController < BaseController
|
|
27
|
-
# Authenticated by API key rather than by the host app's sessions
|
|
27
|
+
# Authenticated by API key rather than by the host app's sessions, so
|
|
28
|
+
# there is no session cookie for a cross-site request to ride on.
|
|
28
29
|
allow_unauthenticated_access
|
|
30
|
+
skip_forgery_protection
|
|
29
31
|
before_action :authenticate_api_key!, except: [ :unsupported ]
|
|
30
32
|
|
|
31
33
|
PROTOCOL_VERSION = "2025-03-26"
|
|
@@ -92,12 +92,15 @@ module ActionAgent
|
|
|
92
92
|
nil
|
|
93
93
|
end
|
|
94
94
|
|
|
95
|
+
# The whole catalog, never a prefix of it: OpenRouter serves several
|
|
96
|
+
# hundred models, and a cap after sorting left everything late in the
|
|
97
|
+
# alphabet (openai/*, qwen/*, ...) unselectable. The editor filters it.
|
|
95
98
|
def live_openrouter_models
|
|
96
99
|
data = Rails.cache.fetch("provider_models:openrouter", expires_in: 1.hour) do
|
|
97
100
|
fetch_json(URI.parse("https://openrouter.ai/api/v1/models"))
|
|
98
101
|
end
|
|
99
102
|
ids = Array(data&.dig("data")).filter_map { |model| model["id"] }
|
|
100
|
-
[ ids.sort
|
|
103
|
+
[ ids.sort, "live" ] if ids.any?
|
|
101
104
|
rescue StandardError => e
|
|
102
105
|
Rails.logger.warn("[ProviderModels] openrouter lookup failed: #{e.message}")
|
|
103
106
|
nil
|
|
@@ -16,9 +16,19 @@ module ActionAgent
|
|
|
16
16
|
DEFAULT_LIMIT = 500
|
|
17
17
|
|
|
18
18
|
# GET /api/traces
|
|
19
|
+
#
|
|
20
|
+
# Params:
|
|
21
|
+
# minutes the window in minutes, DEFAULT_WINDOW_MINUTES when absent
|
|
22
|
+
# agent_id an agent the caller can see. Everything in the response,
|
|
23
|
+
# `agents` and `agent_ids` included, is narrowed to that
|
|
24
|
+
# agent's traces (Agent#telemetry_traces). 404 for an agent
|
|
25
|
+
# the caller cannot see.
|
|
26
|
+
# agent an agent_class, narrowing `traces` only
|
|
27
|
+
# service a service_name, narrowing `traces` only
|
|
28
|
+
# status "error" for failed traces only
|
|
19
29
|
def index
|
|
20
30
|
window = params.fetch(:minutes, DEFAULT_WINDOW_MINUTES).to_i.clamp(1, MAX_WINDOW_MINUTES)
|
|
21
|
-
window_scope = traces_scope.for_date_range(window.minutes.ago, Time.current)
|
|
31
|
+
window_scope = agent_scope(traces_scope).for_date_range(window.minutes.ago, Time.current)
|
|
22
32
|
|
|
23
33
|
scope = window_scope
|
|
24
34
|
scope = scope.for_agent(params[:agent]) if params[:agent].present?
|
|
@@ -54,6 +64,15 @@ module ActionAgent
|
|
|
54
64
|
ActionAgent.trace_model.for_account(current_account)
|
|
55
65
|
end
|
|
56
66
|
|
|
67
|
+
# +scope+ narrowed to the traces of the agent `agent_id` names, looked
|
|
68
|
+
# up among the agents the caller can see.
|
|
69
|
+
def agent_scope(scope)
|
|
70
|
+
agent_id = integer_param(:agent_id)
|
|
71
|
+
return scope unless agent_id
|
|
72
|
+
|
|
73
|
+
owner_agents.find(agent_id).telemetry_traces(scope)
|
|
74
|
+
end
|
|
75
|
+
|
|
57
76
|
# One grouped query. A class can appear under several agent records
|
|
58
77
|
# (same class, different action); the most recently active one wins,
|
|
59
78
|
# since that's what the operator most likely means by "this agent".
|
|
@@ -34,8 +34,8 @@ module ActionAgent
|
|
|
34
34
|
# }
|
|
35
35
|
#
|
|
36
36
|
class TracesController < ActionController::API
|
|
37
|
-
|
|
38
|
-
|
|
37
|
+
include IngestAuthentication
|
|
38
|
+
|
|
39
39
|
before_action :enforce_ingest_quota!
|
|
40
40
|
|
|
41
41
|
# POST <mount>/api/traces (e.g. /activeagents/api/traces)
|
|
@@ -75,64 +75,15 @@ module ActionAgent
|
|
|
75
75
|
|
|
76
76
|
private
|
|
77
77
|
|
|
78
|
-
#
|
|
79
|
-
# Only used in multi-tenant mode.
|
|
80
|
-
def authenticate_api_key!
|
|
81
|
-
token = extract_bearer_token
|
|
82
|
-
|
|
83
|
-
if token.blank?
|
|
84
|
-
render json: { error: "Missing Authorization header" }, status: :unauthorized
|
|
85
|
-
return
|
|
86
|
-
end
|
|
87
|
-
|
|
88
|
-
account_class = ActionAgent.account_class.constantize
|
|
89
|
-
@account = account_class.find_by(telemetry_api_key: token)
|
|
90
|
-
|
|
91
|
-
if @account.nil?
|
|
92
|
-
render json: { error: "Invalid API key" }, status: :unauthorized
|
|
93
|
-
return
|
|
94
|
-
end
|
|
95
|
-
|
|
96
|
-
# Track usage for rate limiting (if the account responds to it)
|
|
97
|
-
@account.increment_telemetry_usage! if @account.respond_to?(:increment_telemetry_usage!)
|
|
98
|
-
end
|
|
99
|
-
|
|
100
|
-
# Requires the configured single-tenant ingest key when one is set.
|
|
101
|
-
# The telemetry reporter and ruby_llm_telemetry both send their
|
|
102
|
-
# api_key as a Bearer header, so remote apps work unchanged.
|
|
103
|
-
def authenticate_ingest_key!
|
|
104
|
-
expected = ActionAgent.ingest_api_key
|
|
105
|
-
return if expected.blank?
|
|
106
|
-
|
|
107
|
-
token = extract_bearer_token
|
|
108
|
-
return if token.present? && ActiveSupport::SecurityUtils.secure_compare(token, expected)
|
|
109
|
-
|
|
110
|
-
render json: { error: "Invalid API key" }, status: :unauthorized
|
|
111
|
-
end
|
|
112
|
-
|
|
113
|
-
# The host app's quota checker, asked with kind :trace_ingest — the
|
|
114
|
-
# counterpart to Api::BaseController#enforce_quota!, which asks with
|
|
115
|
-
# :execution. Denials are 429 here rather than 402: a reporter that is
|
|
116
|
-
# over its ingest allowance should back off, not upgrade mid-flush.
|
|
117
|
-
# Same body shape, so a checker's message or Hash payload reads the
|
|
118
|
-
# same on both.
|
|
78
|
+
# The host app's quota checker, asked with kind :trace_ingest.
|
|
119
79
|
def enforce_ingest_quota!
|
|
120
|
-
|
|
121
|
-
return if denial.blank?
|
|
122
|
-
|
|
123
|
-
body = { error: "Trace ingest limit reached" }
|
|
124
|
-
body = denial.is_a?(Hash) ? body.merge(denial) : body.merge(message: denial)
|
|
125
|
-
|
|
126
|
-
render json: body, status: :too_many_requests
|
|
80
|
+
enforce_ingest_quota_for!(:trace_ingest, "Trace ingest limit reached")
|
|
127
81
|
end
|
|
128
82
|
|
|
129
|
-
#
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
match = auth_header.match(/^Bearer\s+(.+)$/i)
|
|
135
|
-
match[1] if match
|
|
83
|
+
# Counts the request against the tenant, when its account model
|
|
84
|
+
# defines the hook.
|
|
85
|
+
def record_ingest_request
|
|
86
|
+
@account.increment_telemetry_usage! if @account.respond_to?(:increment_telemetry_usage!)
|
|
136
87
|
end
|
|
137
88
|
|
|
138
89
|
# Process traces synchronously for local development.
|
|
@@ -5,6 +5,10 @@ module ActionAgent
|
|
|
5
5
|
#
|
|
6
6
|
# Handles authentication and provides helper methods for multi-tenant mode.
|
|
7
7
|
class ApplicationController < ActionController::Base
|
|
8
|
+
# Ahead of the engine's own callbacks, so a concern's before_action or
|
|
9
|
+
# around_action runs before the dashboard authenticates.
|
|
10
|
+
ActionAgent.controller_concern_modules.each { |concern| include concern }
|
|
11
|
+
|
|
8
12
|
protect_from_forgery with: :exception
|
|
9
13
|
|
|
10
14
|
before_action :authenticate_dashboard!
|
|
@@ -0,0 +1,94 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module ActionAgent
|
|
4
|
+
module Api
|
|
5
|
+
# Bearer-token authentication for the endpoints other applications post
|
|
6
|
+
# to: trace ingest (Api::TracesController) and published evaluation
|
|
7
|
+
# reports (Api::EvaluationReportsController). Neither is a dashboard
|
|
8
|
+
# controller, so neither sees the host's session or controller concerns.
|
|
9
|
+
#
|
|
10
|
+
# A multi-tenant install authenticates the tenant's key and sets
|
|
11
|
+
# `@account` to the tenant it names. A single-tenant install requires
|
|
12
|
+
# ActionAgent.ingest_api_key when one is set and leaves `@account` nil.
|
|
13
|
+
module IngestAuthentication
|
|
14
|
+
extend ActiveSupport::Concern
|
|
15
|
+
|
|
16
|
+
included do
|
|
17
|
+
before_action :authenticate_api_key!, if: -> { ActionAgent.multi_tenant? }
|
|
18
|
+
before_action :authenticate_ingest_key!, unless: -> { ActionAgent.multi_tenant? }
|
|
19
|
+
end
|
|
20
|
+
|
|
21
|
+
private
|
|
22
|
+
|
|
23
|
+
# Finds the tenant whose `telemetry_api_key` is the bearer token.
|
|
24
|
+
# Only used in multi-tenant mode.
|
|
25
|
+
def authenticate_api_key!
|
|
26
|
+
token = extract_bearer_token
|
|
27
|
+
|
|
28
|
+
if token.blank?
|
|
29
|
+
render json: { error: "Missing Authorization header" }, status: :unauthorized
|
|
30
|
+
return
|
|
31
|
+
end
|
|
32
|
+
|
|
33
|
+
account_class = ActionAgent.account_class.constantize
|
|
34
|
+
@account = account_class.find_by(telemetry_api_key: token)
|
|
35
|
+
|
|
36
|
+
if @account.nil?
|
|
37
|
+
render json: { error: "Invalid API key" }, status: :unauthorized
|
|
38
|
+
return
|
|
39
|
+
end
|
|
40
|
+
|
|
41
|
+
record_ingest_request
|
|
42
|
+
end
|
|
43
|
+
|
|
44
|
+
# Called once the tenant has authenticated. Counts nothing here: trace
|
|
45
|
+
# ingest counts the request through the tenant's
|
|
46
|
+
# increment_telemetry_usage!, and the report collector records a stored
|
|
47
|
+
# report through ActionAgent.usage_recorder instead.
|
|
48
|
+
def record_ingest_request; end
|
|
49
|
+
|
|
50
|
+
# Requires the configured single-tenant ingest key when one is set.
|
|
51
|
+
# The telemetry reporter, ruby_llm_telemetry and
|
|
52
|
+
# ActiveAgent::Evals::Publisher all send their api_key as a Bearer
|
|
53
|
+
# header, so remote apps work unchanged.
|
|
54
|
+
def authenticate_ingest_key!
|
|
55
|
+
expected = ActionAgent.ingest_api_key
|
|
56
|
+
return if expected.blank?
|
|
57
|
+
|
|
58
|
+
token = extract_bearer_token
|
|
59
|
+
return if token.present? && ActiveSupport::SecurityUtils.secure_compare(token, expected)
|
|
60
|
+
|
|
61
|
+
render json: { error: "Invalid API key" }, status: :unauthorized
|
|
62
|
+
end
|
|
63
|
+
|
|
64
|
+
# Asks the host app's quota checker whether the tenant may do +kind+,
|
|
65
|
+
# and renders 429 with +error+ when it may not. The counterpart to
|
|
66
|
+
# Api::BaseController#enforce_quota!, which answers 402: a client that
|
|
67
|
+
# is over its ingest allowance should back off, not upgrade mid-flush.
|
|
68
|
+
# Same body shape, so a checker's message or Hash payload reads the
|
|
69
|
+
# same on both.
|
|
70
|
+
def enforce_ingest_quota_for!(kind, error)
|
|
71
|
+
denial = ActionAgent.quota_denial(@account, kind)
|
|
72
|
+
return if denial.blank?
|
|
73
|
+
|
|
74
|
+
render json: quota_denial_body(denial, error), status: :too_many_requests
|
|
75
|
+
end
|
|
76
|
+
|
|
77
|
+
# The 429 body for a quota checker's +denial+: its Hash merged in, or
|
|
78
|
+
# its String as the message.
|
|
79
|
+
def quota_denial_body(denial, error)
|
|
80
|
+
body = { error: error }
|
|
81
|
+
denial.is_a?(Hash) ? body.merge(denial) : body.merge(message: denial)
|
|
82
|
+
end
|
|
83
|
+
|
|
84
|
+
# Extracts Bearer token from Authorization header.
|
|
85
|
+
def extract_bearer_token
|
|
86
|
+
auth_header = request.headers["Authorization"]
|
|
87
|
+
return nil if auth_header.blank?
|
|
88
|
+
|
|
89
|
+
match = auth_header.match(/^Bearer\s+(.+)$/i)
|
|
90
|
+
match[1] if match
|
|
91
|
+
end
|
|
92
|
+
end
|
|
93
|
+
end
|
|
94
|
+
end
|