actionagent 1.6.4 → 1.7.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +5 -0
- data/app/assets/builds/action_agent.css +1 -1
- data/app/assets/builds/action_agent.js +57 -57
- data/app/controllers/action_agent/api/agents_controller.rb +28 -3
- data/app/controllers/action_agent/api/base_controller.rb +20 -4
- data/app/controllers/action_agent/api/dashboard_assistant_controller.rb +0 -11
- data/app/controllers/action_agent/api/evaluation_reports_controller.rb +134 -0
- data/app/controllers/action_agent/api/evaluations_controller.rb +88 -12
- data/app/controllers/action_agent/api/interactions_controller.rb +2 -3
- data/app/controllers/action_agent/api/mcp_controller.rb +3 -1
- data/app/controllers/action_agent/api/provider_models_controller.rb +39 -5
- data/app/controllers/action_agent/api/trace_reports_controller.rb +20 -1
- data/app/controllers/action_agent/api/traces_controller.rb +8 -57
- data/app/controllers/action_agent/application_controller.rb +4 -0
- data/app/controllers/concerns/action_agent/api/ingest_authentication.rb +94 -0
- data/app/models/action_agent/agent.rb +71 -3
- data/app/models/action_agent/application_record.rb +4 -0
- data/app/models/action_agent/evaluation_run.rb +61 -13
- data/app/models/action_agent/telemetry_trace.rb +4 -3
- data/app/models/concerns/action_agent/ownable.rb +18 -6
- data/app/queries/action_agent/metrics_report.rb +1 -1
- data/app/services/action_agent/agent_execution_service.rb +67 -2
- data/app/services/action_agent/agent_sync.rb +136 -0
- data/app/services/action_agent/agent_tool_roster.rb +1 -1
- data/app/services/action_agent/evaluation_report_import.rb +743 -0
- data/app/services/action_agent/evaluation_runner_service.rb +141 -19
- data/app/services/action_agent/evaluation_tool_resolver.rb +3 -3
- data/app/services/action_agent/scenario_evaluation_runner.rb +16 -7
- data/config/routes.rb +8 -0
- data/lib/action_agent/version.rb +1 -1
- data/lib/action_agent.rb +110 -13
- data/lib/generators/action_agent/install_generator.rb +26 -3
- data/lib/generators/action_agent/templates/action_agent.rb.erb +19 -3
- data/lib/generators/action_agent/templates/add_agent_releases.rb.erb +18 -13
- data/lib/generators/action_agent/templates/add_evaluation_report_identity.rb.erb +54 -0
- data/lib/generators/action_agent/templates/create_active_agent_dashboard_tables.rb.erb +34 -0
- data/lib/generators/action_agent/templates/ensure_agent_release_columns.rb.erb +51 -0
- metadata +8 -2
|
@@ -7,33 +7,38 @@
|
|
|
7
7
|
# migration carries the same columns for a fresh install.
|
|
8
8
|
#
|
|
9
9
|
# Each step is guarded so the migration is safe to re-run against a database
|
|
10
|
-
# that already has some of these columns.
|
|
10
|
+
# that already has some of these columns. Table names follow
|
|
11
|
+
# ActionAgent.table_name_prefix, except the trace table, whose name is fixed.
|
|
11
12
|
class AddAgentReleases < ActiveRecord::Migration<%= migration_version %>
|
|
12
13
|
def up
|
|
13
|
-
add_column_unless_exists :
|
|
14
|
+
add_column_unless_exists table(:agents), :release_digest, :string
|
|
14
15
|
|
|
15
|
-
add_column_unless_exists :
|
|
16
|
-
add_column_unless_exists :
|
|
17
|
-
add_index_unless_exists :
|
|
16
|
+
add_column_unless_exists table(:agent_versions), :release_digest, :string
|
|
17
|
+
add_column_unless_exists table(:agent_versions), :revision, :string
|
|
18
|
+
add_index_unless_exists table(:agent_versions), [ :agent_id, :release_digest ]
|
|
18
19
|
|
|
19
|
-
|
|
20
|
-
add_column_unless_exists
|
|
21
|
-
add_index_unless_exists
|
|
20
|
+
[ "active_agent_telemetry_traces", table(:agent_runs), table(:evaluation_runs) ].each do |name|
|
|
21
|
+
add_column_unless_exists name, :agent_version_id, :bigint
|
|
22
|
+
add_index_unless_exists name, :agent_version_id
|
|
22
23
|
end
|
|
23
24
|
end
|
|
24
25
|
|
|
25
26
|
def down
|
|
26
|
-
remove_column :
|
|
27
|
-
remove_column :
|
|
28
|
-
remove_column :
|
|
27
|
+
remove_column table(:agents), :release_digest if column_exists?(table(:agents), :release_digest)
|
|
28
|
+
remove_column table(:agent_versions), :release_digest if column_exists?(table(:agent_versions), :release_digest)
|
|
29
|
+
remove_column table(:agent_versions), :revision if column_exists?(table(:agent_versions), :revision)
|
|
29
30
|
|
|
30
|
-
|
|
31
|
-
remove_column
|
|
31
|
+
[ "active_agent_telemetry_traces", table(:agent_runs), table(:evaluation_runs) ].each do |name|
|
|
32
|
+
remove_column name, :agent_version_id if column_exists?(name, :agent_version_id)
|
|
32
33
|
end
|
|
33
34
|
end
|
|
34
35
|
|
|
35
36
|
private
|
|
36
37
|
|
|
38
|
+
def table(name)
|
|
39
|
+
"#{ActionAgent.table_name_prefix}#{name}"
|
|
40
|
+
end
|
|
41
|
+
|
|
37
42
|
def add_column_unless_exists(table, column, type)
|
|
38
43
|
return unless table_exists?(table)
|
|
39
44
|
return if column_exists?(table, column)
|
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
# Published evaluation reports: an application that runs its agents itself
|
|
4
|
+
# can evaluate them in-process and POST the finished report to
|
|
5
|
+
# <mount>/api/evaluation_reports (ActionAgent::EvaluationReportImport), which
|
|
6
|
+
# stores it as an evaluation run. The run is identified by the run_id the
|
|
7
|
+
# application minted, within the tenant whose key published it, and carries
|
|
8
|
+
# the digest of the report, so an identical retry resolves to the same run and
|
|
9
|
+
# different content under the same run_id is refused.
|
|
10
|
+
#
|
|
11
|
+
# The tenant is its id, or "" on a single-tenant install, because a unique
|
|
12
|
+
# index never treats two NULLs as equal. Both are compared exactly, which on
|
|
13
|
+
# MySQL takes a binary collation. All three columns are NULL for a run the
|
|
14
|
+
# dashboard executed. Emitted for an install whose dashboard tables predate
|
|
15
|
+
# published reports; the create-table migration carries the same columns for
|
|
16
|
+
# a fresh install. Each step is guarded, so the migration is safe to re-run.
|
|
17
|
+
class AddEvaluationReportIdentity < ActiveRecord::Migration<%= migration_version %>
|
|
18
|
+
IDENTITY = [ :external_tenant, :external_run_id ].freeze
|
|
19
|
+
|
|
20
|
+
def up
|
|
21
|
+
return unless table_exists?(table)
|
|
22
|
+
|
|
23
|
+
IDENTITY.each do |column|
|
|
24
|
+
add_column table, column, :string, **exact_collation unless column_exists?(table, column)
|
|
25
|
+
end
|
|
26
|
+
add_column table, :external_report_digest, :string unless column_exists?(table, :external_report_digest)
|
|
27
|
+
add_index table, IDENTITY, unique: true, name: index_name unless index_exists?(table, IDENTITY, name: index_name)
|
|
28
|
+
end
|
|
29
|
+
|
|
30
|
+
def down
|
|
31
|
+
return unless table_exists?(table)
|
|
32
|
+
|
|
33
|
+
remove_index table, name: index_name if index_exists?(table, IDENTITY, name: index_name)
|
|
34
|
+
%i[external_tenant external_run_id external_report_digest].each do |column|
|
|
35
|
+
remove_column table, column if column_exists?(table, column)
|
|
36
|
+
end
|
|
37
|
+
end
|
|
38
|
+
|
|
39
|
+
private
|
|
40
|
+
|
|
41
|
+
def table
|
|
42
|
+
"#{ActionAgent.table_name_prefix}evaluation_runs"
|
|
43
|
+
end
|
|
44
|
+
|
|
45
|
+
def index_name
|
|
46
|
+
"index_#{ActionAgent.table_name_prefix}evaluation_runs_on_external_identity"
|
|
47
|
+
end
|
|
48
|
+
|
|
49
|
+
# MySQL's default collation ignores case and accents, so "nightly-a" and
|
|
50
|
+
# "Nightly-A" would be one run_id there and two everywhere else.
|
|
51
|
+
def exact_collation
|
|
52
|
+
connection.adapter_name.to_s.downcase.match?(/mysql|trilogy/) ? { collation: "utf8mb4_bin" } : {}
|
|
53
|
+
end
|
|
54
|
+
end
|
|
@@ -39,6 +39,8 @@ class CreateActiveAgentDashboardTables < ActiveRecord::Migration<%= migration_ve
|
|
|
39
39
|
t.string :source
|
|
40
40
|
t.datetime :first_observed_at
|
|
41
41
|
t.datetime :last_observed_at
|
|
42
|
+
# The digest of the release the agent last shipped in (Agent#record_release!).
|
|
43
|
+
t.string :release_digest
|
|
42
44
|
|
|
43
45
|
t.bigint :user_id
|
|
44
46
|
t.bigint :account_id
|
|
@@ -55,8 +57,13 @@ class CreateActiveAgentDashboardTables < ActiveRecord::Migration<%= migration_ve
|
|
|
55
57
|
t.column :configuration_snapshot, json_type, **json_default({}, null: false)
|
|
56
58
|
t.string :change_summary
|
|
57
59
|
t.string :created_by
|
|
60
|
+
# A version cut from a release carries its digest and the deploy it
|
|
61
|
+
# shipped in.
|
|
62
|
+
t.string :release_digest
|
|
63
|
+
t.string :revision
|
|
58
64
|
t.timestamps
|
|
59
65
|
t.index [ :agent_id, :version_number ], unique: true
|
|
66
|
+
t.index [ :agent_id, :release_digest ]
|
|
60
67
|
end
|
|
61
68
|
|
|
62
69
|
create_table "#{prefix}agent_runs" do |t|
|
|
@@ -77,10 +84,13 @@ class CreateActiveAgentDashboardTables < ActiveRecord::Migration<%= migration_ve
|
|
|
77
84
|
t.integer :duration_ms
|
|
78
85
|
t.datetime :started_at
|
|
79
86
|
t.datetime :completed_at
|
|
87
|
+
# The version of the agent the run executed.
|
|
88
|
+
t.bigint :agent_version_id
|
|
80
89
|
t.timestamps
|
|
81
90
|
t.index :agent_id
|
|
82
91
|
t.index :trace_id
|
|
83
92
|
t.index :created_at
|
|
93
|
+
t.index :agent_version_id
|
|
84
94
|
end
|
|
85
95
|
|
|
86
96
|
create_table "#{prefix}agent_templates" do |t|
|
|
@@ -197,8 +207,22 @@ class CreateActiveAgentDashboardTables < ActiveRecord::Migration<%= migration_ve
|
|
|
197
207
|
t.integer :samples_passed, default: 0
|
|
198
208
|
t.text :error_message
|
|
199
209
|
t.datetime :completed_at
|
|
210
|
+
# The version of the agent the run scored.
|
|
211
|
+
t.bigint :agent_version_id
|
|
212
|
+
# Set only on a run an application executed itself and published
|
|
213
|
+
# (ActionAgent::EvaluationReportImport): the tenant whose key published
|
|
214
|
+
# it, the run_id the application minted, and the digest of the report.
|
|
215
|
+
# The tenant is its id, or "" on a single-tenant install, because a
|
|
216
|
+
# unique index never treats two NULLs as equal. Both are compared
|
|
217
|
+
# exactly, which on MySQL takes a binary collation.
|
|
218
|
+
t.string :external_tenant, **exact_collation
|
|
219
|
+
t.string :external_run_id, **exact_collation
|
|
220
|
+
t.string :external_report_digest
|
|
200
221
|
t.timestamps
|
|
201
222
|
t.index :evaluation_id
|
|
223
|
+
t.index :agent_version_id
|
|
224
|
+
t.index [ :external_tenant, :external_run_id ], unique: true,
|
|
225
|
+
name: "index_#{prefix}evaluation_runs_on_external_identity"
|
|
202
226
|
end
|
|
203
227
|
|
|
204
228
|
create_table "#{prefix}sandbox_sessions" do |t|
|
|
@@ -328,7 +352,17 @@ class CreateActiveAgentDashboardTables < ActiveRecord::Migration<%= migration_ve
|
|
|
328
352
|
null.nil? ? { default: value } : { default: value, null: null }
|
|
329
353
|
end
|
|
330
354
|
|
|
355
|
+
# MySQL's default collation ignores case and accents, so "nightly-a" and
|
|
356
|
+
# "Nightly-A" would be one run_id there and two everywhere else.
|
|
357
|
+
def exact_collation
|
|
358
|
+
mysql? ? { collation: "utf8mb4_bin" } : {}
|
|
359
|
+
end
|
|
360
|
+
|
|
331
361
|
def postgres?
|
|
332
362
|
connection.adapter_name.to_s.downcase.include?("postgres")
|
|
333
363
|
end
|
|
364
|
+
|
|
365
|
+
def mysql?
|
|
366
|
+
connection.adapter_name.to_s.downcase.match?(/mysql|trilogy/)
|
|
367
|
+
end
|
|
334
368
|
end
|
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
# Adds the agent release columns wherever they are missing: release_digest on
|
|
4
|
+
# agents, release_digest and revision on agent versions, and agent_version_id
|
|
5
|
+
# on traces, agent runs and evaluation runs.
|
|
6
|
+
#
|
|
7
|
+
# Two kinds of install lack them. One generated fresh by an actionagent
|
|
8
|
+
# release whose add_agent_releases ran before its dashboard tables existed,
|
|
9
|
+
# so every step of it was skipped. And one whose tables carry a custom
|
|
10
|
+
# ActionAgent.table_name_prefix, which the copy of add_agent_releases it
|
|
11
|
+
# holds does not read. Each step is guarded, so this is a no-op where the
|
|
12
|
+
# columns already exist, and it is emitted after the dashboard tables.
|
|
13
|
+
class EnsureAgentReleaseColumns < ActiveRecord::Migration<%= migration_version %>
|
|
14
|
+
def up
|
|
15
|
+
add_column_unless_exists table(:agents), :release_digest, :string
|
|
16
|
+
|
|
17
|
+
add_column_unless_exists table(:agent_versions), :release_digest, :string
|
|
18
|
+
add_column_unless_exists table(:agent_versions), :revision, :string
|
|
19
|
+
add_index_unless_exists table(:agent_versions), [ :agent_id, :release_digest ]
|
|
20
|
+
|
|
21
|
+
# The trace table keeps its literal name whatever the prefix
|
|
22
|
+
# (ActionAgent::TelemetryTrace.table_name).
|
|
23
|
+
[ "active_agent_telemetry_traces", table(:agent_runs), table(:evaluation_runs) ].each do |name|
|
|
24
|
+
add_column_unless_exists name, :agent_version_id, :bigint
|
|
25
|
+
add_index_unless_exists name, :agent_version_id
|
|
26
|
+
end
|
|
27
|
+
end
|
|
28
|
+
|
|
29
|
+
# The columns may predate this migration, so rolling it back leaves them.
|
|
30
|
+
def down; end
|
|
31
|
+
|
|
32
|
+
private
|
|
33
|
+
|
|
34
|
+
def table(name)
|
|
35
|
+
"#{ActionAgent.table_name_prefix}#{name}"
|
|
36
|
+
end
|
|
37
|
+
|
|
38
|
+
def add_column_unless_exists(table, column, type)
|
|
39
|
+
return unless table_exists?(table)
|
|
40
|
+
return if column_exists?(table, column)
|
|
41
|
+
|
|
42
|
+
add_column table, column, type
|
|
43
|
+
end
|
|
44
|
+
|
|
45
|
+
def add_index_unless_exists(table, columns)
|
|
46
|
+
return unless table_exists?(table)
|
|
47
|
+
return if index_exists?(table, columns)
|
|
48
|
+
|
|
49
|
+
add_index table, columns
|
|
50
|
+
end
|
|
51
|
+
end
|
metadata
CHANGED
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: actionagent
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 1.
|
|
4
|
+
version: 1.7.1
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Justin Bowen
|
|
8
8
|
autorequire:
|
|
9
9
|
bindir: bin
|
|
10
10
|
cert_chain: []
|
|
11
|
-
date: 2026-09-
|
|
11
|
+
date: 2026-09-30 00:00:00.000000000 Z
|
|
12
12
|
dependencies:
|
|
13
13
|
- !ruby/object:Gem::Dependency
|
|
14
14
|
name: activeagent
|
|
@@ -130,6 +130,7 @@ files:
|
|
|
130
130
|
- app/controllers/action_agent/api/api_keys_controller.rb
|
|
131
131
|
- app/controllers/action_agent/api/base_controller.rb
|
|
132
132
|
- app/controllers/action_agent/api/dashboard_assistant_controller.rb
|
|
133
|
+
- app/controllers/action_agent/api/evaluation_reports_controller.rb
|
|
133
134
|
- app/controllers/action_agent/api/evaluations_controller.rb
|
|
134
135
|
- app/controllers/action_agent/api/instance_tiers_controller.rb
|
|
135
136
|
- app/controllers/action_agent/api/interaction_messages_controller.rb
|
|
@@ -150,6 +151,7 @@ files:
|
|
|
150
151
|
- app/controllers/action_agent/dashboard_controller.rb
|
|
151
152
|
- app/controllers/action_agent/traces_controller.rb
|
|
152
153
|
- app/controllers/concerns/action_agent/api/agent_serialization.rb
|
|
154
|
+
- app/controllers/concerns/action_agent/api/ingest_authentication.rb
|
|
153
155
|
- app/jobs/action_agent/agent_execution_job.rb
|
|
154
156
|
- app/jobs/action_agent/application_job.rb
|
|
155
157
|
- app/jobs/action_agent/evaluation_run_job.rb
|
|
@@ -195,10 +197,12 @@ files:
|
|
|
195
197
|
- app/services/action_agent/agent_registrar.rb
|
|
196
198
|
- app/services/action_agent/agent_release.rb
|
|
197
199
|
- app/services/action_agent/agent_scorecard.rb
|
|
200
|
+
- app/services/action_agent/agent_sync.rb
|
|
198
201
|
- app/services/action_agent/agent_tool_roster.rb
|
|
199
202
|
- app/services/action_agent/agent_toolbox.rb
|
|
200
203
|
- app/services/action_agent/dashboard_assistant_service.rb
|
|
201
204
|
- app/services/action_agent/evaluation_evidence.rb
|
|
205
|
+
- app/services/action_agent/evaluation_report_import.rb
|
|
202
206
|
- app/services/action_agent/evaluation_runner_service.rb
|
|
203
207
|
- app/services/action_agent/evaluation_tool_resolver.rb
|
|
204
208
|
- app/services/action_agent/mcp_catalog.rb
|
|
@@ -229,9 +233,11 @@ files:
|
|
|
229
233
|
- lib/generators/action_agent/templates/action_agent.rb.erb
|
|
230
234
|
- lib/generators/action_agent/templates/add_agent_id_to_active_agent_telemetry_traces.rb.erb
|
|
231
235
|
- lib/generators/action_agent/templates/add_agent_releases.rb.erb
|
|
236
|
+
- lib/generators/action_agent/templates/add_evaluation_report_identity.rb.erb
|
|
232
237
|
- lib/generators/action_agent/templates/create_active_agent_dashboard_tables.rb.erb
|
|
233
238
|
- lib/generators/action_agent/templates/create_active_agent_evaluation_scenarios.rb.erb
|
|
234
239
|
- lib/generators/action_agent/templates/create_active_agent_telemetry_traces.rb.erb
|
|
240
|
+
- lib/generators/action_agent/templates/ensure_agent_release_columns.rb.erb
|
|
235
241
|
- lib/tasks/action_agent.rake
|
|
236
242
|
homepage: https://activeagents.ai
|
|
237
243
|
licenses:
|