activeagent 1.9.3 → 1.9.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +165 -0
- data/README.md +7 -1
- data/exe/activeagent +8 -0
- data/lib/active_agent/cli/client.rb +227 -0
- data/lib/active_agent/cli/credentials.rb +79 -0
- data/lib/active_agent/cli/eval_commands.rb +264 -0
- data/lib/active_agent/cli/resource_commands.rb +154 -0
- data/lib/active_agent/cli/session_commands.rb +106 -0
- data/lib/active_agent/cli/trace_commands.rb +129 -0
- data/lib/active_agent/cli.rb +300 -0
- data/lib/active_agent/concerns/preview.rb +2 -2
- data/lib/active_agent/evals/design_tokens.rb +12 -1
- data/lib/active_agent/evals/publisher.rb +72 -17
- data/lib/active_agent/inline_preview_interceptor.rb +1 -0
- data/lib/active_agent/preview/templates/layouts/active_agent/preview.html.erb +17 -0
- data/lib/active_agent/preview/templates/rails/agents/index.html.erb +14 -0
- data/lib/active_agent/preview/templates/rails/agents/preview.html.erb +3 -0
- data/lib/active_agent/railtie.rb +11 -5
- data/lib/active_agent/version.rb +1 -1
- data/lib/generators/test_unit/agent_generator.rb +1 -1
- data/lib/generators/test_unit/templates/functional_test.rb.tt +1 -1
- data/lib/generators/test_unit/templates/preview.rb.tt +3 -3
- data/lib/rails/agents_controller.rb +43 -0
- metadata +17 -4
|
@@ -0,0 +1,300 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "json"
|
|
4
|
+
require "optparse"
|
|
5
|
+
require_relative "version"
|
|
6
|
+
|
|
7
|
+
module ActiveAgent
|
|
8
|
+
# The `activeagent` executable: a dashboard's evaluations, scenario
|
|
9
|
+
# catalogs, agents and traces from a terminal or a CI job, over HTTP with a
|
|
10
|
+
# dashboard API key. Plain Ruby (OptionParser and Net::HTTP); it loads
|
|
11
|
+
# neither Rails nor the framework, so it starts fast and runs anywhere the
|
|
12
|
+
# gem is installed.
|
|
13
|
+
#
|
|
14
|
+
# activeagent login --url https://ops.example.com/activeagents --key aa_...
|
|
15
|
+
# activeagent evals run "Support orders" --wait --min-pass-rate 0.8
|
|
16
|
+
#
|
|
17
|
+
# Every command answers through an API-key-authenticated endpoint: the MCP
|
|
18
|
+
# facade's dashboard tools (<mount>/mcp) with the dashboard API key, or the
|
|
19
|
+
# ingest endpoints the framework's own reporters post to (the report
|
|
20
|
+
# collector at <mount>/api/evaluation_reports, trace ingest at
|
|
21
|
+
# <mount>/api/traces) with the ingest key. A dashboard that does not offer
|
|
22
|
+
# what a command needs (an older engine, or one that turned the dashboard
|
|
23
|
+
# tools off) exits EXIT_UNAVAILABLE saying so.
|
|
24
|
+
#
|
|
25
|
+
# CLI.start returns the exit code rather than exiting, so it can be called
|
|
26
|
+
# from Ruby and tested in process.
|
|
27
|
+
class CLI
|
|
28
|
+
EXIT_SUCCESS = 0
|
|
29
|
+
# A request failed or the dashboard refused it.
|
|
30
|
+
EXIT_FAILURE = 1
|
|
31
|
+
# The command line was wrong.
|
|
32
|
+
EXIT_USAGE = 2
|
|
33
|
+
# No dashboard URL or key to use.
|
|
34
|
+
EXIT_CONFIG = 3
|
|
35
|
+
# An evaluation run failed, or passed fewer scenarios than --min-pass-rate.
|
|
36
|
+
EXIT_GATE = 4
|
|
37
|
+
# The dashboard does not offer what the command needs: an older engine,
|
|
38
|
+
# or one with its dashboard tools turned off.
|
|
39
|
+
EXIT_UNAVAILABLE = 5
|
|
40
|
+
|
|
41
|
+
# Every failure a command reports: the message is printed and the exit
|
|
42
|
+
# code returned. +status+ is the HTTP status or JSON-RPC code behind it.
|
|
43
|
+
class Error < StandardError
|
|
44
|
+
attr_reader :exit_code, :status
|
|
45
|
+
|
|
46
|
+
def initialize(message = nil, exit_code: EXIT_FAILURE, status: nil)
|
|
47
|
+
super(message)
|
|
48
|
+
@exit_code = exit_code
|
|
49
|
+
@status = status
|
|
50
|
+
end
|
|
51
|
+
end
|
|
52
|
+
|
|
53
|
+
def self.version
|
|
54
|
+
ActiveAgent::VERSION
|
|
55
|
+
end
|
|
56
|
+
|
|
57
|
+
def self.start(argv = ARGV, **options)
|
|
58
|
+
new(**options).run(argv.dup)
|
|
59
|
+
end
|
|
60
|
+
|
|
61
|
+
# +sleeper+ and +clock+ let tests drive `evals run --wait` without
|
|
62
|
+
# waiting.
|
|
63
|
+
def initialize(out: $stdout, err: $stderr, input: $stdin, env: ENV, credentials_path: nil,
|
|
64
|
+
sleeper: ->(seconds) { sleep(seconds) }, clock: -> { Process.clock_gettime(Process::CLOCK_MONOTONIC) })
|
|
65
|
+
@out = out
|
|
66
|
+
@err = err
|
|
67
|
+
@input = input
|
|
68
|
+
@env = env
|
|
69
|
+
@credentials = Credentials.new(credentials_path || Credentials.default_path(env))
|
|
70
|
+
@sleeper = sleeper
|
|
71
|
+
@clock = clock
|
|
72
|
+
@options = {}
|
|
73
|
+
end
|
|
74
|
+
|
|
75
|
+
def run(argv)
|
|
76
|
+
global_parser.order!(argv)
|
|
77
|
+
return print_version if @options[:version]
|
|
78
|
+
|
|
79
|
+
command = argv.shift
|
|
80
|
+
return print_help(global_parser) if command.nil? || command == "help" || @options[:help]
|
|
81
|
+
|
|
82
|
+
dispatch(command, argv)
|
|
83
|
+
rescue OptionParser::ParseError => e
|
|
84
|
+
@err.puts "activeagent: #{e.message}"
|
|
85
|
+
@err.puts "Run `activeagent help` for usage."
|
|
86
|
+
EXIT_USAGE
|
|
87
|
+
rescue Error => e
|
|
88
|
+
@err.puts "activeagent: #{e.message}"
|
|
89
|
+
e.exit_code
|
|
90
|
+
end
|
|
91
|
+
|
|
92
|
+
COMMANDS = {
|
|
93
|
+
"login" => "Store a dashboard URL and API key as a profile",
|
|
94
|
+
"logout" => "Remove a stored profile",
|
|
95
|
+
"status" => "Check the URL and keys against the dashboard",
|
|
96
|
+
"evals" => "Evaluations: list, runs, run, report, import",
|
|
97
|
+
"catalogs" => "Scenario catalogs: list, import (or preview), export",
|
|
98
|
+
"agents" => "Agents: list, versions",
|
|
99
|
+
"traces" => "Telemetry traces: list, get, export, import",
|
|
100
|
+
"version" => "Print the version"
|
|
101
|
+
}.freeze
|
|
102
|
+
|
|
103
|
+
private
|
|
104
|
+
|
|
105
|
+
def dispatch(command, argv)
|
|
106
|
+
case command
|
|
107
|
+
when "login" then login(argv)
|
|
108
|
+
when "logout" then logout(argv)
|
|
109
|
+
when "status" then status(argv)
|
|
110
|
+
when "version" then print_version
|
|
111
|
+
when "evals" then subcommand("evals", argv, %w[list runs run report import])
|
|
112
|
+
when "catalogs" then subcommand("catalogs", argv, %w[list import export])
|
|
113
|
+
when "agents" then subcommand("agents", argv, %w[list versions])
|
|
114
|
+
when "traces" then subcommand("traces", argv, %w[list get export import])
|
|
115
|
+
else
|
|
116
|
+
raise Error.new("unknown command '#{command}'. Commands: #{COMMANDS.keys.join(", ")}", exit_code: EXIT_USAGE)
|
|
117
|
+
end
|
|
118
|
+
end
|
|
119
|
+
|
|
120
|
+
def subcommand(group, argv, names)
|
|
121
|
+
name = argv.shift
|
|
122
|
+
if name.nil? || %w[help -h --help].include?(name)
|
|
123
|
+
@out.puts "Usage: activeagent #{group} <#{names.join("|")}> [options]"
|
|
124
|
+
return name.nil? ? EXIT_USAGE : EXIT_SUCCESS
|
|
125
|
+
end
|
|
126
|
+
unless names.include?(name)
|
|
127
|
+
raise Error.new("unknown command '#{group} #{name}'. Try: #{names.map { |n| "#{group} #{n}" }.join(", ")}", exit_code: EXIT_USAGE)
|
|
128
|
+
end
|
|
129
|
+
|
|
130
|
+
send(:"#{group}_#{name}", argv)
|
|
131
|
+
end
|
|
132
|
+
|
|
133
|
+
def global_parser
|
|
134
|
+
@global_parser ||= OptionParser.new do |parser|
|
|
135
|
+
parser.banner = "Usage: activeagent <command> [options]"
|
|
136
|
+
parser.separator ""
|
|
137
|
+
parser.separator "Commands:"
|
|
138
|
+
COMMANDS.each { |name, description| parser.separator format(" %-10s %s", name, description) }
|
|
139
|
+
parser.separator ""
|
|
140
|
+
parser.separator "Options:"
|
|
141
|
+
common_options(parser)
|
|
142
|
+
parser.on("-v", "--version", "Print the version") { @options[:version] = true }
|
|
143
|
+
parser.separator ""
|
|
144
|
+
parser.separator "Environment: ACTIVEAGENT_URL, ACTIVEAGENT_API_KEY, ACTIVEAGENT_INGEST_KEY and"
|
|
145
|
+
parser.separator "ACTIVEAGENT_PROFILE override the stored profile (for CI)."
|
|
146
|
+
parser.separator "Exit codes: 0 ok, 1 failed, 2 usage, 3 not configured, 4 evaluation gate failed,"
|
|
147
|
+
parser.separator "5 not available on this dashboard (an older engine, or its dashboard tools turned off)."
|
|
148
|
+
end
|
|
149
|
+
end
|
|
150
|
+
|
|
151
|
+
# Options every command takes, before or after the command name.
|
|
152
|
+
def common_options(parser)
|
|
153
|
+
parser.on("--profile NAME", "Use this stored profile (default: ACTIVEAGENT_PROFILE or 'default')") { |value| @options[:profile] = value }
|
|
154
|
+
parser.on("--url URL", "The dashboard mount, e.g. https://ops.example.com/activeagents") { |value| @options[:url] = value }
|
|
155
|
+
parser.on("--json", "Print JSON for scripts instead of tables") { @options[:json] = true }
|
|
156
|
+
parser.on("-h", "--help", "Show help") { @options[:help] = true }
|
|
157
|
+
end
|
|
158
|
+
|
|
159
|
+
# Parses +argv+ with the command's own options plus the common ones and
|
|
160
|
+
# returns the remaining positional arguments, printing the command's help
|
|
161
|
+
# and returning nil for --help.
|
|
162
|
+
def parse(argv, banner)
|
|
163
|
+
parser = OptionParser.new do |p|
|
|
164
|
+
p.banner = "Usage: activeagent #{banner}"
|
|
165
|
+
yield p if block_given?
|
|
166
|
+
common_options(p)
|
|
167
|
+
end
|
|
168
|
+
parser.parse!(argv)
|
|
169
|
+
if @options[:help]
|
|
170
|
+
@out.puts parser
|
|
171
|
+
return nil
|
|
172
|
+
end
|
|
173
|
+
argv
|
|
174
|
+
end
|
|
175
|
+
|
|
176
|
+
def print_help(parser)
|
|
177
|
+
@out.puts parser
|
|
178
|
+
EXIT_SUCCESS
|
|
179
|
+
end
|
|
180
|
+
|
|
181
|
+
def print_version
|
|
182
|
+
@out.puts(@options[:json] ? JSON.generate(version: self.class.version) : "activeagent #{self.class.version}")
|
|
183
|
+
EXIT_SUCCESS
|
|
184
|
+
end
|
|
185
|
+
|
|
186
|
+
def profile_name
|
|
187
|
+
@options[:profile] || @env["ACTIVEAGENT_PROFILE"].to_s.then { |name| name.empty? ? "default" : name }
|
|
188
|
+
end
|
|
189
|
+
|
|
190
|
+
# The profile's stored values with the environment and --url over them.
|
|
191
|
+
def profile
|
|
192
|
+
stored = @credentials[profile_name]
|
|
193
|
+
if stored.empty? && @options[:profile] && !@env["ACTIVEAGENT_URL"]
|
|
194
|
+
raise Error.new("No profile '#{profile_name}' in #{@credentials.path}; run `activeagent login --profile #{profile_name}`",
|
|
195
|
+
exit_code: EXIT_CONFIG)
|
|
196
|
+
end
|
|
197
|
+
|
|
198
|
+
{
|
|
199
|
+
"url" => @options[:url] || env_value("ACTIVEAGENT_URL") || stored["url"],
|
|
200
|
+
"api_key" => env_value("ACTIVEAGENT_API_KEY") || stored["api_key"],
|
|
201
|
+
"ingest_key" => env_value("ACTIVEAGENT_INGEST_KEY") || stored["ingest_key"]
|
|
202
|
+
}
|
|
203
|
+
end
|
|
204
|
+
|
|
205
|
+
def env_value(name)
|
|
206
|
+
value = @env[name].to_s.strip
|
|
207
|
+
value.empty? ? nil : value
|
|
208
|
+
end
|
|
209
|
+
|
|
210
|
+
def client
|
|
211
|
+
@client ||= begin
|
|
212
|
+
values = profile
|
|
213
|
+
if values["url"].to_s.empty?
|
|
214
|
+
raise Error.new("No dashboard URL: run `activeagent login --url <mount URL>`, or set ACTIVEAGENT_URL", exit_code: EXIT_CONFIG)
|
|
215
|
+
end
|
|
216
|
+
Client.new(url: values["url"], api_key: values["api_key"], ingest_key: values["ingest_key"])
|
|
217
|
+
end
|
|
218
|
+
end
|
|
219
|
+
|
|
220
|
+
# Prints +data+ as JSON with --json, otherwise yields for the human form.
|
|
221
|
+
def emit(data)
|
|
222
|
+
if @options[:json]
|
|
223
|
+
@out.puts JSON.pretty_generate(data)
|
|
224
|
+
else
|
|
225
|
+
yield
|
|
226
|
+
end
|
|
227
|
+
EXIT_SUCCESS
|
|
228
|
+
end
|
|
229
|
+
|
|
230
|
+
# Prints +rows+ as aligned columns. +columns+ maps each header to a
|
|
231
|
+
# proc reading a row's cell.
|
|
232
|
+
def table(rows, columns, empty: "Nothing to show.")
|
|
233
|
+
return @out.puts(empty) if rows.empty?
|
|
234
|
+
|
|
235
|
+
cells = rows.map { |row| columns.values.map { |reader| cell(reader.call(row)) } }
|
|
236
|
+
widths = columns.keys.each_with_index.map { |header, i| ([ header ] + cells.map { |c| c[i] }).map(&:length).max }
|
|
237
|
+
line = ->(values) { values.each_with_index.map { |value, i| value.ljust(widths[i]) }.join(" ").rstrip }
|
|
238
|
+
@out.puts line.call(columns.keys)
|
|
239
|
+
cells.each { |c| @out.puts line.call(c) }
|
|
240
|
+
end
|
|
241
|
+
|
|
242
|
+
def cell(value)
|
|
243
|
+
value.nil? ? "-" : value.to_s.gsub(/[[:space:]\p{C}]+/, " ")
|
|
244
|
+
end
|
|
245
|
+
|
|
246
|
+
# Writes +body+ to +path+, or prints it, ending in one newline either
|
|
247
|
+
# way. With a path, the block's line is said on standard error (not with
|
|
248
|
+
# --json, whose standard output is the only thing a script reads).
|
|
249
|
+
def write_output(body, path)
|
|
250
|
+
body = "#{body}\n" unless body.end_with?("\n")
|
|
251
|
+
if path
|
|
252
|
+
File.write(path, body)
|
|
253
|
+
@err.puts yield if block_given? && !@options[:json]
|
|
254
|
+
else
|
|
255
|
+
@out.print body
|
|
256
|
+
end
|
|
257
|
+
EXIT_SUCCESS
|
|
258
|
+
end
|
|
259
|
+
|
|
260
|
+
def read_text_file(path)
|
|
261
|
+
File.read(path)
|
|
262
|
+
rescue Errno::ENOENT, Errno::EACCES, Errno::EISDIR => e
|
|
263
|
+
raise Error.new("Cannot read #{path} (#{e.class})", exit_code: EXIT_USAGE)
|
|
264
|
+
end
|
|
265
|
+
|
|
266
|
+
def read_json_file(path)
|
|
267
|
+
document = JSON.parse(read_text_file(path))
|
|
268
|
+
raise Error.new("#{path} must hold a JSON object", exit_code: EXIT_USAGE) unless document.is_a?(Hash)
|
|
269
|
+
|
|
270
|
+
document
|
|
271
|
+
rescue JSON::ParserError
|
|
272
|
+
raise Error.new("#{path} is not valid JSON", exit_code: EXIT_USAGE), cause: nil
|
|
273
|
+
end
|
|
274
|
+
|
|
275
|
+
def require_argument(argv, name, usage)
|
|
276
|
+
value = argv.shift
|
|
277
|
+
raise Error.new("missing #{name}. Usage: activeagent #{usage}", exit_code: EXIT_USAGE) if value.to_s.empty?
|
|
278
|
+
raise Error.new("unexpected arguments: #{argv.join(" ")}", exit_code: EXIT_USAGE) unless argv.empty?
|
|
279
|
+
|
|
280
|
+
value
|
|
281
|
+
end
|
|
282
|
+
|
|
283
|
+
def positive_integer(value, name)
|
|
284
|
+
Integer(value, 10).tap { |n| raise ArgumentError unless n.positive? }
|
|
285
|
+
rescue ArgumentError, TypeError
|
|
286
|
+
raise Error.new("#{name} must be a positive integer", exit_code: EXIT_USAGE)
|
|
287
|
+
end
|
|
288
|
+
|
|
289
|
+
def list_option(value)
|
|
290
|
+
value.split(",").map(&:strip).reject(&:empty?)
|
|
291
|
+
end
|
|
292
|
+
end
|
|
293
|
+
end
|
|
294
|
+
|
|
295
|
+
require_relative "cli/credentials"
|
|
296
|
+
require_relative "cli/client"
|
|
297
|
+
require_relative "cli/session_commands"
|
|
298
|
+
require_relative "cli/eval_commands"
|
|
299
|
+
require_relative "cli/resource_commands"
|
|
300
|
+
require_relative "cli/trace_commands"
|
|
@@ -68,7 +68,7 @@ module ActiveAgent
|
|
|
68
68
|
class << self
|
|
69
69
|
# Returns all agent preview classes.
|
|
70
70
|
def all
|
|
71
|
-
load_previews
|
|
71
|
+
load_previews
|
|
72
72
|
descendants.sort_by { |agent| agent.name.titleize }
|
|
73
73
|
end
|
|
74
74
|
|
|
@@ -111,7 +111,7 @@ module ActiveAgent
|
|
|
111
111
|
|
|
112
112
|
def load_previews
|
|
113
113
|
preview_paths.each do |preview_path|
|
|
114
|
-
Dir["#{preview_path}/**/*_preview.rb"].sort.each { |file|
|
|
114
|
+
Dir["#{preview_path}/**/*_preview.rb"].sort.each { |file| require_dependency file }
|
|
115
115
|
end
|
|
116
116
|
end
|
|
117
117
|
|
|
@@ -58,6 +58,13 @@ module ActiveAgent
|
|
|
58
58
|
"--color-info" => "#3b82f6",
|
|
59
59
|
"--color-info-soft" => "#dbeafe",
|
|
60
60
|
"--color-info-text" => "#1e40af",
|
|
61
|
+
# The system and developer message roles: badge tones like the four above.
|
|
62
|
+
"--color-system" => "#8b5cf6",
|
|
63
|
+
"--color-system-soft" => "#ede9fe",
|
|
64
|
+
"--color-system-text" => "#6d28d9",
|
|
65
|
+
"--color-dev" => "#14b8a6",
|
|
66
|
+
"--color-dev-soft" => "#ccfbf1",
|
|
67
|
+
"--color-dev-text" => "#0f766e",
|
|
61
68
|
|
|
62
69
|
# Trace span colors (observability)
|
|
63
70
|
"--span-root" => "#9ca3af",
|
|
@@ -111,7 +118,11 @@ module ActiveAgent
|
|
|
111
118
|
"--color-error-soft" => "rgba(220,38,38,0.15)",
|
|
112
119
|
"--color-error-text" => "#f87171",
|
|
113
120
|
"--color-info-soft" => "rgba(59,130,246,0.15)",
|
|
114
|
-
"--color-info-text" => "#93c5fd"
|
|
121
|
+
"--color-info-text" => "#93c5fd",
|
|
122
|
+
"--color-system-soft" => "rgba(139,92,246,0.15)",
|
|
123
|
+
"--color-system-text" => "#c4b5fd",
|
|
124
|
+
"--color-dev-soft" => "rgba(20,184,166,0.15)",
|
|
125
|
+
"--color-dev-text" => "#5eead4"
|
|
115
126
|
}.freeze
|
|
116
127
|
|
|
117
128
|
# One CSS rule declaring `tokens` as custom properties on `scope`, with
|
|
@@ -10,9 +10,11 @@ module ActiveAgent
|
|
|
10
10
|
module Evals
|
|
11
11
|
# Publishes a completed report without replaying the agent. The caller must
|
|
12
12
|
# retain run_id when retrying: compatible collectors treat that identity as
|
|
13
|
-
# immutable within the authenticated account
|
|
14
|
-
# not follow redirects with the account's
|
|
15
|
-
# the collector whether it is up and
|
|
13
|
+
# immutable within the authenticated account once the report is complete.
|
|
14
|
+
# Delivery is blocking and does not follow redirects with the account's
|
|
15
|
+
# bearer credential. +verify!+ asks the collector whether it is up and
|
|
16
|
+
# accepts the key before a run is paid for, and +progress+ shows a run's
|
|
17
|
+
# results so far while it is still running.
|
|
16
18
|
#
|
|
17
19
|
# Every failure to deliver raises Error. Invalid arguments raise
|
|
18
20
|
# ArgumentError before anything is sent.
|
|
@@ -94,19 +96,35 @@ module ActiveAgent
|
|
|
94
96
|
# report may be a Report or its saved JSON hash. Full prompts, answers and
|
|
95
97
|
# tool results are included; applications should make publication opt-in.
|
|
96
98
|
def call(report:, run_id:, source:, agent_name:, suite:)
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
deliver("retain the report and run_id for retry") do
|
|
101
|
-
response = post(body)
|
|
102
|
-
raise rejection(response) unless %w[200 201].include?(response.code)
|
|
99
|
+
envelope = self.class.envelope(report: report, run_id: run_id, source: source, agent_name: agent_name, suite: suite)
|
|
100
|
+
publish(envelope, accepted: %w[complete], receipt: "completion receipt")
|
|
101
|
+
end
|
|
103
102
|
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
103
|
+
# Returns the collector's receipt for the results +report+ holds so far,
|
|
104
|
+
# which the collector shows as a running run under +run_id+. Each call
|
|
105
|
+
# replaces the results the previous one sent; +call+ with the finished
|
|
106
|
+
# report under the same run_id completes the run. The receipt's status
|
|
107
|
+
# is "running", or "complete" when the run already finished.
|
|
108
|
+
#
|
|
109
|
+
# Raises Error, without sending the report, when the collector does not
|
|
110
|
+
# list "progress" among its capabilities (see +supports_progress?+): a
|
|
111
|
+
# collector that predates progress would store the partial report as
|
|
112
|
+
# complete.
|
|
113
|
+
def progress(report:, run_id:, source:, agent_name:, suite:)
|
|
114
|
+
unless supports_progress?
|
|
115
|
+
raise Error, "Evaluation collector does not accept progress; publish the finished report with call"
|
|
109
116
|
end
|
|
117
|
+
|
|
118
|
+
envelope = self.class.envelope(report: report, run_id: run_id, source: source, agent_name: agent_name, suite: suite, status: "running")
|
|
119
|
+
publish(envelope, accepted: %w[running complete], receipt: "progress receipt")
|
|
120
|
+
end
|
|
121
|
+
|
|
122
|
+
# Returns true when the collector accepts +progress+. Asks it once, with
|
|
123
|
+
# the probe +verify!+ sends, unless +verify!+ already has; raises as
|
|
124
|
+
# +verify!+ does.
|
|
125
|
+
def supports_progress?
|
|
126
|
+
verify! if @capabilities.nil?
|
|
127
|
+
@capabilities.include?("progress")
|
|
110
128
|
end
|
|
111
129
|
|
|
112
130
|
# Returns true when the collector is up and accepts the API key, without
|
|
@@ -117,6 +135,8 @@ module ActiveAgent
|
|
|
117
135
|
# a stopped collector or a refused key fails before the first model call
|
|
118
136
|
# rather than after the last one.
|
|
119
137
|
#
|
|
138
|
+
# The refusal's +capabilities+ is what +supports_progress?+ reads.
|
|
139
|
+
#
|
|
120
140
|
# Raises Error for anything else: a rejection other than 422, with the
|
|
121
141
|
# status, detail and guidance +call+ would carry (401 for a refused key,
|
|
122
142
|
# 404 when nothing at the endpoint takes reports); a retryable delivery
|
|
@@ -128,7 +148,10 @@ module ActiveAgent
|
|
|
128
148
|
response = post("{}")
|
|
129
149
|
if response.code == "422"
|
|
130
150
|
detail = collector_detail(response.body)
|
|
131
|
-
|
|
151
|
+
if detail&.match?(/\bversion\b/)
|
|
152
|
+
@capabilities = collector_capabilities(response.body)
|
|
153
|
+
next true
|
|
154
|
+
end
|
|
132
155
|
|
|
133
156
|
raise Error.new("Evaluation collector answered HTTP 422 without refusing the empty envelope as a version-1 report, " \
|
|
134
157
|
"so it is not a compatible collector; check the endpoint", status: 422, detail: detail)
|
|
@@ -144,9 +167,12 @@ module ActiveAgent
|
|
|
144
167
|
# collector's export of a stored run returns, so the export can be
|
|
145
168
|
# posted again. report may be a Report or its saved JSON hash.
|
|
146
169
|
#
|
|
170
|
+
# `status: "running"` marks the envelope +progress+ sends; a complete
|
|
171
|
+
# report's envelope carries no status.
|
|
172
|
+
#
|
|
147
173
|
# Raises ArgumentError for an identity that is not a nonempty string,
|
|
148
174
|
# or a report that is neither.
|
|
149
|
-
def self.envelope(report:, run_id:, source:, agent_name:, suite:)
|
|
175
|
+
def self.envelope(report:, run_id:, source:, agent_name:, suite:, status: nil)
|
|
150
176
|
identities = { "run_id" => run_id, "source" => source, "agent_name" => agent_name, "suite" => suite }
|
|
151
177
|
identities.each do |key, value|
|
|
152
178
|
raise ArgumentError, "#{key} must be a nonempty string" unless value.is_a?(String) && !value.strip.empty?
|
|
@@ -154,11 +180,40 @@ module ActiveAgent
|
|
|
154
180
|
hash = report.to_h if report.respond_to?(:to_h) && !report.nil? && !report.is_a?(Array)
|
|
155
181
|
raise ArgumentError, "report must be a Report or its saved JSON hash" unless hash.is_a?(Hash)
|
|
156
182
|
|
|
157
|
-
identities.merge("version" => 1, "report" => hash)
|
|
183
|
+
envelope = identities.merge("version" => 1, "report" => hash)
|
|
184
|
+
status ? envelope.merge("status" => status) : envelope
|
|
158
185
|
end
|
|
159
186
|
|
|
160
187
|
private
|
|
161
188
|
|
|
189
|
+
# Delivers +envelope+ and returns the collector's receipt, which must name
|
|
190
|
+
# the envelope's run_id, a stored run, and one of the +accepted+ statuses.
|
|
191
|
+
def publish(envelope, accepted:, receipt:)
|
|
192
|
+
body = encode(envelope)
|
|
193
|
+
raise Error, "Evaluation report exceeds the 2 MiB delivery limit; publish a smaller selection" if body.bytesize > MAX_BYTES
|
|
194
|
+
|
|
195
|
+
deliver("retain the report and run_id for retry") do
|
|
196
|
+
response = post(body)
|
|
197
|
+
raise rejection(response) unless %w[200 201].include?(response.code)
|
|
198
|
+
|
|
199
|
+
answer = JSON.parse(response.body.to_s)
|
|
200
|
+
unless answer.is_a?(Hash) && answer["run_id"] == envelope["run_id"] && accepted.include?(answer["status"]) &&
|
|
201
|
+
answer["id"] && answer["evaluation_id"]
|
|
202
|
+
raise Error.new("Evaluation collector returned an invalid #{receipt}; retain the report and run_id for retry", retryable: true)
|
|
203
|
+
end
|
|
204
|
+
answer
|
|
205
|
+
end
|
|
206
|
+
end
|
|
207
|
+
|
|
208
|
+
# The string entries of a JSON object body's +capabilities+ array, or none.
|
|
209
|
+
def collector_capabilities(body)
|
|
210
|
+
parsed = JSON.parse(body.to_s)
|
|
211
|
+
list = parsed["capabilities"] if parsed.is_a?(Hash)
|
|
212
|
+
list.is_a?(Array) ? list.grep(String).freeze : [].freeze
|
|
213
|
+
rescue JSON::ParserError, EncodingError
|
|
214
|
+
[].freeze
|
|
215
|
+
end
|
|
216
|
+
|
|
162
217
|
# Posts +body+ to the endpoint with the bearer key and the delivery timeouts.
|
|
163
218
|
def post(body)
|
|
164
219
|
http = Net::HTTP.new(@uri.hostname, @uri.port)
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
<!DOCTYPE html>
|
|
2
|
+
<html lang="en">
|
|
3
|
+
<head>
|
|
4
|
+
<meta charset="utf-8">
|
|
5
|
+
<meta name="viewport" content="width=device-width, initial-scale=1">
|
|
6
|
+
<title>ActiveAgent Previews</title>
|
|
7
|
+
<style>
|
|
8
|
+
body { color-scheme: light dark; font: 16px/1.5 system-ui, sans-serif; max-width: 1000px; padding: 1rem; margin: auto; }
|
|
9
|
+
pre { white-space: pre-wrap; overflow-wrap: anywhere; border: 1px solid; border-radius: .25rem; padding: 1rem; }
|
|
10
|
+
a { text-underline-offset: .2em; }
|
|
11
|
+
</style>
|
|
12
|
+
</head>
|
|
13
|
+
<body>
|
|
14
|
+
<nav><%= link_to "All agent previews", rails_agent_previews_path %></nav>
|
|
15
|
+
<%= yield %>
|
|
16
|
+
</body>
|
|
17
|
+
</html>
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
<h1>Agent Previews</h1>
|
|
2
|
+
<% if @previews.empty? %>
|
|
3
|
+
<p>No previews found. Generate an agent or add an ActiveAgent::Preview subclass under test/agents/previews.</p>
|
|
4
|
+
<% end %>
|
|
5
|
+
<% @previews.each do |preview| %>
|
|
6
|
+
<section>
|
|
7
|
+
<h2><%= preview.preview_name %></h2>
|
|
8
|
+
<ul>
|
|
9
|
+
<% preview.prompts.each do |prompt| %>
|
|
10
|
+
<li><%= link_to prompt, rails_agent_preview_path("#{preview.preview_name}/#{prompt}") %></li>
|
|
11
|
+
<% end %>
|
|
12
|
+
</ul>
|
|
13
|
+
</section>
|
|
14
|
+
<% end %>
|
data/lib/active_agent/railtie.rb
CHANGED
|
@@ -6,6 +6,8 @@ require "active_agent"
|
|
|
6
6
|
require "active_agent/railtie/schema_generator_extension"
|
|
7
7
|
require "abstract_controller/railties/routes_helpers"
|
|
8
8
|
|
|
9
|
+
Rails.autoload :AgentsController, "rails/agents_controller"
|
|
10
|
+
|
|
9
11
|
module ActiveAgent
|
|
10
12
|
class Railtie < Rails::Railtie # :nodoc:
|
|
11
13
|
config.active_agent = ActiveSupport::OrderedOptions.new
|
|
@@ -101,8 +103,12 @@ module ActiveAgent
|
|
|
101
103
|
end
|
|
102
104
|
|
|
103
105
|
initializer "active_agent.set_autoload_paths", before: :set_autoload_paths do |app|
|
|
104
|
-
|
|
105
|
-
#
|
|
106
|
+
options = app.config.active_agent
|
|
107
|
+
options.preview_paths |= [ "#{app.root}/test/agents/previews", "#{app.root}/test/docs/previews" ]
|
|
108
|
+
# This runs before set_configs assigns show_previews its development default.
|
|
109
|
+
if options.show_previews.nil? ? Rails.env.development? : options.show_previews
|
|
110
|
+
app.config.autoload_paths.concat(options.preview_paths)
|
|
111
|
+
end
|
|
106
112
|
end
|
|
107
113
|
|
|
108
114
|
initializer "active_agent.compile_config_methods" do
|
|
@@ -132,9 +138,9 @@ module ActiveAgent
|
|
|
132
138
|
|
|
133
139
|
if options.show_previews
|
|
134
140
|
app.routes.prepend do
|
|
135
|
-
get "/rails/agents" => "rails/agents#index", :internal => true
|
|
136
|
-
get "/rails/agents/download/*path" => "rails/agents#download", :internal => true
|
|
137
|
-
get "/rails/agents/*path" => "rails/agents#preview", :internal => true
|
|
141
|
+
get "/rails/agents" => "rails/agents#index", :internal => true, :as => :rails_agent_previews
|
|
142
|
+
get "/rails/agents/download/*path" => "rails/agents#download", :internal => true, :as => :download_rails_agent_preview, :format => false
|
|
143
|
+
get "/rails/agents/*path" => "rails/agents#preview", :internal => true, :as => :rails_agent_preview, :format => false
|
|
138
144
|
end
|
|
139
145
|
end
|
|
140
146
|
end
|
data/lib/active_agent/version.rb
CHANGED
|
@@ -17,7 +17,7 @@ module TestUnit # :nodoc:
|
|
|
17
17
|
end
|
|
18
18
|
|
|
19
19
|
def create_preview_files
|
|
20
|
-
template "preview.rb", File.join("test/
|
|
20
|
+
template "preview.rb", File.join("test/agents/previews", class_path, "#{file_name}_agent_preview.rb")
|
|
21
21
|
end
|
|
22
22
|
|
|
23
23
|
private
|
|
@@ -1,13 +1,13 @@
|
|
|
1
1
|
<% module_namespacing do -%>
|
|
2
|
-
# Preview all agent views/prompts templates at http://localhost:3000/
|
|
2
|
+
# Preview all agent views/prompts templates at http://localhost:3000/rails/agents/<%= file_path %>_agent
|
|
3
3
|
class <%= class_name %>AgentPreview < ActiveAgent::Preview
|
|
4
4
|
<% actions.each_with_index do |action, index| -%>
|
|
5
5
|
<% if index != 0 -%>
|
|
6
6
|
|
|
7
7
|
<% end -%>
|
|
8
|
-
# Preview this
|
|
8
|
+
# Preview this prompt at http://localhost:3000/rails/agents/<%= file_path %>_agent/<%= action %>
|
|
9
9
|
def <%= action %>
|
|
10
|
-
<%= class_name %>Agent.<%= action %>
|
|
10
|
+
<%= class_name %>Agent.with(params).<%= action %>
|
|
11
11
|
end
|
|
12
12
|
<% end -%>
|
|
13
13
|
end
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "rails/application_controller"
|
|
4
|
+
|
|
5
|
+
class Rails::AgentsController < Rails::ApplicationController # :nodoc:
|
|
6
|
+
prepend_view_path File.expand_path("../active_agent/preview/templates", __dir__)
|
|
7
|
+
layout "active_agent/preview"
|
|
8
|
+
before_action :require_local!
|
|
9
|
+
|
|
10
|
+
def index
|
|
11
|
+
@previews = ActiveAgent::Preview.all
|
|
12
|
+
end
|
|
13
|
+
|
|
14
|
+
def preview
|
|
15
|
+
if found = ActiveAgent::Preview.find(params[:path])
|
|
16
|
+
@previews = [ found ]
|
|
17
|
+
render :index
|
|
18
|
+
elsif load_preview
|
|
19
|
+
render :preview
|
|
20
|
+
end
|
|
21
|
+
end
|
|
22
|
+
|
|
23
|
+
def download
|
|
24
|
+
return unless load_preview
|
|
25
|
+
|
|
26
|
+
send_data @content, type: "text/markdown", disposition: "attachment",
|
|
27
|
+
filename: "#{@preview.preview_name.tr('/', '-')}-#{@prompt}.md"
|
|
28
|
+
end
|
|
29
|
+
|
|
30
|
+
private
|
|
31
|
+
|
|
32
|
+
def load_preview
|
|
33
|
+
name, _, @prompt = params[:path].to_s.rpartition("/")
|
|
34
|
+
@preview = ActiveAgent::Preview.find(name)
|
|
35
|
+
unless @preview&.prompt_exists?(@prompt)
|
|
36
|
+
head :not_found
|
|
37
|
+
return false
|
|
38
|
+
end
|
|
39
|
+
|
|
40
|
+
@content = @preview.call(@prompt, request.query_parameters).prompt_preview
|
|
41
|
+
true
|
|
42
|
+
end
|
|
43
|
+
end
|