actionagent 1.9.8 → 1.9.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 5f8ad45415f0faa3e84cd38c32ceeb8f1b59a2240b5a9bbcced702a2ddd6d246
|
|
4
|
+
data.tar.gz: 5e8315eca89021eb8a54abb1282f6c74721eff39c380e14bcadd8904930c70b2
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 65b3d15741034542a8f72a99022da4bc79bd0c6ef510a75482453b02c069f96b6085d20a1c27aa0a14eaa77b1a9c629d4d53752974664aa60e12beff90fb7c33
|
|
7
|
+
data.tar.gz: a6bc5006b281eaed9717ce3dbb6225b6e005ef81b3e6a37b29463a9b55f963c526fb5ab8734d6f5961d99aeaa07eef23fde6d37c11a0ce2f1c59b48daabe8055
|
|
@@ -8,7 +8,9 @@ module ActionAgent
|
|
|
8
8
|
#
|
|
9
9
|
# A failure outside the replay is stored as the pair's errored result
|
|
10
10
|
# rather than raised, so the run still completes; a scenario removed from
|
|
11
|
-
# the suite during the run is left out of it.
|
|
11
|
+
# the suite during the run is left out of it. The same runner records the
|
|
12
|
+
# failure, since it knows whether this delivery claimed the pair and
|
|
13
|
+
# stored its result (see ScenarioEvaluationRunner#fail_result!).
|
|
12
14
|
class EvaluationReplayJob < ApplicationJob
|
|
13
15
|
queue_as :agents
|
|
14
16
|
|
|
@@ -21,7 +23,7 @@ module ActionAgent
|
|
|
21
23
|
runner.replay_scenario!(run, scenario_id, spec)
|
|
22
24
|
rescue StandardError => e
|
|
23
25
|
Rails.logger.warn("[ActionAgent] evaluation run #{run_id}: replay of scenario #{scenario_id} failed: #{e.class}: #{e.message}")
|
|
24
|
-
|
|
26
|
+
runner.fail_result!(run.reload, scenario_id, spec, e)
|
|
25
27
|
end
|
|
26
28
|
end
|
|
27
29
|
end
|
|
@@ -3,7 +3,9 @@
|
|
|
3
3
|
module ActionAgent
|
|
4
4
|
# Executes an evaluation run created by Evaluation#run_later!. The run
|
|
5
5
|
# record already exists (pending), so the dashboard can show it while the
|
|
6
|
-
# job is queued; the runner
|
|
6
|
+
# job is queued; the runner claims it as running, then marks it complete
|
|
7
|
+
# or failed. A job delivered again finds the run claimed and replays
|
|
8
|
+
# nothing, even when two workers load it as pending at once.
|
|
7
9
|
class EvaluationRunJob < ApplicationJob
|
|
8
10
|
queue_as :agents
|
|
9
11
|
|
|
@@ -47,6 +47,7 @@ module ActionAgent
|
|
|
47
47
|
class ScenarioEvaluationRunner < EvaluationRunnerService
|
|
48
48
|
Evals = ActiveAgent::Evals
|
|
49
49
|
JUDGE_LEASE_DURATION = 30.minutes
|
|
50
|
+
REPLAY_LEASE_DURATION = 30.minutes
|
|
50
51
|
|
|
51
52
|
# The providers a candidate model name resolves against. `mock` is the
|
|
52
53
|
# framework's test double, accepted so the test suite can compare cohorts
|
|
@@ -82,10 +83,15 @@ module ActionAgent
|
|
|
82
83
|
end
|
|
83
84
|
|
|
84
85
|
def call
|
|
86
|
+
pending = @run || @evaluation.evaluation_runs.create!(status: :pending)
|
|
87
|
+
# Another worker started this run (a job delivered twice, or handed to
|
|
88
|
+
# two workers): it is theirs to finish, so nothing here writes to it.
|
|
89
|
+
return pending.reload unless claim!(pending)
|
|
90
|
+
|
|
91
|
+
run = pending
|
|
85
92
|
scenarios = selected_scenarios
|
|
86
93
|
specs = model_specs
|
|
87
|
-
run
|
|
88
|
-
run.update!(status: :running, selection: selection_summary(scenarios, specs))
|
|
94
|
+
run.update!(selection: selection_summary(scenarios, specs))
|
|
89
95
|
record_pass_thresholds!(run)
|
|
90
96
|
ensure_sandbox_live!
|
|
91
97
|
|
|
@@ -173,6 +179,13 @@ module ActionAgent
|
|
|
173
179
|
# Replays one scenario under one model and stores the answer as a pending
|
|
174
180
|
# result, then enqueues its scoring. A result already stored for the
|
|
175
181
|
# pair (a retried job) is not replayed again.
|
|
182
|
+
#
|
|
183
|
+
# The replay is claimed before the agent runs, so a second delivery of
|
|
184
|
+
# the job (a queue redelivering it, or handing it to two workers) does
|
|
185
|
+
# not call the provider and the agent's tools a second time. A delivery
|
|
186
|
+
# that finds the pair claimed comes back when the claim expires (see
|
|
187
|
+
# #claim_replay!): by then the owner has stored the result, or stopped
|
|
188
|
+
# and left the pair to it.
|
|
176
189
|
def replay_scenario!(run, scenario_id, spec_value)
|
|
177
190
|
return unless run.running?
|
|
178
191
|
|
|
@@ -180,9 +193,21 @@ module ActionAgent
|
|
|
180
193
|
spec = model_spec_from(spec_value)
|
|
181
194
|
row = stored_result(run, scenario, spec)
|
|
182
195
|
unless row
|
|
196
|
+
key = replay_key(scenario, spec)
|
|
197
|
+
held_until = claim_replay!(run, key)
|
|
198
|
+
if held_until
|
|
199
|
+
EvaluationReplayJob.set(wait_until: held_until).perform_later(run.id, scenario.id, spec_value, @selection.to_h)
|
|
200
|
+
return
|
|
201
|
+
end
|
|
202
|
+
return unless @replay_token # the run stopped running
|
|
203
|
+
|
|
183
204
|
task = Evals::Scenario.from_hash(scenario.as_json_summary)
|
|
184
205
|
replay = fan_out_replay(task, spec)
|
|
185
|
-
row = run.with_lock
|
|
206
|
+
row = run.with_lock do
|
|
207
|
+
stored = stored_result(run, scenario, spec) || store_replay(run, scenario, task, spec, replay).tap { |created| @stored_result_id = created.id }
|
|
208
|
+
release_replay!(run, key)
|
|
209
|
+
stored
|
|
210
|
+
end
|
|
186
211
|
end
|
|
187
212
|
EvaluationJudgeJob.perform_later(row.id, @selection.to_h, run.id) if row.pending?
|
|
188
213
|
row
|
|
@@ -227,6 +252,12 @@ module ActionAgent
|
|
|
227
252
|
# replay (which records its own errors), so the run still completes. A
|
|
228
253
|
# scenario removed from the suite during the run has no result to
|
|
229
254
|
# store; the run completes without it (see #finish_fan_out!).
|
|
255
|
+
#
|
|
256
|
+
# Only this delivery's own pair is failed: one with no result yet and
|
|
257
|
+
# no other delivery replaying it, or the result this runner stored. A
|
|
258
|
+
# result another delivery stored is that delivery's to score, and a
|
|
259
|
+
# pair another delivery has claimed is its to store, so a duplicate
|
|
260
|
+
# delivery's failure cannot turn a good replay into an error.
|
|
230
261
|
def fail_result!(run, scenario_id, spec_value, error)
|
|
231
262
|
return unless run.running?
|
|
232
263
|
|
|
@@ -235,8 +266,15 @@ module ActionAgent
|
|
|
235
266
|
spec = model_spec_from(spec_value)
|
|
236
267
|
run.with_lock do
|
|
237
268
|
row = stored_result(run, scenario, spec)
|
|
238
|
-
|
|
239
|
-
|
|
269
|
+
if row.nil?
|
|
270
|
+
next if replay_claimed_elsewhere?(run, replay_key(scenario, spec))
|
|
271
|
+
|
|
272
|
+
task = Evals::Scenario.from_hash(scenario.as_json_summary)
|
|
273
|
+
row = store_replay(run, scenario, task, spec, Evals::Replay.failed(error))
|
|
274
|
+
@stored_result_id = row.id
|
|
275
|
+
end
|
|
276
|
+
next unless row.id == @stored_result_id
|
|
277
|
+
|
|
240
278
|
attributes = { status: :errored, error_message: row.error_message.presence || error_text(error) }
|
|
241
279
|
if scoring_leases?
|
|
242
280
|
# Another replay delivery may fail while this result is being
|
|
@@ -291,6 +329,24 @@ module ActionAgent
|
|
|
291
329
|
|
|
292
330
|
private
|
|
293
331
|
|
|
332
|
+
# Moves the run from pending to running in one conditional update, as
|
|
333
|
+
# AgentExecutionJob#claim_pending claims an agent run, and returns
|
|
334
|
+
# whether this runner is the one that did. Of two workers that loaded
|
|
335
|
+
# the run as pending, only the first to write replays the suite (or fans
|
|
336
|
+
# it out); the other replays nothing and leaves the run alone. The claim
|
|
337
|
+
# comes before the selection is read, so a selection that cannot be
|
|
338
|
+
# resolved fails the run rather than leaving it pending. Nothing is held
|
|
339
|
+
# over the replays.
|
|
340
|
+
def claim!(run)
|
|
341
|
+
now = Time.current
|
|
342
|
+
claimed = EvaluationRun.where(id: run.id, status: EvaluationRun.statuses[:pending])
|
|
343
|
+
.update_all(status: EvaluationRun.statuses[:running], updated_at: now)
|
|
344
|
+
return false if claimed.zero?
|
|
345
|
+
|
|
346
|
+
run.reload
|
|
347
|
+
true
|
|
348
|
+
end
|
|
349
|
+
|
|
294
350
|
def fan_out?(adapter)
|
|
295
351
|
ActionAgent.scenario_evaluation_fan_out == true && adapter.nil? && !browser_requested? &&
|
|
296
352
|
Evals::Runner.instance_method(:initialize).parameters.any? { |_type, name| name == :refine_gate }
|
|
@@ -404,6 +460,70 @@ module ActionAgent
|
|
|
404
460
|
(%w[scoring_token scoring_expires_at] - EvaluationScenarioResult.column_names).empty?
|
|
405
461
|
end
|
|
406
462
|
|
|
463
|
+
# A pair's replay claim, kept with the run's fan-out bookkeeping
|
|
464
|
+
# (scores["_fan_out"]["replays"]) until the replay is stored, so no
|
|
465
|
+
# migration is needed. The result row cannot hold it: the row is
|
|
466
|
+
# created only once there is an answer to store, and nothing makes a
|
|
467
|
+
# pair's row unique to create it earlier.
|
|
468
|
+
def replay_key(scenario, spec)
|
|
469
|
+
"#{scenario.id}:#{spec.provider}/#{spec.model}"
|
|
470
|
+
end
|
|
471
|
+
|
|
472
|
+
# Claims the pair's replay for this runner (its token in @replay_token)
|
|
473
|
+
# and returns nil, or returns when another delivery's live claim
|
|
474
|
+
# expires. An expired claim is taken over: its worker stopped, or is
|
|
475
|
+
# still replaying after REPLAY_LEASE_DURATION, in which case the two
|
|
476
|
+
# replays overlap and the first to store its answer is kept. A run no
|
|
477
|
+
# longer running claims nothing (@replay_token stays nil).
|
|
478
|
+
#
|
|
479
|
+
# This holds one active replay per pair across ordinary duplicate
|
|
480
|
+
# deliveries. It is not exactly once: a worker that stops after the
|
|
481
|
+
# provider answered and before the result is stored is replayed again.
|
|
482
|
+
def claim_replay!(run, key)
|
|
483
|
+
@replay_token = nil
|
|
484
|
+
token = SecureRandom.uuid
|
|
485
|
+
held_until = nil
|
|
486
|
+
update_fan_out_state!(run) do |state|
|
|
487
|
+
now = Time.current
|
|
488
|
+
replays = state["replays"].is_a?(Hash) ? state["replays"] : {}
|
|
489
|
+
expires_at = replay_expiry(replays[key])
|
|
490
|
+
if expires_at && expires_at > now
|
|
491
|
+
held_until = expires_at
|
|
492
|
+
next state
|
|
493
|
+
end
|
|
494
|
+
|
|
495
|
+
@replay_token = token
|
|
496
|
+
state.merge("replays" => replays.merge(key => { "token" => token, "expires_at" => (now + REPLAY_LEASE_DURATION).iso8601(6) }))
|
|
497
|
+
end
|
|
498
|
+
held_until
|
|
499
|
+
end
|
|
500
|
+
|
|
501
|
+
# Drops this runner's claim once its replay is stored. A claim another
|
|
502
|
+
# delivery took over is left to that delivery. Called under the run's
|
|
503
|
+
# lock, after the store.
|
|
504
|
+
def release_replay!(run, key)
|
|
505
|
+
return unless run.running?
|
|
506
|
+
|
|
507
|
+
state = fan_out_state(run)
|
|
508
|
+
replays = state["replays"].is_a?(Hash) ? state["replays"] : {}
|
|
509
|
+
return unless @replay_token && replays.dig(key, "token") == @replay_token
|
|
510
|
+
|
|
511
|
+
scores = run.scores.is_a?(Hash) ? run.scores : {}
|
|
512
|
+
run.update!(scores: scores.merge("_fan_out" => state.merge("replays" => replays.except(key))))
|
|
513
|
+
end
|
|
514
|
+
|
|
515
|
+
def replay_claimed_elsewhere?(run, key)
|
|
516
|
+
claim = fan_out_state(run)["replays"].to_h[key]
|
|
517
|
+
expires_at = replay_expiry(claim)
|
|
518
|
+
expires_at.present? && expires_at > Time.current && claim["token"] != @replay_token
|
|
519
|
+
end
|
|
520
|
+
|
|
521
|
+
def replay_expiry(claim)
|
|
522
|
+
Time.iso8601(claim["expires_at"].to_s) if claim.is_a?(Hash)
|
|
523
|
+
rescue ArgumentError
|
|
524
|
+
nil
|
|
525
|
+
end
|
|
526
|
+
|
|
407
527
|
# What Evals::Runner does with a replay that raises: an errored replay,
|
|
408
528
|
# scored like any other.
|
|
409
529
|
def fan_out_replay(task, spec)
|
data/lib/action_agent/version.rb
CHANGED
metadata
CHANGED
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: actionagent
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 1.9.
|
|
4
|
+
version: 1.9.9
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Justin Bowen
|
|
8
8
|
autorequire:
|
|
9
9
|
bindir: bin
|
|
10
10
|
cert_chain: []
|
|
11
|
-
date: 2026-11-
|
|
11
|
+
date: 2026-11-19 00:00:00.000000000 Z
|
|
12
12
|
dependencies:
|
|
13
13
|
- !ruby/object:Gem::Dependency
|
|
14
14
|
name: activeagent
|