actionagent 1.9.8 → 1.9.9

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: ca1b54d9b2bfde62c92c8aac8b6667f64fa13a002d44585027fc3846dff1683d
4
- data.tar.gz: 173618e8827d584682a9a45e6444faaf709d86bb562fee05a4e7793aa3df9203
3
+ metadata.gz: 5f8ad45415f0faa3e84cd38c32ceeb8f1b59a2240b5a9bbcced702a2ddd6d246
4
+ data.tar.gz: 5e8315eca89021eb8a54abb1282f6c74721eff39c380e14bcadd8904930c70b2
5
5
  SHA512:
6
- metadata.gz: 749f555544301b7018f665eab994088fce59757fbaa69ae142f81fd93d52411d47733030acca8527e4ad2adbae6ebb2b8d8d9eab763418fe4d176f8c9c85129c
7
- data.tar.gz: 9edd3988cb8695981eb80432a77973b24fde33fb8a02dd63942eb89f199f2ad4c1825ef3f09e59e43fd7cf10f6544f9bb3f5cfd193ea1f01d6fd75af08ac3ee1
6
+ metadata.gz: 65b3d15741034542a8f72a99022da4bc79bd0c6ef510a75482453b02c069f96b6085d20a1c27aa0a14eaa77b1a9c629d4d53752974664aa60e12beff90fb7c33
7
+ data.tar.gz: a6bc5006b281eaed9717ce3dbb6225b6e005ef81b3e6a37b29463a9b55f963c526fb5ab8734d6f5961d99aeaa07eef23fde6d37c11a0ce2f1c59b48daabe8055
@@ -8,7 +8,9 @@ module ActionAgent
8
8
  #
9
9
  # A failure outside the replay is stored as the pair's errored result
10
10
  # rather than raised, so the run still completes; a scenario removed from
11
- # the suite during the run is left out of it.
11
+ # the suite during the run is left out of it. The same runner records the
12
+ # failure, since it knows whether this delivery claimed the pair and
13
+ # stored its result (see ScenarioEvaluationRunner#fail_result!).
12
14
  class EvaluationReplayJob < ApplicationJob
13
15
  queue_as :agents
14
16
 
@@ -21,7 +23,7 @@ module ActionAgent
21
23
  runner.replay_scenario!(run, scenario_id, spec)
22
24
  rescue StandardError => e
23
25
  Rails.logger.warn("[ActionAgent] evaluation run #{run_id}: replay of scenario #{scenario_id} failed: #{e.class}: #{e.message}")
24
- ScenarioEvaluationRunner.new(run.evaluation, selection: selection, run: run).fail_result!(run.reload, scenario_id, spec, e)
26
+ runner.fail_result!(run.reload, scenario_id, spec, e)
25
27
  end
26
28
  end
27
29
  end
@@ -3,7 +3,9 @@
3
3
  module ActionAgent
4
4
  # Executes an evaluation run created by Evaluation#run_later!. The run
5
5
  # record already exists (pending), so the dashboard can show it while the
6
- # job is queued; the runner marks it running, then complete or failed.
6
+ # job is queued; the runner claims it as running, then marks it complete
7
+ # or failed. A job delivered again finds the run claimed and replays
8
+ # nothing, even when two workers load it as pending at once.
7
9
  class EvaluationRunJob < ApplicationJob
8
10
  queue_as :agents
9
11
 
@@ -47,6 +47,7 @@ module ActionAgent
47
47
  class ScenarioEvaluationRunner < EvaluationRunnerService
48
48
  Evals = ActiveAgent::Evals
49
49
  JUDGE_LEASE_DURATION = 30.minutes
50
+ REPLAY_LEASE_DURATION = 30.minutes
50
51
 
51
52
  # The providers a candidate model name resolves against. `mock` is the
52
53
  # framework's test double, accepted so the test suite can compare cohorts
@@ -82,10 +83,15 @@ module ActionAgent
82
83
  end
83
84
 
84
85
  def call
86
+ pending = @run || @evaluation.evaluation_runs.create!(status: :pending)
87
+ # Another worker started this run (a job delivered twice, or handed to
88
+ # two workers): it is theirs to finish, so nothing here writes to it.
89
+ return pending.reload unless claim!(pending)
90
+
91
+ run = pending
85
92
  scenarios = selected_scenarios
86
93
  specs = model_specs
87
- run = @run || @evaluation.evaluation_runs.create!(status: :pending)
88
- run.update!(status: :running, selection: selection_summary(scenarios, specs))
94
+ run.update!(selection: selection_summary(scenarios, specs))
89
95
  record_pass_thresholds!(run)
90
96
  ensure_sandbox_live!
91
97
 
@@ -173,6 +179,13 @@ module ActionAgent
173
179
  # Replays one scenario under one model and stores the answer as a pending
174
180
  # result, then enqueues its scoring. A result already stored for the
175
181
  # pair (a retried job) is not replayed again.
182
+ #
183
+ # The replay is claimed before the agent runs, so a second delivery of
184
+ # the job (a queue redelivering it, or handing it to two workers) does
185
+ # not call the provider and the agent's tools a second time. A delivery
186
+ # that finds the pair claimed comes back when the claim expires (see
187
+ # #claim_replay!): by then the owner has stored the result, or stopped
188
+ # and left the pair to it.
176
189
  def replay_scenario!(run, scenario_id, spec_value)
177
190
  return unless run.running?
178
191
 
@@ -180,9 +193,21 @@ module ActionAgent
180
193
  spec = model_spec_from(spec_value)
181
194
  row = stored_result(run, scenario, spec)
182
195
  unless row
196
+ key = replay_key(scenario, spec)
197
+ held_until = claim_replay!(run, key)
198
+ if held_until
199
+ EvaluationReplayJob.set(wait_until: held_until).perform_later(run.id, scenario.id, spec_value, @selection.to_h)
200
+ return
201
+ end
202
+ return unless @replay_token # the run stopped running
203
+
183
204
  task = Evals::Scenario.from_hash(scenario.as_json_summary)
184
205
  replay = fan_out_replay(task, spec)
185
- row = run.with_lock { stored_result(run, scenario, spec) || store_replay(run, scenario, task, spec, replay) }
206
+ row = run.with_lock do
207
+ stored = stored_result(run, scenario, spec) || store_replay(run, scenario, task, spec, replay).tap { |created| @stored_result_id = created.id }
208
+ release_replay!(run, key)
209
+ stored
210
+ end
186
211
  end
187
212
  EvaluationJudgeJob.perform_later(row.id, @selection.to_h, run.id) if row.pending?
188
213
  row
@@ -227,6 +252,12 @@ module ActionAgent
227
252
  # replay (which records its own errors), so the run still completes. A
228
253
  # scenario removed from the suite during the run has no result to
229
254
  # store; the run completes without it (see #finish_fan_out!).
255
+ #
256
+ # Only this delivery's own pair is failed: one with no result yet and
257
+ # no other delivery replaying it, or the result this runner stored. A
258
+ # result another delivery stored is that delivery's to score, and a
259
+ # pair another delivery has claimed is its to store, so a duplicate
260
+ # delivery's failure cannot turn a good replay into an error.
230
261
  def fail_result!(run, scenario_id, spec_value, error)
231
262
  return unless run.running?
232
263
 
@@ -235,8 +266,15 @@ module ActionAgent
235
266
  spec = model_spec_from(spec_value)
236
267
  run.with_lock do
237
268
  row = stored_result(run, scenario, spec)
238
- task = Evals::Scenario.from_hash(scenario.as_json_summary)
239
- row ||= store_replay(run, scenario, task, spec, Evals::Replay.failed(error))
269
+ if row.nil?
270
+ next if replay_claimed_elsewhere?(run, replay_key(scenario, spec))
271
+
272
+ task = Evals::Scenario.from_hash(scenario.as_json_summary)
273
+ row = store_replay(run, scenario, task, spec, Evals::Replay.failed(error))
274
+ @stored_result_id = row.id
275
+ end
276
+ next unless row.id == @stored_result_id
277
+
240
278
  attributes = { status: :errored, error_message: row.error_message.presence || error_text(error) }
241
279
  if scoring_leases?
242
280
  # Another replay delivery may fail while this result is being
@@ -291,6 +329,24 @@ module ActionAgent
291
329
 
292
330
  private
293
331
 
332
+ # Moves the run from pending to running in one conditional update, as
333
+ # AgentExecutionJob#claim_pending claims an agent run, and returns
334
+ # whether this runner is the one that did. Of two workers that loaded
335
+ # the run as pending, only the first to write replays the suite (or fans
336
+ # it out); the other replays nothing and leaves the run alone. The claim
337
+ # comes before the selection is read, so a selection that cannot be
338
+ # resolved fails the run rather than leaving it pending. Nothing is held
339
+ # over the replays.
340
+ def claim!(run)
341
+ now = Time.current
342
+ claimed = EvaluationRun.where(id: run.id, status: EvaluationRun.statuses[:pending])
343
+ .update_all(status: EvaluationRun.statuses[:running], updated_at: now)
344
+ return false if claimed.zero?
345
+
346
+ run.reload
347
+ true
348
+ end
349
+
294
350
  def fan_out?(adapter)
295
351
  ActionAgent.scenario_evaluation_fan_out == true && adapter.nil? && !browser_requested? &&
296
352
  Evals::Runner.instance_method(:initialize).parameters.any? { |_type, name| name == :refine_gate }
@@ -404,6 +460,70 @@ module ActionAgent
404
460
  (%w[scoring_token scoring_expires_at] - EvaluationScenarioResult.column_names).empty?
405
461
  end
406
462
 
463
+ # A pair's replay claim, kept with the run's fan-out bookkeeping
464
+ # (scores["_fan_out"]["replays"]) until the replay is stored, so no
465
+ # migration is needed. The result row cannot hold it: the row is
466
+ # created only once there is an answer to store, and nothing makes a
467
+ # pair's row unique to create it earlier.
468
+ def replay_key(scenario, spec)
469
+ "#{scenario.id}:#{spec.provider}/#{spec.model}"
470
+ end
471
+
472
+ # Claims the pair's replay for this runner (its token in @replay_token)
473
+ # and returns nil, or returns when another delivery's live claim
474
+ # expires. An expired claim is taken over: its worker stopped, or is
475
+ # still replaying after REPLAY_LEASE_DURATION, in which case the two
476
+ # replays overlap and the first to store its answer is kept. A run no
477
+ # longer running claims nothing (@replay_token stays nil).
478
+ #
479
+ # This holds one active replay per pair across ordinary duplicate
480
+ # deliveries. It is not exactly once: a worker that stops after the
481
+ # provider answered and before the result is stored is replayed again.
482
+ def claim_replay!(run, key)
483
+ @replay_token = nil
484
+ token = SecureRandom.uuid
485
+ held_until = nil
486
+ update_fan_out_state!(run) do |state|
487
+ now = Time.current
488
+ replays = state["replays"].is_a?(Hash) ? state["replays"] : {}
489
+ expires_at = replay_expiry(replays[key])
490
+ if expires_at && expires_at > now
491
+ held_until = expires_at
492
+ next state
493
+ end
494
+
495
+ @replay_token = token
496
+ state.merge("replays" => replays.merge(key => { "token" => token, "expires_at" => (now + REPLAY_LEASE_DURATION).iso8601(6) }))
497
+ end
498
+ held_until
499
+ end
500
+
501
+ # Drops this runner's claim once its replay is stored. A claim another
502
+ # delivery took over is left to that delivery. Called under the run's
503
+ # lock, after the store.
504
+ def release_replay!(run, key)
505
+ return unless run.running?
506
+
507
+ state = fan_out_state(run)
508
+ replays = state["replays"].is_a?(Hash) ? state["replays"] : {}
509
+ return unless @replay_token && replays.dig(key, "token") == @replay_token
510
+
511
+ scores = run.scores.is_a?(Hash) ? run.scores : {}
512
+ run.update!(scores: scores.merge("_fan_out" => state.merge("replays" => replays.except(key))))
513
+ end
514
+
515
+ def replay_claimed_elsewhere?(run, key)
516
+ claim = fan_out_state(run)["replays"].to_h[key]
517
+ expires_at = replay_expiry(claim)
518
+ expires_at.present? && expires_at > Time.current && claim["token"] != @replay_token
519
+ end
520
+
521
+ def replay_expiry(claim)
522
+ Time.iso8601(claim["expires_at"].to_s) if claim.is_a?(Hash)
523
+ rescue ArgumentError
524
+ nil
525
+ end
526
+
407
527
  # What Evals::Runner does with a replay that raises: an errored replay,
408
528
  # scored like any other.
409
529
  def fan_out_replay(task, spec)
@@ -1,5 +1,5 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  module ActionAgent
4
- VERSION = "1.9.8"
4
+ VERSION = "1.9.9"
5
5
  end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: actionagent
3
3
  version: !ruby/object:Gem::Version
4
- version: 1.9.8
4
+ version: 1.9.9
5
5
  platform: ruby
6
6
  authors:
7
7
  - Justin Bowen
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2026-11-12 00:00:00.000000000 Z
11
+ date: 2026-11-19 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: activeagent