@ai-dossier/sched 0.22.1 → 0.23.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +80 -13
- package/dist/batch-dispatch.d.ts +39 -1
- package/dist/batch-dispatch.d.ts.map +1 -1
- package/dist/batch-dispatch.js +157 -55
- package/dist/batch-dispatch.js.map +1 -1
- package/dist/engine.d.ts.map +1 -1
- package/dist/engine.js +205 -59
- package/dist/engine.js.map +1 -1
- package/dist/groundtruth.d.ts +40 -0
- package/dist/groundtruth.d.ts.map +1 -1
- package/dist/groundtruth.js +100 -5
- package/dist/groundtruth.js.map +1 -1
- package/dist/index.d.ts +2 -2
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +5 -3
- package/dist/index.js.map +1 -1
- package/dist/state.d.ts +18 -0
- package/dist/state.d.ts.map +1 -1
- package/dist/state.js +45 -9
- package/dist/state.js.map +1 -1
- package/dist/types.d.ts +49 -4
- package/dist/types.d.ts.map +1 -1
- package/dist/types.js +3 -2
- package/dist/types.js.map +1 -1
- package/package.json +1 -1
package/README.md
CHANGED
|
@@ -99,7 +99,8 @@ where every mechanical supervision decision is code, not remembered prose:
|
|
|
99
99
|
An unverified exit rides the recovery ladder like a stall. A `report done` milestone
|
|
100
100
|
must also postdate the slot's own `spawned_at` (±60s clock-skew tolerance, #575) — a
|
|
101
101
|
re-enqueued issue's PREVIOUS run's report milestone is ignored (journaled
|
|
102
|
-
`stale-milestone-ignored
|
|
102
|
+
`stale-milestone-ignored`, at most once per dispatch since #610) rather than instantly
|
|
103
|
+
completing a freshly-spawned agent on
|
|
103
104
|
its first reconcile tick; a legacy slot with no `spawned_at` degrades to the old,
|
|
104
105
|
unfenced check. Batch members get the same fence on their own completion signal
|
|
105
106
|
(`isMemberComplete`, `phase=review status=done mode=slot`).
|
|
@@ -115,7 +116,18 @@ where every mechanical supervision decision is code, not remembered prose:
|
|
|
115
116
|
pre-resolves every tier once per tick, so a mixed-CLI ladder rescues on a different
|
|
116
117
|
agent CLI, not just a different `--model` flag on the same one. Cap 2
|
|
117
118
|
escalations — or a stall at the strongest tier — fails the unit and blocks its
|
|
118
|
-
TRANSITIVE dependents (`dep-failed:<issue>`).
|
|
119
|
+
TRANSITIVE dependents (`dep-failed:<issue>`). One exception (#596): before failing
|
|
120
|
+
terminally for an UNVERIFIED EXIT (never a stall — a hung agent never had the chance
|
|
121
|
+
to open anything), the engine asks ground truth whether the slot's branch already
|
|
122
|
+
carries an open PR the milestone trail never recorded
|
|
123
|
+
(`GroundTruth.openPrForBranch` → `gh pr list --head <branch> --state open`). A
|
|
124
|
+
confirmed number parks the unit instead and the watcher owns it from there. It fails
|
|
125
|
+
closed: a report slot (no branch of its own), an unknown branch, an unusable payload
|
|
126
|
+
or an unreachable lookup all take the terminal path — only a confirmed PR the fleet
|
|
127
|
+
itself opened parks. Which of those it was is recorded on the `unit-failed` entry as
|
|
128
|
+
`pr_check=none|unreachable|no-branch`, so "we checked, there was nothing" is
|
|
129
|
+
distinguishable from "`gh` was down and we wrote off a unit whose PR may have been
|
|
130
|
+
mergeable" — the ambiguity that stranded imboard-monorepo#3999. The timeout is **phase-aware** (#495):
|
|
119
131
|
the `implement` phase alone can run 1-3h on a large monorepo with zero intermediate
|
|
120
132
|
milestone or pushed commit, so it gets a longer built-in default (90 min,
|
|
121
133
|
`DEFAULT_PHASE_STALL_TIMEOUT_MS`) than every other phase's 30-min default — selected by
|
|
@@ -229,6 +241,17 @@ Two engine-safety policies were explicit product decisions on #464:
|
|
|
229
241
|
milestone. The rejection is journaled as `stale-milestone-ignored`; `spawned_at=null`
|
|
230
242
|
(a legacy slot) degrades to the old, unfenced check.
|
|
231
243
|
|
|
244
|
+
#610: the event is emitted at most ONCE PER DISPATCH, on both rails. The decision is
|
|
245
|
+
stamped on `SlotEntry.stale_milestone_ignored_for` and compared against `spawned_at`
|
|
246
|
+
itself, so a redispatch's new `spawned_at` re-arms it for free — no reset at any spawn
|
|
247
|
+
site. Previously it re-fired every reconcile tick for as long as the stale milestone
|
|
248
|
+
stayed latest (~20 identical lines for a 40-minute unit, each forwarded to Telegram by
|
|
249
|
+
`tick.sh`, burying the events an operator is actually watching for). The entry's `at`
|
|
250
|
+
is the time the ENGINE made the decision and `milestone_at` is the ignored milestone's
|
|
251
|
+
own timestamp — one vocabulary for both emitters (`journalStaleMilestoneIfIgnored` in
|
|
252
|
+
`engine.ts`, `reconcileMemberSlot` in `batch-dispatch.ts`), so nothing reading
|
|
253
|
+
`events.jsonl` has to know which rail produced a line to know what `at` means.
|
|
254
|
+
|
|
232
255
|
This applies to `issue:<n>` unit dispatch (`dispatchAssignments`). `batch:<id>` units run
|
|
233
256
|
through a separate pass with its own claim/reconcile logic — see
|
|
234
257
|
[Batch dispatch (#523)](#batch-dispatch-523) below.
|
|
@@ -337,7 +360,12 @@ awaiting-merge (CONFLICTING | auto-merge-blocked)
|
|
|
337
360
|
[Batch dispatch (#523)](#batch-dispatch-523) — appends to `evictions[]`) is a no-op
|
|
338
361
|
for a repeat issue and journals `eviction-duplicate` instead; the repeat requeue is
|
|
339
362
|
skipped with it, so a second call cannot overwrite the first eviction's
|
|
340
|
-
`failure_evidence` or kill a live re-dispatch of that member.
|
|
363
|
+
`failure_evidence` or kill a live re-dispatch of that member. Since #613 the duplicate
|
|
364
|
+
is a no-op for the CALLER too, not just for the append: `evictMemberDirectly` returns
|
|
365
|
+
`{ dissolved, duplicate }` and, on `duplicate: true`, `evictMemberAndContinue` journals
|
|
366
|
+
no `unit-failed` and does not advance the batch — record and journal share one
|
|
367
|
+
lock-protected claim, so they can never disagree about which member the batch advanced
|
|
368
|
+
past. **Read
|
|
341
369
|
`docs/agent-traps.md` before "fixing" the dissolve rule:** the dissolve trigger has
|
|
342
370
|
counted DISTINCT member ids since #572 (`evictedMemberIds`) and was never inflatable
|
|
343
371
|
by a duplicate. What a duplicate did inflate is the raw `evictions[]` array itself —
|
|
@@ -371,8 +399,9 @@ awaiting-merge (CONFLICTING | auto-merge-blocked)
|
|
|
371
399
|
|
|
372
400
|
Thirteen journal events carry the detail: `suite-failed`, `attributed`, `fix-dispatched`,
|
|
373
401
|
`fix-resolved`, `member-evicted`, `eviction-duplicate` (#595 — a second eviction call
|
|
374
|
-
named a member already in `evictions[]`; the append
|
|
375
|
-
|
|
402
|
+
named a member already in `evictions[]`; the append, the requeue, the caller's own
|
|
403
|
+
`unit-failed` and the batch advance are ALL no-ops — since #613 the duplicate claim is the
|
|
404
|
+
single gate on all four — and the attempt is journaled rather than dropped), `revert-conflict`, `batch-rebased`,
|
|
376
405
|
`batch-dissolved`,
|
|
377
406
|
`batch-preserved` (#563 — the dissolve threshold was crossed but the survivors' re-run
|
|
378
407
|
suite came back green, so the batch ships them instead of dissolving), `batch-blocked`
|
|
@@ -549,6 +578,15 @@ never stored it on the entry, so this also retroactively covers the #562 case).
|
|
|
549
578
|
states migrate on load: no gate has ever produced a non-`ok` verdict, and no batch has
|
|
550
579
|
ever been blocked, under them, so `{}`/`null` is the exact backfill, not a guess.
|
|
551
580
|
|
|
581
|
+
Schema 1.12.0 (#610): `SlotEntry` gains `stale_milestone_ignored_for` (ISO string or
|
|
582
|
+
null — the `spawned_at` already covered by a `stale-milestone-ignored` journal entry, so
|
|
583
|
+
the event fires once per dispatch rather than once per reconcile tick). Held as the
|
|
584
|
+
timestamp rather than a boolean: a fresh dispatch stamps a NEW `spawned_at`, so the
|
|
585
|
+
marker goes stale automatically and needs no explicit reset at any spawn site. 1.11.0
|
|
586
|
+
states migrate on load, backfilling `null` — nothing was ever recorded per-dispatch
|
|
587
|
+
under the old once-per-tick behavior, so null is exact, not a guess. Resets with the
|
|
588
|
+
slot on release (`CLEARED_SLOT_FIELDS`).
|
|
589
|
+
|
|
552
590
|
New journal events: `batch-setup-done`, `batch-setup-failed`, `member-advanced`,
|
|
553
591
|
`batch-warmup-done`, `batch-warmup-failed` (#561 — the cold-path warm step only; a pool
|
|
554
592
|
claim emits neither). `gate-inconclusive` (#583 — the incremental gate came back
|
|
@@ -556,7 +594,12 @@ claim emits neither). `gate-inconclusive` (#583 — the incremental gate came ba
|
|
|
556
594
|
sits alongside `batch-blocked` as the per-member analogue of the aggregate suite's
|
|
557
595
|
"block, don't dissolve" precedent; #594 routes an unevidenced `task-failed` here too).
|
|
558
596
|
`eviction-duplicate` (#595 — `evictMemberDirectly` and `evictMembers` both emit it when
|
|
559
|
-
asked to evict a member already in `evictions[]
|
|
597
|
+
asked to evict a member already in `evictions[]`; since #613 it also suppresses the caller's
|
|
598
|
+
`unit-failed`/`member-advanced` pair and the member advance, so the pair is emitted exactly
|
|
599
|
+
once per member and names the member the batch is advancing FROM) and
|
|
600
|
+
`member-advance-skipped` (#613 — a resolution that lost the one-shot claim on
|
|
601
|
+
`executing_member` and so advanced nothing; journaled rather than dropped, so a batch that
|
|
602
|
+
stops advancing never does so silently). Member/tail/report/fix-agent spawn, progress,
|
|
560
603
|
completion and park events reuse the existing unit-generic names (`assigned`/`spawned`/`unit-failed`/
|
|
561
604
|
`external-advance`/`pr-parked`/`merge-accepted`/`report-dispatched`/`teardown-done`/
|
|
562
605
|
`teardown-failed`) with `unit = batch:<id>`.
|
|
@@ -617,6 +660,7 @@ import {
|
|
|
617
660
|
isParkedMilestone, // ship-phase awaiting-merge + pr= → the park signal
|
|
618
661
|
prOfMilestone, // a milestone's pr= key as a positive integer
|
|
619
662
|
parsePrViewJson, // gh pr view --json → PR truth (mergedAt/mergeable/blocked label)
|
|
663
|
+
parseOpenPrListJson, // gh pr list --head <b> --state open → the open PR we opened (#596)
|
|
620
664
|
parseSetupInfo, // gh issue view --json comments → teardown inputs
|
|
621
665
|
runTeardown, // #468 script teardown for a merged unit (pool return / worktree remove)
|
|
622
666
|
isSafeWorktree, // worktree-path containment check (CWE-22)
|
|
@@ -657,9 +701,13 @@ import {
|
|
|
657
701
|
Journal, // append-only events.jsonl
|
|
658
702
|
appendJsonl, // the shared mkdir+append+swallow JSONL write
|
|
659
703
|
transitionIssue, transitionBatch, transitionSlot, // typed §D transitions
|
|
704
|
+
patchSlot, patchBatch, // §D.3 METADATA patches without a status change — `id`/`status`
|
|
705
|
+
// excluded (and stripped at runtime): status goes through the
|
|
706
|
+
// typed rails above. patchSlot moved out of engine.ts in #610
|
|
707
|
+
// so batch-dispatch.ts can share it.
|
|
660
708
|
TRANSITIONS, // the transition tables themselves (for previews)
|
|
661
709
|
buildStatusReport, // machine-readable status incl. blocked/failed sets
|
|
662
|
-
validateState, // strict persisted-state validation (1.0.0-1.
|
|
710
|
+
validateState, // strict persisted-state validation (1.0.0-1.11.0 files migrate)
|
|
663
711
|
DEFAULT_ISSUE_PRIORITY, DEFAULT_BATCH_PRIORITY, // priority defaults (0 / 10, #565)
|
|
664
712
|
IllegalTransitionError, EnqueueError, CorruptStateError, LockTimeoutError,
|
|
665
713
|
SchedNotFoundError,
|
|
@@ -744,8 +792,16 @@ attributes to a concrete cause (e.g. `Monitor`) without opening the transcript.
|
|
|
744
792
|
non-terminal `verify-incomplete` event on every unverified exit and, once the escalation
|
|
745
793
|
ladder is exhausted, the terminal `unit-failed` (`agent-exited-unverified` /
|
|
746
794
|
`unverified-exit-at-strongest-tier`) as `last_tool`; a stall-timeout kill carries it too. Only
|
|
747
|
-
the dead-pid detection rail and the stall kill
|
|
748
|
-
|
|
795
|
+
the dead-pid detection rail and the stall kill RECORD a fresh log slice (a `runs.jsonl`
|
|
796
|
+
entry) in the same tick — `recordDispatchRunLog` is once-per-dispatch and refuses to
|
|
797
|
+
append a second entry over the same slice. Since #620 a slot already `exited`/`verifying`
|
|
798
|
+
when reconciled again still ATTRIBUTES one: `readLastToolForSlot` re-parses the same
|
|
799
|
+
static slice for the tool name alone, writing nothing, so it carries no exactly-once
|
|
800
|
+
constraint. That matters because the verify decision lands on a LATER tick than the
|
|
801
|
+
dead-pid detection whenever ground truth was unreachable in between — previously the
|
|
802
|
+
tool name was simply lost on exactly the runs hardest to diagnose. The read is deferred
|
|
803
|
+
behind a thunk so only the tick that reaches the unverified-exit decision pays for it,
|
|
804
|
+
never the ticks that return early on an outage.
|
|
749
805
|
|
|
750
806
|
The dispatch log (`runs/<unit>.log`) is per-UNIT and opened in append mode
|
|
751
807
|
(`createSpawnDeps`), so a redispatched unit's second agent writes its output AFTER the
|
|
@@ -815,14 +871,16 @@ after-the-fact recovery, not a missing-data bug.
|
|
|
815
871
|
queue data.
|
|
816
872
|
- **Schema**: state/config files from #460 (schema 1.0.0), #464 (1.1.0), #468 (1.2.0),
|
|
817
873
|
#472 (1.3.0), #500 (1.4.0), #505 (1.5.0), #504 (1.6.0), #523 (1.7.0) and #524 (1.8.0)
|
|
818
|
-
load and migrate to 1.
|
|
874
|
+
load and migrate to 1.12.0 automatically (slot `branch`/`last_head`/`pid_start`, slot `role` (inferred from the
|
|
819
875
|
unit's queue entry, with the persisted `phase` as a fallback — #500), entry
|
|
820
876
|
`pr`/`cleanup`/`failure_evidence`, batch `anchor`/`branch`/`run_id`/`eviction_groups`/
|
|
821
877
|
`evictions`/`fix_attempts`/`rebase_attempts`, state-level `last_pr_poll_at` backfill to
|
|
822
878
|
null, state-level `consecutive_suspect_dispatches`/`last_suspect_dispatch_unit`
|
|
823
879
|
backfill to `0`/`null` — #505, slot `gen`/`fenced_at` backfill to `0`/`null` — #504, and
|
|
824
|
-
slot `spawned_at`/`log_offset_at_spawn` backfill to `null`/`null` — #524,
|
|
825
|
-
state-level `last_label_poll_at` backfill to `null` — #544
|
|
880
|
+
slot `spawned_at`/`log_offset_at_spawn` backfill to `null`/`null` — #524,
|
|
881
|
+
state-level `last_label_poll_at` backfill to `null` — #544, entry `priority` and batch
|
|
882
|
+
`member_gates`/`blocked_reason` — #565/#583, and slot
|
|
883
|
+
`stale_milestone_ignored_for` backfill to `null` — #610).
|
|
826
884
|
- **`max_slots`** bounds live units (`assigned | running | recovering`); dependency
|
|
827
885
|
edges gate readiness — an issue with an unmerged dependency, and a batch behind an
|
|
828
886
|
unmerged batch, are never runnable.
|
|
@@ -891,6 +949,14 @@ prompt instructs it) and exit. The engine owns everything after the park:
|
|
|
891
949
|
phase's `awaiting-merge` (with `pr=`) is a VERIFIED park, not an unverified
|
|
892
950
|
exit: the entry moves to `parked`, the slot is released (a waiting unit
|
|
893
951
|
consumes zero slots), and the watcher takes over.
|
|
952
|
+
1b. **Recovery-adopted park (#596)** — a unit about to fail TERMINALLY for an
|
|
953
|
+
unverified exit, whose branch ground truth reports an open PR, parks on
|
|
954
|
+
that PR instead of dying as `unverified-exit-at-strongest-tier`. The
|
|
955
|
+
`pr-parked` event carries `detail: "unverified-exit-recovered-open-pr"`
|
|
956
|
+
and the `branch` the PR was found on, so a milestone-verified park (no
|
|
957
|
+
`detail`) and a recovery-adopted one are distinguishable in
|
|
958
|
+
`events.jsonl` without cross-referencing the ladder. Report slots are
|
|
959
|
+
excluded — a report agent has no branch of its own.
|
|
894
960
|
2. **PR watching (AC1)** — parked PRs are polled every `pr_poll_interval_ms`
|
|
895
961
|
(default 150 s — "every 2–3 min", persisted `last_pr_poll_at` so a restart
|
|
896
962
|
honors the cadence; checked on each reconcile tick when due, so a
|
|
@@ -947,7 +1013,8 @@ prompt instructs it) and exit. The engine owns everything after the park:
|
|
|
947
1013
|
|
|
948
1014
|
`sched status` shows parked PRs (zero slots, with the last poll's age), a
|
|
949
1015
|
`pr` column and a `cleanup` column on the queue; every watcher decision lands
|
|
950
|
-
in `events.jsonl` (`pr-parked
|
|
1016
|
+
in `events.jsonl` (`pr-parked` — two paths, see 1 and 1b above,
|
|
1017
|
+
`merge-accepted`, `pr-watch-failed`,
|
|
951
1018
|
`pr-watch-waiting`, `teardown-done`/`teardown-failed`, `report-dispatched`,
|
|
952
1019
|
`report-failed`, `ground-truth-unreachable`, `stale-failure-reconciled`).
|
|
953
1020
|
|
package/dist/batch-dispatch.d.ts
CHANGED
|
@@ -67,7 +67,7 @@ import type { SchedStore } from './persist';
|
|
|
67
67
|
import type { ExecFn } from './project';
|
|
68
68
|
import { type SuiteResult } from './recovery';
|
|
69
69
|
import { type FsExists } from './teardown';
|
|
70
|
-
import type { CapabilityGateResult, SchedConfig } from './types';
|
|
70
|
+
import type { BatchEntry, CapabilityGateResult, SchedConfig } from './types';
|
|
71
71
|
export type { CapOutcome } from './types';
|
|
72
72
|
/** Everything batch dispatch needs from the outside world. */
|
|
73
73
|
export interface BatchDispatchDeps {
|
|
@@ -128,6 +128,44 @@ export interface BatchTickResult {
|
|
|
128
128
|
/** Issue numbers requeued full-cycle by a dissolve — matches `TickResult.blocked`'s shape. */
|
|
129
129
|
blocked: number[];
|
|
130
130
|
}
|
|
131
|
+
/**
|
|
132
|
+
* Why a member is being evicted: the `reason` recorded in `evictions[]` and the operator-
|
|
133
|
+
* facing `detail`/`extraKv` journaled alongside it. One object rather than three adjacent
|
|
134
|
+
* positional arguments — `reason` and `detail` are both strings, and transposing them
|
|
135
|
+
* compiles cleanly while writing a prose sentence into the eviction record's `reason`,
|
|
136
|
+
* corrupting exactly the field #613 exists to keep trustworthy.
|
|
137
|
+
*/
|
|
138
|
+
export interface MemberFailure {
|
|
139
|
+
reason: string;
|
|
140
|
+
detail: string;
|
|
141
|
+
extraKv?: Record<string, string>;
|
|
142
|
+
}
|
|
143
|
+
/**
|
|
144
|
+
* Evict the current member and either dissolve, or continue the batch via
|
|
145
|
+
* `advanceMemberOrValidate` — the shared tail of both member-failure rails
|
|
146
|
+
* (self-reported blocked, and the incremental gate below).
|
|
147
|
+
*
|
|
148
|
+
* #613: `evictMemberDirectly`'s duplicate check (#595) is the one atomic,
|
|
149
|
+
* lock-protected claim on "did THIS member's eviction already happen" — so
|
|
150
|
+
* the `unit-failed` journal is emitted inside `evictMemberDirectly`, the
|
|
151
|
+
* moment that claim succeeds and BEFORE the dissolve that claim may trigger,
|
|
152
|
+
* rather than by each caller before it ever calls this function. A caller
|
|
153
|
+
* that journaled `unit-failed` unconditionally, before the claim, could fire
|
|
154
|
+
* it twice for one member (and once for the next member never at all) under
|
|
155
|
+
* a duplicate/re-entrant resolve — the record and the journal must share the
|
|
156
|
+
* same gate or they can disagree about which member the batch is advancing
|
|
157
|
+
* past. Emitting it inside also keeps cause before effect in the journal and
|
|
158
|
+
* survives a kill during the dissolve's shell-outs, which would otherwise
|
|
159
|
+
* leave the eviction record on disk with no line saying why.
|
|
160
|
+
*
|
|
161
|
+
* Exported (not part of the package's `index.ts` public surface — imported
|
|
162
|
+
* directly by `batch-integration.test.ts`) so #613's regression test can
|
|
163
|
+
* call it twice with one stale `BatchEntry` snapshot: the exact "another
|
|
164
|
+
* resolution already claimed this member" condition, which the public
|
|
165
|
+
* `runBatchTick`/`resumeBlockedGate` entry points cannot reproduce since
|
|
166
|
+
* both always read state fresh.
|
|
167
|
+
*/
|
|
168
|
+
export declare function evictMemberAndContinue(deps: BatchDispatchDeps, config: SchedConfig, dispatch: ResolvedDispatch, batchId: string, batch: BatchEntry, memberIssue: number, failure: MemberFailure, now: Date, result: BatchTickResult): void;
|
|
131
169
|
/**
|
|
132
170
|
* `sched resume --batch <id>` (#583 AC4): an operator-triggered, synchronous
|
|
133
171
|
* one-shot recheck of a batch blocked on `gate-inconclusive:<capabilityId>`
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"batch-dispatch.d.ts","sourceRoot":"","sources":["../src/batch-dispatch.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA6DG;AAmBH,OAAO,EAUL,KAAK,gBAAgB,EAErB,KAAK,SAAS,EACf,MAAM,YAAY,CAAC;AACpB,OAAO,EACL,KAAK,WAAW,EAOjB,MAAM,eAAe,CAAC;AACvB,OAAO,EAAE,KAAK,OAAO,EAAa,MAAM,WAAW,CAAC;AACpD,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,WAAW,CAAC;AAC5C,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,WAAW,CAAC;AAExC,OAAO,EAUL,KAAK,WAAW,EACjB,MAAM,YAAY,CAAC;
|
|
1
|
+
{"version":3,"file":"batch-dispatch.d.ts","sourceRoot":"","sources":["../src/batch-dispatch.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA6DG;AAmBH,OAAO,EAUL,KAAK,gBAAgB,EAErB,KAAK,SAAS,EACf,MAAM,YAAY,CAAC;AACpB,OAAO,EACL,KAAK,WAAW,EAOjB,MAAM,eAAe,CAAC;AACvB,OAAO,EAAE,KAAK,OAAO,EAAa,MAAM,WAAW,CAAC;AACpD,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,WAAW,CAAC;AAC5C,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,WAAW,CAAC;AAExC,OAAO,EAUL,KAAK,WAAW,EACjB,MAAM,YAAY,CAAC;AAiBpB,OAAO,EAAE,KAAK,QAAQ,EAA2D,MAAM,YAAY,CAAC;AACpG,OAAO,KAAK,EAEV,UAAU,EAEV,oBAAoB,EAKpB,WAAW,EAGZ,MAAM,SAAS,CAAC;AAWjB,YAAY,EAAE,UAAU,EAAE,MAAM,SAAS,CAAC;AAE1C,8DAA8D;AAC9D,MAAM,WAAW,iBAAiB;IAChC,KAAK,EAAE,UAAU,CAAC;IAClB,OAAO,EAAE,OAAO,CAAC;IACjB,WAAW,EAAE,WAAW,CAAC;IACzB,SAAS,EAAE,SAAS,CAAC;IACrB,GAAG,EAAE,MAAM,IAAI,CAAC;IAChB,uFAAuF;IACvF,OAAO,EAAE,MAAM,CAAC;IAChB,0FAA0F;IAC1F,IAAI,EAAE,MAAM,CAAC;IACb;;;;;;OAMG;IACH,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,sGAAsG;IACtG,QAAQ,EAAE,CAAC,QAAQ,EAAE,MAAM,KAAK,WAAW,CAAC;IAC5C;;;;;;;;;;;;;;;;;;;OAmBG;IACH,aAAa,CAAC,EAAE,CAAC,QAAQ,EAAE,MAAM,EAAE,YAAY,EAAE,MAAM,KAAK,oBAAoB,CAAC;IACjF,QAAQ,CAAC,EAAE,QAAQ,CAAC;IACpB;;;OAGG;IACH,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB;AAED,8FAA8F;AAC9F,MAAM,WAAW,eAAe;IAC9B,OAAO,EAAE,MAAM,EAAE,CAAC;IAClB,SAAS,EAAE,MAAM,EAAE,CAAC;IACpB,MAAM,EAAE,MAAM,EAAE,CAAC;IACjB,aAAa,EAAE,MAAM,EAAE,CAAC;IACxB,MAAM,EAAE,MAAM,EAAE,CAAC;IACjB,8FAA8F;IAC9F,OAAO,EAAE,MAAM,EAAE,CAAC;CACnB;AAED;;;;;;GAMG;AACH,MAAM,WAAW,aAAa;IAC5B,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;CAClC;AA6oCD;;;;;;;;;;;;;;;;;;;;;;;;GAwBG;AACH,wBAAgB,sBAAsB,CACpC,IAAI,EAAE,iBAAiB,EACvB,MAAM,EAAE,WAAW,EACnB,QAAQ,EAAE,gBAAgB,EAC1B,OAAO,EAAE,MAAM,EACf,KAAK,EAAE,UAAU,EACjB,WAAW,EAAE,MAAM,EACnB,OAAO,EAAE,aAAa,EACtB,GAAG,EAAE,IAAI,EACT,MAAM,EAAE,eAAe,GACtB,IAAI,CAyBN;AA0KD;;;;;;;;;;;;;;;;;;;;;;;GAuBG;AACH,wBAAgB,iBAAiB,CAC/B,IAAI,EAAE,iBAAiB,EACvB,MAAM,EAAE,WAAW,EACnB,QAAQ,EAAE,gBAAgB,EAC1B,OAAO,EAAE,MAAM,EACf,GAAG,EAAE,IAAI,GACR;IACD,OAAO,EAAE,eAAe,GAAG,SAAS,GAAG,WAAW,CAAC;IACnD,UAAU,EAAE,MAAM,CAAC;IACnB;;;;;OAKG;IACH,SAAS,CAAC,EAAE,cAAc,GAAG,qBAAqB,CAAC;IACnD,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB,CAyGA;AA8kBD;;;;;;;;;;;;;GAaG;AACH,wBAAgB,YAAY,CAC1B,IAAI,EAAE,iBAAiB,EACvB,MAAM,EAAE,WAAW,EACnB,QAAQ,EAAE,gBAAgB,GACzB,eAAe,CA2GjB"}
|
package/dist/batch-dispatch.js
CHANGED
|
@@ -95,6 +95,7 @@ var __importStar = (this && this.__importStar) || (function () {
|
|
|
95
95
|
};
|
|
96
96
|
})();
|
|
97
97
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
98
|
+
exports.evictMemberAndContinue = evictMemberAndContinue;
|
|
98
99
|
exports.resumeBlockedGate = resumeBlockedGate;
|
|
99
100
|
exports.runBatchTick = runBatchTick;
|
|
100
101
|
const fs = __importStar(require("node:fs"));
|
|
@@ -942,34 +943,67 @@ function evictOffender(deps, config, batchId, offender, attribution, now, result
|
|
|
942
943
|
});
|
|
943
944
|
}
|
|
944
945
|
}
|
|
945
|
-
// --- Reconcile a batch currently holding a live/exited slot ---
|
|
946
946
|
/**
|
|
947
|
-
*
|
|
948
|
-
*
|
|
949
|
-
*
|
|
950
|
-
*
|
|
947
|
+
* The one-shot claim on resolving `currentMember` (#613). Transitions the batch only while
|
|
948
|
+
* it is still `executing` on exactly that member; `false` means another call already
|
|
949
|
+
* claimed it (a duplicate dispatch of the same eviction/completion, or a re-entrant tick)
|
|
950
|
+
* and this caller must journal nothing and spawn nothing. Advancing anyway would validate a
|
|
951
|
+
* batch that already validated, or skip past a member that has not run at all — which is
|
|
952
|
+
* precisely how one member's eviction record ends up naming the member the batch already
|
|
953
|
+
* advanced past while the next member gets no record.
|
|
954
|
+
*
|
|
955
|
+
* Written once and called from both arms so the two can never drift apart: they are the
|
|
956
|
+
* same claim, differing only in what they transition to.
|
|
951
957
|
*/
|
|
958
|
+
function claimMemberResolution(deps, batchId, currentMember, now, next) {
|
|
959
|
+
return deps.store.withLock((s) => {
|
|
960
|
+
const b = (0, state_1.findBatch)(s, batchId);
|
|
961
|
+
if (!b || b.status !== 'executing' || b.executing_member !== currentMember) {
|
|
962
|
+
return {
|
|
963
|
+
state: s,
|
|
964
|
+
result: {
|
|
965
|
+
claimed: false,
|
|
966
|
+
observed: b?.executing_member ?? null,
|
|
967
|
+
status: b?.status ?? null,
|
|
968
|
+
},
|
|
969
|
+
};
|
|
970
|
+
}
|
|
971
|
+
const { to, patch } = next(b);
|
|
972
|
+
return {
|
|
973
|
+
state: (0, state_1.transitionBatch)(s, batchId, to, patch, now),
|
|
974
|
+
result: { claimed: true, observed: b.executing_member, status: b.status },
|
|
975
|
+
};
|
|
976
|
+
});
|
|
977
|
+
}
|
|
978
|
+
/** Journal a lost claim so a batch that stops advancing never does so silently (#613). */
|
|
979
|
+
function journalAdvanceSkipped(deps, batchId, memberIssue, currentMember, claim) {
|
|
980
|
+
journalEvent(deps, 'member-advance-skipped', unit(batchId), {
|
|
981
|
+
issue: memberIssue,
|
|
982
|
+
detail: `another resolution already advanced past member ${currentMember} (batch is now executing_member=${claim.observed ?? 'gone'}, status=${claim.status ?? 'gone'}) — no second member-advanced and no second continuation`,
|
|
983
|
+
});
|
|
984
|
+
}
|
|
952
985
|
function advanceMemberOrValidate(deps, config, dispatch, batchId, memberCount, currentMember, memberIssue, now, result) {
|
|
953
986
|
const isLast = currentMember >= memberCount;
|
|
954
987
|
if (isLast) {
|
|
955
|
-
deps
|
|
956
|
-
|
|
957
|
-
|
|
958
|
-
|
|
959
|
-
|
|
960
|
-
|
|
988
|
+
const claim = claimMemberResolution(deps, batchId, currentMember, now, () => ({
|
|
989
|
+
to: 'validating',
|
|
990
|
+
patch: {},
|
|
991
|
+
}));
|
|
992
|
+
if (!claim.claimed) {
|
|
993
|
+
journalAdvanceSkipped(deps, batchId, memberIssue, currentMember, claim);
|
|
994
|
+
return;
|
|
995
|
+
}
|
|
961
996
|
runValidate(deps, config, dispatch, batchId, now, result);
|
|
962
997
|
return;
|
|
963
998
|
}
|
|
964
|
-
deps
|
|
965
|
-
|
|
966
|
-
|
|
967
|
-
|
|
968
|
-
|
|
969
|
-
|
|
970
|
-
|
|
971
|
-
|
|
972
|
-
});
|
|
999
|
+
const claim = claimMemberResolution(deps, batchId, currentMember, now, (b) => ({
|
|
1000
|
+
to: 'executing',
|
|
1001
|
+
patch: { executing_member: b.executing_member + 1 },
|
|
1002
|
+
}));
|
|
1003
|
+
if (!claim.claimed) {
|
|
1004
|
+
journalAdvanceSkipped(deps, batchId, memberIssue, currentMember, claim);
|
|
1005
|
+
return;
|
|
1006
|
+
}
|
|
973
1007
|
journalEvent(deps, 'member-advanced', unit(batchId), { issue: memberIssue });
|
|
974
1008
|
spawnMemberContinuation(deps, config, dispatch, batchId, now, result);
|
|
975
1009
|
}
|
|
@@ -977,9 +1011,31 @@ function advanceMemberOrValidate(deps, config, dispatch, batchId, memberCount, c
|
|
|
977
1011
|
* Evict the current member and either dissolve, or continue the batch via
|
|
978
1012
|
* `advanceMemberOrValidate` — the shared tail of both member-failure rails
|
|
979
1013
|
* (self-reported blocked, and the incremental gate below).
|
|
1014
|
+
*
|
|
1015
|
+
* #613: `evictMemberDirectly`'s duplicate check (#595) is the one atomic,
|
|
1016
|
+
* lock-protected claim on "did THIS member's eviction already happen" — so
|
|
1017
|
+
* the `unit-failed` journal is emitted inside `evictMemberDirectly`, the
|
|
1018
|
+
* moment that claim succeeds and BEFORE the dissolve that claim may trigger,
|
|
1019
|
+
* rather than by each caller before it ever calls this function. A caller
|
|
1020
|
+
* that journaled `unit-failed` unconditionally, before the claim, could fire
|
|
1021
|
+
* it twice for one member (and once for the next member never at all) under
|
|
1022
|
+
* a duplicate/re-entrant resolve — the record and the journal must share the
|
|
1023
|
+
* same gate or they can disagree about which member the batch is advancing
|
|
1024
|
+
* past. Emitting it inside also keeps cause before effect in the journal and
|
|
1025
|
+
* survives a kill during the dissolve's shell-outs, which would otherwise
|
|
1026
|
+
* leave the eviction record on disk with no line saying why.
|
|
1027
|
+
*
|
|
1028
|
+
* Exported (not part of the package's `index.ts` public surface — imported
|
|
1029
|
+
* directly by `batch-integration.test.ts`) so #613's regression test can
|
|
1030
|
+
* call it twice with one stale `BatchEntry` snapshot: the exact "another
|
|
1031
|
+
* resolution already claimed this member" condition, which the public
|
|
1032
|
+
* `runBatchTick`/`resumeBlockedGate` entry points cannot reproduce since
|
|
1033
|
+
* both always read state fresh.
|
|
980
1034
|
*/
|
|
981
|
-
function evictMemberAndContinue(deps, config, dispatch, batchId, batch, memberIssue,
|
|
982
|
-
const dissolved = evictMemberDirectly(deps, config, batchId, memberIssue,
|
|
1035
|
+
function evictMemberAndContinue(deps, config, dispatch, batchId, batch, memberIssue, failure, now, result) {
|
|
1036
|
+
const { dissolved, duplicate } = evictMemberDirectly(deps, config, batchId, memberIssue, failure, now);
|
|
1037
|
+
if (duplicate)
|
|
1038
|
+
return;
|
|
983
1039
|
if (dissolved) {
|
|
984
1040
|
result.failed.push(unit(batchId));
|
|
985
1041
|
return;
|
|
@@ -1040,13 +1096,11 @@ function runIncrementalGate(deps, config, dispatch, batchId, batch, memberIssue,
|
|
|
1040
1096
|
const reason = `incremental-gate-failed:${earnedFailure.id}`;
|
|
1041
1097
|
writeGateLog(deps, batchId, earnedFailure.id, memberIssue, earnedFailure.outputTail);
|
|
1042
1098
|
const excerpt = gateDetailExcerpt(earnedFailure.outputTail, earnedFailure.reason);
|
|
1043
|
-
|
|
1044
|
-
|
|
1099
|
+
deps.store.withLock((s) => ({ state: releaseSlot(s, batchId, now), result: undefined }));
|
|
1100
|
+
evictMemberAndContinue(deps, config, dispatch, batchId, batch, memberIssue, {
|
|
1045
1101
|
reason,
|
|
1046
1102
|
detail: withExcerpt(`cap run ${earnedFailure.id} reported task-failed with failing-test evidence after member review done`, excerpt),
|
|
1047
|
-
});
|
|
1048
|
-
deps.store.withLock((s) => ({ state: releaseSlot(s, batchId, now), result: undefined }));
|
|
1049
|
-
evictMemberAndContinue(deps, config, dispatch, batchId, batch, memberIssue, reason, now, result);
|
|
1103
|
+
}, now, result);
|
|
1050
1104
|
return true;
|
|
1051
1105
|
}
|
|
1052
1106
|
const inconclusive = gateInconclusive ?? unevidencedFailure;
|
|
@@ -1188,12 +1242,10 @@ function resumeBlockedGate(deps, config, dispatch, batchId, now) {
|
|
|
1188
1242
|
if (recheck.outcome === 'task-failed') {
|
|
1189
1243
|
const reason = `incremental-gate-failed:${capabilityId}`;
|
|
1190
1244
|
writeGateLog(deps, batchId, capabilityId, memberIssue, recheck.outputTail);
|
|
1191
|
-
|
|
1192
|
-
issue: memberIssue,
|
|
1245
|
+
evictMemberAndContinue(deps, config, dispatch, batchId, batch, memberIssue, {
|
|
1193
1246
|
reason,
|
|
1194
1247
|
detail: withExcerpt(`sched resume --batch: cap run ${capabilityId} reported task-failed on recheck`, excerpt),
|
|
1195
|
-
});
|
|
1196
|
-
evictMemberAndContinue(deps, config, dispatch, batchId, batch, memberIssue, reason, now, result);
|
|
1248
|
+
}, now, result);
|
|
1197
1249
|
return { outcome: 'evicted', capability: capabilityId, detail: excerpt };
|
|
1198
1250
|
}
|
|
1199
1251
|
journalEvent(deps, 'external-advance', unit(batchId), {
|
|
@@ -1279,14 +1331,27 @@ function reconcileMemberSlot(deps, config, dispatch, batchId, slot, now, result)
|
|
|
1279
1331
|
const staleTerminal = milestone !== null &&
|
|
1280
1332
|
(((0, groundtruth_1.isMemberComplete)(milestone) && !(0, groundtruth_1.isMemberComplete)(milestone, slot.spawned_at)) ||
|
|
1281
1333
|
((0, groundtruth_1.isMemberBlocked)(milestone) && !(0, groundtruth_1.isMemberBlocked)(milestone, slot.spawned_at)));
|
|
1282
|
-
|
|
1334
|
+
// #610: this dispatch's own spawned_at IS the fence value the two
|
|
1335
|
+
// predicates above already key on — reusing it here as the "already
|
|
1336
|
+
// journalled" marker means a fresh dispatch (new spawned_at) invalidates
|
|
1337
|
+
// the marker for free, with no reset needed at any spawn site. Without the
|
|
1338
|
+
// gate this re-fired every tick for as long as the stale milestone stayed
|
|
1339
|
+
// latest — once every reconcile interval for the member's whole run.
|
|
1340
|
+
if (staleTerminal && slot.stale_milestone_ignored_for !== slot.spawned_at) {
|
|
1283
1341
|
journalEvent(deps, 'stale-milestone-ignored', unit(batchId), {
|
|
1284
1342
|
issue: memberIssue,
|
|
1285
1343
|
slot: slot.id,
|
|
1286
1344
|
run: milestone.run,
|
|
1287
|
-
|
|
1345
|
+
// The engine's OWN decision time — `milestone.at` is kept separately so
|
|
1346
|
+
// an operator can still see how old the stale milestone actually is.
|
|
1347
|
+
at: now.toISOString(),
|
|
1348
|
+
milestone_at: milestone.at,
|
|
1288
1349
|
detail: `${milestone.status === 'blocked' ? 'blocked' : 'complete'} milestone predates dispatch spawned_at=${slot.spawned_at}`,
|
|
1289
1350
|
});
|
|
1351
|
+
deps.store.withLock((s) => ({
|
|
1352
|
+
state: (0, state_1.patchSlot)(s, slot.id, { stale_milestone_ignored_for: slot.spawned_at }, now),
|
|
1353
|
+
result: undefined,
|
|
1354
|
+
}));
|
|
1290
1355
|
}
|
|
1291
1356
|
if ((0, groundtruth_1.isMemberComplete)(milestone, slot.spawned_at)) {
|
|
1292
1357
|
deps.journal.append((0, journal_1.unitEvent)('external-advance', unit(batchId), {
|
|
@@ -1323,17 +1388,6 @@ function reconcileMemberSlot(deps, config, dispatch, batchId, slot, now, result)
|
|
|
1323
1388
|
? 'agent-exited-unverified'
|
|
1324
1389
|
: 'member-blocked';
|
|
1325
1390
|
const lastTool = recordMemberRunLog(deps, dispatch, state0, batchId, batch, memberIssue, slot, now);
|
|
1326
|
-
journalEvent(deps, 'unit-failed', unit(batchId), {
|
|
1327
|
-
issue: memberIssue,
|
|
1328
|
-
reason,
|
|
1329
|
-
detail: 'member blocked',
|
|
1330
|
-
// #591: attributes an unverified member exit to a concrete cause (e.g.
|
|
1331
|
-
// `Monitor`) without opening the transcript. Gated on the RESOLVED reason, not
|
|
1332
|
-
// `dead` — a member can be simultaneously `dead` AND carry a milestone-posted
|
|
1333
|
-
// `reason` (it posted `blocked` and then exited), and that real block has no
|
|
1334
|
-
// log-derived cause to attribute; only `agent-exited-unverified` does.
|
|
1335
|
-
...(reason === 'agent-exited-unverified' && lastTool !== null ? { last_tool: lastTool } : {}),
|
|
1336
|
-
});
|
|
1337
1391
|
deps.store.withLock((s) => ({ state: releaseSlot(s, batchId, now), result: undefined }));
|
|
1338
1392
|
// A member that never went green (RFC F.1) evicts DIRECTLY — no aggregate
|
|
1339
1393
|
// suite has run yet, so there is nothing for `attributing`/`evicting` (the
|
|
@@ -1343,7 +1397,23 @@ function reconcileMemberSlot(deps, config, dispatch, batchId, slot, now, result)
|
|
|
1343
1397
|
// attributing → evicting` is not even a legal edge from mid-`executing`
|
|
1344
1398
|
// (BATCH_TRANSITIONS has no `executing → evicting`) — this stays entirely
|
|
1345
1399
|
// within `executing`/`dissolving`, both of which ARE legal from here.
|
|
1346
|
-
evictMemberAndContinue(deps, config, dispatch, batchId, batch, memberIssue,
|
|
1400
|
+
evictMemberAndContinue(deps, config, dispatch, batchId, batch, memberIssue, {
|
|
1401
|
+
reason,
|
|
1402
|
+
// This rail covers a self-reported block AND a member that simply died, so the
|
|
1403
|
+
// detail must follow the RESOLVED reason — `member blocked` on an
|
|
1404
|
+
// `agent-exited-unverified` eviction contradicts the reason on its own journal line.
|
|
1405
|
+
detail: reason === 'agent-exited-unverified'
|
|
1406
|
+
? 'member agent exited without posting a terminal milestone'
|
|
1407
|
+
: 'member blocked',
|
|
1408
|
+
// #591: attributes an unverified member exit to a concrete cause (e.g.
|
|
1409
|
+
// `Monitor`) without opening the transcript. Gated on the RESOLVED reason, not
|
|
1410
|
+
// `dead` — a member can be simultaneously `dead` AND carry a milestone-posted
|
|
1411
|
+
// `reason` (it posted `blocked` and then exited), and that real block has no
|
|
1412
|
+
// log-derived cause to attribute; only `agent-exited-unverified` does.
|
|
1413
|
+
extraKv: reason === 'agent-exited-unverified' && lastTool !== null
|
|
1414
|
+
? { last_tool: lastTool }
|
|
1415
|
+
: undefined,
|
|
1416
|
+
}, now, result);
|
|
1347
1417
|
}
|
|
1348
1418
|
}
|
|
1349
1419
|
/**
|
|
@@ -1359,20 +1429,34 @@ function reconcileMemberSlot(deps, config, dispatch, batchId, slot, now, result)
|
|
|
1359
1429
|
* and the queue entry disagreeing about why the member was evicted — and, if
|
|
1360
1430
|
* the member has since been re-dispatched full-cycle, would take the
|
|
1361
1431
|
* `executing → evicted → requeued` rail and kill that live run.
|
|
1432
|
+
*
|
|
1433
|
+
* Returns `{ dissolved, duplicate }`: `dissolved` is whether this eviction tipped the batch
|
|
1434
|
+
* past its threshold and the batch is gone; `duplicate: true` means another resolution had
|
|
1435
|
+
* already claimed this member's eviction, and the caller must journal nothing and advance
|
|
1436
|
+
* nothing on top of it (#613 — that is exactly how a record ends up naming the member the
|
|
1437
|
+
* batch already advanced past).
|
|
1362
1438
|
*/
|
|
1363
|
-
function evictMemberDirectly(deps, config, batchId, memberIssue,
|
|
1439
|
+
function evictMemberDirectly(deps, config, batchId, memberIssue, failure, now) {
|
|
1440
|
+
const { reason } = failure;
|
|
1364
1441
|
const dissolvePolicy = (0, types_1.resolveDissolvePolicy)(config.dissolve_policy);
|
|
1365
1442
|
// Pass 1 (pure — requeue + record the eviction): safe to run entirely
|
|
1366
1443
|
// inside the lock, unlike `dissolveBatch` below, which shells out
|
|
1367
1444
|
// (`deps.exec`/`postMilestone`) and so must NOT hold the lock while it runs.
|
|
1368
|
-
const { triggered, duplicate } = deps.store.withLock((s) => {
|
|
1445
|
+
const { triggered, duplicate, prior } = deps.store.withLock((s) => {
|
|
1369
1446
|
const b = (0, state_1.findBatch)(s, batchId);
|
|
1370
|
-
if (!b)
|
|
1371
|
-
return { state: s, result: { triggered: false, duplicate: false } };
|
|
1372
|
-
|
|
1447
|
+
if (!b) {
|
|
1448
|
+
return { state: s, result: { triggered: false, duplicate: false, prior: undefined } };
|
|
1449
|
+
}
|
|
1450
|
+
const priorRecord = b.evictions.find((e) => e.issue === memberIssue);
|
|
1451
|
+
if (priorRecord) {
|
|
1452
|
+
// A duplicate never re-evaluates the dissolve threshold: the resolution that WROTE
|
|
1453
|
+
// this record already did, under this same lock. (#595: a duplicate must not count
|
|
1454
|
+
// twice; #613: it must not act on the batch at all.) Note this also means a dissolve
|
|
1455
|
+
// interrupted between pass 1 and pass 2 is not retried here — re-entering
|
|
1456
|
+
// `dissolveBatch` on an already-terminal batch throws `IllegalTransitionError`.
|
|
1373
1457
|
return {
|
|
1374
1458
|
state: s,
|
|
1375
|
-
result: { triggered:
|
|
1459
|
+
result: { triggered: false, duplicate: true, prior: priorRecord },
|
|
1376
1460
|
};
|
|
1377
1461
|
}
|
|
1378
1462
|
const evidence = {
|
|
@@ -1404,24 +1488,42 @@ function evictMemberDirectly(deps, config, batchId, memberIssue, reason, now) {
|
|
|
1404
1488
|
result: {
|
|
1405
1489
|
triggered: updated !== undefined && (0, recovery_1.checkDissolveTrigger)(updated, dissolvePolicy),
|
|
1406
1490
|
duplicate: duplicateRecords.length > 0,
|
|
1491
|
+
prior: undefined,
|
|
1407
1492
|
},
|
|
1408
1493
|
};
|
|
1409
1494
|
});
|
|
1410
1495
|
if (duplicate) {
|
|
1411
1496
|
journalEvent(deps, 'eviction-duplicate', unit(batchId), {
|
|
1412
1497
|
issue: memberIssue,
|
|
1413
|
-
detail: (0, state_1.duplicateEvictionDetail)(memberIssue, reason),
|
|
1498
|
+
detail: (0, state_1.duplicateEvictionDetail)(memberIssue, reason, prior),
|
|
1414
1499
|
});
|
|
1500
|
+
// Another resolution already claimed this member's eviction record — the
|
|
1501
|
+
// caller must not journal its own `unit-failed`/advance the batch again
|
|
1502
|
+
// on top of that one (#613: that is exactly how a record ends up naming
|
|
1503
|
+
// the member the batch already advanced past).
|
|
1504
|
+
return { dissolved: false, duplicate: true };
|
|
1415
1505
|
}
|
|
1506
|
+
// This call owns the member's resolution, so it owns recording WHY — before pass 2 below
|
|
1507
|
+
// can dissolve, tear the worktree down, or be killed mid-shell-out and leave the eviction
|
|
1508
|
+
// record on disk with no journal line naming its cause (#613). The caller's extras go
|
|
1509
|
+
// first so the authoritative keys always win: `extraKv` is a wide Record, and a future
|
|
1510
|
+
// caller passing `reason`/`detail`/`issue` must not be able to rewrite the record's own
|
|
1511
|
+
// identity.
|
|
1512
|
+
journalEvent(deps, 'unit-failed', unit(batchId), {
|
|
1513
|
+
...(failure.extraKv ?? {}),
|
|
1514
|
+
issue: memberIssue,
|
|
1515
|
+
reason,
|
|
1516
|
+
detail: failure.detail,
|
|
1517
|
+
});
|
|
1416
1518
|
if (!triggered)
|
|
1417
|
-
return false;
|
|
1519
|
+
return { dissolved: false, duplicate: false };
|
|
1418
1520
|
// Pass 2 (outside the lock — dissolveBatch shells out): re-load fresh
|
|
1419
1521
|
// (pass 1's write already landed), dissolve, then re-apply just this
|
|
1420
1522
|
// batch's + the requeued members' state under a fresh lock.
|
|
1421
1523
|
const state = deps.store.load();
|
|
1422
1524
|
const batch = (0, state_1.findBatch)(state, batchId);
|
|
1423
1525
|
if (!batch)
|
|
1424
|
-
return false;
|
|
1526
|
+
return { dissolved: false, duplicate: false };
|
|
1425
1527
|
const rDeps = recoveryDeps(deps, config, batch, now);
|
|
1426
1528
|
const outcome = (0, recovery_1.dissolveBatch)(state, batchId, { strategy: 'full', reason: 'eviction-threshold' }, rDeps);
|
|
1427
1529
|
deps.store.withLock((s) => ({
|
|
@@ -1429,7 +1531,7 @@ function evictMemberDirectly(deps, config, batchId, memberIssue, reason, now) {
|
|
|
1429
1531
|
result: undefined,
|
|
1430
1532
|
}));
|
|
1431
1533
|
teardownBatch(deps, batchId);
|
|
1432
|
-
return true;
|
|
1534
|
+
return { dissolved: true, duplicate: false };
|
|
1433
1535
|
}
|
|
1434
1536
|
function reconcileFixSlot(deps, config, batchId, slot, now, result) {
|
|
1435
1537
|
if (slot.pid !== null && deps.spawnDeps.isAlive(slot.pid, slot.pid_start ?? undefined))
|