@ai-dossier/sched 0.55.1 → 0.57.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -164,9 +164,18 @@ function sanitizeUntrustedText(value) {
164
164
  function journalEvent(deps, event, unitId, extra = {}) {
165
165
  deps.journal.append((0, journal_1.unitEvent)(event, unitId, extra), deps.now());
166
166
  }
167
- /** Stop the slot's agent if it is still alive (pid-start-safe: a reused pid is never signalled). */
167
+ /** Whether the slot's agent is still alive (pid-start-safe: a reused pid reads dead). */
168
+ function agentAlive(deps, slot) {
169
+ return slot.pid !== null && deps.spawnDeps.isAlive(slot.pid, slot.pid_start ?? undefined);
170
+ }
171
+ /**
172
+ * Stop the slot's agent if it is still alive (pid-start-safe: a reused pid is
173
+ * never signalled). SIGTERM only — no wait, no escalation. A decision that
174
+ * must not run alongside the agent uses `stopAgentBeforeDeciding` instead,
175
+ * which escalates to SIGKILL (#844).
176
+ */
168
177
  function killLiveAgent(deps, slot) {
169
- if (slot.pid !== null && deps.spawnDeps.isAlive(slot.pid, slot.pid_start ?? undefined)) {
178
+ if (agentAlive(deps, slot)) {
170
179
  deps.spawnDeps.kill(slot.pid, slot.pid_start ?? undefined);
171
180
  }
172
181
  }
@@ -806,9 +815,11 @@ function landMemberBranch(deps, batchId, now) {
806
815
  * `-d` after a landing (fully merged into the integration branch), `-D` on
807
816
  * the eviction path (its commits never landed; since #810 the parked entry
808
817
  * records the branch and an operator requeue continues from the REMOTE copy).
809
- * The REMOTE member branch is deleted only on the landed path, and
810
- * deliberately KEPT on eviction — the pushed sha is the evicted work's only
811
- * recoverable copy.
818
+ * The REMOTE member branch is never deleted here (#840): on eviction the
819
+ * pushed sha is the evicted work's only recoverable copy, and after a landing
820
+ * it is what a later aggregate-suite eviction parks the member on. Batch
821
+ * teardown deletes it once the batch ships or dissolves
822
+ * (`deleteMemberBranches`).
812
823
  *
813
824
  * Best-effort and idempotent: null fields are a no-op, and a failed cleanup
814
825
  * journals `teardown-failed` rather than throwing into the caller's
@@ -894,9 +905,6 @@ issue) {
894
905
  deps.exec('git', ['branch', flag, branch], deps.repoDir) === null
895
906
  ? `failed-branch-delete-${flag}`
896
907
  : `branch-deleted-${flag}`;
897
- if (landed) {
898
- deps.exec('git', ['push', 'origin', '--delete', branch], deps.repoDir);
899
- }
900
908
  }
901
909
  const failed = !treeCleared || branchCleanup.startsWith('failed');
902
910
  journalEvent(deps, failed ? 'teardown-failed' : 'member-worktree-torn-down', unit(batchId), {
@@ -1285,6 +1293,9 @@ function spawnMemberContinuation(deps, config, dispatch, batchId, now, result) {
1285
1293
  // contract `prepareMemberWorktree` documents). The wedge arm of
1286
1294
  // `runBatchTick` calls this EVERY tick while the batch is `executing`
1287
1295
  // with no slot, so the cheap gates run first:
1296
+ // - recovery (#844): a current member already in `evictions[]` is advanced
1297
+ // past, never respawned — checked before capacity, since the advance
1298
+ // needs no slot.
1288
1299
  // - capacity: prep before the free-capacity check would do pool/git/npm
1289
1300
  // work (and journal) per tick against a full scheduler — the #610/#630/
1290
1301
  // #632 per-tick emission trap. `claimAndSpawn` re-checks under the lock;
@@ -1301,6 +1312,30 @@ function spawnMemberContinuation(deps, config, dispatch, batchId, now, result) {
1301
1312
  const memberIssue = batch.members[batch.executing_member - 1];
1302
1313
  if (memberIssue === undefined)
1303
1314
  return;
1315
+ const evicted = batch.evictions.find((e) => e.issue === memberIssue);
1316
+ if (evicted !== undefined) {
1317
+ // #844: the member's eviction (and its slot release, in the same write)
1318
+ // landed, but the engine died before advancing past it — possibly before
1319
+ // journaling its cause, too, so the record's cause goes on this line.
1320
+ // Respawning it would dispatch an evicted member again — finish what the
1321
+ // eviction would have done instead: the dissolve its threshold tripped,
1322
+ // or the advance.
1323
+ const kind = evicted.kind ?? 'evicted';
1324
+ journalEvent(deps, 'member-advance-recovered', unit(batchId), {
1325
+ issue: memberIssue,
1326
+ reason: evicted.reason,
1327
+ kind,
1328
+ evicted_at: evicted.at,
1329
+ detail: `member ${batch.executing_member} is already parked ${kind} (${evicted.reason}) but the batch never advanced past it (engine exit mid-eviction) — continuing instead of respawning it`,
1330
+ });
1331
+ if ((0, recovery_1.checkDissolveTrigger)(batch, (0, types_1.resolveDissolvePolicy)(config.dissolve_policy)) &&
1332
+ settleEvictionThreshold(deps, config, batchId, memberIssue, (0, state_1.validatedMembersOf)(state, batch), now)) {
1333
+ result.failed.push(unit(batchId));
1334
+ return;
1335
+ }
1336
+ continuePastEvictedMember(deps, config, dispatch, batchId, batch, memberIssue, now, result);
1337
+ return;
1338
+ }
1304
1339
  if ((0, scheduler_1.freeCapacity)(state, config) === 0)
1305
1340
  return;
1306
1341
  const expectedBranch = memberBranchFor(batchId, batch.executing_member, memberIssue);
@@ -1669,7 +1704,7 @@ function evictOffender(deps, config, batchId, offender, attribution, now, result
1669
1704
  const rDeps = recoveryDeps(deps, config, batch, now);
1670
1705
  const outcome = (0, recovery_1.evictMembers)(state, batchId, { issues: [offender], reason: 'suite-red-after-fix', attribution, ranges: batch.ranges }, rDeps);
1671
1706
  deps.store.withLock((s) => ({
1672
- state: applyBatchAndIssues(s, outcome.state, batchId, [...outcome.requeued, ...(outcome.dissolve?.parked ?? [])], batch.members),
1707
+ state: applyBatchAndIssues(s, outcome.state, batchId, [...new Set([...outcome.requeued, ...outcome.parked, ...(outcome.dissolve?.parked ?? [])])], batch.members),
1673
1708
  result: undefined,
1674
1709
  }));
1675
1710
  // #810: `dissolve.blocked` — a full dissolve refused over validated
@@ -1772,8 +1807,14 @@ serialTeardown) {
1772
1807
  }
1773
1808
  /**
1774
1809
  * Evict the current member and either dissolve, or continue the batch via
1775
- * `advanceMemberOrValidate` — the shared tail of both member-failure rails
1776
- * (self-reported blocked, and the incremental gate below).
1810
+ * `advanceMemberOrValidate` — the shared tail of the serial member-failure
1811
+ * rails (dead/blocked, the incremental gate below, the wrong-procedure
1812
+ * evictions, and `sched resume --batch`'s recheck).
1813
+ *
1814
+ * #844: a caller whose member still holds its slot MUST pass its release as
1815
+ * `preWrite` — never release under a separate lock first. `evictMemberDirectly` commits
1816
+ * the release with the park/eviction record in one write, so an engine exit
1817
+ * cannot leave the member slotless and un-evicted (and so respawned).
1777
1818
  *
1778
1819
  * #613: `evictMemberDirectly`'s duplicate check (#595) is the one atomic,
1779
1820
  * lock-protected claim on "did THIS member's eviction already happen" — so
@@ -1795,14 +1836,27 @@ serialTeardown) {
1795
1836
  * `runBatchTick`/`resumeBlockedGate` entry points cannot reproduce since
1796
1837
  * both always read state fresh.
1797
1838
  */
1798
- function evictMemberAndContinue(deps, config, dispatch, batchId, batch, memberIssue, failure, now, result) {
1799
- const { batchEnded, duplicate } = evictMemberDirectly(deps, config, batchId, memberIssue, failure, now);
1839
+ function evictMemberAndContinue(deps, config, dispatch, batchId, batch, memberIssue, failure, now, result,
1840
+ /** #844: the member's slot release (or any pre-write) — committed in the eviction's own write. */
1841
+ preWrite) {
1842
+ const { batchEnded, duplicate } = evictMemberDirectly(deps, config, batchId, memberIssue, failure, now, preWrite);
1800
1843
  if (duplicate)
1801
1844
  return;
1802
1845
  if (batchEnded) {
1803
1846
  result.failed.push(unit(batchId));
1804
1847
  return;
1805
1848
  }
1849
+ continuePastEvictedMember(deps, config, dispatch, batchId, batch, memberIssue, now, result);
1850
+ }
1851
+ /**
1852
+ * The serial rail after a member's eviction has been recorded: tear its tree
1853
+ * down and advance. Also the crash recovery for an engine that died between
1854
+ * the eviction's write and this advance (#844) — `spawnMemberContinuation`
1855
+ * finds the batch still pointing at a member already in `evictions[]` and
1856
+ * continues from here instead of respawning it. `advanceMemberOrValidate`'s
1857
+ * one-shot claim (#613) keeps a re-entry from advancing twice.
1858
+ */
1859
+ function continuePastEvictedMember(deps, config, dispatch, batchId, batch, memberIssue, now, result) {
1806
1860
  // #677: the evicted member's commits never landed (eviction is the
1807
1861
  // pre-landing rail), so there is nothing to revert on the integration
1808
1862
  // branch — but its worktree/branch must go, or the next tick's prep for
@@ -1842,7 +1896,9 @@ function evictMemberAndContinue(deps, config, dispatch, batchId, batch, memberIs
1842
1896
  * configured or both checks came back `ok`, meaning the caller should treat
1843
1897
  * the member as genuinely complete.
1844
1898
  */
1845
- function runIncrementalGate(deps, config, dispatch, batchId, batch, memberIssue, now, result) {
1899
+ function runIncrementalGate(deps, config, dispatch, batchId, batch, memberIssue, now, result,
1900
+ /** The member's slot release — handed to the eviction on `failed` (#844). */
1901
+ release) {
1846
1902
  if (batch.worktree === null || !deps.runCapability)
1847
1903
  return false;
1848
1904
  // #677: the gate judges the member's OWN work — pre-landing, its diff
@@ -1854,11 +1910,10 @@ function runIncrementalGate(deps, config, dispatch, batchId, batch, memberIssue,
1854
1910
  if (verdict.kind === 'pass')
1855
1911
  return false;
1856
1912
  if (verdict.kind === 'failed') {
1857
- deps.store.withLock((s) => ({ state: releaseSlot(s, batchId, now), result: undefined }));
1858
- evictMemberAndContinue(deps, config, dispatch, batchId, batch, memberIssue, { reason: verdict.reason, detail: verdict.detail }, now, result);
1913
+ evictMemberAndContinue(deps, config, dispatch, batchId, batch, memberIssue, { reason: verdict.reason, detail: verdict.detail }, now, result, release);
1859
1914
  return true;
1860
1915
  }
1861
- deps.store.withLock((s) => ({ state: releaseSlot(s, batchId, now), result: undefined }));
1916
+ deps.store.withLock((s) => ({ state: release(s), result: undefined }));
1862
1917
  // #677: F.11's block semantics are "the member's commit stays on the
1863
1918
  // branch, nothing requeued/reverted" — under member worktrees that is
1864
1919
  // true only if the member LANDS before the batch blocks. The block means
@@ -2176,19 +2231,21 @@ function resumeBlockedGate(deps, config, dispatch, batchId, now) {
2176
2231
  detail: excerpt,
2177
2232
  };
2178
2233
  }
2179
- deps.store.withLock((s) => ({
2180
- state: (0, state_1.patchBatch)((0, state_1.transitionBatch)(s, batchId, 'executing', {}, now), batchId, { blocked_reason: null }, now),
2181
- result: undefined,
2182
- }));
2234
+ const unblock = (s) => (0, state_1.patchBatch)((0, state_1.transitionBatch)(s, batchId, 'executing', {}, now), batchId, { blocked_reason: null }, now);
2183
2235
  if (recheck.outcome === 'task-failed') {
2184
2236
  const reason = `incremental-gate-failed:${capabilityId}`;
2185
2237
  writeGateLog(deps, batchId, capabilityId, memberIssue, recheck.outputTail);
2186
2238
  evictMemberAndContinue(deps, config, batchDispatch, batchId, batch, memberIssue, {
2187
2239
  reason,
2188
2240
  detail: withExcerpt(`sched resume --batch: cap run ${capabilityId} reported task-failed on recheck`, excerpt),
2189
- }, now, result);
2241
+ }, now, result,
2242
+ // #844: the unblock commits with the eviction — two writes left a
2243
+ // slotless `executing` batch whose un-evicted member the wedge arm
2244
+ // then respawned.
2245
+ unblock);
2190
2246
  return { outcome: 'evicted', capability: capabilityId, detail: excerpt };
2191
2247
  }
2248
+ deps.store.withLock((s) => ({ state: unblock(s), result: undefined }));
2192
2249
  journalEvent(deps, 'external-advance', unit(batchId), {
2193
2250
  issue: memberIssue,
2194
2251
  detail: `sched resume --batch: cap run ${capabilityId} reported ok on recheck`,
@@ -2389,6 +2446,84 @@ function wrongProcedureCounts(batch, memberIssue, read) {
2389
2446
  const repromptedAt = Date.parse(prior.reprompted_at);
2390
2447
  return Number.isNaN(at) || Number.isNaN(repromptedAt) ? false : at > repromptedAt;
2391
2448
  }
2449
+ /**
2450
+ * A kill timestamp that belongs to the slot's CURRENT dispatch, or null
2451
+ * (#844): one stamped before `spawned_at` was for an earlier dispatch in the
2452
+ * same slot, and an unparseable or future one is no anchor at all.
2453
+ */
2454
+ function currentDispatchStamp(stamp, spawnedAt, now) {
2455
+ if (stamp === null)
2456
+ return null;
2457
+ const at = Date.parse(stamp);
2458
+ if (Number.isNaN(at))
2459
+ return null;
2460
+ if (spawnedAt !== null && at < Date.parse(spawnedAt))
2461
+ return null;
2462
+ // A future stamp (clock step, hand edit) would make the elapsed time
2463
+ // negative and switch the escalation off for good — re-stamp instead.
2464
+ if (at > now.getTime())
2465
+ return null;
2466
+ return stamp;
2467
+ }
2468
+ /**
2469
+ * Stop the slot's agent before a decision that must not run alongside it
2470
+ * (#844). Returns `true` while it is still alive — the caller waits and
2471
+ * decides on a later tick — and `false` once it is gone (or never had a pid).
2472
+ *
2473
+ * The first sight sends SIGTERM, stamps `kill_sent_at` and journals
2474
+ * `member-stop-requested` once; each later tick
2475
+ * re-sends SIGTERM until the agent has outlived it by `KILL_ESCALATION_MS`,
2476
+ * then sends SIGKILL (to its process group — `SpawnDeps.kill`) and journals
2477
+ * `kill-escalated` once (`kill_escalated_at`). Pre-#844 this path re-sent
2478
+ * SIGTERM forever, so an agent ignoring it held its member indefinitely.
2479
+ */
2480
+ function stopAgentBeforeDeciding(deps, batchId, memberIssue, slot, now) {
2481
+ if (!agentAlive(deps, slot))
2482
+ return false;
2483
+ const pid = slot.pid;
2484
+ const pidStart = slot.pid_start ?? undefined;
2485
+ const sentAt = currentDispatchStamp(slot.kill_sent_at, slot.spawned_at, now);
2486
+ const aliveForMs = sentAt === null ? 0 : now.getTime() - Date.parse(sentAt);
2487
+ const escalate = sentAt !== null && aliveForMs >= dispatch_1.KILL_ESCALATION_MS;
2488
+ // `kill` refuses a pid whose recorded identity no longer matches (reused by
2489
+ // another process): the agent we spawned is gone, so stop waiting on it.
2490
+ if (!deps.spawnDeps.kill(pid, pidStart, escalate ? 'SIGKILL' : 'SIGTERM'))
2491
+ return false;
2492
+ const firstEscalation = escalate && currentDispatchStamp(slot.kill_escalated_at, slot.spawned_at, now) === null;
2493
+ if (sentAt === null || firstEscalation) {
2494
+ deps.store.withLock((s) => {
2495
+ const live = s.slots.find((x) => x.id === slot.id);
2496
+ // Only the slot still holding THIS agent — never a later dispatch's.
2497
+ if (!live || live.unit !== slot.unit || live.pid !== pid) {
2498
+ return { state: s, result: undefined };
2499
+ }
2500
+ const stamp = now.toISOString();
2501
+ return {
2502
+ state: (0, state_1.patchSlot)(s, slot.id, sentAt === null
2503
+ ? { kill_sent_at: stamp, kill_escalated_at: null }
2504
+ : { kill_escalated_at: stamp }, now),
2505
+ result: undefined,
2506
+ };
2507
+ });
2508
+ }
2509
+ if (sentAt === null) {
2510
+ journalEvent(deps, 'member-stop-requested', unit(batchId), {
2511
+ issue: memberIssue,
2512
+ pid,
2513
+ slot: slot.id,
2514
+ detail: `SIGTERM sent — waiting for the agent to exit before deciding (SIGKILL after ${dispatch_1.KILL_ESCALATION_MS / 1000}s)`,
2515
+ });
2516
+ }
2517
+ if (firstEscalation) {
2518
+ journalEvent(deps, 'kill-escalated', unit(batchId), {
2519
+ issue: memberIssue,
2520
+ pid,
2521
+ slot: slot.id,
2522
+ detail: `agent still alive ${Math.round(aliveForMs / 1000)}s after SIGTERM (bound ${dispatch_1.KILL_ESCALATION_MS / 1000}s) — sent SIGKILL (to its process group where available)`,
2523
+ });
2524
+ }
2525
+ return true;
2526
+ }
2392
2527
  /**
2393
2528
  * #822 (#810 proposal 3): the member's dispatch ran the wrong procedure (a
2394
2529
  * full-cycle-shaped trail — imboard #4174 ran full-cycle as a batch member).
@@ -2398,23 +2533,27 @@ function wrongProcedureCounts(batch, memberIssue, read) {
2398
2533
  * `reprompted_members` — the wedge arm (serial) or `spawnParallelMembers`
2399
2534
  * (parallel) respawns it in place, in the same worktree, with
2400
2535
  * `wrongProcedureDirective` appended. Returns `'evict'` when the member was
2401
- * already re-prompted once: the caller evicts it `wrong-procedure`.
2536
+ * already re-prompted once: the caller evicts it `wrong-procedure`. On
2537
+ * `'evict'` the slot is NOT released here — the caller passes `release` to
2538
+ * `evictMemberAndContinue`/`evictParallelMember`, which commits it in the
2539
+ * eviction's own write (#844).
2402
2540
  */
2403
2541
  function repromptWrongProcedure(deps, batchId, memberIssue, milestoneAt, slot, release, now) {
2404
2542
  // Never respawn into (or release) a worktree the old agent is still in: a
2405
2543
  // dying full-cycle run could still push, post, or open a PR. Signal it and
2406
2544
  // decide on a later tick, once it is gone — and so the re-prompt's
2407
2545
  // `reprompted_at` fence postdates everything it could still post.
2408
- if (slot.pid !== null && deps.spawnDeps.isAlive(slot.pid, slot.pid_start ?? undefined)) {
2409
- deps.spawnDeps.kill(slot.pid, slot.pid_start ?? undefined);
2546
+ if (stopAgentBeforeDeciding(deps, batchId, memberIssue, slot, now))
2410
2547
  return 'waiting';
2411
- }
2412
2548
  const outcome = deps.store.withLock((s) => {
2413
- const released = release(s);
2414
- const b = (0, state_1.findBatch)(released, batchId);
2549
+ const b = (0, state_1.findBatch)(s, batchId);
2415
2550
  if (!b || b.reprompted_members.some((r) => r.issue === memberIssue)) {
2416
- return { state: released, result: 'evict' };
2551
+ // #844: NOT released here — the caller's eviction commits the release
2552
+ // in its own write, so a crash in between cannot leave the member
2553
+ // slotless and un-evicted (and so respawned).
2554
+ return { state: s, result: 'evict' };
2417
2555
  }
2556
+ const released = release(s);
2418
2557
  return {
2419
2558
  state: (0, state_1.patchBatch)(released, batchId, {
2420
2559
  reprompted_members: [
@@ -2455,18 +2594,16 @@ function wrongProcedureFailure(lastTool, shippedPr = null) {
2455
2594
  * #822: the one wrong-procedure decision shared by both member rails —
2456
2595
  * `'waiting'` (the old agent is still dying), `'reprompted'`, or the
2457
2596
  * eviction failure to apply. A run that already shipped is evicted without
2458
- * a re-prompt.
2597
+ * a re-prompt. On an eviction the slot is still held: the caller passes
2598
+ * `release` to the eviction, which commits both in one write (#844).
2459
2599
  */
2460
2600
  function decideWrongProcedure(deps, batchId, memberIssue, milestone, slot, release, lastTool, now) {
2461
2601
  const shipped = (0, groundtruth_1.wrongProcedureShippedPr)(milestone);
2462
2602
  if (shipped !== null) {
2463
- if (slot.pid !== null && deps.spawnDeps.isAlive(slot.pid, slot.pid_start ?? undefined)) {
2464
- deps.spawnDeps.kill(slot.pid, slot.pid_start ?? undefined);
2603
+ if (stopAgentBeforeDeciding(deps, batchId, memberIssue, slot, now))
2465
2604
  return 'waiting';
2466
- }
2467
- const tool = lastTool();
2468
- deps.store.withLock((s) => ({ state: release(s), result: undefined }));
2469
- return wrongProcedureFailure(tool, shipped);
2605
+ // #844: the caller's eviction releases the slot in its own write.
2606
+ return wrongProcedureFailure(lastTool(), shipped);
2470
2607
  }
2471
2608
  const verdict = repromptWrongProcedure(deps, batchId, memberIssue, milestone.at, slot, release, now);
2472
2609
  if (verdict === 'waiting')
@@ -2521,6 +2658,9 @@ function reconcileMemberSlot(deps, config, dispatch, batchId, slot, now, result)
2521
2658
  const memberIssue = batch.members[batch.executing_member - 1];
2522
2659
  if (memberIssue === undefined)
2523
2660
  return;
2661
+ // #844: every rail below that evicts hands THIS to the eviction, which
2662
+ // commits it in the same write — never released under a lock of its own.
2663
+ const release = (s) => releaseSlot(s, batchId, now);
2524
2664
  const read = readMemberDispatch(deps, batchId, memberIssue, slot, now);
2525
2665
  if (read.kind === 'unreachable')
2526
2666
  return;
@@ -2538,7 +2678,7 @@ function reconcileMemberSlot(deps, config, dispatch, batchId, slot, now, result)
2538
2678
  // gate already decided the member's fate (evicted or blocked) and
2539
2679
  // returned early; `false` means both checks were `ok` and the member is
2540
2680
  // genuinely done.
2541
- if (runIncrementalGate(deps, config, dispatch, batchId, batch, memberIssue, now, result)) {
2681
+ if (runIncrementalGate(deps, config, dispatch, batchId, batch, memberIssue, now, result, release)) {
2542
2682
  return;
2543
2683
  }
2544
2684
  completeMemberGate(deps, config, dispatch, batchId, batch, memberIssue, now, result);
@@ -2548,9 +2688,9 @@ function reconcileMemberSlot(deps, config, dispatch, batchId, slot, now, result)
2548
2688
  // before the dead/blocked rail, which would otherwise evict it as an
2549
2689
  // unverified exit at the first sight.
2550
2690
  if (wrongProcedureCounts(batch, memberIssue, read) && read.milestone !== null) {
2551
- const decision = decideWrongProcedure(deps, batchId, memberIssue, read.milestone, slot, (s) => releaseSlot(s, batchId, now), () => recordMemberRunLog(deps, dispatch, state0, batchId, batch.executing_member, memberIssue, slot, now), now);
2691
+ const decision = decideWrongProcedure(deps, batchId, memberIssue, read.milestone, slot, release, () => recordMemberRunLog(deps, dispatch, state0, batchId, batch.executing_member, memberIssue, slot, now), now);
2552
2692
  if (typeof decision === 'object') {
2553
- evictMemberAndContinue(deps, config, dispatch, batchId, batch, memberIssue, decision, now, result);
2693
+ evictMemberAndContinue(deps, config, dispatch, batchId, batch, memberIssue, decision, now, result, release);
2554
2694
  }
2555
2695
  return;
2556
2696
  }
@@ -2566,7 +2706,6 @@ function reconcileMemberSlot(deps, config, dispatch, batchId, slot, now, result)
2566
2706
  }
2567
2707
  if (read.blockedNow || read.dead) {
2568
2708
  const lastTool = recordMemberRunLog(deps, dispatch, state0, batchId, batch.executing_member, memberIssue, slot, now);
2569
- deps.store.withLock((s) => ({ state: releaseSlot(s, batchId, now), result: undefined }));
2570
2709
  // A member that never went green (RFC F.1) evicts DIRECTLY — no aggregate
2571
2710
  // suite has run yet, so there is nothing for `attributing`/`evicting` (the
2572
2711
  // AGGREGATE-suite-red pipeline, RFC F.2) to attribute or revert: the
@@ -2575,7 +2714,9 @@ function reconcileMemberSlot(deps, config, dispatch, batchId, slot, now, result)
2575
2714
  // attributing → evicting` is not even a legal edge from mid-`executing`
2576
2715
  // (BATCH_TRANSITIONS has no `executing → evicting`) — this stays entirely
2577
2716
  // within `executing`/`dissolving`, both of which ARE legal from here.
2578
- evictMemberAndContinue(deps, config, dispatch, batchId, batch, memberIssue, memberFailureFor(read, lastTool), now, result);
2717
+ evictMemberAndContinue(deps, config, dispatch, batchId, batch, memberIssue, memberFailureFor(read, lastTool), now, result,
2718
+ // #844: released in the eviction's own write, never before it.
2719
+ release);
2579
2720
  }
2580
2721
  }
2581
2722
  /**
@@ -2616,14 +2757,26 @@ function reconcileMemberSlot(deps, config, dispatch, batchId, slot, now, result)
2616
2757
  * retried by a re-entry: the re-entry takes the duplicate path (pre-existing,
2617
2758
  * #595/#613) and the batch continues.
2618
2759
  */
2619
- function evictMemberDirectly(deps, config, batchId, memberIssue, failure, now) {
2760
+ function evictMemberDirectly(deps, config, batchId, memberIssue, failure, now,
2761
+ /**
2762
+ * #844: a state transform applied inside pass 1's lock in every branch —
2763
+ * typically the member's slot release (plus, on the parallel rail, its
2764
+ * run's `evicted` status; on `sched resume --batch`, the batch's
2765
+ * `blocked → executing` transition) — so it and the park/eviction record
2766
+ * commit as ONE write. Pre-#844 callers wrote it under a separate lock
2767
+ * first, and an engine exit between the two left the member in-work with
2768
+ * no slot, which the wedge arm (serial) or `spawnParallelMembers`
2769
+ * (parallel) then respawned.
2770
+ */
2771
+ preWrite) {
2620
2772
  const { reason } = failure;
2621
2773
  const kind = failure.kind ?? 'evicted';
2622
2774
  const dissolvePolicy = (0, types_1.resolveDissolvePolicy)(config.dissolve_policy);
2623
2775
  // Pass 1 (pure — park + record the exit): safe to run entirely inside the
2624
2776
  // lock, unlike `dissolveBatch` below, which shells out
2625
2777
  // (`deps.exec`/`postMilestone`) and so must NOT hold the lock while it runs.
2626
- const { triggered, duplicate, prior, branch, validated } = deps.store.withLock((s) => {
2778
+ const { triggered, duplicate, prior, branch, validated } = deps.store.withLock((held) => {
2779
+ const s = preWrite ? preWrite(held) : held;
2627
2780
  const b = (0, state_1.findBatch)(s, batchId);
2628
2781
  if (!b) {
2629
2782
  return {
@@ -2708,6 +2861,20 @@ function evictMemberDirectly(deps, config, batchId, memberIssue, failure, now) {
2708
2861
  });
2709
2862
  if (!triggered)
2710
2863
  return { batchEnded: false, duplicate: false };
2864
+ return {
2865
+ batchEnded: settleEvictionThreshold(deps, config, batchId, memberIssue, validated, now),
2866
+ duplicate: false,
2867
+ };
2868
+ }
2869
+ /**
2870
+ * Pass 2 of an eviction that tipped the batch past its dissolve threshold:
2871
+ * dissolve it — unless members are already VALIDATED (#810: never dissolve
2872
+ * over landed work; journal `dissolve-suppressed` and continue). Returns
2873
+ * whether the batch's run ended. Shared by `evictMemberDirectly` and #844's
2874
+ * crash recovery in `spawnMemberContinuation`, which re-runs it for an
2875
+ * eviction whose pass 1 landed but whose engine exited before this ran.
2876
+ */
2877
+ function settleEvictionThreshold(deps, config, batchId, memberIssue, validated, now) {
2711
2878
  if (validated.length > 0) {
2712
2879
  // #810: never dissolve over validated work — it stays landed and ships
2713
2880
  // with the batch; the members that failed are parked, not requeued.
@@ -2715,19 +2882,19 @@ function evictMemberDirectly(deps, config, batchId, memberIssue, failure, now) {
2715
2882
  issue: memberIssue,
2716
2883
  detail: `eviction threshold reached, but member(s) ${validated.join(',')} are validated on the integration branch — keeping them and continuing with the remaining members instead of dissolving`,
2717
2884
  });
2718
- return { batchEnded: false, duplicate: false };
2885
+ return false;
2719
2886
  }
2720
- // Pass 2 (outside the lock — dissolveBatch shells out): re-load fresh
2721
- // (pass 1's write already landed), dissolve, then re-apply just this
2722
- // batch's + the requeued members' state under a fresh lock.
2887
+ // Outside the lock — dissolveBatch shells out: re-load fresh (pass 1's
2888
+ // write already landed), dissolve, then re-apply just this batch's + the
2889
+ // requeued members' state under a fresh lock.
2723
2890
  const state = deps.store.load();
2724
2891
  const batch = (0, state_1.findBatch)(state, batchId);
2725
2892
  if (!batch)
2726
- return { batchEnded: false, duplicate: false };
2893
+ return false;
2727
2894
  const rDeps = recoveryDeps(deps, config, batch, now);
2728
2895
  const outcome = (0, recovery_1.dissolveBatch)(state, batchId, { strategy: 'full', reason: 'eviction-threshold' }, rDeps);
2729
2896
  applyDissolveOutcome(deps, batchId, outcome, now);
2730
- return { batchEnded: true, duplicate: false };
2897
+ return true;
2731
2898
  }
2732
2899
  function reconcileFixSlot(deps, config, batchId, slot, now, result) {
2733
2900
  if (slot.pid !== null && deps.spawnDeps.isAlive(slot.pid, slot.pid_start ?? undefined))
@@ -3853,6 +4020,7 @@ function teardownBatch(deps, batchId) {
3853
4020
  true;
3854
4021
  const serial = teardownMemberWorktree(deps, batchId, deps.now(), lastMemberLanded);
3855
4022
  teardownParallelRuns(deps, batchId, serial);
4023
+ deleteMemberBranches(deps, batchId);
3856
4024
  const state = deps.store.load();
3857
4025
  const batch = (0, state_1.findBatch)(state, batchId);
3858
4026
  if (!batch || batch.worktree === null)
@@ -3876,6 +4044,30 @@ function teardownBatch(deps, batchId) {
3876
4044
  worktree: batch.worktree,
3877
4045
  });
3878
4046
  }
4047
+ /**
4048
+ * #840: `teardownBatch`'s member-branch cleanup — `pruneMemberBranches`
4049
+ * (recovery.ts) plus the journal line. Best-effort: a failed listing or delete
4050
+ * is journaled, never thrown into the teardown path; a kept batch
4051
+ * (`worktree_kept`) never gets here.
4052
+ */
4053
+ function deleteMemberBranches(deps, batchId) {
4054
+ const pruned = (0, recovery_1.pruneMemberBranches)(deps.exec, deps.repoDir, deps.store.load(), batchId);
4055
+ if (pruned === null) {
4056
+ journalEvent(deps, 'teardown-failed', unit(batchId), {
4057
+ cleanup: 'member-branches-unlisted',
4058
+ detail: `could not list origin's batch/${batchId}-m* member branches — none deleted`,
4059
+ });
4060
+ return;
4061
+ }
4062
+ const { deleted, kept, failed } = pruned;
4063
+ if (deleted.length + kept.length + failed.length === 0)
4064
+ return;
4065
+ journalEvent(deps, failed.length === 0 ? 'member-branches-deleted' : 'teardown-failed', unit(batchId), {
4066
+ ...(failed.length === 0 ? {} : { cleanup: 'member-branches-delete-failed' }),
4067
+ detail: `deleted=${deleted.join(',') || 'none'} kept=${kept.join(',') || 'none'}` +
4068
+ (failed.length > 0 ? ` failed=${failed.join(',')}` : ''),
4069
+ });
4070
+ }
3879
4071
  // --- Parallel member dispatch (#809) ---
3880
4072
  /**
3881
4073
  * How a batch should dispatch its members (#809) — decided once, at the
@@ -4054,7 +4246,10 @@ function spawnParallelMembers(deps, config, dispatch, batchId, now, result) {
4054
4246
  evictMemberDirectly(deps, config, batchId, next.issue, {
4055
4247
  reason: `member-worktree-prep-failed:${prep.reason}`,
4056
4248
  detail: `member worktree prep failed: ${prep.reason} (member ${next.index} #${next.issue}, branch ${memberBranchFor(batchId, next.index, next.issue)})`,
4057
- }, now);
4249
+ }, now,
4250
+ // #844: a run already recorded (a respawn whose tree vanished) must
4251
+ // not stay `running` once its member is evicted.
4252
+ (s) => patchRun(s, batchId, next.issue, { status: 'evicted', gate_inconclusive: null }, now));
4058
4253
  result.failed.push((0, state_1.batchMemberUnit)(batchId, next.issue));
4059
4254
  continue;
4060
4255
  }
@@ -4115,17 +4310,19 @@ function spawnParallelMembers(deps, config, dispatch, batchId, now, result) {
4115
4310
  * evicted work's only recoverable copy, as in serial mode). A dissolve this
4116
4311
  * tips over is torn down by `teardownBatch`, which covers every live run.
4117
4312
  */
4118
- function evictParallelMember(deps, config, batchId, run, failure, now, result) {
4119
- const { batchEnded, duplicate } = evictMemberDirectly(deps, config, batchId, run.issue, failure, now);
4313
+ function evictParallelMember(deps, config, batchId, run, failure, now, result,
4314
+ /** #844: the member's own slot release — committed in the eviction's write. */
4315
+ release) {
4316
+ const { batchEnded, duplicate } = evictMemberDirectly(deps, config, batchId, run.issue, failure, now,
4317
+ // #844: the run's `evicted` status lands in the eviction's write too — a
4318
+ // separate write left an evicted member's run `running` (slotless, its
4319
+ // tree kept until batch end) when the engine exited in between.
4320
+ (s) => patchRun(release ? release(s) : s, batchId, run.issue, { status: 'evicted', gate_inconclusive: null }, now));
4120
4321
  if (duplicate)
4121
4322
  return;
4122
4323
  result.failed.push(batchEnded ? unit(batchId) : (0, state_1.batchMemberUnit)(batchId, run.issue));
4123
4324
  if (batchEnded)
4124
4325
  return;
4125
- deps.store.withLock((s) => ({
4126
- state: patchRun(s, batchId, run.issue, { status: 'evicted', gate_inconclusive: null }, now),
4127
- result: undefined,
4128
- }));
4129
4326
  teardownRunTree(deps, batchId, run, false, now);
4130
4327
  }
4131
4328
  /**
@@ -4156,8 +4353,7 @@ function reconcileParallelMemberSlot(deps, config, dispatch, batchId, run, slot,
4156
4353
  recordMemberRunLog(deps, dispatch, state0, batchId, run.index, run.issue, slot, now);
4157
4354
  const verdict = evaluateIncrementalGate(deps, batchId, run.worktree, run.issue, now);
4158
4355
  if (verdict.kind === 'failed') {
4159
- deps.store.withLock((s) => ({ state: release(s), result: undefined }));
4160
- evictParallelMember(deps, config, batchId, run, { reason: verdict.reason, detail: verdict.detail }, now, result);
4356
+ evictParallelMember(deps, config, batchId, run, { reason: verdict.reason, detail: verdict.detail }, now, result, release);
4161
4357
  return;
4162
4358
  }
4163
4359
  deps.store.withLock((s) => ({
@@ -4178,7 +4374,7 @@ function reconcileParallelMemberSlot(deps, config, dispatch, batchId, run, slot,
4178
4374
  read.milestone !== null) {
4179
4375
  const decision = decideWrongProcedure(deps, batchId, run.issue, read.milestone, slot, release, () => recordMemberRunLog(deps, dispatch, state0, batchId, run.index, run.issue, slot, now), now);
4180
4376
  if (typeof decision === 'object') {
4181
- evictParallelMember(deps, config, batchId, run, decision, now, result);
4377
+ evictParallelMember(deps, config, batchId, run, decision, now, result, release);
4182
4378
  }
4183
4379
  return;
4184
4380
  }
@@ -4191,8 +4387,7 @@ function reconcileParallelMemberSlot(deps, config, dispatch, batchId, run, slot,
4191
4387
  }
4192
4388
  if (read.blockedNow || read.dead) {
4193
4389
  const lastTool = recordMemberRunLog(deps, dispatch, state0, batchId, run.index, run.issue, slot, now);
4194
- deps.store.withLock((s) => ({ state: release(s), result: undefined }));
4195
- evictParallelMember(deps, config, batchId, run, memberFailureFor(read, lastTool), now, result);
4390
+ evictParallelMember(deps, config, batchId, run, memberFailureFor(read, lastTool), now, result, release);
4196
4391
  }
4197
4392
  }
4198
4393
  /**
@@ -4595,12 +4790,8 @@ function runBatchTick(deps, config, dispatch) {
4595
4790
  const dissolvable = fresh !== undefined && (0, state_1.allowedBatchTransitions)(fresh.status).includes('dissolving');
4596
4791
  if (dissolvable) {
4597
4792
  // #809: a parallel batch holds one slot per running member too.
4598
- for (const liveSlot of (0, state_1.slotsForBatch)(current, batch.id)) {
4599
- if (liveSlot.pid !== null &&
4600
- deps.spawnDeps.isAlive(liveSlot.pid, liveSlot.pid_start ?? undefined)) {
4601
- deps.spawnDeps.kill(liveSlot.pid, liveSlot.pid_start ?? undefined);
4602
- }
4603
- }
4793
+ for (const liveSlot of (0, state_1.slotsForBatch)(current, batch.id))
4794
+ killLiveAgent(deps, liveSlot);
4604
4795
  const outcome = (0, recovery_1.dissolveBatch)(current, batch.id, {
4605
4796
  strategy: 'full',
4606
4797
  reason: `dispatch-profile-missing:${batch.dispatch_profile}`,