@ai-dossier/sched 0.46.0 → 0.47.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -12,7 +12,8 @@
12
12
  * at batch-phase granularity instead of per-issue-phase granularity:
13
13
  *
14
14
  * ```
15
- * ready → executing(member i/N) ⟲ → validating → reviewing → shipping
15
+ * ready → executing(serial: member i/N ⟲ | parallel: members ∥ + ordered landing)
16
+ * → validating → reviewing → shipping
16
17
  * → awaiting-merge → merged → deployed → reported → done
17
18
  * failure rails (RFC F.2/F.8/F.9):
18
19
  * executing → dissolving (a member self-reports blocked, RFC F.1)
@@ -98,6 +99,7 @@ Object.defineProperty(exports, "__esModule", { value: true });
98
99
  exports.memberBranchFor = memberBranchFor;
99
100
  exports.evictMemberAndContinue = evictMemberAndContinue;
100
101
  exports.resumeBlockedGate = resumeBlockedGate;
102
+ exports.memberDispatchModeFor = memberDispatchModeFor;
101
103
  exports.runBatchTick = runBatchTick;
102
104
  const fs = __importStar(require("node:fs"));
103
105
  const path = __importStar(require("node:path"));
@@ -197,7 +199,9 @@ function handleDeadDispatchApiError(deps, batchId, logFile, offset, now, options
197
199
  return false;
198
200
  deps.store.withLock((s) => {
199
201
  let n = (0, dispatch_health_1.recordDispatchApiError)((event, unitId, extra) => journalEvent(deps, event, unitId, extra), s, unit(batchId), apiError);
200
- if (options.release ?? true)
202
+ if (options.releaseWith)
203
+ n = options.releaseWith(n);
204
+ else if (options.release ?? true)
201
205
  n = releaseSlot(n, batchId, now);
202
206
  return { state: n, result: undefined };
203
207
  });
@@ -789,13 +793,9 @@ landed) {
789
793
  });
790
794
  return;
791
795
  }
792
- // Route through `runTeardown` for the same guarantees `teardownBatch`
793
- // gets: pool returns are verify-first idempotent and refuse to claim
794
- // success unless the pool's own self-check reports the entry `warm`;
795
- // cold removes post-verify the path is gone and unlisted. Nulls first —
796
- // clear the fields BEFORE the exec so a crash mid-teardown doesn't retry
797
- // into a half-cleaned tree (the re-derived path is deterministic if a
798
- // leftover does survive).
796
+ // Nulls first — clear the fields BEFORE the exec so a crash mid-teardown
797
+ // doesn't retry into a half-cleaned tree (the re-derived path is
798
+ // deterministic if a leftover does survive).
799
799
  deps.store.withLock((s) => {
800
800
  const b = (0, state_1.findBatch)(s, batchId);
801
801
  if (!b)
@@ -805,7 +805,29 @@ landed) {
805
805
  result: undefined,
806
806
  };
807
807
  });
808
- const t = (0, teardown_1.runTeardown)(deps.exec, deps.repoDir, { worktree, poolClaimed: batch.member_pool_claimed === true, branch }, deps.fsExists);
808
+ teardownMemberTree(deps, batchId, { worktree, branch, poolClaimed: batch.member_pool_claimed === true }, landed);
809
+ }
810
+ /**
811
+ * The exec half of a member teardown, shared by the serial rail
812
+ * (`teardownMemberWorktree`, which first clears the batch's member fields) and
813
+ * the parallel rail (#809: `evictParallelMember`, `advanceParallelBatch`,
814
+ * `teardownParallelRuns`). See
815
+ * `teardownMemberWorktree` for the landed/evicted branch policy.
816
+ */
817
+ function teardownMemberTree(deps, batchId, tree, landed) {
818
+ const { worktree, branch } = tree;
819
+ if (branch !== null && !attribution_1.SAFE_REF_RE.test(branch)) {
820
+ journalEvent(deps, 'teardown-failed', unit(batchId), {
821
+ cleanup: 'failed-invalid-branch-name',
822
+ detail: branch,
823
+ });
824
+ return;
825
+ }
826
+ // Route through `runTeardown` for the same guarantees `teardownBatch`
827
+ // gets: pool returns are verify-first idempotent and refuse to claim
828
+ // success unless the pool's own self-check reports the entry `warm`;
829
+ // cold removes post-verify the path is gone and unlisted.
830
+ const t = (0, teardown_1.runTeardown)(deps.exec, deps.repoDir, { worktree, poolClaimed: tree.poolClaimed, branch }, deps.fsExists);
809
831
  const treeCleared = !t.cleanup.startsWith('failed');
810
832
  let branchCleanup = 'skipped';
811
833
  if (branch !== null && treeCleared) {
@@ -858,9 +880,10 @@ function blockLandingFailure(deps, config, batch, reason, now, result) {
858
880
  blockBatchForOperator(deps, config, batch.id, { reason }, now, result);
859
881
  }
860
882
  /**
861
- * Spawn one batch member's member-cycle agent (#677) into the slot a prior
862
- * member (or batch-setup) just released — each member in its OWN worktree
863
- * off the integration branch, dispatched serially as before. Prepared by
883
+ * Spawn one SERIAL batch member's member-cycle agent (#677) into the slot a
884
+ * prior member (or batch-setup) just released — each member in its OWN
885
+ * worktree off the integration branch. Parallel batches (#809) spawn through
886
+ * `spawnParallelMembers` instead, one slot per member. Prepared by
864
887
  * {@link prepareMemberWorktree} at the spawn site (outside the lock); this
865
888
  * function only patches state and spawns.
866
889
  */
@@ -902,6 +925,42 @@ function advanceMemberToValidated(state, memberIssue, now) {
902
925
  return next;
903
926
  }
904
927
  function spawnMember(deps, dispatch, state, slot, batchId, now, result, member) {
928
+ const batch = (0, state_1.findBatch)(state, batchId);
929
+ const memberIssue = batch?.members[(batch?.executing_member ?? 0) - 1];
930
+ if (batch && memberIssue === undefined) {
931
+ journalEvent(deps, 'unit-failed', unit(batchId), {
932
+ reason: 'no-member',
933
+ detail: `spawnMember: executing_member ${batch.executing_member} has no member issue`,
934
+ });
935
+ return releaseSlot(state, batchId, now);
936
+ }
937
+ const spawned = spawnMemberAgent(deps, dispatch, state, slot, batchId, now, result, {
938
+ index: batch?.executing_member ?? 0,
939
+ issue: memberIssue ?? 0,
940
+ member,
941
+ unitId: unit(batchId),
942
+ release: (s) => releaseSlot(s, batchId, now),
943
+ });
944
+ if (!spawned.ok)
945
+ return spawned.state;
946
+ // #677: persist the member's worktree context — a takeover redispatch or a
947
+ // `sched resume --batch` recheck after an engine restart must land in the
948
+ // SAME worktree/branch this prompt names, not re-derive (or worse,
949
+ // re-create) it.
950
+ return (0, state_1.patchBatch)(spawned.state, batchId, {
951
+ member_branch: member.branch,
952
+ member_worktree: member.worktree,
953
+ member_pool_claimed: member.poolClaimed,
954
+ }, now);
955
+ }
956
+ /**
957
+ * Spawn ONE member's member-cycle agent into `slot` — the shared core of the
958
+ * serial rail (`spawnMember`, the batch's own slot) and the parallel rail
959
+ * (`spawnParallelMembers`, #809: the member's own `batch:<id>#<issue>` slot).
960
+ * Patches the slot and the member's queue entry; the caller persists its own
961
+ * member context. On failure the slot is released through `target.release`.
962
+ */
963
+ function spawnMemberAgent(deps, dispatch, state, slot, batchId, now, result, target) {
905
964
  const batch = (0, state_1.findBatch)(state, batchId);
906
965
  if (!batch || batch.worktree === null || batch.branch === null) {
907
966
  // A leaked `assigned` slot with `pid: null` is invisible to `dead`
@@ -911,16 +970,9 @@ function spawnMember(deps, dispatch, state, slot, batchId, now, result, member)
911
970
  reason: 'no-worktree',
912
971
  detail: 'spawnMember: batch has no worktree/branch — batch-setup has not landed',
913
972
  });
914
- return releaseSlot(state, batchId, now);
915
- }
916
- const memberIssue = batch.members[batch.executing_member - 1];
917
- if (memberIssue === undefined) {
918
- journalEvent(deps, 'unit-failed', unit(batchId), {
919
- reason: 'no-member',
920
- detail: `spawnMember: executing_member ${batch.executing_member} has no member issue`,
921
- });
922
- return releaseSlot(state, batchId, now);
973
+ return { ok: false, state: target.release(state) };
923
974
  }
975
+ const memberIssue = target.issue;
924
976
  const withStatus = advanceMemberToInWork(state, memberIssue, now);
925
977
  const memberEntry = (0, state_1.findEntry)(withStatus, memberIssue);
926
978
  // #771: a `review=full` member dispatches at `strong` minimum (within this
@@ -932,8 +984,8 @@ function spawnMember(deps, dispatch, state, slot, batchId, now, result, member)
932
984
  // #677: the prompt carries THIS member's own worktree and the integration
933
985
  // branch it was cut from — `member-cycle`'s `worktree`/`integration_branch`
934
986
  // inputs. The shared batch worktree is no longer a member surface at all.
935
- const prompt = (0, dispatch_1.buildMemberPrompt)(dispatch.memberPrompt, memberIssue, batchId, member.worktree, batch.branch, review);
936
- const logFile = (0, dispatch_1.batchMemberLogPath)(deps.store.runsDir, batchId, batch.executing_member, memberIssue);
987
+ const prompt = (0, dispatch_1.buildMemberPrompt)(dispatch.memberPrompt, memberIssue, batchId, target.member.worktree, batch.branch, review);
988
+ const logFile = (0, dispatch_1.batchMemberLogPath)(deps.store.runsDir, batchId, target.index, memberIssue);
937
989
  // #629: captured BEFORE spawning, mirroring `engine.ts`'s own
938
990
  // `spawnAndRecord` — the log is per-role and append-mode, so the size at
939
991
  // this instant is exactly where THIS dispatch's own output starts. Fences
@@ -949,8 +1001,8 @@ function spawnMember(deps, dispatch, state, slot, batchId, now, result, member)
949
1001
  reason: 'spawn-error',
950
1002
  detail: `member #${memberIssue} spawn failed: ${err.message}`,
951
1003
  });
952
- result.failed.push(unit(batchId));
953
- return releaseSlot(withStatus, batchId, now);
1004
+ result.failed.push(target.unitId);
1005
+ return { ok: false, state: target.release(withStatus) };
954
1006
  }
955
1007
  const patch = {
956
1008
  pid,
@@ -963,18 +1015,9 @@ function spawnMember(deps, dispatch, state, slot, batchId, now, result, member)
963
1015
  spawned_at: now.toISOString(),
964
1016
  log_offset_at_spawn: logOffset,
965
1017
  };
966
- let next = slot.status === 'assigned' || slot.status === 'recovering'
1018
+ const next = slot.status === 'assigned' || slot.status === 'recovering'
967
1019
  ? (0, state_1.transitionSlot)(withStatus, slot.id, 'running', patch, now)
968
1020
  : withStatus;
969
- // #677: persist the member's worktree context — a takeover redispatch or a
970
- // `sched resume --batch` recheck after an engine restart must land in the
971
- // SAME worktree/branch this prompt names, not re-derive (or worse,
972
- // re-create) it.
973
- next = (0, state_1.patchBatch)(next, batchId, {
974
- member_branch: member.branch,
975
- member_worktree: member.worktree,
976
- member_pool_claimed: member.poolClaimed,
977
- }, now);
978
1021
  deps.journal.append((0, journal_1.unitEvent)('spawned', unit(batchId), {
979
1022
  pid,
980
1023
  tier,
@@ -983,11 +1026,11 @@ function spawnMember(deps, dispatch, state, slot, batchId, now, result, member)
983
1026
  issue: memberIssue,
984
1027
  ...(0, dispatch_1.journalCmdModelFields)(spawnSpec),
985
1028
  log: logFile,
986
- worktree: member.worktree,
987
- detail: `member ${batch.executing_member}/${batch.members.length}`,
1029
+ worktree: target.member.worktree,
1030
+ detail: `member ${target.index}/${batch.members.length}${batch.member_dispatch === 'parallel' ? ' (parallel)' : ''}`,
988
1031
  }), now);
989
- result.spawned.push(unit(batchId));
990
- return next;
1032
+ result.spawned.push(target.unitId);
1033
+ return { ok: true, state: next };
991
1034
  }
992
1035
  /**
993
1036
  * First claim of a `ready` batch: assign it a slot, run batch-setup (real
@@ -1054,7 +1097,38 @@ function claimAndSetup(deps, config, dispatch, batchId, now, result) {
1054
1097
  pool_claimed: setup.poolClaimed ? 'true' : 'false',
1055
1098
  },
1056
1099
  });
1057
- deps.journal.append((0, journal_1.unitEvent)('batch-setup-done', unit(batchId), { detail: setup.worktree }), now);
1100
+ // #809: decided ONCE, here, and persisted — a batch never changes mode mid-run.
1101
+ const mode = memberDispatchModeFor(deps.store.load(), batch, config);
1102
+ deps.journal.append((0, journal_1.unitEvent)('batch-setup-done', unit(batchId), {
1103
+ detail: `${setup.worktree} (members: ${mode})`,
1104
+ }), now);
1105
+ if (mode === 'parallel') {
1106
+ // Parallel members each claim their OWN slot (`batch:<id>#<issue>`), so
1107
+ // the setup slot is released here and the members' worktrees are prepared
1108
+ // by `spawnParallelMembers` (outside the lock, per member).
1109
+ deps.store.withLock((s) => {
1110
+ const b = (0, state_1.findBatch)(s, batchId);
1111
+ if (!b || b.status !== 'ready') {
1112
+ journalEvent(deps, 'unit-failed', unit(batchId), {
1113
+ reason: 'batch-left-ready-during-setup',
1114
+ detail: `worktree ${setup.worktree} created but batch is now '${b?.status ?? 'gone'}' — orphaned, manual cleanup required`,
1115
+ });
1116
+ return { state: releaseSlot(s, batchId, now), result: undefined };
1117
+ }
1118
+ let next = (0, state_1.patchBatch)(s, batchId, {
1119
+ branch: setup.branch,
1120
+ worktree: setup.worktree,
1121
+ run_id: setup.runId,
1122
+ pool_claimed: setup.poolClaimed,
1123
+ member_dispatch: 'parallel',
1124
+ member_runs: [],
1125
+ }, now);
1126
+ next = (0, state_1.transitionBatch)(next, batchId, 'executing', { executing_member: 0 }, now);
1127
+ return { state: releaseSlot(next, batchId, now), result: undefined };
1128
+ });
1129
+ spawnParallelMembers(deps, config, dispatch, batchId, now, result);
1130
+ return;
1131
+ }
1058
1132
  // #677: prepare member 1's OWN worktree before claiming the spawn — real
1059
1133
  // git/network work, so OUTSIDE the store lock (the same contract
1060
1134
  // `runBatchSetup` just followed). All-or-nothing: a failed prep posts the
@@ -1114,6 +1188,7 @@ function claimAndSetup(deps, config, dispatch, batchId, now, result) {
1114
1188
  worktree: setup.worktree,
1115
1189
  run_id: setup.runId,
1116
1190
  pool_claimed: setup.poolClaimed,
1191
+ member_dispatch: 'serial',
1117
1192
  }, now);
1118
1193
  next = (0, state_1.transitionBatch)(next, batchId, 'executing', { executing_member: 1 }, now);
1119
1194
  const slot = slotFor(next, batchId);
@@ -1369,14 +1444,20 @@ function runValidate(deps, config, dispatch, batchId, now, result) {
1369
1444
  return { state: s, result: false };
1370
1445
  if (b.executing_member < b.members.length) {
1371
1446
  return {
1372
- state: (0, state_1.transitionBatch)(s, batchId, 'executing', { executing_member: b.executing_member + 1 }, now),
1447
+ state: (0, state_1.transitionBatch)(s, batchId, 'executing',
1448
+ // #809: a parallel batch spawns every run-less member itself.
1449
+ isParallelBatch(b) ? {} : { executing_member: b.executing_member + 1 }, now),
1373
1450
  result: true,
1374
1451
  };
1375
1452
  }
1376
1453
  return { state: (0, state_1.transitionBatch)(s, batchId, 'reviewing', {}, now), result: undefined };
1377
1454
  });
1378
1455
  if (admitted) {
1379
- spawnMemberContinuation(deps, config, dispatch, batchId, now, result);
1456
+ const b = (0, state_1.findBatch)(deps.store.load(), batchId);
1457
+ if (b && isParallelBatch(b))
1458
+ spawnParallelMembers(deps, config, dispatch, batchId, now, result);
1459
+ else
1460
+ spawnMemberContinuation(deps, config, dispatch, batchId, now, result);
1380
1461
  return;
1381
1462
  }
1382
1463
  spawnTailAgent(deps, config, dispatch, batchId, now, result);
@@ -1487,7 +1568,7 @@ function evictOffender(deps, config, batchId, offender, attribution, now, result
1487
1568
  return { state: s, result: undefined };
1488
1569
  if (b.executing_member < b.members.length) {
1489
1570
  return {
1490
- state: (0, state_1.transitionBatch)(s, batchId, 'executing', { executing_member: b.executing_member + 1 }, now),
1571
+ state: (0, state_1.transitionBatch)(s, batchId, 'executing', isParallelBatch(b) ? {} : { executing_member: b.executing_member + 1 }, now),
1491
1572
  result: undefined,
1492
1573
  };
1493
1574
  }
@@ -1535,6 +1616,15 @@ function journalAdvanceSkipped(deps, batchId, memberIssue, currentMember, claim)
1535
1616
  });
1536
1617
  }
1537
1618
  function advanceMemberOrValidate(deps, config, dispatch, batchId, memberCount, currentMember, memberIssue, now, result) {
1619
+ // #809: a parallel batch reaches this serial rail only through `sched resume
1620
+ // --batch` resolving its gate-inconclusive member — hand back to the
1621
+ // parallel rail instead of advancing a serial member pointer.
1622
+ const current = (0, state_1.findBatch)(deps.store.load(), batchId);
1623
+ if (current && isParallelBatch(current)) {
1624
+ settleParallelResume(deps, batchId, memberIssue, now);
1625
+ advanceParallelBatch(deps, config, dispatch, batchId, now, result);
1626
+ return;
1627
+ }
1538
1628
  const isLast = currentMember >= memberCount;
1539
1629
  if (isLast) {
1540
1630
  const claim = claimMemberResolution(deps, batchId, currentMember, now, () => ({
@@ -1639,7 +1729,41 @@ function runIncrementalGate(deps, config, dispatch, batchId, batch, memberIssue,
1639
1729
  // the fallback for a batch dispatched before member worktrees existed (its
1640
1730
  // members' commits landed directly on the integration branch).
1641
1731
  const worktree = batch.member_worktree ?? batch.worktree;
1732
+ const verdict = evaluateIncrementalGate(deps, batchId, worktree, memberIssue, now);
1733
+ if (verdict.kind === 'pass')
1734
+ return false;
1735
+ if (verdict.kind === 'failed') {
1736
+ deps.store.withLock((s) => ({ state: releaseSlot(s, batchId, now), result: undefined }));
1737
+ evictMemberAndContinue(deps, config, dispatch, batchId, batch, memberIssue, { reason: verdict.reason, detail: verdict.detail }, now, result);
1738
+ return true;
1739
+ }
1740
+ deps.store.withLock((s) => ({ state: releaseSlot(s, batchId, now), result: undefined }));
1741
+ // #677: F.11's block semantics are "the member's commit stays on the
1742
+ // branch, nothing requeued/reverted" — under member worktrees that is
1743
+ // true only if the member LANDS before the batch blocks. The block means
1744
+ // the gate could not reach a verdict, not that the work is bad, so the
1745
+ // verified member joins the integration branch and is there for an
1746
+ // operator (or the #686 out-of-band reconcile) to act on. A landing
1747
+ // failure blocks the batch with its own reason — same operator outcome.
1748
+ const landed = landMemberBranch(deps, batchId, now);
1749
+ if (!landed.ok) {
1750
+ blockLandingFailure(deps, config, batch, landed.reason, now, result);
1751
+ return true;
1752
+ }
1753
+ blockBatchForOperator(deps, config, batchId, { reason: verdict.reason, milestonePhase: 'batch-review' }, now, result);
1754
+ return true;
1755
+ }
1756
+ /**
1757
+ * Run the incremental gate's capabilities against one member worktree and
1758
+ * reach a verdict — journaling, gate logs and `member_gates` included, but NO
1759
+ * state action: serial (`runIncrementalGate`) and parallel
1760
+ * (`reconcileParallelMemberSlot`, #809) members act on the verdict
1761
+ * differently. See `runIncrementalGate`'s doc for the policy.
1762
+ */
1763
+ function evaluateIncrementalGate(deps, batchId, worktree, memberIssue, now) {
1642
1764
  const runCapability = deps.runCapability;
1765
+ if (!runCapability)
1766
+ return { kind: 'pass' };
1643
1767
  const gateResults = ['typecheck.run', 'test.focused'].map((id) => ({
1644
1768
  id,
1645
1769
  ...runCapability(worktree, id),
@@ -1680,13 +1804,10 @@ function runIncrementalGate(deps, config, dispatch, batchId, batch, memberIssue,
1680
1804
  // trap (#594's shape — absence reading as a verdict). Silence must never
1681
1805
  // be mistaken for a pass.
1682
1806
  //
1683
- // #632: confirmed this fires once per member, not once per tick.
1684
- // `runIncrementalGate` is called only from the `isMemberComplete` branch
1685
- // of `reconcileMemberSlot`, and both of that branch's callees
1686
- // (`completeMemberGate`, `evictMemberAndContinue`) advance
1687
- // `batch.executing_member` before returning — so the very next tick
1688
- // reads a DIFFERENT `memberIssue` from `batch.members[executing_member -
1689
- // 1]`, and this gate never runs twice against the same member.
1807
+ // #632: confirmed this fires once per member, not once per tick: the
1808
+ // gate runs only on a member dispatch's verified completion, and both
1809
+ // rails resolve that member (serial: the pointer advances; parallel:
1810
+ // the run leaves `running` and its slot is released) before returning.
1690
1811
  journalEvent(deps, 'gate-skipped', unit(batchId), {
1691
1812
  issue: memberIssue,
1692
1813
  reason: `gate-skipped:${skipped.id}`,
@@ -1734,15 +1855,13 @@ function runIncrementalGate(deps, config, dispatch, batchId, batch, memberIssue,
1734
1855
  recordMemberGate(deps, batchId, memberIssue, decisiveGate, now);
1735
1856
  }
1736
1857
  if (earnedFailure) {
1737
- const reason = `incremental-gate-failed:${earnedFailure.id}`;
1738
1858
  writeGateLog(deps, batchId, earnedFailure.id, memberIssue, earnedFailure.outputTail);
1739
1859
  const excerpt = gateDetailExcerpt(earnedFailure.outputTail, earnedFailure.reason);
1740
- deps.store.withLock((s) => ({ state: releaseSlot(s, batchId, now), result: undefined }));
1741
- evictMemberAndContinue(deps, config, dispatch, batchId, batch, memberIssue, {
1742
- reason,
1860
+ return {
1861
+ kind: 'failed',
1862
+ reason: `incremental-gate-failed:${earnedFailure.id}`,
1743
1863
  detail: withExcerpt(`cap run ${earnedFailure.id} reported task-failed with failing-test evidence after member review done`, excerpt),
1744
- }, now, result);
1745
- return true;
1864
+ };
1746
1865
  }
1747
1866
  const inconclusive = gateInconclusive ?? unevidencedFailure;
1748
1867
  if (inconclusive) {
@@ -1758,23 +1877,9 @@ function runIncrementalGate(deps, config, dispatch, batchId, batch, memberIssue,
1758
1877
  reason,
1759
1878
  detail: withExcerpt(`cap run ${inconclusive.id} ${describeInconclusive(inconclusive)} after member review done`, excerpt),
1760
1879
  });
1761
- deps.store.withLock((s) => ({ state: releaseSlot(s, batchId, now), result: undefined }));
1762
- // #677: F.11's block semantics are "the member's commit stays on the
1763
- // branch, nothing requeued/reverted" — under member worktrees that is
1764
- // true only if the member LANDS before the batch blocks. The block means
1765
- // the gate could not reach a verdict, not that the work is bad, so the
1766
- // verified member joins the integration branch and is there for an
1767
- // operator (or the #686 out-of-band reconcile) to act on. A landing
1768
- // failure blocks the batch with its own reason — same operator outcome.
1769
- const landed = landMemberBranch(deps, batchId, now);
1770
- if (!landed.ok) {
1771
- blockLandingFailure(deps, config, batch, landed.reason, now, result);
1772
- return true;
1773
- }
1774
- blockBatchForOperator(deps, config, batchId, { reason, milestonePhase: 'batch-review' }, now, result);
1775
- return true;
1880
+ return { kind: 'inconclusive', reason, capability: inconclusive.id };
1776
1881
  }
1777
- return false;
1882
+ return { kind: 'pass' };
1778
1883
  }
1779
1884
  /**
1780
1885
  * The member is verifiably done — self-reported complete AND (when checked)
@@ -1988,7 +2093,9 @@ function resumeBlockedGate(deps, config, dispatch, batchId, now) {
1988
2093
  * instead), so unlike `engine.ts`'s per-unit log, a member's log file is
1989
2094
  * always one-shot and reading from offset 0 is always correct.
1990
2095
  */
1991
- function recordMemberRunLog(deps, dispatch, state, batchId, batch, memberIssue, slot, now) {
2096
+ function recordMemberRunLog(deps, dispatch, state, batchId,
2097
+ /** 1-based member index — names the per-member log file (serial: `executing_member`; parallel: `MemberRun.index`). */
2098
+ memberIndex, memberIssue, slot, now) {
1992
2099
  if (slot.status !== 'running' || slot.spawned_at === null) {
1993
2100
  journalEvent(deps, 'run-log-skipped', unit(batchId), {
1994
2101
  issue: memberIssue,
@@ -2002,7 +2109,7 @@ function recordMemberRunLog(deps, dispatch, state, batchId, batch, memberIssue,
2002
2109
  const memberEntry = (0, state_1.findEntry)(state, memberIssue);
2003
2110
  const tier = memberEntry ? (0, dispatch_1.memberDispatchTier)(memberEntry) : 'mid';
2004
2111
  const { cmd, model } = (0, dispatch_1.resolveTierSpawn)(dispatch, tier, memberIssue);
2005
- const logFile = (0, dispatch_1.batchMemberLogPath)(deps.store.runsDir, batchId, batch.executing_member, memberIssue);
2112
+ const logFile = (0, dispatch_1.batchMemberLogPath)(deps.store.runsDir, batchId, memberIndex, memberIssue);
2006
2113
  // #629: fenced to THIS dispatch's slice (`log_offset_at_spawn`, stamped by
2007
2114
  // `spawnMember`/`spawnMemberContinuation` at spawn time) rather than a
2008
2115
  // fixed byte-0 read. Before #629 a member log never outlived one attempt
@@ -2070,14 +2177,12 @@ function terminalMilestoneForDispatch(deps, memberIssue, slot) {
2070
2177
  }
2071
2178
  return latest;
2072
2179
  }
2073
- function reconcileMemberSlot(deps, config, dispatch, batchId, slot, now, result) {
2074
- const state0 = deps.store.load();
2075
- const batch = (0, state_1.findBatch)(state0, batchId);
2076
- if (!batch)
2077
- return;
2078
- const memberIssue = batch.members[batch.executing_member - 1];
2079
- if (memberIssue === undefined)
2080
- return;
2180
+ /**
2181
+ * Read one member dispatch's fate: the fenced terminal milestone (#575/#605/
2182
+ * #622) plus liveness, journaling a stale terminal milestone once per
2183
+ * dispatch (#610). Pure reading — the caller acts on the result.
2184
+ */
2185
+ function readMemberDispatch(deps, batchId, memberIssue, slot, now) {
2081
2186
  const dead = slot.pid !== null && !deps.spawnDeps.isAlive(slot.pid, slot.pid_start ?? undefined);
2082
2187
  // #622: the milestone this DISPATCH's fate should be read from — the last
2083
2188
  // TERMINAL one it posted, not merely the newest one on the issue.
@@ -2096,7 +2201,7 @@ function reconcileMemberSlot(deps, config, dispatch, batchId, slot, now, result)
2096
2201
  // poll) this degrades to the previous latest-only behaviour.
2097
2202
  const milestone = terminalMilestoneForDispatch(deps, memberIssue, slot);
2098
2203
  if (milestone === undefined)
2099
- return; // unreachable — pause this batch's decisions
2204
+ return { kind: 'unreachable' }; // pause this batch's decisions
2100
2205
  // #575 / #605: fence to THIS member dispatch's `spawned_at` — a member
2101
2206
  // re-added to a fresh batch run after a PREVIOUS batch already posted a
2102
2207
  // terminal `mode=slot` milestone (pilot re-run, requeue-with-context, and
@@ -2135,7 +2240,61 @@ function reconcileMemberSlot(deps, config, dispatch, batchId, slot, now, result)
2135
2240
  result: undefined,
2136
2241
  }));
2137
2242
  }
2138
- if ((0, groundtruth_1.isMemberComplete)(milestone, slot.spawned_at)) {
2243
+ if ((0, groundtruth_1.isMemberComplete)(milestone, slot.spawned_at))
2244
+ return { kind: 'complete' };
2245
+ return {
2246
+ kind: 'pending',
2247
+ milestone,
2248
+ dead,
2249
+ blockedNow: (0, groundtruth_1.isMemberBlocked)(milestone, slot.spawned_at),
2250
+ };
2251
+ }
2252
+ /**
2253
+ * The eviction a blocked-or-dead member dispatch earns (shared by both rails).
2254
+ * #605: only a milestone THIS dispatch posted may name the reason. A dead
2255
+ * agent whose issue carries only a stale `blocked` milestone is an unverified
2256
+ * exit, not a re-run of the previous run's block — reporting the old
2257
+ * `reason=` here is what made the journal misdirect, naming a precondition
2258
+ * that had already been fixed on a run where it no longer applied.
2259
+ * #804: `memberBlockedReason` also names a `review partial` hand-back.
2260
+ */
2261
+ function memberFailureFor(read, lastTool) {
2262
+ const rawReason = read.blockedNow && read.milestone ? (0, groundtruth_1.memberBlockedReason)(read.milestone) : undefined;
2263
+ const reason = typeof rawReason === 'string' && rawReason.length > 0
2264
+ ? sanitizeUntrustedText(rawReason)
2265
+ : read.dead
2266
+ ? 'agent-exited-unverified'
2267
+ : 'member-blocked';
2268
+ return {
2269
+ reason,
2270
+ // This rail covers a self-reported block AND a member that simply died, so the
2271
+ // detail must follow the RESOLVED reason — `member blocked` on an
2272
+ // `agent-exited-unverified` eviction contradicts the reason on its own journal line.
2273
+ detail: reason === 'agent-exited-unverified'
2274
+ ? 'member agent exited without posting a terminal milestone'
2275
+ : 'member blocked',
2276
+ // #591: attributes an unverified member exit to a concrete cause (e.g.
2277
+ // `Monitor`) without opening the transcript. Gated on the RESOLVED reason, not
2278
+ // `dead` — a member can be simultaneously `dead` AND carry a milestone-posted
2279
+ // `reason` (it posted `blocked` and then exited), and that real block has no
2280
+ // log-derived cause to attribute; only `agent-exited-unverified` does.
2281
+ extraKv: reason === 'agent-exited-unverified' && lastTool !== null
2282
+ ? { last_tool: lastTool }
2283
+ : undefined,
2284
+ };
2285
+ }
2286
+ function reconcileMemberSlot(deps, config, dispatch, batchId, slot, now, result) {
2287
+ const state0 = deps.store.load();
2288
+ const batch = (0, state_1.findBatch)(state0, batchId);
2289
+ if (!batch)
2290
+ return;
2291
+ const memberIssue = batch.members[batch.executing_member - 1];
2292
+ if (memberIssue === undefined)
2293
+ return;
2294
+ const read = readMemberDispatch(deps, batchId, memberIssue, slot, now);
2295
+ if (read.kind === 'unreachable')
2296
+ return;
2297
+ if (read.kind === 'complete') {
2139
2298
  deps.journal.append((0, journal_1.unitEvent)('external-advance', unit(batchId), {
2140
2299
  issue: memberIssue,
2141
2300
  detail: 'member review done',
@@ -2143,7 +2302,7 @@ function reconcileMemberSlot(deps, config, dispatch, batchId, slot, now, result)
2143
2302
  // The member's own agent process is done regardless of what the
2144
2303
  // incremental gate below decides — record its telemetry once here (#564)
2145
2304
  // rather than at each of this branch's two later exit points.
2146
- recordMemberRunLog(deps, dispatch, state0, batchId, batch, memberIssue, slot, now);
2305
+ recordMemberRunLog(deps, dispatch, state0, batchId, batch.executing_member, memberIssue, slot, now);
2147
2306
  // Incremental gate (#523 AC2, revised #583) — see `runIncrementalGate`'s
2148
2307
  // own doc comment for the three-way policy. A `true` return means the
2149
2308
  // gate already decided the member's fate (evicted or blocked) and
@@ -2155,9 +2314,8 @@ function reconcileMemberSlot(deps, config, dispatch, batchId, slot, now, result)
2155
2314
  completeMemberGate(deps, config, dispatch, batchId, batch, memberIssue, now, result);
2156
2315
  return;
2157
2316
  }
2158
- const blockedNow = (0, groundtruth_1.isMemberBlocked)(milestone, slot.spawned_at);
2159
- if (dead &&
2160
- !blockedNow &&
2317
+ if (read.dead &&
2318
+ !read.blockedNow &&
2161
2319
  handleDeadDispatchApiError(deps, batchId, (0, dispatch_1.batchMemberLogPath)(deps.store.runsDir, batchId, batch.executing_member, memberIssue), slot.log_offset_at_spawn ?? 0, now)) {
2162
2320
  // #629: a confirmed provider API error is not a real member failure —
2163
2321
  // evicting the member for an account-wide spend wall would silently
@@ -2166,20 +2324,8 @@ function reconcileMemberSlot(deps, config, dispatch, batchId, slot, now, result)
2166
2324
  // place, keeping its position.
2167
2325
  return;
2168
2326
  }
2169
- if (blockedNow || dead) {
2170
- // #605: only a milestone THIS dispatch posted may name the reason. A dead
2171
- // agent whose issue carries only a stale `blocked` milestone is an
2172
- // unverified exit, not a re-run of the previous run's block — reporting
2173
- // the old `reason=` here is what made the journal misdirect, naming a
2174
- // precondition that had already been fixed on a run where it no longer
2175
- // applied.
2176
- const rawReason = blockedNow ? milestone?.keys.reason : undefined;
2177
- const reason = typeof rawReason === 'string' && rawReason.length > 0
2178
- ? sanitizeUntrustedText(rawReason)
2179
- : dead
2180
- ? 'agent-exited-unverified'
2181
- : 'member-blocked';
2182
- const lastTool = recordMemberRunLog(deps, dispatch, state0, batchId, batch, memberIssue, slot, now);
2327
+ if (read.blockedNow || read.dead) {
2328
+ const lastTool = recordMemberRunLog(deps, dispatch, state0, batchId, batch.executing_member, memberIssue, slot, now);
2183
2329
  deps.store.withLock((s) => ({ state: releaseSlot(s, batchId, now), result: undefined }));
2184
2330
  // A member that never went green (RFC F.1) evicts DIRECTLY — no aggregate
2185
2331
  // suite has run yet, so there is nothing for `attributing`/`evicting` (the
@@ -2189,23 +2335,7 @@ function reconcileMemberSlot(deps, config, dispatch, batchId, slot, now, result)
2189
2335
  // attributing → evicting` is not even a legal edge from mid-`executing`
2190
2336
  // (BATCH_TRANSITIONS has no `executing → evicting`) — this stays entirely
2191
2337
  // within `executing`/`dissolving`, both of which ARE legal from here.
2192
- evictMemberAndContinue(deps, config, dispatch, batchId, batch, memberIssue, {
2193
- reason,
2194
- // This rail covers a self-reported block AND a member that simply died, so the
2195
- // detail must follow the RESOLVED reason — `member blocked` on an
2196
- // `agent-exited-unverified` eviction contradicts the reason on its own journal line.
2197
- detail: reason === 'agent-exited-unverified'
2198
- ? 'member agent exited without posting a terminal milestone'
2199
- : 'member blocked',
2200
- // #591: attributes an unverified member exit to a concrete cause (e.g.
2201
- // `Monitor`) without opening the transcript. Gated on the RESOLVED reason, not
2202
- // `dead` — a member can be simultaneously `dead` AND carry a milestone-posted
2203
- // `reason` (it posted `blocked` and then exited), and that real block has no
2204
- // log-derived cause to attribute; only `agent-exited-unverified` does.
2205
- extraKv: reason === 'agent-exited-unverified' && lastTool !== null
2206
- ? { last_tool: lastTool }
2207
- : undefined,
2208
- }, now, result);
2338
+ evictMemberAndContinue(deps, config, dispatch, batchId, batch, memberIssue, memberFailureFor(read, lastTool), now, result);
2209
2339
  }
2210
2340
  }
2211
2341
  /**
@@ -3006,7 +3136,9 @@ function teardownBatch(deps, batchId) {
3006
3136
  const terminal = (0, state_1.findBatch)(deps.store.load(), batchId);
3007
3137
  const lastMemberLanded = terminal?.ranges.some((r) => r.issue === terminal.members[terminal.executing_member - 1]) ===
3008
3138
  true;
3139
+ const serialTree = terminal?.member_worktree ?? null;
3009
3140
  teardownMemberWorktree(deps, batchId, deps.now(), lastMemberLanded);
3141
+ teardownParallelRuns(deps, batchId, serialTree);
3010
3142
  const state = deps.store.load();
3011
3143
  const batch = (0, state_1.findBatch)(state, batchId);
3012
3144
  if (!batch || batch.worktree === null)
@@ -3030,6 +3162,615 @@ function teardownBatch(deps, batchId) {
3030
3162
  worktree: batch.worktree,
3031
3163
  });
3032
3164
  }
3165
+ // --- Parallel member dispatch (#809) ---
3166
+ /**
3167
+ * How a batch should dispatch its members (#809) — decided once, at the
3168
+ * `ready → executing` claim, and persisted on `BatchEntry.member_dispatch`.
3169
+ *
3170
+ * Serial when the operator set `member_parallelism: 1`, or the batch itself
3171
+ * says its members are not independent: an eviction group (members that must
3172
+ * revert together — e.g. one built on another's API) or a member whose `deps`
3173
+ * names another member (it must see that member's landed work before it is
3174
+ * cut). Parallel otherwise — the batch-cycle skill's and prep dossier's
3175
+ * design (RFC-0001 §J.3: each member works off the integration branch and
3176
+ * sees no one else's changes until the parent lands them).
3177
+ */
3178
+ /** @internal exported for tests. */
3179
+ function memberDispatchModeFor(state, batch, config) {
3180
+ if (config.member_parallelism === 1)
3181
+ return 'serial';
3182
+ if (batch.eviction_groups.some((group) => group.length > 1))
3183
+ return 'serial';
3184
+ const members = new Set(batch.members);
3185
+ for (const issue of batch.members) {
3186
+ if ((0, state_1.findEntry)(state, issue)?.deps.some((dep) => members.has(dep)))
3187
+ return 'serial';
3188
+ }
3189
+ return 'parallel';
3190
+ }
3191
+ /** Whether `batch` runs the parallel member rail — `null` (pre-#809, or not yet claimed) is serial. */
3192
+ function isParallelBatch(batch) {
3193
+ return batch.member_dispatch === 'parallel';
3194
+ }
3195
+ /** At most this many members of one batch run at once (#809) — `max_slots` when unset. */
3196
+ function memberParallelism(config) {
3197
+ // Both are validated ≥ 1 at config load (`member_parallelism`, `max_slots`).
3198
+ return config.member_parallelism ?? config.max_slots;
3199
+ }
3200
+ /** The teardown target of a member run (or a freshly prepared member tree). */
3201
+ function runTree(r) {
3202
+ return 'pool_claimed' in r
3203
+ ? { worktree: r.worktree, branch: r.branch, poolClaimed: r.pool_claimed }
3204
+ : { worktree: r.worktree, branch: r.branch, poolClaimed: r.poolClaimed };
3205
+ }
3206
+ /**
3207
+ * Record that a run's tree is gone. Written AFTER the teardown, so a crash in
3208
+ * between re-runs the (idempotent) teardown from `teardownParallelRuns`
3209
+ * instead of leaking the tree.
3210
+ */
3211
+ function markTornDown(deps, batchId, issue, now) {
3212
+ deps.store.withLock((s) => ({
3213
+ state: patchRun(s, batchId, issue, { torn_down: true }, now),
3214
+ result: undefined,
3215
+ }));
3216
+ }
3217
+ /** Merge `patch` into member `issue`'s run record (no-op when it has none). */
3218
+ function patchRun(state, batchId, issue, patch, now) {
3219
+ const batch = (0, state_1.findBatch)(state, batchId);
3220
+ if (!batch || !batch.member_runs.some((r) => r.issue === issue))
3221
+ return state;
3222
+ return (0, state_1.patchBatch)(state, batchId, { member_runs: batch.member_runs.map((r) => (r.issue === issue ? { ...r, ...patch } : r)) }, now);
3223
+ }
3224
+ /** Insert or replace member `run.issue`'s run record. */
3225
+ function upsertRun(state, batchId, run, now) {
3226
+ const batch = (0, state_1.findBatch)(state, batchId);
3227
+ if (!batch)
3228
+ return state;
3229
+ const others = batch.member_runs.filter((r) => r.issue !== run.issue);
3230
+ return (0, state_1.patchBatch)(state, batchId, { member_runs: [...others, run].sort((a, b) => a.index - b.index) }, now);
3231
+ }
3232
+ /**
3233
+ * Whether `issue` is still an active member of `batch` — not evicted, and its
3234
+ * queue entry (when it has one) still a live slot-mode entry of THIS batch. An
3235
+ * evicted member's entry is requeued `mode: 'full', batch: null`, so both
3236
+ * checks agree; the entry check also covers a member stopped or abandoned out
3237
+ * from under the batch.
3238
+ */
3239
+ function memberStillInBatch(state, batch, issue) {
3240
+ if (batch.evictions.some((e) => e.issue === issue))
3241
+ return false;
3242
+ const entry = (0, state_1.findEntry)(state, issue);
3243
+ if (entry === undefined)
3244
+ return true;
3245
+ return (entry.mode === 'slot' && entry.batch === batch.id && !types_1.TERMINAL_ISSUE_STATUSES.has(entry.status));
3246
+ }
3247
+ /** Whether member `issue`'s own slot currently holds a live agent. */
3248
+ function hasLiveMemberSlot(state, batchId, issue) {
3249
+ const slot = (0, state_1.slotForBatchMember)(state, batchId, issue);
3250
+ return slot !== undefined && types_1.LIVE_SLOT_STATUSES.has(slot.status);
3251
+ }
3252
+ /**
3253
+ * The next member to (re)spawn: first a run still `running` that holds no slot
3254
+ * (an api-error hold released it, its spawn threw, or the engine restarted) —
3255
+ * respawned in place, in its own worktree — then the first member in order
3256
+ * that has no run yet.
3257
+ */
3258
+ function nextParallelCandidate(state, batch, attempted) {
3259
+ for (const run of batch.member_runs) {
3260
+ if (run.status === 'running' &&
3261
+ !attempted.has(run.issue) &&
3262
+ (0, state_1.slotForBatchMember)(state, batch.id, run.issue) === undefined &&
3263
+ memberStillInBatch(state, batch, run.issue)) {
3264
+ return { issue: run.issue, index: run.index, run };
3265
+ }
3266
+ }
3267
+ for (const [i, issue] of batch.members.entries()) {
3268
+ if (attempted.has(issue) || batch.member_runs.some((r) => r.issue === issue))
3269
+ continue;
3270
+ if (!memberStillInBatch(state, batch, issue))
3271
+ continue;
3272
+ return { issue, index: i + 1 };
3273
+ }
3274
+ return undefined;
3275
+ }
3276
+ /**
3277
+ * Fill the batch's member concurrency: spawn members into their OWN slots
3278
+ * (`batch:<id>#<issue>`) while fewer than `member_parallelism` run and the
3279
+ * scheduler has free capacity. Each gets its own worktree off the integration
3280
+ * branch (`prepareMemberWorktree`, outside the lock) — the integration branch
3281
+ * does not move until members land, so concurrent members are all cut from the
3282
+ * same base. A member whose worktree cannot be prepared is evicted (requeued
3283
+ * full-cycle, which does its own setup) rather than retried every tick in
3284
+ * front of the members behind it.
3285
+ */
3286
+ function spawnParallelMembers(deps, config, dispatch, batchId, now, result) {
3287
+ const limit = memberParallelism(config);
3288
+ const fsExists = deps.fsExists ?? ((p) => fs.existsSync(p));
3289
+ const attempted = new Set();
3290
+ // Bounded by the member count: every iteration marks one member attempted.
3291
+ for (;;) {
3292
+ const state = deps.store.load();
3293
+ const batch = (0, state_1.findBatch)(state, batchId);
3294
+ if (!batch || batch.status !== 'executing' || !isParallelBatch(batch))
3295
+ return;
3296
+ const live = batch.member_runs.filter((r) => hasLiveMemberSlot(state, batchId, r.issue)).length;
3297
+ if (live >= limit || (0, scheduler_1.freeCapacity)(state, config) === 0)
3298
+ return;
3299
+ const next = nextParallelCandidate(state, batch, attempted);
3300
+ if (next === undefined)
3301
+ return;
3302
+ attempted.add(next.issue);
3303
+ let member;
3304
+ const reusable = next.run !== undefined &&
3305
+ fsExists(next.run.worktree) &&
3306
+ (next.run.pool_claimed ||
3307
+ (0, teardown_1.isSafeWorktree)(path.resolve(deps.exec('git', ['rev-parse', '--show-toplevel'], deps.repoDir) ?? deps.repoDir), next.run.worktree));
3308
+ if (next.run !== undefined && reusable) {
3309
+ member = {
3310
+ branch: next.run.branch,
3311
+ worktree: next.run.worktree,
3312
+ poolClaimed: next.run.pool_claimed,
3313
+ };
3314
+ }
3315
+ else {
3316
+ // The serial-member fields describe no member of a parallel batch —
3317
+ // blank them so the reuse branch re-derives THIS member's branch/path.
3318
+ const prep = prepareMemberWorktree(deps, { ...batch, member_branch: null, member_worktree: null, member_pool_claimed: false }, next.index, next.issue, now);
3319
+ if (!prep.ok) {
3320
+ evictMemberDirectly(deps, config, batchId, next.issue, {
3321
+ reason: `member-worktree-prep-failed:${prep.reason}`,
3322
+ detail: `member worktree prep failed: ${prep.reason} (member ${next.index} #${next.issue}, branch ${memberBranchFor(batchId, next.index, next.issue)})`,
3323
+ }, now);
3324
+ result.failed.push((0, state_1.batchMemberUnit)(batchId, next.issue));
3325
+ continue;
3326
+ }
3327
+ member = prep;
3328
+ }
3329
+ const recorded = deps.store.withLock((s) => {
3330
+ const b = (0, state_1.findBatch)(s, batchId);
3331
+ if (!b || b.status !== 'executing' || (0, state_1.slotForBatchMember)(s, batchId, next.issue)) {
3332
+ return { state: s, result: false };
3333
+ }
3334
+ // The run is recorded whether or not a slot is free and the spawn
3335
+ // succeeds: its worktree exists either way, and a `running` run with no
3336
+ // slot is exactly what a later pass respawns in place.
3337
+ let n = upsertRun(s, batchId, {
3338
+ issue: next.issue,
3339
+ index: next.index,
3340
+ branch: member.branch,
3341
+ worktree: member.worktree,
3342
+ pool_claimed: member.poolClaimed,
3343
+ status: 'running',
3344
+ gate_inconclusive: null,
3345
+ torn_down: false,
3346
+ }, now);
3347
+ if ((0, scheduler_1.freeCapacity)(n, config) === 0)
3348
+ return { state: n, result: true };
3349
+ const memberUnit = (0, state_1.batchMemberUnit)(batchId, next.issue);
3350
+ const assigned = (0, scheduler_1.assignToIdleSlot)(n, memberUnit, 'member', now, 'cycle');
3351
+ const slot = assigned.state.slots.find((x) => x.id === assigned.slotId);
3352
+ if (!slot)
3353
+ return { state: n, result: true };
3354
+ n = spawnMemberAgent(deps, dispatch, assigned.state, slot, batchId, now, result, {
3355
+ index: next.index,
3356
+ issue: next.issue,
3357
+ member,
3358
+ unitId: memberUnit,
3359
+ release: (st) => (0, state_1.releaseBatchMemberSlot)(st, batchId, next.issue, now),
3360
+ }).state;
3361
+ n = (0, state_1.patchBatch)(n, batchId, { executing_member: Math.max(b.executing_member, next.index) }, now);
3362
+ return { state: n, result: true };
3363
+ });
3364
+ if (!recorded && next.run === undefined) {
3365
+ // The batch left `executing` (stopped/abandoned/dissolved) while the
3366
+ // tree was being prepared outside the lock — nothing records it, so
3367
+ // give it back now rather than leak a worktree/pool claim.
3368
+ journalEvent(deps, 'unit-failed', unit(batchId), {
3369
+ issue: next.issue,
3370
+ reason: 'member-worktree-orphaned',
3371
+ detail: `batch left executing while member ${next.index}'s tree ${member.worktree} was prepared — tearing it down`,
3372
+ });
3373
+ teardownMemberTree(deps, batchId, runTree(member), false);
3374
+ }
3375
+ }
3376
+ }
3377
+ /**
3378
+ * Evict one parallel member directly (RFC F.1 — its commits never landed) and
3379
+ * tear its worktree down; the pushed member branch is KEPT on origin (the
3380
+ * evicted work's only recoverable copy, as in serial mode). A dissolve this
3381
+ * tips over is torn down by `teardownBatch`, which covers every live run.
3382
+ */
3383
+ function evictParallelMember(deps, config, batchId, run, failure, now, result) {
3384
+ const { dissolved, duplicate } = evictMemberDirectly(deps, config, batchId, run.issue, failure, now);
3385
+ if (duplicate)
3386
+ return;
3387
+ result.failed.push(dissolved ? unit(batchId) : (0, state_1.batchMemberUnit)(batchId, run.issue));
3388
+ if (dissolved)
3389
+ return;
3390
+ deps.store.withLock((s) => ({
3391
+ state: patchRun(s, batchId, run.issue, { status: 'evicted', gate_inconclusive: null }, now),
3392
+ result: undefined,
3393
+ }));
3394
+ teardownMemberTree(deps, batchId, runTree(run), false);
3395
+ markTornDown(deps, batchId, run.issue, now);
3396
+ }
3397
+ /**
3398
+ * Reconcile one parallel member's slot — the parallel twin of
3399
+ * `reconcileMemberSlot`: the same fenced milestone read, the same incremental
3400
+ * gate (in the member's OWN worktree), the same eviction rail. A verified
3401
+ * member does NOT land here: landing is ordered (`advanceParallelBatch`), so
3402
+ * it only leaves `running` and gives its slot back.
3403
+ */
3404
+ function reconcileParallelMemberSlot(deps, config, dispatch, batchId, run, slot, now, result) {
3405
+ const state0 = deps.store.load();
3406
+ const read = readMemberDispatch(deps, batchId, run.issue, slot, now);
3407
+ if (read.kind === 'unreachable')
3408
+ return;
3409
+ const memberUnit = (0, state_1.batchMemberUnit)(batchId, run.issue);
3410
+ const release = (s) => (0, state_1.releaseBatchMemberSlot)(s, batchId, run.issue, now);
3411
+ if (read.kind === 'complete') {
3412
+ // Its slot is about to be released and its tree rebased/force-pushed at
3413
+ // landing: an agent still running after `review done` would keep writing
3414
+ // into that tree with its pid no longer tracked anywhere (so neither a
3415
+ // dissolve nor `sched stop --batch` could stop it). Its work is done —
3416
+ // stop it first.
3417
+ if (slot.pid !== null && deps.spawnDeps.isAlive(slot.pid, slot.pid_start ?? undefined)) {
3418
+ deps.spawnDeps.kill(slot.pid, slot.pid_start ?? undefined);
3419
+ }
3420
+ journalEvent(deps, 'external-advance', unit(batchId), {
3421
+ issue: run.issue,
3422
+ detail: 'member review done',
3423
+ });
3424
+ recordMemberRunLog(deps, dispatch, state0, batchId, run.index, run.issue, slot, now);
3425
+ const verdict = evaluateIncrementalGate(deps, batchId, run.worktree, run.issue, now);
3426
+ if (verdict.kind === 'failed') {
3427
+ deps.store.withLock((s) => ({ state: release(s), result: undefined }));
3428
+ evictParallelMember(deps, config, batchId, run, { reason: verdict.reason, detail: verdict.detail }, now, result);
3429
+ return;
3430
+ }
3431
+ deps.store.withLock((s) => ({
3432
+ // #629: a verified member completion is proof dispatch is healthy.
3433
+ state: patchRun((0, dispatch_health_1.resetDispatchApiErrorStreak)(release(s)), batchId, run.issue, {
3434
+ status: 'verified',
3435
+ gate_inconclusive: verdict.kind === 'inconclusive' ? verdict.capability : null,
3436
+ }, now),
3437
+ result: undefined,
3438
+ }));
3439
+ result.completed.push(memberUnit);
3440
+ return;
3441
+ }
3442
+ if (read.dead &&
3443
+ !read.blockedNow &&
3444
+ handleDeadDispatchApiError(deps, batchId, (0, dispatch_1.batchMemberLogPath)(deps.store.runsDir, batchId, run.index, run.issue), slot.log_offset_at_spawn ?? 0, now, { releaseWith: release })) {
3445
+ // #629: not a member failure — the run stays `running` with no slot, and
3446
+ // `spawnParallelMembers` (pause-gated) respawns it in place.
3447
+ return;
3448
+ }
3449
+ if (read.blockedNow || read.dead) {
3450
+ const lastTool = recordMemberRunLog(deps, dispatch, state0, batchId, run.index, run.issue, slot, now);
3451
+ deps.store.withLock((s) => ({ state: release(s), result: undefined }));
3452
+ evictParallelMember(deps, config, batchId, run, memberFailureFor(read, lastTool), now, result);
3453
+ }
3454
+ }
3455
+ /**
3456
+ * Land one verified parallel member onto the integration branch (#809): rebase
3457
+ * its commits onto the integration TIP in its own worktree (earlier members may
3458
+ * have landed since it was cut), refresh the pushed member branch, then
3459
+ * `merge --ff-only` + push in the batch worktree — the same linear,
3460
+ * `(#<issue>)`-trailed history serial landing produces, so `memberRanges`
3461
+ * attribution and per-member commits are unchanged.
3462
+ *
3463
+ * `conflict` — the member's diff does not apply on top of the members landed
3464
+ * before it (the rebase is aborted, leaving its pushed branch untouched); the
3465
+ * caller evicts it, batch-integrate's eviction verdict. `failed` — a mechanical
3466
+ * step failed after a clean rebase; the caller blocks the batch, like serial.
3467
+ */
3468
+ function landParallelRun(deps, batch, run, now) {
3469
+ const fail = (reason, detail) => {
3470
+ journalEvent(deps, 'landing-failed', unit(batch.id), { issue: run.issue, reason, detail });
3471
+ return { kind: 'failed', reason };
3472
+ };
3473
+ if (batch.worktree === null || batch.branch === null) {
3474
+ return fail('no-integration-branch', 'batch has no integration worktree/branch');
3475
+ }
3476
+ // Defense in depth (CWE-88): persisted refs reach git argv below.
3477
+ if (!attribution_1.SAFE_REF_RE.test(run.branch) || !attribution_1.SAFE_REF_RE.test(batch.branch)) {
3478
+ return fail('invalid-branch-name', `persisted member/integration branch failed SAFE_REF_RE: ${run.branch} → ${batch.branch}`);
3479
+ }
3480
+ // The rebase rewrites whatever is checked out in `run.worktree` — confirm
3481
+ // it is this member's own tree, present, on this member's branch first.
3482
+ if (!(deps.fsExists ?? ((p) => fs.existsSync(p)))(run.worktree)) {
3483
+ return fail('member-worktree-missing', `member worktree ${run.worktree} is gone from disk`);
3484
+ }
3485
+ if (!run.pool_claimed) {
3486
+ const root = deps.exec('git', ['rev-parse', '--show-toplevel'], deps.repoDir) ?? deps.repoDir;
3487
+ if (!(0, teardown_1.isSafeWorktree)(path.resolve(root), run.worktree)) {
3488
+ return fail('unsafe-member-worktree', `member worktree ${run.worktree} is outside the repo's worktree roots`);
3489
+ }
3490
+ }
3491
+ // A rebase an engine crash left half-done would make the next one fail
3492
+ // ("rebase already in progress") and read as a conflict — clear it first
3493
+ // (a no-op failure when none is in progress).
3494
+ deps.exec('git', ['rebase', '--abort'], run.worktree);
3495
+ const head = deps.exec('git', ['symbolic-ref', '--short', 'HEAD'], run.worktree)?.trim();
3496
+ if (head !== run.branch) {
3497
+ return fail('member-worktree-off-branch', `member worktree ${run.worktree} has ${head ?? 'no branch'} checked out, expected ${run.branch} — not rebasing someone else's checkout`);
3498
+ }
3499
+ // The force-push below must never overwrite work pushed to the member
3500
+ // branch that is not part of what was verified here. Read origin's tip NOW
3501
+ // (a bare `--force-with-lease` would trust `origin/<branch>`, which any fetch
3502
+ // in the shared repo refreshes) and require it to be an ancestor of the
3503
+ // verified local branch (the member may not have pushed its last commit);
3504
+ // the lease then pins exactly that tip.
3505
+ const verified = deps
3506
+ .exec('git', ['rev-parse', `refs/heads/${run.branch}`], run.worktree)
3507
+ ?.trim();
3508
+ const remoteLine = deps.exec('git', ['ls-remote', 'origin', `refs/heads/${run.branch}`], run.worktree);
3509
+ if (!verified || !groundtruth_1.GIT_OID_RE.test(verified) || remoteLine === null) {
3510
+ return fail('member-branch-unresolved', `could not resolve ${run.branch} locally and on origin before landing`);
3511
+ }
3512
+ const remoteTip = remoteLine.trim().split(/\s+/)[0] ?? '';
3513
+ if (remoteTip !== '' &&
3514
+ (!groundtruth_1.GIT_OID_RE.test(remoteTip) ||
3515
+ deps.exec('git', ['merge-base', '--is-ancestor', remoteTip, verified], run.worktree) === null)) {
3516
+ return fail('member-branch-diverged', `origin/${run.branch} (${remoteTip.slice(0, 12)}) holds commits the verified ${verified.slice(0, 12)} does not — something was pushed after verification; blocking rather than overwriting it`);
3517
+ }
3518
+ // `--autostash`: a stray uncommitted file in the member tree is not the
3519
+ // member's deliverable and must not read as a conflict.
3520
+ if (deps.exec('git', ['rebase', '--autostash', batch.branch], run.worktree) === null) {
3521
+ // Only a rebase stopped on unmerged paths is a CONFLICT (eviction); any
3522
+ // other failure is mechanical and blocks like the other landing steps.
3523
+ const unmerged = (deps.exec('git', ['diff', '--name-only', '--diff-filter=U'], run.worktree) ?? '')
3524
+ .split('\n')
3525
+ .map((l) => l.trim())
3526
+ .filter((l) => l.length > 0);
3527
+ deps.exec('git', ['rebase', '--abort'], run.worktree);
3528
+ if (unmerged.length === 0) {
3529
+ return fail('member-rebase-failed', `git rebase of ${run.branch} onto ${batch.branch} failed with no conflicting paths — blocking for an operator`);
3530
+ }
3531
+ const detail = `${run.branch} does not rebase cleanly onto ${batch.branch} — conflicting with members landed before it in: ${unmerged.slice(0, 10).join(', ')}; evicted with its pushed branch intact`;
3532
+ journalEvent(deps, 'landing-failed', unit(batch.id), {
3533
+ issue: run.issue,
3534
+ reason: 'landing-conflict',
3535
+ detail,
3536
+ });
3537
+ return { kind: 'conflict', detail };
3538
+ }
3539
+ if (deps.exec('git', [
3540
+ 'push',
3541
+ `--force-with-lease=refs/heads/${run.branch}:${remoteTip}`,
3542
+ 'origin',
3543
+ '--',
3544
+ run.branch,
3545
+ ], run.worktree) === null) {
3546
+ return fail('member-push-failed', `rebased ${run.branch} onto ${batch.branch} but could not push it (origin moved since ${remoteTip.slice(0, 12) || 'absent'}, or the push failed) — blocking so nothing pushed after verification is overwritten`);
3547
+ }
3548
+ if (deps.exec('git', ['merge', '--ff-only', run.branch], batch.worktree) === null) {
3549
+ return fail('landing-merge-failed', `git merge --ff-only ${run.branch} into ${batch.branch} failed right after a clean rebase — blocking for an operator`);
3550
+ }
3551
+ if (deps.exec('git', ['push', 'origin', '--', batch.branch], batch.worktree) === null) {
3552
+ return fail('landing-push-failed', `landed ${run.branch} on ${batch.branch} locally but the push failed — blocking so origin stays the durable copy`);
3553
+ }
3554
+ deps.journal.append((0, journal_1.unitEvent)('member-landed', unit(batch.id), {
3555
+ issue: run.issue,
3556
+ detail: `${run.branch} rebased onto and fast-forwarded into ${batch.branch} (member ${run.index}, parallel)`,
3557
+ }), now);
3558
+ return { kind: 'landed' };
3559
+ }
3560
+ /**
3561
+ * Walk members IN ORDER: skip every resolved one (landed, evicted, or no longer
3562
+ * in the batch); the first unresolved one is either landable (`verified`) or
3563
+ * something the walk must wait for (not yet spawned, or still running) — later
3564
+ * members never land ahead of it, so the integration branch always holds a
3565
+ * member-order prefix.
3566
+ */
3567
+ function nextLandingStep(state, batch) {
3568
+ for (const issue of batch.members) {
3569
+ const run = batch.member_runs.find((r) => r.issue === issue);
3570
+ if (run?.status === 'landed' || run?.status === 'evicted')
3571
+ continue;
3572
+ if (!memberStillInBatch(state, batch, issue))
3573
+ continue;
3574
+ if (run === undefined || run.status === 'running')
3575
+ return { kind: 'wait' };
3576
+ return { kind: 'land', run };
3577
+ }
3578
+ return { kind: 'done' };
3579
+ }
3580
+ /**
3581
+ * Land every landable member in order, then — once every member is resolved —
3582
+ * either block on a gate-inconclusive member (serial parity: it landed, the
3583
+ * batch blocks for `sched resume --batch`) or move to `validating` and run the
3584
+ * aggregate suite once.
3585
+ */
3586
+ function advanceParallelBatch(deps, config, dispatch, batchId, now, result) {
3587
+ // Bounded: every iteration lands or evicts one member.
3588
+ for (;;) {
3589
+ const state = deps.store.load();
3590
+ const batch = (0, state_1.findBatch)(state, batchId);
3591
+ if (!batch || batch.status !== 'executing' || !isParallelBatch(batch))
3592
+ return;
3593
+ const step = nextLandingStep(state, batch);
3594
+ if (step.kind === 'wait')
3595
+ return;
3596
+ if (step.kind === 'done') {
3597
+ finalizeParallelMembers(deps, config, dispatch, batchId, now, result);
3598
+ return;
3599
+ }
3600
+ const { run } = step;
3601
+ const landed = landParallelRun(deps, batch, run, now);
3602
+ if (landed.kind === 'conflict') {
3603
+ evictParallelMember(deps, config, batchId, run, { reason: 'landing-conflict', detail: landed.detail }, now, result);
3604
+ continue;
3605
+ }
3606
+ if (landed.kind === 'failed') {
3607
+ stopRunningMembers(deps, batchId, now);
3608
+ blockBatchForOperator(deps, config, batchId, { reason: landed.reason }, now, result);
3609
+ return;
3610
+ }
3611
+ // `git log` outside the lock (same invariant as `completeMemberGate`).
3612
+ const ranges = (0, attribution_1.memberRanges)(boundaryCommits(deps, batch));
3613
+ deps.store.withLock((s) => {
3614
+ let n = (0, state_1.patchBatch)(s, batchId, { ranges }, now);
3615
+ n = advanceMemberToValidated(n, run.issue, now);
3616
+ n = patchRun(n, batchId, run.issue, { status: 'landed' }, now);
3617
+ return { state: n, result: undefined };
3618
+ });
3619
+ // A gate-inconclusive member keeps its tree for `sched resume --batch`'s
3620
+ // recheck; everyone else's is done serving.
3621
+ if (run.gate_inconclusive === null) {
3622
+ teardownMemberTree(deps, batchId, runTree(run), true);
3623
+ markTornDown(deps, batchId, run.issue, now);
3624
+ }
3625
+ }
3626
+ }
3627
+ /** Every member is resolved — block on a gate-inconclusive one, or validate. */
3628
+ function finalizeParallelMembers(deps, config, dispatch, batchId, now, result) {
3629
+ const batch = (0, state_1.findBatch)(deps.store.load(), batchId);
3630
+ if (!batch)
3631
+ return;
3632
+ const blockedOn = batch.member_runs.find((r) => r.status === 'landed' && r.gate_inconclusive !== null);
3633
+ if (blockedOn !== undefined) {
3634
+ // Point the serial "current member" fields at it: `sched resume --batch`
3635
+ // (`resumeBlockedGate`) rechecks `member_worktree` and completes/evicts
3636
+ // `members[executing_member - 1]`, then hands back via
3637
+ // `advanceMemberOrValidate` → `settleParallelResume`.
3638
+ deps.store.withLock((s) => ({
3639
+ state: (0, state_1.patchBatch)(s, batchId, {
3640
+ executing_member: blockedOn.index,
3641
+ member_branch: blockedOn.branch,
3642
+ member_worktree: blockedOn.worktree,
3643
+ member_pool_claimed: blockedOn.pool_claimed,
3644
+ }, now),
3645
+ result: undefined,
3646
+ }));
3647
+ blockBatchForOperator(deps, config, batchId, {
3648
+ reason: `gate-inconclusive:${blockedOn.gate_inconclusive}`,
3649
+ milestonePhase: 'batch-review',
3650
+ }, now, result);
3651
+ return;
3652
+ }
3653
+ const claimed = deps.store.withLock((s) => {
3654
+ const b = (0, state_1.findBatch)(s, batchId);
3655
+ if (!b || b.status !== 'executing')
3656
+ return { state: s, result: false };
3657
+ return {
3658
+ state: (0, state_1.transitionBatch)(s, batchId, 'validating', { executing_member: b.members.length }, now),
3659
+ result: true,
3660
+ };
3661
+ });
3662
+ if (claimed)
3663
+ runValidate(deps, config, dispatch, batchId, now, result);
3664
+ }
3665
+ /**
3666
+ * `sched resume --batch` resolved the gate-inconclusive member of a parallel
3667
+ * batch (completed or evicted it through the serial rail): record that on its
3668
+ * run so the landing walk treats it as resolved, then continue the parallel
3669
+ * rail from where it blocked.
3670
+ */
3671
+ function settleParallelResume(deps, batchId, memberIssue, now) {
3672
+ deps.store.withLock((s) => {
3673
+ const b = (0, state_1.findBatch)(s, batchId);
3674
+ if (!b)
3675
+ return { state: s, result: undefined };
3676
+ const evicted = b.evictions.some((e) => e.issue === memberIssue);
3677
+ return {
3678
+ state: patchRun(s, batchId, memberIssue, {
3679
+ gate_inconclusive: null,
3680
+ // The serial rail's `teardownMemberWorktree` removed its tree.
3681
+ torn_down: true,
3682
+ ...(evicted ? { status: 'evicted' } : {}),
3683
+ }, now),
3684
+ result: undefined,
3685
+ };
3686
+ });
3687
+ }
3688
+ /**
3689
+ * One tick of a parallel batch in `executing`: reconcile every member slot,
3690
+ * land what can land in order, then fill free member concurrency (unless the
3691
+ * scheduler is paused — a spawn is a provider dispatch).
3692
+ */
3693
+ function reconcileParallelBatch(deps, config, dispatch, batchId, now, result, paused) {
3694
+ const batch = (0, state_1.findBatch)(deps.store.load(), batchId);
3695
+ if (!batch)
3696
+ return;
3697
+ for (const { issue } of batch.member_runs) {
3698
+ // Fresh per member: an earlier member's eviction may have dissolved the
3699
+ // batch (killing and releasing every other member) in this same pass.
3700
+ const state = deps.store.load();
3701
+ const fresh = (0, state_1.findBatch)(state, batchId);
3702
+ if (!fresh || fresh.status !== 'executing')
3703
+ return;
3704
+ const run = fresh.member_runs.find((r) => r.issue === issue);
3705
+ if (run?.status !== 'running')
3706
+ continue;
3707
+ const slot = (0, state_1.slotForBatchMember)(state, batchId, issue);
3708
+ if (slot && (slot.status === 'running' || slot.status === 'assigned')) {
3709
+ reconcileParallelMemberSlot(deps, config, dispatch, batchId, run, slot, now, result);
3710
+ }
3711
+ }
3712
+ advanceParallelBatch(deps, config, dispatch, batchId, now, result);
3713
+ if (!paused)
3714
+ spawnParallelMembers(deps, config, dispatch, batchId, now, result);
3715
+ }
3716
+ /**
3717
+ * #809: stop and tear down every parallel member whose tree is not yet torn
3718
+ * down — a dissolve can land while other members are mid-run (their agents are
3719
+ * killed: their entries were just requeued full-cycle, so a live member agent
3720
+ * would be working a unit nothing tracks any more), `sched stop`/`abandon`
3721
+ * make a batch terminal without a teardown, and a crash between a run's
3722
+ * resolution and its teardown leaves `torn_down: false` behind. Teardown is
3723
+ * idempotent, so re-running it is always safe. `alreadyTornDown` names the
3724
+ * serial-field tree `teardownMemberWorktree` just handled (a gate-inconclusive
3725
+ * member of a blocked parallel batch).
3726
+ */
3727
+ function teardownParallelRuns(deps, batchId, alreadyTornDown) {
3728
+ const batch = (0, state_1.findBatch)(deps.store.load(), batchId);
3729
+ if (!batch)
3730
+ return;
3731
+ const now = deps.now();
3732
+ for (const run of batch.member_runs) {
3733
+ if (run.torn_down)
3734
+ continue;
3735
+ stopMemberAgent(deps, batchId, run.issue, now);
3736
+ const landed = run.status === 'landed';
3737
+ deps.store.withLock((s) => ({
3738
+ state: patchRun(s, batchId, run.issue, { status: landed ? 'landed' : 'evicted', gate_inconclusive: null }, now),
3739
+ result: undefined,
3740
+ }));
3741
+ if (run.worktree !== alreadyTornDown) {
3742
+ teardownMemberTree(deps, batchId, runTree(run), landed);
3743
+ }
3744
+ markTornDown(deps, batchId, run.issue, now);
3745
+ }
3746
+ }
3747
+ /** Kill a parallel member's live agent (pid-start checked) and release its slot. */
3748
+ function stopMemberAgent(deps, batchId, issue, now) {
3749
+ const slot = (0, state_1.slotForBatchMember)(deps.store.load(), batchId, issue);
3750
+ if (slot === undefined)
3751
+ return;
3752
+ if (slot.pid !== null && deps.spawnDeps.isAlive(slot.pid, slot.pid_start ?? undefined)) {
3753
+ deps.spawnDeps.kill(slot.pid, slot.pid_start ?? undefined);
3754
+ }
3755
+ deps.store.withLock((s) => ({
3756
+ state: (0, state_1.releaseBatchMemberSlot)(s, batchId, issue, now),
3757
+ result: undefined,
3758
+ }));
3759
+ }
3760
+ /**
3761
+ * Stop every running member of a parallel batch that is about to BLOCK (a
3762
+ * landing failure, a reconcile error): nothing reconciles a blocked batch's
3763
+ * member slots, so a still-running agent would hold capacity and keep working
3764
+ * a unit an operator is about to abandon/requeue. Trees are kept for the
3765
+ * operator; `teardownBatch` removes them when the batch ends.
3766
+ */
3767
+ function stopRunningMembers(deps, batchId, now) {
3768
+ const batch = (0, state_1.findBatch)(deps.store.load(), batchId);
3769
+ for (const run of batch?.member_runs ?? []) {
3770
+ if (run.status === 'running')
3771
+ stopMemberAgent(deps, batchId, run.issue, now);
3772
+ }
3773
+ }
3033
3774
  // --- Entry point ---
3034
3775
  /**
3035
3776
  * One batch reconcile+refill pass. Called from `engine.ts`'s `tick()` after
@@ -3092,11 +3833,12 @@ function runBatchTick(deps, config, dispatch) {
3092
3833
  const fresh = (0, state_1.findBatch)(current, batch.id);
3093
3834
  const dissolvable = fresh !== undefined && (0, state_1.allowedBatchTransitions)(fresh.status).includes('dissolving');
3094
3835
  if (dissolvable) {
3095
- const liveSlot = slotFor(current, batch.id);
3096
- if (liveSlot !== undefined &&
3097
- liveSlot.pid !== null &&
3098
- deps.spawnDeps.isAlive(liveSlot.pid, liveSlot.pid_start ?? undefined)) {
3099
- deps.spawnDeps.kill(liveSlot.pid, liveSlot.pid_start ?? undefined);
3836
+ // #809: a parallel batch holds one slot per running member too.
3837
+ for (const liveSlot of (0, state_1.slotsForBatch)(current, batch.id)) {
3838
+ if (liveSlot.pid !== null &&
3839
+ deps.spawnDeps.isAlive(liveSlot.pid, liveSlot.pid_start ?? undefined)) {
3840
+ deps.spawnDeps.kill(liveSlot.pid, liveSlot.pid_start ?? undefined);
3841
+ }
3100
3842
  }
3101
3843
  const outcome = (0, recovery_1.dissolveBatch)(current, batch.id, { strategy: 'full', reason: `dispatch-profile-missing:${batch.dispatch_profile}` }, recoveryDeps(deps, config, fresh, now));
3102
3844
  deps.store.withLock((s) => ({
@@ -3125,6 +3867,13 @@ function runBatchTick(deps, config, dispatch) {
3125
3867
  };
3126
3868
  const containFailure = (batchId, err) => {
3127
3869
  const detail = `${err.name}: ${err.message}`;
3870
+ // #809: a blocked batch's member slots are never reconciled — stop the agents too.
3871
+ try {
3872
+ stopRunningMembers(deps, batchId, now);
3873
+ }
3874
+ catch {
3875
+ // best-effort: the block below must still land
3876
+ }
3128
3877
  // A corrupt unit must not turn into a project-wide scheduler outage.
3129
3878
  // Keep the failed batch inspectable, release its slot, and continue.
3130
3879
  deps.store.withLock((state) => {
@@ -3133,7 +3882,7 @@ function runBatchTick(deps, config, dispatch) {
3133
3882
  return { state, result: undefined };
3134
3883
  }
3135
3884
  const blocked = (0, state_1.transitionBatch)(state, batchId, 'blocked', { blocked_reason: 'reconcile-error' }, now);
3136
- return { state: (0, state_1.releaseBatchSlot)(blocked, batchId, now), result: undefined };
3885
+ return { state: (0, state_1.releaseAllBatchSlots)(blocked, batchId, now), result: undefined };
3137
3886
  });
3138
3887
  journalEvent(deps, 'unit-failed', unit(batchId), { reason: 'reconcile-error', detail });
3139
3888
  result.failed.push(unit(batchId));
@@ -3185,6 +3934,8 @@ function runBatchTick(deps, config, dispatch) {
3185
3934
  }
3186
3935
  for (const batch of deps.store.load().batches) {
3187
3936
  const slot = slotFor(deps.store.load(), batch.id);
3937
+ // #809: a terminal batch may also leak parallel member slots.
3938
+ const heldSlots = (0, state_1.slotsForBatch)(deps.store.load(), batch.id);
3188
3939
  // #609: a TERMINAL batch (`done`/`dissolved`) still holding a slot matches
3189
3940
  // none of the status arms below and would fall through their `continue`,
3190
3941
  // so nothing ever looks at that slot again — not even the orphaned-pid
@@ -3196,15 +3947,29 @@ function runBatchTick(deps, config, dispatch) {
3196
3947
  // files that already carry a leaked slot, which is the only way an
3197
3948
  // operator gets those three hours of held capacity back.
3198
3949
  if (types_1.TERMINAL_BATCH_STATUSES.has(batch.status)) {
3199
- if (slot) {
3950
+ // #809: `sched stop`/`abandon` end a batch without a teardown — give
3951
+ // back every member tree (and stop any agent still running in one).
3952
+ if (batch.member_runs.some((r) => !r.torn_down)) {
3953
+ try {
3954
+ teardownParallelRuns(deps, batch.id, null);
3955
+ }
3956
+ catch (err) {
3957
+ journalEvent(deps, 'teardown-failed', unit(batch.id), {
3958
+ detail: `member teardown: ${err.message}`,
3959
+ });
3960
+ }
3961
+ }
3962
+ if (heldSlots.length > 0) {
3200
3963
  deps.store.withLock((s) => ({
3201
- state: (0, state_1.releaseBatchSlot)(s, batch.id, now),
3964
+ state: (0, state_1.releaseAllBatchSlots)(s, batch.id, now),
3202
3965
  result: undefined,
3203
3966
  }));
3204
- journalEvent(deps, 'slot-released', unit(batch.id), {
3205
- slot: slot.id,
3206
- detail: `terminal batch (${batch.status}) held slot ${slot.id}`,
3207
- });
3967
+ for (const held of heldSlots) {
3968
+ journalEvent(deps, 'slot-released', unit(batch.id), {
3969
+ slot: held.id,
3970
+ detail: `terminal batch (${batch.status}) held slot ${held.id}${held.unit === unit(batch.id) ? '' : ` (${held.unit})`}`,
3971
+ });
3972
+ }
3208
3973
  }
3209
3974
  continue;
3210
3975
  }
@@ -3216,6 +3981,12 @@ function runBatchTick(deps, config, dispatch) {
3216
3981
  const batchDispatch = dispatchFor(batch);
3217
3982
  if (batchDispatch === null)
3218
3983
  continue;
3984
+ // #809: a parallel batch's members hold their OWN slots — its whole
3985
+ // `executing` phase is one arm.
3986
+ if (batch.status === 'executing' && isParallelBatch(batch)) {
3987
+ reconcileParallelBatch(deps, config, batchDispatch, batch.id, now, result, paused);
3988
+ continue;
3989
+ }
3219
3990
  if (slot && (slot.status === 'running' || slot.status === 'assigned')) {
3220
3991
  if (batch.status === 'executing') {
3221
3992
  reconcileMemberSlot(deps, config, batchDispatch, batch.id, slot, now, result);