@ai-dossier/sched 0.54.1 → 0.55.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +20 -5
- package/dist/anchor-close.d.ts +26 -16
- package/dist/anchor-close.d.ts.map +1 -1
- package/dist/anchor-close.js +112 -75
- package/dist/anchor-close.js.map +1 -1
- package/dist/batch-dispatch.d.ts +17 -24
- package/dist/batch-dispatch.d.ts.map +1 -1
- package/dist/batch-dispatch.js +149 -74
- package/dist/batch-dispatch.js.map +1 -1
- package/dist/index.d.ts +1 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +2 -2
- package/dist/index.js.map +1 -1
- package/dist/state.d.ts +11 -1
- package/dist/state.d.ts.map +1 -1
- package/dist/state.js +33 -0
- package/dist/state.js.map +1 -1
- package/dist/status.d.ts +11 -8
- package/dist/status.d.ts.map +1 -1
- package/dist/status.js +20 -14
- package/dist/status.js.map +1 -1
- package/dist/types.d.ts +35 -5
- package/dist/types.d.ts.map +1 -1
- package/dist/types.js +5 -1
- package/dist/types.js.map +1 -1
- package/package.json +1 -1
package/dist/batch-dispatch.js
CHANGED
|
@@ -820,7 +820,7 @@ landed) {
|
|
|
820
820
|
const state = deps.store.load();
|
|
821
821
|
const batch = (0, state_1.findBatch)(state, batchId);
|
|
822
822
|
if (!batch || batch.member_worktree === null)
|
|
823
|
-
return;
|
|
823
|
+
return null;
|
|
824
824
|
const worktree = batch.member_worktree;
|
|
825
825
|
const branch = batch.member_branch;
|
|
826
826
|
// #677 review (defense in depth, matching `branchMergedIntoBase`'s bar):
|
|
@@ -831,7 +831,7 @@ landed) {
|
|
|
831
831
|
cleanup: 'failed-invalid-branch-name',
|
|
832
832
|
detail: branch,
|
|
833
833
|
});
|
|
834
|
-
return;
|
|
834
|
+
return { path: worktree, cleared: false };
|
|
835
835
|
}
|
|
836
836
|
// Nulls first — clear the fields BEFORE the exec so a crash mid-teardown
|
|
837
837
|
// doesn't retry into a half-cleaned tree (the re-derived path is
|
|
@@ -845,23 +845,39 @@ landed) {
|
|
|
845
845
|
result: undefined,
|
|
846
846
|
};
|
|
847
847
|
});
|
|
848
|
-
teardownMemberTree(deps, batchId, { worktree, branch, poolClaimed: batch.member_pool_claimed === true }, landed);
|
|
848
|
+
const cleared = teardownMemberTree(deps, batchId, { worktree, branch, poolClaimed: batch.member_pool_claimed === true }, landed, batch.members[batch.executing_member - 1]);
|
|
849
|
+
return { path: worktree, cleared };
|
|
849
850
|
}
|
|
851
|
+
/** #855: the `teardown-failed` tail for a member tree that stays on disk. */
|
|
852
|
+
const TREE_LEFT_FOR_OPERATOR = 'tree left on disk for the operator and NOT retried (it may hold work); inspect it, then remove it (`git worktree remove`, or `worktree-pool return` if pool-claimed) — `sched status` names it';
|
|
850
853
|
/**
|
|
851
854
|
* The exec half of a member teardown, shared by the serial rail
|
|
852
855
|
* (`teardownMemberWorktree`, which first clears the batch's member fields) and
|
|
853
856
|
* the parallel rail (#809: `evictParallelMember`, `advanceParallelBatch`,
|
|
854
857
|
* `teardownParallelRuns`). See
|
|
855
858
|
* `teardownMemberWorktree` for the landed/evicted branch policy.
|
|
859
|
+
*
|
|
860
|
+
* Returns whether the TREE is verifiably gone (#855) — `runTeardown` reports
|
|
861
|
+
* `done` only after the path is gone and unlisted, or the pool's self-check
|
|
862
|
+
* reports the entry returned. A failed branch delete after that does not
|
|
863
|
+
* un-clear the tree. Journals exactly one line per call; a failed tree is
|
|
864
|
+
* never re-attempted by any caller, so the line says what the operator does.
|
|
856
865
|
*/
|
|
857
|
-
function teardownMemberTree(deps, batchId, tree, landed
|
|
866
|
+
function teardownMemberTree(deps, batchId, tree, landed,
|
|
867
|
+
/** The member whose tree this is — named in the journal line. */
|
|
868
|
+
issue) {
|
|
858
869
|
const { worktree, branch } = tree;
|
|
870
|
+
const context = {
|
|
871
|
+
...(issue !== undefined ? { issue } : {}),
|
|
872
|
+
worktree,
|
|
873
|
+
};
|
|
859
874
|
if (branch !== null && !attribution_1.SAFE_REF_RE.test(branch)) {
|
|
860
875
|
journalEvent(deps, 'teardown-failed', unit(batchId), {
|
|
876
|
+
...context,
|
|
861
877
|
cleanup: 'failed-invalid-branch-name',
|
|
862
|
-
detail: branch
|
|
878
|
+
detail: `${branch} — ${TREE_LEFT_FOR_OPERATOR}`,
|
|
863
879
|
});
|
|
864
|
-
return;
|
|
880
|
+
return false;
|
|
865
881
|
}
|
|
866
882
|
// Route through `runTeardown` for the same guarantees `teardownBatch`
|
|
867
883
|
// gets: pool returns are verify-first idempotent and refuse to claim
|
|
@@ -884,11 +900,12 @@ function teardownMemberTree(deps, batchId, tree, landed) {
|
|
|
884
900
|
}
|
|
885
901
|
const failed = !treeCleared || branchCleanup.startsWith('failed');
|
|
886
902
|
journalEvent(deps, failed ? 'teardown-failed' : 'member-worktree-torn-down', unit(batchId), {
|
|
903
|
+
...context,
|
|
887
904
|
cleanup: t.cleanup,
|
|
888
905
|
branch_cleanup: branchCleanup,
|
|
889
|
-
detail: t.detail
|
|
890
|
-
worktree,
|
|
906
|
+
detail: treeCleared ? t.detail : `${t.detail} — ${TREE_LEFT_FOR_OPERATOR}`,
|
|
891
907
|
});
|
|
908
|
+
return treeCleared;
|
|
892
909
|
}
|
|
893
910
|
/**
|
|
894
911
|
* The shared "block the batch for an operator" tail (#677 review — the same
|
|
@@ -1717,13 +1734,15 @@ function journalAdvanceSkipped(deps, batchId, memberIssue, currentMember, claim)
|
|
|
1717
1734
|
detail: `another resolution already advanced past member ${currentMember} (batch is now executing_member=${claim.observed ?? 'gone'}, status=${claim.status ?? 'gone'}) — no second member-advanced and no second continuation`,
|
|
1718
1735
|
});
|
|
1719
1736
|
}
|
|
1720
|
-
function advanceMemberOrValidate(deps, config, dispatch, batchId, memberCount, currentMember, memberIssue, now, result
|
|
1737
|
+
function advanceMemberOrValidate(deps, config, dispatch, batchId, memberCount, currentMember, memberIssue, now, result,
|
|
1738
|
+
/** What the caller's `teardownMemberWorktree` did with the member's tree (#855). */
|
|
1739
|
+
serialTeardown) {
|
|
1721
1740
|
// #809: a parallel batch reaches this serial rail only through `sched resume
|
|
1722
1741
|
// --batch` resolving its gate-inconclusive member — hand back to the
|
|
1723
1742
|
// parallel rail instead of advancing a serial member pointer.
|
|
1724
1743
|
const current = (0, state_1.findBatch)(deps.store.load(), batchId);
|
|
1725
1744
|
if (current && isParallelBatch(current)) {
|
|
1726
|
-
settleParallelResume(deps, batchId, memberIssue, now);
|
|
1745
|
+
settleParallelResume(deps, batchId, memberIssue, now, serialTeardown);
|
|
1727
1746
|
advanceParallelBatch(deps, config, dispatch, batchId, now, result);
|
|
1728
1747
|
return;
|
|
1729
1748
|
}
|
|
@@ -1789,8 +1808,8 @@ function evictMemberAndContinue(deps, config, dispatch, batchId, batch, memberIs
|
|
|
1789
1808
|
// branch — but its worktree/branch must go, or the next tick's prep for
|
|
1790
1809
|
// the NEXT member collides with a stale tree. `teardownBatch` already
|
|
1791
1810
|
// handled it on the dissolved path (the fields are cleared there).
|
|
1792
|
-
teardownMemberWorktree(deps, batchId, now, false);
|
|
1793
|
-
advanceMemberOrValidate(deps, config, dispatch, batchId, batch.members.length, batch.executing_member, memberIssue, now, result);
|
|
1811
|
+
const serialTeardown = teardownMemberWorktree(deps, batchId, now, false);
|
|
1812
|
+
advanceMemberOrValidate(deps, config, dispatch, batchId, batch.members.length, batch.executing_member, memberIssue, now, result, serialTeardown);
|
|
1794
1813
|
}
|
|
1795
1814
|
/**
|
|
1796
1815
|
* The incremental gate (#523 AC2, revised #583): typecheck + focused tests
|
|
@@ -2027,8 +2046,8 @@ function completeMemberGate(deps, config, dispatch, batchId, batch, memberIssue,
|
|
|
2027
2046
|
// #677: the member's tree is done serving — landed, ranges recomputed.
|
|
2028
2047
|
// Teardown before the advance so the next member's prep cannot collide
|
|
2029
2048
|
// with this tree's cleanup.
|
|
2030
|
-
teardownMemberWorktree(deps, batchId, now, true);
|
|
2031
|
-
advanceMemberOrValidate(deps, config, dispatch, batchId, batch.members.length, batch.executing_member, memberIssue, now, result);
|
|
2049
|
+
const serialTeardown = teardownMemberWorktree(deps, batchId, now, true);
|
|
2050
|
+
advanceMemberOrValidate(deps, config, dispatch, batchId, batch.members.length, batch.executing_member, memberIssue, now, result, serialTeardown);
|
|
2032
2051
|
}
|
|
2033
2052
|
/**
|
|
2034
2053
|
* `sched resume --batch <id>` (#583 AC4): an operator-triggered, synchronous
|
|
@@ -2920,7 +2939,13 @@ function reconcileTailSlot(deps, config, batchId, slot, now, result) {
|
|
|
2920
2939
|
result.failed.push(unit(batchId));
|
|
2921
2940
|
}
|
|
2922
2941
|
}
|
|
2923
|
-
function reconcileReportSlot(deps,
|
|
2942
|
+
function reconcileReportSlot(deps,
|
|
2943
|
+
// Unused — kept so this function's signature matches its `reconcile*Slot`
|
|
2944
|
+
// siblings (`reconcileMemberSlot`, `reconcileFixSlot`, `reconcileTailSlot`),
|
|
2945
|
+
// all called uniformly as `reconcile*Slot(deps, config, ...)` from the same
|
|
2946
|
+
// dispatch loop (#830 — pre-existing Biome `noUnusedFunctionParameters`,
|
|
2947
|
+
// independently fixed the same way by #855/#857).
|
|
2948
|
+
_config, batchId, slot, now, result) {
|
|
2924
2949
|
const state = deps.store.load();
|
|
2925
2950
|
const batch = (0, state_1.findBatch)(state, batchId);
|
|
2926
2951
|
if (!batch || batch.anchor === null)
|
|
@@ -3430,6 +3455,10 @@ function reconcileStaleBlockedBatches(deps, now, result, anchorCtx) {
|
|
|
3430
3455
|
// clear into this same transition instead of a separate lock write.
|
|
3431
3456
|
...state_1.CLEARED_PR_DETECT_AMBIGUOUS_FIELDS,
|
|
3432
3457
|
...(prWasRecorded ? { pr: detectedPr } : {}),
|
|
3458
|
+
// #855: persist the keep decision in the same transition, so every
|
|
3459
|
+
// later teardown path — the terminal-batch safety net included —
|
|
3460
|
+
// reads it for `member_runs[]` too.
|
|
3461
|
+
...(keepWorktree ? { worktree_kept: true } : {}),
|
|
3433
3462
|
}, now);
|
|
3434
3463
|
n = (0, state_1.transitionBatch)(n, batch.id, 'deployed', {}, now);
|
|
3435
3464
|
unwalkable = [];
|
|
@@ -3520,46 +3549,42 @@ function reconcileStaleBlockedBatches(deps, now, result, anchorCtx) {
|
|
|
3520
3549
|
* #834: `MemberRun.worktree` is a required `string` (unlike
|
|
3521
3550
|
* `BatchEntry.worktree: string | null`) — there is no null to clear it to,
|
|
3522
3551
|
* so "cleared" for a member run means `torn_down: true`, the same flag
|
|
3523
|
-
* `
|
|
3524
|
-
* thing evidentially (nothing is holding this path
|
|
3525
|
-
* by definitive evidence instead of by running
|
|
3552
|
+
* `recordRunTeardown` sets after a teardown that verifiably landed. Set
|
|
3553
|
+
* here it means the same thing evidentially (nothing is holding this path
|
|
3554
|
+
* any more), just reached by definitive evidence instead of by running
|
|
3555
|
+
* `teardownParallelRuns`.
|
|
3526
3556
|
*
|
|
3527
|
-
* #834
|
|
3528
|
-
*
|
|
3529
|
-
*
|
|
3530
|
-
*
|
|
3531
|
-
*
|
|
3532
|
-
* `
|
|
3533
|
-
*
|
|
3534
|
-
*
|
|
3535
|
-
*
|
|
3536
|
-
*
|
|
3537
|
-
*
|
|
3538
|
-
* is effectively a backstop, not the primary path: by the time it runs,
|
|
3539
|
-
* `torn_down` is usually already `true` — correctly when the tree really
|
|
3540
|
-
* is gone, but also when a real teardown attempt failed and left the tree
|
|
3541
|
-
* on disk — filed as #855 (out of scope here; #834 only extends the
|
|
3542
|
-
* evidence-based ledger reconcile, it does not change teardown behavior).
|
|
3543
|
-
* This function's own branch still matters for the window before that
|
|
3544
|
-
* safety net's next tick runs, and for `sched status` reads taken while
|
|
3545
|
-
* the scheduler isn't ticking at all.
|
|
3557
|
+
* #834/#855: a terminal batch's live `member_runs[]` entries are ALSO
|
|
3558
|
+
* reached by the terminal-batch arm in `runBatchTick` (search
|
|
3559
|
+
* `TERMINAL_BATCH_STATUSES.has(batch.status)` above the call site below),
|
|
3560
|
+
* added by #809 for `sched stop`/`abandon`, which runs earlier in the same
|
|
3561
|
+
* tick than this function. Since #855 that arm respects the keep decision
|
|
3562
|
+
* (`worktree_kept`) and marks a run torn down only after a teardown that
|
|
3563
|
+
* verifiably landed, so on a KEPT batch, or a run whose teardown failed
|
|
3564
|
+
* (`teardown_failed_at`), `torn_down` stays false and this function is the
|
|
3565
|
+
* only thing that ever clears it — once the operator has removed or
|
|
3566
|
+
* returned the tree. On a non-kept batch the arm usually tears the tree
|
|
3567
|
+
* down (and records it) first.
|
|
3546
3568
|
*
|
|
3547
3569
|
* Exported (not part of the package's public `index.ts` surface — same
|
|
3548
3570
|
* test-only convention as `reconcileStaleBlockedBatches`) so `member_runs[]`
|
|
3549
3571
|
* tests can call it directly, isolated from the terminal-batch safety net
|
|
3550
|
-
* described above, which
|
|
3551
|
-
* both run inside one `runBatchTick` call.
|
|
3572
|
+
* described above, which reaches a non-kept batch's `member_runs[]` first
|
|
3573
|
+
* when both run inside one `runBatchTick` call.
|
|
3552
3574
|
*/
|
|
3553
3575
|
function reconcileKeptWorktrees(deps, now) {
|
|
3554
3576
|
const state = deps.store.load();
|
|
3555
3577
|
const fsExists = deps.fsExists ?? ((p) => fs.existsSync(p));
|
|
3556
|
-
|
|
3557
|
-
|
|
3578
|
+
// #855: a `stopped`/`dissolved` batch's run whose teardown failed is a
|
|
3579
|
+
// leftover too (`isLeftoverMemberRun`) — its ledger clears on the same
|
|
3580
|
+
// evidence once the operator has dealt with it.
|
|
3581
|
+
const candidates = state.batches.filter((b) => (b.status === 'done' && (b.worktree !== null || b.member_worktree !== null)) ||
|
|
3582
|
+
b.member_runs.some((r) => (0, state_1.isLeftoverMemberRun)(b, r)));
|
|
3558
3583
|
if (candidates.length === 0)
|
|
3559
3584
|
return;
|
|
3560
3585
|
const needsPoolCheck = candidates.some((b) => (b.worktree !== null && b.pool_claimed) ||
|
|
3561
3586
|
(b.member_worktree !== null && b.member_pool_claimed) ||
|
|
3562
|
-
b.member_runs.some((r) =>
|
|
3587
|
+
b.member_runs.some((r) => r.pool_claimed && (0, state_1.isLeftoverMemberRun)(b, r)));
|
|
3563
3588
|
// One pool query per TICK, not per candidate — `worktree-pool status` is a
|
|
3564
3589
|
// real subprocess (`npx ...`), and this reconcile is meant to be cheap.
|
|
3565
3590
|
const poolStatusJson = needsPoolCheck
|
|
@@ -3576,12 +3601,14 @@ function reconcileKeptWorktrees(deps, now) {
|
|
|
3576
3601
|
for (const batch of candidates) {
|
|
3577
3602
|
const clears = {};
|
|
3578
3603
|
const clearedFields = [];
|
|
3579
|
-
|
|
3604
|
+
const done = batch.status === 'done';
|
|
3605
|
+
if (done && batch.worktree !== null && goneOrReturned(batch.worktree, batch.pool_claimed)) {
|
|
3580
3606
|
clears.worktree = null;
|
|
3581
3607
|
clears.pool_claimed = false;
|
|
3582
3608
|
clearedFields.push('worktree', 'pool_claimed');
|
|
3583
3609
|
}
|
|
3584
|
-
if (
|
|
3610
|
+
if (done &&
|
|
3611
|
+
batch.member_worktree !== null &&
|
|
3585
3612
|
goneOrReturned(batch.member_worktree, batch.member_pool_claimed)) {
|
|
3586
3613
|
clears.member_worktree = null;
|
|
3587
3614
|
clears.member_pool_claimed = false;
|
|
@@ -3589,7 +3616,7 @@ function reconcileKeptWorktrees(deps, now) {
|
|
|
3589
3616
|
}
|
|
3590
3617
|
// #834: unlike `worktree`/`member_worktree`, `member_runs[]` is actively
|
|
3591
3618
|
// mutated by many OTHER code paths while a tick runs (member dispatch,
|
|
3592
|
-
// eviction, `
|
|
3619
|
+
// eviction, `recordRunTeardown`, `sched attach-pr`, ...), so its evidence
|
|
3593
3620
|
// check must NOT be computed from this function's own outer, unlocked
|
|
3594
3621
|
// `batch` snapshot: `patchBatch` merges `member_runs` as a whole-array
|
|
3595
3622
|
// replacement, not per-entry, so writing an array built from a stale
|
|
@@ -3599,7 +3626,7 @@ function reconcileKeptWorktrees(deps, now) {
|
|
|
3599
3626
|
// to look at" pre-check runs out here, to decide whether entering the
|
|
3600
3627
|
// lock is worth it at all; the real per-run evidence check re-reads
|
|
3601
3628
|
// `member_runs` fresh, from the batch `withLock` reloads below.
|
|
3602
|
-
const mightClearRuns = batch.member_runs.some((r) =>
|
|
3629
|
+
const mightClearRuns = batch.member_runs.some((r) => (0, state_1.isLeftoverMemberRun)(batch, r));
|
|
3603
3630
|
if (clearedFields.length === 0 && !mightClearRuns)
|
|
3604
3631
|
continue;
|
|
3605
3632
|
const clearedRunIssues = [];
|
|
@@ -3607,12 +3634,12 @@ function reconcileKeptWorktrees(deps, now) {
|
|
|
3607
3634
|
const b = (0, state_1.findBatch)(s, batch.id);
|
|
3608
3635
|
// Re-check under the lock: another tick, or an operator command, may
|
|
3609
3636
|
// have already moved this batch or cleared the fields itself.
|
|
3610
|
-
if (!b || b.status !==
|
|
3637
|
+
if (!b || b.status !== batch.status)
|
|
3611
3638
|
return { state: s, result: undefined };
|
|
3612
3639
|
const runClears = {};
|
|
3613
3640
|
if (mightClearRuns) {
|
|
3614
3641
|
const updatedRuns = b.member_runs.map((run) => {
|
|
3615
|
-
if (
|
|
3642
|
+
if (!(0, state_1.isLeftoverMemberRun)(b, run))
|
|
3616
3643
|
return run;
|
|
3617
3644
|
if (!goneOrReturned(run.worktree, run.pool_claimed))
|
|
3618
3645
|
return run;
|
|
@@ -3808,6 +3835,12 @@ function recordAnchorCloseFailed(deps, batch, outcome, shipped, now) {
|
|
|
3808
3835
|
* left on disk forever, since nothing else ever calls this for it.
|
|
3809
3836
|
*/
|
|
3810
3837
|
function teardownBatch(deps, batchId) {
|
|
3838
|
+
// #855: a kept batch's trees are the operator's — every teardown path reads
|
|
3839
|
+
// the same persisted decision. Only its live agents are stopped.
|
|
3840
|
+
if ((0, state_1.findBatch)(deps.store.load(), batchId)?.worktree_kept === true) {
|
|
3841
|
+
teardownParallelRuns(deps, batchId, null);
|
|
3842
|
+
return;
|
|
3843
|
+
}
|
|
3811
3844
|
// #677: the current member's worktree goes first — it is batch-owned
|
|
3812
3845
|
// state on the same teardown path as the shared tree, and a blocked or
|
|
3813
3846
|
// dissolved batch must not leak a member tree (its branch/fields are
|
|
@@ -3818,9 +3851,8 @@ function teardownBatch(deps, batchId) {
|
|
|
3818
3851
|
const terminal = (0, state_1.findBatch)(deps.store.load(), batchId);
|
|
3819
3852
|
const lastMemberLanded = terminal?.ranges.some((r) => r.issue === terminal.members[terminal.executing_member - 1]) ===
|
|
3820
3853
|
true;
|
|
3821
|
-
const
|
|
3822
|
-
|
|
3823
|
-
teardownParallelRuns(deps, batchId, serialTree);
|
|
3854
|
+
const serial = teardownMemberWorktree(deps, batchId, deps.now(), lastMemberLanded);
|
|
3855
|
+
teardownParallelRuns(deps, batchId, serial);
|
|
3824
3856
|
const state = deps.store.load();
|
|
3825
3857
|
const batch = (0, state_1.findBatch)(state, batchId);
|
|
3826
3858
|
if (!batch || batch.worktree === null)
|
|
@@ -3886,13 +3918,30 @@ function runTree(r) {
|
|
|
3886
3918
|
: { worktree: r.worktree, branch: r.branch, poolClaimed: r.poolClaimed };
|
|
3887
3919
|
}
|
|
3888
3920
|
/**
|
|
3889
|
-
*
|
|
3921
|
+
* Tear one parallel member run's tree down and record what actually happened
|
|
3922
|
+
* (#855) — the one teardown-and-record path for the evict, land and
|
|
3923
|
+
* `teardownParallelRuns` rails.
|
|
3924
|
+
*/
|
|
3925
|
+
function teardownRunTree(deps, batchId, run, landed, now) {
|
|
3926
|
+
const cleared = teardownMemberTree(deps, batchId, runTree(run), landed, run.issue);
|
|
3927
|
+
recordRunTeardown(deps, batchId, run.issue, cleared, now);
|
|
3928
|
+
}
|
|
3929
|
+
/** The one mapping from a member-tree teardown outcome to its run record (#855). */
|
|
3930
|
+
function teardownOutcomePatch(cleared, now) {
|
|
3931
|
+
return cleared ? { torn_down: true } : { teardown_failed_at: now.toISOString() };
|
|
3932
|
+
}
|
|
3933
|
+
/**
|
|
3934
|
+
* Record a run's teardown outcome. Written AFTER the teardown, so a crash in
|
|
3890
3935
|
* between re-runs the (idempotent) teardown from `teardownParallelRuns`
|
|
3891
|
-
* instead of leaking the tree.
|
|
3936
|
+
* instead of leaking the tree. #855: `torn_down` is set only when the
|
|
3937
|
+
* teardown verifiably landed; a failed one records `teardown_failed_at`
|
|
3938
|
+
* instead and leaves the tree on disk for the operator (never retried — see
|
|
3939
|
+
* `MemberRun.teardown_failed_at`). The caller's teardown already journaled
|
|
3940
|
+
* the failure, once.
|
|
3892
3941
|
*/
|
|
3893
|
-
function
|
|
3942
|
+
function recordRunTeardown(deps, batchId, issue, cleared, now) {
|
|
3894
3943
|
deps.store.withLock((s) => ({
|
|
3895
|
-
state: patchRun(s, batchId, issue,
|
|
3944
|
+
state: patchRun(s, batchId, issue, teardownOutcomePatch(cleared, now), now),
|
|
3896
3945
|
result: undefined,
|
|
3897
3946
|
}));
|
|
3898
3947
|
}
|
|
@@ -4028,6 +4077,7 @@ function spawnParallelMembers(deps, config, dispatch, batchId, now, result) {
|
|
|
4028
4077
|
status: 'running',
|
|
4029
4078
|
gate_inconclusive: null,
|
|
4030
4079
|
torn_down: false,
|
|
4080
|
+
teardown_failed_at: null,
|
|
4031
4081
|
}, now);
|
|
4032
4082
|
if ((0, scheduler_1.freeCapacity)(n, config) === 0)
|
|
4033
4083
|
return { state: n, result: true };
|
|
@@ -4055,7 +4105,7 @@ function spawnParallelMembers(deps, config, dispatch, batchId, now, result) {
|
|
|
4055
4105
|
reason: 'member-worktree-orphaned',
|
|
4056
4106
|
detail: `batch left executing while member ${next.index}'s tree ${member.worktree} was prepared — tearing it down`,
|
|
4057
4107
|
});
|
|
4058
|
-
teardownMemberTree(deps, batchId, runTree(member), false);
|
|
4108
|
+
teardownMemberTree(deps, batchId, runTree(member), false, next.issue);
|
|
4059
4109
|
}
|
|
4060
4110
|
}
|
|
4061
4111
|
}
|
|
@@ -4076,8 +4126,7 @@ function evictParallelMember(deps, config, batchId, run, failure, now, result) {
|
|
|
4076
4126
|
state: patchRun(s, batchId, run.issue, { status: 'evicted', gate_inconclusive: null }, now),
|
|
4077
4127
|
result: undefined,
|
|
4078
4128
|
}));
|
|
4079
|
-
|
|
4080
|
-
markTornDown(deps, batchId, run.issue, now);
|
|
4129
|
+
teardownRunTree(deps, batchId, run, false, now);
|
|
4081
4130
|
}
|
|
4082
4131
|
/**
|
|
4083
4132
|
* Reconcile one parallel member's slot — the parallel twin of
|
|
@@ -4313,8 +4362,7 @@ function advanceParallelBatch(deps, config, dispatch, batchId, now, result) {
|
|
|
4313
4362
|
// A gate-inconclusive member keeps its tree for `sched resume --batch`'s
|
|
4314
4363
|
// recheck; everyone else's is done serving.
|
|
4315
4364
|
if (run.gate_inconclusive === null) {
|
|
4316
|
-
|
|
4317
|
-
markTornDown(deps, batchId, run.issue, now);
|
|
4365
|
+
teardownRunTree(deps, batchId, run, true, now);
|
|
4318
4366
|
}
|
|
4319
4367
|
}
|
|
4320
4368
|
}
|
|
@@ -4362,7 +4410,7 @@ function finalizeParallelMembers(deps, config, dispatch, batchId, now, result) {
|
|
|
4362
4410
|
* run so the landing walk treats it as resolved, then continue the parallel
|
|
4363
4411
|
* rail from where it blocked.
|
|
4364
4412
|
*/
|
|
4365
|
-
function settleParallelResume(deps, batchId, memberIssue, now) {
|
|
4413
|
+
function settleParallelResume(deps, batchId, memberIssue, now, serialTeardown) {
|
|
4366
4414
|
deps.store.withLock((s) => {
|
|
4367
4415
|
const b = (0, state_1.findBatch)(s, batchId);
|
|
4368
4416
|
if (!b)
|
|
@@ -4371,8 +4419,10 @@ function settleParallelResume(deps, batchId, memberIssue, now) {
|
|
|
4371
4419
|
return {
|
|
4372
4420
|
state: patchRun(s, batchId, memberIssue, {
|
|
4373
4421
|
gate_inconclusive: null,
|
|
4374
|
-
// The serial rail's `teardownMemberWorktree`
|
|
4375
|
-
|
|
4422
|
+
// The serial rail's `teardownMemberWorktree` tore its tree down —
|
|
4423
|
+
// recorded as torn down only when that verifiably landed (#855).
|
|
4424
|
+
// No serial tree at all leaves the run live for `teardownBatch`.
|
|
4425
|
+
...(serialTeardown !== null ? teardownOutcomePatch(serialTeardown.cleared, now) : {}),
|
|
4376
4426
|
...(evicted ? { status: 'evicted' } : {}),
|
|
4377
4427
|
}, now),
|
|
4378
4428
|
result: undefined,
|
|
@@ -4414,11 +4464,19 @@ function reconcileParallelBatch(deps, config, dispatch, batchId, now, result, pa
|
|
|
4414
4464
|
* would be working a unit nothing tracks any more), `sched stop`/`abandon`
|
|
4415
4465
|
* make a batch terminal without a teardown, and a crash between a run's
|
|
4416
4466
|
* resolution and its teardown leaves `torn_down: false` behind. Teardown is
|
|
4417
|
-
* idempotent, so re-running it is always safe. `
|
|
4418
|
-
*
|
|
4419
|
-
* member of a blocked parallel batch)
|
|
4467
|
+
* idempotent, so re-running it is always safe. `serial` is what
|
|
4468
|
+
* `teardownMemberWorktree` just did with the serial-field tree (a
|
|
4469
|
+
* gate-inconclusive member of a blocked parallel batch) — its outcome is
|
|
4470
|
+
* recorded on the matching run without a second attempt.
|
|
4471
|
+
*
|
|
4472
|
+
* #855: default to KEEPING. A kept batch's (`worktree_kept`, #768's
|
|
4473
|
+
* members-closed reconcile) trees may hold unpushed operator work: its live
|
|
4474
|
+
* agents are still stopped (non-destructive), but no tree is removed or
|
|
4475
|
+
* pool-returned and `torn_down` stays false, so `sched status`'s
|
|
4476
|
+
* `kept-worktree` warning names them. A run whose earlier teardown failed
|
|
4477
|
+
* (`teardown_failed_at`) is not re-attempted either.
|
|
4420
4478
|
*/
|
|
4421
|
-
function teardownParallelRuns(deps, batchId,
|
|
4479
|
+
function teardownParallelRuns(deps, batchId, serial) {
|
|
4422
4480
|
const batch = (0, state_1.findBatch)(deps.store.load(), batchId);
|
|
4423
4481
|
if (!batch)
|
|
4424
4482
|
return;
|
|
@@ -4427,15 +4485,25 @@ function teardownParallelRuns(deps, batchId, alreadyTornDown) {
|
|
|
4427
4485
|
if (run.torn_down)
|
|
4428
4486
|
continue;
|
|
4429
4487
|
stopMemberAgent(deps, batchId, run.issue, now);
|
|
4488
|
+
if (run.teardown_failed_at !== null)
|
|
4489
|
+
continue;
|
|
4430
4490
|
const landed = run.status === 'landed';
|
|
4431
|
-
|
|
4432
|
-
|
|
4433
|
-
|
|
4434
|
-
|
|
4435
|
-
|
|
4436
|
-
|
|
4491
|
+
const status = landed ? 'landed' : 'evicted';
|
|
4492
|
+
// Only when it changes: a kept batch's runs pass through here every tick.
|
|
4493
|
+
if (run.status !== status || run.gate_inconclusive !== null) {
|
|
4494
|
+
deps.store.withLock((s) => ({
|
|
4495
|
+
state: patchRun(s, batchId, run.issue, { status, gate_inconclusive: null }, now),
|
|
4496
|
+
result: undefined,
|
|
4497
|
+
}));
|
|
4498
|
+
}
|
|
4499
|
+
if (batch.worktree_kept)
|
|
4500
|
+
continue;
|
|
4501
|
+
if (serial !== null && run.worktree === serial.path) {
|
|
4502
|
+
recordRunTeardown(deps, batchId, run.issue, serial.cleared, now);
|
|
4503
|
+
}
|
|
4504
|
+
else {
|
|
4505
|
+
teardownRunTree(deps, batchId, run, landed, now);
|
|
4437
4506
|
}
|
|
4438
|
-
markTornDown(deps, batchId, run.issue, now);
|
|
4439
4507
|
}
|
|
4440
4508
|
}
|
|
4441
4509
|
/** Kill a parallel member's live agent (pid-start checked) and release its slot. */
|
|
@@ -4643,6 +4711,13 @@ function runBatchTick(deps, config, dispatch) {
|
|
|
4643
4711
|
if (types_1.TERMINAL_BATCH_STATUSES.has(batch.status)) {
|
|
4644
4712
|
// #809: `sched stop`/`abandon` end a batch without a teardown — give
|
|
4645
4713
|
// back every member tree (and stop any agent still running in one).
|
|
4714
|
+
// #855: except a KEPT batch's trees (`worktree_kept`, the members-closed
|
|
4715
|
+
// reconcile's decision — the same one `worktree`/`member_worktree`
|
|
4716
|
+
// get) and a run whose teardown already failed: `teardownParallelRuns`
|
|
4717
|
+
// leaves both on disk with `torn_down: false`, and marks a run torn
|
|
4718
|
+
// down only after a teardown that verifiably landed. Such a batch
|
|
4719
|
+
// re-enters here every tick by design — cheap (a state read per run),
|
|
4720
|
+
// and it stops any agent that is somehow still running in a kept tree.
|
|
4646
4721
|
if (batch.member_runs.some((r) => !r.torn_down)) {
|
|
4647
4722
|
try {
|
|
4648
4723
|
teardownParallelRuns(deps, batch.id, null);
|