@ai-dossier/sched 0.54.1 → 0.55.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +18 -3
- package/dist/batch-dispatch.d.ts +17 -24
- package/dist/batch-dispatch.d.ts.map +1 -1
- package/dist/batch-dispatch.js +143 -74
- package/dist/batch-dispatch.js.map +1 -1
- package/dist/state.d.ts +11 -1
- package/dist/state.d.ts.map +1 -1
- package/dist/state.js +33 -0
- package/dist/state.js.map +1 -1
- package/dist/status.d.ts +10 -7
- package/dist/status.d.ts.map +1 -1
- package/dist/status.js +20 -14
- package/dist/status.js.map +1 -1
- package/dist/types.d.ts +35 -5
- package/dist/types.d.ts.map +1 -1
- package/dist/types.js +5 -1
- package/dist/types.js.map +1 -1
- package/package.json +1 -1
package/README.md
CHANGED
|
@@ -807,6 +807,11 @@ failure rails: executing → dissolving (a member self-reports blocked)
|
|
|
807
807
|
an eviction group, or a member `deps` on another member; a batch already past `ready`
|
|
808
808
|
with no recorded mode (claimed by a pre-1.23.0 engine) stays serial across the upgrade.
|
|
809
809
|
Dissolve, `sched stop --batch` and `abandon` stop/release every member slot.
|
|
810
|
+
A run's `torn_down` is set only after its tree teardown verifiably landed (path gone
|
|
811
|
+
and unlisted, or the pool reports it returned); a failed one records
|
|
812
|
+
`teardown_failed_at`, journals `teardown-failed` once and is never retried — the tree
|
|
813
|
+
stays for the operator and `sched status` names it (`kept-worktree`) (#855). A kept batch's (`worktree_kept`) runs are never torn
|
|
814
|
+
down by the scheduler.
|
|
810
815
|
- **Serial mode: members run `member-cycle` one fresh agent at a time — each in its OWN
|
|
811
816
|
worktree** on its OWN branch `batch/<id>-m<n>-<issue>` (#677, RFC-0001 §J.3), cut off
|
|
812
817
|
the integration branch, warmed, and pushed before the agent spawns; the agent never
|
|
@@ -1426,7 +1431,7 @@ after-the-fact recovery, not a missing-data bug.
|
|
|
1426
1431
|
queue data.
|
|
1427
1432
|
- **Schema**: state/config files from #460 (schema 1.0.0), #464 (1.1.0), #468 (1.2.0),
|
|
1428
1433
|
#472 (1.3.0), #500 (1.4.0), #505 (1.5.0), #504 (1.6.0), #523 (1.7.0) and #524 (1.8.0)
|
|
1429
|
-
load and migrate to the current schema (1.
|
|
1434
|
+
load and migrate to the current schema (1.28.0 — 1.24.0 was #810: no backfill,
|
|
1430
1435
|
`kind`/`branch` optional, absent = an `evicted` record with no branch) automatically
|
|
1431
1436
|
(slot `branch`/`last_head`/`pid_start`, slot `role` (inferred from the
|
|
1432
1437
|
unit's queue entry, with the persisted `phase` as a fallback — #500), entry
|
|
@@ -1449,6 +1454,10 @@ after-the-fact recovery, not a missing-data bug.
|
|
|
1449
1454
|
respawn counter); `null` is backfilled on load.
|
|
1450
1455
|
Schema 1.27.0 (#822): `BatchEntry` gains `reprompted_members`
|
|
1451
1456
|
(`{ issue, milestone_at, reprompted_at }` records; `[]` backfilled).
|
|
1457
|
+
Schema 1.28.0 (#855): `BatchEntry` gains `worktree_kept` (the members-closed keep
|
|
1458
|
+
decision, persisted; `false` backfilled) and `MemberRun` gains `teardown_failed_at`
|
|
1459
|
+
(`null` backfilled). A 1.27.0 `done` batch still carrying a live member run loads as
|
|
1460
|
+
kept — the old engine never recorded the decision, so the default is to keep.
|
|
1452
1461
|
- **`max_slots`** bounds live units (`assigned | running | recovering`); dependency
|
|
1453
1462
|
edges gate readiness — an issue with an unmerged dependency, and a batch behind an
|
|
1454
1463
|
unmerged batch, are never runnable.
|
|
@@ -1486,7 +1495,11 @@ to skip the check entirely):
|
|
|
1486
1495
|
- **`kept-worktree`** (#791) — a `done` batch whose `worktree` or `member_worktree` is
|
|
1487
1496
|
still set in the ledger: the `members-closed` stale-blocked reconcile below
|
|
1488
1497
|
intentionally leaves it in place (it may hold unpushed operator repair work), and
|
|
1489
|
-
nothing else surfaced it until now.
|
|
1498
|
+
nothing else surfaced it until now. Also (#834/#855) each parallel `member_runs[]`
|
|
1499
|
+
tree left on disk: every not-torn-down run of a `done` batch (a kept batch's, or one
|
|
1500
|
+
a crash stranded), and a run of any ended batch whose teardown FAILED
|
|
1501
|
+
(`teardown_failed_at` — the message says so; the cause is in the `teardown-failed`
|
|
1502
|
+
journal line, and the scheduler never retries it). For each kept path, `sched status` reports:
|
|
1490
1503
|
whether it still exists on disk; whether it is pool-claimed (the claim is held
|
|
1491
1504
|
indefinitely until returned); and, via two local, read-only `git` probes scoped to
|
|
1492
1505
|
that worktree's own `HEAD` (`git --no-optional-locks status --porcelain`, `git log
|
|
@@ -1544,7 +1557,9 @@ predicate in `anchor-close.ts`) over `blocked`/`done` batches touched within the
|
|
|
1544
1557
|
for #686's stale-blocked reconcile (`members-closed`), which survives the mandated
|
|
1545
1558
|
rebase-merge that makes the ancestry probe structurally dead. It finishes the terminal
|
|
1546
1559
|
rail inline but never tears the blocked worktree down (it may hold unpushed work);
|
|
1547
|
-
|
|
1560
|
+
since #855 the decision is persisted as `worktree_kept`, so the terminal-batch safety
|
|
1561
|
+
net leaves the batch's parallel `member_runs[]` trees in place too (`torn_down` stays
|
|
1562
|
+
false). The journal line says the worktree was kept, and `sched status` then raises a
|
|
1548
1563
|
`kept-worktree` warning for it on every run until the ledger field clears — see
|
|
1549
1564
|
above and `reconcileKeptWorktrees` (#791).
|
|
1550
1565
|
- **PR-detection evidence (#789).** When `batch.pr` was never recorded at all — a PR
|
package/dist/batch-dispatch.d.ts
CHANGED
|
@@ -319,35 +319,28 @@ export declare function reconcileStaleBlockedBatches(deps: BatchDispatchDeps, no
|
|
|
319
319
|
* #834: `MemberRun.worktree` is a required `string` (unlike
|
|
320
320
|
* `BatchEntry.worktree: string | null`) — there is no null to clear it to,
|
|
321
321
|
* so "cleared" for a member run means `torn_down: true`, the same flag
|
|
322
|
-
* `
|
|
323
|
-
* thing evidentially (nothing is holding this path
|
|
324
|
-
* by definitive evidence instead of by running
|
|
322
|
+
* `recordRunTeardown` sets after a teardown that verifiably landed. Set
|
|
323
|
+
* here it means the same thing evidentially (nothing is holding this path
|
|
324
|
+
* any more), just reached by definitive evidence instead of by running
|
|
325
|
+
* `teardownParallelRuns`.
|
|
325
326
|
*
|
|
326
|
-
* #834
|
|
327
|
-
*
|
|
328
|
-
*
|
|
329
|
-
*
|
|
330
|
-
*
|
|
331
|
-
* `
|
|
332
|
-
*
|
|
333
|
-
*
|
|
334
|
-
*
|
|
335
|
-
*
|
|
336
|
-
*
|
|
337
|
-
* is effectively a backstop, not the primary path: by the time it runs,
|
|
338
|
-
* `torn_down` is usually already `true` — correctly when the tree really
|
|
339
|
-
* is gone, but also when a real teardown attempt failed and left the tree
|
|
340
|
-
* on disk — filed as #855 (out of scope here; #834 only extends the
|
|
341
|
-
* evidence-based ledger reconcile, it does not change teardown behavior).
|
|
342
|
-
* This function's own branch still matters for the window before that
|
|
343
|
-
* safety net's next tick runs, and for `sched status` reads taken while
|
|
344
|
-
* the scheduler isn't ticking at all.
|
|
327
|
+
* #834/#855: a terminal batch's live `member_runs[]` entries are ALSO
|
|
328
|
+
* reached by the terminal-batch arm in `runBatchTick` (search
|
|
329
|
+
* `TERMINAL_BATCH_STATUSES.has(batch.status)` above the call site below),
|
|
330
|
+
* added by #809 for `sched stop`/`abandon`, which runs earlier in the same
|
|
331
|
+
* tick than this function. Since #855 that arm respects the keep decision
|
|
332
|
+
* (`worktree_kept`) and marks a run torn down only after a teardown that
|
|
333
|
+
* verifiably landed, so on a KEPT batch, or a run whose teardown failed
|
|
334
|
+
* (`teardown_failed_at`), `torn_down` stays false and this function is the
|
|
335
|
+
* only thing that ever clears it — once the operator has removed or
|
|
336
|
+
* returned the tree. On a non-kept batch the arm usually tears the tree
|
|
337
|
+
* down (and records it) first.
|
|
345
338
|
*
|
|
346
339
|
* Exported (not part of the package's public `index.ts` surface — same
|
|
347
340
|
* test-only convention as `reconcileStaleBlockedBatches`) so `member_runs[]`
|
|
348
341
|
* tests can call it directly, isolated from the terminal-batch safety net
|
|
349
|
-
* described above, which
|
|
350
|
-
* both run inside one `runBatchTick` call.
|
|
342
|
+
* described above, which reaches a non-kept batch's `member_runs[]` first
|
|
343
|
+
* when both run inside one `runBatchTick` call.
|
|
351
344
|
*/
|
|
352
345
|
export declare function reconcileKeptWorktrees(deps: BatchDispatchDeps, now: Date): void;
|
|
353
346
|
/**
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"batch-dispatch.d.ts","sourceRoot":"","sources":["../src/batch-dispatch.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA8DG;AAUH,OAAO,EAIL,KAAK,YAAY,EAGjB,KAAK,gBAAgB,EAGtB,MAAM,gBAAgB,CAAC;AAUxB,OAAO,EAaL,KAAK,gBAAgB,EAGrB,KAAK,SAAS,EAEf,MAAM,YAAY,CAAC;AAEpB,OAAO,EAGL,KAAK,WAAW,EAcjB,MAAM,eAAe,CAAC;AACvB,OAAO,EAAE,KAAK,OAAO,EAAa,MAAM,WAAW,CAAC;AACpD,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,WAAW,CAAC;AAC5C,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,WAAW,CAAC;AAExC,OAAO,EACL,KAAK,iBAAiB,EAYtB,KAAK,WAAW,EACjB,MAAM,YAAY,CAAC;AAGpB,OAAO,
|
|
1
|
+
{"version":3,"file":"batch-dispatch.d.ts","sourceRoot":"","sources":["../src/batch-dispatch.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA8DG;AAUH,OAAO,EAIL,KAAK,YAAY,EAGjB,KAAK,gBAAgB,EAGtB,MAAM,gBAAgB,CAAC;AAUxB,OAAO,EAaL,KAAK,gBAAgB,EAGrB,KAAK,SAAS,EAEf,MAAM,YAAY,CAAC;AAEpB,OAAO,EAGL,KAAK,WAAW,EAcjB,MAAM,eAAe,CAAC;AACvB,OAAO,EAAE,KAAK,OAAO,EAAa,MAAM,WAAW,CAAC;AACpD,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,WAAW,CAAC;AAC5C,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,WAAW,CAAC;AAExC,OAAO,EACL,KAAK,iBAAiB,EAYtB,KAAK,WAAW,EACjB,MAAM,YAAY,CAAC;AAGpB,OAAO,EAyBL,kBAAkB,EAKnB,MAAM,SAAS,CAAC;AACjB,OAAO,EACL,KAAK,QAAQ,EAMd,MAAM,YAAY,CAAC;AACpB,OAAO,KAAK,EAGV,UAAU,EAEV,oBAAoB,EAKpB,kBAAkB,EAClB,cAAc,EAId,WAAW,EACX,UAAU,EAEX,MAAM,SAAS,CAAC;AAcjB,OAAO,EAAE,kBAAkB,EAAE,CAAC;AAK9B,YAAY,EAAE,UAAU,EAAE,MAAM,SAAS,CAAC;AAE1C,8DAA8D;AAC9D,MAAM,WAAW,iBAAiB;IAChC,KAAK,EAAE,UAAU,CAAC;IAClB,OAAO,EAAE,OAAO,CAAC;IACjB,WAAW,EAAE,WAAW,CAAC;IACzB,SAAS,EAAE,SAAS,CAAC;IACrB,GAAG,EAAE,MAAM,IAAI,CAAC;IAChB,uFAAuF;IACvF,OAAO,EAAE,MAAM,CAAC;IAChB,0FAA0F;IAC1F,IAAI,EAAE,MAAM,CAAC;IACb;;;;;;OAMG;IACH,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB;;;;;OAKG;IACH,QAAQ,EAAE,CAAC,QAAQ,EAAE,MAAM,EAAE,GAAG,CAAC,EAAE,iBAAiB,KAAK,WAAW,CAAC;IACrE;;;;;;;;;;;;;;;;;;;OAmBG;IACH,aAAa,CAAC,EAAE,CAAC,QAAQ,EAAE,MAAM,EAAE,YAAY,EAAE,MAAM,KAAK,oBAAoB,CAAC;IACjF,QAAQ,CAAC,EAAE,QAAQ,CAAC;IACpB;;;OAGG;IACH,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB;;;;;;OAMG;IACH,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB;AAED,8FAA8F;AAC9F,MAAM,WAAW,eAAe;IAC9B,OAAO,EAAE,MAAM,EAAE,CAAC;IAClB,SAAS,EAAE,MAAM,EAAE,CAAC;IACpB,MAAM,EAAE,MAAM,EAAE,CAAC;IACjB,aAAa,EAAE,MAAM,EAAE,CAAC;IACxB,MAAM,EAAE,MAAM,EAAE,CAAC;IACjB,uFAAuF;IACvF,qBAAqB,EAAE,MAAM,EAAE,CAAC;IAChC,8FAA8F;IAC9F,OAAO,EAAE,MAAM,EAAE,CAAC;CACnB;AAED;;;;;;GAMG;AACH,MAAM,WAAW,aAAa;IAC5B,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACjC;;;;OAIG;IACH,IAAI,CAAC,EAAE,cAAc,CAAC;CACvB;AAylBD;;;;;;;GAOG;AACH,wBAAgB,eAAe,CAAC,OAAO,EAAE,MAAM,EAAE,WAAW,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,MAAM,CAE3F;AAs+CD;;;;;;;;;;;;;;;;;;;;;;;;GAwBG;AACH,wBAAgB,sBAAsB,CACpC,IAAI,EAAE,iBAAiB,EACvB,MAAM,EAAE,WAAW,EACnB,QAAQ,EAAE,gBAAgB,EAC1B,OAAO,EAAE,MAAM,EACf,KAAK,EAAE,UAAU,EACjB,WAAW,EAAE,MAAM,EACnB,OAAO,EAAE,aAAa,EACtB,GAAG,EAAE,IAAI,EACT,MAAM,EAAE,eAAe,GACtB,IAAI,CAgCN;AA4TD;;;;;;;;;;;;;;;;;;;;;;;;;;;GA2BG;AACH,wBAAgB,iBAAiB,CAC/B,IAAI,EAAE,iBAAiB,EACvB,MAAM,EAAE,WAAW,EACnB,QAAQ,EAAE,gBAAgB,EAC1B,OAAO,EAAE,MAAM,EACf,GAAG,EAAE,IAAI,GACR;IACD,OAAO,EAAE,eAAe,GAAG,SAAS,GAAG,WAAW,GAAG,SAAS,CAAC;IAC/D,UAAU,EAAE,MAAM,CAAC;IACnB;;;;;OAKG;IACH,SAAS,CAAC,EAAE,cAAc,GAAG,qBAAqB,CAAC;IACnD,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB,CA2KA;AAuxBD;;;;;;;;GAQG;AACH,eAAO,MAAM,wBAAwB,IAAI,CAAC;AAkf1C;;;;;;GAMG;AACH,wBAAgB,4BAA4B,CAAC,KAAK,EAAE,UAAU,EAAE,KAAK,EAAE,UAAU,GAAG,MAAM,EAAE,CAuB3F;AA6HD;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAwCG;AACH,wBAAgB,4BAA4B,CAC1C,IAAI,EAAE,iBAAiB,EACvB,GAAG,EAAE,IAAI,EACT,MAAM,EAAE,eAAe,EACvB,SAAS,EAAE,iBAAiB,GAAG,SAAS,GACvC,IAAI,CAmMN;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAyCG;AACH,wBAAgB,sBAAsB,CAAC,IAAI,EAAE,iBAAiB,EAAE,GAAG,EAAE,IAAI,GAAG,IAAI,CAuG/E;AAED;;;;;;GAMG;AACH,UAAU,iBAAiB;IACzB,IAAI,EAAE,MAAM,CAAC;IACb,gDAAgD;IAChD,IAAI,EAAE,gBAAgB,CAAC;IACvB,uEAAuE;IACvE,SAAS,EAAE,gBAAgB,CAAC;IAC5B,YAAY,EAAE,YAAY,CAAC;CAC5B;AAoRD;;;;;;;;;;;GAWG;AACH,oCAAoC;AACpC,wBAAgB,qBAAqB,CACnC,KAAK,EAAE,UAAU,EACjB,KAAK,EAAE,UAAU,EACjB,MAAM,EAAE,WAAW,GAClB,kBAAkB,CAQpB;AAi3BD;;;;;;;;;;;;;GAaG;AACH,wBAAgB,YAAY,CAC1B,IAAI,EAAE,iBAAiB,EACvB,MAAM,EAAE,WAAW,EACnB,QAAQ,EAAE,gBAAgB,GACzB,eAAe,CAuSjB"}
|
package/dist/batch-dispatch.js
CHANGED
|
@@ -820,7 +820,7 @@ landed) {
|
|
|
820
820
|
const state = deps.store.load();
|
|
821
821
|
const batch = (0, state_1.findBatch)(state, batchId);
|
|
822
822
|
if (!batch || batch.member_worktree === null)
|
|
823
|
-
return;
|
|
823
|
+
return null;
|
|
824
824
|
const worktree = batch.member_worktree;
|
|
825
825
|
const branch = batch.member_branch;
|
|
826
826
|
// #677 review (defense in depth, matching `branchMergedIntoBase`'s bar):
|
|
@@ -831,7 +831,7 @@ landed) {
|
|
|
831
831
|
cleanup: 'failed-invalid-branch-name',
|
|
832
832
|
detail: branch,
|
|
833
833
|
});
|
|
834
|
-
return;
|
|
834
|
+
return { path: worktree, cleared: false };
|
|
835
835
|
}
|
|
836
836
|
// Nulls first — clear the fields BEFORE the exec so a crash mid-teardown
|
|
837
837
|
// doesn't retry into a half-cleaned tree (the re-derived path is
|
|
@@ -845,23 +845,39 @@ landed) {
|
|
|
845
845
|
result: undefined,
|
|
846
846
|
};
|
|
847
847
|
});
|
|
848
|
-
teardownMemberTree(deps, batchId, { worktree, branch, poolClaimed: batch.member_pool_claimed === true }, landed);
|
|
848
|
+
const cleared = teardownMemberTree(deps, batchId, { worktree, branch, poolClaimed: batch.member_pool_claimed === true }, landed, batch.members[batch.executing_member - 1]);
|
|
849
|
+
return { path: worktree, cleared };
|
|
849
850
|
}
|
|
851
|
+
/** #855: the `teardown-failed` tail for a member tree that stays on disk. */
|
|
852
|
+
const TREE_LEFT_FOR_OPERATOR = 'tree left on disk for the operator and NOT retried (it may hold work); inspect it, then remove it (`git worktree remove`, or `worktree-pool return` if pool-claimed) — `sched status` names it';
|
|
850
853
|
/**
|
|
851
854
|
* The exec half of a member teardown, shared by the serial rail
|
|
852
855
|
* (`teardownMemberWorktree`, which first clears the batch's member fields) and
|
|
853
856
|
* the parallel rail (#809: `evictParallelMember`, `advanceParallelBatch`,
|
|
854
857
|
* `teardownParallelRuns`). See
|
|
855
858
|
* `teardownMemberWorktree` for the landed/evicted branch policy.
|
|
859
|
+
*
|
|
860
|
+
* Returns whether the TREE is verifiably gone (#855) — `runTeardown` reports
|
|
861
|
+
* `done` only after the path is gone and unlisted, or the pool's self-check
|
|
862
|
+
* reports the entry returned. A failed branch delete after that does not
|
|
863
|
+
* un-clear the tree. Journals exactly one line per call; a failed tree is
|
|
864
|
+
* never re-attempted by any caller, so the line says what the operator does.
|
|
856
865
|
*/
|
|
857
|
-
function teardownMemberTree(deps, batchId, tree, landed
|
|
866
|
+
function teardownMemberTree(deps, batchId, tree, landed,
|
|
867
|
+
/** The member whose tree this is — named in the journal line. */
|
|
868
|
+
issue) {
|
|
858
869
|
const { worktree, branch } = tree;
|
|
870
|
+
const context = {
|
|
871
|
+
...(issue !== undefined ? { issue } : {}),
|
|
872
|
+
worktree,
|
|
873
|
+
};
|
|
859
874
|
if (branch !== null && !attribution_1.SAFE_REF_RE.test(branch)) {
|
|
860
875
|
journalEvent(deps, 'teardown-failed', unit(batchId), {
|
|
876
|
+
...context,
|
|
861
877
|
cleanup: 'failed-invalid-branch-name',
|
|
862
|
-
detail: branch
|
|
878
|
+
detail: `${branch} — ${TREE_LEFT_FOR_OPERATOR}`,
|
|
863
879
|
});
|
|
864
|
-
return;
|
|
880
|
+
return false;
|
|
865
881
|
}
|
|
866
882
|
// Route through `runTeardown` for the same guarantees `teardownBatch`
|
|
867
883
|
// gets: pool returns are verify-first idempotent and refuse to claim
|
|
@@ -884,11 +900,12 @@ function teardownMemberTree(deps, batchId, tree, landed) {
|
|
|
884
900
|
}
|
|
885
901
|
const failed = !treeCleared || branchCleanup.startsWith('failed');
|
|
886
902
|
journalEvent(deps, failed ? 'teardown-failed' : 'member-worktree-torn-down', unit(batchId), {
|
|
903
|
+
...context,
|
|
887
904
|
cleanup: t.cleanup,
|
|
888
905
|
branch_cleanup: branchCleanup,
|
|
889
|
-
detail: t.detail
|
|
890
|
-
worktree,
|
|
906
|
+
detail: treeCleared ? t.detail : `${t.detail} — ${TREE_LEFT_FOR_OPERATOR}`,
|
|
891
907
|
});
|
|
908
|
+
return treeCleared;
|
|
892
909
|
}
|
|
893
910
|
/**
|
|
894
911
|
* The shared "block the batch for an operator" tail (#677 review — the same
|
|
@@ -1717,13 +1734,15 @@ function journalAdvanceSkipped(deps, batchId, memberIssue, currentMember, claim)
|
|
|
1717
1734
|
detail: `another resolution already advanced past member ${currentMember} (batch is now executing_member=${claim.observed ?? 'gone'}, status=${claim.status ?? 'gone'}) — no second member-advanced and no second continuation`,
|
|
1718
1735
|
});
|
|
1719
1736
|
}
|
|
1720
|
-
function advanceMemberOrValidate(deps, config, dispatch, batchId, memberCount, currentMember, memberIssue, now, result
|
|
1737
|
+
function advanceMemberOrValidate(deps, config, dispatch, batchId, memberCount, currentMember, memberIssue, now, result,
|
|
1738
|
+
/** What the caller's `teardownMemberWorktree` did with the member's tree (#855). */
|
|
1739
|
+
serialTeardown) {
|
|
1721
1740
|
// #809: a parallel batch reaches this serial rail only through `sched resume
|
|
1722
1741
|
// --batch` resolving its gate-inconclusive member — hand back to the
|
|
1723
1742
|
// parallel rail instead of advancing a serial member pointer.
|
|
1724
1743
|
const current = (0, state_1.findBatch)(deps.store.load(), batchId);
|
|
1725
1744
|
if (current && isParallelBatch(current)) {
|
|
1726
|
-
settleParallelResume(deps, batchId, memberIssue, now);
|
|
1745
|
+
settleParallelResume(deps, batchId, memberIssue, now, serialTeardown);
|
|
1727
1746
|
advanceParallelBatch(deps, config, dispatch, batchId, now, result);
|
|
1728
1747
|
return;
|
|
1729
1748
|
}
|
|
@@ -1789,8 +1808,8 @@ function evictMemberAndContinue(deps, config, dispatch, batchId, batch, memberIs
|
|
|
1789
1808
|
// branch — but its worktree/branch must go, or the next tick's prep for
|
|
1790
1809
|
// the NEXT member collides with a stale tree. `teardownBatch` already
|
|
1791
1810
|
// handled it on the dissolved path (the fields are cleared there).
|
|
1792
|
-
teardownMemberWorktree(deps, batchId, now, false);
|
|
1793
|
-
advanceMemberOrValidate(deps, config, dispatch, batchId, batch.members.length, batch.executing_member, memberIssue, now, result);
|
|
1811
|
+
const serialTeardown = teardownMemberWorktree(deps, batchId, now, false);
|
|
1812
|
+
advanceMemberOrValidate(deps, config, dispatch, batchId, batch.members.length, batch.executing_member, memberIssue, now, result, serialTeardown);
|
|
1794
1813
|
}
|
|
1795
1814
|
/**
|
|
1796
1815
|
* The incremental gate (#523 AC2, revised #583): typecheck + focused tests
|
|
@@ -2027,8 +2046,8 @@ function completeMemberGate(deps, config, dispatch, batchId, batch, memberIssue,
|
|
|
2027
2046
|
// #677: the member's tree is done serving — landed, ranges recomputed.
|
|
2028
2047
|
// Teardown before the advance so the next member's prep cannot collide
|
|
2029
2048
|
// with this tree's cleanup.
|
|
2030
|
-
teardownMemberWorktree(deps, batchId, now, true);
|
|
2031
|
-
advanceMemberOrValidate(deps, config, dispatch, batchId, batch.members.length, batch.executing_member, memberIssue, now, result);
|
|
2049
|
+
const serialTeardown = teardownMemberWorktree(deps, batchId, now, true);
|
|
2050
|
+
advanceMemberOrValidate(deps, config, dispatch, batchId, batch.members.length, batch.executing_member, memberIssue, now, result, serialTeardown);
|
|
2032
2051
|
}
|
|
2033
2052
|
/**
|
|
2034
2053
|
* `sched resume --batch <id>` (#583 AC4): an operator-triggered, synchronous
|
|
@@ -2920,7 +2939,7 @@ function reconcileTailSlot(deps, config, batchId, slot, now, result) {
|
|
|
2920
2939
|
result.failed.push(unit(batchId));
|
|
2921
2940
|
}
|
|
2922
2941
|
}
|
|
2923
|
-
function reconcileReportSlot(deps,
|
|
2942
|
+
function reconcileReportSlot(deps, _config, batchId, slot, now, result) {
|
|
2924
2943
|
const state = deps.store.load();
|
|
2925
2944
|
const batch = (0, state_1.findBatch)(state, batchId);
|
|
2926
2945
|
if (!batch || batch.anchor === null)
|
|
@@ -3430,6 +3449,10 @@ function reconcileStaleBlockedBatches(deps, now, result, anchorCtx) {
|
|
|
3430
3449
|
// clear into this same transition instead of a separate lock write.
|
|
3431
3450
|
...state_1.CLEARED_PR_DETECT_AMBIGUOUS_FIELDS,
|
|
3432
3451
|
...(prWasRecorded ? { pr: detectedPr } : {}),
|
|
3452
|
+
// #855: persist the keep decision in the same transition, so every
|
|
3453
|
+
// later teardown path — the terminal-batch safety net included —
|
|
3454
|
+
// reads it for `member_runs[]` too.
|
|
3455
|
+
...(keepWorktree ? { worktree_kept: true } : {}),
|
|
3433
3456
|
}, now);
|
|
3434
3457
|
n = (0, state_1.transitionBatch)(n, batch.id, 'deployed', {}, now);
|
|
3435
3458
|
unwalkable = [];
|
|
@@ -3520,46 +3543,42 @@ function reconcileStaleBlockedBatches(deps, now, result, anchorCtx) {
|
|
|
3520
3543
|
* #834: `MemberRun.worktree` is a required `string` (unlike
|
|
3521
3544
|
* `BatchEntry.worktree: string | null`) — there is no null to clear it to,
|
|
3522
3545
|
* so "cleared" for a member run means `torn_down: true`, the same flag
|
|
3523
|
-
* `
|
|
3524
|
-
* thing evidentially (nothing is holding this path
|
|
3525
|
-
* by definitive evidence instead of by running
|
|
3546
|
+
* `recordRunTeardown` sets after a teardown that verifiably landed. Set
|
|
3547
|
+
* here it means the same thing evidentially (nothing is holding this path
|
|
3548
|
+
* any more), just reached by definitive evidence instead of by running
|
|
3549
|
+
* `teardownParallelRuns`.
|
|
3526
3550
|
*
|
|
3527
|
-
* #834
|
|
3528
|
-
*
|
|
3529
|
-
*
|
|
3530
|
-
*
|
|
3531
|
-
*
|
|
3532
|
-
* `
|
|
3533
|
-
*
|
|
3534
|
-
*
|
|
3535
|
-
*
|
|
3536
|
-
*
|
|
3537
|
-
*
|
|
3538
|
-
* is effectively a backstop, not the primary path: by the time it runs,
|
|
3539
|
-
* `torn_down` is usually already `true` — correctly when the tree really
|
|
3540
|
-
* is gone, but also when a real teardown attempt failed and left the tree
|
|
3541
|
-
* on disk — filed as #855 (out of scope here; #834 only extends the
|
|
3542
|
-
* evidence-based ledger reconcile, it does not change teardown behavior).
|
|
3543
|
-
* This function's own branch still matters for the window before that
|
|
3544
|
-
* safety net's next tick runs, and for `sched status` reads taken while
|
|
3545
|
-
* the scheduler isn't ticking at all.
|
|
3551
|
+
* #834/#855: a terminal batch's live `member_runs[]` entries are ALSO
|
|
3552
|
+
* reached by the terminal-batch arm in `runBatchTick` (search
|
|
3553
|
+
* `TERMINAL_BATCH_STATUSES.has(batch.status)` above the call site below),
|
|
3554
|
+
* added by #809 for `sched stop`/`abandon`, which runs earlier in the same
|
|
3555
|
+
* tick than this function. Since #855 that arm respects the keep decision
|
|
3556
|
+
* (`worktree_kept`) and marks a run torn down only after a teardown that
|
|
3557
|
+
* verifiably landed, so on a KEPT batch, or a run whose teardown failed
|
|
3558
|
+
* (`teardown_failed_at`), `torn_down` stays false and this function is the
|
|
3559
|
+
* only thing that ever clears it — once the operator has removed or
|
|
3560
|
+
* returned the tree. On a non-kept batch the arm usually tears the tree
|
|
3561
|
+
* down (and records it) first.
|
|
3546
3562
|
*
|
|
3547
3563
|
* Exported (not part of the package's public `index.ts` surface — same
|
|
3548
3564
|
* test-only convention as `reconcileStaleBlockedBatches`) so `member_runs[]`
|
|
3549
3565
|
* tests can call it directly, isolated from the terminal-batch safety net
|
|
3550
|
-
* described above, which
|
|
3551
|
-
* both run inside one `runBatchTick` call.
|
|
3566
|
+
* described above, which reaches a non-kept batch's `member_runs[]` first
|
|
3567
|
+
* when both run inside one `runBatchTick` call.
|
|
3552
3568
|
*/
|
|
3553
3569
|
function reconcileKeptWorktrees(deps, now) {
|
|
3554
3570
|
const state = deps.store.load();
|
|
3555
3571
|
const fsExists = deps.fsExists ?? ((p) => fs.existsSync(p));
|
|
3556
|
-
|
|
3557
|
-
|
|
3572
|
+
// #855: a `stopped`/`dissolved` batch's run whose teardown failed is a
|
|
3573
|
+
// leftover too (`isLeftoverMemberRun`) — its ledger clears on the same
|
|
3574
|
+
// evidence once the operator has dealt with it.
|
|
3575
|
+
const candidates = state.batches.filter((b) => (b.status === 'done' && (b.worktree !== null || b.member_worktree !== null)) ||
|
|
3576
|
+
b.member_runs.some((r) => (0, state_1.isLeftoverMemberRun)(b, r)));
|
|
3558
3577
|
if (candidates.length === 0)
|
|
3559
3578
|
return;
|
|
3560
3579
|
const needsPoolCheck = candidates.some((b) => (b.worktree !== null && b.pool_claimed) ||
|
|
3561
3580
|
(b.member_worktree !== null && b.member_pool_claimed) ||
|
|
3562
|
-
b.member_runs.some((r) =>
|
|
3581
|
+
b.member_runs.some((r) => r.pool_claimed && (0, state_1.isLeftoverMemberRun)(b, r)));
|
|
3563
3582
|
// One pool query per TICK, not per candidate — `worktree-pool status` is a
|
|
3564
3583
|
// real subprocess (`npx ...`), and this reconcile is meant to be cheap.
|
|
3565
3584
|
const poolStatusJson = needsPoolCheck
|
|
@@ -3576,12 +3595,14 @@ function reconcileKeptWorktrees(deps, now) {
|
|
|
3576
3595
|
for (const batch of candidates) {
|
|
3577
3596
|
const clears = {};
|
|
3578
3597
|
const clearedFields = [];
|
|
3579
|
-
|
|
3598
|
+
const done = batch.status === 'done';
|
|
3599
|
+
if (done && batch.worktree !== null && goneOrReturned(batch.worktree, batch.pool_claimed)) {
|
|
3580
3600
|
clears.worktree = null;
|
|
3581
3601
|
clears.pool_claimed = false;
|
|
3582
3602
|
clearedFields.push('worktree', 'pool_claimed');
|
|
3583
3603
|
}
|
|
3584
|
-
if (
|
|
3604
|
+
if (done &&
|
|
3605
|
+
batch.member_worktree !== null &&
|
|
3585
3606
|
goneOrReturned(batch.member_worktree, batch.member_pool_claimed)) {
|
|
3586
3607
|
clears.member_worktree = null;
|
|
3587
3608
|
clears.member_pool_claimed = false;
|
|
@@ -3589,7 +3610,7 @@ function reconcileKeptWorktrees(deps, now) {
|
|
|
3589
3610
|
}
|
|
3590
3611
|
// #834: unlike `worktree`/`member_worktree`, `member_runs[]` is actively
|
|
3591
3612
|
// mutated by many OTHER code paths while a tick runs (member dispatch,
|
|
3592
|
-
// eviction, `
|
|
3613
|
+
// eviction, `recordRunTeardown`, `sched attach-pr`, ...), so its evidence
|
|
3593
3614
|
// check must NOT be computed from this function's own outer, unlocked
|
|
3594
3615
|
// `batch` snapshot: `patchBatch` merges `member_runs` as a whole-array
|
|
3595
3616
|
// replacement, not per-entry, so writing an array built from a stale
|
|
@@ -3599,7 +3620,7 @@ function reconcileKeptWorktrees(deps, now) {
|
|
|
3599
3620
|
// to look at" pre-check runs out here, to decide whether entering the
|
|
3600
3621
|
// lock is worth it at all; the real per-run evidence check re-reads
|
|
3601
3622
|
// `member_runs` fresh, from the batch `withLock` reloads below.
|
|
3602
|
-
const mightClearRuns = batch.member_runs.some((r) =>
|
|
3623
|
+
const mightClearRuns = batch.member_runs.some((r) => (0, state_1.isLeftoverMemberRun)(batch, r));
|
|
3603
3624
|
if (clearedFields.length === 0 && !mightClearRuns)
|
|
3604
3625
|
continue;
|
|
3605
3626
|
const clearedRunIssues = [];
|
|
@@ -3607,12 +3628,12 @@ function reconcileKeptWorktrees(deps, now) {
|
|
|
3607
3628
|
const b = (0, state_1.findBatch)(s, batch.id);
|
|
3608
3629
|
// Re-check under the lock: another tick, or an operator command, may
|
|
3609
3630
|
// have already moved this batch or cleared the fields itself.
|
|
3610
|
-
if (!b || b.status !==
|
|
3631
|
+
if (!b || b.status !== batch.status)
|
|
3611
3632
|
return { state: s, result: undefined };
|
|
3612
3633
|
const runClears = {};
|
|
3613
3634
|
if (mightClearRuns) {
|
|
3614
3635
|
const updatedRuns = b.member_runs.map((run) => {
|
|
3615
|
-
if (
|
|
3636
|
+
if (!(0, state_1.isLeftoverMemberRun)(b, run))
|
|
3616
3637
|
return run;
|
|
3617
3638
|
if (!goneOrReturned(run.worktree, run.pool_claimed))
|
|
3618
3639
|
return run;
|
|
@@ -3808,6 +3829,12 @@ function recordAnchorCloseFailed(deps, batch, outcome, shipped, now) {
|
|
|
3808
3829
|
* left on disk forever, since nothing else ever calls this for it.
|
|
3809
3830
|
*/
|
|
3810
3831
|
function teardownBatch(deps, batchId) {
|
|
3832
|
+
// #855: a kept batch's trees are the operator's — every teardown path reads
|
|
3833
|
+
// the same persisted decision. Only its live agents are stopped.
|
|
3834
|
+
if ((0, state_1.findBatch)(deps.store.load(), batchId)?.worktree_kept === true) {
|
|
3835
|
+
teardownParallelRuns(deps, batchId, null);
|
|
3836
|
+
return;
|
|
3837
|
+
}
|
|
3811
3838
|
// #677: the current member's worktree goes first — it is batch-owned
|
|
3812
3839
|
// state on the same teardown path as the shared tree, and a blocked or
|
|
3813
3840
|
// dissolved batch must not leak a member tree (its branch/fields are
|
|
@@ -3818,9 +3845,8 @@ function teardownBatch(deps, batchId) {
|
|
|
3818
3845
|
const terminal = (0, state_1.findBatch)(deps.store.load(), batchId);
|
|
3819
3846
|
const lastMemberLanded = terminal?.ranges.some((r) => r.issue === terminal.members[terminal.executing_member - 1]) ===
|
|
3820
3847
|
true;
|
|
3821
|
-
const
|
|
3822
|
-
|
|
3823
|
-
teardownParallelRuns(deps, batchId, serialTree);
|
|
3848
|
+
const serial = teardownMemberWorktree(deps, batchId, deps.now(), lastMemberLanded);
|
|
3849
|
+
teardownParallelRuns(deps, batchId, serial);
|
|
3824
3850
|
const state = deps.store.load();
|
|
3825
3851
|
const batch = (0, state_1.findBatch)(state, batchId);
|
|
3826
3852
|
if (!batch || batch.worktree === null)
|
|
@@ -3886,13 +3912,30 @@ function runTree(r) {
|
|
|
3886
3912
|
: { worktree: r.worktree, branch: r.branch, poolClaimed: r.poolClaimed };
|
|
3887
3913
|
}
|
|
3888
3914
|
/**
|
|
3889
|
-
*
|
|
3915
|
+
* Tear one parallel member run's tree down and record what actually happened
|
|
3916
|
+
* (#855) — the one teardown-and-record path for the evict, land and
|
|
3917
|
+
* `teardownParallelRuns` rails.
|
|
3918
|
+
*/
|
|
3919
|
+
function teardownRunTree(deps, batchId, run, landed, now) {
|
|
3920
|
+
const cleared = teardownMemberTree(deps, batchId, runTree(run), landed, run.issue);
|
|
3921
|
+
recordRunTeardown(deps, batchId, run.issue, cleared, now);
|
|
3922
|
+
}
|
|
3923
|
+
/** The one mapping from a member-tree teardown outcome to its run record (#855). */
|
|
3924
|
+
function teardownOutcomePatch(cleared, now) {
|
|
3925
|
+
return cleared ? { torn_down: true } : { teardown_failed_at: now.toISOString() };
|
|
3926
|
+
}
|
|
3927
|
+
/**
|
|
3928
|
+
* Record a run's teardown outcome. Written AFTER the teardown, so a crash in
|
|
3890
3929
|
* between re-runs the (idempotent) teardown from `teardownParallelRuns`
|
|
3891
|
-
* instead of leaking the tree.
|
|
3930
|
+
* instead of leaking the tree. #855: `torn_down` is set only when the
|
|
3931
|
+
* teardown verifiably landed; a failed one records `teardown_failed_at`
|
|
3932
|
+
* instead and leaves the tree on disk for the operator (never retried — see
|
|
3933
|
+
* `MemberRun.teardown_failed_at`). The caller's teardown already journaled
|
|
3934
|
+
* the failure, once.
|
|
3892
3935
|
*/
|
|
3893
|
-
function
|
|
3936
|
+
function recordRunTeardown(deps, batchId, issue, cleared, now) {
|
|
3894
3937
|
deps.store.withLock((s) => ({
|
|
3895
|
-
state: patchRun(s, batchId, issue,
|
|
3938
|
+
state: patchRun(s, batchId, issue, teardownOutcomePatch(cleared, now), now),
|
|
3896
3939
|
result: undefined,
|
|
3897
3940
|
}));
|
|
3898
3941
|
}
|
|
@@ -4028,6 +4071,7 @@ function spawnParallelMembers(deps, config, dispatch, batchId, now, result) {
|
|
|
4028
4071
|
status: 'running',
|
|
4029
4072
|
gate_inconclusive: null,
|
|
4030
4073
|
torn_down: false,
|
|
4074
|
+
teardown_failed_at: null,
|
|
4031
4075
|
}, now);
|
|
4032
4076
|
if ((0, scheduler_1.freeCapacity)(n, config) === 0)
|
|
4033
4077
|
return { state: n, result: true };
|
|
@@ -4055,7 +4099,7 @@ function spawnParallelMembers(deps, config, dispatch, batchId, now, result) {
|
|
|
4055
4099
|
reason: 'member-worktree-orphaned',
|
|
4056
4100
|
detail: `batch left executing while member ${next.index}'s tree ${member.worktree} was prepared — tearing it down`,
|
|
4057
4101
|
});
|
|
4058
|
-
teardownMemberTree(deps, batchId, runTree(member), false);
|
|
4102
|
+
teardownMemberTree(deps, batchId, runTree(member), false, next.issue);
|
|
4059
4103
|
}
|
|
4060
4104
|
}
|
|
4061
4105
|
}
|
|
@@ -4076,8 +4120,7 @@ function evictParallelMember(deps, config, batchId, run, failure, now, result) {
|
|
|
4076
4120
|
state: patchRun(s, batchId, run.issue, { status: 'evicted', gate_inconclusive: null }, now),
|
|
4077
4121
|
result: undefined,
|
|
4078
4122
|
}));
|
|
4079
|
-
|
|
4080
|
-
markTornDown(deps, batchId, run.issue, now);
|
|
4123
|
+
teardownRunTree(deps, batchId, run, false, now);
|
|
4081
4124
|
}
|
|
4082
4125
|
/**
|
|
4083
4126
|
* Reconcile one parallel member's slot — the parallel twin of
|
|
@@ -4313,8 +4356,7 @@ function advanceParallelBatch(deps, config, dispatch, batchId, now, result) {
|
|
|
4313
4356
|
// A gate-inconclusive member keeps its tree for `sched resume --batch`'s
|
|
4314
4357
|
// recheck; everyone else's is done serving.
|
|
4315
4358
|
if (run.gate_inconclusive === null) {
|
|
4316
|
-
|
|
4317
|
-
markTornDown(deps, batchId, run.issue, now);
|
|
4359
|
+
teardownRunTree(deps, batchId, run, true, now);
|
|
4318
4360
|
}
|
|
4319
4361
|
}
|
|
4320
4362
|
}
|
|
@@ -4362,7 +4404,7 @@ function finalizeParallelMembers(deps, config, dispatch, batchId, now, result) {
|
|
|
4362
4404
|
* run so the landing walk treats it as resolved, then continue the parallel
|
|
4363
4405
|
* rail from where it blocked.
|
|
4364
4406
|
*/
|
|
4365
|
-
function settleParallelResume(deps, batchId, memberIssue, now) {
|
|
4407
|
+
function settleParallelResume(deps, batchId, memberIssue, now, serialTeardown) {
|
|
4366
4408
|
deps.store.withLock((s) => {
|
|
4367
4409
|
const b = (0, state_1.findBatch)(s, batchId);
|
|
4368
4410
|
if (!b)
|
|
@@ -4371,8 +4413,10 @@ function settleParallelResume(deps, batchId, memberIssue, now) {
|
|
|
4371
4413
|
return {
|
|
4372
4414
|
state: patchRun(s, batchId, memberIssue, {
|
|
4373
4415
|
gate_inconclusive: null,
|
|
4374
|
-
// The serial rail's `teardownMemberWorktree`
|
|
4375
|
-
|
|
4416
|
+
// The serial rail's `teardownMemberWorktree` tore its tree down —
|
|
4417
|
+
// recorded as torn down only when that verifiably landed (#855).
|
|
4418
|
+
// No serial tree at all leaves the run live for `teardownBatch`.
|
|
4419
|
+
...(serialTeardown !== null ? teardownOutcomePatch(serialTeardown.cleared, now) : {}),
|
|
4376
4420
|
...(evicted ? { status: 'evicted' } : {}),
|
|
4377
4421
|
}, now),
|
|
4378
4422
|
result: undefined,
|
|
@@ -4414,11 +4458,19 @@ function reconcileParallelBatch(deps, config, dispatch, batchId, now, result, pa
|
|
|
4414
4458
|
* would be working a unit nothing tracks any more), `sched stop`/`abandon`
|
|
4415
4459
|
* make a batch terminal without a teardown, and a crash between a run's
|
|
4416
4460
|
* resolution and its teardown leaves `torn_down: false` behind. Teardown is
|
|
4417
|
-
* idempotent, so re-running it is always safe. `
|
|
4418
|
-
*
|
|
4419
|
-
* member of a blocked parallel batch)
|
|
4461
|
+
* idempotent, so re-running it is always safe. `serial` is what
|
|
4462
|
+
* `teardownMemberWorktree` just did with the serial-field tree (a
|
|
4463
|
+
* gate-inconclusive member of a blocked parallel batch) — its outcome is
|
|
4464
|
+
* recorded on the matching run without a second attempt.
|
|
4465
|
+
*
|
|
4466
|
+
* #855: default to KEEPING. A kept batch's (`worktree_kept`, #768's
|
|
4467
|
+
* members-closed reconcile) trees may hold unpushed operator work: its live
|
|
4468
|
+
* agents are still stopped (non-destructive), but no tree is removed or
|
|
4469
|
+
* pool-returned and `torn_down` stays false, so `sched status`'s
|
|
4470
|
+
* `kept-worktree` warning names them. A run whose earlier teardown failed
|
|
4471
|
+
* (`teardown_failed_at`) is not re-attempted either.
|
|
4420
4472
|
*/
|
|
4421
|
-
function teardownParallelRuns(deps, batchId,
|
|
4473
|
+
function teardownParallelRuns(deps, batchId, serial) {
|
|
4422
4474
|
const batch = (0, state_1.findBatch)(deps.store.load(), batchId);
|
|
4423
4475
|
if (!batch)
|
|
4424
4476
|
return;
|
|
@@ -4427,15 +4479,25 @@ function teardownParallelRuns(deps, batchId, alreadyTornDown) {
|
|
|
4427
4479
|
if (run.torn_down)
|
|
4428
4480
|
continue;
|
|
4429
4481
|
stopMemberAgent(deps, batchId, run.issue, now);
|
|
4482
|
+
if (run.teardown_failed_at !== null)
|
|
4483
|
+
continue;
|
|
4430
4484
|
const landed = run.status === 'landed';
|
|
4431
|
-
|
|
4432
|
-
|
|
4433
|
-
|
|
4434
|
-
|
|
4435
|
-
|
|
4436
|
-
|
|
4485
|
+
const status = landed ? 'landed' : 'evicted';
|
|
4486
|
+
// Only when it changes: a kept batch's runs pass through here every tick.
|
|
4487
|
+
if (run.status !== status || run.gate_inconclusive !== null) {
|
|
4488
|
+
deps.store.withLock((s) => ({
|
|
4489
|
+
state: patchRun(s, batchId, run.issue, { status, gate_inconclusive: null }, now),
|
|
4490
|
+
result: undefined,
|
|
4491
|
+
}));
|
|
4492
|
+
}
|
|
4493
|
+
if (batch.worktree_kept)
|
|
4494
|
+
continue;
|
|
4495
|
+
if (serial !== null && run.worktree === serial.path) {
|
|
4496
|
+
recordRunTeardown(deps, batchId, run.issue, serial.cleared, now);
|
|
4497
|
+
}
|
|
4498
|
+
else {
|
|
4499
|
+
teardownRunTree(deps, batchId, run, landed, now);
|
|
4437
4500
|
}
|
|
4438
|
-
markTornDown(deps, batchId, run.issue, now);
|
|
4439
4501
|
}
|
|
4440
4502
|
}
|
|
4441
4503
|
/** Kill a parallel member's live agent (pid-start checked) and release its slot. */
|
|
@@ -4643,6 +4705,13 @@ function runBatchTick(deps, config, dispatch) {
|
|
|
4643
4705
|
if (types_1.TERMINAL_BATCH_STATUSES.has(batch.status)) {
|
|
4644
4706
|
// #809: `sched stop`/`abandon` end a batch without a teardown — give
|
|
4645
4707
|
// back every member tree (and stop any agent still running in one).
|
|
4708
|
+
// #855: except a KEPT batch's trees (`worktree_kept`, the members-closed
|
|
4709
|
+
// reconcile's decision — the same one `worktree`/`member_worktree`
|
|
4710
|
+
// get) and a run whose teardown already failed: `teardownParallelRuns`
|
|
4711
|
+
// leaves both on disk with `torn_down: false`, and marks a run torn
|
|
4712
|
+
// down only after a teardown that verifiably landed. Such a batch
|
|
4713
|
+
// re-enters here every tick by design — cheap (a state read per run),
|
|
4714
|
+
// and it stops any agent that is somehow still running in a kept tree.
|
|
4646
4715
|
if (batch.member_runs.some((r) => !r.torn_down)) {
|
|
4647
4716
|
try {
|
|
4648
4717
|
teardownParallelRuns(deps, batch.id, null);
|