@ai-dossier/sched 0.54.0 → 0.55.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +18 -3
- package/dist/batch-dispatch.d.ts +43 -0
- package/dist/batch-dispatch.d.ts.map +1 -1
- package/dist/batch-dispatch.js +193 -49
- package/dist/batch-dispatch.js.map +1 -1
- package/dist/state.d.ts +11 -1
- package/dist/state.d.ts.map +1 -1
- package/dist/state.js +33 -0
- package/dist/state.js.map +1 -1
- package/dist/status.d.ts +21 -13
- package/dist/status.d.ts.map +1 -1
- package/dist/status.js +47 -15
- package/dist/status.js.map +1 -1
- package/dist/types.d.ts +35 -5
- package/dist/types.d.ts.map +1 -1
- package/dist/types.js +5 -1
- package/dist/types.js.map +1 -1
- package/package.json +1 -1
package/README.md
CHANGED
|
@@ -807,6 +807,11 @@ failure rails: executing → dissolving (a member self-reports blocked)
|
|
|
807
807
|
an eviction group, or a member `deps` on another member; a batch already past `ready`
|
|
808
808
|
with no recorded mode (claimed by a pre-1.23.0 engine) stays serial across the upgrade.
|
|
809
809
|
Dissolve, `sched stop --batch` and `abandon` stop/release every member slot.
|
|
810
|
+
A run's `torn_down` is set only after its tree teardown verifiably landed (path gone
|
|
811
|
+
and unlisted, or the pool reports it returned); a failed one records
|
|
812
|
+
`teardown_failed_at`, journals `teardown-failed` once and is never retried — the tree
|
|
813
|
+
stays for the operator and `sched status` names it (`kept-worktree`) (#855). A kept batch's (`worktree_kept`) runs are never torn
|
|
814
|
+
down by the scheduler.
|
|
810
815
|
- **Serial mode: members run `member-cycle` one fresh agent at a time — each in its OWN
|
|
811
816
|
worktree** on its OWN branch `batch/<id>-m<n>-<issue>` (#677, RFC-0001 §J.3), cut off
|
|
812
817
|
the integration branch, warmed, and pushed before the agent spawns; the agent never
|
|
@@ -1426,7 +1431,7 @@ after-the-fact recovery, not a missing-data bug.
|
|
|
1426
1431
|
queue data.
|
|
1427
1432
|
- **Schema**: state/config files from #460 (schema 1.0.0), #464 (1.1.0), #468 (1.2.0),
|
|
1428
1433
|
#472 (1.3.0), #500 (1.4.0), #505 (1.5.0), #504 (1.6.0), #523 (1.7.0) and #524 (1.8.0)
|
|
1429
|
-
load and migrate to the current schema (1.
|
|
1434
|
+
load and migrate to the current schema (1.28.0 — 1.24.0 was #810: no backfill,
|
|
1430
1435
|
`kind`/`branch` optional, absent = an `evicted` record with no branch) automatically
|
|
1431
1436
|
(slot `branch`/`last_head`/`pid_start`, slot `role` (inferred from the
|
|
1432
1437
|
unit's queue entry, with the persisted `phase` as a fallback — #500), entry
|
|
@@ -1449,6 +1454,10 @@ after-the-fact recovery, not a missing-data bug.
|
|
|
1449
1454
|
respawn counter); `null` is backfilled on load.
|
|
1450
1455
|
Schema 1.27.0 (#822): `BatchEntry` gains `reprompted_members`
|
|
1451
1456
|
(`{ issue, milestone_at, reprompted_at }` records; `[]` backfilled).
|
|
1457
|
+
Schema 1.28.0 (#855): `BatchEntry` gains `worktree_kept` (the members-closed keep
|
|
1458
|
+
decision, persisted; `false` backfilled) and `MemberRun` gains `teardown_failed_at`
|
|
1459
|
+
(`null` backfilled). A 1.27.0 `done` batch still carrying a live member run loads as
|
|
1460
|
+
kept — the old engine never recorded the decision, so the default is to keep.
|
|
1452
1461
|
- **`max_slots`** bounds live units (`assigned | running | recovering`); dependency
|
|
1453
1462
|
edges gate readiness — an issue with an unmerged dependency, and a batch behind an
|
|
1454
1463
|
unmerged batch, are never runnable.
|
|
@@ -1486,7 +1495,11 @@ to skip the check entirely):
|
|
|
1486
1495
|
- **`kept-worktree`** (#791) — a `done` batch whose `worktree` or `member_worktree` is
|
|
1487
1496
|
still set in the ledger: the `members-closed` stale-blocked reconcile below
|
|
1488
1497
|
intentionally leaves it in place (it may hold unpushed operator repair work), and
|
|
1489
|
-
nothing else surfaced it until now.
|
|
1498
|
+
nothing else surfaced it until now. Also (#834/#855) each parallel `member_runs[]`
|
|
1499
|
+
tree left on disk: every not-torn-down run of a `done` batch (a kept batch's, or one
|
|
1500
|
+
a crash stranded), and a run of any ended batch whose teardown FAILED
|
|
1501
|
+
(`teardown_failed_at` — the message says so; the cause is in the `teardown-failed`
|
|
1502
|
+
journal line, and the scheduler never retries it). For each kept path, `sched status` reports:
|
|
1490
1503
|
whether it still exists on disk; whether it is pool-claimed (the claim is held
|
|
1491
1504
|
indefinitely until returned); and, via two local, read-only `git` probes scoped to
|
|
1492
1505
|
that worktree's own `HEAD` (`git --no-optional-locks status --porcelain`, `git log
|
|
@@ -1544,7 +1557,9 @@ predicate in `anchor-close.ts`) over `blocked`/`done` batches touched within the
|
|
|
1544
1557
|
for #686's stale-blocked reconcile (`members-closed`), which survives the mandated
|
|
1545
1558
|
rebase-merge that makes the ancestry probe structurally dead. It finishes the terminal
|
|
1546
1559
|
rail inline but never tears the blocked worktree down (it may hold unpushed work);
|
|
1547
|
-
|
|
1560
|
+
since #855 the decision is persisted as `worktree_kept`, so the terminal-batch safety
|
|
1561
|
+
net leaves the batch's parallel `member_runs[]` trees in place too (`torn_down` stays
|
|
1562
|
+
false). The journal line says the worktree was kept, and `sched status` then raises a
|
|
1548
1563
|
`kept-worktree` warning for it on every run until the ledger field clears — see
|
|
1549
1564
|
above and `reconcileKeptWorktrees` (#791).
|
|
1550
1565
|
- **PR-detection evidence (#789).** When `batch.pr` was never recorded at all — a PR
|
package/dist/batch-dispatch.d.ts
CHANGED
|
@@ -300,6 +300,49 @@ export declare function undispatchedSurvivingMembers(state: SchedState, batch: B
|
|
|
300
300
|
* #632 lesson: no journal inside a loop that doesn't terminate the unit).
|
|
301
301
|
*/
|
|
302
302
|
export declare function reconcileStaleBlockedBatches(deps: BatchDispatchDeps, now: Date, result: BatchTickResult, anchorCtx: AnchorTickContext | undefined): void;
|
|
303
|
+
/**
|
|
304
|
+
* #791: once a `done` batch's kept worktree (left in place by the
|
|
305
|
+
* `members-closed` branch above) is DEFINITIVELY gone — the path no longer
|
|
306
|
+
* exists on disk, or the pool itself now reports it back as a warm spare —
|
|
307
|
+
* clear the ledger's `worktree`/`pool_claimed` (and, separately,
|
|
308
|
+
* `member_worktree`/`member_pool_claimed`, and — #834 — each live
|
|
309
|
+
* `member_runs[]` entry's `torn_down`) fields so `sched status`'s
|
|
310
|
+
* `kept-worktree` warning does not persist after the operator's cleanup
|
|
311
|
+
* already happened. Ledger-only, by construction: this function reads
|
|
312
|
+
* `fsExists` and `worktree-pool status --json` (both read-only) and never
|
|
313
|
+
* builds a `git worktree remove` / `worktree-pool return` argv — the actual
|
|
314
|
+
* removal stays entirely the operator's call, exactly like the warning
|
|
315
|
+
* itself. Bounded the same way the warning's own candidate scan is: only
|
|
316
|
+
* `done` batches carrying a kept worktree are considered, which in practice
|
|
317
|
+
* is a small set.
|
|
318
|
+
*
|
|
319
|
+
* #834: `MemberRun.worktree` is a required `string` (unlike
|
|
320
|
+
* `BatchEntry.worktree: string | null`) — there is no null to clear it to,
|
|
321
|
+
* so "cleared" for a member run means `torn_down: true`, the same flag
|
|
322
|
+
* `recordRunTeardown` sets after a teardown that verifiably landed. Set
|
|
323
|
+
* here it means the same thing evidentially (nothing is holding this path
|
|
324
|
+
* any more), just reached by definitive evidence instead of by running
|
|
325
|
+
* `teardownParallelRuns`.
|
|
326
|
+
*
|
|
327
|
+
* #834/#855: a terminal batch's live `member_runs[]` entries are ALSO
|
|
328
|
+
* reached by the terminal-batch arm in `runBatchTick` (search
|
|
329
|
+
* `TERMINAL_BATCH_STATUSES.has(batch.status)` above the call site below),
|
|
330
|
+
* added by #809 for `sched stop`/`abandon`, which runs earlier in the same
|
|
331
|
+
* tick than this function. Since #855 that arm respects the keep decision
|
|
332
|
+
* (`worktree_kept`) and marks a run torn down only after a teardown that
|
|
333
|
+
* verifiably landed, so on a KEPT batch, or a run whose teardown failed
|
|
334
|
+
* (`teardown_failed_at`), `torn_down` stays false and this function is the
|
|
335
|
+
* only thing that ever clears it — once the operator has removed or
|
|
336
|
+
* returned the tree. On a non-kept batch the arm usually tears the tree
|
|
337
|
+
* down (and records it) first.
|
|
338
|
+
*
|
|
339
|
+
* Exported (not part of the package's public `index.ts` surface — same
|
|
340
|
+
* test-only convention as `reconcileStaleBlockedBatches`) so `member_runs[]`
|
|
341
|
+
* tests can call it directly, isolated from the terminal-batch safety net
|
|
342
|
+
* described above, which reaches a non-kept batch's `member_runs[]` first
|
|
343
|
+
* when both run inside one `runBatchTick` call.
|
|
344
|
+
*/
|
|
345
|
+
export declare function reconcileKeptWorktrees(deps: BatchDispatchDeps, now: Date): void;
|
|
303
346
|
/**
|
|
304
347
|
* #768: what one tick's anchor work reads GitHub and git through — built once
|
|
305
348
|
* per tick so the stale-blocked reconcile and the anchor close never read the
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"batch-dispatch.d.ts","sourceRoot":"","sources":["../src/batch-dispatch.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA8DG;AAUH,OAAO,EAIL,KAAK,YAAY,EAGjB,KAAK,gBAAgB,EAGtB,MAAM,gBAAgB,CAAC;AAUxB,OAAO,EAaL,KAAK,gBAAgB,EAGrB,KAAK,SAAS,EAEf,MAAM,YAAY,CAAC;AAEpB,OAAO,EAGL,KAAK,WAAW,EAcjB,MAAM,eAAe,CAAC;AACvB,OAAO,EAAE,KAAK,OAAO,EAAa,MAAM,WAAW,CAAC;AACpD,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,WAAW,CAAC;AAC5C,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,WAAW,CAAC;AAExC,OAAO,EACL,KAAK,iBAAiB,EAYtB,KAAK,WAAW,EACjB,MAAM,YAAY,CAAC;AAGpB,OAAO,
|
|
1
|
+
{"version":3,"file":"batch-dispatch.d.ts","sourceRoot":"","sources":["../src/batch-dispatch.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA8DG;AAUH,OAAO,EAIL,KAAK,YAAY,EAGjB,KAAK,gBAAgB,EAGtB,MAAM,gBAAgB,CAAC;AAUxB,OAAO,EAaL,KAAK,gBAAgB,EAGrB,KAAK,SAAS,EAEf,MAAM,YAAY,CAAC;AAEpB,OAAO,EAGL,KAAK,WAAW,EAcjB,MAAM,eAAe,CAAC;AACvB,OAAO,EAAE,KAAK,OAAO,EAAa,MAAM,WAAW,CAAC;AACpD,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,WAAW,CAAC;AAC5C,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,WAAW,CAAC;AAExC,OAAO,EACL,KAAK,iBAAiB,EAYtB,KAAK,WAAW,EACjB,MAAM,YAAY,CAAC;AAGpB,OAAO,EAyBL,kBAAkB,EAKnB,MAAM,SAAS,CAAC;AACjB,OAAO,EACL,KAAK,QAAQ,EAMd,MAAM,YAAY,CAAC;AACpB,OAAO,KAAK,EAGV,UAAU,EAEV,oBAAoB,EAKpB,kBAAkB,EAClB,cAAc,EAId,WAAW,EACX,UAAU,EAEX,MAAM,SAAS,CAAC;AAcjB,OAAO,EAAE,kBAAkB,EAAE,CAAC;AAK9B,YAAY,EAAE,UAAU,EAAE,MAAM,SAAS,CAAC;AAE1C,8DAA8D;AAC9D,MAAM,WAAW,iBAAiB;IAChC,KAAK,EAAE,UAAU,CAAC;IAClB,OAAO,EAAE,OAAO,CAAC;IACjB,WAAW,EAAE,WAAW,CAAC;IACzB,SAAS,EAAE,SAAS,CAAC;IACrB,GAAG,EAAE,MAAM,IAAI,CAAC;IAChB,uFAAuF;IACvF,OAAO,EAAE,MAAM,CAAC;IAChB,0FAA0F;IAC1F,IAAI,EAAE,MAAM,CAAC;IACb;;;;;;OAMG;IACH,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB;;;;;OAKG;IACH,QAAQ,EAAE,CAAC,QAAQ,EAAE,MAAM,EAAE,GAAG,CAAC,EAAE,iBAAiB,KAAK,WAAW,CAAC;IACrE;;;;;;;;;;;;;;;;;;;OAmBG;IACH,aAAa,CAAC,EAAE,CAAC,QAAQ,EAAE,MAAM,EAAE,YAAY,EAAE,MAAM,KAAK,oBAAoB,CAAC;IACjF,QAAQ,CAAC,EAAE,QAAQ,CAAC;IACpB;;;OAGG;IACH,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB;;;;;;OAMG;IACH,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB;AAED,8FAA8F;AAC9F,MAAM,WAAW,eAAe;IAC9B,OAAO,EAAE,MAAM,EAAE,CAAC;IAClB,SAAS,EAAE,MAAM,EAAE,CAAC;IACpB,MAAM,EAAE,MAAM,EAAE,CAAC;IACjB,aAAa,EAAE,MAAM,EAAE,CAAC;IACxB,MAAM,EAAE,MAAM,EAAE,CAAC;IACjB,uFAAuF;IACvF,qBAAqB,EAAE,MAAM,EAAE,CAAC;IAChC,8FAA8F;IAC9F,OAAO,EAAE,MAAM,EAAE,CAAC;CACnB;AAED;;;;;;GAMG;AACH,MAAM,WAAW,aAAa;IAC5B,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACjC;;;;OAIG;IACH,IAAI,CAAC,EAAE,cAAc,CAAC;CACvB;AAylBD;;;;;;;GAOG;AACH,wBAAgB,eAAe,CAAC,OAAO,EAAE,MAAM,EAAE,WAAW,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,MAAM,CAE3F;AAs+CD;;;;;;;;;;;;;;;;;;;;;;;;GAwBG;AACH,wBAAgB,sBAAsB,CACpC,IAAI,EAAE,iBAAiB,EACvB,MAAM,EAAE,WAAW,EACnB,QAAQ,EAAE,gBAAgB,EAC1B,OAAO,EAAE,MAAM,EACf,KAAK,EAAE,UAAU,EACjB,WAAW,EAAE,MAAM,EACnB,OAAO,EAAE,aAAa,EACtB,GAAG,EAAE,IAAI,EACT,MAAM,EAAE,eAAe,GACtB,IAAI,CAgCN;AA4TD;;;;;;;;;;;;;;;;;;;;;;;;;;;GA2BG;AACH,wBAAgB,iBAAiB,CAC/B,IAAI,EAAE,iBAAiB,EACvB,MAAM,EAAE,WAAW,EACnB,QAAQ,EAAE,gBAAgB,EAC1B,OAAO,EAAE,MAAM,EACf,GAAG,EAAE,IAAI,GACR;IACD,OAAO,EAAE,eAAe,GAAG,SAAS,GAAG,WAAW,GAAG,SAAS,CAAC;IAC/D,UAAU,EAAE,MAAM,CAAC;IACnB;;;;;OAKG;IACH,SAAS,CAAC,EAAE,cAAc,GAAG,qBAAqB,CAAC;IACnD,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB,CA2KA;AAuxBD;;;;;;;;GAQG;AACH,eAAO,MAAM,wBAAwB,IAAI,CAAC;AAkf1C;;;;;;GAMG;AACH,wBAAgB,4BAA4B,CAAC,KAAK,EAAE,UAAU,EAAE,KAAK,EAAE,UAAU,GAAG,MAAM,EAAE,CAuB3F;AA6HD;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAwCG;AACH,wBAAgB,4BAA4B,CAC1C,IAAI,EAAE,iBAAiB,EACvB,GAAG,EAAE,IAAI,EACT,MAAM,EAAE,eAAe,EACvB,SAAS,EAAE,iBAAiB,GAAG,SAAS,GACvC,IAAI,CAmMN;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAyCG;AACH,wBAAgB,sBAAsB,CAAC,IAAI,EAAE,iBAAiB,EAAE,GAAG,EAAE,IAAI,GAAG,IAAI,CAuG/E;AAED;;;;;;GAMG;AACH,UAAU,iBAAiB;IACzB,IAAI,EAAE,MAAM,CAAC;IACb,gDAAgD;IAChD,IAAI,EAAE,gBAAgB,CAAC;IACvB,uEAAuE;IACvE,SAAS,EAAE,gBAAgB,CAAC;IAC5B,YAAY,EAAE,YAAY,CAAC;CAC5B;AAoRD;;;;;;;;;;;GAWG;AACH,oCAAoC;AACpC,wBAAgB,qBAAqB,CACnC,KAAK,EAAE,UAAU,EACjB,KAAK,EAAE,UAAU,EACjB,MAAM,EAAE,WAAW,GAClB,kBAAkB,CAQpB;AAi3BD;;;;;;;;;;;;;GAaG;AACH,wBAAgB,YAAY,CAC1B,IAAI,EAAE,iBAAiB,EACvB,MAAM,EAAE,WAAW,EACnB,QAAQ,EAAE,gBAAgB,GACzB,eAAe,CAuSjB"}
|
package/dist/batch-dispatch.js
CHANGED
|
@@ -102,6 +102,7 @@ exports.evictMemberAndContinue = evictMemberAndContinue;
|
|
|
102
102
|
exports.resumeBlockedGate = resumeBlockedGate;
|
|
103
103
|
exports.undispatchedSurvivingMembers = undispatchedSurvivingMembers;
|
|
104
104
|
exports.reconcileStaleBlockedBatches = reconcileStaleBlockedBatches;
|
|
105
|
+
exports.reconcileKeptWorktrees = reconcileKeptWorktrees;
|
|
105
106
|
exports.memberDispatchModeFor = memberDispatchModeFor;
|
|
106
107
|
exports.runBatchTick = runBatchTick;
|
|
107
108
|
const fs = __importStar(require("node:fs"));
|
|
@@ -819,7 +820,7 @@ landed) {
|
|
|
819
820
|
const state = deps.store.load();
|
|
820
821
|
const batch = (0, state_1.findBatch)(state, batchId);
|
|
821
822
|
if (!batch || batch.member_worktree === null)
|
|
822
|
-
return;
|
|
823
|
+
return null;
|
|
823
824
|
const worktree = batch.member_worktree;
|
|
824
825
|
const branch = batch.member_branch;
|
|
825
826
|
// #677 review (defense in depth, matching `branchMergedIntoBase`'s bar):
|
|
@@ -830,7 +831,7 @@ landed) {
|
|
|
830
831
|
cleanup: 'failed-invalid-branch-name',
|
|
831
832
|
detail: branch,
|
|
832
833
|
});
|
|
833
|
-
return;
|
|
834
|
+
return { path: worktree, cleared: false };
|
|
834
835
|
}
|
|
835
836
|
// Nulls first — clear the fields BEFORE the exec so a crash mid-teardown
|
|
836
837
|
// doesn't retry into a half-cleaned tree (the re-derived path is
|
|
@@ -844,23 +845,39 @@ landed) {
|
|
|
844
845
|
result: undefined,
|
|
845
846
|
};
|
|
846
847
|
});
|
|
847
|
-
teardownMemberTree(deps, batchId, { worktree, branch, poolClaimed: batch.member_pool_claimed === true }, landed);
|
|
848
|
+
const cleared = teardownMemberTree(deps, batchId, { worktree, branch, poolClaimed: batch.member_pool_claimed === true }, landed, batch.members[batch.executing_member - 1]);
|
|
849
|
+
return { path: worktree, cleared };
|
|
848
850
|
}
|
|
851
|
+
/** #855: the `teardown-failed` tail for a member tree that stays on disk. */
|
|
852
|
+
const TREE_LEFT_FOR_OPERATOR = 'tree left on disk for the operator and NOT retried (it may hold work); inspect it, then remove it (`git worktree remove`, or `worktree-pool return` if pool-claimed) — `sched status` names it';
|
|
849
853
|
/**
|
|
850
854
|
* The exec half of a member teardown, shared by the serial rail
|
|
851
855
|
* (`teardownMemberWorktree`, which first clears the batch's member fields) and
|
|
852
856
|
* the parallel rail (#809: `evictParallelMember`, `advanceParallelBatch`,
|
|
853
857
|
* `teardownParallelRuns`). See
|
|
854
858
|
* `teardownMemberWorktree` for the landed/evicted branch policy.
|
|
859
|
+
*
|
|
860
|
+
* Returns whether the TREE is verifiably gone (#855) — `runTeardown` reports
|
|
861
|
+
* `done` only after the path is gone and unlisted, or the pool's self-check
|
|
862
|
+
* reports the entry returned. A failed branch delete after that does not
|
|
863
|
+
* un-clear the tree. Journals exactly one line per call; a failed tree is
|
|
864
|
+
* never re-attempted by any caller, so the line says what the operator does.
|
|
855
865
|
*/
|
|
856
|
-
function teardownMemberTree(deps, batchId, tree, landed
|
|
866
|
+
function teardownMemberTree(deps, batchId, tree, landed,
|
|
867
|
+
/** The member whose tree this is — named in the journal line. */
|
|
868
|
+
issue) {
|
|
857
869
|
const { worktree, branch } = tree;
|
|
870
|
+
const context = {
|
|
871
|
+
...(issue !== undefined ? { issue } : {}),
|
|
872
|
+
worktree,
|
|
873
|
+
};
|
|
858
874
|
if (branch !== null && !attribution_1.SAFE_REF_RE.test(branch)) {
|
|
859
875
|
journalEvent(deps, 'teardown-failed', unit(batchId), {
|
|
876
|
+
...context,
|
|
860
877
|
cleanup: 'failed-invalid-branch-name',
|
|
861
|
-
detail: branch
|
|
878
|
+
detail: `${branch} — ${TREE_LEFT_FOR_OPERATOR}`,
|
|
862
879
|
});
|
|
863
|
-
return;
|
|
880
|
+
return false;
|
|
864
881
|
}
|
|
865
882
|
// Route through `runTeardown` for the same guarantees `teardownBatch`
|
|
866
883
|
// gets: pool returns are verify-first idempotent and refuse to claim
|
|
@@ -883,11 +900,12 @@ function teardownMemberTree(deps, batchId, tree, landed) {
|
|
|
883
900
|
}
|
|
884
901
|
const failed = !treeCleared || branchCleanup.startsWith('failed');
|
|
885
902
|
journalEvent(deps, failed ? 'teardown-failed' : 'member-worktree-torn-down', unit(batchId), {
|
|
903
|
+
...context,
|
|
886
904
|
cleanup: t.cleanup,
|
|
887
905
|
branch_cleanup: branchCleanup,
|
|
888
|
-
detail: t.detail
|
|
889
|
-
worktree,
|
|
906
|
+
detail: treeCleared ? t.detail : `${t.detail} — ${TREE_LEFT_FOR_OPERATOR}`,
|
|
890
907
|
});
|
|
908
|
+
return treeCleared;
|
|
891
909
|
}
|
|
892
910
|
/**
|
|
893
911
|
* The shared "block the batch for an operator" tail (#677 review — the same
|
|
@@ -1716,13 +1734,15 @@ function journalAdvanceSkipped(deps, batchId, memberIssue, currentMember, claim)
|
|
|
1716
1734
|
detail: `another resolution already advanced past member ${currentMember} (batch is now executing_member=${claim.observed ?? 'gone'}, status=${claim.status ?? 'gone'}) — no second member-advanced and no second continuation`,
|
|
1717
1735
|
});
|
|
1718
1736
|
}
|
|
1719
|
-
function advanceMemberOrValidate(deps, config, dispatch, batchId, memberCount, currentMember, memberIssue, now, result
|
|
1737
|
+
function advanceMemberOrValidate(deps, config, dispatch, batchId, memberCount, currentMember, memberIssue, now, result,
|
|
1738
|
+
/** What the caller's `teardownMemberWorktree` did with the member's tree (#855). */
|
|
1739
|
+
serialTeardown) {
|
|
1720
1740
|
// #809: a parallel batch reaches this serial rail only through `sched resume
|
|
1721
1741
|
// --batch` resolving its gate-inconclusive member — hand back to the
|
|
1722
1742
|
// parallel rail instead of advancing a serial member pointer.
|
|
1723
1743
|
const current = (0, state_1.findBatch)(deps.store.load(), batchId);
|
|
1724
1744
|
if (current && isParallelBatch(current)) {
|
|
1725
|
-
settleParallelResume(deps, batchId, memberIssue, now);
|
|
1745
|
+
settleParallelResume(deps, batchId, memberIssue, now, serialTeardown);
|
|
1726
1746
|
advanceParallelBatch(deps, config, dispatch, batchId, now, result);
|
|
1727
1747
|
return;
|
|
1728
1748
|
}
|
|
@@ -1788,8 +1808,8 @@ function evictMemberAndContinue(deps, config, dispatch, batchId, batch, memberIs
|
|
|
1788
1808
|
// branch — but its worktree/branch must go, or the next tick's prep for
|
|
1789
1809
|
// the NEXT member collides with a stale tree. `teardownBatch` already
|
|
1790
1810
|
// handled it on the dissolved path (the fields are cleared there).
|
|
1791
|
-
teardownMemberWorktree(deps, batchId, now, false);
|
|
1792
|
-
advanceMemberOrValidate(deps, config, dispatch, batchId, batch.members.length, batch.executing_member, memberIssue, now, result);
|
|
1811
|
+
const serialTeardown = teardownMemberWorktree(deps, batchId, now, false);
|
|
1812
|
+
advanceMemberOrValidate(deps, config, dispatch, batchId, batch.members.length, batch.executing_member, memberIssue, now, result, serialTeardown);
|
|
1793
1813
|
}
|
|
1794
1814
|
/**
|
|
1795
1815
|
* The incremental gate (#523 AC2, revised #583): typecheck + focused tests
|
|
@@ -2026,8 +2046,8 @@ function completeMemberGate(deps, config, dispatch, batchId, batch, memberIssue,
|
|
|
2026
2046
|
// #677: the member's tree is done serving — landed, ranges recomputed.
|
|
2027
2047
|
// Teardown before the advance so the next member's prep cannot collide
|
|
2028
2048
|
// with this tree's cleanup.
|
|
2029
|
-
teardownMemberWorktree(deps, batchId, now, true);
|
|
2030
|
-
advanceMemberOrValidate(deps, config, dispatch, batchId, batch.members.length, batch.executing_member, memberIssue, now, result);
|
|
2049
|
+
const serialTeardown = teardownMemberWorktree(deps, batchId, now, true);
|
|
2050
|
+
advanceMemberOrValidate(deps, config, dispatch, batchId, batch.members.length, batch.executing_member, memberIssue, now, result, serialTeardown);
|
|
2031
2051
|
}
|
|
2032
2052
|
/**
|
|
2033
2053
|
* `sched resume --batch <id>` (#583 AC4): an operator-triggered, synchronous
|
|
@@ -2919,7 +2939,7 @@ function reconcileTailSlot(deps, config, batchId, slot, now, result) {
|
|
|
2919
2939
|
result.failed.push(unit(batchId));
|
|
2920
2940
|
}
|
|
2921
2941
|
}
|
|
2922
|
-
function reconcileReportSlot(deps,
|
|
2942
|
+
function reconcileReportSlot(deps, _config, batchId, slot, now, result) {
|
|
2923
2943
|
const state = deps.store.load();
|
|
2924
2944
|
const batch = (0, state_1.findBatch)(state, batchId);
|
|
2925
2945
|
if (!batch || batch.anchor === null)
|
|
@@ -3429,6 +3449,10 @@ function reconcileStaleBlockedBatches(deps, now, result, anchorCtx) {
|
|
|
3429
3449
|
// clear into this same transition instead of a separate lock write.
|
|
3430
3450
|
...state_1.CLEARED_PR_DETECT_AMBIGUOUS_FIELDS,
|
|
3431
3451
|
...(prWasRecorded ? { pr: detectedPr } : {}),
|
|
3452
|
+
// #855: persist the keep decision in the same transition, so every
|
|
3453
|
+
// later teardown path — the terminal-batch safety net included —
|
|
3454
|
+
// reads it for `member_runs[]` too.
|
|
3455
|
+
...(keepWorktree ? { worktree_kept: true } : {}),
|
|
3432
3456
|
}, now);
|
|
3433
3457
|
n = (0, state_1.transitionBatch)(n, batch.id, 'deployed', {}, now);
|
|
3434
3458
|
unwalkable = [];
|
|
@@ -3505,7 +3529,8 @@ function reconcileStaleBlockedBatches(deps, now, result, anchorCtx) {
|
|
|
3505
3529
|
* `members-closed` branch above) is DEFINITIVELY gone — the path no longer
|
|
3506
3530
|
* exists on disk, or the pool itself now reports it back as a warm spare —
|
|
3507
3531
|
* clear the ledger's `worktree`/`pool_claimed` (and, separately,
|
|
3508
|
-
* `member_worktree`/`member_pool_claimed
|
|
3532
|
+
* `member_worktree`/`member_pool_claimed`, and — #834 — each live
|
|
3533
|
+
* `member_runs[]` entry's `torn_down`) fields so `sched status`'s
|
|
3509
3534
|
* `kept-worktree` warning does not persist after the operator's cleanup
|
|
3510
3535
|
* already happened. Ledger-only, by construction: this function reads
|
|
3511
3536
|
* `fsExists` and `worktree-pool status --json` (both read-only) and never
|
|
@@ -3514,15 +3539,46 @@ function reconcileStaleBlockedBatches(deps, now, result, anchorCtx) {
|
|
|
3514
3539
|
* itself. Bounded the same way the warning's own candidate scan is: only
|
|
3515
3540
|
* `done` batches carrying a kept worktree are considered, which in practice
|
|
3516
3541
|
* is a small set.
|
|
3542
|
+
*
|
|
3543
|
+
* #834: `MemberRun.worktree` is a required `string` (unlike
|
|
3544
|
+
* `BatchEntry.worktree: string | null`) — there is no null to clear it to,
|
|
3545
|
+
* so "cleared" for a member run means `torn_down: true`, the same flag
|
|
3546
|
+
* `recordRunTeardown` sets after a teardown that verifiably landed. Set
|
|
3547
|
+
* here it means the same thing evidentially (nothing is holding this path
|
|
3548
|
+
* any more), just reached by definitive evidence instead of by running
|
|
3549
|
+
* `teardownParallelRuns`.
|
|
3550
|
+
*
|
|
3551
|
+
* #834/#855: a terminal batch's live `member_runs[]` entries are ALSO
|
|
3552
|
+
* reached by the terminal-batch arm in `runBatchTick` (search
|
|
3553
|
+
* `TERMINAL_BATCH_STATUSES.has(batch.status)` above the call site below),
|
|
3554
|
+
* added by #809 for `sched stop`/`abandon`, which runs earlier in the same
|
|
3555
|
+
* tick than this function. Since #855 that arm respects the keep decision
|
|
3556
|
+
* (`worktree_kept`) and marks a run torn down only after a teardown that
|
|
3557
|
+
* verifiably landed, so on a KEPT batch, or a run whose teardown failed
|
|
3558
|
+
* (`teardown_failed_at`), `torn_down` stays false and this function is the
|
|
3559
|
+
* only thing that ever clears it — once the operator has removed or
|
|
3560
|
+
* returned the tree. On a non-kept batch the arm usually tears the tree
|
|
3561
|
+
* down (and records it) first.
|
|
3562
|
+
*
|
|
3563
|
+
* Exported (not part of the package's public `index.ts` surface — same
|
|
3564
|
+
* test-only convention as `reconcileStaleBlockedBatches`) so `member_runs[]`
|
|
3565
|
+
* tests can call it directly, isolated from the terminal-batch safety net
|
|
3566
|
+
* described above, which reaches a non-kept batch's `member_runs[]` first
|
|
3567
|
+
* when both run inside one `runBatchTick` call.
|
|
3517
3568
|
*/
|
|
3518
3569
|
function reconcileKeptWorktrees(deps, now) {
|
|
3519
3570
|
const state = deps.store.load();
|
|
3520
3571
|
const fsExists = deps.fsExists ?? ((p) => fs.existsSync(p));
|
|
3521
|
-
|
|
3572
|
+
// #855: a `stopped`/`dissolved` batch's run whose teardown failed is a
|
|
3573
|
+
// leftover too (`isLeftoverMemberRun`) — its ledger clears on the same
|
|
3574
|
+
// evidence once the operator has dealt with it.
|
|
3575
|
+
const candidates = state.batches.filter((b) => (b.status === 'done' && (b.worktree !== null || b.member_worktree !== null)) ||
|
|
3576
|
+
b.member_runs.some((r) => (0, state_1.isLeftoverMemberRun)(b, r)));
|
|
3522
3577
|
if (candidates.length === 0)
|
|
3523
3578
|
return;
|
|
3524
3579
|
const needsPoolCheck = candidates.some((b) => (b.worktree !== null && b.pool_claimed) ||
|
|
3525
|
-
(b.member_worktree !== null && b.member_pool_claimed)
|
|
3580
|
+
(b.member_worktree !== null && b.member_pool_claimed) ||
|
|
3581
|
+
b.member_runs.some((r) => r.pool_claimed && (0, state_1.isLeftoverMemberRun)(b, r)));
|
|
3526
3582
|
// One pool query per TICK, not per candidate — `worktree-pool status` is a
|
|
3527
3583
|
// real subprocess (`npx ...`), and this reconcile is meant to be cheap.
|
|
3528
3584
|
const poolStatusJson = needsPoolCheck
|
|
@@ -3539,27 +3595,67 @@ function reconcileKeptWorktrees(deps, now) {
|
|
|
3539
3595
|
for (const batch of candidates) {
|
|
3540
3596
|
const clears = {};
|
|
3541
3597
|
const clearedFields = [];
|
|
3542
|
-
|
|
3598
|
+
const done = batch.status === 'done';
|
|
3599
|
+
if (done && batch.worktree !== null && goneOrReturned(batch.worktree, batch.pool_claimed)) {
|
|
3543
3600
|
clears.worktree = null;
|
|
3544
3601
|
clears.pool_claimed = false;
|
|
3545
3602
|
clearedFields.push('worktree', 'pool_claimed');
|
|
3546
3603
|
}
|
|
3547
|
-
if (
|
|
3604
|
+
if (done &&
|
|
3605
|
+
batch.member_worktree !== null &&
|
|
3548
3606
|
goneOrReturned(batch.member_worktree, batch.member_pool_claimed)) {
|
|
3549
3607
|
clears.member_worktree = null;
|
|
3550
3608
|
clears.member_pool_claimed = false;
|
|
3551
3609
|
clearedFields.push('member_worktree', 'member_pool_claimed');
|
|
3552
3610
|
}
|
|
3553
|
-
|
|
3611
|
+
// #834: unlike `worktree`/`member_worktree`, `member_runs[]` is actively
|
|
3612
|
+
// mutated by many OTHER code paths while a tick runs (member dispatch,
|
|
3613
|
+
// eviction, `recordRunTeardown`, `sched attach-pr`, ...), so its evidence
|
|
3614
|
+
// check must NOT be computed from this function's own outer, unlocked
|
|
3615
|
+
// `batch` snapshot: `patchBatch` merges `member_runs` as a whole-array
|
|
3616
|
+
// replacement, not per-entry, so writing an array built from a stale
|
|
3617
|
+
// snapshot would silently discard a concurrent writer's change to a
|
|
3618
|
+
// SIBLING entry in the gap between this snapshot and the lock below
|
|
3619
|
+
// (a lost update — found in review). Only a coarse "is there anything
|
|
3620
|
+
// to look at" pre-check runs out here, to decide whether entering the
|
|
3621
|
+
// lock is worth it at all; the real per-run evidence check re-reads
|
|
3622
|
+
// `member_runs` fresh, from the batch `withLock` reloads below.
|
|
3623
|
+
const mightClearRuns = batch.member_runs.some((r) => (0, state_1.isLeftoverMemberRun)(batch, r));
|
|
3624
|
+
if (clearedFields.length === 0 && !mightClearRuns)
|
|
3554
3625
|
continue;
|
|
3626
|
+
const clearedRunIssues = [];
|
|
3555
3627
|
deps.store.withLock((s) => {
|
|
3556
3628
|
const b = (0, state_1.findBatch)(s, batch.id);
|
|
3557
3629
|
// Re-check under the lock: another tick, or an operator command, may
|
|
3558
3630
|
// have already moved this batch or cleared the fields itself.
|
|
3559
|
-
if (!b || b.status !==
|
|
3631
|
+
if (!b || b.status !== batch.status)
|
|
3632
|
+
return { state: s, result: undefined };
|
|
3633
|
+
const runClears = {};
|
|
3634
|
+
if (mightClearRuns) {
|
|
3635
|
+
const updatedRuns = b.member_runs.map((run) => {
|
|
3636
|
+
if (!(0, state_1.isLeftoverMemberRun)(b, run))
|
|
3637
|
+
return run;
|
|
3638
|
+
if (!goneOrReturned(run.worktree, run.pool_claimed))
|
|
3639
|
+
return run;
|
|
3640
|
+
clearedRunIssues.push(run.issue);
|
|
3641
|
+
return { ...run, torn_down: true };
|
|
3642
|
+
});
|
|
3643
|
+
if (clearedRunIssues.length > 0)
|
|
3644
|
+
runClears.member_runs = updatedRuns;
|
|
3645
|
+
}
|
|
3646
|
+
if (Object.keys(clears).length === 0 && Object.keys(runClears).length === 0) {
|
|
3560
3647
|
return { state: s, result: undefined };
|
|
3561
|
-
|
|
3648
|
+
}
|
|
3649
|
+
return {
|
|
3650
|
+
state: (0, state_1.patchBatch)(s, batch.id, { ...clears, ...runClears }, now, false),
|
|
3651
|
+
result: undefined,
|
|
3652
|
+
};
|
|
3562
3653
|
});
|
|
3654
|
+
if (clearedRunIssues.length > 0) {
|
|
3655
|
+
clearedFields.push(`member_runs[${clearedRunIssues.join(',')}].torn_down`);
|
|
3656
|
+
}
|
|
3657
|
+
if (clearedFields.length === 0)
|
|
3658
|
+
continue;
|
|
3563
3659
|
journalEvent(deps, 'kept-worktree-cleared', unit(batch.id), {
|
|
3564
3660
|
cleared: clearedFields.join(','),
|
|
3565
3661
|
detail: `kept-worktree ledger field(s) cleared on definitive evidence (${clearedFields.join(', ')}) — path gone or pool reports it returned; no removal command was run`,
|
|
@@ -3733,6 +3829,12 @@ function recordAnchorCloseFailed(deps, batch, outcome, shipped, now) {
|
|
|
3733
3829
|
* left on disk forever, since nothing else ever calls this for it.
|
|
3734
3830
|
*/
|
|
3735
3831
|
function teardownBatch(deps, batchId) {
|
|
3832
|
+
// #855: a kept batch's trees are the operator's — every teardown path reads
|
|
3833
|
+
// the same persisted decision. Only its live agents are stopped.
|
|
3834
|
+
if ((0, state_1.findBatch)(deps.store.load(), batchId)?.worktree_kept === true) {
|
|
3835
|
+
teardownParallelRuns(deps, batchId, null);
|
|
3836
|
+
return;
|
|
3837
|
+
}
|
|
3736
3838
|
// #677: the current member's worktree goes first — it is batch-owned
|
|
3737
3839
|
// state on the same teardown path as the shared tree, and a blocked or
|
|
3738
3840
|
// dissolved batch must not leak a member tree (its branch/fields are
|
|
@@ -3743,9 +3845,8 @@ function teardownBatch(deps, batchId) {
|
|
|
3743
3845
|
const terminal = (0, state_1.findBatch)(deps.store.load(), batchId);
|
|
3744
3846
|
const lastMemberLanded = terminal?.ranges.some((r) => r.issue === terminal.members[terminal.executing_member - 1]) ===
|
|
3745
3847
|
true;
|
|
3746
|
-
const
|
|
3747
|
-
|
|
3748
|
-
teardownParallelRuns(deps, batchId, serialTree);
|
|
3848
|
+
const serial = teardownMemberWorktree(deps, batchId, deps.now(), lastMemberLanded);
|
|
3849
|
+
teardownParallelRuns(deps, batchId, serial);
|
|
3749
3850
|
const state = deps.store.load();
|
|
3750
3851
|
const batch = (0, state_1.findBatch)(state, batchId);
|
|
3751
3852
|
if (!batch || batch.worktree === null)
|
|
@@ -3811,13 +3912,30 @@ function runTree(r) {
|
|
|
3811
3912
|
: { worktree: r.worktree, branch: r.branch, poolClaimed: r.poolClaimed };
|
|
3812
3913
|
}
|
|
3813
3914
|
/**
|
|
3814
|
-
*
|
|
3915
|
+
* Tear one parallel member run's tree down and record what actually happened
|
|
3916
|
+
* (#855) — the one teardown-and-record path for the evict, land and
|
|
3917
|
+
* `teardownParallelRuns` rails.
|
|
3918
|
+
*/
|
|
3919
|
+
function teardownRunTree(deps, batchId, run, landed, now) {
|
|
3920
|
+
const cleared = teardownMemberTree(deps, batchId, runTree(run), landed, run.issue);
|
|
3921
|
+
recordRunTeardown(deps, batchId, run.issue, cleared, now);
|
|
3922
|
+
}
|
|
3923
|
+
/** The one mapping from a member-tree teardown outcome to its run record (#855). */
|
|
3924
|
+
function teardownOutcomePatch(cleared, now) {
|
|
3925
|
+
return cleared ? { torn_down: true } : { teardown_failed_at: now.toISOString() };
|
|
3926
|
+
}
|
|
3927
|
+
/**
|
|
3928
|
+
* Record a run's teardown outcome. Written AFTER the teardown, so a crash in
|
|
3815
3929
|
* between re-runs the (idempotent) teardown from `teardownParallelRuns`
|
|
3816
|
-
* instead of leaking the tree.
|
|
3930
|
+
* instead of leaking the tree. #855: `torn_down` is set only when the
|
|
3931
|
+
* teardown verifiably landed; a failed one records `teardown_failed_at`
|
|
3932
|
+
* instead and leaves the tree on disk for the operator (never retried — see
|
|
3933
|
+
* `MemberRun.teardown_failed_at`). The caller's teardown already journaled
|
|
3934
|
+
* the failure, once.
|
|
3817
3935
|
*/
|
|
3818
|
-
function
|
|
3936
|
+
function recordRunTeardown(deps, batchId, issue, cleared, now) {
|
|
3819
3937
|
deps.store.withLock((s) => ({
|
|
3820
|
-
state: patchRun(s, batchId, issue,
|
|
3938
|
+
state: patchRun(s, batchId, issue, teardownOutcomePatch(cleared, now), now),
|
|
3821
3939
|
result: undefined,
|
|
3822
3940
|
}));
|
|
3823
3941
|
}
|
|
@@ -3953,6 +4071,7 @@ function spawnParallelMembers(deps, config, dispatch, batchId, now, result) {
|
|
|
3953
4071
|
status: 'running',
|
|
3954
4072
|
gate_inconclusive: null,
|
|
3955
4073
|
torn_down: false,
|
|
4074
|
+
teardown_failed_at: null,
|
|
3956
4075
|
}, now);
|
|
3957
4076
|
if ((0, scheduler_1.freeCapacity)(n, config) === 0)
|
|
3958
4077
|
return { state: n, result: true };
|
|
@@ -3980,7 +4099,7 @@ function spawnParallelMembers(deps, config, dispatch, batchId, now, result) {
|
|
|
3980
4099
|
reason: 'member-worktree-orphaned',
|
|
3981
4100
|
detail: `batch left executing while member ${next.index}'s tree ${member.worktree} was prepared — tearing it down`,
|
|
3982
4101
|
});
|
|
3983
|
-
teardownMemberTree(deps, batchId, runTree(member), false);
|
|
4102
|
+
teardownMemberTree(deps, batchId, runTree(member), false, next.issue);
|
|
3984
4103
|
}
|
|
3985
4104
|
}
|
|
3986
4105
|
}
|
|
@@ -4001,8 +4120,7 @@ function evictParallelMember(deps, config, batchId, run, failure, now, result) {
|
|
|
4001
4120
|
state: patchRun(s, batchId, run.issue, { status: 'evicted', gate_inconclusive: null }, now),
|
|
4002
4121
|
result: undefined,
|
|
4003
4122
|
}));
|
|
4004
|
-
|
|
4005
|
-
markTornDown(deps, batchId, run.issue, now);
|
|
4123
|
+
teardownRunTree(deps, batchId, run, false, now);
|
|
4006
4124
|
}
|
|
4007
4125
|
/**
|
|
4008
4126
|
* Reconcile one parallel member's slot — the parallel twin of
|
|
@@ -4238,8 +4356,7 @@ function advanceParallelBatch(deps, config, dispatch, batchId, now, result) {
|
|
|
4238
4356
|
// A gate-inconclusive member keeps its tree for `sched resume --batch`'s
|
|
4239
4357
|
// recheck; everyone else's is done serving.
|
|
4240
4358
|
if (run.gate_inconclusive === null) {
|
|
4241
|
-
|
|
4242
|
-
markTornDown(deps, batchId, run.issue, now);
|
|
4359
|
+
teardownRunTree(deps, batchId, run, true, now);
|
|
4243
4360
|
}
|
|
4244
4361
|
}
|
|
4245
4362
|
}
|
|
@@ -4287,7 +4404,7 @@ function finalizeParallelMembers(deps, config, dispatch, batchId, now, result) {
|
|
|
4287
4404
|
* run so the landing walk treats it as resolved, then continue the parallel
|
|
4288
4405
|
* rail from where it blocked.
|
|
4289
4406
|
*/
|
|
4290
|
-
function settleParallelResume(deps, batchId, memberIssue, now) {
|
|
4407
|
+
function settleParallelResume(deps, batchId, memberIssue, now, serialTeardown) {
|
|
4291
4408
|
deps.store.withLock((s) => {
|
|
4292
4409
|
const b = (0, state_1.findBatch)(s, batchId);
|
|
4293
4410
|
if (!b)
|
|
@@ -4296,8 +4413,10 @@ function settleParallelResume(deps, batchId, memberIssue, now) {
|
|
|
4296
4413
|
return {
|
|
4297
4414
|
state: patchRun(s, batchId, memberIssue, {
|
|
4298
4415
|
gate_inconclusive: null,
|
|
4299
|
-
// The serial rail's `teardownMemberWorktree`
|
|
4300
|
-
|
|
4416
|
+
// The serial rail's `teardownMemberWorktree` tore its tree down —
|
|
4417
|
+
// recorded as torn down only when that verifiably landed (#855).
|
|
4418
|
+
// No serial tree at all leaves the run live for `teardownBatch`.
|
|
4419
|
+
...(serialTeardown !== null ? teardownOutcomePatch(serialTeardown.cleared, now) : {}),
|
|
4301
4420
|
...(evicted ? { status: 'evicted' } : {}),
|
|
4302
4421
|
}, now),
|
|
4303
4422
|
result: undefined,
|
|
@@ -4339,11 +4458,19 @@ function reconcileParallelBatch(deps, config, dispatch, batchId, now, result, pa
|
|
|
4339
4458
|
* would be working a unit nothing tracks any more), `sched stop`/`abandon`
|
|
4340
4459
|
* make a batch terminal without a teardown, and a crash between a run's
|
|
4341
4460
|
* resolution and its teardown leaves `torn_down: false` behind. Teardown is
|
|
4342
|
-
* idempotent, so re-running it is always safe. `
|
|
4343
|
-
*
|
|
4344
|
-
* member of a blocked parallel batch)
|
|
4461
|
+
* idempotent, so re-running it is always safe. `serial` is what
|
|
4462
|
+
* `teardownMemberWorktree` just did with the serial-field tree (a
|
|
4463
|
+
* gate-inconclusive member of a blocked parallel batch) — its outcome is
|
|
4464
|
+
* recorded on the matching run without a second attempt.
|
|
4465
|
+
*
|
|
4466
|
+
* #855: default to KEEPING. A kept batch's (`worktree_kept`, #768's
|
|
4467
|
+
* members-closed reconcile) trees may hold unpushed operator work: its live
|
|
4468
|
+
* agents are still stopped (non-destructive), but no tree is removed or
|
|
4469
|
+
* pool-returned and `torn_down` stays false, so `sched status`'s
|
|
4470
|
+
* `kept-worktree` warning names them. A run whose earlier teardown failed
|
|
4471
|
+
* (`teardown_failed_at`) is not re-attempted either.
|
|
4345
4472
|
*/
|
|
4346
|
-
function teardownParallelRuns(deps, batchId,
|
|
4473
|
+
function teardownParallelRuns(deps, batchId, serial) {
|
|
4347
4474
|
const batch = (0, state_1.findBatch)(deps.store.load(), batchId);
|
|
4348
4475
|
if (!batch)
|
|
4349
4476
|
return;
|
|
@@ -4352,15 +4479,25 @@ function teardownParallelRuns(deps, batchId, alreadyTornDown) {
|
|
|
4352
4479
|
if (run.torn_down)
|
|
4353
4480
|
continue;
|
|
4354
4481
|
stopMemberAgent(deps, batchId, run.issue, now);
|
|
4482
|
+
if (run.teardown_failed_at !== null)
|
|
4483
|
+
continue;
|
|
4355
4484
|
const landed = run.status === 'landed';
|
|
4356
|
-
|
|
4357
|
-
|
|
4358
|
-
|
|
4359
|
-
|
|
4360
|
-
|
|
4361
|
-
|
|
4485
|
+
const status = landed ? 'landed' : 'evicted';
|
|
4486
|
+
// Only when it changes: a kept batch's runs pass through here every tick.
|
|
4487
|
+
if (run.status !== status || run.gate_inconclusive !== null) {
|
|
4488
|
+
deps.store.withLock((s) => ({
|
|
4489
|
+
state: patchRun(s, batchId, run.issue, { status, gate_inconclusive: null }, now),
|
|
4490
|
+
result: undefined,
|
|
4491
|
+
}));
|
|
4492
|
+
}
|
|
4493
|
+
if (batch.worktree_kept)
|
|
4494
|
+
continue;
|
|
4495
|
+
if (serial !== null && run.worktree === serial.path) {
|
|
4496
|
+
recordRunTeardown(deps, batchId, run.issue, serial.cleared, now);
|
|
4497
|
+
}
|
|
4498
|
+
else {
|
|
4499
|
+
teardownRunTree(deps, batchId, run, landed, now);
|
|
4362
4500
|
}
|
|
4363
|
-
markTornDown(deps, batchId, run.issue, now);
|
|
4364
4501
|
}
|
|
4365
4502
|
}
|
|
4366
4503
|
/** Kill a parallel member's live agent (pid-start checked) and release its slot. */
|
|
@@ -4568,6 +4705,13 @@ function runBatchTick(deps, config, dispatch) {
|
|
|
4568
4705
|
if (types_1.TERMINAL_BATCH_STATUSES.has(batch.status)) {
|
|
4569
4706
|
// #809: `sched stop`/`abandon` end a batch without a teardown — give
|
|
4570
4707
|
// back every member tree (and stop any agent still running in one).
|
|
4708
|
+
// #855: except a KEPT batch's trees (`worktree_kept`, the members-closed
|
|
4709
|
+
// reconcile's decision — the same one `worktree`/`member_worktree`
|
|
4710
|
+
// get) and a run whose teardown already failed: `teardownParallelRuns`
|
|
4711
|
+
// leaves both on disk with `torn_down: false`, and marks a run torn
|
|
4712
|
+
// down only after a teardown that verifiably landed. Such a batch
|
|
4713
|
+
// re-enters here every tick by design — cheap (a state read per run),
|
|
4714
|
+
// and it stops any agent that is somehow still running in a kept tree.
|
|
4571
4715
|
if (batch.member_runs.some((r) => !r.torn_down)) {
|
|
4572
4716
|
try {
|
|
4573
4717
|
teardownParallelRuns(deps, batch.id, null);
|