@ai-dossier/sched 0.31.0 → 0.33.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +44 -13
- package/dist/batch-dispatch.d.ts +10 -1
- package/dist/batch-dispatch.d.ts.map +1 -1
- package/dist/batch-dispatch.js +721 -23
- package/dist/batch-dispatch.js.map +1 -1
- package/dist/dispatch.d.ts +28 -10
- package/dist/dispatch.d.ts.map +1 -1
- package/dist/dispatch.js +61 -16
- package/dist/dispatch.js.map +1 -1
- package/dist/engine.d.ts +3 -2
- package/dist/engine.d.ts.map +1 -1
- package/dist/engine.js +31 -24
- package/dist/engine.js.map +1 -1
- package/dist/groundtruth.d.ts +27 -15
- package/dist/groundtruth.d.ts.map +1 -1
- package/dist/groundtruth.js +43 -21
- package/dist/groundtruth.js.map +1 -1
- package/dist/index.d.ts +1 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +5 -4
- package/dist/index.js.map +1 -1
- package/dist/state.d.ts.map +1 -1
- package/dist/state.js +26 -9
- package/dist/state.js.map +1 -1
- package/dist/types.d.ts +58 -14
- package/dist/types.d.ts.map +1 -1
- package/dist/types.js +5 -1
- package/dist/types.js.map +1 -1
- package/package.json +1 -1
package/dist/batch-dispatch.js
CHANGED
|
@@ -95,6 +95,7 @@ var __importStar = (this && this.__importStar) || (function () {
|
|
|
95
95
|
};
|
|
96
96
|
})();
|
|
97
97
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
98
|
+
exports.memberBranchFor = memberBranchFor;
|
|
98
99
|
exports.evictMemberAndContinue = evictMemberAndContinue;
|
|
99
100
|
exports.resumeBlockedGate = resumeBlockedGate;
|
|
100
101
|
exports.runBatchTick = runBatchTick;
|
|
@@ -529,7 +530,312 @@ function runBatchSetup(deps, batch, now) {
|
|
|
529
530
|
}
|
|
530
531
|
return { ok: true, branch, worktree, runId: mintedRunId, poolClaimed: false };
|
|
531
532
|
}
|
|
532
|
-
|
|
533
|
+
// --- Member worktrees (#677, RFC-0001 §J.3) ---
|
|
534
|
+
/**
|
|
535
|
+
* The current member's own branch: `batch/<id>-m<n>-<issue>` (#677) — member
|
|
536
|
+
* index and issue both in the name, and deterministic from persisted state,
|
|
537
|
+
* so a takeover redispatch, a `sched resume --batch` recheck, and teardown
|
|
538
|
+
* all re-derive the same string without a lookup table. Distinct from the
|
|
539
|
+
* integration branch (`batch/<id>-<date>`): the member commits here, the
|
|
540
|
+
* scheduler lands it there.
|
|
541
|
+
*/
|
|
542
|
+
function memberBranchFor(batchId, memberIndex, issue) {
|
|
543
|
+
return `batch/${batchId}-m${memberIndex}-${issue}`;
|
|
544
|
+
}
|
|
545
|
+
/** The worktree path a cold member-worktree prep creates for one member. */
|
|
546
|
+
function memberWorktreePathFor(deps, batchId, memberIndex, issue) {
|
|
547
|
+
return path.join(deps.repoDir, 'worktrees', `batch-${batchId}-m${memberIndex}-${issue}`);
|
|
548
|
+
}
|
|
549
|
+
/**
|
|
550
|
+
* Prepare the CURRENT member's own worktree and branch (#677, AC5): a
|
|
551
|
+
* worktree whose checked-out branch is `memberBranchFor(...)` created OFF the
|
|
552
|
+
* integration branch, warm before the member's first command, and pushed so
|
|
553
|
+
* the member agent can commit and push to it — `member-cycle`'s Step 0
|
|
554
|
+
* preconditions (inside a worktree, on YOUR member branch, clean, env warm).
|
|
555
|
+
* The agent never creates either; a prompt change without this preparation
|
|
556
|
+
* fails every member on precondition 2 (the reason #677 exists).
|
|
557
|
+
*
|
|
558
|
+
* Pool claim first — warm by construction, mirroring `runBatchSetup`. A pool
|
|
559
|
+
* spare sits on the default branch, so the claimed tree is re-pointed at the
|
|
560
|
+
* member branch off the INTEGRATION branch (`git checkout -B` against the
|
|
561
|
+
* `origin/<integration>` ref `runBatchSetup`'s push already created
|
|
562
|
+
* repo-wide); checkout preserves the installed `node_modules`. Cold fallback:
|
|
563
|
+
* `git worktree add` at the integration branch tip + `warmColdBatchWorktree`
|
|
564
|
+
* (#561 — a cold member is an evicted member).
|
|
565
|
+
*
|
|
566
|
+
* An on-disk worktree for the SAME member is reused as-is (takeover
|
|
567
|
+
* redispatch, api-error hold retry): the branch is already checked out, the
|
|
568
|
+
* tree already warm, and a takeover agent resumes in place — possibly dirty,
|
|
569
|
+
* which is the parent's state to reason about, not something to reset.
|
|
570
|
+
*
|
|
571
|
+
* All-or-nothing like `runBatchSetup`; the exec calls run OUTSIDE any store
|
|
572
|
+
* lock (the caller's contract), with results landed as pure data.
|
|
573
|
+
*/
|
|
574
|
+
/**
|
|
575
|
+
* The pool CLI's stdout hardening shared by `runBatchSetup` and the member
|
|
576
|
+
* prep (no NUL/newline, absolute, resolved, on disk) — our own CLI's
|
|
577
|
+
* output, but garbage would otherwise be trusted as a `BatchEntry`
|
|
578
|
+
* worktree path and used as a spawn/exec cwd.
|
|
579
|
+
*/
|
|
580
|
+
function usablePoolClaim(deps, w) {
|
|
581
|
+
return (!!w &&
|
|
582
|
+
!w.includes('\0') &&
|
|
583
|
+
!w.includes('\n') &&
|
|
584
|
+
path.isAbsolute(w) &&
|
|
585
|
+
path.resolve(w) === w &&
|
|
586
|
+
(deps.fsExists ?? ((p) => fs.existsSync(p)))(w));
|
|
587
|
+
}
|
|
588
|
+
function prepareMemberWorktree(deps, batch, memberIndex, issue, now) {
|
|
589
|
+
if (batch.branch === null)
|
|
590
|
+
return { ok: false, reason: 'no-integration-branch' };
|
|
591
|
+
const branch = memberBranchFor(batch.id, memberIndex, issue);
|
|
592
|
+
const worktree = memberWorktreePathFor(deps, batch.id, memberIndex, issue);
|
|
593
|
+
if (!attribution_1.SAFE_REF_RE.test(branch))
|
|
594
|
+
return { ok: false, reason: 'invalid-member-branch-name' };
|
|
595
|
+
const fsExists = deps.fsExists ?? ((p) => fs.existsSync(p));
|
|
596
|
+
const root = deps.exec('git', ['rev-parse', '--show-toplevel'], deps.repoDir) ?? deps.repoDir;
|
|
597
|
+
if (!(0, teardown_1.isSafeWorktree)(path.resolve(root), worktree)) {
|
|
598
|
+
return { ok: false, reason: 'invalid-member-worktree-path' };
|
|
599
|
+
}
|
|
600
|
+
// All-or-nothing rollback for the cold path (same contract as
|
|
601
|
+
// `runBatchSetup`): a warm or push failure otherwise leaves a cold
|
|
602
|
+
// worktree the NEXT retry's exists-check reuses forever cold — exactly
|
|
603
|
+
// the env-cold eviction #561 removed.
|
|
604
|
+
const rollbackColdPrep = () => {
|
|
605
|
+
deps.exec('git', ['worktree', 'remove', '--force', '--', worktree], deps.repoDir);
|
|
606
|
+
deps.exec('git', ['branch', '-D', branch], deps.repoDir);
|
|
607
|
+
};
|
|
608
|
+
const exists = fsExists(worktree);
|
|
609
|
+
if (exists) {
|
|
610
|
+
// Crash-window reuse: the derived-path tree exists on disk but the
|
|
611
|
+
// spawn never landed (fields null) — OR a pool-claimed tree whose
|
|
612
|
+
// directory name equals the derived slug (the pool renames claims to
|
|
613
|
+
// the branch slug, so a pool tree CAN sit at the derived path; the
|
|
614
|
+
// persisted `member_pool_claimed` is the only record of which). In
|
|
615
|
+
// either case the on-disk tree wins: reuse it rather than re-branching.
|
|
616
|
+
// #632: fires once per crash window, not per tick — the call sites run
|
|
617
|
+
// at spawn time (a transition point), and `claimMemberResolution`
|
|
618
|
+
// gates the member advance; the capacity pre-check in
|
|
619
|
+
// `spawnMemberContinuation` keeps the wedge path from re-entering
|
|
620
|
+
// while no slot is free.
|
|
621
|
+
deps.journal.append((0, journal_1.unitEvent)('member-worktree-reused', unit(batch.id), {
|
|
622
|
+
issue,
|
|
623
|
+
detail: worktree,
|
|
624
|
+
}), now);
|
|
625
|
+
return {
|
|
626
|
+
ok: true,
|
|
627
|
+
branch: batch.member_branch ?? branch,
|
|
628
|
+
worktree: batch.member_worktree ?? worktree,
|
|
629
|
+
poolClaimed: batch.member_pool_claimed,
|
|
630
|
+
};
|
|
631
|
+
}
|
|
632
|
+
const claimed = deps.exec(teardown_1.POOL_BIN, [...teardown_1.POOL_ARGS_PREFIX, 'claim', '--issue', String(issue), '--branch', branch], deps.repoDir);
|
|
633
|
+
const claimedWorktree = claimed?.trim();
|
|
634
|
+
if (claimedWorktree && usablePoolClaim(deps, claimedWorktree)) {
|
|
635
|
+
// Re-point the warm spare at the member branch off the integration
|
|
636
|
+
// branch — `origin/<integration>` exists repo-wide (runBatchSetup pushed
|
|
637
|
+
// it), and `checkout -B` preserves the installed node_modules.
|
|
638
|
+
if (deps.exec('git', ['checkout', '-B', branch, `origin/${batch.branch}`], claimedWorktree) ===
|
|
639
|
+
null) {
|
|
640
|
+
deps.exec(teardown_1.POOL_BIN, [...teardown_1.POOL_ARGS_PREFIX, 'return', '--path', claimedWorktree, '--json'], deps.repoDir);
|
|
641
|
+
return { ok: false, reason: 'member-branch-checkout-failed' };
|
|
642
|
+
}
|
|
643
|
+
if (deps.exec('git', ['push', '-u', 'origin', '--', branch], claimedWorktree) === null) {
|
|
644
|
+
deps.exec(teardown_1.POOL_BIN, [...teardown_1.POOL_ARGS_PREFIX, 'return', '--path', claimedWorktree, '--json'], deps.repoDir);
|
|
645
|
+
return { ok: false, reason: 'member-branch-push-failed' };
|
|
646
|
+
}
|
|
647
|
+
return { ok: true, branch, worktree: claimedWorktree, poolClaimed: true };
|
|
648
|
+
}
|
|
649
|
+
const poolNote = claimedWorktree ? 'pool-claim-invalid:' : '';
|
|
650
|
+
const added = deps.exec('git', ['worktree', 'add', '-b', branch, worktree, batch.branch], deps.repoDir) !== null;
|
|
651
|
+
if (!added) {
|
|
652
|
+
// Self-heal the crash window a teardown died in (branch kept, tree
|
|
653
|
+
// gone): `worktree add -b` fails while the stale branch exists. The
|
|
654
|
+
// member branch is disposable by construction — a member whose branch
|
|
655
|
+
// still matters is CURRENT, and a current member's tree is reused by
|
|
656
|
+
// the exists-check above, never re-added. Force-delete the stale ref
|
|
657
|
+
// and retry the add once.
|
|
658
|
+
if (deps.exec('git', ['branch', '-D', branch], deps.repoDir) === null) {
|
|
659
|
+
return { ok: false, reason: 'member-worktree-add-failed' };
|
|
660
|
+
}
|
|
661
|
+
if (deps.exec('git', ['worktree', 'add', '-b', branch, worktree, batch.branch], deps.repoDir) === null) {
|
|
662
|
+
return { ok: false, reason: 'member-worktree-add-failed' };
|
|
663
|
+
}
|
|
664
|
+
}
|
|
665
|
+
const warmed = warmColdBatchWorktree(deps, batch, worktree, now, poolNote);
|
|
666
|
+
if (!warmed.ok) {
|
|
667
|
+
rollbackColdPrep();
|
|
668
|
+
return { ok: false, reason: warmed.reason };
|
|
669
|
+
}
|
|
670
|
+
if (deps.exec('git', ['push', '-u', 'origin', '--', branch], worktree) === null) {
|
|
671
|
+
rollbackColdPrep();
|
|
672
|
+
return { ok: false, reason: 'member-branch-push-failed' };
|
|
673
|
+
}
|
|
674
|
+
return { ok: true, branch, worktree, poolClaimed: false };
|
|
675
|
+
}
|
|
676
|
+
/**
|
|
677
|
+
* Land the CURRENT member's work on the integration branch (#677, §J.3: "a
|
|
678
|
+
* member lands its commit onto the integration branch when its own
|
|
679
|
+
* verification passes"): `git merge --ff-only <member_branch>` in the shared
|
|
680
|
+
* batch worktree (which holds the integration branch), then push.
|
|
681
|
+
*
|
|
682
|
+
* Fast-forward is always available under the SERIAL dispatch model — only
|
|
683
|
+
* the member moved its branch since it was cut, and nothing else moves the
|
|
684
|
+
* integration branch mid-`executing` (fix agents only run from the
|
|
685
|
+
* post-last-member aggregate validation). A failed merge therefore means the
|
|
686
|
+
* serial invariant broke, not a conflict to resolve: the batch BLOCKS for an
|
|
687
|
+
* operator instead of silently merging or reverting. Linear history is also
|
|
688
|
+
* what `boundaryCommits`/`memberRanges` attribution expects — the member's
|
|
689
|
+
* `(#<issue>)`-trailed subjects stay first-parent-readable after the landing.
|
|
690
|
+
*/
|
|
691
|
+
function landMemberBranch(deps, batchId, now) {
|
|
692
|
+
const state = deps.store.load();
|
|
693
|
+
const batch = (0, state_1.findBatch)(state, batchId);
|
|
694
|
+
if (!batch || batch.worktree === null || batch.branch === null || batch.member_branch === null) {
|
|
695
|
+
return { ok: true };
|
|
696
|
+
}
|
|
697
|
+
const memberIssue = batch.members[batch.executing_member - 1];
|
|
698
|
+
// #677 review (defense in depth, matching `branchMergedIntoBase`'s bar):
|
|
699
|
+
// the persisted ref reaches git argv below — re-validate its shape.
|
|
700
|
+
if (!attribution_1.SAFE_REF_RE.test(batch.member_branch) || !attribution_1.SAFE_REF_RE.test(batch.branch)) {
|
|
701
|
+
journalEvent(deps, 'landing-failed', unit(batchId), {
|
|
702
|
+
issue: memberIssue,
|
|
703
|
+
reason: 'invalid-branch-name',
|
|
704
|
+
detail: `persisted member/integration branch failed SAFE_REF_RE: ${batch.member_branch} → ${batch.branch}`,
|
|
705
|
+
});
|
|
706
|
+
return { ok: false, reason: 'invalid-branch-name' };
|
|
707
|
+
}
|
|
708
|
+
if (deps.exec('git', ['merge', '--ff-only', batch.member_branch], batch.worktree) === null) {
|
|
709
|
+
journalEvent(deps, 'landing-failed', unit(batchId), {
|
|
710
|
+
issue: memberIssue,
|
|
711
|
+
reason: 'landing-merge-failed',
|
|
712
|
+
detail: `git merge --ff-only ${batch.member_branch} into ${batch.branch} failed — serial-landing invariant broken; blocking for an operator`,
|
|
713
|
+
});
|
|
714
|
+
return { ok: false, reason: 'landing-merge-failed' };
|
|
715
|
+
}
|
|
716
|
+
if (deps.exec('git', ['push', 'origin', '--', batch.branch], batch.worktree) === null) {
|
|
717
|
+
journalEvent(deps, 'landing-failed', unit(batchId), {
|
|
718
|
+
issue: memberIssue,
|
|
719
|
+
reason: 'landing-push-failed',
|
|
720
|
+
detail: `landed ${batch.member_branch} on ${batch.branch} locally but the push failed — blocking so origin stays the durable copy`,
|
|
721
|
+
});
|
|
722
|
+
return { ok: false, reason: 'landing-push-failed' };
|
|
723
|
+
}
|
|
724
|
+
deps.journal.append((0, journal_1.unitEvent)('member-landed', unit(batchId), {
|
|
725
|
+
issue: memberIssue,
|
|
726
|
+
detail: `${batch.member_branch} fast-forwarded onto ${batch.branch}`,
|
|
727
|
+
}), now);
|
|
728
|
+
return { ok: true };
|
|
729
|
+
}
|
|
730
|
+
/**
|
|
731
|
+
* Tear the CURRENT member's worktree down and clear the member fields
|
|
732
|
+
* (#677): pool-claimed trees are RETURNED to the pool (a raw remove leaves a
|
|
733
|
+
* dangling pool entry — the corrupted-entry state `worktree-pool status`
|
|
734
|
+
* reports), cold trees are removed. The member branch is deleted locally —
|
|
735
|
+
* `-d` after a landing (fully merged into the integration branch), `-D` on
|
|
736
|
+
* the eviction path (its commits never landed; the requeue carries the work
|
|
737
|
+
* forward). The REMOTE member branch is deleted only on the landed path, and
|
|
738
|
+
* deliberately KEPT on eviction — the pushed sha is the evicted work's only
|
|
739
|
+
* recoverable copy.
|
|
740
|
+
*
|
|
741
|
+
* Best-effort and idempotent: null fields are a no-op, and a failed cleanup
|
|
742
|
+
* journals `teardown-failed` rather than throwing into the caller's
|
|
743
|
+
* advance/dissolve path.
|
|
744
|
+
*/
|
|
745
|
+
function teardownMemberWorktree(deps, batchId, now,
|
|
746
|
+
/** Whether the member's branch was ff-landed onto the integration branch. Passed explicitly by the callers that KNOW — deriving it from `batch.ranges` would silently force-delete a merged branch if a caller ever reached teardown after the member pointer advanced. */
|
|
747
|
+
landed) {
|
|
748
|
+
const state = deps.store.load();
|
|
749
|
+
const batch = (0, state_1.findBatch)(state, batchId);
|
|
750
|
+
if (!batch || batch.member_worktree === null)
|
|
751
|
+
return;
|
|
752
|
+
const worktree = batch.member_worktree;
|
|
753
|
+
const branch = batch.member_branch;
|
|
754
|
+
// #677 review (defense in depth, matching `branchMergedIntoBase`'s bar):
|
|
755
|
+
// persisted state reaches git argv here — re-validate the ref shape before
|
|
756
|
+
// interpolating it.
|
|
757
|
+
if (branch !== null && !attribution_1.SAFE_REF_RE.test(branch)) {
|
|
758
|
+
journalEvent(deps, 'teardown-failed', unit(batchId), {
|
|
759
|
+
cleanup: 'failed-invalid-branch-name',
|
|
760
|
+
detail: branch,
|
|
761
|
+
});
|
|
762
|
+
return;
|
|
763
|
+
}
|
|
764
|
+
// Route through `runTeardown` for the same guarantees `teardownBatch`
|
|
765
|
+
// gets: pool returns are verify-first idempotent and refuse to claim
|
|
766
|
+
// success unless the pool's own self-check reports the entry `warm`;
|
|
767
|
+
// cold removes post-verify the path is gone and unlisted. Nulls first —
|
|
768
|
+
// clear the fields BEFORE the exec so a crash mid-teardown doesn't retry
|
|
769
|
+
// into a half-cleaned tree (the re-derived path is deterministic if a
|
|
770
|
+
// leftover does survive).
|
|
771
|
+
deps.store.withLock((s) => {
|
|
772
|
+
const b = (0, state_1.findBatch)(s, batchId);
|
|
773
|
+
if (!b)
|
|
774
|
+
return { state: s, result: undefined };
|
|
775
|
+
return {
|
|
776
|
+
state: (0, state_1.patchBatch)(s, batchId, { member_branch: null, member_worktree: null, member_pool_claimed: false }, now),
|
|
777
|
+
result: undefined,
|
|
778
|
+
};
|
|
779
|
+
});
|
|
780
|
+
const t = (0, teardown_1.runTeardown)(deps.exec, deps.repoDir, { worktree, poolClaimed: batch.member_pool_claimed === true, branch }, deps.fsExists);
|
|
781
|
+
const treeCleared = !t.cleanup.startsWith('failed');
|
|
782
|
+
let branchCleanup = 'skipped';
|
|
783
|
+
if (branch !== null && treeCleared) {
|
|
784
|
+
// `-d` refuses an unmerged branch; the eviction path forces it
|
|
785
|
+
// deliberately (see doc). Run from repoDir — the worktree is gone.
|
|
786
|
+
const flag = landed ? '-d' : '-D';
|
|
787
|
+
branchCleanup =
|
|
788
|
+
deps.exec('git', ['branch', flag, branch], deps.repoDir) === null
|
|
789
|
+
? `failed-branch-delete-${flag}`
|
|
790
|
+
: `branch-deleted-${flag}`;
|
|
791
|
+
if (landed) {
|
|
792
|
+
deps.exec('git', ['push', 'origin', '--delete', branch], deps.repoDir);
|
|
793
|
+
}
|
|
794
|
+
}
|
|
795
|
+
const failed = !treeCleared || branchCleanup.startsWith('failed');
|
|
796
|
+
journalEvent(deps, failed ? 'teardown-failed' : 'member-worktree-torn-down', unit(batchId), {
|
|
797
|
+
cleanup: t.cleanup,
|
|
798
|
+
branch_cleanup: branchCleanup,
|
|
799
|
+
detail: t.detail,
|
|
800
|
+
worktree,
|
|
801
|
+
});
|
|
802
|
+
}
|
|
803
|
+
/**
|
|
804
|
+
* The shared "block the batch for an operator" tail (#677 review — the same
|
|
805
|
+
* load → `recoveryDeps` → `blockBatch` → re-apply sequence
|
|
806
|
+
* `runValidate`'s suite-unreadable path and `runIncrementalGate`'s
|
|
807
|
+
* inconclusive path each grew): releases nothing itself — callers release
|
|
808
|
+
* their own slot first — and always reports the unit under `failed`.
|
|
809
|
+
*/
|
|
810
|
+
function blockBatchForOperator(deps, config, batchId, opts, now, result) {
|
|
811
|
+
const stateNow = deps.store.load();
|
|
812
|
+
const fresh = (0, state_1.findBatch)(stateNow, batchId);
|
|
813
|
+
if (!fresh)
|
|
814
|
+
return;
|
|
815
|
+
const blocked = (0, recovery_1.blockBatch)(stateNow, batchId, opts, recoveryDeps(deps, config, fresh, now));
|
|
816
|
+
deps.store.withLock((s) => ({
|
|
817
|
+
state: applyBatchAndIssues(s, blocked.state, batchId, []),
|
|
818
|
+
result: undefined,
|
|
819
|
+
}));
|
|
820
|
+
result.failed.push(unit(batchId));
|
|
821
|
+
}
|
|
822
|
+
/**
|
|
823
|
+
* Block the batch on a mechanical landing failure (#677): release the slot,
|
|
824
|
+
* persist `blocked_reason`, and journal — the same operator-inspection path
|
|
825
|
+
* `suite-unreadable` uses. The member's work is safe (its branch is pushed);
|
|
826
|
+
* only the integration step failed.
|
|
827
|
+
*/
|
|
828
|
+
function blockLandingFailure(deps, config, batch, reason, now, result) {
|
|
829
|
+
deps.store.withLock((s) => ({ state: releaseSlot(s, batch.id, now), result: undefined }));
|
|
830
|
+
blockBatchForOperator(deps, config, batch.id, { reason }, now, result);
|
|
831
|
+
}
|
|
832
|
+
/**
|
|
833
|
+
* Spawn one batch member's member-cycle agent (#677) into the slot a prior
|
|
834
|
+
* member (or batch-setup) just released — each member in its OWN worktree
|
|
835
|
+
* off the integration branch, dispatched serially as before. Prepared by
|
|
836
|
+
* {@link prepareMemberWorktree} at the spawn site (outside the lock); this
|
|
837
|
+
* function only patches state and spawns.
|
|
838
|
+
*/
|
|
533
839
|
/**
|
|
534
840
|
* Drive a member's `QueueEntry` through the D.1 slot-line states it must pass
|
|
535
841
|
* through before `shipped-in-batch` becomes a legal edge (`validated` is the
|
|
@@ -567,15 +873,15 @@ function advanceMemberToValidated(state, memberIssue, now) {
|
|
|
567
873
|
}
|
|
568
874
|
return next;
|
|
569
875
|
}
|
|
570
|
-
function spawnMember(deps, dispatch, state, slot, batchId, now, result) {
|
|
876
|
+
function spawnMember(deps, dispatch, state, slot, batchId, now, result, member) {
|
|
571
877
|
const batch = (0, state_1.findBatch)(state, batchId);
|
|
572
|
-
if (!batch || batch.worktree === null) {
|
|
878
|
+
if (!batch || batch.worktree === null || batch.branch === null) {
|
|
573
879
|
// A leaked `assigned` slot with `pid: null` is invisible to `dead`
|
|
574
880
|
// detection (nothing ever kills/reclaims it) — release it here rather
|
|
575
881
|
// than leaving the batch permanently down one slot of capacity.
|
|
576
882
|
journalEvent(deps, 'unit-failed', unit(batchId), {
|
|
577
883
|
reason: 'no-worktree',
|
|
578
|
-
detail: 'spawnMember: batch has no worktree — batch-setup has not landed',
|
|
884
|
+
detail: 'spawnMember: batch has no worktree/branch — batch-setup has not landed',
|
|
579
885
|
});
|
|
580
886
|
return releaseSlot(state, batchId, now);
|
|
581
887
|
}
|
|
@@ -591,7 +897,10 @@ function spawnMember(deps, dispatch, state, slot, batchId, now, result) {
|
|
|
591
897
|
const tier = (0, state_1.findEntry)(withStatus, memberIssue)?.tier ?? 'mid';
|
|
592
898
|
const spawnSpec = (0, dispatch_1.resolveTierSpawn)(dispatch, tier, memberIssue);
|
|
593
899
|
const cmd = spawnSpec.cmd;
|
|
594
|
-
|
|
900
|
+
// #677: the prompt carries THIS member's own worktree and the integration
|
|
901
|
+
// branch it was cut from — `member-cycle`'s `worktree`/`integration_branch`
|
|
902
|
+
// inputs. The shared batch worktree is no longer a member surface at all.
|
|
903
|
+
const prompt = (0, dispatch_1.buildMemberPrompt)(dispatch.memberPrompt, memberIssue, batchId, member.worktree, batch.branch);
|
|
595
904
|
const logFile = (0, dispatch_1.batchMemberLogPath)(deps.store.runsDir, batchId, batch.executing_member, memberIssue);
|
|
596
905
|
// #629: captured BEFORE spawning, mirroring `engine.ts`'s own
|
|
597
906
|
// `spawnAndRecord` — the log is per-role and append-mode, so the size at
|
|
@@ -622,9 +931,18 @@ function spawnMember(deps, dispatch, state, slot, batchId, now, result) {
|
|
|
622
931
|
spawned_at: now.toISOString(),
|
|
623
932
|
log_offset_at_spawn: logOffset,
|
|
624
933
|
};
|
|
625
|
-
|
|
934
|
+
let next = slot.status === 'assigned' || slot.status === 'recovering'
|
|
626
935
|
? (0, state_1.transitionSlot)(withStatus, slot.id, 'running', patch, now)
|
|
627
936
|
: withStatus;
|
|
937
|
+
// #677: persist the member's worktree context — a takeover redispatch or a
|
|
938
|
+
// `sched resume --batch` recheck after an engine restart must land in the
|
|
939
|
+
// SAME worktree/branch this prompt names, not re-derive (or worse,
|
|
940
|
+
// re-create) it.
|
|
941
|
+
next = (0, state_1.patchBatch)(next, batchId, {
|
|
942
|
+
member_branch: member.branch,
|
|
943
|
+
member_worktree: member.worktree,
|
|
944
|
+
member_pool_claimed: member.poolClaimed,
|
|
945
|
+
}, now);
|
|
628
946
|
deps.journal.append((0, journal_1.unitEvent)('spawned', unit(batchId), {
|
|
629
947
|
pid,
|
|
630
948
|
tier,
|
|
@@ -632,6 +950,7 @@ function spawnMember(deps, dispatch, state, slot, batchId, now, result) {
|
|
|
632
950
|
issue: memberIssue,
|
|
633
951
|
...(0, dispatch_1.journalCmdModelFields)(spawnSpec),
|
|
634
952
|
log: logFile,
|
|
953
|
+
worktree: member.worktree,
|
|
635
954
|
detail: `member ${batch.executing_member}/${batch.members.length}`,
|
|
636
955
|
}), now);
|
|
637
956
|
result.spawned.push(unit(batchId));
|
|
@@ -703,6 +1022,45 @@ function claimAndSetup(deps, config, dispatch, batchId, now, result) {
|
|
|
703
1022
|
},
|
|
704
1023
|
});
|
|
705
1024
|
deps.journal.append((0, journal_1.unitEvent)('batch-setup-done', unit(batchId), { detail: setup.worktree }), now);
|
|
1025
|
+
// #677: prepare member 1's OWN worktree before claiming the spawn — real
|
|
1026
|
+
// git/network work, so OUTSIDE the store lock (the same contract
|
|
1027
|
+
// `runBatchSetup` just followed). All-or-nothing: a failed prep posts the
|
|
1028
|
+
// blocked milestone and releases the slot rather than spawning into a
|
|
1029
|
+
// worktree that does not exist (every member would fail Step 0).
|
|
1030
|
+
const firstIssue = batch.members[0];
|
|
1031
|
+
// Shared abort tail of every setup-stage failure (#677 review): post the
|
|
1032
|
+
// blocked milestone when a run id exists, journal, report, release.
|
|
1033
|
+
const abortSetup = (kvReason, detail) => {
|
|
1034
|
+
if (batch.anchor !== null && setup.runId !== null) {
|
|
1035
|
+
poster(batch.anchor, setup.runId, {
|
|
1036
|
+
phase: 'batch-setup',
|
|
1037
|
+
status: 'blocked',
|
|
1038
|
+
kv: { reason: kvReason },
|
|
1039
|
+
});
|
|
1040
|
+
}
|
|
1041
|
+
else {
|
|
1042
|
+
journalEvent(deps, 'milestone-post-failed', unit(batchId), {
|
|
1043
|
+
detail: `batch-setup blocked (${kvReason}) — no run id to post to yet`,
|
|
1044
|
+
});
|
|
1045
|
+
}
|
|
1046
|
+
deps.journal.append((0, journal_1.unitEvent)('batch-setup-failed', unit(batchId), { detail }), now);
|
|
1047
|
+
result.failed.push(unit(batchId));
|
|
1048
|
+
deps.store.withLock((s) => ({ state: releaseSlot(s, batchId, now), result: undefined }));
|
|
1049
|
+
};
|
|
1050
|
+
if (firstIssue === undefined) {
|
|
1051
|
+
abortSetup('no-member', 'claimAndSetup: batch has no members');
|
|
1052
|
+
return;
|
|
1053
|
+
}
|
|
1054
|
+
const freshForPrep = (0, state_1.findBatch)(deps.store.load(), batchId);
|
|
1055
|
+
const memberPrep = prepareMemberWorktree(deps,
|
|
1056
|
+
// The setup patch (branch/worktree/run_id) lands under the lock BELOW —
|
|
1057
|
+
// prep needs the integration branch, so overlay `setup.branch` on the
|
|
1058
|
+
// fresh state rather than waiting for the patch.
|
|
1059
|
+
{ ...(freshForPrep ?? batch), branch: setup.branch }, 1, firstIssue, now);
|
|
1060
|
+
if (!memberPrep.ok) {
|
|
1061
|
+
abortSetup(`member-worktree-prep-failed:${memberPrep.reason}`, `member worktree prep failed: ${memberPrep.reason} (member 1 #${firstIssue}, branch ${memberBranchFor(batchId, 1, firstIssue)})`);
|
|
1062
|
+
return;
|
|
1063
|
+
}
|
|
706
1064
|
deps.store.withLock((s) => {
|
|
707
1065
|
const b = (0, state_1.findBatch)(s, batchId);
|
|
708
1066
|
if (!b || b.status !== 'ready') {
|
|
@@ -727,7 +1085,7 @@ function claimAndSetup(deps, config, dispatch, batchId, now, result) {
|
|
|
727
1085
|
next = (0, state_1.transitionBatch)(next, batchId, 'executing', { executing_member: 1 }, now);
|
|
728
1086
|
const slot = slotFor(next, batchId);
|
|
729
1087
|
if (slot)
|
|
730
|
-
next = spawnMember(deps, dispatch, next, slot, batchId, now, result);
|
|
1088
|
+
next = spawnMember(deps, dispatch, next, slot, batchId, now, result, memberPrep);
|
|
731
1089
|
return { state: next, result: undefined };
|
|
732
1090
|
});
|
|
733
1091
|
}
|
|
@@ -753,7 +1111,65 @@ function claimAndSpawn(deps, config, batchId, phase, now, spawn) {
|
|
|
753
1111
|
});
|
|
754
1112
|
}
|
|
755
1113
|
function spawnMemberContinuation(deps, config, dispatch, batchId, now, result) {
|
|
756
|
-
|
|
1114
|
+
// #677: make sure the current member's worktree exists before claiming a
|
|
1115
|
+
// slot for the redispatch — git exec work, so OUTSIDE the store lock (the
|
|
1116
|
+
// contract `prepareMemberWorktree` documents). The wedge arm of
|
|
1117
|
+
// `runBatchTick` calls this EVERY tick while the batch is `executing`
|
|
1118
|
+
// with no slot, so the cheap gates run first:
|
|
1119
|
+
// - capacity: prep before the free-capacity check would do pool/git/npm
|
|
1120
|
+
// work (and journal) per tick against a full scheduler — the #610/#630/
|
|
1121
|
+
// #632 per-tick emission trap. `claimAndSpawn` re-checks under the lock;
|
|
1122
|
+
// this pre-check only avoids wasted prep, it is not the gate.
|
|
1123
|
+
// - identity: persisted member fields are reused only when they belong to
|
|
1124
|
+
// the CURRENT member (they can be stale after a failed teardown that
|
|
1125
|
+
// kept them, or a crash between an eviction and its teardown) and the
|
|
1126
|
+
// tree still EXISTS on disk (state-first, because a pool-claimed path
|
|
1127
|
+
// is not re-derivable from batchId+index+issue).
|
|
1128
|
+
const state = deps.store.load();
|
|
1129
|
+
const batch = (0, state_1.findBatch)(state, batchId);
|
|
1130
|
+
if (!batch || batch.status !== 'executing' || batch.worktree === null)
|
|
1131
|
+
return;
|
|
1132
|
+
const memberIssue = batch.members[batch.executing_member - 1];
|
|
1133
|
+
if (memberIssue === undefined)
|
|
1134
|
+
return;
|
|
1135
|
+
if ((0, scheduler_1.freeCapacity)(state, config) === 0)
|
|
1136
|
+
return;
|
|
1137
|
+
const expectedBranch = memberBranchFor(batchId, batch.executing_member, memberIssue);
|
|
1138
|
+
const fsExists = deps.fsExists ?? ((p) => fs.existsSync(p));
|
|
1139
|
+
const reusable = batch.member_worktree !== null &&
|
|
1140
|
+
batch.member_branch === expectedBranch &&
|
|
1141
|
+
fsExists(batch.member_worktree);
|
|
1142
|
+
let member;
|
|
1143
|
+
if (reusable) {
|
|
1144
|
+
member = {
|
|
1145
|
+
branch: batch.member_branch,
|
|
1146
|
+
worktree: batch.member_worktree,
|
|
1147
|
+
poolClaimed: batch.member_pool_claimed,
|
|
1148
|
+
};
|
|
1149
|
+
}
|
|
1150
|
+
else {
|
|
1151
|
+
if (batch.member_worktree !== null) {
|
|
1152
|
+
// Stale context for a DIFFERENT member (or a vanished tree): journal
|
|
1153
|
+
// it before overwriting — the disposition of that tree is the
|
|
1154
|
+
// operator's to inspect if it was real.
|
|
1155
|
+
journalEvent(deps, 'stale-member-worktree', unit(batchId), {
|
|
1156
|
+
issue: memberIssue,
|
|
1157
|
+
detail: `persisted ${batch.member_branch ?? '?'} @ ${batch.member_worktree} does not match expected ${expectedBranch} (or tree gone) — preparing fresh`,
|
|
1158
|
+
});
|
|
1159
|
+
}
|
|
1160
|
+
const prep = prepareMemberWorktree(deps, batch, batch.executing_member, memberIssue, now);
|
|
1161
|
+
if (!prep.ok) {
|
|
1162
|
+
journalEvent(deps, 'unit-failed', unit(batchId), {
|
|
1163
|
+
issue: memberIssue,
|
|
1164
|
+
reason: 'member-worktree-prep-failed',
|
|
1165
|
+
detail: `member worktree prep failed on continuation: ${prep.reason} (branch ${expectedBranch})`,
|
|
1166
|
+
});
|
|
1167
|
+
result.failed.push(unit(batchId));
|
|
1168
|
+
return;
|
|
1169
|
+
}
|
|
1170
|
+
member = prep;
|
|
1171
|
+
}
|
|
1172
|
+
claimAndSpawn(deps, config, batchId, 'member', now, (nextState, slot) => spawnMember(deps, dispatch, nextState, slot, batchId, now, result, member));
|
|
757
1173
|
}
|
|
758
1174
|
/**
|
|
759
1175
|
* Tail/report/fix dispatches (this function, `spawnReportAgent`,
|
|
@@ -1124,6 +1540,12 @@ function evictMemberAndContinue(deps, config, dispatch, batchId, batch, memberIs
|
|
|
1124
1540
|
result.failed.push(unit(batchId));
|
|
1125
1541
|
return;
|
|
1126
1542
|
}
|
|
1543
|
+
// #677: the evicted member's commits never landed (eviction is the
|
|
1544
|
+
// pre-landing rail), so there is nothing to revert on the integration
|
|
1545
|
+
// branch — but its worktree/branch must go, or the next tick's prep for
|
|
1546
|
+
// the NEXT member collides with a stale tree. `teardownBatch` already
|
|
1547
|
+
// handled it on the dissolved path (the fields are cleared there).
|
|
1548
|
+
teardownMemberWorktree(deps, batchId, now, false);
|
|
1127
1549
|
advanceMemberOrValidate(deps, config, dispatch, batchId, batch.members.length, batch.executing_member, memberIssue, now, result);
|
|
1128
1550
|
}
|
|
1129
1551
|
/**
|
|
@@ -1160,7 +1582,11 @@ function evictMemberAndContinue(deps, config, dispatch, batchId, batch, memberIs
|
|
|
1160
1582
|
function runIncrementalGate(deps, config, dispatch, batchId, batch, memberIssue, now, result) {
|
|
1161
1583
|
if (batch.worktree === null || !deps.runCapability)
|
|
1162
1584
|
return false;
|
|
1163
|
-
|
|
1585
|
+
// #677: the gate judges the member's OWN work — pre-landing, its diff
|
|
1586
|
+
// exists only in the member worktree/branch. The shared batch worktree is
|
|
1587
|
+
// the fallback for a batch dispatched before member worktrees existed (its
|
|
1588
|
+
// members' commits landed directly on the integration branch).
|
|
1589
|
+
const worktree = batch.member_worktree ?? batch.worktree;
|
|
1164
1590
|
const runCapability = deps.runCapability;
|
|
1165
1591
|
const gateResults = ['typecheck.run', 'test.focused'].map((id) => ({
|
|
1166
1592
|
id,
|
|
@@ -1180,9 +1606,9 @@ function runIncrementalGate(deps, config, dispatch, batchId, batch, memberIssue,
|
|
|
1180
1606
|
// extend to an id nobody wrote down.
|
|
1181
1607
|
//
|
|
1182
1608
|
// `warmColdBatchWorktree` in this same file already draws exactly this
|
|
1183
|
-
// distinction for `worktree.prepare`, and
|
|
1184
|
-
//
|
|
1185
|
-
// treated it as fatal.
|
|
1609
|
+
// distinction for `worktree.prepare`, and the member workflow (member-cycle,
|
|
1610
|
+
// #677) runs its own relevance-scoped tests rather than treating a broken
|
|
1611
|
+
// capability as fatal. This was the one place that treated it as fatal.
|
|
1186
1612
|
//
|
|
1187
1613
|
// Skipping costs EARLY detection, not correctness: the aggregate `test.full`
|
|
1188
1614
|
// gate still runs before ship, CI still runs on the batch PR, and #562's
|
|
@@ -1281,14 +1707,19 @@ function runIncrementalGate(deps, config, dispatch, batchId, batch, memberIssue,
|
|
|
1281
1707
|
detail: withExcerpt(`cap run ${inconclusive.id} ${describeInconclusive(inconclusive)} after member review done`, excerpt),
|
|
1282
1708
|
});
|
|
1283
1709
|
deps.store.withLock((s) => ({ state: releaseSlot(s, batchId, now), result: undefined }));
|
|
1284
|
-
|
|
1285
|
-
|
|
1286
|
-
|
|
1287
|
-
|
|
1288
|
-
|
|
1289
|
-
|
|
1290
|
-
|
|
1291
|
-
|
|
1710
|
+
// #677: F.11's block semantics are "the member's commit stays on the
|
|
1711
|
+
// branch, nothing requeued/reverted" — under member worktrees that is
|
|
1712
|
+
// true only if the member LANDS before the batch blocks. The block means
|
|
1713
|
+
// the gate could not reach a verdict, not that the work is bad, so the
|
|
1714
|
+
// verified member joins the integration branch and is there for an
|
|
1715
|
+
// operator (or the #686 out-of-band reconcile) to act on. A landing
|
|
1716
|
+
// failure blocks the batch with its own reason — same operator outcome.
|
|
1717
|
+
const landed = landMemberBranch(deps, batchId, now);
|
|
1718
|
+
if (!landed.ok) {
|
|
1719
|
+
blockLandingFailure(deps, config, batch, landed.reason, now, result);
|
|
1720
|
+
return true;
|
|
1721
|
+
}
|
|
1722
|
+
blockBatchForOperator(deps, config, batchId, { reason, milestonePhase: 'batch-review' }, now, result);
|
|
1292
1723
|
return true;
|
|
1293
1724
|
}
|
|
1294
1725
|
return false;
|
|
@@ -1304,6 +1735,16 @@ function runIncrementalGate(deps, config, dispatch, batchId, batch, memberIssue,
|
|
|
1304
1735
|
* this is safe to call whether or not the caller already released it.
|
|
1305
1736
|
*/
|
|
1306
1737
|
function completeMemberGate(deps, config, dispatch, batchId, batch, memberIssue, now, result) {
|
|
1738
|
+
// #677 (§J.3): land the member's verified work on the integration branch
|
|
1739
|
+
// BEFORE anything reads it — the range recompute below runs `git log` on
|
|
1740
|
+
// the batch branch, which only sees the member's commits once landed. The
|
|
1741
|
+
// exec calls run OUTSIDE the lock (same invariant as the range recompute's
|
|
1742
|
+
// own comment below).
|
|
1743
|
+
const landed = landMemberBranch(deps, batchId, now);
|
|
1744
|
+
if (!landed.ok) {
|
|
1745
|
+
blockLandingFailure(deps, config, batch, landed.reason, now, result);
|
|
1746
|
+
return;
|
|
1747
|
+
}
|
|
1307
1748
|
// The commit-range recompute (`git log`) is a blocking subprocess call — it
|
|
1308
1749
|
// must run OUTSIDE the lock, like every other exec in this module; the
|
|
1309
1750
|
// result then lands as a pure data patch under the lock (Convention review:
|
|
@@ -1324,6 +1765,10 @@ function completeMemberGate(deps, config, dispatch, batchId, batch, memberIssue,
|
|
|
1324
1765
|
return { state: n, result: undefined };
|
|
1325
1766
|
});
|
|
1326
1767
|
result.completed.push(unit(batchId));
|
|
1768
|
+
// #677: the member's tree is done serving — landed, ranges recomputed.
|
|
1769
|
+
// Teardown before the advance so the next member's prep cannot collide
|
|
1770
|
+
// with this tree's cleanup.
|
|
1771
|
+
teardownMemberWorktree(deps, batchId, now, true);
|
|
1327
1772
|
advanceMemberOrValidate(deps, config, dispatch, batchId, batch.members.length, batch.executing_member, memberIssue, now, result);
|
|
1328
1773
|
}
|
|
1329
1774
|
/**
|
|
@@ -1360,6 +1805,12 @@ function resumeBlockedGate(deps, config, dispatch, batchId, now) {
|
|
|
1360
1805
|
if (!batch)
|
|
1361
1806
|
throw new types_1.SchedNotFoundError(`Batch not found: ${batchId}`);
|
|
1362
1807
|
if (batch.status !== 'blocked' || !batch.blocked_reason?.startsWith('gate-inconclusive:')) {
|
|
1808
|
+
// #677 review: the landing-failure blocks have no resume verb (same as
|
|
1809
|
+
// `suite-unreadable`) — name the actual reason and the operator path
|
|
1810
|
+
// instead of a bare illegal-transition error that reads like a bug.
|
|
1811
|
+
if (batch.status === 'blocked') {
|
|
1812
|
+
throw new types_1.SchedNotFoundError(`Batch ${batchId} is blocked on '${batch.blocked_reason ?? '?'}', which has no gate recheck: inspect the batch worktree/state named in its journal, then \`sched abandon --batch ${batchId}\` if it is a dead end`);
|
|
1813
|
+
}
|
|
1363
1814
|
throw new types_1.IllegalTransitionError('batch', batch.status, 'executing');
|
|
1364
1815
|
}
|
|
1365
1816
|
const capabilityId = batch.blocked_reason.slice('gate-inconclusive:'.length);
|
|
@@ -1367,7 +1818,13 @@ function resumeBlockedGate(deps, config, dispatch, batchId, now) {
|
|
|
1367
1818
|
if (batch.worktree === null || !deps.runCapability || memberIssue === undefined) {
|
|
1368
1819
|
throw new types_1.SchedNotFoundError(`Batch ${batchId} has no worktree/current member/gate hook to recheck`);
|
|
1369
1820
|
}
|
|
1370
|
-
const recheck = deps.runCapability(
|
|
1821
|
+
const recheck = deps.runCapability(
|
|
1822
|
+
// #677: recheck the member's OWN tree — pre-landing, the member's diff
|
|
1823
|
+
// exists only in its worktree; a batch-worktree recheck would test a
|
|
1824
|
+
// tree missing the change and trivially pass it. The fallback preserves
|
|
1825
|
+
// the pre-#677 behavior for a batch blocked before member worktrees
|
|
1826
|
+
// existed (its member's commits were already on the integration branch).
|
|
1827
|
+
batch.member_worktree ?? batch.worktree, capabilityId);
|
|
1371
1828
|
const result = emptyResult();
|
|
1372
1829
|
const excerpt = gateDetailExcerpt(recheck.outputTail, recheck.reason);
|
|
1373
1830
|
// #681: a recheck can classify what the batch blocked on as a TIMEOUT —
|
|
@@ -2048,6 +2505,232 @@ function reconcilePrWatch(deps, now, result) {
|
|
|
2048
2505
|
}
|
|
2049
2506
|
}
|
|
2050
2507
|
}
|
|
2508
|
+
// --- #686: stale-BLOCKED batch reconciliation (ground truth beats the ledger) ---
|
|
2509
|
+
/**
|
|
2510
|
+
* #686: how long after a batch `blocked` its merge evidence may still arrive
|
|
2511
|
+
* and reconcile it — measured from `updated_at`, which is when the batch
|
|
2512
|
+
* became blocked (dedup writes don't touch it). Same window semantics as the
|
|
2513
|
+
* per-issue path's `STALE_RECONCILE_WINDOW_MS` (engine.ts): past it, an
|
|
2514
|
+
* abandoned batch is left as-is rather than polled (one `gh pr view` plus git
|
|
2515
|
+
* probes) forever. A local constant rather than an import: engine.ts already
|
|
2516
|
+
* imports THIS module, so the arrow points one way only.
|
|
2517
|
+
*/
|
|
2518
|
+
const STALE_BLOCKED_RECONCILE_WINDOW_MS = 7 * 24 * 60 * 60 * 1000;
|
|
2519
|
+
/**
|
|
2520
|
+
* The forward rail a surviving member walks to `done` on a stale-blocked
|
|
2521
|
+
* reconcile — every step an ALREADY-LEGAL `ISSUE_BASE_TRANSITIONS` edge, so
|
|
2522
|
+
* the walk invents no new state-machine surface (AC3: a member whose commits
|
|
2523
|
+
* are in the base must not stay `in-work`, and `in-work → shipped-in-batch`
|
|
2524
|
+
* is not an edge — but `in-work → committed → validated → shipped-in-batch →
|
|
2525
|
+
* done` is). Members whose current status has no entry here (`waiting`, i.e.
|
|
2526
|
+
* never dispatched) are not walked: no recorded work, nothing shipped.
|
|
2527
|
+
*/
|
|
2528
|
+
const MEMBER_RECONCILE_CHAIN = {
|
|
2529
|
+
'in-work': 'committed',
|
|
2530
|
+
committed: 'validated',
|
|
2531
|
+
validated: 'shipped-in-batch',
|
|
2532
|
+
'shipped-in-batch': 'done',
|
|
2533
|
+
};
|
|
2534
|
+
/**
|
|
2535
|
+
* Whether every SURVIVING member of `batch` was actually dispatched — i.e. no
|
|
2536
|
+
* surviving member is still sitting in a pre-dispatch status (`queued`,
|
|
2537
|
+
* `classified`, `batched`, `waiting`). Surviving = a `batch.members` issue
|
|
2538
|
+
* that is still a slot-mode entry, not evicted (its work was reverted and it
|
|
2539
|
+
* requeued full-cycle), and not already preserved (shipped/terminal through
|
|
2540
|
+
* another rail).
|
|
2541
|
+
*
|
|
2542
|
+
* This is the safety rail behind AC1: a member that never ran has NO work on
|
|
2543
|
+
* the branch, so a batch carrying one is NOT reconcilable to shipped —
|
|
2544
|
+
* reconciling it would call an issue `done` that shipped nothing, minting the
|
|
2545
|
+
* exact ledger/ground-truth drift (#675-class) this issue exists to cure,
|
|
2546
|
+
* just with the signs flipped. A dispatched member, by contrast, put its work
|
|
2547
|
+
* ON the branch (even when the gate blocked before its range was recorded —
|
|
2548
|
+
* the b-20260909-01 shape, where the only surviving member sat `in-work`),
|
|
2549
|
+
* and the branch is exactly what the merge evidence below covers.
|
|
2550
|
+
*/
|
|
2551
|
+
function everySurvivingMemberDispatched(state, batch) {
|
|
2552
|
+
const evicted = new Set(batch.evictions.map((e) => e.issue));
|
|
2553
|
+
for (const issue of batch.members) {
|
|
2554
|
+
const entry = (0, state_1.findEntry)(state, issue);
|
|
2555
|
+
if (!entry)
|
|
2556
|
+
continue;
|
|
2557
|
+
if (entry.mode !== 'slot' || evicted.has(issue) || (0, state_1.isPreservedMember)(entry))
|
|
2558
|
+
continue;
|
|
2559
|
+
// Pre-dispatch statuses: queued/classified/batched/waiting. `in-work` is
|
|
2560
|
+
// stamped at SPAWN (`spawnMember`), `committed`/`validated` at verified
|
|
2561
|
+
// completion — everything the entry can reach afterwards means the
|
|
2562
|
+
// member was dispatched and its work is on the branch.
|
|
2563
|
+
if (!DISPATCHED_MEMBER_STATUSES.has(entry.status))
|
|
2564
|
+
return false;
|
|
2565
|
+
}
|
|
2566
|
+
return true;
|
|
2567
|
+
}
|
|
2568
|
+
/** Slot-line statuses that prove a member was actually dispatched (see above). */
|
|
2569
|
+
const DISPATCHED_MEMBER_STATUSES = new Set([
|
|
2570
|
+
'in-work',
|
|
2571
|
+
'committed',
|
|
2572
|
+
'validated',
|
|
2573
|
+
]);
|
|
2574
|
+
/**
|
|
2575
|
+
* Whether the batch branch has landed in the base — the b-20260909-01 shape
|
|
2576
|
+
* of merge evidence ("The batch branch is in `main` at c277ff59b", `pr=None`):
|
|
2577
|
+
* the branch was merged out of band, so its commits are IN the base even
|
|
2578
|
+
* though the batch never shipped a PR of its own. Two probes against the
|
|
2579
|
+
* remote-tracking ref (refreshed best-effort first — a merge that landed on
|
|
2580
|
+
* the remote is invisible to `rev-list` until fetched; fetch failure leaves
|
|
2581
|
+
* the stale ref, the same degradation the per-issue path tolerates):
|
|
2582
|
+
*
|
|
2583
|
+
* - `git rev-list --count origin/<base>..<branch>` is 0 — nothing of the
|
|
2584
|
+
* branch is missing from the base, i.e. every member commit on it is an
|
|
2585
|
+
* ancestor of the base (AC1's wording);
|
|
2586
|
+
* - the branch tip is NOT the base tip — the branch actually carried work.
|
|
2587
|
+
* A batch branch starts AT the base tip, so a branch that never received a
|
|
2588
|
+
* member commit still probes 0 on the range; requiring a different tip is
|
|
2589
|
+
* what separates "shipped" from "vacuously empty".
|
|
2590
|
+
*/
|
|
2591
|
+
function branchMergedIntoBase(deps, batch) {
|
|
2592
|
+
if (batch.worktree === null || batch.branch === null)
|
|
2593
|
+
return false;
|
|
2594
|
+
// CWE-88 (the repo's own trap index): both refs are interpolated into git
|
|
2595
|
+
// argv below. They are state values written by batch-setup, not remote
|
|
2596
|
+
// input, but `runBatchSetup` refuses to CREATE anything this pattern would
|
|
2597
|
+
// reject (SAFE_REF_RE at the branch-creation site) — hold the reader to
|
|
2598
|
+
// the same bar instead of trusting upstream state shape.
|
|
2599
|
+
if (!attribution_1.SAFE_REF_RE.test(batch.branch) || !attribution_1.SAFE_REF_RE.test(batch.base_branch))
|
|
2600
|
+
return false;
|
|
2601
|
+
const base = `origin/${batch.base_branch}`;
|
|
2602
|
+
deps.exec('git', ['fetch', 'origin', batch.base_branch], deps.repoDir);
|
|
2603
|
+
const ahead = deps.exec('git', ['rev-list', '--count', `${base}..${batch.branch}`], deps.repoDir);
|
|
2604
|
+
if ((ahead ?? '').trim() !== '0')
|
|
2605
|
+
return false;
|
|
2606
|
+
const branchTip = deps.exec('git', ['rev-parse', batch.branch], deps.repoDir);
|
|
2607
|
+
const baseTip = deps.exec('git', ['rev-parse', base], deps.repoDir);
|
|
2608
|
+
return branchTip !== null && baseTip !== null && branchTip.trim() !== '' && branchTip !== baseTip;
|
|
2609
|
+
}
|
|
2610
|
+
/**
|
|
2611
|
+
* #686: reconcile a `blocked` batch whose work demonstrably shipped anyway.
|
|
2612
|
+
* `reconcilePrWatch` above only watches `awaiting-merge` batches, and
|
|
2613
|
+
* `resumeBlockedGate` only re-runs the gate — so a batch blocked on a stale
|
|
2614
|
+
* verdict (the concrete case: `gate-inconclusive:test.focused`, itself often
|
|
2615
|
+
* a #690-declined gate TIMEOUT) whose PR merged out of band — or whose every
|
|
2616
|
+
* member commit is already in the base — stayed blocked FOREVER: no report,
|
|
2617
|
+
* no teardown, an operator-visible `blocked` row, and members stuck
|
|
2618
|
+
* `in-work`. Sibling of the per-issue `reconcileStaleFailedParks`
|
|
2619
|
+
* (engine.ts) under the same principle: ground truth beats the ledger, for
|
|
2620
|
+
* ANY stale verdict — the blocked_reason is deliberately not examined.
|
|
2621
|
+
*
|
|
2622
|
+
* Evidence (either suffices, cheapest first): the batch PR is MERGED with a
|
|
2623
|
+
* real timestamp (a merged batch PR contains the whole batch branch), or the
|
|
2624
|
+
* batch branch has fully landed in the base branch. A survivability guard
|
|
2625
|
+
* applies to BOTH paths (see `everySurvivingMemberDispatched`) — a batch with
|
|
2626
|
+
* a surviving member that was never dispatched is not reconcilable.
|
|
2627
|
+
*
|
|
2628
|
+
* On reconcile the batch joins the SAME rail a normally-merged batch takes:
|
|
2629
|
+
* `blocked → merged → deployed` (the `blocked → merged` edge exists for
|
|
2630
|
+
* exactly this caller), surviving members walked to `done` along legal
|
|
2631
|
+
* edges, and — when `batch.pr` is set — the ordinary `deployed` machinery
|
|
2632
|
+
* dispatches the report agent next tick, with teardown after `batch-report
|
|
2633
|
+
* done`. With `pr === null` no report agent can be prompted (it names the
|
|
2634
|
+
* PR), so the same lock continues `deployed → reported → done` and the
|
|
2635
|
+
* worktree is torn down here instead; the journal line says why no report
|
|
2636
|
+
* agent was dispatched rather than leaving the gap silent.
|
|
2637
|
+
*
|
|
2638
|
+
* Bounded by construction: only `blocked` batches within the window are
|
|
2639
|
+
* examined, the PR check is one `gh` call, git runs only when PR evidence
|
|
2640
|
+
* didn't already reconcile, and a reconciled batch leaves `blocked` — the
|
|
2641
|
+
* one journal line fires on the one-way transition, never per-tick (the
|
|
2642
|
+
* #632 lesson: no journal inside a loop that doesn't terminate the unit).
|
|
2643
|
+
*/
|
|
2644
|
+
function reconcileStaleBlockedBatches(deps, now, result) {
|
|
2645
|
+
const state = deps.store.load();
|
|
2646
|
+
const nowMs = now.getTime();
|
|
2647
|
+
for (const batch of state.batches) {
|
|
2648
|
+
if (batch.status !== 'blocked')
|
|
2649
|
+
continue;
|
|
2650
|
+
if (nowMs - Date.parse(batch.updated_at) >= STALE_BLOCKED_RECONCILE_WINDOW_MS)
|
|
2651
|
+
continue;
|
|
2652
|
+
if (!everySurvivingMemberDispatched(state, batch))
|
|
2653
|
+
continue;
|
|
2654
|
+
let evidence = null;
|
|
2655
|
+
if (batch.pr !== null) {
|
|
2656
|
+
const truth = deps.groundTruth.prState(batch.pr);
|
|
2657
|
+
if (truth !== undefined && truth.state === 'MERGED' && truth.mergedAt !== null) {
|
|
2658
|
+
evidence = { kind: 'pr-merged', pr: batch.pr, mergedAt: truth.mergedAt };
|
|
2659
|
+
}
|
|
2660
|
+
}
|
|
2661
|
+
if (evidence === null && branchMergedIntoBase(deps, batch)) {
|
|
2662
|
+
evidence = { kind: 'commits-in-base' };
|
|
2663
|
+
}
|
|
2664
|
+
if (evidence === null)
|
|
2665
|
+
continue;
|
|
2666
|
+
const blockedReason = batch.blocked_reason ?? 'blocked';
|
|
2667
|
+
const failedAt = batch.updated_at;
|
|
2668
|
+
let reconciled = false;
|
|
2669
|
+
let unwalkable = [];
|
|
2670
|
+
deps.store.withLock((s) => {
|
|
2671
|
+
const b = (0, state_1.findBatch)(s, batch.id);
|
|
2672
|
+
// Re-check under the lock: `sched resume --batch` (a passing recheck)
|
|
2673
|
+
// or `sched abandon` may have moved the batch out from under the
|
|
2674
|
+
// snapshot this pass started from.
|
|
2675
|
+
if (!b || b.status !== 'blocked')
|
|
2676
|
+
return { state: s, result: undefined };
|
|
2677
|
+
let n = (0, state_1.transitionBatch)(s, batch.id, 'merged', {}, now);
|
|
2678
|
+
n = (0, state_1.transitionBatch)(n, batch.id, 'deployed', {}, now);
|
|
2679
|
+
unwalkable = [];
|
|
2680
|
+
for (const issue of b.members) {
|
|
2681
|
+
let entry = (0, state_1.findEntry)(n, issue);
|
|
2682
|
+
if (!entry || (0, state_1.isPreservedMember)(entry))
|
|
2683
|
+
continue;
|
|
2684
|
+
let guard = 0;
|
|
2685
|
+
while (entry !== undefined) {
|
|
2686
|
+
const nextStatus = MEMBER_RECONCILE_CHAIN[entry.status];
|
|
2687
|
+
if (nextStatus === undefined || guard++ > 8)
|
|
2688
|
+
break; // chain is 4 long; never loop forever on a bad table
|
|
2689
|
+
try {
|
|
2690
|
+
n = (0, state_1.transitionIssue)(n, issue, nextStatus, {}, now);
|
|
2691
|
+
}
|
|
2692
|
+
catch {
|
|
2693
|
+
// Not reachable along legal edges from here — name it in the
|
|
2694
|
+
// journal rather than forcing an illegal transition.
|
|
2695
|
+
unwalkable.push(issue);
|
|
2696
|
+
break;
|
|
2697
|
+
}
|
|
2698
|
+
entry = (0, state_1.findEntry)(n, issue);
|
|
2699
|
+
}
|
|
2700
|
+
}
|
|
2701
|
+
if (b.pr === null) {
|
|
2702
|
+
// No PR → no report agent is possible (its prompt names the PR).
|
|
2703
|
+
// Finish the terminal rail and let teardown run below.
|
|
2704
|
+
n = (0, state_1.transitionBatch)(n, batch.id, 'reported', {}, now);
|
|
2705
|
+
n = (0, state_1.transitionBatch)(n, batch.id, 'done', {}, now);
|
|
2706
|
+
}
|
|
2707
|
+
reconciled = true;
|
|
2708
|
+
return { state: n, result: undefined };
|
|
2709
|
+
});
|
|
2710
|
+
if (!reconciled)
|
|
2711
|
+
continue;
|
|
2712
|
+
journalEvent(deps, 'stale-failure-reconciled', unit(batch.id), {
|
|
2713
|
+
pr: batch.pr ?? undefined,
|
|
2714
|
+
reason: blockedReason,
|
|
2715
|
+
failedAt,
|
|
2716
|
+
evidence: evidence.kind,
|
|
2717
|
+
...(evidence.kind === 'pr-merged' ? { mergedAt: evidence.mergedAt } : {}),
|
|
2718
|
+
detail: (evidence.kind === 'pr-merged'
|
|
2719
|
+
? `PR #${evidence.pr} is MERGED — ledger reconciled blocked (${blockedReason}) to the merged rail (blocked at ${failedAt}); report and teardown will now dispatch`
|
|
2720
|
+
: `every member commit is an ancestor of ${batch.base_branch} — ledger reconciled blocked (${blockedReason}) to shipped (blocked at ${failedAt})`) +
|
|
2721
|
+
(batch.pr === null
|
|
2722
|
+
? ' — no PR of its own, so no report agent can be prompted; teardown dispatched inline'
|
|
2723
|
+
: '') +
|
|
2724
|
+
(unwalkable.length > 0
|
|
2725
|
+
? `; members NOT walked (no legal edge): ${unwalkable.join(',')}`
|
|
2726
|
+
: ''),
|
|
2727
|
+
});
|
|
2728
|
+
result.mergeAccepted.push(unit(batch.id));
|
|
2729
|
+
if (batch.pr === null) {
|
|
2730
|
+
teardownBatch(deps, batch.id);
|
|
2731
|
+
}
|
|
2732
|
+
}
|
|
2733
|
+
}
|
|
2051
2734
|
/**
|
|
2052
2735
|
* Remove the batch's shared worktree — called both on the happy path
|
|
2053
2736
|
* (`reconcileReportSlot`, after `batch-report done`) and on every dissolve
|
|
@@ -2056,6 +2739,17 @@ function reconcilePrWatch(deps, now, result) {
|
|
|
2056
2739
|
* left on disk forever, since nothing else ever calls this for it.
|
|
2057
2740
|
*/
|
|
2058
2741
|
function teardownBatch(deps, batchId) {
|
|
2742
|
+
// #677: the current member's worktree goes first — it is batch-owned
|
|
2743
|
+
// state on the same teardown path as the shared tree, and a blocked or
|
|
2744
|
+
// dissolved batch must not leak a member tree (its branch/fields are
|
|
2745
|
+
// cleared by the helper; a gate-inconclusive BLOCK that intends to resume
|
|
2746
|
+
// never reaches this function). The batch is terminal here, so the
|
|
2747
|
+
// ranges-recorded membership of the last current member is a safe landed
|
|
2748
|
+
// signal (a range exists only for a member whose verified work landed).
|
|
2749
|
+
const terminal = (0, state_1.findBatch)(deps.store.load(), batchId);
|
|
2750
|
+
const lastMemberLanded = terminal?.ranges.some((r) => r.issue === terminal.members[terminal.executing_member - 1]) ===
|
|
2751
|
+
true;
|
|
2752
|
+
teardownMemberWorktree(deps, batchId, deps.now(), lastMemberLanded);
|
|
2059
2753
|
const state = deps.store.load();
|
|
2060
2754
|
const batch = (0, state_1.findBatch)(state, batchId);
|
|
2061
2755
|
if (!batch || batch.worktree === null)
|
|
@@ -2188,8 +2882,11 @@ function runBatchTick(deps, config, dispatch) {
|
|
|
2188
2882
|
// A prior spawn threw, or `claimAndSpawn` found zero free capacity —
|
|
2189
2883
|
// either way the batch is stuck mid-member with no slot and nothing
|
|
2190
2884
|
// else will ever retry it (Conformance review AC5 caveat; Supportability
|
|
2191
|
-
// review #12). Retrying every tick is
|
|
2192
|
-
// the capacity
|
|
2885
|
+
// review #12). Retrying every tick is cheap-by-contract (#677 review):
|
|
2886
|
+
// `spawnMemberContinuation` runs the free-capacity and member-identity
|
|
2887
|
+
// pre-checks BEFORE any pool/git work, so a full scheduler (or stale
|
|
2888
|
+
// member context) exits here without exec'ing, and `claimAndSpawn`
|
|
2889
|
+
// remains the authoritative capacity gate.
|
|
2193
2890
|
spawnMemberContinuation(deps, config, dispatch, batch.id, now, result);
|
|
2194
2891
|
}
|
|
2195
2892
|
else if (batch.status === 'reviewing' || batch.status === 'shipping') {
|
|
@@ -2219,6 +2916,7 @@ function runBatchTick(deps, config, dispatch) {
|
|
|
2219
2916
|
}
|
|
2220
2917
|
}
|
|
2221
2918
|
reconcilePrWatch(deps, now, result);
|
|
2919
|
+
reconcileStaleBlockedBatches(deps, now, result);
|
|
2222
2920
|
return result;
|
|
2223
2921
|
}
|
|
2224
2922
|
//# sourceMappingURL=batch-dispatch.js.map
|