@ai-dossier/sched 0.32.0 → 0.34.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +70 -15
- package/dist/batch-dispatch.d.ts +10 -1
- package/dist/batch-dispatch.d.ts.map +1 -1
- package/dist/batch-dispatch.js +570 -30
- package/dist/batch-dispatch.js.map +1 -1
- package/dist/dispatch.d.ts +54 -10
- package/dist/dispatch.d.ts.map +1 -1
- package/dist/dispatch.js +122 -16
- package/dist/dispatch.js.map +1 -1
- package/dist/engine.d.ts +3 -2
- package/dist/engine.d.ts.map +1 -1
- package/dist/engine.js +3 -2
- package/dist/engine.js.map +1 -1
- package/dist/enqueue.d.ts +12 -0
- package/dist/enqueue.d.ts.map +1 -1
- package/dist/enqueue.js +25 -2
- package/dist/enqueue.js.map +1 -1
- package/dist/groundtruth.d.ts +27 -15
- package/dist/groundtruth.d.ts.map +1 -1
- package/dist/groundtruth.js +43 -21
- package/dist/groundtruth.js.map +1 -1
- package/dist/index.d.ts +4 -4
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +9 -4
- package/dist/index.js.map +1 -1
- package/dist/persist.d.ts.map +1 -1
- package/dist/persist.js +59 -0
- package/dist/persist.js.map +1 -1
- package/dist/recovery.d.ts +2 -0
- package/dist/recovery.d.ts.map +1 -1
- package/dist/recovery.js +4 -1
- package/dist/recovery.js.map +1 -1
- package/dist/state.d.ts +9 -1
- package/dist/state.d.ts.map +1 -1
- package/dist/state.js +36 -0
- package/dist/state.js.map +1 -1
- package/dist/status.d.ts +6 -0
- package/dist/status.d.ts.map +1 -1
- package/dist/status.js +7 -1
- package/dist/status.js.map +1 -1
- package/dist/types.d.ts +101 -11
- package/dist/types.d.ts.map +1 -1
- package/dist/types.js +11 -2
- package/dist/types.js.map +1 -1
- package/package.json +1 -1
package/dist/batch-dispatch.js
CHANGED
|
@@ -95,6 +95,7 @@ var __importStar = (this && this.__importStar) || (function () {
|
|
|
95
95
|
};
|
|
96
96
|
})();
|
|
97
97
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
98
|
+
exports.memberBranchFor = memberBranchFor;
|
|
98
99
|
exports.evictMemberAndContinue = evictMemberAndContinue;
|
|
99
100
|
exports.resumeBlockedGate = resumeBlockedGate;
|
|
100
101
|
exports.runBatchTick = runBatchTick;
|
|
@@ -529,7 +530,312 @@ function runBatchSetup(deps, batch, now) {
|
|
|
529
530
|
}
|
|
530
531
|
return { ok: true, branch, worktree, runId: mintedRunId, poolClaimed: false };
|
|
531
532
|
}
|
|
532
|
-
|
|
533
|
+
// --- Member worktrees (#677, RFC-0001 §J.3) ---
|
|
534
|
+
/**
|
|
535
|
+
* The current member's own branch: `batch/<id>-m<n>-<issue>` (#677) — member
|
|
536
|
+
* index and issue both in the name, and deterministic from persisted state,
|
|
537
|
+
* so a takeover redispatch, a `sched resume --batch` recheck, and teardown
|
|
538
|
+
* all re-derive the same string without a lookup table. Distinct from the
|
|
539
|
+
* integration branch (`batch/<id>-<date>`): the member commits here, the
|
|
540
|
+
* scheduler lands it there.
|
|
541
|
+
*/
|
|
542
|
+
function memberBranchFor(batchId, memberIndex, issue) {
|
|
543
|
+
return `batch/${batchId}-m${memberIndex}-${issue}`;
|
|
544
|
+
}
|
|
545
|
+
/** The worktree path a cold member-worktree prep creates for one member. */
|
|
546
|
+
function memberWorktreePathFor(deps, batchId, memberIndex, issue) {
|
|
547
|
+
return path.join(deps.repoDir, 'worktrees', `batch-${batchId}-m${memberIndex}-${issue}`);
|
|
548
|
+
}
|
|
549
|
+
/**
|
|
550
|
+
* Prepare the CURRENT member's own worktree and branch (#677, AC5): a
|
|
551
|
+
* worktree whose checked-out branch is `memberBranchFor(...)` created OFF the
|
|
552
|
+
* integration branch, warm before the member's first command, and pushed so
|
|
553
|
+
* the member agent can commit and push to it — `member-cycle`'s Step 0
|
|
554
|
+
* preconditions (inside a worktree, on YOUR member branch, clean, env warm).
|
|
555
|
+
* The agent never creates either; a prompt change without this preparation
|
|
556
|
+
* fails every member on precondition 2 (the reason #677 exists).
|
|
557
|
+
*
|
|
558
|
+
* Pool claim first — warm by construction, mirroring `runBatchSetup`. A pool
|
|
559
|
+
* spare sits on the default branch, so the claimed tree is re-pointed at the
|
|
560
|
+
* member branch off the INTEGRATION branch (`git checkout -B` against the
|
|
561
|
+
* `origin/<integration>` ref `runBatchSetup`'s push already created
|
|
562
|
+
* repo-wide); checkout preserves the installed `node_modules`. Cold fallback:
|
|
563
|
+
* `git worktree add` at the integration branch tip + `warmColdBatchWorktree`
|
|
564
|
+
* (#561 — a cold member is an evicted member).
|
|
565
|
+
*
|
|
566
|
+
* An on-disk worktree for the SAME member is reused as-is (takeover
|
|
567
|
+
* redispatch, api-error hold retry): the branch is already checked out, the
|
|
568
|
+
* tree already warm, and a takeover agent resumes in place — possibly dirty,
|
|
569
|
+
* which is the parent's state to reason about, not something to reset.
|
|
570
|
+
*
|
|
571
|
+
* All-or-nothing like `runBatchSetup`; the exec calls run OUTSIDE any store
|
|
572
|
+
* lock (the caller's contract), with results landed as pure data.
|
|
573
|
+
*/
|
|
574
|
+
/**
|
|
575
|
+
* The pool CLI's stdout hardening shared by `runBatchSetup` and the member
|
|
576
|
+
* prep (no NUL/newline, absolute, resolved, on disk) — our own CLI's
|
|
577
|
+
* output, but garbage would otherwise be trusted as a `BatchEntry`
|
|
578
|
+
* worktree path and used as a spawn/exec cwd.
|
|
579
|
+
*/
|
|
580
|
+
function usablePoolClaim(deps, w) {
|
|
581
|
+
return (!!w &&
|
|
582
|
+
!w.includes('\0') &&
|
|
583
|
+
!w.includes('\n') &&
|
|
584
|
+
path.isAbsolute(w) &&
|
|
585
|
+
path.resolve(w) === w &&
|
|
586
|
+
(deps.fsExists ?? ((p) => fs.existsSync(p)))(w));
|
|
587
|
+
}
|
|
588
|
+
function prepareMemberWorktree(deps, batch, memberIndex, issue, now) {
|
|
589
|
+
if (batch.branch === null)
|
|
590
|
+
return { ok: false, reason: 'no-integration-branch' };
|
|
591
|
+
const branch = memberBranchFor(batch.id, memberIndex, issue);
|
|
592
|
+
const worktree = memberWorktreePathFor(deps, batch.id, memberIndex, issue);
|
|
593
|
+
if (!attribution_1.SAFE_REF_RE.test(branch))
|
|
594
|
+
return { ok: false, reason: 'invalid-member-branch-name' };
|
|
595
|
+
const fsExists = deps.fsExists ?? ((p) => fs.existsSync(p));
|
|
596
|
+
const root = deps.exec('git', ['rev-parse', '--show-toplevel'], deps.repoDir) ?? deps.repoDir;
|
|
597
|
+
if (!(0, teardown_1.isSafeWorktree)(path.resolve(root), worktree)) {
|
|
598
|
+
return { ok: false, reason: 'invalid-member-worktree-path' };
|
|
599
|
+
}
|
|
600
|
+
// All-or-nothing rollback for the cold path (same contract as
|
|
601
|
+
// `runBatchSetup`): a warm or push failure otherwise leaves a cold
|
|
602
|
+
// worktree the NEXT retry's exists-check reuses forever cold — exactly
|
|
603
|
+
// the env-cold eviction #561 removed.
|
|
604
|
+
const rollbackColdPrep = () => {
|
|
605
|
+
deps.exec('git', ['worktree', 'remove', '--force', '--', worktree], deps.repoDir);
|
|
606
|
+
deps.exec('git', ['branch', '-D', branch], deps.repoDir);
|
|
607
|
+
};
|
|
608
|
+
const exists = fsExists(worktree);
|
|
609
|
+
if (exists) {
|
|
610
|
+
// Crash-window reuse: the derived-path tree exists on disk but the
|
|
611
|
+
// spawn never landed (fields null) — OR a pool-claimed tree whose
|
|
612
|
+
// directory name equals the derived slug (the pool renames claims to
|
|
613
|
+
// the branch slug, so a pool tree CAN sit at the derived path; the
|
|
614
|
+
// persisted `member_pool_claimed` is the only record of which). In
|
|
615
|
+
// either case the on-disk tree wins: reuse it rather than re-branching.
|
|
616
|
+
// #632: fires once per crash window, not per tick — the call sites run
|
|
617
|
+
// at spawn time (a transition point), and `claimMemberResolution`
|
|
618
|
+
// gates the member advance; the capacity pre-check in
|
|
619
|
+
// `spawnMemberContinuation` keeps the wedge path from re-entering
|
|
620
|
+
// while no slot is free.
|
|
621
|
+
deps.journal.append((0, journal_1.unitEvent)('member-worktree-reused', unit(batch.id), {
|
|
622
|
+
issue,
|
|
623
|
+
detail: worktree,
|
|
624
|
+
}), now);
|
|
625
|
+
return {
|
|
626
|
+
ok: true,
|
|
627
|
+
branch: batch.member_branch ?? branch,
|
|
628
|
+
worktree: batch.member_worktree ?? worktree,
|
|
629
|
+
poolClaimed: batch.member_pool_claimed,
|
|
630
|
+
};
|
|
631
|
+
}
|
|
632
|
+
const claimed = deps.exec(teardown_1.POOL_BIN, [...teardown_1.POOL_ARGS_PREFIX, 'claim', '--issue', String(issue), '--branch', branch], deps.repoDir);
|
|
633
|
+
const claimedWorktree = claimed?.trim();
|
|
634
|
+
if (claimedWorktree && usablePoolClaim(deps, claimedWorktree)) {
|
|
635
|
+
// Re-point the warm spare at the member branch off the integration
|
|
636
|
+
// branch — `origin/<integration>` exists repo-wide (runBatchSetup pushed
|
|
637
|
+
// it), and `checkout -B` preserves the installed node_modules.
|
|
638
|
+
if (deps.exec('git', ['checkout', '-B', branch, `origin/${batch.branch}`], claimedWorktree) ===
|
|
639
|
+
null) {
|
|
640
|
+
deps.exec(teardown_1.POOL_BIN, [...teardown_1.POOL_ARGS_PREFIX, 'return', '--path', claimedWorktree, '--json'], deps.repoDir);
|
|
641
|
+
return { ok: false, reason: 'member-branch-checkout-failed' };
|
|
642
|
+
}
|
|
643
|
+
if (deps.exec('git', ['push', '-u', 'origin', '--', branch], claimedWorktree) === null) {
|
|
644
|
+
deps.exec(teardown_1.POOL_BIN, [...teardown_1.POOL_ARGS_PREFIX, 'return', '--path', claimedWorktree, '--json'], deps.repoDir);
|
|
645
|
+
return { ok: false, reason: 'member-branch-push-failed' };
|
|
646
|
+
}
|
|
647
|
+
return { ok: true, branch, worktree: claimedWorktree, poolClaimed: true };
|
|
648
|
+
}
|
|
649
|
+
const poolNote = claimedWorktree ? 'pool-claim-invalid:' : '';
|
|
650
|
+
const added = deps.exec('git', ['worktree', 'add', '-b', branch, worktree, batch.branch], deps.repoDir) !== null;
|
|
651
|
+
if (!added) {
|
|
652
|
+
// Self-heal the crash window a teardown died in (branch kept, tree
|
|
653
|
+
// gone): `worktree add -b` fails while the stale branch exists. The
|
|
654
|
+
// member branch is disposable by construction — a member whose branch
|
|
655
|
+
// still matters is CURRENT, and a current member's tree is reused by
|
|
656
|
+
// the exists-check above, never re-added. Force-delete the stale ref
|
|
657
|
+
// and retry the add once.
|
|
658
|
+
if (deps.exec('git', ['branch', '-D', branch], deps.repoDir) === null) {
|
|
659
|
+
return { ok: false, reason: 'member-worktree-add-failed' };
|
|
660
|
+
}
|
|
661
|
+
if (deps.exec('git', ['worktree', 'add', '-b', branch, worktree, batch.branch], deps.repoDir) === null) {
|
|
662
|
+
return { ok: false, reason: 'member-worktree-add-failed' };
|
|
663
|
+
}
|
|
664
|
+
}
|
|
665
|
+
const warmed = warmColdBatchWorktree(deps, batch, worktree, now, poolNote);
|
|
666
|
+
if (!warmed.ok) {
|
|
667
|
+
rollbackColdPrep();
|
|
668
|
+
return { ok: false, reason: warmed.reason };
|
|
669
|
+
}
|
|
670
|
+
if (deps.exec('git', ['push', '-u', 'origin', '--', branch], worktree) === null) {
|
|
671
|
+
rollbackColdPrep();
|
|
672
|
+
return { ok: false, reason: 'member-branch-push-failed' };
|
|
673
|
+
}
|
|
674
|
+
return { ok: true, branch, worktree, poolClaimed: false };
|
|
675
|
+
}
|
|
676
|
+
/**
|
|
677
|
+
* Land the CURRENT member's work on the integration branch (#677, §J.3: "a
|
|
678
|
+
* member lands its commit onto the integration branch when its own
|
|
679
|
+
* verification passes"): `git merge --ff-only <member_branch>` in the shared
|
|
680
|
+
* batch worktree (which holds the integration branch), then push.
|
|
681
|
+
*
|
|
682
|
+
* Fast-forward is always available under the SERIAL dispatch model — only
|
|
683
|
+
* the member moved its branch since it was cut, and nothing else moves the
|
|
684
|
+
* integration branch mid-`executing` (fix agents only run from the
|
|
685
|
+
* post-last-member aggregate validation). A failed merge therefore means the
|
|
686
|
+
* serial invariant broke, not a conflict to resolve: the batch BLOCKS for an
|
|
687
|
+
* operator instead of silently merging or reverting. Linear history is also
|
|
688
|
+
* what `boundaryCommits`/`memberRanges` attribution expects — the member's
|
|
689
|
+
* `(#<issue>)`-trailed subjects stay first-parent-readable after the landing.
|
|
690
|
+
*/
|
|
691
|
+
function landMemberBranch(deps, batchId, now) {
|
|
692
|
+
const state = deps.store.load();
|
|
693
|
+
const batch = (0, state_1.findBatch)(state, batchId);
|
|
694
|
+
if (!batch || batch.worktree === null || batch.branch === null || batch.member_branch === null) {
|
|
695
|
+
return { ok: true };
|
|
696
|
+
}
|
|
697
|
+
const memberIssue = batch.members[batch.executing_member - 1];
|
|
698
|
+
// #677 review (defense in depth, matching `branchMergedIntoBase`'s bar):
|
|
699
|
+
// the persisted ref reaches git argv below — re-validate its shape.
|
|
700
|
+
if (!attribution_1.SAFE_REF_RE.test(batch.member_branch) || !attribution_1.SAFE_REF_RE.test(batch.branch)) {
|
|
701
|
+
journalEvent(deps, 'landing-failed', unit(batchId), {
|
|
702
|
+
issue: memberIssue,
|
|
703
|
+
reason: 'invalid-branch-name',
|
|
704
|
+
detail: `persisted member/integration branch failed SAFE_REF_RE: ${batch.member_branch} → ${batch.branch}`,
|
|
705
|
+
});
|
|
706
|
+
return { ok: false, reason: 'invalid-branch-name' };
|
|
707
|
+
}
|
|
708
|
+
if (deps.exec('git', ['merge', '--ff-only', batch.member_branch], batch.worktree) === null) {
|
|
709
|
+
journalEvent(deps, 'landing-failed', unit(batchId), {
|
|
710
|
+
issue: memberIssue,
|
|
711
|
+
reason: 'landing-merge-failed',
|
|
712
|
+
detail: `git merge --ff-only ${batch.member_branch} into ${batch.branch} failed — serial-landing invariant broken; blocking for an operator`,
|
|
713
|
+
});
|
|
714
|
+
return { ok: false, reason: 'landing-merge-failed' };
|
|
715
|
+
}
|
|
716
|
+
if (deps.exec('git', ['push', 'origin', '--', batch.branch], batch.worktree) === null) {
|
|
717
|
+
journalEvent(deps, 'landing-failed', unit(batchId), {
|
|
718
|
+
issue: memberIssue,
|
|
719
|
+
reason: 'landing-push-failed',
|
|
720
|
+
detail: `landed ${batch.member_branch} on ${batch.branch} locally but the push failed — blocking so origin stays the durable copy`,
|
|
721
|
+
});
|
|
722
|
+
return { ok: false, reason: 'landing-push-failed' };
|
|
723
|
+
}
|
|
724
|
+
deps.journal.append((0, journal_1.unitEvent)('member-landed', unit(batchId), {
|
|
725
|
+
issue: memberIssue,
|
|
726
|
+
detail: `${batch.member_branch} fast-forwarded onto ${batch.branch}`,
|
|
727
|
+
}), now);
|
|
728
|
+
return { ok: true };
|
|
729
|
+
}
|
|
730
|
+
/**
|
|
731
|
+
* Tear the CURRENT member's worktree down and clear the member fields
|
|
732
|
+
* (#677): pool-claimed trees are RETURNED to the pool (a raw remove leaves a
|
|
733
|
+
* dangling pool entry — the corrupted-entry state `worktree-pool status`
|
|
734
|
+
* reports), cold trees are removed. The member branch is deleted locally —
|
|
735
|
+
* `-d` after a landing (fully merged into the integration branch), `-D` on
|
|
736
|
+
* the eviction path (its commits never landed; the requeue carries the work
|
|
737
|
+
* forward). The REMOTE member branch is deleted only on the landed path, and
|
|
738
|
+
* deliberately KEPT on eviction — the pushed sha is the evicted work's only
|
|
739
|
+
* recoverable copy.
|
|
740
|
+
*
|
|
741
|
+
* Best-effort and idempotent: null fields are a no-op, and a failed cleanup
|
|
742
|
+
* journals `teardown-failed` rather than throwing into the caller's
|
|
743
|
+
* advance/dissolve path.
|
|
744
|
+
*/
|
|
745
|
+
function teardownMemberWorktree(deps, batchId, now,
|
|
746
|
+
/** Whether the member's branch was ff-landed onto the integration branch. Passed explicitly by the callers that KNOW — deriving it from `batch.ranges` would silently force-delete a merged branch if a caller ever reached teardown after the member pointer advanced. */
|
|
747
|
+
landed) {
|
|
748
|
+
const state = deps.store.load();
|
|
749
|
+
const batch = (0, state_1.findBatch)(state, batchId);
|
|
750
|
+
if (!batch || batch.member_worktree === null)
|
|
751
|
+
return;
|
|
752
|
+
const worktree = batch.member_worktree;
|
|
753
|
+
const branch = batch.member_branch;
|
|
754
|
+
// #677 review (defense in depth, matching `branchMergedIntoBase`'s bar):
|
|
755
|
+
// persisted state reaches git argv here — re-validate the ref shape before
|
|
756
|
+
// interpolating it.
|
|
757
|
+
if (branch !== null && !attribution_1.SAFE_REF_RE.test(branch)) {
|
|
758
|
+
journalEvent(deps, 'teardown-failed', unit(batchId), {
|
|
759
|
+
cleanup: 'failed-invalid-branch-name',
|
|
760
|
+
detail: branch,
|
|
761
|
+
});
|
|
762
|
+
return;
|
|
763
|
+
}
|
|
764
|
+
// Route through `runTeardown` for the same guarantees `teardownBatch`
|
|
765
|
+
// gets: pool returns are verify-first idempotent and refuse to claim
|
|
766
|
+
// success unless the pool's own self-check reports the entry `warm`;
|
|
767
|
+
// cold removes post-verify the path is gone and unlisted. Nulls first —
|
|
768
|
+
// clear the fields BEFORE the exec so a crash mid-teardown doesn't retry
|
|
769
|
+
// into a half-cleaned tree (the re-derived path is deterministic if a
|
|
770
|
+
// leftover does survive).
|
|
771
|
+
deps.store.withLock((s) => {
|
|
772
|
+
const b = (0, state_1.findBatch)(s, batchId);
|
|
773
|
+
if (!b)
|
|
774
|
+
return { state: s, result: undefined };
|
|
775
|
+
return {
|
|
776
|
+
state: (0, state_1.patchBatch)(s, batchId, { member_branch: null, member_worktree: null, member_pool_claimed: false }, now),
|
|
777
|
+
result: undefined,
|
|
778
|
+
};
|
|
779
|
+
});
|
|
780
|
+
const t = (0, teardown_1.runTeardown)(deps.exec, deps.repoDir, { worktree, poolClaimed: batch.member_pool_claimed === true, branch }, deps.fsExists);
|
|
781
|
+
const treeCleared = !t.cleanup.startsWith('failed');
|
|
782
|
+
let branchCleanup = 'skipped';
|
|
783
|
+
if (branch !== null && treeCleared) {
|
|
784
|
+
// `-d` refuses an unmerged branch; the eviction path forces it
|
|
785
|
+
// deliberately (see doc). Run from repoDir — the worktree is gone.
|
|
786
|
+
const flag = landed ? '-d' : '-D';
|
|
787
|
+
branchCleanup =
|
|
788
|
+
deps.exec('git', ['branch', flag, branch], deps.repoDir) === null
|
|
789
|
+
? `failed-branch-delete-${flag}`
|
|
790
|
+
: `branch-deleted-${flag}`;
|
|
791
|
+
if (landed) {
|
|
792
|
+
deps.exec('git', ['push', 'origin', '--delete', branch], deps.repoDir);
|
|
793
|
+
}
|
|
794
|
+
}
|
|
795
|
+
const failed = !treeCleared || branchCleanup.startsWith('failed');
|
|
796
|
+
journalEvent(deps, failed ? 'teardown-failed' : 'member-worktree-torn-down', unit(batchId), {
|
|
797
|
+
cleanup: t.cleanup,
|
|
798
|
+
branch_cleanup: branchCleanup,
|
|
799
|
+
detail: t.detail,
|
|
800
|
+
worktree,
|
|
801
|
+
});
|
|
802
|
+
}
|
|
803
|
+
/**
|
|
804
|
+
* The shared "block the batch for an operator" tail (#677 review — the same
|
|
805
|
+
* load → `recoveryDeps` → `blockBatch` → re-apply sequence
|
|
806
|
+
* `runValidate`'s suite-unreadable path and `runIncrementalGate`'s
|
|
807
|
+
* inconclusive path each grew): releases nothing itself — callers release
|
|
808
|
+
* their own slot first — and always reports the unit under `failed`.
|
|
809
|
+
*/
|
|
810
|
+
function blockBatchForOperator(deps, config, batchId, opts, now, result) {
|
|
811
|
+
const stateNow = deps.store.load();
|
|
812
|
+
const fresh = (0, state_1.findBatch)(stateNow, batchId);
|
|
813
|
+
if (!fresh)
|
|
814
|
+
return;
|
|
815
|
+
const blocked = (0, recovery_1.blockBatch)(stateNow, batchId, opts, recoveryDeps(deps, config, fresh, now));
|
|
816
|
+
deps.store.withLock((s) => ({
|
|
817
|
+
state: applyBatchAndIssues(s, blocked.state, batchId, []),
|
|
818
|
+
result: undefined,
|
|
819
|
+
}));
|
|
820
|
+
result.failed.push(unit(batchId));
|
|
821
|
+
}
|
|
822
|
+
/**
|
|
823
|
+
* Block the batch on a mechanical landing failure (#677): release the slot,
|
|
824
|
+
* persist `blocked_reason`, and journal — the same operator-inspection path
|
|
825
|
+
* `suite-unreadable` uses. The member's work is safe (its branch is pushed);
|
|
826
|
+
* only the integration step failed.
|
|
827
|
+
*/
|
|
828
|
+
function blockLandingFailure(deps, config, batch, reason, now, result) {
|
|
829
|
+
deps.store.withLock((s) => ({ state: releaseSlot(s, batch.id, now), result: undefined }));
|
|
830
|
+
blockBatchForOperator(deps, config, batch.id, { reason }, now, result);
|
|
831
|
+
}
|
|
832
|
+
/**
|
|
833
|
+
* Spawn one batch member's member-cycle agent (#677) into the slot a prior
|
|
834
|
+
* member (or batch-setup) just released — each member in its OWN worktree
|
|
835
|
+
* off the integration branch, dispatched serially as before. Prepared by
|
|
836
|
+
* {@link prepareMemberWorktree} at the spawn site (outside the lock); this
|
|
837
|
+
* function only patches state and spawns.
|
|
838
|
+
*/
|
|
533
839
|
/**
|
|
534
840
|
* Drive a member's `QueueEntry` through the D.1 slot-line states it must pass
|
|
535
841
|
* through before `shipped-in-batch` becomes a legal edge (`validated` is the
|
|
@@ -567,15 +873,15 @@ function advanceMemberToValidated(state, memberIssue, now) {
|
|
|
567
873
|
}
|
|
568
874
|
return next;
|
|
569
875
|
}
|
|
570
|
-
function spawnMember(deps, dispatch, state, slot, batchId, now, result) {
|
|
876
|
+
function spawnMember(deps, dispatch, state, slot, batchId, now, result, member) {
|
|
571
877
|
const batch = (0, state_1.findBatch)(state, batchId);
|
|
572
|
-
if (!batch || batch.worktree === null) {
|
|
878
|
+
if (!batch || batch.worktree === null || batch.branch === null) {
|
|
573
879
|
// A leaked `assigned` slot with `pid: null` is invisible to `dead`
|
|
574
880
|
// detection (nothing ever kills/reclaims it) — release it here rather
|
|
575
881
|
// than leaving the batch permanently down one slot of capacity.
|
|
576
882
|
journalEvent(deps, 'unit-failed', unit(batchId), {
|
|
577
883
|
reason: 'no-worktree',
|
|
578
|
-
detail: 'spawnMember: batch has no worktree — batch-setup has not landed',
|
|
884
|
+
detail: 'spawnMember: batch has no worktree/branch — batch-setup has not landed',
|
|
579
885
|
});
|
|
580
886
|
return releaseSlot(state, batchId, now);
|
|
581
887
|
}
|
|
@@ -591,7 +897,10 @@ function spawnMember(deps, dispatch, state, slot, batchId, now, result) {
|
|
|
591
897
|
const tier = (0, state_1.findEntry)(withStatus, memberIssue)?.tier ?? 'mid';
|
|
592
898
|
const spawnSpec = (0, dispatch_1.resolveTierSpawn)(dispatch, tier, memberIssue);
|
|
593
899
|
const cmd = spawnSpec.cmd;
|
|
594
|
-
|
|
900
|
+
// #677: the prompt carries THIS member's own worktree and the integration
|
|
901
|
+
// branch it was cut from — `member-cycle`'s `worktree`/`integration_branch`
|
|
902
|
+
// inputs. The shared batch worktree is no longer a member surface at all.
|
|
903
|
+
const prompt = (0, dispatch_1.buildMemberPrompt)(dispatch.memberPrompt, memberIssue, batchId, member.worktree, batch.branch);
|
|
595
904
|
const logFile = (0, dispatch_1.batchMemberLogPath)(deps.store.runsDir, batchId, batch.executing_member, memberIssue);
|
|
596
905
|
// #629: captured BEFORE spawning, mirroring `engine.ts`'s own
|
|
597
906
|
// `spawnAndRecord` — the log is per-role and append-mode, so the size at
|
|
@@ -622,9 +931,18 @@ function spawnMember(deps, dispatch, state, slot, batchId, now, result) {
|
|
|
622
931
|
spawned_at: now.toISOString(),
|
|
623
932
|
log_offset_at_spawn: logOffset,
|
|
624
933
|
};
|
|
625
|
-
|
|
934
|
+
let next = slot.status === 'assigned' || slot.status === 'recovering'
|
|
626
935
|
? (0, state_1.transitionSlot)(withStatus, slot.id, 'running', patch, now)
|
|
627
936
|
: withStatus;
|
|
937
|
+
// #677: persist the member's worktree context — a takeover redispatch or a
|
|
938
|
+
// `sched resume --batch` recheck after an engine restart must land in the
|
|
939
|
+
// SAME worktree/branch this prompt names, not re-derive (or worse,
|
|
940
|
+
// re-create) it.
|
|
941
|
+
next = (0, state_1.patchBatch)(next, batchId, {
|
|
942
|
+
member_branch: member.branch,
|
|
943
|
+
member_worktree: member.worktree,
|
|
944
|
+
member_pool_claimed: member.poolClaimed,
|
|
945
|
+
}, now);
|
|
628
946
|
deps.journal.append((0, journal_1.unitEvent)('spawned', unit(batchId), {
|
|
629
947
|
pid,
|
|
630
948
|
tier,
|
|
@@ -632,6 +950,7 @@ function spawnMember(deps, dispatch, state, slot, batchId, now, result) {
|
|
|
632
950
|
issue: memberIssue,
|
|
633
951
|
...(0, dispatch_1.journalCmdModelFields)(spawnSpec),
|
|
634
952
|
log: logFile,
|
|
953
|
+
worktree: member.worktree,
|
|
635
954
|
detail: `member ${batch.executing_member}/${batch.members.length}`,
|
|
636
955
|
}), now);
|
|
637
956
|
result.spawned.push(unit(batchId));
|
|
@@ -703,6 +1022,45 @@ function claimAndSetup(deps, config, dispatch, batchId, now, result) {
|
|
|
703
1022
|
},
|
|
704
1023
|
});
|
|
705
1024
|
deps.journal.append((0, journal_1.unitEvent)('batch-setup-done', unit(batchId), { detail: setup.worktree }), now);
|
|
1025
|
+
// #677: prepare member 1's OWN worktree before claiming the spawn — real
|
|
1026
|
+
// git/network work, so OUTSIDE the store lock (the same contract
|
|
1027
|
+
// `runBatchSetup` just followed). All-or-nothing: a failed prep posts the
|
|
1028
|
+
// blocked milestone and releases the slot rather than spawning into a
|
|
1029
|
+
// worktree that does not exist (every member would fail Step 0).
|
|
1030
|
+
const firstIssue = batch.members[0];
|
|
1031
|
+
// Shared abort tail of every setup-stage failure (#677 review): post the
|
|
1032
|
+
// blocked milestone when a run id exists, journal, report, release.
|
|
1033
|
+
const abortSetup = (kvReason, detail) => {
|
|
1034
|
+
if (batch.anchor !== null && setup.runId !== null) {
|
|
1035
|
+
poster(batch.anchor, setup.runId, {
|
|
1036
|
+
phase: 'batch-setup',
|
|
1037
|
+
status: 'blocked',
|
|
1038
|
+
kv: { reason: kvReason },
|
|
1039
|
+
});
|
|
1040
|
+
}
|
|
1041
|
+
else {
|
|
1042
|
+
journalEvent(deps, 'milestone-post-failed', unit(batchId), {
|
|
1043
|
+
detail: `batch-setup blocked (${kvReason}) — no run id to post to yet`,
|
|
1044
|
+
});
|
|
1045
|
+
}
|
|
1046
|
+
deps.journal.append((0, journal_1.unitEvent)('batch-setup-failed', unit(batchId), { detail }), now);
|
|
1047
|
+
result.failed.push(unit(batchId));
|
|
1048
|
+
deps.store.withLock((s) => ({ state: releaseSlot(s, batchId, now), result: undefined }));
|
|
1049
|
+
};
|
|
1050
|
+
if (firstIssue === undefined) {
|
|
1051
|
+
abortSetup('no-member', 'claimAndSetup: batch has no members');
|
|
1052
|
+
return;
|
|
1053
|
+
}
|
|
1054
|
+
const freshForPrep = (0, state_1.findBatch)(deps.store.load(), batchId);
|
|
1055
|
+
const memberPrep = prepareMemberWorktree(deps,
|
|
1056
|
+
// The setup patch (branch/worktree/run_id) lands under the lock BELOW —
|
|
1057
|
+
// prep needs the integration branch, so overlay `setup.branch` on the
|
|
1058
|
+
// fresh state rather than waiting for the patch.
|
|
1059
|
+
{ ...(freshForPrep ?? batch), branch: setup.branch }, 1, firstIssue, now);
|
|
1060
|
+
if (!memberPrep.ok) {
|
|
1061
|
+
abortSetup(`member-worktree-prep-failed:${memberPrep.reason}`, `member worktree prep failed: ${memberPrep.reason} (member 1 #${firstIssue}, branch ${memberBranchFor(batchId, 1, firstIssue)})`);
|
|
1062
|
+
return;
|
|
1063
|
+
}
|
|
706
1064
|
deps.store.withLock((s) => {
|
|
707
1065
|
const b = (0, state_1.findBatch)(s, batchId);
|
|
708
1066
|
if (!b || b.status !== 'ready') {
|
|
@@ -727,7 +1085,7 @@ function claimAndSetup(deps, config, dispatch, batchId, now, result) {
|
|
|
727
1085
|
next = (0, state_1.transitionBatch)(next, batchId, 'executing', { executing_member: 1 }, now);
|
|
728
1086
|
const slot = slotFor(next, batchId);
|
|
729
1087
|
if (slot)
|
|
730
|
-
next = spawnMember(deps, dispatch, next, slot, batchId, now, result);
|
|
1088
|
+
next = spawnMember(deps, dispatch, next, slot, batchId, now, result, memberPrep);
|
|
731
1089
|
return { state: next, result: undefined };
|
|
732
1090
|
});
|
|
733
1091
|
}
|
|
@@ -753,7 +1111,65 @@ function claimAndSpawn(deps, config, batchId, phase, now, spawn) {
|
|
|
753
1111
|
});
|
|
754
1112
|
}
|
|
755
1113
|
function spawnMemberContinuation(deps, config, dispatch, batchId, now, result) {
|
|
756
|
-
|
|
1114
|
+
// #677: make sure the current member's worktree exists before claiming a
|
|
1115
|
+
// slot for the redispatch — git exec work, so OUTSIDE the store lock (the
|
|
1116
|
+
// contract `prepareMemberWorktree` documents). The wedge arm of
|
|
1117
|
+
// `runBatchTick` calls this EVERY tick while the batch is `executing`
|
|
1118
|
+
// with no slot, so the cheap gates run first:
|
|
1119
|
+
// - capacity: prep before the free-capacity check would do pool/git/npm
|
|
1120
|
+
// work (and journal) per tick against a full scheduler — the #610/#630/
|
|
1121
|
+
// #632 per-tick emission trap. `claimAndSpawn` re-checks under the lock;
|
|
1122
|
+
// this pre-check only avoids wasted prep, it is not the gate.
|
|
1123
|
+
// - identity: persisted member fields are reused only when they belong to
|
|
1124
|
+
// the CURRENT member (they can be stale after a failed teardown that
|
|
1125
|
+
// kept them, or a crash between an eviction and its teardown) and the
|
|
1126
|
+
// tree still EXISTS on disk (state-first, because a pool-claimed path
|
|
1127
|
+
// is not re-derivable from batchId+index+issue).
|
|
1128
|
+
const state = deps.store.load();
|
|
1129
|
+
const batch = (0, state_1.findBatch)(state, batchId);
|
|
1130
|
+
if (!batch || batch.status !== 'executing' || batch.worktree === null)
|
|
1131
|
+
return;
|
|
1132
|
+
const memberIssue = batch.members[batch.executing_member - 1];
|
|
1133
|
+
if (memberIssue === undefined)
|
|
1134
|
+
return;
|
|
1135
|
+
if ((0, scheduler_1.freeCapacity)(state, config) === 0)
|
|
1136
|
+
return;
|
|
1137
|
+
const expectedBranch = memberBranchFor(batchId, batch.executing_member, memberIssue);
|
|
1138
|
+
const fsExists = deps.fsExists ?? ((p) => fs.existsSync(p));
|
|
1139
|
+
const reusable = batch.member_worktree !== null &&
|
|
1140
|
+
batch.member_branch === expectedBranch &&
|
|
1141
|
+
fsExists(batch.member_worktree);
|
|
1142
|
+
let member;
|
|
1143
|
+
if (reusable) {
|
|
1144
|
+
member = {
|
|
1145
|
+
branch: batch.member_branch,
|
|
1146
|
+
worktree: batch.member_worktree,
|
|
1147
|
+
poolClaimed: batch.member_pool_claimed,
|
|
1148
|
+
};
|
|
1149
|
+
}
|
|
1150
|
+
else {
|
|
1151
|
+
if (batch.member_worktree !== null) {
|
|
1152
|
+
// Stale context for a DIFFERENT member (or a vanished tree): journal
|
|
1153
|
+
// it before overwriting — the disposition of that tree is the
|
|
1154
|
+
// operator's to inspect if it was real.
|
|
1155
|
+
journalEvent(deps, 'stale-member-worktree', unit(batchId), {
|
|
1156
|
+
issue: memberIssue,
|
|
1157
|
+
detail: `persisted ${batch.member_branch ?? '?'} @ ${batch.member_worktree} does not match expected ${expectedBranch} (or tree gone) — preparing fresh`,
|
|
1158
|
+
});
|
|
1159
|
+
}
|
|
1160
|
+
const prep = prepareMemberWorktree(deps, batch, batch.executing_member, memberIssue, now);
|
|
1161
|
+
if (!prep.ok) {
|
|
1162
|
+
journalEvent(deps, 'unit-failed', unit(batchId), {
|
|
1163
|
+
issue: memberIssue,
|
|
1164
|
+
reason: 'member-worktree-prep-failed',
|
|
1165
|
+
detail: `member worktree prep failed on continuation: ${prep.reason} (branch ${expectedBranch})`,
|
|
1166
|
+
});
|
|
1167
|
+
result.failed.push(unit(batchId));
|
|
1168
|
+
return;
|
|
1169
|
+
}
|
|
1170
|
+
member = prep;
|
|
1171
|
+
}
|
|
1172
|
+
claimAndSpawn(deps, config, batchId, 'member', now, (nextState, slot) => spawnMember(deps, dispatch, nextState, slot, batchId, now, result, member));
|
|
757
1173
|
}
|
|
758
1174
|
/**
|
|
759
1175
|
* Tail/report/fix dispatches (this function, `spawnReportAgent`,
|
|
@@ -958,7 +1374,10 @@ function runValidate(deps, config, dispatch, batchId, now, result) {
|
|
|
958
1374
|
return;
|
|
959
1375
|
}
|
|
960
1376
|
const offender = outcome.offenders[0];
|
|
961
|
-
const { state: fixing, dispatch: fixDispatch } = (0, recovery_1.beginFixAttempt)(attributed, batchId, offender, rDeps,
|
|
1377
|
+
const { state: fixing, dispatch: fixDispatch } = (0, recovery_1.beginFixAttempt)(attributed, batchId, offender, rDeps,
|
|
1378
|
+
// #707: `dispatch` is the batch's own (profile-resolved) dispatch — the
|
|
1379
|
+
// fix agent rides the same family the members ran on.
|
|
1380
|
+
{ config, tests: outcome.attributed.get(offender) ?? [], dispatch });
|
|
962
1381
|
deps.store.withLock((s) => ({
|
|
963
1382
|
state: applyBatchAndIssues(s, fixing, batchId, []),
|
|
964
1383
|
result: undefined,
|
|
@@ -1124,6 +1543,12 @@ function evictMemberAndContinue(deps, config, dispatch, batchId, batch, memberIs
|
|
|
1124
1543
|
result.failed.push(unit(batchId));
|
|
1125
1544
|
return;
|
|
1126
1545
|
}
|
|
1546
|
+
// #677: the evicted member's commits never landed (eviction is the
|
|
1547
|
+
// pre-landing rail), so there is nothing to revert on the integration
|
|
1548
|
+
// branch — but its worktree/branch must go, or the next tick's prep for
|
|
1549
|
+
// the NEXT member collides with a stale tree. `teardownBatch` already
|
|
1550
|
+
// handled it on the dissolved path (the fields are cleared there).
|
|
1551
|
+
teardownMemberWorktree(deps, batchId, now, false);
|
|
1127
1552
|
advanceMemberOrValidate(deps, config, dispatch, batchId, batch.members.length, batch.executing_member, memberIssue, now, result);
|
|
1128
1553
|
}
|
|
1129
1554
|
/**
|
|
@@ -1160,7 +1585,11 @@ function evictMemberAndContinue(deps, config, dispatch, batchId, batch, memberIs
|
|
|
1160
1585
|
function runIncrementalGate(deps, config, dispatch, batchId, batch, memberIssue, now, result) {
|
|
1161
1586
|
if (batch.worktree === null || !deps.runCapability)
|
|
1162
1587
|
return false;
|
|
1163
|
-
|
|
1588
|
+
// #677: the gate judges the member's OWN work — pre-landing, its diff
|
|
1589
|
+
// exists only in the member worktree/branch. The shared batch worktree is
|
|
1590
|
+
// the fallback for a batch dispatched before member worktrees existed (its
|
|
1591
|
+
// members' commits landed directly on the integration branch).
|
|
1592
|
+
const worktree = batch.member_worktree ?? batch.worktree;
|
|
1164
1593
|
const runCapability = deps.runCapability;
|
|
1165
1594
|
const gateResults = ['typecheck.run', 'test.focused'].map((id) => ({
|
|
1166
1595
|
id,
|
|
@@ -1180,9 +1609,9 @@ function runIncrementalGate(deps, config, dispatch, batchId, batch, memberIssue,
|
|
|
1180
1609
|
// extend to an id nobody wrote down.
|
|
1181
1610
|
//
|
|
1182
1611
|
// `warmColdBatchWorktree` in this same file already draws exactly this
|
|
1183
|
-
// distinction for `worktree.prepare`, and
|
|
1184
|
-
//
|
|
1185
|
-
// treated it as fatal.
|
|
1612
|
+
// distinction for `worktree.prepare`, and the member workflow (member-cycle,
|
|
1613
|
+
// #677) runs its own relevance-scoped tests rather than treating a broken
|
|
1614
|
+
// capability as fatal. This was the one place that treated it as fatal.
|
|
1186
1615
|
//
|
|
1187
1616
|
// Skipping costs EARLY detection, not correctness: the aggregate `test.full`
|
|
1188
1617
|
// gate still runs before ship, CI still runs on the batch PR, and #562's
|
|
@@ -1281,14 +1710,19 @@ function runIncrementalGate(deps, config, dispatch, batchId, batch, memberIssue,
|
|
|
1281
1710
|
detail: withExcerpt(`cap run ${inconclusive.id} ${describeInconclusive(inconclusive)} after member review done`, excerpt),
|
|
1282
1711
|
});
|
|
1283
1712
|
deps.store.withLock((s) => ({ state: releaseSlot(s, batchId, now), result: undefined }));
|
|
1284
|
-
|
|
1285
|
-
|
|
1286
|
-
|
|
1287
|
-
|
|
1288
|
-
|
|
1289
|
-
|
|
1290
|
-
|
|
1291
|
-
|
|
1713
|
+
// #677: F.11's block semantics are "the member's commit stays on the
|
|
1714
|
+
// branch, nothing requeued/reverted" — under member worktrees that is
|
|
1715
|
+
// true only if the member LANDS before the batch blocks. The block means
|
|
1716
|
+
// the gate could not reach a verdict, not that the work is bad, so the
|
|
1717
|
+
// verified member joins the integration branch and is there for an
|
|
1718
|
+
// operator (or the #686 out-of-band reconcile) to act on. A landing
|
|
1719
|
+
// failure blocks the batch with its own reason — same operator outcome.
|
|
1720
|
+
const landed = landMemberBranch(deps, batchId, now);
|
|
1721
|
+
if (!landed.ok) {
|
|
1722
|
+
blockLandingFailure(deps, config, batch, landed.reason, now, result);
|
|
1723
|
+
return true;
|
|
1724
|
+
}
|
|
1725
|
+
blockBatchForOperator(deps, config, batchId, { reason, milestonePhase: 'batch-review' }, now, result);
|
|
1292
1726
|
return true;
|
|
1293
1727
|
}
|
|
1294
1728
|
return false;
|
|
@@ -1304,6 +1738,16 @@ function runIncrementalGate(deps, config, dispatch, batchId, batch, memberIssue,
|
|
|
1304
1738
|
* this is safe to call whether or not the caller already released it.
|
|
1305
1739
|
*/
|
|
1306
1740
|
function completeMemberGate(deps, config, dispatch, batchId, batch, memberIssue, now, result) {
|
|
1741
|
+
// #677 (§J.3): land the member's verified work on the integration branch
|
|
1742
|
+
// BEFORE anything reads it — the range recompute below runs `git log` on
|
|
1743
|
+
// the batch branch, which only sees the member's commits once landed. The
|
|
1744
|
+
// exec calls run OUTSIDE the lock (same invariant as the range recompute's
|
|
1745
|
+
// own comment below).
|
|
1746
|
+
const landed = landMemberBranch(deps, batchId, now);
|
|
1747
|
+
if (!landed.ok) {
|
|
1748
|
+
blockLandingFailure(deps, config, batch, landed.reason, now, result);
|
|
1749
|
+
return;
|
|
1750
|
+
}
|
|
1307
1751
|
// The commit-range recompute (`git log`) is a blocking subprocess call — it
|
|
1308
1752
|
// must run OUTSIDE the lock, like every other exec in this module; the
|
|
1309
1753
|
// result then lands as a pure data patch under the lock (Convention review:
|
|
@@ -1324,6 +1768,10 @@ function completeMemberGate(deps, config, dispatch, batchId, batch, memberIssue,
|
|
|
1324
1768
|
return { state: n, result: undefined };
|
|
1325
1769
|
});
|
|
1326
1770
|
result.completed.push(unit(batchId));
|
|
1771
|
+
// #677: the member's tree is done serving — landed, ranges recomputed.
|
|
1772
|
+
// Teardown before the advance so the next member's prep cannot collide
|
|
1773
|
+
// with this tree's cleanup.
|
|
1774
|
+
teardownMemberWorktree(deps, batchId, now, true);
|
|
1327
1775
|
advanceMemberOrValidate(deps, config, dispatch, batchId, batch.members.length, batch.executing_member, memberIssue, now, result);
|
|
1328
1776
|
}
|
|
1329
1777
|
/**
|
|
@@ -1360,6 +1808,12 @@ function resumeBlockedGate(deps, config, dispatch, batchId, now) {
|
|
|
1360
1808
|
if (!batch)
|
|
1361
1809
|
throw new types_1.SchedNotFoundError(`Batch not found: ${batchId}`);
|
|
1362
1810
|
if (batch.status !== 'blocked' || !batch.blocked_reason?.startsWith('gate-inconclusive:')) {
|
|
1811
|
+
// #677 review: the landing-failure blocks have no resume verb (same as
|
|
1812
|
+
// `suite-unreadable`) — name the actual reason and the operator path
|
|
1813
|
+
// instead of a bare illegal-transition error that reads like a bug.
|
|
1814
|
+
if (batch.status === 'blocked') {
|
|
1815
|
+
throw new types_1.SchedNotFoundError(`Batch ${batchId} is blocked on '${batch.blocked_reason ?? '?'}', which has no gate recheck: inspect the batch worktree/state named in its journal, then \`sched abandon --batch ${batchId}\` if it is a dead end`);
|
|
1816
|
+
}
|
|
1363
1817
|
throw new types_1.IllegalTransitionError('batch', batch.status, 'executing');
|
|
1364
1818
|
}
|
|
1365
1819
|
const capabilityId = batch.blocked_reason.slice('gate-inconclusive:'.length);
|
|
@@ -1367,7 +1821,13 @@ function resumeBlockedGate(deps, config, dispatch, batchId, now) {
|
|
|
1367
1821
|
if (batch.worktree === null || !deps.runCapability || memberIssue === undefined) {
|
|
1368
1822
|
throw new types_1.SchedNotFoundError(`Batch ${batchId} has no worktree/current member/gate hook to recheck`);
|
|
1369
1823
|
}
|
|
1370
|
-
const recheck = deps.runCapability(
|
|
1824
|
+
const recheck = deps.runCapability(
|
|
1825
|
+
// #677: recheck the member's OWN tree — pre-landing, the member's diff
|
|
1826
|
+
// exists only in its worktree; a batch-worktree recheck would test a
|
|
1827
|
+
// tree missing the change and trivially pass it. The fallback preserves
|
|
1828
|
+
// the pre-#677 behavior for a batch blocked before member worktrees
|
|
1829
|
+
// existed (its member's commits were already on the integration branch).
|
|
1830
|
+
batch.member_worktree ?? batch.worktree, capabilityId);
|
|
1371
1831
|
const result = emptyResult();
|
|
1372
1832
|
const excerpt = gateDetailExcerpt(recheck.outputTail, recheck.reason);
|
|
1373
1833
|
// #681: a recheck can classify what the batch blocked on as a TIMEOUT —
|
|
@@ -2282,6 +2742,17 @@ function reconcileStaleBlockedBatches(deps, now, result) {
|
|
|
2282
2742
|
* left on disk forever, since nothing else ever calls this for it.
|
|
2283
2743
|
*/
|
|
2284
2744
|
function teardownBatch(deps, batchId) {
|
|
2745
|
+
// #677: the current member's worktree goes first — it is batch-owned
|
|
2746
|
+
// state on the same teardown path as the shared tree, and a blocked or
|
|
2747
|
+
// dissolved batch must not leak a member tree (its branch/fields are
|
|
2748
|
+
// cleared by the helper; a gate-inconclusive BLOCK that intends to resume
|
|
2749
|
+
// never reaches this function). The batch is terminal here, so the
|
|
2750
|
+
// ranges-recorded membership of the last current member is a safe landed
|
|
2751
|
+
// signal (a range exists only for a member whose verified work landed).
|
|
2752
|
+
const terminal = (0, state_1.findBatch)(deps.store.load(), batchId);
|
|
2753
|
+
const lastMemberLanded = terminal?.ranges.some((r) => r.issue === terminal.members[terminal.executing_member - 1]) ===
|
|
2754
|
+
true;
|
|
2755
|
+
teardownMemberWorktree(deps, batchId, deps.now(), lastMemberLanded);
|
|
2285
2756
|
const state = deps.store.load();
|
|
2286
2757
|
const batch = (0, state_1.findBatch)(state, batchId);
|
|
2287
2758
|
if (!batch || batch.worktree === null)
|
|
@@ -2323,6 +2794,67 @@ function teardownBatch(deps, batchId) {
|
|
|
2323
2794
|
function runBatchTick(deps, config, dispatch) {
|
|
2324
2795
|
const result = emptyResult();
|
|
2325
2796
|
const now = deps.now();
|
|
2797
|
+
// #707: each batch dispatches through ITS OWN recorded profile, not the
|
|
2798
|
+
// engine's default. Resolution is cached per tick (per runBatchTick call):
|
|
2799
|
+
// within one pass a profile resolves at most once, and a failed resolution
|
|
2800
|
+
// reacts at most once. A profile that no longer resolves (config edited
|
|
2801
|
+
// mid-run) never silently falls back — the silent fallback is precisely
|
|
2802
|
+
// the #680 incident shape (a run that looked like one agent family,
|
|
2803
|
+
// executed as another). The reaction depends on how far the batch got:
|
|
2804
|
+
// - pre-merge (`dissolving` is a legal edge): DISSOLVE with reason
|
|
2805
|
+
// `dispatch-profile-missing:<name>` — loud, deterministic, one-shot, and
|
|
2806
|
+
// the members requeue as full-cycle units on the config default, which is
|
|
2807
|
+
// a coherent recovery rather than a mislabelled continuation.
|
|
2808
|
+
// - post-merge (PR parked/merged — the product already shipped): dissolving
|
|
2809
|
+
// would discard a landed PR's bookkeeping, so the remaining dispatch (the
|
|
2810
|
+
// report agent) runs on the default and journals
|
|
2811
|
+
// `dispatch-profile-missing` every tick it holds the decision open — a
|
|
2812
|
+
// bounded phase (deployed → reported), so the event cannot flood.
|
|
2813
|
+
const profileCache = new Map();
|
|
2814
|
+
const dispatchFor = (batch) => {
|
|
2815
|
+
if (batch.dispatch_profile === null)
|
|
2816
|
+
return dispatch;
|
|
2817
|
+
const cached = profileCache.get(batch.dispatch_profile);
|
|
2818
|
+
if (cached !== undefined)
|
|
2819
|
+
return cached;
|
|
2820
|
+
let resolved;
|
|
2821
|
+
try {
|
|
2822
|
+
resolved = (0, dispatch_1.resolveProfiledDispatch)(config, batch.dispatch_profile);
|
|
2823
|
+
}
|
|
2824
|
+
catch (err) {
|
|
2825
|
+
if (!(err instanceof dispatch_1.DispatchProfileError))
|
|
2826
|
+
throw err;
|
|
2827
|
+
const current = deps.store.load();
|
|
2828
|
+
const fresh = (0, state_1.findBatch)(current, batch.id);
|
|
2829
|
+
const dissolvable = fresh !== undefined && (0, state_1.allowedBatchTransitions)(fresh.status).includes('dissolving');
|
|
2830
|
+
if (dissolvable) {
|
|
2831
|
+
const liveSlot = slotFor(current, batch.id);
|
|
2832
|
+
if (liveSlot !== undefined &&
|
|
2833
|
+
liveSlot.pid !== null &&
|
|
2834
|
+
deps.spawnDeps.isAlive(liveSlot.pid, liveSlot.pid_start ?? undefined)) {
|
|
2835
|
+
deps.spawnDeps.kill(liveSlot.pid, liveSlot.pid_start ?? undefined);
|
|
2836
|
+
}
|
|
2837
|
+
const outcome = (0, recovery_1.dissolveBatch)(current, batch.id, { strategy: 'full', reason: `dispatch-profile-missing:${batch.dispatch_profile}` }, recoveryDeps(deps, config, fresh, now));
|
|
2838
|
+
deps.store.withLock((s) => ({
|
|
2839
|
+
state: applyBatchAndIssues(s, outcome.state, batch.id, outcome.requeued),
|
|
2840
|
+
result: undefined,
|
|
2841
|
+
}));
|
|
2842
|
+
result.failed.push(unit(batch.id));
|
|
2843
|
+
// Resolution failed: skip this batch for the rest of the pass with the
|
|
2844
|
+
// base dispatch as a harmless placeholder — the batch is dissolved, so
|
|
2845
|
+
// no further arm can claim a slot for it this tick.
|
|
2846
|
+
resolved = dispatch;
|
|
2847
|
+
}
|
|
2848
|
+
else {
|
|
2849
|
+
journalEvent(deps, 'dispatch-profile-missing', unit(batch.id), {
|
|
2850
|
+
detail: `profile '${batch.dispatch_profile}' is no longer configured; the post-merge tail runs on the default dispatch`,
|
|
2851
|
+
});
|
|
2852
|
+
resolved = dispatch;
|
|
2853
|
+
}
|
|
2854
|
+
}
|
|
2855
|
+
profileCache.set(batch.dispatch_profile, resolved);
|
|
2856
|
+
return resolved;
|
|
2857
|
+
};
|
|
2326
2858
|
// #565: when more ready batches exist than free capacity, claim them in
|
|
2327
2859
|
// the same priority order `runnableUnits` would (desc priority → asc
|
|
2328
2860
|
// readiness age → anchor) — this loop never goes through
|
|
@@ -2354,10 +2886,15 @@ function runBatchTick(deps, config, dispatch) {
|
|
|
2354
2886
|
const state = deps.store.load();
|
|
2355
2887
|
const batch = (0, state_1.findBatch)(state, batchId);
|
|
2356
2888
|
if (batch && batch.status === 'ready' && slotFor(state, batchId) === undefined) {
|
|
2357
|
-
|
|
2889
|
+
// #707: batch-setup dispatches through the batch's own profile too — it
|
|
2890
|
+
// is the batch's first agent, not an engine-internal step.
|
|
2891
|
+
claimAndSetup(deps, config, dispatchFor(batch), batchId, now, result);
|
|
2358
2892
|
}
|
|
2359
2893
|
}
|
|
2360
2894
|
for (const batch of deps.store.load().batches) {
|
|
2895
|
+
// #707: resolve this batch's profile once per pass — or dissolve/fall
|
|
2896
|
+
// back loudly when it no longer resolves (see `dispatchFor` above).
|
|
2897
|
+
const batchDispatch = dispatchFor(batch);
|
|
2361
2898
|
const slot = slotFor(deps.store.load(), batch.id);
|
|
2362
2899
|
// #609: a TERMINAL batch (`done`/`dissolved`) still holding a slot matches
|
|
2363
2900
|
// none of the status arms below and would fall through their `continue`,
|
|
@@ -2382,7 +2919,7 @@ function runBatchTick(deps, config, dispatch) {
|
|
|
2382
2919
|
}
|
|
2383
2920
|
if (slot && (slot.status === 'running' || slot.status === 'assigned')) {
|
|
2384
2921
|
if (batch.status === 'executing') {
|
|
2385
|
-
reconcileMemberSlot(deps, config,
|
|
2922
|
+
reconcileMemberSlot(deps, config, batchDispatch, batch.id, slot, now, result);
|
|
2386
2923
|
}
|
|
2387
2924
|
else if (batch.status === 'fixing') {
|
|
2388
2925
|
reconcileFixSlot(deps, config, batch.id, slot, now, result);
|
|
@@ -2399,7 +2936,7 @@ function runBatchTick(deps, config, dispatch) {
|
|
|
2399
2936
|
continue; // live but neither running/assigned (e.g. mid-verify) — next tick
|
|
2400
2937
|
if (batch.status === 'validating') {
|
|
2401
2938
|
// A local suite run, not a provider dispatch — unaffected by `paused`.
|
|
2402
|
-
runValidate(deps, config,
|
|
2939
|
+
runValidate(deps, config, batchDispatch, batch.id, now, result);
|
|
2403
2940
|
continue;
|
|
2404
2941
|
}
|
|
2405
2942
|
// #629: every branch below spawns a provider agent — hold the wedge
|
|
@@ -2408,19 +2945,22 @@ function runBatchTick(deps, config, dispatch) {
|
|
|
2408
2945
|
if (paused)
|
|
2409
2946
|
continue;
|
|
2410
2947
|
if (batch.status === 'deployed') {
|
|
2411
|
-
spawnReportAgent(deps, config,
|
|
2948
|
+
spawnReportAgent(deps, config, batchDispatch, batch.id, now, result);
|
|
2412
2949
|
}
|
|
2413
2950
|
else if (batch.status === 'executing') {
|
|
2414
2951
|
// A prior spawn threw, or `claimAndSpawn` found zero free capacity —
|
|
2415
2952
|
// either way the batch is stuck mid-member with no slot and nothing
|
|
2416
2953
|
// else will ever retry it (Conformance review AC5 caveat; Supportability
|
|
2417
|
-
// review #12). Retrying every tick is
|
|
2418
|
-
// the capacity
|
|
2419
|
-
|
|
2954
|
+
// review #12). Retrying every tick is cheap-by-contract (#677 review):
|
|
2955
|
+
// `spawnMemberContinuation` runs the free-capacity and member-identity
|
|
2956
|
+
// pre-checks BEFORE any pool/git work, so a full scheduler (or stale
|
|
2957
|
+
// member context) exits here without exec'ing, and `claimAndSpawn`
|
|
2958
|
+
// remains the authoritative capacity gate.
|
|
2959
|
+
spawnMemberContinuation(deps, config, batchDispatch, batch.id, now, result);
|
|
2420
2960
|
}
|
|
2421
2961
|
else if (batch.status === 'reviewing' || batch.status === 'shipping') {
|
|
2422
2962
|
// Same wedge, for a dead-or-never-claimed tail agent.
|
|
2423
|
-
spawnTailAgent(deps, config,
|
|
2963
|
+
spawnTailAgent(deps, config, batchDispatch, batch.id, now, result);
|
|
2424
2964
|
}
|
|
2425
2965
|
else if (batch.status === 'fixing') {
|
|
2426
2966
|
// `beginFixAttempt` already recorded this member's ONE attempt as
|