@ai-dossier/sched 0.59.0 → 0.60.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/batch-dispatch.d.ts +9 -0
- package/dist/batch-dispatch.d.ts.map +1 -1
- package/dist/batch-dispatch.js +213 -114
- package/dist/batch-dispatch.js.map +1 -1
- package/dist/dispatch.d.ts +12 -0
- package/dist/dispatch.d.ts.map +1 -1
- package/dist/dispatch.js +14 -0
- package/dist/dispatch.js.map +1 -1
- package/dist/engine.d.ts.map +1 -1
- package/dist/engine.js +2 -5
- package/dist/engine.js.map +1 -1
- package/dist/index.d.ts +1 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +6 -5
- package/dist/index.js.map +1 -1
- package/dist/types.d.ts +1 -1
- package/dist/types.d.ts.map +1 -1
- package/package.json +1 -1
package/dist/batch-dispatch.d.ts
CHANGED
|
@@ -380,6 +380,15 @@ interface AnchorTickContext {
|
|
|
380
380
|
*/
|
|
381
381
|
/** @internal exported for tests. */
|
|
382
382
|
export declare function memberDispatchModeFor(state: SchedState, batch: BatchEntry, config: SchedConfig): MemberDispatchMode;
|
|
383
|
+
/**
|
|
384
|
+
* #815: how many consecutive ticks a parallel member's landing may fail on a
|
|
385
|
+
* PUSH (`member-push-failed` / `landing-push-failed`) before the batch blocks.
|
|
386
|
+
* A network blip should not cost an operator round-trip; the landing is
|
|
387
|
+
* idempotent (the rebase is a no-op once done, the lease pins the same tip,
|
|
388
|
+
* `merge --ff-only` succeeds when already merged), so the next tick simply
|
|
389
|
+
* runs it again.
|
|
390
|
+
*/
|
|
391
|
+
export declare const MAX_LAND_PUSH_RETRIES = 3;
|
|
383
392
|
/**
|
|
384
393
|
* One batch reconcile+refill pass. Called from `engine.ts`'s `tick()` after
|
|
385
394
|
* the issue-level pass — this pass never claims a slot the issue pass already
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"batch-dispatch.d.ts","sourceRoot":"","sources":["../src/batch-dispatch.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA8DG;AAUH,OAAO,EAIL,KAAK,YAAY,EAGjB,KAAK,gBAAgB,EAGtB,MAAM,gBAAgB,CAAC;AAUxB,OAAO,
|
|
1
|
+
{"version":3,"file":"batch-dispatch.d.ts","sourceRoot":"","sources":["../src/batch-dispatch.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA8DG;AAUH,OAAO,EAIL,KAAK,YAAY,EAGjB,KAAK,gBAAgB,EAGtB,MAAM,gBAAgB,CAAC;AAUxB,OAAO,EAeL,KAAK,gBAAgB,EAGrB,KAAK,SAAS,EAEf,MAAM,YAAY,CAAC;AAGpB,OAAO,EAGL,KAAK,WAAW,EAcjB,MAAM,eAAe,CAAC;AACvB,OAAO,EAAE,KAAK,OAAO,EAAa,MAAM,WAAW,CAAC;AACpD,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,WAAW,CAAC;AAC5C,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,WAAW,CAAC;AAExC,OAAO,EACL,KAAK,iBAAiB,EActB,KAAK,WAAW,EACjB,MAAM,YAAY,CAAC;AAGpB,OAAO,EA2BL,kBAAkB,EAKnB,MAAM,SAAS,CAAC;AACjB,OAAO,EACL,KAAK,QAAQ,EAMd,MAAM,YAAY,CAAC;AACpB,OAAO,KAAK,EAGV,UAAU,EAEV,oBAAoB,EAKpB,kBAAkB,EAClB,cAAc,EAId,WAAW,EACX,UAAU,EAEX,MAAM,SAAS,CAAC;AAajB,YAAY,EAAE,UAAU,EAAE,MAAM,SAAS,CAAC;AAI1C,OAAO,EAAE,kBAAkB,EAAE,CAAC;AAE9B,8DAA8D;AAC9D,MAAM,WAAW,iBAAiB;IAChC,KAAK,EAAE,UAAU,CAAC;IAClB,OAAO,EAAE,OAAO,CAAC;IACjB,WAAW,EAAE,WAAW,CAAC;IACzB,SAAS,EAAE,SAAS,CAAC;IACrB,GAAG,EAAE,MAAM,IAAI,CAAC;IAChB,uFAAuF;IACvF,OAAO,EAAE,MAAM,CAAC;IAChB,0FAA0F;IAC1F,IAAI,EAAE,MAAM,CAAC;IACb;;;;;;OAMG;IACH,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB;;;;;OAKG;IACH,QAAQ,EAAE,CAAC,QAAQ,EAAE,MAAM,EAAE,GAAG,CAAC,EAAE,iBAAiB,KAAK,WAAW,CAAC;IACrE;;;;;;;;;;;;;;;;;;;OAmBG;IACH,aAAa,CAAC,EAAE,CAAC,QAAQ,EAAE,MAAM,EAAE,YAAY,EAAE,MAAM,KAAK,oBAAoB,CAAC;IACjF,QAAQ,CAAC,EAAE,QAAQ,CAAC;IACpB;;;OAGG;IACH,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB;;;;;;OAMG;IACH,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB;AAED,8FAA8F;AAC9F,MAAM,WAAW,eAAe;IAC9B,OAAO,EAAE,MAAM,EAAE,CAAC;IAClB,SAAS,EAAE,MAAM,EAAE,CAAC;IACpB,MAAM,EAAE,MAAM,EAAE,CAAC;IACjB,aAAa,EAAE,MAAM,EAAE,CAAC;IACxB,MAAM,EAAE,MAAM,EAAE,CAAC;IACjB,uFAAuF;IACvF,qBAAqB,EAAE,MAAM,EAAE,CAAC;IAChC,8FAA8F;IAC9F,OAAO,EAAE,MAAM,EAAE,CAAC;CACnB;AAED;;;;;;GAMG;AACH,MAAM,WAAW,aAAa;IAC5B,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACjC;;;;OAIG;IACH,IAAI,CAAC,EAAE,cAAc,CAAC;CACvB;AAimBD;;;;;;;GAOG;AACH,wBAAgB,eAAe,CAAC,OAAO,EAAE,MAAM,EAAE,WAAW,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,MAAM,CAE3F;AA8qDD;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA8BG;AACH,wBAAgB,sBAAsB,CACpC,IAAI,EAAE,iBAAiB,EACvB,MAAM,EAAE,WAAW,EACnB,QAAQ,EAAE,gBAAgB,EAC1B,OAAO,EAAE,MAAM,EACf,KAAK,EAAE,UAAU,EACjB,WAAW,EAAE,MAAM,EACnB,OAAO,EAAE,aAAa,EACtB,GAAG,EAAE,IAAI,EACT,MAAM,EAAE,eAAe;AACvB,kGAAkG;AAClG,QAAQ,CAAC,EAAE,CAAC,CAAC,EAAE,UAAU,KAAK,UAAU,EACxC,UAAU,CAAC,EAAE,MAAM,IAAI,GACtB,IAAI,CAiBN;AAoWD;;;;;;;;;;;;;;;;;;;;;;;;;;;GA2BG;AACH,wBAAgB,iBAAiB,CAC/B,IAAI,EAAE,iBAAiB,EACvB,MAAM,EAAE,WAAW,EACnB,QAAQ,EAAE,gBAAgB,EAC1B,OAAO,EAAE,MAAM,EACf,GAAG,EAAE,IAAI,GACR;IACD,OAAO,EAAE,eAAe,GAAG,SAAS,GAAG,WAAW,GAAG,SAAS,CAAC;IAC/D,UAAU,EAAE,MAAM,CAAC;IACnB;;;;;OAKG;IACH,SAAS,CAAC,EAAE,cAAc,GAAG,qBAAqB,CAAC;IACnD,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB,CA+KA;AAm9BD;;;;;;;;GAQG;AACH,eAAO,MAAM,wBAAwB,IAAI,CAAC;AAgkB1C;;;;;;GAMG;AACH,wBAAgB,4BAA4B,CAAC,KAAK,EAAE,UAAU,EAAE,KAAK,EAAE,UAAU,GAAG,MAAM,EAAE,CAuB3F;AA6HD;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAwCG;AACH,wBAAgB,4BAA4B,CAC1C,IAAI,EAAE,iBAAiB,EACvB,GAAG,EAAE,IAAI,EACT,MAAM,EAAE,eAAe,EACvB,SAAS,EAAE,iBAAiB,GAAG,SAAS,GACvC,IAAI,CAmMN;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAyCG;AACH,wBAAgB,sBAAsB,CAAC,IAAI,EAAE,iBAAiB,EAAE,GAAG,EAAE,IAAI,GAAG,IAAI,CAuG/E;AAED;;;;;;GAMG;AACH,UAAU,iBAAiB;IACzB,IAAI,EAAE,MAAM,CAAC;IACb,gDAAgD;IAChD,IAAI,EAAE,gBAAgB,CAAC;IACvB,uEAAuE;IACvE,SAAS,EAAE,gBAAgB,CAAC;IAC5B,YAAY,EAAE,YAAY,CAAC;CAC5B;AAoTD;;;;;;;;;;;GAWG;AACH,oCAAoC;AACpC,wBAAgB,qBAAqB,CACnC,KAAK,EAAE,UAAU,EACjB,KAAK,EAAE,UAAU,EACjB,MAAM,EAAE,WAAW,GAClB,kBAAkB,CAQpB;AAymBD;;;;;;;GAOG;AACH,eAAO,MAAM,qBAAqB,IAAI,CAAC;AA8UvC;;;;;;;;;;;;;GAaG;AACH,wBAAgB,YAAY,CAC1B,IAAI,EAAE,iBAAiB,EACvB,MAAM,EAAE,WAAW,EACnB,QAAQ,EAAE,gBAAgB,GACzB,eAAe,CAgSjB"}
|
package/dist/batch-dispatch.js
CHANGED
|
@@ -96,7 +96,7 @@ var __importStar = (this && this.__importStar) || (function () {
|
|
|
96
96
|
};
|
|
97
97
|
})();
|
|
98
98
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
99
|
-
exports.MAX_BATCH_AGENT_RESPAWNS = exports.tailMembersRefusal = void 0;
|
|
99
|
+
exports.MAX_LAND_PUSH_RETRIES = exports.MAX_BATCH_AGENT_RESPAWNS = exports.tailMembersRefusal = void 0;
|
|
100
100
|
exports.memberBranchFor = memberBranchFor;
|
|
101
101
|
exports.evictMemberAndContinue = evictMemberAndContinue;
|
|
102
102
|
exports.resumeBlockedGate = resumeBlockedGate;
|
|
@@ -176,9 +176,7 @@ function agentAlive(deps, slot) {
|
|
|
176
176
|
* which escalates to SIGKILL (#844).
|
|
177
177
|
*/
|
|
178
178
|
function killLiveAgent(deps, slot) {
|
|
179
|
-
|
|
180
|
-
deps.spawnDeps.kill(slot.pid, slot.pid_start ?? undefined);
|
|
181
|
-
}
|
|
179
|
+
(0, dispatch_1.killSlotAgent)(deps.spawnDeps, slot);
|
|
182
180
|
}
|
|
183
181
|
/**
|
|
184
182
|
* Read a dead batch dispatch's own log and classify it (#629 AC3) — tail,
|
|
@@ -393,7 +391,7 @@ function applyDissolveOutcome(deps, batchId, outcome, now, membersBeforeRecovery
|
|
|
393
391
|
stopAndReleaseBlocked(deps, batchId, now);
|
|
394
392
|
return;
|
|
395
393
|
}
|
|
396
|
-
teardownBatch(deps, batchId);
|
|
394
|
+
teardownBatch(deps, batchId, now);
|
|
397
395
|
}
|
|
398
396
|
/** Stop every live agent of a batch that just blocked, and release all its slots (#810). */
|
|
399
397
|
function stopAndReleaseBlocked(deps, batchId, now) {
|
|
@@ -676,6 +674,30 @@ function usablePoolClaim(deps, w) {
|
|
|
676
674
|
path.resolve(w) === w &&
|
|
677
675
|
(deps.fsExists ?? ((p) => fs.existsSync(p)))(w));
|
|
678
676
|
}
|
|
677
|
+
/**
|
|
678
|
+
* The path of a worktree, other than the repo's own or the batch's, that has
|
|
679
|
+
* `branch` checked out — an earlier prep's pool claim orphaned by an engine
|
|
680
|
+
* exit before its run was recorded (#815). `null` when there is none. Member
|
|
681
|
+
* branch names are unique per batch/index/issue, so a match can only be this
|
|
682
|
+
* member's own leftover.
|
|
683
|
+
*/
|
|
684
|
+
function orphanedMemberClaim(deps, root, batch, branch) {
|
|
685
|
+
const listing = deps.exec('git', ['worktree', 'list', '--porcelain'], deps.repoDir);
|
|
686
|
+
if (listing === null)
|
|
687
|
+
return null;
|
|
688
|
+
let current = null;
|
|
689
|
+
for (const line of listing.split('\n')) {
|
|
690
|
+
if (line.startsWith('worktree '))
|
|
691
|
+
current = line.slice('worktree '.length).trim();
|
|
692
|
+
else if (line.trim() === `branch refs/heads/${branch}` && current !== null) {
|
|
693
|
+
const candidate = current;
|
|
694
|
+
if (candidate === root || candidate === batch.worktree)
|
|
695
|
+
return null;
|
|
696
|
+
return usablePoolClaim(deps, candidate) ? candidate : null;
|
|
697
|
+
}
|
|
698
|
+
}
|
|
699
|
+
return null;
|
|
700
|
+
}
|
|
679
701
|
function prepareMemberWorktree(deps, batch, memberIndex, issue, now) {
|
|
680
702
|
if (batch.branch === null)
|
|
681
703
|
return { ok: false, reason: 'no-integration-branch' };
|
|
@@ -720,7 +742,22 @@ function prepareMemberWorktree(deps, batch, memberIndex, issue, now) {
|
|
|
720
742
|
poolClaimed: batch.member_pool_claimed,
|
|
721
743
|
};
|
|
722
744
|
}
|
|
723
|
-
|
|
745
|
+
// #815: a prior prep that died between its pool claim and the run record
|
|
746
|
+
// left the claim checked out on THIS member's branch at a pool path that is
|
|
747
|
+
// not re-derivable (the pool renames the claim to the branch slug). Claiming
|
|
748
|
+
// again would take a SECOND tree, and `checkout -B` would then fail because
|
|
749
|
+
// the branch is checked out in the orphan — an eviction over a blip. No run
|
|
750
|
+
// was recorded, so no agent ever started in it: take the orphan over and
|
|
751
|
+
// re-point it exactly like a fresh claim.
|
|
752
|
+
const orphan = orphanedMemberClaim(deps, root, batch, branch);
|
|
753
|
+
const claimed = orphan ??
|
|
754
|
+
deps.exec(teardown_1.POOL_BIN, [...teardown_1.POOL_ARGS_PREFIX, 'claim', '--issue', String(issue), '--branch', branch], deps.repoDir);
|
|
755
|
+
if (orphan !== null) {
|
|
756
|
+
deps.journal.append((0, journal_1.unitEvent)('member-claim-reused', unit(batch.id), {
|
|
757
|
+
issue,
|
|
758
|
+
detail: `${orphan} — a pool claim on ${branch} that no run recorded (engine exit mid-prep)`,
|
|
759
|
+
}), now);
|
|
760
|
+
}
|
|
724
761
|
const claimedWorktree = claimed?.trim();
|
|
725
762
|
if (claimedWorktree && usablePoolClaim(deps, claimedWorktree)) {
|
|
726
763
|
// Re-point the warm spare at the member branch off the integration
|
|
@@ -796,26 +833,45 @@ function landMemberBranch(deps, batchId, now) {
|
|
|
796
833
|
});
|
|
797
834
|
return { ok: false, reason: 'invalid-branch-name' };
|
|
798
835
|
}
|
|
799
|
-
|
|
800
|
-
|
|
801
|
-
|
|
802
|
-
|
|
803
|
-
|
|
836
|
+
return fastForwardOntoIntegration(deps, {
|
|
837
|
+
batchId,
|
|
838
|
+
issue: memberIssue,
|
|
839
|
+
memberBranch: batch.member_branch,
|
|
840
|
+
integrationBranch: batch.branch,
|
|
841
|
+
worktree: batch.worktree,
|
|
842
|
+
now,
|
|
843
|
+
mergeFailedDetail: 'serial-landing invariant broken; blocking for an operator',
|
|
844
|
+
landedDetail: `${batch.member_branch} fast-forwarded onto ${batch.branch}`,
|
|
845
|
+
});
|
|
846
|
+
}
|
|
847
|
+
/**
|
|
848
|
+
* The tail every landing shares (#815 — serial `landMemberBranch` and
|
|
849
|
+
* parallel `landParallelRun` once each carried a copy): `git merge --ff-only
|
|
850
|
+
* <member branch>` in the shared batch worktree (which holds the integration
|
|
851
|
+
* branch), push the integration branch, journal `member-landed`. Both steps
|
|
852
|
+
* are idempotent, so re-running it after a partial failure (merged locally,
|
|
853
|
+
* push failed) is safe. A failure journals `landing-failed` with its reason
|
|
854
|
+
* and returns it; the caller decides whether that blocks or retries.
|
|
855
|
+
*/
|
|
856
|
+
function fastForwardOntoIntegration(deps, land) {
|
|
857
|
+
const failed = (reason, detail) => {
|
|
858
|
+
journalEvent(deps, 'landing-failed', unit(land.batchId), {
|
|
859
|
+
issue: land.issue,
|
|
860
|
+
reason,
|
|
861
|
+
detail,
|
|
804
862
|
});
|
|
805
|
-
return { ok: false, reason
|
|
863
|
+
return { ok: false, reason };
|
|
864
|
+
};
|
|
865
|
+
if (deps.exec('git', ['merge', '--ff-only', land.memberBranch], land.worktree) === null) {
|
|
866
|
+
return failed('landing-merge-failed', `git merge --ff-only ${land.memberBranch} into ${land.integrationBranch} failed — ${land.mergeFailedDetail}`);
|
|
806
867
|
}
|
|
807
|
-
if (deps.exec('git', ['push', 'origin', '--',
|
|
808
|
-
|
|
809
|
-
issue: memberIssue,
|
|
810
|
-
reason: 'landing-push-failed',
|
|
811
|
-
detail: `landed ${batch.member_branch} on ${batch.branch} locally but the push failed — blocking so origin stays the durable copy`,
|
|
812
|
-
});
|
|
813
|
-
return { ok: false, reason: 'landing-push-failed' };
|
|
868
|
+
if (deps.exec('git', ['push', 'origin', '--', land.integrationBranch], land.worktree) === null) {
|
|
869
|
+
return failed('landing-push-failed', `landed ${land.memberBranch} on ${land.integrationBranch} locally but the push failed — blocking so origin stays the durable copy`);
|
|
814
870
|
}
|
|
815
|
-
deps.journal.append((0, journal_1.unitEvent)('member-landed', unit(batchId), {
|
|
816
|
-
issue:
|
|
817
|
-
detail:
|
|
818
|
-
}), now);
|
|
871
|
+
deps.journal.append((0, journal_1.unitEvent)('member-landed', unit(land.batchId), {
|
|
872
|
+
issue: land.issue,
|
|
873
|
+
detail: land.landedDetail,
|
|
874
|
+
}), land.now);
|
|
819
875
|
return { ok: true };
|
|
820
876
|
}
|
|
821
877
|
/**
|
|
@@ -1225,26 +1281,10 @@ function claimAndSetup(deps, config, dispatch, batchId, now, result) {
|
|
|
1225
1281
|
// Parallel members each claim their OWN slot (`batch:<id>#<issue>`), so
|
|
1226
1282
|
// the setup slot is released here and the members' worktrees are prepared
|
|
1227
1283
|
// by `spawnParallelMembers` (outside the lock, per member).
|
|
1228
|
-
deps.store.withLock((s) => {
|
|
1229
|
-
|
|
1230
|
-
|
|
1231
|
-
|
|
1232
|
-
reason: 'batch-left-ready-during-setup',
|
|
1233
|
-
detail: `worktree ${setup.worktree} created but batch is now '${b?.status ?? 'gone'}' — orphaned, manual cleanup required`,
|
|
1234
|
-
});
|
|
1235
|
-
return { state: releaseSlot(s, batchId, now), result: undefined };
|
|
1236
|
-
}
|
|
1237
|
-
let next = (0, state_1.patchBatch)(s, batchId, {
|
|
1238
|
-
branch: setup.branch,
|
|
1239
|
-
worktree: setup.worktree,
|
|
1240
|
-
run_id: setup.runId,
|
|
1241
|
-
pool_claimed: setup.poolClaimed,
|
|
1242
|
-
member_dispatch: 'parallel',
|
|
1243
|
-
member_runs: [],
|
|
1244
|
-
}, now);
|
|
1245
|
-
next = (0, state_1.transitionBatch)(next, batchId, 'executing', { executing_member: 0 }, now);
|
|
1246
|
-
return { state: releaseSlot(next, batchId, now), result: undefined };
|
|
1247
|
-
});
|
|
1284
|
+
deps.store.withLock((s) => ({
|
|
1285
|
+
state: landSetup(deps, s, batchId, setup, now, { member_dispatch: 'parallel', member_runs: [] }, 0, (next) => releaseSlot(next, batchId, now)),
|
|
1286
|
+
result: undefined,
|
|
1287
|
+
}));
|
|
1248
1288
|
spawnParallelMembers(deps, config, dispatch, batchId, now, result);
|
|
1249
1289
|
return;
|
|
1250
1290
|
}
|
|
@@ -1287,34 +1327,44 @@ function claimAndSetup(deps, config, dispatch, batchId, now, result) {
|
|
|
1287
1327
|
abortSetup(`member-worktree-prep-failed:${memberPrep.reason}`, `member worktree prep failed: ${memberPrep.reason} (member 1 #${firstIssue}, branch ${memberBranchFor(batchId, 1, firstIssue)})`);
|
|
1288
1328
|
return;
|
|
1289
1329
|
}
|
|
1290
|
-
deps.store.withLock((s) => {
|
|
1291
|
-
|
|
1292
|
-
|
|
1293
|
-
|
|
1294
|
-
|
|
1295
|
-
|
|
1296
|
-
|
|
1297
|
-
|
|
1298
|
-
|
|
1299
|
-
|
|
1300
|
-
|
|
1301
|
-
|
|
1302
|
-
|
|
1303
|
-
|
|
1304
|
-
|
|
1305
|
-
|
|
1306
|
-
|
|
1307
|
-
|
|
1308
|
-
|
|
1309
|
-
|
|
1310
|
-
|
|
1311
|
-
|
|
1312
|
-
|
|
1313
|
-
|
|
1314
|
-
|
|
1315
|
-
|
|
1316
|
-
|
|
1317
|
-
|
|
1330
|
+
deps.store.withLock((s) => ({
|
|
1331
|
+
state: landSetup(deps, s, batchId, setup, now, { member_dispatch: 'serial' }, 1, (next) => {
|
|
1332
|
+
const slot = slotFor(next, batchId);
|
|
1333
|
+
return slot
|
|
1334
|
+
? spawnMember(deps, dispatch, next, slot, batchId, now, result, memberPrep)
|
|
1335
|
+
: next;
|
|
1336
|
+
}),
|
|
1337
|
+
result: undefined,
|
|
1338
|
+
}));
|
|
1339
|
+
}
|
|
1340
|
+
/**
|
|
1341
|
+
* Land a completed `runBatchSetup` on the batch entry (inside the caller's
|
|
1342
|
+
* lock): patch branch/worktree/run id, move `ready → executing`, then hand the
|
|
1343
|
+
* result to `after` (serial: spawn member 1; parallel: release the setup slot).
|
|
1344
|
+
* A batch that left `ready` between the claim and here (dissolved/abandoned)
|
|
1345
|
+
* leaves a real worktree nothing else knows about and a slot still `assigned`
|
|
1346
|
+
* with no agent in it — release the slot rather than leak capacity; the
|
|
1347
|
+
* worktree is orphaned (named in the journal for manual cleanup — it is not
|
|
1348
|
+
* this rare-race path's job to guess whether reusing or removing it is safe).
|
|
1349
|
+
* Shared by both dispatch modes so the two landings cannot drift (#815).
|
|
1350
|
+
*/
|
|
1351
|
+
function landSetup(deps, s, batchId, setup, now, modePatch, executingMember, after) {
|
|
1352
|
+
const b = (0, state_1.findBatch)(s, batchId);
|
|
1353
|
+
if (!b || b.status !== 'ready') {
|
|
1354
|
+
journalEvent(deps, 'unit-failed', unit(batchId), {
|
|
1355
|
+
reason: 'batch-left-ready-during-setup',
|
|
1356
|
+
detail: `worktree ${setup.worktree} created but batch is now '${b?.status ?? 'gone'}' — orphaned, manual cleanup required`,
|
|
1357
|
+
});
|
|
1358
|
+
return releaseSlot(s, batchId, now);
|
|
1359
|
+
}
|
|
1360
|
+
const next = (0, state_1.transitionBatch)((0, state_1.patchBatch)(s, batchId, {
|
|
1361
|
+
branch: setup.branch,
|
|
1362
|
+
worktree: setup.worktree,
|
|
1363
|
+
run_id: setup.runId,
|
|
1364
|
+
pool_claimed: setup.poolClaimed,
|
|
1365
|
+
...modePatch,
|
|
1366
|
+
}, now), batchId, 'executing', { executing_member: executingMember }, now);
|
|
1367
|
+
return after(next);
|
|
1318
1368
|
}
|
|
1319
1369
|
// --- Continuation: claim a fresh slot for the next live step ---
|
|
1320
1370
|
/**
|
|
@@ -1643,18 +1693,8 @@ function runValidate(deps, config, dispatch, batchId, now, result) {
|
|
|
1643
1693
|
}
|
|
1644
1694
|
deps.journal.append((0, journal_1.unitEvent)('verify-complete', unit(batchId), { detail: suite.detail ?? 'suite green' }), now);
|
|
1645
1695
|
const admitted = deps.store.withLock((s) => {
|
|
1646
|
-
const
|
|
1647
|
-
|
|
1648
|
-
return { state: s, result: false };
|
|
1649
|
-
if (b.executing_member < b.members.length) {
|
|
1650
|
-
return {
|
|
1651
|
-
state: (0, state_1.transitionBatch)(s, batchId, 'executing',
|
|
1652
|
-
// #809: a parallel batch spawns every run-less member itself.
|
|
1653
|
-
isParallelBatch(b) ? {} : { executing_member: b.executing_member + 1 }, now),
|
|
1654
|
-
result: true,
|
|
1655
|
-
};
|
|
1656
|
-
}
|
|
1657
|
-
return { state: (0, state_1.transitionBatch)(s, batchId, 'reviewing', {}, now), result: undefined };
|
|
1696
|
+
const out = readmitAfterValidation(s, batchId, now);
|
|
1697
|
+
return { state: out.state, result: out.admitted };
|
|
1658
1698
|
});
|
|
1659
1699
|
if (admitted) {
|
|
1660
1700
|
const b = (0, state_1.findBatch)(deps.store.load(), batchId);
|
|
@@ -1766,19 +1806,32 @@ function evictOffender(deps, config, batchId, offender, attribution, now, result
|
|
|
1766
1806
|
return;
|
|
1767
1807
|
}
|
|
1768
1808
|
if (outcome.suite?.ok) {
|
|
1769
|
-
deps.store.withLock((s) => {
|
|
1770
|
-
|
|
1771
|
-
|
|
1772
|
-
|
|
1773
|
-
|
|
1774
|
-
|
|
1775
|
-
|
|
1776
|
-
|
|
1777
|
-
|
|
1778
|
-
|
|
1779
|
-
|
|
1780
|
-
|
|
1809
|
+
deps.store.withLock((s) => ({
|
|
1810
|
+
state: readmitAfterValidation(s, batchId, now).state,
|
|
1811
|
+
result: undefined,
|
|
1812
|
+
}));
|
|
1813
|
+
}
|
|
1814
|
+
}
|
|
1815
|
+
/**
|
|
1816
|
+
* The green-gate exit of `validating` (#815 — one step for `runValidate` and
|
|
1817
|
+
* `evictOffender`): a member admitted while the suite ran sends the batch back
|
|
1818
|
+
* to `executing` (the suite must run again before final review); otherwise it
|
|
1819
|
+
* moves on to `reviewing`. A batch no longer `validating` is left untouched
|
|
1820
|
+
* (`admitted: false`, so the caller falls through to the tail spawn).
|
|
1821
|
+
*/
|
|
1822
|
+
function readmitAfterValidation(s, batchId, now) {
|
|
1823
|
+
const b = (0, state_1.findBatch)(s, batchId);
|
|
1824
|
+
if (!b || b.status !== 'validating')
|
|
1825
|
+
return { state: s, admitted: false };
|
|
1826
|
+
if (b.executing_member < b.members.length) {
|
|
1827
|
+
return {
|
|
1828
|
+
state: (0, state_1.transitionBatch)(s, batchId, 'executing',
|
|
1829
|
+
// #809: a parallel batch spawns every run-less member itself.
|
|
1830
|
+
isParallelBatch(b) ? {} : { executing_member: b.executing_member + 1 }, now),
|
|
1831
|
+
admitted: true,
|
|
1832
|
+
};
|
|
1781
1833
|
}
|
|
1834
|
+
return { state: (0, state_1.transitionBatch)(s, batchId, 'reviewing', {}, now), admitted: false };
|
|
1782
1835
|
}
|
|
1783
1836
|
/**
|
|
1784
1837
|
* The one-shot claim on resolving `currentMember` (#613). Transitions the batch only while
|
|
@@ -3104,7 +3157,7 @@ function closeWithoutReport(deps, batchId, why, now, result) {
|
|
|
3104
3157
|
detail: `${why} — batch closed without its report (the PR is merged); post the batch report by hand if one is wanted`,
|
|
3105
3158
|
});
|
|
3106
3159
|
result.failed.push(unit(batchId));
|
|
3107
|
-
teardownBatch(deps, batchId);
|
|
3160
|
+
teardownBatch(deps, batchId, now);
|
|
3108
3161
|
}
|
|
3109
3162
|
function reconcileTailSlot(deps, config, batchId, slot, now, result) {
|
|
3110
3163
|
const state = deps.store.load();
|
|
@@ -3213,7 +3266,7 @@ _config, batchId, slot, now, result) {
|
|
|
3213
3266
|
return { state: n, result: undefined };
|
|
3214
3267
|
});
|
|
3215
3268
|
result.completed.push(unit(batchId));
|
|
3216
|
-
teardownBatch(deps, batchId);
|
|
3269
|
+
teardownBatch(deps, batchId, now);
|
|
3217
3270
|
return;
|
|
3218
3271
|
}
|
|
3219
3272
|
if (dead) {
|
|
@@ -3823,7 +3876,7 @@ function reconcileStaleBlockedBatches(deps, now, result, anchorCtx) {
|
|
|
3823
3876
|
});
|
|
3824
3877
|
result.mergeAccepted.push(unit(batch.id));
|
|
3825
3878
|
if (finishInline && !keepWorktree) {
|
|
3826
|
-
teardownBatch(deps, batch.id);
|
|
3879
|
+
teardownBatch(deps, batch.id, now);
|
|
3827
3880
|
}
|
|
3828
3881
|
}
|
|
3829
3882
|
}
|
|
@@ -4132,11 +4185,11 @@ function recordAnchorCloseFailed(deps, batch, outcome, shipped, now) {
|
|
|
4132
4185
|
* eviction-threshold dissolve): a dissolved batch's worktree is otherwise
|
|
4133
4186
|
* left on disk forever, since nothing else ever calls this for it.
|
|
4134
4187
|
*/
|
|
4135
|
-
function teardownBatch(deps, batchId) {
|
|
4188
|
+
function teardownBatch(deps, batchId, now) {
|
|
4136
4189
|
// #855: a kept batch's trees are the operator's — every teardown path reads
|
|
4137
4190
|
// the same persisted decision. Only its live agents are stopped.
|
|
4138
4191
|
if ((0, state_1.findBatch)(deps.store.load(), batchId)?.worktree_kept === true) {
|
|
4139
|
-
teardownParallelRuns(deps, batchId, null);
|
|
4192
|
+
teardownParallelRuns(deps, batchId, null, now);
|
|
4140
4193
|
return;
|
|
4141
4194
|
}
|
|
4142
4195
|
// #677: the current member's worktree goes first — it is batch-owned
|
|
@@ -4149,8 +4202,8 @@ function teardownBatch(deps, batchId) {
|
|
|
4149
4202
|
const terminal = (0, state_1.findBatch)(deps.store.load(), batchId);
|
|
4150
4203
|
const lastMemberLanded = terminal?.ranges.some((r) => r.issue === terminal.members[terminal.executing_member - 1]) ===
|
|
4151
4204
|
true;
|
|
4152
|
-
const serial = teardownMemberWorktree(deps, batchId,
|
|
4153
|
-
teardownParallelRuns(deps, batchId, serial);
|
|
4205
|
+
const serial = teardownMemberWorktree(deps, batchId, now, lastMemberLanded);
|
|
4206
|
+
teardownParallelRuns(deps, batchId, serial, now);
|
|
4154
4207
|
deleteMemberBranches(deps, batchId);
|
|
4155
4208
|
const state = deps.store.load();
|
|
4156
4209
|
const batch = (0, state_1.findBatch)(state, batchId);
|
|
@@ -4619,17 +4672,60 @@ function landParallelRun(deps, batch, run, now) {
|
|
|
4619
4672
|
], run.worktree) === null) {
|
|
4620
4673
|
return fail('member-push-failed', `rebased ${run.branch} onto ${batch.branch} but could not push it (origin moved since ${remoteTip.slice(0, 12) || 'absent'}, or the push failed) — blocking so nothing pushed after verification is overwritten`);
|
|
4621
4674
|
}
|
|
4622
|
-
|
|
4623
|
-
|
|
4624
|
-
|
|
4625
|
-
|
|
4626
|
-
|
|
4627
|
-
|
|
4628
|
-
|
|
4675
|
+
const ff = fastForwardOntoIntegration(deps, {
|
|
4676
|
+
batchId: batch.id,
|
|
4677
|
+
issue: run.issue,
|
|
4678
|
+
memberBranch: run.branch,
|
|
4679
|
+
integrationBranch: batch.branch,
|
|
4680
|
+
worktree: batch.worktree,
|
|
4681
|
+
now,
|
|
4682
|
+
mergeFailedDetail: 'right after a clean rebase — blocking for an operator',
|
|
4683
|
+
landedDetail: `${run.branch} rebased onto and fast-forwarded into ${batch.branch} (member ${run.index}, parallel)`,
|
|
4684
|
+
});
|
|
4685
|
+
return ff.ok ? { kind: 'landed' } : { kind: 'failed', reason: ff.reason };
|
|
4686
|
+
}
|
|
4687
|
+
/**
|
|
4688
|
+
* #815: how many consecutive ticks a parallel member's landing may fail on a
|
|
4689
|
+
* PUSH (`member-push-failed` / `landing-push-failed`) before the batch blocks.
|
|
4690
|
+
* A network blip should not cost an operator round-trip; the landing is
|
|
4691
|
+
* idempotent (the rebase is a no-op once done, the lease pins the same tip,
|
|
4692
|
+
* `merge --ff-only` succeeds when already merged), so the next tick simply
|
|
4693
|
+
* runs it again.
|
|
4694
|
+
*/
|
|
4695
|
+
exports.MAX_LAND_PUSH_RETRIES = 3;
|
|
4696
|
+
/** Reasons that mean "the push failed", as opposed to a broken invariant that retrying cannot fix. */
|
|
4697
|
+
const TRANSIENT_LAND_REASONS = new Set([
|
|
4698
|
+
'member-push-failed',
|
|
4699
|
+
'landing-push-failed',
|
|
4700
|
+
]);
|
|
4701
|
+
/**
|
|
4702
|
+
* Consecutive push-failure ticks per parallel member, keyed by state dir +
|
|
4703
|
+
* batch + issue. In-memory on purpose: a restart merely grants a fresh budget
|
|
4704
|
+
* (still bounded), and it keeps the persisted schema untouched.
|
|
4705
|
+
*/
|
|
4706
|
+
const landPushRetries = new Map();
|
|
4707
|
+
function landRetryKey(deps, batchId, issue) {
|
|
4708
|
+
return `${deps.store.dir}\0${batchId}\0${issue}`;
|
|
4709
|
+
}
|
|
4710
|
+
/**
|
|
4711
|
+
* Whether a failed parallel landing should be retried next tick instead of
|
|
4712
|
+
* blocking the batch: only push failures, at most `MAX_LAND_PUSH_RETRIES`
|
|
4713
|
+
* times in a row. Journals `landing-retry` with the position in the budget.
|
|
4714
|
+
*/
|
|
4715
|
+
function deferTransientPushFailure(deps, batchId, run, reason, now) {
|
|
4716
|
+
if (!TRANSIENT_LAND_REASONS.has(reason))
|
|
4717
|
+
return false;
|
|
4718
|
+
const key = landRetryKey(deps, batchId, run.issue);
|
|
4719
|
+
const attempt = (landPushRetries.get(key) ?? 0) + 1;
|
|
4720
|
+
if (attempt > exports.MAX_LAND_PUSH_RETRIES)
|
|
4721
|
+
return false;
|
|
4722
|
+
landPushRetries.set(key, attempt);
|
|
4723
|
+
deps.journal.append((0, journal_1.unitEvent)('landing-retry', unit(batchId), {
|
|
4629
4724
|
issue: run.issue,
|
|
4630
|
-
|
|
4725
|
+
reason,
|
|
4726
|
+
detail: `${reason} for member ${run.index} (#${run.issue}) — retrying next tick (${attempt}/${exports.MAX_LAND_PUSH_RETRIES}) before blocking`,
|
|
4631
4727
|
}), now);
|
|
4632
|
-
return
|
|
4728
|
+
return true;
|
|
4633
4729
|
}
|
|
4634
4730
|
/**
|
|
4635
4731
|
* Walk members IN ORDER: skip every resolved one (landed, evicted, or no longer
|
|
@@ -4678,10 +4774,14 @@ function advanceParallelBatch(deps, config, dispatch, batchId, now, result) {
|
|
|
4678
4774
|
continue;
|
|
4679
4775
|
}
|
|
4680
4776
|
if (landed.kind === 'failed') {
|
|
4777
|
+
if (deferTransientPushFailure(deps, batchId, run, landed.reason, now))
|
|
4778
|
+
return;
|
|
4779
|
+
landPushRetries.delete(landRetryKey(deps, batchId, run.issue));
|
|
4681
4780
|
stopRunningMembers(deps, batchId, now);
|
|
4682
4781
|
blockBatchForOperator(deps, config, batchId, { reason: landed.reason }, now, result);
|
|
4683
4782
|
return;
|
|
4684
4783
|
}
|
|
4784
|
+
landPushRetries.delete(landRetryKey(deps, batchId, run.issue));
|
|
4685
4785
|
// `git log` outside the lock (same invariant as `completeMemberGate`).
|
|
4686
4786
|
const ranges = (0, attribution_1.memberRanges)(boundaryCommits(deps, batch));
|
|
4687
4787
|
deps.store.withLock((s) => {
|
|
@@ -4807,11 +4907,10 @@ function reconcileParallelBatch(deps, config, dispatch, batchId, now, result, pa
|
|
|
4807
4907
|
* `kept-worktree` warning names them. A run whose earlier teardown failed
|
|
4808
4908
|
* (`teardown_failed_at`) is not re-attempted either.
|
|
4809
4909
|
*/
|
|
4810
|
-
function teardownParallelRuns(deps, batchId, serial) {
|
|
4910
|
+
function teardownParallelRuns(deps, batchId, serial, now) {
|
|
4811
4911
|
const batch = (0, state_1.findBatch)(deps.store.load(), batchId);
|
|
4812
4912
|
if (!batch)
|
|
4813
4913
|
return;
|
|
4814
|
-
const now = deps.now();
|
|
4815
4914
|
for (const run of batch.member_runs) {
|
|
4816
4915
|
if (run.torn_down)
|
|
4817
4916
|
continue;
|
|
@@ -5047,7 +5146,7 @@ function runBatchTick(deps, config, dispatch) {
|
|
|
5047
5146
|
// and it stops any agent that is somehow still running in a kept tree.
|
|
5048
5147
|
if (batch.member_runs.some((r) => !r.torn_down)) {
|
|
5049
5148
|
try {
|
|
5050
|
-
teardownParallelRuns(deps, batch.id, null);
|
|
5149
|
+
teardownParallelRuns(deps, batch.id, null, now);
|
|
5051
5150
|
}
|
|
5052
5151
|
catch (err) {
|
|
5053
5152
|
journalEvent(deps, 'teardown-failed', unit(batch.id), {
|