@ai-dossier/sched 0.46.1 → 0.47.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +33 -11
- package/dist/batch-dispatch.d.ts +17 -2
- package/dist/batch-dispatch.d.ts.map +1 -1
- package/dist/batch-dispatch.js +903 -133
- package/dist/batch-dispatch.js.map +1 -1
- package/dist/engine.d.ts +3 -2
- package/dist/engine.d.ts.map +1 -1
- package/dist/engine.js +3 -2
- package/dist/engine.js.map +1 -1
- package/dist/index.d.ts +2 -2
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +5 -2
- package/dist/index.js.map +1 -1
- package/dist/journal.d.ts +4 -1
- package/dist/journal.d.ts.map +1 -1
- package/dist/journal.js +5 -2
- package/dist/journal.js.map +1 -1
- package/dist/persist.d.ts.map +1 -1
- package/dist/persist.js +10 -1
- package/dist/persist.js.map +1 -1
- package/dist/scheduler.d.ts.map +1 -1
- package/dist/scheduler.js +5 -5
- package/dist/scheduler.js.map +1 -1
- package/dist/state.d.ts +25 -0
- package/dist/state.d.ts.map +1 -1
- package/dist/state.js +155 -12
- package/dist/state.js.map +1 -1
- package/dist/status.d.ts.map +1 -1
- package/dist/status.js +5 -2
- package/dist/status.js.map +1 -1
- package/dist/types.d.ts +89 -4
- package/dist/types.d.ts.map +1 -1
- package/dist/types.js +9 -2
- package/dist/types.js.map +1 -1
- package/package.json +1 -1
package/dist/batch-dispatch.js
CHANGED
|
@@ -12,7 +12,8 @@
|
|
|
12
12
|
* at batch-phase granularity instead of per-issue-phase granularity:
|
|
13
13
|
*
|
|
14
14
|
* ```
|
|
15
|
-
* ready → executing(member i/N
|
|
15
|
+
* ready → executing(serial: member i/N ⟲ | parallel: members ∥ + ordered landing)
|
|
16
|
+
* → validating → reviewing → shipping
|
|
16
17
|
* → awaiting-merge → merged → deployed → reported → done
|
|
17
18
|
* failure rails (RFC F.2/F.8/F.9):
|
|
18
19
|
* executing → dissolving (a member self-reports blocked, RFC F.1)
|
|
@@ -98,6 +99,7 @@ Object.defineProperty(exports, "__esModule", { value: true });
|
|
|
98
99
|
exports.memberBranchFor = memberBranchFor;
|
|
99
100
|
exports.evictMemberAndContinue = evictMemberAndContinue;
|
|
100
101
|
exports.resumeBlockedGate = resumeBlockedGate;
|
|
102
|
+
exports.memberDispatchModeFor = memberDispatchModeFor;
|
|
101
103
|
exports.runBatchTick = runBatchTick;
|
|
102
104
|
const fs = __importStar(require("node:fs"));
|
|
103
105
|
const path = __importStar(require("node:path"));
|
|
@@ -197,7 +199,9 @@ function handleDeadDispatchApiError(deps, batchId, logFile, offset, now, options
|
|
|
197
199
|
return false;
|
|
198
200
|
deps.store.withLock((s) => {
|
|
199
201
|
let n = (0, dispatch_health_1.recordDispatchApiError)((event, unitId, extra) => journalEvent(deps, event, unitId, extra), s, unit(batchId), apiError);
|
|
200
|
-
if (options.
|
|
202
|
+
if (options.releaseWith)
|
|
203
|
+
n = options.releaseWith(n);
|
|
204
|
+
else if (options.release ?? true)
|
|
201
205
|
n = releaseSlot(n, batchId, now);
|
|
202
206
|
return { state: n, result: undefined };
|
|
203
207
|
});
|
|
@@ -789,13 +793,9 @@ landed) {
|
|
|
789
793
|
});
|
|
790
794
|
return;
|
|
791
795
|
}
|
|
792
|
-
//
|
|
793
|
-
//
|
|
794
|
-
//
|
|
795
|
-
// cold removes post-verify the path is gone and unlisted. Nulls first —
|
|
796
|
-
// clear the fields BEFORE the exec so a crash mid-teardown doesn't retry
|
|
797
|
-
// into a half-cleaned tree (the re-derived path is deterministic if a
|
|
798
|
-
// leftover does survive).
|
|
796
|
+
// Nulls first — clear the fields BEFORE the exec so a crash mid-teardown
|
|
797
|
+
// doesn't retry into a half-cleaned tree (the re-derived path is
|
|
798
|
+
// deterministic if a leftover does survive).
|
|
799
799
|
deps.store.withLock((s) => {
|
|
800
800
|
const b = (0, state_1.findBatch)(s, batchId);
|
|
801
801
|
if (!b)
|
|
@@ -805,7 +805,29 @@ landed) {
|
|
|
805
805
|
result: undefined,
|
|
806
806
|
};
|
|
807
807
|
});
|
|
808
|
-
|
|
808
|
+
teardownMemberTree(deps, batchId, { worktree, branch, poolClaimed: batch.member_pool_claimed === true }, landed);
|
|
809
|
+
}
|
|
810
|
+
/**
|
|
811
|
+
* The exec half of a member teardown, shared by the serial rail
|
|
812
|
+
* (`teardownMemberWorktree`, which first clears the batch's member fields) and
|
|
813
|
+
* the parallel rail (#809: `evictParallelMember`, `advanceParallelBatch`,
|
|
814
|
+
* `teardownParallelRuns`). See
|
|
815
|
+
* `teardownMemberWorktree` for the landed/evicted branch policy.
|
|
816
|
+
*/
|
|
817
|
+
function teardownMemberTree(deps, batchId, tree, landed) {
|
|
818
|
+
const { worktree, branch } = tree;
|
|
819
|
+
if (branch !== null && !attribution_1.SAFE_REF_RE.test(branch)) {
|
|
820
|
+
journalEvent(deps, 'teardown-failed', unit(batchId), {
|
|
821
|
+
cleanup: 'failed-invalid-branch-name',
|
|
822
|
+
detail: branch,
|
|
823
|
+
});
|
|
824
|
+
return;
|
|
825
|
+
}
|
|
826
|
+
// Route through `runTeardown` for the same guarantees `teardownBatch`
|
|
827
|
+
// gets: pool returns are verify-first idempotent and refuse to claim
|
|
828
|
+
// success unless the pool's own self-check reports the entry `warm`;
|
|
829
|
+
// cold removes post-verify the path is gone and unlisted.
|
|
830
|
+
const t = (0, teardown_1.runTeardown)(deps.exec, deps.repoDir, { worktree, poolClaimed: tree.poolClaimed, branch }, deps.fsExists);
|
|
809
831
|
const treeCleared = !t.cleanup.startsWith('failed');
|
|
810
832
|
let branchCleanup = 'skipped';
|
|
811
833
|
if (branch !== null && treeCleared) {
|
|
@@ -858,9 +880,10 @@ function blockLandingFailure(deps, config, batch, reason, now, result) {
|
|
|
858
880
|
blockBatchForOperator(deps, config, batch.id, { reason }, now, result);
|
|
859
881
|
}
|
|
860
882
|
/**
|
|
861
|
-
* Spawn one batch member's member-cycle agent (#677) into the slot a
|
|
862
|
-
* member (or batch-setup) just released — each member in its OWN
|
|
863
|
-
* off the integration branch
|
|
883
|
+
* Spawn one SERIAL batch member's member-cycle agent (#677) into the slot a
|
|
884
|
+
* prior member (or batch-setup) just released — each member in its OWN
|
|
885
|
+
* worktree off the integration branch. Parallel batches (#809) spawn through
|
|
886
|
+
* `spawnParallelMembers` instead, one slot per member. Prepared by
|
|
864
887
|
* {@link prepareMemberWorktree} at the spawn site (outside the lock); this
|
|
865
888
|
* function only patches state and spawns.
|
|
866
889
|
*/
|
|
@@ -902,6 +925,42 @@ function advanceMemberToValidated(state, memberIssue, now) {
|
|
|
902
925
|
return next;
|
|
903
926
|
}
|
|
904
927
|
function spawnMember(deps, dispatch, state, slot, batchId, now, result, member) {
|
|
928
|
+
const batch = (0, state_1.findBatch)(state, batchId);
|
|
929
|
+
const memberIssue = batch?.members[(batch?.executing_member ?? 0) - 1];
|
|
930
|
+
if (batch && memberIssue === undefined) {
|
|
931
|
+
journalEvent(deps, 'unit-failed', unit(batchId), {
|
|
932
|
+
reason: 'no-member',
|
|
933
|
+
detail: `spawnMember: executing_member ${batch.executing_member} has no member issue`,
|
|
934
|
+
});
|
|
935
|
+
return releaseSlot(state, batchId, now);
|
|
936
|
+
}
|
|
937
|
+
const spawned = spawnMemberAgent(deps, dispatch, state, slot, batchId, now, result, {
|
|
938
|
+
index: batch?.executing_member ?? 0,
|
|
939
|
+
issue: memberIssue ?? 0,
|
|
940
|
+
member,
|
|
941
|
+
unitId: unit(batchId),
|
|
942
|
+
release: (s) => releaseSlot(s, batchId, now),
|
|
943
|
+
});
|
|
944
|
+
if (!spawned.ok)
|
|
945
|
+
return spawned.state;
|
|
946
|
+
// #677: persist the member's worktree context — a takeover redispatch or a
|
|
947
|
+
// `sched resume --batch` recheck after an engine restart must land in the
|
|
948
|
+
// SAME worktree/branch this prompt names, not re-derive (or worse,
|
|
949
|
+
// re-create) it.
|
|
950
|
+
return (0, state_1.patchBatch)(spawned.state, batchId, {
|
|
951
|
+
member_branch: member.branch,
|
|
952
|
+
member_worktree: member.worktree,
|
|
953
|
+
member_pool_claimed: member.poolClaimed,
|
|
954
|
+
}, now);
|
|
955
|
+
}
|
|
956
|
+
/**
|
|
957
|
+
* Spawn ONE member's member-cycle agent into `slot` — the shared core of the
|
|
958
|
+
* serial rail (`spawnMember`, the batch's own slot) and the parallel rail
|
|
959
|
+
* (`spawnParallelMembers`, #809: the member's own `batch:<id>#<issue>` slot).
|
|
960
|
+
* Patches the slot and the member's queue entry; the caller persists its own
|
|
961
|
+
* member context. On failure the slot is released through `target.release`.
|
|
962
|
+
*/
|
|
963
|
+
function spawnMemberAgent(deps, dispatch, state, slot, batchId, now, result, target) {
|
|
905
964
|
const batch = (0, state_1.findBatch)(state, batchId);
|
|
906
965
|
if (!batch || batch.worktree === null || batch.branch === null) {
|
|
907
966
|
// A leaked `assigned` slot with `pid: null` is invisible to `dead`
|
|
@@ -911,16 +970,9 @@ function spawnMember(deps, dispatch, state, slot, batchId, now, result, member)
|
|
|
911
970
|
reason: 'no-worktree',
|
|
912
971
|
detail: 'spawnMember: batch has no worktree/branch — batch-setup has not landed',
|
|
913
972
|
});
|
|
914
|
-
return
|
|
915
|
-
}
|
|
916
|
-
const memberIssue = batch.members[batch.executing_member - 1];
|
|
917
|
-
if (memberIssue === undefined) {
|
|
918
|
-
journalEvent(deps, 'unit-failed', unit(batchId), {
|
|
919
|
-
reason: 'no-member',
|
|
920
|
-
detail: `spawnMember: executing_member ${batch.executing_member} has no member issue`,
|
|
921
|
-
});
|
|
922
|
-
return releaseSlot(state, batchId, now);
|
|
973
|
+
return { ok: false, state: target.release(state) };
|
|
923
974
|
}
|
|
975
|
+
const memberIssue = target.issue;
|
|
924
976
|
const withStatus = advanceMemberToInWork(state, memberIssue, now);
|
|
925
977
|
const memberEntry = (0, state_1.findEntry)(withStatus, memberIssue);
|
|
926
978
|
// #771: a `review=full` member dispatches at `strong` minimum (within this
|
|
@@ -932,8 +984,8 @@ function spawnMember(deps, dispatch, state, slot, batchId, now, result, member)
|
|
|
932
984
|
// #677: the prompt carries THIS member's own worktree and the integration
|
|
933
985
|
// branch it was cut from — `member-cycle`'s `worktree`/`integration_branch`
|
|
934
986
|
// inputs. The shared batch worktree is no longer a member surface at all.
|
|
935
|
-
const prompt = (0, dispatch_1.buildMemberPrompt)(dispatch.memberPrompt, memberIssue, batchId, member.worktree, batch.branch, review);
|
|
936
|
-
const logFile = (0, dispatch_1.batchMemberLogPath)(deps.store.runsDir, batchId,
|
|
987
|
+
const prompt = (0, dispatch_1.buildMemberPrompt)(dispatch.memberPrompt, memberIssue, batchId, target.member.worktree, batch.branch, review);
|
|
988
|
+
const logFile = (0, dispatch_1.batchMemberLogPath)(deps.store.runsDir, batchId, target.index, memberIssue);
|
|
937
989
|
// #629: captured BEFORE spawning, mirroring `engine.ts`'s own
|
|
938
990
|
// `spawnAndRecord` — the log is per-role and append-mode, so the size at
|
|
939
991
|
// this instant is exactly where THIS dispatch's own output starts. Fences
|
|
@@ -949,8 +1001,8 @@ function spawnMember(deps, dispatch, state, slot, batchId, now, result, member)
|
|
|
949
1001
|
reason: 'spawn-error',
|
|
950
1002
|
detail: `member #${memberIssue} spawn failed: ${err.message}`,
|
|
951
1003
|
});
|
|
952
|
-
result.failed.push(
|
|
953
|
-
return
|
|
1004
|
+
result.failed.push(target.unitId);
|
|
1005
|
+
return { ok: false, state: target.release(withStatus) };
|
|
954
1006
|
}
|
|
955
1007
|
const patch = {
|
|
956
1008
|
pid,
|
|
@@ -963,18 +1015,9 @@ function spawnMember(deps, dispatch, state, slot, batchId, now, result, member)
|
|
|
963
1015
|
spawned_at: now.toISOString(),
|
|
964
1016
|
log_offset_at_spawn: logOffset,
|
|
965
1017
|
};
|
|
966
|
-
|
|
1018
|
+
const next = slot.status === 'assigned' || slot.status === 'recovering'
|
|
967
1019
|
? (0, state_1.transitionSlot)(withStatus, slot.id, 'running', patch, now)
|
|
968
1020
|
: withStatus;
|
|
969
|
-
// #677: persist the member's worktree context — a takeover redispatch or a
|
|
970
|
-
// `sched resume --batch` recheck after an engine restart must land in the
|
|
971
|
-
// SAME worktree/branch this prompt names, not re-derive (or worse,
|
|
972
|
-
// re-create) it.
|
|
973
|
-
next = (0, state_1.patchBatch)(next, batchId, {
|
|
974
|
-
member_branch: member.branch,
|
|
975
|
-
member_worktree: member.worktree,
|
|
976
|
-
member_pool_claimed: member.poolClaimed,
|
|
977
|
-
}, now);
|
|
978
1021
|
deps.journal.append((0, journal_1.unitEvent)('spawned', unit(batchId), {
|
|
979
1022
|
pid,
|
|
980
1023
|
tier,
|
|
@@ -983,11 +1026,11 @@ function spawnMember(deps, dispatch, state, slot, batchId, now, result, member)
|
|
|
983
1026
|
issue: memberIssue,
|
|
984
1027
|
...(0, dispatch_1.journalCmdModelFields)(spawnSpec),
|
|
985
1028
|
log: logFile,
|
|
986
|
-
worktree: member.worktree,
|
|
987
|
-
detail: `member ${
|
|
1029
|
+
worktree: target.member.worktree,
|
|
1030
|
+
detail: `member ${target.index}/${batch.members.length}${batch.member_dispatch === 'parallel' ? ' (parallel)' : ''}`,
|
|
988
1031
|
}), now);
|
|
989
|
-
result.spawned.push(
|
|
990
|
-
return next;
|
|
1032
|
+
result.spawned.push(target.unitId);
|
|
1033
|
+
return { ok: true, state: next };
|
|
991
1034
|
}
|
|
992
1035
|
/**
|
|
993
1036
|
* First claim of a `ready` batch: assign it a slot, run batch-setup (real
|
|
@@ -1054,7 +1097,38 @@ function claimAndSetup(deps, config, dispatch, batchId, now, result) {
|
|
|
1054
1097
|
pool_claimed: setup.poolClaimed ? 'true' : 'false',
|
|
1055
1098
|
},
|
|
1056
1099
|
});
|
|
1057
|
-
|
|
1100
|
+
// #809: decided ONCE, here, and persisted — a batch never changes mode mid-run.
|
|
1101
|
+
const mode = memberDispatchModeFor(deps.store.load(), batch, config);
|
|
1102
|
+
deps.journal.append((0, journal_1.unitEvent)('batch-setup-done', unit(batchId), {
|
|
1103
|
+
detail: `${setup.worktree} (members: ${mode})`,
|
|
1104
|
+
}), now);
|
|
1105
|
+
if (mode === 'parallel') {
|
|
1106
|
+
// Parallel members each claim their OWN slot (`batch:<id>#<issue>`), so
|
|
1107
|
+
// the setup slot is released here and the members' worktrees are prepared
|
|
1108
|
+
// by `spawnParallelMembers` (outside the lock, per member).
|
|
1109
|
+
deps.store.withLock((s) => {
|
|
1110
|
+
const b = (0, state_1.findBatch)(s, batchId);
|
|
1111
|
+
if (!b || b.status !== 'ready') {
|
|
1112
|
+
journalEvent(deps, 'unit-failed', unit(batchId), {
|
|
1113
|
+
reason: 'batch-left-ready-during-setup',
|
|
1114
|
+
detail: `worktree ${setup.worktree} created but batch is now '${b?.status ?? 'gone'}' — orphaned, manual cleanup required`,
|
|
1115
|
+
});
|
|
1116
|
+
return { state: releaseSlot(s, batchId, now), result: undefined };
|
|
1117
|
+
}
|
|
1118
|
+
let next = (0, state_1.patchBatch)(s, batchId, {
|
|
1119
|
+
branch: setup.branch,
|
|
1120
|
+
worktree: setup.worktree,
|
|
1121
|
+
run_id: setup.runId,
|
|
1122
|
+
pool_claimed: setup.poolClaimed,
|
|
1123
|
+
member_dispatch: 'parallel',
|
|
1124
|
+
member_runs: [],
|
|
1125
|
+
}, now);
|
|
1126
|
+
next = (0, state_1.transitionBatch)(next, batchId, 'executing', { executing_member: 0 }, now);
|
|
1127
|
+
return { state: releaseSlot(next, batchId, now), result: undefined };
|
|
1128
|
+
});
|
|
1129
|
+
spawnParallelMembers(deps, config, dispatch, batchId, now, result);
|
|
1130
|
+
return;
|
|
1131
|
+
}
|
|
1058
1132
|
// #677: prepare member 1's OWN worktree before claiming the spawn — real
|
|
1059
1133
|
// git/network work, so OUTSIDE the store lock (the same contract
|
|
1060
1134
|
// `runBatchSetup` just followed). All-or-nothing: a failed prep posts the
|
|
@@ -1114,6 +1188,7 @@ function claimAndSetup(deps, config, dispatch, batchId, now, result) {
|
|
|
1114
1188
|
worktree: setup.worktree,
|
|
1115
1189
|
run_id: setup.runId,
|
|
1116
1190
|
pool_claimed: setup.poolClaimed,
|
|
1191
|
+
member_dispatch: 'serial',
|
|
1117
1192
|
}, now);
|
|
1118
1193
|
next = (0, state_1.transitionBatch)(next, batchId, 'executing', { executing_member: 1 }, now);
|
|
1119
1194
|
const slot = slotFor(next, batchId);
|
|
@@ -1369,14 +1444,20 @@ function runValidate(deps, config, dispatch, batchId, now, result) {
|
|
|
1369
1444
|
return { state: s, result: false };
|
|
1370
1445
|
if (b.executing_member < b.members.length) {
|
|
1371
1446
|
return {
|
|
1372
|
-
state: (0, state_1.transitionBatch)(s, batchId, 'executing',
|
|
1447
|
+
state: (0, state_1.transitionBatch)(s, batchId, 'executing',
|
|
1448
|
+
// #809: a parallel batch spawns every run-less member itself.
|
|
1449
|
+
isParallelBatch(b) ? {} : { executing_member: b.executing_member + 1 }, now),
|
|
1373
1450
|
result: true,
|
|
1374
1451
|
};
|
|
1375
1452
|
}
|
|
1376
1453
|
return { state: (0, state_1.transitionBatch)(s, batchId, 'reviewing', {}, now), result: undefined };
|
|
1377
1454
|
});
|
|
1378
1455
|
if (admitted) {
|
|
1379
|
-
|
|
1456
|
+
const b = (0, state_1.findBatch)(deps.store.load(), batchId);
|
|
1457
|
+
if (b && isParallelBatch(b))
|
|
1458
|
+
spawnParallelMembers(deps, config, dispatch, batchId, now, result);
|
|
1459
|
+
else
|
|
1460
|
+
spawnMemberContinuation(deps, config, dispatch, batchId, now, result);
|
|
1380
1461
|
return;
|
|
1381
1462
|
}
|
|
1382
1463
|
spawnTailAgent(deps, config, dispatch, batchId, now, result);
|
|
@@ -1487,7 +1568,7 @@ function evictOffender(deps, config, batchId, offender, attribution, now, result
|
|
|
1487
1568
|
return { state: s, result: undefined };
|
|
1488
1569
|
if (b.executing_member < b.members.length) {
|
|
1489
1570
|
return {
|
|
1490
|
-
state: (0, state_1.transitionBatch)(s, batchId, 'executing', { executing_member: b.executing_member + 1 }, now),
|
|
1571
|
+
state: (0, state_1.transitionBatch)(s, batchId, 'executing', isParallelBatch(b) ? {} : { executing_member: b.executing_member + 1 }, now),
|
|
1491
1572
|
result: undefined,
|
|
1492
1573
|
};
|
|
1493
1574
|
}
|
|
@@ -1535,6 +1616,15 @@ function journalAdvanceSkipped(deps, batchId, memberIssue, currentMember, claim)
|
|
|
1535
1616
|
});
|
|
1536
1617
|
}
|
|
1537
1618
|
function advanceMemberOrValidate(deps, config, dispatch, batchId, memberCount, currentMember, memberIssue, now, result) {
|
|
1619
|
+
// #809: a parallel batch reaches this serial rail only through `sched resume
|
|
1620
|
+
// --batch` resolving its gate-inconclusive member — hand back to the
|
|
1621
|
+
// parallel rail instead of advancing a serial member pointer.
|
|
1622
|
+
const current = (0, state_1.findBatch)(deps.store.load(), batchId);
|
|
1623
|
+
if (current && isParallelBatch(current)) {
|
|
1624
|
+
settleParallelResume(deps, batchId, memberIssue, now);
|
|
1625
|
+
advanceParallelBatch(deps, config, dispatch, batchId, now, result);
|
|
1626
|
+
return;
|
|
1627
|
+
}
|
|
1538
1628
|
const isLast = currentMember >= memberCount;
|
|
1539
1629
|
if (isLast) {
|
|
1540
1630
|
const claim = claimMemberResolution(deps, batchId, currentMember, now, () => ({
|
|
@@ -1639,7 +1729,41 @@ function runIncrementalGate(deps, config, dispatch, batchId, batch, memberIssue,
|
|
|
1639
1729
|
// the fallback for a batch dispatched before member worktrees existed (its
|
|
1640
1730
|
// members' commits landed directly on the integration branch).
|
|
1641
1731
|
const worktree = batch.member_worktree ?? batch.worktree;
|
|
1732
|
+
const verdict = evaluateIncrementalGate(deps, batchId, worktree, memberIssue, now);
|
|
1733
|
+
if (verdict.kind === 'pass')
|
|
1734
|
+
return false;
|
|
1735
|
+
if (verdict.kind === 'failed') {
|
|
1736
|
+
deps.store.withLock((s) => ({ state: releaseSlot(s, batchId, now), result: undefined }));
|
|
1737
|
+
evictMemberAndContinue(deps, config, dispatch, batchId, batch, memberIssue, { reason: verdict.reason, detail: verdict.detail }, now, result);
|
|
1738
|
+
return true;
|
|
1739
|
+
}
|
|
1740
|
+
deps.store.withLock((s) => ({ state: releaseSlot(s, batchId, now), result: undefined }));
|
|
1741
|
+
// #677: F.11's block semantics are "the member's commit stays on the
|
|
1742
|
+
// branch, nothing requeued/reverted" — under member worktrees that is
|
|
1743
|
+
// true only if the member LANDS before the batch blocks. The block means
|
|
1744
|
+
// the gate could not reach a verdict, not that the work is bad, so the
|
|
1745
|
+
// verified member joins the integration branch and is there for an
|
|
1746
|
+
// operator (or the #686 out-of-band reconcile) to act on. A landing
|
|
1747
|
+
// failure blocks the batch with its own reason — same operator outcome.
|
|
1748
|
+
const landed = landMemberBranch(deps, batchId, now);
|
|
1749
|
+
if (!landed.ok) {
|
|
1750
|
+
blockLandingFailure(deps, config, batch, landed.reason, now, result);
|
|
1751
|
+
return true;
|
|
1752
|
+
}
|
|
1753
|
+
blockBatchForOperator(deps, config, batchId, { reason: verdict.reason, milestonePhase: 'batch-review' }, now, result);
|
|
1754
|
+
return true;
|
|
1755
|
+
}
|
|
1756
|
+
/**
|
|
1757
|
+
* Run the incremental gate's capabilities against one member worktree and
|
|
1758
|
+
* reach a verdict — journaling, gate logs and `member_gates` included, but NO
|
|
1759
|
+
* state action: serial (`runIncrementalGate`) and parallel
|
|
1760
|
+
* (`reconcileParallelMemberSlot`, #809) members act on the verdict
|
|
1761
|
+
* differently. See `runIncrementalGate`'s doc for the policy.
|
|
1762
|
+
*/
|
|
1763
|
+
function evaluateIncrementalGate(deps, batchId, worktree, memberIssue, now) {
|
|
1642
1764
|
const runCapability = deps.runCapability;
|
|
1765
|
+
if (!runCapability)
|
|
1766
|
+
return { kind: 'pass' };
|
|
1643
1767
|
const gateResults = ['typecheck.run', 'test.focused'].map((id) => ({
|
|
1644
1768
|
id,
|
|
1645
1769
|
...runCapability(worktree, id),
|
|
@@ -1680,13 +1804,10 @@ function runIncrementalGate(deps, config, dispatch, batchId, batch, memberIssue,
|
|
|
1680
1804
|
// trap (#594's shape — absence reading as a verdict). Silence must never
|
|
1681
1805
|
// be mistaken for a pass.
|
|
1682
1806
|
//
|
|
1683
|
-
// #632: confirmed this fires once per member, not once per tick
|
|
1684
|
-
//
|
|
1685
|
-
//
|
|
1686
|
-
//
|
|
1687
|
-
// `batch.executing_member` before returning — so the very next tick
|
|
1688
|
-
// reads a DIFFERENT `memberIssue` from `batch.members[executing_member -
|
|
1689
|
-
// 1]`, and this gate never runs twice against the same member.
|
|
1807
|
+
// #632: confirmed this fires once per member, not once per tick: the
|
|
1808
|
+
// gate runs only on a member dispatch's verified completion, and both
|
|
1809
|
+
// rails resolve that member (serial: the pointer advances; parallel:
|
|
1810
|
+
// the run leaves `running` and its slot is released) before returning.
|
|
1690
1811
|
journalEvent(deps, 'gate-skipped', unit(batchId), {
|
|
1691
1812
|
issue: memberIssue,
|
|
1692
1813
|
reason: `gate-skipped:${skipped.id}`,
|
|
@@ -1734,15 +1855,13 @@ function runIncrementalGate(deps, config, dispatch, batchId, batch, memberIssue,
|
|
|
1734
1855
|
recordMemberGate(deps, batchId, memberIssue, decisiveGate, now);
|
|
1735
1856
|
}
|
|
1736
1857
|
if (earnedFailure) {
|
|
1737
|
-
const reason = `incremental-gate-failed:${earnedFailure.id}`;
|
|
1738
1858
|
writeGateLog(deps, batchId, earnedFailure.id, memberIssue, earnedFailure.outputTail);
|
|
1739
1859
|
const excerpt = gateDetailExcerpt(earnedFailure.outputTail, earnedFailure.reason);
|
|
1740
|
-
|
|
1741
|
-
|
|
1742
|
-
reason
|
|
1860
|
+
return {
|
|
1861
|
+
kind: 'failed',
|
|
1862
|
+
reason: `incremental-gate-failed:${earnedFailure.id}`,
|
|
1743
1863
|
detail: withExcerpt(`cap run ${earnedFailure.id} reported task-failed with failing-test evidence after member review done`, excerpt),
|
|
1744
|
-
}
|
|
1745
|
-
return true;
|
|
1864
|
+
};
|
|
1746
1865
|
}
|
|
1747
1866
|
const inconclusive = gateInconclusive ?? unevidencedFailure;
|
|
1748
1867
|
if (inconclusive) {
|
|
@@ -1758,23 +1877,9 @@ function runIncrementalGate(deps, config, dispatch, batchId, batch, memberIssue,
|
|
|
1758
1877
|
reason,
|
|
1759
1878
|
detail: withExcerpt(`cap run ${inconclusive.id} ${describeInconclusive(inconclusive)} after member review done`, excerpt),
|
|
1760
1879
|
});
|
|
1761
|
-
|
|
1762
|
-
// #677: F.11's block semantics are "the member's commit stays on the
|
|
1763
|
-
// branch, nothing requeued/reverted" — under member worktrees that is
|
|
1764
|
-
// true only if the member LANDS before the batch blocks. The block means
|
|
1765
|
-
// the gate could not reach a verdict, not that the work is bad, so the
|
|
1766
|
-
// verified member joins the integration branch and is there for an
|
|
1767
|
-
// operator (or the #686 out-of-band reconcile) to act on. A landing
|
|
1768
|
-
// failure blocks the batch with its own reason — same operator outcome.
|
|
1769
|
-
const landed = landMemberBranch(deps, batchId, now);
|
|
1770
|
-
if (!landed.ok) {
|
|
1771
|
-
blockLandingFailure(deps, config, batch, landed.reason, now, result);
|
|
1772
|
-
return true;
|
|
1773
|
-
}
|
|
1774
|
-
blockBatchForOperator(deps, config, batchId, { reason, milestonePhase: 'batch-review' }, now, result);
|
|
1775
|
-
return true;
|
|
1880
|
+
return { kind: 'inconclusive', reason, capability: inconclusive.id };
|
|
1776
1881
|
}
|
|
1777
|
-
return
|
|
1882
|
+
return { kind: 'pass' };
|
|
1778
1883
|
}
|
|
1779
1884
|
/**
|
|
1780
1885
|
* The member is verifiably done — self-reported complete AND (when checked)
|
|
@@ -1988,7 +2093,9 @@ function resumeBlockedGate(deps, config, dispatch, batchId, now) {
|
|
|
1988
2093
|
* instead), so unlike `engine.ts`'s per-unit log, a member's log file is
|
|
1989
2094
|
* always one-shot and reading from offset 0 is always correct.
|
|
1990
2095
|
*/
|
|
1991
|
-
function recordMemberRunLog(deps, dispatch, state, batchId,
|
|
2096
|
+
function recordMemberRunLog(deps, dispatch, state, batchId,
|
|
2097
|
+
/** 1-based member index — names the per-member log file (serial: `executing_member`; parallel: `MemberRun.index`). */
|
|
2098
|
+
memberIndex, memberIssue, slot, now) {
|
|
1992
2099
|
if (slot.status !== 'running' || slot.spawned_at === null) {
|
|
1993
2100
|
journalEvent(deps, 'run-log-skipped', unit(batchId), {
|
|
1994
2101
|
issue: memberIssue,
|
|
@@ -2002,7 +2109,7 @@ function recordMemberRunLog(deps, dispatch, state, batchId, batch, memberIssue,
|
|
|
2002
2109
|
const memberEntry = (0, state_1.findEntry)(state, memberIssue);
|
|
2003
2110
|
const tier = memberEntry ? (0, dispatch_1.memberDispatchTier)(memberEntry) : 'mid';
|
|
2004
2111
|
const { cmd, model } = (0, dispatch_1.resolveTierSpawn)(dispatch, tier, memberIssue);
|
|
2005
|
-
const logFile = (0, dispatch_1.batchMemberLogPath)(deps.store.runsDir, batchId,
|
|
2112
|
+
const logFile = (0, dispatch_1.batchMemberLogPath)(deps.store.runsDir, batchId, memberIndex, memberIssue);
|
|
2006
2113
|
// #629: fenced to THIS dispatch's slice (`log_offset_at_spawn`, stamped by
|
|
2007
2114
|
// `spawnMember`/`spawnMemberContinuation` at spawn time) rather than a
|
|
2008
2115
|
// fixed byte-0 read. Before #629 a member log never outlived one attempt
|
|
@@ -2070,14 +2177,12 @@ function terminalMilestoneForDispatch(deps, memberIssue, slot) {
|
|
|
2070
2177
|
}
|
|
2071
2178
|
return latest;
|
|
2072
2179
|
}
|
|
2073
|
-
|
|
2074
|
-
|
|
2075
|
-
|
|
2076
|
-
|
|
2077
|
-
|
|
2078
|
-
|
|
2079
|
-
if (memberIssue === undefined)
|
|
2080
|
-
return;
|
|
2180
|
+
/**
|
|
2181
|
+
* Read one member dispatch's fate: the fenced terminal milestone (#575/#605/
|
|
2182
|
+
* #622) plus liveness, journaling a stale terminal milestone once per
|
|
2183
|
+
* dispatch (#610). Pure reading — the caller acts on the result.
|
|
2184
|
+
*/
|
|
2185
|
+
function readMemberDispatch(deps, batchId, memberIssue, slot, now) {
|
|
2081
2186
|
const dead = slot.pid !== null && !deps.spawnDeps.isAlive(slot.pid, slot.pid_start ?? undefined);
|
|
2082
2187
|
// #622: the milestone this DISPATCH's fate should be read from — the last
|
|
2083
2188
|
// TERMINAL one it posted, not merely the newest one on the issue.
|
|
@@ -2096,7 +2201,7 @@ function reconcileMemberSlot(deps, config, dispatch, batchId, slot, now, result)
|
|
|
2096
2201
|
// poll) this degrades to the previous latest-only behaviour.
|
|
2097
2202
|
const milestone = terminalMilestoneForDispatch(deps, memberIssue, slot);
|
|
2098
2203
|
if (milestone === undefined)
|
|
2099
|
-
return
|
|
2204
|
+
return { kind: 'unreachable' }; // pause this batch's decisions
|
|
2100
2205
|
// #575 / #605: fence to THIS member dispatch's `spawned_at` — a member
|
|
2101
2206
|
// re-added to a fresh batch run after a PREVIOUS batch already posted a
|
|
2102
2207
|
// terminal `mode=slot` milestone (pilot re-run, requeue-with-context, and
|
|
@@ -2135,7 +2240,61 @@ function reconcileMemberSlot(deps, config, dispatch, batchId, slot, now, result)
|
|
|
2135
2240
|
result: undefined,
|
|
2136
2241
|
}));
|
|
2137
2242
|
}
|
|
2138
|
-
if ((0, groundtruth_1.isMemberComplete)(milestone, slot.spawned_at))
|
|
2243
|
+
if ((0, groundtruth_1.isMemberComplete)(milestone, slot.spawned_at))
|
|
2244
|
+
return { kind: 'complete' };
|
|
2245
|
+
return {
|
|
2246
|
+
kind: 'pending',
|
|
2247
|
+
milestone,
|
|
2248
|
+
dead,
|
|
2249
|
+
blockedNow: (0, groundtruth_1.isMemberBlocked)(milestone, slot.spawned_at),
|
|
2250
|
+
};
|
|
2251
|
+
}
|
|
2252
|
+
/**
|
|
2253
|
+
* The eviction a blocked-or-dead member dispatch earns (shared by both rails).
|
|
2254
|
+
* #605: only a milestone THIS dispatch posted may name the reason. A dead
|
|
2255
|
+
* agent whose issue carries only a stale `blocked` milestone is an unverified
|
|
2256
|
+
* exit, not a re-run of the previous run's block — reporting the old
|
|
2257
|
+
* `reason=` here is what made the journal misdirect, naming a precondition
|
|
2258
|
+
* that had already been fixed on a run where it no longer applied.
|
|
2259
|
+
* #804: `memberBlockedReason` also names a `review partial` hand-back.
|
|
2260
|
+
*/
|
|
2261
|
+
function memberFailureFor(read, lastTool) {
|
|
2262
|
+
const rawReason = read.blockedNow && read.milestone ? (0, groundtruth_1.memberBlockedReason)(read.milestone) : undefined;
|
|
2263
|
+
const reason = typeof rawReason === 'string' && rawReason.length > 0
|
|
2264
|
+
? sanitizeUntrustedText(rawReason)
|
|
2265
|
+
: read.dead
|
|
2266
|
+
? 'agent-exited-unverified'
|
|
2267
|
+
: 'member-blocked';
|
|
2268
|
+
return {
|
|
2269
|
+
reason,
|
|
2270
|
+
// This rail covers a self-reported block AND a member that simply died, so the
|
|
2271
|
+
// detail must follow the RESOLVED reason — `member blocked` on an
|
|
2272
|
+
// `agent-exited-unverified` eviction contradicts the reason on its own journal line.
|
|
2273
|
+
detail: reason === 'agent-exited-unverified'
|
|
2274
|
+
? 'member agent exited without posting a terminal milestone'
|
|
2275
|
+
: 'member blocked',
|
|
2276
|
+
// #591: attributes an unverified member exit to a concrete cause (e.g.
|
|
2277
|
+
// `Monitor`) without opening the transcript. Gated on the RESOLVED reason, not
|
|
2278
|
+
// `dead` — a member can be simultaneously `dead` AND carry a milestone-posted
|
|
2279
|
+
// `reason` (it posted `blocked` and then exited), and that real block has no
|
|
2280
|
+
// log-derived cause to attribute; only `agent-exited-unverified` does.
|
|
2281
|
+
extraKv: reason === 'agent-exited-unverified' && lastTool !== null
|
|
2282
|
+
? { last_tool: lastTool }
|
|
2283
|
+
: undefined,
|
|
2284
|
+
};
|
|
2285
|
+
}
|
|
2286
|
+
function reconcileMemberSlot(deps, config, dispatch, batchId, slot, now, result) {
|
|
2287
|
+
const state0 = deps.store.load();
|
|
2288
|
+
const batch = (0, state_1.findBatch)(state0, batchId);
|
|
2289
|
+
if (!batch)
|
|
2290
|
+
return;
|
|
2291
|
+
const memberIssue = batch.members[batch.executing_member - 1];
|
|
2292
|
+
if (memberIssue === undefined)
|
|
2293
|
+
return;
|
|
2294
|
+
const read = readMemberDispatch(deps, batchId, memberIssue, slot, now);
|
|
2295
|
+
if (read.kind === 'unreachable')
|
|
2296
|
+
return;
|
|
2297
|
+
if (read.kind === 'complete') {
|
|
2139
2298
|
deps.journal.append((0, journal_1.unitEvent)('external-advance', unit(batchId), {
|
|
2140
2299
|
issue: memberIssue,
|
|
2141
2300
|
detail: 'member review done',
|
|
@@ -2143,7 +2302,7 @@ function reconcileMemberSlot(deps, config, dispatch, batchId, slot, now, result)
|
|
|
2143
2302
|
// The member's own agent process is done regardless of what the
|
|
2144
2303
|
// incremental gate below decides — record its telemetry once here (#564)
|
|
2145
2304
|
// rather than at each of this branch's two later exit points.
|
|
2146
|
-
recordMemberRunLog(deps, dispatch, state0, batchId, batch, memberIssue, slot, now);
|
|
2305
|
+
recordMemberRunLog(deps, dispatch, state0, batchId, batch.executing_member, memberIssue, slot, now);
|
|
2147
2306
|
// Incremental gate (#523 AC2, revised #583) — see `runIncrementalGate`'s
|
|
2148
2307
|
// own doc comment for the three-way policy. A `true` return means the
|
|
2149
2308
|
// gate already decided the member's fate (evicted or blocked) and
|
|
@@ -2155,9 +2314,8 @@ function reconcileMemberSlot(deps, config, dispatch, batchId, slot, now, result)
|
|
|
2155
2314
|
completeMemberGate(deps, config, dispatch, batchId, batch, memberIssue, now, result);
|
|
2156
2315
|
return;
|
|
2157
2316
|
}
|
|
2158
|
-
|
|
2159
|
-
|
|
2160
|
-
!blockedNow &&
|
|
2317
|
+
if (read.dead &&
|
|
2318
|
+
!read.blockedNow &&
|
|
2161
2319
|
handleDeadDispatchApiError(deps, batchId, (0, dispatch_1.batchMemberLogPath)(deps.store.runsDir, batchId, batch.executing_member, memberIssue), slot.log_offset_at_spawn ?? 0, now)) {
|
|
2162
2320
|
// #629: a confirmed provider API error is not a real member failure —
|
|
2163
2321
|
// evicting the member for an account-wide spend wall would silently
|
|
@@ -2166,21 +2324,8 @@ function reconcileMemberSlot(deps, config, dispatch, batchId, slot, now, result)
|
|
|
2166
2324
|
// place, keeping its position.
|
|
2167
2325
|
return;
|
|
2168
2326
|
}
|
|
2169
|
-
if (blockedNow || dead) {
|
|
2170
|
-
|
|
2171
|
-
// agent whose issue carries only a stale `blocked` milestone is an
|
|
2172
|
-
// unverified exit, not a re-run of the previous run's block — reporting
|
|
2173
|
-
// the old `reason=` here is what made the journal misdirect, naming a
|
|
2174
|
-
// precondition that had already been fixed on a run where it no longer
|
|
2175
|
-
// applied.
|
|
2176
|
-
// #804: `memberBlockedReason` also names a `review partial` hand-back.
|
|
2177
|
-
const rawReason = blockedNow && milestone ? (0, groundtruth_1.memberBlockedReason)(milestone) : undefined;
|
|
2178
|
-
const reason = typeof rawReason === 'string' && rawReason.length > 0
|
|
2179
|
-
? sanitizeUntrustedText(rawReason)
|
|
2180
|
-
: dead
|
|
2181
|
-
? 'agent-exited-unverified'
|
|
2182
|
-
: 'member-blocked';
|
|
2183
|
-
const lastTool = recordMemberRunLog(deps, dispatch, state0, batchId, batch, memberIssue, slot, now);
|
|
2327
|
+
if (read.blockedNow || read.dead) {
|
|
2328
|
+
const lastTool = recordMemberRunLog(deps, dispatch, state0, batchId, batch.executing_member, memberIssue, slot, now);
|
|
2184
2329
|
deps.store.withLock((s) => ({ state: releaseSlot(s, batchId, now), result: undefined }));
|
|
2185
2330
|
// A member that never went green (RFC F.1) evicts DIRECTLY — no aggregate
|
|
2186
2331
|
// suite has run yet, so there is nothing for `attributing`/`evicting` (the
|
|
@@ -2190,23 +2335,7 @@ function reconcileMemberSlot(deps, config, dispatch, batchId, slot, now, result)
|
|
|
2190
2335
|
// attributing → evicting` is not even a legal edge from mid-`executing`
|
|
2191
2336
|
// (BATCH_TRANSITIONS has no `executing → evicting`) — this stays entirely
|
|
2192
2337
|
// within `executing`/`dissolving`, both of which ARE legal from here.
|
|
2193
|
-
evictMemberAndContinue(deps, config, dispatch, batchId, batch, memberIssue,
|
|
2194
|
-
reason,
|
|
2195
|
-
// This rail covers a self-reported block AND a member that simply died, so the
|
|
2196
|
-
// detail must follow the RESOLVED reason — `member blocked` on an
|
|
2197
|
-
// `agent-exited-unverified` eviction contradicts the reason on its own journal line.
|
|
2198
|
-
detail: reason === 'agent-exited-unverified'
|
|
2199
|
-
? 'member agent exited without posting a terminal milestone'
|
|
2200
|
-
: 'member blocked',
|
|
2201
|
-
// #591: attributes an unverified member exit to a concrete cause (e.g.
|
|
2202
|
-
// `Monitor`) without opening the transcript. Gated on the RESOLVED reason, not
|
|
2203
|
-
// `dead` — a member can be simultaneously `dead` AND carry a milestone-posted
|
|
2204
|
-
// `reason` (it posted `blocked` and then exited), and that real block has no
|
|
2205
|
-
// log-derived cause to attribute; only `agent-exited-unverified` does.
|
|
2206
|
-
extraKv: reason === 'agent-exited-unverified' && lastTool !== null
|
|
2207
|
-
? { last_tool: lastTool }
|
|
2208
|
-
: undefined,
|
|
2209
|
-
}, now, result);
|
|
2338
|
+
evictMemberAndContinue(deps, config, dispatch, batchId, batch, memberIssue, memberFailureFor(read, lastTool), now, result);
|
|
2210
2339
|
}
|
|
2211
2340
|
}
|
|
2212
2341
|
/**
|
|
@@ -3007,7 +3136,9 @@ function teardownBatch(deps, batchId) {
|
|
|
3007
3136
|
const terminal = (0, state_1.findBatch)(deps.store.load(), batchId);
|
|
3008
3137
|
const lastMemberLanded = terminal?.ranges.some((r) => r.issue === terminal.members[terminal.executing_member - 1]) ===
|
|
3009
3138
|
true;
|
|
3139
|
+
const serialTree = terminal?.member_worktree ?? null;
|
|
3010
3140
|
teardownMemberWorktree(deps, batchId, deps.now(), lastMemberLanded);
|
|
3141
|
+
teardownParallelRuns(deps, batchId, serialTree);
|
|
3011
3142
|
const state = deps.store.load();
|
|
3012
3143
|
const batch = (0, state_1.findBatch)(state, batchId);
|
|
3013
3144
|
if (!batch || batch.worktree === null)
|
|
@@ -3031,6 +3162,615 @@ function teardownBatch(deps, batchId) {
|
|
|
3031
3162
|
worktree: batch.worktree,
|
|
3032
3163
|
});
|
|
3033
3164
|
}
|
|
3165
|
+
// --- Parallel member dispatch (#809) ---
|
|
3166
|
+
/**
|
|
3167
|
+
* How a batch should dispatch its members (#809) — decided once, at the
|
|
3168
|
+
* `ready → executing` claim, and persisted on `BatchEntry.member_dispatch`.
|
|
3169
|
+
*
|
|
3170
|
+
* Serial when the operator set `member_parallelism: 1`, or the batch itself
|
|
3171
|
+
* says its members are not independent: an eviction group (members that must
|
|
3172
|
+
* revert together — e.g. one built on another's API) or a member whose `deps`
|
|
3173
|
+
* names another member (it must see that member's landed work before it is
|
|
3174
|
+
* cut). Parallel otherwise — the batch-cycle skill's and prep dossier's
|
|
3175
|
+
* design (RFC-0001 §J.3: each member works off the integration branch and
|
|
3176
|
+
* sees no one else's changes until the parent lands them).
|
|
3177
|
+
*/
|
|
3178
|
+
/** @internal exported for tests. */
|
|
3179
|
+
function memberDispatchModeFor(state, batch, config) {
|
|
3180
|
+
if (config.member_parallelism === 1)
|
|
3181
|
+
return 'serial';
|
|
3182
|
+
if (batch.eviction_groups.some((group) => group.length > 1))
|
|
3183
|
+
return 'serial';
|
|
3184
|
+
const members = new Set(batch.members);
|
|
3185
|
+
for (const issue of batch.members) {
|
|
3186
|
+
if ((0, state_1.findEntry)(state, issue)?.deps.some((dep) => members.has(dep)))
|
|
3187
|
+
return 'serial';
|
|
3188
|
+
}
|
|
3189
|
+
return 'parallel';
|
|
3190
|
+
}
|
|
3191
|
+
/** Whether `batch` runs the parallel member rail — `null` (pre-#809, or not yet claimed) is serial. */
|
|
3192
|
+
function isParallelBatch(batch) {
|
|
3193
|
+
return batch.member_dispatch === 'parallel';
|
|
3194
|
+
}
|
|
3195
|
+
/** At most this many members of one batch run at once (#809) — `max_slots` when unset. */
|
|
3196
|
+
function memberParallelism(config) {
|
|
3197
|
+
// Both are validated ≥ 1 at config load (`member_parallelism`, `max_slots`).
|
|
3198
|
+
return config.member_parallelism ?? config.max_slots;
|
|
3199
|
+
}
|
|
3200
|
+
/** The teardown target of a member run (or a freshly prepared member tree). */
|
|
3201
|
+
function runTree(r) {
|
|
3202
|
+
return 'pool_claimed' in r
|
|
3203
|
+
? { worktree: r.worktree, branch: r.branch, poolClaimed: r.pool_claimed }
|
|
3204
|
+
: { worktree: r.worktree, branch: r.branch, poolClaimed: r.poolClaimed };
|
|
3205
|
+
}
|
|
3206
|
+
/**
|
|
3207
|
+
* Record that a run's tree is gone. Written AFTER the teardown, so a crash in
|
|
3208
|
+
* between re-runs the (idempotent) teardown from `teardownParallelRuns`
|
|
3209
|
+
* instead of leaking the tree.
|
|
3210
|
+
*/
|
|
3211
|
+
function markTornDown(deps, batchId, issue, now) {
|
|
3212
|
+
deps.store.withLock((s) => ({
|
|
3213
|
+
state: patchRun(s, batchId, issue, { torn_down: true }, now),
|
|
3214
|
+
result: undefined,
|
|
3215
|
+
}));
|
|
3216
|
+
}
|
|
3217
|
+
/** Merge `patch` into member `issue`'s run record (no-op when it has none). */
|
|
3218
|
+
function patchRun(state, batchId, issue, patch, now) {
|
|
3219
|
+
const batch = (0, state_1.findBatch)(state, batchId);
|
|
3220
|
+
if (!batch || !batch.member_runs.some((r) => r.issue === issue))
|
|
3221
|
+
return state;
|
|
3222
|
+
return (0, state_1.patchBatch)(state, batchId, { member_runs: batch.member_runs.map((r) => (r.issue === issue ? { ...r, ...patch } : r)) }, now);
|
|
3223
|
+
}
|
|
3224
|
+
/** Insert or replace member `run.issue`'s run record. */
|
|
3225
|
+
function upsertRun(state, batchId, run, now) {
|
|
3226
|
+
const batch = (0, state_1.findBatch)(state, batchId);
|
|
3227
|
+
if (!batch)
|
|
3228
|
+
return state;
|
|
3229
|
+
const others = batch.member_runs.filter((r) => r.issue !== run.issue);
|
|
3230
|
+
return (0, state_1.patchBatch)(state, batchId, { member_runs: [...others, run].sort((a, b) => a.index - b.index) }, now);
|
|
3231
|
+
}
|
|
3232
|
+
/**
|
|
3233
|
+
* Whether `issue` is still an active member of `batch` — not evicted, and its
|
|
3234
|
+
* queue entry (when it has one) still a live slot-mode entry of THIS batch. An
|
|
3235
|
+
* evicted member's entry is requeued `mode: 'full', batch: null`, so both
|
|
3236
|
+
* checks agree; the entry check also covers a member stopped or abandoned out
|
|
3237
|
+
* from under the batch.
|
|
3238
|
+
*/
|
|
3239
|
+
function memberStillInBatch(state, batch, issue) {
|
|
3240
|
+
if (batch.evictions.some((e) => e.issue === issue))
|
|
3241
|
+
return false;
|
|
3242
|
+
const entry = (0, state_1.findEntry)(state, issue);
|
|
3243
|
+
if (entry === undefined)
|
|
3244
|
+
return true;
|
|
3245
|
+
return (entry.mode === 'slot' && entry.batch === batch.id && !types_1.TERMINAL_ISSUE_STATUSES.has(entry.status));
|
|
3246
|
+
}
|
|
3247
|
+
/** Whether member `issue`'s own slot currently holds a live agent. */
|
|
3248
|
+
function hasLiveMemberSlot(state, batchId, issue) {
|
|
3249
|
+
const slot = (0, state_1.slotForBatchMember)(state, batchId, issue);
|
|
3250
|
+
return slot !== undefined && types_1.LIVE_SLOT_STATUSES.has(slot.status);
|
|
3251
|
+
}
|
|
3252
|
+
/**
|
|
3253
|
+
* The next member to (re)spawn: first a run still `running` that holds no slot
|
|
3254
|
+
* (an api-error hold released it, its spawn threw, or the engine restarted) —
|
|
3255
|
+
* respawned in place, in its own worktree — then the first member in order
|
|
3256
|
+
* that has no run yet.
|
|
3257
|
+
*/
|
|
3258
|
+
function nextParallelCandidate(state, batch, attempted) {
|
|
3259
|
+
for (const run of batch.member_runs) {
|
|
3260
|
+
if (run.status === 'running' &&
|
|
3261
|
+
!attempted.has(run.issue) &&
|
|
3262
|
+
(0, state_1.slotForBatchMember)(state, batch.id, run.issue) === undefined &&
|
|
3263
|
+
memberStillInBatch(state, batch, run.issue)) {
|
|
3264
|
+
return { issue: run.issue, index: run.index, run };
|
|
3265
|
+
}
|
|
3266
|
+
}
|
|
3267
|
+
for (const [i, issue] of batch.members.entries()) {
|
|
3268
|
+
if (attempted.has(issue) || batch.member_runs.some((r) => r.issue === issue))
|
|
3269
|
+
continue;
|
|
3270
|
+
if (!memberStillInBatch(state, batch, issue))
|
|
3271
|
+
continue;
|
|
3272
|
+
return { issue, index: i + 1 };
|
|
3273
|
+
}
|
|
3274
|
+
return undefined;
|
|
3275
|
+
}
|
|
3276
|
+
/**
|
|
3277
|
+
* Fill the batch's member concurrency: spawn members into their OWN slots
|
|
3278
|
+
* (`batch:<id>#<issue>`) while fewer than `member_parallelism` run and the
|
|
3279
|
+
* scheduler has free capacity. Each gets its own worktree off the integration
|
|
3280
|
+
* branch (`prepareMemberWorktree`, outside the lock) — the integration branch
|
|
3281
|
+
* does not move until members land, so concurrent members are all cut from the
|
|
3282
|
+
* same base. A member whose worktree cannot be prepared is evicted (requeued
|
|
3283
|
+
* full-cycle, which does its own setup) rather than retried every tick in
|
|
3284
|
+
* front of the members behind it.
|
|
3285
|
+
*/
|
|
3286
|
+
function spawnParallelMembers(deps, config, dispatch, batchId, now, result) {
|
|
3287
|
+
const limit = memberParallelism(config);
|
|
3288
|
+
const fsExists = deps.fsExists ?? ((p) => fs.existsSync(p));
|
|
3289
|
+
const attempted = new Set();
|
|
3290
|
+
// Bounded by the member count: every iteration marks one member attempted.
|
|
3291
|
+
for (;;) {
|
|
3292
|
+
const state = deps.store.load();
|
|
3293
|
+
const batch = (0, state_1.findBatch)(state, batchId);
|
|
3294
|
+
if (!batch || batch.status !== 'executing' || !isParallelBatch(batch))
|
|
3295
|
+
return;
|
|
3296
|
+
const live = batch.member_runs.filter((r) => hasLiveMemberSlot(state, batchId, r.issue)).length;
|
|
3297
|
+
if (live >= limit || (0, scheduler_1.freeCapacity)(state, config) === 0)
|
|
3298
|
+
return;
|
|
3299
|
+
const next = nextParallelCandidate(state, batch, attempted);
|
|
3300
|
+
if (next === undefined)
|
|
3301
|
+
return;
|
|
3302
|
+
attempted.add(next.issue);
|
|
3303
|
+
let member;
|
|
3304
|
+
const reusable = next.run !== undefined &&
|
|
3305
|
+
fsExists(next.run.worktree) &&
|
|
3306
|
+
(next.run.pool_claimed ||
|
|
3307
|
+
(0, teardown_1.isSafeWorktree)(path.resolve(deps.exec('git', ['rev-parse', '--show-toplevel'], deps.repoDir) ?? deps.repoDir), next.run.worktree));
|
|
3308
|
+
if (next.run !== undefined && reusable) {
|
|
3309
|
+
member = {
|
|
3310
|
+
branch: next.run.branch,
|
|
3311
|
+
worktree: next.run.worktree,
|
|
3312
|
+
poolClaimed: next.run.pool_claimed,
|
|
3313
|
+
};
|
|
3314
|
+
}
|
|
3315
|
+
else {
|
|
3316
|
+
// The serial-member fields describe no member of a parallel batch —
|
|
3317
|
+
// blank them so the reuse branch re-derives THIS member's branch/path.
|
|
3318
|
+
const prep = prepareMemberWorktree(deps, { ...batch, member_branch: null, member_worktree: null, member_pool_claimed: false }, next.index, next.issue, now);
|
|
3319
|
+
if (!prep.ok) {
|
|
3320
|
+
evictMemberDirectly(deps, config, batchId, next.issue, {
|
|
3321
|
+
reason: `member-worktree-prep-failed:${prep.reason}`,
|
|
3322
|
+
detail: `member worktree prep failed: ${prep.reason} (member ${next.index} #${next.issue}, branch ${memberBranchFor(batchId, next.index, next.issue)})`,
|
|
3323
|
+
}, now);
|
|
3324
|
+
result.failed.push((0, state_1.batchMemberUnit)(batchId, next.issue));
|
|
3325
|
+
continue;
|
|
3326
|
+
}
|
|
3327
|
+
member = prep;
|
|
3328
|
+
}
|
|
3329
|
+
const recorded = deps.store.withLock((s) => {
|
|
3330
|
+
const b = (0, state_1.findBatch)(s, batchId);
|
|
3331
|
+
if (!b || b.status !== 'executing' || (0, state_1.slotForBatchMember)(s, batchId, next.issue)) {
|
|
3332
|
+
return { state: s, result: false };
|
|
3333
|
+
}
|
|
3334
|
+
// The run is recorded whether or not a slot is free and the spawn
|
|
3335
|
+
// succeeds: its worktree exists either way, and a `running` run with no
|
|
3336
|
+
// slot is exactly what a later pass respawns in place.
|
|
3337
|
+
let n = upsertRun(s, batchId, {
|
|
3338
|
+
issue: next.issue,
|
|
3339
|
+
index: next.index,
|
|
3340
|
+
branch: member.branch,
|
|
3341
|
+
worktree: member.worktree,
|
|
3342
|
+
pool_claimed: member.poolClaimed,
|
|
3343
|
+
status: 'running',
|
|
3344
|
+
gate_inconclusive: null,
|
|
3345
|
+
torn_down: false,
|
|
3346
|
+
}, now);
|
|
3347
|
+
if ((0, scheduler_1.freeCapacity)(n, config) === 0)
|
|
3348
|
+
return { state: n, result: true };
|
|
3349
|
+
const memberUnit = (0, state_1.batchMemberUnit)(batchId, next.issue);
|
|
3350
|
+
const assigned = (0, scheduler_1.assignToIdleSlot)(n, memberUnit, 'member', now, 'cycle');
|
|
3351
|
+
const slot = assigned.state.slots.find((x) => x.id === assigned.slotId);
|
|
3352
|
+
if (!slot)
|
|
3353
|
+
return { state: n, result: true };
|
|
3354
|
+
n = spawnMemberAgent(deps, dispatch, assigned.state, slot, batchId, now, result, {
|
|
3355
|
+
index: next.index,
|
|
3356
|
+
issue: next.issue,
|
|
3357
|
+
member,
|
|
3358
|
+
unitId: memberUnit,
|
|
3359
|
+
release: (st) => (0, state_1.releaseBatchMemberSlot)(st, batchId, next.issue, now),
|
|
3360
|
+
}).state;
|
|
3361
|
+
n = (0, state_1.patchBatch)(n, batchId, { executing_member: Math.max(b.executing_member, next.index) }, now);
|
|
3362
|
+
return { state: n, result: true };
|
|
3363
|
+
});
|
|
3364
|
+
if (!recorded && next.run === undefined) {
|
|
3365
|
+
// The batch left `executing` (stopped/abandoned/dissolved) while the
|
|
3366
|
+
// tree was being prepared outside the lock — nothing records it, so
|
|
3367
|
+
// give it back now rather than leak a worktree/pool claim.
|
|
3368
|
+
journalEvent(deps, 'unit-failed', unit(batchId), {
|
|
3369
|
+
issue: next.issue,
|
|
3370
|
+
reason: 'member-worktree-orphaned',
|
|
3371
|
+
detail: `batch left executing while member ${next.index}'s tree ${member.worktree} was prepared — tearing it down`,
|
|
3372
|
+
});
|
|
3373
|
+
teardownMemberTree(deps, batchId, runTree(member), false);
|
|
3374
|
+
}
|
|
3375
|
+
}
|
|
3376
|
+
}
|
|
3377
|
+
/**
|
|
3378
|
+
* Evict one parallel member directly (RFC F.1 — its commits never landed) and
|
|
3379
|
+
* tear its worktree down; the pushed member branch is KEPT on origin (the
|
|
3380
|
+
* evicted work's only recoverable copy, as in serial mode). A dissolve this
|
|
3381
|
+
* tips over is torn down by `teardownBatch`, which covers every live run.
|
|
3382
|
+
*/
|
|
3383
|
+
function evictParallelMember(deps, config, batchId, run, failure, now, result) {
|
|
3384
|
+
const { dissolved, duplicate } = evictMemberDirectly(deps, config, batchId, run.issue, failure, now);
|
|
3385
|
+
if (duplicate)
|
|
3386
|
+
return;
|
|
3387
|
+
result.failed.push(dissolved ? unit(batchId) : (0, state_1.batchMemberUnit)(batchId, run.issue));
|
|
3388
|
+
if (dissolved)
|
|
3389
|
+
return;
|
|
3390
|
+
deps.store.withLock((s) => ({
|
|
3391
|
+
state: patchRun(s, batchId, run.issue, { status: 'evicted', gate_inconclusive: null }, now),
|
|
3392
|
+
result: undefined,
|
|
3393
|
+
}));
|
|
3394
|
+
teardownMemberTree(deps, batchId, runTree(run), false);
|
|
3395
|
+
markTornDown(deps, batchId, run.issue, now);
|
|
3396
|
+
}
|
|
3397
|
+
/**
|
|
3398
|
+
* Reconcile one parallel member's slot — the parallel twin of
|
|
3399
|
+
* `reconcileMemberSlot`: the same fenced milestone read, the same incremental
|
|
3400
|
+
* gate (in the member's OWN worktree), the same eviction rail. A verified
|
|
3401
|
+
* member does NOT land here: landing is ordered (`advanceParallelBatch`), so
|
|
3402
|
+
* it only leaves `running` and gives its slot back.
|
|
3403
|
+
*/
|
|
3404
|
+
function reconcileParallelMemberSlot(deps, config, dispatch, batchId, run, slot, now, result) {
|
|
3405
|
+
const state0 = deps.store.load();
|
|
3406
|
+
const read = readMemberDispatch(deps, batchId, run.issue, slot, now);
|
|
3407
|
+
if (read.kind === 'unreachable')
|
|
3408
|
+
return;
|
|
3409
|
+
const memberUnit = (0, state_1.batchMemberUnit)(batchId, run.issue);
|
|
3410
|
+
const release = (s) => (0, state_1.releaseBatchMemberSlot)(s, batchId, run.issue, now);
|
|
3411
|
+
if (read.kind === 'complete') {
|
|
3412
|
+
// Its slot is about to be released and its tree rebased/force-pushed at
|
|
3413
|
+
// landing: an agent still running after `review done` would keep writing
|
|
3414
|
+
// into that tree with its pid no longer tracked anywhere (so neither a
|
|
3415
|
+
// dissolve nor `sched stop --batch` could stop it). Its work is done —
|
|
3416
|
+
// stop it first.
|
|
3417
|
+
if (slot.pid !== null && deps.spawnDeps.isAlive(slot.pid, slot.pid_start ?? undefined)) {
|
|
3418
|
+
deps.spawnDeps.kill(slot.pid, slot.pid_start ?? undefined);
|
|
3419
|
+
}
|
|
3420
|
+
journalEvent(deps, 'external-advance', unit(batchId), {
|
|
3421
|
+
issue: run.issue,
|
|
3422
|
+
detail: 'member review done',
|
|
3423
|
+
});
|
|
3424
|
+
recordMemberRunLog(deps, dispatch, state0, batchId, run.index, run.issue, slot, now);
|
|
3425
|
+
const verdict = evaluateIncrementalGate(deps, batchId, run.worktree, run.issue, now);
|
|
3426
|
+
if (verdict.kind === 'failed') {
|
|
3427
|
+
deps.store.withLock((s) => ({ state: release(s), result: undefined }));
|
|
3428
|
+
evictParallelMember(deps, config, batchId, run, { reason: verdict.reason, detail: verdict.detail }, now, result);
|
|
3429
|
+
return;
|
|
3430
|
+
}
|
|
3431
|
+
deps.store.withLock((s) => ({
|
|
3432
|
+
// #629: a verified member completion is proof dispatch is healthy.
|
|
3433
|
+
state: patchRun((0, dispatch_health_1.resetDispatchApiErrorStreak)(release(s)), batchId, run.issue, {
|
|
3434
|
+
status: 'verified',
|
|
3435
|
+
gate_inconclusive: verdict.kind === 'inconclusive' ? verdict.capability : null,
|
|
3436
|
+
}, now),
|
|
3437
|
+
result: undefined,
|
|
3438
|
+
}));
|
|
3439
|
+
result.completed.push(memberUnit);
|
|
3440
|
+
return;
|
|
3441
|
+
}
|
|
3442
|
+
if (read.dead &&
|
|
3443
|
+
!read.blockedNow &&
|
|
3444
|
+
handleDeadDispatchApiError(deps, batchId, (0, dispatch_1.batchMemberLogPath)(deps.store.runsDir, batchId, run.index, run.issue), slot.log_offset_at_spawn ?? 0, now, { releaseWith: release })) {
|
|
3445
|
+
// #629: not a member failure — the run stays `running` with no slot, and
|
|
3446
|
+
// `spawnParallelMembers` (pause-gated) respawns it in place.
|
|
3447
|
+
return;
|
|
3448
|
+
}
|
|
3449
|
+
if (read.blockedNow || read.dead) {
|
|
3450
|
+
const lastTool = recordMemberRunLog(deps, dispatch, state0, batchId, run.index, run.issue, slot, now);
|
|
3451
|
+
deps.store.withLock((s) => ({ state: release(s), result: undefined }));
|
|
3452
|
+
evictParallelMember(deps, config, batchId, run, memberFailureFor(read, lastTool), now, result);
|
|
3453
|
+
}
|
|
3454
|
+
}
|
|
3455
|
+
/**
|
|
3456
|
+
* Land one verified parallel member onto the integration branch (#809): rebase
|
|
3457
|
+
* its commits onto the integration TIP in its own worktree (earlier members may
|
|
3458
|
+
* have landed since it was cut), refresh the pushed member branch, then
|
|
3459
|
+
* `merge --ff-only` + push in the batch worktree — the same linear,
|
|
3460
|
+
* `(#<issue>)`-trailed history serial landing produces, so `memberRanges`
|
|
3461
|
+
* attribution and per-member commits are unchanged.
|
|
3462
|
+
*
|
|
3463
|
+
* `conflict` — the member's diff does not apply on top of the members landed
|
|
3464
|
+
* before it (the rebase is aborted, leaving its pushed branch untouched); the
|
|
3465
|
+
* caller evicts it, batch-integrate's eviction verdict. `failed` — a mechanical
|
|
3466
|
+
* step failed after a clean rebase; the caller blocks the batch, like serial.
|
|
3467
|
+
*/
|
|
3468
|
+
function landParallelRun(deps, batch, run, now) {
|
|
3469
|
+
const fail = (reason, detail) => {
|
|
3470
|
+
journalEvent(deps, 'landing-failed', unit(batch.id), { issue: run.issue, reason, detail });
|
|
3471
|
+
return { kind: 'failed', reason };
|
|
3472
|
+
};
|
|
3473
|
+
if (batch.worktree === null || batch.branch === null) {
|
|
3474
|
+
return fail('no-integration-branch', 'batch has no integration worktree/branch');
|
|
3475
|
+
}
|
|
3476
|
+
// Defense in depth (CWE-88): persisted refs reach git argv below.
|
|
3477
|
+
if (!attribution_1.SAFE_REF_RE.test(run.branch) || !attribution_1.SAFE_REF_RE.test(batch.branch)) {
|
|
3478
|
+
return fail('invalid-branch-name', `persisted member/integration branch failed SAFE_REF_RE: ${run.branch} → ${batch.branch}`);
|
|
3479
|
+
}
|
|
3480
|
+
// The rebase rewrites whatever is checked out in `run.worktree` — confirm
|
|
3481
|
+
// it is this member's own tree, present, on this member's branch first.
|
|
3482
|
+
if (!(deps.fsExists ?? ((p) => fs.existsSync(p)))(run.worktree)) {
|
|
3483
|
+
return fail('member-worktree-missing', `member worktree ${run.worktree} is gone from disk`);
|
|
3484
|
+
}
|
|
3485
|
+
if (!run.pool_claimed) {
|
|
3486
|
+
const root = deps.exec('git', ['rev-parse', '--show-toplevel'], deps.repoDir) ?? deps.repoDir;
|
|
3487
|
+
if (!(0, teardown_1.isSafeWorktree)(path.resolve(root), run.worktree)) {
|
|
3488
|
+
return fail('unsafe-member-worktree', `member worktree ${run.worktree} is outside the repo's worktree roots`);
|
|
3489
|
+
}
|
|
3490
|
+
}
|
|
3491
|
+
// A rebase an engine crash left half-done would make the next one fail
|
|
3492
|
+
// ("rebase already in progress") and read as a conflict — clear it first
|
|
3493
|
+
// (a no-op failure when none is in progress).
|
|
3494
|
+
deps.exec('git', ['rebase', '--abort'], run.worktree);
|
|
3495
|
+
const head = deps.exec('git', ['symbolic-ref', '--short', 'HEAD'], run.worktree)?.trim();
|
|
3496
|
+
if (head !== run.branch) {
|
|
3497
|
+
return fail('member-worktree-off-branch', `member worktree ${run.worktree} has ${head ?? 'no branch'} checked out, expected ${run.branch} — not rebasing someone else's checkout`);
|
|
3498
|
+
}
|
|
3499
|
+
// The force-push below must never overwrite work pushed to the member
|
|
3500
|
+
// branch that is not part of what was verified here. Read origin's tip NOW
|
|
3501
|
+
// (a bare `--force-with-lease` would trust `origin/<branch>`, which any fetch
|
|
3502
|
+
// in the shared repo refreshes) and require it to be an ancestor of the
|
|
3503
|
+
// verified local branch (the member may not have pushed its last commit);
|
|
3504
|
+
// the lease then pins exactly that tip.
|
|
3505
|
+
const verified = deps
|
|
3506
|
+
.exec('git', ['rev-parse', `refs/heads/${run.branch}`], run.worktree)
|
|
3507
|
+
?.trim();
|
|
3508
|
+
const remoteLine = deps.exec('git', ['ls-remote', 'origin', `refs/heads/${run.branch}`], run.worktree);
|
|
3509
|
+
if (!verified || !groundtruth_1.GIT_OID_RE.test(verified) || remoteLine === null) {
|
|
3510
|
+
return fail('member-branch-unresolved', `could not resolve ${run.branch} locally and on origin before landing`);
|
|
3511
|
+
}
|
|
3512
|
+
const remoteTip = remoteLine.trim().split(/\s+/)[0] ?? '';
|
|
3513
|
+
if (remoteTip !== '' &&
|
|
3514
|
+
(!groundtruth_1.GIT_OID_RE.test(remoteTip) ||
|
|
3515
|
+
deps.exec('git', ['merge-base', '--is-ancestor', remoteTip, verified], run.worktree) === null)) {
|
|
3516
|
+
return fail('member-branch-diverged', `origin/${run.branch} (${remoteTip.slice(0, 12)}) holds commits the verified ${verified.slice(0, 12)} does not — something was pushed after verification; blocking rather than overwriting it`);
|
|
3517
|
+
}
|
|
3518
|
+
// `--autostash`: a stray uncommitted file in the member tree is not the
|
|
3519
|
+
// member's deliverable and must not read as a conflict.
|
|
3520
|
+
if (deps.exec('git', ['rebase', '--autostash', batch.branch], run.worktree) === null) {
|
|
3521
|
+
// Only a rebase stopped on unmerged paths is a CONFLICT (eviction); any
|
|
3522
|
+
// other failure is mechanical and blocks like the other landing steps.
|
|
3523
|
+
const unmerged = (deps.exec('git', ['diff', '--name-only', '--diff-filter=U'], run.worktree) ?? '')
|
|
3524
|
+
.split('\n')
|
|
3525
|
+
.map((l) => l.trim())
|
|
3526
|
+
.filter((l) => l.length > 0);
|
|
3527
|
+
deps.exec('git', ['rebase', '--abort'], run.worktree);
|
|
3528
|
+
if (unmerged.length === 0) {
|
|
3529
|
+
return fail('member-rebase-failed', `git rebase of ${run.branch} onto ${batch.branch} failed with no conflicting paths — blocking for an operator`);
|
|
3530
|
+
}
|
|
3531
|
+
const detail = `${run.branch} does not rebase cleanly onto ${batch.branch} — conflicting with members landed before it in: ${unmerged.slice(0, 10).join(', ')}; evicted with its pushed branch intact`;
|
|
3532
|
+
journalEvent(deps, 'landing-failed', unit(batch.id), {
|
|
3533
|
+
issue: run.issue,
|
|
3534
|
+
reason: 'landing-conflict',
|
|
3535
|
+
detail,
|
|
3536
|
+
});
|
|
3537
|
+
return { kind: 'conflict', detail };
|
|
3538
|
+
}
|
|
3539
|
+
if (deps.exec('git', [
|
|
3540
|
+
'push',
|
|
3541
|
+
`--force-with-lease=refs/heads/${run.branch}:${remoteTip}`,
|
|
3542
|
+
'origin',
|
|
3543
|
+
'--',
|
|
3544
|
+
run.branch,
|
|
3545
|
+
], run.worktree) === null) {
|
|
3546
|
+
return fail('member-push-failed', `rebased ${run.branch} onto ${batch.branch} but could not push it (origin moved since ${remoteTip.slice(0, 12) || 'absent'}, or the push failed) — blocking so nothing pushed after verification is overwritten`);
|
|
3547
|
+
}
|
|
3548
|
+
if (deps.exec('git', ['merge', '--ff-only', run.branch], batch.worktree) === null) {
|
|
3549
|
+
return fail('landing-merge-failed', `git merge --ff-only ${run.branch} into ${batch.branch} failed right after a clean rebase — blocking for an operator`);
|
|
3550
|
+
}
|
|
3551
|
+
if (deps.exec('git', ['push', 'origin', '--', batch.branch], batch.worktree) === null) {
|
|
3552
|
+
return fail('landing-push-failed', `landed ${run.branch} on ${batch.branch} locally but the push failed — blocking so origin stays the durable copy`);
|
|
3553
|
+
}
|
|
3554
|
+
deps.journal.append((0, journal_1.unitEvent)('member-landed', unit(batch.id), {
|
|
3555
|
+
issue: run.issue,
|
|
3556
|
+
detail: `${run.branch} rebased onto and fast-forwarded into ${batch.branch} (member ${run.index}, parallel)`,
|
|
3557
|
+
}), now);
|
|
3558
|
+
return { kind: 'landed' };
|
|
3559
|
+
}
|
|
3560
|
+
/**
|
|
3561
|
+
* Walk members IN ORDER: skip every resolved one (landed, evicted, or no longer
|
|
3562
|
+
* in the batch); the first unresolved one is either landable (`verified`) or
|
|
3563
|
+
* something the walk must wait for (not yet spawned, or still running) — later
|
|
3564
|
+
* members never land ahead of it, so the integration branch always holds a
|
|
3565
|
+
* member-order prefix.
|
|
3566
|
+
*/
|
|
3567
|
+
function nextLandingStep(state, batch) {
|
|
3568
|
+
for (const issue of batch.members) {
|
|
3569
|
+
const run = batch.member_runs.find((r) => r.issue === issue);
|
|
3570
|
+
if (run?.status === 'landed' || run?.status === 'evicted')
|
|
3571
|
+
continue;
|
|
3572
|
+
if (!memberStillInBatch(state, batch, issue))
|
|
3573
|
+
continue;
|
|
3574
|
+
if (run === undefined || run.status === 'running')
|
|
3575
|
+
return { kind: 'wait' };
|
|
3576
|
+
return { kind: 'land', run };
|
|
3577
|
+
}
|
|
3578
|
+
return { kind: 'done' };
|
|
3579
|
+
}
|
|
3580
|
+
/**
|
|
3581
|
+
* Land every landable member in order, then — once every member is resolved —
|
|
3582
|
+
* either block on a gate-inconclusive member (serial parity: it landed, the
|
|
3583
|
+
* batch blocks for `sched resume --batch`) or move to `validating` and run the
|
|
3584
|
+
* aggregate suite once.
|
|
3585
|
+
*/
|
|
3586
|
+
function advanceParallelBatch(deps, config, dispatch, batchId, now, result) {
|
|
3587
|
+
// Bounded: every iteration lands or evicts one member.
|
|
3588
|
+
for (;;) {
|
|
3589
|
+
const state = deps.store.load();
|
|
3590
|
+
const batch = (0, state_1.findBatch)(state, batchId);
|
|
3591
|
+
if (!batch || batch.status !== 'executing' || !isParallelBatch(batch))
|
|
3592
|
+
return;
|
|
3593
|
+
const step = nextLandingStep(state, batch);
|
|
3594
|
+
if (step.kind === 'wait')
|
|
3595
|
+
return;
|
|
3596
|
+
if (step.kind === 'done') {
|
|
3597
|
+
finalizeParallelMembers(deps, config, dispatch, batchId, now, result);
|
|
3598
|
+
return;
|
|
3599
|
+
}
|
|
3600
|
+
const { run } = step;
|
|
3601
|
+
const landed = landParallelRun(deps, batch, run, now);
|
|
3602
|
+
if (landed.kind === 'conflict') {
|
|
3603
|
+
evictParallelMember(deps, config, batchId, run, { reason: 'landing-conflict', detail: landed.detail }, now, result);
|
|
3604
|
+
continue;
|
|
3605
|
+
}
|
|
3606
|
+
if (landed.kind === 'failed') {
|
|
3607
|
+
stopRunningMembers(deps, batchId, now);
|
|
3608
|
+
blockBatchForOperator(deps, config, batchId, { reason: landed.reason }, now, result);
|
|
3609
|
+
return;
|
|
3610
|
+
}
|
|
3611
|
+
// `git log` outside the lock (same invariant as `completeMemberGate`).
|
|
3612
|
+
const ranges = (0, attribution_1.memberRanges)(boundaryCommits(deps, batch));
|
|
3613
|
+
deps.store.withLock((s) => {
|
|
3614
|
+
let n = (0, state_1.patchBatch)(s, batchId, { ranges }, now);
|
|
3615
|
+
n = advanceMemberToValidated(n, run.issue, now);
|
|
3616
|
+
n = patchRun(n, batchId, run.issue, { status: 'landed' }, now);
|
|
3617
|
+
return { state: n, result: undefined };
|
|
3618
|
+
});
|
|
3619
|
+
// A gate-inconclusive member keeps its tree for `sched resume --batch`'s
|
|
3620
|
+
// recheck; everyone else's is done serving.
|
|
3621
|
+
if (run.gate_inconclusive === null) {
|
|
3622
|
+
teardownMemberTree(deps, batchId, runTree(run), true);
|
|
3623
|
+
markTornDown(deps, batchId, run.issue, now);
|
|
3624
|
+
}
|
|
3625
|
+
}
|
|
3626
|
+
}
|
|
3627
|
+
/** Every member is resolved — block on a gate-inconclusive one, or validate. */
|
|
3628
|
+
function finalizeParallelMembers(deps, config, dispatch, batchId, now, result) {
|
|
3629
|
+
const batch = (0, state_1.findBatch)(deps.store.load(), batchId);
|
|
3630
|
+
if (!batch)
|
|
3631
|
+
return;
|
|
3632
|
+
const blockedOn = batch.member_runs.find((r) => r.status === 'landed' && r.gate_inconclusive !== null);
|
|
3633
|
+
if (blockedOn !== undefined) {
|
|
3634
|
+
// Point the serial "current member" fields at it: `sched resume --batch`
|
|
3635
|
+
// (`resumeBlockedGate`) rechecks `member_worktree` and completes/evicts
|
|
3636
|
+
// `members[executing_member - 1]`, then hands back via
|
|
3637
|
+
// `advanceMemberOrValidate` → `settleParallelResume`.
|
|
3638
|
+
deps.store.withLock((s) => ({
|
|
3639
|
+
state: (0, state_1.patchBatch)(s, batchId, {
|
|
3640
|
+
executing_member: blockedOn.index,
|
|
3641
|
+
member_branch: blockedOn.branch,
|
|
3642
|
+
member_worktree: blockedOn.worktree,
|
|
3643
|
+
member_pool_claimed: blockedOn.pool_claimed,
|
|
3644
|
+
}, now),
|
|
3645
|
+
result: undefined,
|
|
3646
|
+
}));
|
|
3647
|
+
blockBatchForOperator(deps, config, batchId, {
|
|
3648
|
+
reason: `gate-inconclusive:${blockedOn.gate_inconclusive}`,
|
|
3649
|
+
milestonePhase: 'batch-review',
|
|
3650
|
+
}, now, result);
|
|
3651
|
+
return;
|
|
3652
|
+
}
|
|
3653
|
+
const claimed = deps.store.withLock((s) => {
|
|
3654
|
+
const b = (0, state_1.findBatch)(s, batchId);
|
|
3655
|
+
if (!b || b.status !== 'executing')
|
|
3656
|
+
return { state: s, result: false };
|
|
3657
|
+
return {
|
|
3658
|
+
state: (0, state_1.transitionBatch)(s, batchId, 'validating', { executing_member: b.members.length }, now),
|
|
3659
|
+
result: true,
|
|
3660
|
+
};
|
|
3661
|
+
});
|
|
3662
|
+
if (claimed)
|
|
3663
|
+
runValidate(deps, config, dispatch, batchId, now, result);
|
|
3664
|
+
}
|
|
3665
|
+
/**
|
|
3666
|
+
* `sched resume --batch` resolved the gate-inconclusive member of a parallel
|
|
3667
|
+
* batch (completed or evicted it through the serial rail): record that on its
|
|
3668
|
+
* run so the landing walk treats it as resolved, then continue the parallel
|
|
3669
|
+
* rail from where it blocked.
|
|
3670
|
+
*/
|
|
3671
|
+
function settleParallelResume(deps, batchId, memberIssue, now) {
|
|
3672
|
+
deps.store.withLock((s) => {
|
|
3673
|
+
const b = (0, state_1.findBatch)(s, batchId);
|
|
3674
|
+
if (!b)
|
|
3675
|
+
return { state: s, result: undefined };
|
|
3676
|
+
const evicted = b.evictions.some((e) => e.issue === memberIssue);
|
|
3677
|
+
return {
|
|
3678
|
+
state: patchRun(s, batchId, memberIssue, {
|
|
3679
|
+
gate_inconclusive: null,
|
|
3680
|
+
// The serial rail's `teardownMemberWorktree` removed its tree.
|
|
3681
|
+
torn_down: true,
|
|
3682
|
+
...(evicted ? { status: 'evicted' } : {}),
|
|
3683
|
+
}, now),
|
|
3684
|
+
result: undefined,
|
|
3685
|
+
};
|
|
3686
|
+
});
|
|
3687
|
+
}
|
|
3688
|
+
/**
|
|
3689
|
+
* One tick of a parallel batch in `executing`: reconcile every member slot,
|
|
3690
|
+
* land what can land in order, then fill free member concurrency (unless the
|
|
3691
|
+
* scheduler is paused — a spawn is a provider dispatch).
|
|
3692
|
+
*/
|
|
3693
|
+
function reconcileParallelBatch(deps, config, dispatch, batchId, now, result, paused) {
|
|
3694
|
+
const batch = (0, state_1.findBatch)(deps.store.load(), batchId);
|
|
3695
|
+
if (!batch)
|
|
3696
|
+
return;
|
|
3697
|
+
for (const { issue } of batch.member_runs) {
|
|
3698
|
+
// Fresh per member: an earlier member's eviction may have dissolved the
|
|
3699
|
+
// batch (killing and releasing every other member) in this same pass.
|
|
3700
|
+
const state = deps.store.load();
|
|
3701
|
+
const fresh = (0, state_1.findBatch)(state, batchId);
|
|
3702
|
+
if (!fresh || fresh.status !== 'executing')
|
|
3703
|
+
return;
|
|
3704
|
+
const run = fresh.member_runs.find((r) => r.issue === issue);
|
|
3705
|
+
if (run?.status !== 'running')
|
|
3706
|
+
continue;
|
|
3707
|
+
const slot = (0, state_1.slotForBatchMember)(state, batchId, issue);
|
|
3708
|
+
if (slot && (slot.status === 'running' || slot.status === 'assigned')) {
|
|
3709
|
+
reconcileParallelMemberSlot(deps, config, dispatch, batchId, run, slot, now, result);
|
|
3710
|
+
}
|
|
3711
|
+
}
|
|
3712
|
+
advanceParallelBatch(deps, config, dispatch, batchId, now, result);
|
|
3713
|
+
if (!paused)
|
|
3714
|
+
spawnParallelMembers(deps, config, dispatch, batchId, now, result);
|
|
3715
|
+
}
|
|
3716
|
+
/**
|
|
3717
|
+
* #809: stop and tear down every parallel member whose tree is not yet torn
|
|
3718
|
+
* down — a dissolve can land while other members are mid-run (their agents are
|
|
3719
|
+
* killed: their entries were just requeued full-cycle, so a live member agent
|
|
3720
|
+
* would be working a unit nothing tracks any more), `sched stop`/`abandon`
|
|
3721
|
+
* make a batch terminal without a teardown, and a crash between a run's
|
|
3722
|
+
* resolution and its teardown leaves `torn_down: false` behind. Teardown is
|
|
3723
|
+
* idempotent, so re-running it is always safe. `alreadyTornDown` names the
|
|
3724
|
+
* serial-field tree `teardownMemberWorktree` just handled (a gate-inconclusive
|
|
3725
|
+
* member of a blocked parallel batch).
|
|
3726
|
+
*/
|
|
3727
|
+
function teardownParallelRuns(deps, batchId, alreadyTornDown) {
|
|
3728
|
+
const batch = (0, state_1.findBatch)(deps.store.load(), batchId);
|
|
3729
|
+
if (!batch)
|
|
3730
|
+
return;
|
|
3731
|
+
const now = deps.now();
|
|
3732
|
+
for (const run of batch.member_runs) {
|
|
3733
|
+
if (run.torn_down)
|
|
3734
|
+
continue;
|
|
3735
|
+
stopMemberAgent(deps, batchId, run.issue, now);
|
|
3736
|
+
const landed = run.status === 'landed';
|
|
3737
|
+
deps.store.withLock((s) => ({
|
|
3738
|
+
state: patchRun(s, batchId, run.issue, { status: landed ? 'landed' : 'evicted', gate_inconclusive: null }, now),
|
|
3739
|
+
result: undefined,
|
|
3740
|
+
}));
|
|
3741
|
+
if (run.worktree !== alreadyTornDown) {
|
|
3742
|
+
teardownMemberTree(deps, batchId, runTree(run), landed);
|
|
3743
|
+
}
|
|
3744
|
+
markTornDown(deps, batchId, run.issue, now);
|
|
3745
|
+
}
|
|
3746
|
+
}
|
|
3747
|
+
/** Kill a parallel member's live agent (pid-start checked) and release its slot. */
|
|
3748
|
+
function stopMemberAgent(deps, batchId, issue, now) {
|
|
3749
|
+
const slot = (0, state_1.slotForBatchMember)(deps.store.load(), batchId, issue);
|
|
3750
|
+
if (slot === undefined)
|
|
3751
|
+
return;
|
|
3752
|
+
if (slot.pid !== null && deps.spawnDeps.isAlive(slot.pid, slot.pid_start ?? undefined)) {
|
|
3753
|
+
deps.spawnDeps.kill(slot.pid, slot.pid_start ?? undefined);
|
|
3754
|
+
}
|
|
3755
|
+
deps.store.withLock((s) => ({
|
|
3756
|
+
state: (0, state_1.releaseBatchMemberSlot)(s, batchId, issue, now),
|
|
3757
|
+
result: undefined,
|
|
3758
|
+
}));
|
|
3759
|
+
}
|
|
3760
|
+
/**
|
|
3761
|
+
* Stop every running member of a parallel batch that is about to BLOCK (a
|
|
3762
|
+
* landing failure, a reconcile error): nothing reconciles a blocked batch's
|
|
3763
|
+
* member slots, so a still-running agent would hold capacity and keep working
|
|
3764
|
+
* a unit an operator is about to abandon/requeue. Trees are kept for the
|
|
3765
|
+
* operator; `teardownBatch` removes them when the batch ends.
|
|
3766
|
+
*/
|
|
3767
|
+
function stopRunningMembers(deps, batchId, now) {
|
|
3768
|
+
const batch = (0, state_1.findBatch)(deps.store.load(), batchId);
|
|
3769
|
+
for (const run of batch?.member_runs ?? []) {
|
|
3770
|
+
if (run.status === 'running')
|
|
3771
|
+
stopMemberAgent(deps, batchId, run.issue, now);
|
|
3772
|
+
}
|
|
3773
|
+
}
|
|
3034
3774
|
// --- Entry point ---
|
|
3035
3775
|
/**
|
|
3036
3776
|
* One batch reconcile+refill pass. Called from `engine.ts`'s `tick()` after
|
|
@@ -3093,11 +3833,12 @@ function runBatchTick(deps, config, dispatch) {
|
|
|
3093
3833
|
const fresh = (0, state_1.findBatch)(current, batch.id);
|
|
3094
3834
|
const dissolvable = fresh !== undefined && (0, state_1.allowedBatchTransitions)(fresh.status).includes('dissolving');
|
|
3095
3835
|
if (dissolvable) {
|
|
3096
|
-
|
|
3097
|
-
|
|
3098
|
-
liveSlot.pid !== null &&
|
|
3099
|
-
|
|
3100
|
-
|
|
3836
|
+
// #809: a parallel batch holds one slot per running member too.
|
|
3837
|
+
for (const liveSlot of (0, state_1.slotsForBatch)(current, batch.id)) {
|
|
3838
|
+
if (liveSlot.pid !== null &&
|
|
3839
|
+
deps.spawnDeps.isAlive(liveSlot.pid, liveSlot.pid_start ?? undefined)) {
|
|
3840
|
+
deps.spawnDeps.kill(liveSlot.pid, liveSlot.pid_start ?? undefined);
|
|
3841
|
+
}
|
|
3101
3842
|
}
|
|
3102
3843
|
const outcome = (0, recovery_1.dissolveBatch)(current, batch.id, { strategy: 'full', reason: `dispatch-profile-missing:${batch.dispatch_profile}` }, recoveryDeps(deps, config, fresh, now));
|
|
3103
3844
|
deps.store.withLock((s) => ({
|
|
@@ -3126,6 +3867,13 @@ function runBatchTick(deps, config, dispatch) {
|
|
|
3126
3867
|
};
|
|
3127
3868
|
const containFailure = (batchId, err) => {
|
|
3128
3869
|
const detail = `${err.name}: ${err.message}`;
|
|
3870
|
+
// #809: a blocked batch's member slots are never reconciled — stop the agents too.
|
|
3871
|
+
try {
|
|
3872
|
+
stopRunningMembers(deps, batchId, now);
|
|
3873
|
+
}
|
|
3874
|
+
catch {
|
|
3875
|
+
// best-effort: the block below must still land
|
|
3876
|
+
}
|
|
3129
3877
|
// A corrupt unit must not turn into a project-wide scheduler outage.
|
|
3130
3878
|
// Keep the failed batch inspectable, release its slot, and continue.
|
|
3131
3879
|
deps.store.withLock((state) => {
|
|
@@ -3134,7 +3882,7 @@ function runBatchTick(deps, config, dispatch) {
|
|
|
3134
3882
|
return { state, result: undefined };
|
|
3135
3883
|
}
|
|
3136
3884
|
const blocked = (0, state_1.transitionBatch)(state, batchId, 'blocked', { blocked_reason: 'reconcile-error' }, now);
|
|
3137
|
-
return { state: (0, state_1.
|
|
3885
|
+
return { state: (0, state_1.releaseAllBatchSlots)(blocked, batchId, now), result: undefined };
|
|
3138
3886
|
});
|
|
3139
3887
|
journalEvent(deps, 'unit-failed', unit(batchId), { reason: 'reconcile-error', detail });
|
|
3140
3888
|
result.failed.push(unit(batchId));
|
|
@@ -3186,6 +3934,8 @@ function runBatchTick(deps, config, dispatch) {
|
|
|
3186
3934
|
}
|
|
3187
3935
|
for (const batch of deps.store.load().batches) {
|
|
3188
3936
|
const slot = slotFor(deps.store.load(), batch.id);
|
|
3937
|
+
// #809: a terminal batch may also leak parallel member slots.
|
|
3938
|
+
const heldSlots = (0, state_1.slotsForBatch)(deps.store.load(), batch.id);
|
|
3189
3939
|
// #609: a TERMINAL batch (`done`/`dissolved`) still holding a slot matches
|
|
3190
3940
|
// none of the status arms below and would fall through their `continue`,
|
|
3191
3941
|
// so nothing ever looks at that slot again — not even the orphaned-pid
|
|
@@ -3197,15 +3947,29 @@ function runBatchTick(deps, config, dispatch) {
|
|
|
3197
3947
|
// files that already carry a leaked slot, which is the only way an
|
|
3198
3948
|
// operator gets those three hours of held capacity back.
|
|
3199
3949
|
if (types_1.TERMINAL_BATCH_STATUSES.has(batch.status)) {
|
|
3200
|
-
|
|
3950
|
+
// #809: `sched stop`/`abandon` end a batch without a teardown — give
|
|
3951
|
+
// back every member tree (and stop any agent still running in one).
|
|
3952
|
+
if (batch.member_runs.some((r) => !r.torn_down)) {
|
|
3953
|
+
try {
|
|
3954
|
+
teardownParallelRuns(deps, batch.id, null);
|
|
3955
|
+
}
|
|
3956
|
+
catch (err) {
|
|
3957
|
+
journalEvent(deps, 'teardown-failed', unit(batch.id), {
|
|
3958
|
+
detail: `member teardown: ${err.message}`,
|
|
3959
|
+
});
|
|
3960
|
+
}
|
|
3961
|
+
}
|
|
3962
|
+
if (heldSlots.length > 0) {
|
|
3201
3963
|
deps.store.withLock((s) => ({
|
|
3202
|
-
state: (0, state_1.
|
|
3964
|
+
state: (0, state_1.releaseAllBatchSlots)(s, batch.id, now),
|
|
3203
3965
|
result: undefined,
|
|
3204
3966
|
}));
|
|
3205
|
-
|
|
3206
|
-
slot
|
|
3207
|
-
|
|
3208
|
-
|
|
3967
|
+
for (const held of heldSlots) {
|
|
3968
|
+
journalEvent(deps, 'slot-released', unit(batch.id), {
|
|
3969
|
+
slot: held.id,
|
|
3970
|
+
detail: `terminal batch (${batch.status}) held slot ${held.id}${held.unit === unit(batch.id) ? '' : ` (${held.unit})`}`,
|
|
3971
|
+
});
|
|
3972
|
+
}
|
|
3209
3973
|
}
|
|
3210
3974
|
continue;
|
|
3211
3975
|
}
|
|
@@ -3217,6 +3981,12 @@ function runBatchTick(deps, config, dispatch) {
|
|
|
3217
3981
|
const batchDispatch = dispatchFor(batch);
|
|
3218
3982
|
if (batchDispatch === null)
|
|
3219
3983
|
continue;
|
|
3984
|
+
// #809: a parallel batch's members hold their OWN slots — its whole
|
|
3985
|
+
// `executing` phase is one arm.
|
|
3986
|
+
if (batch.status === 'executing' && isParallelBatch(batch)) {
|
|
3987
|
+
reconcileParallelBatch(deps, config, batchDispatch, batch.id, now, result, paused);
|
|
3988
|
+
continue;
|
|
3989
|
+
}
|
|
3220
3990
|
if (slot && (slot.status === 'running' || slot.status === 'assigned')) {
|
|
3221
3991
|
if (batch.status === 'executing') {
|
|
3222
3992
|
reconcileMemberSlot(deps, config, batchDispatch, batch.id, slot, now, result);
|