@ai-dossier/sched 0.46.1 → 0.48.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +102 -17
- package/dist/anchor-close.d.ts.map +1 -1
- package/dist/anchor-close.js +2 -1
- package/dist/anchor-close.js.map +1 -1
- package/dist/batch-dispatch.d.ts +23 -2
- package/dist/batch-dispatch.d.ts.map +1 -1
- package/dist/batch-dispatch.js +1057 -209
- package/dist/batch-dispatch.js.map +1 -1
- package/dist/dispatch.d.ts +17 -0
- package/dist/dispatch.d.ts.map +1 -1
- package/dist/dispatch.js +31 -0
- package/dist/dispatch.js.map +1 -1
- package/dist/engine.d.ts +3 -2
- package/dist/engine.d.ts.map +1 -1
- package/dist/engine.js +13 -4
- package/dist/engine.js.map +1 -1
- package/dist/index.d.ts +6 -6
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +17 -4
- package/dist/index.js.map +1 -1
- package/dist/journal.d.ts +4 -1
- package/dist/journal.d.ts.map +1 -1
- package/dist/journal.js +5 -2
- package/dist/journal.js.map +1 -1
- package/dist/persist.d.ts.map +1 -1
- package/dist/persist.js +10 -1
- package/dist/persist.js.map +1 -1
- package/dist/recovery.d.ts +39 -3
- package/dist/recovery.d.ts.map +1 -1
- package/dist/recovery.js +206 -46
- package/dist/recovery.js.map +1 -1
- package/dist/scheduler.d.ts +15 -1
- package/dist/scheduler.d.ts.map +1 -1
- package/dist/scheduler.js +31 -5
- package/dist/scheduler.js.map +1 -1
- package/dist/state.d.ts +94 -8
- package/dist/state.d.ts.map +1 -1
- package/dist/state.js +332 -26
- package/dist/state.js.map +1 -1
- package/dist/status.d.ts +29 -1
- package/dist/status.d.ts.map +1 -1
- package/dist/status.js +63 -3
- package/dist/status.js.map +1 -1
- package/dist/types.d.ts +133 -8
- package/dist/types.d.ts.map +1 -1
- package/dist/types.js +27 -2
- package/dist/types.js.map +1 -1
- package/package.json +1 -1
package/dist/batch-dispatch.js
CHANGED
|
@@ -12,7 +12,8 @@
|
|
|
12
12
|
* at batch-phase granularity instead of per-issue-phase granularity:
|
|
13
13
|
*
|
|
14
14
|
* ```
|
|
15
|
-
* ready → executing(member i/N
|
|
15
|
+
* ready → executing(serial: member i/N ⟲ | parallel: members ∥ + ordered landing)
|
|
16
|
+
* → validating → reviewing → shipping
|
|
16
17
|
* → awaiting-merge → merged → deployed → reported → done
|
|
17
18
|
* failure rails (RFC F.2/F.8/F.9):
|
|
18
19
|
* executing → dissolving (a member self-reports blocked, RFC F.1)
|
|
@@ -98,6 +99,7 @@ Object.defineProperty(exports, "__esModule", { value: true });
|
|
|
98
99
|
exports.memberBranchFor = memberBranchFor;
|
|
99
100
|
exports.evictMemberAndContinue = evictMemberAndContinue;
|
|
100
101
|
exports.resumeBlockedGate = resumeBlockedGate;
|
|
102
|
+
exports.memberDispatchModeFor = memberDispatchModeFor;
|
|
101
103
|
exports.runBatchTick = runBatchTick;
|
|
102
104
|
const fs = __importStar(require("node:fs"));
|
|
103
105
|
const path = __importStar(require("node:path"));
|
|
@@ -140,8 +142,11 @@ function unit(batchId) {
|
|
|
140
142
|
*/
|
|
141
143
|
function sanitizeUntrustedText(value) {
|
|
142
144
|
return (value
|
|
145
|
+
// C0 + DEL + C1 (incl. the single-byte CSI \u009B) and the bidi
|
|
146
|
+
// overrides/isolates (#810: the reason now also renders in `sched status`'s
|
|
147
|
+
// Parked members rows).
|
|
143
148
|
// biome-ignore lint/suspicious/noControlCharactersInRegex: flattening control characters is the point
|
|
144
|
-
.replace(/[\u0000-\u001F\u007F]/g, ' ')
|
|
149
|
+
.replace(/[\u0000-\u001F\u007F-\u009F\u202A-\u202E\u2066-\u2069]/g, ' ')
|
|
145
150
|
.slice(0, 200));
|
|
146
151
|
}
|
|
147
152
|
/**
|
|
@@ -197,7 +202,9 @@ function handleDeadDispatchApiError(deps, batchId, logFile, offset, now, options
|
|
|
197
202
|
return false;
|
|
198
203
|
deps.store.withLock((s) => {
|
|
199
204
|
let n = (0, dispatch_health_1.recordDispatchApiError)((event, unitId, extra) => journalEvent(deps, event, unitId, extra), s, unit(batchId), apiError);
|
|
200
|
-
if (options.
|
|
205
|
+
if (options.releaseWith)
|
|
206
|
+
n = options.releaseWith(n);
|
|
207
|
+
else if (options.release ?? true)
|
|
201
208
|
n = releaseSlot(n, batchId, now);
|
|
202
209
|
return { state: n, result: undefined };
|
|
203
210
|
});
|
|
@@ -347,6 +354,31 @@ function applyBatchAndIssues(fresh, computed, batchId, issues, membersBeforeReco
|
|
|
347
354
|
});
|
|
348
355
|
return { ...fresh, batches, entries };
|
|
349
356
|
}
|
|
357
|
+
/**
|
|
358
|
+
* #810: land a `dissolveBatch` outcome — the one tail every dissolve caller
|
|
359
|
+
* shares. Re-applies the batch plus every entry the dissolve touched
|
|
360
|
+
* (requeued AND parked — a parked entry left out here would silently revert
|
|
361
|
+
* to its pre-dissolve status). A dissolved batch is torn down; a dissolve
|
|
362
|
+
* REFUSED over validated members (`outcome.blocked`) keeps its worktree for
|
|
363
|
+
* the operator, but its member agents are stopped and its slots released —
|
|
364
|
+
* nothing reconciles a blocked batch's member slots.
|
|
365
|
+
*/
|
|
366
|
+
function applyDissolveOutcome(deps, batchId, outcome, now, membersBeforeRecovery) {
|
|
367
|
+
deps.store.withLock((s) => ({
|
|
368
|
+
state: applyBatchAndIssues(s, outcome.state, batchId, [...outcome.requeued, ...outcome.parked], membersBeforeRecovery),
|
|
369
|
+
result: undefined,
|
|
370
|
+
}));
|
|
371
|
+
if (outcome.blocked) {
|
|
372
|
+
stopAndReleaseBlocked(deps, batchId, now);
|
|
373
|
+
return;
|
|
374
|
+
}
|
|
375
|
+
teardownBatch(deps, batchId);
|
|
376
|
+
}
|
|
377
|
+
/** Stop every live agent of a batch that just blocked, and release all its slots (#810). */
|
|
378
|
+
function stopAndReleaseBlocked(deps, batchId, now) {
|
|
379
|
+
stopRunningMembers(deps, batchId, now);
|
|
380
|
+
deps.store.withLock((s) => ({ state: (0, state_1.releaseAllBatchSlots)(s, batchId, now), result: undefined }));
|
|
381
|
+
}
|
|
350
382
|
/**
|
|
351
383
|
* Run the aggregate suite, treating a THROWING runner as a red suite with no
|
|
352
384
|
* failing tests — `recovery.ts`'s own internal `runSuite` wrapper already
|
|
@@ -761,8 +793,9 @@ function landMemberBranch(deps, batchId, now) {
|
|
|
761
793
|
* dangling pool entry — the corrupted-entry state `worktree-pool status`
|
|
762
794
|
* reports), cold trees are removed. The member branch is deleted locally —
|
|
763
795
|
* `-d` after a landing (fully merged into the integration branch), `-D` on
|
|
764
|
-
* the eviction path (its commits never landed;
|
|
765
|
-
*
|
|
796
|
+
* the eviction path (its commits never landed; since #810 the parked entry
|
|
797
|
+
* records the branch and an operator requeue continues from the REMOTE copy).
|
|
798
|
+
* The REMOTE member branch is deleted only on the landed path, and
|
|
766
799
|
* deliberately KEPT on eviction — the pushed sha is the evicted work's only
|
|
767
800
|
* recoverable copy.
|
|
768
801
|
*
|
|
@@ -789,13 +822,9 @@ landed) {
|
|
|
789
822
|
});
|
|
790
823
|
return;
|
|
791
824
|
}
|
|
792
|
-
//
|
|
793
|
-
//
|
|
794
|
-
//
|
|
795
|
-
// cold removes post-verify the path is gone and unlisted. Nulls first —
|
|
796
|
-
// clear the fields BEFORE the exec so a crash mid-teardown doesn't retry
|
|
797
|
-
// into a half-cleaned tree (the re-derived path is deterministic if a
|
|
798
|
-
// leftover does survive).
|
|
825
|
+
// Nulls first — clear the fields BEFORE the exec so a crash mid-teardown
|
|
826
|
+
// doesn't retry into a half-cleaned tree (the re-derived path is
|
|
827
|
+
// deterministic if a leftover does survive).
|
|
799
828
|
deps.store.withLock((s) => {
|
|
800
829
|
const b = (0, state_1.findBatch)(s, batchId);
|
|
801
830
|
if (!b)
|
|
@@ -805,7 +834,29 @@ landed) {
|
|
|
805
834
|
result: undefined,
|
|
806
835
|
};
|
|
807
836
|
});
|
|
808
|
-
|
|
837
|
+
teardownMemberTree(deps, batchId, { worktree, branch, poolClaimed: batch.member_pool_claimed === true }, landed);
|
|
838
|
+
}
|
|
839
|
+
/**
|
|
840
|
+
* The exec half of a member teardown, shared by the serial rail
|
|
841
|
+
* (`teardownMemberWorktree`, which first clears the batch's member fields) and
|
|
842
|
+
* the parallel rail (#809: `evictParallelMember`, `advanceParallelBatch`,
|
|
843
|
+
* `teardownParallelRuns`). See
|
|
844
|
+
* `teardownMemberWorktree` for the landed/evicted branch policy.
|
|
845
|
+
*/
|
|
846
|
+
function teardownMemberTree(deps, batchId, tree, landed) {
|
|
847
|
+
const { worktree, branch } = tree;
|
|
848
|
+
if (branch !== null && !attribution_1.SAFE_REF_RE.test(branch)) {
|
|
849
|
+
journalEvent(deps, 'teardown-failed', unit(batchId), {
|
|
850
|
+
cleanup: 'failed-invalid-branch-name',
|
|
851
|
+
detail: branch,
|
|
852
|
+
});
|
|
853
|
+
return;
|
|
854
|
+
}
|
|
855
|
+
// Route through `runTeardown` for the same guarantees `teardownBatch`
|
|
856
|
+
// gets: pool returns are verify-first idempotent and refuse to claim
|
|
857
|
+
// success unless the pool's own self-check reports the entry `warm`;
|
|
858
|
+
// cold removes post-verify the path is gone and unlisted.
|
|
859
|
+
const t = (0, teardown_1.runTeardown)(deps.exec, deps.repoDir, { worktree, poolClaimed: tree.poolClaimed, branch }, deps.fsExists);
|
|
809
860
|
const treeCleared = !t.cleanup.startsWith('failed');
|
|
810
861
|
let branchCleanup = 'skipped';
|
|
811
862
|
if (branch !== null && treeCleared) {
|
|
@@ -858,9 +909,10 @@ function blockLandingFailure(deps, config, batch, reason, now, result) {
|
|
|
858
909
|
blockBatchForOperator(deps, config, batch.id, { reason }, now, result);
|
|
859
910
|
}
|
|
860
911
|
/**
|
|
861
|
-
* Spawn one batch member's member-cycle agent (#677) into the slot a
|
|
862
|
-
* member (or batch-setup) just released — each member in its OWN
|
|
863
|
-
* off the integration branch
|
|
912
|
+
* Spawn one SERIAL batch member's member-cycle agent (#677) into the slot a
|
|
913
|
+
* prior member (or batch-setup) just released — each member in its OWN
|
|
914
|
+
* worktree off the integration branch. Parallel batches (#809) spawn through
|
|
915
|
+
* `spawnParallelMembers` instead, one slot per member. Prepared by
|
|
864
916
|
* {@link prepareMemberWorktree} at the spawn site (outside the lock); this
|
|
865
917
|
* function only patches state and spawns.
|
|
866
918
|
*/
|
|
@@ -902,6 +954,42 @@ function advanceMemberToValidated(state, memberIssue, now) {
|
|
|
902
954
|
return next;
|
|
903
955
|
}
|
|
904
956
|
function spawnMember(deps, dispatch, state, slot, batchId, now, result, member) {
|
|
957
|
+
const batch = (0, state_1.findBatch)(state, batchId);
|
|
958
|
+
const memberIssue = batch?.members[(batch?.executing_member ?? 0) - 1];
|
|
959
|
+
if (batch && memberIssue === undefined) {
|
|
960
|
+
journalEvent(deps, 'unit-failed', unit(batchId), {
|
|
961
|
+
reason: 'no-member',
|
|
962
|
+
detail: `spawnMember: executing_member ${batch.executing_member} has no member issue`,
|
|
963
|
+
});
|
|
964
|
+
return releaseSlot(state, batchId, now);
|
|
965
|
+
}
|
|
966
|
+
const spawned = spawnMemberAgent(deps, dispatch, state, slot, batchId, now, result, {
|
|
967
|
+
index: batch?.executing_member ?? 0,
|
|
968
|
+
issue: memberIssue ?? 0,
|
|
969
|
+
member,
|
|
970
|
+
unitId: unit(batchId),
|
|
971
|
+
release: (s) => releaseSlot(s, batchId, now),
|
|
972
|
+
});
|
|
973
|
+
if (!spawned.ok)
|
|
974
|
+
return spawned.state;
|
|
975
|
+
// #677: persist the member's worktree context — a takeover redispatch or a
|
|
976
|
+
// `sched resume --batch` recheck after an engine restart must land in the
|
|
977
|
+
// SAME worktree/branch this prompt names, not re-derive (or worse,
|
|
978
|
+
// re-create) it.
|
|
979
|
+
return (0, state_1.patchBatch)(spawned.state, batchId, {
|
|
980
|
+
member_branch: member.branch,
|
|
981
|
+
member_worktree: member.worktree,
|
|
982
|
+
member_pool_claimed: member.poolClaimed,
|
|
983
|
+
}, now);
|
|
984
|
+
}
|
|
985
|
+
/**
|
|
986
|
+
* Spawn ONE member's member-cycle agent into `slot` — the shared core of the
|
|
987
|
+
* serial rail (`spawnMember`, the batch's own slot) and the parallel rail
|
|
988
|
+
* (`spawnParallelMembers`, #809: the member's own `batch:<id>#<issue>` slot).
|
|
989
|
+
* Patches the slot and the member's queue entry; the caller persists its own
|
|
990
|
+
* member context. On failure the slot is released through `target.release`.
|
|
991
|
+
*/
|
|
992
|
+
function spawnMemberAgent(deps, dispatch, state, slot, batchId, now, result, target) {
|
|
905
993
|
const batch = (0, state_1.findBatch)(state, batchId);
|
|
906
994
|
if (!batch || batch.worktree === null || batch.branch === null) {
|
|
907
995
|
// A leaked `assigned` slot with `pid: null` is invisible to `dead`
|
|
@@ -911,16 +999,9 @@ function spawnMember(deps, dispatch, state, slot, batchId, now, result, member)
|
|
|
911
999
|
reason: 'no-worktree',
|
|
912
1000
|
detail: 'spawnMember: batch has no worktree/branch — batch-setup has not landed',
|
|
913
1001
|
});
|
|
914
|
-
return
|
|
915
|
-
}
|
|
916
|
-
const memberIssue = batch.members[batch.executing_member - 1];
|
|
917
|
-
if (memberIssue === undefined) {
|
|
918
|
-
journalEvent(deps, 'unit-failed', unit(batchId), {
|
|
919
|
-
reason: 'no-member',
|
|
920
|
-
detail: `spawnMember: executing_member ${batch.executing_member} has no member issue`,
|
|
921
|
-
});
|
|
922
|
-
return releaseSlot(state, batchId, now);
|
|
1002
|
+
return { ok: false, state: target.release(state) };
|
|
923
1003
|
}
|
|
1004
|
+
const memberIssue = target.issue;
|
|
924
1005
|
const withStatus = advanceMemberToInWork(state, memberIssue, now);
|
|
925
1006
|
const memberEntry = (0, state_1.findEntry)(withStatus, memberIssue);
|
|
926
1007
|
// #771: a `review=full` member dispatches at `strong` minimum (within this
|
|
@@ -932,8 +1013,8 @@ function spawnMember(deps, dispatch, state, slot, batchId, now, result, member)
|
|
|
932
1013
|
// #677: the prompt carries THIS member's own worktree and the integration
|
|
933
1014
|
// branch it was cut from — `member-cycle`'s `worktree`/`integration_branch`
|
|
934
1015
|
// inputs. The shared batch worktree is no longer a member surface at all.
|
|
935
|
-
const prompt = (0, dispatch_1.buildMemberPrompt)(dispatch.memberPrompt, memberIssue, batchId, member.worktree, batch.branch, review);
|
|
936
|
-
const logFile = (0, dispatch_1.batchMemberLogPath)(deps.store.runsDir, batchId,
|
|
1016
|
+
const prompt = (0, dispatch_1.buildMemberPrompt)(dispatch.memberPrompt, memberIssue, batchId, target.member.worktree, batch.branch, review);
|
|
1017
|
+
const logFile = (0, dispatch_1.batchMemberLogPath)(deps.store.runsDir, batchId, target.index, memberIssue);
|
|
937
1018
|
// #629: captured BEFORE spawning, mirroring `engine.ts`'s own
|
|
938
1019
|
// `spawnAndRecord` — the log is per-role and append-mode, so the size at
|
|
939
1020
|
// this instant is exactly where THIS dispatch's own output starts. Fences
|
|
@@ -949,8 +1030,8 @@ function spawnMember(deps, dispatch, state, slot, batchId, now, result, member)
|
|
|
949
1030
|
reason: 'spawn-error',
|
|
950
1031
|
detail: `member #${memberIssue} spawn failed: ${err.message}`,
|
|
951
1032
|
});
|
|
952
|
-
result.failed.push(
|
|
953
|
-
return
|
|
1033
|
+
result.failed.push(target.unitId);
|
|
1034
|
+
return { ok: false, state: target.release(withStatus) };
|
|
954
1035
|
}
|
|
955
1036
|
const patch = {
|
|
956
1037
|
pid,
|
|
@@ -963,18 +1044,9 @@ function spawnMember(deps, dispatch, state, slot, batchId, now, result, member)
|
|
|
963
1044
|
spawned_at: now.toISOString(),
|
|
964
1045
|
log_offset_at_spawn: logOffset,
|
|
965
1046
|
};
|
|
966
|
-
|
|
1047
|
+
const next = slot.status === 'assigned' || slot.status === 'recovering'
|
|
967
1048
|
? (0, state_1.transitionSlot)(withStatus, slot.id, 'running', patch, now)
|
|
968
1049
|
: withStatus;
|
|
969
|
-
// #677: persist the member's worktree context — a takeover redispatch or a
|
|
970
|
-
// `sched resume --batch` recheck after an engine restart must land in the
|
|
971
|
-
// SAME worktree/branch this prompt names, not re-derive (or worse,
|
|
972
|
-
// re-create) it.
|
|
973
|
-
next = (0, state_1.patchBatch)(next, batchId, {
|
|
974
|
-
member_branch: member.branch,
|
|
975
|
-
member_worktree: member.worktree,
|
|
976
|
-
member_pool_claimed: member.poolClaimed,
|
|
977
|
-
}, now);
|
|
978
1050
|
deps.journal.append((0, journal_1.unitEvent)('spawned', unit(batchId), {
|
|
979
1051
|
pid,
|
|
980
1052
|
tier,
|
|
@@ -983,11 +1055,11 @@ function spawnMember(deps, dispatch, state, slot, batchId, now, result, member)
|
|
|
983
1055
|
issue: memberIssue,
|
|
984
1056
|
...(0, dispatch_1.journalCmdModelFields)(spawnSpec),
|
|
985
1057
|
log: logFile,
|
|
986
|
-
worktree: member.worktree,
|
|
987
|
-
detail: `member ${
|
|
1058
|
+
worktree: target.member.worktree,
|
|
1059
|
+
detail: `member ${target.index}/${batch.members.length}${batch.member_dispatch === 'parallel' ? ' (parallel)' : ''}`,
|
|
988
1060
|
}), now);
|
|
989
|
-
result.spawned.push(
|
|
990
|
-
return next;
|
|
1061
|
+
result.spawned.push(target.unitId);
|
|
1062
|
+
return { ok: true, state: next };
|
|
991
1063
|
}
|
|
992
1064
|
/**
|
|
993
1065
|
* First claim of a `ready` batch: assign it a slot, run batch-setup (real
|
|
@@ -1054,7 +1126,38 @@ function claimAndSetup(deps, config, dispatch, batchId, now, result) {
|
|
|
1054
1126
|
pool_claimed: setup.poolClaimed ? 'true' : 'false',
|
|
1055
1127
|
},
|
|
1056
1128
|
});
|
|
1057
|
-
|
|
1129
|
+
// #809: decided ONCE, here, and persisted — a batch never changes mode mid-run.
|
|
1130
|
+
const mode = memberDispatchModeFor(deps.store.load(), batch, config);
|
|
1131
|
+
deps.journal.append((0, journal_1.unitEvent)('batch-setup-done', unit(batchId), {
|
|
1132
|
+
detail: `${setup.worktree} (members: ${mode})`,
|
|
1133
|
+
}), now);
|
|
1134
|
+
if (mode === 'parallel') {
|
|
1135
|
+
// Parallel members each claim their OWN slot (`batch:<id>#<issue>`), so
|
|
1136
|
+
// the setup slot is released here and the members' worktrees are prepared
|
|
1137
|
+
// by `spawnParallelMembers` (outside the lock, per member).
|
|
1138
|
+
deps.store.withLock((s) => {
|
|
1139
|
+
const b = (0, state_1.findBatch)(s, batchId);
|
|
1140
|
+
if (!b || b.status !== 'ready') {
|
|
1141
|
+
journalEvent(deps, 'unit-failed', unit(batchId), {
|
|
1142
|
+
reason: 'batch-left-ready-during-setup',
|
|
1143
|
+
detail: `worktree ${setup.worktree} created but batch is now '${b?.status ?? 'gone'}' — orphaned, manual cleanup required`,
|
|
1144
|
+
});
|
|
1145
|
+
return { state: releaseSlot(s, batchId, now), result: undefined };
|
|
1146
|
+
}
|
|
1147
|
+
let next = (0, state_1.patchBatch)(s, batchId, {
|
|
1148
|
+
branch: setup.branch,
|
|
1149
|
+
worktree: setup.worktree,
|
|
1150
|
+
run_id: setup.runId,
|
|
1151
|
+
pool_claimed: setup.poolClaimed,
|
|
1152
|
+
member_dispatch: 'parallel',
|
|
1153
|
+
member_runs: [],
|
|
1154
|
+
}, now);
|
|
1155
|
+
next = (0, state_1.transitionBatch)(next, batchId, 'executing', { executing_member: 0 }, now);
|
|
1156
|
+
return { state: releaseSlot(next, batchId, now), result: undefined };
|
|
1157
|
+
});
|
|
1158
|
+
spawnParallelMembers(deps, config, dispatch, batchId, now, result);
|
|
1159
|
+
return;
|
|
1160
|
+
}
|
|
1058
1161
|
// #677: prepare member 1's OWN worktree before claiming the spawn — real
|
|
1059
1162
|
// git/network work, so OUTSIDE the store lock (the same contract
|
|
1060
1163
|
// `runBatchSetup` just followed). All-or-nothing: a failed prep posts the
|
|
@@ -1114,6 +1217,7 @@ function claimAndSetup(deps, config, dispatch, batchId, now, result) {
|
|
|
1114
1217
|
worktree: setup.worktree,
|
|
1115
1218
|
run_id: setup.runId,
|
|
1116
1219
|
pool_claimed: setup.poolClaimed,
|
|
1220
|
+
member_dispatch: 'serial',
|
|
1117
1221
|
}, now);
|
|
1118
1222
|
next = (0, state_1.transitionBatch)(next, batchId, 'executing', { executing_member: 1 }, now);
|
|
1119
1223
|
const slot = slotFor(next, batchId);
|
|
@@ -1369,14 +1473,20 @@ function runValidate(deps, config, dispatch, batchId, now, result) {
|
|
|
1369
1473
|
return { state: s, result: false };
|
|
1370
1474
|
if (b.executing_member < b.members.length) {
|
|
1371
1475
|
return {
|
|
1372
|
-
state: (0, state_1.transitionBatch)(s, batchId, 'executing',
|
|
1476
|
+
state: (0, state_1.transitionBatch)(s, batchId, 'executing',
|
|
1477
|
+
// #809: a parallel batch spawns every run-less member itself.
|
|
1478
|
+
isParallelBatch(b) ? {} : { executing_member: b.executing_member + 1 }, now),
|
|
1373
1479
|
result: true,
|
|
1374
1480
|
};
|
|
1375
1481
|
}
|
|
1376
1482
|
return { state: (0, state_1.transitionBatch)(s, batchId, 'reviewing', {}, now), result: undefined };
|
|
1377
1483
|
});
|
|
1378
1484
|
if (admitted) {
|
|
1379
|
-
|
|
1485
|
+
const b = (0, state_1.findBatch)(deps.store.load(), batchId);
|
|
1486
|
+
if (b && isParallelBatch(b))
|
|
1487
|
+
spawnParallelMembers(deps, config, dispatch, batchId, now, result);
|
|
1488
|
+
else
|
|
1489
|
+
spawnMemberContinuation(deps, config, dispatch, batchId, now, result);
|
|
1380
1490
|
return;
|
|
1381
1491
|
}
|
|
1382
1492
|
spawnTailAgent(deps, config, dispatch, batchId, now, result);
|
|
@@ -1407,11 +1517,7 @@ function runValidate(deps, config, dispatch, batchId, now, result) {
|
|
|
1407
1517
|
const { state: attributed, outcome } = (0, recovery_1.beginAttribution)(state, batchId, { failing: suite.failing, footprints: memberFootprints(deps, batch) }, rDeps);
|
|
1408
1518
|
if (outcome.offenders.length === 0) {
|
|
1409
1519
|
const dissolve = (0, recovery_1.dissolveBatch)(attributed, batchId, { strategy: 'full', reason: 'unattributable-suite-failure' }, rDeps);
|
|
1410
|
-
deps.
|
|
1411
|
-
state: applyBatchAndIssues(s, dissolve.state, batchId, dissolve.requeued, batch.members),
|
|
1412
|
-
result: undefined,
|
|
1413
|
-
}));
|
|
1414
|
-
teardownBatch(deps, batchId);
|
|
1520
|
+
applyDissolveOutcome(deps, batchId, dissolve, now, batch.members);
|
|
1415
1521
|
result.blocked.push(...dissolve.requeued);
|
|
1416
1522
|
result.failed.push(unit(batchId));
|
|
1417
1523
|
return;
|
|
@@ -1473,10 +1579,14 @@ function evictOffender(deps, config, batchId, offender, attribution, now, result
|
|
|
1473
1579
|
const rDeps = recoveryDeps(deps, config, batch, now);
|
|
1474
1580
|
const outcome = (0, recovery_1.evictMembers)(state, batchId, { issues: [offender], reason: 'suite-red-after-fix', attribution, ranges: batch.ranges }, rDeps);
|
|
1475
1581
|
deps.store.withLock((s) => ({
|
|
1476
|
-
state: applyBatchAndIssues(s, outcome.state, batchId, outcome.requeued, batch.members),
|
|
1582
|
+
state: applyBatchAndIssues(s, outcome.state, batchId, [...outcome.requeued, ...(outcome.dissolve?.parked ?? [])], batch.members),
|
|
1477
1583
|
result: undefined,
|
|
1478
1584
|
}));
|
|
1479
|
-
|
|
1585
|
+
// #810: `dissolve.blocked` — a full dissolve refused over validated
|
|
1586
|
+
// members; the batch is blocked for an operator, not validating.
|
|
1587
|
+
if (outcome.dissolved || outcome.dissolve?.blocked === true) {
|
|
1588
|
+
if (outcome.dissolve?.blocked === true)
|
|
1589
|
+
stopAndReleaseBlocked(deps, batchId, now);
|
|
1480
1590
|
result.failed.push(unit(batchId));
|
|
1481
1591
|
return;
|
|
1482
1592
|
}
|
|
@@ -1487,7 +1597,7 @@ function evictOffender(deps, config, batchId, offender, attribution, now, result
|
|
|
1487
1597
|
return { state: s, result: undefined };
|
|
1488
1598
|
if (b.executing_member < b.members.length) {
|
|
1489
1599
|
return {
|
|
1490
|
-
state: (0, state_1.transitionBatch)(s, batchId, 'executing', { executing_member: b.executing_member + 1 }, now),
|
|
1600
|
+
state: (0, state_1.transitionBatch)(s, batchId, 'executing', isParallelBatch(b) ? {} : { executing_member: b.executing_member + 1 }, now),
|
|
1491
1601
|
result: undefined,
|
|
1492
1602
|
};
|
|
1493
1603
|
}
|
|
@@ -1535,6 +1645,15 @@ function journalAdvanceSkipped(deps, batchId, memberIssue, currentMember, claim)
|
|
|
1535
1645
|
});
|
|
1536
1646
|
}
|
|
1537
1647
|
function advanceMemberOrValidate(deps, config, dispatch, batchId, memberCount, currentMember, memberIssue, now, result) {
|
|
1648
|
+
// #809: a parallel batch reaches this serial rail only through `sched resume
|
|
1649
|
+
// --batch` resolving its gate-inconclusive member — hand back to the
|
|
1650
|
+
// parallel rail instead of advancing a serial member pointer.
|
|
1651
|
+
const current = (0, state_1.findBatch)(deps.store.load(), batchId);
|
|
1652
|
+
if (current && isParallelBatch(current)) {
|
|
1653
|
+
settleParallelResume(deps, batchId, memberIssue, now);
|
|
1654
|
+
advanceParallelBatch(deps, config, dispatch, batchId, now, result);
|
|
1655
|
+
return;
|
|
1656
|
+
}
|
|
1538
1657
|
const isLast = currentMember >= memberCount;
|
|
1539
1658
|
if (isLast) {
|
|
1540
1659
|
const claim = claimMemberResolution(deps, batchId, currentMember, now, () => ({
|
|
@@ -1585,10 +1704,10 @@ function advanceMemberOrValidate(deps, config, dispatch, batchId, memberCount, c
|
|
|
1585
1704
|
* both always read state fresh.
|
|
1586
1705
|
*/
|
|
1587
1706
|
function evictMemberAndContinue(deps, config, dispatch, batchId, batch, memberIssue, failure, now, result) {
|
|
1588
|
-
const {
|
|
1707
|
+
const { batchEnded, duplicate } = evictMemberDirectly(deps, config, batchId, memberIssue, failure, now);
|
|
1589
1708
|
if (duplicate)
|
|
1590
1709
|
return;
|
|
1591
|
-
if (
|
|
1710
|
+
if (batchEnded) {
|
|
1592
1711
|
result.failed.push(unit(batchId));
|
|
1593
1712
|
return;
|
|
1594
1713
|
}
|
|
@@ -1639,7 +1758,41 @@ function runIncrementalGate(deps, config, dispatch, batchId, batch, memberIssue,
|
|
|
1639
1758
|
// the fallback for a batch dispatched before member worktrees existed (its
|
|
1640
1759
|
// members' commits landed directly on the integration branch).
|
|
1641
1760
|
const worktree = batch.member_worktree ?? batch.worktree;
|
|
1761
|
+
const verdict = evaluateIncrementalGate(deps, batchId, worktree, memberIssue, now);
|
|
1762
|
+
if (verdict.kind === 'pass')
|
|
1763
|
+
return false;
|
|
1764
|
+
if (verdict.kind === 'failed') {
|
|
1765
|
+
deps.store.withLock((s) => ({ state: releaseSlot(s, batchId, now), result: undefined }));
|
|
1766
|
+
evictMemberAndContinue(deps, config, dispatch, batchId, batch, memberIssue, { reason: verdict.reason, detail: verdict.detail }, now, result);
|
|
1767
|
+
return true;
|
|
1768
|
+
}
|
|
1769
|
+
deps.store.withLock((s) => ({ state: releaseSlot(s, batchId, now), result: undefined }));
|
|
1770
|
+
// #677: F.11's block semantics are "the member's commit stays on the
|
|
1771
|
+
// branch, nothing requeued/reverted" — under member worktrees that is
|
|
1772
|
+
// true only if the member LANDS before the batch blocks. The block means
|
|
1773
|
+
// the gate could not reach a verdict, not that the work is bad, so the
|
|
1774
|
+
// verified member joins the integration branch and is there for an
|
|
1775
|
+
// operator (or the #686 out-of-band reconcile) to act on. A landing
|
|
1776
|
+
// failure blocks the batch with its own reason — same operator outcome.
|
|
1777
|
+
const landed = landMemberBranch(deps, batchId, now);
|
|
1778
|
+
if (!landed.ok) {
|
|
1779
|
+
blockLandingFailure(deps, config, batch, landed.reason, now, result);
|
|
1780
|
+
return true;
|
|
1781
|
+
}
|
|
1782
|
+
blockBatchForOperator(deps, config, batchId, { reason: verdict.reason, milestonePhase: 'batch-review' }, now, result);
|
|
1783
|
+
return true;
|
|
1784
|
+
}
|
|
1785
|
+
/**
|
|
1786
|
+
* Run the incremental gate's capabilities against one member worktree and
|
|
1787
|
+
* reach a verdict — journaling, gate logs and `member_gates` included, but NO
|
|
1788
|
+
* state action: serial (`runIncrementalGate`) and parallel
|
|
1789
|
+
* (`reconcileParallelMemberSlot`, #809) members act on the verdict
|
|
1790
|
+
* differently. See `runIncrementalGate`'s doc for the policy.
|
|
1791
|
+
*/
|
|
1792
|
+
function evaluateIncrementalGate(deps, batchId, worktree, memberIssue, now) {
|
|
1642
1793
|
const runCapability = deps.runCapability;
|
|
1794
|
+
if (!runCapability)
|
|
1795
|
+
return { kind: 'pass' };
|
|
1643
1796
|
const gateResults = ['typecheck.run', 'test.focused'].map((id) => ({
|
|
1644
1797
|
id,
|
|
1645
1798
|
...runCapability(worktree, id),
|
|
@@ -1680,13 +1833,10 @@ function runIncrementalGate(deps, config, dispatch, batchId, batch, memberIssue,
|
|
|
1680
1833
|
// trap (#594's shape — absence reading as a verdict). Silence must never
|
|
1681
1834
|
// be mistaken for a pass.
|
|
1682
1835
|
//
|
|
1683
|
-
// #632: confirmed this fires once per member, not once per tick
|
|
1684
|
-
//
|
|
1685
|
-
//
|
|
1686
|
-
//
|
|
1687
|
-
// `batch.executing_member` before returning — so the very next tick
|
|
1688
|
-
// reads a DIFFERENT `memberIssue` from `batch.members[executing_member -
|
|
1689
|
-
// 1]`, and this gate never runs twice against the same member.
|
|
1836
|
+
// #632: confirmed this fires once per member, not once per tick: the
|
|
1837
|
+
// gate runs only on a member dispatch's verified completion, and both
|
|
1838
|
+
// rails resolve that member (serial: the pointer advances; parallel:
|
|
1839
|
+
// the run leaves `running` and its slot is released) before returning.
|
|
1690
1840
|
journalEvent(deps, 'gate-skipped', unit(batchId), {
|
|
1691
1841
|
issue: memberIssue,
|
|
1692
1842
|
reason: `gate-skipped:${skipped.id}`,
|
|
@@ -1734,15 +1884,13 @@ function runIncrementalGate(deps, config, dispatch, batchId, batch, memberIssue,
|
|
|
1734
1884
|
recordMemberGate(deps, batchId, memberIssue, decisiveGate, now);
|
|
1735
1885
|
}
|
|
1736
1886
|
if (earnedFailure) {
|
|
1737
|
-
const reason = `incremental-gate-failed:${earnedFailure.id}`;
|
|
1738
1887
|
writeGateLog(deps, batchId, earnedFailure.id, memberIssue, earnedFailure.outputTail);
|
|
1739
1888
|
const excerpt = gateDetailExcerpt(earnedFailure.outputTail, earnedFailure.reason);
|
|
1740
|
-
|
|
1741
|
-
|
|
1742
|
-
reason
|
|
1889
|
+
return {
|
|
1890
|
+
kind: 'failed',
|
|
1891
|
+
reason: `incremental-gate-failed:${earnedFailure.id}`,
|
|
1743
1892
|
detail: withExcerpt(`cap run ${earnedFailure.id} reported task-failed with failing-test evidence after member review done`, excerpt),
|
|
1744
|
-
}
|
|
1745
|
-
return true;
|
|
1893
|
+
};
|
|
1746
1894
|
}
|
|
1747
1895
|
const inconclusive = gateInconclusive ?? unevidencedFailure;
|
|
1748
1896
|
if (inconclusive) {
|
|
@@ -1758,23 +1906,9 @@ function runIncrementalGate(deps, config, dispatch, batchId, batch, memberIssue,
|
|
|
1758
1906
|
reason,
|
|
1759
1907
|
detail: withExcerpt(`cap run ${inconclusive.id} ${describeInconclusive(inconclusive)} after member review done`, excerpt),
|
|
1760
1908
|
});
|
|
1761
|
-
|
|
1762
|
-
// #677: F.11's block semantics are "the member's commit stays on the
|
|
1763
|
-
// branch, nothing requeued/reverted" — under member worktrees that is
|
|
1764
|
-
// true only if the member LANDS before the batch blocks. The block means
|
|
1765
|
-
// the gate could not reach a verdict, not that the work is bad, so the
|
|
1766
|
-
// verified member joins the integration branch and is there for an
|
|
1767
|
-
// operator (or the #686 out-of-band reconcile) to act on. A landing
|
|
1768
|
-
// failure blocks the batch with its own reason — same operator outcome.
|
|
1769
|
-
const landed = landMemberBranch(deps, batchId, now);
|
|
1770
|
-
if (!landed.ok) {
|
|
1771
|
-
blockLandingFailure(deps, config, batch, landed.reason, now, result);
|
|
1772
|
-
return true;
|
|
1773
|
-
}
|
|
1774
|
-
blockBatchForOperator(deps, config, batchId, { reason, milestonePhase: 'batch-review' }, now, result);
|
|
1775
|
-
return true;
|
|
1909
|
+
return { kind: 'inconclusive', reason, capability: inconclusive.id };
|
|
1776
1910
|
}
|
|
1777
|
-
return
|
|
1911
|
+
return { kind: 'pass' };
|
|
1778
1912
|
}
|
|
1779
1913
|
/**
|
|
1780
1914
|
* The member is verifiably done — self-reported complete AND (when checked)
|
|
@@ -1984,11 +2118,13 @@ function resumeBlockedGate(deps, config, dispatch, batchId, now) {
|
|
|
1984
2118
|
* Exactly-once per dispatch, mirroring `recordDispatchRunLog`'s own
|
|
1985
2119
|
* invariant: called from both of `reconcileMemberSlot`'s exit branches
|
|
1986
2120
|
* (member complete, member blocked/dead) — a batch never redispatches the
|
|
1987
|
-
* same member slot (
|
|
1988
|
-
*
|
|
2121
|
+
* same member slot (an evicted member is parked, #810, and any later
|
|
2122
|
+
* full-cycle run is an independent unit), so unlike `engine.ts`'s per-unit log, a member's log file is
|
|
1989
2123
|
* always one-shot and reading from offset 0 is always correct.
|
|
1990
2124
|
*/
|
|
1991
|
-
function recordMemberRunLog(deps, dispatch, state, batchId,
|
|
2125
|
+
function recordMemberRunLog(deps, dispatch, state, batchId,
|
|
2126
|
+
/** 1-based member index — names the per-member log file (serial: `executing_member`; parallel: `MemberRun.index`). */
|
|
2127
|
+
memberIndex, memberIssue, slot, now) {
|
|
1992
2128
|
if (slot.status !== 'running' || slot.spawned_at === null) {
|
|
1993
2129
|
journalEvent(deps, 'run-log-skipped', unit(batchId), {
|
|
1994
2130
|
issue: memberIssue,
|
|
@@ -2002,7 +2138,7 @@ function recordMemberRunLog(deps, dispatch, state, batchId, batch, memberIssue,
|
|
|
2002
2138
|
const memberEntry = (0, state_1.findEntry)(state, memberIssue);
|
|
2003
2139
|
const tier = memberEntry ? (0, dispatch_1.memberDispatchTier)(memberEntry) : 'mid';
|
|
2004
2140
|
const { cmd, model } = (0, dispatch_1.resolveTierSpawn)(dispatch, tier, memberIssue);
|
|
2005
|
-
const logFile = (0, dispatch_1.batchMemberLogPath)(deps.store.runsDir, batchId,
|
|
2141
|
+
const logFile = (0, dispatch_1.batchMemberLogPath)(deps.store.runsDir, batchId, memberIndex, memberIssue);
|
|
2006
2142
|
// #629: fenced to THIS dispatch's slice (`log_offset_at_spawn`, stamped by
|
|
2007
2143
|
// `spawnMember`/`spawnMemberContinuation` at spawn time) rather than a
|
|
2008
2144
|
// fixed byte-0 read. Before #629 a member log never outlived one attempt
|
|
@@ -2070,14 +2206,12 @@ function terminalMilestoneForDispatch(deps, memberIssue, slot) {
|
|
|
2070
2206
|
}
|
|
2071
2207
|
return latest;
|
|
2072
2208
|
}
|
|
2073
|
-
|
|
2074
|
-
|
|
2075
|
-
|
|
2076
|
-
|
|
2077
|
-
|
|
2078
|
-
|
|
2079
|
-
if (memberIssue === undefined)
|
|
2080
|
-
return;
|
|
2209
|
+
/**
|
|
2210
|
+
* Read one member dispatch's fate: the fenced terminal milestone (#575/#605/
|
|
2211
|
+
* #622) plus liveness, journaling a stale terminal milestone once per
|
|
2212
|
+
* dispatch (#610). Pure reading — the caller acts on the result.
|
|
2213
|
+
*/
|
|
2214
|
+
function readMemberDispatch(deps, batchId, memberIssue, slot, now) {
|
|
2081
2215
|
const dead = slot.pid !== null && !deps.spawnDeps.isAlive(slot.pid, slot.pid_start ?? undefined);
|
|
2082
2216
|
// #622: the milestone this DISPATCH's fate should be read from — the last
|
|
2083
2217
|
// TERMINAL one it posted, not merely the newest one on the issue.
|
|
@@ -2096,7 +2230,7 @@ function reconcileMemberSlot(deps, config, dispatch, batchId, slot, now, result)
|
|
|
2096
2230
|
// poll) this degrades to the previous latest-only behaviour.
|
|
2097
2231
|
const milestone = terminalMilestoneForDispatch(deps, memberIssue, slot);
|
|
2098
2232
|
if (milestone === undefined)
|
|
2099
|
-
return
|
|
2233
|
+
return { kind: 'unreachable' }; // pause this batch's decisions
|
|
2100
2234
|
// #575 / #605: fence to THIS member dispatch's `spawned_at` — a member
|
|
2101
2235
|
// re-added to a fresh batch run after a PREVIOUS batch already posted a
|
|
2102
2236
|
// terminal `mode=slot` milestone (pilot re-run, requeue-with-context, and
|
|
@@ -2135,7 +2269,65 @@ function reconcileMemberSlot(deps, config, dispatch, batchId, slot, now, result)
|
|
|
2135
2269
|
result: undefined,
|
|
2136
2270
|
}));
|
|
2137
2271
|
}
|
|
2138
|
-
if ((0, groundtruth_1.isMemberComplete)(milestone, slot.spawned_at))
|
|
2272
|
+
if ((0, groundtruth_1.isMemberComplete)(milestone, slot.spawned_at))
|
|
2273
|
+
return { kind: 'complete' };
|
|
2274
|
+
return {
|
|
2275
|
+
kind: 'pending',
|
|
2276
|
+
milestone,
|
|
2277
|
+
dead,
|
|
2278
|
+
blockedNow: (0, groundtruth_1.isMemberBlocked)(milestone, slot.spawned_at),
|
|
2279
|
+
};
|
|
2280
|
+
}
|
|
2281
|
+
/**
|
|
2282
|
+
* The eviction a blocked-or-dead member dispatch earns (shared by both rails).
|
|
2283
|
+
* #605: only a milestone THIS dispatch posted may name the reason. A dead
|
|
2284
|
+
* agent whose issue carries only a stale `blocked` milestone is an unverified
|
|
2285
|
+
* exit, not a re-run of the previous run's block — reporting the old
|
|
2286
|
+
* `reason=` here is what made the journal misdirect, naming a precondition
|
|
2287
|
+
* that had already been fixed on a run where it no longer applied.
|
|
2288
|
+
* #804: `memberBlockedReason` also names a `review partial` hand-back.
|
|
2289
|
+
*/
|
|
2290
|
+
function memberFailureFor(read, lastTool) {
|
|
2291
|
+
const rawReason = read.blockedNow && read.milestone ? (0, groundtruth_1.memberBlockedReason)(read.milestone) : undefined;
|
|
2292
|
+
const reason = typeof rawReason === 'string' && rawReason.length > 0
|
|
2293
|
+
? sanitizeUntrustedText(rawReason)
|
|
2294
|
+
: read.dead
|
|
2295
|
+
? 'agent-exited-unverified'
|
|
2296
|
+
: 'member-blocked';
|
|
2297
|
+
return {
|
|
2298
|
+
reason,
|
|
2299
|
+
// #810: a block THIS dispatch posted is the member's own hand-back — parked
|
|
2300
|
+
// `handed-back`, never counted toward the dissolve threshold. A member that
|
|
2301
|
+
// died without one is an engine-decided eviction (`agent-exited-unverified`).
|
|
2302
|
+
kind: read.blockedNow ? 'handed-back' : 'evicted',
|
|
2303
|
+
// This rail covers a self-reported block AND a member that simply died, so the
|
|
2304
|
+
// detail must follow the RESOLVED reason — `member blocked` on an
|
|
2305
|
+
// `agent-exited-unverified` eviction contradicts the reason on its own journal line.
|
|
2306
|
+
detail: reason === 'agent-exited-unverified'
|
|
2307
|
+
? 'member agent exited without posting a terminal milestone'
|
|
2308
|
+
: 'member blocked',
|
|
2309
|
+
// #591: attributes an unverified member exit to a concrete cause (e.g.
|
|
2310
|
+
// `Monitor`) without opening the transcript. Gated on the RESOLVED reason, not
|
|
2311
|
+
// `dead` — a member can be simultaneously `dead` AND carry a milestone-posted
|
|
2312
|
+
// `reason` (it posted `blocked` and then exited), and that real block has no
|
|
2313
|
+
// log-derived cause to attribute; only `agent-exited-unverified` does.
|
|
2314
|
+
extraKv: reason === 'agent-exited-unverified' && lastTool !== null
|
|
2315
|
+
? { last_tool: lastTool }
|
|
2316
|
+
: undefined,
|
|
2317
|
+
};
|
|
2318
|
+
}
|
|
2319
|
+
function reconcileMemberSlot(deps, config, dispatch, batchId, slot, now, result) {
|
|
2320
|
+
const state0 = deps.store.load();
|
|
2321
|
+
const batch = (0, state_1.findBatch)(state0, batchId);
|
|
2322
|
+
if (!batch)
|
|
2323
|
+
return;
|
|
2324
|
+
const memberIssue = batch.members[batch.executing_member - 1];
|
|
2325
|
+
if (memberIssue === undefined)
|
|
2326
|
+
return;
|
|
2327
|
+
const read = readMemberDispatch(deps, batchId, memberIssue, slot, now);
|
|
2328
|
+
if (read.kind === 'unreachable')
|
|
2329
|
+
return;
|
|
2330
|
+
if (read.kind === 'complete') {
|
|
2139
2331
|
deps.journal.append((0, journal_1.unitEvent)('external-advance', unit(batchId), {
|
|
2140
2332
|
issue: memberIssue,
|
|
2141
2333
|
detail: 'member review done',
|
|
@@ -2143,7 +2335,7 @@ function reconcileMemberSlot(deps, config, dispatch, batchId, slot, now, result)
|
|
|
2143
2335
|
// The member's own agent process is done regardless of what the
|
|
2144
2336
|
// incremental gate below decides — record its telemetry once here (#564)
|
|
2145
2337
|
// rather than at each of this branch's two later exit points.
|
|
2146
|
-
recordMemberRunLog(deps, dispatch, state0, batchId, batch, memberIssue, slot, now);
|
|
2338
|
+
recordMemberRunLog(deps, dispatch, state0, batchId, batch.executing_member, memberIssue, slot, now);
|
|
2147
2339
|
// Incremental gate (#523 AC2, revised #583) — see `runIncrementalGate`'s
|
|
2148
2340
|
// own doc comment for the three-way policy. A `true` return means the
|
|
2149
2341
|
// gate already decided the member's fate (evicted or blocked) and
|
|
@@ -2155,9 +2347,8 @@ function reconcileMemberSlot(deps, config, dispatch, batchId, slot, now, result)
|
|
|
2155
2347
|
completeMemberGate(deps, config, dispatch, batchId, batch, memberIssue, now, result);
|
|
2156
2348
|
return;
|
|
2157
2349
|
}
|
|
2158
|
-
|
|
2159
|
-
|
|
2160
|
-
!blockedNow &&
|
|
2350
|
+
if (read.dead &&
|
|
2351
|
+
!read.blockedNow &&
|
|
2161
2352
|
handleDeadDispatchApiError(deps, batchId, (0, dispatch_1.batchMemberLogPath)(deps.store.runsDir, batchId, batch.executing_member, memberIssue), slot.log_offset_at_spawn ?? 0, now)) {
|
|
2162
2353
|
// #629: a confirmed provider API error is not a real member failure —
|
|
2163
2354
|
// evicting the member for an account-wide spend wall would silently
|
|
@@ -2166,21 +2357,8 @@ function reconcileMemberSlot(deps, config, dispatch, batchId, slot, now, result)
|
|
|
2166
2357
|
// place, keeping its position.
|
|
2167
2358
|
return;
|
|
2168
2359
|
}
|
|
2169
|
-
if (blockedNow || dead) {
|
|
2170
|
-
|
|
2171
|
-
// agent whose issue carries only a stale `blocked` milestone is an
|
|
2172
|
-
// unverified exit, not a re-run of the previous run's block — reporting
|
|
2173
|
-
// the old `reason=` here is what made the journal misdirect, naming a
|
|
2174
|
-
// precondition that had already been fixed on a run where it no longer
|
|
2175
|
-
// applied.
|
|
2176
|
-
// #804: `memberBlockedReason` also names a `review partial` hand-back.
|
|
2177
|
-
const rawReason = blockedNow && milestone ? (0, groundtruth_1.memberBlockedReason)(milestone) : undefined;
|
|
2178
|
-
const reason = typeof rawReason === 'string' && rawReason.length > 0
|
|
2179
|
-
? sanitizeUntrustedText(rawReason)
|
|
2180
|
-
: dead
|
|
2181
|
-
? 'agent-exited-unverified'
|
|
2182
|
-
: 'member-blocked';
|
|
2183
|
-
const lastTool = recordMemberRunLog(deps, dispatch, state0, batchId, batch, memberIssue, slot, now);
|
|
2360
|
+
if (read.blockedNow || read.dead) {
|
|
2361
|
+
const lastTool = recordMemberRunLog(deps, dispatch, state0, batchId, batch.executing_member, memberIssue, slot, now);
|
|
2184
2362
|
deps.store.withLock((s) => ({ state: releaseSlot(s, batchId, now), result: undefined }));
|
|
2185
2363
|
// A member that never went green (RFC F.1) evicts DIRECTLY — no aggregate
|
|
2186
2364
|
// suite has run yet, so there is nothing for `attributing`/`evicting` (the
|
|
@@ -2190,78 +2368,88 @@ function reconcileMemberSlot(deps, config, dispatch, batchId, slot, now, result)
|
|
|
2190
2368
|
// attributing → evicting` is not even a legal edge from mid-`executing`
|
|
2191
2369
|
// (BATCH_TRANSITIONS has no `executing → evicting`) — this stays entirely
|
|
2192
2370
|
// within `executing`/`dissolving`, both of which ARE legal from here.
|
|
2193
|
-
evictMemberAndContinue(deps, config, dispatch, batchId, batch, memberIssue,
|
|
2194
|
-
reason,
|
|
2195
|
-
// This rail covers a self-reported block AND a member that simply died, so the
|
|
2196
|
-
// detail must follow the RESOLVED reason — `member blocked` on an
|
|
2197
|
-
// `agent-exited-unverified` eviction contradicts the reason on its own journal line.
|
|
2198
|
-
detail: reason === 'agent-exited-unverified'
|
|
2199
|
-
? 'member agent exited without posting a terminal milestone'
|
|
2200
|
-
: 'member blocked',
|
|
2201
|
-
// #591: attributes an unverified member exit to a concrete cause (e.g.
|
|
2202
|
-
// `Monitor`) without opening the transcript. Gated on the RESOLVED reason, not
|
|
2203
|
-
// `dead` — a member can be simultaneously `dead` AND carry a milestone-posted
|
|
2204
|
-
// `reason` (it posted `blocked` and then exited), and that real block has no
|
|
2205
|
-
// log-derived cause to attribute; only `agent-exited-unverified` does.
|
|
2206
|
-
extraKv: reason === 'agent-exited-unverified' && lastTool !== null
|
|
2207
|
-
? { last_tool: lastTool }
|
|
2208
|
-
: undefined,
|
|
2209
|
-
}, now, result);
|
|
2371
|
+
evictMemberAndContinue(deps, config, dispatch, batchId, batch, memberIssue, memberFailureFor(read, lastTool), now, result);
|
|
2210
2372
|
}
|
|
2211
2373
|
}
|
|
2212
2374
|
/**
|
|
2213
|
-
*
|
|
2214
|
-
* the batch past its `dissolve_policy` threshold (#563;
|
|
2215
|
-
* going through `recovery.ts`'s `evictMembers` (which
|
|
2216
|
-
* `attributing`/`evicting` status and a commit range to revert; a
|
|
2217
|
-
* evicted here has neither). Returns whether the batch dissolved.
|
|
2375
|
+
* Park a member that left the batch before landing, record the exit, and
|
|
2376
|
+
* dissolve if this tips the batch past its `dissolve_policy` threshold (#563;
|
|
2377
|
+
* RFC F.1/F.8) — WITHOUT going through `recovery.ts`'s `evictMembers` (which
|
|
2378
|
+
* needs `attributing`/`evicting` status and a commit range to revert; a
|
|
2379
|
+
* member evicted here has neither). Returns whether the batch dissolved.
|
|
2380
|
+
*
|
|
2381
|
+
* #810 — the member is PARKED, never auto-requeued:
|
|
2382
|
+
* - `failure.kind === 'handed-back'` (the member posted its own `blocked` /
|
|
2383
|
+
* `review partial` milestone) → entry `handed-back`, record
|
|
2384
|
+
* `kind: 'handed-back'`, journal `member-handed-back`. A hand-back is a
|
|
2385
|
+
* valued outcome, not a batch failure: it never counts toward the dissolve
|
|
2386
|
+
* threshold.
|
|
2387
|
+
* - otherwise (unverified exit, gate task-failed, landing conflict, worktree
|
|
2388
|
+
* prep) → entry `evicted`, journal `unit-failed`, counted.
|
|
2389
|
+
* Either way the member branch is recorded on the record and the entry's
|
|
2390
|
+
* `failure_evidence`, and the batch's dispatch profile on the entry — so an
|
|
2391
|
+
* operator's `sched requeue` continues the work from that branch on the same
|
|
2392
|
+
* provider. The pre-#810 rail requeued `mode: 'full', dispatch_profile: null`
|
|
2393
|
+
* from the base branch, discarding the member's commits and its profile.
|
|
2394
|
+
*
|
|
2395
|
+
* #810: when the threshold trips while members are already VALIDATED (landed
|
|
2396
|
+
* on the integration branch), the batch does not dissolve — dissolving would
|
|
2397
|
+
* requeue validated work from scratch. It journals `dissolve-suppressed` and
|
|
2398
|
+
* continues with the remaining members; its validated members still ship.
|
|
2218
2399
|
*
|
|
2219
2400
|
* A member already in `evictions[]` is a NO-OP here, not merely a skipped
|
|
2220
|
-
* record (#595 AC1): the
|
|
2221
|
-
* overwrite the first eviction's `failure_evidence` — leaving `evictions[]`
|
|
2222
|
-
* and the queue entry disagreeing about why the member was evicted — and, if
|
|
2223
|
-
* the member has since been re-dispatched full-cycle, would take the
|
|
2224
|
-
* `executing → evicted → requeued` rail and kill that live run.
|
|
2401
|
+
* record (#595 AC1): the park must not run a second time either.
|
|
2225
2402
|
*
|
|
2226
|
-
* Returns `{
|
|
2227
|
-
*
|
|
2228
|
-
*
|
|
2229
|
-
*
|
|
2230
|
-
*
|
|
2403
|
+
* Returns `{ batchEnded, duplicate }`: `batchEnded` is whether this exit ended
|
|
2404
|
+
* the batch's run — it dissolved, or (a `full` dissolve refused over validated
|
|
2405
|
+
* members, #810) it blocked for an operator; either way the caller must not
|
|
2406
|
+
* advance it. `duplicate: true` means another resolution had already claimed
|
|
2407
|
+
* this member's exit, and the caller must journal nothing and advance nothing
|
|
2408
|
+
* on top of it (#613). A dissolve interrupted between pass 1 and pass 2 is not
|
|
2409
|
+
* retried by a re-entry: the re-entry takes the duplicate path (pre-existing,
|
|
2410
|
+
* #595/#613) and the batch continues.
|
|
2231
2411
|
*/
|
|
2232
2412
|
function evictMemberDirectly(deps, config, batchId, memberIssue, failure, now) {
|
|
2233
2413
|
const { reason } = failure;
|
|
2414
|
+
const kind = failure.kind ?? 'evicted';
|
|
2234
2415
|
const dissolvePolicy = (0, types_1.resolveDissolvePolicy)(config.dissolve_policy);
|
|
2235
|
-
// Pass 1 (pure —
|
|
2236
|
-
//
|
|
2416
|
+
// Pass 1 (pure — park + record the exit): safe to run entirely inside the
|
|
2417
|
+
// lock, unlike `dissolveBatch` below, which shells out
|
|
2237
2418
|
// (`deps.exec`/`postMilestone`) and so must NOT hold the lock while it runs.
|
|
2238
|
-
const { triggered, duplicate, prior } = deps.store.withLock((s) => {
|
|
2419
|
+
const { triggered, duplicate, prior, branch, validated } = deps.store.withLock((s) => {
|
|
2239
2420
|
const b = (0, state_1.findBatch)(s, batchId);
|
|
2240
2421
|
if (!b) {
|
|
2241
|
-
return {
|
|
2422
|
+
return {
|
|
2423
|
+
state: s,
|
|
2424
|
+
result: {
|
|
2425
|
+
triggered: false,
|
|
2426
|
+
duplicate: false,
|
|
2427
|
+
prior: undefined,
|
|
2428
|
+
branch: null,
|
|
2429
|
+
validated: [],
|
|
2430
|
+
},
|
|
2431
|
+
};
|
|
2242
2432
|
}
|
|
2243
2433
|
const priorRecord = b.evictions.find((e) => e.issue === memberIssue);
|
|
2244
2434
|
if (priorRecord) {
|
|
2245
2435
|
// A duplicate never re-evaluates the dissolve threshold: the resolution that WROTE
|
|
2246
2436
|
// this record already did, under this same lock. (#595: a duplicate must not count
|
|
2247
|
-
// twice; #613: it must not act on the batch at all.)
|
|
2248
|
-
// interrupted between pass 1 and pass 2 is not retried here — re-entering
|
|
2249
|
-
// `dissolveBatch` on an already-terminal batch throws `IllegalTransitionError`.
|
|
2437
|
+
// twice; #613: it must not act on the batch at all.)
|
|
2250
2438
|
return {
|
|
2251
2439
|
state: s,
|
|
2252
|
-
result: {
|
|
2440
|
+
result: {
|
|
2441
|
+
triggered: false,
|
|
2442
|
+
duplicate: true,
|
|
2443
|
+
prior: priorRecord,
|
|
2444
|
+
branch: null,
|
|
2445
|
+
validated: [],
|
|
2446
|
+
},
|
|
2253
2447
|
};
|
|
2254
2448
|
}
|
|
2255
|
-
const
|
|
2256
|
-
|
|
2257
|
-
reason,
|
|
2258
|
-
|
|
2259
|
-
attribution: 'none',
|
|
2260
|
-
reverted_commits: [],
|
|
2261
|
-
at: now.toISOString(),
|
|
2262
|
-
};
|
|
2263
|
-
const requeueResult = (0, state_1.requeueMember)(s, memberIssue, { mode: 'full', batch: null }, reason, now, { failure_evidence: evidence });
|
|
2264
|
-
let next = requeueResult.state;
|
|
2449
|
+
const memberBranch = (0, state_1.recordedMemberBranch)(b, memberIssue);
|
|
2450
|
+
let next = (0, state_1.parkMember)(s, memberIssue, kind, reason, now, {
|
|
2451
|
+
failure_evidence: (0, state_1.memberExitEvidence)(batchId, reason, memberBranch, now),
|
|
2452
|
+
}).state;
|
|
2265
2453
|
// `appendEvictions` stays the state-level backstop even though the
|
|
2266
2454
|
// duplicate is already short-circuited above — it is the one append path.
|
|
2267
2455
|
const { evictions, duplicate: duplicateRecords } = (0, state_1.appendEvictions)(b, [
|
|
@@ -2271,6 +2459,8 @@ function evictMemberDirectly(deps, config, batchId, memberIssue, failure, now) {
|
|
|
2271
2459
|
attribution: 'none',
|
|
2272
2460
|
reverted_commits: [],
|
|
2273
2461
|
group: [],
|
|
2462
|
+
kind,
|
|
2463
|
+
branch: memberBranch,
|
|
2274
2464
|
at: now.toISOString(),
|
|
2275
2465
|
},
|
|
2276
2466
|
]);
|
|
@@ -2282,6 +2472,8 @@ function evictMemberDirectly(deps, config, batchId, memberIssue, failure, now) {
|
|
|
2282
2472
|
triggered: updated !== undefined && (0, recovery_1.checkDissolveTrigger)(updated, dissolvePolicy),
|
|
2283
2473
|
duplicate: duplicateRecords.length > 0,
|
|
2284
2474
|
prior: undefined,
|
|
2475
|
+
branch: memberBranch,
|
|
2476
|
+
validated: updated !== undefined ? (0, state_1.validatedMembersOf)(next, updated) : [],
|
|
2285
2477
|
},
|
|
2286
2478
|
};
|
|
2287
2479
|
});
|
|
@@ -2290,41 +2482,45 @@ function evictMemberDirectly(deps, config, batchId, memberIssue, failure, now) {
|
|
|
2290
2482
|
issue: memberIssue,
|
|
2291
2483
|
detail: (0, state_1.duplicateEvictionDetail)(memberIssue, reason, prior),
|
|
2292
2484
|
});
|
|
2293
|
-
// Another resolution already claimed this member's
|
|
2485
|
+
// Another resolution already claimed this member's exit record — the
|
|
2294
2486
|
// caller must not journal its own `unit-failed`/advance the batch again
|
|
2295
|
-
// on top of that one (#613
|
|
2296
|
-
|
|
2297
|
-
return { dissolved: false, duplicate: true };
|
|
2487
|
+
// on top of that one (#613).
|
|
2488
|
+
return { batchEnded: false, duplicate: true };
|
|
2298
2489
|
}
|
|
2299
2490
|
// This call owns the member's resolution, so it owns recording WHY — before pass 2 below
|
|
2300
|
-
// can dissolve, tear the worktree down, or be killed mid-shell-out and leave the
|
|
2301
|
-
//
|
|
2302
|
-
//
|
|
2303
|
-
|
|
2304
|
-
// identity.
|
|
2305
|
-
journalEvent(deps, 'unit-failed', unit(batchId), {
|
|
2491
|
+
// can dissolve, tear the worktree down, or be killed mid-shell-out and leave the record
|
|
2492
|
+
// on disk with no journal line naming its cause (#613). The caller's extras go first so
|
|
2493
|
+
// the authoritative keys always win.
|
|
2494
|
+
journalEvent(deps, kind === 'handed-back' ? 'member-handed-back' : 'unit-failed', unit(batchId), {
|
|
2306
2495
|
...(failure.extraKv ?? {}),
|
|
2307
2496
|
issue: memberIssue,
|
|
2308
2497
|
reason,
|
|
2309
|
-
detail: failure.detail
|
|
2498
|
+
detail: `${failure.detail} — parked ${kind}` +
|
|
2499
|
+
(branch !== null ? ` (work on ${branch})` : '') +
|
|
2500
|
+
`; remedy: sched requeue --issue ${memberIssue}`,
|
|
2310
2501
|
});
|
|
2311
2502
|
if (!triggered)
|
|
2312
|
-
return {
|
|
2503
|
+
return { batchEnded: false, duplicate: false };
|
|
2504
|
+
if (validated.length > 0) {
|
|
2505
|
+
// #810: never dissolve over validated work — it stays landed and ships
|
|
2506
|
+
// with the batch; the members that failed are parked, not requeued.
|
|
2507
|
+
journalEvent(deps, 'dissolve-suppressed', unit(batchId), {
|
|
2508
|
+
issue: memberIssue,
|
|
2509
|
+
detail: `eviction threshold reached, but member(s) ${validated.join(',')} are validated on the integration branch — keeping them and continuing with the remaining members instead of dissolving`,
|
|
2510
|
+
});
|
|
2511
|
+
return { batchEnded: false, duplicate: false };
|
|
2512
|
+
}
|
|
2313
2513
|
// Pass 2 (outside the lock — dissolveBatch shells out): re-load fresh
|
|
2314
2514
|
// (pass 1's write already landed), dissolve, then re-apply just this
|
|
2315
2515
|
// batch's + the requeued members' state under a fresh lock.
|
|
2316
2516
|
const state = deps.store.load();
|
|
2317
2517
|
const batch = (0, state_1.findBatch)(state, batchId);
|
|
2318
2518
|
if (!batch)
|
|
2319
|
-
return {
|
|
2519
|
+
return { batchEnded: false, duplicate: false };
|
|
2320
2520
|
const rDeps = recoveryDeps(deps, config, batch, now);
|
|
2321
2521
|
const outcome = (0, recovery_1.dissolveBatch)(state, batchId, { strategy: 'full', reason: 'eviction-threshold' }, rDeps);
|
|
2322
|
-
deps
|
|
2323
|
-
|
|
2324
|
-
result: undefined,
|
|
2325
|
-
}));
|
|
2326
|
-
teardownBatch(deps, batchId);
|
|
2327
|
-
return { dissolved: true, duplicate: false };
|
|
2522
|
+
applyDissolveOutcome(deps, batchId, outcome, now);
|
|
2523
|
+
return { batchEnded: true, duplicate: false };
|
|
2328
2524
|
}
|
|
2329
2525
|
function reconcileFixSlot(deps, config, batchId, slot, now, result) {
|
|
2330
2526
|
if (slot.pid !== null && deps.spawnDeps.isAlive(slot.pid, slot.pid_start ?? undefined))
|
|
@@ -2596,8 +2792,8 @@ const MEMBER_RECONCILE_CHAIN = {
|
|
|
2596
2792
|
* Whether every SURVIVING member of `batch` was actually dispatched — i.e. no
|
|
2597
2793
|
* surviving member is still sitting in a pre-dispatch status (`queued`,
|
|
2598
2794
|
* `classified`, `batched`, `waiting`). Surviving = a `batch.members` issue
|
|
2599
|
-
* that is still a slot-mode entry, not evicted
|
|
2600
|
-
*
|
|
2795
|
+
* that is still a slot-mode entry, not evicted / handed back (parked, #810),
|
|
2796
|
+
* and not already preserved (shipped/terminal through
|
|
2601
2797
|
* another rail).
|
|
2602
2798
|
*
|
|
2603
2799
|
* This is the safety rail behind AC1: a member that never ran has NO work on
|
|
@@ -2615,8 +2811,14 @@ function everySurvivingMemberDispatched(state, batch) {
|
|
|
2615
2811
|
const entry = (0, state_1.findEntry)(state, issue);
|
|
2616
2812
|
if (!entry)
|
|
2617
2813
|
continue;
|
|
2618
|
-
|
|
2814
|
+
// #810: a PARKED member (evicted / handed back) left the batch before
|
|
2815
|
+
// landing — not a survivor, whether or not it has an `evictions[]` record.
|
|
2816
|
+
if (entry.mode !== 'slot' ||
|
|
2817
|
+
evicted.has(issue) ||
|
|
2818
|
+
(0, state_1.isPreservedMember)(entry) ||
|
|
2819
|
+
state_1.PARKED_MEMBER_STATUSES.has(entry.status)) {
|
|
2619
2820
|
continue;
|
|
2821
|
+
}
|
|
2620
2822
|
// Pre-dispatch statuses: queued/classified/batched/waiting. `in-work` is
|
|
2621
2823
|
// stamped at SPAWN (`spawnMember`), `committed`/`validated` at verified
|
|
2622
2824
|
// completion — everything the entry can reach afterwards means the
|
|
@@ -3007,7 +3209,9 @@ function teardownBatch(deps, batchId) {
|
|
|
3007
3209
|
const terminal = (0, state_1.findBatch)(deps.store.load(), batchId);
|
|
3008
3210
|
const lastMemberLanded = terminal?.ranges.some((r) => r.issue === terminal.members[terminal.executing_member - 1]) ===
|
|
3009
3211
|
true;
|
|
3212
|
+
const serialTree = terminal?.member_worktree ?? null;
|
|
3010
3213
|
teardownMemberWorktree(deps, batchId, deps.now(), lastMemberLanded);
|
|
3214
|
+
teardownParallelRuns(deps, batchId, serialTree);
|
|
3011
3215
|
const state = deps.store.load();
|
|
3012
3216
|
const batch = (0, state_1.findBatch)(state, batchId);
|
|
3013
3217
|
if (!batch || batch.worktree === null)
|
|
@@ -3031,6 +3235,618 @@ function teardownBatch(deps, batchId) {
|
|
|
3031
3235
|
worktree: batch.worktree,
|
|
3032
3236
|
});
|
|
3033
3237
|
}
|
|
3238
|
+
// --- Parallel member dispatch (#809) ---
|
|
3239
|
+
/**
|
|
3240
|
+
* How a batch should dispatch its members (#809) — decided once, at the
|
|
3241
|
+
* `ready → executing` claim, and persisted on `BatchEntry.member_dispatch`.
|
|
3242
|
+
*
|
|
3243
|
+
* Serial when the operator set `member_parallelism: 1`, or the batch itself
|
|
3244
|
+
* says its members are not independent: an eviction group (members that must
|
|
3245
|
+
* revert together — e.g. one built on another's API) or a member whose `deps`
|
|
3246
|
+
* names another member (it must see that member's landed work before it is
|
|
3247
|
+
* cut). Parallel otherwise — the batch-cycle skill's and prep dossier's
|
|
3248
|
+
* design (RFC-0001 §J.3: each member works off the integration branch and
|
|
3249
|
+
* sees no one else's changes until the parent lands them).
|
|
3250
|
+
*/
|
|
3251
|
+
/** @internal exported for tests. */
|
|
3252
|
+
function memberDispatchModeFor(state, batch, config) {
|
|
3253
|
+
if (config.member_parallelism === 1)
|
|
3254
|
+
return 'serial';
|
|
3255
|
+
if (batch.eviction_groups.some((group) => group.length > 1))
|
|
3256
|
+
return 'serial';
|
|
3257
|
+
const members = new Set(batch.members);
|
|
3258
|
+
for (const issue of batch.members) {
|
|
3259
|
+
if ((0, state_1.findEntry)(state, issue)?.deps.some((dep) => members.has(dep)))
|
|
3260
|
+
return 'serial';
|
|
3261
|
+
}
|
|
3262
|
+
return 'parallel';
|
|
3263
|
+
}
|
|
3264
|
+
/** Whether `batch` runs the parallel member rail — `null` (pre-#809, or not yet claimed) is serial. */
|
|
3265
|
+
function isParallelBatch(batch) {
|
|
3266
|
+
return batch.member_dispatch === 'parallel';
|
|
3267
|
+
}
|
|
3268
|
+
/** At most this many members of one batch run at once (#809) — `max_slots` when unset. */
|
|
3269
|
+
function memberParallelism(config) {
|
|
3270
|
+
// Both are validated ≥ 1 at config load (`member_parallelism`, `max_slots`).
|
|
3271
|
+
return config.member_parallelism ?? config.max_slots;
|
|
3272
|
+
}
|
|
3273
|
+
/** The teardown target of a member run (or a freshly prepared member tree). */
|
|
3274
|
+
function runTree(r) {
|
|
3275
|
+
return 'pool_claimed' in r
|
|
3276
|
+
? { worktree: r.worktree, branch: r.branch, poolClaimed: r.pool_claimed }
|
|
3277
|
+
: { worktree: r.worktree, branch: r.branch, poolClaimed: r.poolClaimed };
|
|
3278
|
+
}
|
|
3279
|
+
/**
|
|
3280
|
+
* Record that a run's tree is gone. Written AFTER the teardown, so a crash in
|
|
3281
|
+
* between re-runs the (idempotent) teardown from `teardownParallelRuns`
|
|
3282
|
+
* instead of leaking the tree.
|
|
3283
|
+
*/
|
|
3284
|
+
function markTornDown(deps, batchId, issue, now) {
|
|
3285
|
+
deps.store.withLock((s) => ({
|
|
3286
|
+
state: patchRun(s, batchId, issue, { torn_down: true }, now),
|
|
3287
|
+
result: undefined,
|
|
3288
|
+
}));
|
|
3289
|
+
}
|
|
3290
|
+
/** Merge `patch` into member `issue`'s run record (no-op when it has none). */
|
|
3291
|
+
function patchRun(state, batchId, issue, patch, now) {
|
|
3292
|
+
const batch = (0, state_1.findBatch)(state, batchId);
|
|
3293
|
+
if (!batch || !batch.member_runs.some((r) => r.issue === issue))
|
|
3294
|
+
return state;
|
|
3295
|
+
return (0, state_1.patchBatch)(state, batchId, { member_runs: batch.member_runs.map((r) => (r.issue === issue ? { ...r, ...patch } : r)) }, now);
|
|
3296
|
+
}
|
|
3297
|
+
/** Insert or replace member `run.issue`'s run record. */
|
|
3298
|
+
function upsertRun(state, batchId, run, now) {
|
|
3299
|
+
const batch = (0, state_1.findBatch)(state, batchId);
|
|
3300
|
+
if (!batch)
|
|
3301
|
+
return state;
|
|
3302
|
+
const others = batch.member_runs.filter((r) => r.issue !== run.issue);
|
|
3303
|
+
return (0, state_1.patchBatch)(state, batchId, { member_runs: [...others, run].sort((a, b) => a.index - b.index) }, now);
|
|
3304
|
+
}
|
|
3305
|
+
/**
|
|
3306
|
+
* Whether `issue` is still an active member of `batch` — not evicted / handed back
|
|
3307
|
+
* (its `evictions[]` record — a #810 parked entry keeps `mode: 'slot'` and its
|
|
3308
|
+
* batch, so the record is what takes it out), and its queue entry (when it has
|
|
3309
|
+
* one) still a live slot-mode entry of THIS batch — which also covers a member
|
|
3310
|
+
* stopped, abandoned or operator-requeued out from under the batch.
|
|
3311
|
+
*/
|
|
3312
|
+
function memberStillInBatch(state, batch, issue) {
|
|
3313
|
+
if (batch.evictions.some((e) => e.issue === issue))
|
|
3314
|
+
return false;
|
|
3315
|
+
const entry = (0, state_1.findEntry)(state, issue);
|
|
3316
|
+
if (entry === undefined)
|
|
3317
|
+
return true;
|
|
3318
|
+
return (entry.mode === 'slot' &&
|
|
3319
|
+
entry.batch === batch.id &&
|
|
3320
|
+
!types_1.TERMINAL_ISSUE_STATUSES.has(entry.status) &&
|
|
3321
|
+
!state_1.PARKED_MEMBER_STATUSES.has(entry.status));
|
|
3322
|
+
}
|
|
3323
|
+
/** Whether member `issue`'s own slot currently holds a live agent. */
|
|
3324
|
+
function hasLiveMemberSlot(state, batchId, issue) {
|
|
3325
|
+
const slot = (0, state_1.slotForBatchMember)(state, batchId, issue);
|
|
3326
|
+
return slot !== undefined && types_1.LIVE_SLOT_STATUSES.has(slot.status);
|
|
3327
|
+
}
|
|
3328
|
+
/**
|
|
3329
|
+
* The next member to (re)spawn: first a run still `running` that holds no slot
|
|
3330
|
+
* (an api-error hold released it, its spawn threw, or the engine restarted) —
|
|
3331
|
+
* respawned in place, in its own worktree — then the first member in order
|
|
3332
|
+
* that has no run yet.
|
|
3333
|
+
*/
|
|
3334
|
+
function nextParallelCandidate(state, batch, attempted) {
|
|
3335
|
+
for (const run of batch.member_runs) {
|
|
3336
|
+
if (run.status === 'running' &&
|
|
3337
|
+
!attempted.has(run.issue) &&
|
|
3338
|
+
(0, state_1.slotForBatchMember)(state, batch.id, run.issue) === undefined &&
|
|
3339
|
+
memberStillInBatch(state, batch, run.issue)) {
|
|
3340
|
+
return { issue: run.issue, index: run.index, run };
|
|
3341
|
+
}
|
|
3342
|
+
}
|
|
3343
|
+
for (const [i, issue] of batch.members.entries()) {
|
|
3344
|
+
if (attempted.has(issue) || batch.member_runs.some((r) => r.issue === issue))
|
|
3345
|
+
continue;
|
|
3346
|
+
if (!memberStillInBatch(state, batch, issue))
|
|
3347
|
+
continue;
|
|
3348
|
+
return { issue, index: i + 1 };
|
|
3349
|
+
}
|
|
3350
|
+
return undefined;
|
|
3351
|
+
}
|
|
3352
|
+
/**
|
|
3353
|
+
* Fill the batch's member concurrency: spawn members into their OWN slots
|
|
3354
|
+
* (`batch:<id>#<issue>`) while fewer than `member_parallelism` run and the
|
|
3355
|
+
* scheduler has free capacity. Each gets its own worktree off the integration
|
|
3356
|
+
* branch (`prepareMemberWorktree`, outside the lock) — the integration branch
|
|
3357
|
+
* does not move until members land, so concurrent members are all cut from the
|
|
3358
|
+
* same base. A member whose worktree cannot be prepared is evicted (parked, #810 —
|
|
3359
|
+
* an operator's `sched requeue` runs it full-cycle, which does its own setup)
|
|
3360
|
+
* rather than retried every tick in front of the members behind it.
|
|
3361
|
+
*/
|
|
3362
|
+
function spawnParallelMembers(deps, config, dispatch, batchId, now, result) {
|
|
3363
|
+
const limit = memberParallelism(config);
|
|
3364
|
+
const fsExists = deps.fsExists ?? ((p) => fs.existsSync(p));
|
|
3365
|
+
const attempted = new Set();
|
|
3366
|
+
// Bounded by the member count: every iteration marks one member attempted.
|
|
3367
|
+
for (;;) {
|
|
3368
|
+
const state = deps.store.load();
|
|
3369
|
+
const batch = (0, state_1.findBatch)(state, batchId);
|
|
3370
|
+
if (!batch || batch.status !== 'executing' || !isParallelBatch(batch))
|
|
3371
|
+
return;
|
|
3372
|
+
const live = batch.member_runs.filter((r) => hasLiveMemberSlot(state, batchId, r.issue)).length;
|
|
3373
|
+
if (live >= limit || (0, scheduler_1.freeCapacity)(state, config) === 0)
|
|
3374
|
+
return;
|
|
3375
|
+
const next = nextParallelCandidate(state, batch, attempted);
|
|
3376
|
+
if (next === undefined)
|
|
3377
|
+
return;
|
|
3378
|
+
attempted.add(next.issue);
|
|
3379
|
+
let member;
|
|
3380
|
+
const reusable = next.run !== undefined &&
|
|
3381
|
+
fsExists(next.run.worktree) &&
|
|
3382
|
+
(next.run.pool_claimed ||
|
|
3383
|
+
(0, teardown_1.isSafeWorktree)(path.resolve(deps.exec('git', ['rev-parse', '--show-toplevel'], deps.repoDir) ?? deps.repoDir), next.run.worktree));
|
|
3384
|
+
if (next.run !== undefined && reusable) {
|
|
3385
|
+
member = {
|
|
3386
|
+
branch: next.run.branch,
|
|
3387
|
+
worktree: next.run.worktree,
|
|
3388
|
+
poolClaimed: next.run.pool_claimed,
|
|
3389
|
+
};
|
|
3390
|
+
}
|
|
3391
|
+
else {
|
|
3392
|
+
// The serial-member fields describe no member of a parallel batch —
|
|
3393
|
+
// blank them so the reuse branch re-derives THIS member's branch/path.
|
|
3394
|
+
const prep = prepareMemberWorktree(deps, { ...batch, member_branch: null, member_worktree: null, member_pool_claimed: false }, next.index, next.issue, now);
|
|
3395
|
+
if (!prep.ok) {
|
|
3396
|
+
evictMemberDirectly(deps, config, batchId, next.issue, {
|
|
3397
|
+
reason: `member-worktree-prep-failed:${prep.reason}`,
|
|
3398
|
+
detail: `member worktree prep failed: ${prep.reason} (member ${next.index} #${next.issue}, branch ${memberBranchFor(batchId, next.index, next.issue)})`,
|
|
3399
|
+
}, now);
|
|
3400
|
+
result.failed.push((0, state_1.batchMemberUnit)(batchId, next.issue));
|
|
3401
|
+
continue;
|
|
3402
|
+
}
|
|
3403
|
+
member = prep;
|
|
3404
|
+
}
|
|
3405
|
+
const recorded = deps.store.withLock((s) => {
|
|
3406
|
+
const b = (0, state_1.findBatch)(s, batchId);
|
|
3407
|
+
if (!b || b.status !== 'executing' || (0, state_1.slotForBatchMember)(s, batchId, next.issue)) {
|
|
3408
|
+
return { state: s, result: false };
|
|
3409
|
+
}
|
|
3410
|
+
// The run is recorded whether or not a slot is free and the spawn
|
|
3411
|
+
// succeeds: its worktree exists either way, and a `running` run with no
|
|
3412
|
+
// slot is exactly what a later pass respawns in place.
|
|
3413
|
+
let n = upsertRun(s, batchId, {
|
|
3414
|
+
issue: next.issue,
|
|
3415
|
+
index: next.index,
|
|
3416
|
+
branch: member.branch,
|
|
3417
|
+
worktree: member.worktree,
|
|
3418
|
+
pool_claimed: member.poolClaimed,
|
|
3419
|
+
status: 'running',
|
|
3420
|
+
gate_inconclusive: null,
|
|
3421
|
+
torn_down: false,
|
|
3422
|
+
}, now);
|
|
3423
|
+
if ((0, scheduler_1.freeCapacity)(n, config) === 0)
|
|
3424
|
+
return { state: n, result: true };
|
|
3425
|
+
const memberUnit = (0, state_1.batchMemberUnit)(batchId, next.issue);
|
|
3426
|
+
const assigned = (0, scheduler_1.assignToIdleSlot)(n, memberUnit, 'member', now, 'cycle');
|
|
3427
|
+
const slot = assigned.state.slots.find((x) => x.id === assigned.slotId);
|
|
3428
|
+
if (!slot)
|
|
3429
|
+
return { state: n, result: true };
|
|
3430
|
+
n = spawnMemberAgent(deps, dispatch, assigned.state, slot, batchId, now, result, {
|
|
3431
|
+
index: next.index,
|
|
3432
|
+
issue: next.issue,
|
|
3433
|
+
member,
|
|
3434
|
+
unitId: memberUnit,
|
|
3435
|
+
release: (st) => (0, state_1.releaseBatchMemberSlot)(st, batchId, next.issue, now),
|
|
3436
|
+
}).state;
|
|
3437
|
+
n = (0, state_1.patchBatch)(n, batchId, { executing_member: Math.max(b.executing_member, next.index) }, now);
|
|
3438
|
+
return { state: n, result: true };
|
|
3439
|
+
});
|
|
3440
|
+
if (!recorded && next.run === undefined) {
|
|
3441
|
+
// The batch left `executing` (stopped/abandoned/dissolved) while the
|
|
3442
|
+
// tree was being prepared outside the lock — nothing records it, so
|
|
3443
|
+
// give it back now rather than leak a worktree/pool claim.
|
|
3444
|
+
journalEvent(deps, 'unit-failed', unit(batchId), {
|
|
3445
|
+
issue: next.issue,
|
|
3446
|
+
reason: 'member-worktree-orphaned',
|
|
3447
|
+
detail: `batch left executing while member ${next.index}'s tree ${member.worktree} was prepared — tearing it down`,
|
|
3448
|
+
});
|
|
3449
|
+
teardownMemberTree(deps, batchId, runTree(member), false);
|
|
3450
|
+
}
|
|
3451
|
+
}
|
|
3452
|
+
}
|
|
3453
|
+
/**
|
|
3454
|
+
* Evict one parallel member directly (RFC F.1 — its commits never landed) and
|
|
3455
|
+
* tear its worktree down; the pushed member branch is KEPT on origin (the
|
|
3456
|
+
* evicted work's only recoverable copy, as in serial mode). A dissolve this
|
|
3457
|
+
* tips over is torn down by `teardownBatch`, which covers every live run.
|
|
3458
|
+
*/
|
|
3459
|
+
function evictParallelMember(deps, config, batchId, run, failure, now, result) {
|
|
3460
|
+
const { batchEnded, duplicate } = evictMemberDirectly(deps, config, batchId, run.issue, failure, now);
|
|
3461
|
+
if (duplicate)
|
|
3462
|
+
return;
|
|
3463
|
+
result.failed.push(batchEnded ? unit(batchId) : (0, state_1.batchMemberUnit)(batchId, run.issue));
|
|
3464
|
+
if (batchEnded)
|
|
3465
|
+
return;
|
|
3466
|
+
deps.store.withLock((s) => ({
|
|
3467
|
+
state: patchRun(s, batchId, run.issue, { status: 'evicted', gate_inconclusive: null }, now),
|
|
3468
|
+
result: undefined,
|
|
3469
|
+
}));
|
|
3470
|
+
teardownMemberTree(deps, batchId, runTree(run), false);
|
|
3471
|
+
markTornDown(deps, batchId, run.issue, now);
|
|
3472
|
+
}
|
|
3473
|
+
/**
|
|
3474
|
+
* Reconcile one parallel member's slot — the parallel twin of
|
|
3475
|
+
* `reconcileMemberSlot`: the same fenced milestone read, the same incremental
|
|
3476
|
+
* gate (in the member's OWN worktree), the same eviction rail. A verified
|
|
3477
|
+
* member does NOT land here: landing is ordered (`advanceParallelBatch`), so
|
|
3478
|
+
* it only leaves `running` and gives its slot back.
|
|
3479
|
+
*/
|
|
3480
|
+
function reconcileParallelMemberSlot(deps, config, dispatch, batchId, run, slot, now, result) {
|
|
3481
|
+
const state0 = deps.store.load();
|
|
3482
|
+
const read = readMemberDispatch(deps, batchId, run.issue, slot, now);
|
|
3483
|
+
if (read.kind === 'unreachable')
|
|
3484
|
+
return;
|
|
3485
|
+
const memberUnit = (0, state_1.batchMemberUnit)(batchId, run.issue);
|
|
3486
|
+
const release = (s) => (0, state_1.releaseBatchMemberSlot)(s, batchId, run.issue, now);
|
|
3487
|
+
if (read.kind === 'complete') {
|
|
3488
|
+
// Its slot is about to be released and its tree rebased/force-pushed at
|
|
3489
|
+
// landing: an agent still running after `review done` would keep writing
|
|
3490
|
+
// into that tree with its pid no longer tracked anywhere (so neither a
|
|
3491
|
+
// dissolve nor `sched stop --batch` could stop it). Its work is done —
|
|
3492
|
+
// stop it first.
|
|
3493
|
+
if (slot.pid !== null && deps.spawnDeps.isAlive(slot.pid, slot.pid_start ?? undefined)) {
|
|
3494
|
+
deps.spawnDeps.kill(slot.pid, slot.pid_start ?? undefined);
|
|
3495
|
+
}
|
|
3496
|
+
journalEvent(deps, 'external-advance', unit(batchId), {
|
|
3497
|
+
issue: run.issue,
|
|
3498
|
+
detail: 'member review done',
|
|
3499
|
+
});
|
|
3500
|
+
recordMemberRunLog(deps, dispatch, state0, batchId, run.index, run.issue, slot, now);
|
|
3501
|
+
const verdict = evaluateIncrementalGate(deps, batchId, run.worktree, run.issue, now);
|
|
3502
|
+
if (verdict.kind === 'failed') {
|
|
3503
|
+
deps.store.withLock((s) => ({ state: release(s), result: undefined }));
|
|
3504
|
+
evictParallelMember(deps, config, batchId, run, { reason: verdict.reason, detail: verdict.detail }, now, result);
|
|
3505
|
+
return;
|
|
3506
|
+
}
|
|
3507
|
+
deps.store.withLock((s) => ({
|
|
3508
|
+
// #629: a verified member completion is proof dispatch is healthy.
|
|
3509
|
+
state: patchRun((0, dispatch_health_1.resetDispatchApiErrorStreak)(release(s)), batchId, run.issue, {
|
|
3510
|
+
status: 'verified',
|
|
3511
|
+
gate_inconclusive: verdict.kind === 'inconclusive' ? verdict.capability : null,
|
|
3512
|
+
}, now),
|
|
3513
|
+
result: undefined,
|
|
3514
|
+
}));
|
|
3515
|
+
result.completed.push(memberUnit);
|
|
3516
|
+
return;
|
|
3517
|
+
}
|
|
3518
|
+
if (read.dead &&
|
|
3519
|
+
!read.blockedNow &&
|
|
3520
|
+
handleDeadDispatchApiError(deps, batchId, (0, dispatch_1.batchMemberLogPath)(deps.store.runsDir, batchId, run.index, run.issue), slot.log_offset_at_spawn ?? 0, now, { releaseWith: release })) {
|
|
3521
|
+
// #629: not a member failure — the run stays `running` with no slot, and
|
|
3522
|
+
// `spawnParallelMembers` (pause-gated) respawns it in place.
|
|
3523
|
+
return;
|
|
3524
|
+
}
|
|
3525
|
+
if (read.blockedNow || read.dead) {
|
|
3526
|
+
const lastTool = recordMemberRunLog(deps, dispatch, state0, batchId, run.index, run.issue, slot, now);
|
|
3527
|
+
deps.store.withLock((s) => ({ state: release(s), result: undefined }));
|
|
3528
|
+
evictParallelMember(deps, config, batchId, run, memberFailureFor(read, lastTool), now, result);
|
|
3529
|
+
}
|
|
3530
|
+
}
|
|
3531
|
+
/**
|
|
3532
|
+
* Land one verified parallel member onto the integration branch (#809): rebase
|
|
3533
|
+
* its commits onto the integration TIP in its own worktree (earlier members may
|
|
3534
|
+
* have landed since it was cut), refresh the pushed member branch, then
|
|
3535
|
+
* `merge --ff-only` + push in the batch worktree — the same linear,
|
|
3536
|
+
* `(#<issue>)`-trailed history serial landing produces, so `memberRanges`
|
|
3537
|
+
* attribution and per-member commits are unchanged.
|
|
3538
|
+
*
|
|
3539
|
+
* `conflict` — the member's diff does not apply on top of the members landed
|
|
3540
|
+
* before it (the rebase is aborted, leaving its pushed branch untouched); the
|
|
3541
|
+
* caller evicts it, batch-integrate's eviction verdict. `failed` — a mechanical
|
|
3542
|
+
* step failed after a clean rebase; the caller blocks the batch, like serial.
|
|
3543
|
+
*/
|
|
3544
|
+
function landParallelRun(deps, batch, run, now) {
|
|
3545
|
+
const fail = (reason, detail) => {
|
|
3546
|
+
journalEvent(deps, 'landing-failed', unit(batch.id), { issue: run.issue, reason, detail });
|
|
3547
|
+
return { kind: 'failed', reason };
|
|
3548
|
+
};
|
|
3549
|
+
if (batch.worktree === null || batch.branch === null) {
|
|
3550
|
+
return fail('no-integration-branch', 'batch has no integration worktree/branch');
|
|
3551
|
+
}
|
|
3552
|
+
// Defense in depth (CWE-88): persisted refs reach git argv below.
|
|
3553
|
+
if (!attribution_1.SAFE_REF_RE.test(run.branch) || !attribution_1.SAFE_REF_RE.test(batch.branch)) {
|
|
3554
|
+
return fail('invalid-branch-name', `persisted member/integration branch failed SAFE_REF_RE: ${run.branch} → ${batch.branch}`);
|
|
3555
|
+
}
|
|
3556
|
+
// The rebase rewrites whatever is checked out in `run.worktree` — confirm
|
|
3557
|
+
// it is this member's own tree, present, on this member's branch first.
|
|
3558
|
+
if (!(deps.fsExists ?? ((p) => fs.existsSync(p)))(run.worktree)) {
|
|
3559
|
+
return fail('member-worktree-missing', `member worktree ${run.worktree} is gone from disk`);
|
|
3560
|
+
}
|
|
3561
|
+
if (!run.pool_claimed) {
|
|
3562
|
+
const root = deps.exec('git', ['rev-parse', '--show-toplevel'], deps.repoDir) ?? deps.repoDir;
|
|
3563
|
+
if (!(0, teardown_1.isSafeWorktree)(path.resolve(root), run.worktree)) {
|
|
3564
|
+
return fail('unsafe-member-worktree', `member worktree ${run.worktree} is outside the repo's worktree roots`);
|
|
3565
|
+
}
|
|
3566
|
+
}
|
|
3567
|
+
// A rebase an engine crash left half-done would make the next one fail
|
|
3568
|
+
// ("rebase already in progress") and read as a conflict — clear it first
|
|
3569
|
+
// (a no-op failure when none is in progress).
|
|
3570
|
+
deps.exec('git', ['rebase', '--abort'], run.worktree);
|
|
3571
|
+
const head = deps.exec('git', ['symbolic-ref', '--short', 'HEAD'], run.worktree)?.trim();
|
|
3572
|
+
if (head !== run.branch) {
|
|
3573
|
+
return fail('member-worktree-off-branch', `member worktree ${run.worktree} has ${head ?? 'no branch'} checked out, expected ${run.branch} — not rebasing someone else's checkout`);
|
|
3574
|
+
}
|
|
3575
|
+
// The force-push below must never overwrite work pushed to the member
|
|
3576
|
+
// branch that is not part of what was verified here. Read origin's tip NOW
|
|
3577
|
+
// (a bare `--force-with-lease` would trust `origin/<branch>`, which any fetch
|
|
3578
|
+
// in the shared repo refreshes) and require it to be an ancestor of the
|
|
3579
|
+
// verified local branch (the member may not have pushed its last commit);
|
|
3580
|
+
// the lease then pins exactly that tip.
|
|
3581
|
+
const verified = deps
|
|
3582
|
+
.exec('git', ['rev-parse', `refs/heads/${run.branch}`], run.worktree)
|
|
3583
|
+
?.trim();
|
|
3584
|
+
const remoteLine = deps.exec('git', ['ls-remote', 'origin', `refs/heads/${run.branch}`], run.worktree);
|
|
3585
|
+
if (!verified || !groundtruth_1.GIT_OID_RE.test(verified) || remoteLine === null) {
|
|
3586
|
+
return fail('member-branch-unresolved', `could not resolve ${run.branch} locally and on origin before landing`);
|
|
3587
|
+
}
|
|
3588
|
+
const remoteTip = remoteLine.trim().split(/\s+/)[0] ?? '';
|
|
3589
|
+
if (remoteTip !== '' &&
|
|
3590
|
+
(!groundtruth_1.GIT_OID_RE.test(remoteTip) ||
|
|
3591
|
+
deps.exec('git', ['merge-base', '--is-ancestor', remoteTip, verified], run.worktree) === null)) {
|
|
3592
|
+
return fail('member-branch-diverged', `origin/${run.branch} (${remoteTip.slice(0, 12)}) holds commits the verified ${verified.slice(0, 12)} does not — something was pushed after verification; blocking rather than overwriting it`);
|
|
3593
|
+
}
|
|
3594
|
+
// `--autostash`: a stray uncommitted file in the member tree is not the
|
|
3595
|
+
// member's deliverable and must not read as a conflict.
|
|
3596
|
+
if (deps.exec('git', ['rebase', '--autostash', batch.branch], run.worktree) === null) {
|
|
3597
|
+
// Only a rebase stopped on unmerged paths is a CONFLICT (eviction); any
|
|
3598
|
+
// other failure is mechanical and blocks like the other landing steps.
|
|
3599
|
+
const unmerged = (deps.exec('git', ['diff', '--name-only', '--diff-filter=U'], run.worktree) ?? '')
|
|
3600
|
+
.split('\n')
|
|
3601
|
+
.map((l) => l.trim())
|
|
3602
|
+
.filter((l) => l.length > 0);
|
|
3603
|
+
deps.exec('git', ['rebase', '--abort'], run.worktree);
|
|
3604
|
+
if (unmerged.length === 0) {
|
|
3605
|
+
return fail('member-rebase-failed', `git rebase of ${run.branch} onto ${batch.branch} failed with no conflicting paths — blocking for an operator`);
|
|
3606
|
+
}
|
|
3607
|
+
const detail = `${run.branch} does not rebase cleanly onto ${batch.branch} — conflicting with members landed before it in: ${unmerged.slice(0, 10).join(', ')}; evicted with its pushed branch intact`;
|
|
3608
|
+
journalEvent(deps, 'landing-failed', unit(batch.id), {
|
|
3609
|
+
issue: run.issue,
|
|
3610
|
+
reason: 'landing-conflict',
|
|
3611
|
+
detail,
|
|
3612
|
+
});
|
|
3613
|
+
return { kind: 'conflict', detail };
|
|
3614
|
+
}
|
|
3615
|
+
if (deps.exec('git', [
|
|
3616
|
+
'push',
|
|
3617
|
+
`--force-with-lease=refs/heads/${run.branch}:${remoteTip}`,
|
|
3618
|
+
'origin',
|
|
3619
|
+
'--',
|
|
3620
|
+
run.branch,
|
|
3621
|
+
], run.worktree) === null) {
|
|
3622
|
+
return fail('member-push-failed', `rebased ${run.branch} onto ${batch.branch} but could not push it (origin moved since ${remoteTip.slice(0, 12) || 'absent'}, or the push failed) — blocking so nothing pushed after verification is overwritten`);
|
|
3623
|
+
}
|
|
3624
|
+
if (deps.exec('git', ['merge', '--ff-only', run.branch], batch.worktree) === null) {
|
|
3625
|
+
return fail('landing-merge-failed', `git merge --ff-only ${run.branch} into ${batch.branch} failed right after a clean rebase — blocking for an operator`);
|
|
3626
|
+
}
|
|
3627
|
+
if (deps.exec('git', ['push', 'origin', '--', batch.branch], batch.worktree) === null) {
|
|
3628
|
+
return fail('landing-push-failed', `landed ${run.branch} on ${batch.branch} locally but the push failed — blocking so origin stays the durable copy`);
|
|
3629
|
+
}
|
|
3630
|
+
deps.journal.append((0, journal_1.unitEvent)('member-landed', unit(batch.id), {
|
|
3631
|
+
issue: run.issue,
|
|
3632
|
+
detail: `${run.branch} rebased onto and fast-forwarded into ${batch.branch} (member ${run.index}, parallel)`,
|
|
3633
|
+
}), now);
|
|
3634
|
+
return { kind: 'landed' };
|
|
3635
|
+
}
|
|
3636
|
+
/**
|
|
3637
|
+
* Walk members IN ORDER: skip every resolved one (landed, evicted, or no longer
|
|
3638
|
+
* in the batch); the first unresolved one is either landable (`verified`) or
|
|
3639
|
+
* something the walk must wait for (not yet spawned, or still running) — later
|
|
3640
|
+
* members never land ahead of it, so the integration branch always holds a
|
|
3641
|
+
* member-order prefix.
|
|
3642
|
+
*/
|
|
3643
|
+
function nextLandingStep(state, batch) {
|
|
3644
|
+
for (const issue of batch.members) {
|
|
3645
|
+
const run = batch.member_runs.find((r) => r.issue === issue);
|
|
3646
|
+
if (run?.status === 'landed' || run?.status === 'evicted')
|
|
3647
|
+
continue;
|
|
3648
|
+
if (!memberStillInBatch(state, batch, issue))
|
|
3649
|
+
continue;
|
|
3650
|
+
if (run === undefined || run.status === 'running')
|
|
3651
|
+
return { kind: 'wait' };
|
|
3652
|
+
return { kind: 'land', run };
|
|
3653
|
+
}
|
|
3654
|
+
return { kind: 'done' };
|
|
3655
|
+
}
|
|
3656
|
+
/**
|
|
3657
|
+
* Land every landable member in order, then — once every member is resolved —
|
|
3658
|
+
* either block on a gate-inconclusive member (serial parity: it landed, the
|
|
3659
|
+
* batch blocks for `sched resume --batch`) or move to `validating` and run the
|
|
3660
|
+
* aggregate suite once.
|
|
3661
|
+
*/
|
|
3662
|
+
function advanceParallelBatch(deps, config, dispatch, batchId, now, result) {
|
|
3663
|
+
// Bounded: every iteration lands or evicts one member.
|
|
3664
|
+
for (;;) {
|
|
3665
|
+
const state = deps.store.load();
|
|
3666
|
+
const batch = (0, state_1.findBatch)(state, batchId);
|
|
3667
|
+
if (!batch || batch.status !== 'executing' || !isParallelBatch(batch))
|
|
3668
|
+
return;
|
|
3669
|
+
const step = nextLandingStep(state, batch);
|
|
3670
|
+
if (step.kind === 'wait')
|
|
3671
|
+
return;
|
|
3672
|
+
if (step.kind === 'done') {
|
|
3673
|
+
finalizeParallelMembers(deps, config, dispatch, batchId, now, result);
|
|
3674
|
+
return;
|
|
3675
|
+
}
|
|
3676
|
+
const { run } = step;
|
|
3677
|
+
const landed = landParallelRun(deps, batch, run, now);
|
|
3678
|
+
if (landed.kind === 'conflict') {
|
|
3679
|
+
evictParallelMember(deps, config, batchId, run, { reason: 'landing-conflict', detail: landed.detail }, now, result);
|
|
3680
|
+
continue;
|
|
3681
|
+
}
|
|
3682
|
+
if (landed.kind === 'failed') {
|
|
3683
|
+
stopRunningMembers(deps, batchId, now);
|
|
3684
|
+
blockBatchForOperator(deps, config, batchId, { reason: landed.reason }, now, result);
|
|
3685
|
+
return;
|
|
3686
|
+
}
|
|
3687
|
+
// `git log` outside the lock (same invariant as `completeMemberGate`).
|
|
3688
|
+
const ranges = (0, attribution_1.memberRanges)(boundaryCommits(deps, batch));
|
|
3689
|
+
deps.store.withLock((s) => {
|
|
3690
|
+
let n = (0, state_1.patchBatch)(s, batchId, { ranges }, now);
|
|
3691
|
+
n = advanceMemberToValidated(n, run.issue, now);
|
|
3692
|
+
n = patchRun(n, batchId, run.issue, { status: 'landed' }, now);
|
|
3693
|
+
return { state: n, result: undefined };
|
|
3694
|
+
});
|
|
3695
|
+
// A gate-inconclusive member keeps its tree for `sched resume --batch`'s
|
|
3696
|
+
// recheck; everyone else's is done serving.
|
|
3697
|
+
if (run.gate_inconclusive === null) {
|
|
3698
|
+
teardownMemberTree(deps, batchId, runTree(run), true);
|
|
3699
|
+
markTornDown(deps, batchId, run.issue, now);
|
|
3700
|
+
}
|
|
3701
|
+
}
|
|
3702
|
+
}
|
|
3703
|
+
/** Every member is resolved — block on a gate-inconclusive one, or validate. */
|
|
3704
|
+
function finalizeParallelMembers(deps, config, dispatch, batchId, now, result) {
|
|
3705
|
+
const batch = (0, state_1.findBatch)(deps.store.load(), batchId);
|
|
3706
|
+
if (!batch)
|
|
3707
|
+
return;
|
|
3708
|
+
const blockedOn = batch.member_runs.find((r) => r.status === 'landed' && r.gate_inconclusive !== null);
|
|
3709
|
+
if (blockedOn !== undefined) {
|
|
3710
|
+
// Point the serial "current member" fields at it: `sched resume --batch`
|
|
3711
|
+
// (`resumeBlockedGate`) rechecks `member_worktree` and completes/evicts
|
|
3712
|
+
// `members[executing_member - 1]`, then hands back via
|
|
3713
|
+
// `advanceMemberOrValidate` → `settleParallelResume`.
|
|
3714
|
+
deps.store.withLock((s) => ({
|
|
3715
|
+
state: (0, state_1.patchBatch)(s, batchId, {
|
|
3716
|
+
executing_member: blockedOn.index,
|
|
3717
|
+
member_branch: blockedOn.branch,
|
|
3718
|
+
member_worktree: blockedOn.worktree,
|
|
3719
|
+
member_pool_claimed: blockedOn.pool_claimed,
|
|
3720
|
+
}, now),
|
|
3721
|
+
result: undefined,
|
|
3722
|
+
}));
|
|
3723
|
+
blockBatchForOperator(deps, config, batchId, {
|
|
3724
|
+
reason: `gate-inconclusive:${blockedOn.gate_inconclusive}`,
|
|
3725
|
+
milestonePhase: 'batch-review',
|
|
3726
|
+
}, now, result);
|
|
3727
|
+
return;
|
|
3728
|
+
}
|
|
3729
|
+
const claimed = deps.store.withLock((s) => {
|
|
3730
|
+
const b = (0, state_1.findBatch)(s, batchId);
|
|
3731
|
+
if (!b || b.status !== 'executing')
|
|
3732
|
+
return { state: s, result: false };
|
|
3733
|
+
return {
|
|
3734
|
+
state: (0, state_1.transitionBatch)(s, batchId, 'validating', { executing_member: b.members.length }, now),
|
|
3735
|
+
result: true,
|
|
3736
|
+
};
|
|
3737
|
+
});
|
|
3738
|
+
if (claimed)
|
|
3739
|
+
runValidate(deps, config, dispatch, batchId, now, result);
|
|
3740
|
+
}
|
|
3741
|
+
/**
|
|
3742
|
+
* `sched resume --batch` resolved the gate-inconclusive member of a parallel
|
|
3743
|
+
* batch (completed or evicted it through the serial rail): record that on its
|
|
3744
|
+
* run so the landing walk treats it as resolved, then continue the parallel
|
|
3745
|
+
* rail from where it blocked.
|
|
3746
|
+
*/
|
|
3747
|
+
function settleParallelResume(deps, batchId, memberIssue, now) {
|
|
3748
|
+
deps.store.withLock((s) => {
|
|
3749
|
+
const b = (0, state_1.findBatch)(s, batchId);
|
|
3750
|
+
if (!b)
|
|
3751
|
+
return { state: s, result: undefined };
|
|
3752
|
+
const evicted = b.evictions.some((e) => e.issue === memberIssue);
|
|
3753
|
+
return {
|
|
3754
|
+
state: patchRun(s, batchId, memberIssue, {
|
|
3755
|
+
gate_inconclusive: null,
|
|
3756
|
+
// The serial rail's `teardownMemberWorktree` removed its tree.
|
|
3757
|
+
torn_down: true,
|
|
3758
|
+
...(evicted ? { status: 'evicted' } : {}),
|
|
3759
|
+
}, now),
|
|
3760
|
+
result: undefined,
|
|
3761
|
+
};
|
|
3762
|
+
});
|
|
3763
|
+
}
|
|
3764
|
+
/**
|
|
3765
|
+
* One tick of a parallel batch in `executing`: reconcile every member slot,
|
|
3766
|
+
* land what can land in order, then fill free member concurrency (unless the
|
|
3767
|
+
* scheduler is paused — a spawn is a provider dispatch).
|
|
3768
|
+
*/
|
|
3769
|
+
function reconcileParallelBatch(deps, config, dispatch, batchId, now, result, paused) {
|
|
3770
|
+
const batch = (0, state_1.findBatch)(deps.store.load(), batchId);
|
|
3771
|
+
if (!batch)
|
|
3772
|
+
return;
|
|
3773
|
+
for (const { issue } of batch.member_runs) {
|
|
3774
|
+
// Fresh per member: an earlier member's eviction may have dissolved the
|
|
3775
|
+
// batch (killing and releasing every other member) in this same pass.
|
|
3776
|
+
const state = deps.store.load();
|
|
3777
|
+
const fresh = (0, state_1.findBatch)(state, batchId);
|
|
3778
|
+
if (!fresh || fresh.status !== 'executing')
|
|
3779
|
+
return;
|
|
3780
|
+
const run = fresh.member_runs.find((r) => r.issue === issue);
|
|
3781
|
+
if (run?.status !== 'running')
|
|
3782
|
+
continue;
|
|
3783
|
+
const slot = (0, state_1.slotForBatchMember)(state, batchId, issue);
|
|
3784
|
+
if (slot && (slot.status === 'running' || slot.status === 'assigned')) {
|
|
3785
|
+
reconcileParallelMemberSlot(deps, config, dispatch, batchId, run, slot, now, result);
|
|
3786
|
+
}
|
|
3787
|
+
}
|
|
3788
|
+
advanceParallelBatch(deps, config, dispatch, batchId, now, result);
|
|
3789
|
+
if (!paused)
|
|
3790
|
+
spawnParallelMembers(deps, config, dispatch, batchId, now, result);
|
|
3791
|
+
}
|
|
3792
|
+
/**
|
|
3793
|
+
* #809: stop and tear down every parallel member whose tree is not yet torn
|
|
3794
|
+
* down — a dissolve can land while other members are mid-run (their agents are
|
|
3795
|
+
* killed: their entries were just parked or requeued, so a live member agent
|
|
3796
|
+
* would be working a unit nothing tracks any more), `sched stop`/`abandon`
|
|
3797
|
+
* make a batch terminal without a teardown, and a crash between a run's
|
|
3798
|
+
* resolution and its teardown leaves `torn_down: false` behind. Teardown is
|
|
3799
|
+
* idempotent, so re-running it is always safe. `alreadyTornDown` names the
|
|
3800
|
+
* serial-field tree `teardownMemberWorktree` just handled (a gate-inconclusive
|
|
3801
|
+
* member of a blocked parallel batch).
|
|
3802
|
+
*/
|
|
3803
|
+
function teardownParallelRuns(deps, batchId, alreadyTornDown) {
|
|
3804
|
+
const batch = (0, state_1.findBatch)(deps.store.load(), batchId);
|
|
3805
|
+
if (!batch)
|
|
3806
|
+
return;
|
|
3807
|
+
const now = deps.now();
|
|
3808
|
+
for (const run of batch.member_runs) {
|
|
3809
|
+
if (run.torn_down)
|
|
3810
|
+
continue;
|
|
3811
|
+
stopMemberAgent(deps, batchId, run.issue, now);
|
|
3812
|
+
const landed = run.status === 'landed';
|
|
3813
|
+
deps.store.withLock((s) => ({
|
|
3814
|
+
state: patchRun(s, batchId, run.issue, { status: landed ? 'landed' : 'evicted', gate_inconclusive: null }, now),
|
|
3815
|
+
result: undefined,
|
|
3816
|
+
}));
|
|
3817
|
+
if (run.worktree !== alreadyTornDown) {
|
|
3818
|
+
teardownMemberTree(deps, batchId, runTree(run), landed);
|
|
3819
|
+
}
|
|
3820
|
+
markTornDown(deps, batchId, run.issue, now);
|
|
3821
|
+
}
|
|
3822
|
+
}
|
|
3823
|
+
/** Kill a parallel member's live agent (pid-start checked) and release its slot. */
|
|
3824
|
+
function stopMemberAgent(deps, batchId, issue, now) {
|
|
3825
|
+
const slot = (0, state_1.slotForBatchMember)(deps.store.load(), batchId, issue);
|
|
3826
|
+
if (slot === undefined)
|
|
3827
|
+
return;
|
|
3828
|
+
if (slot.pid !== null && deps.spawnDeps.isAlive(slot.pid, slot.pid_start ?? undefined)) {
|
|
3829
|
+
deps.spawnDeps.kill(slot.pid, slot.pid_start ?? undefined);
|
|
3830
|
+
}
|
|
3831
|
+
deps.store.withLock((s) => ({
|
|
3832
|
+
state: (0, state_1.releaseBatchMemberSlot)(s, batchId, issue, now),
|
|
3833
|
+
result: undefined,
|
|
3834
|
+
}));
|
|
3835
|
+
}
|
|
3836
|
+
/**
|
|
3837
|
+
* Stop every running member of a parallel batch that is about to BLOCK (a
|
|
3838
|
+
* landing failure, a reconcile error): nothing reconciles a blocked batch's
|
|
3839
|
+
* member slots, so a still-running agent would hold capacity and keep working
|
|
3840
|
+
* a unit an operator is about to abandon/requeue. Trees are kept for the
|
|
3841
|
+
* operator; `teardownBatch` removes them when the batch ends.
|
|
3842
|
+
*/
|
|
3843
|
+
function stopRunningMembers(deps, batchId, now) {
|
|
3844
|
+
const batch = (0, state_1.findBatch)(deps.store.load(), batchId);
|
|
3845
|
+
for (const run of batch?.member_runs ?? []) {
|
|
3846
|
+
if (run.status === 'running')
|
|
3847
|
+
stopMemberAgent(deps, batchId, run.issue, now);
|
|
3848
|
+
}
|
|
3849
|
+
}
|
|
3034
3850
|
// --- Entry point ---
|
|
3035
3851
|
/**
|
|
3036
3852
|
* One batch reconcile+refill pass. Called from `engine.ts`'s `tick()` after
|
|
@@ -3059,7 +3875,8 @@ function runBatchTick(deps, config, dispatch) {
|
|
|
3059
3875
|
// - pre-merge (`dissolving` is a legal edge): DISSOLVE with reason
|
|
3060
3876
|
// `dispatch-profile-missing:<name>` — loud, deterministic, one-shot, and
|
|
3061
3877
|
// the members requeue as full-cycle units on the config default, which is
|
|
3062
|
-
// a coherent recovery rather than a mislabelled continuation
|
|
3878
|
+
// a coherent recovery rather than a mislabelled continuation (#810: in-flight
|
|
3879
|
+
// members park on the default; validated members block the batch instead).
|
|
3063
3880
|
// - post-merge (PR parked/merged — the product already shipped): dissolving
|
|
3064
3881
|
// would discard a landed PR's bookkeeping, so the remaining dispatch (the
|
|
3065
3882
|
// report agent) runs on the default and journals
|
|
@@ -3093,18 +3910,20 @@ function runBatchTick(deps, config, dispatch) {
|
|
|
3093
3910
|
const fresh = (0, state_1.findBatch)(current, batch.id);
|
|
3094
3911
|
const dissolvable = fresh !== undefined && (0, state_1.allowedBatchTransitions)(fresh.status).includes('dissolving');
|
|
3095
3912
|
if (dissolvable) {
|
|
3096
|
-
|
|
3097
|
-
|
|
3098
|
-
liveSlot.pid !== null &&
|
|
3099
|
-
|
|
3100
|
-
|
|
3913
|
+
// #809: a parallel batch holds one slot per running member too.
|
|
3914
|
+
for (const liveSlot of (0, state_1.slotsForBatch)(current, batch.id)) {
|
|
3915
|
+
if (liveSlot.pid !== null &&
|
|
3916
|
+
deps.spawnDeps.isAlive(liveSlot.pid, liveSlot.pid_start ?? undefined)) {
|
|
3917
|
+
deps.spawnDeps.kill(liveSlot.pid, liveSlot.pid_start ?? undefined);
|
|
3918
|
+
}
|
|
3101
3919
|
}
|
|
3102
|
-
const outcome = (0, recovery_1.dissolveBatch)(current, batch.id, {
|
|
3103
|
-
|
|
3104
|
-
|
|
3105
|
-
|
|
3106
|
-
|
|
3107
|
-
|
|
3920
|
+
const outcome = (0, recovery_1.dissolveBatch)(current, batch.id, {
|
|
3921
|
+
strategy: 'full',
|
|
3922
|
+
reason: `dispatch-profile-missing:${batch.dispatch_profile}`,
|
|
3923
|
+
// #810: the profile is what is broken — requeue on the default.
|
|
3924
|
+
carryDispatchProfile: false,
|
|
3925
|
+
}, recoveryDeps(deps, config, fresh, now));
|
|
3926
|
+
applyDissolveOutcome(deps, batch.id, outcome, now);
|
|
3108
3927
|
result.failed.push(unit(batch.id));
|
|
3109
3928
|
// Resolution failed: clean up any batch/member worktrees, then skip
|
|
3110
3929
|
// this batch for the rest of the pass. No further arm can claim it.
|
|
@@ -3126,6 +3945,13 @@ function runBatchTick(deps, config, dispatch) {
|
|
|
3126
3945
|
};
|
|
3127
3946
|
const containFailure = (batchId, err) => {
|
|
3128
3947
|
const detail = `${err.name}: ${err.message}`;
|
|
3948
|
+
// #809: a blocked batch's member slots are never reconciled — stop the agents too.
|
|
3949
|
+
try {
|
|
3950
|
+
stopRunningMembers(deps, batchId, now);
|
|
3951
|
+
}
|
|
3952
|
+
catch {
|
|
3953
|
+
// best-effort: the block below must still land
|
|
3954
|
+
}
|
|
3129
3955
|
// A corrupt unit must not turn into a project-wide scheduler outage.
|
|
3130
3956
|
// Keep the failed batch inspectable, release its slot, and continue.
|
|
3131
3957
|
deps.store.withLock((state) => {
|
|
@@ -3134,7 +3960,7 @@ function runBatchTick(deps, config, dispatch) {
|
|
|
3134
3960
|
return { state, result: undefined };
|
|
3135
3961
|
}
|
|
3136
3962
|
const blocked = (0, state_1.transitionBatch)(state, batchId, 'blocked', { blocked_reason: 'reconcile-error' }, now);
|
|
3137
|
-
return { state: (0, state_1.
|
|
3963
|
+
return { state: (0, state_1.releaseAllBatchSlots)(blocked, batchId, now), result: undefined };
|
|
3138
3964
|
});
|
|
3139
3965
|
journalEvent(deps, 'unit-failed', unit(batchId), { reason: 'reconcile-error', detail });
|
|
3140
3966
|
result.failed.push(unit(batchId));
|
|
@@ -3186,6 +4012,8 @@ function runBatchTick(deps, config, dispatch) {
|
|
|
3186
4012
|
}
|
|
3187
4013
|
for (const batch of deps.store.load().batches) {
|
|
3188
4014
|
const slot = slotFor(deps.store.load(), batch.id);
|
|
4015
|
+
// #809: a terminal batch may also leak parallel member slots.
|
|
4016
|
+
const heldSlots = (0, state_1.slotsForBatch)(deps.store.load(), batch.id);
|
|
3189
4017
|
// #609: a TERMINAL batch (`done`/`dissolved`) still holding a slot matches
|
|
3190
4018
|
// none of the status arms below and would fall through their `continue`,
|
|
3191
4019
|
// so nothing ever looks at that slot again — not even the orphaned-pid
|
|
@@ -3197,15 +4025,29 @@ function runBatchTick(deps, config, dispatch) {
|
|
|
3197
4025
|
// files that already carry a leaked slot, which is the only way an
|
|
3198
4026
|
// operator gets those three hours of held capacity back.
|
|
3199
4027
|
if (types_1.TERMINAL_BATCH_STATUSES.has(batch.status)) {
|
|
3200
|
-
|
|
4028
|
+
// #809: `sched stop`/`abandon` end a batch without a teardown — give
|
|
4029
|
+
// back every member tree (and stop any agent still running in one).
|
|
4030
|
+
if (batch.member_runs.some((r) => !r.torn_down)) {
|
|
4031
|
+
try {
|
|
4032
|
+
teardownParallelRuns(deps, batch.id, null);
|
|
4033
|
+
}
|
|
4034
|
+
catch (err) {
|
|
4035
|
+
journalEvent(deps, 'teardown-failed', unit(batch.id), {
|
|
4036
|
+
detail: `member teardown: ${err.message}`,
|
|
4037
|
+
});
|
|
4038
|
+
}
|
|
4039
|
+
}
|
|
4040
|
+
if (heldSlots.length > 0) {
|
|
3201
4041
|
deps.store.withLock((s) => ({
|
|
3202
|
-
state: (0, state_1.
|
|
4042
|
+
state: (0, state_1.releaseAllBatchSlots)(s, batch.id, now),
|
|
3203
4043
|
result: undefined,
|
|
3204
4044
|
}));
|
|
3205
|
-
|
|
3206
|
-
slot
|
|
3207
|
-
|
|
3208
|
-
|
|
4045
|
+
for (const held of heldSlots) {
|
|
4046
|
+
journalEvent(deps, 'slot-released', unit(batch.id), {
|
|
4047
|
+
slot: held.id,
|
|
4048
|
+
detail: `terminal batch (${batch.status}) held slot ${held.id}${held.unit === unit(batch.id) ? '' : ` (${held.unit})`}`,
|
|
4049
|
+
});
|
|
4050
|
+
}
|
|
3209
4051
|
}
|
|
3210
4052
|
continue;
|
|
3211
4053
|
}
|
|
@@ -3217,6 +4059,12 @@ function runBatchTick(deps, config, dispatch) {
|
|
|
3217
4059
|
const batchDispatch = dispatchFor(batch);
|
|
3218
4060
|
if (batchDispatch === null)
|
|
3219
4061
|
continue;
|
|
4062
|
+
// #809: a parallel batch's members hold their OWN slots — its whole
|
|
4063
|
+
// `executing` phase is one arm.
|
|
4064
|
+
if (batch.status === 'executing' && isParallelBatch(batch)) {
|
|
4065
|
+
reconcileParallelBatch(deps, config, batchDispatch, batch.id, now, result, paused);
|
|
4066
|
+
continue;
|
|
4067
|
+
}
|
|
3220
4068
|
if (slot && (slot.status === 'running' || slot.status === 'assigned')) {
|
|
3221
4069
|
if (batch.status === 'executing') {
|
|
3222
4070
|
reconcileMemberSlot(deps, config, batchDispatch, batch.id, slot, now, result);
|