@ai-dossier/sched 0.55.0 → 0.56.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +32 -4
- package/dist/anchor-close.d.ts +26 -16
- package/dist/anchor-close.d.ts.map +1 -1
- package/dist/anchor-close.js +112 -75
- package/dist/anchor-close.js.map +1 -1
- package/dist/batch-dispatch.d.ts +11 -3
- package/dist/batch-dispatch.d.ts.map +1 -1
- package/dist/batch-dispatch.js +234 -61
- package/dist/batch-dispatch.js.map +1 -1
- package/dist/dispatch.d.ts +20 -2
- package/dist/dispatch.d.ts.map +1 -1
- package/dist/dispatch.js +45 -4
- package/dist/dispatch.js.map +1 -1
- package/dist/index.d.ts +2 -2
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +8 -7
- package/dist/index.js.map +1 -1
- package/dist/state.d.ts +2 -0
- package/dist/state.d.ts.map +1 -1
- package/dist/state.js +10 -2
- package/dist/state.js.map +1 -1
- package/dist/status.d.ts +1 -1
- package/dist/status.d.ts.map +1 -1
- package/dist/types.d.ts +21 -2
- package/dist/types.d.ts.map +1 -1
- package/dist/types.js +5 -1
- package/dist/types.js.map +1 -1
- package/package.json +1 -1
package/dist/batch-dispatch.js
CHANGED
|
@@ -164,9 +164,18 @@ function sanitizeUntrustedText(value) {
|
|
|
164
164
|
function journalEvent(deps, event, unitId, extra = {}) {
|
|
165
165
|
deps.journal.append((0, journal_1.unitEvent)(event, unitId, extra), deps.now());
|
|
166
166
|
}
|
|
167
|
-
/**
|
|
167
|
+
/** Whether the slot's agent is still alive (pid-start-safe: a reused pid reads dead). */
|
|
168
|
+
function agentAlive(deps, slot) {
|
|
169
|
+
return slot.pid !== null && deps.spawnDeps.isAlive(slot.pid, slot.pid_start ?? undefined);
|
|
170
|
+
}
|
|
171
|
+
/**
|
|
172
|
+
* Stop the slot's agent if it is still alive (pid-start-safe: a reused pid is
|
|
173
|
+
* never signalled). SIGTERM only — no wait, no escalation. A decision that
|
|
174
|
+
* must not run alongside the agent uses `stopAgentBeforeDeciding` instead,
|
|
175
|
+
* which escalates to SIGKILL (#844).
|
|
176
|
+
*/
|
|
168
177
|
function killLiveAgent(deps, slot) {
|
|
169
|
-
if (
|
|
178
|
+
if (agentAlive(deps, slot)) {
|
|
170
179
|
deps.spawnDeps.kill(slot.pid, slot.pid_start ?? undefined);
|
|
171
180
|
}
|
|
172
181
|
}
|
|
@@ -1285,6 +1294,9 @@ function spawnMemberContinuation(deps, config, dispatch, batchId, now, result) {
|
|
|
1285
1294
|
// contract `prepareMemberWorktree` documents). The wedge arm of
|
|
1286
1295
|
// `runBatchTick` calls this EVERY tick while the batch is `executing`
|
|
1287
1296
|
// with no slot, so the cheap gates run first:
|
|
1297
|
+
// - recovery (#844): a current member already in `evictions[]` is advanced
|
|
1298
|
+
// past, never respawned — checked before capacity, since the advance
|
|
1299
|
+
// needs no slot.
|
|
1288
1300
|
// - capacity: prep before the free-capacity check would do pool/git/npm
|
|
1289
1301
|
// work (and journal) per tick against a full scheduler — the #610/#630/
|
|
1290
1302
|
// #632 per-tick emission trap. `claimAndSpawn` re-checks under the lock;
|
|
@@ -1301,6 +1313,30 @@ function spawnMemberContinuation(deps, config, dispatch, batchId, now, result) {
|
|
|
1301
1313
|
const memberIssue = batch.members[batch.executing_member - 1];
|
|
1302
1314
|
if (memberIssue === undefined)
|
|
1303
1315
|
return;
|
|
1316
|
+
const evicted = batch.evictions.find((e) => e.issue === memberIssue);
|
|
1317
|
+
if (evicted !== undefined) {
|
|
1318
|
+
// #844: the member's eviction (and its slot release, in the same write)
|
|
1319
|
+
// landed, but the engine died before advancing past it — possibly before
|
|
1320
|
+
// journaling its cause, too, so the record's cause goes on this line.
|
|
1321
|
+
// Respawning it would dispatch an evicted member again — finish what the
|
|
1322
|
+
// eviction would have done instead: the dissolve its threshold tripped,
|
|
1323
|
+
// or the advance.
|
|
1324
|
+
const kind = evicted.kind ?? 'evicted';
|
|
1325
|
+
journalEvent(deps, 'member-advance-recovered', unit(batchId), {
|
|
1326
|
+
issue: memberIssue,
|
|
1327
|
+
reason: evicted.reason,
|
|
1328
|
+
kind,
|
|
1329
|
+
evicted_at: evicted.at,
|
|
1330
|
+
detail: `member ${batch.executing_member} is already parked ${kind} (${evicted.reason}) but the batch never advanced past it (engine exit mid-eviction) — continuing instead of respawning it`,
|
|
1331
|
+
});
|
|
1332
|
+
if ((0, recovery_1.checkDissolveTrigger)(batch, (0, types_1.resolveDissolvePolicy)(config.dissolve_policy)) &&
|
|
1333
|
+
settleEvictionThreshold(deps, config, batchId, memberIssue, (0, state_1.validatedMembersOf)(state, batch), now)) {
|
|
1334
|
+
result.failed.push(unit(batchId));
|
|
1335
|
+
return;
|
|
1336
|
+
}
|
|
1337
|
+
continuePastEvictedMember(deps, config, dispatch, batchId, batch, memberIssue, now, result);
|
|
1338
|
+
return;
|
|
1339
|
+
}
|
|
1304
1340
|
if ((0, scheduler_1.freeCapacity)(state, config) === 0)
|
|
1305
1341
|
return;
|
|
1306
1342
|
const expectedBranch = memberBranchFor(batchId, batch.executing_member, memberIssue);
|
|
@@ -1772,8 +1808,14 @@ serialTeardown) {
|
|
|
1772
1808
|
}
|
|
1773
1809
|
/**
|
|
1774
1810
|
* Evict the current member and either dissolve, or continue the batch via
|
|
1775
|
-
* `advanceMemberOrValidate` — the shared tail of
|
|
1776
|
-
* (
|
|
1811
|
+
* `advanceMemberOrValidate` — the shared tail of the serial member-failure
|
|
1812
|
+
* rails (dead/blocked, the incremental gate below, the wrong-procedure
|
|
1813
|
+
* evictions, and `sched resume --batch`'s recheck).
|
|
1814
|
+
*
|
|
1815
|
+
* #844: a caller whose member still holds its slot MUST pass its release as
|
|
1816
|
+
* `preWrite` — never release under a separate lock first. `evictMemberDirectly` commits
|
|
1817
|
+
* the release with the park/eviction record in one write, so an engine exit
|
|
1818
|
+
* cannot leave the member slotless and un-evicted (and so respawned).
|
|
1777
1819
|
*
|
|
1778
1820
|
* #613: `evictMemberDirectly`'s duplicate check (#595) is the one atomic,
|
|
1779
1821
|
* lock-protected claim on "did THIS member's eviction already happen" — so
|
|
@@ -1795,14 +1837,27 @@ serialTeardown) {
|
|
|
1795
1837
|
* `runBatchTick`/`resumeBlockedGate` entry points cannot reproduce since
|
|
1796
1838
|
* both always read state fresh.
|
|
1797
1839
|
*/
|
|
1798
|
-
function evictMemberAndContinue(deps, config, dispatch, batchId, batch, memberIssue, failure, now, result
|
|
1799
|
-
|
|
1840
|
+
function evictMemberAndContinue(deps, config, dispatch, batchId, batch, memberIssue, failure, now, result,
|
|
1841
|
+
/** #844: the member's slot release (or any pre-write) — committed in the eviction's own write. */
|
|
1842
|
+
preWrite) {
|
|
1843
|
+
const { batchEnded, duplicate } = evictMemberDirectly(deps, config, batchId, memberIssue, failure, now, preWrite);
|
|
1800
1844
|
if (duplicate)
|
|
1801
1845
|
return;
|
|
1802
1846
|
if (batchEnded) {
|
|
1803
1847
|
result.failed.push(unit(batchId));
|
|
1804
1848
|
return;
|
|
1805
1849
|
}
|
|
1850
|
+
continuePastEvictedMember(deps, config, dispatch, batchId, batch, memberIssue, now, result);
|
|
1851
|
+
}
|
|
1852
|
+
/**
|
|
1853
|
+
* The serial rail after a member's eviction has been recorded: tear its tree
|
|
1854
|
+
* down and advance. Also the crash recovery for an engine that died between
|
|
1855
|
+
* the eviction's write and this advance (#844) — `spawnMemberContinuation`
|
|
1856
|
+
* finds the batch still pointing at a member already in `evictions[]` and
|
|
1857
|
+
* continues from here instead of respawning it. `advanceMemberOrValidate`'s
|
|
1858
|
+
* one-shot claim (#613) keeps a re-entry from advancing twice.
|
|
1859
|
+
*/
|
|
1860
|
+
function continuePastEvictedMember(deps, config, dispatch, batchId, batch, memberIssue, now, result) {
|
|
1806
1861
|
// #677: the evicted member's commits never landed (eviction is the
|
|
1807
1862
|
// pre-landing rail), so there is nothing to revert on the integration
|
|
1808
1863
|
// branch — but its worktree/branch must go, or the next tick's prep for
|
|
@@ -1842,7 +1897,9 @@ function evictMemberAndContinue(deps, config, dispatch, batchId, batch, memberIs
|
|
|
1842
1897
|
* configured or both checks came back `ok`, meaning the caller should treat
|
|
1843
1898
|
* the member as genuinely complete.
|
|
1844
1899
|
*/
|
|
1845
|
-
function runIncrementalGate(deps, config, dispatch, batchId, batch, memberIssue, now, result
|
|
1900
|
+
function runIncrementalGate(deps, config, dispatch, batchId, batch, memberIssue, now, result,
|
|
1901
|
+
/** The member's slot release — handed to the eviction on `failed` (#844). */
|
|
1902
|
+
release) {
|
|
1846
1903
|
if (batch.worktree === null || !deps.runCapability)
|
|
1847
1904
|
return false;
|
|
1848
1905
|
// #677: the gate judges the member's OWN work — pre-landing, its diff
|
|
@@ -1854,11 +1911,10 @@ function runIncrementalGate(deps, config, dispatch, batchId, batch, memberIssue,
|
|
|
1854
1911
|
if (verdict.kind === 'pass')
|
|
1855
1912
|
return false;
|
|
1856
1913
|
if (verdict.kind === 'failed') {
|
|
1857
|
-
deps
|
|
1858
|
-
evictMemberAndContinue(deps, config, dispatch, batchId, batch, memberIssue, { reason: verdict.reason, detail: verdict.detail }, now, result);
|
|
1914
|
+
evictMemberAndContinue(deps, config, dispatch, batchId, batch, memberIssue, { reason: verdict.reason, detail: verdict.detail }, now, result, release);
|
|
1859
1915
|
return true;
|
|
1860
1916
|
}
|
|
1861
|
-
deps.store.withLock((s) => ({ state:
|
|
1917
|
+
deps.store.withLock((s) => ({ state: release(s), result: undefined }));
|
|
1862
1918
|
// #677: F.11's block semantics are "the member's commit stays on the
|
|
1863
1919
|
// branch, nothing requeued/reverted" — under member worktrees that is
|
|
1864
1920
|
// true only if the member LANDS before the batch blocks. The block means
|
|
@@ -2176,19 +2232,21 @@ function resumeBlockedGate(deps, config, dispatch, batchId, now) {
|
|
|
2176
2232
|
detail: excerpt,
|
|
2177
2233
|
};
|
|
2178
2234
|
}
|
|
2179
|
-
|
|
2180
|
-
state: (0, state_1.patchBatch)((0, state_1.transitionBatch)(s, batchId, 'executing', {}, now), batchId, { blocked_reason: null }, now),
|
|
2181
|
-
result: undefined,
|
|
2182
|
-
}));
|
|
2235
|
+
const unblock = (s) => (0, state_1.patchBatch)((0, state_1.transitionBatch)(s, batchId, 'executing', {}, now), batchId, { blocked_reason: null }, now);
|
|
2183
2236
|
if (recheck.outcome === 'task-failed') {
|
|
2184
2237
|
const reason = `incremental-gate-failed:${capabilityId}`;
|
|
2185
2238
|
writeGateLog(deps, batchId, capabilityId, memberIssue, recheck.outputTail);
|
|
2186
2239
|
evictMemberAndContinue(deps, config, batchDispatch, batchId, batch, memberIssue, {
|
|
2187
2240
|
reason,
|
|
2188
2241
|
detail: withExcerpt(`sched resume --batch: cap run ${capabilityId} reported task-failed on recheck`, excerpt),
|
|
2189
|
-
}, now, result
|
|
2242
|
+
}, now, result,
|
|
2243
|
+
// #844: the unblock commits with the eviction — two writes left a
|
|
2244
|
+
// slotless `executing` batch whose un-evicted member the wedge arm
|
|
2245
|
+
// then respawned.
|
|
2246
|
+
unblock);
|
|
2190
2247
|
return { outcome: 'evicted', capability: capabilityId, detail: excerpt };
|
|
2191
2248
|
}
|
|
2249
|
+
deps.store.withLock((s) => ({ state: unblock(s), result: undefined }));
|
|
2192
2250
|
journalEvent(deps, 'external-advance', unit(batchId), {
|
|
2193
2251
|
issue: memberIssue,
|
|
2194
2252
|
detail: `sched resume --batch: cap run ${capabilityId} reported ok on recheck`,
|
|
@@ -2389,6 +2447,84 @@ function wrongProcedureCounts(batch, memberIssue, read) {
|
|
|
2389
2447
|
const repromptedAt = Date.parse(prior.reprompted_at);
|
|
2390
2448
|
return Number.isNaN(at) || Number.isNaN(repromptedAt) ? false : at > repromptedAt;
|
|
2391
2449
|
}
|
|
2450
|
+
/**
|
|
2451
|
+
* A kill timestamp that belongs to the slot's CURRENT dispatch, or null
|
|
2452
|
+
* (#844): one stamped before `spawned_at` was for an earlier dispatch in the
|
|
2453
|
+
* same slot, and an unparseable or future one is no anchor at all.
|
|
2454
|
+
*/
|
|
2455
|
+
function currentDispatchStamp(stamp, spawnedAt, now) {
|
|
2456
|
+
if (stamp === null)
|
|
2457
|
+
return null;
|
|
2458
|
+
const at = Date.parse(stamp);
|
|
2459
|
+
if (Number.isNaN(at))
|
|
2460
|
+
return null;
|
|
2461
|
+
if (spawnedAt !== null && at < Date.parse(spawnedAt))
|
|
2462
|
+
return null;
|
|
2463
|
+
// A future stamp (clock step, hand edit) would make the elapsed time
|
|
2464
|
+
// negative and switch the escalation off for good — re-stamp instead.
|
|
2465
|
+
if (at > now.getTime())
|
|
2466
|
+
return null;
|
|
2467
|
+
return stamp;
|
|
2468
|
+
}
|
|
2469
|
+
/**
|
|
2470
|
+
* Stop the slot's agent before a decision that must not run alongside it
|
|
2471
|
+
* (#844). Returns `true` while it is still alive — the caller waits and
|
|
2472
|
+
* decides on a later tick — and `false` once it is gone (or never had a pid).
|
|
2473
|
+
*
|
|
2474
|
+
* The first sight sends SIGTERM, stamps `kill_sent_at` and journals
|
|
2475
|
+
* `member-stop-requested` once; each later tick
|
|
2476
|
+
* re-sends SIGTERM until the agent has outlived it by `KILL_ESCALATION_MS`,
|
|
2477
|
+
* then sends SIGKILL (to its process group — `SpawnDeps.kill`) and journals
|
|
2478
|
+
* `kill-escalated` once (`kill_escalated_at`). Pre-#844 this path re-sent
|
|
2479
|
+
* SIGTERM forever, so an agent ignoring it held its member indefinitely.
|
|
2480
|
+
*/
|
|
2481
|
+
function stopAgentBeforeDeciding(deps, batchId, memberIssue, slot, now) {
|
|
2482
|
+
if (!agentAlive(deps, slot))
|
|
2483
|
+
return false;
|
|
2484
|
+
const pid = slot.pid;
|
|
2485
|
+
const pidStart = slot.pid_start ?? undefined;
|
|
2486
|
+
const sentAt = currentDispatchStamp(slot.kill_sent_at, slot.spawned_at, now);
|
|
2487
|
+
const aliveForMs = sentAt === null ? 0 : now.getTime() - Date.parse(sentAt);
|
|
2488
|
+
const escalate = sentAt !== null && aliveForMs >= dispatch_1.KILL_ESCALATION_MS;
|
|
2489
|
+
// `kill` refuses a pid whose recorded identity no longer matches (reused by
|
|
2490
|
+
// another process): the agent we spawned is gone, so stop waiting on it.
|
|
2491
|
+
if (!deps.spawnDeps.kill(pid, pidStart, escalate ? 'SIGKILL' : 'SIGTERM'))
|
|
2492
|
+
return false;
|
|
2493
|
+
const firstEscalation = escalate && currentDispatchStamp(slot.kill_escalated_at, slot.spawned_at, now) === null;
|
|
2494
|
+
if (sentAt === null || firstEscalation) {
|
|
2495
|
+
deps.store.withLock((s) => {
|
|
2496
|
+
const live = s.slots.find((x) => x.id === slot.id);
|
|
2497
|
+
// Only the slot still holding THIS agent — never a later dispatch's.
|
|
2498
|
+
if (!live || live.unit !== slot.unit || live.pid !== pid) {
|
|
2499
|
+
return { state: s, result: undefined };
|
|
2500
|
+
}
|
|
2501
|
+
const stamp = now.toISOString();
|
|
2502
|
+
return {
|
|
2503
|
+
state: (0, state_1.patchSlot)(s, slot.id, sentAt === null
|
|
2504
|
+
? { kill_sent_at: stamp, kill_escalated_at: null }
|
|
2505
|
+
: { kill_escalated_at: stamp }, now),
|
|
2506
|
+
result: undefined,
|
|
2507
|
+
};
|
|
2508
|
+
});
|
|
2509
|
+
}
|
|
2510
|
+
if (sentAt === null) {
|
|
2511
|
+
journalEvent(deps, 'member-stop-requested', unit(batchId), {
|
|
2512
|
+
issue: memberIssue,
|
|
2513
|
+
pid,
|
|
2514
|
+
slot: slot.id,
|
|
2515
|
+
detail: `SIGTERM sent — waiting for the agent to exit before deciding (SIGKILL after ${dispatch_1.KILL_ESCALATION_MS / 1000}s)`,
|
|
2516
|
+
});
|
|
2517
|
+
}
|
|
2518
|
+
if (firstEscalation) {
|
|
2519
|
+
journalEvent(deps, 'kill-escalated', unit(batchId), {
|
|
2520
|
+
issue: memberIssue,
|
|
2521
|
+
pid,
|
|
2522
|
+
slot: slot.id,
|
|
2523
|
+
detail: `agent still alive ${Math.round(aliveForMs / 1000)}s after SIGTERM (bound ${dispatch_1.KILL_ESCALATION_MS / 1000}s) — sent SIGKILL (to its process group where available)`,
|
|
2524
|
+
});
|
|
2525
|
+
}
|
|
2526
|
+
return true;
|
|
2527
|
+
}
|
|
2392
2528
|
/**
|
|
2393
2529
|
* #822 (#810 proposal 3): the member's dispatch ran the wrong procedure (a
|
|
2394
2530
|
* full-cycle-shaped trail — imboard #4174 ran full-cycle as a batch member).
|
|
@@ -2398,23 +2534,27 @@ function wrongProcedureCounts(batch, memberIssue, read) {
|
|
|
2398
2534
|
* `reprompted_members` — the wedge arm (serial) or `spawnParallelMembers`
|
|
2399
2535
|
* (parallel) respawns it in place, in the same worktree, with
|
|
2400
2536
|
* `wrongProcedureDirective` appended. Returns `'evict'` when the member was
|
|
2401
|
-
* already re-prompted once: the caller evicts it `wrong-procedure`.
|
|
2537
|
+
* already re-prompted once: the caller evicts it `wrong-procedure`. On
|
|
2538
|
+
* `'evict'` the slot is NOT released here — the caller passes `release` to
|
|
2539
|
+
* `evictMemberAndContinue`/`evictParallelMember`, which commits it in the
|
|
2540
|
+
* eviction's own write (#844).
|
|
2402
2541
|
*/
|
|
2403
2542
|
function repromptWrongProcedure(deps, batchId, memberIssue, milestoneAt, slot, release, now) {
|
|
2404
2543
|
// Never respawn into (or release) a worktree the old agent is still in: a
|
|
2405
2544
|
// dying full-cycle run could still push, post, or open a PR. Signal it and
|
|
2406
2545
|
// decide on a later tick, once it is gone — and so the re-prompt's
|
|
2407
2546
|
// `reprompted_at` fence postdates everything it could still post.
|
|
2408
|
-
if (
|
|
2409
|
-
deps.spawnDeps.kill(slot.pid, slot.pid_start ?? undefined);
|
|
2547
|
+
if (stopAgentBeforeDeciding(deps, batchId, memberIssue, slot, now))
|
|
2410
2548
|
return 'waiting';
|
|
2411
|
-
}
|
|
2412
2549
|
const outcome = deps.store.withLock((s) => {
|
|
2413
|
-
const
|
|
2414
|
-
const b = (0, state_1.findBatch)(released, batchId);
|
|
2550
|
+
const b = (0, state_1.findBatch)(s, batchId);
|
|
2415
2551
|
if (!b || b.reprompted_members.some((r) => r.issue === memberIssue)) {
|
|
2416
|
-
|
|
2552
|
+
// #844: NOT released here — the caller's eviction commits the release
|
|
2553
|
+
// in its own write, so a crash in between cannot leave the member
|
|
2554
|
+
// slotless and un-evicted (and so respawned).
|
|
2555
|
+
return { state: s, result: 'evict' };
|
|
2417
2556
|
}
|
|
2557
|
+
const released = release(s);
|
|
2418
2558
|
return {
|
|
2419
2559
|
state: (0, state_1.patchBatch)(released, batchId, {
|
|
2420
2560
|
reprompted_members: [
|
|
@@ -2455,18 +2595,16 @@ function wrongProcedureFailure(lastTool, shippedPr = null) {
|
|
|
2455
2595
|
* #822: the one wrong-procedure decision shared by both member rails —
|
|
2456
2596
|
* `'waiting'` (the old agent is still dying), `'reprompted'`, or the
|
|
2457
2597
|
* eviction failure to apply. A run that already shipped is evicted without
|
|
2458
|
-
* a re-prompt.
|
|
2598
|
+
* a re-prompt. On an eviction the slot is still held: the caller passes
|
|
2599
|
+
* `release` to the eviction, which commits both in one write (#844).
|
|
2459
2600
|
*/
|
|
2460
2601
|
function decideWrongProcedure(deps, batchId, memberIssue, milestone, slot, release, lastTool, now) {
|
|
2461
2602
|
const shipped = (0, groundtruth_1.wrongProcedureShippedPr)(milestone);
|
|
2462
2603
|
if (shipped !== null) {
|
|
2463
|
-
if (
|
|
2464
|
-
deps.spawnDeps.kill(slot.pid, slot.pid_start ?? undefined);
|
|
2604
|
+
if (stopAgentBeforeDeciding(deps, batchId, memberIssue, slot, now))
|
|
2465
2605
|
return 'waiting';
|
|
2466
|
-
|
|
2467
|
-
|
|
2468
|
-
deps.store.withLock((s) => ({ state: release(s), result: undefined }));
|
|
2469
|
-
return wrongProcedureFailure(tool, shipped);
|
|
2606
|
+
// #844: the caller's eviction releases the slot in its own write.
|
|
2607
|
+
return wrongProcedureFailure(lastTool(), shipped);
|
|
2470
2608
|
}
|
|
2471
2609
|
const verdict = repromptWrongProcedure(deps, batchId, memberIssue, milestone.at, slot, release, now);
|
|
2472
2610
|
if (verdict === 'waiting')
|
|
@@ -2521,6 +2659,9 @@ function reconcileMemberSlot(deps, config, dispatch, batchId, slot, now, result)
|
|
|
2521
2659
|
const memberIssue = batch.members[batch.executing_member - 1];
|
|
2522
2660
|
if (memberIssue === undefined)
|
|
2523
2661
|
return;
|
|
2662
|
+
// #844: every rail below that evicts hands THIS to the eviction, which
|
|
2663
|
+
// commits it in the same write — never released under a lock of its own.
|
|
2664
|
+
const release = (s) => releaseSlot(s, batchId, now);
|
|
2524
2665
|
const read = readMemberDispatch(deps, batchId, memberIssue, slot, now);
|
|
2525
2666
|
if (read.kind === 'unreachable')
|
|
2526
2667
|
return;
|
|
@@ -2538,7 +2679,7 @@ function reconcileMemberSlot(deps, config, dispatch, batchId, slot, now, result)
|
|
|
2538
2679
|
// gate already decided the member's fate (evicted or blocked) and
|
|
2539
2680
|
// returned early; `false` means both checks were `ok` and the member is
|
|
2540
2681
|
// genuinely done.
|
|
2541
|
-
if (runIncrementalGate(deps, config, dispatch, batchId, batch, memberIssue, now, result)) {
|
|
2682
|
+
if (runIncrementalGate(deps, config, dispatch, batchId, batch, memberIssue, now, result, release)) {
|
|
2542
2683
|
return;
|
|
2543
2684
|
}
|
|
2544
2685
|
completeMemberGate(deps, config, dispatch, batchId, batch, memberIssue, now, result);
|
|
@@ -2548,9 +2689,9 @@ function reconcileMemberSlot(deps, config, dispatch, batchId, slot, now, result)
|
|
|
2548
2689
|
// before the dead/blocked rail, which would otherwise evict it as an
|
|
2549
2690
|
// unverified exit at the first sight.
|
|
2550
2691
|
if (wrongProcedureCounts(batch, memberIssue, read) && read.milestone !== null) {
|
|
2551
|
-
const decision = decideWrongProcedure(deps, batchId, memberIssue, read.milestone, slot,
|
|
2692
|
+
const decision = decideWrongProcedure(deps, batchId, memberIssue, read.milestone, slot, release, () => recordMemberRunLog(deps, dispatch, state0, batchId, batch.executing_member, memberIssue, slot, now), now);
|
|
2552
2693
|
if (typeof decision === 'object') {
|
|
2553
|
-
evictMemberAndContinue(deps, config, dispatch, batchId, batch, memberIssue, decision, now, result);
|
|
2694
|
+
evictMemberAndContinue(deps, config, dispatch, batchId, batch, memberIssue, decision, now, result, release);
|
|
2554
2695
|
}
|
|
2555
2696
|
return;
|
|
2556
2697
|
}
|
|
@@ -2566,7 +2707,6 @@ function reconcileMemberSlot(deps, config, dispatch, batchId, slot, now, result)
|
|
|
2566
2707
|
}
|
|
2567
2708
|
if (read.blockedNow || read.dead) {
|
|
2568
2709
|
const lastTool = recordMemberRunLog(deps, dispatch, state0, batchId, batch.executing_member, memberIssue, slot, now);
|
|
2569
|
-
deps.store.withLock((s) => ({ state: releaseSlot(s, batchId, now), result: undefined }));
|
|
2570
2710
|
// A member that never went green (RFC F.1) evicts DIRECTLY — no aggregate
|
|
2571
2711
|
// suite has run yet, so there is nothing for `attributing`/`evicting` (the
|
|
2572
2712
|
// AGGREGATE-suite-red pipeline, RFC F.2) to attribute or revert: the
|
|
@@ -2575,7 +2715,9 @@ function reconcileMemberSlot(deps, config, dispatch, batchId, slot, now, result)
|
|
|
2575
2715
|
// attributing → evicting` is not even a legal edge from mid-`executing`
|
|
2576
2716
|
// (BATCH_TRANSITIONS has no `executing → evicting`) — this stays entirely
|
|
2577
2717
|
// within `executing`/`dissolving`, both of which ARE legal from here.
|
|
2578
|
-
evictMemberAndContinue(deps, config, dispatch, batchId, batch, memberIssue, memberFailureFor(read, lastTool), now, result
|
|
2718
|
+
evictMemberAndContinue(deps, config, dispatch, batchId, batch, memberIssue, memberFailureFor(read, lastTool), now, result,
|
|
2719
|
+
// #844: released in the eviction's own write, never before it.
|
|
2720
|
+
release);
|
|
2579
2721
|
}
|
|
2580
2722
|
}
|
|
2581
2723
|
/**
|
|
@@ -2616,14 +2758,26 @@ function reconcileMemberSlot(deps, config, dispatch, batchId, slot, now, result)
|
|
|
2616
2758
|
* retried by a re-entry: the re-entry takes the duplicate path (pre-existing,
|
|
2617
2759
|
* #595/#613) and the batch continues.
|
|
2618
2760
|
*/
|
|
2619
|
-
function evictMemberDirectly(deps, config, batchId, memberIssue, failure, now
|
|
2761
|
+
function evictMemberDirectly(deps, config, batchId, memberIssue, failure, now,
|
|
2762
|
+
/**
|
|
2763
|
+
* #844: a state transform applied inside pass 1's lock in every branch —
|
|
2764
|
+
* typically the member's slot release (plus, on the parallel rail, its
|
|
2765
|
+
* run's `evicted` status; on `sched resume --batch`, the batch's
|
|
2766
|
+
* `blocked → executing` transition) — so it and the park/eviction record
|
|
2767
|
+
* commit as ONE write. Pre-#844 callers wrote it under a separate lock
|
|
2768
|
+
* first, and an engine exit between the two left the member in-work with
|
|
2769
|
+
* no slot, which the wedge arm (serial) or `spawnParallelMembers`
|
|
2770
|
+
* (parallel) then respawned.
|
|
2771
|
+
*/
|
|
2772
|
+
preWrite) {
|
|
2620
2773
|
const { reason } = failure;
|
|
2621
2774
|
const kind = failure.kind ?? 'evicted';
|
|
2622
2775
|
const dissolvePolicy = (0, types_1.resolveDissolvePolicy)(config.dissolve_policy);
|
|
2623
2776
|
// Pass 1 (pure — park + record the exit): safe to run entirely inside the
|
|
2624
2777
|
// lock, unlike `dissolveBatch` below, which shells out
|
|
2625
2778
|
// (`deps.exec`/`postMilestone`) and so must NOT hold the lock while it runs.
|
|
2626
|
-
const { triggered, duplicate, prior, branch, validated } = deps.store.withLock((
|
|
2779
|
+
const { triggered, duplicate, prior, branch, validated } = deps.store.withLock((held) => {
|
|
2780
|
+
const s = preWrite ? preWrite(held) : held;
|
|
2627
2781
|
const b = (0, state_1.findBatch)(s, batchId);
|
|
2628
2782
|
if (!b) {
|
|
2629
2783
|
return {
|
|
@@ -2708,6 +2862,20 @@ function evictMemberDirectly(deps, config, batchId, memberIssue, failure, now) {
|
|
|
2708
2862
|
});
|
|
2709
2863
|
if (!triggered)
|
|
2710
2864
|
return { batchEnded: false, duplicate: false };
|
|
2865
|
+
return {
|
|
2866
|
+
batchEnded: settleEvictionThreshold(deps, config, batchId, memberIssue, validated, now),
|
|
2867
|
+
duplicate: false,
|
|
2868
|
+
};
|
|
2869
|
+
}
|
|
2870
|
+
/**
|
|
2871
|
+
* Pass 2 of an eviction that tipped the batch past its dissolve threshold:
|
|
2872
|
+
* dissolve it — unless members are already VALIDATED (#810: never dissolve
|
|
2873
|
+
* over landed work; journal `dissolve-suppressed` and continue). Returns
|
|
2874
|
+
* whether the batch's run ended. Shared by `evictMemberDirectly` and #844's
|
|
2875
|
+
* crash recovery in `spawnMemberContinuation`, which re-runs it for an
|
|
2876
|
+
* eviction whose pass 1 landed but whose engine exited before this ran.
|
|
2877
|
+
*/
|
|
2878
|
+
function settleEvictionThreshold(deps, config, batchId, memberIssue, validated, now) {
|
|
2711
2879
|
if (validated.length > 0) {
|
|
2712
2880
|
// #810: never dissolve over validated work — it stays landed and ships
|
|
2713
2881
|
// with the batch; the members that failed are parked, not requeued.
|
|
@@ -2715,19 +2883,19 @@ function evictMemberDirectly(deps, config, batchId, memberIssue, failure, now) {
|
|
|
2715
2883
|
issue: memberIssue,
|
|
2716
2884
|
detail: `eviction threshold reached, but member(s) ${validated.join(',')} are validated on the integration branch — keeping them and continuing with the remaining members instead of dissolving`,
|
|
2717
2885
|
});
|
|
2718
|
-
return
|
|
2886
|
+
return false;
|
|
2719
2887
|
}
|
|
2720
|
-
//
|
|
2721
|
-
//
|
|
2722
|
-
//
|
|
2888
|
+
// Outside the lock — dissolveBatch shells out: re-load fresh (pass 1's
|
|
2889
|
+
// write already landed), dissolve, then re-apply just this batch's + the
|
|
2890
|
+
// requeued members' state under a fresh lock.
|
|
2723
2891
|
const state = deps.store.load();
|
|
2724
2892
|
const batch = (0, state_1.findBatch)(state, batchId);
|
|
2725
2893
|
if (!batch)
|
|
2726
|
-
return
|
|
2894
|
+
return false;
|
|
2727
2895
|
const rDeps = recoveryDeps(deps, config, batch, now);
|
|
2728
2896
|
const outcome = (0, recovery_1.dissolveBatch)(state, batchId, { strategy: 'full', reason: 'eviction-threshold' }, rDeps);
|
|
2729
2897
|
applyDissolveOutcome(deps, batchId, outcome, now);
|
|
2730
|
-
return
|
|
2898
|
+
return true;
|
|
2731
2899
|
}
|
|
2732
2900
|
function reconcileFixSlot(deps, config, batchId, slot, now, result) {
|
|
2733
2901
|
if (slot.pid !== null && deps.spawnDeps.isAlive(slot.pid, slot.pid_start ?? undefined))
|
|
@@ -2939,7 +3107,13 @@ function reconcileTailSlot(deps, config, batchId, slot, now, result) {
|
|
|
2939
3107
|
result.failed.push(unit(batchId));
|
|
2940
3108
|
}
|
|
2941
3109
|
}
|
|
2942
|
-
function reconcileReportSlot(deps,
|
|
3110
|
+
function reconcileReportSlot(deps,
|
|
3111
|
+
// Unused — kept so this function's signature matches its `reconcile*Slot`
|
|
3112
|
+
// siblings (`reconcileMemberSlot`, `reconcileFixSlot`, `reconcileTailSlot`),
|
|
3113
|
+
// all called uniformly as `reconcile*Slot(deps, config, ...)` from the same
|
|
3114
|
+
// dispatch loop (#830 — pre-existing Biome `noUnusedFunctionParameters`,
|
|
3115
|
+
// independently fixed the same way by #855/#857).
|
|
3116
|
+
_config, batchId, slot, now, result) {
|
|
2943
3117
|
const state = deps.store.load();
|
|
2944
3118
|
const batch = (0, state_1.findBatch)(state, batchId);
|
|
2945
3119
|
if (!batch || batch.anchor === null)
|
|
@@ -4048,7 +4222,10 @@ function spawnParallelMembers(deps, config, dispatch, batchId, now, result) {
|
|
|
4048
4222
|
evictMemberDirectly(deps, config, batchId, next.issue, {
|
|
4049
4223
|
reason: `member-worktree-prep-failed:${prep.reason}`,
|
|
4050
4224
|
detail: `member worktree prep failed: ${prep.reason} (member ${next.index} #${next.issue}, branch ${memberBranchFor(batchId, next.index, next.issue)})`,
|
|
4051
|
-
}, now
|
|
4225
|
+
}, now,
|
|
4226
|
+
// #844: a run already recorded (a respawn whose tree vanished) must
|
|
4227
|
+
// not stay `running` once its member is evicted.
|
|
4228
|
+
(s) => patchRun(s, batchId, next.issue, { status: 'evicted', gate_inconclusive: null }, now));
|
|
4052
4229
|
result.failed.push((0, state_1.batchMemberUnit)(batchId, next.issue));
|
|
4053
4230
|
continue;
|
|
4054
4231
|
}
|
|
@@ -4109,17 +4286,19 @@ function spawnParallelMembers(deps, config, dispatch, batchId, now, result) {
|
|
|
4109
4286
|
* evicted work's only recoverable copy, as in serial mode). A dissolve this
|
|
4110
4287
|
* tips over is torn down by `teardownBatch`, which covers every live run.
|
|
4111
4288
|
*/
|
|
4112
|
-
function evictParallelMember(deps, config, batchId, run, failure, now, result
|
|
4113
|
-
|
|
4289
|
+
function evictParallelMember(deps, config, batchId, run, failure, now, result,
|
|
4290
|
+
/** #844: the member's own slot release — committed in the eviction's write. */
|
|
4291
|
+
release) {
|
|
4292
|
+
const { batchEnded, duplicate } = evictMemberDirectly(deps, config, batchId, run.issue, failure, now,
|
|
4293
|
+
// #844: the run's `evicted` status lands in the eviction's write too — a
|
|
4294
|
+
// separate write left an evicted member's run `running` (slotless, its
|
|
4295
|
+
// tree kept until batch end) when the engine exited in between.
|
|
4296
|
+
(s) => patchRun(release ? release(s) : s, batchId, run.issue, { status: 'evicted', gate_inconclusive: null }, now));
|
|
4114
4297
|
if (duplicate)
|
|
4115
4298
|
return;
|
|
4116
4299
|
result.failed.push(batchEnded ? unit(batchId) : (0, state_1.batchMemberUnit)(batchId, run.issue));
|
|
4117
4300
|
if (batchEnded)
|
|
4118
4301
|
return;
|
|
4119
|
-
deps.store.withLock((s) => ({
|
|
4120
|
-
state: patchRun(s, batchId, run.issue, { status: 'evicted', gate_inconclusive: null }, now),
|
|
4121
|
-
result: undefined,
|
|
4122
|
-
}));
|
|
4123
4302
|
teardownRunTree(deps, batchId, run, false, now);
|
|
4124
4303
|
}
|
|
4125
4304
|
/**
|
|
@@ -4150,8 +4329,7 @@ function reconcileParallelMemberSlot(deps, config, dispatch, batchId, run, slot,
|
|
|
4150
4329
|
recordMemberRunLog(deps, dispatch, state0, batchId, run.index, run.issue, slot, now);
|
|
4151
4330
|
const verdict = evaluateIncrementalGate(deps, batchId, run.worktree, run.issue, now);
|
|
4152
4331
|
if (verdict.kind === 'failed') {
|
|
4153
|
-
deps
|
|
4154
|
-
evictParallelMember(deps, config, batchId, run, { reason: verdict.reason, detail: verdict.detail }, now, result);
|
|
4332
|
+
evictParallelMember(deps, config, batchId, run, { reason: verdict.reason, detail: verdict.detail }, now, result, release);
|
|
4155
4333
|
return;
|
|
4156
4334
|
}
|
|
4157
4335
|
deps.store.withLock((s) => ({
|
|
@@ -4172,7 +4350,7 @@ function reconcileParallelMemberSlot(deps, config, dispatch, batchId, run, slot,
|
|
|
4172
4350
|
read.milestone !== null) {
|
|
4173
4351
|
const decision = decideWrongProcedure(deps, batchId, run.issue, read.milestone, slot, release, () => recordMemberRunLog(deps, dispatch, state0, batchId, run.index, run.issue, slot, now), now);
|
|
4174
4352
|
if (typeof decision === 'object') {
|
|
4175
|
-
evictParallelMember(deps, config, batchId, run, decision, now, result);
|
|
4353
|
+
evictParallelMember(deps, config, batchId, run, decision, now, result, release);
|
|
4176
4354
|
}
|
|
4177
4355
|
return;
|
|
4178
4356
|
}
|
|
@@ -4185,8 +4363,7 @@ function reconcileParallelMemberSlot(deps, config, dispatch, batchId, run, slot,
|
|
|
4185
4363
|
}
|
|
4186
4364
|
if (read.blockedNow || read.dead) {
|
|
4187
4365
|
const lastTool = recordMemberRunLog(deps, dispatch, state0, batchId, run.index, run.issue, slot, now);
|
|
4188
|
-
deps
|
|
4189
|
-
evictParallelMember(deps, config, batchId, run, memberFailureFor(read, lastTool), now, result);
|
|
4366
|
+
evictParallelMember(deps, config, batchId, run, memberFailureFor(read, lastTool), now, result, release);
|
|
4190
4367
|
}
|
|
4191
4368
|
}
|
|
4192
4369
|
/**
|
|
@@ -4589,12 +4766,8 @@ function runBatchTick(deps, config, dispatch) {
|
|
|
4589
4766
|
const dissolvable = fresh !== undefined && (0, state_1.allowedBatchTransitions)(fresh.status).includes('dissolving');
|
|
4590
4767
|
if (dissolvable) {
|
|
4591
4768
|
// #809: a parallel batch holds one slot per running member too.
|
|
4592
|
-
for (const liveSlot of (0, state_1.slotsForBatch)(current, batch.id))
|
|
4593
|
-
|
|
4594
|
-
deps.spawnDeps.isAlive(liveSlot.pid, liveSlot.pid_start ?? undefined)) {
|
|
4595
|
-
deps.spawnDeps.kill(liveSlot.pid, liveSlot.pid_start ?? undefined);
|
|
4596
|
-
}
|
|
4597
|
-
}
|
|
4769
|
+
for (const liveSlot of (0, state_1.slotsForBatch)(current, batch.id))
|
|
4770
|
+
killLiveAgent(deps, liveSlot);
|
|
4598
4771
|
const outcome = (0, recovery_1.dissolveBatch)(current, batch.id, {
|
|
4599
4772
|
strategy: 'full',
|
|
4600
4773
|
reason: `dispatch-profile-missing:${batch.dispatch_profile}`,
|