@ai-dossier/sched 0.55.1 → 0.57.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +75 -10
- package/dist/batch-dispatch.d.ts +11 -3
- package/dist/batch-dispatch.d.ts.map +1 -1
- package/dist/batch-dispatch.js +258 -67
- package/dist/batch-dispatch.js.map +1 -1
- package/dist/dispatch.d.ts +36 -3
- package/dist/dispatch.d.ts.map +1 -1
- package/dist/dispatch.js +97 -9
- package/dist/dispatch.js.map +1 -1
- package/dist/engine.d.ts +9 -0
- package/dist/engine.d.ts.map +1 -1
- package/dist/engine.js +90 -2
- package/dist/engine.js.map +1 -1
- package/dist/index.d.ts +3 -2
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +11 -5
- package/dist/index.js.map +1 -1
- package/dist/recovery.d.ts +66 -5
- package/dist/recovery.d.ts.map +1 -1
- package/dist/recovery.js +317 -67
- package/dist/recovery.js.map +1 -1
- package/dist/resume-seed.d.ts +36 -0
- package/dist/resume-seed.d.ts.map +1 -0
- package/dist/resume-seed.js +83 -0
- package/dist/resume-seed.js.map +1 -0
- package/dist/state.d.ts +2 -0
- package/dist/state.d.ts.map +1 -1
- package/dist/state.js +20 -3
- package/dist/state.js.map +1 -1
- package/dist/types.d.ts +40 -13
- package/dist/types.d.ts.map +1 -1
- package/dist/types.js +14 -10
- package/dist/types.js.map +1 -1
- package/package.json +1 -1
package/dist/batch-dispatch.js
CHANGED
|
@@ -164,9 +164,18 @@ function sanitizeUntrustedText(value) {
|
|
|
164
164
|
function journalEvent(deps, event, unitId, extra = {}) {
|
|
165
165
|
deps.journal.append((0, journal_1.unitEvent)(event, unitId, extra), deps.now());
|
|
166
166
|
}
|
|
167
|
-
/**
|
|
167
|
+
/** Whether the slot's agent is still alive (pid-start-safe: a reused pid reads dead). */
|
|
168
|
+
function agentAlive(deps, slot) {
|
|
169
|
+
return slot.pid !== null && deps.spawnDeps.isAlive(slot.pid, slot.pid_start ?? undefined);
|
|
170
|
+
}
|
|
171
|
+
/**
|
|
172
|
+
* Stop the slot's agent if it is still alive (pid-start-safe: a reused pid is
|
|
173
|
+
* never signalled). SIGTERM only — no wait, no escalation. A decision that
|
|
174
|
+
* must not run alongside the agent uses `stopAgentBeforeDeciding` instead,
|
|
175
|
+
* which escalates to SIGKILL (#844).
|
|
176
|
+
*/
|
|
168
177
|
function killLiveAgent(deps, slot) {
|
|
169
|
-
if (
|
|
178
|
+
if (agentAlive(deps, slot)) {
|
|
170
179
|
deps.spawnDeps.kill(slot.pid, slot.pid_start ?? undefined);
|
|
171
180
|
}
|
|
172
181
|
}
|
|
@@ -806,9 +815,11 @@ function landMemberBranch(deps, batchId, now) {
|
|
|
806
815
|
* `-d` after a landing (fully merged into the integration branch), `-D` on
|
|
807
816
|
* the eviction path (its commits never landed; since #810 the parked entry
|
|
808
817
|
* records the branch and an operator requeue continues from the REMOTE copy).
|
|
809
|
-
* The REMOTE member branch is deleted
|
|
810
|
-
*
|
|
811
|
-
*
|
|
818
|
+
* The REMOTE member branch is never deleted here (#840): on eviction the
|
|
819
|
+
* pushed sha is the evicted work's only recoverable copy, and after a landing
|
|
820
|
+
* it is what a later aggregate-suite eviction parks the member on. Batch
|
|
821
|
+
* teardown deletes it once the batch ships or dissolves
|
|
822
|
+
* (`deleteMemberBranches`).
|
|
812
823
|
*
|
|
813
824
|
* Best-effort and idempotent: null fields are a no-op, and a failed cleanup
|
|
814
825
|
* journals `teardown-failed` rather than throwing into the caller's
|
|
@@ -894,9 +905,6 @@ issue) {
|
|
|
894
905
|
deps.exec('git', ['branch', flag, branch], deps.repoDir) === null
|
|
895
906
|
? `failed-branch-delete-${flag}`
|
|
896
907
|
: `branch-deleted-${flag}`;
|
|
897
|
-
if (landed) {
|
|
898
|
-
deps.exec('git', ['push', 'origin', '--delete', branch], deps.repoDir);
|
|
899
|
-
}
|
|
900
908
|
}
|
|
901
909
|
const failed = !treeCleared || branchCleanup.startsWith('failed');
|
|
902
910
|
journalEvent(deps, failed ? 'teardown-failed' : 'member-worktree-torn-down', unit(batchId), {
|
|
@@ -1285,6 +1293,9 @@ function spawnMemberContinuation(deps, config, dispatch, batchId, now, result) {
|
|
|
1285
1293
|
// contract `prepareMemberWorktree` documents). The wedge arm of
|
|
1286
1294
|
// `runBatchTick` calls this EVERY tick while the batch is `executing`
|
|
1287
1295
|
// with no slot, so the cheap gates run first:
|
|
1296
|
+
// - recovery (#844): a current member already in `evictions[]` is advanced
|
|
1297
|
+
// past, never respawned — checked before capacity, since the advance
|
|
1298
|
+
// needs no slot.
|
|
1288
1299
|
// - capacity: prep before the free-capacity check would do pool/git/npm
|
|
1289
1300
|
// work (and journal) per tick against a full scheduler — the #610/#630/
|
|
1290
1301
|
// #632 per-tick emission trap. `claimAndSpawn` re-checks under the lock;
|
|
@@ -1301,6 +1312,30 @@ function spawnMemberContinuation(deps, config, dispatch, batchId, now, result) {
|
|
|
1301
1312
|
const memberIssue = batch.members[batch.executing_member - 1];
|
|
1302
1313
|
if (memberIssue === undefined)
|
|
1303
1314
|
return;
|
|
1315
|
+
const evicted = batch.evictions.find((e) => e.issue === memberIssue);
|
|
1316
|
+
if (evicted !== undefined) {
|
|
1317
|
+
// #844: the member's eviction (and its slot release, in the same write)
|
|
1318
|
+
// landed, but the engine died before advancing past it — possibly before
|
|
1319
|
+
// journaling its cause, too, so the record's cause goes on this line.
|
|
1320
|
+
// Respawning it would dispatch an evicted member again — finish what the
|
|
1321
|
+
// eviction would have done instead: the dissolve its threshold tripped,
|
|
1322
|
+
// or the advance.
|
|
1323
|
+
const kind = evicted.kind ?? 'evicted';
|
|
1324
|
+
journalEvent(deps, 'member-advance-recovered', unit(batchId), {
|
|
1325
|
+
issue: memberIssue,
|
|
1326
|
+
reason: evicted.reason,
|
|
1327
|
+
kind,
|
|
1328
|
+
evicted_at: evicted.at,
|
|
1329
|
+
detail: `member ${batch.executing_member} is already parked ${kind} (${evicted.reason}) but the batch never advanced past it (engine exit mid-eviction) — continuing instead of respawning it`,
|
|
1330
|
+
});
|
|
1331
|
+
if ((0, recovery_1.checkDissolveTrigger)(batch, (0, types_1.resolveDissolvePolicy)(config.dissolve_policy)) &&
|
|
1332
|
+
settleEvictionThreshold(deps, config, batchId, memberIssue, (0, state_1.validatedMembersOf)(state, batch), now)) {
|
|
1333
|
+
result.failed.push(unit(batchId));
|
|
1334
|
+
return;
|
|
1335
|
+
}
|
|
1336
|
+
continuePastEvictedMember(deps, config, dispatch, batchId, batch, memberIssue, now, result);
|
|
1337
|
+
return;
|
|
1338
|
+
}
|
|
1304
1339
|
if ((0, scheduler_1.freeCapacity)(state, config) === 0)
|
|
1305
1340
|
return;
|
|
1306
1341
|
const expectedBranch = memberBranchFor(batchId, batch.executing_member, memberIssue);
|
|
@@ -1669,7 +1704,7 @@ function evictOffender(deps, config, batchId, offender, attribution, now, result
|
|
|
1669
1704
|
const rDeps = recoveryDeps(deps, config, batch, now);
|
|
1670
1705
|
const outcome = (0, recovery_1.evictMembers)(state, batchId, { issues: [offender], reason: 'suite-red-after-fix', attribution, ranges: batch.ranges }, rDeps);
|
|
1671
1706
|
deps.store.withLock((s) => ({
|
|
1672
|
-
state: applyBatchAndIssues(s, outcome.state, batchId, [...outcome.requeued, ...(outcome.dissolve?.parked ?? [])], batch.members),
|
|
1707
|
+
state: applyBatchAndIssues(s, outcome.state, batchId, [...new Set([...outcome.requeued, ...outcome.parked, ...(outcome.dissolve?.parked ?? [])])], batch.members),
|
|
1673
1708
|
result: undefined,
|
|
1674
1709
|
}));
|
|
1675
1710
|
// #810: `dissolve.blocked` — a full dissolve refused over validated
|
|
@@ -1772,8 +1807,14 @@ serialTeardown) {
|
|
|
1772
1807
|
}
|
|
1773
1808
|
/**
|
|
1774
1809
|
* Evict the current member and either dissolve, or continue the batch via
|
|
1775
|
-
* `advanceMemberOrValidate` — the shared tail of
|
|
1776
|
-
* (
|
|
1810
|
+
* `advanceMemberOrValidate` — the shared tail of the serial member-failure
|
|
1811
|
+
* rails (dead/blocked, the incremental gate below, the wrong-procedure
|
|
1812
|
+
* evictions, and `sched resume --batch`'s recheck).
|
|
1813
|
+
*
|
|
1814
|
+
* #844: a caller whose member still holds its slot MUST pass its release as
|
|
1815
|
+
* `preWrite` — never release under a separate lock first. `evictMemberDirectly` commits
|
|
1816
|
+
* the release with the park/eviction record in one write, so an engine exit
|
|
1817
|
+
* cannot leave the member slotless and un-evicted (and so respawned).
|
|
1777
1818
|
*
|
|
1778
1819
|
* #613: `evictMemberDirectly`'s duplicate check (#595) is the one atomic,
|
|
1779
1820
|
* lock-protected claim on "did THIS member's eviction already happen" — so
|
|
@@ -1795,14 +1836,27 @@ serialTeardown) {
|
|
|
1795
1836
|
* `runBatchTick`/`resumeBlockedGate` entry points cannot reproduce since
|
|
1796
1837
|
* both always read state fresh.
|
|
1797
1838
|
*/
|
|
1798
|
-
function evictMemberAndContinue(deps, config, dispatch, batchId, batch, memberIssue, failure, now, result
|
|
1799
|
-
|
|
1839
|
+
function evictMemberAndContinue(deps, config, dispatch, batchId, batch, memberIssue, failure, now, result,
|
|
1840
|
+
/** #844: the member's slot release (or any pre-write) — committed in the eviction's own write. */
|
|
1841
|
+
preWrite) {
|
|
1842
|
+
const { batchEnded, duplicate } = evictMemberDirectly(deps, config, batchId, memberIssue, failure, now, preWrite);
|
|
1800
1843
|
if (duplicate)
|
|
1801
1844
|
return;
|
|
1802
1845
|
if (batchEnded) {
|
|
1803
1846
|
result.failed.push(unit(batchId));
|
|
1804
1847
|
return;
|
|
1805
1848
|
}
|
|
1849
|
+
continuePastEvictedMember(deps, config, dispatch, batchId, batch, memberIssue, now, result);
|
|
1850
|
+
}
|
|
1851
|
+
/**
|
|
1852
|
+
* The serial rail after a member's eviction has been recorded: tear its tree
|
|
1853
|
+
* down and advance. Also the crash recovery for an engine that died between
|
|
1854
|
+
* the eviction's write and this advance (#844) — `spawnMemberContinuation`
|
|
1855
|
+
* finds the batch still pointing at a member already in `evictions[]` and
|
|
1856
|
+
* continues from here instead of respawning it. `advanceMemberOrValidate`'s
|
|
1857
|
+
* one-shot claim (#613) keeps a re-entry from advancing twice.
|
|
1858
|
+
*/
|
|
1859
|
+
function continuePastEvictedMember(deps, config, dispatch, batchId, batch, memberIssue, now, result) {
|
|
1806
1860
|
// #677: the evicted member's commits never landed (eviction is the
|
|
1807
1861
|
// pre-landing rail), so there is nothing to revert on the integration
|
|
1808
1862
|
// branch — but its worktree/branch must go, or the next tick's prep for
|
|
@@ -1842,7 +1896,9 @@ function evictMemberAndContinue(deps, config, dispatch, batchId, batch, memberIs
|
|
|
1842
1896
|
* configured or both checks came back `ok`, meaning the caller should treat
|
|
1843
1897
|
* the member as genuinely complete.
|
|
1844
1898
|
*/
|
|
1845
|
-
function runIncrementalGate(deps, config, dispatch, batchId, batch, memberIssue, now, result
|
|
1899
|
+
function runIncrementalGate(deps, config, dispatch, batchId, batch, memberIssue, now, result,
|
|
1900
|
+
/** The member's slot release — handed to the eviction on `failed` (#844). */
|
|
1901
|
+
release) {
|
|
1846
1902
|
if (batch.worktree === null || !deps.runCapability)
|
|
1847
1903
|
return false;
|
|
1848
1904
|
// #677: the gate judges the member's OWN work — pre-landing, its diff
|
|
@@ -1854,11 +1910,10 @@ function runIncrementalGate(deps, config, dispatch, batchId, batch, memberIssue,
|
|
|
1854
1910
|
if (verdict.kind === 'pass')
|
|
1855
1911
|
return false;
|
|
1856
1912
|
if (verdict.kind === 'failed') {
|
|
1857
|
-
deps
|
|
1858
|
-
evictMemberAndContinue(deps, config, dispatch, batchId, batch, memberIssue, { reason: verdict.reason, detail: verdict.detail }, now, result);
|
|
1913
|
+
evictMemberAndContinue(deps, config, dispatch, batchId, batch, memberIssue, { reason: verdict.reason, detail: verdict.detail }, now, result, release);
|
|
1859
1914
|
return true;
|
|
1860
1915
|
}
|
|
1861
|
-
deps.store.withLock((s) => ({ state:
|
|
1916
|
+
deps.store.withLock((s) => ({ state: release(s), result: undefined }));
|
|
1862
1917
|
// #677: F.11's block semantics are "the member's commit stays on the
|
|
1863
1918
|
// branch, nothing requeued/reverted" — under member worktrees that is
|
|
1864
1919
|
// true only if the member LANDS before the batch blocks. The block means
|
|
@@ -2176,19 +2231,21 @@ function resumeBlockedGate(deps, config, dispatch, batchId, now) {
|
|
|
2176
2231
|
detail: excerpt,
|
|
2177
2232
|
};
|
|
2178
2233
|
}
|
|
2179
|
-
|
|
2180
|
-
state: (0, state_1.patchBatch)((0, state_1.transitionBatch)(s, batchId, 'executing', {}, now), batchId, { blocked_reason: null }, now),
|
|
2181
|
-
result: undefined,
|
|
2182
|
-
}));
|
|
2234
|
+
const unblock = (s) => (0, state_1.patchBatch)((0, state_1.transitionBatch)(s, batchId, 'executing', {}, now), batchId, { blocked_reason: null }, now);
|
|
2183
2235
|
if (recheck.outcome === 'task-failed') {
|
|
2184
2236
|
const reason = `incremental-gate-failed:${capabilityId}`;
|
|
2185
2237
|
writeGateLog(deps, batchId, capabilityId, memberIssue, recheck.outputTail);
|
|
2186
2238
|
evictMemberAndContinue(deps, config, batchDispatch, batchId, batch, memberIssue, {
|
|
2187
2239
|
reason,
|
|
2188
2240
|
detail: withExcerpt(`sched resume --batch: cap run ${capabilityId} reported task-failed on recheck`, excerpt),
|
|
2189
|
-
}, now, result
|
|
2241
|
+
}, now, result,
|
|
2242
|
+
// #844: the unblock commits with the eviction — two writes left a
|
|
2243
|
+
// slotless `executing` batch whose un-evicted member the wedge arm
|
|
2244
|
+
// then respawned.
|
|
2245
|
+
unblock);
|
|
2190
2246
|
return { outcome: 'evicted', capability: capabilityId, detail: excerpt };
|
|
2191
2247
|
}
|
|
2248
|
+
deps.store.withLock((s) => ({ state: unblock(s), result: undefined }));
|
|
2192
2249
|
journalEvent(deps, 'external-advance', unit(batchId), {
|
|
2193
2250
|
issue: memberIssue,
|
|
2194
2251
|
detail: `sched resume --batch: cap run ${capabilityId} reported ok on recheck`,
|
|
@@ -2389,6 +2446,84 @@ function wrongProcedureCounts(batch, memberIssue, read) {
|
|
|
2389
2446
|
const repromptedAt = Date.parse(prior.reprompted_at);
|
|
2390
2447
|
return Number.isNaN(at) || Number.isNaN(repromptedAt) ? false : at > repromptedAt;
|
|
2391
2448
|
}
|
|
2449
|
+
/**
|
|
2450
|
+
* A kill timestamp that belongs to the slot's CURRENT dispatch, or null
|
|
2451
|
+
* (#844): one stamped before `spawned_at` was for an earlier dispatch in the
|
|
2452
|
+
* same slot, and an unparseable or future one is no anchor at all.
|
|
2453
|
+
*/
|
|
2454
|
+
function currentDispatchStamp(stamp, spawnedAt, now) {
|
|
2455
|
+
if (stamp === null)
|
|
2456
|
+
return null;
|
|
2457
|
+
const at = Date.parse(stamp);
|
|
2458
|
+
if (Number.isNaN(at))
|
|
2459
|
+
return null;
|
|
2460
|
+
if (spawnedAt !== null && at < Date.parse(spawnedAt))
|
|
2461
|
+
return null;
|
|
2462
|
+
// A future stamp (clock step, hand edit) would make the elapsed time
|
|
2463
|
+
// negative and switch the escalation off for good — re-stamp instead.
|
|
2464
|
+
if (at > now.getTime())
|
|
2465
|
+
return null;
|
|
2466
|
+
return stamp;
|
|
2467
|
+
}
|
|
2468
|
+
/**
|
|
2469
|
+
* Stop the slot's agent before a decision that must not run alongside it
|
|
2470
|
+
* (#844). Returns `true` while it is still alive — the caller waits and
|
|
2471
|
+
* decides on a later tick — and `false` once it is gone (or never had a pid).
|
|
2472
|
+
*
|
|
2473
|
+
* The first sight sends SIGTERM, stamps `kill_sent_at` and journals
|
|
2474
|
+
* `member-stop-requested` once; each later tick
|
|
2475
|
+
* re-sends SIGTERM until the agent has outlived it by `KILL_ESCALATION_MS`,
|
|
2476
|
+
* then sends SIGKILL (to its process group — `SpawnDeps.kill`) and journals
|
|
2477
|
+
* `kill-escalated` once (`kill_escalated_at`). Pre-#844 this path re-sent
|
|
2478
|
+
* SIGTERM forever, so an agent ignoring it held its member indefinitely.
|
|
2479
|
+
*/
|
|
2480
|
+
function stopAgentBeforeDeciding(deps, batchId, memberIssue, slot, now) {
|
|
2481
|
+
if (!agentAlive(deps, slot))
|
|
2482
|
+
return false;
|
|
2483
|
+
const pid = slot.pid;
|
|
2484
|
+
const pidStart = slot.pid_start ?? undefined;
|
|
2485
|
+
const sentAt = currentDispatchStamp(slot.kill_sent_at, slot.spawned_at, now);
|
|
2486
|
+
const aliveForMs = sentAt === null ? 0 : now.getTime() - Date.parse(sentAt);
|
|
2487
|
+
const escalate = sentAt !== null && aliveForMs >= dispatch_1.KILL_ESCALATION_MS;
|
|
2488
|
+
// `kill` refuses a pid whose recorded identity no longer matches (reused by
|
|
2489
|
+
// another process): the agent we spawned is gone, so stop waiting on it.
|
|
2490
|
+
if (!deps.spawnDeps.kill(pid, pidStart, escalate ? 'SIGKILL' : 'SIGTERM'))
|
|
2491
|
+
return false;
|
|
2492
|
+
const firstEscalation = escalate && currentDispatchStamp(slot.kill_escalated_at, slot.spawned_at, now) === null;
|
|
2493
|
+
if (sentAt === null || firstEscalation) {
|
|
2494
|
+
deps.store.withLock((s) => {
|
|
2495
|
+
const live = s.slots.find((x) => x.id === slot.id);
|
|
2496
|
+
// Only the slot still holding THIS agent — never a later dispatch's.
|
|
2497
|
+
if (!live || live.unit !== slot.unit || live.pid !== pid) {
|
|
2498
|
+
return { state: s, result: undefined };
|
|
2499
|
+
}
|
|
2500
|
+
const stamp = now.toISOString();
|
|
2501
|
+
return {
|
|
2502
|
+
state: (0, state_1.patchSlot)(s, slot.id, sentAt === null
|
|
2503
|
+
? { kill_sent_at: stamp, kill_escalated_at: null }
|
|
2504
|
+
: { kill_escalated_at: stamp }, now),
|
|
2505
|
+
result: undefined,
|
|
2506
|
+
};
|
|
2507
|
+
});
|
|
2508
|
+
}
|
|
2509
|
+
if (sentAt === null) {
|
|
2510
|
+
journalEvent(deps, 'member-stop-requested', unit(batchId), {
|
|
2511
|
+
issue: memberIssue,
|
|
2512
|
+
pid,
|
|
2513
|
+
slot: slot.id,
|
|
2514
|
+
detail: `SIGTERM sent — waiting for the agent to exit before deciding (SIGKILL after ${dispatch_1.KILL_ESCALATION_MS / 1000}s)`,
|
|
2515
|
+
});
|
|
2516
|
+
}
|
|
2517
|
+
if (firstEscalation) {
|
|
2518
|
+
journalEvent(deps, 'kill-escalated', unit(batchId), {
|
|
2519
|
+
issue: memberIssue,
|
|
2520
|
+
pid,
|
|
2521
|
+
slot: slot.id,
|
|
2522
|
+
detail: `agent still alive ${Math.round(aliveForMs / 1000)}s after SIGTERM (bound ${dispatch_1.KILL_ESCALATION_MS / 1000}s) — sent SIGKILL (to its process group where available)`,
|
|
2523
|
+
});
|
|
2524
|
+
}
|
|
2525
|
+
return true;
|
|
2526
|
+
}
|
|
2392
2527
|
/**
|
|
2393
2528
|
* #822 (#810 proposal 3): the member's dispatch ran the wrong procedure (a
|
|
2394
2529
|
* full-cycle-shaped trail — imboard #4174 ran full-cycle as a batch member).
|
|
@@ -2398,23 +2533,27 @@ function wrongProcedureCounts(batch, memberIssue, read) {
|
|
|
2398
2533
|
* `reprompted_members` — the wedge arm (serial) or `spawnParallelMembers`
|
|
2399
2534
|
* (parallel) respawns it in place, in the same worktree, with
|
|
2400
2535
|
* `wrongProcedureDirective` appended. Returns `'evict'` when the member was
|
|
2401
|
-
* already re-prompted once: the caller evicts it `wrong-procedure`.
|
|
2536
|
+
* already re-prompted once: the caller evicts it `wrong-procedure`. On
|
|
2537
|
+
* `'evict'` the slot is NOT released here — the caller passes `release` to
|
|
2538
|
+
* `evictMemberAndContinue`/`evictParallelMember`, which commits it in the
|
|
2539
|
+
* eviction's own write (#844).
|
|
2402
2540
|
*/
|
|
2403
2541
|
function repromptWrongProcedure(deps, batchId, memberIssue, milestoneAt, slot, release, now) {
|
|
2404
2542
|
// Never respawn into (or release) a worktree the old agent is still in: a
|
|
2405
2543
|
// dying full-cycle run could still push, post, or open a PR. Signal it and
|
|
2406
2544
|
// decide on a later tick, once it is gone — and so the re-prompt's
|
|
2407
2545
|
// `reprompted_at` fence postdates everything it could still post.
|
|
2408
|
-
if (
|
|
2409
|
-
deps.spawnDeps.kill(slot.pid, slot.pid_start ?? undefined);
|
|
2546
|
+
if (stopAgentBeforeDeciding(deps, batchId, memberIssue, slot, now))
|
|
2410
2547
|
return 'waiting';
|
|
2411
|
-
}
|
|
2412
2548
|
const outcome = deps.store.withLock((s) => {
|
|
2413
|
-
const
|
|
2414
|
-
const b = (0, state_1.findBatch)(released, batchId);
|
|
2549
|
+
const b = (0, state_1.findBatch)(s, batchId);
|
|
2415
2550
|
if (!b || b.reprompted_members.some((r) => r.issue === memberIssue)) {
|
|
2416
|
-
|
|
2551
|
+
// #844: NOT released here — the caller's eviction commits the release
|
|
2552
|
+
// in its own write, so a crash in between cannot leave the member
|
|
2553
|
+
// slotless and un-evicted (and so respawned).
|
|
2554
|
+
return { state: s, result: 'evict' };
|
|
2417
2555
|
}
|
|
2556
|
+
const released = release(s);
|
|
2418
2557
|
return {
|
|
2419
2558
|
state: (0, state_1.patchBatch)(released, batchId, {
|
|
2420
2559
|
reprompted_members: [
|
|
@@ -2455,18 +2594,16 @@ function wrongProcedureFailure(lastTool, shippedPr = null) {
|
|
|
2455
2594
|
* #822: the one wrong-procedure decision shared by both member rails —
|
|
2456
2595
|
* `'waiting'` (the old agent is still dying), `'reprompted'`, or the
|
|
2457
2596
|
* eviction failure to apply. A run that already shipped is evicted without
|
|
2458
|
-
* a re-prompt.
|
|
2597
|
+
* a re-prompt. On an eviction the slot is still held: the caller passes
|
|
2598
|
+
* `release` to the eviction, which commits both in one write (#844).
|
|
2459
2599
|
*/
|
|
2460
2600
|
function decideWrongProcedure(deps, batchId, memberIssue, milestone, slot, release, lastTool, now) {
|
|
2461
2601
|
const shipped = (0, groundtruth_1.wrongProcedureShippedPr)(milestone);
|
|
2462
2602
|
if (shipped !== null) {
|
|
2463
|
-
if (
|
|
2464
|
-
deps.spawnDeps.kill(slot.pid, slot.pid_start ?? undefined);
|
|
2603
|
+
if (stopAgentBeforeDeciding(deps, batchId, memberIssue, slot, now))
|
|
2465
2604
|
return 'waiting';
|
|
2466
|
-
|
|
2467
|
-
|
|
2468
|
-
deps.store.withLock((s) => ({ state: release(s), result: undefined }));
|
|
2469
|
-
return wrongProcedureFailure(tool, shipped);
|
|
2605
|
+
// #844: the caller's eviction releases the slot in its own write.
|
|
2606
|
+
return wrongProcedureFailure(lastTool(), shipped);
|
|
2470
2607
|
}
|
|
2471
2608
|
const verdict = repromptWrongProcedure(deps, batchId, memberIssue, milestone.at, slot, release, now);
|
|
2472
2609
|
if (verdict === 'waiting')
|
|
@@ -2521,6 +2658,9 @@ function reconcileMemberSlot(deps, config, dispatch, batchId, slot, now, result)
|
|
|
2521
2658
|
const memberIssue = batch.members[batch.executing_member - 1];
|
|
2522
2659
|
if (memberIssue === undefined)
|
|
2523
2660
|
return;
|
|
2661
|
+
// #844: every rail below that evicts hands THIS to the eviction, which
|
|
2662
|
+
// commits it in the same write — never released under a lock of its own.
|
|
2663
|
+
const release = (s) => releaseSlot(s, batchId, now);
|
|
2524
2664
|
const read = readMemberDispatch(deps, batchId, memberIssue, slot, now);
|
|
2525
2665
|
if (read.kind === 'unreachable')
|
|
2526
2666
|
return;
|
|
@@ -2538,7 +2678,7 @@ function reconcileMemberSlot(deps, config, dispatch, batchId, slot, now, result)
|
|
|
2538
2678
|
// gate already decided the member's fate (evicted or blocked) and
|
|
2539
2679
|
// returned early; `false` means both checks were `ok` and the member is
|
|
2540
2680
|
// genuinely done.
|
|
2541
|
-
if (runIncrementalGate(deps, config, dispatch, batchId, batch, memberIssue, now, result)) {
|
|
2681
|
+
if (runIncrementalGate(deps, config, dispatch, batchId, batch, memberIssue, now, result, release)) {
|
|
2542
2682
|
return;
|
|
2543
2683
|
}
|
|
2544
2684
|
completeMemberGate(deps, config, dispatch, batchId, batch, memberIssue, now, result);
|
|
@@ -2548,9 +2688,9 @@ function reconcileMemberSlot(deps, config, dispatch, batchId, slot, now, result)
|
|
|
2548
2688
|
// before the dead/blocked rail, which would otherwise evict it as an
|
|
2549
2689
|
// unverified exit at the first sight.
|
|
2550
2690
|
if (wrongProcedureCounts(batch, memberIssue, read) && read.milestone !== null) {
|
|
2551
|
-
const decision = decideWrongProcedure(deps, batchId, memberIssue, read.milestone, slot,
|
|
2691
|
+
const decision = decideWrongProcedure(deps, batchId, memberIssue, read.milestone, slot, release, () => recordMemberRunLog(deps, dispatch, state0, batchId, batch.executing_member, memberIssue, slot, now), now);
|
|
2552
2692
|
if (typeof decision === 'object') {
|
|
2553
|
-
evictMemberAndContinue(deps, config, dispatch, batchId, batch, memberIssue, decision, now, result);
|
|
2693
|
+
evictMemberAndContinue(deps, config, dispatch, batchId, batch, memberIssue, decision, now, result, release);
|
|
2554
2694
|
}
|
|
2555
2695
|
return;
|
|
2556
2696
|
}
|
|
@@ -2566,7 +2706,6 @@ function reconcileMemberSlot(deps, config, dispatch, batchId, slot, now, result)
|
|
|
2566
2706
|
}
|
|
2567
2707
|
if (read.blockedNow || read.dead) {
|
|
2568
2708
|
const lastTool = recordMemberRunLog(deps, dispatch, state0, batchId, batch.executing_member, memberIssue, slot, now);
|
|
2569
|
-
deps.store.withLock((s) => ({ state: releaseSlot(s, batchId, now), result: undefined }));
|
|
2570
2709
|
// A member that never went green (RFC F.1) evicts DIRECTLY — no aggregate
|
|
2571
2710
|
// suite has run yet, so there is nothing for `attributing`/`evicting` (the
|
|
2572
2711
|
// AGGREGATE-suite-red pipeline, RFC F.2) to attribute or revert: the
|
|
@@ -2575,7 +2714,9 @@ function reconcileMemberSlot(deps, config, dispatch, batchId, slot, now, result)
|
|
|
2575
2714
|
// attributing → evicting` is not even a legal edge from mid-`executing`
|
|
2576
2715
|
// (BATCH_TRANSITIONS has no `executing → evicting`) — this stays entirely
|
|
2577
2716
|
// within `executing`/`dissolving`, both of which ARE legal from here.
|
|
2578
|
-
evictMemberAndContinue(deps, config, dispatch, batchId, batch, memberIssue, memberFailureFor(read, lastTool), now, result
|
|
2717
|
+
evictMemberAndContinue(deps, config, dispatch, batchId, batch, memberIssue, memberFailureFor(read, lastTool), now, result,
|
|
2718
|
+
// #844: released in the eviction's own write, never before it.
|
|
2719
|
+
release);
|
|
2579
2720
|
}
|
|
2580
2721
|
}
|
|
2581
2722
|
/**
|
|
@@ -2616,14 +2757,26 @@ function reconcileMemberSlot(deps, config, dispatch, batchId, slot, now, result)
|
|
|
2616
2757
|
* retried by a re-entry: the re-entry takes the duplicate path (pre-existing,
|
|
2617
2758
|
* #595/#613) and the batch continues.
|
|
2618
2759
|
*/
|
|
2619
|
-
function evictMemberDirectly(deps, config, batchId, memberIssue, failure, now
|
|
2760
|
+
function evictMemberDirectly(deps, config, batchId, memberIssue, failure, now,
|
|
2761
|
+
/**
|
|
2762
|
+
* #844: a state transform applied inside pass 1's lock in every branch —
|
|
2763
|
+
* typically the member's slot release (plus, on the parallel rail, its
|
|
2764
|
+
* run's `evicted` status; on `sched resume --batch`, the batch's
|
|
2765
|
+
* `blocked → executing` transition) — so it and the park/eviction record
|
|
2766
|
+
* commit as ONE write. Pre-#844 callers wrote it under a separate lock
|
|
2767
|
+
* first, and an engine exit between the two left the member in-work with
|
|
2768
|
+
* no slot, which the wedge arm (serial) or `spawnParallelMembers`
|
|
2769
|
+
* (parallel) then respawned.
|
|
2770
|
+
*/
|
|
2771
|
+
preWrite) {
|
|
2620
2772
|
const { reason } = failure;
|
|
2621
2773
|
const kind = failure.kind ?? 'evicted';
|
|
2622
2774
|
const dissolvePolicy = (0, types_1.resolveDissolvePolicy)(config.dissolve_policy);
|
|
2623
2775
|
// Pass 1 (pure — park + record the exit): safe to run entirely inside the
|
|
2624
2776
|
// lock, unlike `dissolveBatch` below, which shells out
|
|
2625
2777
|
// (`deps.exec`/`postMilestone`) and so must NOT hold the lock while it runs.
|
|
2626
|
-
const { triggered, duplicate, prior, branch, validated } = deps.store.withLock((
|
|
2778
|
+
const { triggered, duplicate, prior, branch, validated } = deps.store.withLock((held) => {
|
|
2779
|
+
const s = preWrite ? preWrite(held) : held;
|
|
2627
2780
|
const b = (0, state_1.findBatch)(s, batchId);
|
|
2628
2781
|
if (!b) {
|
|
2629
2782
|
return {
|
|
@@ -2708,6 +2861,20 @@ function evictMemberDirectly(deps, config, batchId, memberIssue, failure, now) {
|
|
|
2708
2861
|
});
|
|
2709
2862
|
if (!triggered)
|
|
2710
2863
|
return { batchEnded: false, duplicate: false };
|
|
2864
|
+
return {
|
|
2865
|
+
batchEnded: settleEvictionThreshold(deps, config, batchId, memberIssue, validated, now),
|
|
2866
|
+
duplicate: false,
|
|
2867
|
+
};
|
|
2868
|
+
}
|
|
2869
|
+
/**
|
|
2870
|
+
* Pass 2 of an eviction that tipped the batch past its dissolve threshold:
|
|
2871
|
+
* dissolve it — unless members are already VALIDATED (#810: never dissolve
|
|
2872
|
+
* over landed work; journal `dissolve-suppressed` and continue). Returns
|
|
2873
|
+
* whether the batch's run ended. Shared by `evictMemberDirectly` and #844's
|
|
2874
|
+
* crash recovery in `spawnMemberContinuation`, which re-runs it for an
|
|
2875
|
+
* eviction whose pass 1 landed but whose engine exited before this ran.
|
|
2876
|
+
*/
|
|
2877
|
+
function settleEvictionThreshold(deps, config, batchId, memberIssue, validated, now) {
|
|
2711
2878
|
if (validated.length > 0) {
|
|
2712
2879
|
// #810: never dissolve over validated work — it stays landed and ships
|
|
2713
2880
|
// with the batch; the members that failed are parked, not requeued.
|
|
@@ -2715,19 +2882,19 @@ function evictMemberDirectly(deps, config, batchId, memberIssue, failure, now) {
|
|
|
2715
2882
|
issue: memberIssue,
|
|
2716
2883
|
detail: `eviction threshold reached, but member(s) ${validated.join(',')} are validated on the integration branch — keeping them and continuing with the remaining members instead of dissolving`,
|
|
2717
2884
|
});
|
|
2718
|
-
return
|
|
2885
|
+
return false;
|
|
2719
2886
|
}
|
|
2720
|
-
//
|
|
2721
|
-
//
|
|
2722
|
-
//
|
|
2887
|
+
// Outside the lock — dissolveBatch shells out: re-load fresh (pass 1's
|
|
2888
|
+
// write already landed), dissolve, then re-apply just this batch's + the
|
|
2889
|
+
// requeued members' state under a fresh lock.
|
|
2723
2890
|
const state = deps.store.load();
|
|
2724
2891
|
const batch = (0, state_1.findBatch)(state, batchId);
|
|
2725
2892
|
if (!batch)
|
|
2726
|
-
return
|
|
2893
|
+
return false;
|
|
2727
2894
|
const rDeps = recoveryDeps(deps, config, batch, now);
|
|
2728
2895
|
const outcome = (0, recovery_1.dissolveBatch)(state, batchId, { strategy: 'full', reason: 'eviction-threshold' }, rDeps);
|
|
2729
2896
|
applyDissolveOutcome(deps, batchId, outcome, now);
|
|
2730
|
-
return
|
|
2897
|
+
return true;
|
|
2731
2898
|
}
|
|
2732
2899
|
function reconcileFixSlot(deps, config, batchId, slot, now, result) {
|
|
2733
2900
|
if (slot.pid !== null && deps.spawnDeps.isAlive(slot.pid, slot.pid_start ?? undefined))
|
|
@@ -3853,6 +4020,7 @@ function teardownBatch(deps, batchId) {
|
|
|
3853
4020
|
true;
|
|
3854
4021
|
const serial = teardownMemberWorktree(deps, batchId, deps.now(), lastMemberLanded);
|
|
3855
4022
|
teardownParallelRuns(deps, batchId, serial);
|
|
4023
|
+
deleteMemberBranches(deps, batchId);
|
|
3856
4024
|
const state = deps.store.load();
|
|
3857
4025
|
const batch = (0, state_1.findBatch)(state, batchId);
|
|
3858
4026
|
if (!batch || batch.worktree === null)
|
|
@@ -3876,6 +4044,30 @@ function teardownBatch(deps, batchId) {
|
|
|
3876
4044
|
worktree: batch.worktree,
|
|
3877
4045
|
});
|
|
3878
4046
|
}
|
|
4047
|
+
/**
|
|
4048
|
+
* #840: `teardownBatch`'s member-branch cleanup — `pruneMemberBranches`
|
|
4049
|
+
* (recovery.ts) plus the journal line. Best-effort: a failed listing or delete
|
|
4050
|
+
* is journaled, never thrown into the teardown path; a kept batch
|
|
4051
|
+
* (`worktree_kept`) never gets here.
|
|
4052
|
+
*/
|
|
4053
|
+
function deleteMemberBranches(deps, batchId) {
|
|
4054
|
+
const pruned = (0, recovery_1.pruneMemberBranches)(deps.exec, deps.repoDir, deps.store.load(), batchId);
|
|
4055
|
+
if (pruned === null) {
|
|
4056
|
+
journalEvent(deps, 'teardown-failed', unit(batchId), {
|
|
4057
|
+
cleanup: 'member-branches-unlisted',
|
|
4058
|
+
detail: `could not list origin's batch/${batchId}-m* member branches — none deleted`,
|
|
4059
|
+
});
|
|
4060
|
+
return;
|
|
4061
|
+
}
|
|
4062
|
+
const { deleted, kept, failed } = pruned;
|
|
4063
|
+
if (deleted.length + kept.length + failed.length === 0)
|
|
4064
|
+
return;
|
|
4065
|
+
journalEvent(deps, failed.length === 0 ? 'member-branches-deleted' : 'teardown-failed', unit(batchId), {
|
|
4066
|
+
...(failed.length === 0 ? {} : { cleanup: 'member-branches-delete-failed' }),
|
|
4067
|
+
detail: `deleted=${deleted.join(',') || 'none'} kept=${kept.join(',') || 'none'}` +
|
|
4068
|
+
(failed.length > 0 ? ` failed=${failed.join(',')}` : ''),
|
|
4069
|
+
});
|
|
4070
|
+
}
|
|
3879
4071
|
// --- Parallel member dispatch (#809) ---
|
|
3880
4072
|
/**
|
|
3881
4073
|
* How a batch should dispatch its members (#809) — decided once, at the
|
|
@@ -4054,7 +4246,10 @@ function spawnParallelMembers(deps, config, dispatch, batchId, now, result) {
|
|
|
4054
4246
|
evictMemberDirectly(deps, config, batchId, next.issue, {
|
|
4055
4247
|
reason: `member-worktree-prep-failed:${prep.reason}`,
|
|
4056
4248
|
detail: `member worktree prep failed: ${prep.reason} (member ${next.index} #${next.issue}, branch ${memberBranchFor(batchId, next.index, next.issue)})`,
|
|
4057
|
-
}, now
|
|
4249
|
+
}, now,
|
|
4250
|
+
// #844: a run already recorded (a respawn whose tree vanished) must
|
|
4251
|
+
// not stay `running` once its member is evicted.
|
|
4252
|
+
(s) => patchRun(s, batchId, next.issue, { status: 'evicted', gate_inconclusive: null }, now));
|
|
4058
4253
|
result.failed.push((0, state_1.batchMemberUnit)(batchId, next.issue));
|
|
4059
4254
|
continue;
|
|
4060
4255
|
}
|
|
@@ -4115,17 +4310,19 @@ function spawnParallelMembers(deps, config, dispatch, batchId, now, result) {
|
|
|
4115
4310
|
* evicted work's only recoverable copy, as in serial mode). A dissolve this
|
|
4116
4311
|
* tips over is torn down by `teardownBatch`, which covers every live run.
|
|
4117
4312
|
*/
|
|
4118
|
-
function evictParallelMember(deps, config, batchId, run, failure, now, result
|
|
4119
|
-
|
|
4313
|
+
function evictParallelMember(deps, config, batchId, run, failure, now, result,
|
|
4314
|
+
/** #844: the member's own slot release — committed in the eviction's write. */
|
|
4315
|
+
release) {
|
|
4316
|
+
const { batchEnded, duplicate } = evictMemberDirectly(deps, config, batchId, run.issue, failure, now,
|
|
4317
|
+
// #844: the run's `evicted` status lands in the eviction's write too — a
|
|
4318
|
+
// separate write left an evicted member's run `running` (slotless, its
|
|
4319
|
+
// tree kept until batch end) when the engine exited in between.
|
|
4320
|
+
(s) => patchRun(release ? release(s) : s, batchId, run.issue, { status: 'evicted', gate_inconclusive: null }, now));
|
|
4120
4321
|
if (duplicate)
|
|
4121
4322
|
return;
|
|
4122
4323
|
result.failed.push(batchEnded ? unit(batchId) : (0, state_1.batchMemberUnit)(batchId, run.issue));
|
|
4123
4324
|
if (batchEnded)
|
|
4124
4325
|
return;
|
|
4125
|
-
deps.store.withLock((s) => ({
|
|
4126
|
-
state: patchRun(s, batchId, run.issue, { status: 'evicted', gate_inconclusive: null }, now),
|
|
4127
|
-
result: undefined,
|
|
4128
|
-
}));
|
|
4129
4326
|
teardownRunTree(deps, batchId, run, false, now);
|
|
4130
4327
|
}
|
|
4131
4328
|
/**
|
|
@@ -4156,8 +4353,7 @@ function reconcileParallelMemberSlot(deps, config, dispatch, batchId, run, slot,
|
|
|
4156
4353
|
recordMemberRunLog(deps, dispatch, state0, batchId, run.index, run.issue, slot, now);
|
|
4157
4354
|
const verdict = evaluateIncrementalGate(deps, batchId, run.worktree, run.issue, now);
|
|
4158
4355
|
if (verdict.kind === 'failed') {
|
|
4159
|
-
deps
|
|
4160
|
-
evictParallelMember(deps, config, batchId, run, { reason: verdict.reason, detail: verdict.detail }, now, result);
|
|
4356
|
+
evictParallelMember(deps, config, batchId, run, { reason: verdict.reason, detail: verdict.detail }, now, result, release);
|
|
4161
4357
|
return;
|
|
4162
4358
|
}
|
|
4163
4359
|
deps.store.withLock((s) => ({
|
|
@@ -4178,7 +4374,7 @@ function reconcileParallelMemberSlot(deps, config, dispatch, batchId, run, slot,
|
|
|
4178
4374
|
read.milestone !== null) {
|
|
4179
4375
|
const decision = decideWrongProcedure(deps, batchId, run.issue, read.milestone, slot, release, () => recordMemberRunLog(deps, dispatch, state0, batchId, run.index, run.issue, slot, now), now);
|
|
4180
4376
|
if (typeof decision === 'object') {
|
|
4181
|
-
evictParallelMember(deps, config, batchId, run, decision, now, result);
|
|
4377
|
+
evictParallelMember(deps, config, batchId, run, decision, now, result, release);
|
|
4182
4378
|
}
|
|
4183
4379
|
return;
|
|
4184
4380
|
}
|
|
@@ -4191,8 +4387,7 @@ function reconcileParallelMemberSlot(deps, config, dispatch, batchId, run, slot,
|
|
|
4191
4387
|
}
|
|
4192
4388
|
if (read.blockedNow || read.dead) {
|
|
4193
4389
|
const lastTool = recordMemberRunLog(deps, dispatch, state0, batchId, run.index, run.issue, slot, now);
|
|
4194
|
-
deps
|
|
4195
|
-
evictParallelMember(deps, config, batchId, run, memberFailureFor(read, lastTool), now, result);
|
|
4390
|
+
evictParallelMember(deps, config, batchId, run, memberFailureFor(read, lastTool), now, result, release);
|
|
4196
4391
|
}
|
|
4197
4392
|
}
|
|
4198
4393
|
/**
|
|
@@ -4595,12 +4790,8 @@ function runBatchTick(deps, config, dispatch) {
|
|
|
4595
4790
|
const dissolvable = fresh !== undefined && (0, state_1.allowedBatchTransitions)(fresh.status).includes('dissolving');
|
|
4596
4791
|
if (dissolvable) {
|
|
4597
4792
|
// #809: a parallel batch holds one slot per running member too.
|
|
4598
|
-
for (const liveSlot of (0, state_1.slotsForBatch)(current, batch.id))
|
|
4599
|
-
|
|
4600
|
-
deps.spawnDeps.isAlive(liveSlot.pid, liveSlot.pid_start ?? undefined)) {
|
|
4601
|
-
deps.spawnDeps.kill(liveSlot.pid, liveSlot.pid_start ?? undefined);
|
|
4602
|
-
}
|
|
4603
|
-
}
|
|
4793
|
+
for (const liveSlot of (0, state_1.slotsForBatch)(current, batch.id))
|
|
4794
|
+
killLiveAgent(deps, liveSlot);
|
|
4604
4795
|
const outcome = (0, recovery_1.dissolveBatch)(current, batch.id, {
|
|
4605
4796
|
strategy: 'full',
|
|
4606
4797
|
reason: `dispatch-profile-missing:${batch.dispatch_profile}`,
|