@ai-dossier/sched 0.55.1 → 0.56.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +30 -2
- package/dist/batch-dispatch.d.ts +11 -3
- package/dist/batch-dispatch.d.ts.map +1 -1
- package/dist/batch-dispatch.js +227 -60
- package/dist/batch-dispatch.js.map +1 -1
- package/dist/dispatch.d.ts +20 -2
- package/dist/dispatch.d.ts.map +1 -1
- package/dist/dispatch.js +45 -4
- package/dist/dispatch.js.map +1 -1
- package/dist/index.d.ts +1 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +6 -5
- package/dist/index.js.map +1 -1
- package/dist/state.d.ts +2 -0
- package/dist/state.d.ts.map +1 -1
- package/dist/state.js +10 -2
- package/dist/state.js.map +1 -1
- package/dist/types.d.ts +21 -2
- package/dist/types.d.ts.map +1 -1
- package/dist/types.js +5 -1
- package/dist/types.js.map +1 -1
- package/package.json +1 -1
package/README.md
CHANGED
|
@@ -1097,7 +1097,13 @@ hand-back shape (`blocked`, `review partial`) never counts: it keeps its own pat
|
|
|
1097
1097
|
member rails then:
|
|
1098
1098
|
|
|
1099
1099
|
- stop a live agent and wait for it to be gone before anything else (a dying full-cycle run
|
|
1100
|
-
could still push, post or open a PR in that worktree)
|
|
1100
|
+
could still push, post or open a PR in that worktree). The first sight sends SIGTERM
|
|
1101
|
+
(re-sent each tick), stamps `SlotEntry.kill_sent_at` and journals `member-stop-requested`
|
|
1102
|
+
once; an agent still alive `KILL_ESCALATION_MS` (120 s) later
|
|
1103
|
+
is sent SIGKILL — to its process group when its recorded start time confirms its identity
|
|
1104
|
+
(every agent is spawned detached, so it leads one), else to the pid alone — and
|
|
1105
|
+
`kill-escalated` is journaled once (#844). Pre-#844 an agent ignoring SIGTERM held its
|
|
1106
|
+
member here indefinitely;
|
|
1101
1107
|
- if the milestone shows the run already SHIPPED (a `ship`/`report` phase or a `pr=` key —
|
|
1102
1108
|
`wrongProcedureShippedPr`), evict it at once `wrong-procedure-shipped`, naming the stray PR
|
|
1103
1109
|
(`stray_pr`) for the operator to close or reconcile — a re-prompt cannot undo a PR;
|
|
@@ -1109,6 +1115,21 @@ member rails then:
|
|
|
1109
1115
|
the killed first dispatch's milestones would otherwise still count inside the 60s fence)
|
|
1110
1116
|
evicts it `wrong-procedure` (parked `evicted`, branch kept, #810).
|
|
1111
1117
|
|
|
1118
|
+
### A member's slot release and its eviction are one write (#844)
|
|
1119
|
+
|
|
1120
|
+
Every member-failure rail — serial dead/blocked, the incremental gate's `task-failed`, both
|
|
1121
|
+
wrong-procedure evictions, and their parallel twins — hands the slot release to
|
|
1122
|
+
`evictMemberDirectly`, which applies it inside the same lock that parks the member and
|
|
1123
|
+
appends its `evictions[]` record. Pre-#844 the release was its own earlier write, and an
|
|
1124
|
+
engine exit between the two left the member in-work with no slot: the serial wedge arm or
|
|
1125
|
+
`spawnParallelMembers` then dispatched it once more, with no eviction in the journal. An exit
|
|
1126
|
+
AFTER the combined write but before the serial advance is recovered too: the wedge arm finds
|
|
1127
|
+
the current member already in `evictions[]`, journals `member-advance-recovered` (carrying the
|
|
1128
|
+
record's `reason`/`kind`, since the exit may have beaten the `unit-failed` line), and finishes
|
|
1129
|
+
what the eviction would have done — the dissolve its threshold tripped, or the advance —
|
|
1130
|
+
instead of respawning it. The same one-write rule covers the parallel run's `evicted` status
|
|
1131
|
+
and `sched resume --batch`'s `blocked → executing` transition before a recheck eviction.
|
|
1132
|
+
|
|
1112
1133
|
## API surface
|
|
1113
1134
|
|
|
1114
1135
|
```ts
|
|
@@ -1137,6 +1158,10 @@ import {
|
|
|
1137
1158
|
// truth/clock/repoDir/teardownExec/fencer/batchExec/runBatchSuite/
|
|
1138
1159
|
// runBatchCapability — #523)
|
|
1139
1160
|
createSpawnDeps, // real detached-spawn process I/O
|
|
1161
|
+
KILL_ESCALATION_MS, // #844: SIGTERM → SIGKILL bound (120 s) for an agent the batch
|
|
1162
|
+
// wrong-procedure wait is stopping
|
|
1163
|
+
type KillSignal, // #844: 'SIGTERM' | 'SIGKILL' — SpawnDeps.kill's optional 3rd arg
|
|
1164
|
+
// (SIGKILL targets the agent's process group on confirmed identity)
|
|
1140
1165
|
createExecGroundTruth, // runstate/gh/git ground truth via subprocesses (injectable exec);
|
|
1141
1166
|
// since #468 also gh pr view PR state + setup info from comments;
|
|
1142
1167
|
// since #789 also mergedPrForBranch, and since #824
|
|
@@ -1431,7 +1456,7 @@ after-the-fact recovery, not a missing-data bug.
|
|
|
1431
1456
|
queue data.
|
|
1432
1457
|
- **Schema**: state/config files from #460 (schema 1.0.0), #464 (1.1.0), #468 (1.2.0),
|
|
1433
1458
|
#472 (1.3.0), #500 (1.4.0), #505 (1.5.0), #504 (1.6.0), #523 (1.7.0) and #524 (1.8.0)
|
|
1434
|
-
load and migrate to the current schema (1.
|
|
1459
|
+
load and migrate to the current schema (1.29.0 — 1.24.0 was #810: no backfill,
|
|
1435
1460
|
`kind`/`branch` optional, absent = an `evicted` record with no branch) automatically
|
|
1436
1461
|
(slot `branch`/`last_head`/`pid_start`, slot `role` (inferred from the
|
|
1437
1462
|
unit's queue entry, with the persisted `phase` as a fallback — #500), entry
|
|
@@ -1458,6 +1483,9 @@ after-the-fact recovery, not a missing-data bug.
|
|
|
1458
1483
|
decision, persisted; `false` backfilled) and `MemberRun` gains `teardown_failed_at`
|
|
1459
1484
|
(`null` backfilled). A 1.27.0 `done` batch still carrying a live member run loads as
|
|
1460
1485
|
kept — the old engine never recorded the decision, so the default is to keep.
|
|
1486
|
+
Schema 1.29.0 (#844): `SlotEntry` gains `kill_sent_at`/`kill_escalated_at` (the
|
|
1487
|
+
SIGTERM → SIGKILL escalation anchor and its journal dedup marker; `null`/`null`
|
|
1488
|
+
backfilled, cleared when the slot goes idle).
|
|
1461
1489
|
- **`max_slots`** bounds live units (`assigned | running | recovering`); dependency
|
|
1462
1490
|
edges gate readiness — an issue with an unmerged dependency, and a batch behind an
|
|
1463
1491
|
unmerged batch, are never runnable.
|
package/dist/batch-dispatch.d.ts
CHANGED
|
@@ -176,8 +176,14 @@ export interface MemberFailure {
|
|
|
176
176
|
export declare function memberBranchFor(batchId: string, memberIndex: number, issue: number): string;
|
|
177
177
|
/**
|
|
178
178
|
* Evict the current member and either dissolve, or continue the batch via
|
|
179
|
-
* `advanceMemberOrValidate` — the shared tail of
|
|
180
|
-
* (
|
|
179
|
+
* `advanceMemberOrValidate` — the shared tail of the serial member-failure
|
|
180
|
+
* rails (dead/blocked, the incremental gate below, the wrong-procedure
|
|
181
|
+
* evictions, and `sched resume --batch`'s recheck).
|
|
182
|
+
*
|
|
183
|
+
* #844: a caller whose member still holds its slot MUST pass its release as
|
|
184
|
+
* `preWrite` — never release under a separate lock first. `evictMemberDirectly` commits
|
|
185
|
+
* the release with the park/eviction record in one write, so an engine exit
|
|
186
|
+
* cannot leave the member slotless and un-evicted (and so respawned).
|
|
181
187
|
*
|
|
182
188
|
* #613: `evictMemberDirectly`'s duplicate check (#595) is the one atomic,
|
|
183
189
|
* lock-protected claim on "did THIS member's eviction already happen" — so
|
|
@@ -199,7 +205,9 @@ export declare function memberBranchFor(batchId: string, memberIndex: number, is
|
|
|
199
205
|
* `runBatchTick`/`resumeBlockedGate` entry points cannot reproduce since
|
|
200
206
|
* both always read state fresh.
|
|
201
207
|
*/
|
|
202
|
-
export declare function evictMemberAndContinue(deps: BatchDispatchDeps, config: SchedConfig, dispatch: ResolvedDispatch, batchId: string, batch: BatchEntry, memberIssue: number, failure: MemberFailure, now: Date, result: BatchTickResult
|
|
208
|
+
export declare function evictMemberAndContinue(deps: BatchDispatchDeps, config: SchedConfig, dispatch: ResolvedDispatch, batchId: string, batch: BatchEntry, memberIssue: number, failure: MemberFailure, now: Date, result: BatchTickResult,
|
|
209
|
+
/** #844: the member's slot release (or any pre-write) — committed in the eviction's own write. */
|
|
210
|
+
preWrite?: (s: SchedState) => SchedState): void;
|
|
203
211
|
/**
|
|
204
212
|
* `sched resume --batch <id>` (#583 AC4): an operator-triggered, synchronous
|
|
205
213
|
* one-shot recheck of a batch blocked on `gate-inconclusive:<capabilityId>`
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"batch-dispatch.d.ts","sourceRoot":"","sources":["../src/batch-dispatch.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA8DG;AAUH,OAAO,EAIL,KAAK,YAAY,EAGjB,KAAK,gBAAgB,EAGtB,MAAM,gBAAgB,CAAC;AAUxB,OAAO,
|
|
1
|
+
{"version":3,"file":"batch-dispatch.d.ts","sourceRoot":"","sources":["../src/batch-dispatch.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA8DG;AAUH,OAAO,EAIL,KAAK,YAAY,EAGjB,KAAK,gBAAgB,EAGtB,MAAM,gBAAgB,CAAC;AAUxB,OAAO,EAcL,KAAK,gBAAgB,EAGrB,KAAK,SAAS,EAEf,MAAM,YAAY,CAAC;AAEpB,OAAO,EAGL,KAAK,WAAW,EAcjB,MAAM,eAAe,CAAC;AACvB,OAAO,EAAE,KAAK,OAAO,EAAa,MAAM,WAAW,CAAC;AACpD,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,WAAW,CAAC;AAC5C,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,WAAW,CAAC;AAExC,OAAO,EACL,KAAK,iBAAiB,EAYtB,KAAK,WAAW,EACjB,MAAM,YAAY,CAAC;AAGpB,OAAO,EAyBL,kBAAkB,EAKnB,MAAM,SAAS,CAAC;AACjB,OAAO,EACL,KAAK,QAAQ,EAMd,MAAM,YAAY,CAAC;AACpB,OAAO,KAAK,EAGV,UAAU,EAEV,oBAAoB,EAKpB,kBAAkB,EAClB,cAAc,EAId,WAAW,EACX,UAAU,EAEX,MAAM,SAAS,CAAC;AAcjB,OAAO,EAAE,kBAAkB,EAAE,CAAC;AAK9B,YAAY,EAAE,UAAU,EAAE,MAAM,SAAS,CAAC;AAE1C,8DAA8D;AAC9D,MAAM,WAAW,iBAAiB;IAChC,KAAK,EAAE,UAAU,CAAC;IAClB,OAAO,EAAE,OAAO,CAAC;IACjB,WAAW,EAAE,WAAW,CAAC;IACzB,SAAS,EAAE,SAAS,CAAC;IACrB,GAAG,EAAE,MAAM,IAAI,CAAC;IAChB,uFAAuF;IACvF,OAAO,EAAE,MAAM,CAAC;IAChB,0FAA0F;IAC1F,IAAI,EAAE,MAAM,CAAC;IACb;;;;;;OAMG;IACH,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB;;;;;OAKG;IACH,QAAQ,EAAE,CAAC,QAAQ,EAAE,MAAM,EAAE,GAAG,CAAC,EAAE,iBAAiB,KAAK,WAAW,CAAC;IACrE;;;;;;;;;;;;;;;;;;;OAmBG;IACH,aAAa,CAAC,EAAE,CAAC,QAAQ,EAAE,MAAM,EAAE,YAAY,EAAE,MAAM,KAAK,oBAAoB,CAAC;IACjF,QAAQ,CAAC,EAAE,QAAQ,CAAC;IACpB;;;OAGG;IACH,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB;;;;;;OAMG;IACH,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB;AAED,8FAA8F;AAC9F,MAAM,WAAW,eAAe;IAC9B,OAAO,EAAE,MAAM,EAAE,CAAC;IAClB,SAAS,EAAE,MAAM,EAAE,CAAC;IACpB,MAAM,EAAE,MAAM,EAAE,CAAC;IACjB,aAAa,EAAE,MAAM,EAAE,CAAC;IACxB,MAAM,EAAE,MAAM,EAAE,CAAC;IACjB,uFAAuF;IACvF,qBAAqB,EAAE,MAAM,EAAE,CAAC;IAChC,8FAA8F;IAC9F,OAAO,EAAE,MAAM,EAAE,CAAC;CACnB;AAED;;;;;;GAMG;AACH,MAAM,WAAW,aAAa;IAC5B,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACjC;;;;OAIG;IACH,IAAI,CAAC,EAAE,cAAc,CAAC;CACvB;AAmmBD;;;;;;;GAOG;AACH,wBAAgB,eAAe,CAAC,OAAO,EAAE,MAAM,EAAE,WAAW,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,MAAM,CAE3F;AA0gDD;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA8BG;AACH,wBAAgB,sBAAsB,CACpC,IAAI,EAAE,iBAAiB,EACvB,MAAM,EAAE,WAAW,EACnB,QAAQ,EAAE,gBAAgB,EAC1B,OAAO,EAAE,MAAM,EACf,KAAK,EAAE,UAAU,EACjB,WAAW,EAAE,MAAM,EACnB,OAAO,EAAE,aAAa,EACtB,GAAG,EAAE,IAAI,EACT,MAAM,EAAE,eAAe;AACvB,kGAAkG;AAClG,QAAQ,CAAC,EAAE,CAAC,CAAC,EAAE,UAAU,KAAK,UAAU,GACvC,IAAI,CAgBN;AAoWD;;;;;;;;;;;;;;;;;;;;;;;;;;;GA2BG;AACH,wBAAgB,iBAAiB,CAC/B,IAAI,EAAE,iBAAiB,EACvB,MAAM,EAAE,WAAW,EACnB,QAAQ,EAAE,gBAAgB,EAC1B,OAAO,EAAE,MAAM,EACf,GAAG,EAAE,IAAI,GACR;IACD,OAAO,EAAE,eAAe,GAAG,SAAS,GAAG,WAAW,GAAG,SAAS,CAAC;IAC/D,UAAU,EAAE,MAAM,CAAC;IACnB;;;;;OAKG;IACH,SAAS,CAAC,EAAE,cAAc,GAAG,qBAAqB,CAAC;IACnD,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB,CA+KA;AAw5BD;;;;;;;;GAQG;AACH,eAAO,MAAM,wBAAwB,IAAI,CAAC;AAuf1C;;;;;;GAMG;AACH,wBAAgB,4BAA4B,CAAC,KAAK,EAAE,UAAU,EAAE,KAAK,EAAE,UAAU,GAAG,MAAM,EAAE,CAuB3F;AA6HD;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAwCG;AACH,wBAAgB,4BAA4B,CAC1C,IAAI,EAAE,iBAAiB,EACvB,GAAG,EAAE,IAAI,EACT,MAAM,EAAE,eAAe,EACvB,SAAS,EAAE,iBAAiB,GAAG,SAAS,GACvC,IAAI,CAmMN;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAyCG;AACH,wBAAgB,sBAAsB,CAAC,IAAI,EAAE,iBAAiB,EAAE,GAAG,EAAE,IAAI,GAAG,IAAI,CAuG/E;AAED;;;;;;GAMG;AACH,UAAU,iBAAiB;IACzB,IAAI,EAAE,MAAM,CAAC;IACb,gDAAgD;IAChD,IAAI,EAAE,gBAAgB,CAAC;IACvB,uEAAuE;IACvE,SAAS,EAAE,gBAAgB,CAAC;IAC5B,YAAY,EAAE,YAAY,CAAC;CAC5B;AAoRD;;;;;;;;;;;GAWG;AACH,oCAAoC;AACpC,wBAAgB,qBAAqB,CACnC,KAAK,EAAE,UAAU,EACjB,KAAK,EAAE,UAAU,EACjB,MAAM,EAAE,WAAW,GAClB,kBAAkB,CAQpB;AAs4BD;;;;;;;;;;;;;GAaG;AACH,wBAAgB,YAAY,CAC1B,IAAI,EAAE,iBAAiB,EACvB,MAAM,EAAE,WAAW,EACnB,QAAQ,EAAE,gBAAgB,GACzB,eAAe,CAgSjB"}
|
package/dist/batch-dispatch.js
CHANGED
|
@@ -164,9 +164,18 @@ function sanitizeUntrustedText(value) {
|
|
|
164
164
|
function journalEvent(deps, event, unitId, extra = {}) {
|
|
165
165
|
deps.journal.append((0, journal_1.unitEvent)(event, unitId, extra), deps.now());
|
|
166
166
|
}
|
|
167
|
-
/**
|
|
167
|
+
/** Whether the slot's agent is still alive (pid-start-safe: a reused pid reads dead). */
|
|
168
|
+
function agentAlive(deps, slot) {
|
|
169
|
+
return slot.pid !== null && deps.spawnDeps.isAlive(slot.pid, slot.pid_start ?? undefined);
|
|
170
|
+
}
|
|
171
|
+
/**
|
|
172
|
+
* Stop the slot's agent if it is still alive (pid-start-safe: a reused pid is
|
|
173
|
+
* never signalled). SIGTERM only — no wait, no escalation. A decision that
|
|
174
|
+
* must not run alongside the agent uses `stopAgentBeforeDeciding` instead,
|
|
175
|
+
* which escalates to SIGKILL (#844).
|
|
176
|
+
*/
|
|
168
177
|
function killLiveAgent(deps, slot) {
|
|
169
|
-
if (
|
|
178
|
+
if (agentAlive(deps, slot)) {
|
|
170
179
|
deps.spawnDeps.kill(slot.pid, slot.pid_start ?? undefined);
|
|
171
180
|
}
|
|
172
181
|
}
|
|
@@ -1285,6 +1294,9 @@ function spawnMemberContinuation(deps, config, dispatch, batchId, now, result) {
|
|
|
1285
1294
|
// contract `prepareMemberWorktree` documents). The wedge arm of
|
|
1286
1295
|
// `runBatchTick` calls this EVERY tick while the batch is `executing`
|
|
1287
1296
|
// with no slot, so the cheap gates run first:
|
|
1297
|
+
// - recovery (#844): a current member already in `evictions[]` is advanced
|
|
1298
|
+
// past, never respawned — checked before capacity, since the advance
|
|
1299
|
+
// needs no slot.
|
|
1288
1300
|
// - capacity: prep before the free-capacity check would do pool/git/npm
|
|
1289
1301
|
// work (and journal) per tick against a full scheduler — the #610/#630/
|
|
1290
1302
|
// #632 per-tick emission trap. `claimAndSpawn` re-checks under the lock;
|
|
@@ -1301,6 +1313,30 @@ function spawnMemberContinuation(deps, config, dispatch, batchId, now, result) {
|
|
|
1301
1313
|
const memberIssue = batch.members[batch.executing_member - 1];
|
|
1302
1314
|
if (memberIssue === undefined)
|
|
1303
1315
|
return;
|
|
1316
|
+
const evicted = batch.evictions.find((e) => e.issue === memberIssue);
|
|
1317
|
+
if (evicted !== undefined) {
|
|
1318
|
+
// #844: the member's eviction (and its slot release, in the same write)
|
|
1319
|
+
// landed, but the engine died before advancing past it — possibly before
|
|
1320
|
+
// journaling its cause, too, so the record's cause goes on this line.
|
|
1321
|
+
// Respawning it would dispatch an evicted member again — finish what the
|
|
1322
|
+
// eviction would have done instead: the dissolve its threshold tripped,
|
|
1323
|
+
// or the advance.
|
|
1324
|
+
const kind = evicted.kind ?? 'evicted';
|
|
1325
|
+
journalEvent(deps, 'member-advance-recovered', unit(batchId), {
|
|
1326
|
+
issue: memberIssue,
|
|
1327
|
+
reason: evicted.reason,
|
|
1328
|
+
kind,
|
|
1329
|
+
evicted_at: evicted.at,
|
|
1330
|
+
detail: `member ${batch.executing_member} is already parked ${kind} (${evicted.reason}) but the batch never advanced past it (engine exit mid-eviction) — continuing instead of respawning it`,
|
|
1331
|
+
});
|
|
1332
|
+
if ((0, recovery_1.checkDissolveTrigger)(batch, (0, types_1.resolveDissolvePolicy)(config.dissolve_policy)) &&
|
|
1333
|
+
settleEvictionThreshold(deps, config, batchId, memberIssue, (0, state_1.validatedMembersOf)(state, batch), now)) {
|
|
1334
|
+
result.failed.push(unit(batchId));
|
|
1335
|
+
return;
|
|
1336
|
+
}
|
|
1337
|
+
continuePastEvictedMember(deps, config, dispatch, batchId, batch, memberIssue, now, result);
|
|
1338
|
+
return;
|
|
1339
|
+
}
|
|
1304
1340
|
if ((0, scheduler_1.freeCapacity)(state, config) === 0)
|
|
1305
1341
|
return;
|
|
1306
1342
|
const expectedBranch = memberBranchFor(batchId, batch.executing_member, memberIssue);
|
|
@@ -1772,8 +1808,14 @@ serialTeardown) {
|
|
|
1772
1808
|
}
|
|
1773
1809
|
/**
|
|
1774
1810
|
* Evict the current member and either dissolve, or continue the batch via
|
|
1775
|
-
* `advanceMemberOrValidate` — the shared tail of
|
|
1776
|
-
* (
|
|
1811
|
+
* `advanceMemberOrValidate` — the shared tail of the serial member-failure
|
|
1812
|
+
* rails (dead/blocked, the incremental gate below, the wrong-procedure
|
|
1813
|
+
* evictions, and `sched resume --batch`'s recheck).
|
|
1814
|
+
*
|
|
1815
|
+
* #844: a caller whose member still holds its slot MUST pass its release as
|
|
1816
|
+
* `preWrite` — never release under a separate lock first. `evictMemberDirectly` commits
|
|
1817
|
+
* the release with the park/eviction record in one write, so an engine exit
|
|
1818
|
+
* cannot leave the member slotless and un-evicted (and so respawned).
|
|
1777
1819
|
*
|
|
1778
1820
|
* #613: `evictMemberDirectly`'s duplicate check (#595) is the one atomic,
|
|
1779
1821
|
* lock-protected claim on "did THIS member's eviction already happen" — so
|
|
@@ -1795,14 +1837,27 @@ serialTeardown) {
|
|
|
1795
1837
|
* `runBatchTick`/`resumeBlockedGate` entry points cannot reproduce since
|
|
1796
1838
|
* both always read state fresh.
|
|
1797
1839
|
*/
|
|
1798
|
-
function evictMemberAndContinue(deps, config, dispatch, batchId, batch, memberIssue, failure, now, result
|
|
1799
|
-
|
|
1840
|
+
function evictMemberAndContinue(deps, config, dispatch, batchId, batch, memberIssue, failure, now, result,
|
|
1841
|
+
/** #844: the member's slot release (or any pre-write) — committed in the eviction's own write. */
|
|
1842
|
+
preWrite) {
|
|
1843
|
+
const { batchEnded, duplicate } = evictMemberDirectly(deps, config, batchId, memberIssue, failure, now, preWrite);
|
|
1800
1844
|
if (duplicate)
|
|
1801
1845
|
return;
|
|
1802
1846
|
if (batchEnded) {
|
|
1803
1847
|
result.failed.push(unit(batchId));
|
|
1804
1848
|
return;
|
|
1805
1849
|
}
|
|
1850
|
+
continuePastEvictedMember(deps, config, dispatch, batchId, batch, memberIssue, now, result);
|
|
1851
|
+
}
|
|
1852
|
+
/**
|
|
1853
|
+
* The serial rail after a member's eviction has been recorded: tear its tree
|
|
1854
|
+
* down and advance. Also the crash recovery for an engine that died between
|
|
1855
|
+
* the eviction's write and this advance (#844) — `spawnMemberContinuation`
|
|
1856
|
+
* finds the batch still pointing at a member already in `evictions[]` and
|
|
1857
|
+
* continues from here instead of respawning it. `advanceMemberOrValidate`'s
|
|
1858
|
+
* one-shot claim (#613) keeps a re-entry from advancing twice.
|
|
1859
|
+
*/
|
|
1860
|
+
function continuePastEvictedMember(deps, config, dispatch, batchId, batch, memberIssue, now, result) {
|
|
1806
1861
|
// #677: the evicted member's commits never landed (eviction is the
|
|
1807
1862
|
// pre-landing rail), so there is nothing to revert on the integration
|
|
1808
1863
|
// branch — but its worktree/branch must go, or the next tick's prep for
|
|
@@ -1842,7 +1897,9 @@ function evictMemberAndContinue(deps, config, dispatch, batchId, batch, memberIs
|
|
|
1842
1897
|
* configured or both checks came back `ok`, meaning the caller should treat
|
|
1843
1898
|
* the member as genuinely complete.
|
|
1844
1899
|
*/
|
|
1845
|
-
function runIncrementalGate(deps, config, dispatch, batchId, batch, memberIssue, now, result
|
|
1900
|
+
function runIncrementalGate(deps, config, dispatch, batchId, batch, memberIssue, now, result,
|
|
1901
|
+
/** The member's slot release — handed to the eviction on `failed` (#844). */
|
|
1902
|
+
release) {
|
|
1846
1903
|
if (batch.worktree === null || !deps.runCapability)
|
|
1847
1904
|
return false;
|
|
1848
1905
|
// #677: the gate judges the member's OWN work — pre-landing, its diff
|
|
@@ -1854,11 +1911,10 @@ function runIncrementalGate(deps, config, dispatch, batchId, batch, memberIssue,
|
|
|
1854
1911
|
if (verdict.kind === 'pass')
|
|
1855
1912
|
return false;
|
|
1856
1913
|
if (verdict.kind === 'failed') {
|
|
1857
|
-
deps
|
|
1858
|
-
evictMemberAndContinue(deps, config, dispatch, batchId, batch, memberIssue, { reason: verdict.reason, detail: verdict.detail }, now, result);
|
|
1914
|
+
evictMemberAndContinue(deps, config, dispatch, batchId, batch, memberIssue, { reason: verdict.reason, detail: verdict.detail }, now, result, release);
|
|
1859
1915
|
return true;
|
|
1860
1916
|
}
|
|
1861
|
-
deps.store.withLock((s) => ({ state:
|
|
1917
|
+
deps.store.withLock((s) => ({ state: release(s), result: undefined }));
|
|
1862
1918
|
// #677: F.11's block semantics are "the member's commit stays on the
|
|
1863
1919
|
// branch, nothing requeued/reverted" — under member worktrees that is
|
|
1864
1920
|
// true only if the member LANDS before the batch blocks. The block means
|
|
@@ -2176,19 +2232,21 @@ function resumeBlockedGate(deps, config, dispatch, batchId, now) {
|
|
|
2176
2232
|
detail: excerpt,
|
|
2177
2233
|
};
|
|
2178
2234
|
}
|
|
2179
|
-
|
|
2180
|
-
state: (0, state_1.patchBatch)((0, state_1.transitionBatch)(s, batchId, 'executing', {}, now), batchId, { blocked_reason: null }, now),
|
|
2181
|
-
result: undefined,
|
|
2182
|
-
}));
|
|
2235
|
+
const unblock = (s) => (0, state_1.patchBatch)((0, state_1.transitionBatch)(s, batchId, 'executing', {}, now), batchId, { blocked_reason: null }, now);
|
|
2183
2236
|
if (recheck.outcome === 'task-failed') {
|
|
2184
2237
|
const reason = `incremental-gate-failed:${capabilityId}`;
|
|
2185
2238
|
writeGateLog(deps, batchId, capabilityId, memberIssue, recheck.outputTail);
|
|
2186
2239
|
evictMemberAndContinue(deps, config, batchDispatch, batchId, batch, memberIssue, {
|
|
2187
2240
|
reason,
|
|
2188
2241
|
detail: withExcerpt(`sched resume --batch: cap run ${capabilityId} reported task-failed on recheck`, excerpt),
|
|
2189
|
-
}, now, result
|
|
2242
|
+
}, now, result,
|
|
2243
|
+
// #844: the unblock commits with the eviction — two writes left a
|
|
2244
|
+
// slotless `executing` batch whose un-evicted member the wedge arm
|
|
2245
|
+
// then respawned.
|
|
2246
|
+
unblock);
|
|
2190
2247
|
return { outcome: 'evicted', capability: capabilityId, detail: excerpt };
|
|
2191
2248
|
}
|
|
2249
|
+
deps.store.withLock((s) => ({ state: unblock(s), result: undefined }));
|
|
2192
2250
|
journalEvent(deps, 'external-advance', unit(batchId), {
|
|
2193
2251
|
issue: memberIssue,
|
|
2194
2252
|
detail: `sched resume --batch: cap run ${capabilityId} reported ok on recheck`,
|
|
@@ -2389,6 +2447,84 @@ function wrongProcedureCounts(batch, memberIssue, read) {
|
|
|
2389
2447
|
const repromptedAt = Date.parse(prior.reprompted_at);
|
|
2390
2448
|
return Number.isNaN(at) || Number.isNaN(repromptedAt) ? false : at > repromptedAt;
|
|
2391
2449
|
}
|
|
2450
|
+
/**
|
|
2451
|
+
* A kill timestamp that belongs to the slot's CURRENT dispatch, or null
|
|
2452
|
+
* (#844): one stamped before `spawned_at` was for an earlier dispatch in the
|
|
2453
|
+
* same slot, and an unparseable or future one is no anchor at all.
|
|
2454
|
+
*/
|
|
2455
|
+
function currentDispatchStamp(stamp, spawnedAt, now) {
|
|
2456
|
+
if (stamp === null)
|
|
2457
|
+
return null;
|
|
2458
|
+
const at = Date.parse(stamp);
|
|
2459
|
+
if (Number.isNaN(at))
|
|
2460
|
+
return null;
|
|
2461
|
+
if (spawnedAt !== null && at < Date.parse(spawnedAt))
|
|
2462
|
+
return null;
|
|
2463
|
+
// A future stamp (clock step, hand edit) would make the elapsed time
|
|
2464
|
+
// negative and switch the escalation off for good — re-stamp instead.
|
|
2465
|
+
if (at > now.getTime())
|
|
2466
|
+
return null;
|
|
2467
|
+
return stamp;
|
|
2468
|
+
}
|
|
2469
|
+
/**
|
|
2470
|
+
* Stop the slot's agent before a decision that must not run alongside it
|
|
2471
|
+
* (#844). Returns `true` while it is still alive — the caller waits and
|
|
2472
|
+
* decides on a later tick — and `false` once it is gone (or never had a pid).
|
|
2473
|
+
*
|
|
2474
|
+
* The first sight sends SIGTERM, stamps `kill_sent_at` and journals
|
|
2475
|
+
* `member-stop-requested` once; each later tick
|
|
2476
|
+
* re-sends SIGTERM until the agent has outlived it by `KILL_ESCALATION_MS`,
|
|
2477
|
+
* then sends SIGKILL (to its process group — `SpawnDeps.kill`) and journals
|
|
2478
|
+
* `kill-escalated` once (`kill_escalated_at`). Pre-#844 this path re-sent
|
|
2479
|
+
* SIGTERM forever, so an agent ignoring it held its member indefinitely.
|
|
2480
|
+
*/
|
|
2481
|
+
function stopAgentBeforeDeciding(deps, batchId, memberIssue, slot, now) {
|
|
2482
|
+
if (!agentAlive(deps, slot))
|
|
2483
|
+
return false;
|
|
2484
|
+
const pid = slot.pid;
|
|
2485
|
+
const pidStart = slot.pid_start ?? undefined;
|
|
2486
|
+
const sentAt = currentDispatchStamp(slot.kill_sent_at, slot.spawned_at, now);
|
|
2487
|
+
const aliveForMs = sentAt === null ? 0 : now.getTime() - Date.parse(sentAt);
|
|
2488
|
+
const escalate = sentAt !== null && aliveForMs >= dispatch_1.KILL_ESCALATION_MS;
|
|
2489
|
+
// `kill` refuses a pid whose recorded identity no longer matches (reused by
|
|
2490
|
+
// another process): the agent we spawned is gone, so stop waiting on it.
|
|
2491
|
+
if (!deps.spawnDeps.kill(pid, pidStart, escalate ? 'SIGKILL' : 'SIGTERM'))
|
|
2492
|
+
return false;
|
|
2493
|
+
const firstEscalation = escalate && currentDispatchStamp(slot.kill_escalated_at, slot.spawned_at, now) === null;
|
|
2494
|
+
if (sentAt === null || firstEscalation) {
|
|
2495
|
+
deps.store.withLock((s) => {
|
|
2496
|
+
const live = s.slots.find((x) => x.id === slot.id);
|
|
2497
|
+
// Only the slot still holding THIS agent — never a later dispatch's.
|
|
2498
|
+
if (!live || live.unit !== slot.unit || live.pid !== pid) {
|
|
2499
|
+
return { state: s, result: undefined };
|
|
2500
|
+
}
|
|
2501
|
+
const stamp = now.toISOString();
|
|
2502
|
+
return {
|
|
2503
|
+
state: (0, state_1.patchSlot)(s, slot.id, sentAt === null
|
|
2504
|
+
? { kill_sent_at: stamp, kill_escalated_at: null }
|
|
2505
|
+
: { kill_escalated_at: stamp }, now),
|
|
2506
|
+
result: undefined,
|
|
2507
|
+
};
|
|
2508
|
+
});
|
|
2509
|
+
}
|
|
2510
|
+
if (sentAt === null) {
|
|
2511
|
+
journalEvent(deps, 'member-stop-requested', unit(batchId), {
|
|
2512
|
+
issue: memberIssue,
|
|
2513
|
+
pid,
|
|
2514
|
+
slot: slot.id,
|
|
2515
|
+
detail: `SIGTERM sent — waiting for the agent to exit before deciding (SIGKILL after ${dispatch_1.KILL_ESCALATION_MS / 1000}s)`,
|
|
2516
|
+
});
|
|
2517
|
+
}
|
|
2518
|
+
if (firstEscalation) {
|
|
2519
|
+
journalEvent(deps, 'kill-escalated', unit(batchId), {
|
|
2520
|
+
issue: memberIssue,
|
|
2521
|
+
pid,
|
|
2522
|
+
slot: slot.id,
|
|
2523
|
+
detail: `agent still alive ${Math.round(aliveForMs / 1000)}s after SIGTERM (bound ${dispatch_1.KILL_ESCALATION_MS / 1000}s) — sent SIGKILL (to its process group where available)`,
|
|
2524
|
+
});
|
|
2525
|
+
}
|
|
2526
|
+
return true;
|
|
2527
|
+
}
|
|
2392
2528
|
/**
|
|
2393
2529
|
* #822 (#810 proposal 3): the member's dispatch ran the wrong procedure (a
|
|
2394
2530
|
* full-cycle-shaped trail — imboard #4174 ran full-cycle as a batch member).
|
|
@@ -2398,23 +2534,27 @@ function wrongProcedureCounts(batch, memberIssue, read) {
|
|
|
2398
2534
|
* `reprompted_members` — the wedge arm (serial) or `spawnParallelMembers`
|
|
2399
2535
|
* (parallel) respawns it in place, in the same worktree, with
|
|
2400
2536
|
* `wrongProcedureDirective` appended. Returns `'evict'` when the member was
|
|
2401
|
-
* already re-prompted once: the caller evicts it `wrong-procedure`.
|
|
2537
|
+
* already re-prompted once: the caller evicts it `wrong-procedure`. On
|
|
2538
|
+
* `'evict'` the slot is NOT released here — the caller passes `release` to
|
|
2539
|
+
* `evictMemberAndContinue`/`evictParallelMember`, which commits it in the
|
|
2540
|
+
* eviction's own write (#844).
|
|
2402
2541
|
*/
|
|
2403
2542
|
function repromptWrongProcedure(deps, batchId, memberIssue, milestoneAt, slot, release, now) {
|
|
2404
2543
|
// Never respawn into (or release) a worktree the old agent is still in: a
|
|
2405
2544
|
// dying full-cycle run could still push, post, or open a PR. Signal it and
|
|
2406
2545
|
// decide on a later tick, once it is gone — and so the re-prompt's
|
|
2407
2546
|
// `reprompted_at` fence postdates everything it could still post.
|
|
2408
|
-
if (
|
|
2409
|
-
deps.spawnDeps.kill(slot.pid, slot.pid_start ?? undefined);
|
|
2547
|
+
if (stopAgentBeforeDeciding(deps, batchId, memberIssue, slot, now))
|
|
2410
2548
|
return 'waiting';
|
|
2411
|
-
}
|
|
2412
2549
|
const outcome = deps.store.withLock((s) => {
|
|
2413
|
-
const
|
|
2414
|
-
const b = (0, state_1.findBatch)(released, batchId);
|
|
2550
|
+
const b = (0, state_1.findBatch)(s, batchId);
|
|
2415
2551
|
if (!b || b.reprompted_members.some((r) => r.issue === memberIssue)) {
|
|
2416
|
-
|
|
2552
|
+
// #844: NOT released here — the caller's eviction commits the release
|
|
2553
|
+
// in its own write, so a crash in between cannot leave the member
|
|
2554
|
+
// slotless and un-evicted (and so respawned).
|
|
2555
|
+
return { state: s, result: 'evict' };
|
|
2417
2556
|
}
|
|
2557
|
+
const released = release(s);
|
|
2418
2558
|
return {
|
|
2419
2559
|
state: (0, state_1.patchBatch)(released, batchId, {
|
|
2420
2560
|
reprompted_members: [
|
|
@@ -2455,18 +2595,16 @@ function wrongProcedureFailure(lastTool, shippedPr = null) {
|
|
|
2455
2595
|
* #822: the one wrong-procedure decision shared by both member rails —
|
|
2456
2596
|
* `'waiting'` (the old agent is still dying), `'reprompted'`, or the
|
|
2457
2597
|
* eviction failure to apply. A run that already shipped is evicted without
|
|
2458
|
-
* a re-prompt.
|
|
2598
|
+
* a re-prompt. On an eviction the slot is still held: the caller passes
|
|
2599
|
+
* `release` to the eviction, which commits both in one write (#844).
|
|
2459
2600
|
*/
|
|
2460
2601
|
function decideWrongProcedure(deps, batchId, memberIssue, milestone, slot, release, lastTool, now) {
|
|
2461
2602
|
const shipped = (0, groundtruth_1.wrongProcedureShippedPr)(milestone);
|
|
2462
2603
|
if (shipped !== null) {
|
|
2463
|
-
if (
|
|
2464
|
-
deps.spawnDeps.kill(slot.pid, slot.pid_start ?? undefined);
|
|
2604
|
+
if (stopAgentBeforeDeciding(deps, batchId, memberIssue, slot, now))
|
|
2465
2605
|
return 'waiting';
|
|
2466
|
-
|
|
2467
|
-
|
|
2468
|
-
deps.store.withLock((s) => ({ state: release(s), result: undefined }));
|
|
2469
|
-
return wrongProcedureFailure(tool, shipped);
|
|
2606
|
+
// #844: the caller's eviction releases the slot in its own write.
|
|
2607
|
+
return wrongProcedureFailure(lastTool(), shipped);
|
|
2470
2608
|
}
|
|
2471
2609
|
const verdict = repromptWrongProcedure(deps, batchId, memberIssue, milestone.at, slot, release, now);
|
|
2472
2610
|
if (verdict === 'waiting')
|
|
@@ -2521,6 +2659,9 @@ function reconcileMemberSlot(deps, config, dispatch, batchId, slot, now, result)
|
|
|
2521
2659
|
const memberIssue = batch.members[batch.executing_member - 1];
|
|
2522
2660
|
if (memberIssue === undefined)
|
|
2523
2661
|
return;
|
|
2662
|
+
// #844: every rail below that evicts hands THIS to the eviction, which
|
|
2663
|
+
// commits it in the same write — never released under a lock of its own.
|
|
2664
|
+
const release = (s) => releaseSlot(s, batchId, now);
|
|
2524
2665
|
const read = readMemberDispatch(deps, batchId, memberIssue, slot, now);
|
|
2525
2666
|
if (read.kind === 'unreachable')
|
|
2526
2667
|
return;
|
|
@@ -2538,7 +2679,7 @@ function reconcileMemberSlot(deps, config, dispatch, batchId, slot, now, result)
|
|
|
2538
2679
|
// gate already decided the member's fate (evicted or blocked) and
|
|
2539
2680
|
// returned early; `false` means both checks were `ok` and the member is
|
|
2540
2681
|
// genuinely done.
|
|
2541
|
-
if (runIncrementalGate(deps, config, dispatch, batchId, batch, memberIssue, now, result)) {
|
|
2682
|
+
if (runIncrementalGate(deps, config, dispatch, batchId, batch, memberIssue, now, result, release)) {
|
|
2542
2683
|
return;
|
|
2543
2684
|
}
|
|
2544
2685
|
completeMemberGate(deps, config, dispatch, batchId, batch, memberIssue, now, result);
|
|
@@ -2548,9 +2689,9 @@ function reconcileMemberSlot(deps, config, dispatch, batchId, slot, now, result)
|
|
|
2548
2689
|
// before the dead/blocked rail, which would otherwise evict it as an
|
|
2549
2690
|
// unverified exit at the first sight.
|
|
2550
2691
|
if (wrongProcedureCounts(batch, memberIssue, read) && read.milestone !== null) {
|
|
2551
|
-
const decision = decideWrongProcedure(deps, batchId, memberIssue, read.milestone, slot,
|
|
2692
|
+
const decision = decideWrongProcedure(deps, batchId, memberIssue, read.milestone, slot, release, () => recordMemberRunLog(deps, dispatch, state0, batchId, batch.executing_member, memberIssue, slot, now), now);
|
|
2552
2693
|
if (typeof decision === 'object') {
|
|
2553
|
-
evictMemberAndContinue(deps, config, dispatch, batchId, batch, memberIssue, decision, now, result);
|
|
2694
|
+
evictMemberAndContinue(deps, config, dispatch, batchId, batch, memberIssue, decision, now, result, release);
|
|
2554
2695
|
}
|
|
2555
2696
|
return;
|
|
2556
2697
|
}
|
|
@@ -2566,7 +2707,6 @@ function reconcileMemberSlot(deps, config, dispatch, batchId, slot, now, result)
|
|
|
2566
2707
|
}
|
|
2567
2708
|
if (read.blockedNow || read.dead) {
|
|
2568
2709
|
const lastTool = recordMemberRunLog(deps, dispatch, state0, batchId, batch.executing_member, memberIssue, slot, now);
|
|
2569
|
-
deps.store.withLock((s) => ({ state: releaseSlot(s, batchId, now), result: undefined }));
|
|
2570
2710
|
// A member that never went green (RFC F.1) evicts DIRECTLY — no aggregate
|
|
2571
2711
|
// suite has run yet, so there is nothing for `attributing`/`evicting` (the
|
|
2572
2712
|
// AGGREGATE-suite-red pipeline, RFC F.2) to attribute or revert: the
|
|
@@ -2575,7 +2715,9 @@ function reconcileMemberSlot(deps, config, dispatch, batchId, slot, now, result)
|
|
|
2575
2715
|
// attributing → evicting` is not even a legal edge from mid-`executing`
|
|
2576
2716
|
// (BATCH_TRANSITIONS has no `executing → evicting`) — this stays entirely
|
|
2577
2717
|
// within `executing`/`dissolving`, both of which ARE legal from here.
|
|
2578
|
-
evictMemberAndContinue(deps, config, dispatch, batchId, batch, memberIssue, memberFailureFor(read, lastTool), now, result
|
|
2718
|
+
evictMemberAndContinue(deps, config, dispatch, batchId, batch, memberIssue, memberFailureFor(read, lastTool), now, result,
|
|
2719
|
+
// #844: released in the eviction's own write, never before it.
|
|
2720
|
+
release);
|
|
2579
2721
|
}
|
|
2580
2722
|
}
|
|
2581
2723
|
/**
|
|
@@ -2616,14 +2758,26 @@ function reconcileMemberSlot(deps, config, dispatch, batchId, slot, now, result)
|
|
|
2616
2758
|
* retried by a re-entry: the re-entry takes the duplicate path (pre-existing,
|
|
2617
2759
|
* #595/#613) and the batch continues.
|
|
2618
2760
|
*/
|
|
2619
|
-
function evictMemberDirectly(deps, config, batchId, memberIssue, failure, now
|
|
2761
|
+
function evictMemberDirectly(deps, config, batchId, memberIssue, failure, now,
|
|
2762
|
+
/**
|
|
2763
|
+
* #844: a state transform applied inside pass 1's lock in every branch —
|
|
2764
|
+
* typically the member's slot release (plus, on the parallel rail, its
|
|
2765
|
+
* run's `evicted` status; on `sched resume --batch`, the batch's
|
|
2766
|
+
* `blocked → executing` transition) — so it and the park/eviction record
|
|
2767
|
+
* commit as ONE write. Pre-#844 callers wrote it under a separate lock
|
|
2768
|
+
* first, and an engine exit between the two left the member in-work with
|
|
2769
|
+
* no slot, which the wedge arm (serial) or `spawnParallelMembers`
|
|
2770
|
+
* (parallel) then respawned.
|
|
2771
|
+
*/
|
|
2772
|
+
preWrite) {
|
|
2620
2773
|
const { reason } = failure;
|
|
2621
2774
|
const kind = failure.kind ?? 'evicted';
|
|
2622
2775
|
const dissolvePolicy = (0, types_1.resolveDissolvePolicy)(config.dissolve_policy);
|
|
2623
2776
|
// Pass 1 (pure — park + record the exit): safe to run entirely inside the
|
|
2624
2777
|
// lock, unlike `dissolveBatch` below, which shells out
|
|
2625
2778
|
// (`deps.exec`/`postMilestone`) and so must NOT hold the lock while it runs.
|
|
2626
|
-
const { triggered, duplicate, prior, branch, validated } = deps.store.withLock((
|
|
2779
|
+
const { triggered, duplicate, prior, branch, validated } = deps.store.withLock((held) => {
|
|
2780
|
+
const s = preWrite ? preWrite(held) : held;
|
|
2627
2781
|
const b = (0, state_1.findBatch)(s, batchId);
|
|
2628
2782
|
if (!b) {
|
|
2629
2783
|
return {
|
|
@@ -2708,6 +2862,20 @@ function evictMemberDirectly(deps, config, batchId, memberIssue, failure, now) {
|
|
|
2708
2862
|
});
|
|
2709
2863
|
if (!triggered)
|
|
2710
2864
|
return { batchEnded: false, duplicate: false };
|
|
2865
|
+
return {
|
|
2866
|
+
batchEnded: settleEvictionThreshold(deps, config, batchId, memberIssue, validated, now),
|
|
2867
|
+
duplicate: false,
|
|
2868
|
+
};
|
|
2869
|
+
}
|
|
2870
|
+
/**
|
|
2871
|
+
* Pass 2 of an eviction that tipped the batch past its dissolve threshold:
|
|
2872
|
+
* dissolve it — unless members are already VALIDATED (#810: never dissolve
|
|
2873
|
+
* over landed work; journal `dissolve-suppressed` and continue). Returns
|
|
2874
|
+
* whether the batch's run ended. Shared by `evictMemberDirectly` and #844's
|
|
2875
|
+
* crash recovery in `spawnMemberContinuation`, which re-runs it for an
|
|
2876
|
+
* eviction whose pass 1 landed but whose engine exited before this ran.
|
|
2877
|
+
*/
|
|
2878
|
+
function settleEvictionThreshold(deps, config, batchId, memberIssue, validated, now) {
|
|
2711
2879
|
if (validated.length > 0) {
|
|
2712
2880
|
// #810: never dissolve over validated work — it stays landed and ships
|
|
2713
2881
|
// with the batch; the members that failed are parked, not requeued.
|
|
@@ -2715,19 +2883,19 @@ function evictMemberDirectly(deps, config, batchId, memberIssue, failure, now) {
|
|
|
2715
2883
|
issue: memberIssue,
|
|
2716
2884
|
detail: `eviction threshold reached, but member(s) ${validated.join(',')} are validated on the integration branch — keeping them and continuing with the remaining members instead of dissolving`,
|
|
2717
2885
|
});
|
|
2718
|
-
return
|
|
2886
|
+
return false;
|
|
2719
2887
|
}
|
|
2720
|
-
//
|
|
2721
|
-
//
|
|
2722
|
-
//
|
|
2888
|
+
// Outside the lock — dissolveBatch shells out: re-load fresh (pass 1's
|
|
2889
|
+
// write already landed), dissolve, then re-apply just this batch's + the
|
|
2890
|
+
// requeued members' state under a fresh lock.
|
|
2723
2891
|
const state = deps.store.load();
|
|
2724
2892
|
const batch = (0, state_1.findBatch)(state, batchId);
|
|
2725
2893
|
if (!batch)
|
|
2726
|
-
return
|
|
2894
|
+
return false;
|
|
2727
2895
|
const rDeps = recoveryDeps(deps, config, batch, now);
|
|
2728
2896
|
const outcome = (0, recovery_1.dissolveBatch)(state, batchId, { strategy: 'full', reason: 'eviction-threshold' }, rDeps);
|
|
2729
2897
|
applyDissolveOutcome(deps, batchId, outcome, now);
|
|
2730
|
-
return
|
|
2898
|
+
return true;
|
|
2731
2899
|
}
|
|
2732
2900
|
function reconcileFixSlot(deps, config, batchId, slot, now, result) {
|
|
2733
2901
|
if (slot.pid !== null && deps.spawnDeps.isAlive(slot.pid, slot.pid_start ?? undefined))
|
|
@@ -4054,7 +4222,10 @@ function spawnParallelMembers(deps, config, dispatch, batchId, now, result) {
|
|
|
4054
4222
|
evictMemberDirectly(deps, config, batchId, next.issue, {
|
|
4055
4223
|
reason: `member-worktree-prep-failed:${prep.reason}`,
|
|
4056
4224
|
detail: `member worktree prep failed: ${prep.reason} (member ${next.index} #${next.issue}, branch ${memberBranchFor(batchId, next.index, next.issue)})`,
|
|
4057
|
-
}, now
|
|
4225
|
+
}, now,
|
|
4226
|
+
// #844: a run already recorded (a respawn whose tree vanished) must
|
|
4227
|
+
// not stay `running` once its member is evicted.
|
|
4228
|
+
(s) => patchRun(s, batchId, next.issue, { status: 'evicted', gate_inconclusive: null }, now));
|
|
4058
4229
|
result.failed.push((0, state_1.batchMemberUnit)(batchId, next.issue));
|
|
4059
4230
|
continue;
|
|
4060
4231
|
}
|
|
@@ -4115,17 +4286,19 @@ function spawnParallelMembers(deps, config, dispatch, batchId, now, result) {
|
|
|
4115
4286
|
* evicted work's only recoverable copy, as in serial mode). A dissolve this
|
|
4116
4287
|
* tips over is torn down by `teardownBatch`, which covers every live run.
|
|
4117
4288
|
*/
|
|
4118
|
-
function evictParallelMember(deps, config, batchId, run, failure, now, result
|
|
4119
|
-
|
|
4289
|
+
function evictParallelMember(deps, config, batchId, run, failure, now, result,
|
|
4290
|
+
/** #844: the member's own slot release — committed in the eviction's write. */
|
|
4291
|
+
release) {
|
|
4292
|
+
const { batchEnded, duplicate } = evictMemberDirectly(deps, config, batchId, run.issue, failure, now,
|
|
4293
|
+
// #844: the run's `evicted` status lands in the eviction's write too — a
|
|
4294
|
+
// separate write left an evicted member's run `running` (slotless, its
|
|
4295
|
+
// tree kept until batch end) when the engine exited in between.
|
|
4296
|
+
(s) => patchRun(release ? release(s) : s, batchId, run.issue, { status: 'evicted', gate_inconclusive: null }, now));
|
|
4120
4297
|
if (duplicate)
|
|
4121
4298
|
return;
|
|
4122
4299
|
result.failed.push(batchEnded ? unit(batchId) : (0, state_1.batchMemberUnit)(batchId, run.issue));
|
|
4123
4300
|
if (batchEnded)
|
|
4124
4301
|
return;
|
|
4125
|
-
deps.store.withLock((s) => ({
|
|
4126
|
-
state: patchRun(s, batchId, run.issue, { status: 'evicted', gate_inconclusive: null }, now),
|
|
4127
|
-
result: undefined,
|
|
4128
|
-
}));
|
|
4129
4302
|
teardownRunTree(deps, batchId, run, false, now);
|
|
4130
4303
|
}
|
|
4131
4304
|
/**
|
|
@@ -4156,8 +4329,7 @@ function reconcileParallelMemberSlot(deps, config, dispatch, batchId, run, slot,
|
|
|
4156
4329
|
recordMemberRunLog(deps, dispatch, state0, batchId, run.index, run.issue, slot, now);
|
|
4157
4330
|
const verdict = evaluateIncrementalGate(deps, batchId, run.worktree, run.issue, now);
|
|
4158
4331
|
if (verdict.kind === 'failed') {
|
|
4159
|
-
deps
|
|
4160
|
-
evictParallelMember(deps, config, batchId, run, { reason: verdict.reason, detail: verdict.detail }, now, result);
|
|
4332
|
+
evictParallelMember(deps, config, batchId, run, { reason: verdict.reason, detail: verdict.detail }, now, result, release);
|
|
4161
4333
|
return;
|
|
4162
4334
|
}
|
|
4163
4335
|
deps.store.withLock((s) => ({
|
|
@@ -4178,7 +4350,7 @@ function reconcileParallelMemberSlot(deps, config, dispatch, batchId, run, slot,
|
|
|
4178
4350
|
read.milestone !== null) {
|
|
4179
4351
|
const decision = decideWrongProcedure(deps, batchId, run.issue, read.milestone, slot, release, () => recordMemberRunLog(deps, dispatch, state0, batchId, run.index, run.issue, slot, now), now);
|
|
4180
4352
|
if (typeof decision === 'object') {
|
|
4181
|
-
evictParallelMember(deps, config, batchId, run, decision, now, result);
|
|
4353
|
+
evictParallelMember(deps, config, batchId, run, decision, now, result, release);
|
|
4182
4354
|
}
|
|
4183
4355
|
return;
|
|
4184
4356
|
}
|
|
@@ -4191,8 +4363,7 @@ function reconcileParallelMemberSlot(deps, config, dispatch, batchId, run, slot,
|
|
|
4191
4363
|
}
|
|
4192
4364
|
if (read.blockedNow || read.dead) {
|
|
4193
4365
|
const lastTool = recordMemberRunLog(deps, dispatch, state0, batchId, run.index, run.issue, slot, now);
|
|
4194
|
-
deps
|
|
4195
|
-
evictParallelMember(deps, config, batchId, run, memberFailureFor(read, lastTool), now, result);
|
|
4366
|
+
evictParallelMember(deps, config, batchId, run, memberFailureFor(read, lastTool), now, result, release);
|
|
4196
4367
|
}
|
|
4197
4368
|
}
|
|
4198
4369
|
/**
|
|
@@ -4595,12 +4766,8 @@ function runBatchTick(deps, config, dispatch) {
|
|
|
4595
4766
|
const dissolvable = fresh !== undefined && (0, state_1.allowedBatchTransitions)(fresh.status).includes('dissolving');
|
|
4596
4767
|
if (dissolvable) {
|
|
4597
4768
|
// #809: a parallel batch holds one slot per running member too.
|
|
4598
|
-
for (const liveSlot of (0, state_1.slotsForBatch)(current, batch.id))
|
|
4599
|
-
|
|
4600
|
-
deps.spawnDeps.isAlive(liveSlot.pid, liveSlot.pid_start ?? undefined)) {
|
|
4601
|
-
deps.spawnDeps.kill(liveSlot.pid, liveSlot.pid_start ?? undefined);
|
|
4602
|
-
}
|
|
4603
|
-
}
|
|
4769
|
+
for (const liveSlot of (0, state_1.slotsForBatch)(current, batch.id))
|
|
4770
|
+
killLiveAgent(deps, liveSlot);
|
|
4604
4771
|
const outcome = (0, recovery_1.dissolveBatch)(current, batch.id, {
|
|
4605
4772
|
strategy: 'full',
|
|
4606
4773
|
reason: `dispatch-profile-missing:${batch.dispatch_profile}`,
|