@ai-dossier/sched 0.50.0 → 0.52.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +111 -7
- package/dist/batch-dispatch.d.ts +12 -0
- package/dist/batch-dispatch.d.ts.map +1 -1
- package/dist/batch-dispatch.js +339 -22
- package/dist/batch-dispatch.js.map +1 -1
- package/dist/dispatch.d.ts +9 -0
- package/dist/dispatch.d.ts.map +1 -1
- package/dist/dispatch.js +19 -1
- package/dist/dispatch.js.map +1 -1
- package/dist/groundtruth.d.ts +29 -1
- package/dist/groundtruth.d.ts.map +1 -1
- package/dist/groundtruth.js +64 -0
- package/dist/groundtruth.js.map +1 -1
- package/dist/index.d.ts +5 -5
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +15 -6
- package/dist/index.js.map +1 -1
- package/dist/recovery.d.ts +5 -0
- package/dist/recovery.d.ts.map +1 -1
- package/dist/recovery.js +11 -6
- package/dist/recovery.js.map +1 -1
- package/dist/scheduler.d.ts +34 -0
- package/dist/scheduler.d.ts.map +1 -1
- package/dist/scheduler.js +91 -3
- package/dist/scheduler.js.map +1 -1
- package/dist/state.d.ts +17 -1
- package/dist/state.d.ts.map +1 -1
- package/dist/state.js +70 -5
- package/dist/state.js.map +1 -1
- package/dist/status.d.ts.map +1 -1
- package/dist/status.js +39 -3
- package/dist/status.js.map +1 -1
- package/dist/types.d.ts +54 -5
- package/dist/types.d.ts.map +1 -1
- package/dist/types.js +8 -1
- package/dist/types.js.map +1 -1
- package/package.json +1 -1
package/dist/batch-dispatch.js
CHANGED
|
@@ -96,6 +96,7 @@ var __importStar = (this && this.__importStar) || (function () {
|
|
|
96
96
|
};
|
|
97
97
|
})();
|
|
98
98
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
99
|
+
exports.MAX_BATCH_AGENT_RESPAWNS = exports.tailMembersRefusal = void 0;
|
|
99
100
|
exports.memberBranchFor = memberBranchFor;
|
|
100
101
|
exports.evictMemberAndContinue = evictMemberAndContinue;
|
|
101
102
|
exports.resumeBlockedGate = resumeBlockedGate;
|
|
@@ -117,6 +118,7 @@ const recovery_1 = require("./recovery");
|
|
|
117
118
|
const run_log_1 = require("./run-log");
|
|
118
119
|
const scheduler_1 = require("./scheduler");
|
|
119
120
|
const state_1 = require("./state");
|
|
121
|
+
Object.defineProperty(exports, "tailMembersRefusal", { enumerable: true, get: function () { return state_1.tailMembersRefusal; } });
|
|
120
122
|
const teardown_1 = require("./teardown");
|
|
121
123
|
const types_1 = require("./types");
|
|
122
124
|
function emptyResult() {
|
|
@@ -160,6 +162,12 @@ function sanitizeUntrustedText(value) {
|
|
|
160
162
|
function journalEvent(deps, event, unitId, extra = {}) {
|
|
161
163
|
deps.journal.append((0, journal_1.unitEvent)(event, unitId, extra), deps.now());
|
|
162
164
|
}
|
|
165
|
+
/** Stop the slot's agent if it is still alive (pid-start-safe: a reused pid is never signalled). */
|
|
166
|
+
function killLiveAgent(deps, slot) {
|
|
167
|
+
if (slot.pid !== null && deps.spawnDeps.isAlive(slot.pid, slot.pid_start ?? undefined)) {
|
|
168
|
+
deps.spawnDeps.kill(slot.pid, slot.pid_start ?? undefined);
|
|
169
|
+
}
|
|
170
|
+
}
|
|
163
171
|
/**
|
|
164
172
|
* Read a dead batch dispatch's own log and classify it (#629 AC3) — tail,
|
|
165
173
|
* member, fix and report all reach a `dead` agent the same way (`pid` no
|
|
@@ -1014,7 +1022,11 @@ function spawnMemberAgent(deps, dispatch, state, slot, batchId, now, result, tar
|
|
|
1014
1022
|
// #677: the prompt carries THIS member's own worktree and the integration
|
|
1015
1023
|
// branch it was cut from — `member-cycle`'s `worktree`/`integration_branch`
|
|
1016
1024
|
// inputs. The shared batch worktree is no longer a member surface at all.
|
|
1017
|
-
const prompt = (0, dispatch_1.buildMemberPrompt)(dispatch.memberPrompt, memberIssue, batchId, target.member.worktree, batch.branch, review)
|
|
1025
|
+
const prompt = (0, dispatch_1.buildMemberPrompt)(dispatch.memberPrompt, memberIssue, batchId, target.member.worktree, batch.branch, review) +
|
|
1026
|
+
// #822: a member respawned after running the wrong procedure.
|
|
1027
|
+
(batch.reprompted_members.some((r) => r.issue === memberIssue)
|
|
1028
|
+
? (0, dispatch_1.wrongProcedureDirective)(memberIssue, batchId)
|
|
1029
|
+
: '');
|
|
1018
1030
|
const logFile = (0, dispatch_1.batchMemberLogPath)(deps.store.runsDir, batchId, target.index, memberIssue);
|
|
1019
1031
|
// #629: captured BEFORE spawning, mirroring `engine.ts`'s own
|
|
1020
1032
|
// `spawnAndRecord` — the log is per-role and append-mode, so the size at
|
|
@@ -1309,6 +1321,20 @@ function spawnMemberContinuation(deps, config, dispatch, batchId, now, result) {
|
|
|
1309
1321
|
}
|
|
1310
1322
|
claimAndSpawn(deps, config, batchId, 'member', now, (nextState, slot) => spawnMember(deps, dispatch, nextState, slot, batchId, now, result, member));
|
|
1311
1323
|
}
|
|
1324
|
+
/**
|
|
1325
|
+
* #832: the one pre-dispatch verdict for a tail — checked before a slot is
|
|
1326
|
+
* claimed and again under the claim's lock. A tail whose respawns are spent
|
|
1327
|
+
* (a crash landed between counting the exit that spent them and blocking the
|
|
1328
|
+
* batch) is refused, as is a members mismatch. Only a batch in a tail phase
|
|
1329
|
+
* is judged; anything else is not the tail's to refuse.
|
|
1330
|
+
*/
|
|
1331
|
+
function tailDispatchRefusal(state, batch) {
|
|
1332
|
+
if (batch.status !== 'reviewing' && batch.status !== 'shipping')
|
|
1333
|
+
return null;
|
|
1334
|
+
if (agentRespawnsSpent(batch, 'tail'))
|
|
1335
|
+
return 'respawn-cap:tail';
|
|
1336
|
+
return (0, state_1.tailMembersRefusal)(batch, (0, state_1.validatedMembersOf)(state, batch));
|
|
1337
|
+
}
|
|
1312
1338
|
/**
|
|
1313
1339
|
* Tail/report/fix dispatches (this function, `spawnReportAgent`,
|
|
1314
1340
|
* `reconcileFixSlot`) are NOT recorded live to `runs.jsonl` — only member
|
|
@@ -1317,11 +1343,27 @@ function spawnMemberContinuation(deps, config, dispatch, batchId, now, result) {
|
|
|
1317
1343
|
* the only way to see their cost today, reconstructed from the raw log
|
|
1318
1344
|
* after the fact. Wiring in live recording for these later means repeating
|
|
1319
1345
|
* `spawnMember`'s own #564 fix first: none of these three spawn functions'
|
|
1320
|
-
* patches
|
|
1346
|
+
* patches stamped `SlotEntry.spawned_at` either, so a `recordXRunLog` guarded
|
|
1321
1347
|
* on `spawned_at !== null` (mirroring `recordMemberRunLog`) would silently
|
|
1322
|
-
* no-op forever, exactly like the original bug.
|
|
1348
|
+
* no-op forever, exactly like the original bug. #832 made the tail and report
|
|
1349
|
+
* spawns stamp it (their hand-back fence needs it); the fix agent still does
|
|
1350
|
+
* not.
|
|
1323
1351
|
*/
|
|
1324
1352
|
function spawnTailAgent(deps, config, dispatch, batchId, now, result) {
|
|
1353
|
+
// #832: the tail runs over the LANDED members only — never `batch.members`,
|
|
1354
|
+
// which still lists a member that was stopped, handed back, evicted or
|
|
1355
|
+
// parked. A refusal blocks the batch before any agent is spawned (and
|
|
1356
|
+
// before a slot is claimed — `blockBatchForOperator` posts a milestone, an
|
|
1357
|
+
// exec that must not run under the store lock).
|
|
1358
|
+
const before = deps.store.load();
|
|
1359
|
+
const pending = (0, state_1.findBatch)(before, batchId);
|
|
1360
|
+
if (pending) {
|
|
1361
|
+
const refusal = tailDispatchRefusal(before, pending);
|
|
1362
|
+
if (refusal !== null) {
|
|
1363
|
+
blockBatchForOperator(deps, config, batchId, { reason: refusal, milestonePhase: 'batch-review' }, now, result);
|
|
1364
|
+
return;
|
|
1365
|
+
}
|
|
1366
|
+
}
|
|
1325
1367
|
claimAndSpawn(deps, config, batchId, 'reviewing', now, (state, slot) => {
|
|
1326
1368
|
const batch = (0, state_1.findBatch)(state, batchId);
|
|
1327
1369
|
if (!batch || batch.worktree === null || batch.anchor === null) {
|
|
@@ -1331,9 +1373,20 @@ function spawnTailAgent(deps, config, dispatch, batchId, now, result) {
|
|
|
1331
1373
|
});
|
|
1332
1374
|
return releaseSlot(state, batchId, now);
|
|
1333
1375
|
}
|
|
1376
|
+
const landed = (0, state_1.validatedMembersOf)(state, batch);
|
|
1377
|
+
const lateRefusal = tailDispatchRefusal(state, batch);
|
|
1378
|
+
if (lateRefusal !== null) {
|
|
1379
|
+
// The state changed between the pre-check and the lock (a concurrent
|
|
1380
|
+
// stop/park) — release, and the next tick's pre-check (reading the same
|
|
1381
|
+
// persisted state) blocks the batch with this reason.
|
|
1382
|
+
journalEvent(deps, 'slot-released', unit(batchId), {
|
|
1383
|
+
detail: `tail not dispatched: ${lateRefusal} (state changed before the claim; the next tick blocks the batch)`,
|
|
1384
|
+
});
|
|
1385
|
+
return releaseSlot(state, batchId, now);
|
|
1386
|
+
}
|
|
1334
1387
|
const spawnSpec = (0, dispatch_1.resolveTierSpawn)(dispatch, 'strong', batch.anchor);
|
|
1335
1388
|
const cmd = spawnSpec.cmd;
|
|
1336
|
-
const prompt = (0, dispatch_1.buildBatchTailPrompt)(dispatch.batchTailPrompt, batchId, batch.anchor,
|
|
1389
|
+
const prompt = (0, dispatch_1.buildBatchTailPrompt)(dispatch.batchTailPrompt, batchId, batch.anchor, landed, batch.worktree);
|
|
1337
1390
|
const logFile = (0, dispatch_1.batchTailLogPath)(deps.store.runsDir, batchId);
|
|
1338
1391
|
// #629: fences `handleDeadDispatchApiError`'s classification to this
|
|
1339
1392
|
// dispatch's slice — see `spawnMember`'s identical comment.
|
|
@@ -1356,6 +1409,14 @@ function spawnTailAgent(deps, config, dispatch, batchId, now, result) {
|
|
|
1356
1409
|
phase: 'reviewing',
|
|
1357
1410
|
last_progress_at: now.toISOString(),
|
|
1358
1411
|
log_offset_at_spawn: logOffset,
|
|
1412
|
+
// #832: the fence `reconcileTailSlot` reads — without it a
|
|
1413
|
+
// `batch-review blocked` left on the anchor by an EARLIER dispatch
|
|
1414
|
+
// (or a previous batch on the same anchor, #605) would block this one.
|
|
1415
|
+
// The fence tolerates `DISPATCH_FENCE_TOLERANCE_MS` (60s) of clock skew,
|
|
1416
|
+
// so a blocked milestone posted less than a minute before this spawn
|
|
1417
|
+
// still counts — only a resume that re-dispatches the tail that fast
|
|
1418
|
+
// could trip it.
|
|
1419
|
+
spawned_at: now.toISOString(),
|
|
1359
1420
|
};
|
|
1360
1421
|
const next = slot.status === 'assigned' ? (0, state_1.transitionSlot)(state, slot.id, 'running', patch, now) : state;
|
|
1361
1422
|
deps.journal.append((0, journal_1.unitEvent)('spawned', unit(batchId), {
|
|
@@ -1370,6 +1431,13 @@ function spawnTailAgent(deps, config, dispatch, batchId, now, result) {
|
|
|
1370
1431
|
});
|
|
1371
1432
|
}
|
|
1372
1433
|
function spawnReportAgent(deps, config, dispatch, batchId, now, result) {
|
|
1434
|
+
// #832: respawns already spent (a crash landed between counting the exit
|
|
1435
|
+
// that spent them and closing the batch) — close it now, never spawn.
|
|
1436
|
+
const pending = (0, state_1.findBatch)(deps.store.load(), batchId);
|
|
1437
|
+
if (pending && pending.status === 'deployed' && agentRespawnsSpent(pending, 'report')) {
|
|
1438
|
+
closeWithoutReport(deps, batchId, 'respawn-cap:report', now, result);
|
|
1439
|
+
return;
|
|
1440
|
+
}
|
|
1373
1441
|
claimAndSpawn(deps, config, batchId, 'report', now, (state, slot) => {
|
|
1374
1442
|
const batch = (0, state_1.findBatch)(state, batchId);
|
|
1375
1443
|
if (!batch || batch.anchor === null) {
|
|
@@ -1406,6 +1474,8 @@ function spawnReportAgent(deps, config, dispatch, batchId, now, result) {
|
|
|
1406
1474
|
phase: 'report',
|
|
1407
1475
|
last_progress_at: now.toISOString(),
|
|
1408
1476
|
log_offset_at_spawn: logOffset,
|
|
1477
|
+
// #832: the fence `reconcileReportSlot`'s hand-back check reads.
|
|
1478
|
+
spawned_at: now.toISOString(),
|
|
1409
1479
|
};
|
|
1410
1480
|
const next = slot.status === 'assigned'
|
|
1411
1481
|
? (0, state_1.transitionSlot)(state, slot.id, 'running', patchState, now)
|
|
@@ -2001,7 +2071,7 @@ function resumeBlockedGate(deps, config, dispatch, batchId, now) {
|
|
|
2001
2071
|
// `suite-unreadable`) — name the actual reason and the operator path
|
|
2002
2072
|
// instead of a bare illegal-transition error that reads like a bug.
|
|
2003
2073
|
if (batch.status === 'blocked') {
|
|
2004
|
-
throw new types_1.SchedNotFoundError(`Batch ${batchId} is blocked on '${batch.blocked_reason ?? '?'}', which has no gate recheck: inspect the batch worktree/state named in its journal, then \`sched abandon --batch ${batchId}\` if it is a dead end`);
|
|
2074
|
+
throw new types_1.SchedNotFoundError(`Batch ${batchId} is blocked on '${batch.blocked_reason ?? '?'}', which has no gate recheck (a batch blocked over landed work — dissolve-refused / tail-blocked / members-mismatch / respawn-cap:tail — resumes via \`sched resume --batch\`, #822): inspect the batch worktree/state named in its journal, then \`sched abandon --batch ${batchId}\` if it is a dead end`);
|
|
2005
2075
|
}
|
|
2006
2076
|
throw new types_1.IllegalTransitionError('batch', batch.status, 'executing');
|
|
2007
2077
|
}
|
|
@@ -2277,8 +2347,113 @@ function readMemberDispatch(deps, batchId, memberIssue, slot, now) {
|
|
|
2277
2347
|
milestone,
|
|
2278
2348
|
dead,
|
|
2279
2349
|
blockedNow: (0, groundtruth_1.isMemberBlocked)(milestone, slot.spawned_at),
|
|
2350
|
+
wrongProcedure: (0, groundtruth_1.isWrongProcedureMilestone)(milestone, slot.spawned_at),
|
|
2351
|
+
};
|
|
2352
|
+
}
|
|
2353
|
+
/**
|
|
2354
|
+
* #822: whether a wrong-procedure read counts against THIS dispatch of
|
|
2355
|
+
* `memberIssue`. After a re-prompt, only a milestone posted strictly after
|
|
2356
|
+
* the re-prompt does: the killed first dispatch's milestones still sit
|
|
2357
|
+
* inside the respawn's 60s fence tolerance and would otherwise evict the
|
|
2358
|
+
* member the moment it respawns. No tolerance here — the engine and the
|
|
2359
|
+
* member's CLI stamp times on the same host.
|
|
2360
|
+
*/
|
|
2361
|
+
function wrongProcedureCounts(batch, memberIssue, read) {
|
|
2362
|
+
if (!read.wrongProcedure || read.milestone === null)
|
|
2363
|
+
return false;
|
|
2364
|
+
const prior = batch.reprompted_members.find((r) => r.issue === memberIssue);
|
|
2365
|
+
if (prior === undefined)
|
|
2366
|
+
return true;
|
|
2367
|
+
const at = Date.parse(read.milestone.at);
|
|
2368
|
+
const repromptedAt = Date.parse(prior.reprompted_at);
|
|
2369
|
+
return Number.isNaN(at) || Number.isNaN(repromptedAt) ? false : at > repromptedAt;
|
|
2370
|
+
}
|
|
2371
|
+
/**
|
|
2372
|
+
* #822 (#810 proposal 3): the member's dispatch ran the wrong procedure (a
|
|
2373
|
+
* full-cycle-shaped trail — imboard #4174 ran full-cycle as a batch member).
|
|
2374
|
+
* That is a dispatch mistake, not the member's work failing, so the first
|
|
2375
|
+
* time it is RE-PROMPTED: a live agent is stopped (a full-cycle run left going
|
|
2376
|
+
* would open its own PR), the slot is released, and the member is recorded in
|
|
2377
|
+
* `reprompted_members` — the wedge arm (serial) or `spawnParallelMembers`
|
|
2378
|
+
* (parallel) respawns it in place, in the same worktree, with
|
|
2379
|
+
* `wrongProcedureDirective` appended. Returns `'evict'` when the member was
|
|
2380
|
+
* already re-prompted once: the caller evicts it `wrong-procedure`.
|
|
2381
|
+
*/
|
|
2382
|
+
function repromptWrongProcedure(deps, batchId, memberIssue, milestoneAt, slot, release, now) {
|
|
2383
|
+
// Never respawn into (or release) a worktree the old agent is still in: a
|
|
2384
|
+
// dying full-cycle run could still push, post, or open a PR. Signal it and
|
|
2385
|
+
// decide on a later tick, once it is gone — and so the re-prompt's
|
|
2386
|
+
// `reprompted_at` fence postdates everything it could still post.
|
|
2387
|
+
if (slot.pid !== null && deps.spawnDeps.isAlive(slot.pid, slot.pid_start ?? undefined)) {
|
|
2388
|
+
deps.spawnDeps.kill(slot.pid, slot.pid_start ?? undefined);
|
|
2389
|
+
return 'waiting';
|
|
2390
|
+
}
|
|
2391
|
+
const outcome = deps.store.withLock((s) => {
|
|
2392
|
+
const released = release(s);
|
|
2393
|
+
const b = (0, state_1.findBatch)(released, batchId);
|
|
2394
|
+
if (!b || b.reprompted_members.some((r) => r.issue === memberIssue)) {
|
|
2395
|
+
return { state: released, result: 'evict' };
|
|
2396
|
+
}
|
|
2397
|
+
return {
|
|
2398
|
+
state: (0, state_1.patchBatch)(released, batchId, {
|
|
2399
|
+
reprompted_members: [
|
|
2400
|
+
...b.reprompted_members,
|
|
2401
|
+
{ issue: memberIssue, milestone_at: milestoneAt, reprompted_at: now.toISOString() },
|
|
2402
|
+
],
|
|
2403
|
+
}, now),
|
|
2404
|
+
result: 'reprompted',
|
|
2405
|
+
};
|
|
2406
|
+
});
|
|
2407
|
+
if (outcome === 'reprompted') {
|
|
2408
|
+
journalEvent(deps, 'member-reprompted', unit(batchId), {
|
|
2409
|
+
issue: memberIssue,
|
|
2410
|
+
reason: 'wrong-procedure',
|
|
2411
|
+
milestone_at: milestoneAt,
|
|
2412
|
+
detail: `${WRONG_PROCEDURE_SHAPE} — respawning it once in place with a member-cycle directive`,
|
|
2413
|
+
});
|
|
2414
|
+
}
|
|
2415
|
+
return outcome;
|
|
2416
|
+
}
|
|
2417
|
+
/** #822: the evidence line every wrong-procedure journal/eviction names. */
|
|
2418
|
+
const WRONG_PROCEDURE_SHAPE = 'member posted a full-cycle-shaped milestone (no batch=/mode=slot key)';
|
|
2419
|
+
/** #822: the eviction a member earns for running the wrong procedure after its one re-prompt. */
|
|
2420
|
+
function wrongProcedureFailure(lastTool, shippedPr = null) {
|
|
2421
|
+
return {
|
|
2422
|
+
reason: shippedPr === null ? 'wrong-procedure' : 'wrong-procedure-shipped',
|
|
2423
|
+
detail: shippedPr === null
|
|
2424
|
+
? `${WRONG_PROCEDURE_SHAPE} again after its one re-prompt`
|
|
2425
|
+
: `${WRONG_PROCEDURE_SHAPE} and its full-cycle run already reached ship (PR ${shippedPr === 'unknown' ? 'not named' : `#${shippedPr}`}) — evicted without a re-prompt; close or reconcile that PR by hand`,
|
|
2426
|
+
kind: 'evicted',
|
|
2427
|
+
extraKv: {
|
|
2428
|
+
...(lastTool ? { last_tool: lastTool } : {}),
|
|
2429
|
+
...(typeof shippedPr === 'number' ? { stray_pr: String(shippedPr) } : {}),
|
|
2430
|
+
},
|
|
2280
2431
|
};
|
|
2281
2432
|
}
|
|
2433
|
+
/**
|
|
2434
|
+
* #822: the one wrong-procedure decision shared by both member rails —
|
|
2435
|
+
* `'waiting'` (the old agent is still dying), `'reprompted'`, or the
|
|
2436
|
+
* eviction failure to apply. A run that already shipped is evicted without
|
|
2437
|
+
* a re-prompt.
|
|
2438
|
+
*/
|
|
2439
|
+
function decideWrongProcedure(deps, batchId, memberIssue, milestone, slot, release, lastTool, now) {
|
|
2440
|
+
const shipped = (0, groundtruth_1.wrongProcedureShippedPr)(milestone);
|
|
2441
|
+
if (shipped !== null) {
|
|
2442
|
+
if (slot.pid !== null && deps.spawnDeps.isAlive(slot.pid, slot.pid_start ?? undefined)) {
|
|
2443
|
+
deps.spawnDeps.kill(slot.pid, slot.pid_start ?? undefined);
|
|
2444
|
+
return 'waiting';
|
|
2445
|
+
}
|
|
2446
|
+
const tool = lastTool();
|
|
2447
|
+
deps.store.withLock((s) => ({ state: release(s), result: undefined }));
|
|
2448
|
+
return wrongProcedureFailure(tool, shipped);
|
|
2449
|
+
}
|
|
2450
|
+
const verdict = repromptWrongProcedure(deps, batchId, memberIssue, milestone.at, slot, release, now);
|
|
2451
|
+
if (verdict === 'waiting')
|
|
2452
|
+
return verdict;
|
|
2453
|
+
// The finished dispatch's run-log entry, exactly once (never while waiting).
|
|
2454
|
+
const tool = lastTool();
|
|
2455
|
+
return verdict === 'reprompted' ? verdict : wrongProcedureFailure(tool);
|
|
2456
|
+
}
|
|
2282
2457
|
/**
|
|
2283
2458
|
* The eviction a blocked-or-dead member dispatch earns (shared by both rails).
|
|
2284
2459
|
* #605: only a milestone THIS dispatch posted may name the reason. A dead
|
|
@@ -2348,6 +2523,16 @@ function reconcileMemberSlot(deps, config, dispatch, batchId, slot, now, result)
|
|
|
2348
2523
|
completeMemberGate(deps, config, dispatch, batchId, batch, memberIssue, now, result);
|
|
2349
2524
|
return;
|
|
2350
2525
|
}
|
|
2526
|
+
// #822: a wrong-procedure trail is re-prompted once, then evicted — decided
|
|
2527
|
+
// before the dead/blocked rail, which would otherwise evict it as an
|
|
2528
|
+
// unverified exit at the first sight.
|
|
2529
|
+
if (wrongProcedureCounts(batch, memberIssue, read) && read.milestone !== null) {
|
|
2530
|
+
const decision = decideWrongProcedure(deps, batchId, memberIssue, read.milestone, slot, (s) => releaseSlot(s, batchId, now), () => recordMemberRunLog(deps, dispatch, state0, batchId, batch.executing_member, memberIssue, slot, now), now);
|
|
2531
|
+
if (typeof decision === 'object') {
|
|
2532
|
+
evictMemberAndContinue(deps, config, dispatch, batchId, batch, memberIssue, decision, now, result);
|
|
2533
|
+
}
|
|
2534
|
+
return;
|
|
2535
|
+
}
|
|
2351
2536
|
if (read.dead &&
|
|
2352
2537
|
!read.blockedNow &&
|
|
2353
2538
|
handleDeadDispatchApiError(deps, batchId, (0, dispatch_1.batchMemberLogPath)(deps.store.runsDir, batchId, batch.executing_member, memberIssue), slot.log_offset_at_spawn ?? 0, now)) {
|
|
@@ -2565,7 +2750,98 @@ function reconcileFixSlot(deps, config, batchId, slot, now, result) {
|
|
|
2565
2750
|
}
|
|
2566
2751
|
evictOffender(deps, config, batchId, offenderRecord.issue, 'overlap', now, result);
|
|
2567
2752
|
}
|
|
2568
|
-
|
|
2753
|
+
/**
|
|
2754
|
+
* #832: how many times one tail-work phase's agent (`tail` or `report`) is
|
|
2755
|
+
* RESPAWNED after exiting unverified — the first dispatch plus this many
|
|
2756
|
+
* respawns; the next unverified exit ends the phase instead. The #629 class:
|
|
2757
|
+
* the tick loop's "same wedge" arm respawns a dead tail/report agent on its
|
|
2758
|
+
* own, so without a cap an agent that exits without a verdict is
|
|
2759
|
+
* redispatched forever (b-20260924-04: the strong-tier tail four times,
|
|
2760
|
+
* ~375k tokens, stopped only by hand).
|
|
2761
|
+
*/
|
|
2762
|
+
exports.MAX_BATCH_AGENT_RESPAWNS = 2;
|
|
2763
|
+
/** #832: whether `batch` has already spent its respawns for `phase`. */
|
|
2764
|
+
function agentRespawnsSpent(batch, phase) {
|
|
2765
|
+
return (batch.agent_exits !== null &&
|
|
2766
|
+
batch.agent_exits.phase === phase &&
|
|
2767
|
+
batch.agent_exits.count > exports.MAX_BATCH_AGENT_RESPAWNS);
|
|
2768
|
+
}
|
|
2769
|
+
/**
|
|
2770
|
+
* #832: count one unverified exit of `phase`'s agent, release its slot, and
|
|
2771
|
+
* journal the exit with its position against the cap — one lock, so the count
|
|
2772
|
+
* and the release cannot come apart. Returns `'capped'` when this exit spent
|
|
2773
|
+
* the phase's last respawn (the caller ends the phase), else `'respawn'`.
|
|
2774
|
+
* The count is per phase — another phase's exits start it over — and verified
|
|
2775
|
+
* progress clears it (`clearAgentExits`).
|
|
2776
|
+
*/
|
|
2777
|
+
function releaseAndCountAgentExit(deps, batchId, phase, failed, now) {
|
|
2778
|
+
const count = deps.store.withLock((s) => {
|
|
2779
|
+
const b = (0, state_1.findBatch)(s, batchId);
|
|
2780
|
+
const prior = b?.agent_exits;
|
|
2781
|
+
const next = prior && prior.phase === phase ? prior.count + 1 : 1;
|
|
2782
|
+
let n = releaseSlot(s, batchId, now);
|
|
2783
|
+
if (b)
|
|
2784
|
+
n = (0, state_1.patchBatch)(n, batchId, { agent_exits: { phase, count: next } }, now);
|
|
2785
|
+
return { state: n, result: next };
|
|
2786
|
+
});
|
|
2787
|
+
journalEvent(deps, failed.event, unit(batchId), {
|
|
2788
|
+
...failed.extra,
|
|
2789
|
+
detail: `${String(failed.extra.detail ?? 'unverified exit')} ${count}/${exports.MAX_BATCH_AGENT_RESPAWNS + 1} (log: ${failed.log})`,
|
|
2790
|
+
});
|
|
2791
|
+
return count > exports.MAX_BATCH_AGENT_RESPAWNS ? 'capped' : 'respawn';
|
|
2792
|
+
}
|
|
2793
|
+
/** #832: verified progress in a tail-work phase — the respawn count starts over. */
|
|
2794
|
+
function clearAgentExits(s, batchId, now) {
|
|
2795
|
+
const b = (0, state_1.findBatch)(s, batchId);
|
|
2796
|
+
return b && b.agent_exits !== null ? (0, state_1.patchBatch)(s, batchId, { agent_exits: null }, now) : s;
|
|
2797
|
+
}
|
|
2798
|
+
/**
|
|
2799
|
+
* #832: the tail posted its own `batch-review`/`batch-ship blocked` — release
|
|
2800
|
+
* its slot and block the batch with that reason in ONE lock (no milestone to
|
|
2801
|
+
* post: the agent's is already the anchor's record, so no exec runs under the
|
|
2802
|
+
* lock), so a crash can never leave the slot released with the batch still
|
|
2803
|
+
* `reviewing` for the wedge arm to respawn. The agent decided; respawning it
|
|
2804
|
+
* would only get the same answer again.
|
|
2805
|
+
*/
|
|
2806
|
+
function blockOnTailHandBack(deps, config, batch, reason, now, result) {
|
|
2807
|
+
const rDeps = recoveryDeps(deps, config, batch, now);
|
|
2808
|
+
deps.store.withLock((s) => {
|
|
2809
|
+
const released = releaseSlot(s, batch.id, now);
|
|
2810
|
+
const b = (0, state_1.findBatch)(released, batch.id);
|
|
2811
|
+
if (!b || types_1.TERMINAL_BATCH_STATUSES.has(b.status) || b.status === 'blocked') {
|
|
2812
|
+
return { state: released, result: undefined };
|
|
2813
|
+
}
|
|
2814
|
+
const blocked = (0, recovery_1.blockBatch)(released, batch.id, { reason, milestonePhase: 'batch-review', milestoneAlreadyPosted: true }, rDeps);
|
|
2815
|
+
return { state: blocked.state, result: undefined };
|
|
2816
|
+
});
|
|
2817
|
+
result.failed.push(unit(batch.id));
|
|
2818
|
+
}
|
|
2819
|
+
/**
|
|
2820
|
+
* #832: close a MERGED batch whose report agent spent its respawns, without
|
|
2821
|
+
* its report. The report is the batch's summary, not its work — the PR is
|
|
2822
|
+
* merged and deployed — so the batch goes `deployed → reported → done` and
|
|
2823
|
+
* is torn down like any finished batch. Blocking it instead would not stop
|
|
2824
|
+
* anything: `reconcileStaleBlockedBatches` walks a blocked batch with a
|
|
2825
|
+
* merged PR straight back to `deployed`, and the report agent would respawn.
|
|
2826
|
+
*/
|
|
2827
|
+
function closeWithoutReport(deps, batchId, why, now, result) {
|
|
2828
|
+
const closed = deps.store.withLock((s) => {
|
|
2829
|
+
const b = (0, state_1.findBatch)(s, batchId);
|
|
2830
|
+
if (!b || b.status !== 'deployed')
|
|
2831
|
+
return { state: s, result: false };
|
|
2832
|
+
let n = (0, state_1.transitionBatch)(s, batchId, 'reported', { agent_exits: null }, now);
|
|
2833
|
+
n = (0, state_1.transitionBatch)(n, batchId, 'done', {}, now);
|
|
2834
|
+
return { state: n, result: true };
|
|
2835
|
+
});
|
|
2836
|
+
if (!closed)
|
|
2837
|
+
return;
|
|
2838
|
+
journalEvent(deps, 'report-failed', unit(batchId), {
|
|
2839
|
+
detail: `${why} — batch closed without its report (the PR is merged); post the batch report by hand if one is wanted`,
|
|
2840
|
+
});
|
|
2841
|
+
result.failed.push(unit(batchId));
|
|
2842
|
+
teardownBatch(deps, batchId);
|
|
2843
|
+
}
|
|
2844
|
+
function reconcileTailSlot(deps, config, batchId, slot, now, result) {
|
|
2569
2845
|
const state = deps.store.load();
|
|
2570
2846
|
const batch = (0, state_1.findBatch)(state, batchId);
|
|
2571
2847
|
if (!batch || batch.anchor === null)
|
|
@@ -2580,10 +2856,28 @@ function reconcileTailSlot(deps, batchId, slot, now, result) {
|
|
|
2580
2856
|
const b = (0, state_1.findBatch)(s, batchId);
|
|
2581
2857
|
if (!b || b.status !== 'reviewing')
|
|
2582
2858
|
return { state: s, result: undefined };
|
|
2583
|
-
|
|
2859
|
+
const n = clearAgentExits(s, batchId, now);
|
|
2860
|
+
return { state: (0, state_1.transitionBatch)(n, batchId, 'shipping', {}, now), result: undefined };
|
|
2584
2861
|
});
|
|
2585
2862
|
return;
|
|
2586
2863
|
}
|
|
2864
|
+
// #832: the tail handed back (`batch-review`/`batch-ship blocked`, e.g.
|
|
2865
|
+
// `members-mismatch`) — block the batch with its reason, never respawn.
|
|
2866
|
+
// Checked before liveness: the agent is done deciding whether or not its
|
|
2867
|
+
// process has exited yet, and the block releases its slot. Only for a slot
|
|
2868
|
+
// that recorded its dispatch time: a tail spawned before #832 stamped
|
|
2869
|
+
// `spawned_at` has no fence, and an unfenced read would let an older
|
|
2870
|
+
// `blocked` milestone on the anchor block this dispatch — that exit falls
|
|
2871
|
+
// through to the (capped) unverified path instead.
|
|
2872
|
+
const handedBack = slot.spawned_at === null
|
|
2873
|
+
? null
|
|
2874
|
+
: (0, groundtruth_1.batchPhaseBlockedReason)(milestone, ['batch-review', 'batch-ship'], slot.spawned_at);
|
|
2875
|
+
if (handedBack !== null) {
|
|
2876
|
+
if (!dead)
|
|
2877
|
+
killLiveAgent(deps, slot);
|
|
2878
|
+
blockOnTailHandBack(deps, config, batch, `tail-blocked:${handedBack}`, now, result);
|
|
2879
|
+
return;
|
|
2880
|
+
}
|
|
2587
2881
|
if ((0, groundtruth_1.isBatchTailParked)(milestone)) {
|
|
2588
2882
|
const pr = (0, groundtruth_1.prOfMilestone)(milestone);
|
|
2589
2883
|
journalEvent(deps, 'pr-parked', unit(batchId), { pr: pr ?? undefined });
|
|
@@ -2595,6 +2889,7 @@ function reconcileTailSlot(deps, batchId, slot, now, result) {
|
|
|
2595
2889
|
const b = (0, state_1.findBatch)(n, batchId);
|
|
2596
2890
|
if (!b)
|
|
2597
2891
|
return { state: n, result: undefined };
|
|
2892
|
+
n = clearAgentExits(n, batchId, now);
|
|
2598
2893
|
n = b.status === 'reviewing' ? (0, state_1.transitionBatch)(n, batchId, 'shipping', {}, now) : n;
|
|
2599
2894
|
n = (0, state_1.transitionBatch)(n, batchId, 'awaiting-merge', { pr }, now);
|
|
2600
2895
|
return { state: n, result: undefined };
|
|
@@ -2610,12 +2905,20 @@ function reconcileTailSlot(deps, batchId, slot, now, result) {
|
|
|
2610
2905
|
if (handleDeadDispatchApiError(deps, batchId, (0, dispatch_1.batchTailLogPath)(deps.store.runsDir, batchId), slot.log_offset_at_spawn ?? 0, now)) {
|
|
2611
2906
|
return;
|
|
2612
2907
|
}
|
|
2613
|
-
|
|
2614
|
-
|
|
2908
|
+
// #832: counted — once the respawns are spent the batch blocks instead.
|
|
2909
|
+
const outcome = releaseAndCountAgentExit(deps, batchId, 'tail', {
|
|
2910
|
+
event: 'unit-failed',
|
|
2911
|
+
extra: { reason: 'tail-agent-exited-unverified' },
|
|
2912
|
+
log: (0, dispatch_1.batchTailLogPath)(deps.store.runsDir, batchId),
|
|
2913
|
+
}, now);
|
|
2914
|
+
if (outcome === 'capped') {
|
|
2915
|
+
blockBatchForOperator(deps, config, batchId, { reason: 'respawn-cap:tail', milestonePhase: 'batch-review' }, now, result);
|
|
2916
|
+
return;
|
|
2917
|
+
}
|
|
2615
2918
|
result.failed.push(unit(batchId));
|
|
2616
2919
|
}
|
|
2617
2920
|
}
|
|
2618
|
-
function reconcileReportSlot(deps, batchId, slot, now, result) {
|
|
2921
|
+
function reconcileReportSlot(deps, config, batchId, slot, now, result) {
|
|
2619
2922
|
const state = deps.store.load();
|
|
2620
2923
|
const batch = (0, state_1.findBatch)(state, batchId);
|
|
2621
2924
|
if (!batch || batch.anchor === null)
|
|
@@ -2630,7 +2933,7 @@ function reconcileReportSlot(deps, batchId, slot, now, result) {
|
|
|
2630
2933
|
// #629: a verified report completion is proof dispatch is healthy —
|
|
2631
2934
|
// reset the confirmed-failure streak, mirroring the per-issue path's
|
|
2632
2935
|
// own verified-complete branch.
|
|
2633
|
-
let n = (0, dispatch_health_1.resetDispatchApiErrorStreak)(releaseSlot(s, batchId, now));
|
|
2936
|
+
let n = clearAgentExits((0, dispatch_health_1.resetDispatchApiErrorStreak)(releaseSlot(s, batchId, now)), batchId, now);
|
|
2634
2937
|
const b = (0, state_1.findBatch)(n, batchId);
|
|
2635
2938
|
if (!b || b.status !== 'deployed')
|
|
2636
2939
|
return { state: n, result: undefined };
|
|
@@ -2650,8 +2953,15 @@ function reconcileReportSlot(deps, batchId, slot, now, result) {
|
|
|
2650
2953
|
if (handleDeadDispatchApiError(deps, batchId, (0, dispatch_1.batchReportLogPath)(deps.store.runsDir, batchId), slot.log_offset_at_spawn ?? 0, now)) {
|
|
2651
2954
|
return;
|
|
2652
2955
|
}
|
|
2653
|
-
|
|
2654
|
-
|
|
2956
|
+
// #832: counted — once the respawns are spent the merged batch closes
|
|
2957
|
+
// without its report (see `closeWithoutReport` for why not a block).
|
|
2958
|
+
const outcome = releaseAndCountAgentExit(deps, batchId, 'report', {
|
|
2959
|
+
event: 'report-failed',
|
|
2960
|
+
extra: { detail: 'unverified exit' },
|
|
2961
|
+
log: (0, dispatch_1.batchReportLogPath)(deps.store.runsDir, batchId),
|
|
2962
|
+
}, now);
|
|
2963
|
+
if (outcome === 'capped')
|
|
2964
|
+
closeWithoutReport(deps, batchId, 'respawn-cap:report', now, result);
|
|
2655
2965
|
}
|
|
2656
2966
|
}
|
|
2657
2967
|
// --- PR watch for `awaiting-merge` batches (mirrors `pollParkedPrs`/`reconcileParked`) ---
|
|
@@ -3689,9 +3999,7 @@ function reconcileParallelMemberSlot(deps, config, dispatch, batchId, run, slot,
|
|
|
3689
3999
|
// into that tree with its pid no longer tracked anywhere (so neither a
|
|
3690
4000
|
// dissolve nor `sched stop --batch` could stop it). Its work is done —
|
|
3691
4001
|
// stop it first.
|
|
3692
|
-
|
|
3693
|
-
deps.spawnDeps.kill(slot.pid, slot.pid_start ?? undefined);
|
|
3694
|
-
}
|
|
4002
|
+
killLiveAgent(deps, slot);
|
|
3695
4003
|
journalEvent(deps, 'external-advance', unit(batchId), {
|
|
3696
4004
|
issue: run.issue,
|
|
3697
4005
|
detail: 'member review done',
|
|
@@ -3714,6 +4022,17 @@ function reconcileParallelMemberSlot(deps, config, dispatch, batchId, run, slot,
|
|
|
3714
4022
|
result.completed.push(memberUnit);
|
|
3715
4023
|
return;
|
|
3716
4024
|
}
|
|
4025
|
+
// #822: re-prompt once, then evict — see the serial rail.
|
|
4026
|
+
const parallelBatch = (0, state_1.findBatch)(state0, batchId);
|
|
4027
|
+
if (parallelBatch &&
|
|
4028
|
+
wrongProcedureCounts(parallelBatch, run.issue, read) &&
|
|
4029
|
+
read.milestone !== null) {
|
|
4030
|
+
const decision = decideWrongProcedure(deps, batchId, run.issue, read.milestone, slot, release, () => recordMemberRunLog(deps, dispatch, state0, batchId, run.index, run.issue, slot, now), now);
|
|
4031
|
+
if (typeof decision === 'object') {
|
|
4032
|
+
evictParallelMember(deps, config, batchId, run, decision, now, result);
|
|
4033
|
+
}
|
|
4034
|
+
return;
|
|
4035
|
+
}
|
|
3717
4036
|
if (read.dead &&
|
|
3718
4037
|
!read.blockedNow &&
|
|
3719
4038
|
handleDeadDispatchApiError(deps, batchId, (0, dispatch_1.batchMemberLogPath)(deps.store.runsDir, batchId, run.index, run.issue), slot.log_offset_at_spawn ?? 0, now, { releaseWith: release })) {
|
|
@@ -4024,9 +4343,7 @@ function stopMemberAgent(deps, batchId, issue, now) {
|
|
|
4024
4343
|
const slot = (0, state_1.slotForBatchMember)(deps.store.load(), batchId, issue);
|
|
4025
4344
|
if (slot === undefined)
|
|
4026
4345
|
return;
|
|
4027
|
-
|
|
4028
|
-
deps.spawnDeps.kill(slot.pid, slot.pid_start ?? undefined);
|
|
4029
|
-
}
|
|
4346
|
+
killLiveAgent(deps, slot);
|
|
4030
4347
|
deps.store.withLock((s) => ({
|
|
4031
4348
|
state: (0, state_1.releaseBatchMemberSlot)(s, batchId, issue, now),
|
|
4032
4349
|
result: undefined,
|
|
@@ -4158,7 +4475,7 @@ function runBatchTick(deps, config, dispatch) {
|
|
|
4158
4475
|
if (!fresh || types_1.TERMINAL_BATCH_STATUSES.has(fresh.status) || fresh.status === 'blocked') {
|
|
4159
4476
|
return { state, result: undefined };
|
|
4160
4477
|
}
|
|
4161
|
-
const blocked = (0, state_1.transitionBatch)(state, batchId, 'blocked', { blocked_reason: 'reconcile-error' }, now);
|
|
4478
|
+
const blocked = (0, state_1.transitionBatch)(state, batchId, 'blocked', { blocked_reason: 'reconcile-error', agent_exits: null }, now);
|
|
4162
4479
|
return { state: (0, state_1.releaseAllBatchSlots)(blocked, batchId, now), result: undefined };
|
|
4163
4480
|
});
|
|
4164
4481
|
journalEvent(deps, 'unit-failed', unit(batchId), { reason: 'reconcile-error', detail });
|
|
@@ -4272,10 +4589,10 @@ function runBatchTick(deps, config, dispatch) {
|
|
|
4272
4589
|
reconcileFixSlot(deps, config, batch.id, slot, now, result);
|
|
4273
4590
|
}
|
|
4274
4591
|
else if (batch.status === 'reviewing' || batch.status === 'shipping') {
|
|
4275
|
-
reconcileTailSlot(deps, batch.id, slot, now, result);
|
|
4592
|
+
reconcileTailSlot(deps, config, batch.id, slot, now, result);
|
|
4276
4593
|
}
|
|
4277
4594
|
else if (batch.status === 'deployed') {
|
|
4278
|
-
reconcileReportSlot(deps, batch.id, slot, now, result);
|
|
4595
|
+
reconcileReportSlot(deps, config, batch.id, slot, now, result);
|
|
4279
4596
|
}
|
|
4280
4597
|
continue;
|
|
4281
4598
|
}
|