@ai-dossier/sched 0.47.0 → 0.48.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +103 -13
- package/dist/anchor-close.d.ts.map +1 -1
- package/dist/anchor-close.js +2 -1
- package/dist/anchor-close.js.map +1 -1
- package/dist/batch-dispatch.d.ts +7 -1
- package/dist/batch-dispatch.d.ts.map +1 -1
- package/dist/batch-dispatch.js +234 -89
- package/dist/batch-dispatch.js.map +1 -1
- package/dist/dispatch.d.ts +17 -0
- package/dist/dispatch.d.ts.map +1 -1
- package/dist/dispatch.js +31 -0
- package/dist/dispatch.js.map +1 -1
- package/dist/engine.d.ts.map +1 -1
- package/dist/engine.js +10 -2
- package/dist/engine.js.map +1 -1
- package/dist/index.d.ts +7 -7
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +22 -4
- package/dist/index.js.map +1 -1
- package/dist/recovery.d.ts +39 -3
- package/dist/recovery.d.ts.map +1 -1
- package/dist/recovery.js +206 -46
- package/dist/recovery.js.map +1 -1
- package/dist/scheduler.d.ts +15 -1
- package/dist/scheduler.d.ts.map +1 -1
- package/dist/scheduler.js +26 -0
- package/dist/scheduler.js.map +1 -1
- package/dist/state.d.ts +69 -8
- package/dist/state.d.ts.map +1 -1
- package/dist/state.js +177 -14
- package/dist/state.js.map +1 -1
- package/dist/status.d.ts +107 -5
- package/dist/status.d.ts.map +1 -1
- package/dist/status.js +291 -4
- package/dist/status.js.map +1 -1
- package/dist/teardown.d.ts +9 -0
- package/dist/teardown.d.ts.map +1 -1
- package/dist/teardown.js +19 -10
- package/dist/teardown.js.map +1 -1
- package/dist/types.d.ts +45 -5
- package/dist/types.d.ts.map +1 -1
- package/dist/types.js +20 -2
- package/dist/types.js.map +1 -1
- package/package.json +1 -1
package/dist/batch-dispatch.js
CHANGED
|
@@ -142,8 +142,11 @@ function unit(batchId) {
|
|
|
142
142
|
*/
|
|
143
143
|
function sanitizeUntrustedText(value) {
|
|
144
144
|
return (value
|
|
145
|
+
// C0 + DEL + C1 (incl. the single-byte CSI \u009B) and the bidi
|
|
146
|
+
// overrides/isolates (#810: the reason now also renders in `sched status`'s
|
|
147
|
+
// Parked members rows).
|
|
145
148
|
// biome-ignore lint/suspicious/noControlCharactersInRegex: flattening control characters is the point
|
|
146
|
-
.replace(/[\u0000-\u001F\u007F]/g, ' ')
|
|
149
|
+
.replace(/[\u0000-\u001F\u007F-\u009F\u202A-\u202E\u2066-\u2069]/g, ' ')
|
|
147
150
|
.slice(0, 200));
|
|
148
151
|
}
|
|
149
152
|
/**
|
|
@@ -351,6 +354,31 @@ function applyBatchAndIssues(fresh, computed, batchId, issues, membersBeforeReco
|
|
|
351
354
|
});
|
|
352
355
|
return { ...fresh, batches, entries };
|
|
353
356
|
}
|
|
357
|
+
/**
|
|
358
|
+
* #810: land a `dissolveBatch` outcome — the one tail every dissolve caller
|
|
359
|
+
* shares. Re-applies the batch plus every entry the dissolve touched
|
|
360
|
+
* (requeued AND parked — a parked entry left out here would silently revert
|
|
361
|
+
* to its pre-dissolve status). A dissolved batch is torn down; a dissolve
|
|
362
|
+
* REFUSED over validated members (`outcome.blocked`) keeps its worktree for
|
|
363
|
+
* the operator, but its member agents are stopped and its slots released —
|
|
364
|
+
* nothing reconciles a blocked batch's member slots.
|
|
365
|
+
*/
|
|
366
|
+
function applyDissolveOutcome(deps, batchId, outcome, now, membersBeforeRecovery) {
|
|
367
|
+
deps.store.withLock((s) => ({
|
|
368
|
+
state: applyBatchAndIssues(s, outcome.state, batchId, [...outcome.requeued, ...outcome.parked], membersBeforeRecovery),
|
|
369
|
+
result: undefined,
|
|
370
|
+
}));
|
|
371
|
+
if (outcome.blocked) {
|
|
372
|
+
stopAndReleaseBlocked(deps, batchId, now);
|
|
373
|
+
return;
|
|
374
|
+
}
|
|
375
|
+
teardownBatch(deps, batchId);
|
|
376
|
+
}
|
|
377
|
+
/** Stop every live agent of a batch that just blocked, and release all its slots (#810). */
|
|
378
|
+
function stopAndReleaseBlocked(deps, batchId, now) {
|
|
379
|
+
stopRunningMembers(deps, batchId, now);
|
|
380
|
+
deps.store.withLock((s) => ({ state: (0, state_1.releaseAllBatchSlots)(s, batchId, now), result: undefined }));
|
|
381
|
+
}
|
|
354
382
|
/**
|
|
355
383
|
* Run the aggregate suite, treating a THROWING runner as a red suite with no
|
|
356
384
|
* failing tests — `recovery.ts`'s own internal `runSuite` wrapper already
|
|
@@ -765,8 +793,9 @@ function landMemberBranch(deps, batchId, now) {
|
|
|
765
793
|
* dangling pool entry — the corrupted-entry state `worktree-pool status`
|
|
766
794
|
* reports), cold trees are removed. The member branch is deleted locally —
|
|
767
795
|
* `-d` after a landing (fully merged into the integration branch), `-D` on
|
|
768
|
-
* the eviction path (its commits never landed;
|
|
769
|
-
*
|
|
796
|
+
* the eviction path (its commits never landed; since #810 the parked entry
|
|
797
|
+
* records the branch and an operator requeue continues from the REMOTE copy).
|
|
798
|
+
* The REMOTE member branch is deleted only on the landed path, and
|
|
770
799
|
* deliberately KEPT on eviction — the pushed sha is the evicted work's only
|
|
771
800
|
* recoverable copy.
|
|
772
801
|
*
|
|
@@ -1488,11 +1517,7 @@ function runValidate(deps, config, dispatch, batchId, now, result) {
|
|
|
1488
1517
|
const { state: attributed, outcome } = (0, recovery_1.beginAttribution)(state, batchId, { failing: suite.failing, footprints: memberFootprints(deps, batch) }, rDeps);
|
|
1489
1518
|
if (outcome.offenders.length === 0) {
|
|
1490
1519
|
const dissolve = (0, recovery_1.dissolveBatch)(attributed, batchId, { strategy: 'full', reason: 'unattributable-suite-failure' }, rDeps);
|
|
1491
|
-
deps.
|
|
1492
|
-
state: applyBatchAndIssues(s, dissolve.state, batchId, dissolve.requeued, batch.members),
|
|
1493
|
-
result: undefined,
|
|
1494
|
-
}));
|
|
1495
|
-
teardownBatch(deps, batchId);
|
|
1520
|
+
applyDissolveOutcome(deps, batchId, dissolve, now, batch.members);
|
|
1496
1521
|
result.blocked.push(...dissolve.requeued);
|
|
1497
1522
|
result.failed.push(unit(batchId));
|
|
1498
1523
|
return;
|
|
@@ -1554,10 +1579,14 @@ function evictOffender(deps, config, batchId, offender, attribution, now, result
|
|
|
1554
1579
|
const rDeps = recoveryDeps(deps, config, batch, now);
|
|
1555
1580
|
const outcome = (0, recovery_1.evictMembers)(state, batchId, { issues: [offender], reason: 'suite-red-after-fix', attribution, ranges: batch.ranges }, rDeps);
|
|
1556
1581
|
deps.store.withLock((s) => ({
|
|
1557
|
-
state: applyBatchAndIssues(s, outcome.state, batchId, outcome.requeued, batch.members),
|
|
1582
|
+
state: applyBatchAndIssues(s, outcome.state, batchId, [...outcome.requeued, ...(outcome.dissolve?.parked ?? [])], batch.members),
|
|
1558
1583
|
result: undefined,
|
|
1559
1584
|
}));
|
|
1560
|
-
|
|
1585
|
+
// #810: `dissolve.blocked` — a full dissolve refused over validated
|
|
1586
|
+
// members; the batch is blocked for an operator, not validating.
|
|
1587
|
+
if (outcome.dissolved || outcome.dissolve?.blocked === true) {
|
|
1588
|
+
if (outcome.dissolve?.blocked === true)
|
|
1589
|
+
stopAndReleaseBlocked(deps, batchId, now);
|
|
1561
1590
|
result.failed.push(unit(batchId));
|
|
1562
1591
|
return;
|
|
1563
1592
|
}
|
|
@@ -1675,10 +1704,10 @@ function advanceMemberOrValidate(deps, config, dispatch, batchId, memberCount, c
|
|
|
1675
1704
|
* both always read state fresh.
|
|
1676
1705
|
*/
|
|
1677
1706
|
function evictMemberAndContinue(deps, config, dispatch, batchId, batch, memberIssue, failure, now, result) {
|
|
1678
|
-
const {
|
|
1707
|
+
const { batchEnded, duplicate } = evictMemberDirectly(deps, config, batchId, memberIssue, failure, now);
|
|
1679
1708
|
if (duplicate)
|
|
1680
1709
|
return;
|
|
1681
|
-
if (
|
|
1710
|
+
if (batchEnded) {
|
|
1682
1711
|
result.failed.push(unit(batchId));
|
|
1683
1712
|
return;
|
|
1684
1713
|
}
|
|
@@ -2089,8 +2118,8 @@ function resumeBlockedGate(deps, config, dispatch, batchId, now) {
|
|
|
2089
2118
|
* Exactly-once per dispatch, mirroring `recordDispatchRunLog`'s own
|
|
2090
2119
|
* invariant: called from both of `reconcileMemberSlot`'s exit branches
|
|
2091
2120
|
* (member complete, member blocked/dead) — a batch never redispatches the
|
|
2092
|
-
* same member slot (
|
|
2093
|
-
*
|
|
2121
|
+
* same member slot (an evicted member is parked, #810, and any later
|
|
2122
|
+
* full-cycle run is an independent unit), so unlike `engine.ts`'s per-unit log, a member's log file is
|
|
2094
2123
|
* always one-shot and reading from offset 0 is always correct.
|
|
2095
2124
|
*/
|
|
2096
2125
|
function recordMemberRunLog(deps, dispatch, state, batchId,
|
|
@@ -2267,6 +2296,10 @@ function memberFailureFor(read, lastTool) {
|
|
|
2267
2296
|
: 'member-blocked';
|
|
2268
2297
|
return {
|
|
2269
2298
|
reason,
|
|
2299
|
+
// #810: a block THIS dispatch posted is the member's own hand-back — parked
|
|
2300
|
+
// `handed-back`, never counted toward the dissolve threshold. A member that
|
|
2301
|
+
// died without one is an engine-decided eviction (`agent-exited-unverified`).
|
|
2302
|
+
kind: read.blockedNow ? 'handed-back' : 'evicted',
|
|
2270
2303
|
// This rail covers a self-reported block AND a member that simply died, so the
|
|
2271
2304
|
// detail must follow the RESOLVED reason — `member blocked` on an
|
|
2272
2305
|
// `agent-exited-unverified` eviction contradicts the reason on its own journal line.
|
|
@@ -2339,58 +2372,84 @@ function reconcileMemberSlot(deps, config, dispatch, batchId, slot, now, result)
|
|
|
2339
2372
|
}
|
|
2340
2373
|
}
|
|
2341
2374
|
/**
|
|
2342
|
-
*
|
|
2343
|
-
* the batch past its `dissolve_policy` threshold (#563;
|
|
2344
|
-
* going through `recovery.ts`'s `evictMembers` (which
|
|
2345
|
-
* `attributing`/`evicting` status and a commit range to revert; a
|
|
2346
|
-
* evicted here has neither). Returns whether the batch dissolved.
|
|
2375
|
+
* Park a member that left the batch before landing, record the exit, and
|
|
2376
|
+
* dissolve if this tips the batch past its `dissolve_policy` threshold (#563;
|
|
2377
|
+
* RFC F.1/F.8) — WITHOUT going through `recovery.ts`'s `evictMembers` (which
|
|
2378
|
+
* needs `attributing`/`evicting` status and a commit range to revert; a
|
|
2379
|
+
* member evicted here has neither). Returns whether the batch dissolved.
|
|
2380
|
+
*
|
|
2381
|
+
* #810 — the member is PARKED, never auto-requeued:
|
|
2382
|
+
* - `failure.kind === 'handed-back'` (the member posted its own `blocked` /
|
|
2383
|
+
* `review partial` milestone) → entry `handed-back`, record
|
|
2384
|
+
* `kind: 'handed-back'`, journal `member-handed-back`. A hand-back is a
|
|
2385
|
+
* valued outcome, not a batch failure: it never counts toward the dissolve
|
|
2386
|
+
* threshold.
|
|
2387
|
+
* - otherwise (unverified exit, gate task-failed, landing conflict, worktree
|
|
2388
|
+
* prep) → entry `evicted`, journal `unit-failed`, counted.
|
|
2389
|
+
* Either way the member branch is recorded on the record and the entry's
|
|
2390
|
+
* `failure_evidence`, and the batch's dispatch profile on the entry — so an
|
|
2391
|
+
* operator's `sched requeue` continues the work from that branch on the same
|
|
2392
|
+
* provider. The pre-#810 rail requeued `mode: 'full', dispatch_profile: null`
|
|
2393
|
+
* from the base branch, discarding the member's commits and its profile.
|
|
2394
|
+
*
|
|
2395
|
+
* #810: when the threshold trips while members are already VALIDATED (landed
|
|
2396
|
+
* on the integration branch), the batch does not dissolve — dissolving would
|
|
2397
|
+
* requeue validated work from scratch. It journals `dissolve-suppressed` and
|
|
2398
|
+
* continues with the remaining members; its validated members still ship.
|
|
2347
2399
|
*
|
|
2348
2400
|
* A member already in `evictions[]` is a NO-OP here, not merely a skipped
|
|
2349
|
-
* record (#595 AC1): the
|
|
2350
|
-
* overwrite the first eviction's `failure_evidence` — leaving `evictions[]`
|
|
2351
|
-
* and the queue entry disagreeing about why the member was evicted — and, if
|
|
2352
|
-
* the member has since been re-dispatched full-cycle, would take the
|
|
2353
|
-
* `executing → evicted → requeued` rail and kill that live run.
|
|
2401
|
+
* record (#595 AC1): the park must not run a second time either.
|
|
2354
2402
|
*
|
|
2355
|
-
* Returns `{
|
|
2356
|
-
*
|
|
2357
|
-
*
|
|
2358
|
-
*
|
|
2359
|
-
*
|
|
2403
|
+
* Returns `{ batchEnded, duplicate }`: `batchEnded` is whether this exit ended
|
|
2404
|
+
* the batch's run — it dissolved, or (a `full` dissolve refused over validated
|
|
2405
|
+
* members, #810) it blocked for an operator; either way the caller must not
|
|
2406
|
+
* advance it. `duplicate: true` means another resolution had already claimed
|
|
2407
|
+
* this member's exit, and the caller must journal nothing and advance nothing
|
|
2408
|
+
* on top of it (#613). A dissolve interrupted between pass 1 and pass 2 is not
|
|
2409
|
+
* retried by a re-entry: the re-entry takes the duplicate path (pre-existing,
|
|
2410
|
+
* #595/#613) and the batch continues.
|
|
2360
2411
|
*/
|
|
2361
2412
|
function evictMemberDirectly(deps, config, batchId, memberIssue, failure, now) {
|
|
2362
2413
|
const { reason } = failure;
|
|
2414
|
+
const kind = failure.kind ?? 'evicted';
|
|
2363
2415
|
const dissolvePolicy = (0, types_1.resolveDissolvePolicy)(config.dissolve_policy);
|
|
2364
|
-
// Pass 1 (pure —
|
|
2365
|
-
//
|
|
2416
|
+
// Pass 1 (pure — park + record the exit): safe to run entirely inside the
|
|
2417
|
+
// lock, unlike `dissolveBatch` below, which shells out
|
|
2366
2418
|
// (`deps.exec`/`postMilestone`) and so must NOT hold the lock while it runs.
|
|
2367
|
-
const { triggered, duplicate, prior } = deps.store.withLock((s) => {
|
|
2419
|
+
const { triggered, duplicate, prior, branch, validated } = deps.store.withLock((s) => {
|
|
2368
2420
|
const b = (0, state_1.findBatch)(s, batchId);
|
|
2369
2421
|
if (!b) {
|
|
2370
|
-
return {
|
|
2422
|
+
return {
|
|
2423
|
+
state: s,
|
|
2424
|
+
result: {
|
|
2425
|
+
triggered: false,
|
|
2426
|
+
duplicate: false,
|
|
2427
|
+
prior: undefined,
|
|
2428
|
+
branch: null,
|
|
2429
|
+
validated: [],
|
|
2430
|
+
},
|
|
2431
|
+
};
|
|
2371
2432
|
}
|
|
2372
2433
|
const priorRecord = b.evictions.find((e) => e.issue === memberIssue);
|
|
2373
2434
|
if (priorRecord) {
|
|
2374
2435
|
// A duplicate never re-evaluates the dissolve threshold: the resolution that WROTE
|
|
2375
2436
|
// this record already did, under this same lock. (#595: a duplicate must not count
|
|
2376
|
-
// twice; #613: it must not act on the batch at all.)
|
|
2377
|
-
// interrupted between pass 1 and pass 2 is not retried here — re-entering
|
|
2378
|
-
// `dissolveBatch` on an already-terminal batch throws `IllegalTransitionError`.
|
|
2437
|
+
// twice; #613: it must not act on the batch at all.)
|
|
2379
2438
|
return {
|
|
2380
2439
|
state: s,
|
|
2381
|
-
result: {
|
|
2440
|
+
result: {
|
|
2441
|
+
triggered: false,
|
|
2442
|
+
duplicate: true,
|
|
2443
|
+
prior: priorRecord,
|
|
2444
|
+
branch: null,
|
|
2445
|
+
validated: [],
|
|
2446
|
+
},
|
|
2382
2447
|
};
|
|
2383
2448
|
}
|
|
2384
|
-
const
|
|
2385
|
-
|
|
2386
|
-
reason,
|
|
2387
|
-
|
|
2388
|
-
attribution: 'none',
|
|
2389
|
-
reverted_commits: [],
|
|
2390
|
-
at: now.toISOString(),
|
|
2391
|
-
};
|
|
2392
|
-
const requeueResult = (0, state_1.requeueMember)(s, memberIssue, { mode: 'full', batch: null }, reason, now, { failure_evidence: evidence });
|
|
2393
|
-
let next = requeueResult.state;
|
|
2449
|
+
const memberBranch = (0, state_1.recordedMemberBranch)(b, memberIssue);
|
|
2450
|
+
let next = (0, state_1.parkMember)(s, memberIssue, kind, reason, now, {
|
|
2451
|
+
failure_evidence: (0, state_1.memberExitEvidence)(batchId, reason, memberBranch, now),
|
|
2452
|
+
}).state;
|
|
2394
2453
|
// `appendEvictions` stays the state-level backstop even though the
|
|
2395
2454
|
// duplicate is already short-circuited above — it is the one append path.
|
|
2396
2455
|
const { evictions, duplicate: duplicateRecords } = (0, state_1.appendEvictions)(b, [
|
|
@@ -2400,6 +2459,8 @@ function evictMemberDirectly(deps, config, batchId, memberIssue, failure, now) {
|
|
|
2400
2459
|
attribution: 'none',
|
|
2401
2460
|
reverted_commits: [],
|
|
2402
2461
|
group: [],
|
|
2462
|
+
kind,
|
|
2463
|
+
branch: memberBranch,
|
|
2403
2464
|
at: now.toISOString(),
|
|
2404
2465
|
},
|
|
2405
2466
|
]);
|
|
@@ -2411,6 +2472,8 @@ function evictMemberDirectly(deps, config, batchId, memberIssue, failure, now) {
|
|
|
2411
2472
|
triggered: updated !== undefined && (0, recovery_1.checkDissolveTrigger)(updated, dissolvePolicy),
|
|
2412
2473
|
duplicate: duplicateRecords.length > 0,
|
|
2413
2474
|
prior: undefined,
|
|
2475
|
+
branch: memberBranch,
|
|
2476
|
+
validated: updated !== undefined ? (0, state_1.validatedMembersOf)(next, updated) : [],
|
|
2414
2477
|
},
|
|
2415
2478
|
};
|
|
2416
2479
|
});
|
|
@@ -2419,41 +2482,45 @@ function evictMemberDirectly(deps, config, batchId, memberIssue, failure, now) {
|
|
|
2419
2482
|
issue: memberIssue,
|
|
2420
2483
|
detail: (0, state_1.duplicateEvictionDetail)(memberIssue, reason, prior),
|
|
2421
2484
|
});
|
|
2422
|
-
// Another resolution already claimed this member's
|
|
2485
|
+
// Another resolution already claimed this member's exit record — the
|
|
2423
2486
|
// caller must not journal its own `unit-failed`/advance the batch again
|
|
2424
|
-
// on top of that one (#613
|
|
2425
|
-
|
|
2426
|
-
return { dissolved: false, duplicate: true };
|
|
2487
|
+
// on top of that one (#613).
|
|
2488
|
+
return { batchEnded: false, duplicate: true };
|
|
2427
2489
|
}
|
|
2428
2490
|
// This call owns the member's resolution, so it owns recording WHY — before pass 2 below
|
|
2429
|
-
// can dissolve, tear the worktree down, or be killed mid-shell-out and leave the
|
|
2430
|
-
//
|
|
2431
|
-
//
|
|
2432
|
-
|
|
2433
|
-
// identity.
|
|
2434
|
-
journalEvent(deps, 'unit-failed', unit(batchId), {
|
|
2491
|
+
// can dissolve, tear the worktree down, or be killed mid-shell-out and leave the record
|
|
2492
|
+
// on disk with no journal line naming its cause (#613). The caller's extras go first so
|
|
2493
|
+
// the authoritative keys always win.
|
|
2494
|
+
journalEvent(deps, kind === 'handed-back' ? 'member-handed-back' : 'unit-failed', unit(batchId), {
|
|
2435
2495
|
...(failure.extraKv ?? {}),
|
|
2436
2496
|
issue: memberIssue,
|
|
2437
2497
|
reason,
|
|
2438
|
-
detail: failure.detail
|
|
2498
|
+
detail: `${failure.detail} — parked ${kind}` +
|
|
2499
|
+
(branch !== null ? ` (work on ${branch})` : '') +
|
|
2500
|
+
`; remedy: sched requeue --issue ${memberIssue}`,
|
|
2439
2501
|
});
|
|
2440
2502
|
if (!triggered)
|
|
2441
|
-
return {
|
|
2503
|
+
return { batchEnded: false, duplicate: false };
|
|
2504
|
+
if (validated.length > 0) {
|
|
2505
|
+
// #810: never dissolve over validated work — it stays landed and ships
|
|
2506
|
+
// with the batch; the members that failed are parked, not requeued.
|
|
2507
|
+
journalEvent(deps, 'dissolve-suppressed', unit(batchId), {
|
|
2508
|
+
issue: memberIssue,
|
|
2509
|
+
detail: `eviction threshold reached, but member(s) ${validated.join(',')} are validated on the integration branch — keeping them and continuing with the remaining members instead of dissolving`,
|
|
2510
|
+
});
|
|
2511
|
+
return { batchEnded: false, duplicate: false };
|
|
2512
|
+
}
|
|
2442
2513
|
// Pass 2 (outside the lock — dissolveBatch shells out): re-load fresh
|
|
2443
2514
|
// (pass 1's write already landed), dissolve, then re-apply just this
|
|
2444
2515
|
// batch's + the requeued members' state under a fresh lock.
|
|
2445
2516
|
const state = deps.store.load();
|
|
2446
2517
|
const batch = (0, state_1.findBatch)(state, batchId);
|
|
2447
2518
|
if (!batch)
|
|
2448
|
-
return {
|
|
2519
|
+
return { batchEnded: false, duplicate: false };
|
|
2449
2520
|
const rDeps = recoveryDeps(deps, config, batch, now);
|
|
2450
2521
|
const outcome = (0, recovery_1.dissolveBatch)(state, batchId, { strategy: 'full', reason: 'eviction-threshold' }, rDeps);
|
|
2451
|
-
deps
|
|
2452
|
-
|
|
2453
|
-
result: undefined,
|
|
2454
|
-
}));
|
|
2455
|
-
teardownBatch(deps, batchId);
|
|
2456
|
-
return { dissolved: true, duplicate: false };
|
|
2522
|
+
applyDissolveOutcome(deps, batchId, outcome, now);
|
|
2523
|
+
return { batchEnded: true, duplicate: false };
|
|
2457
2524
|
}
|
|
2458
2525
|
function reconcileFixSlot(deps, config, batchId, slot, now, result) {
|
|
2459
2526
|
if (slot.pid !== null && deps.spawnDeps.isAlive(slot.pid, slot.pid_start ?? undefined))
|
|
@@ -2725,8 +2792,8 @@ const MEMBER_RECONCILE_CHAIN = {
|
|
|
2725
2792
|
* Whether every SURVIVING member of `batch` was actually dispatched — i.e. no
|
|
2726
2793
|
* surviving member is still sitting in a pre-dispatch status (`queued`,
|
|
2727
2794
|
* `classified`, `batched`, `waiting`). Surviving = a `batch.members` issue
|
|
2728
|
-
* that is still a slot-mode entry, not evicted
|
|
2729
|
-
*
|
|
2795
|
+
* that is still a slot-mode entry, not evicted / handed back (parked, #810),
|
|
2796
|
+
* and not already preserved (shipped/terminal through
|
|
2730
2797
|
* another rail).
|
|
2731
2798
|
*
|
|
2732
2799
|
* This is the safety rail behind AC1: a member that never ran has NO work on
|
|
@@ -2744,8 +2811,14 @@ function everySurvivingMemberDispatched(state, batch) {
|
|
|
2744
2811
|
const entry = (0, state_1.findEntry)(state, issue);
|
|
2745
2812
|
if (!entry)
|
|
2746
2813
|
continue;
|
|
2747
|
-
|
|
2814
|
+
// #810: a PARKED member (evicted / handed back) left the batch before
|
|
2815
|
+
// landing — not a survivor, whether or not it has an `evictions[]` record.
|
|
2816
|
+
if (entry.mode !== 'slot' ||
|
|
2817
|
+
evicted.has(issue) ||
|
|
2818
|
+
(0, state_1.isPreservedMember)(entry) ||
|
|
2819
|
+
state_1.PARKED_MEMBER_STATUSES.has(entry.status)) {
|
|
2748
2820
|
continue;
|
|
2821
|
+
}
|
|
2749
2822
|
// Pre-dispatch statuses: queued/classified/batched/waiting. `in-work` is
|
|
2750
2823
|
// stamped at SPAWN (`spawnMember`), `committed`/`validated` at verified
|
|
2751
2824
|
// completion — everything the entry can reach afterwards means the
|
|
@@ -2951,6 +3024,72 @@ function reconcileStaleBlockedBatches(deps, now, result, anchorCtx) {
|
|
|
2951
3024
|
}
|
|
2952
3025
|
}
|
|
2953
3026
|
}
|
|
3027
|
+
/**
|
|
3028
|
+
* #791: once a `done` batch's kept worktree (left in place by the
|
|
3029
|
+
* `members-closed` branch above) is DEFINITIVELY gone — the path no longer
|
|
3030
|
+
* exists on disk, or the pool itself now reports it back as a warm spare —
|
|
3031
|
+
* clear the ledger's `worktree`/`pool_claimed` (and, separately,
|
|
3032
|
+
* `member_worktree`/`member_pool_claimed`) fields so `sched status`'s
|
|
3033
|
+
* `kept-worktree` warning does not persist after the operator's cleanup
|
|
3034
|
+
* already happened. Ledger-only, by construction: this function reads
|
|
3035
|
+
* `fsExists` and `worktree-pool status --json` (both read-only) and never
|
|
3036
|
+
* builds a `git worktree remove` / `worktree-pool return` argv — the actual
|
|
3037
|
+
* removal stays entirely the operator's call, exactly like the warning
|
|
3038
|
+
* itself. Bounded the same way the warning's own candidate scan is: only
|
|
3039
|
+
* `done` batches carrying a kept worktree are considered, which in practice
|
|
3040
|
+
* is a small set.
|
|
3041
|
+
*/
|
|
3042
|
+
function reconcileKeptWorktrees(deps, now) {
|
|
3043
|
+
const state = deps.store.load();
|
|
3044
|
+
const fsExists = deps.fsExists ?? ((p) => fs.existsSync(p));
|
|
3045
|
+
const candidates = state.batches.filter((b) => b.status === 'done' && (b.worktree !== null || b.member_worktree !== null));
|
|
3046
|
+
if (candidates.length === 0)
|
|
3047
|
+
return;
|
|
3048
|
+
const needsPoolCheck = candidates.some((b) => (b.worktree !== null && b.pool_claimed) ||
|
|
3049
|
+
(b.member_worktree !== null && b.member_pool_claimed));
|
|
3050
|
+
// One pool query per TICK, not per candidate — `worktree-pool status` is a
|
|
3051
|
+
// real subprocess (`npx ...`), and this reconcile is meant to be cheap.
|
|
3052
|
+
const poolStatusJson = needsPoolCheck
|
|
3053
|
+
? deps.exec(teardown_1.POOL_BIN, [...teardown_1.POOL_ARGS_PREFIX, 'status', '--json'], deps.repoDir)
|
|
3054
|
+
: null;
|
|
3055
|
+
const goneOrReturned = (worktree, poolClaimed) => {
|
|
3056
|
+
if (!fsExists(worktree))
|
|
3057
|
+
return true;
|
|
3058
|
+
if (poolClaimed && poolStatusJson !== null) {
|
|
3059
|
+
return (0, teardown_1.poolEntryFor)(poolStatusJson, worktree) === 'warm';
|
|
3060
|
+
}
|
|
3061
|
+
return false;
|
|
3062
|
+
};
|
|
3063
|
+
for (const batch of candidates) {
|
|
3064
|
+
const clears = {};
|
|
3065
|
+
const clearedFields = [];
|
|
3066
|
+
if (batch.worktree !== null && goneOrReturned(batch.worktree, batch.pool_claimed)) {
|
|
3067
|
+
clears.worktree = null;
|
|
3068
|
+
clears.pool_claimed = false;
|
|
3069
|
+
clearedFields.push('worktree', 'pool_claimed');
|
|
3070
|
+
}
|
|
3071
|
+
if (batch.member_worktree !== null &&
|
|
3072
|
+
goneOrReturned(batch.member_worktree, batch.member_pool_claimed)) {
|
|
3073
|
+
clears.member_worktree = null;
|
|
3074
|
+
clears.member_pool_claimed = false;
|
|
3075
|
+
clearedFields.push('member_worktree', 'member_pool_claimed');
|
|
3076
|
+
}
|
|
3077
|
+
if (clearedFields.length === 0)
|
|
3078
|
+
continue;
|
|
3079
|
+
deps.store.withLock((s) => {
|
|
3080
|
+
const b = (0, state_1.findBatch)(s, batch.id);
|
|
3081
|
+
// Re-check under the lock: another tick, or an operator command, may
|
|
3082
|
+
// have already moved this batch or cleared the fields itself.
|
|
3083
|
+
if (!b || b.status !== 'done')
|
|
3084
|
+
return { state: s, result: undefined };
|
|
3085
|
+
return { state: (0, state_1.patchBatch)(s, batch.id, clears, now, false), result: undefined };
|
|
3086
|
+
});
|
|
3087
|
+
journalEvent(deps, 'kept-worktree-cleared', unit(batch.id), {
|
|
3088
|
+
cleared: clearedFields.join(','),
|
|
3089
|
+
detail: `kept-worktree ledger field(s) cleared on definitive evidence (${clearedFields.join(', ')}) — path gone or pool reports it returned; no removal command was run`,
|
|
3090
|
+
});
|
|
3091
|
+
}
|
|
3092
|
+
}
|
|
2954
3093
|
function anchorTickContext(deps) {
|
|
2955
3094
|
const reader = (0, groundtruth_1.issueCloseReader)(deps.groundTruth);
|
|
2956
3095
|
if (reader === undefined || deps.anchorRepo === undefined)
|
|
@@ -3230,11 +3369,11 @@ function upsertRun(state, batchId, run, now) {
|
|
|
3230
3369
|
return (0, state_1.patchBatch)(state, batchId, { member_runs: [...others, run].sort((a, b) => a.index - b.index) }, now);
|
|
3231
3370
|
}
|
|
3232
3371
|
/**
|
|
3233
|
-
* Whether `issue` is still an active member of `batch` — not evicted
|
|
3234
|
-
*
|
|
3235
|
-
*
|
|
3236
|
-
*
|
|
3237
|
-
* from under the batch.
|
|
3372
|
+
* Whether `issue` is still an active member of `batch` — not evicted / handed back
|
|
3373
|
+
* (its `evictions[]` record — a #810 parked entry keeps `mode: 'slot'` and its
|
|
3374
|
+
* batch, so the record is what takes it out), and its queue entry (when it has
|
|
3375
|
+
* one) still a live slot-mode entry of THIS batch — which also covers a member
|
|
3376
|
+
* stopped, abandoned or operator-requeued out from under the batch.
|
|
3238
3377
|
*/
|
|
3239
3378
|
function memberStillInBatch(state, batch, issue) {
|
|
3240
3379
|
if (batch.evictions.some((e) => e.issue === issue))
|
|
@@ -3242,7 +3381,10 @@ function memberStillInBatch(state, batch, issue) {
|
|
|
3242
3381
|
const entry = (0, state_1.findEntry)(state, issue);
|
|
3243
3382
|
if (entry === undefined)
|
|
3244
3383
|
return true;
|
|
3245
|
-
return (entry.mode === 'slot' &&
|
|
3384
|
+
return (entry.mode === 'slot' &&
|
|
3385
|
+
entry.batch === batch.id &&
|
|
3386
|
+
!types_1.TERMINAL_ISSUE_STATUSES.has(entry.status) &&
|
|
3387
|
+
!state_1.PARKED_MEMBER_STATUSES.has(entry.status));
|
|
3246
3388
|
}
|
|
3247
3389
|
/** Whether member `issue`'s own slot currently holds a live agent. */
|
|
3248
3390
|
function hasLiveMemberSlot(state, batchId, issue) {
|
|
@@ -3279,9 +3421,9 @@ function nextParallelCandidate(state, batch, attempted) {
|
|
|
3279
3421
|
* scheduler has free capacity. Each gets its own worktree off the integration
|
|
3280
3422
|
* branch (`prepareMemberWorktree`, outside the lock) — the integration branch
|
|
3281
3423
|
* does not move until members land, so concurrent members are all cut from the
|
|
3282
|
-
* same base. A member whose worktree cannot be prepared is evicted (
|
|
3283
|
-
* full-cycle, which does its own setup)
|
|
3284
|
-
* front of the members behind it.
|
|
3424
|
+
* same base. A member whose worktree cannot be prepared is evicted (parked, #810 —
|
|
3425
|
+
* an operator's `sched requeue` runs it full-cycle, which does its own setup)
|
|
3426
|
+
* rather than retried every tick in front of the members behind it.
|
|
3285
3427
|
*/
|
|
3286
3428
|
function spawnParallelMembers(deps, config, dispatch, batchId, now, result) {
|
|
3287
3429
|
const limit = memberParallelism(config);
|
|
@@ -3381,11 +3523,11 @@ function spawnParallelMembers(deps, config, dispatch, batchId, now, result) {
|
|
|
3381
3523
|
* tips over is torn down by `teardownBatch`, which covers every live run.
|
|
3382
3524
|
*/
|
|
3383
3525
|
function evictParallelMember(deps, config, batchId, run, failure, now, result) {
|
|
3384
|
-
const {
|
|
3526
|
+
const { batchEnded, duplicate } = evictMemberDirectly(deps, config, batchId, run.issue, failure, now);
|
|
3385
3527
|
if (duplicate)
|
|
3386
3528
|
return;
|
|
3387
|
-
result.failed.push(
|
|
3388
|
-
if (
|
|
3529
|
+
result.failed.push(batchEnded ? unit(batchId) : (0, state_1.batchMemberUnit)(batchId, run.issue));
|
|
3530
|
+
if (batchEnded)
|
|
3389
3531
|
return;
|
|
3390
3532
|
deps.store.withLock((s) => ({
|
|
3391
3533
|
state: patchRun(s, batchId, run.issue, { status: 'evicted', gate_inconclusive: null }, now),
|
|
@@ -3716,7 +3858,7 @@ function reconcileParallelBatch(deps, config, dispatch, batchId, now, result, pa
|
|
|
3716
3858
|
/**
|
|
3717
3859
|
* #809: stop and tear down every parallel member whose tree is not yet torn
|
|
3718
3860
|
* down — a dissolve can land while other members are mid-run (their agents are
|
|
3719
|
-
* killed: their entries were just requeued
|
|
3861
|
+
* killed: their entries were just parked or requeued, so a live member agent
|
|
3720
3862
|
* would be working a unit nothing tracks any more), `sched stop`/`abandon`
|
|
3721
3863
|
* make a batch terminal without a teardown, and a crash between a run's
|
|
3722
3864
|
* resolution and its teardown leaves `torn_down: false` behind. Teardown is
|
|
@@ -3799,7 +3941,8 @@ function runBatchTick(deps, config, dispatch) {
|
|
|
3799
3941
|
// - pre-merge (`dissolving` is a legal edge): DISSOLVE with reason
|
|
3800
3942
|
// `dispatch-profile-missing:<name>` — loud, deterministic, one-shot, and
|
|
3801
3943
|
// the members requeue as full-cycle units on the config default, which is
|
|
3802
|
-
// a coherent recovery rather than a mislabelled continuation
|
|
3944
|
+
// a coherent recovery rather than a mislabelled continuation (#810: in-flight
|
|
3945
|
+
// members park on the default; validated members block the batch instead).
|
|
3803
3946
|
// - post-merge (PR parked/merged — the product already shipped): dissolving
|
|
3804
3947
|
// would discard a landed PR's bookkeeping, so the remaining dispatch (the
|
|
3805
3948
|
// report agent) runs on the default and journals
|
|
@@ -3840,12 +3983,13 @@ function runBatchTick(deps, config, dispatch) {
|
|
|
3840
3983
|
deps.spawnDeps.kill(liveSlot.pid, liveSlot.pid_start ?? undefined);
|
|
3841
3984
|
}
|
|
3842
3985
|
}
|
|
3843
|
-
const outcome = (0, recovery_1.dissolveBatch)(current, batch.id, {
|
|
3844
|
-
|
|
3845
|
-
|
|
3846
|
-
|
|
3847
|
-
|
|
3848
|
-
|
|
3986
|
+
const outcome = (0, recovery_1.dissolveBatch)(current, batch.id, {
|
|
3987
|
+
strategy: 'full',
|
|
3988
|
+
reason: `dispatch-profile-missing:${batch.dispatch_profile}`,
|
|
3989
|
+
// #810: the profile is what is broken — requeue on the default.
|
|
3990
|
+
carryDispatchProfile: false,
|
|
3991
|
+
}, recoveryDeps(deps, config, fresh, now));
|
|
3992
|
+
applyDissolveOutcome(deps, batch.id, outcome, now);
|
|
3849
3993
|
result.failed.push(unit(batch.id));
|
|
3850
3994
|
// Resolution failed: clean up any batch/member worktrees, then skip
|
|
3851
3995
|
// this batch for the rest of the pass. No further arm can claim it.
|
|
@@ -4062,6 +4206,7 @@ function runBatchTick(deps, config, dispatch) {
|
|
|
4062
4206
|
const anchorCtx = anchorTickContext(deps);
|
|
4063
4207
|
reconcileStaleBlockedBatches(deps, now, result, anchorCtx);
|
|
4064
4208
|
reconcileAnchorClosure(deps, now, anchorCtx);
|
|
4209
|
+
reconcileKeptWorktrees(deps, now);
|
|
4065
4210
|
return result;
|
|
4066
4211
|
}
|
|
4067
4212
|
//# sourceMappingURL=batch-dispatch.js.map
|