claude-code-session-manager 0.80.0 → 0.82.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/assets/{AgentLibrary-zS3jw_1e.js → AgentLibrary-pwlkAFb3.js} +1 -1
- package/dist/assets/{DataModel-Cy_vxTpi.js → DataModel-BZK9PXFD.js} +1 -1
- package/dist/assets/{History-C6JRuqfT.js → History-BVRjxJjS.js} +1 -1
- package/dist/assets/{Hooks-BafPy9mB.js → Hooks-CNuwHeGx.js} +1 -1
- package/dist/assets/{HostBilko-BZwhQOFt.js → HostBilko-cjwNodhV.js} +1 -1
- package/dist/assets/{Library-C8JDDliz.js → Library-YPNm9W92.js} +1 -1
- package/dist/assets/{ListDetail-CqiOdwLc.js → ListDetail-CY4GM1Om.js} +1 -1
- package/dist/assets/{MarkdownEditor-CyLyP67L.js → MarkdownEditor-BF4y2Jiz.js} +1 -1
- package/dist/assets/{McpServers-BzMv-_84.js → McpServers-CmBdWtX_.js} +1 -1
- package/dist/assets/{Memory-DSBYQdJR.js → Memory-CvkIXNl1.js} +1 -1
- package/dist/assets/{Panel-CLUhkNNA.js → Panel-D93o-sxe.js} +1 -1
- package/dist/assets/{Permissions-BfC2-HN4.js → Permissions-DQipg16I.js} +1 -1
- package/dist/assets/{Plugins-BKi40jT5.js → Plugins-B6NwzPfK.js} +2 -2
- package/dist/assets/{ProvenanceBadge-BzFw4KhD.js → ProvenanceBadge-DACVJhrB.js} +1 -1
- package/dist/assets/{SaveBar-avk2p9jv.js → SaveBar-Cg4lbChb.js} +1 -1
- package/dist/assets/{Scheduler-Bf_6MdJo.js → Scheduler-Dr5ZcBLe.js} +1 -1
- package/dist/assets/{ScopeSwitcher-C-RwYUVZ.js → ScopeSwitcher-C-locvy0.js} +1 -1
- package/dist/assets/{Settings-Djd8OoBA.js → Settings-BVrAle90.js} +1 -1
- package/dist/assets/{SkillReferenceGraph-DuogY6s7.js → SkillReferenceGraph-DDzuYgSK.js} +1 -1
- package/dist/assets/{Skills-D_qAqxZ_.js → Skills-DFvhiOAQ.js} +1 -1
- package/dist/assets/{SystemPrompt-DbHFLQV3.js → SystemPrompt-8PiTyUyL.js} +1 -1
- package/dist/assets/{TagLibrary-C2y91BT0.js → TagLibrary-DruUYaAc.js} +1 -1
- package/dist/assets/{TiptapBody-D9iz4xQx.js → TiptapBody-Dr4a--42.js} +1 -1
- package/dist/assets/{Toggle-BGnFL2E5.js → Toggle-B_EH2TFb.js} +1 -1
- package/dist/assets/{index-_2ARyFDj.js → index-DApB4DHS.js} +4 -4
- package/dist/assets/{settingsSchema-JK15eJU8.js → settingsSchema-BKa-xk8g.js} +1 -1
- package/dist/index.html +1 -1
- package/package.json +1 -1
- package/plugins/session-manager-dev/skills/develop/standards.md +1 -0
- package/scripts/project-pages-logic/dist/logic.cjs +12 -12
- package/scripts/render-project-pages/dist/renderer.cjs +22 -22
- package/src/main/__tests__/machineProfile.test.cjs +134 -0
- package/src/main/__tests__/rcaReport.test.cjs +24 -0
- package/src/main/__tests__/runVerify-blocked-by-foreign-wip.test.cjs +58 -0
- package/src/main/__tests__/runVerify-policy-denial.test.cjs +89 -0
- package/src/main/__tests__/scheduler-already-satisfied-on-main.test.cjs +105 -0
- package/src/main/__tests__/scheduler-blocked-by-foreign-wip.test.cjs +107 -0
- package/src/main/__tests__/scheduler-finalize-dispatch-guards.test.cjs +229 -0
- package/src/main/__tests__/scheduler-looks-done.test.cjs +141 -1
- package/src/main/__tests__/scheduler-periodic-reverify-guard.test.cjs +85 -0
- package/src/main/__tests__/scheduler-rate-limit-pause.test.cjs +81 -0
- package/src/main/__tests__/scheduler-reap-dead-running-jobs.test.cjs +205 -2
- package/src/main/__tests__/telemetrySettings.test.cjs +178 -0
- package/src/main/lib/__tests__/branchSweep.test.cjs +164 -0
- package/src/main/lib/__tests__/fixtures/204-mercury-steam-horse.log.txt +13 -0
- package/src/main/lib/__tests__/landedSinceRun.test.cjs +61 -1
- package/src/main/lib/__tests__/rateLimitWindow.test.cjs +88 -0
- package/src/main/lib/__tests__/reaperHelpers.test.cjs +120 -1
- package/src/main/lib/branchSweep.cjs +127 -0
- package/src/main/lib/gitWorktree.cjs +20 -0
- package/src/main/lib/landedSinceRun.cjs +41 -1
- package/src/main/lib/machineProfile.cjs +144 -0
- package/src/main/lib/rateLimitWindow.cjs +62 -0
- package/src/main/lib/rcaReport.cjs +18 -3
- package/src/main/lib/reaperHelpers.cjs +169 -3
- package/src/main/lib/scheduleJobTransitions.cjs +16 -3
- package/src/main/lib/telemetrySettings.cjs +171 -0
- package/src/main/runVerify.cjs +71 -3
- package/src/main/scheduler.cjs +768 -34
package/src/main/scheduler.cjs
CHANGED
|
@@ -57,9 +57,14 @@ const { resolveClaudeBin, probeClaudeVersion } = require('./lib/claudeBin.cjs');
|
|
|
57
57
|
const launchFailure = require('./lib/launchFailure.cjs');
|
|
58
58
|
const { appendError } = require('./lib/opsErrorLog.cjs');
|
|
59
59
|
const { readTail } = require('./lib/fileTail.cjs');
|
|
60
|
-
const {
|
|
60
|
+
const {
|
|
61
|
+
claudePidAlive, classifyRunOutcome, mapOutcomeToGateOutcome, ORPHAN_REQUEUE_CAP, selectReapableJobs,
|
|
62
|
+
findLiveProcessForJob, logHasOutput, resolvePidlessGateOutcome, resolveCommitGuardOutcome,
|
|
63
|
+
} = require('./lib/reaperHelpers.cjs');
|
|
64
|
+
const { sweepStrandedJobBranches } = require('./lib/branchSweep.cjs');
|
|
61
65
|
const { detectRateLimitInLog } = require('./lib/rateLimitDetect.cjs');
|
|
62
66
|
const { stripAppOwnedChurn } = require('./lib/jobDirtFilter.cjs');
|
|
67
|
+
const { resolveBindingRateLimitReset } = require('./lib/rateLimitWindow.cjs');
|
|
63
68
|
const { computeQueueHealth } = require('./lib/queueHealth.cjs');
|
|
64
69
|
const { createLoadGate, topCpuConsumers } = require('./lib/loadGate.cjs');
|
|
65
70
|
const { openLog, withChildAndLog } = require('./lib/childWithLog.cjs');
|
|
@@ -71,10 +76,10 @@ const { enqueueExternalPrompt } = require('./chatRunner.cjs');
|
|
|
71
76
|
const { appendResponseEventIfKnown } = require('./promptSessionEvents.cjs');
|
|
72
77
|
const { maybeEnqueueValidationPrompt } = require('./lib/epicValidationHook.cjs');
|
|
73
78
|
const promptSessionTranscript = require('./promptSessionTranscript.cjs');
|
|
74
|
-
const { verifyRun } = require('./runVerify.cjs');
|
|
79
|
+
const { verifyRun, parseLog, scanSentinel, scanForeignWipPathsClaim } = require('./runVerify.cjs');
|
|
75
80
|
const { latestTerminalOutcomeForSlug, COMPLETED_EQUIVALENT_VERDICTS } = require('./lib/terminalRunOutcome.cjs');
|
|
76
81
|
const { isFixPlanSlug, classifyDiscoveredFixPlan, resolveIsFixPlan } = require('./lib/fixPlanSlug.cjs');
|
|
77
|
-
const { landedSinceRun } = require('./lib/landedSinceRun.cjs');
|
|
82
|
+
const { landedSinceRun, landedOnMainSince } = require('./lib/landedSinceRun.cjs');
|
|
78
83
|
const { declaredPathsForPrd } = require('./lib/prdDeclaredPaths.cjs');
|
|
79
84
|
const logs = require('./logs.cjs');
|
|
80
85
|
const { schemas, validated, SCHEDULE_SLUG_RE } = require('./ipcSchemas.cjs');
|
|
@@ -268,9 +273,10 @@ deferring it.
|
|
|
268
273
|
Your own work must still never be left uncommitted — this only changes
|
|
269
274
|
which paths get staged, never whether you commit.
|
|
270
275
|
5. VERDICT SENTINEL — as the LAST LINE of your final result text, emit exactly
|
|
271
|
-
one of these
|
|
276
|
+
one of these lines (no trailing text after it):
|
|
272
277
|
SCHEDULER_VERDICT: PASS
|
|
273
278
|
SCHEDULER_VERDICT: FAIL <one-line reason>
|
|
279
|
+
SCHEDULER_VERDICT: BLOCKED_BY_FOREIGN_WIP
|
|
274
280
|
Print PASS only when the AC gate is green AND the commit from step 4 landed.
|
|
275
281
|
Print FAIL (and exit 1) if the AC gate was red or the commit could not land.
|
|
276
282
|
NEVER print PASS on a red AC gate — a lying PASS turns the verifier from a
|
|
@@ -278,6 +284,22 @@ deferring it.
|
|
|
278
284
|
landed commit lets the verifier override incidental transcript noise (grep
|
|
279
285
|
results containing "Error", a TDD red-test run early in the session, debug
|
|
280
286
|
Tracebacks) so those do not false-trip a needs_review downgrade.
|
|
287
|
+
Print BLOCKED_BY_FOREIGN_WIP (and exit 1) ONLY when your own AC gate failed
|
|
288
|
+
because it ran against a SIBLING job's in-flight, uncommitted file — never
|
|
289
|
+
because of your own regression — AND every failing path is one this prompt
|
|
290
|
+
already disclosed to you as foreign (see the "FOREIGN WORKING-TREE STATE"
|
|
291
|
+
section above, if present). It MUST be accompanied by a second line naming
|
|
292
|
+
every such path:
|
|
293
|
+
FOREIGN_WIP_PATHS: <path1>, <path2>, ...
|
|
294
|
+
The scheduler independently validates every listed path against the exact
|
|
295
|
+
foreign-WIP manifest it disclosed to you. Only list a path that (a) this
|
|
296
|
+
prompt already told you is foreign WIP, not yours, AND (b) is why your own
|
|
297
|
+
AC gate failed — never a path you own, and never a path that failed for
|
|
298
|
+
some other reason of your own making. Any listed path NOT in that manifest
|
|
299
|
+
downgrades this whole verdict back to FAIL, with your claim rejected. This
|
|
300
|
+
is not an escape hatch for your own broken code: claiming it for a
|
|
301
|
+
regression you introduced, or for a path you never received as foreign
|
|
302
|
+
WIP, is a lying verdict exactly like a false PASS.
|
|
281
303
|
|
|
282
304
|
A job that exits with uncommitted changes is treated as INCOMPLETE and flagged
|
|
283
305
|
for review. Do NOT add work beyond the acceptance criteria — this protocol is the
|
|
@@ -571,6 +593,52 @@ async function computeCommittedDuringRun(cwd, headBefore, headAfter, startedAt,
|
|
|
571
593
|
return module.exports.committedInWindow(cwd, startedAt, untilIso);
|
|
572
594
|
}
|
|
573
595
|
|
|
596
|
+
/**
|
|
597
|
+
* Read-only check: is a job's `sm-job/<slug>` worktree branch already fully
|
|
598
|
+
* integrated into `cwd`'s current HEAD — i.e. every commit on the branch is
|
|
599
|
+
* an ancestor of (or equal to) HEAD? Mirrors gitWorktree.cjs's own
|
|
600
|
+
* integrateBranch() no-op detection (`mergeBase === branchHead`) exactly, but
|
|
601
|
+
* deliberately never calls integrateBranch itself: this is used by
|
|
602
|
+
* reapDeadRunningJobs to PROVE a dead job's work already landed, never to
|
|
603
|
+
* perform the landing — attempting a real merge from an audit check is a
|
|
604
|
+
* mutating action with its own conflict/timing risk, and "land the still-
|
|
605
|
+
* stranded branch" is explicitly a separate, later decision (mechanical
|
|
606
|
+
* recovery, or a human), not something a reap pass should do on its own
|
|
607
|
+
* initiative. See reapDeadRunningJobs's own header comment for why the
|
|
608
|
+
* conservative failure direction here is needs_review, never a silent merge.
|
|
609
|
+
*
|
|
610
|
+
* Returns:
|
|
611
|
+
* - true — branch exists and is fully contained in HEAD (already
|
|
612
|
+
* integrated by a prior pass, or a genuine no-op branch that never
|
|
613
|
+
* diverged from its base — both cases legitimately "landed").
|
|
614
|
+
* - false — branch exists and still holds commits not in HEAD (a real
|
|
615
|
+
* stranded deliverable, e.g. the PRD 1118 shape).
|
|
616
|
+
* - null — branch does not exist at all (never a worktree run — an
|
|
617
|
+
* in-place run, or SM_JOB_WORKTREE_DISABLE), or the git calls errored.
|
|
618
|
+
* Never throws.
|
|
619
|
+
*/
|
|
620
|
+
function isBranchAlreadyIntegrated(cwd, branch) {
|
|
621
|
+
return new Promise((resolve) => {
|
|
622
|
+
if (!cwd || !branch) { resolve(null); return; }
|
|
623
|
+
execFile(
|
|
624
|
+
'git', ['-C', cwd, 'rev-parse', '--verify', branch],
|
|
625
|
+
{ timeout: 10_000, windowsHide: true },
|
|
626
|
+
(err, branchHeadOut) => {
|
|
627
|
+
if (err) { resolve(null); return; } // branch doesn't exist — not a worktree run
|
|
628
|
+
const branchHead = String(branchHeadOut || '').trim();
|
|
629
|
+
execFile(
|
|
630
|
+
'git', ['-C', cwd, 'merge-base', 'HEAD', branch],
|
|
631
|
+
{ timeout: 10_000, windowsHide: true },
|
|
632
|
+
(mbErr, mbOut) => {
|
|
633
|
+
const mergeBase = mbErr ? '' : String(mbOut || '').trim();
|
|
634
|
+
resolve(mergeBase !== '' && mergeBase === branchHead);
|
|
635
|
+
},
|
|
636
|
+
);
|
|
637
|
+
},
|
|
638
|
+
);
|
|
639
|
+
});
|
|
640
|
+
}
|
|
641
|
+
|
|
574
642
|
/**
|
|
575
643
|
* Override for a SIGTERM'd (143) run when a commit landed in its window.
|
|
576
644
|
* Exit 143 alone doesn't prove the deliverable is missing — the 776/779
|
|
@@ -592,6 +660,74 @@ function classifySigtermWithCommit(exitCode, commitFoundInWindow) {
|
|
|
592
660
|
};
|
|
593
661
|
}
|
|
594
662
|
|
|
663
|
+
/**
|
|
664
|
+
* Pure decision for spawnJob's finalize mutate (2026-09-06 incident): given
|
|
665
|
+
* the row this run is trying to finalize (or its absence) and this run's own
|
|
666
|
+
* identity, decide whether the finalize must be DROPPED instead of applied.
|
|
667
|
+
* Never mutates anything itself — callers apply `stampLandedCommit` and emit
|
|
668
|
+
* the audit event/log line.
|
|
669
|
+
*
|
|
670
|
+
* - row missing entirely (i2 === -1 upstream): always drop, nothing to
|
|
671
|
+
* stamp — there is no row left to carry the fact.
|
|
672
|
+
* - row present but not 'running': drop the STATUS change (a cancelled row,
|
|
673
|
+
* PRD 1024, must never be re-legalized to completed/needs_review by a
|
|
674
|
+
* stale exit handler) but still let a genuinely-landed commit be stamped
|
|
675
|
+
* as a fact — never a transition — when this run owns the row's current
|
|
676
|
+
* runId, or the row doesn't already carry a newer one of its own.
|
|
677
|
+
* - row present and 'running': not a drop; caller proceeds to finalize.
|
|
678
|
+
*/
|
|
679
|
+
function evaluateFinalizeDrop({ rowExists, rowStatus, rowRunId, rowLandedCommit, runId, landedCommit }) {
|
|
680
|
+
if (!rowExists) {
|
|
681
|
+
return { drop: true, reason: 'row-missing', stampLandedCommit: null };
|
|
682
|
+
}
|
|
683
|
+
if (rowStatus !== 'running') {
|
|
684
|
+
const stampLandedCommit = landedCommit && (rowRunId === runId || !rowLandedCommit) ? landedCommit : null;
|
|
685
|
+
return { drop: true, reason: 'row-not-running', stampLandedCommit };
|
|
686
|
+
}
|
|
687
|
+
return { drop: false, reason: null, stampLandedCommit: null };
|
|
688
|
+
}
|
|
689
|
+
|
|
690
|
+
/**
|
|
691
|
+
* Pure decision for spawnJob's dispatch mutate: given a PENDING row about to
|
|
692
|
+
* be dispatched and the newest terminalRunOutcome sidecar result for its
|
|
693
|
+
* slug, decide whether this dispatch should be SKIPPED because a prior run
|
|
694
|
+
* already completed this exact slug — the gap left by reconcile()'s own
|
|
695
|
+
* anti-resurrection guard, which only ever sees a slug BEFORE it first lands
|
|
696
|
+
* in s.jobs (2026-09-06 incident: an existing pending row was re-dispatched
|
|
697
|
+
* three times against already-shipped work).
|
|
698
|
+
*
|
|
699
|
+
* Compares the sidecar's finishedAt against THIS row's own last pending
|
|
700
|
+
* transition (or queuedAt, when the row has never been reset) rather than
|
|
701
|
+
* merely "a completed sidecar exists somewhere" — so a deliberate human
|
|
702
|
+
* re-queue of the same slug in a LATER episode (whose queuedAt/pending
|
|
703
|
+
* transition postdates the earlier completion) still runs.
|
|
704
|
+
*/
|
|
705
|
+
function evaluateDispatchSidecarReconcile({ rowStatus, rowRunId, statusHistory, queuedAt, outcome }) {
|
|
706
|
+
if (rowStatus !== 'pending') return { skip: false };
|
|
707
|
+
if (!outcome || outcome.status !== 'completed') return { skip: false };
|
|
708
|
+
if (outcome.runId === rowRunId) return { skip: false };
|
|
709
|
+
const lastPendingEntry = Array.isArray(statusHistory)
|
|
710
|
+
? [...statusHistory].reverse().find((h) => h.to === 'pending')
|
|
711
|
+
: null;
|
|
712
|
+
const lastPendingAt = lastPendingEntry?.at ?? queuedAt ?? null;
|
|
713
|
+
if (lastPendingAt && (!outcome.finishedAt || outcome.finishedAt < lastPendingAt)) {
|
|
714
|
+
return { skip: false };
|
|
715
|
+
}
|
|
716
|
+
return { skip: true, runId: outcome.runId, finishedAt: outcome.finishedAt };
|
|
717
|
+
}
|
|
718
|
+
|
|
719
|
+
/**
|
|
720
|
+
* Pure detector for mutate()'s write-path regression check: transitionJob
|
|
721
|
+
* only ever APPENDS to statusHistory (capped at STATUS_HISTORY_CAP, never
|
|
722
|
+
* shrunk below it once reached) — so a running->pending change whose
|
|
723
|
+
* statusHistory got SHORTER is not a real transition, it's evidence this
|
|
724
|
+
* row's prior state was lost (e.g. a finalize mutate working off a stale
|
|
725
|
+
* in-memory snapshot).
|
|
726
|
+
*/
|
|
727
|
+
function isQueueRowRegression({ statusBefore, statusAfter, historyLenBefore, historyLenAfter }) {
|
|
728
|
+
return statusBefore === 'running' && statusAfter === 'pending' && historyLenAfter < historyLenBefore;
|
|
729
|
+
}
|
|
730
|
+
|
|
595
731
|
const ROOT = path.join(os.homedir(), '.claude', 'session-manager', 'scheduled-plans');
|
|
596
732
|
const PRDS_DIR = path.join(ROOT, 'prds');
|
|
597
733
|
const RUNS_DIR = path.join(ROOT, 'runs');
|
|
@@ -1565,7 +1701,42 @@ function mutate(fn) {
|
|
|
1565
1701
|
if (state.unreadable) {
|
|
1566
1702
|
throw new Error(`queue mutation skipped: queue.json unreadable (${state.unreadable})`);
|
|
1567
1703
|
}
|
|
1704
|
+
const beforeBySlug = new Map(
|
|
1705
|
+
(state.jobs || []).map((j) => [
|
|
1706
|
+
j.slug,
|
|
1707
|
+
{ status: j.status, historyLen: Array.isArray(j.statusHistory) ? j.statusHistory.length : 0 },
|
|
1708
|
+
]),
|
|
1709
|
+
);
|
|
1568
1710
|
const ret = await fn(state);
|
|
1711
|
+
// Detection-only regression check: transitionJob only ever APPENDS to
|
|
1712
|
+
// statusHistory (capped at STATUS_HISTORY_CAP, never shrunk below it) —
|
|
1713
|
+
// so a running->pending change whose statusHistory got SHORTER is not a
|
|
1714
|
+
// real transition, it's evidence this row's prior state was lost (e.g. a
|
|
1715
|
+
// finalize mutate working off a stale in-memory snapshot). Never
|
|
1716
|
+
// repaired here — mutate()'s job is to persist, not reconcile.
|
|
1717
|
+
for (const j of state.jobs || []) {
|
|
1718
|
+
const prior = beforeBySlug.get(j.slug);
|
|
1719
|
+
if (!prior) continue;
|
|
1720
|
+
const afterLen = Array.isArray(j.statusHistory) ? j.statusHistory.length : 0;
|
|
1721
|
+
if (isQueueRowRegression({
|
|
1722
|
+
statusBefore: prior.status,
|
|
1723
|
+
statusAfter: j.status,
|
|
1724
|
+
historyLenBefore: prior.historyLen,
|
|
1725
|
+
historyLenAfter: afterLen,
|
|
1726
|
+
})) {
|
|
1727
|
+
console.error(
|
|
1728
|
+
`[scheduler] queue row regressed: slug=${j.slug} running->pending with `
|
|
1729
|
+
+ `statusHistory ${prior.historyLen} -> ${afterLen}`,
|
|
1730
|
+
);
|
|
1731
|
+
appendAuditEvent('queue_row_regressed', {
|
|
1732
|
+
slug: j.slug,
|
|
1733
|
+
from: 'running',
|
|
1734
|
+
to: 'pending',
|
|
1735
|
+
historyBefore: prior.historyLen,
|
|
1736
|
+
historyAfter: afterLen,
|
|
1737
|
+
});
|
|
1738
|
+
}
|
|
1739
|
+
}
|
|
1569
1740
|
await writeQueue(state);
|
|
1570
1741
|
return ret;
|
|
1571
1742
|
});
|
|
@@ -2166,6 +2337,12 @@ async function reconcile(state) {
|
|
|
2166
2337
|
state.jobs = sorted;
|
|
2167
2338
|
}
|
|
2168
2339
|
|
|
2340
|
+
// Auto-requeue any job parked 'skipped' over a validated BLOCKED_BY_FOREIGN_WIP
|
|
2341
|
+
// verdict once the paths that blocked it are no longer dirty in its own cwd —
|
|
2342
|
+
// the only place this check runs, so a sibling landing its commit resumes the
|
|
2343
|
+
// blocked job with no human action, on the very next reconcile() pass.
|
|
2344
|
+
await requeueForeignWipBlockedJobs(state.jobs);
|
|
2345
|
+
|
|
2169
2346
|
// Auto-archive completed PRDs' .md files out of the live prds/ dir. Runs
|
|
2170
2347
|
// AFTER the history append above (which is awaited) so a job's queue row
|
|
2171
2348
|
// is always durably in history.jsonl before its file can be moved — a
|
|
@@ -2208,6 +2385,20 @@ function getNextResetCached() {
|
|
|
2208
2385
|
return cachedNextReset;
|
|
2209
2386
|
}
|
|
2210
2387
|
|
|
2388
|
+
/**
|
|
2389
|
+
* Pure: picks the reset to pause against for a rate-limited run (PRD 1118).
|
|
2390
|
+
* Prefers the BINDING window read off the run's own log — refreshNextReset()
|
|
2391
|
+
* only ever reports five_hour, which is the wrong clock when a
|
|
2392
|
+
* seven_day/seven_day_overage_included window is what actually 429'd
|
|
2393
|
+
* (five_hour can read 0% utilization at the very same moment). Falls back
|
|
2394
|
+
* to the billing-endpoint-derived reset only when the log yields nothing.
|
|
2395
|
+
*/
|
|
2396
|
+
function resolveRateLimitPauseReset(logPath, billingResetIso) {
|
|
2397
|
+
const logReset = resolveBindingRateLimitReset(logPath);
|
|
2398
|
+
if (logReset != null) return new Date(logReset * 1000).toISOString();
|
|
2399
|
+
return billingResetIso ?? null;
|
|
2400
|
+
}
|
|
2401
|
+
|
|
2211
2402
|
// ---------- health / poll state ----------
|
|
2212
2403
|
|
|
2213
2404
|
let bootedAt = Date.now();
|
|
@@ -2491,6 +2682,33 @@ function nextRapidRateLimitCount(prevCount, { rateLimited, durationMs }) {
|
|
|
2491
2682
|
return prevCount || 0;
|
|
2492
2683
|
}
|
|
2493
2684
|
|
|
2685
|
+
/**
|
|
2686
|
+
* Pure: decides the resumeAt actually armed for a pause. 'network' and
|
|
2687
|
+
* 'rate_limit' (PRD 1118) both get a bounded 30-minute fallback when no
|
|
2688
|
+
* explicit resumeAt is supplied — the live rate_limit failure mode is the
|
|
2689
|
+
* billing usage endpoint itself 429ing while the log yields no binding
|
|
2690
|
+
* window either, which used to leave an indefinite pause with no resume
|
|
2691
|
+
* timer at all (a queue that never comes back on its own).
|
|
2692
|
+
*/
|
|
2693
|
+
function computeEffectiveResumeAt(reason, resumeAtIso, nowMs = Date.now()) {
|
|
2694
|
+
if (resumeAtIso) return resumeAtIso;
|
|
2695
|
+
if (reason === 'network' || reason === 'rate_limit') {
|
|
2696
|
+
return new Date(nowMs + 30 * 60_000).toISOString();
|
|
2697
|
+
}
|
|
2698
|
+
return null;
|
|
2699
|
+
}
|
|
2700
|
+
|
|
2701
|
+
/**
|
|
2702
|
+
* Pure: the setTimeout delay for a resume timer, plus whether it overflows
|
|
2703
|
+
* setTimeout's signed-32-bit max (~24.8 days) and must not be armed.
|
|
2704
|
+
* Resume fires 30s after the reset to give the auth/billing endpoint time
|
|
2705
|
+
* to flip.
|
|
2706
|
+
*/
|
|
2707
|
+
function computeResumeDelay(effectiveResumeAtIso, nowMs = Date.now()) {
|
|
2708
|
+
const delayMs = Math.max(30_000, new Date(effectiveResumeAtIso).getTime() - nowMs + 30_000);
|
|
2709
|
+
return { delayMs, tooFar: delayMs > 0x7fffffff };
|
|
2710
|
+
}
|
|
2711
|
+
|
|
2494
2712
|
async function setPaused(reason, resumeAtIso, opts = {}) {
|
|
2495
2713
|
const { observedAt = null, force = false } = opts;
|
|
2496
2714
|
// Honor manual-override cooldown: if the user cleared a pause within the
|
|
@@ -2507,11 +2725,7 @@ async function setPaused(reason, resumeAtIso, opts = {}) {
|
|
|
2507
2725
|
console.log(`[scheduler] setPaused(${reason}) engaging despite manual override cooldown — triggering run started after the manual clear`);
|
|
2508
2726
|
}
|
|
2509
2727
|
|
|
2510
|
-
|
|
2511
|
-
let effectiveResumeAt = resumeAtIso;
|
|
2512
|
-
if (reason === 'network' && !resumeAtIso) {
|
|
2513
|
-
effectiveResumeAt = new Date(Date.now() + 30 * 60_000).toISOString();
|
|
2514
|
-
}
|
|
2728
|
+
const effectiveResumeAt = computeEffectiveResumeAt(reason, resumeAtIso);
|
|
2515
2729
|
|
|
2516
2730
|
await mutate((s) => {
|
|
2517
2731
|
if (s.paused && s.paused.reason === reason) {
|
|
@@ -2525,9 +2739,8 @@ async function setPaused(reason, resumeAtIso, opts = {}) {
|
|
|
2525
2739
|
if (resumeTimer) { clearTimeout(resumeTimer); resumeTimer = null; }
|
|
2526
2740
|
if (!effectiveResumeAt) return;
|
|
2527
2741
|
|
|
2528
|
-
|
|
2529
|
-
|
|
2530
|
-
if (delay > 0x7fffffff) {
|
|
2742
|
+
const { delayMs: delay, tooFar } = computeResumeDelay(effectiveResumeAt);
|
|
2743
|
+
if (tooFar) {
|
|
2531
2744
|
console.warn(`[scheduler] paused (${reason}); resumeAt too far for setTimeout (${delay}ms)`);
|
|
2532
2745
|
return;
|
|
2533
2746
|
}
|
|
@@ -2636,6 +2849,13 @@ function resetJobFields(job, errorMsg, opts = {}) {
|
|
|
2636
2849
|
delete job.leftoverCount;
|
|
2637
2850
|
delete job.leftoverPathsTruncated;
|
|
2638
2851
|
delete job.preRunDirtyPaths;
|
|
2852
|
+
// A manual/force reset (e.g. a human clearing a 3-strikes needs_review
|
|
2853
|
+
// park) must not leave the row permanently excluded from the auto-fix
|
|
2854
|
+
// chain (selectAutoFixTargets checks job.blockedByForeignWip) — this run's
|
|
2855
|
+
// BLOCKED_BY_FOREIGN_WIP history is done, the human is taking over.
|
|
2856
|
+
delete job.blockedByForeignWip;
|
|
2857
|
+
delete job.foreignWipBlockedPaths;
|
|
2858
|
+
delete job.foreignWipBlockCount;
|
|
2639
2859
|
// Deliberately NOT deleting job.landedCommit: it must outlive a reset so a
|
|
2640
2860
|
// re-fired run of this same slug can pass it to verifyRun as
|
|
2641
2861
|
// priorLandedCommit (pass_no_commit_prior_run_verified exemption).
|
|
@@ -3102,6 +3322,44 @@ function classifyFailureOutcome({ exitCode, networkError, durationMs, transientR
|
|
|
3102
3322
|
return { action: 'retry', transientKind, retries };
|
|
3103
3323
|
}
|
|
3104
3324
|
|
|
3325
|
+
/**
|
|
3326
|
+
* Evidence gathering for the commit-guard's already-satisfied-on-main
|
|
3327
|
+
* exemption (PRD 1136): does a commit reachable from `main`, landed AFTER
|
|
3328
|
+
* this job's `queuedAt`, touch a path this PRD itself declares? Scoped to
|
|
3329
|
+
* the PRD's own declared paths via declaredPathsForPrd — the same
|
|
3330
|
+
* path-extraction computeLooksDone already uses — so an unrelated commit
|
|
3331
|
+
* elsewhere in the repo is never credited to this job. Returns `[]` (never
|
|
3332
|
+
* fabricates evidence) when the PRD names no paths or has no `queuedAt`.
|
|
3333
|
+
*
|
|
3334
|
+
* Requires EVERY declared path to have landed, not just one — declaredPathsForPrd's
|
|
3335
|
+
* regex matches any backtick-quoted path in the PRD's Implementation notes or
|
|
3336
|
+
* Acceptance criteria, including a path cited only as context (e.g.
|
|
3337
|
+
* `` `src/main/scheduler.cjs:1234` `` pointing at a call site, not a file this
|
|
3338
|
+
* PRD's own work touches). Unlike computeLooksDone's identical path-overlap
|
|
3339
|
+
* heuristic — which only ever annotates a still-needs_review row for a human
|
|
3340
|
+
* to confirm — this check drives an UNATTENDED transition straight to
|
|
3341
|
+
* 'completed', so a single incidental hot-file citation must never be
|
|
3342
|
+
* sufficient evidence on its own (code-review finding, 2026-09-07: a PRD
|
|
3343
|
+
* that cites both its real target file and one hot file purely as context
|
|
3344
|
+
* would auto-complete off any unrelated commit touching that hot file).
|
|
3345
|
+
* Requiring full coverage of the declared-path set trades recall for safety
|
|
3346
|
+
* exactly as this PRD's own constraint demands — a PRD that fails this
|
|
3347
|
+
* stricter check still falls back to the existing needs_review park, never
|
|
3348
|
+
* a false 'completed'.
|
|
3349
|
+
*
|
|
3350
|
+
* @returns {Promise<string[]>} full commit SHAs reachable from main, newest first
|
|
3351
|
+
*/
|
|
3352
|
+
async function findSatisfyingCommitOnMain(job) {
|
|
3353
|
+
if (!job?.queuedAt) return [];
|
|
3354
|
+
const prdPath = (await resolveVerifyPrdPath(job)) ?? archivedPrdPathForJob(job);
|
|
3355
|
+
const paths = declaredPathsForPrd(prdPath);
|
|
3356
|
+
if (!paths.length) return [];
|
|
3357
|
+
await fetchAllRefs(job.cwd);
|
|
3358
|
+
const perPathCommits = await Promise.all(paths.map((p) => landedOnMainSince(job.cwd, job.queuedAt, [p])));
|
|
3359
|
+
if (perPathCommits.some((commits) => commits.length === 0)) return [];
|
|
3360
|
+
return landedOnMainSince(job.cwd, job.queuedAt, paths);
|
|
3361
|
+
}
|
|
3362
|
+
|
|
3105
3363
|
/**
|
|
3106
3364
|
* Commit-guard verdict decision. Pure/no I/O so the false-positive defenses
|
|
3107
3365
|
* can be unit-tested directly rather than only through a live spawnJob run.
|
|
@@ -3272,6 +3530,73 @@ function buildForeignWipSection({ preRunDirtyPaths, carriedPaths } = {}) {
|
|
|
3272
3530
|
return '';
|
|
3273
3531
|
}
|
|
3274
3532
|
|
|
3533
|
+
// A job whose validated BLOCKED_BY_FOREIGN_WIP claim keeps recurring against
|
|
3534
|
+
// the same sibling WIP is not making progress by re-firing forever — cap the
|
|
3535
|
+
// auto-requeue cycle and hand it to a human instead, naming the paths that
|
|
3536
|
+
// never went clean. Matches TRANSIENT_RETRY_CAP's "bounded self-heal, then
|
|
3537
|
+
// escalate" shape.
|
|
3538
|
+
const FOREIGN_WIP_BLOCK_STREAK_LIMIT = 3;
|
|
3539
|
+
|
|
3540
|
+
/**
|
|
3541
|
+
* Pure: validate an executor's BLOCKED_BY_FOREIGN_WIP claim against the exact
|
|
3542
|
+
* foreign-WIP manifest THIS job was disclosed at dispatch time
|
|
3543
|
+
* (preRunDirtyPaths / carriedPaths — PRD 1105). Every claimed path must be a
|
|
3544
|
+
* member of that manifest; a path the job was never told was foreign cannot
|
|
3545
|
+
* be laundered into a block, whether that's a genuine regression the
|
|
3546
|
+
* executor is trying to dodge or an honest mistake. `job` is any object
|
|
3547
|
+
* carrying those two array fields (the live queue row at finalize time).
|
|
3548
|
+
*
|
|
3549
|
+
* Returns `ok: false` for an empty claim too — a BLOCKED_BY_FOREIGN_WIP
|
|
3550
|
+
* verdict with no FOREIGN_WIP_PATHS evidence at all is exactly as
|
|
3551
|
+
* unsubstantiated as one naming an unlisted path.
|
|
3552
|
+
*/
|
|
3553
|
+
function validateForeignWipBlockClaim(claimedPaths, job) {
|
|
3554
|
+
const manifest = new Set([
|
|
3555
|
+
...(Array.isArray(job?.preRunDirtyPaths) ? job.preRunDirtyPaths : []),
|
|
3556
|
+
...(Array.isArray(job?.carriedPaths) ? job.carriedPaths : []),
|
|
3557
|
+
]);
|
|
3558
|
+
const claimed = Array.isArray(claimedPaths) ? claimedPaths.filter(Boolean) : [];
|
|
3559
|
+
const validPaths = claimed.filter((p) => manifest.has(p));
|
|
3560
|
+
const invalidPaths = claimed.filter((p) => !manifest.has(p));
|
|
3561
|
+
return { ok: claimed.length > 0 && invalidPaths.length === 0, validPaths, invalidPaths };
|
|
3562
|
+
}
|
|
3563
|
+
|
|
3564
|
+
/**
|
|
3565
|
+
* Auto-requeue jobs parked 'skipped' with a validated BLOCKED_BY_FOREIGN_WIP
|
|
3566
|
+
* verdict once none of the paths that blocked them are still dirty in their
|
|
3567
|
+
* own `cwd` — so a sibling job landing its commit (or a human resolving their
|
|
3568
|
+
* own WIP) resumes the blocked job with no human action needed, on the very
|
|
3569
|
+
* next reconcile() pass. `getDirtyPaths` is injectable (defaults to the real
|
|
3570
|
+
* `uncommittedChanges` git-status wrapper) so this is unit-testable without a
|
|
3571
|
+
* real git repo. Mutates eligible jobs in place via transitionJob; returns
|
|
3572
|
+
* nothing.
|
|
3573
|
+
*/
|
|
3574
|
+
async function requeueForeignWipBlockedJobs(jobs, { getDirtyPaths = uncommittedChanges } = {}) {
|
|
3575
|
+
const candidates = (Array.isArray(jobs) ? jobs : []).filter(
|
|
3576
|
+
(j) => j && j.status === 'skipped' && j.blockedByForeignWip === true,
|
|
3577
|
+
);
|
|
3578
|
+
for (const job of candidates) {
|
|
3579
|
+
const blockedPaths = Array.isArray(job.foreignWipBlockedPaths) ? job.foreignWipBlockedPaths : [];
|
|
3580
|
+
if (!blockedPaths.length) continue;
|
|
3581
|
+
const dirty = await getDirtyPaths(job.cwd);
|
|
3582
|
+
if (dirty === null) continue; // non-git cwd or git error — best-effort, leave parked
|
|
3583
|
+
const dirtySet = new Set(dirty);
|
|
3584
|
+
const stillDirty = blockedPaths.filter((p) => dirtySet.has(p));
|
|
3585
|
+
if (stillDirty.length === 0) {
|
|
3586
|
+
// resetJobFields (not a bare transitionJob) so the row re-enters
|
|
3587
|
+
// 'pending' with none of the blocked run's stale exitCode/error/
|
|
3588
|
+
// verifierVerdict/finishedAt/leftoverPaths left on it for a viewer to
|
|
3589
|
+
// misread against the new 'pending' status — same reasoning as every
|
|
3590
|
+
// other terminal->pending path in this file. force:true is required
|
|
3591
|
+
// here: resetJobFields refuses skipped->pending without it.
|
|
3592
|
+
resetJobFields(job, `foreign WIP cleared (was blocked on: ${blockedPaths.join(', ')}) — auto-requeued`, {
|
|
3593
|
+
source: 'reconcile-foreign-wip-clear',
|
|
3594
|
+
force: true,
|
|
3595
|
+
});
|
|
3596
|
+
}
|
|
3597
|
+
}
|
|
3598
|
+
}
|
|
3599
|
+
|
|
3275
3600
|
/**
|
|
3276
3601
|
* Resume-first recovery (PRD 1111). A job parked in needs_review with verdict
|
|
3277
3602
|
* 'uncommitted_changes' has a live claude session (job.sessionId, minted by
|
|
@@ -4297,6 +4622,11 @@ function readRunOutcomeSidecars(runDir, slug) {
|
|
|
4297
4622
|
return {
|
|
4298
4623
|
meta: readJson(path.join(runDir, `${slug}.meta.json`)),
|
|
4299
4624
|
verdicts: readJson(path.join(runDir, `${slug}.verdicts.json`)),
|
|
4625
|
+
// outcome.json (launchFailure.writeOutcomeSidecar) is the one sidecar
|
|
4626
|
+
// that carries landedCommit — reused by the dispatch-time sidecar-
|
|
4627
|
+
// reconcile guard and the pre-dispatch landedCommit backfill below,
|
|
4628
|
+
// rather than growing a second reader for the same directory.
|
|
4629
|
+
outcome: readJson(path.join(runDir, `${slug}.outcome.json`)),
|
|
4300
4630
|
};
|
|
4301
4631
|
}
|
|
4302
4632
|
|
|
@@ -4895,9 +5225,64 @@ async function spawnJob(job, runId, runDir, defaultCwd, resumeTarget = null) {
|
|
|
4895
5225
|
// started after the manual clear, not whatever startedAt this row
|
|
4896
5226
|
// carried from a prior run.
|
|
4897
5227
|
let dispatchStartedAtMs = null;
|
|
5228
|
+
let dispatchSkippedAlreadyCompleted = false;
|
|
4898
5229
|
await mutate((s) => {
|
|
4899
5230
|
const idx = s.jobs.findIndex((x) => x.slug === job.slug);
|
|
4900
5231
|
if (idx >= 0) {
|
|
5232
|
+
// Anti-resurrection guard for an EXISTING pending row (the gap left
|
|
5233
|
+
// by reconcile()'s own guard, which only ever sees a slug BEFORE it
|
|
5234
|
+
// first lands in s.jobs — 2026-09-06 incident: a finalize dropped
|
|
5235
|
+
// silently three times over left this same slug 'pending' and the
|
|
5236
|
+
// dispatcher re-fired it three more times against work that had
|
|
5237
|
+
// already shipped). Skipped for a resume-recovery dispatch (that
|
|
5238
|
+
// targets a specific prior session on purpose) and for anything not
|
|
5239
|
+
// currently 'pending' (e.g. a needs_review->running recovery row).
|
|
5240
|
+
if (!resumeTarget && s.jobs[idx].status === 'pending') {
|
|
5241
|
+
const outcome = latestTerminalOutcomeForSlug(job.slug, { runsDir: RUNS_DIR });
|
|
5242
|
+
const reconcileDecision = evaluateDispatchSidecarReconcile({
|
|
5243
|
+
rowStatus: s.jobs[idx].status,
|
|
5244
|
+
rowRunId: s.jobs[idx].runId ?? null,
|
|
5245
|
+
statusHistory: s.jobs[idx].statusHistory,
|
|
5246
|
+
queuedAt: s.jobs[idx].queuedAt ?? null,
|
|
5247
|
+
outcome,
|
|
5248
|
+
});
|
|
5249
|
+
if (reconcileDecision.skip) {
|
|
5250
|
+
const sidecar = readRunOutcomeSidecars(path.join(RUNS_DIR, reconcileDecision.runId), job.slug);
|
|
5251
|
+
transitionJob(s.jobs[idx], 'completed', {
|
|
5252
|
+
reason: `prior run ${reconcileDecision.runId} already completed this slug (sidecar-reconciled)`,
|
|
5253
|
+
source: 'spawnJob:dispatch-sidecar-reconcile',
|
|
5254
|
+
});
|
|
5255
|
+
s.jobs[idx].runId = reconcileDecision.runId;
|
|
5256
|
+
s.jobs[idx].finishedAt = reconcileDecision.finishedAt;
|
|
5257
|
+
s.jobs[idx].exitCode = 0;
|
|
5258
|
+
if (sidecar.outcome?.landedCommit) {
|
|
5259
|
+
s.jobs[idx].landedCommit = sidecar.outcome.landedCommit;
|
|
5260
|
+
}
|
|
5261
|
+
appendAuditEvent('job_dispatch_skipped_already_completed', {
|
|
5262
|
+
slug: job.slug,
|
|
5263
|
+
priorRunId: reconcileDecision.runId,
|
|
5264
|
+
cwd: job.cwd || defaultCwd,
|
|
5265
|
+
});
|
|
5266
|
+
console.warn(
|
|
5267
|
+
`[scheduler] ${job.slug}: dispatch skipped — prior run ${reconcileDecision.runId} already `
|
|
5268
|
+
+ 'completed this slug (sidecar-reconciled)',
|
|
5269
|
+
);
|
|
5270
|
+
dispatchSkippedAlreadyCompleted = true;
|
|
5271
|
+
return;
|
|
5272
|
+
}
|
|
5273
|
+
// Belt-and-braces (PRD fix-plan step 3): a row about to dispatch
|
|
5274
|
+
// with no landedCommit of its own, whose newest sidecar for this
|
|
5275
|
+
// slug DOES record one, gets it backfilled before spawn so
|
|
5276
|
+
// verifyRun receives a real priorLandedCommit and the
|
|
5277
|
+
// pass_no_commit_prior_run_verified exemption can fire on this
|
|
5278
|
+
// run if it turns out to be another no-op re-verification.
|
|
5279
|
+
if (!s.jobs[idx].landedCommit && outcome?.runId) {
|
|
5280
|
+
const sidecar = readRunOutcomeSidecars(path.join(RUNS_DIR, outcome.runId), job.slug);
|
|
5281
|
+
if (sidecar.outcome?.landedCommit) {
|
|
5282
|
+
s.jobs[idx].landedCommit = sidecar.outcome.landedCommit;
|
|
5283
|
+
}
|
|
5284
|
+
}
|
|
5285
|
+
}
|
|
4901
5286
|
transitionJob(s.jobs[idx], 'running', {
|
|
4902
5287
|
reason: resumeTarget ? 'dispatched for resume-recovery' : 'dispatched for execution',
|
|
4903
5288
|
source: 'spawnJob:dispatch',
|
|
@@ -4921,6 +5306,7 @@ async function spawnJob(job, runId, runDir, defaultCwd, resumeTarget = null) {
|
|
|
4921
5306
|
}
|
|
4922
5307
|
});
|
|
4923
5308
|
await broadcast({ flush: true });
|
|
5309
|
+
if (dispatchSkippedAlreadyCompleted) return;
|
|
4924
5310
|
|
|
4925
5311
|
// Commit-guard baseline: snapshot the working tree BEFORE the run so the
|
|
4926
5312
|
// post-run check flags only paths THIS job left dirty, not pre-existing WIP.
|
|
@@ -5164,7 +5550,9 @@ async function spawnJob(job, runId, runDir, defaultCwd, resumeTarget = null) {
|
|
|
5164
5550
|
}
|
|
5165
5551
|
|
|
5166
5552
|
if (res.rateLimited) {
|
|
5167
|
-
const
|
|
5553
|
+
const logPath = path.join(runDir, `${job.slug}.log`);
|
|
5554
|
+
const billingResetIso = await refreshNextReset().catch(() => cachedNextReset);
|
|
5555
|
+
const resetIso = resolveRateLimitPauseReset(logPath, billingResetIso);
|
|
5168
5556
|
const observedAt = dispatchStartedAtMs;
|
|
5169
5557
|
const prevCount = consecutiveRapidRateLimitsBySlug.get(job.slug) || 0;
|
|
5170
5558
|
const nextCount = nextRapidRateLimitCount(prevCount, { rateLimited: true, durationMs: res.durationMs });
|
|
@@ -5310,9 +5698,22 @@ async function spawnJob(job, runId, runDir, defaultCwd, resumeTarget = null) {
|
|
|
5310
5698
|
salvagePatch,
|
|
5311
5699
|
});
|
|
5312
5700
|
if (guardVerdict) {
|
|
5313
|
-
|
|
5314
|
-
|
|
5315
|
-
|
|
5701
|
+
// Already-satisfied-on-main exemption (PRD 1136): resolveCommitGuardOutcome
|
|
5702
|
+
// only ever touches the clean-tree/no-commit shape ('silent_no_op') —
|
|
5703
|
+
// a job that left dirty files behind is a genuine finish-protocol
|
|
5704
|
+
// violation regardless of what already landed on main, so it is
|
|
5705
|
+
// never routed through this check (see that function's own doc).
|
|
5706
|
+
const satisfyingCommits = guardVerdict.verdict === 'silent_no_op'
|
|
5707
|
+
? await findSatisfyingCommitOnMain(job)
|
|
5708
|
+
: [];
|
|
5709
|
+
const finalVerdict = resolveCommitGuardOutcome(guardVerdict, satisfyingCommits);
|
|
5710
|
+
verifyResult = finalVerdict;
|
|
5711
|
+
if (finalVerdict.verdict === 'already_satisfied_on_main') {
|
|
5712
|
+
console.log(`[scheduler] already-satisfied-on-main: ${job.slug} → completed (${finalVerdict.satisfyingSha})`);
|
|
5713
|
+
} else {
|
|
5714
|
+
const what = newlyDirty.length > 0 ? `left ${newlyDirty.length} files uncommitted` : 'made no commit on an already-clean tree';
|
|
5715
|
+
console.log(`[scheduler] commit-guard: ${job.slug} ${what} → needs_review`);
|
|
5716
|
+
}
|
|
5316
5717
|
}
|
|
5317
5718
|
}
|
|
5318
5719
|
}
|
|
@@ -5378,6 +5779,25 @@ async function spawnJob(job, runId, runDir, defaultCwd, resumeTarget = null) {
|
|
|
5378
5779
|
);
|
|
5379
5780
|
}
|
|
5380
5781
|
|
|
5782
|
+
// BLOCKED_BY_FOREIGN_WIP claim scan: the executor exits non-zero for this
|
|
5783
|
+
// outcome (same as FAIL — see FINISH_PROTOCOL), so it is never seen by the
|
|
5784
|
+
// exit=0-only verifyRun call above; scanned here, against THIS run's own
|
|
5785
|
+
// log, before the generic non-zero-exit -> 'failed' classification below.
|
|
5786
|
+
// Read outside mutate() (I/O); actual manifest validation happens inside
|
|
5787
|
+
// mutate(), against the LIVE row's preRunDirtyPaths/carriedPaths, so a
|
|
5788
|
+
// stale local `job` snapshot can never be the source of truth for it.
|
|
5789
|
+
let foreignWipClaimedPaths = null;
|
|
5790
|
+
if (res.exitCode !== 0 && !res.rateLimited) {
|
|
5791
|
+
try {
|
|
5792
|
+
const { resultEvent, events } = parseLog(path.join(runDir, `${job.slug}.log`));
|
|
5793
|
+
if (scanSentinel(resultEvent, events) === 'blocked_by_foreign_wip') {
|
|
5794
|
+
foreignWipClaimedPaths = scanForeignWipPathsClaim(resultEvent, events);
|
|
5795
|
+
}
|
|
5796
|
+
} catch (e) {
|
|
5797
|
+
console.warn(`[scheduler] ${job.slug}: foreign-WIP verdict scan failed, falling through to ordinary failed classification`, e?.message);
|
|
5798
|
+
}
|
|
5799
|
+
}
|
|
5800
|
+
|
|
5381
5801
|
let actuallyFailed = false;
|
|
5382
5802
|
let failedJobSnapshot = null;
|
|
5383
5803
|
let needsInvestigationNow = false;
|
|
@@ -5402,7 +5822,44 @@ async function spawnJob(job, runId, runDir, defaultCwd, resumeTarget = null) {
|
|
|
5402
5822
|
// scheduleJobTransitions.cjs's LEGAL_TRANSITIONS) and silently
|
|
5403
5823
|
// undo the cancellation. Skip — the row already reflects its real
|
|
5404
5824
|
// terminal state.
|
|
5405
|
-
|
|
5825
|
+
const finalizeDrop = evaluateFinalizeDrop({
|
|
5826
|
+
rowExists: i2 >= 0,
|
|
5827
|
+
rowStatus: i2 >= 0 ? s.jobs[i2].status : null,
|
|
5828
|
+
rowRunId: i2 >= 0 ? (s.jobs[i2].runId ?? null) : null,
|
|
5829
|
+
rowLandedCommit: i2 >= 0 ? (s.jobs[i2].landedCommit ?? null) : null,
|
|
5830
|
+
runId,
|
|
5831
|
+
landedCommit: jobLandedCommitThisRun ?? null,
|
|
5832
|
+
});
|
|
5833
|
+
if (finalizeDrop.drop) {
|
|
5834
|
+
// Never silent (2026-09-06 incident: a bare early-return here dropped
|
|
5835
|
+
// three legitimate no-op verifications of an already-shipped PRD with
|
|
5836
|
+
// no trace at all, leaving the row stuck 'pending' so the dispatcher
|
|
5837
|
+
// re-fired it three more times). 'row-not-running' covers a job
|
|
5838
|
+
// already moved off 'running' by someone else (namely
|
|
5839
|
+
// remote.cancelJob, PRD 1024) — re-finalizing anyway could
|
|
5840
|
+
// re-legalize the row via a legal failed->completed/needs_review edge
|
|
5841
|
+
// and silently undo the cancellation, so the STATUS change is still
|
|
5842
|
+
// skipped; only a genuinely-landed commit is stamped as a fact.
|
|
5843
|
+
const logFn = finalizeDrop.reason === 'row-missing' ? console.error : console.warn;
|
|
5844
|
+
logFn(
|
|
5845
|
+
`[scheduler] finalize dropped: slug=${job.slug} runId=${runId} reason=${finalizeDrop.reason} `
|
|
5846
|
+
+ `actualStatus=${i2 >= 0 ? s.jobs[i2].status : '(row-missing)'} `
|
|
5847
|
+
+ `rowRunId=${i2 >= 0 ? (s.jobs[i2].runId ?? '(none)') : '(none)'}`,
|
|
5848
|
+
);
|
|
5849
|
+
appendAuditEvent('job_finalize_dropped', {
|
|
5850
|
+
slug: job.slug,
|
|
5851
|
+
runId,
|
|
5852
|
+
reason: finalizeDrop.reason,
|
|
5853
|
+
actualStatus: i2 >= 0 ? s.jobs[i2].status : null,
|
|
5854
|
+
rowRunId: i2 >= 0 ? (s.jobs[i2].runId ?? null) : null,
|
|
5855
|
+
landedCommit: jobLandedCommitThisRun ?? null,
|
|
5856
|
+
exitCode: res.exitCode,
|
|
5857
|
+
});
|
|
5858
|
+
if (finalizeDrop.stampLandedCommit) {
|
|
5859
|
+
s.jobs[i2].landedCommit = finalizeDrop.stampLandedCommit;
|
|
5860
|
+
}
|
|
5861
|
+
return;
|
|
5862
|
+
}
|
|
5406
5863
|
if (i2 >= 0) {
|
|
5407
5864
|
const treatAsPending = res.rateLimited || (s.paused && s.paused.reason === 'rate_limit');
|
|
5408
5865
|
if (treatAsPending) {
|
|
@@ -5414,14 +5871,66 @@ async function spawnJob(job, runId, runDir, defaultCwd, resumeTarget = null) {
|
|
|
5414
5871
|
const sigtermOverride = res.exitCode !== 0
|
|
5415
5872
|
? classifySigtermWithCommit(res.exitCode, sigtermCommitFound)
|
|
5416
5873
|
: null;
|
|
5874
|
+
// Validated against the LIVE row's own foreign-WIP manifest — never
|
|
5875
|
+
// the stale outer `job` snapshot — so an unlisted path cannot
|
|
5876
|
+
// launder a real regression into a block (PRD: give the executor a
|
|
5877
|
+
// first-class verdict for "the gate failed on a sibling's in-flight
|
|
5878
|
+
// file", but VALIDATE the claim rather than trust it).
|
|
5879
|
+
const foreignWipValidation = (!sigtermOverride && foreignWipClaimedPaths !== null)
|
|
5880
|
+
? validateForeignWipBlockClaim(foreignWipClaimedPaths, s.jobs[i2])
|
|
5881
|
+
: null;
|
|
5882
|
+
// Consecutive-block streak: cleared by default on every outcome and
|
|
5883
|
+
// only re-established below inside the validated-block branch —
|
|
5884
|
+
// so a completed run, an ordinary failure, or any other outcome
|
|
5885
|
+
// between two blocks always resets "in a row" back to zero.
|
|
5886
|
+
const priorForeignWipBlockCount = s.jobs[i2].foreignWipBlockCount ?? 0;
|
|
5887
|
+
delete s.jobs[i2].foreignWipBlockCount;
|
|
5417
5888
|
if (sigtermOverride) {
|
|
5418
5889
|
effectiveStatus = sigtermOverride.status;
|
|
5419
5890
|
sigtermOverrideReason = sigtermOverride.reason;
|
|
5891
|
+
} else if (foreignWipValidation && foreignWipValidation.ok) {
|
|
5892
|
+
const blockCount = priorForeignWipBlockCount + 1;
|
|
5893
|
+
s.jobs[i2].foreignWipBlockCount = blockCount;
|
|
5894
|
+
s.jobs[i2].blockedByForeignWip = true;
|
|
5895
|
+
s.jobs[i2].foreignWipBlockedPaths = foreignWipValidation.validPaths;
|
|
5896
|
+
if (blockCount >= FOREIGN_WIP_BLOCK_STREAK_LIMIT) {
|
|
5897
|
+
// Blocked FOREIGN_WIP_BLOCK_STREAK_LIMIT times in a row on the
|
|
5898
|
+
// same tree: auto-requeuing again would spin forever against
|
|
5899
|
+
// paths that never go clean. Park for a human instead — never
|
|
5900
|
+
// routed through the auto-fix chain (selectAutoFixTargets
|
|
5901
|
+
// excludes any job.blockedByForeignWip row), since there is no
|
|
5902
|
+
// fix-plan to author against another job's WIP.
|
|
5903
|
+
effectiveStatus = 'needs_review';
|
|
5904
|
+
s.jobs[i2].verifierVerdict = 'blocked_by_foreign_wip_streak';
|
|
5905
|
+
sigtermOverrideReason = `blocked by foreign WIP ${blockCount} times in a row on persistently-dirty path(s): ${foreignWipValidation.validPaths.join(', ')} — auto-requeue exhausted, parked for human review`;
|
|
5906
|
+
} else {
|
|
5907
|
+
// Terminal-but-retryable, same shape as the existing 'skipped'
|
|
5908
|
+
// status (never counts as failed, never enters the auto-fix
|
|
5909
|
+
// chain, reconcile()'s requeueForeignWipBlockedJobs promotes it
|
|
5910
|
+
// straight back to 'pending' once these exact paths go clean).
|
|
5911
|
+
effectiveStatus = 'skipped';
|
|
5912
|
+
sigtermOverrideReason = `blocked by foreign WIP (attempt ${blockCount}/${FOREIGN_WIP_BLOCK_STREAK_LIMIT}): ${foreignWipValidation.validPaths.join(', ')} — will auto-requeue once these path(s) are no longer dirty`;
|
|
5913
|
+
}
|
|
5914
|
+
} else if (foreignWipValidation && !foreignWipValidation.ok) {
|
|
5915
|
+
// Claimed BLOCKED_BY_FOREIGN_WIP but named a path outside this
|
|
5916
|
+
// job's own disclosed manifest (or named none at all) — downgrade
|
|
5917
|
+
// to an ordinary failure and log the offending paths loudly so the
|
|
5918
|
+
// rejection is never silent.
|
|
5919
|
+
console.warn(`[scheduler] ${job.slug}: SCHEDULER_VERDICT: BLOCKED_BY_FOREIGN_WIP downgraded to FAIL — claimed path(s) not in this job's foreign-WIP manifest: ${foreignWipValidation.invalidPaths.join(', ') || '(no FOREIGN_WIP_PATHS line)'}`);
|
|
5920
|
+
effectiveStatus = 'failed';
|
|
5921
|
+
sigtermOverrideReason = `SCHEDULER_VERDICT: BLOCKED_BY_FOREIGN_WIP rejected — unlisted path(s) not in the disclosed foreign-WIP manifest: ${foreignWipValidation.invalidPaths.join(', ') || '(no FOREIGN_WIP_PATHS line)'}`;
|
|
5420
5922
|
} else if (res.exitCode !== 0) {
|
|
5421
5923
|
effectiveStatus = 'failed';
|
|
5422
5924
|
} else if (
|
|
5423
5925
|
!verifyResult
|
|
5424
5926
|
|| COMPLETED_EQUIVALENT_VERDICTS.has(verifyResult.verdict)
|
|
5927
|
+
// Already-satisfied-on-main (PRD 1136): a second, independently-
|
|
5928
|
+
// evidenced route to 'completed' alongside COMPLETED_EQUIVALENT_
|
|
5929
|
+
// VERDICTS above — kept as its own explicit check rather than
|
|
5930
|
+
// folded into that shared Set so it can never leak into
|
|
5931
|
+
// reverifyNeedsReview's or runVerify.cjs's unrelated healing
|
|
5932
|
+
// decisions, which consult that Set for a different purpose.
|
|
5933
|
+
|| verifyResult.verdict === 'already_satisfied_on_main'
|
|
5425
5934
|
) {
|
|
5426
5935
|
effectiveStatus = 'completed';
|
|
5427
5936
|
} else if (verifyResult.downgradeTo === 'pending') {
|
|
@@ -5433,7 +5942,14 @@ async function spawnJob(job, runId, runDir, defaultCwd, resumeTarget = null) {
|
|
|
5433
5942
|
effectiveStatus = 'needs_review';
|
|
5434
5943
|
}
|
|
5435
5944
|
|
|
5436
|
-
|
|
5945
|
+
// already_satisfied_on_main names the satisfying sha in its own
|
|
5946
|
+
// reason — surface that on the completed row's statusHistory
|
|
5947
|
+
// instead of the generic "run finished with exit 0" every other
|
|
5948
|
+
// completed run gets.
|
|
5949
|
+
const finalizeReason = (effectiveStatus === 'completed' && verifyResult?.verdict === 'already_satisfied_on_main')
|
|
5950
|
+
? verifyResult.reason
|
|
5951
|
+
: (sigtermOverrideReason ?? `run finished with exit ${res.exitCode}`);
|
|
5952
|
+
transitionJob(s.jobs[i2], effectiveStatus, { reason: finalizeReason, source: 'spawnJob:finalize' });
|
|
5437
5953
|
s.jobs[i2].finishedAt = new Date().toISOString();
|
|
5438
5954
|
s.jobs[i2].exitCode = res.exitCode;
|
|
5439
5955
|
s.jobs[i2].leakedDescendants = res.leakedDescendants ?? [];
|
|
@@ -5442,7 +5958,7 @@ async function spawnJob(job, runId, runDir, defaultCwd, resumeTarget = null) {
|
|
|
5442
5958
|
} else {
|
|
5443
5959
|
delete s.jobs[i2].salvagePatch;
|
|
5444
5960
|
}
|
|
5445
|
-
s.jobs[i2].error = effectiveStatus === 'needs_review'
|
|
5961
|
+
s.jobs[i2].error = (effectiveStatus === 'needs_review' || s.jobs[i2].blockedByForeignWip === true)
|
|
5446
5962
|
? (verifyResult?.reason ?? sigtermOverrideReason ?? null)
|
|
5447
5963
|
// A failed job (non-zero exit) never consults verifyResult above,
|
|
5448
5964
|
// but a worktree integration failure is still worth surfacing on
|
|
@@ -5459,9 +5975,12 @@ async function spawnJob(job, runId, runDir, defaultCwd, resumeTarget = null) {
|
|
|
5459
5975
|
s.jobs[i2].landedCommit = jobLandedCommitThisRun;
|
|
5460
5976
|
}
|
|
5461
5977
|
// Persist the verifier's verdict string so the renderer can show it.
|
|
5978
|
+
// 'blocked_by_foreign_wip_streak' is set above from the sigterm/
|
|
5979
|
+
// exit-code path, never from verifyResult (which stays null on a
|
|
5980
|
+
// non-zero exit) — never clobber it here.
|
|
5462
5981
|
if (verifyResult?.verdict && verifyResult.verdict !== 'clean') {
|
|
5463
5982
|
s.jobs[i2].verifierVerdict = verifyResult.verdict;
|
|
5464
|
-
} else {
|
|
5983
|
+
} else if (s.jobs[i2].verifierVerdict !== 'blocked_by_foreign_wip_streak') {
|
|
5465
5984
|
delete s.jobs[i2].verifierVerdict;
|
|
5466
5985
|
}
|
|
5467
5986
|
// Closed-set outcome taxonomy (issue #11 list A2) so a queue row
|
|
@@ -6166,6 +6685,55 @@ function runQueueHealthSweep(jobs) {
|
|
|
6166
6685
|
}
|
|
6167
6686
|
}
|
|
6168
6687
|
|
|
6688
|
+
// Cross-cycle "already attempted" memory for the branch sweep (PRD 1135) —
|
|
6689
|
+
// in-memory only, keyed `"<cwd>::<branch>"`. Resets on restart, same as
|
|
6690
|
+
// consecutiveRapidRateLimitsBySlug above; the goal is "don't retry a
|
|
6691
|
+
// permanently-conflicting branch every cycle forever" while the process
|
|
6692
|
+
// stays up, not durability across a restart.
|
|
6693
|
+
const branchSweepAttempted = new Set();
|
|
6694
|
+
|
|
6695
|
+
/**
|
|
6696
|
+
* Recover any `sm-job/*` branch left stranded (unmerged, owning row terminal
|
|
6697
|
+
* or gone) across every known project — the other half of PRD 1135's
|
|
6698
|
+
* invariant alongside the reaper's liveness check above: that check stops
|
|
6699
|
+
* NEW stranding, this sweep recovers anything stranded before it (or by a
|
|
6700
|
+
* crash the liveness check can't cover). Read-only reporting plus AT MOST one
|
|
6701
|
+
* bounded integrateBranch attempt per branch per process lifetime (see
|
|
6702
|
+
* branchSweepAttempted) — never forces anything, never deletes a branch.
|
|
6703
|
+
* Hangs off the same cadence as runQueueHealthSweep. Never throws.
|
|
6704
|
+
*/
|
|
6705
|
+
async function runBranchSweep(jobs) {
|
|
6706
|
+
if (process.env.SM_BRANCH_SWEEP_DISABLE === '1') return;
|
|
6707
|
+
try {
|
|
6708
|
+
for (const cwd of allProjectCwds()) {
|
|
6709
|
+
let sweep;
|
|
6710
|
+
try {
|
|
6711
|
+
sweep = await sweepStrandedJobBranches({ cwd, jobs, attemptedBranches: branchSweepAttempted });
|
|
6712
|
+
} catch (e) {
|
|
6713
|
+
console.warn(`[scheduler] branch sweep error for ${cwd}`, e?.message);
|
|
6714
|
+
continue;
|
|
6715
|
+
}
|
|
6716
|
+
for (const r of sweep.results) {
|
|
6717
|
+
if (r.action === 'integrated') {
|
|
6718
|
+
console.log(`[scheduler] branch-sweep: merged stranded branch ${r.branch} into ${cwd} HEAD`);
|
|
6719
|
+
appendAuditEvent('branch_sweep_integrated', { cwd, branch: r.branch, slug: r.slug });
|
|
6720
|
+
} else if (r.action === 'conflict') {
|
|
6721
|
+
console.warn(`[scheduler] branch-sweep: ${r.branch} could not be integrated: ${r.integration?.reason}`);
|
|
6722
|
+
appendAuditEvent('branch_sweep_conflict', { cwd, branch: r.branch, slug: r.slug, reason: r.integration?.reason });
|
|
6723
|
+
if (r.slug) {
|
|
6724
|
+
await mutate((s) => {
|
|
6725
|
+
const j = s.jobs.find((x) => x.slug === r.slug);
|
|
6726
|
+
if (j) j.strandedBranch = r.branch;
|
|
6727
|
+
});
|
|
6728
|
+
}
|
|
6729
|
+
}
|
|
6730
|
+
}
|
|
6731
|
+
}
|
|
6732
|
+
} catch (e) {
|
|
6733
|
+
console.warn('[scheduler] branch sweep error', e?.message);
|
|
6734
|
+
}
|
|
6735
|
+
}
|
|
6736
|
+
|
|
6169
6737
|
/**
|
|
6170
6738
|
* Scan running jobs, identify those whose claude process is provably dead OR
|
|
6171
6739
|
* whose spawn never got far enough to record a runtime.pid in the first
|
|
@@ -6183,14 +6751,35 @@ async function reapDeadRunningJobs() {
|
|
|
6183
6751
|
// status:"running" with no slug left in runningSet to trigger reconciliation.
|
|
6184
6752
|
// queue.json is the source of truth for which jobs are actually running.
|
|
6185
6753
|
const state = await readQueue();
|
|
6186
|
-
const { reapable, warnings } = selectReapableJobs(state.jobs, Date.now(), {
|
|
6754
|
+
const { reapable, warnings, recovered } = selectReapableJobs(state.jobs, Date.now(), {
|
|
6187
6755
|
pidAlive: claudePidAlive,
|
|
6188
6756
|
grace: PIDLESS_SPAWN_GRACE_MS,
|
|
6757
|
+
findLiveProcess: (j) => findLiveProcessForJob(j, {
|
|
6758
|
+
worktreeDir: jobWorktree.worktreeDirFor(j.cwd || state.config?.defaultCwd || DEFAULT_PROJECT_CWD, j.slug),
|
|
6759
|
+
}),
|
|
6189
6760
|
});
|
|
6190
6761
|
for (const w of warnings) {
|
|
6191
6762
|
console.warn(`[scheduler] reapDeadRunningJobs: ${w.reason} slug=${w.slug} — leaving row alone`);
|
|
6192
6763
|
}
|
|
6193
6764
|
|
|
6765
|
+
// A pidless row whose process was proven alive by the /proc liveness
|
|
6766
|
+
// scan must never be terminalized (2026-09-06 incident — see
|
|
6767
|
+
// findLiveProcessForJob's header). Re-stamp the recovered pid so future
|
|
6768
|
+
// cycles see it as an ordinary live-pid row, and stop here for it.
|
|
6769
|
+
if (recovered.length) {
|
|
6770
|
+
await mutate(async (s) => {
|
|
6771
|
+
for (const r of recovered) {
|
|
6772
|
+
const idx = s.jobs.findIndex((x) => x.slug === r.slug);
|
|
6773
|
+
if (idx < 0 || s.jobs[idx].status !== 'running') continue;
|
|
6774
|
+
s.jobs[idx].runtime = { ...(s.jobs[idx].runtime || {}), pid: r.pid };
|
|
6775
|
+
}
|
|
6776
|
+
});
|
|
6777
|
+
for (const r of recovered) {
|
|
6778
|
+
console.log(`[scheduler] reapDeadRunningJobs: pid=${r.pid} recovered by /proc liveness scan for slug=${r.slug} — row stays running`);
|
|
6779
|
+
appendAuditEvent('job_pid_recovered_by_liveness_scan', { slug: r.slug, pid: r.pid });
|
|
6780
|
+
}
|
|
6781
|
+
}
|
|
6782
|
+
|
|
6194
6783
|
const dead = [];
|
|
6195
6784
|
for (const { slug, pid, pidless, reason } of reapable) {
|
|
6196
6785
|
const j = state.jobs.find((x) => x.slug === slug);
|
|
@@ -6201,15 +6790,18 @@ async function reapDeadRunningJobs() {
|
|
|
6201
6790
|
// 'no_result' → non-success below → filed as failed, never completed.
|
|
6202
6791
|
const outcome = logPath ? classifyRunOutcome(logPath) : 'unknown';
|
|
6203
6792
|
// A pidless reap means the spawn never got far enough to record a
|
|
6204
|
-
// pid — the gate could not possibly have run
|
|
6205
|
-
//
|
|
6206
|
-
|
|
6207
|
-
|
|
6793
|
+
// pid — the gate could not possibly have run. But `never_ran` is only
|
|
6794
|
+
// a true claim when the run dir produced no log output at all; a log
|
|
6795
|
+
// with real content proves the job DID run (see
|
|
6796
|
+
// resolvePidlessGateOutcome's header).
|
|
6797
|
+
const gateOutcome = pidless ? resolvePidlessGateOutcome(outcome, logHasOutput(logPath)) : mapOutcomeToGateOutcome(outcome);
|
|
6798
|
+
dead.push({ slug, pid, outcome, gateOutcome, pidless, reason, logPath });
|
|
6208
6799
|
}
|
|
6209
6800
|
|
|
6210
6801
|
queueHealthSweepCycle += 1;
|
|
6211
6802
|
if (queueHealthSweepCycle % QUEUE_HEALTH_SWEEP_EVERY_N_CYCLES === 0) {
|
|
6212
6803
|
runQueueHealthSweep(state.jobs);
|
|
6804
|
+
await runBranchSweep(state.jobs);
|
|
6213
6805
|
}
|
|
6214
6806
|
|
|
6215
6807
|
if (dead.length === 0) return;
|
|
@@ -6221,8 +6813,9 @@ async function reapDeadRunningJobs() {
|
|
|
6221
6813
|
// the same still-active rate limit — the spin loop this PRD exists to
|
|
6222
6814
|
// stop. Done once, outside mutate(), before finalizing any row below.
|
|
6223
6815
|
if (dead.some((d) => d.outcome === 'rate_limited')) {
|
|
6224
|
-
const resetIso = await refreshNextReset().catch(() => cachedNextReset);
|
|
6225
6816
|
const triggering = dead.find((d) => d.outcome === 'rate_limited');
|
|
6817
|
+
const billingResetIso = await refreshNextReset().catch(() => cachedNextReset);
|
|
6818
|
+
const resetIso = resolveRateLimitPauseReset(triggering.logPath, billingResetIso);
|
|
6226
6819
|
const triggeringRow = triggering ? state.jobs.find((x) => x.slug === triggering.slug) : null;
|
|
6227
6820
|
const observedAtMs = triggeringRow?.startedAt ? Date.parse(triggeringRow.startedAt) : null;
|
|
6228
6821
|
// Same rapid-repeat circuit breaker spawnJob's own res.rateLimited
|
|
@@ -6242,6 +6835,64 @@ async function reapDeadRunningJobs() {
|
|
|
6242
6835
|
await setPaused('rate_limit', resetIso, { observedAt: observedAtMs, force: forceHardPause });
|
|
6243
6836
|
}
|
|
6244
6837
|
|
|
6838
|
+
// Prove integration BEFORE entering mutate() (PRD 1133): the check below
|
|
6839
|
+
// shells out to git — including a full `git fetch --all --prune` (up to
|
|
6840
|
+
// 20s) via computeCommittedDuringRun's committedInWindow, plus up to two
|
|
6841
|
+
// 2s retry sleeps when nothing landed — for every dead job that needs the
|
|
6842
|
+
// in-place fallback. mutate() serializes through ONE global mutateTail
|
|
6843
|
+
// promise chain shared by every project's dispatch/admin/cancel mutation,
|
|
6844
|
+
// same reason the rate-limit handling above already runs outside it — so
|
|
6845
|
+
// doing this git work inside the mutate() callback would stall the whole
|
|
6846
|
+
// scheduler's queue writes for the sum of these calls across every dead
|
|
6847
|
+
// job in the batch (e.g. an app-restart reap that dead-letters several
|
|
6848
|
+
// running jobs at once).
|
|
6849
|
+
const integrationResults = new Map();
|
|
6850
|
+
for (const d of dead) {
|
|
6851
|
+
if (d.outcome !== 'success') continue;
|
|
6852
|
+
const row = state.jobs.find((x) => x.slug === d.slug);
|
|
6853
|
+
if (!row) continue;
|
|
6854
|
+
const rowCwd = row.cwd || state.config?.defaultCwd || DEFAULT_PROJECT_CWD;
|
|
6855
|
+
if (!isGitRepoSync(rowCwd)) continue;
|
|
6856
|
+
const branch = jobWorktree.branchNameFor(d.slug);
|
|
6857
|
+
const integrated = await isBranchAlreadyIntegrated(rowCwd, branch);
|
|
6858
|
+
const headNow = await gitHead(rowCwd);
|
|
6859
|
+
const guardHeadBeforeVal = row.guardHeadBefore || null;
|
|
6860
|
+
let landedCommit = null;
|
|
6861
|
+
let notLandedInfo = null;
|
|
6862
|
+
let effectiveSuccess = true;
|
|
6863
|
+
if (integrated === null) {
|
|
6864
|
+
// No sm-job/<slug> branch — an in-place run (or worktree isolation
|
|
6865
|
+
// disabled). Prove landing via the exact same HEAD-advance evidence
|
|
6866
|
+
// the live commit-guard already uses — reused, not re-derived.
|
|
6867
|
+
const committed = await computeCommittedDuringRun(
|
|
6868
|
+
rowCwd, guardHeadBeforeVal, headNow, row.startedAt, new Date().toISOString(),
|
|
6869
|
+
);
|
|
6870
|
+
if (committed) {
|
|
6871
|
+
if (guardHeadBeforeVal && headNow && headNow !== guardHeadBeforeVal) landedCommit = headNow;
|
|
6872
|
+
} else {
|
|
6873
|
+
effectiveSuccess = false;
|
|
6874
|
+
notLandedInfo = {
|
|
6875
|
+
verdict: 'reaped_without_integration',
|
|
6876
|
+
reason: 'no commit landed during the run window — HEAD never advanced',
|
|
6877
|
+
};
|
|
6878
|
+
}
|
|
6879
|
+
} else if (integrated === true) {
|
|
6880
|
+
if (guardHeadBeforeVal && headNow && headNow !== guardHeadBeforeVal) landedCommit = headNow;
|
|
6881
|
+
} else {
|
|
6882
|
+
// Branch exists and still holds commits never merged into rowCwd's
|
|
6883
|
+
// HEAD — the exact PRD 1118 shape. Never merge it here (see
|
|
6884
|
+
// isBranchAlreadyIntegrated's header comment); park for review
|
|
6885
|
+
// instead, naming the branch so the work is easy to find and land by
|
|
6886
|
+
// hand or via mechanical recovery.
|
|
6887
|
+
effectiveSuccess = false;
|
|
6888
|
+
notLandedInfo = {
|
|
6889
|
+
verdict: 'reaped_without_integration',
|
|
6890
|
+
reason: `worktree branch ${branch} still holds unintegrated work — never merged into ${rowCwd}`,
|
|
6891
|
+
};
|
|
6892
|
+
}
|
|
6893
|
+
integrationResults.set(d.slug, { effectiveSuccess, landedCommit, notLandedInfo });
|
|
6894
|
+
}
|
|
6895
|
+
|
|
6245
6896
|
await mutate(async (s) => {
|
|
6246
6897
|
for (const { slug, pid, outcome, gateOutcome, pidless, reason } of dead) {
|
|
6247
6898
|
const idx = s.jobs.findIndex((x) => x.slug === slug);
|
|
@@ -6280,12 +6931,38 @@ async function reapDeadRunningJobs() {
|
|
|
6280
6931
|
console.error(`[scheduler] reapDeadRunningJobs: in-place salvage failed for ${slug}`, e);
|
|
6281
6932
|
}
|
|
6282
6933
|
}
|
|
6934
|
+
// Prove integration before this reap is allowed to say 'completed'
|
|
6935
|
+
// (PRD 1133): a reaped job's owning process vanished before
|
|
6936
|
+
// spawnJob's own post-run integration/commit-guard ever ran, so
|
|
6937
|
+
// 'outcome=success' alone (a clean result event in the log) is not
|
|
6938
|
+
// proof anything actually landed — the two live incidents this PRD
|
|
6939
|
+
// exists for (1118, starry-night-ships 224) both had exactly that
|
|
6940
|
+
// shape. Only evaluated for a genuinely successful, non-rate-limited
|
|
6941
|
+
// outcome; a non-git cwd (isGitRepoSync false) skips this entirely,
|
|
6942
|
+
// preserving today's behaviour exactly. The actual git work already
|
|
6943
|
+
// ran ABOVE, before this mutate() call, into integrationResults — see
|
|
6944
|
+
// that block's own header comment for why it must not run in here.
|
|
6945
|
+
let effectiveSuccess = success;
|
|
6946
|
+
let landedCommit = null;
|
|
6947
|
+
let notLandedInfo = null;
|
|
6948
|
+
if (success && !rateLimited) {
|
|
6949
|
+
const ir = integrationResults.get(slug);
|
|
6950
|
+
if (ir) {
|
|
6951
|
+
effectiveSuccess = ir.effectiveSuccess;
|
|
6952
|
+
landedCommit = ir.landedCommit;
|
|
6953
|
+
notLandedInfo = ir.notLandedInfo;
|
|
6954
|
+
}
|
|
6955
|
+
}
|
|
6956
|
+
|
|
6283
6957
|
const leftoverSuffix = deltaPaths && deltaPaths.length
|
|
6284
6958
|
? ` — left ${deltaPaths.length} files uncommitted`
|
|
6285
6959
|
: '';
|
|
6960
|
+
const baseReason = notLandedInfo
|
|
6961
|
+
? `reaped: ${notLandedInfo.reason}`
|
|
6962
|
+
: (pidless ? reason : `reaped: process gone (outcome=${outcome})`);
|
|
6286
6963
|
const transitionReason = rateLimited
|
|
6287
6964
|
? `reaped: rate limit detected — reset to pending, not failed (outcome=${outcome})${leftoverSuffix}`
|
|
6288
|
-
:
|
|
6965
|
+
: baseReason + leftoverSuffix;
|
|
6289
6966
|
|
|
6290
6967
|
if (rateLimited) {
|
|
6291
6968
|
// Retryable, never terminal (PRD 1117) — same resetJobFields path
|
|
@@ -6294,11 +6971,18 @@ async function reapDeadRunningJobs() {
|
|
|
6294
6971
|
// paused-for-rate-limit reset: fresh runId/startedAt/exitCode.
|
|
6295
6972
|
resetJobFields(s.jobs[idx], transitionReason, { source: 'reapDeadRunningJobs:rate-limit' });
|
|
6296
6973
|
} else {
|
|
6297
|
-
|
|
6298
|
-
s.jobs[idx]
|
|
6974
|
+
const targetStatus = effectiveSuccess ? 'completed' : (notLandedInfo ? 'needs_review' : 'failed');
|
|
6975
|
+
transitionJob(s.jobs[idx], targetStatus, { reason: transitionReason, source: 'reapDeadRunningJobs' });
|
|
6976
|
+
s.jobs[idx].exitCode = effectiveSuccess ? 0 : (s.jobs[idx].exitCode ?? 1);
|
|
6299
6977
|
s.jobs[idx].finishedAt = new Date().toISOString();
|
|
6300
|
-
s.jobs[idx].error =
|
|
6978
|
+
s.jobs[idx].error = effectiveSuccess ? null : `${transitionReason} (outcome=${outcome})`;
|
|
6301
6979
|
s.jobs[idx].gateOutcome = gateOutcome;
|
|
6980
|
+
if (notLandedInfo) {
|
|
6981
|
+
s.jobs[idx].verifierVerdict = notLandedInfo.verdict;
|
|
6982
|
+
} else {
|
|
6983
|
+
delete s.jobs[idx].verifierVerdict;
|
|
6984
|
+
}
|
|
6985
|
+
if (landedCommit) s.jobs[idx].landedCommit = landedCommit;
|
|
6302
6986
|
}
|
|
6303
6987
|
delete s.jobs[idx].runtime;
|
|
6304
6988
|
delete s.jobs[idx].guardBaseline;
|
|
@@ -6726,6 +7410,27 @@ function isRescanCandidate(job) {
|
|
|
6726
7410
|
return false;
|
|
6727
7411
|
}
|
|
6728
7412
|
|
|
7413
|
+
/**
|
|
7414
|
+
* Cheap-guard for the 10-minute periodic reverify tick. MUST be expressed in
|
|
7415
|
+
* terms of isRescanCandidate — not a hand-written status test — because the
|
|
7416
|
+
* two drifted once already: the guard only looked for a `needs_review` row
|
|
7417
|
+
* while isRescanCandidate had grown a `failed`/isFailedUnverifiedShaped arm,
|
|
7418
|
+
* so reverifyNeedsReview's failed → needs_review looksDone branch was
|
|
7419
|
+
* unreachable from the periodic path on any queue holding zero needs_review
|
|
7420
|
+
* rows. A `failed` row with no run log sat wedged for five days with the
|
|
7421
|
+
* self-heal that exists to inspect it never firing (only an app restart, via
|
|
7422
|
+
* the unguarded boot call, could reach it). Reported 2026-09-10 from
|
|
7423
|
+
* social-signals-trader.
|
|
7424
|
+
*
|
|
7425
|
+
* Cost: one extra classifyRunOutcome() log stat per `failed` row, and only
|
|
7426
|
+
* for rows that reach isFailedUnverifiedShaped's log check — bounded by the
|
|
7427
|
+
* failed-row count, and strictly cheaper than firing the whole pass (which
|
|
7428
|
+
* does git fetches via computeLooksDone) on every tick.
|
|
7429
|
+
*/
|
|
7430
|
+
function shouldRunPeriodicReverify(jobs) {
|
|
7431
|
+
return Array.isArray(jobs) && jobs.some((j) => isRescanCandidate(j));
|
|
7432
|
+
}
|
|
7433
|
+
|
|
6729
7434
|
/**
|
|
6730
7435
|
* Self-healing pass over needs_review jobs. The verifier runs in-process, so a
|
|
6731
7436
|
* fix to runVerify.cjs only takes effect for jobs verified AFTER an app
|
|
@@ -6778,6 +7483,11 @@ function selectAutoFixTargets(jobs, { fixSlugExists, resolveJobRunId = resolveRu
|
|
|
6778
7483
|
const slugsInQueue = new Set(jobs.map((j) => j.slug));
|
|
6779
7484
|
return jobs.filter((job) => {
|
|
6780
7485
|
if (job.status !== 'needs_review') return false;
|
|
7486
|
+
// A job parked here because it was blocked by a sibling's foreign WIP
|
|
7487
|
+
// three times in a row (never its own regression) has nothing for a
|
|
7488
|
+
// fix-plan investigation to diagnose — there is no code defect to
|
|
7489
|
+
// author a PRD against, only another job's still-uncommitted tree.
|
|
7490
|
+
if (job.blockedByForeignWip === true) return false;
|
|
6781
7491
|
// A stale re-run whose work already shipped (rcaReport's 'already-shipped'
|
|
6782
7492
|
// class) must never buy a fix-plan PRD — there is nothing to fix, and the
|
|
6783
7493
|
// correct recovery (archiving the PRD) is a human/reconcile action, not
|
|
@@ -6920,7 +7630,15 @@ async function reverifyNeedsReview() {
|
|
|
6920
7630
|
// Still needs_review after the existing heal pass — widen the evidence
|
|
6921
7631
|
// window before giving up on it entirely (unchanged heal semantics for
|
|
6922
7632
|
// rows that already qualified above; this only adds an annotation).
|
|
6923
|
-
|
|
7633
|
+
// Skipped when a fix-plan investigation was already minted for this row
|
|
7634
|
+
// (job.autoFixAttempted) — PRD 1136: 'looks done, confirm before
|
|
7635
|
+
// archiving' and 'a -fix- child is already investigating this' are two
|
|
7636
|
+
// different claims about the SAME evidence, and stamping both leaves a
|
|
7637
|
+
// human reading two contradictory signals off one row. autoFixAttempted
|
|
7638
|
+
// is stamped synchronously in spawnJob's same-tick auto-fix branch,
|
|
7639
|
+
// always before this periodic/boot pass can run against the same row, so
|
|
7640
|
+
// this check reliably catches the only order that can occur.
|
|
7641
|
+
if (stillOpen && job.autoFixAttempted !== true) {
|
|
6924
7642
|
const looksDone = await computeLooksDone(job);
|
|
6925
7643
|
if (looksDone) {
|
|
6926
7644
|
looksDoneUpdates.push({ slug: job.slug, cwd: job.cwd, looksDone, fromFailed: false });
|
|
@@ -7659,14 +8377,16 @@ async function init() {
|
|
|
7659
8377
|
const s = readQueueSync();
|
|
7660
8378
|
// Periodic self-heal: re-run the verifier over stale needs_review jobs so a
|
|
7661
8379
|
// job whose work actually landed (committed in-window, no FAIL sentinel)
|
|
7662
|
-
// auto-clears WITHOUT waiting for the next app restart. Cheap-guarded
|
|
7663
|
-
//
|
|
8380
|
+
// auto-clears WITHOUT waiting for the next app restart. Cheap-guarded by
|
|
8381
|
+
// shouldRunPeriodicReverify, which reuses isRescanCandidate so the guard
|
|
8382
|
+
// and the candidate filter can never drift apart again (they did once —
|
|
8383
|
+
// see that function's comment). Kill-switch:
|
|
7664
8384
|
// SM_REVERIFY_PERIODIC_DISABLE=1 (boot reverify above stays always-on).
|
|
7665
8385
|
// reverifyNeedsReview's auto-fix loop is capped downstream by
|
|
7666
8386
|
// MAX_CONCURRENT_INVESTIGATIONS (spawnInvestigation queues/early-returns
|
|
7667
8387
|
// past it), so this interval firing cannot fan out investigations.
|
|
7668
8388
|
if (process.env.SM_REVERIFY_PERIODIC_DISABLE !== '1') {
|
|
7669
|
-
if (s.jobs
|
|
8389
|
+
if (shouldRunPeriodicReverify(s.jobs)) {
|
|
7670
8390
|
reverifyNeedsReview().catch(() => {});
|
|
7671
8391
|
}
|
|
7672
8392
|
// A quarantined row only ever promotes to 'pending' through
|
|
@@ -8444,10 +9164,12 @@ module.exports = {
|
|
|
8444
9164
|
pickNextBatch,
|
|
8445
9165
|
pickForProject,
|
|
8446
9166
|
reapDeadRunningJobs,
|
|
9167
|
+
runBranchSweep,
|
|
8447
9168
|
pollRecoveryClearSource,
|
|
8448
9169
|
memoryLimitedBatchSize,
|
|
8449
9170
|
availableForJobs,
|
|
8450
9171
|
reverifyNeedsReview,
|
|
9172
|
+
shouldRunPeriodicReverify,
|
|
8451
9173
|
isRescanCandidate,
|
|
8452
9174
|
isFailedUnverifiedShaped,
|
|
8453
9175
|
computeLooksDone,
|
|
@@ -8467,6 +9189,10 @@ module.exports = {
|
|
|
8467
9189
|
committedInWindow,
|
|
8468
9190
|
computeCommittedDuringRun,
|
|
8469
9191
|
classifySigtermWithCommit,
|
|
9192
|
+
evaluateFinalizeDrop,
|
|
9193
|
+
evaluateDispatchSidecarReconcile,
|
|
9194
|
+
isQueueRowRegression,
|
|
9195
|
+
readRunOutcomeSidecars,
|
|
8470
9196
|
isFixPlanSlug,
|
|
8471
9197
|
classifyDiscoveredFixPlan,
|
|
8472
9198
|
resolveIsFixPlan,
|
|
@@ -8478,6 +9204,7 @@ module.exports = {
|
|
|
8478
9204
|
detectRateLimitInLog,
|
|
8479
9205
|
classifyFailureOutcome,
|
|
8480
9206
|
commitGuardVerdict,
|
|
9207
|
+
findSatisfyingCommitOnMain,
|
|
8481
9208
|
leftoverFieldsFrom,
|
|
8482
9209
|
applyLeftoverFields,
|
|
8483
9210
|
LEFTOVER_PATHS_CAP,
|
|
@@ -8534,6 +9261,7 @@ module.exports = {
|
|
|
8534
9261
|
checkSharedTreeGuard,
|
|
8535
9262
|
uncommittedChanges,
|
|
8536
9263
|
gitHead,
|
|
9264
|
+
isBranchAlreadyIntegrated,
|
|
8537
9265
|
selectResumeRecoveryTarget,
|
|
8538
9266
|
buildResumeRecoveryPreamble,
|
|
8539
9267
|
buildClaudeSpawnArgs,
|
|
@@ -8560,4 +9288,10 @@ module.exports = {
|
|
|
8560
9288
|
MANUAL_PAUSE_COOLDOWN_MS,
|
|
8561
9289
|
RUNS_DIR,
|
|
8562
9290
|
pickRunDir,
|
|
9291
|
+
resolveRateLimitPauseReset,
|
|
9292
|
+
computeEffectiveResumeAt,
|
|
9293
|
+
computeResumeDelay,
|
|
9294
|
+
FOREIGN_WIP_BLOCK_STREAK_LIMIT,
|
|
9295
|
+
validateForeignWipBlockClaim,
|
|
9296
|
+
requeueForeignWipBlockedJobs,
|
|
8563
9297
|
};
|