@ai-dossier/sched 0.23.5 → 0.25.0

package/dist/engine.js

@@ -73,6 +73,7 @@ exports.runLoop = runLoop;
 const core_1 = require("@ai-dossier/core");
 const batch_dispatch_1 = require("./batch-dispatch");
 const dispatch_1 = require("./dispatch");
+const dispatch_health_1 = require("./dispatch-health");
 const groundtruth_1 = require("./groundtruth");
 const journal_1 = require("./journal");
 const labels_1 = require("./labels");
@@ -725,7 +726,14 @@ function writeFence(ctx, unit, issue, slot, truth) {
  * `unit-failed` line otherwise — and that indistinguishability is exactly
  * what cost imboard-monorepo#3999 (docs/agent-traps.md).
  */
-function failOrAdoptOpenPr(ctx, state, unit, slot, report, causeEvent, cause, evidence) {
+function failOrAdoptOpenPr(ctx, state, unit, slot, report, 
+// #629: never `'dispatch-failure'` in practice — `enterRecovery` only
+// reaches this function under `escalate: true` (its cap-check is itself
+// gated on `escalate`), and a `dispatch-failure` cause always passes
+// `escalate: false`. Kept as the two-member union `enterRecovery` uses
+// before its escalate check, so a maintainer reading this signature does
+// not read "a spend wall can terminally fail a unit" as a live path.
+causeEvent, cause, evidence) {
     let prCheck = 'skipped';
     if (!report && causeEvent === 'verify-incomplete') {
         if (slot.branch === null) {
@@ -776,7 +784,7 @@ function failOrAdoptOpenPr(ctx, state, unit, slot, report, causeEvent, cause, ev
     // with `causeEvent === 'verify-incomplete'` thread it through
     // `completeUnitOrRecover`: `reconcileRunning`'s dead-pid rail, from
     // `recordDispatchRunLog`'s own read, and (since #620) `reconcileSlots`'
-    // `exited`/`verifying` rail, from `readLastToolForSlot`. The two agree —
+    // `exited`/`verifying` rail, from `readDispatchSignalsForSlot`. The two agree —
     // the dispatch log is static once the agent has exited — so the tool name
     // survives regardless of which tick reaches this decision. It stays
     // OPTIONAL: a slice with no parseable `tool_use` yields null, hence the
@@ -798,7 +806,14 @@ function failOrAdoptOpenPr(ctx, state, unit, slot, report, causeEvent, cause, ev
  * mechanical-starting ladder and fail MERGED-AWARE at the cap: the PR is
  * already merged, so dependents stay released.
  */
-function enterRecovery(ctx, state, unit, causeEvent, cause, truth, evidence = {}) {
+function enterRecovery(ctx, state, unit, causeEvent, cause, truth, evidence = {}, 
+// #629: `escalate: false` for a CONFIRMED provider API error — it must
+// never consume the per-unit escalation ladder or the ESCALATION_CAP
+// (a spend/rate wall is not the issue's fault), so the respawn keeps the
+// slot's CURRENT tier and `recoveries` count unchanged. Every other caller
+// keeps the default (`true`), unaffected.
+options = {}) {
+    const escalate = options.escalate ?? true;
     const issue = (0, journal_1.issueOfUnit)(unit);
     if (issue === null)
         return state;
@@ -809,21 +824,36 @@ function enterRecovery(ctx, state, unit, causeEvent, cause, truth, evidence = {}
         return state;
     killUnitAgent(ctx, state, unit);
     // #524: only the STALL path kills a still-live, not-yet-recorded agent —
-    // `causeEvent === 'verify-incomplete'` arrives from `completeUnitOrRecover`
-    // AFTER the agent's exit was already detected and recorded by the dead-pid
-    // branch of `reconcileRunning`; recording again here would double-count
-    // that same dispatch. #591: a stall kill still has a fresh log slice worth
-    // reading for `last_tool` — a hung agent's last tool is exactly what tells
-    // an operator what it hung in.
+    // `causeEvent === 'verify-incomplete'`/`'dispatch-failure'` arrive from
+    // `completeUnitOrRecover` AFTER the agent's exit was already detected and
+    // recorded by the dead-pid branch of `reconcileRunning`; recording again
+    // here would double-count that same dispatch. #591: a stall kill still has
+    // a fresh log slice worth reading for `last_tool` — a hung agent's last
+    // tool is exactly what tells an operator what it hung in.
     if (causeEvent === 'stalled') {
-        const stallLastTool = recordDispatchRunLog(ctx, state, slot, unit);
+        const stallLastTool = recordDispatchRunLog(ctx, state, slot, unit).lastTool;
         if (stallLastTool !== null)
             evidence = { ...evidence, last_tool: stallLastTool };
     }
     const report = (0, state_1.isReportSlot)(slot);
-    const nextTier = report ? (0, dispatch_1.reportTierFor)(slot.recoveries + 1) : (0, dispatch_1.escalateTier)(entry.tier);
-    if (slot.recoveries >= types_1.ESCALATION_CAP || nextTier === null) {
-        return failOrAdoptOpenPr(ctx, state, unit, slot, report, causeEvent, cause, evidence);
+    let resolvedTier;
+    if (escalate) {
+        const escalated = report ? (0, dispatch_1.reportTierFor)(slot.recoveries + 1) : (0, dispatch_1.escalateTier)(entry.tier);
+        if (slot.recoveries >= types_1.ESCALATION_CAP || escalated === null) {
+            // #629: unreachable with `causeEvent === 'dispatch-failure'` — that
+            // cause always passes `escalate: false` above, so this branch (and the
+            // narrower `causeEvent` type `failOrAdoptOpenPr` declares) is never
+            // actually asked to terminally fail a unit over a provider wall.
+            return failOrAdoptOpenPr(ctx, state, unit, slot, report, causeEvent, cause, evidence);
+        }
+        resolvedTier = escalated;
+    }
+    else {
+        // An unescalated redispatch keeps the CURRENT tier — `reportTierFor`
+        // evaluated at the unchanged `slot.recoveries` for a report slot (mirrors
+        // `recordDispatchRunLog`'s own tier expression), `entry.tier` verbatim
+        // otherwise — never `escalateTier`/`recoveries + 1`.
+        resolvedTier = report ? ((0, dispatch_1.reportTierFor)(slot.recoveries) ?? entry.tier) : entry.tier;
     }
     // Fence BEFORE the respawn (#504 AC1/AC4): `killUnitAgent` above only reaches a pid
     // this process can see and signal, and #472 proved that is not the same as a dead
@@ -832,17 +862,17 @@ function enterRecovery(ctx, state, unit, causeEvent, cause, truth, evidence = {}
     const fenced = writeFence(ctx, unit, issue, slot, truth);
     let next = (0, state_1.transitionSlot)(state, slot.id, 'recovering', {
         pid: null,
-        recoveries: slot.recoveries + 1,
+        recoveries: escalate ? slot.recoveries + 1 : slot.recoveries,
         gen: fenced ?? slot.gen,
         // Only a fence that actually landed starts the short takeover watch: an
         // unfenced redispatch is already degraded, and cutting its allowance down
         // would compound one failure with another.
         fenced_at: fenced === null ? null : now.toISOString(),
     }, now);
-    if (!report) {
+    if (!report && escalate) {
         next = {
             ...next,
-            entries: next.entries.map((e) => e.issue === issue ? { ...e, tier: nextTier, updated_at: now.toISOString() } : e),
+            entries: next.entries.map((e) => e.issue === issue ? { ...e, tier: resolvedTier, updated_at: now.toISOString() } : e),
         };
     }
     journal(ctx, causeEvent, unit, {
@@ -851,10 +881,22 @@ function enterRecovery(ctx, state, unit, causeEvent, cause, truth, evidence = {}
         ...(slot.last_progress_at !== null ? { last_progress_at: slot.last_progress_at } : {}),
         ...evidence,
     });
+    // #629: an UNESCALATED redispatch (a confirmed provider API error) has no
+    // natural bound — `recoveries`/`ESCALATION_CAP` never advance for it — so
+    // once the dispatch-health pause has fired, respawning it every tick would
+    // reproduce the exact incident this fix exists to stop, just on the
+    // per-issue rail instead of the batch one (`runBatchTick` gets the
+    // equivalent gate below). Hold the slot in `recovering`, unspawned, until
+    // `sched resume`: `reconcileRecovering` (below) is what retries it once the
+    // pause clears — no separate re-classification happens while parked, since
+    // this function is reached exactly once per dead dispatch.
+    if (!escalate && next.paused) {
+        return next;
+    }
     journal(ctx, 'redispatched', unit, {
-        tier: nextTier,
+        tier: resolvedTier,
         slot: slot.id,
-        ...(0, dispatch_1.journalCmdModelFields)((0, dispatch_1.resolveTierSpawn)(ctx.dispatch, nextTier, issue)),
+        ...(0, dispatch_1.journalCmdModelFields)((0, dispatch_1.resolveTierSpawn)(ctx.dispatch, resolvedTier, issue)),
     });
     ctx.result.redispatched.push(unit);
     // Respawn immediately on the recovering rail — recovering → running. A
@@ -1125,22 +1167,24 @@ function journalStaleMilestoneIfIgnored(ctx, state, unit, slot, truth, verifiedC
     });
     return (0, state_1.patchSlot)(state, slot.id, { stale_milestone_ignored_for: slot.spawned_at }, now);
 }
+const NO_DISPATCH_SIGNALS = { lastTool: null, apiError: null };
 /**
- * Read THIS dispatch's last tool call from its log slice (#591, #620) — a
- * pure, side-effect-free parse, safe to call from any slot status and any
- * number of times: unlike `recordDispatchRunLog` below, it writes nothing to
- * `runs.jsonl`, so it carries no exactly-once constraint. Exists because a
- * `verify-incomplete`/`unit-failed` decision can land on a LATER tick than
- * the one that detected the dead pid (e.g. ground truth was unreachable in
- * between) — by then the slot has moved past `running` and
- * `recordDispatchRunLog`'s guard refuses to re-read, but the dispatch's log
- * file is static once the agent has exited, so re-parsing it here yields the
- * same answer every time.
+ * Read THIS dispatch's last tool call and API-error classification (#591,
+ * #620, #629) from its log slice — a pure, side-effect-free parse, safe to
+ * call from any slot status and any number of times: unlike
+ * `recordDispatchRunLog` below, it writes nothing to `runs.jsonl`, so it
+ * carries no exactly-once constraint. Exists because a `verify-incomplete`/
+ * `dispatch-failure` decision can land on a LATER tick than the one that
+ * detected the dead pid (e.g. ground truth was unreachable in between) — by
+ * then the slot has moved past `running` and `recordDispatchRunLog`'s guard
+ * refuses to re-read, but the dispatch's log file is static once the agent
+ * has exited, so re-parsing it here yields the same answer every time.
  */
-function readLastToolForSlot(ctx, slot, unit) {
+function readDispatchSignalsForSlot(ctx, slot, unit) {
     if (slot.spawned_at === null)
-        return null;
-    return (0, core_1.parseLastToolUse)(dispatchLogSlice(ctx, slot, unit).content);
+        return NO_DISPATCH_SIGNALS;
+    const { content } = dispatchLogSlice(ctx, slot, unit);
+    return { lastTool: (0, core_1.parseLastToolUse)(content), apiError: (0, core_1.parseDispatchApiError)(content) };
 }
 /**
  * THIS dispatch's slice of the per-unit log: path, start offset, contents.
@@ -1187,7 +1231,7 @@ function recordDispatchRunLog(ctx, state, slot, unit) {
             reason: slot.spawned_at === null ? 'never-spawned' : `already-recorded-${slot.status}`,
             slot: slot.id,
         });
-        return null;
+        return NO_DISPATCH_SIGNALS;
     }
     const issue = (0, journal_1.issueOfUnit)(unit);
     const entry = issue === null ? undefined : (0, state_1.findEntry)(state, issue);
@@ -1199,7 +1243,7 @@ function recordDispatchRunLog(ctx, state, slot, unit) {
         journal(ctx, 'run-log-skipped', unit, {
             reason: issue === null ? 'not-an-issue-unit' : 'entry-gone',
         });
-        return null;
+        return NO_DISPATCH_SIGNALS;
     }
     // Report slots ride the same tier the cycle escalation ladder set at
     // dispatch time, EXCEPT `entry.tier` is deliberately not updated for a
@@ -1228,7 +1272,9 @@ function recordDispatchRunLog(ctx, state, slot, unit) {
     (0, run_log_1.finalizeRunLogEntry)(runEntry, logContent, ctx.deps.homeDir, (event, extra) => journal(ctx, event, unit, extra), { log: logFile, offset });
     // #591: the last tool this dispatch called, so an unverified exit attributes to a
     // concrete cause (e.g. `Monitor`) without opening the transcript — see `enterRecovery`.
-    return (0, core_1.parseLastToolUse)(logContent);
+    // #629: whether this SAME dispatch was a confirmed provider API error — see
+    // `completeUnitOrRecover`.
+    return { lastTool: (0, core_1.parseLastToolUse)(logContent), apiError: (0, core_1.parseDispatchApiError)(logContent) };
 }
 /** Reconcile one running slot against its polled ground truth. */
 function reconcileRunning(ctx, state, slot, truth, unit) {
@@ -1237,9 +1283,9 @@ function reconcileRunning(ctx, state, slot, truth, unit) {
     // is DETECTED, never trusted as completion (AC2/AC3).
     if (slot.pid !== null && !ctx.deps.spawnDeps.isAlive(slot.pid, slot.pid_start ?? undefined)) {
         journal(ctx, 'exit-detected', unit, { pid: slot.pid, slot: slot.id });
-        const lastTool = recordDispatchRunLog(ctx, state, slot, unit);
+        const signals = recordDispatchRunLog(ctx, state, slot, unit);
         const exited = (0, state_1.transitionSlot)(state, slot.id, 'exited', {}, now);
-        return completeUnitOrRecover(ctx, exited, unit, truth, 'verify-complete', lastTool);
+        return completeUnitOrRecover(ctx, exited, unit, truth, 'verify-complete', signals);
     }
     // The milestone poll FAILED (gh outage, missing binary) — unreachable is NOT
     // known-absent (decision 2, option A): stall and advance decisions pause for
@@ -1365,11 +1411,13 @@ function recordDispatchOutcome(ctx, state, unit, slot, suspect) {
  * event.
  */
 function completeUnitOrRecover(ctx, state, unit, truth, via, 
-// #591: the last tool this dispatch called (from `recordDispatchRunLog`'s dead-pid
-// reading), threaded through to `enterRecovery`'s `unverified-exit` evidence. The
-// `external-advance` caller passes none — it DOES record a run log (`reconcileRunning`,
-// #524), but ground truth already confirmed completion, so this path never reaches the
-// `unverified-exit` branch and the tool name has nothing to attribute.
+// #591/#629: the last tool called AND whether the exit was a confirmed
+// provider API error (from `recordDispatchRunLog`'s dead-pid reading),
+// threaded through to the unverified-exit decision below. The
+// `external-advance` caller passes none — it DOES record a run log
+// (`reconcileRunning`, #524), but ground truth already confirmed
+// completion, so this path never reaches that decision and neither signal
+// has anything to attribute.
 //
 // #620: a THUNK, not a value, for the deferred `exited`/`verifying` rail — that caller
 // has no read of its own to hand over and would otherwise have to read the dispatch log
@@ -1378,7 +1426,7 @@ function completeUnitOrRecover(ctx, state, unit, truth, via,
 // MAX_DISPATCH_LOG_BYTES (32 MiB) and its size is set by the spawned agent, so an
 // eager read is up to 32 MiB per slot per tick for as long as an outage holds slots in
 // `verifying`. Resolved once, only on the branch that consumes it.
-lastTool = null) {
+dispatchSignals = NO_DISPATCH_SIGNALS) {
     const now = ctx.deps.now();
     let next = state;
     let slot = slotOf(next, unit);
@@ -1403,12 +1451,12 @@ lastTool = null) {
     // takes the unit (AC2's "never inferred from agent exit" cut both ways:
     // the park IS the milestone, the merge is not).
     if (entry !== undefined && entry.status === 'dispatched' && (0, groundtruth_1.isParkedMilestone)(truth.milestone)) {
-        // A verified park is also proof dispatch is healthy (#505) — reset the
-        // streak exactly like the sibling `completeUnit` branch below, or a
-        // healthy park sandwiched between two unrelated units' suspect exits
-        // would be invisible to the cross-unit correlation and could still tip
-        // it into a false-positive pause.
-        const parked = recordDispatchOutcome(ctx, next, unit, slot, false);
+        // A verified park is also proof dispatch is healthy (#505/#629) — reset
+        // both streaks exactly like the sibling `completeUnit` branch below, or a
+        // healthy park sandwiched between two unrelated units' suspect/api-error
+        // exits would be invisible to the correlation and could still tip it
+        // into a false-positive pause.
+        const parked = (0, dispatch_health_1.resetDispatchApiErrorStreak)(recordDispatchOutcome(ctx, next, unit, slot, false));
         const pr = (0, groundtruth_1.prOfMilestone)(truth.milestone); // non-null: isParkedMilestone guarantees it
         if (pr === null) {
             // Structurally unreachable, and deliberately not silent if it ever is:
@@ -1442,18 +1490,30 @@ lastTool = null) {
     const verifiedComplete = (0, groundtruth_1.isVerifiedComplete)(truth.milestone, closedSignal, slot.spawned_at);
     next = journalStaleMilestoneIfIgnored(ctx, next, unit, slot, truth, verifiedComplete, closedSignal);
     if (verifiedComplete) {
-        return completeUnit(ctx, recordDispatchOutcome(ctx, next, unit, slot, false), unit, via);
+        const completed = (0, dispatch_health_1.resetDispatchApiErrorStreak)(recordDispatchOutcome(ctx, next, unit, slot, false));
+        return completeUnit(ctx, completed, unit, via);
+    }
+    const resolved = typeof dispatchSignals === 'function' ? dispatchSignals() : dispatchSignals;
+    // #629: a CONFIRMED provider API error is a dispatch failure, never an
+    // unverified exit — it skips `recordDispatchOutcome`'s timing-based
+    // suspect check entirely (the classification is deterministic, not a
+    // heuristic) and never reaches `enterRecovery`'s escalation.
+    if (resolved.apiError) {
+        // `journalFailure: false` — `enterRecovery` journals its own `causeEvent`
+        // (`'dispatch-failure'`) with this same evidence immediately below; without
+        // the flag this rail double-journals every confirmed error (#629 review).
+        const recorded = (0, dispatch_health_1.recordDispatchApiError)((event, u, extra) => journal(ctx, event, u, extra), next, unit, resolved.apiError, { journalFailure: false });
+        return enterRecovery(ctx, recorded, unit, 'dispatch-failure', (0, dispatch_health_1.dispatchApiErrorDetail)(resolved.apiError), truth, (0, dispatch_health_1.dispatchApiErrorFields)(resolved.apiError), { escalate: false });
     }
     const suspect = msSinceLastProgress(slot, now) < types_1.SUSPECT_DISPATCH_WINDOW_MS;
     next = recordDispatchOutcome(ctx, next, unit, slot, suspect);
-    const resolvedLastTool = typeof lastTool === 'function' ? lastTool() : lastTool;
     return enterRecovery(ctx, next, unit, 'verify-incomplete', 'unverified-exit', truth, {
         observed: truth.milestone
             ? `milestone ${truth.milestone.phase}/${truth.milestone.status}; closed=${truth.closed}`
             : `no milestone; closed=${truth.closed}`,
         // #591: attributes the unverified exit to a concrete cause (e.g. `Monitor`)
         // without opening the transcript. Omitted when the log yielded no tool_use.
-        ...(resolvedLastTool !== null ? { last_tool: resolvedLastTool } : {}),
+        ...(resolved.lastTool !== null ? { last_tool: resolved.lastTool } : {}),
     });
 }
 /** Re-attach or spawn a slot left `assigned` by a crash between assign and spawn. */
@@ -1472,6 +1532,14 @@ function reconcileAssigned(ctx, state, slot, unit) {
 }
 /** Reconcile a `recovering` slot: respawn with the escalated tier. */
 function reconcileRecovering(ctx, state, unit) {
+    // #629: while paused, do not resume a `recovering` slot's respawn — this is
+    // the crash-recovery rail (a sched restart caught a slot between
+    // `enterRecovery`'s transition and its own `spawnUnit` call, OR `enterRecovery`
+    // itself deliberately parked an unescalated redispatch here, see its own
+    // pause check). Either way, `sched resume` is what lets the tick loop reach
+    // this function again and actually respawn.
+    if (state.paused)
+        return state;
     return spawnUnit(ctx, state, unit);
 }
 // --- Tick phases ---
@@ -1513,7 +1581,7 @@ function reconcileSlots(ctx, state, polled) {
                 // whatever `reconcileRunning` saw. Passed as a thunk so the read
                 // happens only on the tick that actually reaches the unverified-exit
                 // decision, not on every tick an outage holds the slot here.
-                next = completeUnitOrRecover(ctx, next, unit, truth, 'verify-complete', () => readLastToolForSlot(ctx, slot, unit));
+                next = completeUnitOrRecover(ctx, next, unit, truth, 'verify-complete', () => readDispatchSignalsForSlot(ctx, slot, unit));
                 break;
             case 'recovering':
                 next = reconcileRecovering(ctx, next, unit);
@@ -2040,10 +2108,14 @@ function sleep(ms, shouldStop) {
                 resolve();
             }
         }, Math.min(dispatch_1.STOP_POLL_MAX_MS, Math.max(dispatch_1.STOP_POLL_MIN_MS, Math.floor(ms / 10))));
-        if (typeof timer.unref === 'function')
-            timer.unref();
-        if (typeof stopCheck.unref === 'function')
-            stopCheck.unref();
+        // NEVER unref these handles (#679). They are what holds the event loop
+        // open between ticks: spawned agents are detached and unref'd on purpose
+        // (dispatch.ts), so if the sleep timers are unref'd too, nothing keeps
+        // the process alive once a tick's async work settles — the loop drains
+        // and `sched start` exits cleanly after its first tick, silently
+        // behaving like `--once`. Ref'd timers cost nothing here: SIGINT still
+        // stops the engine promptly, because the ref'd stop-check polls
+        // shouldStop() and resolves this sleep early.
     });
 }
 //# sourceMappingURL=engine.js.map