@ai-dossier/sched 0.23.2 → 0.24.0

package/dist/engine.js

@@ -73,6 +73,7 @@ exports.runLoop = runLoop;
 const core_1 = require("@ai-dossier/core");
 const batch_dispatch_1 = require("./batch-dispatch");
 const dispatch_1 = require("./dispatch");
+const dispatch_health_1 = require("./dispatch-health");
 const groundtruth_1 = require("./groundtruth");
 const journal_1 = require("./journal");
 const labels_1 = require("./labels");
@@ -405,7 +406,9 @@ function reconcileLabelBlocks(ctx, state, poll) {
             const ownStillPresent = labels.some((name) => name.toLowerCase() === blockedBy.toLowerCase());
             const current = (0, labels_1.pickHardBlockLabel)(labels);
             if (!ownStillPresent && current === null) {
-                next = (0, state_1.transitionIssue)(next, issue, 'queued', { reason: null }, now);
+                // #633: an unblocked entry is a fresh attempt too — see
+                // `requeueOrphanedDispatches`.
+                next = (0, state_1.transitionIssue)(next, issue, 'queued', { reason: null, ...state_1.CLEARED_ENTRY_DEDUP_MARKERS }, now);
                 journal(ctx, 'label-cleared', unit, { reason: (0, labels_1.labelBlockReason)(blockedBy) });
                 ctx.result.labelCleared.push(unit);
             }
@@ -723,7 +726,14 @@ function writeFence(ctx, unit, issue, slot, truth) {
  * `unit-failed` line otherwise — and that indistinguishability is exactly
  * what cost imboard-monorepo#3999 (docs/agent-traps.md).
  */
-function failOrAdoptOpenPr(ctx, state, unit, slot, report, causeEvent, cause, evidence) {
+function failOrAdoptOpenPr(ctx, state, unit, slot, report, 
+// #629: never `'dispatch-failure'` in practice — `enterRecovery` only
+// reaches this function under `escalate: true` (its cap-check is itself
+// gated on `escalate`), and a `dispatch-failure` cause always passes
+// `escalate: false`. Kept as the two-member union `enterRecovery` uses
+// before its escalate check, so a maintainer reading this signature does
+// not read "a spend wall can terminally fail a unit" as a live path.
+causeEvent, cause, evidence) {
     let prCheck = 'skipped';
     if (!report && causeEvent === 'verify-incomplete') {
         if (slot.branch === null) {
@@ -746,6 +756,13 @@ function failOrAdoptOpenPr(ctx, state, unit, slot, report, causeEvent, cause, ev
                 });
             }
             prCheck = openPr === undefined ? 'unreachable' : 'none';
+            // #632: confirmed NOT one of the per-tick re-emit sites — this
+            // function always ends by either parking the unit (the branch above)
+            // or falling through to the unconditional `failUnit` below, so the
+            // unit is terminal (or parked) by the time this call returns. There is
+            // no "next tick" on which the SAME dispatch reaches this check again,
+            // so no dedup marker is needed here; the line fires exactly once, on
+            // the one terminal decision it accompanies.
             if (prCheck === 'unreachable') {
                 journal(ctx, 'ground-truth-unreachable', unit, {
                     slot: slot.id,
@@ -767,7 +784,7 @@ function failOrAdoptOpenPr(ctx, state, unit, slot, report, causeEvent, cause, ev
     // with `causeEvent === 'verify-incomplete'` thread it through
     // `completeUnitOrRecover`: `reconcileRunning`'s dead-pid rail, from
     // `recordDispatchRunLog`'s own read, and (since #620) `reconcileSlots`'
-    // `exited`/`verifying` rail, from `readLastToolForSlot`. The two agree —
+    // `exited`/`verifying` rail, from `readDispatchSignalsForSlot`. The two agree —
     // the dispatch log is static once the agent has exited — so the tool name
     // survives regardless of which tick reaches this decision. It stays
     // OPTIONAL: a slice with no parseable `tool_use` yields null, hence the
@@ -789,7 +806,14 @@ function failOrAdoptOpenPr(ctx, state, unit, slot, report, causeEvent, cause, ev
  * mechanical-starting ladder and fail MERGED-AWARE at the cap: the PR is
  * already merged, so dependents stay released.
  */
-function enterRecovery(ctx, state, unit, causeEvent, cause, truth, evidence = {}) {
+function enterRecovery(ctx, state, unit, causeEvent, cause, truth, evidence = {}, 
+// #629: `escalate: false` for a CONFIRMED provider API error — it must
+// never consume the per-unit escalation ladder or the ESCALATION_CAP
+// (a spend/rate wall is not the issue's fault), so the respawn keeps the
+// slot's CURRENT tier and `recoveries` count unchanged. Every other caller
+// keeps the default (`true`), unaffected.
+options = {}) {
+    const escalate = options.escalate ?? true;
     const issue = (0, journal_1.issueOfUnit)(unit);
     if (issue === null)
         return state;
@@ -800,21 +824,36 @@ function enterRecovery(ctx, state, unit, causeEvent, cause, truth, evidence = {}
         return state;
     killUnitAgent(ctx, state, unit);
     // #524: only the STALL path kills a still-live, not-yet-recorded agent —
-    // `causeEvent === 'verify-incomplete'` arrives from `completeUnitOrRecover`
-    // AFTER the agent's exit was already detected and recorded by the dead-pid
-    // branch of `reconcileRunning`; recording again here would double-count
-    // that same dispatch. #591: a stall kill still has a fresh log slice worth
-    // reading for `last_tool` — a hung agent's last tool is exactly what tells
-    // an operator what it hung in.
+    // `causeEvent === 'verify-incomplete'`/`'dispatch-failure'` arrive from
+    // `completeUnitOrRecover` AFTER the agent's exit was already detected and
+    // recorded by the dead-pid branch of `reconcileRunning`; recording again
+    // here would double-count that same dispatch. #591: a stall kill still has
+    // a fresh log slice worth reading for `last_tool` — a hung agent's last
+    // tool is exactly what tells an operator what it hung in.
     if (causeEvent === 'stalled') {
-        const stallLastTool = recordDispatchRunLog(ctx, state, slot, unit);
+        const stallLastTool = recordDispatchRunLog(ctx, state, slot, unit).lastTool;
         if (stallLastTool !== null)
             evidence = { ...evidence, last_tool: stallLastTool };
     }
     const report = (0, state_1.isReportSlot)(slot);
-    const nextTier = report ? (0, dispatch_1.reportTierFor)(slot.recoveries + 1) : (0, dispatch_1.escalateTier)(entry.tier);
-    if (slot.recoveries >= types_1.ESCALATION_CAP || nextTier === null) {
-        return failOrAdoptOpenPr(ctx, state, unit, slot, report, causeEvent, cause, evidence);
+    let resolvedTier;
+    if (escalate) {
+        const escalated = report ? (0, dispatch_1.reportTierFor)(slot.recoveries + 1) : (0, dispatch_1.escalateTier)(entry.tier);
+        if (slot.recoveries >= types_1.ESCALATION_CAP || escalated === null) {
+            // #629: unreachable with `causeEvent === 'dispatch-failure'` — that
+            // cause always passes `escalate: false` above, so this branch (and the
+            // narrower `causeEvent` type `failOrAdoptOpenPr` declares) is never
+            // actually asked to terminally fail a unit over a provider wall.
+            return failOrAdoptOpenPr(ctx, state, unit, slot, report, causeEvent, cause, evidence);
+        }
+        resolvedTier = escalated;
+    }
+    else {
+        // An unescalated redispatch keeps the CURRENT tier — `reportTierFor`
+        // evaluated at the unchanged `slot.recoveries` for a report slot (mirrors
+        // `recordDispatchRunLog`'s own tier expression), `entry.tier` verbatim
+        // otherwise — never `escalateTier`/`recoveries + 1`.
+        resolvedTier = report ? ((0, dispatch_1.reportTierFor)(slot.recoveries) ?? entry.tier) : entry.tier;
     }
     // Fence BEFORE the respawn (#504 AC1/AC4): `killUnitAgent` above only reaches a pid
     // this process can see and signal, and #472 proved that is not the same as a dead
@@ -823,17 +862,17 @@ function enterRecovery(ctx, state, unit, causeEvent, cause, truth, evidence = {}
     const fenced = writeFence(ctx, unit, issue, slot, truth);
     let next = (0, state_1.transitionSlot)(state, slot.id, 'recovering', {
         pid: null,
-        recoveries: slot.recoveries + 1,
+        recoveries: escalate ? slot.recoveries + 1 : slot.recoveries,
         gen: fenced ?? slot.gen,
         // Only a fence that actually landed starts the short takeover watch: an
         // unfenced redispatch is already degraded, and cutting its allowance down
         // would compound one failure with another.
         fenced_at: fenced === null ? null : now.toISOString(),
     }, now);
-    if (!report) {
+    if (!report && escalate) {
         next = {
             ...next,
-            entries: next.entries.map((e) => e.issue === issue ? { ...e, tier: nextTier, updated_at: now.toISOString() } : e),
+            entries: next.entries.map((e) => e.issue === issue ? { ...e, tier: resolvedTier, updated_at: now.toISOString() } : e),
         };
     }
     journal(ctx, causeEvent, unit, {
@@ -842,10 +881,22 @@ function enterRecovery(ctx, state, unit, causeEvent, cause, truth, evidence = {}
         ...(slot.last_progress_at !== null ? { last_progress_at: slot.last_progress_at } : {}),
         ...evidence,
     });
+    // #629: an UNESCALATED redispatch (a confirmed provider API error) has no
+    // natural bound — `recoveries`/`ESCALATION_CAP` never advance for it — so
+    // once the dispatch-health pause has fired, respawning it every tick would
+    // reproduce the exact incident this fix exists to stop, just on the
+    // per-issue rail instead of the batch one (`runBatchTick` gets the
+    // equivalent gate below). Hold the slot in `recovering`, unspawned, until
+    // `sched resume`: `reconcileRecovering` (below) is what retries it once the
+    // pause clears — no separate re-classification happens while parked, since
+    // this function is reached exactly once per dead dispatch.
+    if (!escalate && next.paused) {
+        return next;
+    }
     journal(ctx, 'redispatched', unit, {
-        tier: nextTier,
+        tier: resolvedTier,
         slot: slot.id,
-        ...(0, dispatch_1.journalCmdModelFields)((0, dispatch_1.resolveTierSpawn)(ctx.dispatch, nextTier, issue)),
+        ...(0, dispatch_1.journalCmdModelFields)((0, dispatch_1.resolveTierSpawn)(ctx.dispatch, resolvedTier, issue)),
     });
     ctx.result.redispatched.push(unit);
     // Respawn immediately on the recovering rail — recovering → running. A
@@ -980,6 +1031,96 @@ function applyProgressSignals(ctx, state, slot, truth, unit) {
 function effectiveClosedSignal(slot, truth) {
     return (0, state_1.isReportSlot)(slot) ? false : truth.closed;
 }
+/**
+ * Journal `event` for `unit` on the first tick of a new streak, then again
+ * only every `JOURNAL_DEDUP_REANNOUNCE_TICKS` ticks while it persists (#632).
+ * That window is a TICK count, not a duration: the sites reached every
+ * reconcile re-announce at ~20 min on the default 60 s interval, while
+ * `reconcileParked`/`reconcileStaleFailedParks` are gated on `prPoll.ran` and
+ * so advance once per 150 s PR poll, ~50 min. Both intervals are
+ * operator-tunable — which is why each entry also carries `since`
+ * — `ground-truth-unreachable` and `pr-watch-waiting` were previously
+ * journaled every tick the condition held, for as long as it lasted (an
+ * outage produced one entry every reconcile interval, per affected unit, for
+ * up to `max_slots` units at once).
+ *
+ * `sinceOf`/`ticksOf` read the entry's marker for THIS event family and
+ * `withMarker` writes it back — `ground-truth-unreachable` and
+ * `pr-watch-waiting` dedup independently via two separate marker pairs on
+ * `QueueEntry` (mirrors #630's `pr_watch_failed_*` on `BatchEntry`, scoped
+ * to the issue instead of the batch: these sites span both slot-held units
+ * and parked/stale-failed ones with no live slot, and `QueueEntry` is the
+ * one record every unit has either way).
+ *
+ * A caller with no `QueueEntry` to key on (should not happen for a real
+ * issue-dispatch unit) journals unconditionally rather than silently drop
+ * the line — the pre-#632 behavior, and safe: it can only make the journal
+ * more verbose, never hide a real condition.
+ *
+ * Returns the patched state — callers must thread it, or the marker is lost
+ * and the event re-fires next tick as if nothing had been recorded.
+ */
+function journalConditionIfDue(ctx, state, unit, event, sinceOf, ticksOf, withMarker, extra) {
+    const issue = (0, journal_1.issueOfUnit)(unit);
+    const entry = issue === null ? undefined : (0, state_1.findEntry)(state, issue);
+    if (issue === null || entry === undefined) {
+        journal(ctx, event, unit, extra);
+        return state;
+    }
+    const isNewStreak = sinceOf(entry) === null;
+    const ticks = isNewStreak ? 1 : ticksOf(entry) + 1;
+    const now = ctx.deps.now();
+    const since = isNewStreak ? now.toISOString() : sinceOf(entry);
+    if (isNewStreak || ticks % types_1.JOURNAL_DEDUP_REANNOUNCE_TICKS === 0) {
+        // `at` is the decision clock (AC3); `since` is the streak's onset. Both
+        // are needed: `ticks_persisted` is a TICK count, and the two rails tick at
+        // different, operator-tunable rates, so it maps to no fixed wall-clock.
+        journal(ctx, event, unit, {
+            ...extra,
+            at: now.toISOString(),
+            since,
+            ticks_persisted: ticks,
+        });
+    }
+    // `touchUpdatedAt: false` — this marker is dedup bookkeeping, not a
+    // substantive change to the entry; `QueueEntry.updated_at` is relied on
+    // elsewhere (`isStaleFailedPark`'s window, `status.ts`'s "since" display,
+    // `readiness.ts`'s dispatch tiebreak) as a clock that must not reset on a
+    // silent tick.
+    return (0, state_1.patchEntry)(state, issue, withMarker(since, ticks), now, false);
+}
+function journalGroundTruthUnreachableIfDue(ctx, state, unit, extra) {
+    return journalConditionIfDue(ctx, state, unit, 'ground-truth-unreachable', (e) => e.ground_truth_unreachable_since, (e) => e.ground_truth_unreachable_ticks, (since, ticks) => ({
+        ground_truth_unreachable_since: since,
+        ground_truth_unreachable_ticks: ticks,
+    }), extra);
+}
+function journalPrWatchWaitingIfDue(ctx, state, unit, extra) {
+    return journalConditionIfDue(ctx, state, unit, 'pr-watch-waiting', (e) => e.pr_watch_waiting_since, (e) => e.pr_watch_waiting_ticks, (since, ticks) => ({ pr_watch_waiting_since: since, pr_watch_waiting_ticks: ticks }), extra);
+}
+/**
+ * Clear a `QueueEntry`'s `ground-truth-unreachable` marker once truth
+ * answers again (#632) — a no-op when nothing was set, so callers can call
+ * this unconditionally on every tick truth is healthy without churning
+ * `updated_at` for entries that were never in a streak.
+ */
+function clearGroundTruthUnreachable(state, unit, now) {
+    const issue = (0, journal_1.issueOfUnit)(unit);
+    const entry = issue === null ? undefined : (0, state_1.findEntry)(state, issue);
+    if (issue === null || entry === undefined || entry.ground_truth_unreachable_since === null) {
+        return state;
+    }
+    return (0, state_1.patchEntry)(state, issue, { ground_truth_unreachable_since: null, ground_truth_unreachable_ticks: 0 }, now, false);
+}
+/** Same as `clearGroundTruthUnreachable`, for the `pr-watch-waiting` marker. */
+function clearPrWatchWaiting(state, unit, now) {
+    const issue = (0, journal_1.issueOfUnit)(unit);
+    const entry = issue === null ? undefined : (0, state_1.findEntry)(state, issue);
+    if (issue === null || entry === undefined || entry.pr_watch_waiting_since === null) {
+        return state;
+    }
+    return (0, state_1.patchEntry)(state, issue, { pr_watch_waiting_since: null, pr_watch_waiting_ticks: 0 }, now, false);
+}
 /**
  * #575: journal the ignored stale milestone — shared by `reconcileRunning`'s
  * external-advance check and `completeUnitOrRecover`'s verify-complete check,
@@ -1026,22 +1167,24 @@ function journalStaleMilestoneIfIgnored(ctx, state, unit, slot, truth, verifiedC
     });
     return (0, state_1.patchSlot)(state, slot.id, { stale_milestone_ignored_for: slot.spawned_at }, now);
 }
+const NO_DISPATCH_SIGNALS = { lastTool: null, apiError: null };
 /**
- * Read THIS dispatch's last tool call from its log slice (#591, #620) — a
- * pure, side-effect-free parse, safe to call from any slot status and any
- * number of times: unlike `recordDispatchRunLog` below, it writes nothing to
- * `runs.jsonl`, so it carries no exactly-once constraint. Exists because a
- * `verify-incomplete`/`unit-failed` decision can land on a LATER tick than
- * the one that detected the dead pid (e.g. ground truth was unreachable in
- * between) — by then the slot has moved past `running` and
- * `recordDispatchRunLog`'s guard refuses to re-read, but the dispatch's log
- * file is static once the agent has exited, so re-parsing it here yields the
- * same answer every time.
+ * Read THIS dispatch's last tool call and API-error classification (#591,
+ * #620, #629) from its log slice — a pure, side-effect-free parse, safe to
+ * call from any slot status and any number of times: unlike
+ * `recordDispatchRunLog` below, it writes nothing to `runs.jsonl`, so it
+ * carries no exactly-once constraint. Exists because a `verify-incomplete`/
+ * `dispatch-failure` decision can land on a LATER tick than the one that
+ * detected the dead pid (e.g. ground truth was unreachable in between) — by
+ * then the slot has moved past `running` and `recordDispatchRunLog`'s guard
+ * refuses to re-read, but the dispatch's log file is static once the agent
+ * has exited, so re-parsing it here yields the same answer every time.
  */
-function readLastToolForSlot(ctx, slot, unit) {
+function readDispatchSignalsForSlot(ctx, slot, unit) {
     if (slot.spawned_at === null)
-        return null;
-    return (0, core_1.parseLastToolUse)(dispatchLogSlice(ctx, slot, unit).content);
+        return NO_DISPATCH_SIGNALS;
+    const { content } = dispatchLogSlice(ctx, slot, unit);
+    return { lastTool: (0, core_1.parseLastToolUse)(content), apiError: (0, core_1.parseDispatchApiError)(content) };
 }
 /**
  * THIS dispatch's slice of the per-unit log: path, start offset, contents.
@@ -1088,7 +1231,7 @@ function recordDispatchRunLog(ctx, state, slot, unit) {
             reason: slot.spawned_at === null ? 'never-spawned' : `already-recorded-${slot.status}`,
             slot: slot.id,
         });
-        return null;
+        return NO_DISPATCH_SIGNALS;
     }
     const issue = (0, journal_1.issueOfUnit)(unit);
     const entry = issue === null ? undefined : (0, state_1.findEntry)(state, issue);
@@ -1100,7 +1243,7 @@ function recordDispatchRunLog(ctx, state, slot, unit) {
         journal(ctx, 'run-log-skipped', unit, {
             reason: issue === null ? 'not-an-issue-unit' : 'entry-gone',
         });
-        return null;
+        return NO_DISPATCH_SIGNALS;
     }
     // Report slots ride the same tier the cycle escalation ladder set at
     // dispatch time, EXCEPT `entry.tier` is deliberately not updated for a
@@ -1129,7 +1272,9 @@ function recordDispatchRunLog(ctx, state, slot, unit) {
     (0, run_log_1.finalizeRunLogEntry)(runEntry, logContent, ctx.deps.homeDir, (event, extra) => journal(ctx, event, unit, extra), { log: logFile, offset });
     // #591: the last tool this dispatch called, so an unverified exit attributes to a
     // concrete cause (e.g. `Monitor`) without opening the transcript — see `enterRecovery`.
-    return (0, core_1.parseLastToolUse)(logContent);
+    // #629: whether this SAME dispatch was a confirmed provider API error — see
+    // `completeUnitOrRecover`.
+    return { lastTool: (0, core_1.parseLastToolUse)(logContent), apiError: (0, core_1.parseDispatchApiError)(logContent) };
 }
 /** Reconcile one running slot against its polled ground truth. */
 function reconcileRunning(ctx, state, slot, truth, unit) {
@@ -1138,21 +1283,25 @@ function reconcileRunning(ctx, state, slot, truth, unit) {
     // is DETECTED, never trusted as completion (AC2/AC3).
     if (slot.pid !== null && !ctx.deps.spawnDeps.isAlive(slot.pid, slot.pid_start ?? undefined)) {
         journal(ctx, 'exit-detected', unit, { pid: slot.pid, slot: slot.id });
-        const lastTool = recordDispatchRunLog(ctx, state, slot, unit);
+        const signals = recordDispatchRunLog(ctx, state, slot, unit);
         const exited = (0, state_1.transitionSlot)(state, slot.id, 'exited', {}, now);
-        return completeUnitOrRecover(ctx, exited, unit, truth, 'verify-complete', lastTool);
+        return completeUnitOrRecover(ctx, exited, unit, truth, 'verify-complete', signals);
     }
     // The milestone poll FAILED (gh outage, missing binary) — unreachable is NOT
     // known-absent (decision 2, option A): stall and advance decisions pause for
     // this unit until truth returns. The dead-pid rail above still ran — local
     // truth needs no network.
     if (!truth.reachable) {
-        journal(ctx, 'ground-truth-unreachable', unit, {
+        return journalGroundTruthUnreachableIfDue(ctx, state, unit, {
             slot: slot.id,
             detail: 'stall/advance decisions paused until truth returns',
         });
-        return state;
     }
+    // #632: truth answered this tick — any unreachable streak recorded against
+    // a PREVIOUS tick is over.
+    // Named for what it holds — a SchedState with the streak cleared — not for
+    // `truth.reachable`, the boolean five lines up.
+    const cleared = clearGroundTruthUnreachable(state, unit, now);
     // Ground truth says the unit is DONE while the agent still holds the slot —
     // externally-advanced state (AC3): reclaim the slot, kill the leftover agent.
     // A parked milestone is deliberately NOT an advance: a detached run parks
@@ -1165,7 +1314,7 @@ function reconcileRunning(ctx, state, slot, truth, unit) {
     const verifiedComplete = (0, groundtruth_1.isVerifiedComplete)(truth.milestone, closedSignal, slot.spawned_at);
     // Threaded, not discarded: the once-per-dispatch marker lives in the state
     // this returns.
-    const marked = journalStaleMilestoneIfIgnored(ctx, state, unit, slot, truth, verifiedComplete, closedSignal);
+    const marked = journalStaleMilestoneIfIgnored(ctx, cleared, unit, slot, truth, verifiedComplete, closedSignal);
     if (verifiedComplete && !(0, groundtruth_1.isParkedMilestone)(truth.milestone)) {
         journal(ctx, 'external-advance', unit, {
             pid: slot.pid,
@@ -1262,11 +1411,13 @@ function recordDispatchOutcome(ctx, state, unit, slot, suspect) {
  * event.
  */
 function completeUnitOrRecover(ctx, state, unit, truth, via, 
-// #591: the last tool this dispatch called (from `recordDispatchRunLog`'s dead-pid
-// reading), threaded through to `enterRecovery`'s `unverified-exit` evidence. The
-// `external-advance` caller passes none — it DOES record a run log (`reconcileRunning`,
-// #524), but ground truth already confirmed completion, so this path never reaches the
-// `unverified-exit` branch and the tool name has nothing to attribute.
+// #591/#629: the last tool called AND whether the exit was a confirmed
+// provider API error (from `recordDispatchRunLog`'s dead-pid reading),
+// threaded through to the unverified-exit decision below. The
+// `external-advance` caller passes none — it DOES record a run log
+// (`reconcileRunning`, #524), but ground truth already confirmed
+// completion, so this path never reaches that decision and neither signal
+// has anything to attribute.
 //
 // #620: a THUNK, not a value, for the deferred `exited`/`verifying` rail — that caller
 // has no read of its own to hand over and would otherwise have to read the dispatch log
@@ -1275,7 +1426,7 @@ function completeUnitOrRecover(ctx, state, unit, truth, via,
 // MAX_DISPATCH_LOG_BYTES (32 MiB) and its size is set by the spawned agent, so an
 // eager read is up to 32 MiB per slot per tick for as long as an outage holds slots in
 // `verifying`. Resolved once, only on the branch that consumes it.
-lastTool = null) {
+dispatchSignals = NO_DISPATCH_SIGNALS) {
     const now = ctx.deps.now();
     let next = state;
     let slot = slotOf(next, unit);
@@ -1289,11 +1440,10 @@ lastTool = null) {
     // hold the exit in `verifying` until truth returns, then decide. The agent
     // is already gone; no slot work is lost by waiting.
     if (!truth.reachable) {
-        journal(ctx, 'ground-truth-unreachable', unit, {
+        return journalGroundTruthUnreachableIfDue(ctx, next, unit, {
             slot: slot.id,
             detail: 'exit verification paused until truth returns',
         });
-        return next;
     }
     const issue = (0, journal_1.issueOfUnit)(unit);
     const entry = issue !== null ? (0, state_1.findEntry)(next, issue) : undefined;
@@ -1301,12 +1451,12 @@ lastTool = null) {
     // takes the unit (AC2's "never inferred from agent exit" cut both ways:
     // the park IS the milestone, the merge is not).
     if (entry !== undefined && entry.status === 'dispatched' && (0, groundtruth_1.isParkedMilestone)(truth.milestone)) {
-        // A verified park is also proof dispatch is healthy (#505) — reset the
-        // streak exactly like the sibling `completeUnit` branch below, or a
-        // healthy park sandwiched between two unrelated units' suspect exits
-        // would be invisible to the cross-unit correlation and could still tip
-        // it into a false-positive pause.
-        const parked = recordDispatchOutcome(ctx, next, unit, slot, false);
+        // A verified park is also proof dispatch is healthy (#505/#629) — reset
+        // both streaks exactly like the sibling `completeUnit` branch below, or a
+        // healthy park sandwiched between two unrelated units' suspect/api-error
+        // exits would be invisible to the correlation and could still tip it
+        // into a false-positive pause.
+        const parked = (0, dispatch_health_1.resetDispatchApiErrorStreak)(recordDispatchOutcome(ctx, next, unit, slot, false));
         const pr = (0, groundtruth_1.prOfMilestone)(truth.milestone); // non-null: isParkedMilestone guarantees it
         if (pr === null) {
             // Structurally unreachable, and deliberately not silent if it ever is:
@@ -1314,15 +1464,24 @@ lastTool = null) {
             // journaled leaves it re-deciding the same way every tick, forever,
             // against an empty trail. Return the recorded state (the pre-#596 form
             // did — `parkUnit`'s internal guard returned the state it was handed),
-            // not the un-recorded `next`.
-            journal(ctx, 'ground-truth-unreachable', unit, {
+            // not the un-recorded `next`. #632: dedup like the sibling check above
+            // — same marker, since only one of the two can be live for this unit
+            // on a given tick (this branch is reached only after `truth.reachable`
+            // already held).
+            return journalGroundTruthUnreachableIfDue(ctx, parked, unit, {
                 slot: slot.id,
                 detail: `parked milestone (run=${truth.milestone?.run ?? 'unknown'}) carries no parseable pr= key — holding in verifying`,
             });
-            return parked;
         }
-        return parkUnit(ctx, parked, unit, pr);
+        // #632: a pr= key was found — both flavors of this unit's
+        // ground-truth-unreachable streak (unreachable poll, unparseable pr=)
+        // are resolved.
+        return parkUnit(ctx, clearGroundTruthUnreachable(parked, unit, now), unit, pr);
     }
+    // #632: truth answered this tick and this unit isn't stuck on the
+    // missing-pr= edge above — any streak recorded against a PREVIOUS tick
+    // (either flavor) is over.
+    next = clearGroundTruthUnreachable(next, unit, now);
     // #575: fence to THIS dispatch's `spawned_at` — an agent that exited having
     // posted nothing new must not read as complete against the issue's
     // PREVIOUS run's report milestone; it falls through to the recovery ladder
@@ -1331,18 +1490,30 @@ lastTool = null) {
     const verifiedComplete = (0, groundtruth_1.isVerifiedComplete)(truth.milestone, closedSignal, slot.spawned_at);
     next = journalStaleMilestoneIfIgnored(ctx, next, unit, slot, truth, verifiedComplete, closedSignal);
     if (verifiedComplete) {
-        return completeUnit(ctx, recordDispatchOutcome(ctx, next, unit, slot, false), unit, via);
+        const completed = (0, dispatch_health_1.resetDispatchApiErrorStreak)(recordDispatchOutcome(ctx, next, unit, slot, false));
+        return completeUnit(ctx, completed, unit, via);
+    }
+    const resolved = typeof dispatchSignals === 'function' ? dispatchSignals() : dispatchSignals;
+    // #629: a CONFIRMED provider API error is a dispatch failure, never an
+    // unverified exit — it skips `recordDispatchOutcome`'s timing-based
+    // suspect check entirely (the classification is deterministic, not a
+    // heuristic) and never reaches `enterRecovery`'s escalation.
+    if (resolved.apiError) {
+        // `journalFailure: false` — `enterRecovery` journals its own `causeEvent`
+        // (`'dispatch-failure'`) with this same evidence immediately below; without
+        // the flag this rail double-journals every confirmed error (#629 review).
+        const recorded = (0, dispatch_health_1.recordDispatchApiError)((event, u, extra) => journal(ctx, event, u, extra), next, unit, resolved.apiError, { journalFailure: false });
+        return enterRecovery(ctx, recorded, unit, 'dispatch-failure', (0, dispatch_health_1.dispatchApiErrorDetail)(resolved.apiError), truth, (0, dispatch_health_1.dispatchApiErrorFields)(resolved.apiError), { escalate: false });
     }
     const suspect = msSinceLastProgress(slot, now) < types_1.SUSPECT_DISPATCH_WINDOW_MS;
     next = recordDispatchOutcome(ctx, next, unit, slot, suspect);
-    const resolvedLastTool = typeof lastTool === 'function' ? lastTool() : lastTool;
     return enterRecovery(ctx, next, unit, 'verify-incomplete', 'unverified-exit', truth, {
         observed: truth.milestone
             ? `milestone ${truth.milestone.phase}/${truth.milestone.status}; closed=${truth.closed}`
             : `no milestone; closed=${truth.closed}`,
         // #591: attributes the unverified exit to a concrete cause (e.g. `Monitor`)
         // without opening the transcript. Omitted when the log yielded no tool_use.
-        ...(resolvedLastTool !== null ? { last_tool: resolvedLastTool } : {}),
+        ...(resolved.lastTool !== null ? { last_tool: resolved.lastTool } : {}),
     });
 }
 /** Re-attach or spawn a slot left `assigned` by a crash between assign and spawn. */
@@ -1361,6 +1532,14 @@ function reconcileAssigned(ctx, state, slot, unit) {
 }
 /** Reconcile a `recovering` slot: respawn with the escalated tier. */
 function reconcileRecovering(ctx, state, unit) {
+    // #629: while paused, do not resume a `recovering` slot's respawn — this is
+    // the crash-recovery rail (a sched restart caught a slot between
+    // `enterRecovery`'s transition and its own `spawnUnit` call, OR `enterRecovery`
+    // itself deliberately parked an unescalated redispatch here, see its own
+    // pause check). Either way, `sched resume` is what lets the tick loop reach
+    // this function again and actually respawn.
+    if (state.paused)
+        return state;
     return spawnUnit(ctx, state, unit);
 }
 // --- Tick phases ---
@@ -1402,7 +1581,7 @@ function reconcileSlots(ctx, state, polled) {
                 // whatever `reconcileRunning` saw. Passed as a thunk so the read
                 // happens only on the tick that actually reaches the unverified-exit
                 // decision, not on every tick an outage holds the slot here.
-                next = completeUnitOrRecover(ctx, next, unit, truth, 'verify-complete', () => readLastToolForSlot(ctx, slot, unit));
+                next = completeUnitOrRecover(ctx, next, unit, truth, 'verify-complete', () => readDispatchSignalsForSlot(ctx, slot, unit));
                 break;
             case 'recovering':
                 next = reconcileRecovering(ctx, next, unit);
@@ -1422,7 +1601,12 @@ function requeueOrphanedDispatches(ctx, state) {
             continue;
         const now = ctx.deps.now();
         next = (0, state_1.transitionIssue)(next, entry.issue, 'blocked', { reason: 'orphaned-dispatch' }, now);
-        next = (0, state_1.transitionIssue)(next, entry.issue, 'queued', { reason: null }, now);
+        // #633: same rule as `requeueMember` — a requeue is a fresh attempt, so a
+        // dedup streak recorded against the PREVIOUS dispatch must not suppress
+        // that condition's first occurrence on this one. Without it, a crash
+        // during an outage leaves `ground_truth_unreachable_ticks` mid-streak and
+        // the redispatched unit stays silent until the next re-announcement.
+        next = (0, state_1.transitionIssue)(next, entry.issue, 'queued', { reason: null, ...state_1.CLEARED_ENTRY_DEDUP_MARKERS }, now);
         journal(ctx, 'requeued', `issue:${entry.issue}`, {
             detail: 'orphaned dispatch requeued after restart',
         });
@@ -1457,14 +1641,18 @@ function reconcileParked(ctx, state, prPoll) {
             if (!prPoll.truths.has(issue)) {
                 continue; // parked AFTER the poll ran (this tick) — next cadence picks it up
             }
-            journal(ctx, 'ground-truth-unreachable', unit, {
+            next = journalGroundTruthUnreachableIfDue(ctx, next, unit, {
                 detail: 'pr watch paused until truth returns',
             });
             continue;
         }
+        // #632: truth answered this tick — any unreachable streak recorded
+        // against a PREVIOUS tick is over.
+        next = clearGroundTruthUnreachable(next, unit, ctx.deps.now());
         const failWatch = (reason) => {
             journal(ctx, 'pr-watch-failed', unit, { reason, pr: entry.pr });
-            return failUnit(ctx, next, unit, reason);
+            // #632: the watch is ending (terminal failure) — no more "waiting".
+            return clearPrWatchWaiting(failUnit(ctx, next, unit, reason), unit, ctx.deps.now());
         };
         // #501: MERGED is checked FIRST, before any failure rail. A PR that is
         // genuinely merged is merged regardless of a leftover `auto-merge-blocked`
@@ -1478,7 +1666,7 @@ function reconcileParked(ctx, state, prPoll) {
             // AC1: the issue must ALSO be closed (a merged PR auto-closes it) —
             // until GitHub propagates, the unit stays parked and keeps watching.
             if (prPoll.closed.get(issue) !== true) {
-                journal(ctx, 'pr-watch-waiting', unit, {
+                next = journalPrWatchWaitingIfDue(ctx, next, unit, {
                     pr: entry.pr,
                     mergedAt: truth.mergedAt,
                     detail: 'merge seen but issue not closed — keep watching',
@@ -1486,6 +1674,7 @@ function reconcileParked(ctx, state, prPoll) {
                 continue;
             }
             next = (0, state_1.transitionIssue)(next, issue, 'shipped', { reason: null }, ctx.deps.now());
+            next = clearPrWatchWaiting(next, unit, ctx.deps.now());
             journal(ctx, 'merge-accepted', unit, { pr: entry.pr, mergedAt: truth.mergedAt });
             ctx.result.mergeAccepted.push(unit);
             continue;
@@ -1501,7 +1690,12 @@ function reconcileParked(ctx, state, prPoll) {
         if (truth.state === 'CLOSED' && truth.mergedAt === null) {
             next = failWatch('pr-closed-unmerged');
         }
-        // OPEN (or mergeable UNKNOWN) — keep watching.
+        else {
+            // OPEN (or mergeable UNKNOWN) — keep watching. #632: this tick's truth
+            // is NOT "merge seen but not closed", so a waiting streak from an
+            // earlier tick's merge-then-reverted flicker is over.
+            next = clearPrWatchWaiting(next, unit, ctx.deps.now());
+        }
     }
     return next;
 }
@@ -1537,15 +1731,23 @@ function reconcileStaleFailedParks(ctx, state, prPoll) {
             if (!prPoll.truths.has(issue)) {
                 continue; // failed AFTER the poll ran this tick — next cadence picks it up
             }
-            journal(ctx, 'ground-truth-unreachable', unit, {
+            next = journalGroundTruthUnreachableIfDue(ctx, next, unit, {
                 detail: 'stale-failure reconcile paused until truth returns',
             });
             continue;
         }
-        if (!isPrMerged(truth))
-            continue; // still blocked/open/conflicting — stays failed
+        // #632: truth answered this tick — any unreachable streak recorded
+        // against a PREVIOUS tick is over.
+        next = clearGroundTruthUnreachable(next, unit, ctx.deps.now());
+        if (!isPrMerged(truth)) {
+            // Still blocked/open/conflicting — stays failed. #632: not "merge seen
+            // but not closed" either, so a waiting streak from an earlier tick's
+            // merge-then-reverted flicker is over.
+            next = clearPrWatchWaiting(next, unit, ctx.deps.now());
+            continue;
+        }
         if (prPoll.closed.get(issue) !== true) {
-            journal(ctx, 'pr-watch-waiting', unit, {
+            next = journalPrWatchWaitingIfDue(ctx, next, unit, {
                 pr: entry.pr,
                 mergedAt: truth.mergedAt,
                 reason: AUTO_MERGE_BLOCKED_REASON,
@@ -1555,6 +1757,7 @@ function reconcileStaleFailedParks(ctx, state, prPoll) {
         }
         const failedAt = entry.updated_at;
         next = (0, state_1.transitionIssue)(next, issue, 'shipped', { reason: null }, ctx.deps.now());
+        next = clearPrWatchWaiting(next, unit, ctx.deps.now());
         journal(ctx, 'stale-failure-reconciled', unit, {
             pr: entry.pr,
             mergedAt: truth.mergedAt,
@@ -1656,6 +1859,17 @@ function runTeardownFor(deps, issue) {
     const info = deps.groundTruth.setupInfo(issue);
     const unit = `issue:${issue}`;
     if (info === undefined) {
+        // #633: AUDITED, NOT DEDUPED. This is a tenth site with #632's shape —
+        // `teardownPendingIssues` reaches it every tick and a `null` return leaves
+        // `cleanup` unset, so an unreachable `setupInfo` re-emits this line once
+        // per reconcile interval for as long as the outage lasts.
+        //
+        // Left as-is deliberately: #632 enumerated nine sites and its plan makes
+        // the boundary load-bearing ("a site outside the table is a NEW issue, not
+        // a reason to widen this one") — widening is what dissolved b-07. The fix
+        // is tracked in #636, and is not a one-liner here: `runTeardownFor` runs
+        // OUTSIDE the store lock and has no `SchedState` to patch, so the marker
+        // has to be threaded through the caller's second lock pass.
         deps.journal.append((0, journal_1.unitEvent)('ground-truth-unreachable', unit, {
             detail: 'teardown paused until truth returns',
         }), deps.now());