@tokenfactory/acc-runner 0.40.27 → 0.40.29

package/dist/watch.js

@@ -55,6 +55,7 @@ import { getChatEngine } from "./engines/registry.js";
 // a `watch` runner — the runner serves its bound user's chat turns behind their
 // companion. Glue lives in watch-chat/wire.ts to keep this file the task loop.
 import { startFleetChatServing, } from "./watch-chat/wire.js";
+import { ClaimDeadman, readDispatchSignal, DEFAULT_DEADMAN_CHECK_MS, DEFAULT_FALLBACK_POLL_MS, CLAIM_CHANNEL_WEDGED_VERB, CLAIM_CHANNEL_RECOVERED_VERB, CLAIM_CHANNEL_REBUILD_VERB, CLAIM_CHANNEL_FALLBACK_VERB, } from "./claim-deadman/index.js";
 import { unknownAccountIdentity } from "./engines/account-identity.js";
 import { checkVersion, compareSemver } from "./version-check.js";
 import { PACKAGE_VERSION } from "./pkg-version.js";
@@ -120,6 +121,22 @@ const DEFAULT_CLAIM_WATCHDOG_CHECK_MS = 30_000;
 // regressions of the same shape. Default 60s — prompt recovery without a third
 // frequent DB poll racing the watchdog. Knob: `claim_backstop_interval_ms`.
 const DEFAULT_CLAIM_BACKSTOP_INTERVAL_MS = 60_000;
+// CLAIM-DEADMAN: dead-man switch for a WEDGED claim channel
+// that the 90s watchdog resubscribe can't clear. Live incident 2026-07-22: a
+// runner heartbeated fine (hb_age 0min) but its Realtime claim subscription was
+// dead AND the watchdog resubscribe had itself stopped firing — 12 queued / 0
+// running with two online runners, cleared only by a manual restart. The deadman
+// tracks pending-dispatch (work the server is offering) vs the last processed
+// claim; when work is pending and no claim lands for CLAIM_DEADMAN_WEDGE_MS
+// (default 120s, > the 90s watchdog so the cheap resubscribe gets first crack)
+// it (1) TEARS DOWN + FULLY REBUILDS the Realtime client up to
+// CLAIM_DEADMAN_MAX_REBUILDS times, then (2) falls back to degraded DB-poll
+// claiming every CLAIM_DEADMAN_FALLBACK_POLL_MS. CLAIM_DEADMAN_CHECK_MS is the
+// evaluation cadence. Tests override all of these for speed. See
+// packages/acc-runner/src/claim-deadman/deadman.ts + docs/acc/CLAIM_DEADMAN.md.
+const DEFAULT_CLAIM_DEADMAN_WEDGE_MS = 120_000;
+const DEFAULT_CLAIM_DEADMAN_REBUILD_GRACE_MS = 15_000;
+const DEFAULT_CLAIM_DEADMAN_MAX_REBUILDS = 2;
 // GIT-AUTH-ALERT: how often the runner probes its base clone's git credential
 // (a masked `ls-remote origin` dry-run). The same 60s cadence detects the first
 // failure AND polls for recovery — while degraded, task-lane claiming is paused
@@ -208,6 +225,13 @@ function enqueue(state, taskId, factory) {
     state.seen.add(taskId);
     state.queue.push(taskId);
     touchActivity(state);
+    // CLAIM-DEADMAN: a claim was processed via SOME path
+    // (realtime / poll / scan / watchdog / the deadman's own DB-poll fallback).
+    // Reset the deadman's staleness clock so a runner that IS claiming — even
+    // slowly, even degraded — is never treated as wedged. Recovery itself is
+    // declared by the offered backlog draining (observeDispatchSignal), so this
+    // never prematurely clears a wedge episode.
+    state.claimDeadman?.observeClaimProcessed();
     void pump(state, factory);
     return true;
 }
@@ -656,6 +680,230 @@ async function claimBackstopScan(state, factory) {
         state.backstopScanning = false;
     }
 }
+/**
+ * CLAIM-DEADMAN: a runner may have free capacity but be
+ * deliberately not claiming (quarantined, capacity-paused, review-claim-paused,
+ * or already at its effective task-claim ceiling). In those states pending work
+ * SHOULD sit queued, so it must NOT be read as a wedge. Mirrors the
+ * claimBackstopScan guard. Returns true only when the runner both wants to and
+ * can claim another task.
+ */
+function deadmanCanClaim(state) {
+    if (state.quarantined || state.pausedCapacity || state.capacityClaimPaused) {
+        return false;
+    }
+    const limit = computeTaskClaimLimit({
+        concurrencyLimit: state.concurrencyLimit,
+        reservedReviewSlots: state.reservedReviewSlots,
+        capacityProbe: state.capacityProbe,
+        claimPaused: state.capacityClaimPaused,
+    });
+    return state.running.size < limit;
+}
+/**
+ * CLAIM-DEADMAN: parse the pending-dispatch fields off a `heartbeat_runner`
+ * response. Tolerant of a scalar row, a single-row array, or null (today's
+ * void return) — returns null unless a numeric `pending_dispatch` is present, so
+ * a pre-extension server leaves the deadman on its assigned-queued read.
+ */
+export function parseHeartbeatDeadmanFields(data) {
+    const row = Array.isArray(data) ? data[0] : data;
+    if (!row || typeof row !== "object")
+        return null;
+    const r = row;
+    if (typeof r.pending_dispatch !== "number")
+        return null;
+    return {
+        pending_dispatch: Math.max(0, Math.floor(r.pending_dispatch)),
+        suspect: Boolean(r.suspect),
+    };
+}
+/**
+ * CLAIM-DEADMAN: emit the audit event + operator-facing alert for a claim-
+ * channel state change. `runner.claim_channel_wedged` / `..._recovered` are the
+ * UI alert breadcrumbs; the payload NAMES the runner (id + label) so the derived
+ * notifications read model can surface "runner X's claim channel is wedged"
+ * without a join. Best-effort by contract — a logging failure never disturbs the
+ * watch loop. Also mirrors to stderr so a locally-tailed runner shows it.
+ */
+function emitDeadmanEvent(state, verb, payload, humanLine) {
+    process.stderr.write(`[acc-runner] ${humanLine}\n`);
+    void (async () => {
+        try {
+            await state.supabase.rpc("log_activity", {
+                p_verb: verb,
+                p_target_id: state.session.runner_id,
+                p_payload: {
+                    // NAMES the runner so the derived notifications read model can surface
+                    // "runner <id>'s claim channel is wedged" without a join (the UI
+                    // resolves the display name from acc.runners by this id).
+                    runner_id: state.session.runner_id,
+                    version: PACKAGE_VERSION,
+                    ...payload,
+                },
+                p_target_type: "runner",
+            });
+        }
+        catch {
+            /* best-effort breadcrumb */
+        }
+    })();
+}
+/**
+ * CLAIM-DEADMAN: tear down + FULLY REBUILD the Realtime client — not just a
+ * resubscribe. The v0.68 resubscribe / v0.73 watchdog both remove+re-add a
+ * channel on the SAME Supabase client, so a wedged underlying socket / Realtime
+ * instance survives them (the incident's failure mode). This discards the whole
+ * client and builds a fresh one around the same JWT + realtime config (mirrors
+ * doRefresh/doFailover's client rebuild, minus the token change), then recovers
+ * anything the dead channel missed. Single-flight via `deadmanRebuilding`.
+ */
+export async function rebuildRealtimeClient(state, factory, attempt) {
+    if (state.stopped || state.deadmanRebuilding)
+        return;
+    state.deadmanRebuilding = true;
+    try {
+        // Tear down the old channel + socket as fully as the client allows.
+        try {
+            await state.supabase.removeChannel(state.channel);
+        }
+        catch {
+            /* channel may already be dead */
+        }
+        try {
+            await state.supabase.removeAllChannels?.();
+        }
+        catch {
+            /* best-effort */
+        }
+        try {
+            // any-allowed: `.realtime` is not on the narrowed RunnerSupabaseClient type
+            // but supabase-js exposes a RealtimeClient with disconnect() here — this is
+            // the "fully rebuild, not just resubscribe" teardown of the live socket.
+            // eslint-disable-next-line @typescript-eslint/no-explicit-any
+            state.supabase.realtime?.disconnect?.();
+        }
+        catch {
+            /* best-effort */
+        }
+        // Fresh client around the current (still-valid) credential.
+        state.supabase = createRunnerClient(state.cfg, state.session.access_token, state.session.realtime_config);
+        subscribeChannel(state, factory);
+        // FLEET-SERVE (wire): hot-swap the chat lane onto the fresh client so chat
+        // serving survives the rebuild (mirrors doRefresh/doFailover).
+        await state.fleetChat?.refreshClient(state.supabase, state.session.access_token);
+        emitDeadmanEvent(state, CLAIM_CHANNEL_REBUILD_VERB, { attempt }, `claim-deadman: rebuilding Realtime client (attempt ${attempt}) — ` +
+            `claim channel wedged with a live heartbeat`);
+        // Recover anything the wedged channel missed. The seen/seenReviews guards
+        // dedupe against a late re-delivery on the fresh channel.
+        void reassertRunningClaims(state);
+        void pollOnce(state, factory);
+        void claimScan(state, factory, "periodic");
+        void reviewScan(state, "periodic");
+    }
+    catch (err) {
+        process.stderr.write(`[acc-runner] claim-deadman rebuild failed: ${err.message}\n`);
+    }
+    finally {
+        state.deadmanRebuilding = false;
+    }
+}
+/**
+ * CLAIM-DEADMAN: the degraded DB-poll fallback loop. Once the rebuild budget is
+ * spent the runner claims by polling the claim path directly — a full epoch-
+ * watermark claim scan plus the `seen`-reconcile backstop — every
+ * `deadmanFallbackPollMs`. Degraded (no realtime latency) but FUNCTIONAL: tasks
+ * get claimed with zero human action while the WS stays dead. Idempotent —
+ * starting an already-running loop is a no-op.
+ */
+function startDeadmanFallback(state, factory) {
+    if (state.stopped || state.deadmanFallbackTimer)
+        return;
+    emitDeadmanEvent(state, CLAIM_CHANNEL_FALLBACK_VERB, { poll_ms: state.deadmanFallbackPollMs }, `claim-deadman: Realtime rebuild budget spent — falling back to DB-poll ` +
+        `claiming every ${state.deadmanFallbackPollMs}ms (degraded but functional)`);
+    const tick = () => {
+        void (async () => {
+            if (state.stopped)
+                return;
+            await claimScan(state, factory, "periodic");
+            await claimBackstopScan(state, factory);
+        })();
+    };
+    tick(); // claim immediately, don't wait a full interval
+    state.deadmanFallbackTimer = setInterval(tick, state.deadmanFallbackPollMs);
+    state.deadmanFallbackTimer.unref?.();
+}
+/** CLAIM-DEADMAN: stop the degraded DB-poll fallback loop (on recovery / stop). */
+function stopDeadmanFallback(state) {
+    if (state.deadmanFallbackTimer) {
+        clearInterval(state.deadmanFallbackTimer);
+        state.deadmanFallbackTimer = null;
+    }
+}
+/**
+ * CLAIM-DEADMAN: the dead-man evaluation. Reads how much
+ * work the server is offering this runner (pending-dispatch — from the heartbeat
+ * response when the server supplies it, else the assigned-queued read over the
+ * same HTTP path that survives a dead WS), feeds it + the last-claim clock to the
+ * pure ClaimDeadman, and acts on its decision:
+ *
+ *   - recovered (offered backlog drained after a wedge episode) → stop the
+ *     fallback loop + emit `runner.claim_channel_recovered`.
+ *   - rebuild(N) → full Realtime-client rebuild (rebuildRealtimeClient).
+ *   - fallback → engage/keep the degraded DB-poll claim loop.
+ *
+ * A heartbeat failure short-circuits to observeHeartbeatFailure (a network/auth
+ * outage is a different failure with its own handling — not a wedge). Bounded
+ * and side-effect-safe: never throws out of the loop. Single-flight via
+ * `claimDeadmanChecking`.
+ */
+async function claimDeadmanScan(state, factory) {
+    if (state.stopped || state.claimDeadmanChecking)
+        return;
+    state.claimDeadmanChecking = true;
+    try {
+        if (state.heartbeatFailures > 0) {
+            state.claimDeadman.observeHeartbeatFailure();
+            return;
+        }
+        // Only pending work the runner WOULD claim counts as offered — a runner that
+        // is deliberately not claiming (paused/quarantined/at-capacity) is not wedged.
+        let signal;
+        if (deadmanCanClaim(state)) {
+            signal = await readDispatchSignal(state.supabase, state.session.runner_id, state.lastHeartbeatDeadmanFields);
+        }
+        else {
+            signal = { pendingDispatch: 0, suspect: false };
+        }
+        const { recovered } = state.claimDeadman.observeDispatchSignal(signal);
+        if (recovered) {
+            stopDeadmanFallback(state);
+            emitDeadmanEvent(state, CLAIM_CHANNEL_RECOVERED_VERB, {}, `claim-deadman: claim channel recovered — offered backlog drained, ` +
+                `runner is claiming normally again`);
+        }
+        const decision = state.claimDeadman.tick();
+        if (decision.wedged) {
+            emitDeadmanEvent(state, CLAIM_CHANNEL_WEDGED_VERB, {
+                pending_dispatch: decision.pendingDispatch,
+                ms_since_claim: decision.msSinceClaim,
+            }, `claim-deadman: claim channel WEDGED — ${decision.pendingDispatch} ` +
+                `task(s) offered but none claimed for ${Math.round(decision.msSinceClaim / 1000)}s ` +
+                `with a live heartbeat; self-healing`);
+        }
+        if (decision.action === "rebuild") {
+            await rebuildRealtimeClient(state, factory, decision.rebuildAttempt ?? 1);
+        }
+        else if (decision.action === "fallback") {
+            startDeadmanFallback(state, factory);
+        }
+    }
+    catch (err) {
+        process.stderr.write(`[acc-runner] claim deadman scan failed: ${err.message}\n`);
+    }
+    finally {
+        state.claimDeadmanChecking = false;
+    }
+}
 /**
  * v0.65 (T-65-2): full review scan — the review twin of claimScan. Lists EVERY
  * pending review assigned to this runner with an epoch watermark (`1970-…Z`)
@@ -923,6 +1171,7 @@ export async function watchCommand(options = {}) {
                 reviewScanOnce: async () => { },
                 claimWatchdogOnce: async () => { },
                 claimBackstopOnce: async () => { },
+                claimDeadmanOnce: async () => { },
                 gitAuthProbeOnce: async () => false,
                 runningCount: () => 0,
             };
@@ -1105,6 +1354,20 @@ export async function watchCommand(options = {}) {
         claimBackstopTimer: undefined,
         claimBackstopIntervalMs: options.claimBackstopIntervalMs ?? DEFAULT_CLAIM_BACKSTOP_INTERVAL_MS,
         backstopScanning: false,
+        // CLAIM-DEADMAN: wedge dead-man switch. The pure state
+        // machine carries its own clock (Date.now); tests inject tiny thresholds.
+        claimDeadman: new ClaimDeadman({
+            wedgeThresholdMs: options.claimDeadmanWedgeMs ?? DEFAULT_CLAIM_DEADMAN_WEDGE_MS,
+            rebuildGraceMs: options.claimDeadmanRebuildGraceMs ?? DEFAULT_CLAIM_DEADMAN_REBUILD_GRACE_MS,
+            maxRebuilds: options.claimDeadmanMaxRebuilds ?? DEFAULT_CLAIM_DEADMAN_MAX_REBUILDS,
+        }),
+        claimDeadmanTimer: undefined,
+        claimDeadmanCheckMs: options.claimDeadmanCheckMs ?? DEFAULT_DEADMAN_CHECK_MS,
+        claimDeadmanChecking: false,
+        deadmanRebuilding: false,
+        deadmanFallbackTimer: null,
+        deadmanFallbackPollMs: options.claimDeadmanFallbackPollMs ?? DEFAULT_FALLBACK_POLL_MS,
+        lastHeartbeatDeadmanFields: null,
         // GIT-AUTH-ALERT: controller wired below once `state` exists (its emit
         // callbacks close over `state`); timer armed in the timer-setup block.
         gitAuthController: null,
@@ -1263,6 +1526,18 @@ export async function watchCommand(options = {}) {
     state.claimBackstopTimer = setInterval(() => {
         void claimBackstopScan(state, taskRunnerFactory);
     }, state.claimBackstopIntervalMs);
+    // CLAIM-DEADMAN: the wedge dead-man switch. Escalates
+    // BEYOND the 90s watchdog resubscribe (which only re-adds a channel on the
+    // same client and, per the incident, can itself stop firing): tracks pending-
+    // dispatch vs the last processed claim, and when work is offered but nothing
+    // is claimed for the wedge threshold it FULLY REBUILDS the Realtime client
+    // (twice) then degrades to DB-poll claiming — self-healing the "heartbeat-
+    // alive but claim-dead" runner with zero human action. Independent timer +
+    // single-flight inside claimDeadmanScan().
+    state.claimDeadmanTimer = setInterval(() => {
+        void claimDeadmanScan(state, taskRunnerFactory);
+    }, state.claimDeadmanCheckMs);
+    state.claimDeadmanTimer.unref?.();
     // GIT-AUTH-ALERT: periodic git-credential health probe. Armed only when the
     // controller is wired (a base clone is resolvable). The same cadence detects
     // the first auth failure AND polls for recovery while degraded. Single-flight
@@ -1368,6 +1643,8 @@ export async function watchCommand(options = {}) {
         clearInterval(state.reviewScanTimer); // v0.65 (T-65-2)
         clearInterval(state.claimWatchdogTimer); // v0.73 (T-73-2)
         clearInterval(state.claimBackstopTimer); // FIX-D
+        clearInterval(state.claimDeadmanTimer); // CLAIM-DEADMAN
+        stopDeadmanFallback(state); // CLAIM-DEADMAN
         if (state.gitAuthTimer)
             clearInterval(state.gitAuthTimer); // GIT-AUTH-ALERT
         if (state.resubscribeTimer)
@@ -1480,6 +1757,7 @@ export async function watchCommand(options = {}) {
         reviewScanOnce: () => reviewScan(state, "periodic"), // v0.65 (T-65-2)
         claimWatchdogOnce: () => claimWatchdogScan(state, taskRunnerFactory), // v0.73 (T-73-2)
         claimBackstopOnce: () => claimBackstopScan(state, taskRunnerFactory), // FIX-D
+        claimDeadmanOnce: () => claimDeadmanScan(state, taskRunnerFactory), // CLAIM-DEADMAN
         gitAuthProbeOnce: () => gitAuthProbeScan(state), // GIT-AUTH-ALERT
         runningCount: () => state.running.size,
     };
@@ -1572,12 +1850,20 @@ function scheduleHeartbeat(state, taskRunnerFactory, baseMs) {
     const delay = nextHeartbeatDelayMs(state.heartbeatFailures, baseMs);
     state.heartbeatTimer = setTimeout(() => {
         void (async () => {
-            const { error } = await state.supabase.rpc("heartbeat_runner", {
+            const { data, error } = await state.supabase.rpc("heartbeat_runner", {
                 p_id: state.session.runner_id,
                 p_version: PACKAGE_VERSION,
             });
             if (!error) {
                 state.heartbeatFailures = 0;
+                // CLAIM-DEADMAN: capture the pending-dispatch count
+                // + suspect flag when the heartbeat RESPONSE carries them, so the runner
+                // learns it is being offered work even with a dead WS. Forward-compatible:
+                // today `heartbeat_runner` returns void and this is a no-op (the deadman
+                // reads the assigned-queued count over HTTP instead). Once the RPC is
+                // extended (Track A migration) this is the authoritative signal — the
+                // deadman prefers it in readDispatchSignal.
+                state.lastHeartbeatDeadmanFields = parseHeartbeatDeadmanFields(data);
                 // RUNCFG-1: pull the operator's server-set concurrency and apply it to
                 // the live ceiling within this heartbeat. Runs only on a healthy
                 // heartbeat (a network/auth failure would fail this RPC too). Awaited so
@@ -1611,6 +1897,8 @@ function scheduleHeartbeat(state, taskRunnerFactory, baseMs) {
                     clearInterval(state.reviewScanTimer); // v0.65 (T-65-2)
                     clearInterval(state.claimWatchdogTimer); // v0.73 (T-73-2)
                     clearInterval(state.claimBackstopTimer); // FIX-D
+                    clearInterval(state.claimDeadmanTimer); // CLAIM-DEADMAN
+                    stopDeadmanFallback(state); // CLAIM-DEADMAN
                     if (state.resubscribeTimer)
                         clearTimeout(state.resubscribeTimer); // v0.68 (T-68-1)
                     process.exit(1);