@tokenfactory/acc-runner 0.41.3 → 0.41.5

package/dist/watch.js

@@ -42,7 +42,7 @@ import { TASK_PAUSED_EVENT, captureWorktreeCheckpoint, parseTaskPausedBroadcast,
 import { worktreePath } from "./runtime/worktree.js";
 import { runTask } from "./task-runner.js";
 import { runReview, } from "./runtime/reviewer.js";
-import { getQuarantine, setQuarantine, } from "./runtime/quarantine.js";
+import { getQuarantine, setQuarantine, clearQuarantine, } from "./runtime/quarantine.js";
 import { acquireSingletonLock, singletonRunnerId, SingletonLockHeldError, } from "./runtime/singleton.js";
 import { getClaudeVersion, recordTaskClaudeVersion, } from "./runtime/version-drift.js";
 import { autoUpgradeDisabled, maybeSelfUpgrade, maybeSelfUpdateOn426, } from "./runtime/self-upgrade.js";
@@ -54,7 +54,7 @@ import { GitAuthController, makeGitAuthProbe, RUNNER_GIT_AUTH_FAILED_VERB, RUNNE
 import { ResumeController } from "./capacity/resume-controller.js";
 import { RUNNER_CAPACITY_PAUSED_VERB, RUNNER_CAPACITY_RESUME_VERB, } from "./capacity/fleet-pause-verbs.js";
 import { AccountProbeScheduler } from "./engines/account-probe.js";
-import { getChatEngine } from "./engines/registry.js";
+import { getChatEngine, getEngine } from "./engines/registry.js";
 // FLEET-SERVE (wire): fleet-fallback chat serving runs ALONGSIDE the task pump on
 // a `watch` runner — the runner serves its bound user's chat turns behind their
 // companion. Glue lives in watch-chat/wire.ts to keep this file the task loop.
@@ -127,6 +127,10 @@ const DEFAULT_CLAIM_WATCHDOG_CHECK_MS = 30_000;
 // regressions of the same shape. Default 60s — prompt recovery without a third
 // frequent DB poll racing the watchdog. Knob: `claim_backstop_interval_ms`.
 const DEFAULT_CLAIM_BACKSTOP_INTERVAL_MS = 60_000;
+// v0.41.4: cadence for the env_broken quarantine-recovery re-probe. 60s gives a
+// prompt auto-recovery once claude is reachable again (e.g. after a self-update
+// window) without hammering the health probe.
+const DEFAULT_QUARANTINE_RECOVERY_INTERVAL_MS = 60_000;
 // CLAIM-DEADMAN: dead-man switch for a WEDGED claim channel
 // that the 90s watchdog resubscribe can't clear. Live incident 2026-07-22: a
 // runner heartbeated fine (hb_age 0min) but its Realtime claim subscription was
@@ -581,6 +585,16 @@ async function claimWatchdogScan(state, factory) {
             state.claimWatchdogSince.clear();
             return;
         }
+        // v0.41.4: never answer a DELIBERATE not-claiming state with a transport
+        // remedy. While quarantined or capacity-paused the runner is SUPPOSED to
+        // leave its tasks queued; a resubscribe changes nothing and produced the
+        // resubscribe storm interleaved with QUARANTINED in the claim-livelock
+        // incident. Mirror the backstop's guard and drop the stale timers so an
+        // age never spans the pause and falsely trips the instant it lifts.
+        if (state.quarantined || state.pausedCapacity) {
+            state.claimWatchdogSince.clear();
+            return;
+        }
         const { data, error } = await state.supabase.rpc("list_assigned_queued_tasks", {
             p_runner_id: state.session.runner_id,
             p_since: "1970-01-01T00:00:00.000Z",
@@ -601,20 +615,52 @@ async function claimWatchdogScan(state, factory) {
             if (!currentIds.has(id))
                 state.claimWatchdogSince.delete(id);
         }
+        // v0.41.4: same reset for the contention tracker — a task that left the
+        // queue (claimed or unassigned) is no longer contended.
+        for (const id of [...state.claimContentionSince.keys()]) {
+            if (!currentIds.has(id))
+                state.claimContentionSince.delete(id);
+        }
         // Observe: stamp the first-seen instant for newly assigned-unclaimed tasks;
-        // collect any that have now been stale past the threshold.
+        // collect any that have now been stale past the threshold, splitting off
+        // those that are merely waiting on file-lock contention (v0.41.4).
         const stale = [];
+        const contended = [];
         for (const id of currentIds) {
             const since = state.claimWatchdogSince.get(id);
             if (since === undefined) {
                 state.claimWatchdogSince.set(id, now);
+                continue;
+            }
+            if (now - since < state.claimWatchdogStaleMs)
+                continue;
+            // Stale past threshold. A recorded claim_locks conflict means the claim
+            // IS reaching the broker and being refused on a lock a live task holds —
+            // contention, not a wedged transport. Resubscribing cannot help; the fix
+            // is the holder releasing (or the orphan-lock self-heal / server sweep).
+            // The stamp is cleared the moment the task leaves the assigned+queued set
+            // (claimed or unassigned, in the reset loop above), so its presence means
+            // "conflicted and STILL queued" — no time window needed. The periodic
+            // backstop keeps re-driving a genuinely-claimable task, so suppressing
+            // the resubscribe here never strands it.
+            if (state.claimContentionSince.has(id)) {
+                contended.push(id);
             }
-            else if (now - since >= state.claimWatchdogStaleMs) {
+            else {
                 stale.push(id);
             }
         }
-        if (stale.length === 0)
+        if (stale.length === 0) {
+            if (contended.length > 0) {
+                // Quiet, throttled breadcrumb (re-arm the timers so it fires at most
+                // once per threshold window instead of every check).
+                for (const id of contended)
+                    state.claimWatchdogSince.set(id, now);
+                process.stderr.write(`[acc-runner] claim watchdog: task(s) ${contended.join(", ")} waiting on ` +
+                    `file-lock contention (holder still active) — not resubscribing.\n`);
+            }
             return;
+        }
         // Single-flight: never stack on a resubscribe already in flight or
         // scheduled (the existing T-68-1 guard/backoff owns the recovery).
         if (state.resubscribing || state.resubscribeTimer)
@@ -746,6 +792,59 @@ async function claimBackstopScan(state, factory) {
         state.backstopScanning = false;
     }
 }
+/**
+ * v0.41.4 (claim-livelock hardening): env_broken quarantine auto-recovery.
+ *
+ * A quarantine is a sticky sentinel — it persists until an operator runs
+ * `acc-runner quarantine clear`. That is correct for a genuinely broken host,
+ * but `env_broken` also fires on TRANSIENT conditions (a claude self-update
+ * window, a momentary "claude --version exited signal") that resolve on their
+ * own — leaving the runner permanently dark long after claude recovered (the
+ * incident). This pass re-probes claude while quarantined under `env_broken`
+ * and, when the probe passes, clears the quarantine and resumes claiming.
+ *
+ * Scoped to `env_broken` ONLY: usage_limit clears via the capacity resume path
+ * and auth_expired via a token refresh — a claude re-probe must not paper over
+ * those. Single-flight; never throws out of the loop.
+ */
+async function quarantineRecoveryScan(state, factory) {
+    if (state.stopped || state.quarantineRecovering)
+        return;
+    if (!state.quarantined || state.quarantineCause !== "env_broken")
+        return;
+    state.quarantineRecovering = true;
+    try {
+        const probe = await state.healthProbe();
+        if (!probe.ok)
+            return; // still broken — stay quarantined, try again next tick.
+        // claude is reachable again — lift the quarantine and resume.
+        await clearQuarantine().catch(() => { });
+        state.quarantined = false;
+        state.quarantineCause = null;
+        process.stderr.write(chalk.green(`[acc-runner] quarantine auto-cleared: claude health probe recovered ` +
+            `(${probe.detail}). Resuming task claims.\n`));
+        void (async () => {
+            try {
+                await state.supabase.rpc("log_activity", {
+                    p_verb: "runner.quarantine_auto_cleared",
+                    p_target_id: state.session.runner_id,
+                    p_payload: { cause: "env_broken", detail: probe.detail, version: PACKAGE_VERSION },
+                    p_target_type: "runner",
+                });
+            }
+            catch { /* best-effort breadcrumb */ }
+        })();
+        // Recover any work that stranded while quarantined: re-list + pump.
+        void pollOnce(state, factory);
+        void pump(state, factory);
+    }
+    catch (err) {
+        process.stderr.write(`[acc-runner] quarantine recovery scan failed: ${err.message}\n`);
+    }
+    finally {
+        state.quarantineRecovering = false;
+    }
+}
 /**
  * CLAIM-DEADMAN: a runner may have free capacity but be
  * deliberately not claiming (quarantined, capacity-paused, review-claim-paused,
@@ -1158,10 +1257,12 @@ export async function watchCommand(options = {}) {
     const enforceSingleton = options.enforceSingleton ?? !options.taskRunnerFactory;
     let singletonLock = null;
     if (enforceSingleton) {
-        // v1.02-A: the lock is keyed by (repo path + ACC_RUNNER_ID). A second
-        // runner with a DISTINCT ACC_RUNNER_ID (and, by default, its own
-        // namespaced cache dir) acquires its own lock and coexists; only a true
-        // duplicate — same repo, same identity — is rejected here.
+        // v1.02-A / v0.41.4: the lock is keyed by the runner IDENTITY, not the
+        // repo path — task assignment is per identity, so two processes sharing an
+        // identity contend regardless of checkout. A second runner with a DISTINCT
+        // ACC_RUNNER_ID (and, by default, its own namespaced cache dir) acquires
+        // its own lock and coexists; a duplicate of the same identity (including a
+        // second runner with no ACC_RUNNER_ID) is rejected here.
         const runnerId = singletonRunnerId();
         try {
             singletonLock = await acquireSingletonLock(cfg.repoPath, process.pid, runnerId);
@@ -1169,12 +1270,14 @@ export async function watchCommand(options = {}) {
         catch (err) {
             if (err instanceof SingletonLockHeldError) {
                 process.stderr.write(chalk.red.bold("\n[acc-runner] ANOTHER RUNNER IS ALREADY ACTIVE.\n"));
-                process.stderr.write(chalk.red(`Repo path ${err.holder.repoPath} is served by pid ${err.holder.pid}` +
-                    (err.holder.runnerId ? ` as ${err.holder.runnerId}` : "") +
-                    ` (acc-runner v${err.holder.version}, since ${err.holder.acquiredAt}).\n` +
-                    "Refusing to start a second runner with the same identity for the " +
-                    "same repo — concurrent same-identity runners corrupt worktrees and " +
-                    "race git refs.\n"));
+                process.stderr.write(chalk.red(`Identity ${err.holder.runnerId || "(default / no ACC_RUNNER_ID)"} is ` +
+                    `already served by pid ${err.holder.pid} on repo path ` +
+                    `${err.holder.repoPath} (acc-runner v${err.holder.version}, since ` +
+                    `${err.holder.acquiredAt}).\n` +
+                    "Refusing to start a second runner with the same identity — they " +
+                    "claim the same tasks and fight over the same file-locks (the claim " +
+                    "livelock), and concurrent same-identity runners corrupt worktrees " +
+                    "and race git refs.\n"));
                 process.stderr.write(chalk.gray(`To stop the other runner:  kill ${err.holder.pid}\n` +
                     `If it is already dead, remove ${err.lockFile} (or just retry — ` +
                     "stale locks self-reclaim).\n" +
@@ -1239,6 +1342,7 @@ export async function watchCommand(options = {}) {
                 claimBackstopOnce: async () => { },
                 claimDeadmanOnce: async () => { },
                 gitAuthProbeOnce: async () => false,
+                quarantineRecoveryOnce: async () => { },
                 runningCount: () => 0,
             };
         }
@@ -1416,10 +1520,18 @@ export async function watchCommand(options = {}) {
         claimWatchdogStaleMs: options.claimWatchdogStaleMs ?? DEFAULT_CLAIM_WATCHDOG_STALE_MS,
         claimWatchdogSince: new Map(),
         claimWatchdogChecking: false,
+        // v0.41.4: file-lock contention tracker (see WatchState doc).
+        claimContentionSince: new Map(),
         // FIX-D: periodic DB-poll claim backstop.
         claimBackstopTimer: undefined,
         claimBackstopIntervalMs: options.claimBackstopIntervalMs ?? DEFAULT_CLAIM_BACKSTOP_INTERVAL_MS,
         backstopScanning: false,
+        // v0.41.4: env_broken quarantine recovery.
+        quarantineCause: startupQuarantine?.cause ?? null,
+        quarantineRecoveryTimer: undefined,
+        quarantineRecoveryIntervalMs: options.quarantineRecoveryIntervalMs ?? DEFAULT_QUARANTINE_RECOVERY_INTERVAL_MS,
+        healthProbe: options.healthProbe ?? (() => getEngine().healthProbe()),
+        quarantineRecovering: false,
         // CLAIM-DEADMAN: wedge dead-man switch. The pure state
         // machine carries its own clock (Date.now); tests inject tiny thresholds.
         claimDeadman: new ClaimDeadman({
@@ -1593,6 +1705,10 @@ export async function watchCommand(options = {}) {
     state.claimBackstopTimer = setInterval(() => {
         void claimBackstopScan(state, taskRunnerFactory);
     }, state.claimBackstopIntervalMs);
+    // v0.41.4: env_broken quarantine auto-recovery re-probe.
+    state.quarantineRecoveryTimer = setInterval(() => {
+        void quarantineRecoveryScan(state, taskRunnerFactory);
+    }, state.quarantineRecoveryIntervalMs);
     // CLAIM-DEADMAN: the wedge dead-man switch. Escalates
     // BEYOND the 90s watchdog resubscribe (which only re-adds a channel on the
     // same client and, per the incident, can itself stop firing): tracks pending-
@@ -1710,6 +1826,7 @@ export async function watchCommand(options = {}) {
         clearInterval(state.reviewScanTimer); // v0.65 (T-65-2)
         clearInterval(state.claimWatchdogTimer); // v0.73 (T-73-2)
         clearInterval(state.claimBackstopTimer); // FIX-D
+        clearInterval(state.quarantineRecoveryTimer); // v0.41.4
         clearInterval(state.claimDeadmanTimer); // CLAIM-DEADMAN
         stopDeadmanFallback(state); // CLAIM-DEADMAN
         if (state.gitAuthTimer)
@@ -1826,6 +1943,7 @@ export async function watchCommand(options = {}) {
         claimBackstopOnce: () => claimBackstopScan(state, taskRunnerFactory), // FIX-D
         claimDeadmanOnce: () => claimDeadmanScan(state, taskRunnerFactory), // CLAIM-DEADMAN
         gitAuthProbeOnce: () => gitAuthProbeScan(state), // GIT-AUTH-ALERT
+        quarantineRecoveryOnce: () => quarantineRecoveryScan(state, taskRunnerFactory), // v0.41.4
         runningCount: () => state.running.size,
     };
 }
@@ -2012,6 +2130,7 @@ function scheduleHeartbeat(state, taskRunnerFactory, baseMs, lastRoundMs = 0) {
                     clearInterval(state.reviewScanTimer); // v0.65 (T-65-2)
                     clearInterval(state.claimWatchdogTimer); // v0.73 (T-73-2)
                     clearInterval(state.claimBackstopTimer); // FIX-D
+                    clearInterval(state.quarantineRecoveryTimer); // v0.41.4
                     clearInterval(state.claimDeadmanTimer); // CLAIM-DEADMAN
                     stopDeadmanFallback(state); // CLAIM-DEADMAN
                     if (state.resubscribeTimer)
@@ -2330,6 +2449,7 @@ consecutive) {
     if (state.quarantined)
         return;
     state.quarantined = true;
+    state.quarantineCause = cause; // v0.41.4: gates env_broken auto-recovery.
     const qState = {
         cause,
         classifiedAt: new Date().toISOString(),
@@ -2726,6 +2846,14 @@ async function pump(state, factory) {
                     const reason = outcome.error?.trim() ||
                         (outcome.exitCode != null ? `exit ${outcome.exitCode}` : "unknown");
                     process.stderr.write(`[acc-runner] task ${next} phase=${outcome.phase ?? "unknown"} failed: ${reason}\n`);
+                    // v0.41.4: a claim_locks failure is file-lock CONTENTION — the
+                    // claim reached the broker and was refused because a live task
+                    // holds an overlapping path. Stamp it so the claim watchdog treats
+                    // this task as legitimately waiting, not transport-wedged, and
+                    // does not answer it with a resubscribe storm.
+                    if (outcome.phase === "claim_locks") {
+                        state.claimContentionSince.set(next, Date.now());
+                    }
                     // v0.48: machine-level failure → quarantine. Guard with
                     // !state.quarantined so concurrent tasks don't fire duplicate
                     // quarantine events when two machine-level failures land