@tokenfactory/acc-runner 0.8.1 → 0.12.0

package/dist/watch.js

@@ -1,8 +1,8 @@
 /**
  * `acc-runner watch` — long-running command. Subscribes to broadcast
- * tasks, runs them serially, heartbeats every 4s. v0.2.5 refreshes the
- * access token on startup + every 30 min, and exits loudly after 5
- * consecutive heartbeat failures instead of silent-zombie-ing.
+ * tasks, dispatches them concurrently (v0.10+), heartbeats every 4s.
+ * v0.2.5 refreshes the access token on startup + every 30 min, and exits
+ * loudly after 5 consecutive heartbeat failures.
  *
  * v0.3 adds a 30s polling fallback: if a Realtime broadcast is dropped
  * during a reconnect, the next poll pass picks the task up via the
@@ -11,17 +11,29 @@
  *
  * v0.6.2 (v0.12-REVIEW-LOCAL) listens for `review_assigned` on the same
  * `runner:<id>` channel and dispatches to runtime/reviewer.ts. Reviews
- * are independently tracked (separate `seenReviews` set) so a task
- * and a review for that task can both run in this process — though
- * not concurrently with each other: the existing single-flight pump()
- * serializes task work, and reviews queue behind whatever is running.
+ * are independently tracked (separate `seenReviews` set).
+ *
+ * v0.10 (T-49-2): concurrent task dispatch. pump() now launches up to
+ * `concurrencyLimit` (default 2) tasks simultaneously. Each task runs in
+ * its own isolated git worktree so concurrent tasks never share files.
+ * `createTaskAsAdmin` in api/_lib/supabase-admin.ts is the sanctioned
+ * Manager seed path that feeds this dispatch loop.
+ *
+ * v0.12.0 (T-52-7): ephemeral-runner support. Sessions load through the
+ * token-provider abstraction (keychain by default — byte-identical desktop
+ * behaviour; ACC_RUNNER_TOKEN_MODE=env for containers). Env-token runners
+ * register an acc-eph-<random> identity on boot, and ACC_RUNNER_IDLE_TTL_MIN
+ * arms a clean self-termination after n minutes without work.
  */
 import chalk from "chalk";
 import { loadConfig } from "./config.js";
-import { loadSession, saveSession } from "./keychain.js";
+import { getTokenProvider } from "./token-provider.js";
+import { machineIdentity, detectInstalledModels, RUNNER_CAPS } from "./login.js";
 import { createRunnerClient } from "./supabase.js";
 import { runTask } from "./task-runner.js";
 import { runReview, } from "./runtime/reviewer.js";
+import { getQuarantine, setQuarantine, } from "./runtime/quarantine.js";
+import { postRunnerStateMessage } from "./messaging.js";
 import { checkVersion, compareSemver } from "./version-check.js";
 import { PROTOCOL_VERSION } from "./types.js";
 import { PACKAGE_VERSION } from "./pkg-version.js";
@@ -33,6 +45,9 @@ export class RefreshReuseDetectedError extends Error {
 }
 const ONE_HOUR_MS = 60 * 60 * 1000;
 const REFRESH_CHECK_MS = 30 * 60 * 1000;
+// v0.12.0 (T-52-7): idle-TTL check cadence ceiling. The actual cadence is
+// min(IDLE_CHECK_MS, ttl) so a tiny test TTL still gets checked in time.
+const IDLE_CHECK_MS = 30_000;
 const HEARTBEAT_REFRESH_AFTER = 3;
 const HEARTBEAT_EXIT_AFTER = 5;
 const POLL_INTERVAL_MS = 30_000;
@@ -107,6 +122,26 @@ export function nextHeartbeatDelayMs(consecutiveFailures, baseMs) {
     const idx = Math.min(consecutiveFailures - 1, HEARTBEAT_BACKOFF_MS.length - 1);
     return Math.max(baseMs, HEARTBEAT_BACKOFF_MS[idx]);
 }
+/**
+ * v0.12.0 (T-52-7): resolve ACC_RUNNER_IDLE_TTL_MIN into ms. Returns null
+ * (TTL disabled) when unset; warns and returns null on junk so a typo'd
+ * env never strands a container in "never exits" mode silently.
+ */
+export function resolveIdleTtlMs(env = process.env) {
+    const raw = env.ACC_RUNNER_IDLE_TTL_MIN?.trim();
+    if (!raw)
+        return null;
+    const minutes = Number(raw);
+    if (!Number.isFinite(minutes) || minutes <= 0) {
+        process.stderr.write(`[acc-runner] ACC_RUNNER_IDLE_TTL_MIN=${JSON.stringify(raw)} is not a positive number; idle TTL disabled\n`);
+        return null;
+    }
+    return minutes * 60_000;
+}
+/** v0.12.0 (T-52-7): record work-related activity for the idle TTL clock. */
+function touchActivity(state) {
+    state.lastActivityMs = Date.now();
+}
 function expiresSoon(session, leadMs = ONE_HOUR_MS) {
     const expires = Date.parse(session.access_expires_at);
     if (!Number.isFinite(expires))
@@ -159,12 +194,14 @@ function failOnReuse(err) {
 function enqueue(state, taskId, factory) {
     if (state.seen.has(taskId))
         return false;
-    if (state.current && state.current.taskId === taskId) {
+    // v0.10 T-49-2: check the running Map (replaces v0.9 state.current check).
+    if (state.running.has(taskId)) {
         state.seen.add(taskId);
         return false;
     }
     state.seen.add(taskId);
     state.queue.push(taskId);
+    touchActivity(state);
     void pump(state, factory);
     return true;
 }
@@ -180,6 +217,7 @@ function enqueueReview(state, payload) {
         task_id: p.task_id,
         pr_number: p.pr_number,
     });
+    touchActivity(state);
     void pumpReviews(state);
 }
 function subscribeChannel(state, taskRunnerFactory) {
@@ -196,8 +234,10 @@ function subscribeChannel(state, taskRunnerFactory) {
         const taskId = payload?.task_id;
         if (!taskId)
             return;
-        if (state.current && state.current.taskId === taskId) {
-            state.current.cancel();
+        // v0.10 T-49-2: cancel from the running Map; fall back to queue removal.
+        const ctrl = state.running.get(taskId);
+        if (ctrl) {
+            ctrl.cancel();
         }
         else {
             state.queue = state.queue.filter((t) => t !== taskId);
@@ -248,13 +288,20 @@ async function pollOnce(state, factory) {
 // Single-flight refresh. Recreates the Supabase client (so future RPC +
 // channel auth use the new JWT) and resubscribes the broadcast channel.
 async function doRefresh(state, taskRunnerFactory) {
+    // v0.12.0 (T-52-7): an env-token runner without ACC_RUNNER_REFRESH_TOKEN
+    // cannot rotate. Refusing here (instead of POSTing an empty token) keeps
+    // the failure mode a clear stderr line rather than a server-side 400.
+    if (!state.tokenProvider.canRefresh()) {
+        process.stderr.write("[acc-runner] token refresh unavailable (no refresh token in env); runner will exit when the access token expires\n");
+        return false;
+    }
     if (state.refreshing)
         return state.refreshing;
     state.refreshing = (async () => {
         try {
             const updated = await refreshAccessToken(state.cfg, state.session);
             state.session = updated;
-            await saveSession(updated);
+            await state.tokenProvider.save(updated);
             try {
                 await state.supabase.removeChannel(state.channel);
             }
@@ -279,9 +326,12 @@ async function doRefresh(state, taskRunnerFactory) {
 }
 export async function watchCommand(options = {}) {
     const cfg = loadConfig();
-    let session = await loadSession();
+    const tokenProvider = getTokenProvider();
+    let session = await tokenProvider.load();
     if (!session) {
-        process.stderr.write("Not logged in. Run `acc-runner login`.\n");
+        process.stderr.write(tokenProvider.mode === "env"
+            ? "No session in env. Set ACC_RUNNER_ACCESS_TOKEN (token mode: env).\n"
+            : "Not logged in. Run `acc-runner login`.\n");
         process.exit(1);
     }
     const v = await checkVersion(cfg.publicUrl);
@@ -297,10 +347,13 @@ export async function watchCommand(options = {}) {
     }
     // Startup refresh: if the access token expires in less than an hour,
     // grab a fresh one before opening the realtime connection.
-    if (expiresSoon(session)) {
+    // v0.12.0 (T-52-7): skipped when the provider has no refresh token —
+    // an env-token runner just runs out its 8h access token (the idle TTL
+    // normally exits it long before then).
+    if (tokenProvider.canRefresh() && expiresSoon(session)) {
         try {
             session = await refreshAccessToken(cfg, session);
-            await saveSession(session);
+            await tokenProvider.save(session);
         }
         catch (err) {
             if (err instanceof RefreshReuseDetectedError)
@@ -309,19 +362,61 @@ export async function watchCommand(options = {}) {
             process.exit(1);
         }
     }
+    // v0.12.0 (T-52-7): env-token (ephemeral) runners had no `login` flow, so
+    // nothing registered an acc.runners row yet. Register on boot under the
+    // acc-eph-<random> identity. register_runner may resolve to an existing
+    // row (same bound user + machine, e.g. a restarted container with a
+    // pinned hostname) — adopt whatever id it returns so the realtime
+    // channel, heartbeats, and polling all agree.
+    if (tokenProvider.mode === "env") {
+        const boot = createRunnerClient(cfg, session.access_token);
+        const models = await detectInstalledModels();
+        const { data: resolvedId, error: regErr } = await boot.rpc("register_runner", {
+            p_id: session.runner_id,
+            p_name: session.runner_id,
+            p_owner: session.email ?? session.user_id,
+            p_machine: machineIdentity(),
+            p_models: models,
+            p_caps: RUNNER_CAPS,
+            p_version: PACKAGE_VERSION,
+        });
+        if (regErr) {
+            process.stderr.write(`[acc-runner] ephemeral register_runner failed: ${regErr.message}\n`);
+            process.exit(1);
+        }
+        if (typeof resolvedId === "string" && resolvedId && resolvedId !== session.runner_id) {
+            session = { ...session, runner_id: resolvedId };
+        }
+        await tokenProvider.save(session);
+        console.log(chalk.green(`✓ Registered ephemeral runner: ${session.runner_id}`));
+    }
     const taskRunnerFactory = options.taskRunnerFactory ?? runTask;
     const heartbeatMs = options.heartbeatMs ?? 4_000;
     const pollMs = options.pollMs ?? POLL_INTERVAL_MS;
     const reviewerFactory = options.reviewerFactory ?? ((assignment, deps) => runReview(assignment, deps));
+    // v0.48: check for an existing quarantine file before opening the
+    // realtime connection. If quarantined, emit a loud warning — the
+    // runner will not claim tasks until `acc-runner quarantine clear` runs.
+    const startupQuarantine = await getQuarantine().catch(() => null);
+    if (startupQuarantine) {
+        process.stderr.write(chalk.red.bold(`[acc-runner] QUARANTINED (${startupQuarantine.cause}): ${startupQuarantine.detail}\n`));
+        process.stderr.write(chalk.red(`[acc-runner] Runner will NOT claim tasks. Resolve the issue then run ` +
+            `\`acc-runner quarantine clear\`.\n`));
+    }
+    // v0.12.0 (T-52-7): idle TTL — option override wins (tests), then env.
+    const idleTtlMs = options.idleTtlMs !== undefined ? options.idleTtlMs : resolveIdleTtlMs();
     const state = {
         cfg,
         session,
+        tokenProvider,
         supabase: createRunnerClient(cfg, session.access_token),
         channel: undefined,
         heartbeatTimer: undefined,
         refreshTimer: undefined,
         pollTimer: undefined,
-        current: null,
+        // v0.10 T-49-2: concurrent running Map replaces v0.9 single `current`.
+        running: new Map(),
+        concurrencyLimit: options.concurrencyLimit ?? 2,
         queue: [],
         seen: new Set(),
         pumping: false,
@@ -334,6 +429,12 @@ export async function watchCommand(options = {}) {
         seenReviews: new Set(),
         reviewPumping: false,
         runReview: reviewerFactory,
+        quarantined: startupQuarantine !== null,
+        authRetriedTasks: new Set(),
+        idleTtlMs,
+        lastActivityMs: Date.now(),
+        idleTimer: null,
+        idleExiting: false,
     };
     subscribeChannel(state, taskRunnerFactory);
     // v0.34-B: one-shot startup claim scan. The 5-minute pollSince watermark
@@ -376,8 +477,10 @@ export async function watchCommand(options = {}) {
     // attempts. setInterval would keep firing every heartbeatMs even
     // while the link is down.
     scheduleHeartbeat(state, taskRunnerFactory, heartbeatMs);
+    // v0.12.0 (T-52-7): no refresh timer when the provider can't rotate —
+    // doRefresh would just emit the same "unavailable" line every 30 min.
     state.refreshTimer = setInterval(() => {
-        if (expiresSoon(state.session)) {
+        if (state.tokenProvider.canRefresh() && expiresSoon(state.session)) {
             void doRefresh(state, taskRunnerFactory);
         }
     }, REFRESH_CHECK_MS);
@@ -416,8 +519,12 @@ export async function watchCommand(options = {}) {
         clearTimeout(state.heartbeatTimer);
         clearInterval(state.refreshTimer);
         clearInterval(state.pollTimer);
-        if (state.current)
-            state.current.cancel();
+        if (state.idleTimer)
+            clearInterval(state.idleTimer);
+        // v0.10 T-49-2: cancel every in-flight task (replaces v0.9 single cancel).
+        for (const ctrl of state.running.values()) {
+            ctrl.cancel();
+        }
         try {
             await state.supabase.removeChannel(state.channel);
         }
@@ -444,6 +551,19 @@ export async function watchCommand(options = {}) {
         }
         catch { /* network may already be gone */ }
     };
+    // v0.12.0 (T-52-7): arm the idle-TTL timer. Desktop runners (no TTL env,
+    // no option) never reach this branch — zero behaviour change. Armed after
+    // `stop` exists so the exit sequence can reuse the one shutdown path.
+    if (state.idleTtlMs !== null && state.idleTtlMs > 0) {
+        const exitFn = options.exit ?? ((code) => process.exit(code));
+        const checkMs = options.idleCheckMs ?? Math.min(IDLE_CHECK_MS, state.idleTtlMs);
+        state.idleTimer = setInterval(() => {
+            void maybeIdleExit(state, stop, exitFn);
+        }, checkMs);
+        // Don't hold the event loop open for the idle check alone.
+        state.idleTimer.unref?.();
+        console.log(chalk.gray(`Idle TTL armed: exiting after ${Math.round(state.idleTtlMs / 60_000)} min without work.`));
+    }
     if (!options.taskRunnerFactory) {
         // v0.33-C: best-effort terminal status flush on runner shutdown. If
         // launchd (or another OS-driven kill) sends SIGTERM while a task is
@@ -455,16 +575,19 @@ export async function watchCommand(options = {}) {
         // safety net; this just shrinks the window when the kill is catchable.
         const onSignal = (signal) => {
             void (async () => {
-                if (signal === "SIGTERM" && state.current) {
-                    try {
-                        await state.supabase.rpc("transition_task", {
-                            p_task_id: state.current.taskId,
-                            p_new_status: "failed",
-                        });
-                    }
-                    catch {
-                        // best-effort — the sweep covers us if this RPC can't get out
-                    }
+                // v0.10 T-49-2: on SIGTERM, transition ALL in-flight tasks to
+                // failed (replaces v0.9 single-task transition). SIGINT (Ctrl-C)
+                // still skips the flush — the 5-min sweep handles that path.
+                if (signal === "SIGTERM" && state.running.size > 0) {
+                    await Promise.allSettled([...state.running.keys()].map(async (taskId) => {
+                        try {
+                            await state.supabase.rpc("transition_task", {
+                                p_task_id: taskId,
+                                p_new_status: "failed",
+                            });
+                        }
+                        catch { /* best-effort — sweep covers us if RPC can't get out */ }
+                    }));
                 }
                 await stop();
                 process.exit(0);
@@ -473,7 +596,11 @@ export async function watchCommand(options = {}) {
         process.once("SIGINT", () => onSignal("SIGINT"));
         process.once("SIGTERM", () => onSignal("SIGTERM"));
     }
-    return { stop, pollOnce: () => pollOnce(state, taskRunnerFactory) };
+    return {
+        stop,
+        pollOnce: () => pollOnce(state, taskRunnerFactory),
+        runningCount: () => state.running.size,
+    };
 }
 function scheduleHeartbeat(state, taskRunnerFactory, baseMs) {
     if (state.stopped)
@@ -513,6 +640,57 @@ function scheduleHeartbeat(state, taskRunnerFactory, baseMs) {
         })();
     }, delay);
 }
+/**
+ * v0.12.0 (T-52-7): idle-TTL exit check. Fires from the idle timer; exits
+ * the process cleanly when the runner has had no queued or in-flight work
+ * (tasks OR reviews) for idleTtlMs. The exit sequence:
+ *
+ *   1. log_activity runner.idle_ttl_exit — the audit breadcrumb the
+ *      autoscale sweep and operators read ("why did acc-eph-x vanish?").
+ *   2. final heartbeat_runner — so the runners panel shows a fresh
+ *      timestamp right up to the clean exit (vs. a stale-heartbeat crash).
+ *   3. stop() — shutdown_intent + set_runner_status('offline'), the same
+ *      deregistration path `acc-runner logout` and SIGTERM use.
+ *   4. exit(0).
+ *
+ * In-flight work always wins: any running task, queued task, or queued
+ * review resets the decision to "not idle" — the TTL never kills a runner
+ * mid-task.
+ */
+async function maybeIdleExit(state, stop, exitFn) {
+    if (state.stopped || state.idleExiting || state.idleTtlMs === null)
+        return;
+    if (state.running.size > 0 ||
+        state.queue.length > 0 ||
+        state.reviewQueue.length > 0 ||
+        state.reviewPumping) {
+        return;
+    }
+    const idleMs = Date.now() - state.lastActivityMs;
+    if (idleMs < state.idleTtlMs)
+        return;
+    state.idleExiting = true;
+    const idleMinutes = Math.round(idleMs / 60_000);
+    console.log(chalk.gray(`[acc-runner] idle TTL reached (${idleMinutes} min without work); exiting cleanly.`));
+    try {
+        await state.supabase.rpc("log_activity", {
+            p_verb: "runner.idle_ttl_exit",
+            p_target_id: state.session.runner_id,
+            p_payload: { idle_minutes: idleMinutes, version: PACKAGE_VERSION },
+            p_target_type: "runner",
+        });
+    }
+    catch { /* best-effort */ }
+    try {
+        await state.supabase.rpc("heartbeat_runner", {
+            p_id: state.session.runner_id,
+            p_version: PACKAGE_VERSION,
+        });
+    }
+    catch { /* best-effort */ }
+    await stop();
+    exitFn(0);
+}
 async function pumpReviews(state) {
     if (state.reviewPumping)
         return;
@@ -530,22 +708,103 @@ async function pumpReviews(state) {
             catch (err) {
                 process.stderr.write(`[acc-runner] review ${next.review_id} crashed: ${err.message}\n`);
             }
+            finally {
+                touchActivity(state); // v0.12.0 (T-52-7): idle clock restarts at review end
+            }
         }
     }
     finally {
         state.reviewPumping = false;
     }
 }
+/**
+ * v0.48 / v0.49 T-49-6: enter quarantine after a machine-level failure.
+ *
+ * Extracted from pump()'s completion handler so both the immediate
+ * quarantine path and the auth_expired "refresh failed → quarantine"
+ * path share one implementation. Idempotent: a no-op once the runner is
+ * already quarantined, so concurrent machine-level failures don't fire
+ * duplicate audit events. Sets the in-memory flag synchronously (pump()
+ * reads it without an async hop) and best-effort persists + announces.
+ */
+function enterQuarantine(state, taskId, cause, detail) {
+    if (state.quarantined)
+        return;
+    state.quarantined = true;
+    const qState = {
+        cause,
+        classifiedAt: new Date().toISOString(),
+        taskId,
+        detail,
+    };
+    void setQuarantine(qState).catch((err) => {
+        process.stderr.write(`[acc-runner] setQuarantine failed: ${err.message}\n`);
+    });
+    process.stderr.write(chalk.red.bold(`[acc-runner] QUARANTINE: runner ${state.session.runner_id} quarantined ` +
+        `(${cause}) after task ${taskId} failed.\n`));
+    void postRunnerStateMessage(state.supabase, {
+        task_id: taskId,
+        sender_id: state.session.runner_id,
+        state: "blocked",
+        protocol: "error_context",
+        payload: {
+            capacity_alert: true,
+            quarantine_cause: cause,
+            runner_id: state.session.runner_id,
+            detail,
+        },
+        kind: "blocked",
+    }).catch((err) => {
+        process.stderr.write(`[acc-runner] capacity alert post failed: ${err.message}\n`);
+    });
+    void (async () => {
+        try {
+            await state.supabase.rpc("log_activity", {
+                p_verb: "runner.quarantine_enter",
+                p_target_id: state.session.runner_id,
+                p_payload: { cause, task_id: taskId, detail },
+                p_target_type: "runner",
+            });
+        }
+        catch { /* best-effort */ }
+    })();
+}
+/**
+ * v0.10 T-49-2: concurrent task pump.
+ *
+ * Launches up to state.concurrencyLimit tasks in parallel. Each task
+ * gets its own isolated git worktree (prepareTaskWorktree in task-runner)
+ * so there is zero shared-file risk between concurrent runs. Database-side
+ * `claim_task_with_locks` provides a second guard: it rejects a claim when
+ * the task's file scope overlaps another in-flight task.
+ *
+ * Design: rather than `await`ing each task inside a while loop (v0.9
+ * serial behaviour), tasks are launched fire-and-forget. Each task's
+ * `.then()` handler removes it from `state.running` and re-fires pump()
+ * so newly freed capacity immediately picks up the next queued item.
+ * The `pumping` mutex prevents two concurrent pump() invocations from
+ * both launching into the same concurrencyLimit slot.
+ */
 async function pump(state, factory) {
     if (state.pumping)
         return;
     state.pumping = true;
     try {
-        while (!state.stopped && state.queue.length > 0) {
+        // v0.48: check quarantine before claiming any task.
+        if (state.quarantined) {
+            process.stderr.write(`[acc-runner] QUARANTINED: runner ${state.session.runner_id} is not claiming tasks.\n` +
+                `[acc-runner] Fix the underlying issue then run \`acc-runner quarantine clear\`.\n`);
+            return;
+        }
+        // Fill available concurrency slots from the queue.
+        while (!state.stopped &&
+            !state.quarantined &&
+            state.queue.length > 0 &&
+            state.running.size < state.concurrencyLimit) {
             const next = state.queue.shift();
             if (!next)
                 continue;
-            state.current = factory(next, {
+            const ctrl = factory(next, {
                 supabase: state.supabase,
                 cfg: state.cfg,
                 session: {
@@ -554,24 +813,78 @@ async function pump(state, factory) {
                 },
                 publicUrl: state.cfg.publicUrl,
             });
-            try {
-                const outcome = await state.current.promise;
-                // v0.6.0 REG-294: surface a one-liner when runTask resolves
-                // with status:'failed' (not thrown). Before, an operator had
-                // no signal that a multi-minute task ended in a 'failed' RPC
-                // call — only thrown exceptions ever hit stderr.
+            state.running.set(next, ctrl);
+            // Detached completion handler. Runs AFTER pump() returns so the
+            // running Map slot is occupied by the time the while condition
+            // re-checks running.size on the next iteration.
+            void ctrl.promise
+                .then((outcome) => {
+                state.running.delete(next);
+                touchActivity(state); // v0.12.0 (T-52-7): idle clock restarts at task end
                 if (outcome.status === "failed") {
                     const reason = outcome.error?.trim() ||
                         (outcome.exitCode != null ? `exit ${outcome.exitCode}` : "unknown");
                     process.stderr.write(`[acc-runner] task ${next} phase=${outcome.phase ?? "unknown"} failed: ${reason}\n`);
+                    // v0.48: machine-level failure → quarantine. Guard with
+                    // !state.quarantined so concurrent tasks don't fire duplicate
+                    // quarantine events when two machine-level failures land
+                    // simultaneously (both already in-flight; only the first to
+                    // resolve sets the flag and writes the audit trail).
+                    if (outcome.quarantine_cause && !state.quarantined) {
+                        const cause = outcome.quarantine_cause;
+                        // v0.49 T-49-6: a transient auth_expired (the access token
+                        // lapsed while the task ran) gets ONE JWT refresh before we
+                        // quarantine. If the refresh recovers, the runner stays
+                        // online and resumes claiming queued work; if it fails — or
+                        // this task already burned its one retry — we quarantine.
+                        // usage_limit / env_broken skip this path: a token refresh
+                        // can't fix a spent quota or a broken host.
+                        if (cause === "auth_expired" && !state.authRetriedTasks.has(next)) {
+                            state.authRetriedTasks.add(next);
+                            process.stderr.write(chalk.yellow(`[acc-runner] auth_expired after task ${next}; attempting one ` +
+                                `token refresh before quarantine.\n`));
+                            void doRefresh(state, factory).then((ok) => {
+                                if (ok) {
+                                    process.stderr.write(chalk.green(`[acc-runner] token refresh recovered auth after task ${next}; ` +
+                                        `staying online.\n`));
+                                    void (async () => {
+                                        try {
+                                            await state.supabase.rpc("log_activity", {
+                                                p_verb: "runner.auth_retry_recovered",
+                                                p_target_id: state.session.runner_id,
+                                                p_payload: { task_id: next, cause },
+                                                p_target_type: "runner",
+                                            });
+                                        }
+                                        catch { /* best-effort */ }
+                                    })();
+                                    // Resume claiming: the freed slot can pick up queued work
+                                    // now that the JWT is fresh.
+                                    void pump(state, factory);
+                                }
+                                else {
+                                    enterQuarantine(state, next, cause, reason);
+                                }
+                            });
+                            return;
+                        }
+                        enterQuarantine(state, next, cause, reason);
+                        // Quarantined — do not re-fire pump for new tasks.
+                        // Other in-flight tasks (still in state.running) complete
+                        // normally; we just stop accepting new work.
+                        return;
+                    }
                 }
-            }
-            catch (err) {
+                // Task done (ok, failed without quarantine, or cancelled).
+                // Re-fire pump so the next queued task can claim the freed slot.
+                void pump(state, factory);
+            })
+                .catch((err) => {
+                state.running.delete(next);
+                touchActivity(state); // v0.12.0 (T-52-7): idle clock restarts at task end
                 process.stderr.write(`[acc-runner] task ${next} crashed: ${err.message}\n`);
-            }
-            finally {
-                state.current = null;
-            }
+                void pump(state, factory);
+            });
         }
     }
     finally {