@tokenfactory/acc-runner 0.40.14 → 0.40.16

package/dist/watch.js

@@ -43,6 +43,14 @@ import { acquireSingletonLock, singletonRunnerId, SingletonLockHeldError, } from
 import { getClaudeVersion, recordTaskClaudeVersion, } from "./runtime/version-drift.js";
 import { autoUpgradeDisabled, maybeSelfUpgrade, maybeSelfUpdateOn426, } from "./runtime/self-upgrade.js";
 import { postRunnerStateMessage } from "./messaging.js";
+// AS-3 (RESUME-WIRE): drive the live capacity pause/resume through the
+// ResumeController (pause-until reset OR account-change OR ping) instead of a
+// fixed pause-until-reset setTimeout, and off the AS-2 account-probe cadence.
+import { ResumeController } from "./capacity/resume-controller.js";
+import { RUNNER_CAPACITY_PAUSED_VERB, RUNNER_CAPACITY_RESUME_VERB, } from "./capacity/fleet-pause-verbs.js";
+import { AccountProbeScheduler } from "./engines/account-probe.js";
+import { getChatEngine } from "./engines/registry.js";
+import { unknownAccountIdentity } from "./engines/account-identity.js";
 import { checkVersion, compareSemver } from "./version-check.js";
 import { PACKAGE_VERSION } from "./pkg-version.js";
 import { resolveAuthMode, describeAuthMode, authPrecedenceHint } from "./provider-auth.js";
@@ -1119,12 +1127,24 @@ export async function watchCommand(options = {}) {
         pausedCapacity: false,
         pausedUntil: null,
         capacityTriggerTaskId: null,
-        pauseTimer: null,
+        capacityPauseDetail: "",
+        // AS-3 (RESUME-WIRE): assigned immediately below once `state` exists (the
+        // controller's seams close over `state`). `undefined as unknown as` mirrors
+        // the channel/timer bootstrap fields above.
+        resumeController: undefined,
+        accountProbe: null,
+        resumeAccountKey: null,
         capacityProbe: false,
         capacityBackoffMs: options.capacityBackoffMs ?? DEFAULT_CAPACITY_BACKOFF_MS,
         resumeJitterMs: options.resumeJitterMs ?? DEFAULT_RESUME_JITTER_MS,
         singletonLock,
     };
+    // AS-3 (RESUME-WIRE): build the capacity resume governor + account-probe
+    // cadence NOW, before subscribeChannel / the first pump, so a capacity signal
+    // on the very first claim routes through the controller. Awaited so the
+    // starting account is seeded before any claim can 429. Both timers unref so
+    // the probe/poll cadence never holds the process open on its own.
+    await wireCapacityResume(state, taskRunnerFactory, options);
     subscribeChannel(state, taskRunnerFactory);
     // v0.34-B / v0.62 (T-62-1): one-shot startup claim scan. The 5-minute
     // pollSince watermark (v0.30-B) can miss a task dispatched (runner_id set)
@@ -1279,8 +1299,10 @@ export async function watchCommand(options = {}) {
             clearInterval(state.idleTimer);
         if (state.selfUpgradeTimer)
             clearInterval(state.selfUpgradeTimer); // FIX-K
-        if (state.pauseTimer)
-            clearTimeout(state.pauseTimer); // v0.56 (T-56-1)
+        // AS-3 (RESUME-WIRE): cancel the resume poll + account-probe cadence
+        // (replaces the v0.56 fixed pause-until-reset timer).
+        state.resumeController.stop();
+        state.accountProbe?.stop();
         // v0.10 T-49-2: cancel every in-flight task (replaces v0.9 single cancel).
         for (const ctrl of state.running.values()) {
             ctrl.cancel();
@@ -1763,142 +1785,234 @@ consecutive) {
     })();
 }
 /**
- * v0.56 (T-56-1): enter a capacity pause.
+ * AS-3 (RESUME-WIRE): enter a capacity pause.
  *
- * Triggered when a task OR a review reports capacity_exhausted (Claude
- * account out of session/usage capacity). The runner:
- *   - stops claiming tasks AND reviews (pump/pumpReviews bail on the flag);
- *   - schedules an automatic resume at the stated reset time (or a default
- *     backoff when none was parseable) plus a small jitter;
- *   - keeps heartbeating so the board shows online-but-paused, and emits a
- *     `runner.paused_capacity` activity event + a runner-level bus message
- *     carrying `resume_at` so health/board surfaces can show WHY the fleet
- *     is idle (heartbeat_runner's RPC shape is frozen, so the detail is
- *     carried additively through these existing channels).
+ * Triggered when a task OR a review reports capacity_exhausted (Claude account
+ * out of session/usage capacity). Since AS-3 the actual pause is owned by the
+ * ResumeController: this helper resolves the reset instant (stated reset, else
+ * the default backoff, plus the v0.56 spread jitter), stashes the trigger task +
+ * cause for the pause seams, and hands the cap to `reportCapacity`. The
+ * controller folds it onto the current account's window (extend-only), ENTERS
+ * the pause through the stopClaims/setServerPause seams (which stop claiming and
+ * emit the canonical `runner.capacity_paused` event + bus message), and owns the
+ * resume timing — reset elapsed OR a logged-in-account change — REPLACING the
+ * fixed pause-until-reset `setTimeout`.
  *
- * Idempotent: a second capacity hit while already paused only ever pushes
- * the resume later, never earlier, so a straggler task that 429s a moment
- * after the first one can't shorten the wait.
+ * Idempotent: a straggler cap while already paused only ever pushes the reset
+ * later (the window fold's extend-only rule), never shortens the wait.
  */
 function enterCapacityPause(state, triggerTaskId, resumeAtIso, detail) {
     const now = Date.now();
     const parsed = resumeAtIso ? Date.parse(resumeAtIso) : NaN;
-    const resumeMs = Number.isFinite(parsed) && parsed > now ? parsed : now + state.capacityBackoffMs;
-    const source = Number.isFinite(parsed) && parsed > now ? "reset_time" : "default_backoff";
-    // Already paused: only extend the window, never shorten it.
-    if (state.pausedCapacity && state.pausedUntil !== null && resumeMs <= state.pausedUntil) {
-        return;
-    }
-    state.pausedCapacity = true;
-    state.pausedUntil = resumeMs;
-    // FH-1a: remember which task drove the pause so resumeFromCapacity can emit
-    // a task-scoped `task.capacity_resumed` for it once the window reopens.
-    state.capacityTriggerTaskId = triggerTaskId;
-    if (state.pauseTimer) {
-        clearTimeout(state.pauseTimer);
-        state.pauseTimer = null;
-    }
+    const baseResumeMs = Number.isFinite(parsed) && parsed > now ? parsed : now + state.capacityBackoffMs;
+    // Preserve the v0.56 resume jitter (spreads a fleet's simultaneous resume)
+    // by folding it into the reset instant the controller waits on.
     const jitter = state.resumeJitterMs > 0 ? Math.floor(Math.random() * state.resumeJitterMs) : 0;
-    const delay = Math.max(0, resumeMs - now) + jitter;
-    const resumeAtFinal = new Date(now + delay).toISOString();
-    process.stderr.write(chalk.yellow(`[acc-runner] PAUSED (capacity): ${detail}. Not claiming tasks or reviews; ` +
-        `resuming at ${resumeAtFinal} (${source}).\n`));
-    void (async () => {
-        try {
-            await state.supabase.rpc("log_activity", {
-                p_verb: "runner.paused_capacity",
-                p_target_id: state.session.runner_id,
-                p_payload: {
-                    resume_at: resumeAtFinal,
-                    source,
-                    detail,
-                    version: PACKAGE_VERSION,
-                },
-                p_target_type: "runner",
-            });
-        }
-        catch { /* best-effort */ }
-    })();
-    // Runner-level bus message so operators see the paused status + reason on
-    // /messages even though heartbeat_runner itself can't carry a detail field.
-    // Anchored to the triggering task so the RPC can resolve the org scope
-    // (post_agent_message looks up org_id from acc.tasks for a non-null id).
-    void postRunnerStateMessage(state.supabase, {
-        task_id: triggerTaskId,
-        sender_id: state.session.runner_id,
-        state: "blocked",
-        protocol: "info",
-        payload: {
-            capacity_paused: true,
-            runner_id: state.session.runner_id,
-            resume_at: resumeAtFinal,
-            source,
-            detail,
-        },
-        kind: "blocked",
-    }).catch((err) => {
-        process.stderr.write(`[acc-runner] capacity pause bus post failed: ${err.message}\n`);
+    const resumeMs = baseResumeMs + jitter;
+    // FH-1a: remember which task drove the pause so the resume path can emit a
+    // task-scoped `task.capacity_resumed` for it once the window reopens; stash
+    // the non-secret cause so the setServerPause seam can carry it into the
+    // canonical pause payload + bus message (the seam only sees {account, reset}).
+    state.capacityTriggerTaskId = triggerTaskId;
+    state.capacityPauseDetail = detail;
+    // Stop claiming SYNCHRONOUSLY (the controller's stopClaims seam runs on the
+    // serialized action tail a microtask later; pump()/pumpReviews() read this
+    // flag between their own awaits, so setting it here closes the gap and keeps
+    // the v0.56 "no new claim the instant a 429 lands" guarantee byte-identical).
+    state.pausedCapacity = true;
+    state.resumeController.reportCapacity({
+        accountKey: state.resumeAccountKey ?? state.session.runner_id,
+        resetAtMs: resumeMs,
+        detail,
     });
-    state.pauseTimer = setTimeout(() => {
-        void resumeFromCapacity(state);
-    }, delay);
-    state.pauseTimer.unref?.();
 }
 /**
- * v0.56 (T-56-1): resume from a capacity pause.
+ * AS-3 (RESUME-WIRE): the ResumeController side-effect seams, backed by the REAL
+ * runner path (log_activity RPCs + pollOnce/pump). Exported so the account-switch
+ * rig / real-path tests drive the controller through the SAME seams the live
+ * runner uses, not hand-written fakes.
+ *
+ *   - setServerPause   → emit the canonical `runner.capacity_paused` activity
+ *                        event (the verb api/_lib/fleet-pause.ts actually reads,
+ *                        fixing the drift) + the operator bus message; mark the
+ *                        account probe paused so it re-probes at the pause cadence.
+ *   - stopClaims       → set pausedCapacity so pump()/pumpReviews() stop claiming.
+ *   - clearServerPause → emit the canonical `runner.capacity_resume` (clears the
+ *                        derived fleet_pause) + the FH-1a task-scoped
+ *                        `task.capacity_resumed`; unpause the probe.
+ *   - restartClaims    → arm the single-probe clamp and re-drive pollOnce + pump
+ *                        (the clamp bounds the first claim to one task; a
+ *                        non-capacity completion releases the held reviews).
  *
- * Clears the pause and arms the single-probe clamp: the runner claims at
- * most ONE task to confirm the window actually reopened before resuming
- * full task concurrency and reviews. If the probe itself 429s, the
- * completion handler re-enters the pause; if it succeeds, the probe clamp
- * clears and normal claiming resumes.
+ * In-flight sessions are NEVER touched — the seams only gate CLAIMING, so a task
+ * already running finishes normally across a pause / resume / account swap.
  */
-async function resumeFromCapacity(state) {
-    if (state.stopped)
-        return;
-    state.pausedCapacity = false;
-    state.pausedUntil = null;
-    // RVU-2 (FU-RVU2R): a full pause supersedes the soft claim-pause; clear it on
-    // resume so the probe window (which pins the task limit to 1) isn't zeroed out.
-    state.capacityClaimPaused = false;
-    if (state.pauseTimer) {
-        clearTimeout(state.pauseTimer);
-        state.pauseTimer = null;
-    }
-    state.capacityProbe = true;
-    console.log(chalk.gray(`[acc-runner] capacity window reopened — resuming with a single probe task.`));
-    try {
-        await state.supabase.rpc("log_activity", {
-            p_verb: "runner.resume_capacity",
-            p_target_id: state.session.runner_id,
-            p_payload: { probe: true, version: PACKAGE_VERSION },
-            p_target_type: "runner",
-        });
-    }
-    catch { /* best-effort */ }
-    // FH-1a: task-scoped resume audit for the task that drove the pause. The
-    // mid-task 429 cleanly requeued it (running→queued) and emitted
-    // task.capacity_paused; this closes the pair so a task timeline shows the
-    // window reopening. Best-effort + cleared after so a later review-triggered
-    // pause doesn't re-emit a stale task id.
-    const resumedTaskId = state.capacityTriggerTaskId;
-    state.capacityTriggerTaskId = null;
-    if (resumedTaskId) {
-        try {
-            await state.supabase.rpc("log_activity", {
-                p_verb: "task.capacity_resumed",
-                p_target_id: resumedTaskId,
-                p_payload: { task_id: resumedTaskId, probe: true, version: PACKAGE_VERSION },
-                p_target_type: "task",
+export function buildResumeControllerDeps(state, factory) {
+    return {
+        setServerPause: async ({ account, resetAtMs }) => {
+            state.pausedUntil = resetAtMs;
+            const resumeAtIso = resetAtMs !== null ? new Date(resetAtMs).toISOString() : null;
+            const detail = state.capacityPauseDetail;
+            const triggerTaskId = state.capacityTriggerTaskId;
+            state.accountProbe?.setPaused(true);
+            process.stderr.write(chalk.yellow(`[acc-runner] PAUSED (capacity) on ${account}: ${detail}. Not claiming ` +
+                `tasks or reviews; resuming at ${resumeAtIso ?? "reset / account change"}.\n`));
+            try {
+                await state.supabase.rpc("log_activity", {
+                    p_verb: RUNNER_CAPACITY_PAUSED_VERB,
+                    p_target_id: state.session.runner_id,
+                    p_payload: {
+                        runner_id: state.session.runner_id,
+                        resume_at: resumeAtIso,
+                        reason: detail,
+                        account,
+                        version: PACKAGE_VERSION,
+                    },
+                    p_target_type: "runner",
+                });
+            }
+            catch { /* best-effort */ }
+            // Runner-level bus message so operators see the paused status + reason on
+            // /messages. Anchored to the triggering task so the RPC can resolve the
+            // org scope (post_agent_message looks up org_id from acc.tasks for a
+            // non-null id); skipped for a review/probe pause with no task anchor.
+            if (triggerTaskId) {
+                await postRunnerStateMessage(state.supabase, {
+                    task_id: triggerTaskId,
+                    sender_id: state.session.runner_id,
+                    state: "blocked",
+                    protocol: "info",
+                    payload: {
+                        capacity_paused: true,
+                        runner_id: state.session.runner_id,
+                        resume_at: resumeAtIso,
+                        detail,
+                    },
+                    kind: "blocked",
+                }).catch((err) => {
+                    process.stderr.write(`[acc-runner] capacity pause bus post failed: ${err.message}\n`);
+                });
+            }
+        },
+        stopClaims: () => {
+            state.pausedCapacity = true;
+        },
+        clearServerPause: async () => {
+            try {
+                await state.supabase.rpc("log_activity", {
+                    p_verb: RUNNER_CAPACITY_RESUME_VERB,
+                    p_target_id: state.session.runner_id,
+                    p_payload: { runner_id: state.session.runner_id, version: PACKAGE_VERSION },
+                    p_target_type: "runner",
+                });
+            }
+            catch { /* best-effort */ }
+            // FH-1a: task-scoped resume audit for the task that drove the pause, then
+            // clear it so a later review-triggered pause doesn't re-emit a stale id.
+            const resumedTaskId = state.capacityTriggerTaskId;
+            state.capacityTriggerTaskId = null;
+            if (resumedTaskId) {
+                try {
+                    await state.supabase.rpc("log_activity", {
+                        p_verb: "task.capacity_resumed",
+                        p_target_id: resumedTaskId,
+                        p_payload: { task_id: resumedTaskId, probe: true, version: PACKAGE_VERSION },
+                        p_target_type: "task",
+                    });
+                }
+                catch { /* best-effort */ }
+            }
+            state.accountProbe?.setPaused(false);
+        },
+        restartClaims: async () => {
+            if (state.stopped)
+                return;
+            state.pausedCapacity = false;
+            state.pausedUntil = null;
+            // RVU-2 (FU-RVU2R): a full pause supersedes the soft claim-pause; clear it
+            // so the probe window (task limit pinned to 1) isn't zeroed out.
+            state.capacityClaimPaused = false;
+            // Single-probe clamp — claim exactly ONE task to confirm the window
+            // reopened before unleashing full concurrency + reviews.
+            state.capacityProbe = true;
+            console.log(chalk.gray(`[acc-runner] capacity window reopened — resuming with a single probe task.`));
+            // Poll first (a released task may have been re-queued elsewhere), then
+            // pump — the probe clamp bounds it to one task.
+            await pollOnce(state, factory);
+            void pump(state, factory);
+        },
+        onLog: (line) => {
+            process.stderr.write(`${line}\n`);
+        },
+    };
+}
+/**
+ * AS-3 (RESUME-WIRE): construct + start the ResumeController and the AS-1/AS-2
+ * account-probe cadence, and stitch them together — each account-probe broadcast
+ * feeds `observeAccount()` so a keychain login swap converges within one probe
+ * cycle instead of waiting on the stale account's reset. Both use an UNREF'd
+ * timer so the poll / probe cadence never holds the process open by itself.
+ */
+async function wireCapacityResume(state, factory, options) {
+    const unrefTimer = {
+        setTimer: (fn, ms) => {
+            const h = setTimeout(fn, ms);
+            h.unref?.();
+            return h;
+        },
+        clearTimer: (h) => clearTimeout(h),
+    };
+    // The account-identity probe: injected in tests; the resolved chat engine's
+    // accountIdentity() otherwise (an honest per-host synthetic account when no
+    // real login introspects). Never throws — a failed probe degrades to unknown
+    // here, and to a logged no-op on the scheduler's later cadence probes.
+    const chatEngine = getChatEngine();
+    const probe = options.accountProbe ??
+        (async () => {
+            if (chatEngine.accountIdentity)
+                return chatEngine.accountIdentity();
+            return unknownAccountIdentity({
+                engineId: chatEngine.id,
+                channel: chatEngine.authMode,
+                detail: "no account-identity probe on the resolved chat engine",
             });
-        }
-        catch { /* best-effort */ }
-    }
-    // Poll first (the stale-running sweep may have re-queued the released task
-    // to another runner; this picks up whatever is still assigned here), then
-    // pump — the probe clamp bounds it to one task.
-    await pollOnce(state, state.taskRunnerFactory);
-    void pump(state, state.taskRunnerFactory);
+        });
+    // Seed the controller's starting account BEFORE it starts, so a cap that
+    // lands before the scheduler's first cadence probe still folds onto — and
+    // pauses — the right account (reportCapacity only pauses the observed account).
+    const initial = await probe().catch(() => null);
+    state.resumeAccountKey = initial?.synthetic_account ?? null;
+    state.resumeController = new ResumeController({
+        timer: unrefTimer,
+        pollIntervalMs: options.capacityPollMs ?? 60_000,
+        ...(options.capacityAdoptionDebounceMs !== undefined
+            ? { adoptionDebounceMs: options.capacityAdoptionDebounceMs }
+            : {}),
+        // A login swap resolves via the account-change branch (no request burned);
+        // reset-elapsed drives the same-account resume. No probe ping wired.
+        ...buildResumeControllerDeps(state, factory),
+    });
+    state.resumeController.start(state.resumeAccountKey);
+    state.accountProbe = new AccountProbeScheduler({
+        probe,
+        timer: unrefTimer,
+        pauseTickMs: options.accountProbeTickMs ?? 60_000,
+        ...(options.accountProbeHeartbeatMs !== undefined
+            ? { heartbeatMs: options.accountProbeHeartbeatMs }
+            : {}),
+        emit: (broadcast) => {
+            state.resumeAccountKey = broadcast.identity.synthetic_account;
+            // Fast path for a login swap: an immediate re-evaluation so a swap
+            // converges within one probe cycle rather than the stale account's reset.
+            state.resumeController.observeAccount(broadcast.identity.synthetic_account);
+        },
+        onLog: (line) => {
+            process.stderr.write(`${line}\n`);
+        },
+    });
+    // Start the probe cadence (fires the startup probe immediately, then the
+    // pause-tick / heartbeat cadence per its four-trigger schedule).
+    state.accountProbe.start();
 }
 /**
  * v0.10 T-49-2: concurrent task pump.