@phnx-labs/agents-cli 1.22.46 → 1.22.49

package/dist/lib/traces/worker-template.js

@@ -0,0 +1,159 @@
+// The Cloudflare Worker that fronts the R2 traces bucket.
+//
+// SECURITY: unlike agents-share, there is NO public GET path here. Every route
+// — PUT and GET — requires a Phoenix bearer. The caller must be the record's
+// owner (path prefix == verified userId). A missing bearer → 401; a wrong
+// owner → 403; public cache headers are NEVER emitted.
+//
+// Routes (all require Phoenix bearer + userId owner match):
+//   PUT  /<userId>/<device>/index.json            — per-device stats shard
+//   PUT  /<userId>/<device>/sessions/<id>.json    — per-session SessionDetail
+//   GET  /<userId>/...                            — returns the stored object
+//   GET  /                                        — 200 description (no data)
+//
+// Emitted as a string so it compiles into dist/** and ships with no
+// package.json#files change. `provisionTraces.ts` uploads this verbatim as an
+// ES-module Worker with a BUCKET (R2) binding + a PHOENIX_ID_BASE secret.
+/**
+ * Render the traces Worker source. Pure — bindings are wired at deploy time.
+ */
+export function renderTracesWorkerScript() {
+    return `// GENERATED by agents-cli agents traces — do not edit; edit
+// src/lib/traces/worker-template.ts and re-provision.
+export const hooks = {
+  verifyPhoenixToken: defaultVerifyPhoenixToken,
+};
+
+export default {
+  async fetch(request, env) {
+    const url = new URL(request.url);
+    const path = decodeURIComponent(url.pathname.replace(/^\\/+/, ''));
+
+    if (!path) {
+      return new Response(
+        'agents traces — PUT /<userId>/<device>/index.json or sessions/<id>.json to upload; GET the same path to read. All routes require a Phoenix bearer.',
+        { status: 200, headers: { 'content-type': 'text/plain; charset=utf-8', 'cache-control': 'private, no-store' } },
+      );
+    }
+
+    if (request.method === 'PUT') {
+      return handlePut(request, env, path);
+    }
+
+    if (request.method === 'GET' || request.method === 'HEAD') {
+      return handleGet(request, env, path);
+    }
+
+    return json({ error: 'method not allowed' }, 405);
+  },
+};
+
+async function handlePut(request, env, path) {
+  const auth = await authorizeWrite(request, env);
+  if (auth.error) return auth.error;
+
+  const segments = path.split('/').filter(Boolean);
+  const expected = auth.owner;
+  if (!expected || segments[0] !== expected) {
+    return json({ error: 'namespace mismatch', owner: expected }, 403);
+  }
+
+  const contentType = request.headers.get('content-type') || 'application/json; charset=utf-8';
+  await env.BUCKET.put(path, request.body, {
+    httpMetadata: { contentType },
+    customMetadata: {
+      owner: auth.owner,
+      uploadedAt: new Date().toISOString(),
+    },
+  });
+
+  const headers = new Headers();
+  headers.set('cache-control', 'private, no-store');
+  return new Response(JSON.stringify({ ok: true, key: path }), {
+    status: 200,
+    headers,
+  });
+}
+
+async function handleGet(request, env, path) {
+  const auth = await authorizeRead(request, env, path);
+  if (auth.error) return auth.error;
+
+  const object = await env.BUCKET.get(path);
+  if (!object) return json({ error: 'not found' }, 404);
+
+  const headers = new Headers(object.httpMetadata ?? {});
+  headers.set('cache-control', 'private, no-store');
+  return new Response(object.body, { headers });
+}
+
+// Require a Phoenix bearer that owns the path prefix.
+async function authorizeRead(request, env, path) {
+  const claims = await hooks.verifyPhoenixToken(request, env);
+  if (!claims) return { error: json({ error: 'unauthorized' }, 401) };
+  const ownerId = path.split('/').filter(Boolean)[0] || '';
+  if (!ownerId || ownerId !== claims.userId) {
+    return { error: json({ error: 'forbidden' }, 403) };
+  }
+  return { claims };
+}
+
+// Two legitimate principals for writes: static WRITE_TOKEN first (BYO),
+// otherwise Phoenix bearer. 401 when neither authenticates.
+async function authorizeWrite(request, env) {
+  const presented = (request.headers.get('authorization') || '').replace(/^Bearer\\s+/i, '');
+  if (!presented) return { error: json({ error: 'unauthorized' }, 401) };
+  if (env.WRITE_TOKEN && safeEqual(presented, env.WRITE_TOKEN)) {
+    return { kind: 'byo', owner: env.TRACES_NAMESPACE || 'byo' };
+  }
+  const claims = await hooks.verifyPhoenixToken(request, env);
+  if (claims && typeof claims.userId === 'string' && claims.userId) {
+    return { kind: 'phoenix', owner: claims.userId };
+  }
+  return { error: json({ error: 'unauthorized' }, 401) };
+}
+
+async function defaultVerifyPhoenixToken(request, env) {
+  const presented = (request.headers.get('authorization') || '').replace(/^Bearer\\s+/i, '');
+  if (!presented) return null;
+  const base = typeof env.PHOENIX_ID_BASE === 'string' ? env.PHOENIX_ID_BASE.replace(/\\/+$/, '') : '';
+  if (!base) return null;
+  let res;
+  try {
+    res = await fetch(base + '/api/v1/auth/me', {
+      method: 'GET',
+      headers: { Authorization: 'Bearer ' + presented },
+    });
+  } catch {
+    return null;
+  }
+  if (!res.ok) return null;
+  let body;
+  try {
+    body = await res.json();
+  } catch {
+    return null;
+  }
+  if (!body || typeof body.userId !== 'string' || !body.userId) return null;
+  return { userId: body.userId, email: typeof body.email === 'string' ? body.email : '' };
+}
+
+function json(body, status = 200) {
+  return new Response(JSON.stringify(body), {
+    status,
+    headers: {
+      'content-type': 'application/json; charset=utf-8',
+      'cache-control': 'private, no-store',
+    },
+  });
+}
+
+// Constant-time comparison so timing attacks can't probe a static WRITE_TOKEN.
+function safeEqual(a, b) {
+  if (a.length !== b.length) return false;
+  let diff = 0;
+  for (let i = 0; i < a.length; i++) diff |= a.charCodeAt(i) ^ b.charCodeAt(i);
+  return diff === 0;
+}
+`;
+}

package/dist/lib/types.d.ts

@@ -29,6 +29,20 @@ export declare const RUN_AUTO_KEYWORD = "auto";
  * process.env (which exec.ts `runAutoDefaultsToAffinity` reads) never sees them.
  */
 export declare const RUN_AUTO_HOST_RESOLVED_ENV = "AGENTS_RUN_AUTO_HOST_RESOLVED";
+/**
+ * Env var a host dispatcher exports into the remote SHELL when it opens an
+ * INTERACTIVE `--device` run: tells the remote CLI that this agent's stdio is a
+ * network link, so the run must be detached (tmux-wrapped) to outlive it.
+ *
+ * Rides the same shell-export prelude as {@link RUN_AUTO_HOST_RESOLVED_ENV},
+ * and for the same reason — exec.ts `resolveTmuxWrap` reads the remote CLI's own
+ * `process.env`, which `--env` flags (they reach only the spawned AGENT) never
+ * touch.
+ *
+ * Set ONLY on the interactive path. A headless `--device` dispatch is already
+ * detached with setsid by `launchDetached`, so it neither needs nor gets this.
+ */
+export declare const REMOTE_INTERACTIVE_ENV = "AGENTS_REMOTE_INTERACTIVE";
 /** Per-agent run strategy config. */
 export interface AgentRunConfig {
     strategy?: RunStrategy;
@@ -277,6 +291,22 @@ export interface HookMatches {
     git_dirty?: boolean;
     cwd_includes?: string | string[];
     project_has?: string;
+    /**
+     * Permission modes the hook fires in (e.g. `plan`). Unlike the other
+     * predicates this one is fail-open on absence: an input that carries no
+     * permission_mode/permissionMode field passes, because only some harnesses
+     * (Claude Code) report the live mode — an explicit non-listed value skips.
+     */
+    permission_mode?: string | string[];
+    /**
+     * Permission modes the hook must NOT fire in (e.g. `plan`). The inverse of
+     * `permission_mode`, and the correct predicate for gating a guard off in one
+     * mode: expressing that with the allowlist means enumerating every other
+     * mode, which silently stops firing when a harness adds or renames one. Same
+     * fail-open-on-absence rule — an input with no mode field still fires — so an
+     * unknown mode errs toward running the hook, never toward skipping it.
+     */
+    permission_mode_not?: string | string[];
 }
 /**
  * Cache scoping. Determines which cache file a hook invocation reads/writes:
@@ -942,30 +972,15 @@ export interface Meta {
      */
     hooks?: Record<string, ManifestHook>;
     /**
-     * Browser profile definitions keyed by profile name. Portable user config
-     * that syncs with `agents repo push/pull`. Runtime state (chrome-data, pids)
-     * lives separately in ~/.agents/.cache/browser/<profile>/.
-     */
-    browser?: Record<string, BrowserProfileConfig>;
-    /**
-     * Machine-local browser profiles — written to `config:`-adjacent
-     * `browser:` in `~/.agents/devices/<machine>/agents.yaml`, never the synced
-     * file.
-     *
-     * The auto-detected `default` profile lives here because it is inherently
-     * machine-specific: it carries an absolute `binary:` path and a port chosen by
-     * scanning THIS machine. Sharing it fleet-wide meant a macOS box wrote
-     * `/Applications/...`, a Linux box found it unlaunchable and rewrote
-     * `/usr/bin/chromium-browser`, and the two flipped the tracked agents.yaml back
-     * and forth on every `agents browser start` — the single largest source of
-     * churn on that file. User-created named profiles stay central and still sync.
+     * Browser profiles declared by this machine. Written as `browser:` in
+     * `~/.agents/devices/<machine>/agents.yaml`. The fleet registry is the union
+     * of every device file; central `agents.yaml` is not a browser-profile store.
      */
     deviceBrowser?: Record<string, BrowserProfileConfig>;
     /**
      * User-scope config block (`config:` in central agents.yaml). Holds the
      * user-scope keys from the device-config registry (`lib/device-config.ts`) —
-     * currently `interactiveHost` and `usagePrimaryHost`. Syncs fleet-wide via
-     * `agents repo push/pull`.
+     * currently `interactiveHost`. Syncs fleet-wide via `agents repo push/pull`.
      * Device-scope keys live in the per-device doc
      * `devices/<name>/agents.yaml` `config:` block, layered over the fleet-wide
      * defaults in {@link Meta.fleet} (`fleet.defaults.config`).

package/dist/lib/types.js

@@ -22,6 +22,20 @@ export const RUN_AUTO_KEYWORD = 'auto';
  * process.env (which exec.ts `runAutoDefaultsToAffinity` reads) never sees them.
  */
 export const RUN_AUTO_HOST_RESOLVED_ENV = 'AGENTS_RUN_AUTO_HOST_RESOLVED';
+/**
+ * Env var a host dispatcher exports into the remote SHELL when it opens an
+ * INTERACTIVE `--device` run: tells the remote CLI that this agent's stdio is a
+ * network link, so the run must be detached (tmux-wrapped) to outlive it.
+ *
+ * Rides the same shell-export prelude as {@link RUN_AUTO_HOST_RESOLVED_ENV},
+ * and for the same reason — exec.ts `resolveTmuxWrap` reads the remote CLI's own
+ * `process.env`, which `--env` flags (they reach only the spawned AGENT) never
+ * touch.
+ *
+ * Set ONLY on the interactive path. A headless `--device` dispatch is already
+ * detached with setsid by `launchDetached`, so it neither needs nor gets this.
+ */
+export const REMOTE_INTERACTIVE_ENV = 'AGENTS_REMOTE_INTERACTIVE';
 /** Every canonical mode in declaration order. Useful for iteration / validation. */
 export const ALL_MODES = ['plan', 'edit', 'auto', 'skip'];
 /** Canonical system repo cloned into ~/.agents/.system/. */

package/dist/lib/usage-backoff.d.ts

@@ -15,6 +15,7 @@ export declare function parseRetryAfterMs(header: string | null | undefined, now
 export declare function noteUsageRateLimited(agent: AgentId, retryAfter: string | null | undefined, opts?: {
     now?: number;
     fallbackMs?: number;
+    account?: string | null;
 }): void;
 /**
  * Epoch ms until which `agent`'s usage endpoint should not be called, or null
@@ -24,6 +25,6 @@ export declare function noteUsageRateLimited(agent: AgentId, retryAfter: string
  * Sweeps elapsed files while it is here: they can only accumulate at the rate
  * penalties are issued, and this is the one place that already lists them.
  */
-export declare function usageRateLimitedUntil(agent: AgentId, now?: number): number | null;
+export declare function usageRateLimitedUntil(agent: AgentId, now?: number, account?: string | null): number | null;
 /** Human-readable remaining backoff, for the error a skipped read returns. */
 export declare function formatBackoffRemaining(untilMs: number, now?: number): string;

package/dist/lib/usage-backoff.js

@@ -39,9 +39,15 @@
  * one — monotonicity is structural rather than argued, and there is no lock to
  * go stale on a path every usage read touches. Elapsed files are swept on read.
  *
- * Deliberately per-provider, not per-account: the endpoint throttles the caller,
- * and the observed 429 hit all five accounts on the box at once. Backing off one
- * account while the others keep firing would not clear the penalty.
+ * Scoped per ACCOUNT when the caller knows one, per provider otherwise
+ * (RUSH-3036). The first cut was deliberately provider-wide — the 2026-08-03
+ * incident 429'd every account on the box at once, so per-provider looked
+ * sufficient. It was not: the quotas proved per-account, and a provider-wide
+ * park let the first throttled account in a refresh pass starve every account
+ * after it in the loop's fixed iteration order — the same four accounts read
+ * 'usage unavailable' across passes while their siblings refreshed. An
+ * account-scoped penalty parks only that account; a penalty recorded with no
+ * account identity stays provider-wide and still parks everything.
  */
 import * as fs from 'fs';
 import * as path from 'path';
@@ -84,8 +90,26 @@ export function parseRetryAfterMs(header, now = Date.now()) {
     const ms = at - now;
     return ms > 0 ? Math.min(ms, MAX_BACKOFF_MS) : null;
 }
-/** Every recorded deadline for `agent`, newest-first. Never throws. */
-function deadlinesFor(agent) {
+/**
+ * File-name scope for a penalty. Provider-wide penalties are `<agent>`;
+ * account-scoped ones are `<agent>@<slug>` (RUSH-3036), where the slug is the
+ * account's usage key with path separators neutralized (usage keys carry `:`,
+ * `=`, `.` and `@` freely — all filename-safe on the platforms we run on).
+ * `@` cannot collide with the provider scope because provider files are always
+ * `<agent>.<digits>` and agent ids never contain `@`.
+ */
+function backoffScope(agent, account) {
+    if (!account)
+        return agent;
+    return `${agent}@${account.replace(/[/\\]/g, '_')}`;
+}
+/**
+ * Every recorded deadline for `scope`. A file belongs to the scope only when
+ * everything after `<scope>.` is pure digits — account slugs contain dots
+ * (emails inside usage keys), so a bare prefix match would let scope
+ * `claude@a` swallow `claude@a.b`'s files.
+ */
+function deadlinesFor(scope) {
     let names;
     try {
         names = fs.readdirSync(backoffDir());
@@ -94,12 +118,15 @@ function deadlinesFor(agent) {
         // No directory yet: nothing is throttled.
         return [];
     }
-    const prefix = `${agent}.`;
+    const prefix = `${scope}.`;
     const out = [];
     for (const name of names) {
         if (!name.startsWith(prefix))
             continue;
-        const at = Number(name.slice(prefix.length));
+        const rest = name.slice(prefix.length);
+        if (!/^\d+$/.test(rest))
+            continue;
+        const at = Number(rest);
         if (Number.isFinite(at))
             out.push(at);
     }
@@ -120,7 +147,14 @@ export function noteUsageRateLimited(agent, retryAfter, opts) {
         fs.mkdirSync(backoffDir(), { recursive: true });
         // Empty file: the name carries the whole value, so there is no content a
         // concurrent reader could catch half-written, and no document to merge.
-        fs.writeFileSync(path.join(backoffDir(), `${agent}.${deadline}`), '');
+        //
+        // With an account, the penalty is scoped to THAT account (RUSH-3036): the
+        // observed 429s are per-account quotas, and a provider-wide park let the
+        // first throttled account starve every account after it in the refresh
+        // loop's fixed order — the same 4 accounts stayed 'usage unavailable'
+        // across passes while their siblings refreshed. Callers with no account
+        // identity still record provider-wide, which continues to park everything.
+        fs.writeFileSync(path.join(backoffDir(), `${backoffScope(agent, opts?.account)}.${deadline}`), '');
     }
     catch {
         // Best-effort. An unwritable cache dir costs the cross-process backoff, not
@@ -135,19 +169,26 @@ export function noteUsageRateLimited(agent, retryAfter, opts) {
  * Sweeps elapsed files while it is here: they can only accumulate at the rate
  * penalties are issued, and this is the one place that already lists them.
  */
-export function usageRateLimitedUntil(agent, now = Date.now()) {
+export function usageRateLimitedUntil(agent, now = Date.now(), account) {
+    // A provider-wide penalty parks every account; an account-scoped read also
+    // honors that account's own penalties. A bare (no-account) read deliberately
+    // ignores account-scoped penalties — one pinned account must not park its
+    // siblings (RUSH-3036).
+    const scopes = account ? [backoffScope(agent, null), backoffScope(agent, account)] : [backoffScope(agent, null)];
     let latest = null;
-    for (const at of deadlinesFor(agent)) {
-        if (at > now) {
-            if (latest === null || at > latest)
-                latest = at;
-        }
-        else {
-            try {
-                fs.rmSync(path.join(backoffDir(), `${agent}.${at}`), { force: true });
+    for (const scope of scopes) {
+        for (const at of deadlinesFor(scope)) {
+            if (at > now) {
+                if (latest === null || at > latest)
+                    latest = at;
             }
-            catch {
-                /* another process may have swept it already */
+            else {
+                try {
+                    fs.rmSync(path.join(backoffDir(), `${scope}.${at}`), { force: true });
+                }
+                catch {
+                    /* another process may have swept it already */
+                }
             }
         }
     }

package/dist/lib/usage-refresh.d.ts

@@ -18,6 +18,10 @@ export declare const REFRESH_BURN_DIVISOR = 4;
 export declare const HOURLY_CALL_CAP = 12;
 /** How often the daemon wakes to *consider* a refresh pass (due accounts only). */
 export declare const USAGE_REFRESH_TICK_MS: number;
+/** Consecutive failed live reads before one broken account is quarantined. */
+export declare const FAILURE_QUARANTINE_THRESHOLD = 3;
+/** A chronic offender waits this long while healthy siblings keep their cadence. */
+export declare const FAILURE_QUARANTINE_MS: number;
 /**
  * One account's refresh state + published headroom. `sessionUsedPercent` /
  * `capturedAt` are the prior sample the NEXT tick projects the burn rate from;
@@ -36,6 +40,8 @@ export interface HeadroomEntry {
     callTimestamps: number[];
     /** Epoch ms this entry was written. */
     computedAt: number;
+    /** Consecutive live-fetch misses; absent on entries written before this field. */
+    consecutiveFailures?: number;
 }
 export declare function setHeadroomCachePathForTest(cachePath: string | null): string | null;
 /** Read the whole headroom cache (best-effort; missing/corrupt → empty map). */
@@ -79,13 +85,15 @@ export interface LocalUsageAccount {
     /** Live-fetch this account's usage; the daemon passes the real network fetch. */
     fetch: () => Promise<UsageInfo>;
 }
+/** Cold accounts lead each pass; both cold and cached groups rotate every tick. */
+export declare function orderUsageAccounts(accounts: LocalUsageAccount[], cache: Record<string, HeadroomEntry>, tick: number): LocalUsageAccount[];
 /**
  * Enumerate the usage accounts whose credentials live on THIS host — one
  * per unique usage key, deduped to the most-recently-active version (the same
  * canonicalization `getUsageInfoByIdentity` uses). Each carries a closure that
  * live-fetches its usage. This is the daemon's `listAccounts`; because it only
  * ever lists local, signed-in accounts, each host is the sole writer for its own
- * accounts' caches. `runUsageRefreshTick` calls this only on the fleet publisher.
+ * accounts' caches. `runUsageRefreshTick` calls this on every host.
  */
 export declare function buildLocalUsageAccounts(): Promise<LocalUsageAccount[]>;
 /** Injectable side effects, so `runUsageRefresh` is drivable without the daemon. */
@@ -95,8 +103,14 @@ export interface UsageRefreshDeps {
     listAccounts: () => Promise<LocalUsageAccount[]>;
     /** Persist a fresh snapshot to the usage cache (writeClaudeUsageCache). */
     writeUsageCache: (usageKey: string, snapshot: UsageSnapshot) => void;
-    /** Epoch ms a provider is backed off until, or null when free (usageRateLimitedUntil). */
-    backoffUntil: (agentId: AgentId) => number | null;
+    /**
+     * Epoch ms this provider — or, when `usageKey` is given, this specific
+     * account — is backed off until; null when free (usageRateLimitedUntil).
+     * Per-account scope (RUSH-3036): one throttled account must not park its
+     * siblings, which previously starved every account after the first 429 in
+     * this loop's fixed iteration order.
+     */
+    backoffUntil: (agentId: AgentId, usageKey?: string) => number | null;
 }
 export interface UsageRefreshResult {
     refreshed: number;
@@ -107,7 +121,7 @@ export interface UsageRefreshResult {
 }
 /**
  * One refresher tick: for each local account that is due, under its hourly cap,
- * and not provider-backed-off, live-fetch its usage, update the cache, and
+ * and not backed off, live-fetch its usage, update the cache, and
  * reschedule from the new burn projection. Never throws — a single account's
  * failed fetch leaves its cache untouched and counts as `failed`.
  */

package/dist/lib/usage-refresh.js

@@ -1,17 +1,17 @@
 /**
- * Daemon-owned usage refresher (publisher host).
+ * Daemon-owned usage refresher — runs on every host.
  *
  * The routing hot path (`agents run` → collectRunCandidates) reads usage
  * CACHE-ONLY (`getUsageInfoForIdentity`, RUSH-2061) and never blocks
  * on a provider fetch. Something still has to keep that cache fresh — this
- * module is that something, running inside the daemon on the configured
- * `usage.primary-host` (or locally when no primary is configured).
+ * module is that something, running inside the daemon on every host
+ * (RUSH-3193 #15: no primary/subscriber split — each host refreshes only
+ * the accounts it holds credentials for).
  *
  * Design:
  *
- *  - **Fleet sole writer.** Only the configured primary lists credential-backed
- *    accounts and calls providers. It exports derived snapshots/headroom;
- *    subscriber hosts import that token-free envelope (`usage-fleet.ts`).
+ *  - **Per-host writer.** Each host lists its own credential-backed accounts
+ *    and calls providers directly; there is no cross-host broadcast.
  *  - **Fixed 5-minute cadence** (`REFRESH_INTERVAL_MS`). Enough to keep
  *    balanced/`agents view` off multi-hour stale data without thrashing
  *    provider APIs when the user runs agents frequently. The delay helpers
@@ -19,8 +19,8 @@
  *    floor and ceiling are both 5 minutes.
  *  - **Hard hourly cap** (`HOURLY_CALL_CAP`) so a stuck "due" loop cannot
  *    hammer an endpoint past ~12 calls/account/hour.
- *  - **429 backoff.** A provider under `usageRateLimitedUntil` is skipped
- *    entirely — no live fetch, no Touch ID, no re-armed penalty.
+ *  - **429 backoff.** An account (or provider-wide scope) under
+ *    `usageRateLimitedUntil` is skipped — no live fetch or re-armed penalty.
  *  - **File-only credentials on the daemon path.** Refresh never opens the
  *    ACL-bound macOS keychain item (Touch ID storm). It uses the no-ACL
  *    access-token cache / setup-token / `.credentials.json` only
@@ -37,8 +37,8 @@
  *     serializes read-modify-write; no lost updates.
  *  3. **macOS keychain ACL / Touch ID** — fileOnly refresh never calls
  *     `security find-generic-password` on Claude's ACL item.
- *  4. **Provider 429** — whole provider skipped until Retry-After; cache
- *     freezes (visible as aged `capturedAt`) rather than hammering.
+ *  4. **Account 429** — that account is skipped until Retry-After while its
+ *     siblings continue; a provider-wide penalty still skips all accounts.
  *  5. **Expired access token (no refresh)** — usage path never rotates
  *     single-use refresh tokens; counts as `failed`, reschedules 5m later.
  *  6. **No file credential on this host** — account skipped / failed; no
@@ -71,6 +71,12 @@ export const REFRESH_BURN_DIVISOR = 4;
 export const HOURLY_CALL_CAP = 12;
 /** How often the daemon wakes to *consider* a refresh pass (due accounts only). */
 export const USAGE_REFRESH_TICK_MS = 60 * 1000;
+/** Consecutive failed live reads before one broken account is quarantined. */
+export const FAILURE_QUARANTINE_THRESHOLD = 3;
+/** A chronic offender waits this long while healthy siblings keep their cadence. */
+export const FAILURE_QUARANTINE_MS = 30 * 60 * 1000;
+const SKIP_JITTER_MIN_MS = 2_000;
+const SKIP_JITTER_RANGE_MS = 3_001;
 const HOUR_MS = 60 * 60 * 1000;
 /** Test seam for the headroom cache path (see usage.ts `setClaudeUsageCachePathForTest`). */
 let headroomCachePathOverride = null;
@@ -169,15 +175,54 @@ export function nextHeadroomEntry(prev, snapshot, now) {
         nextRefreshAt: now + computeNextRefreshDelayMs(headroom.minutesToLimit),
         callTimestamps,
         computedAt: now,
+        consecutiveFailures: snapshot ? 0 : (prev?.consecutiveFailures ?? 0) + 1,
     };
 }
+/** Stable per-account delay in [2s, 5s], used to spread skipped accounts. */
+function skipJitterMs(usageKey) {
+    let hash = 0;
+    for (const char of usageKey)
+        hash = (hash * 31 + char.charCodeAt(0)) >>> 0;
+    return SKIP_JITTER_MIN_MS + (hash % SKIP_JITTER_RANGE_MS);
+}
+function skippedHeadroomEntry(prev, usageKey, now, index) {
+    return {
+        status: prev?.status ?? null,
+        minutesToLimit: prev?.minutesToLimit ?? null,
+        sessionUsedPercent: prev?.sessionUsedPercent ?? null,
+        capturedAt: prev?.capturedAt ?? null,
+        nextRefreshAt: now + (index + 1) * skipJitterMs(usageKey),
+        callTimestamps: pruneCallTimestamps(prev?.callTimestamps ?? [], now),
+        computedAt: now,
+        consecutiveFailures: prev?.consecutiveFailures ?? 0,
+    };
+}
+function failedHeadroomEntry(prev, now) {
+    const next = nextHeadroomEntry(prev, null, now);
+    if ((next.consecutiveFailures ?? 0) >= FAILURE_QUARANTINE_THRESHOLD) {
+        next.nextRefreshAt = now + FAILURE_QUARANTINE_MS;
+    }
+    return next;
+}
+/** Cold accounts lead each pass; both cold and cached groups rotate every tick. */
+export function orderUsageAccounts(accounts, cache, tick) {
+    const rotate = (group) => {
+        if (group.length < 2)
+            return group;
+        const start = tick % group.length;
+        return [...group.slice(start), ...group.slice(0, start)];
+    };
+    const cold = accounts.filter((account) => cache[account.usageKey] == null);
+    const cached = accounts.filter((account) => cache[account.usageKey] != null);
+    return [...rotate(cold), ...rotate(cached)];
+}
 /**
  * Enumerate the usage accounts whose credentials live on THIS host — one
  * per unique usage key, deduped to the most-recently-active version (the same
  * canonicalization `getUsageInfoByIdentity` uses). Each carries a closure that
  * live-fetches its usage. This is the daemon's `listAccounts`; because it only
  * ever lists local, signed-in accounts, each host is the sole writer for its own
- * accounts' caches. `runUsageRefreshTick` calls this only on the fleet publisher.
+ * accounts' caches. `runUsageRefreshTick` calls this on every host.
  */
 export async function buildLocalUsageAccounts() {
     const accounts = [];
@@ -217,7 +262,7 @@ export async function buildLocalUsageAccounts() {
 }
 /**
  * One refresher tick: for each local account that is due, under its hourly cap,
- * and not provider-backed-off, live-fetch its usage, update the cache, and
+ * and not backed off, live-fetch its usage, update the cache, and
  * reschedule from the new burn projection. Never throws — a single account's
  * failed fetch leaves its cache untouched and counts as `failed`.
  */
@@ -230,14 +275,15 @@ export async function runUsageRefresh(deps) {
         skippedCap: 0,
         failed: 0,
     };
-    const accounts = await deps.listAccounts();
     const cache = readHeadroomCache();
+    const accounts = orderUsageAccounts(await deps.listAccounts(), cache, Math.floor(now / USAGE_REFRESH_TICK_MS));
     const updates = {};
-    for (const account of accounts) {
+    for (const [index, account] of accounts.entries()) {
         const entry = cache[account.usageKey] ?? null;
-        // A provider under a 429 penalty is off-limits — poking it re-arms the
+        // A penalized account/provider is off-limits — poking it re-arms the
         // penalty (the whole reason usage-backoff exists).
-        if ((deps.backoffUntil(account.agentId) ?? 0) > now) {
+        if ((deps.backoffUntil(account.agentId, account.usageKey) ?? 0) > now) {
+            updates[account.usageKey] = skippedHeadroomEntry(entry, account.usageKey, now, index);
             result.skippedBackoff += 1;
             continue;
         }
@@ -250,7 +296,10 @@ export async function runUsageRefresh(deps) {
         }
         try {
             const usage = await account.fetch();
-            if (usage.snapshot?.source === 'live') {
+            if (usage.snapshot) {
+                // `source` is provenance, not freshness. A forced collection that just
+                // reread a local harness event returns `last_seen`; that is still a
+                // successful collection and belongs in the shared read cache.
                 deps.writeUsageCache(account.usageKey, usage.snapshot);
                 updates[account.usageKey] = nextHeadroomEntry(entry, usage.snapshot, now);
                 result.refreshed += 1;
@@ -259,12 +308,12 @@ export async function runUsageRefresh(deps) {
                 // No live snapshot (expired token / fetch miss): don't rewrite the usage
                 // cache, but still record the call + reschedule so a broken account
                 // isn't retried every tick.
-                updates[account.usageKey] = nextHeadroomEntry(entry, null, now);
+                updates[account.usageKey] = failedHeadroomEntry(entry, now);
                 result.failed += 1;
             }
         }
         catch {
-            updates[account.usageKey] = nextHeadroomEntry(entry, null, now);
+            updates[account.usageKey] = failedHeadroomEntry(entry, now);
             result.failed += 1;
         }
     }

package/dist/lib/watchdog/log.d.ts

@@ -1,7 +1,7 @@
 /** Same path the Fleet reader pulls (apps/ext/src/core/watchdogLog.ts). */
 export declare const WATCHDOG_LOG_PATH: string;
 /** Kinds the Factory reader accepts. Keep in lockstep with watchdogLog.ts. */
-export type WatchdogEventKind = 'tick' | 'decision' | 'nudge' | 'rotate' | 'error';
+export type WatchdogEventKind = 'tick' | 'decision' | 'nudge' | 'undelivered' | 'rotate' | 'error';
 /** JSONL row shape — a verbatim replica of the Factory reader's WatchdogEvent. */
 export interface WatchdogEvent {
     ts: number;

package/dist/lib/watchdog/log.js

@@ -18,7 +18,7 @@ import * as path from 'path';
 import { withFileLock, atomicWriteFileSync, ensureLockTarget } from '../fs-atomic.js';
 /** Same path the Fleet reader pulls (apps/ext/src/core/watchdogLog.ts). */
 export const WATCHDOG_LOG_PATH = path.join(os.homedir(), '.agents', '.cache', 'logs', 'watchdog.log');
-const WATCHDOG_EVENT_KINDS = new Set(['tick', 'decision', 'nudge', 'rotate', 'error']);
+const WATCHDOG_EVENT_KINDS = new Set(['tick', 'decision', 'nudge', 'undelivered', 'rotate', 'error']);
 /** Parse the audit log without letting a partial final append hide valid rows. */
 export function parseWatchdogEvents(text) {
     const events = [];