@phnx-labs/agents-cli 1.22.64 → 1.22.65

package/dist/lib/usage-refresh.d.ts

@@ -18,6 +18,56 @@ export declare const REFRESH_BURN_DIVISOR = 4;
 export declare const HOURLY_CALL_CAP = 12;
 /** How often the daemon wakes to *consider* a refresh pass (due accounts only). */
 export declare const USAGE_REFRESH_TICK_MS: number;
+/**
+ * Minimum wall-clock spacing between two live usage fetches to ONE network
+ * provider, across all of its accounts. This is the pacing primitive: refreshes
+ * are issued round-robin (stalest account first) no faster than one per spacing,
+ * so aggregate endpoint load is a smooth, fixed rate — never the synchronized
+ * burst-then-stall a plain rolling-hour cap produces when every account falls
+ * due on the same tick. Set to two daemon ticks so the floor-based pacing lands
+ * on exact tick boundaries (no drift): one refresh every other tick ⇒ 30/hr.
+ */
+export declare const PROVIDER_MIN_REFRESH_SPACING_MS: number;
+/**
+ * Aggregate live fetches this daemon may spend on ONE network provider's usage
+ * endpoint per rolling hour, across ALL of that provider's local accounts —
+ * derived from {@link PROVIDER_MIN_REFRESH_SPACING_MS} so the two are always
+ * consistent (HOUR / 120s = 30).
+ *
+ * The per-account {@link HOURLY_CALL_CAP} alone scales linearly with account
+ * count — 8 Claude accounts × 12/hr = ~96 usage calls/hr from one box — and
+ * Anthropic's `/api/oauth/usage` rate-limits around ~100/hr (see the
+ * `usage-backoff.ts` header). That tripped the endpoint into per-account 429s
+ * with Retry-After penalties up to an hour: measured live on `zion`, 7 of 8
+ * Claude accounts sat parked, never refreshed inside their 5h window, so
+ * `agents view` showed `S: unavailable` and balanced routing read stale/absent
+ * usage. It got WORSE with every account added.
+ *
+ * 30/hr is a fixed rate that does NOT grow with account count, and leaves ample
+ * headroom under the ~100/hr ceiling for the auth probe (same endpoint, ~3/hr
+ * per account, RUSH-2998) and foreground `agents view` bursts. Because refreshes
+ * are paced round-robin (stalest first), each account's worst-case proactive
+ * cadence is bounded at N × spacing (8 accounts ⇒ 16 min; 16 ⇒ 32 min) — kept
+ * deliberately under the {@link USAGE_STALE_REFUSAL_MAX_AGE_MS} routing window so
+ * a budget-paced account never reads as "genuinely stale". A slightly
+ * older-but-present reading beats a 45-minute 429 park. Network providers only;
+ * grok/codex read local logs and have no rate-limited endpoint.
+ */
+export declare const PROVIDER_HOURLY_BUDGET: number;
+/**
+ * Most refreshes a single tick may catch up after the daemon has been idle/down
+ * (elapsed ≫ spacing). Without this clamp a long gap would grant many tokens at
+ * once and re-synchronize every account into the very burst the spacing exists
+ * to prevent. A small catch-up keeps the load smooth even after a restart.
+ */
+export declare const PROVIDER_CATCHUP_MAX = 2;
+/**
+ * A usage row this recently captured (by the free statusline ingest of a live
+ * `agents run`, or any writer) is already fresh — do not spend an API call to
+ * re-refresh it. Actively-used accounts stay current at zero endpoint cost, so
+ * the proactive budget is reserved for genuinely idle accounts.
+ */
+export declare const STATUSLINE_FRESH_MS: number;
 /** Consecutive failed live reads before one broken account is quarantined. */
 export declare const FAILURE_QUARANTINE_THRESHOLD = 3;
 /** A chronic offender waits this long while healthy siblings keep their cadence. */
@@ -78,6 +128,24 @@ export declare function shouldRefreshAccount(entry: HeadroomEntry | null | undef
  * projection, and record this call for the hourly cap.
  */
 export declare function nextHeadroomEntry(prev: HeadroomEntry | null | undefined, snapshot: UsageSnapshot | null, now: number): HeadroomEntry;
+/**
+ * Most-recent live-fetch time per network provider (the max call timestamp
+ * across its accounts, 0 when none), which the smooth per-provider pacing spaces
+ * the next refresh from. Non-network providers are omitted — they have no
+ * rate-limited endpoint to pace.
+ */
+export declare function providerLastCall(accounts: LocalUsageAccount[], cache: Record<string, HeadroomEntry>): Map<AgentId, number>;
+/**
+ * How many live fetches the smooth pacing permits a provider THIS tick: one per
+ * elapsed {@link PROVIDER_MIN_REFRESH_SPACING_MS} since its last fetch, clamped
+ * to {@link PROVIDER_CATCHUP_MAX} so a long idle gap (or a cold provider with no
+ * prior fetch) cannot re-burst the whole due set at once. At the daemon's 60 s
+ * tick this yields at most one fetch every other tick in steady state (⇒ the
+ * hourly budget), while a small fleet whose total demand fits under budget is
+ * never throttled — the {@link PROVIDER_HOURLY_BUDGET} rolling cap is the only
+ * gate that binds it.
+ */
+export declare function providerSpacingTokens(lastCallMs: number, now: number): number;
 /** An account whose credentials live on the publisher host. */
 export interface LocalUsageAccount {
     usageKey: string;
@@ -85,8 +153,27 @@ export interface LocalUsageAccount {
     /** Live-fetch this account's usage; the daemon passes the real network fetch. */
     fetch: () => Promise<UsageInfo>;
 }
-/** Cold accounts lead each pass; both cold and cached groups rotate every tick. */
+/**
+ * Order a pass STALEST-FIRST so a scarce per-provider budget
+ * ({@link PROVIDER_HOURLY_BUDGET}) is always spent on the accounts most in need
+ * of a fresh reading, and no account is starved indefinitely.
+ *
+ *  - **Cold accounts** (never refreshed → no cache entry) are maximally stale
+ *    and lead the pass. They rotate by `tick` so, when the budget can't cover
+ *    them all in one tick, a different cold account leads each tick.
+ *  - **Cached accounts** follow, oldest `capturedAt` first (a null capture time
+ *    counts as maximally stale). As accounts refresh their `capturedAt` advances,
+ *    so the next pass naturally rotates to whoever is now most out of date.
+ */
 export declare function orderUsageAccounts(accounts: LocalUsageAccount[], cache: Record<string, HeadroomEntry>, tick: number): LocalUsageAccount[];
+/**
+ * Aggregate live calls a network provider has already spent in the trailing hour,
+ * summed across the accounts in this pass. Seeds the per-provider budget counter
+ * so {@link PROVIDER_HOURLY_BUDGET} bounds the rolling-hour total, not just this
+ * one tick. Non-network providers (grok/codex, local logs) are excluded — they
+ * have no rate-limited endpoint to budget.
+ */
+export declare function providerRecentCalls(accounts: LocalUsageAccount[], cache: Record<string, HeadroomEntry>, now: number): Map<AgentId, number>;
 /**
  * Enumerate the usage accounts whose credentials live on THIS host — one
  * per unique usage key, deduped to the most-recently-active version (the same
@@ -111,12 +198,24 @@ export interface UsageRefreshDeps {
      * this loop's fixed iteration order.
      */
     backoffUntil: (agentId: AgentId, usageKey?: string) => number | null;
+    /**
+     * The account's current usage row from the shared cache (the row the routing
+     * hot path reads), or null when absent. Lets the refresher see the FREE
+     * statusline ingest of a live `agents run` and, when that row is recent, skip a
+     * redundant API refresh while still re-deriving headroom from it — instead of
+     * spending scarce provider budget re-fetching an already-current account.
+     */
+    readCachedSnapshot?: (usageKey: string) => UsageSnapshot | null;
 }
 export interface UsageRefreshResult {
     refreshed: number;
     skippedNotDue: number;
     skippedBackoff: number;
     skippedCap: number;
+    /** Skipped because the provider's rolling-hour budget was already spent. */
+    skippedBudget: number;
+    /** Skipped because a free statusline ingest already captured it recently. */
+    skippedFresh: number;
     failed: number;
 }
 /**

package/dist/lib/usage-refresh.js

@@ -50,7 +50,7 @@ import * as fs from 'fs';
 import * as path from 'path';
 import { getCacheDir } from './state.js';
 import { atomicWriteFileSync, ensureLockTarget, withFileLock } from './fs-atomic.js';
-import { deriveUsageHeadroom, buildCanonicalUsageContext, USAGE_SOURCE_AGENT_IDS, } from './accounting/usage.js';
+import { deriveUsageHeadroom, buildCanonicalUsageContext, agentUsesNetworkUsage, USAGE_SOURCE_AGENT_IDS, } from './accounting/usage.js';
 import { getAccountInfo } from './agents.js';
 import { listInstalledVersions, getVersionHomePath } from './installations/versions.js';
 /**
@@ -71,13 +71,63 @@ export const REFRESH_BURN_DIVISOR = 4;
 export const HOURLY_CALL_CAP = 12;
 /** How often the daemon wakes to *consider* a refresh pass (due accounts only). */
 export const USAGE_REFRESH_TICK_MS = 60 * 1000;
+const HOUR_MS = 60 * 60 * 1000;
+/**
+ * Minimum wall-clock spacing between two live usage fetches to ONE network
+ * provider, across all of its accounts. This is the pacing primitive: refreshes
+ * are issued round-robin (stalest account first) no faster than one per spacing,
+ * so aggregate endpoint load is a smooth, fixed rate — never the synchronized
+ * burst-then-stall a plain rolling-hour cap produces when every account falls
+ * due on the same tick. Set to two daemon ticks so the floor-based pacing lands
+ * on exact tick boundaries (no drift): one refresh every other tick ⇒ 30/hr.
+ */
+export const PROVIDER_MIN_REFRESH_SPACING_MS = 2 * USAGE_REFRESH_TICK_MS;
+/**
+ * Aggregate live fetches this daemon may spend on ONE network provider's usage
+ * endpoint per rolling hour, across ALL of that provider's local accounts —
+ * derived from {@link PROVIDER_MIN_REFRESH_SPACING_MS} so the two are always
+ * consistent (HOUR / 120s = 30).
+ *
+ * The per-account {@link HOURLY_CALL_CAP} alone scales linearly with account
+ * count — 8 Claude accounts × 12/hr = ~96 usage calls/hr from one box — and
+ * Anthropic's `/api/oauth/usage` rate-limits around ~100/hr (see the
+ * `usage-backoff.ts` header). That tripped the endpoint into per-account 429s
+ * with Retry-After penalties up to an hour: measured live on `zion`, 7 of 8
+ * Claude accounts sat parked, never refreshed inside their 5h window, so
+ * `agents view` showed `S: unavailable` and balanced routing read stale/absent
+ * usage. It got WORSE with every account added.
+ *
+ * 30/hr is a fixed rate that does NOT grow with account count, and leaves ample
+ * headroom under the ~100/hr ceiling for the auth probe (same endpoint, ~3/hr
+ * per account, RUSH-2998) and foreground `agents view` bursts. Because refreshes
+ * are paced round-robin (stalest first), each account's worst-case proactive
+ * cadence is bounded at N × spacing (8 accounts ⇒ 16 min; 16 ⇒ 32 min) — kept
+ * deliberately under the {@link USAGE_STALE_REFUSAL_MAX_AGE_MS} routing window so
+ * a budget-paced account never reads as "genuinely stale". A slightly
+ * older-but-present reading beats a 45-minute 429 park. Network providers only;
+ * grok/codex read local logs and have no rate-limited endpoint.
+ */
+export const PROVIDER_HOURLY_BUDGET = HOUR_MS / PROVIDER_MIN_REFRESH_SPACING_MS;
+/**
+ * Most refreshes a single tick may catch up after the daemon has been idle/down
+ * (elapsed ≫ spacing). Without this clamp a long gap would grant many tokens at
+ * once and re-synchronize every account into the very burst the spacing exists
+ * to prevent. A small catch-up keeps the load smooth even after a restart.
+ */
+export const PROVIDER_CATCHUP_MAX = 2;
+/**
+ * A usage row this recently captured (by the free statusline ingest of a live
+ * `agents run`, or any writer) is already fresh — do not spend an API call to
+ * re-refresh it. Actively-used accounts stay current at zero endpoint cost, so
+ * the proactive budget is reserved for genuinely idle accounts.
+ */
+export const STATUSLINE_FRESH_MS = REFRESH_INTERVAL_MS;
 /** Consecutive failed live reads before one broken account is quarantined. */
 export const FAILURE_QUARANTINE_THRESHOLD = 3;
 /** A chronic offender waits this long while healthy siblings keep their cadence. */
 export const FAILURE_QUARANTINE_MS = 30 * 60 * 1000;
 const SKIP_JITTER_MIN_MS = 2_000;
 const SKIP_JITTER_RANGE_MS = 3_001;
-const HOUR_MS = 60 * 60 * 1000;
 /** Test seam for the headroom cache path (see usage.ts `setClaudeUsageCachePathForTest`). */
 let headroomCachePathOverride = null;
 export function setHeadroomCachePathForTest(cachePath) {
@@ -197,6 +247,71 @@ function skippedHeadroomEntry(prev, usageKey, now, index) {
         consecutiveFailures: prev?.consecutiveFailures ?? 0,
     };
 }
+/**
+ * Reschedule an account we skipped because a free statusline ingest already
+ * captured it inside {@link STATUSLINE_FRESH_MS}. The statusline row IS a real,
+ * live sample, so RE-DERIVE headroom (status / minutesToLimit) from it against
+ * the prior sample — otherwise `status`/`minutesToLimit` would freeze at their
+ * last API-refresh value forever for exactly the actively-used accounts that
+ * stay statusline-fresh, and `capacityWeight` reads `minutesToLimit`. No call
+ * timestamp is recorded (this cost zero API budget); the next proactive attempt
+ * is pushed to one interval past the free capture.
+ */
+function freshHeadroomEntry(prev, snapshot, now, capturedAtMs) {
+    const headroom = deriveUsageHeadroom(snapshot, prev && prev.capturedAt !== null && prev.sessionUsedPercent !== null
+        ? { capturedAt: prev.capturedAt, usedPercent: prev.sessionUsedPercent }
+        : null);
+    const session = snapshot.windows.find((window) => window.key === 'session') ?? null;
+    return {
+        status: headroom.status,
+        minutesToLimit: headroom.minutesToLimit,
+        sessionUsedPercent: session?.usedPercent ?? prev?.sessionUsedPercent ?? null,
+        capturedAt: snapshot.capturedAt?.getTime() ?? prev?.capturedAt ?? null,
+        nextRefreshAt: capturedAtMs + REFRESH_INTERVAL_MS,
+        // Not an API call — do NOT record a timestamp (would wrongly spend budget).
+        callTimestamps: pruneCallTimestamps(prev?.callTimestamps ?? [], now),
+        computedAt: now,
+        consecutiveFailures: 0,
+    };
+}
+/**
+ * Most-recent live-fetch time per network provider (the max call timestamp
+ * across its accounts, 0 when none), which the smooth per-provider pacing spaces
+ * the next refresh from. Non-network providers are omitted — they have no
+ * rate-limited endpoint to pace.
+ */
+export function providerLastCall(accounts, cache) {
+    const last = new Map();
+    for (const account of accounts) {
+        if (!agentUsesNetworkUsage(account.agentId))
+            continue;
+        if (!last.has(account.agentId))
+            last.set(account.agentId, 0);
+        for (const ts of cache[account.usageKey]?.callTimestamps ?? []) {
+            if (ts > (last.get(account.agentId) ?? 0))
+                last.set(account.agentId, ts);
+        }
+    }
+    return last;
+}
+/**
+ * How many live fetches the smooth pacing permits a provider THIS tick: one per
+ * elapsed {@link PROVIDER_MIN_REFRESH_SPACING_MS} since its last fetch, clamped
+ * to {@link PROVIDER_CATCHUP_MAX} so a long idle gap (or a cold provider with no
+ * prior fetch) cannot re-burst the whole due set at once. At the daemon's 60 s
+ * tick this yields at most one fetch every other tick in steady state (⇒ the
+ * hourly budget), while a small fleet whose total demand fits under budget is
+ * never throttled — the {@link PROVIDER_HOURLY_BUDGET} rolling cap is the only
+ * gate that binds it.
+ */
+export function providerSpacingTokens(lastCallMs, now) {
+    // A cold provider (never fetched) is treated as maximally idle: grant the
+    // catch-up ceiling so a couple of accounts warm immediately without bursting.
+    const elapsed = lastCallMs <= 0 ? Infinity : now - lastCallMs;
+    if (elapsed < PROVIDER_MIN_REFRESH_SPACING_MS)
+        return 0;
+    return Math.min(PROVIDER_CATCHUP_MAX, Math.floor(elapsed / PROVIDER_MIN_REFRESH_SPACING_MS));
+}
 function failedHeadroomEntry(prev, now) {
     const next = nextHeadroomEntry(prev, null, now);
     if ((next.consecutiveFailures ?? 0) >= FAILURE_QUARANTINE_THRESHOLD) {
@@ -204,7 +319,18 @@ function failedHeadroomEntry(prev, now) {
     }
     return next;
 }
-/** Cold accounts lead each pass; both cold and cached groups rotate every tick. */
+/**
+ * Order a pass STALEST-FIRST so a scarce per-provider budget
+ * ({@link PROVIDER_HOURLY_BUDGET}) is always spent on the accounts most in need
+ * of a fresh reading, and no account is starved indefinitely.
+ *
+ *  - **Cold accounts** (never refreshed → no cache entry) are maximally stale
+ *    and lead the pass. They rotate by `tick` so, when the budget can't cover
+ *    them all in one tick, a different cold account leads each tick.
+ *  - **Cached accounts** follow, oldest `capturedAt` first (a null capture time
+ *    counts as maximally stale). As accounts refresh their `capturedAt` advances,
+ *    so the next pass naturally rotates to whoever is now most out of date.
+ */
 export function orderUsageAccounts(accounts, cache, tick) {
     const rotate = (group) => {
         if (group.length < 2)
@@ -214,7 +340,26 @@ export function orderUsageAccounts(accounts, cache, tick) {
     };
     const cold = accounts.filter((account) => cache[account.usageKey] == null);
     const cached = accounts.filter((account) => cache[account.usageKey] != null);
-    return [...rotate(cold), ...rotate(cached)];
+    const staleness = (account) => cache[account.usageKey]?.capturedAt ?? 0;
+    const byStalest = [...cached].sort((a, b) => staleness(a) - staleness(b));
+    return [...rotate(cold), ...byStalest];
+}
+/**
+ * Aggregate live calls a network provider has already spent in the trailing hour,
+ * summed across the accounts in this pass. Seeds the per-provider budget counter
+ * so {@link PROVIDER_HOURLY_BUDGET} bounds the rolling-hour total, not just this
+ * one tick. Non-network providers (grok/codex, local logs) are excluded — they
+ * have no rate-limited endpoint to budget.
+ */
+export function providerRecentCalls(accounts, cache, now) {
+    const counts = new Map();
+    for (const account of accounts) {
+        if (!agentUsesNetworkUsage(account.agentId))
+            continue;
+        const recent = pruneCallTimestamps(cache[account.usageKey]?.callTimestamps ?? [], now);
+        counts.set(account.agentId, (counts.get(account.agentId) ?? 0) + recent.length);
+    }
+    return counts;
 }
 /**
  * Enumerate the usage accounts whose credentials live on THIS host — one
@@ -274,13 +419,31 @@ export async function runUsageRefresh(deps) {
         skippedNotDue: 0,
         skippedBackoff: 0,
         skippedCap: 0,
+        skippedBudget: 0,
+        skippedFresh: 0,
         failed: 0,
     };
     const cache = readHeadroomCache();
     const accounts = orderUsageAccounts(await deps.listAccounts(), cache, Math.floor(now / USAGE_REFRESH_TICK_MS));
+    // Per-provider pacing. Two gates keep aggregate endpoint load smooth and bounded:
+    //  - a rolling-hour ceiling (PROVIDER_HOURLY_BUDGET) — the hard cap, seeded
+    //    with calls already spent in the trailing hour;
+    //  - a min-spacing token count (PROVIDER_MIN_REFRESH_SPACING_MS) — the smoother,
+    //    which issues refreshes round-robin at a fixed rate instead of the
+    //    synchronized burst-then-stall a plain rolling cap produces when every
+    //    account falls due on the same tick.
+    // Both are per-provider and network-only; accounts are ordered stalest-first, so
+    // the scarce budget always serves the account most in need and none is starved.
+    const budgetSpent = providerRecentCalls(accounts, cache, now);
+    const lastCall = providerLastCall(accounts, cache);
+    const spacingTokens = new Map();
+    for (const [agent, last] of lastCall)
+        spacingTokens.set(agent, providerSpacingTokens(last, now));
+    const spacingUsed = new Map();
     const updates = {};
     for (const [index, account] of accounts.entries()) {
         const entry = cache[account.usageKey] ?? null;
+        const network = agentUsesNetworkUsage(account.agentId);
         // A penalized account/provider is off-limits — poking it re-arms the
         // penalty (the whole reason usage-backoff exists).
         if ((deps.backoffUntil(account.agentId, account.usageKey) ?? 0) > now) {
@@ -295,6 +458,34 @@ export async function runUsageRefresh(deps) {
                 result.skippedCap += 1;
             continue;
         }
+        // A live `agents run` already refreshed this account's usage row for free via
+        // the statusline ingest — re-derive headroom from that row and skip the API
+        // call. Network providers only: a local-log provider's cache is always its
+        // own last write, so this must not suppress its refresh (grok/codex).
+        if (network) {
+            const cached = deps.readCachedSnapshot?.(account.usageKey) ?? null;
+            const capturedAtMs = cached?.capturedAt?.getTime() ?? null;
+            if (cached && capturedAtMs !== null && now - capturedAtMs < STATUSLINE_FRESH_MS) {
+                updates[account.usageKey] = freshHeadroomEntry(entry, cached, now, capturedAtMs);
+                result.skippedFresh += 1;
+                continue;
+            }
+        }
+        // Global per-provider budget: cap aggregate endpoint traffic so it does not
+        // scale linearly with account count and trip the ~100/hr rate limit, and pace
+        // it smoothly. Non-network providers (local logs) have no endpoint to protect.
+        if (network) {
+            const overHourly = (budgetSpent.get(account.agentId) ?? 0) >= PROVIDER_HOURLY_BUDGET;
+            const overSpacing = (spacingUsed.get(account.agentId) ?? 0) >= (spacingTokens.get(account.agentId) ?? 0);
+            if (overHourly || overSpacing) {
+                // Leave the entry untouched so this still-due account competes again next
+                // tick, when budget/spacing frees — never starved (stalest-first serves it).
+                result.skippedBudget += 1;
+                continue;
+            }
+            budgetSpent.set(account.agentId, (budgetSpent.get(account.agentId) ?? 0) + 1);
+            spacingUsed.set(account.agentId, (spacingUsed.get(account.agentId) ?? 0) + 1);
+        }
         try {
             const usage = await account.fetch();
             if (usage.snapshot) {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@phnx-labs/agents-cli",
-  "version": "1.22.64",
+  "version": "1.22.65",
   "description": "One CLI for all your AI coding agents - versions, config, cloud dispatch, sessions, and teams (now with first-class Grok Build CLI support)",
   "type": "module",
   "main": "dist/index.js",