@opsee/cli 0.11.13 → 0.11.18

package/src/foreman/usage-poller.ts

@@ -0,0 +1,489 @@
+import { isQuarantined, type Account } from "./account.js";
+import { readCredentialText, realCredentialDeps, type CredentialDeps } from "./credential-store.js";
+import type { UsageWindow } from "./usage.js";
+import type { UsageStore } from "./usage-store.js";
+
+/**
+ * The usage poller (the multi-account headroom design, §3).
+ *
+ * **This is the only module in the package permitted to read a vendor credential, and the only
+ * reason ADR-0013 was amended.** What the amendment allows is exactly what happens here: a token is
+ * read in order to ask that same vendor, on the user's own machine, how much of the user's own quota
+ * is left. It is never copied, never stored, never logged, never put in an error message, and never
+ * sent anywhere but `ANTHROPIC_USAGE_URL`. `usage-poller-boundary.test.ts` holds it to that.
+ *
+ * Why a poller at all, when the Worker's own stream already reports usage: in-band numbers only
+ * arrive for an Account that is *running*. An idle Account emits nothing, so "switch to the one with
+ * the most quota left" cannot be answered from in-band signals alone. This fills in the idle ones.
+ *
+ * The endpoint's shape is not invented. It is pinned from claude-swap's MIT-licensed `oauth.py`
+ * (`request_usage_data`, `build_usage_result`), and the one trap it holds is the scale: **this
+ * endpoint reports `utilization` as a percentage (0–100)**, where the in-band
+ * `rate_limit_info.utilization` is a fraction (0–1).
+ */
+
+export const ANTHROPIC_USAGE_URL = "https://api.anthropic.com/api/oauth/usage";
+
+/** The beta header the OAuth endpoints require. */
+export const OAUTH_BETA_HEADER = "oauth-2025-04-20";
+
+/** What the poller calls itself to the vendor. */
+const USER_AGENT = "opsee-foreman";
+
+/**
+ * The usage endpoint has a **request cap**, and every number below exists to stay under it.
+ *
+ * Roughly 28-30 requests an hour per identity, over a **trailing window** rather than a refilling
+ * bucket: capacity comes back only as old requests age out of the hour behind you, so a burst
+ * saturates the identity for a full hour and waiting does not return it early. (Measured and
+ * documented by claude-swap's `poll_policy.py`, whose floors these mirror.)
+ *
+ * The first cut of this asked every two minutes — about thirty requests an hour per Account, level
+ * with the cap and leaving nothing for a human running a command by hand. These numbers target
+ * twelve an hour at worst, which leaves most of the cap unspent.
+ *
+ * "Cap" rather than "budget" deliberately: `cli/CONTEXT.md` puts *Budget* on Headroom's avoid list,
+ * and a second thing in this package wearing an avoided word is how the vocabulary that file exists
+ * to keep straight comes apart.
+ */
+
+/** The hard floor. Nothing may ask about one Account more often than this, for any reason. */
+export const MIN_POLL_INTERVAL_MS = 3 * 60_000;
+
+/** How long a measured Account is left alone. Idle Accounts are what the poller is *for* — a
+ * running one reports in band — and an idle Account's usage moves slowly by definition. */
+export const IDLE_POLL_MS = 5 * 60_000;
+
+/** How long an Account at or past its limit is left alone: nothing about it changes until its window
+ * resets, so asking more often spends requests to learn the same number. */
+export const EXHAUSTED_POLL_MS = 10 * 60_000;
+
+/**
+ * How long an Account is left alone after the endpoint rate-limits us, when it names no interval.
+ *
+ * An hour, because that is the window that produced the refusal: with a trailing window, capacity
+ * returns as the burst ages out, and a retry in five minutes spends requests with no chance of
+ * having recovered. The endpoint's own `Retry-After` is preferred when it sends one.
+ */
+export const RATE_LIMITED_BACKOFF_MS = 60 * 60_000;
+
+/**
+ * The longest any backoff may last, whatever the endpoint asks for.
+ *
+ * `Retry-After: 999999999` is thirty-one years. Honouring it would take the Account out of every
+ * schedule for the rest of the machine's life, with nothing short of deleting the usage file to
+ * bring it back — a header from a misconfigured proxy would be indistinguishable from a permanent
+ * deregistration. Six hours is far past any real limit window and still recovers by itself.
+ */
+export const MAX_BACKOFF_MS = 6 * 60 * 60_000;
+
+/**
+ * How long a stored number may be printed without asking again.
+ *
+ * What `account usage --refresh` honours: a human asking to refresh still must not be able to spend
+ * the hour's requests by running a command repeatedly, and a number three minutes old is not one
+ * worth a request.
+ *
+ * The same value as `MIN_POLL_INTERVAL_MS` today, and kept separate because they answer different
+ * questions — one is "how old may a printed number be", the other "how often may this be asked" —
+ * and a future reason to move one is not a reason to move the other.
+ */
+export const SERVE_TTL_MS = 3 * 60_000;
+
+/** How far an Account's own schedule is nudged off every other Account's, as a fraction of its
+ * interval. Deterministic per name rather than random: a fleet still spreads out, and a test does
+ * not depend on a dice roll. */
+const JITTER_FRACTION = 0.1;
+
+/** Above this, an Account counts as exhausted for cadence purposes. Deliberately not the dispatch
+ * threshold: this is about how often to ask, not about whether to use the Account. */
+const EXHAUSTED_PCT = 95;
+
+/** How many Accounts are polled in one pass. A fleet should not become a burst of requests. */
+export const POLL_BATCH = 2;
+
+/** How many `--refresh` may ask about at once. Larger than a poll pass, because a human has just
+ * asked and is waiting — but still bounded, so a fleet is not one burst. */
+export const REFRESH_BATCH = 6;
+
+export type TokenRead = { ok: true; token: string } | { ok: false; reason: string };
+
+export type UsageFetch =
+  | { ok: true; windows: Record<string, UsageWindow> }
+  /** `retryAfterMs` is set when the endpoint rate-limited us: the interval it named, or the window
+   * that produced the refusal. Nothing asks again inside it. */
+  | { ok: false; reason: string; retryAfterMs?: number };
+
+/** The seam every consumer of usage sees. `FakeUsageSource` in tests, `AnthropicUsageSource` in
+ * the daemon; nothing above this interface needs a credential or a network. */
+export interface UsageSource {
+  fetch(account: Account, now: number): Promise<UsageFetch>;
+}
+
+/**
+ * The OAuth access token of one config directory.
+ *
+ * Exported because it is the credential boundary made explicit: this function, through
+ * `credential-store.ts`, is the only place a credential is read. Every failure is a one-line reason
+ * that quotes **nothing** of what was read — a poller failure is written to the usage store,
+ * printed by `account usage` and logged, so a reason carrying the credential would leak a token
+ * into all three.
+ */
+export async function accessTokenFrom(configDir: string, deps: CredentialDeps): Promise<TokenRead> {
+  // Wherever the vendor actually put it. On macOS that is a Keychain item derived from this config
+  // directory, not a file — the assumption that it was always a file is what made every poll on a
+  // Mac fail and every Account read "never observed".
+  const stored = await readCredentialText("claude", configDir, deps);
+  if (!stored.ok) return stored;
+  let parsed: unknown;
+  try {
+    parsed = JSON.parse(stored.text);
+  } catch {
+    return { ok: false, reason: "the stored credential is not valid JSON" };
+  }
+  const token = (parsed as { claudeAiOauth?: { accessToken?: unknown } } | null)?.claudeAiOauth?.accessToken;
+  if (typeof token !== "string" || token === "") {
+    return { ok: false, reason: "the stored credential holds no OAuth access token (an API-key login reports no usage)" };
+  }
+  return { ok: true, token };
+}
+
+export interface AnthropicUsageSourceDeps {
+  /** Defaults to `accessTokenFrom` over the real seams; a test hands in its own so no real
+   * credential is involved. */
+  readToken?: (configDir: string) => Promise<TokenRead> | TokenRead;
+  /** Defaults to `fetch`. Takes the headers separately so a test can assert where the token went. */
+  fetchJson?: (url: string, headers: Record<string, string>) => Promise<{ ok: true; body: unknown } | { ok: false; reason: string; retryAfterMs?: number }>;
+}
+
+export class AnthropicUsageSource implements UsageSource {
+  private readonly readToken: (configDir: string) => Promise<TokenRead> | TokenRead;
+  private readonly fetchJson: NonNullable<AnthropicUsageSourceDeps["fetchJson"]>;
+
+  constructor(deps: AnthropicUsageSourceDeps = {}) {
+    this.readToken = deps.readToken ?? ((configDir) => accessTokenFrom(configDir, realCredentialDeps));
+    this.fetchJson = deps.fetchJson ?? defaultFetchJson;
+  }
+
+  async fetch(account: Account, now: number): Promise<UsageFetch> {
+    // An API-key Account has no config directory and no OAuth token: there is nothing to read and
+    // nothing to ask. Refused before `readToken` is reached, so the boundary is not even approached
+    // for an Account this could never measure.
+    if (account.type !== "subscription") {
+      return { ok: false, reason: "an API-key Account reports no usage" };
+    }
+    const token = await this.readToken(account.configDir);
+    if (!token.ok) return { ok: false, reason: token.reason };
+
+    const response = await this.fetchJson(ANTHROPIC_USAGE_URL, {
+      Authorization: `Bearer ${token.token}`,
+      "anthropic-beta": OAUTH_BETA_HEADER,
+      "User-Agent": USER_AGENT,
+    });
+    if (!response.ok) return { ok: false, reason: response.reason, ...(response.retryAfterMs === undefined ? {} : { retryAfterMs: response.retryAfterMs }) };
+
+    const windows = windowsFromUsageResponse(response.body, new Date(now).toISOString());
+    // An empty success would write nothing and clear the previous failure, which reads as "measured
+    // and found nothing". A response with no window in it is a response we did not understand.
+    if (!windows) return { ok: false, reason: "the usage response held no window with both a utilization and a reset" };
+    return { ok: true, windows };
+  }
+}
+
+async function defaultFetchJson(url: string, headers: Record<string, string>): Promise<{ ok: true; body: unknown } | { ok: false; reason: string; retryAfterMs?: number }> {
+  try {
+    const response = await fetch(url, { headers, signal: AbortSignal.timeout(5_000) });
+    if (response.status === 429) {
+      // The endpoint's own interval when it names one, else the window that produced the refusal.
+      const named = Number(response.headers.get("retry-after"));
+      // Capped: `Retry-After` is whatever the far end says, and an absurd one must not be a life
+      // sentence (`MAX_BACKOFF_MS`). An HTTP-date rather than seconds parses as NaN and takes the
+      // default, which is the right answer for a header this does not understand.
+      const retryAfterMs = Number.isFinite(named) && named > 0 ? Math.min(named * 1_000, MAX_BACKOFF_MS) : RATE_LIMITED_BACKOFF_MS;
+      return { ok: false, reason: "HTTP 429", retryAfterMs };
+    }
+    if (!response.ok) return { ok: false, reason: `HTTP ${response.status}` };
+    return { ok: true, body: await response.json() };
+  } catch (error) {
+    // The vendor's message, not the request: nothing here may echo a header.
+    return { ok: false, reason: error instanceof Error ? error.message : "the usage request failed" };
+  }
+}
+
+interface RawWindow {
+  utilization?: unknown;
+  resets_at?: unknown;
+}
+
+/**
+ * The endpoint's response as usage observations, or undefined when it held nothing usable.
+ *
+ * The window keys are deliberately the vendor's own — `five_hour`, `seven_day` — because those are
+ * exactly the keys the in-band stream uses. If they differed, the same window would be stored twice
+ * and `headroom` would read whichever copy was staler as a real limit.
+ *
+ * Per-model weekly limits live in a newer `limits` array rather than under those two keys, so each
+ * is surfaced as `weekly_<model>`; without this, a Fable weekly limit would be invisible to every
+ * decision. The `extra_usage` spend axis is deliberately ignored: it is money, not a rate-limit
+ * window, and holding an Account back because of it would be a different product decision.
+ */
+export function windowsFromUsageResponse(body: unknown, observedAt: string): Record<string, UsageWindow> | undefined {
+  if (!body || typeof body !== "object") return undefined;
+  const data = body as { five_hour?: RawWindow; seven_day?: RawWindow; limits?: unknown };
+  const windows: Record<string, UsageWindow> = {};
+
+  for (const name of ["five_hour", "seven_day"] as const) {
+    const window = toUsageWindow(data[name]?.utilization, data[name]?.resets_at, observedAt);
+    if (window) windows[name] = window;
+  }
+
+  if (Array.isArray(data.limits)) {
+    for (const limit of data.limits as { scope?: { model?: { display_name?: unknown } }; percent?: unknown; resets_at?: unknown }[]) {
+      if (!limit || typeof limit !== "object") continue;
+      const model = limit.scope?.model?.display_name;
+      if (typeof model !== "string" || model === "") continue;
+      const window = toUsageWindow(limit.percent, limit.resets_at, observedAt);
+      if (window) windows[`weekly_${model.toLowerCase().replace(/[^a-z0-9]+/g, "_")}`] = window;
+    }
+  }
+
+  return Object.keys(windows).length === 0 ? undefined : windows;
+}
+
+/** One window, or undefined unless it has both a numeric utilization and a parseable reset. A
+ * utilization with no reset describes no interval; a reset with no utilization is not a measurement. */
+function toUsageWindow(utilization: unknown, resetsAt: unknown, observedAt: string): UsageWindow | undefined {
+  if (typeof utilization !== "number" || !Number.isFinite(utilization)) return undefined;
+  if (typeof resetsAt !== "string" || !Number.isFinite(Date.parse(resetsAt))) return undefined;
+  return {
+    // Taken as given, not multiplied: this endpoint already reports a percentage. The in-band
+    // stream is the one that reports a fraction (claude-worker-adapter.ts `usageWindowsFrom`).
+    usedPct: Math.max(0, Math.min(100, Math.round(utilization * 10) / 10)),
+    resetsAt,
+    observedAt,
+    source: "polled",
+  };
+}
+
+/**
+ * Which Accounts to poll in this pass, most overdue first, at most `limit` of them.
+ *
+ * Adaptive rather than a flat interval, for the reason claude-swap gives: an Account nobody has
+ * measured is worth a request now, a busy one is worth watching, and an exhausted one has nothing
+ * to say until it resets. Pure, so the cadence is testable with a fake clock.
+ *
+ * Three kinds of Account are never polled at all: an API-key Account (no window exists), a
+ * quarantined one (its credential is dead, so the request can only fail), and another vendor's
+ * (no such endpoint). A *Paused* Account is polled at the exhausted cadence — it is Paused because
+ * it hit a limit, and its reset is the one thing still worth confirming.
+ */
+export function chooseToPoll(
+  accounts: readonly Account[],
+  usage: ReadonlyMap<string, PollableRecord>,
+  now: number,
+  limit: number = POLL_BATCH,
+): Account[] {
+  // An exhausted Account is the slowest case: a failure must never make one *more* eager, which is
+  // what picking a failure interval ahead of the exhausted one did.
+  return overdueBy(accounts, usage, now, (record) => (isExhausted(record, now) ? EXHAUSTED_POLL_MS : IDLE_POLL_MS))
+    .slice(0, Math.max(0, limit))
+    .map((d) => d.account);
+}
+
+/**
+ * The Accounts whose numbers are too old to print without asking again (`SERVE_TTL_MS`).
+ *
+ * What `account usage --refresh` uses. A shorter interval than the poller's, because a human has
+ * just said they want it now — but the same floors apply underneath: an Account inside a rate-limit
+ * window is not asked however explicitly it was requested, since the requests it would spend are the
+ * one the Foreman needs to keep working.
+ */
+export function staleFor(accounts: readonly Account[], usage: ReadonlyMap<string, PollableRecord>, now: number, limit: number = REFRESH_BATCH): Account[] {
+  return overdueBy(accounts, usage, now, () => SERVE_TTL_MS)
+    .slice(0, Math.max(0, limit))
+    .map((d) => d.account);
+}
+
+/** What either schedule needs of a stored record. */
+export type PollableRecord = { windows: Record<string, UsageWindow>; pollFailedAt?: string; pollBackoffUntil?: string };
+
+/**
+ * Whether an Account is one the poller speaks to at all: the vendor that has a usage endpoint, a
+ * subscription rather than a key, and not quarantined.
+ *
+ * A predicate rather than an inline condition because it is asked in two places — here, to decide
+ * what to measure, and by the daemon, to decide whether "nothing can be measured" is even true. The
+ * daemon had its own copy, which is the one that would have gone stale the day a second vendor
+ * reports usage, and gone stale silently: it only decides what gets said.
+ */
+export function pollable(account: Account): boolean {
+  return account.vendor === "claude" && account.type === "subscription" && !isQuarantined(account);
+}
+
+/** When an Account's endpoint-imposed rate-limit window lifts, or `undefined` if it is not in one.
+ * Unparseable and already-past deadlines are both "not in one", so a corrupt record cannot wedge an
+ * Account out of the schedule for good. */
+function backoffUntil(record: PollableRecord | undefined, now: number): number | undefined {
+  const at = record?.pollBackoffUntil ? Date.parse(record.pollBackoffUntil) : Number.NaN;
+  return Number.isFinite(at) && at > now ? at : undefined;
+}
+
+/**
+ * When every pollable Account is inside a rate-limit window, the moment the first of them lifts.
+ * `undefined` when anything at all could be measured now, and when there is nothing to measure.
+ *
+ * The daemon's one question about an empty pass: is this the ordinary "nothing is due yet", or the
+ * state the poller cannot leave by itself? Answered here, beside the schedule that creates it,
+ * rather than re-derived from the store by whoever wants to print it.
+ */
+export function allBackedOffUntil(accounts: readonly Account[], usage: ReadonlyMap<string, PollableRecord>, now: number): number | undefined {
+  let soonest: number | undefined;
+  let any = false;
+  for (const account of accounts) {
+    if (!pollable(account)) continue;
+    any = true;
+    const until = backoffUntil(usage.get(account.name), now);
+    if (until === undefined) return undefined;
+    soonest = soonest === undefined ? until : Math.min(soonest, until);
+  }
+  return any ? soonest : undefined;
+}
+
+/**
+ * The Accounts due under `interval`, most overdue first.
+ *
+ * Both schedules share this so the guards can only be written once: the vendors and states that are
+ * never polled, the hard floor, the rate-limit window, and the per-Account offset.
+ */
+function overdueBy(
+  accounts: readonly Account[],
+  usage: ReadonlyMap<string, PollableRecord>,
+  now: number,
+  interval: (record: PollableRecord | undefined) => number,
+): { account: Account; overdue: number }[] {
+  const due: { account: Account; overdue: number }[] = [];
+  for (const account of accounts) {
+    if (!pollable(account)) continue;
+    const record = usage.get(account.name);
+    // A window the endpoint itself imposed. Nothing goes inside it: not the poller, not a human.
+    if (backoffUntil(record, now) !== undefined) continue;
+    const last = lastLookAt(record, now);
+    if (last === undefined) {
+      // Never observed: the most valuable request there is, since every decision about this Account
+      // is currently being made blind. `Infinity` puts it ahead of anything merely stale.
+      due.push({ account, overdue: Number.POSITIVE_INFINITY });
+      continue;
+    }
+    // The offset is a fraction of the wait actually used, not of the one asked for: derived from
+    // the un-floored interval it would silently shrink if an interval were ever set below the floor,
+    // collapsing the spread this exists to create.
+    const wait = Math.max(MIN_POLL_INTERVAL_MS, interval(record));
+    const overdue = now - last - (wait + offsetFor(account.name, wait));
+    if (overdue >= 0) due.push({ account, overdue });
+  }
+  return due.sort((a, b) => b.overdue - a.overdue || a.account.name.localeCompare(b.account.name));
+}
+
+/**
+ * A stable per-Account nudge, so a fleet registered at once does not then ask in lockstep for ever.
+ *
+ * Derived from the name, so it is the same on every run and in every test — a fleet spreads without
+ * a test depending on a dice roll. FNV-1a rather than the usual `hash * 31 + c`, which barely moves
+ * for short names: the obvious version put every one-letter Account within 200ms of the others,
+ * which is lockstep with extra steps.
+ */
+function offsetFor(name: string, interval: number): number {
+  let hash = 0x811c9dc5;
+  for (let i = 0; i < name.length; i++) {
+    hash ^= name.charCodeAt(i);
+    hash = Math.imul(hash, 0x01000193) >>> 0;
+  }
+  // A finalizer, because FNV alone does not avalanche a short name: one round after a single
+  // character leaves the high bits almost unchanged, so "a" through "h" all landed within 1.5s of
+  // each other — lockstep with extra steps, which is the thing this is here to prevent.
+  hash ^= hash >>> 16;
+  hash = Math.imul(hash, 0x7feb352d) >>> 0;
+  hash ^= hash >>> 15;
+  hash = Math.imul(hash, 0x846ca68b) >>> 0;
+  hash = (hash ^ (hash >>> 16)) >>> 0;
+  return Math.round((hash / 0xffffffff) * interval * JITTER_FRACTION);
+}
+
+/**
+ * When this Account was last looked at, successfully or not; undefined when it never has been.
+ *
+ * A stamp in the future is disregarded rather than clamped: it is not evidence of a recent look at
+ * all, it is a skewed clock or a hand-edited record. Clamping it to `now` would still read as
+ * "measured this instant" and hold the Account out for a whole interval; dropping it leaves the
+ * record looking unobserved, which is the truth and puts the Account first in the queue.
+ */
+function lastLookAt(record: PollableRecord | undefined, now: number): number | undefined {
+  if (!record) return undefined;
+  const times = [
+    ...Object.values(record.windows).map((w) => Date.parse(w.observedAt)),
+    ...(record.pollFailedAt ? [Date.parse(record.pollFailedAt)] : []),
+  ].filter((t) => Number.isFinite(t) && t <= now);
+  return times.length === 0 ? undefined : Math.max(...times);
+}
+
+function isExhausted(record: { windows: Record<string, UsageWindow> } | undefined, now: number): boolean {
+  return Object.values(record?.windows ?? {}).some((w) => Date.parse(w.resetsAt) > now && w.usedPct >= EXHAUSTED_PCT);
+}
+
+/**
+ * A failure reason with anything token-shaped taken out, and clipped to one readable line.
+ *
+ * Every reason this module builds is already clean — that is asserted — but a reason can also come
+ * from a transport's own thrown error, and those are outside our control: a library that puts the
+ * request headers in its message would otherwise write the token into the usage store, into
+ * `account usage`, and into the daemon's log at once. So nothing reaches the store unredacted.
+ */
+export function redactReason(reason: string): string {
+  const cleaned = reason
+    .replace(/sk-[A-Za-z0-9_-]{4,}/g, "sk-***")
+    .replace(/Bearer\s+\S+/gi, "Bearer ***")
+    .replace(/\b[A-Za-z0-9_-]{40,}\b/g, "***");
+  const line = cleaned.replace(/\s+/g, " ").trim();
+  return line.length > 200 ? `${line.slice(0, 197)}...` : line;
+}
+
+export interface PollRequest {
+  store: UsageStore;
+  source: UsageSource;
+  accounts: readonly Account[];
+  now?: () => number;
+}
+
+/**
+ * Polls each Account handed in and writes the result, whichever way it went.
+ *
+ * A failure is recorded and nothing else: it never clears a window, never Pauses anything, and —
+ * the important one — never counts as a credential failure. `noteCredentialFailure` and the
+ * two-strike quarantine behind it exist for a Worker turn the vendor refused, which proves a login
+ * is dead. One Anthropic API blip would otherwise quarantine every Account on the machine, and a
+ * quarantine needs a human to lift.
+ */
+export async function pollUsage(request: PollRequest): Promise<{ polled: number; failed: number }> {
+  const now = (request.now ?? Date.now)();
+  let polled = 0;
+  let failed = 0;
+  for (const account of request.accounts) {
+    let result: UsageFetch;
+    try {
+      result = await request.source.fetch(account, now);
+    } catch (error) {
+      // One Account's transport throwing must not end the pass: the next Account is the one a
+      // dispatch may be about to choose.
+      result = { ok: false, reason: error instanceof Error ? error.message : "the usage request failed" };
+    }
+    if (result.ok) {
+      request.store.observe(account.name, result.windows, now);
+      polled++;
+    } else {
+      const backoff = result.retryAfterMs === undefined ? undefined : new Date(now + Math.min(result.retryAfterMs, MAX_BACKOFF_MS));
+      request.store.noteFailure(account.name, new Date(now), redactReason(result.reason), backoff);
+      failed++;
+    }
+  }
+  return { polled, failed };
+}