@centerforagenticai/pi-multi-account 0.1.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (124) hide show
  1. package/LICENSE +21 -0
  2. package/NOTICE +29 -0
  3. package/README.md +999 -0
  4. package/config/models/pi-multi-account.v1.json +32 -0
  5. package/config/subscription-plans.v1.json +122 -0
  6. package/package.json +76 -0
  7. package/packages/pi-anthropic-oauth/LICENSE +21 -0
  8. package/packages/pi-anthropic-oauth/package.json +54 -0
  9. package/packages/pi-anthropic-oauth/src/auth.ts +396 -0
  10. package/packages/pi-anthropic-oauth/src/context.ts +116 -0
  11. package/packages/pi-anthropic-oauth/src/convert.ts +303 -0
  12. package/packages/pi-anthropic-oauth/src/index.ts +37 -0
  13. package/packages/pi-anthropic-oauth/src/prompt.ts +137 -0
  14. package/packages/pi-anthropic-oauth/src/stream.ts +476 -0
  15. package/packages/pi-antigravity/LICENSE +21 -0
  16. package/packages/pi-antigravity/package.json +77 -0
  17. package/packages/pi-antigravity/src/auth/index.ts +14 -0
  18. package/packages/pi-antigravity/src/auth/oauth.ts +442 -0
  19. package/packages/pi-antigravity/src/client/client.ts +561 -0
  20. package/packages/pi-antigravity/src/client/index.ts +1 -0
  21. package/packages/pi-antigravity/src/context.ts +110 -0
  22. package/packages/pi-antigravity/src/diagnostics/diagnostics.ts +96 -0
  23. package/packages/pi-antigravity/src/diagnostics/index.ts +1 -0
  24. package/packages/pi-antigravity/src/image/image.ts +336 -0
  25. package/packages/pi-antigravity/src/image/index.ts +1 -0
  26. package/packages/pi-antigravity/src/index.ts +280 -0
  27. package/packages/pi-antigravity/src/models/discovery.ts +154 -0
  28. package/packages/pi-antigravity/src/models/grouping.ts +424 -0
  29. package/packages/pi-antigravity/src/models/index.ts +3 -0
  30. package/packages/pi-antigravity/src/models/models.ts +500 -0
  31. package/packages/pi-antigravity/src/stream/index.ts +1 -0
  32. package/packages/pi-antigravity/src/stream/stream.ts +1478 -0
  33. package/packages/pi-antigravity/src/types/enums.ts +42 -0
  34. package/packages/pi-antigravity/src/types/index.ts +2 -0
  35. package/packages/pi-antigravity/src/types/types.ts +292 -0
  36. package/packages/pi-antigravity/src/usage/index.ts +1 -0
  37. package/packages/pi-antigravity/src/usage/usage.ts +416 -0
  38. package/packages/pi-antigravity/src/utils/http.ts +91 -0
  39. package/packages/pi-antigravity/src/utils/index.ts +3 -0
  40. package/packages/pi-antigravity/src/utils/security.ts +73 -0
  41. package/packages/pi-antigravity/src/utils/util.ts +132 -0
  42. package/scripts/multi-account.mjs +44 -0
  43. package/src/account-labels.ts +223 -0
  44. package/src/account-plan-assignment.ts +340 -0
  45. package/src/account-rate-history.ts +372 -0
  46. package/src/anthropic-adaptive-stream.ts +531 -0
  47. package/src/anthropic-alias-stream.ts +140 -0
  48. package/src/anthropic-context-compat.ts +80 -0
  49. package/src/api-pricing.ts +579 -0
  50. package/src/bounded-file-lines.ts +97 -0
  51. package/src/catalog-rebinding.ts +177 -0
  52. package/src/catalog-registration-probe.ts +111 -0
  53. package/src/codex-adapter.ts +345 -0
  54. package/src/codex-model-defaults.ts +785 -0
  55. package/src/command-completions.ts +404 -0
  56. package/src/commands.ts +2000 -0
  57. package/src/compaction.ts +14 -0
  58. package/src/config.ts +1317 -0
  59. package/src/continuation.ts +569 -0
  60. package/src/cooldowns.ts +110 -0
  61. package/src/cost-digest-store.ts +332 -0
  62. package/src/cost-digest.ts +1044 -0
  63. package/src/cost-history.ts +251 -0
  64. package/src/cost-period-closer.ts +160 -0
  65. package/src/cost-report-json.ts +318 -0
  66. package/src/cost-report-reader.ts +368 -0
  67. package/src/cost-report-render.ts +207 -0
  68. package/src/cost-report.ts +1104 -0
  69. package/src/coverage-attestation.ts +397 -0
  70. package/src/credential-lifecycle.ts +169 -0
  71. package/src/credential-refresh.ts +248 -0
  72. package/src/declaration-notice-marker.ts +238 -0
  73. package/src/diagnostic-store.ts +276 -0
  74. package/src/diagnostics.ts +309 -0
  75. package/src/discovery.ts +471 -0
  76. package/src/duration.ts +13 -0
  77. package/src/error-classification.ts +256 -0
  78. package/src/fuzzy.ts +15 -0
  79. package/src/group-policy.ts +81 -0
  80. package/src/history-store.ts +897 -0
  81. package/src/index.ts +5572 -0
  82. package/src/lifecycle.ts +378 -0
  83. package/src/logical-dispatch.ts +279 -0
  84. package/src/logical-model-selector.ts +254 -0
  85. package/src/logical-model-switcher.ts +430 -0
  86. package/src/logical-provider-attribution.ts +544 -0
  87. package/src/logical-provider.ts +1237 -0
  88. package/src/logical-route-indicator.ts +215 -0
  89. package/src/machine-lease.ts +445 -0
  90. package/src/model-support.ts +66 -0
  91. package/src/models-declaration.ts +1091 -0
  92. package/src/openai-adapter.ts +117 -0
  93. package/src/openrouter-budget.ts +304 -0
  94. package/src/openrouter-fallback.ts +146 -0
  95. package/src/period-boundaries.ts +376 -0
  96. package/src/pi-anthropic-oauth.d.ts +6 -0
  97. package/src/preflight.ts +253 -0
  98. package/src/pricing-cache.ts +235 -0
  99. package/src/project-identity.ts +100 -0
  100. package/src/provider-registration.ts +942 -0
  101. package/src/rate-formula.ts +163 -0
  102. package/src/recovery-engine.ts +853 -0
  103. package/src/recovery-output.ts +837 -0
  104. package/src/recovery-plan.ts +239 -0
  105. package/src/report-range.ts +203 -0
  106. package/src/route-resolver.ts +789 -0
  107. package/src/routing-config-transaction.ts +232 -0
  108. package/src/routing.ts +1163 -0
  109. package/src/runtime-state.ts +630 -0
  110. package/src/session-account-groups.ts +284 -0
  111. package/src/session-restore.ts +287 -0
  112. package/src/shared-usage.ts +1392 -0
  113. package/src/standalone-cli.ts +720 -0
  114. package/src/status-view.ts +578 -0
  115. package/src/subscription-plan-catalog.ts +346 -0
  116. package/src/tier-model-resolver.ts +46 -0
  117. package/src/upstream-anthropic.ts +315 -0
  118. package/src/upstream-antigravity.ts +327 -0
  119. package/src/usage-fetch.ts +1634 -0
  120. package/src/usage.ts +1026 -0
  121. package/src/vendor.ts +87 -0
  122. package/src/warmer.ts +231 -0
  123. package/src/watchdog.ts +219 -0
  124. package/src/window-history.ts +270 -0
@@ -0,0 +1,1634 @@
1
+ import { createHash } from "node:crypto";
2
+ import type { AllowedFamily, MultiAccountConfig } from "./config.js";
3
+ import type { CredentialType } from "./discovery.js";
4
+ import {
5
+ acquireMachineLease,
6
+ type MachineLeaseHandle,
7
+ } from "./machine-lease.js";
8
+ import {
9
+ normalizeUsageEndpointPercent,
10
+ type SharedUsageAttemptRecord,
11
+ type SharedUsageStore,
12
+ type UsageFailureDetail,
13
+ } from "./shared-usage.js";
14
+ import type { UsageLedger } from "./usage.js";
15
+ import { isCanonicalManagedProviderId } from "./runtime-state.js";
16
+ import { loadUpstreamAntigravityPrimitives } from "./upstream-antigravity.js";
17
+ import {
18
+ markWindowUnusable,
19
+ remainingFractionFromUtilization,
20
+ writeHistoryWindowSample,
21
+ type WindowHistoryWriteOptions,
22
+ type WindowSample,
23
+ } from "./window-history.js";
24
+
25
+ export const USAGE_FETCH_INTERVAL_MS = 5 * 60_000;
26
+ export const WINDOW_SAMPLE_INTERVAL_MS = 15 * 60_000;
27
+
28
+ /**
29
+ * Sentinel window id marking a stretch of time the fetcher could not cover, so
30
+ * an unmeasured period is distinguishable from a genuinely idle one. Without it,
31
+ * "no data" and "no usage" render identically and a cost report claims $0.00 for
32
+ * a window it never saw.
33
+ *
34
+ * INTERNAL. Operator surfaces must suppress it: it names nothing an operator can
35
+ * act on, and the double underscores read as broken markup.
36
+ */
37
+ export const GAP_WINDOW_ID = "__gap__";
38
+ export const USAGE_FETCH_TIMEOUT_MS = 10_000;
39
+ const MAX_RESPONSE_BYTES = 128 * 1024;
40
+ const MAX_ERROR_CHARS = 512;
41
+ const BASE_BACKOFF_MS = 30_000;
42
+ const MAX_BACKOFF_MS = 15 * 60_000;
43
+ const BACKOFF_LADDER_STEPS =
44
+ Math.ceil(Math.log2(MAX_BACKOFF_MS / BASE_BACKOFF_MS)) + 1;
45
+ /** Disable only after the capped rung has failed once more. */
46
+ export const USAGE_FETCH_DISABLE_AFTER_FAILURES = BACKOFF_LADDER_STEPS + 1;
47
+
48
+ const CODEX_USAGE_URL = "https://chatgpt.com/backend-api/wham/usage";
49
+ const ANTHROPIC_USAGE_URL = "https://api.anthropic.com/api/oauth/usage";
50
+ const ALLOWED_USAGE_URLS = new Set([CODEX_USAGE_URL, ANTHROPIC_USAGE_URL]);
51
+
52
+ const MAX_ANTIGRAVITY_PROJECT_ID_BYTES = 4_096;
53
+ const MAX_ANTIGRAVITY_WINDOWS = 64;
54
+ /**
55
+ * Hard bound on projection work for one quota path. Entries beyond the scan cap
56
+ * are ignored; real fork responses contain only tens of models.
57
+ */
58
+ const MAX_ANTIGRAVITY_SCANNED_ENTRIES = 4_096;
59
+ const MAX_ANTIGRAVITY_WINDOW_ID_CHARS = 128;
60
+ const ANTIGRAVITY_PROJECT_DIGEST_PREFIX = "antigravity-project-";
61
+ const ANTIGRAVITY_PROJECT_DIGEST_PATTERN = new RegExp(
62
+ `^${ANTIGRAVITY_PROJECT_DIGEST_PREFIX}[0-9a-f]{24}$`,
63
+ );
64
+
65
+ /**
66
+ * Projects a raw Google Cloud project id (from `pi-antigravity`'s decoded
67
+ * `AccountUsage.projectId`) into the only representation that may reach a
68
+ * status view, diagnostic, usage record, cost record, or history sample. The
69
+ * digest mirrors `project-identity.ts#deriveProjectKey`'s established shape
70
+ * (sha256, hex, truncated, tagged prefix) so this repository has one bounded,
71
+ * non-identifying digest convention rather than two.
72
+ */
73
+ export function deriveAntigravityProjectDigest(
74
+ rawProjectId: unknown,
75
+ ): string | undefined {
76
+ if (typeof rawProjectId !== "string" || rawProjectId.length === 0) {
77
+ return undefined;
78
+ }
79
+ if (Buffer.byteLength(rawProjectId, "utf8") > MAX_ANTIGRAVITY_PROJECT_ID_BYTES) {
80
+ return undefined;
81
+ }
82
+ const digest = createHash("sha256").update(rawProjectId, "utf8").digest("hex");
83
+ return `${ANTIGRAVITY_PROJECT_DIGEST_PREFIX}${digest.slice(0, 24)}`;
84
+ }
85
+
86
+ export function isAntigravityProjectDigest(value: string): boolean {
87
+ return ANTIGRAVITY_PROJECT_DIGEST_PATTERN.test(value);
88
+ }
89
+
90
+ export interface UsageFetchAccount {
91
+ readonly providerId: string;
92
+ readonly family: AllowedFamily;
93
+ /**
94
+ * Discovered credential type. An `api_key` account yields the unmeasured
95
+ * `not-supported` outcome before any OAuth usage call (#24). Optional so
96
+ * existing call sites and fixtures stay valid; absent is treated as the
97
+ * established OAuth path.
98
+ */
99
+ readonly credentialType?: CredentialType;
100
+ }
101
+
102
+ export interface UsageFetchStatus {
103
+ readonly enabled: boolean;
104
+ readonly disabled: boolean;
105
+ readonly failureCount: number;
106
+ readonly nextAttemptAtMs?: number;
107
+ readonly disabledReason?:
108
+ | "rate-limit"
109
+ | "server-error"
110
+ | "credential-unavailable"
111
+ | "malformed-response"
112
+ | "network-error";
113
+ }
114
+
115
+ export type UsageFetchResultStatus =
116
+ | "fetched"
117
+ | "disabled"
118
+ | "backoff"
119
+ | "not-due"
120
+ | "lease-unavailable"
121
+ | "disabled-by-config"
122
+ | "credential-unavailable"
123
+ | "failed"
124
+ /**
125
+ * The account's credential type has no supported usage endpoint (an
126
+ * `api_key` account, #24). Unmeasured, not a failure: no OAuth call is made,
127
+ * the failure ladder does not advance, and the account is not disabled.
128
+ */
129
+ | "not-supported";
130
+
131
+ export interface UsageFetchResult {
132
+ readonly providerId: string;
133
+ readonly status: UsageFetchResultStatus;
134
+ readonly utilization?: number;
135
+ readonly capturedAtMs?: number;
136
+ }
137
+
138
+ interface UsageFetchResponse {
139
+ readonly status: number;
140
+ readonly headers: {
141
+ get(name: string): string | null;
142
+ };
143
+ text(): Promise<string>;
144
+ }
145
+
146
+ export type UsageFetchImplementation = (
147
+ input: string,
148
+ init: RequestInit,
149
+ ) => Promise<UsageFetchResponse>;
150
+
151
+ export interface UsageWindowReading {
152
+ readonly windowId: string;
153
+ readonly utilization?: number;
154
+ readonly remainingFraction?: number;
155
+ readonly resetAtMs?: number;
156
+ readonly resetEpoch?: number;
157
+ readonly usable: boolean;
158
+ }
159
+
160
+ type UsageReading = {
161
+ readonly utilization: number;
162
+ readonly recoveryAtMs?: number;
163
+ readonly windows: readonly UsageWindowReading[];
164
+ /** Present only for a family whose usage endpoint names a scoped project. */
165
+ readonly projectDigest?: string;
166
+ };
167
+
168
+ function sanitizeFailureDetail(value: unknown): UsageFailureDetail | undefined {
169
+ switch (value) {
170
+ case "not-object":
171
+ case "no-quota-groups":
172
+ case "quota-summary-error":
173
+ return value;
174
+ default:
175
+ return undefined;
176
+ }
177
+ }
178
+
179
+ class UsageEndpointError extends Error {
180
+ readonly status: number | undefined;
181
+ readonly retryAfterMs: number | undefined;
182
+ readonly kind: UsageFetchStatus["disabledReason"];
183
+ readonly detail: UsageFailureDetail | undefined;
184
+
185
+ constructor(
186
+ message: string,
187
+ kind: UsageFetchStatus["disabledReason"],
188
+ status?: number,
189
+ retryAfterMs?: number,
190
+ detail?: string,
191
+ ) {
192
+ super(message);
193
+ this.name = "UsageEndpointError";
194
+ this.kind = kind;
195
+ this.status = status;
196
+ this.retryAfterMs = retryAfterMs;
197
+ this.detail = sanitizeFailureDetail(detail);
198
+ }
199
+ }
200
+
201
+ function responseHeader(
202
+ response: UsageFetchResponse,
203
+ name: string,
204
+ ): string | undefined {
205
+ try {
206
+ const value = response.headers.get(name);
207
+ return value === null ? undefined : value;
208
+ } catch {
209
+ return undefined;
210
+ }
211
+ }
212
+
213
+ function retryAfterMs(response: UsageFetchResponse): number | undefined {
214
+ const value = responseHeader(response, "retry-after");
215
+ if (value === undefined || !/^\d+(?:\.\d+)?$/.test(value.trim())) {
216
+ return undefined;
217
+ }
218
+ const seconds = Number(value);
219
+ if (!Number.isFinite(seconds) || seconds < 0) return undefined;
220
+ return Math.min(MAX_BACKOFF_MS, Math.max(0, Math.ceil(seconds * 1_000)));
221
+ }
222
+
223
+ /** Redacts bearer and access-token values before an error can escape this module. */
224
+ export function redactErrorBody(body: string): string {
225
+ const redacted = body
226
+ .replace(/Bearer\s+[A-Za-z0-9._~+/=-]+/gi, "Bearer <redacted>")
227
+ .replace(/("?access_token"?\s*[:=]\s*["']?)[^,\s"'}]+/gi, "$1<redacted>")
228
+ .trim();
229
+ return redacted.length <= MAX_ERROR_CHARS
230
+ ? redacted
231
+ : `${redacted.slice(0, MAX_ERROR_CHARS - 1)}…`;
232
+ }
233
+
234
+ function asObject(value: unknown): Record<string, unknown> | undefined {
235
+ return typeof value === "object" && value !== null && !Array.isArray(value)
236
+ ? (value as Record<string, unknown>)
237
+ : undefined;
238
+ }
239
+
240
+ function asNumber(value: unknown): number | undefined {
241
+ if (typeof value === "number" && Number.isFinite(value)) return value;
242
+ if (typeof value === "string" && value.trim() !== "") {
243
+ const parsed = Number(value);
244
+ return Number.isFinite(parsed) ? parsed : undefined;
245
+ }
246
+ return undefined;
247
+ }
248
+
249
+ function asResetMs(value: unknown): number | undefined {
250
+ const number = asNumber(value);
251
+ if (number !== undefined && number >= 0) {
252
+ return number < 100_000_000_000 ? number * 1_000 : number;
253
+ }
254
+ if (typeof value === "string") {
255
+ const parsed = Date.parse(value);
256
+ return Number.isFinite(parsed) && parsed >= 0 ? parsed : undefined;
257
+ }
258
+ return undefined;
259
+ }
260
+
261
+ function readingFromWindow(
262
+ value: unknown,
263
+ windowId: string,
264
+ ): UsageWindowReading | undefined {
265
+ const window = asObject(value);
266
+ if (!window) return undefined;
267
+ const percent =
268
+ asNumber(window.used_percent) ??
269
+ asNumber(window.used_percentage) ??
270
+ asNumber(window.usedPercent) ??
271
+ asNumber(window.utilization);
272
+ const utilization =
273
+ percent === undefined ? undefined : normalizeUsageEndpointPercent(percent);
274
+ const resetAtMs =
275
+ asResetMs(window.reset_at) ??
276
+ asResetMs(window.resetAt) ??
277
+ asResetMs(window.resets_at) ??
278
+ asResetMs(window.resetsAt);
279
+ const remainingFraction =
280
+ utilization === undefined
281
+ ? undefined
282
+ : remainingFractionFromUtilization(utilization);
283
+ return {
284
+ windowId,
285
+ ...(utilization === undefined ? {} : { utilization }),
286
+ ...(remainingFraction === undefined ? {} : { remainingFraction }),
287
+ ...(resetAtMs === undefined ? {} : { resetAtMs, resetEpoch: resetAtMs }),
288
+ usable:
289
+ utilization !== undefined &&
290
+ remainingFraction !== undefined &&
291
+ resetAtMs !== undefined,
292
+ };
293
+ }
294
+
295
+ function windowEntries(
296
+ container: Record<string, unknown> | undefined,
297
+ knownIds: readonly string[],
298
+ ): UsageWindowReading[] {
299
+ if (!container) return [];
300
+ const entries = new Map<string, UsageWindowReading>();
301
+ for (const key of knownIds) {
302
+ const reading = readingFromWindow(container[key], key);
303
+ if (reading) entries.set(key, reading);
304
+ }
305
+ for (const [key, value] of Object.entries(container)) {
306
+ if (entries.has(key)) continue;
307
+ const object = asObject(value);
308
+ if (
309
+ !object ||
310
+ ![
311
+ "used_percent",
312
+ "used_percentage",
313
+ "usedPercent",
314
+ "utilization",
315
+ "reset_at",
316
+ "resetAt",
317
+ "resets_at",
318
+ "resetsAt",
319
+ ].some((field) => field in object)
320
+ ) {
321
+ continue;
322
+ }
323
+ const reading = readingFromWindow(object, key);
324
+ if (reading) entries.set(key, reading);
325
+ }
326
+ return [...entries.values()];
327
+ }
328
+
329
+ function primaryReading(
330
+ windows: readonly UsageWindowReading[],
331
+ ): UsageReading | undefined {
332
+ const usable = windows.find((window) => window.utilization !== undefined);
333
+ if (!usable || usable.utilization === undefined) return undefined;
334
+ // `recoveryAtMs` means "this account is out and recovers at T", and
335
+ // `snapshotIndicatesExhaustion` treats any future `recoveryAtMs` as
336
+ // exhaustion. A window's `resetAtMs` is its ROUTINE reset time and is present
337
+ // whether or not the window has capacity, so carrying it unconditionally
338
+ // marked a healthy account (e.g. Codex primary_window at 17% used, weekly
339
+ // reset ~6 days out) exhausted and demoted `unified` to the owning-vendor API
340
+ // tier until that reset (#72). Only an actually-spent window contributes a
341
+ // recovery time; a window with capacity to spare recovers nothing.
342
+ const exhausted = usable.utilization >= 1 || usable.remainingFraction === 0;
343
+ return {
344
+ utilization: usable.utilization,
345
+ ...(exhausted && usable.resetAtMs !== undefined
346
+ ? { recoveryAtMs: usable.resetAtMs }
347
+ : {}),
348
+ windows,
349
+ };
350
+ }
351
+
352
+ /**
353
+ * One quota row from `pi-antigravity`'s decoded `AccountUsage`. The upstream
354
+ * shape is intentionally treated as unknown here (the pinned package owns
355
+ * usage decoding; this repository owns projection), so every field is
356
+ * defensively narrowed the same way `readingFromWindow` narrows the
357
+ * Codex/Anthropic raw JSON above -- never guessed, never passed through.
358
+ */
359
+ function antigravityQuotaReading(
360
+ value: unknown,
361
+ idField: "bucketId" | "modelId",
362
+ ): UsageWindowReading | undefined {
363
+ const quota = asObject(value);
364
+ if (!quota) return undefined;
365
+ const rawId = quota[idField];
366
+ const windowId =
367
+ typeof rawId === "string" &&
368
+ rawId.length > 0 &&
369
+ rawId.length <= MAX_ANTIGRAVITY_WINDOW_ID_CHARS
370
+ ? rawId
371
+ : undefined;
372
+ if (windowId === undefined) return undefined;
373
+ const rawRemaining = asNumber(quota.remainingFraction);
374
+ const utilization =
375
+ rawRemaining === undefined
376
+ ? undefined
377
+ : Math.max(0, Math.min(1, 1 - rawRemaining));
378
+ const remainingFraction =
379
+ utilization === undefined ? undefined : remainingFractionFromUtilization(utilization);
380
+ const resetAtMs = asResetMs(quota.resetTime);
381
+ return {
382
+ windowId,
383
+ ...(utilization === undefined ? {} : { utilization }),
384
+ ...(remainingFraction === undefined ? {} : { remainingFraction }),
385
+ ...(resetAtMs === undefined ? {} : { resetAtMs, resetEpoch: resetAtMs }),
386
+ usable:
387
+ utilization !== undefined &&
388
+ remainingFraction !== undefined &&
389
+ resetAtMs !== undefined,
390
+ };
391
+ }
392
+
393
+ function boundedMostConstrainedAntigravityWindows(
394
+ values: Iterable<unknown>,
395
+ idField: "bucketId" | "modelId",
396
+ ): UsageWindowReading[] {
397
+ const candidates: UsageWindowReading[] = [];
398
+ const seenWindowIds = new Set<string>();
399
+ let scannedEntries = 0;
400
+ for (const value of values) {
401
+ if (scannedEntries >= MAX_ANTIGRAVITY_SCANNED_ENTRIES) break;
402
+ scannedEntries += 1;
403
+ const reading = antigravityQuotaReading(value, idField);
404
+ if (reading === undefined || seenWindowIds.has(reading.windowId)) continue;
405
+ seenWindowIds.add(reading.windowId);
406
+ candidates.push(reading);
407
+ }
408
+ // Stable sort preserves the first-seen input order when utilization ties.
409
+ return candidates
410
+ .sort((left, right) => {
411
+ const leftUtilization = left.utilization ?? Number.NEGATIVE_INFINITY;
412
+ const rightUtilization = right.utilization ?? Number.NEGATIVE_INFINITY;
413
+ return leftUtilization === rightUtilization
414
+ ? 0
415
+ : rightUtilization - leftUtilization;
416
+ })
417
+ .slice(0, MAX_ANTIGRAVITY_WINDOWS);
418
+ }
419
+
420
+ /**
421
+ * Group containers count toward the same scan budget as the buckets they hold,
422
+ * so an oversized response of empty groups cannot bypass the entry cap.
423
+ */
424
+ function* antigravityGroupBuckets(groups: readonly unknown[]): Iterable<unknown> {
425
+ let visited = 0;
426
+ for (const group of groups) {
427
+ if (++visited > MAX_ANTIGRAVITY_SCANNED_ENTRIES) return;
428
+ const groupRecord = asObject(group);
429
+ const buckets =
430
+ groupRecord && Array.isArray(groupRecord.buckets) ? groupRecord.buckets : [];
431
+ for (const bucket of buckets) {
432
+ if (++visited > MAX_ANTIGRAVITY_SCANNED_ENTRIES) return;
433
+ yield bucket;
434
+ }
435
+ }
436
+ }
437
+
438
+ /**
439
+ * Projects the decoded `AccountUsage` result into exactly the named, bounded
440
+ * facts this repository retains: a small set of group- or model-quota windows
441
+ * and a digest of the account-scoped project id. Every other upstream field
442
+ * (plan labels, display names, tier names, raw endpoint, raw project id) is
443
+ * dropped here, at the seam, before any status, diagnostic, usage, cost, or
444
+ * history surface can see it.
445
+ */
446
+ export function projectAntigravityUsage(
447
+ raw: unknown,
448
+ ): { windows: UsageWindowReading[]; projectDigest?: string } | undefined {
449
+ const usage = asObject(raw);
450
+ if (!usage) return undefined;
451
+ const projectDigest = deriveAntigravityProjectDigest(usage.projectId);
452
+ const groups = Array.isArray(usage.groups) ? usage.groups : [];
453
+ const boundedWindows = boundedMostConstrainedAntigravityWindows(
454
+ antigravityGroupBuckets(groups),
455
+ "bucketId",
456
+ );
457
+ if (!boundedWindows.some((window) => window.usable)) {
458
+ const models = Array.isArray(usage.models) ? usage.models : [];
459
+ const boundedModelWindows = boundedMostConstrainedAntigravityWindows(
460
+ models,
461
+ "modelId",
462
+ );
463
+ if (boundedModelWindows.length > 0) {
464
+ return {
465
+ windows: boundedModelWindows,
466
+ ...(projectDigest === undefined ? {} : { projectDigest }),
467
+ };
468
+ }
469
+ }
470
+ return {
471
+ windows: boundedWindows,
472
+ ...(projectDigest === undefined ? {} : { projectDigest }),
473
+ };
474
+ }
475
+
476
+ /**
477
+ * Injectable Antigravity usage transport; defaults to the reviewed upstream
478
+ * primitive. The optional `signal` is forwarded to the fork's
479
+ * `fetchAccountUsage`, which propagates it into every one of its three
480
+ * internal legs (`loadCodeAssist`, quota summary, available-models catalog)
481
+ * and drains every settlement via `Promise.allSettled` before rejecting on
482
+ * abort -- see `pi-antigravity`'s `src/usage/usage.ts`. Aborting this signal
483
+ * is therefore the only way this repository can actually cancel the real
484
+ * upstream call rather than merely giving up on waiting for it.
485
+ */
486
+ export type AntigravityUsageFetchImplementation = (
487
+ apiKey: string,
488
+ options?: { readonly signal?: AbortSignal },
489
+ ) => Promise<unknown>;
490
+
491
+ async function defaultFetchAntigravityUsage(
492
+ apiKey: string,
493
+ options?: { readonly signal?: AbortSignal },
494
+ ): Promise<unknown> {
495
+ if (process.env.VITEST === "true") {
496
+ throw new UsageEndpointError(
497
+ "network access is disabled in tests",
498
+ "network-error",
499
+ );
500
+ }
501
+ const primitives = await loadUpstreamAntigravityPrimitives();
502
+ return primitives.fetchUsage(
503
+ apiKey,
504
+ options?.signal === undefined ? {} : { signal: options.signal },
505
+ );
506
+ }
507
+
508
+ function parsePayload(text: string): Record<string, unknown> {
509
+ if (Buffer.byteLength(text, "utf8") > MAX_RESPONSE_BYTES) {
510
+ throw new UsageEndpointError(
511
+ "usage endpoint response exceeded the bounded response size",
512
+ "malformed-response",
513
+ );
514
+ }
515
+ let payload: unknown;
516
+ try {
517
+ payload = JSON.parse(text);
518
+ } catch {
519
+ throw new UsageEndpointError(
520
+ "usage endpoint response was not valid JSON",
521
+ "malformed-response",
522
+ );
523
+ }
524
+ const object = asObject(payload);
525
+ if (!object) {
526
+ throw new UsageEndpointError(
527
+ "usage endpoint response was not an object",
528
+ "malformed-response",
529
+ );
530
+ }
531
+ return object;
532
+ }
533
+
534
+ export function normalizeCodexUsagePayload(
535
+ payload: Record<string, unknown>,
536
+ ): UsageReading {
537
+ const rateLimit = asObject(payload.rate_limit);
538
+ const windows = windowEntries(rateLimit, [
539
+ "primary_window",
540
+ "secondary_window",
541
+ ]);
542
+ const topLevelWindows = windowEntries(payload, [
543
+ "primary_window",
544
+ "secondary_window",
545
+ ]);
546
+ const byId = new Map(windows.map((window) => [window.windowId, window]));
547
+ for (const window of topLevelWindows) {
548
+ if (!byId.has(window.windowId)) byId.set(window.windowId, window);
549
+ }
550
+ const reading = primaryReading([...byId.values()]);
551
+ if (reading) return reading;
552
+ throw new UsageEndpointError(
553
+ "Codex usage endpoint returned no bounded rate-limit window",
554
+ "malformed-response",
555
+ );
556
+ }
557
+
558
+ export function normalizeAnthropicUsagePayload(
559
+ payload: Record<string, unknown>,
560
+ ): UsageReading {
561
+ const windows = windowEntries(payload, [
562
+ "five_hour",
563
+ "fiveHour",
564
+ "primary",
565
+ "weekly",
566
+ "seven_day",
567
+ "sevenDay",
568
+ "seven_day_opus",
569
+ ]);
570
+ const reading = primaryReading(windows);
571
+ if (reading) return reading;
572
+ throw new UsageEndpointError(
573
+ "Anthropic usage endpoint returned no bounded rate-limit window",
574
+ "malformed-response",
575
+ );
576
+ }
577
+
578
+ async function fetchWithTimeout(
579
+ fetchImpl: UsageFetchImplementation,
580
+ url: string,
581
+ init: RequestInit,
582
+ timeoutMs: number,
583
+ ): Promise<UsageFetchResponse> {
584
+ const controller = new AbortController();
585
+ const timeout = setTimeout(() => controller.abort(), timeoutMs);
586
+ try {
587
+ return await fetchImpl(url, { ...init, signal: controller.signal });
588
+ } catch (error) {
589
+ if (controller.signal.aborted) {
590
+ throw new UsageEndpointError(
591
+ `usage endpoint timed out after ${Math.round(timeoutMs / 1_000)}s`,
592
+ "network-error",
593
+ );
594
+ }
595
+ throw new UsageEndpointError(
596
+ redactErrorBody(error instanceof Error ? error.message : String(error)),
597
+ "network-error",
598
+ );
599
+ } finally {
600
+ clearTimeout(timeout);
601
+ }
602
+ }
603
+
604
+ async function queryEndpoint(
605
+ account: UsageFetchAccount,
606
+ access: string,
607
+ fetchImpl: UsageFetchImplementation,
608
+ timeoutMs: number,
609
+ ): Promise<UsageReading> {
610
+ const headers: Record<string, string> =
611
+ account.family === "openai-codex"
612
+ ? { Authorization: `Bearer ${access}` }
613
+ : {
614
+ Authorization: `Bearer ${access}`,
615
+ "anthropic-beta": "oauth-2025-04-20",
616
+ };
617
+ const url =
618
+ account.family === "openai-codex" ? CODEX_USAGE_URL : ANTHROPIC_USAGE_URL;
619
+ const response = await fetchWithTimeout(
620
+ fetchImpl,
621
+ url,
622
+ { headers },
623
+ timeoutMs,
624
+ );
625
+ if (response.status < 200 || response.status >= 300) {
626
+ const body = await response.text().catch(() => "");
627
+ const retryMs = retryAfterMs(response);
628
+ throw new UsageEndpointError(
629
+ `usage endpoint returned ${response.status}: ${redactErrorBody(body)}`,
630
+ response.status === 429
631
+ ? "rate-limit"
632
+ : response.status >= 500
633
+ ? "server-error"
634
+ : "network-error",
635
+ response.status,
636
+ retryMs,
637
+ );
638
+ }
639
+ const payload = parsePayload(await response.text());
640
+ return account.family === "openai-codex"
641
+ ? normalizeCodexUsagePayload(payload)
642
+ : normalizeAnthropicUsagePayload(payload);
643
+ }
644
+
645
+ function statusFromAttempt(
646
+ attempt: SharedUsageAttemptRecord | undefined,
647
+ enabled: boolean,
648
+ ): UsageFetchStatus {
649
+ const nextAttemptAtMs =
650
+ attempt?.failureCount === 0 ? undefined : attempt?.nextAttemptAtMs;
651
+ const disabledReason =
652
+ attempt?.failureReason as UsageFetchStatus["disabledReason"];
653
+ return {
654
+ enabled,
655
+ disabled: attempt?.disabled ?? false,
656
+ failureCount: attempt?.failureCount ?? 0,
657
+ ...(nextAttemptAtMs === undefined ? {} : { nextAttemptAtMs }),
658
+ ...(disabledReason === undefined ? {} : { disabledReason }),
659
+ };
660
+ }
661
+
662
+ function failureReason(
663
+ error: UsageEndpointError,
664
+ ): NonNullable<UsageFetchStatus["disabledReason"]> {
665
+ return error.kind ?? "network-error";
666
+ }
667
+
668
+ /**
669
+ * Lets an Antigravity attempt's bounded per-attempt deadline hand the
670
+ * already-acquired machine lease off to a background drain instead of the
671
+ * caller releasing it immediately, when the real upstream call is still
672
+ * outstanding at that deadline. `handedOff` starts `false`; the caller's own
673
+ * `finally { if (!leaseGuard.handedOff) lease.release(); }` is a no-op once
674
+ * it flips `true`, so ownership only ever moves one direction and is never
675
+ * released twice.
676
+ */
677
+ interface AntigravityLeaseGuard {
678
+ readonly lease: MachineLeaseHandle;
679
+ handedOff: boolean;
680
+ }
681
+
682
+ /** Detached, lease-coordinated authoritative usage fetcher. */
683
+ export class UsageFetcher {
684
+ readonly #lockPath: string;
685
+ readonly #usage: UsageLedger;
686
+ readonly #sharedStore: SharedUsageStore;
687
+ readonly #resolveCredential: (
688
+ providerId: string,
689
+ ) => Promise<string | undefined>;
690
+ readonly #fetchImpl: UsageFetchImplementation;
691
+ readonly #fetchAntigravityUsage: AntigravityUsageFetchImplementation;
692
+ readonly #now: () => number;
693
+ readonly #timeoutMs: number;
694
+ readonly #onUsageRecorded: ((providerId: string) => void) | undefined;
695
+ readonly #windowHistoryOptions: WindowHistoryWriteOptions | undefined;
696
+ readonly #lastWindowFetchAtMs = new Map<string, number>();
697
+ /**
698
+ * In-flight failure-triggered refreshes, keyed by account.
699
+ *
700
+ * Claimed synchronously before the first `await`, so two failures arriving in
701
+ * the same tick join one refresh instead of racing to the durable checks.
702
+ * The durable markers cover the cross-process case; this covers our own.
703
+ */
704
+ readonly #inFlightFailureRefresh = new Map<
705
+ string,
706
+ Promise<UsageFetchResult>
707
+ >();
708
+ /**
709
+ * One real, in-flight `pi-antigravity` `fetchAccountUsage` call per
710
+ * canonical account, shared by every `UsageFetcher` instance in this
711
+ * process.
712
+ *
713
+ * `fetchAccountUsage` accepts an `AbortSignal` and propagates it into every
714
+ * one of its three internal legs, draining every settlement via
715
+ * `Promise.allSettled` before rejecting on abort. Each entry therefore owns
716
+ * a dedicated `AbortController`, created once when a genuinely new call
717
+ * starts, so `#fetchAntigravityUsageWithDeadline` can actually cancel the
718
+ * real upstream call at its bounded per-attempt deadline instead of merely
719
+ * giving up on waiting for it. Reusing the pending entry across joiners
720
+ * means at most one real call -- and one controller -- is outstanding per
721
+ * account in THIS process; a joiner never gets its own controller, so its
722
+ * own deadline aborts the SAME shared call other still-pending joiners are
723
+ * awaiting too. That is deliberate: the call is genuinely one shared
724
+ * operation, not one per caller.
725
+ *
726
+ * The entry is cleared only when the real promise actually settles
727
+ * (resolves or rejects), never when a caller's per-attempt deadline merely
728
+ * gives up on it -- a still-pending upstream call must stay reachable so a
729
+ * later poll can await its eventual result instead of duplicating the
730
+ * call. A late settlement after every waiting poll already timed out is
731
+ * simply observed by the cleanup handler and produces no write: nothing
732
+ * outside that handler still awaits the promise at that point.
733
+ *
734
+ * The entry holds only the promise and its controller, never the
735
+ * credential that started it. It cannot: telling "same account, refreshed
736
+ * token" apart from "a different account now occupies this slot" would
737
+ * need retaining the raw credential for comparison, or hashing it into a
738
+ * fingerprint -- both forbidden by this package's credential-retention
739
+ * discipline, and no legitimate boundary here yields an opaque, non-secret
740
+ * generation number to use instead. So a credential change while a call is
741
+ * pending does not start a second call under the new value; the poll joins
742
+ * the one already in flight, same as any other still-pending poll. This is
743
+ * a known, accepted limit, not a claimed guarantee: if an operator
744
+ * re-logs a slot into a different account while a call for the old one is
745
+ * still outstanding, that call's eventual result is still attributed to
746
+ * this slot. Closing it would need the credential-resolution boundary
747
+ * itself to hand back an opaque per-slot generation alongside the
748
+ * credential; adding one is outside this file's touches.
749
+ *
750
+ * This map is a class-static field: EVERY `UsageFetcher` instance in this
751
+ * process shares it, so two instances constructed in the same process
752
+ * cannot each start their own concurrent real call for the same account.
753
+ * It is still not by itself a machine-wide or fleet-wide guard -- a peer OS
754
+ * process has its own module state and its own map. The per-attempt
755
+ * machine lease (`acquireMachineLease`) is the cross-process
756
+ * serialization, and it now stays held -- renewed at its own
757
+ * `renewalIntervalMs` -- through this entry's ACTUAL settlement rather
758
+ * than releasing at the earlier bounded deadline, so a peer process can
759
+ * never acquire the lease and start a second real call while this
760
+ * process's real call is still genuinely outstanding. A same-process
761
+ * concurrent attempt for an account whose lease is still held this way
762
+ * gets the ordinary `lease-unavailable` busy/skip outcome, the same as any
763
+ * other lease contention; it never joins a call an earlier deadline already
764
+ * abandoned.
765
+ */
766
+ static readonly #inFlightAntigravityRawFetch = new Map<
767
+ string,
768
+ { readonly promise: Promise<unknown>; readonly controller: AbortController }
769
+ >();
770
+ readonly #lastWindowGapAtMs = new Map<string, number>();
771
+
772
+ constructor(options: {
773
+ readonly lockPath: string;
774
+ readonly usage: UsageLedger;
775
+ readonly sharedStore: SharedUsageStore;
776
+ readonly resolveCredential: (
777
+ providerId: string,
778
+ ) => Promise<string | undefined>;
779
+ readonly fetchImpl?: UsageFetchImplementation;
780
+ readonly fetchAntigravityUsage?: AntigravityUsageFetchImplementation;
781
+ readonly now?: () => number;
782
+ readonly timeoutMs?: number;
783
+ readonly onUsageRecorded?: (providerId: string) => void;
784
+ readonly windowHistoryOptions?: WindowHistoryWriteOptions;
785
+ }) {
786
+ this.#lockPath = options.lockPath;
787
+ this.#usage = options.usage;
788
+ this.#sharedStore = options.sharedStore;
789
+ this.#resolveCredential = options.resolveCredential;
790
+ this.#fetchImpl =
791
+ options.fetchImpl ??
792
+ ((input, init) => {
793
+ if (process.env.VITEST === "true") {
794
+ return Promise.reject(
795
+ new UsageEndpointError(
796
+ "network access is disabled in tests",
797
+ "network-error",
798
+ ),
799
+ );
800
+ }
801
+ if (!ALLOWED_USAGE_URLS.has(input)) {
802
+ return Promise.reject(
803
+ new UsageEndpointError(
804
+ "usage endpoint URL was not allowlisted",
805
+ "network-error",
806
+ ),
807
+ );
808
+ }
809
+ return globalThis.fetch(input, init);
810
+ });
811
+ this.#fetchAntigravityUsage =
812
+ options.fetchAntigravityUsage ?? defaultFetchAntigravityUsage;
813
+ this.#now = options.now ?? Date.now;
814
+ this.#timeoutMs = options.timeoutMs ?? USAGE_FETCH_TIMEOUT_MS;
815
+ this.#onUsageRecorded = options.onUsageRecorded;
816
+ this.#windowHistoryOptions = options.windowHistoryOptions;
817
+ }
818
+
819
+ #notifyUsageRecorded(providerId: string): void {
820
+ try {
821
+ this.#onUsageRecorded?.(providerId);
822
+ } catch {
823
+ // Observation is advisory; callback failure cannot reclassify a fetch.
824
+ }
825
+ }
826
+
827
+ status(
828
+ providerId: string,
829
+ family: AllowedFamily,
830
+ config: MultiAccountConfig,
831
+ ): UsageFetchStatus {
832
+ const enabled = config.usageFetchEnabled?.[family] ?? true;
833
+ return statusFromAttempt(
834
+ this.#sharedStore.latestAttempt(providerId, family),
835
+ enabled,
836
+ );
837
+ }
838
+
839
+ #accountKey(account: UsageFetchAccount): string {
840
+ return `${account.family}:${account.providerId}`;
841
+ }
842
+
843
+ async #persistWindowGap(
844
+ account: UsageFetchAccount,
845
+ recordedAtMs: number,
846
+ reason: string,
847
+ ): Promise<void> {
848
+ const key = this.#accountKey(account);
849
+ const previous = this.#lastWindowGapAtMs.get(key);
850
+ if (
851
+ previous !== undefined &&
852
+ recordedAtMs - previous < WINDOW_SAMPLE_INTERVAL_MS
853
+ ) {
854
+ return;
855
+ }
856
+ const gap = markWindowUnusable({
857
+ providerId: account.providerId,
858
+ accountId: account.providerId,
859
+ family: account.family,
860
+ windowId: GAP_WINDOW_ID,
861
+ recordedAtMs,
862
+ schemaVersion: 1,
863
+ buildProvenance: `node@${process.version}`,
864
+ observerId: `process-${process.pid}`,
865
+ unusableReason: reason,
866
+ });
867
+ if (await writeHistoryWindowSample(gap, this.#windowHistoryOptions)) {
868
+ this.#lastWindowGapAtMs.set(key, recordedAtMs);
869
+ }
870
+ }
871
+
872
+ async #persistWindowSamples(
873
+ account: UsageFetchAccount,
874
+ reading: UsageReading,
875
+ recordedAtMs: number,
876
+ ): Promise<boolean> {
877
+ let persisted = true;
878
+ for (const window of reading.windows) {
879
+ const baseSample = {
880
+ providerId: account.providerId,
881
+ accountId: account.providerId,
882
+ family: account.family,
883
+ windowId: window.windowId,
884
+ recordedAtMs,
885
+ ...(window.resetAtMs === undefined
886
+ ? {}
887
+ : { resetAtMs: window.resetAtMs }),
888
+ ...(window.resetEpoch === undefined
889
+ ? {}
890
+ : { resetEpoch: window.resetEpoch }),
891
+ ...(window.remainingFraction === undefined
892
+ ? {}
893
+ : { remainingFraction: window.remainingFraction }),
894
+ ...(reading.projectDigest === undefined
895
+ ? {}
896
+ : { projectDigest: reading.projectDigest }),
897
+ };
898
+ const sample: WindowSample = window.usable
899
+ ? {
900
+ ...baseSample,
901
+ schemaVersion: 1,
902
+ buildProvenance: `node@${process.version}`,
903
+ observerId: `process-${process.pid}`,
904
+ usable: true,
905
+ }
906
+ : markWindowUnusable({
907
+ ...baseSample,
908
+ schemaVersion: 1,
909
+ buildProvenance: `node@${process.version}`,
910
+ observerId: `process-${process.pid}`,
911
+ });
912
+ if (!(await writeHistoryWindowSample(sample, this.#windowHistoryOptions))) {
913
+ persisted = false;
914
+ }
915
+ }
916
+ if (persisted && reading.windows.length > 0) {
917
+ this.#lastWindowFetchAtMs.set(this.#accountKey(account), recordedAtMs);
918
+ }
919
+ return persisted;
920
+ }
921
+
922
+ async fetchAccount(
923
+ account: UsageFetchAccount,
924
+ config: MultiAccountConfig,
925
+ ): Promise<UsageFetchResult> {
926
+ return this.#fetchAccount(account, config, false);
927
+ }
928
+
929
+ /**
930
+ * Refreshes usage after the provider refused a request for quota reasons.
931
+ *
932
+ * Unlike the cadence poll this bypasses the fresh-header guard, because the
933
+ * header that looked fresh is exactly the reading the 429 just contradicted.
934
+ * It reuses the existing `bypassHeaderFreshness` parameter rather than
935
+ * inventing a mechanism: `fetchAccounts` already passes `true` for window
936
+ * sampling.
937
+ *
938
+ * At most one extra poll per failure, in this process or across the fleet.
939
+ * `failedAtMs` is the failure's own time, not now: a poll that ran after the
940
+ * failure but before settlement has already answered this failure's question
941
+ * and must suppress.
942
+ */
943
+ async refreshAfterFailure(
944
+ account: UsageFetchAccount,
945
+ config: MultiAccountConfig,
946
+ failedAtMs: number,
947
+ ): Promise<UsageFetchResult> {
948
+ const key = this.#accountKey(account);
949
+ // Step 2: claim synchronously, before any await, so a second failure in
950
+ // the same tick joins this refresh rather than starting its own.
951
+ const inFlight = this.#inFlightFailureRefresh.get(key);
952
+ if (inFlight !== undefined) return inFlight;
953
+ const pending = this.#refreshAfterFailure(account, config, failedAtMs);
954
+ this.#inFlightFailureRefresh.set(key, pending);
955
+ try {
956
+ return await pending;
957
+ } finally {
958
+ // Release only our own token: a later refresh may already have
959
+ // replaced it, and deleting that one would reopen the window.
960
+ if (this.#inFlightFailureRefresh.get(key) === pending) {
961
+ this.#inFlightFailureRefresh.delete(key);
962
+ }
963
+ }
964
+ }
965
+
966
+ /**
967
+ * Durable suppression checks for a failure-triggered refresh.
968
+ *
969
+ * Run twice: once before taking the lease, and again under it. The second
970
+ * run is not redundant -- a peer can write an attempt between our first read
971
+ * and our lease acquisition, and without the recheck both processes would
972
+ * poll the same account for the same failure.
973
+ *
974
+ * Each clause suppresses for a different reason, and they must stay
975
+ * distinguishable:
976
+ *
977
+ * - endpoint backoff: an absolute deadline from a real failure ladder.
978
+ * `failureCount > 0` matters because a zero-failure record with a future
979
+ * `nextAttemptAtMs` is an ordinary cadence reservation, not a ladder.
980
+ * - already answered: any attempt at or after our failure has already asked
981
+ * the endpoint the question this failure raises.
982
+ * - debounce: a previous failure-triggered refresh set a deadline we are
983
+ * still inside.
984
+ */
985
+ #failureRefreshSuppressedBy(
986
+ attempt: SharedUsageAttemptRecord | undefined,
987
+ failedAtMs: number,
988
+ nowMs: number,
989
+ ): "backoff" | "already-answered" | "debounced" | undefined {
990
+ if (attempt === undefined) return undefined;
991
+ // An attempt anchored in the future cannot suppress anything.
992
+ //
993
+ // Round 2 found the first version of this bound was applied only inside
994
+ // the debounce clause, so `already-answered` and `backoff` returned
995
+ // before it ran: an attempt with `observedAtMs` centuries ahead is
996
+ // trivially `>= failedAtMs`, and suppressed every failure refresh
997
+ // forever. The bound belongs here, on the record, before any clause
998
+ // reads it.
999
+ //
1000
+ // `nextAttemptAtMs` is allowed one debounce interval of headroom because
1001
+ // Only the ORIGIN timestamps are bounded, not `nextAttemptAtMs`: a real
1002
+ // failure ladder legitimately schedules its next rung far ahead, up to
1003
+ // MAX_BACKOFF_MS, and bounding that would break genuine backoff
1004
+ // suppression. An honest record cannot have been OBSERVED in the future.
1005
+ if (
1006
+ attempt.observedAtMs > nowMs ||
1007
+ (attempt.failureTriggeredAtMs !== undefined &&
1008
+ attempt.failureTriggeredAtMs > nowMs)
1009
+ ) {
1010
+ return undefined;
1011
+ }
1012
+ if (attempt.failureCount > 0 && attempt.nextAttemptAtMs > nowMs) {
1013
+ return "backoff";
1014
+ }
1015
+ if (attempt.observedAtMs >= failedAtMs) return "already-answered";
1016
+ if (
1017
+ attempt.refreshDebounceUntilMs !== undefined &&
1018
+ attempt.refreshDebounceUntilMs > nowMs &&
1019
+ // Same far-future bypass as the hold: the stored span can be a
1020
+ // legitimate five minutes while its origin sits centuries ahead, which
1021
+ // would suppress every failure refresh forever. A deadline further
1022
+ // than one debounce interval from now cannot be honest.
1023
+ attempt.refreshDebounceUntilMs <= nowMs + USAGE_FETCH_INTERVAL_MS
1024
+ ) {
1025
+ return "debounced";
1026
+ }
1027
+ return undefined;
1028
+ }
1029
+
1030
+ async #refreshAfterFailure(
1031
+ account: UsageFetchAccount,
1032
+ config: MultiAccountConfig,
1033
+ failedAtMs: number,
1034
+ ): Promise<UsageFetchResult> {
1035
+ if (!isCanonicalManagedProviderId(account.providerId, account.family)) {
1036
+ return { providerId: account.providerId, status: "failed" };
1037
+ }
1038
+ // An api_key account has no OAuth usage endpoint; unmeasured, not failed.
1039
+ if (account.credentialType === "api_key") {
1040
+ return { providerId: account.providerId, status: "not-supported" };
1041
+ }
1042
+ if (!(config.usageFetchEnabled?.[account.family] ?? true)) {
1043
+ return { providerId: account.providerId, status: "disabled-by-config" };
1044
+ }
1045
+
1046
+ // Step 4/5: durable checks before the lease.
1047
+ const beforeLease = this.#sharedStore.latestAttempt(
1048
+ account.providerId,
1049
+ account.family,
1050
+ );
1051
+ if (
1052
+ this.#failureRefreshSuppressedBy(
1053
+ beforeLease,
1054
+ failedAtMs,
1055
+ this.#now(),
1056
+ ) !== undefined
1057
+ ) {
1058
+ return { providerId: account.providerId, status: "not-due" };
1059
+ }
1060
+
1061
+ // Step 6: the lease. Failure means another usage operation is running,
1062
+ // which is itself a reason to suppress rather than queue behind it.
1063
+ const lease = acquireMachineLease({
1064
+ lockPath: this.#lockPath,
1065
+ ttlMs: Math.max(2_000, this.#timeoutMs + 2_000),
1066
+ now: this.#now,
1067
+ reclaimMalformed: true,
1068
+ });
1069
+ if (!lease) {
1070
+ return { providerId: account.providerId, status: "lease-unavailable" };
1071
+ }
1072
+ const leaseGuard: AntigravityLeaseGuard = { lease, handedOff: false };
1073
+ try {
1074
+ // Step 7: the same checks again, now that nobody else can write.
1075
+ const underLease = this.#sharedStore.latestAttempt(
1076
+ account.providerId,
1077
+ account.family,
1078
+ );
1079
+ if (
1080
+ this.#failureRefreshSuppressedBy(
1081
+ underLease,
1082
+ failedAtMs,
1083
+ this.#now(),
1084
+ ) !== undefined
1085
+ ) {
1086
+ return { providerId: account.providerId, status: "not-due" };
1087
+ }
1088
+
1089
+ // Step 8: persist the marked reservation BEFORE resolving the
1090
+ // credential. If the ledger cannot be written, make no endpoint call:
1091
+ // an unbounded call with no durable backoff state is how a stampede
1092
+ // starts.
1093
+ const attemptedAtMs = this.#now();
1094
+ const reserved = this.#sharedStore.append({
1095
+ recordType: "usage-attempt",
1096
+ tokens: null,
1097
+ providerId: account.providerId,
1098
+ family: account.family,
1099
+ observedAtMs: attemptedAtMs,
1100
+ observerId: this.#sharedStore.observerId,
1101
+ failureCount: underLease?.failureCount ?? 0,
1102
+ nextAttemptAtMs: attemptedAtMs + USAGE_FETCH_INTERVAL_MS,
1103
+ disabled: underLease?.disabled ?? false,
1104
+ ...(underLease?.failureReason === undefined
1105
+ ? {}
1106
+ : { failureReason: underLease.failureReason }),
1107
+ ...(underLease?.failureDetail === undefined
1108
+ ? {}
1109
+ : { failureDetail: underLease.failureDetail }),
1110
+ failureTriggeredAtMs: failedAtMs,
1111
+ refreshDebounceUntilMs: failedAtMs + USAGE_FETCH_INTERVAL_MS,
1112
+ });
1113
+ if (!reserved) {
1114
+ return { providerId: account.providerId, status: "failed" };
1115
+ }
1116
+
1117
+ // Step 9: the ordinary ladder. A failed refresh is not a no-op -- it
1118
+ // advances failureCount and can disable the account, exactly as a
1119
+ // cadence poll would.
1120
+ // `underLease` must be passed: the catch computes the next rung as
1121
+ // `prior.failureCount + 1`, so omitting it restarts an existing ladder
1122
+ // at 1 and can clear a `disabled` flag. A dead endpoint would then be
1123
+ // retried harder after every failure and might never reach the disable
1124
+ // threshold.
1125
+ return await this.#completeAttempt(
1126
+ account,
1127
+ attemptedAtMs,
1128
+ leaseGuard,
1129
+ {
1130
+ failureTriggeredAtMs: failedAtMs,
1131
+ refreshDebounceUntilMs: failedAtMs + USAGE_FETCH_INTERVAL_MS,
1132
+ },
1133
+ underLease,
1134
+ );
1135
+ } finally {
1136
+ if (!leaseGuard.handedOff) lease.release();
1137
+ }
1138
+ }
1139
+
1140
+ async #fetchAccount(
1141
+ account: UsageFetchAccount,
1142
+ config: MultiAccountConfig,
1143
+ bypassHeaderFreshness: boolean,
1144
+ ): Promise<UsageFetchResult> {
1145
+ if (!isCanonicalManagedProviderId(account.providerId, account.family)) {
1146
+ return { providerId: account.providerId, status: "failed" };
1147
+ }
1148
+ // #24: an api_key account has no OAuth usage endpoint. Return an
1149
+ // unmeasured outcome BEFORE any config gate, header check, backoff, lease,
1150
+ // or network call, so it never accrues a failure, never advances the
1151
+ // ladder, and is never disabled. Missing usage stays a coverage gap.
1152
+ if (account.credentialType === "api_key") {
1153
+ return { providerId: account.providerId, status: "not-supported" };
1154
+ }
1155
+ if (!(config.usageFetchEnabled?.[account.family] ?? true)) {
1156
+ return { providerId: account.providerId, status: "disabled-by-config" };
1157
+ }
1158
+ const nowMs = this.#now();
1159
+ if (
1160
+ !bypassHeaderFreshness &&
1161
+ this.#usage.hasFreshHeaderObservation(
1162
+ account.providerId,
1163
+ account.family,
1164
+ nowMs,
1165
+ )
1166
+ ) {
1167
+ return { providerId: account.providerId, status: "not-due" };
1168
+ }
1169
+ const prior = this.#sharedStore.latestAttempt(
1170
+ account.providerId,
1171
+ account.family,
1172
+ );
1173
+ // Backoff is checked BEFORE `disabled`, deliberately.
1174
+ //
1175
+ // Every failure writes a correctly-computed `nextAttemptAtMs`, but the
1176
+ // disabled gate used to come first and returned unconditionally, so that
1177
+ // timestamp was never read once an account had been disabled. `disabled`
1178
+ // therefore meant "forever": nothing anywhere sets it false or resets
1179
+ // `failureCount`.
1180
+ //
1181
+ // The consequences invert the intent. The reasons that disable a fetcher --
1182
+ // rate-limit above all -- are TRANSIENT, and the poller is the only
1183
+ // quota-free way to observe that the account recovered. So a briefly
1184
+ // throttled account was cut off permanently from the mechanism that would
1185
+ // have shown it was healthy again. Observed live: anthropic-account-2
1186
+ // exhausted its ladder on 27 July with `failureReason: "rate-limit"`, and
1187
+ // its `nextAttemptAtMs` came due seven days ago while the account sat
1188
+ // usable and invisible.
1189
+ //
1190
+ // Once the recorded backoff has elapsed, the account is retried whether or
1191
+ // not it was disabled. A retry that fails again simply re-arms the ladder at
1192
+ // its capped rung, so a genuinely dead endpoint is polled at most once per
1193
+ // MAX_BACKOFF_MS rather than hammered.
1194
+ if (prior !== undefined && prior.nextAttemptAtMs > nowMs) {
1195
+ await this.#persistWindowGap(
1196
+ account,
1197
+ nowMs,
1198
+ prior.disabled ? "auto-disabled" : "backoff",
1199
+ );
1200
+ return {
1201
+ providerId: account.providerId,
1202
+ status: prior.disabled ? "disabled" : "not-due",
1203
+ };
1204
+ }
1205
+ const lease = acquireMachineLease({
1206
+ lockPath: this.#lockPath,
1207
+ ttlMs: Math.max(2_000, this.#timeoutMs + 2_000),
1208
+ now: this.#now,
1209
+ // A wedged usage-fetch.lock would otherwise suppress every poll.
1210
+ reclaimMalformed: true,
1211
+ });
1212
+ if (!lease) {
1213
+ await this.#persistWindowGap(account, nowMs, "lease-unavailable");
1214
+ return { providerId: account.providerId, status: "lease-unavailable" };
1215
+ }
1216
+ const leaseGuard: AntigravityLeaseGuard = { lease, handedOff: false };
1217
+ try {
1218
+ const current = this.#sharedStore.latestAttempt(
1219
+ account.providerId,
1220
+ account.family,
1221
+ );
1222
+ // Same ordering as the pre-lease check above: an elapsed backoff wins
1223
+ // over a stale `disabled` flag, so a recovered account can be observed
1224
+ // again. Re-read under the lease because a peer may have attempted in
1225
+ // between.
1226
+ if (current !== undefined && current.nextAttemptAtMs > this.#now()) {
1227
+ await this.#persistWindowGap(
1228
+ account,
1229
+ this.#now(),
1230
+ current.disabled ? "auto-disabled" : "backoff",
1231
+ );
1232
+ return {
1233
+ providerId: account.providerId,
1234
+ status: current.disabled ? "disabled" : "not-due",
1235
+ };
1236
+ }
1237
+ const attemptedAtMs = this.#now();
1238
+ const reserved = this.#sharedStore.append({
1239
+ recordType: "usage-attempt",
1240
+ tokens: null,
1241
+ providerId: account.providerId,
1242
+ family: account.family,
1243
+ observedAtMs: attemptedAtMs,
1244
+ observerId: this.#sharedStore.observerId,
1245
+ failureCount: current?.failureCount ?? 0,
1246
+ nextAttemptAtMs: attemptedAtMs + USAGE_FETCH_INTERVAL_MS,
1247
+ disabled: current?.disabled ?? false,
1248
+ ...(current?.failureReason === undefined
1249
+ ? {}
1250
+ : { failureReason: current.failureReason }),
1251
+ ...(current?.failureDetail === undefined
1252
+ ? {}
1253
+ : { failureDetail: current.failureDetail }),
1254
+ });
1255
+ if (!reserved) {
1256
+ // The attempt ledger is the fleet-wide stampede barrier. If it
1257
+ // cannot be persisted, fail closed rather than making an
1258
+ // unbounded endpoint call with no durable backoff state.
1259
+ await this.#persistWindowGap(account, attemptedAtMs, "attempt-persistence");
1260
+ return { providerId: account.providerId, status: "failed" };
1261
+ }
1262
+ return await this.#completeAttempt(
1263
+ account,
1264
+ attemptedAtMs,
1265
+ leaseGuard,
1266
+ {},
1267
+ current,
1268
+ );
1269
+ } finally {
1270
+ if (!leaseGuard.handedOff) lease.release();
1271
+ }
1272
+ }
1273
+
1274
+ /**
1275
+ * Reuse (or start) the one real, in-flight raw fetch for this account,
1276
+ * shared process-wide. See `#inFlightAntigravityRawFetch` for why this
1277
+ * exists and its exact limits -- in particular, this deliberately never
1278
+ * compares `apiKey` against a stored value: doing so would mean retaining
1279
+ * or fingerprinting the raw credential, which this package's
1280
+ * credential-retention discipline forbids. `apiKey` is used only to start
1281
+ * a genuinely new call -- with a fresh `AbortController` -- when no pending
1282
+ * one exists for this account.
1283
+ */
1284
+ #antigravityRawFetch(
1285
+ account: UsageFetchAccount,
1286
+ apiKey: string,
1287
+ ): { readonly promise: Promise<unknown>; readonly controller: AbortController } {
1288
+ const key = this.#accountKey(account);
1289
+ const existing = UsageFetcher.#inFlightAntigravityRawFetch.get(key);
1290
+ if (existing !== undefined) {
1291
+ return existing;
1292
+ }
1293
+ const controller = new AbortController();
1294
+ const promise = this.#fetchAntigravityUsage(apiKey, {
1295
+ signal: controller.signal,
1296
+ });
1297
+ const entry = { promise, controller };
1298
+ UsageFetcher.#inFlightAntigravityRawFetch.set(key, entry);
1299
+ const clearOnSettle = () => {
1300
+ // Only clear our own entry: a settlement racing a fresh start (after
1301
+ // an earlier settlement already cleared and replaced this one) must
1302
+ // not delete the newer entry.
1303
+ if (UsageFetcher.#inFlightAntigravityRawFetch.get(key) === entry) {
1304
+ UsageFetcher.#inFlightAntigravityRawFetch.delete(key);
1305
+ }
1306
+ };
1307
+ promise.then(clearOnSettle, clearOnSettle);
1308
+ return entry;
1309
+ }
1310
+
1311
+ /**
1312
+ * Antigravity has no raw-HTTP usage endpoint of its own: the reviewed
1313
+ * `pi-antigravity` primitive makes several internal calls and resolves one
1314
+ * decoded `AccountUsage` object. This wraps that call with the same bounded
1315
+ * per-attempt deadline every other family's usage fetch gets, then projects
1316
+ * the result through `projectAntigravityUsage` before anything is recorded.
1317
+ */
1318
+ async #queryAntigravityUsage(
1319
+ account: UsageFetchAccount,
1320
+ apiKey: string,
1321
+ leaseGuard: AntigravityLeaseGuard,
1322
+ ): Promise<UsageReading> {
1323
+ let raw: unknown;
1324
+ try {
1325
+ raw = await this.#fetchAntigravityUsageWithDeadline(
1326
+ account,
1327
+ apiKey,
1328
+ leaseGuard,
1329
+ );
1330
+ } catch (error) {
1331
+ if (error instanceof UsageEndpointError) throw error;
1332
+ throw new UsageEndpointError(
1333
+ redactErrorBody(error instanceof Error ? error.message : String(error)),
1334
+ "network-error",
1335
+ );
1336
+ }
1337
+ const projection = projectAntigravityUsage(raw);
1338
+ if (projection === undefined) {
1339
+ throw new UsageEndpointError(
1340
+ "Antigravity usage endpoint response was not a bounded object",
1341
+ "malformed-response",
1342
+ undefined,
1343
+ undefined,
1344
+ "not-object",
1345
+ );
1346
+ }
1347
+ const reading = primaryReading(
1348
+ projection.windows.filter((window) => window.usable),
1349
+ );
1350
+ if (reading === undefined) {
1351
+ const usage = asObject(raw);
1352
+ throw new UsageEndpointError(
1353
+ "Antigravity usage endpoint returned no bounded quota bucket",
1354
+ "malformed-response",
1355
+ undefined,
1356
+ undefined,
1357
+ typeof usage?.quotaSummaryError === "string"
1358
+ ? "quota-summary-error"
1359
+ : "no-quota-groups",
1360
+ );
1361
+ }
1362
+ return {
1363
+ ...reading,
1364
+ ...(projection.projectDigest === undefined
1365
+ ? {}
1366
+ : { projectDigest: projection.projectDigest }),
1367
+ };
1368
+ }
1369
+
1370
+ /**
1371
+ * Races the shared in-flight raw fetch against this attempt's bounded
1372
+ * deadline. Losing that race calls `entry.controller.abort()`, so the abort
1373
+ * this repository issues is exactly the abort the fork's own three legs
1374
+ * observe -- but this process still awaits that call's OWN settlement, not
1375
+ * merely the deadline, before releasing the machine-shared lease: hitting
1376
+ * the deadline hands lease ownership to `leaseGuard`, which
1377
+ * `#drainAntigravityLease` renews at the lease's own `renewalIntervalMs`
1378
+ * until the real promise actually settles, then releases. The caller's own
1379
+ * `finally { if (!leaseGuard.handedOff) lease.release(); }` becomes a no-op
1380
+ * once handed off, so the lease is never released early. A late settlement
1381
+ * -- success or failure -- reaches only the drain continuation from that
1382
+ * point on: this function has already rejected the deadline's caller, and
1383
+ * nothing here writes usage, cost, or history from that late value.
1384
+ */
1385
+ async #fetchAntigravityUsageWithDeadline(
1386
+ account: UsageFetchAccount,
1387
+ apiKey: string,
1388
+ leaseGuard: AntigravityLeaseGuard,
1389
+ ): Promise<unknown> {
1390
+ const entry = this.#antigravityRawFetch(account, apiKey);
1391
+ let timedOut = false;
1392
+ let timer: ReturnType<typeof setTimeout> | undefined;
1393
+ const deadline = new Promise<never>((_resolve, reject) => {
1394
+ timer = setTimeout(() => {
1395
+ timedOut = true;
1396
+ entry.controller.abort();
1397
+ reject(
1398
+ new UsageEndpointError(
1399
+ `Antigravity usage fetch timed out after ${Math.round(this.#timeoutMs / 1_000)}s`,
1400
+ "network-error",
1401
+ ),
1402
+ );
1403
+ }, this.#timeoutMs);
1404
+ });
1405
+ // Attach a no-op rejection handler so a slow transport that rejects AFTER
1406
+ // the deadline has already won the race cannot surface as an unhandled
1407
+ // rejection; the caller only ever observes whichever settles first.
1408
+ // `#antigravityRawFetch` already attached its own settle handler too, so
1409
+ // this is belt-and-suspenders, not the only protection.
1410
+ entry.promise.catch(() => {});
1411
+ try {
1412
+ return await Promise.race([entry.promise, deadline]);
1413
+ } finally {
1414
+ if (timer !== undefined) clearTimeout(timer);
1415
+ if (timedOut) {
1416
+ leaseGuard.handedOff = true;
1417
+ this.#drainAntigravityLease(leaseGuard.lease, entry.promise);
1418
+ }
1419
+ }
1420
+ }
1421
+
1422
+ /**
1423
+ * Holds the machine-shared usage lease open until `pending` -- the real,
1424
+ * still-outstanding `fetchAccountUsage` call a bounded deadline already gave
1425
+ * up on -- actually settles. This reuses the lease's existing renewal
1426
+ * mechanism (`renew()` / `renewalIntervalMs`) rather than inventing a
1427
+ * second one: one drain-scoped interval, cleared the moment the real call
1428
+ * is done, not a standing per-agent timer.
1429
+ *
1430
+ * The first renewal happens synchronously, right here, before the interval
1431
+ * is ever scheduled. `setInterval`'s own first tick fires only after a
1432
+ * full `renewalIntervalMs`, and at the production defaults
1433
+ * (`USAGE_FETCH_TIMEOUT_MS=10_000` -> `ttlMs=12_000` ->
1434
+ * `renewalIntervalMs=4_000`) that first tick would land at
1435
+ * deadline+4_000ms -- two full seconds after the lease already expired on
1436
+ * disk at deadline+2_000ms. The lease's own ttl leaves exactly that
1437
+ * `ttlMs - timeoutMs` buffer past the deadline for this handoff to land
1438
+ * one immediate renewal inside, so calling `renew()` now (still inside
1439
+ * that buffer) closes the gap before the interval ever needs to run.
1440
+ *
1441
+ * `renew()`'s boolean result is never discarded: a `false` here or on a
1442
+ * later tick means the lease could not be kept -- already expired, or
1443
+ * reclaimed by a peer -- and this drain has nothing left to renew, so
1444
+ * renewal stops rather than continuing to poll a lease it no longer
1445
+ * safely holds. `pending` is still awaited either way, so a late
1446
+ * settlement never becomes an unhandled rejection, and `lease.release()`
1447
+ * still runs at the end either way: `release()` itself refuses to touch a
1448
+ * record whose token no longer matches the one this handle acquired, so
1449
+ * it can never delete a peer's lease even if renewal was lost earlier.
1450
+ */
1451
+ #drainAntigravityLease(
1452
+ lease: MachineLeaseHandle,
1453
+ pending: Promise<unknown>,
1454
+ ): void {
1455
+ let interval: ReturnType<typeof setInterval> | undefined;
1456
+ const stopRenewing = (): void => {
1457
+ if (interval !== undefined) {
1458
+ clearInterval(interval);
1459
+ interval = undefined;
1460
+ }
1461
+ };
1462
+ if (lease.renew()) {
1463
+ interval = setInterval(() => {
1464
+ if (!lease.renew()) stopRenewing();
1465
+ }, lease.renewalIntervalMs);
1466
+ }
1467
+ void pending
1468
+ .catch(() => {
1469
+ // A late failure from the abandoned call is observed only to know
1470
+ // the lease can now be released -- never written to usage, cost,
1471
+ // or history.
1472
+ })
1473
+ .finally(() => {
1474
+ stopRenewing();
1475
+ lease.release();
1476
+ });
1477
+ }
1478
+
1479
+ /**
1480
+ * The shared post-reservation ladder: call the endpoint, record the reading,
1481
+ * and on failure advance failureCount/backoff and possibly disable.
1482
+ *
1483
+ * Extracted so a failure-triggered refresh completes through exactly the same
1484
+ * path as a cadence poll. A second copy would be free to drift, and the one
1485
+ * that drifted would be the one nobody tests.
1486
+ *
1487
+ * `markers` is carried onto both outcome records, so the debounce deadline
1488
+ * survives the attempt completing. Without that a later failure would see an
1489
+ * unmarked record and poll again immediately.
1490
+ */
1491
+ async #completeAttempt(
1492
+ account: UsageFetchAccount,
1493
+ attemptedAtMs: number,
1494
+ leaseGuard: AntigravityLeaseGuard,
1495
+ markers: {
1496
+ readonly failureTriggeredAtMs?: number;
1497
+ readonly refreshDebounceUntilMs?: number;
1498
+ } = {},
1499
+ prior?: SharedUsageAttemptRecord,
1500
+ ): Promise<UsageFetchResult> {
1501
+ try {
1502
+ const credential = await this.#resolveCredential(account.providerId);
1503
+ if (typeof credential !== "string" || credential.length === 0) {
1504
+ throw new UsageEndpointError(
1505
+ "credential unavailable",
1506
+ "credential-unavailable",
1507
+ );
1508
+ }
1509
+ const reading =
1510
+ account.family === "google-antigravity"
1511
+ ? await this.#queryAntigravityUsage(account, credential, leaseGuard)
1512
+ : await queryEndpoint(
1513
+ account,
1514
+ credential,
1515
+ this.#fetchImpl,
1516
+ this.#timeoutMs,
1517
+ );
1518
+ const capturedAtMs = this.#now();
1519
+ this.#usage.record({
1520
+ providerId: account.providerId,
1521
+ family: account.family,
1522
+ observedAtMs: capturedAtMs,
1523
+ rateLimit: {
1524
+ ...(reading.recoveryAtMs === undefined
1525
+ ? {}
1526
+ : { recoveryAtMs: reading.recoveryAtMs }),
1527
+ utilization: reading.utilization,
1528
+ utilizationSource: "usage-endpoint",
1529
+ },
1530
+ });
1531
+ this.#notifyUsageRecorded(account.providerId);
1532
+ await this.#persistWindowSamples(account, reading, capturedAtMs);
1533
+ this.#sharedStore.append({
1534
+ recordType: "usage-attempt",
1535
+ tokens: null,
1536
+ providerId: account.providerId,
1537
+ family: account.family,
1538
+ observedAtMs: capturedAtMs,
1539
+ observerId: this.#sharedStore.observerId,
1540
+ failureCount: 0,
1541
+ nextAttemptAtMs: capturedAtMs + USAGE_FETCH_INTERVAL_MS,
1542
+ disabled: false,
1543
+ ...markers,
1544
+ });
1545
+ return {
1546
+ providerId: account.providerId,
1547
+ status: "fetched",
1548
+ utilization: reading.utilization,
1549
+ capturedAtMs,
1550
+ };
1551
+ } catch (error) {
1552
+ const failure =
1553
+ error instanceof UsageEndpointError
1554
+ ? error
1555
+ : new UsageEndpointError("usage endpoint failed", "network-error");
1556
+ const failureCount = (prior?.failureCount ?? 0) + 1;
1557
+ const backoff = Math.min(
1558
+ MAX_BACKOFF_MS,
1559
+ BASE_BACKOFF_MS * 2 ** Math.max(0, failureCount - 1),
1560
+ );
1561
+ const disabled = failureCount >= USAGE_FETCH_DISABLE_AFTER_FAILURES;
1562
+ this.#sharedStore.append({
1563
+ recordType: "usage-attempt",
1564
+ tokens: null,
1565
+ providerId: account.providerId,
1566
+ family: account.family,
1567
+ observedAtMs: attemptedAtMs,
1568
+ observerId: this.#sharedStore.observerId,
1569
+ failureCount,
1570
+ nextAttemptAtMs: Math.max(
1571
+ attemptedAtMs + USAGE_FETCH_INTERVAL_MS,
1572
+ attemptedAtMs + backoff,
1573
+ attemptedAtMs + (failure.retryAfterMs ?? 0),
1574
+ ),
1575
+ disabled,
1576
+ failureReason: failureReason(failure),
1577
+ ...(failure.detail === undefined
1578
+ ? {}
1579
+ : { failureDetail: failure.detail }),
1580
+ ...markers,
1581
+ });
1582
+ await this.#persistWindowGap(
1583
+ account,
1584
+ attemptedAtMs,
1585
+ disabled ? "auto-disabled" : "backoff",
1586
+ );
1587
+ return {
1588
+ providerId: account.providerId,
1589
+ status:
1590
+ failure.kind === "credential-unavailable"
1591
+ ? "credential-unavailable"
1592
+ : "failed",
1593
+ };
1594
+ }
1595
+ }
1596
+
1597
+ async fetchAccounts(
1598
+ accounts: readonly UsageFetchAccount[],
1599
+ config: MultiAccountConfig,
1600
+ ): Promise<readonly UsageFetchResult[]> {
1601
+ const results: UsageFetchResult[] = [];
1602
+ for (const account of accounts) {
1603
+ const result = await this.fetchAccount(account, config);
1604
+ results.push(result);
1605
+
1606
+ // Window cadence is deliberately a call-driven comparison, not a timer.
1607
+ // A normal fetch above records its own window sample. When a fresh
1608
+ // header suppressed it, this second path bypasses ONLY that header
1609
+ // check; #fetchAccount still applies every durable/config/lease/network
1610
+ // guard in the same order.
1611
+ const last = this.#lastWindowFetchAtMs.get(this.#accountKey(account));
1612
+ if (
1613
+ last !== undefined &&
1614
+ this.#now() - last < WINDOW_SAMPLE_INTERVAL_MS
1615
+ ) {
1616
+ continue;
1617
+ }
1618
+ // REQ-WINDOW-CADENCE: enforce cross-process 15-minute floor.
1619
+ // Check durable attempt record to prevent peer-process fetch within the window.
1620
+ const prior = this.#sharedStore.latestAttempt(
1621
+ account.providerId,
1622
+ account.family,
1623
+ );
1624
+ if (
1625
+ prior !== undefined &&
1626
+ this.#now() - prior.observedAtMs < WINDOW_SAMPLE_INTERVAL_MS
1627
+ ) {
1628
+ continue;
1629
+ }
1630
+ await this.#fetchAccount(account, config, true);
1631
+ }
1632
+ return results;
1633
+ }
1634
+ }