@centerforagenticai/pi-multi-account 0.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/NOTICE +29 -0
- package/README.md +999 -0
- package/config/models/pi-multi-account.v1.json +32 -0
- package/config/subscription-plans.v1.json +122 -0
- package/package.json +76 -0
- package/packages/pi-anthropic-oauth/LICENSE +21 -0
- package/packages/pi-anthropic-oauth/package.json +54 -0
- package/packages/pi-anthropic-oauth/src/auth.ts +396 -0
- package/packages/pi-anthropic-oauth/src/context.ts +116 -0
- package/packages/pi-anthropic-oauth/src/convert.ts +303 -0
- package/packages/pi-anthropic-oauth/src/index.ts +37 -0
- package/packages/pi-anthropic-oauth/src/prompt.ts +137 -0
- package/packages/pi-anthropic-oauth/src/stream.ts +476 -0
- package/packages/pi-antigravity/LICENSE +21 -0
- package/packages/pi-antigravity/package.json +77 -0
- package/packages/pi-antigravity/src/auth/index.ts +14 -0
- package/packages/pi-antigravity/src/auth/oauth.ts +442 -0
- package/packages/pi-antigravity/src/client/client.ts +561 -0
- package/packages/pi-antigravity/src/client/index.ts +1 -0
- package/packages/pi-antigravity/src/context.ts +110 -0
- package/packages/pi-antigravity/src/diagnostics/diagnostics.ts +96 -0
- package/packages/pi-antigravity/src/diagnostics/index.ts +1 -0
- package/packages/pi-antigravity/src/image/image.ts +336 -0
- package/packages/pi-antigravity/src/image/index.ts +1 -0
- package/packages/pi-antigravity/src/index.ts +280 -0
- package/packages/pi-antigravity/src/models/discovery.ts +154 -0
- package/packages/pi-antigravity/src/models/grouping.ts +424 -0
- package/packages/pi-antigravity/src/models/index.ts +3 -0
- package/packages/pi-antigravity/src/models/models.ts +500 -0
- package/packages/pi-antigravity/src/stream/index.ts +1 -0
- package/packages/pi-antigravity/src/stream/stream.ts +1478 -0
- package/packages/pi-antigravity/src/types/enums.ts +42 -0
- package/packages/pi-antigravity/src/types/index.ts +2 -0
- package/packages/pi-antigravity/src/types/types.ts +292 -0
- package/packages/pi-antigravity/src/usage/index.ts +1 -0
- package/packages/pi-antigravity/src/usage/usage.ts +416 -0
- package/packages/pi-antigravity/src/utils/http.ts +91 -0
- package/packages/pi-antigravity/src/utils/index.ts +3 -0
- package/packages/pi-antigravity/src/utils/security.ts +73 -0
- package/packages/pi-antigravity/src/utils/util.ts +132 -0
- package/scripts/multi-account.mjs +44 -0
- package/src/account-labels.ts +223 -0
- package/src/account-plan-assignment.ts +340 -0
- package/src/account-rate-history.ts +372 -0
- package/src/anthropic-adaptive-stream.ts +531 -0
- package/src/anthropic-alias-stream.ts +140 -0
- package/src/anthropic-context-compat.ts +80 -0
- package/src/api-pricing.ts +579 -0
- package/src/bounded-file-lines.ts +97 -0
- package/src/catalog-rebinding.ts +177 -0
- package/src/catalog-registration-probe.ts +111 -0
- package/src/codex-adapter.ts +345 -0
- package/src/codex-model-defaults.ts +785 -0
- package/src/command-completions.ts +404 -0
- package/src/commands.ts +2000 -0
- package/src/compaction.ts +14 -0
- package/src/config.ts +1317 -0
- package/src/continuation.ts +569 -0
- package/src/cooldowns.ts +110 -0
- package/src/cost-digest-store.ts +332 -0
- package/src/cost-digest.ts +1044 -0
- package/src/cost-history.ts +251 -0
- package/src/cost-period-closer.ts +160 -0
- package/src/cost-report-json.ts +318 -0
- package/src/cost-report-reader.ts +368 -0
- package/src/cost-report-render.ts +207 -0
- package/src/cost-report.ts +1104 -0
- package/src/coverage-attestation.ts +397 -0
- package/src/credential-lifecycle.ts +169 -0
- package/src/credential-refresh.ts +248 -0
- package/src/declaration-notice-marker.ts +238 -0
- package/src/diagnostic-store.ts +276 -0
- package/src/diagnostics.ts +309 -0
- package/src/discovery.ts +471 -0
- package/src/duration.ts +13 -0
- package/src/error-classification.ts +256 -0
- package/src/fuzzy.ts +15 -0
- package/src/group-policy.ts +81 -0
- package/src/history-store.ts +897 -0
- package/src/index.ts +5572 -0
- package/src/lifecycle.ts +378 -0
- package/src/logical-dispatch.ts +279 -0
- package/src/logical-model-selector.ts +254 -0
- package/src/logical-model-switcher.ts +430 -0
- package/src/logical-provider-attribution.ts +544 -0
- package/src/logical-provider.ts +1237 -0
- package/src/logical-route-indicator.ts +215 -0
- package/src/machine-lease.ts +445 -0
- package/src/model-support.ts +66 -0
- package/src/models-declaration.ts +1091 -0
- package/src/openai-adapter.ts +117 -0
- package/src/openrouter-budget.ts +304 -0
- package/src/openrouter-fallback.ts +146 -0
- package/src/period-boundaries.ts +376 -0
- package/src/pi-anthropic-oauth.d.ts +6 -0
- package/src/preflight.ts +253 -0
- package/src/pricing-cache.ts +235 -0
- package/src/project-identity.ts +100 -0
- package/src/provider-registration.ts +942 -0
- package/src/rate-formula.ts +163 -0
- package/src/recovery-engine.ts +853 -0
- package/src/recovery-output.ts +837 -0
- package/src/recovery-plan.ts +239 -0
- package/src/report-range.ts +203 -0
- package/src/route-resolver.ts +789 -0
- package/src/routing-config-transaction.ts +232 -0
- package/src/routing.ts +1163 -0
- package/src/runtime-state.ts +630 -0
- package/src/session-account-groups.ts +284 -0
- package/src/session-restore.ts +287 -0
- package/src/shared-usage.ts +1392 -0
- package/src/standalone-cli.ts +720 -0
- package/src/status-view.ts +578 -0
- package/src/subscription-plan-catalog.ts +346 -0
- package/src/tier-model-resolver.ts +46 -0
- package/src/upstream-anthropic.ts +315 -0
- package/src/upstream-antigravity.ts +327 -0
- package/src/usage-fetch.ts +1634 -0
- package/src/usage.ts +1026 -0
- package/src/vendor.ts +87 -0
- package/src/warmer.ts +231 -0
- package/src/watchdog.ts +219 -0
- package/src/window-history.ts +270 -0
|
@@ -0,0 +1,1634 @@
|
|
|
1
|
+
import { createHash } from "node:crypto";
|
|
2
|
+
import type { AllowedFamily, MultiAccountConfig } from "./config.js";
|
|
3
|
+
import type { CredentialType } from "./discovery.js";
|
|
4
|
+
import {
|
|
5
|
+
acquireMachineLease,
|
|
6
|
+
type MachineLeaseHandle,
|
|
7
|
+
} from "./machine-lease.js";
|
|
8
|
+
import {
|
|
9
|
+
normalizeUsageEndpointPercent,
|
|
10
|
+
type SharedUsageAttemptRecord,
|
|
11
|
+
type SharedUsageStore,
|
|
12
|
+
type UsageFailureDetail,
|
|
13
|
+
} from "./shared-usage.js";
|
|
14
|
+
import type { UsageLedger } from "./usage.js";
|
|
15
|
+
import { isCanonicalManagedProviderId } from "./runtime-state.js";
|
|
16
|
+
import { loadUpstreamAntigravityPrimitives } from "./upstream-antigravity.js";
|
|
17
|
+
import {
|
|
18
|
+
markWindowUnusable,
|
|
19
|
+
remainingFractionFromUtilization,
|
|
20
|
+
writeHistoryWindowSample,
|
|
21
|
+
type WindowHistoryWriteOptions,
|
|
22
|
+
type WindowSample,
|
|
23
|
+
} from "./window-history.js";
|
|
24
|
+
|
|
25
|
+
export const USAGE_FETCH_INTERVAL_MS = 5 * 60_000;
|
|
26
|
+
export const WINDOW_SAMPLE_INTERVAL_MS = 15 * 60_000;
|
|
27
|
+
|
|
28
|
+
/**
|
|
29
|
+
* Sentinel window id marking a stretch of time the fetcher could not cover, so
|
|
30
|
+
* an unmeasured period is distinguishable from a genuinely idle one. Without it,
|
|
31
|
+
* "no data" and "no usage" render identically and a cost report claims $0.00 for
|
|
32
|
+
* a window it never saw.
|
|
33
|
+
*
|
|
34
|
+
* INTERNAL. Operator surfaces must suppress it: it names nothing an operator can
|
|
35
|
+
* act on, and the double underscores read as broken markup.
|
|
36
|
+
*/
|
|
37
|
+
export const GAP_WINDOW_ID = "__gap__";
|
|
38
|
+
export const USAGE_FETCH_TIMEOUT_MS = 10_000;
|
|
39
|
+
const MAX_RESPONSE_BYTES = 128 * 1024;
|
|
40
|
+
const MAX_ERROR_CHARS = 512;
|
|
41
|
+
const BASE_BACKOFF_MS = 30_000;
|
|
42
|
+
const MAX_BACKOFF_MS = 15 * 60_000;
|
|
43
|
+
const BACKOFF_LADDER_STEPS =
|
|
44
|
+
Math.ceil(Math.log2(MAX_BACKOFF_MS / BASE_BACKOFF_MS)) + 1;
|
|
45
|
+
/** Disable only after the capped rung has failed once more. */
|
|
46
|
+
export const USAGE_FETCH_DISABLE_AFTER_FAILURES = BACKOFF_LADDER_STEPS + 1;
|
|
47
|
+
|
|
48
|
+
const CODEX_USAGE_URL = "https://chatgpt.com/backend-api/wham/usage";
|
|
49
|
+
const ANTHROPIC_USAGE_URL = "https://api.anthropic.com/api/oauth/usage";
|
|
50
|
+
const ALLOWED_USAGE_URLS = new Set([CODEX_USAGE_URL, ANTHROPIC_USAGE_URL]);
|
|
51
|
+
|
|
52
|
+
const MAX_ANTIGRAVITY_PROJECT_ID_BYTES = 4_096;
|
|
53
|
+
const MAX_ANTIGRAVITY_WINDOWS = 64;
|
|
54
|
+
/**
|
|
55
|
+
* Hard bound on projection work for one quota path. Entries beyond the scan cap
|
|
56
|
+
* are ignored; real fork responses contain only tens of models.
|
|
57
|
+
*/
|
|
58
|
+
const MAX_ANTIGRAVITY_SCANNED_ENTRIES = 4_096;
|
|
59
|
+
const MAX_ANTIGRAVITY_WINDOW_ID_CHARS = 128;
|
|
60
|
+
const ANTIGRAVITY_PROJECT_DIGEST_PREFIX = "antigravity-project-";
|
|
61
|
+
const ANTIGRAVITY_PROJECT_DIGEST_PATTERN = new RegExp(
|
|
62
|
+
`^${ANTIGRAVITY_PROJECT_DIGEST_PREFIX}[0-9a-f]{24}$`,
|
|
63
|
+
);
|
|
64
|
+
|
|
65
|
+
/**
|
|
66
|
+
* Projects a raw Google Cloud project id (from `pi-antigravity`'s decoded
|
|
67
|
+
* `AccountUsage.projectId`) into the only representation that may reach a
|
|
68
|
+
* status view, diagnostic, usage record, cost record, or history sample. The
|
|
69
|
+
* digest mirrors `project-identity.ts#deriveProjectKey`'s established shape
|
|
70
|
+
* (sha256, hex, truncated, tagged prefix) so this repository has one bounded,
|
|
71
|
+
* non-identifying digest convention rather than two.
|
|
72
|
+
*/
|
|
73
|
+
export function deriveAntigravityProjectDigest(
|
|
74
|
+
rawProjectId: unknown,
|
|
75
|
+
): string | undefined {
|
|
76
|
+
if (typeof rawProjectId !== "string" || rawProjectId.length === 0) {
|
|
77
|
+
return undefined;
|
|
78
|
+
}
|
|
79
|
+
if (Buffer.byteLength(rawProjectId, "utf8") > MAX_ANTIGRAVITY_PROJECT_ID_BYTES) {
|
|
80
|
+
return undefined;
|
|
81
|
+
}
|
|
82
|
+
const digest = createHash("sha256").update(rawProjectId, "utf8").digest("hex");
|
|
83
|
+
return `${ANTIGRAVITY_PROJECT_DIGEST_PREFIX}${digest.slice(0, 24)}`;
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
export function isAntigravityProjectDigest(value: string): boolean {
|
|
87
|
+
return ANTIGRAVITY_PROJECT_DIGEST_PATTERN.test(value);
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
export interface UsageFetchAccount {
|
|
91
|
+
readonly providerId: string;
|
|
92
|
+
readonly family: AllowedFamily;
|
|
93
|
+
/**
|
|
94
|
+
* Discovered credential type. An `api_key` account yields the unmeasured
|
|
95
|
+
* `not-supported` outcome before any OAuth usage call (#24). Optional so
|
|
96
|
+
* existing call sites and fixtures stay valid; absent is treated as the
|
|
97
|
+
* established OAuth path.
|
|
98
|
+
*/
|
|
99
|
+
readonly credentialType?: CredentialType;
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
export interface UsageFetchStatus {
|
|
103
|
+
readonly enabled: boolean;
|
|
104
|
+
readonly disabled: boolean;
|
|
105
|
+
readonly failureCount: number;
|
|
106
|
+
readonly nextAttemptAtMs?: number;
|
|
107
|
+
readonly disabledReason?:
|
|
108
|
+
| "rate-limit"
|
|
109
|
+
| "server-error"
|
|
110
|
+
| "credential-unavailable"
|
|
111
|
+
| "malformed-response"
|
|
112
|
+
| "network-error";
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
export type UsageFetchResultStatus =
|
|
116
|
+
| "fetched"
|
|
117
|
+
| "disabled"
|
|
118
|
+
| "backoff"
|
|
119
|
+
| "not-due"
|
|
120
|
+
| "lease-unavailable"
|
|
121
|
+
| "disabled-by-config"
|
|
122
|
+
| "credential-unavailable"
|
|
123
|
+
| "failed"
|
|
124
|
+
/**
|
|
125
|
+
* The account's credential type has no supported usage endpoint (an
|
|
126
|
+
* `api_key` account, #24). Unmeasured, not a failure: no OAuth call is made,
|
|
127
|
+
* the failure ladder does not advance, and the account is not disabled.
|
|
128
|
+
*/
|
|
129
|
+
| "not-supported";
|
|
130
|
+
|
|
131
|
+
export interface UsageFetchResult {
|
|
132
|
+
readonly providerId: string;
|
|
133
|
+
readonly status: UsageFetchResultStatus;
|
|
134
|
+
readonly utilization?: number;
|
|
135
|
+
readonly capturedAtMs?: number;
|
|
136
|
+
}
|
|
137
|
+
|
|
138
|
+
interface UsageFetchResponse {
|
|
139
|
+
readonly status: number;
|
|
140
|
+
readonly headers: {
|
|
141
|
+
get(name: string): string | null;
|
|
142
|
+
};
|
|
143
|
+
text(): Promise<string>;
|
|
144
|
+
}
|
|
145
|
+
|
|
146
|
+
export type UsageFetchImplementation = (
|
|
147
|
+
input: string,
|
|
148
|
+
init: RequestInit,
|
|
149
|
+
) => Promise<UsageFetchResponse>;
|
|
150
|
+
|
|
151
|
+
export interface UsageWindowReading {
|
|
152
|
+
readonly windowId: string;
|
|
153
|
+
readonly utilization?: number;
|
|
154
|
+
readonly remainingFraction?: number;
|
|
155
|
+
readonly resetAtMs?: number;
|
|
156
|
+
readonly resetEpoch?: number;
|
|
157
|
+
readonly usable: boolean;
|
|
158
|
+
}
|
|
159
|
+
|
|
160
|
+
type UsageReading = {
|
|
161
|
+
readonly utilization: number;
|
|
162
|
+
readonly recoveryAtMs?: number;
|
|
163
|
+
readonly windows: readonly UsageWindowReading[];
|
|
164
|
+
/** Present only for a family whose usage endpoint names a scoped project. */
|
|
165
|
+
readonly projectDigest?: string;
|
|
166
|
+
};
|
|
167
|
+
|
|
168
|
+
function sanitizeFailureDetail(value: unknown): UsageFailureDetail | undefined {
|
|
169
|
+
switch (value) {
|
|
170
|
+
case "not-object":
|
|
171
|
+
case "no-quota-groups":
|
|
172
|
+
case "quota-summary-error":
|
|
173
|
+
return value;
|
|
174
|
+
default:
|
|
175
|
+
return undefined;
|
|
176
|
+
}
|
|
177
|
+
}
|
|
178
|
+
|
|
179
|
+
class UsageEndpointError extends Error {
|
|
180
|
+
readonly status: number | undefined;
|
|
181
|
+
readonly retryAfterMs: number | undefined;
|
|
182
|
+
readonly kind: UsageFetchStatus["disabledReason"];
|
|
183
|
+
readonly detail: UsageFailureDetail | undefined;
|
|
184
|
+
|
|
185
|
+
constructor(
|
|
186
|
+
message: string,
|
|
187
|
+
kind: UsageFetchStatus["disabledReason"],
|
|
188
|
+
status?: number,
|
|
189
|
+
retryAfterMs?: number,
|
|
190
|
+
detail?: string,
|
|
191
|
+
) {
|
|
192
|
+
super(message);
|
|
193
|
+
this.name = "UsageEndpointError";
|
|
194
|
+
this.kind = kind;
|
|
195
|
+
this.status = status;
|
|
196
|
+
this.retryAfterMs = retryAfterMs;
|
|
197
|
+
this.detail = sanitizeFailureDetail(detail);
|
|
198
|
+
}
|
|
199
|
+
}
|
|
200
|
+
|
|
201
|
+
function responseHeader(
|
|
202
|
+
response: UsageFetchResponse,
|
|
203
|
+
name: string,
|
|
204
|
+
): string | undefined {
|
|
205
|
+
try {
|
|
206
|
+
const value = response.headers.get(name);
|
|
207
|
+
return value === null ? undefined : value;
|
|
208
|
+
} catch {
|
|
209
|
+
return undefined;
|
|
210
|
+
}
|
|
211
|
+
}
|
|
212
|
+
|
|
213
|
+
function retryAfterMs(response: UsageFetchResponse): number | undefined {
|
|
214
|
+
const value = responseHeader(response, "retry-after");
|
|
215
|
+
if (value === undefined || !/^\d+(?:\.\d+)?$/.test(value.trim())) {
|
|
216
|
+
return undefined;
|
|
217
|
+
}
|
|
218
|
+
const seconds = Number(value);
|
|
219
|
+
if (!Number.isFinite(seconds) || seconds < 0) return undefined;
|
|
220
|
+
return Math.min(MAX_BACKOFF_MS, Math.max(0, Math.ceil(seconds * 1_000)));
|
|
221
|
+
}
|
|
222
|
+
|
|
223
|
+
/** Redacts bearer and access-token values before an error can escape this module. */
|
|
224
|
+
export function redactErrorBody(body: string): string {
|
|
225
|
+
const redacted = body
|
|
226
|
+
.replace(/Bearer\s+[A-Za-z0-9._~+/=-]+/gi, "Bearer <redacted>")
|
|
227
|
+
.replace(/("?access_token"?\s*[:=]\s*["']?)[^,\s"'}]+/gi, "$1<redacted>")
|
|
228
|
+
.trim();
|
|
229
|
+
return redacted.length <= MAX_ERROR_CHARS
|
|
230
|
+
? redacted
|
|
231
|
+
: `${redacted.slice(0, MAX_ERROR_CHARS - 1)}…`;
|
|
232
|
+
}
|
|
233
|
+
|
|
234
|
+
function asObject(value: unknown): Record<string, unknown> | undefined {
|
|
235
|
+
return typeof value === "object" && value !== null && !Array.isArray(value)
|
|
236
|
+
? (value as Record<string, unknown>)
|
|
237
|
+
: undefined;
|
|
238
|
+
}
|
|
239
|
+
|
|
240
|
+
function asNumber(value: unknown): number | undefined {
|
|
241
|
+
if (typeof value === "number" && Number.isFinite(value)) return value;
|
|
242
|
+
if (typeof value === "string" && value.trim() !== "") {
|
|
243
|
+
const parsed = Number(value);
|
|
244
|
+
return Number.isFinite(parsed) ? parsed : undefined;
|
|
245
|
+
}
|
|
246
|
+
return undefined;
|
|
247
|
+
}
|
|
248
|
+
|
|
249
|
+
function asResetMs(value: unknown): number | undefined {
|
|
250
|
+
const number = asNumber(value);
|
|
251
|
+
if (number !== undefined && number >= 0) {
|
|
252
|
+
return number < 100_000_000_000 ? number * 1_000 : number;
|
|
253
|
+
}
|
|
254
|
+
if (typeof value === "string") {
|
|
255
|
+
const parsed = Date.parse(value);
|
|
256
|
+
return Number.isFinite(parsed) && parsed >= 0 ? parsed : undefined;
|
|
257
|
+
}
|
|
258
|
+
return undefined;
|
|
259
|
+
}
|
|
260
|
+
|
|
261
|
+
function readingFromWindow(
|
|
262
|
+
value: unknown,
|
|
263
|
+
windowId: string,
|
|
264
|
+
): UsageWindowReading | undefined {
|
|
265
|
+
const window = asObject(value);
|
|
266
|
+
if (!window) return undefined;
|
|
267
|
+
const percent =
|
|
268
|
+
asNumber(window.used_percent) ??
|
|
269
|
+
asNumber(window.used_percentage) ??
|
|
270
|
+
asNumber(window.usedPercent) ??
|
|
271
|
+
asNumber(window.utilization);
|
|
272
|
+
const utilization =
|
|
273
|
+
percent === undefined ? undefined : normalizeUsageEndpointPercent(percent);
|
|
274
|
+
const resetAtMs =
|
|
275
|
+
asResetMs(window.reset_at) ??
|
|
276
|
+
asResetMs(window.resetAt) ??
|
|
277
|
+
asResetMs(window.resets_at) ??
|
|
278
|
+
asResetMs(window.resetsAt);
|
|
279
|
+
const remainingFraction =
|
|
280
|
+
utilization === undefined
|
|
281
|
+
? undefined
|
|
282
|
+
: remainingFractionFromUtilization(utilization);
|
|
283
|
+
return {
|
|
284
|
+
windowId,
|
|
285
|
+
...(utilization === undefined ? {} : { utilization }),
|
|
286
|
+
...(remainingFraction === undefined ? {} : { remainingFraction }),
|
|
287
|
+
...(resetAtMs === undefined ? {} : { resetAtMs, resetEpoch: resetAtMs }),
|
|
288
|
+
usable:
|
|
289
|
+
utilization !== undefined &&
|
|
290
|
+
remainingFraction !== undefined &&
|
|
291
|
+
resetAtMs !== undefined,
|
|
292
|
+
};
|
|
293
|
+
}
|
|
294
|
+
|
|
295
|
+
function windowEntries(
|
|
296
|
+
container: Record<string, unknown> | undefined,
|
|
297
|
+
knownIds: readonly string[],
|
|
298
|
+
): UsageWindowReading[] {
|
|
299
|
+
if (!container) return [];
|
|
300
|
+
const entries = new Map<string, UsageWindowReading>();
|
|
301
|
+
for (const key of knownIds) {
|
|
302
|
+
const reading = readingFromWindow(container[key], key);
|
|
303
|
+
if (reading) entries.set(key, reading);
|
|
304
|
+
}
|
|
305
|
+
for (const [key, value] of Object.entries(container)) {
|
|
306
|
+
if (entries.has(key)) continue;
|
|
307
|
+
const object = asObject(value);
|
|
308
|
+
if (
|
|
309
|
+
!object ||
|
|
310
|
+
![
|
|
311
|
+
"used_percent",
|
|
312
|
+
"used_percentage",
|
|
313
|
+
"usedPercent",
|
|
314
|
+
"utilization",
|
|
315
|
+
"reset_at",
|
|
316
|
+
"resetAt",
|
|
317
|
+
"resets_at",
|
|
318
|
+
"resetsAt",
|
|
319
|
+
].some((field) => field in object)
|
|
320
|
+
) {
|
|
321
|
+
continue;
|
|
322
|
+
}
|
|
323
|
+
const reading = readingFromWindow(object, key);
|
|
324
|
+
if (reading) entries.set(key, reading);
|
|
325
|
+
}
|
|
326
|
+
return [...entries.values()];
|
|
327
|
+
}
|
|
328
|
+
|
|
329
|
+
function primaryReading(
|
|
330
|
+
windows: readonly UsageWindowReading[],
|
|
331
|
+
): UsageReading | undefined {
|
|
332
|
+
const usable = windows.find((window) => window.utilization !== undefined);
|
|
333
|
+
if (!usable || usable.utilization === undefined) return undefined;
|
|
334
|
+
// `recoveryAtMs` means "this account is out and recovers at T", and
|
|
335
|
+
// `snapshotIndicatesExhaustion` treats any future `recoveryAtMs` as
|
|
336
|
+
// exhaustion. A window's `resetAtMs` is its ROUTINE reset time and is present
|
|
337
|
+
// whether or not the window has capacity, so carrying it unconditionally
|
|
338
|
+
// marked a healthy account (e.g. Codex primary_window at 17% used, weekly
|
|
339
|
+
// reset ~6 days out) exhausted and demoted `unified` to the owning-vendor API
|
|
340
|
+
// tier until that reset (#72). Only an actually-spent window contributes a
|
|
341
|
+
// recovery time; a window with capacity to spare recovers nothing.
|
|
342
|
+
const exhausted = usable.utilization >= 1 || usable.remainingFraction === 0;
|
|
343
|
+
return {
|
|
344
|
+
utilization: usable.utilization,
|
|
345
|
+
...(exhausted && usable.resetAtMs !== undefined
|
|
346
|
+
? { recoveryAtMs: usable.resetAtMs }
|
|
347
|
+
: {}),
|
|
348
|
+
windows,
|
|
349
|
+
};
|
|
350
|
+
}
|
|
351
|
+
|
|
352
|
+
/**
|
|
353
|
+
* One quota row from `pi-antigravity`'s decoded `AccountUsage`. The upstream
|
|
354
|
+
* shape is intentionally treated as unknown here (the pinned package owns
|
|
355
|
+
* usage decoding; this repository owns projection), so every field is
|
|
356
|
+
* defensively narrowed the same way `readingFromWindow` narrows the
|
|
357
|
+
* Codex/Anthropic raw JSON above -- never guessed, never passed through.
|
|
358
|
+
*/
|
|
359
|
+
function antigravityQuotaReading(
|
|
360
|
+
value: unknown,
|
|
361
|
+
idField: "bucketId" | "modelId",
|
|
362
|
+
): UsageWindowReading | undefined {
|
|
363
|
+
const quota = asObject(value);
|
|
364
|
+
if (!quota) return undefined;
|
|
365
|
+
const rawId = quota[idField];
|
|
366
|
+
const windowId =
|
|
367
|
+
typeof rawId === "string" &&
|
|
368
|
+
rawId.length > 0 &&
|
|
369
|
+
rawId.length <= MAX_ANTIGRAVITY_WINDOW_ID_CHARS
|
|
370
|
+
? rawId
|
|
371
|
+
: undefined;
|
|
372
|
+
if (windowId === undefined) return undefined;
|
|
373
|
+
const rawRemaining = asNumber(quota.remainingFraction);
|
|
374
|
+
const utilization =
|
|
375
|
+
rawRemaining === undefined
|
|
376
|
+
? undefined
|
|
377
|
+
: Math.max(0, Math.min(1, 1 - rawRemaining));
|
|
378
|
+
const remainingFraction =
|
|
379
|
+
utilization === undefined ? undefined : remainingFractionFromUtilization(utilization);
|
|
380
|
+
const resetAtMs = asResetMs(quota.resetTime);
|
|
381
|
+
return {
|
|
382
|
+
windowId,
|
|
383
|
+
...(utilization === undefined ? {} : { utilization }),
|
|
384
|
+
...(remainingFraction === undefined ? {} : { remainingFraction }),
|
|
385
|
+
...(resetAtMs === undefined ? {} : { resetAtMs, resetEpoch: resetAtMs }),
|
|
386
|
+
usable:
|
|
387
|
+
utilization !== undefined &&
|
|
388
|
+
remainingFraction !== undefined &&
|
|
389
|
+
resetAtMs !== undefined,
|
|
390
|
+
};
|
|
391
|
+
}
|
|
392
|
+
|
|
393
|
+
function boundedMostConstrainedAntigravityWindows(
|
|
394
|
+
values: Iterable<unknown>,
|
|
395
|
+
idField: "bucketId" | "modelId",
|
|
396
|
+
): UsageWindowReading[] {
|
|
397
|
+
const candidates: UsageWindowReading[] = [];
|
|
398
|
+
const seenWindowIds = new Set<string>();
|
|
399
|
+
let scannedEntries = 0;
|
|
400
|
+
for (const value of values) {
|
|
401
|
+
if (scannedEntries >= MAX_ANTIGRAVITY_SCANNED_ENTRIES) break;
|
|
402
|
+
scannedEntries += 1;
|
|
403
|
+
const reading = antigravityQuotaReading(value, idField);
|
|
404
|
+
if (reading === undefined || seenWindowIds.has(reading.windowId)) continue;
|
|
405
|
+
seenWindowIds.add(reading.windowId);
|
|
406
|
+
candidates.push(reading);
|
|
407
|
+
}
|
|
408
|
+
// Stable sort preserves the first-seen input order when utilization ties.
|
|
409
|
+
return candidates
|
|
410
|
+
.sort((left, right) => {
|
|
411
|
+
const leftUtilization = left.utilization ?? Number.NEGATIVE_INFINITY;
|
|
412
|
+
const rightUtilization = right.utilization ?? Number.NEGATIVE_INFINITY;
|
|
413
|
+
return leftUtilization === rightUtilization
|
|
414
|
+
? 0
|
|
415
|
+
: rightUtilization - leftUtilization;
|
|
416
|
+
})
|
|
417
|
+
.slice(0, MAX_ANTIGRAVITY_WINDOWS);
|
|
418
|
+
}
|
|
419
|
+
|
|
420
|
+
/**
|
|
421
|
+
* Group containers count toward the same scan budget as the buckets they hold,
|
|
422
|
+
* so an oversized response of empty groups cannot bypass the entry cap.
|
|
423
|
+
*/
|
|
424
|
+
function* antigravityGroupBuckets(groups: readonly unknown[]): Iterable<unknown> {
|
|
425
|
+
let visited = 0;
|
|
426
|
+
for (const group of groups) {
|
|
427
|
+
if (++visited > MAX_ANTIGRAVITY_SCANNED_ENTRIES) return;
|
|
428
|
+
const groupRecord = asObject(group);
|
|
429
|
+
const buckets =
|
|
430
|
+
groupRecord && Array.isArray(groupRecord.buckets) ? groupRecord.buckets : [];
|
|
431
|
+
for (const bucket of buckets) {
|
|
432
|
+
if (++visited > MAX_ANTIGRAVITY_SCANNED_ENTRIES) return;
|
|
433
|
+
yield bucket;
|
|
434
|
+
}
|
|
435
|
+
}
|
|
436
|
+
}
|
|
437
|
+
|
|
438
|
+
/**
|
|
439
|
+
* Projects the decoded `AccountUsage` result into exactly the named, bounded
|
|
440
|
+
* facts this repository retains: a small set of group- or model-quota windows
|
|
441
|
+
* and a digest of the account-scoped project id. Every other upstream field
|
|
442
|
+
* (plan labels, display names, tier names, raw endpoint, raw project id) is
|
|
443
|
+
* dropped here, at the seam, before any status, diagnostic, usage, cost, or
|
|
444
|
+
* history surface can see it.
|
|
445
|
+
*/
|
|
446
|
+
export function projectAntigravityUsage(
|
|
447
|
+
raw: unknown,
|
|
448
|
+
): { windows: UsageWindowReading[]; projectDigest?: string } | undefined {
|
|
449
|
+
const usage = asObject(raw);
|
|
450
|
+
if (!usage) return undefined;
|
|
451
|
+
const projectDigest = deriveAntigravityProjectDigest(usage.projectId);
|
|
452
|
+
const groups = Array.isArray(usage.groups) ? usage.groups : [];
|
|
453
|
+
const boundedWindows = boundedMostConstrainedAntigravityWindows(
|
|
454
|
+
antigravityGroupBuckets(groups),
|
|
455
|
+
"bucketId",
|
|
456
|
+
);
|
|
457
|
+
if (!boundedWindows.some((window) => window.usable)) {
|
|
458
|
+
const models = Array.isArray(usage.models) ? usage.models : [];
|
|
459
|
+
const boundedModelWindows = boundedMostConstrainedAntigravityWindows(
|
|
460
|
+
models,
|
|
461
|
+
"modelId",
|
|
462
|
+
);
|
|
463
|
+
if (boundedModelWindows.length > 0) {
|
|
464
|
+
return {
|
|
465
|
+
windows: boundedModelWindows,
|
|
466
|
+
...(projectDigest === undefined ? {} : { projectDigest }),
|
|
467
|
+
};
|
|
468
|
+
}
|
|
469
|
+
}
|
|
470
|
+
return {
|
|
471
|
+
windows: boundedWindows,
|
|
472
|
+
...(projectDigest === undefined ? {} : { projectDigest }),
|
|
473
|
+
};
|
|
474
|
+
}
|
|
475
|
+
|
|
476
|
+
/**
|
|
477
|
+
* Injectable Antigravity usage transport; defaults to the reviewed upstream
|
|
478
|
+
* primitive. The optional `signal` is forwarded to the fork's
|
|
479
|
+
* `fetchAccountUsage`, which propagates it into every one of its three
|
|
480
|
+
* internal legs (`loadCodeAssist`, quota summary, available-models catalog)
|
|
481
|
+
* and drains every settlement via `Promise.allSettled` before rejecting on
|
|
482
|
+
* abort -- see `pi-antigravity`'s `src/usage/usage.ts`. Aborting this signal
|
|
483
|
+
* is therefore the only way this repository can actually cancel the real
|
|
484
|
+
* upstream call rather than merely giving up on waiting for it.
|
|
485
|
+
*/
|
|
486
|
+
export type AntigravityUsageFetchImplementation = (
|
|
487
|
+
apiKey: string,
|
|
488
|
+
options?: { readonly signal?: AbortSignal },
|
|
489
|
+
) => Promise<unknown>;
|
|
490
|
+
|
|
491
|
+
async function defaultFetchAntigravityUsage(
|
|
492
|
+
apiKey: string,
|
|
493
|
+
options?: { readonly signal?: AbortSignal },
|
|
494
|
+
): Promise<unknown> {
|
|
495
|
+
if (process.env.VITEST === "true") {
|
|
496
|
+
throw new UsageEndpointError(
|
|
497
|
+
"network access is disabled in tests",
|
|
498
|
+
"network-error",
|
|
499
|
+
);
|
|
500
|
+
}
|
|
501
|
+
const primitives = await loadUpstreamAntigravityPrimitives();
|
|
502
|
+
return primitives.fetchUsage(
|
|
503
|
+
apiKey,
|
|
504
|
+
options?.signal === undefined ? {} : { signal: options.signal },
|
|
505
|
+
);
|
|
506
|
+
}
|
|
507
|
+
|
|
508
|
+
function parsePayload(text: string): Record<string, unknown> {
|
|
509
|
+
if (Buffer.byteLength(text, "utf8") > MAX_RESPONSE_BYTES) {
|
|
510
|
+
throw new UsageEndpointError(
|
|
511
|
+
"usage endpoint response exceeded the bounded response size",
|
|
512
|
+
"malformed-response",
|
|
513
|
+
);
|
|
514
|
+
}
|
|
515
|
+
let payload: unknown;
|
|
516
|
+
try {
|
|
517
|
+
payload = JSON.parse(text);
|
|
518
|
+
} catch {
|
|
519
|
+
throw new UsageEndpointError(
|
|
520
|
+
"usage endpoint response was not valid JSON",
|
|
521
|
+
"malformed-response",
|
|
522
|
+
);
|
|
523
|
+
}
|
|
524
|
+
const object = asObject(payload);
|
|
525
|
+
if (!object) {
|
|
526
|
+
throw new UsageEndpointError(
|
|
527
|
+
"usage endpoint response was not an object",
|
|
528
|
+
"malformed-response",
|
|
529
|
+
);
|
|
530
|
+
}
|
|
531
|
+
return object;
|
|
532
|
+
}
|
|
533
|
+
|
|
534
|
+
export function normalizeCodexUsagePayload(
|
|
535
|
+
payload: Record<string, unknown>,
|
|
536
|
+
): UsageReading {
|
|
537
|
+
const rateLimit = asObject(payload.rate_limit);
|
|
538
|
+
const windows = windowEntries(rateLimit, [
|
|
539
|
+
"primary_window",
|
|
540
|
+
"secondary_window",
|
|
541
|
+
]);
|
|
542
|
+
const topLevelWindows = windowEntries(payload, [
|
|
543
|
+
"primary_window",
|
|
544
|
+
"secondary_window",
|
|
545
|
+
]);
|
|
546
|
+
const byId = new Map(windows.map((window) => [window.windowId, window]));
|
|
547
|
+
for (const window of topLevelWindows) {
|
|
548
|
+
if (!byId.has(window.windowId)) byId.set(window.windowId, window);
|
|
549
|
+
}
|
|
550
|
+
const reading = primaryReading([...byId.values()]);
|
|
551
|
+
if (reading) return reading;
|
|
552
|
+
throw new UsageEndpointError(
|
|
553
|
+
"Codex usage endpoint returned no bounded rate-limit window",
|
|
554
|
+
"malformed-response",
|
|
555
|
+
);
|
|
556
|
+
}
|
|
557
|
+
|
|
558
|
+
export function normalizeAnthropicUsagePayload(
|
|
559
|
+
payload: Record<string, unknown>,
|
|
560
|
+
): UsageReading {
|
|
561
|
+
const windows = windowEntries(payload, [
|
|
562
|
+
"five_hour",
|
|
563
|
+
"fiveHour",
|
|
564
|
+
"primary",
|
|
565
|
+
"weekly",
|
|
566
|
+
"seven_day",
|
|
567
|
+
"sevenDay",
|
|
568
|
+
"seven_day_opus",
|
|
569
|
+
]);
|
|
570
|
+
const reading = primaryReading(windows);
|
|
571
|
+
if (reading) return reading;
|
|
572
|
+
throw new UsageEndpointError(
|
|
573
|
+
"Anthropic usage endpoint returned no bounded rate-limit window",
|
|
574
|
+
"malformed-response",
|
|
575
|
+
);
|
|
576
|
+
}
|
|
577
|
+
|
|
578
|
+
async function fetchWithTimeout(
|
|
579
|
+
fetchImpl: UsageFetchImplementation,
|
|
580
|
+
url: string,
|
|
581
|
+
init: RequestInit,
|
|
582
|
+
timeoutMs: number,
|
|
583
|
+
): Promise<UsageFetchResponse> {
|
|
584
|
+
const controller = new AbortController();
|
|
585
|
+
const timeout = setTimeout(() => controller.abort(), timeoutMs);
|
|
586
|
+
try {
|
|
587
|
+
return await fetchImpl(url, { ...init, signal: controller.signal });
|
|
588
|
+
} catch (error) {
|
|
589
|
+
if (controller.signal.aborted) {
|
|
590
|
+
throw new UsageEndpointError(
|
|
591
|
+
`usage endpoint timed out after ${Math.round(timeoutMs / 1_000)}s`,
|
|
592
|
+
"network-error",
|
|
593
|
+
);
|
|
594
|
+
}
|
|
595
|
+
throw new UsageEndpointError(
|
|
596
|
+
redactErrorBody(error instanceof Error ? error.message : String(error)),
|
|
597
|
+
"network-error",
|
|
598
|
+
);
|
|
599
|
+
} finally {
|
|
600
|
+
clearTimeout(timeout);
|
|
601
|
+
}
|
|
602
|
+
}
|
|
603
|
+
|
|
604
|
+
async function queryEndpoint(
|
|
605
|
+
account: UsageFetchAccount,
|
|
606
|
+
access: string,
|
|
607
|
+
fetchImpl: UsageFetchImplementation,
|
|
608
|
+
timeoutMs: number,
|
|
609
|
+
): Promise<UsageReading> {
|
|
610
|
+
const headers: Record<string, string> =
|
|
611
|
+
account.family === "openai-codex"
|
|
612
|
+
? { Authorization: `Bearer ${access}` }
|
|
613
|
+
: {
|
|
614
|
+
Authorization: `Bearer ${access}`,
|
|
615
|
+
"anthropic-beta": "oauth-2025-04-20",
|
|
616
|
+
};
|
|
617
|
+
const url =
|
|
618
|
+
account.family === "openai-codex" ? CODEX_USAGE_URL : ANTHROPIC_USAGE_URL;
|
|
619
|
+
const response = await fetchWithTimeout(
|
|
620
|
+
fetchImpl,
|
|
621
|
+
url,
|
|
622
|
+
{ headers },
|
|
623
|
+
timeoutMs,
|
|
624
|
+
);
|
|
625
|
+
if (response.status < 200 || response.status >= 300) {
|
|
626
|
+
const body = await response.text().catch(() => "");
|
|
627
|
+
const retryMs = retryAfterMs(response);
|
|
628
|
+
throw new UsageEndpointError(
|
|
629
|
+
`usage endpoint returned ${response.status}: ${redactErrorBody(body)}`,
|
|
630
|
+
response.status === 429
|
|
631
|
+
? "rate-limit"
|
|
632
|
+
: response.status >= 500
|
|
633
|
+
? "server-error"
|
|
634
|
+
: "network-error",
|
|
635
|
+
response.status,
|
|
636
|
+
retryMs,
|
|
637
|
+
);
|
|
638
|
+
}
|
|
639
|
+
const payload = parsePayload(await response.text());
|
|
640
|
+
return account.family === "openai-codex"
|
|
641
|
+
? normalizeCodexUsagePayload(payload)
|
|
642
|
+
: normalizeAnthropicUsagePayload(payload);
|
|
643
|
+
}
|
|
644
|
+
|
|
645
|
+
function statusFromAttempt(
|
|
646
|
+
attempt: SharedUsageAttemptRecord | undefined,
|
|
647
|
+
enabled: boolean,
|
|
648
|
+
): UsageFetchStatus {
|
|
649
|
+
const nextAttemptAtMs =
|
|
650
|
+
attempt?.failureCount === 0 ? undefined : attempt?.nextAttemptAtMs;
|
|
651
|
+
const disabledReason =
|
|
652
|
+
attempt?.failureReason as UsageFetchStatus["disabledReason"];
|
|
653
|
+
return {
|
|
654
|
+
enabled,
|
|
655
|
+
disabled: attempt?.disabled ?? false,
|
|
656
|
+
failureCount: attempt?.failureCount ?? 0,
|
|
657
|
+
...(nextAttemptAtMs === undefined ? {} : { nextAttemptAtMs }),
|
|
658
|
+
...(disabledReason === undefined ? {} : { disabledReason }),
|
|
659
|
+
};
|
|
660
|
+
}
|
|
661
|
+
|
|
662
|
+
function failureReason(
|
|
663
|
+
error: UsageEndpointError,
|
|
664
|
+
): NonNullable<UsageFetchStatus["disabledReason"]> {
|
|
665
|
+
return error.kind ?? "network-error";
|
|
666
|
+
}
|
|
667
|
+
|
|
668
|
+
/**
|
|
669
|
+
* Lets an Antigravity attempt's bounded per-attempt deadline hand the
|
|
670
|
+
* already-acquired machine lease off to a background drain instead of the
|
|
671
|
+
* caller releasing it immediately, when the real upstream call is still
|
|
672
|
+
* outstanding at that deadline. `handedOff` starts `false`; the caller's own
|
|
673
|
+
* `finally { if (!leaseGuard.handedOff) lease.release(); }` is a no-op once
|
|
674
|
+
* it flips `true`, so ownership only ever moves one direction and is never
|
|
675
|
+
* released twice.
|
|
676
|
+
*/
|
|
677
|
+
interface AntigravityLeaseGuard {
|
|
678
|
+
readonly lease: MachineLeaseHandle;
|
|
679
|
+
handedOff: boolean;
|
|
680
|
+
}
|
|
681
|
+
|
|
682
|
+
/** Detached, lease-coordinated authoritative usage fetcher. */
|
|
683
|
+
export class UsageFetcher {
|
|
684
|
+
readonly #lockPath: string;
|
|
685
|
+
readonly #usage: UsageLedger;
|
|
686
|
+
readonly #sharedStore: SharedUsageStore;
|
|
687
|
+
readonly #resolveCredential: (
|
|
688
|
+
providerId: string,
|
|
689
|
+
) => Promise<string | undefined>;
|
|
690
|
+
readonly #fetchImpl: UsageFetchImplementation;
|
|
691
|
+
readonly #fetchAntigravityUsage: AntigravityUsageFetchImplementation;
|
|
692
|
+
readonly #now: () => number;
|
|
693
|
+
readonly #timeoutMs: number;
|
|
694
|
+
readonly #onUsageRecorded: ((providerId: string) => void) | undefined;
|
|
695
|
+
readonly #windowHistoryOptions: WindowHistoryWriteOptions | undefined;
|
|
696
|
+
readonly #lastWindowFetchAtMs = new Map<string, number>();
|
|
697
|
+
/**
|
|
698
|
+
* In-flight failure-triggered refreshes, keyed by account.
|
|
699
|
+
*
|
|
700
|
+
* Claimed synchronously before the first `await`, so two failures arriving in
|
|
701
|
+
* the same tick join one refresh instead of racing to the durable checks.
|
|
702
|
+
* The durable markers cover the cross-process case; this covers our own.
|
|
703
|
+
*/
|
|
704
|
+
readonly #inFlightFailureRefresh = new Map<
|
|
705
|
+
string,
|
|
706
|
+
Promise<UsageFetchResult>
|
|
707
|
+
>();
|
|
708
|
+
/**
|
|
709
|
+
* One real, in-flight `pi-antigravity` `fetchAccountUsage` call per
|
|
710
|
+
* canonical account, shared by every `UsageFetcher` instance in this
|
|
711
|
+
* process.
|
|
712
|
+
*
|
|
713
|
+
* `fetchAccountUsage` accepts an `AbortSignal` and propagates it into every
|
|
714
|
+
* one of its three internal legs, draining every settlement via
|
|
715
|
+
* `Promise.allSettled` before rejecting on abort. Each entry therefore owns
|
|
716
|
+
* a dedicated `AbortController`, created once when a genuinely new call
|
|
717
|
+
* starts, so `#fetchAntigravityUsageWithDeadline` can actually cancel the
|
|
718
|
+
* real upstream call at its bounded per-attempt deadline instead of merely
|
|
719
|
+
* giving up on waiting for it. Reusing the pending entry across joiners
|
|
720
|
+
* means at most one real call -- and one controller -- is outstanding per
|
|
721
|
+
* account in THIS process; a joiner never gets its own controller, so its
|
|
722
|
+
* own deadline aborts the SAME shared call other still-pending joiners are
|
|
723
|
+
* awaiting too. That is deliberate: the call is genuinely one shared
|
|
724
|
+
* operation, not one per caller.
|
|
725
|
+
*
|
|
726
|
+
* The entry is cleared only when the real promise actually settles
|
|
727
|
+
* (resolves or rejects), never when a caller's per-attempt deadline merely
|
|
728
|
+
* gives up on it -- a still-pending upstream call must stay reachable so a
|
|
729
|
+
* later poll can await its eventual result instead of duplicating the
|
|
730
|
+
* call. A late settlement after every waiting poll already timed out is
|
|
731
|
+
* simply observed by the cleanup handler and produces no write: nothing
|
|
732
|
+
* outside that handler still awaits the promise at that point.
|
|
733
|
+
*
|
|
734
|
+
* The entry holds only the promise and its controller, never the
|
|
735
|
+
* credential that started it. It cannot: telling "same account, refreshed
|
|
736
|
+
* token" apart from "a different account now occupies this slot" would
|
|
737
|
+
* need retaining the raw credential for comparison, or hashing it into a
|
|
738
|
+
* fingerprint -- both forbidden by this package's credential-retention
|
|
739
|
+
* discipline, and no legitimate boundary here yields an opaque, non-secret
|
|
740
|
+
* generation number to use instead. So a credential change while a call is
|
|
741
|
+
* pending does not start a second call under the new value; the poll joins
|
|
742
|
+
* the one already in flight, same as any other still-pending poll. This is
|
|
743
|
+
* a known, accepted limit, not a claimed guarantee: if an operator
|
|
744
|
+
* re-logs a slot into a different account while a call for the old one is
|
|
745
|
+
* still outstanding, that call's eventual result is still attributed to
|
|
746
|
+
* this slot. Closing it would need the credential-resolution boundary
|
|
747
|
+
* itself to hand back an opaque per-slot generation alongside the
|
|
748
|
+
* credential; adding one is outside this file's touches.
|
|
749
|
+
*
|
|
750
|
+
* This map is a class-static field: EVERY `UsageFetcher` instance in this
|
|
751
|
+
* process shares it, so two instances constructed in the same process
|
|
752
|
+
* cannot each start their own concurrent real call for the same account.
|
|
753
|
+
* It is still not by itself a machine-wide or fleet-wide guard -- a peer OS
|
|
754
|
+
* process has its own module state and its own map. The per-attempt
|
|
755
|
+
* machine lease (`acquireMachineLease`) is the cross-process
|
|
756
|
+
* serialization, and it now stays held -- renewed at its own
|
|
757
|
+
* `renewalIntervalMs` -- through this entry's ACTUAL settlement rather
|
|
758
|
+
* than releasing at the earlier bounded deadline, so a peer process can
|
|
759
|
+
* never acquire the lease and start a second real call while this
|
|
760
|
+
* process's real call is still genuinely outstanding. A same-process
|
|
761
|
+
* concurrent attempt for an account whose lease is still held this way
|
|
762
|
+
* gets the ordinary `lease-unavailable` busy/skip outcome, the same as any
|
|
763
|
+
* other lease contention; it never joins a call an earlier deadline already
|
|
764
|
+
* abandoned.
|
|
765
|
+
*/
|
|
766
|
+
static readonly #inFlightAntigravityRawFetch = new Map<
|
|
767
|
+
string,
|
|
768
|
+
{ readonly promise: Promise<unknown>; readonly controller: AbortController }
|
|
769
|
+
>();
|
|
770
|
+
readonly #lastWindowGapAtMs = new Map<string, number>();
|
|
771
|
+
|
|
772
|
+
constructor(options: {
|
|
773
|
+
readonly lockPath: string;
|
|
774
|
+
readonly usage: UsageLedger;
|
|
775
|
+
readonly sharedStore: SharedUsageStore;
|
|
776
|
+
readonly resolveCredential: (
|
|
777
|
+
providerId: string,
|
|
778
|
+
) => Promise<string | undefined>;
|
|
779
|
+
readonly fetchImpl?: UsageFetchImplementation;
|
|
780
|
+
readonly fetchAntigravityUsage?: AntigravityUsageFetchImplementation;
|
|
781
|
+
readonly now?: () => number;
|
|
782
|
+
readonly timeoutMs?: number;
|
|
783
|
+
readonly onUsageRecorded?: (providerId: string) => void;
|
|
784
|
+
readonly windowHistoryOptions?: WindowHistoryWriteOptions;
|
|
785
|
+
}) {
|
|
786
|
+
this.#lockPath = options.lockPath;
|
|
787
|
+
this.#usage = options.usage;
|
|
788
|
+
this.#sharedStore = options.sharedStore;
|
|
789
|
+
this.#resolveCredential = options.resolveCredential;
|
|
790
|
+
this.#fetchImpl =
|
|
791
|
+
options.fetchImpl ??
|
|
792
|
+
((input, init) => {
|
|
793
|
+
if (process.env.VITEST === "true") {
|
|
794
|
+
return Promise.reject(
|
|
795
|
+
new UsageEndpointError(
|
|
796
|
+
"network access is disabled in tests",
|
|
797
|
+
"network-error",
|
|
798
|
+
),
|
|
799
|
+
);
|
|
800
|
+
}
|
|
801
|
+
if (!ALLOWED_USAGE_URLS.has(input)) {
|
|
802
|
+
return Promise.reject(
|
|
803
|
+
new UsageEndpointError(
|
|
804
|
+
"usage endpoint URL was not allowlisted",
|
|
805
|
+
"network-error",
|
|
806
|
+
),
|
|
807
|
+
);
|
|
808
|
+
}
|
|
809
|
+
return globalThis.fetch(input, init);
|
|
810
|
+
});
|
|
811
|
+
this.#fetchAntigravityUsage =
|
|
812
|
+
options.fetchAntigravityUsage ?? defaultFetchAntigravityUsage;
|
|
813
|
+
this.#now = options.now ?? Date.now;
|
|
814
|
+
this.#timeoutMs = options.timeoutMs ?? USAGE_FETCH_TIMEOUT_MS;
|
|
815
|
+
this.#onUsageRecorded = options.onUsageRecorded;
|
|
816
|
+
this.#windowHistoryOptions = options.windowHistoryOptions;
|
|
817
|
+
}
|
|
818
|
+
|
|
819
|
+
#notifyUsageRecorded(providerId: string): void {
|
|
820
|
+
try {
|
|
821
|
+
this.#onUsageRecorded?.(providerId);
|
|
822
|
+
} catch {
|
|
823
|
+
// Observation is advisory; callback failure cannot reclassify a fetch.
|
|
824
|
+
}
|
|
825
|
+
}
|
|
826
|
+
|
|
827
|
+
status(
|
|
828
|
+
providerId: string,
|
|
829
|
+
family: AllowedFamily,
|
|
830
|
+
config: MultiAccountConfig,
|
|
831
|
+
): UsageFetchStatus {
|
|
832
|
+
const enabled = config.usageFetchEnabled?.[family] ?? true;
|
|
833
|
+
return statusFromAttempt(
|
|
834
|
+
this.#sharedStore.latestAttempt(providerId, family),
|
|
835
|
+
enabled,
|
|
836
|
+
);
|
|
837
|
+
}
|
|
838
|
+
|
|
839
|
+
#accountKey(account: UsageFetchAccount): string {
|
|
840
|
+
return `${account.family}:${account.providerId}`;
|
|
841
|
+
}
|
|
842
|
+
|
|
843
|
+
async #persistWindowGap(
|
|
844
|
+
account: UsageFetchAccount,
|
|
845
|
+
recordedAtMs: number,
|
|
846
|
+
reason: string,
|
|
847
|
+
): Promise<void> {
|
|
848
|
+
const key = this.#accountKey(account);
|
|
849
|
+
const previous = this.#lastWindowGapAtMs.get(key);
|
|
850
|
+
if (
|
|
851
|
+
previous !== undefined &&
|
|
852
|
+
recordedAtMs - previous < WINDOW_SAMPLE_INTERVAL_MS
|
|
853
|
+
) {
|
|
854
|
+
return;
|
|
855
|
+
}
|
|
856
|
+
const gap = markWindowUnusable({
|
|
857
|
+
providerId: account.providerId,
|
|
858
|
+
accountId: account.providerId,
|
|
859
|
+
family: account.family,
|
|
860
|
+
windowId: GAP_WINDOW_ID,
|
|
861
|
+
recordedAtMs,
|
|
862
|
+
schemaVersion: 1,
|
|
863
|
+
buildProvenance: `node@${process.version}`,
|
|
864
|
+
observerId: `process-${process.pid}`,
|
|
865
|
+
unusableReason: reason,
|
|
866
|
+
});
|
|
867
|
+
if (await writeHistoryWindowSample(gap, this.#windowHistoryOptions)) {
|
|
868
|
+
this.#lastWindowGapAtMs.set(key, recordedAtMs);
|
|
869
|
+
}
|
|
870
|
+
}
|
|
871
|
+
|
|
872
|
+
async #persistWindowSamples(
|
|
873
|
+
account: UsageFetchAccount,
|
|
874
|
+
reading: UsageReading,
|
|
875
|
+
recordedAtMs: number,
|
|
876
|
+
): Promise<boolean> {
|
|
877
|
+
let persisted = true;
|
|
878
|
+
for (const window of reading.windows) {
|
|
879
|
+
const baseSample = {
|
|
880
|
+
providerId: account.providerId,
|
|
881
|
+
accountId: account.providerId,
|
|
882
|
+
family: account.family,
|
|
883
|
+
windowId: window.windowId,
|
|
884
|
+
recordedAtMs,
|
|
885
|
+
...(window.resetAtMs === undefined
|
|
886
|
+
? {}
|
|
887
|
+
: { resetAtMs: window.resetAtMs }),
|
|
888
|
+
...(window.resetEpoch === undefined
|
|
889
|
+
? {}
|
|
890
|
+
: { resetEpoch: window.resetEpoch }),
|
|
891
|
+
...(window.remainingFraction === undefined
|
|
892
|
+
? {}
|
|
893
|
+
: { remainingFraction: window.remainingFraction }),
|
|
894
|
+
...(reading.projectDigest === undefined
|
|
895
|
+
? {}
|
|
896
|
+
: { projectDigest: reading.projectDigest }),
|
|
897
|
+
};
|
|
898
|
+
const sample: WindowSample = window.usable
|
|
899
|
+
? {
|
|
900
|
+
...baseSample,
|
|
901
|
+
schemaVersion: 1,
|
|
902
|
+
buildProvenance: `node@${process.version}`,
|
|
903
|
+
observerId: `process-${process.pid}`,
|
|
904
|
+
usable: true,
|
|
905
|
+
}
|
|
906
|
+
: markWindowUnusable({
|
|
907
|
+
...baseSample,
|
|
908
|
+
schemaVersion: 1,
|
|
909
|
+
buildProvenance: `node@${process.version}`,
|
|
910
|
+
observerId: `process-${process.pid}`,
|
|
911
|
+
});
|
|
912
|
+
if (!(await writeHistoryWindowSample(sample, this.#windowHistoryOptions))) {
|
|
913
|
+
persisted = false;
|
|
914
|
+
}
|
|
915
|
+
}
|
|
916
|
+
if (persisted && reading.windows.length > 0) {
|
|
917
|
+
this.#lastWindowFetchAtMs.set(this.#accountKey(account), recordedAtMs);
|
|
918
|
+
}
|
|
919
|
+
return persisted;
|
|
920
|
+
}
|
|
921
|
+
|
|
922
|
+
async fetchAccount(
|
|
923
|
+
account: UsageFetchAccount,
|
|
924
|
+
config: MultiAccountConfig,
|
|
925
|
+
): Promise<UsageFetchResult> {
|
|
926
|
+
return this.#fetchAccount(account, config, false);
|
|
927
|
+
}
|
|
928
|
+
|
|
929
|
+
/**
|
|
930
|
+
* Refreshes usage after the provider refused a request for quota reasons.
|
|
931
|
+
*
|
|
932
|
+
* Unlike the cadence poll this bypasses the fresh-header guard, because the
|
|
933
|
+
* header that looked fresh is exactly the reading the 429 just contradicted.
|
|
934
|
+
* It reuses the existing `bypassHeaderFreshness` parameter rather than
|
|
935
|
+
* inventing a mechanism: `fetchAccounts` already passes `true` for window
|
|
936
|
+
* sampling.
|
|
937
|
+
*
|
|
938
|
+
* At most one extra poll per failure, in this process or across the fleet.
|
|
939
|
+
* `failedAtMs` is the failure's own time, not now: a poll that ran after the
|
|
940
|
+
* failure but before settlement has already answered this failure's question
|
|
941
|
+
* and must suppress.
|
|
942
|
+
*/
|
|
943
|
+
async refreshAfterFailure(
|
|
944
|
+
account: UsageFetchAccount,
|
|
945
|
+
config: MultiAccountConfig,
|
|
946
|
+
failedAtMs: number,
|
|
947
|
+
): Promise<UsageFetchResult> {
|
|
948
|
+
const key = this.#accountKey(account);
|
|
949
|
+
// Step 2: claim synchronously, before any await, so a second failure in
|
|
950
|
+
// the same tick joins this refresh rather than starting its own.
|
|
951
|
+
const inFlight = this.#inFlightFailureRefresh.get(key);
|
|
952
|
+
if (inFlight !== undefined) return inFlight;
|
|
953
|
+
const pending = this.#refreshAfterFailure(account, config, failedAtMs);
|
|
954
|
+
this.#inFlightFailureRefresh.set(key, pending);
|
|
955
|
+
try {
|
|
956
|
+
return await pending;
|
|
957
|
+
} finally {
|
|
958
|
+
// Release only our own token: a later refresh may already have
|
|
959
|
+
// replaced it, and deleting that one would reopen the window.
|
|
960
|
+
if (this.#inFlightFailureRefresh.get(key) === pending) {
|
|
961
|
+
this.#inFlightFailureRefresh.delete(key);
|
|
962
|
+
}
|
|
963
|
+
}
|
|
964
|
+
}
|
|
965
|
+
|
|
966
|
+
/**
|
|
967
|
+
* Durable suppression checks for a failure-triggered refresh.
|
|
968
|
+
*
|
|
969
|
+
* Run twice: once before taking the lease, and again under it. The second
|
|
970
|
+
* run is not redundant -- a peer can write an attempt between our first read
|
|
971
|
+
* and our lease acquisition, and without the recheck both processes would
|
|
972
|
+
* poll the same account for the same failure.
|
|
973
|
+
*
|
|
974
|
+
* Each clause suppresses for a different reason, and they must stay
|
|
975
|
+
* distinguishable:
|
|
976
|
+
*
|
|
977
|
+
* - endpoint backoff: an absolute deadline from a real failure ladder.
|
|
978
|
+
* `failureCount > 0` matters because a zero-failure record with a future
|
|
979
|
+
* `nextAttemptAtMs` is an ordinary cadence reservation, not a ladder.
|
|
980
|
+
* - already answered: any attempt at or after our failure has already asked
|
|
981
|
+
* the endpoint the question this failure raises.
|
|
982
|
+
* - debounce: a previous failure-triggered refresh set a deadline we are
|
|
983
|
+
* still inside.
|
|
984
|
+
*/
|
|
985
|
+
#failureRefreshSuppressedBy(
|
|
986
|
+
attempt: SharedUsageAttemptRecord | undefined,
|
|
987
|
+
failedAtMs: number,
|
|
988
|
+
nowMs: number,
|
|
989
|
+
): "backoff" | "already-answered" | "debounced" | undefined {
|
|
990
|
+
if (attempt === undefined) return undefined;
|
|
991
|
+
// An attempt anchored in the future cannot suppress anything.
|
|
992
|
+
//
|
|
993
|
+
// Round 2 found the first version of this bound was applied only inside
|
|
994
|
+
// the debounce clause, so `already-answered` and `backoff` returned
|
|
995
|
+
// before it ran: an attempt with `observedAtMs` centuries ahead is
|
|
996
|
+
// trivially `>= failedAtMs`, and suppressed every failure refresh
|
|
997
|
+
// forever. The bound belongs here, on the record, before any clause
|
|
998
|
+
// reads it.
|
|
999
|
+
//
|
|
1000
|
+
// `nextAttemptAtMs` is allowed one debounce interval of headroom because
|
|
1001
|
+
// Only the ORIGIN timestamps are bounded, not `nextAttemptAtMs`: a real
|
|
1002
|
+
// failure ladder legitimately schedules its next rung far ahead, up to
|
|
1003
|
+
// MAX_BACKOFF_MS, and bounding that would break genuine backoff
|
|
1004
|
+
// suppression. An honest record cannot have been OBSERVED in the future.
|
|
1005
|
+
if (
|
|
1006
|
+
attempt.observedAtMs > nowMs ||
|
|
1007
|
+
(attempt.failureTriggeredAtMs !== undefined &&
|
|
1008
|
+
attempt.failureTriggeredAtMs > nowMs)
|
|
1009
|
+
) {
|
|
1010
|
+
return undefined;
|
|
1011
|
+
}
|
|
1012
|
+
if (attempt.failureCount > 0 && attempt.nextAttemptAtMs > nowMs) {
|
|
1013
|
+
return "backoff";
|
|
1014
|
+
}
|
|
1015
|
+
if (attempt.observedAtMs >= failedAtMs) return "already-answered";
|
|
1016
|
+
if (
|
|
1017
|
+
attempt.refreshDebounceUntilMs !== undefined &&
|
|
1018
|
+
attempt.refreshDebounceUntilMs > nowMs &&
|
|
1019
|
+
// Same far-future bypass as the hold: the stored span can be a
|
|
1020
|
+
// legitimate five minutes while its origin sits centuries ahead, which
|
|
1021
|
+
// would suppress every failure refresh forever. A deadline further
|
|
1022
|
+
// than one debounce interval from now cannot be honest.
|
|
1023
|
+
attempt.refreshDebounceUntilMs <= nowMs + USAGE_FETCH_INTERVAL_MS
|
|
1024
|
+
) {
|
|
1025
|
+
return "debounced";
|
|
1026
|
+
}
|
|
1027
|
+
return undefined;
|
|
1028
|
+
}
|
|
1029
|
+
|
|
1030
|
+
async #refreshAfterFailure(
|
|
1031
|
+
account: UsageFetchAccount,
|
|
1032
|
+
config: MultiAccountConfig,
|
|
1033
|
+
failedAtMs: number,
|
|
1034
|
+
): Promise<UsageFetchResult> {
|
|
1035
|
+
if (!isCanonicalManagedProviderId(account.providerId, account.family)) {
|
|
1036
|
+
return { providerId: account.providerId, status: "failed" };
|
|
1037
|
+
}
|
|
1038
|
+
// An api_key account has no OAuth usage endpoint; unmeasured, not failed.
|
|
1039
|
+
if (account.credentialType === "api_key") {
|
|
1040
|
+
return { providerId: account.providerId, status: "not-supported" };
|
|
1041
|
+
}
|
|
1042
|
+
if (!(config.usageFetchEnabled?.[account.family] ?? true)) {
|
|
1043
|
+
return { providerId: account.providerId, status: "disabled-by-config" };
|
|
1044
|
+
}
|
|
1045
|
+
|
|
1046
|
+
// Step 4/5: durable checks before the lease.
|
|
1047
|
+
const beforeLease = this.#sharedStore.latestAttempt(
|
|
1048
|
+
account.providerId,
|
|
1049
|
+
account.family,
|
|
1050
|
+
);
|
|
1051
|
+
if (
|
|
1052
|
+
this.#failureRefreshSuppressedBy(
|
|
1053
|
+
beforeLease,
|
|
1054
|
+
failedAtMs,
|
|
1055
|
+
this.#now(),
|
|
1056
|
+
) !== undefined
|
|
1057
|
+
) {
|
|
1058
|
+
return { providerId: account.providerId, status: "not-due" };
|
|
1059
|
+
}
|
|
1060
|
+
|
|
1061
|
+
// Step 6: the lease. Failure means another usage operation is running,
|
|
1062
|
+
// which is itself a reason to suppress rather than queue behind it.
|
|
1063
|
+
const lease = acquireMachineLease({
|
|
1064
|
+
lockPath: this.#lockPath,
|
|
1065
|
+
ttlMs: Math.max(2_000, this.#timeoutMs + 2_000),
|
|
1066
|
+
now: this.#now,
|
|
1067
|
+
reclaimMalformed: true,
|
|
1068
|
+
});
|
|
1069
|
+
if (!lease) {
|
|
1070
|
+
return { providerId: account.providerId, status: "lease-unavailable" };
|
|
1071
|
+
}
|
|
1072
|
+
const leaseGuard: AntigravityLeaseGuard = { lease, handedOff: false };
|
|
1073
|
+
try {
|
|
1074
|
+
// Step 7: the same checks again, now that nobody else can write.
|
|
1075
|
+
const underLease = this.#sharedStore.latestAttempt(
|
|
1076
|
+
account.providerId,
|
|
1077
|
+
account.family,
|
|
1078
|
+
);
|
|
1079
|
+
if (
|
|
1080
|
+
this.#failureRefreshSuppressedBy(
|
|
1081
|
+
underLease,
|
|
1082
|
+
failedAtMs,
|
|
1083
|
+
this.#now(),
|
|
1084
|
+
) !== undefined
|
|
1085
|
+
) {
|
|
1086
|
+
return { providerId: account.providerId, status: "not-due" };
|
|
1087
|
+
}
|
|
1088
|
+
|
|
1089
|
+
// Step 8: persist the marked reservation BEFORE resolving the
|
|
1090
|
+
// credential. If the ledger cannot be written, make no endpoint call:
|
|
1091
|
+
// an unbounded call with no durable backoff state is how a stampede
|
|
1092
|
+
// starts.
|
|
1093
|
+
const attemptedAtMs = this.#now();
|
|
1094
|
+
const reserved = this.#sharedStore.append({
|
|
1095
|
+
recordType: "usage-attempt",
|
|
1096
|
+
tokens: null,
|
|
1097
|
+
providerId: account.providerId,
|
|
1098
|
+
family: account.family,
|
|
1099
|
+
observedAtMs: attemptedAtMs,
|
|
1100
|
+
observerId: this.#sharedStore.observerId,
|
|
1101
|
+
failureCount: underLease?.failureCount ?? 0,
|
|
1102
|
+
nextAttemptAtMs: attemptedAtMs + USAGE_FETCH_INTERVAL_MS,
|
|
1103
|
+
disabled: underLease?.disabled ?? false,
|
|
1104
|
+
...(underLease?.failureReason === undefined
|
|
1105
|
+
? {}
|
|
1106
|
+
: { failureReason: underLease.failureReason }),
|
|
1107
|
+
...(underLease?.failureDetail === undefined
|
|
1108
|
+
? {}
|
|
1109
|
+
: { failureDetail: underLease.failureDetail }),
|
|
1110
|
+
failureTriggeredAtMs: failedAtMs,
|
|
1111
|
+
refreshDebounceUntilMs: failedAtMs + USAGE_FETCH_INTERVAL_MS,
|
|
1112
|
+
});
|
|
1113
|
+
if (!reserved) {
|
|
1114
|
+
return { providerId: account.providerId, status: "failed" };
|
|
1115
|
+
}
|
|
1116
|
+
|
|
1117
|
+
// Step 9: the ordinary ladder. A failed refresh is not a no-op -- it
|
|
1118
|
+
// advances failureCount and can disable the account, exactly as a
|
|
1119
|
+
// cadence poll would.
|
|
1120
|
+
// `underLease` must be passed: the catch computes the next rung as
|
|
1121
|
+
// `prior.failureCount + 1`, so omitting it restarts an existing ladder
|
|
1122
|
+
// at 1 and can clear a `disabled` flag. A dead endpoint would then be
|
|
1123
|
+
// retried harder after every failure and might never reach the disable
|
|
1124
|
+
// threshold.
|
|
1125
|
+
return await this.#completeAttempt(
|
|
1126
|
+
account,
|
|
1127
|
+
attemptedAtMs,
|
|
1128
|
+
leaseGuard,
|
|
1129
|
+
{
|
|
1130
|
+
failureTriggeredAtMs: failedAtMs,
|
|
1131
|
+
refreshDebounceUntilMs: failedAtMs + USAGE_FETCH_INTERVAL_MS,
|
|
1132
|
+
},
|
|
1133
|
+
underLease,
|
|
1134
|
+
);
|
|
1135
|
+
} finally {
|
|
1136
|
+
if (!leaseGuard.handedOff) lease.release();
|
|
1137
|
+
}
|
|
1138
|
+
}
|
|
1139
|
+
|
|
1140
|
+
async #fetchAccount(
|
|
1141
|
+
account: UsageFetchAccount,
|
|
1142
|
+
config: MultiAccountConfig,
|
|
1143
|
+
bypassHeaderFreshness: boolean,
|
|
1144
|
+
): Promise<UsageFetchResult> {
|
|
1145
|
+
if (!isCanonicalManagedProviderId(account.providerId, account.family)) {
|
|
1146
|
+
return { providerId: account.providerId, status: "failed" };
|
|
1147
|
+
}
|
|
1148
|
+
// #24: an api_key account has no OAuth usage endpoint. Return an
|
|
1149
|
+
// unmeasured outcome BEFORE any config gate, header check, backoff, lease,
|
|
1150
|
+
// or network call, so it never accrues a failure, never advances the
|
|
1151
|
+
// ladder, and is never disabled. Missing usage stays a coverage gap.
|
|
1152
|
+
if (account.credentialType === "api_key") {
|
|
1153
|
+
return { providerId: account.providerId, status: "not-supported" };
|
|
1154
|
+
}
|
|
1155
|
+
if (!(config.usageFetchEnabled?.[account.family] ?? true)) {
|
|
1156
|
+
return { providerId: account.providerId, status: "disabled-by-config" };
|
|
1157
|
+
}
|
|
1158
|
+
const nowMs = this.#now();
|
|
1159
|
+
if (
|
|
1160
|
+
!bypassHeaderFreshness &&
|
|
1161
|
+
this.#usage.hasFreshHeaderObservation(
|
|
1162
|
+
account.providerId,
|
|
1163
|
+
account.family,
|
|
1164
|
+
nowMs,
|
|
1165
|
+
)
|
|
1166
|
+
) {
|
|
1167
|
+
return { providerId: account.providerId, status: "not-due" };
|
|
1168
|
+
}
|
|
1169
|
+
const prior = this.#sharedStore.latestAttempt(
|
|
1170
|
+
account.providerId,
|
|
1171
|
+
account.family,
|
|
1172
|
+
);
|
|
1173
|
+
// Backoff is checked BEFORE `disabled`, deliberately.
|
|
1174
|
+
//
|
|
1175
|
+
// Every failure writes a correctly-computed `nextAttemptAtMs`, but the
|
|
1176
|
+
// disabled gate used to come first and returned unconditionally, so that
|
|
1177
|
+
// timestamp was never read once an account had been disabled. `disabled`
|
|
1178
|
+
// therefore meant "forever": nothing anywhere sets it false or resets
|
|
1179
|
+
// `failureCount`.
|
|
1180
|
+
//
|
|
1181
|
+
// The consequences invert the intent. The reasons that disable a fetcher --
|
|
1182
|
+
// rate-limit above all -- are TRANSIENT, and the poller is the only
|
|
1183
|
+
// quota-free way to observe that the account recovered. So a briefly
|
|
1184
|
+
// throttled account was cut off permanently from the mechanism that would
|
|
1185
|
+
// have shown it was healthy again. Observed live: anthropic-account-2
|
|
1186
|
+
// exhausted its ladder on 27 July with `failureReason: "rate-limit"`, and
|
|
1187
|
+
// its `nextAttemptAtMs` came due seven days ago while the account sat
|
|
1188
|
+
// usable and invisible.
|
|
1189
|
+
//
|
|
1190
|
+
// Once the recorded backoff has elapsed, the account is retried whether or
|
|
1191
|
+
// not it was disabled. A retry that fails again simply re-arms the ladder at
|
|
1192
|
+
// its capped rung, so a genuinely dead endpoint is polled at most once per
|
|
1193
|
+
// MAX_BACKOFF_MS rather than hammered.
|
|
1194
|
+
if (prior !== undefined && prior.nextAttemptAtMs > nowMs) {
|
|
1195
|
+
await this.#persistWindowGap(
|
|
1196
|
+
account,
|
|
1197
|
+
nowMs,
|
|
1198
|
+
prior.disabled ? "auto-disabled" : "backoff",
|
|
1199
|
+
);
|
|
1200
|
+
return {
|
|
1201
|
+
providerId: account.providerId,
|
|
1202
|
+
status: prior.disabled ? "disabled" : "not-due",
|
|
1203
|
+
};
|
|
1204
|
+
}
|
|
1205
|
+
const lease = acquireMachineLease({
|
|
1206
|
+
lockPath: this.#lockPath,
|
|
1207
|
+
ttlMs: Math.max(2_000, this.#timeoutMs + 2_000),
|
|
1208
|
+
now: this.#now,
|
|
1209
|
+
// A wedged usage-fetch.lock would otherwise suppress every poll.
|
|
1210
|
+
reclaimMalformed: true,
|
|
1211
|
+
});
|
|
1212
|
+
if (!lease) {
|
|
1213
|
+
await this.#persistWindowGap(account, nowMs, "lease-unavailable");
|
|
1214
|
+
return { providerId: account.providerId, status: "lease-unavailable" };
|
|
1215
|
+
}
|
|
1216
|
+
const leaseGuard: AntigravityLeaseGuard = { lease, handedOff: false };
|
|
1217
|
+
try {
|
|
1218
|
+
const current = this.#sharedStore.latestAttempt(
|
|
1219
|
+
account.providerId,
|
|
1220
|
+
account.family,
|
|
1221
|
+
);
|
|
1222
|
+
// Same ordering as the pre-lease check above: an elapsed backoff wins
|
|
1223
|
+
// over a stale `disabled` flag, so a recovered account can be observed
|
|
1224
|
+
// again. Re-read under the lease because a peer may have attempted in
|
|
1225
|
+
// between.
|
|
1226
|
+
if (current !== undefined && current.nextAttemptAtMs > this.#now()) {
|
|
1227
|
+
await this.#persistWindowGap(
|
|
1228
|
+
account,
|
|
1229
|
+
this.#now(),
|
|
1230
|
+
current.disabled ? "auto-disabled" : "backoff",
|
|
1231
|
+
);
|
|
1232
|
+
return {
|
|
1233
|
+
providerId: account.providerId,
|
|
1234
|
+
status: current.disabled ? "disabled" : "not-due",
|
|
1235
|
+
};
|
|
1236
|
+
}
|
|
1237
|
+
const attemptedAtMs = this.#now();
|
|
1238
|
+
const reserved = this.#sharedStore.append({
|
|
1239
|
+
recordType: "usage-attempt",
|
|
1240
|
+
tokens: null,
|
|
1241
|
+
providerId: account.providerId,
|
|
1242
|
+
family: account.family,
|
|
1243
|
+
observedAtMs: attemptedAtMs,
|
|
1244
|
+
observerId: this.#sharedStore.observerId,
|
|
1245
|
+
failureCount: current?.failureCount ?? 0,
|
|
1246
|
+
nextAttemptAtMs: attemptedAtMs + USAGE_FETCH_INTERVAL_MS,
|
|
1247
|
+
disabled: current?.disabled ?? false,
|
|
1248
|
+
...(current?.failureReason === undefined
|
|
1249
|
+
? {}
|
|
1250
|
+
: { failureReason: current.failureReason }),
|
|
1251
|
+
...(current?.failureDetail === undefined
|
|
1252
|
+
? {}
|
|
1253
|
+
: { failureDetail: current.failureDetail }),
|
|
1254
|
+
});
|
|
1255
|
+
if (!reserved) {
|
|
1256
|
+
// The attempt ledger is the fleet-wide stampede barrier. If it
|
|
1257
|
+
// cannot be persisted, fail closed rather than making an
|
|
1258
|
+
// unbounded endpoint call with no durable backoff state.
|
|
1259
|
+
await this.#persistWindowGap(account, attemptedAtMs, "attempt-persistence");
|
|
1260
|
+
return { providerId: account.providerId, status: "failed" };
|
|
1261
|
+
}
|
|
1262
|
+
return await this.#completeAttempt(
|
|
1263
|
+
account,
|
|
1264
|
+
attemptedAtMs,
|
|
1265
|
+
leaseGuard,
|
|
1266
|
+
{},
|
|
1267
|
+
current,
|
|
1268
|
+
);
|
|
1269
|
+
} finally {
|
|
1270
|
+
if (!leaseGuard.handedOff) lease.release();
|
|
1271
|
+
}
|
|
1272
|
+
}
|
|
1273
|
+
|
|
1274
|
+
/**
|
|
1275
|
+
* Reuse (or start) the one real, in-flight raw fetch for this account,
|
|
1276
|
+
* shared process-wide. See `#inFlightAntigravityRawFetch` for why this
|
|
1277
|
+
* exists and its exact limits -- in particular, this deliberately never
|
|
1278
|
+
* compares `apiKey` against a stored value: doing so would mean retaining
|
|
1279
|
+
* or fingerprinting the raw credential, which this package's
|
|
1280
|
+
* credential-retention discipline forbids. `apiKey` is used only to start
|
|
1281
|
+
* a genuinely new call -- with a fresh `AbortController` -- when no pending
|
|
1282
|
+
* one exists for this account.
|
|
1283
|
+
*/
|
|
1284
|
+
#antigravityRawFetch(
|
|
1285
|
+
account: UsageFetchAccount,
|
|
1286
|
+
apiKey: string,
|
|
1287
|
+
): { readonly promise: Promise<unknown>; readonly controller: AbortController } {
|
|
1288
|
+
const key = this.#accountKey(account);
|
|
1289
|
+
const existing = UsageFetcher.#inFlightAntigravityRawFetch.get(key);
|
|
1290
|
+
if (existing !== undefined) {
|
|
1291
|
+
return existing;
|
|
1292
|
+
}
|
|
1293
|
+
const controller = new AbortController();
|
|
1294
|
+
const promise = this.#fetchAntigravityUsage(apiKey, {
|
|
1295
|
+
signal: controller.signal,
|
|
1296
|
+
});
|
|
1297
|
+
const entry = { promise, controller };
|
|
1298
|
+
UsageFetcher.#inFlightAntigravityRawFetch.set(key, entry);
|
|
1299
|
+
const clearOnSettle = () => {
|
|
1300
|
+
// Only clear our own entry: a settlement racing a fresh start (after
|
|
1301
|
+
// an earlier settlement already cleared and replaced this one) must
|
|
1302
|
+
// not delete the newer entry.
|
|
1303
|
+
if (UsageFetcher.#inFlightAntigravityRawFetch.get(key) === entry) {
|
|
1304
|
+
UsageFetcher.#inFlightAntigravityRawFetch.delete(key);
|
|
1305
|
+
}
|
|
1306
|
+
};
|
|
1307
|
+
promise.then(clearOnSettle, clearOnSettle);
|
|
1308
|
+
return entry;
|
|
1309
|
+
}
|
|
1310
|
+
|
|
1311
|
+
/**
|
|
1312
|
+
* Antigravity has no raw-HTTP usage endpoint of its own: the reviewed
|
|
1313
|
+
* `pi-antigravity` primitive makes several internal calls and resolves one
|
|
1314
|
+
* decoded `AccountUsage` object. This wraps that call with the same bounded
|
|
1315
|
+
* per-attempt deadline every other family's usage fetch gets, then projects
|
|
1316
|
+
* the result through `projectAntigravityUsage` before anything is recorded.
|
|
1317
|
+
*/
|
|
1318
|
+
async #queryAntigravityUsage(
|
|
1319
|
+
account: UsageFetchAccount,
|
|
1320
|
+
apiKey: string,
|
|
1321
|
+
leaseGuard: AntigravityLeaseGuard,
|
|
1322
|
+
): Promise<UsageReading> {
|
|
1323
|
+
let raw: unknown;
|
|
1324
|
+
try {
|
|
1325
|
+
raw = await this.#fetchAntigravityUsageWithDeadline(
|
|
1326
|
+
account,
|
|
1327
|
+
apiKey,
|
|
1328
|
+
leaseGuard,
|
|
1329
|
+
);
|
|
1330
|
+
} catch (error) {
|
|
1331
|
+
if (error instanceof UsageEndpointError) throw error;
|
|
1332
|
+
throw new UsageEndpointError(
|
|
1333
|
+
redactErrorBody(error instanceof Error ? error.message : String(error)),
|
|
1334
|
+
"network-error",
|
|
1335
|
+
);
|
|
1336
|
+
}
|
|
1337
|
+
const projection = projectAntigravityUsage(raw);
|
|
1338
|
+
if (projection === undefined) {
|
|
1339
|
+
throw new UsageEndpointError(
|
|
1340
|
+
"Antigravity usage endpoint response was not a bounded object",
|
|
1341
|
+
"malformed-response",
|
|
1342
|
+
undefined,
|
|
1343
|
+
undefined,
|
|
1344
|
+
"not-object",
|
|
1345
|
+
);
|
|
1346
|
+
}
|
|
1347
|
+
const reading = primaryReading(
|
|
1348
|
+
projection.windows.filter((window) => window.usable),
|
|
1349
|
+
);
|
|
1350
|
+
if (reading === undefined) {
|
|
1351
|
+
const usage = asObject(raw);
|
|
1352
|
+
throw new UsageEndpointError(
|
|
1353
|
+
"Antigravity usage endpoint returned no bounded quota bucket",
|
|
1354
|
+
"malformed-response",
|
|
1355
|
+
undefined,
|
|
1356
|
+
undefined,
|
|
1357
|
+
typeof usage?.quotaSummaryError === "string"
|
|
1358
|
+
? "quota-summary-error"
|
|
1359
|
+
: "no-quota-groups",
|
|
1360
|
+
);
|
|
1361
|
+
}
|
|
1362
|
+
return {
|
|
1363
|
+
...reading,
|
|
1364
|
+
...(projection.projectDigest === undefined
|
|
1365
|
+
? {}
|
|
1366
|
+
: { projectDigest: projection.projectDigest }),
|
|
1367
|
+
};
|
|
1368
|
+
}
|
|
1369
|
+
|
|
1370
|
+
/**
|
|
1371
|
+
* Races the shared in-flight raw fetch against this attempt's bounded
|
|
1372
|
+
* deadline. Losing that race calls `entry.controller.abort()`, so the abort
|
|
1373
|
+
* this repository issues is exactly the abort the fork's own three legs
|
|
1374
|
+
* observe -- but this process still awaits that call's OWN settlement, not
|
|
1375
|
+
* merely the deadline, before releasing the machine-shared lease: hitting
|
|
1376
|
+
* the deadline hands lease ownership to `leaseGuard`, which
|
|
1377
|
+
* `#drainAntigravityLease` renews at the lease's own `renewalIntervalMs`
|
|
1378
|
+
* until the real promise actually settles, then releases. The caller's own
|
|
1379
|
+
* `finally { if (!leaseGuard.handedOff) lease.release(); }` becomes a no-op
|
|
1380
|
+
* once handed off, so the lease is never released early. A late settlement
|
|
1381
|
+
* -- success or failure -- reaches only the drain continuation from that
|
|
1382
|
+
* point on: this function has already rejected the deadline's caller, and
|
|
1383
|
+
* nothing here writes usage, cost, or history from that late value.
|
|
1384
|
+
*/
|
|
1385
|
+
async #fetchAntigravityUsageWithDeadline(
|
|
1386
|
+
account: UsageFetchAccount,
|
|
1387
|
+
apiKey: string,
|
|
1388
|
+
leaseGuard: AntigravityLeaseGuard,
|
|
1389
|
+
): Promise<unknown> {
|
|
1390
|
+
const entry = this.#antigravityRawFetch(account, apiKey);
|
|
1391
|
+
let timedOut = false;
|
|
1392
|
+
let timer: ReturnType<typeof setTimeout> | undefined;
|
|
1393
|
+
const deadline = new Promise<never>((_resolve, reject) => {
|
|
1394
|
+
timer = setTimeout(() => {
|
|
1395
|
+
timedOut = true;
|
|
1396
|
+
entry.controller.abort();
|
|
1397
|
+
reject(
|
|
1398
|
+
new UsageEndpointError(
|
|
1399
|
+
`Antigravity usage fetch timed out after ${Math.round(this.#timeoutMs / 1_000)}s`,
|
|
1400
|
+
"network-error",
|
|
1401
|
+
),
|
|
1402
|
+
);
|
|
1403
|
+
}, this.#timeoutMs);
|
|
1404
|
+
});
|
|
1405
|
+
// Attach a no-op rejection handler so a slow transport that rejects AFTER
|
|
1406
|
+
// the deadline has already won the race cannot surface as an unhandled
|
|
1407
|
+
// rejection; the caller only ever observes whichever settles first.
|
|
1408
|
+
// `#antigravityRawFetch` already attached its own settle handler too, so
|
|
1409
|
+
// this is belt-and-suspenders, not the only protection.
|
|
1410
|
+
entry.promise.catch(() => {});
|
|
1411
|
+
try {
|
|
1412
|
+
return await Promise.race([entry.promise, deadline]);
|
|
1413
|
+
} finally {
|
|
1414
|
+
if (timer !== undefined) clearTimeout(timer);
|
|
1415
|
+
if (timedOut) {
|
|
1416
|
+
leaseGuard.handedOff = true;
|
|
1417
|
+
this.#drainAntigravityLease(leaseGuard.lease, entry.promise);
|
|
1418
|
+
}
|
|
1419
|
+
}
|
|
1420
|
+
}
|
|
1421
|
+
|
|
1422
|
+
/**
|
|
1423
|
+
* Holds the machine-shared usage lease open until `pending` -- the real,
|
|
1424
|
+
* still-outstanding `fetchAccountUsage` call a bounded deadline already gave
|
|
1425
|
+
* up on -- actually settles. This reuses the lease's existing renewal
|
|
1426
|
+
* mechanism (`renew()` / `renewalIntervalMs`) rather than inventing a
|
|
1427
|
+
* second one: one drain-scoped interval, cleared the moment the real call
|
|
1428
|
+
* is done, not a standing per-agent timer.
|
|
1429
|
+
*
|
|
1430
|
+
* The first renewal happens synchronously, right here, before the interval
|
|
1431
|
+
* is ever scheduled. `setInterval`'s own first tick fires only after a
|
|
1432
|
+
* full `renewalIntervalMs`, and at the production defaults
|
|
1433
|
+
* (`USAGE_FETCH_TIMEOUT_MS=10_000` -> `ttlMs=12_000` ->
|
|
1434
|
+
* `renewalIntervalMs=4_000`) that first tick would land at
|
|
1435
|
+
* deadline+4_000ms -- two full seconds after the lease already expired on
|
|
1436
|
+
* disk at deadline+2_000ms. The lease's own ttl leaves exactly that
|
|
1437
|
+
* `ttlMs - timeoutMs` buffer past the deadline for this handoff to land
|
|
1438
|
+
* one immediate renewal inside, so calling `renew()` now (still inside
|
|
1439
|
+
* that buffer) closes the gap before the interval ever needs to run.
|
|
1440
|
+
*
|
|
1441
|
+
* `renew()`'s boolean result is never discarded: a `false` here or on a
|
|
1442
|
+
* later tick means the lease could not be kept -- already expired, or
|
|
1443
|
+
* reclaimed by a peer -- and this drain has nothing left to renew, so
|
|
1444
|
+
* renewal stops rather than continuing to poll a lease it no longer
|
|
1445
|
+
* safely holds. `pending` is still awaited either way, so a late
|
|
1446
|
+
* settlement never becomes an unhandled rejection, and `lease.release()`
|
|
1447
|
+
* still runs at the end either way: `release()` itself refuses to touch a
|
|
1448
|
+
* record whose token no longer matches the one this handle acquired, so
|
|
1449
|
+
* it can never delete a peer's lease even if renewal was lost earlier.
|
|
1450
|
+
*/
|
|
1451
|
+
#drainAntigravityLease(
|
|
1452
|
+
lease: MachineLeaseHandle,
|
|
1453
|
+
pending: Promise<unknown>,
|
|
1454
|
+
): void {
|
|
1455
|
+
let interval: ReturnType<typeof setInterval> | undefined;
|
|
1456
|
+
const stopRenewing = (): void => {
|
|
1457
|
+
if (interval !== undefined) {
|
|
1458
|
+
clearInterval(interval);
|
|
1459
|
+
interval = undefined;
|
|
1460
|
+
}
|
|
1461
|
+
};
|
|
1462
|
+
if (lease.renew()) {
|
|
1463
|
+
interval = setInterval(() => {
|
|
1464
|
+
if (!lease.renew()) stopRenewing();
|
|
1465
|
+
}, lease.renewalIntervalMs);
|
|
1466
|
+
}
|
|
1467
|
+
void pending
|
|
1468
|
+
.catch(() => {
|
|
1469
|
+
// A late failure from the abandoned call is observed only to know
|
|
1470
|
+
// the lease can now be released -- never written to usage, cost,
|
|
1471
|
+
// or history.
|
|
1472
|
+
})
|
|
1473
|
+
.finally(() => {
|
|
1474
|
+
stopRenewing();
|
|
1475
|
+
lease.release();
|
|
1476
|
+
});
|
|
1477
|
+
}
|
|
1478
|
+
|
|
1479
|
+
/**
|
|
1480
|
+
* The shared post-reservation ladder: call the endpoint, record the reading,
|
|
1481
|
+
* and on failure advance failureCount/backoff and possibly disable.
|
|
1482
|
+
*
|
|
1483
|
+
* Extracted so a failure-triggered refresh completes through exactly the same
|
|
1484
|
+
* path as a cadence poll. A second copy would be free to drift, and the one
|
|
1485
|
+
* that drifted would be the one nobody tests.
|
|
1486
|
+
*
|
|
1487
|
+
* `markers` is carried onto both outcome records, so the debounce deadline
|
|
1488
|
+
* survives the attempt completing. Without that a later failure would see an
|
|
1489
|
+
* unmarked record and poll again immediately.
|
|
1490
|
+
*/
|
|
1491
|
+
async #completeAttempt(
|
|
1492
|
+
account: UsageFetchAccount,
|
|
1493
|
+
attemptedAtMs: number,
|
|
1494
|
+
leaseGuard: AntigravityLeaseGuard,
|
|
1495
|
+
markers: {
|
|
1496
|
+
readonly failureTriggeredAtMs?: number;
|
|
1497
|
+
readonly refreshDebounceUntilMs?: number;
|
|
1498
|
+
} = {},
|
|
1499
|
+
prior?: SharedUsageAttemptRecord,
|
|
1500
|
+
): Promise<UsageFetchResult> {
|
|
1501
|
+
try {
|
|
1502
|
+
const credential = await this.#resolveCredential(account.providerId);
|
|
1503
|
+
if (typeof credential !== "string" || credential.length === 0) {
|
|
1504
|
+
throw new UsageEndpointError(
|
|
1505
|
+
"credential unavailable",
|
|
1506
|
+
"credential-unavailable",
|
|
1507
|
+
);
|
|
1508
|
+
}
|
|
1509
|
+
const reading =
|
|
1510
|
+
account.family === "google-antigravity"
|
|
1511
|
+
? await this.#queryAntigravityUsage(account, credential, leaseGuard)
|
|
1512
|
+
: await queryEndpoint(
|
|
1513
|
+
account,
|
|
1514
|
+
credential,
|
|
1515
|
+
this.#fetchImpl,
|
|
1516
|
+
this.#timeoutMs,
|
|
1517
|
+
);
|
|
1518
|
+
const capturedAtMs = this.#now();
|
|
1519
|
+
this.#usage.record({
|
|
1520
|
+
providerId: account.providerId,
|
|
1521
|
+
family: account.family,
|
|
1522
|
+
observedAtMs: capturedAtMs,
|
|
1523
|
+
rateLimit: {
|
|
1524
|
+
...(reading.recoveryAtMs === undefined
|
|
1525
|
+
? {}
|
|
1526
|
+
: { recoveryAtMs: reading.recoveryAtMs }),
|
|
1527
|
+
utilization: reading.utilization,
|
|
1528
|
+
utilizationSource: "usage-endpoint",
|
|
1529
|
+
},
|
|
1530
|
+
});
|
|
1531
|
+
this.#notifyUsageRecorded(account.providerId);
|
|
1532
|
+
await this.#persistWindowSamples(account, reading, capturedAtMs);
|
|
1533
|
+
this.#sharedStore.append({
|
|
1534
|
+
recordType: "usage-attempt",
|
|
1535
|
+
tokens: null,
|
|
1536
|
+
providerId: account.providerId,
|
|
1537
|
+
family: account.family,
|
|
1538
|
+
observedAtMs: capturedAtMs,
|
|
1539
|
+
observerId: this.#sharedStore.observerId,
|
|
1540
|
+
failureCount: 0,
|
|
1541
|
+
nextAttemptAtMs: capturedAtMs + USAGE_FETCH_INTERVAL_MS,
|
|
1542
|
+
disabled: false,
|
|
1543
|
+
...markers,
|
|
1544
|
+
});
|
|
1545
|
+
return {
|
|
1546
|
+
providerId: account.providerId,
|
|
1547
|
+
status: "fetched",
|
|
1548
|
+
utilization: reading.utilization,
|
|
1549
|
+
capturedAtMs,
|
|
1550
|
+
};
|
|
1551
|
+
} catch (error) {
|
|
1552
|
+
const failure =
|
|
1553
|
+
error instanceof UsageEndpointError
|
|
1554
|
+
? error
|
|
1555
|
+
: new UsageEndpointError("usage endpoint failed", "network-error");
|
|
1556
|
+
const failureCount = (prior?.failureCount ?? 0) + 1;
|
|
1557
|
+
const backoff = Math.min(
|
|
1558
|
+
MAX_BACKOFF_MS,
|
|
1559
|
+
BASE_BACKOFF_MS * 2 ** Math.max(0, failureCount - 1),
|
|
1560
|
+
);
|
|
1561
|
+
const disabled = failureCount >= USAGE_FETCH_DISABLE_AFTER_FAILURES;
|
|
1562
|
+
this.#sharedStore.append({
|
|
1563
|
+
recordType: "usage-attempt",
|
|
1564
|
+
tokens: null,
|
|
1565
|
+
providerId: account.providerId,
|
|
1566
|
+
family: account.family,
|
|
1567
|
+
observedAtMs: attemptedAtMs,
|
|
1568
|
+
observerId: this.#sharedStore.observerId,
|
|
1569
|
+
failureCount,
|
|
1570
|
+
nextAttemptAtMs: Math.max(
|
|
1571
|
+
attemptedAtMs + USAGE_FETCH_INTERVAL_MS,
|
|
1572
|
+
attemptedAtMs + backoff,
|
|
1573
|
+
attemptedAtMs + (failure.retryAfterMs ?? 0),
|
|
1574
|
+
),
|
|
1575
|
+
disabled,
|
|
1576
|
+
failureReason: failureReason(failure),
|
|
1577
|
+
...(failure.detail === undefined
|
|
1578
|
+
? {}
|
|
1579
|
+
: { failureDetail: failure.detail }),
|
|
1580
|
+
...markers,
|
|
1581
|
+
});
|
|
1582
|
+
await this.#persistWindowGap(
|
|
1583
|
+
account,
|
|
1584
|
+
attemptedAtMs,
|
|
1585
|
+
disabled ? "auto-disabled" : "backoff",
|
|
1586
|
+
);
|
|
1587
|
+
return {
|
|
1588
|
+
providerId: account.providerId,
|
|
1589
|
+
status:
|
|
1590
|
+
failure.kind === "credential-unavailable"
|
|
1591
|
+
? "credential-unavailable"
|
|
1592
|
+
: "failed",
|
|
1593
|
+
};
|
|
1594
|
+
}
|
|
1595
|
+
}
|
|
1596
|
+
|
|
1597
|
+
async fetchAccounts(
|
|
1598
|
+
accounts: readonly UsageFetchAccount[],
|
|
1599
|
+
config: MultiAccountConfig,
|
|
1600
|
+
): Promise<readonly UsageFetchResult[]> {
|
|
1601
|
+
const results: UsageFetchResult[] = [];
|
|
1602
|
+
for (const account of accounts) {
|
|
1603
|
+
const result = await this.fetchAccount(account, config);
|
|
1604
|
+
results.push(result);
|
|
1605
|
+
|
|
1606
|
+
// Window cadence is deliberately a call-driven comparison, not a timer.
|
|
1607
|
+
// A normal fetch above records its own window sample. When a fresh
|
|
1608
|
+
// header suppressed it, this second path bypasses ONLY that header
|
|
1609
|
+
// check; #fetchAccount still applies every durable/config/lease/network
|
|
1610
|
+
// guard in the same order.
|
|
1611
|
+
const last = this.#lastWindowFetchAtMs.get(this.#accountKey(account));
|
|
1612
|
+
if (
|
|
1613
|
+
last !== undefined &&
|
|
1614
|
+
this.#now() - last < WINDOW_SAMPLE_INTERVAL_MS
|
|
1615
|
+
) {
|
|
1616
|
+
continue;
|
|
1617
|
+
}
|
|
1618
|
+
// REQ-WINDOW-CADENCE: enforce cross-process 15-minute floor.
|
|
1619
|
+
// Check durable attempt record to prevent peer-process fetch within the window.
|
|
1620
|
+
const prior = this.#sharedStore.latestAttempt(
|
|
1621
|
+
account.providerId,
|
|
1622
|
+
account.family,
|
|
1623
|
+
);
|
|
1624
|
+
if (
|
|
1625
|
+
prior !== undefined &&
|
|
1626
|
+
this.#now() - prior.observedAtMs < WINDOW_SAMPLE_INTERVAL_MS
|
|
1627
|
+
) {
|
|
1628
|
+
continue;
|
|
1629
|
+
}
|
|
1630
|
+
await this.#fetchAccount(account, config, true);
|
|
1631
|
+
}
|
|
1632
|
+
return results;
|
|
1633
|
+
}
|
|
1634
|
+
}
|