@centerforagenticai/pi-multi-account 0.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/NOTICE +29 -0
- package/README.md +999 -0
- package/config/models/pi-multi-account.v1.json +32 -0
- package/config/subscription-plans.v1.json +122 -0
- package/package.json +76 -0
- package/packages/pi-anthropic-oauth/LICENSE +21 -0
- package/packages/pi-anthropic-oauth/package.json +54 -0
- package/packages/pi-anthropic-oauth/src/auth.ts +396 -0
- package/packages/pi-anthropic-oauth/src/context.ts +116 -0
- package/packages/pi-anthropic-oauth/src/convert.ts +303 -0
- package/packages/pi-anthropic-oauth/src/index.ts +37 -0
- package/packages/pi-anthropic-oauth/src/prompt.ts +137 -0
- package/packages/pi-anthropic-oauth/src/stream.ts +476 -0
- package/packages/pi-antigravity/LICENSE +21 -0
- package/packages/pi-antigravity/package.json +77 -0
- package/packages/pi-antigravity/src/auth/index.ts +14 -0
- package/packages/pi-antigravity/src/auth/oauth.ts +442 -0
- package/packages/pi-antigravity/src/client/client.ts +561 -0
- package/packages/pi-antigravity/src/client/index.ts +1 -0
- package/packages/pi-antigravity/src/context.ts +110 -0
- package/packages/pi-antigravity/src/diagnostics/diagnostics.ts +96 -0
- package/packages/pi-antigravity/src/diagnostics/index.ts +1 -0
- package/packages/pi-antigravity/src/image/image.ts +336 -0
- package/packages/pi-antigravity/src/image/index.ts +1 -0
- package/packages/pi-antigravity/src/index.ts +280 -0
- package/packages/pi-antigravity/src/models/discovery.ts +154 -0
- package/packages/pi-antigravity/src/models/grouping.ts +424 -0
- package/packages/pi-antigravity/src/models/index.ts +3 -0
- package/packages/pi-antigravity/src/models/models.ts +500 -0
- package/packages/pi-antigravity/src/stream/index.ts +1 -0
- package/packages/pi-antigravity/src/stream/stream.ts +1478 -0
- package/packages/pi-antigravity/src/types/enums.ts +42 -0
- package/packages/pi-antigravity/src/types/index.ts +2 -0
- package/packages/pi-antigravity/src/types/types.ts +292 -0
- package/packages/pi-antigravity/src/usage/index.ts +1 -0
- package/packages/pi-antigravity/src/usage/usage.ts +416 -0
- package/packages/pi-antigravity/src/utils/http.ts +91 -0
- package/packages/pi-antigravity/src/utils/index.ts +3 -0
- package/packages/pi-antigravity/src/utils/security.ts +73 -0
- package/packages/pi-antigravity/src/utils/util.ts +132 -0
- package/scripts/multi-account.mjs +44 -0
- package/src/account-labels.ts +223 -0
- package/src/account-plan-assignment.ts +340 -0
- package/src/account-rate-history.ts +372 -0
- package/src/anthropic-adaptive-stream.ts +531 -0
- package/src/anthropic-alias-stream.ts +140 -0
- package/src/anthropic-context-compat.ts +80 -0
- package/src/api-pricing.ts +579 -0
- package/src/bounded-file-lines.ts +97 -0
- package/src/catalog-rebinding.ts +177 -0
- package/src/catalog-registration-probe.ts +111 -0
- package/src/codex-adapter.ts +345 -0
- package/src/codex-model-defaults.ts +785 -0
- package/src/command-completions.ts +404 -0
- package/src/commands.ts +2000 -0
- package/src/compaction.ts +14 -0
- package/src/config.ts +1317 -0
- package/src/continuation.ts +569 -0
- package/src/cooldowns.ts +110 -0
- package/src/cost-digest-store.ts +332 -0
- package/src/cost-digest.ts +1044 -0
- package/src/cost-history.ts +251 -0
- package/src/cost-period-closer.ts +160 -0
- package/src/cost-report-json.ts +318 -0
- package/src/cost-report-reader.ts +368 -0
- package/src/cost-report-render.ts +207 -0
- package/src/cost-report.ts +1104 -0
- package/src/coverage-attestation.ts +397 -0
- package/src/credential-lifecycle.ts +169 -0
- package/src/credential-refresh.ts +248 -0
- package/src/declaration-notice-marker.ts +238 -0
- package/src/diagnostic-store.ts +276 -0
- package/src/diagnostics.ts +309 -0
- package/src/discovery.ts +471 -0
- package/src/duration.ts +13 -0
- package/src/error-classification.ts +256 -0
- package/src/fuzzy.ts +15 -0
- package/src/group-policy.ts +81 -0
- package/src/history-store.ts +897 -0
- package/src/index.ts +5572 -0
- package/src/lifecycle.ts +378 -0
- package/src/logical-dispatch.ts +279 -0
- package/src/logical-model-selector.ts +254 -0
- package/src/logical-model-switcher.ts +430 -0
- package/src/logical-provider-attribution.ts +544 -0
- package/src/logical-provider.ts +1237 -0
- package/src/logical-route-indicator.ts +215 -0
- package/src/machine-lease.ts +445 -0
- package/src/model-support.ts +66 -0
- package/src/models-declaration.ts +1091 -0
- package/src/openai-adapter.ts +117 -0
- package/src/openrouter-budget.ts +304 -0
- package/src/openrouter-fallback.ts +146 -0
- package/src/period-boundaries.ts +376 -0
- package/src/pi-anthropic-oauth.d.ts +6 -0
- package/src/preflight.ts +253 -0
- package/src/pricing-cache.ts +235 -0
- package/src/project-identity.ts +100 -0
- package/src/provider-registration.ts +942 -0
- package/src/rate-formula.ts +163 -0
- package/src/recovery-engine.ts +853 -0
- package/src/recovery-output.ts +837 -0
- package/src/recovery-plan.ts +239 -0
- package/src/report-range.ts +203 -0
- package/src/route-resolver.ts +789 -0
- package/src/routing-config-transaction.ts +232 -0
- package/src/routing.ts +1163 -0
- package/src/runtime-state.ts +630 -0
- package/src/session-account-groups.ts +284 -0
- package/src/session-restore.ts +287 -0
- package/src/shared-usage.ts +1392 -0
- package/src/standalone-cli.ts +720 -0
- package/src/status-view.ts +578 -0
- package/src/subscription-plan-catalog.ts +346 -0
- package/src/tier-model-resolver.ts +46 -0
- package/src/upstream-anthropic.ts +315 -0
- package/src/upstream-antigravity.ts +327 -0
- package/src/usage-fetch.ts +1634 -0
- package/src/usage.ts +1026 -0
- package/src/vendor.ts +87 -0
- package/src/warmer.ts +231 -0
- package/src/watchdog.ts +219 -0
- package/src/window-history.ts +270 -0
|
@@ -0,0 +1,1392 @@
|
|
|
1
|
+
import {
|
|
2
|
+
appendFileSync,
|
|
3
|
+
chmodSync,
|
|
4
|
+
closeSync,
|
|
5
|
+
fstatSync,
|
|
6
|
+
mkdirSync,
|
|
7
|
+
openSync,
|
|
8
|
+
readSync,
|
|
9
|
+
renameSync,
|
|
10
|
+
statSync,
|
|
11
|
+
unlinkSync,
|
|
12
|
+
writeFileSync,
|
|
13
|
+
} from "node:fs";
|
|
14
|
+
import { randomUUID } from "node:crypto";
|
|
15
|
+
import { hostname } from "node:os";
|
|
16
|
+
import { dirname, join } from "node:path";
|
|
17
|
+
import { iterateBoundedFileLines } from "./bounded-file-lines.js";
|
|
18
|
+
import { isAllowedFamily, type AllowedFamily } from "./config.js";
|
|
19
|
+
import { acquireMachineLease, type MachineLeaseHandle } from "./machine-lease.js";
|
|
20
|
+
import { isCanonicalManagedProviderId } from "./runtime-state.js";
|
|
21
|
+
// The one exhaustion rule, imported rather than restated. A newer reading that
|
|
22
|
+
// this predicate does NOT call exhausted supersedes an older durable recovery
|
|
23
|
+
// time for the same account, so a recovered account is released without waiting
|
|
24
|
+
// for a stale `recoveryAtMs` to elapse. `routing.ts` does not import this
|
|
25
|
+
// module, so this adds no cycle.
|
|
26
|
+
import { snapshotIndicatesExhaustion } from "./routing.js";
|
|
27
|
+
|
|
28
|
+
export const SHARED_USAGE_TTL_MS = 5 * 60_000;
|
|
29
|
+
/**
|
|
30
|
+
* How far ahead a recovery time may plausibly sit.
|
|
31
|
+
*
|
|
32
|
+
* `validTimestamp` accepts any finite non-negative number, so a malformed
|
|
33
|
+
* provider response or a corrupted retained record can carry something like
|
|
34
|
+
* `1e308`. Nothing downstream would ever see that instant pass, so an account
|
|
35
|
+
* excluded by it would stay excluded across every process and every restart --
|
|
36
|
+
* exactly the transient-becomes-permanent failure the routing rules exist to
|
|
37
|
+
* prevent, and the one risk a durable exhaustion signal genuinely adds.
|
|
38
|
+
*
|
|
39
|
+
* Thirty-five days clears the longest real recovery this extension has
|
|
40
|
+
* observed, a monthly quota window, with room to spare. A value beyond it is
|
|
41
|
+
* not a long outage; it is a broken reading, and a broken reading must not be
|
|
42
|
+
* able to retire an account.
|
|
43
|
+
*/
|
|
44
|
+
export const MAX_RECOVERY_HORIZON_MS = 35 * 24 * 60 * 60_000;
|
|
45
|
+
/**
|
|
46
|
+
* How long an account stays out of routing after the provider reported it
|
|
47
|
+
* exhausted without giving a recovery time.
|
|
48
|
+
*
|
|
49
|
+
* Sixty minutes is a policy choice measured against retained observations, not
|
|
50
|
+
* a constant borrowed from elsewhere in this file: of 28 header-reported
|
|
51
|
+
* exhaustions carrying no recovery time, 15 had an authoritative recovery
|
|
52
|
+
* reading within the hour and 13 did not. The long tail is deliberately left
|
|
53
|
+
* uncovered -- the hold is a time-boxed guess that lapses on its own, not a
|
|
54
|
+
* claim to know the account is still spent.
|
|
55
|
+
*/
|
|
56
|
+
export const EXHAUSTION_HOLD_MS = 60 * 60_000;
|
|
57
|
+
|
|
58
|
+
/**
|
|
59
|
+
* Ceiling on a failure-triggered refresh debounce, enforced on append and read.
|
|
60
|
+
*
|
|
61
|
+
* The debounce itself is `USAGE_FETCH_INTERVAL_MS`, defined in `usage-fetch.ts`
|
|
62
|
+
* where the cadence lives. This is only the outer bound a persisted deadline
|
|
63
|
+
* may claim, kept here because validation cannot import from that module. It is
|
|
64
|
+
* deliberately loose: its job is to refuse an absurd value, not to restate the
|
|
65
|
+
* policy.
|
|
66
|
+
*/
|
|
67
|
+
const MAX_REFRESH_DEBOUNCE_MS = 60 * 60_000;
|
|
68
|
+
|
|
69
|
+
export const SHARED_USAGE_MAX_BYTES = 512 * 1024;
|
|
70
|
+
const MAX_RECORD_BYTES = 4_096;
|
|
71
|
+
const MAX_OBSERVER_ID_LENGTH = 256;
|
|
72
|
+
/**
|
|
73
|
+
* Shape a field name must have inside a record type this build does not
|
|
74
|
+
* recognise, applied when deciding whether compaction may carry it forward
|
|
75
|
+
* rather than delete it.
|
|
76
|
+
*
|
|
77
|
+
* A length bound alone was the first attempt and round 3 refuted it: a key
|
|
78
|
+
* called `Bearer SECRET` is short, so the credential simply moved from the
|
|
79
|
+
* value into the key. Field names this project writes are lower-camel
|
|
80
|
+
* identifiers.
|
|
81
|
+
*
|
|
82
|
+
* There is no companion value-length bound: unknown string VALUES are refused
|
|
83
|
+
* outright rather than length-capped, because a bounded short string is
|
|
84
|
+
* exactly the shape of a leaked bearer token.
|
|
85
|
+
*/
|
|
86
|
+
const CARRYABLE_FIELD_NAME = /^[a-z][A-Za-z0-9]{0,63}$/;
|
|
87
|
+
|
|
88
|
+
export type UsageObservationSource = "rate-limit-header" | "usage-endpoint";
|
|
89
|
+
|
|
90
|
+
export interface SharedUsageTokens {
|
|
91
|
+
readonly inputTokens?: number;
|
|
92
|
+
readonly outputTokens?: number;
|
|
93
|
+
readonly cacheCreationInputTokens?: number;
|
|
94
|
+
readonly cacheReadInputTokens?: number;
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
export interface SharedUsageRateLimit {
|
|
98
|
+
readonly remainingRequests?: number;
|
|
99
|
+
readonly remainingTokens?: number;
|
|
100
|
+
readonly recoveryAtMs?: number;
|
|
101
|
+
/** Always normalized to a 0-1 fraction, regardless of source API units. */
|
|
102
|
+
readonly utilization?: number;
|
|
103
|
+
readonly utilizationSource?: UsageObservationSource;
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
export interface SharedUsageRecord {
|
|
107
|
+
readonly providerId: string;
|
|
108
|
+
readonly family: AllowedFamily;
|
|
109
|
+
readonly observedAtMs: number;
|
|
110
|
+
readonly observerId: string;
|
|
111
|
+
readonly tokens?: SharedUsageTokens;
|
|
112
|
+
readonly rateLimit?: SharedUsageRateLimit;
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
export type UsageFailureDetail =
|
|
116
|
+
| "not-object"
|
|
117
|
+
| "no-quota-groups"
|
|
118
|
+
| "quota-summary-error";
|
|
119
|
+
|
|
120
|
+
const USAGE_FAILURE_DETAILS = new Set<UsageFailureDetail>([
|
|
121
|
+
"not-object",
|
|
122
|
+
"no-quota-groups",
|
|
123
|
+
"quota-summary-error",
|
|
124
|
+
]);
|
|
125
|
+
|
|
126
|
+
/** Machine-global fetch-attempt state; deliberately ignored by usage aggregation. */
|
|
127
|
+
export interface SharedUsageAttemptRecord {
|
|
128
|
+
readonly recordType: "usage-attempt";
|
|
129
|
+
/** Null or missing makes pre-0012 readers reject this as a usage observation. */
|
|
130
|
+
readonly tokens?: null;
|
|
131
|
+
readonly providerId: string;
|
|
132
|
+
readonly family: AllowedFamily;
|
|
133
|
+
readonly observedAtMs: number;
|
|
134
|
+
readonly observerId: string;
|
|
135
|
+
readonly failureCount: number;
|
|
136
|
+
readonly nextAttemptAtMs: number;
|
|
137
|
+
readonly disabled: boolean;
|
|
138
|
+
readonly failureReason?: string;
|
|
139
|
+
/** Fixed, sanitized classification detail; never an upstream error body. */
|
|
140
|
+
readonly failureDetail?: UsageFailureDetail;
|
|
141
|
+
/**
|
|
142
|
+
* The failure that triggered this attempt, when it was failure-triggered.
|
|
143
|
+
*
|
|
144
|
+
* Distinguishes a refresh provoked by a rate-limit failure from an ordinary
|
|
145
|
+
* cadence poll, so a later failure can tell whether the account has already
|
|
146
|
+
* been probed since *its own* failure rather than merely recently.
|
|
147
|
+
*/
|
|
148
|
+
readonly failureTriggeredAtMs?: number;
|
|
149
|
+
/**
|
|
150
|
+
* When another failure-triggered refresh may run. Bounded relative to
|
|
151
|
+
* `failureTriggeredAtMs` on read, so a corrupt far-future value cannot
|
|
152
|
+
* suppress refreshes permanently.
|
|
153
|
+
*/
|
|
154
|
+
readonly refreshDebounceUntilMs?: number;
|
|
155
|
+
}
|
|
156
|
+
|
|
157
|
+
/**
|
|
158
|
+
* A bounded, machine-global hold that keeps an account out of routing after the
|
|
159
|
+
* provider reported it exhausted without saying when it recovers.
|
|
160
|
+
*
|
|
161
|
+
* This is deliberately a separate fact rather than a synthesised `recoveryAtMs`.
|
|
162
|
+
* That field means "the provider said so", and aggregation already treats it as
|
|
163
|
+
* authoritative; writing a guessed duration into it would conflate an estimate
|
|
164
|
+
* with authority. A hold is an admitted guess with an expiry.
|
|
165
|
+
*/
|
|
166
|
+
export interface SharedUsageExhaustionHoldRecord {
|
|
167
|
+
readonly recordType: "usage-exhaustion-hold";
|
|
168
|
+
/** Null or missing makes older readers reject this as a usage observation. */
|
|
169
|
+
readonly tokens?: null;
|
|
170
|
+
readonly providerId: string;
|
|
171
|
+
readonly family: AllowedFamily;
|
|
172
|
+
readonly observedAtMs: number;
|
|
173
|
+
readonly observerId: string;
|
|
174
|
+
/** When the failure that installed this hold was classified. */
|
|
175
|
+
readonly failedAtMs: number;
|
|
176
|
+
/** Exclusion end. Bounded relative to `failedAtMs` on append and on read. */
|
|
177
|
+
readonly holdUntilMs: number;
|
|
178
|
+
}
|
|
179
|
+
|
|
180
|
+
export type SharedUsageLogRecord =
|
|
181
|
+
| SharedUsageRecord
|
|
182
|
+
| SharedUsageAttemptRecord
|
|
183
|
+
| SharedUsageExhaustionHoldRecord;
|
|
184
|
+
|
|
185
|
+
export interface SharedUsageSnapshot {
|
|
186
|
+
readonly providerId: string;
|
|
187
|
+
readonly family: AllowedFamily;
|
|
188
|
+
readonly snapshotAtMs: number;
|
|
189
|
+
readonly ageMs: number;
|
|
190
|
+
readonly observerId: string;
|
|
191
|
+
readonly inputTokens: number;
|
|
192
|
+
readonly outputTokens: number;
|
|
193
|
+
readonly cacheCreationInputTokens: number;
|
|
194
|
+
readonly cacheReadInputTokens: number;
|
|
195
|
+
readonly remainingRequests?: number;
|
|
196
|
+
readonly remainingTokens?: number;
|
|
197
|
+
readonly recoveryAtMs?: number;
|
|
198
|
+
readonly utilization?: number;
|
|
199
|
+
readonly utilizationSource?: UsageObservationSource;
|
|
200
|
+
}
|
|
201
|
+
|
|
202
|
+
export interface SharedUsageAggregate {
|
|
203
|
+
readonly providerId: string;
|
|
204
|
+
readonly family: AllowedFamily;
|
|
205
|
+
readonly session?: SharedUsageSnapshot;
|
|
206
|
+
readonly fleet?: SharedUsageSnapshot;
|
|
207
|
+
readonly stale?: SharedUsageSnapshot;
|
|
208
|
+
/**
|
|
209
|
+
* Newest record whose recovery time is still ahead, from any observer and of
|
|
210
|
+
* any age.
|
|
211
|
+
*
|
|
212
|
+
* Separate from the three above because it answers a different question.
|
|
213
|
+
* They ask "what did usage last look like, and who measured it", which the
|
|
214
|
+
* TTL rightly ages out. This asks "is the account known to be out right
|
|
215
|
+
* now", which a five-minute window cannot decide: the answer carries its own
|
|
216
|
+
* expiry in `recoveryAtMs`.
|
|
217
|
+
*
|
|
218
|
+
* Absent once that instant passes, so it can never keep an account excluded
|
|
219
|
+
* after it recovers.
|
|
220
|
+
*/
|
|
221
|
+
readonly durableExhaustion?: SharedUsageSnapshot;
|
|
222
|
+
}
|
|
223
|
+
|
|
224
|
+
function nonNegativeInteger(value: unknown): value is number {
|
|
225
|
+
return Number.isSafeInteger(value) && (value as number) >= 0;
|
|
226
|
+
}
|
|
227
|
+
|
|
228
|
+
function validFraction(value: unknown): value is number {
|
|
229
|
+
return (
|
|
230
|
+
typeof value === "number" &&
|
|
231
|
+
Number.isFinite(value) &&
|
|
232
|
+
value >= 0 &&
|
|
233
|
+
value <= 1
|
|
234
|
+
);
|
|
235
|
+
}
|
|
236
|
+
|
|
237
|
+
function validTimestamp(value: unknown): value is number {
|
|
238
|
+
return typeof value === "number" && Number.isFinite(value) && value >= 0;
|
|
239
|
+
}
|
|
240
|
+
|
|
241
|
+
/**
|
|
242
|
+
* Whether a record this build cannot interpret may survive compaction.
|
|
243
|
+
*
|
|
244
|
+
* Compaction rebuilds the file from recognised records, so anything not carried
|
|
245
|
+
* here is deleted. That is how an older build destroys a record type added
|
|
246
|
+
* after it. Carrying unknown records forward keeps a newer build's state alive
|
|
247
|
+
* across a mixed-version fleet.
|
|
248
|
+
*
|
|
249
|
+
* The filter exists because "unrecognised" also covers records this build
|
|
250
|
+
* rejected as malformed, including the credential-bearing ones that
|
|
251
|
+
* `append` refuses and compaction currently scrubs. Carrying those would turn a
|
|
252
|
+
* durability fix into a privacy regression, so a carried record must still look
|
|
253
|
+
* like a usage record for a known account: a `recordType` string this build
|
|
254
|
+
* does not know, the same account identity fields every record carries, and no
|
|
255
|
+
* field outside that shape. A future record type satisfies this; a leaked
|
|
256
|
+
* authorization header does not.
|
|
257
|
+
*/
|
|
258
|
+
function carryableUnknownRecord(value: unknown): boolean {
|
|
259
|
+
if (typeof value !== "object" || value === null || Array.isArray(value))
|
|
260
|
+
return false;
|
|
261
|
+
const record = value as Record<string, unknown>;
|
|
262
|
+
// `recordType` must look like a record type, not merely be non-empty.
|
|
263
|
+
//
|
|
264
|
+
// Round 2 proved "non-empty string" is not a constraint: `Bearer <token>`
|
|
265
|
+
// is a non-empty string, so a credential placed here survived compaction.
|
|
266
|
+
// A real record type is a lower-kebab identifier, and nothing that fails
|
|
267
|
+
// this shape is a record type this build should carry blind.
|
|
268
|
+
if (
|
|
269
|
+
typeof record.recordType !== "string" ||
|
|
270
|
+
!CARRYABLE_RECORD_TYPE.test(record.recordType)
|
|
271
|
+
)
|
|
272
|
+
return false;
|
|
273
|
+
if (
|
|
274
|
+
typeof record.providerId !== "string" ||
|
|
275
|
+
typeof record.family !== "string" ||
|
|
276
|
+
!isAllowedFamily(record.family) ||
|
|
277
|
+
!isCanonicalManagedProviderId(record.providerId, record.family) ||
|
|
278
|
+
!validTimestamp(record.observedAtMs) ||
|
|
279
|
+
!validObserverId(record.observerId) ||
|
|
280
|
+
// Stricter than `validObserverId` on purpose: that only bounds length,
|
|
281
|
+
// and a bearer token is a bounded string. See CARRYABLE_OBSERVER_ID.
|
|
282
|
+
!CARRYABLE_OBSERVER_ID.test(record.observerId)
|
|
283
|
+
)
|
|
284
|
+
return false;
|
|
285
|
+
// Beyond the identity fields above, a carried record may hold only
|
|
286
|
+
// NON-STRING values.
|
|
287
|
+
//
|
|
288
|
+
// The first version of this filter bounded value shape -- primitives only,
|
|
289
|
+
// length-capped -- and review proved it unsound: `{ authorization: "Bearer
|
|
290
|
+
// SHORT" }` is a bounded primitive and survived compaction, which is exactly
|
|
291
|
+
// the privacy regression the carry-through was not allowed to create.
|
|
292
|
+
//
|
|
293
|
+
// Refusing unknown strings outright is the only defensible rule here. A
|
|
294
|
+
// denylist of sensitive-looking field names would be a guess about what a
|
|
295
|
+
// future record type calls its fields, and every credential this project
|
|
296
|
+
// handles is a string. Timestamps, counts, fractions and flags -- what a
|
|
297
|
+
// forward-compatible usage record actually needs -- are unaffected. A future
|
|
298
|
+
// type that genuinely needs a string field must teach this build about
|
|
299
|
+
// itself rather than rely on being carried blind.
|
|
300
|
+
for (const [key, entry] of Object.entries(record)) {
|
|
301
|
+
// Field NAMES are constrained by shape, not only length. Round 3 found
|
|
302
|
+
// a length bound alone lets a key called `Bearer SECRET` through: the
|
|
303
|
+
// credential rides in the key rather than the value. A field name in a
|
|
304
|
+
// JSON record written by this project is a lower-camel identifier.
|
|
305
|
+
if (!CARRYABLE_FIELD_NAME.test(key)) return false;
|
|
306
|
+
if (CARRYABLE_IDENTITY_FIELDS.has(key)) continue;
|
|
307
|
+
if (!carryableUnknownValue(entry)) return false;
|
|
308
|
+
}
|
|
309
|
+
return true;
|
|
310
|
+
}
|
|
311
|
+
|
|
312
|
+
/**
|
|
313
|
+
* Record types compaction may carry forward: the project's own namespace.
|
|
314
|
+
*
|
|
315
|
+
* Two weaker rules were tried and both refuted. "Non-empty" fell to
|
|
316
|
+
* `Bearer <token>` in round 2. A lower-kebab shape fell in round 3 to
|
|
317
|
+
* `sk-ant-api03-deadbeef`, which IS lower-kebab -- an API key and a record
|
|
318
|
+
* type are not distinguishable by shape, so no amount of character-class
|
|
319
|
+
* tightening can separate them.
|
|
320
|
+
*
|
|
321
|
+
* A namespace can. Every record type this file writes is `usage-`-prefixed
|
|
322
|
+
* (`usage-attempt`, `usage-exhaustion-hold`), so a future type from a newer
|
|
323
|
+
* build will be too. That is a property of the writer rather than a guess
|
|
324
|
+
* about what a credential looks like, which is why it holds where the shape
|
|
325
|
+
* checks did not.
|
|
326
|
+
*/
|
|
327
|
+
const CARRYABLE_RECORD_TYPE = /^usage-[a-z][a-z0-9-]{0,56}$/;
|
|
328
|
+
|
|
329
|
+
/**
|
|
330
|
+
* Shape a carried record's `observerId` must have.
|
|
331
|
+
*
|
|
332
|
+
* `validObserverId` only bounds length, and round 2 proved that insufficient:
|
|
333
|
+
* a bearer token is a bounded string. This restricts the CHARACTER SET instead,
|
|
334
|
+
* which is what makes the field unusable for smuggling while still accepting
|
|
335
|
+
* everything the producer can emit.
|
|
336
|
+
*
|
|
337
|
+
* The colon is deliberately NOT required. `defaultObserverId` builds
|
|
338
|
+
* `${hostname()}:${process.pid}` and then truncates to MAX_OBSERVER_ID_LENGTH,
|
|
339
|
+
* so on a host with a very long name the pid -- and the colon with it -- is cut
|
|
340
|
+
* off entirely. Round 3 found an earlier version of this expression required
|
|
341
|
+
* the colon, which would have made compaction DELETE legitimate records on such
|
|
342
|
+
* a machine: the precise data loss this carry-through exists to prevent, caused
|
|
343
|
+
* by the fix for it. A verified probe produced a 256-character id with no colon
|
|
344
|
+
* at all.
|
|
345
|
+
*
|
|
346
|
+
* The length bound matches MAX_OBSERVER_ID_LENGTH rather than guessing a
|
|
347
|
+
* narrower one, so the accepted domain covers every value the producer can
|
|
348
|
+
* actually return.
|
|
349
|
+
*/
|
|
350
|
+
const CARRYABLE_OBSERVER_ID = /^[A-Za-z0-9._:-]{1,256}$/;
|
|
351
|
+
|
|
352
|
+
/**
|
|
353
|
+
* Identity fields every record carries. They are the only strings a carried
|
|
354
|
+
* unknown record may contain, and each is format-checked above rather than
|
|
355
|
+
* merely bounded.
|
|
356
|
+
*/
|
|
357
|
+
const CARRYABLE_IDENTITY_FIELDS = new Set([
|
|
358
|
+
"recordType",
|
|
359
|
+
"providerId",
|
|
360
|
+
"family",
|
|
361
|
+
"observerId",
|
|
362
|
+
]);
|
|
363
|
+
|
|
364
|
+
function carryableUnknownValue(value: unknown): boolean {
|
|
365
|
+
return (
|
|
366
|
+
value === null || typeof value === "boolean" || typeof value === "number"
|
|
367
|
+
);
|
|
368
|
+
}
|
|
369
|
+
|
|
370
|
+
function validObserverId(value: unknown): value is string {
|
|
371
|
+
return (
|
|
372
|
+
typeof value === "string" &&
|
|
373
|
+
value.length > 0 &&
|
|
374
|
+
value.length <= MAX_OBSERVER_ID_LENGTH
|
|
375
|
+
);
|
|
376
|
+
}
|
|
377
|
+
|
|
378
|
+
const TOKEN_FIELDS = [
|
|
379
|
+
"inputTokens",
|
|
380
|
+
"outputTokens",
|
|
381
|
+
"cacheCreationInputTokens",
|
|
382
|
+
"cacheReadInputTokens",
|
|
383
|
+
] as const;
|
|
384
|
+
const RATE_LIMIT_FIELDS = [
|
|
385
|
+
"remainingRequests",
|
|
386
|
+
"remainingTokens",
|
|
387
|
+
"recoveryAtMs",
|
|
388
|
+
"utilization",
|
|
389
|
+
"utilizationSource",
|
|
390
|
+
] as const;
|
|
391
|
+
|
|
392
|
+
function validRecord(value: unknown): value is SharedUsageRecord {
|
|
393
|
+
if (typeof value !== "object" || value === null || Array.isArray(value))
|
|
394
|
+
return false;
|
|
395
|
+
const record = value as Record<string, unknown>;
|
|
396
|
+
if (
|
|
397
|
+
record.recordType !== undefined ||
|
|
398
|
+
typeof record.providerId !== "string" ||
|
|
399
|
+
typeof record.family !== "string" ||
|
|
400
|
+
!isAllowedFamily(record.family) ||
|
|
401
|
+
!isCanonicalManagedProviderId(record.providerId, record.family) ||
|
|
402
|
+
!validTimestamp(record.observedAtMs) ||
|
|
403
|
+
!validObserverId(record.observerId)
|
|
404
|
+
)
|
|
405
|
+
return false;
|
|
406
|
+
if (record.tokens !== undefined) {
|
|
407
|
+
if (typeof record.tokens !== "object" || record.tokens === null)
|
|
408
|
+
return false;
|
|
409
|
+
for (const field of TOKEN_FIELDS) {
|
|
410
|
+
const value = (record.tokens as Record<string, unknown>)[field];
|
|
411
|
+
if (value !== undefined && !nonNegativeInteger(value)) return false;
|
|
412
|
+
}
|
|
413
|
+
}
|
|
414
|
+
if (record.rateLimit !== undefined) {
|
|
415
|
+
if (typeof record.rateLimit !== "object" || record.rateLimit === null)
|
|
416
|
+
return false;
|
|
417
|
+
const rateLimit = record.rateLimit as Record<string, unknown>;
|
|
418
|
+
for (const field of ["remainingRequests", "remainingTokens"] as const) {
|
|
419
|
+
const value = rateLimit[field];
|
|
420
|
+
if (value !== undefined && !nonNegativeInteger(value)) return false;
|
|
421
|
+
}
|
|
422
|
+
// A recovery time implausibly far after its own observation is a broken
|
|
423
|
+
// reading, and this is the only place that says so. Letting one in would
|
|
424
|
+
// mean every reader had to remember to distrust it.
|
|
425
|
+
//
|
|
426
|
+
// The `observedAtMs` anchor is what makes the question decidable here:
|
|
427
|
+
// "365 days after someone looked" is wrong on sight, whereas "within 35
|
|
428
|
+
// days of now" has no answer at the moment a record is written. It is
|
|
429
|
+
// also what keeps the verdict stable. Anchored to the clock instead, the
|
|
430
|
+
// same value is refused today and admitted months later when it drifts
|
|
431
|
+
// into range -- not a bound, but a delayed admission of a record already
|
|
432
|
+
// judged broken once.
|
|
433
|
+
//
|
|
434
|
+
// `validRecord` guards the read path as well as the append path, so a
|
|
435
|
+
// record already on disk -- written before this check existed, or
|
|
436
|
+
// corrupted since -- is dropped when it is read. That is why no second
|
|
437
|
+
// check guards selection: there is no route by which such a record
|
|
438
|
+
// reaches a reader, and an unreachable guard is a claim no test can keep
|
|
439
|
+
// honest.
|
|
440
|
+
if (
|
|
441
|
+
rateLimit.recoveryAtMs !== undefined &&
|
|
442
|
+
(!validTimestamp(rateLimit.recoveryAtMs) ||
|
|
443
|
+
rateLimit.recoveryAtMs >
|
|
444
|
+
record.observedAtMs + MAX_RECOVERY_HORIZON_MS)
|
|
445
|
+
)
|
|
446
|
+
return false;
|
|
447
|
+
if (
|
|
448
|
+
rateLimit.utilization !== undefined &&
|
|
449
|
+
!validFraction(rateLimit.utilization)
|
|
450
|
+
)
|
|
451
|
+
return false;
|
|
452
|
+
if (
|
|
453
|
+
rateLimit.utilizationSource !== undefined &&
|
|
454
|
+
rateLimit.utilizationSource !== "rate-limit-header" &&
|
|
455
|
+
rateLimit.utilizationSource !== "usage-endpoint"
|
|
456
|
+
)
|
|
457
|
+
return false;
|
|
458
|
+
}
|
|
459
|
+
return true;
|
|
460
|
+
}
|
|
461
|
+
|
|
462
|
+
function validAttemptRecord(value: unknown): value is SharedUsageAttemptRecord {
|
|
463
|
+
if (typeof value !== "object" || value === null || Array.isArray(value))
|
|
464
|
+
return false;
|
|
465
|
+
const record = value as Record<string, unknown>;
|
|
466
|
+
return (
|
|
467
|
+
record.recordType === "usage-attempt" &&
|
|
468
|
+
(record.tokens === undefined || record.tokens === null) &&
|
|
469
|
+
typeof record.providerId === "string" &&
|
|
470
|
+
typeof record.family === "string" &&
|
|
471
|
+
isAllowedFamily(record.family) &&
|
|
472
|
+
isCanonicalManagedProviderId(record.providerId, record.family) &&
|
|
473
|
+
validTimestamp(record.observedAtMs) &&
|
|
474
|
+
validObserverId(record.observerId) &&
|
|
475
|
+
nonNegativeInteger(record.failureCount) &&
|
|
476
|
+
validTimestamp(record.nextAttemptAtMs) &&
|
|
477
|
+
typeof record.disabled === "boolean" &&
|
|
478
|
+
(record.failureReason === undefined ||
|
|
479
|
+
(typeof record.failureReason === "string" &&
|
|
480
|
+
record.failureReason.length <= 128)) &&
|
|
481
|
+
(record.failureDetail === undefined ||
|
|
482
|
+
(typeof record.failureDetail === "string" &&
|
|
483
|
+
USAGE_FAILURE_DETAILS.has(record.failureDetail as UsageFailureDetail))) &&
|
|
484
|
+
(record.failureTriggeredAtMs === undefined ||
|
|
485
|
+
validTimestamp(record.failureTriggeredAtMs)) &&
|
|
486
|
+
// The debounce deadline is bounded against the failure that set it, not
|
|
487
|
+
// the clock. Without this a corrupt far-future value would suppress every
|
|
488
|
+
// failure-triggered refresh for that account forever, turning a
|
|
489
|
+
// five-minute debounce into a permanent one.
|
|
490
|
+
(record.refreshDebounceUntilMs === undefined ||
|
|
491
|
+
(validTimestamp(record.refreshDebounceUntilMs) &&
|
|
492
|
+
record.failureTriggeredAtMs !== undefined &&
|
|
493
|
+
record.refreshDebounceUntilMs >= record.failureTriggeredAtMs &&
|
|
494
|
+
record.refreshDebounceUntilMs - record.failureTriggeredAtMs <=
|
|
495
|
+
MAX_REFRESH_DEBOUNCE_MS))
|
|
496
|
+
);
|
|
497
|
+
}
|
|
498
|
+
|
|
499
|
+
/**
|
|
500
|
+
* Validates an exhaustion hold on both append and read.
|
|
501
|
+
*
|
|
502
|
+
* The duration bound is anchored to `failedAtMs`, the record's own observation
|
|
503
|
+
* of when the failure happened, rather than to the current clock. A
|
|
504
|
+
* clock-anchored bound is a sliding window: it silently admits a far-future
|
|
505
|
+
* value once enough time passes. Anchoring to the record makes "longer than the
|
|
506
|
+
* hold we would ever install" decidable from the record alone, so a corrupt or
|
|
507
|
+
* hostile `holdUntilMs` cannot park an account out of routing indefinitely.
|
|
508
|
+
*/
|
|
509
|
+
function validExhaustionHoldRecord(
|
|
510
|
+
value: unknown,
|
|
511
|
+
): value is SharedUsageExhaustionHoldRecord {
|
|
512
|
+
if (typeof value !== "object" || value === null || Array.isArray(value))
|
|
513
|
+
return false;
|
|
514
|
+
const record = value as Record<string, unknown>;
|
|
515
|
+
return (
|
|
516
|
+
record.recordType === "usage-exhaustion-hold" &&
|
|
517
|
+
(record.tokens === undefined || record.tokens === null) &&
|
|
518
|
+
typeof record.providerId === "string" &&
|
|
519
|
+
typeof record.family === "string" &&
|
|
520
|
+
isAllowedFamily(record.family) &&
|
|
521
|
+
isCanonicalManagedProviderId(record.providerId, record.family) &&
|
|
522
|
+
validTimestamp(record.observedAtMs) &&
|
|
523
|
+
validObserverId(record.observerId) &&
|
|
524
|
+
validTimestamp(record.failedAtMs) &&
|
|
525
|
+
validTimestamp(record.holdUntilMs) &&
|
|
526
|
+
record.holdUntilMs >= record.failedAtMs &&
|
|
527
|
+
record.holdUntilMs - record.failedAtMs <= EXHAUSTION_HOLD_MS
|
|
528
|
+
);
|
|
529
|
+
}
|
|
530
|
+
|
|
531
|
+
function projectExhaustionHold(
|
|
532
|
+
record: SharedUsageExhaustionHoldRecord,
|
|
533
|
+
): SharedUsageExhaustionHoldRecord {
|
|
534
|
+
return {
|
|
535
|
+
recordType: "usage-exhaustion-hold",
|
|
536
|
+
tokens: null,
|
|
537
|
+
providerId: record.providerId,
|
|
538
|
+
family: record.family,
|
|
539
|
+
observedAtMs: record.observedAtMs,
|
|
540
|
+
observerId: record.observerId.slice(0, MAX_OBSERVER_ID_LENGTH),
|
|
541
|
+
failedAtMs: record.failedAtMs,
|
|
542
|
+
holdUntilMs: record.holdUntilMs,
|
|
543
|
+
};
|
|
544
|
+
}
|
|
545
|
+
|
|
546
|
+
function defaultStorePath(): string {
|
|
547
|
+
const agentDir =
|
|
548
|
+
process.env.PI_CODING_AGENT_DIR ??
|
|
549
|
+
join(process.env.HOME ?? ".", ".pi", "agent");
|
|
550
|
+
// Package storage identity stays decoupled from the logical provider ID.
|
|
551
|
+
return join(agentDir, "pi-multi-account", "usage.ndjson");
|
|
552
|
+
}
|
|
553
|
+
|
|
554
|
+
/** Identity is bounded metadata only; it contains no credential-derived value. */
|
|
555
|
+
export function defaultObserverId(): string {
|
|
556
|
+
return `${hostname()}:${process.pid}`.slice(0, MAX_OBSERVER_ID_LENGTH);
|
|
557
|
+
}
|
|
558
|
+
|
|
559
|
+
function projectRecord(record: SharedUsageRecord): SharedUsageRecord {
|
|
560
|
+
const tokens = record.tokens;
|
|
561
|
+
const rateLimit = record.rateLimit;
|
|
562
|
+
const projectedTokens =
|
|
563
|
+
tokens === undefined
|
|
564
|
+
? undefined
|
|
565
|
+
: (Object.fromEntries(
|
|
566
|
+
TOKEN_FIELDS.flatMap((field) =>
|
|
567
|
+
tokens[field] === undefined ? [] : [[field, tokens[field]]],
|
|
568
|
+
),
|
|
569
|
+
) as SharedUsageTokens);
|
|
570
|
+
const projectedRateLimit =
|
|
571
|
+
rateLimit === undefined
|
|
572
|
+
? undefined
|
|
573
|
+
: (Object.fromEntries(
|
|
574
|
+
RATE_LIMIT_FIELDS.flatMap((field) =>
|
|
575
|
+
rateLimit[field] === undefined ? [] : [[field, rateLimit[field]]],
|
|
576
|
+
),
|
|
577
|
+
) as SharedUsageRateLimit);
|
|
578
|
+
return {
|
|
579
|
+
providerId: record.providerId,
|
|
580
|
+
family: record.family,
|
|
581
|
+
observedAtMs: record.observedAtMs,
|
|
582
|
+
observerId: record.observerId.slice(0, MAX_OBSERVER_ID_LENGTH),
|
|
583
|
+
...(projectedTokens === undefined ? {} : { tokens: projectedTokens }),
|
|
584
|
+
...(projectedRateLimit === undefined
|
|
585
|
+
? {}
|
|
586
|
+
: { rateLimit: projectedRateLimit }),
|
|
587
|
+
};
|
|
588
|
+
}
|
|
589
|
+
|
|
590
|
+
function projectAttempt(
|
|
591
|
+
record: SharedUsageAttemptRecord,
|
|
592
|
+
): SharedUsageAttemptRecord {
|
|
593
|
+
return {
|
|
594
|
+
recordType: "usage-attempt",
|
|
595
|
+
tokens: null,
|
|
596
|
+
providerId: record.providerId,
|
|
597
|
+
family: record.family,
|
|
598
|
+
observedAtMs: record.observedAtMs,
|
|
599
|
+
observerId: record.observerId.slice(0, MAX_OBSERVER_ID_LENGTH),
|
|
600
|
+
failureCount: record.failureCount,
|
|
601
|
+
nextAttemptAtMs: record.nextAttemptAtMs,
|
|
602
|
+
disabled: record.disabled,
|
|
603
|
+
...(record.failureReason === undefined
|
|
604
|
+
? {}
|
|
605
|
+
: { failureReason: record.failureReason }),
|
|
606
|
+
...(record.failureDetail === undefined
|
|
607
|
+
? {}
|
|
608
|
+
: { failureDetail: record.failureDetail }),
|
|
609
|
+
// Carried through the attempt's success and failure outcomes: a debounce
|
|
610
|
+
// that vanished when the attempt completed would let the next failure
|
|
611
|
+
// poll again immediately, which is the stampede this exists to stop.
|
|
612
|
+
...(record.failureTriggeredAtMs === undefined
|
|
613
|
+
? {}
|
|
614
|
+
: { failureTriggeredAtMs: record.failureTriggeredAtMs }),
|
|
615
|
+
...(record.refreshDebounceUntilMs === undefined
|
|
616
|
+
? {}
|
|
617
|
+
: { refreshDebounceUntilMs: record.refreshDebounceUntilMs }),
|
|
618
|
+
};
|
|
619
|
+
}
|
|
620
|
+
|
|
621
|
+
function* completeUsageLines(
|
|
622
|
+
path: string,
|
|
623
|
+
maxBytes?: number,
|
|
624
|
+
): Generator<string> {
|
|
625
|
+
const limit = 10_000;
|
|
626
|
+
const ring = new Array<string>(limit);
|
|
627
|
+
let count = 0;
|
|
628
|
+
let next = 0;
|
|
629
|
+
for (const line of iterateBoundedFileLines(path, {
|
|
630
|
+
maxLineBytes: MAX_RECORD_BYTES,
|
|
631
|
+
includeIncompleteFinalLine: false,
|
|
632
|
+
...(maxBytes === undefined ? {} : { maxBytes }),
|
|
633
|
+
})) {
|
|
634
|
+
ring[next] = line;
|
|
635
|
+
next = (next + 1) % limit;
|
|
636
|
+
count = Math.min(count + 1, limit);
|
|
637
|
+
}
|
|
638
|
+
const start = count === limit ? next : 0;
|
|
639
|
+
for (let index = 0; index < count; index += 1) {
|
|
640
|
+
const line = ring[(start + index) % limit];
|
|
641
|
+
if (line !== undefined) yield line;
|
|
642
|
+
}
|
|
643
|
+
}
|
|
644
|
+
|
|
645
|
+
function parseRecords(lines: Iterable<string>): readonly SharedUsageRecord[] {
|
|
646
|
+
const records: SharedUsageRecord[] = [];
|
|
647
|
+
for (const line of lines) {
|
|
648
|
+
try {
|
|
649
|
+
const parsed: unknown = JSON.parse(line);
|
|
650
|
+
if (validRecord(parsed)) records.push(projectRecord(parsed));
|
|
651
|
+
} catch {
|
|
652
|
+
// One malformed record must not hide the rest of the append log.
|
|
653
|
+
}
|
|
654
|
+
}
|
|
655
|
+
return records;
|
|
656
|
+
}
|
|
657
|
+
|
|
658
|
+
function parseAttempts(lines: Iterable<string>): readonly SharedUsageAttemptRecord[] {
|
|
659
|
+
const attempts: SharedUsageAttemptRecord[] = [];
|
|
660
|
+
for (const line of lines) {
|
|
661
|
+
try {
|
|
662
|
+
const parsed: unknown = JSON.parse(line);
|
|
663
|
+
if (validAttemptRecord(parsed)) attempts.push(projectAttempt(parsed));
|
|
664
|
+
} catch {
|
|
665
|
+
// Corrupt state is ignored; callers then degrade to local behaviour.
|
|
666
|
+
}
|
|
667
|
+
}
|
|
668
|
+
return attempts;
|
|
669
|
+
}
|
|
670
|
+
|
|
671
|
+
function parseExhaustionHolds(
|
|
672
|
+
lines: Iterable<string>,
|
|
673
|
+
): readonly SharedUsageExhaustionHoldRecord[] {
|
|
674
|
+
const holds: SharedUsageExhaustionHoldRecord[] = [];
|
|
675
|
+
for (const line of lines) {
|
|
676
|
+
try {
|
|
677
|
+
const parsed: unknown = JSON.parse(line);
|
|
678
|
+
if (validExhaustionHoldRecord(parsed))
|
|
679
|
+
holds.push(projectExhaustionHold(parsed));
|
|
680
|
+
} catch {
|
|
681
|
+
// Corrupt state is ignored; callers then degrade to local behaviour.
|
|
682
|
+
}
|
|
683
|
+
}
|
|
684
|
+
return holds;
|
|
685
|
+
}
|
|
686
|
+
|
|
687
|
+
function snapshotFromRecord(
|
|
688
|
+
record: SharedUsageRecord,
|
|
689
|
+
nowMs: number,
|
|
690
|
+
): SharedUsageSnapshot {
|
|
691
|
+
const ageMs = Math.max(0, nowMs - record.observedAtMs);
|
|
692
|
+
return {
|
|
693
|
+
providerId: record.providerId,
|
|
694
|
+
family: record.family,
|
|
695
|
+
snapshotAtMs: record.observedAtMs,
|
|
696
|
+
ageMs,
|
|
697
|
+
observerId: record.observerId,
|
|
698
|
+
inputTokens: record.tokens?.inputTokens ?? 0,
|
|
699
|
+
outputTokens: record.tokens?.outputTokens ?? 0,
|
|
700
|
+
cacheCreationInputTokens: record.tokens?.cacheCreationInputTokens ?? 0,
|
|
701
|
+
cacheReadInputTokens: record.tokens?.cacheReadInputTokens ?? 0,
|
|
702
|
+
...(record.rateLimit?.remainingRequests === undefined
|
|
703
|
+
? {}
|
|
704
|
+
: { remainingRequests: record.rateLimit.remainingRequests }),
|
|
705
|
+
...(record.rateLimit?.remainingTokens === undefined
|
|
706
|
+
? {}
|
|
707
|
+
: { remainingTokens: record.rateLimit.remainingTokens }),
|
|
708
|
+
...(record.rateLimit?.recoveryAtMs === undefined
|
|
709
|
+
? {}
|
|
710
|
+
: { recoveryAtMs: record.rateLimit.recoveryAtMs }),
|
|
711
|
+
...(record.rateLimit?.utilization === undefined
|
|
712
|
+
? {}
|
|
713
|
+
: { utilization: record.rateLimit.utilization }),
|
|
714
|
+
...(record.rateLimit?.utilizationSource === undefined
|
|
715
|
+
? {}
|
|
716
|
+
: { utilizationSource: record.rateLimit.utilizationSource }),
|
|
717
|
+
};
|
|
718
|
+
}
|
|
719
|
+
|
|
720
|
+
function appendState(path: string): { readonly size: number; readonly separator: string } {
|
|
721
|
+
let descriptor: number | undefined;
|
|
722
|
+
try {
|
|
723
|
+
const size = statSync(path).size;
|
|
724
|
+
if (size === 0) return { size, separator: "" };
|
|
725
|
+
descriptor = openSync(path, "r");
|
|
726
|
+
const finalByte = Buffer.allocUnsafe(1);
|
|
727
|
+
const bytesRead = readSync(descriptor, finalByte, 0, 1, size - 1);
|
|
728
|
+
return {
|
|
729
|
+
size,
|
|
730
|
+
separator: bytesRead === 1 && finalByte[0] === 0x0a ? "" : "\n",
|
|
731
|
+
};
|
|
732
|
+
} catch (error) {
|
|
733
|
+
if ((error as NodeJS.ErrnoException).code === "ENOENT") {
|
|
734
|
+
return { size: 0, separator: "" };
|
|
735
|
+
}
|
|
736
|
+
throw error;
|
|
737
|
+
} finally {
|
|
738
|
+
if (descriptor !== undefined) closeSync(descriptor);
|
|
739
|
+
}
|
|
740
|
+
}
|
|
741
|
+
|
|
742
|
+
function completeUsagePrefixBytes(path: string, size: number): number | undefined {
|
|
743
|
+
let descriptor: number | undefined;
|
|
744
|
+
try {
|
|
745
|
+
if (size === 0) return 0;
|
|
746
|
+
const bytesToRead = Math.min(size, MAX_RECORD_BYTES + 1);
|
|
747
|
+
const start = size - bytesToRead;
|
|
748
|
+
const buffer = Buffer.allocUnsafe(bytesToRead);
|
|
749
|
+
descriptor = openSync(path, "r");
|
|
750
|
+
const bytesRead = readSync(descriptor, buffer, 0, bytesToRead, start);
|
|
751
|
+
const bytes = buffer.subarray(0, bytesRead);
|
|
752
|
+
if (bytes.at(-1) === 0x0a) return size;
|
|
753
|
+
const lastNewline = bytes.lastIndexOf(0x0a);
|
|
754
|
+
return start > 0 && lastNewline < 0 ? undefined : start + lastNewline + 1;
|
|
755
|
+
} catch {
|
|
756
|
+
return undefined;
|
|
757
|
+
} finally {
|
|
758
|
+
if (descriptor !== undefined) closeSync(descriptor);
|
|
759
|
+
}
|
|
760
|
+
}
|
|
761
|
+
|
|
762
|
+
function stableUsageTail(options: {
|
|
763
|
+
readonly path: string;
|
|
764
|
+
readonly start: number;
|
|
765
|
+
readonly maxBytes: number;
|
|
766
|
+
readonly expectedDevice: number;
|
|
767
|
+
readonly expectedInode: number;
|
|
768
|
+
}): { readonly bytes: Buffer; readonly sourceSize: number } | undefined {
|
|
769
|
+
for (let attempt = 0; attempt < 3; attempt += 1) {
|
|
770
|
+
let descriptor: number | undefined;
|
|
771
|
+
try {
|
|
772
|
+
const before = statSync(options.path);
|
|
773
|
+
if (
|
|
774
|
+
before.dev !== options.expectedDevice ||
|
|
775
|
+
before.ino !== options.expectedInode ||
|
|
776
|
+
before.size < options.start ||
|
|
777
|
+
before.size - options.start > options.maxBytes
|
|
778
|
+
) {
|
|
779
|
+
return undefined;
|
|
780
|
+
}
|
|
781
|
+
descriptor = openSync(options.path, "r");
|
|
782
|
+
const opened = fstatSync(descriptor);
|
|
783
|
+
if (opened.dev !== before.dev || opened.ino !== before.ino) continue;
|
|
784
|
+
const bytes = Buffer.allocUnsafe(before.size - options.start);
|
|
785
|
+
let offset = 0;
|
|
786
|
+
while (offset < bytes.length) {
|
|
787
|
+
const bytesRead = readSync(
|
|
788
|
+
descriptor,
|
|
789
|
+
bytes,
|
|
790
|
+
offset,
|
|
791
|
+
bytes.length - offset,
|
|
792
|
+
options.start + offset,
|
|
793
|
+
);
|
|
794
|
+
if (bytesRead === 0) break;
|
|
795
|
+
offset += bytesRead;
|
|
796
|
+
}
|
|
797
|
+
const after = statSync(options.path);
|
|
798
|
+
if (
|
|
799
|
+
offset === bytes.length &&
|
|
800
|
+
after.dev === before.dev &&
|
|
801
|
+
after.ino === before.ino &&
|
|
802
|
+
after.size === before.size
|
|
803
|
+
) {
|
|
804
|
+
return { bytes, sourceSize: before.size };
|
|
805
|
+
}
|
|
806
|
+
} catch {
|
|
807
|
+
// Retry a moving append snapshot; publication remains fail-soft.
|
|
808
|
+
} finally {
|
|
809
|
+
if (descriptor !== undefined) closeSync(descriptor);
|
|
810
|
+
}
|
|
811
|
+
}
|
|
812
|
+
return undefined;
|
|
813
|
+
}
|
|
814
|
+
|
|
815
|
+
function compactUsageFile(options: {
|
|
816
|
+
readonly path: string;
|
|
817
|
+
readonly maxBytes: number;
|
|
818
|
+
readonly beforeRename?: () => void;
|
|
819
|
+
readonly mutationLease: MachineLeaseHandle;
|
|
820
|
+
readonly ownerLease?: { renew(): boolean };
|
|
821
|
+
}): boolean {
|
|
822
|
+
try {
|
|
823
|
+
const initial = statSync(options.path);
|
|
824
|
+
if (!initial.isFile()) return false;
|
|
825
|
+
const completePrefixBytes = completeUsagePrefixBytes(options.path, initial.size);
|
|
826
|
+
if (completePrefixBytes === undefined) return false;
|
|
827
|
+
const latest = new Map<string, SharedUsageRecord>();
|
|
828
|
+
const latestTokens = new Map<string, SharedUsageRecord>();
|
|
829
|
+
const latestRateLimits = new Map<string, SharedUsageRecord>();
|
|
830
|
+
const latestAttempts = new Map<string, SharedUsageAttemptRecord>();
|
|
831
|
+
const latestHolds = new Map<string, SharedUsageExhaustionHoldRecord>();
|
|
832
|
+
// Records this build has no reader for are carried through verbatim.
|
|
833
|
+
//
|
|
834
|
+
// Compaction rebuilds the file from what it recognises, so without this a
|
|
835
|
+
// process running an older build silently deletes every record type added
|
|
836
|
+
// after it -- including the exhaustion holds that keep a spent account out
|
|
837
|
+
// of rotation. The account then looks healthy
|
|
838
|
+
// to the next process and gets routed to again.
|
|
839
|
+
//
|
|
840
|
+
// Carrying them verbatim rather than re-serialising keeps this build from
|
|
841
|
+
// imposing a shape on data it does not understand, and matches how the
|
|
842
|
+
// writer tail below is copied byte-for-byte.
|
|
843
|
+
//
|
|
844
|
+
// This is deliberately limited to whole unrecognised *records*.
|
|
845
|
+
// Unrecognised *fields* on a recognised record are still stripped by
|
|
846
|
+
// projectRecord/projectAttempt: that projection is the credential scrub
|
|
847
|
+
// required by AGENTS.md, and widening it here would trade a privacy
|
|
848
|
+
// guarantee for a durability one.
|
|
849
|
+
const unrecognisedLines: string[] = [];
|
|
850
|
+
|
|
851
|
+
for (const line of completeUsageLines(options.path, completePrefixBytes)) {
|
|
852
|
+
let parsed: unknown;
|
|
853
|
+
try {
|
|
854
|
+
parsed = JSON.parse(line);
|
|
855
|
+
} catch {
|
|
856
|
+
continue;
|
|
857
|
+
}
|
|
858
|
+
if (validRecord(parsed)) {
|
|
859
|
+
const record = projectRecord(parsed);
|
|
860
|
+
const key = JSON.stringify([record.providerId, record.observerId]);
|
|
861
|
+
const previous = latest.get(key);
|
|
862
|
+
if (!previous || record.observedAtMs > previous.observedAtMs) {
|
|
863
|
+
latest.set(key, record);
|
|
864
|
+
}
|
|
865
|
+
if (record.tokens !== undefined) {
|
|
866
|
+
const previousTokens = latestTokens.get(key);
|
|
867
|
+
if (!previousTokens || record.observedAtMs > previousTokens.observedAtMs) {
|
|
868
|
+
latestTokens.set(key, record);
|
|
869
|
+
}
|
|
870
|
+
}
|
|
871
|
+
if (record.rateLimit !== undefined) {
|
|
872
|
+
const previousRateLimit = latestRateLimits.get(key);
|
|
873
|
+
if (
|
|
874
|
+
!previousRateLimit ||
|
|
875
|
+
record.observedAtMs > previousRateLimit.observedAtMs
|
|
876
|
+
) {
|
|
877
|
+
latestRateLimits.set(key, record);
|
|
878
|
+
}
|
|
879
|
+
}
|
|
880
|
+
} else if (validAttemptRecord(parsed)) {
|
|
881
|
+
const attempt = projectAttempt(parsed);
|
|
882
|
+
const previous = latestAttempts.get(attempt.providerId);
|
|
883
|
+
if (!previous || attempt.observedAtMs >= previous.observedAtMs) {
|
|
884
|
+
latestAttempts.set(attempt.providerId, attempt);
|
|
885
|
+
}
|
|
886
|
+
} else if (validExhaustionHoldRecord(parsed)) {
|
|
887
|
+
// Keep the newest hold per account. An expired hold is retained
|
|
888
|
+
// until compaction rather than dropped here: readers decide
|
|
889
|
+
// expiry against the current clock, and deleting one early would
|
|
890
|
+
// hide the record from a reader whose clock disagrees.
|
|
891
|
+
const hold = projectExhaustionHold(parsed);
|
|
892
|
+
const previous = latestHolds.get(hold.providerId);
|
|
893
|
+
if (!previous || hold.observedAtMs >= previous.observedAtMs) {
|
|
894
|
+
latestHolds.set(hold.providerId, hold);
|
|
895
|
+
}
|
|
896
|
+
} else if (carryableUnknownRecord(parsed)) {
|
|
897
|
+
unrecognisedLines.push(line);
|
|
898
|
+
}
|
|
899
|
+
|
|
900
|
+
}
|
|
901
|
+
const compactedRecords = new Set([
|
|
902
|
+
...latest.values(),
|
|
903
|
+
...latestTokens.values(),
|
|
904
|
+
...latestRateLimits.values(),
|
|
905
|
+
...latestAttempts.values(),
|
|
906
|
+
...latestHolds.values(),
|
|
907
|
+
]);
|
|
908
|
+
// Carried records go first and verbatim, so a build that cannot interpret
|
|
909
|
+
// them neither reorders them relative to each other nor reformats them.
|
|
910
|
+
const compacted = [
|
|
911
|
+
...unrecognisedLines.map((line) => `${line}\n`),
|
|
912
|
+
...[...compactedRecords].map((record) => `${JSON.stringify(record)}\n`),
|
|
913
|
+
].join("");
|
|
914
|
+
const compactedBytes = Buffer.byteLength(compacted, "utf8");
|
|
915
|
+
if (compactedBytes > options.maxBytes) return false;
|
|
916
|
+
|
|
917
|
+
options.beforeRename?.();
|
|
918
|
+
const tail = stableUsageTail({
|
|
919
|
+
path: options.path,
|
|
920
|
+
start: completePrefixBytes,
|
|
921
|
+
maxBytes: options.maxBytes - compactedBytes,
|
|
922
|
+
expectedDevice: initial.dev,
|
|
923
|
+
expectedInode: initial.ino,
|
|
924
|
+
});
|
|
925
|
+
if (tail === undefined) return false;
|
|
926
|
+
const encoded = Buffer.concat([Buffer.from(compacted, "utf8"), tail.bytes]);
|
|
927
|
+
|
|
928
|
+
const temporaryPath = `${options.path}.${process.pid}.${randomUUID()}.tmp`;
|
|
929
|
+
try {
|
|
930
|
+
writeFileSync(temporaryPath, encoded, { mode: 0o600 });
|
|
931
|
+
if (
|
|
932
|
+
!options.mutationLease.renew() ||
|
|
933
|
+
(options.ownerLease !== undefined && !options.ownerLease.renew())
|
|
934
|
+
) {
|
|
935
|
+
return false;
|
|
936
|
+
}
|
|
937
|
+
const current = statSync(options.path);
|
|
938
|
+
if (
|
|
939
|
+
current.dev !== initial.dev ||
|
|
940
|
+
current.ino !== initial.ino ||
|
|
941
|
+
current.size !== tail.sourceSize
|
|
942
|
+
) {
|
|
943
|
+
return false;
|
|
944
|
+
}
|
|
945
|
+
renameSync(temporaryPath, options.path);
|
|
946
|
+
chmodSync(options.path, 0o600);
|
|
947
|
+
return true;
|
|
948
|
+
} finally {
|
|
949
|
+
try {
|
|
950
|
+
unlinkSync(temporaryPath);
|
|
951
|
+
} catch {
|
|
952
|
+
// A published replacement no longer has a temporary path.
|
|
953
|
+
}
|
|
954
|
+
}
|
|
955
|
+
} catch {
|
|
956
|
+
return false;
|
|
957
|
+
}
|
|
958
|
+
}
|
|
959
|
+
|
|
960
|
+
/**
|
|
961
|
+
* Machine-local append-only usage store. A short mutation lease serializes the
|
|
962
|
+
* projected-size check with append or compaction, so cooperating processes cannot
|
|
963
|
+
* race past the documented byte cap. The warming owner remains the ordinary
|
|
964
|
+
* background compaction trigger.
|
|
965
|
+
*/
|
|
966
|
+
export class SharedUsageStore {
|
|
967
|
+
readonly #path: string;
|
|
968
|
+
readonly #observerId: string;
|
|
969
|
+
readonly #ttlMs: number;
|
|
970
|
+
readonly #maxBytes: number;
|
|
971
|
+
readonly #mutationLockPath: string;
|
|
972
|
+
readonly #beforeCompactionRename: (() => void) | undefined;
|
|
973
|
+
|
|
974
|
+
constructor(
|
|
975
|
+
options: {
|
|
976
|
+
readonly path?: string;
|
|
977
|
+
readonly observerId?: string;
|
|
978
|
+
readonly ttlMs?: number;
|
|
979
|
+
readonly maxBytes?: number;
|
|
980
|
+
readonly mutationLockPath?: string;
|
|
981
|
+
/** Test seam for appending while compaction holds the mutation lease. */
|
|
982
|
+
readonly beforeCompactionRename?: () => void;
|
|
983
|
+
} = {},
|
|
984
|
+
) {
|
|
985
|
+
this.#path = options.path ?? defaultStorePath();
|
|
986
|
+
this.#observerId = options.observerId ?? defaultObserverId();
|
|
987
|
+
this.#ttlMs = options.ttlMs ?? SHARED_USAGE_TTL_MS;
|
|
988
|
+
this.#maxBytes = options.maxBytes ?? SHARED_USAGE_MAX_BYTES;
|
|
989
|
+
this.#mutationLockPath = options.mutationLockPath ?? `${this.#path}.lock`;
|
|
990
|
+
this.#beforeCompactionRename = options.beforeCompactionRename;
|
|
991
|
+
if (!validObserverId(this.#observerId))
|
|
992
|
+
throw new RangeError("observerId must be bounded metadata.");
|
|
993
|
+
if (!Number.isFinite(this.#ttlMs) || this.#ttlMs < 1)
|
|
994
|
+
throw new RangeError("ttlMs must be positive.");
|
|
995
|
+
if (
|
|
996
|
+
!Number.isSafeInteger(this.#maxBytes) ||
|
|
997
|
+
this.#maxBytes < MAX_RECORD_BYTES
|
|
998
|
+
)
|
|
999
|
+
throw new RangeError("maxBytes is too small.");
|
|
1000
|
+
}
|
|
1001
|
+
|
|
1002
|
+
get path(): string {
|
|
1003
|
+
return this.#path;
|
|
1004
|
+
}
|
|
1005
|
+
|
|
1006
|
+
get observerId(): string {
|
|
1007
|
+
return this.#observerId;
|
|
1008
|
+
}
|
|
1009
|
+
|
|
1010
|
+
append(record: SharedUsageLogRecord): boolean {
|
|
1011
|
+
try {
|
|
1012
|
+
const projected = validExhaustionHoldRecord(record)
|
|
1013
|
+
? projectExhaustionHold(record)
|
|
1014
|
+
: validAttemptRecord(record)
|
|
1015
|
+
? projectAttempt(record)
|
|
1016
|
+
: projectRecord(record);
|
|
1017
|
+
if (
|
|
1018
|
+
!validRecord(projected) &&
|
|
1019
|
+
!validAttemptRecord(projected) &&
|
|
1020
|
+
!validExhaustionHoldRecord(projected)
|
|
1021
|
+
)
|
|
1022
|
+
return false;
|
|
1023
|
+
const allowed = validExhaustionHoldRecord(record)
|
|
1024
|
+
? [
|
|
1025
|
+
"recordType",
|
|
1026
|
+
"tokens",
|
|
1027
|
+
"providerId",
|
|
1028
|
+
"family",
|
|
1029
|
+
"observedAtMs",
|
|
1030
|
+
"observerId",
|
|
1031
|
+
"failedAtMs",
|
|
1032
|
+
"holdUntilMs",
|
|
1033
|
+
]
|
|
1034
|
+
: validAttemptRecord(record)
|
|
1035
|
+
? [
|
|
1036
|
+
"recordType",
|
|
1037
|
+
"tokens",
|
|
1038
|
+
"providerId",
|
|
1039
|
+
"family",
|
|
1040
|
+
"observedAtMs",
|
|
1041
|
+
"observerId",
|
|
1042
|
+
"failureCount",
|
|
1043
|
+
"nextAttemptAtMs",
|
|
1044
|
+
"disabled",
|
|
1045
|
+
"failureReason",
|
|
1046
|
+
"failureDetail",
|
|
1047
|
+
"failureTriggeredAtMs",
|
|
1048
|
+
"refreshDebounceUntilMs",
|
|
1049
|
+
]
|
|
1050
|
+
: [
|
|
1051
|
+
"providerId",
|
|
1052
|
+
"family",
|
|
1053
|
+
"observedAtMs",
|
|
1054
|
+
"observerId",
|
|
1055
|
+
"tokens",
|
|
1056
|
+
"rateLimit",
|
|
1057
|
+
];
|
|
1058
|
+
if (Object.keys(record as object).some((key) => !allowed.includes(key)))
|
|
1059
|
+
return false;
|
|
1060
|
+
const line = `${JSON.stringify(projected)}\n`;
|
|
1061
|
+
if (Buffer.byteLength(line, "utf8") > MAX_RECORD_BYTES) return false;
|
|
1062
|
+
const directory = dirname(this.#path);
|
|
1063
|
+
mkdirSync(directory, { recursive: true, mode: 0o700 });
|
|
1064
|
+
chmodSync(directory, 0o700);
|
|
1065
|
+
const mutationLease = acquireMachineLease({
|
|
1066
|
+
lockPath: this.#mutationLockPath,
|
|
1067
|
+
reclaimMalformed: true,
|
|
1068
|
+
});
|
|
1069
|
+
if (mutationLease === undefined) return false;
|
|
1070
|
+
try {
|
|
1071
|
+
let state = appendState(this.#path);
|
|
1072
|
+
let appendBytes = Buffer.byteLength(`${state.separator}${line}`, "utf8");
|
|
1073
|
+
if (state.size + appendBytes > this.#maxBytes) {
|
|
1074
|
+
if (
|
|
1075
|
+
!compactUsageFile({
|
|
1076
|
+
path: this.#path,
|
|
1077
|
+
maxBytes: this.#maxBytes,
|
|
1078
|
+
...(this.#beforeCompactionRename === undefined
|
|
1079
|
+
? {}
|
|
1080
|
+
: { beforeRename: this.#beforeCompactionRename }),
|
|
1081
|
+
mutationLease,
|
|
1082
|
+
})
|
|
1083
|
+
) {
|
|
1084
|
+
return false;
|
|
1085
|
+
}
|
|
1086
|
+
state = appendState(this.#path);
|
|
1087
|
+
appendBytes = Buffer.byteLength(`${state.separator}${line}`, "utf8");
|
|
1088
|
+
}
|
|
1089
|
+
if (state.size + appendBytes > this.#maxBytes) return false;
|
|
1090
|
+
// A crashed writer may leave a torn final line. Keep it as an
|
|
1091
|
+
// independent malformed record so this append remains visible.
|
|
1092
|
+
appendFileSync(this.#path, `${state.separator}${line}`, {
|
|
1093
|
+
encoding: "utf8",
|
|
1094
|
+
mode: 0o600,
|
|
1095
|
+
});
|
|
1096
|
+
chmodSync(this.#path, 0o600);
|
|
1097
|
+
return true;
|
|
1098
|
+
} finally {
|
|
1099
|
+
mutationLease.release();
|
|
1100
|
+
}
|
|
1101
|
+
} catch {
|
|
1102
|
+
return false;
|
|
1103
|
+
}
|
|
1104
|
+
}
|
|
1105
|
+
|
|
1106
|
+
readRecords(): readonly SharedUsageRecord[] {
|
|
1107
|
+
try {
|
|
1108
|
+
return parseRecords(completeUsageLines(this.#path));
|
|
1109
|
+
} catch {
|
|
1110
|
+
return [];
|
|
1111
|
+
}
|
|
1112
|
+
}
|
|
1113
|
+
|
|
1114
|
+
readAttempts(): readonly SharedUsageAttemptRecord[] {
|
|
1115
|
+
try {
|
|
1116
|
+
return parseAttempts(completeUsageLines(this.#path));
|
|
1117
|
+
} catch {
|
|
1118
|
+
return [];
|
|
1119
|
+
}
|
|
1120
|
+
}
|
|
1121
|
+
|
|
1122
|
+
readExhaustionHolds(): readonly SharedUsageExhaustionHoldRecord[] {
|
|
1123
|
+
try {
|
|
1124
|
+
return parseExhaustionHolds(completeUsageLines(this.#path));
|
|
1125
|
+
} catch {
|
|
1126
|
+
return [];
|
|
1127
|
+
}
|
|
1128
|
+
}
|
|
1129
|
+
|
|
1130
|
+
/**
|
|
1131
|
+
* The end of an unexpired hold for this account, or undefined.
|
|
1132
|
+
*
|
|
1133
|
+
* Expiry is decided here against the caller's clock rather than by deleting
|
|
1134
|
+
* records, so a hold that has lapsed simply stops being reported. Callers get
|
|
1135
|
+
* a time to compare, not a boolean, because routing has to combine it with an
|
|
1136
|
+
* authoritative recovery time and take whichever is later.
|
|
1137
|
+
*/
|
|
1138
|
+
activeExhaustionHoldUntilMs(
|
|
1139
|
+
providerId: string,
|
|
1140
|
+
family: AllowedFamily,
|
|
1141
|
+
nowMs: number,
|
|
1142
|
+
): number | undefined {
|
|
1143
|
+
let latest: number | undefined;
|
|
1144
|
+
for (const hold of this.readExhaustionHolds()) {
|
|
1145
|
+
if (hold.providerId !== providerId || hold.family !== family) continue;
|
|
1146
|
+
if (hold.holdUntilMs <= nowMs) continue;
|
|
1147
|
+
// A relative bound alone is not enough. `holdUntilMs - failedAtMs` can
|
|
1148
|
+
// be a legitimate 60 minutes while `failedAtMs` itself sits in the year
|
|
1149
|
+
// 3138, which would exclude the account for centuries. No honest hold
|
|
1150
|
+
// can end more than its own duration from now, so anything beyond that
|
|
1151
|
+
// is ignored here as well as refused on append.
|
|
1152
|
+
if (hold.holdUntilMs > nowMs + EXHAUSTION_HOLD_MS) continue;
|
|
1153
|
+
if (latest === undefined || hold.holdUntilMs > latest)
|
|
1154
|
+
latest = hold.holdUntilMs;
|
|
1155
|
+
}
|
|
1156
|
+
return latest;
|
|
1157
|
+
}
|
|
1158
|
+
|
|
1159
|
+
latestAttempt(
|
|
1160
|
+
providerId: string,
|
|
1161
|
+
family: AllowedFamily,
|
|
1162
|
+
): SharedUsageAttemptRecord | undefined {
|
|
1163
|
+
let latest: SharedUsageAttemptRecord | undefined;
|
|
1164
|
+
for (const record of this.readAttempts()) {
|
|
1165
|
+
if (
|
|
1166
|
+
record.providerId === providerId &&
|
|
1167
|
+
record.family === family &&
|
|
1168
|
+
(latest === undefined || record.observedAtMs >= latest.observedAtMs)
|
|
1169
|
+
) {
|
|
1170
|
+
latest = record;
|
|
1171
|
+
}
|
|
1172
|
+
}
|
|
1173
|
+
return latest;
|
|
1174
|
+
}
|
|
1175
|
+
|
|
1176
|
+
aggregate(
|
|
1177
|
+
providerId: string,
|
|
1178
|
+
family: AllowedFamily,
|
|
1179
|
+
nowMs: number,
|
|
1180
|
+
): SharedUsageAggregate {
|
|
1181
|
+
if (
|
|
1182
|
+
!isCanonicalManagedProviderId(providerId, family) ||
|
|
1183
|
+
!validTimestamp(nowMs)
|
|
1184
|
+
) {
|
|
1185
|
+
return { providerId, family };
|
|
1186
|
+
}
|
|
1187
|
+
const records = this.readRecords()
|
|
1188
|
+
.filter(
|
|
1189
|
+
(record) =>
|
|
1190
|
+
record.providerId === providerId && record.family === family,
|
|
1191
|
+
)
|
|
1192
|
+
.sort((a, b) => b.observedAtMs - a.observedAtMs);
|
|
1193
|
+
const fresh = records.filter(
|
|
1194
|
+
(record) => nowMs - record.observedAtMs <= this.#ttlMs,
|
|
1195
|
+
);
|
|
1196
|
+
// Prefer the newest fresh record that CARRIES a rate-limit observation.
|
|
1197
|
+
//
|
|
1198
|
+
// The store interleaves two record shapes for the same account: token totals
|
|
1199
|
+
// (written per response) and rate-limit observations (written when a
|
|
1200
|
+
// response exposes usage headers). Token records are far more numerous, so
|
|
1201
|
+
// taking the newest record outright usually lands on one with no
|
|
1202
|
+
// `rateLimit`, and the resulting snapshot reports `utilization: undefined`.
|
|
1203
|
+
// Observed live: anthropic-account-3 had 45 fresh records carrying
|
|
1204
|
+
// utilization 0.36 while the operator surface showed no figure at all,
|
|
1205
|
+
// because the single newest record happened to be token-only.
|
|
1206
|
+
//
|
|
1207
|
+
// Falls back to the newest fresh record of either shape, so token totals
|
|
1208
|
+
// still surface for an account that has never reported rate-limit headers.
|
|
1209
|
+
const newestWithRateLimit = (
|
|
1210
|
+
candidates: readonly SharedUsageRecord[],
|
|
1211
|
+
): SharedUsageRecord | undefined =>
|
|
1212
|
+
candidates.find((record) => record.rateLimit !== undefined) ??
|
|
1213
|
+
candidates[0];
|
|
1214
|
+
const local = newestWithRateLimit(
|
|
1215
|
+
fresh.filter((record) => record.observerId === this.#observerId),
|
|
1216
|
+
);
|
|
1217
|
+
const peer = newestWithRateLimit(
|
|
1218
|
+
fresh.filter((record) => record.observerId !== this.#observerId),
|
|
1219
|
+
);
|
|
1220
|
+
// The retained reading exists to tell an operator what the account's
|
|
1221
|
+
// utilization last looked like, so prefer the newest expired record that
|
|
1222
|
+
// actually CARRIES a rate-limit observation.
|
|
1223
|
+
//
|
|
1224
|
+
// Taking the newest expired record outright picks a token-only record --
|
|
1225
|
+
// they are far more numerous -- and yields a snapshot whose `utilization` is
|
|
1226
|
+
// undefined. Observed live: anthropic-account-2 returned a 19-minute-old
|
|
1227
|
+
// token record while the 0.99 utilization reading sat 59 minutes back, so
|
|
1228
|
+
// the surface still showed no usage figure after the retained-data fallback
|
|
1229
|
+
// was added. Falls back to the newest expired record of any kind, so token
|
|
1230
|
+
// totals still surface when no rate-limit observation was ever retained.
|
|
1231
|
+
const expired = records.filter(
|
|
1232
|
+
(record) => nowMs - record.observedAtMs > this.#ttlMs,
|
|
1233
|
+
);
|
|
1234
|
+
const stale =
|
|
1235
|
+
expired.find((record) => record.rateLimit !== undefined) ?? expired[0];
|
|
1236
|
+
// A KNOWN FUTURE RECOVERY TIME IS NOT PERISHABLE, so it is chosen from
|
|
1237
|
+
// every record rather than only the fresh ones, and from any observer
|
|
1238
|
+
// rather than only peers.
|
|
1239
|
+
//
|
|
1240
|
+
// `SHARED_USAGE_TTL_MS` and `USAGE_FETCH_INTERVAL_MS` are both five
|
|
1241
|
+
// minutes, so a rate-limit record ages out of `fresh` exactly as its
|
|
1242
|
+
// replacement falls due. In that gap the selections above settle for the
|
|
1243
|
+
// newest record of any shape -- normally a token-only one -- and routing
|
|
1244
|
+
// sees no `recoveryAtMs` at all. Observed live on `openai-codex`: a
|
|
1245
|
+
// three-day outage looked like a healthy account and took two turns before
|
|
1246
|
+
// anything noticed (#97).
|
|
1247
|
+
//
|
|
1248
|
+
// A token count really does expire in five minutes. "This account is out
|
|
1249
|
+
// until T" does not: it carries its own expiry and stays true until T
|
|
1250
|
+
// passes. The TTL is the wrong instrument for it.
|
|
1251
|
+
//
|
|
1252
|
+
// Observer-blind ON PURPOSE. The `session`/`fleet` split exists so a caller
|
|
1253
|
+
// can tell who measured something, which matters for a token total. A
|
|
1254
|
+
// recovery time is a fact about the ACCOUNT, so our own reading and a
|
|
1255
|
+
// peer's are equally admissible -- and excluding our own is the other half
|
|
1256
|
+
// of the same defect, which is why routing could not avoid the account it
|
|
1257
|
+
// had just measured itself.
|
|
1258
|
+
//
|
|
1259
|
+
// SELF-EXPIRING, and that is load-bearing. `recoveryAtMs > nowMs` is what
|
|
1260
|
+
// keeps this from turning a transient condition into a permanent one: once
|
|
1261
|
+
// the recovery instant passes, no record qualifies and the account is
|
|
1262
|
+
// eligible again with no reset and no new observation. Weakening that
|
|
1263
|
+
// comparison to a presence check would strand a recovered account forever.
|
|
1264
|
+
//
|
|
1265
|
+
// BOUNDED ABOVE TOO, but at the door rather than here: `validRecord`
|
|
1266
|
+
// rejects a recovery time more than `MAX_RECOVERY_HORIZON_MS` after its
|
|
1267
|
+
// own observation, on both the write and read paths, so such a record
|
|
1268
|
+
// never reaches this scan. `validTimestamp` alone admits any finite
|
|
1269
|
+
// number, and a value no clock will ever reach would exclude an account
|
|
1270
|
+
// permanently.
|
|
1271
|
+
//
|
|
1272
|
+
// That bound is anchored to `observedAtMs`, never to `nowMs`. A
|
|
1273
|
+
// clock-anchored bound is a sliding window: it refuses a year-out reading
|
|
1274
|
+
// today and silently admits the same broken value about 330 days later,
|
|
1275
|
+
// when it drifts inside the window and outranks every newer healthy
|
|
1276
|
+
// reading. A plausibility verdict that reverses itself with no new
|
|
1277
|
+
// observation is not a bound at all. Measured from the observation it is
|
|
1278
|
+
// a property of the record and never changes.
|
|
1279
|
+
//
|
|
1280
|
+
// SUPERSEDED BY A NEWER READING FROM THE SAME INSTRUMENT. A durable
|
|
1281
|
+
// recovery time survives the TTL, but it does not survive a
|
|
1282
|
+
// strictly-newer reading, FROM THE SOURCE THAT OBSERVED IT, that reports
|
|
1283
|
+
// capacity again. A provider window can reset ahead of the recorded
|
|
1284
|
+
// `recoveryAtMs`; without this, the account stayed excluded until that
|
|
1285
|
+
// stale timestamp elapsed even though every fresh poll re-measured it
|
|
1286
|
+
// healthy. Observed live: `openai-codex-account-2` sat pinned to the
|
|
1287
|
+
// owning-vendor API tier ~40h past its real reset while its newest
|
|
1288
|
+
// usage-endpoint records read utilization 0.
|
|
1289
|
+
//
|
|
1290
|
+
// SAME SOURCE IS LOAD-BEARING, and is why this does not reopen #97's
|
|
1291
|
+
// other half. The two instruments see different limits: the usage
|
|
1292
|
+
// endpoint tracks the QUOTA window and is blind to a SESSION 429, and a
|
|
1293
|
+
// rate-limit header is the reverse. A healthy reading from the OTHER
|
|
1294
|
+
// instrument is not evidence that THIS exhaustion cleared -- that is the
|
|
1295
|
+
// documented `usage-endpoint util:0 while 429ing` case. Only the same
|
|
1296
|
+
// instrument re-measuring its own window can retire its own recovery
|
|
1297
|
+
// time. A token-only record carries no rate-limit reading at all, so it
|
|
1298
|
+
// is silent about exhaustion rather than evidence of health.
|
|
1299
|
+
//
|
|
1300
|
+
// The NEWEST same-source reading decides, judged by the one shared
|
|
1301
|
+
// exhaustion predicate, so a newer reading that is itself exhausted -- a
|
|
1302
|
+
// fresh 429, or its own future recovery -- never releases the account.
|
|
1303
|
+
const durableCandidate = records.find(
|
|
1304
|
+
(record) =>
|
|
1305
|
+
record.rateLimit?.recoveryAtMs !== undefined &&
|
|
1306
|
+
record.rateLimit.recoveryAtMs > nowMs,
|
|
1307
|
+
);
|
|
1308
|
+
const durableSource = durableCandidate?.rateLimit?.utilizationSource;
|
|
1309
|
+
const newerSameSourceReading =
|
|
1310
|
+
durableCandidate === undefined || durableSource === undefined
|
|
1311
|
+
? undefined
|
|
1312
|
+
: records.find(
|
|
1313
|
+
(record) =>
|
|
1314
|
+
record.rateLimit?.utilizationSource === durableSource &&
|
|
1315
|
+
record.observedAtMs > durableCandidate.observedAtMs,
|
|
1316
|
+
);
|
|
1317
|
+
const durableExhaustion =
|
|
1318
|
+
newerSameSourceReading !== undefined &&
|
|
1319
|
+
!snapshotIndicatesExhaustion(
|
|
1320
|
+
newerSameSourceReading.rateLimit,
|
|
1321
|
+
nowMs,
|
|
1322
|
+
"all-observed",
|
|
1323
|
+
)
|
|
1324
|
+
? undefined
|
|
1325
|
+
: durableCandidate;
|
|
1326
|
+
|
|
1327
|
+
return {
|
|
1328
|
+
providerId,
|
|
1329
|
+
family,
|
|
1330
|
+
...(local === undefined
|
|
1331
|
+
? {}
|
|
1332
|
+
: { session: snapshotFromRecord(local, nowMs) }),
|
|
1333
|
+
...(peer === undefined ? {} : { fleet: snapshotFromRecord(peer, nowMs) }),
|
|
1334
|
+
...(stale === undefined
|
|
1335
|
+
? {}
|
|
1336
|
+
: { stale: snapshotFromRecord(stale, nowMs) }),
|
|
1337
|
+
...(durableExhaustion === undefined
|
|
1338
|
+
? {}
|
|
1339
|
+
: {
|
|
1340
|
+
durableExhaustion: snapshotFromRecord(durableExhaustion, nowMs),
|
|
1341
|
+
}),
|
|
1342
|
+
};
|
|
1343
|
+
}
|
|
1344
|
+
|
|
1345
|
+
/** Compact only complete records and only when the caller proves lease ownership. */
|
|
1346
|
+
compactUnderLease(lease: {
|
|
1347
|
+
readonly record: { readonly token: string };
|
|
1348
|
+
renew(): boolean;
|
|
1349
|
+
}): boolean {
|
|
1350
|
+
if (!lease.record.token || !lease.renew()) return false;
|
|
1351
|
+
try {
|
|
1352
|
+
if (statSync(this.#path).size <= this.#maxBytes) return false;
|
|
1353
|
+
} catch {
|
|
1354
|
+
return false;
|
|
1355
|
+
}
|
|
1356
|
+
const mutationLease = acquireMachineLease({
|
|
1357
|
+
lockPath: this.#mutationLockPath,
|
|
1358
|
+
reclaimMalformed: true,
|
|
1359
|
+
});
|
|
1360
|
+
if (mutationLease === undefined) return false;
|
|
1361
|
+
try {
|
|
1362
|
+
return compactUsageFile({
|
|
1363
|
+
path: this.#path,
|
|
1364
|
+
maxBytes: this.#maxBytes,
|
|
1365
|
+
...(this.#beforeCompactionRename === undefined
|
|
1366
|
+
? {}
|
|
1367
|
+
: { beforeRename: this.#beforeCompactionRename }),
|
|
1368
|
+
mutationLease,
|
|
1369
|
+
ownerLease: lease,
|
|
1370
|
+
});
|
|
1371
|
+
} finally {
|
|
1372
|
+
mutationLease.release();
|
|
1373
|
+
}
|
|
1374
|
+
}
|
|
1375
|
+
}
|
|
1376
|
+
|
|
1377
|
+
/** Header values are already 0-1 fractions. */
|
|
1378
|
+
export function normalizeHeaderUtilization(value: number): number | undefined {
|
|
1379
|
+
return validFraction(value) ? value : undefined;
|
|
1380
|
+
}
|
|
1381
|
+
|
|
1382
|
+
/** Usage endpoint bodies report 0-100 percent; persist the normalized fraction. */
|
|
1383
|
+
export function normalizeUsageEndpointPercent(
|
|
1384
|
+
value: number,
|
|
1385
|
+
): number | undefined {
|
|
1386
|
+
return typeof value === "number" &&
|
|
1387
|
+
Number.isFinite(value) &&
|
|
1388
|
+
value >= 0 &&
|
|
1389
|
+
value <= 100
|
|
1390
|
+
? value / 100
|
|
1391
|
+
: undefined;
|
|
1392
|
+
}
|