@volter/twin 0.1.0 → 0.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +16 -2
- package/inject.cjs +453 -59
- package/package.json +12 -22
- package/src/actions.ts +234 -49
- package/src/blob-store.ts +136 -0
- package/src/changeset.ts +807 -0
- package/src/cli.ts +60 -10
- package/src/connector.ts +30 -7
- package/src/control-plane.ts +17 -1
- package/src/emit.ts +242 -0
- package/src/fork.ts +19 -7
- package/src/index.ts +139 -6
- package/src/lease.ts +4 -6
- package/src/lifecycle.ts +8 -0
- package/src/packRegistry.ts +248 -2
- package/src/plan.ts +131 -23
- package/src/proxy.ts +5 -2
- package/src/pushLedger.ts +116 -11
- package/src/queueLifecycle.ts +3 -4
- package/src/rateBudget.ts +1115 -0
- package/src/refs.ts +9 -10
- package/src/remote-execute.ts +16 -0
- package/src/scenario.ts +387 -0
- package/src/serve.ts +397 -15
- package/src/shadow.ts +86 -7
- package/src/storage.ts +76 -147
- package/src/sync.ts +63 -17
- package/src/twin-fetch.ts +115 -0
- package/src/validate.ts +6 -5
- package/src/world-clock.ts +33 -0
- package/src/world-store.ts +482 -0
- package/src/worldConfig.ts +4 -3
- package/dist/src/actions.d.ts +0 -138
- package/dist/src/actions.js +0 -201
- package/dist/src/args.d.ts +0 -3
- package/dist/src/args.js +0 -12
- package/dist/src/cli.d.ts +0 -2
- package/dist/src/cli.js +0 -425
- package/dist/src/connector.d.ts +0 -106
- package/dist/src/connector.js +0 -129
- package/dist/src/control-plane.d.ts +0 -21
- package/dist/src/control-plane.js +0 -40
- package/dist/src/egress.d.ts +0 -93
- package/dist/src/egress.js +0 -264
- package/dist/src/fork.d.ts +0 -126
- package/dist/src/fork.js +0 -206
- package/dist/src/index.d.ts +0 -42
- package/dist/src/index.js +0 -52
- package/dist/src/lease.d.ts +0 -50
- package/dist/src/lease.js +0 -80
- package/dist/src/packRegistry.d.ts +0 -34
- package/dist/src/packRegistry.js +0 -22
- package/dist/src/plan.d.ts +0 -97
- package/dist/src/plan.js +0 -151
- package/dist/src/proxy.d.ts +0 -25
- package/dist/src/proxy.js +0 -152
- package/dist/src/pushLedger.d.ts +0 -81
- package/dist/src/pushLedger.js +0 -130
- package/dist/src/queueLifecycle.d.ts +0 -62
- package/dist/src/queueLifecycle.js +0 -95
- package/dist/src/reconcile.d.ts +0 -58
- package/dist/src/reconcile.js +0 -137
- package/dist/src/refs.d.ts +0 -29
- package/dist/src/refs.js +0 -68
- package/dist/src/schemas.d.ts +0 -78
- package/dist/src/schemas.js +0 -50
- package/dist/src/serve.d.ts +0 -44
- package/dist/src/serve.js +0 -93
- package/dist/src/shadow.d.ts +0 -77
- package/dist/src/shadow.js +0 -138
- package/dist/src/status.d.ts +0 -31
- package/dist/src/status.js +0 -42
- package/dist/src/storage.d.ts +0 -119
- package/dist/src/storage.js +0 -535
- package/dist/src/sync.d.ts +0 -91
- package/dist/src/sync.js +0 -121
- package/dist/src/types.d.ts +0 -40
- package/dist/src/types.js +0 -1
- package/dist/src/validate.d.ts +0 -27
- package/dist/src/validate.js +0 -68
- package/dist/src/visualizer.d.ts +0 -13
- package/dist/src/visualizer.js +0 -133
- package/dist/src/worldConfig.d.ts +0 -9
- package/dist/src/worldConfig.js +0 -16
|
@@ -0,0 +1,1115 @@
|
|
|
1
|
+
// CLIENT-SIDE RATE BUDGET — a persistent, FAIL-CLOSED spend ledger that sits in front of every
|
|
2
|
+
// real vendor call a twin pack makes. VENDOR-AGNOSTIC: the mechanism lives here, every vendor
|
|
3
|
+
// specific (the window, the ceiling, the per-endpoint weights) is DATA a pack declares, exactly
|
|
4
|
+
// like `TwinPack.browserRouting`. There is no vendor name, and no vendor branch, in this file.
|
|
5
|
+
//
|
|
6
|
+
// ── WHY THIS EXISTS ─────────────────────────────────────────────────────────────────────────
|
|
7
|
+
// Call-conservation doctrine (cache-first, version-gated, batched, renders opt-in) is good
|
|
8
|
+
// discipline, but discipline only binds the code that follows it. A real ~4.5-DAY token lockout
|
|
9
|
+
// happened on one vendor because raw API calls were made OUTSIDE the pack's connector — no cache,
|
|
10
|
+
// no version gate, no batching. Discipline cannot stop that; a BUDGET can. This module is the
|
|
11
|
+
// structural backstop: a durable ledger of what has been spent in the last rolling window,
|
|
12
|
+
// consulted BEFORE the request goes out, that THROWS rather than calls when the ceiling is
|
|
13
|
+
// reached, and that persists across processes so a fresh process does NOT get a fresh allowance.
|
|
14
|
+
// Failing closed is the whole point — a refused call costs a retry, a lockout costs days.
|
|
15
|
+
//
|
|
16
|
+
// It was built for one vendor first and promoted here because nothing about it was vendor-specific
|
|
17
|
+
// except the numbers. Every pack that talks to a live vendor can now be protected by declaring
|
|
18
|
+
// those numbers.
|
|
19
|
+
//
|
|
20
|
+
// ── WHERE THE NUMBERS COME FROM ─────────────────────────────────────────────────────────────
|
|
21
|
+
// From the PACK, never from here. A pack calls `declareRateBudget(vendor, { window, ceiling,
|
|
22
|
+
// weights, reason })` (or ships the same object as `TwinPack.rateBudget`, which `registerPack`
|
|
23
|
+
// forwards). The kernel only knows how to price a call against declared rules and how to keep
|
|
24
|
+
// the ledger honest.
|
|
25
|
+
//
|
|
26
|
+
// A vendor with NO declaration does NOT become unlimited. It falls back to
|
|
27
|
+
// `DEFAULT_RATE_BUDGET`, a deliberately austere allowance (see its docstring) — "nobody
|
|
28
|
+
// configured it" must never mean "no limit". There is deliberately NO opt-out: an escape hatch is
|
|
29
|
+
// a permanent hole that the next careless caller reaches for, and the cost asymmetry is stark —
|
|
30
|
+
// a too-tight default costs a refused call plus a one-line declaration, a too-loose one costs days
|
|
31
|
+
// of lockout.
|
|
32
|
+
//
|
|
33
|
+
// ── CONCURRENCY / ATOMICITY — the honest guarantee ──────────────────────────────────────────
|
|
34
|
+
// Every ledger mutation is a read-modify-write performed while holding an exclusive lock file
|
|
35
|
+
// created with O_EXCL (`openSync(..., 'wx')`), and the ledger itself is replaced by an atomic
|
|
36
|
+
// `rename()` of a temp file, so a reader never observes a half-written JSON document.
|
|
37
|
+
// `checkBudget()` does not merely *look* — it RESERVES, charging the weight to the ledger inside
|
|
38
|
+
// the same lock hold before returning. That is what makes a burst genuinely refused: two processes
|
|
39
|
+
// racing cannot both see room, because the first one's spend is already committed before the
|
|
40
|
+
// second one reads. `recordCall()` then SETTLES that reservation rather than charging twice.
|
|
41
|
+
// Everything inside `withLock` is SYNCHRONOUS, so N concurrent async callers in one process cannot
|
|
42
|
+
// interleave between the read and the write either.
|
|
43
|
+
//
|
|
44
|
+
// ── WHAT THIS DOES *NOT* GUARANTEE (read this before trusting it) ───────────────────────────
|
|
45
|
+
// • It binds only calls made through the guarded client a pack wires it into. A caller who
|
|
46
|
+
// hand-rolls `fetch` against the vendor host is unaffected — that is exactly the incident this
|
|
47
|
+
// exists to prevent, so the ban on raw vendor calls (ARCHITECTURE.md D8) is enforced separately
|
|
48
|
+
// by per-pack repo-wide gates, not by this module.
|
|
49
|
+
// • `path` / `root` / `now` are CALLER-REACHABLE RESETS. A different ledger path, or an injected
|
|
50
|
+
// clock that fast-forwards, restores the full allowance. They exist because tests must inject
|
|
51
|
+
// them, and there is no way to offer that seam to a test and deny it to a determined caller in
|
|
52
|
+
// the same process. `ceiling` / `windowMs` / `maxRetryAfterSeconds` passed at CONSTRUCTION are
|
|
53
|
+
// clamped so a configuration MISTAKE cannot widen the budget, the DECLARATION itself is
|
|
54
|
+
// clamped to sane bounds and may never be re-declared more permissively, and the clock is
|
|
55
|
+
// sanity-checked for the common ACCIDENTS (seconds instead of ms, a monotonic
|
|
56
|
+
// `performance.now()`-style clock) — but no clamp defeats intent. This guards CARELESSNESS,
|
|
57
|
+
// NOT MALICE, and that claim is not upgraded anywhere.
|
|
58
|
+
// • The default ledger is keyed by VENDOR **and** a hash of the TOKEN, and lives in the user's
|
|
59
|
+
// home dir, because vendors rate-limit per credential. Passing `root` scopes it to one project
|
|
60
|
+
// instead, which means two project roots sharing one token get two allowances — use it only
|
|
61
|
+
// when that is what you want.
|
|
62
|
+
// • O_EXCL locking is reliable on a local filesystem; over NFS or a network share it is not.
|
|
63
|
+
// • DELETING the ledger restores a full allowance with no cooldown: a missing file is the
|
|
64
|
+
// legitimate first run and cannot be told apart from a wiped one. A *corrupt* file, by
|
|
65
|
+
// contrast, costs a full window. The asymmetry is deliberate but worth knowing.
|
|
66
|
+
// • A call that dies between reserve and settle stays charged for the rest of the window. That
|
|
67
|
+
// is deliberate: over-charging is the safe direction.
|
|
68
|
+
// • `checkBudget` blocks the event loop for up to ~1s under lock contention (synchronous fs +
|
|
69
|
+
// `Atomics.wait`). Fine for a CLI or a connector pull; it is not for a hot request path.
|
|
70
|
+
import { closeSync, existsSync, fstatSync, mkdirSync, openSync, readFileSync, renameSync, rmSync, statSync, writeFileSync, writeSync } from 'node:fs';
|
|
71
|
+
import { createHash, randomUUID } from 'node:crypto';
|
|
72
|
+
import { homedir } from 'node:os';
|
|
73
|
+
import { dirname, join } from 'node:path';
|
|
74
|
+
import { worldPaths } from './storage.ts';
|
|
75
|
+
|
|
76
|
+
// ── bounds the kernel enforces on ANY declaration ───────────────────────────────────────────
|
|
77
|
+
|
|
78
|
+
/** Shortest window a declaration may claim. A sub-second window is a counter reset, not a budget. */
|
|
79
|
+
export const MIN_RATE_BUDGET_WINDOW_MS = 1_000;
|
|
80
|
+
/** Longest window a declaration may claim — beyond an hour the ledger stops being a rolling window. */
|
|
81
|
+
export const MAX_RATE_BUDGET_WINDOW_MS = 60 * 60 * 1_000;
|
|
82
|
+
/**
|
|
83
|
+
* The most weighted units any pack may declare per window. Not a vendor fact — a blast radius cap,
|
|
84
|
+
* so a typo (`ceiling: 100000`) cannot silently disarm the guard for a whole vendor. Every real
|
|
85
|
+
* declaration is far under it.
|
|
86
|
+
*/
|
|
87
|
+
export const MAX_RATE_BUDGET_CEILING = 1_000;
|
|
88
|
+
/**
|
|
89
|
+
* The sub-window every declaration's BURST is measured over, regardless of its own window length.
|
|
90
|
+
* A rolling window bounds total spend but says nothing about how fast it may be spent: a one-hour
|
|
91
|
+
* window with a 600-unit ceiling admits all 600 in a single millisecond, twenty times what the
|
|
92
|
+
* austere fallback allows in a minute, and neither `MAX_RATE_BUDGET_CEILING` nor
|
|
93
|
+
* `MAX_RATE_BUDGET_UNITS_PER_SECOND` (which evaluates to 0.167/s there) reaches it. So a declaration
|
|
94
|
+
* whose window is longer than this MUST also declare `burstCeiling`, and the kernel refuses against
|
|
95
|
+
* it independently. (§9 round 2, 2026-07-26: the property had been asserted in PROSE, by a test that
|
|
96
|
+
* accepted a declaration's own longer window as the justification for the burst that window created
|
|
97
|
+
* — circular, and it waved through a hostile 1h/1000 declaration.)
|
|
98
|
+
*/
|
|
99
|
+
export const RATE_BUDGET_BURST_WINDOW_MS = 60_000;
|
|
100
|
+
|
|
101
|
+
/**
|
|
102
|
+
* The most weighted units per SECOND any declaration may imply (`ceiling / (windowMs/1000)`).
|
|
103
|
+
* The per-dimension caps above do not bound the RATE — `ceiling: 1000, windowMs: 1000` satisfies
|
|
104
|
+
* both and still buys 1000 units a second, which is not a budget. This is the bound that makes the
|
|
105
|
+
* "blast radius cap" claim true. Every real declaration is far under it — the packs that declare one
|
|
106
|
+
* today sit between roughly 0.3 and 5 units/second. (Named examples were removed here once more than
|
|
107
|
+
* two packs declared: a list in the kernel goes stale, and the kernel must not know its vendors.)
|
|
108
|
+
*/
|
|
109
|
+
export const MAX_RATE_BUDGET_UNITS_PER_SECOND = 20;
|
|
110
|
+
|
|
111
|
+
/**
|
|
112
|
+
* The cheapest a single call may be priced. NOTHING IS FREE: a zero-weight call would let an
|
|
113
|
+
* unbounded loop through, and the ledger's "entries can never outnumber the ceiling" invariant
|
|
114
|
+
* (which bounds `load()`) depends on every entry costing at least one unit.
|
|
115
|
+
*/
|
|
116
|
+
export const MIN_RATE_BUDGET_WEIGHT = 1;
|
|
117
|
+
|
|
118
|
+
/** A cooldown is capped only to keep an absurd/garbage header from producing a nonsense date. */
|
|
119
|
+
const MAX_COOLDOWN_MS = 7 * 24 * 60 * 60 * 1000;
|
|
120
|
+
|
|
121
|
+
/**
|
|
122
|
+
* How long to back off when the vendor says "slow down" without saying for how long. The
|
|
123
|
+
* conventional answer for a bare 429, and also the cap on the "exhausted, reset unreadable" path —
|
|
124
|
+
* an unreadable hint must not cost MORE than no hint at all.
|
|
125
|
+
*/
|
|
126
|
+
const DEFAULT_BARE_429_BACKOFF_S = 60;
|
|
127
|
+
|
|
128
|
+
/**
|
|
129
|
+
* A valid ledger cannot hold more entries than the ceiling (every entry costs at least 1 and the
|
|
130
|
+
* live sum never exceeds the ceiling, which is itself capped at `MAX_RATE_BUDGET_CEILING`), so
|
|
131
|
+
* anything past this is corruption or a griefing write — and an enormous array is also what would
|
|
132
|
+
* make `load()` slow enough to matter for the lock.
|
|
133
|
+
*/
|
|
134
|
+
const MAX_LEDGER_ENTRIES = MAX_RATE_BUDGET_CEILING;
|
|
135
|
+
|
|
136
|
+
/** Earliest plausible ms-epoch (2001-09-09). Below this the clock is in seconds, or not an epoch. */
|
|
137
|
+
const MIN_PLAUSIBLE_EPOCH_MS = 1_000_000_000_000;
|
|
138
|
+
|
|
139
|
+
/** How long a lock may look abandoned before a live-process check is even considered. */
|
|
140
|
+
const STALE_LOCK_MS = 10_000;
|
|
141
|
+
|
|
142
|
+
// ── the per-vendor DATA a pack declares ─────────────────────────────────────────────────────
|
|
143
|
+
|
|
144
|
+
/**
|
|
145
|
+
* One pricing rule. `match` is a RegExp SOURCE string tested against the *call key* — whatever
|
|
146
|
+
* string the pack's guarded client uses to name the call it is about to make (a REST path for an
|
|
147
|
+
* HTTP pack, a dotted SDK method name for an SDK-shaped pack). First matching rule wins, so order
|
|
148
|
+
* matters; a call matching nothing is priced at `defaultWeight`.
|
|
149
|
+
*
|
|
150
|
+
* `whenQueryPresent` / `whenQueryAbsent` let one path split by request shape (e.g. a cheap probe
|
|
151
|
+
* distinguished from a full fetch only by a query parameter) without needing a vendor-specific
|
|
152
|
+
* predicate FUNCTION in the declaration — declarations stay pure, inspectable data.
|
|
153
|
+
*/
|
|
154
|
+
export type RateBudgetWeightRule = {
|
|
155
|
+
/** RegExp source, tested against the call key. */
|
|
156
|
+
match: string;
|
|
157
|
+
/** Weighted cost when this rule matches. At least `MIN_RATE_BUDGET_WEIGHT`. */
|
|
158
|
+
weight: number;
|
|
159
|
+
/** Only match when EVERY named query key is present. */
|
|
160
|
+
whenQueryPresent?: string[];
|
|
161
|
+
/** Only match when EVERY named query key is absent. */
|
|
162
|
+
whenQueryAbsent?: string[];
|
|
163
|
+
};
|
|
164
|
+
|
|
165
|
+
/** What a pack declares. Pure data — no functions, no vendor code in the kernel. */
|
|
166
|
+
export type RateBudgetDeclaration = {
|
|
167
|
+
/** Rolling window, in ms. Spend older than this is pruned. */
|
|
168
|
+
windowMs: number;
|
|
169
|
+
/** Weighted units allowed inside one window. */
|
|
170
|
+
ceiling: number;
|
|
171
|
+
/**
|
|
172
|
+
* Weighted units allowed inside any 60-SECOND sub-window, enforced independently of `ceiling`.
|
|
173
|
+
*
|
|
174
|
+
* REQUIRED when `windowMs` is longer than 60s, and refused as meaningless when it is not (a
|
|
175
|
+
* window that is already a minute has its ceiling as its burst). This is what stops a long window
|
|
176
|
+
* from buying an enormous instantaneous burst as a side effect of bounding a long-horizon limit —
|
|
177
|
+
* the thing github and linear legitimately need, and the thing a careless declaration would get by
|
|
178
|
+
* accident. Must be ≤ `ceiling`; like every other axis it may only ever be TIGHTENED.
|
|
179
|
+
*/
|
|
180
|
+
burstCeiling?: number;
|
|
181
|
+
/** Cost of a call no rule prices. Never zero — an unknown endpoint must never be free. */
|
|
182
|
+
defaultWeight: number;
|
|
183
|
+
/**
|
|
184
|
+
* Seconds. A `Retry-After` above this is not something to sleep off — it means the credential is
|
|
185
|
+
* in real trouble, so it FAILS LOUDLY rather than scheduling a long sleep.
|
|
186
|
+
*/
|
|
187
|
+
maxRetryAfterSeconds?: number;
|
|
188
|
+
/** Ordered pricing rules; first match wins. */
|
|
189
|
+
rules?: RateBudgetWeightRule[];
|
|
190
|
+
/**
|
|
191
|
+
* WHY these numbers — the vendor's documented (or observed) limits, and the reasoning that
|
|
192
|
+
* turned them into this ceiling. Required: a budget nobody can explain is a budget nobody can
|
|
193
|
+
* review, and the ceiling is the one number that can cost days if it is wrong.
|
|
194
|
+
*/
|
|
195
|
+
reason: string;
|
|
196
|
+
};
|
|
197
|
+
|
|
198
|
+
/**
|
|
199
|
+
* A declaration bound to the vendor it governs, with every pattern already compiled. The `RegExp`s
|
|
200
|
+
* are built ONCE here rather than per call: `priceCall` runs in front of every vendor request, and
|
|
201
|
+
* recompiling a pattern per call is both wasted work and a needless amplifier for a pathological
|
|
202
|
+
* pattern. (Declarations are repo source, not user input — this is a performance fix, not a
|
|
203
|
+
* sanitizer.)
|
|
204
|
+
*/
|
|
205
|
+
export type RateBudgetPolicy = Required<Omit<RateBudgetDeclaration, 'rules'>> & {
|
|
206
|
+
vendor: string;
|
|
207
|
+
rules: RateBudgetWeightRule[];
|
|
208
|
+
/** `compiled[i]` is `rules[i].match`, compiled. Same length, same order. */
|
|
209
|
+
compiled: RegExp[];
|
|
210
|
+
};
|
|
211
|
+
|
|
212
|
+
/**
|
|
213
|
+
* The allowance a vendor gets when NOBODY declared one.
|
|
214
|
+
*
|
|
215
|
+
* 60 weighted units per 60s at a default weight of 2 — 30 calls a minute, one every two seconds.
|
|
216
|
+
* That is comfortably above any human- or agent-driven explicit pull (a few dozen calls), and far
|
|
217
|
+
* below the shape that causes lockouts (a loop emitting hundreds of calls a minute). It is
|
|
218
|
+
* deliberately NOT generous: an undeclared vendor is one nobody has thought about, and the safe
|
|
219
|
+
* assumption about an unexamined vendor is that its limits are tight.
|
|
220
|
+
*
|
|
221
|
+
* There is no opt-out. A pack that genuinely needs more says so, in its declaration, with a reason.
|
|
222
|
+
*
|
|
223
|
+
* ⚠️ HONESTLY: this is tighter than a real declaration in CALL COUNT, but it is NOT uniformly
|
|
224
|
+
* tighter, because it prices every call the same. A vendor whose pack prices one endpoint high (a
|
|
225
|
+
* render at 20, a recursive walk at 2) gets that endpoint priced at 2 here — cheaper, not dearer.
|
|
226
|
+
* That is why `RateBudget` reads the policy LIVE rather than snapshotting it at construction, and
|
|
227
|
+
* why a pack's binding (its `XBudget` subclass, whose module declares on import) is the only
|
|
228
|
+
* construction path you should use. (§9 finding, 2026-07-26.)
|
|
229
|
+
*/
|
|
230
|
+
export const DEFAULT_RATE_BUDGET: RateBudgetDeclaration = {
|
|
231
|
+
windowMs: 60_000,
|
|
232
|
+
ceiling: 60,
|
|
233
|
+
defaultWeight: 2,
|
|
234
|
+
maxRetryAfterSeconds: 300,
|
|
235
|
+
rules: [],
|
|
236
|
+
reason:
|
|
237
|
+
'no pack declared a budget for this vendor — this is the conservative fallback (30 calls/min at ' +
|
|
238
|
+
'the default weight), not a considered limit. Declare one on the pack with the vendor\'s real limits.',
|
|
239
|
+
};
|
|
240
|
+
|
|
241
|
+
// ── declaration registry ────────────────────────────────────────────────────────────────────
|
|
242
|
+
|
|
243
|
+
const declarations = new Map<string, RateBudgetPolicy>();
|
|
244
|
+
|
|
245
|
+
function fail(what: string): never {
|
|
246
|
+
throw new Error(`rate budget: ${what}`);
|
|
247
|
+
}
|
|
248
|
+
|
|
249
|
+
function finitePositive(n: unknown): n is number {
|
|
250
|
+
return typeof n === 'number' && Number.isFinite(n) && n > 0;
|
|
251
|
+
}
|
|
252
|
+
|
|
253
|
+
/**
|
|
254
|
+
* Validate + clamp a declaration. Clamping is the ONLY direction that matters: a declaration may
|
|
255
|
+
* ask for a shorter window or a bigger ceiling than the kernel permits, and it gets the kernel's
|
|
256
|
+
* bound instead. Anything structurally wrong (a zero-weight rule, an unparseable pattern, a missing
|
|
257
|
+
* reason) is a THROW, not a clamp — those are mistakes, not preferences.
|
|
258
|
+
*/
|
|
259
|
+
function normalize(vendor: string, decl: RateBudgetDeclaration): RateBudgetPolicy {
|
|
260
|
+
if (typeof vendor !== 'string' || !/^[a-z0-9-]+$/.test(vendor)) fail(`invalid vendor id: ${String(vendor)}`);
|
|
261
|
+
if (!decl || typeof decl !== 'object') fail(`${vendor}: declaration must be an object`);
|
|
262
|
+
if (typeof decl.reason !== 'string' || decl.reason.trim() === '') {
|
|
263
|
+
fail(`${vendor}: a declaration must carry a \`reason\` — the vendor limits that justify this ceiling`);
|
|
264
|
+
}
|
|
265
|
+
if (!finitePositive(decl.windowMs)) fail(`${vendor}: windowMs must be a positive number`);
|
|
266
|
+
if (!finitePositive(decl.ceiling)) fail(`${vendor}: ceiling must be a positive number`);
|
|
267
|
+
if (!finitePositive(decl.defaultWeight)) fail(`${vendor}: defaultWeight must be a positive number`);
|
|
268
|
+
if (decl.defaultWeight < MIN_RATE_BUDGET_WEIGHT) {
|
|
269
|
+
fail(`${vendor}: defaultWeight ${decl.defaultWeight} is below ${MIN_RATE_BUDGET_WEIGHT} — an unclassified call must never be free`);
|
|
270
|
+
}
|
|
271
|
+
if (decl.maxRetryAfterSeconds !== undefined && !finitePositive(decl.maxRetryAfterSeconds)) {
|
|
272
|
+
fail(`${vendor}: maxRetryAfterSeconds must be a positive number`);
|
|
273
|
+
}
|
|
274
|
+
// A window LONGER than the kernel's bound is refused, not clamped down: clamping it would hand
|
|
275
|
+
// the declaration a HIGHER rate than it asked for (a 24h/500 declaration silently becoming
|
|
276
|
+
// 1h/500), which is the one direction normalization must never move. (§9 finding, 2026-07-26.)
|
|
277
|
+
if (decl.windowMs > MAX_RATE_BUDGET_WINDOW_MS) {
|
|
278
|
+
fail(`${vendor}: windowMs ${decl.windowMs} is beyond the ${MAX_RATE_BUDGET_WINDOW_MS}ms bound — refusing rather than clamping it down, which would RAISE the rate this declaration asked for`);
|
|
279
|
+
}
|
|
280
|
+
// A window longer than the burst sub-window MUST bound its burst explicitly; a window that is
|
|
281
|
+
// already the sub-window may not, because there its ceiling IS the burst and a second number could
|
|
282
|
+
// only disagree with the first.
|
|
283
|
+
const longWindow = decl.windowMs > RATE_BUDGET_BURST_WINDOW_MS;
|
|
284
|
+
if (longWindow && !finitePositive(decl.burstCeiling)) {
|
|
285
|
+
fail(`${vendor}: a window longer than ${RATE_BUDGET_BURST_WINDOW_MS / 1000}s must also declare \`burstCeiling\` — otherwise the whole window can be spent in one instant`);
|
|
286
|
+
}
|
|
287
|
+
if (!longWindow && decl.burstCeiling !== undefined) {
|
|
288
|
+
fail(`${vendor}: burstCeiling is meaningless for a window of ${decl.windowMs / 1000}s or less — the ceiling already is the burst`);
|
|
289
|
+
}
|
|
290
|
+
if (decl.burstCeiling !== undefined && decl.burstCeiling > decl.ceiling) {
|
|
291
|
+
fail(`${vendor}: burstCeiling ${decl.burstCeiling} exceeds the ceiling ${decl.ceiling}, which bounds it anyway`);
|
|
292
|
+
}
|
|
293
|
+
const rules: RateBudgetWeightRule[] = [];
|
|
294
|
+
const compiled: RegExp[] = [];
|
|
295
|
+
for (const r of decl.rules ?? []) {
|
|
296
|
+
if (!r || typeof r !== 'object' || typeof r.match !== 'string') fail(`${vendor}: every rule needs a \`match\` pattern`);
|
|
297
|
+
if (!finitePositive(r.weight) || r.weight < MIN_RATE_BUDGET_WEIGHT) {
|
|
298
|
+
fail(`${vendor}: rule ${r.match} has weight ${String(r.weight)} — every priced call must cost at least ${MIN_RATE_BUDGET_WEIGHT}`);
|
|
299
|
+
}
|
|
300
|
+
try {
|
|
301
|
+
compiled.push(new RegExp(r.match));
|
|
302
|
+
} catch (e) {
|
|
303
|
+
fail(`${vendor}: rule ${r.match} is not a valid pattern (${(e as Error).message})`);
|
|
304
|
+
}
|
|
305
|
+
rules.push({
|
|
306
|
+
match: r.match,
|
|
307
|
+
weight: r.weight,
|
|
308
|
+
...(r.whenQueryPresent ? { whenQueryPresent: [...r.whenQueryPresent] } : {}),
|
|
309
|
+
...(r.whenQueryAbsent ? { whenQueryAbsent: [...r.whenQueryAbsent] } : {}),
|
|
310
|
+
});
|
|
311
|
+
}
|
|
312
|
+
// Clamped into the kernel's bounds — a typo cannot buy a wider budget than the cap. Both legs
|
|
313
|
+
// move in the TIGHTENING direction only (a shorter window is lengthened, a bigger ceiling is
|
|
314
|
+
// lowered); the widening leg is the `fail()` above.
|
|
315
|
+
const windowMs = Math.max(decl.windowMs, MIN_RATE_BUDGET_WINDOW_MS);
|
|
316
|
+
const ceiling = Math.min(decl.ceiling, MAX_RATE_BUDGET_CEILING);
|
|
317
|
+
const perSecond = ceiling / (windowMs / 1000);
|
|
318
|
+
if (perSecond > MAX_RATE_BUDGET_UNITS_PER_SECOND) {
|
|
319
|
+
fail(
|
|
320
|
+
`${vendor}: ceiling ${ceiling} per ${windowMs}ms is ${perSecond.toFixed(1)} units/second, past the ` +
|
|
321
|
+
`${MAX_RATE_BUDGET_UNITS_PER_SECOND}/s cap. No real vendor budget is that fast; this is a typo or a budget that isn't one.`,
|
|
322
|
+
);
|
|
323
|
+
}
|
|
324
|
+
// The burst bound is clamped to the ceiling too — it can only ever be the tighter of the two.
|
|
325
|
+
const burstCeiling = windowMs > RATE_BUDGET_BURST_WINDOW_MS
|
|
326
|
+
? Math.min(decl.burstCeiling!, ceiling)
|
|
327
|
+
: ceiling;
|
|
328
|
+
return {
|
|
329
|
+
vendor,
|
|
330
|
+
reason: decl.reason,
|
|
331
|
+
windowMs,
|
|
332
|
+
ceiling,
|
|
333
|
+
burstCeiling,
|
|
334
|
+
defaultWeight: decl.defaultWeight,
|
|
335
|
+
maxRetryAfterSeconds: decl.maxRetryAfterSeconds ?? DEFAULT_RATE_BUDGET.maxRetryAfterSeconds!,
|
|
336
|
+
rules,
|
|
337
|
+
compiled,
|
|
338
|
+
};
|
|
339
|
+
}
|
|
340
|
+
|
|
341
|
+
/** Do two rules select exactly the same calls? (Same pattern, same query conditions, same order.) */
|
|
342
|
+
function sameSelector(a: RateBudgetWeightRule, b: RateBudgetWeightRule): boolean {
|
|
343
|
+
const set = (xs?: string[]) => [...(xs ?? [])].sort().join(',');
|
|
344
|
+
return a.match === b.match
|
|
345
|
+
&& set(a.whenQueryPresent) === set(b.whenQueryPresent)
|
|
346
|
+
&& set(a.whenQueryAbsent) === set(b.whenQueryAbsent);
|
|
347
|
+
}
|
|
348
|
+
|
|
349
|
+
/**
|
|
350
|
+
* Is `next` no more permissive than `prev`? (Identical counts as "no more".)
|
|
351
|
+
*
|
|
352
|
+
* PRICING IS PART OF THIS, and it is the part that matters most: the ceiling says how many units
|
|
353
|
+
* fit, the rules say how many units a call COSTS, so dropping an expensive rule multiplies the
|
|
354
|
+
* number of expensive calls that fit without touching a single number. (§9 finding, 2026-07-26 —
|
|
355
|
+
* before this, re-declaring figma with `rules: []` passed every check and quietly turned 5
|
|
356
|
+
* renders/minute into 50, on the exact endpoint whose burst caused the lockout.)
|
|
357
|
+
*
|
|
358
|
+
* The rule kept here is deliberately BLUNT rather than clever: a re-declaration must present the
|
|
359
|
+
* SAME selectors in the SAME order, and may only raise their weights. Deciding whether an
|
|
360
|
+
* arbitrarily reordered/extended rule list prices every possible key at least as high is not
|
|
361
|
+
* decidable in general, and a "clever" approximation is exactly where a bypass hides. A pack that
|
|
362
|
+
* genuinely needs a different rule SHAPE changes its source declaration — which is a reviewable
|
|
363
|
+
* act — rather than re-declaring at runtime.
|
|
364
|
+
*/
|
|
365
|
+
function noMorePermissive(prev: RateBudgetPolicy, next: RateBudgetPolicy): boolean {
|
|
366
|
+
if (next.ceiling > prev.ceiling) return false;
|
|
367
|
+
if (next.burstCeiling > prev.burstCeiling) return false;
|
|
368
|
+
if (next.windowMs < prev.windowMs) return false;
|
|
369
|
+
if (next.defaultWeight < prev.defaultWeight) return false;
|
|
370
|
+
if (next.maxRetryAfterSeconds > prev.maxRetryAfterSeconds) return false;
|
|
371
|
+
if (next.rules.length !== prev.rules.length) return false;
|
|
372
|
+
return prev.rules.every((r, i) => sameSelector(r, next.rules[i]!) && next.rules[i]!.weight >= r.weight);
|
|
373
|
+
}
|
|
374
|
+
|
|
375
|
+
/**
|
|
376
|
+
* DECLARE a vendor's budget. Idempotent for an identical declaration; a REPEAT declaration that
|
|
377
|
+
* would WIDEN the allowance (higher ceiling, shorter window, cheaper default, longer tolerated
|
|
378
|
+
* back-off) is refused. Without that rule, "declare a bigger budget just before constructing the
|
|
379
|
+
* client" would be the clean way around every guarantee below.
|
|
380
|
+
*/
|
|
381
|
+
export function declareRateBudget(vendor: string, decl: RateBudgetDeclaration): RateBudgetPolicy {
|
|
382
|
+
const next = normalize(vendor, decl);
|
|
383
|
+
const prev = declarations.get(vendor);
|
|
384
|
+
if (prev && !noMorePermissive(prev, next)) {
|
|
385
|
+
fail(
|
|
386
|
+
`${vendor}: refusing to re-declare a MORE PERMISSIVE budget (was ceiling ${prev.ceiling}/${prev.windowMs}ms ` +
|
|
387
|
+
`over ${prev.rules.length} priced rule(s), now ${next.ceiling}/${next.windowMs}ms over ${next.rules.length}). ` +
|
|
388
|
+
'A budget may be tightened — lower ceiling, longer window, HIGHER weights — never widened at runtime, and ' +
|
|
389
|
+
'the pricing rules count: dropping an expensive rule widens the budget just as surely as raising the ceiling.',
|
|
390
|
+
);
|
|
391
|
+
}
|
|
392
|
+
declarations.set(vendor, next);
|
|
393
|
+
return next;
|
|
394
|
+
}
|
|
395
|
+
|
|
396
|
+
/** The declared policy for a vendor, or the conservative fallback bound to that vendor. */
|
|
397
|
+
export function rateBudgetPolicy(vendor: string): RateBudgetPolicy {
|
|
398
|
+
return declarations.get(vendor) ?? normalize(vendor, DEFAULT_RATE_BUDGET);
|
|
399
|
+
}
|
|
400
|
+
|
|
401
|
+
/** Has anyone declared a budget for this vendor? (`false` ⇒ it runs on `DEFAULT_RATE_BUDGET`.) */
|
|
402
|
+
export function hasRateBudgetDeclaration(vendor: string): boolean {
|
|
403
|
+
return declarations.has(vendor);
|
|
404
|
+
}
|
|
405
|
+
|
|
406
|
+
/** Every declared policy, sorted by vendor (deterministic) — for operator/audit listings. */
|
|
407
|
+
export function listRateBudgets(): RateBudgetPolicy[] {
|
|
408
|
+
return [...declarations.values()].sort((a, b) => a.vendor.localeCompare(b.vendor));
|
|
409
|
+
}
|
|
410
|
+
|
|
411
|
+
// ── pricing ─────────────────────────────────────────────────────────────────────────────────
|
|
412
|
+
|
|
413
|
+
/**
|
|
414
|
+
* Price one call against a policy. Keyed off the request the client is ABOUT to make, so an
|
|
415
|
+
* unclassified key still costs `defaultWeight` — an unknown endpoint must never be free.
|
|
416
|
+
*/
|
|
417
|
+
export function priceCall(policy: RateBudgetPolicy, key: string, query?: Record<string, string>): number {
|
|
418
|
+
for (const [i, rule] of policy.rules.entries()) {
|
|
419
|
+
if (!(policy.compiled[i] ?? new RegExp(rule.match)).test(key)) continue;
|
|
420
|
+
if (rule.whenQueryPresent && !rule.whenQueryPresent.every((k) => query?.[k] !== undefined)) continue;
|
|
421
|
+
if (rule.whenQueryAbsent && !rule.whenQueryAbsent.every((k) => query?.[k] === undefined)) continue;
|
|
422
|
+
return rule.weight;
|
|
423
|
+
}
|
|
424
|
+
return policy.defaultWeight;
|
|
425
|
+
}
|
|
426
|
+
|
|
427
|
+
/** Price one call for a vendor, using its declared policy (or the fallback). */
|
|
428
|
+
export function rateBudgetWeight(vendor: string, key: string, query?: Record<string, string>): number {
|
|
429
|
+
return priceCall(rateBudgetPolicy(vendor), key, query);
|
|
430
|
+
}
|
|
431
|
+
|
|
432
|
+
// ── the error ───────────────────────────────────────────────────────────────────────────────
|
|
433
|
+
|
|
434
|
+
export type RateBudgetErrorKind =
|
|
435
|
+
/** The rolling-window ceiling would be exceeded by this call. */
|
|
436
|
+
| 'ceiling'
|
|
437
|
+
/** A prior response told us to back off and the cooldown has not elapsed. */
|
|
438
|
+
| 'cooldown'
|
|
439
|
+
/** The ledger could not be read/parsed — treated as a full window (see below). */
|
|
440
|
+
| 'ledger-unreadable'
|
|
441
|
+
/** The ledger could not be persisted, so spend cannot be tracked. Refuse rather than fly blind. */
|
|
442
|
+
| 'ledger-unwritable'
|
|
443
|
+
/** The injected clock is not a millisecond epoch — pruning would be nonsense, so refuse. */
|
|
444
|
+
| 'clock-invalid'
|
|
445
|
+
/** The 60-second BURST bound would be exceeded, even though the long window has room. */
|
|
446
|
+
| 'burst'
|
|
447
|
+
/** A `Retry-After` beyond the cap — fail loudly instead of sleeping it off. */
|
|
448
|
+
| 'retry-after-too-large';
|
|
449
|
+
|
|
450
|
+
/**
|
|
451
|
+
* Thrown INSTEAD OF calling the vendor. Never swallowed, never converted to a retry: the caller is
|
|
452
|
+
* meant to stop. Carries how long to wait so a caller can schedule rather than spin.
|
|
453
|
+
*/
|
|
454
|
+
export class RateBudgetError extends Error {
|
|
455
|
+
readonly kind: RateBudgetErrorKind;
|
|
456
|
+
/** The vendor whose budget refused — so a multi-vendor caller can tell which one stopped. */
|
|
457
|
+
readonly vendor: string;
|
|
458
|
+
/** Milliseconds until the budget could plausibly admit this call again. */
|
|
459
|
+
readonly retryAfterMs: number;
|
|
460
|
+
constructor(kind: RateBudgetErrorKind, vendor: string, message: string, retryAfterMs: number) {
|
|
461
|
+
super(message);
|
|
462
|
+
this.name = 'RateBudgetError';
|
|
463
|
+
this.kind = kind;
|
|
464
|
+
this.vendor = vendor;
|
|
465
|
+
this.retryAfterMs = retryAfterMs;
|
|
466
|
+
}
|
|
467
|
+
}
|
|
468
|
+
|
|
469
|
+
// ── the ledger ──────────────────────────────────────────────────────────────────────────────
|
|
470
|
+
|
|
471
|
+
type LedgerEntry = { t: number; w: number; id: string };
|
|
472
|
+
type Ledger = { v: 1; cooldownUntil: number; entries: LedgerEntry[] };
|
|
473
|
+
|
|
474
|
+
/** A charge already committed to the ledger by `checkBudget`, settled later by `recordCall`. */
|
|
475
|
+
export type RateBudgetReservation = { id: string; weight: number; at: number };
|
|
476
|
+
|
|
477
|
+
export type RateBudgetSnapshot = {
|
|
478
|
+
vendor: string;
|
|
479
|
+
spend: number;
|
|
480
|
+
/** Weighted spend in the last 60s — the quantity `burstCeiling` bounds. */
|
|
481
|
+
burstSpend: number;
|
|
482
|
+
ceiling: number;
|
|
483
|
+
burstCeiling: number;
|
|
484
|
+
windowMs: number;
|
|
485
|
+
cooldownUntil: number;
|
|
486
|
+
entries: number;
|
|
487
|
+
path: string;
|
|
488
|
+
};
|
|
489
|
+
|
|
490
|
+
export type RateBudgetOptions = {
|
|
491
|
+
/** The vendor whose allowance this is. Required — the ledger is keyed by it. */
|
|
492
|
+
vendor: string;
|
|
493
|
+
/** Ledger file path. Injectable so tests never touch the real one. */
|
|
494
|
+
path?: string;
|
|
495
|
+
/**
|
|
496
|
+
* Scope the ledger to ONE project root instead of the home dir. Note that two roots sharing a
|
|
497
|
+
* credential then get two allowances — prefer the token-keyed default unless you specifically
|
|
498
|
+
* want per-world accounting.
|
|
499
|
+
*/
|
|
500
|
+
root?: string;
|
|
501
|
+
/** The credential whose quota this is. Only its HASH is used, to name the default ledger file. */
|
|
502
|
+
token?: string;
|
|
503
|
+
/** Injected clock (ms since epoch). Tests advance it by hand; production passes nothing. */
|
|
504
|
+
now?: () => number;
|
|
505
|
+
/** Window in ms. Can only be made LONGER than the declared one (shorter would widen spend). */
|
|
506
|
+
windowMs?: number;
|
|
507
|
+
/** Weighted ceiling. Can only be made LOWER than the declared one. */
|
|
508
|
+
ceiling?: number;
|
|
509
|
+
/** Seconds; a `Retry-After` above this fails loudly. Can only be made SMALLER. */
|
|
510
|
+
maxRetryAfterSeconds?: number;
|
|
511
|
+
};
|
|
512
|
+
|
|
513
|
+
/**
|
|
514
|
+
* Where the ledger lives. Vendors rate-limit PER CREDENTIAL, so the default is keyed by VENDOR and
|
|
515
|
+
* a hash of the token, and lives in the user's home directory — deliberately NOT under the project
|
|
516
|
+
* state root, because a cwd-scoped ledger hands the same credential a fresh allowance in every
|
|
517
|
+
* checkout, worktree and CI matrix leg. Pass `root` to opt into world-scoped accounting anyway.
|
|
518
|
+
* Only the hash is ever written to disk; the token itself never is.
|
|
519
|
+
*
|
|
520
|
+
* Two vendors never collide (the vendor is a path segment) and two credentials for one vendor never
|
|
521
|
+
* collide (the hash is the filename).
|
|
522
|
+
*/
|
|
523
|
+
export function rateBudgetPath(opts: { vendor: string; root?: string; token?: string }): string {
|
|
524
|
+
if (opts.root !== undefined) return join(worldPaths(opts.vendor, opts.root).dir, 'rate-budget.json');
|
|
525
|
+
const key = opts.token ? createHash('sha256').update(opts.token).digest('hex').slice(0, 16) : 'no-token';
|
|
526
|
+
return join(homedir(), '.volter', opts.vendor, `rate-budget-${key}.json`);
|
|
527
|
+
}
|
|
528
|
+
|
|
529
|
+
function sleepSync(ms: number): void {
|
|
530
|
+
// Works on the main thread in both Bun and Node (unlike a browser main thread).
|
|
531
|
+
Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, ms);
|
|
532
|
+
}
|
|
533
|
+
|
|
534
|
+
/** Is a pid still running? `kill(pid, 0)` throws ESRCH only when it is gone. */
|
|
535
|
+
function pidAlive(pid: number): boolean {
|
|
536
|
+
if (!Number.isInteger(pid) || pid <= 0) return false;
|
|
537
|
+
try {
|
|
538
|
+
process.kill(pid, 0);
|
|
539
|
+
return true;
|
|
540
|
+
} catch (e) {
|
|
541
|
+
return (e as NodeJS.ErrnoException).code === 'EPERM'; // alive, just not ours
|
|
542
|
+
}
|
|
543
|
+
}
|
|
544
|
+
|
|
545
|
+
/**
|
|
546
|
+
* The guard. One instance per ledger path; cheap to construct, so callers may build one per client.
|
|
547
|
+
* All state lives in the FILE, never in the instance — that is what makes a second instance (or a
|
|
548
|
+
* second process) see the first one's spend. There is no memoization anywhere, by design.
|
|
549
|
+
*/
|
|
550
|
+
/**
|
|
551
|
+
* Refuse a "budget" whose GUARD HAS BEEN REPLACED, and hand back the same object when it is intact.
|
|
552
|
+
*
|
|
553
|
+
* Every pack's guarded factory takes an optional `budget` so several clients can share one ledger,
|
|
554
|
+
* and each one used to validate it with `budget instanceof XBudget`. §9 (2026-07-26) showed that is
|
|
555
|
+
* not a check at all — the subclass is one line, and `checkBudget` is an ordinary prototype method:
|
|
556
|
+
*
|
|
557
|
+
* class Loose extends GithubBudget { checkBudget(w) { return { id: 'x', weight: w, at: 0 }; } }
|
|
558
|
+
* liveGithubExecute(pat, undefined, { budget: new Loose() }) // 5,000 requests, zero refusals
|
|
559
|
+
*
|
|
560
|
+
* `instanceof` said yes; so would a `Proxy` over a genuine budget whose `get` returns a no-op. Both
|
|
561
|
+
* are exactly the "one clean way around the guard" those factories claimed to have closed.
|
|
562
|
+
*
|
|
563
|
+
* So the check is on the METHODS, not the prototype chain: the three functions a guarded factory
|
|
564
|
+
* actually calls must be the very functions `RateBudget.prototype` defines. An own-property
|
|
565
|
+
* override, a subclass override and a `get`-trapping proxy all fail that identity comparison. The
|
|
566
|
+
* exact-prototype test additionally pins the vendor binding, so another vendor's budget subclass
|
|
567
|
+
* cannot stand in for this one's.
|
|
568
|
+
*
|
|
569
|
+
* HONEST LIMIT, since the point of this function is not over-claiming: a determined caller can still
|
|
570
|
+
* defeat it (a proxy that returns the real function to the probe and a no-op afterwards, a
|
|
571
|
+
* `Object.defineProperty` on `RateBudget.prototype` itself, an injected clock, a ledger path in a
|
|
572
|
+
* temp dir). Nothing reachable from inside this process can stop that, and the module header says so
|
|
573
|
+
* up front. What this closes is the ACCIDENT and the one-liner — the shapes that actually happen.
|
|
574
|
+
*/
|
|
575
|
+
export function assertBudgetGuardIntact<T extends RateBudget>(
|
|
576
|
+
budget: unknown,
|
|
577
|
+
expected: new (...args: never[]) => T,
|
|
578
|
+
where: string,
|
|
579
|
+
): T {
|
|
580
|
+
const what = `${where}: \`budget\` must be an unmodified ${expected.name}`;
|
|
581
|
+
if (!(budget instanceof expected)) {
|
|
582
|
+
throw new Error(`${what} — refusing to build a client around an unverified rate guard`);
|
|
583
|
+
}
|
|
584
|
+
if (Object.getPrototypeOf(budget) !== expected.prototype) {
|
|
585
|
+
throw new Error(`${what} — a SUBCLASS may override the guard, so it is refused rather than trusted`);
|
|
586
|
+
}
|
|
587
|
+
const b = budget as unknown as Record<string, unknown>;
|
|
588
|
+
const proto = RateBudget.prototype as unknown as Record<string, unknown>;
|
|
589
|
+
for (const method of ['checkBudget', 'recordCall', 'weightFor'] as const) {
|
|
590
|
+
if (b[method] !== proto[method]) {
|
|
591
|
+
throw new Error(`${what} — its \`${method}\` is not the kernel's, so the ceiling it reports cannot be trusted`);
|
|
592
|
+
}
|
|
593
|
+
}
|
|
594
|
+
// …and it must actually have been CONSTRUCTED. `Object.create(XBudget.prototype)` has the right
|
|
595
|
+
// prototype and inherits the real methods, so everything above passes — but it has no `vendor` and
|
|
596
|
+
// no `path`, and the first call dies inside `node:fs` with "the \"path\" property must be of type
|
|
597
|
+
// string". That already fails CLOSED (zero requests reach the vendor), so this is not a hole; it is
|
|
598
|
+
// a legibility fix, turning an inscrutable fs error into a refusal that names the real problem.
|
|
599
|
+
if (typeof b.vendor !== 'string' || b.vendor === '' || typeof b.path !== 'string' || b.path === '') {
|
|
600
|
+
throw new Error(`${what} — it was never constructed (no vendor/ledger path), so it has no ledger to charge`);
|
|
601
|
+
}
|
|
602
|
+
return budget;
|
|
603
|
+
}
|
|
604
|
+
|
|
605
|
+
export class RateBudget {
|
|
606
|
+
readonly vendor: string;
|
|
607
|
+
readonly path: string;
|
|
608
|
+
private readonly clock: () => number;
|
|
609
|
+
private readonly askedWindowMs: number | undefined;
|
|
610
|
+
private readonly askedCeiling: number | undefined;
|
|
611
|
+
private readonly askedMaxRetryAfterSeconds: number | undefined;
|
|
612
|
+
|
|
613
|
+
constructor(opts: RateBudgetOptions) {
|
|
614
|
+
if (!opts || typeof opts.vendor !== 'string' || opts.vendor === '') {
|
|
615
|
+
fail('a budget must name the vendor it guards — the ledger is keyed by it');
|
|
616
|
+
}
|
|
617
|
+
this.vendor = opts.vendor;
|
|
618
|
+
// Resolving the policy here would VALIDATE the vendor id early; do that explicitly instead,
|
|
619
|
+
// since the policy itself is now read live (see `policy`).
|
|
620
|
+
rateBudgetPolicy(opts.vendor);
|
|
621
|
+
this.path = opts.path ?? rateBudgetPath({
|
|
622
|
+
vendor: opts.vendor,
|
|
623
|
+
...(opts.root !== undefined ? { root: opts.root } : {}),
|
|
624
|
+
...(opts.token !== undefined ? { token: opts.token } : {}),
|
|
625
|
+
});
|
|
626
|
+
this.clock = opts.now ?? (() => Date.now());
|
|
627
|
+
this.askedWindowMs = opts.windowMs;
|
|
628
|
+
this.askedCeiling = opts.ceiling;
|
|
629
|
+
this.askedMaxRetryAfterSeconds = opts.maxRetryAfterSeconds;
|
|
630
|
+
}
|
|
631
|
+
|
|
632
|
+
/**
|
|
633
|
+
* The effective policy, read LIVE from the declaration registry on every use rather than captured
|
|
634
|
+
* at construction.
|
|
635
|
+
*
|
|
636
|
+
* That matters because of an ordering hazard §9 found (2026-07-26): a budget built for a vendor
|
|
637
|
+
* whose declaration module has not been imported yet falls back to `DEFAULT_RATE_BUDGET`, and the
|
|
638
|
+
* fallback is NOT uniformly tighter than a pack's own numbers — its ceiling is lower, but it
|
|
639
|
+
* prices every call at `defaultWeight`, which for a vendor with an expensive endpoint (a render,
|
|
640
|
+
* a recursive walk) is far CHEAPER than the pack's declared weight for it. Capturing that
|
|
641
|
+
* snapshot would freeze the wrong prices for the instance's whole life. Reading live means the
|
|
642
|
+
* pack's real weights take effect the moment its declaration lands.
|
|
643
|
+
*
|
|
644
|
+
* A caller cannot hand in a policy object — it comes only from the registry, and the registry
|
|
645
|
+
* refuses a widening re-declaration — so "construct a client around a huge made-up budget" is
|
|
646
|
+
* still not a move.
|
|
647
|
+
*/
|
|
648
|
+
get policy(): RateBudgetPolicy {
|
|
649
|
+
return rateBudgetPolicy(this.vendor);
|
|
650
|
+
}
|
|
651
|
+
|
|
652
|
+
/** Window in ms. The caller may only LENGTHEN it; a shorter one would widen spend. */
|
|
653
|
+
get windowMs(): number {
|
|
654
|
+
const declared = this.policy.windowMs;
|
|
655
|
+
return Math.max(this.askedWindowMs ?? declared, declared);
|
|
656
|
+
}
|
|
657
|
+
|
|
658
|
+
/** Weighted ceiling. The caller may only LOWER it. */
|
|
659
|
+
get ceiling(): number {
|
|
660
|
+
const declared = this.policy.ceiling;
|
|
661
|
+
return Math.min(this.askedCeiling ?? declared, declared);
|
|
662
|
+
}
|
|
663
|
+
|
|
664
|
+
/**
|
|
665
|
+
* Weighted units allowed in any 60-SECOND sub-window, enforced independently of `ceiling`. For a
|
|
666
|
+
* pack whose window already IS 60s this equals the ceiling and the second check is a no-op; for a
|
|
667
|
+
* longer window it is the bound that stops the whole window being spent in an instant. Never
|
|
668
|
+
* looser than `ceiling` (a lowered ceiling lowers it too).
|
|
669
|
+
*/
|
|
670
|
+
get burstCeiling(): number {
|
|
671
|
+
return Math.min(this.policy.burstCeiling, this.ceiling);
|
|
672
|
+
}
|
|
673
|
+
|
|
674
|
+
/** Tolerated back-off, in seconds. The caller may only SHRINK it. */
|
|
675
|
+
get maxRetryAfterSeconds(): number {
|
|
676
|
+
const declared = this.policy.maxRetryAfterSeconds;
|
|
677
|
+
return Math.min(this.askedMaxRetryAfterSeconds ?? declared, declared);
|
|
678
|
+
}
|
|
679
|
+
|
|
680
|
+
/** Price a call against THIS vendor's declared rules, as they stand right now. */
|
|
681
|
+
weightFor(key: string, query?: Record<string, string>): number {
|
|
682
|
+
return priceCall(this.policy, key, query);
|
|
683
|
+
}
|
|
684
|
+
|
|
685
|
+
/**
|
|
686
|
+
* Read the clock and sanity-check it. Catches the ACCIDENT class that silently disables pruning:
|
|
687
|
+
* a clock in seconds, a `performance.now()`-style monotonic clock, or NaN. Entry timestamps
|
|
688
|
+
* written under one epoch prune instantly under another, which would void the whole budget
|
|
689
|
+
* without anyone noticing. (A deliberate fast-forwarding clock is NOT stopped by this — nothing
|
|
690
|
+
* can stop that while the seam exists; it is disclosed at the top instead of pretended away.)
|
|
691
|
+
*/
|
|
692
|
+
private now(): number {
|
|
693
|
+
const t = this.clock();
|
|
694
|
+
if (!Number.isFinite(t) || t < MIN_PLAUSIBLE_EPOCH_MS) {
|
|
695
|
+
throw new RateBudgetError(
|
|
696
|
+
'clock-invalid',
|
|
697
|
+
this.vendor,
|
|
698
|
+
`${this.vendor} budget: injected clock returned ${String(t)}, which is not a millisecond epoch — refusing rather than pruning the ledger against a nonsense clock`,
|
|
699
|
+
this.windowMs,
|
|
700
|
+
);
|
|
701
|
+
}
|
|
702
|
+
return t;
|
|
703
|
+
}
|
|
704
|
+
|
|
705
|
+
// ── locking + durable IO ──────────────────────────────────────────────────────────────────
|
|
706
|
+
|
|
707
|
+
private lockPath(): string { return `${this.path}.lock`; }
|
|
708
|
+
|
|
709
|
+
/** Acquire the exclusive lock, run `fn`, always release. Bounded wait, dead locks broken. */
|
|
710
|
+
private withLock<T>(fn: () => T): T {
|
|
711
|
+
const lock = this.lockPath();
|
|
712
|
+
mkdirSync(dirname(this.path), { recursive: true });
|
|
713
|
+
let fd: number | undefined;
|
|
714
|
+
// ~1s total: 200 attempts x 5ms. Contention here is a few fs ops long, so this is generous.
|
|
715
|
+
for (let attempt = 0; attempt < 200; attempt += 1) {
|
|
716
|
+
try {
|
|
717
|
+
fd = openSync(lock, 'wx');
|
|
718
|
+
// Stamp the holder so a waiter can tell "crashed" from "slow". An age check alone would
|
|
719
|
+
// let a waiter delete a LIVE holder's lock during a long hold (suspend, fs stall) — two
|
|
720
|
+
// writers, a lost update, and the ledger under-reporting spend, i.e. failing OPEN.
|
|
721
|
+
try { writeSync(fd, JSON.stringify({ pid: process.pid, at: Date.now() })); } catch { /* advisory only */ }
|
|
722
|
+
break;
|
|
723
|
+
} catch {
|
|
724
|
+
this.breakDeadLock(lock);
|
|
725
|
+
sleepSync(5);
|
|
726
|
+
}
|
|
727
|
+
}
|
|
728
|
+
if (fd === undefined) {
|
|
729
|
+
// Could not serialize the read-modify-write ⇒ cannot guarantee the ceiling ⇒ refuse.
|
|
730
|
+
throw new RateBudgetError('ledger-unwritable', this.vendor, `${this.vendor} budget: could not acquire ${lock}`, this.windowMs);
|
|
731
|
+
}
|
|
732
|
+
try {
|
|
733
|
+
return fn();
|
|
734
|
+
} finally {
|
|
735
|
+
// Release the lock WE hold, identified by inode — never by path. A holder that was itself
|
|
736
|
+
// broken must not delete its successor's lock on the way out (a stomp cascade).
|
|
737
|
+
try {
|
|
738
|
+
const mine = fstatSync(fd).ino;
|
|
739
|
+
try { if (statSync(lock).ino === mine) rmSync(lock, { force: true }); } catch { /* already gone */ }
|
|
740
|
+
} catch { /* cannot stat our own fd; leave it for the dead-lock breaker */ }
|
|
741
|
+
try { closeSync(fd); } catch { /* already closed */ }
|
|
742
|
+
}
|
|
743
|
+
}
|
|
744
|
+
|
|
745
|
+
/** Remove a lock whose owner is provably gone. Never removes one that might still be live. */
|
|
746
|
+
private breakDeadLock(lock: string): void {
|
|
747
|
+
try {
|
|
748
|
+
const age = Date.now() - statSync(lock).mtimeMs;
|
|
749
|
+
if (age <= STALE_LOCK_MS) return;
|
|
750
|
+
// Old is NOT enough — a slow holder is old too. Require the owning process to be gone.
|
|
751
|
+
let owner: number | undefined;
|
|
752
|
+
try { owner = (JSON.parse(readFileSync(lock, 'utf8')) as { pid?: number }).pid; } catch { /* unstamped */ }
|
|
753
|
+
// An unstamped lock predates this scheme (or its stamp write failed); age alone must do.
|
|
754
|
+
if (owner !== undefined && pidAlive(owner)) return;
|
|
755
|
+
rmSync(lock, { force: true });
|
|
756
|
+
} catch { /* raced with the holder releasing it; just retry */ }
|
|
757
|
+
}
|
|
758
|
+
|
|
759
|
+
/**
|
|
760
|
+
* Read the ledger. A MISSING file is the legitimate first-run case (no state ⇒ no spend). A file
|
|
761
|
+
* that exists but cannot be read or parsed is the dangerous case, and it FAILS CLOSED: we cannot
|
|
762
|
+
* know what was already spent, so the window is treated as fully consumed. A fresh ledger
|
|
763
|
+
* carrying a one-window cooldown is written in its place, so the guard self-heals after the
|
|
764
|
+
* window instead of needing a human to delete a file.
|
|
765
|
+
*/
|
|
766
|
+
private load(now: number): Ledger {
|
|
767
|
+
if (!existsSync(this.path)) return { v: 1, cooldownUntil: 0, entries: [] };
|
|
768
|
+
let raw: string;
|
|
769
|
+
try {
|
|
770
|
+
raw = readFileSync(this.path, 'utf8');
|
|
771
|
+
} catch (e) {
|
|
772
|
+
throw this.quarantine(now, `unreadable (${(e as Error).message})`);
|
|
773
|
+
}
|
|
774
|
+
let parsed: unknown;
|
|
775
|
+
try {
|
|
776
|
+
parsed = JSON.parse(raw);
|
|
777
|
+
} catch (e) {
|
|
778
|
+
throw this.quarantine(now, `malformed JSON (${(e as Error).message})`);
|
|
779
|
+
}
|
|
780
|
+
const l = parsed as Partial<Ledger>;
|
|
781
|
+
if (!l || typeof l !== 'object' || l.v !== 1 || !Array.isArray(l.entries)) {
|
|
782
|
+
throw this.quarantine(now, 'unrecognized ledger shape');
|
|
783
|
+
}
|
|
784
|
+
if (l.entries.length > MAX_LEDGER_ENTRIES) {
|
|
785
|
+
throw this.quarantine(now, `${l.entries.length} entries, far past the ${MAX_LEDGER_ENTRIES} a valid window can hold`);
|
|
786
|
+
}
|
|
787
|
+
const entries: LedgerEntry[] = [];
|
|
788
|
+
for (const e of l.entries as unknown[]) {
|
|
789
|
+
const row = e as Partial<LedgerEntry>;
|
|
790
|
+
// A row we cannot price is not skipped — an entry that exists must cost something, or a
|
|
791
|
+
// partially-corrupt ledger would silently forgive spend.
|
|
792
|
+
if (!row || typeof row !== 'object') throw this.quarantine(now, 'non-object ledger entry');
|
|
793
|
+
const t = Number(row.t);
|
|
794
|
+
const w = Number(row.w);
|
|
795
|
+
if (!Number.isFinite(t) || !Number.isFinite(w) || w < 0) throw this.quarantine(now, 'non-numeric ledger entry');
|
|
796
|
+
entries.push({ t, w, id: typeof row.id === 'string' ? row.id : randomUUID() });
|
|
797
|
+
}
|
|
798
|
+
// A garbage cooldown must not silently DISARM a back-off a 429 armed — that is the one
|
|
799
|
+
// corruption that would fail open on the cooldown path, so it quarantines like the rest.
|
|
800
|
+
const cooldownUntil = Number(l.cooldownUntil);
|
|
801
|
+
if (!Number.isFinite(cooldownUntil) || cooldownUntil < 0) throw this.quarantine(now, 'non-numeric cooldownUntil');
|
|
802
|
+
return { v: 1, cooldownUntil, entries };
|
|
803
|
+
}
|
|
804
|
+
|
|
805
|
+
/** Replace a corrupt ledger with a valid one that is already in cooldown, and refuse this call. */
|
|
806
|
+
private quarantine(now: number, why: string): RateBudgetError {
|
|
807
|
+
const until = now + this.windowMs;
|
|
808
|
+
try {
|
|
809
|
+
this.save({ v: 1, cooldownUntil: until, entries: [] });
|
|
810
|
+
} catch { /* best effort — the throw below is what actually protects the credential */ }
|
|
811
|
+
return new RateBudgetError(
|
|
812
|
+
'ledger-unreadable',
|
|
813
|
+
this.vendor,
|
|
814
|
+
`${this.vendor} budget: ledger ${this.path} is ${why} — refusing the call and cooling down for ${Math.round(this.windowMs / 1000)}s ` +
|
|
815
|
+
'rather than assuming zero spend',
|
|
816
|
+
this.windowMs,
|
|
817
|
+
);
|
|
818
|
+
}
|
|
819
|
+
|
|
820
|
+
/** Atomic replace: write a temp file, then rename over the target. No partial ledger is visible. */
|
|
821
|
+
private save(ledger: Ledger): void {
|
|
822
|
+
const tmp = `${this.path}.${process.pid}.${randomUUID().slice(0, 8)}.tmp`;
|
|
823
|
+
try {
|
|
824
|
+
mkdirSync(dirname(this.path), { recursive: true });
|
|
825
|
+
writeFileSync(tmp, `${JSON.stringify(ledger)}\n`, 'utf8');
|
|
826
|
+
renameSync(tmp, this.path);
|
|
827
|
+
} catch (e) {
|
|
828
|
+
try { rmSync(tmp, { force: true }); } catch { /* nothing to clean */ }
|
|
829
|
+
// If spend cannot be persisted we would be flying blind on the next call. Refuse instead.
|
|
830
|
+
throw new RateBudgetError(
|
|
831
|
+
'ledger-unwritable',
|
|
832
|
+
this.vendor,
|
|
833
|
+
`${this.vendor} budget: cannot persist ${this.path} (${(e as Error).message}) — refusing rather than tracking nothing`,
|
|
834
|
+
this.windowMs,
|
|
835
|
+
);
|
|
836
|
+
}
|
|
837
|
+
}
|
|
838
|
+
|
|
839
|
+
private prune(ledger: Ledger, now: number): Ledger {
|
|
840
|
+
return { ...ledger, entries: ledger.entries.filter((e) => now - e.t < this.windowMs) };
|
|
841
|
+
}
|
|
842
|
+
|
|
843
|
+
private static spendOf(ledger: Ledger): number {
|
|
844
|
+
return ledger.entries.reduce((n, e) => n + e.w, 0);
|
|
845
|
+
}
|
|
846
|
+
|
|
847
|
+
// ── the API ───────────────────────────────────────────────────────────────────────────────
|
|
848
|
+
|
|
849
|
+
/**
|
|
850
|
+
* THE GATE. Prunes the window, refuses if we are in cooldown or if `weight` would breach the
|
|
851
|
+
* ceiling, and otherwise RESERVES the weight (committing it to the ledger) before returning.
|
|
852
|
+
* When it throws, the caller MUST NOT call the vendor — that is the entire contract.
|
|
853
|
+
*/
|
|
854
|
+
checkBudget(weight: number): RateBudgetReservation {
|
|
855
|
+
const w = Number.isFinite(weight) && weight > 0 ? weight : this.policy.defaultWeight;
|
|
856
|
+
return this.withLock(() => {
|
|
857
|
+
const now = this.now();
|
|
858
|
+
const ledger = this.prune(this.load(now), now);
|
|
859
|
+
|
|
860
|
+
if (ledger.cooldownUntil > now) {
|
|
861
|
+
const wait = ledger.cooldownUntil - now;
|
|
862
|
+
throw new RateBudgetError(
|
|
863
|
+
'cooldown',
|
|
864
|
+
this.vendor,
|
|
865
|
+
`${this.vendor} budget: in cooldown for another ${Math.ceil(wait / 1000)}s (a previous response asked us to back off) — not calling the vendor`,
|
|
866
|
+
wait,
|
|
867
|
+
);
|
|
868
|
+
}
|
|
869
|
+
|
|
870
|
+
// THE BURST BOUND, checked first because it is the tighter one whenever it differs. A long
|
|
871
|
+
// window bounds how much may be spent per hour; it says nothing about how fast, and "all of it
|
|
872
|
+
// in one millisecond" is exactly the shape a runaway loop has. (§9 round 2, 2026-07-26.)
|
|
873
|
+
const burstCeiling = this.burstCeiling;
|
|
874
|
+
if (burstCeiling < this.ceiling) {
|
|
875
|
+
const since = now - RATE_BUDGET_BURST_WINDOW_MS;
|
|
876
|
+
const recent = ledger.entries.filter((e) => e.t > since);
|
|
877
|
+
const burstSpend = recent.reduce((n, e) => n + e.w, 0);
|
|
878
|
+
if (burstSpend + w > burstCeiling) {
|
|
879
|
+
const oldest = recent.reduce((min, e) => Math.min(min, e.t), now);
|
|
880
|
+
const wait = Math.max(0, RATE_BUDGET_BURST_WINDOW_MS - (now - oldest));
|
|
881
|
+
throw new RateBudgetError(
|
|
882
|
+
'burst',
|
|
883
|
+
this.vendor,
|
|
884
|
+
`${this.vendor} budget: this call costs ${w} and ${burstSpend}/${burstCeiling} weighted units are already spent in the last ` +
|
|
885
|
+
`${RATE_BUDGET_BURST_WINDOW_MS / 1000}s — refusing the call. The ${Math.round(this.windowMs / 1000)}s window still has room; ` +
|
|
886
|
+
`this is the BURST bound, which exists so a long window cannot be spent all at once. Retry in ~${Math.ceil(wait / 1000)}s.`,
|
|
887
|
+
wait,
|
|
888
|
+
);
|
|
889
|
+
}
|
|
890
|
+
}
|
|
891
|
+
|
|
892
|
+
const spend = RateBudget.spendOf(ledger);
|
|
893
|
+
if (spend + w > this.ceiling) {
|
|
894
|
+
// Wait only as long as it takes for enough of the window to age out.
|
|
895
|
+
const oldest = ledger.entries.reduce((min, e) => Math.min(min, e.t), now);
|
|
896
|
+
const wait = Math.max(0, this.windowMs - (now - oldest));
|
|
897
|
+
throw new RateBudgetError(
|
|
898
|
+
'ceiling',
|
|
899
|
+
this.vendor,
|
|
900
|
+
`${this.vendor} budget: this call costs ${w} and ${spend}/${this.ceiling} weighted units are already spent in the last ` +
|
|
901
|
+
`${Math.round(this.windowMs / 1000)}s — refusing the call. Retry in ~${Math.ceil(wait / 1000)}s; ` +
|
|
902
|
+
'cached work is preserved, so a re-run resumes where this stopped.',
|
|
903
|
+
wait,
|
|
904
|
+
);
|
|
905
|
+
}
|
|
906
|
+
|
|
907
|
+
const reservation: RateBudgetReservation = { id: randomUUID(), weight: w, at: now };
|
|
908
|
+
ledger.entries.push({ t: now, w, id: reservation.id });
|
|
909
|
+
this.save(ledger);
|
|
910
|
+
return reservation;
|
|
911
|
+
});
|
|
912
|
+
}
|
|
913
|
+
|
|
914
|
+
/**
|
|
915
|
+
* Record the OUTCOME of a call. Settles the reservation `checkBudget` made (never charging
|
|
916
|
+
* twice); when called without one — a direct caller, or a call made outside the guarded path —
|
|
917
|
+
* it appends the charge itself, so spend is recorded either way.
|
|
918
|
+
*
|
|
919
|
+
* If the response carried `Retry-After`, a 429, or an `X-…-RateLimit-Remaining: 0` exhaustion
|
|
920
|
+
* signal, a COOLDOWN is persisted: every later call then fails fast in `checkBudget` WITHOUT
|
|
921
|
+
* touching the vendor. A `Retry-After` beyond the cap additionally throws — that is not something
|
|
922
|
+
* to sleep off.
|
|
923
|
+
*/
|
|
924
|
+
recordCall(
|
|
925
|
+
weight: number,
|
|
926
|
+
headers?: Record<string, string>,
|
|
927
|
+
opts: { status?: number; reservation?: RateBudgetReservation | null } = {},
|
|
928
|
+
): void {
|
|
929
|
+
const w = Number.isFinite(weight) && weight > 0 ? weight : this.policy.defaultWeight;
|
|
930
|
+
// The EFFECTIVE window, not the declared one. `windowMs` honours a caller's lengthening; using
|
|
931
|
+
// the declared value would hand a caller who deliberately widened their window the SHORTER
|
|
932
|
+
// fallback cooldown — the only loosening direction left in a class where every other clamp
|
|
933
|
+
// tightens. (§9 finding, 2026-07-26.)
|
|
934
|
+
//
|
|
935
|
+
// THIS budget's clock, too, not the wall clock. An ABSOLUTE epoch reset header has to be
|
|
936
|
+
// subtracted from the same clock the ledger is timestamped against; reading it against
|
|
937
|
+
// `Date.now()` while the ledger runs on an injected clock silently mixes two time bases, and the
|
|
938
|
+
// difference between them lands directly in the cooldown. (§9 finding, 2026-07-26.)
|
|
939
|
+
const nowMs = this.now();
|
|
940
|
+
const backoff = backoffSeconds(headers, opts.status, this.windowMs, nowMs);
|
|
941
|
+
const thrown = this.withLock(() => {
|
|
942
|
+
const now = this.now();
|
|
943
|
+
const ledger = this.prune(this.load(now), now);
|
|
944
|
+
|
|
945
|
+
const held = opts.reservation ? ledger.entries.find((e) => e.id === opts.reservation!.id) : undefined;
|
|
946
|
+
if (held) held.t = now; // settle in place — already charged, do not charge again
|
|
947
|
+
else ledger.entries.push({ t: now, w, id: randomUUID() });
|
|
948
|
+
|
|
949
|
+
if (backoff !== null) {
|
|
950
|
+
const until = now + Math.min(backoff * 1000, MAX_COOLDOWN_MS);
|
|
951
|
+
ledger.cooldownUntil = Math.max(ledger.cooldownUntil, until);
|
|
952
|
+
}
|
|
953
|
+
this.save(ledger);
|
|
954
|
+
|
|
955
|
+
if (backoff !== null && backoff > this.maxRetryAfterSeconds) {
|
|
956
|
+
return new RateBudgetError(
|
|
957
|
+
'retry-after-too-large',
|
|
958
|
+
this.vendor,
|
|
959
|
+
`${this.vendor} budget: the vendor asked for a ${backoff}s backoff, beyond the ${this.maxRetryAfterSeconds}s cap — ` +
|
|
960
|
+
'failing immediately instead of sleeping it off. The credential is being throttled hard; stop calling.',
|
|
961
|
+
backoff * 1000,
|
|
962
|
+
);
|
|
963
|
+
}
|
|
964
|
+
return null;
|
|
965
|
+
});
|
|
966
|
+
if (thrown) throw thrown;
|
|
967
|
+
}
|
|
968
|
+
|
|
969
|
+
/** Current weighted spend in the window. Reads the FILE, so it sees other processes' spend. */
|
|
970
|
+
spend(): number {
|
|
971
|
+
return this.withLock(() => {
|
|
972
|
+
const now = this.now();
|
|
973
|
+
return RateBudget.spendOf(this.prune(this.load(now), now));
|
|
974
|
+
});
|
|
975
|
+
}
|
|
976
|
+
|
|
977
|
+
/** Everything an operator (or a test) wants to see, in one read. */
|
|
978
|
+
snapshot(): RateBudgetSnapshot {
|
|
979
|
+
return this.withLock(() => {
|
|
980
|
+
const now = this.now();
|
|
981
|
+
const ledger = this.prune(this.load(now), now);
|
|
982
|
+
const since = now - RATE_BUDGET_BURST_WINDOW_MS;
|
|
983
|
+
return {
|
|
984
|
+
vendor: this.vendor,
|
|
985
|
+
spend: RateBudget.spendOf(ledger),
|
|
986
|
+
burstSpend: ledger.entries.filter((e) => e.t > since).reduce((n, e) => n + e.w, 0),
|
|
987
|
+
ceiling: this.ceiling,
|
|
988
|
+
burstCeiling: this.burstCeiling,
|
|
989
|
+
windowMs: this.windowMs,
|
|
990
|
+
cooldownUntil: ledger.cooldownUntil,
|
|
991
|
+
entries: ledger.entries.length,
|
|
992
|
+
path: this.path,
|
|
993
|
+
};
|
|
994
|
+
});
|
|
995
|
+
}
|
|
996
|
+
}
|
|
997
|
+
|
|
998
|
+
/**
|
|
999
|
+
* Seconds to back off for, or `null` if the response carried no back-off signal.
|
|
1000
|
+
*
|
|
1001
|
+
* Recognizes the two forms that are effectively universal across REST vendors — the standard
|
|
1002
|
+
* `Retry-After` header and a bare 429 — plus the widely-used `*-ratelimit-remaining: 0` /
|
|
1003
|
+
* `*-ratelimit-reset` convention, matched by SUFFIX so any vendor's prefix works without the
|
|
1004
|
+
* kernel knowing the vendor's name. Every one of them is read IF PRESENT and never assumed.
|
|
1005
|
+
*/
|
|
1006
|
+
function backoffSeconds(
|
|
1007
|
+
headers: Record<string, string> | undefined,
|
|
1008
|
+
status: number | undefined,
|
|
1009
|
+
windowMs: number,
|
|
1010
|
+
nowMs: number,
|
|
1011
|
+
): number | null {
|
|
1012
|
+
const h: Record<string, string> = {};
|
|
1013
|
+
for (const [k, v] of Object.entries(headers ?? {})) h[k.toLowerCase()] = v;
|
|
1014
|
+
|
|
1015
|
+
const retryAfter = Number(h['retry-after']);
|
|
1016
|
+
if (h['retry-after'] !== undefined && h['retry-after'] !== '' && Number.isFinite(retryAfter) && retryAfter >= 0) {
|
|
1017
|
+
return retryAfter;
|
|
1018
|
+
}
|
|
1019
|
+
// An explicit "you have nothing left" signal, with a reset hint when one is offered. The header
|
|
1020
|
+
// NAMES vary by vendor, so match STRUCTURALLY rather than against a table of vendor spellings:
|
|
1021
|
+
// a header counts if its dash-delimited words contain a `ratelimit`/`rate-limit` token AND the
|
|
1022
|
+
// word `remaining` (or `reset`). That covers every shape observed in practice — and, crucially,
|
|
1023
|
+
// ALL THREE WORD ORDERS, which a suffix match did not (§9 finding, 2026-07-26, class-A wiring):
|
|
1024
|
+
// • `x-ratelimit-remaining` (GitHub, PostHog, Supabase, Cal.com)
|
|
1025
|
+
// • `x-<vendor>-rate-limit-remaining` (Figma, Sentry)
|
|
1026
|
+
// • `<vendor>-ratelimit-<dimension>-remaining` (Anthropic, Linear) ← infix, was DROPPED
|
|
1027
|
+
// • `x-ratelimit-remaining-<dimension>` (OpenAI) ← trailing, was DROPPED
|
|
1028
|
+
// Dropping one is a FAIL-OPEN: the ledger keeps cheerfully spending into a credential the vendor
|
|
1029
|
+
// has already told us is exhausted. The kernel still knows no vendor's name — only the shape.
|
|
1030
|
+
//
|
|
1031
|
+
// EVERY matching header is examined, not the first one found (§9 finding, 2026-07-26): a proxy or
|
|
1032
|
+
// CDN in front of the vendor can emit its own `x-ratelimit-remaining: 100` ahead of the vendor's
|
|
1033
|
+
// `x-<vendor>-rate-limit-remaining: 0`, and picking the first match would silently DROP the
|
|
1034
|
+
// exhaustion signal — a fail-open the exact-name lookup this replaced could not produce. If any
|
|
1035
|
+
// of them says zero, we are out.
|
|
1036
|
+
const hasWord = (k: string, w: string) => new RegExp(`(^|-)${w}(-|$)`).test(k);
|
|
1037
|
+
const isLimitHeader = (k: string) => /(^|-)rate-?limit(-|$)/.test(k);
|
|
1038
|
+
const remainingKeys = Object.keys(h).filter((k) => isLimitHeader(k) && hasWord(k, 'remaining'));
|
|
1039
|
+
// A PRESENT-BUT-EMPTY value is ABSENT, not zero. `Number('') === 0` and `Number(' ') === 0`, so a
|
|
1040
|
+
// blank header from a proxy that reserves the name without filling it would otherwise read as "you
|
|
1041
|
+
// are exhausted" and refuse a perfectly healthy credential for a whole window — a false positive
|
|
1042
|
+
// the widened matcher above makes far more reachable. (§9 finding, 2026-07-26.)
|
|
1043
|
+
const numeric = (v: string | undefined): number => (v === undefined || v.trim() === '' ? NaN : Number(v));
|
|
1044
|
+
const exhausted = remainingKeys.filter((k) => numeric(h[k]) === 0);
|
|
1045
|
+
if (exhausted.length > 0) {
|
|
1046
|
+
// Prefer the reset header that names the SAME limit as the exhausted one — pairing an
|
|
1047
|
+
// exhaustion signal with an unrelated intermediary's reset is how a full-window cooldown
|
|
1048
|
+
// becomes a 1s one. Two headers name the same limit when they agree on every word except
|
|
1049
|
+
// `remaining`/`reset`, which is word-order-independent and so survives all four shapes above.
|
|
1050
|
+
const resetKeys = Object.keys(h).filter((k) => isLimitHeader(k) && hasWord(k, 'reset'));
|
|
1051
|
+
const prefixOf = (k: string) => k.split('-').filter((w) => w !== 'remaining' && w !== 'reset').join('-');
|
|
1052
|
+
// The LONGEST paired back-off wins, not the first one in header order (§9 finding, 2026-07-26):
|
|
1053
|
+
// a vendor that limits several dimensions at once (requests AND tokens) can report two exhausted
|
|
1054
|
+
// buckets with different resets, and taking whichever happened to be inserted first would obey
|
|
1055
|
+
// the 5-second one and ignore the 3,600-second one. Every candidate is converted first, because
|
|
1056
|
+
// the shortest STRING is not necessarily the shortest DURATION once units are resolved.
|
|
1057
|
+
const candidates = resetKeys
|
|
1058
|
+
.filter((k) => exhausted.some((e) => prefixOf(k) === prefixOf(e)))
|
|
1059
|
+
.map((k) => resetToSeconds(numeric(h[k]), nowMs))
|
|
1060
|
+
.filter((s): s is number => s !== null);
|
|
1061
|
+
if (candidates.length > 0) return Math.max(...candidates);
|
|
1062
|
+
// No trustworthy reset hint. Which back-off is right depends on WHETHER THE VENDOR ACTUALLY
|
|
1063
|
+
// REFUSED (§9 round 2, 2026-07-26 — the two cases had been collapsed, and the cap was a narrow
|
|
1064
|
+
// fail-open for hour-window packs):
|
|
1065
|
+
// • the response was an ERROR (4xx/5xx) ⇒ the vendor turned this call away. Back off for a
|
|
1066
|
+
// whole window. Re-probing an exhausted credential every minute is precisely what GitHub's
|
|
1067
|
+
// secondary-limit guidance warns against, and "the ledger's ceiling bounds it underneath"
|
|
1068
|
+
// only holds once the ceiling is nearly spent — exhaustion can arrive with a nearly empty
|
|
1069
|
+
// ledger when the credential is shared (a team PAT, a CI fleet).
|
|
1070
|
+
// • the response SUCCEEDED and merely reported `remaining: 0` ⇒ we got what we asked for and
|
|
1071
|
+
// the next call MIGHT be refused. Cap at the conventional bare-429 back-off, so a header
|
|
1072
|
+
// that is rounded, blank-ish or emitted by an intermediary cannot cost a healthy credential
|
|
1073
|
+
// a full hour. This is the leg that keeps the false-positive protection.
|
|
1074
|
+
return status !== undefined && status >= 400
|
|
1075
|
+
? windowMs / 1000
|
|
1076
|
+
: Math.min(windowMs / 1000, DEFAULT_BARE_429_BACKOFF_S);
|
|
1077
|
+
}
|
|
1078
|
+
// A 429 with no guidance: the conventional default back-off.
|
|
1079
|
+
if (status === 429) return DEFAULT_BARE_429_BACKOFF_S;
|
|
1080
|
+
return null;
|
|
1081
|
+
}
|
|
1082
|
+
|
|
1083
|
+
/**
|
|
1084
|
+
* A `*-ratelimit-reset` value → seconds to wait, or `null` when it cannot be trusted.
|
|
1085
|
+
*
|
|
1086
|
+
* Vendors disagree about the UNIT, and getting it wrong is not a rounding error — it is a multi-day
|
|
1087
|
+
* self-inflicted outage. Three shapes are seen in practice, distinguished by MAGNITUDE because none
|
|
1088
|
+
* of them is labelled:
|
|
1089
|
+
* • a small number → a delta, already in seconds (`x-ratelimit-reset: 45`)
|
|
1090
|
+
* • ~1e9-1e10 → an absolute epoch in SECONDS (GitHub)
|
|
1091
|
+
* • ~1e12-1e13 → an absolute epoch in MILLISECONDS (Linear documents exactly this)
|
|
1092
|
+
*
|
|
1093
|
+
* Reading a millisecond epoch as a second epoch yields a back-off of roughly fifty thousand YEARS,
|
|
1094
|
+
* which the cooldown cap then clamps to a week — so ONE exhausted response would lock the credential
|
|
1095
|
+
* out client-side for seven days and need a human to delete the ledger file. That is the module
|
|
1096
|
+
* inflicting the very outage it exists to prevent. (§9 finding, 2026-07-26.)
|
|
1097
|
+
*
|
|
1098
|
+
* Anything that still resolves to more than a day is treated as UNPARSEABLE rather than obeyed:
|
|
1099
|
+
* no real rate limit resets a day out, so such a value means the unit guess was wrong, and the
|
|
1100
|
+
* honest answer is `null` (the caller falls back to a one-window cooldown) rather than a number
|
|
1101
|
+
* nobody can justify. Non-finite and negative values are likewise `null`.
|
|
1102
|
+
*/
|
|
1103
|
+
export function resetToSeconds(reset: number, nowMs: number = Date.now()): number | null {
|
|
1104
|
+
if (!Number.isFinite(reset) || reset <= 0) return null;
|
|
1105
|
+
const nowS = Math.floor(nowMs / 1000);
|
|
1106
|
+
// Thresholds sit between the plausible ranges, not at their edges: 1e11 s is year 5138 (so no
|
|
1107
|
+
// real epoch-seconds value reaches it) and 1e11 ms is 1973 (so every real epoch-ms value passes it).
|
|
1108
|
+
const seconds = reset > 1e11
|
|
1109
|
+
? Math.max(0, Math.floor(reset / 1000) - nowS) // absolute epoch, milliseconds
|
|
1110
|
+
: reset > 1e6
|
|
1111
|
+
? Math.max(0, reset - nowS) // absolute epoch, seconds
|
|
1112
|
+
: reset; // a delta, already seconds
|
|
1113
|
+
if (!Number.isFinite(seconds) || seconds <= 0) return null;
|
|
1114
|
+
return seconds > 24 * 60 * 60 ? null : seconds;
|
|
1115
|
+
}
|