@naced/llm-multi-rate-limiter 0.0.0-stage → 0.1.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +78 -0
- package/LICENSE +22 -0
- package/README.md +112 -2
- package/README.zh.md +115 -0
- package/cordis.patch.yml +3 -0
- package/lib/client.js +956 -0
- package/lib/config-resolve.js +259 -0
- package/lib/index.d.ts +91 -0
- package/lib/index.js +552 -0
- package/lib/status-rpc.js +285 -0
- package/lib/strategies/sliding-window.js +174 -0
- package/lib/strategies/token-bucket.js +196 -0
- package/lib/types/config.js +126 -0
- package/lib/types/index.js +1 -0
- package/package.json +81 -4
|
@@ -0,0 +1,285 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* dsh-llm-multi-rate-limiter — status RPC channel (Host half).
|
|
3
|
+
*
|
|
4
|
+
* Exposes live rate-limiter statistics to the browser over the framework's
|
|
5
|
+
* generic Connection RPC channel. The framework supplies POST+JSON transport,
|
|
6
|
+
* the Host/Origin fence (403) and browser authentication (401); this module
|
|
7
|
+
* only answers endpoints and shapes envelopes.
|
|
8
|
+
*
|
|
9
|
+
* Two mounting paths exist, and both speak the *same* wire protocol so the
|
|
10
|
+
* browser half never needs to know which one is active:
|
|
11
|
+
*
|
|
12
|
+
* 1. `connection.rpc.handle(channel, handler)` — the framework's own generic
|
|
13
|
+
* channel registry. Preferred: the framework owns the route, the request
|
|
14
|
+
* validation, and the fiber-scoped withdrawal.
|
|
15
|
+
*
|
|
16
|
+
* 2. `createChannelRoute({ channel, handler, reject })` — a self-registered
|
|
17
|
+
* prefix route for hosts where path 1 is broken. DSH 0.1.5-rc.3 changed
|
|
18
|
+
* `@deepseek-ai/dsh-client-connection`'s own `inject` from
|
|
19
|
+
* `["webServer", "credentials"]` to `["credentials"]` while
|
|
20
|
+
* `HostConnectionService.register()` still dereferences
|
|
21
|
+
* `owner.webServer`, so `rpc.handle` throws
|
|
22
|
+
* `cannot get property "webServer" without inject`. This route is
|
|
23
|
+
* registered on our own fiber (which *can* see `webServer`) and reuses the
|
|
24
|
+
* connection service's public `requestRejection()` fence, so the 403/401
|
|
25
|
+
* policy is still the framework's.
|
|
26
|
+
*
|
|
27
|
+
* Envelope contract (mirrors the Connection wire schema):
|
|
28
|
+
* success: { ok: true, value: <JSON-safe> }
|
|
29
|
+
* failure: { ok: false, error: { code, message, details } }
|
|
30
|
+
*
|
|
31
|
+
* @module dsh-llm-multi-rate-limiter/status-rpc
|
|
32
|
+
*/
|
|
33
|
+
|
|
34
|
+
/**
|
|
35
|
+
* Channel name. Must satisfy the framework's `assertChannel` pattern
|
|
36
|
+
* `/^\/[A-Za-z0-9._~-]+$/` and must not be the reserved `/api` channel.
|
|
37
|
+
*/
|
|
38
|
+
const CHANNEL = "/llm-multi-rate-limiter";
|
|
39
|
+
|
|
40
|
+
/**
|
|
41
|
+
* Endpoint grammar for one path segment, copied verbatim from the framework's
|
|
42
|
+
* `ENDPOINT_SEGMENT_PATTERN` in dsh-client-connection. Keeping it identical is
|
|
43
|
+
* what makes the self-registered route accept and reject exactly the same URLs
|
|
44
|
+
* as `rpc.handle` would.
|
|
45
|
+
*/
|
|
46
|
+
const ENDPOINT_SEGMENT_PATTERN = /^[A-Za-z0-9_$.-]+$/;
|
|
47
|
+
|
|
48
|
+
/**
|
|
49
|
+
* Buffered-body cap for the self-registered route. Both endpoints are called
|
|
50
|
+
* with an empty payload, so this is a defensive memory bound rather than a
|
|
51
|
+
* real limit (the framework uses its own, much larger, carrier cap).
|
|
52
|
+
*/
|
|
53
|
+
const MAX_REQUEST_BODY_BYTES = 1024 * 1024;
|
|
54
|
+
|
|
55
|
+
/**
|
|
56
|
+
* Correlation id echoed when the request envelope cannot be parsed at all —
|
|
57
|
+
* the same literal the framework uses (`INVALID_REQUEST_RPC_ID`).
|
|
58
|
+
*/
|
|
59
|
+
const INVALID_REQUEST_RPC_ID = "invalid-request";
|
|
60
|
+
|
|
61
|
+
/**
|
|
62
|
+
* Build a failure envelope. Field-for-field the same shape the Connection
|
|
63
|
+
* host accepts, so callers can rely on `ok` discrimination alone.
|
|
64
|
+
* @param {string} code - stable machine-readable error code.
|
|
65
|
+
* @param {string} message - human-readable detail.
|
|
66
|
+
* @param {Record<string, unknown>} [details] - optional structured context.
|
|
67
|
+
* @returns {{ ok: false, error: { code: string, message: string, details: Record<string, unknown> } }}
|
|
68
|
+
*/
|
|
69
|
+
function failure(code, message, details = {}) {
|
|
70
|
+
return { ok: false, error: { code, message, details } };
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
/** Build a success envelope. */
|
|
74
|
+
function success(value) {
|
|
75
|
+
return { ok: true, value };
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
/**
|
|
79
|
+
* Create the channel handler.
|
|
80
|
+
*
|
|
81
|
+
* @param {object} deps
|
|
82
|
+
* @param {() => object} deps.buildSnapshot - live snapshot builder.
|
|
83
|
+
* @param {() => void} deps.resetTotals - zero the counters (and bump `rev`).
|
|
84
|
+
* @returns {(endpoint: string, payload: unknown, signal?: AbortSignal) => Promise<object>}
|
|
85
|
+
*/
|
|
86
|
+
function createStatusChannel({ buildSnapshot, resetTotals }) {
|
|
87
|
+
return async function statusChannel(endpoint) {
|
|
88
|
+
if (endpoint === "snapshot") return success(buildSnapshot());
|
|
89
|
+
if (endpoint === "reset") {
|
|
90
|
+
resetTotals();
|
|
91
|
+
return success({});
|
|
92
|
+
}
|
|
93
|
+
return failure(
|
|
94
|
+
"llm-multi-rate-limiter/unknown-endpoint",
|
|
95
|
+
`unknown endpoint: ${String(endpoint)}`,
|
|
96
|
+
);
|
|
97
|
+
};
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
/* ── self-registered route (the DSH 0.1.5-rc.3 fallback) ───────────────── */
|
|
101
|
+
|
|
102
|
+
/** Narrow an unknown JSON value to a plain object. */
|
|
103
|
+
function isRecord(value) {
|
|
104
|
+
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
/**
|
|
108
|
+
* Extract the endpoint from a channel-relative pathname, mirroring the
|
|
109
|
+
* framework's `endpointFromPath`: a single `channel/<endpoint>` segment whose
|
|
110
|
+
* segments all satisfy {@link ENDPOINT_SEGMENT_PATTERN}.
|
|
111
|
+
*
|
|
112
|
+
* @param {string} channel - the channel prefix (e.g. `/llm-multi-rate-limiter`).
|
|
113
|
+
* @param {string} pathname - request pathname.
|
|
114
|
+
* @returns {string | undefined} the endpoint, or undefined when the URL is not
|
|
115
|
+
* a well-formed member of this channel.
|
|
116
|
+
*/
|
|
117
|
+
function endpointFromPath(channel, pathname) {
|
|
118
|
+
if (!pathname.startsWith(`${channel}/`)) return undefined;
|
|
119
|
+
const endpoint = pathname.slice(channel.length + 1);
|
|
120
|
+
const segments = endpoint.split("/");
|
|
121
|
+
if (segments.some((segment) => segment === "" || segment === "." || segment === ".." || !ENDPOINT_SEGMENT_PATTERN.test(segment))) return undefined;
|
|
122
|
+
return endpoint;
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
/**
|
|
126
|
+
* Read and decode the request body, refusing anything above `limit` bytes so a
|
|
127
|
+
* hostile caller cannot make the host buffer without bound.
|
|
128
|
+
*
|
|
129
|
+
* @param {import("node:http").IncomingMessage} req - node request stream.
|
|
130
|
+
* @param {number} limit - maximum accepted byte count.
|
|
131
|
+
* @returns {Promise<string>} the decoded UTF-8 body.
|
|
132
|
+
* @throws {Error & { code: "TOO_LARGE" }} when the body exceeds `limit`.
|
|
133
|
+
*/
|
|
134
|
+
async function readBody(req, limit) {
|
|
135
|
+
const chunks = [];
|
|
136
|
+
let received = 0;
|
|
137
|
+
for await (const chunk of req) {
|
|
138
|
+
const buffer = typeof chunk === "string" ? Buffer.from(chunk) : chunk;
|
|
139
|
+
received += buffer.byteLength;
|
|
140
|
+
if (received > limit) {
|
|
141
|
+
const error = new Error("request body too large");
|
|
142
|
+
error.code = "TOO_LARGE";
|
|
143
|
+
throw error;
|
|
144
|
+
}
|
|
145
|
+
chunks.push(buffer);
|
|
146
|
+
}
|
|
147
|
+
return Buffer.concat(chunks).toString("utf8");
|
|
148
|
+
}
|
|
149
|
+
|
|
150
|
+
/**
|
|
151
|
+
* Write one `server-response` envelope as JSON — byte-for-byte the shape
|
|
152
|
+
* `Response.json(...)` produces on the framework's own route.
|
|
153
|
+
*
|
|
154
|
+
* @param {import("node:http").ServerResponse} res - response to own.
|
|
155
|
+
* @param {string} rpcId - correlation id to echo.
|
|
156
|
+
* @param {object} result - the `{ ok, value }` / `{ ok, error }` envelope.
|
|
157
|
+
*/
|
|
158
|
+
function sendEnvelope(res, rpcId, result) {
|
|
159
|
+
res.writeHead(200, { "content-type": "application/json" });
|
|
160
|
+
res.end(JSON.stringify({ type: "server-response", rpcId, result }));
|
|
161
|
+
}
|
|
162
|
+
|
|
163
|
+
/**
|
|
164
|
+
* Build a webserver prefix route that speaks the Connection RPC wire protocol.
|
|
165
|
+
*
|
|
166
|
+
* This is the fallback carrier for {@link createStatusChannel} on hosts where
|
|
167
|
+
* `connection.rpc.handle` throws. Every request-shape decision mirrors the
|
|
168
|
+
* framework's `rpcFetchHandler` so the browser half is byte-compatible with the
|
|
169
|
+
* preferred path:
|
|
170
|
+
*
|
|
171
|
+
* | condition | response |
|
|
172
|
+
* |---|---|
|
|
173
|
+
* | fence/auth rejected by `reject` | `403` (or `401`) + `forbidden`/`unauthorized` |
|
|
174
|
+
* | non-POST, or URL outside the channel | `404 not found` |
|
|
175
|
+
* | `content-type` not `application/json` | `415` |
|
|
176
|
+
* | body over {@link MAX_REQUEST_BODY_BYTES} | `413` |
|
|
177
|
+
* | body not JSON | `400 body is not JSON` |
|
|
178
|
+
* | envelope not a `client-request` | `gateway/bad-request` envelope |
|
|
179
|
+
* | `method` ≠ endpoint | `gateway/bad-request` envelope |
|
|
180
|
+
* | handler threw | `500 handler failure: ...` |
|
|
181
|
+
* | otherwise | `200` + `server-response` envelope |
|
|
182
|
+
*
|
|
183
|
+
* @param {object} deps
|
|
184
|
+
* @param {string} deps.channel - channel prefix to own (e.g. `/llm-multi-rate-limiter`).
|
|
185
|
+
* @param {(endpoint: string, payload: unknown, signal?: AbortSignal) => Promise<object>} deps.handler - endpoint dispatcher.
|
|
186
|
+
* @param {(req: import("node:http").IncomingMessage) => number | undefined} deps.reject - the framework's `connection.requestRejection` fence; returns an HTTP status to refuse with, or undefined to allow.
|
|
187
|
+
* @returns {{ kind: "prefix", path: string, handler: (req: import("node:http").IncomingMessage, res: import("node:http").ServerResponse) => Promise<void> }}
|
|
188
|
+
*/
|
|
189
|
+
function createChannelRoute({ channel, handler, reject }) {
|
|
190
|
+
return {
|
|
191
|
+
kind: "prefix",
|
|
192
|
+
path: channel,
|
|
193
|
+
handler: async (req, res) => {
|
|
194
|
+
// 1. The framework's own fence: Host/Origin trust (403) then browser
|
|
195
|
+
// authentication (401). Reusing it keeps the security policy identical.
|
|
196
|
+
const rejection = reject(req);
|
|
197
|
+
if (rejection !== undefined) {
|
|
198
|
+
res.writeHead(rejection);
|
|
199
|
+
res.end(rejection === 401 ? "unauthorized" : "forbidden");
|
|
200
|
+
return;
|
|
201
|
+
}
|
|
202
|
+
|
|
203
|
+
// 2. POST-only, and only inside this channel (framework: 404 otherwise).
|
|
204
|
+
const url = new URL(req.url ?? "/", "http://dsh.internal");
|
|
205
|
+
const endpoint = endpointFromPath(channel, url.pathname);
|
|
206
|
+
if (req.method !== "POST" || endpoint === undefined) {
|
|
207
|
+
res.writeHead(404);
|
|
208
|
+
res.end("not found");
|
|
209
|
+
return;
|
|
210
|
+
}
|
|
211
|
+
|
|
212
|
+
// 3. JSON only (framework: 415).
|
|
213
|
+
const mediaType = String(req.headers["content-type"] ?? "").split(";", 1)[0].trim().toLowerCase();
|
|
214
|
+
if (mediaType !== "application/json") {
|
|
215
|
+
res.writeHead(415);
|
|
216
|
+
res.end("content type must be application/json");
|
|
217
|
+
return;
|
|
218
|
+
}
|
|
219
|
+
|
|
220
|
+
// 4. Buffer the body under a defensive cap, then parse it.
|
|
221
|
+
let raw;
|
|
222
|
+
try {
|
|
223
|
+
raw = await readBody(req, MAX_REQUEST_BODY_BYTES);
|
|
224
|
+
} catch (error) {
|
|
225
|
+
if (error?.code === "TOO_LARGE") {
|
|
226
|
+
res.writeHead(413, { connection: "close" });
|
|
227
|
+
res.end();
|
|
228
|
+
req.destroy?.();
|
|
229
|
+
return;
|
|
230
|
+
}
|
|
231
|
+
throw error;
|
|
232
|
+
}
|
|
233
|
+
let body;
|
|
234
|
+
try {
|
|
235
|
+
body = JSON.parse(raw);
|
|
236
|
+
} catch {
|
|
237
|
+
res.writeHead(400);
|
|
238
|
+
res.end("body is not JSON");
|
|
239
|
+
return;
|
|
240
|
+
}
|
|
241
|
+
|
|
242
|
+
// 5. Validate the client-request envelope; echo the correlation id
|
|
243
|
+
// whenever it is a string, exactly like the framework does.
|
|
244
|
+
const rpcId = typeof body?.rpcId === "string" ? body.rpcId : INVALID_REQUEST_RPC_ID;
|
|
245
|
+
if (!isRecord(body) || body.type !== "client-request" || typeof body.rpcId !== "string" || typeof body.method !== "string") {
|
|
246
|
+
sendEnvelope(res, rpcId, failure("gateway/bad-request", "invalid client-request message", { issues: [] }));
|
|
247
|
+
return;
|
|
248
|
+
}
|
|
249
|
+
if (body.method !== endpoint) {
|
|
250
|
+
sendEnvelope(res, rpcId, failure(
|
|
251
|
+
"gateway/bad-request",
|
|
252
|
+
`method ${JSON.stringify(body.method)} does not match endpoint ${JSON.stringify(endpoint)}`,
|
|
253
|
+
{ issues: [] },
|
|
254
|
+
));
|
|
255
|
+
return;
|
|
256
|
+
}
|
|
257
|
+
|
|
258
|
+
// 6. Dispatch. Tie the signal to the response lifetime, as the framework's
|
|
259
|
+
// http bridge does, so a dropped client can cancel the work.
|
|
260
|
+
const abort = new AbortController();
|
|
261
|
+
res.on?.("close", () => {
|
|
262
|
+
if (!res.writableEnded) abort.abort();
|
|
263
|
+
});
|
|
264
|
+
let result;
|
|
265
|
+
try {
|
|
266
|
+
result = await handler(endpoint, body.payload, abort.signal);
|
|
267
|
+
} catch (error) {
|
|
268
|
+
res.writeHead(500);
|
|
269
|
+
res.end(`handler failure: ${String(error)}`);
|
|
270
|
+
return;
|
|
271
|
+
}
|
|
272
|
+
sendEnvelope(res, rpcId, result);
|
|
273
|
+
},
|
|
274
|
+
};
|
|
275
|
+
}
|
|
276
|
+
|
|
277
|
+
export {
|
|
278
|
+
CHANNEL,
|
|
279
|
+
MAX_REQUEST_BODY_BYTES,
|
|
280
|
+
createChannelRoute,
|
|
281
|
+
createStatusChannel,
|
|
282
|
+
endpointFromPath,
|
|
283
|
+
failure,
|
|
284
|
+
success,
|
|
285
|
+
};
|
|
@@ -0,0 +1,174 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Sliding-Window rate limiter strategy.
|
|
3
|
+
*
|
|
4
|
+
* Counts requests inside a rolling window of `windowMs` milliseconds.
|
|
5
|
+
* The window slides continuously: each request's timestamp is recorded,
|
|
6
|
+
* and the count is the number of timestamps inside the window.
|
|
7
|
+
*
|
|
8
|
+
* Concurrency is controlled separately (same as TokenBucketStrategy).
|
|
9
|
+
*
|
|
10
|
+
* @module dsh-llm-multi-rate-limiter/strategies/sliding-window
|
|
11
|
+
*/
|
|
12
|
+
|
|
13
|
+
export class SlidingWindowStrategy {
|
|
14
|
+
/** @type {{ windowMs: number, maxRpm: number, maxConcurrent: number }} */
|
|
15
|
+
#cfg;
|
|
16
|
+
#concurrent = 0;
|
|
17
|
+
/**
|
|
18
|
+
* Slots handed to a waiter that has not called `acquireSlot()` yet.
|
|
19
|
+
*
|
|
20
|
+
* `#drain()` wakes several waiters in one pass and each of them takes its
|
|
21
|
+
* concurrent slot only after the promise settles; counting reservations keeps
|
|
22
|
+
* one release from unlocking the entire queue past `maxConcurrent`.
|
|
23
|
+
*/
|
|
24
|
+
#reserved = 0;
|
|
25
|
+
/** @type {number[]} — timestamps (Date.now()) inside the window */
|
|
26
|
+
#timestamps = [];
|
|
27
|
+
/** @type {Array<{ resolve: (r: { granted: boolean }) => void, abort: () => void }>} */
|
|
28
|
+
#waiters = [];
|
|
29
|
+
#timer;
|
|
30
|
+
|
|
31
|
+
/**
|
|
32
|
+
* @param {object} config — `{ windowMs, maxRpm, maxConcurrent }`
|
|
33
|
+
*/
|
|
34
|
+
constructor(config) {
|
|
35
|
+
this.#cfg = { ...config };
|
|
36
|
+
// Prune stale timestamps every second.
|
|
37
|
+
this.#timer = setInterval(() => this.#prune(), 1_000);
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
/* ── public API ─────────────────────────────────────── */
|
|
41
|
+
|
|
42
|
+
acquire({ signal, timeoutMs }) {
|
|
43
|
+
const t0 = Date.now();
|
|
44
|
+
|
|
45
|
+
if (signal?.aborted) return Promise.resolve({ granted: false });
|
|
46
|
+
|
|
47
|
+
// Fast path: concurrency AND window capacity both available.
|
|
48
|
+
if (this.#concurrent < this.#cfg.maxConcurrent && this.#count() < this.#cfg.maxRpm) {
|
|
49
|
+
this.#timestamps.push(Date.now());
|
|
50
|
+
return Promise.resolve({ granted: true });
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
if (this.#cfg.maxConcurrent <= 0) return Promise.resolve({ granted: false });
|
|
54
|
+
|
|
55
|
+
return new Promise((resolve) => {
|
|
56
|
+
// A waiter can be granted by `#drain()` in the same tick its timeout
|
|
57
|
+
// fires; settling once keeps the window/slot accounting exact.
|
|
58
|
+
let settled = false;
|
|
59
|
+
const onDone = (result) => {
|
|
60
|
+
if (settled) return;
|
|
61
|
+
settled = true;
|
|
62
|
+
clearTimeout(timer);
|
|
63
|
+
signal?.removeEventListener("abort", onAbort);
|
|
64
|
+
const idx = this.#waiters.indexOf(entry);
|
|
65
|
+
if (idx >= 0) this.#waiters.splice(idx, 1);
|
|
66
|
+
resolve({ ...result, waitMs: Date.now() - t0 });
|
|
67
|
+
};
|
|
68
|
+
|
|
69
|
+
const onAbort = () => onDone({ granted: false });
|
|
70
|
+
const timer = setTimeout(() => onDone({ granted: false }), timeoutMs);
|
|
71
|
+
|
|
72
|
+
const entry = {
|
|
73
|
+
resolve: () => onDone({ granted: true }),
|
|
74
|
+
abort: onAbort,
|
|
75
|
+
};
|
|
76
|
+
|
|
77
|
+
this.#waiters.push(entry);
|
|
78
|
+
signal?.addEventListener("abort", onAbort, { once: true });
|
|
79
|
+
});
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
/**
|
|
83
|
+
* Non-blocking acquisition: record the request in the window and take an
|
|
84
|
+
* implicit concurrent slot if both are available right now, without parking
|
|
85
|
+
* in the queue.
|
|
86
|
+
*
|
|
87
|
+
* Used by "reject" throttle mode, which must not wait, but must still count
|
|
88
|
+
* the request so the rate limit is enforced.
|
|
89
|
+
*
|
|
90
|
+
* @returns {{ granted: boolean }} — granted means the request was counted
|
|
91
|
+
* and a concurrent slot taken; the caller MUST pair it with `releaseSlot()`.
|
|
92
|
+
*/
|
|
93
|
+
acquireNonBlocking() {
|
|
94
|
+
if (this.#concurrent < this.#cfg.maxConcurrent && this.#count() < this.#cfg.maxRpm) {
|
|
95
|
+
this.#timestamps.push(Date.now());
|
|
96
|
+
this.#concurrent += 1;
|
|
97
|
+
return { granted: true };
|
|
98
|
+
}
|
|
99
|
+
return { granted: false };
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
/**
|
|
103
|
+
* Mark one concurrent slot as in-use (converting a `#drain()` reservation
|
|
104
|
+
* when there is one, so the slot is never counted twice).
|
|
105
|
+
*/
|
|
106
|
+
acquireSlot() {
|
|
107
|
+
if (this.#reserved > 0) this.#reserved -= 1;
|
|
108
|
+
this.#concurrent += 1;
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
releaseSlot() {
|
|
112
|
+
this.#concurrent = Math.max(0, this.#concurrent - 1);
|
|
113
|
+
this.#drain();
|
|
114
|
+
}
|
|
115
|
+
|
|
116
|
+
getStatus() {
|
|
117
|
+
return {
|
|
118
|
+
strategy: "sliding-window",
|
|
119
|
+
windowMs: this.#cfg.windowMs,
|
|
120
|
+
countInWindow: this.#count(),
|
|
121
|
+
maxRpm: this.#cfg.maxRpm,
|
|
122
|
+
concurrent: this.#concurrent,
|
|
123
|
+
reserved: this.#reserved,
|
|
124
|
+
maxConcurrent: this.#cfg.maxConcurrent,
|
|
125
|
+
queued: this.#waiters.length,
|
|
126
|
+
};
|
|
127
|
+
}
|
|
128
|
+
|
|
129
|
+
update(config) {
|
|
130
|
+
Object.assign(this.#cfg, config);
|
|
131
|
+
}
|
|
132
|
+
|
|
133
|
+
dispose() {
|
|
134
|
+
clearInterval(this.#timer);
|
|
135
|
+
for (const w of this.#waiters) w.abort();
|
|
136
|
+
this.#waiters = [];
|
|
137
|
+
}
|
|
138
|
+
|
|
139
|
+
/* ── internals ─────────────────────────────────────── */
|
|
140
|
+
|
|
141
|
+
/** Number of timestamps inside the current window. */
|
|
142
|
+
#count() {
|
|
143
|
+
const cutoff = Date.now() - this.#cfg.windowMs;
|
|
144
|
+
// timestamps are pushed in order, so find the first that's still inside.
|
|
145
|
+
let i = 0;
|
|
146
|
+
while (i < this.#timestamps.length && this.#timestamps[i] <= cutoff) i += 1;
|
|
147
|
+
return this.#timestamps.length - i;
|
|
148
|
+
}
|
|
149
|
+
|
|
150
|
+
/** Remove timestamps that fell out of the window. */
|
|
151
|
+
#prune() {
|
|
152
|
+
const cutoff = Date.now() - this.#cfg.windowMs;
|
|
153
|
+
while (this.#timestamps.length > 0 && this.#timestamps[0] <= cutoff) {
|
|
154
|
+
this.#timestamps.shift();
|
|
155
|
+
}
|
|
156
|
+
this.#drain();
|
|
157
|
+
}
|
|
158
|
+
|
|
159
|
+
/** Drain waiters as long as capacity is available. */
|
|
160
|
+
#drain() {
|
|
161
|
+
while (
|
|
162
|
+
this.#waiters.length > 0
|
|
163
|
+
&& this.#count() < this.#cfg.maxRpm
|
|
164
|
+
&& this.#concurrent + this.#reserved < this.#cfg.maxConcurrent
|
|
165
|
+
) {
|
|
166
|
+
this.#timestamps.push(Date.now());
|
|
167
|
+
// Reserve the concurrent slot for the waiter we are about to wake; it is
|
|
168
|
+
// converted into a live slot by `acquireSlot()` a microtask later.
|
|
169
|
+
this.#reserved += 1;
|
|
170
|
+
const waiter = this.#waiters.shift();
|
|
171
|
+
waiter?.resolve();
|
|
172
|
+
}
|
|
173
|
+
}
|
|
174
|
+
}
|
|
@@ -0,0 +1,196 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Token-Bucket rate limiter strategy.
|
|
3
|
+
*
|
|
4
|
+
* Each model gets its own bucket. A bucket has:
|
|
5
|
+
* - `burstSize` — maximum tokens the bucket can hold
|
|
6
|
+
* - `refillRate` — tokens added per second
|
|
7
|
+
* - a concurrent-slot counter (separate from tokens, so burst + concurrency are orthogonal)
|
|
8
|
+
*
|
|
9
|
+
* When the bucket is empty the request is queued (a promise resolver parked in
|
|
10
|
+
* `waiters[]`). As tokens refill, waiters are drained in FIFO order.
|
|
11
|
+
*
|
|
12
|
+
* @module dsh-llm-multi-rate-limiter/strategies/token-bucket
|
|
13
|
+
*/
|
|
14
|
+
|
|
15
|
+
export class TokenBucketStrategy {
|
|
16
|
+
/** @type {{ burstSize: number, refillRate: number, maxConcurrent: number }} */
|
|
17
|
+
#cfg;
|
|
18
|
+
#tokens;
|
|
19
|
+
#lastRefill;
|
|
20
|
+
#concurrent = 0;
|
|
21
|
+
/**
|
|
22
|
+
* Slots handed to a waiter that has not called `acquireSlot()` yet.
|
|
23
|
+
*
|
|
24
|
+
* `#drain()` resolves several waiters in one pass, and each of them takes its
|
|
25
|
+
* concurrent slot only after the promise settles. Counting reservations keeps
|
|
26
|
+
* the drain loop from granting more waiters than `maxConcurrent` allows —
|
|
27
|
+
* without this, one `releaseSlot()` could release the whole queue at once.
|
|
28
|
+
*/
|
|
29
|
+
#reserved = 0;
|
|
30
|
+
/** @type {Array<{ resolve: (r: { granted: boolean }) => void, abort: () => void }>} */
|
|
31
|
+
#waiters = [];
|
|
32
|
+
#timer;
|
|
33
|
+
|
|
34
|
+
/**
|
|
35
|
+
* @param {object} config — `{ burstSize, refillRate, maxConcurrent }`
|
|
36
|
+
*/
|
|
37
|
+
constructor(config) {
|
|
38
|
+
this.#cfg = { ...config };
|
|
39
|
+
this.#tokens = config.burstSize;
|
|
40
|
+
this.#lastRefill = Date.now();
|
|
41
|
+
this.#startRefill();
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
/* ── public API ─────────────────────────────────────── */
|
|
45
|
+
|
|
46
|
+
/**
|
|
47
|
+
* Wait until a request is allowed.
|
|
48
|
+
* Resolves `{ granted: true }` immediately or after the queue drains.
|
|
49
|
+
* Resolves `{ granted: false }` on abort or timeout (timeout is the caller's
|
|
50
|
+
* responsibility — it races this promise against a timer).
|
|
51
|
+
*
|
|
52
|
+
* @param {{ signal?: AbortSignal, timeoutMs: number }} opts
|
|
53
|
+
* @returns {Promise<{ granted: boolean, waitMs?: number }>}
|
|
54
|
+
*/
|
|
55
|
+
acquire({ signal, timeoutMs }) {
|
|
56
|
+
const t0 = Date.now();
|
|
57
|
+
|
|
58
|
+
// Already aborted — fail fast.
|
|
59
|
+
if (signal?.aborted) {
|
|
60
|
+
return Promise.resolve({ granted: false });
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
// Fast path: concurrency room AND a token available.
|
|
64
|
+
if (this.#concurrent < this.#cfg.maxConcurrent && this.#tokens >= 1) {
|
|
65
|
+
this.#tokens -= 1;
|
|
66
|
+
return Promise.resolve({ granted: true });
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
// Slow path: park in the waiter queue.
|
|
70
|
+
if (this.#cfg.maxConcurrent <= 0) {
|
|
71
|
+
return Promise.resolve({ granted: false });
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
return new Promise((resolve) => {
|
|
75
|
+
// A waiter can be granted by `#drain()` in the same tick its timeout
|
|
76
|
+
// fires; settling once keeps the token/slot accounting exact.
|
|
77
|
+
let settled = false;
|
|
78
|
+
const onDone = (result) => {
|
|
79
|
+
if (settled) return;
|
|
80
|
+
settled = true;
|
|
81
|
+
clearTimeout(timer);
|
|
82
|
+
signal?.removeEventListener("abort", onAbort);
|
|
83
|
+
const idx = this.#waiters.indexOf(entry);
|
|
84
|
+
if (idx >= 0) this.#waiters.splice(idx, 1);
|
|
85
|
+
resolve({ ...result, waitMs: Date.now() - t0 });
|
|
86
|
+
};
|
|
87
|
+
|
|
88
|
+
const onAbort = () => onDone({ granted: false });
|
|
89
|
+
|
|
90
|
+
const timer = setTimeout(() => onDone({ granted: false }), timeoutMs);
|
|
91
|
+
|
|
92
|
+
const entry = {
|
|
93
|
+
resolve: () => {
|
|
94
|
+
// Token already deducted by _drain.
|
|
95
|
+
onDone({ granted: true });
|
|
96
|
+
},
|
|
97
|
+
abort: onAbort,
|
|
98
|
+
};
|
|
99
|
+
|
|
100
|
+
this.#waiters.push(entry);
|
|
101
|
+
signal?.addEventListener("abort", onAbort, { once: true });
|
|
102
|
+
});
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
/**
|
|
106
|
+
* Non-blocking acquisition: consume a token and an implicit concurrent slot
|
|
107
|
+
* if both are available right now, without parking in the queue.
|
|
108
|
+
*
|
|
109
|
+
* Used by "reject" throttle mode, which must not wait, but must still deduct
|
|
110
|
+
* tokens so the rate limit is enforced (otherwise only the concurrency cap
|
|
111
|
+
* would bite).
|
|
112
|
+
*
|
|
113
|
+
* @returns {{ granted: boolean }} — granted means a token was consumed and a
|
|
114
|
+
* concurrent slot taken; the caller MUST pair it with `releaseSlot()`.
|
|
115
|
+
*/
|
|
116
|
+
acquireNonBlocking() {
|
|
117
|
+
if (this.#concurrent < this.#cfg.maxConcurrent && this.#tokens >= 1) {
|
|
118
|
+
this.#tokens -= 1;
|
|
119
|
+
this.#concurrent += 1;
|
|
120
|
+
return { granted: true };
|
|
121
|
+
}
|
|
122
|
+
return { granted: false };
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
/**
|
|
126
|
+
* Mark one concurrent slot as in-use.
|
|
127
|
+
*
|
|
128
|
+
* When the slot was already reserved by `#drain()`, this converts the
|
|
129
|
+
* reservation into a live slot instead of taking a second one.
|
|
130
|
+
*/
|
|
131
|
+
acquireSlot() {
|
|
132
|
+
if (this.#reserved > 0) this.#reserved -= 1;
|
|
133
|
+
this.#concurrent += 1;
|
|
134
|
+
}
|
|
135
|
+
|
|
136
|
+
/** Release one concurrent slot and try to drain waiters. */
|
|
137
|
+
releaseSlot() {
|
|
138
|
+
this.#concurrent = Math.max(0, this.#concurrent - 1);
|
|
139
|
+
this.#drain();
|
|
140
|
+
}
|
|
141
|
+
|
|
142
|
+
/** Live snapshot for the GUI. */
|
|
143
|
+
getStatus() {
|
|
144
|
+
return {
|
|
145
|
+
strategy: "token-bucket",
|
|
146
|
+
tokens: Math.round(this.#tokens * 100) / 100,
|
|
147
|
+
burstSize: this.#cfg.burstSize,
|
|
148
|
+
refillRate: this.#cfg.refillRate,
|
|
149
|
+
concurrent: this.#concurrent,
|
|
150
|
+
reserved: this.#reserved,
|
|
151
|
+
maxConcurrent: this.#cfg.maxConcurrent,
|
|
152
|
+
queued: this.#waiters.length,
|
|
153
|
+
};
|
|
154
|
+
}
|
|
155
|
+
|
|
156
|
+
/** Replace config at runtime (hot-reload). */
|
|
157
|
+
update(config) {
|
|
158
|
+
Object.assign(this.#cfg, config);
|
|
159
|
+
}
|
|
160
|
+
|
|
161
|
+
/** Dispose — release all waiters and stop the refill timer. */
|
|
162
|
+
dispose() {
|
|
163
|
+
clearInterval(this.#timer);
|
|
164
|
+
for (const w of this.#waiters) w.abort();
|
|
165
|
+
this.#waiters = [];
|
|
166
|
+
}
|
|
167
|
+
|
|
168
|
+
/* ── internals ─────────────────────────────────────── */
|
|
169
|
+
|
|
170
|
+
#startRefill() {
|
|
171
|
+
// Refill every 50 ms for smooth granularity.
|
|
172
|
+
this.#timer = setInterval(() => {
|
|
173
|
+
const now = Date.now();
|
|
174
|
+
const elapsed = (now - this.#lastRefill) / 1000;
|
|
175
|
+
this.#tokens = Math.min(this.#cfg.burstSize, this.#tokens + elapsed * this.#cfg.refillRate);
|
|
176
|
+
this.#lastRefill = now;
|
|
177
|
+
this.#drain();
|
|
178
|
+
}, 50);
|
|
179
|
+
}
|
|
180
|
+
|
|
181
|
+
/** Drain waiters as long as we have tokens and concurrency room. */
|
|
182
|
+
#drain() {
|
|
183
|
+
while (
|
|
184
|
+
this.#waiters.length > 0
|
|
185
|
+
&& this.#tokens >= 1
|
|
186
|
+
&& this.#concurrent + this.#reserved < this.#cfg.maxConcurrent
|
|
187
|
+
) {
|
|
188
|
+
this.#tokens -= 1;
|
|
189
|
+
// Reserve the concurrent slot for the waiter we are about to wake; it is
|
|
190
|
+
// converted into a live slot by `acquireSlot()` a microtask later.
|
|
191
|
+
this.#reserved += 1;
|
|
192
|
+
const waiter = this.#waiters.shift();
|
|
193
|
+
waiter?.resolve();
|
|
194
|
+
}
|
|
195
|
+
}
|
|
196
|
+
}
|