@naced/llm-multi-rate-limiter 0.1.0 → 0.1.2

package/lib/strategies/sliding-window.js

@@ -14,6 +14,14 @@ export class SlidingWindowStrategy {
   /** @type {{ windowMs: number, maxRpm: number, maxConcurrent: number }} */
   #cfg;
   #concurrent = 0;
+  /**
+   * Slots handed to a waiter that has not called `acquireSlot()` yet.
+   *
+   * `#drain()` wakes several waiters in one pass and each of them takes its
+   * concurrent slot only after the promise settles; counting reservations keeps
+   * one release from unlocking the entire queue past `maxConcurrent`.
+   */
+  #reserved = 0;
   /** @type {number[]} — timestamps (Date.now()) inside the window */
   #timestamps = [];
   /** @type {Array<{ resolve: (r: { granted: boolean }) => void, abort: () => void }>} */
@@ -45,7 +53,12 @@ export class SlidingWindowStrategy {
     if (this.#cfg.maxConcurrent <= 0) return Promise.resolve({ granted: false });
 
     return new Promise((resolve) => {
+      // A waiter can be granted by `#drain()` in the same tick its timeout
+      // fires; settling once keeps the window/slot accounting exact.
+      let settled = false;
       const onDone = (result) => {
+        if (settled) return;
+        settled = true;
         clearTimeout(timer);
         signal?.removeEventListener("abort", onAbort);
         const idx = this.#waiters.indexOf(entry);
@@ -86,7 +99,12 @@ export class SlidingWindowStrategy {
     return { granted: false };
   }
 
+  /**
+   * Mark one concurrent slot as in-use (converting a `#drain()` reservation
+   * when there is one, so the slot is never counted twice).
+   */
   acquireSlot() {
+    if (this.#reserved > 0) this.#reserved -= 1;
     this.#concurrent += 1;
   }
 
@@ -102,6 +120,7 @@ export class SlidingWindowStrategy {
       countInWindow: this.#count(),
       maxRpm: this.#cfg.maxRpm,
       concurrent: this.#concurrent,
+      reserved: this.#reserved,
       maxConcurrent: this.#cfg.maxConcurrent,
       queued: this.#waiters.length,
     };
@@ -139,8 +158,15 @@ export class SlidingWindowStrategy {
 
   /** Drain waiters as long as capacity is available. */
   #drain() {
-    while (this.#waiters.length > 0 && this.#count() < this.#cfg.maxRpm && this.#concurrent < this.#cfg.maxConcurrent) {
+    while (
+      this.#waiters.length > 0
+      && this.#count() < this.#cfg.maxRpm
+      && this.#concurrent + this.#reserved < this.#cfg.maxConcurrent
+    ) {
       this.#timestamps.push(Date.now());
+      // Reserve the concurrent slot for the waiter we are about to wake; it is
+      // converted into a live slot by `acquireSlot()` a microtask later.
+      this.#reserved += 1;
       const waiter = this.#waiters.shift();
       waiter?.resolve();
     }

package/lib/strategies/token-bucket.js

@@ -18,6 +18,15 @@ export class TokenBucketStrategy {
   #tokens;
   #lastRefill;
   #concurrent = 0;
+  /**
+   * Slots handed to a waiter that has not called `acquireSlot()` yet.
+   *
+   * `#drain()` resolves several waiters in one pass, and each of them takes its
+   * concurrent slot only after the promise settles. Counting reservations keeps
+   * the drain loop from granting more waiters than `maxConcurrent` allows —
+   * without this, one `releaseSlot()` could release the whole queue at once.
+   */
+  #reserved = 0;
   /** @type {Array<{ resolve: (r: { granted: boolean }) => void, abort: () => void }>} */
   #waiters = [];
   #timer;
@@ -63,7 +72,12 @@ export class TokenBucketStrategy {
     }
 
     return new Promise((resolve) => {
+      // A waiter can be granted by `#drain()` in the same tick its timeout
+      // fires; settling once keeps the token/slot accounting exact.
+      let settled = false;
       const onDone = (result) => {
+        if (settled) return;
+        settled = true;
         clearTimeout(timer);
         signal?.removeEventListener("abort", onAbort);
         const idx = this.#waiters.indexOf(entry);
@@ -108,8 +122,14 @@ export class TokenBucketStrategy {
     return { granted: false };
   }
 
-  /** Mark one concurrent slot as in-use. */
+  /**
+   * Mark one concurrent slot as in-use.
+   *
+   * When the slot was already reserved by `#drain()`, this converts the
+   * reservation into a live slot instead of taking a second one.
+   */
   acquireSlot() {
+    if (this.#reserved > 0) this.#reserved -= 1;
     this.#concurrent += 1;
   }
 
@@ -127,6 +147,7 @@ export class TokenBucketStrategy {
       burstSize: this.#cfg.burstSize,
       refillRate: this.#cfg.refillRate,
       concurrent: this.#concurrent,
+      reserved: this.#reserved,
       maxConcurrent: this.#cfg.maxConcurrent,
       queued: this.#waiters.length,
     };
@@ -159,8 +180,15 @@ export class TokenBucketStrategy {
 
   /** Drain waiters as long as we have tokens and concurrency room. */
   #drain() {
-    while (this.#waiters.length > 0 && this.#tokens >= 1 && this.#concurrent < this.#cfg.maxConcurrent) {
+    while (
+      this.#waiters.length > 0
+      && this.#tokens >= 1
+      && this.#concurrent + this.#reserved < this.#cfg.maxConcurrent
+    ) {
       this.#tokens -= 1;
+      // Reserve the concurrent slot for the waiter we are about to wake; it is
+      // converted into a live slot by `acquireSlot()` a microtask later.
+      this.#reserved += 1;
       const waiter = this.#waiters.shift();
       waiter?.resolve();
     }

package/lib/types/config.js

@@ -34,6 +34,20 @@ const ModelOverride = Schema.object({
   enabled: Schema.boolean().description("Enable/disable rate limiting for this model"),
 });
 
+/**
+ * Per-provider overrides — key is the provider id (e.g. `"openai"`).
+ *
+ * Sits between the global `defaults` and the per-model tier: one provider's
+ * models usually share a single quota, so this is where that quota belongs.
+ * Wildcards are accepted here too (`"mb-int-*"`).
+ */
+const ProviderOverride = Schema.object({
+  maxConcurrent: Schema.number().step(1).min(1).description("Max concurrent requests for this provider"),
+  maxRpm: Schema.number().step(1).min(1).description("Max requests per minute for this provider"),
+  burstSize: Schema.number().step(1).min(1).description("Burst capacity for this provider (token-bucket)"),
+  refillRate: Schema.number().min(0.1).description("Tokens refilled per second (token-bucket)"),
+});
+
 /**
  * Per-model defaults — applied to every model unless overridden.
  * The object is volatile AS A WHOLE so `config.defaults.get()` yields one
@@ -46,6 +60,20 @@ const Defaults = Schema.object({
   refillRate: Schema.number().min(0.1).default(1).description("Tokens per second (token-bucket)"),
 });
 
+/**
+ * Adaptive back-off after the provider answers 429.
+ *
+ * The limiter cannot see the provider's own quota arithmetic, so a `RATE_LIMIT`
+ * failure is treated as a hint: the model's rate is scaled by `factor` for
+ * `cooldownMs`, then restored after `recoverAfter` clean calls.
+ */
+const Cooldown = Schema.object({
+  enabled: Schema.boolean().default(true).description("Slow a model down after the provider returns 429"),
+  cooldownMs: Schema.number().step(1).min(1000).default(30_000).description("How long the reduced rate lasts"),
+  factor: Schema.number().min(0.05).max(0.9).step(0.05).default(0.5).description("Rate multiplier while cooling"),
+  recoverAfter: Schema.number().step(1).min(1).default(3).description("Clean calls before the full rate returns"),
+});
+
 export const Config = Schema.object({
   /** Master switch */
   enabled: Schema.boolean().default(true).volatile().description("Enable the rate limiter globally"),
@@ -60,7 +88,17 @@ export const Config = Schema.object({
   defaults: Defaults.default({}).volatile().description("Default limits applied to all models"),
 
   /**
-   * Per-model overrides — key is "provider/model".
+   * Per-provider overrides — key is the provider id, wildcards allowed.
+   *
+   * Volatile as a whole block, for the same reason as `models` below: two
+   * volatile layers on one path are rejected by schemastery.
+   */
+  providers: Schema.dict(ProviderOverride).default({}).volatile()
+    .description("Per-provider rate limit overrides"),
+
+  /**
+   * Per-model overrides — key is "provider/model"; `*` matches any run of
+   * characters inside one path segment, so `openai/*` covers every OpenAI model.
    *
    * WHOLE-BLOCK volatile: `dict(inner).volatile()`, never
    * `dict(inner.volatile())`. The latter marks the dict's `inner` while the
@@ -74,6 +112,9 @@ export const Config = Schema.object({
    */
   models: Schema.dict(ModelOverride).default({}).volatile().description("Per-model rate limit overrides"),
 
+  /** Adaptive back-off after a provider-side 429 */
+  cooldown: Cooldown.default({}).volatile().description("Slow down after the provider returns 429"),
+
   /** Queue behaviour when a request is throttled */
   onThrottled: Schema.union([
     Schema.const("queue").description("Wait in queue until a slot opens"),

package/package.json

@@ -1,17 +1,35 @@
 {
   "name": "@naced/llm-multi-rate-limiter",
-  "version": "0.1.0",
+  "version": "0.1.2",
   "description": "Per-model LLM call rate limiter for DeepSeek Harness with a Settings section UI, queueing and a live status panel",
   "type": "module",
   "main": "lib/index.js",
+  "types": "./lib/index.d.ts",
   "exports": {
-    ".": "./lib/index.js",
+    ".": {
+      "types": "./lib/index.d.ts",
+      "default": "./lib/index.js"
+    },
     "./client": "./lib/client.js",
     "./package.json": "./package.json"
   },
   "engines": {
+    "node": ">=20",
     "dsh": ">=0.2.1-rc.2"
   },
+  "author": "Naced <869521569@qq.com>",
+  "license": "MIT",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/NacedWang/dsh-llm-multi-rate-limiter.git"
+  },
+  "homepage": "https://github.com/NacedWang/dsh-llm-multi-rate-limiter#readme",
+  "bugs": {
+    "url": "https://github.com/NacedWang/dsh-llm-multi-rate-limiter/issues"
+  },
+  "publishConfig": {
+    "access": "public"
+  },
   "dsh": {
     "bundle": {
       "patch": "./cordis.patch.yml"
@@ -26,7 +44,11 @@
     }
   },
   "scripts": {
-    "build": "node --check lib/index.js && node --check lib/client.js"
+    "build": "node --check lib/index.js && node --check lib/client.js && node --check lib/config-resolve.js",
+    "test": "node --test",
+    "lint": "eslint .",
+    "check": "npm run build && npm run lint && npm test",
+    "prepublishOnly": "npm run build && npm test"
   },
   "keywords": [
     "dsh",
@@ -37,7 +59,6 @@
     "sliding-window",
     "cordis-plugin"
   ],
-  "license": "MIT",
   "peerDependencies": {
     "@deepseek-ai/cordis": ">=4.0.0",
     "@deepseek-ai/dsh-settings": ">=0.1.7-rc.1",
@@ -47,14 +68,15 @@
     "@deepseek-ai/schemastery": "^3.18.4"
   },
   "devDependencies": {
-    "@deepseek-ai/dsh-settings": "^0.1.7-rc.1"
+    "@deepseek-ai/dsh-settings": "^0.1.7-rc.1",
+    "eslint": "^9.15.0"
   },
   "files": [
     "lib/",
     "cordis.patch.yml",
     "package.json",
-    "README.md",
     "README.zh.md",
+    "README.md",
     "CHANGELOG.md",
     "LICENSE"
   ]