@mlx-node/server 0.0.9 → 0.0.12

package/dist/registry.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"registry.d.ts","sourceRoot":"","sources":["../src/registry.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAqCG;AAEH,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,gBAAgB,CAAC;AACjD,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,cAAc,CAAC;AAExD,OAAO,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AAExD,yEAAyE;AACzE,MAAM,MAAM,aAAa,GAAG,mBAAmB,CAAC;AAEhD,0CAA0C;AAC1C,MAAM,WAAW,UAAU;IACzB,EAAE,EAAE,MAAM,CAAC;IACX,KAAK,EAAE,aAAa,CAAC;IACrB,SAAS,EAAE,MAAM,CAAC;IAClB;;;OAGG;IACH,eAAe,EAAE,eAAe,CAAC;CAClC;AAsCD;;GAEG;AACH,MAAM,WAAW,oBAAoB;IACnC;;;;;;OAMG;IACH,aAAa,CAAC,EAAE,MAAM,CAAC;CACxB;AAED;;GAEG;AACH,MAAM,WAAW,eAAe;IAC9B;;;;;;;;;;;;;;;OAeG;IACH,gBAAgB,CAAC,EAAE,UAAU,CAAC;IAC9B;;;;;OAKG;IACH,eAAe,CAAC,EAAE,MAAM,CAAC;CAC1B;AAED,qBAAa,aAAa;IACxB,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAqB;IACnD,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAiC;IACxD;;;;;;;OAOG;IACH,OAAO,CAAC,QAAQ,CAAC,wBAAwB,CAAoD;IAC7F;;;;;OAKG;IACH,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAoC;IAChE,yDAAyD;IACzD,OAAO,CAAC,cAAc,CAAK;IAC3B;;;;;;;;;;;;;;;;;;;;;;;;;;;;OA4BG;IACH,OAAO,CAAC,QAAQ,CAAC,kBAAkB,CAAkF;IAErH,YAAY,IAAI,CAAC,EAAE,oBAAoB,EAEtC;IAED;;;;;;;;;;;;;;;;;;;;;;;OAuBG;IACH,QAAQ,CAAC,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,aAAa,EAAE,IAAI,CAAC,EAAE,eAAe,GAAG,IAAI,CAwFzE;IAED;;;;;;;;;;OAUG;IACH,UAAU,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAMhC;IAED;;;;;;;;;;;;OAYG;IACH,OAAO,CAAC,iBAAiB;IAiBzB;;;;;;;;;;;;;;;OAeG;IACH,OAAO,CAAC,uBAAuB;IAK/B;;;;;;;;;;;;;;;;;;OAkBG;IACH,oBAAoB,CAClB,IAAI,EAAE,MAAM,GACX;QAAE,KAAK,EAAE,aAAa,CAAC;QAAC,QAAQ,EAAE,eAAe,CAAC;QAAC,UAAU,EAAE,MAAM,CAAA;KAAE,GAAG,SAAS,CASrF;IAED;;;;;;;;OAQG;IACH,oBAAoB,CAAC,KAAK,EAAE,aAAa,GAAG,IAAI,CAQ/C;IAED;;;;;;;;;;;;;;;;;;;;;;;;;OAyBG;IACH,aAAa,CAAC,KAAK,EAAE,aAAa,GAAG,IAAI,CAIxC;IAED;;;;;;;;OAQG;IACH,cAAc,CAAC,KAAK,EAAE,aAAa,GAAG,IAAI,CAQzC;IAED;;;;;;;;;;;;;;;;;;;;;OAqBG;IACH,+BAA+B,CAAC,KAAK,EAAE,aAAa,GAAG;QAAE,UAAU,EAAE,MAAM,CAAA;KAAE,GAAG,SAAS,CAUxF;IAED;;;;;;;;;;OAUG;IACH,gBAAgB,CAAC,KAAK,EAAE,aAAa,GAAG,IAAI,CAO3C;IAED;;OAEG;IACH,GAAG,CAAC,IAAI,EAAE,MAAM,GAAG,aAAa,GAAG,SAAS,CAE3C;IAED;;;;;;;;;;;;;;OAcG;IACH,aAAa,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,GAAG,SAAS,CAI9C;IAED;;;;;;;;;;;;;;OAcG;IACH,kBAAkB,CAAC,IAAI,EAAE,MAAM,GAAG,eAAe,GAAG,SAAS,CAE5D;IAED;;;;;;;;;OASG;IACH,qBAAqB,IAAI,eAAe,EAAE,CAMzC;IAED;;OAEG;IACH,IAAI,IAAI;QAAE,EAAE,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAC;QAAC,OAAO,EAAE,MAAM,CAAC;QAAC,QAAQ,EAAE,MAAM,CAAA;KAAE,EAAE,CAgB1E;IAED;;;;;;;;;OASG;IACH,gBAAgB,CAAC,KAAK,EAAE,aAAa,GAAG,OAAO,CAG9C;CACF"}
+{"version":3,"file":"registry.d.ts","sourceRoot":"","sources":["../src/registry.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAqCG;AAEH,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,gBAAgB,CAAC;AACjD,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,cAAc,CAAC;AAExD,OAAO,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AAExD,UAAU,6BAA6B;IACrC,yBAAyB,CAAC,OAAO,EAAE,MAAM,EAAE,QAAQ,EAAE,eAAe,GAAG,IAAI,CAAC;IAC5E,2BAA2B,CAAC,OAAO,EAAE,MAAM,EAAE,QAAQ,EAAE,eAAe,GAAG,IAAI,CAAC;CAC/E;AASD,yEAAyE;AACzE,MAAM,MAAM,aAAa,GAAG,mBAAmB,CAAC;AAEhD,0CAA0C;AAC1C,MAAM,WAAW,UAAU;IACzB,EAAE,EAAE,MAAM,CAAC;IACX,KAAK,EAAE,aAAa,CAAC;IACrB,SAAS,EAAE,MAAM,CAAC;IAClB;;;OAGG;IACH,eAAe,EAAE,eAAe,CAAC;CAClC;AAsCD;;GAEG;AACH,MAAM,WAAW,oBAAoB;IACnC;;;;;;OAMG;IACH,aAAa,CAAC,EAAE,MAAM,CAAC;CACxB;AAED;;GAEG;AACH,MAAM,WAAW,eAAe;IAC9B;;;;;;;;;;;;;;;OAeG;IACH,gBAAgB,CAAC,EAAE,UAAU,CAAC;IAC9B;;;;;OAKG;IACH,eAAe,CAAC,EAAE,MAAM,CAAC;CAC1B;AAED,qBAAa,aAAa;IACxB,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAqB;IACnD,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAiC;IACxD;;;;;;;OAOG;IACH,OAAO,CAAC,QAAQ,CAAC,wBAAwB,CAAoD;IAC7F;;;;;OAKG;IACH,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAoC;IAChE,yDAAyD;IACzD,OAAO,CAAC,cAAc,CAAK;IAC3B;;;;;;;;;;;;;;;;;;;;;;;;;;;;OA4BG;IACH,OAAO,CAAC,QAAQ,CAAC,kBAAkB,CAAkF;IACrH,OAAO,CAAC,6BAA6B,CAA4C;IAEjF,YAAY,IAAI,CAAC,EAAE,oBAAoB,EAEtC;IAED,sEAAsE;IACtE,IAAI,eAAe,IAAI,MAAM,GAAG,SAAS,CAExC;IAED;;;;;OAKG;IACH,gCAAgC,CAAC,WAAW,EAAE,6BAA6B,GAAG,SAAS,GAAG,IAAI,CAa7F;IAED;;;;;;;;;;;;;;;;;;;;;;;OAuBG;IACH,QAAQ,CAAC,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,aAAa,EAAE,IAAI,CAAC,EAAE,eAAe,GAAG,IAAI,CA4FzE;IAED;;;;;;;;;;OAUG;IACH,UAAU,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAOhC;IAED;;;;;;;;;;;;OAYG;IACH,OAAO,CAAC,iBAAiB;IAiBzB;;;;;;;;;;;;;;;OAeG;IACH,OAAO,CAAC,uBAAuB;IAgB/B;;;;;;;;;;;;;;;;;;OAkBG;IACH,oBAAoB,CAClB,IAAI,EAAE,MAAM,GACX;QAAE,KAAK,EAAE,aAAa,CAAC;QAAC,QAAQ,EAAE,eAAe,CAAC;QAAC,UAAU,EAAE,MAAM,CAAA;KAAE,GAAG,SAAS,CASrF;IAED;;;;;;;;OAQG;IACH,oBAAoB,CAAC,KAAK,EAAE,aAAa,GAAG,IAAI,CAQ/C;IAED;;;;;;;;;;;;;;;;;;;;;;;;;OAyBG;IACH,aAAa,CAAC,KAAK,EAAE,aAAa,GAAG,IAAI,CAIxC;IAED;;;;;;;;OAQG;IACH,cAAc,CAAC,KAAK,EAAE,aAAa,GAAG,IAAI,CAQzC;IAED;;;;;;;;;;;;;;;;;;;;;OAqBG;IACH,+BAA+B,CAAC,KAAK,EAAE,aAAa,GAAG;QAAE,UAAU,EAAE,MAAM,CAAA;KAAE,GAAG,SAAS,CAUxF;IAED;;;;;;;;;;OAUG;IACH,gBAAgB,CAAC,KAAK,EAAE,aAAa,GAAG,IAAI,CAO3C;IAED;;OAEG;IACH,GAAG,CAAC,IAAI,EAAE,MAAM,GAAG,aAAa,GAAG,SAAS,CAE3C;IAED;;;;;;;;;;;;;;OAcG;IACH,aAAa,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,GAAG,SAAS,CAI9C;IAED;;;;;;;;;;;;;;OAcG;IACH,kBAAkB,CAAC,IAAI,EAAE,MAAM,GAAG,eAAe,GAAG,SAAS,CAE5D;IAED;;;;;;;;;OASG;IACH,qBAAqB,IAAI,eAAe,EAAE,CAMzC;IAED;;OAEG;IACH,IAAI,IAAI;QAAE,EAAE,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAC;QAAC,OAAO,EAAE,MAAM,CAAC;QAAC,QAAQ,EAAE,MAAM,CAAA;KAAE,EAAE,CAgB1E;IAED;;;;;;;;;OASG;IACH,gBAAgB,CAAC,KAAK,EAAE,aAAa,GAAG,OAAO,CAG9C;CACF"}

package/dist/registry.js

@@ -37,6 +37,14 @@
  *     and the continuation is accepted.
  */
 import { SessionRegistry } from './session-registry.js';
+function concurrentDispatchCapacity(model) {
+    if (model.hasBlockPagedCache?.() !== true)
+        return 1;
+    const reported = model.maxConcurrentSequences?.();
+    if (reported === undefined || !Number.isSafeInteger(reported) || reported < 2)
+        return 1;
+    return reported;
+}
 export class ModelRegistry {
     maxQueueDepth;
     models = new Map();
@@ -88,9 +96,35 @@ export class ModelRegistry {
      * hard-timeouts have fired.
      */
     retiredInstanceIds = new WeakMap();
+    modelLoadAdmissionCoordinator;
     constructor(opts) {
         this.maxQueueDepth = opts?.maxQueueDepth;
     }
+    /** Configured per-model waiter cap, or `undefined` when unbounded. */
+    get queueDepthLimit() {
+        return this.maxQueueDepth;
+    }
+    /**
+     * Connect cold-load admission to name registration. The coordinator is
+     * attached by the HTTP handler before dispatch; existing names are
+     * published immediately and future `register`/`unregister` calls keep the
+     * mapping current.
+     */
+    setModelLoadAdmissionCoordinator(coordinator) {
+        if (this.modelLoadAdmissionCoordinator === coordinator)
+            return;
+        if (this.modelLoadAdmissionCoordinator) {
+            for (const [name, entry] of this.models) {
+                this.modelLoadAdmissionCoordinator.unbindRequestLoadAdmissions(name, entry.sessionRegistry);
+            }
+        }
+        this.modelLoadAdmissionCoordinator = coordinator;
+        if (coordinator) {
+            for (const [name, entry] of this.models) {
+                coordinator.bindRequestLoadAdmissions(name, entry.sessionRegistry);
+            }
+        }
+    }
     /**
      * Register a model under a given name.
      *
@@ -132,11 +166,13 @@ export class ModelRegistry {
             if (opts && 'maxOutputTokens' in opts) {
                 existing.sessionRegistry.setMaxOutputTokens(maxOutputTokens);
             }
+            this.modelLoadAdmissionCoordinator?.bindRequestLoadAdmissions(name, existing.sessionRegistry);
             return;
         }
         if (existing) {
             // Same name, different model: release the old model's refcount
             // before installing the new binding.
+            this.modelLoadAdmissionCoordinator?.unbindRequestLoadAdmissions(name, existing.sessionRegistry);
             this.dropNameReference(existing.model);
         }
         // Look up or allocate the shared binding. If it is still alive
@@ -151,6 +187,7 @@ export class ModelRegistry {
                 registry: new SessionRegistry({
                     model,
                     maxQueueDepth: this.maxQueueDepth,
+                    maxConcurrentDispatches: concurrentDispatchCapacity(model),
                     samplingDefaults,
                     maxOutputTokens,
                 }),
@@ -203,6 +240,7 @@ export class ModelRegistry {
             createdAt: Math.floor(Date.now() / 1000),
             sessionRegistry: binding.registry,
         });
+        this.modelLoadAdmissionCoordinator?.bindRequestLoadAdmissions(name, binding.registry);
     }
     /**
      * Unregister a model by name.
@@ -219,6 +257,7 @@ export class ModelRegistry {
         const entry = this.models.get(name);
         if (!entry)
             return false;
+        this.modelLoadAdmissionCoordinator?.unbindRequestLoadAdmissions(name, entry.sessionRegistry);
         this.models.delete(name);
         this.dropNameReference(entry.model);
         return true;
@@ -270,6 +309,18 @@ export class ModelRegistry {
      * persists that crossed the safety breaker.
      */
     finalizeBindingTeardown(model) {
+        const binding = this.sessionRegistriesByModel.get(model);
+        if (!binding)
+            return;
+        binding.registry.clear();
+        // The binding maps are removed synchronously below, so no later request can
+        // discover this registry and call flushPendingDisposals() for it. Start the
+        // flush while the registry is still in hand: the promise retains it until
+        // every pending disposal has settled and every transient failure observed
+        // during this flush has received its one bounded retry.
+        void binding.registry.flushPendingDisposals().catch((error) => {
+            console.error('[server] failed to flush final chat-session owner disposals:', error);
+        });
         this.sessionRegistriesByModel.delete(model);
         this.instanceIds.delete(model);
     }

package/dist/server.d.ts

@@ -6,6 +6,21 @@ import { type ServerHealth } from './health.js';
 import { type LoadModelOptions } from './load-model.js';
 import { ModelWorkCoordinator } from './model-work-coordinator.js';
 import { ModelRegistry } from './registry.js';
+/**
+ * Default per-model queue-depth cap (waiters-only; the actively running
+ * dispatch does not count). Applied when neither
+ * {@link ServerConfig.maxQueueDepthPerModel} nor the
+ * `MLX_MAX_QUEUE_DEPTH_PER_MODEL` env var is set; the config sentinel
+ * `'unbounded'` opts out entirely.
+ *
+ * 16, not 8: an agent fan-out (e.g. Claude Code bursting parallel tool
+ * calls) can put well over 8 concurrent requests behind one model from a
+ * SINGLE client, so 8 risked 429s on a lone user. Over-cap requests get
+ * HTTP 429 with `Retry-After: 1`, which the OpenAI/Anthropic SDK retry
+ * loops honor — the cap sheds pile-up instead of failing well-behaved
+ * clients.
+ */
+export declare const DEFAULT_MAX_QUEUE_DEPTH_PER_MODEL = 16;
 /**
  * Parse a positive integer seconds value; returns undefined for unset/invalid so caller can apply its own default.
  *
@@ -101,10 +116,13 @@ export interface ServerConfig {
      * `Retry-After: 1` header so clients can back off instead of piling
      * into an unbounded queue.
      *
-     * Default: `undefined` (unbounded — current behaviour). Env override:
-     * `MLX_MAX_QUEUE_DEPTH_PER_MODEL` (positive integer).
+     * Default: {@link DEFAULT_MAX_QUEUE_DEPTH_PER_MODEL} (16). Env
+     * override: `MLX_MAX_QUEUE_DEPTH_PER_MODEL` (positive integer). Pass
+     * `'unbounded'` to opt out of the cap entirely (the pre-cap
+     * behaviour); the sentinel also bypasses the env override, because an
+     * explicit config value always wins over env.
      */
-    maxQueueDepthPerModel?: number;
+    maxQueueDepthPerModel?: number | 'unbounded';
     /**
      * Milliseconds of HTTP inactivity (no request arrivals or completions)
      * before the server issues a single `clearCache()` to drain the MLX

package/dist/server.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"server.d.ts","sourceRoot":"","sources":["../src/server.ts"],"names":[],"mappings":"AAAA,0HAA0H;AAI1H,OAAO,KAAK,EAAE,MAAM,EAAkB,MAAM,WAAW,CAAC;AAIxD,OAAO,EAAE,aAAa,EAAE,MAAM,gBAAgB,CAAC;AAE/C,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,cAAc,CAAC;AAErD,OAAO,EAAwB,KAAK,YAAY,EAAE,MAAM,aAAa,CAAC;AAEtE,OAAO,EAAuB,KAAK,gBAAgB,EAAE,MAAM,iBAAiB,CAAC;AAC7E,OAAO,EAAE,oBAAoB,EAAE,MAAM,6BAA6B,CAAC;AACnE,OAAO,EAAE,aAAa,EAAE,MAAM,eAAe,CAAC;AA2B9C;;;;;;;;;;;;GAYG;AACH,wBAAgB,eAAe,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,GAAG,SAAS,CAOhE;AAED;;;;;;;GAOG;AACH,wBAAgB,mBAAmB,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,GAAG,SAAS,CAOpE;AAuBD;;;;;;;;;;;;;;;;;;;;;;;;;GAyBG;AACH,wBAAgB,gBAAgB,CAAC,QAAQ,EAAE,MAAM,GAAG,SAAS,GAAG,MAAM,GAAG,SAAS,CAUjF;AAED,MAAM,WAAW,YAAY;IAC3B,yCAAyC;IACzC,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,kDAAkD;IAClD,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,6EAA6E;IAC7E,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,0DAA0D;IAC1D,YAAY,CAAC,EAAE,OAAO,CAAC;IACvB;;;;;;OAMG;IACH,IAAI,CAAC,EAAE,OAAO,CAAC;IACf;;;;;;;;;;OAUG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB;;;;;;;OAOG;IACH,oBAAoB,CAAC,EAAE,MAAM,CAAC;IAC9B;;;;;;;;;OASG;IACH,qBAAqB,CAAC,EAAE,MAAM,CAAC;IAC/B;;;;;;;;;;;;;OAaG;IACH,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B;;;;;;OAMG;IACH,YAAY,CAAC,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;IAC/C;;;;;OAKG;IACH,UAAU,CAAC,EAAE,MAAM,gBAAgB,EAAE,CAAC;CACvC;AAED,gDAAgD;AAChD,MAAM,WAAW,YAAY;IAC3B;;;;;;OAMG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAED,+CAA+C;AAC/C,MAAM,WAAW,WAAW;IAC1B,2EAA2E;IAC3E,MAAM,EAAE,OAAO,CAAC;IAChB,iFAAiF;IACjF,cAAc,EAAE,MAAM,CAAC;IACvB,2CAA2C;IAC3C,UAAU,EAAE,MAAM,CAAC;CACpB;AAED,MAAM,WAAW,cAAc;IAC7B,MAAM,EAAE,MAAM,CAAC;IACf,gDAAgD;IAChD,QAAQ,EAAE,aAAa,CAAC;IACxB,0BAA0B;IAC1B,KAAK,EAAE,aAAa,GAAG,IAAI,CAAC;IAC5B;;;;;;OAMG;IACH,QAAQ,CAAC,SAAS,EAAE,oBAAoB,CAAC;IACzC;;;OAGG;IACH,MAAM,IAAI,YAAY,CAAC;IACvB;;;;;;;;;;;;;;;;;;OAkBG;IACH,KAAK,CAAC,IAAI,CAAC,EAAE,YAAY,GAAG,OAAO,CAAC,WAAW,CAAC,CAAC;IACjD;;;;;;;;;;;;;OAaG;IACH,SAAS,CAAC,IAAI,EAAE,gBAAgB,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IACjD;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OAgCG;IACH,mBAAmB,CAAC,CAAC,EAAE,EAAE,EAAE,MAAM,OAAO,CAAC,CAAC,CAAC,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC;IACzD,mBAAmB,CAAC,CAAC,EAAE,EAAE,EAAE,MAAM,CAAC,GAAG,CAAC,CAAC;IACvC;;;;;;;;OAQG;IACH,aAAa,IAAI,MAAM,IAAI,CAAC;CAC7B;AAED;;;;;;;;;GASG;AACH,wBAAsB,YAAY,CAAC,MAAM,CAAC,EAAE,YAAY,GAAG,OAAO,CAAC,cAAc,CAAC,CA4KjF"}
+{"version":3,"file":"server.d.ts","sourceRoot":"","sources":["../src/server.ts"],"names":[],"mappings":"AAAA,0HAA0H;AAI1H,OAAO,KAAK,EAAE,MAAM,EAAkB,MAAM,WAAW,CAAC;AAIxD,OAAO,EAAE,aAAa,EAAE,MAAM,gBAAgB,CAAC;AAE/C,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,cAAc,CAAC;AAErD,OAAO,EAAwB,KAAK,YAAY,EAAE,MAAM,aAAa,CAAC;AAEtE,OAAO,EAAuB,KAAK,gBAAgB,EAAE,MAAM,iBAAiB,CAAC;AAC7E,OAAO,EAAE,oBAAoB,EAAE,MAAM,6BAA6B,CAAC;AACnE,OAAO,EAAE,aAAa,EAAE,MAAM,eAAe,CAAC;AA2B9C;;;;;;;;;;;;;GAaG;AACH,eAAO,MAAM,iCAAiC,KAAK,CAAC;AAEpD;;;;;;;;;;;;GAYG;AACH,wBAAgB,eAAe,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,GAAG,SAAS,CAOhE;AAED;;;;;;;GAOG;AACH,wBAAgB,mBAAmB,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,GAAG,SAAS,CAOpE;AAuBD;;;;;;;;;;;;;;;;;;;;;;;;;GAyBG;AACH,wBAAgB,gBAAgB,CAAC,QAAQ,EAAE,MAAM,GAAG,SAAS,GAAG,MAAM,GAAG,SAAS,CAUjF;AAED,MAAM,WAAW,YAAY;IAC3B,yCAAyC;IACzC,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,kDAAkD;IAClD,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,6EAA6E;IAC7E,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,0DAA0D;IAC1D,YAAY,CAAC,EAAE,OAAO,CAAC;IACvB;;;;;;OAMG;IACH,IAAI,CAAC,EAAE,OAAO,CAAC;IACf;;;;;;;;;;OAUG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB;;;;;;;OAOG;IACH,oBAAoB,CAAC,EAAE,MAAM,CAAC;IAC9B;;;;;;;;;;;;OAYG;IACH,qBAAqB,CAAC,EAAE,MAAM,GAAG,WAAW,CAAC;IAC7C;;;;;;;;;;;;;OAaG;IACH,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B;;;;;;OAMG;IACH,YAAY,CAAC,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;IAC/C;;;;;OAKG;IACH,UAAU,CAAC,EAAE,MAAM,gBAAgB,EAAE,CAAC;CACvC;AAED,gDAAgD;AAChD,MAAM,WAAW,YAAY;IAC3B;;;;;;OAMG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAED,+CAA+C;AAC/C,MAAM,WAAW,WAAW;IAC1B,2EAA2E;IAC3E,MAAM,EAAE,OAAO,CAAC;IAChB,iFAAiF;IACjF,cAAc,EAAE,MAAM,CAAC;IACvB,2CAA2C;IAC3C,UAAU,EAAE,MAAM,CAAC;CACpB;AAED,MAAM,WAAW,cAAc;IAC7B,MAAM,EAAE,MAAM,CAAC;IACf,gDAAgD;IAChD,QAAQ,EAAE,aAAa,CAAC;IACxB,0BAA0B;IAC1B,KAAK,EAAE,aAAa,GAAG,IAAI,CAAC;IAC5B;;;;;;OAMG;IACH,QAAQ,CAAC,SAAS,EAAE,oBAAoB,CAAC;IACzC;;;OAGG;IACH,MAAM,IAAI,YAAY,CAAC;IACvB;;;;;;;;;;;;;;;;;;OAkBG;IACH,KAAK,CAAC,IAAI,CAAC,EAAE,YAAY,GAAG,OAAO,CAAC,WAAW,CAAC,CAAC;IACjD;;;;;;;;;;;;;OAaG;IACH,SAAS,CAAC,IAAI,EAAE,gBAAgB,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IACjD;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OAgCG;IACH,mBAAmB,CAAC,CAAC,EAAE,EAAE,EAAE,MAAM,OAAO,CAAC,CAAC,CAAC,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC;IACzD,mBAAmB,CAAC,CAAC,EAAE,EAAE,EAAE,MAAM,CAAC,GAAG,CAAC,CAAC;IACvC;;;;;;;;OAQG;IACH,aAAa,IAAI,MAAM,IAAI,CAAC;CAC7B;AAED;;;;;;;;;GASG;AACH,wBAAsB,YAAY,CAAC,MAAM,CAAC,EAAE,YAAY,GAAG,OAAO,CAAC,cAAc,CAAC,CAsLjF"}

package/dist/server.js

@@ -32,6 +32,21 @@ const DEFAULT_CLOSE_TIMEOUT_MS = 5000;
  * cost of a one-time prefill on recovery.
  */
 const DEFAULT_RESPONSE_RETENTION_SECONDS = 7 * 24 * 60 * 60; // 7 days
+/**
+ * Default per-model queue-depth cap (waiters-only; the actively running
+ * dispatch does not count). Applied when neither
+ * {@link ServerConfig.maxQueueDepthPerModel} nor the
+ * `MLX_MAX_QUEUE_DEPTH_PER_MODEL` env var is set; the config sentinel
+ * `'unbounded'` opts out entirely.
+ *
+ * 16, not 8: an agent fan-out (e.g. Claude Code bursting parallel tool
+ * calls) can put well over 8 concurrent requests behind one model from a
+ * SINGLE client, so 8 risked 429s on a lone user. Over-cap requests get
+ * HTTP 429 with `Retry-After: 1`, which the OpenAI/Anthropic SDK retry
+ * loops honor — the cap sheds pile-up instead of failing well-behaved
+ * clients.
+ */
+export const DEFAULT_MAX_QUEUE_DEPTH_PER_MODEL = 16;
 /**
  * Parse a positive integer seconds value; returns undefined for unset/invalid so caller can apply its own default.
  *
@@ -156,11 +171,20 @@ export async function createServer(config) {
     // `normalizePositiveIntConfig` for the failure modes we're guarding.
     const configRetentionSec = normalizePositiveIntConfig(config?.responseRetentionSec, 'responseRetentionSec');
     const responseRetentionSec = configRetentionSec ?? parseEnvSeconds('MLX_RESPONSE_RETENTION_SECONDS') ?? DEFAULT_RESPONSE_RETENTION_SECONDS;
-    // Opt-in queue-depth cap; resolved exactly once at server construction
-    // so the registry (and its per-model `SessionRegistry` instances
-    // allocated on `register()`) all share a single effective value.
-    const configMaxQueueDepth = normalizePositiveIntConfig(config?.maxQueueDepthPerModel, 'maxQueueDepthPerModel');
-    const maxQueueDepthPerModel = configMaxQueueDepth ?? parseEnvPositiveInt('MLX_MAX_QUEUE_DEPTH_PER_MODEL');
+    // Queue-depth cap; resolved exactly once at server construction so the
+    // registry (and its per-model `SessionRegistry` instances allocated on
+    // `register()`) all share a single effective value. Precedence:
+    // explicit config (the `'unbounded'` sentinel disables the cap
+    // outright, short-circuiting env) wins over env wins over the default
+    // of {@link DEFAULT_MAX_QUEUE_DEPTH_PER_MODEL}. Bogus numeric config
+    // still fails fast in `normalizePositiveIntConfig` before any env /
+    // default fallback applies.
+    const maxQueueDepthConfig = config?.maxQueueDepthPerModel;
+    const maxQueueDepthPerModel = maxQueueDepthConfig === 'unbounded'
+        ? undefined
+        : (normalizePositiveIntConfig(maxQueueDepthConfig, 'maxQueueDepthPerModel') ??
+            parseEnvPositiveInt('MLX_MAX_QUEUE_DEPTH_PER_MODEL') ??
+            DEFAULT_MAX_QUEUE_DEPTH_PER_MODEL);
     // Idle sweeper wiring. `0` is a legal explicit "off" value so we
     // cannot reuse `normalizePositiveIntConfig` (which rejects 0).
     // Precedence: explicit config value wins over env wins over default.
@@ -172,7 +196,7 @@ export async function createServer(config) {
     // request will not fire `clearCache()`. See the `idle-sweeper.ts`
     // module doc (section "Drain is post-request only") for rationale.
     const registry = new ModelRegistry({ maxQueueDepth: maxQueueDepthPerModel });
-    const modelWorkCoordinator = new ModelWorkCoordinator();
+    const modelWorkCoordinator = new ModelWorkCoordinator(maxQueueDepthPerModel);
     let store = null;
     if (!disableStore) {
         const storePath = config?.storePath ?? join(homedir(), '.mlx-node', 'responses.db');