@mlx-node/server 0.0.8 → 0.0.10

package/dist/server.js

@@ -5,11 +5,23 @@ import { homedir } from 'node:os';
 import { join } from 'node:path';
 import { ResponseStore } from '@mlx-node/core';
 import { createHandler } from './handler.js';
+import { createHealthReporter } from './health.js';
 import { createIdleSweeper, DEFAULT_IDLE_CLEAR_CACHE_MS, parseIdleClearCacheEnv } from './idle-sweeper.js';
+import { runGuardedModelLoad } from './load-model.js';
 import { ModelWorkCoordinator } from './model-work-coordinator.js';
 import { ModelRegistry } from './registry.js';
+import { activeSSEStreamCountForResponses } from './streaming.js';
 /** Cleanup interval for expired responses (ms). */
 const CLEANUP_INTERVAL_MS = 5 * 60 * 1000; // 5 minutes
+/**
+ * Default grace period before {@link ServerInstance.close} destroys
+ * still-open connections.
+ *
+ * 5 s is deliberately a little longer than the ~5 s GPU watchdog window: a
+ * decode loop that is about to yield its next token should get the chance to
+ * unwind cleanly rather than being cut off a hair early.
+ */
+const DEFAULT_CLOSE_TIMEOUT_MS = 5000;
 /**
  * Default retention for persisted response rows in SQLite, in seconds.
  *
@@ -84,6 +96,42 @@ function normalizePositiveIntConfig(value, name) {
     }
     return value;
 }
+/**
+ * Resolve the effective auth token from an explicit value plus the env
+ * fallback.
+ *
+ * Exported because `createInferenceHost` has to answer "is this server going
+ * to be protected?" BEFORE it binds, in order to refuse a non-loopback bind
+ * that would serve anonymously. Two independent copies of this rule would
+ * drift, and the direction it would drift is a host that refuses to start
+ * while `MLX_SERVER_AUTH_TOKEN` is sitting right there in the environment.
+ *
+ * An empty env var means "not set". An accidental `MLX_SERVER_AUTH_TOKEN=` in
+ * a launcher script must not enable auth with an empty secret that every
+ * credential-less request would then fail against.
+ *
+ * An empty EXPLICIT token is a different case and is rejected outright, for the
+ * same reason {@link normalizePositiveIntConfig} rejects a bogus explicit knob:
+ * somebody asked for a token and supplied nothing, which is
+ * `--auth-token "$TOKEN"` with `TOKEN` unset. Returning `''` made the bind
+ * guard read "auth is configured" and allow `0.0.0.0`, while the comparator
+ * accepted an empty `x-api-key` because both strings were empty — a wildcard
+ * bind published under a credential anyone can guess. Quietly downgrading to
+ * "no auth" instead would be fail-open in the other direction: on loopback it
+ * hands back the unauthenticated, wildcard-CORS server the operator was
+ * explicitly trying not to start. Throwing is the only answer that is wrong in
+ * neither bind mode, and it happens before anything binds or loads.
+ */
+export function resolveAuthToken(explicit) {
+    if (explicit === '') {
+        throw new Error('authToken was set to an empty string; pass a real secret or omit it entirely ' +
+            '(an unset shell variable in `--auth-token "$VAR"` is the usual cause).');
+    }
+    if (explicit !== undefined)
+        return explicit;
+    const fromEnv = process.env.MLX_SERVER_AUTH_TOKEN;
+    return fromEnv != null && fromEnv !== '' ? fromEnv : undefined;
+}
 /**
  * Start an MLX-Node HTTP server exposing `POST /v1/responses`,
  * `POST /v1/messages`, and `GET /v1/models`.
@@ -97,7 +145,10 @@ function normalizePositiveIntConfig(value, name) {
 export async function createServer(config) {
     const port = config?.port ?? 8080;
     const host = config?.host ?? '127.0.0.1';
-    const cors = config?.cors ?? true;
+    const authToken = resolveAuthToken(config?.authToken);
+    // Forwarded unresolved so `createHandler` applies the auth-aware default
+    // (`true` without a token, `false` with one) in exactly one place.
+    const cors = config?.cors;
     const disableStore = config?.disableStore ?? false;
     // Validate caller-supplied numeric knobs BEFORE consulting env fallbacks
     // so a bogus explicit value surfaces as a descriptive error instead of
@@ -139,6 +190,9 @@ export async function createServer(config) {
         }
     }, CLEANUP_INTERVAL_MS);
     cleanupTimer.unref();
+    // One reporter shared by the HTTP endpoint and `ServerInstance.health()`
+    // so both report the same uptime origin.
+    const health = createHealthReporter({ registry, idleSweeper, modelWorkCoordinator });
     const handler = createHandler(registry, {
         cors,
         store,
@@ -147,8 +201,22 @@ export async function createServer(config) {
         resolveModel: config?.resolveModel,
         modelWorkCoordinator,
         listModels: config?.listModels,
+        authToken,
+        health,
+    });
+    /**
+     * Responses owned by THIS HTTP server. The SSE registry is deliberately
+     * process-wide because `beginSSE` is also used by standalone handlers;
+     * shutdown accounting intersects it with this weak ownership set so one
+     * server cannot claim streams that another server leaves live. A WeakSet
+     * needs no second response cleanup lifecycle.
+     */
+    const ownedResponses = new WeakSet();
+    const server = httpCreateServer((req, res) => {
+        // Synchronous, before `handler` can reach either endpoint's `beginSSE`.
+        ownedResponses.add(res);
+        void handler(req, res);
     });
-    const server = httpCreateServer(handler);
     await new Promise((resolve, reject) => {
         const onError = (err) => {
             server.removeListener('error', onError);
@@ -160,21 +228,69 @@ export async function createServer(config) {
             resolve();
         });
     });
-    return {
-        server,
-        registry,
-        store,
-        async close() {
+    // Memoized so a second `close()` returns the first call's promise instead
+    // of re-entering `server.close()` (which invokes its callback with
+    // ERR_SERVER_NOT_RUNNING once the server is already down).
+    let closePromise = null;
+    const close = (opts) => {
+        if (closePromise !== null)
+            return closePromise;
+        const startedAt = Date.now();
+        const requested = opts?.timeoutMs;
+        const timeoutMs = typeof requested === 'number' && Number.isFinite(requested) && requested >= 0
+            ? requested
+            : DEFAULT_CLOSE_TIMEOUT_MS;
+        closePromise = (async () => {
             clearInterval(cleanupTimer);
             idleSweeper.close();
-            await new Promise((resolve, reject) => {
+            let forced = false;
+            let streamsAborted = 0;
+            // Arm the wait BEFORE dropping idle sockets so nothing can complete in
+            // the gap and settle `server.close()` before we are listening.
+            const serverClosed = new Promise((resolve, reject) => {
                 server.close((err) => {
-                    if (err)
+                    // Tolerated defensively: memoization should make this unreachable,
+                    // but a caller who also closed `instance.server` directly would
+                    // otherwise turn a successful shutdown into a rejection.
+                    if (err && err.code !== 'ERR_SERVER_NOT_RUNNING')
                         reject(err);
                     else
                         resolve();
                 });
             });
+            // Keep-alive sockets parked in a client pool hold no request; without
+            // this the shutdown would sit on them until the client's own timeout.
+            server.closeIdleConnections();
+            const forceTimer = setTimeout(() => {
+                forced = true;
+                // Snapshot BEFORE destroying: `closeAllConnections()` fires each
+                // response's `'close'` event, which unregisters it from the global SSE
+                // registry used by the intersection.
+                streamsAborted = activeSSEStreamCountForResponses(ownedResponses);
+                // Destroying the socket fires exactly the `res.on('close')` path a
+                // client disconnect fires, so the endpoint's AbortController cancels
+                // the native stream handle rather than leaking a running decode.
+                server.closeAllConnections();
+            }, timeoutMs);
+            try {
+                await serverClosed;
+            }
+            finally {
+                clearTimeout(forceTimer);
+            }
+            return { forced, streamsAborted, durationMs: Date.now() - startedAt };
+        })();
+        return closePromise;
+    };
+    return {
+        server,
+        registry,
+        store,
+        modelWork: modelWorkCoordinator,
+        health,
+        close,
+        loadModel(opts) {
+            return runGuardedModelLoad({ idleSweeper, modelWorkCoordinator, registry }, opts);
         },
         withSuspendedDrains(fn) {
             return idleSweeper.withSuspendedDrains(fn);

package/dist/session-registry.d.ts

@@ -281,6 +281,13 @@ export declare class SessionRegistry {
      * `fn`. Primarily for tests and diagnostics.
      */
     get queueDepth(): number;
+    /**
+     * Configured waiter cap for this model's execution mutex, or `undefined`
+     * when unbounded. Paired with {@link queueDepth} so a readiness probe can
+     * tell "3 waiters, unbounded" (fine) from "3 waiters, cap of 3" (the next
+     * request gets a 429) without reaching into private state.
+     */
+    get queueDepthLimit(): number | undefined;
     /**
      * Current sampling defaults applied to every new `ChatSession` this
      * registry allocates. Exposed primarily for tests and diagnostics.

package/dist/session-registry.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"session-registry.d.ts","sourceRoot":"","sources":["../src/session-registry.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA4FG;AAIH,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,gBAAgB,CAAC;AACjD,OAAO,EAAE,WAAW,EAAE,KAAK,mBAAmB,EAAE,MAAM,cAAc,CAAC;AAyErE;;;;;;;;;;;;;;;;GAgBG;AACH,wBAAgB,kCAAkC,IAAI,IAAI,CAGzD;AAED;;;;;;;;;GASG;AACH,wBAAgB,kCAAkC,IAAI,MAAM,CAE3D;AAuDD;;;;;;;;;;;;;;GAcG;AACH,wBAAgB,iCAAiC,CAAC,MAAM,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,GAAG,IAAI,CA2BzF;AAED,uDAAuD;AACvD,MAAM,WAAW,sBAAsB;IACrC,sFAAsF;IACtF,KAAK,EAAE,mBAAmB,CAAC;IAC3B,kFAAkF;IAClF,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB;;;;;;;;;;;OAWG;IACH,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB;;;;;;;;;;;;OAYG;IACH,gBAAgB,CAAC,EAAE,UAAU,CAAC;IAC9B;;;OAGG;IACH,eAAe,CAAC,EAAE,MAAM,CAAC;CAC1B;AAED;;;;;GAKG;AACH,qBAAa,cAAe,SAAQ,KAAK;IACvC,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IAEvB,YAAY,WAAW,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAK7C;CACF;AAED;;;;;;GAMG;AACH,MAAM,WAAW,mBAAmB;IAClC,OAAO,EAAE,WAAW,CAAC,mBAAmB,CAAC,CAAC;IAC1C,GAAG,EAAE,OAAO,CAAC;CACd;AAsCD,qBAAa,eAAe;IAC1B,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAsB;IAC5C,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAS;IAChC,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAqB;IACnD;;;;;OAKG;IACH,OAAO,CAAC,gBAAgB,CAAyB;IACjD,OAAO,CAAC,eAAe,CAAqB;IAC5C;;;;;;;;;;;;;;;;;;;OAmBG;IACH,OAAO,CAAC,WAAW,CAAK;IACxB;;;;;;OAMG;IACH,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAwC;IAChE;;;;;;;;;OASG;IACH,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAoC;IAChE;;;;;;;;;;;;OAYG;IACH,OAAO,CAAC,QAAQ,CAAmC;IAEnD,YAAY,IAAI,EAAE,sBAAsB,EAMvC;IAED;;;;;;;OAOG;IACH,OAAO,CAAC,UAAU;IASlB;;;;OAIG;IACH,IAAI,UAAU,IAAI,MAAM,CAEvB;IAED;;;OAGG;IACH,IAAI,qBAAqB,IAAI,UAAU,GAAG,SAAS,CAElD;IAED,IAAI,gBAAgB,IAAI,MAAM,GAAG,SAAS,CAEzC;IAED;;;;;;;;OAQG;IACH,mBAAmB,CAAC,QAAQ,EAAE,UAAU,GAAG,SAAS,GAAG,IAAI,CAE1D;IAED,kBAAkB,CAAC,KAAK,EAAE,MAAM,GAAG,SAAS,GAAG,IAAI,CAElD;IAED,+FAA+F;IAC/F,IAAI,IAAI,IAAI,MAAM,CAEjB;IAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OAkDG;IACH,WAAW,CACT,kBAAkB,EAAE,MAAM,GAAG,IAAI,EACjC,qBAAqB,EAAE,MAAM,GAAG,IAAI,EACpC,cAAc,GAAE,MAAM,GAAG,IAAW,GACnC,mBAAmB,CAsErB;IAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OA6BG;IACH,kBAAkB,IAAI,mBAAmB,CAExC;IAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OA8EG;IACH,kBAAkB,CAAC,qBAAqB,EAAE,MAAM,GAAG,IAAI,GAAG,mBAAmB,CAiB5E;IAED;;;;;;;;;;;;;;;;;OAiBG;IACH,KAAK,CACH,UAAU,EAAE,MAAM,EAClB,OAAO,EAAE,WAAW,CAAC,mBAAmB,CAAC,EACzC,YAAY,EAAE,MAAM,GAAG,IAAI,EAC3B,cAAc,GAAE,MAAM,GAAG,IAAI,GAAG,SAAgB,GAC/C,IAAI,CAcN;IAED;;OAEG;IACH,IAAI,CAAC,UAAU,EAAE,MAAM,GAAG,IAAI,CAE7B;IAED;;;;OAIG;IACH,KAAK,IAAI,IAAI,CAOZ;IAED,2DAA2D;IAC3D,KAAK,IAAI,IAAI,CAEZ;IAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OAuCG;IACH,aAAa,CAAC,CAAC,EAAE,EAAE,EAAE,MAAM,OAAO,CAAC,CAAC,CAAC,GAAG,OAAO,CAAC,CAAC,CAAC,CA4BjD;IAED;;;;;;;;;;OAUG;YACW,aAAa;CAiD5B"}
+{"version":3,"file":"session-registry.d.ts","sourceRoot":"","sources":["../src/session-registry.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA4FG;AAIH,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,gBAAgB,CAAC;AACjD,OAAO,EAAE,WAAW,EAAE,KAAK,mBAAmB,EAAE,MAAM,cAAc,CAAC;AAyErE;;;;;;;;;;;;;;;;GAgBG;AACH,wBAAgB,kCAAkC,IAAI,IAAI,CAGzD;AAED;;;;;;;;;GASG;AACH,wBAAgB,kCAAkC,IAAI,MAAM,CAE3D;AAuDD;;;;;;;;;;;;;;GAcG;AACH,wBAAgB,iCAAiC,CAAC,MAAM,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,GAAG,IAAI,CA2BzF;AAED,uDAAuD;AACvD,MAAM,WAAW,sBAAsB;IACrC,sFAAsF;IACtF,KAAK,EAAE,mBAAmB,CAAC;IAC3B,kFAAkF;IAClF,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB;;;;;;;;;;;OAWG;IACH,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB;;;;;;;;;;;;OAYG;IACH,gBAAgB,CAAC,EAAE,UAAU,CAAC;IAC9B;;;OAGG;IACH,eAAe,CAAC,EAAE,MAAM,CAAC;CAC1B;AAED;;;;;GAKG;AACH,qBAAa,cAAe,SAAQ,KAAK;IACvC,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IAEvB,YAAY,WAAW,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAK7C;CACF;AAED;;;;;;GAMG;AACH,MAAM,WAAW,mBAAmB;IAClC,OAAO,EAAE,WAAW,CAAC,mBAAmB,CAAC,CAAC;IAC1C,GAAG,EAAE,OAAO,CAAC;CACd;AAsCD,qBAAa,eAAe;IAC1B,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAsB;IAC5C,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAS;IAChC,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAqB;IACnD;;;;;OAKG;IACH,OAAO,CAAC,gBAAgB,CAAyB;IACjD,OAAO,CAAC,eAAe,CAAqB;IAC5C;;;;;;;;;;;;;;;;;;;OAmBG;IACH,OAAO,CAAC,WAAW,CAAK;IACxB;;;;;;OAMG;IACH,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAwC;IAChE;;;;;;;;;OASG;IACH,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAoC;IAChE;;;;;;;;;;;;OAYG;IACH,OAAO,CAAC,QAAQ,CAAmC;IAEnD,YAAY,IAAI,EAAE,sBAAsB,EAMvC;IAED;;;;;;;OAOG;IACH,OAAO,CAAC,UAAU;IASlB;;;;OAIG;IACH,IAAI,UAAU,IAAI,MAAM,CAEvB;IAED;;;;;OAKG;IACH,IAAI,eAAe,IAAI,MAAM,GAAG,SAAS,CAExC;IAED;;;OAGG;IACH,IAAI,qBAAqB,IAAI,UAAU,GAAG,SAAS,CAElD;IAED,IAAI,gBAAgB,IAAI,MAAM,GAAG,SAAS,CAEzC;IAED;;;;;;;;OAQG;IACH,mBAAmB,CAAC,QAAQ,EAAE,UAAU,GAAG,SAAS,GAAG,IAAI,CAE1D;IAED,kBAAkB,CAAC,KAAK,EAAE,MAAM,GAAG,SAAS,GAAG,IAAI,CAElD;IAED,+FAA+F;IAC/F,IAAI,IAAI,IAAI,MAAM,CAEjB;IAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OAkDG;IACH,WAAW,CACT,kBAAkB,EAAE,MAAM,GAAG,IAAI,EACjC,qBAAqB,EAAE,MAAM,GAAG,IAAI,EACpC,cAAc,GAAE,MAAM,GAAG,IAAW,GACnC,mBAAmB,CAsErB;IAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OA6BG;IACH,kBAAkB,IAAI,mBAAmB,CAExC;IAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OA8EG;IACH,kBAAkB,CAAC,qBAAqB,EAAE,MAAM,GAAG,IAAI,GAAG,mBAAmB,CAiB5E;IAED;;;;;;;;;;;;;;;;;OAiBG;IACH,KAAK,CACH,UAAU,EAAE,MAAM,EAClB,OAAO,EAAE,WAAW,CAAC,mBAAmB,CAAC,EACzC,YAAY,EAAE,MAAM,GAAG,IAAI,EAC3B,cAAc,GAAE,MAAM,GAAG,IAAI,GAAG,SAAgB,GAC/C,IAAI,CAcN;IAED;;OAEG;IACH,IAAI,CAAC,UAAU,EAAE,MAAM,GAAG,IAAI,CAE7B;IAED;;;;OAIG;IACH,KAAK,IAAI,IAAI,CAOZ;IAED,2DAA2D;IAC3D,KAAK,IAAI,IAAI,CAEZ;IAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OAuCG;IACH,aAAa,CAAC,CAAC,EAAE,EAAE,EAAE,MAAM,OAAO,CAAC,CAAC,CAAC,GAAG,OAAO,CAAC,CAAC,CAAC,CA4BjD;IAED;;;;;;;;;;OAUG;YACW,aAAa;CAiD5B"}

package/dist/session-registry.js

@@ -406,6 +406,15 @@ export class SessionRegistry {
     get queueDepth() {
         return this.queuedCount;
     }
+    /**
+     * Configured waiter cap for this model's execution mutex, or `undefined`
+     * when unbounded. Paired with {@link queueDepth} so a readiness probe can
+     * tell "3 waiters, unbounded" (fine) from "3 waiters, cap of 3" (the next
+     * request gets a 429) without reaching into private state.
+     */
+    get queueDepthLimit() {
+        return this.maxQueueDepth;
+    }
     /**
      * Current sampling defaults applied to every new `ChatSession` this
      * registry allocates. Exposed primarily for tests and diagnostics.

package/dist/streaming.d.ts

@@ -4,4 +4,18 @@ export declare function beginSSE(res: ServerResponse): void;
 /** Write one SSE event. Injects `type: eventType` into the payload (data's own `type` wins) for OpenAI SDK compatibility. */
 export declare function writeSSEEvent(res: ServerResponse, eventType: string, data: object): void;
 export declare function endSSE(res: ServerResponse): void;
+/**
+ * Number of SSE streams currently open process-wide. Diagnostics and
+ * shutdown accounting only.
+ */
+export declare function activeSSEStreamCount(): number;
+/**
+ * Number of active SSE streams among a caller-owned collection of responses.
+ *
+ * `createServer()` uses this to intersect the process-wide SSE registry with
+ * the responses accepted by one `node:http` Server. Keep the no-argument
+ * {@link activeSSEStreamCount} above for process-wide diagnostics and
+ * standalone `createHandler()` consumers.
+ */
+export declare function activeSSEStreamCountForResponses(responses: WeakSet<ServerResponse>): number;
 //# sourceMappingURL=streaming.d.ts.map

package/dist/streaming.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"streaming.d.ts","sourceRoot":"","sources":["../src/streaming.ts"],"names":[],"mappings":"AAAA,4BAA4B;AAE5B,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,WAAW,CAAC;AAEhD,wBAAgB,QAAQ,CAAC,GAAG,EAAE,cAAc,GAAG,IAAI,CAMlD;AAED,6HAA6H;AAC7H,wBAAgB,aAAa,CAAC,GAAG,EAAE,cAAc,EAAE,SAAS,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,GAAG,IAAI,CAGxF;AAED,wBAAgB,MAAM,CAAC,GAAG,EAAE,cAAc,GAAG,IAAI,CAEhD"}
+{"version":3,"file":"streaming.d.ts","sourceRoot":"","sources":["../src/streaming.ts"],"names":[],"mappings":"AAAA,4BAA4B;AAE5B,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,WAAW,CAAC;AAgBhD,wBAAgB,QAAQ,CAAC,GAAG,EAAE,cAAc,GAAG,IAAI,CAclD;AAED,6HAA6H;AAC7H,wBAAgB,aAAa,CAAC,GAAG,EAAE,cAAc,EAAE,SAAS,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,GAAG,IAAI,CAGxF;AAED,wBAAgB,MAAM,CAAC,GAAG,EAAE,cAAc,GAAG,IAAI,CAGhD;AAED;;;GAGG;AACH,wBAAgB,oBAAoB,IAAI,MAAM,CAE7C;AAED;;;;;;;GAOG;AACH,wBAAgB,gCAAgC,CAAC,SAAS,EAAE,OAAO,CAAC,cAAc,CAAC,GAAG,MAAM,CAM3F"}

package/dist/streaming.js

@@ -1,5 +1,26 @@
 /** SSE writer utilities. */
+/**
+ * Responses that have committed to SSE (`beginSSE`) but have not yet been
+ * ended (`endSSE`) or had their connection torn down.
+ *
+ * Purely an accounting aid for {@link activeSSEStreamCount}, which graceful
+ * shutdown reads to report how many streams a forced close cut short. Nothing
+ * in the request path branches on membership, so a miscount can only skew a
+ * diagnostic number — never the behaviour of a live stream.
+ *
+ * Module-scoped because `@mlx-node/server` is loaded once per process and
+ * `beginSSE`/`endSSE` are free functions called from both endpoints.
+ */
+const activeSSEResponses = new Set();
 export function beginSSE(res) {
+    activeSSEResponses.add(res);
+    // Belt-and-braces cleanup: a stream torn down by a client disconnect (or by
+    // `server.closeAllConnections()`) may unwind through an error path that
+    // never reaches `endSSE`. Without this the entry would leak for the life of
+    // the process and inflate the count forever.
+    res.once('close', () => {
+        activeSSEResponses.delete(res);
+    });
     res.writeHead(200, {
         'Content-Type': 'text/event-stream',
         'Cache-Control': 'no-cache',
@@ -12,5 +33,29 @@ export function writeSSEEvent(res, eventType, data) {
     res.write(`event: ${eventType}\ndata: ${JSON.stringify(payload)}\n\n`);
 }
 export function endSSE(res) {
+    activeSSEResponses.delete(res);
     res.end();
 }
+/**
+ * Number of SSE streams currently open process-wide. Diagnostics and
+ * shutdown accounting only.
+ */
+export function activeSSEStreamCount() {
+    return activeSSEResponses.size;
+}
+/**
+ * Number of active SSE streams among a caller-owned collection of responses.
+ *
+ * `createServer()` uses this to intersect the process-wide SSE registry with
+ * the responses accepted by one `node:http` Server. Keep the no-argument
+ * {@link activeSSEStreamCount} above for process-wide diagnostics and
+ * standalone `createHandler()` consumers.
+ */
+export function activeSSEStreamCountForResponses(responses) {
+    let count = 0;
+    for (const response of activeSSEResponses) {
+        if (responses.has(response))
+            count += 1;
+    }
+    return count;
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@mlx-node/server",
-  "version": "0.0.8",
+  "version": "0.0.10",
   "homepage": "https://github.com/mlx-node/mlx-node",
   "bugs": {
     "url": "https://github.com/mlx-node/mlx-node/issues"
@@ -21,14 +21,26 @@
     ".": {
       "types": "./dist/index.d.ts",
       "import": "./dist/index.js"
+    },
+    "./host": {
+      "types": "./dist/host/index.d.ts",
+      "import": "./dist/host/index.js"
+    },
+    "./host/env-policy": {
+      "types": "./dist/host/env-policy.d.ts",
+      "import": "./dist/host/env-policy.js"
+    },
+    "./host/paths": {
+      "types": "./dist/host/paths.d.ts",
+      "import": "./dist/host/paths.js"
     }
   },
   "scripts": {
     "build": "tsc -b"
   },
   "dependencies": {
-    "@mlx-node/core": "0.0.8",
-    "@mlx-node/lm": "0.0.8"
+    "@mlx-node/core": "0.0.10",
+    "@mlx-node/lm": "0.0.10"
   },
   "devDependencies": {
     "@types/node": "@types/node@24.12.2"