@mlx-node/server 0.0.0

package/dist/registry.js

@@ -0,0 +1,497 @@
+/**
+ * ModelRegistry -- maps friendly model names to loaded model instances.
+ *
+ * All models exposing the chat-session surface (see `SessionCapableModel`
+ * from `@mlx-node/lm`) are eligible for serving. Every registered model
+ * is paired with a `SessionRegistry` — an LRU+TTL cache of live
+ * `ChatSession` instances keyed by server-allocated response id.
+ *
+ * **Model-instance identity, not name.** Session registries are keyed
+ * by MODEL OBJECT identity. The single-warm-session invariant
+ * enforced by `SessionRegistry` is a property of the underlying
+ * `SessionCapableModel` (one shared native KV cache per instance), so
+ * registering the SAME model object under two names MUST yield the
+ * SAME `SessionRegistry` — otherwise each alias's local single-warm
+ * cache would hand out warm wrappers while the other alias silently
+ * stomps them via the shared native state. `register()` looks the
+ * model up in an identity-keyed map and reuses the existing registry
+ * on alias, or allocates a fresh one on first sight.
+ *
+ * **Monotonic per-instance ids.** Every distinct model object gets a
+ * monotonic `instanceId` on first registration, reused across aliases,
+ * and dropped when the binding is fully torn down. The responses
+ * endpoint persists this id alongside each stored record and, on a
+ * `previous_response_id` continuation, compares the stored id against
+ * the live id for `body.model`. This closes two holes a friendly-name
+ * check leaves open:
+ *
+ *  1. A name hot-swap — `register("foo", modelA)` then
+ *     `register("foo", modelB)` — would pass a string check, so a
+ *     chain produced by `modelA` could be silently replayed through
+ *     `modelB`'s tokenizer / chat template / KV layout. With instance
+ *     ids the stored id (modelA's) no longer matches the live id
+ *     (modelB's) and the continuation is rejected with 400.
+ *  2. Two NAMES aliasing the SAME model object would be spuriously
+ *     rejected by a string check comparing the stored name against
+ *     `body.model`. Instance ids recognise them as the same binding
+ *     and the continuation is accepted.
+ */
+import { SessionRegistry } from './session-registry.js';
+export class ModelRegistry {
+    maxQueueDepth;
+    models = new Map();
+    /**
+     * Identity-keyed (WeakMap semantics, but strong refs because the
+     * registry already holds the model through its ModelEntry) map
+     * from a model instance to its shared `SessionRegistry` binding.
+     * Every name that references the same model object resolves to
+     * the same binding — an alias of a registered model shares its
+     * session cache and therefore its single-warm invariant.
+     */
+    sessionRegistriesByModel = new Map();
+    /**
+     * Identity-keyed map from a model instance to its monotonic
+     * instance id. Entries are allocated on first registration,
+     * reused across aliasing, and dropped when the last binding
+     * releases (mirrors `sessionRegistriesByModel` lifetime exactly).
+     */
+    instanceIds = new Map();
+    /** Monotonic counter for `instanceIds`. Never reused. */
+    nextInstanceId = 1;
+    /**
+     * Tombstone map for instance ids retired by the hard-timeout
+     * breaker. When the responses endpoint force-releases a wedged
+     * persist's `retainBinding`, the breaker calls
+     * `retireInstanceIdForForceRelease(model)` BEFORE dropping the
+     * retain so the live id (already stamped into the pending record)
+     * is preserved here. A subsequent `register()` of the SAME model
+     * object that arrives AFTER the binding has fully torn down
+     * inherits the retired id instead of minting a fresh one — so a
+     * late-landing persist's row stays chainable. A true hot-swap
+     * (different model object) has no tombstone for the new model,
+     * so a fresh id is minted and the stale stored row is correctly
+     * rejected with 400 instance-mismatch.
+     *
+     * `WeakMap`-keyed on the model object so entries do not keep the
+     * model alive; if the model is GC'd the tombstone is cleaned up
+     * automatically.
+     *
+     * Lifetime is refcounted: store ONE `{ instanceId, outstandingCount }`
+     * entry per model. `retireInstanceIdForForceRelease` increments
+     * (creating the entry on first retire); `releaseTombstone`
+     * decrements and drops the entry when count hits zero. Because
+     * `register()` inherits the retired id whenever the tombstone
+     * exists, concurrent breakers on the same model all target the
+     * SAME numeric `instanceId` — one shared refcount keeps the
+     * tombstone alive as long as ANY pending persist still needs it,
+     * and memory is bounded at O(1) per model regardless of how many
+     * hard-timeouts have fired.
+     */
+    retiredInstanceIds = new WeakMap();
+    constructor(opts) {
+        this.maxQueueDepth = opts?.maxQueueDepth;
+    }
+    /**
+     * Register a model under a given name.
+     *
+     * If the name is already registered and the new model is a
+     * DIFFERENT instance, the old binding's refcount is decremented
+     * (and dropped if no other alias references it) before the new
+     * binding is taken. Re-registering with the SAME model instance
+     * leaves the binding unchanged.
+     *
+     * On first sight of a model object a fresh `SessionRegistry` is
+     * allocated. On alias the existing registry is reused so the
+     * single-warm invariant spans both names.
+     *
+     * Tombstone-inherit path: if the binding was previously torn down
+     * AND the hard-timeout breaker called
+     * `retireInstanceIdForForceRelease(model)` before teardown fired,
+     * the fresh binding inherits the retired instance id from
+     * `retiredInstanceIds` — so a late-landing persist's record stays
+     * chainable. A hot-swap (different model object) has no tombstone,
+     * so a fresh id is minted and the stale record fails
+     * `previous_response_id` with 400. The aliasing path naturally
+     * preserves the id because `instanceIds.has(model)` is already
+     * true.
+     */
+    register(name, model) {
+        const existing = this.models.get(name);
+        if (existing && existing.model === model) {
+            // Same name + same model object: leave the binding and refcount
+            // alone. Refresh createdAt so `/v1/models` surfaces the most
+            // recent registration time.
+            existing.createdAt = Math.floor(Date.now() / 1000);
+            return;
+        }
+        if (existing) {
+            // Same name, different model: release the old model's refcount
+            // before installing the new binding.
+            this.dropNameReference(existing.model);
+        }
+        // Look up or allocate the shared binding. If it is still alive
+        // but flagged `pendingTeardown`, clear the flag and reuse it —
+        // the fresh registration revives the binding before teardown
+        // runs so the shared `SessionRegistry` / mutex chain stays
+        // identical and any new dispatch serializes behind the
+        // in-flight one.
+        let binding = this.sessionRegistriesByModel.get(model);
+        if (!binding) {
+            binding = {
+                registry: new SessionRegistry({ model, maxQueueDepth: this.maxQueueDepth }),
+                refCount: 0,
+                inFlight: 0,
+                pendingPersists: 0,
+                pendingTeardown: false,
+            };
+            this.sessionRegistriesByModel.set(model, binding);
+        }
+        else if (binding.pendingTeardown) {
+            binding.pendingTeardown = false;
+        }
+        binding.refCount += 1;
+        // Allocate a fresh monotonic instance id on first sight of this
+        // model object; reuse the existing id on every alias thereafter.
+        // Id lifetime mirrors the binding's — see `finalizeBindingTeardown`.
+        // If the binding was fully torn down but the hard-timeout breaker
+        // retired the previous id for the same model object, inherit it
+        // from the tombstone instead of minting fresh.
+        if (!this.instanceIds.has(model)) {
+            // Tombstone is refcounted; we do NOT decrement here — the
+            // still-pending persists own the outstanding count and balance
+            // it via `releaseTombstone` in their own `.finally(...)`.
+            const tombstone = this.retiredInstanceIds.get(model);
+            if (tombstone) {
+                this.instanceIds.set(model, tombstone.instanceId);
+            }
+            else {
+                this.instanceIds.set(model, this.nextInstanceId);
+                this.nextInstanceId += 1;
+            }
+        }
+        this.models.set(name, {
+            id: name,
+            model,
+            createdAt: Math.floor(Date.now() / 1000),
+            sessionRegistry: binding.registry,
+        });
+    }
+    /**
+     * Unregister a model by name.
+     *
+     * Drops the name -> ModelEntry mapping and decrements the shared
+     * session-registry binding's refcount. When the refcount hits zero
+     * (no other alias references this model object) the binding — and
+     * the `SessionRegistry` it owns — is dropped entirely so cached
+     * sessions for the now-unreferenced model are released.
+     *
+     * @returns true if the model was removed.
+     */
+    unregister(name) {
+        const entry = this.models.get(name);
+        if (!entry)
+            return false;
+        this.models.delete(name);
+        this.dropNameReference(entry.model);
+        return true;
+    }
+    /**
+     * Decrement the refcount on a model binding; drop it at zero iff
+     * no dispatch holds a lease AND no post-commit persist is still
+     * retaining it. When either counter is non-zero the teardown is
+     * deferred via `pendingTeardown` so the `SessionRegistry` (and
+     * its `execLock` FIFO) stays alive until the last holder releases.
+     *
+     * A concurrent `register(sameModel)` between `dropNameReference()`
+     * and the final release clears `pendingTeardown` and reuses the
+     * still-live binding, preserving `modelInstanceId` so any row the
+     * pending persist is about to land still resolves to a live id
+     * when the next continuation arrives.
+     */
+    dropNameReference(model) {
+        const binding = this.sessionRegistriesByModel.get(model);
+        if (!binding)
+            return;
+        binding.refCount -= 1;
+        if (binding.refCount <= 0) {
+            if (binding.inFlight > 0 || binding.pendingPersists > 0) {
+                // Defer until the last lease AND the last persist retention
+                // drop. The binding and its instance id stay in the maps so
+                // a same-object re-registration before finalisation can
+                // revive it in place.
+                binding.pendingTeardown = true;
+                return;
+            }
+            this.finalizeBindingTeardown(model);
+        }
+    }
+    /**
+     * Drop `model`'s binding and instance id from the registry.
+     * Shared teardown step; a subsequent re-registration usually mints
+     * a FRESH instance id — once the last alias, lease, and persist
+     * retention all release, any previously stored record referencing
+     * this id belongs to a logically dead binding and a continuation
+     * against it must fall through to the `currentInstanceId === undefined`
+     * rejection path so the stale chain cannot be replayed.
+     *
+     * Tombstone exception: if the hard-timeout breaker retired the
+     * previous id via `retireInstanceIdForForceRelease` before the
+     * forced release, a subsequent same-object `register()` inherits
+     * the retired id from `retiredInstanceIds` instead of minting
+     * fresh — this preserves chain continuity for late-landing
+     * persists that crossed the safety breaker.
+     */
+    finalizeBindingTeardown(model) {
+        this.sessionRegistriesByModel.delete(model);
+        this.instanceIds.delete(model);
+    }
+    /**
+     * Acquire a dispatch lease on the session registry bound to `name`.
+     * Returns the live `SessionRegistry` and the binding's instance id,
+     * or `undefined` if the name is not registered. Every successful
+     * acquisition MUST be balanced with exactly one
+     * `releaseDispatchLease(model)` call (typically via try/finally).
+     *
+     * The lease keeps the binding alive past a concurrent
+     * `unregister()` / `register(differentModel)` sequence: the
+     * `SessionRegistry` and its `execLock` FIFO remain valid while any
+     * lease is outstanding, so a newly registered same-model alias
+     * will rebind to the SAME registry and its `withExclusive` will
+     * serialize behind the in-flight dispatch.
+     *
+     * The returned `model` handle is what the caller passes to
+     * `releaseDispatchLease()` — the lease binds to the model OBJECT
+     * (not the friendly name) because the name can be hot-swapped
+     * while the lease is held.
+     */
+    acquireDispatchLease(name) {
+        const entry = this.models.get(name);
+        if (!entry)
+            return undefined;
+        const binding = this.sessionRegistriesByModel.get(entry.model);
+        if (!binding)
+            return undefined;
+        const instanceId = this.instanceIds.get(entry.model);
+        if (instanceId === undefined)
+            return undefined;
+        binding.inFlight += 1;
+        return { model: entry.model, registry: binding.registry, instanceId };
+    }
+    /**
+     * Release a dispatch lease previously obtained via
+     * `acquireDispatchLease()`. Decrements the binding's in-flight
+     * counter and, if the binding has been flagged for teardown (its
+     * refcount hit zero while the lease was held), drops it once the
+     * last lease releases. Safe to call exactly once per acquired
+     * lease; calling it on a model whose binding has already been
+     * fully torn down is a no-op.
+     */
+    releaseDispatchLease(model) {
+        const binding = this.sessionRegistriesByModel.get(model);
+        if (!binding)
+            return;
+        binding.inFlight -= 1;
+        if (binding.inFlight < 0)
+            binding.inFlight = 0;
+        if (binding.pendingTeardown && binding.refCount <= 0 && binding.inFlight === 0 && binding.pendingPersists === 0) {
+            this.finalizeBindingTeardown(model);
+        }
+    }
+    /**
+     * Retain the binding for the duration of a post-commit persist.
+     *
+     * The responses endpoint starts `store.store(record)` synchronously
+     * inside `withExclusive` so the pending-writes tracker observes
+     * the in-flight write before the mutex releases, but does NOT
+     * await it on the critical path. The write still carries the
+     * binding's `modelInstanceId` (stamped into `configJson` by
+     * `buildResponseRecord`); without a retention, a same-model
+     * unregister + re-register completing while the write is in flight
+     * would delete the instance id and the re-registration would mint
+     * a fresh one, so the row — when it finally lands — would
+     * reference a dead id and the next continuation would be rejected
+     * with 400 instance-mismatch.
+     *
+     * The retention counter is CHECKED in every teardown gate
+     * (`dropNameReference`, `releaseDispatchLease`, `releaseBinding`).
+     * It is orthogonal to `inFlight` so the dispatch lease can release
+     * eagerly after `withExclusive` returns while the binding stays
+     * pinned long enough for the backgrounded `store.store(...)` to
+     * settle.
+     *
+     * Safe to call on a model whose binding has already been torn down
+     * (no-op). The matching `releaseBinding(model)` MUST still run in
+     * the persist's `.finally(...)` so the counter stays balanced.
+     */
+    retainBinding(model) {
+        const binding = this.sessionRegistriesByModel.get(model);
+        if (!binding)
+            return;
+        binding.pendingPersists += 1;
+    }
+    /**
+     * Balance a prior `retainBinding()` call. Decrements the persist
+     * retention counter and, if the binding has been flagged for
+     * teardown (refcount hit zero while the retention was held AND
+     * every dispatch lease has already released), drops it once the
+     * last retention releases. Safe to call exactly once per retain;
+     * calling it on a model whose binding has already been fully torn
+     * down is a no-op.
+     */
+    releaseBinding(model) {
+        const binding = this.sessionRegistriesByModel.get(model);
+        if (!binding)
+            return;
+        binding.pendingPersists -= 1;
+        if (binding.pendingPersists < 0)
+            binding.pendingPersists = 0;
+        if (binding.pendingTeardown && binding.refCount <= 0 && binding.inFlight === 0 && binding.pendingPersists === 0) {
+            this.finalizeBindingTeardown(model);
+        }
+    }
+    /**
+     * Tombstone installer, invoked exclusively by the responses
+     * endpoint's hard-timeout breaker (see
+     * `getPostCommitPersistHardTimeoutMs` in `endpoints/responses.ts`)
+     * when it force-releases the `retainBinding` on a wedged persist.
+     * Must be called BEFORE the idempotent
+     * `persistRetainBox.release?.()` so `instanceIds.get(model)`
+     * still returns the live id that the already-stamped record
+     * carries.
+     *
+     * Returns the retired id so the caller can capture it and scope
+     * the tombstone's lifetime to the specific wedged persist via
+     * `releaseTombstone(model)` inside that persist's `.finally(...)`.
+     * Returns `undefined` when the model has no current instance id
+     * assignment (caller raced the natural teardown path).
+     *
+     * Refcounted: each call increments a shared
+     * `{ instanceId, outstandingCount }` entry per model. Overlapping
+     * breakers share one slot (they all target the same numeric id
+     * because `register()` inherits the retired id whenever the
+     * tombstone is present), so memory stays O(1) per model.
+     */
+    retireInstanceIdForForceRelease(model) {
+        const id = this.instanceIds.get(model);
+        if (id === undefined)
+            return undefined;
+        const existing = this.retiredInstanceIds.get(model);
+        if (existing) {
+            existing.outstandingCount += 1;
+            return { instanceId: existing.instanceId };
+        }
+        this.retiredInstanceIds.set(model, { instanceId: id, outstandingCount: 1 });
+        return { instanceId: id };
+    }
+    /**
+     * Tombstone cleanup. Called from the post-commit persist's
+     * `.finally(...)` to balance exactly one prior
+     * `retireInstanceIdForForceRelease(model)` call. Decrements the
+     * shared refcount and drops the entry at zero so the next natural
+     * teardown mints a fresh id.
+     *
+     * Safe to call on a model whose tombstone has already been drained
+     * (no-op). The counter is clamped non-negative so spurious
+     * releases cannot underflow and re-enable inheritance.
+     */
+    releaseTombstone(model) {
+        const entry = this.retiredInstanceIds.get(model);
+        if (!entry)
+            return;
+        entry.outstandingCount -= 1;
+        if (entry.outstandingCount <= 0) {
+            this.retiredInstanceIds.delete(model);
+        }
+    }
+    /**
+     * Retrieve a model instance by name.
+     */
+    get(name) {
+        return this.models.get(name)?.model;
+    }
+    /**
+     * Retrieve the monotonic instance id for the model currently bound
+     * to `name`, or `undefined` if the name isn't registered.
+     *
+     * Two names that alias the same model object return the SAME id
+     * (they share a binding), and a name that has been hot-swapped to
+     * a different model object returns a DIFFERENT id than before the
+     * swap (the prior binding's id was dropped by `dropNameReference`
+     * and a fresh id was minted for the new model on re-registration).
+     *
+     * The responses endpoint uses this to key the
+     * `previous_response_id` cross-chain guard on instance identity
+     * instead of friendly name, so hot swaps are caught and safe
+     * aliases are not spuriously rejected.
+     */
+    getInstanceId(name) {
+        const entry = this.models.get(name);
+        if (!entry)
+            return undefined;
+        return this.instanceIds.get(entry.model);
+    }
+    /**
+     * Retrieve the session registry for a given model name, or
+     * `undefined` if the name is not registered.
+     *
+     * Every name that points at the same model instance returns the
+     * SAME `SessionRegistry` object. Two aliases `a` and `b` of one
+     * model therefore satisfy
+     * `registry.getSessionRegistry('a') === registry.getSessionRegistry('b')`,
+     * which is what the single-warm invariant requires: any turn
+     * through either alias advances the same cache's state, so a later
+     * lookup via either alias sees the current warm wrapper (if
+     * freshly adopted) or misses and cold-replays (if it was leased
+     * out by the other alias) — never a stale wrapper pointing at
+     * stomped native state.
+     */
+    getSessionRegistry(name) {
+        return this.models.get(name)?.sessionRegistry;
+    }
+    /**
+     * Iterate every DISTINCT session registry currently in use.
+     *
+     * Two aliases of the same model share one `SessionRegistry`, so
+     * naively walking every `ModelEntry` would yield duplicates. We
+     * walk the identity-keyed bindings instead so each registry
+     * appears exactly once, which is what the periodic `sweep()`
+     * scheduler in `server.ts` needs to avoid redundantly sweeping
+     * the same cache multiple times per tick.
+     */
+    listSessionRegistries() {
+        const out = [];
+        for (const binding of this.sessionRegistriesByModel.values()) {
+            out.push(binding.registry);
+        }
+        return out;
+    }
+    /**
+     * List all registered models in the OpenAI /v1/models format.
+     */
+    list() {
+        const result = [];
+        for (const entry of this.models.values()) {
+            result.push({
+                id: entry.id,
+                object: 'model',
+                created: entry.createdAt,
+                owned_by: 'mlx-node',
+            });
+        }
+        return result;
+    }
+    /**
+     * Check whether a model supports streaming.
+     *
+     * Every `SessionCapableModel` structurally exposes
+     * `chatStreamSessionStart`, so this is universally `true` for any
+     * properly-typed model registered through the session-capable
+     * interface. Kept as a belt-and-suspenders duck-type so a partially
+     * stubbed test double (pre-migration or intentionally non-streaming)
+     * can still opt out by omitting the method.
+     */
+    hasStreamSupport(model) {
+        const fn = model['chatStreamSessionStart'];
+        return typeof fn === 'function';
+    }
+}

package/dist/router.d.ts

@@ -0,0 +1,6 @@
+/** Path-based router for /v1/* endpoints. */
+import type { IncomingMessage, ServerResponse } from 'node:http';
+import type { ResponseStore } from '@mlx-node/core';
+import type { ModelRegistry } from './registry.js';
+export declare function routeRequest(req: IncomingMessage, res: ServerResponse, registry: ModelRegistry, store: ResponseStore | null, responseRetentionSec?: number): Promise<void>;
+//# sourceMappingURL=router.d.ts.map

package/dist/router.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"router.d.ts","sourceRoot":"","sources":["../src/router.ts"],"names":[],"mappings":"AAAA,6CAA6C;AAE7C,OAAO,KAAK,EAAE,eAAe,EAAE,cAAc,EAAE,MAAM,WAAW,CAAC;AAEjE,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,gBAAgB,CAAC;AAYpD,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,eAAe,CAAC;AAyBnD,wBAAsB,YAAY,CAChC,GAAG,EAAE,eAAe,EACpB,GAAG,EAAE,cAAc,EACnB,QAAQ,EAAE,aAAa,EACvB,KAAK,EAAE,aAAa,GAAG,IAAI,EAC3B,oBAAoB,CAAC,EAAE,MAAM,GAC5B,OAAO,CAAC,IAAI,CAAC,CA8Df"}

package/dist/router.js

@@ -0,0 +1,78 @@
+/** Path-based router for /v1/* endpoints. */
+import { handleCreateMessage } from './endpoints/messages.js';
+import { handleListModels } from './endpoints/models.js';
+import { handleCreateResponse } from './endpoints/responses.js';
+import { sendAnthropicBadRequest, sendAnthropicMethodNotAllowed, sendBadRequest, sendMethodNotAllowed, sendNotFound, } from './errors.js';
+/** Max request body size (10 MB). */
+const MAX_BODY_BYTES = 10 * 1024 * 1024;
+function readBody(req) {
+    return new Promise((resolve, reject) => {
+        const chunks = [];
+        let totalBytes = 0;
+        req.on('data', (chunk) => {
+            totalBytes += chunk.length;
+            if (totalBytes > MAX_BODY_BYTES) {
+                reject(new Error('Request body too large'));
+                req.destroy();
+                return;
+            }
+            chunks.push(chunk);
+        });
+        req.on('end', () => resolve(Buffer.concat(chunks).toString('utf-8')));
+        req.on('error', reject);
+    });
+}
+export async function routeRequest(req, res, registry, store, responseRetentionSec) {
+    const url = new URL(req.url ?? '/', `http://${req.headers.host ?? 'localhost'}`);
+    const path = url.pathname;
+    if (path === '/v1/models') {
+        if (req.method !== 'GET') {
+            sendMethodNotAllowed(res, 'GET');
+            return;
+        }
+        handleListModels(res, registry);
+        return;
+    }
+    if (path === '/v1/responses') {
+        if (req.method !== 'POST') {
+            sendMethodNotAllowed(res, 'POST');
+            return;
+        }
+        let body;
+        try {
+            const raw = await readBody(req);
+            body = JSON.parse(raw);
+        }
+        catch (err) {
+            const msg = err instanceof Error && err.message === 'Request body too large' ? err.message : 'Invalid JSON in request body';
+            sendBadRequest(res, msg);
+            return;
+        }
+        await handleCreateResponse(res, body, registry, store, req, responseRetentionSec);
+        return;
+    }
+    if (path === '/v1/messages') {
+        if (req.method !== 'POST') {
+            sendAnthropicMethodNotAllowed(res, 'POST');
+            return;
+        }
+        let body;
+        try {
+            const raw = await readBody(req);
+            body = JSON.parse(raw);
+        }
+        catch (err) {
+            const msg = err instanceof Error && err.message === 'Request body too large' ? err.message : 'Invalid JSON in request body';
+            sendAnthropicBadRequest(res, msg);
+            return;
+        }
+        await handleCreateMessage(res, body, registry, req);
+        return;
+    }
+    if (path === '/health' || path === '/v1/health') {
+        res.writeHead(200, { 'Content-Type': 'application/json' });
+        res.end(JSON.stringify({ status: 'ok' }));
+        return;
+    }
+    sendNotFound(res, `No route matches ${req.method} ${path}`);
+}

package/dist/server.d.ts

@@ -0,0 +1,80 @@
+/** Full HTTP server lifecycle: wires up the handler and periodically sweeps expired `ResponseStore` rows and sessions. */
+import type { Server } from 'node:http';
+import { ResponseStore } from '@mlx-node/core';
+import { ModelRegistry } from './registry.js';
+/**
+ * Parse a positive integer seconds value; returns undefined for unset/invalid so caller can apply its own default.
+ *
+ * Non-integer positive values (e.g. `"1.5"`) are rejected rather than
+ * silently truncated — a typo like `"1.5"` meant as `"15"` would otherwise
+ * be accepted as 1 second, expiring persisted response rows almost
+ * immediately and breaking `previous_response_id` continuity. We prefer
+ * falling through to the caller's default over crashing on startup so a
+ * config-template typo in a Dockerfile / CI manifest does not take the
+ * service down.
+ *
+ * Exported for unit tests.
+ */
+export declare function parseEnvSeconds(name: string): number | undefined;
+/**
+ * Parse a positive integer count from env; shares the reject-unset-or-invalid
+ * semantics used by {@link parseEnvSeconds} (including the non-integer
+ * reject) so callers can fall back to their own default when the var is
+ * missing or malformed.
+ *
+ * Exported for unit tests.
+ */
+export declare function parseEnvPositiveInt(name: string): number | undefined;
+export interface ServerConfig {
+    /** Port to listen on (default: 8080). */
+    port?: number;
+    /** Hostname to bind to (default: '127.0.0.1'). */
+    host?: string;
+    /** Path to the SQLite response store (default: ~/.mlx-node/responses.db). */
+    storePath?: string;
+    /** Disable response storage entirely (default: false). */
+    disableStore?: boolean;
+    /** Enable CORS headers (default: true). */
+    cors?: boolean;
+    /**
+     * Retention for persisted response rows, in seconds. Stamped as `expires_at`
+     * on each committed response; controls how long `previous_response_id`
+     * cold-replay from SQLite remains possible after the warm session is evicted.
+     *
+     * Default: 7 days. Env override: `MLX_RESPONSE_RETENTION_SECONDS`. Ignored
+     * when `disableStore` is true.
+     */
+    responseRetentionSec?: number;
+    /**
+     * Maximum number of concurrent requests that may be WAITING for the
+     * per-model execution mutex (the one actively running does not count).
+     * When the cap is reached, further requests return HTTP 429 with a
+     * `Retry-After: 1` header so clients can back off instead of piling
+     * into an unbounded queue.
+     *
+     * Default: `undefined` (unbounded — current behaviour). Env override:
+     * `MLX_MAX_QUEUE_DEPTH_PER_MODEL` (positive integer).
+     */
+    maxQueueDepthPerModel?: number;
+}
+export interface ServerInstance {
+    server: Server;
+    /** Register models before or after starting. */
+    registry: ModelRegistry;
+    /** Null when disabled. */
+    store: ResponseStore | null;
+    /** Graceful shutdown. */
+    close(): Promise<void>;
+}
+/**
+ * Start an MLX-Node HTTP server exposing `POST /v1/responses`,
+ * `POST /v1/messages`, and `GET /v1/models`.
+ *
+ * @example
+ * ```typescript
+ * const { registry, close } = await createServer({ port: 8080 });
+ * registry.register('qwen3.5-3b', await Qwen35Model.load('./models/qwen3.5-3b'));
+ * ```
+ */
+export declare function createServer(config?: ServerConfig): Promise<ServerInstance>;
+//# sourceMappingURL=server.d.ts.map

package/dist/server.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"server.d.ts","sourceRoot":"","sources":["../src/server.ts"],"names":[],"mappings":"AAAA,0HAA0H;AAI1H,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,WAAW,CAAC;AAIxC,OAAO,EAAE,aAAa,EAAE,MAAM,gBAAgB,CAAC;AAG/C,OAAO,EAAE,aAAa,EAAE,MAAM,eAAe,CAAC;AAgB9C;;;;;;;;;;;;GAYG;AACH,wBAAgB,eAAe,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,GAAG,SAAS,CAOhE;AAED;;;;;;;GAOG;AACH,wBAAgB,mBAAmB,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,GAAG,SAAS,CAOpE;AAuBD,MAAM,WAAW,YAAY;IAC3B,yCAAyC;IACzC,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,kDAAkD;IAClD,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,6EAA6E;IAC7E,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,0DAA0D;IAC1D,YAAY,CAAC,EAAE,OAAO,CAAC;IACvB,2CAA2C;IAC3C,IAAI,CAAC,EAAE,OAAO,CAAC;IACf;;;;;;;OAOG;IACH,oBAAoB,CAAC,EAAE,MAAM,CAAC;IAC9B;;;;;;;;;OASG;IACH,qBAAqB,CAAC,EAAE,MAAM,CAAC;CAChC;AAED,MAAM,WAAW,cAAc;IAC7B,MAAM,EAAE,MAAM,CAAC;IACf,gDAAgD;IAChD,QAAQ,EAAE,aAAa,CAAC;IACxB,0BAA0B;IAC1B,KAAK,EAAE,aAAa,GAAG,IAAI,CAAC;IAC5B,yBAAyB;IACzB,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;CACxB;AAED;;;;;;;;;GASG;AACH,wBAAsB,YAAY,CAAC,MAAM,CAAC,EAAE,YAAY,GAAG,OAAO,CAAC,cAAc,CAAC,CAoEjF"}