@bendyline/gezel-inference 0.0.0-stage → 1.0.0

package/dist/providers/mlx/config-parser.d.ts

@@ -0,0 +1,29 @@
+/**
+ * MLX model metadata parsing.
+ *
+ * Unlike llama.cpp's single-file GGUF format, MLX models are a directory
+ * of HuggingFace-standard files:
+ *   - `config.json`            architecture, hidden sizes, contextWindow
+ *   - `tokenizer_config.json`  chat_template (when present)
+ *   - `tokenizer.json`         raw tokenizer
+ *   - `model.safetensors`(+.index.json when sharded)  the weights
+ *
+ * This helper extracts the metadata the rest of the provider needs —
+ * context window (to set `mlx_lm.server --max-tokens` / pass-through in
+ * the API call) and chat-template presence (surfaced as a UI warning
+ * same as llama.cpp).
+ */
+export interface MlxModelSummary {
+    /** Detected context window from config.json, or null when not findable. */
+    contextWindow: number | null;
+    /** Architecture family (`llama`, `gemma`, `qwen2`, …) — surfaced in the UI. */
+    architecture: string | null;
+    /** True when `tokenizer_config.json` has a non-empty `chat_template` field. */
+    chatTemplatePresent: boolean;
+}
+/**
+ * Read the metadata files in an MLX model directory. Returns conservative
+ * defaults when individual files are missing or malformed — we never raise
+ * because a weird HF repo shouldn't render the model unusable.
+ */
+export declare function readMlxSummary(modelDir: string): Promise<MlxModelSummary>;

package/dist/providers/mlx/config-parser.js

@@ -0,0 +1,81 @@
+/**
+ * MLX model metadata parsing.
+ *
+ * Unlike llama.cpp's single-file GGUF format, MLX models are a directory
+ * of HuggingFace-standard files:
+ *   - `config.json`            architecture, hidden sizes, contextWindow
+ *   - `tokenizer_config.json`  chat_template (when present)
+ *   - `tokenizer.json`         raw tokenizer
+ *   - `model.safetensors`(+.index.json when sharded)  the weights
+ *
+ * This helper extracts the metadata the rest of the provider needs —
+ * context window (to set `mlx_lm.server --max-tokens` / pass-through in
+ * the API call) and chat-template presence (surfaced as a UI warning
+ * same as llama.cpp).
+ */
+import { readFile } from 'node:fs/promises';
+import { join } from 'node:path';
+/**
+ * Read the metadata files in an MLX model directory. Returns conservative
+ * defaults when individual files are missing or malformed — we never raise
+ * because a weird HF repo shouldn't render the model unusable.
+ */
+export async function readMlxSummary(modelDir) {
+    const config = await readJsonSafe(join(modelDir, 'config.json'));
+    const tokenizerConfig = await readJsonSafe(join(modelDir, 'tokenizer_config.json'));
+    return {
+        contextWindow: extractContextWindow(config),
+        architecture: extractArchitecture(config),
+        chatTemplatePresent: extractChatTemplate(tokenizerConfig),
+    };
+}
+async function readJsonSafe(path) {
+    try {
+        const raw = await readFile(path, 'utf8');
+        return JSON.parse(raw);
+    }
+    catch {
+        return null;
+    }
+}
+/**
+ * HuggingFace configs are inconsistent about which field carries the
+ * context window. Check the half-dozen common spellings in order of
+ * precedence; return the first positive integer found.
+ */
+function extractContextWindow(config) {
+    if (!config)
+        return null;
+    const candidates = [
+        'max_position_embeddings',
+        'n_positions',
+        'n_ctx',
+        'max_sequence_length',
+        'seq_length',
+        'model_max_length',
+    ];
+    for (const key of candidates) {
+        const v = config[key];
+        if (typeof v === 'number' && v > 0 && Number.isFinite(v))
+            return Math.floor(v);
+    }
+    return null;
+}
+function extractArchitecture(config) {
+    if (!config)
+        return null;
+    const arch = config.architectures;
+    if (Array.isArray(arch) && arch.length > 0 && typeof arch[0] === 'string') {
+        return arch[0];
+    }
+    const modelType = config.model_type;
+    if (typeof modelType === 'string' && modelType.length > 0)
+        return modelType;
+    return null;
+}
+function extractChatTemplate(tokenizerConfig) {
+    if (!tokenizerConfig)
+        return false;
+    const tpl = tokenizerConfig.chat_template;
+    return typeof tpl === 'string' && tpl.length > 0;
+}

package/dist/providers/mlx/engine-gate.d.ts

@@ -0,0 +1,51 @@
+/** Who is waiting on an engine request — see `SendAndWaitOpts.queue.enginePriority`. */
+export type EngineRequestPriority = 'interactive' | 'background';
+/**
+ * Width-N gate over actual engine requests. The width is the engine's batch
+ * capability (`batchMaxConcurrency`). At width 1 this is strict FIFO with one
+ * HTTP request in flight, while the sidecar still uses its snapshot-capable
+ * BatchGenerator internally. A wider gate lets up to N requests reach the
+ * sidecar for one static batched wave.
+ *
+ * Interactive requests (a person is waiting) are handed slots before
+ * background ones, and get ONE slot beyond the width when every request in
+ * flight is background. The sidecar parks a wave of background work for a
+ * waiting interactive request (python/wave_policy.py), but only a request
+ * that reached it can be served that way: with every slot held by task
+ * steps, a person's message used to wait here, invisible to the engine,
+ * until a whole task turn finished. The overflow never adds a concurrent
+ * generation — the parked wave is suspended while the interactive one runs —
+ * and the sidecar still refuses to park when live memory is tight.
+ */
+export declare class MlxEngineGate {
+    readonly width: number;
+    private normal;
+    private overflow;
+    private interactiveHeld;
+    private readonly waiters;
+    /** Exclusive claims ({@link acquireAll}) in progress; they close the overflow slot. */
+    private exclusivePending;
+    private overflowDrained;
+    constructor(width: number);
+    /** True while a physical request is being served or is queued for a slot. */
+    get busy(): boolean;
+    private slotFor;
+    private take;
+    /** Grant every waiter the rules now allow, interactive first, FIFO within a priority. */
+    private drain;
+    private enqueue;
+    acquire(label: string, signal?: AbortSignal, onWait?: (info: {
+        aheadOf: number;
+    }) => void, priority?: EngineRequestPriority): Promise<() => void>;
+    /**
+     * Every slot at once, for work that must not overlap any request — an
+     * engine reload. The N claims are queued back to back in one synchronous
+     * pass, so interactive requests already waiting are served first and
+     * nothing queued later can slip in between two of them. A reload is made
+     * for a person's request (an image to look at), so its claims rank as
+     * interactive: background work already waiting goes after it. The
+     * overflow slot is closed for the duration and drained before this
+     * resolves.
+     */
+    acquireAll(label: string): Promise<() => void>;
+}

package/dist/providers/mlx/engine-gate.js

@@ -0,0 +1,207 @@
+import { createLogger } from '@bendyline/gezel';
+import { QUEUE_WAIT_NOTICE_DELAY_MS, QUEUE_WAIT_NOTICE_REPEAT_MS } from '../queue.js';
+const log = createLogger('mlx');
+/**
+ * Width-N gate over actual engine requests. The width is the engine's batch
+ * capability (`batchMaxConcurrency`). At width 1 this is strict FIFO with one
+ * HTTP request in flight, while the sidecar still uses its snapshot-capable
+ * BatchGenerator internally. A wider gate lets up to N requests reach the
+ * sidecar for one static batched wave.
+ *
+ * Interactive requests (a person is waiting) are handed slots before
+ * background ones, and get ONE slot beyond the width when every request in
+ * flight is background. The sidecar parks a wave of background work for a
+ * waiting interactive request (python/wave_policy.py), but only a request
+ * that reached it can be served that way: with every slot held by task
+ * steps, a person's message used to wait here, invisible to the engine,
+ * until a whole task turn finished. The overflow never adds a concurrent
+ * generation — the parked wave is suspended while the interactive one runs —
+ * and the sidecar still refuses to park when live memory is tight.
+ */
+export class MlxEngineGate {
+    width;
+    normal = 0;
+    overflow = 0;
+    interactiveHeld = 0;
+    waiters = [];
+    /** Exclusive claims ({@link acquireAll}) in progress; they close the overflow slot. */
+    exclusivePending = 0;
+    overflowDrained = [];
+    constructor(width) {
+        this.width = width;
+    }
+    /** True while a physical request is being served or is queued for a slot. */
+    get busy() {
+        return this.normal + this.overflow > 0 || this.waiters.length > 0;
+    }
+    slotFor(priority) {
+        if (this.normal < this.width)
+            return 'normal';
+        if (priority === 'interactive' &&
+            this.overflow === 0 &&
+            this.interactiveHeld === 0 &&
+            this.exclusivePending === 0) {
+            return 'overflow';
+        }
+        return null;
+    }
+    take(slot, priority) {
+        if (slot === 'normal')
+            this.normal++;
+        else
+            this.overflow++;
+        if (priority === 'interactive')
+            this.interactiveHeld++;
+    }
+    /** Grant every waiter the rules now allow, interactive first, FIFO within a priority. */
+    drain() {
+        for (let i = 0; i < this.waiters.length;) {
+            const waiter = this.waiters[i];
+            const slot = this.slotFor(waiter.priority);
+            if (!slot) {
+                i++;
+                continue;
+            }
+            this.waiters.splice(i, 1);
+            if (waiter.signal && waiter.onAbort) {
+                waiter.signal.removeEventListener('abort', waiter.onAbort);
+            }
+            this.take(slot, waiter.priority);
+            waiter.resolve(slot);
+        }
+    }
+    enqueue(waiter) {
+        if (waiter.priority === 'background') {
+            this.waiters.push(waiter);
+            return;
+        }
+        const firstBackground = this.waiters.findIndex((w) => w.priority === 'background');
+        if (firstBackground === -1)
+            this.waiters.push(waiter);
+        else
+            this.waiters.splice(firstBackground, 0, waiter);
+    }
+    async acquire(label, signal, onWait, priority = 'interactive') {
+        const waitStartedAt = Date.now();
+        let slot = this.waiters.length === 0 ? this.slotFor(priority) : null;
+        // Waiters ahead in priority order keep their place: a newcomer may only
+        // skip the line through the overflow slot, which no waiter ahead of it
+        // could have used.
+        if (!slot &&
+            priority === 'interactive' &&
+            !this.waiters.some((w) => w.priority === 'interactive')) {
+            const overflow = this.slotFor('interactive');
+            if (overflow === 'overflow')
+                slot = overflow;
+        }
+        if (slot) {
+            this.take(slot, priority);
+        }
+        else {
+            let waitNotice = null;
+            let waitNoticeDelay = null;
+            try {
+                slot = await new Promise((resolve, reject) => {
+                    const waiter = {
+                        priority,
+                        resolve,
+                        reject,
+                        ...(signal ? { signal } : {}),
+                    };
+                    if (signal) {
+                        waiter.onAbort = () => {
+                            const idx = this.waiters.indexOf(waiter);
+                            if (idx === -1)
+                                return;
+                            this.waiters.splice(idx, 1);
+                            signal.removeEventListener('abort', waiter.onAbort);
+                            reject(new DOMException(`MLX engine request ${label} aborted`, 'AbortError'));
+                        };
+                        signal.addEventListener('abort', waiter.onAbort, { once: true });
+                    }
+                    this.enqueue(waiter);
+                    // Announce the park. Same reasoning as the llama.cpp gate: a turn
+                    // that already cleared the ProviderQueue can still wait here for
+                    // the length of another session's round-trip, and a wait with no
+                    // signal reads to the user as a wedged model.
+                    if (onWait) {
+                        const publish = () => {
+                            const idx = this.waiters.indexOf(waiter);
+                            if (idx === -1)
+                                return;
+                            onWait({ aheadOf: this.normal + this.overflow + idx });
+                        };
+                        waitNoticeDelay = setTimeout(() => {
+                            publish();
+                            waitNotice = setInterval(publish, QUEUE_WAIT_NOTICE_REPEAT_MS);
+                            waitNotice.unref?.();
+                        }, QUEUE_WAIT_NOTICE_DELAY_MS);
+                        waitNoticeDelay.unref?.();
+                    }
+                });
+            }
+            finally {
+                if (waitNoticeDelay)
+                    clearTimeout(waitNoticeDelay);
+                if (waitNotice)
+                    clearInterval(waitNotice);
+            }
+        }
+        const waitedMs = Date.now() - waitStartedAt;
+        if (waitedMs > 1_000) {
+            log.debug(`engine request ${label} waited ${waitedMs}ms for an MLX engine slot`);
+        }
+        if (slot === 'overflow') {
+            log.info(`engine request ${label} took the interactive overflow slot — every slot was held by background work`);
+        }
+        const held = slot;
+        let released = false;
+        return () => {
+            if (released)
+                return;
+            released = true;
+            if (held === 'normal')
+                this.normal--;
+            else
+                this.overflow--;
+            if (priority === 'interactive')
+                this.interactiveHeld--;
+            if (held === 'overflow' && this.overflow === 0) {
+                const drained = this.overflowDrained;
+                this.overflowDrained = [];
+                for (const resolve of drained)
+                    resolve();
+            }
+            this.drain();
+        };
+    }
+    /**
+     * Every slot at once, for work that must not overlap any request — an
+     * engine reload. The N claims are queued back to back in one synchronous
+     * pass, so interactive requests already waiting are served first and
+     * nothing queued later can slip in between two of them. A reload is made
+     * for a person's request (an image to look at), so its claims rank as
+     * interactive: background work already waiting goes after it. The
+     * overflow slot is closed for the duration and drained before this
+     * resolves.
+     */
+    async acquireAll(label) {
+        this.exclusivePending++;
+        let releases;
+        try {
+            releases = await Promise.all(Array.from({ length: this.width }, (_, i) => this.acquire(`${label}:${i + 1}/${this.width}`, undefined, undefined, 'interactive')));
+            if (this.overflow > 0) {
+                await new Promise((resolve) => this.overflowDrained.push(resolve));
+            }
+        }
+        catch (err) {
+            this.exclusivePending--;
+            throw err;
+        }
+        return () => {
+            this.exclusivePending--;
+            for (const release of releases)
+                release();
+        };
+    }
+}

package/dist/providers/mlx/engine-hold.d.ts

@@ -0,0 +1,41 @@
+/**
+ * Engine-side holds. The sidecar runs requests in waves and prints a
+ * `waiting` marker for every request it is holding behind other work (and
+ * `paused` for a background wave that stepped aside for a chat), from its
+ * worker loop, every few seconds. Each one is proof the engine is alive and
+ * busy, so the session re-arms whichever idle bound is live: a turn waiting
+ * its turn is not a stall, however long the work ahead of it takes. Measured
+ * before this: a 300-token question was labelled "Processing prompt" behind a
+ * 55k-token re-prefill and aborted at 407s by the pre-first-byte bound without
+ * ever starting. The turn's own deadline still bounds the whole wait.
+ */
+import type { EnginePhaseEvent } from '../streaming-session.js';
+export type EngineQueueMarker = NonNullable<EnginePhaseEvent['engineQueue']>;
+/**
+ * The pill's words for a request the engine is holding. Plain language: the
+ * person needs to know their turn has not started and why, not which engine
+ * phase it is in.
+ */
+export declare function engineHoldLabelFor(queue: EngineQueueMarker): string;
+/**
+ * One session's hold state before its first byte. While held, {@link label}
+ * wins over the prefill heartbeat's label, which would otherwise keep calling
+ * a queued turn "Processing prompt" — the label a person read for seven
+ * minutes before the watchdog killed a turn that had never started.
+ */
+export declare class EngineHold {
+    private heldLabel;
+    private aheadOf;
+    private noticeTimer;
+    get label(): string | null;
+    /**
+     * Mark the request held and publish the queue notice. The chat bubble's
+     * "model queue" state expires a few seconds after the last notice, and one
+     * engine step can outlast that on a long prefill, so the notice is
+     * re-asserted on a timer until {@link end}.
+     */
+    begin(queue: EngineQueueMarker, onQueueWait?: (info: {
+        aheadOf: number;
+    }) => void): string;
+    end(): void;
+}

package/dist/providers/mlx/engine-hold.js

@@ -0,0 +1,59 @@
+/**
+ * How often a turn the engine is holding re-asserts the chat's "model queue"
+ * notice. Inside the UI's freshness window (12s) so the notice holds steady
+ * even when one engine step outlasts the sidecar's own marker cadence.
+ */
+const ENGINE_HOLD_NOTICE_REPEAT_MS = 5_000;
+function othersAhead(queue) {
+    return Math.max(1, queue.behind?.length ?? 1) + (queue.ahead ?? 0);
+}
+/**
+ * The pill's words for a request the engine is holding. Plain language: the
+ * person needs to know their turn has not started and why, not which engine
+ * phase it is in.
+ */
+export function engineHoldLabelFor(queue) {
+    if (queue.state === 'paused')
+        return 'Paused while a chat goes first';
+    const others = othersAhead(queue);
+    return others === 1
+        ? 'Waiting for another chat to finish'
+        : `Waiting for ${others} other chats to finish`;
+}
+/**
+ * One session's hold state before its first byte. While held, {@link label}
+ * wins over the prefill heartbeat's label, which would otherwise keep calling
+ * a queued turn "Processing prompt" — the label a person read for seven
+ * minutes before the watchdog killed a turn that had never started.
+ */
+export class EngineHold {
+    heldLabel = null;
+    aheadOf = 0;
+    noticeTimer = null;
+    get label() {
+        return this.heldLabel;
+    }
+    /**
+     * Mark the request held and publish the queue notice. The chat bubble's
+     * "model queue" state expires a few seconds after the last notice, and one
+     * engine step can outlast that on a long prefill, so the notice is
+     * re-asserted on a timer until {@link end}.
+     */
+    begin(queue, onQueueWait) {
+        this.heldLabel = engineHoldLabelFor(queue);
+        this.aheadOf = othersAhead(queue);
+        const publish = () => onQueueWait?.({ aheadOf: this.aheadOf });
+        publish();
+        if (!this.noticeTimer) {
+            this.noticeTimer = setInterval(publish, ENGINE_HOLD_NOTICE_REPEAT_MS);
+            this.noticeTimer.unref?.();
+        }
+        return this.heldLabel;
+    }
+    end() {
+        this.heldLabel = null;
+        if (this.noticeTimer)
+            clearInterval(this.noticeTimer);
+        this.noticeTimer = null;
+    }
+}

package/dist/providers/mlx/engine-log-router.d.ts

@@ -0,0 +1,84 @@
+import type { EnginePhaseEvent } from '../streaming-session.js';
+import { type MlxFatalError } from './stdout-parser.js';
+/**
+ * Turns the supervised engine's stdout into per-session phase events, and
+ * decides when a line means the engine is broken.
+ *
+ * Extracted from `MlxProvider` because it is one concern with its own state
+ * machine and its own hard-won rules, and because a supervised engine has ONE
+ * stdout shared by every session on it — so "who hears this line" is a real
+ * question, not an implementation detail of the provider.
+ */
+export interface EngineLogRouterDeps {
+    /**
+     * Friendly model name for the long weight-load window, so the pill reads
+     * "Loading model — Qwen 3.5" rather than a bare "Loading weights". Read
+     * lazily: the catalog may resolve it after the router is built.
+     */
+    modelDisplayName?: () => string | undefined;
+    /** Hand a classified phase to whoever should hear it. */
+    deliver: (phase: EnginePhaseEvent) => void;
+    /**
+     * The engine broke before it ever served. Fires once per fatal (first-seen
+     * wins); the caller tears the child down so the next `ensureRunning()`
+     * respawns fresh.
+     */
+    onFatal: (fatal: MlxFatalError) => void;
+    /** A `ready` phase survived the repeat filter. */
+    onReady: () => void;
+}
+export declare class EngineLogRouter {
+    private readonly deps;
+    /** Last phase delivered, for the repeat filter. Cleared by `ready`. */
+    private lastStartupPhase;
+    /**
+     * Latest fatal error seen before the engine served. The server's httpd keeps
+     * accepting connections after `_generate` dies, so `/health` cannot be
+     * trusted; while this is non-null a send short-circuits instead of hanging
+     * on an unresponsive completions endpoint. Cleared on `starting`.
+     */
+    private fatalError;
+    /**
+     * Whether the engine has reported itself ready and is therefore serving.
+     * Gates fatal handling: the classifier reads any `SomeError: …` leaf line as
+     * engine death, which is right during startup and wrong afterwards. A
+     * caught, contained Python exception prints the same shape as a fatal one,
+     * and killing the engine for it takes down every *other* session sharing it
+     * — a spec-decode wave that failed one request, and answered that request,
+     * SIGKILLed a 27B mid-turn and dropped an unrelated session with it.
+     */
+    private serving;
+    /**
+     * Most recent error line seen while serving. Not fatal — kept so a turn that
+     * dies on a dropped stream can name the real cause instead of "something
+     * closed the HTTP request."
+     */
+    private lastRuntimeError;
+    constructor(deps: EngineLogRouterDeps);
+    /** One raw line from the supervised child. */
+    onLine(line: string): void;
+    /**
+     * Last delivered phase, replayed to a session that registers mid-run so a
+     * chat opened during a long weight load still shows what the engine is
+     * doing. Null once `ready` clears it — there is nothing to catch up on.
+     */
+    lastPhase(): EnginePhaseEvent | null;
+    /**
+     * Consume the startup fatal, if one was seen. Consumes on read: the caller
+     * still throws it, so the *current* send fails fast, but a later send gets a
+     * fresh shot at respawning — otherwise a fix made out-of-process
+     * (reinstalled deps, a different model, a venv reset) would not take effect
+     * until gezeld itself restarted.
+     */
+    takeFatalError(): MlxFatalError | null;
+    /** Consume the last serving-time error, if one was seen. */
+    takeRuntimeError(): MlxFatalError | null;
+    /**
+     * An error from an engine that is up and serving. Deliberately does NOT tear
+     * the engine down: the sidecar prints these from handlers that already
+     * contained the failure and answered the affected request, so the engine is
+     * still correct for everyone else on it.
+     */
+    private noteRuntimeError;
+    private handleFatalError;
+}