@bendyline/gezel-inference 0.1.0 → 1.0.1

package/dist/http/routes/v1-models.js

@@ -1,6 +1,8 @@
-import { externalGezelModelId, isRecommendedModel, resolveDistributionProfile, } from '@bendyline/gezel';
+import { estimateManifestKvBytes, externalGezelModelId, isRecommendedModel, localContextFloorTokens, rateModelsForDevice, resolveDistributionProfile, } from '@bendyline/gezel';
 import { resolveOnDeviceProvider } from '@bendyline/gezel/native';
 import { Hono } from 'hono';
+import { CapacityBroker } from '../../providers/native/capacity-broker.js';
+import { detectMemoryProfileCached } from '../../system/memory.js';
 import { appleSystemModel } from '../openai-compat/apple-model.js';
 import { resolveFallbackGezelId } from '../openai-compat/chat-target.js';
 /**
@@ -30,12 +32,35 @@ const PROVIDERS_TO_ENUMERATE = [
     'ds4',
 ];
 const LOCAL_PROVIDERS = ['llama-cpp', 'mlx', 'ds4'];
+/**
+ * Engines that enforce `response_format` with a grammar: llama-server's
+ * `json_schema`, and the MLX sidecar's llguidance JSON grammar
+ * (`providers/mlx/python/response_format.py`). ds4 shares llama.cpp's request
+ * map, but nothing here shows its server enforces a schema, so it is not
+ * advertised.
+ */
+function enforcesResponseFormat(provider) {
+    return provider === 'llama-cpp' || provider === 'mlx';
+}
+function catalogCapabilities(manifest, provider) {
+    return {
+        text: true,
+        tools: manifest.supportsTools,
+        structuredOutput: enforcesResponseFormat(provider),
+        images: false,
+        foregroundOnly: false,
+    };
+}
 function catalogBackend(manifest) {
+    // ds4 has no portable fallback: `discoverNativeBinaries` stamps its binary
+    // only where a build for this host is on disk, which is the gate the
+    // Settings availability check reads too. Windows has none.
+    const ds4 = manifest.ds4 && process.env.GEZEL_DS4_SERVER_BIN ? 'ds4' : null;
     // Store hosts have no provisioned Python runtime. Only advertise models
     // backed by bundled native engines until a frozen MLX runtime ships.
     const store = resolveDistributionProfile().pythonProvisioning === 'frozen-only';
     if (store)
-        return manifest.llamaCpp ? 'llama-cpp' : manifest.ds4 ? 'ds4' : null;
+        return manifest.llamaCpp ? 'llama-cpp' : ds4;
     const preferred = resolveOnDeviceProvider(process.platform, process.arch);
     if (preferred === 'mlx' && manifest.mlx && !manifest.mlx.disabledReason)
         return 'mlx';
@@ -45,19 +70,71 @@ function catalogBackend(manifest) {
         return 'llama-cpp';
     if (manifest.mlx && !manifest.mlx.disabledReason && process.platform === 'darwin')
         return 'mlx';
-    if (manifest.ds4)
-        return 'ds4';
-    return null;
+    return ds4;
+}
+/**
+ * What loading a catalog model would hold, priced as the capacity broker
+ * prices it: the catalog's measured pin when it has one, else the engine's
+ * estimate, plus KV at the context floor this machine launches with. ds4's
+ * figure already bounds its streamed working set, so it takes no extra KV.
+ */
+function footprintBytes({ manifest, provider, source }, memory) {
+    const mmprojBytes = provider === 'llama-cpp' ? manifest.llamaCpp?.mmproj?.sizeBytes : undefined;
+    const weights = source.residentBytes ??
+        CapacityBroker.estimateResidentBytes(provider, source.approxSizeBytes, mmprojBytes ? { mmprojBytes } : undefined);
+    if (provider === 'ds4')
+        return weights;
+    const ctxTokens = localContextFloorTokens({
+        totalRamBytes: memory.totalRamBytes,
+        gpuVramBytes: fastGpuBytes(memory),
+    });
+    // MLX keeps its cache in f16; llama.cpp's default q8_0 is the helper's default.
+    return (weights +
+        estimateManifestKvBytes(manifest, {
+            ctxTokens,
+            ...(provider === 'mlx' ? { cacheScale: 1 } : {}),
+        }));
+}
+/**
+ * An integrated adapter's advertised allocation is shared system RAM, not a
+ * fast pool beside it, so fit treats that host as RAM-paced (as Settings does).
+ */
+function fastGpuBytes(memory) {
+    return memory.gpuMemoryKind === 'integrated' ? null : memory.gpuVramBytes;
+}
+function recoDevice(memory) {
+    return {
+        platform: memory.platform,
+        gpuVramBytes: fastGpuBytes(memory),
+        gpuMemoryKind: memory.gpuMemoryKind,
+        totalRamBytes: memory.totalRamBytes,
+        usableBytes: memory.usableBytes,
+        budgetBytes: memory.budgetBytes,
+    };
 }
-async function downloadableCatalogEntries(ctx, created, existingIds) {
-    const items = await ctx.catalog.list('chat-model').catch(() => []);
-    return items
-        .flatMap((item) => {
+/** Curated order, for when this machine's memory could not be read. */
+function byCuratedScore(a, b) {
+    const aRecommended = isRecommendedModel(a.manifest);
+    const bRecommended = isRecommendedModel(b.manifest);
+    if (aRecommended !== bRecommended)
+        return aRecommended ? -1 : 1;
+    const score = (b.manifest.recoScore ?? 0) - (a.manifest.recoScore ?? 0);
+    return score || (a.entry.name ?? a.entry.id).localeCompare(b.entry.name ?? b.entry.id);
+}
+/**
+ * Downloadable models, each rated against this machine (`fit`,
+ * `recommendation`) and listed best first. A model too big to run here stays
+ * listed with `fit.tier: 'too-big'` and a 0 score, so a client decides
+ * whether to hide it.
+ */
+async function downloadableCatalogEntries(items, created, existingIds, memoryProfile) {
+    const candidates = items.flatMap((item) => {
         const manifest = item.manifest;
         if (manifest.kind !== 'chat-model')
             return [];
         const provider = catalogBackend(manifest);
-        if (!provider)
+        const source = provider === 'mlx' ? manifest.mlx : provider === 'ds4' ? manifest.ds4 : manifest.llamaCpp;
+        if (!provider || !source)
             return [];
         const id = `${provider}:${manifest.id}`;
         if (existingIds.has(id))
@@ -65,6 +142,8 @@ async function downloadableCatalogEntries(ctx, created, existingIds) {
         return [
             {
                 manifest,
+                provider,
+                source,
                 entry: {
                     id,
                     object: 'model',
@@ -78,26 +157,41 @@ async function downloadableCatalogEntries(ctx, created, existingIds) {
                     locality: 'on-device',
                     download_bytes: manifest.approxSizeBytes,
                     ...(manifest.contextWindow ? { context_window: manifest.contextWindow } : {}),
-                    capabilities: {
-                        text: true,
-                        tools: manifest.supportsTools,
-                        structuredOutput: false,
-                        images: false,
-                        foregroundOnly: false,
-                    },
+                    capabilities: catalogCapabilities(manifest, provider),
                 },
             },
         ];
-    })
-        .sort((a, b) => {
-        const aRecommended = isRecommendedModel(a.manifest);
-        const bRecommended = isRecommendedModel(b.manifest);
-        if (aRecommended !== bRecommended)
-            return aRecommended ? -1 : 1;
-        const score = (b.manifest.recoScore ?? 0) - (a.manifest.recoScore ?? 0);
-        return score || (a.entry.name ?? a.entry.id).localeCompare(b.entry.name ?? b.entry.id);
-    })
-        .map(({ entry }) => entry);
+    });
+    if (candidates.length === 0)
+        return [];
+    const memory = await memoryProfile().catch(() => null);
+    if (!memory)
+        return candidates.sort(byCuratedScore).map(({ entry }) => entry);
+    const rated = candidates.map((candidate) => {
+        const { entry, manifest, source } = candidate;
+        return {
+            id: entry.id,
+            residentBytes: footprintBytes(candidate, memory),
+            tags: manifest.tags,
+            supportsTools: manifest.supportsTools,
+            ...(manifest.recoScore !== undefined ? { recoScore: manifest.recoScore } : {}),
+            ...(manifest.licenseClass ? { licenseClass: manifest.licenseClass } : {}),
+            ...(source.quantization ? { quantization: source.quantization } : {}),
+        };
+    });
+    const entries = new Map(candidates.map(({ entry }) => [entry.id, entry]));
+    return rateModelsForDevice(rated, recoDevice(memory)).flatMap((rating) => {
+        const entry = entries.get(rating.id);
+        if (!entry)
+            return [];
+        return [
+            {
+                ...entry,
+                fit: { tier: rating.fit.tier, label: rating.fit.label, detail: rating.fit.detail },
+                recommendation: { score: rating.score, recommended: rating.recommended },
+            },
+        ];
+    });
 }
 async function buildModelEntries(ctx, created, options) {
     // Advertise every gezel as a selectable OpenAI "model". Keep the
@@ -128,6 +222,14 @@ async function buildModelEntries(ctx, created, options) {
             ...(gezel.id === fallbackGezelId ? { is_fallback: true } : {}),
         }));
     }
+    const catalogItems = await ctx.catalog.list('chat-model').catch(() => []);
+    // An installed model carries the capabilities its catalog entry
+    // advertised before download; a model with no entry stays unflagged
+    // rather than guessing its tools or images support.
+    const manifests = new Map();
+    for (const item of catalogItems)
+        if (item.manifest.kind === 'chat-model')
+            manifests.set(item.manifest.id, item.manifest);
     const providers = (options.localOnly ? LOCAL_PROVIDERS : PROVIDERS_TO_ENUMERATE).filter((provider) => provider !== 'mlx' || resolveDistributionProfile().pythonProvisioning !== 'frozen-only');
     const buckets = await Promise.all(providers.map(async (provider) => {
         try {
@@ -146,6 +248,9 @@ async function buildModelEntries(ctx, created, options) {
                     : {}),
                 ...(m.contextWindow ? { context_window: m.contextWindow } : {}),
                 ...(m.supportsReasoning ? { supports_reasoning: true } : {}),
+                ...(enforcesResponseFormat(provider) && manifests.has(m.id)
+                    ? { capabilities: catalogCapabilities(manifests.get(m.id), provider) }
+                    : {}),
             }));
         }
         catch {
@@ -156,7 +261,7 @@ async function buildModelEntries(ctx, created, options) {
     }));
     const availableEntries = buckets.flat();
     const existingIds = new Set([...gezelEntries, ...availableEntries].map((entry) => entry.id));
-    const downloadable = await downloadableCatalogEntries(ctx, created, existingIds);
+    const downloadable = await downloadableCatalogEntries(catalogItems, created, existingIds, options.memoryProfile ?? (() => detectMemoryProfileCached()));
     const apple = await appleSystemModel(created);
     return [...gezelEntries, ...apple, ...availableEntries, ...downloadable];
 }

package/dist/http/routes/v1-responses.js

@@ -3,10 +3,11 @@ import { Hono } from 'hono';
 import { bodyLimit } from 'hono/body-limit';
 import { streamSSE } from 'hono/streaming';
 import { ZodError } from 'zod';
+import { UnsupportedResponseFormatError } from '../../providers/mlx/runtime-diagnostics.js';
 import { ExternalToolsUnsupportedError, ModelNotInstalledError, } from '../../providers/types.js';
 import { appendCallerOwnedActionLedger } from '../openai-compat/action-ledger.js';
 import { profileForCallerOwnedInference } from '../openai-compat/caller-owned-profile.js';
-import { resolveChatTarget } from '../openai-compat/chat-target.js';
+import { resolveChatTarget, resolvePinnedModelTarget, } from '../openai-compat/chat-target.js';
 import { runResponsesNonStreaming, runResponsesStreaming, } from '../openai-compat/responses-streaming.js';
 import { ResponsesRequestSchema, translateResponsesRequest, } from '../openai-compat/responses-translate.js';
 import { flattenTranscriptIntoPrompt, parseGezelModelRef, resolveModelTarget, } from '../openai-compat/translate.js';
@@ -259,6 +260,9 @@ export function v1ResponsesRoutes(ctx, opts = {}) {
             if (err instanceof ExternalToolsUnsupportedError) {
                 return invalidRequest(c, err.message, 'tools_not_supported_for_provider');
             }
+            if (err instanceof UnsupportedResponseFormatError) {
+                return invalidRequest(c, err.message, err.code);
+            }
             const message = err instanceof Error ? err.message : String(err);
             if (message.includes('messages must') ||
                 message.includes('empty') ||
@@ -284,7 +288,10 @@ async function resolveResponsesTarget(model, ctx) {
             systemPrefix: '',
         };
     }
-    throw new Error(`Unknown model "${model}". Use an explicit <provider>:<model> id or advertised gezel:<role>-<name> alias.`);
+    const pinned = await resolvePinnedModelTarget(ctx, model);
+    if (pinned)
+        return pinned;
+    throw new Error(`Unknown model "${model}". Use an explicit <provider>:<model> id, a model pinned as a default, or an advertised gezel:<role>-<name> alias.`);
 }
 function prependSystemPrefix(input, prefix) {
     if (!prefix)

package/dist/in-flight-requests.d.ts

@@ -0,0 +1,19 @@
+/**
+ * The in-process transport's open requests, and the one handle that keeps Node
+ * running while any of them is open.
+ *
+ * The direct transport has no socket, so nothing a request awaits is
+ * guaranteed to hold the event loop: engine supervisors and workers unref their
+ * timers and threads so that an idle host never blocks exit. A plain Node
+ * script exited with code 13 ("unsettled top-level await") inside `chat()`
+ * while llama-server cold-started; Electron's own handles had hidden it.
+ */
+export declare class InFlightRequests {
+    private readonly open;
+    private keepAlive;
+    get size(): number;
+    add(controller: AbortController): void;
+    delete(controller: AbortController): void;
+    /** Abort and forget every open request, including a body nobody will read. */
+    abortAll(): void;
+}

package/dist/in-flight-requests.js

@@ -0,0 +1,34 @@
+/**
+ * The in-process transport's open requests, and the one handle that keeps Node
+ * running while any of them is open.
+ *
+ * The direct transport has no socket, so nothing a request awaits is
+ * guaranteed to hold the event loop: engine supervisors and workers unref their
+ * timers and threads so that an idle host never blocks exit. A plain Node
+ * script exited with code 13 ("unsettled top-level await") inside `chat()`
+ * while llama-server cold-started; Electron's own handles had hidden it.
+ */
+export class InFlightRequests {
+    open = new Set();
+    keepAlive;
+    get size() {
+        return this.open.size;
+    }
+    add(controller) {
+        this.open.add(controller);
+        this.keepAlive ??= setInterval(() => { }, 2 ** 30);
+    }
+    delete(controller) {
+        if (!this.open.delete(controller) || this.open.size > 0)
+            return;
+        clearInterval(this.keepAlive);
+        this.keepAlive = undefined;
+    }
+    /** Abort and forget every open request, including a body nobody will read. */
+    abortAll() {
+        for (const controller of [...this.open]) {
+            controller.abort();
+            this.delete(controller);
+        }
+    }
+}

package/dist/inference-service.js

@@ -18,6 +18,7 @@ import { v1KnowledgeRoutes } from './http/routes/v1-knowledge.js';
 import { v1ModelsEnsureRoutes } from './http/routes/v1-models-ensure.js';
 import { v1ModelsRoutes } from './http/routes/v1-models.js';
 import { v1ResponsesRoutes } from './http/routes/v1-responses.js';
+import { InFlightRequests } from './in-flight-requests.js';
 import { KnowledgeManager } from './knowledge/manager.js';
 import { createWorkerCatalogHost } from './knowledge/worker-host.js';
 import { embeddingsHealth, openEmbeddings, shutdownEmbeddings } from './memory/embeddings.js';
@@ -44,14 +45,13 @@ export async function startService(options = {}) {
     const runtimeDir = join(home, 'runtime');
     const lock = await acquireSingleInstanceLock({ runtimeDir, lockPath: join(runtimeDir, 'lock') });
     const cleanup = [() => lock.release()];
-    const requests = new Set();
+    const requests = new InFlightRequests();
     let stopping;
     let closed = false;
     const stop = () => {
         stopping ??= (async () => {
             closed = true;
-            for (const controller of requests)
-                controller.abort();
+            requests.abortAll();
             const errors = [];
             for (const close of cleanup.reverse())
                 try {

package/dist/providers/llama-cpp/build-provider.js

@@ -8,7 +8,7 @@ import { effectiveEngineRelease, isEnginePinned } from '../../engines/native-man
 import { engineInstallPhase } from '../../engines/registry.js';
 import { localDeviceSettingsLabel, noModelYetMessage, notReadyError, } from '../active-install-message.js';
 import { resolveCatalogLlamaCppEngineConfig, resolveCatalogReasoningBudget, } from '../catalog-model-config.js';
-import { CapacityDeniedError, NOT_ENOUGH_MEMORY_MESSAGE, availableSystemRamBytes, formatContextCapacityDenial, minViableLocalContextTokens, planAdaptiveContextGrowth, pressureIdleGraceMs, resolveLlamaCppContextRequirement, } from '../native/capacity-broker.js';
+import { CapacityDeniedError, NOT_ENOUGH_MEMORY_MESSAGE, autoKvCacheMaxBytes, availableSystemRamBytes, formatContextCapacityDenial, minViableLocalContextTokens, planAdaptiveContextGrowth, pressureIdleGraceMs, resolveLlamaCppContextRequirement, } from '../native/capacity-broker.js';
 import { measureAvailableGpuBytes } from '../native/device-capacity.js';
 import { engineApiKey, withEngineApiKey } from '../native/engine-api-key.js';
 import { makeEngineKey } from '../native/engine-key.js';
@@ -803,7 +803,13 @@ export async function buildLlamaCppProvider(opts) {
                         architecture: modelCatalogInfo?.architecture,
                         modelId: defaultModelId ?? undefined,
                     });
+                // A full-size cache past the automatic KV ceiling is declined even
+                // where it fits: gemma4-31b's is 55 GB at 64K, 11x its windowed
+                // cache, for a cross-request reuse gain nobody had measured.
+                const kvCeilingBytes = autoKvCacheMaxBytes();
+                const fullCacheOverCeiling = fullKvBytesAtRequested !== undefined && fullKvBytesAtRequested > kvCeilingBytes;
                 const fullKvOverBudget = fullCacheFitsFastMemory === false ||
+                    fullCacheOverCeiling ||
                     !admission.minimumSatisfied ||
                     admission.slots < slots ||
                     admission.clamped;
@@ -867,13 +873,32 @@ export async function buildLlamaCppProvider(opts) {
                         const fastFitDetail = fullCacheFitsFastMemory === false && fullKvBytesAtRequested !== undefined
                             ? `projected weights + full KV ${((residentBytes + fullKvBytesAtRequested) / 1024 ** 3).toFixed(1)} GiB exceeds available fast memory ${(Math.max(0, fastBudgetBytes - committedOtherBytes) / 1024 ** 3).toFixed(1)} GiB`
                             : undefined;
-                        log.warn([
-                            `[llama-cpp] ${modelCatalogInfo?.id ?? 'model'}: the full SWA working set at the requested `,
-                            `${effectiveNumCtx * slots}-token total context does not fit fast memory — keeping the full `,
-                            'context and declining the --swa-full auto-default instead; the windowed KV cache ',
-                            'is what actually fits (cross-request prefix reuse unavailable). ',
-                            `Fit detail: ${fastFitDetail ?? admission.reason ?? 'over budget at the requested slot count'}`,
-                        ].join(''));
+                        const overCeilingOnly = fullCacheOverCeiling &&
+                            fullCacheFitsFastMemory !== false &&
+                            admission.minimumSatisfied &&
+                            admission.slots >= slots &&
+                            !admission.clamped;
+                        if (overCeilingOnly && fullKvBytesAtRequested !== undefined) {
+                            // A policy choice on a machine that could hold the cache, not
+                            // a shortage: info, not warn.
+                            log.info([
+                                `[llama-cpp] ${modelCatalogInfo?.id ?? 'model'}: the full SWA cache at the requested `,
+                                `${effectiveNumCtx * slots}-token total context would take `,
+                                `${(fullKvBytesAtRequested / 1024 ** 3).toFixed(1)} GiB, past the `,
+                                `${(kvCeilingBytes / 1024 ** 3).toFixed(1)} GiB automatic KV ceiling — keeping the full `,
+                                'context and declining the --swa-full auto-default instead; the windowed KV cache ',
+                                'holds it (cross-request prefix reuse unavailable).',
+                            ].join(''));
+                        }
+                        else {
+                            log.warn([
+                                `[llama-cpp] ${modelCatalogInfo?.id ?? 'model'}: the full SWA working set at the requested `,
+                                `${effectiveNumCtx * slots}-token total context does not fit fast memory — keeping the full `,
+                                'context and declining the --swa-full auto-default instead; the windowed KV cache ',
+                                'is what actually fits (cross-request prefix reuse unavailable). ',
+                                `Fit detail: ${fastFitDetail ?? admission.reason ?? 'over budget at the requested slot count'}`,
+                            ].join(''));
+                        }
                     }
                     if (windowed) {
                         ladderPlan = planCtxTokensForMemory({

package/dist/providers/llama-cpp/cache-adapter.d.ts

@@ -149,6 +149,11 @@ export declare class LlamaCppCacheAdapter implements EngineCacheAdapter {
      * may have been pruned in between — re-seeding keeps it warm).
      */
     private readonly restoredSessions;
+    /**
+     * Sessions that will never be resumed (see `markEphemeral`): a save seeds
+     * their gezel prefix file when it is missing and writes nothing else.
+     */
+    private readonly ephemeralSessions;
     /**
      * Latched off when this llama-server refuses slot save/restore (501
      * "This feature is not supported by multimodal" — fires for mmproj-
@@ -190,6 +195,7 @@ export declare class LlamaCppCacheAdapter implements EngineCacheAdapter {
     prepareForSend(sessionId: string, systemPrompt?: string, layers?: SystemPromptLayers): Promise<void>;
     buildRequestExtras(sessionId: string): Record<string, unknown>;
     evict(sessionIds: readonly string[]): Promise<void>;
+    markEphemeral(sessionId: string): void;
     reportUsage(): Promise<readonly EngineCacheUsage[]>;
     warm(_sessionId: string, _messages: readonly CacheWarmMessage[]): Promise<void>;
     /**
@@ -227,6 +233,12 @@ export declare class LlamaCppCacheAdapter implements EngineCacheAdapter {
     private pruneDiskCache;
     private saveSlotForSession;
     private saveSlotForSessionUnlocked;
+    /**
+     * A one-shot session's only lasting value is the prompt prefix it shares
+     * with its gezel's next session. Save the slot straight to that prefix file
+     * when none exists yet; otherwise write nothing. Returns whether it wrote.
+     */
+    private seedPrefixFromEphemeralSlot;
     /** Try restore-by-session, then restore-by-prefix. Returns true if
      *  any restore succeeded. */
     private tryRestoreForSession;

package/dist/providers/llama-cpp/cache-adapter.js

@@ -149,6 +149,11 @@ export class LlamaCppCacheAdapter {
      * may have been pruned in between — re-seeding keeps it warm).
      */
     restoredSessions = new Set();
+    /**
+     * Sessions that will never be resumed (see `markEphemeral`): a save seeds
+     * their gezel prefix file when it is missing and writes nothing else.
+     */
+    ephemeralSessions = new Set();
     /**
      * Latched off when this llama-server refuses slot save/restore (501
      * "This feature is not supported by multimodal" — fires for mmproj-
@@ -263,8 +268,12 @@ export class LlamaCppCacheAdapter {
             this.sessionPrefix.delete(id);
             this.sessionLayerPrefixes.delete(id);
             this.restoredSessions.delete(id);
+            this.ephemeralSessions.delete(id);
         }
     }
+    markEphemeral(sessionId) {
+        this.ephemeralSessions.add(sessionId);
+    }
     async reportUsage() {
         const generation = this.engineGeneration;
         const baseUrl = await this.resolveBaseUrl().catch(() => null);
@@ -410,6 +419,7 @@ export class LlamaCppCacheAdapter {
                 log.debug(`slot ${chosen} recycled: evicting session ${victimSession.slice(0, 8)} for ${sessionId.slice(0, 8)} (disk save: ${saved ? 'ok' : 'MISSED'})`);
                 this.sessionToSlot.delete(victimSession);
                 this.slotToSession.delete(chosen);
+                this.ephemeralSessions.delete(victimSession);
             }
         }
         // We now have a slot id `chosen` that's free of any current binding.
@@ -534,6 +544,9 @@ export class LlamaCppCacheAdapter {
         const baseUrl = await this.resolveBaseUrl().catch(() => null);
         if (!baseUrl || generation !== this.engineGeneration)
             return false;
+        if (this.ephemeralSessions.has(sessionId)) {
+            return this.seedPrefixFromEphemeralSlot(baseUrl, slot, sessionId, generation);
+        }
         const filename = sessFilename(sessionId);
         const ok = await this.invokeSlotAction(baseUrl, slot, 'save', filename, generation);
         if (!ok)
@@ -571,6 +584,27 @@ export class LlamaCppCacheAdapter {
         }
         return true;
     }
+    /**
+     * A one-shot session's only lasting value is the prompt prefix it shares
+     * with its gezel's next session. Save the slot straight to that prefix file
+     * when none exists yet; otherwise write nothing. Returns whether it wrote.
+     */
+    async seedPrefixFromEphemeralSlot(baseUrl, slot, sessionId, generation) {
+        if (!this.slotSavePath)
+            return false;
+        const prefixId = this.sessionPrefix.get(sessionId);
+        if (!prefixId || this.seededPrefixes.has(prefixId))
+            return false;
+        const name = prefixFilename(prefixId);
+        if (await stat(join(this.slotSavePath, name)).catch(() => null)) {
+            this.seededPrefixes.add(prefixId);
+            return false;
+        }
+        const ok = await this.invokeSlotAction(baseUrl, slot, 'save', name, generation);
+        if (ok)
+            this.seededPrefixes.add(prefixId);
+        return ok;
+    }
     /** Try restore-by-session, then restore-by-prefix. Returns true if
      *  any restore succeeded. */
     async tryRestoreForSession(slot, sessionId) {

package/dist/providers/llama-cpp/gguf-metadata-async.js

@@ -122,6 +122,9 @@ async function runInWorker(path, options) {
     const id = nextId++;
     return new Promise((resolveSummary, reject) => {
         pending.set(id, { resolve: resolveSummary, reject });
+        // Held while busy: an unreferenced worker let a plain Node host exit
+        // mid-await during an engine cold start.
+        target.ref();
         target.postMessage({ id, path, ...(options ? { options } : {}) });
     });
 }
@@ -141,6 +144,8 @@ function ensureWorker() {
             else if (worker === next)
                 worker = null;
         });
+        // An idle reader must never hold the event loop open; runInWorker refs it
+        // only while a request is in flight.
         next.unref();
         worker = next;
         return next;
@@ -158,6 +163,8 @@ function onMessage(reply) {
     if (!request)
         return;
     pending.delete(reply.id);
+    if (pending.size === 0)
+        worker?.unref();
     if (reply.error)
         request.reject(new Error(reply.error));
     else if (reply.summary)

package/dist/providers/mlx/provider.js

@@ -58,9 +58,9 @@ import { MlxEngineGate } from './engine-gate.js';
 import { EngineHold, engineHoldLabelFor } from './engine-hold.js';
 import { EngineLogRouter } from './engine-log-router.js';
 import { MlxEngineStatsSampler } from './engine-stats-sampler.js';
-import { StreamingReasoningSplit, requestOpensReasoning, } from './reasoning-stream.js';
+import { REASONING_OPEN_HEADER, StreamingReasoningSplit, engineOpensReasoning, requestOpensReasoning, } from './reasoning-stream.js';
 import { applyMlxRequestShape } from './request-shape.js';
-import { PRE_FIRST_BYTE_BASE_MS, buildMidStreamDropMessage, buildPreFirstByteAbortMessage, computePreFirstByteBudgetMs, estimatePromptTokens, formatTps, isMidStreamConnectionDrop, translateMlxHttpError, } from './runtime-diagnostics.js';
+import { PRE_FIRST_BYTE_BASE_MS, buildMidStreamDropMessage, buildPreFirstByteAbortMessage, computePreFirstByteBudgetMs, estimatePromptTokens, formatTps, isMidStreamConnectionDrop, mlxHttpError, } from './runtime-diagnostics.js';
 import { classifyMlxFatalErrorLine, classifyMlxStartupLine, } from './stdout-parser.js';
 import { salvageTextualToolCalls } from './textual-tool-call-salvage.js';
 import { APPEND_TO_FILE_CONTINUATION_TOOL, MlxToolCallAccumulator, applyCallableToolGrammar, callableRequestToolNames, chatCompletionToolName, externalToolsAsChatCompletions, hermesRequiredArgGrammarRequested, missingTopLevelRequiredToolArgs, toChatCompletionsTools, validatorReportedMissingRequiredArgs, } from './tool-call-protocol.js';
@@ -1571,7 +1571,7 @@ class MlxSession extends StreamingSessionBase {
                     this.deps.markUsed();
                     const txt = await res.text().catch(() => '');
                     log.error(`turn#${seq}.${turn} HTTP-ERR status=${res.status} ${res.statusText}`);
-                    throw new Error(translateMlxHttpError(res.status, res.statusText, txt));
+                    throw mlxHttpError(res.status, res.statusText, txt);
                 }
                 if (debugOn) {
                     log.debug(`turn#${seq}.${turn} HTTP-OK status=${res.status} ` +
@@ -1616,10 +1616,14 @@ class MlxSession extends StreamingSessionBase {
                 // `turn.ramble-detection`; per-model thresholds come from
                 // the validated config. Absent → detector disabled.
                 //
-                // Read from the request, not only the template: a thinking-off
-                // request renders the block already closed, and seeding from the
-                // template streamed a whole `write_file` call into the reasoning pane.
-                const opensInReasoning = requestOpensReasoning(this.deps.templateOpensReasoning, body);
+                // The engine reports what the rendered prompt did; the template guess
+                // is only a fallback. A thinking-off request renders the block already
+                // closed (seeding from the template streamed a whole `write_file` call
+                // into the reasoning pane), and Qwen 3.5 0.8B's template opens it only
+                // when thinking is explicitly on (guessing closed streamed its thoughts
+                // and a stray `</think>` as the reply).
+                const opensInReasoning = engineOpensReasoning(res.headers.get(REASONING_OPEN_HEADER)) ??
+                    requestOpensReasoning(this.deps.templateOpensReasoning, body);
                 const rambleConfig = profileBehaviorConfig(this.deps.profile, 'turn.ramble-detection');
                 const ramble = outputIsConstrained(this.deps.tuning)
                     ? inertRambleDetector()

package/dist/providers/mlx/python/batch_processors.py

@@ -0,0 +1,26 @@
+"""batch_processors — per-sequence logits-processor lists for mlx_lm's
+BatchGenerator that stay aligned across waves.
+
+mlx_lm 0.31.3's `GenerationBatch.filter()` trims `logits_processors` only
+`if any(self.logits_processors)`. Once every live sequence carries `[]`, the
+finished ones' entries survive the filter, and the next wave's sequences are
+appended after them at the wrong index: a request's tool grammar, think
+budget or response format then ran for its first token only, and in a mixed
+wave one sequence's grammar could land on another's output. Caught
+2026-10-13: a JSON-schema request right after a plain one came back as free
+text. Giving every sequence a non-empty list keeps `any()` true, so the
+filter always trims. The pass-through costs nothing measurable (A/B on
+gemma4-e4b, 2026-10-14: 64.3 vs 62.7 tok/s median, within noise).
+"""
+
+from typing import Any, Callable, List, Sequence
+
+
+def pass_through_logits(tokens: Any, logits: Any) -> Any:
+    """A no-op processor that keeps a sequence's processor list non-empty."""
+    return logits
+
+
+def per_sequence_processors(processors: Sequence[Callable[[Any, Any], Any]]) -> List[Callable[[Any, Any], Any]]:
+    """The processor list to hand BatchGenerator for one sequence; never empty."""
+    return list(processors) or [pass_through_logits]