@bendyline/gezel-inference 1.0.2 → 1.0.4

package/dist/providers/image/types.d.ts

@@ -0,0 +1,235 @@
+import type { ImageAspectRatio, ImageModelAuxiliaryRole, ImageQuality } from '@bendyline/gezel';
+export type { ImageModelAuxiliaryRole } from '@bendyline/gezel';
+/**
+ * ImageProvider — the service-side abstraction for local image generation.
+ *
+ * Mirrors the role `LLMProvider` plays for text. Deliberately simpler:
+ * image gen is one-shot rather than session-based, so there is no
+ * `createSession`/`sendAndWait` stream — just a single `generate` call
+ * that returns a PNG buffer plus metadata.
+ *
+ * Implementations today:
+ *   - `MockImageProvider` — deterministic 1x1 PNG for tests and
+ *     GEZEL_MOCK_PROVIDER=1 flows.
+ *   - `StableDiffusionCppProvider` — talks to a `sd-server` binary over HTTP.
+ */
+/**
+ * One source image for editing / reference-driven generation, after
+ * the route has resolved any artifact paths to bytes. Provider sees
+ * a uniform `{ data: Buffer; mimeType }` shape.
+ */
+export interface ImageInputBytes {
+    data: Buffer;
+    mimeType: string;
+}
+/**
+ * Where a local generation is. sd-server loads weights lazily inside the
+ * first request after a launch, so `loading_model` happens mid-request, and
+ * its tiled VAE decode prints a progress bar of its own after sampling.
+ */
+export type ImageGenerationPhase = 'queued' | 'loading_model' | 'sampling' | 'decoding';
+/**
+ * Progress info reported mid-generate by providers that can observe
+ * per-step advancement. Today only the local sd-cpp provider emits
+ * these (parsed from sd-server's `|==> | 3/20 - 18.20s/it` stdout).
+ * Cloud providers stay silent — their HTTP APIs are one-shot, with
+ * no streaming step counter to surface.
+ *
+ * `step`/`totalSteps` count within the phase: tensors loaded while
+ * `loading_model`, sampling steps while `sampling`, tiles while `decoding`.
+ * Both are 0 while `queued`. A consumer that only draws a sampling bar
+ * reads events whose `phase` is `sampling` or absent.
+ */
+export interface ImageGenerationProgress {
+    /** Unit the engine just completed in this phase (1-indexed). */
+    step: number;
+    /** Units in this phase. */
+    totalSteps: number;
+    /** Wall-clock seconds for the most recent step, when reported. */
+    secondsPerStep?: number;
+    /** Absent from providers that only report sampling. */
+    phase?: ImageGenerationPhase;
+}
+export interface ImageGenerationInput {
+    prompt: string;
+    negativePrompt?: string;
+    /** Manifest id of the model to use. Provider falls back to its default. */
+    model?: string;
+    width?: number;
+    height?: number;
+    /**
+     * The shape and size tier the caller asked for. Local engines render the
+     * pixels the route already resolved from these; cloud providers map them
+     * to their own aspect ratios and size tiers, which explicit pixels
+     * override.
+     */
+    aspectRatio?: ImageAspectRatio;
+    quality?: ImageQuality;
+    steps?: number;
+    seed?: number;
+    /**
+     * Source images for image editing / reference-driven generation.
+     * Cloud providers accept multiple; sd-cpp's img2img path uses the
+     * first only (the others are dropped with a warning).
+     */
+    inputImages?: ImageInputBytes[];
+    /** Img2img strength (0 = keep source, 1 = ignore source). sd-cpp only. */
+    strength?: number;
+    /**
+     * Optional callback fired as the engine advances through sampling
+     * steps. Today only the local sd-cpp provider emits these; cloud
+     * providers ignore the field. Errors thrown from the callback are
+     * swallowed so a buggy listener can't take down a generation.
+     */
+    onProgress?: (progress: ImageGenerationProgress) => void;
+    /**
+     * Cancels the generation. A queued request leaves without touching the
+     * engine. sd-server cannot interrupt a render it has started, and keeps
+     * rendering after its client disconnects, so the local provider stops the
+     * engine process instead; the next generation pays a fresh weight load.
+     */
+    signal?: AbortSignal;
+}
+/** The caller's signal aborted the generation. */
+export declare class ImageGenerationCancelledError extends Error {
+    readonly code = "image_generation_cancelled";
+    constructor();
+}
+/**
+ * The engine could not fit the render into graphics memory. sd-server plans
+ * placement at launch and can misjudge free memory held by another engine;
+ * it then fails at the first sampling step with a bare "no results".
+ */
+export declare class ImageCapacityError extends Error {
+    readonly code = "image_capacity";
+    constructor(detail?: string);
+}
+export interface ImageGenerationMeta {
+    model: string;
+    seed: number;
+    steps: number;
+    widthPx: number;
+    heightPx: number;
+    durationMs: number;
+    /**
+     * Set when `inputImages` were supplied but the resolved model does not
+     * support img2img — the provider dropped the sources and generated from
+     * the prompt alone. Mirrors the core wire schema so the route can
+     * forward it untouched.
+     */
+    img2imgSkippedReason?: string;
+}
+export interface ImageGenerationOutput {
+    png: Buffer;
+    meta: ImageGenerationMeta;
+}
+export interface InstalledImageModelInfo {
+    id: string;
+    name: string;
+    approxSizeBytes: number;
+    installedAt: string;
+    /** Loading shape recorded at pull time; absent on pre-capability installs. */
+    weightsKind?: 'checkpoint' | 'diffusion-model';
+    /** Explicit catalog capability recorded at pull time; absent → resolve via the ladder. */
+    supportsImg2Img?: boolean;
+}
+export type ImageModelPullEvent = {
+    type: 'progress';
+    bytesWritten: number;
+    totalBytes?: number;
+}
+/**
+ * Surfaced when the shared `downloadWithRetry` helper hit a transient
+ * network error and is about to try again. The UI can render
+ * "Connection lost — retrying in 4s (attempt 3/5)…" instead of a
+ * fatal error banner. `reason` is already a user-friendly sentence;
+ * don't wrap it.
+ */
+ | {
+    type: 'retrying';
+    attempt: number;
+    maxAttempts: number;
+    delayMs: number;
+    reason: string;
+} | {
+    type: 'error';
+    error: string;
+} | {
+    type: 'done';
+    id: string;
+};
+export interface ImageModelAuxiliaryPullSpec {
+    role: ImageModelAuxiliaryRole;
+    downloadUrl: string;
+    sha256: string;
+    approxSizeBytes: number;
+}
+export interface ImageModelPullSpec {
+    downloadUrl: string;
+    sha256: string;
+    approxSizeBytes: number;
+    /** Human-readable name copied into the local metadata. */
+    name: string;
+    /**
+     * `checkpoint` — single-file (SD 1.x / SDXL / SD3.5 safetensors). The
+     * supervisor passes `--model <weights>` at sd-server launch.
+     * `diffusion-model` — unet-only (FLUX / SD3 GGUF); supervisor passes
+     * `--diffusion-model <weights>` plus one `--<role>` flag per aux file.
+     */
+    weightsKind: 'checkpoint' | 'diffusion-model';
+    /**
+     * Explicit img2img capability from the catalog manifest, persisted
+     * into the installed metadata so local state stays self-describing
+     * even if the catalog entry later disappears. Absent when the catalog
+     * didn't declare it — resolution falls back to the assessment map +
+     * weights-kind default.
+     */
+    supportsImg2Img?: boolean;
+    /** Auxiliary files (VAE, text encoders) downloaded alongside the unet. */
+    auxiliaryFiles: ImageModelAuxiliaryPullSpec[];
+}
+export interface ImageEngineHealth {
+    /**
+     * `ok` — the engine is reachable and ready to accept generation requests.
+     * `unreachable` — the configured engine URL is not responding (no
+     *   process listening, network error, etc).
+     * `not-configured` — no engine binary or URL is wired up; the provider
+     *   would fall back to a default loopback URL that nothing is going to
+     *   answer. UI surfaces this distinctly so the user gets actionable
+     *   guidance rather than a generic "unreachable" state.
+     */
+    status: 'ok' | 'unreachable' | 'not-configured';
+    /** Resolved base URL the provider is pointed at, for diagnostic display. */
+    baseUrl: string;
+    /** Free-form reason when `status !== 'ok'`. */
+    error?: string;
+}
+export interface ImageProvider {
+    readonly name: string;
+    generate(input: ImageGenerationInput): Promise<ImageGenerationOutput>;
+    listInstalledModels(): Promise<InstalledImageModelInfo[]>;
+    /**
+     * The model a generate that names none will use. Callers that look up
+     * per-model defaults (the route's recommended steps) must ask this rather
+     * than guess, or the defaults of one model are applied to another.
+     * Providers without a local default omit it.
+     */
+    resolveDefaultModelId?(): Promise<string | undefined>;
+    /**
+     * Stream the install of one catalog model. When `signal` is supplied
+     * and aborts mid-download, providers should leave any `.partial` file
+     * in place so a follow-up pull resumes via Range requests — disconnect
+     * ≠ delete. Cloud providers ignore the field.
+     */
+    pullModel(id: string, spec: ImageModelPullSpec, signal?: AbortSignal): AsyncIterable<ImageModelPullEvent>;
+    deleteModel(id: string): Promise<void>;
+    /**
+     * Probe the underlying engine for reachability without doing real
+     * work. Implementations should be fast (sub-second) and never throw
+     * — return `unreachable` / `not-configured` instead. Called by the
+     * Settings UI to render an honest readiness pill.
+     */
+    health(): Promise<ImageEngineHealth>;
+    /** Best-effort shutdown of any backing engine. Called when the service stops. */
+    shutdown?(): Promise<void>;
+}

package/dist/providers/image/types.js

@@ -0,0 +1,20 @@
+/** The caller's signal aborted the generation. */
+export class ImageGenerationCancelledError extends Error {
+    code = 'image_generation_cancelled';
+    constructor() {
+        super('Image generation was cancelled');
+        this.name = 'ImageGenerationCancelledError';
+    }
+}
+/**
+ * The engine could not fit the render into graphics memory. sd-server plans
+ * placement at launch and can misjudge free memory held by another engine;
+ * it then fails at the first sampling step with a bare "no results".
+ */
+export class ImageCapacityError extends Error {
+    code = 'image_capacity';
+    constructor(detail) {
+        super(`Not enough graphics memory to render this image${detail ? ` (${detail})` : ''}. Close other models or try a smaller size.`);
+        this.name = 'ImageCapacityError';
+    }
+}

package/dist/providers/llama-cpp/backend-health.d.ts

@@ -11,6 +11,13 @@
  */
 import type { LlamaCppBackendHealthResponse } from '@bendyline/gezel';
 import type { GpuUnused } from './gpu-startup-watch.js';
+type MissingRuntimeState = NonNullable<LlamaCppBackendHealthResponse['missingRuntime']>;
+/**
+ * Windows: whether this PC can load the runtime every engine imports, read from
+ * the first engine binary discovery found. One is enough: they all link the
+ * same runtime.
+ */
+export declare function readMissingRuntime(now?: number, platform?: NodeJS.Platform): MissingRuntimeState | null;
 /** Record what the latest start of a build found; null clears it. */
 export declare function noteGpuStartup(verdict: GpuUnused | null, now?: Date): void;
 export declare function noteHostOutOfMemory(input: {

package/dist/providers/llama-cpp/backend-health.js

@@ -9,7 +9,44 @@
  * Process-wide like the backend env it sits beside: every llama provider in
  * this process launches the same build.
  */
-import { readLlamaCudaBlocker } from '@bendyline/gezel/native';
+import { missingMachineRuntime, readLlamaCudaBlocker, vcRedistributableUrl, } from '@bendyline/gezel/native';
+/**
+ * Reading an engine's import tables takes tens of milliseconds of synchronous
+ * file reads, and the answer only changes when someone installs the runtime.
+ */
+const RUNTIME_CHECK_TTL_MS = 60_000;
+let runtimeCheck = null;
+/**
+ * Windows: whether this PC can load the runtime every engine imports, read from
+ * the first engine binary discovery found. One is enough: they all link the
+ * same runtime.
+ */
+export function readMissingRuntime(now = Date.now(), platform = process.platform) {
+    if (platform !== 'win32')
+        return null;
+    const binary = [
+        process.env.GEZEL_LLAMA_SERVER_BIN,
+        process.env.GEZEL_SD_SERVER_BIN,
+        process.env.GEZEL_WHISPER_SERVER_BIN,
+    ].find((path) => Boolean(path));
+    if (!binary)
+        return null;
+    if (runtimeCheck?.binary === binary && now - runtimeCheck.at < RUNTIME_CHECK_TTL_MS) {
+        return runtimeCheck.value;
+    }
+    const failure = missingMachineRuntime(binary, process.env);
+    const arch = failure?.arch === 'arm64' || failure?.arch === 'x64' ? failure.arch : null;
+    const value = failure && arch
+        ? {
+            arch,
+            dll: failure.dll ?? 'VCRUNTIME140.dll',
+            downloadUrl: vcRedistributableUrl(arch),
+            detail: `${binary}: ${failure.reason}`,
+        }
+        : null;
+    runtimeCheck = { binary, at: now, value };
+    return value;
+}
 let gpuUnused = null;
 let outOfMemory = null;
 /** Record what the latest start of a build found; null clears it. */
@@ -51,14 +88,17 @@ export function llamaBackendHealthView() {
     // A verdict about a build the process has since moved off (a quarantine
     // fallback, a lifted quarantine) no longer describes what runs.
     const unused = gpuUnused && gpuUnused.backend === running ? gpuUnused : null;
+    const missingRuntime = readMissingRuntime();
     return {
         ...(running ? { running } : {}),
         ...(cudaBlocker ? { cudaBlocker } : {}),
         ...(unused ? { gpuUnused: unused } : {}),
+        ...(missingRuntime ? { missingRuntime } : {}),
         ...(outOfMemory ? { outOfMemory } : {}),
     };
 }
 export function __resetLlamaBackendHealthForTest() {
     gpuUnused = null;
     outOfMemory = null;
+    runtimeCheck = null;
 }

package/dist/providers/llama-cpp/build-provider.js

@@ -24,7 +24,7 @@ import { readGgufSummaryAsync } from './gguf-metadata-async.js';
 import { effectiveLazyTensorBytes } from './gguf-metadata.js';
 import { GpuStartupWatch } from './gpu-startup-watch.js';
 import { isGemmaModel, planLlamaCppKv, resolveLlamaCppKvCacheType, } from './kv-cache-type.js';
-import { degradeDenseFfnOffloadDecision, degradeMoeOffloadDecision, estimateExactPerSlotKvBytesF16, estimateKvReserveBytes, estimateWindowedKvLinearization, planDenseFfnOffload, planMoeOffload, } from './offload-planner.js';
+import { degradeDenseFfnOffloadDecision, degradeMoeOffloadDecision, estimateExactPerSlotKvBytesF16, estimateKvReserveBytes, estimateWindowedKvLinearization, planDenseFfnOffload, planMoeOffload, shedSlotsForGpuResidency, } from './offload-planner.js';
 import { LlamaCppProvider } from './provider.js';
 import { reasoningLaunchOverridesFromEnv } from './reasoning-launch.js';
 import { resolveSpecDraft } from './spec-draft.js';
@@ -347,13 +347,6 @@ export async function buildLlamaCppProvider(opts) {
     // path still clamps to the native train context.
     const explicitCtx = envNumCtx ?? perModelCtxOverride ?? config.llamaCppNumCtx;
     const hostFloorTokens = minViableLocalContextTokens();
-    // A per-model override below the host floor is deliberate user intent —
-    // lower the admission floor to the override instead of silently raising
-    // the request back to 64K. Env / machine-wide values keep their
-    // historical floor semantics.
-    const minViableTokens = perModelCtxOverride !== undefined && explicitCtx === perModelCtxOverride
-        ? Math.min(hostFloorTokens, perModelCtxOverride)
-        : hostFloorTokens;
     const contextRequirement = resolveLlamaCppContextRequirement({
         modelContextWindow: modelCatalogInfo?.contextWindow,
         ...(explicitCtx !== undefined ? { explicitContextWindow: explicitCtx } : {}),
@@ -362,7 +355,7 @@ export async function buildLlamaCppProvider(opts) {
             ? { manifestContextSize: manifestEngineConfig.contextSize }
             : {}),
         contextSizing: config.llamaCppContextSizing ?? 'adaptive',
-        minViableContextTokens: minViableTokens,
+        minViableContextTokens: hostFloorTokens,
     });
     // `let`: RAM-aware admission below may lower this (but never below the
     // model-aware minimum) before anything launch-visible consumes it.
@@ -650,6 +643,10 @@ export async function buildLlamaCppProvider(opts) {
     // mode. Undefined when the GGUF is unreadable — the pool builder then
     // falls back to the legacy weights-multiplier reservation.
     let plannedReservationBytes;
+    // What prices that reservation, kept for the out-of-memory ladder: when it
+    // drops a slot, the relaunch is charged for the KV it actually allocates.
+    let reservationKvFor;
+    let reservationResidentBytes;
     // Graphics memory the MoE offload plan puts on the card. Device admission
     // charges this rather than the whole card; undefined keeps the old charge.
     let plannedGpuBytes;
@@ -979,9 +976,9 @@ export async function buildLlamaCppProvider(opts) {
                     ssmConvKernel: summary.ssmConvKernel,
                     kvCacheType,
                 });
-            const kvReserveBytes = windowedKvPlan
-                ? Math.round(windowedKvPlan.fixedBytes * slots +
-                    windowedKvPlan.bytesPerToken * effectiveNumCtx * slots)
+            const kvReserveFor = (slotCount) => windowedKvPlan
+                ? Math.round(windowedKvPlan.fixedBytes * slotCount +
+                    windowedKvPlan.bytesPerToken * effectiveNumCtx * slotCount)
                 : estimateKvReserveBytes({
                     blockCount: summary.blockCount,
                     embeddingLength: summary.embeddingLength,
@@ -999,43 +996,47 @@ export async function buildLlamaCppProvider(opts) {
                     ssmInnerSize: summary.ssmInnerSize,
                     ssmStateSize: summary.ssmStateSize,
                     ssmConvKernel: summary.ssmConvKernel,
-                    ctxTokens: effectiveNumCtx * slots,
+                    ctxTokens: effectiveNumCtx * slotCount,
                     kvCacheType,
                 });
-            // The broker-ledger reservation for this launch: resident weights
-            // plus the KV the engine will allocate at the granted window (M1 —
-            // the weights-only ledger under-reserved dense models whose KV
-            // rivals their weights: qwen3.5-4b at 64K carries more KV than
-            // parameters). Consumed by the pool builder via
-            // `plannedReservationBytes` so co-residency admission can see it.
-            if (kvReserveBytes !== undefined) {
-                plannedReservationBytes = residentBytes + kvReserveBytes;
-            }
+            // Plan against the card as it is now. Sized to the whole card, a plan
+            // counts memory other programs hold: MoE admission then charged the
+            // model the whole card and refused it (2026-10-11), and a dense
+            // 31B planned into 30.3 GiB of a 32 GB card that the desktop and its
+            // apps were already holding ~5 GB of died with a Vulkan out-of-memory
+            // (2026-10-19).
+            const freeGpuBytes = vramBytes > 0 ? await measureAvailableGpuBytes().catch(() => undefined) : undefined;
+            // A lane count the person chose is not the planner's to spend, and
+            // eval runs under GEZEL_LLAMA_NUM_CTX accept the memory consequences.
+            const mayShedSlots = configuredSlots === undefined && envNumCtx === undefined;
+            const placeSlots = (gpuBytes, plan) => mayShedSlots
+                ? shedSlotsForGpuResidency({ slots, vramBytes: gpuBytes, plan })
+                : { slots, decision: plan(slots, gpuBytes), shed: false };
+            let placement;
             if (isMoE) {
-                // Plan against the card as it is now. Sized to the whole card, the
-                // plan counted memory other programs held, and admission then had
-                // to charge the model the whole card and refuse it (2026-10-11).
-                const freeGpuBytes = vramBytes > 0 ? await measureAvailableGpuBytes().catch(() => undefined) : undefined;
-                const moePlan = planMoeOffload({
-                    isMoE,
-                    residentBytes,
-                    vramBytes: freeGpuBytes !== undefined ? Math.min(vramBytes, freeGpuBytes) : vramBytes,
-                    ...(split
-                        ? {
-                            split,
-                            ...(summary.blockCount !== undefined ? { blockCount: summary.blockCount } : {}),
-                            ...(kvReserveBytes !== undefined ? { kvReserveBytes } : {}),
-                        }
-                        : {}),
+                placement = placeSlots(freeGpuBytes !== undefined ? Math.min(vramBytes, freeGpuBytes) : vramBytes, (slotCount, gpuBytes) => {
+                    const kvReserveBytes = kvReserveFor(slotCount);
+                    return planMoeOffload({
+                        isMoE,
+                        residentBytes,
+                        vramBytes: gpuBytes,
+                        ...(split
+                            ? {
+                                split,
+                                ...(summary.blockCount !== undefined ? { blockCount: summary.blockCount } : {}),
+                                ...(kvReserveBytes !== undefined ? { kvReserveBytes } : {}),
+                            }
+                            : {}),
+                    });
                 });
-                offloadDecision = moePlan;
+                offloadDecision = placement.decision;
                 // An explicit offload setting shadows the planner's flags, so its
                 // footprint would describe a launch that is not the one made.
                 const plannerPlacesExperts = (config.llamaCppCpuMoe ?? resolvedManifestEngineConfig?.cpuMoe) === undefined &&
                     (config.llamaCppNCpuMoe ?? resolvedManifestEngineConfig?.nCpuMoe) === undefined &&
                     (config.llamaCppNGpuLayers ?? resolvedManifestEngineConfig?.nGpuLayers) === undefined;
                 if (plannerPlacesExperts)
-                    plannedGpuBytes = moePlan.gpuBytes;
+                    plannedGpuBytes = placement.decision.gpuBytes;
             }
             else {
                 const explicitNCpuFfn = config.llamaCppNCpuFfn ?? resolvedManifestEngineConfig?.nCpuFfn;
@@ -1044,16 +1045,21 @@ export async function buildLlamaCppProvider(opts) {
                 // A finite whole-layer GPU count is a different placement strategy;
                 // don't layer an automatic tensor override on top of it. `-1` means
                 // all and is compatible with the planner's own attention-on-GPU plan.
-                offloadDecision =
-                    explicitNCpuFfn === undefined &&
-                        (explicitNGpuLayers === undefined || explicitNGpuLayers === -1)
-                        ? planDenseFfnOffload({
+                if (explicitNCpuFfn === undefined &&
+                    (explicitNGpuLayers === undefined || explicitNGpuLayers === -1)) {
+                    placement = placeSlots(
+                    // Respect the usable accelerator pool, any explicit capacity
+                    // ceiling, co-resident reservations, and what other programs
+                    // hold on the card now. The raw probe still gates this to a real
+                    // discrete-GPU build rather than treating a CPU/unified fast pool
+                    // as VRAM.
+                    vramBytes > 0
+                        ? Math.max(0, Math.min(budgetBytes - committedOtherBytes, freeGpuBytes ?? Number.POSITIVE_INFINITY))
+                        : 0, (slotCount, gpuBytes) => {
+                        const kvReserveBytes = kvReserveFor(slotCount);
+                        return planDenseFfnOffload({
                             isMoE,
-                            // Respect the usable accelerator pool, any explicit capacity
-                            // ceiling, and co-resident reservations. The raw probe still
-                            // gates this to a real discrete-GPU build rather than treating
-                            // a CPU/unified fast pool as VRAM.
-                            vramBytes: vramBytes > 0 ? Math.max(0, budgetBytes - committedOtherBytes) : 0,
+                            vramBytes: gpuBytes,
                             ...(denseFfnSplit ? { split: denseFfnSplit } : {}),
                             ...(summary.blockCount !== undefined ? { blockCount: summary.blockCount } : {}),
                             ...(kvReserveBytes !== undefined ? { kvReserveBytes } : {}),
@@ -1062,9 +1068,30 @@ export async function buildLlamaCppProvider(opts) {
                                 ? brokerSnap.pools.ramShareBytes
                                 : hostCapacity.ramShareBytes,
                             freeSystemRamBytes: availableSystemRamBytes(),
-                        })
-                        : undefined;
+                        });
+                    });
+                    offloadDecision = placement.decision;
+                }
+                else {
+                    offloadDecision = undefined;
+                }
+            }
+            if (placement?.shed) {
+                log.info(`[llama-cpp] ${modelCatalogInfo?.id ?? 'model'}: reducing engine slots ${slots} -> 1 to keep the model on the GPU with room to spare (other programs' graphics memory moves)`);
+                slots = 1;
+            }
+            // The broker-ledger reservation for this launch: resident weights
+            // plus the KV the engine will allocate at the granted window (M1 —
+            // the weights-only ledger under-reserved dense models whose KV
+            // rivals their weights: qwen3.5-4b at 64K carries more KV than
+            // parameters). Consumed by the pool builder via
+            // `plannedReservationBytes` so co-residency admission can see it.
+            const kvReserveBytes = kvReserveFor(slots);
+            if (kvReserveBytes !== undefined) {
+                plannedReservationBytes = residentBytes + kvReserveBytes;
             }
+            reservationKvFor = kvReserveFor;
+            reservationResidentBytes = residentBytes;
             if (offloadDecision?.reason) {
                 log.info(`[llama-cpp] offload plan (${modelCatalogInfo?.id ?? 'model'}): ${offloadDecision.reason}`);
             }
@@ -1228,16 +1255,34 @@ export async function buildLlamaCppProvider(opts) {
             gpuWatch?.observe(line);
             providerHolder.current?.onStdoutLine(line);
         },
-        // GPU-OOM recovery ladder: when a start dies of VRAM exhaustion and the
-        // planner's offload decision was in play, degrade it one step
-        // (partial split → all expert/FFN weights to RAM → engine-owned fit) and let the
-        // supervisor retry — `resolveLaunch` below re-reads `offloadDecision`
-        // on every spawn. The degraded plan sticks for later restarts of this
-        // provider, so a recovered engine doesn't re-OOM on its next boot.
+        // GPU-OOM recovery ladder: when a start dies of VRAM exhaustion, first
+        // give back the extra slots' KV (every weight stays on the GPU), then, if
+        // the planner's offload decision was in play, degrade it one step
+        // (partial split → all expert/FFN weights to RAM → engine-owned fit) and
+        // let the supervisor retry — `resolveLaunch` below re-reads `slots` and
+        // `offloadDecision` on every spawn. The degraded plan sticks for later
+        // restarts of this provider, so a recovered engine doesn't re-OOM on its
+        // next boot.
         recoverStartup: ({ panicKind }) => {
             if (panicKind !== 'cuda-out-of-memory' && panicKind !== 'vulkan-out-of-memory') {
                 return false;
             }
+            // A full-residency plan has no offload rung below it, so before this a
+            // two-slot launch that missed by one KV allocation failed outright
+            // (2026-10-19, Gemma 4 31B on a 32 GB card). This rung spends one of
+            // the supervisor's two retries; an offload rung still has the other.
+            if (slots > 1 && configuredSlots === undefined) {
+                const from = slots;
+                if (providerHolder.current?.narrowEngineSlots(1)) {
+                    slots = 1;
+                    const kvReserveBytes = reservationKvFor?.(1);
+                    if (kvReserveBytes !== undefined && reservationResidentBytes !== undefined) {
+                        plannedReservationBytes = reservationResidentBytes + kvReserveBytes;
+                    }
+                    log.warn(`[llama-cpp] ${panicKind} while starting ${modelCatalogInfo?.id ?? 'model'} — retrying with one engine slot instead of ${from}`);
+                    return true;
+                }
+            }
             // Explicit config/manifest offload settings shadow the planner
             // per-field (see `buildLlamaCppEngineArgs`); when every field the
             // ladder would change is pinned, a retry replays the same argv.
@@ -1274,6 +1319,12 @@ export async function buildLlamaCppProvider(opts) {
         },
         resolveLaunch: async () => {
             gpuWatch = new GpuStartupWatch(binaryBackend);
+            // Again at every launch, not only at build: llama.cpp from the 0.1.51
+            // release refuses a --slot-save-path that does not exist ("not a
+            // directory") and exits before ready, and an eval preflight lost its
+            // build-time directory during a 50 s capacity wait (2026-10-18). The
+            // person saw "no model is installed yet".
+            await mkdirAsync(slotSavePath, { recursive: true }).catch(() => { });
             const port = cachedPort ?? (await pickFreePort());
             cachedPort = port;
             // Prepend the binary's own directory to PATH so the OS finds the

package/dist/providers/llama-cpp/cache-adapter.d.ts

@@ -123,7 +123,7 @@ export declare class LlamaCppCacheAdapter implements EngineCacheAdapter {
      */
     private allocationTail;
     private engineGeneration;
-    private readonly slotCount;
+    private slotCount;
     private readonly resolveBaseUrl;
     private readonly resolveAuthToken;
     private readonly fetchImpl;
@@ -168,6 +168,12 @@ export declare class LlamaCppCacheAdapter implements EngineCacheAdapter {
      */
     private slotActionsUnsupported;
     constructor(opts: LlamaCppCacheAdapterOptions);
+    /**
+     * Follow an engine relaunched with a different `--parallel` count. Every
+     * binding goes with it: a session pinned to a slot the new engine lacks
+     * would send an `id_slot` the server rejects.
+     */
+    setSlotCount(slotCount: number): void;
     /** The provider survives engine restarts, but its resident slot contents do not. */
     resetEngineState(): void;
     /**

package/dist/providers/llama-cpp/cache-adapter.js

@@ -181,6 +181,15 @@ export class LlamaCppCacheAdapter {
             throw new Error('Invalid llama disk cache budget');
         this.diskCacheBudgetBytes = budget * 1024 * 1024;
     }
+    /**
+     * Follow an engine relaunched with a different `--parallel` count. Every
+     * binding goes with it: a session pinned to a slot the new engine lacks
+     * would send an `id_slot` the server rejects.
+     */
+    setSlotCount(slotCount) {
+        this.slotCount = Math.max(1, Math.floor(slotCount));
+        this.resetEngineState();
+    }
     /** The provider survives engine restarts, but its resident slot contents do not. */
     resetEngineState() {
         this.engineGeneration++;

package/dist/providers/llama-cpp/devices.d.ts

@@ -37,6 +37,8 @@ export interface LlamaDeviceProbe {
      * means the cached devices describe a machine that no longer exists.
      */
     systemRamBytes?: number;
+    /** {@link llamaBinaryFingerprint} of the build that answered. */
+    binary?: string;
 }
 export interface NvidiaRuntimeDevice {
     index: number;
@@ -75,11 +77,20 @@ export declare function maxGpuVramBytes(devices: LlamaDevice[]): number;
  * (e.g. "AMD Radeon RX 7900 XTX") in a tier rationale. Pure — unit-tested.
  */
 export declare function pickBestGpuDevice(devices: LlamaDevice[]): LlamaDevice | null;
+/**
+ * Which build a cached listing came from: its path (the backend) and its
+ * size and mtime (the release, since a native update rewrites the files).
+ * A listing taken from the CPU build, or from the build an update replaced,
+ * said "no GPU" about a 32 GB Radeon and was served for the rest of the day
+ * (2026-10-19).
+ */
+export declare function llamaBinaryFingerprint(binaryPath: string): string;
 /**
  * Probe device memory by running `<binaryPath> --list-devices`, caching
  * the result. Reads the cache when present (VRAM capacity is stable per
- * machine) and the OS still reports the RAM it was probed with; pass
- * `force` to re-probe. Any failure (spawn error, no
+ * machine), the OS still reports the RAM it was probed with, and the same
+ * build answered it; an empty listing is re-asked hourly. Pass `force` to
+ * re-probe. Any failure (spawn error, no
  * cache, unparseable output) resolves to an empty device list — the
  * planner then simply does nothing and the engine's own `--fit`/`-ngl
  * auto` takes over. `nowIso` and `systemRamBytes` are injectable for