@bendyline/gezel-inference 1.0.8 → 1.0.9

package/dist/providers/llama-cpp/engine-flags.js

@@ -49,7 +49,7 @@ function gpuResidentThreads(hostCpuCount) {
         : GPU_RESIDENT_THREADS;
     return Math.max(1, Math.min(GPU_RESIDENT_THREADS, available));
 }
-const GPU_BACKENDS = new Set(['cuda', 'vulkan', 'metal']);
+const GPU_BACKENDS = new Set(['cuda', 'rocm', 'vulkan', 'metal']);
 /**
  * `llamaCppExtraArgs` keys that either choose a thread pool or may move
  * weights to the CPU; any of them makes the GPU-resident default stand down.
@@ -223,7 +223,7 @@ export function buildLlamaCppEngineArgs(input) {
     // MTP is ON by default for a GGUF that carries the head, having cleared the
     // A/B gate this comment used to wait for (reports/llama-mtp-eval-20260829.md):
     // +12.7% / +4.1% / +15.6% / +19.3% decode at 5.5k / 21.7k / 54k / 87k
-    // context, never negative, byte-identical greedy output, +1.35 GB resident.
+    // context, never negative, +1.35 GB resident.
     // The gain GROWS with context — long-context decode is bound by reading the
     // KV cache, and verifying K drafted tokens costs one pass over it instead of
     // K — which is the regime agent turns actually run in.
@@ -232,6 +232,8 @@ export function buildLlamaCppEngineArgs(input) {
     // FATALLY when draft-mtp is selected for a model with no MTP tensors, so
     // the default may only ever apply where the metadata confirms the head.
     // Explicit config and manifest `spec.type` still win, including 'none'.
+    // `perModel` arrives with its `spec.byBackend` already applied by
+    // `resolveSpecDraft`, which also decided whether a drafter is in play.
     const specType = config.llamaCppSpecType ?? perModel?.spec?.type ?? (ggufHasMtp ? 'draft-mtp' : undefined);
     const safeSpecType = specType === 'draft-mtp' && !ggufHasMtp ? undefined : specType;
     if (safeSpecType && safeSpecType !== 'none') {
@@ -244,12 +246,15 @@ export function buildLlamaCppEngineArgs(input) {
         const nMax = config.llamaCppSpecDraftNMax ?? perModel?.spec?.nMax;
         if (typeof nMax === 'number')
             args.push('--spec-draft-n-max', String(nMax));
-        // v0.7.0 supports probabilistic selection only for MTP/simple drafts
-        // (DFlash/LiLiCorr ignore it).
-        // Leave the upstream greedy default untouched unless explicitly set.
-        if ((safeSpecType === 'draft-mtp' || safeSpecType === 'draft-simple') &&
-            config.llamaCppSpecDraftSampling) {
-            args.push('--spec-draft-sampling', config.llamaCppSpecDraftSampling);
+        // MTP, simple and DFlash2 honor probabilistic selection; a DFlash 1 or
+        // LiLiCorr drafter takes the flag and ignores it. Leave the upstream greedy
+        // default untouched unless someone chose otherwise.
+        const draftSampling = config.llamaCppSpecDraftSampling ?? perModel?.spec?.sampling;
+        if ((safeSpecType === 'draft-mtp' ||
+            safeSpecType === 'draft-simple' ||
+            safeSpecType === 'draft-dflash') &&
+            draftSampling) {
+            args.push('--spec-draft-sampling', draftSampling);
         }
     }
     // ── Server-side output parsing (`--reasoning-format`) ─────────────

package/dist/providers/llama-cpp/gpu-startup-watch.d.ts

@@ -7,8 +7,8 @@
  * word. A CUDA backend that loads but cannot initialize prints one error line
  * and registers no devices. Either way the engine starts, answers, and runs
  * every token on the CPU — at a fraction of the speed the person's hardware
- * and Settings promise — while gezel reports the CUDA or Vulkan build it
- * launched.
+ * and Settings promise — while gezel reports the CUDA, ROCm or Vulkan build
+ * it launched.
  *
  * llama.cpp names every GPU it will use while loading a model
  * (`using device CUDA0 (NVIDIA GB10) …`), integrated GPUs included. A GPU
@@ -16,14 +16,14 @@
  * without its GPU. The explicit lines say why when there is a why to say.
  */
 export type GpuUnusedReason = 
-/** The CUDA runtime loaded and refused to start: `failed to initialize CUDA`. */
+/** The CUDA or ROCm runtime loaded and refused to start: `failed to initialize CUDA`. */
 'init-failed'
 /** The backend loaded and found no device it could use. */
  | 'no-devices'
 /** No sign of the GPU backend at all: it failed to load, silently. */
  | 'not-loaded';
 export interface GpuUnused {
-    backend: 'cuda' | 'vulkan';
+    backend: 'cuda' | 'rocm' | 'vulkan';
     reason: GpuUnusedReason;
     /** The engine's own words, when it gave any. */
     detail?: string;

package/dist/providers/llama-cpp/gpu-startup-watch.js

@@ -7,8 +7,8 @@
  * word. A CUDA backend that loads but cannot initialize prints one error line
  * and registers no devices. Either way the engine starts, answers, and runs
  * every token on the CPU — at a fraction of the speed the person's hardware
- * and Settings promise — while gezel reports the CUDA or Vulkan build it
- * launched.
+ * and Settings promise — while gezel reports the CUDA, ROCm or Vulkan build
+ * it launched.
  *
  * llama.cpp names every GPU it will use while loading a model
  * (`using device CUDA0 (NVIDIA GB10) …`), integrated GPUs included. A GPU
@@ -16,7 +16,8 @@
  * without its GPU. The explicit lines say why when there is a why to say.
  */
 const USING_GPU = /\busing device (?:CUDA|Vulkan|ROCm|SYCL|MTL)\d+\b/;
-const CUDA_INIT_FAILED = /ggml_cuda_init: failed to initialize CUDA:\s*(.+?)\s*$/;
+/** ggml's ROCm backend is its CUDA code under HIP, and names itself `ROCm` here. */
+const CUDA_INIT_FAILED = /ggml_cuda_init: failed to initialize (?:CUDA|ROCm):\s*(.+?)\s*$/;
 const VULKAN_NO_DEVICES = /ggml_vulkan: No devices found/;
 /**
  * Lines llama.cpp prints at the same verbosity as `using device`, while
@@ -46,9 +47,10 @@ export class GpuStartupWatch {
     /** At readiness: what kept the GPU out, or null when it is in use or this is no GPU build. */
     verdict() {
         const backend = this.backend;
-        if ((backend !== 'cuda' && backend !== 'vulkan') || this.usingGpu)
+        if ((backend !== 'cuda' && backend !== 'rocm' && backend !== 'vulkan') || this.usingGpu) {
             return null;
-        if (backend === 'cuda' && this.initFailure) {
+        }
+        if ((backend === 'cuda' || backend === 'rocm') && this.initFailure) {
             return { backend, reason: 'init-failed', detail: this.initFailure };
         }
         if (backend === 'vulkan' && this.noDevices)

package/dist/providers/llama-cpp/models.d.ts

@@ -73,6 +73,8 @@ export interface InstalledLlamaCppModel {
      * `--spec-draft-model <path>` for the selected draft algorithm.
      */
     draftModelPath?: string;
+    /** On-disk size of {@link draftModelPath}, for the memory plan. */
+    draftModelSizeBytes?: number;
     /** From the GGUF metadata — useful for the supervisor's `-c` flag (Phase 3). */
     contextWindow?: number;
     /** Quantization tag (Q4_K_M / Q8_0 / …) lifted from the catalog source block. */
@@ -389,6 +391,14 @@ export declare class LlamaCppModelManager {
      * a recorded digest plus an unchanged stat is identity.
      */
     private planReusableFiles;
+    /**
+     * A drafter is trained against a model, not a quant, so every quant of that
+     * model pins the same file. Link a copy another installed quant verified
+     * rather than downloading a gigabyte again. A hard link keeps each model's
+     * directory complete on its own: deleting one quant leaves the other's
+     * drafter in place. A volume that cannot link just downloads.
+     */
+    private adoptSiblingDraftModels;
     /** Warn once per model directory per process, then stay quiet. */
     private warnSkip;
     private loadInstalled;

package/dist/providers/llama-cpp/models.js

@@ -33,9 +33,9 @@
  */
 import { createHash } from 'node:crypto';
 import { createReadStream } from 'node:fs';
-import { mkdir, readFile, readdir, rename, rm, stat, writeFile } from 'node:fs/promises';
+import { link, mkdir, readFile, readdir, rename, rm, stat, writeFile } from 'node:fs/promises';
 import { basename, join, resolve } from 'node:path';
-import { createLogger, llamaCppBackendExclusion, llamaCppBackendLabel } from '@bendyline/gezel';
+import { createLogger, llamaCppBackendExclusion, llamaCppBackendLabel, llamaCppDraftModelApplies, } from '@bendyline/gezel';
 import { listBundleModelFiles, publishStagedModel, safeBundleModelPath, } from '../../models/bundle-storage.js';
 import { evaluateCatalogDrift } from '../../models/catalog-drift.js';
 import { catalogPayloadFingerprint, describeCatalogPayload, } from '../../models/catalog-payload-identity.js';
@@ -291,6 +291,7 @@ export class LlamaCppModelManager {
             modelDir,
             manifest: detail.manifest,
             installedVersion,
+            llamaCppBackend: this.resolveBackend(),
             installed: {
                 ...(parsed.huggingfaceRepo ? { huggingfaceRepo: parsed.huggingfaceRepo } : {}),
                 ...(parsed.payloadFingerprint ? { payloadFingerprint: parsed.payloadFingerprint } : {}),
@@ -474,7 +475,7 @@ export class LlamaCppModelManager {
         await assertModelStorePathSafe(this.modelsRoot, itemDir);
         await mkdir(itemDir, { recursive: true });
         await assertModelStorePathSafe(this.modelsRoot, itemDir);
-        const plan = planDownloads(src, includeMmproj);
+        const plan = planDownloads(src, includeMmproj, this.resolveBackend());
         if (plan.length === 0) {
             yield {
                 type: 'error',
@@ -495,6 +496,7 @@ export class LlamaCppModelManager {
         // the start, keep their digests, and are left untouched by the publish
         // rename below.
         const reusable = await this.planReusableFiles(itemDir, plan);
+        await this.adoptSiblingDraftModels(catalogId, itemDir, plan, reusable);
         let bytesCompleted = 0;
         for (const entry of plan) {
             const sha = reusable.get(entry.destFilename);
@@ -709,7 +711,7 @@ export class LlamaCppModelManager {
         // that only edits metadata can be recognized as such without re-reading a
         // byte. Skipped for a skipSha install: those bytes are knowingly not the
         // ones the catalog describes.
-        const catalogPayload = describeCatalogPayload(manifest, this.engine);
+        const catalogPayload = describeCatalogPayload(manifest, this.engine, this.resolveBackend());
         const payloadFingerprint = catalogPayload && !skipSha ? catalogPayloadFingerprint(catalogPayload) : undefined;
         // Recorded next to the catalog's tag, never instead of it: the catalog
         // names the build a user chose, the header names what the bytes are.
@@ -795,6 +797,48 @@ export class LlamaCppModelManager {
         }
         return reusable;
     }
+    /**
+     * A drafter is trained against a model, not a quant, so every quant of that
+     * model pins the same file. Link a copy another installed quant verified
+     * rather than downloading a gigabyte again. A hard link keeps each model's
+     * directory complete on its own: deleting one quant leaves the other's
+     * drafter in place. A volume that cannot link just downloads.
+     */
+    async adoptSiblingDraftModels(catalogId, itemDir, plan, reusable) {
+        const drafters = plan.filter((entry) => entry.role === 'draft-model' && !reusable.has(entry.destFilename));
+        if (drafters.length === 0)
+            return;
+        const siblings = await readdir(this.modelsRoot).catch(() => []);
+        for (const entry of drafters) {
+            for (const sibling of siblings) {
+                if (sibling === catalogId)
+                    continue;
+                const siblingDir = join(this.modelsRoot, sibling);
+                const recorded = await readFile(join(siblingDir, 'manifest.json'), 'utf8')
+                    .then((raw) => JSON.parse(raw).fileSha256)
+                    .catch(() => undefined);
+                if (recorded?.[entry.destFilename]?.toLowerCase() !== entry.sha256.toLowerCase())
+                    continue;
+                const source = join(siblingDir, entry.destFilename);
+                const size = await stat(source)
+                    .then((info) => (info.isFile() ? info.size : -1))
+                    .catch(() => -1);
+                if (size !== entry.sizeBytes)
+                    continue;
+                const dest = join(itemDir, entry.destFilename);
+                try {
+                    await rm(dest, { force: true });
+                    await link(source, dest);
+                }
+                catch {
+                    continue;
+                }
+                reusable.set(entry.destFilename, entry.sha256.toLowerCase());
+                log.info(`[models] [${this.engine}] "${catalogId}": linked ${entry.destFilename} from "${sibling}"`);
+                break;
+            }
+        }
+    }
     /** Warn once per model directory per process, then stay quiet. */
     warnSkip(id, reason) {
         if (this.skipWarned.has(id))
@@ -854,6 +898,11 @@ export class LlamaCppModelManager {
                 .then((s) => s.size)
                 .catch(() => 0)
             : 0;
+        const draftModelSizeBytes = parsed.draftModelFilename
+            ? await stat(join(root, id, parsed.draftModelFilename))
+                .then((s) => s.size)
+                .catch(() => 0)
+            : 0;
         // Absent on every copy installed before the field existed, and on those
         // the catalog tag is the only thing the table has to show.
         const ggufQuantization = parsed.ggufQuantization ??
@@ -884,7 +933,10 @@ export class LlamaCppModelManager {
                 }
                 : {}),
             ...(parsed.draftModelFilename
-                ? { draftModelPath: join(root, id, parsed.draftModelFilename) }
+                ? {
+                    draftModelPath: join(root, id, parsed.draftModelFilename),
+                    ...(draftModelSizeBytes > 0 ? { draftModelSizeBytes } : {}),
+                }
                 : {}),
             ...(parsed.contextWindow ? { contextWindow: parsed.contextWindow } : {}),
             ...(parsed.quantization ? { quantization: parsed.quantization } : {}),
@@ -928,7 +980,9 @@ function planDownloads(src,
  * This option only governs llama.cpp's `mmproj`; a declared ds4
  * `visionEncoder` is part of that model's install and is always fetched.
  */
-includeMmproj = true) {
+includeMmproj = true, 
+/** Decides a drafter limited to some llama-server backends. */
+backend) {
     const out = [];
     if (src.shards && src.shards.length > 0) {
         src.shards.forEach((shard, i) => {
@@ -976,10 +1030,10 @@ includeMmproj = true) {
         });
     }
     // Draft companions are part of the model's selected decoding path, not
-    // an optional modality. Download them whenever the catalog ships one so
-    // an explicitly selected `draft-mtp`/other draft mode can work immediately
-    // after install without a second download.
-    if (src.draftModel) {
+    // an optional modality. Download them whenever the catalog ships one for
+    // this backend, so an explicitly selected `draft-mtp`/other draft mode can
+    // work immediately after install without a second download.
+    if (src.draftModel && llamaCppDraftModelApplies(src.draftModel, backend)) {
         out.push({
             ...(src.draftModel.huggingfaceRepo
                 ? { huggingfaceRepo: src.draftModel.huggingfaceRepo }

package/dist/providers/llama-cpp/offload-planner.d.ts

@@ -94,6 +94,12 @@ export interface MoeOffloadInput {
      * consulted on the exact path; 0 / absent = weights-only budgeting.
      */
     kvReserveBytes?: number;
+    /**
+     * Weights outside the GGUF split that stay on the GPU, such as a
+     * speculative drafter. The exact path adds them to its reserves;
+     * `residentBytes` already counts them.
+     */
+    additionalGpuBytes?: number;
 }
 export interface MoeOffloadDecision {
     /** `--n-gpu-layers` (−1 = all). Undefined = leave to the engine. */

package/dist/providers/llama-cpp/offload-planner.js

@@ -167,7 +167,7 @@ function planFromSplit(input, split, margin) {
     const kvReserve = Math.max(0, input.kvReserveBytes ?? 0);
     const expertTotal = split.expertBytesByLayer.reduce((sum, v) => sum + v, 0);
     const weightsTotal = split.nonExpertBytes + expertTotal;
-    const reserves = margin + kvReserve + COMPUTE_RESERVE_BYTES;
+    const reserves = margin + kvReserve + COMPUTE_RESERVE_BYTES + Math.max(0, input.additionalGpuBytes ?? 0);
     if (weightsTotal + reserves <= input.vramBytes) {
         return {
             fullGpuResidency: true,

package/dist/providers/llama-cpp/spec-draft.d.ts

@@ -1,37 +1,59 @@
-/**
- * Speculative-decoding draft resolution, extracted from the chat/manager
- * launcher so it can be unit-tested in isolation.
- *
- * A model manifest names its `draft-simple` draft by CATALOG id
- * (`spec.draftModelId`) so it stays portable across installs — a user's GGUF
- * path differs from any hardcoded one. But engine-flags passes
- * `--spec-draft-model` verbatim and a `draft-simple` spec with no draft model
- * is a FATAL llama-server launch, so the id must be resolved to the installed
- * GGUF path (or the spec disabled) BEFORE the argv is built. This module owns
- * exactly that decision:
- *
- *   - global `config.llamaCppDraftModelPath` set → operator's own path wins,
- *     nothing to resolve.
- *   - manifest draft resolves to an installed path → rewrite `draftModelId`
- *     to that path.
- *   - manifest-driven spec whose draft isn't installed → STRIP the spec so
- *     the launch falls back to plain decoding instead of crashing. The
- *     +decode win is opportunistic on the draft being present, never a hard
- *     dependency.
- *   - operator explicitly forced `config.llamaCppSpecType=draft-simple` but
- *     no draft path resolves → leave it and warn; the operator owns it.
- */
 import type { PerModelLlamaCppEngineConfig } from './engine-flags.js';
 export interface SpecDraftResolveInput {
     /** The catalog manifest's `tuning.engine.llamaCpp` block, if any. */
     perModel: PerModelLlamaCppEngineConfig | undefined;
+    /** The llama-server build's backend, for the manifest's `spec.byBackend`. */
+    backend?: string | undefined;
     /** Global `config.llamaCppSpecType`, if the operator set one. */
     configSpecType?: string | undefined;
     /** Global `config.llamaCppDraftModelPath`, if the operator set one. */
     configDraftModelPath?: string | undefined;
+    /** The catalog `draftModel` sidecar on disk beside the weights, if any. */
+    installedDraftModelPath?: string | undefined;
     /** Resolve a catalog draft id → installed GGUF path (null if not installed). */
     resolveDraftPath: (id: string) => Promise<string | null>;
 }
+/**
+ * Whether the sidecar beside the weights is what this launch loads. A separate
+ * MTP assistant loads whenever MTP runs. A DFlash drafter sits beside a target
+ * that may have its own head, or on a backend its catalog entry did not turn it
+ * on for; handing it to MTP through `--spec-draft-model` would load the wrong
+ * file. `perBackend` is the manifest carrying a `spec.byBackend` table.
+ */
+export declare function llamaCppSidecarLoads(input: {
+    specType: string | undefined;
+    targetHasMtp: boolean;
+    perBackend: boolean;
+}): boolean;
+/**
+ * Engine and drafter pairs that took an engine down, so a later launch in this
+ * daemon goes without instead of failing the same way first. Keyed by both,
+ * because a drafter that one engine build cannot load may suit the next.
+ */
+export declare const brokenDrafters: Set<string>;
+/** The parts of a KV plan a drafter must not change. */
+export interface DrafterKvPlan {
+    kvCacheType: string;
+    ctxCapTokens?: number | undefined;
+}
+/**
+ * Whether a drafter would cost the session something it runs out of. The
+ * target window and KV precision outlast a faster decode: a drafter that
+ * makes the plan cap the target, drop to a coarser KV cache, or stop fitting
+ * at all is left out. Adaptive growth above the target and slots pay for it,
+ * as they do for every resident cost.
+ */
+export declare function drafterCostsContext(withDrafter: DrafterKvPlan, withoutDrafter: DrafterKvPlan, fitsWithDrafter: boolean): boolean;
+/**
+ * Whether a drafter would push the launch past the graphics memory it has.
+ * `gpuPoolBytes` is what a discrete card leaves this launch now (free memory
+ * as device admission measures it, within the card's usable share), and
+ * undefined elsewhere. Checked against the card's total alone, a drafter
+ * stayed while admission refused the model it was meant to speed up: the
+ * 74B Bolt and the 27B on a 32 GB card beside a desktop holding 6.5 GB, both
+ * of which start without it (2026-11-07).
+ */
+export declare function drafterCostsGpuResidency(withDrafterBytes: number, gpuPoolBytes: number | undefined): boolean;
 export interface SpecDraftResolution {
     /** The perModel to hand to `buildLlamaCppEngineArgs` (spec rewritten or stripped). */
     perModel: PerModelLlamaCppEngineConfig | undefined;

package/dist/providers/llama-cpp/spec-draft.js

@@ -1,44 +1,133 @@
+/**
+ * Speculative-decoding draft resolution, extracted from the chat/manager
+ * launcher so it can be unit-tested in isolation.
+ *
+ * A catalog entry can ask for a separate draft GGUF two ways: a `draftModel`
+ * sidecar installed beside the weights, or a `spec.draftModelId` naming another
+ * catalog model. engine-flags passes `--spec-draft-model` verbatim, and a draft
+ * algorithm with no draft model is a FATAL llama-server launch, so this has to
+ * be settled BEFORE the argv is built. This module owns that decision, and
+ * applies the catalog's per-backend overlay first so everything downstream of
+ * it reads one spec:
+ *
+ *   - global `config.llamaCppDraftModelPath` set → operator's own path wins,
+ *     nothing to resolve.
+ *   - sidecar installed → use it.
+ *   - manifest draft id resolves to an installed path → rewrite `draftModelId`
+ *     to that path.
+ *   - manifest-driven spec with no draft on disk → fall back so the launch
+ *     decodes the way it would without the drafter instead of crashing. A live
+ *     Gilde update reaches this: the new manifest takes effect at once, and the
+ *     drafter arrives only when the person updates the model. The +decode win
+ *     is opportunistic on the draft being present, never a hard dependency.
+ *   - operator explicitly forced a draft `config.llamaCppSpecType` but no draft
+ *     path resolves → leave it and warn; the operator owns it.
+ */
+import { llamaCppSpecForBackend, llamaCppSpecNeedsDraftModel } from '@bendyline/gezel';
+/**
+ * Whether the sidecar beside the weights is what this launch loads. A separate
+ * MTP assistant loads whenever MTP runs. A DFlash drafter sits beside a target
+ * that may have its own head, or on a backend its catalog entry did not turn it
+ * on for; handing it to MTP through `--spec-draft-model` would load the wrong
+ * file. `perBackend` is the manifest carrying a `spec.byBackend` table.
+ */
+export function llamaCppSidecarLoads(input) {
+    const { specType, targetHasMtp, perBackend } = input;
+    if (llamaCppSpecNeedsDraftModel(specType))
+        return true;
+    if (specType && specType !== 'draft-mtp')
+        return false;
+    return !targetHasMtp && !perBackend;
+}
+/**
+ * Engine and drafter pairs that took an engine down, so a later launch in this
+ * daemon goes without instead of failing the same way first. Keyed by both,
+ * because a drafter that one engine build cannot load may suit the next.
+ */
+export const brokenDrafters = new Set();
+/**
+ * Whether a drafter would cost the session something it runs out of. The
+ * target window and KV precision outlast a faster decode: a drafter that
+ * makes the plan cap the target, drop to a coarser KV cache, or stop fitting
+ * at all is left out. Adaptive growth above the target and slots pay for it,
+ * as they do for every resident cost.
+ */
+export function drafterCostsContext(withDrafter, withoutDrafter, fitsWithDrafter) {
+    return (!fitsWithDrafter ||
+        withDrafter.kvCacheType !== withoutDrafter.kvCacheType ||
+        withDrafter.ctxCapTokens !== withoutDrafter.ctxCapTokens);
+}
+/**
+ * Whether a drafter would push the launch past the graphics memory it has.
+ * `gpuPoolBytes` is what a discrete card leaves this launch now (free memory
+ * as device admission measures it, within the card's usable share), and
+ * undefined elsewhere. Checked against the card's total alone, a drafter
+ * stayed while admission refused the model it was meant to speed up: the
+ * 74B Bolt and the 27B on a 32 GB card beside a desktop holding 6.5 GB, both
+ * of which start without it (2026-11-07).
+ */
+export function drafterCostsGpuResidency(withDrafterBytes, gpuPoolBytes) {
+    return gpuPoolBytes !== undefined && withDrafterBytes > gpuPoolBytes;
+}
+/** The base spec with no overlay, or `perModel` with no spec at all. */
+function withoutOverlay(perModel) {
+    const base = llamaCppSpecForBackend(perModel.spec, undefined);
+    return base ? { ...perModel, spec: base } : perModel;
+}
+function withoutSpec(perModel) {
+    const { spec: _spec, ...rest } = perModel;
+    return rest;
+}
 export async function resolveSpecDraft(input) {
-    const { perModel, configSpecType, configDraftModelPath, resolveDraftPath } = input;
-    const effectiveSpecType = configSpecType ?? perModel?.spec?.type;
-    // Only the manifest-draftModelId path needs resolving. An explicit global
-    // draft path, a non-draft-simple spec, or no draft id → nothing to do.
-    if (effectiveSpecType !== 'draft-simple' ||
-        !perModel?.spec?.draftModelId ||
-        configDraftModelPath) {
+    const { perModel, backend, configSpecType, configDraftModelPath } = input;
+    const { installedDraftModelPath, resolveDraftPath } = input;
+    if (!perModel?.spec)
         return { perModel, log: null };
+    const overlay = backend ? perModel.spec.byBackend?.[backend] : undefined;
+    // A person who forced another algorithm has not asked for this overlay's
+    // tuning: MTP must not inherit a sampling mode chosen for a drafter.
+    const overlayStands = !configSpecType || !overlay?.type || overlay.type === configSpecType;
+    const spec = llamaCppSpecForBackend(perModel.spec, overlayStands ? backend : undefined);
+    const applied = perModel.spec.byBackend ? { ...perModel, spec } : perModel;
+    const effectiveSpecType = configSpecType ?? spec?.type;
+    if (!llamaCppSpecNeedsDraftModel(effectiveSpecType) ||
+        configDraftModelPath ||
+        installedDraftModelPath) {
+        return { perModel: applied, log: null };
     }
-    const draftId = perModel.spec.draftModelId;
-    const draftPath = await resolveDraftPath(draftId).catch(() => null);
-    if (draftPath) {
+    const draftId = spec?.draftModelId;
+    const draftPath = draftId ? await resolveDraftPath(draftId).catch(() => null) : null;
+    if (draftPath && spec) {
         return {
-            perModel: { ...perModel, spec: { ...perModel.spec, draftModelId: draftPath } },
+            perModel: { ...applied, spec: { ...spec, draftModelId: draftPath } },
             log: {
                 level: 'info',
                 message: `[llama-cpp] speculative decoding: draft "${draftId}" → ${draftPath}`,
             },
         };
     }
-    if (!configSpecType) {
-        // Manifest-driven spec, draft absent → strip so we don't fatal-launch a
-        // draft-simple with no draft model.
-        const { spec: _spec, ...rest } = perModel;
+    const name = draftId ? `"${draftId}"` : 'sidecar';
+    if (configSpecType) {
+        // Operator forced the spec type via global config but gave no resolvable
+        // draft — leave it and let llama-server surface the error rather than
+        // silently overriding an explicit request.
         return {
-            perModel: rest,
+            perModel: applied,
             log: {
-                level: 'info',
-                message: `[llama-cpp] speculative decoding (manifest draft "${draftId}") skipped: draft model not installed; running without it.`,
+                level: 'warn',
+                message: `[llama-cpp] config.llamaCppSpecType=${configSpecType} but draft ${name} is not installed and no llamaCppDraftModelPath is set; the launch may fail.`,
             },
         };
     }
-    // Operator forced the spec type via global config but gave no resolvable
-    // draft — leave it and let llama-server surface the error rather than
-    // silently overriding an explicit request.
+    // Manifest-driven spec, draft absent → the overlay's base if that decodes
+    // without one, else no spec at all.
+    const base = overlay ? withoutOverlay(perModel) : undefined;
+    const fallback = base && !llamaCppSpecNeedsDraftModel(base.spec?.type) ? base : withoutSpec(perModel);
     return {
-        perModel,
+        perModel: fallback,
         log: {
-            level: 'warn',
-            message: `[llama-cpp] config.llamaCppSpecType=draft-simple but draft "${draftId}" is not installed and no llamaCppDraftModelPath is set; the launch may fail.`,
+            level: 'info',
+            message: `[llama-cpp] speculative decoding (manifest draft ${name}) skipped: draft model not installed; running without it.`,
         },
     };
 }

package/dist/providers/mlx/python/spec_decode.py

@@ -682,6 +682,21 @@ def abort_verify_round(verify) -> None:
         abort()
 
 
+def shared_kv_after_round(verify, rejected: int, slice_after_reject):
+    """The drafter's shared-KV view for the next round.
+
+    mlx-vlm 0.7.12 lets a target rebuild that view from its committed caches
+    (Gemma 4's ``shared_kv_from_cache``): ``verify.commit`` then empties the
+    verify window and records ``committed_shared_kv``. Slicing the emptied
+    window would hand the drafter no context, which costs acceptance, never
+    exactness. Targets without the hook (the Qwen 3.5 family) keep the slice.
+    """
+    committed = getattr(verify, "committed_shared_kv", None)
+    if committed is not None:
+        return committed
+    return slice_after_reject(verify.shared_kv_states, rejected)
+
+
 def assisted_rounds(model, spec: SpecState, cache, hidden_tokens: List[int],
                     last_output, first_bonus: int, max_tokens: int, *,
                     sampler, processors, history):
@@ -734,9 +749,13 @@ def assisted_rounds(model, spec: SpecState, cache, hidden_tokens: List[int],
     hidden = _mtp_draft_hidden(lm, hidden)
 
     kv_offset = _mtp_cache_offset_max(cache)
+    rebuild_shared_kv = getattr(lm, "shared_kv_from_cache", None)
+    if callable(rebuild_shared_kv):
+        shared_kv = rebuild_shared_kv(cache)
     drafter.set_shared_kv(
         shared_kv, kv_offset, position=_mtp_draft_position(kv_offset), kv_valid_len=kv_offset
     )
+    release_shared_kv = getattr(drafter, "release_shared_kv", None)
 
     b = int(first_bonus)
     emitted = 1
@@ -753,6 +772,8 @@ def assisted_rounds(model, spec: SpecState, cache, hidden_tokens: List[int],
                 b, hidden, None, bs, draw, token_dtype, **draft_kwargs
             )
             mx.async_eval(draft_tokens)
+            if callable(release_shared_kv):
+                release_shared_kv()
 
             verify_input = mx.concatenate(
                 [mx.array([[b]], dtype=token_dtype), draft_tokens], axis=1
@@ -806,8 +827,8 @@ def assisted_rounds(model, spec: SpecState, cache, hidden_tokens: List[int],
         hidden = _mtp_draft_hidden(lm, verify.hidden[:, accepted : accepted + 1, :])
         b = new_tokens[-1] if new_tokens else b
 
-        shared_kv = _slice_shared_kv_after_reject(
-            verify.shared_kv_states, bs - (accepted + 1)
+        shared_kv = shared_kv_after_round(
+            verify, bs - (accepted + 1), _slice_shared_kv_after_reject
         )
         kv_offset += accepted + 1
         drafter.set_shared_kv(