@bendyline/gezel-inference 1.0.7 → 1.0.9

package/dist/providers/llama-cpp/models.js

@@ -33,9 +33,9 @@
  */
 import { createHash } from 'node:crypto';
 import { createReadStream } from 'node:fs';
-import { mkdir, readFile, readdir, rename, rm, stat, writeFile } from 'node:fs/promises';
+import { link, mkdir, readFile, readdir, rename, rm, stat, writeFile } from 'node:fs/promises';
 import { basename, join, resolve } from 'node:path';
-import { createLogger, llamaCppBackendExclusion, llamaCppBackendLabel } from '@bendyline/gezel';
+import { createLogger, llamaCppBackendExclusion, llamaCppBackendLabel, llamaCppDraftModelApplies, } from '@bendyline/gezel';
 import { listBundleModelFiles, publishStagedModel, safeBundleModelPath, } from '../../models/bundle-storage.js';
 import { evaluateCatalogDrift } from '../../models/catalog-drift.js';
 import { catalogPayloadFingerprint, describeCatalogPayload, } from '../../models/catalog-payload-identity.js';
@@ -291,6 +291,7 @@ export class LlamaCppModelManager {
             modelDir,
             manifest: detail.manifest,
             installedVersion,
+            llamaCppBackend: this.resolveBackend(),
             installed: {
                 ...(parsed.huggingfaceRepo ? { huggingfaceRepo: parsed.huggingfaceRepo } : {}),
                 ...(parsed.payloadFingerprint ? { payloadFingerprint: parsed.payloadFingerprint } : {}),
@@ -474,7 +475,7 @@ export class LlamaCppModelManager {
         await assertModelStorePathSafe(this.modelsRoot, itemDir);
         await mkdir(itemDir, { recursive: true });
         await assertModelStorePathSafe(this.modelsRoot, itemDir);
-        const plan = planDownloads(src, includeMmproj);
+        const plan = planDownloads(src, includeMmproj, this.resolveBackend());
         if (plan.length === 0) {
             yield {
                 type: 'error',
@@ -495,6 +496,7 @@ export class LlamaCppModelManager {
         // the start, keep their digests, and are left untouched by the publish
         // rename below.
         const reusable = await this.planReusableFiles(itemDir, plan);
+        await this.adoptSiblingDraftModels(catalogId, itemDir, plan, reusable);
         let bytesCompleted = 0;
         for (const entry of plan) {
             const sha = reusable.get(entry.destFilename);
@@ -709,7 +711,7 @@ export class LlamaCppModelManager {
         // that only edits metadata can be recognized as such without re-reading a
         // byte. Skipped for a skipSha install: those bytes are knowingly not the
         // ones the catalog describes.
-        const catalogPayload = describeCatalogPayload(manifest, this.engine);
+        const catalogPayload = describeCatalogPayload(manifest, this.engine, this.resolveBackend());
         const payloadFingerprint = catalogPayload && !skipSha ? catalogPayloadFingerprint(catalogPayload) : undefined;
         // Recorded next to the catalog's tag, never instead of it: the catalog
         // names the build a user chose, the header names what the bytes are.
@@ -795,6 +797,48 @@ export class LlamaCppModelManager {
         }
         return reusable;
     }
+    /**
+     * A drafter is trained against a model, not a quant, so every quant of that
+     * model pins the same file. Link a copy another installed quant verified
+     * rather than downloading a gigabyte again. A hard link keeps each model's
+     * directory complete on its own: deleting one quant leaves the other's
+     * drafter in place. A volume that cannot link just downloads.
+     */
+    async adoptSiblingDraftModels(catalogId, itemDir, plan, reusable) {
+        const drafters = plan.filter((entry) => entry.role === 'draft-model' && !reusable.has(entry.destFilename));
+        if (drafters.length === 0)
+            return;
+        const siblings = await readdir(this.modelsRoot).catch(() => []);
+        for (const entry of drafters) {
+            for (const sibling of siblings) {
+                if (sibling === catalogId)
+                    continue;
+                const siblingDir = join(this.modelsRoot, sibling);
+                const recorded = await readFile(join(siblingDir, 'manifest.json'), 'utf8')
+                    .then((raw) => JSON.parse(raw).fileSha256)
+                    .catch(() => undefined);
+                if (recorded?.[entry.destFilename]?.toLowerCase() !== entry.sha256.toLowerCase())
+                    continue;
+                const source = join(siblingDir, entry.destFilename);
+                const size = await stat(source)
+                    .then((info) => (info.isFile() ? info.size : -1))
+                    .catch(() => -1);
+                if (size !== entry.sizeBytes)
+                    continue;
+                const dest = join(itemDir, entry.destFilename);
+                try {
+                    await rm(dest, { force: true });
+                    await link(source, dest);
+                }
+                catch {
+                    continue;
+                }
+                reusable.set(entry.destFilename, entry.sha256.toLowerCase());
+                log.info(`[models] [${this.engine}] "${catalogId}": linked ${entry.destFilename} from "${sibling}"`);
+                break;
+            }
+        }
+    }
     /** Warn once per model directory per process, then stay quiet. */
     warnSkip(id, reason) {
         if (this.skipWarned.has(id))
@@ -854,6 +898,11 @@ export class LlamaCppModelManager {
                 .then((s) => s.size)
                 .catch(() => 0)
             : 0;
+        const draftModelSizeBytes = parsed.draftModelFilename
+            ? await stat(join(root, id, parsed.draftModelFilename))
+                .then((s) => s.size)
+                .catch(() => 0)
+            : 0;
         // Absent on every copy installed before the field existed, and on those
         // the catalog tag is the only thing the table has to show.
         const ggufQuantization = parsed.ggufQuantization ??
@@ -884,7 +933,10 @@ export class LlamaCppModelManager {
                 }
                 : {}),
             ...(parsed.draftModelFilename
-                ? { draftModelPath: join(root, id, parsed.draftModelFilename) }
+                ? {
+                    draftModelPath: join(root, id, parsed.draftModelFilename),
+                    ...(draftModelSizeBytes > 0 ? { draftModelSizeBytes } : {}),
+                }
                 : {}),
             ...(parsed.contextWindow ? { contextWindow: parsed.contextWindow } : {}),
             ...(parsed.quantization ? { quantization: parsed.quantization } : {}),
@@ -928,7 +980,9 @@ function planDownloads(src,
  * This option only governs llama.cpp's `mmproj`; a declared ds4
  * `visionEncoder` is part of that model's install and is always fetched.
  */
-includeMmproj = true) {
+includeMmproj = true, 
+/** Decides a drafter limited to some llama-server backends. */
+backend) {
     const out = [];
     if (src.shards && src.shards.length > 0) {
         src.shards.forEach((shard, i) => {
@@ -976,10 +1030,10 @@ includeMmproj = true) {
         });
     }
     // Draft companions are part of the model's selected decoding path, not
-    // an optional modality. Download them whenever the catalog ships one so
-    // an explicitly selected `draft-mtp`/other draft mode can work immediately
-    // after install without a second download.
-    if (src.draftModel) {
+    // an optional modality. Download them whenever the catalog ships one for
+    // this backend, so an explicitly selected `draft-mtp`/other draft mode can
+    // work immediately after install without a second download.
+    if (src.draftModel && llamaCppDraftModelApplies(src.draftModel, backend)) {
         out.push({
             ...(src.draftModel.huggingfaceRepo
                 ? { huggingfaceRepo: src.draftModel.huggingfaceRepo }

package/dist/providers/llama-cpp/offload-planner.d.ts

@@ -94,6 +94,12 @@ export interface MoeOffloadInput {
      * consulted on the exact path; 0 / absent = weights-only budgeting.
      */
     kvReserveBytes?: number;
+    /**
+     * Weights outside the GGUF split that stay on the GPU, such as a
+     * speculative drafter. The exact path adds them to its reserves;
+     * `residentBytes` already counts them.
+     */
+    additionalGpuBytes?: number;
 }
 export interface MoeOffloadDecision {
     /** `--n-gpu-layers` (−1 = all). Undefined = leave to the engine. */

package/dist/providers/llama-cpp/offload-planner.js

@@ -167,7 +167,7 @@ function planFromSplit(input, split, margin) {
     const kvReserve = Math.max(0, input.kvReserveBytes ?? 0);
     const expertTotal = split.expertBytesByLayer.reduce((sum, v) => sum + v, 0);
     const weightsTotal = split.nonExpertBytes + expertTotal;
-    const reserves = margin + kvReserve + COMPUTE_RESERVE_BYTES;
+    const reserves = margin + kvReserve + COMPUTE_RESERVE_BYTES + Math.max(0, input.additionalGpuBytes ?? 0);
     if (weightsTotal + reserves <= input.vramBytes) {
         return {
             fullGpuResidency: true,

package/dist/providers/llama-cpp/spec-draft.d.ts

@@ -1,37 +1,59 @@
-/**
- * Speculative-decoding draft resolution, extracted from the chat/manager
- * launcher so it can be unit-tested in isolation.
- *
- * A model manifest names its `draft-simple` draft by CATALOG id
- * (`spec.draftModelId`) so it stays portable across installs — a user's GGUF
- * path differs from any hardcoded one. But engine-flags passes
- * `--spec-draft-model` verbatim and a `draft-simple` spec with no draft model
- * is a FATAL llama-server launch, so the id must be resolved to the installed
- * GGUF path (or the spec disabled) BEFORE the argv is built. This module owns
- * exactly that decision:
- *
- *   - global `config.llamaCppDraftModelPath` set → operator's own path wins,
- *     nothing to resolve.
- *   - manifest draft resolves to an installed path → rewrite `draftModelId`
- *     to that path.
- *   - manifest-driven spec whose draft isn't installed → STRIP the spec so
- *     the launch falls back to plain decoding instead of crashing. The
- *     +decode win is opportunistic on the draft being present, never a hard
- *     dependency.
- *   - operator explicitly forced `config.llamaCppSpecType=draft-simple` but
- *     no draft path resolves → leave it and warn; the operator owns it.
- */
 import type { PerModelLlamaCppEngineConfig } from './engine-flags.js';
 export interface SpecDraftResolveInput {
     /** The catalog manifest's `tuning.engine.llamaCpp` block, if any. */
     perModel: PerModelLlamaCppEngineConfig | undefined;
+    /** The llama-server build's backend, for the manifest's `spec.byBackend`. */
+    backend?: string | undefined;
     /** Global `config.llamaCppSpecType`, if the operator set one. */
     configSpecType?: string | undefined;
     /** Global `config.llamaCppDraftModelPath`, if the operator set one. */
     configDraftModelPath?: string | undefined;
+    /** The catalog `draftModel` sidecar on disk beside the weights, if any. */
+    installedDraftModelPath?: string | undefined;
     /** Resolve a catalog draft id → installed GGUF path (null if not installed). */
     resolveDraftPath: (id: string) => Promise<string | null>;
 }
+/**
+ * Whether the sidecar beside the weights is what this launch loads. A separate
+ * MTP assistant loads whenever MTP runs. A DFlash drafter sits beside a target
+ * that may have its own head, or on a backend its catalog entry did not turn it
+ * on for; handing it to MTP through `--spec-draft-model` would load the wrong
+ * file. `perBackend` is the manifest carrying a `spec.byBackend` table.
+ */
+export declare function llamaCppSidecarLoads(input: {
+    specType: string | undefined;
+    targetHasMtp: boolean;
+    perBackend: boolean;
+}): boolean;
+/**
+ * Engine and drafter pairs that took an engine down, so a later launch in this
+ * daemon goes without instead of failing the same way first. Keyed by both,
+ * because a drafter that one engine build cannot load may suit the next.
+ */
+export declare const brokenDrafters: Set<string>;
+/** The parts of a KV plan a drafter must not change. */
+export interface DrafterKvPlan {
+    kvCacheType: string;
+    ctxCapTokens?: number | undefined;
+}
+/**
+ * Whether a drafter would cost the session something it runs out of. The
+ * target window and KV precision outlast a faster decode: a drafter that
+ * makes the plan cap the target, drop to a coarser KV cache, or stop fitting
+ * at all is left out. Adaptive growth above the target and slots pay for it,
+ * as they do for every resident cost.
+ */
+export declare function drafterCostsContext(withDrafter: DrafterKvPlan, withoutDrafter: DrafterKvPlan, fitsWithDrafter: boolean): boolean;
+/**
+ * Whether a drafter would push the launch past the graphics memory it has.
+ * `gpuPoolBytes` is what a discrete card leaves this launch now (free memory
+ * as device admission measures it, within the card's usable share), and
+ * undefined elsewhere. Checked against the card's total alone, a drafter
+ * stayed while admission refused the model it was meant to speed up: the
+ * 74B Bolt and the 27B on a 32 GB card beside a desktop holding 6.5 GB, both
+ * of which start without it (2026-11-07).
+ */
+export declare function drafterCostsGpuResidency(withDrafterBytes: number, gpuPoolBytes: number | undefined): boolean;
 export interface SpecDraftResolution {
     /** The perModel to hand to `buildLlamaCppEngineArgs` (spec rewritten or stripped). */
     perModel: PerModelLlamaCppEngineConfig | undefined;

package/dist/providers/llama-cpp/spec-draft.js

@@ -1,44 +1,133 @@
+/**
+ * Speculative-decoding draft resolution, extracted from the chat/manager
+ * launcher so it can be unit-tested in isolation.
+ *
+ * A catalog entry can ask for a separate draft GGUF two ways: a `draftModel`
+ * sidecar installed beside the weights, or a `spec.draftModelId` naming another
+ * catalog model. engine-flags passes `--spec-draft-model` verbatim, and a draft
+ * algorithm with no draft model is a FATAL llama-server launch, so this has to
+ * be settled BEFORE the argv is built. This module owns that decision, and
+ * applies the catalog's per-backend overlay first so everything downstream of
+ * it reads one spec:
+ *
+ *   - global `config.llamaCppDraftModelPath` set → operator's own path wins,
+ *     nothing to resolve.
+ *   - sidecar installed → use it.
+ *   - manifest draft id resolves to an installed path → rewrite `draftModelId`
+ *     to that path.
+ *   - manifest-driven spec with no draft on disk → fall back so the launch
+ *     decodes the way it would without the drafter instead of crashing. A live
+ *     Gilde update reaches this: the new manifest takes effect at once, and the
+ *     drafter arrives only when the person updates the model. The +decode win
+ *     is opportunistic on the draft being present, never a hard dependency.
+ *   - operator explicitly forced a draft `config.llamaCppSpecType` but no draft
+ *     path resolves → leave it and warn; the operator owns it.
+ */
+import { llamaCppSpecForBackend, llamaCppSpecNeedsDraftModel } from '@bendyline/gezel';
+/**
+ * Whether the sidecar beside the weights is what this launch loads. A separate
+ * MTP assistant loads whenever MTP runs. A DFlash drafter sits beside a target
+ * that may have its own head, or on a backend its catalog entry did not turn it
+ * on for; handing it to MTP through `--spec-draft-model` would load the wrong
+ * file. `perBackend` is the manifest carrying a `spec.byBackend` table.
+ */
+export function llamaCppSidecarLoads(input) {
+    const { specType, targetHasMtp, perBackend } = input;
+    if (llamaCppSpecNeedsDraftModel(specType))
+        return true;
+    if (specType && specType !== 'draft-mtp')
+        return false;
+    return !targetHasMtp && !perBackend;
+}
+/**
+ * Engine and drafter pairs that took an engine down, so a later launch in this
+ * daemon goes without instead of failing the same way first. Keyed by both,
+ * because a drafter that one engine build cannot load may suit the next.
+ */
+export const brokenDrafters = new Set();
+/**
+ * Whether a drafter would cost the session something it runs out of. The
+ * target window and KV precision outlast a faster decode: a drafter that
+ * makes the plan cap the target, drop to a coarser KV cache, or stop fitting
+ * at all is left out. Adaptive growth above the target and slots pay for it,
+ * as they do for every resident cost.
+ */
+export function drafterCostsContext(withDrafter, withoutDrafter, fitsWithDrafter) {
+    return (!fitsWithDrafter ||
+        withDrafter.kvCacheType !== withoutDrafter.kvCacheType ||
+        withDrafter.ctxCapTokens !== withoutDrafter.ctxCapTokens);
+}
+/**
+ * Whether a drafter would push the launch past the graphics memory it has.
+ * `gpuPoolBytes` is what a discrete card leaves this launch now (free memory
+ * as device admission measures it, within the card's usable share), and
+ * undefined elsewhere. Checked against the card's total alone, a drafter
+ * stayed while admission refused the model it was meant to speed up: the
+ * 74B Bolt and the 27B on a 32 GB card beside a desktop holding 6.5 GB, both
+ * of which start without it (2026-11-07).
+ */
+export function drafterCostsGpuResidency(withDrafterBytes, gpuPoolBytes) {
+    return gpuPoolBytes !== undefined && withDrafterBytes > gpuPoolBytes;
+}
+/** The base spec with no overlay, or `perModel` with no spec at all. */
+function withoutOverlay(perModel) {
+    const base = llamaCppSpecForBackend(perModel.spec, undefined);
+    return base ? { ...perModel, spec: base } : perModel;
+}
+function withoutSpec(perModel) {
+    const { spec: _spec, ...rest } = perModel;
+    return rest;
+}
 export async function resolveSpecDraft(input) {
-    const { perModel, configSpecType, configDraftModelPath, resolveDraftPath } = input;
-    const effectiveSpecType = configSpecType ?? perModel?.spec?.type;
-    // Only the manifest-draftModelId path needs resolving. An explicit global
-    // draft path, a non-draft-simple spec, or no draft id → nothing to do.
-    if (effectiveSpecType !== 'draft-simple' ||
-        !perModel?.spec?.draftModelId ||
-        configDraftModelPath) {
+    const { perModel, backend, configSpecType, configDraftModelPath } = input;
+    const { installedDraftModelPath, resolveDraftPath } = input;
+    if (!perModel?.spec)
         return { perModel, log: null };
+    const overlay = backend ? perModel.spec.byBackend?.[backend] : undefined;
+    // A person who forced another algorithm has not asked for this overlay's
+    // tuning: MTP must not inherit a sampling mode chosen for a drafter.
+    const overlayStands = !configSpecType || !overlay?.type || overlay.type === configSpecType;
+    const spec = llamaCppSpecForBackend(perModel.spec, overlayStands ? backend : undefined);
+    const applied = perModel.spec.byBackend ? { ...perModel, spec } : perModel;
+    const effectiveSpecType = configSpecType ?? spec?.type;
+    if (!llamaCppSpecNeedsDraftModel(effectiveSpecType) ||
+        configDraftModelPath ||
+        installedDraftModelPath) {
+        return { perModel: applied, log: null };
     }
-    const draftId = perModel.spec.draftModelId;
-    const draftPath = await resolveDraftPath(draftId).catch(() => null);
-    if (draftPath) {
+    const draftId = spec?.draftModelId;
+    const draftPath = draftId ? await resolveDraftPath(draftId).catch(() => null) : null;
+    if (draftPath && spec) {
         return {
-            perModel: { ...perModel, spec: { ...perModel.spec, draftModelId: draftPath } },
+            perModel: { ...applied, spec: { ...spec, draftModelId: draftPath } },
             log: {
                 level: 'info',
                 message: `[llama-cpp] speculative decoding: draft "${draftId}" → ${draftPath}`,
             },
         };
     }
-    if (!configSpecType) {
-        // Manifest-driven spec, draft absent → strip so we don't fatal-launch a
-        // draft-simple with no draft model.
-        const { spec: _spec, ...rest } = perModel;
+    const name = draftId ? `"${draftId}"` : 'sidecar';
+    if (configSpecType) {
+        // Operator forced the spec type via global config but gave no resolvable
+        // draft — leave it and let llama-server surface the error rather than
+        // silently overriding an explicit request.
         return {
-            perModel: rest,
+            perModel: applied,
             log: {
-                level: 'info',
-                message: `[llama-cpp] speculative decoding (manifest draft "${draftId}") skipped: draft model not installed; running without it.`,
+                level: 'warn',
+                message: `[llama-cpp] config.llamaCppSpecType=${configSpecType} but draft ${name} is not installed and no llamaCppDraftModelPath is set; the launch may fail.`,
             },
         };
     }
-    // Operator forced the spec type via global config but gave no resolvable
-    // draft — leave it and let llama-server surface the error rather than
-    // silently overriding an explicit request.
+    // Manifest-driven spec, draft absent → the overlay's base if that decodes
+    // without one, else no spec at all.
+    const base = overlay ? withoutOverlay(perModel) : undefined;
+    const fallback = base && !llamaCppSpecNeedsDraftModel(base.spec?.type) ? base : withoutSpec(perModel);
     return {
-        perModel,
+        perModel: fallback,
         log: {
-            level: 'warn',
-            message: `[llama-cpp] config.llamaCppSpecType=draft-simple but draft "${draftId}" is not installed and no llamaCppDraftModelPath is set; the launch may fail.`,
+            level: 'info',
+            message: `[llama-cpp] speculative decoding (manifest draft ${name}) skipped: draft model not installed; running without it.`,
         },
     };
 }

package/dist/providers/mlx/async-file-handoff.d.ts

@@ -7,5 +7,13 @@
  * recipients.
  */
 export declare function isSuccessfulAsyncHandoff(toolName: string, output: string): boolean;
-export declare function asyncHandoffClosing(count: number): string;
+/** The gezel a successful handoff result names, when it names one. */
+export declare function asyncHandoffRecipient(output: string): string | undefined;
+/**
+ * The reply the person sees when the runtime ends a delegating turn, which the
+ * model also reads back as its own last message. The tool result already told
+ * the model to end its turn, so this line only says what happened. Each entry
+ * is one handoff's recipient, or undefined when its result named nobody.
+ */
+export declare function asyncHandoffClosing(recipients: readonly (string | undefined)[]): string;
 export declare function immediateFileWriteClosing(paths: string[]): string;

package/dist/providers/mlx/async-file-handoff.js

@@ -10,10 +10,38 @@ export function isSuccessfulAsyncHandoff(toolName, output) {
     return (!output.startsWith('ERROR:') &&
         (toolName === 'message_gezel' || toolName.startsWith('delegate_')));
 }
-export function asyncHandoffClosing(count) {
-    return count === 1
-        ? 'I sent the handoff and ended my turn so the recipient can use the provider queue. Their reply will arrive asynchronously.'
-        : `I sent ${count} handoffs and ended my turn so the recipients can use the provider queue. Their replies will arrive asynchronously.`;
+/**
+ * The handoff results (core's `messageGezelText`, the MCP `delegate_*` tools)
+ * name the recipient after a fixed lead-in and again as "<name>'s turn".
+ * Requiring both means a reworded result, or a name holding a period, gives
+ * no name rather than half of one.
+ */
+const HANDOFF_RECIPIENT = /(?:Accepted the (?:message|handoff) for|Dispatched the (?:message|handoff) to|An identical file handoff is already pending with) (.+?)(?= \(|[.;])[\s\S]*?\1's turn/;
+/** The gezel a successful handoff result names, when it names one. */
+export function asyncHandoffRecipient(output) {
+    return HANDOFF_RECIPIENT.exec(output.slice(0, 2_000))?.[1]?.trim() || undefined;
+}
+function joinNames(names) {
+    if (names.length <= 2)
+        return names.join(' and ');
+    return `${names.slice(0, -1).join(', ')}, and ${names[names.length - 1]}`;
+}
+/**
+ * The reply the person sees when the runtime ends a delegating turn, which the
+ * model also reads back as its own last message. The tool result already told
+ * the model to end its turn, so this line only says what happened. Each entry
+ * is one handoff's recipient, or undefined when its result named nobody.
+ */
+export function asyncHandoffClosing(recipients) {
+    const named = [...new Set(recipients.filter((name) => Boolean(name)))];
+    if (named.length > 0 && !recipients.includes(undefined)) {
+        return named.length === 1
+            ? `${named[0]} is on it. Their reply will appear here.`
+            : `${joinNames(named)} are on it. Their replies will appear here.`;
+    }
+    return recipients.length > 1
+        ? "I've passed this on. The replies will appear here."
+        : "I've passed this on. The reply will appear here.";
 }
 export function immediateFileWriteClosing(paths) {
     const unique = [...new Set(paths.filter((path) => path.trim().length > 0))];

package/dist/providers/mlx/provider.js

@@ -51,7 +51,7 @@ import { coerceToolCallArgs } from '../tool-arg-schema-coercion.js';
 import { createToolExecutor } from '../tool-executor.js';
 import { ToolFailureTracker } from '../tool-failure-tracker.js';
 import { ToolRepeatTracker } from '../tool-repeat-tracker.js';
-import { asyncHandoffClosing, immediateFileWriteClosing, isSuccessfulAsyncHandoff, } from './async-file-handoff.js';
+import { asyncHandoffClosing, asyncHandoffRecipient, immediateFileWriteClosing, isSuccessfulAsyncHandoff, } from './async-file-handoff.js';
 export { isSuccessfulAsyncHandoff };
 import { setChatTemplateKwarg, } from './chat-protocol.js';
 import { MlxEngineGate } from './engine-gate.js';
@@ -60,7 +60,7 @@ import { EngineLogRouter } from './engine-log-router.js';
 import { MlxEngineStatsSampler } from './engine-stats-sampler.js';
 import { REASONING_OPEN_HEADER, StreamingReasoningSplit, engineOpensReasoning, reopenTemplateReasoning, requestOpensReasoning, } from './reasoning-stream.js';
 import { applyMlxRequestShape } from './request-shape.js';
-import { PRE_FIRST_BYTE_BASE_MS, buildMidStreamDropMessage, buildPreFirstByteAbortMessage, computePreFirstByteBudgetMs, estimatePromptTokens, formatTps, isMidStreamConnectionDrop, mlxHttpError, } from './runtime-diagnostics.js';
+import { MLX_IMAGE_RESERVED_CHARS, PRE_FIRST_BYTE_BASE_MS, buildMidStreamDropMessage, buildPreFirstByteAbortMessage, computePreFirstByteBudgetMs, estimatePromptTokens, formatTps, isMidStreamConnectionDrop, mlxHttpError, } from './runtime-diagnostics.js';
 import { classifyMlxFatalErrorLine, classifyMlxStartupLine, } from './stdout-parser.js';
 import { salvageTextualToolCalls } from './textual-tool-call-salvage.js';
 import { APPEND_TO_FILE_CONTINUATION_TOOL, MlxToolCallAccumulator, applyCallableToolGrammar, callableRequestToolNames, chatCompletionToolName, externalToolsAsChatCompletions, hermesRequiredArgGrammarRequested, missingTopLevelRequiredToolArgs, toChatCompletionsTools, validatorReportedMissingRequiredArgs, } from './tool-call-protocol.js';
@@ -646,7 +646,7 @@ class MlxSession extends StreamingSessionBase {
                 total += m.content.length;
             // The sidecar bounds images to 1024px. Reserve patch tokens without
             // counting base64 bytes as language tokens or treating pixels as free.
-            total += (m.images?.length ?? 0) * 8192;
+            total += (m.images?.length ?? 0) * MLX_IMAGE_RESERVED_CHARS;
             if (m.tool_calls) {
                 for (const tc of m.tool_calls)
                     total += tc.function.arguments.length + tc.function.name.length;
@@ -1427,7 +1427,8 @@ class MlxSession extends StreamingSessionBase {
                 //   - STREAMING_IDLE_MS: budget between two consecutive SSE
                 //     content events once tokens are flowing. Generous enough
                 //     to cover a heavy reasoning gap on a 26B model.
-                const approxPromptTokens = estimatePromptTokens(this.messages, body.tools);
+                // The request as sent: an image retired from it costs no prefill.
+                const approxPromptTokens = estimatePromptTokens(body.messages, body.tools);
                 const PRE_FIRST_BYTE_IDLE_MS = computePreFirstByteBudgetMs(approxPromptTokens);
                 const PRE_FIRST_BYTE_PER_PROGRESS_MS = PRE_FIRST_BYTE_IDLE_MS;
                 const STREAMING_IDLE_MS = 300_000;
@@ -2168,6 +2169,11 @@ class MlxSession extends StreamingSessionBase {
                 });
                 turnContent = salvaged.turnContent;
                 const { proseRepaired, xmlTagRepaired, claudeInvokeRepaired, glmRepaired, hermesRepaired, shellRepaired, envelopeRepaired, envelopeTruncated, unknownCall, truncatedSalvage, bareInvokeRepaired, } = salvaged;
+                // A prose-shaped call such as lookup_record({}) was ordinary TEXT
+                // to the Hermes grammar, not a constrained tool envelope. Missing
+                // arguments in that salvage cannot establish a decoding violation.
+                // Keep provenance by id through coercion and duplicate collapse.
+                const proseCallIds = new Set(proseRepaired.map((call) => call.id));
                 const rawTurnContentBeforeReasoning = turnContent;
                 // Reasoning-tag extraction applies regardless of which (if any)
                 // salvage path fired. Qwen 3.5/3.6 leak `<think>…</think>`
@@ -2602,7 +2608,7 @@ class MlxSession extends StreamingSessionBase {
                     tool_calls: toolCalls,
                 });
                 let abortDueToFailureLoop = null;
-                let asyncHandoffCount = 0;
+                const asyncHandoffRecipients = [];
                 let terminalActionClosing = null;
                 const toolImages = [];
                 const immediateFileWritePaths = [];
@@ -2701,16 +2707,21 @@ class MlxSession extends StreamingSessionBase {
                         this.deps.bridges.hasTool(call.function.name)) {
                         const missingRequired = missingTopLevelRequiredToolArgs(toolArgSchemas.get(call.function.name), args);
                         const grammarContractViolation = missingRequired.length > 0 &&
+                            !proseCallIds.has(call.id) &&
                             hermesRequiredArgGrammarRequested(body) &&
                             !this.forceSequentialToolGrammar;
-                        if (grammarContractViolation) {
-                            this.forceSequentialToolGrammar = true;
-                            log.warn(`turn#${seq}.${turn} required-argument grammar contract violation tool=${call.function.name} missing=${missingRequired.join(',')} — latching this session to sequential decode`);
+                        if (missingRequired.length > 0) {
+                            if (grammarContractViolation) {
+                                this.forceSequentialToolGrammar = true;
+                                log.warn(`turn#${seq}.${turn} required-argument grammar contract violation tool=${call.function.name} missing=${missingRequired.join(',')} — latching this session to sequential decode`);
+                            }
                             // Do not send a call that the advertised schema proves cannot
-                            // pass validation. Keeping it out of the bridge also avoids
-                            // poisoning the session-wide unresolved-failure ledger before
-                            // the sequential retry gets a chance to recover.
-                            output = `ERROR: ${call.function.name} was not executed because the constrained decoder omitted required fields: ${missingRequired.join(', ')}. The runtime switched this session to sequential decoding; retry the call once.`;
+                            // pass validation, regardless of how it was parsed. A rejected
+                            // prose salvage gets ordinary corrective feedback and retains
+                            // speculation; native grammar violations keep the circuit breaker.
+                            output = grammarContractViolation
+                                ? `ERROR: ${call.function.name} was not executed because the constrained decoder omitted required fields: ${missingRequired.join(', ')}. The runtime switched this session to sequential decoding; retry the call once.`
+                                : `ERROR: ${call.function.name} was not executed because its arguments are missing required fields: ${missingRequired.join(', ')}. Retry the call with all declared required fields.`;
                         }
                         else {
                             try {
@@ -2741,6 +2752,7 @@ class MlxSession extends StreamingSessionBase {
                             // the advertised schema. Sequential decode may still recover on
                             // the next iteration, and the log now makes the skew explicit.
                             if (validatorReportedMissingRequiredArgs(output) &&
+                                !proseCallIds.has(call.id) &&
                                 hermesRequiredArgGrammarRequested(body) &&
                                 !this.forceSequentialToolGrammar) {
                                 this.forceSequentialToolGrammar = true;
@@ -2755,7 +2767,7 @@ class MlxSession extends StreamingSessionBase {
                             askedQuestionThisTurn = true;
                         }
                         if (isSuccessfulAsyncHandoff(call.function.name, output)) {
-                            asyncHandoffCount++;
+                            asyncHandoffRecipients.push(asyncHandoffRecipient(output));
                         }
                         if ((immediateFileWriteTurn || writeContinuationActive) &&
                             (call.function.name === 'write_file' || call.function.name === 'append_to_file') &&
@@ -2930,12 +2942,12 @@ class MlxSession extends StreamingSessionBase {
                         `${writeContinuations > 0 ? ` continuations=${writeContinuations}` : ''}`);
                     return fullText;
                 }
-                if (asyncHandoffCount > 0) {
-                    const closingText = asyncHandoffClosing(asyncHandoffCount);
+                if (asyncHandoffRecipients.length > 0) {
+                    const closingText = asyncHandoffClosing(asyncHandoffRecipients);
                     this.messages.push({ role: 'assistant', content: closingText });
                     fullText = closingText;
                     log.info(`turn#${seq} END async-handoff-bail afterMs=${Date.now() - start} ` +
-                        `handoffs=${asyncHandoffCount} loopTurns=${turn + 1}`);
+                        `handoffs=${asyncHandoffRecipients.length} loopTurns=${turn + 1}`);
                     return fullText;
                 }
                 if (forceProjectMacroBail) {

package/dist/providers/mlx/python/spec_decode.py

@@ -682,6 +682,21 @@ def abort_verify_round(verify) -> None:
         abort()
 
 
+def shared_kv_after_round(verify, rejected: int, slice_after_reject):
+    """The drafter's shared-KV view for the next round.
+
+    mlx-vlm 0.7.12 lets a target rebuild that view from its committed caches
+    (Gemma 4's ``shared_kv_from_cache``): ``verify.commit`` then empties the
+    verify window and records ``committed_shared_kv``. Slicing the emptied
+    window would hand the drafter no context, which costs acceptance, never
+    exactness. Targets without the hook (the Qwen 3.5 family) keep the slice.
+    """
+    committed = getattr(verify, "committed_shared_kv", None)
+    if committed is not None:
+        return committed
+    return slice_after_reject(verify.shared_kv_states, rejected)
+
+
 def assisted_rounds(model, spec: SpecState, cache, hidden_tokens: List[int],
                     last_output, first_bonus: int, max_tokens: int, *,
                     sampler, processors, history):
@@ -734,9 +749,13 @@ def assisted_rounds(model, spec: SpecState, cache, hidden_tokens: List[int],
     hidden = _mtp_draft_hidden(lm, hidden)
 
     kv_offset = _mtp_cache_offset_max(cache)
+    rebuild_shared_kv = getattr(lm, "shared_kv_from_cache", None)
+    if callable(rebuild_shared_kv):
+        shared_kv = rebuild_shared_kv(cache)
     drafter.set_shared_kv(
         shared_kv, kv_offset, position=_mtp_draft_position(kv_offset), kv_valid_len=kv_offset
     )
+    release_shared_kv = getattr(drafter, "release_shared_kv", None)
 
     b = int(first_bonus)
     emitted = 1
@@ -753,6 +772,8 @@ def assisted_rounds(model, spec: SpecState, cache, hidden_tokens: List[int],
                 b, hidden, None, bs, draw, token_dtype, **draft_kwargs
             )
             mx.async_eval(draft_tokens)
+            if callable(release_shared_kv):
+                release_shared_kv()
 
             verify_input = mx.concatenate(
                 [mx.array([[b]], dtype=token_dtype), draft_tokens], axis=1
@@ -806,8 +827,8 @@ def assisted_rounds(model, spec: SpecState, cache, hidden_tokens: List[int],
         hidden = _mtp_draft_hidden(lm, verify.hidden[:, accepted : accepted + 1, :])
         b = new_tokens[-1] if new_tokens else b
 
-        shared_kv = _slice_shared_kv_after_reject(
-            verify.shared_kv_states, bs - (accepted + 1)
+        shared_kv = shared_kv_after_round(
+            verify, bs - (accepted + 1), _slice_shared_kv_after_reject
         )
         kv_offset += accepted + 1
         drafter.set_shared_kv(