@bendyline/gezel-inference 1.0.6 → 1.0.8

package/dist/providers/image/types.d.ts

@@ -1,4 +1,4 @@
-import type { ImageAspectRatio, ImageModelAuxiliaryRole, ImageQuality } from '@bendyline/gezel';
+import type { ImageAspectRatio, ImageLoraPrompt, ImageLoraPurpose, ImageLoraRender, ImageModelAuxiliaryRole, ImageQuality } from '@bendyline/gezel';
 export type { ImageModelAuxiliaryRole } from '@bendyline/gezel';
 /**
  * ImageProvider — the service-side abstraction for local image generation.
@@ -69,12 +69,21 @@ export interface ImageGenerationInput {
     seed?: number;
     /**
      * Source images for image editing / reference-driven generation.
-     * Cloud providers accept multiple; sd-cpp's img2img path uses the
-     * first only (the others are dropped with a warning).
+     * Cloud providers accept multiple; a local model that reads references
+     * takes up to its limit, and sd-cpp's img2img path uses the first only
+     * (the others are dropped with a warning).
      */
     inputImages?: ImageInputBytes[];
-    /** Img2img strength (0 = keep source, 1 = ignore source). sd-cpp only. */
+    /** Img2img strength (0 = keep source, 1 = ignore source). sd-cpp img2img only. */
     strength?: number;
+    /**
+     * LoRAs to apply, by catalog id: a style, a fast mode. The engine reads each
+     * one's file, strength, prompt words and render settings from its own
+     * install, so an engine that predates LoRAs renders correctly without them.
+     */
+    loras?: Array<{
+        id: string;
+    }>;
     /**
      * Optional callback fired as the engine advances through sampling
      * steps. Today only the local sd-cpp provider emits these; cloud
@@ -95,6 +104,11 @@ export declare class ImageGenerationCancelledError extends Error {
     readonly code = "image_generation_cancelled";
     constructor();
 }
+/** A model made only for editing was asked for a picture with nothing to edit. */
+export declare class ImageSourceRequiredError extends Error {
+    readonly code = "image_source_required";
+    constructor(modelName: string);
+}
 /**
  * The engine could not fit the render into graphics memory. sd-server plans
  * placement at launch and can misjudge free memory held by another engine;
@@ -120,6 +134,11 @@ export interface ImageGenerationMeta {
     img2imgSkippedReason?: string;
     /** Provider limits or adjustments to the requested image settings. */
     settingsNotes?: string[];
+    /** The LoRAs the engine applied; absent when none. */
+    loras?: Array<{
+        id: string;
+        name: string;
+    }>;
 }
 export interface ImageGenerationOutput {
     png: Buffer;
@@ -134,6 +153,45 @@ export interface InstalledImageModelInfo {
     weightsKind?: 'checkpoint' | 'diffusion-model';
     /** Explicit catalog capability recorded at pull time; absent → resolve via the ladder. */
     supportsImg2Img?: boolean;
+    /**
+     * Resolved editing capability (see `resolveImageEditCapability`), so a
+     * caller across the machine broker or a paired server can choose a model
+     * without the engine's assessment map.
+     */
+    editMode?: 'reference' | 'img2img';
+    maxSourceImages?: number;
+    editOnly?: boolean;
+    /**
+     * Installed LoRAs that fit this model. Carried on the model listing because
+     * that listing already reaches the product daemon from a machine broker or
+     * a paired server; an engine that predates LoRAs lists none.
+     */
+    loras?: InstalledImageLoraSummary[];
+}
+/** An installed LoRA as the model listing names it. */
+export interface InstalledImageLoraSummary {
+    id: string;
+    name: string;
+    purpose: ImageLoraPurpose;
+    /** The steps a speed LoRA renders in. */
+    steps?: number;
+}
+export interface InstalledImageLoraInfo extends InstalledImageLoraSummary {
+    baseModels: string[];
+    approxSizeBytes: number;
+    installedAt: string;
+}
+/** What a LoRA install needs from its catalog entry; persisted beside the file. */
+export interface ImageLoraPullSpec {
+    downloadUrl: string;
+    sha256: string;
+    approxSizeBytes: number;
+    name: string;
+    baseModels: string[];
+    purpose: ImageLoraPurpose;
+    strength: number;
+    prompt?: ImageLoraPrompt;
+    render?: ImageLoraRender;
 }
 export type ImageModelPullEvent = {
     type: 'progress';
@@ -187,6 +245,12 @@ export interface ImageModelPullSpec {
      * weights-kind default.
      */
     supportsImg2Img?: boolean;
+    /** Catalog `referenceImages`, persisted the same way. */
+    referenceImages?: {
+        max: number;
+    };
+    /** Catalog `editOnly`, persisted the same way. */
+    editOnly?: boolean;
     /** Auxiliary files (VAE, text encoders) downloaded alongside the unet. */
     auxiliaryFiles: ImageModelAuxiliaryPullSpec[];
 }
@@ -225,6 +289,14 @@ export interface ImageProvider {
      */
     pullModel(id: string, spec: ImageModelPullSpec, signal?: AbortSignal): AsyncIterable<ImageModelPullEvent>;
     deleteModel(id: string): Promise<void>;
+    /**
+     * LoRA management for an engine that applies them (the local sd-cpp
+     * engine). Rendering never needs these: the model listing carries the
+     * installed LoRAs and `generate` takes their ids.
+     */
+    pullLora?(id: string, spec: ImageLoraPullSpec, signal?: AbortSignal): AsyncIterable<ImageModelPullEvent>;
+    listLoras?(): Promise<InstalledImageLoraInfo[]>;
+    deleteLora?(id: string): Promise<void>;
     /**
      * Probe the underlying engine for reachability without doing real
      * work. Implementations should be fast (sub-second) and never throw

package/dist/providers/image/types.js

@@ -6,6 +6,14 @@ export class ImageGenerationCancelledError extends Error {
         this.name = 'ImageGenerationCancelledError';
     }
 }
+/** A model made only for editing was asked for a picture with nothing to edit. */
+export class ImageSourceRequiredError extends Error {
+    code = 'image_source_required';
+    constructor(modelName) {
+        super(`${modelName} changes a picture you give it. Attach the picture to change, or pick a model that draws new ones.`);
+        this.name = 'ImageSourceRequiredError';
+    }
+}
 /**
  * The engine could not fit the render into graphics memory. sd-server plans
  * placement at launch and can misjudge free memory held by another engine;

package/dist/providers/llama-cpp/build-provider.js

@@ -1,4 +1,3 @@
-import { availableParallelism } from 'node:os';
 import { delimiter, dirname, join } from 'node:path';
 import { DEFAULT_LOCAL_ENGINE_IDLE_TIMEOUT_MS, createLogger, estimateLlamaCppResidentBytes, llamaCppBackendExclusion, llamaCppBackendLabel, } from '@bendyline/gezel';
 import { LLAMA_QUARANTINE_STRIKES, clearLlamaStartupStrikes, recordLlamaStartupCrash, unrunnableBuildExit, } from '@bendyline/gezel/native';
@@ -6,6 +5,7 @@ import { gezelPaths } from '@bendyline/gezel/paths';
 import { mmprojBudgetBytes, nativeVisionEnabledFor } from '../../chat/vision-capability.js';
 import { effectiveEngineRelease, isEnginePinned } from '../../engines/native-manifest.js';
 import { engineInstallPhase } from '../../engines/registry.js';
+import { availableCpuCount, containerCpuLimit } from '../../system/host-memory.js';
 import { localDeviceSettingsLabel, noModelYetMessage, notReadyError, } from '../active-install-message.js';
 import { resolveCatalogLlamaCppEngineConfig, resolveCatalogReasoningBudget, } from '../catalog-model-config.js';
 import { CapacityDeniedError, NOT_ENOUGH_MEMORY_MESSAGE, availableSystemRamBytes, formatContextCapacityDenial, minViableLocalContextTokens, planAdaptiveContextGrowth, pressureIdleGraceMs, resolveLlamaCppContextRequirement, } from '../native/capacity-broker.js';
@@ -1352,7 +1352,8 @@ export async function buildLlamaCppProvider(opts) {
                 reasoningFormat: process.env.GEZEL_LLAMA_REASONING_FORMAT?.trim() || undefined,
                 reasoningPreserve,
                 backend: llamaBuildMetadata?.backend ?? process.env.GEZEL_LLAMA_SERVER_BACKEND,
-                hostCpuCount: availableParallelism(),
+                hostCpuCount: availableCpuCount(),
+                containerCpuLimit: containerCpuLimit() ?? undefined,
             });
             return {
                 command: binary,

package/dist/providers/llama-cpp/cache-adapter.d.ts

@@ -201,6 +201,7 @@ export declare class LlamaCppCacheAdapter implements EngineCacheAdapter {
     prepareForSend(sessionId: string, systemPrompt?: string, layers?: SystemPromptLayers): Promise<void>;
     buildRequestExtras(sessionId: string): Record<string, unknown>;
     evict(sessionIds: readonly string[]): Promise<void>;
+    private evictUnlocked;
     markEphemeral(sessionId: string): void;
     reportUsage(): Promise<readonly EngineCacheUsage[]>;
     warm(_sessionId: string, _messages: readonly CacheWarmMessage[]): Promise<void>;
@@ -210,6 +211,7 @@ export declare class LlamaCppCacheAdapter implements EngineCacheAdapter {
      * sessions for the next boot.
      */
     flushAll(): Promise<number>;
+    private flushAllUnlocked;
     /**
      * Async path: allocate a slot for `sessionId`, save the LRU victim
      * (if any) to disk, and try to restore the new slot from disk
@@ -217,6 +219,7 @@ export declare class LlamaCppCacheAdapter implements EngineCacheAdapter {
      * {@link prepareForSend}.
      */
     private allocateSlot;
+    private withAllocationLock;
     private assertEngineGeneration;
     private allocateSlotUnlocked;
     /**

package/dist/providers/llama-cpp/cache-adapter.js

@@ -258,7 +258,11 @@ export class LlamaCppCacheAdapter {
         };
     }
     async evict(sessionIds) {
-        const generation = this.engineGeneration;
+        return this.withAllocationLock((generation) => this.evictUnlocked(sessionIds, generation));
+    }
+    async evictUnlocked(sessionIds, generation) {
+        if (generation !== this.engineGeneration)
+            return;
         // Save before forgetting so the session can resume on next access.
         // Persistence is best-effort; failures are logged and the local
         // mapping is still cleared (controller view stays consistent).
@@ -352,7 +356,11 @@ export class LlamaCppCacheAdapter {
     async flushAll() {
         if (!this.slotSavePath)
             return 0;
-        const generation = this.engineGeneration;
+        return this.withAllocationLock((generation) => this.flushAllUnlocked(generation));
+    }
+    async flushAllUnlocked(generation) {
+        if (generation !== this.engineGeneration)
+            return 0;
         let saved = 0;
         for (const [sessionId, slot] of this.sessionToSlot) {
             if (await this.saveSlotForSession(slot, sessionId))
@@ -369,6 +377,13 @@ export class LlamaCppCacheAdapter {
      * {@link prepareForSend}.
      */
     async allocateSlot(sessionId) {
+        return this.withAllocationLock((generation) => {
+            this.assertEngineGeneration(generation);
+            return this.allocateSlotUnlocked(sessionId, generation);
+        });
+    }
+    // Eviction and idle flushing must not save a slot after a handoff has rebound it.
+    async withAllocationLock(work) {
         const generation = this.engineGeneration;
         const previous = this.allocationTail;
         let release;
@@ -377,8 +392,7 @@ export class LlamaCppCacheAdapter {
         });
         await previous;
         try {
-            this.assertEngineGeneration(generation);
-            return await this.allocateSlotUnlocked(sessionId, generation);
+            return await work(generation);
         }
         finally {
             release();

package/dist/providers/llama-cpp/devices.js

@@ -17,11 +17,11 @@
  */
 import { execFile } from 'node:child_process';
 import { existsSync, mkdirSync, readFileSync, statSync, writeFileSync } from 'node:fs';
-import { totalmem } from 'node:os';
 import { delimiter, dirname, join } from 'node:path';
 import { promisify } from 'node:util';
 import { windowsHeadlessSpawnOptions } from '@bendyline/gezel/native';
 import { gezelHome } from '@bendyline/gezel/paths';
+import { totalMemoryBytes } from '../../system/host-memory.js';
 const execFileAsync = promisify(execFile);
 const LINE_RE = /^\s*(\S+?):\s*(.+?)\s*\((\d+)\s*MiB,\s*(\d+)\s*MiB\s+free\)\s*$/;
 /**
@@ -158,7 +158,7 @@ export function llamaBinaryFingerprint(binaryPath) {
 export async function probeLlamaDevices(opts) {
     const { binaryPath, home, force } = opts;
     const now = opts.nowIso ?? (() => new Date().toISOString());
-    const systemRamBytes = opts.systemRamBytes ?? totalmem();
+    const systemRamBytes = opts.systemRamBytes ?? totalMemoryBytes();
     const path = cachePath(home);
     const binary = llamaBinaryFingerprint(binaryPath);
     if (!force && existsSync(path)) {

package/dist/providers/llama-cpp/engine-flags.d.ts

@@ -133,6 +133,11 @@ export interface EngineFlagInput {
     backend?: string | undefined;
     /** Logical CPUs available to the daemon; caps the GPU-resident thread default. */
     hostCpuCount?: number | undefined;
+    /**
+     * CPUs a container leaves the daemon, when fewer than the host has (see
+     * `containerCpuLimit`). Without it llama.cpp sizes its pool from the host.
+     */
+    containerCpuLimit?: number | undefined;
 }
 /** Default `--cache-reuse` chunk when neither config nor manifest set it. */
 export declare const DEFAULT_CACHE_REUSE = 256;

package/dist/providers/llama-cpp/engine-flags.js

@@ -100,7 +100,7 @@ function normalizeFlagKey(key) {
  * `args` array.
  */
 export function buildLlamaCppEngineArgs(input) {
-    const { config, perModel, planner, kvCacheType, slots, ggufHasMtp, installedDraftModelPath, reasoningFormat, reasoningPreserve, backend, hostCpuCount, } = input;
+    const { config, perModel, planner, kvCacheType, slots, ggufHasMtp, installedDraftModelPath, reasoningFormat, reasoningPreserve, backend, hostCpuCount, containerCpuLimit, } = input;
     const args = [];
     // ── GPU-layer offload (`--n-gpu-layers`) ──────────────────────────
     const nGpuLayers = config.llamaCppNGpuLayers ?? perModel?.nGpuLayers ?? planner?.nGpuLayers;
@@ -203,9 +203,12 @@ export function buildLlamaCppEngineArgs(input) {
         !(typeof nCpuMoe === 'number' && nCpuMoe > 0) &&
         !(typeof nCpuFfn === 'number' && nCpuFfn > 0) &&
         !extraArgsOwnThreads(config.llamaCppExtraArgs);
+    // In a container that allows fewer CPUs than the host has, the all-cores
+    // default oversubscribes the quota and spends each period throttled.
     const threads = config.llamaCppThreads ??
         perModel?.threads ??
-        (gpuResident ? gpuResidentThreads(hostCpuCount) : undefined);
+        (gpuResident ? gpuResidentThreads(hostCpuCount) : undefined) ??
+        (extraArgsOwnThreads(config.llamaCppExtraArgs) ? undefined : containerCpuLimit);
     if (typeof threads === 'number')
         args.push('--threads', String(threads));
     const batchSize = config.llamaCppBatchSize ?? perModel?.batchSize;

package/dist/providers/mlx/async-file-handoff.d.ts

@@ -7,5 +7,13 @@
  * recipients.
  */
 export declare function isSuccessfulAsyncHandoff(toolName: string, output: string): boolean;
-export declare function asyncHandoffClosing(count: number): string;
+/** The gezel a successful handoff result names, when it names one. */
+export declare function asyncHandoffRecipient(output: string): string | undefined;
+/**
+ * The reply the person sees when the runtime ends a delegating turn, which the
+ * model also reads back as its own last message. The tool result already told
+ * the model to end its turn, so this line only says what happened. Each entry
+ * is one handoff's recipient, or undefined when its result named nobody.
+ */
+export declare function asyncHandoffClosing(recipients: readonly (string | undefined)[]): string;
 export declare function immediateFileWriteClosing(paths: string[]): string;

package/dist/providers/mlx/async-file-handoff.js

@@ -10,10 +10,38 @@ export function isSuccessfulAsyncHandoff(toolName, output) {
     return (!output.startsWith('ERROR:') &&
         (toolName === 'message_gezel' || toolName.startsWith('delegate_')));
 }
-export function asyncHandoffClosing(count) {
-    return count === 1
-        ? 'I sent the handoff and ended my turn so the recipient can use the provider queue. Their reply will arrive asynchronously.'
-        : `I sent ${count} handoffs and ended my turn so the recipients can use the provider queue. Their replies will arrive asynchronously.`;
+/**
+ * The handoff results (core's `messageGezelText`, the MCP `delegate_*` tools)
+ * name the recipient after a fixed lead-in and again as "<name>'s turn".
+ * Requiring both means a reworded result, or a name holding a period, gives
+ * no name rather than half of one.
+ */
+const HANDOFF_RECIPIENT = /(?:Accepted the (?:message|handoff) for|Dispatched the (?:message|handoff) to|An identical file handoff is already pending with) (.+?)(?= \(|[.;])[\s\S]*?\1's turn/;
+/** The gezel a successful handoff result names, when it names one. */
+export function asyncHandoffRecipient(output) {
+    return HANDOFF_RECIPIENT.exec(output.slice(0, 2_000))?.[1]?.trim() || undefined;
+}
+function joinNames(names) {
+    if (names.length <= 2)
+        return names.join(' and ');
+    return `${names.slice(0, -1).join(', ')}, and ${names[names.length - 1]}`;
+}
+/**
+ * The reply the person sees when the runtime ends a delegating turn, which the
+ * model also reads back as its own last message. The tool result already told
+ * the model to end its turn, so this line only says what happened. Each entry
+ * is one handoff's recipient, or undefined when its result named nobody.
+ */
+export function asyncHandoffClosing(recipients) {
+    const named = [...new Set(recipients.filter((name) => Boolean(name)))];
+    if (named.length > 0 && !recipients.includes(undefined)) {
+        return named.length === 1
+            ? `${named[0]} is on it. Their reply will appear here.`
+            : `${joinNames(named)} are on it. Their replies will appear here.`;
+    }
+    return recipients.length > 1
+        ? "I've passed this on. The replies will appear here."
+        : "I've passed this on. The reply will appear here.";
 }
 export function immediateFileWriteClosing(paths) {
     const unique = [...new Set(paths.filter((path) => path.trim().length > 0))];

package/dist/providers/mlx/provider.js

@@ -51,7 +51,7 @@ import { coerceToolCallArgs } from '../tool-arg-schema-coercion.js';
 import { createToolExecutor } from '../tool-executor.js';
 import { ToolFailureTracker } from '../tool-failure-tracker.js';
 import { ToolRepeatTracker } from '../tool-repeat-tracker.js';
-import { asyncHandoffClosing, immediateFileWriteClosing, isSuccessfulAsyncHandoff, } from './async-file-handoff.js';
+import { asyncHandoffClosing, asyncHandoffRecipient, immediateFileWriteClosing, isSuccessfulAsyncHandoff, } from './async-file-handoff.js';
 export { isSuccessfulAsyncHandoff };
 import { setChatTemplateKwarg, } from './chat-protocol.js';
 import { MlxEngineGate } from './engine-gate.js';
@@ -60,7 +60,7 @@ import { EngineLogRouter } from './engine-log-router.js';
 import { MlxEngineStatsSampler } from './engine-stats-sampler.js';
 import { REASONING_OPEN_HEADER, StreamingReasoningSplit, engineOpensReasoning, reopenTemplateReasoning, requestOpensReasoning, } from './reasoning-stream.js';
 import { applyMlxRequestShape } from './request-shape.js';
-import { PRE_FIRST_BYTE_BASE_MS, buildMidStreamDropMessage, buildPreFirstByteAbortMessage, computePreFirstByteBudgetMs, estimatePromptTokens, formatTps, isMidStreamConnectionDrop, mlxHttpError, } from './runtime-diagnostics.js';
+import { MLX_IMAGE_RESERVED_CHARS, PRE_FIRST_BYTE_BASE_MS, buildMidStreamDropMessage, buildPreFirstByteAbortMessage, computePreFirstByteBudgetMs, estimatePromptTokens, formatTps, isMidStreamConnectionDrop, mlxHttpError, } from './runtime-diagnostics.js';
 import { classifyMlxFatalErrorLine, classifyMlxStartupLine, } from './stdout-parser.js';
 import { salvageTextualToolCalls } from './textual-tool-call-salvage.js';
 import { APPEND_TO_FILE_CONTINUATION_TOOL, MlxToolCallAccumulator, applyCallableToolGrammar, callableRequestToolNames, chatCompletionToolName, externalToolsAsChatCompletions, hermesRequiredArgGrammarRequested, missingTopLevelRequiredToolArgs, toChatCompletionsTools, validatorReportedMissingRequiredArgs, } from './tool-call-protocol.js';
@@ -73,7 +73,7 @@ export { buildMidStreamDropMessage, computePreFirstByteBudgetMs, estimatePromptT
 import { FILE_REPAIR_MUTATION_TOOLS, FILE_REPAIR_READ_TOOLS, applyConstrainedTurnShape, disableTemplateThinking, fileRepairTargetPath, isSourceFileRepairPrompt, requiredToolOnly, writeFileOnlyTools, } from '../constrained-turn.js';
 import { hasExplicitFullFileRewriteWording } from '../direct-file-work-prompt.js';
 import { buildImmediateFileWriteNoMutationNudge, buildPrerequisiteRepairReadNudge, buildScenarioRepairNoMutationNudge, completeWorkspaceReadPaths, extractPrerequisiteRepairReadPaths, normalizeWorkspacePathForCompare, remainingPrerequisiteRepairReadPaths, } from '../file-repair-policy.js';
-import { LOCAL_TURN_LIMITS, LocalTurnPolicy, compactLocalTurn, planFileTurn, repairTools, turnCheckpoint, } from '../local-turn-policy.js';
+import { ConstrainedWriteRecovery, LOCAL_TURN_LIMITS, LocalTurnPolicy, compactLocalTurn, planFileTurn, repairTools, turnCheckpoint, } from '../local-turn-policy.js';
 const MAX_TOOL_LOOP_TURNS = LOCAL_TURN_LIMITS.iterations;
 // Hard per-turn ceiling when the caller doesn't pass `opts.timeoutMs`.
 // 10 minutes matches what the manager passes for local providers in
@@ -646,7 +646,7 @@ class MlxSession extends StreamingSessionBase {
                 total += m.content.length;
             // The sidecar bounds images to 1024px. Reserve patch tokens without
             // counting base64 bytes as language tokens or treating pixels as free.
-            total += (m.images?.length ?? 0) * 8192;
+            total += (m.images?.length ?? 0) * MLX_IMAGE_RESERVED_CHARS;
             if (m.tool_calls) {
                 for (const tc of m.tool_calls)
                     total += tc.function.arguments.length + tc.function.name.length;
@@ -984,6 +984,7 @@ class MlxSession extends StreamingSessionBase {
         // validation error every iteration, "diagnosing" the schema in
         // chat, and re-trying the same wrong shape forever. See the
         // tracker's docstring for the threshold rationale.
+        const writeRecovery = new ConstrainedWriteRecovery();
         const failureTracker = new ToolFailureTracker({
             surgicalEditsAvailable,
             delegationAvailable,
@@ -1164,7 +1165,9 @@ class MlxSession extends StreamingSessionBase {
                         log.debug(`turn#${seq}.${turn} required-call turn: ${opts.requiredTool} only`);
                     }
                 }
-                const fileTurnPlan = planFileTurn(prompt, tools, opts?.fileTurnIntent);
+                const fileTurnPlan = writeRecovery.active
+                    ? { kind: 'ordinary' }
+                    : planFileTurn(prompt, tools, opts?.fileTurnIntent);
                 const unreadInputs = unreadRequiredInputs(requiredInputs, requiredInputReads);
                 const immediateFileWriteTurn = fileTurnPlan.kind === 'create-file' && unreadInputs.length === 0;
                 if (fileTurnPlan.kind === 'create-file' &&
@@ -1219,6 +1222,11 @@ class MlxSession extends StreamingSessionBase {
                     (typeof body.max_tokens !== 'number' || body.max_tokens < FILE_WRITE_MIN_TOKENS)) {
                     body.max_tokens = FILE_WRITE_MIN_TOKENS;
                 }
+                if (writeRecovery.lookupPending) {
+                    requestTools = tools?.filter((tool) => chatCompletionToolName(tool) === writeRecovery.lookupPending);
+                    applyConstrainedTurnShape(body);
+                    log.debug(`turn#${seq}.${turn} grounding recovery: research-only surface=${writeRecovery.lookupPending}`);
+                }
                 if (requestTools && requestTools.length > 0)
                     body.tools = requestTools;
                 // Tool-schema size breakdown (opt-in: GEZEL_PROMPT_BREAKDOWN=1). The
@@ -1419,7 +1427,8 @@ class MlxSession extends StreamingSessionBase {
                 //   - STREAMING_IDLE_MS: budget between two consecutive SSE
                 //     content events once tokens are flowing. Generous enough
                 //     to cover a heavy reasoning gap on a 26B model.
-                const approxPromptTokens = estimatePromptTokens(this.messages, body.tools);
+                // The request as sent: an image retired from it costs no prefill.
+                const approxPromptTokens = estimatePromptTokens(body.messages, body.tools);
                 const PRE_FIRST_BYTE_IDLE_MS = computePreFirstByteBudgetMs(approxPromptTokens);
                 const PRE_FIRST_BYTE_PER_PROGRESS_MS = PRE_FIRST_BYTE_IDLE_MS;
                 const STREAMING_IDLE_MS = 300_000;
@@ -2160,6 +2169,11 @@ class MlxSession extends StreamingSessionBase {
                 });
                 turnContent = salvaged.turnContent;
                 const { proseRepaired, xmlTagRepaired, claudeInvokeRepaired, glmRepaired, hermesRepaired, shellRepaired, envelopeRepaired, envelopeTruncated, unknownCall, truncatedSalvage, bareInvokeRepaired, } = salvaged;
+                // A prose-shaped call such as lookup_record({}) was ordinary TEXT
+                // to the Hermes grammar, not a constrained tool envelope. Missing
+                // arguments in that salvage cannot establish a decoding violation.
+                // Keep provenance by id through coercion and duplicate collapse.
+                const proseCallIds = new Set(proseRepaired.map((call) => call.id));
                 const rawTurnContentBeforeReasoning = turnContent;
                 // Reasoning-tag extraction applies regardless of which (if any)
                 // salvage path fired. Qwen 3.5/3.6 leak `<think>…</think>`
@@ -2594,7 +2608,7 @@ class MlxSession extends StreamingSessionBase {
                     tool_calls: toolCalls,
                 });
                 let abortDueToFailureLoop = null;
-                let asyncHandoffCount = 0;
+                const asyncHandoffRecipients = [];
                 let terminalActionClosing = null;
                 const toolImages = [];
                 const immediateFileWritePaths = [];
@@ -2693,16 +2707,21 @@ class MlxSession extends StreamingSessionBase {
                         this.deps.bridges.hasTool(call.function.name)) {
                         const missingRequired = missingTopLevelRequiredToolArgs(toolArgSchemas.get(call.function.name), args);
                         const grammarContractViolation = missingRequired.length > 0 &&
+                            !proseCallIds.has(call.id) &&
                             hermesRequiredArgGrammarRequested(body) &&
                             !this.forceSequentialToolGrammar;
-                        if (grammarContractViolation) {
-                            this.forceSequentialToolGrammar = true;
-                            log.warn(`turn#${seq}.${turn} required-argument grammar contract violation tool=${call.function.name} missing=${missingRequired.join(',')} — latching this session to sequential decode`);
+                        if (missingRequired.length > 0) {
+                            if (grammarContractViolation) {
+                                this.forceSequentialToolGrammar = true;
+                                log.warn(`turn#${seq}.${turn} required-argument grammar contract violation tool=${call.function.name} missing=${missingRequired.join(',')} — latching this session to sequential decode`);
+                            }
                             // Do not send a call that the advertised schema proves cannot
-                            // pass validation. Keeping it out of the bridge also avoids
-                            // poisoning the session-wide unresolved-failure ledger before
-                            // the sequential retry gets a chance to recover.
-                            output = `ERROR: ${call.function.name} was not executed because the constrained decoder omitted required fields: ${missingRequired.join(', ')}. The runtime switched this session to sequential decoding; retry the call once.`;
+                            // pass validation, regardless of how it was parsed. A rejected
+                            // prose salvage gets ordinary corrective feedback and retains
+                            // speculation; native grammar violations keep the circuit breaker.
+                            output = grammarContractViolation
+                                ? `ERROR: ${call.function.name} was not executed because the constrained decoder omitted required fields: ${missingRequired.join(', ')}. The runtime switched this session to sequential decoding; retry the call once.`
+                                : `ERROR: ${call.function.name} was not executed because its arguments are missing required fields: ${missingRequired.join(', ')}. Retry the call with all declared required fields.`;
                         }
                         else {
                             try {
@@ -2733,6 +2752,7 @@ class MlxSession extends StreamingSessionBase {
                             // the advertised schema. Sequential decode may still recover on
                             // the next iteration, and the log now makes the skew explicit.
                             if (validatorReportedMissingRequiredArgs(output) &&
+                                !proseCallIds.has(call.id) &&
                                 hermesRequiredArgGrammarRequested(body) &&
                                 !this.forceSequentialToolGrammar) {
                                 this.forceSequentialToolGrammar = true;
@@ -2747,7 +2767,7 @@ class MlxSession extends StreamingSessionBase {
                             askedQuestionThisTurn = true;
                         }
                         if (isSuccessfulAsyncHandoff(call.function.name, output)) {
-                            asyncHandoffCount++;
+                            asyncHandoffRecipients.push(asyncHandoffRecipient(output));
                         }
                         if ((immediateFileWriteTurn || writeContinuationActive) &&
                             (call.function.name === 'write_file' || call.function.name === 'append_to_file') &&
@@ -2803,6 +2823,7 @@ class MlxSession extends StreamingSessionBase {
                         }
                     }
                     const tracked = failureTracker.recordResult(call.function.name, output);
+                    writeRecovery.recordResult(call.function.name, output, tracked, tools);
                     terminalActionClosing ??= terminalToolClosingText(this.deps.terminalToolPolicy, call.function.name, args, output);
                     // Layer the same-args repeat tracker on top of the failure
                     // tracker. The failure tracker catches "tool keeps erroring";
@@ -2877,7 +2898,7 @@ class MlxSession extends StreamingSessionBase {
                     });
                 }
                 this.turnPolicy.checkpoint();
-                if (fileRepairTurn && repairMutationSucceeded) {
+                if (fileRepairTurn && repairMutationSucceeded && !writeRecovery.active) {
                     const closing = 'Updated the requested file.';
                     this.messages.push({ role: 'assistant', content: closing });
                     return closing;
@@ -2898,7 +2919,7 @@ class MlxSession extends StreamingSessionBase {
                     log.info(`turn#${seq} END terminal-tool-bail afterMs=${Date.now() - start} loopTurns=${turn + 1}`);
                     return fullText;
                 }
-                if (immediateFileWritePaths.length > 0) {
+                if (immediateFileWritePaths.length > 0 && !writeRecovery.active) {
                     // File was truncated mid-content and we still have continuation
                     // budget — keep the turn alive instead of bailing on a partial.
                     // The tool result already carries the "append the rest" hint;
@@ -2921,12 +2942,12 @@ class MlxSession extends StreamingSessionBase {
                         `${writeContinuations > 0 ? ` continuations=${writeContinuations}` : ''}`);
                     return fullText;
                 }
-                if (asyncHandoffCount > 0) {
-                    const closingText = asyncHandoffClosing(asyncHandoffCount);
+                if (asyncHandoffRecipients.length > 0) {
+                    const closingText = asyncHandoffClosing(asyncHandoffRecipients);
                     this.messages.push({ role: 'assistant', content: closingText });
                     fullText = closingText;
                     log.info(`turn#${seq} END async-handoff-bail afterMs=${Date.now() - start} ` +
-                        `handoffs=${asyncHandoffCount} loopTurns=${turn + 1}`);
+                        `handoffs=${asyncHandoffRecipients.length} loopTurns=${turn + 1}`);
                     return fullText;
                 }
                 if (forceProjectMacroBail) {

package/dist/providers/mlx/python/gezel_mlx_server.py

@@ -2759,10 +2759,17 @@ class BatchEngine:
                     cache_seed.trim_layers(cache_layers, excess)
             except Exception:  # noqa: BLE001 — snapshot path covers hybrids
                 pass
+            cache_tokens = full_prompt + [int(t) for t in sub.token_ids]
+            if cache_seed.all_trimmable(cache_layers):
+                cache_tokens = spec_decode.cached_token_prefix(cache_layers, cache_tokens)
+                if cache_tokens is None:
+                    # Leave any prior valid entry intact rather than persisting
+                    # cache layers whose token boundary cannot be established.
+                    cache_layers = None
             r = types.SimpleNamespace(
                 finish_reason=finish_reason,
-                prompt_cache=list(cache_layers),
-                all_tokens=full_prompt + [int(t) for t in sub.token_ids],
+                prompt_cache=list(cache_layers) if cache_layers is not None else None,
+                all_tokens=cache_tokens,
                 token=sub.token_ids[-1] if sub.token_ids else 0,
             )
             self._finish(sub, r)

package/dist/providers/mlx/python/spec_decode.py

@@ -117,8 +117,11 @@ def resolve_spec(args, model, text_tower: Optional[str], log=print) -> Optional[
     except Exception as exc:  # noqa: BLE001
         _off(log, f"drafter incompatible with target: {exc}")
         return None
-    if not install_chunked_draft_prefill(
-        drafter, int(getattr(args, "prefill_step_size", 0) or 0), log=log
+    if (
+        callable(getattr(drafter, "prefill_from_target_hidden", None))
+        and not install_chunked_draft_prefill(
+            drafter, int(getattr(args, "prefill_step_size", 0) or 0), log=log
+        )
     ):
         log(
             "[spec] drafter has no incremental prefill surface — long prompts "
@@ -240,6 +243,11 @@ def draft_prefill_gate(spec: Optional[SpecState], prompt_len: int) -> Tuple[bool
     throw that work away."""
     if spec is None:
         return True, ""
+    # Shared-KV drafters (e.g. Gemma's assistant) have no separate prompt
+    # prefill. Both round loops skip this optional hook, so there is no
+    # quadratic drafter prefill to limit for that capability shape.
+    if not callable(getattr(spec.drafter, "prefill_from_target_hidden", None)):
+        return True, ""
     if getattr(spec.drafter, "_gezel_chunked_prefill", False):
         return True, ""
     if int(prompt_len) > UNCHUNKED_DRAFT_PREFILL_LIMIT:
@@ -278,9 +286,21 @@ def cache_matches_tower(layers, tower) -> bool:
         return False
     if layers is None or len(layers) != len(expect):
         return False
-    return all(
-        type(a).__name__ == type(b).__name__ for a, b in zip(layers, expect)
-    )
+
+    def matches(actual, expected):
+        if type(actual).__name__ == type(expected).__name__:
+            return True
+        # Upstream MTP deliberately wraps a rotating cache in this subclass
+        # to roll back rejected drafts. It is still the same target layout;
+        # rejecting our own saved cache here forces every follow-up cold.
+        # Do not admit arbitrary subclasses (e.g. batch-shaped cache adapters).
+        return (
+            type(expected).__name__ == "RotatingKVCache"
+            and type(actual).__name__ == "BufferedRotatingKVCache"
+            and isinstance(actual, type(expected))
+        )
+
+    return all(matches(a, b) for a, b in zip(layers, expect))
 
 
 def clone_layers(layers) -> List[Any]:
@@ -294,6 +314,25 @@ def clone_layers(layers) -> List[Any]:
     return out
 
 
+def cached_token_prefix(layers, tokens: Sequence[int]) -> Optional[List[int]]:
+    """Describe only tokens actually fed into a trimmable target cache.
+
+    A final bonus token can be emitted before it is fed back to the target.
+    Saving it as cached makes the next seed fail its offset consistency guard.
+    Unknown or disagreeing offsets must remain a miss, never guessed state.
+    """
+    try:
+        offsets = [int(c.offset) for c in layers]
+    except (AttributeError, TypeError, ValueError):
+        return None
+    if not offsets or len(set(offsets)) != 1:
+        return None
+    count = offsets[0]
+    if count <= 0 or count > len(tokens):
+        return None
+    return list(tokens[:count])
+
+
 def chunked_prefill_steps(
     tower,
     cache,