@bendyline/gezel-inference 1.0.5 → 1.0.7

package/dist/providers/image/types.d.ts

@@ -1,4 +1,4 @@
-import type { ImageAspectRatio, ImageModelAuxiliaryRole, ImageQuality } from '@bendyline/gezel';
+import type { ImageAspectRatio, ImageLoraPrompt, ImageLoraPurpose, ImageLoraRender, ImageModelAuxiliaryRole, ImageQuality } from '@bendyline/gezel';
 export type { ImageModelAuxiliaryRole } from '@bendyline/gezel';
 /**
  * ImageProvider — the service-side abstraction for local image generation.
@@ -69,12 +69,21 @@ export interface ImageGenerationInput {
     seed?: number;
     /**
      * Source images for image editing / reference-driven generation.
-     * Cloud providers accept multiple; sd-cpp's img2img path uses the
-     * first only (the others are dropped with a warning).
+     * Cloud providers accept multiple; a local model that reads references
+     * takes up to its limit, and sd-cpp's img2img path uses the first only
+     * (the others are dropped with a warning).
      */
     inputImages?: ImageInputBytes[];
-    /** Img2img strength (0 = keep source, 1 = ignore source). sd-cpp only. */
+    /** Img2img strength (0 = keep source, 1 = ignore source). sd-cpp img2img only. */
     strength?: number;
+    /**
+     * LoRAs to apply, by catalog id: a style, a fast mode. The engine reads each
+     * one's file, strength, prompt words and render settings from its own
+     * install, so an engine that predates LoRAs renders correctly without them.
+     */
+    loras?: Array<{
+        id: string;
+    }>;
     /**
      * Optional callback fired as the engine advances through sampling
      * steps. Today only the local sd-cpp provider emits these; cloud
@@ -95,6 +104,11 @@ export declare class ImageGenerationCancelledError extends Error {
     readonly code = "image_generation_cancelled";
     constructor();
 }
+/** A model made only for editing was asked for a picture with nothing to edit. */
+export declare class ImageSourceRequiredError extends Error {
+    readonly code = "image_source_required";
+    constructor(modelName: string);
+}
 /**
  * The engine could not fit the render into graphics memory. sd-server plans
  * placement at launch and can misjudge free memory held by another engine;
@@ -120,6 +134,11 @@ export interface ImageGenerationMeta {
     img2imgSkippedReason?: string;
     /** Provider limits or adjustments to the requested image settings. */
     settingsNotes?: string[];
+    /** The LoRAs the engine applied; absent when none. */
+    loras?: Array<{
+        id: string;
+        name: string;
+    }>;
 }
 export interface ImageGenerationOutput {
     png: Buffer;
@@ -134,6 +153,45 @@ export interface InstalledImageModelInfo {
     weightsKind?: 'checkpoint' | 'diffusion-model';
     /** Explicit catalog capability recorded at pull time; absent → resolve via the ladder. */
     supportsImg2Img?: boolean;
+    /**
+     * Resolved editing capability (see `resolveImageEditCapability`), so a
+     * caller across the machine broker or a paired server can choose a model
+     * without the engine's assessment map.
+     */
+    editMode?: 'reference' | 'img2img';
+    maxSourceImages?: number;
+    editOnly?: boolean;
+    /**
+     * Installed LoRAs that fit this model. Carried on the model listing because
+     * that listing already reaches the product daemon from a machine broker or
+     * a paired server; an engine that predates LoRAs lists none.
+     */
+    loras?: InstalledImageLoraSummary[];
+}
+/** An installed LoRA as the model listing names it. */
+export interface InstalledImageLoraSummary {
+    id: string;
+    name: string;
+    purpose: ImageLoraPurpose;
+    /** The steps a speed LoRA renders in. */
+    steps?: number;
+}
+export interface InstalledImageLoraInfo extends InstalledImageLoraSummary {
+    baseModels: string[];
+    approxSizeBytes: number;
+    installedAt: string;
+}
+/** What a LoRA install needs from its catalog entry; persisted beside the file. */
+export interface ImageLoraPullSpec {
+    downloadUrl: string;
+    sha256: string;
+    approxSizeBytes: number;
+    name: string;
+    baseModels: string[];
+    purpose: ImageLoraPurpose;
+    strength: number;
+    prompt?: ImageLoraPrompt;
+    render?: ImageLoraRender;
 }
 export type ImageModelPullEvent = {
     type: 'progress';
@@ -187,6 +245,12 @@ export interface ImageModelPullSpec {
      * weights-kind default.
      */
     supportsImg2Img?: boolean;
+    /** Catalog `referenceImages`, persisted the same way. */
+    referenceImages?: {
+        max: number;
+    };
+    /** Catalog `editOnly`, persisted the same way. */
+    editOnly?: boolean;
     /** Auxiliary files (VAE, text encoders) downloaded alongside the unet. */
     auxiliaryFiles: ImageModelAuxiliaryPullSpec[];
 }
@@ -225,6 +289,14 @@ export interface ImageProvider {
      */
     pullModel(id: string, spec: ImageModelPullSpec, signal?: AbortSignal): AsyncIterable<ImageModelPullEvent>;
     deleteModel(id: string): Promise<void>;
+    /**
+     * LoRA management for an engine that applies them (the local sd-cpp
+     * engine). Rendering never needs these: the model listing carries the
+     * installed LoRAs and `generate` takes their ids.
+     */
+    pullLora?(id: string, spec: ImageLoraPullSpec, signal?: AbortSignal): AsyncIterable<ImageModelPullEvent>;
+    listLoras?(): Promise<InstalledImageLoraInfo[]>;
+    deleteLora?(id: string): Promise<void>;
     /**
      * Probe the underlying engine for reachability without doing real
      * work. Implementations should be fast (sub-second) and never throw

package/dist/providers/image/types.js

@@ -6,6 +6,14 @@ export class ImageGenerationCancelledError extends Error {
         this.name = 'ImageGenerationCancelledError';
     }
 }
+/** A model made only for editing was asked for a picture with nothing to edit. */
+export class ImageSourceRequiredError extends Error {
+    code = 'image_source_required';
+    constructor(modelName) {
+        super(`${modelName} changes a picture you give it. Attach the picture to change, or pick a model that draws new ones.`);
+        this.name = 'ImageSourceRequiredError';
+    }
+}
 /**
  * The engine could not fit the render into graphics memory. sd-server plans
  * placement at launch and can misjudge free memory held by another engine;

package/dist/providers/llama-cpp/build-provider.js

@@ -1,4 +1,3 @@
-import { availableParallelism } from 'node:os';
 import { delimiter, dirname, join } from 'node:path';
 import { DEFAULT_LOCAL_ENGINE_IDLE_TIMEOUT_MS, createLogger, estimateLlamaCppResidentBytes, llamaCppBackendExclusion, llamaCppBackendLabel, } from '@bendyline/gezel';
 import { LLAMA_QUARANTINE_STRIKES, clearLlamaStartupStrikes, recordLlamaStartupCrash, unrunnableBuildExit, } from '@bendyline/gezel/native';
@@ -6,6 +5,7 @@ import { gezelPaths } from '@bendyline/gezel/paths';
 import { mmprojBudgetBytes, nativeVisionEnabledFor } from '../../chat/vision-capability.js';
 import { effectiveEngineRelease, isEnginePinned } from '../../engines/native-manifest.js';
 import { engineInstallPhase } from '../../engines/registry.js';
+import { availableCpuCount, containerCpuLimit } from '../../system/host-memory.js';
 import { localDeviceSettingsLabel, noModelYetMessage, notReadyError, } from '../active-install-message.js';
 import { resolveCatalogLlamaCppEngineConfig, resolveCatalogReasoningBudget, } from '../catalog-model-config.js';
 import { CapacityDeniedError, NOT_ENOUGH_MEMORY_MESSAGE, availableSystemRamBytes, formatContextCapacityDenial, minViableLocalContextTokens, planAdaptiveContextGrowth, pressureIdleGraceMs, resolveLlamaCppContextRequirement, } from '../native/capacity-broker.js';
@@ -1352,7 +1352,8 @@ export async function buildLlamaCppProvider(opts) {
                 reasoningFormat: process.env.GEZEL_LLAMA_REASONING_FORMAT?.trim() || undefined,
                 reasoningPreserve,
                 backend: llamaBuildMetadata?.backend ?? process.env.GEZEL_LLAMA_SERVER_BACKEND,
-                hostCpuCount: availableParallelism(),
+                hostCpuCount: availableCpuCount(),
+                containerCpuLimit: containerCpuLimit() ?? undefined,
             });
             return {
                 command: binary,

package/dist/providers/llama-cpp/cache-adapter.d.ts

@@ -201,6 +201,7 @@ export declare class LlamaCppCacheAdapter implements EngineCacheAdapter {
     prepareForSend(sessionId: string, systemPrompt?: string, layers?: SystemPromptLayers): Promise<void>;
     buildRequestExtras(sessionId: string): Record<string, unknown>;
     evict(sessionIds: readonly string[]): Promise<void>;
+    private evictUnlocked;
     markEphemeral(sessionId: string): void;
     reportUsage(): Promise<readonly EngineCacheUsage[]>;
     warm(_sessionId: string, _messages: readonly CacheWarmMessage[]): Promise<void>;
@@ -210,6 +211,7 @@ export declare class LlamaCppCacheAdapter implements EngineCacheAdapter {
      * sessions for the next boot.
      */
     flushAll(): Promise<number>;
+    private flushAllUnlocked;
     /**
      * Async path: allocate a slot for `sessionId`, save the LRU victim
      * (if any) to disk, and try to restore the new slot from disk
@@ -217,6 +219,7 @@ export declare class LlamaCppCacheAdapter implements EngineCacheAdapter {
      * {@link prepareForSend}.
      */
     private allocateSlot;
+    private withAllocationLock;
     private assertEngineGeneration;
     private allocateSlotUnlocked;
     /**

package/dist/providers/llama-cpp/cache-adapter.js

@@ -258,7 +258,11 @@ export class LlamaCppCacheAdapter {
         };
     }
     async evict(sessionIds) {
-        const generation = this.engineGeneration;
+        return this.withAllocationLock((generation) => this.evictUnlocked(sessionIds, generation));
+    }
+    async evictUnlocked(sessionIds, generation) {
+        if (generation !== this.engineGeneration)
+            return;
         // Save before forgetting so the session can resume on next access.
         // Persistence is best-effort; failures are logged and the local
         // mapping is still cleared (controller view stays consistent).
@@ -352,7 +356,11 @@ export class LlamaCppCacheAdapter {
     async flushAll() {
         if (!this.slotSavePath)
             return 0;
-        const generation = this.engineGeneration;
+        return this.withAllocationLock((generation) => this.flushAllUnlocked(generation));
+    }
+    async flushAllUnlocked(generation) {
+        if (generation !== this.engineGeneration)
+            return 0;
         let saved = 0;
         for (const [sessionId, slot] of this.sessionToSlot) {
             if (await this.saveSlotForSession(slot, sessionId))
@@ -369,6 +377,13 @@ export class LlamaCppCacheAdapter {
      * {@link prepareForSend}.
      */
     async allocateSlot(sessionId) {
+        return this.withAllocationLock((generation) => {
+            this.assertEngineGeneration(generation);
+            return this.allocateSlotUnlocked(sessionId, generation);
+        });
+    }
+    // Eviction and idle flushing must not save a slot after a handoff has rebound it.
+    async withAllocationLock(work) {
         const generation = this.engineGeneration;
         const previous = this.allocationTail;
         let release;
@@ -377,8 +392,7 @@ export class LlamaCppCacheAdapter {
         });
         await previous;
         try {
-            this.assertEngineGeneration(generation);
-            return await this.allocateSlotUnlocked(sessionId, generation);
+            return await work(generation);
         }
         finally {
             release();

package/dist/providers/llama-cpp/devices.js

@@ -17,11 +17,11 @@
  */
 import { execFile } from 'node:child_process';
 import { existsSync, mkdirSync, readFileSync, statSync, writeFileSync } from 'node:fs';
-import { totalmem } from 'node:os';
 import { delimiter, dirname, join } from 'node:path';
 import { promisify } from 'node:util';
 import { windowsHeadlessSpawnOptions } from '@bendyline/gezel/native';
 import { gezelHome } from '@bendyline/gezel/paths';
+import { totalMemoryBytes } from '../../system/host-memory.js';
 const execFileAsync = promisify(execFile);
 const LINE_RE = /^\s*(\S+?):\s*(.+?)\s*\((\d+)\s*MiB,\s*(\d+)\s*MiB\s+free\)\s*$/;
 /**
@@ -158,7 +158,7 @@ export function llamaBinaryFingerprint(binaryPath) {
 export async function probeLlamaDevices(opts) {
     const { binaryPath, home, force } = opts;
     const now = opts.nowIso ?? (() => new Date().toISOString());
-    const systemRamBytes = opts.systemRamBytes ?? totalmem();
+    const systemRamBytes = opts.systemRamBytes ?? totalMemoryBytes();
     const path = cachePath(home);
     const binary = llamaBinaryFingerprint(binaryPath);
     if (!force && existsSync(path)) {

package/dist/providers/llama-cpp/engine-flags.d.ts

@@ -133,6 +133,11 @@ export interface EngineFlagInput {
     backend?: string | undefined;
     /** Logical CPUs available to the daemon; caps the GPU-resident thread default. */
     hostCpuCount?: number | undefined;
+    /**
+     * CPUs a container leaves the daemon, when fewer than the host has (see
+     * `containerCpuLimit`). Without it llama.cpp sizes its pool from the host.
+     */
+    containerCpuLimit?: number | undefined;
 }
 /** Default `--cache-reuse` chunk when neither config nor manifest set it. */
 export declare const DEFAULT_CACHE_REUSE = 256;

package/dist/providers/llama-cpp/engine-flags.js

@@ -100,7 +100,7 @@ function normalizeFlagKey(key) {
  * `args` array.
  */
 export function buildLlamaCppEngineArgs(input) {
-    const { config, perModel, planner, kvCacheType, slots, ggufHasMtp, installedDraftModelPath, reasoningFormat, reasoningPreserve, backend, hostCpuCount, } = input;
+    const { config, perModel, planner, kvCacheType, slots, ggufHasMtp, installedDraftModelPath, reasoningFormat, reasoningPreserve, backend, hostCpuCount, containerCpuLimit, } = input;
     const args = [];
     // ── GPU-layer offload (`--n-gpu-layers`) ──────────────────────────
     const nGpuLayers = config.llamaCppNGpuLayers ?? perModel?.nGpuLayers ?? planner?.nGpuLayers;
@@ -203,9 +203,12 @@ export function buildLlamaCppEngineArgs(input) {
         !(typeof nCpuMoe === 'number' && nCpuMoe > 0) &&
         !(typeof nCpuFfn === 'number' && nCpuFfn > 0) &&
         !extraArgsOwnThreads(config.llamaCppExtraArgs);
+    // In a container that allows fewer CPUs than the host has, the all-cores
+    // default oversubscribes the quota and spends each period throttled.
     const threads = config.llamaCppThreads ??
         perModel?.threads ??
-        (gpuResident ? gpuResidentThreads(hostCpuCount) : undefined);
+        (gpuResident ? gpuResidentThreads(hostCpuCount) : undefined) ??
+        (extraArgsOwnThreads(config.llamaCppExtraArgs) ? undefined : containerCpuLimit);
     if (typeof threads === 'number')
         args.push('--threads', String(threads));
     const batchSize = config.llamaCppBatchSize ?? perModel?.batchSize;

package/dist/providers/llama-cpp/provider.js

@@ -32,6 +32,13 @@ import { classifyStartupLine } from './stdout-parser.js';
 export * from '@bendyline/gezel/local-loop';
 export { isFileRepairTurn as isScenarioFileRepairTurn } from '@bendyline/gezel/local-loop';
 const log = createLogger('llama-cpp');
+/**
+ * How long shutdown waits for the slot caches to reach disk. The app gives
+ * the whole daemon 15 s to quit, and a save the engine never answers must
+ * not spend all of it: an unsaved or half-written slot only means a fresh
+ * prefill next time, because a restore that fails falls back to one.
+ */
+const SHUTDOWN_FLUSH_BUDGET_MS = 8_000;
 export class LlamaCppProvider {
     name = 'llama-cpp';
     queue;
@@ -627,12 +634,21 @@ export class LlamaCppProvider {
         // llama-server exits the slot save endpoint is gone. flushAll is
         // best-effort; failures during shutdown shouldn't block the stop.
         if (this.cacheAdapter) {
+            let timer;
+            const budget = new Promise((resolve) => {
+                timer = setTimeout(() => resolve('timeout'), SHUTDOWN_FLUSH_BUDGET_MS);
+                timer.unref?.();
+            });
             try {
-                await this.cacheAdapter.flushAll();
+                // Best-effort persistence: a failure only means the next boot pays
+                // the prefill cost on resume.
+                const outcome = await Promise.race([this.cacheAdapter.flushAll().catch(() => 0), budget]);
+                if (outcome === 'timeout') {
+                    log.warn(`slot cache save still running after ${SHUTDOWN_FLUSH_BUDGET_MS}ms at shutdown; stopping the engine`);
+                }
             }
-            catch {
-                // Best-effort persistence — failure here just means the next
-                // boot pays the prefill cost on resume, same as before.
+            finally {
+                clearTimeout(timer);
             }
         }
         await this.supervisor?.stop();

package/dist/providers/mlx/provider.js

@@ -73,7 +73,7 @@ export { buildMidStreamDropMessage, computePreFirstByteBudgetMs, estimatePromptT
 import { FILE_REPAIR_MUTATION_TOOLS, FILE_REPAIR_READ_TOOLS, applyConstrainedTurnShape, disableTemplateThinking, fileRepairTargetPath, isSourceFileRepairPrompt, requiredToolOnly, writeFileOnlyTools, } from '../constrained-turn.js';
 import { hasExplicitFullFileRewriteWording } from '../direct-file-work-prompt.js';
 import { buildImmediateFileWriteNoMutationNudge, buildPrerequisiteRepairReadNudge, buildScenarioRepairNoMutationNudge, completeWorkspaceReadPaths, extractPrerequisiteRepairReadPaths, normalizeWorkspacePathForCompare, remainingPrerequisiteRepairReadPaths, } from '../file-repair-policy.js';
-import { LOCAL_TURN_LIMITS, LocalTurnPolicy, compactLocalTurn, planFileTurn, repairTools, turnCheckpoint, } from '../local-turn-policy.js';
+import { ConstrainedWriteRecovery, LOCAL_TURN_LIMITS, LocalTurnPolicy, compactLocalTurn, planFileTurn, repairTools, turnCheckpoint, } from '../local-turn-policy.js';
 const MAX_TOOL_LOOP_TURNS = LOCAL_TURN_LIMITS.iterations;
 // Hard per-turn ceiling when the caller doesn't pass `opts.timeoutMs`.
 // 10 minutes matches what the manager passes for local providers in
@@ -984,6 +984,7 @@ class MlxSession extends StreamingSessionBase {
         // validation error every iteration, "diagnosing" the schema in
         // chat, and re-trying the same wrong shape forever. See the
         // tracker's docstring for the threshold rationale.
+        const writeRecovery = new ConstrainedWriteRecovery();
         const failureTracker = new ToolFailureTracker({
             surgicalEditsAvailable,
             delegationAvailable,
@@ -1164,7 +1165,9 @@ class MlxSession extends StreamingSessionBase {
                         log.debug(`turn#${seq}.${turn} required-call turn: ${opts.requiredTool} only`);
                     }
                 }
-                const fileTurnPlan = planFileTurn(prompt, tools, opts?.fileTurnIntent);
+                const fileTurnPlan = writeRecovery.active
+                    ? { kind: 'ordinary' }
+                    : planFileTurn(prompt, tools, opts?.fileTurnIntent);
                 const unreadInputs = unreadRequiredInputs(requiredInputs, requiredInputReads);
                 const immediateFileWriteTurn = fileTurnPlan.kind === 'create-file' && unreadInputs.length === 0;
                 if (fileTurnPlan.kind === 'create-file' &&
@@ -1219,6 +1222,11 @@ class MlxSession extends StreamingSessionBase {
                     (typeof body.max_tokens !== 'number' || body.max_tokens < FILE_WRITE_MIN_TOKENS)) {
                     body.max_tokens = FILE_WRITE_MIN_TOKENS;
                 }
+                if (writeRecovery.lookupPending) {
+                    requestTools = tools?.filter((tool) => chatCompletionToolName(tool) === writeRecovery.lookupPending);
+                    applyConstrainedTurnShape(body);
+                    log.debug(`turn#${seq}.${turn} grounding recovery: research-only surface=${writeRecovery.lookupPending}`);
+                }
                 if (requestTools && requestTools.length > 0)
                     body.tools = requestTools;
                 // Tool-schema size breakdown (opt-in: GEZEL_PROMPT_BREAKDOWN=1). The
@@ -2803,6 +2811,7 @@ class MlxSession extends StreamingSessionBase {
                         }
                     }
                     const tracked = failureTracker.recordResult(call.function.name, output);
+                    writeRecovery.recordResult(call.function.name, output, tracked, tools);
                     terminalActionClosing ??= terminalToolClosingText(this.deps.terminalToolPolicy, call.function.name, args, output);
                     // Layer the same-args repeat tracker on top of the failure
                     // tracker. The failure tracker catches "tool keeps erroring";
@@ -2877,7 +2886,7 @@ class MlxSession extends StreamingSessionBase {
                     });
                 }
                 this.turnPolicy.checkpoint();
-                if (fileRepairTurn && repairMutationSucceeded) {
+                if (fileRepairTurn && repairMutationSucceeded && !writeRecovery.active) {
                     const closing = 'Updated the requested file.';
                     this.messages.push({ role: 'assistant', content: closing });
                     return closing;
@@ -2898,7 +2907,7 @@ class MlxSession extends StreamingSessionBase {
                     log.info(`turn#${seq} END terminal-tool-bail afterMs=${Date.now() - start} loopTurns=${turn + 1}`);
                     return fullText;
                 }
-                if (immediateFileWritePaths.length > 0) {
+                if (immediateFileWritePaths.length > 0 && !writeRecovery.active) {
                     // File was truncated mid-content and we still have continuation
                     // budget — keep the turn alive instead of bailing on a partial.
                     // The tool result already carries the "append the rest" hint;

package/dist/providers/mlx/python/gezel_mlx_server.py

@@ -2759,10 +2759,17 @@ class BatchEngine:
                     cache_seed.trim_layers(cache_layers, excess)
             except Exception:  # noqa: BLE001 — snapshot path covers hybrids
                 pass
+            cache_tokens = full_prompt + [int(t) for t in sub.token_ids]
+            if cache_seed.all_trimmable(cache_layers):
+                cache_tokens = spec_decode.cached_token_prefix(cache_layers, cache_tokens)
+                if cache_tokens is None:
+                    # Leave any prior valid entry intact rather than persisting
+                    # cache layers whose token boundary cannot be established.
+                    cache_layers = None
             r = types.SimpleNamespace(
                 finish_reason=finish_reason,
-                prompt_cache=list(cache_layers),
-                all_tokens=full_prompt + [int(t) for t in sub.token_ids],
+                prompt_cache=list(cache_layers) if cache_layers is not None else None,
+                all_tokens=cache_tokens,
                 token=sub.token_ids[-1] if sub.token_ids else 0,
             )
             self._finish(sub, r)

package/dist/providers/mlx/python/spec_decode.py

@@ -117,8 +117,11 @@ def resolve_spec(args, model, text_tower: Optional[str], log=print) -> Optional[
     except Exception as exc:  # noqa: BLE001
         _off(log, f"drafter incompatible with target: {exc}")
         return None
-    if not install_chunked_draft_prefill(
-        drafter, int(getattr(args, "prefill_step_size", 0) or 0), log=log
+    if (
+        callable(getattr(drafter, "prefill_from_target_hidden", None))
+        and not install_chunked_draft_prefill(
+            drafter, int(getattr(args, "prefill_step_size", 0) or 0), log=log
+        )
     ):
         log(
             "[spec] drafter has no incremental prefill surface — long prompts "
@@ -240,6 +243,11 @@ def draft_prefill_gate(spec: Optional[SpecState], prompt_len: int) -> Tuple[bool
     throw that work away."""
     if spec is None:
         return True, ""
+    # Shared-KV drafters (e.g. Gemma's assistant) have no separate prompt
+    # prefill. Both round loops skip this optional hook, so there is no
+    # quadratic drafter prefill to limit for that capability shape.
+    if not callable(getattr(spec.drafter, "prefill_from_target_hidden", None)):
+        return True, ""
     if getattr(spec.drafter, "_gezel_chunked_prefill", False):
         return True, ""
     if int(prompt_len) > UNCHUNKED_DRAFT_PREFILL_LIMIT:
@@ -278,9 +286,21 @@ def cache_matches_tower(layers, tower) -> bool:
         return False
     if layers is None or len(layers) != len(expect):
         return False
-    return all(
-        type(a).__name__ == type(b).__name__ for a, b in zip(layers, expect)
-    )
+
+    def matches(actual, expected):
+        if type(actual).__name__ == type(expected).__name__:
+            return True
+        # Upstream MTP deliberately wraps a rotating cache in this subclass
+        # to roll back rejected drafts. It is still the same target layout;
+        # rejecting our own saved cache here forces every follow-up cold.
+        # Do not admit arbitrary subclasses (e.g. batch-shaped cache adapters).
+        return (
+            type(expected).__name__ == "RotatingKVCache"
+            and type(actual).__name__ == "BufferedRotatingKVCache"
+            and isinstance(actual, type(expected))
+        )
+
+    return all(matches(a, b) for a, b in zip(layers, expect))
 
 
 def clone_layers(layers) -> List[Any]:
@@ -294,6 +314,25 @@ def clone_layers(layers) -> List[Any]:
     return out
 
 
+def cached_token_prefix(layers, tokens: Sequence[int]) -> Optional[List[int]]:
+    """Describe only tokens actually fed into a trimmable target cache.
+
+    A final bonus token can be emitted before it is fed back to the target.
+    Saving it as cached makes the next seed fail its offset consistency guard.
+    Unknown or disagreeing offsets must remain a miss, never guessed state.
+    """
+    try:
+        offsets = [int(c.offset) for c in layers]
+    except (AttributeError, TypeError, ValueError):
+        return None
+    if not offsets or len(set(offsets)) != 1:
+        return None
+    count = offsets[0]
+    if count <= 0 or count > len(tokens):
+        return None
+    return list(tokens[:count])
+
+
 def chunked_prefill_steps(
     tower,
     cache,

package/dist/providers/native/capacity-broker.d.ts

@@ -31,7 +31,7 @@ export interface CapacityBrokerOptions {
      * tests and for users who want to opt out of the budget entirely.
      */
     budgetBytes?: number;
-    /** Test seam — defaults to {@link totalmem}. */
+    /** Test seam — defaults to {@link totalMemoryBytes}. */
     systemRamBytes?: () => number;
     /**
      * Whether this host shares one memory pool between CPU and GPU, which

package/dist/providers/native/capacity-broker.js

@@ -18,8 +18,8 @@
  */
 import { execFileSync } from 'node:child_process';
 import { readFileSync } from 'node:fs';
-import { freemem, totalmem } from 'node:os';
 import { CONSTRAINED_LOCAL_CONTEXT_FLOOR_TOKENS, LLAMA_CPP_COMPUTE_HEADROOM, LOCAL_CONTEXT_FLOOR_TOKENS, MLX_COMPUTE_HEADROOM, createLogger, estimateLlamaCppResidentBytes, estimateMlxResidentBytes, localContextFloorTokens, } from '@bendyline/gezel';
+import { capToContainer, freeMemoryBytes, totalMemoryBytes } from '../../system/host-memory.js';
 import { autoAllowRamSpillover, computeCapacityBudget, fastMemoryBudgetBytes, getDetectedGpuVramBytes, } from './capacity-budget.js';
 export { autoAllowRamSpillover, autoDetectBudgetBytes, computeCapacityBudget, getDetectedGpuVramBytes, pressureIdleGraceMs, setDetectedGpuVramBytes, } from './capacity-budget.js';
 export { fastMemoryBudgetBytes };
@@ -156,7 +156,7 @@ export class CapacityBroker {
     explicitAllowRamSpillover;
     reservations = new Map();
     constructor(opts = {}) {
-        this.systemRamBytes = (opts.systemRamBytes ?? totalmem)();
+        this.systemRamBytes = (opts.systemRamBytes ?? totalMemoryBytes)();
         this.unifiedMemory = opts.unifiedMemory;
         this.gpuVramBytes = opts.gpuVramBytes;
         this.reservedGpuBytes = opts.reservedGpuBytes;
@@ -467,24 +467,24 @@ export function availableSystemRamBytes() {
     // vm_stat categories overlap (file-backed pages span active+inactive),
     // so the raw sum can exceed physical RAM; cap at 90% of totalmem — the
     // consumers already hold back their own OS reserves on top.
-    const cap = totalmem() * 0.9;
+    const cap = totalMemoryBytes() * 0.9;
     try {
         if (process.platform === 'darwin') {
             const out = execFileSync('/usr/bin/vm_stat', { encoding: 'utf8', timeout: 3_000 });
             const parsed = parseVmStatAvailableBytes(out);
             if (parsed !== null)
-                return Math.max(Math.min(parsed, cap), freemem());
+                return capToContainer(Math.max(Math.min(parsed, cap), freeMemoryBytes()));
         }
         else if (process.platform === 'linux') {
             const parsed = parseMeminfoAvailableBytes(readFileSync('/proc/meminfo', 'utf8'));
             if (parsed !== null)
-                return Math.max(Math.min(parsed, cap), freemem());
+                return capToContainer(Math.max(Math.min(parsed, cap), freeMemoryBytes()));
         }
     }
     catch {
         // Probe failure falls through to the conservative number.
     }
-    return freemem();
+    return freeMemoryBytes();
 }
 /** Pure vm_stat parser (exported for tests). Returns null when the output
  *  doesn't look like vm_stat. */
@@ -664,7 +664,7 @@ export function llamaCppSlotCeiling(opts) {
  * going to free 5 GiB, because nothing was holding it but the estimate.
  * Planning and admission must read one number.
  */
-export function liveRamOsReserveBytes(totalRamBytes = totalmem()) {
+export function liveRamOsReserveBytes(totalRamBytes = totalMemoryBytes()) {
     return Math.min(4 * GIB, Math.max(2 * GIB, totalRamBytes * 0.1));
 }
 /**
@@ -769,7 +769,7 @@ export function minViableLocalContextTokens(opts) {
     const override = envMinContextTokens();
     if (override !== null)
         return override;
-    const systemRamBytes = opts?.systemRamBytes ?? totalmem();
+    const systemRamBytes = opts?.systemRamBytes ?? totalMemoryBytes();
     const budget = opts?.budget ?? computeCapacityBudget({ systemRamBytes });
     return localContextFloorTokens({
         totalRamBytes: systemRamBytes,

package/dist/providers/native/capacity-budget.js

@@ -1,4 +1,5 @@
-import { arch, platform, totalmem } from 'node:os';
+import { arch, platform } from 'node:os';
+import { totalMemoryBytes } from '../../system/host-memory.js';
 const GIB = 1024 ** 3;
 /**
  * Ceiling on the auto-derived SYSTEM-RAM share, however much RAM the host
@@ -152,7 +153,7 @@ export function isUnifiedMemoryHost() {
  * `GEZEL_CAPACITY_BUDGET_GB` overrides the total, including the setting.
  */
 export function computeCapacityBudget(input = {}) {
-    const systemRamBytes = input.systemRamBytes ?? totalmem();
+    const systemRamBytes = input.systemRamBytes ?? totalMemoryBytes();
     // Reading the ambient probe is for callers that know nothing about the
     // host. One that passes `unifiedMemory` has already described it.
     const measuredVram = input.gpuVramBytes !== undefined
@@ -269,7 +270,7 @@ function envOverrideBytes() {
  * (Settings → the engine pages) is the dial for owners who want that traded
  * either way.
  */
-export function autoDetectBudgetBytes(systemRamBytes = totalmem(), opts = {}) {
+export function autoDetectBudgetBytes(systemRamBytes = totalMemoryBytes(), opts = {}) {
     return computeCapacityBudget({ systemRamBytes, ...opts }).budgetBytes;
 }
 /**
@@ -277,7 +278,7 @@ export function autoDetectBudgetBytes(systemRamBytes = totalmem(), opts = {}) {
  * the whole budget otherwise. KV slots, prompt caches, and anything else
  * that must live where the compute happens size against this.
  */
-export function fastMemoryBudgetBytes(systemRamBytes = totalmem(), opts = {}) {
+export function fastMemoryBudgetBytes(systemRamBytes = totalMemoryBytes(), opts = {}) {
     return computeCapacityBudget({ systemRamBytes, ...opts }).fastBytes;
 }
 /** Fast pool at or below which a host is treated as accelerator-constrained. */

package/dist/providers/native/device-capacity-ledger.d.ts

@@ -1,4 +1,5 @@
 import type { NativeCapacityCommand, NativeCapacityReply } from '@bendyline/gezel';
+import { readProcessIdentities } from './process-identity.js';
 export interface DeviceCapacitySample {
     budgetBytes: number;
     gpuBudgetBytes: number;
@@ -25,6 +26,7 @@ export declare class DeviceCapacityLedger {
         directory: string;
         sample: () => Promise<DeviceCapacitySample>;
         alive?: (pid: number) => boolean;
+        identities?: typeof readProcessIdentities;
     });
     execute(command: NativeCapacityCommand): Promise<NativeCapacityReply>;
     private transact;