@bendyline/gezel-inference 1.0.8 → 1.0.9

package/dist/providers/ds4/residency.js

@@ -6,6 +6,10 @@ export { DS4_FULL_RESIDENCY_HEADROOM_BYTES };
  * Conservative floor for the full-residency working set. Exclusivity is an
  * explicit device lease at process startup, not a consequence of this number:
  * the automatic model budget can exceed 96 GiB on larger machines.
+ *
+ * Only for a launch nobody measured: an entry with `fullResidencyMemory` is
+ * reserved at that working set ({@link planDs4MeasuredLaunch}). Flooring a
+ * 45 GiB Qwen3.8 launch here put it over a 96 GiB Mac's 80 GiB budget.
  */
 export const DS4_FULL_RESIDENCY_RESERVATION_BYTES = 96 * GB;
 /**
@@ -80,20 +84,97 @@ export function shouldUseDs4SsdStreaming(opts = {}) {
  * the old RAM-tier heuristic which incorrectly gave both models 64 GiB on a
  * 128 GiB Mac. Manual values are still accepted, but clamped so fixed model
  * state + the cache cannot consume the runtime/OS reserve.
+ *
+ * The 24 GiB headroom guards a footprint guessed as `residentBytes −
+ * cacheExpertsBytes`, which omits graph and staging memory ds4 also holds.
+ * An entry with a measured streamed footprint has no such gap, so its cache
+ * may use whatever this device's local-model budget leaves beside it. A fixed
+ * 24 GiB refused Qwen3.8 on 32 GiB Macs whose budget held it with room to spare.
  */
 export function planDs4ExpertCache(opts) {
     const totalRamBytes = opts.totalRamBytes ?? totalMemoryBytes();
     const catalogCacheBytes = opts.catalogCacheBytes ?? 0;
-    const fixedResidentBytes = Math.max(0, (opts.catalogResidentBytes ?? 0) - catalogCacheBytes);
     const fallbackGb = totalRamBytes >= 120 * GB ? 64 : totalRamBytes >= 88 * GB ? 48 : 32;
     const requestedGb = opts.configuredGb ?? (catalogCacheBytes > 0 ? catalogCacheBytes / GB : fallbackGb);
-    const safeCacheBytes = totalRamBytes - fixedResidentBytes - DS4_FULL_RESIDENCY_HEADROOM_BYTES;
+    const modelBudget = opts.measuredFixedBytes !== undefined && opts.budgetBytes !== undefined;
+    const safeCacheBytes = modelBudget
+        ? opts.budgetBytes - opts.measuredFixedBytes
+        : totalRamBytes -
+            Math.max(0, (opts.catalogResidentBytes ?? 0) - catalogCacheBytes) -
+            DS4_FULL_RESIDENCY_HEADROOM_BYTES;
     const safeCacheGb = Math.floor(safeCacheBytes / GB);
-    if (safeCacheGb < 1) {
-        return { cacheGb: 1, requestedGb, clamped: true, safe: false };
+    const minCacheGb = Math.max(1, Math.ceil((opts.minCacheBytes ?? 0) / GB));
+    const limit = modelBudget ? 'model-budget' : 'headroom';
+    if (safeCacheGb < minCacheGb) {
+        return { cacheGb: minCacheGb, requestedGb, clamped: true, safe: false, limit };
+    }
+    const cacheGb = Math.max(minCacheGb, Math.min(requestedGb, safeCacheGb));
+    return { cacheGb, requestedGb, clamped: cacheGb !== requestedGb, safe: true, limit };
+}
+/** Evaluate a measured line at a window, never below its context-free part. */
+export function ds4MeasuredBytesAt(line, ctxTokens) {
+    const contextFree = Math.max(0, line.bytes - line.bytesPerToken * line.ctxTokens);
+    return Math.round(contextFree + line.bytesPerToken * Math.max(0, ctxTokens));
+}
+function fullResidencyLine(source) {
+    const m = source?.fullResidencyMemory;
+    return m
+        ? { bytes: m.workingSetBytes, ctxTokens: m.ctxTokens, bytesPerToken: m.bytesPerToken }
+        : undefined;
+}
+function streamingLine(source) {
+    const m = source?.streamingMemory;
+    return m
+        ? { bytes: m.fixedBytes, ctxTokens: m.ctxTokens, bytesPerToken: m.bytesPerToken }
+        : undefined;
+}
+/** Measured streamed footprint without the expert cache, at a window; undefined when unmeasured. */
+export function ds4StreamingFixedBytes(source, ctxTokens) {
+    const line = streamingLine(source);
+    return line ? ds4MeasuredBytesAt(line, ctxTokens) : undefined;
+}
+/**
+ * Price a launch from the entry's measurement for the chosen mode. Undefined
+ * when that mode is unmeasured (or a streamed launch has no budget to size
+ * against), so callers keep their catalog-`residentBytes` path, including the
+ * 96 GiB full-residency floor that stands in for a missing measurement.
+ */
+export function planDs4MeasuredLaunch(opts) {
+    const sidecarBytes = opts.sidecarBytes ?? 0;
+    if (!opts.ssdStreaming) {
+        const line = fullResidencyLine(opts.source);
+        if (!line)
+            return undefined;
+        const contextFreeBytes = ds4MeasuredBytesAt(line, 0) + sidecarBytes;
+        return {
+            workingSetBytes: contextFreeBytes + line.bytesPerToken * Math.max(0, opts.numCtx),
+            contextFreeBytes,
+            bytesPerToken: line.bytesPerToken,
+        };
     }
-    const cacheGb = Math.max(1, Math.min(requestedGb, safeCacheGb));
-    return { cacheGb, requestedGb, clamped: cacheGb !== requestedGb, safe: true };
+    const line = streamingLine(opts.source);
+    if (!line || opts.budgetBytes === undefined)
+        return undefined;
+    const fixedBytes = ds4MeasuredBytesAt(line, opts.numCtx) + sidecarBytes;
+    const cache = planDs4ExpertCache({
+        ...(opts.configuredCacheGb !== undefined ? { configuredGb: opts.configuredCacheGb } : {}),
+        ...(opts.source?.cacheExpertsBytes !== undefined
+            ? { catalogCacheBytes: opts.source.cacheExpertsBytes }
+            : {}),
+        totalRamBytes: opts.totalRamBytes,
+        measuredFixedBytes: fixedBytes,
+        budgetBytes: opts.budgetBytes,
+        ...(opts.source?.streamingMemory?.minCacheBytes !== undefined
+            ? { minCacheBytes: opts.source.streamingMemory.minCacheBytes }
+            : {}),
+    });
+    const cacheBytes = cache.cacheGb * GB;
+    return {
+        workingSetBytes: fixedBytes + cacheBytes,
+        contextFreeBytes: ds4MeasuredBytesAt(line, 0) + sidecarBytes + cacheBytes,
+        bytesPerToken: line.bytesPerToken,
+        cache,
+    };
 }
 /** Effective capacity-broker reservation for the chosen DS4 residency mode. */
 export function ds4ResidentBytesForMode(catalogResidentBytes, ssdStreaming, ssdStreamingSupported = true) {

package/dist/providers/image/sd-cpp.js

@@ -285,8 +285,9 @@ export class StableDiffusionCppProvider {
             this.launchState.modelId = modelId;
             await this.supervisor.stop();
         }
-        // A render the caller abandons keeps running inside sd-server, which
-        // cannot interrupt one it has started, so cancelling stops the engine.
+        // A render the caller abandons keeps running inside sd-server, whose
+        // `/sdapi` endpoints cannot interrupt one they have started, so
+        // cancelling stops the engine.
         const request = new AbortController();
         const stopRender = () => {
             request.abort();

package/dist/providers/llama-cpp/backend-health.d.ts

@@ -1,7 +1,8 @@
 /**
  * Whether llama-server is getting the hardware this machine has, as the
  * engine process sees it: the CUDA build discovery passed over for the
- * driver (`GEZEL_LLAMA_CUDA_BLOCKER`), a GPU build that started without its
+ * driver (`GEZEL_LLAMA_CUDA_BLOCKER`), the ROCm build it passed over for this
+ * machine's AMD GPUs (`GEZEL_LLAMA_ROCM_BLOCKER`), a GPU build that started without its
  * GPU (gpu-startup-watch.ts), and Linux's out-of-memory killer stopping the
  * engine. Each is invisible otherwise — the engine keeps answering, slower,
  * or a chat fails with a crash that reads like any other.
@@ -24,7 +25,7 @@ export declare function noteHostOutOfMemory(input: {
     model?: string | undefined;
     whileStarting: boolean;
 }, now?: Date): void;
-declare const BACKENDS: readonly ["cuda", "vulkan", "metal", "cpu"];
+declare const BACKENDS: readonly ["cuda", "rocm", "vulkan", "metal", "cpu"];
 /**
  * The backend this process's on-device llama-server runs, for catalog
  * backend exclusions: the build discovery resolved, else what the probe

package/dist/providers/llama-cpp/backend-health.js

@@ -1,7 +1,8 @@
 /**
  * Whether llama-server is getting the hardware this machine has, as the
  * engine process sees it: the CUDA build discovery passed over for the
- * driver (`GEZEL_LLAMA_CUDA_BLOCKER`), a GPU build that started without its
+ * driver (`GEZEL_LLAMA_CUDA_BLOCKER`), the ROCm build it passed over for this
+ * machine's AMD GPUs (`GEZEL_LLAMA_ROCM_BLOCKER`), a GPU build that started without its
  * GPU (gpu-startup-watch.ts), and Linux's out-of-memory killer stopping the
  * engine. Each is invisible otherwise — the engine keeps answering, slower,
  * or a chat fails with a crash that reads like any other.
@@ -9,7 +10,7 @@
  * Process-wide like the backend env it sits beside: every llama provider in
  * this process launches the same build.
  */
-import { missingMachineRuntime, readLlamaCudaBlocker, vcRedistributableUrl, } from '@bendyline/gezel/native';
+import { missingMachineRuntime, readLlamaCudaBlocker, readRocmBlocker, vcRedistributableUrl, } from '@bendyline/gezel/native';
 /**
  * Reading an engine's import tables takes tens of milliseconds of synchronous
  * file reads, and the answer only changes when someone installs the runtime.
@@ -61,7 +62,7 @@ export function noteHostOutOfMemory(input, now = new Date()) {
         whileStarting: input.whileStarting,
     };
 }
-const BACKENDS = ['cuda', 'vulkan', 'metal', 'cpu'];
+const BACKENDS = ['cuda', 'rocm', 'vulkan', 'metal', 'cpu'];
 /**
  * The backend this process's on-device llama-server runs, for catalog
  * backend exclusions: the build discovery resolved, else what the probe
@@ -85,6 +86,7 @@ export function llamaBackendHealthView() {
     const runningRaw = process.env.GEZEL_LLAMA_SERVER_BACKEND;
     const running = BACKENDS.find((backend) => backend === runningRaw);
     const cudaBlocker = readLlamaCudaBlocker();
+    const rocmBlocker = readRocmBlocker();
     // A verdict about a build the process has since moved off (a quarantine
     // fallback, a lifted quarantine) no longer describes what runs.
     const unused = gpuUnused && gpuUnused.backend === running ? gpuUnused : null;
@@ -92,6 +94,7 @@ export function llamaBackendHealthView() {
     return {
         ...(running ? { running } : {}),
         ...(cudaBlocker ? { cudaBlocker } : {}),
+        ...(rocmBlocker ? { rocmBlocker } : {}),
         ...(unused ? { gpuUnused: unused } : {}),
         ...(missingRuntime ? { missingRuntime } : {}),
         ...(outOfMemory ? { outOfMemory } : {}),

package/dist/providers/llama-cpp/backend-quarantine.js

@@ -11,7 +11,7 @@
  * next engine launch instead of the next app start.
  */
 import { LLAMA_QUARANTINE_STRIKES, clearLlamaQuarantine, createLlamaBuildCheck, isBinaryQuarantined, llamaVariantBinaryPath, publishLlamaCudaBlocker, readLlamaQuarantine, resolveLlamaFallbackBinary, resolveLlamaVariantBinary, } from '@bendyline/gezel/native';
-const BACKENDS = ['cuda', 'vulkan', 'metal', 'cpu'];
+const BACKENDS = ['cuda', 'rocm', 'vulkan', 'metal', 'cpu'];
 export function isLlamaBackend(value) {
     return BACKENDS.some((backend) => backend === value);
 }

package/dist/providers/llama-cpp/build-provider.js

@@ -1,10 +1,11 @@
-import { delimiter, dirname, join } from 'node:path';
-import { DEFAULT_LOCAL_ENGINE_IDLE_TIMEOUT_MS, createLogger, estimateLlamaCppResidentBytes, llamaCppBackendExclusion, llamaCppBackendLabel, } from '@bendyline/gezel';
-import { LLAMA_QUARANTINE_STRIKES, clearLlamaStartupStrikes, recordLlamaStartupCrash, unrunnableBuildExit, } from '@bendyline/gezel/native';
+import { basename, delimiter, dirname, join } from 'node:path';
+import { DEFAULT_LOCAL_ENGINE_IDLE_TIMEOUT_MS, createLogger, estimateLlamaCppResidentBytes, llamaCppBackendExclusion, llamaCppBackendLabel, llamaCppDraftResidentBytes, llamaCppSpecNeedsDraftModel, } from '@bendyline/gezel';
+import { LLAMA_QUARANTINE_STRIKES, clearLlamaStartupStrikes, recordLlamaStartupCrash, rocmLaunchEnv, unrunnableBuildExit, } from '@bendyline/gezel/native';
 import { gezelPaths } from '@bendyline/gezel/paths';
 import { mmprojBudgetBytes, nativeVisionEnabledFor } from '../../chat/vision-capability.js';
 import { effectiveEngineRelease, isEnginePinned } from '../../engines/native-manifest.js';
 import { engineInstallPhase } from '../../engines/registry.js';
+import { llamaDownloadVariant } from '../../engines/resolver.js';
 import { availableCpuCount, containerCpuLimit } from '../../system/host-memory.js';
 import { localDeviceSettingsLabel, noModelYetMessage, notReadyError, } from '../active-install-message.js';
 import { resolveCatalogLlamaCppEngineConfig, resolveCatalogReasoningBudget, } from '../catalog-model-config.js';
@@ -27,7 +28,7 @@ import { isGemmaModel, planLlamaCppKv, resolveLlamaCppKvCacheType, } from './kv-
 import { degradeDenseFfnOffloadDecision, degradeMoeOffloadDecision, estimateExactPerSlotKvBytesF16, estimateKvReserveBytes, estimateWindowedKvLinearization, planDenseFfnOffload, planMoeOffload, shedSlotsForGpuResidency, } from './offload-planner.js';
 import { LlamaCppProvider } from './provider.js';
 import { reasoningLaunchOverridesFromEnv } from './reasoning-launch.js';
-import { resolveSpecDraft } from './spec-draft.js';
+import { brokenDrafters, drafterCostsContext, drafterCostsGpuResidency, llamaCppSidecarLoads, resolveSpecDraft, } from './spec-draft.js';
 const log = createLogger('chat');
 function activeLlamaInstall(models) {
     return models?.getActiveInstalls().find((i) => i.phase !== undefined) ?? null;
@@ -50,7 +51,7 @@ export function ensureLlamaEngineStatus(registry, config) {
     const override = config.llamaCppBackendOverride;
     const backend = override && override !== 'auto'
         ? override
-        : process.env.GEZEL_LLAMA_DETECTED_BACKEND || undefined;
+        : llamaDownloadVariant(process.env.GEZEL_LLAMA_DETECTED_BACKEND || undefined);
     const { snapshot } = registry.ensure('llama-server', backend);
     if (snapshot.error) {
         return { detail: `On-device engine couldn't be downloaded: ${snapshot.error}`, failed: true };
@@ -283,10 +284,13 @@ export async function buildLlamaCppProvider(opts) {
     }
     // Install refuses an excluded backend; this catches a model installed
     // before the exclusion was authored, or copied in by hand.
+    let catalogDraftResidentBytes;
     if (modelCatalogInfo && opts.catalog) {
         const engineBackend = binaryBackend ?? currentLlamaCppBackend();
         const detail = await opts.catalog.get('chat-model', modelCatalogInfo.id).catch(() => null);
-        const excluded = llamaCppBackendExclusion(detail?.manifest.kind === 'chat-model' ? detail.manifest.llamaCpp : undefined, engineBackend);
+        const catalogSource = detail?.manifest.kind === 'chat-model' ? detail.manifest.llamaCpp : undefined;
+        catalogDraftResidentBytes = llamaCppDraftResidentBytes(catalogSource?.draftModel, modelCatalogInfo.draftModelPath);
+        const excluded = llamaCppBackendExclusion(catalogSource, engineBackend);
         if (excluded && engineBackend) {
             const err = new Error(`Local model: ${modelCatalogInfo.name} cannot run on the ${llamaCppBackendLabel(engineBackend)} llama.cpp engine. ${excluded} Pick another local model in Settings → ${localDeviceSettingsLabel()}.`);
             err.isActionable = true;
@@ -338,6 +342,41 @@ export async function buildLlamaCppProvider(opts) {
     const manifestEngineConfig = opts.catalog
         ? await resolveCatalogLlamaCppEngineConfig(opts.catalog, opts.modelOverride?.modelId ?? defaultModelId)
         : undefined;
+    // Speculative-decoding resolution — the catalog's per-backend overlay, and
+    // whether the drafter it names is on disk. Settled here, above the memory
+    // planning, for the same reason as vision: the launch and the budget must
+    // agree on whether a drafter is resident. See providers/llama-cpp/spec-draft.ts.
+    const llamaBuildMetadata = binary ? await readLlamaCppBuildMetadata(binary) : null;
+    const llamaBackend = llamaBuildMetadata?.backend ?? process.env.GEZEL_LLAMA_SERVER_BACKEND;
+    const resolveSpec = (installedDraftModelPath) => resolveSpecDraft({
+        perModel: manifestEngineConfig,
+        backend: llamaBackend,
+        configSpecType: config.llamaCppSpecType,
+        configDraftModelPath: config.llamaCppDraftModelPath,
+        installedDraftModelPath,
+        resolveDraftPath: (id) => opts.llamaCppModels ? opts.llamaCppModels.resolveModelPath(id) : Promise.resolve(null),
+    });
+    const brokenDrafterKey = `${binary ?? ''}|${modelCatalogInfo?.draftModelPath ?? ''}`;
+    let drafterInUse = brokenDrafters.has(brokenDrafterKey)
+        ? undefined
+        : modelCatalogInfo?.draftModelPath;
+    let specDraft = await resolveSpec(drafterInUse);
+    if (specDraft.log)
+        log[specDraft.log.level](specDraft.log.message);
+    let resolvedManifestEngineConfig = specDraft.perModel;
+    const specTypeNow = () => config.llamaCppSpecType ?? resolvedManifestEngineConfig?.spec?.type;
+    const draftBudgetBytes = () => (drafterInUse ? (modelCatalogInfo?.draftModelSizeBytes ?? 0) : 0);
+    // A drafter the catalog chose, which this launch can run without. One someone
+    // forced stays theirs to answer for.
+    const catalogDrafterLoaded = () => !!drafterInUse &&
+        !config.llamaCppSpecType &&
+        !config.llamaCppDraftModelPath &&
+        llamaCppSpecNeedsDraftModel(resolvedManifestEngineConfig?.spec?.type);
+    const leaveDrafterOut = async () => {
+        drafterInUse = undefined;
+        specDraft = await resolveSpec(undefined);
+        resolvedManifestEngineConfig = specDraft.perModel;
+    };
     const PREFERRED_CTX_DEFAULT = 65_536;
     // Explicit numeric values win: env, then the per-model override, then the
     // machine-wide config. Otherwise Adaptive uses the per-model manifest
@@ -413,6 +452,13 @@ export async function buildLlamaCppProvider(opts) {
             headerSummary = null;
         }
     }
+    if (!llamaCppSidecarLoads({
+        specType: specTypeNow(),
+        targetHasMtp: (headerSummary?.nextnPredictLayers ?? 0) > 0,
+        perBackend: !!manifestEngineConfig?.spec?.byBackend,
+    })) {
+        drafterInUse = undefined;
+    }
     const exactPerSlotKvF16 = headerSummary
         ? estimateExactPerSlotKvBytesF16({
             blockCount: headerSummary.blockCount,
@@ -462,8 +508,10 @@ export async function buildLlamaCppProvider(opts) {
     // contract in planLlamaCppKv. Priced from the cached exact GGUF walk above,
     // including any v0.4 lazy-tensor deduction.
     const PLAN_REFERENCE_CTX = 4096;
-    const planResidentBytes = estimateLlamaCppResidentBytes(planWeightsBytes, {
+    const planResidentBytes = () => estimateLlamaCppResidentBytes(planWeightsBytes, {
         mmprojBytes: visionBudgetBytes,
+        draftBytes: draftBudgetBytes(),
+        draftResidentBytes: catalogDraftResidentBytes,
     });
     const planCapacity = hostCapacity;
     const planKvBytesPerToken = (kv) => {
@@ -506,7 +554,7 @@ export async function buildLlamaCppProvider(opts) {
             slots: slotCount,
             minimumPerTurnCtxTokens: ctxTokens,
             kvBytesPerToken: planKvBytesPerToken(kv),
-            weightsResidentBytes: planResidentBytes,
+            weightsResidentBytes: planResidentBytes(),
             budgetBytes: brokerSnap?.enforced ? brokerSnap.budgetBytes : planCapacity.budgetBytes,
             committedOtherBytes,
             freeSystemRamBytes: availableSystemRamBytes(),
@@ -514,7 +562,51 @@ export async function buildLlamaCppProvider(opts) {
         });
         return probe.minimumSatisfied && probe.slots >= slotCount;
     };
-    const kvPlan = planLlamaCppKv({
+    // One slot's KV at `ctxTokens`, priced the way the launch reservation
+    // prices it below: the windowed cache unless `--swa-full` is pinned.
+    const planSlotKvReserveBytes = (kv, ctxTokens) => {
+        if (!headerSummary)
+            return planKvBytesPerToken(kv) * ctxTokens;
+        const geometry = {
+            blockCount: headerSummary.blockCount,
+            embeddingLength: headerSummary.embeddingLength,
+            headCount: headerSummary.headCount,
+            headCountKv: headerSummary.headCountKv,
+            headCountKvPerLayer: headerSummary.headCountKvPerLayer,
+            slidingWindowPattern: headerSummary.slidingWindowPattern,
+            sharedKvLayers: headerSummary.sharedKvLayers,
+            loopCount: headerSummary.loopCount,
+            keyLength: headerSummary.keyLength,
+            valueLength: headerSummary.valueLength,
+            keyLengthSwa: headerSummary.keyLengthSwa,
+            valueLengthSwa: headerSummary.valueLengthSwa,
+            fullAttentionInterval: headerSummary.fullAttentionInterval,
+            ssmInnerSize: headerSummary.ssmInnerSize,
+            ssmStateSize: headerSummary.ssmStateSize,
+            ssmConvKernel: headerSummary.ssmConvKernel,
+            kvCacheType: kv,
+        };
+        const windowed = (config.llamaCppSwaFull ?? manifestEngineConfig?.swaFull ?? false)
+            ? undefined
+            : estimateWindowedKvLinearization({
+                ...geometry,
+                slidingWindow: headerSummary.slidingWindow,
+            });
+        return windowed
+            ? windowed.fixedBytes + windowed.bytesPerToken * ctxTokens
+            : (estimateKvReserveBytes({ ...geometry, ctxTokens }) ?? planKvBytesPerToken(kv) * ctxTokens);
+    };
+    // What a discrete card leaves this launch now: free graphics memory as
+    // device admission measures it, within the card's usable share. Undefined
+    // off discrete cards, where there is no separate pool to run out of.
+    const planGpuPoolBytes = async () => {
+        const kind = brokerSnap?.enforced ? brokerSnap.pools.kind : hostCapacity.kind;
+        if (kind !== 'discrete-gpu')
+            return undefined;
+        const freeGpu = await measureAvailableGpuBytes().catch(() => undefined);
+        return Math.max(0, Math.min(budgetBytes - committedOtherBytes, freeGpu ?? Number.POSITIVE_INFINITY));
+    };
+    const planKv = () => planLlamaCppKv({
         architecture: modelCatalogInfo?.architecture,
         modelId: defaultModelId ?? undefined,
         override: config.llamaCppKvCacheType,
@@ -527,6 +619,42 @@ export async function buildLlamaCppProvider(opts) {
         fitsAt: planFitsAt,
         maxSlots: defaultLocalEngineSlots(budgetBytes),
     });
+    let kvPlan = planKv();
+    // A drafter buys speed; context is what a session runs out of. When a
+    // drafter the catalog chose would cap the target window or cost its KV
+    // precision, it goes and the model decodes the way it does without one.
+    // Growth above the target, sized later, simply has less room. On a
+    // discrete card it must also leave the model wholly on the GPU in the
+    // memory free now, which is what device admission charges the launch.
+    if (draftBudgetBytes() > 0 && !config.llamaCppSpecType && !config.llamaCppDraftModelPath) {
+        const targetCtx = kvPlan.ctxCapTokens ?? effectiveNumCtx;
+        const fitsWithDrafter = planFitsAt(kvPlan.kvCacheType, targetCtx, 1);
+        const drafter = drafterInUse;
+        drafterInUse = undefined;
+        const withoutDrafter = planKv();
+        drafterInUse = drafter;
+        let drafterCost;
+        if (drafterCostsContext(kvPlan, withoutDrafter, fitsWithDrafter)) {
+            drafterCost = `beside it the ${effectiveNumCtx}-token window would not keep its size and KV precision`;
+        }
+        else {
+            const gpuPoolBytes = await planGpuPoolBytes();
+            const withDrafterBytes = planResidentBytes() + planSlotKvReserveBytes(kvPlan.kvCacheType, targetCtx);
+            if (drafterCostsGpuResidency(withDrafterBytes, gpuPoolBytes)) {
+                drafterCost = `beside it the model needs ${(withDrafterBytes / 1024 ** 3).toFixed(1)} GiB of graphics memory and ${((gpuPoolBytes ?? 0) / 1024 ** 3).toFixed(1)} GiB is available`;
+            }
+        }
+        if (drafterCost) {
+            log.info(`[llama-cpp] ${modelCatalogInfo?.id ?? defaultModelId ?? 'model'}: leaving the speculative drafter out; ${drafterCost}`);
+            await leaveDrafterOut();
+            kvPlan = withoutDrafter;
+        }
+    }
+    // What a loaded drafter holds on the GPU, as the resident estimate prices it.
+    const drafterGpuBytes = () => estimateLlamaCppResidentBytes(0, {
+        draftBytes: draftBudgetBytes(),
+        draftResidentBytes: catalogDraftResidentBytes,
+    }) - estimateLlamaCppResidentBytes(0);
     kvCacheType = kvPlan.kvCacheType;
     if (kvPlan.upgraded) {
         log.info(`[llama-cpp] ${modelCatalogInfo?.id ?? defaultModelId ?? 'model'}: trading f16 KV for q8_0 to fit a second engine slot (single-slot SWA session alternation re-prefills wholesale; KV A/B 2026-08-03 showed no measurable q8_0 fidelity cost)`);
@@ -552,7 +680,7 @@ export async function buildLlamaCppProvider(opts) {
     // concurrent requests correctly with zero engine errors, so the failure this
     // clamp guards against looks stale — but one probe is not grounds for
     // deleting a safety measure, so it stays for anyone who opted in explicitly.
-    const selectedSpecType = config.llamaCppSpecType ?? manifestEngineConfig?.spec?.type;
+    const selectedSpecType = specTypeNow();
     if (selectedSpecType === 'draft-mtp' && slots > 1) {
         log.info(`[llama-cpp] limiting ${modelCatalogInfo?.id ?? 'MTP model'} to one slot because draft-mtp is not reliable with --parallel > 1 in the bundled engine`);
         slots = 1;
@@ -614,18 +742,6 @@ export async function buildLlamaCppProvider(opts) {
         : catalogReasoningBudgetTokens !== undefined
             ? 'catalog'
             : 'unbounded';
-    // Speculative-decoding draft resolution — a manifest names its draft by
-    // catalog id; resolve it to the installed GGUF path (or disable) before the
-    // argv is built. See providers/llama-cpp/spec-draft.ts for the full rules.
-    const specDraft = await resolveSpecDraft({
-        perModel: manifestEngineConfig,
-        configSpecType: config.llamaCppSpecType,
-        configDraftModelPath: config.llamaCppDraftModelPath,
-        resolveDraftPath: (id) => opts.llamaCppModels ? opts.llamaCppModels.resolveModelPath(id) : Promise.resolve(null),
-    });
-    const resolvedManifestEngineConfig = specDraft.perModel;
-    if (specDraft.log)
-        log[specDraft.log.level](specDraft.log.message);
     // Phase v4 — hardware-aware structured-weight offload. Probe device VRAM (cached to
     // `engines/llama-cpp/devices.json` via `--list-devices`) and read the
     // model's MoE metadata from its GGUF header; the planner decides
@@ -654,7 +770,6 @@ export async function buildLlamaCppProvider(opts) {
     // cross-check for an explicit draft-mtp request.
     let ggufHasMtp = false;
     let mtpLayerCount = 0;
-    const llamaBuildMetadata = binary ? await readLlamaCppBuildMetadata(binary) : null;
     let llamaGpuDevice = null;
     let nvidiaRuntimeDevice;
     let llamaDevices = [];
@@ -677,8 +792,10 @@ export async function buildLlamaCppProvider(opts) {
             offloadBlockCount = summary.blockCount;
             mtpLayerCount = summary.nextnPredictLayers ?? 0;
             ggufHasMtp = mtpLayerCount > 0;
-            if (!ggufHasMtp && modelCatalogInfo?.draftModelPath) {
-                const draftSummary = await readGgufSummaryAsync(modelCatalogInfo.draftModelPath);
+            // Only a companion this launch loads can supply the head: one left out
+            // for memory would turn MTP on with nothing to run it.
+            if (!ggufHasMtp && drafterInUse) {
+                const draftSummary = await readGgufSummaryAsync(drafterInUse);
                 mtpLayerCount = draftSummary.nextnPredictLayers ?? 0;
                 ggufHasMtp = mtpLayerCount > 0;
             }
@@ -687,6 +804,8 @@ export async function buildLlamaCppProvider(opts) {
             const residentWeightBytes = Math.max(0, approxBytes - lazyTensorBytes);
             const residentBytes = estimateLlamaCppResidentBytes(residentWeightBytes, {
                 mmprojBytes: visionBudgetBytes,
+                draftBytes: draftBudgetBytes(),
+                draftResidentBytes: catalogDraftResidentBytes,
             });
             if (lazyTensorBytes > 0) {
                 log.info(`[llama-cpp] ${modelCatalogInfo?.id ?? defaultModelId ?? 'model'}: lazy-reading ${(lazyTensorBytes / 1024 ** 3).toFixed(1)} GiB of row-addressable embeddings from the GGUF; budgeting ${(residentWeightBytes / 1024 ** 3).toFixed(1)} GiB as resident model weights`);
@@ -1025,6 +1144,7 @@ export async function buildLlamaCppProvider(opts) {
                                 split,
                                 ...(summary.blockCount !== undefined ? { blockCount: summary.blockCount } : {}),
                                 ...(kvReserveBytes !== undefined ? { kvReserveBytes } : {}),
+                                additionalGpuBytes: drafterGpuBytes(),
                             }
                             : {}),
                     });
@@ -1063,7 +1183,7 @@ export async function buildLlamaCppProvider(opts) {
                             ...(denseFfnSplit ? { split: denseFfnSplit } : {}),
                             ...(summary.blockCount !== undefined ? { blockCount: summary.blockCount } : {}),
                             ...(kvReserveBytes !== undefined ? { kvReserveBytes } : {}),
-                            additionalGpuBytes: visionBudgetBytes,
+                            additionalGpuBytes: visionBudgetBytes + drafterGpuBytes(),
                             ramBudgetBytes: brokerSnap?.enforced
                                 ? brokerSnap.pools.ramShareBytes
                                 : hostCapacity.ramShareBytes,
@@ -1100,8 +1220,8 @@ export async function buildLlamaCppProvider(opts) {
             // metadata gate remains load-bearing even though supported GGUFs are
             // now default-on.
             if (ggufHasMtp) {
-                log.info(`[llama-cpp] ${modelCatalogInfo?.id ?? 'model'} ships an MTP head (nextn_predict_layers=${mtpLayerCount}); speculative decoding ${config.llamaCppSpecType || manifestEngineConfig?.spec?.type
-                    ? 'follows the explicit setting'
+                log.info(`[llama-cpp] ${modelCatalogInfo?.id ?? 'model'} ships an MTP head (nextn_predict_layers=${mtpLayerCount}); speculative decoding ${specTypeNow()
+                    ? `runs ${specTypeNow()} (${config.llamaCppSpecType ? 'Settings' : 'catalog'})`
                     : 'is ON by default (+4-19% decode; disable with llamaCppSpecType: "none")'}.`);
             }
         }
@@ -1183,6 +1303,8 @@ export async function buildLlamaCppProvider(opts) {
             providerHolder.current?.getCacheAdapter()?.resetEngineState();
             if (snapshot.expected)
                 return;
+            if (catalogDrafterLoaded())
+                brokenDrafters.add(brokenDrafterKey);
             logFile.writeIncident(snapshot);
             if (snapshot.panicKind === 'host-out-of-memory') {
                 noteHostOutOfMemory({
@@ -1264,6 +1386,15 @@ export async function buildLlamaCppProvider(opts) {
         // restarts of this provider, so a recovered engine doesn't re-OOM on its
         // next boot.
         recoverStartup: ({ panicKind }) => {
+            // The drafter is the one part of this launch it can do without, and
+            // drafters have changed format between engine releases. Without this a
+            // bad one reads as a build that cannot run, and three of those quarantine
+            // the whole GPU backend; `resolveLaunch` drops it from the retry.
+            if (catalogDrafterLoaded()) {
+                brokenDrafters.add(brokenDrafterKey);
+                log.warn(`[llama-cpp] ${panicKind ?? 'failed start'} while starting ${modelCatalogInfo?.id ?? 'model'} with its speculative drafter — retrying without it`);
+                return true;
+            }
             if (panicKind !== 'cuda-out-of-memory' && panicKind !== 'vulkan-out-of-memory') {
                 return false;
             }
@@ -1318,6 +1449,9 @@ export async function buildLlamaCppProvider(opts) {
             return true;
         },
         resolveLaunch: async () => {
+            if (catalogDrafterLoaded() && brokenDrafters.has(brokenDrafterKey)) {
+                await leaveDrafterOut();
+            }
             gpuWatch = new GpuStartupWatch(binaryBackend);
             // Again at every launch, not only at build: llama.cpp from the 0.1.51
             // release refuses a --slot-save-path that does not exist ("not a
@@ -1345,7 +1479,7 @@ export async function buildLlamaCppProvider(opts) {
                 kvCacheType,
                 slots,
                 ggufHasMtp,
-                installedDraftModelPath: modelCatalogInfo?.draftModelPath,
+                installedDraftModelPath: drafterInUse,
                 // Opt-in A/B lever: `GEZEL_LLAMA_REASONING_FORMAT=none`
                 // disables llama-server's chat-template output parsing so mangled
                 // model output reaches `delta.content` for provider-side salvage.
@@ -1483,6 +1617,8 @@ export async function buildLlamaCppProvider(opts) {
                     ubatchSize: lastArgValue(resolvedAdvancedArgs, '--ubatch-size') ?? 'default',
                     flashAttention: lastArgValue(resolvedAdvancedArgs, '--flash-attn') ?? 'default',
                     threads: lastArgValue(resolvedAdvancedArgs, '--threads') ?? 'default',
+                    specType: lastArgValue(resolvedAdvancedArgs, '--spec-type') ?? 'default',
+                    specDraftModel: basename(lastArgValue(resolvedAdvancedArgs, '--spec-draft-model') ?? 'none'),
                     reasoningBudgetTokens: reasoningBudgetTokens ?? 'unbounded',
                     reasoningBudgetSource,
                     reasoningPreserve,
@@ -1493,6 +1629,9 @@ export async function buildLlamaCppProvider(opts) {
                     // Equivalent to `--api-key`, but out of `argv` so it does not show
                     // up in `ps` on a shared machine.
                     LLAMA_API_KEY: engineApiKey(),
+                    // A ROCm build's kernel packs, rocBLAS routing, and the AMD GPUs it
+                    // has no code for kept out of sight (rocm-devices.ts).
+                    ...rocmLaunchEnv(home, binary, llamaBuildMetadata?.backend ?? binaryBackend),
                 },
                 baseUrl: `http://127.0.0.1:${port}`,
             };

package/dist/providers/llama-cpp/devices.js

@@ -19,7 +19,7 @@ import { execFile } from 'node:child_process';
 import { existsSync, mkdirSync, readFileSync, statSync, writeFileSync } from 'node:fs';
 import { delimiter, dirname, join } from 'node:path';
 import { promisify } from 'node:util';
-import { windowsHeadlessSpawnOptions } from '@bendyline/gezel/native';
+import { rocmLaunchEnv, windowsHeadlessSpawnOptions } from '@bendyline/gezel/native';
 import { gezelHome } from '@bendyline/gezel/paths';
 import { totalMemoryBytes } from '../../system/host-memory.js';
 const execFileAsync = promisify(execFile);
@@ -191,6 +191,9 @@ export async function probeLlamaDevices(opts) {
             env: {
                 ...process.env,
                 PATH: inheritedPath ? `${binDir}${delimiter}${inheritedPath}` : binDir,
+                // A ROCm build lists only the GPUs its launch will see, so the VRAM
+                // the planner reads is the VRAM the engine gets.
+                ...rocmLaunchEnv(home, binaryPath),
             },
             // This is an awaited probe, not an independent process. Hide its
             // Windows console without detaching it from the daemon.

package/dist/providers/llama-cpp/engine-flags.d.ts

@@ -64,6 +64,12 @@ export interface PerModelLlamaCppEngineConfig {
         mtp?: boolean;
         draftModelId?: string;
         nMax?: number;
+        sampling?: 'greedy' | 'probabilistic';
+        byBackend?: Partial<Record<string, {
+            type?: string;
+            nMax?: number;
+            sampling?: 'greedy' | 'probabilistic';
+        }>>;
     };
 }
 /** Phase v2 hardware-aware offload decision (lowest precedence). */
@@ -106,7 +112,8 @@ export interface EngineFlagInput {
     /**
      * Absolute path to a catalog-installed speculative draft sidecar.
      * Separate MTP assistants (Gemma) and other draft algorithms need this;
-     * combined MTP GGUFs (Qwen) intentionally leave it unset.
+     * combined MTP GGUFs (Qwen) leave it unset, even with a DFlash drafter on
+     * disk: `--spec-draft-model` would make MTP load the wrong file.
      */
     installedDraftModelPath?: string | undefined;
     /**