@bendyline/gezel-inference 1.0.7 → 1.0.9

package/dist/providers/llama-cpp/build-provider.js

@@ -1,10 +1,11 @@
-import { delimiter, dirname, join } from 'node:path';
-import { DEFAULT_LOCAL_ENGINE_IDLE_TIMEOUT_MS, createLogger, estimateLlamaCppResidentBytes, llamaCppBackendExclusion, llamaCppBackendLabel, } from '@bendyline/gezel';
-import { LLAMA_QUARANTINE_STRIKES, clearLlamaStartupStrikes, recordLlamaStartupCrash, unrunnableBuildExit, } from '@bendyline/gezel/native';
+import { basename, delimiter, dirname, join } from 'node:path';
+import { DEFAULT_LOCAL_ENGINE_IDLE_TIMEOUT_MS, createLogger, estimateLlamaCppResidentBytes, llamaCppBackendExclusion, llamaCppBackendLabel, llamaCppDraftResidentBytes, llamaCppSpecNeedsDraftModel, } from '@bendyline/gezel';
+import { LLAMA_QUARANTINE_STRIKES, clearLlamaStartupStrikes, recordLlamaStartupCrash, rocmLaunchEnv, unrunnableBuildExit, } from '@bendyline/gezel/native';
 import { gezelPaths } from '@bendyline/gezel/paths';
 import { mmprojBudgetBytes, nativeVisionEnabledFor } from '../../chat/vision-capability.js';
 import { effectiveEngineRelease, isEnginePinned } from '../../engines/native-manifest.js';
 import { engineInstallPhase } from '../../engines/registry.js';
+import { llamaDownloadVariant } from '../../engines/resolver.js';
 import { availableCpuCount, containerCpuLimit } from '../../system/host-memory.js';
 import { localDeviceSettingsLabel, noModelYetMessage, notReadyError, } from '../active-install-message.js';
 import { resolveCatalogLlamaCppEngineConfig, resolveCatalogReasoningBudget, } from '../catalog-model-config.js';
@@ -27,7 +28,7 @@ import { isGemmaModel, planLlamaCppKv, resolveLlamaCppKvCacheType, } from './kv-
 import { degradeDenseFfnOffloadDecision, degradeMoeOffloadDecision, estimateExactPerSlotKvBytesF16, estimateKvReserveBytes, estimateWindowedKvLinearization, planDenseFfnOffload, planMoeOffload, shedSlotsForGpuResidency, } from './offload-planner.js';
 import { LlamaCppProvider } from './provider.js';
 import { reasoningLaunchOverridesFromEnv } from './reasoning-launch.js';
-import { resolveSpecDraft } from './spec-draft.js';
+import { brokenDrafters, drafterCostsContext, drafterCostsGpuResidency, llamaCppSidecarLoads, resolveSpecDraft, } from './spec-draft.js';
 const log = createLogger('chat');
 function activeLlamaInstall(models) {
     return models?.getActiveInstalls().find((i) => i.phase !== undefined) ?? null;
@@ -50,7 +51,7 @@ export function ensureLlamaEngineStatus(registry, config) {
     const override = config.llamaCppBackendOverride;
     const backend = override && override !== 'auto'
         ? override
-        : process.env.GEZEL_LLAMA_DETECTED_BACKEND || undefined;
+        : llamaDownloadVariant(process.env.GEZEL_LLAMA_DETECTED_BACKEND || undefined);
     const { snapshot } = registry.ensure('llama-server', backend);
     if (snapshot.error) {
         return { detail: `On-device engine couldn't be downloaded: ${snapshot.error}`, failed: true };
@@ -283,10 +284,13 @@ export async function buildLlamaCppProvider(opts) {
     }
     // Install refuses an excluded backend; this catches a model installed
     // before the exclusion was authored, or copied in by hand.
+    let catalogDraftResidentBytes;
     if (modelCatalogInfo && opts.catalog) {
         const engineBackend = binaryBackend ?? currentLlamaCppBackend();
         const detail = await opts.catalog.get('chat-model', modelCatalogInfo.id).catch(() => null);
-        const excluded = llamaCppBackendExclusion(detail?.manifest.kind === 'chat-model' ? detail.manifest.llamaCpp : undefined, engineBackend);
+        const catalogSource = detail?.manifest.kind === 'chat-model' ? detail.manifest.llamaCpp : undefined;
+        catalogDraftResidentBytes = llamaCppDraftResidentBytes(catalogSource?.draftModel, modelCatalogInfo.draftModelPath);
+        const excluded = llamaCppBackendExclusion(catalogSource, engineBackend);
         if (excluded && engineBackend) {
             const err = new Error(`Local model: ${modelCatalogInfo.name} cannot run on the ${llamaCppBackendLabel(engineBackend)} llama.cpp engine. ${excluded} Pick another local model in Settings → ${localDeviceSettingsLabel()}.`);
             err.isActionable = true;
@@ -338,6 +342,41 @@ export async function buildLlamaCppProvider(opts) {
     const manifestEngineConfig = opts.catalog
         ? await resolveCatalogLlamaCppEngineConfig(opts.catalog, opts.modelOverride?.modelId ?? defaultModelId)
         : undefined;
+    // Speculative-decoding resolution — the catalog's per-backend overlay, and
+    // whether the drafter it names is on disk. Settled here, above the memory
+    // planning, for the same reason as vision: the launch and the budget must
+    // agree on whether a drafter is resident. See providers/llama-cpp/spec-draft.ts.
+    const llamaBuildMetadata = binary ? await readLlamaCppBuildMetadata(binary) : null;
+    const llamaBackend = llamaBuildMetadata?.backend ?? process.env.GEZEL_LLAMA_SERVER_BACKEND;
+    const resolveSpec = (installedDraftModelPath) => resolveSpecDraft({
+        perModel: manifestEngineConfig,
+        backend: llamaBackend,
+        configSpecType: config.llamaCppSpecType,
+        configDraftModelPath: config.llamaCppDraftModelPath,
+        installedDraftModelPath,
+        resolveDraftPath: (id) => opts.llamaCppModels ? opts.llamaCppModels.resolveModelPath(id) : Promise.resolve(null),
+    });
+    const brokenDrafterKey = `${binary ?? ''}|${modelCatalogInfo?.draftModelPath ?? ''}`;
+    let drafterInUse = brokenDrafters.has(brokenDrafterKey)
+        ? undefined
+        : modelCatalogInfo?.draftModelPath;
+    let specDraft = await resolveSpec(drafterInUse);
+    if (specDraft.log)
+        log[specDraft.log.level](specDraft.log.message);
+    let resolvedManifestEngineConfig = specDraft.perModel;
+    const specTypeNow = () => config.llamaCppSpecType ?? resolvedManifestEngineConfig?.spec?.type;
+    const draftBudgetBytes = () => (drafterInUse ? (modelCatalogInfo?.draftModelSizeBytes ?? 0) : 0);
+    // A drafter the catalog chose, which this launch can run without. One someone
+    // forced stays theirs to answer for.
+    const catalogDrafterLoaded = () => !!drafterInUse &&
+        !config.llamaCppSpecType &&
+        !config.llamaCppDraftModelPath &&
+        llamaCppSpecNeedsDraftModel(resolvedManifestEngineConfig?.spec?.type);
+    const leaveDrafterOut = async () => {
+        drafterInUse = undefined;
+        specDraft = await resolveSpec(undefined);
+        resolvedManifestEngineConfig = specDraft.perModel;
+    };
     const PREFERRED_CTX_DEFAULT = 65_536;
     // Explicit numeric values win: env, then the per-model override, then the
     // machine-wide config. Otherwise Adaptive uses the per-model manifest
@@ -413,6 +452,13 @@ export async function buildLlamaCppProvider(opts) {
             headerSummary = null;
         }
     }
+    if (!llamaCppSidecarLoads({
+        specType: specTypeNow(),
+        targetHasMtp: (headerSummary?.nextnPredictLayers ?? 0) > 0,
+        perBackend: !!manifestEngineConfig?.spec?.byBackend,
+    })) {
+        drafterInUse = undefined;
+    }
     const exactPerSlotKvF16 = headerSummary
         ? estimateExactPerSlotKvBytesF16({
             blockCount: headerSummary.blockCount,
@@ -462,8 +508,10 @@ export async function buildLlamaCppProvider(opts) {
     // contract in planLlamaCppKv. Priced from the cached exact GGUF walk above,
     // including any v0.4 lazy-tensor deduction.
     const PLAN_REFERENCE_CTX = 4096;
-    const planResidentBytes = estimateLlamaCppResidentBytes(planWeightsBytes, {
+    const planResidentBytes = () => estimateLlamaCppResidentBytes(planWeightsBytes, {
         mmprojBytes: visionBudgetBytes,
+        draftBytes: draftBudgetBytes(),
+        draftResidentBytes: catalogDraftResidentBytes,
     });
     const planCapacity = hostCapacity;
     const planKvBytesPerToken = (kv) => {
@@ -506,7 +554,7 @@ export async function buildLlamaCppProvider(opts) {
             slots: slotCount,
             minimumPerTurnCtxTokens: ctxTokens,
             kvBytesPerToken: planKvBytesPerToken(kv),
-            weightsResidentBytes: planResidentBytes,
+            weightsResidentBytes: planResidentBytes(),
             budgetBytes: brokerSnap?.enforced ? brokerSnap.budgetBytes : planCapacity.budgetBytes,
             committedOtherBytes,
             freeSystemRamBytes: availableSystemRamBytes(),
@@ -514,7 +562,51 @@ export async function buildLlamaCppProvider(opts) {
         });
         return probe.minimumSatisfied && probe.slots >= slotCount;
     };
-    const kvPlan = planLlamaCppKv({
+    // One slot's KV at `ctxTokens`, priced the way the launch reservation
+    // prices it below: the windowed cache unless `--swa-full` is pinned.
+    const planSlotKvReserveBytes = (kv, ctxTokens) => {
+        if (!headerSummary)
+            return planKvBytesPerToken(kv) * ctxTokens;
+        const geometry = {
+            blockCount: headerSummary.blockCount,
+            embeddingLength: headerSummary.embeddingLength,
+            headCount: headerSummary.headCount,
+            headCountKv: headerSummary.headCountKv,
+            headCountKvPerLayer: headerSummary.headCountKvPerLayer,
+            slidingWindowPattern: headerSummary.slidingWindowPattern,
+            sharedKvLayers: headerSummary.sharedKvLayers,
+            loopCount: headerSummary.loopCount,
+            keyLength: headerSummary.keyLength,
+            valueLength: headerSummary.valueLength,
+            keyLengthSwa: headerSummary.keyLengthSwa,
+            valueLengthSwa: headerSummary.valueLengthSwa,
+            fullAttentionInterval: headerSummary.fullAttentionInterval,
+            ssmInnerSize: headerSummary.ssmInnerSize,
+            ssmStateSize: headerSummary.ssmStateSize,
+            ssmConvKernel: headerSummary.ssmConvKernel,
+            kvCacheType: kv,
+        };
+        const windowed = (config.llamaCppSwaFull ?? manifestEngineConfig?.swaFull ?? false)
+            ? undefined
+            : estimateWindowedKvLinearization({
+                ...geometry,
+                slidingWindow: headerSummary.slidingWindow,
+            });
+        return windowed
+            ? windowed.fixedBytes + windowed.bytesPerToken * ctxTokens
+            : (estimateKvReserveBytes({ ...geometry, ctxTokens }) ?? planKvBytesPerToken(kv) * ctxTokens);
+    };
+    // What a discrete card leaves this launch now: free graphics memory as
+    // device admission measures it, within the card's usable share. Undefined
+    // off discrete cards, where there is no separate pool to run out of.
+    const planGpuPoolBytes = async () => {
+        const kind = brokerSnap?.enforced ? brokerSnap.pools.kind : hostCapacity.kind;
+        if (kind !== 'discrete-gpu')
+            return undefined;
+        const freeGpu = await measureAvailableGpuBytes().catch(() => undefined);
+        return Math.max(0, Math.min(budgetBytes - committedOtherBytes, freeGpu ?? Number.POSITIVE_INFINITY));
+    };
+    const planKv = () => planLlamaCppKv({
         architecture: modelCatalogInfo?.architecture,
         modelId: defaultModelId ?? undefined,
         override: config.llamaCppKvCacheType,
@@ -527,6 +619,42 @@ export async function buildLlamaCppProvider(opts) {
         fitsAt: planFitsAt,
         maxSlots: defaultLocalEngineSlots(budgetBytes),
     });
+    let kvPlan = planKv();
+    // A drafter buys speed; context is what a session runs out of. When a
+    // drafter the catalog chose would cap the target window or cost its KV
+    // precision, it goes and the model decodes the way it does without one.
+    // Growth above the target, sized later, simply has less room. On a
+    // discrete card it must also leave the model wholly on the GPU in the
+    // memory free now, which is what device admission charges the launch.
+    if (draftBudgetBytes() > 0 && !config.llamaCppSpecType && !config.llamaCppDraftModelPath) {
+        const targetCtx = kvPlan.ctxCapTokens ?? effectiveNumCtx;
+        const fitsWithDrafter = planFitsAt(kvPlan.kvCacheType, targetCtx, 1);
+        const drafter = drafterInUse;
+        drafterInUse = undefined;
+        const withoutDrafter = planKv();
+        drafterInUse = drafter;
+        let drafterCost;
+        if (drafterCostsContext(kvPlan, withoutDrafter, fitsWithDrafter)) {
+            drafterCost = `beside it the ${effectiveNumCtx}-token window would not keep its size and KV precision`;
+        }
+        else {
+            const gpuPoolBytes = await planGpuPoolBytes();
+            const withDrafterBytes = planResidentBytes() + planSlotKvReserveBytes(kvPlan.kvCacheType, targetCtx);
+            if (drafterCostsGpuResidency(withDrafterBytes, gpuPoolBytes)) {
+                drafterCost = `beside it the model needs ${(withDrafterBytes / 1024 ** 3).toFixed(1)} GiB of graphics memory and ${((gpuPoolBytes ?? 0) / 1024 ** 3).toFixed(1)} GiB is available`;
+            }
+        }
+        if (drafterCost) {
+            log.info(`[llama-cpp] ${modelCatalogInfo?.id ?? defaultModelId ?? 'model'}: leaving the speculative drafter out; ${drafterCost}`);
+            await leaveDrafterOut();
+            kvPlan = withoutDrafter;
+        }
+    }
+    // What a loaded drafter holds on the GPU, as the resident estimate prices it.
+    const drafterGpuBytes = () => estimateLlamaCppResidentBytes(0, {
+        draftBytes: draftBudgetBytes(),
+        draftResidentBytes: catalogDraftResidentBytes,
+    }) - estimateLlamaCppResidentBytes(0);
     kvCacheType = kvPlan.kvCacheType;
     if (kvPlan.upgraded) {
         log.info(`[llama-cpp] ${modelCatalogInfo?.id ?? defaultModelId ?? 'model'}: trading f16 KV for q8_0 to fit a second engine slot (single-slot SWA session alternation re-prefills wholesale; KV A/B 2026-08-03 showed no measurable q8_0 fidelity cost)`);
@@ -552,7 +680,7 @@ export async function buildLlamaCppProvider(opts) {
     // concurrent requests correctly with zero engine errors, so the failure this
     // clamp guards against looks stale — but one probe is not grounds for
     // deleting a safety measure, so it stays for anyone who opted in explicitly.
-    const selectedSpecType = config.llamaCppSpecType ?? manifestEngineConfig?.spec?.type;
+    const selectedSpecType = specTypeNow();
     if (selectedSpecType === 'draft-mtp' && slots > 1) {
         log.info(`[llama-cpp] limiting ${modelCatalogInfo?.id ?? 'MTP model'} to one slot because draft-mtp is not reliable with --parallel > 1 in the bundled engine`);
         slots = 1;
@@ -614,18 +742,6 @@ export async function buildLlamaCppProvider(opts) {
         : catalogReasoningBudgetTokens !== undefined
             ? 'catalog'
             : 'unbounded';
-    // Speculative-decoding draft resolution — a manifest names its draft by
-    // catalog id; resolve it to the installed GGUF path (or disable) before the
-    // argv is built. See providers/llama-cpp/spec-draft.ts for the full rules.
-    const specDraft = await resolveSpecDraft({
-        perModel: manifestEngineConfig,
-        configSpecType: config.llamaCppSpecType,
-        configDraftModelPath: config.llamaCppDraftModelPath,
-        resolveDraftPath: (id) => opts.llamaCppModels ? opts.llamaCppModels.resolveModelPath(id) : Promise.resolve(null),
-    });
-    const resolvedManifestEngineConfig = specDraft.perModel;
-    if (specDraft.log)
-        log[specDraft.log.level](specDraft.log.message);
     // Phase v4 — hardware-aware structured-weight offload. Probe device VRAM (cached to
     // `engines/llama-cpp/devices.json` via `--list-devices`) and read the
     // model's MoE metadata from its GGUF header; the planner decides
@@ -654,7 +770,6 @@ export async function buildLlamaCppProvider(opts) {
     // cross-check for an explicit draft-mtp request.
     let ggufHasMtp = false;
     let mtpLayerCount = 0;
-    const llamaBuildMetadata = binary ? await readLlamaCppBuildMetadata(binary) : null;
     let llamaGpuDevice = null;
     let nvidiaRuntimeDevice;
     let llamaDevices = [];
@@ -677,8 +792,10 @@ export async function buildLlamaCppProvider(opts) {
             offloadBlockCount = summary.blockCount;
             mtpLayerCount = summary.nextnPredictLayers ?? 0;
             ggufHasMtp = mtpLayerCount > 0;
-            if (!ggufHasMtp && modelCatalogInfo?.draftModelPath) {
-                const draftSummary = await readGgufSummaryAsync(modelCatalogInfo.draftModelPath);
+            // Only a companion this launch loads can supply the head: one left out
+            // for memory would turn MTP on with nothing to run it.
+            if (!ggufHasMtp && drafterInUse) {
+                const draftSummary = await readGgufSummaryAsync(drafterInUse);
                 mtpLayerCount = draftSummary.nextnPredictLayers ?? 0;
                 ggufHasMtp = mtpLayerCount > 0;
             }
@@ -687,6 +804,8 @@ export async function buildLlamaCppProvider(opts) {
             const residentWeightBytes = Math.max(0, approxBytes - lazyTensorBytes);
             const residentBytes = estimateLlamaCppResidentBytes(residentWeightBytes, {
                 mmprojBytes: visionBudgetBytes,
+                draftBytes: draftBudgetBytes(),
+                draftResidentBytes: catalogDraftResidentBytes,
             });
             if (lazyTensorBytes > 0) {
                 log.info(`[llama-cpp] ${modelCatalogInfo?.id ?? defaultModelId ?? 'model'}: lazy-reading ${(lazyTensorBytes / 1024 ** 3).toFixed(1)} GiB of row-addressable embeddings from the GGUF; budgeting ${(residentWeightBytes / 1024 ** 3).toFixed(1)} GiB as resident model weights`);
@@ -1025,6 +1144,7 @@ export async function buildLlamaCppProvider(opts) {
                                 split,
                                 ...(summary.blockCount !== undefined ? { blockCount: summary.blockCount } : {}),
                                 ...(kvReserveBytes !== undefined ? { kvReserveBytes } : {}),
+                                additionalGpuBytes: drafterGpuBytes(),
                             }
                             : {}),
                     });
@@ -1063,7 +1183,7 @@ export async function buildLlamaCppProvider(opts) {
                             ...(denseFfnSplit ? { split: denseFfnSplit } : {}),
                             ...(summary.blockCount !== undefined ? { blockCount: summary.blockCount } : {}),
                             ...(kvReserveBytes !== undefined ? { kvReserveBytes } : {}),
-                            additionalGpuBytes: visionBudgetBytes,
+                            additionalGpuBytes: visionBudgetBytes + drafterGpuBytes(),
                             ramBudgetBytes: brokerSnap?.enforced
                                 ? brokerSnap.pools.ramShareBytes
                                 : hostCapacity.ramShareBytes,
@@ -1100,8 +1220,8 @@ export async function buildLlamaCppProvider(opts) {
             // metadata gate remains load-bearing even though supported GGUFs are
             // now default-on.
             if (ggufHasMtp) {
-                log.info(`[llama-cpp] ${modelCatalogInfo?.id ?? 'model'} ships an MTP head (nextn_predict_layers=${mtpLayerCount}); speculative decoding ${config.llamaCppSpecType || manifestEngineConfig?.spec?.type
-                    ? 'follows the explicit setting'
+                log.info(`[llama-cpp] ${modelCatalogInfo?.id ?? 'model'} ships an MTP head (nextn_predict_layers=${mtpLayerCount}); speculative decoding ${specTypeNow()
+                    ? `runs ${specTypeNow()} (${config.llamaCppSpecType ? 'Settings' : 'catalog'})`
                     : 'is ON by default (+4-19% decode; disable with llamaCppSpecType: "none")'}.`);
             }
         }
@@ -1183,6 +1303,8 @@ export async function buildLlamaCppProvider(opts) {
             providerHolder.current?.getCacheAdapter()?.resetEngineState();
             if (snapshot.expected)
                 return;
+            if (catalogDrafterLoaded())
+                brokenDrafters.add(brokenDrafterKey);
             logFile.writeIncident(snapshot);
             if (snapshot.panicKind === 'host-out-of-memory') {
                 noteHostOutOfMemory({
@@ -1264,6 +1386,15 @@ export async function buildLlamaCppProvider(opts) {
         // restarts of this provider, so a recovered engine doesn't re-OOM on its
         // next boot.
         recoverStartup: ({ panicKind }) => {
+            // The drafter is the one part of this launch it can do without, and
+            // drafters have changed format between engine releases. Without this a
+            // bad one reads as a build that cannot run, and three of those quarantine
+            // the whole GPU backend; `resolveLaunch` drops it from the retry.
+            if (catalogDrafterLoaded()) {
+                brokenDrafters.add(brokenDrafterKey);
+                log.warn(`[llama-cpp] ${panicKind ?? 'failed start'} while starting ${modelCatalogInfo?.id ?? 'model'} with its speculative drafter — retrying without it`);
+                return true;
+            }
             if (panicKind !== 'cuda-out-of-memory' && panicKind !== 'vulkan-out-of-memory') {
                 return false;
             }
@@ -1318,6 +1449,9 @@ export async function buildLlamaCppProvider(opts) {
             return true;
         },
         resolveLaunch: async () => {
+            if (catalogDrafterLoaded() && brokenDrafters.has(brokenDrafterKey)) {
+                await leaveDrafterOut();
+            }
             gpuWatch = new GpuStartupWatch(binaryBackend);
             // Again at every launch, not only at build: llama.cpp from the 0.1.51
             // release refuses a --slot-save-path that does not exist ("not a
@@ -1345,7 +1479,7 @@ export async function buildLlamaCppProvider(opts) {
                 kvCacheType,
                 slots,
                 ggufHasMtp,
-                installedDraftModelPath: modelCatalogInfo?.draftModelPath,
+                installedDraftModelPath: drafterInUse,
                 // Opt-in A/B lever: `GEZEL_LLAMA_REASONING_FORMAT=none`
                 // disables llama-server's chat-template output parsing so mangled
                 // model output reaches `delta.content` for provider-side salvage.
@@ -1483,6 +1617,8 @@ export async function buildLlamaCppProvider(opts) {
                     ubatchSize: lastArgValue(resolvedAdvancedArgs, '--ubatch-size') ?? 'default',
                     flashAttention: lastArgValue(resolvedAdvancedArgs, '--flash-attn') ?? 'default',
                     threads: lastArgValue(resolvedAdvancedArgs, '--threads') ?? 'default',
+                    specType: lastArgValue(resolvedAdvancedArgs, '--spec-type') ?? 'default',
+                    specDraftModel: basename(lastArgValue(resolvedAdvancedArgs, '--spec-draft-model') ?? 'none'),
                     reasoningBudgetTokens: reasoningBudgetTokens ?? 'unbounded',
                     reasoningBudgetSource,
                     reasoningPreserve,
@@ -1493,6 +1629,9 @@ export async function buildLlamaCppProvider(opts) {
                     // Equivalent to `--api-key`, but out of `argv` so it does not show
                     // up in `ps` on a shared machine.
                     LLAMA_API_KEY: engineApiKey(),
+                    // A ROCm build's kernel packs, rocBLAS routing, and the AMD GPUs it
+                    // has no code for kept out of sight (rocm-devices.ts).
+                    ...rocmLaunchEnv(home, binary, llamaBuildMetadata?.backend ?? binaryBackend),
                 },
                 baseUrl: `http://127.0.0.1:${port}`,
             };

package/dist/providers/llama-cpp/devices.js

@@ -19,7 +19,7 @@ import { execFile } from 'node:child_process';
 import { existsSync, mkdirSync, readFileSync, statSync, writeFileSync } from 'node:fs';
 import { delimiter, dirname, join } from 'node:path';
 import { promisify } from 'node:util';
-import { windowsHeadlessSpawnOptions } from '@bendyline/gezel/native';
+import { rocmLaunchEnv, windowsHeadlessSpawnOptions } from '@bendyline/gezel/native';
 import { gezelHome } from '@bendyline/gezel/paths';
 import { totalMemoryBytes } from '../../system/host-memory.js';
 const execFileAsync = promisify(execFile);
@@ -191,6 +191,9 @@ export async function probeLlamaDevices(opts) {
             env: {
                 ...process.env,
                 PATH: inheritedPath ? `${binDir}${delimiter}${inheritedPath}` : binDir,
+                // A ROCm build lists only the GPUs its launch will see, so the VRAM
+                // the planner reads is the VRAM the engine gets.
+                ...rocmLaunchEnv(home, binaryPath),
             },
             // This is an awaited probe, not an independent process. Hide its
             // Windows console without detaching it from the daemon.

package/dist/providers/llama-cpp/engine-flags.d.ts

@@ -64,6 +64,12 @@ export interface PerModelLlamaCppEngineConfig {
         mtp?: boolean;
         draftModelId?: string;
         nMax?: number;
+        sampling?: 'greedy' | 'probabilistic';
+        byBackend?: Partial<Record<string, {
+            type?: string;
+            nMax?: number;
+            sampling?: 'greedy' | 'probabilistic';
+        }>>;
     };
 }
 /** Phase v2 hardware-aware offload decision (lowest precedence). */
@@ -106,7 +112,8 @@ export interface EngineFlagInput {
     /**
      * Absolute path to a catalog-installed speculative draft sidecar.
      * Separate MTP assistants (Gemma) and other draft algorithms need this;
-     * combined MTP GGUFs (Qwen) intentionally leave it unset.
+     * combined MTP GGUFs (Qwen) leave it unset, even with a DFlash drafter on
+     * disk: `--spec-draft-model` would make MTP load the wrong file.
      */
     installedDraftModelPath?: string | undefined;
     /**

package/dist/providers/llama-cpp/engine-flags.js

@@ -49,7 +49,7 @@ function gpuResidentThreads(hostCpuCount) {
         : GPU_RESIDENT_THREADS;
     return Math.max(1, Math.min(GPU_RESIDENT_THREADS, available));
 }
-const GPU_BACKENDS = new Set(['cuda', 'vulkan', 'metal']);
+const GPU_BACKENDS = new Set(['cuda', 'rocm', 'vulkan', 'metal']);
 /**
  * `llamaCppExtraArgs` keys that either choose a thread pool or may move
  * weights to the CPU; any of them makes the GPU-resident default stand down.
@@ -223,7 +223,7 @@ export function buildLlamaCppEngineArgs(input) {
     // MTP is ON by default for a GGUF that carries the head, having cleared the
     // A/B gate this comment used to wait for (reports/llama-mtp-eval-20260829.md):
     // +12.7% / +4.1% / +15.6% / +19.3% decode at 5.5k / 21.7k / 54k / 87k
-    // context, never negative, byte-identical greedy output, +1.35 GB resident.
+    // context, never negative, +1.35 GB resident.
     // The gain GROWS with context — long-context decode is bound by reading the
     // KV cache, and verifying K drafted tokens costs one pass over it instead of
     // K — which is the regime agent turns actually run in.
@@ -232,6 +232,8 @@ export function buildLlamaCppEngineArgs(input) {
     // FATALLY when draft-mtp is selected for a model with no MTP tensors, so
     // the default may only ever apply where the metadata confirms the head.
     // Explicit config and manifest `spec.type` still win, including 'none'.
+    // `perModel` arrives with its `spec.byBackend` already applied by
+    // `resolveSpecDraft`, which also decided whether a drafter is in play.
     const specType = config.llamaCppSpecType ?? perModel?.spec?.type ?? (ggufHasMtp ? 'draft-mtp' : undefined);
     const safeSpecType = specType === 'draft-mtp' && !ggufHasMtp ? undefined : specType;
     if (safeSpecType && safeSpecType !== 'none') {
@@ -244,12 +246,15 @@ export function buildLlamaCppEngineArgs(input) {
         const nMax = config.llamaCppSpecDraftNMax ?? perModel?.spec?.nMax;
         if (typeof nMax === 'number')
             args.push('--spec-draft-n-max', String(nMax));
-        // v0.7.0 supports probabilistic selection only for MTP/simple drafts
-        // (DFlash/LiLiCorr ignore it).
-        // Leave the upstream greedy default untouched unless explicitly set.
-        if ((safeSpecType === 'draft-mtp' || safeSpecType === 'draft-simple') &&
-            config.llamaCppSpecDraftSampling) {
-            args.push('--spec-draft-sampling', config.llamaCppSpecDraftSampling);
+        // MTP, simple and DFlash2 honor probabilistic selection; a DFlash 1 or
+        // LiLiCorr drafter takes the flag and ignores it. Leave the upstream greedy
+        // default untouched unless someone chose otherwise.
+        const draftSampling = config.llamaCppSpecDraftSampling ?? perModel?.spec?.sampling;
+        if ((safeSpecType === 'draft-mtp' ||
+            safeSpecType === 'draft-simple' ||
+            safeSpecType === 'draft-dflash') &&
+            draftSampling) {
+            args.push('--spec-draft-sampling', draftSampling);
         }
     }
     // ── Server-side output parsing (`--reasoning-format`) ─────────────

package/dist/providers/llama-cpp/gpu-startup-watch.d.ts

@@ -7,8 +7,8 @@
  * word. A CUDA backend that loads but cannot initialize prints one error line
  * and registers no devices. Either way the engine starts, answers, and runs
  * every token on the CPU — at a fraction of the speed the person's hardware
- * and Settings promise — while gezel reports the CUDA or Vulkan build it
- * launched.
+ * and Settings promise — while gezel reports the CUDA, ROCm or Vulkan build
+ * it launched.
  *
  * llama.cpp names every GPU it will use while loading a model
  * (`using device CUDA0 (NVIDIA GB10) …`), integrated GPUs included. A GPU
@@ -16,14 +16,14 @@
  * without its GPU. The explicit lines say why when there is a why to say.
  */
 export type GpuUnusedReason = 
-/** The CUDA runtime loaded and refused to start: `failed to initialize CUDA`. */
+/** The CUDA or ROCm runtime loaded and refused to start: `failed to initialize CUDA`. */
 'init-failed'
 /** The backend loaded and found no device it could use. */
  | 'no-devices'
 /** No sign of the GPU backend at all: it failed to load, silently. */
  | 'not-loaded';
 export interface GpuUnused {
-    backend: 'cuda' | 'vulkan';
+    backend: 'cuda' | 'rocm' | 'vulkan';
     reason: GpuUnusedReason;
     /** The engine's own words, when it gave any. */
     detail?: string;

package/dist/providers/llama-cpp/gpu-startup-watch.js

@@ -7,8 +7,8 @@
  * word. A CUDA backend that loads but cannot initialize prints one error line
  * and registers no devices. Either way the engine starts, answers, and runs
  * every token on the CPU — at a fraction of the speed the person's hardware
- * and Settings promise — while gezel reports the CUDA or Vulkan build it
- * launched.
+ * and Settings promise — while gezel reports the CUDA, ROCm or Vulkan build
+ * it launched.
  *
  * llama.cpp names every GPU it will use while loading a model
  * (`using device CUDA0 (NVIDIA GB10) …`), integrated GPUs included. A GPU
@@ -16,7 +16,8 @@
  * without its GPU. The explicit lines say why when there is a why to say.
  */
 const USING_GPU = /\busing device (?:CUDA|Vulkan|ROCm|SYCL|MTL)\d+\b/;
-const CUDA_INIT_FAILED = /ggml_cuda_init: failed to initialize CUDA:\s*(.+?)\s*$/;
+/** ggml's ROCm backend is its CUDA code under HIP, and names itself `ROCm` here. */
+const CUDA_INIT_FAILED = /ggml_cuda_init: failed to initialize (?:CUDA|ROCm):\s*(.+?)\s*$/;
 const VULKAN_NO_DEVICES = /ggml_vulkan: No devices found/;
 /**
  * Lines llama.cpp prints at the same verbosity as `using device`, while
@@ -46,9 +47,10 @@ export class GpuStartupWatch {
     /** At readiness: what kept the GPU out, or null when it is in use or this is no GPU build. */
     verdict() {
         const backend = this.backend;
-        if ((backend !== 'cuda' && backend !== 'vulkan') || this.usingGpu)
+        if ((backend !== 'cuda' && backend !== 'rocm' && backend !== 'vulkan') || this.usingGpu) {
             return null;
-        if (backend === 'cuda' && this.initFailure) {
+        }
+        if ((backend === 'cuda' || backend === 'rocm') && this.initFailure) {
             return { backend, reason: 'init-failed', detail: this.initFailure };
         }
         if (backend === 'vulkan' && this.noDevices)

package/dist/providers/llama-cpp/models.d.ts

@@ -73,6 +73,8 @@ export interface InstalledLlamaCppModel {
      * `--spec-draft-model <path>` for the selected draft algorithm.
      */
     draftModelPath?: string;
+    /** On-disk size of {@link draftModelPath}, for the memory plan. */
+    draftModelSizeBytes?: number;
     /** From the GGUF metadata — useful for the supervisor's `-c` flag (Phase 3). */
     contextWindow?: number;
     /** Quantization tag (Q4_K_M / Q8_0 / …) lifted from the catalog source block. */
@@ -389,6 +391,14 @@ export declare class LlamaCppModelManager {
      * a recorded digest plus an unchanged stat is identity.
      */
     private planReusableFiles;
+    /**
+     * A drafter is trained against a model, not a quant, so every quant of that
+     * model pins the same file. Link a copy another installed quant verified
+     * rather than downloading a gigabyte again. A hard link keeps each model's
+     * directory complete on its own: deleting one quant leaves the other's
+     * drafter in place. A volume that cannot link just downloads.
+     */
+    private adoptSiblingDraftModels;
     /** Warn once per model directory per process, then stay quiet. */
     private warnSkip;
     private loadInstalled;