@bendyline/gezel-inference 1.0.8 → 1.0.9

package/dist/providers/mlx/python/tool_args_json.py

@@ -22,13 +22,17 @@ chunk by chunk. Everything else (ints, floats, bools, objects, arrays) is
 small and needs the whole value for `_convert_param_value`-style coercion, so
 those are buffered per-parameter and emitted when their `</parameter>` lands.
 
-Semantics are copied from mlx_vlm 0.6.6 `tool_parsers/qwen3_coder.py` and must
-stay identical; `test_tool_args_json.py` asserts that by differential-testing
-every case against the reference parser. In particular:
+Semantics are copied from mlx_vlm 0.7.12 `tools/parsers/qwen3_coder.py` and
+must stay identical; `tool_args_json_test.py` asserts that by differential-
+testing every case against the reference parser. In particular:
 
   * exactly ONE leading and ONE trailing "\\n" is stripped from a value
-  * a literal "null" (any case) becomes JSON null
+  * a literal "null" (any case) becomes JSON null, except in a declared
+    string parameter, which keeps the text unless the schema says
+    `"nullable": true`
   * a parameter absent from the schema is treated as a string
+  * a declared parameter without a type decodes JSON objects and arrays and
+    keeps anything else as text
   * unknown/odd types fall through to `ast.literal_eval`, as upstream does
 
 Chunk boundaries are the hazard throughout: `</parameter>`, `</function>` and
@@ -68,29 +72,59 @@ def arguments_config(func_name: str, tools: Optional[Any]) -> Dict[str, Any]:
     return {}
 
 
+def _declared_string(param: Any) -> bool:
+    return (
+        isinstance(param, dict)
+        and isinstance(param.get("type"), str)
+        and param["type"].strip().lower() in _STRING_TYPES
+    )
+
+
 def is_string_param(param_name: str, param_config: Dict[str, Any]) -> bool:
     """Can this parameter stream, or must its value be buffered whole?
 
     Unknown parameters count as strings — upstream returns the raw value when
-    the schema has no entry, and raw-string is the streamable case.
+    the schema has no entry, and raw-string is the streamable case. A declared
+    parameter without a type is buffered: its value may decode to an object.
     """
-    param = param_config.get(param_name)
-    if not param:
-        return True
-    declared = param.get("type")
-    if declared is None:
+    if param_name not in param_config:
         return True
-    return str(declared).strip().lower() in _STRING_TYPES
+    param = param_config[param_name]
+    if not isinstance(param, dict) or "type" not in param:
+        return False
+    return str(param["type"]).strip().lower() in _STRING_TYPES
+
+
+def null_text_is_none(param_name: str, param_config: Dict[str, Any]) -> bool:
+    """Whether a complete value of "null" is JSON null for this parameter.
+
+    A declared string keeps the text unless the schema marks it nullable;
+    every other parameter, unknown ones included, reads it as null.
+    """
+    param = param_config.get(param_name)
+    if _declared_string(param):
+        return param.get("nullable") is True
+    return True
 
 
 def convert_param_value(value: str, param_name: str, param_config: Dict[str, Any]) -> Any:
     """Coerce a complete value, mirroring `_convert_param_value`."""
+    param = param_config.get(param_name)
+    if _declared_string(param):
+        if value.lower() == "null" and param.get("nullable") is True:
+            return None
+        return value
     if value.lower() == "null":
         return None
-    param = param_config.get(param_name)
-    if not param:
+    if param_name not in param_config:
         return value
-    param_type = str(param["type"]).strip().lower() if "type" in param else "string"
+    if not isinstance(param, dict) or "type" not in param:
+        try:
+            decoded = json.loads(value)
+        except json.JSONDecodeError:
+            return value
+        return decoded if isinstance(decoded, (dict, list)) else value
+    param_type = str(param["type"]).strip().lower()
     if param_type in _STRING_TYPES:
         return value
     if param_type.startswith(("int", "uint", "long", "short", "unsigned")):
@@ -194,10 +228,10 @@ class ToolArgsJsonEncoder:
                 self._leading_stripped = False
                 self._opened_quote = False
                 # NOTE: the opening quote is deliberately NOT emitted yet.
-                # `_convert_param_value` maps a whole value of "null" to JSON
-                # null BEFORE consulting the schema, so even a string-typed
-                # parameter can turn out to be null — and by then a quote
-                # would already be on the wire. Hold the first few bytes.
+                # A whole value of "null" is JSON null for an unknown or
+                # nullable string parameter (`null_text_is_none`) — and by
+                # then a quote would already be on the wire. Hold the first
+                # few bytes.
                 continue
 
             end = self._buf.find(_PARAM_CLOSE)
@@ -228,8 +262,8 @@ class ToolArgsJsonEncoder:
             #   * strip exactly one leading newline
             #   * strip exactly one trailing newline (so the final newline is
             #     held until we know whether anything follows it)
-            #   * a complete value of "null" is JSON null, not a string, so
-            #     the opening quote waits until the value is longer than
+            #   * a complete value of "null" can be JSON null, not a string,
+            #     so the opening quote waits until the value is longer than
             #     "null" or the parameter closes
             if not self._leading_stripped:
                 if chunk.startswith("\n"):
@@ -241,7 +275,11 @@ class ToolArgsJsonEncoder:
                 self._value_buf += chunk
                 if end < 0 and len(self._value_buf) <= 4:
                     break
-                if end >= 0 and self._value_buf.lower() == "null":
+                if (
+                    end >= 0
+                    and self._value_buf.lower() == "null"
+                    and null_text_is_none(self._value_name, self.param_config)
+                ):
                     out.append(f"{json.dumps(self._value_name)}:null")
                     self._in_value = False
                     continue

package/dist/providers/mlx/venv.d.ts

@@ -78,9 +78,27 @@
  * instead of consuming the Metal working set. Do not treat the library pin
  * alone as proof that every community conversion is correct or fits.
  *
+ * 0.7.12 (2026-11-05): the project moved to Neywa-Labs/mlx-vlm; same PyPI
+ * name. It resolves mlx 0.32.4 (mlx-audio pins it exactly) and transformers
+ * 5.22. Every private `speculative.mtp` helper the assisted round loop calls
+ * is unchanged for Qwen 3.5; Gemma 4 targets now rebuild the drafter's shared
+ * KV from committed caches, which `shared_kv_after_round` follows. Its
+ * qwen3_coder parser keeps "null" as text in a string parameter, which
+ * `tool_args_json.py` now mirrors. Verified on an M5 Max against 0.7.1
+ * through the real sidecar: gemma4-e4b and qwen3.8-27b + MTP drafter render
+ * identical prompt token counts and identical tool-call, JSON and vision
+ * output; 0.7.1-written disk caches load and extend (Gemma 4 rotating,
+ * Qwen 3.5 hybrid). Greedy MTP matches greedy no-spec on code at 0-30k
+ * tokens; on prose both versions flip bf16 one-step ties (top-2 gap 0.125
+ * nats), which two non-speculative decode paths also disagree on, so prose
+ * is not an exactness probe. Clean A/B/B/A decode on qwen3.8-27b-q4: MTP on
+ * the vlm tower at parity to 10k and +19% at 50k; vlm tower without MTP 2.0x
+ * at 50k (7.3 -> 14.8 tok/s), now level with the mlx_lm tower; mlx_lm tower
+ * 0.94x at 0-10k, within run-to-run noise.
+ *
  * Keep upgrades explicit and re-sweep the MLX catalog whenever this pin
  * moves. */
-export declare const MLX_DEFAULT_PACKAGE_SPEC = "mlx-vlm==0.7.1";
+export declare const MLX_DEFAULT_PACKAGE_SPEC = "mlx-vlm==0.7.12";
 /** Venv name passed to `UvRuntime.ensureVenv` (and its dir basename). */
 export declare const MLX_VENV_NAME = "mlx";
 /**

package/dist/providers/mlx/venv.js

@@ -78,9 +78,27 @@
  * instead of consuming the Metal working set. Do not treat the library pin
  * alone as proof that every community conversion is correct or fits.
  *
+ * 0.7.12 (2026-11-05): the project moved to Neywa-Labs/mlx-vlm; same PyPI
+ * name. It resolves mlx 0.32.4 (mlx-audio pins it exactly) and transformers
+ * 5.22. Every private `speculative.mtp` helper the assisted round loop calls
+ * is unchanged for Qwen 3.5; Gemma 4 targets now rebuild the drafter's shared
+ * KV from committed caches, which `shared_kv_after_round` follows. Its
+ * qwen3_coder parser keeps "null" as text in a string parameter, which
+ * `tool_args_json.py` now mirrors. Verified on an M5 Max against 0.7.1
+ * through the real sidecar: gemma4-e4b and qwen3.8-27b + MTP drafter render
+ * identical prompt token counts and identical tool-call, JSON and vision
+ * output; 0.7.1-written disk caches load and extend (Gemma 4 rotating,
+ * Qwen 3.5 hybrid). Greedy MTP matches greedy no-spec on code at 0-30k
+ * tokens; on prose both versions flip bf16 one-step ties (top-2 gap 0.125
+ * nats), which two non-speculative decode paths also disagree on, so prose
+ * is not an exactness probe. Clean A/B/B/A decode on qwen3.8-27b-q4: MTP on
+ * the vlm tower at parity to 10k and +19% at 50k; vlm tower without MTP 2.0x
+ * at 50k (7.3 -> 14.8 tok/s), now level with the mlx_lm tower; mlx_lm tower
+ * 0.94x at 0-10k, within run-to-run noise.
+ *
  * Keep upgrades explicit and re-sweep the MLX catalog whenever this pin
  * moves. */
-export const MLX_DEFAULT_PACKAGE_SPEC = 'mlx-vlm==0.7.1';
+export const MLX_DEFAULT_PACKAGE_SPEC = 'mlx-vlm==0.7.12';
 /** Venv name passed to `UvRuntime.ensureVenv` (and its dir basename). */
 export const MLX_VENV_NAME = 'mlx';
 /**
@@ -100,7 +118,8 @@ export function mlxVenvPackages(packageSpec) {
     // a dependency (0.6.6 did, which is the only reason the old list
     // worked). Dropping this pin kills the batch engine at import time.
     // 0.31.3 is also the combination used for the 0.7.1 Qwen4-Exp batch
-    // compatibility check above; keep it independent of mlx-vlm's deps.
+    // compatibility check and the 0.7.12 verification above; keep it
+    // independent of mlx-vlm's deps.
     return [
         packageSpec ?? MLX_DEFAULT_PACKAGE_SPEC,
         'mlx-lm==0.31.3',

package/dist/providers/native/device-capacity.d.ts

@@ -77,6 +77,10 @@ export declare function staticCapacityCeilings(budget: Pick<CapacityBudget, 'bud
     budgetBytes: number;
     gpuBudgetBytes: number;
 };
+/** The RAM ceiling the device ledger admits one launch under, for sizing a launch to fit it. */
+export declare function deviceModelBudgetBytes(config: {
+    localEngineMemoryGb?: number | null | undefined;
+}): Promise<number>;
 export declare function sampleDeviceCapacity(home?: string): Promise<DeviceCapacitySample>;
 export declare function localDeviceCapacity(home: string): DeviceCapacityLedger;
 /** Conservative fallback for media engines that do not expose a launch planner. */

package/dist/providers/native/device-capacity.js

@@ -155,6 +155,11 @@ export function staticCapacityCeilings(budget, ramBytes, configuredBytes, platfo
             : budget.fastBytes),
     };
 }
+/** The RAM ceiling the device ledger admits one launch under, for sizing a launch to fit it. */
+export async function deviceModelBudgetBytes(config) {
+    const ram = totalMemoryBytes();
+    return staticCapacityCeilings(await measuredCapacityBudget(), ram, configuredEngineMemoryBytes(config)).budgetBytes;
+}
 export async function sampleDeviceCapacity(home) {
     const budget = await measuredCapacityBudget();
     const ram = totalMemoryBytes();

package/dist/providers/native/local-engine-runtime.d.ts

@@ -37,7 +37,7 @@ export declare class LocalEngineRuntime {
     /** Product chat may prefill its own sessions; the engine runtime never resolves them. */
     protected prefillCacheSession(_sessionId: string): Promise<boolean>;
     /** Cached provider instances keyed by provider name. */
-    protected readonly providers: Map<"copilot" | "openai" | "anthropic" | "anthropic-cli" | "codex-cli" | "ollama" | "llama-cpp" | "mlx" | "ds4" | "apple-foundation-models" | "android-mlkit" | "remote", LLMProvider>;
+    protected readonly providers: Map<"copilot" | "openai" | "anthropic" | "anthropic-cli" | "codex-cli" | "ollama" | "llama-cpp" | "mlx" | "ds4" | "remote" | "apple-foundation-models" | "android-mlkit", LLMProvider>;
     protected readonly catalog: CatalogService;
     protected readonly llamaCppModels?: import('../llama-cpp/index.js').LlamaCppModelManager;
     protected readonly ds4Models?: import('../llama-cpp/index.js').LlamaCppModelManager;
@@ -384,6 +384,21 @@ export declare class LocalEngineRuntime {
          * the only guard against a window the engine cannot actually serve.
          */
         contextCeilingTokens?: number;
+        /** ds4 only: whether the launch would stream routed experts from SSD. */
+        ssdStreaming?: boolean;
+        /** ds4 only: `plannedResidentBytes` comes from the entry's measurement for this mode. */
+        memoryMeasured?: boolean;
+        /**
+         * ds4 only, streamed launches with a measured footprint: the expert cache
+         * the launcher would choose, so the model list's fit verdict comes from the
+         * same plan instead of a copy of its headroom rule.
+         */
+        expertCache?: {
+            cacheBytes: number;
+            requestedBytes: number;
+            clamped: boolean;
+            safe: boolean;
+        };
     }>;
     /**
      * Cancel a pending provider-queue entry by (provider, id). Resolves

package/dist/providers/native/local-engine-runtime.js

@@ -1,15 +1,17 @@
 import { randomUUID } from 'node:crypto';
 import { join } from 'node:path';
-import { createLogger } from '@bendyline/gezel';
-import { estimateLlamaCppResidentBytes } from '@bendyline/gezel';
+import { createLogger, ds4SsdStreamingSupported } from '@bendyline/gezel';
+import { estimateLlamaCppResidentBytes, llamaCppDraftResidentBytes, } from '@bendyline/gezel';
 import { resolveCatalogLlamaCppEngineConfig } from '../catalog-model-config.js';
-import { buildDs4Provider, resolveDs4LaunchCtx, resolveDs4VisionLaunch, } from '../ds4/build-provider.js';
+import { buildDs4Provider, ds4RamTieredContext, resolveDs4LaunchCtx, resolveDs4VisionLaunch, } from '../ds4/build-provider.js';
 import { ds4VisionResidentBytes } from '../ds4/residency.js';
+import { currentLlamaCppBackend } from '../llama-cpp/backend-health.js';
 import { buildLlamaCppProvider, ensureLlamaEngineStatus } from '../llama-cpp/build-provider.js';
 import { readGgufSummaryAsync } from '../llama-cpp/gguf-metadata-async.js';
 import { effectiveLazyTensorBytes } from '../llama-cpp/gguf-metadata.js';
 import { isGemmaModel, planLlamaCppKv, resolveLlamaCppKvCacheType, } from '../llama-cpp/kv-cache-type.js';
 import { estimateExactPerSlotKvBytesF16, estimateKvReserveBytes, estimateWindowedKvLinearization, } from '../llama-cpp/offload-planner.js';
+import { drafterCostsContext, llamaCppSidecarLoads, resolveSpecDraft, } from '../llama-cpp/spec-draft.js';
 import { buildMlxProvider, resolveMlxEffectiveNumCtx } from '../mlx/build-provider.js';
 import { readMlxModelGeometry } from '../mlx/model-geometry.js';
 import { ModelNotInstalledError } from '../types.js';
@@ -594,7 +596,8 @@ export class LocalEngineRuntime {
                         const config = await this.store.readConfig();
                         const externalBaseUrl = process.env.GEZEL_DS4_SERVER_URL ?? config.ds4BaseUrl;
                         if (!externalBaseUrl) {
-                            const { ds4ProjectedResidentBytes, ds4ResidentBytesForMode, ds4ResidentLine, shouldUseDs4SsdStreaming, } = await import('../ds4/residency.js');
+                            const { ds4ProjectedResidentBytes, ds4ResidentBytesForMode, ds4ResidentLine, planDs4MeasuredLaunch, shouldUseDs4SsdStreaming, } = await import('../ds4/residency.js');
+                            const { totalMemoryBytes } = await import('../../system/host-memory.js');
                             // Bill the window this model will actually launch with. The
                             // authored footprint is a measurement at `residentCtxTokens`, so
                             // without this a raised context reserves the old number and the
@@ -603,29 +606,51 @@ export class LocalEngineRuntime {
                             const ds4Block = cm.ds4;
                             const installed = await this.ds4Models?.resolveModel(modelId).catch(() => null);
                             const visionResidentBytes = await this.resolveDs4VisionResidentBytes(config, modelId, installed, ds4Block);
-                            const line = ds4ResidentLine({
-                                residentBytes: bytes,
-                                kvBytesPerToken: ds4Block?.kvBytesPerToken,
-                                residentCtxTokens: ds4Block?.residentCtxTokens,
+                            const totalRamBytes = totalMemoryBytes();
+                            const ds4Override = config.modelContextOverrides?.[`ds4:${modelId}`];
+                            const contextFloor = minViableLocalContextTokens();
+                            const numCtx = resolveDs4LaunchCtx({
+                                configured: ds4Override ?? config.ds4NumCtx,
+                                ramTieredCtx: ds4RamTieredContext(totalRamBytes),
+                                catalogMaxCtx: ds4Block?.maxLaunchCtx,
+                                minViableContextTokens: ds4Override !== undefined ? Math.min(contextFloor, ds4Override) : contextFloor,
                             });
-                            if (line) {
-                                const { totalMemoryBytes } = await import('../../system/host-memory.js');
-                                bytes = ds4ProjectedResidentBytes(line, resolveDs4LaunchCtx({
-                                    configured: config.modelContextOverrides?.[`ds4:${modelId}`] ?? config.ds4NumCtx,
-                                    ramTieredCtx: totalMemoryBytes() / 1024 ** 3 >= 192 ? 262_144 : 131_072,
-                                    catalogMaxCtx: ds4Block?.maxLaunchCtx,
-                                    minViableContextTokens: minViableLocalContextTokens(),
-                                }));
-                            }
-                            bytes = ds4ResidentBytesForMode(bytes, shouldUseDs4SsdStreaming({
+                            const streamingSupported = ds4SsdStreamingSupported(ds4Block, process.platform);
+                            const ssdStreaming = shouldUseDs4SsdStreaming({
                                 configured: config.ds4SsdStreaming,
-                                ...(ds4Block?.ssdStreamingSupported !== undefined
-                                    ? { ssdStreamingSupported: ds4Block.ssdStreamingSupported }
+                                ...(streamingSupported !== undefined
+                                    ? { ssdStreamingSupported: streamingSupported }
                                     : {}),
                                 modelSizeBytes: ds4Block?.residentWeightBytes ?? block.approxSizeBytes,
+                                totalRamBytes,
                                 ...(visionResidentBytes > 0 ? { companionBytes: visionResidentBytes } : {}),
-                            }), ds4Block?.ssdStreamingSupported);
-                            bytes += visionResidentBytes;
+                            });
+                            const measured = planDs4MeasuredLaunch({
+                                source: ds4Block,
+                                numCtx,
+                                ssdStreaming,
+                                totalRamBytes,
+                                budgetBytes: ssdStreaming && ds4Block?.streamingMemory
+                                    ? await (await import('./device-capacity.js')).deviceModelBudgetBytes(config)
+                                    : undefined,
+                                sidecarBytes: visionResidentBytes,
+                                configuredCacheGb: config.ds4CacheExpertsGb,
+                            });
+                            if (measured) {
+                                bytes = measured.workingSetBytes;
+                            }
+                            else {
+                                const line = ds4ResidentLine({
+                                    residentBytes: bytes,
+                                    kvBytesPerToken: ds4Block?.kvBytesPerToken,
+                                    residentCtxTokens: ds4Block?.residentCtxTokens,
+                                });
+                                if (line)
+                                    bytes = ds4ProjectedResidentBytes(line, numCtx);
+                                bytes =
+                                    ds4ResidentBytesForMode(bytes, ssdStreaming, streamingSupported) +
+                                        visionResidentBytes;
+                            }
                         }
                     }
                 }
@@ -1047,7 +1072,9 @@ export class LocalEngineRuntime {
                 throw new ModelNotInstalledError(name, modelId);
             }
             const { totalMemoryBytes } = await import('../../system/host-memory.js');
-            const ramTieredCtx = totalMemoryBytes() / 1024 ** 3 >= 192 ? 262_144 : 131_072;
+            // The launcher's own tier: 64K below 96 GiB, so a smaller Mac is quoted
+            // the window and footprint it will actually launch with.
+            const ramTieredCtx = ds4RamTieredContext(totalMemoryBytes());
             const ds4Override = config.modelContextOverrides?.[`ds4:${modelId}`];
             const ds4Floor = ds4Override !== undefined ? Math.min(contextFloor, ds4Override) : contextFloor;
             const effective = resolveDs4LaunchCtx({
@@ -1086,36 +1113,69 @@ export class LocalEngineRuntime {
             // `fixed + slope × ctx` line llama.cpp/MLX rows carry, so the slider
             // prices a drag instead of claiming the footprint never moves. Entries
             // without a measured slope keep the flat number and say so.
-            const { ds4ProjectedResidentBytes, ds4ResidentBytesForMode, ds4ResidentLine, shouldUseDs4SsdStreaming, } = await import('../ds4/residency.js');
+            const { ds4ProjectedResidentBytes, ds4ResidentBytesForMode, ds4ResidentLine, planDs4MeasuredLaunch, shouldUseDs4SsdStreaming, } = await import('../ds4/residency.js');
             const ds4Line = ds4ResidentLine({
                 residentBytes: ds4Source?.residentBytes,
                 kvBytesPerToken: ds4Source?.kvBytesPerToken,
                 residentCtxTokens: ds4Source?.residentCtxTokens,
             });
             const visionResidentBytes = await this.resolveDs4VisionResidentBytes(config, modelId, installed, ds4Source);
+            const streamingSupported = ds4SsdStreamingSupported(ds4Source, process.platform);
             const ssdStreaming = shouldUseDs4SsdStreaming({
                 configured: config.ds4SsdStreaming,
-                ...(ds4Source?.ssdStreamingSupported !== undefined
-                    ? { ssdStreamingSupported: ds4Source.ssdStreamingSupported }
-                    : {}),
+                ...(streamingSupported !== undefined ? { ssdStreamingSupported: streamingSupported } : {}),
                 modelSizeBytes: ds4Source?.residentWeightBytes ??
                     ds4Source?.approxSizeBytes ??
                     installed?.approxSizeBytes,
                 ...(visionResidentBytes > 0 ? { companionBytes: visionResidentBytes } : {}),
             });
-            const plannedResidentBytes = ds4Line
-                ? ds4ResidentBytesForMode(ds4ProjectedResidentBytes(ds4Line, effective), ssdStreaming, ds4Source?.ssdStreamingSupported) + visionResidentBytes
-                : await this.resolveResidentBytes('ds4', modelId);
+            // A measurement for this launch's mode prices it exactly — the window
+            // moves only that mode's slope, and a streamed launch carries the cache
+            // the launcher would choose. Unmeasured entries keep the authored line.
+            const measured = planDs4MeasuredLaunch({
+                source: ds4Source,
+                numCtx: effective,
+                ssdStreaming,
+                totalRamBytes: totalMemoryBytes(),
+                budgetBytes: ssdStreaming && ds4Source?.streamingMemory
+                    ? await (await import('./device-capacity.js')).deviceModelBudgetBytes(config)
+                    : undefined,
+                sidecarBytes: visionResidentBytes,
+                configuredCacheGb: config.ds4CacheExpertsGb,
+            });
+            const plannedResidentBytes = measured
+                ? measured.workingSetBytes
+                : ds4Line
+                    ? ds4ResidentBytesForMode(ds4ProjectedResidentBytes(ds4Line, effective), ssdStreaming, streamingSupported) + visionResidentBytes
+                    : await this.resolveResidentBytes('ds4', modelId);
             return {
                 contextWindow: useResidentOr(effective, Math.min(effective, ds4Floor)),
                 plannedResidentBytes,
-                ...(ds4Line
+                ssdStreaming,
+                ...(measured
                     ? {
-                        weightsResidentBytes: ds4ResidentBytesForMode(ds4Line.contextFreeBytes, ssdStreaming, ds4Source?.ssdStreamingSupported) + visionResidentBytes,
+                        memoryMeasured: true,
+                        weightsResidentBytes: measured.contextFreeBytes,
                         kvFixedBytesPerSlot: 0,
-                        kvBytesPerTokenPerSlot: ds4Line.kvBytesPerToken,
+                        kvBytesPerTokenPerSlot: measured.bytesPerToken,
+                        ...(measured.cache
+                            ? {
+                                expertCache: {
+                                    cacheBytes: measured.cache.cacheGb * 1024 ** 3,
+                                    requestedBytes: measured.cache.requestedGb * 1024 ** 3,
+                                    clamped: measured.cache.clamped,
+                                    safe: measured.cache.safe,
+                                },
+                            }
+                            : {}),
                     }
-                    : {}),
+                    : ds4Line
+                        ? {
+                            weightsResidentBytes: ds4ResidentBytesForMode(ds4Line.contextFreeBytes, ssdStreaming, streamingSupported) + visionResidentBytes,
+                            kvFixedBytesPerSlot: 0,
+                            kvBytesPerTokenPerSlot: ds4Line.kvBytesPerToken,
+                        }
+                        : {}),
                 ...(nativeWindow ? { nativeContextWindow: nativeWindow } : {}),
                 ...(ds4Override !== undefined ? { overrideContextTokens: ds4Override } : {}),
                 ...(ds4AutoContextWindow !== undefined ? { autoContextWindow: ds4AutoContextWindow } : {}),
@@ -1190,9 +1250,32 @@ export class LocalEngineRuntime {
             ? effectiveLazyTensorBytes(summary, config.llamaCppLazyMode ?? manifestEngineConfig?.lazyMode ?? 'auto')
             : 0;
         const residentWeightFileBytes = Math.max(0, installed.approxSizeBytes - lazyTensorBytes);
-        const residentWeightsBytes = estimateLlamaCppResidentBytes(residentWeightFileBytes, {
+        const { perModel: previewEngineConfig } = await resolveSpecDraft({
+            perModel: manifestEngineConfig,
+            backend: currentLlamaCppBackend(),
+            configSpecType: config.llamaCppSpecType,
+            configDraftModelPath: config.llamaCppDraftModelPath,
+            installedDraftModelPath: installed.draftModelPath,
+            resolveDraftPath: async (id) => (await this.llamaCppModels?.resolveModelPath(id)) ?? null,
+        });
+        const previewSpecType = config.llamaCppSpecType ?? previewEngineConfig?.spec?.type;
+        const draftBudgetBytes = llamaCppSidecarLoads({
+            specType: previewSpecType,
+            targetHasMtp: (summary?.nextnPredictLayers ?? 0) > 0,
+            perBackend: !!manifestEngineConfig?.spec?.byBackend,
+        }) && installed.draftModelPath
+            ? (installed.draftModelSizeBytes ?? 0)
+            : 0;
+        const catalogDetail = draftBudgetBytes > 0 ? await this.catalog.get('chat-model', modelId).catch(() => null) : null;
+        const draftResidentBytes = llamaCppDraftResidentBytes(catalogDetail?.manifest.kind === 'chat-model'
+            ? catalogDetail.manifest.llamaCpp?.draftModel
+            : undefined, installed.draftModelPath);
+        const residentBytesWith = (draftBytes) => estimateLlamaCppResidentBytes(residentWeightFileBytes, {
             mmprojBytes: visionBudgetBytes,
+            draftBytes,
+            draftResidentBytes,
         });
+        let residentWeightsBytes = residentBytesWith(draftBudgetBytes);
         const exactPerSlotKvF16 = summary
             ? estimateExactPerSlotKvBytesF16({
                 blockCount: summary.blockCount,
@@ -1452,6 +1535,7 @@ export class LocalEngineRuntime {
                 maxSlots: defaultLocalEngineSlots(fastBudgetBytes),
             });
             kv = kvPlan.kvCacheType;
+            const draftFits = planFitsAt(kv, kvPlan.ctxCapTokens ?? grantedCtx, 1);
             // Mirror the launch: the f16-by-context-cap trade shrinks the granted
             // window here too, so the settings preview shows what will really run.
             if (kvPlan.ctxCapTokens !== undefined && kvPlan.ctxCapTokens < grantedCtx) {
@@ -1462,7 +1546,7 @@ export class LocalEngineRuntime {
                 ceiling: passCeiling(kv, grantedCtx),
                 tierDefault: defaultLocalEngineSlots(fastBudgetBytes),
             });
-            if ((config.llamaCppSpecType ?? manifestEngineConfig?.spec?.type) === 'draft-mtp')
+            if (previewSpecType === 'draft-mtp')
                 slots = 1;
             try {
                 if (!summary)
@@ -1544,7 +1628,14 @@ export class LocalEngineRuntime {
                         // SWA model without a readable layout, over budget on full
                         // math: the launch path leaves such a launch untouched — preview
                         // the requested window.
-                        return { grantedCtx, slots, kvCacheType: kv, plannedFieldsOk: false };
+                        return {
+                            grantedCtx,
+                            slots,
+                            kvCacheType: kv,
+                            plannedFieldsOk: false,
+                            draftFits,
+                            kvPlan,
+                        };
                     }
                 }
                 if (!admission.minimumSatisfied) {
@@ -1588,9 +1679,21 @@ export class LocalEngineRuntime {
                     throw error;
                 log.warn(`[llama-cpp] could not inspect ${modelId} while previewing admission: ${error instanceof Error ? error.message : String(error)}`);
             }
-            return { grantedCtx, slots, kvCacheType: kv, plannedFieldsOk: true };
+            return { grantedCtx, slots, kvCacheType: kv, plannedFieldsOk: true, draftFits, kvPlan };
         };
-        const plan = llamaPlanPass(explicitContextWindow, contextFloor);
+        let plan = llamaPlanPass(explicitContextWindow, contextFloor);
+        // The launch leaves out a catalog drafter that would cap the target window
+        // or cost its KV precision; the preview has to land on the same plan.
+        if (draftBudgetBytes > 0 && !config.llamaCppSpecType && !config.llamaCppDraftModelPath) {
+            residentWeightsBytes = residentBytesWith(0);
+            const withoutDrafter = llamaPlanPass(explicitContextWindow, contextFloor);
+            if (drafterCostsContext(plan.kvPlan, withoutDrafter.kvPlan, plan.draftFits)) {
+                plan = withoutDrafter;
+            }
+            else {
+                residentWeightsBytes = residentBytesWith(draftBudgetBytes);
+            }
+        }
         let autoContextWindow;
         if (overrideActive) {
             try {

package/dist/providers/native/supervisor.d.ts

@@ -516,7 +516,14 @@ export declare class NativeEngineSupervisor {
     private clearFreezeTimer;
     private clearHealthTimer;
 }
-/** High-precision fatal signatures; attribution happens only if the child exits. */
+/**
+ * High-precision fatal signatures; attribution happens only if the child exits.
+ *
+ * The `cuda-*` kinds also cover ROCm builds: ggml's ROCm backend is its CUDA
+ * code compiled with HIP, and it prints the same errors under the name `ROCm`
+ * (`ROCm error: out of memory`). The kinds are wire values, so they keep their
+ * names rather than growing a ROCm twin each.
+ */
 export declare function classifyNativeEnginePanic(line: string): {
     kind: NativeEnginePanicKind;
     line: string;

package/dist/providers/native/supervisor.js

@@ -1415,13 +1415,20 @@ export class NativeEngineSupervisor {
         }
     }
 }
-/** High-precision fatal signatures; attribution happens only if the child exits. */
+/**
+ * High-precision fatal signatures; attribution happens only if the child exits.
+ *
+ * The `cuda-*` kinds also cover ROCm builds: ggml's ROCm backend is its CUDA
+ * code compiled with HIP, and it prints the same errors under the name `ROCm`
+ * (`ROCm error: out of memory`). The kinds are wire values, so they keep their
+ * names rather than growing a ROCm twin each.
+ */
 export function classifyNativeEnginePanic(line) {
     const clipped = line.trim().slice(0, 1_000);
-    if (/CUDA error:\s*invalid (?:configuration )?argument/i.test(line)) {
+    if (/(?:CUDA|ROCm) error:\s*invalid (?:configuration )?argument/i.test(line)) {
         return { kind: 'cuda-invalid-argument', line: clipped };
     }
-    if (/CUDA error:\s*(?:out of memory|memory allocation)/i.test(line)) {
+    if (/(?:CUDA|ROCm) error:\s*(?:out of memory|memory allocation)/i.test(line)) {
         return { kind: 'cuda-out-of-memory', line: clipped };
     }
     // Vulkan's spellings of the same condition — the backend a non-CUDA
@@ -1429,16 +1436,18 @@ export function classifyNativeEnginePanic(line) {
     if (/ErrorOutOfDeviceMemory|OutOfDeviceMemoryError|ggml_vulkan:.*(?:allocation|memory).*failed/i.test(line)) {
         return { kind: 'vulkan-out-of-memory', line: clipped };
     }
-    if (/CUDA error:\s*(?:an )?illegal memory access/i.test(line)) {
+    if (/(?:CUDA|ROCm) error:\s*(?:an )?illegal memory access/i.test(line)) {
         return { kind: 'cuda-illegal-memory-access', line: clipped };
     }
-    if (/CUDA error:.*device-side assert/i.test(line)) {
+    if (/(?:CUDA|ROCm) error:.*device-side assert/i.test(line)) {
         return { kind: 'cuda-device-assert', line: clipped };
     }
-    if (/CUDA error:\s*(?:no kernel image is available|the provided PTX was compiled with an unsupported toolchain)/i.test(line)) {
+    if (/(?:CUDA|ROCm) error:\s*(?:no kernel image is available|invalid device function|the provided PTX was compiled with an unsupported toolchain)/i.test(line) ||
+        // rocBLAS names the architecture it has no kernel library for, then aborts.
+        /rocBLAS error: Cannot read .*TensileLibrary.*for GPU arch/i.test(line)) {
         return { kind: 'cuda-unsupported-device', line: clipped };
     }
-    if (/CUDA error:/i.test(line))
+    if (/(?:CUDA|ROCm) error:/i.test(line))
         return { kind: 'cuda-error', line: clipped };
     if (/assert(?:ion)? failed|GGML_ASSERT/i.test(line)) {
         return { kind: 'assertion-failed', line: clipped };

package/dist/providers/types.d.ts

@@ -600,6 +600,10 @@ export interface EngineLaunchSnapshot {
 export interface ModelInfo {
     id: string;
     name: string;
+    /** One short line a picker shows beside the name. */
+    description?: string;
+    /** Heading a picker lists this model under; see core's `ModelInfoSchema`. */
+    group?: string;
     supportsReasoning?: boolean;
     reasoningEfforts?: string[];
     defaultReasoningEffort?: string;

package/dist/remotes/types.d.ts

@@ -11,7 +11,7 @@ export interface PairedRemote {
     remoteId: string;
     /** Base URL A reaches the server at, e.g. `https://192.168.1.50:6229`. */
     baseUrl: string;
-    /** Human label shown in A's model picker / Remote Servers panel. */
+    /** Human label shown in A's model picker / Remote servers panel. */
     displayName: string;
     /** Per-device bearer token the server issued A (scope `remote-inference`). */
     token: string;