@bendyline/gezel-inference 0.0.0-stage → 0.1.0

package/dist/providers/llama-cpp/engine-flags.d.ts

@@ -0,0 +1,181 @@
+/**
+ * Pure builder for llama-server's ENGINE launch flags (argv), kept
+ * separate from the launcher in `chat/manager.ts` so it can be
+ * unit-tested in isolation. It owns only the flags that were NOT
+ * already computed by the launcher (`--model`, `--ctx-size`,
+ * `--parallel`, `--slot-save-path`, `--cache-type-k/v`, `--jinja`,
+ * `--mmproj` and `--reasoning-budget` stay in the launcher).
+ *
+ * Three input layers, merged per-field with this precedence (first
+ * defined wins):
+ *
+ *   1. `config`   — user's global `GezelConfig.llamaCpp*` overrides.
+ *   2. `perModel` — the catalog manifest's `tuning.engine.llamaCpp`.
+ *   3. `planner`  — Phase v2's hardware-aware offload decision.
+ *
+ * …then `config.llamaCppExtraArgs` is appended LAST so it can override
+ * anything above (llama-server honours the last occurrence of a flag).
+ *
+ * Types are declared structurally here (not imported from
+ * `@bendyline/gezel-core`) so the helper and its tests carry no schema
+ * dependency; the real `GezelConfig` / `LlamaCppEngineConfig` satisfy
+ * these shapes by construction.
+ */
+/** The subset of `GezelConfig` this builder reads (global overrides). */
+export interface GlobalLlamaCppFlags {
+    llamaCppFlashAttn?: boolean | 'on' | 'off' | 'auto';
+    llamaCppNGpuLayers?: number;
+    llamaCppCpuMoe?: boolean;
+    llamaCppNCpuMoe?: number;
+    llamaCppNCpuFfn?: number;
+    llamaCppLoadMode?: 'auto' | 'none' | 'mmap' | 'mlock' | 'mmap+mlock' | 'dio';
+    /** Legacy input, translated to v0.4.0's `--load-mode mlock`. */
+    llamaCppMlock?: boolean;
+    llamaCppLazyMode?: 'on' | 'auto' | 'off';
+    llamaCppCacheReuse?: number;
+    llamaCppSwaFull?: boolean;
+    llamaCppThreads?: number;
+    llamaCppBatchSize?: number;
+    llamaCppUbatchSize?: number;
+    llamaCppSpecType?: string;
+    llamaCppDraftModelPath?: string;
+    llamaCppSpecDraftNMax?: number;
+    llamaCppSpecDraftSampling?: 'greedy' | 'probabilistic';
+    llamaCppExtraArgs?: Record<string, string | number | boolean>;
+}
+/** The catalog manifest's per-model `tuning.engine.llamaCpp` block. */
+export interface PerModelLlamaCppEngineConfig {
+    nGpuLayers?: number;
+    cpuMoe?: boolean;
+    nCpuMoe?: number;
+    nCpuFfn?: number;
+    loadMode?: 'auto' | 'none' | 'mmap' | 'mlock' | 'mmap+mlock' | 'dio';
+    lazyMode?: 'on' | 'auto' | 'off';
+    cacheReuse?: number;
+    swaFull?: boolean;
+    flashAttn?: 'on' | 'off' | 'auto';
+    ubatchSize?: number;
+    contextSize?: number;
+    chatTemplate?: string;
+    threads?: number;
+    batchSize?: number;
+    spec?: {
+        type?: string;
+        mtp?: boolean;
+        draftModelId?: string;
+        nMax?: number;
+    };
+}
+/** Phase v2 hardware-aware offload decision (lowest precedence). */
+export interface PlannerOffloadDecision {
+    nGpuLayers?: number;
+    cpuMoe?: boolean;
+    nCpuMoe?: number;
+    nCpuFfn?: number;
+    /** Every weight fits the GPU pool (see `MoeOffloadDecision.fullGpuResidency`). */
+    fullGpuResidency?: true;
+    /** Human-readable reason, for the decision log (not emitted as a flag). */
+    reason?: string;
+}
+export interface EngineFlagInput {
+    config: GlobalLlamaCppFlags;
+    perModel?: PerModelLlamaCppEngineConfig | undefined;
+    planner?: PlannerOffloadDecision | undefined;
+    /**
+     * The resolved KV-cache type the launcher is passing to
+     * `--cache-type-k/v`. Used only for the flash-attn coherence rule:
+     * a quantized KV cache wants FA on for the fast path, so when FA is
+     * otherwise unset we force it on.
+     */
+    kvCacheType?: string | undefined;
+    /**
+     * Server slot count (`--parallel`). b9843 rejects `--cache-reuse`
+     * under multi-slot non-unified KV, so the auto-on cache-reuse default
+     * is applied ONLY when this is exactly 1. An explicit config/manifest
+     * `cacheReuse` is passed regardless.
+     */
+    slots?: number | undefined;
+    /**
+     * Whether the model's GGUF actually ships MTP (`nextn`) layers, read
+     * from its header. A safety cross-check for an explicit `draft-mtp`
+     * request: selecting it for a model without MTP layers makes llama-server
+     * exit with a fatal load error, so the flag is emitted only when this
+     * confirms the capability.
+     */
+    ggufHasMtp?: boolean | undefined;
+    /**
+     * Absolute path to a catalog-installed speculative draft sidecar.
+     * Separate MTP assistants (Gemma) and other draft algorithms need this;
+     * combined MTP GGUFs (Qwen) intentionally leave it unset.
+     */
+    installedDraftModelPath?: string | undefined;
+    /**
+     * `--reasoning-format <value>` opt-in (resolved from the
+     * `GEZEL_LLAMA_REASONING_FORMAT` env var at the call site). `none`
+     * stops llama-server from parsing chat-template tool-call / channel
+     * markup server-side, so raw model output reaches `delta.content`
+     * where the salvage layer can catch it — the A/B lever for the
+     * peg-gemma4 parser dropping mangled `<|tool_call>` turns entirely
+     * pre-SSE (cbmx-20260720-195716, ~74 trials). Omitted when unset:
+     * default behavior is unchanged.
+     */
+    reasoningFormat?: string | undefined;
+    /**
+     * Preserve assistant reasoning across tool-loop/history turns. This is an
+     * explicit experiment because it changes the rendered prompt and only works
+     * when the client also replays `reasoning_content`.
+     */
+    reasoningPreserve?: boolean | undefined;
+    /**
+     * The model's GGUF `general.architecture`, for the Gemma-family auto
+     * defaults (sliding-window full cache). Undefined disables the arch
+     * heuristic; an explicit config/manifest `swaFull` still applies.
+     */
+    architecture?: string | undefined;
+    /** The resolved model id, a fallback for `architecture` (see isGemmaModel). */
+    modelId?: string | undefined;
+    /**
+     * Whether resident weights plus the full SWA KV cache fit the host's
+     * fast-memory pool (VRAM on a discrete GPU). The exact estimate honors
+     * per-layer SWA dimensions and shared-KV layers. `false` declines the
+     * auto-default so a low-memory device keeps the model's full context on
+     * the windowed cache instead of spilling or OOMing. `undefined` (unknown
+     * host/metadata) and `true` retain the compatibility default. An explicit
+     * config/manifest `swaFull` wins regardless.
+     */
+    swaFullAutoFits?: boolean | undefined;
+    /**
+     * The llama-server build's backend (`cuda`, `vulkan`, `metal`, `cpu`).
+     * Only a GPU backend can receive the GPU-resident thread default.
+     */
+    backend?: string | undefined;
+    /** Logical CPUs available to the daemon; caps the GPU-resident thread default. */
+    hostCpuCount?: number | undefined;
+}
+/** Default `--cache-reuse` chunk when neither config nor manifest set it. */
+export declare const DEFAULT_CACHE_REUSE = 256;
+/**
+ * Upper bound on `--threads` for an engine whose every weight is on the GPU,
+ * when neither config nor manifest chose a count.
+ *
+ * Left alone, llama.cpp sizes its CPU pool to every physical core: all 20 on
+ * a DGX Spark, where nothing is held back because there is no SMT and no
+ * efficiency-core exclusion on Linux ARM. The pool spins between steps even
+ * though the GPU does the model's work, and from the bundled 0.1.48 build
+ * (upstream v0.6.0) on it spins hard enough that a pool covering every core
+ * starves the thread driving the GPU. Measured on the Spark: gemma4-e2b
+ * decoded at 72 t/s with the default 20 threads and ~120 t/s anywhere from 1
+ * to 16; 10 threads pinned to 10 cores fell to 58 t/s, so the cause is the
+ * missing spare core, not the core type. gemma4-e4b, granite-vision and
+ * qwen3.8-flash-next lost 15-41% the same way. Decode and prefill were flat
+ * across small pools while CPU time grew linearly with the thread count, so
+ * a small pool with headroom is both the fastest and the cheapest setting.
+ */
+export declare const GPU_RESIDENT_THREADS = 4;
+/**
+ * Build the engine-flag argv fragment. Deterministic ordering: the
+ * first-class flags in a fixed sequence, then `extraArgs` in insertion
+ * order. Returns a flat `string[]` ready to splice into the launcher's
+ * `args` array.
+ */
+export declare function buildLlamaCppEngineArgs(input: EngineFlagInput): string[];

package/dist/providers/llama-cpp/engine-flags.js

@@ -0,0 +1,279 @@
+/**
+ * Pure builder for llama-server's ENGINE launch flags (argv), kept
+ * separate from the launcher in `chat/manager.ts` so it can be
+ * unit-tested in isolation. It owns only the flags that were NOT
+ * already computed by the launcher (`--model`, `--ctx-size`,
+ * `--parallel`, `--slot-save-path`, `--cache-type-k/v`, `--jinja`,
+ * `--mmproj` and `--reasoning-budget` stay in the launcher).
+ *
+ * Three input layers, merged per-field with this precedence (first
+ * defined wins):
+ *
+ *   1. `config`   — user's global `GezelConfig.llamaCpp*` overrides.
+ *   2. `perModel` — the catalog manifest's `tuning.engine.llamaCpp`.
+ *   3. `planner`  — Phase v2's hardware-aware offload decision.
+ *
+ * …then `config.llamaCppExtraArgs` is appended LAST so it can override
+ * anything above (llama-server honours the last occurrence of a flag).
+ *
+ * Types are declared structurally here (not imported from
+ * `@bendyline/gezel-core`) so the helper and its tests carry no schema
+ * dependency; the real `GezelConfig` / `LlamaCppEngineConfig` satisfy
+ * these shapes by construction.
+ */
+import { isGemmaModel } from './kv-cache-type.js';
+/** Default `--cache-reuse` chunk when neither config nor manifest set it. */
+export const DEFAULT_CACHE_REUSE = 256;
+/**
+ * Upper bound on `--threads` for an engine whose every weight is on the GPU,
+ * when neither config nor manifest chose a count.
+ *
+ * Left alone, llama.cpp sizes its CPU pool to every physical core: all 20 on
+ * a DGX Spark, where nothing is held back because there is no SMT and no
+ * efficiency-core exclusion on Linux ARM. The pool spins between steps even
+ * though the GPU does the model's work, and from the bundled 0.1.48 build
+ * (upstream v0.6.0) on it spins hard enough that a pool covering every core
+ * starves the thread driving the GPU. Measured on the Spark: gemma4-e2b
+ * decoded at 72 t/s with the default 20 threads and ~120 t/s anywhere from 1
+ * to 16; 10 threads pinned to 10 cores fell to 58 t/s, so the cause is the
+ * missing spare core, not the core type. gemma4-e4b, granite-vision and
+ * qwen3.8-flash-next lost 15-41% the same way. Decode and prefill were flat
+ * across small pools while CPU time grew linearly with the thread count, so
+ * a small pool with headroom is both the fastest and the cheapest setting.
+ */
+export const GPU_RESIDENT_THREADS = 4;
+/** Cores left free for the server, sampling and GPU-driver threads. */
+const GPU_RESIDENT_SPARE_CORES = 2;
+function gpuResidentThreads(hostCpuCount) {
+    const available = typeof hostCpuCount === 'number' && hostCpuCount > 0
+        ? hostCpuCount - GPU_RESIDENT_SPARE_CORES
+        : GPU_RESIDENT_THREADS;
+    return Math.max(1, Math.min(GPU_RESIDENT_THREADS, available));
+}
+const GPU_BACKENDS = new Set(['cuda', 'vulkan', 'metal']);
+/**
+ * `llamaCppExtraArgs` keys that either choose a thread pool or may move
+ * weights to the CPU; any of them makes the GPU-resident default stand down.
+ */
+const EXTRA_ARGS_OWNING_THREADS = new Set([
+    'threads',
+    't',
+    'threads-batch',
+    'tb',
+    'cpu-mask',
+    'cpu-range',
+    'n-gpu-layers',
+    'gpu-layers',
+    'ngl',
+    'cpu-moe',
+    'n-cpu-moe',
+    'n-cpu-ffn',
+    'override-tensor',
+    'ot',
+]);
+function extraArgsOwnThreads(extraArgs) {
+    if (!extraArgs)
+        return false;
+    return Object.keys(extraArgs).some((key) => EXTRA_ARGS_OWNING_THREADS.has(key.replace(/^-+/, '')));
+}
+/**
+ * Normalize the tri-state-or-legacy-boolean flash-attn config into the
+ * string llama-server wants, or `undefined` to omit the flag.
+ * Back-compat: legacy `true` → force `on`; legacy `false` → omit (its
+ * historical meaning was "don't pass the flag", i.e. server default),
+ * NOT force-off. Force-off is reachable only via the explicit `'off'`.
+ */
+function normalizeFlashAttn(v) {
+    if (v === true)
+        return 'on';
+    if (v === false || v === undefined)
+        return undefined;
+    return v;
+}
+/** Strip any leading dashes and re-prefix with `--`. */
+function normalizeFlagKey(key) {
+    return `--${key.replace(/^-+/, '')}`;
+}
+/**
+ * Build the engine-flag argv fragment. Deterministic ordering: the
+ * first-class flags in a fixed sequence, then `extraArgs` in insertion
+ * order. Returns a flat `string[]` ready to splice into the launcher's
+ * `args` array.
+ */
+export function buildLlamaCppEngineArgs(input) {
+    const { config, perModel, planner, kvCacheType, slots, ggufHasMtp, installedDraftModelPath, reasoningFormat, reasoningPreserve, architecture, modelId, swaFullAutoFits, backend, hostCpuCount, } = input;
+    const args = [];
+    // ── GPU-layer offload (`--n-gpu-layers`) ──────────────────────────
+    const nGpuLayers = config.llamaCppNGpuLayers ?? perModel?.nGpuLayers ?? planner?.nGpuLayers;
+    if (typeof nGpuLayers === 'number') {
+        args.push('--n-gpu-layers', nGpuLayers === -1 ? 'all' : String(nGpuLayers));
+    }
+    // ── MoE expert offload (`--cpu-moe` / `--n-cpu-moe`) ──────────────
+    // Tri-state: an explicit `true`/`false` in config or manifest wins;
+    // `undefined` at both layers falls through to the planner's auto
+    // decision. An explicit `false` is "force experts onto the GPU" — it
+    // must also suppress the planner's partial `--n-cpu-moe`, not just the
+    // all-experts `--cpu-moe`, or "Off" would silently still offload.
+    const explicitCpuMoe = config.llamaCppCpuMoe ?? perModel?.cpuMoe;
+    const cpuMoe = explicitCpuMoe ?? planner?.cpuMoe;
+    // An explicit partial split (config/manifest) is a specific opinion that
+    // always applies. Only the planner's suggested partial split is
+    // suppressed when experts are explicitly forced off.
+    const explicitNCpuMoe = config.llamaCppNCpuMoe ?? perModel?.nCpuMoe;
+    const nCpuMoe = explicitNCpuMoe ?? (explicitCpuMoe === false ? undefined : planner?.nCpuMoe);
+    if (cpuMoe) {
+        // All experts on CPU — the granular `--n-cpu-moe` would be redundant.
+        args.push('--cpu-moe');
+    }
+    else if (typeof nCpuMoe === 'number') {
+        args.push('--n-cpu-moe', String(nCpuMoe));
+    }
+    // ── Dense FFN offload (`--n-cpu-ffn`) ────────────────────────────
+    // Zero is an explicit "do not offload" sentinel: omit the CLI flag but
+    // suppress the planner. Positive values pin the first N blocks' dense
+    // FFN tensors to CPU while `-ngl all` can keep attention on the GPU.
+    const explicitNCpuFfn = config.llamaCppNCpuFfn ?? perModel?.nCpuFfn;
+    const nCpuFfn = explicitNCpuFfn ?? planner?.nCpuFfn;
+    if (typeof nCpuFfn === 'number' && nCpuFfn > 0) {
+        args.push('--n-cpu-ffn', String(nCpuFfn));
+    }
+    // ── Model loading (`--load-mode` / `--lazy-mode`) ────────────────
+    // v0.4.0 consolidates mmap/mlock/direct-I/O under --load-mode. Preserve
+    // the old `llamaCppMlock` setting without emitting its deprecated flag.
+    const loadMode = config.llamaCppLoadMode ?? (config.llamaCppMlock ? 'mlock' : undefined) ?? perModel?.loadMode;
+    if (loadMode)
+        args.push('--load-mode', loadMode);
+    const lazyMode = config.llamaCppLazyMode ?? perModel?.lazyMode;
+    if (lazyMode)
+        args.push('--lazy-mode', lazyMode);
+    // ── Flash attention (`--flash-attn on|off|auto`) ──────────────────
+    let flashAttn = normalizeFlashAttn(config.llamaCppFlashAttn) ?? perModel?.flashAttn;
+    // Coherence: quantized KV cache wants FA on for the fast path. Only
+    // applied when neither config nor manifest expressed an opinion.
+    if (flashAttn === undefined && kvCacheType && kvCacheType !== 'f16') {
+        flashAttn = 'on';
+    }
+    if (flashAttn !== undefined)
+        args.push('--flash-attn', flashAttn);
+    // ── Cross-request prefix KV reuse (`--cache-reuse`) ───────────────
+    // Passed regardless of slot count. The old rule withheld the default
+    // whenever `slots !== 1`, because b9843 was believed to reject
+    // `--cache-reuse` under multi-slot non-unified KV. Re-measured against
+    // the bundled 0.1.36 engine (upstream f8def7f) by launching it directly:
+    //
+    //   dense model, --parallel 1  -> accepted, no warning, no error
+    //   dense model, --parallel 4  -> accepted, no warning, no error
+    //
+    // `kv_unified='false'` turns out to be normal at EVERY slot count — the
+    // dense control reports it at `--parallel 1` too — so it was never the
+    // discriminator the old comment took it for. What actually decides
+    // support is whether the context's cache can be partially rewound:
+    //
+    //   dense (granite 4b)            -> supported
+    //   hybrid recurrent (qwen3.5+)   -> "not supported ... will be disabled"
+    //   gemma windowed (SWA)          -> "not supported ... will be disabled"
+    //   gemma + `--swa-full`          -> supported
+    //
+    // So the engine already declines it precisely when it cannot honour it,
+    // which is what the warning is for. Withholding the flag ourselves only
+    // cost reuse on multi-slot dense engines. Note the coupling this exposes:
+    // the memory-driven `--swa-full` decision below also silently decides
+    // whether Gemma gets any cross-request prefix reuse at all.
+    const explicitCacheReuse = config.llamaCppCacheReuse ?? perModel?.cacheReuse;
+    const cacheReuse = explicitCacheReuse ?? DEFAULT_CACHE_REUSE;
+    if (cacheReuse > 0)
+        args.push('--cache-reuse', String(cacheReuse));
+    // ── SWA full cache (`--swa-full`) ─────────────────────────────────
+    // Tri-state: explicit config/manifest `true`/`false` wins; `undefined`
+    // at both layers falls through to the Gemma-family auto-default. Gemma's
+    // sliding-window attention refuses cross-request prefix reuse under the
+    // memory-efficient windowed cache; the full cache preserves old SWA KV
+    // for cache operations by allocating full-context storage on every
+    // cache-owning layer. The cost varies sharply with model geometry, so
+    // Auto is gated on `swaFullAutoFits`: when weights + full KV do not fit
+    // fast memory, Gemma launches with the windowed cache instead.
+    const swaFull = config.llamaCppSwaFull ??
+        perModel?.swaFull ??
+        (swaFullAutoFits !== false && isGemmaModel({ architecture, modelId }));
+    if (swaFull)
+        args.push('--swa-full');
+    // ── Thread / batch overrides ──────────────────────────────────────
+    // The GPU-resident default needs positive evidence that no model compute
+    // runs on the CPU: a GPU build, a planner that measured a full fit, and no
+    // CPU placement from any layer. Anything less keeps llama.cpp's own
+    // all-cores default, because CPU-resident layers need every thread.
+    const gpuResident = backend !== undefined &&
+        GPU_BACKENDS.has(backend) &&
+        planner?.fullGpuResidency === true &&
+        (nGpuLayers === undefined || nGpuLayers === -1) &&
+        !cpuMoe &&
+        !(typeof nCpuMoe === 'number' && nCpuMoe > 0) &&
+        !(typeof nCpuFfn === 'number' && nCpuFfn > 0) &&
+        !extraArgsOwnThreads(config.llamaCppExtraArgs);
+    const threads = config.llamaCppThreads ??
+        perModel?.threads ??
+        (gpuResident ? gpuResidentThreads(hostCpuCount) : undefined);
+    if (typeof threads === 'number')
+        args.push('--threads', String(threads));
+    const batchSize = config.llamaCppBatchSize ?? perModel?.batchSize;
+    if (typeof batchSize === 'number')
+        args.push('--batch-size', String(batchSize));
+    const ubatchSize = config.llamaCppUbatchSize ?? perModel?.ubatchSize;
+    if (typeof ubatchSize === 'number')
+        args.push('--ubatch-size', String(ubatchSize));
+    if (perModel?.chatTemplate)
+        args.push('--chat-template', perModel.chatTemplate);
+    // ── Speculative decoding (`--spec-type` + draft knobs) ────────────
+    // MTP is ON by default for a GGUF that carries the head, having cleared the
+    // A/B gate this comment used to wait for (reports/llama-mtp-eval-20260829.md):
+    // +12.7% / +4.1% / +15.6% / +19.3% decode at 5.5k / 21.7k / 54k / 87k
+    // context, never negative, byte-identical greedy output, +1.35 GB resident.
+    // The gain GROWS with context — long-context decode is bound by reading the
+    // KV cache, and verifying K drafted tokens costs one pass over it instead of
+    // K — which is the regime agent turns actually run in.
+    //
+    // `ggufHasMtp` is the load-bearing gate either way: llama-server exits
+    // FATALLY when draft-mtp is selected for a model with no MTP tensors, so
+    // the default may only ever apply where the metadata confirms the head.
+    // Explicit config and manifest `spec.type` still win, including 'none'.
+    const specType = config.llamaCppSpecType ?? perModel?.spec?.type ?? (ggufHasMtp ? 'draft-mtp' : undefined);
+    const safeSpecType = specType === 'draft-mtp' && !ggufHasMtp ? undefined : specType;
+    if (safeSpecType && safeSpecType !== 'none') {
+        args.push('--spec-type', safeSpecType);
+        if (safeSpecType.startsWith('draft-')) {
+            const draftModel = config.llamaCppDraftModelPath ?? installedDraftModelPath ?? perModel?.spec?.draftModelId;
+            if (draftModel)
+                args.push('--spec-draft-model', draftModel);
+        }
+        const nMax = config.llamaCppSpecDraftNMax ?? perModel?.spec?.nMax;
+        if (typeof nMax === 'number')
+            args.push('--spec-draft-n-max', String(nMax));
+        // v0.6.0 supports probabilistic selection only for MTP/simple drafts.
+        // Leave the upstream greedy default untouched unless explicitly set.
+        if ((safeSpecType === 'draft-mtp' || safeSpecType === 'draft-simple') &&
+            config.llamaCppSpecDraftSampling) {
+            args.push('--spec-draft-sampling', config.llamaCppSpecDraftSampling);
+        }
+    }
+    // ── Server-side output parsing (`--reasoning-format`) ─────────────
+    if (reasoningFormat)
+        args.push('--reasoning-format', reasoningFormat);
+    // ── Cross-turn reasoning history (`--reasoning-preserve`) ─────────
+    if (reasoningPreserve === true)
+        args.push('--reasoning-preserve');
+    else if (reasoningPreserve === false)
+        args.push('--no-reasoning-preserve');
+    // ── Escape hatch (`llamaCppExtraArgs`) — applied last, wins ────────
+    if (config.llamaCppExtraArgs) {
+        for (const [rawKey, value] of Object.entries(config.llamaCppExtraArgs)) {
+            if (value === false)
+                continue; // false = omit the flag entirely
+            const key = normalizeFlagKey(rawKey);
+            if (value === true)
+                args.push(key);
+            else
+                args.push(key, String(value));
+        }
+    }
+    return args;
+}

package/dist/providers/llama-cpp/gguf-metadata-async.d.ts

@@ -0,0 +1,11 @@
+import { type GgufSummary } from './gguf-metadata.js';
+export interface GgufSummaryOptions {
+    includeTensors?: boolean;
+    includeTensorSizes?: boolean;
+}
+/**
+ * Inspect a GGUF without blocking the daemon event loop. Results are keyed by
+ * file identity and option set, so Settings' model-list polling only parses a
+ * given installed payload once and a replaced model invalidates naturally.
+ */
+export declare function readGgufSummaryAsync(path: string, options?: GgufSummaryOptions): Promise<GgufSummary>;

package/dist/providers/llama-cpp/gguf-metadata-async.js

@@ -0,0 +1,180 @@
+import { stat } from 'node:fs/promises';
+import { basename, dirname, join, resolve } from 'node:path';
+import { Worker } from 'node:worker_threads';
+import { createLogger } from '@bendyline/gezel';
+import { findServiceWorkerEntry } from '../../utils/service-worker-entry.js';
+import { readGgufSummary } from './gguf-metadata.js';
+const log = createLogger('gguf:metadata-worker');
+const MAX_CACHE_ENTRIES = 32;
+let worker = null;
+let nextId = 1;
+const pending = new Map();
+const cache = new Map();
+const inFlight = new Map();
+/**
+ * Inspect a GGUF without blocking the daemon event loop. Results are keyed by
+ * file identity and option set, so Settings' model-list polling only parses a
+ * given installed payload once and a replaced model invalidates naturally.
+ */
+export async function readGgufSummaryAsync(path, options) {
+    const shardPaths = await discoverShardPaths(path);
+    const infos = await Promise.all(shardPaths.map((shardPath) => stat(shardPath)));
+    const key = [
+        ...shardPaths.flatMap((shardPath, index) => [
+            resolve(shardPath),
+            infos[index].size,
+            infos[index].mtimeMs,
+        ]),
+        options?.includeTensors === true ? 1 : 0,
+        options?.includeTensorSizes === true ? 1 : 0,
+    ].join('\n');
+    const cached = cache.get(key);
+    if (cached)
+        return cached;
+    const active = inFlight.get(key);
+    if (active)
+        return active;
+    const request = Promise.all(shardPaths.map((shardPath) => runInWorker(shardPath, options))).then((summaries) => {
+        const summary = aggregateShardSummaries(summaries);
+        cache.set(key, summary);
+        while (cache.size > MAX_CACHE_ENTRIES)
+            cache.delete(cache.keys().next().value);
+        return summary;
+    });
+    inFlight.set(key, request);
+    try {
+        return await request;
+    }
+    finally {
+        if (inFlight.get(key) === request)
+            inFlight.delete(key);
+    }
+}
+const SPLIT_GGUF_RE = /^(.*)-(\d+)-of-(\d+)\.gguf$/i;
+/** Resolve every sibling of a standard `-00001-of-00003.gguf` split. */
+async function discoverShardPaths(path) {
+    const match = SPLIT_GGUF_RE.exec(basename(path));
+    if (!match)
+        return [path];
+    const count = Number(match[3]);
+    const width = match[2].length;
+    if (!Number.isSafeInteger(count) || count < 1 || count > 10_000)
+        return [path];
+    const paths = Array.from({ length: count }, (_, index) => join(dirname(path), `${match[1]}-${String(index + 1).padStart(width, '0')}-of-${match[3]}.gguf`));
+    try {
+        await Promise.all(paths.map((shardPath) => stat(shardPath)));
+        return paths;
+    }
+    catch {
+        // An in-progress/partial install is not a valid model yet, but preserving
+        // single-file inspection keeps download validation and Settings previews
+        // useful instead of turning a missing sibling into a parser failure.
+        return [path];
+    }
+}
+function sumField(summaries, field) {
+    const values = summaries
+        .map((summary) => summary[field])
+        .filter((value) => value !== undefined);
+    return values.length > 0 ? values.reduce((sum, value) => sum + value, 0) : undefined;
+}
+function sumLayerField(summaries, field) {
+    const arrays = summaries
+        .map((summary) => summary[field])
+        .filter((value) => value !== undefined);
+    const length = Math.max(0, ...arrays.map((values) => values.length));
+    if (arrays.length === 0)
+        return undefined;
+    return Array.from({ length }, (_, index) => arrays.reduce((sum, values) => sum + (values[index] ?? 0), 0));
+}
+function aggregateShardSummaries(summaries) {
+    const first = summaries[0];
+    if (summaries.length === 1)
+        return first;
+    const tensorNames = summaries.flatMap((summary) => summary.tensorNames ?? []);
+    return {
+        ...first,
+        tensorCount: summaries.reduce((sum, summary) => sum + summary.tensorCount, 0n),
+        fileSizeBytes: summaries.reduce((sum, summary) => sum + summary.fileSizeBytes, 0),
+        bytesRead: summaries.reduce((sum, summary) => sum + summary.bytesRead, 0),
+        readOperations: summaries.reduce((sum, summary) => sum + summary.readOperations, 0),
+        shardCount: summaries.length,
+        ...(tensorNames.length > 0 ? { tensorNames } : {}),
+        expertBytesTotal: sumField(summaries, 'expertBytesTotal'),
+        nonExpertBytes: sumField(summaries, 'nonExpertBytes'),
+        expertBytesByLayer: sumLayerField(summaries, 'expertBytesByLayer'),
+        denseFfnBytesTotal: sumField(summaries, 'denseFfnBytesTotal'),
+        nonDenseFfnBytes: sumField(summaries, 'nonDenseFfnBytes'),
+        denseFfnBytesByLayer: sumLayerField(summaries, 'denseFfnBytesByLayer'),
+        lazyTensorBytesTotal: sumField(summaries, 'lazyTensorBytesTotal'),
+    };
+}
+async function runInWorker(path, options) {
+    // Vitest runs the TS source tree without emitted worker entries. Keep its
+    // synthetic fixtures simple and deterministic; production builds must ship
+    // the worker and fail this advisory preview if packaging regresses.
+    if (process.env.VITEST)
+        return readGgufSummary(path, options);
+    const target = ensureWorker();
+    if (!target) {
+        throw new Error('GGUF metadata worker is missing; rebuild @bendyline/gezel-service before starting the daemon');
+    }
+    const id = nextId++;
+    return new Promise((resolveSummary, reject) => {
+        pending.set(id, { resolve: resolveSummary, reject });
+        target.postMessage({ id, path, ...(options ? { options } : {}) });
+    });
+}
+function ensureWorker() {
+    if (worker)
+        return worker;
+    const entry = workerEntry();
+    if (!entry)
+        return null;
+    try {
+        const next = new Worker(entry);
+        next.on('message', onMessage);
+        next.on('error', (error) => onWorkerDown(error.message));
+        next.on('exit', (code) => {
+            if (code !== 0)
+                onWorkerDown(`worker exited with code ${code}`);
+            else if (worker === next)
+                worker = null;
+        });
+        next.unref();
+        worker = next;
+        return next;
+    }
+    catch (error) {
+        log.error(`failed to start GGUF metadata worker: ${describe(error)}`);
+        return null;
+    }
+}
+function workerEntry() {
+    return findServiceWorkerEntry(import.meta.url, 'gguf-metadata');
+}
+function onMessage(reply) {
+    const request = pending.get(reply.id);
+    if (!request)
+        return;
+    pending.delete(reply.id);
+    if (reply.error)
+        request.reject(new Error(reply.error));
+    else if (reply.summary)
+        request.resolve(reply.summary);
+    else
+        request.reject(new Error('GGUF metadata worker returned no summary'));
+}
+function onWorkerDown(reason) {
+    const dead = worker;
+    worker = null;
+    if (dead)
+        void dead.terminate().catch(() => { });
+    const error = new Error(`GGUF metadata worker stopped: ${reason}`);
+    for (const request of pending.values())
+        request.reject(error);
+    pending.clear();
+}
+function describe(error) {
+    return error instanceof Error ? error.message : String(error);
+}

package/dist/providers/llama-cpp/gguf-metadata-worker.d.ts

@@ -0,0 +1,7 @@
+/** Worker-thread host for GGUF metadata inspection.
+ *
+ * Large tokenizer vocabularies are cheap in bytes but expensive to walk. Keep
+ * even the first uncached inspection away from Electron's main thread when the
+ * service is embedded in a development app.
+ */
+export {};