@bendyline/gezel-inference 0.0.0-stage → 0.1.0

package/dist/providers/llama-cpp/gguf-quantization-backfill.js

@@ -0,0 +1,75 @@
+/**
+ * "What quantization is this, really?" — asked of the model file when the
+ * catalog's answer isn't one.
+ *
+ * A catalog `quantization` string is hand-authored content, and
+ * `muse-glimmer-30b-q4` shipped `K-Quant-17GB` — the upstream GGUF filename,
+ * which names no bit depth and rendered verbatim in a column of `~4` / `~8`.
+ * Installs from here on record the file's own `general.file_type` tag
+ * alongside the catalog's, but every copy already on disk predates the field,
+ * so this reads the header once and writes the answer down.
+ *
+ * Deliberately narrow: it runs only when the catalog tag carries no bit depth
+ * at all. A GGUF header read is cheap next to the weights but not free, and a
+ * model whose catalog label already says `Q4_K_M` has nothing to learn from
+ * asking the file.
+ */
+import { readFile, rename, writeFile } from 'node:fs/promises';
+import { join } from 'node:path';
+import { createLogger, quantizationBitDepths } from '@bendyline/gezel';
+import { makeSharedModelReadable } from '../../models/storage-roots.js';
+import { ggufQuantizationTag, readGgufSummary } from './gguf-metadata.js';
+const log = createLogger('models');
+/**
+ * In-process memo of what the header said, keyed by manifest path. Model
+ * inventory is polled, and a read-only overlay copy can never persist its
+ * answer — without this, every poll would re-read the header of every such
+ * model forever. `null` records "asked, and the file has no usable tag".
+ */
+const resolved = new Map();
+export async function backfillGgufQuantization(opts) {
+    if (quantizationBitDepths(opts.catalogQuantization).length > 0)
+        return undefined;
+    const manifestPath = join(opts.modelDir, 'manifest.json');
+    const memo = resolved.get(manifestPath);
+    if (memo !== undefined)
+        return memo ?? undefined;
+    let tag;
+    try {
+        tag = ggufQuantizationTag(readGgufSummary(join(opts.modelDir, opts.weightsFilename)));
+    }
+    catch (err) {
+        // The listing is not the place to fail over unreadable metadata — the
+        // payload verification above it already gates whether this model runs.
+        log.debug(`[${opts.engine}] could not read the GGUF quantization of "${opts.id}": ${err instanceof Error ? err.message : String(err)}`);
+        resolved.set(manifestPath, null);
+        return undefined;
+    }
+    resolved.set(manifestPath, tag ?? null);
+    if (!tag || !opts.writable)
+        return tag;
+    try {
+        const parsed = JSON.parse(await readFile(manifestPath, 'utf8'));
+        if (parsed.ggufQuantization === tag)
+            return tag;
+        const tmpPath = `${manifestPath}.quant.tmp`;
+        await writeFile(tmpPath, `${JSON.stringify({ ...parsed, ggufQuantization: tag }, null, 2)}\n`, {
+            encoding: 'utf8',
+        });
+        await rename(tmpPath, manifestPath);
+        // A machine service writes into the shared asset store under a private
+        // umask; without this the healed manifest becomes unreadable to the very
+        // desktop clients that list the model.
+        await makeSharedModelReadable(opts.modelDir);
+        log.info(`[${opts.engine}] "${opts.id}" declares ${tag}; recorded it beside the catalog's "${opts.catalogQuantization ?? '(none)'}"`);
+    }
+    catch (err) {
+        // A heal we couldn't write is a heal we redo next boot.
+        log.debug(`[${opts.engine}] could not record the GGUF quantization of "${opts.id}": ${err instanceof Error ? err.message : String(err)}`);
+    }
+    return tag;
+}
+/** Test seam — the memo is process-wide by design. */
+export function resetGgufQuantizationMemo() {
+    resolved.clear();
+}

package/dist/providers/llama-cpp/index.d.ts

@@ -0,0 +1,10 @@
+/**
+ * Public surface of the llama.cpp provider package. Other parts of
+ * the service import from this barrel rather than reaching into
+ * sub-files; the layout (`provider.ts`, `gguf-metadata.ts`,
+ * `models.ts`, …) is internal.
+ */
+export { LlamaCppProvider, NativeEngineCrashedError, ToolCallAccumulator, } from './provider.js';
+export { readSseEvents } from '../openai-compatible/sse.js';
+export { readGgufSummary, type GgufSummary } from './gguf-metadata.js';
+export { LlamaCppModelManager, type InstallEvent, type InstalledLlamaCppModel, type LlamaCppModelManagerOptions, } from './models.js';

package/dist/providers/llama-cpp/index.js

@@ -0,0 +1,13 @@
+/**
+ * Public surface of the llama.cpp provider package. Other parts of
+ * the service import from this barrel rather than reaching into
+ * sub-files; the layout (`provider.ts`, `gguf-metadata.ts`,
+ * `models.ts`, …) is internal.
+ */
+export { LlamaCppProvider, NativeEngineCrashedError, ToolCallAccumulator, } from './provider.js';
+// Re-exported for backwards compatibility — actual implementation lives
+// in providers/openai-compatible/sse.ts, shared with MLX and any future
+// OpenAI-compatible provider.
+export { readSseEvents } from '../openai-compatible/sse.js';
+export { readGgufSummary } from './gguf-metadata.js';
+export { LlamaCppModelManager, } from './models.js';

package/dist/providers/llama-cpp/kv-cache-type.d.ts

@@ -0,0 +1,118 @@
+/**
+ * KV-cache precision selection for llama-server (`--cache-type-k/v`).
+ *
+ * Split out from the supervisor launch builder so the family-aware
+ * default is unit-testable in isolation.
+ */
+export type LlamaCppKvCacheType = 'f16' | 'q8_0' | 'q4_0';
+/**
+ * The context we are willing to keep when trading window size for f16 KV
+ * (policy 2026-08-28: "favor f16 if memory allows, even at the sacrifice
+ * of slots or context above 64k"). Above this, an adaptive context is
+ * capped before falling back to a quantized cache; an explicit operator
+ * context is never capped.
+ */
+export declare const F16_PREFERRED_CTX_CAP_TOKENS = 65536;
+/**
+ * Is this model in the Gemma 3/4 family? The single source of truth for
+ * the Gemma-specific engine defaults (f16 KV cache, sliding-window
+ * full-cache). Prefers the GGUF architecture string; falls back to the
+ * model id for installs whose header didn't record an architecture.
+ */
+export declare function isGemmaModel(args: {
+    architecture?: string;
+    modelId?: string;
+}): boolean;
+/**
+ * The family FALLBACK `--cache-type-k/v` value — what a model gets when
+ * memory pressure forces the compact cache. This is no longer the whole
+ * default: {@link planLlamaCppKv} prefers f16 whenever the memory plan
+ * fits it, and only lands here when it does not.
+ *
+ * Gemma 3/4 are unusually sensitive to a quantized KV cache: large
+ * attention head dims (key/value_length = 512), a final logit softcap,
+ * and sliding-window attention mean an 8-bit KV cache corrupts the
+ * *stored prompt tokens* — the model reasons fine but recalls cached
+ * source/text as garbled multilingual tokens. Wild-caught
+ * (gemma4-12b): quoted source lines came back as Korean glyphs + emoji
+ * with `K/V = q8_0`. So the Gemma family falls back to f16; every other
+ * family falls back to q8_0 (~50% KV memory; quality-equivalent per the
+ * 2026-08-03 A/B). An explicit operator override always wins.
+ *
+ * The `modelId` fallback covers installs whose GGUF metadata didn't
+ * record an architecture string — a `gemma`-named model still gets f16.
+ */
+export declare function resolveLlamaCppKvCacheType(args: {
+    architecture?: string;
+    modelId?: string;
+    override?: LlamaCppKvCacheType;
+}): LlamaCppKvCacheType;
+/**
+ * Memory-aware KV precision + the sacrifices it is allowed to make.
+ *
+ * f16 KV is measurably faster than q8_0 on llama.cpp, and the gap grows
+ * with context where it hurts most: the 2026-08-28 ABBA
+ * (muse-glimmer-30b, reports/llama-kv-q8-longctx-20260828.md) measured a
+ * flat ~6% decode tax and a prefill tax growing −6% → −12% → −17% at
+ * 8k/40k/90k. The original q8_0 default was priced on a 2B model whose
+ * KV was too small for the tax to show. Policy (2026-08-28): **prefer
+ * f16 whenever memory allows, sacrificing engine slots first and then
+ * context above 64k tokens; fall back to the family default
+ * ({@link resolveLlamaCppKvCacheType}) only when f16 cannot fit even a
+ * single 64k slot.**
+ *
+ * Explicit operator choices always win and are never sacrificed: a
+ * `kvCacheType` override short-circuits entirely; an explicit slot count
+ * is honored (f16 only if it fits that many); an explicit context
+ * (env / per-model / machine-wide numeric, or `model-max` sizing) is
+ * never capped.
+ *
+ * GEMMA IS THE DELIBERATE EXCEPTION and keeps its long-standing shape:
+ * f16 for correctness (the garbling incident above), traded to q8_0
+ * only when that buys a second slot — because single-slot Gemma
+ * alternation re-prefills the other session's whole context (SWA models
+ * get nothing from llama-server's prompt cache; wild-caught ~41K tok ≈
+ * 79s per switch), a far larger cost than q8_0's prefill tax. The
+ * general "sacrifice slots for f16" ladder would reinstate exactly that
+ * pathology, so it does not apply to Gemma.
+ */
+export declare function planLlamaCppKv(args: {
+    architecture?: string;
+    modelId?: string;
+    override?: LlamaCppKvCacheType;
+    slotsConfigured: boolean;
+    /** The explicit slot count when `slotsConfigured` (honored verbatim). */
+    configuredSlots?: number;
+    /** Per-turn context the launch is currently asking for. */
+    requestedCtxTokens: number;
+    /** Admission floor — a cap below this is never proposed. */
+    minimumCtxTokens: number;
+    /** True when the context came from explicit operator intent. */
+    ctxConfigured: boolean;
+    /** Memory-ceiling slots for a KV type at a context (caller's budget math). */
+    ceilingFor: (kv: LlamaCppKvCacheType, ctxTokens: number) => number;
+    /**
+     * Does this exact plan clear the admission the launch will really run —
+     * weights + KV against the co-residency budget, at the full window?
+     *
+     * `ceilingFor` cannot answer this and must not be used as if it could. It
+     * is a slot COUNT floored at 1 (`localEngineSlotCeiling`), so it reports
+     * "1 slot" for a plan that does not fit at all, and it is priced against
+     * the FAST pool while admission is priced against the whole budget — for a
+     * RAM-spillover model those are different questions by construction.
+     * Conflating them made every f16 rung below pass on any single-slot
+     * machine, leaving the q8_0 rung unreachable and denying models that fit
+     * comfortably at q8_0.
+     */
+    fitsAt: (kv: LlamaCppKvCacheType, ctxTokens: number, slots: number) => boolean;
+    /** Upper bound on slots regardless of memory (policy default). */
+    maxSlots: number;
+}): {
+    kvCacheType: LlamaCppKvCacheType;
+    /** Gemma f16→q8_0 slot trade fired (pre-existing meaning, kept). */
+    upgraded: boolean;
+    /** Set when f16 was bought by capping context; assign to effectiveNumCtx. */
+    ctxCapTokens?: number;
+    /** One-line rationale for the launch log. */
+    reason: string;
+};

package/dist/providers/llama-cpp/kv-cache-type.js

@@ -0,0 +1,140 @@
+/**
+ * KV-cache precision selection for llama-server (`--cache-type-k/v`).
+ *
+ * Split out from the supervisor launch builder so the family-aware
+ * default is unit-testable in isolation.
+ */
+/**
+ * The context we are willing to keep when trading window size for f16 KV
+ * (policy 2026-08-28: "favor f16 if memory allows, even at the sacrifice
+ * of slots or context above 64k"). Above this, an adaptive context is
+ * capped before falling back to a quantized cache; an explicit operator
+ * context is never capped.
+ */
+export const F16_PREFERRED_CTX_CAP_TOKENS = 65_536;
+/**
+ * Is this model in the Gemma 3/4 family? The single source of truth for
+ * the Gemma-specific engine defaults (f16 KV cache, sliding-window
+ * full-cache). Prefers the GGUF architecture string; falls back to the
+ * model id for installs whose header didn't record an architecture.
+ */
+export function isGemmaModel(args) {
+    const arch = (args.architecture ?? '').toLowerCase();
+    return arch.startsWith('gemma') || /gemma/i.test(args.modelId ?? '');
+}
+/**
+ * The family FALLBACK `--cache-type-k/v` value — what a model gets when
+ * memory pressure forces the compact cache. This is no longer the whole
+ * default: {@link planLlamaCppKv} prefers f16 whenever the memory plan
+ * fits it, and only lands here when it does not.
+ *
+ * Gemma 3/4 are unusually sensitive to a quantized KV cache: large
+ * attention head dims (key/value_length = 512), a final logit softcap,
+ * and sliding-window attention mean an 8-bit KV cache corrupts the
+ * *stored prompt tokens* — the model reasons fine but recalls cached
+ * source/text as garbled multilingual tokens. Wild-caught
+ * (gemma4-12b): quoted source lines came back as Korean glyphs + emoji
+ * with `K/V = q8_0`. So the Gemma family falls back to f16; every other
+ * family falls back to q8_0 (~50% KV memory; quality-equivalent per the
+ * 2026-08-03 A/B). An explicit operator override always wins.
+ *
+ * The `modelId` fallback covers installs whose GGUF metadata didn't
+ * record an architecture string — a `gemma`-named model still gets f16.
+ */
+export function resolveLlamaCppKvCacheType(args) {
+    if (args.override)
+        return args.override;
+    return isGemmaModel(args) ? 'f16' : 'q8_0';
+}
+/**
+ * Memory-aware KV precision + the sacrifices it is allowed to make.
+ *
+ * f16 KV is measurably faster than q8_0 on llama.cpp, and the gap grows
+ * with context where it hurts most: the 2026-08-28 ABBA
+ * (muse-glimmer-30b, reports/llama-kv-q8-longctx-20260828.md) measured a
+ * flat ~6% decode tax and a prefill tax growing −6% → −12% → −17% at
+ * 8k/40k/90k. The original q8_0 default was priced on a 2B model whose
+ * KV was too small for the tax to show. Policy (2026-08-28): **prefer
+ * f16 whenever memory allows, sacrificing engine slots first and then
+ * context above 64k tokens; fall back to the family default
+ * ({@link resolveLlamaCppKvCacheType}) only when f16 cannot fit even a
+ * single 64k slot.**
+ *
+ * Explicit operator choices always win and are never sacrificed: a
+ * `kvCacheType` override short-circuits entirely; an explicit slot count
+ * is honored (f16 only if it fits that many); an explicit context
+ * (env / per-model / machine-wide numeric, or `model-max` sizing) is
+ * never capped.
+ *
+ * GEMMA IS THE DELIBERATE EXCEPTION and keeps its long-standing shape:
+ * f16 for correctness (the garbling incident above), traded to q8_0
+ * only when that buys a second slot — because single-slot Gemma
+ * alternation re-prefills the other session's whole context (SWA models
+ * get nothing from llama-server's prompt cache; wild-caught ~41K tok ≈
+ * 79s per switch), a far larger cost than q8_0's prefill tax. The
+ * general "sacrifice slots for f16" ladder would reinstate exactly that
+ * pathology, so it does not apply to Gemma.
+ */
+export function planLlamaCppKv(args) {
+    if (args.override) {
+        return { kvCacheType: args.override, upgraded: false, reason: 'operator kvCacheType override' };
+    }
+    const fallback = resolveLlamaCppKvCacheType(args);
+    if (isGemmaModel(args)) {
+        // Long-standing Gemma shape, verbatim (see the doc block above).
+        if (args.slotsConfigured || args.maxSlots < 2) {
+            return { kvCacheType: fallback, upgraded: false, reason: 'gemma f16 (correctness default)' };
+        }
+        const f16Slots = Math.min(args.maxSlots, args.ceilingFor('f16', args.requestedCtxTokens));
+        if (f16Slots >= 2) {
+            return { kvCacheType: fallback, upgraded: false, reason: 'gemma f16 fits multi-slot' };
+        }
+        const q8Slots = Math.min(args.maxSlots, args.ceilingFor('q8_0', args.requestedCtxTokens));
+        if (q8Slots >= 2) {
+            return {
+                kvCacheType: 'q8_0',
+                upgraded: true,
+                reason: 'gemma f16 traded for a second slot (SWA re-prefill pathology)',
+            };
+        }
+        return { kvCacheType: fallback, upgraded: false, reason: 'gemma f16 (trade buys nothing)' };
+    }
+    // Non-Gemma: f16-first ladder. Every rung clears BOTH gates — the fast-pool
+    // slot ceiling (KV lives on the card) and the admission the launch will
+    // really run. Gating on the ceiling alone is what made this ladder always
+    // answer f16: see the `fitsAt` contract above.
+    const desiredSlots = args.slotsConfigured ? (args.configuredSlots ?? 1) : args.maxSlots;
+    const f16AtRequested = args.ceilingFor('f16', args.requestedCtxTokens);
+    if (f16AtRequested >= desiredSlots && args.fitsAt('f16', args.requestedCtxTokens, desiredSlots)) {
+        return { kvCacheType: 'f16', upgraded: false, reason: 'f16 fits the full plan' };
+    }
+    if (!args.slotsConfigured &&
+        f16AtRequested >= 1 &&
+        args.fitsAt('f16', args.requestedCtxTokens, 1)) {
+        return {
+            kvCacheType: 'f16',
+            upgraded: false,
+            reason: `f16 preferred at ${f16AtRequested} slot(s) instead of q8_0 at ${desiredSlots}`,
+        };
+    }
+    const canCapCtx = !args.ctxConfigured &&
+        args.requestedCtxTokens > F16_PREFERRED_CTX_CAP_TOKENS &&
+        F16_PREFERRED_CTX_CAP_TOKENS >= args.minimumCtxTokens;
+    if (canCapCtx) {
+        const neededSlots = args.slotsConfigured ? desiredSlots : 1;
+        if (args.ceilingFor('f16', F16_PREFERRED_CTX_CAP_TOKENS) >= neededSlots &&
+            args.fitsAt('f16', F16_PREFERRED_CTX_CAP_TOKENS, neededSlots)) {
+            return {
+                kvCacheType: 'f16',
+                upgraded: false,
+                ctxCapTokens: F16_PREFERRED_CTX_CAP_TOKENS,
+                reason: `f16 preferred with context capped ${args.requestedCtxTokens} -> ${F16_PREFERRED_CTX_CAP_TOKENS}`,
+            };
+        }
+    }
+    return {
+        kvCacheType: fallback,
+        upgraded: false,
+        reason: 'f16 does not fit the memory plan — q8_0 for the memory saving',
+    };
+}

package/dist/providers/llama-cpp/log.d.ts

@@ -0,0 +1,92 @@
+import type { NativeEngineExitSnapshot } from '../native/supervisor.js';
+/**
+ * Suppress a native llama.cpp busy-loop without hiding real progress.
+ *
+ * Some engine builds can emit the same prompt-progress state tens of
+ * thousands of times per second (the uptime prefix changes, but task, token
+ * count, and progress do not). Logging every repaint can fill gigabytes in a
+ * few minutes. Distinct progress states still pass immediately; a stuck state
+ * is sampled periodically and gets an explicit suppression summary.
+ */
+export declare class LlamaProgressLogThrottle {
+    private readonly intervalMs;
+    private readonly now;
+    private lastSignature;
+    private lastEmittedAt;
+    private suppressed;
+    constructor(intervalMs?: number, now?: () => number);
+    accept(line: string): string[];
+    private takeSummary;
+}
+/**
+ * Rolling append-only log for llama-server stdout/stderr. Mirrors the
+ * per-day pattern in packages/app/src/supervisor/log-rotator.ts but
+ * lives inside the service package so the Electron → service
+ * dependency direction stays one-way.
+ *
+ * Files live at `<dir>/llama-server-YYYY-MM-DD.log`, roll to
+ * `-1.log` / `-2.log` when the active file crosses 10 MB, and ones
+ * older than 7 days are swept on open.
+ *
+ * Design note: we duplicate (rather than factor-out) the small
+ * rotator because the alternative is a shared package. Two files of
+ * trivial rotation logic is cheaper than a new package boundary for
+ * the size of this codebase.
+ */
+export declare class LlamaCppLogFile {
+    private readonly dir;
+    private readonly baseName;
+    private stream;
+    private currentPath;
+    private currentDay;
+    private rollIndex;
+    private bytesWritten;
+    private readonly maxBytes;
+    private readonly retentionDays;
+    private readonly readyPromise;
+    private pendingWrites;
+    constructor(dir: string, baseName?: string);
+    private init;
+    private today;
+    private openForToday;
+    private rollIfNeeded;
+    /**
+     * Fire-and-forget append. Serializes writes via an internal promise
+     * chain so rotation + ordering stay correct even under bursty
+     * stdout. The caller doesn't need to await individual lines; any
+     * subsequent `tail()` / `close()` awaits the drain.
+     */
+    write(line: string): void;
+    /**
+     * Persist an unexpected native exit before the provider releases the
+     * correlated request error. This synchronous, single-line write closes the
+     * race where an eval snapshots its isolated home immediately after a crash.
+     * The potentially user-derived stdout/stderr tail deliberately stays only in
+     * memory; the panic classifier contributes at most one redacted error line.
+     */
+    writeIncident(snapshot: NativeEngineExitSnapshot): void;
+    /** Wait for all in-flight writes to flush, then return. */
+    flush(): Promise<void>;
+    close(): Promise<void>;
+    private closeStream;
+    /**
+     * Read the last `bytes` chars of the current file. Used by the
+     * Settings → On-device → Engine log disclosure. Returns an empty
+     * string if the file doesn't exist yet (first launch, nothing
+     * logged).
+     */
+    tail(bytes: number): Promise<string>;
+    /** Absolute path of the current log file (for surfacing in UI). */
+    currentFile(): string | null;
+    private sweepOldFiles;
+}
+/**
+ * Read the newest retained engine log without constructing a writer. This is
+ * used by diagnostics after the provider has been evicted or the app was
+ * force-quit — exactly when an in-memory provider handle is unavailable but
+ * the last native stdout/stderr is most useful.
+ */
+export declare function tailLatestEngineLog(dir: string, baseName: string, bytes: number): Promise<{
+    path: string | null;
+    tail: string;
+}>;