@bendyline/gezel-inference 0.0.0-stage → 1.0.0

package/dist/providers/llama-cpp/devices.d.ts

@@ -0,0 +1,130 @@
+/**
+ * Device (VRAM) probe for the bundled `llama-server`.
+ *
+ * b9843's `--list-devices` prints one line per backend device with its
+ * total + free memory, e.g.
+ *
+ *   Available devices:
+ *     CUDA0: NVIDIA GeForce RTX 4070 (12282 MiB, 11987 MiB free)
+ *     MTL0: Apple M4 Max (53084 MiB, 53083 MiB free)
+ *     BLAS: Accelerate (0 MiB, 0 MiB free)
+ *
+ * That's the ground-truth VRAM number we need for the MoE offload
+ * planner — no native NVML/DXGI/Metal binding required. We spawn it
+ * once and cache the result next to the backend cache
+ * (`<home>/engines/llama-cpp/devices.json`), mirroring the
+ * `backend.json` pattern in `core/native/llama-backend.ts`.
+ */
+export interface LlamaDevice {
+    /** Backend device id, e.g. `CUDA0`, `Vulkan0`, `MTL0`, `BLAS`. */
+    id: string;
+    /** Human name, e.g. `NVIDIA GeForce RTX 4070`. */
+    name: string;
+    /** Total device memory in MiB. */
+    totalMiB: number;
+    /** Free device memory in MiB at probe time. */
+    freeMiB: number;
+}
+export interface LlamaDeviceProbe {
+    devices: LlamaDevice[];
+    /** ISO timestamp the probe ran (or was cached). */
+    probedAt: string;
+    /** True when returned from the on-disk cache rather than a fresh spawn. */
+    cached: boolean;
+}
+export interface NvidiaRuntimeDevice {
+    index: number;
+    name: string;
+    computeCapability: string;
+    driverVersion: string;
+}
+/**
+ * Parse `--list-devices` stdout into structured device records. Pure —
+ * the unit-tested half of the probe. Lines that don't match the
+ * `id: name (N MiB, M MiB free)` shape (the `Available devices:` header,
+ * blank lines, log noise) are ignored.
+ */
+export declare function parseLlamaDevices(stdout: string): LlamaDevice[];
+/** Parse the stable CSV shape requested from nvidia-smi for crash diagnostics. */
+export declare function parseNvidiaRuntimeDevices(stdout: string): NvidiaRuntimeDevice[];
+/**
+ * Best-effort NVIDIA driver/device facts. This is diagnostic only: failure or
+ * absence never changes backend selection and returns an empty list.
+ */
+export declare function probeNvidiaRuntimeDevices(): Promise<NvidiaRuntimeDevice[]>;
+/** Match llama-server's selected CUDA device to nvidia-smi diagnostics. */
+export declare function matchNvidiaRuntimeDevice(llamaDevice: LlamaDevice | null, nvidiaDevices: NvidiaRuntimeDevice[]): NvidiaRuntimeDevice | undefined;
+/**
+ * The largest single-device VRAM pool, in BYTES. Excludes the CPU/BLAS
+ * pseudo-devices (which report 0 MiB). Returns 0 when there's no GPU
+ * device — the planner reads that as "no offload target, leave it to
+ * the engine". We use the max (not the sum) because a model loads onto
+ * one device by default; multi-GPU splitting is a separate knob.
+ */
+export declare function maxGpuVramBytes(devices: LlamaDevice[]): number;
+/**
+ * The largest real GPU device (by total memory), or null when the list
+ * holds only CPU/BLAS pseudo-devices (which report 0 MiB). Same selection
+ * as `maxGpuVramBytes`, but returns the device so callers can name the card
+ * (e.g. "AMD Radeon RX 7900 XTX") in a tier rationale. Pure — unit-tested.
+ */
+export declare function pickBestGpuDevice(devices: LlamaDevice[]): LlamaDevice | null;
+/**
+ * Probe device memory by running `<binaryPath> --list-devices`, caching
+ * the result. Reads the cache when present (VRAM capacity is stable per
+ * machine); pass `force` to re-probe. Any failure (spawn error, no
+ * cache, unparseable output) resolves to an empty device list — the
+ * planner then simply does nothing and the engine's own `--fit`/`-ngl
+ * auto` takes over. `nowIso` is injectable for deterministic tests.
+ */
+export declare function probeLlamaDevices(opts: {
+    binaryPath: string;
+    home: string;
+    force?: boolean;
+    nowIso?: () => string;
+}): Promise<LlamaDeviceProbe>;
+/**
+ * Best-effort VRAM for a non-NVIDIA GPU (AMD Radeon, Intel Arc, any Vulkan
+ * device) via the bundled `llama-server --list-devices` — the one VRAM
+ * source in the tree that isn't `nvidia-smi`. Returns null when the engine
+ * binary isn't resolved yet (`GEZEL_LLAMA_SERVER_BIN` unset — e.g. a
+ * standalone gezeld before its first engine launch) or no GPU is reported.
+ * The underlying probe caches to `devices.json`, so this spawns at most
+ * once per host.
+ *
+ * `detectMemoryProfile` and `detectModelTier` call this AFTER `nvidia-smi`
+ * misses, so a Radeon box gets a real GPU budget + model tier instead of
+ * falling through to the nvidia-only "no GPU detected → system RAM" path.
+ */
+export type GpuVendor = 'amd' | 'nvidia' | 'intel';
+export type LlamaGpuMemoryKind = 'discrete' | 'integrated' | 'unknown';
+/**
+ * Infer the GPU vendor from the device NAME `--list-devices` reports (e.g.
+ * "AMD Radeon AI PRO R9700", "Intel(R) Arc(TM) A770 Graphics", "NVIDIA
+ * GeForce RTX 4070"). Deriving it from the same string that named the card
+ * — rather than a separate driver-DLL probe — guarantees the label can't
+ * disagree with the device whose VRAM we actually picked: an Intel Arc on
+ * the Vulkan backend reads "Intel", never "AMD". Unknown or software
+ * devices (llvmpipe, SwiftShader, …) return undefined, and the caller shows
+ * a generic "GPU" rather than guessing a brand. Pure — unit-tested.
+ */
+export declare function gpuVendorFromName(name: string): GpuVendor | undefined;
+/**
+ * Best-effort classification of the memory reported by llama.cpp's Vulkan
+ * device probe. Integrated adapters commonly expose a large slice of system
+ * RAM as their `totalMiB`; treating that number as dedicated VRAM counts the
+ * same physical memory twice.
+ *
+ * Names intentionally identify only strong, well-known families. Unknown
+ * hardware keeps the historical behavior rather than being silently demoted.
+ * Intel is the useful exception: consumer Intel Graphics/UHD/Iris adapters
+ * are integrated unless the name carries a discrete Arc board SKU (A770,
+ * B580, Pro A60, etc.).
+ */
+export declare function gpuMemoryKindFromName(name: string): LlamaGpuMemoryKind;
+export declare function detectLlamaGpuVram(homeOverride?: string): Promise<{
+    vramBytes: number;
+    name: string;
+    vendor?: GpuVendor;
+    memoryKind: LlamaGpuMemoryKind;
+} | null>;

package/dist/providers/llama-cpp/devices.js

@@ -0,0 +1,256 @@
+/**
+ * Device (VRAM) probe for the bundled `llama-server`.
+ *
+ * b9843's `--list-devices` prints one line per backend device with its
+ * total + free memory, e.g.
+ *
+ *   Available devices:
+ *     CUDA0: NVIDIA GeForce RTX 4070 (12282 MiB, 11987 MiB free)
+ *     MTL0: Apple M4 Max (53084 MiB, 53083 MiB free)
+ *     BLAS: Accelerate (0 MiB, 0 MiB free)
+ *
+ * That's the ground-truth VRAM number we need for the MoE offload
+ * planner — no native NVML/DXGI/Metal binding required. We spawn it
+ * once and cache the result next to the backend cache
+ * (`<home>/engines/llama-cpp/devices.json`), mirroring the
+ * `backend.json` pattern in `core/native/llama-backend.ts`.
+ */
+import { execFile } from 'node:child_process';
+import { existsSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs';
+import { delimiter, dirname, join } from 'node:path';
+import { promisify } from 'node:util';
+import { windowsHeadlessSpawnOptions } from '@bendyline/gezel/native';
+import { gezelHome } from '@bendyline/gezel/paths';
+const execFileAsync = promisify(execFile);
+const LINE_RE = /^\s*(\S+?):\s*(.+?)\s*\((\d+)\s*MiB,\s*(\d+)\s*MiB\s+free\)\s*$/;
+/**
+ * Parse `--list-devices` stdout into structured device records. Pure —
+ * the unit-tested half of the probe. Lines that don't match the
+ * `id: name (N MiB, M MiB free)` shape (the `Available devices:` header,
+ * blank lines, log noise) are ignored.
+ */
+export function parseLlamaDevices(stdout) {
+    const devices = [];
+    for (const line of stdout.split(/\r?\n/)) {
+        const m = LINE_RE.exec(line);
+        if (!m)
+            continue;
+        const [, id, name, total, free] = m;
+        devices.push({
+            id: id,
+            name: name.trim(),
+            totalMiB: Number(total),
+            freeMiB: Number(free),
+        });
+    }
+    return devices;
+}
+/** Parse the stable CSV shape requested from nvidia-smi for crash diagnostics. */
+export function parseNvidiaRuntimeDevices(stdout) {
+    const devices = [];
+    for (const rawLine of stdout.split(/\r?\n/)) {
+        const fields = rawLine.split(',').map((field) => field.trim());
+        if (fields.length !== 4)
+            continue;
+        const index = Number.parseInt(fields[0] ?? '', 10);
+        const name = fields[1] ?? '';
+        const computeCapability = fields[2] ?? '';
+        const driverVersion = fields[3] ?? '';
+        if (!Number.isFinite(index) || !name || !computeCapability || !driverVersion)
+            continue;
+        devices.push({ index, name, computeCapability, driverVersion });
+    }
+    return devices;
+}
+/**
+ * Best-effort NVIDIA driver/device facts. This is diagnostic only: failure or
+ * absence never changes backend selection and returns an empty list.
+ */
+export async function probeNvidiaRuntimeDevices() {
+    try {
+        const { stdout } = await execFileAsync('nvidia-smi', ['--query-gpu=index,name,compute_cap,driver_version', '--format=csv,noheader,nounits'], { timeout: 5_000, ...windowsHeadlessSpawnOptions() });
+        return parseNvidiaRuntimeDevices(stdout);
+    }
+    catch {
+        return [];
+    }
+}
+/** Match llama-server's selected CUDA device to nvidia-smi diagnostics. */
+export function matchNvidiaRuntimeDevice(llamaDevice, nvidiaDevices) {
+    const cudaIndex = llamaDevice?.id.match(/^CUDA(\d+)$/i)?.[1];
+    return ((cudaIndex !== undefined
+        ? nvidiaDevices.find((device) => device.index === Number(cudaIndex))
+        : undefined) ??
+        (llamaDevice ? nvidiaDevices.find((device) => device.name === llamaDevice.name) : undefined) ??
+        nvidiaDevices[0]);
+}
+/**
+ * The largest single-device VRAM pool, in BYTES. Excludes the CPU/BLAS
+ * pseudo-devices (which report 0 MiB). Returns 0 when there's no GPU
+ * device — the planner reads that as "no offload target, leave it to
+ * the engine". We use the max (not the sum) because a model loads onto
+ * one device by default; multi-GPU splitting is a separate knob.
+ */
+export function maxGpuVramBytes(devices) {
+    let maxMiB = 0;
+    for (const d of devices) {
+        if (d.totalMiB > maxMiB)
+            maxMiB = d.totalMiB;
+    }
+    return maxMiB * 1024 * 1024;
+}
+/**
+ * The largest real GPU device (by total memory), or null when the list
+ * holds only CPU/BLAS pseudo-devices (which report 0 MiB). Same selection
+ * as `maxGpuVramBytes`, but returns the device so callers can name the card
+ * (e.g. "AMD Radeon RX 7900 XTX") in a tier rationale. Pure — unit-tested.
+ */
+export function pickBestGpuDevice(devices) {
+    const dedicated = devices.filter((device) => device.totalMiB > 0 && gpuMemoryKindFromName(device.name) === 'discrete');
+    // An iGPU can advertise a larger shared allocation than a real card's
+    // dedicated VRAM. Prefer a known discrete device first so a 16-GiB shared
+    // Intel heap cannot hide a 12-GiB Radeon board.
+    const candidates = dedicated.length > 0 ? dedicated : devices;
+    let best = null;
+    for (const d of candidates) {
+        if (d.totalMiB > 0 && (!best || d.totalMiB > best.totalMiB))
+            best = d;
+    }
+    return best;
+}
+function cachePath(home) {
+    return join(home, 'engines', 'llama-cpp', 'devices.json');
+}
+/**
+ * Probe device memory by running `<binaryPath> --list-devices`, caching
+ * the result. Reads the cache when present (VRAM capacity is stable per
+ * machine); pass `force` to re-probe. Any failure (spawn error, no
+ * cache, unparseable output) resolves to an empty device list — the
+ * planner then simply does nothing and the engine's own `--fit`/`-ngl
+ * auto` takes over. `nowIso` is injectable for deterministic tests.
+ */
+export async function probeLlamaDevices(opts) {
+    const { binaryPath, home, force } = opts;
+    const now = opts.nowIso ?? (() => new Date().toISOString());
+    const path = cachePath(home);
+    if (!force && existsSync(path)) {
+        try {
+            const cached = JSON.parse(readFileSync(path, 'utf8'));
+            if (Array.isArray(cached.devices))
+                return { ...cached, cached: true };
+        }
+        catch {
+            // fall through to a fresh probe
+        }
+    }
+    let devices = [];
+    try {
+        // Prepend the binary's dir to PATH so its peer GGML/CUDA DLLs
+        // resolve on Windows (same reason the launcher does it).
+        const binDir = dirname(binaryPath);
+        const inheritedPath = process.env.PATH ?? '';
+        const { stdout } = await execFileAsync(binaryPath, ['--list-devices'], {
+            timeout: 20_000,
+            env: {
+                ...process.env,
+                PATH: inheritedPath ? `${binDir}${delimiter}${inheritedPath}` : binDir,
+            },
+            // This is an awaited probe, not an independent process. Hide its
+            // Windows console without detaching it from the daemon.
+            ...windowsHeadlessSpawnOptions(),
+        });
+        devices = parseLlamaDevices(stdout);
+    }
+    catch {
+        // Leave devices empty — caller treats "no devices" as "no offload target".
+        return { devices, probedAt: now(), cached: false };
+    }
+    const probe = { devices, probedAt: now(), cached: false };
+    try {
+        mkdirSync(dirname(path), { recursive: true });
+        writeFileSync(path, JSON.stringify(probe, null, 2), 'utf8');
+    }
+    catch {
+        // Caching is best-effort; a write failure doesn't invalidate the probe.
+    }
+    return probe;
+}
+/**
+ * Infer the GPU vendor from the device NAME `--list-devices` reports (e.g.
+ * "AMD Radeon AI PRO R9700", "Intel(R) Arc(TM) A770 Graphics", "NVIDIA
+ * GeForce RTX 4070"). Deriving it from the same string that named the card
+ * — rather than a separate driver-DLL probe — guarantees the label can't
+ * disagree with the device whose VRAM we actually picked: an Intel Arc on
+ * the Vulkan backend reads "Intel", never "AMD". Unknown or software
+ * devices (llvmpipe, SwiftShader, …) return undefined, and the caller shows
+ * a generic "GPU" rather than guessing a brand. Pure — unit-tested.
+ */
+export function gpuVendorFromName(name) {
+    const n = name.toLowerCase();
+    if (n.includes('nvidia') || n.includes('geforce') || n.includes('rtx'))
+        return 'nvidia';
+    if (n.includes('amd') || n.includes('radeon'))
+        return 'amd';
+    if (n.includes('intel') || n.includes('arc'))
+        return 'intel';
+    return undefined;
+}
+/**
+ * Best-effort classification of the memory reported by llama.cpp's Vulkan
+ * device probe. Integrated adapters commonly expose a large slice of system
+ * RAM as their `totalMiB`; treating that number as dedicated VRAM counts the
+ * same physical memory twice.
+ *
+ * Names intentionally identify only strong, well-known families. Unknown
+ * hardware keeps the historical behavior rather than being silently demoted.
+ * Intel is the useful exception: consumer Intel Graphics/UHD/Iris adapters
+ * are integrated unless the name carries a discrete Arc board SKU (A770,
+ * B580, Pro A60, etc.).
+ */
+export function gpuMemoryKindFromName(name) {
+    const n = name.toLowerCase().replaceAll('(tm)', ' ').replaceAll('(r)', ' ');
+    if (n.includes('qualcomm') || n.includes('adreno'))
+        return 'integrated';
+    if (n.includes('intel')) {
+        // Discrete Intel Arc boards include a letter+number SKU. Plain "Intel Arc
+        // Graphics" is the integrated Meteor/Lunar Lake branding.
+        if (/\barc\b.*\b(?:pro\s+)?[ab]\d{2,4}\b/.test(n))
+            return 'discrete';
+        return 'integrated';
+    }
+    if (n.includes('amd') || n.includes('radeon')) {
+        if (/\bradeon\s+(?:rx\s*\d|vii\b|pro\s+w\d|ai\s+pro\b)/.test(n) ||
+            /\b(?:instinct|firepro)\b/.test(n)) {
+            return 'discrete';
+        }
+        if (/\b(?:vega|radeon\s+graphics)\b/.test(n) || /\bradeon\s+\d{3,4}[ms]\b/.test(n)) {
+            return 'integrated';
+        }
+    }
+    if (n.includes('nvidia') || n.includes('geforce') || n.includes('quadro')) {
+        return 'discrete';
+    }
+    return 'unknown';
+}
+export async function detectLlamaGpuVram(homeOverride) {
+    const binaryPath = process.env.GEZEL_LLAMA_SERVER_BIN;
+    if (!binaryPath)
+        return null;
+    try {
+        const home = homeOverride ?? gezelHome();
+        const probe = await probeLlamaDevices({ binaryPath, home });
+        const best = pickBestGpuDevice(probe.devices);
+        if (!best)
+            return null;
+        const vendor = gpuVendorFromName(best.name);
+        return {
+            vramBytes: best.totalMiB * 1024 * 1024,
+            name: best.name,
+            memoryKind: gpuMemoryKindFromName(best.name),
+            ...(vendor ? { vendor } : {}),
+        };
+    }
+    catch {
+        return null;
+    }
+}

package/dist/providers/llama-cpp/disk-cache.d.ts

@@ -0,0 +1,8 @@
+export declare const DEFAULT_LLAMA_DISK_CACHE_MB = 8192;
+/** Save, restore and eviction must not race another replica's pruning. */
+export declare function withLlamaDiskCache<T>(root: string, work: () => Promise<T>): Promise<T>;
+export declare function pruneLlamaDiskCache(root: string, budgetBytes: number): Promise<{
+    bytes: number;
+    removedBytes: number;
+    removedFiles: number;
+}>;

package/dist/providers/llama-cpp/disk-cache.js

@@ -0,0 +1,112 @@
+/** Regenerable llama.cpp snapshots share one bounded cache across model replicas. */
+import { lstat, readdir, unlink } from 'node:fs/promises';
+import { join, resolve } from 'node:path';
+export const DEFAULT_LLAMA_DISK_CACHE_MB = 8192;
+const operations = new Map();
+/** Save, restore and eviction must not race another replica's pruning. */
+export async function withLlamaDiskCache(root, work) {
+    const absolute = resolve(root);
+    const key = process.platform === 'win32' ? absolute.toLowerCase() : absolute;
+    const previous = operations.get(key) ?? Promise.resolve();
+    let release;
+    const tail = new Promise((done) => {
+        release = done;
+    });
+    operations.set(key, tail);
+    await previous;
+    try {
+        return await work();
+    }
+    finally {
+        release();
+        if (operations.get(key) === tail)
+            operations.delete(key);
+    }
+}
+export async function pruneLlamaDiskCache(root, budgetBytes) {
+    if (!Number.isSafeInteger(budgetBytes) || budgetBytes < 0)
+        throw new Error('Invalid llama disk cache budget');
+    const result = { bytes: 0, removedBytes: 0, removedFiles: 0 };
+    if (budgetBytes === 0)
+        return result; // Explicit operator opt-out.
+    const files = [];
+    const directories = new Map();
+    async function visit(dir, depth, parents) {
+        // Never traverse directory symlinks/junctions, including a replaced root.
+        const info = await lstat(dir).catch((error) => {
+            if (error.code === 'ENOENT')
+                return null;
+            throw error;
+        });
+        if (!info?.isDirectory() || info.isSymbolicLink())
+            return;
+        directories.set(dir, { ino: info.ino, dev: info.dev });
+        const ancestors = [...parents, dir];
+        for (const entry of await readdir(dir, { withFileTypes: true })) {
+            if (entry.isSymbolicLink())
+                continue;
+            const path = join(dir, entry.name);
+            if (entry.isDirectory()) {
+                if ((depth === 0 && /^[a-f0-9]{24}$/.test(entry.name)) ||
+                    (depth === 1 && /^replica-[1-9]\d*$/.test(entry.name)))
+                    await visit(path, depth + 1, ancestors);
+                continue;
+            }
+            if (!entry.isFile() ||
+                !/^(?:sess-[A-Za-z0-9._-]+|prefix-(?:(?:gp|gezel)-)?[a-f0-9]{16})\.bin$/.test(entry.name))
+                continue;
+            const stat = await lstat(path).catch(() => null);
+            if (!stat?.isFile() || stat.isSymbolicLink())
+                continue;
+            files.push({
+                path,
+                size: stat.size,
+                mtimeMs: stat.mtimeMs,
+                ino: stat.ino,
+                dev: stat.dev,
+                parents: ancestors,
+            });
+            result.bytes += stat.size;
+        }
+    }
+    await visit(resolve(root), 0, []);
+    files.sort((a, b) => a.mtimeMs - b.mtimeMs || a.path.localeCompare(b.path));
+    for (const file of files) {
+        if (result.bytes <= budgetBytes)
+            break;
+        try {
+            let safe = true;
+            for (const parent of file.parents) {
+                const expected = directories.get(parent);
+                const current = await lstat(parent);
+                if (!current.isDirectory() ||
+                    current.isSymbolicLink() ||
+                    current.ino !== expected.ino ||
+                    current.dev !== expected.dev) {
+                    safe = false;
+                    break;
+                }
+            }
+            if (!safe)
+                continue;
+            // External writers are not in the process-local lock. Do not evict an
+            // entry whose identity or contents changed since the inventory scan.
+            const current = await lstat(file.path);
+            if (!current.isFile() ||
+                current.isSymbolicLink() ||
+                current.ino !== file.ino ||
+                current.dev !== file.dev ||
+                current.size !== file.size ||
+                current.mtimeMs !== file.mtimeMs)
+                continue;
+            await unlink(file.path);
+            result.bytes -= file.size;
+            result.removedBytes += file.size;
+            result.removedFiles++;
+        }
+        catch {
+            // A missing/locked cache is harmless; retry retention on the next turn.
+        }
+    }
+    return result;
+}