@bendyline/gezel-inference 0.0.0-stage → 1.0.0

package/dist/providers/gpu-arbiter.js

@@ -0,0 +1,301 @@
+/**
+ * GpuArbiter — thermal admission plus the user's explicit cross-slot swap
+ * preference. Automatic memory admission belongs to DeviceCapacityLedger,
+ * shared by all native supervisors and cooperating daemon processes.
+ *
+ * Policies:
+ *   - `coexist` — no slot eviction; supervisors admit residency by bytes.
+ *   - `swap` — `acquire(slot)` evicts the *other* slot first. The
+ *     evicted engine's lazy-restart kicks in on its next request,
+ *     so callers see a small one-time latency on the turn after a
+ *     swap, not on every turn.
+ *
+ * Engines register an evictor via {@link registerEvictor}; the
+ * llama-cpp + sd-cpp providers each call this once at construction
+ * with their supervisor's `stop()`. When no engine has registered
+ * for a slot (e.g. cloud LLM + local image gen), `acquire()` for
+ * that slot is a no-op — there's nothing to evict.
+ *
+ * Designed to be light: no internal state machine, no per-request
+ * tracking. The supervisor's own state machine is the source of
+ * truth for "is the engine running"; an unconditional `stop()`
+ * call when the supervisor is already stopped is a tested no-op.
+ */
+import { createLogger } from '@bendyline/gezel';
+const arbiterLog = createLogger('gpu-arbiter');
+/**
+ * Ceiling above which a held lease is assumed leaked rather than slow.
+ *
+ * Sized against the longest *legitimate* holder, not against typical use: the
+ * video provider's own request timeout is 1 h, so anything at or below that
+ * would break real renders mid-decode — precisely what the lease exists to
+ * prevent. Every legitimate holder is bounded by its own operation timeout, so
+ * a lease older than this one has no owner that will ever release it.
+ *
+ * This is a backstop, not the fix. A turn that throws now releases its lease in
+ * a `finally` (see the llama-cpp provider's tool loop). Before that, a leaked
+ * lease parked every later acquirer forever and the whole daemon stopped
+ * serving until gezeld restarted; this bounds that outage instead of letting it
+ * run indefinitely.
+ */
+const STALE_LEASE_BREAK_MS = 90 * 60_000;
+/** How often a blocked acquirer re-reports that it is still waiting. */
+const LEASE_WAIT_POLL_MS = 60_000;
+export class GpuArbiter {
+    policy;
+    /**
+     * Evictors per slot, keyed by owner. A slot can have multiple
+     * registered engines at once — the engine pool runs several
+     * llama-server replicas (one per resident model), each registering
+     * under its engine key — and an eviction of the slot must stop ALL
+     * of them, not just the most recently registered.
+     */
+    evictors = new Map();
+    log;
+    healthGate;
+    activeLease = null;
+    activeLeaseSince = null;
+    leaseWaiters = [];
+    staleLeaseBreakMs;
+    constructor(opts) {
+        this.policy = opts.policy;
+        this.log = opts.log ?? ((msg) => arbiterLog.info(msg));
+        if (opts.healthGate)
+            this.healthGate = opts.healthGate;
+        this.staleLeaseBreakMs = opts.staleLeaseBreakMs ?? STALE_LEASE_BREAK_MS;
+    }
+    /**
+     * Drop a lease the holder can no longer release, so blocked acquirers make
+     * progress. Returns true when a lease was actually broken.
+     */
+    breakStaleLease() {
+        if (this.activeLease === null || this.activeLeaseSince === null)
+            return false;
+        const heldMs = Date.now() - this.activeLeaseSince;
+        if (heldMs < this.staleLeaseBreakMs)
+            return false;
+        arbiterLog.error(`[gpu-arbiter] breaking stale ${this.activeLease} lease held ${Math.round(heldMs / 1000)}s (ceiling ${Math.round(this.staleLeaseBreakMs / 1000)}s) — its holder never released it. This is a leak: every GPU acquirer was blocked behind it.`);
+        this.activeLease = null;
+        this.activeLeaseSince = null;
+        this.wakeLeaseWaiters();
+        return true;
+    }
+    /**
+     * Register the function the arbiter should call to evict an engine
+     * from the GPU. Typically the supervisor's `stop()`. Re-registering
+     * the same `(slot, ownerId)` replaces the previous evictor — useful
+     * when a provider is rebuilt (e.g. user switched llama-cpp models
+     * in Settings) and the old supervisor is gone. Distinct owners on
+     * the same slot coexist (pool replicas); pass the engine key as
+     * `ownerId` so each replica can be unregistered independently.
+     */
+    registerEvictor(slot, evict, ownerId = 'default') {
+        let owners = this.evictors.get(slot);
+        if (!owners) {
+            owners = new Map();
+            this.evictors.set(slot, owners);
+        }
+        owners.set(ownerId, evict);
+    }
+    /**
+     * Forget an evictor. Called when a provider is shut down for good
+     * (e.g. user switched from local to cloud, or the pool evicted a
+     * replica). After this, `acquire()` for the *other* slot won't try
+     * to evict here.
+     */
+    unregisterEvictor(slot, ownerId = 'default') {
+        const owners = this.evictors.get(slot);
+        if (!owners)
+            return;
+        owners.delete(ownerId);
+        if (owners.size === 0)
+            this.evictors.delete(slot);
+    }
+    /**
+     * Called by a provider just before it intends to use the GPU. In
+     * `swap` mode, evicts every other registered slot first; in
+     * `coexist` mode, returns immediately. Failures during eviction are
+     * logged and swallowed — the requesting engine still gets to run,
+     * which is the user-visible promise.
+     */
+    async acquire(slot) {
+        if (this.getPolicy() === 'swap') {
+            await this.waitForForeignLease(slot);
+            if (this.getPolicy() === 'swap')
+                await this.evictOthers(slot);
+        }
+        await this.healthGate?.admit(`${slot} workload`);
+    }
+    /**
+     * Acquire the GPU and keep an in-flight operation protected until the
+     * returned release function is called. Long local image generations use
+     * this so a chat nudge cannot evict sd-server halfway through VAE decode.
+     */
+    async acquireLease(slot) {
+        if (this.getPolicy() === 'coexist') {
+            await this.healthGate?.admit(`${slot} workload`);
+            return () => { };
+        }
+        while (this.getPolicy() === 'swap' && this.activeLease !== null) {
+            if (this.breakStaleLease())
+                break;
+            this.log(`[gpu-arbiter] ${slot} requested GPU — waiting for active ${this.activeLease} job`);
+            await this.waitForLeaseRelease();
+        }
+        if (this.getPolicy() === 'coexist')
+            return () => { };
+        this.activeLease = slot;
+        this.activeLeaseSince = Date.now();
+        try {
+            await this.evictOthers(slot);
+            await this.healthGate?.admit(`${slot} workload`);
+        }
+        catch (error) {
+            if (this.activeLease === slot) {
+                this.activeLease = null;
+                this.activeLeaseSince = null;
+                this.wakeLeaseWaiters();
+            }
+            throw error;
+        }
+        let released = false;
+        return () => {
+            if (released)
+                return;
+            released = true;
+            if (this.activeLease === slot) {
+                this.activeLease = null;
+                this.activeLeaseSince = null;
+                this.wakeLeaseWaiters();
+            }
+        };
+    }
+    async waitForForeignLease(slot) {
+        while (this.getPolicy() === 'swap' && this.activeLease !== null && this.activeLease !== slot) {
+            if (this.breakStaleLease())
+                break;
+            this.log(`[gpu-arbiter] ${slot} requested GPU — waiting for active ${this.activeLease} job`);
+            await this.waitForLeaseRelease();
+        }
+    }
+    /**
+     * Resolves on the next release, or after {@link LEASE_WAIT_POLL_MS} so the
+     * caller re-evaluates. The timed wake is what makes the stale-lease breaker
+     * reachable: a leaked lease produces no release, so a wake-on-release-only
+     * wait would never re-check and the caller would park forever.
+     */
+    waitForLeaseRelease() {
+        return new Promise((resolve) => {
+            let settled = false;
+            const settleOnce = () => {
+                if (settled)
+                    return;
+                settled = true;
+                clearTimeout(timer);
+                resolve();
+            };
+            const timer = setTimeout(settleOnce, Math.min(LEASE_WAIT_POLL_MS, this.staleLeaseBreakMs));
+            // Never hold the process open on a lease wait.
+            timer.unref?.();
+            this.leaseWaiters.push(settleOnce);
+        });
+    }
+    wakeLeaseWaiters() {
+        const waiters = this.leaseWaiters;
+        this.leaseWaiters = [];
+        for (const resolve of waiters)
+            resolve();
+    }
+    async evictOthers(slot) {
+        const others = [...this.evictors.entries()].filter(([s]) => s !== slot);
+        if (others.length === 0)
+            return;
+        for (const [otherSlot, owners] of others) {
+            for (const [ownerId, evict] of owners) {
+                this.log(`[gpu-arbiter] ${slot} requested GPU — evicting ${otherSlot} (${ownerId})`);
+                try {
+                    await evict();
+                }
+                catch (err) {
+                    // The other engine may already be stopped, in which case its
+                    // supervisor's `stop()` is a fast no-op. A real error
+                    // shouldn't block the requesting engine from proceeding —
+                    // worst case the OS handles the VRAM contention.
+                    this.log(`[gpu-arbiter] evict ${otherSlot} (${ownerId}) threw (continuing): ${err instanceof Error ? err.message : String(err)}`);
+                }
+            }
+        }
+    }
+    getPolicy() {
+        return this.policy;
+    }
+    /**
+     * Hot-swap the policy. Used by the config PUT handler when the user
+     * flips the toggle in Settings. Doesn't preemptively load or evict
+     * anything — the new policy applies on the next `acquire()`.
+     */
+    setPolicy(policy) {
+        if (policy === this.policy)
+            return;
+        this.log(`[gpu-arbiter] policy ${this.policy} → ${policy}`);
+        this.policy = policy;
+        if (policy === 'coexist')
+            this.wakeLeaseWaiters();
+    }
+    /** Hot-swap native device-safety thresholds for the next admission. */
+    setDeviceSafetyPolicy(policy) {
+        this.healthGate?.setPolicy(policy);
+    }
+    /** Latest normalized machine-health snapshot for the authenticated UI. */
+    async getDeviceHealthStatus(maxAgeMs) {
+        return this.healthGate?.status(maxAgeMs);
+    }
+    /**
+     * Low-free-VRAM signal for idle engine supervisors. This is deliberately a
+     * release hint, not an admission denial: active inference always finishes,
+     * then an idle model gets one minute to be reused before caches flush and
+     * its process exits.
+     */
+    async getMemoryPressureStatus(maxAgeMs = 1_000) {
+        const status = await this.healthGate?.status(maxAgeMs);
+        const readings = (status?.readings ?? []).filter((reading) => typeof reading.memoryTotalMb === 'number' &&
+            Number.isFinite(reading.memoryTotalMb) &&
+            reading.memoryTotalMb > 0 &&
+            typeof reading.memoryUsedMb === 'number' &&
+            Number.isFinite(reading.memoryUsedMb) &&
+            reading.memoryUsedMb >= 0);
+        if (readings.length === 0)
+            return { pressured: false };
+        const totalBytes = readings.reduce((sum, reading) => sum + (reading.memoryTotalMb ?? 0), 0) * 1024 ** 2;
+        const usedBytes = readings.reduce((sum, reading) => sum + (reading.memoryUsedMb ?? 0), 0) * 1024 ** 2;
+        const freeBytes = Math.max(0, totalBytes - usedBytes);
+        const thresholdBytes = Math.max(1024 ** 3, totalBytes * 0.08);
+        const pressured = freeBytes <= thresholdBytes;
+        const gib = (bytes) => (bytes / 1024 ** 3).toFixed(1);
+        return {
+            pressured,
+            freeBytes,
+            totalBytes,
+            ...(pressured ? { detail: `${gib(freeBytes)} GB free of ${gib(totalBytes)} GB VRAM` } : {}),
+        };
+    }
+}
+/**
+ * Automatic co-residency is now admitted by the device memory ledger at actual
+ * engine startup. The slot arbiter retains explicit `swap` as an operator
+ * preference; platform/RAM heuristics must not serialize models that fit.
+ */
+export function detectGpuPolicy(_opts) {
+    return 'coexist';
+}
+/**
+ * Resolve the user-facing setting (which may be `'auto'`) into a
+ * concrete policy by consulting {@link detectGpuPolicy} for the
+ * `'auto'` case. Centralized so the service entrypoint and the
+ * config-change path agree on the same mapping.
+ */
+export function resolveGpuPolicy(setting) {
+    if (setting === 'coexist' || setting === 'swap')
+        return setting;
+    return detectGpuPolicy();
+}

package/dist/providers/image-persister-contract.d.ts

@@ -0,0 +1,15 @@
+export interface ToolImagePersister {
+    /**
+     * Write all images from a single tool result to the project artifacts
+     * tree. The persister owns an internal monotonic counter so multiple
+     * calls of the same tool inside one session don't collide on filename.
+     * Returns the saved relative paths, parallel to the input array.
+     */
+    persist(images: ReadonlyArray<{
+        base64: string;
+        mimeType: string;
+    }>): Promise<Array<{
+        path: string;
+        mimeType: string;
+    }>>;
+}

package/dist/providers/image-persister-contract.js

@@ -0,0 +1 @@
+export {};

package/dist/providers/llama-cpp/build-metadata.d.ts

@@ -0,0 +1,13 @@
+export interface LlamaCppBuildMetadata {
+    schemaVersion: 1;
+    engine: 'llama-cpp';
+    revision: string;
+    platform: string;
+    backend: string;
+    cudaArchitectures?: string[];
+    cudaToolkit?: string;
+}
+/** Read build facts emitted beside gezel-llama-server; legacy bundles return null. */
+export declare function readLlamaCppBuildMetadata(binaryPath: string): Promise<LlamaCppBuildMetadata | null>;
+/** Last occurrence wins, matching llama-server's command-line parser. */
+export declare function lastArgValue(args: string[], flag: string): string | undefined;

package/dist/providers/llama-cpp/build-metadata.js

@@ -0,0 +1,39 @@
+import { readFile } from 'node:fs/promises';
+import { dirname, join } from 'node:path';
+/** Read build facts emitted beside gezel-llama-server; legacy bundles return null. */
+export async function readLlamaCppBuildMetadata(binaryPath) {
+    try {
+        const parsed = JSON.parse(await readFile(join(dirname(binaryPath), 'gezel-llama-build.json'), 'utf8'));
+        if (parsed.schemaVersion !== 1 ||
+            parsed.engine !== 'llama-cpp' ||
+            typeof parsed.revision !== 'string' ||
+            typeof parsed.platform !== 'string' ||
+            typeof parsed.backend !== 'string') {
+            return null;
+        }
+        return {
+            schemaVersion: 1,
+            engine: 'llama-cpp',
+            revision: parsed.revision,
+            platform: parsed.platform,
+            backend: parsed.backend,
+            ...(Array.isArray(parsed.cudaArchitectures) &&
+                parsed.cudaArchitectures.every((value) => typeof value === 'string')
+                ? { cudaArchitectures: parsed.cudaArchitectures }
+                : {}),
+            ...(typeof parsed.cudaToolkit === 'string' ? { cudaToolkit: parsed.cudaToolkit } : {}),
+        };
+    }
+    catch {
+        return null;
+    }
+}
+/** Last occurrence wins, matching llama-server's command-line parser. */
+export function lastArgValue(args, flag) {
+    let value;
+    for (let index = 0; index < args.length; index++) {
+        if (args[index] === flag && index + 1 < args.length)
+            value = args[index + 1];
+    }
+    return value;
+}

package/dist/providers/llama-cpp/build-provider.d.ts

@@ -0,0 +1,66 @@
+import { type GezelConfig } from '@bendyline/gezel';
+import type { CatalogService } from '@bendyline/gezel-catalog';
+import { LlamaCppProvider } from './provider.js';
+/** What the engine-resolver hook reports for a turn that found no engine binary. */
+export interface EngineStatus {
+    detail: string;
+    /** The download failed: a fault, not a wait. */
+    failed?: boolean;
+}
+/**
+ * Lazy on-device engine-resolver hook for {@link buildLlamaCppProvider}.
+ * Kicks (or attaches to) a background download of llama-server for the
+ * detected/overridden GPU backend and returns an actionable status to
+ * surface this turn. Returns `undefined` when auto-download is disabled —
+ * the caller then falls through to the plain "install the engine" error.
+ */
+export declare function ensureLlamaEngineStatus(registry: import('../../engines/registry.js').EngineBinaryRegistry, config: GezelConfig): EngineStatus | undefined;
+/**
+ * Build the llama.cpp provider for either an externally managed server or a
+ * supervised local engine. The supervised path resolves the installed model,
+ * plans context/KV/offload against available capacity, and owns the engine's
+ * lifecycle through {@link NativeEngineSupervisor}.
+ */
+export declare function buildLlamaCppProvider(opts: {
+    config: GezelConfig;
+    affinity: boolean | undefined;
+    home: string;
+    llamaCppModels?: import('./index.js').LlamaCppModelManager;
+    arbiter?: import('../gpu-arbiter.js').GpuArbiter;
+    /**
+     * Optional catalog service — used to read per-model launch-time
+     * tuning that can't be expressed per-request, like
+     * `tuning.reasoning.thinkingBudget` → `--reasoning-budget` CLI flag.
+     * When absent (CLI/external boots), the server starts with its
+     * defaults.
+     */
+    catalog?: CatalogService;
+    /**
+     * Pool-driven multi-engine routing. When set, the build resolves
+     * `modelId` from the catalog (ignoring `config.defaultModel` /
+     * `config.llamaCppModelPath` / `GEZEL_LLAMA_CPP_MODEL`), and uses
+     * `replicaIdx` to isolate the slot-save-path so concurrent
+     * replicas don't collide. Unset = today's singleton behavior.
+     */
+    modelOverride?: {
+        modelId: string;
+        replicaIdx: number;
+    };
+    /**
+     * Lazy engine-resolver hook. When the on-device binary is missing (and
+     * no external URL is configured), this is consulted before erroring: it
+     * may kick a background download and return an actionable status to
+     * surface this turn. Absent → the plain "install the engine" error.
+     */
+    ensureEngine?: () => EngineStatus | undefined;
+    /**
+     * Capacity broker (pool path only). When present, the supervised slot
+     * ceiling subtracts already-committed co-resident model reservations
+     * from the budget so a second resident model doesn't over-slot into
+     * memory the first one already holds. The pool reserves THIS model only
+     * after this builder returns, so its committed total is purely the other
+     * residents. Absent on the singleton path → the ceiling uses the full
+     * auto-detected budget.
+     */
+    broker?: import('../native/capacity-broker.js').CapacityBroker;
+}): Promise<LlamaCppProvider>;