@bendyline/gezel-inference 0.0.0-stage → 1.0.0

package/dist/providers/llama-cpp/provider.d.ts

@@ -0,0 +1,493 @@
+/**
+ * LlamaCppProvider — talks to a local `llama-server` binary over HTTP.
+ * The server speaks OpenAI-compatible `/v1/chat/completions` with SSE
+ * streaming and standard tool-calling when launched with `--jinja`.
+ *
+ * Session model is stateless (like Ollama): we keep the full transcript
+ * in memory and resend it each turn. `ProviderSessionState` is empty —
+ * the session record on disk is the source of truth.
+ *
+ * Lifecycle of the llama-server process itself is managed by a
+ * {@link NativeEngineSupervisor}: lazy-start on first turn, idle-stop,
+ * health-watch, restart budget. The provider asks `ensureRunning()`
+ * before each turn and `markUsed()` afterwards. A dev / LAN mode
+ * without a supervisor (just a `baseUrl`) is also supported for when
+ * the user runs their own llama-server.
+ *
+ * Each provider INSTANCE serves a single model (one llama-server
+ * process); multi-model is handled a layer up by the engine pool
+ * (`providers/native/provider-pool.ts`), which keys instances by
+ * `{provider, modelId, replicaIdx}` and spools/evicts them under a
+ * memory budget. An evicted instance is `disposed` — it must never
+ * lazily respawn its process (see {@link shutdown}).
+ */
+import { type DisableThinkingRequestShape, LlamaCppSession, type ReasoningEffortRequestShape, type ToolGrammarFallback } from '@bendyline/gezel/local-loop';
+import type { GpuArbiter } from '../gpu-arbiter.js';
+import type { NativeEngineLifecycleSnapshot, NativeEngineSupervisor } from '../native/supervisor.js';
+import { ProviderQueue } from '../queue.js';
+import type { BatchCapability, EngineLaunchSnapshot, LLMProvider, LLMSession, ModelInfo, SessionOpts } from '../types.js';
+import type { LlamaCppLogFile } from './log.js';
+import { type StartupPhase } from './stdout-parser.js';
+export * from '@bendyline/gezel/local-loop';
+export { isFileRepairTurn as isScenarioFileRepairTurn } from '@bendyline/gezel/local-loop';
+export declare class LlamaCppProvider implements LLMProvider {
+    readonly name: "llama-cpp";
+    readonly queue: ProviderQueue;
+    readonly supportsExternalTools = true;
+    readonly supportsPriorMessages = true;
+    /**
+     * What this engine build's aggregate tool-grammar converter has already
+     * proven it cannot take, remembered across sessions.
+     *
+     * The rejection is deterministic — a fixed build applied to a tool
+     * payload of a given size either compiles or doesn't — but the recovery
+     * ladder in `runSend` used to start from `'none'` on every turn, so a
+     * roster that fails re-paid two rejected round-trips *per turn*, forever.
+     * On one install that was 16 rejections in a single afternoon, every one
+     * of them re-deriving the same answer.
+     *
+     * Keyed on tool count rather than remembered flat, because the limit the
+     * failures actually hit is grammar SIZE ("number of repetitions exceeds
+     * sane defaults"): a 48-tool roster blowing up says nothing about a
+     * 5-tool one, and degrading the small roster's schemas would cost real
+     * tool-argument fidelity for no reason. A request at or above the
+     * smallest count known to fail starts at the tier that recovered it.
+     */
+    private toolGrammarFloor;
+    private readonly supervisor?;
+    /** Explicit base URL when no supervisor is managing the process. */
+    private readonly externalBaseUrl?;
+    private readonly defaultModel;
+    /**
+     * Catalog model id for the on-disk weights llama-server is serving —
+     * `qwen3.5-9b-llama-cpp`, `gemma4-e4b-llama-cpp`, etc. Distinct from
+     * `defaultModel`, which holds the placeholder string llama-server
+     * accepts in the `model` request field (the engine ignores it when
+     * only one model is loaded). Surfaced via {@link getEffectiveModelId}
+     * so the chat manager can resolve a tier for sessions that don't
+     * have an explicit model selection (`record.model` undefined AND
+     * `config.defaultModel.llama-cpp` undefined). Without this, those
+     * sessions classify as `tier:tiny` because the tier resolver has
+     * nothing to parse — and the debug bundle can't print the model
+     * name either, so an engineer reading it sees `tier: tiny` against
+     * a 9B Qwen and is left guessing what model is actually loaded.
+     *
+     * Empty when the provider was constructed in external-baseUrl mode
+     * (we don't know what the user's llama-server is serving) or before
+     * the catalog has resolved a default model. Mirrors the same field
+     * on `MlxProvider`.
+     */
+    private readonly catalogModelId?;
+    /**
+     * Catalog manager — when set, `listModels()` enumerates every model
+     * the user has installed (so `/v1/models` + Settings pickers see the
+     * full set the engine pool can serve), not just the resident one.
+     * Optional because external-baseUrl mode has no local catalog.
+     * Mirrors the same field on `MlxProvider`.
+     */
+    private readonly modelManager?;
+    /** Owner key of our `'llm'` evictor registration; see constructor opts. */
+    private evictorOwnerId?;
+    /**
+     * Set once {@link shutdown} runs. A disposed provider must never
+     * lazily respawn its llama-server: the engine pool evicts replicas
+     * by calling `shutdown()`, and any session still holding the old
+     * instance would otherwise resurrect the process via
+     * `ensureRunning()` — a zombie engine outside the pool's capacity
+     * accounting.
+     */
+    private disposed;
+    private readonly numCtx;
+    private readonly plannedReservation?;
+    private readonly plannedGpu?;
+    /**
+     * When true, append `stream_options:{include_usage:true}` to chat requests so
+     * the engine emits a final usage chunk. Off for llama-server (it surfaces its
+     * own custom timings/usage the session already parses); ds4-server emits
+     * usage ONLY when asked, so {@link buildDs4Provider} opts in — without this,
+     * ds4 turns record zero tokens and tok/s telemetry is null.
+     */
+    private readonly includeUsageInStream;
+    /**
+     * When true, assistant turns committed to the session transcript carry
+     * their verbatim SSE `reasoning_content` so later requests replay it.
+     * Off for llama-server (its templates drop reasoning on re-render, and
+     * echoing it back would *change* the rendered prompt on templates that
+     * honor the field). On for ds4-server, whose DeepSeek-V4 render replays
+     * `<think>{reasoning_content}</think>` for every assistant turn in tool
+     * context and keeps exact per-call DSML by tool-call id: with the
+     * reasoning echoed, the re-rendered history is byte-identical to what
+     * was generated, the engine's live-KV prefix match survives each tool
+     * iteration, and a continuation prefills only the new tool results.
+     * Without it, every iteration logs `live kv cache miss … reason=
+     * token-mismatch`, falls back to the system-prefix disk snapshot, and
+     * re-prefills the whole conversation tail (minutes per turn at DS4's
+     * SSD-streamed prefill speeds).
+     */
+    private readonly replayReasoningContent;
+    /**
+     * Whether this engine was launched with `--mmproj`, i.e. whether the server
+     * can actually decode an image. Gates the typed-content message shape:
+     * sending image parts to a text-only server just inflates the prompt with
+     * base64 the model can't interpret.
+     *
+     * Sourced from the installed model's projector path plus the user's
+     * per-model opt-in, resolved in `buildLlamaCppProvider`.
+     */
+    private readonly visionEnabled;
+    /** Read by the machine engine's `/v1/remote/infer`, which refuses image history otherwise. */
+    get supportsImageInput(): boolean;
+    private readonly disableThinkingRequestShape;
+    /** Request-scoped effort shape for compatible wrappers such as ds4-server. */
+    private readonly reasoningEffortRequestShape;
+    /** Engine batch width; see the `batchMaxConcurrency` constructor opt. */
+    private readonly batchMaxConcurrency;
+    /**
+     * Number of real llama-server request slots (`--parallel`). The provider
+     * queue can intentionally be wider than this by one reserved background
+     * lane so an in-turn one-shot can dispatch without deadlocking behind the
+     * foreground turn that awaits it. This gate is the separate physical
+     * boundary: cache slot save/restore plus the streamed engine request must
+     * never exceed the slots the native server actually owns.
+     */
+    private readonly engineRequestWidth;
+    private engineRequestsActive;
+    private readonly engineRequestWaiters;
+    /**
+     * Mid-stream silence cap (ms). After this many ms with no SSE chunk,
+     * the in-flight `/v1/chat/completions` request is aborted with an
+     * idle-stall error so the runtime publishes `done`, the engine pill
+     * clears, and the user can retry. Without this cap a hung llama-cpp
+     * turn (model stalled, llama-server thread wedged, GPU eviction mid-
+     * generation) would hold the runtime's `await sendAndWait` forever.
+     * Mirrors Ollama's same-named knob (default 5 min); see Ollama's
+     * `streamingIdleMs` for the design notes.
+     */
+    private readonly streamingIdleMs;
+    /**
+     * Pre-first-byte cap (ms). Covers cold model load, prompt prefill,
+     * and the first-token wait. Separate from `streamingIdleMs` so a
+     * legitimately slow first-chunk on a 30B-class model doesn't trip
+     * the streaming-idle watchdog set for active-generation silence.
+     */
+    private readonly preFirstByteIdleMs;
+    /**
+     * Tight post-reasoning silent-stall cap (ms). Armed when the engine
+     * signals it finished thinking; fires if no SSE delta arrives within
+     * the window AND a `/slots` liveness probe shows the KV cache is not
+     * growing. Default 30s; overridable for tests. See the watchdog in
+     * `sendAndWaitInner`.
+     */
+    private readonly postReasoningWatchdogMs;
+    /**
+     * Constrained mutation turns should produce a tool signal quickly after
+     * the model has already inspected inputs. DS4 can otherwise burn the full
+     * turn in hidden/tool thinking with no SSE signal; this timer converts that
+     * into the existing no-mutation corrective retry.
+     */
+    private readonly constrainedToolNoSignalMs;
+    private readonly fetchImpl;
+    private readonly logFile?;
+    /**
+     * Cross-engine GPU coordinator. When set (and we're running a
+     * supervised local llama-server, not an external base URL),
+     * `resolveBaseUrl` acquires the `'llm'` slot before each request,
+     * which evicts the image engine in `swap` mode. The provider also
+     * registers its supervisor's `stop()` as the `'llm'` evictor so
+     * image-gen requests can swap us out symmetrically.
+     */
+    private readonly arbiter?;
+    /**
+     * Sessions currently inside `sendAndWaitInner` — used to fan-out
+     * supervisor-side phase events (model load progress parsed from
+     * llama-server stdout) to every caller waiting on the same engine
+     * startup. Sessions add themselves on turn entry and remove in the
+     * finally block. Empty when the engine is idle.
+     */
+    private readonly activeSessions;
+    /**
+     * Last phase observed from the stdout classifier, kept so newly-
+     * starting sessions can pick up the current state instead of waiting
+     * for the next log line. Cleared on `ready`.
+     */
+    private lastStartupPhase;
+    /**
+     * Running total of bytes allocated across every `load_tensors:` /
+     * `llama_kv_cache_init:` line the stdout classifier has seen since
+     * the current engine start. Reset on `ready` so a later restart's
+     * allocation doesn't double-count. Published once per engine start
+     * as an `engine_stats` event fanning out to active sessions.
+     */
+    private accumulatedRamBytes;
+    /**
+     * Most recent ram total published — used so new sessions joining
+     * mid-lifecycle get the figure replayed (same pattern as
+     * `lastStartupPhase`).
+     */
+    private lastEngineStats;
+    /**
+     * Stdout line → phase classifier used by {@link onStdoutLine}.
+     * Defaults to the llama-server classifier; engine wrappers that
+     * reuse this turn loop against a different binary (ds4-server via
+     * {@link Ds4Provider}) inject their own so phase events reflect
+     * that engine's actual log wording instead of silently matching
+     * nothing.
+     */
+    private readonly classifyLine;
+    constructor(opts: {
+        /**
+         * Supervisor that owns the llama-server child. When set, each
+         * turn calls `supervisor.ensureRunning()` to get the live base
+         * URL and `markUsed()` after. Mutually exclusive with
+         * `baseUrl`.
+         */
+        supervisor?: NativeEngineSupervisor;
+        /**
+         * Fixed base URL of an already-running llama-server. Used in
+         * dev iteration and LAN-shared setups. Mutually exclusive with
+         * `supervisor`.
+         */
+        baseUrl?: string;
+        defaultModel?: string;
+        /** See {@link LlamaCppProvider.catalogModelId} for the contract. */
+        catalogModelId?: string;
+        /** See {@link LlamaCppProvider.modelManager} for the contract. */
+        modelManager?: import('./models.js').LlamaCppModelManager;
+        /**
+         * Context window (tokens) llama-server was booted with. The
+         * supervisor-owned launch path is responsible for matching this
+         * to the `--ctx-size` it passed; the external-baseUrl path
+         * trusts the caller. Surfaced on every session so ChatManager
+         * can pressure-check. Default 16384 when omitted.
+         */
+        numCtx?: number;
+        /**
+         * Broker-ledger reservation for this replica: resident weights plus
+         * the KV the engine will allocate at the granted window and cache
+         * mode, computed by the launch admission pass. The pool builder
+         * prefers this over the catalog/weights-multiplier fallback so
+         * co-residency admission can see KV (M1).
+         */
+        plannedReservationBytes?: number;
+        /** Graphics memory the MoE offload plan puts on the card. See `LLMProvider.plannedGpuBytes`. */
+        plannedGpuBytes?: number;
+        /** See {@link LlamaCppProvider.includeUsageInStream}. Default false. */
+        includeUsageInStream?: boolean;
+        /** See {@link LlamaCppProvider.replayReasoningContent}. Default false. */
+        replayReasoningContent?: boolean;
+        /** See {@link LlamaCppProvider.visionEnabled}. Default false. */
+        visionEnabled?: boolean;
+        /**
+         * Request fields used when constrained local turns need thinking disabled.
+         * llama-server honors chat_template_kwargs.enable_thinking; ds4-server
+         * needs DeepSeek-compatible top-level thinking fields.
+         */
+        disableThinkingRequestShape?: DisableThinkingRequestShape;
+        /**
+         * Forward `SessionOpts.reasoningEffort` on each request. DS4 accepts both
+         * the top-level OpenAI field and Qwen-style chat-template kwargs; ordinary
+         * llama-server providers retain their existing catalog-only behavior.
+         */
+        reasoningEffortRequestShape?: ReasoningEffortRequestShape;
+        concurrency?: number;
+        /**
+         * Keep the queue's normal spare background lane. Defaults to true because
+         * llama-server can accept a queued second request while a foreground turn
+         * releases/reacquires its slot between tool iterations. Specialized
+         * single-tenant wrappers such as ds4 disable this so expensive SSD-streamed
+         * inference is strictly serial and cannot double its I/O pressure.
+         */
+        reserveBackgroundSlot?: boolean;
+        /**
+         * Engine batch width — how many sequences the server can generate at
+         * once. The supervised path passes its `--parallel` slot count, so this
+         * equals {@link concurrency} there; chats may fill all of it, with one
+         * slot withheld from background work so a live turn never waits behind a
+         * background cohort. Default 1, which is what the external-baseUrl path
+         * takes: we don't control that server's `--parallel` and it may be
+         * single-slot. See {@link LLMProvider.batch}.
+         */
+        batchMaxConcurrency?: number;
+        affinity?: boolean;
+        /**
+         * Override the default mid-stream silence cap (ms). Set lower for
+         * tests; raise via `config.llamaCppStreamingIdleSec` for users with
+         * legitimately slow models that need more headroom mid-generation.
+         */
+        streamingIdleMs?: number;
+        /** Override the default pre-first-byte cap (ms). */
+        preFirstByteIdleMs?: number;
+        /** Override the default post-reasoning silent-stall cap (ms). Set lower for tests. */
+        postReasoningWatchdogMs?: number;
+        /** Override the constrained mutation no-tool-signal cap (ms). Set lower for tests. */
+        constrainedToolNoSignalMs?: number;
+        fetchImpl?: typeof fetch;
+        /**
+         * Rolling log-file sink capturing raw llama-server stdout/stderr.
+         * Surfaced via {@link getLogFile} so the Settings → On-device log
+         * viewer can tail it. Only set on the supervised path.
+         */
+        logFile?: LlamaCppLogFile;
+        /**
+         * Cross-engine GPU coordinator. Optional: cloud-LLM installs and
+         * tests don't need it. When supplied alongside a supervisor, the
+         * provider registers its `stop()` so image generation can evict
+         * llama-server in `swap` mode, and acquires the `'llm'` slot
+         * before each request so a still-running image engine gets
+         * evicted first.
+         */
+        arbiter?: GpuArbiter;
+        /**
+         * Owner key for the arbiter's `'llm'`-slot evictor registration.
+         * The engine pool passes the engine key (`llama-cpp/<model>#<n>`)
+         * so concurrently-resident replicas don't clobber each other's
+         * registration; singleton boots omit it and use the arbiter's
+         * `'default'` owner. Unregistered automatically in
+         * {@link shutdown}.
+         */
+        evictorOwnerId?: string;
+        /**
+         * Directory passed to llama-server's `--slot-save-path`. Non-null
+         * only on the supervised path (we know what we passed); the cache
+         * adapter reads it via {@link getSlotSavePath} to wire its disk-
+         * persistence flow.
+         */
+        slotSavePath?: string;
+        /** See {@link LlamaCppProvider.classifyLine}. Default: `classifyStartupLine`. */
+        classifyLine?: (line: string) => StartupPhase | null;
+    });
+    /**
+     * Record that a tool payload of `toolCount` was rejected by the engine's
+     * grammar converter and needed `tier` to get through. See
+     * {@link toolGrammarFloor}.
+     *
+     * Both fields widen monotonically: the smallest failing count (so the
+     * floor covers every payload at least that large) and the most permissive
+     * tier we have had to reach (so a later turn never starts below a rung
+     * already known to be insufficient).
+     */
+    noteToolGrammarFloor(toolCount: number, tier: ToolGrammarFallback): void;
+    /** Tier a request carrying `toolCount` tools should START at. */
+    toolGrammarFloorFor(toolCount: number): ToolGrammarFallback;
+    /**
+     * Set once the engine has rejected a forced tool call for this model, so
+     * every later constrained turn — in this session and every sibling on the
+     * same engine — advertises tools without `tool_choice: "required"` rather
+     * than replaying a rejection the payload cannot fix.
+     *
+     * Engine-scoped and monotonic for the same reason as
+     * {@link toolGrammarFloor}: the incompatibility is a property of the model
+     * plus the server build, not of one turn's tools, so re-probing per turn
+     * would burn a request each time a rescue fires — and rescues fire exactly
+     * when the turn is already in trouble.
+     */
+    private forcedToolChoiceUnsupported;
+    /** Record that this engine rejected `tool_choice: "required"`. */
+    noteForcedToolChoiceUnsupported(): void;
+    /** Whether a constrained turn may ask the engine to force a tool call. */
+    get supportsForcedToolChoice(): boolean;
+    /**
+     * Batch capability — llama-server serves `--parallel N` concurrent KV
+     * slots, so when batched inference is enabled `maxConcurrency` is that
+     * slot count; otherwise 1 (we don't opt into co-batching). See
+     * {@link LLMProvider.batch}.
+     */
+    get batch(): BatchCapability;
+    /**
+     * Cache adapter (Phase 1+) — set by ChatManager after construction
+     * and read by sessions on every send to derive `cache_prompt` +
+     * `id_slot` extras. Null when no controller is wired (cloud-only
+     * install, tests).
+     */
+    private cacheAdapter;
+    private slotSavePath?;
+    private readonly launchedSlots;
+    setCacheAdapter(adapter: import('./cache-adapter.js').LlamaCppCacheAdapter): void;
+    getCacheAdapter(): import('./cache-adapter.js').LlamaCppCacheAdapter | null;
+    /**
+     * Directory passed to llama-server's `--slot-save-path` on launch.
+     * The cache adapter reads this to wire slot save/restore I/O.
+     * Undefined on external-baseUrl mode (we don't control the launch).
+     */
+    getSlotSavePath(): string | undefined;
+    /**
+     * The `--parallel` slot count this provider was constructed for — the
+     * ENGINE's slot space, which is what the cache adapter must size its
+     * slot model to. Deliberately NOT `queue.concurrency`: the queue
+     * reserves a background lane ABOVE the engine slots
+     * (`max(slots, interactive+1)`), so on a single-slot launch the queue
+     * reads 2 — and an adapter sized off it binds sessions to slot ids the
+     * server doesn't have. Wild-caught 2026-08-03: the adapter "bound"
+     * slot 1 on a `--parallel 1` server; every save/restore against it
+     * would 400 silently, and the debug probe's slot narration was fiction.
+     */
+    getLaunchedSlots(): number;
+    /**
+     * Claim one physical llama-server request slot.
+     *
+     * This is deliberately narrower than {@link queue}: a logical turn holds its
+     * ProviderQueue lease across the whole tool loop, while this lease covers one
+     * cache-prepare + `/v1/chat/completions` round-trip. On a one-slot launch that
+     * lets a synchronous background one-shot run between foreground iterations,
+     * but prevents the wild-caught failure where a recovery nudge saved/restored
+     * slot 0 while another session was still generating on slot 0, leaving the
+     * native prompt processor stuck forever.
+     */
+    acquireExclusiveEngineRequest(label: string, signal?: AbortSignal, onWait?: (info: {
+        aheadOf: number;
+    }) => void): Promise<() => void>;
+    /**
+     * Current base URL (when supervised) or the externally-configured
+     * one. `undefined` when the supervised engine hasn't started yet —
+     * the cache adapter treats that as "engine not ready, return empty
+     * usage."
+     */
+    currentBaseUrl(): string | null;
+    /** True only while this native engine is serving or queuing a physical request. */
+    isEngineBusy(): boolean;
+    engineLifecycleSnapshot(): NativeEngineLifecycleSnapshot | undefined;
+    initialize(): Promise<void>;
+    get isDisposed(): boolean;
+    shutdown(): Promise<void>;
+    createSession(opts: SessionOpts): Promise<LLMSession>;
+    getContextWindow(): number;
+    /**
+     * Live engine launch provenance (granted context, slots, KV dtype) from
+     * the supervisor — undefined in external base-URL mode or when no child
+     * process is up. See `LLMProvider.engineLaunchSnapshot`.
+     */
+    engineLaunchSnapshot(): EngineLaunchSnapshot | undefined;
+    /**
+     * Weights + KV at the granted window/cache mode, from the launch
+     * admission pass. See `LLMProvider.plannedReservationBytes`.
+     */
+    plannedReservationBytes(): number | undefined;
+    /** See `LLMProvider.plannedGpuBytes`. */
+    plannedGpuBytes(): number | undefined;
+    /**
+     * Classifier hook the supervisor wires into its `onRawLine`. Runs
+     * on every stdout/stderr line from llama-server; when the line
+     * maps to a known startup phase, fans the descriptor out to every
+     * currently-waiting session so the UI surfaces "loading model 42%"
+     * instead of a silent spinner.
+     *
+     * Public so `buildLlamaCppProvider` can register it on the
+     * supervisor it constructs; not something external callers should
+     * invoke directly.
+     */
+    onStdoutLine(line: string): void;
+    /**
+     * Surface the rolling log-file handle so the service's HTTP layer
+     * can tail it for the Settings → On-device log viewer. Returns
+     * undefined when no supervised process is running (external
+     * baseUrl mode — there's no stdout for us to capture).
+     */
+    getLogFile(): LlamaCppLogFile | undefined;
+    /** Internal — called by LlamaCppSession to register itself for startup fan-out. */
+    _registerActiveSession(session: LlamaCppSession): void;
+    _deregisterActiveSession(session: LlamaCppSession): void;
+    listModels(): Promise<ModelInfo[]>;
+    getEffectiveModelId(): string | undefined;
+    private resolveBaseUrl;
+    private acquireGpuLease;
+}