@bendyline/gezel-inference 0.0.0-stage → 1.0.0

package/dist/providers/llama-cpp/offload-planner.d.ts

@@ -0,0 +1,339 @@
+/**
+ * Hardware-aware structured-weight offload planner (Phase v4 — dense FFN).
+ *
+ * The user's headline case: a big Mixture-of-Experts model on a
+ * constrained discrete GPU (e.g. 4–12 GB VRAM + 32–64 GB system RAM).
+ * The whole model won't fit in VRAM, but its *attention/dense* layers
+ * will — the bulk is the sparse expert weights. b9843's `--cpu-moe`
+ * keeps exactly those experts in system RAM (streamed per token) while
+ * every attention layer stays on the GPU; `--n-cpu-moe N` is the
+ * partial form that keeps only blocks `0..N-1`'s experts in RAM.
+ *
+ * Two tiers of decision, by input quality:
+ *
+ *   • **Exact** (`split` present — per-tensor sums from the GGUF header):
+ *     size the non-expert residue against VRAM, then pack as many
+ *     trailing layers' experts as fit into what's left and emit the
+ *     matching `--n-cpu-moe N`. A card that can hold *some* experts no
+ *     longer strands that VRAM.
+ *
+ *   • **Binary** (no split): the v2 behavior, unchanged — all experts to
+ *     RAM whenever the resident estimate exceeds VRAM.
+ *
+ * Pure and deterministic → unit-tested. All I/O (VRAM probe, GGUF read)
+ * happens in the caller, which passes plain numbers in.
+ *
+ * The returned shape is a subset of `PlannerOffloadDecision` from
+ * `engine-flags.ts`, so it feeds straight into `buildLlamaCppEngineArgs`
+ * as the lowest-precedence layer (global config + manifest still win).
+ *
+ * Dense models get the same treatment through llama.cpp v0.4.0's
+ * `--n-cpu-ffn`: keep attention on the GPU and move only the first N
+ * blocks' dense feed-forward weights to system RAM. This is materially
+ * better than the engine's whole-layer fallback when a dense model is only
+ * somewhat larger than VRAM, but it is enabled only on a discrete-GPU host
+ * whose system-RAM budget can hold the selected prefix.
+ *
+ * When a plan is still too optimistic (KV estimates are coarse, other
+ * processes hold VRAM), the launch path degrades it one step per CUDA/
+ * Vulkan OOM via {@link degradeMoeOffloadDecision} and retries — see the
+ * supervisor's `recoverStartup` hook.
+ */
+export interface MoeExpertSplit {
+    /** Bytes `--cpu-moe` leaves on the GPU (attention, dense FFN, shared experts, embeddings). */
+    nonExpertBytes: number;
+    /** Routed-expert bytes per block, indexed by `blk.N` (0 for dense blocks). */
+    expertBytesByLayer: number[];
+}
+export interface DenseFfnSplit {
+    /** Bytes `--n-cpu-ffn` leaves on the GPU (attention, embeddings, norms, output). */
+    nonFfnBytes: number;
+    /** Dense FFN bytes per block, indexed by `blk.N`. */
+    ffnBytesByLayer: number[];
+}
+export interface DenseFfnOffloadInput {
+    /** Dense-FFN offload deliberately does not apply to MoE models. */
+    isMoE: boolean;
+    /** Largest single-GPU VRAM pool in bytes (0 = no discrete GPU). */
+    vramBytes: number;
+    /** Exact per-tensor dense-FFN/non-FFN sums from the GGUF header. */
+    split?: DenseFfnSplit;
+    /** `<arch>.block_count` — bounds `--n-cpu-ffn N`. */
+    blockCount?: number;
+    /** KV bytes resident on the GPU at the selected context and slot count. */
+    kvReserveBytes?: number;
+    /** Projector or other weights not represented by the primary GGUF split. */
+    additionalGpuBytes?: number;
+    /** Capacity-policy share of system RAM available to local engines. */
+    ramBudgetBytes?: number;
+    /** Live reclaimable system RAM. Used as a second, stricter safety gate. */
+    freeSystemRamBytes?: number;
+    /** Headroom to leave free in each memory pool; defaults to 1 GiB. */
+    marginBytes?: number;
+}
+export interface MoeOffloadInput {
+    /** True when the GGUF declares `<arch>.expert_count > 1`. */
+    isMoE: boolean;
+    /**
+     * Estimated GPU-resident working set of the model if fully offloaded
+     * (weights + KV cache + activations). Callers derive it from
+     * `approxSizeBytes * ~1.2`.
+     */
+    residentBytes: number;
+    /** Largest single-GPU VRAM pool in bytes (0 = no GPU device found). */
+    vramBytes: number;
+    /** Headroom to leave free; defaults to 1 GiB. */
+    marginBytes?: number;
+    /** Exact per-tensor expert/non-expert sums from the GGUF header, when scanned. */
+    split?: MoeExpertSplit;
+    /** `<arch>.block_count` — bounds `--n-cpu-moe N`. Defaults to the split's layer count. */
+    blockCount?: number;
+    /**
+     * VRAM the KV cache will hold at the launch context size (all KV stays
+     * on-GPU under `-ngl all`). See {@link estimateKvReserveBytes}. Only
+     * consulted on the exact path; 0 / absent = weights-only budgeting.
+     */
+    kvReserveBytes?: number;
+}
+export interface MoeOffloadDecision {
+    /** `--n-gpu-layers` (−1 = all). Undefined = leave to the engine. */
+    nGpuLayers?: number;
+    /** `--cpu-moe` — keep all MoE experts in system RAM. */
+    cpuMoe?: boolean;
+    /** `--n-cpu-moe N` — keep blocks `0..N-1`'s experts in system RAM. */
+    nCpuMoe?: number;
+    /** `--n-cpu-ffn N` — keep blocks `0..N-1`'s dense FFN weights in RAM. */
+    nCpuFfn?: number;
+    /**
+     * Set only when every weight fits the GPU pool. The engine-flag builder
+     * reads it as positive evidence that no model compute runs on the CPU,
+     * which is what licenses its GPU-resident thread default. Decisions that
+     * leave placement to the engine omit it on purpose: llama.cpp's own fit
+     * may then put layers on the CPU, where every thread counts.
+     */
+    fullGpuResidency?: true;
+    /**
+     * Graphics memory this plan puts on the card: the always-active residue,
+     * the expert layers kept there, the KV cache and the reserves. Set only on
+     * an exact MoE offload plan. Device admission charges it instead of the
+     * whole card, which refused an offloaded model whenever anything else was
+     * using the GPU.
+     */
+    gpuBytes?: number;
+    /** Human-readable rationale for the decision log (never emitted as a flag). */
+    reason?: string;
+}
+/**
+ * Plan llama.cpp v0.4.0's dense-FFN split on suitable discrete-GPU hosts.
+ *
+ * There is intentionally no estimate-only fallback. The decision pins
+ * `-ngl all`, so it must be based on the exact tensors the accompanying
+ * override moves; an approximate split could turn the optimization into a
+ * startup OOM. If the non-FFN GPU residue or the CPU prefix cannot fit its
+ * respective pool, leave the launch to llama.cpp's conservative `--fit`
+ * whole-layer strategy.
+ */
+export declare function planDenseFfnOffload(input: DenseFfnOffloadInput): MoeOffloadDecision;
+/**
+ * Decide how to split a MoE model between VRAM and system RAM.
+ *
+ * - No GPU / unknown VRAM  → no decision (engine default).
+ * - Dense model           → no decision (its `--fit`/`-ngl auto`
+ *                            layer-dropping is the right tool).
+ * - MoE that fits in VRAM  → no decision (full-GPU residency is fastest).
+ * - MoE that won't fit     → with exact tensor sums: keep the largest
+ *                            suffix of expert layers that fits beside the
+ *                            non-expert residue (`--n-cpu-moe N`), or all
+ *                            experts to RAM when none fit; without exact
+ *                            sums: `-ngl all --cpu-moe`.
+ * - Residue exceeds VRAM   → `--cpu-moe` with NO `-ngl` pin, so the
+ *                            engine's own fit can drop whole layers too.
+ */
+export declare function planMoeOffload(input: MoeOffloadInput): MoeOffloadDecision;
+/**
+ * One step down the launch-recovery ladder after a GPU out-of-memory at
+ * startup. Each step trades speed for admission; `null` means the planner
+ * has nothing safer left (dropping `--cpu-moe` itself would put experts
+ * BACK on the GPU, which cannot help an OOM).
+ *
+ *   `--n-cpu-moe N, -ngl all` → `--cpu-moe, -ngl all` → `--cpu-moe` (engine
+ *   fit may drop whole layers) → null.
+ *
+ * Explicit config/manifest offload settings shadow the planner per-field
+ * (see `buildLlamaCppEngineArgs`), so degrading a fully-shadowed decision
+ * is pointless — the caller checks that before retrying.
+ */
+export declare function degradeMoeOffloadDecision(decision: MoeOffloadDecision | undefined): MoeOffloadDecision | null;
+/** Dense counterpart of {@link degradeMoeOffloadDecision}. */
+export declare function degradeDenseFfnOffloadDecision(decision: MoeOffloadDecision | undefined, blockCount: number | undefined): MoeOffloadDecision | null;
+export interface KvReserveInput {
+    blockCount?: number | undefined;
+    embeddingLength?: number | undefined;
+    headCount?: number | undefined;
+    headCountKv?: number | undefined;
+    /**
+     * Per-layer KV heads, index-aligned with `slidingWindowPattern`
+     * (Gemma 4). When present (with the pattern and SWA dims), the estimate
+     * becomes per-layer exact instead of mean-based.
+     */
+    headCountKvPerLayer?: number[] | undefined;
+    /** Per-layer flags: true = SWA layer, which caches at the `*Swa` dims. */
+    slidingWindowPattern?: boolean[] | undefined;
+    /** Trailing logical layers that share earlier K/V and own no cache tensors. */
+    sharedKvLayers?: number | undefined;
+    /**
+     * `<arch>.num_loops` — a looped transformer runs the block stack this many
+     * times over shared weights, each pass keeping its OWN KV cache (llama.cpp
+     * unrolls them into `n_layer`). Multiplies every per-token KV quantity.
+     * Absent / 1 on ordinary models.
+     */
+    loopCount?: number | undefined;
+    keyLength?: number | undefined;
+    valueLength?: number | undefined;
+    /** SWA-layer head dims when they differ from the global ones (Gemma 4: 256 vs 512). */
+    keyLengthSwa?: number | undefined;
+    valueLengthSwa?: number | undefined;
+    /**
+     * `<arch>.full_attention_interval` — linear-attention hybrids
+     * (`qwen35moe`) make only every Nth layer full attention; the rest keep
+     * a context-independent recurrent state. Counting all layers as
+     * KV-scaling overstates a 40-layer/interval-4 model 4x.
+     */
+    fullAttentionInterval?: number | undefined;
+    /** `<arch>.ssm.*` — recurrent-state geometry on the linear layers. */
+    ssmInnerSize?: number | undefined;
+    ssmStateSize?: number | undefined;
+    ssmConvKernel?: number | undefined;
+    /** Total launch context (`--ctx-size`, i.e. per-slot ctx × slots). */
+    ctxTokens: number;
+    /** The launcher's `--cache-type-k/v` value (assumed symmetric). */
+    kvCacheType?: string | undefined;
+}
+/**
+ * Estimate the KV cache footprint with every cache-owning layer caching the
+ * full context — what llama.cpp allocates for ordinary full-attention
+ * models, and for SWA models under `--swa-full`. Trailing shared-KV layers
+ * reuse an earlier cache and therefore contribute no allocation of their own.
+ *
+ * Per-layer exact when the header supplies the layout: SWA layers cache
+ * at their own head dims (`key/value_length_swa`), so pricing them at the
+ * global dims overstated Gemma 4's true `--swa-full` allocation ~2×
+ * (31b at 64K: ~55 GB real vs ~105 GB mean-based) — enough to decline
+ * the full cache on machines that could genuinely hold it. Falls back to
+ * mean heads × global dims when per-layer data is absent, and to
+ * undefined when even that is missing, so callers budget weights-only
+ * rather than on a guess. For the DEFAULT (windowed) cache of an SWA
+ * model, use {@link estimateWindowedKvLinearization} instead.
+ */
+export declare function estimateKvReserveBytes(input: KvReserveInput): number | undefined;
+export interface WindowedKvInput {
+    blockCount?: number | undefined;
+    embeddingLength?: number | undefined;
+    headCount?: number | undefined;
+    /** Scalar/mean KV heads — used for every layer when no per-layer array exists. */
+    headCountKv?: number | undefined;
+    /** Per-layer KV heads, index-aligned with `slidingWindowPattern` (Gemma 4). */
+    headCountKvPerLayer?: number[] | undefined;
+    /** SWA window size in tokens (`<arch>.attention.sliding_window`). */
+    slidingWindow?: number | undefined;
+    /** Per-layer flags: true = SWA layer, false = global layer. */
+    slidingWindowPattern?: boolean[] | undefined;
+    /** Trailing logical layers that share earlier K/V and own no cache tensors. */
+    sharedKvLayers?: number | undefined;
+    /**
+     * `<arch>.num_loops` — a looped transformer runs the block stack this many
+     * times over shared weights, each pass keeping its OWN KV cache (llama.cpp
+     * unrolls them into `n_layer`). Multiplies every per-token KV quantity.
+     * Absent / 1 on ordinary models.
+     */
+    loopCount?: number | undefined;
+    keyLength?: number | undefined;
+    valueLength?: number | undefined;
+    /** SWA-layer head dims when they differ from the global ones (Gemma 4: 256 vs 512). */
+    keyLengthSwa?: number | undefined;
+    valueLengthSwa?: number | undefined;
+    /**
+     * `<arch>.full_attention_interval` — linear-attention hybrids
+     * (`qwen35moe`) make only every Nth layer full attention; the rest keep
+     * a context-independent recurrent state. Counting all layers as
+     * KV-scaling overstates a 40-layer/interval-4 model 4x.
+     */
+    fullAttentionInterval?: number | undefined;
+    /** `<arch>.ssm.*` — recurrent-state geometry on the linear layers. */
+    ssmInnerSize?: number | undefined;
+    ssmStateSize?: number | undefined;
+    ssmConvKernel?: number | undefined;
+    kvCacheType?: string | undefined;
+}
+export interface WindowedKvLinearization {
+    /**
+     * KV bytes per context token per slot — global (full-attention) layers
+     * only, the sole component that scales with the window.
+     */
+    bytesPerToken: number;
+    /**
+     * Context-independent KV bytes per slot: the SWA layers' window-capped
+     * caches. Add to the resident-weights term when feeding a linear
+     * admission model (`total(ctx) = fixed + bytesPerToken × ctx`).
+     */
+    fixedBytes: number;
+}
+/**
+ * Price the DEFAULT (windowed) KV cache of a sliding-window-attention
+ * model as a linear function of context: SWA layers cache only
+ * ~`slidingWindow` tokens regardless of context (a fixed cost), while the
+ * global layers scale with it. On Gemma 4 the two shapes differ by more
+ * than the layer ratio — global layers carry fewer KV heads at wider head
+ * dims (31b: 10 layers × 4 heads × 512+512 vs 50 layers × 16 heads ×
+ * 256+256) — so full-attention math overstates the windowed cache ~14×
+ * and using it for admission over-clamps or over-denies exactly the
+ * launches the `--swa-full` decline is trying to keep whole (the
+ * 2026-08-04 gemma4-26b sweep shipped 19–56K windows on a 65536 request
+ * this way).
+ *
+ * Returns undefined when the header lacks the SWA layout (no window, no
+ * pattern, length mismatches) so callers fall back to their conservative
+ * path instead of guessing. Shared trailing KV layers are excluded exactly:
+ * llama.cpp does not allocate cache tensors for them.
+ */
+/**
+ * Exact per-slot KV bytes at f16 for the slot-ceiling math: the windowed
+ * linearization when the header shows an SWA layout (the engine-default
+ * cache mode, and the one that yields the honest slot upper bound — a
+ * full-attention number would cut Gemma to one slot for a cache it only
+ * runs when it fits anyway), per-layer full-attention otherwise.
+ * Undefined when the header lacks the dims; callers fall back to the
+ * weights-scaled heuristic.
+ */
+export declare function estimateExactPerSlotKvBytesF16(input: Omit<WindowedKvInput, 'kvCacheType'>, perTurnCtxTokens: number): number | undefined;
+/**
+ * Price a linear-attention hybrid (`qwen35moe`: Qwen 3.5 MoE, Ornith,
+ * BTL-4) as `fixed + slope × ctx`, the same shape the SWA linearization
+ * returns and the same shape `planCtxTokensForMemory` consumes.
+ *
+ * Only the full-attention layers scale with the window; the linear layers
+ * carry a recurrent state whose size does not depend on context at all.
+ * A 40-layer model at interval 4 therefore caches 10 layers, not 40 —
+ * ~20 KB/token instead of ~80 KB/token, which is a 4x difference on the
+ * exact models whose selling point is a cheap 256K context. Pricing them
+ * as full attention would clamp or deny windows the engine holds easily.
+ *
+ * Returns undefined when the header shows no interval (ordinary models)
+ * or lacks the dims, so callers fall back to full-attention math.
+ */
+export declare function estimateLinearHybridKvLinearization(input: WindowedKvInput): WindowedKvLinearization | undefined;
+export declare function estimateWindowedKvLinearization(input: WindowedKvInput): WindowedKvLinearization | undefined;
+export interface SwaFullFastMemoryFitInput {
+    residentWeightsBytes: number;
+    fullKvBytes: number;
+    /** Already safety-discounted fast-memory budget (VRAM on a discrete GPU). */
+    fastBudgetBytes: number;
+    /** Fast-memory reservations held by models that are already resident. */
+    committedOtherBytes?: number | undefined;
+}
+/**
+ * Whether Auto may keep `--swa-full` without spilling the model or KV cache
+ * out of the fast pool. The capacity budget already reserves driver/OS
+ * headroom, so this comparison must not discount it a second time.
+ */
+export declare function fitsSwaFullInFastMemory(input: SwaFullFastMemoryFitInput): boolean;