@elizaos/plugin-local-inference 2.0.0-beta.1 → 2.0.11-beta.7

package/src/services/ffi-streaming-backend.ts

@@ -0,0 +1,382 @@
+/**
+ * In-process FFI streaming backend adapter.
+ *
+ * Implements `LocalInferenceBackend` as the optimized in-process
+ * llama.cpp path used by Eliza-1 on desktop and mobile.
+ *
+ * What this class deliberately does NOT do:
+ *   - Own the FFI context. The runtime provider passed to this class owns
+ *     native load/unload and hands back the binding, context, and tokenizer.
+ *   - Decode image bytes or call mtmd directly. Vision requests are validated
+ *     here, then forwarded to runtimes that expose `describeImage`.
+ */
+
+import type {
+	BackendPlan,
+	GenerateArgs,
+	GenerateResult,
+	LocalGenerateWithUsageResult,
+	LocalInferenceBackend,
+} from "./backend";
+import type { FfiStreamingRunner } from "./ffi-streaming-runner";
+import type {
+	LlmCtxHandle,
+	LlmStreamingBinding,
+} from "./llm-streaming-binding";
+import { resolveGuidedDecodeForParams } from "./structured-output";
+
+/**
+ * Constructor-injected adapter that resolves the FFI binding, context, and
+ * tokenizer for a given load. Two responsibilities:
+ *
+ *   1. Decide whether the FFI path is viable on the current binding
+ *      (`supported()`). Mirrors `LlmStreamingBinding.llmStreamSupported()`
+ *      plus any higher-level constraints (e.g. dylib path exists, build
+ *      target matches the bundle's required kernels).
+ *   2. Lifecycle: `acquire(plan)` returns the FFI runner ready for
+ *      `generate()` against the requested model, plus a tokenizer that
+ *      matches that model's vocab. `release()` tears everything down.
+ *
+ * Production runtime implementation: the fused libelizainference path
+ * (`desktop-fused-ffi-backend-runtime.ts`), which wraps `ElizaInferenceFfi`
+ * via `wrapElizaInferenceFfi()` from `services/llm-streaming-binding.ts`.
+ * libllama has been retired — there is no second runtime behind this slot.
+ */
+export interface FfiBackendRuntime {
+	supported(): boolean;
+	acquire(plan: BackendPlan): Promise<FfiBackendSession>;
+	release(): Promise<void>;
+	/**
+	 * Optional parallel-slot pool surface. When the runtime exposes a
+	 * ctx pool (the desktop libllama path does), `parallelSlots()`
+	 * reports the live count and `resizeParallel(N)` grows/shrinks it.
+	 * Runtimes without a pool report 1 and ignore resize requests.
+	 */
+	parallelSlots?(): number;
+	resizeParallel?(target: number): Promise<boolean>;
+}
+
+/**
+ * Result of `FfiBackendRuntime.acquire()` — a live FFI session bound to a
+ * specific loaded model.
+ */
+export interface FfiBackendSession {
+	readonly binding: LlmStreamingBinding;
+	readonly ctx: LlmCtxHandle;
+	readonly runner: FfiStreamingRunner;
+	/**
+	 * Tokenize a prompt string into model token ids using the loaded model's
+	 * tokenizer. The vocab MUST match the GGUF — mismatches produce gibberish
+	 * silently. The runtime is responsible for asserting this at acquire
+	 * time.
+	 */
+	readonly tokenize: (prompt: string) => Int32Array;
+	/**
+	 * Native MTP speculative-decoding policy from the catalog. `null`
+	 * disables speculative decoding for this session.
+	 */
+	readonly mtp: {
+		specType: "draft-mtp";
+		draftMin: number;
+		draftMax: number;
+		gpuLayers: number | "auto";
+	} | null;
+	/**
+	 * Absolute path to a *separate* MTP drafter GGUF resolved during load.
+	 * `null` means same-file MTP: the NextN head is embedded in the main
+	 * text GGUF and the native runner activates `--spec-type draft-mtp`
+	 * with no `-md`. Speculative decoding is governed by `mtp`, not by the
+	 * presence of this path.
+	 */
+	readonly draftModelPath: string | null;
+	/**
+	 * Multimodal projector (mmproj) GGUF path for vision describe. Resolved
+	 * from `plan.overrides.mmprojPath` at acquire time. `null` disables
+	 * vision — `describeImage` then throws an actionable error.
+	 */
+	readonly mmprojPath: string | null;
+	/**
+	 * Per-load runtime config the fused libelizainference path applies at its
+	 * first `llmStreamOpen` (gpuLayers + KV-cache quant types). The desktop
+	 * libllama runtime applies these at `loadModel()` instead and leaves this
+	 * `null` — the backend forwards them into the runner's per-call config only
+	 * when present, so the fused path mirrors the libllama load decision without
+	 * the libllama path double-applying them.
+	 */
+	readonly loadConfig?: {
+		gpuLayers?: number;
+		cacheTypeK?: string | null;
+		cacheTypeV?: string | null;
+	} | null;
+}
+
+/**
+ * Adapter that satisfies `LocalInferenceBackend` by delegating to
+ * `FfiStreamingRunner`. The `id` is `"llama-cpp"` because this is the
+ * in-process variant of the optimized llama.cpp path.
+ */
+export class FfiStreamingBackend implements LocalInferenceBackend {
+	readonly id = "llama-cpp" as const;
+
+	private session: FfiBackendSession | null = null;
+	private loadedPath: string | null = null;
+
+	constructor(private readonly runtime: FfiBackendRuntime) {}
+
+	async available(): Promise<boolean> {
+		return this.runtime.supported();
+	}
+
+	hasLoadedModel(): boolean {
+		return this.session !== null;
+	}
+
+	currentModelPath(): string | null {
+		return this.loadedPath;
+	}
+
+	async load(plan: BackendPlan): Promise<void> {
+		if (this.session) await this.unload();
+		this.session = await this.runtime.acquire(plan);
+		this.loadedPath = plan.modelPath;
+	}
+
+	async unload(): Promise<void> {
+		// Await the native release BEFORE nulling our refs. If we null first and
+		// release() throws (a raw bun:ffi free can reject), this.session would be
+		// null while the runtime still holds a live session — the next load()
+		// would skip unload() and call acquire(), which throws on its live-session
+		// guard, wedging the backend until process restart. The finally guarantees
+		// our refs are cleared regardless so a failed release can't leave a stale
+		// "loaded" view either.
+		try {
+			await this.runtime.release();
+		} finally {
+			this.session = null;
+			this.loadedPath = null;
+		}
+	}
+
+	async generate(args: GenerateArgs): Promise<GenerateResult> {
+		const result = await this.generateWithUsage(args);
+		return result.text;
+	}
+
+	async generateWithUsage(
+		args: GenerateArgs & { slotId?: number },
+	): Promise<LocalGenerateWithUsageResult> {
+		if (!this.session) {
+			throw new Error(
+				"[ffi-streaming-backend] generate() called before load() — " +
+					"the FFI session has not been acquired.",
+			);
+		}
+		const { runner, tokenize, mtp, draftModelPath, loadConfig } = this.session;
+		// Force the structured-reply envelope: compile the GBNF from the
+		// caller's `responseSkeleton` / explicit `grammar` (precedence handled
+		// by `resolveGuidedDecodeForParams`, mirroring `engine.ts`'s
+		// `resolveBindingGrammarSource`). The native session installs it FIRST
+		// in the sampler chain so every sampled token is grammar-constrained.
+		const gbnfGrammar =
+			resolveGuidedDecodeForParams(args).grammar?.source ?? null;
+		const result = await runner.generateWithUsage({
+			promptTokens: tokenize(args.prompt),
+			slotId: args.slotId ?? -1,
+			cacheKey: args.cacheKey,
+			maxTokens: args.maxTokens ?? 2048,
+			temperature: args.temperature ?? 0.7,
+			topP: args.topP ?? 0.9,
+			topK: 40,
+			repeatPenalty: 1.1,
+			draftMin: mtp?.draftMin ?? 0,
+			draftMax: mtp?.draftMax ?? 0,
+			draftModelPath,
+			gbnfGrammar,
+			gpuLayers: loadConfig?.gpuLayers,
+			cacheTypeK: loadConfig?.cacheTypeK,
+			cacheTypeV: loadConfig?.cacheTypeV,
+			signal: args.signal,
+			onTextChunk: args.onTextChunk,
+			onVerifierEvent: args.onVerifierEvent,
+		});
+		return {
+			text: result.text,
+			slotId: result.slotId,
+			firstTokenMs: result.firstTokenMs,
+			usage: {
+				completion_tokens: result.accepted,
+			},
+			mtpStats: {
+				drafted: result.drafted,
+				accepted: result.accepted,
+				acceptanceRate:
+					result.drafted > 0 ? result.accepted / result.drafted : null,
+			},
+		};
+	}
+
+	// === Optional `LocalInferenceBackend` methods routed through the runner.
+
+	/**
+	 * Persist the active session's KV state to a per-conversation file.
+	 * v1 uses `llama_state_seq_save_file` against seq_id=0. The on-disk file
+	 * path mirrors `ffi-streaming-backend.ts`'s conversation-keyed slot layout
+	 * (`<cacheDir>/<conversationId>/<slotId>.kv`) so a switch between
+	 * FFI and subprocess can resume each other's slots — once both
+	 * paths agree on the file format.
+	 */
+	async persistConversationKv(
+		conversationId: string,
+		slotId: number,
+	): Promise<void> {
+		if (!this.session) return; // no active session to persist
+		const { binding } = this.session;
+		if (!binding.llmStreamSaveSlot) return; // adapter doesn't support save
+		const filename = slotFilename(conversationId, slotId);
+		// llmStreamSaveSlot is per-stream in the binding API; the desktop
+		// adapter currently saves the ctx-wide seq=0 state, so the stream
+		// handle is informational. We pass the runner's most recent
+		// stream id when available; 0n is the binding-level sentinel.
+		binding.llmStreamSaveSlot({ stream: 0n, filename });
+	}
+
+	/** Restore a previously persisted KV state. Mirror of `persistConversationKv`. */
+	async restoreConversationKv(
+		conversationId: string,
+		slotId: number,
+	): Promise<boolean> {
+		if (!this.session) return false;
+		const { binding } = this.session;
+		if (!binding.llmStreamRestoreSlot) return false;
+		const filename = slotFilename(conversationId, slotId);
+		binding.llmStreamRestoreSlot({ stream: 0n, filename });
+		return true;
+	}
+
+	/**
+	 * Pre-decode `promptPrefix` so the next `generate` against the same
+	 * `cacheKey` skips re-prefill. Returns `false` when the prefix is
+	 * empty or no session is loaded. The FFI runner serializes by
+	 * `cacheKey` internally via the `slotInFlight` map.
+	 */
+	async prewarmConversation(
+		promptPrefix: string,
+		opts: { slotId: number; cacheKey: string },
+	): Promise<boolean> {
+		if (!this.session || promptPrefix.length === 0) return false;
+		const { runner, tokenize, mtp, draftModelPath, loadConfig } = this.session;
+		await runner.generateWithUsage({
+			promptTokens: tokenize(promptPrefix),
+			slotId: opts.slotId,
+			cacheKey: opts.cacheKey,
+			maxTokens: 0, // prefill-only: feed prompt, generate nothing
+			temperature: 0,
+			topP: 1,
+			topK: 1,
+			repeatPenalty: 1,
+			draftMin: mtp?.draftMin ?? 0,
+			draftMax: mtp?.draftMax ?? 0,
+			draftModelPath,
+			gpuLayers: loadConfig?.gpuLayers,
+			cacheTypeK: loadConfig?.cacheTypeK,
+			cacheTypeV: loadConfig?.cacheTypeV,
+		});
+		return true;
+	}
+
+	/**
+	 * True when Eliza-1 native MTP is active for the loaded target model.
+	 * Covers both shapes: same-file MTP (NextN head embedded in the text
+	 * GGUF, `draftModelPath` null) and separate-drafter MTP.
+	 */
+	mtpEnabled(): boolean {
+		return Boolean(this.session?.mtp);
+	}
+
+	/**
+	 * Parallel-slot pool size. Routed to the runtime's ctx pool when one
+	 * exists; defaults to 1 otherwise.
+	 */
+	parallelSlots(): number {
+		return this.runtime.parallelSlots?.() ?? 1;
+	}
+
+	/**
+	 * Grow or shrink the runtime's ctx pool to `target` slots. Returns
+	 * false when the runtime has no pool surface (in which case parallel
+	 * resize is ignored — the conversation registry tolerates
+	 * fixed 1-slot operation).
+	 */
+	async resizeParallel(target: number): Promise<boolean> {
+		if (!this.runtime.resizeParallel) return false;
+		return this.runtime.resizeParallel(target);
+	}
+
+	/**
+	 * Vision describe via mmproj. Requires:
+	 *   - The shim built with `-DELIZA_ENABLE_VISION=1` (ELIZA_ENABLE_VISION=1
+	 *     at the build script env). When absent the runtime throws an
+	 *     actionable error.
+	 *   - `plan.overrides.mmprojPath` was passed at load time so the
+	 *     adapter knows which mmproj GGUF to feed clip.
+	 */
+	async describeImage(args: {
+		bytes: Uint8Array;
+		mimeType?: string;
+		prompt?: string;
+		maxTokens?: number;
+		temperature?: number;
+		signal?: AbortSignal;
+	}): Promise<{ text: string; projectorMs?: number; decodeMs?: number }> {
+		if (!this.session) {
+			throw new Error(
+				"[ffi-streaming-backend] describeImage before load — no session acquired",
+			);
+		}
+		if (!this.session.mmprojPath) {
+			throw new Error(
+				"[ffi-streaming-backend] describeImage: no mmproj GGUF loaded for this session. " +
+					"Pass `overrides.mmprojPath` in the BackendPlan when activating a vision-capable bundle.",
+			);
+		}
+		// The runtime adapter has visionSupported() + describeImage(args).
+		// We re-shape `bytes` → `imageBytes` and merge in the resolved
+		// mmprojPath; the rest of args pass through unchanged.
+		const runtime = this.runtime as unknown as {
+			describeImage?: (args: {
+				imageBytes: Uint8Array;
+				mmprojPath: string;
+				prompt?: string;
+				maxTokens?: number;
+				temperature?: number;
+				signal?: AbortSignal;
+			}) => Promise<{ text: string; projectorMs?: number; decodeMs?: number }>;
+		};
+		if (!runtime.describeImage) {
+			throw new Error(
+				"[ffi-streaming-backend] runtime lacks describeImage support",
+			);
+		}
+		return runtime.describeImage({
+			imageBytes: args.bytes,
+			mmprojPath: this.session.mmprojPath,
+			prompt: args.prompt,
+			maxTokens: args.maxTokens,
+			temperature: args.temperature,
+			signal: args.signal,
+		});
+	}
+
+	currentMmprojPath(): string | null {
+		return this.session?.mmprojPath ?? null;
+	}
+}
+
+/**
+ * Conversation-keyed slot file layout. Mirrors `cache-bridge.ts`'s
+ * `slotSavePath` so an `ELIZA_INFERENCE_BACKEND=http` opt-out can resume
+ * an FFI-saved conversation and vice-versa once the file formats align.
+ */
+function slotFilename(conversationId: string, slotId: number): string {
+	return `${conversationId}__slot${slotId}.kv`;
+}

package/src/services/ffi-streaming-runner.d.ts

@@ -0,0 +1,122 @@
+/**
+ * In-process streaming-LLM runner.
+ *
+ * FFI streaming-LLM ABI declared in `ffi-streaming-llm.h`. The
+ * token-by-token loop hands `onTextChunk` accepted chunks and surfaces
+ * verifier events from native MTP.
+ *
+ * This file deliberately does not own the FFI context or the binding
+ * itself. It takes a narrow `LlmStreamingBinding` (see
+ * `services/llm-streaming-binding.ts`) + an opaque `LlmCtxHandle` as
+ * constructor arguments — that way it can be driven by libelizainference
+ * (via `wrapElizaInferenceFfi`) or any desktop libllama shim adapter without
+ * dragging in TTS/ASR surfaces. A single context can host concurrent generation
+ * sessions (one per pinned slot); the runner serialises with
+ * `slotInFlight`.
+ *
+ * Single-flight: lock map keyed by slot id, slot id `-1` unlocked. Two concurrent generates
+ * against the same pinned slot would interleave KV cache state, so the
+ * runner serializes them at the JS layer.
+ */
+import type { LlmCtxHandle, LlmStreamingBinding } from "./llm-streaming-binding";
+import type { LlmStreamHandle, LlmStreamStep } from "./voice/ffi-bindings";
+import type { VerifierStreamEvent } from "./voice/types";
+export interface FfiStreamingGenerateArgs {
+    /** Pre-tokenized prompt — the runner does not detokenize. */
+    promptTokens: Int32Array;
+    /** Pinned slot id; -1 disables pinning (any free slot). */
+    slotId: number;
+    /** Optional prompt cache key used to derive a slot when `slotId === -1`. */
+    cacheKey?: string;
+    maxTokens: number;
+    temperature: number;
+    topP: number;
+    topK: number;
+    repeatPenalty: number;
+    draftMin: number;
+    draftMax: number;
+    /** Reserved for separate draft-model speculation; null for Eliza-1 MTP. */
+    draftModelPath: string | null;
+    /**
+     * Per-load GPU offload (ABI v8). Forwarded into the native session config
+     * on `llmStreamOpen`. The fused libelizainference path loads the text model
+     * once per ctx, so the FIRST session's value wins; later sessions reuse the
+     * resident model. `undefined` selects the runtime default (all layers).
+     * The desktop libllama path already applies gpuLayers at `loadModel()`, so
+     * it ignores this field — it is load-time config, threaded here only so the
+     * fused runner can mirror the libllama load decision.
+     */
+    gpuLayers?: number;
+    /**
+     * KV-cache K/V quant type names (ABI v8), e.g. "qjl1_256" / "q4_polar".
+     * Same load-time semantics as `gpuLayers`: forwarded into the fused
+     * session config so the first `llmStreamOpen` applies the quantized cache.
+     */
+    cacheTypeK?: string | null;
+    cacheTypeV?: string | null;
+    /**
+     * GBNF grammar source forcing the structured-reply envelope. Passed to
+     * the native session's `llmStreamOpen` config so sampling is
+     * grammar-constrained. `null` disables the constraint (free generation).
+     */
+    gbnfGrammar?: string | null;
+    /** Cancellation signal — fires `llmStreamCancel` on the active session. */
+    signal?: AbortSignal;
+    /** Per-chunk text callback. */
+    onTextChunk?: (chunk: string) => void | Promise<void>;
+    /** Speculative accept/reject events from MTP verification. */
+    onVerifierEvent?: (event: VerifierStreamEvent) => void | Promise<void>;
+}
+export interface FfiStreamingGenerateResult {
+    text: string;
+    slotId: number;
+    firstTokenMs: number | null;
+    drafted: number;
+    accepted: number;
+}
+/**
+ * Backend used by the mobile and desktop FFI routes.
+ */
+export declare class FfiStreamingRunner {
+    private readonly ffi;
+    private readonly ctx;
+    private readonly slotInFlight;
+    /**
+     * Constructor takes the narrow `LlmStreamingBinding` (see
+     * `services/llm-streaming-binding.ts`) so both libelizainference (via
+     * `wrapElizaInferenceFfi`) and desktop libllama adapters can
+     * satisfy it. The runner never touches TTS/ASR/mmap surfaces.
+     */
+    constructor(ffi: LlmStreamingBinding, ctx: LlmCtxHandle);
+    /**
+     * Run one generation. Mirrors `MtpLlamaServer.generateWithUsage()`
+     * — same single-flight rule, same callback shape, same result block
+     * minus the metrics scrape (FFI does not have a `/metrics` endpoint).
+     */
+    generateWithUsage(args: FfiStreamingGenerateArgs): Promise<FfiStreamingGenerateResult>;
+    /**
+     * Async-iterable variant. Yields each accepted-token batch as it lands
+     * so callers that want token-grained control (e.g. the voice scheduler
+     * driving phrase-chunking off accept/reject events) don't have to
+     * register a callback. Internally still routes through `generateWithUsage`
+     * via a pump so the single-flight rule applies.
+     */
+    generateStream(args: FfiStreamingGenerateArgs): AsyncIterable<LlmStreamStep>;
+    /**
+     * Save the streaming slot KV state to disk. Best called between turns
+     * — calling mid-stream is racy and the FFI side is allowed to refuse.
+     * Surfaced here so the conversation registry can persist between
+     * mobile backgrounds the same way `MtpLlamaServer.persistSlot` does.
+     */
+    saveSlot(stream: LlmStreamHandle, filename: string): void;
+    /** Restore a previously-saved slot KV file into a fresh session. */
+    restoreSlot(stream: LlmStreamHandle, filename: string): void;
+    private runGenerate;
+    /**
+     * Shared inner loop. Opens the session, runs the prefill + next pump,
+     * forwards each step through `onStep` plus the optional caller
+     * callbacks, and wires abort + cancel.
+     */
+    private runGenerateInner;
+}
+//# sourceMappingURL=ffi-streaming-runner.d.ts.map

package/src/services/ffi-streaming-runner.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"ffi-streaming-runner.d.ts","sourceRoot":"","sources":["ffi-streaming-runner.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;GAmBG;AAIH,OAAO,KAAK,EACX,YAAY,EACZ,mBAAmB,EACnB,MAAM,yBAAyB,CAAC;AACjC,OAAO,KAAK,EAAE,eAAe,EAAE,aAAa,EAAE,MAAM,sBAAsB,CAAC;AAC3E,OAAO,KAAK,EAAa,mBAAmB,EAAE,MAAM,eAAe,CAAC;AAEpE,MAAM,WAAW,wBAAwB;IACxC,6DAA6D;IAC7D,YAAY,EAAE,UAAU,CAAC;IACzB,2DAA2D;IAC3D,MAAM,EAAE,MAAM,CAAC;IACf,4EAA4E;IAC5E,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;IAClB,WAAW,EAAE,MAAM,CAAC;IACpB,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,CAAC;IACb,aAAa,EAAE,MAAM,CAAC;IACtB,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,EAAE,MAAM,CAAC;IACjB,2EAA2E;IAC3E,cAAc,EAAE,MAAM,GAAG,IAAI,CAAC;IAC9B;;;;;;;;OAQG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB;;;;OAIG;IACH,UAAU,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3B,UAAU,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3B;;;;OAIG;IACH,WAAW,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC5B,2EAA2E;IAC3E,MAAM,CAAC,EAAE,WAAW,CAAC;IACrB,+BAA+B;IAC/B,WAAW,CAAC,EAAE,CAAC,KAAK,EAAE,MAAM,KAAK,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IACtD,8DAA8D;IAC9D,eAAe,CAAC,EAAE,CAAC,KAAK,EAAE,mBAAmB,KAAK,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;CACvE;AAED,MAAM,WAAW,0BAA0B;IAC1C,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,MAAM,CAAC;IACf,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;IAC5B,OAAO,EAAE,MAAM,CAAC;IAChB,QAAQ,EAAE,MAAM,CAAC;CACjB;AAMD;;GAEG;AACH,qBAAa,kBAAkB;IAU7B,OAAO,CAAC,QAAQ,CAAC,GAAG;IACpB,OAAO,CAAC,QAAQ,CAAC,GAAG;IAVrB,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAoC;IAEjE;;;;;OAKG;gBAEe,GAAG,EAAE,mBAAmB,EACxB,GAAG,EAAE,YAAY;IAGnC;;;;OAIG;IACG,iBAAiB,CACtB,IAAI,EAAE,wBAAwB,GAC5B,OAAO,CAAC,0BAA0B,CAAC;IAsBtC;;;;;;OAMG;IACI,cAAc,CACpB,IAAI,EAAE,wBAAwB,GAC5B,aAAa,CAAC,aAAa,CAAC;IAmD/B;;;;;OAKG;IACH,QAAQ,CAAC,MAAM,EAAE,eAAe,EAAE,QAAQ,EAAE,MAAM,GAAG,IAAI;IASzD,oEAAoE;IACpE,WAAW,CAAC,MAAM,EAAE,eAAe,EAAE,QAAQ,EAAE,MAAM,GAAG,IAAI;YAW9C,WAAW;IA2BzB;;;;OAIG;YACW,gBAAgB;CAmG9B"}

package/src/services/ffi-streaming-runner.test.ts

@@ -0,0 +1,60 @@
+import { describe, expect, it, vi } from "vitest";
+import { FfiStreamingRunner } from "./ffi-streaming-runner";
+import type {
+	LlmCtxHandle,
+	LlmStreamingBinding,
+} from "./llm-streaming-binding";
+import type { LlmStreamHandle } from "./voice/ffi-bindings";
+
+describe("FfiStreamingRunner prewarm", () => {
+	it("treats maxTokens: 0 as prefill-only and never calls next-token generation", async () => {
+		const stream = 7n as LlmStreamHandle;
+		const binding: LlmStreamingBinding = {
+			llmStreamSupported: () => true,
+			llmStreamOpen: vi.fn().mockReturnValue(stream),
+			llmStreamPrefill: vi.fn(),
+			llmStreamNext: vi.fn().mockReturnValue({
+				tokens: [1],
+				text: "x",
+				done: true,
+				drafterDrafted: 0,
+				drafterAccepted: 0,
+			}),
+			llmStreamCancel: vi.fn(),
+			llmStreamClose: vi.fn(),
+		};
+		const onTextChunk = vi.fn();
+		const runner = new FfiStreamingRunner(binding, 1n as LlmCtxHandle);
+		const promptTokens = new Int32Array([11, 12, 13]);
+
+		const result = await runner.generateWithUsage({
+			promptTokens,
+			slotId: 0,
+			maxTokens: 0,
+			temperature: 0,
+			topP: 1,
+			topK: 0,
+			repeatPenalty: 1,
+			draftMin: 0,
+			draftMax: 0,
+			draftModelPath: null,
+			onTextChunk,
+		});
+
+		expect(binding.llmStreamOpen).toHaveBeenCalledTimes(1);
+		expect(binding.llmStreamPrefill).toHaveBeenCalledWith({
+			stream,
+			tokens: promptTokens,
+		});
+		expect(binding.llmStreamNext).not.toHaveBeenCalled();
+		expect(onTextChunk).not.toHaveBeenCalled();
+		expect(binding.llmStreamClose).toHaveBeenCalledWith(stream);
+		expect(result).toEqual({
+			text: "",
+			slotId: 0,
+			firstTokenMs: null,
+			drafted: 0,
+			accepted: 0,
+		});
+	});
+});