@elizaos/plugin-local-inference 2.0.0-beta.1 → 2.0.3-beta.2

package/src/services/llama-server-metrics.ts

@@ -0,0 +1,304 @@
+/**
+ * Scraper for llama-server's `/metrics` (Prometheus exposition format)
+ * endpoint. Translates the running counters into the
+ * Anthropic-SDK-shaped `usage` block that callers already know how to
+ * consume from the cloud Anthropic plugin.
+ *
+ * llama-server publishes the following counters (per-process, monotonic):
+ *
+ *   llamacpp:n_decode_total           — context tokens decoded (prefill + gen)
+ *   llamacpp:n_tokens_predicted_total — output tokens
+ *   llamacpp:prompt_tokens_total      — total input tokens accepted
+ *   llamacpp:n_past_max               — high-water mark of cached past-tokens
+ *   llamacpp:n_prompt_tokens_processed_total — fresh tokens prefilled
+ *                                       (i.e. cache MISS), excludes cache hits
+ *   llamacpp:kv_cache_tokens          — current size of KV cache (gauge)
+ *   llamacpp:kv_cache_used_cells      — slots with active KV (gauge)
+ *
+ * For MTP speculative decoding, the fork additionally publishes:
+ *
+ *   llamacpp:n_drafted_total          — drafter-emitted tokens
+ *   llamacpp:n_drafted_accepted_total — accepted speculative tokens
+ *
+ * The mapping into Anthropic shape:
+ *
+ *   prompt_tokens_total                              → input_tokens
+ *   n_tokens_predicted_total                         → output_tokens
+ *   n_prompt_tokens_processed_total                  → cache_creation_input_tokens
+ *   prompt_tokens_total - n_prompt_tokens_processed_total → cache_read_input_tokens
+ *   n_drafted_total / n_drafted_accepted_total       → MTP extension fields
+ *
+ * Counters are taken as deltas across two snapshots: take one before
+ * `generate`, one after, and subtract. Losing a few samples to process
+ * restart is acceptable — the deltas are useful for the call's own
+ * usage accounting, not for global monitoring.
+ */
+
+export interface LlamaServerMetricSnapshot {
+	/** Wall-clock ms when the snapshot was taken; useful for diagnostics. */
+	takenAtMs: number;
+	/** True when `/metrics` was fetched and parsed. False means scrape failure. */
+	scrapeOk?: boolean;
+	/** True when the scrape included at least one generation/speculation counter. */
+	hasGenerationCounters?: boolean;
+	promptTokensTotal: number;
+	predictedTokensTotal: number;
+	/** Tokens that had to be freshly prefilled — i.e. cache MISS this turn. */
+	promptTokensProcessedTotal: number;
+	draftedTotal: number;
+	acceptedTotal: number;
+	/** Current size of the KV cache (gauge). */
+	kvCacheTokens: number;
+	/** Number of slots currently holding active KV (gauge). */
+	kvCacheUsedCells: number;
+}
+
+type MetricNumericField = Exclude<
+	keyof LlamaServerMetricSnapshot,
+	"scrapeOk" | "hasGenerationCounters"
+>;
+
+const METRIC_KEYS: Record<string, MetricNumericField> = {
+	"llamacpp:prompt_tokens_total": "promptTokensTotal",
+	"llamacpp:n_tokens_predicted_total": "predictedTokensTotal",
+	"llamacpp:n_prompt_tokens_processed_total": "promptTokensProcessedTotal",
+	"llamacpp:n_drafted_total": "draftedTotal",
+	"llamacpp:n_drafted": "draftedTotal",
+	"llamacpp:n_drafted_accepted_total": "acceptedTotal",
+	"llamacpp:n_drafted_accepted": "acceptedTotal",
+	"llamacpp:n_accepted_total": "acceptedTotal",
+	"llamacpp:n_accepted": "acceptedTotal",
+	"llamacpp:kv_cache_tokens": "kvCacheTokens",
+	"llamacpp:kv_cache_used_cells": "kvCacheUsedCells",
+};
+
+const DEFAULT_METRICS_SCRAPE_TIMEOUT_MS = 2_000;
+
+/**
+ * Parse a Prometheus exposition-format payload into a metric snapshot.
+ * Unknown or malformed lines are silently skipped — counters we don't
+ * recognise are not interesting and metric exporters add new ones over
+ * time.
+ *
+ * llama-server usually exposes one sample per metric (no labels), e.g.
+ *   `llamacpp:prompt_tokens_total 1234`
+ * Some MTP forks expose per-slot labelled samples, e.g.
+ *   `llamacpp:n_drafted_accepted_total{slot_id="0"} 12`
+ * Labelled samples are summed unless an unlabelled total exists for the same
+ * canonical field, in which case the unlabelled total wins.
+ */
+export function parsePrometheusMetrics(
+	body: string,
+	takenAtMs: number = Date.now(),
+): LlamaServerMetricSnapshot {
+	const snapshot: LlamaServerMetricSnapshot = {
+		takenAtMs,
+		scrapeOk: true,
+		hasGenerationCounters: false,
+		promptTokensTotal: 0,
+		predictedTokensTotal: 0,
+		promptTokensProcessedTotal: 0,
+		draftedTotal: 0,
+		acceptedTotal: 0,
+		kvCacheTokens: 0,
+		kvCacheUsedCells: 0,
+	};
+	const buckets = new Map<
+		MetricNumericField,
+		{ unlabeled: number | null; labeledSum: number }
+	>();
+	let hasGenerationCounters = false;
+
+	for (const rawLine of body.split(/\r?\n/)) {
+		const line = rawLine.trim();
+		if (!line || line.startsWith("#")) continue;
+		// Prometheus line format: `name{labels?} value [timestamp]`.
+		const match = line.match(
+			/^([a-zA-Z_:][\w:]*)(\{[^}]*\})?\s+([+-]?\d+(?:\.\d+)?(?:e[+-]?\d+)?)/i,
+		);
+		if (!match) continue;
+		const name = match[1];
+		const labels = match[2];
+		const value = Number(match[3]);
+		if (!Number.isFinite(value) || name === undefined) continue;
+		const field = METRIC_KEYS[name];
+		if (!field) continue;
+		if (
+			field === "promptTokensTotal" ||
+			field === "predictedTokensTotal" ||
+			field === "promptTokensProcessedTotal" ||
+			field === "draftedTotal" ||
+			field === "acceptedTotal"
+		) {
+			hasGenerationCounters = true;
+		}
+		const bucket = buckets.get(field) ?? { unlabeled: null, labeledSum: 0 };
+		if (labels) bucket.labeledSum += value;
+		else bucket.unlabeled = value;
+		buckets.set(field, bucket);
+	}
+
+	for (const [field, bucket] of buckets) {
+		snapshot[field] = bucket.unlabeled ?? bucket.labeledSum;
+	}
+
+	snapshot.hasGenerationCounters = hasGenerationCounters;
+
+	return snapshot;
+}
+
+/**
+ * Anthropic-SDK-shaped usage block, optionally extended with MTP
+ * speculative-decoding metrics. The cloud plugin (plugin-anthropic)
+ * emits the first three fields verbatim; local inference adds the
+ * `mtp_*` fields when speculative decoding is active. Callers that
+ * already handle the cloud `usage` shape need no change.
+ */
+export interface LocalUsageBlock {
+	[key: string]: unknown;
+	input_tokens: number;
+	output_tokens: number;
+	cache_creation_input_tokens: number;
+	cache_read_input_tokens: number;
+	mtp_drafted_tokens?: number;
+	mtp_accepted_tokens?: number;
+	/** 0..1 — proportion of drafted tokens that were accepted. */
+	mtp_acceptance_rate?: number;
+	/** 0..1 — proportion of input tokens that hit a warm slot (cache reuse). */
+	cache_hit_rate?: number;
+}
+
+/**
+ * Compute the Anthropic-shape usage block for a single generation by
+ * differencing two snapshots. `before` is taken just before the request,
+ * `after` just after the response was received. Negative deltas (caused
+ * by a metric reset between snapshots, e.g. server restart) are clamped
+ * to 0 — losing the sample is preferable to surfacing nonsense to the
+ * caller.
+ *
+ * Pass `responseUsage` to override input/output counts when the response
+ * payload itself reports per-call counters that are more accurate than
+ * the metric delta — llama-server's chat completion response includes
+ * `usage.{prompt,completion}_tokens` per request, which is exact while
+ * the metric delta is "everything that happened during the wall-clock
+ * window of the request."
+ */
+export function diffSnapshots(
+	before: LlamaServerMetricSnapshot,
+	after: LlamaServerMetricSnapshot,
+	responseUsage?: { prompt_tokens?: number; completion_tokens?: number },
+): LocalUsageBlock {
+	const promptDelta = clampNonNegative(
+		after.promptTokensTotal - before.promptTokensTotal,
+	);
+	const predictedDelta = clampNonNegative(
+		after.predictedTokensTotal - before.predictedTokensTotal,
+	);
+	const processedDelta = clampNonNegative(
+		after.promptTokensProcessedTotal - before.promptTokensProcessedTotal,
+	);
+	const draftedDelta = clampNonNegative(
+		after.draftedTotal - before.draftedTotal,
+	);
+	const acceptedDelta = clampNonNegative(
+		after.acceptedTotal - before.acceptedTotal,
+	);
+
+	const responsePrompt = responseUsage?.prompt_tokens ?? promptDelta;
+	const responseCompletion = responseUsage?.completion_tokens ?? predictedDelta;
+
+	const inputTokens = responsePrompt;
+	const outputTokens = responseCompletion;
+	// Tokens that had to be freshly prefilled this call. Bounded above by
+	// the per-call input count — a metric-delta wider than the call's own
+	// input is a sampling artifact.
+	const cacheCreation = Math.min(processedDelta, inputTokens);
+	const cacheRead = Math.max(0, inputTokens - cacheCreation);
+
+	const block: LocalUsageBlock = {
+		input_tokens: inputTokens,
+		output_tokens: outputTokens,
+		cache_creation_input_tokens: cacheCreation,
+		cache_read_input_tokens: cacheRead,
+	};
+	if (inputTokens > 0) {
+		block.cache_hit_rate = cacheRead / inputTokens;
+	}
+	if (draftedDelta > 0) {
+		block.mtp_drafted_tokens = draftedDelta;
+		block.mtp_accepted_tokens = acceptedDelta;
+		block.mtp_acceptance_rate = acceptedDelta / draftedDelta;
+	}
+	return block;
+}
+
+function clampNonNegative(value: number): number {
+	if (!Number.isFinite(value)) return 0;
+	return value < 0 ? 0 : value;
+}
+
+/**
+ * GET `/metrics` from a running llama-server and parse it. Errors fall
+ * back to a zero-valued snapshot rather than throwing — observability
+ * MUST NOT break generation. `scrapeOk=false` tells callers that the
+ * zeros are not evidence of absent MTP/KV activity.
+ */
+export async function fetchMetricsSnapshot(
+	baseUrl: string,
+	signal?: AbortSignal,
+	timeoutMs = DEFAULT_METRICS_SCRAPE_TIMEOUT_MS,
+): Promise<LlamaServerMetricSnapshot> {
+	const takenAtMs = Date.now();
+	const empty: LlamaServerMetricSnapshot = {
+		takenAtMs,
+		scrapeOk: false,
+		hasGenerationCounters: false,
+		promptTokensTotal: 0,
+		predictedTokensTotal: 0,
+		promptTokensProcessedTotal: 0,
+		draftedTotal: 0,
+		acceptedTotal: 0,
+		kvCacheTokens: 0,
+		kvCacheUsedCells: 0,
+	};
+	const controller = new AbortController();
+	const abortFromCaller = () => controller.abort(signal?.reason);
+	if (signal?.aborted) {
+		abortFromCaller();
+	} else {
+		signal?.addEventListener("abort", abortFromCaller, { once: true });
+	}
+	const timer = setTimeout(
+		() =>
+			controller.abort(
+				new DOMException(
+					`llama-server metrics scrape timed out after ${timeoutMs}ms`,
+					"TimeoutError",
+				),
+			),
+		Math.max(1, Math.floor(timeoutMs)),
+	);
+	let res: Response | null = null;
+	let bodySettled = false;
+	try {
+		res = await fetch(`${baseUrl.replace(/\/$/, "")}/metrics`, {
+			method: "GET",
+			signal: controller.signal,
+		});
+		if (!res.ok) return empty;
+		const body = await res.text();
+		bodySettled = true;
+		return parsePrometheusMetrics(body, takenAtMs);
+	} catch {
+		// Best effort: a metrics scrape failure must not abort the response
+		// path. Returning an empty snapshot causes diffSnapshots to surface
+		// zero deltas; the caller still sees the response payload usage.
+		return empty;
+	} finally {
+		clearTimeout(timer);
+		signal?.removeEventListener("abort", abortFromCaller);
+		if (res?.body && (!bodySettled || controller.signal.aborted)) {
+			await res.body.cancel(controller.signal.reason).catch(() => undefined);
+		}
+	}
+}

package/src/services/llm-streaming-binding.d.ts

@@ -0,0 +1,96 @@
+/**
+ * Narrow streaming-LLM binding.
+ *
+ * `FfiStreamingRunner` (`services/ffi-streaming-runner.ts`) used to require
+ * the full `ElizaInferenceFfi` surface (TTS + ASR + VAD + mmap regions +
+ * the entire fused libelizainference) just to run text generation. That
+ * surface implies a *bundle-anchored* runtime — libelizainference owns a
+ * context built from a bundle root, not a single GGUF — and ~25 methods
+ * that have nothing to do with LLM streaming.
+ *
+ * This file extracts the actual contract the runner depends on: the seven
+ * `llmStream*` methods plus the (optional) two slot save/restore methods.
+ * Both libelizainference (via a tiny adapter) and the desktop
+ * libllama + eliza-llama-shim path (built by `build-llama-cpp-desktop-dylib.mjs`,
+ * mirroring the AOSP adapter pattern) can implement this narrow contract
+ * without dragging in TTS/ASR.
+ *
+ * See `plugins/plugin-local-inference/FFI_BACKEND_WIREUP_PLAN.md` Step B
+ * for the desktop adapter follow-up that implements this interface against
+ * the libllama symbols.
+ */
+import type { ElizaInferenceContextHandle, ElizaInferenceFfi, LlmStreamConfig, LlmStreamHandle, LlmStreamStep } from "./voice/ffi-bindings";
+/**
+ * Opaque per-context handle. For libelizainference this is the
+ * `ElizaInferenceContextHandle` (a bigint pointer to the bundle context).
+ * For the desktop libllama path it's a bigint pointer to a per-model
+ * llama_context. The runner only needs to pass it through to
+ * `llmStreamOpen`.
+ */
+export type LlmCtxHandle = ElizaInferenceContextHandle;
+/**
+ * The streaming-LLM contract `FfiStreamingRunner` consumes. Methods
+ * mirror the C ABI declared in `tools/omnivoice/include/eliza-inference-ffi.h`
+ * (the `eliza_inference_llm_stream_*` surface), but the binding doesn't have
+ * to come from libelizainference — any implementation that satisfies this
+ * interface works.
+ *
+ * Slot save/restore are optional because the desktop libllama path
+ * does not expose `llama_state_seq_save_file` / `_load_file` through
+ * the shim yet. The runner already guards both methods via
+ * `if (this.ffi.llmStreamSaveSlot === undefined) throw ...`.
+ */
+export interface LlmStreamingBinding {
+    /** Probe — must return `true` for the binding to be usable by the runner. */
+    llmStreamSupported(): boolean;
+    /**
+     * Open a streaming-LLM session against `ctx`. Failure throws an
+     * implementation-specific error (`VoiceLifecycleError` for
+     * libelizainference). Close exactly once via `llmStreamClose`.
+     */
+    llmStreamOpen(args: {
+        ctx: LlmCtxHandle;
+        config: LlmStreamConfig;
+    }): LlmStreamHandle;
+    /** Feed a batch of pre-tokenized prompt tokens before the first `next`. */
+    llmStreamPrefill(args: {
+        stream: LlmStreamHandle;
+        tokens: Int32Array;
+    }): void;
+    /**
+     * Pull the next streaming step. `step.done === true` is the final step.
+     * Implementations may bound the step by `maxTokensPerStep` /
+     * `maxTextBytes`; defaults are runner-side.
+     */
+    llmStreamNext(args: {
+        stream: LlmStreamHandle;
+        maxTokensPerStep?: number;
+        maxTextBytes?: number;
+    }): LlmStreamStep;
+    /** Cancel in-flight generation; the next `_next` returns CANCELLED. */
+    llmStreamCancel(stream: LlmStreamHandle): void;
+    /** Close + free a streaming-LLM session. Idempotent on already-closed handles. */
+    llmStreamClose(stream: LlmStreamHandle): void;
+    /** Optional — persist the session's slot KV state to disk. */
+    llmStreamSaveSlot?(args: {
+        stream: LlmStreamHandle;
+        filename: string;
+    }): void;
+    /** Optional — restore a previously-saved slot KV file. */
+    llmStreamRestoreSlot?(args: {
+        stream: LlmStreamHandle;
+        filename: string;
+    }): void;
+}
+/**
+ * Wrap a full `ElizaInferenceFfi` as a narrow `LlmStreamingBinding`.
+ * The libelizainference path already implements the `llmStream*` methods
+ * as optional properties; this adapter promotes them to required and
+ * throws if the loaded library is too old to expose them.
+ *
+ * Usage:
+ *   const binding = wrapElizaInferenceFfi(ffi);
+ *   const runner = new FfiStreamingRunner(binding, ctxHandle);
+ */
+export declare function wrapElizaInferenceFfi(ffi: ElizaInferenceFfi): LlmStreamingBinding;
+//# sourceMappingURL=llm-streaming-binding.d.ts.map

package/src/services/llm-streaming-binding.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"llm-streaming-binding.d.ts","sourceRoot":"","sources":["llm-streaming-binding.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;GAoBG;AAEH,OAAO,KAAK,EACX,2BAA2B,EAC3B,iBAAiB,EACjB,eAAe,EACf,eAAe,EACf,aAAa,EACb,MAAM,sBAAsB,CAAC;AAE9B;;;;;;GAMG;AACH,MAAM,MAAM,YAAY,GAAG,2BAA2B,CAAC;AAEvD;;;;;;;;;;;GAWG;AACH,MAAM,WAAW,mBAAmB;IACnC,6EAA6E;IAC7E,kBAAkB,IAAI,OAAO,CAAC;IAC9B;;;;OAIG;IACH,aAAa,CAAC,IAAI,EAAE;QACnB,GAAG,EAAE,YAAY,CAAC;QAClB,MAAM,EAAE,eAAe,CAAC;KACxB,GAAG,eAAe,CAAC;IACpB,2EAA2E;IAC3E,gBAAgB,CAAC,IAAI,EAAE;QAAE,MAAM,EAAE,eAAe,CAAC;QAAC,MAAM,EAAE,UAAU,CAAA;KAAE,GAAG,IAAI,CAAC;IAC9E;;;;OAIG;IACH,aAAa,CAAC,IAAI,EAAE;QACnB,MAAM,EAAE,eAAe,CAAC;QACxB,gBAAgB,CAAC,EAAE,MAAM,CAAC;QAC1B,YAAY,CAAC,EAAE,MAAM,CAAC;KACtB,GAAG,aAAa,CAAC;IAClB,uEAAuE;IACvE,eAAe,CAAC,MAAM,EAAE,eAAe,GAAG,IAAI,CAAC;IAC/C,kFAAkF;IAClF,cAAc,CAAC,MAAM,EAAE,eAAe,GAAG,IAAI,CAAC;IAC9C,8DAA8D;IAC9D,iBAAiB,CAAC,CAAC,IAAI,EAAE;QAAE,MAAM,EAAE,eAAe,CAAC;QAAC,QAAQ,EAAE,MAAM,CAAA;KAAE,GAAG,IAAI,CAAC;IAC9E,0DAA0D;IAC1D,oBAAoB,CAAC,CAAC,IAAI,EAAE;QAC3B,MAAM,EAAE,eAAe,CAAC;QACxB,QAAQ,EAAE,MAAM,CAAC;KACjB,GAAG,IAAI,CAAC;CACT;AAED;;;;;;;;;GASG;AACH,wBAAgB,qBAAqB,CACpC,GAAG,EAAE,iBAAiB,GACpB,mBAAmB,CAmCrB"}

package/src/services/llm-streaming-binding.ts

@@ -0,0 +1,136 @@
+/**
+ * Narrow streaming-LLM binding.
+ *
+ * `FfiStreamingRunner` (`services/ffi-streaming-runner.ts`) used to require
+ * the full `ElizaInferenceFfi` surface (TTS + ASR + VAD + mmap regions +
+ * the entire fused libelizainference) just to run text generation. That
+ * surface implies a *bundle-anchored* runtime — libelizainference owns a
+ * context built from a bundle root, not a single GGUF — and ~25 methods
+ * that have nothing to do with LLM streaming.
+ *
+ * This file extracts the actual contract the runner depends on: the seven
+ * `llmStream*` methods plus the (optional) two slot save/restore methods.
+ * Both libelizainference (via a tiny adapter) and the desktop
+ * libllama + eliza-llama-shim path (built by `build-llama-cpp-desktop-dylib.mjs`,
+ * mirroring the AOSP adapter pattern) can implement this narrow contract
+ * without dragging in TTS/ASR.
+ *
+ * See `plugins/plugin-local-inference/FFI_BACKEND_WIREUP_PLAN.md` Step B
+ * for the desktop adapter follow-up that implements this interface against
+ * the libllama symbols.
+ */
+
+import type {
+	ElizaInferenceContextHandle,
+	ElizaInferenceFfi,
+	LlmStreamConfig,
+	LlmStreamHandle,
+	LlmStreamStep,
+} from "./voice/ffi-bindings";
+
+/**
+ * Opaque per-context handle. For libelizainference this is the
+ * `ElizaInferenceContextHandle` (a bigint pointer to the bundle context).
+ * For the desktop libllama path it's a bigint pointer to a per-model
+ * llama_context. The runner only needs to pass it through to
+ * `llmStreamOpen`.
+ */
+export type LlmCtxHandle = ElizaInferenceContextHandle;
+
+/**
+ * The streaming-LLM contract `FfiStreamingRunner` consumes. Methods
+ * mirror the C ABI declared in `tools/omnivoice/include/eliza-inference-ffi.h`
+ * (the `eliza_inference_llm_stream_*` surface), but the binding doesn't have
+ * to come from libelizainference — any implementation that satisfies this
+ * interface works.
+ *
+ * Slot save/restore are optional because the desktop libllama path
+ * does not expose `llama_state_seq_save_file` / `_load_file` through
+ * the shim yet. The runner already guards both methods via
+ * `if (this.ffi.llmStreamSaveSlot === undefined) throw ...`.
+ */
+export interface LlmStreamingBinding {
+	/** Probe — must return `true` for the binding to be usable by the runner. */
+	llmStreamSupported(): boolean;
+	/**
+	 * Open a streaming-LLM session against `ctx`. Failure throws an
+	 * implementation-specific error (`VoiceLifecycleError` for
+	 * libelizainference). Close exactly once via `llmStreamClose`.
+	 */
+	llmStreamOpen(args: {
+		ctx: LlmCtxHandle;
+		config: LlmStreamConfig;
+	}): LlmStreamHandle;
+	/** Feed a batch of pre-tokenized prompt tokens before the first `next`. */
+	llmStreamPrefill(args: { stream: LlmStreamHandle; tokens: Int32Array }): void;
+	/**
+	 * Pull the next streaming step. `step.done === true` is the final step.
+	 * Implementations may bound the step by `maxTokensPerStep` /
+	 * `maxTextBytes`; defaults are runner-side.
+	 */
+	llmStreamNext(args: {
+		stream: LlmStreamHandle;
+		maxTokensPerStep?: number;
+		maxTextBytes?: number;
+	}): LlmStreamStep;
+	/** Cancel in-flight generation; the next `_next` returns CANCELLED. */
+	llmStreamCancel(stream: LlmStreamHandle): void;
+	/** Close + free a streaming-LLM session. Idempotent on already-closed handles. */
+	llmStreamClose(stream: LlmStreamHandle): void;
+	/** Optional — persist the session's slot KV state to disk. */
+	llmStreamSaveSlot?(args: { stream: LlmStreamHandle; filename: string }): void;
+	/** Optional — restore a previously-saved slot KV file. */
+	llmStreamRestoreSlot?(args: {
+		stream: LlmStreamHandle;
+		filename: string;
+	}): void;
+}
+
+/**
+ * Wrap a full `ElizaInferenceFfi` as a narrow `LlmStreamingBinding`.
+ * The libelizainference path already implements the `llmStream*` methods
+ * as optional properties; this adapter promotes them to required and
+ * throws if the loaded library is too old to expose them.
+ *
+ * Usage:
+ *   const binding = wrapElizaInferenceFfi(ffi);
+ *   const runner = new FfiStreamingRunner(binding, ctxHandle);
+ */
+export function wrapElizaInferenceFfi(
+	ffi: ElizaInferenceFfi,
+): LlmStreamingBinding {
+	if (
+		typeof ffi.llmStreamSupported !== "function" ||
+		!ffi.llmStreamSupported() ||
+		typeof ffi.llmStreamOpen !== "function" ||
+		typeof ffi.llmStreamPrefill !== "function" ||
+		typeof ffi.llmStreamNext !== "function" ||
+		typeof ffi.llmStreamCancel !== "function" ||
+		typeof ffi.llmStreamClose !== "function"
+	) {
+		throw new Error(
+			"[llm-streaming-binding] The loaded libelizainference does not expose " +
+				"the streaming-LLM symbol set (llmStreamSupported/Open/Prefill/Next/" +
+				"Cancel/Close). Rebuild the omnivoice fuse against the current " +
+				"eliza-inference-ffi.h (verify-fused-symbols requires this set).",
+		);
+	}
+	// Narrowed function references so the returned object types are
+	// non-optional even though the source surface declares them
+	// optional.
+	const open = ffi.llmStreamOpen;
+	const prefill = ffi.llmStreamPrefill;
+	const next = ffi.llmStreamNext;
+	const cancel = ffi.llmStreamCancel;
+	const close = ffi.llmStreamClose;
+	return {
+		llmStreamSupported: () => true,
+		llmStreamOpen: open,
+		llmStreamPrefill: prefill,
+		llmStreamNext: next,
+		llmStreamCancel: cancel,
+		llmStreamClose: close,
+		llmStreamSaveSlot: ffi.llmStreamSaveSlot,
+		llmStreamRestoreSlot: ffi.llmStreamRestoreSlot,
+	};
+}

package/src/services/load-args.d.ts

@@ -0,0 +1,82 @@
+/**
+ * Per-load override types for the local inference engine.
+ *
+ * Extracted from active-model.ts to break the active-model ↔ engine
+ * circular dependency. Both modules import from here; neither imports
+ * from the other for these definitions.
+ *
+ * @module services/load-args
+ */
+/**
+ * KV cache placement strategy. `capacitor-llama` does not currently expose a
+ * direct KV-cache placement knob distinct from the model-level `gpuLayers`
+ * setting (the KV cache lives wherever the layer that owns it lives). We
+ * keep the type here so the API/UI surface and the upstream out-of-process
+ * `llama-server` backend can plumb a real choice through; the in-process
+ * binding maps any non-default value to a `gpuLayers` override or warns
+ * loudly when the value cannot be honoured.
+ */
+export type KvOffloadMode = "cpu" | "gpu" | "split" | {
+    gpuLayers: number;
+};
+/**
+ * Per-load overrides accepted by `localInferenceLoader.loadModel(...)` and
+ * `POST /api/local-inference/active`. Catalog defaults are merged in
+ * `resolveLocalInferenceLoadArgs`; per-call overrides supplied by the
+ * caller win over both catalog metadata and env-var fallbacks.
+ */
+export interface LocalInferenceLoadArgs {
+    modelPath: string;
+    /**
+     * Catalog id for direct bundle loads where `modelPath` points at a GGUF
+     * inside an Eliza-1 bundle that is not present in the installed-model
+     * registry yet.
+     */
+    modelId?: string;
+    contextSize?: number;
+    useGpu?: boolean;
+    maxThreads?: number;
+    draftModelPath?: string;
+    draftContextSize?: number;
+    draftMin?: number;
+    draftMax?: number;
+    speculativeSamples?: number;
+    mobileSpeculative?: boolean;
+    cacheTypeK?: string;
+    cacheTypeV?: string;
+    disableThinking?: boolean;
+    /**
+     * Number of model layers to offload to the GPU. `"auto"` and `"max"` are
+     * resolved by the backend's own probing — keep the explicit number type
+     * here so the API surface accepts the most common `gpuLayers: 32` shape
+     * without an extra string branch.
+     */
+    gpuLayers?: number;
+    /**
+     * Where to place the KV cache. See `KvOffloadMode`. node-llama-cpp does
+     * not expose this distinct from `gpuLayers`; the backend translates
+     * the request to a `gpuLayers` override or throws when the value
+     * cannot be honoured.
+     */
+    kvOffload?: KvOffloadMode;
+    flashAttention?: boolean;
+    mmap?: boolean;
+    mlock?: boolean;
+    /**
+     * Path to the multi-modal projector GGUF (mmproj-<tier>.gguf), when the
+     * loaded tier supports vision (`catalog.sourceModel.components.vision`
+     * is present AND the file exists on disk). WS2 (vision-describe)
+     * resolves this from the installed bundle root in
+     * `resolveLocalInferenceLoadArgs`. Backends that support vision use the
+     * path verbatim:
+     *   - llama-server: `--mmproj <path>` flag on spawn.
+     *   - node-llama-cpp: `mtmd_init_from_file(<path>)` (planned in fork).
+     *   - AOSP libllama shim: `eliza_llama_mtmd_init_from_file(<path>)`.
+     * Undefined when the tier doesn't ship vision or the file isn't on
+     * disk yet (e.g. downloaded text-only bundle). The text load is NOT
+     * gated on mmproj presence — text+drafter still load and vision is
+     * marked unavailable for that session.
+     */
+    mmprojPath?: string;
+}
+//# sourceMappingURL=load-args.d.ts.map

package/src/services/load-args.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"load-args.d.ts","sourceRoot":"","sources":["load-args.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAEH;;;;;;;;GAQG;AACH,MAAM,MAAM,aAAa,GAAG,KAAK,GAAG,KAAK,GAAG,OAAO,GAAG;IAAE,SAAS,EAAE,MAAM,CAAA;CAAE,CAAC;AAE5E;;;;;GAKG;AACH,MAAM,WAAW,sBAAsB;IACtC,SAAS,EAAE,MAAM,CAAC;IAClB;;;;OAIG;IACH,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAC5B,iBAAiB,CAAC,EAAE,OAAO,CAAC;IAC5B,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,eAAe,CAAC,EAAE,OAAO,CAAC;IAC1B;;;;;OAKG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB;;;;;OAKG;IACH,SAAS,CAAC,EAAE,aAAa,CAAC;IAC1B,cAAc,CAAC,EAAE,OAAO,CAAC;IACzB,IAAI,CAAC,EAAE,OAAO,CAAC;IACf,KAAK,CAAC,EAAE,OAAO,CAAC;IAChB;;;;;;;;;;;;;;OAcG;IACH,UAAU,CAAC,EAAE,MAAM,CAAC;CACpB"}