@elizaos/plugin-local-inference 2.0.0-beta.1 → 2.0.3-beta.3

package/src/adapters/capacitor-llama/types.ts

@@ -0,0 +1,374 @@
+/**
+ * Canonical Capacitor-llama adapter contract.
+ *
+ * This is the SAME shape used by `llama-cpp-capacitor` on mobile (iOS / Android,
+ * including riscv64 once Wave 2's libllama.so is installed) and by the bun:ffi
+ * backend on desktop. Every entry point in this plugin consumes one of these
+ * interfaces — there is no `node-llama-cpp` import anywhere downstream.
+ *
+ * Shape rationale: the interface mirrors `llama-cpp-capacitor`'s
+ * `LlamaContext` class verbatim so the mobile path can implement it with a
+ * straight pass-through. The loader is mobile-only — libllama is retired, so
+ * there is no desktop façade (desktop runs the fused `libelizainference`).
+ *
+ * Why a TS interface rather than `import type { LlamaContext } from
+ * "llama-cpp-capacitor"`? The Capacitor package is mobile-only at install time
+ * (Capacitor SPI'd Java/Swift bindings). Importing its types from the desktop
+ * path would force every desktop tsc run to pull the mobile binding's d.ts. We
+ * keep the interface free-standing so any backend can implement it.
+ */
+
+export interface TokenizerConfig {
+	name: string;
+	type: string;
+}
+
+// === Context / completion params ===========================================
+
+export interface CapacitorLlamaContextParams {
+	/** GGUF model path (absolute or Capacitor-asset URI). */
+	model: string;
+	/** Optional Jinja chat template override. */
+	chat_template?: string;
+	/** When loading from a packaged Android/iOS asset. */
+	is_model_asset?: boolean;
+	use_progress_callback?: boolean;
+	n_ctx?: number;
+	n_batch?: number;
+	n_ubatch?: number;
+	n_threads?: number;
+	/**
+	 * GPU layer offload. `999` = "all layers on GPU" per llama.cpp convention.
+	 * The Capacitor binding rejects > model layer count; desktop path clamps
+	 * to the model's metadata layer count.
+	 */
+	n_gpu_layers?: number;
+	no_gpu_devices?: boolean;
+	flash_attn?: boolean;
+	cache_type_k?:
+		| "f16"
+		| "f32"
+		| "q8_0"
+		| "q4_0"
+		| "q4_1"
+		| "iq4_nl"
+		| "q5_0"
+		| "q5_1";
+	cache_type_v?:
+		| "f16"
+		| "f32"
+		| "q8_0"
+		| "q4_0"
+		| "q4_1"
+		| "iq4_nl"
+		| "q5_0"
+		| "q5_1";
+	use_mlock?: boolean;
+	use_mmap?: boolean;
+	vocab_only?: boolean;
+	lora?: string;
+	lora_scaled?: number;
+	lora_list?: Array<{ path: string; scaled?: number }>;
+	rope_freq_base?: number;
+	rope_freq_scale?: number;
+	pooling_type?: "none" | "mean" | "cls" | "last" | "rank";
+	ctx_shift?: boolean;
+	kv_unified?: boolean;
+	swa_full?: boolean;
+	n_cpu_moe?: number;
+	embedding?: boolean;
+	embd_normalize?: number;
+}
+
+export interface CapacitorLlamaMessagePart {
+	type: string;
+	text?: string;
+	image_url?: { url?: string };
+	input_audio?: { format: string; data?: string; url?: string };
+}
+
+export interface CapacitorLlamaChatMessage {
+	role: string;
+	content?: string | CapacitorLlamaMessagePart[];
+}
+
+export interface CapacitorLlamaResponseFormat {
+	type: "text" | "json_object" | "json_schema";
+	json_schema?: { strict?: boolean; schema: object };
+	schema?: object;
+}
+
+export interface CapacitorLlamaCompletionParams {
+	/** Raw prompt string. Mutually exclusive with `messages`. */
+	prompt?: string;
+	/** OpenAI-style messages; the binding renders the chat template. */
+	messages?: CapacitorLlamaChatMessage[];
+	chat_template?: string;
+	jinja?: boolean;
+	json_schema?: string;
+	/** GBNF grammar source. */
+	grammar?: string;
+	grammar_lazy?: boolean;
+	grammar_triggers?: Array<{ type: number; value: string; token: number }>;
+	enable_thinking?: boolean;
+	thinking_forced_open?: boolean;
+	preserved_tokens?: string[];
+	chat_format?: number;
+	reasoning_format?: string;
+	media_paths?: string[];
+	stop?: string[];
+	n_predict?: number;
+	n_probs?: number;
+	top_k?: number;
+	top_p?: number;
+	min_p?: number;
+	xtc_probability?: number;
+	xtc_threshold?: number;
+	typical_p?: number;
+	temperature?: number;
+	penalty_last_n?: number;
+	penalty_repeat?: number;
+	penalty_freq?: number;
+	penalty_present?: number;
+	mirostat?: number;
+	mirostat_tau?: number;
+	mirostat_eta?: number;
+	dry_multiplier?: number;
+	dry_base?: number;
+	dry_allowed_length?: number;
+	dry_penalty_last_n?: number;
+	dry_sequence_breakers?: string[];
+	top_n_sigma?: number;
+	ignore_eos?: boolean;
+	logit_bias?: Array<[number, number]>;
+	seed?: number;
+	guide_tokens?: number[];
+	tools?: object;
+	parallel_tool_calls?: object;
+	tool_choice?: string;
+	response_format?: CapacitorLlamaResponseFormat;
+	add_generation_prompt?: boolean;
+	chat_template_kwargs?: Record<string, string>;
+	prefill_text?: string;
+}
+
+export interface CapacitorLlamaToolCall {
+	type: "function";
+	id?: string;
+	function: { name: string; arguments: string };
+}
+
+export interface CapacitorLlamaTokenData {
+	token: string;
+	completion_probabilities?: Array<{
+		content: string;
+		probs: Array<{ tok_str: string; prob: number }>;
+	}>;
+	content?: string;
+	reasoning_content?: string;
+	tool_calls?: CapacitorLlamaToolCall[];
+	accumulated_text?: string;
+}
+
+export interface CapacitorLlamaCompletionResult {
+	text: string;
+	reasoning_content: string;
+	tool_calls: CapacitorLlamaToolCall[];
+	content: string;
+	chat_format: number;
+	tokens_predicted: number;
+	tokens_evaluated: number;
+	truncated: boolean;
+	stopped_eos: boolean;
+	stopped_word: string;
+	stopped_limit: number;
+	stopping_word: string;
+	context_full: boolean;
+	interrupted: boolean;
+	tokens_cached: number;
+	timings: {
+		prompt_n: number;
+		prompt_ms: number;
+		prompt_per_token_ms: number;
+		prompt_per_second: number;
+		predicted_n: number;
+		predicted_ms: number;
+		predicted_per_token_ms: number;
+		predicted_per_second: number;
+	};
+	completion_probabilities?: Array<{
+		content: string;
+		probs: Array<{ tok_str: string; prob: number }>;
+	}>;
+	audio_tokens?: number[];
+}
+
+export interface CapacitorLlamaTokenizeResult {
+	tokens: number[];
+	has_images: boolean;
+	bitmap_hashes: number[];
+	chunk_pos: number[];
+	chunk_pos_images: number[];
+}
+
+export interface CapacitorLlamaEmbeddingResult {
+	embedding: number[];
+}
+
+export interface CapacitorLlamaBenchResult {
+	modelDesc: string;
+	modelSize: number;
+	modelNParams: number;
+	ppAvg: number;
+	ppStd: number;
+	tgAvg: number;
+	tgStd: number;
+}
+
+// === Model description (mirrors NativeLlamaContext['model'])  ==============
+
+export interface CapacitorLlamaModelDescriptor {
+	desc: string;
+	size: number;
+	nEmbd: number;
+	nParams: number;
+	chatTemplates: {
+		llamaChat: boolean;
+		minja: {
+			default: boolean;
+			defaultCaps: {
+				tools: boolean;
+				toolCalls: boolean;
+				toolResponses: boolean;
+				systemRole: boolean;
+				parallelToolCalls: boolean;
+				toolCallId: boolean;
+			};
+			toolUse: boolean;
+			toolUseCaps: {
+				tools: boolean;
+				toolCalls: boolean;
+				toolResponses: boolean;
+				systemRole: boolean;
+				parallelToolCalls: boolean;
+				toolCallId: boolean;
+			};
+		};
+	};
+	metadata: object;
+	isChatTemplateSupported: boolean;
+}
+
+// === Canonical context interface ===========================================
+
+/**
+ * The canonical context handle every backend implements. Designed so a single
+ * caller can switch between `llama-cpp-capacitor`'s `LlamaContext` and the
+ * desktop bun:ffi-backed implementation with zero behavioural change at the
+ * TypeScript level.
+ *
+ * Backends that genuinely cannot implement a method (e.g. `bench()` on the
+ * desktop FFI adapter v1) MUST throw a `CapacitorLlamaUnsupportedError` so
+ * callers can route around the gap explicitly rather than silently producing
+ * wrong results.
+ */
+export interface CapacitorLlamaContext {
+	readonly id: number;
+	readonly gpu: boolean;
+	readonly reasonNoGPU: string;
+	readonly model: CapacitorLlamaModelDescriptor;
+
+	completion(
+		params: CapacitorLlamaCompletionParams,
+		callback?: (data: CapacitorLlamaTokenData) => void,
+	): Promise<CapacitorLlamaCompletionResult>;
+
+	stopCompletion(): Promise<void>;
+
+	tokenize(
+		text: string,
+		options?: { media_paths?: string[] },
+	): Promise<CapacitorLlamaTokenizeResult>;
+
+	detokenize(tokens: number[]): Promise<string>;
+
+	embedding(
+		text: string,
+		params?: { embd_normalize?: number },
+	): Promise<CapacitorLlamaEmbeddingResult>;
+
+	bench(
+		pp: number,
+		tg: number,
+		pl: number,
+		nr: number,
+	): Promise<CapacitorLlamaBenchResult>;
+
+	release(): Promise<void>;
+}
+
+/** Thrown by adapter methods that aren't implemented for a given backend. */
+export class CapacitorLlamaUnsupportedError extends Error {
+	constructor(
+		readonly method: string,
+		readonly backend: "mobile" | "desktop-ffi",
+		message?: string,
+	) {
+		super(
+			message ??
+				`[capacitor-llama] ${method} is not supported on ${backend} yet`,
+		);
+		this.name = "CapacitorLlamaUnsupportedError";
+	}
+}
+
+// === Model registry (re-exported from legacy types.ts) =====================
+
+export interface ModelSpec {
+	name: string;
+	repo: string;
+	size: string;
+	quantization: string;
+	contextSize: number;
+	tokenizer: TokenizerConfig;
+}
+
+export interface EmbeddingModelSpec extends ModelSpec {
+	dimensions: number;
+}
+
+/**
+ * Default model bundle. Vision and TTS are owned by `plugin-local-inference`'s
+ * voice and vision subsystems — they don't live on this adapter.
+ */
+export const MODEL_SPECS = {
+	small: {
+		name: "text/eliza-1-2b-128k.gguf",
+		repo: "elizaos/eliza-1",
+		size: "2B",
+		quantization: "fused GGUF",
+		contextSize: 131072,
+		tokenizer: { name: "elizaos/eliza-1", type: "eliza1" },
+	},
+	medium: {
+		name: "text/eliza-1-4b-128k.gguf",
+		repo: "elizaos/eliza-1",
+		size: "4B",
+		quantization: "fused GGUF",
+		contextSize: 131072,
+		tokenizer: { name: "elizaos/eliza-1", type: "eliza1" },
+	},
+	embedding: {
+		name: "gte-small_fp16.gguf",
+		repo: "ChristianAzinn/gte-small-gguf",
+		size: "64 MB",
+		quantization: "fp16 GGUF",
+		contextSize: 512,
+		dimensions: 384,
+		tokenizer: { name: "ChristianAzinn/gte-small-gguf", type: "bert" },
+	},
+} as const satisfies {
+	small: ModelSpec;
+	medium: ModelSpec;
+	embedding: EmbeddingModelSpec;
+};

package/src/adapters/capacitor-llama/voice-turn.ts

@@ -0,0 +1,178 @@
+/**
+ * On-device fused voice-turn entry for the Capacitor / AOSP device bridge.
+ *
+ * This is the production caller for `LocalInferenceEngine.runVoiceTurn` (#8786):
+ * the in-process fused mic→speech loop (ASR → {MTP drafts ∥ target verify} →
+ * phrase chunker → OmniVoice/Kokoro → PCM ring buffer) that the engine's voice
+ * bridge owns. The engine bridge serves ASR + TTS from the fused
+ * `libelizainference` handle; the DEVICE supplies its own text runner so the
+ * draft/verify loop runs on the on-device text model (the
+ * `CapacitorLlamaContext`-backed decode) rather than the desktop dispatcher —
+ * exactly the seam `runVoiceTurn`'s `textRunner` option was built for.
+ *
+ * The native iOS/Android mic-capture layer hands captured PCM to
+ * {@link runDeviceVoiceTurn}; everything downstream is in-process JS + the
+ * fused FFI. No HTTP, no second voice path.
+ */
+
+import type { GenerateArgs } from "../../services/backend";
+import type { VoicePipelineEvents } from "../../services/voice/pipeline";
+import type { MtpTextRunner } from "../../services/voice/pipeline-impls";
+import type {
+	TranscriptionAudio,
+	VerifierStreamEvent,
+} from "../../services/voice/types";
+import type { VoicePreloadPredictor } from "../../services/voice/voice-preload-predictor";
+import type { CapacitorLlamaContext } from "./types";
+
+/** Turn exit reason, mirroring `LocalInferenceEngine.runVoiceTurn`. */
+export type VoiceTurnExitReason = "done" | "token-cap" | "cancelled";
+
+/**
+ * Structural view of `LocalInferenceEngine.runVoiceTurn` so the device bridge
+ * (and tests) depend on the call shape, not the concrete engine class.
+ */
+export interface DeviceVoiceEngine {
+	runVoiceTurn(
+		audio: TranscriptionAudio,
+		opts?: {
+			maxDraftTokens?: number;
+			maxGeneratedTokens?: number;
+			events?: VoicePipelineEvents;
+			textRunner?: MtpTextRunner;
+		},
+	): Promise<VoiceTurnExitReason>;
+}
+
+export interface CapacitorTextRunnerOptions {
+	/** Default sampling temperature when the pipeline does not pin one. */
+	temperature?: number;
+	/** Default nucleus-sampling cutoff when the pipeline does not pin one. */
+	topP?: number;
+}
+
+/**
+ * Adapt a live `CapacitorLlamaContext` to the {@link MtpTextRunner} contract the
+ * fused voice pipeline drives.
+ *
+ * `hasDrafter()` is `false`: the on-device `CapacitorLlamaContext` does not yet
+ * expose a separate speculative-draft window, so the pipeline runs plain decode
+ * through the target verifier (`MtpTargetVerifier` splits the returned text into
+ * tokens when no per-delta verifier events arrive — its documented non-streaming
+ * path). When the on-device fork exposes draft/verify events this flips to
+ * `true` with no change to the call sites.
+ */
+export function createCapacitorMtpTextRunner(
+	context: CapacitorLlamaContext,
+	options: CapacitorTextRunnerOptions = {},
+): MtpTextRunner {
+	return {
+		hasDrafter() {
+			return false;
+		},
+		async generateWithVerifierEvents(
+			args: GenerateArgs & {
+				onVerifierEvent: (event: VerifierStreamEvent) => void | Promise<void>;
+			},
+		): Promise<{ text: string }> {
+			const { signal } = args;
+			if (signal?.aborted) return { text: "" };
+
+			// A barge-in / kill-switch aborts the in-flight on-device decode.
+			let onAbort: (() => void) | undefined;
+			if (signal) {
+				onAbort = () => {
+					void context.stopCompletion();
+				};
+				signal.addEventListener("abort", onAbort, { once: true });
+			}
+
+			try {
+				const result = await context.completion({
+					prompt: args.prompt,
+					n_predict: args.maxTokens,
+					temperature: args.temperature ?? options.temperature,
+					top_p: args.topP ?? options.topP,
+					...(args.stopSequences ? { stop: args.stopSequences } : {}),
+				});
+				return { text: result.text };
+			} finally {
+				if (signal && onAbort) signal.removeEventListener("abort", onAbort);
+			}
+		},
+	};
+}
+
+export interface RunDeviceVoiceTurnArgs {
+	/** The engine that owns the armed voice bridge (`startVoice` + `armVoice`). */
+	engine: DeviceVoiceEngine;
+	/** The on-device text model context that backs the MTP target verifier. */
+	context: CapacitorLlamaContext;
+	/** Captured mic PCM for this turn. */
+	audio: TranscriptionAudio;
+	/** Pipeline events (first-audio, transcript, completion, cancel). */
+	events?: VoicePipelineEvents;
+	/** Upper bound on generated response tokens. */
+	maxGeneratedTokens?: number;
+	/** Default sampling overrides forwarded to the on-device text runner. */
+	generation?: CapacitorTextRunnerOptions;
+	/**
+	 * Next-stage preload predictor (#8809 C5). On the mobile / Capacitor path the
+	 * text-response model is owned by the {@link MemoryArbiter}; this turn's
+	 * stages run sequentially (ASR → text → TTS), so the predictor warms the
+	 * arbiter-managed text model the instant ASR finishes — during the ASR-stage
+	 * page-trim window — collapsing the cold load off the post-ASR critical path.
+	 * Wired to the pipeline's `onAsrComplete` event below. Omit to keep the turn
+	 * unpredicted (the arbiter still loads the text model on first request).
+	 */
+	preloadPredictor?: VoicePreloadPredictor;
+}
+
+/**
+ * Run one fused on-device voice turn. The native capture layer calls this with
+ * the captured mic audio; it builds the device text runner from the loaded
+ * `CapacitorLlamaContext` and drives `engine.runVoiceTurn`, so ASR + MTP +
+ * TTS all run in-process through the fused handle. Resolves with the turn's
+ * exit reason.
+ */
+export function runDeviceVoiceTurn(
+	args: RunDeviceVoiceTurnArgs,
+): Promise<VoiceTurnExitReason> {
+	const textRunner = createCapacitorMtpTextRunner(
+		args.context,
+		args.generation ?? {},
+	);
+	const events = composeAsrCompletePreload(args.events, args.preloadPredictor);
+	return args.engine.runVoiceTurn(args.audio, {
+		textRunner,
+		...(events ? { events } : {}),
+		...(args.maxGeneratedTokens !== undefined
+			? { maxGeneratedTokens: args.maxGeneratedTokens }
+			: {}),
+	});
+}
+
+/**
+ * Compose the caller's pipeline events with the next-stage preload prediction.
+ *
+ * When a predictor is supplied, its `onAsrStageComplete()` is invoked from the
+ * pipeline's `onAsrComplete` hook — the instant ASR emits its final token — so
+ * the arbiter-managed text model is warmed before the verifier issues its first
+ * request. The prediction is fire-and-forget: it must never delay the
+ * drafter/verifier kick-off, and a declined preload (pressure / no headroom) is
+ * a no-op, not an error. The caller's own `onAsrComplete` (if any) still runs.
+ */
+function composeAsrCompletePreload(
+	events: VoicePipelineEvents | undefined,
+	predictor: VoicePreloadPredictor | undefined,
+): VoicePipelineEvents | undefined {
+	if (!predictor) return events;
+	const callerOnAsrComplete = events?.onAsrComplete;
+	return {
+		...events,
+		onAsrComplete(tokens) {
+			callerOnAsrComplete?.(tokens);
+			void predictor.onAsrStageComplete();
+		},
+	};
+}

package/src/backends/apple-foundation.ts

@@ -0,0 +1,127 @@
+/**
+ * Apple Foundation Models adapter — iOS / iPadOS / macOS 26+.
+ *
+ * The Foundation Models framework exposes a managed on-device LLM that
+ * Apple ships with the OS when Apple Intelligence is enabled. We do not
+ * own its weights and cannot ship our own; this adapter is an
+ * **opportunistic fast-path**:
+ *
+ *   - If the device is iOS 26+ and reports availability via the Capacitor
+ *     `ComputerUse` bridge probe, we register a generate function that
+ *     calls into Apple's framework for short prompts.
+ *   - Otherwise we do nothing, and the existing llama-cpp-capacitor
+ *     (Gemma-VL / eliza-1 2b) path remains the active local-inference handler.
+ *
+ * Why this lives in plugin-local-inference (not plugin-capacitor-bridge):
+ *
+ *   - The plugin-capacitor-bridge package is a thin JS↔native shim. The
+ *     decision of *when* to use Apple's model vs. llama.cpp is an
+ *     inference-routing decision, which is plugin-local-inference's job.
+ *     Co-locating the adapter with the rest of the routing logic
+ *     (`backend-selector`, `runtime-dispatcher`, etc.) keeps the routing
+ *     surface inspectable in one place. The bridge only transports
+ *     bytes; it does not decide policy.
+ *
+ * Entitlement: requires `com.apple.developer.kernel.increased-memory-limit`
+ * and Apple Intelligence to be enabled by the user in Settings. See
+ * `docs/IOS_CONSTRAINTS.md`.
+ */
+
+import type {
+	FoundationModelOptions,
+	FoundationModelResult,
+	IosComputerUseBridge,
+} from "@elizaos/plugin-computeruse/mobile/ios-bridge";
+
+export interface AppleFoundationGenerateArgs {
+	readonly prompt: string;
+	readonly options?: FoundationModelOptions;
+}
+
+export interface AppleFoundationAdapter {
+	readonly name: "apple-foundation";
+	available(): boolean;
+	generate(args: AppleFoundationGenerateArgs): Promise<FoundationModelResult>;
+}
+
+/**
+ * Builds the adapter. The bridge getter is lazy so this module remains
+ * Node-importable for tests; in the Capacitor build the runtime resolves
+ * `Capacitor.Plugins.ComputerUse` and passes it in.
+ */
+export function createAppleFoundationAdapter(
+	getBridge: () => IosComputerUseBridge | null,
+): AppleFoundationAdapter {
+	let probedAvailable: boolean | null = null;
+	let probing: Promise<boolean> | null = null;
+
+	async function probe(): Promise<boolean> {
+		const bridge = getBridge();
+		if (!bridge) return false;
+		const result = await bridge.probe();
+		if (!result.ok) return false;
+		return result.data.capabilities.foundationModel === true;
+	}
+
+	return {
+		name: "apple-foundation",
+		available(): boolean {
+			if (probedAvailable !== null) return probedAvailable;
+			if (!probing) {
+				probing = probe().then((v) => {
+					probedAvailable = v;
+					return v;
+				});
+			}
+			// Synchronous semantics: return false until the probe resolves; this
+			// matches the rest of the local-inference availability pattern, which
+			// is allowed to return a slightly stale "unavailable" state until the
+			// first async probe completes.
+			return probedAvailable === true;
+		},
+		async generate(args): Promise<FoundationModelResult> {
+			const bridge = getBridge();
+			if (!bridge) {
+				throw new Error(
+					"apple-foundation adapter invoked but Capacitor ComputerUse plugin is not registered.",
+				);
+			}
+			const result = await bridge.foundationModelGenerate({
+				prompt: args.prompt,
+				...(args.options ? { options: args.options } : {}),
+			});
+			if (!result.ok) {
+				throw new Error(
+					`apple-foundation generate failed: ${result.code} — ${result.message}`,
+				);
+			}
+			return result.data;
+		},
+	};
+}
+
+// ── Registry ─────────────────────────────────────────────────────────────────
+
+let registered: AppleFoundationAdapter | null = null;
+
+/**
+ * Register the adapter so the local-inference runtime picks it up
+ * opportunistically. Idempotent — subsequent calls overwrite the previous
+ * registration so a hot reload of the bridge swaps cleanly.
+ */
+export function registerAppleFoundationAdapter(
+	adapter: AppleFoundationAdapter,
+): void {
+	registered = adapter;
+}
+
+export function getAppleFoundationAdapter(): AppleFoundationAdapter | null {
+	return registered;
+}
+
+/**
+ * Tests use this to ensure each spec starts from a known state.
+ */
+export function _resetAppleFoundationAdapterForTests(): void {
+	registered = null;
+}