@elizaos/plugin-local-inference 2.0.0-beta.1 → 2.0.11-beta.7

package/src/adapters/capacitor-llama/loader.ts

@@ -0,0 +1,109 @@
+/**
+ * Canonical Capacitor-llama loader — MOBILE ONLY.
+ *
+ * `initCapacitorLlama(params)` returns a context that implements
+ * `CapacitorLlamaContext` by dynamic-importing `llama-cpp-capacitor` on
+ * `ELIZA_PLATFORM=android|ios`. The package's `LlamaContext` already implements
+ * the canonical shape verbatim, so we return it directly with an `as` cast (the
+ * d.ts overlaps by construction).
+ *
+ * The Capacitor llama API is mobile-only. libllama has been retired, so there
+ * is no desktop façade: desktop / server inference runs through the fused
+ * `libelizainference` engine (`LocalInferenceEngine` /
+ * `desktopFusedFfiBackendRuntime`), not this loader. `initCapacitorLlama` on a
+ * non-mobile platform throws `CapacitorLlamaUnsupportedError`.
+ */
+
+import { logger } from "@elizaos/core";
+import {
+	type CapacitorLlamaContext,
+	type CapacitorLlamaContextParams,
+	CapacitorLlamaUnsupportedError,
+} from "./types";
+
+// === Mobile shim ===========================================================
+
+interface MobileCapacitorModule {
+	initLlama(
+		params: CapacitorLlamaContextParams,
+		onProgress?: (progress: number) => void,
+	): Promise<CapacitorLlamaContext>;
+	releaseAllLlama(): Promise<void>;
+	setContextLimit(limit: number): Promise<void>;
+	toggleNativeLog(enabled: boolean): Promise<void>;
+}
+
+let cachedMobileModule: MobileCapacitorModule | null = null;
+
+async function loadMobileCapacitor(): Promise<MobileCapacitorModule> {
+	if (cachedMobileModule) return cachedMobileModule;
+	// Dynamic import keeps the desktop / test runtime from trying to resolve
+	// the mobile-only native binding.
+	const spec = "llama-cpp-capacitor";
+	const mod = (await import(spec)) as MobileCapacitorModule;
+	if (typeof mod.initLlama !== "function") {
+		throw new Error(
+			"[capacitor-llama] llama-cpp-capacitor did not expose initLlama — the binding is missing or unavailable.",
+		);
+	}
+	cachedMobileModule = mod;
+	return mod;
+}
+
+// === Public loader =========================================================
+
+export interface InitCapacitorLlamaOptions extends CapacitorLlamaContextParams {
+	/**
+	 * Force a specific backend. Only `mobile` is supported — it requires the
+	 * `llama-cpp-capacitor` package. The desktop façade was removed when
+	 * libllama was retired; desktop inference runs through the fused engine.
+	 */
+	backend?: "mobile";
+}
+
+function detectBackend(env: NodeJS.ProcessEnv = process.env): "mobile" | null {
+	const platform = env.ELIZA_PLATFORM?.trim().toLowerCase();
+	if (platform === "android" || platform === "ios") return "mobile";
+	return null;
+}
+
+/**
+ * Load a Capacitor-shaped llama.cpp context. Mobile-only: on
+ * `ELIZA_PLATFORM=android|ios` it resolves `llama-cpp-capacitor`; on any other
+ * platform it throws `CapacitorLlamaUnsupportedError` (libllama is retired —
+ * desktop/server inference runs through the fused `libelizainference` engine,
+ * not this loader). Callers MUST handle the throw explicitly.
+ */
+export async function initCapacitorLlama(
+	opts: InitCapacitorLlamaOptions,
+): Promise<CapacitorLlamaContext> {
+	const { backend, ...params } = opts;
+	const target = backend ?? detectBackend();
+
+	if (target === "mobile") {
+		const mod = await loadMobileCapacitor();
+		return mod.initLlama(params);
+	}
+
+	throw new CapacitorLlamaUnsupportedError(
+		"initCapacitorLlama",
+		"desktop-ffi",
+		"[capacitor-llama] The Capacitor llama API is mobile-only. libllama has been " +
+			"retired; desktop/server inference runs through the fused libelizainference " +
+			"engine (LocalInferenceEngine / desktopFusedFfiBackendRuntime), not this loader.",
+	);
+}
+
+/** Mobile-only: release every context. No-op on desktop. */
+export async function releaseAllCapacitorLlama(): Promise<void> {
+	if (detectBackend() !== "mobile") return;
+	try {
+		const mod = await loadMobileCapacitor();
+		await mod.releaseAllLlama();
+	} catch (err) {
+		logger.debug(
+			{ err: err instanceof Error ? err.message : String(err) },
+			"[capacitor-llama] releaseAllLlama not available",
+		);
+	}
+}

package/src/adapters/capacitor-llama/structured-output.ts

@@ -0,0 +1,165 @@
+/**
+ * Structured-output planning on the Capacitor surface.
+ *
+ * llama.cpp itself supports GBNF grammars and JSON-Schema-as-grammar natively;
+ * the Capacitor binding exposes both via `CompletionParams.grammar` (raw GBNF
+ * source) and `CompletionParams.response_format` (JSON object or JSON schema).
+ *
+ * Function-calling is exposed differently than in node-llama-cpp: the
+ * Capacitor binding takes a raw `tools` object plus optional `tool_choice` /
+ * `parallel_tool_calls` and the model emits an OpenAI-shaped
+ * `tool_calls[]` array inside the completion result. We project this back
+ * into the elizaOS `ToolCallResult` shape so downstream callers do not care
+ * which backend produced the call.
+ */
+
+import type { JSONSchema, ToolDefinition } from "@elizaos/core";
+import type {
+	CapacitorLlamaCompletionParams,
+	CapacitorLlamaCompletionResult,
+	CapacitorLlamaToolCall,
+} from "./types";
+
+export interface ToolCallResult {
+	id: string;
+	name: string;
+	arguments: Record<string, unknown>;
+	type: "function";
+}
+
+export interface StructuredRequestPlan {
+	kind: "text" | "tools" | "schema" | "json_object";
+	/** Forwarded to `CapacitorLlamaCompletionParams.tools`. */
+	tools?: object;
+	/** Forwarded to `CapacitorLlamaCompletionParams.response_format`. */
+	responseFormat?: CapacitorLlamaCompletionParams["response_format"];
+	/** Optional `tool_choice` directive. */
+	toolChoice?: string;
+}
+
+/**
+ * Build the OpenAI-style `tools` object the Capacitor binding accepts. Each
+ * elizaOS `ToolDefinition` becomes one entry of:
+ *   { type: "function", function: { name, description, parameters } }
+ */
+export function buildCapacitorTools(tools: readonly ToolDefinition[]): object {
+	const out: Array<{
+		type: "function";
+		function: {
+			name: string;
+			description?: string;
+			parameters?: JSONSchema;
+		};
+	}> = [];
+	for (const tool of tools) {
+		if (!tool.name) continue;
+		out.push({
+			type: "function",
+			function: {
+				name: tool.name,
+				...(tool.description ? { description: tool.description } : {}),
+				...(tool.parameters ? { parameters: tool.parameters } : {}),
+			},
+		});
+	}
+	return out;
+}
+
+/**
+ * Extract tool calls from the Capacitor completion result. The binding emits
+ * them on `result.tool_calls`; we parse `arguments` JSON and surface the
+ * elizaOS shape.
+ */
+export function extractToolCalls(
+	result: CapacitorLlamaCompletionResult,
+): ToolCallResult[] {
+	const calls: ToolCallResult[] = [];
+	let i = 0;
+	for (const call of result.tool_calls ?? []) {
+		const parsedArgs = parseToolArguments(call);
+		calls.push({
+			id: call.id ?? `call_${i++}`,
+			name: call.function.name,
+			arguments: parsedArgs,
+			type: "function",
+		});
+	}
+	return calls;
+}
+
+function parseToolArguments(
+	call: CapacitorLlamaToolCall,
+): Record<string, unknown> {
+	const raw = call.function.arguments;
+	if (typeof raw !== "string" || raw.length === 0) return {};
+	try {
+		const parsed = JSON.parse(raw) as unknown;
+		if (parsed && typeof parsed === "object" && !Array.isArray(parsed)) {
+			return parsed as Record<string, unknown>;
+		}
+		return { _: parsed };
+	} catch {
+		// Model emitted invalid JSON; surface raw text so the runtime can
+		// recover or report rather than silently dropping the call.
+		return { _raw: raw };
+	}
+}
+
+/**
+ * Decide which structured-output mode applies. Mirrors the legacy
+ * `planStructuredRequest` semantics: tools > schema > generic json > text.
+ */
+export function planStructuredRequest(params: {
+	tools?: readonly ToolDefinition[];
+	responseSchema?: JSONSchema;
+	responseFormat?: { type: "json_object" | "text" } | string | undefined;
+	toolChoice?: string;
+}): StructuredRequestPlan {
+	if (params.tools && params.tools.length > 0) {
+		return {
+			kind: "tools",
+			tools: buildCapacitorTools(params.tools),
+			...(params.toolChoice ? { toolChoice: params.toolChoice } : {}),
+		};
+	}
+	if (params.responseSchema) {
+		return {
+			kind: "schema",
+			responseFormat: {
+				type: "json_schema",
+				json_schema: {
+					strict: true,
+					schema: params.responseSchema as object,
+				},
+			},
+		};
+	}
+	if (
+		params.responseFormat &&
+		typeof params.responseFormat === "object" &&
+		params.responseFormat.type === "json_object"
+	) {
+		return {
+			kind: "json_object",
+			responseFormat: { type: "json_object" },
+		};
+	}
+	return { kind: "text" };
+}
+
+/**
+ * Apply a `StructuredRequestPlan` onto a `CapacitorLlamaCompletionParams`
+ * object, returning a new merged object. Used by the index module right
+ * before dispatching the completion.
+ */
+export function applyStructuredPlan(
+	params: CapacitorLlamaCompletionParams,
+	plan: StructuredRequestPlan,
+): CapacitorLlamaCompletionParams {
+	if (plan.kind === "text") return params;
+	const next: CapacitorLlamaCompletionParams = { ...params };
+	if (plan.tools) next.tools = plan.tools;
+	if (plan.toolChoice) next.tool_choice = plan.toolChoice;
+	if (plan.responseFormat) next.response_format = plan.responseFormat;
+	return next;
+}

package/src/adapters/capacitor-llama/text-streaming.ts

@@ -0,0 +1,227 @@
+/**
+ * Capacitor → `TextStreamResult` bridge.
+ *
+ * The Capacitor `LlamaContext.completion(params, callback)` API is push-based:
+ * `callback` fires once per token. The elizaOS runtime consumes a pull-based
+ * `TextStreamResult` (`{ textStream, text, usage, finishReason }`). We bridge
+ * the two with a queue-backed async iterator, identical in shape to the
+ * legacy node-llama-cpp bridge.
+ */
+
+import type { TextStreamResult, TokenUsage } from "@elizaos/core";
+import type {
+	CapacitorLlamaCompletionParams,
+	CapacitorLlamaCompletionResult,
+	CapacitorLlamaContext,
+} from "./types";
+
+function indexOfIgnoreCase(haystack: string, needle: string): number {
+	return haystack.toLowerCase().indexOf(needle.toLowerCase());
+}
+
+function createThinkTagStreamFilter(): {
+	push(chunk: string): string[];
+	flush(): string[];
+} {
+	let buffer = "";
+	let inThink = false;
+	const openMarker = "<think";
+	const closeMarker = "</think>";
+	const safeOpenTail = openMarker.length - 1;
+	const safeCloseTail = closeMarker.length - 1;
+
+	const drain = (final: boolean): string[] => {
+		const out: string[] = [];
+		for (;;) {
+			if (inThink) {
+				const closeAt = indexOfIgnoreCase(buffer, closeMarker);
+				if (closeAt === -1) {
+					buffer = final ? "" : buffer.slice(-safeCloseTail);
+					return out;
+				}
+				buffer = buffer.slice(closeAt + closeMarker.length);
+				inThink = false;
+				continue;
+			}
+
+			const openAt = indexOfIgnoreCase(buffer, openMarker);
+			if (openAt === -1) {
+				const emitLength = final
+					? buffer.length
+					: Math.max(0, buffer.length - safeOpenTail);
+				if (emitLength > 0) {
+					out.push(buffer.slice(0, emitLength));
+					buffer = buffer.slice(emitLength);
+				}
+				return out;
+			}
+
+			if (openAt > 0) {
+				out.push(buffer.slice(0, openAt));
+				buffer = buffer.slice(openAt);
+			}
+
+			const tagEnd = buffer.indexOf(">");
+			if (tagEnd === -1) {
+				if (final) {
+					buffer = "";
+					return out;
+				}
+				return out;
+			}
+			buffer = buffer.slice(tagEnd + 1);
+			inThink = true;
+		}
+	};
+
+	return {
+		push(chunk: string): string[] {
+			buffer += chunk;
+			return drain(false);
+		},
+		flush(): string[] {
+			return drain(true);
+		},
+	};
+}
+
+export interface StreamCapacitorPromptArgs {
+	ctx: CapacitorLlamaContext;
+	params: CapacitorLlamaCompletionParams;
+	estimateUsage: (prompt: string, fullText: string) => TokenUsage;
+	onChunk?: (delta: string) => void;
+	onComplete?: (info: { fullText: string; usage: TokenUsage }) => void;
+	postProcess?: (raw: string) => string;
+}
+
+export function streamCapacitorPrompt(
+	args: StreamCapacitorPromptArgs,
+): TextStreamResult {
+	const queue: string[] = [];
+	const streamFilter = createThinkTagStreamFilter();
+	let pendingResolve: ((value: IteratorResult<string>) => void) | null = null;
+	let pendingReject: ((reason: unknown) => void) | null = null;
+	let promptError: unknown = null;
+	let promptDone = false;
+	let rawAccumulated = "";
+
+	const drain = (): void => {
+		if (!pendingResolve) return;
+		if (queue.length > 0) {
+			const next = queue.shift() as string;
+			const resolver = pendingResolve;
+			pendingResolve = null;
+			pendingReject = null;
+			resolver({ value: next, done: false });
+			return;
+		}
+		if (promptError && pendingReject) {
+			const rejector = pendingReject;
+			pendingResolve = null;
+			pendingReject = null;
+			rejector(promptError);
+			return;
+		}
+		if (promptDone) {
+			const resolver = pendingResolve;
+			pendingResolve = null;
+			pendingReject = null;
+			resolver({ value: undefined as unknown as string, done: true });
+		}
+	};
+
+	let completionFinishReason: string | undefined;
+	const completionPromise: Promise<CapacitorLlamaCompletionResult> =
+		(async () => {
+			try {
+				const result = await args.ctx.completion(args.params, (tok) => {
+					const piece = tok.token ?? tok.content ?? "";
+					if (!piece) return;
+					rawAccumulated += piece;
+					const visibleChunks = streamFilter.push(piece);
+					for (const visibleChunk of visibleChunks) {
+						if (!visibleChunk) continue;
+						args.onChunk?.(visibleChunk);
+						queue.push(visibleChunk);
+					}
+					drain();
+				});
+				const tailChunks = streamFilter.flush();
+				for (const visibleChunk of tailChunks) {
+					if (!visibleChunk) continue;
+					args.onChunk?.(visibleChunk);
+					queue.push(visibleChunk);
+				}
+				if (result.stopped_eos) completionFinishReason = "stop";
+				else if (result.stopped_word) completionFinishReason = "stop";
+				else if (result.stopped_limit) completionFinishReason = "length";
+				else if (result.interrupted) completionFinishReason = "abort";
+				return result;
+			} catch (err) {
+				promptError = err;
+				throw err;
+			} finally {
+				promptDone = true;
+				drain();
+			}
+		})();
+
+	completionPromise.catch(() => {
+		/* surfaced through textStream */
+	});
+
+	const textStream: AsyncIterable<string> = {
+		[Symbol.asyncIterator](): AsyncIterator<string> {
+			return {
+				next(): Promise<IteratorResult<string>> {
+					if (promptError) return Promise.reject(promptError);
+					if (queue.length > 0) {
+						const next = queue.shift() as string;
+						return Promise.resolve({ value: next, done: false });
+					}
+					if (promptDone) {
+						return Promise.resolve({
+							value: undefined as unknown as string,
+							done: true,
+						});
+					}
+					return new Promise<IteratorResult<string>>((resolve, reject) => {
+						pendingResolve = resolve;
+						pendingReject = reject;
+					});
+				},
+			};
+		},
+	};
+
+	const fullTextPromise: Promise<string> = completionPromise.then((result) => {
+		const raw = result.text.length > 0 ? result.text : rawAccumulated;
+		return args.postProcess ? args.postProcess(raw) : raw;
+	});
+
+	const usagePromise: Promise<TokenUsage | undefined> = completionPromise
+		.then((result) => {
+			const usage: TokenUsage = {
+				promptTokens: result.tokens_evaluated,
+				completionTokens: result.tokens_predicted,
+				totalTokens: result.tokens_evaluated + result.tokens_predicted,
+			};
+			// Surface to caller once.
+			void fullTextPromise.then((fullText) =>
+				args.onComplete?.({ fullText, usage }),
+			);
+			return usage;
+		})
+		.catch(() => undefined);
+
+	const finishReasonPromise: Promise<string | undefined> = completionPromise
+		.then(() => completionFinishReason ?? "stop")
+		.catch(() => undefined);
+
+	return {
+		textStream,
+		text: fullTextPromise,
+		usage: usagePromise,
+		finishReason: finishReasonPromise,
+	};
+}