@elizaos/plugin-local-inference 2.0.0-beta.1 → 2.0.3-beta.2

package/src/adapters/capacitor-llama/__tests__/index.test.ts

@@ -0,0 +1,68 @@
+import { ModelType } from "@elizaos/core";
+import { describe, expect, it } from "vitest";
+import { configSchema } from "../environment";
+import { localAiPlugin } from "../index";
+import { MODEL_SPECS } from "../types";
+
+describe("Local AI Plugin (Capacitor-llama)", () => {
+	describe("Plugin metadata", () => {
+		it("registers as 'local-ai' for backwards compatibility", () => {
+			expect(localAiPlugin.name).toBe("local-ai");
+		});
+
+		it("describes itself as Capacitor-backed", () => {
+			expect(localAiPlugin.description).toBeDefined();
+			expect(localAiPlugin.description?.toLowerCase()).toContain("capacitor");
+		});
+
+		it("exposes an init function", () => {
+			expect(typeof localAiPlugin.init).toBe("function");
+		});
+	});
+
+	describe("Model handlers", () => {
+		it("registers TEXT_SMALL, TEXT_LARGE, TEXT_EMBEDDING", () => {
+			expect(typeof localAiPlugin.models?.TEXT_SMALL).toBe("function");
+			expect(typeof localAiPlugin.models?.TEXT_LARGE).toBe("function");
+			expect(typeof localAiPlugin.models?.TEXT_EMBEDDING).toBe("function");
+		});
+
+		it("registers other model types as compat thunks", () => {
+			expect(typeof localAiPlugin.models?.[ModelType.IMAGE_DESCRIPTION]).toBe(
+				"function",
+			);
+			expect(typeof localAiPlugin.models?.[ModelType.TRANSCRIPTION]).toBe(
+				"function",
+			);
+			expect(typeof localAiPlugin.models?.[ModelType.TEXT_TO_SPEECH]).toBe(
+				"function",
+			);
+		});
+	});
+
+	describe("Model bundle (MODEL_SPECS)", () => {
+		it("includes small, medium, and embedding specs", () => {
+			expect(MODEL_SPECS.small.name).toContain("eliza-1");
+			expect(MODEL_SPECS.medium.name).toContain("eliza-1");
+			// Embeddings use the compact gte-small model, not an eliza-1 chat GGUF.
+			expect(MODEL_SPECS.embedding.name).toContain("gte-small");
+		});
+
+		it("declares 384 embedding dimensions to match plugin-local-inference", () => {
+			expect(MODEL_SPECS.embedding.dimensions).toBe(384);
+		});
+
+		it("does not declare any node-llama-cpp-specific fields", () => {
+			// No tokenizer hash, no NAPI binding name — pure GGUF.
+			const json = JSON.stringify(MODEL_SPECS);
+			expect(json.toLowerCase()).not.toContain("node-llama-cpp");
+		});
+	});
+
+	describe("environment configSchema", () => {
+		it("parses an empty environment without throwing", () => {
+			const parsed = configSchema.parse({});
+			expect(parsed.LOCAL_SMALL_MODEL).toContain("eliza-1");
+		});
+	});
+});

package/src/adapters/capacitor-llama/__tests__/structured-output.test.ts

@@ -0,0 +1,215 @@
+import { describe, expect, it } from "vitest";
+import {
+	applyStructuredPlan,
+	buildCapacitorTools,
+	extractToolCalls,
+	planStructuredRequest,
+} from "../structured-output";
+import type {
+	CapacitorLlamaCompletionParams,
+	CapacitorLlamaCompletionResult,
+} from "../types";
+
+describe("capacitor-llama / structured-output", () => {
+	describe("planStructuredRequest", () => {
+		it("returns kind=text when no tools/schema/json supplied", () => {
+			const plan = planStructuredRequest({});
+			expect(plan.kind).toBe("text");
+			expect(plan.tools).toBeUndefined();
+			expect(plan.responseFormat).toBeUndefined();
+		});
+
+		it("prioritises tools over schema", () => {
+			const plan = planStructuredRequest({
+				tools: [
+					{ name: "foo", description: "f", parameters: { type: "object" } },
+				],
+				responseSchema: { type: "object" },
+			});
+			expect(plan.kind).toBe("tools");
+			expect(plan.tools).toBeDefined();
+		});
+
+		it("emits json_schema response_format for responseSchema", () => {
+			const plan = planStructuredRequest({
+				responseSchema: {
+					type: "object",
+					properties: { x: { type: "string" } },
+				},
+			});
+			expect(plan.kind).toBe("schema");
+			expect(plan.responseFormat?.type).toBe("json_schema");
+			expect(plan.responseFormat?.json_schema?.schema).toEqual({
+				type: "object",
+				properties: { x: { type: "string" } },
+			});
+		});
+
+		it("emits json_object response_format for responseFormat json_object", () => {
+			const plan = planStructuredRequest({
+				responseFormat: { type: "json_object" },
+			});
+			expect(plan.kind).toBe("json_object");
+			expect(plan.responseFormat?.type).toBe("json_object");
+		});
+
+		it("forwards toolChoice when tools present", () => {
+			const plan = planStructuredRequest({
+				tools: [{ name: "go", description: "go" }],
+				toolChoice: "auto",
+			});
+			expect(plan.kind).toBe("tools");
+			expect(plan.toolChoice).toBe("auto");
+		});
+	});
+
+	describe("buildCapacitorTools", () => {
+		it("skips tools without a name", () => {
+			const out = buildCapacitorTools([
+				{ name: "", description: "skip" },
+				{ name: "ok", description: "keep" },
+			]) as Array<{ function: { name: string } }>;
+			expect(out).toHaveLength(1);
+			expect(out[0].function.name).toBe("ok");
+		});
+
+		it("propagates description and parameters", () => {
+			const out = buildCapacitorTools([
+				{
+					name: "ping",
+					description: "pings",
+					parameters: { type: "object", properties: { x: { type: "number" } } },
+				},
+			]) as Array<{
+				type: "function";
+				function: {
+					name: string;
+					description?: string;
+					parameters?: unknown;
+				};
+			}>;
+			expect(out[0].type).toBe("function");
+			expect(out[0].function.description).toBe("pings");
+			expect(out[0].function.parameters).toEqual({
+				type: "object",
+				properties: { x: { type: "number" } },
+			});
+		});
+	});
+
+	describe("extractToolCalls", () => {
+		const baseResult = (
+			overrides: Partial<CapacitorLlamaCompletionResult>,
+		): CapacitorLlamaCompletionResult => ({
+			text: "",
+			reasoning_content: "",
+			tool_calls: [],
+			content: "",
+			chat_format: 0,
+			tokens_predicted: 0,
+			tokens_evaluated: 0,
+			truncated: false,
+			stopped_eos: false,
+			stopped_word: "",
+			stopped_limit: 0,
+			stopping_word: "",
+			context_full: false,
+			interrupted: false,
+			tokens_cached: 0,
+			timings: {
+				prompt_n: 0,
+				prompt_ms: 0,
+				prompt_per_token_ms: 0,
+				prompt_per_second: 0,
+				predicted_n: 0,
+				predicted_ms: 0,
+				predicted_per_token_ms: 0,
+				predicted_per_second: 0,
+			},
+			...overrides,
+		});
+
+		it("parses valid JSON arguments", () => {
+			const calls = extractToolCalls(
+				baseResult({
+					tool_calls: [
+						{
+							type: "function",
+							id: "call_a",
+							function: { name: "foo", arguments: '{"x":1,"y":"z"}' },
+						},
+					],
+				}),
+			);
+			expect(calls).toEqual([
+				{
+					id: "call_a",
+					name: "foo",
+					arguments: { x: 1, y: "z" },
+					type: "function",
+				},
+			]);
+		});
+
+		it("synthesises a call id when missing", () => {
+			const calls = extractToolCalls(
+				baseResult({
+					tool_calls: [
+						{ type: "function", function: { name: "f", arguments: "{}" } },
+						{ type: "function", function: { name: "g", arguments: "{}" } },
+					],
+				}),
+			);
+			expect(calls.map((c) => c.id)).toEqual(["call_0", "call_1"]);
+		});
+
+		it("surfaces invalid JSON as _raw rather than dropping the call", () => {
+			const calls = extractToolCalls(
+				baseResult({
+					tool_calls: [
+						{
+							type: "function",
+							id: "broken",
+							function: { name: "x", arguments: "not-json" },
+						},
+					],
+				}),
+			);
+			expect(calls).toHaveLength(1);
+			expect(calls[0].arguments).toEqual({ _raw: "not-json" });
+		});
+	});
+
+	describe("applyStructuredPlan", () => {
+		const base: CapacitorLlamaCompletionParams = {
+			prompt: "hi",
+			n_predict: 32,
+		};
+
+		it("returns params untouched for text plans", () => {
+			const out = applyStructuredPlan(base, { kind: "text" });
+			expect(out).toEqual(base);
+		});
+
+		it("merges tools + tool_choice for tool plans", () => {
+			const out = applyStructuredPlan(base, {
+				kind: "tools",
+				tools: [{ name: "foo" }],
+				toolChoice: "auto",
+			});
+			expect(out.tools).toEqual([{ name: "foo" }]);
+			expect(out.tool_choice).toBe("auto");
+		});
+
+		it("merges response_format for schema plans", () => {
+			const out = applyStructuredPlan(base, {
+				kind: "schema",
+				responseFormat: {
+					type: "json_schema",
+					json_schema: { strict: true, schema: { type: "object" } },
+				},
+			});
+			expect(out.response_format?.type).toBe("json_schema");
+		});
+	});
+});

package/src/adapters/capacitor-llama/__tests__/text-streaming.test.ts

@@ -0,0 +1,174 @@
+import type { TokenUsage } from "@elizaos/core";
+import { describe, expect, it, vi } from "vitest";
+import { streamCapacitorPrompt } from "../text-streaming";
+import type {
+	CapacitorLlamaCompletionParams,
+	CapacitorLlamaCompletionResult,
+	CapacitorLlamaContext,
+	CapacitorLlamaTokenData,
+} from "../types";
+
+function makeBaseResult(
+	text: string,
+	overrides: Partial<CapacitorLlamaCompletionResult> = {},
+): CapacitorLlamaCompletionResult {
+	return {
+		text,
+		reasoning_content: "",
+		tool_calls: [],
+		content: text,
+		chat_format: 0,
+		tokens_predicted: 4,
+		tokens_evaluated: 3,
+		truncated: false,
+		stopped_eos: true,
+		stopped_word: "",
+		stopped_limit: 0,
+		stopping_word: "",
+		context_full: false,
+		interrupted: false,
+		tokens_cached: 3,
+		timings: {
+			prompt_n: 0,
+			prompt_ms: 0,
+			prompt_per_token_ms: 0,
+			prompt_per_second: 0,
+			predicted_n: 4,
+			predicted_ms: 10,
+			predicted_per_token_ms: 2.5,
+			predicted_per_second: 400,
+		},
+		...overrides,
+	};
+}
+
+function makeCtx(
+	chunks: string[],
+	overrides: Partial<CapacitorLlamaCompletionResult> = {},
+): CapacitorLlamaContext {
+	return {
+		id: 1,
+		gpu: false,
+		reasonNoGPU: "",
+		model: {} as CapacitorLlamaContext["model"],
+		async completion(
+			_params: CapacitorLlamaCompletionParams,
+			callback?: (data: CapacitorLlamaTokenData) => void,
+		): Promise<CapacitorLlamaCompletionResult> {
+			for (const chunk of chunks) {
+				callback?.({ token: chunk });
+			}
+			return makeBaseResult(chunks.join(""), overrides);
+		},
+		stopCompletion: vi.fn(async () => undefined),
+		tokenize: vi.fn(async () => ({
+			tokens: [],
+			has_images: false,
+			bitmap_hashes: [],
+			chunk_pos: [],
+			chunk_pos_images: [],
+		})),
+		detokenize: vi.fn(async () => ""),
+		embedding: vi.fn(async () => ({ embedding: [] })),
+		bench: vi.fn(async () => ({
+			modelDesc: "",
+			modelSize: 0,
+			modelNParams: 0,
+			ppAvg: 0,
+			ppStd: 0,
+			tgAvg: 0,
+			tgStd: 0,
+		})),
+		release: vi.fn(async () => undefined),
+	};
+}
+
+const estimateUsage = (prompt: string, fullText: string): TokenUsage => ({
+	promptTokens: prompt.length,
+	completionTokens: fullText.length,
+	totalTokens: prompt.length + fullText.length,
+});
+
+describe("capacitor-llama / text-streaming", () => {
+	it("forwards each token through textStream", async () => {
+		const ctx = makeCtx(["hel", "lo", " ", "world"]);
+		const stream = streamCapacitorPrompt({
+			ctx,
+			params: { prompt: "hi" },
+			estimateUsage,
+		});
+		const collected: string[] = [];
+		for await (const chunk of stream.textStream) {
+			collected.push(chunk);
+		}
+		expect(collected.join("")).toBe("hello world");
+		const fullText = await stream.text;
+		expect(fullText).toBe("hello world");
+		const usage = await stream.usage;
+		expect(usage?.promptTokens).toBe(3);
+		expect(usage?.completionTokens).toBe(4);
+	});
+
+	it("strips think tags via postProcess on the resolved text", async () => {
+		const ctx = makeCtx(["<think>secret</think>", "answer"]);
+		const stream = streamCapacitorPrompt({
+			ctx,
+			params: { prompt: "hi" },
+			estimateUsage,
+			postProcess: (raw) => raw.replace(/<think>[\s\S]*?<\/think>\n?/g, ""),
+		});
+		// Drain the iterable.
+		for await (const _ of stream.textStream) {
+			// nothing
+		}
+		const final = await stream.text;
+		expect(final).toBe("answer");
+	});
+
+	it("resolves finishReason to stop on stopped_eos", async () => {
+		const ctx = makeCtx(["a", "b"]);
+		const stream = streamCapacitorPrompt({
+			ctx,
+			params: { prompt: "hi" },
+			estimateUsage,
+		});
+		for await (const _ of stream.textStream) {
+			/* drain */
+		}
+		expect(await stream.finishReason).toBe("stop");
+	});
+
+	it("resolves finishReason to length on stopped_limit", async () => {
+		const ctx = makeCtx(["a"], { stopped_eos: false, stopped_limit: 1 });
+		const stream = streamCapacitorPrompt({
+			ctx,
+			params: { prompt: "hi" },
+			estimateUsage,
+		});
+		for await (const _ of stream.textStream) {
+			/* drain */
+		}
+		expect(await stream.finishReason).toBe("length");
+	});
+
+	it("invokes onChunk with every visible delta and the concatenation matches the full text", async () => {
+		const seen: string[] = [];
+		const ctx = makeCtx(["a", "bc"]);
+		const stream = streamCapacitorPrompt({
+			ctx,
+			params: { prompt: "hi" },
+			estimateUsage,
+			onChunk: (delta) => {
+				seen.push(delta);
+			},
+		});
+		for await (const _ of stream.textStream) {
+			/* drain */
+		}
+		// The think-tag filter coalesces tokens during buffering to keep
+		// `<think>` prefix detection safe. We don't assert chunk boundaries —
+		// only that every visible byte was delivered exactly once.
+		expect(seen.length).toBeGreaterThan(0);
+		expect(seen.join("")).toBe("abc");
+	});
+});

package/src/adapters/capacitor-llama/environment.ts

@@ -0,0 +1,71 @@
+import { logger } from "@elizaos/core";
+import { z } from "zod";
+
+const DEFAULT_SMALL_MODEL = "text/eliza-1-2b-128k.gguf";
+const DEFAULT_LARGE_MODEL = "text/eliza-1-4b-128k.gguf";
+const DEFAULT_EMBEDDING_MODEL = "gte-small_fp16.gguf";
+
+export const configSchema = z.object({
+	LOCAL_SMALL_MODEL: z.string().optional().default(DEFAULT_SMALL_MODEL),
+	LOCAL_LARGE_MODEL: z.string().optional().default(DEFAULT_LARGE_MODEL),
+	LOCAL_EMBEDDING_MODEL: z.string().optional().default(DEFAULT_EMBEDDING_MODEL),
+	MODELS_DIR: z.string().optional(),
+	CACHE_DIR: z.string().optional(),
+	LOCAL_EMBEDDING_DIMENSIONS: z
+		.string()
+		.optional()
+		.default("384")
+		.transform((val) => parseInt(val, 10)),
+});
+
+export type Config = z.infer<typeof configSchema>;
+
+export function validateConfig(): Config {
+	try {
+		const configToParse = {
+			LOCAL_SMALL_MODEL: process.env.LOCAL_SMALL_MODEL,
+			LOCAL_LARGE_MODEL: process.env.LOCAL_LARGE_MODEL,
+			LOCAL_EMBEDDING_MODEL: process.env.LOCAL_EMBEDDING_MODEL,
+			MODELS_DIR: process.env.MODELS_DIR,
+			CACHE_DIR: process.env.CACHE_DIR,
+			LOCAL_EMBEDDING_DIMENSIONS: process.env.LOCAL_EMBEDDING_DIMENSIONS,
+		};
+
+		logger.debug(
+			{
+				LOCAL_SMALL_MODEL: configToParse.LOCAL_SMALL_MODEL,
+				LOCAL_LARGE_MODEL: configToParse.LOCAL_LARGE_MODEL,
+				LOCAL_EMBEDDING_MODEL: configToParse.LOCAL_EMBEDDING_MODEL,
+				MODELS_DIR: configToParse.MODELS_DIR,
+				CACHE_DIR: configToParse.CACHE_DIR,
+				LOCAL_EMBEDDING_DIMENSIONS: configToParse.LOCAL_EMBEDDING_DIMENSIONS,
+			},
+			"Validating configuration for local AI plugin from env:",
+		);
+
+		const validatedConfig = configSchema.parse(configToParse);
+
+		logger.info(
+			validatedConfig as Record<string, unknown>,
+			"Using local AI configuration:",
+		);
+
+		return validatedConfig;
+	} catch (error) {
+		if (error instanceof z.ZodError) {
+			const errorMessages = error.issues
+				.map((issue) => `${issue.path.join(".")}: ${issue.message}`)
+				.join("\n");
+			logger.error(`Zod validation failed: ${errorMessages}`);
+			throw new Error(`Configuration validation failed:\n${errorMessages}`);
+		}
+		logger.error(
+			{
+				error: error instanceof Error ? error.message : String(error),
+				stack: error instanceof Error ? error.stack : undefined,
+			},
+			"Configuration validation failed:",
+		);
+		throw error;
+	}
+}

package/src/adapters/capacitor-llama/index.browser.ts

@@ -0,0 +1,83 @@
+/**
+ * Browser-unavailable Capacitor-llama adapter. Browsers cannot run
+ * `libllama.so` directly; callers should proxy through a server or switch
+ * providers. All model handlers throw with a clear message.
+ */
+
+import type {
+	GenerateTextParams,
+	IAgentRuntime,
+	ImageDescriptionParams,
+	Plugin,
+	TextEmbeddingParams,
+} from "@elizaos/core";
+import { logger, ModelType } from "@elizaos/core";
+
+type ImageDescriptionResult = { title: string; description: string };
+
+const pluginName = "local-ai";
+const unsupportedMessage =
+	"Local AI is not supported in browsers. Use a server proxy or switch providers.";
+
+const warnUnsupported = (modelType: string): void => {
+	logger.warn(
+		`[plugin-${pluginName}] ${modelType} is not available in browsers.`,
+	);
+};
+
+const unsupportedText = (
+	modelType: string,
+	params?: GenerateTextParams,
+): string => {
+	warnUnsupported(modelType);
+	if (params && (params.tools || params.responseSchema || params.toolChoice)) {
+		throw new Error(
+			`[plugin-${pluginName}] Tool calling and structured output require the Node/Bun or Capacitor mobile runtime. ` +
+				"Browsers cannot execute llama.cpp directly — switch providers or proxy through a server.",
+		);
+	}
+	return unsupportedMessage;
+};
+
+const unsupportedImageDescription = (
+	modelType: string,
+): ImageDescriptionResult => {
+	warnUnsupported(modelType);
+	throw new Error(`[plugin-${pluginName}] ${modelType}: ${unsupportedMessage}`);
+};
+
+export const localAiPlugin: Plugin = {
+	name: pluginName,
+	description: "Local AI plugin (unavailable in browsers; use a server proxy)",
+	async init(_config, _runtime: IAgentRuntime): Promise<void> {
+		logger.warn(
+			`[plugin-${pluginName}] Capacitor-llama adapter is not supported directly in browsers.`,
+		);
+	},
+	models: {
+		[ModelType.TEXT_SMALL]: async (
+			_runtime: IAgentRuntime,
+			params: GenerateTextParams,
+		): Promise<string> => unsupportedText(ModelType.TEXT_SMALL, params),
+		[ModelType.TEXT_LARGE]: async (
+			_runtime: IAgentRuntime,
+			params: GenerateTextParams,
+		): Promise<string> => unsupportedText(ModelType.TEXT_LARGE, params),
+		[ModelType.TEXT_EMBEDDING]: async (
+			_runtime: IAgentRuntime,
+			_params: TextEmbeddingParams | string | null,
+		): Promise<number[]> => {
+			warnUnsupported(ModelType.TEXT_EMBEDDING);
+			throw new Error(
+				`[plugin-${pluginName}] ${ModelType.TEXT_EMBEDDING}: ${unsupportedMessage}`,
+			);
+		},
+		[ModelType.IMAGE_DESCRIPTION]: async (
+			_runtime: IAgentRuntime,
+			_params: ImageDescriptionParams | string,
+		): Promise<ImageDescriptionResult> =>
+			unsupportedImageDescription(ModelType.IMAGE_DESCRIPTION),
+	},
+};
+
+export default localAiPlugin;