@elizaos/plugin-local-inference 2.0.0-beta.1 → 2.0.11-beta.7

package/src/adapters/capacitor-llama/__tests__/text-streaming.test.ts

@@ -0,0 +1,174 @@
+import type { TokenUsage } from "@elizaos/core";
+import { describe, expect, it, vi } from "vitest";
+import { streamCapacitorPrompt } from "../text-streaming";
+import type {
+	CapacitorLlamaCompletionParams,
+	CapacitorLlamaCompletionResult,
+	CapacitorLlamaContext,
+	CapacitorLlamaTokenData,
+} from "../types";
+
+function makeBaseResult(
+	text: string,
+	overrides: Partial<CapacitorLlamaCompletionResult> = {},
+): CapacitorLlamaCompletionResult {
+	return {
+		text,
+		reasoning_content: "",
+		tool_calls: [],
+		content: text,
+		chat_format: 0,
+		tokens_predicted: 4,
+		tokens_evaluated: 3,
+		truncated: false,
+		stopped_eos: true,
+		stopped_word: "",
+		stopped_limit: 0,
+		stopping_word: "",
+		context_full: false,
+		interrupted: false,
+		tokens_cached: 3,
+		timings: {
+			prompt_n: 0,
+			prompt_ms: 0,
+			prompt_per_token_ms: 0,
+			prompt_per_second: 0,
+			predicted_n: 4,
+			predicted_ms: 10,
+			predicted_per_token_ms: 2.5,
+			predicted_per_second: 400,
+		},
+		...overrides,
+	};
+}
+
+function makeCtx(
+	chunks: string[],
+	overrides: Partial<CapacitorLlamaCompletionResult> = {},
+): CapacitorLlamaContext {
+	return {
+		id: 1,
+		gpu: false,
+		reasonNoGPU: "",
+		model: {} as CapacitorLlamaContext["model"],
+		async completion(
+			_params: CapacitorLlamaCompletionParams,
+			callback?: (data: CapacitorLlamaTokenData) => void,
+		): Promise<CapacitorLlamaCompletionResult> {
+			for (const chunk of chunks) {
+				callback?.({ token: chunk });
+			}
+			return makeBaseResult(chunks.join(""), overrides);
+		},
+		stopCompletion: vi.fn(async () => undefined),
+		tokenize: vi.fn(async () => ({
+			tokens: [],
+			has_images: false,
+			bitmap_hashes: [],
+			chunk_pos: [],
+			chunk_pos_images: [],
+		})),
+		detokenize: vi.fn(async () => ""),
+		embedding: vi.fn(async () => ({ embedding: [] })),
+		bench: vi.fn(async () => ({
+			modelDesc: "",
+			modelSize: 0,
+			modelNParams: 0,
+			ppAvg: 0,
+			ppStd: 0,
+			tgAvg: 0,
+			tgStd: 0,
+		})),
+		release: vi.fn(async () => undefined),
+	};
+}
+
+const estimateUsage = (prompt: string, fullText: string): TokenUsage => ({
+	promptTokens: prompt.length,
+	completionTokens: fullText.length,
+	totalTokens: prompt.length + fullText.length,
+});
+
+describe("capacitor-llama / text-streaming", () => {
+	it("forwards each token through textStream", async () => {
+		const ctx = makeCtx(["hel", "lo", " ", "world"]);
+		const stream = streamCapacitorPrompt({
+			ctx,
+			params: { prompt: "hi" },
+			estimateUsage,
+		});
+		const collected: string[] = [];
+		for await (const chunk of stream.textStream) {
+			collected.push(chunk);
+		}
+		expect(collected.join("")).toBe("hello world");
+		const fullText = await stream.text;
+		expect(fullText).toBe("hello world");
+		const usage = await stream.usage;
+		expect(usage?.promptTokens).toBe(3);
+		expect(usage?.completionTokens).toBe(4);
+	});
+
+	it("strips think tags via postProcess on the resolved text", async () => {
+		const ctx = makeCtx(["<think>secret</think>", "answer"]);
+		const stream = streamCapacitorPrompt({
+			ctx,
+			params: { prompt: "hi" },
+			estimateUsage,
+			postProcess: (raw) => raw.replace(/<think>[\s\S]*?<\/think>\n?/g, ""),
+		});
+		// Drain the iterable.
+		for await (const _ of stream.textStream) {
+			// nothing
+		}
+		const final = await stream.text;
+		expect(final).toBe("answer");
+	});
+
+	it("resolves finishReason to stop on stopped_eos", async () => {
+		const ctx = makeCtx(["a", "b"]);
+		const stream = streamCapacitorPrompt({
+			ctx,
+			params: { prompt: "hi" },
+			estimateUsage,
+		});
+		for await (const _ of stream.textStream) {
+			/* drain */
+		}
+		expect(await stream.finishReason).toBe("stop");
+	});
+
+	it("resolves finishReason to length on stopped_limit", async () => {
+		const ctx = makeCtx(["a"], { stopped_eos: false, stopped_limit: 1 });
+		const stream = streamCapacitorPrompt({
+			ctx,
+			params: { prompt: "hi" },
+			estimateUsage,
+		});
+		for await (const _ of stream.textStream) {
+			/* drain */
+		}
+		expect(await stream.finishReason).toBe("length");
+	});
+
+	it("invokes onChunk with every visible delta and the concatenation matches the full text", async () => {
+		const seen: string[] = [];
+		const ctx = makeCtx(["a", "bc"]);
+		const stream = streamCapacitorPrompt({
+			ctx,
+			params: { prompt: "hi" },
+			estimateUsage,
+			onChunk: (delta) => {
+				seen.push(delta);
+			},
+		});
+		for await (const _ of stream.textStream) {
+			/* drain */
+		}
+		// The think-tag filter coalesces tokens during buffering to keep
+		// `<think>` prefix detection safe. We don't assert chunk boundaries —
+		// only that every visible byte was delivered exactly once.
+		expect(seen.length).toBeGreaterThan(0);
+		expect(seen.join("")).toBe("abc");
+	});
+});

package/src/adapters/capacitor-llama/environment.ts

@@ -0,0 +1,71 @@
+import { logger } from "@elizaos/core";
+import { z } from "zod";
+
+const DEFAULT_SMALL_MODEL = "text/eliza-1-2b-128k.gguf";
+const DEFAULT_LARGE_MODEL = "text/eliza-1-4b-128k.gguf";
+const DEFAULT_EMBEDDING_MODEL = "gte-small_fp16.gguf";
+
+export const configSchema = z.object({
+	LOCAL_SMALL_MODEL: z.string().optional().default(DEFAULT_SMALL_MODEL),
+	LOCAL_LARGE_MODEL: z.string().optional().default(DEFAULT_LARGE_MODEL),
+	LOCAL_EMBEDDING_MODEL: z.string().optional().default(DEFAULT_EMBEDDING_MODEL),
+	MODELS_DIR: z.string().optional(),
+	CACHE_DIR: z.string().optional(),
+	LOCAL_EMBEDDING_DIMENSIONS: z
+		.string()
+		.optional()
+		.default("384")
+		.transform((val) => parseInt(val, 10)),
+});
+
+export type Config = z.infer<typeof configSchema>;
+
+export function validateConfig(): Config {
+	try {
+		const configToParse = {
+			LOCAL_SMALL_MODEL: process.env.LOCAL_SMALL_MODEL,
+			LOCAL_LARGE_MODEL: process.env.LOCAL_LARGE_MODEL,
+			LOCAL_EMBEDDING_MODEL: process.env.LOCAL_EMBEDDING_MODEL,
+			MODELS_DIR: process.env.MODELS_DIR,
+			CACHE_DIR: process.env.CACHE_DIR,
+			LOCAL_EMBEDDING_DIMENSIONS: process.env.LOCAL_EMBEDDING_DIMENSIONS,
+		};
+
+		logger.debug(
+			{
+				LOCAL_SMALL_MODEL: configToParse.LOCAL_SMALL_MODEL,
+				LOCAL_LARGE_MODEL: configToParse.LOCAL_LARGE_MODEL,
+				LOCAL_EMBEDDING_MODEL: configToParse.LOCAL_EMBEDDING_MODEL,
+				MODELS_DIR: configToParse.MODELS_DIR,
+				CACHE_DIR: configToParse.CACHE_DIR,
+				LOCAL_EMBEDDING_DIMENSIONS: configToParse.LOCAL_EMBEDDING_DIMENSIONS,
+			},
+			"Validating configuration for local AI plugin from env:",
+		);
+
+		const validatedConfig = configSchema.parse(configToParse);
+
+		logger.info(
+			validatedConfig as Record<string, unknown>,
+			"Using local AI configuration:",
+		);
+
+		return validatedConfig;
+	} catch (error) {
+		if (error instanceof z.ZodError) {
+			const errorMessages = error.issues
+				.map((issue) => `${issue.path.join(".")}: ${issue.message}`)
+				.join("\n");
+			logger.error(`Zod validation failed: ${errorMessages}`);
+			throw new Error(`Configuration validation failed:\n${errorMessages}`);
+		}
+		logger.error(
+			{
+				error: error instanceof Error ? error.message : String(error),
+				stack: error instanceof Error ? error.stack : undefined,
+			},
+			"Configuration validation failed:",
+		);
+		throw error;
+	}
+}

package/src/adapters/capacitor-llama/index.browser.ts

@@ -0,0 +1,83 @@
+/**
+ * Browser-unavailable Capacitor-llama adapter. Browsers cannot run
+ * `libllama.so` directly; callers should proxy through a server or switch
+ * providers. All model handlers throw with a clear message.
+ */
+
+import type {
+	GenerateTextParams,
+	IAgentRuntime,
+	ImageDescriptionParams,
+	Plugin,
+	TextEmbeddingParams,
+} from "@elizaos/core";
+import { logger, ModelType } from "@elizaos/core";
+
+type ImageDescriptionResult = { title: string; description: string };
+
+const pluginName = "local-ai";
+const unsupportedMessage =
+	"Local AI is not supported in browsers. Use a server proxy or switch providers.";
+
+const warnUnsupported = (modelType: string): void => {
+	logger.warn(
+		`[plugin-${pluginName}] ${modelType} is not available in browsers.`,
+	);
+};
+
+const unsupportedText = (
+	modelType: string,
+	params?: GenerateTextParams,
+): string => {
+	warnUnsupported(modelType);
+	if (params && (params.tools || params.responseSchema || params.toolChoice)) {
+		throw new Error(
+			`[plugin-${pluginName}] Tool calling and structured output require the Node/Bun or Capacitor mobile runtime. ` +
+				"Browsers cannot execute llama.cpp directly — switch providers or proxy through a server.",
+		);
+	}
+	return unsupportedMessage;
+};
+
+const unsupportedImageDescription = (
+	modelType: string,
+): ImageDescriptionResult => {
+	warnUnsupported(modelType);
+	throw new Error(`[plugin-${pluginName}] ${modelType}: ${unsupportedMessage}`);
+};
+
+export const localAiPlugin: Plugin = {
+	name: pluginName,
+	description: "Local AI plugin (unavailable in browsers; use a server proxy)",
+	async init(_config, _runtime: IAgentRuntime): Promise<void> {
+		logger.warn(
+			`[plugin-${pluginName}] Capacitor-llama adapter is not supported directly in browsers.`,
+		);
+	},
+	models: {
+		[ModelType.TEXT_SMALL]: async (
+			_runtime: IAgentRuntime,
+			params: GenerateTextParams,
+		): Promise<string> => unsupportedText(ModelType.TEXT_SMALL, params),
+		[ModelType.TEXT_LARGE]: async (
+			_runtime: IAgentRuntime,
+			params: GenerateTextParams,
+		): Promise<string> => unsupportedText(ModelType.TEXT_LARGE, params),
+		[ModelType.TEXT_EMBEDDING]: async (
+			_runtime: IAgentRuntime,
+			_params: TextEmbeddingParams | string | null,
+		): Promise<number[]> => {
+			warnUnsupported(ModelType.TEXT_EMBEDDING);
+			throw new Error(
+				`[plugin-${pluginName}] ${ModelType.TEXT_EMBEDDING}: ${unsupportedMessage}`,
+			);
+		},
+		[ModelType.IMAGE_DESCRIPTION]: async (
+			_runtime: IAgentRuntime,
+			_params: ImageDescriptionParams | string,
+		): Promise<ImageDescriptionResult> =>
+			unsupportedImageDescription(ModelType.IMAGE_DESCRIPTION),
+	},
+};
+
+export default localAiPlugin;