@elizaos/plugin-local-inference 2.0.0-beta.1 → 2.0.3-beta.3

package/src/adapters/capacitor-llama/__tests__/structured-output.test.ts

@@ -0,0 +1,215 @@
+import { describe, expect, it } from "vitest";
+import {
+	applyStructuredPlan,
+	buildCapacitorTools,
+	extractToolCalls,
+	planStructuredRequest,
+} from "../structured-output";
+import type {
+	CapacitorLlamaCompletionParams,
+	CapacitorLlamaCompletionResult,
+} from "../types";
+
+describe("capacitor-llama / structured-output", () => {
+	describe("planStructuredRequest", () => {
+		it("returns kind=text when no tools/schema/json supplied", () => {
+			const plan = planStructuredRequest({});
+			expect(plan.kind).toBe("text");
+			expect(plan.tools).toBeUndefined();
+			expect(plan.responseFormat).toBeUndefined();
+		});
+
+		it("prioritises tools over schema", () => {
+			const plan = planStructuredRequest({
+				tools: [
+					{ name: "foo", description: "f", parameters: { type: "object" } },
+				],
+				responseSchema: { type: "object" },
+			});
+			expect(plan.kind).toBe("tools");
+			expect(plan.tools).toBeDefined();
+		});
+
+		it("emits json_schema response_format for responseSchema", () => {
+			const plan = planStructuredRequest({
+				responseSchema: {
+					type: "object",
+					properties: { x: { type: "string" } },
+				},
+			});
+			expect(plan.kind).toBe("schema");
+			expect(plan.responseFormat?.type).toBe("json_schema");
+			expect(plan.responseFormat?.json_schema?.schema).toEqual({
+				type: "object",
+				properties: { x: { type: "string" } },
+			});
+		});
+
+		it("emits json_object response_format for responseFormat json_object", () => {
+			const plan = planStructuredRequest({
+				responseFormat: { type: "json_object" },
+			});
+			expect(plan.kind).toBe("json_object");
+			expect(plan.responseFormat?.type).toBe("json_object");
+		});
+
+		it("forwards toolChoice when tools present", () => {
+			const plan = planStructuredRequest({
+				tools: [{ name: "go", description: "go" }],
+				toolChoice: "auto",
+			});
+			expect(plan.kind).toBe("tools");
+			expect(plan.toolChoice).toBe("auto");
+		});
+	});
+
+	describe("buildCapacitorTools", () => {
+		it("skips tools without a name", () => {
+			const out = buildCapacitorTools([
+				{ name: "", description: "skip" },
+				{ name: "ok", description: "keep" },
+			]) as Array<{ function: { name: string } }>;
+			expect(out).toHaveLength(1);
+			expect(out[0].function.name).toBe("ok");
+		});
+
+		it("propagates description and parameters", () => {
+			const out = buildCapacitorTools([
+				{
+					name: "ping",
+					description: "pings",
+					parameters: { type: "object", properties: { x: { type: "number" } } },
+				},
+			]) as Array<{
+				type: "function";
+				function: {
+					name: string;
+					description?: string;
+					parameters?: unknown;
+				};
+			}>;
+			expect(out[0].type).toBe("function");
+			expect(out[0].function.description).toBe("pings");
+			expect(out[0].function.parameters).toEqual({
+				type: "object",
+				properties: { x: { type: "number" } },
+			});
+		});
+	});
+
+	describe("extractToolCalls", () => {
+		const baseResult = (
+			overrides: Partial<CapacitorLlamaCompletionResult>,
+		): CapacitorLlamaCompletionResult => ({
+			text: "",
+			reasoning_content: "",
+			tool_calls: [],
+			content: "",
+			chat_format: 0,
+			tokens_predicted: 0,
+			tokens_evaluated: 0,
+			truncated: false,
+			stopped_eos: false,
+			stopped_word: "",
+			stopped_limit: 0,
+			stopping_word: "",
+			context_full: false,
+			interrupted: false,
+			tokens_cached: 0,
+			timings: {
+				prompt_n: 0,
+				prompt_ms: 0,
+				prompt_per_token_ms: 0,
+				prompt_per_second: 0,
+				predicted_n: 0,
+				predicted_ms: 0,
+				predicted_per_token_ms: 0,
+				predicted_per_second: 0,
+			},
+			...overrides,
+		});
+
+		it("parses valid JSON arguments", () => {
+			const calls = extractToolCalls(
+				baseResult({
+					tool_calls: [
+						{
+							type: "function",
+							id: "call_a",
+							function: { name: "foo", arguments: '{"x":1,"y":"z"}' },
+						},
+					],
+				}),
+			);
+			expect(calls).toEqual([
+				{
+					id: "call_a",
+					name: "foo",
+					arguments: { x: 1, y: "z" },
+					type: "function",
+				},
+			]);
+		});
+
+		it("synthesises a call id when missing", () => {
+			const calls = extractToolCalls(
+				baseResult({
+					tool_calls: [
+						{ type: "function", function: { name: "f", arguments: "{}" } },
+						{ type: "function", function: { name: "g", arguments: "{}" } },
+					],
+				}),
+			);
+			expect(calls.map((c) => c.id)).toEqual(["call_0", "call_1"]);
+		});
+
+		it("surfaces invalid JSON as _raw rather than dropping the call", () => {
+			const calls = extractToolCalls(
+				baseResult({
+					tool_calls: [
+						{
+							type: "function",
+							id: "broken",
+							function: { name: "x", arguments: "not-json" },
+						},
+					],
+				}),
+			);
+			expect(calls).toHaveLength(1);
+			expect(calls[0].arguments).toEqual({ _raw: "not-json" });
+		});
+	});
+
+	describe("applyStructuredPlan", () => {
+		const base: CapacitorLlamaCompletionParams = {
+			prompt: "hi",
+			n_predict: 32,
+		};
+
+		it("returns params untouched for text plans", () => {
+			const out = applyStructuredPlan(base, { kind: "text" });
+			expect(out).toEqual(base);
+		});
+
+		it("merges tools + tool_choice for tool plans", () => {
+			const out = applyStructuredPlan(base, {
+				kind: "tools",
+				tools: [{ name: "foo" }],
+				toolChoice: "auto",
+			});
+			expect(out.tools).toEqual([{ name: "foo" }]);
+			expect(out.tool_choice).toBe("auto");
+		});
+
+		it("merges response_format for schema plans", () => {
+			const out = applyStructuredPlan(base, {
+				kind: "schema",
+				responseFormat: {
+					type: "json_schema",
+					json_schema: { strict: true, schema: { type: "object" } },
+				},
+			});
+			expect(out.response_format?.type).toBe("json_schema");
+		});
+	});
+});

package/src/adapters/capacitor-llama/__tests__/text-streaming.test.ts

@@ -0,0 +1,174 @@
+import type { TokenUsage } from "@elizaos/core";
+import { describe, expect, it, vi } from "vitest";
+import { streamCapacitorPrompt } from "../text-streaming";
+import type {
+	CapacitorLlamaCompletionParams,
+	CapacitorLlamaCompletionResult,
+	CapacitorLlamaContext,
+	CapacitorLlamaTokenData,
+} from "../types";
+
+function makeBaseResult(
+	text: string,
+	overrides: Partial<CapacitorLlamaCompletionResult> = {},
+): CapacitorLlamaCompletionResult {
+	return {
+		text,
+		reasoning_content: "",
+		tool_calls: [],
+		content: text,
+		chat_format: 0,
+		tokens_predicted: 4,
+		tokens_evaluated: 3,
+		truncated: false,
+		stopped_eos: true,
+		stopped_word: "",
+		stopped_limit: 0,
+		stopping_word: "",
+		context_full: false,
+		interrupted: false,
+		tokens_cached: 3,
+		timings: {
+			prompt_n: 0,
+			prompt_ms: 0,
+			prompt_per_token_ms: 0,
+			prompt_per_second: 0,
+			predicted_n: 4,
+			predicted_ms: 10,
+			predicted_per_token_ms: 2.5,
+			predicted_per_second: 400,
+		},
+		...overrides,
+	};
+}
+
+function makeCtx(
+	chunks: string[],
+	overrides: Partial<CapacitorLlamaCompletionResult> = {},
+): CapacitorLlamaContext {
+	return {
+		id: 1,
+		gpu: false,
+		reasonNoGPU: "",
+		model: {} as CapacitorLlamaContext["model"],
+		async completion(
+			_params: CapacitorLlamaCompletionParams,
+			callback?: (data: CapacitorLlamaTokenData) => void,
+		): Promise<CapacitorLlamaCompletionResult> {
+			for (const chunk of chunks) {
+				callback?.({ token: chunk });
+			}
+			return makeBaseResult(chunks.join(""), overrides);
+		},
+		stopCompletion: vi.fn(async () => undefined),
+		tokenize: vi.fn(async () => ({
+			tokens: [],
+			has_images: false,
+			bitmap_hashes: [],
+			chunk_pos: [],
+			chunk_pos_images: [],
+		})),
+		detokenize: vi.fn(async () => ""),
+		embedding: vi.fn(async () => ({ embedding: [] })),
+		bench: vi.fn(async () => ({
+			modelDesc: "",
+			modelSize: 0,
+			modelNParams: 0,
+			ppAvg: 0,
+			ppStd: 0,
+			tgAvg: 0,
+			tgStd: 0,
+		})),
+		release: vi.fn(async () => undefined),
+	};
+}
+
+const estimateUsage = (prompt: string, fullText: string): TokenUsage => ({
+	promptTokens: prompt.length,
+	completionTokens: fullText.length,
+	totalTokens: prompt.length + fullText.length,
+});
+
+describe("capacitor-llama / text-streaming", () => {
+	it("forwards each token through textStream", async () => {
+		const ctx = makeCtx(["hel", "lo", " ", "world"]);
+		const stream = streamCapacitorPrompt({
+			ctx,
+			params: { prompt: "hi" },
+			estimateUsage,
+		});
+		const collected: string[] = [];
+		for await (const chunk of stream.textStream) {
+			collected.push(chunk);
+		}
+		expect(collected.join("")).toBe("hello world");
+		const fullText = await stream.text;
+		expect(fullText).toBe("hello world");
+		const usage = await stream.usage;
+		expect(usage?.promptTokens).toBe(3);
+		expect(usage?.completionTokens).toBe(4);
+	});
+
+	it("strips think tags via postProcess on the resolved text", async () => {
+		const ctx = makeCtx(["<think>secret</think>", "answer"]);
+		const stream = streamCapacitorPrompt({
+			ctx,
+			params: { prompt: "hi" },
+			estimateUsage,
+			postProcess: (raw) => raw.replace(/<think>[\s\S]*?<\/think>\n?/g, ""),
+		});
+		// Drain the iterable.
+		for await (const _ of stream.textStream) {
+			// nothing
+		}
+		const final = await stream.text;
+		expect(final).toBe("answer");
+	});
+
+	it("resolves finishReason to stop on stopped_eos", async () => {
+		const ctx = makeCtx(["a", "b"]);
+		const stream = streamCapacitorPrompt({
+			ctx,
+			params: { prompt: "hi" },
+			estimateUsage,
+		});
+		for await (const _ of stream.textStream) {
+			/* drain */
+		}
+		expect(await stream.finishReason).toBe("stop");
+	});
+
+	it("resolves finishReason to length on stopped_limit", async () => {
+		const ctx = makeCtx(["a"], { stopped_eos: false, stopped_limit: 1 });
+		const stream = streamCapacitorPrompt({
+			ctx,
+			params: { prompt: "hi" },
+			estimateUsage,
+		});
+		for await (const _ of stream.textStream) {
+			/* drain */
+		}
+		expect(await stream.finishReason).toBe("length");
+	});
+
+	it("invokes onChunk with every visible delta and the concatenation matches the full text", async () => {
+		const seen: string[] = [];
+		const ctx = makeCtx(["a", "bc"]);
+		const stream = streamCapacitorPrompt({
+			ctx,
+			params: { prompt: "hi" },
+			estimateUsage,
+			onChunk: (delta) => {
+				seen.push(delta);
+			},
+		});
+		for await (const _ of stream.textStream) {
+			/* drain */
+		}
+		// The think-tag filter coalesces tokens during buffering to keep
+		// `<think>` prefix detection safe. We don't assert chunk boundaries —
+		// only that every visible byte was delivered exactly once.
+		expect(seen.length).toBeGreaterThan(0);
+		expect(seen.join("")).toBe("abc");
+	});
+});

package/src/adapters/capacitor-llama/__tests__/voice-turn.test.ts

@@ -0,0 +1,293 @@
+/**
+ * Wiring tests for the on-device fused voice-turn entry (#8786). Proves the
+ * device bridge is the real production caller for
+ * `LocalInferenceEngine.runVoiceTurn`: it adapts the loaded
+ * `CapacitorLlamaContext` into the `MtpTextRunner` the fused pipeline drives,
+ * and forwards the turn to the engine. No native model — fakes stand in for the
+ * context + engine so the wiring itself is the unit under test.
+ */
+
+import { describe, expect, it, vi } from "vitest";
+import type { TranscriptionAudio } from "../../../services/voice/types";
+import { NativePcmVoiceTurnCoordinator } from "../native-voice-capture";
+import type { CapacitorLlamaContext } from "../types";
+import {
+	createCapacitorMtpTextRunner,
+	type DeviceVoiceEngine,
+	runDeviceVoiceTurn,
+	type VoiceTurnExitReason,
+} from "../voice-turn";
+
+function fakeContext(
+	overrides: Partial<CapacitorLlamaContext> = {},
+): CapacitorLlamaContext {
+	return {
+		id: 1,
+		gpu: false,
+		reasonNoGPU: "",
+		model: {} as CapacitorLlamaContext["model"],
+		completion: vi.fn(async () => ({
+			text: "four",
+		})) as unknown as CapacitorLlamaContext["completion"],
+		stopCompletion: vi.fn(async () => {}),
+		tokenize: vi.fn(),
+		detokenize: vi.fn(),
+		embedding: vi.fn(),
+		bench: vi.fn(),
+		release: vi.fn(),
+		...overrides,
+	} as CapacitorLlamaContext;
+}
+
+const AUDIO: TranscriptionAudio = {
+	pcm: new Float32Array([0.1, 0.2, 0.3]),
+	sampleRate: 16_000,
+};
+
+describe("createCapacitorMtpTextRunner", () => {
+	it("reports no drafter (on-device plain decode through the verifier)", () => {
+		const runner = createCapacitorMtpTextRunner(fakeContext());
+		expect(runner.hasDrafter()).toBe(false);
+	});
+
+	it("delegates generateWithVerifierEvents to context.completion and returns its text", async () => {
+		const ctx = fakeContext();
+		const runner = createCapacitorMtpTextRunner(ctx, { temperature: 0.5 });
+		const result = await runner.generateWithVerifierEvents({
+			prompt: "what is 2 + 2?",
+			maxTokens: 8,
+			topP: 0.9,
+			stopSequences: ["</s>"],
+			onVerifierEvent: () => {},
+		});
+		expect(result.text).toBe("four");
+		expect(ctx.completion).toHaveBeenCalledTimes(1);
+		expect(ctx.completion).toHaveBeenCalledWith(
+			expect.objectContaining({
+				prompt: "what is 2 + 2?",
+				n_predict: 8,
+				top_p: 0.9,
+				temperature: 0.5,
+				stop: ["</s>"],
+			}),
+		);
+	});
+
+	it("returns empty text without decoding when the signal is already aborted", async () => {
+		const ctx = fakeContext();
+		const runner = createCapacitorMtpTextRunner(ctx);
+		const controller = new AbortController();
+		controller.abort();
+		const result = await runner.generateWithVerifierEvents({
+			prompt: "hello",
+			signal: controller.signal,
+			onVerifierEvent: () => {},
+		});
+		expect(result.text).toBe("");
+		expect(ctx.completion).not.toHaveBeenCalled();
+	});
+
+	it("aborting mid-decode stops the on-device completion (barge-in)", async () => {
+		let resolveCompletion: ((v: { text: string }) => void) | null = null;
+		const ctx = fakeContext({
+			completion: vi.fn(
+				() =>
+					new Promise((resolve) => {
+						resolveCompletion = resolve as (v: { text: string }) => void;
+					}),
+			) as unknown as CapacitorLlamaContext["completion"],
+		});
+		const runner = createCapacitorMtpTextRunner(ctx);
+		const controller = new AbortController();
+		const pending = runner.generateWithVerifierEvents({
+			prompt: "a long answer",
+			signal: controller.signal,
+			onVerifierEvent: () => {},
+		});
+		controller.abort(); // barge-in
+		expect(ctx.stopCompletion).toHaveBeenCalledTimes(1);
+		resolveCompletion?.({ text: "partial" });
+		await expect(pending).resolves.toEqual({ text: "partial" });
+	});
+});
+
+describe("runDeviceVoiceTurn", () => {
+	it("builds the device text runner and drives engine.runVoiceTurn", async () => {
+		const ctx = fakeContext();
+		const events = { onComplete: vi.fn() };
+		const engine: DeviceVoiceEngine = {
+			runVoiceTurn: vi.fn(async (): Promise<VoiceTurnExitReason> => "done"),
+		};
+
+		const exit = await runDeviceVoiceTurn({
+			engine,
+			context: ctx,
+			audio: AUDIO,
+			events,
+			maxGeneratedTokens: 256,
+		});
+
+		expect(exit).toBe("done");
+		expect(engine.runVoiceTurn).toHaveBeenCalledTimes(1);
+		const [audioArg, optsArg] = (
+			engine.runVoiceTurn as unknown as { mock: { calls: unknown[][] } }
+		).mock.calls[0] as [TranscriptionAudio, Record<string, unknown>];
+		expect(audioArg).toBe(AUDIO);
+		expect(optsArg.events).toBe(events);
+		expect(optsArg.maxGeneratedTokens).toBe(256);
+		// The text runner handed to the engine is the device-context-backed one.
+		const runner = optsArg.textRunner as ReturnType<
+			typeof createCapacitorMtpTextRunner
+		>;
+		expect(runner.hasDrafter()).toBe(false);
+		await runner.generateWithVerifierEvents({
+			prompt: "x",
+			onVerifierEvent: () => {},
+		});
+		expect(ctx.completion).toHaveBeenCalledTimes(1);
+	});
+
+	it("fires the next-stage preload predictor from the pipeline's onAsrComplete (#8809 C5)", async () => {
+		const ctx = fakeContext();
+		const callerOnAsrComplete = vi.fn();
+		const onAsrStageComplete = vi.fn(async () => "eliza-1-4b" as string | null);
+		// Structural stand-in for VoicePreloadPredictor — only onAsrStageComplete
+		// is consumed by the wiring; type-cast keeps the test free of the arbiter.
+		const predictor = { onAsrStageComplete } as unknown as Parameters<
+			typeof runDeviceVoiceTurn
+		>[0]["preloadPredictor"];
+
+		const engine: DeviceVoiceEngine = {
+			runVoiceTurn: vi.fn(async (): Promise<VoiceTurnExitReason> => "done"),
+		};
+
+		await runDeviceVoiceTurn({
+			engine,
+			context: ctx,
+			audio: AUDIO,
+			events: { onAsrComplete: callerOnAsrComplete },
+			preloadPredictor: predictor,
+		});
+
+		const [, optsArg] = (
+			engine.runVoiceTurn as unknown as { mock: { calls: unknown[][] } }
+		).mock.calls[0] as [TranscriptionAudio, Record<string, unknown>];
+		const wiredEvents = optsArg.events as {
+			onAsrComplete?: (tokens: ReadonlyArray<unknown>) => void;
+		};
+
+		// The engine drives onAsrComplete the instant ASR finishes; the composed
+		// hook must fire BOTH the caller's hook and the predictor's prediction.
+		expect(onAsrStageComplete).not.toHaveBeenCalled();
+		wiredEvents.onAsrComplete?.([]);
+		expect(callerOnAsrComplete).toHaveBeenCalledTimes(1);
+		expect(onAsrStageComplete).toHaveBeenCalledTimes(1);
+	});
+
+	it("leaves events untouched when no predictor is supplied", async () => {
+		const events = { onComplete: vi.fn() };
+		const engine: DeviceVoiceEngine = {
+			runVoiceTurn: vi.fn(async (): Promise<VoiceTurnExitReason> => "done"),
+		};
+		await runDeviceVoiceTurn({
+			engine,
+			context: fakeContext(),
+			audio: AUDIO,
+			events,
+		});
+		const [, optsArg] = (
+			engine.runVoiceTurn as unknown as { mock: { calls: unknown[][] } }
+		).mock.calls[0] as [TranscriptionAudio, Record<string, unknown>];
+		expect(optsArg.events).toBe(events);
+	});
+
+	it("propagates the engine's exit reason (token-cap / cancelled)", async () => {
+		const engine: DeviceVoiceEngine = {
+			runVoiceTurn: vi.fn(
+				async (): Promise<VoiceTurnExitReason> => "cancelled",
+			),
+		};
+		const exit = await runDeviceVoiceTurn({
+			engine,
+			context: fakeContext(),
+			audio: AUDIO,
+		});
+		expect(exit).toBe("cancelled");
+	});
+});
+
+describe("NativePcmVoiceTurnCoordinator", () => {
+	it("rejects native turns before the capture lifecycle is started", async () => {
+		const coordinator = new NativePcmVoiceTurnCoordinator({
+			engine: {
+				runVoiceTurn: vi.fn(async (): Promise<VoiceTurnExitReason> => "done"),
+			},
+			context: fakeContext(),
+		});
+
+		await expect(coordinator.acceptTurn({ audio: AUDIO })).rejects.toThrow(
+			/start\(\)/,
+		);
+	});
+
+	it("serializes completed native PCM turns through runDeviceVoiceTurn", async () => {
+		const ctx = fakeContext();
+		let releaseFirst: ((reason: VoiceTurnExitReason) => void) | null = null;
+		const engine: DeviceVoiceEngine = {
+			runVoiceTurn: vi.fn(
+				() =>
+					new Promise<VoiceTurnExitReason>((resolve) => {
+						if (!releaseFirst) {
+							releaseFirst = resolve;
+							return;
+						}
+						resolve("token-cap");
+					}),
+			),
+		};
+		const coordinator = new NativePcmVoiceTurnCoordinator({
+			engine,
+			context: ctx,
+			maxGeneratedTokens: 128,
+		});
+
+		coordinator.start();
+		const first = coordinator.acceptTurn({ turnId: "turn-a", audio: AUDIO });
+		const secondAudio: TranscriptionAudio = {
+			pcm: new Float32Array([0.4, 0.5]),
+			sampleRate: 16_000,
+		};
+		const second = coordinator.acceptTurn({
+			turnId: "turn-b",
+			audio: secondAudio,
+			maxGeneratedTokens: 64,
+		});
+
+		await Promise.resolve();
+		expect(engine.runVoiceTurn).toHaveBeenCalledTimes(1);
+		releaseFirst?.("done");
+		await expect(first).resolves.toEqual({
+			turnId: "turn-a",
+			exitReason: "done",
+		});
+		await expect(second).resolves.toEqual({
+			turnId: "turn-b",
+			exitReason: "token-cap",
+		});
+		expect(engine.runVoiceTurn).toHaveBeenCalledTimes(2);
+
+		const firstCall = (
+			engine.runVoiceTurn as unknown as { mock: { calls: unknown[][] } }
+		).mock.calls[0] as [TranscriptionAudio, Record<string, unknown>];
+		const secondCall = (
+			engine.runVoiceTurn as unknown as { mock: { calls: unknown[][] } }
+		).mock.calls[1] as [TranscriptionAudio, Record<string, unknown>];
+		expect(firstCall[0]).toBe(AUDIO);
+		expect(firstCall[1].maxGeneratedTokens).toBe(128);
+		expect(secondCall[0]).toBe(secondAudio);
+		expect(secondCall[1].maxGeneratedTokens).toBe(64);
+
+		await coordinator.stop();
+		expect(coordinator.isRunning).toBe(false);
+	});
+});