@elizaos/plugin-local-inference 2.0.0-beta.1 → 2.0.11-beta.7

package/src/actions/identify-speaker.ts

@@ -0,0 +1,171 @@
+/**
+ * @module plugin-local-inference/actions/identify-speaker
+ *
+ * `IDENTIFY_SPEAKER` agent action — the explicit, user-driven half of the
+ * voice → entity binding (issue #8234, shape #2).
+ *
+ * When the OWNER names a voice the agent just heard but hasn't identified
+ * ("that was Jill", "this is my friend Sam"), this action binds the most
+ * recently observed *unidentified* speaker profile to a named person. It
+ * does not touch the entity graph directly — it emits `VOICE_TURN_OBSERVED`
+ * so the merge engine (plugin-lifeops) creates/merges the Entity, then the
+ * round-trip `VOICE_ENTITY_BOUND` handler persists `entityId` back onto the
+ * voice profile. If no merge-engine plugin is loaded the action is inert
+ * beyond logging intent.
+ *
+ * Target selection: an explicit `profileId` option wins; otherwise the
+ * single most-recently-observed profile whose `entityId` is still `null`
+ * (i.e. "the person who just spoke and isn't known yet").
+ */
+
+import {
+	type Action,
+	type ActionResult,
+	type HandlerCallback,
+	type IAgentRuntime,
+	logger,
+	type Memory,
+} from "@elizaos/core";
+import {
+	emitVoiceTurnObserved,
+	getVoiceProfileStore,
+} from "../runtime/voice-entity-binding.js";
+import type { VoiceProfileRecord } from "../services/voice/profile-store.js";
+
+function extractMessageText(message: Memory | null | undefined): string {
+	const content = message?.content;
+	if (!content) return "";
+	const text = (content as { text?: unknown }).text;
+	return typeof text === "string" ? text : "";
+}
+
+/**
+ * Extract a person name the owner is attaching to a heard voice. Mirrors
+ * the trigger phrases lifeops' `extractSelfNameClaim` understands so the
+ * downstream entity gets the same `preferredName`. Returns the name or
+ * `null`.
+ */
+const NAME = "[A-Z][A-Za-z'.-]{1,40}(?:\\s+[A-Z][A-Za-z'.-]{1,40}){0,2}";
+const SPEAKER_NAME_PATTERNS: RegExp[] = [
+	new RegExp(`\\bthat\\s+(?:was|is)\\s+(${NAME})\\b`, "i"),
+	new RegExp(`\\bthis\\s+is\\s+(?:my\\s+\\w+\\s+)?(${NAME})\\b`, "i"),
+	new RegExp(`\\bcall\\s+(?:him|her|them)\\s+(${NAME})\\b`, "i"),
+	new RegExp(`\\b(?:his|her|their)\\s+name\\s+is\\s+(${NAME})\\b`, "i"),
+	new RegExp(`\\bnamed?\\s+(${NAME})\\b`, "i"),
+	new RegExp(`\\bspeaker\\s+(?:was|is)\\s+(${NAME})\\b`, "i"),
+];
+
+export function extractSpeakerName(text: string): string | null {
+	for (const pattern of SPEAKER_NAME_PATTERNS) {
+		const m = pattern.exec(text);
+		if (m?.[1]) {
+			const cleaned = m[1].replace(/[.,;:!?]+$/, "").trim();
+			if (cleaned.length > 0) return cleaned;
+		}
+	}
+	return null;
+}
+
+function pickTarget(
+	records: VoiceProfileRecord[],
+	explicitProfileId: string | null,
+): VoiceProfileRecord | null {
+	if (explicitProfileId) {
+		return records.find((r) => r.profileId === explicitProfileId) ?? null;
+	}
+	const unbound = records
+		.filter((r) => r.entityId === null)
+		.sort((a, b) => b.lastObservedAt.localeCompare(a.lastObservedAt));
+	return unbound[0] ?? null;
+}
+
+function optionString(options: unknown, key: string): string | null {
+	if (!options || typeof options !== "object") return null;
+	const value = (options as Record<string, unknown>)[key];
+	return typeof value === "string" && value.trim().length > 0
+		? value.trim()
+		: null;
+}
+
+async function handler(
+	runtime: IAgentRuntime,
+	message: Memory,
+	_state?: unknown,
+	options?: unknown,
+	callback?: HandlerCallback,
+): Promise<ActionResult> {
+	const text = extractMessageText(message);
+	const name = optionString(options, "name") ?? extractSpeakerName(text);
+	if (!name) {
+		const out =
+			'Tell me the name to attach to the voice you just heard — e.g. "that was Jill".';
+		await callback?.({ text: out });
+		return { success: false, text: out };
+	}
+
+	const store = await getVoiceProfileStore();
+	const records = await store.list();
+	const target = pickTarget(records, optionString(options, "profileId"));
+	if (!target) {
+		const out = `I don't have an unidentified recent voice to attach "${name}" to yet.`;
+		await callback?.({ text: out });
+		return { success: false, text: out };
+	}
+
+	// Drive the merge engine through the event seam. `emitEvent` awaits all
+	// handlers, so when this resolves the round-trip binding has run.
+	try {
+		await emitVoiceTurnObserved(runtime, {
+			turnId: typeof message.id === "string" ? message.id : undefined,
+			text: `This is ${name}.`,
+			imprintClusterId: target.imprintClusterId,
+			matchConfidence: 1,
+			matchedEntityId: null,
+			isOwner: false,
+		});
+	} catch (err) {
+		logger.error(
+			{ err, profileId: target.profileId, name },
+			"[IDENTIFY_SPEAKER] failed to emit voice-turn observation",
+		);
+		const out = `I couldn't save ${name}'s voice just now.`;
+		await callback?.({ text: out });
+		return { success: false, text: out, error: out };
+	}
+
+	const updated = await store.get(target.profileId);
+	const entityId = updated?.entityId ?? null;
+	const out = entityId
+		? `Got it — I'll remember ${name}'s voice from now on.`
+		: `Noted ${name}. I'll bind the voice once identity sync is available.`;
+	await callback?.({ text: out });
+	return {
+		success: true,
+		text: out,
+		data: {
+			profileId: target.profileId,
+			imprintClusterId: target.imprintClusterId,
+			entityId,
+			name,
+		},
+	};
+}
+
+async function validate(
+	_runtime: IAgentRuntime,
+	message: Memory,
+): Promise<boolean> {
+	return extractMessageText(message).trim().length > 0;
+}
+
+export const identifySpeakerAction: Action = {
+	name: "IDENTIFY_SPEAKER",
+	similes: ["NAME_SPEAKER", "REMEMBER_VOICE", "THIS_IS_SPEAKER", "TAG_VOICE"],
+	description:
+		'Attach a name to the most recently heard, still-unidentified voice so the agent recognizes that person across sessions. Use when the owner says who a recent speaker is ("that was Jill", "this is my friend Sam").',
+	routingHint:
+		"owner names a recent unknown speaker -> IDENTIFY_SPEAKER; not for naming the owner themselves or contacts unrelated to a heard voice",
+	validate,
+	handler,
+	examples: [],
+};

package/src/adapters/capacitor-llama/__tests__/compat-behavior.test.ts

@@ -0,0 +1,218 @@
+import type { IAgentRuntime } from "@elizaos/core";
+import { ModelType } from "@elizaos/core";
+import { beforeEach, describe, expect, it, vi } from "vitest";
+import type {
+	CapacitorLlamaCompletionParams,
+	CapacitorLlamaCompletionResult,
+	CapacitorLlamaContext,
+	CapacitorLlamaTokenData,
+} from "../types";
+
+const mocks = vi.hoisted(() => ({
+	initCapacitorLlama: vi.fn(),
+}));
+
+vi.mock("../..", () => ({
+	createLocalInferenceModelHandlers: vi.fn(() => ({})),
+	isLocalInferenceUnavailableError: vi.fn(() => false),
+}));
+
+vi.mock("../loader", () => ({
+	initCapacitorLlama: mocks.initCapacitorLlama,
+}));
+
+const { localAiPlugin } = await import("../index");
+
+function makeCompletionResult(
+	text: string,
+	overrides: Partial<CapacitorLlamaCompletionResult> = {},
+): CapacitorLlamaCompletionResult {
+	return {
+		text,
+		content: text,
+		reasoning_content: "",
+		tool_calls: [],
+		chat_format: 0,
+		tokens_predicted: 2,
+		tokens_evaluated: 3,
+		truncated: false,
+		stopped_eos: true,
+		stopped_word: "",
+		stopped_limit: 0,
+		stopping_word: "",
+		context_full: false,
+		interrupted: false,
+		tokens_cached: 0,
+		timings: {
+			prompt_n: 0,
+			prompt_ms: 0,
+			prompt_per_token_ms: 0,
+			prompt_per_second: 0,
+			predicted_n: 2,
+			predicted_ms: 0,
+			predicted_per_token_ms: 0,
+			predicted_per_second: 0,
+		},
+		...overrides,
+	};
+}
+
+function makeCtx(
+	onCompletion?: (params: CapacitorLlamaCompletionParams) => void,
+): CapacitorLlamaContext {
+	return {
+		id: 1,
+		gpu: false,
+		reasonNoGPU: "",
+		model: {} as CapacitorLlamaContext["model"],
+		async completion(
+			params: CapacitorLlamaCompletionParams,
+			callback?: (data: CapacitorLlamaTokenData) => void,
+		): Promise<CapacitorLlamaCompletionResult> {
+			onCompletion?.(params);
+			callback?.({ token: "hel" });
+			callback?.({ token: "lo" });
+			return makeCompletionResult("hello");
+		},
+		stopCompletion: vi.fn(async () => undefined),
+		tokenize: vi.fn(async () => ({
+			tokens: [],
+			has_images: false,
+			bitmap_hashes: [],
+			chunk_pos: [],
+			chunk_pos_images: [],
+		})),
+		detokenize: vi.fn(async () => ""),
+		embedding: vi.fn(async (text: string) => ({
+			embedding: text === "embed me" ? [0.1, 0.2, 0.3] : [1],
+		})),
+		bench: vi.fn(async () => ({
+			modelDesc: "",
+			modelSize: 0,
+			modelNParams: 0,
+			ppAvg: 0,
+			ppStd: 0,
+			tgAvg: 0,
+			tgStd: 0,
+		})),
+		release: vi.fn(async () => undefined),
+	};
+}
+
+function makeRuntime(): IAgentRuntime {
+	return {
+		getSetting: vi.fn(() => undefined),
+		emitEvent: vi.fn(async () => undefined),
+	} as unknown as IAgentRuntime;
+}
+
+describe("local-ai compat adapter behavior", () => {
+	beforeEach(() => {
+		vi.clearAllMocks();
+		mocks.initCapacitorLlama.mockImplementation(async () => makeCtx());
+	});
+
+	it.each([
+		null,
+		"",
+		"   ",
+		{ text: "" },
+		{ text: "   " },
+	])("rejects empty embedding input %# instead of returning a fake vector", async (params) => {
+		await expect(
+			localAiPlugin.models?.[ModelType.TEXT_EMBEDDING]?.(
+				makeRuntime(),
+				params as never,
+			),
+		).rejects.toThrow("Embedding text must be a non-empty string");
+	});
+
+	it("routes non-empty embedding input to the real embedding context", async () => {
+		const result = await localAiPlugin.models?.[ModelType.TEXT_EMBEDDING]?.(
+			makeRuntime(),
+			{ text: "embed me" } as never,
+		);
+
+		expect(result).toEqual([0.1, 0.2, 0.3]);
+	});
+
+	it("wires onStreamChunk through the compat text adapter", async () => {
+		mocks.initCapacitorLlama.mockResolvedValueOnce(makeCtx());
+		const onStreamChunk = vi.fn();
+
+		const result = await localAiPlugin.models?.[ModelType.TEXT_SMALL]?.(
+			makeRuntime(),
+			{
+				prompt: "stream this",
+				stream: true,
+				onStreamChunk,
+			} as never,
+		);
+
+		for await (const _chunk of result.textStream) {
+			// drain
+		}
+
+		expect(onStreamChunk).toHaveBeenCalled();
+		expect(onStreamChunk.mock.calls.map(([chunk]) => chunk).join("")).toBe(
+			"hello",
+		);
+	});
+
+	it("sends a desktop-safe prompt and forwards sampler controls", async () => {
+		let completionParams: CapacitorLlamaCompletionParams | undefined;
+		mocks.initCapacitorLlama.mockResolvedValueOnce(
+			makeCtx((params) => {
+				completionParams = params;
+			}),
+		);
+
+		await localAiPlugin.models?.[ModelType.TEXT_SMALL]?.(makeRuntime(), {
+			system: "system prompt",
+			prompt: "user prompt",
+			maxTokens: 42,
+			temperature: 0.2,
+			topP: 0.8,
+			topK: 17,
+			minP: 0.05,
+			seed: 1234,
+			repetitionPenalty: 1.05,
+			frequencyPenalty: 0.3,
+			presencePenalty: 0.4,
+			stopSequences: ["</s>"],
+		} as never);
+
+		expect(completionParams?.prompt).toContain("system: system prompt");
+		expect(completionParams?.prompt).toContain("user: user prompt");
+		expect(completionParams?.messages).toBeUndefined();
+		expect(completionParams).toMatchObject({
+			n_predict: 42,
+			temperature: 0.2,
+			top_p: 0.8,
+			top_k: 17,
+			min_p: 0.05,
+			seed: 1234,
+			penalty_repeat: 1.05,
+			penalty_freq: 0.3,
+			penalty_present: 0.4,
+			stop: ["</s>"],
+		});
+	});
+
+	it("preserves function toolChoice objects as a Capacitor tool_choice", async () => {
+		let completionParams: CapacitorLlamaCompletionParams | undefined;
+		mocks.initCapacitorLlama.mockResolvedValueOnce(
+			makeCtx((params) => {
+				completionParams = params;
+			}),
+		);
+
+		await localAiPlugin.models?.[ModelType.TEXT_SMALL]?.(makeRuntime(), {
+			prompt: "use tool",
+			tools: [{ name: "lookup" }],
+			toolChoice: { type: "function", function: { name: "lookup" } },
+		} as never);
+
+		expect(completionParams?.tool_choice).toBe("lookup");
+	});
+});

package/src/adapters/capacitor-llama/__tests__/index.test.ts

@@ -0,0 +1,68 @@
+import { ModelType } from "@elizaos/core";
+import { describe, expect, it } from "vitest";
+import { configSchema } from "../environment";
+import { localAiPlugin } from "../index";
+import { MODEL_SPECS } from "../types";
+
+describe("Local AI Plugin (Capacitor-llama)", () => {
+	describe("Plugin metadata", () => {
+		it("registers as 'local-ai' for backwards compatibility", () => {
+			expect(localAiPlugin.name).toBe("local-ai");
+		});
+
+		it("describes itself as Capacitor-backed", () => {
+			expect(localAiPlugin.description).toBeDefined();
+			expect(localAiPlugin.description?.toLowerCase()).toContain("capacitor");
+		});
+
+		it("exposes an init function", () => {
+			expect(typeof localAiPlugin.init).toBe("function");
+		});
+	});
+
+	describe("Model handlers", () => {
+		it("registers TEXT_SMALL, TEXT_LARGE, TEXT_EMBEDDING", () => {
+			expect(typeof localAiPlugin.models?.TEXT_SMALL).toBe("function");
+			expect(typeof localAiPlugin.models?.TEXT_LARGE).toBe("function");
+			expect(typeof localAiPlugin.models?.TEXT_EMBEDDING).toBe("function");
+		});
+
+		it("registers other model types as compat thunks", () => {
+			expect(typeof localAiPlugin.models?.[ModelType.IMAGE_DESCRIPTION]).toBe(
+				"function",
+			);
+			expect(typeof localAiPlugin.models?.[ModelType.TRANSCRIPTION]).toBe(
+				"function",
+			);
+			expect(typeof localAiPlugin.models?.[ModelType.TEXT_TO_SPEECH]).toBe(
+				"function",
+			);
+		});
+	});
+
+	describe("Model bundle (MODEL_SPECS)", () => {
+		it("includes small, medium, and embedding specs", () => {
+			expect(MODEL_SPECS.small.name).toContain("eliza-1");
+			expect(MODEL_SPECS.medium.name).toContain("eliza-1");
+			// Embeddings use the compact gte-small model, not an eliza-1 chat GGUF.
+			expect(MODEL_SPECS.embedding.name).toContain("gte-small");
+		});
+
+		it("declares 384 embedding dimensions to match plugin-local-inference", () => {
+			expect(MODEL_SPECS.embedding.dimensions).toBe(384);
+		});
+
+		it("does not declare any node-llama-cpp-specific fields", () => {
+			// No tokenizer hash, no NAPI binding name — pure GGUF.
+			const json = JSON.stringify(MODEL_SPECS);
+			expect(json.toLowerCase()).not.toContain("node-llama-cpp");
+		});
+	});
+
+	describe("environment configSchema", () => {
+		it("parses an empty environment without throwing", () => {
+			const parsed = configSchema.parse({});
+			expect(parsed.LOCAL_SMALL_MODEL).toContain("eliza-1");
+		});
+	});
+});

package/src/adapters/capacitor-llama/__tests__/structured-output.test.ts

@@ -0,0 +1,215 @@
+import { describe, expect, it } from "vitest";
+import {
+	applyStructuredPlan,
+	buildCapacitorTools,
+	extractToolCalls,
+	planStructuredRequest,
+} from "../structured-output";
+import type {
+	CapacitorLlamaCompletionParams,
+	CapacitorLlamaCompletionResult,
+} from "../types";
+
+describe("capacitor-llama / structured-output", () => {
+	describe("planStructuredRequest", () => {
+		it("returns kind=text when no tools/schema/json supplied", () => {
+			const plan = planStructuredRequest({});
+			expect(plan.kind).toBe("text");
+			expect(plan.tools).toBeUndefined();
+			expect(plan.responseFormat).toBeUndefined();
+		});
+
+		it("prioritises tools over schema", () => {
+			const plan = planStructuredRequest({
+				tools: [
+					{ name: "foo", description: "f", parameters: { type: "object" } },
+				],
+				responseSchema: { type: "object" },
+			});
+			expect(plan.kind).toBe("tools");
+			expect(plan.tools).toBeDefined();
+		});
+
+		it("emits json_schema response_format for responseSchema", () => {
+			const plan = planStructuredRequest({
+				responseSchema: {
+					type: "object",
+					properties: { x: { type: "string" } },
+				},
+			});
+			expect(plan.kind).toBe("schema");
+			expect(plan.responseFormat?.type).toBe("json_schema");
+			expect(plan.responseFormat?.json_schema?.schema).toEqual({
+				type: "object",
+				properties: { x: { type: "string" } },
+			});
+		});
+
+		it("emits json_object response_format for responseFormat json_object", () => {
+			const plan = planStructuredRequest({
+				responseFormat: { type: "json_object" },
+			});
+			expect(plan.kind).toBe("json_object");
+			expect(plan.responseFormat?.type).toBe("json_object");
+		});
+
+		it("forwards toolChoice when tools present", () => {
+			const plan = planStructuredRequest({
+				tools: [{ name: "go", description: "go" }],
+				toolChoice: "auto",
+			});
+			expect(plan.kind).toBe("tools");
+			expect(plan.toolChoice).toBe("auto");
+		});
+	});
+
+	describe("buildCapacitorTools", () => {
+		it("skips tools without a name", () => {
+			const out = buildCapacitorTools([
+				{ name: "", description: "skip" },
+				{ name: "ok", description: "keep" },
+			]) as Array<{ function: { name: string } }>;
+			expect(out).toHaveLength(1);
+			expect(out[0].function.name).toBe("ok");
+		});
+
+		it("propagates description and parameters", () => {
+			const out = buildCapacitorTools([
+				{
+					name: "ping",
+					description: "pings",
+					parameters: { type: "object", properties: { x: { type: "number" } } },
+				},
+			]) as Array<{
+				type: "function";
+				function: {
+					name: string;
+					description?: string;
+					parameters?: unknown;
+				};
+			}>;
+			expect(out[0].type).toBe("function");
+			expect(out[0].function.description).toBe("pings");
+			expect(out[0].function.parameters).toEqual({
+				type: "object",
+				properties: { x: { type: "number" } },
+			});
+		});
+	});
+
+	describe("extractToolCalls", () => {
+		const baseResult = (
+			overrides: Partial<CapacitorLlamaCompletionResult>,
+		): CapacitorLlamaCompletionResult => ({
+			text: "",
+			reasoning_content: "",
+			tool_calls: [],
+			content: "",
+			chat_format: 0,
+			tokens_predicted: 0,
+			tokens_evaluated: 0,
+			truncated: false,
+			stopped_eos: false,
+			stopped_word: "",
+			stopped_limit: 0,
+			stopping_word: "",
+			context_full: false,
+			interrupted: false,
+			tokens_cached: 0,
+			timings: {
+				prompt_n: 0,
+				prompt_ms: 0,
+				prompt_per_token_ms: 0,
+				prompt_per_second: 0,
+				predicted_n: 0,
+				predicted_ms: 0,
+				predicted_per_token_ms: 0,
+				predicted_per_second: 0,
+			},
+			...overrides,
+		});
+
+		it("parses valid JSON arguments", () => {
+			const calls = extractToolCalls(
+				baseResult({
+					tool_calls: [
+						{
+							type: "function",
+							id: "call_a",
+							function: { name: "foo", arguments: '{"x":1,"y":"z"}' },
+						},
+					],
+				}),
+			);
+			expect(calls).toEqual([
+				{
+					id: "call_a",
+					name: "foo",
+					arguments: { x: 1, y: "z" },
+					type: "function",
+				},
+			]);
+		});
+
+		it("synthesises a call id when missing", () => {
+			const calls = extractToolCalls(
+				baseResult({
+					tool_calls: [
+						{ type: "function", function: { name: "f", arguments: "{}" } },
+						{ type: "function", function: { name: "g", arguments: "{}" } },
+					],
+				}),
+			);
+			expect(calls.map((c) => c.id)).toEqual(["call_0", "call_1"]);
+		});
+
+		it("surfaces invalid JSON as _raw rather than dropping the call", () => {
+			const calls = extractToolCalls(
+				baseResult({
+					tool_calls: [
+						{
+							type: "function",
+							id: "broken",
+							function: { name: "x", arguments: "not-json" },
+						},
+					],
+				}),
+			);
+			expect(calls).toHaveLength(1);
+			expect(calls[0].arguments).toEqual({ _raw: "not-json" });
+		});
+	});
+
+	describe("applyStructuredPlan", () => {
+		const base: CapacitorLlamaCompletionParams = {
+			prompt: "hi",
+			n_predict: 32,
+		};
+
+		it("returns params untouched for text plans", () => {
+			const out = applyStructuredPlan(base, { kind: "text" });
+			expect(out).toEqual(base);
+		});
+
+		it("merges tools + tool_choice for tool plans", () => {
+			const out = applyStructuredPlan(base, {
+				kind: "tools",
+				tools: [{ name: "foo" }],
+				toolChoice: "auto",
+			});
+			expect(out.tools).toEqual([{ name: "foo" }]);
+			expect(out.tool_choice).toBe("auto");
+		});
+
+		it("merges response_format for schema plans", () => {
+			const out = applyStructuredPlan(base, {
+				kind: "schema",
+				responseFormat: {
+					type: "json_schema",
+					json_schema: { strict: true, schema: { type: "object" } },
+				},
+			});
+			expect(out.response_format?.type).toBe("json_schema");
+		});
+	});
+});