@elizaos/plugin-local-inference 2.0.3-beta.2 → 2.0.3-beta.3

package/src/runtime/mobile-local-inference-gate.test.ts

@@ -1,5 +1,8 @@
-import { describe, expect, it } from "vitest";
-import { shouldEnableMobileLocalInference } from "./mobile-local-inference-gate";
+import { describe, expect, it, vi } from "vitest";
+import {
+	shouldEnableMobileLocalInference,
+	warnIfMobileGateActiveWithoutPlatform,
+} from "./mobile-local-inference-gate";
 
 describe("shouldEnableMobileLocalInference", () => {
 	it("returns false when no env flags and arch is not riscv64", () => {
@@ -22,6 +25,31 @@ describe("shouldEnableMobileLocalInference", () => {
 		).toBe(true);
 	});
 
+	it("returns true when ELIZA_BIONIC_HOST_DELEGATED=1 (Android bionic delegation)", () => {
+		// The agent delegates inference to the in-process bionic Vulkan host over
+		// UDS; without this trigger ensureLocalInferenceHandler is skipped on the
+		// phone, so the bionic loader + TTS/TRANSCRIPTION/IMAGE_DESCRIPTION
+		// handlers never register and only the direct-reply chat path works (#8848).
+		expect(
+			shouldEnableMobileLocalInference(
+				{ ELIZA_BIONIC_HOST_DELEGATED: "1" },
+				"arm64",
+			),
+		).toBe(true);
+	});
+
+	it("ELIZA_DISABLE_FFI_LLAMA=1 does not block the bionic-host path (process-external UDS)", () => {
+		expect(
+			shouldEnableMobileLocalInference(
+				{
+					ELIZA_DISABLE_FFI_LLAMA: "1",
+					ELIZA_BIONIC_HOST_DELEGATED: "1",
+				},
+				"arm64",
+			),
+		).toBe(true);
+	});
+
 	it("auto-fires on riscv64 with no env flags", () => {
 		// node-llama-cpp has no riscv64 prebuild; the FFI loader (which dlopens
 		// the cross-built libllama.so) is the only in-process llama.cpp path
@@ -67,3 +95,58 @@ describe("shouldEnableMobileLocalInference", () => {
 		).toBe(false);
 	});
 });
+
+describe("warnIfMobileGateActiveWithoutPlatform", () => {
+	it("warns when the gate is active (ELIZA_LOCAL_LLAMA=1) but ELIZA_PLATFORM is unset", () => {
+		const warn = vi.fn();
+		const fired = warnIfMobileGateActiveWithoutPlatform({
+			mobilePlatform: false,
+			warn,
+			env: { ELIZA_LOCAL_LLAMA: "1" },
+			arch: "arm64",
+		});
+		expect(fired).toBe(true);
+		expect(warn).toHaveBeenCalledTimes(1);
+		const message = warn.mock.calls[0]?.[0] ?? "";
+		expect(message).toContain("ELIZA_PLATFORM");
+		expect(message).toContain("Kokoro-exclusive mobile");
+	});
+
+	it("warns when the gate is active via the device bridge and ELIZA_PLATFORM is unset", () => {
+		const warn = vi.fn();
+		const fired = warnIfMobileGateActiveWithoutPlatform({
+			mobilePlatform: false,
+			warn,
+			env: { ELIZA_DEVICE_BRIDGE_ENABLED: "1" },
+			arch: "arm64",
+		});
+		expect(fired).toBe(true);
+		expect(warn).toHaveBeenCalledTimes(1);
+	});
+
+	it("does NOT warn when ELIZA_PLATFORM=android (mobilePlatform true)", () => {
+		// On a real phone the platform flag is set, so isMobilePlatform() is true
+		// and the selector already pins Kokoro — no mismatch to report.
+		const warn = vi.fn();
+		const fired = warnIfMobileGateActiveWithoutPlatform({
+			mobilePlatform: true,
+			warn,
+			env: { ELIZA_LOCAL_LLAMA: "1", ELIZA_PLATFORM: "android" },
+			arch: "arm64",
+		});
+		expect(fired).toBe(false);
+		expect(warn).not.toHaveBeenCalled();
+	});
+
+	it("does NOT warn when the gate is inactive (desktop x64, no flags)", () => {
+		const warn = vi.fn();
+		const fired = warnIfMobileGateActiveWithoutPlatform({
+			mobilePlatform: false,
+			warn,
+			env: {},
+			arch: "x64",
+		});
+		expect(fired).toBe(false);
+		expect(warn).not.toHaveBeenCalled();
+	});
+});

package/src/runtime/mobile-local-inference-gate.ts

@@ -31,14 +31,69 @@ export function shouldEnableMobileLocalInference(
 	env: NodeJS.ProcessEnv = process.env,
 	arch: NodeJS.Architecture = process.arch,
 ): boolean {
+	// Bionic-host delegation (Android dynamic-Vulkan): the app shell stages a
+	// dynamic-Vulkan `libelizainference.so` reachable only from the in-process
+	// bionic host and sets `ELIZA_BIONIC_HOST_DELEGATED=1` (suppressing
+	// ELIZA_LOCAL_LLAMA so the musl agent never dlopen's the lib). The agent talks
+	// to that host over the abstract UDS — a process-external backend just like
+	// the device bridge. Without counting it here, ensureLocalInferenceHandler is
+	// skipped on the phone, so `tryRegisterBionicHostLoader` + the
+	// TEXT/embedding (and TTS/TRANSCRIPTION/IMAGE_DESCRIPTION) handlers never
+	// register and only the mobile-local-direct-reply chat path works (#8848).
+	const bionicHost = env.ELIZA_BIONIC_HOST_DELEGATED?.trim() === "1";
 	if (env.ELIZA_DISABLE_FFI_LLAMA?.trim() === "1") {
-		// Operator opted out of the FFI path entirely — the device-bridge
-		// path remains valid because the bridge is process-external and
-		// doesn't depend on `libllama.so` being present in the APK.
-		return env.ELIZA_DEVICE_BRIDGE_ENABLED?.trim() === "1";
+		// Operator opted out of the FFI path entirely — the device-bridge and the
+		// in-process bionic host are both process-external (WS / UDS) and don't
+		// depend on `libllama.so` being dlopen'd in this musl process.
+		return env.ELIZA_DEVICE_BRIDGE_ENABLED?.trim() === "1" || bionicHost;
 	}
 	const deviceBridge = env.ELIZA_DEVICE_BRIDGE_ENABLED?.trim() === "1";
 	const localLlama = env.ELIZA_LOCAL_LLAMA?.trim() === "1";
 	const riscv64Auto = arch === "riscv64";
-	return deviceBridge || localLlama || riscv64Auto;
+	return deviceBridge || localLlama || riscv64Auto || bionicHost;
+}
+
+/**
+ * Boot-time invariant check for the mobile voice backend selector.
+ *
+ * The voice engine passes `mobile: isMobilePlatform()` into `selectVoiceBackend`
+ * (`services/engine.ts`). That flag is what pins mobile to the Kokoro-exclusive
+ * TTS path — OmniVoice is never shipped on a phone. But `isMobilePlatform()`
+ * keys solely off `ELIZA_PLATFORM ∈ {ios, android}`, whereas
+ * {@link shouldEnableMobileLocalInference} also fires on the device-bridge /
+ * `ELIZA_LOCAL_LLAMA` / riscv64 triggers — none of which set `ELIZA_PLATFORM`.
+ * So a build that enables the mobile local-inference gate without exporting
+ * `ELIZA_PLATFORM` leaves `mobile` false in the selector and could pick
+ * OmniVoice on a phone, violating the Kokoro-exclusive mobile voice invariant.
+ *
+ * This emits exactly one boot-time warning when the gate is active but the
+ * platform flag is absent (the real mismatch). It is a diagnostic, not a
+ * behavior change: the fix is to export `ELIZA_PLATFORM` from the offending
+ * build, which this surfaces. The caller MUST evaluate it on a boot path where
+ * BOTH predicates are checked (i.e. outside any `if (isMobilePlatform())`
+ * branch) — passing the already-computed `mobilePlatform` makes that explicit.
+ *
+ * Kept dependency-free (injectable `warn` + the already-resolved
+ * `mobilePlatform` boolean) so it can be unit-tested without the runtime or a
+ * logger instance, matching the `engine.warnIfParallelTooLow({ warn })` pattern.
+ *
+ * @returns `true` when the warning fired (the mismatch is real), else `false`.
+ */
+export function warnIfMobileGateActiveWithoutPlatform(args: {
+	mobilePlatform: boolean;
+	warn: (message: string) => void;
+	env?: NodeJS.ProcessEnv;
+	arch?: NodeJS.Architecture;
+}): boolean {
+	const { mobilePlatform, warn, env = process.env, arch = process.arch } = args;
+	if (mobilePlatform) return false;
+	if (!shouldEnableMobileLocalInference(env, arch)) return false;
+	warn(
+		"[local-inference] ELIZA_PLATFORM is not set while the mobile local-inference gate is active " +
+			"(ELIZA_DEVICE_BRIDGE_ENABLED / ELIZA_LOCAL_LLAMA / riscv64), so the Kokoro-exclusive mobile " +
+			"voice invariant may be missed. The voice backend selector keys `mobile` off " +
+			"ELIZA_PLATFORM ∈ {ios,android}; with it unset the selector may pick OmniVoice on a phone. " +
+			"Set ELIZA_PLATFORM=android (or ios) on this build.",
+	);
+	return true;
 }

package/src/runtime/voice-entity-binding.transcript.test.ts

@@ -66,4 +66,33 @@ describe("handleLiveVoiceAttribution — transcript carry (#8786)", () => {
 		});
 		expect(signal.transcript).toBe("Jill is my wife");
 	});
+
+	it("stamps the resolved speaker entityId onto the turn metadata (#8786)", async () => {
+		const { runtime } = captureRuntime();
+		const out = output();
+		await handleLiveVoiceAttribution(runtime, out, {
+			ownerEntityId: "entity-jill",
+			transcript: "I'm Jill",
+		});
+		// The imprint → entityId match rides on the message so providers/extraction
+		// attribute the turn to the right person, not just the EOT gate.
+		const meta = out.turn.metadata as Record<string, unknown>;
+		expect(meta.speakerEntityId).toBe("entity-jill");
+		expect(meta.voiceTurnSignal).toBeDefined();
+	});
+
+	it("omits speakerEntityId for an unbound speaker (never writes a null speaker)", async () => {
+		const { runtime } = captureRuntime();
+		const out = {
+			turnId: "t2",
+			primarySpeaker: { entityId: null, confidence: 0.2 },
+			observation: undefined,
+			turn: { metadata: {} },
+			segments: [],
+		} as unknown as VoiceAttributionOutput;
+		await handleLiveVoiceAttribution(runtime, out, {});
+		const meta = out.turn.metadata as Record<string, unknown>;
+		expect(meta.speakerEntityId).toBeUndefined();
+		expect(meta.voiceTurnSignal).toBeDefined();
+	});
 });

package/src/runtime/voice-entity-binding.ts

@@ -24,6 +24,10 @@ import {
 	resolveStateDir,
 	type VoiceEntityBoundPayload,
 } from "@elizaos/core";
+import {
+	AGENT_SELF_VOICE_THRESHOLD,
+	ECHO_WINDOW_MS,
+} from "@elizaos/shared/voice/respond-gate";
 import type {
 	VoiceNextSpeaker,
 	VoiceTurnSignal,
@@ -137,6 +141,16 @@ export interface HandleLiveVoiceAttributionOptions {
 	 * (which has both ASR + diarization) passes the real transcript.
 	 */
 	transcript?: string;
+	/**
+	 * Cosine similarity (0..1) between this turn's live WeSpeaker embedding and
+	 * the agent's own TTS-voice centroid. High means the open mic is hearing the
+	 * agent's playback, not a human user.
+	 */
+	selfVoiceSimilarity?: number | null;
+	/** True while the agent is currently playing TTS. */
+	agentSpeaking?: boolean;
+	/** Age of the most recent agent-spoken reply in ms. */
+	replyAgeMs?: number;
 }
 
 /**
@@ -197,6 +211,15 @@ function foldSpeakerIntoSignal(
 	// Wake word overrides bystander doubt — the user deliberately summoned us.
 	if (opts.wakeWordActive === true) agentShouldSpeak = true;
 
+	const replyRecent =
+		opts.agentSpeaking === true ||
+		(opts.replyAgeMs ?? Number.POSITIVE_INFINITY) <= ECHO_WINDOW_MS;
+	const isSelfVoice =
+		typeof opts.selfVoiceSimilarity === "number" &&
+		opts.selfVoiceSimilarity >= AGENT_SELF_VOICE_THRESHOLD &&
+		replyRecent;
+	if (isSelfVoice) agentShouldSpeak = false;
+
 	const eot = base.endOfTurnProbability;
 	const nextSpeaker: VoiceNextSpeaker = !agentShouldSpeak
 		? "user"
@@ -204,9 +227,11 @@ function foldSpeakerIntoSignal(
 			? "user"
 			: "agent";
 
-	const source = opts.wakeWordActive
-		? "voice-bridge+wakeword"
-		: "voice-bridge+diarization";
+	const source = isSelfVoice
+		? "voice-bridge+self-voice"
+		: opts.wakeWordActive
+			? "voice-bridge+wakeword"
+			: "voice-bridge+diarization";
 
 	return {
 		endOfTurnProbability: eot,
@@ -218,8 +243,15 @@ function foldSpeakerIntoSignal(
 		...(base.latencyMs !== undefined ? { latencyMs: base.latencyMs } : {}),
 		// Stash the human-readable provenance so traces show the fold source even
 		// though the typed `source` enum stays "custom".
-		metadata: { provenance: source },
-	} as VoiceTurnSignal & { metadata: { provenance: string } };
+		metadata: {
+			provenance: source,
+			...(typeof opts.selfVoiceSimilarity === "number"
+				? { selfVoiceSimilarity: opts.selfVoiceSimilarity }
+				: {}),
+		},
+	} as VoiceTurnSignal & {
+		metadata: { provenance: string; selfVoiceSimilarity?: number };
+	};
 }
 
 /**
@@ -282,7 +314,15 @@ export async function handleLiveVoiceAttribution(
 	const signal = foldSpeakerIntoSignal(base, standing, opts);
 
 	const turn = output.turn;
-	turn.metadata = { ...(turn.metadata ?? {}), voiceTurnSignal: signal };
+	// Stamp the resolved speaker entity onto the turn (#8786): the imprint →
+	// entityId match rides with the transcript so the server/providers/extraction
+	// attribute the turn to the right person (not just the EOT gate). Omitted when
+	// the speaker is unbound (`entityId == null`) — never write a null speaker.
+	turn.metadata = {
+		...(turn.metadata ?? {}),
+		voiceTurnSignal: signal,
+		...(standing.entityId ? { speakerEntityId: standing.entityId } : {}),
+	};
 
 	return signal;
 }

package/src/runtime/voice-speaker-entity-contract.test.ts

@@ -0,0 +1,149 @@
+import type { IAgentRuntime, Memory } from "@elizaos/core";
+import { ChannelType, type UUID } from "@elizaos/core";
+import { describe, expect, it } from "vitest";
+import type { VoiceAttributionOutput } from "../services/voice/speaker/attribution-pipeline.js";
+import { handleLiveVoiceAttribution } from "./voice-entity-binding.js";
+
+/**
+ * Producer ⇄ consumer contract for the resolved-speaker stamp (#8786, commit
+ * 2cea841d8a).
+ *
+ * The PRODUCER is `handleLiveVoiceAttribution` (this plugin): it stamps the
+ * imprint → entityId match onto `turn.metadata.speakerEntityId`. That turn
+ * metadata is what rides onto the message's `content.metadata` for a VOICE_DM.
+ *
+ * The CONSUMER is the core message handler's private `getVoiceSpeakerEntityId`
+ * reader (`packages/core/src/services/message.ts`), which — per its documented
+ * contract — reads the id from EITHER `content.speakerEntityId` (the top-level
+ * in-process engine path) OR `content.metadata.speakerEntityId` (the nested
+ * chat-client path), then canonicalizes it onto `content.metadata.speaker
+ * EntityId` of the voice turn so providers / extraction / the facts +
+ * relationships stage attribute the turn to the right person.
+ *
+ * The core reader is not exported, so this test locks the contract at the seam
+ * that matters: it (1) runs the real producer, then (2) asserts the producer
+ * writes the id at the exact `metadata.speakerEntityId` key the documented
+ * reader consumes, and (3) exercises a reference of the reader contract against
+ * a producer-shaped VOICE_DM message to prove a recognized speaker's entityId
+ * lands on the message. A future rename of the key on either half breaks this.
+ */
+
+const ROOM = "11111111-1111-1111-1111-111111111111" as UUID;
+const ENTITY = "55555555-5555-5555-5555-555555555555" as UUID;
+
+/**
+ * Reference of the core `getVoiceSpeakerEntityId` reader contract (kept in sync
+ * with `packages/core/src/services/message.ts`). Reads top-level first, then
+ * the nested `content.metadata` entry; trims; null for an unbound speaker.
+ */
+function readSpeakerEntityIdContract(
+	message: Pick<Memory, "content">,
+): string | null {
+	const content = message.content;
+	const nested =
+		content?.metadata &&
+		typeof content.metadata === "object" &&
+		!Array.isArray(content.metadata)
+			? (content.metadata as Record<string, unknown>).speakerEntityId
+			: undefined;
+	const value =
+		(content as { speakerEntityId?: unknown } | undefined)?.speakerEntityId ??
+		nested;
+	if (typeof value !== "string") return null;
+	const trimmed = value.trim();
+	return trimmed.length > 0 ? trimmed : null;
+}
+
+function captureRuntime(): IAgentRuntime {
+	return {
+		emitEvent: async () => {},
+	} as unknown as IAgentRuntime;
+}
+
+function boundOutput(entityId: string): VoiceAttributionOutput {
+	return {
+		turnId: "t-bound",
+		primarySpeaker: { entityId, confidence: 0.7 },
+		observation: { imprintClusterId: "cluster-1", confidence: 0.7, entityId },
+		turn: { metadata: {} },
+		segments: [],
+	} as unknown as VoiceAttributionOutput;
+}
+
+/** Build the VOICE_DM the nested chat-client path produces from a turn. */
+function voiceDmFromTurn(turnMetadata: Record<string, unknown>): Memory {
+	return {
+		id: "66666666-6666-6666-6666-666666666666" as UUID,
+		entityId: ENTITY,
+		roomId: ROOM,
+		content: {
+			text: "what's on my calendar",
+			channelType: ChannelType.VOICE_DM,
+			metadata: turnMetadata,
+		},
+	} as Memory;
+}
+
+describe("voice speaker-entity producer ⇄ consumer contract (#8786)", () => {
+	it("producer stamps the id at the exact key the core reader consumes", async () => {
+		const runtime = captureRuntime();
+		const out = boundOutput("entity-jill");
+		await handleLiveVoiceAttribution(runtime, out, {
+			ownerEntityId: "entity-jill",
+			transcript: "I'm Jill",
+		});
+		// The reader contract resolves the producer's turn metadata directly.
+		const message = voiceDmFromTurn(
+			out.turn.metadata as Record<string, unknown>,
+		);
+		expect(readSpeakerEntityIdContract(message)).toBe("entity-jill");
+	});
+
+	it("a recognized speaker's entityId lands on the VOICE_DM message (nested entry)", () => {
+		const message = voiceDmFromTurn({
+			voiceTurnSignal: { agentShouldSpeak: true },
+			speakerEntityId: "entity-bob",
+		});
+		expect(readSpeakerEntityIdContract(message)).toBe("entity-bob");
+	});
+
+	it("resolves the top-level in-process entry too", () => {
+		const message = {
+			id: "77777777-7777-7777-7777-777777777777" as UUID,
+			entityId: ENTITY,
+			roomId: ROOM,
+			content: {
+				text: "remind me later",
+				channelType: ChannelType.VOICE_DM,
+				speakerEntityId: "entity-top",
+				metadata: {},
+			},
+		} as unknown as Memory;
+		expect(readSpeakerEntityIdContract(message)).toBe("entity-top");
+	});
+
+	it("an unbound speaker resolves to null (never a null speaker on the message)", async () => {
+		const runtime = captureRuntime();
+		const out = {
+			turnId: "t-unbound",
+			primarySpeaker: { entityId: null, confidence: 0.2 },
+			observation: undefined,
+			turn: { metadata: {} },
+			segments: [],
+		} as unknown as VoiceAttributionOutput;
+		await handleLiveVoiceAttribution(runtime, out, {});
+		const message = voiceDmFromTurn(
+			out.turn.metadata as Record<string, unknown>,
+		);
+		expect(readSpeakerEntityIdContract(message)).toBeNull();
+	});
+
+	it("a blank / whitespace id resolves to null", () => {
+		expect(
+			readSpeakerEntityIdContract(voiceDmFromTurn({ speakerEntityId: "   " })),
+		).toBeNull();
+		expect(
+			readSpeakerEntityIdContract(voiceDmFromTurn({ speakerEntityId: "" })),
+		).toBeNull();
+	});
+});

package/src/services/README.md

@@ -45,10 +45,10 @@ and the local files in this directory are thin re-exports:
 ## What stays a twin (and why)
 
 These files exist in both `packages/app-core` and `packages/ui` and are
-intentionally **not bundled:
+intentionally **not bundled**:
 
 - `catalog.ts` — server adds `contextLength`, `optimizations.requiresKernel`,
-  and MTP drafter variants that the UI public catalog does not surface.
+  and same-file MTP metadata that the UI public catalog does not surface.
 - `recommendation.ts` — server has kernel-availability filtering
   (`recommendation.test.ts` covers MTP gating) that depends on
   server-only `LocalRuntimeOptimizations.requiresKernel`.