@elizaos/plugin-local-inference 2.0.0-beta.1 → 2.0.11-beta.7

package/src/runtime/index.d.ts

@@ -0,0 +1,14 @@
+/**
+ * Runtime-side exports for plugin-local-inference.
+ *
+ * Consumers (app-core/runtime/eliza.ts, agent bootstrap) import from
+ * `@elizaos/plugin-local-inference/runtime` to wire boot-time handler
+ * registration, embedding warm-up policy, and the mobile inference gate.
+ */
+export { DEFAULT_MODELS_DIR, type EmbeddingProgressCallback, embeddingGgufFilePresent, ensureModel, findExistingEmbeddingModelForWarmupReuse, isEmbeddingWarmupReuseDisabled, } from "./embedding-manager-support.js";
+export { detectEmbeddingPreset } from "./embedding-presets.js";
+export { shouldWarmupLocalEmbeddingModel } from "./embedding-warmup-policy.js";
+export { ensureLocalInferenceHandler } from "./ensure-local-inference-handler.js";
+export { shouldEnableMobileLocalInference } from "./mobile-local-inference-gate.js";
+export { type EmitVoiceTurnObservedArgs, emitVoiceTurnObserved, getVoiceProfileStore, handleVoiceEntityBound, setVoiceEntityBindingStore, } from "./voice-entity-binding.js";
+//# sourceMappingURL=index.d.ts.map

package/src/runtime/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["index.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAEH,OAAO,EACN,kBAAkB,EAClB,KAAK,yBAAyB,EAC9B,wBAAwB,EACxB,WAAW,EACX,wCAAwC,EACxC,8BAA8B,GAC9B,MAAM,gCAAgC,CAAC;AACxC,OAAO,EAAE,qBAAqB,EAAE,MAAM,wBAAwB,CAAC;AAC/D,OAAO,EAAE,+BAA+B,EAAE,MAAM,8BAA8B,CAAC;AAC/E,OAAO,EAAE,2BAA2B,EAAE,MAAM,qCAAqC,CAAC;AAClF,OAAO,EAAE,gCAAgC,EAAE,MAAM,kCAAkC,CAAC;AACpF,OAAO,EACN,KAAK,yBAAyB,EAC9B,qBAAqB,EACrB,oBAAoB,EACpB,sBAAsB,EACtB,0BAA0B,GAC1B,MAAM,2BAA2B,CAAC"}

package/src/runtime/index.ts

@@ -0,0 +1,27 @@
+/**
+ * Runtime-side exports for plugin-local-inference.
+ *
+ * Consumers (app-core/runtime/eliza.ts, agent bootstrap) import from
+ * `@elizaos/plugin-local-inference/runtime` to wire boot-time handler
+ * registration, embedding warm-up policy, and the mobile inference gate.
+ */
+
+export {
+	DEFAULT_MODELS_DIR,
+	type EmbeddingProgressCallback,
+	embeddingGgufFilePresent,
+	ensureModel,
+	findExistingEmbeddingModelForWarmupReuse,
+	isEmbeddingWarmupReuseDisabled,
+} from "./embedding-manager-support.js";
+export { detectEmbeddingPreset } from "./embedding-presets.js";
+export { shouldWarmupLocalEmbeddingModel } from "./embedding-warmup-policy.js";
+export { ensureLocalInferenceHandler } from "./ensure-local-inference-handler.js";
+export { shouldEnableMobileLocalInference } from "./mobile-local-inference-gate.js";
+export {
+	type EmitVoiceTurnObservedArgs,
+	emitVoiceTurnObserved,
+	getVoiceProfileStore,
+	handleVoiceEntityBound,
+	setVoiceEntityBindingStore,
+} from "./voice-entity-binding.js";

package/src/runtime/mobile-local-inference-gate.d.ts

@@ -0,0 +1,31 @@
+/**
+ * On a mobile platform (`ELIZA_PLATFORM=android` / `ios`) the runtime skips
+ * nearly every boot helper because they shell out to subprocesses,
+ * platform-specific binaries, or optional packages that aren't in the mobile
+ * bundle. Three mobile-safe inference paths need wiring:
+ *
+ *   - `ELIZA_DEVICE_BRIDGE_ENABLED=1`: the agent (this process) hosts the
+ *     device-bridge WSS and dials whichever paired device connects. On the
+ *     Capacitor APK the WebView's `@elizaos/capacitor-llama` is the intended
+ *     dialer over loopback. The Capacitor build always exports this env so
+ *     the bridge is ready as soon as first-run picks the local mode.
+ *
+ *   - `ELIZA_LOCAL_LLAMA=1`: AOSP path that loads `libllama.so` directly
+ *     inside the Android process via `bun:ffi`. Wired here so the gate is
+ *     in place ahead of sub-task 2 — the AOSP build flag flips this on.
+ *
+ *   - `process.arch === "riscv64"`: `capacitor-llama` has no riscv64 prebuild
+ *     and we can't NAPI-build it on-device, so the in-process FFI path
+ *     (same loader contract as the AOSP path) is the only viable option.
+ *     Auto-firing here keeps the riscv64 mobile boot path zero-config; an
+ *     operator can hard-disable via `ELIZA_DISABLE_FFI_LLAMA=1` to skip the
+ *     loader and route inference through Cloud instead. See
+ *     `plugin-aosp-local-inference/src/aosp-llama-adapter.ts:isAospEnabled`
+ *     and `plugin-local-inference/src/runtime/ensure-local-inference-handler.ts:shouldAttemptAospLlamaLoader`
+ *     — the three predicates agree on the trigger set.
+ *
+ * Kept dependency-free so it can be unit-tested without instantiating the
+ * full runtime.
+ */
+export declare function shouldEnableMobileLocalInference(env?: NodeJS.ProcessEnv, arch?: NodeJS.Architecture): boolean;
+//# sourceMappingURL=mobile-local-inference-gate.d.ts.map

package/src/runtime/mobile-local-inference-gate.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"mobile-local-inference-gate.d.ts","sourceRoot":"","sources":["mobile-local-inference-gate.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA4BG;AACH,wBAAgB,gCAAgC,CAC/C,GAAG,GAAE,MAAM,CAAC,UAAwB,EACpC,IAAI,GAAE,MAAM,CAAC,YAA2B,GACtC,OAAO,CAWT"}

package/src/runtime/mobile-local-inference-gate.test.ts

@@ -0,0 +1,69 @@
+import { describe, expect, it } from "vitest";
+import { shouldEnableMobileLocalInference } from "./mobile-local-inference-gate";
+
+describe("shouldEnableMobileLocalInference", () => {
+	it("returns false when no env flags and arch is not riscv64", () => {
+		expect(shouldEnableMobileLocalInference({}, "x64")).toBe(false);
+		expect(shouldEnableMobileLocalInference({}, "arm64")).toBe(false);
+	});
+
+	it("returns true when ELIZA_DEVICE_BRIDGE_ENABLED=1", () => {
+		expect(
+			shouldEnableMobileLocalInference(
+				{ ELIZA_DEVICE_BRIDGE_ENABLED: "1" },
+				"x64",
+			),
+		).toBe(true);
+	});
+
+	it("returns true when ELIZA_LOCAL_LLAMA=1", () => {
+		expect(
+			shouldEnableMobileLocalInference({ ELIZA_LOCAL_LLAMA: "1" }, "x64"),
+		).toBe(true);
+	});
+
+	it("auto-fires on riscv64 with no env flags", () => {
+		// node-llama-cpp has no riscv64 prebuild; the FFI loader (which dlopens
+		// the cross-built libllama.so) is the only in-process llama.cpp path
+		// available on riscv64, so the gate auto-fires there.
+		expect(shouldEnableMobileLocalInference({}, "riscv64")).toBe(true);
+	});
+
+	it("ELIZA_DISABLE_FFI_LLAMA=1 hard-disables the riscv64 auto-fire", () => {
+		// Operator opt-out: route inference to Cloud instead of the on-device
+		// FFI path. The device-bridge is process-external and unaffected.
+		expect(
+			shouldEnableMobileLocalInference(
+				{ ELIZA_DISABLE_FFI_LLAMA: "1" },
+				"riscv64",
+			),
+		).toBe(false);
+	});
+
+	it("ELIZA_DISABLE_FFI_LLAMA=1 does not block the device-bridge path", () => {
+		expect(
+			shouldEnableMobileLocalInference(
+				{
+					ELIZA_DISABLE_FFI_LLAMA: "1",
+					ELIZA_DEVICE_BRIDGE_ENABLED: "1",
+				},
+				"riscv64",
+			),
+		).toBe(true);
+	});
+
+	it("ELIZA_DISABLE_FFI_LLAMA=1 suppresses ELIZA_LOCAL_LLAMA=1 too", () => {
+		// `ELIZA_LOCAL_LLAMA` is the AOSP/FFI in-process trigger, so the
+		// disable flag must override it. Otherwise a riscv64 operator who set
+		// disable would still be forced into the FFI path on an AOSP build.
+		expect(
+			shouldEnableMobileLocalInference(
+				{
+					ELIZA_DISABLE_FFI_LLAMA: "1",
+					ELIZA_LOCAL_LLAMA: "1",
+				},
+				"riscv64",
+			),
+		).toBe(false);
+	});
+});

package/src/runtime/mobile-local-inference-gate.ts

@@ -0,0 +1,44 @@
+/**
+ * On a mobile platform (`ELIZA_PLATFORM=android` / `ios`) the runtime skips
+ * nearly every boot helper because they shell out to subprocesses,
+ * platform-specific binaries, or optional packages that aren't in the mobile
+ * bundle. Three mobile-safe inference paths need wiring:
+ *
+ *   - `ELIZA_DEVICE_BRIDGE_ENABLED=1`: the agent (this process) hosts the
+ *     device-bridge WSS and dials whichever paired device connects. On the
+ *     Capacitor APK the WebView's `@elizaos/capacitor-llama` is the intended
+ *     dialer over loopback. The Capacitor build always exports this env so
+ *     the bridge is ready as soon as first-run picks the local mode.
+ *
+ *   - `ELIZA_LOCAL_LLAMA=1`: AOSP path that loads `libllama.so` directly
+ *     inside the Android process via `bun:ffi`. Wired here so the gate is
+ *     in place ahead of sub-task 2 — the AOSP build flag flips this on.
+ *
+ *   - `process.arch === "riscv64"`: `capacitor-llama` has no riscv64 prebuild
+ *     and we can't NAPI-build it on-device, so the in-process FFI path
+ *     (same loader contract as the AOSP path) is the only viable option.
+ *     Auto-firing here keeps the riscv64 mobile boot path zero-config; an
+ *     operator can hard-disable via `ELIZA_DISABLE_FFI_LLAMA=1` to skip the
+ *     loader and route inference through Cloud instead. See
+ *     `plugin-aosp-local-inference/src/aosp-llama-adapter.ts:isAospEnabled`
+ *     and `plugin-local-inference/src/runtime/ensure-local-inference-handler.ts:shouldAttemptAospLlamaLoader`
+ *     — the three predicates agree on the trigger set.
+ *
+ * Kept dependency-free so it can be unit-tested without instantiating the
+ * full runtime.
+ */
+export function shouldEnableMobileLocalInference(
+	env: NodeJS.ProcessEnv = process.env,
+	arch: NodeJS.Architecture = process.arch,
+): boolean {
+	if (env.ELIZA_DISABLE_FFI_LLAMA?.trim() === "1") {
+		// Operator opted out of the FFI path entirely — the device-bridge
+		// path remains valid because the bridge is process-external and
+		// doesn't depend on `libllama.so` being present in the APK.
+		return env.ELIZA_DEVICE_BRIDGE_ENABLED?.trim() === "1";
+	}
+	const deviceBridge = env.ELIZA_DEVICE_BRIDGE_ENABLED?.trim() === "1";
+	const localLlama = env.ELIZA_LOCAL_LLAMA?.trim() === "1";
+	const riscv64Auto = arch === "riscv64";
+	return deviceBridge || localLlama || riscv64Auto;
+}

package/src/runtime/voice-entity-binding.d.ts

@@ -0,0 +1,103 @@
+/**
+ * Voice ⇄ entity binding seam (producer + round-trip consumer).
+ *
+ * Producer (`emitVoiceTurnObserved`): emit `VOICE_TURN_OBSERVED` so a
+ * merge-engine owner (plugin-lifeops) can fold the recognized voice turn
+ * into the entity/relationship graph. The voice-profile store is owned
+ * here; the entity graph is owned there; the only shared surface is the
+ * core event seam — neither plugin imports the other.
+ *
+ * Consumer (`handleVoiceEntityBound`): when the merge engine reports a
+ * binding via `VOICE_ENTITY_BOUND`, persist the resulting `entityId` onto
+ * every profile in that imprint cluster (`VoiceProfileStore.bindEntity`).
+ * This is the runtime path that was missing in issue #8234 — without it a
+ * profile's `entityId` stayed `null` and recognized speakers never reached
+ * the relationship graph.
+ */
+import { type IAgentRuntime, type VoiceEntityBoundPayload } from "@elizaos/core";
+import type { VoiceTurnSignal } from "../services/voice/eot-classifier.js";
+import { VoiceProfileStore } from "../services/voice/profile-store.js";
+import type { VoiceAttributionOutput } from "../services/voice/speaker/attribution-pipeline.js";
+export declare function setVoiceEntityBindingStore(store: VoiceProfileStore | null): void;
+export declare function getVoiceProfileStore(): Promise<VoiceProfileStore>;
+export interface EmitVoiceTurnObservedArgs {
+    /** Stable utterance id; a random one is minted when omitted. */
+    turnId?: string;
+    /** Recognized text (drives name/partner-claim extraction downstream). */
+    text: string;
+    /** Imprint cluster id from the voice-profile store. */
+    imprintClusterId: string;
+    /** Confidence of the imprint match (0..1). */
+    matchConfidence: number;
+    /** Entity the imprint already resolved to, or `null`/omitted when unbound. */
+    matchedEntityId?: string | null;
+    /** True when the OWNER spoke this turn. */
+    isOwner?: boolean;
+    /** ISO timestamp; defaults to now. */
+    observedAt?: string;
+}
+/**
+ * Emit `VOICE_TURN_OBSERVED`. No-op in effect when no merge-engine plugin
+ * is loaded (the event simply has no handler). `emitEvent` awaits every
+ * handler, so by the time this resolves the binding round-trip (including
+ * `VOICE_ENTITY_BOUND` → profile persist) has completed.
+ */
+export declare function emitVoiceTurnObserved(runtime: IAgentRuntime, args: EmitVoiceTurnObservedArgs): Promise<void>;
+export interface HandleLiveVoiceAttributionOptions {
+    /**
+     * Entity id the agent treats as the device owner / primary enrolled
+     * speaker. A turn attributed to this entity is always allowed to speak.
+     */
+    ownerEntityId?: string | null;
+    /**
+     * Entity ids the agent answers to without a wake word (owner + enrolled
+     * household members). A confident bystander is anyone attributed to an
+     * entity NOT in this set.
+     */
+    knownSpeakerEntityIds?: readonly string[];
+    /**
+     * The EOT-based turn signal the turn-controller already computed for this
+     * turn (from `eot-classifier` / `turn-controller`). The speaker decision is
+     * folded into it. When omitted, a neutral base is synthesized from
+     * `endOfTurnProbability` (default 0.5 — "unknown", fail open).
+     */
+    baseSignal?: VoiceTurnSignal;
+    /** P(turn complete) when no `baseSignal` is supplied (default 0.5). */
+    endOfTurnProbability?: number;
+    /** True when a wake word fired within the recent listen window. */
+    wakeWordActive?: boolean;
+    /**
+     * The ASR transcript for this turn, joined from the streaming-ASR path. When
+     * provided it rides on `VOICE_TURN_OBSERVED` (and the turn signal) so the
+     * merge engine's name/partner extraction (`VoiceObserver.ingestTurn`) runs
+     * from LIVE audio — previously this was hardcoded `""`, so live recognition
+     * could identify *who* spoke but never *what* they said (#8786). Diarization-
+     * only callers (audio-frame path) leave it unset; the in-process voice engine
+     * (which has both ASR + diarization) passes the real transcript.
+     */
+    transcript?: string;
+}
+/**
+ * Handle a live per-turn attribution result. This is the single automatic seam
+ * the engine bridge calls from its `onAttribution` path: any caller that wires a
+ * `profileStore` gets diarization-driven gating for free.
+ *
+ * 1. Emits `VOICE_TURN_OBSERVED` when the turn produced a profile observation
+ *    (so the merge engine can fold the recognized speaker into the entity
+ *    graph and round-trip the binding back onto the profile).
+ * 2. Composes the EOT-based turn signal with the speaker decision and stamps it
+ *    onto `output.turn.metadata.voiceTurnSignal`, which the chat-view producer
+ *    forwards to the server gate verbatim.
+ *
+ * Returns the composed signal (also written onto the turn metadata in place).
+ * Never throws on the emit path — observation emission is best-effort and is
+ * logged, never propagated, so an attribution turn never crashes a voice turn.
+ */
+export declare function handleLiveVoiceAttribution(runtime: IAgentRuntime, output: VoiceAttributionOutput, opts?: HandleLiveVoiceAttributionOptions): Promise<VoiceTurnSignal>;
+/**
+ * Handler for `VOICE_ENTITY_BOUND`. Persists `entityId` onto every profile
+ * in the cluster that is not already bound to it. Returns nothing (the
+ * `EventHandler` contract); the bound count is logged.
+ */
+export declare function handleVoiceEntityBound(payload: VoiceEntityBoundPayload): Promise<void>;
+//# sourceMappingURL=voice-entity-binding.d.ts.map

package/src/runtime/voice-entity-binding.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"voice-entity-binding.d.ts","sourceRoot":"","sources":["voice-entity-binding.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;GAeG;AAIH,OAAO,EAEN,KAAK,aAAa,EAGlB,KAAK,uBAAuB,EAC5B,MAAM,eAAe,CAAC;AACvB,OAAO,KAAK,EAEX,eAAe,EACf,MAAM,qCAAqC,CAAC;AAC7C,OAAO,EAAE,iBAAiB,EAAE,MAAM,oCAAoC,CAAC;AACvE,OAAO,KAAK,EAAE,sBAAsB,EAAE,MAAM,mDAAmD,CAAC;AAQhG,wBAAgB,0BAA0B,CACzC,KAAK,EAAE,iBAAiB,GAAG,IAAI,GAC7B,IAAI,CAEN;AAED,wBAAsB,oBAAoB,IAAI,OAAO,CAAC,iBAAiB,CAAC,CAOvE;AAMD,MAAM,WAAW,yBAAyB;IACzC,gEAAgE;IAChE,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,yEAAyE;IACzE,IAAI,EAAE,MAAM,CAAC;IACb,uDAAuD;IACvD,gBAAgB,EAAE,MAAM,CAAC;IACzB,8CAA8C;IAC9C,eAAe,EAAE,MAAM,CAAC;IACxB,8EAA8E;IAC9E,eAAe,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAChC,2CAA2C;IAC3C,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,sCAAsC;IACtC,UAAU,CAAC,EAAE,MAAM,CAAC;CACpB;AAED;;;;;GAKG;AACH,wBAAsB,qBAAqB,CAC1C,OAAO,EAAE,aAAa,EACtB,IAAI,EAAE,yBAAyB,GAC7B,OAAO,CAAC,IAAI,CAAC,CAWf;AAWD,MAAM,WAAW,iCAAiC;IACjD;;;OAGG;IACH,aAAa,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC9B;;;;OAIG;IACH,qBAAqB,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;IAC1C;;;;;OAKG;IACH,UAAU,CAAC,EAAE,eAAe,CAAC;IAC7B,uEAAuE;IACvE,oBAAoB,CAAC,EAAE,MAAM,CAAC;IAC9B,mEAAmE;IACnE,cAAc,CAAC,EAAE,OAAO,CAAC;IACzB;;;;;;;;OAQG;IACH,UAAU,CAAC,EAAE,MAAM,CAAC;CACpB;AAqFD;;;;;;;;;;;;;;;GAeG;AACH,wBAAsB,0BAA0B,CAC/C,OAAO,EAAE,aAAa,EACtB,MAAM,EAAE,sBAAsB,EAC9B,IAAI,GAAE,iCAAsC,GAC1C,OAAO,CAAC,eAAe,CAAC,CA2C1B;AAMD;;;;GAIG;AACH,wBAAsB,sBAAsB,CAC3C,OAAO,EAAE,uBAAuB,GAC9B,OAAO,CAAC,IAAI,CAAC,CA2Bf"}

package/src/runtime/voice-entity-binding.transcript.test.ts

@@ -0,0 +1,69 @@
+import type { IAgentRuntime } from "@elizaos/core";
+import { EventType } from "@elizaos/core";
+import { describe, expect, it } from "vitest";
+import type { VoiceAttributionOutput } from "../services/voice/speaker/attribution-pipeline.js";
+import { handleLiveVoiceAttribution } from "./voice-entity-binding.js";
+
+/** Minimal attribution output with an observation (drives the emit path). */
+function output(): VoiceAttributionOutput {
+	return {
+		turnId: "t1",
+		primarySpeaker: { entityId: "entity-jill", confidence: 0.6 },
+		observation: {
+			imprintClusterId: "cluster-1",
+			confidence: 0.6,
+			entityId: "entity-jill",
+		},
+		turn: { metadata: {} },
+		segments: [],
+	} as unknown as VoiceAttributionOutput;
+}
+
+function captureRuntime(): {
+	runtime: IAgentRuntime;
+	events: Array<{ type: unknown; payload: Record<string, unknown> }>;
+} {
+	const events: Array<{ type: unknown; payload: Record<string, unknown> }> = [];
+	const runtime = {
+		emitEvent: async (type: unknown, payload: Record<string, unknown>) => {
+			events.push({ type, payload });
+		},
+	} as unknown as IAgentRuntime;
+	return { runtime, events };
+}
+
+describe("handleLiveVoiceAttribution — transcript carry (#8786)", () => {
+	it("rides the real transcript on VOICE_TURN_OBSERVED when provided", async () => {
+		const { runtime, events } = captureRuntime();
+		await handleLiveVoiceAttribution(runtime, output(), {
+			ownerEntityId: "entity-jill",
+			transcript: "I'm Jill",
+		});
+		const observed = events.find(
+			(e) => e.type === EventType.VOICE_TURN_OBSERVED,
+		);
+		expect(observed).toBeDefined();
+		// Was hardcoded "" — the merge engine's name extraction needs the text.
+		expect(observed?.payload.text).toBe("I'm Jill");
+	});
+
+	it("falls back to '' for diarization-only callers (no transcript)", async () => {
+		const { runtime, events } = captureRuntime();
+		await handleLiveVoiceAttribution(runtime, output(), {
+			ownerEntityId: "entity-jill",
+		});
+		const observed = events.find(
+			(e) => e.type === EventType.VOICE_TURN_OBSERVED,
+		);
+		expect(observed?.payload.text).toBe("");
+	});
+
+	it("also stamps the transcript onto the synthesized turn signal", async () => {
+		const { runtime } = captureRuntime();
+		const signal = await handleLiveVoiceAttribution(runtime, output(), {
+			ownerEntityId: "entity-jill",
+			transcript: "Jill is my wife",
+		});
+		expect(signal.transcript).toBe("Jill is my wife");
+	});
+});