@elizaos/plugin-local-inference 2.0.3-beta.2 → 2.0.3-beta.4

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@elizaos/plugin-local-inference",
-  "version": "2.0.3-beta.2",
+  "version": "2.0.3-beta.4",
   "description": "Eliza-1 local provider plus HTTP routes for local model catalog, downloads, hardware detection, and chat commands.",
   "type": "module",
   "main": "dist/index.js",
@@ -56,6 +56,16 @@
       "import": "./dist/services/index.js",
       "default": "./dist/services/index.js"
     },
+    "./voice-workbench": {
+      "types": "./src/voice-workbench.ts",
+      "eliza-source": {
+        "types": "./src/voice-workbench.ts",
+        "import": "./src/voice-workbench.ts",
+        "default": "./src/voice-workbench.ts"
+      },
+      "import": "./dist/voice-workbench.js",
+      "default": "./dist/voice-workbench.js"
+    },
     "./*.css": "./dist/*.css",
     "./*": {
       "types": "./src/*.ts",
@@ -69,6 +79,7 @@
     }
   },
   "files": [
+    "registry-entry.json",
     "dist",
     "src"
   ],
@@ -77,7 +88,15 @@
     "dev": "bun run build.ts",
     "test": "NODE_OPTIONS='--experimental-sqlite' vitest run",
     "test:asr:real": "NODE_OPTIONS='--experimental-sqlite' bun scripts/asr-real-smoke.ts",
+    "test:kokoro:real": "bun scripts/kokoro-real-smoke.ts",
     "probe:sd-cpp": "node scripts/probe-sd-cpp.mjs --human",
+    "memory:benchmark": "bun run scripts/memory-benchmark.ts",
+    "voice:workbench": "bun run scripts/voice-workbench.ts",
+    "corpus:generate": "bun run scripts/generate-voice-corpus.ts",
+    "roundtrip:real": "NODE_OPTIONS='--experimental-sqlite' bun scripts/mixed-real-roundtrip.ts",
+    "robustness:real": "NODE_OPTIONS='--experimental-sqlite' bun scripts/real-robustness-asr.ts",
+    "voicestack:real": "NODE_OPTIONS='--experimental-sqlite' bun scripts/real-voice-stack.ts",
+    "agentvoice:real": "NODE_OPTIONS='--experimental-sqlite' bun scripts/real-agent-voice-overlap.ts",
     "clean": "rm -rf dist .turbo node_modules",
     "lint": "bunx @biomejs/biome check --write --unsafe .",
     "lint:check": "bunx @biomejs/biome check .",
@@ -86,17 +105,17 @@
     "typecheck": "tsgo --noEmit"
   },
   "dependencies": {
-    "@elizaos/core": "2.0.3-beta.2",
-    "@elizaos/plugin-aosp-local-inference": "2.0.3-beta.2",
-    "@elizaos/plugin-capacitor-bridge": "2.0.3-beta.2",
-    "@elizaos/plugin-computeruse": "2.0.3-beta.2",
-    "@elizaos/shared": "2.0.3-beta.2",
+    "@elizaos/core": "2.0.3-beta.4",
+    "@elizaos/plugin-aosp-local-inference": "2.0.3-beta.4",
+    "@elizaos/plugin-capacitor-bridge": "2.0.3-beta.4",
+    "@elizaos/plugin-computeruse": "2.0.3-beta.4",
+    "@elizaos/shared": "2.0.3-beta.4",
     "phonemizer": "^1.2.1",
     "ws": "^8.20.1"
   },
   "peerDependencies": {
-    "@elizaos/core": "2.0.3-beta.2",
-    "@elizaos/shared": "2.0.3-beta.2"
+    "@elizaos/core": "2.0.3-beta.4",
+    "@elizaos/shared": "2.0.3-beta.4"
   },
   "devDependencies": {
     "@biomejs/biome": "^2.4.14",
@@ -108,5 +127,5 @@
   "publishConfig": {
     "access": "public"
   },
-  "gitHead": "82fe0f44215954c2417328203f5bd6510985c1fc"
+  "gitHead": "f76f55793a0fb8d6b869dd8ddce03970de061e34"
 }

package/registry-entry.json

@@ -0,0 +1,137 @@
+{
+	"id": "local-inference",
+	"name": "Local Inference",
+	"description": "Unified Eliza-1 local inference provider — text generation, embeddings, voice (TTS/ASR), and image description via the elizaOS llama.cpp fork. Replaces the deprecated plugin-local-ai / plugin-local-embedding pair.",
+	"npmName": "@elizaos/plugin-local-inference",
+	"version": "2.0.0-beta.2",
+	"source": "bundled",
+	"tags": [
+		"ai-provider",
+		"llm",
+		"local-models",
+		"self-hosted",
+		"local",
+		"embedding",
+		"voice",
+		"tts",
+		"transcription",
+		"eliza-1"
+	],
+	"config": {
+		"MODELS_DIR": {
+			"type": "file-path",
+			"required": false,
+			"sensitive": false,
+			"label": "Models Dir",
+			"help": "Filesystem path to the directory where GGUF models are stored. Defaults to <stateDir>/models.",
+			"advanced": false
+		},
+		"CACHE_DIR": {
+			"type": "file-path",
+			"required": false,
+			"sensitive": false,
+			"label": "Cache Dir",
+			"help": "Filesystem path to the cache directory for model assets (tokenizers, ONNX caches).",
+			"advanced": false
+		},
+		"LOCAL_SMALL_MODEL": {
+			"type": "string",
+			"required": false,
+			"sensitive": false,
+			"default": "text/eliza-1-2b-128k.gguf",
+			"label": "Small Model",
+			"help": "Filename of the small local model (TEXT_SMALL handler).",
+			"advanced": false
+		},
+		"LOCAL_LARGE_MODEL": {
+			"type": "string",
+			"required": false,
+			"sensitive": false,
+			"default": "text/eliza-1-4b-128k.gguf",
+			"label": "Large Model",
+			"help": "Filename of the large local model (TEXT_LARGE handler).",
+			"advanced": false
+		},
+		"LOCAL_EMBEDDING_MODEL": {
+			"type": "string",
+			"required": false,
+			"sensitive": false,
+			"default": "text/eliza-1-2b-128k.gguf",
+			"label": "Embedding Model",
+			"help": "Filename of the embedding model used for vector embeddings.",
+			"advanced": false
+		},
+		"LOCAL_EMBEDDING_DIMENSIONS": {
+			"type": "number",
+			"required": false,
+			"sensitive": false,
+			"default": "1024",
+			"label": "Embedding Dimensions",
+			"help": "Number of dimensions the embedding model outputs.",
+			"advanced": false
+		},
+		"CUDA_VISIBLE_DEVICES": {
+			"type": "string",
+			"required": false,
+			"sensitive": false,
+			"label": "CUDA Visible Devices",
+			"help": "Set to restrict which CUDA-enabled GPUs the runtime sees (e.g. `0,1`).",
+			"advanced": true
+		},
+		"ELIZA_LOCAL_LLAMA": {
+			"type": "string",
+			"required": false,
+			"sensitive": false,
+			"label": "AOSP FFI Loader",
+			"help": "Set to `1` on Android (elizaOS build) to enable the in-process bun:ffi loader (libllama.so + libeliza-llama-shim.so) via the @elizaos/plugin-aosp-local-inference companion plugin.",
+			"advanced": true
+		},
+		"ELIZA_MODEL": {
+			"type": "select",
+			"required": false,
+			"sensitive": false,
+			"label": "Eliza-1 Tier",
+			"help": "Select a published Eliza-1 fine-tune tier. The runtime auto-picks the right quant flavor (gguf/polarquant/fp8/bf16) for your detected GPU and pulls it from HuggingFace on first use.",
+			"options": [
+				{
+					"value": "eliza-1-2b",
+					"label": "Eliza-1 2B (mobile / entry)",
+					"description": "Smallest/entry tier; fits Android/iOS on-device inference and 16 GB consumer GPUs at Q4_K_M."
+				},
+				{
+					"value": "eliza-1-4b",
+					"label": "Eliza-1 4B (16-24 GB GPU)",
+					"description": "Desktop-tier fine-tune for everyday workstation use."
+				},
+				{
+					"value": "eliza-1-9b",
+					"label": "Eliza-1 9B (24-48 GB GPU)",
+					"description": "Workstation-tier fine-tune; Q4_K_M for 16 GB cards, PolarQuant for 24+."
+				},
+				{
+					"value": "eliza-1-27b",
+					"label": "Eliza-1 27B (cloud / 48 GB+)",
+					"description": "Cloud-tier fine-tune; Q6_K GGUF on RTX 5090, fp8/bf16 on H200/B200."
+				}
+			],
+			"advanced": false
+		}
+	},
+	"render": {
+		"visible": true,
+		"pinTo": [],
+		"style": "card",
+		"icon": "Monitor",
+		"group": "ai-provider",
+		"groupOrder": 0,
+		"actions": ["enable", "configure"]
+	},
+	"resources": {
+		"homepage": "https://github.com/elizaOS/eliza/tree/main/plugins/plugin-local-inference#readme",
+		"repository": "https://github.com/elizaOS/eliza",
+		"setupGuideUrl": "https://docs.eliza.ai/plugin-setup-guide#local-inference"
+	},
+	"dependsOn": [],
+	"kind": "plugin",
+	"subtype": "ai-provider"
+}

package/src/adapters/capacitor-llama/__tests__/voice-turn.test.ts

@@ -0,0 +1,293 @@
+/**
+ * Wiring tests for the on-device fused voice-turn entry (#8786). Proves the
+ * device bridge is the real production caller for
+ * `LocalInferenceEngine.runVoiceTurn`: it adapts the loaded
+ * `CapacitorLlamaContext` into the `MtpTextRunner` the fused pipeline drives,
+ * and forwards the turn to the engine. No native model — fakes stand in for the
+ * context + engine so the wiring itself is the unit under test.
+ */
+
+import { describe, expect, it, vi } from "vitest";
+import type { TranscriptionAudio } from "../../../services/voice/types";
+import { NativePcmVoiceTurnCoordinator } from "../native-voice-capture";
+import type { CapacitorLlamaContext } from "../types";
+import {
+	createCapacitorMtpTextRunner,
+	type DeviceVoiceEngine,
+	runDeviceVoiceTurn,
+	type VoiceTurnExitReason,
+} from "../voice-turn";
+
+function fakeContext(
+	overrides: Partial<CapacitorLlamaContext> = {},
+): CapacitorLlamaContext {
+	return {
+		id: 1,
+		gpu: false,
+		reasonNoGPU: "",
+		model: {} as CapacitorLlamaContext["model"],
+		completion: vi.fn(async () => ({
+			text: "four",
+		})) as unknown as CapacitorLlamaContext["completion"],
+		stopCompletion: vi.fn(async () => {}),
+		tokenize: vi.fn(),
+		detokenize: vi.fn(),
+		embedding: vi.fn(),
+		bench: vi.fn(),
+		release: vi.fn(),
+		...overrides,
+	} as CapacitorLlamaContext;
+}
+
+const AUDIO: TranscriptionAudio = {
+	pcm: new Float32Array([0.1, 0.2, 0.3]),
+	sampleRate: 16_000,
+};
+
+describe("createCapacitorMtpTextRunner", () => {
+	it("reports no drafter (on-device plain decode through the verifier)", () => {
+		const runner = createCapacitorMtpTextRunner(fakeContext());
+		expect(runner.hasDrafter()).toBe(false);
+	});
+
+	it("delegates generateWithVerifierEvents to context.completion and returns its text", async () => {
+		const ctx = fakeContext();
+		const runner = createCapacitorMtpTextRunner(ctx, { temperature: 0.5 });
+		const result = await runner.generateWithVerifierEvents({
+			prompt: "what is 2 + 2?",
+			maxTokens: 8,
+			topP: 0.9,
+			stopSequences: ["</s>"],
+			onVerifierEvent: () => {},
+		});
+		expect(result.text).toBe("four");
+		expect(ctx.completion).toHaveBeenCalledTimes(1);
+		expect(ctx.completion).toHaveBeenCalledWith(
+			expect.objectContaining({
+				prompt: "what is 2 + 2?",
+				n_predict: 8,
+				top_p: 0.9,
+				temperature: 0.5,
+				stop: ["</s>"],
+			}),
+		);
+	});
+
+	it("returns empty text without decoding when the signal is already aborted", async () => {
+		const ctx = fakeContext();
+		const runner = createCapacitorMtpTextRunner(ctx);
+		const controller = new AbortController();
+		controller.abort();
+		const result = await runner.generateWithVerifierEvents({
+			prompt: "hello",
+			signal: controller.signal,
+			onVerifierEvent: () => {},
+		});
+		expect(result.text).toBe("");
+		expect(ctx.completion).not.toHaveBeenCalled();
+	});
+
+	it("aborting mid-decode stops the on-device completion (barge-in)", async () => {
+		let resolveCompletion: ((v: { text: string }) => void) | null = null;
+		const ctx = fakeContext({
+			completion: vi.fn(
+				() =>
+					new Promise((resolve) => {
+						resolveCompletion = resolve as (v: { text: string }) => void;
+					}),
+			) as unknown as CapacitorLlamaContext["completion"],
+		});
+		const runner = createCapacitorMtpTextRunner(ctx);
+		const controller = new AbortController();
+		const pending = runner.generateWithVerifierEvents({
+			prompt: "a long answer",
+			signal: controller.signal,
+			onVerifierEvent: () => {},
+		});
+		controller.abort(); // barge-in
+		expect(ctx.stopCompletion).toHaveBeenCalledTimes(1);
+		resolveCompletion?.({ text: "partial" });
+		await expect(pending).resolves.toEqual({ text: "partial" });
+	});
+});
+
+describe("runDeviceVoiceTurn", () => {
+	it("builds the device text runner and drives engine.runVoiceTurn", async () => {
+		const ctx = fakeContext();
+		const events = { onComplete: vi.fn() };
+		const engine: DeviceVoiceEngine = {
+			runVoiceTurn: vi.fn(async (): Promise<VoiceTurnExitReason> => "done"),
+		};
+
+		const exit = await runDeviceVoiceTurn({
+			engine,
+			context: ctx,
+			audio: AUDIO,
+			events,
+			maxGeneratedTokens: 256,
+		});
+
+		expect(exit).toBe("done");
+		expect(engine.runVoiceTurn).toHaveBeenCalledTimes(1);
+		const [audioArg, optsArg] = (
+			engine.runVoiceTurn as unknown as { mock: { calls: unknown[][] } }
+		).mock.calls[0] as [TranscriptionAudio, Record<string, unknown>];
+		expect(audioArg).toBe(AUDIO);
+		expect(optsArg.events).toBe(events);
+		expect(optsArg.maxGeneratedTokens).toBe(256);
+		// The text runner handed to the engine is the device-context-backed one.
+		const runner = optsArg.textRunner as ReturnType<
+			typeof createCapacitorMtpTextRunner
+		>;
+		expect(runner.hasDrafter()).toBe(false);
+		await runner.generateWithVerifierEvents({
+			prompt: "x",
+			onVerifierEvent: () => {},
+		});
+		expect(ctx.completion).toHaveBeenCalledTimes(1);
+	});
+
+	it("fires the next-stage preload predictor from the pipeline's onAsrComplete (#8809 C5)", async () => {
+		const ctx = fakeContext();
+		const callerOnAsrComplete = vi.fn();
+		const onAsrStageComplete = vi.fn(async () => "eliza-1-4b" as string | null);
+		// Structural stand-in for VoicePreloadPredictor — only onAsrStageComplete
+		// is consumed by the wiring; type-cast keeps the test free of the arbiter.
+		const predictor = { onAsrStageComplete } as unknown as Parameters<
+			typeof runDeviceVoiceTurn
+		>[0]["preloadPredictor"];
+
+		const engine: DeviceVoiceEngine = {
+			runVoiceTurn: vi.fn(async (): Promise<VoiceTurnExitReason> => "done"),
+		};
+
+		await runDeviceVoiceTurn({
+			engine,
+			context: ctx,
+			audio: AUDIO,
+			events: { onAsrComplete: callerOnAsrComplete },
+			preloadPredictor: predictor,
+		});
+
+		const [, optsArg] = (
+			engine.runVoiceTurn as unknown as { mock: { calls: unknown[][] } }
+		).mock.calls[0] as [TranscriptionAudio, Record<string, unknown>];
+		const wiredEvents = optsArg.events as {
+			onAsrComplete?: (tokens: ReadonlyArray<unknown>) => void;
+		};
+
+		// The engine drives onAsrComplete the instant ASR finishes; the composed
+		// hook must fire BOTH the caller's hook and the predictor's prediction.
+		expect(onAsrStageComplete).not.toHaveBeenCalled();
+		wiredEvents.onAsrComplete?.([]);
+		expect(callerOnAsrComplete).toHaveBeenCalledTimes(1);
+		expect(onAsrStageComplete).toHaveBeenCalledTimes(1);
+	});
+
+	it("leaves events untouched when no predictor is supplied", async () => {
+		const events = { onComplete: vi.fn() };
+		const engine: DeviceVoiceEngine = {
+			runVoiceTurn: vi.fn(async (): Promise<VoiceTurnExitReason> => "done"),
+		};
+		await runDeviceVoiceTurn({
+			engine,
+			context: fakeContext(),
+			audio: AUDIO,
+			events,
+		});
+		const [, optsArg] = (
+			engine.runVoiceTurn as unknown as { mock: { calls: unknown[][] } }
+		).mock.calls[0] as [TranscriptionAudio, Record<string, unknown>];
+		expect(optsArg.events).toBe(events);
+	});
+
+	it("propagates the engine's exit reason (token-cap / cancelled)", async () => {
+		const engine: DeviceVoiceEngine = {
+			runVoiceTurn: vi.fn(
+				async (): Promise<VoiceTurnExitReason> => "cancelled",
+			),
+		};
+		const exit = await runDeviceVoiceTurn({
+			engine,
+			context: fakeContext(),
+			audio: AUDIO,
+		});
+		expect(exit).toBe("cancelled");
+	});
+});
+
+describe("NativePcmVoiceTurnCoordinator", () => {
+	it("rejects native turns before the capture lifecycle is started", async () => {
+		const coordinator = new NativePcmVoiceTurnCoordinator({
+			engine: {
+				runVoiceTurn: vi.fn(async (): Promise<VoiceTurnExitReason> => "done"),
+			},
+			context: fakeContext(),
+		});
+
+		await expect(coordinator.acceptTurn({ audio: AUDIO })).rejects.toThrow(
+			/start\(\)/,
+		);
+	});
+
+	it("serializes completed native PCM turns through runDeviceVoiceTurn", async () => {
+		const ctx = fakeContext();
+		let releaseFirst: ((reason: VoiceTurnExitReason) => void) | null = null;
+		const engine: DeviceVoiceEngine = {
+			runVoiceTurn: vi.fn(
+				() =>
+					new Promise<VoiceTurnExitReason>((resolve) => {
+						if (!releaseFirst) {
+							releaseFirst = resolve;
+							return;
+						}
+						resolve("token-cap");
+					}),
+			),
+		};
+		const coordinator = new NativePcmVoiceTurnCoordinator({
+			engine,
+			context: ctx,
+			maxGeneratedTokens: 128,
+		});
+
+		coordinator.start();
+		const first = coordinator.acceptTurn({ turnId: "turn-a", audio: AUDIO });
+		const secondAudio: TranscriptionAudio = {
+			pcm: new Float32Array([0.4, 0.5]),
+			sampleRate: 16_000,
+		};
+		const second = coordinator.acceptTurn({
+			turnId: "turn-b",
+			audio: secondAudio,
+			maxGeneratedTokens: 64,
+		});
+
+		await Promise.resolve();
+		expect(engine.runVoiceTurn).toHaveBeenCalledTimes(1);
+		releaseFirst?.("done");
+		await expect(first).resolves.toEqual({
+			turnId: "turn-a",
+			exitReason: "done",
+		});
+		await expect(second).resolves.toEqual({
+			turnId: "turn-b",
+			exitReason: "token-cap",
+		});
+		expect(engine.runVoiceTurn).toHaveBeenCalledTimes(2);
+
+		const firstCall = (
+			engine.runVoiceTurn as unknown as { mock: { calls: unknown[][] } }
+		).mock.calls[0] as [TranscriptionAudio, Record<string, unknown>];
+		const secondCall = (
+			engine.runVoiceTurn as unknown as { mock: { calls: unknown[][] } }
+		).mock.calls[1] as [TranscriptionAudio, Record<string, unknown>];
+		expect(firstCall[0]).toBe(AUDIO);
+		expect(firstCall[1].maxGeneratedTokens).toBe(128);
+		expect(secondCall[0]).toBe(secondAudio);
+		expect(secondCall[1].maxGeneratedTokens).toBe(64);
+
+		await coordinator.stop();
+		expect(coordinator.isRunning).toBe(false);
+	});
+});

package/src/adapters/capacitor-llama/environment.ts

@@ -2,7 +2,7 @@ import { logger } from "@elizaos/core";
 import { z } from "zod";
 
 const DEFAULT_SMALL_MODEL = "text/eliza-1-2b-128k.gguf";
-const DEFAULT_LARGE_MODEL = "text/eliza-1-4b-128k.gguf";
+const DEFAULT_LARGE_MODEL = "text/eliza-1-2b-128k.gguf";
 const DEFAULT_EMBEDDING_MODEL = "gte-small_fp16.gguf";
 
 export const configSchema = z.object({

package/src/adapters/capacitor-llama/index.ts

@@ -60,6 +60,7 @@ import {
 } from "./types";
 
 const DEFAULT_LOCAL_SYSTEM_PROMPT = "Respond to the current request only.";
+const OMIT_MAX_TOKENS_LOCAL_BUDGET = 64_000;
 
 interface ContextEntry {
 	ctx: CapacitorLlamaContext;
@@ -293,12 +294,12 @@ async function tryLocalInferenceModel<T>(
 function buildNativeResult(
 	result: LocalGenerationResult,
 ): LocalNativeTextModelResult {
-	const nativeResult = {
+	const nativeResult = Object.assign(new String(result.text), {
 		text: result.text,
 		toolCalls: result.toolCalls,
 		...(result.finishReason ? { finishReason: result.finishReason } : {}),
-	};
-	return nativeResult as unknown as LocalNativeTextModelResult;
+	});
+	return nativeResult as LocalNativeTextModelResult;
 }
 
 function getLocalModelLabel(
@@ -530,7 +531,9 @@ class LocalAIManager {
 
 		const baseParams: CapacitorLlamaCompletionParams = {
 			prompt: renderCompletionPrompt({ ...params, system: systemPrompt }),
-			n_predict: params.maxTokens ?? 8192,
+			n_predict: params.omitMaxTokens
+				? (params.maxTokens ?? OMIT_MAX_TOKENS_LOCAL_BUDGET)
+				: (params.maxTokens ?? 8192),
 			temperature: params.temperature ?? 0.7,
 			top_p: params.topP ?? 0.9,
 			...(typeof params.topK === "number" ? { top_k: params.topK } : {}),
@@ -805,3 +808,24 @@ export const localAiPlugin: Plugin = {
 };
 
 export default localAiPlugin;
+
+// On-device fused voice-turn entry (#8786): native iOS/Android mic-capture
+// front-ends hand completed PCM turns to `NativePcmVoiceTurnCoordinator`, which
+// serializes them through `runDeviceVoiceTurn` → `LocalInferenceEngine.runVoiceTurn`
+// (ASR + MTP text + speaker-attribution + TTS in one pass). On memory-constrained
+// devices the optional next-stage preload predictor (#8809) warms the response
+// model as soon as ASR completes.
+export {
+	type NativePcmVoiceTurn,
+	NativePcmVoiceTurnCoordinator,
+	type NativePcmVoiceTurnCoordinatorOptions,
+	type NativePcmVoiceTurnResult,
+} from "./native-voice-capture";
+export {
+	type CapacitorTextRunnerOptions,
+	createCapacitorMtpTextRunner,
+	type DeviceVoiceEngine,
+	type RunDeviceVoiceTurnArgs,
+	runDeviceVoiceTurn,
+	type VoiceTurnExitReason,
+} from "./voice-turn";