@elizaos/plugin-local-inference 2.0.0-beta.1 → 2.0.3-beta.3

package/src/services/voice/wake-word-ggml.ts

@@ -0,0 +1,319 @@
+/**
+ * Wake-word detection — native runtime binding.
+ *
+ * Loads the standalone `packages/native/plugins/wakeword-cpp/`
+ * library directly via `bun:ffi` and exposes the `WakeWordModel`
+ * interface as `OpenWakeWordGgmlModel`. The sibling `./wake-word.ts`
+ * is the fused-`libelizainference` path (`GgmlWakeWordModel`, the
+ * `eliza_inference_wakeword_*` FFI surface) and selects between the two
+ * native backends; both implement the same interface so the voice
+ * lifecycle can swap them without changing anything upstream. There is
+ * NO ONNX path on either — `onnxruntime-node` was removed.
+ *
+ * This binding is the standalone-library backend, used when the
+ * `libwakeword.{so,dylib,dll}` shared library and three converted
+ * GGUFs are present. The C runtime is a pure-fp32 reference
+ * implementation of the openWakeWord three-stage pipeline (melspec
+ * → embedding CNN → classifier head) — no ggml link, no SIMD; a
+ * laptop CPU runs it under 1 % of real time. ABI parity with the
+ * upstream openWakeWord ONNX graphs is gated by
+ * `packages/native/plugins/wakeword-cpp/test/wakeword_parity_test.py`.
+ *
+ * Three GGUFs back one session, mirroring openWakeWord's three ONNX
+ * graphs (the C library is the single source of truth on shapes —
+ * see `packages/native/plugins/wakeword-cpp/include/wakeword/wakeword.h`):
+ *
+ *   1. melspec    — 16 kHz PCM → 32-bin log-mel frames.
+ *   2. embedding  — 20-Conv2D CNN over a 76-mel-frame sliding window
+ *                   → 96-dim embedding.
+ *   3. classifier — 4-layer MLP over a 16-embedding window → P(wake)
+ *                   ∈ [0, 1].
+ */
+
+import type { WakeWordModel } from "./types";
+
+/** PCM frame size the streaming pipeline expects (80 ms @ 16 kHz). */
+const FRAME_SAMPLES = 1280;
+const SAMPLE_RATE = 16_000;
+
+/** Three GGUF paths that back one session. */
+export interface WakeWordGgmlPaths {
+	/** Frozen Hann window + mel filter bank + STFT params metadata. */
+	melspec: string;
+	/** Embedding-CNN weights (fp16) + architecture metadata. */
+	embedding: string;
+	/** Classifier-head weights (fp16) + (1, 16, 96) input shape. */
+	classifier: string;
+}
+
+export interface WakeWordGgmlConfig {
+	/** Detection threshold ∈ [0, 1]. Default 0.5 (matches upstream openWakeWord). */
+	threshold?: number;
+}
+
+/**
+ * Thrown when the native ggml backend cannot be used. Distinct from
+ * `WakeWordUnavailableError` in `./wake-word.ts` so callers that fall
+ * back to the legacy ONNX path can tell the two failure modes apart
+ * during the migration.
+ */
+export class WakeWordGgmlUnavailableError extends Error {
+	readonly code:
+		| "not-bun"
+		| "library-load-failed"
+		| "model-load-failed"
+		| "abi-error";
+	constructor(code: WakeWordGgmlUnavailableError["code"], message: string) {
+		super(message);
+		this.name = "WakeWordGgmlUnavailableError";
+		this.code = code;
+	}
+}
+
+/** Runtime detector — `bun:ffi` is Bun-only. */
+function isBunRuntime(): boolean {
+	return typeof (globalThis as { Bun?: unknown }).Bun !== "undefined";
+}
+
+/**
+ * Native session handle. The C side gives us a pointer (`bigint` under
+ * `bun:ffi`); we never inspect it on the JS side beyond passing it
+ * back through the binding.
+ */
+type NativeHandle = bigint;
+
+/** Bound symbols from `libwakeword`. Mirrors `wakeword.h` 1:1. */
+interface WakeWordBindings {
+	wakeword_open: (
+		melspec: unknown,
+		embedding: unknown,
+		classifier: unknown,
+		outHandle: unknown,
+	) => number;
+	wakeword_close: (handle: NativeHandle) => number;
+	wakeword_process: (
+		handle: NativeHandle,
+		pcm: unknown,
+		nSamples: bigint | number,
+		outScore: unknown,
+	) => number;
+	wakeword_set_threshold: (handle: NativeHandle, threshold: number) => number;
+	wakeword_active_backend: () => unknown;
+}
+
+interface BoundLibrary {
+	bindings: WakeWordBindings;
+	close(): void;
+	libraryPath: string;
+	/** `bun:ffi` helper: encode an ArrayBufferView as a native pointer. */
+	ptr(value: ArrayBufferView): unknown;
+}
+
+/** Minimal shape of the bun:ffi module we use here. */
+interface BunFfiModule {
+	dlopen<TSymbols>(
+		path: string,
+		def: Record<string, { args: number[]; returns: number }>,
+	): {
+		symbols: TSymbols;
+		close(): void;
+	};
+	ptr(value: ArrayBufferView): unknown;
+	FFIType: {
+		cstring: number;
+		ptr: number;
+		i32: number;
+		u64: number;
+		f32: number;
+	};
+}
+
+/**
+ * Resolve `bun:ffi` at runtime via the Bun-injected `require`. Mirrors
+ * the loader pattern in `./ffi-bindings.ts` (search for
+ * `loadBunFfiModule`) so plain Node test runs that import this file
+ * for type-only purposes do not blow up at the import site.
+ */
+function loadBunFfiModule(): BunFfiModule {
+	const req: ((id: string) => unknown) | undefined = (
+		globalThis as { Bun?: { __require?: (id: string) => unknown } }
+	).Bun?.__require;
+	if (typeof req === "function") {
+		return req("bun:ffi") as BunFfiModule;
+	}
+	const mod = require("node:module") as {
+		createRequire: (filename: string) => (id: string) => unknown;
+	};
+	const r = mod.createRequire(import.meta.url);
+	return r("bun:ffi") as BunFfiModule;
+}
+
+/**
+ * Load `libwakeword` and bind every symbol declared in `wakeword.h`.
+ * Throws `WakeWordGgmlUnavailableError({code: "library-load-failed"})`
+ * on `dlopen` failure.
+ */
+function loadLibrary(libraryPath: string): BoundLibrary {
+	if (!isBunRuntime()) {
+		throw new WakeWordGgmlUnavailableError(
+			"not-bun",
+			"[wake-word-ggml] bun:ffi is required; current runtime is not Bun",
+		);
+	}
+	if (!libraryPath || libraryPath.length === 0) {
+		throw new WakeWordGgmlUnavailableError(
+			"library-load-failed",
+			"[wake-word-ggml] libraryPath is required",
+		);
+	}
+	const bunFfi = loadBunFfiModule();
+	const T = bunFfi.FFIType;
+	const lib = bunFfi.dlopen<WakeWordBindings>(libraryPath, {
+		wakeword_open: {
+			args: [T.ptr, T.ptr, T.ptr, T.ptr],
+			returns: T.i32,
+		},
+		wakeword_close: {
+			args: [T.u64],
+			returns: T.i32,
+		},
+		wakeword_process: {
+			args: [T.u64, T.ptr, T.u64, T.ptr],
+			returns: T.i32,
+		},
+		wakeword_set_threshold: {
+			args: [T.u64, T.f32],
+			returns: T.i32,
+		},
+		wakeword_active_backend: {
+			args: [],
+			returns: T.cstring,
+		},
+	});
+	return {
+		bindings: lib.symbols,
+		close: () => lib.close(),
+		libraryPath,
+		ptr: (v: ArrayBufferView) => bunFfi.ptr(v),
+	};
+}
+
+/**
+ * Streaming wake-word detector backed by `libwakeword`.
+ *
+ * Implements the same `WakeWordModel` interface as `OpenWakeWordModel`
+ * in `./wake-word.ts` so the voice lifecycle can swap implementations
+ * without changing anything upstream.
+ */
+export class OpenWakeWordGgmlModel implements WakeWordModel {
+	readonly frameSamples = FRAME_SAMPLES;
+	readonly sampleRate = SAMPLE_RATE;
+
+	private constructor(
+		private readonly lib: BoundLibrary,
+		private readonly handle: NativeHandle,
+	) {}
+
+	/**
+	 * Load a wake-word model from its three GGUFs and the
+	 * `libwakeword` shared library. Returns a ready-to-use detector
+	 * with a fresh streaming session. Throws
+	 * `WakeWordGgmlUnavailableError` on dlopen / ABI / GGUF failure.
+	 */
+	static async load(args: {
+		libraryPath: string;
+		paths: WakeWordGgmlPaths;
+		config?: WakeWordGgmlConfig;
+	}): Promise<OpenWakeWordGgmlModel> {
+		const lib = loadLibrary(args.libraryPath);
+		const out = new BigUint64Array(1);
+		/* Each path string needs a NUL terminator; we encode through
+		 * Uint8Array and pass an explicit pointer because bun:ffi does
+		 * not auto-convert Buffer -> cstring on every binding shape. */
+		const enc = new TextEncoder();
+		const melBuf = enc.encode(`${args.paths.melspec}\0`);
+		const embBuf = enc.encode(`${args.paths.embedding}\0`);
+		const clsBuf = enc.encode(`${args.paths.classifier}\0`);
+		const rc = lib.bindings.wakeword_open(
+			lib.ptr(melBuf),
+			lib.ptr(embBuf),
+			lib.ptr(clsBuf),
+			lib.ptr(out),
+		);
+		if (rc !== 0) {
+			lib.close();
+			throw new WakeWordGgmlUnavailableError(
+				"model-load-failed",
+				`[wake-word-ggml] wakeword_open(${args.paths.melspec}, ${args.paths.embedding}, ${args.paths.classifier}) returned ${rc}`,
+			);
+		}
+		const handle = out[0] as NativeHandle;
+		const model = new OpenWakeWordGgmlModel(lib, handle);
+		if (args.config?.threshold !== undefined) {
+			const setRc = lib.bindings.wakeword_set_threshold(
+				handle,
+				args.config.threshold,
+			);
+			if (setRc !== 0) {
+				model.close();
+				throw new WakeWordGgmlUnavailableError(
+					"abi-error",
+					`[wake-word-ggml] wakeword_set_threshold(${args.config.threshold}) returned ${setRc}`,
+				);
+			}
+		}
+		return model;
+	}
+
+	/**
+	 * Score one 1280-sample (80 ms @ 16 kHz) fp32 mono frame and
+	 * return the most recent classifier probability ∈ [0, 1]. Early
+	 * frames (before enough mel + embedding context has accumulated)
+	 * return 0.
+	 */
+	async scoreFrame(frame: Float32Array): Promise<number> {
+		if (frame.length !== FRAME_SAMPLES) {
+			throw new Error(
+				`[wake-word-ggml] scoreFrame expects ${FRAME_SAMPLES} samples; got ${frame.length}`,
+			);
+		}
+		const out = new Float32Array(1);
+		const rc = this.lib.bindings.wakeword_process(
+			this.handle,
+			this.lib.ptr(frame),
+			BigInt(frame.length),
+			this.lib.ptr(out),
+		);
+		if (rc !== 0) {
+			throw new WakeWordGgmlUnavailableError(
+				"abi-error",
+				`[wake-word-ggml] wakeword_process returned ${rc}`,
+			);
+		}
+		const p = out[0] ?? 0;
+		return Math.min(1, Math.max(0, p));
+	}
+
+	/**
+	 * Streaming state lives on the native side. The C ABI does not
+	 * expose a separate `reset` entry point yet (Phase 2 will add one
+	 * if the ggml-backed implementation needs it); for now, reset is
+	 * a no-op on the JS side. Callers that need a hard state clear
+	 * close + reopen the session.
+	 */
+	reset(): void {
+		// Intentionally empty — see jsdoc above.
+	}
+
+	/** Release the native session and the dlopen handle. */
+	close(): void {
+		this.lib.bindings.wakeword_close(this.handle);
+		this.lib.close();
+	}
+
+	/** Diagnostics: `"native-cpu"` on this build (pure-fp32 reference). */
+	activeBackend(): string {
+		const raw = this.lib.bindings.wakeword_active_backend();
+		return typeof raw === "string" ? raw : String(raw ?? "");
+	}
+}

package/src/services/voice/wake-word.test.ts

@@ -0,0 +1,298 @@
+/**
+ * Wake-word tests — openWakeWord streaming detector.
+ *
+ *   - `OpenWakeWordDetector`: refractory debounce + threshold gating,
+ *     driven by a deterministic scripted `WakeWordModel`.
+ *   - `resolveWakeWordModel`: returns null when the bundle has no
+ *     `wake/openwakeword.gguf` (optional asset).
+ *   - `GgmlWakeWordModel`: routes through the `eliza_inference_wakeword_*`
+ *     FFI surface, surfaces a structured `runtime-not-ready` error when
+ *     the fused build does not export the wake-word symbols (the only
+ *     supported "no wake-word backend" path — there is no ONNX
+ *     fallback, see AGENTS.md §3, §8).
+ *
+ * The real on-device pipeline is covered by an integration test in the
+ * fused-build's test suite (one that actually mmaps a bundled
+ * `openwakeword.gguf` and runs frames through it). That test cannot run
+ * in this package without the native library, so the unit suite here
+ * mocks `ElizaInferenceFfi` and asserts the bindings drive the FFI as
+ * advertised.
+ */
+
+import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs";
+import os from "node:os";
+import path from "node:path";
+import { describe, expect, it, vi } from "vitest";
+import type {
+	ElizaInferenceContextHandle,
+	ElizaInferenceFfi,
+	NativeWakeWordHandle,
+} from "./ffi-bindings";
+import {
+	GgmlWakeWordModel,
+	loadBundledWakeWordModel,
+	OpenWakeWordDetector,
+	resolveWakeWordModel,
+	type WakeWordModel,
+	WakeWordUnavailableError,
+} from "./wake-word";
+
+const FRAME = 1280; // 80 ms @ 16 kHz — what openWakeWord consumes per step.
+
+// --- Deterministic scripted model -----------------------------------------
+
+class ScriptedWakeWordModel implements WakeWordModel {
+	readonly frameSamples = FRAME;
+	readonly sampleRate = 16_000;
+	private idx = 0;
+	resets = 0;
+	scored = 0;
+	constructor(private readonly probs: readonly number[]) {}
+	async scoreFrame(frame: Float32Array): Promise<number> {
+		expect(frame.length).toBe(FRAME);
+		this.scored++;
+		const p = this.probs[this.idx] ?? this.probs[this.probs.length - 1] ?? 0;
+		this.idx++;
+		return p;
+	}
+	reset(): void {
+		this.resets++;
+		this.idx = 0;
+	}
+}
+
+function zeroFrame(): Float32Array {
+	return new Float32Array(FRAME);
+}
+
+describe("OpenWakeWordDetector", () => {
+	it("fires onWake once when probability crosses the threshold", async () => {
+		const model = new ScriptedWakeWordModel([0.1, 0.2, 0.9, 0.95, 0.1]);
+		let fired = 0;
+		const det = new OpenWakeWordDetector({
+			model,
+			config: { threshold: 0.5, refractoryFrames: 10 },
+			onWake: () => fired++,
+		});
+		const hits: boolean[] = [];
+		for (let i = 0; i < 5; i++) hits.push(await det.pushFrame(zeroFrame()));
+		expect(fired).toBe(1);
+		expect(hits).toEqual([false, false, true, false, false]);
+		// Scored every frame, including during the refractory window.
+		expect(model.scored).toBe(5);
+	});
+
+	it("debounces a sustained detection during the refractory window", async () => {
+		const model = new ScriptedWakeWordModel([0.9, 0.9, 0.9, 0.9, 0.9]);
+		let fired = 0;
+		const det = new OpenWakeWordDetector({
+			model,
+			config: { threshold: 0.5, refractoryFrames: 25 },
+			onWake: () => fired++,
+		});
+		for (let i = 0; i < 5; i++) await det.pushFrame(zeroFrame());
+		expect(fired).toBe(1); // fire@0, then 4 frames all inside the 25-frame refractory window
+	});
+
+	it("re-arms after the refractory window elapses", async () => {
+		const model = new ScriptedWakeWordModel([0.9, 0.1, 0.1, 0.9]);
+		let fired = 0;
+		const det = new OpenWakeWordDetector({
+			model,
+			config: { threshold: 0.5, refractoryFrames: 2 },
+			onWake: () => fired++,
+		});
+		for (let i = 0; i < 4; i++) await det.pushFrame(zeroFrame());
+		expect(fired).toBe(2); // fire@0, cooldown 2 → frames 1,2 silent, fire@3
+	});
+
+	it("rejects a wrong-length frame", async () => {
+		const det = new OpenWakeWordDetector({
+			model: new ScriptedWakeWordModel([0.1]),
+			onWake: () => {},
+		});
+		await expect(det.pushFrame(new Float32Array(640))).rejects.toThrow(/1280/);
+	});
+
+	it("reset() clears the cooldown and the model state", async () => {
+		const model = new ScriptedWakeWordModel([0.9, 0.9]);
+		let fired = 0;
+		const det = new OpenWakeWordDetector({
+			model,
+			config: { threshold: 0.5, refractoryFrames: 50 },
+			onWake: () => fired++,
+		});
+		await det.pushFrame(zeroFrame()); // fires, long cooldown
+		expect(fired).toBe(1);
+		det.reset();
+		expect(model.resets).toBe(1);
+		await det.pushFrame(zeroFrame()); // cooldown cleared → fires again
+		expect(fired).toBe(2);
+	});
+});
+
+describe("resolveWakeWordModel", () => {
+	it("returns null when the bundle has no wake-word GGUF (optional asset)", () => {
+		expect(
+			resolveWakeWordModel({ bundleRoot: "/nonexistent/bundle" }),
+		).toBeNull();
+	});
+});
+
+// --- Native FFI routing ---------------------------------------------------
+
+/**
+ * Build a minimal `ElizaInferenceFfi` stand-in that exercises the
+ * wake-word path. `supported` flips the capability probe; the other
+ * methods are spies so the test can assert call shape.
+ */
+function makeMockFfi(supported: boolean): ElizaInferenceFfi {
+	const handle: NativeWakeWordHandle = 0xdeadbeefn;
+	const open = vi.fn(() => handle);
+	const score = vi.fn(() => 0.42);
+	const reset = vi.fn(() => undefined);
+	const close = vi.fn(() => undefined);
+	const ctx: ElizaInferenceContextHandle = 0xcafef00dn;
+	return {
+		libraryPath: "/dev/null",
+		libraryAbiVersion: "5",
+		create: () => ctx,
+		destroy: () => {},
+		mmapAcquire: () => {},
+		mmapEvict: () => {},
+		ttsSynthesize: () => 0,
+		asrTranscribe: () => "",
+		ttsStreamSupported: () => false,
+		ttsSynthesizeStream: () => ({ cancelled: false }),
+		cancelTts: () => {},
+		setVerifierCallback: () => ({ close() {} }),
+		asrStreamSupported: () => false,
+		asrStreamOpen: () => 0n,
+		asrStreamFeed: () => {},
+		asrStreamPartial: () => ({ partial: "" }),
+		asrStreamFinish: () => ({ partial: "" }),
+		asrStreamClose: () => {},
+		wakewordSupported: () => supported,
+		wakewordOpen: supported ? open : undefined,
+		wakewordScore: supported ? score : undefined,
+		wakewordReset: supported ? reset : undefined,
+		wakewordClose: supported ? close : undefined,
+		close: () => {},
+	};
+}
+
+describe("GgmlWakeWordModel", () => {
+	it("throws runtime-not-ready when the FFI does not export wake-word", async () => {
+		const ffi = makeMockFfi(false);
+		const ctx: ElizaInferenceContextHandle = 0xcafef00dn;
+		await expect(
+			GgmlWakeWordModel.load({ ffi, ctx, headName: "hey-eliza" }),
+		).rejects.toMatchObject({
+			name: "WakeWordUnavailableError",
+			code: "runtime-not-ready",
+		});
+	});
+
+	it("isSupported() reflects the FFI capability probe", () => {
+		expect(GgmlWakeWordModel.isSupported(null)).toBe(false);
+		expect(GgmlWakeWordModel.isSupported(makeMockFfi(false))).toBe(false);
+		expect(GgmlWakeWordModel.isSupported(makeMockFfi(true))).toBe(true);
+	});
+
+	it("routes scoreFrame through the FFI and surfaces the probability", async () => {
+		const ffi = makeMockFfi(true);
+		const ctx: ElizaInferenceContextHandle = 0xcafef00dn;
+		const model = await GgmlWakeWordModel.load({
+			ffi,
+			ctx,
+			headName: "hey-eliza",
+		});
+		const p = await model.scoreFrame(new Float32Array(FRAME));
+		expect(p).toBe(0.42);
+		expect(ffi.wakewordScore).toHaveBeenCalledTimes(1);
+	});
+
+	it("rejects a wrong-length frame", async () => {
+		const ffi = makeMockFfi(true);
+		const ctx: ElizaInferenceContextHandle = 0xcafef00dn;
+		const model = await GgmlWakeWordModel.load({
+			ffi,
+			ctx,
+			headName: "hey-eliza",
+		});
+		await expect(model.scoreFrame(new Float32Array(FRAME - 1))).rejects.toThrow(
+			/1280/,
+		);
+	});
+
+	it("reset() and close() drive the matching FFI symbols", async () => {
+		const ffi = makeMockFfi(true);
+		const ctx: ElizaInferenceContextHandle = 0xcafef00dn;
+		const model = await GgmlWakeWordModel.load({
+			ffi,
+			ctx,
+			headName: "hey-eliza",
+		});
+		model.reset();
+		expect(ffi.wakewordReset).toHaveBeenCalledTimes(1);
+		model.close();
+		expect(ffi.wakewordClose).toHaveBeenCalledTimes(1);
+		// Idempotent — calling close again is a no-op (no extra FFI call).
+		model.close();
+		expect(ffi.wakewordClose).toHaveBeenCalledTimes(1);
+	});
+
+	it("wraps a head-bind failure as WakeWordUnavailableError(model-load-failed)", async () => {
+		const ffi = makeMockFfi(true);
+		(
+			ffi.wakewordOpen as unknown as ReturnType<typeof vi.fn>
+		).mockImplementation(() => {
+			throw new Error("[ffi-bindings] unknown head 'banana'");
+		});
+		const ctx: ElizaInferenceContextHandle = 0xcafef00dn;
+		await expect(
+			GgmlWakeWordModel.load({ ffi, ctx, headName: "banana" }),
+		).rejects.toMatchObject({
+			name: "WakeWordUnavailableError",
+			code: "model-load-failed",
+		});
+	});
+});
+
+describe("loadBundledWakeWordModel", () => {
+	it("prefers the fused GgmlWakeWordModel when the bundle GGUF is present and FFI supports it", async () => {
+		const dir = mkdtempSync(path.join(os.tmpdir(), "fused-wake-"));
+		try {
+			const gguf = path.join(dir, "wake", "openwakeword.gguf");
+			mkdirSync(path.dirname(gguf), { recursive: true });
+			writeFileSync(gguf, "");
+			const ffi = makeMockFfi(true);
+			const ctx: ElizaInferenceContextHandle = 0xcafef00dn;
+			const model = await loadBundledWakeWordModel({
+				ffi,
+				ctx,
+				bundleRoot: dir,
+			});
+			expect(model).toBeInstanceOf(GgmlWakeWordModel);
+			// The fused path opened a session via the shared FFI handle — the
+			// standalone wakeword-cpp build was never consulted.
+			expect(ffi.wakewordOpen).toHaveBeenCalledTimes(1);
+		} finally {
+			rmSync(dir, { recursive: true, force: true });
+		}
+	});
+
+	it("returns null when the bundle has no wake-word GGUF and no standalone build", async () => {
+		const ffi = makeMockFfi(true);
+		const ctx: ElizaInferenceContextHandle = 0xcafef00dn;
+		const model = await loadBundledWakeWordModel({
+			ffi,
+			ctx,
+			bundleRoot: "/nonexistent/bundle",
+		});
+		expect(model).toBeNull();
+	});
+});
+
+// Suppress unused-import lints when WakeWordUnavailableError isn't directly referenced.
+void WakeWordUnavailableError;