@elizaos/plugin-local-inference 2.0.0-beta.1 → 2.0.11-beta.7

package/src/services/voice/speaker/diarizer.d.ts

@@ -0,0 +1,75 @@
+/**
+ * pyannote-segmentation-3.0 shared types and pure segmentation logic.
+ *
+ * Diarization runs EXCLUSIVELY through the fused `libelizainference`
+ * `eliza_inference_diariz_*` ABI (`FusedDiarizer` in `diarizer-fused.ts`).
+ * The standalone `libvoice_classifier` binding has been removed — there is one
+ * on-device voice runtime.
+ *
+ * This file holds the shared types (`Diarizer`, `LocalSpeakerSegment`,
+ * `DiarizerOutput`), the model-id / window constants, the structured
+ * `DiarizerUnavailableError`, and the pure `classifyFramesToSegments` reducer
+ * the fused diarizer feeds its per-frame labels through.
+ */
+export declare const PYANNOTE_SEGMENTATION_3_INT8_MODEL_ID: "pyannote-segmentation-3.0-int8";
+export declare const PYANNOTE_SEGMENTATION_3_FP32_MODEL_ID: "pyannote-segmentation-3.0-fp32";
+export type PyannoteDiarizerModelId = typeof PYANNOTE_SEGMENTATION_3_INT8_MODEL_ID | typeof PYANNOTE_SEGMENTATION_3_FP32_MODEL_ID;
+/** pyannote 3.0 segmentation window length (seconds) — model-fixed. */
+export declare const PYANNOTE_WINDOW_SECONDS = 5;
+/** Required mono sample rate (matches upstream training config). */
+export declare const PYANNOTE_SAMPLE_RATE = 16000;
+/** Number of output frames per 5 s window (= 293 in the upstream export). */
+export declare const PYANNOTE_FRAMES_PER_WINDOW = 293;
+/** Per-frame stride in milliseconds (5_000ms / 293 frames ≈ 17.06 ms). */
+export declare const PYANNOTE_FRAME_STRIDE_MS: number;
+/** Output class count — 3 single + 3 overlap + 1 silence = 7. */
+export declare const PYANNOTE_CLASS_COUNT = 7;
+/**
+ * Powerset mapping of pyannote-3 segmentation classes. Each class is
+ * the set of local speaker indices active in that frame. Class 0 is the
+ * silence/no-speaker frame. This matches the upstream `Powerset` head
+ * with `max_speakers_per_chunk=3, max_speakers_per_frame=2`.
+ */
+export declare const PYANNOTE_CLASS_TO_SPEAKERS: ReadonlyArray<ReadonlyArray<number>>;
+/** Thrown when the diarizer cannot be constructed. */
+export declare class DiarizerUnavailableError extends Error {
+    readonly code: "ort-missing" | "native-missing" | "library-missing" | "model-missing" | "model-unavailable" | "model-load-failed" | "model-shape-mismatch" | "forward-not-implemented" | "invalid-input";
+    constructor(code: DiarizerUnavailableError["code"], message: string);
+}
+/**
+ * One speaker-tagged span within a diarized window. `localSpeakerId` is
+ * **window-local** (0..2): the same physical speaker gets different
+ * local ids in different windows. The profile store re-clusters local
+ * ids into stable identities via the WeSpeaker embedding cosine.
+ */
+export interface LocalSpeakerSegment {
+    startMs: number;
+    endMs: number;
+    localSpeakerId: number;
+    /** Best class confidence over the span (max softmax). */
+    confidence: number;
+    /** True if the span contains any overlap-class frames. */
+    hasOverlap: boolean;
+}
+export interface DiarizerOutput {
+    segments: LocalSpeakerSegment[];
+    /** Number of distinct local speakers observed in the window. */
+    localSpeakerCount: number;
+    /** Total speech (any-speaker) duration in milliseconds. */
+    speechMs: number;
+}
+export interface Diarizer {
+    readonly modelId: PyannoteDiarizerModelId;
+    readonly sampleRate: number;
+    /** Process one ~5 s window of PCM. */
+    diarizeWindow(pcm: Float32Array): Promise<DiarizerOutput>;
+    dispose(): Promise<void>;
+}
+/**
+ * Reduce a per-frame class probability tensor into one segment per
+ * (local speaker × contiguous frame run). Frames where the silence
+ * class wins are excluded; frames in overlap classes contribute to
+ * **all** speakers in that class.
+ */
+export declare function classifyFramesToSegments(classProbs: Float32Array, frames: number, classCount: number, startMs: number, frameStrideMs: number): DiarizerOutput;
+//# sourceMappingURL=diarizer.d.ts.map

package/src/services/voice/speaker/diarizer.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"diarizer.d.ts","sourceRoot":"","sources":["diarizer.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AAEH,eAAO,MAAM,qCAAqC,EACjD,gCAAyC,CAAC;AAC3C,eAAO,MAAM,qCAAqC,EACjD,gCAAyC,CAAC;AAC3C,MAAM,MAAM,uBAAuB,GAChC,OAAO,qCAAqC,GAC5C,OAAO,qCAAqC,CAAC;AAEhD,uEAAuE;AACvE,eAAO,MAAM,uBAAuB,IAAI,CAAC;AACzC,oEAAoE;AACpE,eAAO,MAAM,oBAAoB,QAAS,CAAC;AAC3C,6EAA6E;AAC7E,eAAO,MAAM,0BAA0B,MAAM,CAAC;AAC9C,0EAA0E;AAC1E,eAAO,MAAM,wBAAwB,QAC0B,CAAC;AAChE,iEAAiE;AACjE,eAAO,MAAM,oBAAoB,IAAI,CAAC;AAEtC;;;;;GAKG;AACH,eAAO,MAAM,0BAA0B,EAAE,aAAa,CAAC,aAAa,CAAC,MAAM,CAAC,CAS1E,CAAC;AAEH,sDAAsD;AACtD,qBAAa,wBAAyB,SAAQ,KAAK;IAClD,QAAQ,CAAC,IAAI,EACV,aAAa,GACb,gBAAgB,GAChB,iBAAiB,GACjB,eAAe,GACf,mBAAmB,GACnB,mBAAmB,GACnB,sBAAsB,GACtB,yBAAyB,GACzB,eAAe,CAAC;gBACP,IAAI,EAAE,wBAAwB,CAAC,MAAM,CAAC,EAAE,OAAO,EAAE,MAAM;CAKnE;AAED;;;;;GAKG;AACH,MAAM,WAAW,mBAAmB;IACnC,OAAO,EAAE,MAAM,CAAC;IAChB,KAAK,EAAE,MAAM,CAAC;IACd,cAAc,EAAE,MAAM,CAAC;IACvB,yDAAyD;IACzD,UAAU,EAAE,MAAM,CAAC;IACnB,0DAA0D;IAC1D,UAAU,EAAE,OAAO,CAAC;CACpB;AAED,MAAM,WAAW,cAAc;IAC9B,QAAQ,EAAE,mBAAmB,EAAE,CAAC;IAChC,gEAAgE;IAChE,iBAAiB,EAAE,MAAM,CAAC;IAC1B,2DAA2D;IAC3D,QAAQ,EAAE,MAAM,CAAC;CACjB;AAED,MAAM,WAAW,QAAQ;IACxB,QAAQ,CAAC,OAAO,EAAE,uBAAuB,CAAC;IAC1C,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,sCAAsC;IACtC,aAAa,CAAC,GAAG,EAAE,YAAY,GAAG,OAAO,CAAC,cAAc,CAAC,CAAC;IAC1D,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;CACzB;AAmBD;;;;;GAKG;AACH,wBAAgB,wBAAwB,CACvC,UAAU,EAAE,YAAY,EACxB,MAAM,EAAE,MAAM,EACd,UAAU,EAAE,MAAM,EAClB,OAAO,EAAE,MAAM,EACf,aAAa,EAAE,MAAM,GACnB,cAAc,CAsFhB"}

package/src/services/voice/speaker/diarizer.ts

@@ -0,0 +1,218 @@
+/**
+ * pyannote-segmentation-3.0 shared types and pure segmentation logic.
+ *
+ * Diarization runs EXCLUSIVELY through the fused `libelizainference`
+ * `eliza_inference_diariz_*` ABI (`FusedDiarizer` in `diarizer-fused.ts`).
+ * The standalone `libvoice_classifier` binding has been removed — there is one
+ * on-device voice runtime.
+ *
+ * This file holds the shared types (`Diarizer`, `LocalSpeakerSegment`,
+ * `DiarizerOutput`), the model-id / window constants, the structured
+ * `DiarizerUnavailableError`, and the pure `classifyFramesToSegments` reducer
+ * the fused diarizer feeds its per-frame labels through.
+ */
+
+export const PYANNOTE_SEGMENTATION_3_INT8_MODEL_ID =
+	"pyannote-segmentation-3.0-int8" as const;
+export const PYANNOTE_SEGMENTATION_3_FP32_MODEL_ID =
+	"pyannote-segmentation-3.0-fp32" as const;
+export type PyannoteDiarizerModelId =
+	| typeof PYANNOTE_SEGMENTATION_3_INT8_MODEL_ID
+	| typeof PYANNOTE_SEGMENTATION_3_FP32_MODEL_ID;
+
+/** pyannote 3.0 segmentation window length (seconds) — model-fixed. */
+export const PYANNOTE_WINDOW_SECONDS = 5;
+/** Required mono sample rate (matches upstream training config). */
+export const PYANNOTE_SAMPLE_RATE = 16_000;
+/** Number of output frames per 5 s window (= 293 in the upstream export). */
+export const PYANNOTE_FRAMES_PER_WINDOW = 293;
+/** Per-frame stride in milliseconds (5_000ms / 293 frames ≈ 17.06 ms). */
+export const PYANNOTE_FRAME_STRIDE_MS =
+	(1_000 * PYANNOTE_WINDOW_SECONDS) / PYANNOTE_FRAMES_PER_WINDOW;
+/** Output class count — 3 single + 3 overlap + 1 silence = 7. */
+export const PYANNOTE_CLASS_COUNT = 7;
+
+/**
+ * Powerset mapping of pyannote-3 segmentation classes. Each class is
+ * the set of local speaker indices active in that frame. Class 0 is the
+ * silence/no-speaker frame. This matches the upstream `Powerset` head
+ * with `max_speakers_per_chunk=3, max_speakers_per_frame=2`.
+ */
+export const PYANNOTE_CLASS_TO_SPEAKERS: ReadonlyArray<ReadonlyArray<number>> =
+	[
+		[], // 0: silence
+		[0], // 1: speaker 0 only
+		[1], // 2: speaker 1 only
+		[2], // 3: speaker 2 only
+		[0, 1], // 4: speakers 0+1 overlap
+		[0, 2], // 5: speakers 0+2 overlap
+		[1, 2], // 6: speakers 1+2 overlap
+	];
+
+/** Thrown when the diarizer cannot be constructed. */
+export class DiarizerUnavailableError extends Error {
+	readonly code:
+		| "ort-missing"
+		| "native-missing"
+		| "library-missing"
+		| "model-missing"
+		| "model-unavailable"
+		| "model-load-failed"
+		| "model-shape-mismatch"
+		| "forward-not-implemented"
+		| "invalid-input";
+	constructor(code: DiarizerUnavailableError["code"], message: string) {
+		super(message);
+		this.name = "DiarizerUnavailableError";
+		this.code = code;
+	}
+}
+
+/**
+ * One speaker-tagged span within a diarized window. `localSpeakerId` is
+ * **window-local** (0..2): the same physical speaker gets different
+ * local ids in different windows. The profile store re-clusters local
+ * ids into stable identities via the WeSpeaker embedding cosine.
+ */
+export interface LocalSpeakerSegment {
+	startMs: number;
+	endMs: number;
+	localSpeakerId: number;
+	/** Best class confidence over the span (max softmax). */
+	confidence: number;
+	/** True if the span contains any overlap-class frames. */
+	hasOverlap: boolean;
+}
+
+export interface DiarizerOutput {
+	segments: LocalSpeakerSegment[];
+	/** Number of distinct local speakers observed in the window. */
+	localSpeakerCount: number;
+	/** Total speech (any-speaker) duration in milliseconds. */
+	speechMs: number;
+}
+
+export interface Diarizer {
+	readonly modelId: PyannoteDiarizerModelId;
+	readonly sampleRate: number;
+	/** Process one ~5 s window of PCM. */
+	diarizeWindow(pcm: Float32Array): Promise<DiarizerOutput>;
+	dispose(): Promise<void>;
+}
+
+/** Numerically-stable softmax over the last axis. */
+function softmax(row: Float32Array): Float32Array {
+	let max = -Infinity;
+	for (let i = 0; i < row.length; i += 1) {
+		if (row[i] > max) max = row[i];
+	}
+	const out = new Float32Array(row.length);
+	let sum = 0;
+	for (let i = 0; i < row.length; i += 1) {
+		out[i] = Math.exp(row[i] - max);
+		sum += out[i];
+	}
+	if (sum === 0) return out;
+	for (let i = 0; i < row.length; i += 1) out[i] /= sum;
+	return out;
+}
+
+/**
+ * Reduce a per-frame class probability tensor into one segment per
+ * (local speaker × contiguous frame run). Frames where the silence
+ * class wins are excluded; frames in overlap classes contribute to
+ * **all** speakers in that class.
+ */
+export function classifyFramesToSegments(
+	classProbs: Float32Array,
+	frames: number,
+	classCount: number,
+	startMs: number,
+	frameStrideMs: number,
+): DiarizerOutput {
+	if (classProbs.length !== frames * classCount) {
+		throw new DiarizerUnavailableError(
+			"model-load-failed",
+			`[pyannote] frame×class tensor mismatch: have ${classProbs.length}, expected ${frames * classCount}`,
+		);
+	}
+	type Active = {
+		startFrame: number;
+		endFrame: number;
+		confSum: number;
+		count: number;
+		hasOverlap: boolean;
+	};
+	// Per-speaker active runs. The pyannote-3 head supports 3 speakers.
+	const open = new Map<number, Active>();
+	const closed: Array<Active & { speakerId: number }> = [];
+
+	let speechFrames = 0;
+
+	for (let f = 0; f < frames; f += 1) {
+		const offset = f * classCount;
+		const row = classProbs.subarray(offset, offset + classCount);
+		const probs = softmax(row);
+		// Pick winning class.
+		let winner = 0;
+		let winnerProb = probs[0];
+		for (let c = 1; c < classCount; c += 1) {
+			if (probs[c] > winnerProb) {
+				winner = c;
+				winnerProb = probs[c];
+			}
+		}
+		const activeSpeakers = PYANNOTE_CLASS_TO_SPEAKERS[winner] ?? [];
+		const isOverlap = activeSpeakers.length > 1;
+		if (activeSpeakers.length > 0) speechFrames += 1;
+
+		// Close runs for speakers not active this frame.
+		for (const [sid, run] of open.entries()) {
+			if (!activeSpeakers.includes(sid)) {
+				closed.push({ ...run, speakerId: sid });
+				open.delete(sid);
+			}
+		}
+		// Open / extend runs for active speakers.
+		for (const sid of activeSpeakers) {
+			const existing = open.get(sid);
+			if (existing) {
+				existing.endFrame = f + 1;
+				existing.confSum += winnerProb;
+				existing.count += 1;
+				existing.hasOverlap = existing.hasOverlap || isOverlap;
+			} else {
+				open.set(sid, {
+					startFrame: f,
+					endFrame: f + 1,
+					confSum: winnerProb,
+					count: 1,
+					hasOverlap: isOverlap,
+				});
+			}
+		}
+	}
+	// Flush remaining open runs.
+	for (const [sid, run] of open.entries()) {
+		closed.push({ ...run, speakerId: sid });
+	}
+
+	const segments = closed
+		.map<LocalSpeakerSegment>((run) => ({
+			startMs: Math.round(startMs + run.startFrame * frameStrideMs),
+			endMs: Math.round(startMs + run.endFrame * frameStrideMs),
+			localSpeakerId: run.speakerId,
+			confidence: run.count > 0 ? run.confSum / run.count : 0,
+			hasOverlap: run.hasOverlap,
+		}))
+		.sort((a, b) =>
+			a.startMs !== b.startMs ? a.startMs - b.startMs : a.endMs - b.endMs,
+		);
+
+	const localSpeakers = new Set(segments.map((s) => s.localSpeakerId));
+	return {
+		segments,
+		localSpeakerCount: localSpeakers.size,
+		speechMs: Math.round(speechFrames * frameStrideMs),
+	};
+}

package/src/services/voice/speaker/encoder-fused.d.ts

@@ -0,0 +1,60 @@
+/**
+ * Speaker-embedding encoder — fused `libelizainference` binding (ABI v6).
+ *
+ * The strategic on-device voice engine is the single fused-FFI
+ * `libelizainference` library (the merged llama.cpp fork — see
+ * `plugins/plugin-local-inference/native/CLAUDE.md` §1). This class drives the
+ * WeSpeaker ResNet34-LM speaker encoder through that one native handle via the
+ * `eliza_inference_speaker_*` ABI. This is the SOLE on-device speaker-encoder
+ * runtime — the same `ffi`/`ctx` pair powers VAD / wake-word / TTS / ASR, so the
+ * whole voice pipeline runs through one library.
+ *
+ * Shape mirrors the legacy `encoder.ts::SpeakerEncoder` contract exactly:
+ *   - 16 kHz mono fp32 PCM in,
+ *   - one L2-normalized 256-d embedding out,
+ *   - `encode(pcm)` / `dispose()`.
+ *
+ * No silent fallback: when the fused build does not export the speaker ABI
+ * (`eliza_inference_speaker_supported() == 0`) `load()` throws a structured
+ * `SpeakerEncoderGgmlUnavailableError` (AGENTS.md §3 — no synthetic
+ * embeddings, no standalone-lib fallback).
+ */
+import type { ElizaInferenceContextHandle, ElizaInferenceFfi } from "../ffi-bindings";
+import type { SpeakerEncoder } from "./encoder";
+export interface FusedSpeakerEncoderOptions {
+    ffi: ElizaInferenceFfi;
+    ctx: ElizaInferenceContextHandle | (() => ElizaInferenceContextHandle);
+    /**
+     * Optional explicit WeSpeaker GGUF path. `null` lets the native runtime
+     * resolve the bundle's `speaker/` dir (the default).
+     */
+    ggufPath?: string | null;
+}
+/**
+ * Fused-`libelizainference` WeSpeaker speaker encoder. Owns one
+ * `eliza_inference_speaker_*` session; `encode()` runs one forward pass over
+ * the supplied 16 kHz PCM and returns the normalized 256-d embedding. The
+ * native side owns the model graph; this class is a thin handle.
+ */
+export declare class FusedSpeakerEncoder implements SpeakerEncoder {
+    private readonly ffi;
+    private readonly handle;
+    readonly embeddingDim = 256;
+    readonly sampleRate = 16000;
+    readonly modelId: "wespeaker-resnet34-lm-int8";
+    private disposed;
+    private constructor();
+    /**
+     * True only when the fused `libelizainference` build exports the speaker
+     * ABI and advertises support at runtime.
+     */
+    static isSupported(ffi: ElizaInferenceFfi | null | undefined): boolean;
+    /**
+     * Open a native speaker-encoder session. Throws
+     * `SpeakerEncoderGgmlUnavailableError` when the runtime is not present.
+     */
+    static load(opts: FusedSpeakerEncoderOptions): Promise<FusedSpeakerEncoder>;
+    encode(pcm: Float32Array): Promise<Float32Array>;
+    dispose(): Promise<void>;
+}
+//# sourceMappingURL=encoder-fused.d.ts.map

package/src/services/voice/speaker/encoder-fused.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"encoder-fused.d.ts","sourceRoot":"","sources":["encoder-fused.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;GAoBG;AAEH,OAAO,KAAK,EACX,2BAA2B,EAC3B,iBAAiB,EAEjB,MAAM,iBAAiB,CAAC;AACzB,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,WAAW,CAAC;AAShD,MAAM,WAAW,0BAA0B;IAC1C,GAAG,EAAE,iBAAiB,CAAC;IACvB,GAAG,EAAE,2BAA2B,GAAG,CAAC,MAAM,2BAA2B,CAAC,CAAC;IACvE;;;OAGG;IACH,QAAQ,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;CACzB;AAED;;;;;GAKG;AACH,qBAAa,mBAAoB,YAAW,cAAc;IAOxD,OAAO,CAAC,QAAQ,CAAC,GAAG;IACpB,OAAO,CAAC,QAAQ,CAAC,MAAM;IAPxB,QAAQ,CAAC,YAAY,OAA8B;IACnD,QAAQ,CAAC,UAAU,SAA4B;IAC/C,QAAQ,CAAC,OAAO,+BAAuC;IACvD,OAAO,CAAC,QAAQ,CAAS;IAEzB,OAAO;IAKP;;;OAGG;IACH,MAAM,CAAC,WAAW,CAAC,GAAG,EAAE,iBAAiB,GAAG,IAAI,GAAG,SAAS,GAAG,OAAO;IAKtE;;;OAGG;WACU,IAAI,CAChB,IAAI,EAAE,0BAA0B,GAC9B,OAAO,CAAC,mBAAmB,CAAC;IAyBzB,MAAM,CAAC,GAAG,EAAE,YAAY,GAAG,OAAO,CAAC,YAAY,CAAC;IA6BhD,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC;CAK9B"}

package/src/services/voice/speaker/encoder-fused.real.test.ts

@@ -0,0 +1,113 @@
+/**
+ * Real-FFI tests for `FusedSpeakerEncoder`: run against the ACTUAL fused
+ * `libelizainference` — loaded, `create`d, and probed for `speakerSupported()`
+ * — never a stub. The speaker encoder is the SOLE on-device speaker runtime
+ * (the `eliza_inference_speaker_*` ABI off the one fused handle).
+ *
+ * Skipped (not faked) when the fused lib is not resolvable, or when it does not
+ * link the WeSpeaker speaker graph. To run them, point `ELIZA_INFERENCE_LIBRARY`
+ * (or `ELIZA_INFERENCE_LIB_DIR`) at a built `libelizainference` with the speaker
+ * ABI, or build one via `packages/app-core/scripts/build-llama-cpp-mtp.mjs`.
+ * Runs in the post-merge `bun test` lane (`*.real.test.ts` is excluded from the
+ * default lane in `vitest.config.ts`).
+ */
+
+import { existsSync, mkdtempSync, rmSync } from "node:fs";
+import os from "node:os";
+import path from "node:path";
+import {
+	afterAll,
+	afterEach,
+	beforeAll,
+	beforeEach,
+	describe,
+	expect,
+	it,
+} from "vitest";
+
+import { resolveFusedLibraryPath } from "../../desktop-fused-ffi-backend-runtime";
+import {
+	type ElizaInferenceContextHandle,
+	type ElizaInferenceFfi,
+	loadElizaInferenceFfi,
+} from "../ffi-bindings";
+import { FusedSpeakerEncoder } from "./encoder-fused";
+
+const EMB_DIM = 256;
+const MIN_SAMPLES = 16_000;
+
+const isBun = typeof (globalThis as { Bun?: unknown }).Bun !== "undefined";
+const LIB_PATH = resolveFusedLibraryPath(null, process.env);
+// The native speaker_open needs a WeSpeaker GGUF. Provide one via
+// ELIZA_TEST_SPEAKER_GGUF (e.g. wespeaker-resnet34-lm.gguf); the encode
+// assertions skip honestly when it isn't supplied — they are never faked.
+const SPEAKER_GGUF = process.env.ELIZA_TEST_SPEAKER_GGUF?.trim();
+const HAVE_MODEL = !!SPEAKER_GGUF && existsSync(SPEAKER_GGUF);
+
+describe.skipIf(!isBun || !LIB_PATH)("FusedSpeakerEncoder — real FFI", () => {
+	let ffi: ElizaInferenceFfi;
+	let ctx: ElizaInferenceContextHandle;
+	let tmp: string;
+
+	beforeAll(() => {
+		// LIB_PATH is non-null inside the skipIf-guarded block.
+		ffi = loadElizaInferenceFfi(LIB_PATH as string);
+	});
+	afterAll(() => {
+		ffi?.close();
+	});
+	beforeEach(() => {
+		tmp = mkdtempSync(path.join(os.tmpdir(), "speaker-fused-real-"));
+		ctx = ffi.create(tmp);
+	});
+	afterEach(() => {
+		ffi.destroy(ctx);
+		rmSync(tmp, { recursive: true, force: true });
+	});
+
+	it("isSupported() reflects the loaded build's speaker ABI", () => {
+		expect(typeof FusedSpeakerEncoder.isSupported(ffi)).toBe("boolean");
+	});
+
+	it.skipIf(!HAVE_MODEL)(
+		"encode() returns a finite 256-d embedding off the real WeSpeaker graph",
+		async () => {
+			const enc = await FusedSpeakerEncoder.load({
+				ffi,
+				ctx,
+				ggufPath: SPEAKER_GGUF,
+			});
+			expect(enc.embeddingDim).toBe(EMB_DIM);
+			expect(enc.sampleRate).toBe(MIN_SAMPLES);
+			// 1 s of a 220 Hz tone — a real, finite input the native graph accepts.
+			const pcm = new Float32Array(MIN_SAMPLES);
+			for (let i = 0; i < pcm.length; i += 1) {
+				pcm[i] = 0.2 * Math.sin((2 * Math.PI * 220 * i) / MIN_SAMPLES);
+			}
+			const emb = await enc.encode(pcm);
+			expect(emb.length).toBe(EMB_DIM);
+			expect(emb.every((v) => Number.isFinite(v))).toBe(true);
+			// A non-degenerate embedding has real magnitude.
+			let norm = 0;
+			for (const v of emb) norm += v * v;
+			expect(Math.sqrt(norm)).toBeGreaterThan(0);
+			await enc.dispose();
+		},
+	);
+
+	it.skipIf(!HAVE_MODEL)(
+		"rejects pcm shorter than the minimum window before hitting the native graph",
+		async () => {
+			const enc = await FusedSpeakerEncoder.load({
+				ffi,
+				ctx,
+				ggufPath: SPEAKER_GGUF,
+			});
+			await expect(enc.encode(new Float32Array(100))).rejects.toMatchObject({
+				name: "SpeakerEncoderGgmlUnavailableError",
+				code: "invalid-input",
+			});
+			await enc.dispose();
+		},
+	);
+});

package/src/services/voice/speaker/encoder-fused.ts

@@ -0,0 +1,138 @@
+/**
+ * Speaker-embedding encoder — fused `libelizainference` binding (ABI v6).
+ *
+ * The strategic on-device voice engine is the single fused-FFI
+ * `libelizainference` library (the merged llama.cpp fork — see
+ * `plugins/plugin-local-inference/native/CLAUDE.md` §1). This class drives the
+ * WeSpeaker ResNet34-LM speaker encoder through that one native handle via the
+ * `eliza_inference_speaker_*` ABI. This is the SOLE on-device speaker-encoder
+ * runtime — the same `ffi`/`ctx` pair powers VAD / wake-word / TTS / ASR, so the
+ * whole voice pipeline runs through one library.
+ *
+ * Shape mirrors the legacy `encoder.ts::SpeakerEncoder` contract exactly:
+ *   - 16 kHz mono fp32 PCM in,
+ *   - one L2-normalized 256-d embedding out,
+ *   - `encode(pcm)` / `dispose()`.
+ *
+ * No silent fallback: when the fused build does not export the speaker ABI
+ * (`eliza_inference_speaker_supported() == 0`) `load()` throws a structured
+ * `SpeakerEncoderGgmlUnavailableError` (AGENTS.md §3 — no synthetic
+ * embeddings, no standalone-lib fallback).
+ */
+
+import type {
+	ElizaInferenceContextHandle,
+	ElizaInferenceFfi,
+	NativeSpeakerHandle,
+} from "../ffi-bindings";
+import type { SpeakerEncoder } from "./encoder";
+import { WESPEAKER_RESNET34_LM_INT8_MODEL_ID } from "./encoder";
+import {
+	SPEAKER_GGML_EMBEDDING_DIM,
+	SPEAKER_GGML_MIN_SAMPLES,
+	SPEAKER_GGML_SAMPLE_RATE,
+	SpeakerEncoderGgmlUnavailableError,
+} from "./encoder-ggml";
+
+export interface FusedSpeakerEncoderOptions {
+	ffi: ElizaInferenceFfi;
+	ctx: ElizaInferenceContextHandle | (() => ElizaInferenceContextHandle);
+	/**
+	 * Optional explicit WeSpeaker GGUF path. `null` lets the native runtime
+	 * resolve the bundle's `speaker/` dir (the default).
+	 */
+	ggufPath?: string | null;
+}
+
+/**
+ * Fused-`libelizainference` WeSpeaker speaker encoder. Owns one
+ * `eliza_inference_speaker_*` session; `encode()` runs one forward pass over
+ * the supplied 16 kHz PCM and returns the normalized 256-d embedding. The
+ * native side owns the model graph; this class is a thin handle.
+ */
+export class FusedSpeakerEncoder implements SpeakerEncoder {
+	readonly embeddingDim = SPEAKER_GGML_EMBEDDING_DIM;
+	readonly sampleRate = SPEAKER_GGML_SAMPLE_RATE;
+	readonly modelId = WESPEAKER_RESNET34_LM_INT8_MODEL_ID;
+	private disposed = false;
+
+	private constructor(
+		private readonly ffi: ElizaInferenceFfi,
+		private readonly handle: NativeSpeakerHandle,
+	) {}
+
+	/**
+	 * True only when the fused `libelizainference` build exports the speaker
+	 * ABI and advertises support at runtime.
+	 */
+	static isSupported(ffi: ElizaInferenceFfi | null | undefined): boolean {
+		if (!ffi || typeof ffi.speakerSupported !== "function") return false;
+		return ffi.speakerSupported();
+	}
+
+	/**
+	 * Open a native speaker-encoder session. Throws
+	 * `SpeakerEncoderGgmlUnavailableError` when the runtime is not present.
+	 */
+	static async load(
+		opts: FusedSpeakerEncoderOptions,
+	): Promise<FusedSpeakerEncoder> {
+		if (!FusedSpeakerEncoder.isSupported(opts.ffi)) {
+			throw new SpeakerEncoderGgmlUnavailableError(
+				"native-missing",
+				"[speaker-fused] The native speaker encoder is not present in this libelizainference build. Rebuild with the WeSpeaker forward graph linked in (eliza_inference_speaker_* symbols).",
+			);
+		}
+		if (
+			!opts.ffi.speakerOpen ||
+			!opts.ffi.speakerEmbed ||
+			!opts.ffi.speakerClose
+		) {
+			throw new SpeakerEncoderGgmlUnavailableError(
+				"model-load-failed",
+				"[speaker-fused] Speaker support probe succeeded, but the required FFI methods are missing on the binding.",
+			);
+		}
+		const ctx = typeof opts.ctx === "function" ? opts.ctx() : opts.ctx;
+		const handle = opts.ffi.speakerOpen({
+			ctx,
+			ggufPath: opts.ggufPath ?? null,
+		});
+		return new FusedSpeakerEncoder(opts.ffi, handle);
+	}
+
+	async encode(pcm: Float32Array): Promise<Float32Array> {
+		if (this.disposed) {
+			throw new SpeakerEncoderGgmlUnavailableError(
+				"model-load-failed",
+				"[speaker-fused] encode called after dispose()",
+			);
+		}
+		if (!(pcm instanceof Float32Array)) {
+			throw new SpeakerEncoderGgmlUnavailableError(
+				"invalid-input",
+				"[speaker-fused] pcm must be a Float32Array",
+			);
+		}
+		if (pcm.length < SPEAKER_GGML_MIN_SAMPLES) {
+			throw new SpeakerEncoderGgmlUnavailableError(
+				"invalid-input",
+				`[speaker-fused] pcm too short: ${pcm.length} samples < ${SPEAKER_GGML_MIN_SAMPLES}`,
+			);
+		}
+		const embed = this.ffi.speakerEmbed;
+		if (!embed) {
+			throw new SpeakerEncoderGgmlUnavailableError(
+				"model-load-failed",
+				"[speaker-fused] encode missing FFI method",
+			);
+		}
+		return embed({ speaker: this.handle, pcm });
+	}
+
+	async dispose(): Promise<void> {
+		if (this.disposed) return;
+		this.disposed = true;
+		this.ffi.speakerClose?.(this.handle);
+	}
+}

package/src/services/voice/speaker/encoder-ggml.d.ts

@@ -0,0 +1,33 @@
+/**
+ * Speaker-embedding encoder — shared constants, error class, and the
+ * embedding-distance helper.
+ *
+ * The speaker encoder runs EXCLUSIVELY through the fused `libelizainference`
+ * `eliza_inference_speaker_*` ABI (`FusedSpeakerEncoder` in `encoder-fused.ts`).
+ * The standalone `libvoice_classifier` binding that previously lived here has
+ * been removed — there is one on-device voice runtime.
+ *
+ * This module retains the pieces the fused path shares:
+ *   - the canonical dims (`SPEAKER_GGML_*`), pinned at 256 to match the C-side
+ *     `VOICE_SPEAKER_EMBEDDING_DIM` and the WeSpeaker ResNet34-LM head,
+ *   - the structured `SpeakerEncoderGgmlUnavailableError` the fused encoder
+ *     throws (no synthetic embedding fallback),
+ *   - the pure `voiceSpeakerDistance` cosine-distance helper.
+ */
+/** Output embedding dim. Matches `VOICE_SPEAKER_EMBEDDING_DIM`. */
+export declare const SPEAKER_GGML_EMBEDDING_DIM = 256;
+/** Required input sample rate. */
+export declare const SPEAKER_GGML_SAMPLE_RATE = 16000;
+/** Minimum useful audio window (~1.0 s). */
+export declare const SPEAKER_GGML_MIN_SAMPLES = 16000;
+export declare class SpeakerEncoderGgmlUnavailableError extends Error {
+    readonly code: "native-missing" | "library-missing" | "model-missing" | "model-load-failed" | "model-shape-mismatch" | "forward-not-implemented" | "invalid-input";
+    constructor(code: SpeakerEncoderGgmlUnavailableError["code"], message: string);
+}
+/**
+ * Cosine distance between two 256-dim speaker embeddings. Defined as
+ * `1 - cos_similarity(a, b)`, range [0, 2]. Mirrors the C-side
+ * `voice_speaker_distance` helper exactly.
+ */
+export declare function voiceSpeakerDistance(a: Float32Array, b: Float32Array): number;
+//# sourceMappingURL=encoder-ggml.d.ts.map

package/src/services/voice/speaker/encoder-ggml.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"encoder-ggml.d.ts","sourceRoot":"","sources":["encoder-ggml.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;GAeG;AAEH,mEAAmE;AACnE,eAAO,MAAM,0BAA0B,MAAM,CAAC;AAE9C,kCAAkC;AAClC,eAAO,MAAM,wBAAwB,QAAS,CAAC;AAE/C,4CAA4C;AAC5C,eAAO,MAAM,wBAAwB,QAAS,CAAC;AAE/C,qBAAa,kCAAmC,SAAQ,KAAK;IAC5D,QAAQ,CAAC,IAAI,EACV,gBAAgB,GAChB,iBAAiB,GACjB,eAAe,GACf,mBAAmB,GACnB,sBAAsB,GACtB,yBAAyB,GACzB,eAAe,CAAC;gBAElB,IAAI,EAAE,kCAAkC,CAAC,MAAM,CAAC,EAChD,OAAO,EAAE,MAAM;CAMhB;AAED;;;;GAIG;AACH,wBAAgB,oBAAoB,CAAC,CAAC,EAAE,YAAY,EAAE,CAAC,EAAE,YAAY,GAAG,MAAM,CA4B7E"}