@elizaos/plugin-local-inference 2.0.0-beta.1 → 2.0.3-beta.3

package/src/services/voice/speaker/encoder-fused.ts

@@ -0,0 +1,138 @@
+/**
+ * Speaker-embedding encoder — fused `libelizainference` binding (ABI v6).
+ *
+ * The strategic on-device voice engine is the single fused-FFI
+ * `libelizainference` library (the merged llama.cpp fork — see
+ * `plugins/plugin-local-inference/native/CLAUDE.md` §1). This class drives the
+ * WeSpeaker ResNet34-LM speaker encoder through that one native handle via the
+ * `eliza_inference_speaker_*` ABI. This is the SOLE on-device speaker-encoder
+ * runtime — the same `ffi`/`ctx` pair powers VAD / wake-word / TTS / ASR, so the
+ * whole voice pipeline runs through one library.
+ *
+ * Shape mirrors the legacy `encoder.ts::SpeakerEncoder` contract exactly:
+ *   - 16 kHz mono fp32 PCM in,
+ *   - one L2-normalized 256-d embedding out,
+ *   - `encode(pcm)` / `dispose()`.
+ *
+ * No silent fallback: when the fused build does not export the speaker ABI
+ * (`eliza_inference_speaker_supported() == 0`) `load()` throws a structured
+ * `SpeakerEncoderGgmlUnavailableError` (AGENTS.md §3 — no synthetic
+ * embeddings, no standalone-lib fallback).
+ */
+
+import type {
+	ElizaInferenceContextHandle,
+	ElizaInferenceFfi,
+	NativeSpeakerHandle,
+} from "../ffi-bindings";
+import type { SpeakerEncoder } from "./encoder";
+import { WESPEAKER_RESNET34_LM_INT8_MODEL_ID } from "./encoder";
+import {
+	SPEAKER_GGML_EMBEDDING_DIM,
+	SPEAKER_GGML_MIN_SAMPLES,
+	SPEAKER_GGML_SAMPLE_RATE,
+	SpeakerEncoderGgmlUnavailableError,
+} from "./encoder-ggml";
+
+export interface FusedSpeakerEncoderOptions {
+	ffi: ElizaInferenceFfi;
+	ctx: ElizaInferenceContextHandle | (() => ElizaInferenceContextHandle);
+	/**
+	 * Optional explicit WeSpeaker GGUF path. `null` lets the native runtime
+	 * resolve the bundle's `speaker/` dir (the default).
+	 */
+	ggufPath?: string | null;
+}
+
+/**
+ * Fused-`libelizainference` WeSpeaker speaker encoder. Owns one
+ * `eliza_inference_speaker_*` session; `encode()` runs one forward pass over
+ * the supplied 16 kHz PCM and returns the normalized 256-d embedding. The
+ * native side owns the model graph; this class is a thin handle.
+ */
+export class FusedSpeakerEncoder implements SpeakerEncoder {
+	readonly embeddingDim = SPEAKER_GGML_EMBEDDING_DIM;
+	readonly sampleRate = SPEAKER_GGML_SAMPLE_RATE;
+	readonly modelId = WESPEAKER_RESNET34_LM_INT8_MODEL_ID;
+	private disposed = false;
+
+	private constructor(
+		private readonly ffi: ElizaInferenceFfi,
+		private readonly handle: NativeSpeakerHandle,
+	) {}
+
+	/**
+	 * True only when the fused `libelizainference` build exports the speaker
+	 * ABI and advertises support at runtime.
+	 */
+	static isSupported(ffi: ElizaInferenceFfi | null | undefined): boolean {
+		if (!ffi || typeof ffi.speakerSupported !== "function") return false;
+		return ffi.speakerSupported();
+	}
+
+	/**
+	 * Open a native speaker-encoder session. Throws
+	 * `SpeakerEncoderGgmlUnavailableError` when the runtime is not present.
+	 */
+	static async load(
+		opts: FusedSpeakerEncoderOptions,
+	): Promise<FusedSpeakerEncoder> {
+		if (!FusedSpeakerEncoder.isSupported(opts.ffi)) {
+			throw new SpeakerEncoderGgmlUnavailableError(
+				"native-missing",
+				"[speaker-fused] The native speaker encoder is not present in this libelizainference build. Rebuild with the WeSpeaker forward graph linked in (eliza_inference_speaker_* symbols).",
+			);
+		}
+		if (
+			!opts.ffi.speakerOpen ||
+			!opts.ffi.speakerEmbed ||
+			!opts.ffi.speakerClose
+		) {
+			throw new SpeakerEncoderGgmlUnavailableError(
+				"model-load-failed",
+				"[speaker-fused] Speaker support probe succeeded, but the required FFI methods are missing on the binding.",
+			);
+		}
+		const ctx = typeof opts.ctx === "function" ? opts.ctx() : opts.ctx;
+		const handle = opts.ffi.speakerOpen({
+			ctx,
+			ggufPath: opts.ggufPath ?? null,
+		});
+		return new FusedSpeakerEncoder(opts.ffi, handle);
+	}
+
+	async encode(pcm: Float32Array): Promise<Float32Array> {
+		if (this.disposed) {
+			throw new SpeakerEncoderGgmlUnavailableError(
+				"model-load-failed",
+				"[speaker-fused] encode called after dispose()",
+			);
+		}
+		if (!(pcm instanceof Float32Array)) {
+			throw new SpeakerEncoderGgmlUnavailableError(
+				"invalid-input",
+				"[speaker-fused] pcm must be a Float32Array",
+			);
+		}
+		if (pcm.length < SPEAKER_GGML_MIN_SAMPLES) {
+			throw new SpeakerEncoderGgmlUnavailableError(
+				"invalid-input",
+				`[speaker-fused] pcm too short: ${pcm.length} samples < ${SPEAKER_GGML_MIN_SAMPLES}`,
+			);
+		}
+		const embed = this.ffi.speakerEmbed;
+		if (!embed) {
+			throw new SpeakerEncoderGgmlUnavailableError(
+				"model-load-failed",
+				"[speaker-fused] encode missing FFI method",
+			);
+		}
+		return embed({ speaker: this.handle, pcm });
+	}
+
+	async dispose(): Promise<void> {
+		if (this.disposed) return;
+		this.disposed = true;
+		this.ffi.speakerClose?.(this.handle);
+	}
+}

package/src/services/voice/speaker/encoder-ggml.test.ts

@@ -0,0 +1,59 @@
+/**
+ * Unit coverage for voiceSpeakerDistance (#9147 self-voice-rejection / speaker ID).
+ *
+ * `voiceSpeakerDistance` is the pure cosine-distance metric (`1 - cos_sim`,
+ * range [0,2]) that the diarizer / self-voice-rejection compares against a
+ * threshold — small distance = same speaker. It mirrors the C-side
+ * `voice_speaker_distance` and shipped untested. No GGUF / FFI / audio.
+ */
+
+import { describe, expect, it } from "vitest";
+import {
+	SPEAKER_GGML_EMBEDDING_DIM as DIM,
+	voiceSpeakerDistance,
+} from "./encoder-ggml";
+
+const vec = (fill: number) => new Float32Array(DIM).fill(fill);
+const unit = (index: number) => {
+	const v = new Float32Array(DIM);
+	v[index] = 1;
+	return v;
+};
+
+describe("voiceSpeakerDistance", () => {
+	it("is 0 for identical embeddings (same speaker)", () => {
+		expect(voiceSpeakerDistance(vec(1), vec(1))).toBeCloseTo(0, 6);
+	});
+
+	it("is scale-invariant (cosine ignores magnitude)", () => {
+		expect(voiceSpeakerDistance(vec(1), vec(2))).toBeCloseTo(0, 6);
+	});
+
+	it("is 1 for orthogonal embeddings", () => {
+		expect(voiceSpeakerDistance(unit(0), unit(1))).toBeCloseTo(1, 6);
+	});
+
+	it("is 2 for opposite embeddings (max distance)", () => {
+		expect(voiceSpeakerDistance(vec(1), vec(-1))).toBeCloseTo(2, 6);
+	});
+
+	it("returns 1 when either embedding is the zero vector", () => {
+		expect(voiceSpeakerDistance(new Float32Array(DIM), vec(1))).toBe(1);
+		expect(voiceSpeakerDistance(vec(1), new Float32Array(DIM))).toBe(1);
+	});
+
+	it("is symmetric", () => {
+		const a = unit(3);
+		const b = vec(1);
+		expect(voiceSpeakerDistance(a, b)).toBe(voiceSpeakerDistance(b, a));
+	});
+
+	it("throws on a wrong-dimension embedding (either side)", () => {
+		expect(() => voiceSpeakerDistance(new Float32Array(8), vec(1))).toThrow(
+			/expected 256/,
+		);
+		expect(() => voiceSpeakerDistance(vec(1), new Float32Array(8))).toThrow(
+			/expected 256/,
+		);
+	});
+});

package/src/services/voice/speaker/encoder-ggml.ts

@@ -0,0 +1,79 @@
+/**
+ * Speaker-embedding encoder — shared constants, error class, and the
+ * embedding-distance helper.
+ *
+ * The speaker encoder runs EXCLUSIVELY through the fused `libelizainference`
+ * `eliza_inference_speaker_*` ABI (`FusedSpeakerEncoder` in `encoder-fused.ts`).
+ * The standalone `libvoice_classifier` binding that previously lived here has
+ * been removed — there is one on-device voice runtime.
+ *
+ * This module retains the pieces the fused path shares:
+ *   - the canonical dims (`SPEAKER_GGML_*`), pinned at 256 to match the C-side
+ *     `VOICE_SPEAKER_EMBEDDING_DIM` and the WeSpeaker ResNet34-LM head,
+ *   - the structured `SpeakerEncoderGgmlUnavailableError` the fused encoder
+ *     throws (no synthetic embedding fallback),
+ *   - the pure `voiceSpeakerDistance` cosine-distance helper.
+ */
+
+/** Output embedding dim. Matches `VOICE_SPEAKER_EMBEDDING_DIM`. */
+export const SPEAKER_GGML_EMBEDDING_DIM = 256;
+
+/** Required input sample rate. */
+export const SPEAKER_GGML_SAMPLE_RATE = 16_000;
+
+/** Minimum useful audio window (~1.0 s). */
+export const SPEAKER_GGML_MIN_SAMPLES = 16_000;
+
+export class SpeakerEncoderGgmlUnavailableError extends Error {
+	readonly code:
+		| "native-missing"
+		| "library-missing"
+		| "model-missing"
+		| "model-load-failed"
+		| "model-shape-mismatch"
+		| "forward-not-implemented"
+		| "invalid-input";
+	constructor(
+		code: SpeakerEncoderGgmlUnavailableError["code"],
+		message: string,
+	) {
+		super(message);
+		this.name = "SpeakerEncoderGgmlUnavailableError";
+		this.code = code;
+	}
+}
+
+/**
+ * Cosine distance between two 256-dim speaker embeddings. Defined as
+ * `1 - cos_similarity(a, b)`, range [0, 2]. Mirrors the C-side
+ * `voice_speaker_distance` helper exactly.
+ */
+export function voiceSpeakerDistance(a: Float32Array, b: Float32Array): number {
+	if (a.length !== SPEAKER_GGML_EMBEDDING_DIM) {
+		throw new SpeakerEncoderGgmlUnavailableError(
+			"invalid-input",
+			`[speaker] left embedding has dim ${a.length}, expected ${SPEAKER_GGML_EMBEDDING_DIM}`,
+		);
+	}
+	if (b.length !== SPEAKER_GGML_EMBEDDING_DIM) {
+		throw new SpeakerEncoderGgmlUnavailableError(
+			"invalid-input",
+			`[speaker] right embedding has dim ${b.length}, expected ${SPEAKER_GGML_EMBEDDING_DIM}`,
+		);
+	}
+	let dot = 0;
+	let normA = 0;
+	let normB = 0;
+	for (let i = 0; i < SPEAKER_GGML_EMBEDDING_DIM; i += 1) {
+		const av = a[i];
+		const bv = b[i];
+		dot += av * bv;
+		normA += av * av;
+		normB += bv * bv;
+	}
+	if (normA <= 0 || normB <= 0) return 1;
+	let cosine = dot / (Math.sqrt(normA) * Math.sqrt(normB));
+	if (cosine > 1) cosine = 1;
+	if (cosine < -1) cosine = -1;
+	return 1 - cosine;
+}

package/src/services/voice/speaker/encoder.ts

@@ -0,0 +1,105 @@
+/**
+ * Speaker-embedding encoder — shared contract, model-id constants, and the
+ * `averageEmbeddings` centroid helper.
+ *
+ * The speaker encoder runs EXCLUSIVELY through the fused `libelizainference`
+ * `eliza_inference_speaker_*` ABI (`FusedSpeakerEncoder` in `encoder-fused.ts`).
+ * The standalone `WespeakerEncoder` (`libvoice_classifier`) binding has been
+ * removed — there is one on-device voice runtime.
+ *
+ * This module retains the cross-cutting pieces every encoder caller shares:
+ *   - the `SpeakerEncoder` interface the fused encoder implements,
+ *   - the stored model-id strings (kept stable so existing voice profiles in
+ *     the database stay valid),
+ *   - the canonical dims (re-exported from `encoder-ggml`),
+ *   - the `SpeakerEncoderUnavailableError` the enrollment routes raise,
+ *   - the pure `averageEmbeddings` centroid helper.
+ */
+
+import { normalizeVoiceEmbedding } from "../speaker-imprint";
+import {
+	SPEAKER_GGML_EMBEDDING_DIM,
+	SPEAKER_GGML_MIN_SAMPLES,
+	SPEAKER_GGML_SAMPLE_RATE,
+} from "./encoder-ggml";
+
+// ---------------------------------------------------------------------------
+// Stored model id constants.
+// The model id strings are kept stable for compatibility with stored profiles
+// (changing them would invalidate any existing voice profiles in the database).
+// ---------------------------------------------------------------------------
+
+export const WESPEAKER_RESNET34_LM_INT8_MODEL_ID =
+	"wespeaker-resnet34-lm-int8" as const;
+export const WESPEAKER_RESNET34_LM_FP32_MODEL_ID =
+	"wespeaker-resnet34-lm-fp32" as const;
+export type WespeakerModelId =
+	| typeof WESPEAKER_RESNET34_LM_INT8_MODEL_ID
+	| typeof WESPEAKER_RESNET34_LM_FP32_MODEL_ID;
+
+export const WESPEAKER_EMBEDDING_DIM = SPEAKER_GGML_EMBEDDING_DIM;
+export const WESPEAKER_SAMPLE_RATE = SPEAKER_GGML_SAMPLE_RATE;
+export const WESPEAKER_MIN_SAMPLES = SPEAKER_GGML_MIN_SAMPLES;
+
+// ---------------------------------------------------------------------------
+// Structured error.
+// ---------------------------------------------------------------------------
+
+export class SpeakerEncoderUnavailableError extends Error {
+	readonly code:
+		| "native-missing"
+		| "library-missing"
+		| "model-missing"
+		| "model-load-failed"
+		| "model-shape-mismatch"
+		| "forward-not-implemented"
+		| "invalid-input";
+	constructor(code: SpeakerEncoderUnavailableError["code"], message: string) {
+		super(message);
+		this.name = "SpeakerEncoderUnavailableError";
+		this.code = code;
+	}
+}
+
+// ---------------------------------------------------------------------------
+// SpeakerEncoder interface.
+// ---------------------------------------------------------------------------
+
+/** The minimal contract every speaker encoder honors. */
+export interface SpeakerEncoder {
+	readonly embeddingDim: number;
+	readonly sampleRate: number;
+	readonly modelId?: string;
+	encode(pcm: Float32Array): Promise<Float32Array>;
+	dispose(): Promise<void>;
+}
+
+// ---------------------------------------------------------------------------
+// averageEmbeddings — pure centroid helper.
+// ---------------------------------------------------------------------------
+
+export function averageEmbeddings(
+	embeddings: readonly Float32Array[],
+): Float32Array {
+	if (embeddings.length === 0) {
+		throw new SpeakerEncoderUnavailableError(
+			"invalid-input",
+			"[wespeaker] averageEmbeddings called with no inputs",
+		);
+	}
+	const dim = embeddings[0].length;
+	const sum = new Float64Array(dim);
+	for (const emb of embeddings) {
+		if (emb.length !== dim) {
+			throw new SpeakerEncoderUnavailableError(
+				"invalid-input",
+				`[wespeaker] embedding dim mismatch: ${emb.length} vs ${dim}`,
+			);
+		}
+		for (let i = 0; i < dim; i += 1) sum[i] += emb[i];
+	}
+	const out = new Float32Array(dim);
+	for (let i = 0; i < dim; i += 1) out[i] = sum[i] / embeddings.length;
+	const normalized = normalizeVoiceEmbedding(out);
+	return Float32Array.from(normalized);
+}

package/src/services/voice/speaker-imprint.test.ts

@@ -0,0 +1,185 @@
+import { describe, expect, it } from "vitest";
+import {
+	attributeVoiceImprintObservations,
+	cosineSimilarity,
+	matchVoiceImprint,
+	normalizeVoiceEmbedding,
+	updateVoiceImprintCentroid,
+	type VoiceImprintProfile,
+	voiceSpeakerFromImprintMatch,
+} from "./speaker-imprint";
+
+describe("speaker-imprint", () => {
+	it("normalizes embeddings and computes cosine similarity", () => {
+		const normalized = normalizeVoiceEmbedding([3, 4]);
+		expect(normalized[0]).toBeCloseTo(0.6, 6);
+		expect(normalized[1]).toBeCloseTo(0.8, 6);
+		expect(cosineSimilarity([2, 0], [4, 0])).toBeCloseTo(1, 6);
+		expect(cosineSimilarity([1, 0], [0, 1])).toBeCloseTo(0, 6);
+	});
+
+	it("matches the nearest compatible voice imprint above threshold", () => {
+		const profiles: VoiceImprintProfile[] = [
+			{
+				id: "cluster-a",
+				label: "Owner",
+				centroidEmbedding: [1, 0],
+				embeddingModel: "eliza-voice-embed-v1",
+				confidence: 0.9,
+				entityId: "entity-owner",
+			},
+			{
+				id: "cluster-b",
+				label: "Guest",
+				centroidEmbedding: [0, 1],
+				embeddingModel: "eliza-voice-embed-v1",
+				confidence: 0.9,
+			},
+		];
+
+		const match = matchVoiceImprint({
+			embedding: [0.98, 0.05],
+			embeddingModel: "eliza-voice-embed-v1",
+			profiles,
+			threshold: 0.8,
+		});
+
+		expect(match?.profile.id).toBe("cluster-a");
+		expect(match?.similarity).toBeGreaterThan(0.99);
+		if (!match) throw new Error("expected voice imprint match");
+		const speaker = voiceSpeakerFromImprintMatch({
+			match,
+			observationId: "obs-1",
+			source: { kind: "local_mic", deviceId: "mic-1" },
+		});
+		expect(speaker.entityId).toBe("entity-owner");
+		expect(speaker.imprintClusterId).toBe("cluster-a");
+		expect(speaker.imprintObservationId).toBe("obs-1");
+		expect(speaker.source?.kind).toBe("local_mic");
+	});
+
+	it("does not match across embedding-model or dimension mismatches", () => {
+		const profiles: VoiceImprintProfile[] = [
+			{
+				id: "cluster-a",
+				centroidEmbedding: [1, 0],
+				embeddingModel: "other-model",
+			},
+			{
+				id: "cluster-b",
+				centroidEmbedding: [1, 0, 0],
+				embeddingModel: "eliza-voice-embed-v1",
+			},
+		];
+
+		expect(
+			matchVoiceImprint({
+				embedding: [1, 0],
+				embeddingModel: "eliza-voice-embed-v1",
+				profiles,
+			}),
+		).toBeNull();
+	});
+
+	it("updates a centroid with weighted observations", () => {
+		const first = updateVoiceImprintCentroid({
+			observationEmbedding: [10, 0],
+			observationConfidence: 0.8,
+		});
+		expect(first.centroidEmbedding).toEqual([1, 0]);
+		expect(first.sampleCount).toBe(1);
+		expect(first.confidence).toBeCloseTo(0.8, 6);
+
+		const second = updateVoiceImprintCentroid({
+			centroidEmbedding: first.centroidEmbedding,
+			sampleCount: first.sampleCount,
+			confidence: first.confidence,
+			observationEmbedding: [0, 10],
+			observationConfidence: 0.5,
+		});
+		expect(second.sampleCount).toBe(2);
+		expect(second.centroidEmbedding[0]).toBeGreaterThan(
+			second.centroidEmbedding[1],
+		);
+		expect(second.confidence).toBeCloseTo(0.65, 6);
+	});
+
+	it("attributes diarized observation embeddings without granting synthesis or identity authority", () => {
+		const profiles: VoiceImprintProfile[] = [
+			{
+				id: "cluster-owner",
+				label: "Owner",
+				displayName: "Owner",
+				entityId: "entity-owner",
+				centroidEmbedding: [1, 0],
+				embeddingModel: "eliza-voice-embed-v1",
+				confidence: 0.9,
+			},
+			{
+				id: "cluster-guest",
+				label: "Guest",
+				entityId: "entity-guest",
+				centroidEmbedding: [0, 1],
+				embeddingModel: "eliza-voice-embed-v1",
+				confidence: 0.8,
+			},
+		];
+		const source = {
+			kind: "local_mic" as const,
+			deviceId: "default-input",
+			roomId: "room-1",
+		};
+
+		const result = attributeVoiceImprintObservations({
+			defaultSource: source,
+			profiles,
+			threshold: 0.8,
+			observations: [
+				{
+					id: "obs-owner-1",
+					segmentId: "seg-owner-1",
+					text: "I am the owner",
+					startMs: 0,
+					endMs: 1200,
+					embedding: [0.99, 0.02],
+					embeddingModel: "eliza-voice-embed-v1",
+				},
+				{
+					id: "obs-guest-1",
+					segmentId: "seg-guest-1",
+					text: "and I am a guest",
+					startMs: 1300,
+					endMs: 2100,
+					embedding: [0.02, 0.99],
+					embeddingModel: "eliza-voice-embed-v1",
+				},
+			],
+		});
+
+		expect(result.summary).toMatchObject({
+			totalObservations: 2,
+			matchedObservations: 2,
+			unmatchedObservations: 0,
+		});
+		expect(result.primarySpeaker?.entityId).toBe("entity-owner");
+		expect(result.segments.map((segment) => segment.speaker?.entityId)).toEqual(
+			["entity-owner", "entity-guest"],
+		);
+		expect(result.segments[0].metadata).toMatchObject({
+			attributionOnly: true,
+			evidenceKind: "voice_imprint_attribution",
+			identityAuthority: false,
+			synthesisAuthorization: false,
+			diarizationMode: "attribution_only",
+			imprintClusterId: "cluster-owner",
+			imprintObservationId: "obs-owner-1",
+			entityId: "entity-owner",
+		});
+		expect(result.segments[0].speaker?.metadata).toMatchObject({
+			attributionOnly: true,
+			evidenceKind: "voice_imprint_attribution",
+			identityAuthority: false,
+			synthesisAuthorization: false,
+		});
+	});
+});