@elizaos/plugin-local-inference 2.0.0-beta.1 → 2.0.11-beta.7

package/src/services/voice/speaker-imprint.ts

@@ -0,0 +1,312 @@
+import type { VoiceInputSource, VoiceSegment, VoiceSpeaker } from "./types";
+
+export const DEFAULT_VOICE_IMPRINT_MATCH_THRESHOLD = 0.78;
+
+export interface VoiceImprintProfile {
+	id: string;
+	centroidEmbedding: ArrayLike<number> | null | undefined;
+	embeddingModel?: string | null;
+	sampleCount?: number | null;
+	confidence?: number | null;
+	label?: string | null;
+	displayName?: string | null;
+	entityId?: string | null;
+	sourceKind?: string | null;
+	sourceScopeId?: string | null;
+	metadata?: Record<string, unknown> | null;
+}
+
+export interface VoiceImprintMatch {
+	profile: VoiceImprintProfile;
+	similarity: number;
+	confidence: number;
+}
+
+export interface VoiceImprintCentroidUpdate {
+	centroidEmbedding: number[];
+	sampleCount: number;
+	confidence: number;
+}
+
+export interface VoiceImprintObservationInput {
+	id: string;
+	segmentId?: string;
+	text: string;
+	startMs: number;
+	endMs: number;
+	embedding: ArrayLike<number>;
+	embeddingModel?: string | null;
+	confidence?: number | null;
+	source?: VoiceInputSource;
+	metadata?: Record<string, unknown> | null;
+}
+
+export interface AttributedVoiceObservation {
+	observation: VoiceImprintObservationInput;
+	match: VoiceImprintMatch | null;
+	speaker: VoiceSpeaker | null;
+	segment: VoiceSegment;
+}
+
+export interface SpeakerAttributionResult {
+	observations: AttributedVoiceObservation[];
+	segments: VoiceSegment[];
+	primarySpeaker?: VoiceSpeaker;
+	summary: {
+		totalObservations: number;
+		matchedObservations: number;
+		unmatchedObservations: number;
+		meanConfidence: number;
+		meanSimilarity: number | null;
+	};
+}
+
+function clamp01(value: number): number {
+	if (!Number.isFinite(value)) return 0;
+	return Math.max(0, Math.min(1, value));
+}
+
+export function normalizeVoiceEmbedding(
+	embedding: ArrayLike<number>,
+): number[] {
+	let sumSq = 0;
+	const out = new Array<number>(embedding.length);
+	for (let i = 0; i < embedding.length; i += 1) {
+		const value = Number(embedding[i]);
+		if (!Number.isFinite(value)) {
+			throw new Error(
+				`[voice-imprint] embedding contains non-finite value at ${i}`,
+			);
+		}
+		out[i] = value;
+		sumSq += value * value;
+	}
+	if (sumSq === 0) return out;
+	const invNorm = 1 / Math.sqrt(sumSq);
+	for (let i = 0; i < out.length; i += 1) out[i] *= invNorm;
+	return out;
+}
+
+export function cosineSimilarity(
+	left: ArrayLike<number>,
+	right: ArrayLike<number>,
+): number {
+	if (left.length !== right.length) {
+		throw new Error(
+			`[voice-imprint] embedding dimension mismatch: ${left.length} != ${right.length}`,
+		);
+	}
+	const a = normalizeVoiceEmbedding(left);
+	const b = normalizeVoiceEmbedding(right);
+	let dot = 0;
+	for (let i = 0; i < a.length; i += 1) dot += a[i] * b[i];
+	return Math.max(-1, Math.min(1, dot));
+}
+
+export function updateVoiceImprintCentroid(args: {
+	centroidEmbedding?: ArrayLike<number> | null;
+	sampleCount?: number | null;
+	confidence?: number | null;
+	observationEmbedding: ArrayLike<number>;
+	observationConfidence?: number | null;
+	maxObservationWeight?: number;
+}): VoiceImprintCentroidUpdate {
+	const observation = normalizeVoiceEmbedding(args.observationEmbedding);
+	const previousCount = Math.max(0, Math.floor(args.sampleCount ?? 0));
+	const observationWeight = clamp01(
+		Math.min(args.maxObservationWeight ?? 1, args.observationConfidence ?? 1),
+	);
+	if (!args.centroidEmbedding || previousCount === 0) {
+		return {
+			centroidEmbedding: observation,
+			sampleCount: 1,
+			confidence: observationWeight,
+		};
+	}
+	if (args.centroidEmbedding.length !== observation.length) {
+		throw new Error(
+			`[voice-imprint] cannot update ${args.centroidEmbedding.length}-dim centroid with ${observation.length}-dim observation`,
+		);
+	}
+	const centroid = normalizeVoiceEmbedding(args.centroidEmbedding);
+	const historicalWeight = Math.max(1, previousCount);
+	const updated = centroid.map(
+		(value, i) => value * historicalWeight + observation[i] * observationWeight,
+	);
+	return {
+		centroidEmbedding: normalizeVoiceEmbedding(updated),
+		sampleCount: previousCount + 1,
+		confidence: clamp01(
+			((args.confidence ?? 0) * previousCount + observationWeight) /
+				(previousCount + 1),
+		),
+	};
+}
+
+export function matchVoiceImprint(args: {
+	embedding: ArrayLike<number>;
+	profiles: readonly VoiceImprintProfile[];
+	embeddingModel?: string | null;
+	threshold?: number;
+}): VoiceImprintMatch | null {
+	const threshold = args.threshold ?? DEFAULT_VOICE_IMPRINT_MATCH_THRESHOLD;
+	let best: VoiceImprintMatch | null = null;
+	for (const profile of args.profiles) {
+		if (!profile.centroidEmbedding || profile.centroidEmbedding.length === 0) {
+			continue;
+		}
+		if (
+			args.embeddingModel &&
+			profile.embeddingModel &&
+			args.embeddingModel !== profile.embeddingModel
+		) {
+			continue;
+		}
+		if (profile.centroidEmbedding.length !== args.embedding.length) {
+			continue;
+		}
+		const similarity = cosineSimilarity(
+			args.embedding,
+			profile.centroidEmbedding,
+		);
+		if (similarity < threshold) continue;
+		const confidence = clamp01(
+			((similarity - threshold) / Math.max(0.0001, 1 - threshold)) *
+				clamp01(profile.confidence ?? 1),
+		);
+		if (!best || similarity > best.similarity) {
+			best = { profile, similarity, confidence };
+		}
+	}
+	return best;
+}
+
+export function voiceSpeakerFromImprintMatch(args: {
+	match: VoiceImprintMatch;
+	source?: VoiceInputSource;
+	observationId?: string;
+}): VoiceSpeaker {
+	const { profile } = args.match;
+	return {
+		id: profile.entityId ?? profile.id,
+		label: profile.label ?? undefined,
+		displayName: profile.displayName ?? profile.label ?? undefined,
+		source: args.source,
+		imprintClusterId: profile.id,
+		imprintObservationId: args.observationId,
+		entityId: profile.entityId ?? undefined,
+		confidence: args.match.confidence,
+		metadata: {
+			...(profile.metadata ?? {}),
+			attributionOnly: true,
+			evidenceKind: "voice_imprint_attribution",
+			identityAuthority: false,
+			synthesisAuthorization: false,
+			matchSimilarity: args.match.similarity,
+			embeddingModel: profile.embeddingModel ?? undefined,
+		},
+	};
+}
+
+export function attributeVoiceImprintObservations(args: {
+	observations: readonly VoiceImprintObservationInput[];
+	profiles: readonly VoiceImprintProfile[];
+	threshold?: number;
+	defaultSource?: VoiceInputSource;
+}): SpeakerAttributionResult {
+	const attributed: AttributedVoiceObservation[] = [];
+	let confidenceSum = 0;
+	let confidenceCount = 0;
+	let similaritySum = 0;
+	let similarityCount = 0;
+
+	for (const observation of args.observations) {
+		const source = observation.source ?? args.defaultSource;
+		const match = matchVoiceImprint({
+			embedding: observation.embedding,
+			embeddingModel: observation.embeddingModel,
+			profiles: args.profiles,
+			threshold: args.threshold,
+		});
+		const speaker = match
+			? voiceSpeakerFromImprintMatch({
+					match,
+					source,
+					observationId: observation.id,
+				})
+			: null;
+		if (speaker?.confidence !== undefined) {
+			confidenceSum += speaker.confidence;
+			confidenceCount += 1;
+		}
+		if (match) {
+			similaritySum += match.similarity;
+			similarityCount += 1;
+		}
+		attributed.push({
+			observation,
+			match,
+			speaker,
+			segment: {
+				id: observation.segmentId ?? observation.id,
+				text: observation.text,
+				startMs: observation.startMs,
+				endMs: observation.endMs,
+				...(speaker ? { speaker, speakerId: speaker.id } : {}),
+				...(source ? { source } : {}),
+				confidence: speaker?.confidence ?? observation.confidence ?? undefined,
+				metadata: {
+					...(observation.metadata ?? {}),
+					attributionOnly: true,
+					evidenceKind: "voice_imprint_attribution",
+					identityAuthority: false,
+					synthesisAuthorization: false,
+					diarizationMode: "attribution_only",
+					embeddingModel: observation.embeddingModel ?? undefined,
+					imprintObservationId: observation.id,
+					imprintClusterId: speaker?.imprintClusterId,
+					entityId: speaker?.entityId,
+					matchSimilarity: match?.similarity,
+				},
+			},
+		});
+	}
+
+	const segments = attributed.map((row) => row.segment);
+	const primarySpeaker = selectPrimarySpeaker(segments);
+	return {
+		observations: attributed,
+		segments,
+		...(primarySpeaker ? { primarySpeaker } : {}),
+		summary: {
+			totalObservations: attributed.length,
+			matchedObservations: attributed.filter((row) => row.match).length,
+			unmatchedObservations: attributed.filter((row) => !row.match).length,
+			meanConfidence:
+				confidenceCount === 0 ? 0 : clamp01(confidenceSum / confidenceCount),
+			meanSimilarity:
+				similarityCount === 0 ? null : similaritySum / similarityCount,
+		},
+	};
+}
+
+function selectPrimarySpeaker(
+	segments: readonly VoiceSegment[],
+): VoiceSpeaker | undefined {
+	const durations = new Map<string, { speaker: VoiceSpeaker; ms: number }>();
+	for (const segment of segments) {
+		if (!segment.speaker) continue;
+		const key = segment.speaker.id;
+		const prev = durations.get(key);
+		const ms = Math.max(0, segment.endMs - segment.startMs);
+		durations.set(key, {
+			speaker: segment.speaker,
+			ms: (prev?.ms ?? 0) + ms,
+		});
+	}
+	let best: { speaker: VoiceSpeaker; ms: number } | undefined;
+	for (const row of durations.values()) {
+		if (!best || row.ms > best.ms) best = row;
+	}
+	return best?.speaker;
+}

package/src/services/voice/speaker-preset-cache.d.ts

@@ -0,0 +1,77 @@
+import type { SpeakerPreset } from "./types";
+import { type VoicePresetSeedPhrase } from "./voice-preset-format";
+export interface PresetBundlePaths {
+    bundleRoot: string;
+    /** Relative path to the *default* voice preset inside the bundle. */
+    cacheRelPath?: string;
+}
+export interface LoadedPresetBundle {
+    preset: SpeakerPreset;
+    /** Phrase-cache seed entries parsed alongside the embedding. The engine
+     *  bridge feeds these into a `PhraseCache.seed(...)` call before the
+     *  scheduler is constructed. */
+    phrases: ReadonlyArray<VoicePresetSeedPhrase>;
+}
+export interface SpeakerPresetCacheOptions {
+    /**
+     * Maximum number of distinct voices held in RAM at once. Speaker presets
+     * are KB-scale (an embedding vector + the seed-phrase PCM), so the default
+     * is generous; the bound exists so a connector that switches voices per
+     * speaker (e.g. a Discord room with many users) does not grow unbounded.
+     * On insert past the bound the least-recently-used voice is evicted.
+     *
+     * Defaults to 8 hot voices (R6 §1: "LRU 8 hot, mmap-backed").
+     */
+    maxVoices?: number;
+}
+export declare const DEFAULT_VOICE_ID = "default";
+export declare const DEFAULT_VOICE_PRESET_REL_PATH: string;
+/**
+ * Resolve the on-disk path of a voice preset inside a bundle. The default
+ * voice lives at `cache/voice-preset-default.bin`; additional voices ship as
+ * `cache/voice-preset-<voiceId>.bin`. Throws on a `voiceId` that is not a safe
+ * single path segment (no `/`, no `..`).
+ */
+export declare function voicePresetPath(bundleRoot: string, voiceId: string): string;
+/**
+ * LRU cache of parsed speaker presets keyed by `voiceId`. Holds the speaker
+ * embedding, the raw preset bytes (for FFI handoff), and the phrase-cache seed
+ * list parsed from the preset file. Multi-voice: `load(bundleRoot, voiceId)`
+ * reads `cache/voice-preset-<voiceId>.bin` from the bundle on a miss.
+ *
+ * v2 preset fields (`refAudioTokens`, `refText`, `instruct`) are surfaced
+ * on the `SpeakerPreset` shape so the FFI bridge can pass them through to
+ * `ov_tts_params` without going through the legacy "instruct == voiceId"
+ * misreading.
+ */
+export declare class SpeakerPresetCache {
+    private readonly entries;
+    private readonly maxVoices;
+    constructor(opts?: SpeakerPresetCacheOptions);
+    /**
+     * Load the bundle's default voice preset (`cache/voice-preset-default.bin`,
+     * or `paths.cacheRelPath` if overridden) and return both the speaker
+     * embedding and the phrase-cache seed entries. Cached for subsequent
+     * `get("default")` lookups (and marked most-recently-used).
+     */
+    loadFromBundle(paths: PresetBundlePaths, voiceId?: string): LoadedPresetBundle;
+    /**
+     * Load an arbitrary voice by id from a bundle root, reading
+     * `cache/voice-preset-<voiceId>.bin` (or `cache/voice-preset-default.bin`
+     * for `"default"`). Returns the cached entry on a hit (marked MRU).
+     */
+    load(bundleRoot: string, voiceId: string): LoadedPresetBundle;
+    /** True if `voiceId` is currently resident in the cache. */
+    has(voiceId: string): boolean;
+    put(preset: SpeakerPreset): void;
+    get(voiceId: string): SpeakerPreset | undefined;
+    /** Seed entries previously loaded for a voice, if any (does not touch LRU order). */
+    getSeed(voiceId: string): ReadonlyArray<VoicePresetSeedPhrase>;
+    /** Number of voices currently resident. */
+    size(): number;
+    /** Drop every cached preset. */
+    clear(): void;
+    private loadFile;
+    private evictOverflow;
+}
+//# sourceMappingURL=speaker-preset-cache.d.ts.map

package/src/services/voice/speaker-preset-cache.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"speaker-preset-cache.d.ts","sourceRoot":"","sources":["speaker-preset-cache.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,SAAS,CAAC;AAC7C,OAAO,EAGN,KAAK,qBAAqB,EAC1B,MAAM,uBAAuB,CAAC;AAE/B,MAAM,WAAW,iBAAiB;IACjC,UAAU,EAAE,MAAM,CAAC;IACnB,qEAAqE;IACrE,YAAY,CAAC,EAAE,MAAM,CAAC;CACtB;AAED,MAAM,WAAW,kBAAkB;IAClC,MAAM,EAAE,aAAa,CAAC;IACtB;;oCAEgC;IAChC,OAAO,EAAE,aAAa,CAAC,qBAAqB,CAAC,CAAC;CAC9C;AAED,MAAM,WAAW,yBAAyB;IACzC;;;;;;;;OAQG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC;CACnB;AAED,eAAO,MAAM,gBAAgB,YAAY,CAAC;AAG1C,eAAO,MAAM,6BAA6B,QAGzC,CAAC;AAEF;;;;;GAKG;AACH,wBAAgB,eAAe,CAAC,UAAU,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,GAAG,MAAM,CAU3E;AAOD;;;;;;;;;;GAUG;AACH,qBAAa,kBAAkB;IAG9B,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAiC;IACzD,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAS;gBAEvB,IAAI,GAAE,yBAA8B;IAOhD;;;;;OAKG;IACH,cAAc,CACb,KAAK,EAAE,iBAAiB,EACxB,OAAO,SAAmB,GACxB,kBAAkB;IASrB;;;;OAIG;IACH,IAAI,CAAC,UAAU,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,GAAG,kBAAkB;IAI7D,4DAA4D;IAC5D,GAAG,CAAC,OAAO,EAAE,MAAM,GAAG,OAAO;IAI7B,GAAG,CAAC,MAAM,EAAE,aAAa,GAAG,IAAI;IAUhC,GAAG,CAAC,OAAO,EAAE,MAAM,GAAG,aAAa,GAAG,SAAS;IAQ/C,qFAAqF;IACrF,OAAO,CAAC,OAAO,EAAE,MAAM,GAAG,aAAa,CAAC,qBAAqB,CAAC;IAI9D,2CAA2C;IAC3C,IAAI,IAAI,MAAM;IAId,gCAAgC;IAChC,KAAK,IAAI,IAAI;IAIb,OAAO,CAAC,QAAQ;IA+BhB,OAAO,CAAC,aAAa;CAOrB"}

package/src/services/voice/speaker-preset-cache.test.ts

@@ -0,0 +1,154 @@
+import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs";
+import { tmpdir } from "node:os";
+import path from "node:path";
+import { afterEach, beforeEach, describe, expect, it } from "vitest";
+import {
+	DEFAULT_VOICE_ID,
+	DEFAULT_VOICE_PRESET_REL_PATH,
+	SpeakerPresetCache,
+	voicePresetPath,
+} from "./speaker-preset-cache";
+import { writeVoicePresetFile } from "./voice-preset-format";
+
+function presetBlob(seedScalar: number): Uint8Array {
+	return writeVoicePresetFile({
+		embedding: new Float32Array([seedScalar, seedScalar + 1, seedScalar + 2]),
+		phrases: [
+			{
+				text: `phrase-${seedScalar}`,
+				sampleRate: 24000,
+				pcm: new Float32Array([seedScalar / 10, -seedScalar / 10]),
+			},
+		],
+	});
+}
+
+describe("voicePresetPath", () => {
+	it("maps the default voice to the canonical relative path", () => {
+		expect(voicePresetPath("/bundle", DEFAULT_VOICE_ID)).toBe(
+			path.join("/bundle", DEFAULT_VOICE_PRESET_REL_PATH),
+		);
+	});
+
+	it("maps named voices to cache/voice-preset-<id>.bin", () => {
+		expect(voicePresetPath("/bundle", "narrator")).toBe(
+			path.join("/bundle", "cache", "voice-preset-narrator.bin"),
+		);
+	});
+
+	it("rejects unsafe voice ids", () => {
+		expect(() => voicePresetPath("/bundle", "../escape")).toThrow();
+		expect(() => voicePresetPath("/bundle", "a/b")).toThrow();
+		expect(() => voicePresetPath("/bundle", "")).toThrow();
+	});
+});
+
+describe("SpeakerPresetCache", () => {
+	let bundleRoot: string;
+
+	beforeEach(() => {
+		bundleRoot = mkdtempSync(path.join(tmpdir(), "eliza-preset-cache-"));
+		mkdirSync(path.join(bundleRoot, "cache"), { recursive: true });
+	});
+
+	afterEach(() => {
+		rmSync(bundleRoot, { recursive: true, force: true });
+	});
+
+	function writeVoiceFile(voiceId: string, scalar: number): void {
+		writeFileSync(voicePresetPath(bundleRoot, voiceId), presetBlob(scalar));
+	}
+
+	it("loads the default preset from a bundle and exposes embedding + seed phrases", () => {
+		writeVoiceFile(DEFAULT_VOICE_ID, 1);
+		const cache = new SpeakerPresetCache();
+		const { preset, phrases } = cache.loadFromBundle({ bundleRoot });
+		expect(preset.voiceId).toBe(DEFAULT_VOICE_ID);
+		expect(Array.from(preset.embedding)).toEqual([1, 2, 3]);
+		expect(phrases).toHaveLength(1);
+		expect(phrases[0].text).toBe("phrase-1");
+		expect(cache.has(DEFAULT_VOICE_ID)).toBe(true);
+		expect(cache.get(DEFAULT_VOICE_ID)?.embedding[0]).toBe(1);
+		expect(cache.getSeed(DEFAULT_VOICE_ID)).toHaveLength(1);
+	});
+
+	it("returns the cached entry on a second load (no re-parse)", () => {
+		writeVoiceFile(DEFAULT_VOICE_ID, 5);
+		const cache = new SpeakerPresetCache();
+		const first = cache.loadFromBundle({ bundleRoot }).preset;
+		const second = cache.loadFromBundle({ bundleRoot }).preset;
+		expect(second).toBe(first);
+	});
+
+	it("loads arbitrary named voices via load(bundleRoot, voiceId)", () => {
+		writeVoiceFile("narrator", 7);
+		writeVoiceFile("whisper", 9);
+		const cache = new SpeakerPresetCache();
+		const narrator = cache.load(bundleRoot, "narrator");
+		const whisper = cache.load(bundleRoot, "whisper");
+		expect(narrator.preset.voiceId).toBe("narrator");
+		expect(Array.from(narrator.preset.embedding)).toEqual([7, 8, 9]);
+		expect(whisper.preset.voiceId).toBe("whisper");
+		expect(Array.from(whisper.preset.embedding)).toEqual([9, 10, 11]);
+		expect(cache.size()).toBe(2);
+	});
+
+	it("throws a clear error when the preset file is missing", () => {
+		const cache = new SpeakerPresetCache();
+		expect(() => cache.load(bundleRoot, "missing")).toThrow(/not found/);
+	});
+
+	it("evicts the least-recently-used voice past maxVoices", () => {
+		writeVoiceFile("a", 1);
+		writeVoiceFile("b", 2);
+		writeVoiceFile("c", 3);
+		const cache = new SpeakerPresetCache({ maxVoices: 2 });
+		cache.load(bundleRoot, "a");
+		cache.load(bundleRoot, "b");
+		expect(cache.size()).toBe(2);
+		// Touch "a" so "b" becomes LRU.
+		expect(cache.get("a")).toBeDefined();
+		cache.load(bundleRoot, "c");
+		expect(cache.size()).toBe(2);
+		expect(cache.has("b")).toBe(false);
+		expect(cache.has("a")).toBe(true);
+		expect(cache.has("c")).toBe(true);
+	});
+
+	it("re-loading an evicted voice re-parses from disk", () => {
+		writeVoiceFile("a", 1);
+		writeVoiceFile("b", 2);
+		const cache = new SpeakerPresetCache({ maxVoices: 1 });
+		const a1 = cache.load(bundleRoot, "a").preset;
+		cache.load(bundleRoot, "b");
+		expect(cache.has("a")).toBe(false);
+		const a2 = cache.load(bundleRoot, "a").preset;
+		expect(a2).not.toBe(a1);
+		expect(Array.from(a2.embedding)).toEqual([1, 2, 3]);
+	});
+
+	it("put() inserts a preset and preserves any previously-loaded seed", () => {
+		writeVoiceFile(DEFAULT_VOICE_ID, 4);
+		const cache = new SpeakerPresetCache();
+		cache.loadFromBundle({ bundleRoot });
+		expect(cache.getSeed(DEFAULT_VOICE_ID)).toHaveLength(1);
+		cache.put({
+			voiceId: DEFAULT_VOICE_ID,
+			embedding: new Float32Array([0]),
+			bytes: new Uint8Array(0),
+		});
+		expect(cache.get(DEFAULT_VOICE_ID)?.embedding[0]).toBe(0);
+		// Seed survives the embedding replacement.
+		expect(cache.getSeed(DEFAULT_VOICE_ID)).toHaveLength(1);
+	});
+
+	it("clear() drops all entries", () => {
+		writeVoiceFile("a", 1);
+		const cache = new SpeakerPresetCache();
+		cache.load(bundleRoot, "a");
+		expect(cache.size()).toBe(1);
+		cache.clear();
+		expect(cache.size()).toBe(0);
+		expect(cache.has("a")).toBe(false);
+	});
+});