@elizaos/plugin-local-inference 2.0.0-beta.1 → 2.0.3-beta.2

package/src/services/voice/voice-emotion-classifier.ts

@@ -0,0 +1,273 @@
+/**
+ * Voice-emotion types and pure projection functions — consumed by the
+ * attribution pipeline (`emotion-attribution.ts`).
+ *
+ * The ONNX-backed `VoiceEmotionClassifier` class was removed when
+ * `onnxruntime-node` was dropped. No resident voice-emotion classifier
+ * remains; only the pure types and projection helpers below are kept.
+ *
+ * Pure exports here:
+ *   - Model id constants (`WAV2SMALL_INT8_MODEL_ID`, etc.)
+ *   - Sample rate / window constants
+ *   - `VoiceEmotionVad`, `VoiceEmotionClassifierOutput` interfaces
+ *   - `VoiceEmotionHead` type
+ *   - `VoiceEmotionClassifierError` error class
+ *   - `projectVadToExpressiveEmotion` — V-A-D → ExpressiveEmotion projection
+ *   - `interpretCls7Output` — 7-class logit → structured output
+ */
+
+import {
+	EXPRESSIVE_EMOTION_TAGS,
+	type ExpressiveEmotion,
+} from "./expressive-tags";
+
+/** Stable identifier for the Wav2Small student head we ship. */
+export const WAV2SMALL_INT8_MODEL_ID = "wav2small-msp-dim-int8" as const;
+/** Stable identifier for the floating-point parent we use in eval. */
+export const WAV2SMALL_FP32_MODEL_ID = "wav2small-msp-dim-fp32" as const;
+export type VoiceEmotionModelId =
+	| typeof WAV2SMALL_INT8_MODEL_ID
+	| typeof WAV2SMALL_FP32_MODEL_ID;
+
+/** Required sample rate for the Wav2Small log-mel front-end. */
+export const WAV2SMALL_SAMPLE_RATE = 16_000;
+/** Hard minimum window: anything shorter is rejected. */
+export const WAV2SMALL_MIN_SAMPLES = WAV2SMALL_SAMPLE_RATE; // 1.0 s
+/** Soft maximum window: longer inputs are truncated to the trailing window. */
+export const WAV2SMALL_MAX_SAMPLES = WAV2SMALL_SAMPLE_RATE * 12; // 12 s
+
+/** Raised when the bundled model file can not be loaded or run. */
+export class VoiceEmotionClassifierError extends Error {
+	constructor(message: string) {
+		super(message);
+		this.name = "VoiceEmotionClassifierError";
+	}
+}
+
+/** Continuous V-A-D output. All three are in [0, 1]. */
+export interface VoiceEmotionVad {
+	valence: number;
+	arousal: number;
+	dominance: number;
+}
+
+/** One classifier inference output. */
+export interface VoiceEmotionClassifierOutput {
+	vad: VoiceEmotionVad;
+	/** Projected discrete label, or null when no projection is confident. */
+	emotion: ExpressiveEmotion | null;
+	/** Confidence in the projected discrete label, [0, 1]. */
+	confidence: number;
+	/** Per-class soft scores aligned with `EXPRESSIVE_EMOTION_TAGS`. */
+	scores: Record<ExpressiveEmotion, number>;
+	/** Model id used for this inference (for the attribution evidence row). */
+	modelId: VoiceEmotionModelId;
+	/** Inference wall-time in ms (CPU side; useful for the bench harness). */
+	latencyMs: number;
+}
+
+/**
+ * Clamp `value` into the unit interval. Non-finite inputs become 0 — the
+ * downstream attribution will see 0-confidence and reject the read.
+ */
+function clamp01(value: number): number {
+	if (!Number.isFinite(value)) return 0;
+	if (value < 0) return 0;
+	if (value > 1) return 1;
+	return value;
+}
+
+/**
+ * Project a continuous V-A-D triple into the 7-class
+ * `ExpressiveEmotion` tag set. Returns soft scores per tag and the best
+ * discrete pick with a confidence score.
+ *
+ * The projection is Plutchik-aligned and deterministic. The thresholds
+ * are tuned against the MSP-Podcast V-A-D mean/std reported in the
+ * audeering model card and Wav2Small paper; small enough to be stable but
+ * wide enough to give every class some mass on conversational speech.
+ *
+ * Sign convention (audeering teacher, mirrored by Wav2Small):
+ *   valence    — high = positive affect (happy, calm), low = negative (sad, angry).
+ *   arousal    — high = energetic (excited, angry), low = subdued (calm, sad).
+ *   dominance  — high = assertive (angry), low = submissive (nervous, whisper).
+ */
+export function projectVadToExpressiveEmotion(vad: VoiceEmotionVad): {
+	emotion: ExpressiveEmotion | null;
+	confidence: number;
+	scores: Record<ExpressiveEmotion, number>;
+} {
+	// Non-finite inputs cannot be reasoned about — abstain explicitly rather
+	// than coerce to a default corner. The classifier is the source of truth
+	// for V-A-D; a non-finite read means the upstream forward pass diverged
+	// and the downstream attribution should not pretend the read happened.
+	if (
+		!Number.isFinite(vad.valence) ||
+		!Number.isFinite(vad.arousal) ||
+		!Number.isFinite(vad.dominance)
+	) {
+		const emptyScores: Record<ExpressiveEmotion, number> = {
+			happy: 0,
+			sad: 0,
+			angry: 0,
+			nervous: 0,
+			calm: 0,
+			excited: 0,
+			whisper: 0,
+		};
+		return { emotion: null, confidence: 0, scores: emptyScores };
+	}
+
+	const v = clamp01(vad.valence);
+	const a = clamp01(vad.arousal);
+	const d = clamp01(vad.dominance);
+
+	// Center each axis at 0.5; magnitudes in [-0.5, 0.5].
+	const vC = v - 0.5;
+	const aC = a - 0.5;
+	const dC = d - 0.5;
+
+	const scores: Record<ExpressiveEmotion, number> = {
+		happy: 0,
+		sad: 0,
+		angry: 0,
+		nervous: 0,
+		calm: 0,
+		excited: 0,
+		whisper: 0,
+	};
+
+	// Each class scores only from off-center signal — a fully-neutral
+	// (0.5, 0.5, 0.5) read produces all-zero scores and we abstain. Magnitudes
+	// are tuned so that a confident corner of V-A-D space lands ≥ 0.5
+	// (the bench gate threshold for "discrete label confident enough to
+	// surface").
+	// happy   — high V, mid-high A, low |D| spread.
+	scores.happy = clamp01(vC * 1.4 + Math.max(0, aC) * 0.6 - Math.abs(dC) * 0.4);
+	// excited — high V, very high A.
+	scores.excited = clamp01(vC * 0.9 + aC * 1.6);
+	// calm    — high V, low A.
+	scores.calm = clamp01(Math.max(0, vC) * 1.4 - aC * 1.2 - Math.abs(dC) * 0.3);
+	// sad     — low V, low A, low D.
+	scores.sad = clamp01(-vC * 1.4 - aC * 0.8 - dC * 0.4);
+	// angry   — low V, high A, high D.
+	scores.angry = clamp01(-vC * 1.1 + aC * 1.2 + dC * 1.0);
+	// nervous — low-mid V, mid-high A, low D.
+	scores.nervous = clamp01(-vC * 0.7 + aC * 0.9 - dC * 1.2);
+	// whisper — very low A and very low D (both at the floor). Valence-agnostic
+	// (we have no energy axis here). The double-negative gating means a low
+	// arousal alone does NOT trigger whisper — only the very low-A + low-D
+	// corner does.
+	scores.whisper = clamp01(-aC * 1.4 - dC * 1.4);
+
+	let best: ExpressiveEmotion | null = null;
+	let bestScore = 0;
+	for (const tag of EXPRESSIVE_EMOTION_TAGS) {
+		if (scores[tag] > bestScore) {
+			bestScore = scores[tag];
+			best = tag;
+		}
+	}
+	// Require a minimum mass before we attribute a discrete label.
+	if (bestScore < 0.35) {
+		return { emotion: null, confidence: bestScore, scores };
+	}
+	return { emotion: best, confidence: bestScore, scores };
+}
+
+/**
+ * Stable model-head identifier — declares whether the model emits
+ * V-A-D triples or 7-class logits. `vad` = continuous [valence, arousal,
+ * dominance]; `cls7` = 7-class logits in `EXPRESSIVE_EMOTION_TAGS` order.
+ */
+export type VoiceEmotionHead = "vad" | "cls7";
+
+/**
+ * Convert the 7-class logits from the `cls7` head into a structured
+ * emotion read. Applies a numerically-stable softmax (max-subtraction)
+ * over `EXPRESSIVE_EMOTION_TAGS` and selects the argmax.
+ *
+ * Confidence is the softmax probability of the picked class (in [0, 1]),
+ * which gives downstream consumers a calibrated mass to compare against
+ * the V-A-D-projection path's 0.35 abstain floor.
+ *
+ * The `vad` field is synthesised at the neutral midpoint (0.5, 0.5, 0.5).
+ * The cls7 head is the ground truth for the picked emotion — the V-A-D
+ * triple is left at neutral because we no longer regress to a V-A-D
+ * target. Consumers that need real V-A-D must use a `head=vad` model.
+ */
+export function interpretCls7Output(
+	logits: Float32Array,
+	modelId: VoiceEmotionModelId,
+	latencyMs: number,
+): VoiceEmotionClassifierOutput {
+	const n = EXPRESSIVE_EMOTION_TAGS.length;
+	if (logits.length !== n) {
+		throw new VoiceEmotionClassifierError(
+			`[voice-emotion] interpretCls7Output: expected ${n} logits, got ${logits.length}`,
+		);
+	}
+	let maxLogit = Number.NEGATIVE_INFINITY;
+	for (let i = 0; i < n; i++) {
+		const v = logits[i] ?? 0;
+		if (Number.isFinite(v) && v > maxLogit) maxLogit = v;
+	}
+	if (!Number.isFinite(maxLogit)) {
+		// All-NaN/Inf logits — abstain rather than coerce.
+		const emptyScores = makeEmptyScoresRecord();
+		return {
+			vad: { valence: 0.5, arousal: 0.5, dominance: 0.5 },
+			emotion: null,
+			confidence: 0,
+			scores: emptyScores,
+			modelId,
+			latencyMs,
+		};
+	}
+	const exps = new Float32Array(n);
+	let sum = 0;
+	for (let i = 0; i < n; i++) {
+		const v = logits[i] ?? 0;
+		const e = Math.exp((Number.isFinite(v) ? v : maxLogit) - maxLogit);
+		exps[i] = e;
+		sum += e;
+	}
+	const denom = sum > 0 ? sum : 1;
+	let bestIdx = 0;
+	let bestProb = 0;
+	const scores = makeEmptyScoresRecord();
+	for (let i = 0; i < n; i++) {
+		const tag = EXPRESSIVE_EMOTION_TAGS[i];
+		if (!tag) continue;
+		const p = (exps[i] ?? 0) / denom;
+		scores[tag] = p;
+		if (p > bestProb) {
+			bestProb = p;
+			bestIdx = i;
+		}
+	}
+	const emotionTag = EXPRESSIVE_EMOTION_TAGS[bestIdx];
+	return {
+		// The cls7 head doesn't regress V-A-D — surface the neutral
+		// midpoint so callers can still destructure but know not to trust
+		// these floats as anything other than "head was cls7".
+		vad: { valence: 0.5, arousal: 0.5, dominance: 0.5 },
+		emotion: emotionTag ?? null,
+		confidence: bestProb,
+		scores,
+		modelId,
+		latencyMs,
+	};
+}
+
+function makeEmptyScoresRecord(): Record<ExpressiveEmotion, number> {
+	return {
+		happy: 0,
+		sad: 0,
+		angry: 0,
+		nervous: 0,
+		calm: 0,
+		excited: 0,
+		whisper: 0,
+	};
+}

package/src/services/voice/voice-preset-format.d.ts

@@ -0,0 +1,158 @@
+/**
+ * Binary format for `cache/voice-preset-*.bin`.
+ *
+ * Two versions are supported:
+ *
+ *   v1 (`magic='ELZ1', version=1`) — legacy two-section layout used by the
+ *   initial Kokoro-style placeholder. Carries a Float32 speaker embedding +
+ *   a phrase-cache seed list. Still read for back-compat (older bundles only
+ *   contain v1).
+ *
+ *   v2 (`magic='ELZ1', version=2`) — superset adopted for the OmniVoice
+ *   freeze. Adds three OmniVoice-specific sections that the v1 layout had
+ *   no room for: pre-encoded `ref_audio_tokens` (int32, shape
+ *   `[K, ref_T]`), a UTF-8 `ref_text` transcript of the reference clip, and
+ *   a closed-vocabulary `instruct` string (the resolved VoiceDesign
+ *   attributes). v2 readers handle v1 files transparently (the new sections
+ *   default to empty). A v1 reader applied to a v2 file fails fast on
+ *   `truncated-header` because the v2 header is larger.
+ *
+ * Layout (little-endian throughout):
+ *
+ *   v1 header (24 bytes):
+ *     +0   4 bytes  magic 'ELZ1' (0x315A4C45)
+ *     +4   4 bytes  format version (uint32)                — 1
+ *     +8   4 bytes  speaker embedding offset (uint32)
+ *     +12  4 bytes  speaker embedding byte length (uint32)
+ *     +16  4 bytes  phrase cache seed offset (uint32)
+ *     +20  4 bytes  phrase cache seed byte length (uint32)
+ *
+ *   v2 header (64 bytes — additive, all section descriptors are
+ *               `(offset:uint32, length:uint32)` pairs):
+ *     +0   4 bytes  magic 'ELZ1' (0x315A4C45)
+ *     +4   4 bytes  format version (uint32)                — 2
+ *     +8   4 bytes  speaker embedding offset
+ *     +12  4 bytes  speaker embedding byte length
+ *     +16  4 bytes  phrase cache seed offset
+ *     +20  4 bytes  phrase cache seed byte length
+ *     +24  4 bytes  ref_audio_tokens offset
+ *     +28  4 bytes  ref_audio_tokens byte length
+ *     +32  4 bytes  ref_text offset
+ *     +36  4 bytes  ref_text byte length
+ *     +40  4 bytes  instruct offset
+ *     +44  4 bytes  instruct byte length
+ *     +48  4 bytes  metadata offset
+ *     +52  4 bytes  metadata byte length
+ *     +56  4 bytes  reserved (must be 0)
+ *     +60  4 bytes  reserved (must be 0)
+ *
+ *   `ref_audio_tokens` payload (v2):
+ *     +0   4 bytes  K            — codebook count (uint32, OmniVoice = 8)
+ *     +4   4 bytes  ref_T        — frames per codebook (uint32)
+ *     +8   ...      int32 LE codebook samples, row-major shape `[K, ref_T]`
+ *
+ *   `ref_text` payload (v2): raw UTF-8 bytes (no NUL terminator).
+ *   `instruct` payload (v2): raw UTF-8 bytes (closed VoiceDesign vocabulary).
+ *   `metadata` payload (v2): raw UTF-8 JSON bytes (codec sha256, corpus
+ *                            hash, etc.); the runtime never relies on
+ *                            metadata for correctness.
+ *
+ *   Phrase cache seed payload (v1 + v2, identical):
+ *     uint32 LE  N (phrase count)
+ *     for each phrase:
+ *       uint16 LE  text_byte_len
+ *       uint8[]    canonicalized text (UTF-8)
+ *       uint32 LE  sample_rate
+ *       uint32 LE  pcm_byte_len
+ *       uint8[]    PCM (Float32 LE samples)
+ *
+ * Per-section invariants:
+ *   - Section bounds may not overlap the header.
+ *   - Section bounds must fit within the file length.
+ *   - A `length=0` section is allowed (means "absent"); the corresponding
+ *     output field is an empty `Float32Array` / `Int32Array` / empty string.
+ *   - `embedding.length % 4 == 0` (Float32).
+ *   - `ref_audio_tokens.length` ≥ 8 (the two header words K, ref_T) and the
+ *     payload is `8 + K*ref_T*4` bytes.
+ */
+export declare const VOICE_PRESET_MAGIC = 828001349;
+/** Header byte counts. */
+export declare const VOICE_PRESET_HEADER_BYTES_V1 = 24;
+export declare const VOICE_PRESET_HEADER_BYTES_V2 = 64;
+/** Supported format versions. v2 is the canonical write path. */
+export declare const VOICE_PRESET_VERSION_V1 = 1;
+export declare const VOICE_PRESET_VERSION_V2 = 2;
+export declare const VOICE_PRESET_VERSION_CURRENT = 2;
+export interface VoicePresetSeedPhrase {
+    /** Canonicalized text (lowercase, single-spaced, trimmed). */
+    text: string;
+    sampleRate: number;
+    pcm: Float32Array;
+}
+/**
+ * OmniVoice reference-audio-tokens payload. `K` is the codebook count (=8 for
+ * OmniVoice / HiggsAudioV2) and `refT` is the number of frames per codebook.
+ * `tokens` is row-major: codebook `k`, frame `t` is at `tokens[k*refT + t]`.
+ * An empty payload (refT=0, K=0, tokens length 0) is valid and means "no
+ * reference audio bound to this preset" (instruct-only voice).
+ */
+export interface RefAudioTokens {
+    K: number;
+    refT: number;
+    tokens: Int32Array;
+}
+export interface VoicePresetFile {
+    version: number;
+    embedding: Float32Array;
+    phrases: ReadonlyArray<VoicePresetSeedPhrase>;
+    /** v2 only — empty for v1 files. */
+    refAudioTokens: RefAudioTokens;
+    /** v2 only — empty for v1 files. */
+    refText: string;
+    /** v2 only — empty for v1 files. */
+    instruct: string;
+    /** v2 only — parsed JSON object, empty `{}` for v1 files. */
+    metadata: Record<string, unknown>;
+}
+export declare class VoicePresetFormatError extends Error {
+    readonly code: "bad-magic" | "bad-version" | "truncated-header" | "truncated-section" | "bad-section-bounds" | "bad-phrase-record" | "bad-embedding-length" | "bad-ref-tokens" | "bad-metadata";
+    constructor(message: string, code: "bad-magic" | "bad-version" | "truncated-header" | "truncated-section" | "bad-section-bounds" | "bad-phrase-record" | "bad-embedding-length" | "bad-ref-tokens" | "bad-metadata");
+}
+/**
+ * Parse a voice-preset binary blob. Throws `VoicePresetFormatError` on any
+ * malformed input — this is the single defensive boundary for the format.
+ * Supports both v1 and v2 files. For v1 files the v2-only fields are
+ * returned as their empty equivalents.
+ */
+export declare function readVoicePresetFile(bytes: Uint8Array): VoicePresetFile;
+/**
+ * Serialize a voice preset to the v1 binary format. The output is a fresh
+ * `Uint8Array` ready to be written to disk.
+ *
+ * Use this only when the caller deliberately wants the legacy v1 shape (e.g.
+ * the existing Kokoro-style placeholder builder). New code should call
+ * `writeVoicePresetFileV2`.
+ */
+export declare function writeVoicePresetFile(file: {
+    embedding: Float32Array;
+    phrases: ReadonlyArray<VoicePresetSeedPhrase>;
+}): Uint8Array;
+/**
+ * Write a voice preset in the v2 (additive) layout. Used by the OmniVoice
+ * freeze pipeline (`freeze-voice.mjs`) and other producers that need to
+ * persist `refAudioTokens` / `refText` / `instruct` alongside the v1
+ * embedding + phrase-seed sections.
+ *
+ * Any field that the caller doesn't need to persist can be omitted (or
+ * passed empty). The on-disk section is then written as length=0 and is
+ * read back as the empty equivalent.
+ */
+export declare function writeVoicePresetFileV2(file: {
+    embedding?: Float32Array;
+    phrases?: ReadonlyArray<VoicePresetSeedPhrase>;
+    refAudioTokens?: RefAudioTokens;
+    refText?: string;
+    instruct?: string;
+    metadata?: Record<string, unknown>;
+}): Uint8Array;
+//# sourceMappingURL=voice-preset-format.d.ts.map

package/src/services/voice/voice-preset-format.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"voice-preset-format.d.ts","sourceRoot":"","sources":["voice-preset-format.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA4EG;AAEH,eAAO,MAAM,kBAAkB,YAAa,CAAC;AAE7C,0BAA0B;AAC1B,eAAO,MAAM,4BAA4B,KAAK,CAAC;AAC/C,eAAO,MAAM,4BAA4B,KAAK,CAAC;AAE/C,iEAAiE;AACjE,eAAO,MAAM,uBAAuB,IAAI,CAAC;AACzC,eAAO,MAAM,uBAAuB,IAAI,CAAC;AACzC,eAAO,MAAM,4BAA4B,IAA0B,CAAC;AAEpE,MAAM,WAAW,qBAAqB;IACrC,8DAA8D;IAC9D,IAAI,EAAE,MAAM,CAAC;IACb,UAAU,EAAE,MAAM,CAAC;IACnB,GAAG,EAAE,YAAY,CAAC;CAClB;AAED;;;;;;GAMG;AACH,MAAM,WAAW,cAAc;IAC9B,CAAC,EAAE,MAAM,CAAC;IACV,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,UAAU,CAAC;CACnB;AAED,MAAM,WAAW,eAAe;IAC/B,OAAO,EAAE,MAAM,CAAC;IAChB,SAAS,EAAE,YAAY,CAAC;IACxB,OAAO,EAAE,aAAa,CAAC,qBAAqB,CAAC,CAAC;IAC9C,oCAAoC;IACpC,cAAc,EAAE,cAAc,CAAC;IAC/B,oCAAoC;IACpC,OAAO,EAAE,MAAM,CAAC;IAChB,oCAAoC;IACpC,QAAQ,EAAE,MAAM,CAAC;IACjB,6DAA6D;IAC7D,QAAQ,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CAClC;AAED,qBAAa,sBAAuB,SAAQ,KAAK;IAG/C,QAAQ,CAAC,IAAI,EACV,WAAW,GACX,aAAa,GACb,kBAAkB,GAClB,mBAAmB,GACnB,oBAAoB,GACpB,mBAAmB,GACnB,sBAAsB,GACtB,gBAAgB,GAChB,cAAc;gBAVjB,OAAO,EAAE,MAAM,EACN,IAAI,EACV,WAAW,GACX,aAAa,GACb,kBAAkB,GAClB,mBAAmB,GACnB,oBAAoB,GACpB,mBAAmB,GACnB,sBAAsB,GACtB,gBAAgB,GAChB,cAAc;CAKlB;AAyTD;;;;;GAKG;AACH,wBAAgB,mBAAmB,CAAC,KAAK,EAAE,UAAU,GAAG,eAAe,CAYtE;AAED;;;;;;;GAOG;AACH,wBAAgB,oBAAoB,CAAC,IAAI,EAAE;IAC1C,SAAS,EAAE,YAAY,CAAC;IACxB,OAAO,EAAE,aAAa,CAAC,qBAAqB,CAAC,CAAC;CAC9C,GAAG,UAAU,CA4Cb;AAmCD;;;;;;;;;GASG;AACH,wBAAgB,sBAAsB,CAAC,IAAI,EAAE;IAC5C,SAAS,CAAC,EAAE,YAAY,CAAC;IACzB,OAAO,CAAC,EAAE,aAAa,CAAC,qBAAqB,CAAC,CAAC;IAC/C,cAAc,CAAC,EAAE,cAAc,CAAC;IAChC,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACnC,GAAG,UAAU,CAuHb"}