@elizaos/plugin-local-inference 2.0.0-beta.1 → 2.0.3-beta.3

package/src/services/voice/ring-buffer.test.ts

@@ -0,0 +1,129 @@
+import { describe, expect, it } from "vitest";
+import { InMemoryAudioSink, PcmRingBuffer } from "./ring-buffer";
+
+function f32(...xs: number[]): Float32Array {
+	return Float32Array.from(xs);
+}
+
+/** Concatenate everything an InMemoryAudioSink received, in order. */
+function sinkData(sink: InMemoryAudioSink): number[] {
+	const out: number[] = [];
+	for (const c of sink.chunks) out.push(...c.pcm);
+	return out;
+}
+
+describe("PcmRingBuffer (bulk-copy)", () => {
+	it("writes then flushes in FIFO order", () => {
+		const sink = new InMemoryAudioSink();
+		const rb = new PcmRingBuffer(8, 16000, sink);
+		rb.write(f32(1, 2, 3));
+		rb.write(f32(4, 5));
+		expect(rb.size()).toBe(5);
+		expect(rb.flushToSink()).toBe(5);
+		expect(sinkData(sink)).toEqual([1, 2, 3, 4, 5]);
+		expect(rb.size()).toBe(0);
+	});
+
+	it("handles a write that wraps the write cursor", () => {
+		const sink = new InMemoryAudioSink();
+		const rb = new PcmRingBuffer(4, 16000, sink);
+		rb.write(f32(1, 2, 3)); // writePos=3
+		expect(rb.flushToSink()).toBe(3); // readPos=writePos=3
+		rb.write(f32(4, 5, 6)); // wraps: positions 3,0,1
+		expect(rb.flushToSink()).toBe(3);
+		expect(sinkData(sink)).toEqual([1, 2, 3, 4, 5, 6]);
+	});
+
+	it("drops oldest samples on overflow and reports the count", () => {
+		const sink = new InMemoryAudioSink();
+		const dropped: number[] = [];
+		const rb = new PcmRingBuffer(4, 16000, sink, {
+			onOverflow: (n) => dropped.push(n),
+		});
+		rb.write(f32(1, 2, 3)); // filled=3
+		rb.write(f32(4, 5, 6)); // filled would be 6 → drop 2 oldest (1,2)
+		expect(dropped).toEqual([2]);
+		expect(rb.size()).toBe(4);
+		expect(rb.flushToSink()).toBe(4);
+		expect(sinkData(sink)).toEqual([3, 4, 5, 6]);
+	});
+
+	it("keeps only the last `capacity` samples when one write exceeds capacity", () => {
+		const sink = new InMemoryAudioSink();
+		const dropped: number[] = [];
+		const rb = new PcmRingBuffer(4, 16000, sink, {
+			onOverflow: (n) => dropped.push(n),
+		});
+		rb.write(f32(1, 2, 3, 4, 5, 6, 7)); // 7 samples into cap-4 → keep 4,5,6,7
+		expect(dropped).toEqual([3]);
+		expect(rb.size()).toBe(4);
+		rb.flushToSink();
+		expect(sinkData(sink)).toEqual([4, 5, 6, 7]);
+	});
+
+	it("exact-capacity write fills without dropping", () => {
+		const sink = new InMemoryAudioSink();
+		const dropped: number[] = [];
+		const rb = new PcmRingBuffer(4, 16000, sink, {
+			onOverflow: (n) => dropped.push(n),
+		});
+		rb.write(f32(1, 2, 3, 4));
+		expect(dropped).toEqual([]);
+		expect(rb.size()).toBe(4);
+		rb.flushToSink();
+		expect(sinkData(sink)).toEqual([1, 2, 3, 4]);
+	});
+
+	it("partial overflow when already partly filled, with wrap", () => {
+		const sink = new InMemoryAudioSink();
+		const rb = new PcmRingBuffer(4, 16000, sink);
+		rb.write(f32(1, 2)); // writePos=2, filled=2
+		rb.write(f32(3, 4, 5)); // 3→pos2, 4→pos3, 5→pos0 (overwrites 1); drop 1
+		// logical order now: 2,3,4,5
+		expect(rb.size()).toBe(4);
+		rb.flushToSink();
+		expect(sinkData(sink)).toEqual([2, 3, 4, 5]);
+	});
+
+	it("pressure reflects fill ratio", () => {
+		const rb = new PcmRingBuffer(10, 16000, new InMemoryAudioSink());
+		expect(rb.pressure()).toBe(0);
+		rb.write(f32(1, 2, 3, 4, 5));
+		expect(rb.pressure()).toBeCloseTo(0.5);
+	});
+
+	it("drain clears without writing to the sink", () => {
+		const sink = new InMemoryAudioSink();
+		const rb = new PcmRingBuffer(4, 16000, sink);
+		rb.write(f32(1, 2, 3));
+		rb.drain();
+		expect(rb.size()).toBe(0);
+		expect(sinkData(sink)).toEqual([]);
+	});
+
+	it("empty write is a no-op", () => {
+		const sink = new InMemoryAudioSink();
+		const rb = new PcmRingBuffer(4, 16000, sink);
+		rb.write(new Float32Array(0));
+		expect(rb.size()).toBe(0);
+		expect(rb.flushToSink()).toBe(0);
+	});
+
+	it("matches the reference per-sample model over a randomized-ish sequence", () => {
+		// Reference oracle implemented straightforwardly; compare logical contents.
+		const cap = 5;
+		const sink = new InMemoryAudioSink();
+		const rb = new PcmRingBuffer(cap, 16000, sink);
+		const ref: number[] = [];
+		const writes = [[1, 2], [3, 4, 5, 6], [7], [8, 9, 10, 11, 12, 13]];
+		const value = 0;
+		for (const w of writes) {
+			void value;
+			rb.write(Float32Array.from(w));
+			ref.push(...w);
+			while (ref.length > cap) ref.shift();
+		}
+		rb.flushToSink();
+		expect(sinkData(sink)).toEqual(ref);
+	});
+});

package/src/services/voice/ring-buffer.ts

@@ -0,0 +1,123 @@
+import type { AudioSink } from "./types";
+
+export interface PcmRingBufferOptions {
+	/**
+	 * Fired when the buffer is full and a write overwrites unread samples.
+	 * Reports the count of dropped samples in this write call. Schedulers
+	 * should use this signal to apply backpressure upstream — silent
+	 * overwrites produce audible glitches.
+	 */
+	onOverflow?: (droppedSamples: number) => void;
+}
+
+export class PcmRingBuffer {
+	private readonly buf: Float32Array;
+	private readPos = 0;
+	private writePos = 0;
+	private filled = 0;
+	private readonly onOverflow?: (droppedSamples: number) => void;
+
+	constructor(
+		private readonly capacity: number,
+		private readonly sampleRate: number,
+		private readonly sink: AudioSink,
+		options: PcmRingBufferOptions = {},
+	) {
+		if (capacity <= 0) {
+			throw new Error("PcmRingBuffer: capacity must be positive");
+		}
+		this.buf = new Float32Array(capacity);
+		this.onOverflow = options.onOverflow;
+	}
+
+	write(pcm: Float32Array): void {
+		const n = pcm.length;
+		if (n === 0) return;
+		// Samples lost to overwrite: everything past capacity once we account for
+		// what's already buffered. Matches the per-sample loop's drop accounting.
+		const dropped = Math.max(0, this.filled + n - this.capacity);
+
+		if (n >= this.capacity) {
+			// The incoming chunk alone fills (or overfills) the ring — only its last
+			// `capacity` samples survive. Bulk-copy them and reset to a full buffer.
+			this.buf.set(pcm.subarray(n - this.capacity, n), 0);
+			this.readPos = 0;
+			this.writePos = 0;
+			this.filled = this.capacity;
+		} else {
+			// Up to two contiguous bulk copies (one before the wrap, one after).
+			const firstSpan = Math.min(n, this.capacity - this.writePos);
+			this.buf.set(pcm.subarray(0, firstSpan), this.writePos);
+			if (n > firstSpan) this.buf.set(pcm.subarray(firstSpan, n), 0);
+			this.writePos = (this.writePos + n) % this.capacity;
+			if (dropped > 0) {
+				// Buffer overran: the read cursor chases the write cursor.
+				this.readPos = (this.readPos + dropped) % this.capacity;
+				this.filled = this.capacity;
+			} else {
+				this.filled += n;
+			}
+		}
+
+		if (dropped > 0 && this.onOverflow) {
+			this.onOverflow(dropped);
+		}
+	}
+
+	/** Fill ratio in [0, 1]. Schedulers can throttle TTS dispatches as this approaches 1. */
+	pressure(): number {
+		return this.filled / this.capacity;
+	}
+
+	flushToSink(): number {
+		if (this.filled === 0) return 0;
+		const n = this.filled;
+		const out = new Float32Array(n);
+		// Read `n` samples from readPos with at most one wrap — two bulk copies.
+		const firstSpan = Math.min(n, this.capacity - this.readPos);
+		out.set(this.buf.subarray(this.readPos, this.readPos + firstSpan), 0);
+		if (n > firstSpan) out.set(this.buf.subarray(0, n - firstSpan), firstSpan);
+		this.readPos = this.writePos;
+		this.filled = 0;
+		this.sink.write(out, this.sampleRate);
+		return n;
+	}
+
+	drain(): void {
+		this.readPos = this.writePos;
+		this.filled = 0;
+		this.sink.drain();
+	}
+
+	size(): number {
+		return this.filled;
+	}
+
+	capacityHint(): number {
+		return this.capacity;
+	}
+}
+
+export class InMemoryAudioSink implements AudioSink {
+	readonly chunks: Array<{ pcm: Float32Array; sampleRate: number }> = [];
+	private buffered = 0;
+
+	write(pcm: Float32Array, sampleRate: number): void {
+		this.chunks.push({ pcm, sampleRate });
+		this.buffered += pcm.length;
+	}
+
+	drain(): void {
+		this.buffered = 0;
+	}
+
+	bufferedSamples(): number {
+		return this.buffered;
+	}
+
+	totalWritten(): number {
+		let n = 0;
+		for (const c of this.chunks) n += c.pcm.length;
+		return n;
+	}
+}

package/src/services/voice/rollback-queue.ts

@@ -0,0 +1,74 @@
+import type { Phrase, RejectedTokenRange } from "./types";
+
+type PhraseState = "queued" | "synthesizing" | "ringbuffered" | "played";
+
+interface TrackedPhrase {
+	phrase: Phrase;
+	state: PhraseState;
+}
+
+export interface RollbackEvent {
+	phraseId: number;
+	reason: "rejected-tokens";
+	rejectedRange: RejectedTokenRange;
+}
+
+export class RollbackQueue {
+	private readonly tracked = new Map<number, TrackedPhrase>();
+
+	track(phrase: Phrase): void {
+		this.tracked.set(phrase.id, { phrase, state: "queued" });
+	}
+
+	markSynthesizing(phraseId: number): void {
+		const entry = this.requireEntry(phraseId);
+		entry.state = "synthesizing";
+	}
+
+	markRingBuffered(phraseId: number): void {
+		const entry = this.requireEntry(phraseId);
+		entry.state = "ringbuffered";
+	}
+
+	markPlayed(phraseId: number): void {
+		const entry = this.requireEntry(phraseId);
+		entry.state = "played";
+	}
+
+	drop(phraseId: number): void {
+		this.tracked.delete(phraseId);
+	}
+
+	onRejected(range: RejectedTokenRange): RollbackEvent[] {
+		const events: RollbackEvent[] = [];
+		for (const entry of this.tracked.values()) {
+			if (entry.state === "played") continue;
+			if (this.overlaps(entry.phrase, range)) {
+				events.push({
+					phraseId: entry.phrase.id,
+					reason: "rejected-tokens",
+					rejectedRange: range,
+				});
+			}
+		}
+		return events;
+	}
+
+	snapshot(): ReadonlyArray<{ phrase: Phrase; state: PhraseState }> {
+		return Array.from(this.tracked.values()).map((e) => ({ ...e }));
+	}
+
+	private requireEntry(phraseId: number): TrackedPhrase {
+		const entry = this.tracked.get(phraseId);
+		if (!entry) {
+			throw new Error(`RollbackQueue: unknown phraseId ${phraseId}`);
+		}
+		return entry;
+	}
+
+	private overlaps(phrase: Phrase, range: RejectedTokenRange): boolean {
+		return (
+			phrase.toIndex >= range.fromIndex && phrase.fromIndex <= range.toIndex
+		);
+	}
+}

package/src/services/voice/samantha-preset-placeholder.test.ts

@@ -0,0 +1,97 @@
+import { mkdtempSync, rmSync, writeFileSync } from "node:fs";
+import os from "node:os";
+import path from "node:path";
+import { afterEach, beforeEach, describe, expect, it } from "vitest";
+
+import {
+	detectSamanthaPlaceholder,
+	SAMANTHA_PLACEHOLDER_BYTE_LENGTH,
+} from "./samantha-preset-placeholder";
+import {
+	VOICE_PRESET_MAGIC,
+	VOICE_PRESET_VERSION_V1,
+	writeVoicePresetFileV2,
+} from "./voice-preset-format";
+
+function writeLegacyV1Placeholder(): Uint8Array {
+	const blob = new Uint8Array(SAMANTHA_PLACEHOLDER_BYTE_LENGTH);
+	const view = new DataView(blob.buffer);
+	view.setUint32(0, VOICE_PRESET_MAGIC, true);
+	view.setUint32(4, VOICE_PRESET_VERSION_V1, true);
+	view.setUint32(8, 24, true);
+	view.setUint32(12, 256 * 4, true);
+	view.setUint32(16, 24 + 256 * 4, true);
+	view.setUint32(20, 4, true);
+	view.setUint32(24 + 256 * 4, 0, true);
+	return blob;
+}
+
+describe("detectSamanthaPlaceholder", () => {
+	let dir: string;
+	beforeEach(() => {
+		dir = mkdtempSync(path.join(os.tmpdir(), "samantha-placeholder-test-"));
+	});
+	afterEach(() => {
+		rmSync(dir, { recursive: true, force: true });
+	});
+
+	it("reports `missing` when the file does not exist", () => {
+		const out = detectSamanthaPlaceholder(path.join(dir, "absent.bin"));
+		expect(out.kind).toBe("missing");
+	});
+
+	it("reports `real-preset` for files of the wrong byte length", () => {
+		const p = path.join(dir, "tiny.bin");
+		writeFileSync(p, Buffer.alloc(64));
+		const out = detectSamanthaPlaceholder(p);
+		expect(out.kind).toBe("real-preset");
+		if (out.kind === "real-preset") {
+			expect(out.reason).toMatch(/byte-length/);
+		}
+	});
+
+	it("does not report a differently-shaped v2 zero preset as the shipped placeholder", () => {
+		// Current v2 serialization has a different byte layout than the
+		// 1052-byte I-wave placeholder staged in installed bundles.
+		const blob = writeVoicePresetFileV2({
+			embedding: new Float32Array(256), // all zeros
+			phrases: [],
+		});
+		expect(blob.byteLength).not.toBe(SAMANTHA_PLACEHOLDER_BYTE_LENGTH);
+		const p = path.join(dir, "voice-preset-default.bin");
+		writeFileSync(p, blob);
+		const out = detectSamanthaPlaceholder(p);
+		expect(out.kind).toBe("real-preset");
+	});
+
+	it("reports `placeholder` for the legacy v1 zero-fill blob staged in current bundles", () => {
+		const blob = writeLegacyV1Placeholder();
+		expect(blob.byteLength).toBe(SAMANTHA_PLACEHOLDER_BYTE_LENGTH);
+		const p = path.join(dir, "legacy-v1-placeholder.bin");
+		writeFileSync(p, blob);
+		const out = detectSamanthaPlaceholder(p);
+		expect(out.kind).toBe("placeholder");
+	});
+
+	it("reports `real-preset` when the embedding has any non-zero sample", () => {
+		const emb = new Float32Array(256);
+		emb[42] = 0.001;
+		const blob = writeVoicePresetFileV2({ embedding: emb, phrases: [] });
+		const p = path.join(dir, "almost.bin");
+		writeFileSync(p, blob);
+		const out = detectSamanthaPlaceholder(p);
+		expect(out.kind).toBe("real-preset");
+	});
+
+	it("reports `real-preset` when ref_audio_tokens or ref_text are populated", () => {
+		const blob = writeVoicePresetFileV2({
+			embedding: new Float32Array(256),
+			phrases: [],
+			refText: "Hello there.",
+		});
+		const p = path.join(dir, "with-ref.bin");
+		writeFileSync(p, blob);
+		const out = detectSamanthaPlaceholder(p);
+		expect(out.kind).toBe("real-preset");
+	});
+});

package/src/services/voice/samantha-preset-placeholder.ts

@@ -0,0 +1,148 @@
+/**
+ * Samantha-preset placeholder detection + on-the-fly regeneration.
+ *
+ * Background: the canonical default voice (Samantha, `af_same`) ships with
+ * the Eliza-1 bundle as `cache/voice-preset-default.bin` (ELZ1 format).
+ * The first I-wave shipped a 1052-byte zero-filled placeholder before the
+ * real preset bytes were produced — the runtime treats this placeholder as
+ * "not yet generated" and synthesises a fresh preset on first boot via the
+ * fused OmniVoice TTS (Path A).
+ *
+ * The detection rule is intentionally narrow: only an ELZ1 v1/v2 file whose
+ * (a) speaker embedding is exactly zero AND (b) ref_audio_tokens / ref_text
+ * are empty AND (c) phrase-cache seed is empty counts as a placeholder. Any
+ * file that has even one non-zero region is considered real and is left
+ * alone — the runtime never silently overwrites operator-supplied presets.
+ *
+ * The fallback chain executed by `EngineVoiceBridge.start()`:
+ *
+ *   1. If `cache/voice-preset-default.bin` is missing → throw (existing
+ *      behaviour, the bundle is malformed).
+ *   2. If the file exists and is NOT a placeholder → load it as-is.
+ *   3. If it IS a placeholder AND the OmniVoice reference-encode FFI is
+ *      available → regenerate the preset using the bundled Samantha
+ *      reference clip + transcript, write the new bytes back to disk,
+ *      then load.
+ *   4. If it IS a placeholder AND OmniVoice reference-encode is unavailable
+ *      → log a loud warning and fall through to the bundled Kokoro default
+ *      voice (`kokoro.defaultVoiceId`) by re-pointing the discovery layer.
+ */
+
+import { readFileSync, statSync } from "node:fs";
+import {
+	readVoicePresetFile,
+	VOICE_PRESET_HEADER_BYTES_V1,
+	VOICE_PRESET_MAGIC,
+	VOICE_PRESET_VERSION_V1,
+	VOICE_PRESET_VERSION_V2,
+	type VoicePresetFile,
+	VoicePresetFormatError,
+} from "./voice-preset-format";
+
+/** The exact byte-length the I-wave placeholder shipped at. Used as a fast
+ *  pre-check before the more expensive structural parse — files of any
+ *  other size cannot be the placeholder. */
+export const SAMANTHA_PLACEHOLDER_BYTE_LENGTH = 1052;
+
+/** Reasons `detectSamanthaPlaceholder` returns. Each is structurally
+ *  distinct so the caller can branch without re-parsing. */
+export type SamanthaPlaceholderState =
+	| { kind: "missing" }
+	| { kind: "real-preset"; reason: string }
+	| { kind: "placeholder" }
+	| { kind: "unreadable"; reason: string };
+
+/**
+ * Inspect `presetPath` and report whether it is the I-wave zero-fill
+ * placeholder, a real preset, missing, or unreadable. Pure I/O — no
+ * mutation of the file.
+ */
+export function detectSamanthaPlaceholder(
+	presetPath: string,
+): SamanthaPlaceholderState {
+	let bytes: Uint8Array;
+	try {
+		const stat = statSync(presetPath);
+		if (!stat.isFile()) {
+			return { kind: "unreadable", reason: "path is not a regular file" };
+		}
+		bytes = new Uint8Array(readFileSync(presetPath));
+	} catch (err) {
+		const code = (err as NodeJS.ErrnoException).code;
+		if (code === "ENOENT") return { kind: "missing" };
+		return {
+			kind: "unreadable",
+			reason: (err as Error).message ?? String(err),
+		};
+	}
+
+	// Fast path: only the exact placeholder byte-length is a candidate.
+	if (bytes.byteLength !== SAMANTHA_PLACEHOLDER_BYTE_LENGTH) {
+		return { kind: "real-preset", reason: "byte-length mismatch" };
+	}
+
+	// Structural path: must parse as ELZ1 v1/v2 with all-zero embedding and
+	// empty ref/phrase sections.
+	if (bytes.byteLength < VOICE_PRESET_HEADER_BYTES_V1) {
+		return { kind: "real-preset", reason: "too short for v1 header" };
+	}
+	const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);
+	if (view.getUint32(0, true) !== VOICE_PRESET_MAGIC) {
+		return { kind: "real-preset", reason: "magic mismatch" };
+	}
+	const version = view.getUint32(4, true);
+	if (
+		version !== VOICE_PRESET_VERSION_V1 &&
+		version !== VOICE_PRESET_VERSION_V2
+	) {
+		return { kind: "real-preset", reason: "version mismatch" };
+	}
+
+	let parsed: VoicePresetFile;
+	try {
+		parsed = readVoicePresetFile(bytes);
+	} catch (err) {
+		// Malformed v2 file — not a placeholder we own; surface as
+		// real-preset so the loader's existing error path runs.
+		if (err instanceof VoicePresetFormatError) {
+			return { kind: "real-preset", reason: `parse failed: ${err.code}` };
+		}
+		throw err;
+	}
+
+	if (parsed.embedding.length === 0) {
+		return { kind: "real-preset", reason: "empty embedding section" };
+	}
+	if (!parsed.embedding.every((v) => v === 0)) {
+		return { kind: "real-preset", reason: "non-zero embedding sample" };
+	}
+	if (parsed.refAudioTokens.tokens.length !== 0) {
+		return { kind: "real-preset", reason: "ref_audio_tokens populated" };
+	}
+	if (parsed.refText.length !== 0) {
+		return { kind: "real-preset", reason: "ref_text populated" };
+	}
+	if (parsed.phrases.length !== 0) {
+		return { kind: "real-preset", reason: "phrase-cache seed populated" };
+	}
+
+	return { kind: "placeholder" };
+}
+
+/**
+ * The text the on-the-fly regeneration uses as the Samantha reference
+ * transcript. Stable across versions so the produced preset is
+ * deterministic — pinning the prompt is part of the cacheability contract
+ * (same inputs -> same output bytes for byte-for-byte comparison).
+ */
+export const SAMANTHA_REFERENCE_TRANSCRIPT =
+	"Hi, I'm Samantha. It's nice to finally talk to you about something real.";
+
+/**
+ * The closed-vocabulary VoiceDesign instruct string baked into the
+ * regenerated Samantha preset. Mirrors the publish-time freeze CLI's
+ * settings so the runtime regen and a future operator-built preset land on
+ * the same surface. Pinned for deterministic regeneration.
+ */
+export const SAMANTHA_INSTRUCT =
+	"female, american accent, young adult, moderate pitch";