@elizaos/plugin-local-inference 2.0.0-beta.1 → 2.0.3-beta.3

package/src/services/voice/__test-helpers__/fake-ffi.ts

@@ -0,0 +1,94 @@
+/**
+ * Test-only `ElizaInferenceFfi` stand-in. Only the methods the voice
+ * pipeline exercises are non-trivial: `asrTranscribe` returns the supplied
+ * fixed transcript; `ttsSynthesize` writes a constant number of samples;
+ * `ttsSynthesizeStream` emits the same PCM as two chunks (one body + one
+ * `isFinal` tail) and honours `onChunk` returning `true` as a cancel.
+ * The ABI-v2 streaming-ASR symbols report "no working decoder" by
+ * default (the same as the C unsupported-build path) so the pipeline routes through the v1
+ * batch path unless a test opts into `asrStreamSupported`. Everything
+ * else is a no-op / identity so a test can wire a "fused" FFI without a
+ * real `.dylib`.
+ */
+import type { ElizaInferenceFfi, TtsStreamChunk } from "../ffi-bindings";
+
+export function fakeFfi(
+	transcript: string,
+	opts: {
+		ttsSamples?: number;
+		ttsStreamSupported?: boolean;
+		asrStreamSupported?: boolean;
+		vadSupported?: boolean;
+		vadProbs?: readonly number[];
+		speakerSupported?: boolean;
+		/** The 256-float embedding `speakerEmbed` returns (defaults to zeros). */
+		speakerEmbedding?: Float32Array;
+		diarizSupported?: boolean;
+		/** The per-frame label sequence `diarizSegment` returns. */
+		diarizLabels?: Int8Array;
+	} = {},
+): ElizaInferenceFfi {
+	const ttsSamples = opts.ttsSamples ?? 8;
+	const ttsStreamSupported = opts.ttsStreamSupported ?? true;
+	const asrStreamSupported = opts.asrStreamSupported ?? false;
+	const vadSupported = opts.vadSupported ?? false;
+	const vadProbs = opts.vadProbs ?? [0];
+	const speakerSupported = opts.speakerSupported ?? false;
+	const diarizSupported = opts.diarizSupported ?? false;
+	let vadIdx = 0;
+	return {
+		libraryPath: "/fake/libelizainference.so",
+		libraryAbiVersion: "3",
+		create: () => 1n,
+		destroy: () => {},
+		mmapAcquire: () => {},
+		mmapEvict: () => {},
+		ttsSynthesize: ({ out }) => {
+			const n = Math.min(ttsSamples, out.length);
+			out.fill(0.1, 0, n);
+			return n;
+		},
+		asrTranscribe: () => transcript,
+		timedAsrSupported: () => false,
+		asrTranscribeTimed: () => ({ text: transcript, words: [] }),
+		ttsStreamSupported: () => ttsStreamSupported,
+		ttsSynthesizeStream: ({ onChunk }) => {
+			const body = new Float32Array(ttsSamples).fill(0.1);
+			const wantCancel = onChunk({
+				pcm: body,
+				isFinal: false,
+			} as TtsStreamChunk);
+			onChunk({ pcm: new Float32Array(0), isFinal: true });
+			return { cancelled: wantCancel === true };
+		},
+		cancelTts: () => {},
+		setVerifierCallback: () => ({ close: () => {} }),
+		encodeReferenceSupported: () => false,
+		vadSupported: () => vadSupported,
+		vadOpen: () => 2n,
+		vadProcess: ({ pcm }) => {
+			if (pcm.length !== 512) throw new Error("fake VAD expected 512 samples");
+			const p = vadProbs[vadIdx] ?? vadProbs[vadProbs.length - 1] ?? 0;
+			vadIdx++;
+			return p;
+		},
+		vadReset: () => {},
+		vadClose: () => {},
+		speakerSupported: () => speakerSupported,
+		speakerOpen: () => 3n,
+		speakerEmbed: () =>
+			(opts.speakerEmbedding ?? new Float32Array(256)).slice(),
+		speakerClose: () => {},
+		diarizSupported: () => diarizSupported,
+		diarizOpen: () => 4n,
+		diarizSegment: () => (opts.diarizLabels ?? new Int8Array(293)).slice(),
+		diarizClose: () => {},
+		asrStreamSupported: () => asrStreamSupported,
+		asrStreamOpen: () => 1n,
+		asrStreamFeed: () => {},
+		asrStreamPartial: () => ({ partial: transcript }),
+		asrStreamFinish: () => ({ partial: transcript }),
+		asrStreamClose: () => {},
+		close: () => {},
+	};
+}

package/src/services/voice/__test-helpers__/synthetic-speech.ts

@@ -0,0 +1,194 @@
+/**
+ * Deterministic-ish speech-like audio generator for VAD/wake-word smoke
+ * tests. Pure synthesis (glottal pulse train through a three-formant
+ * resonator bank with a syllable-rate amplitude envelope and mild f0
+ * jitter) — close enough to real speech in the time/frequency domain that
+ * the Silero VAD reads it as speech, without shipping a recorded WAV.
+ *
+ * `silence + speech + silence` is the canonical smoke fixture: the VAD
+ * should detect exactly one speech segment whose boundaries land inside
+ * the voiced region, and `VadDetector` should drop the leading/trailing
+ * silence windows from its speech-state timeline.
+ */
+
+export interface SpeechFixtureOptions {
+	sampleRate?: number;
+	/** Seconds of leading silence. */
+	leadSilenceSec?: number;
+	/** Seconds of synthesized speech. */
+	speechSec?: number;
+	/** Seconds of trailing silence. */
+	tailSilenceSec?: number;
+	/** Deterministic seed for the f0 jitter. */
+	seed?: number;
+	/**
+	 * Per-speaker voice colour. Two speakers with different timbres have
+	 * measurably different spectral envelopes, so an acoustic diarizer can tell
+	 * them apart from the audio alone. Omit for the default (shared) voice — the
+	 * VAD/wake-word smoke fixtures don't care who is speaking.
+	 */
+	timbre?: SpeakerTimbre;
+}
+
+/** A speaker's voice colour: fundamental frequency + vocal-tract formants. */
+export interface SpeakerTimbre {
+	/** Base fundamental frequency (Hz) — speaker pitch. */
+	f0Hz: number;
+	/** Three `[centerHz, bandwidthHz]` formants — the vocal-tract resonances. */
+	formants: ReadonlyArray<readonly [number, number]>;
+}
+
+export interface SpeechFixture {
+	pcm: Float32Array;
+	sampleRate: number;
+	speechStartSample: number;
+	speechEndSample: number;
+}
+
+function mulberry32(seed: number): () => number {
+	let a = seed >>> 0;
+	return () => {
+		a |= 0;
+		a = (a + 0x6d2b79f5) | 0;
+		let t = Math.imul(a ^ (a >>> 15), 1 | a);
+		t = (t + Math.imul(t ^ (t >>> 7), 61 | t)) ^ t;
+		return ((t ^ (t >>> 14)) >>> 0) / 4294967296;
+	};
+}
+
+/** A three-formant resonator bank state. */
+class FormantBank {
+	private readonly r: number[];
+	private readonly a1: number[];
+	private readonly a2: number[];
+	private readonly z1: number[];
+	private readonly z2: number[];
+	constructor(
+		sampleRate: number,
+		formants: ReadonlyArray<readonly [number, number]>,
+	) {
+		this.r = [];
+		this.a1 = [];
+		this.a2 = [];
+		this.z1 = [];
+		this.z2 = [];
+		for (const [fc, bw] of formants) {
+			const r = Math.exp((-Math.PI * bw) / sampleRate);
+			const theta = (2 * Math.PI * fc) / sampleRate;
+			this.r.push(r);
+			this.a1.push(-2 * r * Math.cos(theta));
+			this.a2.push(r * r);
+			this.z1.push(0);
+			this.z2.push(0);
+		}
+	}
+	step(excitation: number): number {
+		let v = 0;
+		for (let k = 0; k < this.r.length; k++) {
+			const y = excitation - this.a1[k] * this.z1[k] - this.a2[k] * this.z2[k];
+			this.z2[k] = this.z1[k];
+			this.z1[k] = y;
+			v += y * (1 - k * 0.25);
+		}
+		return v;
+	}
+}
+
+const DEFAULT_FORMANTS: ReadonlyArray<readonly [number, number]> = [
+	[700, 80],
+	[1220, 90],
+	[2600, 120],
+];
+
+/** The shared (speaker-agnostic) voice used when no `timbre` is supplied. */
+export const DEFAULT_SPEAKER_TIMBRE: SpeakerTimbre = {
+	f0Hz: 110,
+	formants: DEFAULT_FORMANTS,
+};
+
+/**
+ * Deterministic, distinct voice colour for participant `index` of `count`
+ * speakers in a scenario. The speakers are spread EVENLY across a wide
+ * vocal-tract-length (formant-scaling) and pitch range, so every pair in a
+ * scenario is acoustically far apart — a blind acoustic diarizer can split them
+ * from the audio alone — while one participant always gets one timbre, so the
+ * same speaker clusters together. Spreading by position (not by a label hash)
+ * guarantees the separation; two labels could otherwise hash to near-identical
+ * voices and merge.
+ */
+export function speakerTimbreForIndex(
+	index: number,
+	count: number,
+): SpeakerTimbre {
+	const frac = count <= 1 ? 0.5 : index / (count - 1); // 0..1
+	// Vocal-tract scaling 0.72..1.32 (shorter tract → higher formants).
+	const formantScale = 0.72 + frac * 0.6;
+	// Alternate the second formant up/down so even adjacent slots differ in
+	// formant PATTERN (F2 is the most speaker-discriminative resonance), not just
+	// a global shift.
+	const f2Bias = index % 2 === 0 ? 1.06 : 0.94;
+	const formants = DEFAULT_FORMANTS.map(([fc, bw], i) => {
+		const bias = i === 1 ? f2Bias : 1;
+		return [fc * formantScale * bias, bw] as const;
+	});
+	// Pitch 98..202 Hz.
+	const f0Hz = 98 + frac * 104;
+	return { f0Hz, formants };
+}
+
+/**
+ * The agent's own synthetic TTS voice — a fixed timbre, deliberately placed
+ * outside the speaker-seed range so it is acoustically distinct from every
+ * scenario participant. The corpus synthesizes agent-echo turns with this voice,
+ * and the acoustic self-voice gate enrolls it as the agent's imprint.
+ */
+export const AGENT_VOICE_TIMBRE: SpeakerTimbre = {
+	f0Hz: 250,
+	formants: [
+		[1100, 90],
+		[2400, 110],
+		[3800, 150],
+	],
+};
+
+/** Build a `silence + synthesized speech + silence` PCM buffer. */
+export function makeSpeechWithSilenceFixture(
+	opts: SpeechFixtureOptions = {},
+): SpeechFixture {
+	const sampleRate = opts.sampleRate ?? 16_000;
+	const leadSec = opts.leadSilenceSec ?? 0.5;
+	const speechSec = opts.speechSec ?? 1.2;
+	const tailSec = opts.tailSilenceSec ?? 0.5;
+	const totalSec = leadSec + speechSec + tailSec;
+	const n = Math.floor(totalSec * sampleRate);
+	const pcm = new Float32Array(n);
+	const speechStartSample = Math.floor(leadSec * sampleRate);
+	const speechEndSample = Math.floor((leadSec + speechSec) * sampleRate);
+
+	const rng = mulberry32(opts.seed ?? 0xe11a);
+	const timbre = opts.timbre ?? DEFAULT_SPEAKER_TIMBRE;
+	const bank = new FormantBank(sampleRate, timbre.formants);
+	let phase = 0;
+	for (let i = speechStartSample; i < speechEndSample; i++) {
+		const tInSpeech = (i - speechStartSample) / sampleRate;
+		// Syllable-rate vibrato proportional to the speaker's base pitch (the
+		// original shared voice swung 30 Hz around 110 Hz ≈ ±27%).
+		const f0 =
+			timbre.f0Hz * (1 + 0.27 * Math.sin(2 * Math.PI * 5 * tInSpeech)) +
+			(rng() - 0.5) * 4;
+		phase += f0 / sampleRate;
+		let excitation = 0;
+		if (phase >= 1) {
+			phase -= 1;
+			excitation = 1;
+		}
+		// Syllable-rate amplitude envelope (~4 Hz).
+		const amp = Math.max(
+			0,
+			0.6 * (1 + Math.sin(2 * Math.PI * 4 * tInSpeech - Math.PI / 2)),
+		);
+		excitation *= amp;
+		pcm[i] = bank.step(excitation) * 0.15;
+	}
+	return { pcm, sampleRate, speechStartSample, speechEndSample };
+}

package/src/services/voice/__tests__/checkpoint-manager.test.ts

@@ -0,0 +1,241 @@
+/**
+ * Unit tests for `CheckpointManager` (REST-backed) and
+ * `MockCheckpointManager` (in-memory).
+ *
+ * `CheckpointManager` is verified by replacing the underlying fetch and
+ * asserting the REST URLs / methods. `MockCheckpointManager` is verified
+ * by save/restore/discard round-trips that check the recorded operations
+ * + the snapshot replay semantics.
+ */
+
+import { describe, expect, it } from "vitest";
+import type { CheckpointFetch } from "../../checkpoint-client";
+import {
+	CheckpointHandleInvalidError,
+	CheckpointManager,
+	MockCheckpointManager,
+} from "../checkpoint-manager";
+import { prefillOptimistic } from "../prefill-client";
+
+interface RecordedRequest {
+	url: string;
+	method: string | undefined;
+}
+
+function makeFetch(recorded: RecordedRequest[]): CheckpointFetch {
+	return async (url, init) => {
+		recorded.push({ url: String(url), method: init?.method });
+		return {
+			ok: true,
+			status: 200,
+			statusText: "OK",
+			async text() {
+				return "{}";
+			},
+		};
+	};
+}
+
+function makePrefillFetch(): typeof fetch {
+	const impl = Object.assign(
+		async () => new Response(JSON.stringify({ content: "" }), { status: 200 }),
+		{ preconnect: fetch.preconnect },
+	);
+	return impl as typeof fetch;
+}
+
+describe("CheckpointManager (REST-backed)", () => {
+	it("saveCheckpoint hits POST /slots/<id>/save and returns a live handle", async () => {
+		const recorded: RecordedRequest[] = [];
+		const mgr = new CheckpointManager({
+			baseUrl: "http://127.0.0.1:9999",
+			fetchImpl: makeFetch(recorded),
+			resolveSlotId: () => 3,
+		});
+		const handle = await mgr.saveCheckpoint("conv-1", "pre-draft");
+		expect(handle.slotId).toBe("conv-1");
+		expect(handle.name).toBe("pre-draft");
+		expect(handle.backendRef?.slotId).toBe(3);
+		expect(handle.backendRef?.filename).toMatch(/^C1-conv-1-pre-draft-/);
+		expect(recorded).toHaveLength(1);
+		expect(recorded[0].method).toBe("POST");
+		expect(recorded[0].url).toMatch(/\/slots\/3\/save\?filename=/);
+	});
+
+	it("restoreCheckpoint hits POST /slots/<id>/restore against the saved filename", async () => {
+		const recorded: RecordedRequest[] = [];
+		const mgr = new CheckpointManager({
+			baseUrl: "http://127.0.0.1:9999",
+			fetchImpl: makeFetch(recorded),
+			resolveSlotId: () => 5,
+		});
+		const handle = await mgr.saveCheckpoint("conv-2", "pre-draft");
+		await mgr.restoreCheckpoint(handle);
+		expect(recorded).toHaveLength(2);
+		expect(recorded[1].method).toBe("POST");
+		expect(recorded[1].url).toMatch(/\/slots\/5\/restore\?filename=/);
+		if (!handle.backendRef) {
+			throw new Error("expected checkpoint backend reference");
+		}
+		expect(recorded[1].url).toContain(handle.backendRef.filename);
+	});
+
+	it("discardCheckpoint hits DELETE /slots/<id> and invalidates the handle", async () => {
+		const recorded: RecordedRequest[] = [];
+		const mgr = new CheckpointManager({
+			baseUrl: "http://127.0.0.1:9999",
+			fetchImpl: makeFetch(recorded),
+			resolveSlotId: () => 7,
+		});
+		const handle = await mgr.saveCheckpoint("conv-3", "pre-draft");
+		await mgr.discardCheckpoint(handle);
+		expect(recorded.at(-1)?.method).toBe("DELETE");
+		expect(recorded.at(-1)?.url).toBe("http://127.0.0.1:9999/slots/7");
+		await expect(mgr.restoreCheckpoint(handle)).rejects.toBeInstanceOf(
+			CheckpointHandleInvalidError,
+		);
+		await expect(mgr.discardCheckpoint(handle)).rejects.toBeInstanceOf(
+			CheckpointHandleInvalidError,
+		);
+	});
+
+	it("each save returns a distinct handle even for the same (slotId, name)", async () => {
+		const recorded: RecordedRequest[] = [];
+		const mgr = new CheckpointManager({
+			baseUrl: "http://127.0.0.1:9999",
+			fetchImpl: makeFetch(recorded),
+			resolveSlotId: () => 0,
+		});
+		const a = await mgr.saveCheckpoint("conv-x", "pre-draft");
+		const b = await mgr.saveCheckpoint("conv-x", "pre-draft");
+		expect(a.id).not.toBe(b.id);
+		expect(a.backendRef?.filename).not.toBe(b.backendRef?.filename);
+	});
+
+	it("rejects malformed slotId or checkpoint name", async () => {
+		const mgr = new CheckpointManager({
+			baseUrl: "http://127.0.0.1:9999",
+			fetchImpl: makeFetch([]),
+		});
+		await expect(mgr.saveCheckpoint("", "pre-draft")).rejects.toThrow(
+			/invalid slotId/,
+		);
+		await expect(mgr.saveCheckpoint("conv-1", "../etc/passwd")).rejects.toThrow(
+			/invalid checkpoint name/,
+		);
+	});
+});
+
+describe("MockCheckpointManager", () => {
+	it("records save / restore / discard operations in order", async () => {
+		const mock = new MockCheckpointManager();
+		const handle = await mock.saveCheckpoint("conv-1", "pre-draft");
+		await mock.restoreCheckpoint(handle);
+		await mock.discardCheckpoint(handle);
+		expect(mock.operations).toEqual([
+			{ kind: "save", slotId: "conv-1", name: "pre-draft", handleId: 1 },
+			{ kind: "restore", handleId: 1 },
+			{ kind: "discard", handleId: 1 },
+		]);
+	});
+
+	it("restore replays the snapshot's tokens onto currentTokens", async () => {
+		const mock = new MockCheckpointManager(() => ({
+			tokens: [10, 11, 12],
+			metadata: { partial: "hello wo" },
+		}));
+		const handle = await mock.saveCheckpoint("conv-1", "pre-draft");
+		mock.currentTokens = [99, 99, 99, 99]; // simulate post-pause drafter writes
+		await mock.restoreCheckpoint(handle);
+		expect(mock.currentTokens).toEqual([10, 11, 12]);
+	});
+
+	it("the same handle can be restored more than once (consecutive barge-ins)", async () => {
+		let snapshotCallCount = 0;
+		const mock = new MockCheckpointManager(() => {
+			snapshotCallCount++;
+			return { tokens: [1, 2, 3] };
+		});
+		const handle = await mock.saveCheckpoint("conv-1", "pre-draft");
+		expect(snapshotCallCount).toBe(1);
+		await mock.restoreCheckpoint(handle);
+		await mock.restoreCheckpoint(handle);
+		expect(mock.operations.filter((op) => op.kind === "restore")).toHaveLength(
+			2,
+		);
+		// Handle still live.
+		expect(mock.liveHandleCount()).toBe(1);
+	});
+
+	it("rejects restore/discard for unknown or already-discarded handles", async () => {
+		const mock = new MockCheckpointManager();
+		const handle = await mock.saveCheckpoint("conv-1", "pre-draft");
+		await mock.discardCheckpoint(handle);
+		await expect(mock.restoreCheckpoint(handle)).rejects.toBeInstanceOf(
+			CheckpointHandleInvalidError,
+		);
+		await expect(mock.discardCheckpoint(handle)).rejects.toBeInstanceOf(
+			CheckpointHandleInvalidError,
+		);
+	});
+
+	it("defaults to capturing currentTokens at save time when no snapshotSource is provided", async () => {
+		const mock = new MockCheckpointManager();
+		mock.currentTokens = [7, 8, 9];
+		const handle = await mock.saveCheckpoint("conv-1", "pre-draft");
+		mock.currentTokens = [1];
+		await mock.restoreCheckpoint(handle);
+		expect(mock.currentTokens).toEqual([7, 8, 9]);
+	});
+});
+
+describe("prefillOptimistic", () => {
+	it("delegates to the checkpoint manager and returns the post-prefill handle", async () => {
+		const mock = new MockCheckpointManager(() => ({ tokens: [1, 2, 3] }));
+		const result = await prefillOptimistic(
+			{
+				baseUrl: "http://localhost:8080",
+				slotId: "conv-1",
+				partialText: "hello there",
+				eotProb: 0.7,
+			},
+			{ checkpointManager: mock, fetchImpl: makePrefillFetch() },
+		);
+		expect(result.backend).toBe("slot-save-emulation");
+		expect(result.eotProb).toBe(0.7);
+		expect(result.checkpointHandle.slotId).toBe("conv-1");
+		// The handle should be live and restorable.
+		await mock.restoreCheckpoint(result.checkpointHandle);
+		expect(mock.operations.map((o) => o.kind)).toEqual([
+			"save",
+			"save",
+			"restore",
+		]);
+	});
+
+	it("rejects empty partialText and out-of-range eotProb", async () => {
+		const mock = new MockCheckpointManager();
+		await expect(
+			prefillOptimistic(
+				{
+					baseUrl: "http://localhost:8080",
+					slotId: "conv-1",
+					partialText: "  ",
+					eotProb: 0.5,
+				},
+				{ checkpointManager: mock, fetchImpl: makePrefillFetch() },
+			),
+		).rejects.toThrow(/partialText/);
+		await expect(
+			prefillOptimistic(
+				{
+					baseUrl: "http://localhost:8080",
+					slotId: "conv-1",
+					partialText: "hi",
+					eotProb: 1.5,
+				},
+				{ checkpointManager: mock, fetchImpl: makePrefillFetch() },
+			),
+		).rejects.toThrow(/eotProb/);
+	});
+});