@elizaos/plugin-local-inference 2.0.0-beta.1 → 2.0.3-beta.3

package/src/services/voice/workbench-scenarios.ts

@@ -0,0 +1,407 @@
+/**
+ * Built-in Voice Workbench scenarios + a ground-truth mock services adapter
+ * (#8785).
+ *
+ * The scenario set spans every {@link VoiceScenarioClass} so the headless runner
+ * and the headful spec matrix exercise the whole surface from one source. The
+ * mock adapter echoes each turn's ground truth, so the CI plumbing lane runs the
+ * runner → scorers → report end-to-end and PASSES without any model — separate
+ * from the gated real-backend lane.
+ */
+
+import type { CorpusTurnLabel } from "./corpus-generator";
+import type { VoiceScenario } from "./voice-scenario";
+import type {
+	VoiceTurnObservation,
+	VoiceWorkbenchServices,
+} from "./workbench-headless-runner";
+
+export const VOICE_WORKBENCH_SCENARIOS: VoiceScenario[] = [
+	{
+		id: "multi-voice-greeting",
+		description: "Two distinct voices greet the agent in turn.",
+		classes: ["multi-voice", "diarization"],
+		participants: [
+			{ label: "alice", entityId: "entity-alice", ttsVoiceId: "af_bella" },
+			{ label: "bob", entityId: "entity-bob", ttsVoiceId: "am_adam" },
+		],
+		turns: [
+			{
+				speaker: "alice",
+				text: "Eliza good morning to you",
+				expectRespond: true,
+			},
+			{
+				speaker: "bob",
+				text: "Eliza what is on my calendar",
+				expectRespond: true,
+			},
+		],
+		assertions: { maxWer: 0.2, maxDer: 0.2 },
+	},
+	{
+		id: "respond-vs-bystander",
+		description:
+			"The agent answers a direct address and stays silent on cross-talk.",
+		classes: ["respond-no-respond", "multi-speaker"],
+		// Only the owner is enrolled; bob is a recognized-but-unknown bystander.
+		knownSpeakerEntityIds: ["entity-alice"],
+		participants: [
+			{ label: "alice", entityId: "entity-alice", isOwner: true },
+			{ label: "bob", entityId: "entity-bob" },
+		],
+		turns: [
+			{
+				speaker: "alice",
+				text: "Eliza set a timer for ten minutes",
+				expectRespond: true,
+			},
+			{
+				speaker: "bob",
+				text: "hey alice did you see the game",
+				expectRespond: false,
+			},
+			{
+				speaker: "alice",
+				text: "Eliza thanks that is all",
+				expectRespond: true,
+			},
+		],
+		assertions: { minRespondAccuracy: 0.9 },
+	},
+	{
+		id: "pauses-midutterance",
+		description: "A slow speaker pauses mid-sentence; EOT must not jump in.",
+		classes: ["pauses", "eot"],
+		participants: [{ label: "alice", entityId: "entity-alice" }],
+		turns: [
+			{
+				speaker: "alice",
+				text: "Eliza schedule a meeting with",
+				expectRespond: false,
+				expectEndOfTurn: false,
+				pausesMs: [1200],
+			},
+			{ speaker: "alice", text: "Bob tomorrow at noon", expectRespond: true },
+		],
+		assertions: { maxEotFalseTriggerRate: 0.2 },
+	},
+	{
+		id: "entity-from-speech",
+		description: "Name inference from the live transcript creates an entity.",
+		classes: ["entity-extraction", "voice-recognition"],
+		participants: [{ label: "jill", entityId: "entity-jill", isOwner: true }],
+		turns: [
+			{
+				speaker: "jill",
+				text: "Eliza I am Jill and this is my house",
+				expectRespond: true,
+				expectedEntity: "entity-jill",
+			},
+		],
+		assertions: { minVoiceEntityMatchRate: 0.9 },
+	},
+	{
+		id: "transcription-mode-dictation",
+		description: "Long-form dictation lands silently in transcription mode.",
+		classes: ["transcription-mode", "long-form-monologue"],
+		participants: [{ label: "alice", entityId: "entity-alice" }],
+		turns: [
+			{
+				speaker: "alice",
+				text: `${"I want to capture this thought for later. ".repeat(12)}`.trim(),
+				expectRespond: false,
+			},
+		],
+	},
+	{
+		id: "multi-agent-room-address",
+		description: "In a room with two agents, only the addressed agent replies.",
+		classes: ["multi-agent-room", "respond-no-respond"],
+		participants: [
+			{ label: "owner", entityId: "entity-owner", isOwner: true },
+			{ label: "eliza" },
+			{ label: "aria" },
+		],
+		agents: ["eliza", "aria"],
+		turns: [
+			{
+				speaker: "owner",
+				text: "Eliza what is the weather",
+				expectRespond: true,
+			},
+			{ speaker: "owner", text: "Aria play some music", expectRespond: true },
+		],
+		assertions: { minRespondAccuracy: 0.9 },
+	},
+	{
+		id: "noisy-room-commands",
+		description:
+			"The owner gives commands in a noisy, reverberant room; the agent still answers.",
+		classes: ["robustness", "respond-no-respond"],
+		knownSpeakerEntityIds: ["entity-owner"],
+		// 8 dB SNR room noise + light reverb across the whole scenario.
+		environment: { noiseSnrDb: 8, noiseKind: "pink", reverb: 0.35, seed: 101 },
+		participants: [{ label: "owner", entityId: "entity-owner", isOwner: true }],
+		turns: [
+			{
+				speaker: "owner",
+				text: "Eliza turn on the kitchen lights",
+				expectRespond: true,
+			},
+			{ speaker: "owner", text: "Eliza what time is it", expectRespond: true },
+		],
+		assertions: { minRespondAccuracy: 0.9, maxWer: 0.35, maxDer: 0.3 },
+	},
+	{
+		id: "music-background-commands",
+		description:
+			"The owner gives commands with music playing in the background; the agent answers and the tonal music bed does not false-trigger VAD/respond.",
+		classes: ["robustness", "respond-no-respond"],
+		knownSpeakerEntityIds: ["entity-owner"],
+		// 10 dB SNR music bed across the whole scenario. The `music` noise kind is
+		// tonal/harmonic (unlike white/pink) — this scenario exercises the music
+		// augmentation that landed in corpus-augment.ts (d56efcc80b) but until now
+		// had no scenario driving it (#9147).
+		environment: { noiseSnrDb: 10, noiseKind: "music", reverb: 0.2, seed: 404 },
+		participants: [{ label: "owner", entityId: "entity-owner", isOwner: true }],
+		turns: [
+			{ speaker: "owner", text: "Eliza pause the timer", expectRespond: true },
+			{
+				speaker: "owner",
+				text: "Eliza what is the weather today",
+				expectRespond: true,
+			},
+		],
+		assertions: { minRespondAccuracy: 0.9, maxWer: 0.35, maxDer: 0.3 },
+	},
+	{
+		id: "far-field-reverb",
+		description:
+			"A far, reverberant speaker across the room — quiet and washed out — is still understood.",
+		classes: ["robustness", "respond-no-respond"],
+		knownSpeakerEntityIds: ["entity-owner"],
+		environment: { farFieldDb: 12, reverb: 0.75, noiseSnrDb: 12, seed: 202 },
+		participants: [{ label: "owner", entityId: "entity-owner", isOwner: true }],
+		turns: [
+			{
+				speaker: "owner",
+				text: "Eliza add milk to the shopping list",
+				expectRespond: true,
+			},
+		],
+		assertions: { minRespondAccuracy: 0.9, maxWer: 0.4 },
+	},
+	{
+		id: "background-talkers",
+		description:
+			"Other people are talking in the background while the owner addresses the agent.",
+		classes: ["robustness", "overlapping-speech", "multi-speaker"],
+		knownSpeakerEntityIds: ["entity-owner"],
+		environment: { backgroundTalkersDb: 9, noiseSnrDb: 14, seed: 303 },
+		participants: [{ label: "owner", entityId: "entity-owner", isOwner: true }],
+		turns: [
+			{
+				speaker: "owner",
+				text: "Eliza start a five minute timer",
+				expectRespond: true,
+			},
+		],
+		assertions: { minRespondAccuracy: 0.9, maxWer: 0.4 },
+	},
+	{
+		id: "echo-self-trigger",
+		description:
+			"The agent's own reply bleeds back into the mic; it must not answer itself.",
+		classes: ["echo-rejection", "respond-no-respond"],
+		knownSpeakerEntityIds: ["entity-owner"],
+		participants: [{ label: "owner", entityId: "entity-owner", isOwner: true }],
+		turns: [
+			{
+				speaker: "owner",
+				text: "hey Eliza what is the weather today",
+				expectRespond: true,
+				agentReplyText:
+					"It is sunny and seventy two degrees in San Francisco today",
+			},
+			{
+				// The agent's TTS echoed back through the open mic — NOT a user turn.
+				speaker: "owner",
+				text: "It is sunny and seventy two degrees in San Francisco today",
+				isAgentEcho: true,
+				expectRespond: false,
+			},
+			{ speaker: "owner", text: "hey Eliza thanks", expectRespond: true },
+		],
+		assertions: { minEchoRejectionRate: 1, minRespondAccuracy: 0.9 },
+	},
+	{
+		id: "multi-speaker-name-capture",
+		description:
+			"Two new people in the room introduce themselves; each name is captured and bound to a distinct entity, separate from the owner.",
+		classes: [
+			"diarization",
+			"entity-extraction",
+			"multi-speaker",
+			"voice-recognition",
+		],
+		knownSpeakerEntityIds: ["entity-owner", "entity-marcus", "entity-priya"],
+		participants: [
+			{ label: "owner", entityId: "entity-owner", isOwner: true },
+			{ label: "marcus", entityId: "entity-marcus" },
+			{ label: "priya", entityId: "entity-priya" },
+		],
+		turns: [
+			{
+				speaker: "owner",
+				text: "Eliza meet my two coworkers",
+				expectRespond: true,
+			},
+			{
+				speaker: "marcus",
+				text: "hey Eliza I am Marcus good to meet you",
+				expectRespond: true,
+				expectedEntity: "entity-marcus",
+			},
+			{
+				speaker: "priya",
+				text: "Eliza I am Priya from the design team",
+				expectRespond: true,
+				expectedEntity: "entity-priya",
+			},
+			{
+				speaker: "owner",
+				text: "Eliza remember them for later",
+				expectRespond: true,
+			},
+		],
+		assertions: {
+			minRespondAccuracy: 0.9,
+			maxDer: 0.2,
+			minVoiceEntityMatchRate: 0.9,
+		},
+	},
+	{
+		id: "echo-mistranscribed",
+		description:
+			"The agent's echo is mis-transcribed (no word overlap); the ACOUSTIC self-voice gate still rejects it.",
+		classes: ["echo-rejection"],
+		knownSpeakerEntityIds: ["entity-owner"],
+		participants: [{ label: "owner", entityId: "entity-owner", isOwner: true }],
+		turns: [
+			{
+				speaker: "owner",
+				text: "hey Eliza how many days until friday",
+				expectRespond: true,
+				agentReplyText: "There are three days until Friday",
+			},
+			{
+				// ASR garbled the echoed reply — NO words overlap "three days until
+				// friday", so the transcript guard would miss it. Self-voice catches it.
+				speaker: "owner",
+				text: "thudder ays untl fryeday",
+				isAgentEcho: true,
+				expectRespond: false,
+			},
+		],
+		assertions: { minEchoRejectionRate: 1 },
+	},
+	{
+		id: "owner-enrollment-inference",
+		description:
+			"No owner is enrolled; the agent infers the owner from who speaks to it most.",
+		classes: ["owner-security", "voice-recognition"],
+		knownSpeakerEntityIds: ["entity-owner", "entity-guest"],
+		participants: [
+			{ label: "owner", entityId: "entity-owner", isOwner: true },
+			{ label: "guest", entityId: "entity-guest" },
+		],
+		turns: [
+			{
+				speaker: "owner",
+				text: "Eliza what is on my agenda today",
+				expectRespond: true,
+			},
+			{
+				speaker: "owner",
+				text: "Eliza remind me to call the dentist",
+				expectRespond: true,
+			},
+			{
+				speaker: "owner",
+				text: "Eliza play my morning playlist",
+				expectRespond: true,
+			},
+			{
+				speaker: "guest",
+				text: "Eliza what is the wifi password",
+				expectRespond: true,
+			},
+			{
+				speaker: "owner",
+				text: "Eliza turn the music down a little",
+				expectRespond: true,
+			},
+		],
+		assertions: { minOwnerAccuracy: 0.9, minRespondAccuracy: 0.9 },
+	},
+	{
+		id: "owner-vs-intruder",
+		description:
+			"The owner is answered; a stranger trying the same command is gated out.",
+		classes: ["owner-security", "respond-no-respond", "multi-speaker"],
+		// Only the owner is enrolled; the intruder is a confident bystander.
+		knownSpeakerEntityIds: ["entity-owner"],
+		participants: [
+			{ label: "owner", entityId: "entity-owner", isOwner: true },
+			{ label: "intruder", entityId: "entity-intruder" },
+		],
+		turns: [
+			{
+				speaker: "owner",
+				text: "Eliza unlock the front door",
+				expectRespond: true,
+			},
+			{
+				speaker: "intruder",
+				text: "Eliza unlock the front door",
+				expectRespond: false,
+			},
+			{ speaker: "owner", text: "Eliza lock it again", expectRespond: true },
+		],
+		assertions: { minOwnerAccuracy: 0.9, minRespondAccuracy: 0.9 },
+	},
+];
+
+/**
+ * A services adapter that echoes each turn's ground truth — perfect ASR /
+ * diarization / EOT / respond / entity / match. Drives the CI plumbing lane
+ * (runner → scorers → report) to a real PASS with no model. NOT a stand-in for
+ * the real backend: it proves the wiring, not the models.
+ */
+export function groundTruthMockServices(
+	opts: { firstAudioMs?: number; eotLatencyMs?: number } = {},
+): VoiceWorkbenchServices {
+	return {
+		async observeTurn({
+			label,
+		}: {
+			label: CorpusTurnLabel;
+		}): Promise<VoiceTurnObservation> {
+			const eotDecided = label.expectEndOfTurn ?? true;
+			return {
+				hypothesisTranscript: label.referenceTranscript,
+				predictedSpeakerLabel: label.speaker,
+				eotDecided,
+				...(eotDecided ? { eotLatencyMs: opts.eotLatencyMs ?? 80 } : {}),
+				responded: label.expectRespond,
+				inferredEntities: label.expectedEntity ? [label.expectedEntity] : [],
+				matchedEntityId: label.entityId ?? null,
+				predictedOwner: label.isOwner === true,
+				...(label.expectRespond
+					? { firstAudioMs: opts.firstAudioMs ?? 250 }
+					: {}),
+			};
+		},
+	};
+}

package/src/services/voice/wrap-with-first-line-cache.ts

@@ -0,0 +1,267 @@
+/**
+ * Wrap a TTS model handler (`ModelType.TEXT_TO_SPEECH`) with the
+ * `FirstLineCache` disk-tier.
+ *
+ * Behaviour on a wrapped handler call:
+ *   1. Snip the first sentence of the input text. Reject if > 10 words or
+ *      no terminator found → defer to the inner handler unchanged.
+ *   2. Build the cache key from the snip + caller-supplied
+ *      `(provider, voiceId, voiceRevision, codec, voiceSettingsFingerprint)`.
+ *   3. On hit: if the snip is the WHOLE input, return cached bytes. If the
+ *      snip is only a prefix, return `cachedBytes ++ synthesize(remainder)`.
+ *      The concat path is safe for the codecs we cache today (mp3, opus,
+ *      ogg are self-framed; wav/pcm_f32 are not, and the wrapper falls
+ *      through for those).
+ *   4. On miss: call the inner handler with the full input, return that.
+ *      In the background, call the inner handler with JUST the snip to get
+ *      a cleanly-framed cache entry (no mid-stream-byte alignment hazards).
+ *
+ * The wrapper is provider-agnostic — `WrapOptions.resolveContext` is the
+ * single seam where each TTS plugin teaches the cache its provider name +
+ * voice id + revision + voice-settings fingerprint.
+ */
+
+import type { IAgentRuntime } from "@elizaos/core";
+import { logger } from "@elizaos/core";
+import {
+	FIRST_SENTENCE_SNIP_VERSION,
+	type FirstSentenceSnipResult,
+	firstSentenceSnip,
+} from "@elizaos/shared";
+import {
+	type FirstLineCache,
+	type FirstLineCacheEntry,
+	type FirstLineCacheKey,
+	getSharedFirstLineCache,
+	type PutInput,
+} from "./first-line-cache";
+
+// ---------------------------------------------------------------------------
+// Types
+// ---------------------------------------------------------------------------
+
+export type TtsBytes = Uint8Array | ArrayBuffer | Buffer;
+export type TtsHandlerInput = string | { text: string; [k: string]: unknown };
+export type TtsHandlerOutput = TtsBytes;
+export type TtsHandler = (
+	runtime: IAgentRuntime,
+	input: TtsHandlerInput,
+) => Promise<TtsHandlerOutput>;
+
+/** Caller-supplied context resolver — provider name + voice metadata. */
+export interface TtsResolvedContext {
+	provider: string;
+	voiceId: string;
+	voiceRevision: string;
+	codec: FirstLineCacheKey["codec"];
+	contentType: string;
+	sampleRate: number;
+	voiceSettingsFingerprint: string;
+	/** Optional: if true, bypass the cache entirely on this call. */
+	bypass?: boolean;
+}
+
+export interface WrapOptions {
+	/** Inject for tests. */
+	cache?: FirstLineCache;
+	/**
+	 * Resolve the per-call provider/voice context. Must be cheap (typically
+	 * a settings read + a small sha256). Return `null` to bypass the cache.
+	 */
+	resolveContext: (
+		runtime: IAgentRuntime,
+		input: TtsHandlerInput,
+	) => Promise<TtsResolvedContext | null> | TtsResolvedContext | null;
+	/**
+	 * Whether to attempt to concatenate cached bytes + remainder synthesis.
+	 * Defaults to true for mp3/opus/ogg, false for wav/pcm_f32 (concat would
+	 * corrupt the RIFF / raw stream).
+	 */
+	concatRemainder?: boolean;
+	/** Optional pre-resolved fingerprint of voiceSettings; rarely needed. */
+	enableCachePopulation?: boolean;
+}
+
+// ---------------------------------------------------------------------------
+// Helpers
+// ---------------------------------------------------------------------------
+
+function bytesToUint8Array(b: TtsHandlerOutput): Uint8Array {
+	if (b instanceof Uint8Array) return b;
+	if (b instanceof ArrayBuffer) return new Uint8Array(b);
+	// Buffer is a Uint8Array subclass so the first branch already covers it.
+	return new Uint8Array(b as ArrayBufferLike);
+}
+
+function extractText(input: TtsHandlerInput): string {
+	return typeof input === "string" ? input : input.text;
+}
+
+function withText(input: TtsHandlerInput, text: string): TtsHandlerInput {
+	if (typeof input === "string") return text;
+	return { ...input, text };
+}
+
+function concatU8(a: Uint8Array, b: Uint8Array): Uint8Array {
+	const out = new Uint8Array(a.length + b.length);
+	out.set(a, 0);
+	out.set(b, a.length);
+	return out;
+}
+
+const NEVER_CONCAT_CODECS: ReadonlySet<FirstLineCacheKey["codec"]> = new Set([
+	"wav",
+	"pcm_f32",
+	"flac",
+]);
+
+function canConcat(codec: FirstLineCacheKey["codec"]): boolean {
+	return !NEVER_CONCAT_CODECS.has(codec);
+}
+
+// ---------------------------------------------------------------------------
+// Wrapper
+// ---------------------------------------------------------------------------
+
+/**
+ * Wrap a TTS handler with first-sentence caching. Returns a new handler with
+ * the same signature.
+ *
+ * The wrapper is safe to apply at runtime registration time — if the cache
+ * is disabled (env / no sqlite), the wrapper short-circuits to the inner
+ * handler with zero overhead beyond a snip attempt.
+ */
+export function wrapWithFirstLineCache(
+	inner: TtsHandler,
+	options: WrapOptions,
+): TtsHandler {
+	const cache = options.cache ?? getSharedFirstLineCache();
+
+	return async function cachedTtsHandler(
+		runtime: IAgentRuntime,
+		input: TtsHandlerInput,
+	): Promise<TtsHandlerOutput> {
+		if (!cache.isEnabled) {
+			return inner(runtime, input);
+		}
+
+		const text = extractText(input);
+		if (!text) return inner(runtime, input);
+
+		const snip = firstSentenceSnip(text);
+		if (!snip) return inner(runtime, input);
+
+		const ctx = await options.resolveContext(runtime, input);
+		if (!ctx || ctx.bypass) return inner(runtime, input);
+		if (!ctx.voiceId || !ctx.voiceRevision) return inner(runtime, input);
+
+		const key: FirstLineCacheKey = {
+			algoVersion: FIRST_SENTENCE_SNIP_VERSION,
+			provider: ctx.provider,
+			voiceId: ctx.voiceId,
+			voiceRevision: ctx.voiceRevision,
+			sampleRate: ctx.sampleRate,
+			codec: ctx.codec,
+			voiceSettingsFingerprint: ctx.voiceSettingsFingerprint,
+			normalizedText: snip.normalized,
+		};
+
+		const hit = cache.get(key);
+		if (hit) {
+			return assembleFromHit({
+				hit,
+				snip,
+				originalText: text,
+				input,
+				inner,
+				runtime,
+				options,
+				codec: ctx.codec,
+			});
+		}
+
+		// MISS — synthesize fresh + populate the cache in background.
+		const fullBytes = await inner(runtime, input);
+		const _fullU8 = bytesToUint8Array(fullBytes);
+
+		if (options.enableCachePopulation !== false) {
+			schedulePopulate({
+				cache,
+				inner,
+				runtime,
+				input,
+				snip,
+				key,
+				ctx,
+			});
+		}
+
+		return fullBytes;
+	};
+}
+
+interface AssembleArgs {
+	hit: FirstLineCacheEntry;
+	snip: FirstSentenceSnipResult;
+	originalText: string;
+	input: TtsHandlerInput;
+	inner: TtsHandler;
+	runtime: IAgentRuntime;
+	options: WrapOptions;
+	codec: FirstLineCacheKey["codec"];
+}
+
+async function assembleFromHit(args: AssembleArgs): Promise<TtsHandlerOutput> {
+	const { hit, snip, originalText, input, inner, runtime, options, codec } =
+		args;
+	const remainder = originalText.slice(snip.endOffset).trimStart();
+	const concatEnabled = options.concatRemainder ?? canConcat(codec);
+
+	if (!remainder) return hit.bytes;
+	if (!concatEnabled) {
+		// Codec we can't safely concat (wav/pcm). Fall through to full synth.
+		return inner(runtime, input);
+	}
+	const remainderBytes = await inner(runtime, withText(input, remainder));
+	return concatU8(hit.bytes, bytesToUint8Array(remainderBytes));
+}
+
+interface PopulateArgs {
+	cache: FirstLineCache;
+	inner: TtsHandler;
+	runtime: IAgentRuntime;
+	input: TtsHandlerInput;
+	snip: FirstSentenceSnipResult;
+	key: FirstLineCacheKey;
+	ctx: TtsResolvedContext;
+}
+
+function schedulePopulate(args: PopulateArgs): void {
+	// Fire-and-forget. If the dedicated synthesis fails, we leave the cache
+	// empty and try again next call.
+	void (async () => {
+		try {
+			if (args.cache.has(args.key)) return;
+			const snipBytes = await args.inner(
+				args.runtime,
+				withText(args.input, args.snip.raw),
+			);
+			const u8 = bytesToUint8Array(snipBytes);
+			if (u8.length === 0) return;
+
+			const put: PutInput = {
+				...args.key,
+				bytes: u8,
+				rawText: args.snip.raw,
+				contentType: args.ctx.contentType,
+				durationMs: 0,
+				wordCount: args.snip.wordCount,
+			};
+			args.cache.put(put);
+		} catch (err) {
+			logger.debug(
+				`[tts-cache] first-line populate failed for "${args.snip.normalized}": ${err instanceof Error ? err.message : String(err)}`,
+			);
+		}
+	})();
+}