@elizaos/plugin-local-inference 2.0.0-beta.1 → 2.0.11-beta.7

package/src/services/voice/samantha-preset-regenerator.ts

@@ -0,0 +1,393 @@
+/**
+ * Samantha-preset on-the-fly regeneration via the fused OmniVoice FFI.
+ *
+ * Path A from W3-11: when the bundle ships the I-wave zero-fill placeholder
+ * for `cache/voice-preset-default.bin`, the runtime synthesises a real
+ * preset by encoding the bundled Samantha reference clip through the FFI's
+ * `eliza_inference_encode_reference` entrypoint and writing the resulting
+ * `ref_audio_tokens` + canonical instruct/refText into a v2 preset blob.
+ *
+ * Determinism contract:
+ *   - The reference clip bytes (24 kHz mono fp32 WAV) are pinned in the
+ *     bundle.
+ *   - The reference transcript (`SAMANTHA_REFERENCE_TRANSCRIPT`) is pinned.
+ *   - The instruct string (`SAMANTHA_INSTRUCT`) is pinned.
+ *   - The OmniVoice encode entrypoint does not consume randomness (the
+ *     HuBERT semantic + RVQ codec passes are pure functions of the input
+ *     PCM + the model weights).
+ * Therefore the produced preset bytes are reproducible byte-for-byte across
+ * boots given the same FFI library + bundle.
+ */
+
+import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
+import path from "node:path";
+import {
+	type ElizaInferenceContextHandle,
+	type ElizaInferenceFfi,
+	loadElizaInferenceFfi,
+} from "./ffi-bindings";
+import {
+	detectSamanthaPlaceholder,
+	SAMANTHA_INSTRUCT,
+	SAMANTHA_REFERENCE_TRANSCRIPT,
+} from "./samantha-preset-placeholder";
+import { writeVoicePresetFileV2 } from "./voice-preset-format";
+
+/** Outcome of `ensureSamanthaPresetReady`. Distinct kinds let the caller
+ *  log the right thing at the right level without re-doing detection. */
+export type EnsureSamanthaPresetOutcome =
+	| { kind: "real-preset" }
+	| { kind: "missing-bundle-preset" }
+	| { kind: "regenerated"; bytes: number; K: number; refT: number }
+	| {
+			kind: "placeholder-no-regen";
+			reason:
+				| "missing-reference-wav"
+				| "missing-ffi-library"
+				| "ffi-no-encode-reference"
+				| "encode-reference-failed";
+			detail: string;
+	  };
+
+export interface RegenerateOptions {
+	bundleRoot: string;
+	/** Absolute path the regenerated preset bytes should target. The caller
+	 *  performs the write — this function only produces the bytes + metadata. */
+	presetPath: string;
+	/** Override path to the Samantha reference WAV. Defaults to the bundle's
+	 *  `tts/omnivoice/samantha-ref.wav`. */
+	referenceWav?: string;
+	/** Override the canonical reference transcript. Defaults to the pinned
+	 *  `SAMANTHA_REFERENCE_TRANSCRIPT`. */
+	referenceText?: string;
+}
+
+export interface RegenerateResult {
+	bytes: Uint8Array;
+	K: number;
+	refT: number;
+	embeddingDim: number;
+}
+
+/**
+ * Platform-specific filenames probed when locating the OmniVoice fused
+ * shared library inside a bundle. Mirrors the matching helper inside
+ * `engine-bridge.ts` (kept private there); regenerator and bridge resolve
+ * the same set of names so a bundle that loads at boot also loads at
+ * regeneration time.
+ */
+function libraryFilenames(): string[] {
+	if (process.platform === "darwin") return ["libelizainference.dylib"];
+	if (process.platform === "win32") {
+		return ["elizainference.dll", "libelizainference.dll"];
+	}
+	return ["libelizainference.so"];
+}
+
+function locateBundleLibrary(bundleRoot: string): string {
+	const exact = process.env.ELIZA_INFERENCE_LIBRARY?.trim();
+	if (exact && existsSync(exact)) return exact;
+	const dirs = [
+		path.join(bundleRoot, "lib"),
+		exact ? path.dirname(exact) : null,
+		process.env.ELIZA_INFERENCE_LIB_DIR?.trim() || null,
+	].filter((dir): dir is string => Boolean(dir));
+	for (const dir of dirs) {
+		for (const name of libraryFilenames()) {
+			const candidate = path.join(dir, name);
+			if (existsSync(candidate)) return candidate;
+		}
+	}
+	return path.join(
+		dirs[0] ?? path.join(bundleRoot, "lib"),
+		libraryFilenames()[0] ?? "libelizainference.so",
+	);
+}
+
+/**
+ * Decode a 24 kHz mono Float32 LE WAV file into a Float32Array of PCM
+ * samples. Refuses anything that is not the canonical OmniVoice reference
+ * format — encoders happily accept stereo / 16-bit / 48 kHz inputs and
+ * silently degrade, which is exactly the kind of fallback sludge AGENTS.md
+ * §3 forbids. We require the file be in the right format up front.
+ */
+export function decodeMonoFloat32Wav24kHz(bytes: Uint8Array): Float32Array {
+	if (bytes.byteLength < 44) {
+		throw new Error(
+			`[samantha-regen] reference WAV too small (${bytes.byteLength} bytes, need >= 44 for header)`,
+		);
+	}
+	const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);
+	const riff = String.fromCharCode(...bytes.subarray(0, 4));
+	const wave = String.fromCharCode(...bytes.subarray(8, 12));
+	if (riff !== "RIFF" || wave !== "WAVE") {
+		throw new Error(
+			`[samantha-regen] reference WAV bad magic: RIFF=${JSON.stringify(riff)} WAVE=${JSON.stringify(wave)}`,
+		);
+	}
+
+	// Walk chunks for "fmt " + "data".
+	let cursor = 12;
+	let fmtOffset = -1;
+	let fmtLen = 0;
+	let dataOffset = -1;
+	let dataLen = 0;
+	while (cursor + 8 <= bytes.byteLength) {
+		const id = String.fromCharCode(...bytes.subarray(cursor, cursor + 4));
+		const size = view.getUint32(cursor + 4, true);
+		const payload = cursor + 8;
+		if (id === "fmt ") {
+			fmtOffset = payload;
+			fmtLen = size;
+		} else if (id === "data") {
+			dataOffset = payload;
+			dataLen = size;
+			break;
+		}
+		cursor = payload + size + (size % 2); // pad byte
+	}
+	if (fmtOffset < 0 || dataOffset < 0) {
+		throw new Error("[samantha-regen] reference WAV missing fmt or data chunk");
+	}
+	const audioFormat = view.getUint16(fmtOffset + 0, true);
+	const channels = view.getUint16(fmtOffset + 2, true);
+	const sampleRate = view.getUint32(fmtOffset + 4, true);
+	const bitsPerSample = view.getUint16(fmtOffset + 14, true);
+
+	// Accept WAVE_FORMAT_IEEE_FLOAT (3) or WAVE_FORMAT_EXTENSIBLE (0xFFFE)
+	// with 32-bit float samples.
+	const isFloat =
+		(audioFormat === 3 && bitsPerSample === 32) ||
+		(audioFormat === 0xfffe && bitsPerSample === 32 && fmtLen >= 40);
+	if (!isFloat) {
+		throw new Error(
+			`[samantha-regen] reference WAV must be 32-bit float PCM (got format=${audioFormat}, bps=${bitsPerSample})`,
+		);
+	}
+	if (channels !== 1) {
+		throw new Error(
+			`[samantha-regen] reference WAV must be mono (got ${channels} channels)`,
+		);
+	}
+	if (sampleRate !== 24_000) {
+		throw new Error(
+			`[samantha-regen] reference WAV must be 24 kHz (got ${sampleRate})`,
+		);
+	}
+
+	const sampleCount = Math.floor(dataLen / 4);
+	// Copy into an aligned buffer (the input slice is not guaranteed
+	// 4-aligned; Float32Array constructor requires alignment).
+	const aligned = new Uint8Array(sampleCount * 4);
+	aligned.set(bytes.subarray(dataOffset, dataOffset + sampleCount * 4));
+	return new Float32Array(aligned.buffer);
+}
+
+/**
+ * Run the on-the-fly regeneration. Loads the bundle's OmniVoice FFI, calls
+ * `encodeReference` against the Samantha reference clip, and serialises the
+ * result into an ELZ1 v2 preset blob. The caller writes the bytes to disk.
+ */
+export async function regenerateSamanthaPresetFromBundle(
+	opts: RegenerateOptions,
+): Promise<RegenerateResult> {
+	const refWav =
+		opts.referenceWav ??
+		path.join(opts.bundleRoot, "tts", "omnivoice", "samantha-ref.wav");
+	if (!existsSync(refWav)) {
+		throw new Error(
+			`[samantha-regen] Samantha reference WAV not found at ${refWav}. The bundle is missing the OmniVoice samantha reference clip.`,
+		);
+	}
+
+	const libPath = locateBundleLibrary(opts.bundleRoot);
+	if (!existsSync(libPath)) {
+		throw new Error(
+			`[samantha-regen] OmniVoice FFI library not found under ${path.join(
+				opts.bundleRoot,
+				"lib",
+			)} (tried ${libraryFilenames().join(", ")}). Build via packages/app-core/scripts/build-llama-cpp-mtp.mjs (omnivoice-merged target).`,
+		);
+	}
+
+	const ffi: ElizaInferenceFfi = loadElizaInferenceFfi(libPath);
+	let ctx: ElizaInferenceContextHandle | null = null;
+	let ttsAcquired = false;
+	try {
+		if (
+			typeof ffi.encodeReferenceSupported !== "function" ||
+			!ffi.encodeReferenceSupported()
+		) {
+			throw new Error(
+				"[samantha-regen] this OmniVoice build does not export eliza_inference_encode_reference (ABI v4 required). Rebuild with the encode-reference target.",
+			);
+		}
+		if (typeof ffi.encodeReference !== "function") {
+			throw new Error(
+				"[samantha-regen] FFI binding missing encodeReference method despite encodeReferenceSupported()=true",
+			);
+		}
+
+		ctx = ffi.create(opts.bundleRoot);
+		ffi.mmapAcquire(ctx, "tts");
+		ttsAcquired = true;
+
+		const wavBytes = new Uint8Array(readFileSync(refWav));
+		const pcm = decodeMonoFloat32Wav24kHz(wavBytes);
+
+		const encoded = ffi.encodeReference({
+			ctx,
+			pcm,
+			sampleRateHz: 24_000,
+		});
+		if (encoded.K <= 0 || encoded.refT <= 0) {
+			throw new Error(
+				`[samantha-regen] encode_reference returned empty tensor (K=${encoded.K}, refT=${encoded.refT})`,
+			);
+		}
+
+		// The FFI encode pass produces ref_audio_tokens; the speaker
+		// embedding section stays empty (OmniVoice resolves the speaker
+		// identity from the tokens, not from a separate embedding vector).
+		const embedding = new Float32Array(0);
+		const refText = opts.referenceText ?? SAMANTHA_REFERENCE_TRANSCRIPT;
+		const instruct = SAMANTHA_INSTRUCT;
+		const metadata: Record<string, unknown> = {
+			generator: "samantha-preset-regenerator",
+			generatorVersion: 1,
+			referenceWavPath: path.basename(refWav),
+			referenceWavBytes: wavBytes.byteLength,
+			referenceText: refText,
+			instruct,
+			K: encoded.K,
+			refT: encoded.refT,
+		};
+
+		const bytes = writeVoicePresetFileV2({
+			embedding,
+			phrases: [],
+			refAudioTokens: {
+				K: encoded.K,
+				refT: encoded.refT,
+				tokens: encoded.tokens,
+			},
+			refText,
+			instruct,
+			metadata,
+		});
+
+		return {
+			bytes,
+			K: encoded.K,
+			refT: encoded.refT,
+			embeddingDim: 0,
+		};
+	} finally {
+		if (ctx !== null) {
+			if (ttsAcquired) {
+				try {
+					ffi.mmapEvict(ctx, "tts");
+				} catch {
+					// Evict is best-effort during regeneration; destroy below
+					// tears down the context either way.
+				}
+			}
+			try {
+				ffi.destroy(ctx);
+			} catch {
+				// Destroy is best-effort during regeneration; the OS reclaims
+				// the context on process exit.
+			}
+		}
+		try {
+			ffi.close();
+		} catch {
+			// Same — close is best-effort.
+		}
+	}
+}
+
+/**
+ * Pre-flight: detect a placeholder preset at the bundle's canonical path
+ * and regenerate it via OmniVoice when possible. Called by the engine's
+ * `ensureActiveBundleVoiceReady()` before the synchronous preset load.
+ *
+ * Outcomes:
+ *
+ *   - `real-preset`            — nothing to do; the file is a real preset.
+ *   - `missing-bundle-preset`  — file does not exist; the engine's existing
+ *                                error path runs (loud failure).
+ *   - `regenerated`            — preset bytes were generated and written.
+ *   - `placeholder-no-regen`   — placeholder detected but regen could not
+ *                                run (FFI missing, reference clip missing,
+ *                                etc.). Returned for the caller to log; the
+ *                                engine then falls through to the bundled
+ *                                Kokoro default voice.
+ */
+export async function ensureSamanthaPresetReady(
+	bundleRoot: string,
+): Promise<EnsureSamanthaPresetOutcome> {
+	const presetPath = path.join(bundleRoot, "cache", "voice-preset-default.bin");
+	const state = detectSamanthaPlaceholder(presetPath);
+
+	if (state.kind === "missing") {
+		return { kind: "missing-bundle-preset" };
+	}
+	if (state.kind === "real-preset") {
+		return { kind: "real-preset" };
+	}
+	if (state.kind === "unreadable") {
+		return {
+			kind: "placeholder-no-regen",
+			reason: "missing-ffi-library", // closest match — file is unreadable
+			detail: state.reason,
+		};
+	}
+
+	// Placeholder detected. Try to regenerate.
+	const refWav = path.join(bundleRoot, "tts", "omnivoice", "samantha-ref.wav");
+	if (!existsSync(refWav)) {
+		return {
+			kind: "placeholder-no-regen",
+			reason: "missing-reference-wav",
+			detail: refWav,
+		};
+	}
+	const libPath = locateBundleLibrary(bundleRoot);
+	if (!existsSync(libPath)) {
+		return {
+			kind: "placeholder-no-regen",
+			reason: "missing-ffi-library",
+			detail: libPath,
+		};
+	}
+
+	let result: RegenerateResult;
+	try {
+		result = await regenerateSamanthaPresetFromBundle({
+			bundleRoot,
+			presetPath,
+			referenceWav: refWav,
+			referenceText: SAMANTHA_REFERENCE_TRANSCRIPT,
+		});
+	} catch (err) {
+		const message = err instanceof Error ? err.message : String(err);
+		// Distinguish the FFI-symbol-missing path from a real synth failure
+		// — both are placeholder-no-regen but the operator-facing reason
+		// differs.
+		const reason: "ffi-no-encode-reference" | "encode-reference-failed" =
+			/encode_reference|encodeReferenceSupported|ABI v4/.test(message)
+				? "ffi-no-encode-reference"
+				: "encode-reference-failed";
+		return { kind: "placeholder-no-regen", reason, detail: message };
+	}
+
+	mkdirSync(path.dirname(presetPath), { recursive: true });
+	writeFileSync(presetPath, result.bytes);
+	return {
+		kind: "regenerated",
+		bytes: result.bytes.byteLength,
+		K: result.K,
+		refT: result.refT,
+	};
+}

package/src/services/voice/scheduler.d.ts

@@ -0,0 +1,146 @@
+import { BargeInController } from "./barge-in";
+import type { PhonemeTokenizer } from "./phoneme-tokenizer";
+import { PhraseCache } from "./phrase-cache";
+import { PhraseChunker } from "./phrase-chunker";
+import { PrefixPreservingQueue } from "./prefix-preserving-queue";
+import { PcmRingBuffer } from "./ring-buffer";
+import { RollbackQueue } from "./rollback-queue";
+import type { AudioChunk, AudioSink, OmniVoiceBackend, Phrase, RejectedTokenRange, SchedulerConfig, SpeakerPreset, TextToken, VoiceSchedulerTelemetryListener } from "./types";
+/**
+ * T2 — per-phrase TTS chunk-size telemetry, emitted once per
+ * `synthesizePhraseStream` call when `SchedulerEvents.onChunkMetrics` is
+ * wired. `chunks` is the in-arrival-order distribution of streamed PCM
+ * chunks (size in PCM bytes assuming Float32 samples, duration in ms
+ * derived from samples / sampleRate). Used to debug T1-class chunk-size
+ * pathologies and to verify T3 time-budget effects.
+ */
+export interface TtsPhraseChunkMetrics {
+    phraseId: number;
+    /** Order-preserving list of per-chunk sizes. Empty when no chunks landed. */
+    chunks: ReadonlyArray<{
+        chunkBytes: number;
+        chunkDurationMs: number;
+    }>;
+    /** Sum of chunk durations in ms. */
+    totalDurationMs: number;
+    /** Sum of chunk bytes. */
+    totalBytes: number;
+    /** Whether the phrase synthesis was cancelled mid-stream. */
+    cancelled: boolean;
+}
+export type TtsChunkMetricsListener = (metrics: TtsPhraseChunkMetrics) => void;
+export interface SchedulerEvents {
+    onPhrase?(phrase: Phrase): void;
+    onRollback?(phraseId: number, range: RejectedTokenRange): void;
+    onAudio?(chunk: AudioChunk): void;
+    /**
+     * Barge-in hard-stop: ring buffer drained, chunker reset, in-flight TTS
+     * cancelled. The engine layer's `voiceStreamingArgs` separately threads
+     * the `BargeInCancelToken.signal` (`bargeIn.onSignal` → `hard-stop`)
+     * into `dispatcher.generate` so the LLM/drafter abort too.
+     */
+    onCancel?(): void;
+    /** Provisional barge-in: a VAD voice hit while the agent is speaking paused TTS playback. */
+    onTtsPause?(): void;
+    /** Blip resolved the provisional barge-in — TTS playback resumed. */
+    onTtsResume?(): void;
+    /** Structured scheduler telemetry for latency, cache, rollback, and barge-in metrics. */
+    onTelemetry?: VoiceSchedulerTelemetryListener;
+    /**
+     * T2 — per-phrase TTS chunk-size distribution. Optional; when set, the
+     * scheduler emits one summary per streamed phrase synthesis (success or
+     * cancelled). Lets test harnesses and metrics consumers verify T1/T3
+     * effects without scraping the audio bus.
+     */
+    onChunkMetrics?: TtsChunkMetricsListener;
+}
+export interface SchedulerDeps {
+    backend: OmniVoiceBackend;
+    sink?: AudioSink;
+    phraseCache?: PhraseCache;
+    /** Optional. Required only when `config.chunkerConfig.chunkOn ===
+     *  'phoneme-stream'`. Defaults are available from
+     *  `createDefaultPhonemeTokenizer()`. */
+    phonemeTokenizer?: PhonemeTokenizer;
+}
+export declare class VoiceScheduler {
+    readonly chunker: PhraseChunker;
+    readonly rollback: RollbackQueue;
+    readonly bargeIn: BargeInController;
+    readonly ringBuffer: PcmRingBuffer;
+    readonly sink: AudioSink;
+    readonly preset: SpeakerPreset;
+    /**
+     * Prefix-preserving barge-in queue. When the streaming TTS path is active,
+     * each audio chunk is enqueued here tagged with its token range. On
+     * hard-stop (barge-in), `rollbackAt(divergencePoint)` partitions the
+     * queue: chunks at or before the divergence point are replayed into the
+     * sink; chunks after are dropped. This lets audio that was already
+     * correct play through without re-synthesizing.
+     */
+    readonly prefixQueue: PrefixPreservingQueue;
+    private readonly backend;
+    private readonly phraseCache;
+    private readonly events;
+    private readonly sampleRate;
+    private readonly inFlight;
+    private readonly maxInFlight;
+    private readonly streamingTtsActive;
+    private kernelTicks;
+    private nextStandalonePhraseId;
+    /** True while a provisional barge-in (`pause-tts`) has paused playback. */
+    private paused;
+    /**
+     * The last committed token index — updated whenever a phrase is dispatched
+     * to TTS. Used as the divergence point when a barge-in fires mid-response.
+     */
+    private lastCommittedTokenIndex;
+    private agentSpeakingUntilMs;
+    private agentSpeakingTimer;
+    private phraseFlushTimer;
+    constructor(config: SchedulerConfig, deps: SchedulerDeps, events?: SchedulerEvents);
+    accept(token: TextToken, acceptedAt?: number): Promise<void>;
+    reject(range: RejectedTokenRange): Promise<void>;
+    flushPending(): Promise<void>;
+    waitIdle(): Promise<void>;
+    synthesizeText(text: string, signal?: AbortSignal): Promise<AudioChunk>;
+    prewarmPhrases(texts: ReadonlyArray<string>, opts?: {
+        concurrency?: number;
+    }): Promise<{
+        warmed: number;
+        cached: number;
+    }>;
+    tickKernel(): void;
+    kernelTickCount(): number;
+    /**
+     * Mark the agent as audibly speaking for the duration of audio handed to the
+     * sink. This is the barge-in gate: VAD blips only pause/resume TTS while this
+     * flag is true, and ASR-confirmed words hard-stop playback plus generation.
+     */
+    markAgentSpeakingForAudio(samples: number, sampleRate: number): void;
+    /** True while a provisional barge-in has paused TTS playback. */
+    get ttsPaused(): boolean;
+    /**
+     * Drop not-yet-spoken TTS without signalling a barge-in: drain the ring
+     * buffer, reset the chunker, cancel in-flight synthesis. Used by the turn
+     * controller when a speculative response is invalidated (speech resumed) —
+     * the speculative TTS was streamed off a stale partial transcript, so it
+     * must go, but this is not a user barge-in (`onCancel` is NOT fired).
+     */
+    cancelPendingTts(): void;
+    private dispatchPhrase;
+    private runPhraseSynthesis;
+    private synthesizePhraseStream;
+    private isPhraseTracked;
+    private cancelNativeTts;
+    private commitAudio;
+    private onBargeInSignal;
+    private handleBargeIn;
+    private emitTtsCancel;
+    private emitTelemetry;
+    private armPhraseFlushTimer;
+    private clearPhraseFlushTimer;
+    private armAgentSpeakingTimer;
+    private clearAgentSpeaking;
+}
+//# sourceMappingURL=scheduler.d.ts.map

package/src/services/voice/scheduler.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"scheduler.d.ts","sourceRoot":"","sources":["scheduler.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,iBAAiB,EAAE,MAAM,YAAY,CAAC;AAC/C,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,qBAAqB,CAAC;AAC5D,OAAO,EAAE,WAAW,EAAE,MAAM,gBAAgB,CAAC;AAC7C,OAAO,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAC;AACjD,OAAO,EACN,qBAAqB,EAErB,MAAM,2BAA2B,CAAC;AACnC,OAAO,EAAqB,aAAa,EAAE,MAAM,eAAe,CAAC;AACjE,OAAO,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAC;AACjD,OAAO,KAAK,EAEX,UAAU,EACV,SAAS,EAET,gBAAgB,EAChB,MAAM,EACN,kBAAkB,EAClB,eAAe,EACf,aAAa,EAEb,SAAS,EAKT,+BAA+B,EAE/B,MAAM,SAAS,CAAC;AAEjB;;;;;;;GAOG;AACH,MAAM,WAAW,qBAAqB;IACrC,QAAQ,EAAE,MAAM,CAAC;IACjB,6EAA6E;IAC7E,MAAM,EAAE,aAAa,CAAC;QACrB,UAAU,EAAE,MAAM,CAAC;QACnB,eAAe,EAAE,MAAM,CAAC;KACxB,CAAC,CAAC;IACH,oCAAoC;IACpC,eAAe,EAAE,MAAM,CAAC;IACxB,0BAA0B;IAC1B,UAAU,EAAE,MAAM,CAAC;IACnB,6DAA6D;IAC7D,SAAS,EAAE,OAAO,CAAC;CACnB;AAED,MAAM,MAAM,uBAAuB,GAAG,CAAC,OAAO,EAAE,qBAAqB,KAAK,IAAI,CAAC;AAE/E,MAAM,WAAW,eAAe;IAC/B,QAAQ,CAAC,CAAC,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;IAChC,UAAU,CAAC,CAAC,QAAQ,EAAE,MAAM,EAAE,KAAK,EAAE,kBAAkB,GAAG,IAAI,CAAC;IAC/D,OAAO,CAAC,CAAC,KAAK,EAAE,UAAU,GAAG,IAAI,CAAC;IAClC;;;;;OAKG;IACH,QAAQ,CAAC,IAAI,IAAI,CAAC;IAClB,6FAA6F;IAC7F,UAAU,CAAC,IAAI,IAAI,CAAC;IACpB,qEAAqE;IACrE,WAAW,CAAC,IAAI,IAAI,CAAC;IACrB,yFAAyF;IACzF,WAAW,CAAC,EAAE,+BAA+B,CAAC;IAC9C;;;;;OAKG;IACH,cAAc,CAAC,EAAE,uBAAuB,CAAC;CACzC;AAED,MAAM,WAAW,aAAa;IAC7B,OAAO,EAAE,gBAAgB,CAAC;IAC1B,IAAI,CAAC,EAAE,SAAS,CAAC;IACjB,WAAW,CAAC,EAAE,WAAW,CAAC;IAC1B;;6CAEyC;IACzC,gBAAgB,CAAC,EAAE,gBAAgB,CAAC;CACpC;AAiED,qBAAa,cAAc;IAC1B,QAAQ,CAAC,OAAO,EAAE,aAAa,CAAC;IAChC,QAAQ,CAAC,QAAQ,gBAAuB;IACxC,QAAQ,CAAC,OAAO,oBAA2B;IAC3C,QAAQ,CAAC,UAAU,EAAE,aAAa,CAAC;IACnC,QAAQ,CAAC,IAAI,EAAE,SAAS,CAAC;IACzB,QAAQ,CAAC,MAAM,EAAE,aAAa,CAAC;IAC/B;;;;;;;OAOG;IACH,QAAQ,CAAC,WAAW,wBAA+B;IACnD,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAmB;IAC3C,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAc;IAC1C,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAkB;IACzC,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAS;IACpC,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAA+B;IACxD,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAS;IACrC,OAAO,CAAC,QAAQ,CAAC,kBAAkB,CAAU;IAC7C,OAAO,CAAC,WAAW,CAAK;IACxB,OAAO,CAAC,sBAAsB,CAAM;IACpC,2EAA2E;IAC3E,OAAO,CAAC,MAAM,CAAS;IACvB;;;OAGG;IACH,OAAO,CAAC,uBAAuB,CAAK;IACpC,OAAO,CAAC,oBAAoB,CAAK;IACjC,OAAO,CAAC,kBAAkB,CAA8C;IACxE,OAAO,CAAC,gBAAgB,CAA8C;gBAGrE,MAAM,EAAE,eAAe,EACvB,IAAI,EAAE,aAAa,EACnB,MAAM,GAAE,eAAoB;IAoCvB,MAAM,CAAC,KAAK,EAAE,SAAS,EAAE,UAAU,SAAa,GAAG,OAAO,CAAC,IAAI,CAAC;IAWhE,MAAM,CAAC,KAAK,EAAE,kBAAkB,GAAG,OAAO,CAAC,IAAI,CAAC;IA8BhD,YAAY,IAAI,OAAO,CAAC,IAAI,CAAC;IAQ7B,QAAQ,IAAI,OAAO,CAAC,IAAI,CAAC;IAKzB,cAAc,CACnB,IAAI,EAAE,MAAM,EACZ,MAAM,CAAC,EAAE,WAAW,GAClB,OAAO,CAAC,UAAU,CAAC;IA6FhB,cAAc,CACnB,KAAK,EAAE,aAAa,CAAC,MAAM,CAAC,EAC5B,IAAI,GAAE;QAAE,WAAW,CAAC,EAAE,MAAM,CAAA;KAAO,GACjC,OAAO,CAAC;QAAE,MAAM,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAA;KAAE,CAAC;IA8C9C,UAAU,IAAI,IAAI;IAIlB,eAAe,IAAI,MAAM;IAIzB;;;;OAIG;IACH,yBAAyB,CAAC,OAAO,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,GAAG,IAAI;IAYpE,iEAAiE;IACjE,IAAI,SAAS,IAAI,OAAO,CAEvB;IAED;;;;;;OAMG;IACH,gBAAgB,IAAI,IAAI;YAeV,cAAc;YA4Dd,kBAAkB;YA8ClB,sBAAsB;IAkHpC,OAAO,CAAC,eAAe;IAMvB,OAAO,CAAC,eAAe;IAMvB,OAAO,CAAC,WAAW;IA4CnB,OAAO,CAAC,eAAe;IAgCvB,OAAO,CAAC,aAAa;IAkErB,OAAO,CAAC,aAAa;IASrB,OAAO,CAAC,aAAa;IAIrB,OAAO,CAAC,mBAAmB;IAsB3B,OAAO,CAAC,qBAAqB;IAO7B,OAAO,CAAC,qBAAqB;IAqB7B,OAAO,CAAC,kBAAkB;CAQ1B"}

package/src/services/voice/scheduler.t2.test.ts

@@ -0,0 +1,141 @@
+/**
+ * T2 — TTS chunk-size telemetry tests for `VoiceScheduler`.
+ *
+ * The scheduler streams TTS PCM in chunks; T2 records the per-phrase
+ * distribution of chunk sizes so we can debug T1-class pathologies (one
+ * giant chunk = no streaming) and confirm T3-class effects (more, smaller
+ * phrases = more, smaller chunks). The streaming backend is a fake so
+ * tests stay hermetic.
+ */
+
+import { describe, expect, it } from "vitest";
+import { InMemoryAudioSink } from "./ring-buffer";
+import { type TtsPhraseChunkMetrics, VoiceScheduler } from "./scheduler";
+import type {
+	AudioChunk,
+	OmniVoiceBackend,
+	Phrase,
+	SpeakerPreset,
+	StreamingTtsBackend,
+	TextToken,
+	TtsPcmChunk,
+} from "./types";
+
+function tok(index: number, text: string): TextToken {
+	return { index, text };
+}
+
+function makePreset(): SpeakerPreset {
+	const embedding = new Float32Array([0.1, 0.2]);
+	return {
+		voiceId: "default",
+		embedding,
+		bytes: new Uint8Array(embedding.buffer.slice(0)),
+	};
+}
+
+class ScriptedStreamingBackend
+	implements OmniVoiceBackend, StreamingTtsBackend
+{
+	constructor(private readonly chunks: ReadonlyArray<Float32Array>) {}
+	async synthesize(): Promise<AudioChunk> {
+		throw new Error("not used");
+	}
+	async synthesizeStream(args: {
+		phrase: Phrase;
+		preset: SpeakerPreset;
+		cancelSignal: { cancelled: boolean };
+		onChunk: (chunk: TtsPcmChunk) => boolean | undefined;
+		onKernelTick?: () => void;
+	}): Promise<{ cancelled: boolean }> {
+		for (const pcm of this.chunks) {
+			args.onKernelTick?.();
+			if (args.cancelSignal.cancelled) break;
+			args.onChunk({ pcm, sampleRate: 24000, isFinal: false });
+		}
+		args.onChunk({
+			pcm: new Float32Array(0),
+			sampleRate: 24000,
+			isFinal: true,
+		});
+		return { cancelled: args.cancelSignal.cancelled };
+	}
+}
+
+describe("VoiceScheduler T2 chunk-size telemetry", () => {
+	it("emits one onChunkMetrics summary per phrase with per-chunk byte and duration", async () => {
+		// 240 samples @ 24 kHz = 10 ms per chunk. Two chunks = 20 ms.
+		const backend = new ScriptedStreamingBackend([
+			new Float32Array(240),
+			new Float32Array(480),
+		]);
+		const sink = new InMemoryAudioSink();
+		const metricsLog: TtsPhraseChunkMetrics[] = [];
+		const sched = new VoiceScheduler(
+			{
+				chunkerConfig: { maxTokensPerPhrase: 10 },
+				preset: makePreset(),
+				ringBufferCapacity: 4096,
+				sampleRate: 24000,
+			},
+			{ backend, sink },
+			{ onChunkMetrics: (m) => metricsLog.push(m) },
+		);
+
+		await sched.accept(tok(0, "Hello"));
+		await sched.accept(tok(1, "."));
+		await sched.waitIdle();
+
+		expect(metricsLog).toHaveLength(1);
+		const m = metricsLog[0];
+		expect(m.chunks).toHaveLength(2);
+		// Float32 => 4 bytes / sample.
+		expect(m.chunks[0]).toEqual({ chunkBytes: 240 * 4, chunkDurationMs: 10 });
+		expect(m.chunks[1]).toEqual({ chunkBytes: 480 * 4, chunkDurationMs: 20 });
+		expect(m.totalBytes).toBe((240 + 480) * 4);
+		expect(m.totalDurationMs).toBe(30);
+		expect(m.cancelled).toBe(false);
+	});
+
+	it("reports cancelled=false summary when synthesis completes", async () => {
+		const backend = new ScriptedStreamingBackend([new Float32Array(120)]);
+		const sink = new InMemoryAudioSink();
+		const metricsLog: TtsPhraseChunkMetrics[] = [];
+		const sched = new VoiceScheduler(
+			{
+				chunkerConfig: { maxTokensPerPhrase: 10 },
+				preset: makePreset(),
+				ringBufferCapacity: 4096,
+				sampleRate: 24000,
+			},
+			{ backend, sink },
+			{ onChunkMetrics: (m) => metricsLog.push(m) },
+		);
+		await sched.accept(tok(0, "Hi"));
+		await sched.accept(tok(1, "."));
+		await sched.waitIdle();
+		expect(metricsLog).toHaveLength(1);
+		expect(metricsLog[0].cancelled).toBe(false);
+		expect(metricsLog[0].chunks).toHaveLength(1);
+	});
+
+	it("does not invoke onChunkMetrics when the listener is absent", async () => {
+		const backend = new ScriptedStreamingBackend([new Float32Array(120)]);
+		const sink = new InMemoryAudioSink();
+		// No listener — should not throw or do extra work; the scheduler still
+		// commits audio normally.
+		const sched = new VoiceScheduler(
+			{
+				chunkerConfig: { maxTokensPerPhrase: 10 },
+				preset: makePreset(),
+				ringBufferCapacity: 4096,
+				sampleRate: 24000,
+			},
+			{ backend, sink },
+		);
+		await sched.accept(tok(0, "Hi"));
+		await sched.accept(tok(1, "."));
+		await sched.waitIdle();
+		expect(sink.totalWritten()).toBeGreaterThan(0);
+	});
+});