@oh-my-pi/pi-coding-agent 16.3.0 → 16.3.3

package/src/tts/streaming-player.ts

@@ -4,14 +4,17 @@
  * Replaces the spawn-`afplay`-per-sentence approach (a fresh process per chunk
  * meant audible gaps, per-spawn latency, and no way to interrupt a clip mid-play)
  * with a single persistent player process fed raw 32-bit-float mono PCM over
- * stdin. Chunks are queued and drained by one writer so sentences play back to
+ * stdin. Chunks are queued and drained by one writer so segments play back to
  * back; writes are paced to stay only {@link LEAD_SECONDS} ahead of realtime so
  * ducking and stop take effect promptly instead of after seconds of buffered
  * audio. {@link StreamingAudioPlayer.stop} kills the process for instant silence.
  *
- * Where no streaming backend exists (Windows, or macOS without the bundled
- * ffmpeg), it degrades to the per-file {@link playAudioFile} path so speech still
- * works — just without gapless playback or mid-clip interruption.
+ * Where no streaming backend exists (Windows, or a host without ffmpeg/sox), it
+ * degrades to the per-file {@link playAudioFile} path so speech still works —
+ * just without gapless playback or mid-clip interruption. A backend that spawns
+ * but dies early (e.g. an ffmpeg built without its platform audio device) is
+ * detected via its exit and the session downgrades to per-file playback without
+ * dropping the chunk being played.
  */
 import * as fs from "node:fs/promises";
 import * as os from "node:os";
@@ -41,8 +44,7 @@ export interface StreamingPlayerLookup {
  * and plays it to the default output device. An empty list means no streaming
  * backend is available and the caller should fall back to per-file playback.
  *
- * - darwin: none; `afplay` is file-only, so macOS uses the interruptible
- *   per-file fallback.
+ * - darwin: `ffmpeg` (AudioToolbox output device) → sox's `play` (coreaudio).
  * - linux/other POSIX: `ffmpeg` (`-f pulse` then `-f alsa`) → `paplay`/`aplay`
  *   raw fallbacks.
  * - win32: none (PowerShell `SoundPlayer` is file-only).
@@ -57,7 +59,19 @@ export function streamingPlayerCommandsFor(
 	const rate = String(sampleRate > 0 ? sampleRate : DEFAULT_SAMPLE_RATE);
 	const input = ["-loglevel", "error", "-nostdin", "-f", "f32le", "-ar", rate, "-ac", "1", "-i", "pipe:0"];
 
-	if (platform === "darwin") return [];
+	if (platform === "darwin") {
+		const commands: PlayerCommand[] = [];
+		const ffmpegBin = ffmpeg();
+		if (ffmpegBin) commands.push({ cmd: ffmpegBin, args: [...input, "-f", "audiotoolbox", "default"] });
+		const play = which("play");
+		if (play) {
+			commands.push({
+				cmd: play,
+				args: ["-q", "-t", "raw", "-e", "floating-point", "-b", "32", "-r", rate, "-c", "1", "-"],
+			});
+		}
+		return commands;
+	}
 	if (platform === "win32") {
 		return [];
 	}
@@ -87,6 +101,8 @@ export class StreamingAudioPlayer {
 	#mode: "stream" | "file" = "file";
 	#proc: Subprocess<"pipe", "ignore", "ignore"> | null = null;
 	#sink: FileSink | null = null;
+	/** Streaming backends not yet tried; consumed head-first by {@link #spawnStream}. */
+	#candidates: PlayerCommand[] | null = null;
 	#writtenSec = 0;
 	#startedAt = 0;
 	#started = false;
@@ -133,27 +149,47 @@ export class StreamingAudioPlayer {
 		this.#abortController.abort();
 		this.#signal();
 		try {
-			this.#sink?.end();
+			// end() flushes asynchronously; the SIGKILL below races it, so a broken
+			// pipe here is expected — swallow the rejection (it otherwise surfaces
+			// as an unhandled EPIPE right as speech ends).
+			void Promise.resolve(this.#sink?.end()).catch(() => {});
 		} catch {}
 		try {
 			this.#proc?.kill("SIGKILL");
 		} catch {}
 	}
 
+	/**
+	 * Spawn the next untried streaming backend; false once the list is
+	 * exhausted. A backend that spawns but dies early (e.g. an ffmpeg built
+	 * without this platform's audio output device) would otherwise swallow PCM
+	 * into a dead pipe, so its exit advances to the next candidate — or to
+	 * per-file playback — and #writeStream's failure path replays the
+	 * in-flight chunk.
+	 */
 	#spawnStream(): boolean {
-		for (const command of streamingPlayerCommandsFor(process.platform, this.#sampleRate)) {
+		this.#candidates ??= streamingPlayerCommandsFor(process.platform, this.#sampleRate);
+		for (let command = this.#candidates.shift(); command; command = this.#candidates.shift()) {
+			const { cmd, args } = command;
 			try {
-				const proc = Bun.spawn([command.cmd, ...command.args], {
+				const proc = Bun.spawn([cmd, ...args], {
 					stdin: "pipe",
 					stdout: "ignore",
 					stderr: "ignore",
 				});
 				this.#proc = proc;
 				this.#sink = proc.stdin;
+				void proc.exited.then(code => {
+					if (this.#proc !== proc || this.#stopped || this.#inputClosed) return;
+					logger.debug("tts: streaming backend exited early; trying next backend", { cmd, code });
+					this.#proc = null;
+					this.#sink = null;
+					this.#mode = this.#spawnStream() ? "stream" : "file";
+				});
 				return true;
 			} catch (error) {
 				logger.debug("tts: streaming player spawn failed", {
-					cmd: command.cmd,
+					cmd,
 					error: error instanceof Error ? error.message : String(error),
 				});
 			}
@@ -184,8 +220,20 @@ export class StreamingAudioPlayer {
 						await Bun.sleep((ahead - LEAD_SECONDS) * 1000);
 						if (this.#stopped) return;
 					}
-					this.#writeStream(chunk);
-					this.#writtenSec += chunk.length / this.#sampleRate;
+					if (await this.#writeStream(chunk)) {
+						this.#writtenSec += chunk.length / this.#sampleRate;
+						continue;
+					}
+					// Backend died mid-write: move to the next streaming candidate
+					// (or the file path) and replay this exact chunk so nothing is
+					// dropped.
+					this.#mode = this.#spawnStream() ? "stream" : "file";
+					if (this.#mode === "stream" && (await this.#writeStream(chunk))) {
+						this.#writtenSec += chunk.length / this.#sampleRate;
+					} else {
+						this.#mode = "file";
+						await this.#playFile(chunk);
+					}
 				} else {
 					await this.#playFile(chunk);
 				}
@@ -219,16 +267,23 @@ export class StreamingAudioPlayer {
 		return promise;
 	}
 
-	#writeStream(pcm: Float32Array): void {
+	/**
+	 * Write one chunk into the backend's stdin and await the flush — a broken
+	 * pipe rejects here (not as an unhandled rejection later), so the caller
+	 * can replay the exact chunk on the next backend.
+	 */
+	async #writeStream(pcm: Float32Array): Promise<boolean> {
 		const sink = this.#sink;
-		if (!sink) return;
+		if (!sink) return false;
 		try {
 			sink.write(this.#bytes(pcm));
-			sink.flush();
+			await sink.flush();
+			return true;
 		} catch (error) {
 			logger.debug("tts: streaming write failed", {
 				error: error instanceof Error ? error.message : String(error),
 			});
+			return false;
 		}
 	}
 

package/src/tts/tts-client.ts

@@ -42,7 +42,7 @@ export interface TtsStreamOptions {
 	signal?: AbortSignal;
 }
 
-/** One synthesized sentence of a streaming session, in emission order. */
+/** One synthesized segment of a streaming session, in emission order. */
 export interface TtsAudioChunk {
 	index: number;
 	text: string;
@@ -51,10 +51,10 @@ export interface TtsAudioChunk {
 }
 
 /**
- * A live streaming-synthesis session. Feed text incrementally with {@link push}
- * and close the input with {@link end}; `chunks` yields each synthesized
- * sentence's audio as soon as it is ready, then completes once the worker
- * finishes draining the closed input.
+ * A live streaming-synthesis session. Feed complete speakable segments with
+ * {@link push} (the worker synthesizes each push as-is) and close the input
+ * with {@link end}; `chunks` yields each segment's audio as soon as it is
+ * ready, then completes once the worker finishes draining the closed input.
  */
 export interface TtsStreamHandle {
 	push(text: string): void;
@@ -239,11 +239,12 @@ export class TtsClient {
 	}
 
 	/**
-	 * Open a streaming-synthesis session. Text is fed incrementally through the
-	 * returned handle's `push`/`end`; audio is emitted one synthesized sentence at
-	 * a time via `chunks`, so playback can begin before the full text is known.
-	 * Returns an inert handle (immediately-ended `chunks`) for unknown models or
-	 * an already-aborted signal, and fails the iterator if the worker cannot spawn.
+	 * Open a streaming-synthesis session. Complete speakable segments are fed
+	 * through the returned handle's `push`/`end`; audio is emitted one segment
+	 * at a time via `chunks`, so playback can begin before the full text is
+	 * known. Returns an inert handle (immediately-ended `chunks`) for unknown
+	 * models or an already-aborted signal, and fails the iterator if the worker
+	 * cannot spawn.
 	 */
 	synthesizeStream(modelKey: string, options: TtsStreamOptions = {}): TtsStreamHandle {
 		if (!isTtsLocalModelKey(modelKey) || options.signal?.aborted) {

package/src/tts/tts-protocol.ts

@@ -24,10 +24,12 @@ export type TtsWorkerInbound =
 	| { type: "ping"; id: string }
 	| { type: "synthesize"; id: string; modelKey: TtsLocalModelKey; text: string; voice?: string }
 	| { type: "download"; id: string; modelKey: TtsLocalModelKey }
-	// Streaming synthesis: a session is opened with `stream-start`, fed incrementally
-	// with `stream-push`, and closed with `stream-end`. `stream-cancel` interrupts
-	// without a final drain. The worker emits an `audio-chunk` per synthesized
-	// sentence and a final `stream-done` only for non-cancelled sessions.
+	// Streaming synthesis: a session is opened with `stream-start`, fed complete
+	// speakable segments with `stream-push` (the parent's SpeakableStream does all
+	// splitting/normalization; the worker synthesizes each push as-is), and closed
+	// with `stream-end`. `stream-cancel` interrupts without a final drain. The
+	// worker emits an `audio-chunk` per segment and a final `stream-done` only for
+	// non-cancelled sessions.
 	| { type: "stream-start"; id: string; modelKey: TtsLocalModelKey; voice?: string }
 	| { type: "stream-push"; id: string; text: string }
 	| { type: "stream-end"; id: string }
@@ -40,7 +42,7 @@ export type TtsWorkerOutbound =
 	| { type: "error"; id: string; error: string }
 	| { type: "progress"; id: string; event: TtsProgressEvent }
 	| { type: "log"; level: "debug" | "warn" | "error"; msg: string; meta?: Record<string, unknown> }
-	// One synthesized sentence of a streaming session, in emission order, followed
+	// One synthesized segment of a streaming session, in emission order, followed
 	// by a single `stream-done` once the input stream is closed and drained.
 	| { type: "audio-chunk"; id: string; index: number; text: string; pcm: Float32Array; sampleRate: number }
 	| { type: "stream-done"; id: string };
@@ -56,5 +58,12 @@ export type TtsWorkerOutbound =
  */
 export interface TtsTransport {
 	send(message: TtsWorkerOutbound): void;
+	/**
+	 * Send and resolve once the message has drained into the IPC channel.
+	 * Streaming synthesis awaits this per audio chunk: ONNX inference blocks
+	 * the worker's event loop for seconds at a time, so fire-and-forget sends
+	 * queue unflushed until the session ends and arrive as one burst.
+	 */
+	sendAndFlush(message: TtsWorkerOutbound): Promise<void>;
 	onMessage(handler: (message: TtsWorkerInbound) => void): () => void;
 }

package/src/tts/tts-worker.ts

@@ -39,20 +39,6 @@ type KokoroDevice = "cpu" | "wasm" | "webgpu";
 /** A loaded Kokoro voice synthesizer (subset of `kokoro-js`'s `KokoroTTS`). */
 interface KokoroTtsInstance {
 	generate(text: string, options: { voice: string }): Promise<RawAudio>;
-	stream(
-		text: string | TextSplitterStreamInstance,
-		options: { voice: string },
-	): AsyncGenerator<{ text: string; phonemes: string; audio: RawAudio }, void, void>;
-}
-
-/**
- * Incremental text source for {@link KokoroTtsInstance.stream} (subset of
- * `kokoro-js`'s `TextSplitterStream`). Text pushed at any time is split into
- * complete sentences; `close` flushes the trailing buffer and ends the stream.
- */
-interface TextSplitterStreamInstance {
-	push(...texts: string[]): void;
-	close(): void;
 }
 
 /** `KokoroTTS` static surface used to load a model from the Hugging Face Hub. */
@@ -67,7 +53,6 @@ interface KokoroRuntime {
 			},
 		): Promise<KokoroTtsInstance>;
 	};
-	TextSplitterStream: new () => TextSplitterStreamInstance;
 }
 
 /**
@@ -96,15 +81,18 @@ const kokoroRuntime = new MemoizedRuntime<KokoroRuntime>();
 
 /**
  * In-flight streaming sessions keyed by request id. A session is created on
- * `stream-start` and torn down when its generator finishes. Text pushed before
- * the model finishes loading is held in `buffered` and flushed into the splitter
- * once it exists; pushes after that go straight to the live splitter.
+ * `stream-start` and torn down when its run loop finishes. Each `stream-push`
+ * carries one complete speakable segment (the parent's `SpeakableStream` does
+ * all splitting and normalization); segments queue here and the run loop
+ * synthesizes them in arrival order, waking via `wake` when idle.
  */
 interface StreamSession {
 	modelKey: TtsLocalModelKey;
 	voice: string | undefined;
-	buffered: string[];
-	splitter: TextSplitterStreamInstance | null;
+	/** Speakable segments awaiting synthesis, in arrival order. */
+	queue: string[];
+	/** Resolves the run loop's idle wait when a push/end/cancel arrives. */
+	wake: (() => void) | null;
 	ended: boolean;
 	cancelled: boolean;
 }
@@ -316,43 +304,51 @@ async function handleQueuedRequest(
 }
 
 /**
- * Drive one streaming session to completion: load the model, create the
- * splitter, flush any text pushed before the model was ready, then emit one
- * `audio-chunk` per synthesized sentence followed by a single `stream-done`.
- * Serialized through {@link synthesizeQueue} so it never interleaves model
- * access with a batch synthesize/download.
+ * Drive one streaming session to completion: load the model, then synthesize
+ * each queued segment in arrival order — one `audio-chunk` per segment,
+ * followed by a single `stream-done`. Chunk sends are drained before the next
+ * segment's inference (see the comment at the send site). Serialized through
+ * {@link synthesizeQueue} so it never interleaves model access with a batch
+ * synthesize/download.
  */
 async function runStreamSession(transport: TtsTransport, id: string, session: StreamSession): Promise<void> {
 	try {
-		if (session.cancelled) return;
-		const runtime = await loadKokoroRuntime(transport, id, session.modelKey);
 		if (session.cancelled) return;
 		const synthesizer = await loadModel(session.modelKey, transport, id);
 		if (session.cancelled) return;
 		const spec = getTtsLocalModelSpec(session.modelKey);
-		const splitter = new runtime.TextSplitterStream();
-		// Flush buffered text before exposing the splitter so a push racing this
-		// block can't slip ahead of the already-queued fragments.
-		for (const text of session.buffered) {
-			if (session.cancelled) return;
-			splitter.push(text);
-		}
-		session.buffered = [];
-		session.splitter = splitter;
-		if (session.ended || session.cancelled) splitter.close();
 		const voice = resolveTtsVoice(session.modelKey, session.voice);
 		let index = 0;
-		for await (const chunk of synthesizer.stream(splitter, { voice })) {
+		while (!session.cancelled) {
+			const segment = session.queue.shift();
+			if (segment === undefined) {
+				if (session.ended) break;
+				const { promise, resolve } = Promise.withResolvers<void>();
+				session.wake = resolve;
+				// Re-check after arming: a push/end/cancel racing the empty shift.
+				if (session.queue.length > 0 || session.ended || session.cancelled) {
+					session.wake = null;
+					resolve();
+				}
+				await promise;
+				continue;
+			}
+			const output = await synthesizer.generate(segment, { voice });
 			if (session.cancelled) break;
-			const audio = Array.isArray(chunk.audio.audio) ? chunk.audio.audio[0] : chunk.audio.audio;
+			const audio = Array.isArray(output.audio) ? output.audio[0] : output.audio;
 			if (!audio) continue;
-			transport.send({
+			// Drain the IPC write before the next segment's inference: ONNX
+			// blocks this event loop for seconds at a time, so a fire-and-forget
+			// send would sit in the pipe queue until the session ends and every
+			// chunk would arrive in one burst (long silence, then all segments
+			// at once) instead of streaming per-segment.
+			await transport.sendAndFlush({
 				type: "audio-chunk",
 				id,
 				index: index++,
-				text: chunk.text,
+				text: segment,
 				pcm: audio,
-				sampleRate: chunk.audio.sampling_rate || spec?.sampleRate || 24_000,
+				sampleRate: output.sampling_rate || spec?.sampleRate || 24_000,
 			});
 		}
 		if (!session.cancelled) transport.send({ type: "stream-done", id });
@@ -370,8 +366,8 @@ function startStreamSession(
 	const session: StreamSession = {
 		modelKey: message.modelKey,
 		voice: message.voice,
-		buffered: [],
-		splitter: null,
+		queue: [],
+		wake: null,
 		ended: false,
 		cancelled: false,
 	};
@@ -382,26 +378,33 @@ function startStreamSession(
 	);
 }
 
+/** Wake the session's run loop if it is parked on an empty queue. */
+function wakeStreamSession(session: StreamSession): void {
+	const wake = session.wake;
+	session.wake = null;
+	wake?.();
+}
+
 function pushToStreamSession(id: string, text: string): void {
 	const session = streamSessions.get(id);
 	if (!session || session.cancelled) return;
-	if (session.splitter) session.splitter.push(text);
-	else session.buffered.push(text);
+	session.queue.push(text);
+	wakeStreamSession(session);
 }
 
 function endStreamSession(id: string): void {
 	const session = streamSessions.get(id);
 	if (!session || session.cancelled) return;
 	session.ended = true;
-	session.splitter?.close();
+	wakeStreamSession(session);
 }
 
 function cancelStreamSession(id: string): void {
 	const session = streamSessions.get(id);
 	if (!session) return;
 	session.cancelled = true;
-	session.buffered = [];
-	session.splitter?.close();
+	session.queue.length = 0;
+	wakeStreamSession(session);
 	streamSessions.delete(id);
 }