@makefinks/daemon 0.1.0

package/src/voice/tts/openai-tts-stream.ts

@@ -0,0 +1,222 @@
+/**
+ * OpenAI TTS client using the streaming API.
+ * Generates speech audio and emits PCM chunks as they arrive for real-time playback.
+ */
+
+import { EventEmitter } from "node:events";
+import OpenAI from "openai";
+import type { SpeechCreateParams } from "openai/resources/audio/speech";
+
+export interface OpenAITTSOptions {
+	/** OpenAI API key (defaults to OPENAI_API_KEY env var) */
+	apiKey?: string;
+	/** OpenAI speech model (default: gpt-4o-mini-tts) */
+	model?: SpeechCreateParams["model"];
+	/** Voice name (default: onyx) */
+	voice?: SpeechCreateParams["voice"];
+	/** Output format (default: pcm) */
+	outputFormat?: SpeechCreateParams["response_format"];
+	/** Speech speed multiplier (0.25 to 4.0) */
+	speed?: SpeechCreateParams["speed"];
+	/** Optional style instructions (works with gpt-4o-mini-tts, not tts-1/tts-1-hd) */
+	instructions?: SpeechCreateParams["instructions"];
+}
+
+export interface OpenAITTSStreamEvents {
+	audio: (chunk: Buffer) => void;
+	done: () => void;
+	error: (error: Error) => void;
+}
+
+const DEFAULT_MODEL: SpeechCreateParams["model"] = "gpt-4o-mini-tts";
+const DEFAULT_VOICE: SpeechCreateParams["voice"] = "onyx";
+const DEFAULT_FORMAT: SpeechCreateParams["response_format"] = "pcm";
+const DEFAULT_SPEED: SpeechCreateParams["speed"] = 1.1;
+
+const DEFAULT_INSTRUCTIONS =
+	"Speak with a confident and precise tone inspired by JARVIS from Iron Man with a purposeful and helpful cadence.";
+
+interface SpeechAudioDeltaEvent {
+	type: "speech.audio.delta";
+	audio: string;
+}
+
+interface SpeechAudioDoneEvent {
+	type: "speech.audio.done";
+	usage?: {
+		input_tokens: number;
+		output_tokens: number;
+		total_tokens: number;
+	};
+}
+
+type SpeechStreamEvent = SpeechAudioDeltaEvent | SpeechAudioDoneEvent;
+
+/**
+ * OpenAI TTS client using the streaming API.
+ * Emits 'audio' events with Buffer chunks as they arrive, then 'done' when complete.
+ */
+export class OpenAITTSStream extends EventEmitter {
+	private client: OpenAI;
+	private model: SpeechCreateParams["model"];
+	private voice: SpeechCreateParams["voice"];
+	private outputFormat: SpeechCreateParams["response_format"];
+	private speed: SpeechCreateParams["speed"];
+	private instructions: SpeechCreateParams["instructions"];
+	private _isSpeaking = false;
+	private _aborted = false;
+	private abortController: AbortController | null = null;
+
+	constructor(options: OpenAITTSOptions = {}) {
+		super();
+
+		this.client = new OpenAI({
+			apiKey: options.apiKey,
+		});
+		this.model = options.model ?? DEFAULT_MODEL;
+		this.voice = options.voice ?? DEFAULT_VOICE;
+		this.outputFormat = options.outputFormat ?? DEFAULT_FORMAT;
+		this.speed = options.speed ?? DEFAULT_SPEED;
+		this.instructions = options.instructions ?? DEFAULT_INSTRUCTIONS;
+	}
+
+	get isSpeaking(): boolean {
+		return this._isSpeaking;
+	}
+
+	async speak(text: string): Promise<void> {
+		if (!text.trim()) {
+			this.emit("done");
+			return;
+		}
+
+		this.stop();
+		this._aborted = false;
+		this._isSpeaking = true;
+		this.abortController = new AbortController();
+
+		try {
+			const modelStr = String(this.model);
+			const supportsSSE = !modelStr.startsWith("tts-1");
+
+			const params: SpeechCreateParams = {
+				model: this.model,
+				input: text,
+				voice: this.voice,
+				response_format: this.outputFormat,
+				speed: this.speed,
+			};
+
+			if (supportsSSE) {
+				params.stream_format = "sse";
+				params.instructions = this.instructions;
+			}
+
+			const response = await this.client.audio.speech.create(params, {
+				signal: this.abortController.signal,
+			});
+
+			const body = response.body;
+			if (!body) {
+				throw new Error("No response body from OpenAI TTS API");
+			}
+
+			if (supportsSSE) {
+				await this.processSSEStream(body);
+			} else {
+				await this.processRawAudioStream(body);
+			}
+
+			this._isSpeaking = false;
+			if (!this._aborted) {
+				this.emit("done");
+			}
+		} catch (error) {
+			this._isSpeaking = false;
+			if (!this._aborted) {
+				const err = error instanceof Error ? error : new Error(String(error));
+				this.emit("error", err);
+			}
+		} finally {
+			this.abortController = null;
+		}
+	}
+
+	private async processSSEStream(body: ReadableStream<Uint8Array>): Promise<void> {
+		const reader = body.getReader();
+		const decoder = new TextDecoder();
+		let buffer = "";
+
+		try {
+			while (true) {
+				if (this._aborted) break;
+
+				const { done, value } = await reader.read();
+				if (done) break;
+
+				buffer += decoder.decode(value, { stream: true });
+
+				const lines = buffer.split("\n");
+				buffer = lines.pop() ?? "";
+
+				for (const line of lines) {
+					if (this._aborted) break;
+
+					const trimmed = line.trim();
+					if (!trimmed || trimmed.startsWith(":")) continue;
+
+					if (trimmed.startsWith("data: ")) {
+						const data = trimmed.slice(6);
+						if (data === "[DONE]") continue;
+
+						try {
+							const event = JSON.parse(data) as SpeechStreamEvent;
+
+							if (event.type === "speech.audio.delta") {
+								const audioBuffer = Buffer.from(event.audio, "base64");
+								this.emit("audio", audioBuffer);
+							}
+						} catch {}
+					}
+				}
+			}
+		} finally {
+			reader.releaseLock();
+		}
+	}
+
+	private async processRawAudioStream(body: ReadableStream<Uint8Array>): Promise<void> {
+		const reader = body.getReader();
+
+		try {
+			while (true) {
+				if (this._aborted) break;
+
+				const { done, value } = await reader.read();
+				if (done) break;
+
+				this.emit("audio", Buffer.from(value));
+			}
+		} finally {
+			reader.releaseLock();
+		}
+	}
+
+	stop(): void {
+		this._aborted = true;
+
+		if (this.abortController) {
+			try {
+				this.abortController.abort();
+			} catch {}
+			this.abortController = null;
+		}
+
+		this._isSpeaking = false;
+	}
+
+	destroy(): void {
+		this.stop();
+		this.removeAllListeners();
+	}
+}

package/src/voice/tts/speech-controller.ts

@@ -0,0 +1,64 @@
+import { EventEmitter } from "node:events";
+
+import { destroyTTSPlayer, getTTSPlayer, stopSpeaking } from "./tts-player";
+
+export interface SpeechControllerOptions {
+	speed: number;
+	outputDeviceName?: string;
+}
+
+export interface SpeechControllerEvents {
+	audioLevel: (level: number) => void;
+}
+
+export class SpeechController extends EventEmitter {
+	private abortController: AbortController | null = null;
+
+	async speak(text: string, options: SpeechControllerOptions, signal?: AbortSignal): Promise<void> {
+		if (!text.trim()) return;
+
+		this.stop();
+		this.abortController = new AbortController();
+
+		if (signal) {
+			if (signal.aborted) {
+				this.stop();
+				return;
+			}
+			signal.addEventListener("abort", () => this.stop(), { once: true });
+		}
+
+		const player = getTTSPlayer({
+			openai: {
+				speed: options.speed,
+			},
+			outputDeviceName: options.outputDeviceName,
+		});
+
+		const handleAudioLevel = (level: number) => {
+			this.emit("audioLevel", level);
+		};
+		player.on("audioLevel", handleAudioLevel);
+
+		try {
+			await player.speak(text, this.abortController.signal);
+		} finally {
+			player.off("audioLevel", handleAudioLevel);
+			this.abortController = null;
+		}
+	}
+
+	stop(): void {
+		if (this.abortController) {
+			this.abortController.abort();
+			this.abortController = null;
+		}
+		stopSpeaking();
+	}
+
+	destroy(): void {
+		this.stop();
+		destroyTTSPlayer();
+		this.removeAllListeners();
+	}
+}

package/src/voice/tts/tts-player.ts

@@ -0,0 +1,257 @@
+import { spawn, type ChildProcess } from "node:child_process";
+import { EventEmitter } from "node:events";
+import { OpenAITTSStream, type OpenAITTSOptions } from "./openai-tts-stream";
+import { debug } from "../../utils/debug-logger";
+
+export interface TTSPlayerOptions {
+	openai?: OpenAITTSOptions;
+	outputDeviceName?: string;
+	debug?: boolean;
+}
+
+export interface TTSPlayerEvents {
+	speaking: () => void;
+	done: () => void;
+	error: (error: Error) => void;
+	audioLevel: (level: number) => void;
+}
+
+export class TTSPlayer extends EventEmitter {
+	private options: TTSPlayerOptions;
+	private tts: OpenAITTSStream | null = null;
+	private player: ChildProcess | null = null;
+	private _isSpeaking = false;
+	private audioChunksReceived = 0;
+	private abortController: AbortController | null = null;
+	private audioLevelSmoothed = 0;
+	private audioLevelLastEmitMs = 0;
+
+	constructor(options: TTSPlayerOptions = {}) {
+		super();
+		this.options = options;
+	}
+
+	updateOptions(options: TTSPlayerOptions = {}): void {
+		this.options = {
+			...this.options,
+			...options,
+			openai: {
+				...this.options.openai,
+				...options.openai,
+			},
+		};
+	}
+
+	get isSpeaking(): boolean {
+		return this._isSpeaking;
+	}
+
+	private emitAudioLevelFromChunk(chunk: Buffer): void {
+		if (chunk.length < 2) return;
+
+		let samples = 0;
+		let sumSquares = 0;
+
+		const sampleCount = Math.floor(chunk.length / 2);
+		const view = new Int16Array(chunk.buffer, chunk.byteOffset, sampleCount);
+		for (let i = 0; i < view.length; i++) {
+			const v = view[i] ?? 0;
+			const f = v / 32768;
+			sumSquares += f * f;
+			samples++;
+		}
+		if (!samples) return;
+
+		const rms = Math.sqrt(sumSquares / samples);
+		const noiseFloor = 0.01;
+		const gain = 8;
+		const rawLevel = Math.max(0, (rms - noiseFloor) * gain);
+		const level = Math.min(1, rawLevel);
+
+		const prev = this.audioLevelSmoothed;
+		const alpha = level > prev ? 0.55 : 0.15;
+		this.audioLevelSmoothed = prev + (level - prev) * alpha;
+
+		const now = Date.now();
+		if (now - this.audioLevelLastEmitMs < 16) return;
+		this.audioLevelLastEmitMs = now;
+		this.emit("audioLevel", this.audioLevelSmoothed);
+	}
+
+	async speak(text: string, signal?: AbortSignal): Promise<void> {
+		if (!text.trim()) return;
+
+		this.stop();
+
+		this._isSpeaking = true;
+		this.audioChunksReceived = 0;
+		this.audioLevelSmoothed = 0;
+		this.audioLevelLastEmitMs = 0;
+		this.emit("speaking");
+
+		this.abortController = new AbortController();
+
+		if (signal) {
+			if (signal.aborted) {
+				this.stop();
+				return;
+			}
+			signal.addEventListener("abort", () => this.stop(), { once: true });
+		}
+
+		return new Promise<void>((resolve, reject) => {
+			const cleanup = () => {
+				this._isSpeaking = false;
+				this.emit("done");
+				resolve();
+			};
+
+			const handleError = (error: Error) => {
+				this.stop();
+				this.emit("error", error);
+				reject(error);
+			};
+
+			try {
+				this.tts = new OpenAITTSStream({
+					outputFormat: "pcm",
+					...this.options.openai,
+				});
+
+				const soxArgs: string[] = [
+					"-q",
+					"-t",
+					"raw",
+					"-e",
+					"signed-integer",
+					"-b",
+					"16",
+					"-r",
+					"24000",
+					"-c",
+					"1",
+					"-",
+				];
+
+				if (this.options.outputDeviceName) {
+					soxArgs.push("-t", "coreaudio", this.options.outputDeviceName);
+				} else {
+					soxArgs.push("-d");
+				}
+
+				this.player = spawn("sox", soxArgs, {
+					stdio: ["pipe", "ignore", "pipe"],
+				});
+
+				this.tts.on("audio", (chunk: Buffer) => {
+					this.audioChunksReceived++;
+					if (this.options.debug && this.audioChunksReceived === 1) {
+						debug.info("tts-player-first-audio-chunk");
+					}
+					this.emitAudioLevelFromChunk(chunk);
+					if (this.player?.stdin?.writable) {
+						this.player.stdin.write(chunk);
+					}
+				});
+
+				this.tts.on("done", () => {
+					if (this.options.debug) {
+						debug.info("tts-player-tts-done");
+					}
+					try {
+						this.player?.stdin?.end();
+					} catch {
+						// Ignore
+					}
+				});
+
+				this.tts.on("error", handleError);
+
+				this.player.on("error", (err) => {
+					handleError(new Error(`sox player error: ${err.message}. Ensure sox is installed.`));
+				});
+
+				this.player.on("close", (code) => {
+					if (this.options.debug) {
+						debug.info("tts-player-sox-exited", { code });
+					}
+					cleanup();
+				});
+
+				this.player.stderr?.on("data", (data: Buffer) => {
+					const msg = data.toString();
+					if (this.options.debug) {
+						debug.info("tts-player-sox-stderr", { msg });
+					}
+				});
+
+				if (this.options.debug) {
+					debug.info("tts-player-starting-openai-tts");
+				}
+				this.tts.speak(text).catch(handleError);
+			} catch (error) {
+				handleError(error instanceof Error ? error : new Error(String(error)));
+			}
+		});
+	}
+
+	stop(): void {
+		if (this.abortController) {
+			this.abortController.abort();
+			this.abortController = null;
+		}
+
+		if (this.tts) {
+			this.tts.destroy();
+			this.tts = null;
+		}
+
+		if (this.player) {
+			try {
+				this.player.kill("SIGTERM");
+			} catch {
+				// Ignore
+			}
+			this.player = null;
+		}
+
+		if (this._isSpeaking) {
+			this._isSpeaking = false;
+			this.emit("done");
+		}
+	}
+
+	destroy(): void {
+		this.stop();
+		this.removeAllListeners();
+	}
+}
+
+let player: TTSPlayer | null = null;
+
+export function getTTSPlayer(options?: TTSPlayerOptions): TTSPlayer {
+	if (!player) {
+		player = new TTSPlayer(options);
+	} else if (options) {
+		player.updateOptions(options);
+	}
+	return player;
+}
+
+export function destroyTTSPlayer(): void {
+	if (player) {
+		player.destroy();
+		player = null;
+	}
+}
+
+export async function speak(text: string, signal?: AbortSignal): Promise<void> {
+	const p = getTTSPlayer();
+	return p.speak(text, signal);
+}
+
+export function stopSpeaking(): void {
+	if (player) {
+		player.stop();
+	}
+}

package/src/voice/voice-input-controller.ts

@@ -0,0 +1,96 @@
+import { EventEmitter } from "node:events";
+
+import { getRecorder, destroyRecorder } from "./audio-recorder";
+import { computeMicLevelFromPcm16Chunk, smoothMicLevel } from "./mic-level";
+
+export interface VoiceInputControllerEvents {
+	micLevel: (level: number) => void;
+	error: (error: Error) => void;
+}
+
+export class VoiceInputController extends EventEmitter {
+	private recorderErrorHandler: ((error: Error) => void) | null = null;
+	private recorderDataHandler: ((chunk: Buffer) => void) | null = null;
+	private activeRecorder: ReturnType<typeof getRecorder> | null = null;
+	private micLevelSmoothed = 0;
+	private micLevelLastEmitMs = 0;
+
+	private detachRecorderListeners(): void {
+		if (!this.activeRecorder) return;
+		if (this.recorderErrorHandler) {
+			this.activeRecorder.off("error", this.recorderErrorHandler);
+			this.recorderErrorHandler = null;
+		}
+		if (this.recorderDataHandler) {
+			this.activeRecorder.off("data", this.recorderDataHandler);
+			this.recorderDataHandler = null;
+		}
+	}
+
+	private emitMicLevelFromChunk(chunk: Buffer): void {
+		const level = computeMicLevelFromPcm16Chunk(chunk);
+		if (level === null) return;
+
+		this.micLevelSmoothed = smoothMicLevel(this.micLevelSmoothed, level);
+
+		const now = Date.now();
+		if (now - this.micLevelLastEmitMs < 16) return;
+		this.micLevelLastEmitMs = now;
+		this.emit("micLevel", this.micLevelSmoothed);
+	}
+
+	start(): void {
+		this.detachRecorderListeners();
+
+		const recorder = getRecorder();
+		this.activeRecorder = recorder;
+		this.micLevelSmoothed = 0;
+		this.micLevelLastEmitMs = 0;
+
+		this.recorderDataHandler = (chunk: Buffer) => {
+			this.emitMicLevelFromChunk(chunk);
+		};
+		this.recorderErrorHandler = (err: Error) => {
+			try {
+				recorder.stop();
+			} catch {
+				// Ignore; we're already failing.
+			}
+			this.detachRecorderListeners();
+			this.emit("error", err);
+		};
+
+		recorder.on("data", this.recorderDataHandler);
+		recorder.on("error", this.recorderErrorHandler);
+		recorder.start();
+	}
+
+	async stop(): Promise<{ audioBuffer: Buffer; duration: number }> {
+		const recorder = this.activeRecorder ?? getRecorder();
+		this.detachRecorderListeners();
+		this.activeRecorder = null;
+
+		const duration = recorder.getDuration();
+		const audioBuffer = await recorder.stopAsync();
+
+		return { audioBuffer, duration };
+	}
+
+	cancel(): void {
+		const recorder = this.activeRecorder ?? getRecorder();
+		this.detachRecorderListeners();
+		this.activeRecorder = null;
+		try {
+			recorder.stop();
+		} catch {
+			// Ignore
+		}
+	}
+
+	destroy(): void {
+		this.detachRecorderListeners();
+		this.activeRecorder = null;
+		destroyRecorder();
+		this.removeAllListeners();
+	}
+}