@prometheus-ai/agent 0.5.4 → 0.5.8

package/src/stt/stt-controller.ts

@@ -3,8 +3,17 @@ import * as os from "node:os";
 import * as path from "node:path";
 import { logger, Snowflake } from "@prometheus-ai/utils";
 import { settings } from "../config/settings";
+import { type SttStreamHandle, sttClient } from "./asr-client";
 import { ensureSTTDependencies } from "./downloader";
-import { type RecordingHandle, startRecording, verifyRecordingFile } from "./recorder";
+import { resolveSttModelSpec } from "./models";
+import {
+	detectRecorder,
+	type RecordingHandle,
+	type StreamingRecordingHandle,
+	startRecording,
+	startStreamingRecording,
+	verifyRecordingFile,
+} from "./recorder";
 import { transcribe } from "./transcriber";
 
 export type SttState = "idle" | "recording" | "transcribing";
@@ -13,21 +22,37 @@ interface ToggleOptions {
 	showWarning(msg: string): void;
 	showStatus(msg: string): void;
 	onStateChange(state: SttState): void;
+	/** Force a redraw after async edits to the composer (live segment/preview inserts). */
+	requestRender?(): void;
 }
 
+/** The slice of the composer editor the controller drives. */
 interface Editor {
 	insertText(text: string): void;
+	setVolatileText(text: string): void;
+	clearVolatileText(): void;
+	commitVolatileText(text: string): void;
 }
 
 export class STTController {
 	#state: SttState = "idle";
-	#recordingHandle: RecordingHandle | null = null;
-	#tempFile: string | null = null;
 	#depsResolved = false;
 	#toggling = false;
+	#stopAfterStart = false;
 	#disposed = false;
+
+	// Batch (single-shot) capture.
+	#recordingHandle: RecordingHandle | null = null;
+	#tempFile: string | null = null;
 	#transcriptionAbort: AbortController | null = null;
 
+	// Live streaming capture.
+	#stream: SttStreamHandle | null = null;
+	#streamRecorder: StreamingRecordingHandle | null = null;
+	#streamEditor: Editor | null = null;
+	#streamCommitted = false;
+	#streamAbort: AbortController | null = null;
+
 	get state(): SttState {
 		return this.#state;
 	}
@@ -38,45 +63,192 @@ export class STTController {
 	}
 
 	async toggle(editor: Editor, options: ToggleOptions): Promise<void> {
-		if (this.#toggling) return;
+		if (this.#toggling) {
+			if (this.#state === "idle" || this.#state === "recording") this.#stopAfterStart = true;
+			return;
+		}
 		this.#toggling = true;
 		try {
 			switch (this.#state) {
 				case "idle":
-					await this.#startRecording(options);
+					await this.#start(editor, options);
 					break;
 				case "recording":
-					await this.#stopAndTranscribe(editor, options);
+					await this.#stop(editor, options);
 					break;
 				case "transcribing":
 					options.showStatus("Transcription in progress...");
 					break;
 			}
+			if (this.#stopAfterStart && this.#state === "recording") {
+				this.#stopAfterStart = false;
+				await this.#stop(editor, options);
+			} else if (this.#state !== "recording") {
+				this.#stopAfterStart = false;
+			}
 		} finally {
 			this.#toggling = false;
 		}
 	}
 
-	async #startRecording(options: ToggleOptions): Promise<void> {
-		if (!this.#depsResolved) {
-			try {
-				options.showStatus("Checking STT dependencies...");
-				await ensureSTTDependencies({
-					modelName: settings.get("stt.modelName") as string | undefined,
-					onProgress: p => options.showStatus(p.stage + (p.percent != null ? ` (${p.percent}%)` : "")),
-				});
-				options.showStatus("");
-				this.#depsResolved = true;
-			} catch (err) {
-				const msg = err instanceof Error ? err.message : "Failed to setup STT dependencies";
+	async #ensureDeps(options: ToggleOptions): Promise<boolean> {
+		if (this.#depsResolved) return true;
+		try {
+			options.showStatus("Checking STT dependencies...");
+			await ensureSTTDependencies({
+				modelName: settings.get("stt.modelName") as string | undefined,
+				onProgress: p => options.showStatus(p.stage + (p.percent != null ? ` (${p.percent}%)` : "")),
+			});
+			options.showStatus("");
+			this.#depsResolved = true;
+			return true;
+		} catch (err) {
+			const msg = err instanceof Error ? err.message : "Failed to setup STT dependencies";
+			options.showWarning(msg);
+			logger.error("STT dependency setup failed", { error: msg });
+			return false;
+		}
+	}
+
+	async #start(editor: Editor, options: ToggleOptions): Promise<void> {
+		if (!(await this.#ensureDeps(options))) return;
+		// Live transcription needs a recorder that can pipe PCM; the Windows
+		// PowerShell mci fallback records to a file, so it stays single-shot.
+		if (this.#recorderCanStream()) {
+			await this.#startStreaming(editor, options);
+			return;
+		}
+		await this.#startBatchRecording(options);
+	}
+
+	async #stop(editor: Editor, options: ToggleOptions): Promise<void> {
+		if (this.#stream) {
+			await this.#stopStreaming(options);
+			return;
+		}
+		await this.#stopBatch(editor, options);
+	}
+
+	// ── Live streaming ──────────────────────────────────────────────
+
+	#recorderCanStream(): boolean {
+		const recorder = detectRecorder();
+		return recorder !== null && recorder.tool !== "powershell";
+	}
+
+	/** Segment text gets a leading space once a prior segment is committed, so
+	 *  phrases join naturally; the first phrase is inserted at the cursor as-is. */
+	#prefixed(text: string): string {
+		const normalized = text.replace(/\s+/g, " ").trim();
+		if (!normalized) return "";
+		return this.#streamCommitted ? ` ${normalized}` : normalized;
+	}
+
+	async #startStreaming(editor: Editor, options: ToggleOptions): Promise<void> {
+		const modelKey = resolveSttModelSpec(settings.get("stt.modelName") as string | undefined).key;
+		const language = settings.get("stt.language") as string | undefined;
+		this.#streamEditor = editor;
+		this.#streamCommitted = false;
+		this.#streamAbort = new AbortController();
+		const stream = sttClient.startStream(modelKey, {
+			language: language || undefined,
+			signal: this.#streamAbort.signal,
+			onPartial: text => {
+				if (this.#disposed || this.#state !== "recording") return;
+				this.#streamEditor?.setVolatileText(this.#prefixed(text));
+				options.requestRender?.();
+			},
+			onSegment: text => {
+				if (this.#disposed) return;
+				const prefixed = this.#prefixed(text);
+				if (prefixed) {
+					this.#streamEditor?.commitVolatileText(prefixed);
+					this.#streamCommitted = true;
+				} else {
+					this.#streamEditor?.clearVolatileText();
+				}
+				options.requestRender?.();
+			},
+		});
+		this.#stream = stream;
+		let recorder: StreamingRecordingHandle | null = null;
+		try {
+			recorder = await startStreamingRecording(samples => stream.pushAudio(samples));
+		} catch (err) {
+			logger.warn("STT streaming recorder failed to start; falling back to batch recording", {
+				error: err instanceof Error ? err.message : String(err),
+			});
+		}
+		if (!recorder) {
+			stream.cancel();
+			this.#cleanupStream();
+			await this.#startBatchRecording(options);
+			return;
+		}
+		this.#streamRecorder = recorder;
+		this.#setState("recording", options);
+		logger.debug("STT live recording started", { modelKey });
+	}
+
+	async #stopStreaming(options: ToggleOptions): Promise<void> {
+		const stream = this.#stream;
+		const recorder = this.#streamRecorder;
+		if (!stream) {
+			this.#setState("idle", options);
+			return;
+		}
+		this.#setState("transcribing", options);
+		// Stop the mic first so no further audio is fed, then flush the worker.
+		try {
+			await recorder?.stop();
+		} catch (err) {
+			logger.debug("stt: streaming recorder stop failed", {
+				error: err instanceof Error ? err.message : String(err),
+			});
+		}
+		this.#streamRecorder = null;
+
+		let failed = false;
+		let finalText = "";
+		try {
+			finalText = (await stream.stop()).trim();
+		} catch (err) {
+			failed = true;
+			if (!this.#disposed) {
+				const msg = err instanceof Error ? err.message : "Transcription failed";
 				options.showWarning(msg);
-				logger.error("STT dependency setup failed", { error: msg });
-				return;
+				logger.error("STT live transcription failed", { error: msg });
 			}
 		}
+		if (this.#disposed) {
+			this.#cleanupStream();
+			return;
+		}
+		if (!this.#streamCommitted && finalText) {
+			this.#streamEditor?.commitVolatileText(this.#prefixed(finalText));
+			this.#streamCommitted = true;
+		} else {
+			this.#streamEditor?.clearVolatileText();
+		}
+		options.requestRender?.();
+		if (!failed) options.showStatus(this.#streamCommitted ? "" : "No speech detected.");
+		this.#cleanupStream();
+		this.#setState("idle", options);
+	}
+
+	#cleanupStream(): void {
+		this.#stream = null;
+		this.#streamRecorder = null;
+		this.#streamEditor = null;
+		this.#streamCommitted = false;
+		this.#streamAbort = null;
+	}
+
+	// ── Batch (single-shot) ─────────────────────────────────────────
+
+	async #startBatchRecording(options: ToggleOptions): Promise<void> {
 		const id = Snowflake.next();
 		this.#tempFile = path.join(os.tmpdir(), `prometheus-stt-${id}.wav`);
-
 		try {
 			this.#recordingHandle = await startRecording(this.#tempFile);
 			this.#setState("recording", options);
@@ -89,7 +261,7 @@ export class STTController {
 		}
 	}
 
-	async #stopAndTranscribe(editor: Editor, options: ToggleOptions): Promise<void> {
+	async #stopBatch(editor: Editor, options: ToggleOptions): Promise<void> {
 		const handle = this.#recordingHandle;
 		const tempFile = this.#tempFile;
 		this.#recordingHandle = null;
@@ -146,6 +318,13 @@ export class STTController {
 			this.#transcriptionAbort.abort();
 			this.#transcriptionAbort = null;
 		}
+		if (this.#streamAbort) {
+			this.#streamAbort.abort();
+			this.#streamAbort = null;
+		}
+		this.#stream?.cancel();
+		this.#streamRecorder?.stop().catch(() => {});
+		this.#cleanupStream();
 		if (this.#recordingHandle) {
 			this.#recordingHandle.stop().catch(() => {});
 			this.#recordingHandle = null;

package/src/stt/transcriber.ts

@@ -1,5 +1,7 @@
-import { $which, logger } from "@prometheus-ai/utils";
-import transcribeScript from "./transcribe.py" with { type: "text" };
+import { logger } from "@prometheus-ai/utils";
+import { sttClient } from "./asr-client";
+import { resolveSttModelSpec } from "./models";
+import { decodeWavToMono16k } from "./wav";
 
 export interface TranscribeOptions {
 	modelName?: string;
@@ -10,82 +12,49 @@ export interface TranscribeOptions {
 const TRANSCRIBE_TIMEOUT_MS = 120_000;
 
 /**
- * Find a usable Python command.
- */
-export function resolvePython(): string | null {
-	for (const cmd of ["python", "py", "python3"]) {
-		if ($which(cmd)) return cmd;
-	}
-	return null;
-}
-
-/**
- * Transcribe a WAV file using Python openai-whisper.
+ * Transcribe a WAV file using the local ONNX Whisper worker.
  *
- * Reads the WAV via Python's built-in `wave` module (no ffmpeg needed),
- * resamples to 16 kHz mono, and passes the numpy array directly to whisper.
+ * Decodes the WAV to a 16 kHz mono Float32Array in-process (no Python, no
+ * ffmpeg) and routes it to the warm speech worker, which keeps the model loaded
+ * across calls. Honors `options.signal` (abort) and applies an internal timeout
+ * with the same semantics as the previous Python path.
  */
 export async function transcribe(audioPath: string, options?: TranscribeOptions): Promise<string> {
 	const audioFile = Bun.file(audioPath);
 	if (audioFile.size < 100) {
 		throw new Error(`Audio file is empty or too small (${audioFile.size} bytes). Check microphone.`);
 	}
-
-	const pythonCmd = resolvePython();
-	if (!pythonCmd) {
-		throw new Error("Python not found. Install Python 3.8+ from https://python.org");
-	}
-
-	const modelName = options?.modelName ?? "base.en";
-	const language = options?.language ?? "en";
-
-	logger.debug("Transcribing with Python whisper", { pythonCmd, audioPath, modelName, language });
-
-	const proc = Bun.spawn([pythonCmd, "-c", transcribeScript, audioPath, modelName, language], {
-		stdout: "pipe",
-		stderr: "pipe",
-	});
-
-	if (options?.signal?.aborted) {
-		proc.kill();
-		options.signal.throwIfAborted();
-	}
-
-	const onAbort = () => proc.kill();
-	options?.signal?.addEventListener("abort", onAbort, { once: true });
-
-	let timedOut = false;
-
-	const killTimer = setTimeout(() => {
-		timedOut = true;
-		logger.error("Python whisper transcription timed out, killing process", { timeoutMs: TRANSCRIBE_TIMEOUT_MS });
-		proc.kill();
-	}, TRANSCRIBE_TIMEOUT_MS);
-
-	const exitCode = await proc.exited;
-	clearTimeout(killTimer);
-	options?.signal?.removeEventListener("abort", onAbort);
-
 	options?.signal?.throwIfAborted();
 
-	const stdout = await new Response(proc.stdout).text();
-	const stderr = await new Response(proc.stderr).text();
-
-	if (timedOut) {
-		throw new Error(`Transcription timed out after ${Math.round(TRANSCRIBE_TIMEOUT_MS / 1000)}s`);
-	}
+	const spec = resolveSttModelSpec(options?.modelName);
+	const language = options?.language || undefined;
+	const audio = decodeWavToMono16k(await audioFile.arrayBuffer());
+	if (audio.length === 0) return "";
+
+	logger.debug("Transcribing with local ONNX whisper", {
+		audioPath,
+		modelKey: spec.key,
+		repo: spec.repo,
+		language,
+		samples: audio.length,
+	});
 
-	if (exitCode !== 0) {
-		logger.error("Python whisper transcription failed", { exitCode, stderr: stderr.trim() });
-		if (stderr.includes("No module named 'whisper'")) {
-			throw new Error("openai-whisper not installed. Run: pip install openai-whisper");
+	// Bound runaway inference. Abort the request on timeout; the warm worker
+	// keeps the model loaded (the request promise just rejects).
+	const timeout = new AbortController();
+	const timer = setTimeout(() => timeout.abort(), TRANSCRIBE_TIMEOUT_MS);
+	const signal = options?.signal ? AbortSignal.any([options.signal, timeout.signal]) : timeout.signal;
+	try {
+		const text = (await sttClient.transcribe(spec.key, audio, { language, signal })).trim();
+		logger.debug("Transcription complete", { length: text.length });
+		return text;
+	} catch (error) {
+		if (timeout.signal.aborted && !options?.signal?.aborted) {
+			logger.error("Local whisper transcription timed out", { timeoutMs: TRANSCRIBE_TIMEOUT_MS });
+			throw new Error(`Transcription timed out after ${Math.round(TRANSCRIBE_TIMEOUT_MS / 1000)}s`);
 		}
-		// Show last line of stderr (the actual error, not the full traceback)
-		const lastLine = stderr.trim().split("\n").pop() ?? "";
-		throw new Error(`Transcription failed: ${lastLine}`);
+		throw error;
+	} finally {
+		clearTimeout(timer);
 	}
-
-	const text = stdout.trim();
-	logger.debug("Transcription complete", { length: text.length });
-	return text;
 }

package/src/stt/wav.ts

@@ -0,0 +1,173 @@
+/**
+ * Minimal WAV (RIFF/PCM) decoder producing the Float32Array @ 16 kHz mono that
+ * transformers.js `automatic-speech-recognition` expects. Ports the decode/
+ * mono-mix/resample logic from the retired Python `transcribe.py` (which read
+ * via the stdlib `wave` module) so STT no longer shells out to Python.
+ *
+ * Supported sample formats: PCM uint8 (8-bit), int16 (16-bit), int32 (32-bit),
+ * and IEEE float32 (format tag 3). Any number of channels is mixed down to mono.
+ */
+
+/** transformers.js Whisper feature extractor operates at 16 kHz. */
+export const TARGET_SAMPLE_RATE = 16_000;
+
+const WAV_FORMAT_PCM = 1;
+const WAV_FORMAT_IEEE_FLOAT = 3;
+const WAV_FORMAT_EXTENSIBLE = 0xfffe;
+
+interface WavData {
+	format: number;
+	channels: number;
+	sampleRate: number;
+	bitsPerSample: number;
+	/** Raw PCM/float bytes from the `data` chunk. */
+	samples: DataView;
+}
+
+function readFourCc(view: DataView, offset: number): string {
+	return String.fromCharCode(
+		view.getUint8(offset),
+		view.getUint8(offset + 1),
+		view.getUint8(offset + 2),
+		view.getUint8(offset + 3),
+	);
+}
+
+/** Parse the RIFF container, returning the `fmt ` parameters and `data` bytes. */
+function parseWav(buffer: ArrayBuffer): WavData {
+	const view = new DataView(buffer);
+	if (buffer.byteLength < 12 || readFourCc(view, 0) !== "RIFF" || readFourCc(view, 8) !== "WAVE") {
+		throw new Error("Not a RIFF/WAVE file");
+	}
+
+	let format: number | undefined;
+	let channels = 0;
+	let sampleRate = 0;
+	let bitsPerSample = 0;
+	let samples: DataView | undefined;
+
+	// Chunks begin after the 12-byte RIFF/WAVE header; each is an 8-byte header
+	// (4-char id + uint32 LE size) followed by `size` bytes padded to even.
+	let offset = 12;
+	while (offset + 8 <= buffer.byteLength) {
+		const id = readFourCc(view, offset);
+		const size = view.getUint32(offset + 4, true);
+		const body = offset + 8;
+		if (id === "fmt ") {
+			format = view.getUint16(body, true);
+			channels = view.getUint16(body + 2, true);
+			sampleRate = view.getUint32(body + 4, true);
+			bitsPerSample = view.getUint16(body + 14, true);
+			// WAVE_FORMAT_EXTENSIBLE (ffmpeg & friends): the real codec is the
+			// first 2 bytes of the SubFormat GUID in the fmt extension.
+			if (format === WAV_FORMAT_EXTENSIBLE && size >= 40) format = view.getUint16(body + 24, true);
+		} else if (id === "data") {
+			const length = Math.min(size, buffer.byteLength - body);
+			samples = new DataView(buffer, body, length);
+		}
+		offset = body + size + (size % 2);
+	}
+
+	if (format === undefined || samples === undefined || channels < 1 || sampleRate < 1) {
+		throw new Error("WAV file missing fmt/data chunks");
+	}
+	return { format, channels, sampleRate, bitsPerSample, samples };
+}
+
+/** Decode raw PCM/float bytes into interleaved normalized [-1, 1] float samples. */
+function decodeSamples(wav: WavData): Float32Array {
+	const { format, bitsPerSample, samples } = wav;
+	const view = samples;
+	if (format === WAV_FORMAT_IEEE_FLOAT && bitsPerSample === 32) {
+		const count = Math.floor(view.byteLength / 4);
+		const out = new Float32Array(count);
+		for (let i = 0; i < count; i += 1) out[i] = view.getFloat32(i * 4, true);
+		return out;
+	}
+	if (format !== WAV_FORMAT_PCM) {
+		throw new Error(`Unsupported WAV format tag: ${format}`);
+	}
+	if (bitsPerSample === 16) {
+		const count = Math.floor(view.byteLength / 2);
+		const out = new Float32Array(count);
+		for (let i = 0; i < count; i += 1) out[i] = view.getInt16(i * 2, true) / 32_768;
+		return out;
+	}
+	if (bitsPerSample === 8) {
+		// 8-bit PCM is unsigned, centered at 128.
+		const count = view.byteLength;
+		const out = new Float32Array(count);
+		for (let i = 0; i < count; i += 1) out[i] = (view.getUint8(i) - 128) / 128;
+		return out;
+	}
+	if (bitsPerSample === 32) {
+		const count = Math.floor(view.byteLength / 4);
+		const out = new Float32Array(count);
+		for (let i = 0; i < count; i += 1) out[i] = view.getInt32(i * 4, true) / 2_147_483_648;
+		return out;
+	}
+	throw new Error(`Unsupported PCM sample width: ${bitsPerSample} bits`);
+}
+
+/** Average interleaved channels down to a single mono track. */
+function mixToMono(interleaved: Float32Array, channels: number): Float32Array {
+	if (channels <= 1) return interleaved;
+	const frames = Math.floor(interleaved.length / channels);
+	const out = new Float32Array(frames);
+	for (let frame = 0; frame < frames; frame += 1) {
+		let sum = 0;
+		for (let channel = 0; channel < channels; channel += 1) sum += interleaved[frame * channels + channel]!;
+		out[frame] = sum / channels;
+	}
+	return out;
+}
+
+/**
+ * Resample via linear interpolation, mirroring the Python `np.interp` over
+ * `linspace(0, n-1, targetLen)` against `arange(n)`.
+ */
+export function resampleLinear(input: Float32Array, fromRate: number, toRate: number): Float32Array {
+	if (fromRate === toRate || input.length === 0) return input;
+	const n = input.length;
+	const targetLen = Math.max(1, Math.floor((n * toRate) / fromRate));
+	const out = new Float32Array(targetLen);
+	if (targetLen === 1) {
+		out[0] = input[0]!;
+		return out;
+	}
+	const step = (n - 1) / (targetLen - 1);
+	for (let i = 0; i < targetLen; i += 1) {
+		const pos = i * step;
+		const lo = Math.floor(pos);
+		const hi = Math.min(lo + 1, n - 1);
+		const frac = pos - lo;
+		out[i] = input[lo]! * (1 - frac) + input[hi]! * frac;
+	}
+	return out;
+}
+
+/**
+ * Decode a WAV byte buffer into a 16 kHz mono Float32Array suitable for the
+ * transformers.js Whisper pipeline.
+ */
+export function decodeWavToMono16k(buffer: ArrayBuffer): Float32Array {
+	const wav = parseWav(buffer);
+	const interleaved = decodeSamples(wav);
+	const mono = mixToMono(interleaved, wav.channels);
+	return resampleLinear(mono, wav.sampleRate, TARGET_SAMPLE_RATE);
+}
+
+/**
+ * Decode interleaved little-endian signed 16-bit PCM bytes into normalized
+ * [-1, 1] mono float samples. The live recorder streams raw s16le frames from
+ * sox/ffmpeg/arecord stdout (no RIFF container), so this is the hot-path
+ * counterpart to {@link decodeWavToMono16k}. `bytes` MUST be 2-byte aligned;
+ * callers buffer any trailing odd byte across chunk boundaries.
+ */
+export function decodePcmS16LE(bytes: Uint8Array): Float32Array {
+	const count = bytes.length >>> 1;
+	const view = new DataView(bytes.buffer, bytes.byteOffset, count * 2);
+	const out = new Float32Array(count);
+	for (let i = 0; i < count; i += 1) out[i] = view.getInt16(i * 2, true) / 32_768;
+	return out;
+}