@prometheus-ai/agent 0.5.3 → 0.5.8

package/src/stt/downloader.ts

@@ -1,6 +1,10 @@
-import { $which, logger } from "@prometheus-ai/utils";
-import { $ } from "bun";
-import { resolvePython } from "./transcriber";
+import * as fs from "node:fs/promises";
+import * as path from "node:path";
+import { getTinyModelsCacheDir } from "@prometheus-ai/utils";
+import { sttClient } from "./asr-client";
+import type { SttProgressStatus } from "./asr-protocol";
+import { resolveSttModelSpec } from "./models";
+import { ensureRecorder } from "./recorder";
 
 export interface DownloadProgress {
 	stage: string;
@@ -9,63 +13,119 @@ export interface DownloadProgress {
 
 export interface EnsureOptions {
 	modelName?: string;
+	signal?: AbortSignal;
 	onProgress?: (progress: DownloadProgress) => void;
 }
 
-// ── Recording tool ─────────────────────────────────────────────────
+// ── ONNX Whisper model ─────────────────────────────────────────────
 
-async function ensureRecordingTool(options?: EnsureOptions): Promise<void> {
-	if ($which("sox")) return;
-	if ($which("ffmpeg")) return;
-	if (process.platform === "linux" && $which("arecord")) return;
+/**
+ * Real-progress event for a speech-model download, surfaced to UI callers.
+ * `percent` is an integer 0–100 aggregated across all model files (encoder +
+ * decoder shards), so it advances monotonically toward completion.
+ */
+export interface SttDownloadProgress {
+	status: SttProgressStatus;
+	/** Integer 0–100 aggregated across files. */
+	percent: number;
+	/** Bytes downloaded so far across all files. */
+	loaded: number;
+	/** Total bytes across all files seen so far. */
+	total: number;
+	/** The file currently downloading, when known. */
+	file?: string;
+	repo: string;
+	label: string;
+}
 
-	// Windows: PowerShell mciSendString is always available as fallback
-	if (process.platform === "win32") {
-		// Try to get ffmpeg for better quality, but don't block on failure
-		options?.onProgress?.({ stage: "Trying to install FFmpeg via winget..." });
-		const result = await $`winget install --id Gyan.FFmpeg -e --accept-source-agreements --accept-package-agreements`
-			.quiet()
-			.nothrow();
-		if (result.exitCode === 0) {
-			logger.debug("FFmpeg installed via winget");
+/**
+ * Whether the selected model is already present in the local cache. For
+ * transformers.js Whisper tiers a complete download leaves `config.json` plus
+ * the `onnx/` weight files (a bare `config.json` from an interrupted fetch reads
+ * as not-cached); for sherpa-onnx tiers every model file (encoder/decoder/joiner
+ * + tokens) must be present (`.part` sidecars from an interrupted fetch are
+ * ignored).
+ */
+export async function isSttModelCached(key: string): Promise<boolean> {
+	const spec = resolveSttModelSpec(key);
+	const repoDir = path.join(getTinyModelsCacheDir(), spec.repo);
+	if (spec.engine === "sherpa") {
+		try {
+			const root = new Set(await fs.readdir(repoDir));
+			for (const role in spec.files) {
+				if (!root.has(spec.files[role as keyof typeof spec.files])) return false;
+			}
+			return true;
+		} catch {
+			return false;
 		}
-		return;
 	}
-
-	throw new Error(
-		"No audio recording tool found. Install SoX: sudo apt install sox, or FFmpeg: sudo apt install ffmpeg",
-	);
-}
-
-// ── Python whisper ─────────────────────────────────────────────────
-
-async function ensurePythonWhisper(options?: EnsureOptions): Promise<void> {
-	const pythonCmd = resolvePython();
-	if (!pythonCmd) {
-		throw new Error("Python not found. Install Python 3.8+ from https://python.org");
+	try {
+		const root = await fs.readdir(repoDir);
+		if (!root.includes("config.json")) return false;
+		const onnxFiles = await fs.readdir(path.join(repoDir, "onnx")).catch(() => [] as string[]);
+		return onnxFiles.some(file => file.endsWith(".onnx"));
+	} catch {
+		return false;
 	}
+}
 
-	// Check if whisper module is already importable
-	const check = Bun.spawnSync([pythonCmd, "-c", "import whisper"], {
-		stdout: "pipe",
-		stderr: "pipe",
+/**
+ * Download (or warm from cache) the selected ONNX Whisper model via the speech
+ * worker, resolving once the model is fully present and loaded. Streams real
+ * Hub progress with an aggregated integer percent. Rejects if the worker cannot
+ * obtain the model. Safe to call non-interactively.
+ */
+export async function downloadSttModel(
+	key: string,
+	onProgress?: (progress: SttDownloadProgress) => void,
+	options?: { signal?: AbortSignal },
+): Promise<void> {
+	const spec = resolveSttModelSpec(key);
+	const files = new Map<string, { loaded: number; total: number }>();
+	const ok = await sttClient.downloadModel(spec.key, {
+		signal: options?.signal,
+		onProgress: event => {
+			if ((event.status === "progress" || event.status === "progress_total") && event.file) {
+				if (typeof event.loaded === "number" && typeof event.total === "number" && event.total > 0) {
+					files.set(event.file, { loaded: event.loaded, total: event.total });
+				}
+			}
+			let loaded = 0;
+			let total = 0;
+			for (const file of files.values()) {
+				loaded += file.loaded;
+				total += file.total;
+			}
+			const settled = event.status === "ready" || event.status === "done";
+			const percent = total > 0 ? Math.min(100, Math.round((loaded / total) * 100)) : settled ? 100 : 0;
+			onProgress?.({
+				status: event.status,
+				percent,
+				loaded,
+				total,
+				file: event.file,
+				repo: spec.repo,
+				label: spec.label,
+			});
+		},
 	});
-	if (check.exitCode === 0) return;
-
-	options?.onProgress?.({ stage: "Installing openai-whisper (this may take a few minutes)..." });
-	logger.debug("Installing openai-whisper via pip");
-
-	const install = await $`${pythonCmd} -m pip install -q openai-whisper`.quiet().nothrow();
-	if (install.exitCode !== 0) {
-		const stderr = install.stderr.toString().trim();
-		throw new Error(`Failed to install openai-whisper: ${stderr.split("\n").pop()}`);
-	}
-	logger.debug("openai-whisper installed successfully");
+	if (!ok) throw new Error(`Failed to download speech model (${spec.repo}). Check your network connection.`);
 }
 
 // ── Public API ─────────────────────────────────────────────────────
 
 export async function ensureSTTDependencies(options?: EnsureOptions): Promise<void> {
-	await ensureRecordingTool(options);
-	await ensurePythonWhisper(options);
+	await ensureRecorder(progress => options?.onProgress?.(progress), options?.signal);
+	await downloadSttModel(
+		resolveSttModelSpec(options?.modelName).key,
+		progress => {
+			const stage =
+				progress.status === "ready" || progress.status === "done"
+					? `Speech model ${progress.label} ready`
+					: `Downloading speech model ${progress.label}`;
+			options?.onProgress?.({ stage, percent: progress.percent });
+		},
+		{ signal: options?.signal },
+	);
 }

package/src/stt/endpointer.ts

@@ -0,0 +1,259 @@
+/**
+ * Energy-based speech endpointer for live transcription.
+ *
+ * The on-device ASR models we ship are non-streaming: the sherpa-onnx Parakeet
+ * recognizer and the transformers.js Whisper pipelines both decode a complete
+ * waveform in one shot. To transcribe *while the user is still speaking*, this
+ * splits the continuous 16 kHz mono float stream into speech segments at natural
+ * pauses — each segment is decoded and committed as it finalizes, and the
+ * in-progress segment is re-decoded periodically for a volatile live preview.
+ *
+ * Segmentation is pure short-time-energy VAD with an adaptive noise floor, so it
+ * needs no extra model and is engine-agnostic (it runs the same way whether the
+ * downstream model is sherpa or transformers). It is deliberately simple and
+ * fully deterministic so it can be unit-tested with synthetic signals.
+ */
+
+/** Tunable thresholds for {@link StreamEndpointer}. All durations in ms. */
+export interface EndpointerConfig {
+	/** Input sample rate (the recorder always delivers 16 kHz mono). */
+	sampleRate: number;
+	/** Short-time analysis frame size. */
+	frameMs: number;
+	/** Trailing silence inside a segment that finalizes (commits) it. */
+	endSilenceMs: number;
+	/** Shortest speech run that is committed; shorter runs are discarded as noise. */
+	minSpeechMs: number;
+	/** Hard cap on segment length so long pause-free speech still commits periodically. */
+	maxSegmentMs: number;
+	/** Audio retained before onset so the first phoneme of a segment is never clipped. */
+	preRollMs: number;
+	/** Cadence of volatile partial emissions for the in-progress segment. */
+	partialIntervalMs: number;
+	/** Speech threshold is `max(minThreshold, noiseFloor * energyRatio)`. */
+	energyRatio: number;
+	/** EMA weight tracking the ambient noise floor on non-speech frames. */
+	floorAttack: number;
+	/** Absolute RMS floor so a near-silent room never trips speech detection. */
+	minThreshold: number;
+}
+
+export const DEFAULT_ENDPOINTER_CONFIG: EndpointerConfig = {
+	sampleRate: 16_000,
+	frameMs: 30,
+	endSilenceMs: 600,
+	minSpeechMs: 200,
+	maxSegmentMs: 12_000,
+	preRollMs: 240,
+	partialIntervalMs: 450,
+	energyRatio: 2.5,
+	floorAttack: 0.05,
+	minThreshold: 0.008,
+};
+
+/**
+ * Emitted by {@link StreamEndpointer.push} / {@link StreamEndpointer.flush}.
+ * `partial` is the volatile in-progress segment (decode and show as preview,
+ * never commit); `segment` is a finalized run (decode and commit once).
+ */
+export type EndpointerEvent = { kind: "partial"; audio: Float32Array } | { kind: "segment"; audio: Float32Array };
+
+/** Append-growable Float32 buffer (amortized O(1) push, no per-frame realloc). */
+class FloatBuffer {
+	#data = new Float32Array(0);
+	#len = 0;
+
+	get length(): number {
+		return this.#len;
+	}
+
+	push(samples: Float32Array): void {
+		const needed = this.#len + samples.length;
+		if (needed > this.#data.length) {
+			const next = new Float32Array(Math.max(this.#data.length * 2, needed, 1 << 14));
+			next.set(this.#data.subarray(0, this.#len));
+			this.#data = next;
+		}
+		this.#data.set(samples, this.#len);
+		this.#len += samples.length;
+	}
+
+	/** Copy `[0, end)` into a fresh array the caller can retain. */
+	take(end = this.#len): Float32Array {
+		return this.#data.slice(0, Math.max(0, Math.min(end, this.#len)));
+	}
+
+	reset(): void {
+		this.#len = 0;
+	}
+}
+
+function rms(frame: Float32Array): number {
+	let sum = 0;
+	for (let i = 0; i < frame.length; i += 1) sum += frame[i]! * frame[i]!;
+	return Math.sqrt(sum / Math.max(1, frame.length));
+}
+
+export class StreamEndpointer {
+	readonly #cfg: EndpointerConfig;
+	readonly #frameSamples: number;
+	readonly #preRollSamples: number;
+
+	#leftover = new Float32Array(0);
+	#inSpeech = false;
+	#noiseFloor: number;
+	#silenceMs = 0;
+	#segmentMs = 0;
+	#msSincePartial = 0;
+	#partialDirty = false;
+
+	readonly #segment = new FloatBuffer();
+	/** Ring of the most recent pre-onset frames, used as segment pre-roll. */
+	readonly #preRoll = new FloatBuffer();
+
+	constructor(config: Partial<EndpointerConfig> = {}) {
+		this.#cfg = { ...DEFAULT_ENDPOINTER_CONFIG, ...config };
+		this.#frameSamples = Math.max(1, Math.round((this.#cfg.sampleRate * this.#cfg.frameMs) / 1000));
+		this.#preRollSamples = Math.max(0, Math.round((this.#cfg.sampleRate * this.#cfg.preRollMs) / 1000));
+		this.#noiseFloor = this.#cfg.minThreshold;
+	}
+
+	/** Feed newly-captured samples; returns ordered partial/segment events. */
+	push(samples: Float32Array): EndpointerEvent[] {
+		const events: EndpointerEvent[] = [];
+		// Prepend the carried-over tail, then consume whole frames.
+		let buf: Float32Array;
+		if (this.#leftover.length === 0) {
+			buf = samples;
+		} else {
+			buf = new Float32Array(this.#leftover.length + samples.length);
+			buf.set(this.#leftover, 0);
+			buf.set(samples, this.#leftover.length);
+		}
+		let offset = 0;
+		for (; offset + this.#frameSamples <= buf.length; offset += this.#frameSamples) {
+			this.#processFrame(buf.subarray(offset, offset + this.#frameSamples), events);
+		}
+		this.#leftover = buf.slice(offset);
+		return events;
+	}
+
+	/** End the stream; returns a trailing committed segment if one is pending. */
+	flush(): EndpointerEvent[] {
+		const events: EndpointerEvent[] = [];
+		if (this.#inSpeech && this.#leftover.length > 0) {
+			this.#segment.push(this.#leftover);
+			this.#segmentMs += (this.#leftover.length / this.#cfg.sampleRate) * 1000;
+		}
+		this.#leftover = new Float32Array(0);
+		if (this.#inSpeech) {
+			const speechMs = this.#segmentMs - this.#silenceMs;
+			if (speechMs >= this.#cfg.minSpeechMs) {
+				events.push({ kind: "segment", audio: this.#segment.take(this.#endpointKeep()) });
+			}
+		}
+		this.#reset();
+		return events;
+	}
+
+	#processFrame(frame: Float32Array, events: EndpointerEvent[]): void {
+		const energy = rms(frame);
+		const threshold = Math.max(this.#cfg.minThreshold, this.#noiseFloor * this.#cfg.energyRatio);
+		const voiced = energy > threshold;
+		// Track ambient noise on non-speech frames only, so loud speech never
+		// inflates the floor (which would make the tail of an utterance read as
+		// silence and clip the segment short).
+		if (!voiced) {
+			this.#noiseFloor = this.#noiseFloor * (1 - this.#cfg.floorAttack) + energy * this.#cfg.floorAttack;
+		}
+
+		if (!this.#inSpeech) {
+			this.#preRoll.push(frame);
+			// Keep only the most recent pre-roll window.
+			if (this.#preRoll.length > this.#preRollSamples) {
+				const tail = this.#preRoll.take().slice(this.#preRoll.length - this.#preRollSamples);
+				this.#preRoll.reset();
+				this.#preRoll.push(tail);
+			}
+			if (voiced) this.#beginSegment(frame);
+			return;
+		}
+
+		this.#segment.push(frame);
+		this.#segmentMs += this.#cfg.frameMs;
+		this.#msSincePartial += this.#cfg.frameMs;
+		if (voiced) {
+			this.#silenceMs = 0;
+			this.#partialDirty = true;
+		} else {
+			this.#silenceMs += this.#cfg.frameMs;
+		}
+
+		if (this.#silenceMs >= this.#cfg.endSilenceMs) {
+			this.#finalizeSegment(events);
+			return;
+		}
+		if (this.#segmentMs >= this.#cfg.maxSegmentMs) {
+			// Pause-free long speech: commit what we have and continue a fresh
+			// segment so output keeps flowing.
+			events.push({ kind: "segment", audio: this.#segment.take() });
+			this.#segment.reset();
+			this.#segmentMs = 0;
+			this.#silenceMs = 0;
+			this.#msSincePartial = 0;
+			this.#partialDirty = false;
+			return;
+		}
+		if (this.#partialDirty && this.#msSincePartial >= this.#cfg.partialIntervalMs) {
+			events.push({ kind: "partial", audio: this.#segment.take() });
+			this.#msSincePartial = 0;
+			this.#partialDirty = false;
+		}
+	}
+
+	#beginSegment(onsetFrame: Float32Array): void {
+		this.#inSpeech = true;
+		this.#segment.reset();
+		const preRoll = this.#preRoll.take();
+		if (preRoll.length > 0) this.#segment.push(preRoll);
+		this.#segment.push(onsetFrame);
+		this.#preRoll.reset();
+		this.#silenceMs = 0;
+		this.#segmentMs = (this.#segment.length / this.#cfg.sampleRate) * 1000;
+		this.#msSincePartial = 0;
+		this.#partialDirty = true;
+	}
+
+	#finalizeSegment(events: EndpointerEvent[]): void {
+		const speechMs = this.#segmentMs - this.#silenceMs;
+		if (speechMs >= this.#cfg.minSpeechMs) {
+			events.push({ kind: "segment", audio: this.#segment.take(this.#endpointKeep()) });
+		}
+		this.#inSpeech = false;
+		this.#segment.reset();
+		this.#silenceMs = 0;
+		this.#segmentMs = 0;
+		this.#msSincePartial = 0;
+		this.#partialDirty = false;
+	}
+
+	/** Samples to keep when committing on silence: drop most of the trailing
+	 *  silence but leave a short tail so the final word is not cut. */
+	#endpointKeep(): number {
+		const tailMs = Math.min(this.#silenceMs, 120);
+		const dropMs = Math.max(0, this.#silenceMs - tailMs);
+		const drop = Math.round((this.#cfg.sampleRate * dropMs) / 1000);
+		return Math.max(0, this.#segment.length - drop);
+	}
+
+	#reset(): void {
+		this.#inSpeech = false;
+		this.#segment.reset();
+		this.#preRoll.reset();
+		this.#silenceMs = 0;
+		this.#segmentMs = 0;
+		this.#msSincePartial = 0;
+		this.#partialDirty = false;
+		this.#noiseFloor = this.#cfg.minThreshold;
+	}
+}

package/src/stt/index.ts

@@ -1,3 +1,7 @@
+export * from "./asr-client";
+export * from "./asr-protocol";
 export * from "./downloader";
-export * from "./setup";
+export * from "./models";
 export * from "./stt-controller";
+export * from "./transcriber";
+export * from "./wav";

package/src/stt/models.ts

@@ -0,0 +1,150 @@
+import type { TinyModelDtype } from "../tiny/dtype";
+
+/**
+ * On-device speech-to-text model registry. Each tier maps a stable settings key
+ * onto a locally-runnable ASR model and the engine that loads it:
+ *
+ * - `transformers` — a transformers.js / ONNX Whisper repo, loaded by the
+ *   `@huggingface/transformers` `automatic-speech-recognition` pipeline.
+ * - `sherpa` — a sherpa-onnx (Next-gen Kaldi) offline model, loaded by the
+ *   native `sherpa-onnx-node` addon. Used for NVIDIA Parakeet, the Open ASR
+ *   Leaderboard accuracy/speed leader.
+ *
+ * The worker resolves the spec by key and loads the model lazily (kept warm
+ * afterwards). Both engines run inside the hard-killed subprocess worker.
+ */
+
+/** ASR runtime that loads a given tier's model. */
+export type SttEngine = "transformers" | "sherpa";
+
+interface SttModelBase {
+	/** Stable key persisted in `stt.modelName` and sent over the worker protocol. */
+	key: string;
+	engine: SttEngine;
+	/** Hugging Face repo id (transformers.js ONNX repo, or sherpa-onnx model repo). */
+	repo: string;
+	/** English-only checkpoint: rejects a configured source `language`. */
+	englishOnly: boolean;
+	label: string;
+	description: string;
+	/** Approximate on-disk download size for the shipped weights (UI hint). */
+	sizeHint: string;
+}
+
+/** A Whisper-family tier loaded via the transformers.js ASR pipeline. */
+export interface TransformersSttModelSpec extends SttModelBase {
+	engine: "transformers";
+	/** ONNX precision used unless overridden by `PROMETHEUS_TINY_DTYPE` / `providers.tinyModelDtype`. */
+	dtype: TinyModelDtype;
+}
+
+/** A sherpa-onnx offline tier (e.g. NeMo Parakeet transducer) loaded natively. */
+export interface SherpaSttModelSpec extends SttModelBase {
+	engine: "sherpa";
+	/** sherpa-onnx offline model family (e.g. `nemo_transducer`). */
+	modelType: string;
+	/** Model files (relative to the repo root) fetched into the local cache. */
+	files: { encoder: string; decoder: string; joiner: string; tokens: string };
+}
+
+export type SttModelSpec = TransformersSttModelSpec | SherpaSttModelSpec;
+
+/**
+ * Speech model tiers, ordered light → SoTA. Defaults to {@link DEFAULT_STT_MODEL_KEY}.
+ * `fast`/`balanced`/`turbo` are multilingual Whisper checkpoints on transformers.js;
+ * `parakeet` is NVIDIA Parakeet TDT 0.6B v3 on sherpa-onnx — the Open ASR
+ * Leaderboard leader (lower WER and far higher throughput than Whisper).
+ */
+export const STT_MODELS = [
+	{
+		key: "fast",
+		engine: "transformers",
+		repo: "onnx-community/whisper-base",
+		dtype: "q8",
+		englishOnly: false,
+		label: "Fast (Whisper base)",
+		description: "Whisper base, multilingual. Smallest + fastest; lowest accuracy. Best for low-resource machines.",
+		sizeHint: "~60 MB",
+	},
+	{
+		key: "balanced",
+		engine: "transformers",
+		repo: "onnx-community/whisper-small",
+		dtype: "q8",
+		englishOnly: false,
+		label: "Balanced (Whisper small)",
+		description: "Whisper small, multilingual. More accurate than Fast, still light on CPU/RAM.",
+		sizeHint: "~190 MB",
+	},
+	{
+		key: "turbo",
+		engine: "transformers",
+		repo: "onnx-community/whisper-large-v3-turbo",
+		dtype: "q4",
+		englishOnly: false,
+		label: "Turbo (Whisper large-v3)",
+		description: "Whisper large-v3-turbo, 99 languages. Widest language coverage; large download, slower.",
+		sizeHint: "~600 MB",
+	},
+	{
+		key: "parakeet",
+		engine: "sherpa",
+		repo: "csukuangfj/sherpa-onnx-nemo-parakeet-tdt-0.6b-v3-int8",
+		modelType: "nemo_transducer",
+		files: {
+			encoder: "encoder.int8.onnx",
+			decoder: "decoder.int8.onnx",
+			joiner: "joiner.int8.onnx",
+			tokens: "tokens.txt",
+		},
+		englishOnly: false,
+		label: "Parakeet TDT v3 (SoTA)",
+		description:
+			"NVIDIA Parakeet TDT 0.6B v3, 25 languages. Open ASR Leaderboard leader — best accuracy and far fastest decoding. Default.",
+		sizeHint: "~680 MB",
+	},
+] as const satisfies readonly SttModelSpec[];
+
+/**
+ * SoTA default — NVIDIA Parakeet TDT 0.6B v3 (sherpa-onnx). Tops the Open ASR
+ * Leaderboard on accuracy while decoding ~20× faster than Whisper large-v3.
+ */
+export const DEFAULT_STT_MODEL_KEY = "parakeet";
+
+export type SttModelKey = (typeof STT_MODELS)[number]["key"];
+
+/** A concrete entry from {@link STT_MODELS}; `key` is the literal tier union. */
+export type SttModel = (typeof STT_MODELS)[number];
+
+export const STT_MODEL_VALUES = ["fast", "balanced", "turbo", "parakeet"] as const satisfies readonly SttModelKey[];
+
+type MissingSttModelValue = Exclude<SttModelKey, (typeof STT_MODEL_VALUES)[number]>;
+type ExtraSttModelValue = Exclude<(typeof STT_MODEL_VALUES)[number], SttModelKey>;
+const STT_MODEL_VALUES_MATCH_REGISTRY: MissingSttModelValue extends never
+	? ExtraSttModelValue extends never
+		? true
+		: never
+	: never = true;
+void STT_MODEL_VALUES_MATCH_REGISTRY;
+
+export const STT_MODEL_OPTIONS = STT_MODELS.map(({ key, label, description }) => ({
+	value: key,
+	label,
+	description,
+})) satisfies ReadonlyArray<{ value: SttModelKey; label: string; description: string }>;
+
+export function isSttModelKey(value: string): value is SttModelKey {
+	return STT_MODELS.some(model => model.key === value);
+}
+
+export function getSttModelSpec(key: string): SttModel | undefined {
+	return STT_MODELS.find(model => model.key === key);
+}
+
+/**
+ * Resolve a (possibly stale or legacy) `stt.modelName` value onto a concrete
+ * spec, falling back to the SoTA default when the key is unknown.
+ */
+export function resolveSttModelSpec(key: string | undefined): SttModel {
+	return (key !== undefined ? getSttModelSpec(key) : undefined) ?? getSttModelSpec(DEFAULT_STT_MODEL_KEY)!;
+}