@oh-my-pi/pi-coding-agent 17.0.9 → 17.1.1

package/src/stt/stt-controller.ts

@@ -1,22 +1,10 @@
-import * as fs from "node:fs/promises";
-import * as os from "node:os";
-import * as path from "node:path";
-import { logger, Snowflake } from "@oh-my-pi/pi-utils";
+import { AudioCapture } from "@oh-my-pi/pi-natives";
+import { logger } from "@oh-my-pi/pi-utils";
 import { settings } from "../config/settings";
 import { type SttStreamHandle, sttClient } from "./asr-client";
 import { downloadSttModel, isSttModelCached } from "./downloader";
 import { resolveSttModelSpec } from "./models";
-import {
-	detectRecorder,
-	ensureRecorder,
-	type RecordingHandle,
-	type StreamingRecordingHandle,
-	startRecording,
-	startStreamingRecording,
-	verifyRecordingFile,
-} from "./recorder";
 import { evaluateSubmitTrigger } from "./submit-trigger";
-import { transcribe } from "./transcriber";
 
 export type SttState = "idle" | "recording" | "transcribing";
 
@@ -38,26 +26,34 @@ interface Editor {
 	deleteBeforeCursor(count: number): void;
 }
 
+interface CaptureHandle {
+	stop(): void;
+}
+
+type CaptureFactory = (onAudio: (error: Error | null, samples: Float32Array) => void) => CaptureHandle;
+
+/** Coordinates native microphone capture with incremental local transcription. */
 export class STTController {
 	#state: SttState = "idle";
 	#resolvedModelKey: string | null = null;
 	#toggling = false;
 	#stopAfterStart = false;
 	#disposed = false;
-
-	// Batch (single-shot) capture.
-	#recordingHandle: RecordingHandle | null = null;
-	#tempFile: string | null = null;
-	#transcriptionAbort: AbortController | null = null;
+	readonly #createCapture: CaptureFactory;
 
 	// Live streaming capture.
 	#stream: SttStreamHandle | null = null;
-	#streamRecorder: StreamingRecordingHandle | null = null;
+	#streamRecorder: CaptureHandle | null = null;
 	#streamEditor: Editor | null = null;
 	#streamCommitted = false;
 	#streamAbort: AbortController | null = null;
 	#streamUtterance = "";
 
+	/** Creates a controller; tests may replace the hardware capture boundary. */
+	constructor(createCapture: CaptureFactory = onAudio => new AudioCapture(16_000, onAudio)) {
+		this.#createCapture = createCapture;
+	}
+
 	get state(): SttState {
 		return this.#state;
 	}
@@ -79,7 +75,7 @@ export class STTController {
 					await this.#start(editor, options);
 					break;
 				case "recording":
-					await this.#stop(editor, options);
+					await this.#stop(options);
 					break;
 				case "transcribing":
 					options.showStatus("Transcription in progress...");
@@ -87,7 +83,7 @@ export class STTController {
 			}
 			if (this.#stopAfterStart && this.#state === "recording") {
 				this.#stopAfterStart = false;
-				await this.#stop(editor, options);
+				await this.#stop(options);
 			} else if (this.#state !== "recording") {
 				this.#stopAfterStart = false;
 			}
@@ -103,19 +99,13 @@ export class STTController {
 		// (with progress) instead of blocking silently at stop.
 		if (this.#resolvedModelKey === modelKey) return true;
 		try {
-			// Only clear the status line if we actually wrote to it: the cached
-			// fast path (recorder on PATH, model present) emits nothing, so an
-			// unconditional clear would be a stray write.
+			// Only clear the status line when preflight emitted progress; the
+			// cached-model fast path emits nothing.
 			let wroteStatus = false;
 			const status = (msg: string): void => {
 				wroteStatus = true;
 				options.showStatus(msg);
 			};
-			// A recorder is required to capture audio; startRecording /
-			// startStreamingRecording only *detect* a recorder and throw when none
-			// exists, so provision one here. Instant when sox/ffmpeg/arecord is on
-			// PATH — only a first-run static-ffmpeg download actually blocks.
-			await ensureRecorder(p => status(p.stage + (p.percent != null ? ` (${p.percent}%)` : "")));
 			// Loading the multi-hundred-MB speech model into the worker is what made
 			// the old "Checking STT dependencies…" step slow. Don't pay it before
 			// recording: when the weights are already cached, start now and warm the
@@ -157,30 +147,15 @@ export class STTController {
 
 	async #start(editor: Editor, options: ToggleOptions): Promise<void> {
 		if (!(await this.#ensureDeps(options))) return;
-		// Live transcription needs a recorder that can pipe PCM; the Windows
-		// PowerShell mci fallback records to a file, so it stays single-shot.
-		if (this.#recorderCanStream()) {
-			await this.#startStreaming(editor, options);
-			return;
-		}
-		await this.#startBatchRecording(options);
+		await this.#startStreaming(editor, options);
 	}
 
-	async #stop(editor: Editor, options: ToggleOptions): Promise<void> {
-		if (this.#stream) {
-			await this.#stopStreaming(options);
-			return;
-		}
-		await this.#stopBatch(editor, options);
+	async #stop(options: ToggleOptions): Promise<void> {
+		await this.#stopStreaming(options);
 	}
 
 	// ── Live streaming ──────────────────────────────────────────────
 
-	#recorderCanStream(): boolean {
-		const recorder = detectRecorder();
-		return recorder !== null && recorder.tool !== "powershell";
-	}
-
 	/** Segment text gets a leading space once a prior segment is committed, so
 	 *  phrases join naturally; the first phrase is inserted at the cursor as-is. */
 	#prefixed(text: string): string {
@@ -218,18 +193,38 @@ export class STTController {
 			},
 		});
 		this.#stream = stream;
-		let recorder: StreamingRecordingHandle | null = null;
+		let recorder: CaptureHandle;
 		try {
-			recorder = await startStreamingRecording(samples => stream.pushAudio(samples));
-		} catch (err) {
-			logger.warn("STT streaming recorder failed to start; falling back to batch recording", {
-				error: err instanceof Error ? err.message : String(err),
+			recorder = this.#createCapture((error, samples) => {
+				if (this.#disposed || this.#stream !== stream || this.#state !== "recording") return;
+				if (error) {
+					logger.error("Native microphone capture failed", { error: error.message });
+					const activeRecorder = this.#streamRecorder;
+					this.#streamRecorder = null;
+					try {
+						activeRecorder?.stop();
+					} catch (cause) {
+						logger.debug("stt: microphone cleanup failed", {
+							error: cause instanceof Error ? cause.message : String(cause),
+						});
+					}
+					this.#streamAbort?.abort(error);
+					stream.cancel();
+					this.#streamEditor?.clearVolatileText();
+					options.requestRender?.();
+					this.#cleanupStream();
+					this.#setState("idle", options);
+					options.showWarning(error.message);
+					return;
+				}
+				stream.pushAudio(samples);
 			});
-		}
-		if (!recorder) {
+		} catch (err) {
 			stream.cancel();
 			this.#cleanupStream();
-			await this.#startBatchRecording(options);
+			const msg = err instanceof Error ? err.message : "Failed to start microphone capture";
+			options.showWarning(msg);
+			logger.error("STT recording failed to start", { error: msg });
 			return;
 		}
 		this.#streamRecorder = recorder;
@@ -247,7 +242,7 @@ export class STTController {
 		this.#setState("transcribing", options);
 		// Stop the mic first so no further audio is fed, then flush the worker.
 		try {
-			await recorder?.stop();
+			recorder?.stop();
 		} catch (err) {
 			logger.debug("stt: streaming recorder stop failed", {
 				error: err instanceof Error ? err.message : String(err),
@@ -306,103 +301,19 @@ export class STTController {
 		this.#streamUtterance = "";
 	}
 
-	// ── Batch (single-shot) ─────────────────────────────────────────
-
-	async #startBatchRecording(options: ToggleOptions): Promise<void> {
-		const id = Snowflake.next();
-		this.#tempFile = path.join(os.tmpdir(), `omp-stt-${id}.wav`);
-		try {
-			this.#recordingHandle = await startRecording(this.#tempFile);
-			this.#setState("recording", options);
-			logger.debug("STT recording started", { tempFile: this.#tempFile });
-		} catch (err) {
-			this.#tempFile = null;
-			const msg = err instanceof Error ? err.message : "Failed to start recording";
-			options.showWarning(msg);
-			logger.error("STT recording failed to start", { error: msg });
-		}
-	}
-
-	async #stopBatch(editor: Editor, options: ToggleOptions): Promise<void> {
-		const handle = this.#recordingHandle;
-		const tempFile = this.#tempFile;
-		this.#recordingHandle = null;
-
-		if (!handle || !tempFile) {
-			this.#setState("idle", options);
-			return;
-		}
-
-		try {
-			await handle.stop();
-			// Validate the recording produced a usable file
-			await verifyRecordingFile(tempFile);
-			this.#setState("transcribing", options);
-
-			const sttSettings = {
-				modelName: settings.get("stt.modelName") as string | undefined,
-				language: settings.get("stt.language") as string | undefined,
-			};
-			this.#transcriptionAbort = new AbortController();
-			const text = await transcribe(tempFile, { ...sttSettings, signal: this.#transcriptionAbort.signal });
-			this.#transcriptionAbort = null;
-			if (this.#disposed) return;
-			if (text.length > 0) {
-				const trigger = settings.get("stt.submitTrigger");
-				const { submit, trimTrailing } = evaluateSubmitTrigger(text, trigger);
-				const textToInsert = trimTrailing > 0 ? text.slice(0, -trimTrailing) : text;
-				if (textToInsert.length > 0) {
-					editor.insertText(textToInsert);
-				}
-				options.showStatus("");
-				if (submit) {
-					editor.submit();
-				}
-			} else {
-				options.showStatus("No speech detected.");
-			}
-			if (!this.#disposed) this.#setState("idle", options);
-		} catch (err) {
-			if (this.#disposed) return;
-			if (err instanceof DOMException && err.name === "AbortError") {
-				this.#setState("idle", options);
-				return;
-			}
-			const msg = err instanceof Error ? err.message : "Transcription failed";
-			options.showWarning(msg);
-			logger.error("STT transcription failed", { error: msg });
-			this.#setState("idle", options);
-		} finally {
-			try {
-				await fs.rm(tempFile, { force: true });
-			} catch {
-				// best effort cleanup
-			}
-			this.#tempFile = null;
-		}
-	}
-
 	dispose(): void {
 		this.#disposed = true;
-		if (this.#transcriptionAbort) {
-			this.#transcriptionAbort.abort();
-			this.#transcriptionAbort = null;
-		}
 		if (this.#streamAbort) {
 			this.#streamAbort.abort();
 			this.#streamAbort = null;
 		}
 		this.#stream?.cancel();
-		this.#streamRecorder?.stop().catch(() => {});
-		this.#cleanupStream();
-		if (this.#recordingHandle) {
-			this.#recordingHandle.stop().catch(() => {});
-			this.#recordingHandle = null;
-		}
-		if (this.#tempFile) {
-			fs.rm(this.#tempFile, { force: true }).catch(() => {});
-			this.#tempFile = null;
+		try {
+			this.#streamRecorder?.stop();
+		} catch {
+			// best effort cleanup
 		}
+		this.#cleanupStream();
 		this.#state = "idle";
 		this.#resolvedModelKey = null;
 	}

package/src/system-prompt.ts

@@ -3,6 +3,7 @@
  */
 
 import * as os from "node:os";
+import * as path from "node:path";
 import type { AgentTool } from "@oh-my-pi/pi-agent-core";
 import type { ToolExample, TSchema } from "@oh-my-pi/pi-ai";
 import { renderToolInventory } from "@oh-my-pi/pi-ai/dialect";
@@ -16,6 +17,7 @@ import { expandAtImports } from "./discovery/at-imports";
 import { loadSkills, type Skill } from "./extensibility/skills";
 import { hasObsidian } from "./internal-urls/vault-protocol";
 import activeRepoContextTemplate from "./prompts/system/active-repo-context.md" with { type: "text" };
+import computerSafetyPrompt from "./prompts/system/computer-safety.md" with { type: "text" };
 import customSystemPromptTemplate from "./prompts/system/custom-system-prompt.md" with { type: "text" };
 import defaultPersonality from "./prompts/system/personalities/default.md" with { type: "text" };
 import friendlyPersonality from "./prompts/system/personalities/friendly.md" with { type: "text" };
@@ -466,6 +468,8 @@ export interface BuildSystemPromptOptions {
 	skillsSettings?: SkillsSettings;
 	/** Working directory. Default: getProjectDir() */
 	cwd?: string;
+	/** Additional workspace directories beyond cwd (multi-root), absolute. Injected into the project prompt. */
+	additionalWorkspaceRoots?: string[];
 	/** Pre-loaded context files (skips discovery if provided). */
 	contextFiles?: Array<{ path: string; content: string; depth?: number }>;
 	/** Skills provided directly to system prompt construction. */
@@ -535,6 +539,7 @@ export async function buildSystemPrompt(options: BuildSystemPromptOptions = {}):
 		skillsSettings,
 		toolNames: providedToolNames,
 		cwd,
+		additionalWorkspaceRoots = [],
 		contextFiles: providedContextFiles,
 		skills: providedSkills,
 		rules,
@@ -619,23 +624,42 @@ export async function buildSystemPrompt(options: BuildSystemPromptOptions = {}):
 	const systemPromptCustomizationPromise: Promise<string | null> = callerControlsCustomPrompt
 		? Promise.resolve(null)
 		: logger.time("loadSystemPromptFiles", loadSystemPromptFiles, { cwd: resolvedCwd });
-	const contextFilesPromise = providedContextFiles
-		? Promise.resolve(providedContextFiles)
-		: logger.time("loadProjectContextFiles", loadProjectContextFiles, { cwd: resolvedCwd });
-	const workspaceTreePromise =
-		providedWorkspaceTree !== undefined
-			? Promise.resolve(providedWorkspaceTree)
-			: includeWorkspaceTree
-				? logger.time("buildWorkspaceTree", () =>
-						buildWorkspaceTree(resolvedCwd, { timeoutMs: SYSTEM_PROMPT_PREP_TIMEOUT_MS }),
-					)
-				: Promise.resolve({
-						rootPath: resolvedCwd,
-						rendered: "",
-						truncated: false,
-						totalLines: 0,
-						agentsMdFiles: [],
-					});
+	const contextFilesPromise = (async () => {
+		const primary = providedContextFiles
+			? providedContextFiles
+			: await logger.time("loadProjectContextFiles", loadProjectContextFiles, { cwd: resolvedCwd });
+		// Also discover context files (AGENTS.md, rules, etc.) for each additional workspace root.
+		const additionalRoots = additionalWorkspaceRoots.filter(d => path.resolve(d) !== path.resolve(resolvedCwd));
+		if (additionalRoots.length === 0) return primary;
+		const extra = await Promise.all(
+			additionalRoots.map(root => loadProjectContextFiles({ cwd: root }).catch(() => [])),
+		);
+		return dedupeExactContextFiles([...primary, ...extra.flat()]);
+	})();
+	const additionalRootsForTree = additionalWorkspaceRoots.filter(d => path.resolve(d) !== path.resolve(resolvedCwd));
+	const workspaceTreePromise = (async () => {
+		const primary =
+			providedWorkspaceTree !== undefined
+				? await Promise.resolve(providedWorkspaceTree)
+				: includeWorkspaceTree
+					? await logger.time("buildWorkspaceTree", () =>
+							buildWorkspaceTree(resolvedCwd, { timeoutMs: SYSTEM_PROMPT_PREP_TIMEOUT_MS }),
+						)
+					: { rootPath: resolvedCwd, rendered: "", truncated: false, totalLines: 0, agentsMdFiles: [] };
+		if (additionalRootsForTree.length === 0 || !includeWorkspaceTree) return primary;
+		const extraTrees = await Promise.all(
+			additionalRootsForTree.map(root =>
+				buildWorkspaceTree(root, { timeoutMs: SYSTEM_PROMPT_PREP_TIMEOUT_MS }).catch(() => ({
+					rootPath: root,
+					rendered: "",
+					truncated: false,
+					totalLines: 0,
+					agentsMdFiles: [],
+				})),
+			),
+		);
+		return { ...primary, agentsMdFiles: [...primary.agentsMdFiles, ...extraTrees.flatMap(t => t.agentsMdFiles)] };
+	})();
 	const skillsPromise: Promise<readonly Skill[]> =
 		providedSkills !== undefined
 			? Promise.resolve(providedSkills)
@@ -794,6 +818,7 @@ export async function buildSystemPrompt(options: BuildSystemPromptOptions = {}):
 		date,
 		dateTime,
 		cwd: promptCwd,
+		additionalWorkspaceRoots: additionalWorkspaceRoots.filter(d => path.resolve(d) !== path.resolve(resolvedCwd)),
 		model: includeModelInPrompt ? (model ?? "") : "",
 		useCodexTaskPrompt: usesCodexTaskPrompt(model),
 		personality: personality === "none" ? "" : PERSONALITY_SPECS[personality].trim(),
@@ -815,6 +840,9 @@ export async function buildSystemPrompt(options: BuildSystemPromptOptions = {}):
 	};
 	const rendered = prompt.render(resolvedCustomPrompt ? customSystemPromptTemplate : systemPromptTemplate, data);
 	const systemPrompt = [rendered];
+	if (toolNames.includes("computer")) {
+		systemPrompt.push(computerSafetyPrompt.trim());
+	}
 	// Custom prompt templates already render context files and append text; the
 	// project footer still carries environment, cwd, workspace, and dir-context.
 	const projectPrompt = prompt