@markusylisiurunen/tau 0.2.34 → 0.2.36

package/dist/tui/chat_controller.js

@@ -1,6 +1,6 @@
 import { randomUUID } from "node:crypto";
 import { realpathSync, statSync } from "node:fs";
-import { mkdtemp, writeFile } from "node:fs/promises";
+import { mkdtemp, readFile, unlink, writeFile } from "node:fs/promises";
 import { tmpdir } from "node:os";
 import { join, relative, resolve, sep } from "node:path";
 import { formatCodexAuthError } from "../core/auth/auth_messages.js";
@@ -8,7 +8,7 @@ import { getAuthPath } from "../core/auth/auth_paths.js";
 import { AuthStorage } from "../core/auth/auth_storage.js";
 import { createCredentialResolver, } from "../core/auth/credential_resolver.js";
 import { createCommandRegistry, getRiskLevelDescription, } from "../core/commands/index.js";
-import { createDefaultConfigDeps, loadRuntimeConfig, } from "../core/config/index.js";
+import { createDefaultConfigDeps, getMistralApiKey, loadRuntimeConfig, } from "../core/config/index.js";
 import { createDefaultCoreDeps } from "../core/runtime/deps.js";
 import { createCheckpoint } from "../core/session/checkpoint.js";
 import { CoreSession } from "../core/session/core_session.js";
@@ -59,6 +59,10 @@ const PRUNED_EDIT_ARGUMENT_MARKER = "[content pruned]";
 const PRUNE_EDIT_UNCHANGED_CONTEXT_LINES = 4;
 const PRUNE_PREVIEW_MAX_TOKENS = 512;
 const PRUNE_MAX_OVERAGE_RATIO = 0.1;
+const SPEAK_TEMP_FILE_TEMPLATE = "/tmp/tau-speak.XXXXXX";
+const SPEAK_MISTRAL_TRANSCRIBE_MODEL = "voxtral-mini-latest";
+const SPEAK_RECORDING_MIN_BYTES = 1024;
+const SPEAK_RECORDING_MAX_DURATION_MS = 5 * 60 * 1000;
 export class ChatController {
     view;
     personas;
@@ -114,6 +118,9 @@ export class ChatController {
     lastTurnDurationMs = 0;
     turnTimer;
     lastEmptySubmitAt;
+    speakRecording;
+    isTranscribingSpeak = false;
+    speakTransition;
     constructor(options) {
         this.view = options.view;
         this.deps = options.deps ?? createDefaultCoreDeps();
@@ -211,6 +218,7 @@ export class ChatController {
             pruneLargest: (extra) => this.pruneToolResults("largest", extra),
             pruneSmart: (extra) => this.pruneToolResultsSmart(extra),
             reload: () => this.reloadContent(),
+            speak: () => this.toggleSpeakCapture(),
             risk: (level) => this.setRiskLevel(level),
             persona: (id) => this.switchPersona(id),
             prompt: (id) => this.insertPrompt(id),
@@ -263,6 +271,7 @@ export class ChatController {
             onCtrlR: () => this.cycleRiskLevel(),
             onCtrlP: () => this.cyclePersonality(),
             onCtrlS: () => void this.stashEditorToClipboard(),
+            onCtrlY: () => void this.toggleSpeakCapture(),
             onEscape: () => this.onInterrupt(),
             onCtrlF: () => {
                 this.expandFileMentions().catch((err) => {
@@ -284,6 +293,10 @@ export class ChatController {
     }
     async dispose() {
         this.subagentUnsubscribe?.();
+        if (this.speakTransition) {
+            await this.speakTransition;
+        }
+        await this.cancelSpeakCapture();
         if (!this.toolBackendDispose)
             return;
         await this.toolBackendDispose();
@@ -293,6 +306,10 @@ export class ChatController {
         await this.handleSubmit(text);
     }
     onInterrupt() {
+        if (this.speakRecording) {
+            void this.runSpeakTransition(() => this.stopSpeakCapture());
+            return;
+        }
         this.interruptAssistantTurn();
     }
     onEvent(event) {
@@ -444,6 +461,8 @@ export class ChatController {
         });
     }
     getInputMode() {
+        if (this.speakRecording)
+            return "recording";
         if (this.isBashIncognito)
             return "bash_incognito";
         if (this.isBashMode)
@@ -767,6 +786,8 @@ export class ChatController {
     }
     // Input Handling --------------------------------------------------------------------------------
     beforeSubmit(text) {
+        if (this.speakRecording)
+            return false;
         if (!this.isStreaming)
             return true;
         const trimmed = text.trimStart();
@@ -853,6 +874,8 @@ export class ChatController {
                 return "prune smart-selected tool results and compact edit calls, optional fraction and guidance";
             case "reload":
                 return "reload prompts, skills, themes, bash commands, and AGENTS.md";
+            case "speak":
+                return "toggle microphone recording and transcribe to editor";
             case "risk":
                 return "set risk level: /risk:read-only or /risk:read-write";
             case "bash":
@@ -1089,6 +1112,258 @@ export class ChatController {
         this.view.addMessage({ type: "user", text: trimmed }, historyEntryId);
         await this.runAssistantTurn();
     }
+    async toggleSpeakCapture() {
+        if (this.speakTransition) {
+            this.view.addSystemMessage("speech recording state change already in progress", "warn");
+            return;
+        }
+        if (this.speakRecording) {
+            await this.runSpeakTransition(() => this.stopSpeakCapture());
+            return;
+        }
+        if (this.isTranscribingSpeak) {
+            this.view.addSystemMessage("speech transcription already in progress", "warn");
+            return;
+        }
+        if (this.isStreaming) {
+            this.view.addSystemMessage("wait for the assistant to finish before recording", "warn");
+            return;
+        }
+        await this.runSpeakTransition(() => this.startSpeakCapture());
+    }
+    async runSpeakTransition(task) {
+        if (this.speakTransition) {
+            return;
+        }
+        const transition = task();
+        this.speakTransition = transition;
+        try {
+            await transition;
+        }
+        finally {
+            if (this.speakTransition === transition) {
+                this.speakTransition = undefined;
+            }
+        }
+    }
+    async startSpeakCapture() {
+        const apiKey = getMistralApiKey(this.config, this.deps.env.env());
+        if (!apiKey) {
+            this.view.addSystemMessage("set MISTRAL_API_KEY or apiKeys.mistral to use /speak", "error");
+            return;
+        }
+        let audioPath;
+        try {
+            audioPath = await this.createSpeakTempFilePath();
+            const abortController = new AbortController();
+            const completion = this.deps.spawn("ffmpeg", [
+                "-hide_banner",
+                "-loglevel",
+                "error",
+                "-nostdin",
+                "-f",
+                "avfoundation",
+                "-i",
+                ":0",
+                "-ac",
+                "1",
+                "-ar",
+                "16000",
+                "-c:a",
+                "pcm_s16le",
+                "-f",
+                "wav",
+                "-y",
+                audioPath,
+            ], {
+                detached: true,
+                killProcessGroup: true,
+                signal: abortController.signal,
+                stdio: ["ignore", "ignore", "ignore"],
+            });
+            const recording = {
+                audioPath,
+                stopRequested: false,
+                abortController,
+                completion,
+            };
+            recording.maxDurationTimeout = setTimeout(() => {
+                if (this.speakRecording !== recording || this.speakTransition)
+                    return;
+                void this.runSpeakTransition(() => this.stopSpeakCapture());
+            }, SPEAK_RECORDING_MAX_DURATION_MS);
+            this.speakRecording = recording;
+            this.view.setEditorInputEnabled(false);
+            this.refreshStatus();
+            void this.watchSpeakRecording(recording);
+        }
+        catch (err) {
+            if (audioPath) {
+                await this.cleanupSpeakTempFile(audioPath);
+            }
+            this.view.addSystemMessage(`failed to start recording: ${err.message}`, "error");
+        }
+    }
+    async stopSpeakCapture() {
+        const recording = this.speakRecording;
+        if (!recording)
+            return;
+        recording.stopRequested = true;
+        this.clearSpeakRecordingMaxDurationTimeout(recording);
+        this.speakRecording = undefined;
+        this.view.setEditorInputEnabled(true);
+        this.refreshStatus();
+        recording.abortController.abort();
+        try {
+            await recording.completion;
+        }
+        catch (err) {
+            this.view.addSystemMessage(`recording failed: ${err.message}`, "error");
+            await this.cleanupSpeakTempFile(recording.audioPath);
+            return;
+        }
+        this.isTranscribingSpeak = true;
+        try {
+            const audio = await readFile(recording.audioPath);
+            if (audio.byteLength < SPEAK_RECORDING_MIN_BYTES) {
+                this.view.addSystemMessage("recording too short, try again", "warn");
+                return;
+            }
+            const transcript = await this.transcribeSpeakAudio(audio);
+            const text = transcript.trim();
+            if (!text) {
+                return;
+            }
+            this.view.insertEditorTextAtCursor(text);
+        }
+        catch (err) {
+            this.view.addSystemMessage(`speech transcription failed: ${err.message}`, "error");
+        }
+        finally {
+            this.isTranscribingSpeak = false;
+            await this.cleanupSpeakTempFile(recording.audioPath);
+        }
+    }
+    async cancelSpeakCapture() {
+        const recording = this.speakRecording;
+        if (!recording)
+            return;
+        recording.stopRequested = true;
+        this.clearSpeakRecordingMaxDurationTimeout(recording);
+        this.speakRecording = undefined;
+        this.view.setEditorInputEnabled(true);
+        this.refreshStatus();
+        recording.abortController.abort();
+        try {
+            await recording.completion;
+        }
+        catch {
+            // ignore disposal errors
+        }
+        await this.cleanupSpeakTempFile(recording.audioPath);
+    }
+    async watchSpeakRecording(recording) {
+        try {
+            const result = await recording.completion;
+            this.clearSpeakRecordingMaxDurationTimeout(recording);
+            if (this.speakRecording !== recording || recording.stopRequested)
+                return;
+            this.speakRecording = undefined;
+            this.view.setEditorInputEnabled(true);
+            this.refreshStatus();
+            const detail = result.exitCode !== null
+                ? `ffmpeg exited with code ${result.exitCode}`
+                : result.closeSignal
+                    ? `ffmpeg terminated by signal ${result.closeSignal}`
+                    : "ffmpeg exited";
+            this.view.addSystemMessage(`recording stopped unexpectedly (${detail})`, "error");
+            await this.cleanupSpeakTempFile(recording.audioPath);
+        }
+        catch (err) {
+            this.clearSpeakRecordingMaxDurationTimeout(recording);
+            if (this.speakRecording !== recording || recording.stopRequested)
+                return;
+            this.speakRecording = undefined;
+            this.view.setEditorInputEnabled(true);
+            this.refreshStatus();
+            const error = err;
+            if (error.code === "ENOENT") {
+                this.view.addSystemMessage("ffmpeg not found. install it with: brew install ffmpeg", "error");
+            }
+            else {
+                this.view.addSystemMessage(`recording failed: ${error.message}`, "error");
+            }
+            await this.cleanupSpeakTempFile(recording.audioPath);
+        }
+    }
+    clearSpeakRecordingMaxDurationTimeout(recording) {
+        if (!recording.maxDurationTimeout)
+            return;
+        clearTimeout(recording.maxDurationTimeout);
+        recording.maxDurationTimeout = undefined;
+    }
+    async createSpeakTempFilePath() {
+        const result = await this.deps.spawn("mktemp", [SPEAK_TEMP_FILE_TEMPLATE]);
+        if (result.exitCode !== 0) {
+            const message = result.stderr.trim() || result.stdout.trim() || "mktemp failed";
+            throw new Error(message);
+        }
+        const path = result.stdout.trim().split(/\r?\n/, 1)[0]?.trim();
+        if (!path) {
+            throw new Error("mktemp returned an empty path");
+        }
+        return path;
+    }
+    async transcribeSpeakAudio(audio) {
+        const apiKey = getMistralApiKey(this.config, this.deps.env.env());
+        if (!apiKey) {
+            throw new Error("missing MISTRAL_API_KEY or apiKeys.mistral");
+        }
+        const formData = new FormData();
+        formData.append("model", SPEAK_MISTRAL_TRANSCRIBE_MODEL);
+        formData.append("file", new Blob([audio], { type: "audio/wav" }), "speech.wav");
+        formData.append("language", "en");
+        const response = await fetch("https://api.mistral.ai/v1/audio/transcriptions", {
+            method: "POST",
+            headers: {
+                Authorization: `Bearer ${apiKey}`,
+            },
+            body: formData,
+        });
+        let payload;
+        const responseText = await response.text();
+        if (responseText) {
+            try {
+                payload = JSON.parse(responseText);
+            }
+            catch {
+                payload = undefined;
+            }
+        }
+        if (!response.ok) {
+            const fromObject = payload && typeof payload === "object" && "message" in payload
+                ? payload.message
+                : undefined;
+            const fromString = typeof fromObject === "string" ? fromObject : undefined;
+            const fallback = responseText.trim() || `HTTP ${response.status}`;
+            throw new Error(fromString || fallback);
+        }
+        const text = payload && typeof payload === "object" && "text" in payload
+            ? payload.text
+            : undefined;
+        if (typeof text !== "string") {
+            return "";
+        }
+        return text;
+    }
+    async cleanupSpeakTempFile(path) {
+        try {
+            await unlink(path);
+        }
+        catch {
+            // best-effort cleanup
+        }
+    }
     getMemoryModeFilePath() {
         const cwd = this.deps.env.cwd();
         const home = this.deps.env.home();