@markusylisiurunen/tau 0.3.53 → 0.3.55

package/dist/tui/session_chat_controller.js

@@ -24,9 +24,10 @@ import { InterruptLifecycle } from "./chat_controller/interrupt_lifecycle.js";
 import { formatDurationMs, formatFooterTokenCount, formatSessionCost, } from "./chat_controller/status_format.js";
 import { copyTextToClipboard } from "./clipboard.js";
 import { DOUBLE_PRESS_WINDOW_MS } from "./constants.js";
-import { cleanupListenTempFile, createListenTempFilePath, getSpeechToTextApiKey, getSpeechToTextApiKeyErrorMessage, LISTEN_RECORDING_MAX_DURATION_MS, LISTEN_RECORDING_MIN_BYTES, readListenAudio, startListenAudioCapture, transcribeListenAudio, } from "./listen_capture.js";
+import { cleanupListenTempFile, createListenTempFilePath, createListenTranscription, deleteListenTempFile, getSpeechToTextApiKey, getSpeechToTextApiKeyErrorMessage, LISTEN_CAPTURE_START_TIMEOUT_MS, LISTEN_RECORDING_MAX_DURATION_MS, LISTEN_RECORDING_MIN_BYTES, readListenAudio, startListenAudioCapture, } from "./listen_capture.js";
 import { createSdkDiffSnapshotDeps, createSdkToolExecutionBackend, } from "./session_tool_execution_backend.js";
 import { runSpeechPlaybackTask } from "./speech_playback.js";
+const LISTEN_CAPTURE_START_CANCELLED = Symbol("listen capture start cancelled");
 export class SessionChatController {
     view;
     session;
@@ -37,6 +38,7 @@ export class SessionChatController {
     defaultDiffTool;
     diffToolLauncher;
     deps;
+    speechToTextDeps;
     themeIds;
     commandRegistry;
     commandHandlers;
@@ -73,8 +75,11 @@ export class SessionChatController {
     turnTimer;
     lastEmptySubmitAt;
     listenRecording;
+    retainedListenAudioPath;
     listenTransition;
-    isTranscribingListen = false;
+    listenStartupAbortController;
+    activeListenTranscription;
+    listenActivityLabel;
     speechActivityLabel;
     speakTask;
     constructor(options) {
@@ -89,6 +94,7 @@ export class SessionChatController {
         this.defaultDiffTool = options.defaultDiffTool;
         this.diffToolLauncher = options.diffToolLauncher;
         this.deps = options.deps ?? createDefaultCoreDeps();
+        this.speechToTextDeps = options.speechToTextDeps;
         this.themeIds = options.themeIds ?? [];
         this.observedSessionRevision = options.snapshot.revision;
         this.commandRegistry = createCommandRegistry();
@@ -104,7 +110,7 @@ export class SessionChatController {
             compactSummaryOnly: (extra) => this.compactSession("summary-only", extra),
             compactSummaryAndLast: (extra) => this.compactSession("summary-and-last", extra),
             reload: () => this.reloadContent(),
-            listen: () => this.startListenCaptureFromCommand(),
+            listen: (action) => this.handleListenCommand(action),
             speak: () => this.speakLastAssistantMessage(),
             persona: (id) => this.setPersona(id),
             prompt: (id) => this.insertPrompt(id),
@@ -142,6 +148,8 @@ export class SessionChatController {
         this.ephemeralUnsubscribe?.();
         this.pendingUserMessagesUnsubscribe?.();
         this.subagentActivitiesUnsubscribe?.();
+        this.listenStartupAbortController?.abort(LISTEN_CAPTURE_START_CANCELLED);
+        this.activeListenTranscription?.abort();
         if (this.listenTransition) {
             await this.listenTransition;
         }
@@ -524,6 +532,11 @@ export class SessionChatController {
             this.view.showFooterNotice("interrupted", "default");
             return;
         }
+        if (this.listenStartupAbortController) {
+            this.listenStartupAbortController.abort(LISTEN_CAPTURE_START_CANCELLED);
+            this.view.showFooterNotice("interrupted", "default");
+            return;
+        }
         if (this.listenRecording) {
             void this.runListenTransition(() => this.stopListenCapture());
             return;
@@ -552,9 +565,28 @@ export class SessionChatController {
             ]);
         }
     }
+    async handleListenCommand(action) {
+        switch (action) {
+            case "record":
+                await this.startListenCaptureFromCommand();
+                return;
+            case "retry":
+                await this.retryRetainedListenAudio();
+                return;
+            case "discard":
+                await this.discardRetainedListenAudio();
+                return;
+        }
+    }
     async toggleListenCapture() {
         if (this.listenTransition) {
-            this.view.showFooterNotice("speech recording state change already in progress", "default");
+            if (this.listenStartupAbortController) {
+                this.listenStartupAbortController.abort(LISTEN_CAPTURE_START_CANCELLED);
+                this.view.showFooterNotice("cancelled speech recording startup", "default");
+            }
+            else {
+                this.view.showFooterNotice("speech recording state change already in progress", "default");
+            }
             return;
         }
         if (this.listenRecording) {
@@ -572,7 +604,7 @@ export class SessionChatController {
             this.view.showFooterNotice("speech recording already in progress", "default");
             return;
         }
-        if (this.isTranscribingListen) {
+        if (this.listenActivityLabel) {
             this.view.showFooterNotice("speech transcription already in progress", "default");
             return;
         }
@@ -605,20 +637,55 @@ export class SessionChatController {
             ]);
             return;
         }
+        const retainedAudioPath = this.retainedListenAudioPath;
         let audioPath;
+        let transcription;
+        let abortController;
+        let completion;
         try {
             audioPath = await createListenTempFilePath(this.deps);
-            const abortController = new AbortController();
-            const completion = startListenAudioCapture({
+            transcription = this.createSpeechTranscription("streaming");
+            abortController = new AbortController();
+            this.listenStartupAbortController = abortController;
+            const capture = startListenAudioCapture({
                 deps: this.deps,
                 audioPath,
                 signal: abortController.signal,
+                onAudioChunk: (audio) => transcription?.appendAudio(audio),
             });
+            completion = capture.completion;
+            await this.waitForListenCaptureStart(capture.started, abortController);
+            if (abortController.signal.aborted) {
+                throw abortController.signal.reason;
+            }
+            if (this.listenStartupAbortController === abortController) {
+                this.listenStartupAbortController = undefined;
+            }
+            if (retainedAudioPath) {
+                try {
+                    await deleteListenTempFile(retainedAudioPath);
+                    if (this.retainedListenAudioPath === retainedAudioPath) {
+                        this.retainedListenAudioPath = undefined;
+                    }
+                }
+                catch (error) {
+                    abortController.abort();
+                    transcription.abort();
+                    await completion.catch(() => undefined);
+                    await cleanupListenTempFile(audioPath);
+                    this.view.addTranscriptNotice("failed to replace retained recording", "error", [
+                        error.message,
+                        `recording retained at ${retainedAudioPath}`,
+                    ]);
+                    return;
+                }
+            }
             const recording = {
                 audioPath,
                 stopRequested: false,
                 abortController,
                 completion,
+                transcription,
             };
             recording.maxDurationTimeout = setTimeout(() => {
                 if (this.listenRecording !== recording || this.listenTransition)
@@ -631,10 +698,43 @@ export class SessionChatController {
             void this.watchListenRecording(recording);
         }
         catch (err) {
+            const cancelled = abortController?.signal.reason === LISTEN_CAPTURE_START_CANCELLED;
+            abortController?.abort();
+            transcription?.abort();
+            await completion?.catch(() => undefined);
             if (audioPath) {
                 await cleanupListenTempFile(audioPath);
             }
-            this.view.addTranscriptNotice("failed to start recording", "error", [err.message]);
+            if (!cancelled) {
+                this.view.addTranscriptNotice("failed to start recording", "error", [
+                    err.message,
+                ]);
+            }
+        }
+        finally {
+            if (this.listenStartupAbortController === abortController) {
+                this.listenStartupAbortController = undefined;
+            }
+        }
+    }
+    async waitForListenCaptureStart(started, abortController) {
+        let timeout;
+        try {
+            await Promise.race([
+                started,
+                new Promise((_resolve, reject) => {
+                    timeout = setTimeout(() => {
+                        const error = new Error("timed out waiting for microphone audio");
+                        abortController.abort(error);
+                        reject(error);
+                    }, LISTEN_CAPTURE_START_TIMEOUT_MS);
+                    timeout.unref?.();
+                }),
+            ]);
+        }
+        finally {
+            if (timeout)
+                clearTimeout(timeout);
         }
     }
     async stopListenCapture() {
@@ -651,39 +751,120 @@ export class SessionChatController {
             await recording.completion;
         }
         catch (err) {
+            recording.transcription.abort();
             this.view.addTranscriptNotice("failed to record audio", "error", [err.message]);
             await cleanupListenTempFile(recording.audioPath);
             return;
         }
-        this.isTranscribingListen = true;
-        this.refreshStatus();
+        await this.transcribeListenAudioFile(recording.audioPath, recording.transcription);
+    }
+    async retryRetainedListenAudio() {
+        if (this.listenRecording || this.listenTransition || this.listenActivityLabel) {
+            this.view.showFooterNotice("speech recording state change already in progress", "default");
+            return;
+        }
+        const audioPath = this.retainedListenAudioPath;
+        if (!audioPath) {
+            this.view.showFooterNotice("no failed speech recording to retry", "default");
+            return;
+        }
+        await this.runListenTransition(() => this.transcribeListenAudioFile(audioPath));
+    }
+    async discardRetainedListenAudio() {
+        if (this.listenRecording || this.listenTransition || this.listenActivityLabel) {
+            this.view.showFooterNotice("speech recording state change already in progress", "default");
+            return;
+        }
+        const audioPath = this.retainedListenAudioPath;
+        if (!audioPath) {
+            this.view.showFooterNotice("no failed speech recording to discard", "default");
+            return;
+        }
         try {
-            const audio = await readListenAudio(recording.audioPath);
-            if (audio.byteLength < LISTEN_RECORDING_MIN_BYTES) {
-                this.view.showFooterNotice("recording too short, try again", "default");
-                return;
+            await deleteListenTempFile(audioPath);
+            this.retainedListenAudioPath = undefined;
+            this.view.showFooterNotice("discarded retained speech recording", "default");
+        }
+        catch (error) {
+            this.view.addTranscriptNotice("failed to discard speech recording", "error", [
+                error.message,
+                `recording retained at ${audioPath}`,
+            ]);
+        }
+    }
+    createSpeechTranscription(mode) {
+        return createListenTranscription({
+            config: this.config,
+            deps: this.deps,
+            mode,
+            context: collectSpeechToTextContext(this.snapshot),
+            speechToTextDeps: this.speechToTextDeps,
+        });
+    }
+    async transcribeListenAudioFile(audioPath, transcription) {
+        let audio;
+        try {
+            audio = await readListenAudio(audioPath);
+        }
+        catch (error) {
+            transcription?.abort();
+            this.view.addTranscriptNotice("failed to read speech recording", "error", [
+                error.message,
+            ]);
+            if (this.retainedListenAudioPath === audioPath) {
+                this.retainedListenAudioPath = undefined;
             }
-            const transcript = await transcribeListenAudio({
-                config: this.config,
-                deps: this.deps,
+            await cleanupListenTempFile(audioPath);
+            return;
+        }
+        if (audio.byteLength < LISTEN_RECORDING_MIN_BYTES) {
+            transcription?.abort();
+            this.view.showFooterNotice("recording too short, try again", "default");
+            if (this.retainedListenAudioPath === audioPath) {
+                this.retainedListenAudioPath = undefined;
+            }
+            await cleanupListenTempFile(audioPath);
+            return;
+        }
+        let activeTranscription = transcription;
+        this.listenActivityLabel = "transcribing voice input";
+        this.refreshStatus();
+        try {
+            activeTranscription ??= this.createSpeechTranscription("file");
+            this.activeListenTranscription = activeTranscription;
+            const text = await activeTranscription.finish({
                 audio,
-                context: collectSpeechToTextContext(this.snapshot),
+                mimeType: "audio/wav",
+                fileName: "speech.wav",
+                language: "en",
             });
-            const text = transcript.trim();
-            if (!text) {
-                return;
-            }
             this.view.insertEditorTextAtCursor(text);
+            this.retainedListenAudioPath = undefined;
+            try {
+                await deleteListenTempFile(audioPath);
+            }
+            catch (error) {
+                this.view.addTranscriptNotice("failed to delete speech recording", "error", [
+                    error.message,
+                    `recording remains at ${audioPath}; delete it manually`,
+                ]);
+            }
         }
-        catch (err) {
+        catch (error) {
+            this.retainedListenAudioPath = audioPath;
             this.view.addTranscriptNotice("failed to transcribe speech", "error", [
-                err.message,
+                error.message,
+                `recording retained at ${audioPath}`,
+                "run /listen retry to try again, or /listen discard to delete it",
             ]);
         }
         finally {
-            this.isTranscribingListen = false;
+            activeTranscription?.abort();
+            if (this.activeListenTranscription === activeTranscription) {
+                this.activeListenTranscription = undefined;
+            }
+            this.listenActivityLabel = undefined;
             this.refreshStatus();
-            await cleanupListenTempFile(recording.audioPath);
         }
     }
     async cancelListenCapture() {
@@ -696,6 +877,7 @@ export class SessionChatController {
         this.view.setEditorInputEnabled(true);
         this.refreshStatus();
         recording.abortController.abort();
+        recording.transcription.abort();
         try {
             await recording.completion;
         }
@@ -711,6 +893,7 @@ export class SessionChatController {
             if (this.listenRecording !== recording || recording.stopRequested)
                 return;
             this.listenRecording = undefined;
+            recording.transcription.abort();
             this.view.setEditorInputEnabled(true);
             this.refreshStatus();
             const detail = result.exitCode !== null
@@ -726,6 +909,7 @@ export class SessionChatController {
             if (this.listenRecording !== recording || recording.stopRequested)
                 return;
             this.listenRecording = undefined;
+            recording.transcription.abort();
             this.view.setEditorInputEnabled(true);
             this.refreshStatus();
             const error = err;
@@ -1603,8 +1787,8 @@ export class SessionChatController {
         if (this.manualCompactionInProgress || this.hasRunningCompactionOperation(this.snapshot)) {
             return "compacting context";
         }
-        if (this.isTranscribingListen) {
-            return "transcribing voice input";
+        if (this.listenActivityLabel) {
+            return this.listenActivityLabel;
         }
         return this.diffReviewService.getActivityLabel(this.speechActivityLabel);
     }
@@ -1772,7 +1956,7 @@ export class SessionChatController {
         return (!this.isSessionOperationActive() &&
             !this.listenRecording &&
             !this.listenTransition &&
-            !this.isTranscribingListen &&
+            !this.listenActivityLabel &&
             !this.speakTask);
     }
     resolveDiffToolConfig() {