@markusylisiurunen/tau 0.3.76 → 0.3.78

package/dist/tui/session_chat_controller.js

@@ -1,6 +1,6 @@
 import { dirname } from "node:path/posix";
 import { createCommandRegistry, } from "../core/commands/index.js";
-import { getGoogleApiKey } from "../core/config/index.js";
+import { getElevenLabsApiKey, getOpenAIApiKey } from "../core/config/index.js";
 import { captureDiffReviewSnapshot, DiffReviewBridge, } from "../core/diff_review/index.js";
 import { buildDiffReviewInstructions } from "../core/diff_review/review_instructions.js";
 import { createDefaultCoreDeps } from "../core/runtime/deps.js";
@@ -12,8 +12,9 @@ import { formatAdaptiveNumber, formatCwd, formatTokenWindow } from "../core/util
 import { extractAssistantText } from "../core/utils/messages.js";
 import { getSpeechToTextRecordingMaxDurationMs, getSpeechToTextStreamingSampleRate, SPEECH_TO_TEXT_CLIENT_MAX_DURATION_MS, } from "../core/utils/speech_to_text.js";
 import { collectSpeechToTextContext } from "../core/utils/speech_to_text_context.js";
-import { hasAutoCompactionContinuationMetadata } from "../core/utils/user_metadata.js";
+import { formatTauUserText, hasAutoCompactionContinuationMetadata, splitTauUserText, } from "../core/utils/user_metadata.js";
 import { APP_VERSION } from "../core/version.js";
+import { launchBuiltInDiffTool } from "../diff_tool/launcher.js";
 import { applySessionProtocolDelta, applySessionProtocolSubagentActivitiesMessage, } from "../protocol/session_protocol.js";
 import { TauSessionProtocolResponseError } from "../transport/errors.js";
 import { copyAssistantCodeToClipboard, copyAssistantTextToClipboard, } from "./chat_controller/assistant_clipboard.js";
@@ -25,9 +26,10 @@ import { InterruptLifecycle } from "./chat_controller/interrupt_lifecycle.js";
 import { formatDurationMs, formatFooterTokenCount, formatSessionCost, } from "./chat_controller/status_format.js";
 import { copyTextToClipboard } from "./clipboard.js";
 import { DOUBLE_PRESS_WINDOW_MS } from "./constants.js";
-import { cleanupListenTempFile, createListenTempFilePath, createListenTranscription, deleteListenTempFile, getSpeechToTextApiKey, getSpeechToTextApiKeyErrorMessage, getSpeechToTextProvider, LISTEN_CAPTURE_START_TIMEOUT_MS, LISTEN_RECORDING_MIN_BYTES, readListenAudio, startListenAudioCapture, } from "./listen_capture.js";
+import { cleanupListenTempFile, createListenTempFilePath, createListenTranscription, deleteListenTempFile, getSpeechToTextApiKey, getSpeechToTextApiKeyErrorMessage, LISTEN_CAPTURE_START_TIMEOUT_MS, LISTEN_RECORDING_MIN_BYTES, readListenAudio, startListenAudioCapture, } from "./listen_capture.js";
 import { createSdkDiffSnapshotDeps, createSdkToolExecutionBackend, } from "./session_tool_execution_backend.js";
 import { runSpeechPlaybackTask } from "./speech_playback.js";
+const TRANSCRIPTION_GUIDANCE = "Some or all of this message may have been transcribed from speech and may contain misheard words or other transcription errors.";
 const LISTEN_CAPTURE_START_CANCELLED = Symbol("listen capture start cancelled");
 export class SessionChatController {
     view;
@@ -36,7 +38,6 @@ export class SessionChatController {
     targetLabel;
     configuredClientToolNames;
     config;
-    defaultDiffTool;
     diffToolLauncher;
     deps;
     speechToTextDeps;
@@ -83,6 +84,8 @@ export class SessionChatController {
     activeListenTranscription;
     listenActivityLabel;
     listenPreview;
+    editorHiddenSystemMessages = [];
+    submissionHiddenSystemMessages = [];
     pendingEditorMutations = 0;
     speechActivityLabel;
     speakTask;
@@ -95,7 +98,6 @@ export class SessionChatController {
         this.targetLabel = options.targetLabel;
         this.configuredClientToolNames = options.configuredClientToolNames ?? [];
         this.config = options.config ?? {};
-        this.defaultDiffTool = options.defaultDiffTool;
         this.diffToolLauncher = options.diffToolLauncher;
         this.deps = options.deps ?? createDefaultCoreDeps();
         this.speechToTextDeps = options.speechToTextDeps;
@@ -127,7 +129,6 @@ export class SessionChatController {
             refreshStatus: () => this.refreshStatus(),
             startTurnTimer: () => this.startTurnTimer(),
             stopTurnTimer: () => this.stopTurnTimer(),
-            getDiffToolConfig: () => this.resolveDiffToolConfig(),
             startSession: (args) => this.startDiffReviewBridge(args, this.session, this.snapshot),
             onReviewReturned: (review) => this.handleReturnedDiffReview(review, this.session),
         });
@@ -185,12 +186,21 @@ export class SessionChatController {
             onCtrlP: () => void this.cyclePersonality(),
             onCtrlS: () => void this.stashEditorToClipboard(),
             onCtrlY: () => void this.toggleListenCapture(),
+            onDisabledSubmit: (mode) => {
+                if (this.listenRecording && !this.listenTransition) {
+                    void this.runListenTransition(() => this.stopListenCapture(mode));
+                }
+            },
             onCtrlG: () => this.interruptSelectedSubagent(),
             onEscape: () => void this.interrupt(),
-            beforeSubmit: (text) => this.beforeSubmit(text),
+            beforeSubmit: (text) => {
+                const allowed = this.beforeSubmit(text);
+                this.submissionHiddenSystemMessages = allowed ? [...this.editorHiddenSystemMessages] : [];
+                return allowed;
+            },
             onChange: (text) => this.handleEditorChange(text),
-            onSubmit: (text) => void this.handleSubmit(text),
-            onSteerSubmit: (text) => void this.submitSteeringMessage(text),
+            onSubmit: (text) => void this.handleSubmit(text, this.takeHiddenSystemSubmission()),
+            onQueueSubmit: (text) => void this.handleSubmit(text, this.takeHiddenSystemSubmission(), "queue"),
             onAltUp: () => void this.cancelPendingMessagesIntoEditor(),
             onAltDown: () => this.cycleSubagentSelection(),
         };
@@ -218,7 +228,16 @@ export class SessionChatController {
         }
         return true;
     }
-    async handleSubmit(text) {
+    takeHiddenSystemSubmission() {
+        const hiddenSystemMessages = this.submissionHiddenSystemMessages;
+        this.submissionHiddenSystemMessages = [];
+        this.editorHiddenSystemMessages = [];
+        return hiddenSystemMessages;
+    }
+    formatSubmittedText(text, hiddenSystemMessages) {
+        return formatTauUserText({ text, hiddenSystemMessages });
+    }
+    async handleSubmit(text, hiddenSystemMessages = [], mode = "steer") {
         const trimmed = text.trim();
         if (!trimmed) {
             if (this.isStreaming || this.submittedTurnInProgress) {
@@ -256,7 +275,11 @@ export class SessionChatController {
             if (trimmed.startsWith("!")) {
                 return;
             }
-            this.submitQueuedText(trimmed);
+            const submittedText = this.formatSubmittedText(trimmed, hiddenSystemMessages);
+            if (mode === "steer")
+                this.submitSteeringText(submittedText);
+            else
+                this.submitQueuedText(submittedText);
             return;
         }
         if (trimmed.startsWith("/") && this.isSingleLineInput(text)) {
@@ -286,7 +309,7 @@ export class SessionChatController {
             }
             return;
         }
-        await this.runSessionTurn(() => this.session.submit(trimmed));
+        await this.runSessionTurn(() => this.session.submit(this.formatSubmittedText(trimmed, hiddenSystemMessages)));
     }
     async onUserInput(text) {
         await this.handleSubmit(text);
@@ -307,6 +330,10 @@ export class SessionChatController {
         this.view.showFooterNotice(`theme set to ${themeId}`, "default");
     }
     handleEditorChange(text) {
+        if (!text.trim())
+            this.editorHiddenSystemMessages = [];
+        else
+            this.submissionHiddenSystemMessages = [];
         const wasBash = this.isBashMode;
         const wasBashIncognito = this.isBashIncognito;
         if (text.trim().length > 0) {
@@ -333,7 +360,6 @@ export class SessionChatController {
             themes: () => this.themeIds.map((id) => ({ id })),
             autocompletePaths: async (query, limit) => (await this.session.autocompletePaths({ query, limit })).paths,
             skills: () => this.snapshot.catalog.skills.map((skill) => skill.name),
-            subagents: () => Object.keys(this.getCurrentPersonaSnapshot()?.subagents ?? {}),
         };
     }
     buildStartupIntroTitle() {
@@ -453,21 +479,6 @@ export class SessionChatController {
             this.view.requestRender();
         }
     }
-    async submitSteeringMessage(text) {
-        const trimmed = text.trim();
-        if (!trimmed) {
-            return;
-        }
-        if (!this.isSessionOperationActive()) {
-            await this.runSessionTurn(() => this.session.submit(trimmed));
-            return;
-        }
-        if (this.isBlockingSessionOperationActive()) {
-            this.view.showFooterNotice("wait for tau to become idle before submitting input", "default");
-            return;
-        }
-        this.submitSteeringText(trimmed);
-    }
     submitQueuedText(text) {
         void this.session.queue(text).catch((error) => {
             if (!this.isPendingMessageCancellation(error)) {
@@ -493,8 +504,17 @@ export class SessionChatController {
             if (cancelled.length === 0) {
                 return;
             }
-            const editorText = this.view.getEditorText();
-            this.view.setEditorText([...(editorText ? [editorText] : []), ...cancelled.map((message) => message.text)].join("\n\n---\n\n"));
+            const editor = splitTauUserText(this.view.getEditorText());
+            const restored = cancelled.map((message) => splitTauUserText(message.text));
+            const hiddenSystemMessages = [
+                ...this.editorHiddenSystemMessages,
+                ...[editor, ...restored].flatMap((message) => message.hiddenSystemBlocks.map((block) => block.text)),
+            ];
+            this.view.setEditorText([
+                ...(editor.displayText ? [editor.displayText] : []),
+                ...restored.map((message) => message.displayText),
+            ].join("\n\n---\n\n"));
+            this.editorHiddenSystemMessages = hiddenSystemMessages;
             this.view.requestRender();
         }
         catch (error) {
@@ -651,7 +671,7 @@ export class SessionChatController {
         const apiKey = getSpeechToTextApiKey(this.config, this.deps);
         if (!apiKey) {
             this.view.addTranscriptNotice("speech-to-text is not configured", "error", [
-                getSpeechToTextApiKeyErrorMessage(this.config, "use /listen"),
+                getSpeechToTextApiKeyErrorMessage("use /listen"),
             ]);
             return;
         }
@@ -672,7 +692,7 @@ export class SessionChatController {
                 deps: this.deps,
                 audioPath,
                 signal: abortController.signal,
-                streamingSampleRate: getSpeechToTextStreamingSampleRate(getSpeechToTextProvider(this.config)),
+                streamingSampleRate: getSpeechToTextStreamingSampleRate(),
                 onAudioChunk: (audio) => transcription?.appendAudio(audio),
             });
             completion = capture.completion;
@@ -714,7 +734,7 @@ export class SessionChatController {
                 if (this.listenRecording !== recording || this.listenTransition)
                     return;
                 void this.runListenTransition(() => this.stopListenCapture());
-            }, getSpeechToTextRecordingMaxDurationMs(getSpeechToTextProvider(this.config)));
+            }, getSpeechToTextRecordingMaxDurationMs());
             this.listenRecording = recording;
             this.refreshStatus();
             void this.watchListenRecording(recording);
@@ -761,7 +781,7 @@ export class SessionChatController {
                 clearTimeout(timeout);
         }
     }
-    async stopListenCapture() {
+    async stopListenCapture(mode) {
         const recording = this.listenRecording;
         if (!recording)
             return;
@@ -781,7 +801,9 @@ export class SessionChatController {
             return;
         }
         const durationMs = Math.min(Math.max(Date.now() - recording.startedAt, 0), SPEECH_TO_TEXT_CLIENT_MAX_DURATION_MS);
-        await this.transcribeListenAudioFile(recording.audioPath, durationMs, recording.transcription);
+        const transcribed = await this.transcribeListenAudioFile(recording.audioPath, durationMs, recording.transcription);
+        if (mode && transcribed && !this.disposed)
+            this.view.submitEditor(mode);
     }
     async retryRetainedListenAudio() {
         if (this.pendingEditorMutations > 0) {
@@ -797,7 +819,9 @@ export class SessionChatController {
             this.view.showFooterNotice("no failed speech recording to retry", "default");
             return;
         }
-        await this.runListenTransition(() => this.transcribeListenAudioFile(retainedAudio.audioPath, retainedAudio.durationMs));
+        await this.runListenTransition(async () => {
+            await this.transcribeListenAudioFile(retainedAudio.audioPath, retainedAudio.durationMs);
+        });
     }
     async discardRetainedListenAudio() {
         if (this.listenRecording || this.listenTransition || this.listenActivityLabel) {
@@ -854,7 +878,7 @@ export class SessionChatController {
                 this.retainedListenAudio = undefined;
             }
             await cleanupListenTempFile(audioPath);
-            return;
+            return false;
         }
         if (!this.listenPreview)
             this.beginListenPreview();
@@ -868,7 +892,7 @@ export class SessionChatController {
                     `recording retained at ${audioPath}`,
                     "run /listen discard to delete it",
                 ]);
-                return;
+                return false;
             }
             let audio;
             try {
@@ -883,7 +907,7 @@ export class SessionChatController {
                     this.retainedListenAudio = undefined;
                 }
                 await cleanupListenTempFile(audioPath);
-                return;
+                return false;
             }
             if (audio.byteLength < LISTEN_RECORDING_MIN_BYTES) {
                 transcription?.abort();
@@ -892,7 +916,7 @@ export class SessionChatController {
                     this.retainedListenAudio = undefined;
                 }
                 await cleanupListenTempFile(audioPath);
-                return;
+                return false;
             }
             if (this.disposed) {
                 transcription?.abort();
@@ -900,7 +924,7 @@ export class SessionChatController {
                     this.retainedListenAudio = undefined;
                 }
                 await cleanupListenTempFile(audioPath);
-                return;
+                return false;
             }
             let activeTranscription = transcription;
             this.listenActivityLabel = "transcribing voice input";
@@ -913,8 +937,11 @@ export class SessionChatController {
                     mimeType: "audio/wav",
                 });
                 if (this.listenPreview !== preview)
-                    return;
+                    return false;
                 preview.commit(text);
+                if (text.trim() && !this.editorHiddenSystemMessages.includes(TRANSCRIPTION_GUIDANCE)) {
+                    this.editorHiddenSystemMessages.push(TRANSCRIPTION_GUIDANCE);
+                }
                 this.retainedListenAudio = undefined;
                 try {
                     await deleteListenTempFile(audioPath);
@@ -925,6 +952,7 @@ export class SessionChatController {
                         `recording remains at ${audioPath}; delete it manually`,
                     ]);
                 }
+                return Boolean(text.trim());
             }
             catch (error) {
                 this.retainedListenAudio = { audioPath, durationMs };
@@ -933,6 +961,7 @@ export class SessionChatController {
                     `recording retained at ${audioPath}`,
                     "run /listen retry to try again, or /listen discard to delete it",
                 ]);
+                return false;
             }
             finally {
                 activeTranscription?.abort();
@@ -1143,13 +1172,6 @@ export class SessionChatController {
         switch (snapshot.kind) {
             case "local":
                 return { kind: "local", cwd: snapshot.cwd };
-            case "cloudflare-sandbox":
-                return {
-                    kind: "cloudflare-sandbox",
-                    bridgeId: snapshot.bridgeId,
-                    sandboxId: snapshot.sandboxId,
-                    cwd: snapshot.cwd,
-                };
             case "fly-sprite":
                 return {
                     kind: "fly-sprite",
@@ -1986,10 +2008,11 @@ export class SessionChatController {
             this.view.showFooterNotice("last assistant message was empty.", "default");
             return;
         }
-        const apiKey = getGoogleApiKey(this.config, this.deps.env.env());
-        if (!apiKey) {
+        const openAIApiKey = getOpenAIApiKey(this.config, this.deps.env.env());
+        const elevenLabsApiKey = getElevenLabsApiKey(this.config, this.deps.env.env());
+        if (!openAIApiKey || !elevenLabsApiKey) {
             this.view.addTranscriptNotice("speech synthesis is not configured", "error", [
-                "set GEMINI_API_KEY or apiKeys.google to use /speak",
+                "set OPENAI_API_KEY or apiKeys.openai for rewriting and ELEVENLABS_API_KEY or apiKeys.elevenlabs for synthesis to use /speak",
             ]);
             return;
         }
@@ -1998,7 +2021,8 @@ export class SessionChatController {
         this.view.startWorkingIcon();
         const abortController = new AbortController();
         const completion = this.runSpeakTask({
-            apiKey,
+            openAIApiKey,
+            elevenLabsApiKey,
             sourceText,
             signal: abortController.signal,
         });
@@ -2028,7 +2052,9 @@ export class SessionChatController {
     async runSpeakTask(args) {
         await runSpeechPlaybackTask({
             deps: this.deps,
-            apiKey: args.apiKey,
+            openAIApiKey: args.openAIApiKey,
+            elevenLabsApiKey: args.elevenLabsApiKey,
+            voiceId: this.config.speech?.voiceId,
             sourceText: args.sourceText,
             signal: args.signal,
             onActivityLabel: (hint) => {
@@ -2249,9 +2275,6 @@ export class SessionChatController {
             !this.listenActivityLabel &&
             !this.speakTask);
     }
-    resolveDiffToolConfig() {
-        return this.config.diffTool ?? this.defaultDiffTool;
-    }
     async startDiffReviewBridge(args, session, sessionSnapshot, executionEnvironment = session) {
         const cwd = sessionSnapshot.executionEnvironment.cwd;
         const backend = createSdkToolExecutionBackend({ executionEnvironment, cwd });
@@ -2276,7 +2299,6 @@ export class SessionChatController {
                 ...(options.reasoning !== undefined ? { reasoning: options.reasoning } : {}),
             }),
             deps: this.deps,
-            ...(this.diffToolLauncher ? { toolLauncher: this.diffToolLauncher } : {}),
         });
         const unsubscribeEphemeral = session.onEphemeral((message) => {
             const event = message.event;
@@ -2297,7 +2319,7 @@ export class SessionChatController {
             if (args.signal.aborted) {
                 throw new Error("diff review start aborted");
             }
-            await bridge.launchTool(args.diffTool);
+            await bridge.launchTool(this.diffToolLauncher ?? launchBuiltInDiffTool);
         }
         catch (error) {
             unsubscribeEphemeral();