@markusylisiurunen/tau 0.3.67 → 0.3.69

package/dist/tui/session_chat_controller.js

@@ -10,7 +10,7 @@ import { TOOL_NAME_BASH } from "../core/tools/tool_names.js";
 import { REASONING_LEVELS } from "../core/types.js";
 import { formatAdaptiveNumber, formatCwd, formatTokenWindow } from "../core/utils/format.js";
 import { extractAssistantText } from "../core/utils/messages.js";
-import { getSpeechToTextStreamingSampleRate, } from "../core/utils/speech_to_text.js";
+import { getSpeechToTextStreamingSampleRate, SPEECH_TO_TEXT_CLIENT_MAX_DURATION_MS, } from "../core/utils/speech_to_text.js";
 import { collectSpeechToTextContext } from "../core/utils/speech_to_text_context.js";
 import { hasAutoCompactionContinuationMetadata } from "../core/utils/user_metadata.js";
 import { APP_VERSION } from "../core/version.js";
@@ -25,7 +25,7 @@ import { InterruptLifecycle } from "./chat_controller/interrupt_lifecycle.js";
 import { formatDurationMs, formatFooterTokenCount, formatSessionCost, } from "./chat_controller/status_format.js";
 import { copyTextToClipboard } from "./clipboard.js";
 import { DOUBLE_PRESS_WINDOW_MS } from "./constants.js";
-import { cleanupListenTempFile, createListenTempFilePath, createListenTranscription, deleteListenTempFile, getSpeechToTextApiKey, getSpeechToTextApiKeyErrorMessage, getSpeechToTextProvider, LISTEN_CAPTURE_START_TIMEOUT_MS, LISTEN_RECORDING_MAX_DURATION_MS, LISTEN_RECORDING_MIN_BYTES, readListenAudio, startListenAudioCapture, } from "./listen_capture.js";
+import { cleanupListenTempFile, createListenTempFilePath, createListenTranscription, deleteListenTempFile, getSpeechToTextApiKey, getSpeechToTextApiKeyErrorMessage, getSpeechToTextProvider, LISTEN_CAPTURE_START_TIMEOUT_MS, LISTEN_RECORDING_MIN_BYTES, readListenAudio, startListenAudioCapture, } from "./listen_capture.js";
 import { createSdkDiffSnapshotDeps, createSdkToolExecutionBackend, } from "./session_tool_execution_backend.js";
 import { runSpeechPlaybackTask } from "./speech_playback.js";
 const LISTEN_CAPTURE_START_CANCELLED = Symbol("listen capture start cancelled");
@@ -76,7 +76,7 @@ export class SessionChatController {
     turnTimer;
     lastEmptySubmitAt;
     listenRecording;
-    retainedListenAudioPath;
+    retainedListenAudio;
     listenTransition;
     listenStartupAbortController;
     activeListenTranscription;
@@ -641,7 +641,7 @@ export class SessionChatController {
             ]);
             return;
         }
-        const retainedAudioPath = this.retainedListenAudioPath;
+        const retainedAudio = this.retainedListenAudio;
         let audioPath;
         let transcription;
         let abortController;
@@ -666,11 +666,11 @@ export class SessionChatController {
             if (this.listenStartupAbortController === abortController) {
                 this.listenStartupAbortController = undefined;
             }
-            if (retainedAudioPath) {
+            if (retainedAudio) {
                 try {
-                    await deleteListenTempFile(retainedAudioPath);
-                    if (this.retainedListenAudioPath === retainedAudioPath) {
-                        this.retainedListenAudioPath = undefined;
+                    await deleteListenTempFile(retainedAudio.audioPath);
+                    if (this.retainedListenAudio === retainedAudio) {
+                        this.retainedListenAudio = undefined;
                     }
                 }
                 catch (error) {
@@ -680,13 +680,14 @@ export class SessionChatController {
                     await cleanupListenTempFile(audioPath);
                     this.view.addTranscriptNotice("failed to replace retained recording", "error", [
                         error.message,
-                        `recording retained at ${retainedAudioPath}`,
+                        `recording retained at ${retainedAudio.audioPath}`,
                     ]);
                     return;
                 }
             }
             const recording = {
                 audioPath,
+                startedAt: Date.now(),
                 stopRequested: false,
                 abortController,
                 completion,
@@ -696,7 +697,7 @@ export class SessionChatController {
                 if (this.listenRecording !== recording || this.listenTransition)
                     return;
                 void this.runListenTransition(() => this.stopListenCapture());
-            }, LISTEN_RECORDING_MAX_DURATION_MS);
+            }, SPEECH_TO_TEXT_CLIENT_MAX_DURATION_MS);
             this.listenRecording = recording;
             this.view.setEditorInputEnabled(false);
             this.refreshStatus();
@@ -761,39 +762,42 @@ export class SessionChatController {
             await cleanupListenTempFile(recording.audioPath);
             return;
         }
-        await this.transcribeListenAudioFile(recording.audioPath, recording.transcription);
+        const durationMs = Math.min(Math.max(Date.now() - recording.startedAt, 0), SPEECH_TO_TEXT_CLIENT_MAX_DURATION_MS);
+        await this.transcribeListenAudioFile(recording.audioPath, durationMs, recording.transcription);
     }
     async retryRetainedListenAudio() {
         if (this.listenRecording || this.listenTransition || this.listenActivityLabel) {
             this.view.showFooterNotice("speech recording state change already in progress", "default");
             return;
         }
-        const audioPath = this.retainedListenAudioPath;
-        if (!audioPath) {
+        const retainedAudio = this.retainedListenAudio;
+        if (!retainedAudio) {
             this.view.showFooterNotice("no failed speech recording to retry", "default");
             return;
         }
-        await this.runListenTransition(() => this.transcribeListenAudioFile(audioPath));
+        await this.runListenTransition(() => this.transcribeListenAudioFile(retainedAudio.audioPath, retainedAudio.durationMs));
     }
     async discardRetainedListenAudio() {
         if (this.listenRecording || this.listenTransition || this.listenActivityLabel) {
             this.view.showFooterNotice("speech recording state change already in progress", "default");
             return;
         }
-        const audioPath = this.retainedListenAudioPath;
-        if (!audioPath) {
+        const retainedAudio = this.retainedListenAudio;
+        if (!retainedAudio) {
             this.view.showFooterNotice("no failed speech recording to discard", "default");
             return;
         }
         try {
-            await deleteListenTempFile(audioPath);
-            this.retainedListenAudioPath = undefined;
+            await deleteListenTempFile(retainedAudio.audioPath);
+            if (this.retainedListenAudio === retainedAudio) {
+                this.retainedListenAudio = undefined;
+            }
             this.view.showFooterNotice("discarded retained speech recording", "default");
         }
         catch (error) {
             this.view.addTranscriptNotice("failed to discard speech recording", "error", [
                 error.message,
-                `recording retained at ${audioPath}`,
+                `recording retained at ${retainedAudio.audioPath}`,
             ]);
         }
     }
@@ -806,7 +810,17 @@ export class SessionChatController {
             speechToTextDeps: this.speechToTextDeps,
         });
     }
-    async transcribeListenAudioFile(audioPath, transcription) {
+    async transcribeListenAudioFile(audioPath, durationMs, transcription) {
+        if (durationMs > SPEECH_TO_TEXT_CLIENT_MAX_DURATION_MS) {
+            transcription?.abort();
+            this.retainedListenAudio = { audioPath, durationMs };
+            this.view.addTranscriptNotice("failed to transcribe speech", "error", [
+                "audio is longer than 20 minutes",
+                `recording retained at ${audioPath}`,
+                "run /listen discard to delete it",
+            ]);
+            return;
+        }
         let audio;
         try {
             audio = await readListenAudio(audioPath);
@@ -816,8 +830,8 @@ export class SessionChatController {
             this.view.addTranscriptNotice("failed to read speech recording", "error", [
                 error.message,
             ]);
-            if (this.retainedListenAudioPath === audioPath) {
-                this.retainedListenAudioPath = undefined;
+            if (this.retainedListenAudio?.audioPath === audioPath) {
+                this.retainedListenAudio = undefined;
             }
             await cleanupListenTempFile(audioPath);
             return;
@@ -825,8 +839,8 @@ export class SessionChatController {
         if (audio.byteLength < LISTEN_RECORDING_MIN_BYTES) {
             transcription?.abort();
             this.view.showFooterNotice("recording too short, try again", "default");
-            if (this.retainedListenAudioPath === audioPath) {
-                this.retainedListenAudioPath = undefined;
+            if (this.retainedListenAudio?.audioPath === audioPath) {
+                this.retainedListenAudio = undefined;
             }
             await cleanupListenTempFile(audioPath);
             return;
@@ -842,7 +856,7 @@ export class SessionChatController {
                 mimeType: "audio/wav",
             });
             this.view.insertEditorTextAtCursor(text);
-            this.retainedListenAudioPath = undefined;
+            this.retainedListenAudio = undefined;
             try {
                 await deleteListenTempFile(audioPath);
             }
@@ -854,7 +868,7 @@ export class SessionChatController {
             }
         }
         catch (error) {
-            this.retainedListenAudioPath = audioPath;
+            this.retainedListenAudio = { audioPath, durationMs };
             this.view.addTranscriptNotice("failed to transcribe speech", "error", [
                 error.message,
                 `recording retained at ${audioPath}`,
@@ -1130,10 +1144,13 @@ export class SessionChatController {
             return false;
         }
         try {
+            if (this.tryApplyFastSnapshotStateDelta(delta)) {
+                return true;
+            }
             if (this.tryApplyFastContentAppendDelta(delta)) {
                 return true;
             }
-            if (this.tryApplyFastToolUiDelta(delta)) {
+            if (this.tryApplyFastTimelineDelta(delta)) {
                 return true;
             }
             if (this.tryApplyFastAgentDelta(delta)) {
@@ -1166,6 +1183,33 @@ export class SessionChatController {
         }
         return true;
     }
+    tryApplyFastSnapshotStateDelta(delta) {
+        if (delta.delta.type !== "snapshot.patch" ||
+            delta.delta.changes.length === 0 ||
+            delta.delta.changes.some((change) => {
+                switch (change.type) {
+                    case "agent-state.set":
+                    case "lifecycle.set":
+                    case "goal.set":
+                    case "cost.set":
+                    case "settings.set":
+                    case "turn.set":
+                    case "timeline.advance":
+                    case "operation.set":
+                    case "operation.remove":
+                        return false;
+                    default:
+                        return true;
+                }
+            })) {
+            return false;
+        }
+        this.snapshot = applySessionProtocolDelta(this.snapshot, delta);
+        this.observedSessionRevision = Math.max(this.observedSessionRevision, this.snapshot.revision);
+        this.updateStreamingStateFromSnapshot(this.snapshot);
+        this.refreshStatus();
+        return true;
+    }
     tryApplyFastContentAppendDelta(delta) {
         if (delta.delta.type !== "snapshot.patch" || delta.delta.changes.length !== 1) {
             return false;
@@ -1202,6 +1246,128 @@ export class SessionChatController {
         }
         return true;
     }
+    tryApplyFastTimelineDelta(delta) {
+        if (delta.delta.type !== "snapshot.patch" || delta.delta.changes.length === 0) {
+            return false;
+        }
+        const previousSnapshot = this.snapshot;
+        const affectedMessageIds = new Set();
+        const affectedToolIds = new Set();
+        const appendedNoticeIds = new Set();
+        let hasTimelineChange = false;
+        for (const change of delta.delta.changes) {
+            switch (change.type) {
+                case "agent-state.set":
+                case "lifecycle.set":
+                case "goal.set":
+                case "cost.set":
+                case "settings.set":
+                case "turn.set":
+                case "timeline.advance":
+                case "operation.set":
+                case "operation.remove":
+                    break;
+                case "message.append":
+                case "message.replace":
+                    hasTimelineChange = true;
+                    affectedMessageIds.add(change.message.id);
+                    break;
+                case "message.content.append":
+                    hasTimelineChange = true;
+                    affectedMessageIds.add(change.messageId);
+                    break;
+                case "timeline.append":
+                    hasTimelineChange = true;
+                    if (change.item.type === "message") {
+                        affectedMessageIds.add(change.item.messageId);
+                    }
+                    else if (change.item.type === "tool") {
+                        affectedToolIds.add(change.item.toolId);
+                    }
+                    else if (change.item.type === "notice") {
+                        appendedNoticeIds.add(change.item.id);
+                    }
+                    break;
+                case "tool.set":
+                    hasTimelineChange = true;
+                    affectedToolIds.add(change.tool.id);
+                    break;
+                case "facet.set":
+                    if (change.facet.kind !== "tau.tool-ui-events" || change.facet.subject.type !== "tool") {
+                        return false;
+                    }
+                    hasTimelineChange = true;
+                    affectedToolIds.add(change.facet.subject.id);
+                    break;
+                default:
+                    return false;
+            }
+        }
+        if (!hasTimelineChange) {
+            return false;
+        }
+        const nextSnapshot = applySessionProtocolDelta(previousSnapshot, delta);
+        this.snapshot = nextSnapshot;
+        this.observedSessionRevision = Math.max(this.observedSessionRevision, nextSnapshot.revision);
+        const remainingMessageIds = new Set(affectedMessageIds);
+        const affectedMessages = new Map();
+        for (const messageId of affectedMessageIds) {
+            const message = nextSnapshot.messages.find((entry) => entry.id === messageId);
+            if (message) {
+                affectedMessages.set(messageId, message);
+            }
+        }
+        const timelineItems = [...nextSnapshot.timeline.items, ...this.ephemeralTimelineItems.values()]
+            .filter((item) => {
+            if (item.type === "message") {
+                return affectedMessageIds.has(item.messageId);
+            }
+            if (item.type === "tool") {
+                return affectedToolIds.has(item.toolId);
+            }
+            return item.type === "notice" && appendedNoticeIds.has(item.id);
+        })
+            .sort((left, right) => left.sequence - right.sequence);
+        for (const item of timelineItems) {
+            if (item.type === "message" && remainingMessageIds.delete(item.messageId)) {
+                const message = affectedMessages.get(item.messageId);
+                if (message) {
+                    this.syncProtocolMessage(message);
+                }
+            }
+            else if (item.type === "tool" && affectedToolIds.has(item.toolId)) {
+                const tool = nextSnapshot.tools[item.toolId];
+                if (tool) {
+                    this.upsertRenderedMessage(item.id, {
+                        type: "tool",
+                        tool: buildToolUiModel(nextSnapshot, tool),
+                    });
+                }
+            }
+            else if (item.type === "notice" && appendedNoticeIds.has(item.id)) {
+                this.upsertRenderedMessage(item.id, {
+                    type: "transcript_notice",
+                    title: item.notice.presentation.title,
+                    ...(item.notice.presentation.content
+                        ? { content: item.notice.presentation.content }
+                        : {}),
+                    tone: item.notice.severity === "error" ? "error" : "default",
+                });
+            }
+        }
+        for (const messageId of remainingMessageIds) {
+            const message = affectedMessages.get(messageId);
+            if (message) {
+                this.syncProtocolMessage(message);
+            }
+            else {
+                this.removeProtocolMessage(messageId);
+            }
+        }
+        this.updateStreamingStateFromSnapshot(nextSnapshot);
+        this.refreshStatus();
+        return true;
+    }
     tryApplyFastAgentDelta(delta) {
         if (delta.delta.type !== "snapshot.patch") {
             return false;
@@ -1221,20 +1387,70 @@ export class SessionChatController {
         this.refreshStatus();
         return true;
     }
-    tryApplyFastToolUiDelta(delta) {
-        if (delta.delta.type !== "snapshot.patch" ||
-            delta.delta.changes.length === 0 ||
-            delta.delta.changes.some((change) => change.type !== "tool.set" && change.type !== "facet.set")) {
-            return false;
+    syncProtocolMessage(message) {
+        if (this.hiddenHistoryEntryIds.has(message.id) || !this.isMessageProjected(message.id)) {
+            this.removeProtocolMessage(message.id);
+            return;
         }
-        const facetChanges = delta.delta.changes.filter((change) => change.type === "facet.set");
-        if (facetChanges.some(({ facet }) => facet.kind !== "tau.tool-ui-events" || facet.subject.type !== "tool")) {
-            return false;
+        const model = this.buildProtocolMessageModel(message);
+        if (model) {
+            this.upsertRenderedMessage(message.id, model);
         }
-        const nextSnapshot = applySessionProtocolDelta(this.snapshot, delta);
-        this.syncRenderedHistory(nextSnapshot);
-        this.refreshStatus();
-        return true;
+        else {
+            this.removeRenderedProtocolItems([message.id]);
+        }
+        const outcomeNotice = getMessageOutcomeNotice(message);
+        const outcomeNoticeId = `presentation:failure:${message.id}`;
+        const interruptionNoticeId = `presentation:interruption:${message.id}`;
+        if (outcomeNotice) {
+            this.upsertRenderedMessage(outcomeNotice.id, {
+                type: "transcript_notice",
+                title: outcomeNotice.title,
+                ...(outcomeNotice.content ? { content: outcomeNotice.content } : {}),
+                tone: outcomeNotice.severity === "error" ? "error" : "default",
+            });
+        }
+        if (outcomeNotice?.id !== outcomeNoticeId) {
+            this.removeRenderedProtocolItems([outcomeNoticeId]);
+        }
+        if (outcomeNotice?.id !== interruptionNoticeId) {
+            this.removeRenderedProtocolItems([interruptionNoticeId]);
+        }
+    }
+    removeProtocolMessage(messageId) {
+        this.removeRenderedProtocolItems([
+            messageId,
+            `presentation:failure:${messageId}`,
+            `presentation:interruption:${messageId}`,
+        ]);
+    }
+    upsertRenderedMessage(protocolId, model) {
+        const viewMessageId = this.getViewMessageId(protocolId);
+        if (this.renderedMessageIds.includes(viewMessageId)) {
+            if (model.type === "assistant" || model.type === "assistant_partial") {
+                this.view.updateAssistantMessage(viewMessageId, model);
+            }
+            else {
+                this.view.updateMessage(viewMessageId, model);
+            }
+            return;
+        }
+        this.renderedMessageIds.push(this.view.addMessage(model, viewMessageId));
+    }
+    removeRenderedProtocolItems(protocolIds) {
+        this.removeRenderedMessages(protocolIds.flatMap((id) => {
+            const viewMessageId = this.viewMessageIds.get(id);
+            return viewMessageId ? [viewMessageId] : [];
+        }));
+    }
+    removeRenderedMessages(ids) {
+        const idsToRemove = ids.filter((id) => this.renderedMessageIds.includes(id));
+        if (idsToRemove.length === 0) {
+            return;
+        }
+        const idSet = new Set(idsToRemove);
+        this.renderedMessageIds.splice(0, this.renderedMessageIds.length, ...this.renderedMessageIds.filter((id) => !idSet.has(id)));
+        this.view.removeMessages(idsToRemove);
     }
     getViewMessageId(protocolId) {
         const existingId = this.viewMessageIds.get(protocolId);