@markusylisiurunen/tau 0.3.50 → 0.3.52

package/dist/core/telegram/adapter.js

@@ -9,6 +9,7 @@ import { collectSpeechToTextContext, } from "../utils/speech_to_text_context.js"
 import { formatTauUserText, splitTauUserText } from "../utils/user_metadata.js";
 import { formatZodError } from "../utils/zod.js";
 import { createScopedTelegramSessionManager, TelegramSessionManagerError, } from "./session_manager.js";
+import { generateTelegramVoice } from "./tts.js";
 const DEFAULT_POLL_INTERVAL_MS = 1000;
 const DEFAULT_REQUEST_TIMEOUT_SECONDS = 30;
 const MAX_COMMAND_PREVIEW_CHARS = 128;
@@ -36,6 +37,8 @@ const CALLBACK_ACTION_PREFIX = "tau:action:";
 const MAX_TELEGRAM_ATTACHMENTS_PER_TURN = 10;
 const MAX_TELEGRAM_ATTACHMENT_FILE_BYTES = 20 * 1024 * 1024;
 const MAX_TELEGRAM_ATTACHMENT_TOTAL_BYTES = 50 * 1024 * 1024;
+const MAX_TELEGRAM_VOICE_BYTES = 50 * 1024 * 1024;
+const TELEGRAM_TTS_JOB_TIMEOUT_MS = 5 * 60_000;
 const MAX_TELEGRAM_GROUP_PENDING_MESSAGES = 50;
 const TELEGRAM_ATTACHMENT_TEMP_DIR_PREFIX = "tau-telegram-attachments-";
 const TELEGRAM_REASONING_EFFORTS = [
@@ -617,15 +620,20 @@ function asPermanentTelegramRequestError(error) {
 }
 function createTelegramApi(botToken) {
     const apiUrl = `https://api.telegram.org/bot${botToken}`;
-    async function callTelegramMethod(method, payload, resultSchema, signal) {
+    async function callTelegramMethod(method, payload, resultSchema, signal, upload) {
+        const body = upload ? new FormData() : JSON.stringify(payload);
+        if (body instanceof FormData && upload) {
+            for (const [key, value] of Object.entries(payload)) {
+                body.append(key, typeof value === "string" ? value : JSON.stringify(value));
+            }
+            body.append(upload.field, new Blob([Uint8Array.from(upload.data)], { type: upload.mimeType }), upload.fileName);
+        }
         let response;
         try {
             response = await fetch(`${apiUrl}/${method}`, {
                 method: "POST",
-                headers: {
-                    "content-type": "application/json",
-                },
-                body: JSON.stringify(payload),
+                ...(upload ? {} : { headers: { "content-type": "application/json" } }),
+                body,
                 signal,
             });
         }
@@ -716,6 +724,14 @@ function createTelegramApi(botToken) {
                 ...(options?.replyMarkup ? { reply_markup: options.replyMarkup } : {}),
             }, z.unknown(), options.signal);
         },
+        async sendVoice(chatId, voice, options) {
+            await callTelegramMethod("sendVoice", { chat_id: chatId }, z.unknown(), options.signal, {
+                field: "voice",
+                fileName: "response.ogg",
+                mimeType: "audio/ogg",
+                data: voice,
+            });
+        },
         async sendChatAction(chatId, action) {
             await callTelegramMethod("sendChatAction", {
                 chat_id: chatId,
@@ -777,6 +793,7 @@ class TelegramAdapterImpl {
     allowedProjectIds;
     api;
     fetchImpl;
+    generateVoice;
     onLog;
     commandDefinitions;
     commandHandlers;
@@ -796,6 +813,8 @@ class TelegramAdapterImpl {
     inFlightUpdateTasks = new Set();
     notificationQueueTailByChat = new Map();
     inFlightNotificationTasks = new Set();
+    ttsQueueTailBySession = new Map();
+    inFlightTtsTasks = new Set();
     unsubscribeSessionEvents;
     loopPromise;
     nextUpdateOffset = 0;
@@ -829,6 +848,7 @@ class TelegramAdapterImpl {
         this.allowedProjectIds = Object.keys(options.projects);
         this.api = options.api;
         this.fetchImpl = options.fetchImpl;
+        this.generateVoice = options.generateVoice ?? generateTelegramVoice;
         this.onLog = options.onLog;
         this.commandDefinitions = this.createCommandDefinitions();
         this.commandHandlers = new Map();
@@ -868,6 +888,8 @@ class TelegramAdapterImpl {
         try {
             await this.loopPromise;
             await this.waitForInFlightUpdateTasks();
+            await Promise.allSettled(Array.from(this.inFlightTtsTasks));
+            this.ttsQueueTailBySession.clear();
             await Promise.allSettled(Array.from(this.inFlightNotificationTasks));
             for (const sessionId of Array.from(this.chatsBySession.keys())) {
                 this.stopTypingIndicators(sessionId);
@@ -910,6 +932,16 @@ class TelegramAdapterImpl {
                 callbackAction: "interrupt",
                 handler: async (chatId) => this.handleInterrupt(chatId),
             },
+            {
+                command: "/tts_on",
+                description: "enable voice responses",
+                handler: async (chatId, args) => this.handleTtsPreference(chatId, true, args),
+            },
+            {
+                command: "/tts_off",
+                description: "disable voice responses",
+                handler: async (chatId, args) => this.handleTtsPreference(chatId, false, args),
+            },
         ];
         for (const reasoning of TELEGRAM_REASONING_EFFORTS) {
             definitions.push({
@@ -1588,6 +1620,24 @@ class TelegramAdapterImpl {
             await this.reply(chatId, `failed to save project preference: ${error instanceof Error ? error.message : String(error)}`);
         }
     }
+    async handleTtsPreference(chatId, enabled, args) {
+        const command = enabled ? "/tts_on" : "/tts_off";
+        if (args.length > 0) {
+            await this.reply(chatId, `usage: ${command}`);
+            return;
+        }
+        if (enabled && !this.geminiApiKey) {
+            await this.reply(chatId, "set GEMINI_API_KEY or apiKeys.google to enable voice responses.");
+            return;
+        }
+        try {
+            await this.projectPreferences.setTtsEnabled(this.ownerIdForChat(chatId), enabled);
+            await this.reply(chatId, `voice responses ${enabled ? "enabled" : "disabled"}.`);
+        }
+        catch (error) {
+            await this.reply(chatId, `failed to save voice response preference: ${error instanceof Error ? error.message : String(error)}`);
+        }
+    }
     async handleSetReasoning(chatId, reasoning, args) {
         if (args.length > 0) {
             await this.reply(chatId, `usage: /effort_${reasoning}`);
@@ -2058,6 +2108,12 @@ class TelegramAdapterImpl {
             this.handleSessionProgress(event);
             return;
         }
+        if (event.type === "session-response-completed") {
+            if (this.chatsBySession.has(event.sessionId)) {
+                this.startTtsNotification(event.sessionId, event.text);
+            }
+            return;
+        }
         if (event.type === "session-turn-failed" || event.type === "session-turn-rejected") {
             if (!this.chatsBySession.has(event.sessionId)) {
                 return;
@@ -2147,6 +2203,104 @@ class TelegramAdapterImpl {
             ? [formatSessionHeadline(event.sessionId, "assistant message"), event.progress.text].join("\n")
             : event.progress.text, { rich: true, messageId: event.progress.messageId });
     }
+    startTtsNotification(sessionId, sourceText) {
+        const chatIds = this.chatsBySession.get(sessionId);
+        if (!chatIds ||
+            ![...chatIds].some((chatId) => this.projectPreferences.isTtsEnabled(this.ownerIdForChat(chatId)))) {
+            return;
+        }
+        const previousTask = this.ttsQueueTailBySession.get(sessionId) ?? Promise.resolve();
+        const task = previousTask
+            .then(async () => {
+            if (!this.abortController.signal.aborted) {
+                await this.generateAndSendTtsNotification(sessionId, sourceText);
+            }
+        })
+            .finally(() => {
+            this.inFlightTtsTasks.delete(task);
+            if (this.ttsQueueTailBySession.get(sessionId) === task) {
+                this.ttsQueueTailBySession.delete(sessionId);
+            }
+        });
+        this.ttsQueueTailBySession.set(sessionId, task);
+        this.inFlightTtsTasks.add(task);
+        void task;
+    }
+    async generateAndSendTtsNotification(sessionId, sourceText) {
+        let timedOut = false;
+        const timeoutController = new AbortController();
+        const timeout = setTimeout(() => {
+            timedOut = true;
+            timeoutController.abort();
+        }, TELEGRAM_TTS_JOB_TIMEOUT_MS);
+        timeout.unref?.();
+        try {
+            if (!this.geminiApiKey) {
+                throw new Error("missing Google credential for Telegram voice responses");
+            }
+            const voice = await this.generateVoice({
+                apiKey: this.geminiApiKey,
+                sourceText,
+                fetchImpl: this.fetchImpl,
+                signal: AbortSignal.any([this.abortController.signal, timeoutController.signal]),
+            });
+            if (voice.length > MAX_TELEGRAM_VOICE_BYTES) {
+                throw new Error("generated voice response exceeds Telegram's 50 MB limit");
+            }
+            const chatIds = this.chatsBySession.get(sessionId);
+            if (!chatIds) {
+                return;
+            }
+            await Promise.all([...chatIds]
+                .filter((chatId) => this.projectPreferences.isTtsEnabled(this.ownerIdForChat(chatId)))
+                .map(async (chatId) => {
+                const result = await this.enqueueVoiceNotification(sessionId, chatId, voice, timeoutController.signal);
+                if (result === "timed-out") {
+                    this.log("error", "Telegram voice response job timed out", {
+                        sessionId,
+                        chatId,
+                        cause: "voice response job timed out after 5 minutes",
+                    });
+                }
+                if (result === "failed" || result === "timed-out") {
+                    await this.enqueueTtsFailureNotification(sessionId, chatId);
+                }
+            }));
+        }
+        catch (error) {
+            if (this.abortController.signal.aborted) {
+                return;
+            }
+            this.log("error", "failed to generate Telegram voice response", {
+                sessionId,
+                cause: timedOut
+                    ? "voice generation timed out after 5 minutes"
+                    : error instanceof Error
+                        ? error.message
+                        : String(error),
+            });
+            await this.notifyTtsFailure(sessionId);
+        }
+        finally {
+            clearTimeout(timeout);
+        }
+    }
+    async notifyTtsFailure(sessionId) {
+        const chatIds = this.chatsBySession.get(sessionId);
+        if (!chatIds) {
+            return;
+        }
+        await Promise.all([...chatIds]
+            .filter((chatId) => this.projectPreferences.isTtsEnabled(this.ownerIdForChat(chatId)))
+            .map(async (chatId) => await this.enqueueTtsFailureNotification(sessionId, chatId)));
+    }
+    async enqueueTtsFailureNotification(sessionId, chatId) {
+        if (this.abortController.signal.aborted ||
+            !this.projectPreferences.isTtsEnabled(this.ownerIdForChat(chatId))) {
+            return;
+        }
+        await this.enqueueNotification(sessionId, chatId, "voice response failed. please try again.", {});
+    }
     notifyLifecycle(sessionId, projectId, state) {
         const sessionName = formatTelegramSessionName(sessionId, projectId);
         if (state === "failed") {
@@ -2207,6 +2361,54 @@ class TelegramAdapterImpl {
         this.inFlightNotificationTasks.add(trackedTask);
         return deliveryTask;
     }
+    enqueueVoiceNotification(sessionId, chatId, voice, jobSignal) {
+        const previousTask = this.notificationQueueTailByChat.get(chatId) ?? Promise.resolve();
+        const deliveryTask = previousTask
+            .then(async () => {
+            if (this.abortController.signal.aborted ||
+                !this.projectPreferences.isTtsEnabled(this.ownerIdForChat(chatId))) {
+                return "skipped";
+            }
+            if (jobSignal.aborted) {
+                return "timed-out";
+            }
+            const result = await this.sendWithRetry(async (signal) => await this.api.sendVoice(chatId, voice, { signal }), jobSignal);
+            if (result === ABORTED) {
+                return jobSignal.aborted && !this.abortController.signal.aborted
+                    ? "timed-out"
+                    : "skipped";
+            }
+            return "sent";
+        })
+            .catch((error) => {
+            if (this.abortController.signal.aborted) {
+                return "skipped";
+            }
+            const deliveryError = error instanceof TelegramDeliveryError
+                ? error
+                : new TelegramDeliveryError(error, 1, false);
+            this.log("error", deliveryError.retryable
+                ? "telegram voice response delivery retries exhausted"
+                : "failed to send Telegram voice response", {
+                sessionId,
+                chatId,
+                attempts: deliveryError.attempts,
+                cause: deliveryError.message,
+            });
+            return "failed";
+        });
+        const trackedTask = deliveryTask
+            .then(() => undefined)
+            .finally(() => {
+            this.inFlightNotificationTasks.delete(trackedTask);
+            if (this.notificationQueueTailByChat.get(chatId) === trackedTask) {
+                this.notificationQueueTailByChat.delete(chatId);
+            }
+        });
+        this.notificationQueueTailByChat.set(chatId, trackedTask);
+        this.inFlightNotificationTasks.add(trackedTask);
+        return deliveryTask;
+    }
     startTypingIndicators(sessionId) {
         const chatIds = this.chatsBySession.get(sessionId);
         if (!chatIds) {
@@ -2346,17 +2548,17 @@ class TelegramAdapterImpl {
             }
         }
     }
-    async sendWithRetry(send) {
+    async sendWithRetry(send, externalSignal) {
         for (let attempt = 1;; attempt += 1) {
             try {
-                const result = await this.runDeliveryAttempt(send);
+                const result = await this.runDeliveryAttempt(send, externalSignal);
                 if (result === ABORTED) {
                     return ABORTED;
                 }
                 return;
             }
             catch (error) {
-                if (this.abortController.signal.aborted) {
+                if (this.abortController.signal.aborted || externalSignal?.aborted) {
                     return ABORTED;
                 }
                 const retryDelayMs = TELEGRAM_DELIVERY_RETRY_DELAYS_MS[attempt - 1];
@@ -2364,15 +2566,18 @@ class TelegramAdapterImpl {
                 if (!retryable || retryDelayMs === undefined) {
                     throw new TelegramDeliveryError(error, attempt, retryable);
                 }
-                await this.wait(Math.max(retryDelayMs, error.retryAfterMs ?? 0));
-                if (this.abortController.signal.aborted) {
+                await this.wait(Math.max(retryDelayMs, error.retryAfterMs ?? 0), externalSignal);
+                if (this.abortController.signal.aborted || externalSignal?.aborted) {
                     return ABORTED;
                 }
             }
         }
     }
-    async runDeliveryAttempt(send) {
-        if (this.abortController.signal.aborted) {
+    async runDeliveryAttempt(send, externalSignal) {
+        const deliverySignal = externalSignal
+            ? AbortSignal.any([this.abortController.signal, externalSignal])
+            : this.abortController.signal;
+        if (deliverySignal.aborted) {
             return ABORTED;
         }
         const attemptController = new AbortController();
@@ -2382,7 +2587,7 @@ class TelegramAdapterImpl {
                 resolve(ABORTED);
                 attemptController.abort();
             };
-            this.abortController.signal.addEventListener("abort", abortListener, { once: true });
+            deliverySignal.addEventListener("abort", abortListener, { once: true });
         });
         let timeout;
         const timeoutPromise = new Promise((resolve) => {
@@ -2405,7 +2610,7 @@ class TelegramAdapterImpl {
             clearTimeout(timeout);
         }
         if (abortListener) {
-            this.abortController.signal.removeEventListener("abort", abortListener);
+            deliverySignal.removeEventListener("abort", abortListener);
         }
         if (raceResult === ABORTED) {
             return ABORTED;
@@ -2417,13 +2622,16 @@ class TelegramAdapterImpl {
             throw raceResult.error;
         }
     }
-    async wait(durationMs) {
-        if (durationMs <= 0 || this.abortController.signal.aborted) {
+    async wait(durationMs, externalSignal) {
+        const waitSignal = externalSignal
+            ? AbortSignal.any([this.abortController.signal, externalSignal])
+            : this.abortController.signal;
+        if (durationMs <= 0 || waitSignal.aborted) {
             return;
         }
         await new Promise((resolve) => {
             const timeout = setTimeout(() => {
-                this.abortController.signal.removeEventListener("abort", onAbort);
+                waitSignal.removeEventListener("abort", onAbort);
                 resolve();
             }, durationMs);
             timeout.unref?.();
@@ -2431,7 +2639,7 @@ class TelegramAdapterImpl {
                 clearTimeout(timeout);
                 resolve();
             };
-            this.abortController.signal.addEventListener("abort", onAbort, { once: true });
+            waitSignal.addEventListener("abort", onAbort, { once: true });
         });
     }
 }