@withbuddi/buddi 0.1.0-pre.20 → 0.1.0-pre.22

package/node_modules/@buddi/gateway/dist/telegram/surface.js

@@ -24,6 +24,7 @@ import { isUnknownAgentError } from './types.js';
 import { whereText } from './commands.js';
 import { BURST_GAP_MS, StreamedAnswer, sendBurst, sendRunExtras, toPlainText, } from './outbound.js';
 import { FIRST_RUN_SUFFIX, shouldStartFirstRun } from '../agents/first-run.js';
+import { answerSpoken, applyVoiceArg, claimTextInsteadNote, getChatVoice, languageName, INSTALL_SPEECH_TEXT, MAX_SPOKEN_CHARS, MAX_VOICE_CAPTION_CHARS, NO_WORDS_TEXT, notHeardText, SET_UP_LISTENING_TEXT, setChatVoice, textInsteadText, VOICE_USAGE_TEXT, voiceSetText, voiceStatusText, voiceTurnText, } from './voice.js';
 export const SURFACE = 'telegram';
 /** Proposal cards' callback prefix (`proposals.ts`); here so routing needs no import cycle. */
 export const PROPOSAL_CALLBACK_PREFIX = 'prp';
@@ -96,6 +97,7 @@ export const HELP = [
     'buddi on Telegram.',
     '',
     'Send a statement, a receipt photo or a CSV and tell me what to do with it.',
+    'Send a voice note and I hear it, and answer with one, when the speech plugin is set up.',
     '',
     'Answers here are kept short, because you read them on a phone. Ask for more and you get more.',
     '',
@@ -113,6 +115,8 @@ export const HELP = [
     '/missions — the next five scheduled missions',
     '/goals — each goal with its number and where it stands',
     '/where — the dashboard address, when your phone can reach it',
+    '/voice [spoken|always|off] — answer with a voice when you send one (spoken), every time (always), or never (off)',
+    '/voice [voice|both|text] — send the voice note alone, with the text, or text only',
     '/browser — where the screen stands; /browser stop, /browser resume, /browser release',
     '/host — host execution permissions and running commands',
     '/hoststop — interrupt all host commands',
@@ -705,6 +709,10 @@ export class TelegramSurface {
      * restart drops it back to ordinary routing, never to a wrong agent.
      */
     #pending = new PendingQuestions();
+    /** The chats whose message in hand is a voice note the owner spoke; read by `#runFor`. */
+    #ownerSpoke = new Set();
+    /** The language of the voice note being run now, per chat, by name. */
+    #ownerLanguage = new Map();
     #offset;
     #running = false;
     #abort;
@@ -880,8 +888,11 @@ export class TelegramSurface {
         const incoming = extractAttachment(message);
         if (incoming) {
             // A file arrives with its own intent, caption or not. It is never the
-            // answer to "what time tonight?", so it ends any pending question.
-            this.#pending.clear(chatId);
+            // answer to "what time tonight?", so it ends any pending question. A
+            // voice note can be exactly that answer: it is heard, then routed as
+            // the words it carries.
+            if (incoming.slot !== 'voice' && incoming.slot !== 'audio')
+                this.#pending.clear(chatId);
             const messageId = String(message.message_id);
             this.enqueue(chatId, () => this.handleAttachment(chatId, resolution.ownerId, messageId, incoming));
             return;
@@ -1176,6 +1187,10 @@ export class TelegramSurface {
             await this.#opts.api.sendMessage(chatId, whereText(this.#opts.publicOrigin));
             return;
         }
+        if (command === '/voice') {
+            await this.handleVoice(chatId, text.slice(command.length));
+            return;
+        }
         // `/quiet 1w` — the argument is everything after the command word, so
         // `/quiet` alone parses as the default. The surface never reads a duration.
         if (command === '/quiet') {
@@ -1382,6 +1397,10 @@ export class TelegramSurface {
         // Whether this turn ended by asking the owner something. Read out of the
         // run, not out of what the owner is shown: the note is prepended after.
         let askedOwner = false;
+        // Read aloud? When the owner spoke and the chat says `spoken`, or always.
+        // Only with a speaker to read it; otherwise the agent is not told so.
+        const spoken = await this.#willSpeak(chatId);
+        const spokenLanguage = this.#ownerLanguage.get(chatId);
         // Where a message typed while this run works arrives. It is registered
         // before the first model call and closed after the last, so the window in
         // which the owner can get a word in is exactly the window in which the
@@ -1397,8 +1416,12 @@ export class TelegramSurface {
                     text: prompt,
                     interjections,
                     ...(carried?.attachments.length ? { attachments: carried.attachments } : {}),
+                    ...(spoken ? { spoken: true } : {}),
+                    ...(spokenLanguage ? { spokenLanguage } : {}),
                     onToolCall: (name) => progress.noteToolCall(name),
-                    onTextDelta: (delta) => stream.push(delta),
+                    // A spoken answer is not streamed: it arrives as a voice note with
+                    // its text underneath, and the progress line holds until then.
+                    ...(spoken ? {} : { onTextDelta: (delta) => stream.push(delta) }),
                 });
                 const reply = replyText(produced);
                 askedOwner = turnAskedOwner(typeof produced === 'string' ? undefined : produced.askedOwner, reply);
@@ -1416,7 +1439,7 @@ export class TelegramSurface {
                     ...(typeof produced !== 'string' && produced.question ? { question: produced.question } : {}),
                     ...extrasOf(produced),
                 };
-            }, label, carried ? readingText(label) : undefined, { agentId: agent.id, conversationId, prompt });
+            }, label, carried ? readingText(label) : undefined, { agentId: agent.id, conversationId, prompt }, spoken);
         }
         finally {
             /*
@@ -1552,10 +1575,11 @@ export class TelegramSurface {
             mime: row.mime,
             sizeBytes: row.sizeBytes,
         });
-        // Voice notes are kept, not heard: no run is started over bytes no model
-        // in this build can read, with or without a caption.
+        // A voice note is heard by the speech plugin and run as the owner's
+        // words. Without the plugin, or with listening not set
+        // up, it is kept and the reply says where to fix that.
         if (row.kind === 'audio') {
-            await api.sendMessage(chatId, gotAudioText(filename, row.sizeBytes));
+            await this.#heard(chatId, row, filename, incoming.caption);
             return;
         }
         const caption = incoming.caption;
@@ -1955,12 +1979,28 @@ export class TelegramSurface {
      * single message behind it — a `/recap` is the mission's, not the owner's,
      * and "try again" there would re-run something they never typed.
      */
-    retry) {
+    retry, 
+    /** Send the answer as a voice note, with its text as the caption. */
+    speak = false) {
         const stopTyping = this.#startTyping(chatId);
         const placeholder = placeholderOverride ?? placeholderText(agentName);
-        // An explicit bubble, posted before the provider is called: the owner sees
-        // that the question landed, and the same bubble becomes the answer.
-        const placeholderId = await this.#opts.api
+        // The thinking Blob first, when this bot has its sticker: it cannot become
+        // the answer, so the answer lands as a new message and the sticker goes.
+        // A named placeholder (`placeholderOverride`) says something the Blob
+        // cannot, and stays text.
+        const stickerId = placeholderOverride === undefined ? await this.#sendWorkingSticker(chatId) : undefined;
+        let stickerDropped = false;
+        const dropSticker = async () => {
+            if (stickerId === undefined || stickerDropped)
+                return;
+            stickerDropped = true;
+            await this.#opts.api.deleteMessage(chatId, stickerId).catch((err) => {
+                this.#log(`telegram: the working sticker could not be deleted: ${message(err)}`);
+            });
+        };
+        // Otherwise an explicit bubble, posted before the provider is called: the
+        // owner sees that the question landed, and the same bubble becomes the answer.
+        const placeholderId = stickerId !== undefined ? undefined : await this.#opts.api
             .sendMessage(chatId, placeholder)
             .catch((err) => {
             this.#log(`telegram: placeholder failed: ${message(err)}`);
@@ -1979,6 +2019,7 @@ export class TelegramSurface {
                 progress.silence();
                 await progress.settle();
             },
+            landed: dropSticker,
         });
         try {
             // Safety net, not the mechanism: TELEGRAM_SURFACE already told the model
@@ -1988,9 +2029,16 @@ export class TelegramSurface {
             const drawn = typeof produced === 'string' ? { text: produced, controls: [] } : produced;
             const reply = toPlainText(drawn.text);
             await progress.settle();
-            await stream.finish(reply, drawn.question && drawn.question.options.length > 0
+            const keyboard = drawn.question && drawn.question.options.length > 0
                 ? questionKeyboard(drawn.question)
-                : drawn.controls.length === 0 ? undefined : offersKeyboard(drawn.controls));
+                : drawn.controls.length === 0 ? undefined : offersKeyboard(drawn.controls);
+            // Buttons need a text message to sit under: an answer that carries any
+            // stays text, as cards and questions do.
+            const voice = speak && keyboard === undefined ? await this.#speakAnswer(chatId, placeholderId, reply, progress) : { sent: false };
+            if (!voice.sent)
+                await stream.finish(reply, keyboard);
+            if (voice.note)
+                await this.#opts.api.sendMessage(chatId, voice.note).catch(() => { });
             // Then what else the turn made: the view it drew, the files it saved.
             if (drawn.artifacts || drawn.canvas) {
                 await sendRunExtras({
@@ -2030,6 +2078,193 @@ export class TelegramSurface {
         }
         finally {
             stopTyping();
+            // Whatever the turn ended as — a voice note, a failure — the Blob goes.
+            await dropSticker();
+        }
+    }
+    /** The working sticker, sent; its message id, or undefined for the text placeholder. */
+    async #sendWorkingSticker(chatId) {
+        const resolve = this.#opts.workingSticker;
+        if (!resolve)
+            return undefined;
+        try {
+            const fileId = await resolve(chatId);
+            if (!fileId)
+                return undefined;
+            return await this.#opts.api.sendSticker(chatId, fileId);
+        }
+        catch (err) {
+            this.#log(`telegram: the working sticker failed, sending text: ${message(err)}`);
+            return undefined;
+        }
+    }
+    /* ---------------------------------------------------------------- *
+     * Voice
+     * ---------------------------------------------------------------- */
+    /**
+     * `/voice` alone shows the chat's choice; `/voice spoken|always|off` sets
+     * when, `/voice voice|both|text` what. Written to the chat's row and to the
+     * speech plugin's copy, so Settings → Speech shows the same.
+     */
+    async handleVoice(chatId, arg) {
+        const api = this.#opts.api;
+        const current = await this.#chatVoice(chatId);
+        if (arg.trim() === '') {
+            await api.sendMessage(chatId, voiceStatusText(current));
+            return;
+        }
+        const next = applyVoiceArg(current, arg);
+        if (!next) {
+            await api.sendMessage(chatId, VOICE_USAGE_TEXT);
+            return;
+        }
+        await setChatVoice(this.#opts.pool, SURFACE, chatId, next);
+        await this.#opts.speech?.saveVoicePrefs?.(next).catch(() => false);
+        await api.sendMessage(chatId, voiceSetText(next));
+    }
+    /** The chat's two choices: the speech plugin's copy where it has one, else the chat's row. */
+    async #chatVoice(chatId) {
+        const row = await getChatVoice(this.#opts.pool, SURFACE, chatId);
+        const prefs = (await this.#opts.speech?.voicePrefs?.().catch(() => ({}))) ?? {};
+        return { when: prefs.when ?? row.when, form: prefs.form ?? row.form };
+    }
+    /** Will this chat's next answer be read aloud? */
+    async #willSpeak(chatId) {
+        const speech = this.#opts.speech;
+        if (!speech || !this.#opts.artifacts || !speech.canSpeak())
+            return false;
+        try {
+            return answerSpoken((await this.#chatVoice(chatId)).when, this.#ownerSpoke.has(chatId));
+        }
+        catch (err) {
+            this.#log(`telegram: chat ${chatId} — the voice setting could not be read: ${message(err)}`);
+            return false;
+        }
+    }
+    /**
+     * A voice note, saved: transcribe it and run it as the owner's words (a
+     * caption follows the transcript). Nothing is echoed back: the transcript is
+     * the owner's message in the conversation, where the dashboard shows it.
+     * Refused or failed: one sentence, and the file stays saved.
+     */
+    async #heard(chatId, row, filename, caption) {
+        const api = this.#opts.api;
+        const speech = this.#opts.speech;
+        if (!speech) {
+            await api.sendMessage(chatId, `${gotAudioText(filename, row.sizeBytes)} ${INSTALL_SPEECH_TEXT}`);
+            return;
+        }
+        const stopTyping = this.#startTyping(chatId);
+        let heard;
+        try {
+            heard = await speech.transcribe(row.id);
+        }
+        catch (err) {
+            heard = { ok: false, reason: 'failed', message: message(err) };
+        }
+        finally {
+            stopTyping();
+        }
+        if (!heard.ok) {
+            if (heard.reason === 'failed')
+                this.#log(`telegram: chat ${chatId} — transcription failed: ${heard.message}`);
+            const text = heard.reason === 'missing'
+                ? `${gotAudioText(filename, row.sizeBytes)} ${INSTALL_SPEECH_TEXT}`
+                : heard.reason === 'unconfigured'
+                    ? `${gotAudioText(filename, row.sizeBytes)} ${SET_UP_LISTENING_TEXT}`
+                    : notHeardText(heard.message);
+            await api.sendMessage(chatId, text);
+            return;
+        }
+        if (heard.text === '') {
+            await api.sendMessage(chatId, NO_WORDS_TEXT);
+            return;
+        }
+        this.#ownerSpoke.add(chatId);
+        const language = languageName(heard.language);
+        if (language)
+            this.#ownerLanguage.set(chatId, language);
+        try {
+            await this.handleText(chatId, '', voiceTurnText(heard.text, caption));
+        }
+        finally {
+            this.#ownerSpoke.delete(chatId);
+            this.#ownerLanguage.delete(chatId);
+        }
+    }
+    /**
+     * The answer as a voice note, in place of the placeholder: alone (`/voice
+     * voice`), or with its text as the caption, or right after it past
+     * Telegram's 1,024 characters (`/voice both`). Not
+     * sent: the caller lands the text as usual, and `note` is the once-a-day
+     * line saying why, when there is one to say.
+     */
+    async #speakAnswer(chatId, placeholderId, reply, progress) {
+        const api = this.#opts.api;
+        const speech = this.#opts.speech;
+        const store = this.#opts.artifacts;
+        // A long answer is not read aloud, and nothing is said about it.
+        if (!speech || !store || reply.trim() === '' || reply.length > MAX_SPOKEN_CHARS)
+            return { sent: false };
+        const fallback = async (reason, detail) => {
+            this.#log(`telegram: chat ${chatId} — the answer went as text (${reason}): ${detail}`);
+            const note = await this.#textInsteadNote(chatId, reason);
+            return note ? { sent: false, note } : { sent: false };
+        };
+        api.sendChatAction(chatId, 'record_voice').catch(() => { });
+        const form = (await this.#chatVoice(chatId).catch(() => undefined))?.form ?? 'voice';
+        let said;
+        try {
+            said = await speech.say(reply, this.#handleNames());
+        }
+        catch (err) {
+            said = { ok: false, reason: 'failed', message: message(err) };
+        }
+        if (!said.ok)
+            return fallback(said.reason, said.message);
+        const loaded = await store.load(said.artifactId).catch(() => null);
+        if (!loaded)
+            return fallback('failed', `the voice ${said.artifactId} could not be read back`);
+        const withText = form === 'both';
+        const long = reply.length > MAX_VOICE_CAPTION_CHARS;
+        progress.silence();
+        try {
+            await api.sendVoice(chatId, Buffer.from(loaded.data, 'base64'), {
+                contentType: loaded.mime,
+                filename: loaded.mime === 'audio/mpeg' ? 'voice.mp3' : 'voice.ogg',
+                ...(withText && !long ? { caption: reply } : {}),
+            });
+        }
+        catch (err) {
+            return fallback('failed', message(err));
+        }
+        if (withText && long)
+            await api.sendMessage(chatId, reply);
+        if (placeholderId !== undefined)
+            await api.deleteMessage(chatId, placeholderId).catch(() => { });
+        return { sent: true };
+    }
+    /** Every agent's handle and name, so a spoken `@ledger` is read as its name. */
+    #handleNames() {
+        const names = {};
+        try {
+            for (const agent of this.#opts.catalog.list())
+                if (agent.handle && agent.name)
+                    names[agent.handle] = agent.name;
+        }
+        catch {
+            // No names: handles are read as bare words.
+        }
+        return names;
+    }
+    /** Today's "why this came as text" line for this chat, or nothing when it was said today. */
+    async #textInsteadNote(chatId, reason) {
+        try {
+            const claimed = await claimTextInsteadNote(this.#opts.pool, SURFACE, chatId, new Date(this.#now()), this.#opts.timezone ?? DEFAULT_TIMEZONE);
+            return claimed ? textInsteadText(reason) : undefined;
+        }
+        catch {
+            return undefined;
         }
     }
     /**