agent-nuvira 3.3.3 → 3.3.4

package/dist/gateway/adapters.js

@@ -14,12 +14,14 @@
  */
 import { createServer } from 'node:http';
 import { envBuff } from '../config/paths.js';
+import { logger } from '../utils/logger.js';
 import { createHmac, randomUUID, timingSafeEqual } from 'node:crypto';
 import { connect as netConnect } from 'node:net';
 import { connect as tlsConnect } from 'node:tls';
 import { isPlatformConfigured } from './channel-directory.js';
 import { BaileysBridge, isSelfChatEnabled } from './whatsapp/baileys-bridge.js';
 import { hasWhatsAppSession } from './whatsapp/session.js';
+import { downloadInboundAttachment, MAX_INBOUND_MEDIA_BYTES, } from './inbound-media.js';
 // ─── Helpers ────────────────────────────────────────────────────────────────
 /** POST JSON to an endpoint and treat 2xx as success. */
 async function postJson(url, body, token, extraHeaders) {
@@ -109,18 +111,27 @@ export class TelegramAdapter {
                     const data = (await res.json());
                     for (const u of data.result ?? []) {
                         this.offset = u.update_id + 1;
-                        const text = u.message?.text;
-                        if (!text || !u.message?.chat)
+                        const m = u.message;
+                        if (!m?.chat)
                             continue;
-                        const chat = u.message.chat;
+                        // A document/photo carries its instruction in the CAPTION; reading
+                        // only `text` made those messages look empty and they were dropped.
+                        const text = m.text ?? m.caption ?? '';
+                        const media = await this.inboundMedia(m);
+                        if (!text && !media)
+                            continue;
+                        const chat = m.chat;
                         await this.handler?.({
                             platform: 'telegram',
                             channelId: String(chat.id),
                             text,
-                            from: u.message.from?.first_name ?? 'telegram-user',
+                            from: m.from?.first_name ?? 'telegram-user',
                             // P1: real sender id + group detection (chat.type).
-                            senderId: u.message.from?.id !== undefined ? String(u.message.from.id) : undefined,
+                            senderId: m.from?.id !== undefined ? String(m.from.id) : undefined,
                             isGroup: chat.type !== undefined && chat.type !== 'private',
+                            // The downloaded attachment rides through the SAME hydration path
+                            // the WhatsApp bridge uses (registry → read_extract).
+                            ...(media ? { media } : {}),
                         });
                     }
                 }
@@ -130,6 +141,72 @@ export class TelegramAdapter {
         };
         void loop();
     }
+    /**
+     * Download an inbound attachment's bytes via `getFile` + the file endpoint.
+     * Never throws — returns null on any failure (or when the file is larger than
+     * the extraction cap), so the turn still runs with its caption text.
+     */
+    async downloadFile(fileId, declaredSize) {
+        if (declaredSize !== undefined && declaredSize > MAX_INBOUND_MEDIA_BYTES)
+            return null;
+        try {
+            const res = await this.api('getFile', { file_id: fileId });
+            if (!res.ok)
+                return null;
+            const data = (await res.json());
+            const filePath = data.result?.file_path;
+            if (!filePath)
+                return null;
+            // Telegram caps bot downloads at 20 MB; the declared size is checked
+            // BEFORE the body is fetched where it is known.
+            const size = data.result?.file_size;
+            if (size !== undefined && size > MAX_INBOUND_MEDIA_BYTES)
+                return null;
+            const fileRes = await fetch(`https://api.telegram.org/file/bot${this.token}/${filePath}`);
+            if (!fileRes.ok)
+                return null;
+            const buf = new Uint8Array(await fileRes.arrayBuffer());
+            if (buf.byteLength === 0 || buf.byteLength > MAX_INBOUND_MEDIA_BYTES)
+                return null;
+            return buf;
+        }
+        catch {
+            return null;
+        }
+    }
+    /**
+     * Build the inbound media payload from a Telegram message, or null when it
+     * carries none. Photos arrive as several sizes — the largest is used.
+     */
+    async inboundMedia(m) {
+        if (m.document) {
+            const data = await this.downloadFile(m.document.file_id, m.document.file_size);
+            return data
+                ? { type: 'document', data, filename: m.document.file_name, mimetype: m.document.mime_type, caption: m.caption }
+                : null;
+        }
+        if (m.video) {
+            const data = await this.downloadFile(m.video.file_id, m.video.file_size);
+            return data
+                ? { type: 'video', data, filename: m.video.file_name, mimetype: m.video.mime_type, caption: m.caption }
+                : null;
+        }
+        if (m.audio) {
+            const data = await this.downloadFile(m.audio.file_id, m.audio.file_size);
+            return data
+                ? { type: 'audio', data, filename: m.audio.file_name, mimetype: m.audio.mime_type, caption: m.caption }
+                : null;
+        }
+        const photo = Array.isArray(m.photo) && m.photo.length > 0 ? m.photo[m.photo.length - 1] : undefined;
+        if (photo) {
+            const data = await this.downloadFile(photo.file_id, photo.file_size);
+            // Telegram photos carry no filename; supply the MIME type so the sandbox
+            // name keeps an image extension (which routes to the describe_image hint
+            // rather than an extractor refusal).
+            return data ? { type: 'image', data, filename: 'photo.jpg', mimetype: 'image/jpeg', caption: m.caption } : null;
+        }
+        return null;
+    }
     async stop() {
         this.running = false;
         if (this.timer)
@@ -213,9 +290,36 @@ export class DiscordAdapter extends WebhookChannelAdapter {
         this.token = envBuff('DISCORD_BOT_TOKEN') ?? '';
         this.configured = Boolean(this.webhookUrl || this.token);
     }
+    /** The real-time Gateway connection, when a bot token is configured. */
+    gateway = null;
     describe() {
         return this.configured ? 'Discord (webhook)' : 'Discord (not configured)';
     }
+    /**
+     * Inbound Discord messages arrive over the Gateway WebSocket (a bot cannot
+     * own the Discord webhook), so a bot token starts the real-time source. The
+     * shared webhook receiver still serves outbound-only webhook setups.
+     */
+    async start(onMessage) {
+        if (!this.token) {
+            // Said out loud: without a bot token there is no socket, and a gateway that
+            // answers outbound but never receives is otherwise indistinguishable from
+            // one whose inbound is broken. Names the key and what still works.
+            logger.info('Discord: real-time Gateway inbound skipped — no bot token (BUFF_DISCORD_BOT_TOKEN). ' +
+                'Webhook inbound (public URL required) and outbound both still work.');
+            return;
+        }
+        const { DiscordGatewaySource } = await import('./realtime.js');
+        const source = new DiscordGatewaySource({ token: this.token });
+        this.gateway = source;
+        void source.start(onMessage).catch(() => {
+            // start() only rejects on a programming error; handshakes retry internally.
+        });
+    }
+    async stop() {
+        await this.gateway?.stop();
+        this.gateway = null;
+    }
     sendUrl(channelId) {
         // A webhook URL may embed its own channel; an explicit channelId with a
         // bot token uses the REST API. When only the webhook URL exists, send there.
@@ -266,9 +370,37 @@ export class SlackAdapter extends WebhookChannelAdapter {
         this.token = envBuff('SLACK_BOT_TOKEN') ?? '';
         this.configured = Boolean(this.webhookUrl || this.token);
     }
+    /** The Socket Mode connection, when an app-level token is configured. */
+    socketMode = null;
     describe() {
         return this.configured ? 'Slack (webhook)' : 'Slack (not configured)';
     }
+    /**
+     * Slack Events API inbound needs a public HTTPS endpoint; apps that cannot
+     * expose one use Socket Mode, which carries the same `event_callback`
+     * envelopes. Enabled by an app-level token (`SLACK_APP_TOKEN`, `xapp-…`).
+     */
+    async start(onMessage) {
+        const appToken = envBuff('SLACK_APP_TOKEN') ?? '';
+        if (!appToken) {
+            logger.info('Slack: real-time Socket Mode inbound skipped — no app-level token (BUFF_SLACK_APP_TOKEN). ' +
+                'The Events API path (public Request URL) and outbound both still work.');
+            return;
+        }
+        const { SlackSocketModeSource } = await import('./realtime.js');
+        const source = new SlackSocketModeSource({
+            appToken,
+            botToken: this.token || undefined,
+        });
+        this.socketMode = source;
+        void source.start(onMessage).catch(() => {
+            // Handshake failures retry with backoff inside the source.
+        });
+    }
+    async stop() {
+        await this.socketMode?.stop();
+        this.socketMode = null;
+    }
     sendUrl(channelId) {
         if (this.webhookUrl)
             return this.webhookUrl;
@@ -334,7 +466,7 @@ export class WhatsAppBridgeAdapter {
     }
     async start(onMessage) {
         this.handler = onMessage;
-        await this.bridge.connect((fromJid, text, participant, messageId) => {
+        await this.bridge.connect((fromJid, text, participant, messageId, media) => {
             // `||` (not `??`): Baileys 7 can deliver participant as an EMPTY string
             // for DMs — an empty string is not nullish, so `??` would blank the
             // sender id and the policy gate would refuse every sender.
@@ -350,6 +482,8 @@ export class WhatsAppBridgeAdapter {
                 // Idempotency: WhatsApp's `key.id` is stable across re-deliveries, so
                 // the gateway collapses the bridge's offline backfill into one turn.
                 messageId,
+                // A downloaded document/image (the caption already folded into `text`).
+                ...(media ? { media } : {}),
             });
         });
     }
@@ -1954,6 +2088,29 @@ export class HomeAssistantAdapter {
         return postJson(`${this.opts.url}/api/services/persistent_notification/create`, { title: 'Agent-Nuvira', message }, this.opts.token);
     }
 }
+/**
+ * Classify an attachment by MIME type, falling back to its filename extension.
+ * Exported for the real-time transports (`realtime.ts`), which receive Discord
+ * `content_type` / Slack `mimetype` on a live gateway event rather than a
+ * webhook body.
+ */
+export function mediaKindFromMime(mimetype, filename) {
+    const mt = (mimetype ?? '').toLowerCase();
+    if (mt.startsWith('image/'))
+        return 'image';
+    if (mt.startsWith('audio/'))
+        return 'audio';
+    if (mt.startsWith('video/'))
+        return 'video';
+    const ext = (filename ?? '').toLowerCase().split('.').pop() ?? '';
+    if (['png', 'jpg', 'jpeg', 'gif', 'webp', 'bmp', 'tiff'].includes(ext))
+        return 'image';
+    if (['mp3', 'm4a', 'ogg', 'oga', 'opus', 'wav', 'flac'].includes(ext))
+        return 'audio';
+    if (['mp4', 'mov', 'mkv', 'webm'].includes(ext))
+        return 'video';
+    return 'document';
+}
 /**
  * Parse platform-specific webhook bodies into a normal InboundMessage.
  * Returns null when the payload isn't a user message (e.g. Slack challenge).
@@ -1963,13 +2120,26 @@ export function parseWebhookPayload(platform, body) {
         if (platform === 'discord') {
             const content = body?.content;
             const channelId = body?.channel_id;
-            if (typeof content === 'string' && channelId) {
+            const first = (Array.isArray(body?.attachments) ? body.attachments[0] : undefined);
+            const hasMedia = Boolean(first?.url);
+            // A media-only message has an EMPTY `content` — it must still dispatch so
+            // the attachment is downloaded and extracted, not dropped for lacking text.
+            if ((typeof content === 'string' || hasMedia) && channelId) {
                 return {
                     channelId,
-                    text: content,
+                    text: typeof content === 'string' ? content : '',
                     from: body?.author?.username ?? 'discord-user',
                     // P1: author id (webhooks can't tell DM vs guild channel).
                     senderId: body?.author?.id !== undefined ? String(body.author.id) : undefined,
+                    ...(hasMedia && first?.url
+                        ? {
+                            media: {
+                                type: mediaKindFromMime(first.content_type, first.filename),
+                                attachment: { url: first.url, filename: first.filename, mimetype: first.content_type, size: first.size },
+                                caption: typeof content === 'string' ? content : undefined,
+                            },
+                        }
+                        : {}),
                 };
             }
             return null;
@@ -1980,14 +2150,27 @@ export function parseWebhookPayload(platform, body) {
             if (body?.challenge)
                 return null;
             const event = body?.event;
-            if (event?.type === 'message' && typeof event.text === 'string' && event.channel) {
+            const file = (Array.isArray(event?.files) ? event.files[0] : undefined);
+            const fileUrl = file?.url_private_download ?? file?.url_private;
+            const hasMedia = Boolean(fileUrl);
+            if (event?.type === 'message' && event.channel && (typeof event.text === 'string' || hasMedia)) {
                 return {
                     channelId: event.channel,
-                    text: event.text,
+                    text: typeof event.text === 'string' ? event.text : '',
                     from: event.user ?? 'slack-user',
                     // P1: event.user IS the Slack user id; D-channels are DMs.
                     senderId: event.user ?? undefined,
                     isGroup: typeof event.channel === 'string' && !event.channel.startsWith('D'),
+                    ...(hasMedia && fileUrl
+                        ? {
+                            media: {
+                                type: mediaKindFromMime(file.mimetype, file.name),
+                                // Slack files require the bot token on the download URL.
+                                attachment: { url: fileUrl, filename: file.name, mimetype: file.mimetype, size: file.size, authenticated: true },
+                                caption: typeof event.text === 'string' ? event.text : undefined,
+                            },
+                        }
+                        : {}),
                 };
             }
             return null;
@@ -2075,6 +2258,9 @@ export class WebhookReceiver {
             res.writeHead(200);
             res.end('ok');
             if (parsed) {
+                // Download the attachment BEFORE dispatch, so the turn always sees a
+                // fully-hydrated document (the registry extracts it like any other).
+                const media = parsed.media ? await this.downloadWebhookMedia(parsed.media) : null;
                 await this.handler?.({
                     platform,
                     channelId: parsed.channelId,
@@ -2083,11 +2269,31 @@ export class WebhookReceiver {
                     // P1: real sender id + DM/group detection ride through the payload.
                     senderId: parsed.senderId,
                     isGroup: parsed.isGroup,
+                    ...(media ? { media } : {}),
                 });
             }
         });
         await new Promise((resolve) => this.server.listen(port, host, resolve));
     }
+    /**
+     * Download a webhook attachment's bytes. Never throws — returns null on any
+     * failure, so the text turn still runs. Slack's `url_private*` endpoints need
+     * the bot token; Discord's CDN URL is public.
+     */
+    async downloadWebhookMedia(media) {
+        const att = media.attachment;
+        const downloaded = await downloadInboundAttachment({
+            type: media.type,
+            url: att.url,
+            filename: att.filename,
+            mimetype: att.mimetype,
+            size: att.size,
+            authenticated: att.authenticated,
+            // Slack's url_private* endpoints need the bot token.
+            token: att.authenticated ? (envBuff('SLACK_BOT_TOKEN') ?? '') : undefined,
+        });
+        return downloaded ? { ...downloaded, caption: media.caption } : null;
+    }
     /**
      * Verify a webhook POST's platform signature. Returns true when the
      * platform has NO secret configured (local/trusted setup) or the signature