clawgram 2.4.3 → 2.5.1

package/dist/channel.js

@@ -1,7 +1,18 @@
 "use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.createChannelPlugin = void 0;
 const core_1 = require("openclaw/plugin-sdk/core");
+const node_os_1 = __importDefault(require("node:os"));
+const node_path_1 = __importDefault(require("node:path"));
+const node_fs_1 = require("node:fs");
+/** Attachments above this are left unread: a long recording or a huge image is
+ *  a different conversation from a spoken line or a screenshot, and the
+ *  transfer is not free. */
+const INBOUND_MEDIA_MAX_BYTES = 25 * 1024 * 1024;
+const media_1 = require("./media");
 const channel_runtime_1 = require("openclaw/plugin-sdk/channel-runtime");
 const param_readers_1 = require("openclaw/plugin-sdk/param-readers");
 const tool_send_1 = require("openclaw/plugin-sdk/tool-send");
@@ -55,7 +66,121 @@ function parseOptionalThreadId(value) {
     const parsed = Number.parseInt(trimmed, 10);
     return Number.isFinite(parsed) ? parsed : undefined;
 }
-const createChannelPlugin = (runtimes) => {
+/**
+ * Turns an inbound attachment into text the agent can read.
+ *
+ * The work is deliberately delegated: `runtime.mediaUnderstanding` already
+ * knows which backend this installation uses for speech and for images, so
+ * the channel stays out of that choice — a local model today, something else
+ * tomorrow, without touching this file.
+ *
+ * Failure is not an error worth dropping the message over. An attachment that
+ * could not be read still happened, and the assistant is better off saying
+ * "you sent something I could not read" than staying silent, which is
+ * indistinguishable from being offline.
+ */
+/**
+ * Locates the agent directory that image understanding needs.
+ *
+ * Image models are called with the agent's own credentials, so the pipeline
+ * refuses to run without this path — audio does not need it, which is why
+ * voice notes worked before images did. The platform exposes no resolver to
+ * plugins, so the documented layout is reconstructed here and checked before
+ * use: a wrong guess would fail the read anyway, and returning undefined lets
+ * the caller degrade instead of throwing.
+ */
+function resolveAgentDirForMedia(cfg) {
+    const stateDir = typeof process.env.OPENCLAW_STATE_DIR === "string" && process.env.OPENCLAW_STATE_DIR.trim()
+        ? process.env.OPENCLAW_STATE_DIR.trim()
+        : node_path_1.default.join(node_os_1.default.homedir(), ".openclaw");
+    const configuredId = cfg?.agents?.defaults?.id;
+    const agentId = typeof configuredId === "string" && configuredId.trim() ? configuredId.trim() : "main";
+    const dir = node_path_1.default.join(stateDir, "agents", agentId, "agent");
+    return (0, node_fs_1.existsSync)(dir) ? dir : undefined;
+}
+async function readInboundAttachment(params) {
+    const media = params.runtime?.mediaUnderstanding;
+    const message = params.event?.message;
+    if (!media || !message) {
+        return undefined;
+    }
+    let downloaded;
+    try {
+        downloaded = await (0, media_1.downloadInboundMediaToTempFile)({
+            client: params.gram.getClient(),
+            message,
+            maxBytes: INBOUND_MEDIA_MAX_BYTES,
+            tmpDir: node_os_1.default.tmpdir(),
+        });
+    }
+    catch (err) {
+        params.log?.info?.("clawgram attachment download failed", {
+            accountId: params.accountId,
+            chatId: params.chatId,
+            messageId: params.messageId,
+            error: String(err),
+        });
+        return undefined;
+    }
+    if (!downloaded) {
+        return undefined;
+    }
+    try {
+        const result = downloaded.understanding === "transcript"
+            ? await media.transcribeAudioFile({
+                filePath: downloaded.path,
+                cfg: params.cfg,
+                mime: downloaded.mimeType,
+            })
+            : await media.describeImageFile({
+                filePath: downloaded.path,
+                cfg: params.cfg,
+                mime: downloaded.mimeType,
+                agentDir: resolveAgentDirForMedia(params.cfg),
+            });
+        const read = typeof result?.text === "string" ? result.text.trim() : "";
+        if (!read) {
+            params.log?.info?.("clawgram attachment read empty", {
+                accountId: params.accountId,
+                chatId: params.chatId,
+                messageId: params.messageId,
+                understanding: downloaded.understanding,
+            });
+            return undefined;
+        }
+        params.log?.info?.("clawgram attachment read", {
+            accountId: params.accountId,
+            chatId: params.chatId,
+            messageId: params.messageId,
+            understanding: downloaded.understanding,
+            characters: read.length,
+        });
+        return { text: read, understanding: downloaded.understanding };
+    }
+    catch (err) {
+        params.log?.info?.("clawgram attachment read failed", {
+            accountId: params.accountId,
+            chatId: params.chatId,
+            messageId: params.messageId,
+            understanding: downloaded.understanding,
+            error: String(err),
+        });
+        return undefined;
+    }
+    finally {
+        void (async () => {
+            try {
+                const { rm } = await import("node:fs/promises");
+                const { dirname } = await import("node:path");
+                await rm(dirname(downloaded.path), { recursive: true, force: true });
+            }
+            catch {
+                // Leaving a temp file behind is not worth failing a delivered message.
+            }
+        })();
+    }
+}
+const createChannelPlugin = (runtimes, pluginRuntime) => {
     const resolveRuntimeAccountId = (cfg, preferred) => {
         const configured = (0, helpers_1.resolveConfiguredAccountId)(cfg, preferred);
         if (configured && runtimes.has(configured)) {
@@ -269,7 +394,28 @@ const createChannelPlugin = (runtimes) => {
                             });
                             return;
                         }
-                        const text = normalized.text?.trim();
+                        let text = normalized.text?.trim();
+                        // An attachment carries no text of its own, and dropping it as
+                        // "empty" is how the assistant used to go silent on being spoken
+                        // to or shown something. Read it into the body instead: for a
+                        // voice note and a screenshot alike, the attachment *is* the
+                        // message. A caption is kept and the reading appended, because
+                        // "look at this" plus the picture is one thought, not two.
+                        const attachment = await readInboundAttachment({
+                            gram,
+                            event,
+                            cfg,
+                            runtime: pluginRuntime,
+                            log,
+                            accountId,
+                            chatId: normalized.chatId,
+                            messageId: normalized.messageId,
+                        });
+                        if (attachment) {
+                            const marker = attachment.understanding === "transcript" ? "голосовое" : "изображение";
+                            const read = `[${marker}] ${attachment.text}`;
+                            text = text ? `${text}\n\n${read}` : read;
+                        }
                         if (!text) {
                             log?.info?.("clawgram skipping empty inbound text", {
                                 accountId,

package/dist/index.js

@@ -14,7 +14,9 @@ const plugin = {
             commands: (0, cli_1.getTelegramUserbotCliDescriptors)().map((entry) => entry.name),
             descriptors: (0, cli_1.getTelegramUserbotCliDescriptors)()
         });
-        api.registerChannel({ plugin: (0, channel_1.createChannelPlugin)(runtimes) });
+        // api.runtime carries the media-understanding pipeline; without it an
+        // inbound voice note has nothing to be turned into words with.
+        api.registerChannel({ plugin: (0, channel_1.createChannelPlugin)(runtimes, api?.runtime) });
     }
 };
 exports.default = plugin;

package/dist/media.js

@@ -13,6 +13,8 @@
  */
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.describeMedia = describeMedia;
+exports.inboundMediaUnderstanding = inboundMediaUnderstanding;
+exports.downloadInboundMediaToTempFile = downloadInboundMediaToTempFile;
 /**
  * GramJS carries numbers as `big-integer` objects as often as native numbers —
  * the same shape that once made `senderId` silently undefined. Anything that
@@ -98,3 +100,70 @@ function describeMedia(media) {
     // as "something was attached" — a blank message is the failure being fixed.
     return { kind: "other", telegramType: className };
 }
+/**
+ * Decides whether an attachment is worth fetching, and what reading it means.
+ *
+ * Voice notes and images are the two kinds whose bytes *are* the message:
+ * dropping them leaves the assistant silent on being spoken to or shown
+ * something. Other attachments keep the old treatment — metadata only —
+ * because "spec.pdf, 240 KB" already tells a reader what happened, and
+ * fetching every document would be a different feature with different costs.
+ */
+function inboundMediaUnderstanding(media) {
+    if (!media)
+        return undefined;
+    if (media.kind === "voice" || media.kind === "audio")
+        return "transcript";
+    if (media.kind === "photo")
+        return "description";
+    // A document can be an image sent "as file" — Telegram keeps the pixels,
+    // only the envelope differs, so read it rather than announce it.
+    if (media.kind === "document" && media.mimeType?.startsWith("image/"))
+        return "description";
+    return undefined;
+}
+/**
+ * Downloads an inbound attachment to a temporary file.
+ *
+ * Returns the path, or undefined when the attachment is not one this channel
+ * reads, or is too large to be worth the transfer. The caller owns the file
+ * and is responsible for removing it.
+ */
+async function downloadInboundMediaToTempFile(params) {
+    const described = describeMedia(params.message?.media);
+    const understanding = inboundMediaUnderstanding(described);
+    if (!described || !understanding) {
+        return undefined;
+    }
+    // A cap belongs here rather than in the caller: an oversized attachment
+    // should be reported as such, not fetched and then discarded after the
+    // transfer cost.
+    if (typeof described.size === "number" && described.size > params.maxBytes) {
+        return undefined;
+    }
+    const buffer = await params.client.downloadMedia(params.message, {});
+    if (!buffer || !(buffer instanceof Buffer) || buffer.length === 0) {
+        return undefined;
+    }
+    const extension = extensionFor(described, understanding);
+    const { mkdtemp, writeFile } = await import("node:fs/promises");
+    const { join } = await import("node:path");
+    const dir = await mkdtemp(join(params.tmpDir, "clawgram-media-"));
+    const path = join(dir, `attachment.${extension}`);
+    await writeFile(path, buffer);
+    return { path, mimeType: described.mimeType, understanding };
+}
+function extensionFor(media, understanding) {
+    if (understanding === "description") {
+        if (media.mimeType === "image/png")
+            return "png";
+        if (media.mimeType === "image/webp")
+            return "webp";
+        return "jpg";
+    }
+    if (media.mimeType === "audio/mpeg")
+        return "mp3";
+    if (media.mimeType === "audio/mp4")
+        return "m4a";
+    return "ogg";
+}

package/openclaw.plugin.json

@@ -2,7 +2,7 @@
   "id": "clawgram",
   "name": "Clawgram",
   "description": "Clawgram — personal Telegram (MTProto userbot) channel for OpenClaw. Your AI assistant reads and responds as you.",
-  "version": "2.4.3",
+  "version": "2.5.1",
   "configSchema": {
     "type": "object",
     "additionalProperties": false,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "clawgram",
-  "version": "2.4.3",
+  "version": "2.5.1",
   "description": "Clawgram — personal Telegram (MTProto userbot) channel for OpenClaw. Your AI assistant reads and responds as you.",
   "main": "./dist/index.js",
   "scripts": {