@daolmedo/openclaw-twilio-whatsapp 1.1.22 → 1.1.24

package/dist/index.js

@@ -63,30 +63,63 @@ async function sendTwilioWhatsappMessage(opts) {
   const to = opts.to.startsWith("whatsapp:") ? opts.to : `whatsapp:${opts.to}`;
   const chunks = opts.body ? splitMessage(opts.body) : [void 0];
   let lastSid = "";
-  for (const chunk of chunks) {
-    const msg = await client.messages.create({
-      from,
-      to,
-      ...chunk ? { body: chunk } : {},
-      ...opts.mediaUrl ? { mediaUrl: [opts.mediaUrl] } : {}
-    });
-    lastSid = msg.sid;
+  for (let i = 0; i < chunks.length; i += 1) {
+    const chunk = chunks[i];
+    console.log(
+      `[twilio-whatsapp] Twilio create start from=${from} to=${to} chunk=${i + 1}/${chunks.length} bodyChars=${chunk?.length ?? 0} media=${opts.mediaUrl ? "yes" : "no"}`
+    );
+    try {
+      const msg = await client.messages.create({
+        from,
+        to,
+        ...chunk ? { body: chunk } : {},
+        ...opts.mediaUrl ? { mediaUrl: [opts.mediaUrl] } : {}
+      });
+      lastSid = msg.sid;
+      console.log(
+        `[twilio-whatsapp] Twilio create ok sid=${msg.sid} status=${msg.status} chunk=${i + 1}/${chunks.length}`
+      );
+    } catch (err) {
+      console.error(
+        `[twilio-whatsapp] Twilio create failed chunk=${i + 1}/${chunks.length}:`,
+        err
+      );
+      throw err;
+    }
   }
   return lastSid;
 }
 
 // src/outbound-adapter.ts
+function summarizeText(text) {
+  if (!text) return "chars=0";
+  return `chars=${text.length} preview=${JSON.stringify(text.slice(0, 120))}`;
+}
 async function deliver(ctx) {
   const account = resolveTwilioAccount(ctx.cfg, ctx.accountId);
-  const sid = await sendTwilioWhatsappMessage({
-    accountSid: account.accountSid,
-    authToken: account.authToken,
-    from: account.phoneNumber,
-    to: ctx.to,
-    body: ctx.text || void 0,
-    mediaUrl: ctx.mediaUrl
-  });
-  return sid;
+  console.log(
+    `[twilio-whatsapp] Durable adapter deliver start accountId=${ctx.accountId ?? "default"} from=${account.phoneNumber} to=${ctx.to} ${summarizeText(ctx.text)} media=${ctx.mediaUrl ? "yes" : "no"}`
+  );
+  try {
+    const sid = await sendTwilioWhatsappMessage({
+      accountSid: account.accountSid,
+      authToken: account.authToken,
+      from: account.phoneNumber,
+      to: ctx.to,
+      body: ctx.text || void 0,
+      mediaUrl: ctx.mediaUrl
+    });
+    console.log(
+      `[twilio-whatsapp] Durable adapter deliver ok accountId=${ctx.accountId ?? "default"} outboundSid=${sid}`
+    );
+    return sid;
+  } catch (err) {
+    console.error(
+      `[twilio-whatsapp] Durable adapter deliver failed accountId=${ctx.accountId ?? "default"}:`,
+      err
+    );
+    throw err;
+  }
 }
 var twilioWhatsappOutbound = {
   deliveryMode: "direct",
@@ -211,7 +244,7 @@ var twilioWhatsappPlugin = {
 // src/http-routes.ts
 import { readFile } from "node:fs/promises";
 import path from "node:path";
-import { resolveInboundRouteEnvelopeBuilderWithRuntime } from "openclaw/plugin-sdk/inbound-envelope";
+import { dispatchInboundDirectDmWithRuntime } from "openclaw/plugin-sdk/direct-dm";
 import { saveMediaBuffer } from "openclaw/plugin-sdk/media-runtime";
 
 // src/webhook.ts
@@ -278,6 +311,20 @@ function startTypingKeepalive(opts) {
 
 // src/http-routes.ts
 var MEDIA_MAX_BYTES = 16 * 1024 * 1024;
+function resolveMediaBatchFlushMs() {
+  const raw = Number(process.env.TWILIO_WHATSAPP_MEDIA_BATCH_FLUSH_MS);
+  if (Number.isFinite(raw) && raw >= 0) return Math.min(raw, 6e4);
+  return 3e3;
+}
+function resolveMediaBatchMaxItems() {
+  const raw = Number(process.env.TWILIO_WHATSAPP_MEDIA_BATCH_MAX_ITEMS);
+  if (Number.isFinite(raw) && raw > 0) return Math.floor(Math.min(raw, 50));
+  return 10;
+}
+function summarizeText2(text) {
+  if (!text) return "chars=0";
+  return `chars=${text.length} preview=${JSON.stringify(text.slice(0, 120))}`;
+}
 async function readBody(req) {
   return new Promise((resolve, reject) => {
     const chunks = [];
@@ -296,6 +343,11 @@ function twimlError(res, status, message) {
     `<?xml version="1.0" encoding="UTF-8"?><Response><Message>${message}</Message></Response>`
   );
 }
+var pendingMediaByConversation = /* @__PURE__ */ new Map();
+var dispatchQueueByConversation = /* @__PURE__ */ new Map();
+function resolvePendingMediaKey(accountId, fromPhone) {
+  return `${accountId}:${fromPhone}`;
+}
 async function downloadTwilioMedia(url, account) {
   try {
     const authHeader = "Basic " + Buffer.from(`${account.accountSid}:${account.authToken}`).toString("base64");
@@ -322,6 +374,243 @@ async function downloadTwilioMedia(url, account) {
     return null;
   }
 }
+function enqueueConversationDispatch(key, task) {
+  const previous = dispatchQueueByConversation.get(key) ?? Promise.resolve();
+  const next = previous.catch(() => void 0).then(task).catch((err) => {
+    console.error(`[twilio-whatsapp] Conversation dispatch failed key=${key}:`, err);
+  });
+  dispatchQueueByConversation.set(key, next);
+  void next.finally(() => {
+    if (dispatchQueueByConversation.get(key) === next) {
+      dispatchQueueByConversation.delete(key);
+    }
+  });
+}
+function stopBatchTimer(batch) {
+  if (batch.timer) {
+    clearTimeout(batch.timer);
+    batch.timer = void 0;
+  }
+}
+function armBatchTimer(api, key, batch) {
+  stopBatchTimer(batch);
+  const flushMs = resolveMediaBatchFlushMs();
+  batch.timer = setTimeout(() => {
+    flushPendingMediaBatch(api, key, "timer");
+  }, flushMs);
+  const maybeTimer = batch.timer;
+  maybeTimer.unref?.();
+}
+function combineBatchMessageText(items) {
+  return items.map((item) => item.messageText.trim()).filter(Boolean).join("\n\n");
+}
+function flushPendingMediaBatch(api, key, reason) {
+  const batch = pendingMediaByConversation.get(key);
+  if (!batch) return;
+  stopBatchTimer(batch);
+  pendingMediaByConversation.delete(key);
+  const items = [...batch.items].sort((a, b) => a.receivedAt - b.receivedAt);
+  const last = items[items.length - 1];
+  if (!last) return;
+  const resolvedMedia = items.flatMap((item) => item.resolvedMedia);
+  if (!resolvedMedia.length && !combineBatchMessageText(items)) {
+    return;
+  }
+  const messageSids = items.map((item) => item.fields.messageSid || "unknown");
+  console.log(
+    `[twilio-whatsapp] Flushing media batch key=${key} reason=${reason} messages=${items.length} media=${resolvedMedia.length} sids=${messageSids.join(",")}`
+  );
+  enqueueConversationDispatch(key, async () => {
+    await dispatchTwilioInbound(api, {
+      ...last,
+      messageText: combineBatchMessageText(items),
+      resolvedMedia
+    });
+  });
+}
+function queueMediaBatch(api, key, item) {
+  let batch = pendingMediaByConversation.get(key);
+  if (!batch) {
+    batch = { items: [] };
+    pendingMediaByConversation.set(key, batch);
+  }
+  batch.items.push(item);
+  console.log(
+    `[twilio-whatsapp] Buffered media inbound sid=${item.fields.messageSid || "unknown"} key=${key} pendingMessages=${batch.items.length} pendingMedia=${batch.items.reduce(
+      (count, pending) => count + pending.resolvedMedia.length,
+      0
+    )}`
+  );
+  if (batch.items.length >= resolveMediaBatchMaxItems()) {
+    flushPendingMediaBatch(api, key, "max-items");
+    return;
+  }
+  if (resolveMediaBatchFlushMs() === 0) {
+    flushPendingMediaBatch(api, key, "disabled");
+    return;
+  }
+  armBatchTimer(api, key, batch);
+}
+async function dispatchTwilioInbound(api, params) {
+  const {
+    fields,
+    stored,
+    account,
+    toPhone,
+    fromPhone,
+    senderAddress,
+    recipientAddress,
+    conversationLabel,
+    resolvedMedia
+  } = params;
+  let messageText = params.messageText;
+  const mediaContext = resolvedMedia.length > 0 ? {
+    MediaPath: resolvedMedia[0].path,
+    MediaUrl: resolvedMedia[0].path,
+    MediaType: resolvedMedia[0].contentType,
+    MediaPaths: resolvedMedia.map((m) => m.path),
+    MediaUrls: resolvedMedia.map((m) => m.path),
+    MediaTypes: resolvedMedia.map((m) => m.contentType)
+  } : {};
+  let bodyForAgent = messageText || void 0;
+  const firstAudio = resolvedMedia.find((m) => m.contentType.startsWith("audio/"));
+  if (firstAudio) {
+    try {
+      const cfgAny = stored.cfg;
+      const agentId = cfgAny.bindings?.find(
+        (b) => b.match?.channel === "twilio-whatsapp" && b.match?.accountId === stored.accountId
+      )?.agentId;
+      const agentDir = cfgAny.agents?.list?.find(
+        (a) => a.id === agentId
+      )?.agentDir;
+      const auth = await api.runtime.modelAuth.resolveApiKeyForProvider({
+        provider: "openai",
+        cfg: api.config,
+        agentDir
+      });
+      if (!auth.apiKey) throw new Error("No OpenAI API key available for audio transcription");
+      const audioBuffer = await readFile(firstAudio.path);
+      const form = new FormData();
+      form.append(
+        "file",
+        new Blob([audioBuffer], { type: firstAudio.contentType }),
+        path.basename(firstAudio.path)
+      );
+      form.append("model", "gpt-4o-mini-transcribe");
+      form.append("response_format", "json");
+      const transcribeRes = await fetch("https://api.openai.com/v1/audio/transcriptions", {
+        method: "POST",
+        headers: { Authorization: `Bearer ${auth.apiKey}` },
+        body: form,
+        signal: AbortSignal.timeout(12e4)
+      });
+      if (!transcribeRes.ok) {
+        const errBody = await transcribeRes.text();
+        throw new Error(`OpenAI transcription HTTP ${transcribeRes.status}: ${errBody}`);
+      }
+      const transcribeJson = await transcribeRes.json();
+      const text = transcribeJson.text;
+      console.log("[twilio-whatsapp] Transcription result:", text);
+      if (text) {
+        bodyForAgent = messageText ? `${messageText}
+
+[Audio]
+Transcript:
+${text}` : `[Audio]
+Transcript:
+${text}`;
+        messageText = messageText ? `${messageText}
+
+${text}` : text;
+        mediaContext.Transcript = text;
+      } else {
+        bodyForAgent = messageText || "<media:audio>";
+      }
+    } catch (err) {
+      console.error("[twilio-whatsapp] Audio transcription failed:", err);
+      bodyForAgent = messageText || "<media:audio>";
+    }
+  }
+  console.log(
+    `[twilio-whatsapp] Dispatching inbound sid=${fields.messageSid || "unknown"} bodyForAgent=${summarizeText2(bodyForAgent)} media=${resolvedMedia.length}`
+  );
+  const stopTypingKeepalive = fields.messageSid ? (() => {
+    console.log(`[twilio-whatsapp] Typing keepalive start sid=${fields.messageSid}`);
+    return startTypingKeepalive({
+      accountSid: account.accountSid,
+      authToken: account.authToken,
+      messageId: fields.messageSid
+    });
+  })() : () => {
+  };
+  let typingStopped = false;
+  const stopTyping = () => {
+    if (typingStopped) return;
+    typingStopped = true;
+    stopTypingKeepalive();
+    console.log(`[twilio-whatsapp] Typing keepalive stop sid=${fields.messageSid || "unknown"}`);
+  };
+  try {
+    const timestamp = Date.now();
+    await dispatchInboundDirectDmWithRuntime({
+      cfg: stored.cfg,
+      runtime: { channel: stored.channelRuntime },
+      channel: "twilio-whatsapp",
+      channelLabel: "WhatsApp",
+      accountId: account.accountId,
+      peer: { kind: "direct", id: fromPhone },
+      senderId: fromPhone,
+      senderAddress,
+      recipientAddress,
+      conversationLabel,
+      rawBody: messageText,
+      bodyForAgent,
+      messageId: fields.messageSid,
+      timestamp,
+      commandAuthorized: true,
+      provider: "twilio-whatsapp",
+      surface: "twilio-whatsapp",
+      originatingChannel: "twilio-whatsapp",
+      originatingTo: senderAddress,
+      extraContext: mediaContext,
+      deliver: async (payload) => {
+        const text = payload.text ?? "";
+        const mediaUrl = payload.mediaUrl ?? payload.mediaUrls?.[0];
+        console.log(
+          `[twilio-whatsapp] Direct deliver invoked sid=${fields.messageSid || "unknown"} ${summarizeText2(text)} media=${mediaUrl ? "yes" : "no"}`
+        );
+        if (!text && !mediaUrl) {
+          console.log(
+            `[twilio-whatsapp] Direct deliver skipped empty payload sid=${fields.messageSid || "unknown"}`
+          );
+          return;
+        }
+        console.log(
+          `[twilio-whatsapp] Twilio direct send start sid=${fields.messageSid || "unknown"} from=${recipientAddress} to=${senderAddress} ${summarizeText2(text)} media=${mediaUrl ? "yes" : "no"}`
+        );
+        const messageId = await sendTwilioWhatsappMessage({
+          accountSid: account.accountSid,
+          authToken: account.authToken,
+          from: toPhone,
+          to: fromPhone,
+          body: text || void 0,
+          mediaUrl
+        });
+        console.log(
+          `[twilio-whatsapp] Twilio direct send ok sid=${fields.messageSid || "unknown"} outboundSid=${messageId}`
+        );
+      },
+      onRecordError: (err) => {
+        console.error("[twilio-whatsapp] Session record error:", err);
+      },
+      onDispatchError: (err, info) => {
+        console.error(`[twilio-whatsapp] Dispatch error (${info.kind}):`, err);
+      }
+    });
+  } finally {
+    stopTyping();
+  }
+}
 function registerTwilioWhatsappHttpRoutes(api) {
   api.registerHttpRoute({
     path: "/twilio/whatsapp/events",
@@ -333,6 +622,9 @@ function registerTwilioWhatsappHttpRoutes(api) {
       const webhookUrl = String(headers["x-twilio-webhook-url"] ?? "");
       const fields = parseTwilioWebhook(rawBody);
       const toPhone = normalizePhoneNumber(fields.to);
+      console.log(
+        `[twilio-whatsapp] Webhook received sid=${fields.messageSid || "unknown"} from=${fields.from} to=${fields.to} bodyChars=${fields.body.length} media=${fields.media.length}`
+      );
       const stored = getRuntimeForPhoneNumber(toPhone);
       if (!stored) {
         console.error(`[twilio-whatsapp] No runtime found for To=${fields.to}`);
@@ -358,7 +650,7 @@ function registerTwilioWhatsappHttpRoutes(api) {
           return true;
         }
       } else {
-        console.warn("[twilio-whatsapp] Skipping sig verification \u2014 headers missing");
+        console.warn("[twilio-whatsapp] Skipping sig verification - headers missing");
       }
       const fromPhone = normalizePhoneNumber(fields.from);
       const senderAddress = `whatsapp:${fromPhone}`;
@@ -366,11 +658,15 @@ function registerTwilioWhatsappHttpRoutes(api) {
       const conversationLabel = fields.profileName || fromPhone;
       const messageText = fields.body.trim();
       const hasMedia = fields.media.length > 0;
+      const receivedAt = Date.now();
       if (!messageText && !hasMedia) {
         twimlOk(res);
         return true;
       }
       twimlOk(res);
+      console.log(
+        `[twilio-whatsapp] Webhook acknowledged sid=${fields.messageSid || "unknown"} accountId=${account.accountId}`
+      );
       const resolvedMedia = [];
       if (hasMedia) {
         const results = await Promise.all(
@@ -379,165 +675,49 @@ function registerTwilioWhatsappHttpRoutes(api) {
         for (const r of results) {
           if (r) resolvedMedia.push(r);
         }
+        console.log(
+          `[twilio-whatsapp] Media resolved sid=${fields.messageSid || "unknown"} requested=${fields.media.length} saved=${resolvedMedia.length}`
+        );
       }
-      const mediaContext = resolvedMedia.length > 0 ? {
-        MediaPath: resolvedMedia[0].path,
-        MediaUrl: resolvedMedia[0].path,
-        MediaType: resolvedMedia[0].contentType,
-        MediaPaths: resolvedMedia.map((m) => m.path),
-        MediaUrls: resolvedMedia.map((m) => m.path),
-        MediaTypes: resolvedMedia.map((m) => m.contentType)
-      } : {};
-      let bodyForAgent = messageText || void 0;
-      const firstAudio = resolvedMedia.find((m) => m.contentType.startsWith("audio/"));
-      if (firstAudio) {
-        try {
-          const cfgAny = stored.cfg;
-          const agentId = cfgAny.bindings?.find(
-            (b) => b.match?.channel === "twilio-whatsapp" && b.match?.accountId === stored.accountId
-          )?.agentId;
-          const agentDir = cfgAny.agents?.list?.find(
-            (a) => a.id === agentId
-          )?.agentDir;
-          const auth = await api.runtime.modelAuth.resolveApiKeyForProvider({
-            provider: "openai",
-            cfg: api.config,
-            agentDir
-          });
-          if (!auth.apiKey) throw new Error("No OpenAI API key available for audio transcription");
-          const audioBuffer = await readFile(firstAudio.path);
-          const form = new FormData();
-          form.append(
-            "file",
-            new Blob([audioBuffer], { type: firstAudio.contentType }),
-            path.basename(firstAudio.path)
-          );
-          form.append("model", "gpt-4o-mini-transcribe");
-          form.append("response_format", "json");
-          const transcribeRes = await fetch("https://api.openai.com/v1/audio/transcriptions", {
-            method: "POST",
-            headers: { Authorization: `Bearer ${auth.apiKey}` },
-            body: form,
-            signal: AbortSignal.timeout(12e4)
-          });
-          if (!transcribeRes.ok) {
-            const errBody = await transcribeRes.text();
-            throw new Error(`OpenAI transcription HTTP ${transcribeRes.status}: ${errBody}`);
-          }
-          const transcribeJson = await transcribeRes.json();
-          const text = transcribeJson.text;
-          console.log("[twilio-whatsapp] Transcription result:", text);
-          if (text) {
-            bodyForAgent = `[Audio]
-Transcript:
-${text}`;
-            mediaContext.Transcript = text;
-          } else {
-            bodyForAgent = messageText || "<media:audio>";
-          }
-        } catch (err) {
-          console.error("[twilio-whatsapp] Audio transcription failed:", err);
-          bodyForAgent = messageText || "<media:audio>";
-        }
+      const pendingMediaKey = resolvePendingMediaKey(account.accountId, fromPhone);
+      if (hasMedia && resolvedMedia.length > 0) {
+        queueMediaBatch(api, pendingMediaKey, {
+          fields,
+          stored,
+          account,
+          toPhone,
+          fromPhone,
+          senderAddress,
+          recipientAddress,
+          conversationLabel,
+          messageText,
+          resolvedMedia,
+          receivedAt
+        });
+        return true;
       }
-      console.log("[twilio-whatsapp] Dispatching bodyForAgent:", bodyForAgent);
-      const stopTypingKeepalive = fields.messageSid ? startTypingKeepalive({
-        accountSid: account.accountSid,
-        authToken: account.authToken,
-        messageId: fields.messageSid
-      }) : () => {
-      };
-      let typingStopped = false;
-      const stopTyping = () => {
-        if (typingStopped) return;
-        typingStopped = true;
-        stopTypingKeepalive();
-      };
-      const { route, buildEnvelope } = resolveInboundRouteEnvelopeBuilderWithRuntime({
-        cfg: stored.cfg,
-        channel: "twilio-whatsapp",
-        accountId: account.accountId,
-        peer: { kind: "direct", id: fromPhone },
-        runtime: stored.channelRuntime,
-        sessionStore: stored.cfg.session?.store
-      });
-      const timestamp = Date.now();
-      const rawBodyForContext = mediaContext.Transcript ? String(mediaContext.Transcript) : messageText;
-      const { storePath, body } = buildEnvelope({
-        channel: "WhatsApp",
-        from: conversationLabel,
-        body: rawBodyForContext,
-        timestamp
-      });
-      const ctxPayload = stored.channelRuntime.reply.finalizeInboundContext({
-        Body: body,
-        BodyForAgent: bodyForAgent ?? rawBodyForContext,
-        RawBody: rawBodyForContext,
-        CommandBody: rawBodyForContext,
-        From: senderAddress,
-        To: recipientAddress,
-        SessionKey: route.sessionKey,
-        AccountId: route.accountId ?? account.accountId,
-        ChatType: "direct",
-        ConversationLabel: conversationLabel,
-        SenderId: fromPhone,
-        Provider: "twilio-whatsapp",
-        Surface: "twilio-whatsapp",
-        MessageSid: fields.messageSid,
-        MessageSidFull: fields.messageSid,
-        Timestamp: timestamp,
-        CommandAuthorized: true,
-        OriginatingChannel: "twilio-whatsapp",
-        OriginatingTo: senderAddress,
-        ...mediaContext
-      });
-      stored.channelRuntime.inbound.dispatchReply({
-        cfg: stored.cfg,
-        channel: "twilio-whatsapp",
-        accountId: route.accountId ?? account.accountId,
-        agentId: route.agentId,
-        routeSessionKey: route.sessionKey,
-        storePath,
-        ctxPayload,
-        recordInboundSession: stored.channelRuntime.session.recordInboundSession,
-        dispatchReplyWithBufferedBlockDispatcher: stored.channelRuntime.reply.dispatchReplyWithBufferedBlockDispatcher,
-        delivery: {
-          deliver: async (payload) => {
-            const outbound = payload;
-            const text = outbound.text ?? "";
-            const mediaUrl = outbound.mediaUrl ?? outbound.mediaUrls?.[0];
-            if (!text && !mediaUrl) {
-              return { visibleReplySent: false };
-            }
-            const messageId = await sendTwilioWhatsappMessage({
-              accountSid: account.accountSid,
-              authToken: account.authToken,
-              from: toPhone,
-              to: fromPhone,
-              body: text || void 0,
-              mediaUrl
-            });
-            return {
-              messageIds: [messageId],
-              visibleReplySent: true
-            };
-          },
-          onError: (err, info) => {
-            stopTyping();
-            console.error(`[twilio-whatsapp] Dispatch error (${info.kind}):`, err);
-          }
-        },
-        replyPipeline: {},
-        record: {
-          onRecordError: (err) => {
-            console.error("[twilio-whatsapp] Session record error:", err);
-          }
-        },
-        messageId: fields.messageSid
-      }).catch((err) => {
-        console.error("[twilio-whatsapp] Unhandled dispatch error:", err);
-      }).finally(() => {
-        stopTyping();
+      if (hasMedia && resolvedMedia.length === 0 && !messageText) {
+        console.warn(
+          `[twilio-whatsapp] Dropping media-only inbound with no resolved media sid=${fields.messageSid || "unknown"}`
+        );
+        return true;
+      }
+      if (!hasMedia) {
+        flushPendingMediaBatch(api, pendingMediaKey, "before-text");
+      }
+      enqueueConversationDispatch(pendingMediaKey, async () => {
+        await dispatchTwilioInbound(api, {
+          fields,
+          stored,
+          account,
+          toPhone,
+          fromPhone,
+          senderAddress,
+          recipientAddress,
+          conversationLabel,
+          messageText,
+          resolvedMedia
+        });
       });
       return true;
     }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@daolmedo/openclaw-twilio-whatsapp",
-  "version": "1.1.22",
+  "version": "1.1.24",
   "description": "Twilio WhatsApp channel plugin for OpenClaw (Cody ISV)",
   "type": "module",
   "exports": {