@agent-wechat/agent-wechat 0.13.0 → 0.15.0

package/dist/index.js

@@ -1127,6 +1127,9 @@ function getWeChatRuntime() {
   return runtime;
 }
 
+// src/monitor.ts
+import { createHash as createHash3 } from "node:crypto";
+
 // ../shared/dist/client.js
 function normalizeUrl(base) {
   const url = base.startsWith("http") ? base : `http://${base}`;
@@ -1230,6 +1233,23 @@ var WeChatClient = class {
   async sendMessage(params) {
     return this.post("/api/messages/send", params);
   }
+  async createVoiceJob(chatId, audio, idempotencyKey) {
+    const form = new FormData();
+    form.set("chatId", chatId);
+    form.set("audio", new Blob([new Uint8Array(audio)]), "voice-audio");
+    const headers = { ...this.headers, "Idempotency-Key": idempotencyKey };
+    delete headers["Content-Type"];
+    const res = await fetch(`${this.base}/api/messages/voice`, { method: "POST", headers, body: form });
+    if (!res.ok)
+      throw new Error(`${res.status} ${res.statusText}: ${await res.text()}`);
+    return res.json();
+  }
+  async getVoiceJob(jobId) {
+    return this.get(`/api/messages/voice/${encodeURIComponent(jobId)}`);
+  }
+  async cancelVoiceJob(jobId) {
+    return this.post(`/api/messages/voice/${encodeURIComponent(jobId)}/cancel`);
+  }
   // ---- Debug ----
   async screenshot() {
     return this.get("/api/debug/screenshot");
@@ -5561,11 +5581,25 @@ ${reference}` : reference;
 function safeDisplayName(name) {
   return name.replace(/[\r\n\0]/g, " ").trim() || "attachment";
 }
+function attachmentFallbackFilename(kind, localId, originalBody = "") {
+  if (kind === "file" && originalBody && !originalBody.trimStart().startsWith("<")) {
+    return safeDisplayName(originalBody);
+  }
+  const extension = kind === "image" ? "jpg" : kind === "audio" ? "mp3" : kind === "video" ? "mp4" : "bin";
+  return `message-${localId}.${extension}`;
+}
+function attachmentSourceBody(originalBody, kind) {
+  if (!kind || kind === "file") return originalBody;
+  return originalBody.trimStart().startsWith("<") ? "" : originalBody;
+}
 function renderAttachmentBody(originalBody, attachment, exposeNonVisualPath = true) {
   if (!attachment) return originalBody;
   const name = safeDisplayName(attachment.filename);
   if (attachment.status !== "ready" || !attachment.path) {
     const label = attachment.kind === "file" ? "File" : attachment.kind === "image" ? "Image" : attachment.kind === "audio" ? "Voice message" : "Video";
+    if (attachment.status === "pending") {
+      return appendReference(originalBody, `[${label} pending: ${name}]`);
+    }
     return appendReference(originalBody, `[${label} unavailable: ${name} (${attachment.status})]`);
   }
   if (attachment.kind === "file") {
@@ -5765,6 +5799,96 @@ async function listMessageWindow(client, chatId, previousLastSeen, firstPoll, un
   return messages;
 }
 
+// src/outbound-media.ts
+import { randomUUID as randomUUID2 } from "node:crypto";
+import { readFile } from "node:fs/promises";
+import { basename, extname } from "node:path";
+var EXT_MIME = {
+  png: "image/png",
+  jpg: "image/jpeg",
+  jpeg: "image/jpeg",
+  gif: "image/gif",
+  webp: "image/webp",
+  mp3: "audio/mpeg",
+  wav: "audio/wav",
+  ogg: "audio/ogg",
+  opus: "audio/ogg",
+  m4a: "audio/mp4",
+  flac: "audio/flac",
+  aac: "audio/aac"
+};
+function isTerminal(job) {
+  return ["completed", "failed", "cancelled", "needs_review"].includes(job.status);
+}
+async function waitForVoiceJob(client, jobId) {
+  const deadline = Date.now() + 25 * 6e4;
+  let lastStatusAt = Date.now();
+  while (Date.now() < deadline) {
+    let job;
+    try {
+      job = await client.getVoiceJob(jobId);
+      lastStatusAt = Date.now();
+    } catch (error) {
+      if (Date.now() - lastStatusAt > 3e4) {
+        throw new Error(`Voice job ${jobId} status is unavailable: ${String(error)}. Inspect the job before retrying.`);
+      }
+      await new Promise((resolve) => setTimeout(resolve, 1e3));
+      continue;
+    }
+    if (isTerminal(job)) {
+      if (job.status === "completed") return job;
+      const sent = job.chunks.filter((chunk) => chunk.status === "verified").length;
+      throw new Error(`Voice job ${jobId} ${job.status}: ${sent}/${job.chunks.length} notes verified. ${job.error ?? ""} Do not resend the whole recording.`);
+    }
+    await new Promise((resolve) => setTimeout(resolve, 1e3));
+  }
+  throw new Error(`Voice job ${jobId} exceeded the 25-minute wait. Inspect the job before retrying.`);
+}
+async function sendWeChatMedia(cfg, to, text, mediaUrl, audioAsVoice = false, idempotencyKey) {
+  const account = resolveWeChatAccount(cfg);
+  if (!account?.serverUrl) throw new Error("No serverUrl configured");
+  const client = new WeChatClient({ baseUrl: account.serverUrl, token: account.token });
+  if (!mediaUrl) {
+    if (audioAsVoice) throw new Error("Voice sending requires an audio attachment");
+    const result2 = await client.sendMessage({ chatId: to, text: text || void 0 });
+    if (!result2.success) throw new Error(result2.error ?? "Send failed");
+    return `agent-wechat:${to}:${Date.now()}`;
+  }
+  let bytes;
+  let mimeType;
+  let filename;
+  if (mediaUrl.startsWith("http://") || mediaUrl.startsWith("https://")) {
+    const res = await fetch(mediaUrl);
+    if (!res.ok) throw new Error(`Media fetch failed: ${res.status}`);
+    bytes = new Uint8Array(await res.arrayBuffer());
+    mimeType = res.headers.get("content-type")?.split(";")[0] ?? "application/octet-stream";
+    filename = basename(new URL(mediaUrl).pathname) || "file";
+    if (mimeType === "application/octet-stream") {
+      mimeType = EXT_MIME[extname(filename).toLowerCase().slice(1)] ?? mimeType;
+    }
+  } else {
+    const localPath = mediaUrl.startsWith("file://") ? new URL(mediaUrl) : mediaUrl;
+    const pathText = localPath instanceof URL ? localPath.pathname : localPath;
+    bytes = new Uint8Array(await readFile(localPath));
+    filename = basename(pathText);
+    mimeType = EXT_MIME[extname(filename).toLowerCase().slice(1)] ?? "application/octet-stream";
+  }
+  if (audioAsVoice) {
+    if (!mimeType.startsWith("audio/")) throw new Error(`Voice media must be audio, got ${mimeType}`);
+    const job = await client.createVoiceJob(to, bytes, idempotencyKey ?? randomUUID2());
+    await waitForVoiceJob(client, job.jobId);
+    if (text) {
+      const caption = await client.sendMessage({ chatId: to, text });
+      if (!caption.success) throw new Error(`Voice job ${job.jobId} completed, but caption failed: ${caption.error ?? "unknown error"}`);
+    }
+    return `agent-wechat:voice:${job.jobId}`;
+  }
+  const base64 = Buffer.from(bytes).toString("base64");
+  const result = mimeType.startsWith("image/") ? await client.sendMessage({ chatId: to, text: text || void 0, image: { data: base64, mimeType } }) : await client.sendMessage({ chatId: to, text: text || void 0, file: { data: base64, filename } });
+  if (!result.success) throw new Error(result.error ?? "Send media failed");
+  return `agent-wechat:${to}:${Date.now()}`;
+}
+
 // src/access-control.ts
 import {
   buildChannelKeyCandidates,
@@ -5975,6 +6099,7 @@ function resolveWeChatMentionGate(params) {
 // src/monitor.ts
 var HISTORY_CONTEXT_MARKER = "[Chat messages since your last reply - for context]";
 var CURRENT_MESSAGE_MARKER = "[Current message - respond to this]";
+var voiceTranscriptCache = /* @__PURE__ */ new Map();
 function isProcessedMessage(value) {
   if (!value || typeof value !== "object") return false;
   const message = value;
@@ -6024,17 +6149,18 @@ async function retrieveAttachment(client, chatId, msg, liveAccount, log, maxAtte
   const baseType = msg.type & 2147483647;
   const expectedKind = attachmentKindForMessageType(baseType);
   if (!expectedKind) return void 0;
+  const fallbackFilename = attachmentFallbackFilename(expectedKind, msg.localId, msg.content);
   try {
     const result = await pollMedia(client, chatId, msg.localId, log, maxAttempts);
     if (!result) {
-      return { kind: expectedKind, status: "unavailable", filename: msg.content || `message-${msg.localId}` };
+      return { kind: expectedKind, status: "unavailable", filename: fallbackFilename };
     }
     if (result.type === "unsupported") {
       if (baseType === 49) return void 0;
-      return { kind: expectedKind, status: "unsupported", filename: msg.content || `message-${msg.localId}` };
+      return { kind: expectedKind, status: "unsupported", filename: fallbackFilename };
     }
     const kind = attachmentKindForResult(result, expectedKind);
-    const filename = result.filename || msg.content || `message-${msg.localId}`;
+    const filename = result.filename || fallbackFilename;
     if (!result.data) {
       return { kind, status: result.type === "pending" ? "pending" : "unavailable", filename };
     }
@@ -6058,7 +6184,7 @@ async function retrieveAttachment(client, chatId, msg, liveAccount, log, maxAtte
     return {
       kind: expectedKind,
       status,
-      filename: msg.content || `message-${msg.localId}`,
+      filename: fallbackFilename,
       error: error instanceof Error ? error.message : String(error)
     };
   }
@@ -6228,13 +6354,14 @@ async function prepareMessage(client, msg, chatId, chat, liveAccount, policy, lo
     return null;
   }
   const baseType = msg.type & 2147483647;
+  const messageAttachmentKind = attachmentKindForMessageType(baseType);
   let attachment;
-  if (attachmentKindForMessageType(baseType)) {
+  if (messageAttachmentKind) {
     log?.info?.(`[wechat:${liveAccount.accountId}] Checking media for msg ${msg.localId} (type ${baseType})`);
     attachment = await retrieveAttachment(client, chatId, msg, liveAccount, log);
   }
   const timestamp = new Date(msg.timestamp).getTime();
-  let sourceBody = msg.content || "";
+  let sourceBody = attachmentSourceBody(msg.content || "", messageAttachmentKind);
   if (msg.reply) {
     const replySender = msg.reply.sender ?? "unknown sender";
     const quotedBody = msg.reply.content.length > 50 ? msg.reply.content.slice(0, 50) + "..." : msg.reply.content;
@@ -6344,6 +6471,36 @@ async function dispatchSegment(segment, client, chatId, chat, liveAccount, polic
     return "ignored";
   }
   try {
+    for (const item of segment) {
+      const attachment = item.attachment;
+      if (item.mediaPath || attachment?.kind !== "audio" || attachment.status !== "ready" || !attachment.path) continue;
+      const cacheKey = `${liveAccount.accountId}:${chatId}:${item.msg.localId}:${attachment.path}`;
+      let transcript = voiceTranscriptCache.get(cacheKey);
+      if (!transcript) {
+        try {
+          const result = await core.mediaUnderstanding.transcribeAudioFile({
+            filePath: attachment.path,
+            cfg,
+            mime: attachment.mime
+          });
+          transcript = result.text?.trim();
+          if (transcript) {
+            if (voiceTranscriptCache.size >= 500) voiceTranscriptCache.clear();
+            voiceTranscriptCache.set(cacheKey, transcript);
+          }
+        } catch (error) {
+          log?.error?.(`[wechat:${liveAccount.accountId}] Voice transcription failed for ${item.msg.localId}: ${String(error)}`);
+        }
+      }
+      if (transcript) {
+        item.rawBody += `
+[Voice transcript: ${transcript.slice(0, 1e4)}]`;
+        item.commandBody = normalizeWeChatCommandBody(item.rawBody, {
+          isGroup: item.isGroup,
+          wasMentioned: item.isMentioned
+        });
+      }
+    }
     const route = core.channel.routing.resolveAgentRoute({
       cfg,
       channel: "agent-wechat",
@@ -6372,7 +6529,7 @@ async function dispatchSegment(segment, client, chatId, chat, liveAccount, polic
         timestamp,
         previousTimestamp,
         envelope: envelopeOptions,
-        body: isGroup ? `${senderName}: ${rawBody}` : rawBody
+        body: isGroup ? `${senderName}: ${lastMsg.rawBody}` : lastMsg.rawBody
       });
     } else {
       const historyMessages = segment.slice(0, -1);
@@ -6392,7 +6549,7 @@ async function dispatchSegment(segment, client, chatId, chat, liveAccount, polic
         timestamp,
         previousTimestamp,
         envelope: envelopeOptions,
-        body: isGroup ? `${senderName}: ${rawBody}` : rawBody
+        body: isGroup ? `${senderName}: ${lastMsg.rawBody}` : lastMsg.rawBody
       });
       body = [
         HISTORY_CONTEXT_MARKER,
@@ -6414,8 +6571,8 @@ async function dispatchSegment(segment, client, chatId, chat, liveAccount, polic
     }
     const ctxPayload = core.channel.reply.finalizeInboundContext({
       Body: body,
-      BodyForAgent: rawBody,
-      RawBody: rawBody,
+      BodyForAgent: lastMsg.rawBody,
+      RawBody: lastMsg.rawBody,
       CommandBody: commandBody,
       InboundHistory: inboundHistory,
       From: isGroup ? `wechat:group:${chatId}` : `wechat:${senderId}`,
@@ -6457,12 +6614,14 @@ async function dispatchSegment(segment, client, chatId, chat, liveAccount, polic
       accountId: liveAccount.accountId
     });
     let replyFailed = false;
+    let deliverySequence = 0;
     await core.channel.reply.dispatchReplyWithBufferedBlockDispatcher({
       ctx: ctxPayload,
       cfg,
       dispatcherOptions: {
         ...prefixOptions,
         deliver: async (payload) => {
+          const deliveryIndex = deliverySequence++;
           const mediaList = payload.mediaUrls?.length ? payload.mediaUrls : payload.mediaUrl ? [payload.mediaUrl] : [];
           const tableMode = core.channel.text.resolveMarkdownTableMode({
             cfg,
@@ -6474,49 +6633,17 @@ async function dispatchSegment(segment, client, chatId, chat, liveAccount, polic
             tableMode
           );
           if (mediaList.length > 0) {
-            for (const mediaUrl of mediaList) {
-              try {
-                const fsmod = await import("fs/promises");
-                const pathmod = await import("path");
-                let base64;
-                let mimeType;
-                let filename;
-                if (mediaUrl.startsWith("http://") || mediaUrl.startsWith("https://")) {
-                  const res = await fetch(mediaUrl);
-                  const buffer = await res.arrayBuffer();
-                  base64 = Buffer.from(buffer).toString("base64");
-                  mimeType = res.headers.get("content-type") ?? "application/octet-stream";
-                  const urlPath = new URL(mediaUrl).pathname;
-                  filename = pathmod.basename(urlPath) || "file";
-                } else {
-                  const buf = await fsmod.readFile(mediaUrl);
-                  base64 = buf.toString("base64");
-                  filename = pathmod.basename(mediaUrl);
-                  const ext = pathmod.extname(mediaUrl).toLowerCase().replace(".", "");
-                  const extMime = {
-                    png: "image/png",
-                    jpg: "image/jpeg",
-                    jpeg: "image/jpeg",
-                    gif: "image/gif",
-                    webp: "image/webp"
-                  };
-                  mimeType = extMime[ext] ?? "application/octet-stream";
-                }
-                const isImage = mimeType.startsWith("image/");
-                if (isImage) {
-                  await client.sendMessage({ chatId, image: { data: base64, mimeType } });
-                } else {
-                  await client.sendMessage({ chatId, file: { data: base64, filename } });
-                }
-              } catch (err) {
-                log?.error?.(`[wechat:${liveAccount.accountId}] Failed to send media: ${err}`);
-              }
+            for (let index = 0; index < mediaList.length; index++) {
+              const retryKey = createHash3("sha256").update(`${liveAccount.accountId}\0${chatId}\0${msg.localId}\0${deliveryIndex}\0${index}`).digest("hex");
+              await sendWeChatMedia(cfg, chatId, "", mediaList[index], payload.audioAsVoice === true, retryKey);
             }
             if (text) {
-              await client.sendMessage({ chatId, text });
+              const result = await client.sendMessage({ chatId, text });
+              if (!result.success) throw new Error(result.error ?? "Text send failed");
             }
           } else if (text) {
-            await client.sendMessage({ chatId, text });
+            const result = await client.sendMessage({ chatId, text });
+            if (!result.success) throw new Error(result.error ?? "Text send failed");
           }
         },
         onError: (err, info) => {
@@ -7093,46 +7220,6 @@ async function sendWeChatText(cfg, to, text) {
   if (!result.success) throw new Error(result.error ?? "Send failed");
   return `agent-wechat:${to}:${Date.now()}`;
 }
-async function sendWeChatMedia(cfg, to, text, mediaUrl) {
-  const account = resolveWeChatAccount(cfg);
-  if (!account?.serverUrl) throw new Error("No serverUrl configured");
-  const client = new WeChatClient({ baseUrl: account.serverUrl, token: account.token });
-  if (!mediaUrl) {
-    const result2 = await client.sendMessage({ chatId: to, text: text || void 0 });
-    if (!result2.success) throw new Error(result2.error ?? "Send failed");
-    return `agent-wechat:${to}:${Date.now()}`;
-  }
-  const fsmod = await import("fs/promises");
-  const pathmod = await import("path");
-  let base64;
-  let mimeType;
-  let filename;
-  if (mediaUrl.startsWith("http://") || mediaUrl.startsWith("https://")) {
-    const res = await fetch(mediaUrl);
-    const buffer = await res.arrayBuffer();
-    base64 = Buffer.from(buffer).toString("base64");
-    mimeType = res.headers.get("content-type") ?? "application/octet-stream";
-    const urlPath = new URL(mediaUrl).pathname;
-    filename = pathmod.basename(urlPath) || "file";
-  } else {
-    const buf = await fsmod.readFile(mediaUrl);
-    base64 = buf.toString("base64");
-    filename = pathmod.basename(mediaUrl);
-    const ext = pathmod.extname(mediaUrl).toLowerCase().replace(".", "");
-    const extMime = {
-      png: "image/png",
-      jpg: "image/jpeg",
-      jpeg: "image/jpeg",
-      gif: "image/gif",
-      webp: "image/webp"
-    };
-    mimeType = extMime[ext] ?? "application/octet-stream";
-  }
-  const isImage = mimeType.startsWith("image/");
-  const result = isImage ? await client.sendMessage({ chatId: to, text: text || void 0, image: { data: base64, mimeType } }) : await client.sendMessage({ chatId: to, text: text || void 0, file: { data: base64, filename } });
-  if (!result.success) throw new Error(result.error ?? "Send media failed");
-  return `agent-wechat:${to}:${Date.now()}`;
-}
 var wechatMessageAdapter = createChannelMessageAdapterFromOutbound({
   id: "agent-wechat",
   capabilities: {
@@ -7145,9 +7232,9 @@ var wechatMessageAdapter = createChannelMessageAdapterFromOutbound({
       channel: "agent-wechat",
       messageId: await sendWeChatText(cfg, to, text)
     }),
-    sendMedia: async ({ cfg, to, text, mediaUrl }) => ({
+    sendMedia: async ({ cfg, to, text, mediaUrl, audioAsVoice }) => ({
       channel: "agent-wechat",
-      messageId: await sendWeChatMedia(cfg, to, text, mediaUrl)
+      messageId: await sendWeChatMedia(cfg, to, text, mediaUrl, audioAsVoice)
     })
   }
 });
@@ -7168,7 +7255,8 @@ var wechatPlugin = {
     reactions: false,
     threads: false,
     media: true,
-    reply: true
+    reply: true,
+    tts: { voice: { synthesisTarget: "voice-note", transcodesAudio: true } }
   },
   reload: { configPrefixes: ["channels.agent-wechat"] },
   configSchema: {
@@ -7295,9 +7383,9 @@ var wechatPlugin = {
       channel: "agent-wechat",
       messageId: await sendWeChatText(cfg, to, text)
     }),
-    sendMedia: async ({ cfg, to, text, mediaUrl }) => ({
+    sendMedia: async ({ cfg, to, text, mediaUrl, audioAsVoice }) => ({
       channel: "agent-wechat",
-      messageId: await sendWeChatMedia(cfg, to, text, mediaUrl)
+      messageId: await sendWeChatMedia(cfg, to, text, mediaUrl, audioAsVoice)
     })
   },
   message: wechatMessageAdapter,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@agent-wechat/agent-wechat",
-  "version": "0.13.0",
+  "version": "0.15.0",
   "type": "module",
   "files": [
     "dist",
@@ -11,7 +11,7 @@
     "esbuild": "^0.25.0",
     "openclaw": "^2026.8.1",
     "typescript": "^5.4.5",
-    "@agent-wechat/shared": "0.2.0"
+    "@agent-wechat/shared": "0.3.0"
   },
   "openclaw": {
     "extensions": [