@agent-wechat/agent-wechat 0.14.0 → 0.15.0

package/dist/index.js

@@ -1127,6 +1127,9 @@ function getWeChatRuntime() {
   return runtime;
 }
 
+// src/monitor.ts
+import { createHash as createHash3 } from "node:crypto";
+
 // ../shared/dist/client.js
 function normalizeUrl(base) {
   const url = base.startsWith("http") ? base : `http://${base}`;
@@ -1230,6 +1233,23 @@ var WeChatClient = class {
   async sendMessage(params) {
     return this.post("/api/messages/send", params);
   }
+  async createVoiceJob(chatId, audio, idempotencyKey) {
+    const form = new FormData();
+    form.set("chatId", chatId);
+    form.set("audio", new Blob([new Uint8Array(audio)]), "voice-audio");
+    const headers = { ...this.headers, "Idempotency-Key": idempotencyKey };
+    delete headers["Content-Type"];
+    const res = await fetch(`${this.base}/api/messages/voice`, { method: "POST", headers, body: form });
+    if (!res.ok)
+      throw new Error(`${res.status} ${res.statusText}: ${await res.text()}`);
+    return res.json();
+  }
+  async getVoiceJob(jobId) {
+    return this.get(`/api/messages/voice/${encodeURIComponent(jobId)}`);
+  }
+  async cancelVoiceJob(jobId) {
+    return this.post(`/api/messages/voice/${encodeURIComponent(jobId)}/cancel`);
+  }
   // ---- Debug ----
   async screenshot() {
     return this.get("/api/debug/screenshot");
@@ -5779,6 +5799,96 @@ async function listMessageWindow(client, chatId, previousLastSeen, firstPoll, un
   return messages;
 }
 
+// src/outbound-media.ts
+import { randomUUID as randomUUID2 } from "node:crypto";
+import { readFile } from "node:fs/promises";
+import { basename, extname } from "node:path";
+var EXT_MIME = {
+  png: "image/png",
+  jpg: "image/jpeg",
+  jpeg: "image/jpeg",
+  gif: "image/gif",
+  webp: "image/webp",
+  mp3: "audio/mpeg",
+  wav: "audio/wav",
+  ogg: "audio/ogg",
+  opus: "audio/ogg",
+  m4a: "audio/mp4",
+  flac: "audio/flac",
+  aac: "audio/aac"
+};
+function isTerminal(job) {
+  return ["completed", "failed", "cancelled", "needs_review"].includes(job.status);
+}
+async function waitForVoiceJob(client, jobId) {
+  const deadline = Date.now() + 25 * 6e4;
+  let lastStatusAt = Date.now();
+  while (Date.now() < deadline) {
+    let job;
+    try {
+      job = await client.getVoiceJob(jobId);
+      lastStatusAt = Date.now();
+    } catch (error) {
+      if (Date.now() - lastStatusAt > 3e4) {
+        throw new Error(`Voice job ${jobId} status is unavailable: ${String(error)}. Inspect the job before retrying.`);
+      }
+      await new Promise((resolve) => setTimeout(resolve, 1e3));
+      continue;
+    }
+    if (isTerminal(job)) {
+      if (job.status === "completed") return job;
+      const sent = job.chunks.filter((chunk) => chunk.status === "verified").length;
+      throw new Error(`Voice job ${jobId} ${job.status}: ${sent}/${job.chunks.length} notes verified. ${job.error ?? ""} Do not resend the whole recording.`);
+    }
+    await new Promise((resolve) => setTimeout(resolve, 1e3));
+  }
+  throw new Error(`Voice job ${jobId} exceeded the 25-minute wait. Inspect the job before retrying.`);
+}
+async function sendWeChatMedia(cfg, to, text, mediaUrl, audioAsVoice = false, idempotencyKey) {
+  const account = resolveWeChatAccount(cfg);
+  if (!account?.serverUrl) throw new Error("No serverUrl configured");
+  const client = new WeChatClient({ baseUrl: account.serverUrl, token: account.token });
+  if (!mediaUrl) {
+    if (audioAsVoice) throw new Error("Voice sending requires an audio attachment");
+    const result2 = await client.sendMessage({ chatId: to, text: text || void 0 });
+    if (!result2.success) throw new Error(result2.error ?? "Send failed");
+    return `agent-wechat:${to}:${Date.now()}`;
+  }
+  let bytes;
+  let mimeType;
+  let filename;
+  if (mediaUrl.startsWith("http://") || mediaUrl.startsWith("https://")) {
+    const res = await fetch(mediaUrl);
+    if (!res.ok) throw new Error(`Media fetch failed: ${res.status}`);
+    bytes = new Uint8Array(await res.arrayBuffer());
+    mimeType = res.headers.get("content-type")?.split(";")[0] ?? "application/octet-stream";
+    filename = basename(new URL(mediaUrl).pathname) || "file";
+    if (mimeType === "application/octet-stream") {
+      mimeType = EXT_MIME[extname(filename).toLowerCase().slice(1)] ?? mimeType;
+    }
+  } else {
+    const localPath = mediaUrl.startsWith("file://") ? new URL(mediaUrl) : mediaUrl;
+    const pathText = localPath instanceof URL ? localPath.pathname : localPath;
+    bytes = new Uint8Array(await readFile(localPath));
+    filename = basename(pathText);
+    mimeType = EXT_MIME[extname(filename).toLowerCase().slice(1)] ?? "application/octet-stream";
+  }
+  if (audioAsVoice) {
+    if (!mimeType.startsWith("audio/")) throw new Error(`Voice media must be audio, got ${mimeType}`);
+    const job = await client.createVoiceJob(to, bytes, idempotencyKey ?? randomUUID2());
+    await waitForVoiceJob(client, job.jobId);
+    if (text) {
+      const caption = await client.sendMessage({ chatId: to, text });
+      if (!caption.success) throw new Error(`Voice job ${job.jobId} completed, but caption failed: ${caption.error ?? "unknown error"}`);
+    }
+    return `agent-wechat:voice:${job.jobId}`;
+  }
+  const base64 = Buffer.from(bytes).toString("base64");
+  const result = mimeType.startsWith("image/") ? await client.sendMessage({ chatId: to, text: text || void 0, image: { data: base64, mimeType } }) : await client.sendMessage({ chatId: to, text: text || void 0, file: { data: base64, filename } });
+  if (!result.success) throw new Error(result.error ?? "Send media failed");
+  return `agent-wechat:${to}:${Date.now()}`;
+}
+
 // src/access-control.ts
 import {
   buildChannelKeyCandidates,
@@ -5989,6 +6099,7 @@ function resolveWeChatMentionGate(params) {
 // src/monitor.ts
 var HISTORY_CONTEXT_MARKER = "[Chat messages since your last reply - for context]";
 var CURRENT_MESSAGE_MARKER = "[Current message - respond to this]";
+var voiceTranscriptCache = /* @__PURE__ */ new Map();
 function isProcessedMessage(value) {
   if (!value || typeof value !== "object") return false;
   const message = value;
@@ -6360,6 +6471,36 @@ async function dispatchSegment(segment, client, chatId, chat, liveAccount, polic
     return "ignored";
   }
   try {
+    for (const item of segment) {
+      const attachment = item.attachment;
+      if (item.mediaPath || attachment?.kind !== "audio" || attachment.status !== "ready" || !attachment.path) continue;
+      const cacheKey = `${liveAccount.accountId}:${chatId}:${item.msg.localId}:${attachment.path}`;
+      let transcript = voiceTranscriptCache.get(cacheKey);
+      if (!transcript) {
+        try {
+          const result = await core.mediaUnderstanding.transcribeAudioFile({
+            filePath: attachment.path,
+            cfg,
+            mime: attachment.mime
+          });
+          transcript = result.text?.trim();
+          if (transcript) {
+            if (voiceTranscriptCache.size >= 500) voiceTranscriptCache.clear();
+            voiceTranscriptCache.set(cacheKey, transcript);
+          }
+        } catch (error) {
+          log?.error?.(`[wechat:${liveAccount.accountId}] Voice transcription failed for ${item.msg.localId}: ${String(error)}`);
+        }
+      }
+      if (transcript) {
+        item.rawBody += `
+[Voice transcript: ${transcript.slice(0, 1e4)}]`;
+        item.commandBody = normalizeWeChatCommandBody(item.rawBody, {
+          isGroup: item.isGroup,
+          wasMentioned: item.isMentioned
+        });
+      }
+    }
     const route = core.channel.routing.resolveAgentRoute({
       cfg,
       channel: "agent-wechat",
@@ -6388,7 +6529,7 @@ async function dispatchSegment(segment, client, chatId, chat, liveAccount, polic
         timestamp,
         previousTimestamp,
         envelope: envelopeOptions,
-        body: isGroup ? `${senderName}: ${rawBody}` : rawBody
+        body: isGroup ? `${senderName}: ${lastMsg.rawBody}` : lastMsg.rawBody
       });
     } else {
       const historyMessages = segment.slice(0, -1);
@@ -6408,7 +6549,7 @@ async function dispatchSegment(segment, client, chatId, chat, liveAccount, polic
         timestamp,
         previousTimestamp,
         envelope: envelopeOptions,
-        body: isGroup ? `${senderName}: ${rawBody}` : rawBody
+        body: isGroup ? `${senderName}: ${lastMsg.rawBody}` : lastMsg.rawBody
       });
       body = [
         HISTORY_CONTEXT_MARKER,
@@ -6430,8 +6571,8 @@ async function dispatchSegment(segment, client, chatId, chat, liveAccount, polic
     }
     const ctxPayload = core.channel.reply.finalizeInboundContext({
       Body: body,
-      BodyForAgent: rawBody,
-      RawBody: rawBody,
+      BodyForAgent: lastMsg.rawBody,
+      RawBody: lastMsg.rawBody,
       CommandBody: commandBody,
       InboundHistory: inboundHistory,
       From: isGroup ? `wechat:group:${chatId}` : `wechat:${senderId}`,
@@ -6473,12 +6614,14 @@ async function dispatchSegment(segment, client, chatId, chat, liveAccount, polic
       accountId: liveAccount.accountId
     });
     let replyFailed = false;
+    let deliverySequence = 0;
     await core.channel.reply.dispatchReplyWithBufferedBlockDispatcher({
       ctx: ctxPayload,
       cfg,
       dispatcherOptions: {
         ...prefixOptions,
         deliver: async (payload) => {
+          const deliveryIndex = deliverySequence++;
           const mediaList = payload.mediaUrls?.length ? payload.mediaUrls : payload.mediaUrl ? [payload.mediaUrl] : [];
           const tableMode = core.channel.text.resolveMarkdownTableMode({
             cfg,
@@ -6490,49 +6633,17 @@ async function dispatchSegment(segment, client, chatId, chat, liveAccount, polic
             tableMode
           );
           if (mediaList.length > 0) {
-            for (const mediaUrl of mediaList) {
-              try {
-                const fsmod = await import("fs/promises");
-                const pathmod = await import("path");
-                let base64;
-                let mimeType;
-                let filename;
-                if (mediaUrl.startsWith("http://") || mediaUrl.startsWith("https://")) {
-                  const res = await fetch(mediaUrl);
-                  const buffer = await res.arrayBuffer();
-                  base64 = Buffer.from(buffer).toString("base64");
-                  mimeType = res.headers.get("content-type") ?? "application/octet-stream";
-                  const urlPath = new URL(mediaUrl).pathname;
-                  filename = pathmod.basename(urlPath) || "file";
-                } else {
-                  const buf = await fsmod.readFile(mediaUrl);
-                  base64 = buf.toString("base64");
-                  filename = pathmod.basename(mediaUrl);
-                  const ext = pathmod.extname(mediaUrl).toLowerCase().replace(".", "");
-                  const extMime = {
-                    png: "image/png",
-                    jpg: "image/jpeg",
-                    jpeg: "image/jpeg",
-                    gif: "image/gif",
-                    webp: "image/webp"
-                  };
-                  mimeType = extMime[ext] ?? "application/octet-stream";
-                }
-                const isImage = mimeType.startsWith("image/");
-                if (isImage) {
-                  await client.sendMessage({ chatId, image: { data: base64, mimeType } });
-                } else {
-                  await client.sendMessage({ chatId, file: { data: base64, filename } });
-                }
-              } catch (err) {
-                log?.error?.(`[wechat:${liveAccount.accountId}] Failed to send media: ${err}`);
-              }
+            for (let index = 0; index < mediaList.length; index++) {
+              const retryKey = createHash3("sha256").update(`${liveAccount.accountId}\0${chatId}\0${msg.localId}\0${deliveryIndex}\0${index}`).digest("hex");
+              await sendWeChatMedia(cfg, chatId, "", mediaList[index], payload.audioAsVoice === true, retryKey);
             }
             if (text) {
-              await client.sendMessage({ chatId, text });
+              const result = await client.sendMessage({ chatId, text });
+              if (!result.success) throw new Error(result.error ?? "Text send failed");
             }
           } else if (text) {
-            await client.sendMessage({ chatId, text });
+            const result = await client.sendMessage({ chatId, text });
+            if (!result.success) throw new Error(result.error ?? "Text send failed");
           }
         },
         onError: (err, info) => {
@@ -7109,46 +7220,6 @@ async function sendWeChatText(cfg, to, text) {
   if (!result.success) throw new Error(result.error ?? "Send failed");
   return `agent-wechat:${to}:${Date.now()}`;
 }
-async function sendWeChatMedia(cfg, to, text, mediaUrl) {
-  const account = resolveWeChatAccount(cfg);
-  if (!account?.serverUrl) throw new Error("No serverUrl configured");
-  const client = new WeChatClient({ baseUrl: account.serverUrl, token: account.token });
-  if (!mediaUrl) {
-    const result2 = await client.sendMessage({ chatId: to, text: text || void 0 });
-    if (!result2.success) throw new Error(result2.error ?? "Send failed");
-    return `agent-wechat:${to}:${Date.now()}`;
-  }
-  const fsmod = await import("fs/promises");
-  const pathmod = await import("path");
-  let base64;
-  let mimeType;
-  let filename;
-  if (mediaUrl.startsWith("http://") || mediaUrl.startsWith("https://")) {
-    const res = await fetch(mediaUrl);
-    const buffer = await res.arrayBuffer();
-    base64 = Buffer.from(buffer).toString("base64");
-    mimeType = res.headers.get("content-type") ?? "application/octet-stream";
-    const urlPath = new URL(mediaUrl).pathname;
-    filename = pathmod.basename(urlPath) || "file";
-  } else {
-    const buf = await fsmod.readFile(mediaUrl);
-    base64 = buf.toString("base64");
-    filename = pathmod.basename(mediaUrl);
-    const ext = pathmod.extname(mediaUrl).toLowerCase().replace(".", "");
-    const extMime = {
-      png: "image/png",
-      jpg: "image/jpeg",
-      jpeg: "image/jpeg",
-      gif: "image/gif",
-      webp: "image/webp"
-    };
-    mimeType = extMime[ext] ?? "application/octet-stream";
-  }
-  const isImage = mimeType.startsWith("image/");
-  const result = isImage ? await client.sendMessage({ chatId: to, text: text || void 0, image: { data: base64, mimeType } }) : await client.sendMessage({ chatId: to, text: text || void 0, file: { data: base64, filename } });
-  if (!result.success) throw new Error(result.error ?? "Send media failed");
-  return `agent-wechat:${to}:${Date.now()}`;
-}
 var wechatMessageAdapter = createChannelMessageAdapterFromOutbound({
   id: "agent-wechat",
   capabilities: {
@@ -7161,9 +7232,9 @@ var wechatMessageAdapter = createChannelMessageAdapterFromOutbound({
       channel: "agent-wechat",
       messageId: await sendWeChatText(cfg, to, text)
     }),
-    sendMedia: async ({ cfg, to, text, mediaUrl }) => ({
+    sendMedia: async ({ cfg, to, text, mediaUrl, audioAsVoice }) => ({
       channel: "agent-wechat",
-      messageId: await sendWeChatMedia(cfg, to, text, mediaUrl)
+      messageId: await sendWeChatMedia(cfg, to, text, mediaUrl, audioAsVoice)
     })
   }
 });
@@ -7184,7 +7255,8 @@ var wechatPlugin = {
     reactions: false,
     threads: false,
     media: true,
-    reply: true
+    reply: true,
+    tts: { voice: { synthesisTarget: "voice-note", transcodesAudio: true } }
   },
   reload: { configPrefixes: ["channels.agent-wechat"] },
   configSchema: {
@@ -7311,9 +7383,9 @@ var wechatPlugin = {
       channel: "agent-wechat",
       messageId: await sendWeChatText(cfg, to, text)
     }),
-    sendMedia: async ({ cfg, to, text, mediaUrl }) => ({
+    sendMedia: async ({ cfg, to, text, mediaUrl, audioAsVoice }) => ({
       channel: "agent-wechat",
-      messageId: await sendWeChatMedia(cfg, to, text, mediaUrl)
+      messageId: await sendWeChatMedia(cfg, to, text, mediaUrl, audioAsVoice)
     })
   },
   message: wechatMessageAdapter,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@agent-wechat/agent-wechat",
-  "version": "0.14.0",
+  "version": "0.15.0",
   "type": "module",
   "files": [
     "dist",
@@ -11,7 +11,7 @@
     "esbuild": "^0.25.0",
     "openclaw": "^2026.8.1",
     "typescript": "^5.4.5",
-    "@agent-wechat/shared": "0.2.0"
+    "@agent-wechat/shared": "0.3.0"
   },
   "openclaw": {
     "extensions": [