@openclaw/whatsapp 2026.5.1-beta.1

package/src/auto-reply/monitor/on-message.audio-preflight.test.ts

@@ -0,0 +1,354 @@
+import { beforeEach, describe, expect, it, vi } from "vitest";
+
+const events: string[] = [];
+const transcribeFirstAudioMock = vi.fn();
+const maybeSendAckReactionMock = vi.fn();
+const processMessageMock = vi.fn();
+const maybeBroadcastMessageMock = vi.fn();
+const ackReactionHandle = {
+  ackReactionPromise: Promise.resolve(true),
+  ackReactionValue: "👀",
+  remove: vi.fn(async () => undefined),
+};
+const applyGroupGatingMock = vi.fn();
+
+vi.mock("./audio-preflight.runtime.js", () => ({
+  transcribeFirstAudio: (...args: unknown[]) => transcribeFirstAudioMock(...args),
+}));
+
+vi.mock("./ack-reaction.js", () => ({
+  maybeSendAckReaction: (...args: unknown[]) => maybeSendAckReactionMock(...args),
+}));
+
+vi.mock("./process-message.js", () => ({
+  processMessage: (...args: unknown[]) => processMessageMock(...args),
+}));
+
+vi.mock("./broadcast.js", () => ({
+  maybeBroadcastMessage: (...args: unknown[]) => maybeBroadcastMessageMock(...args),
+}));
+
+vi.mock("./group-gating.js", () => ({
+  applyGroupGating: (...args: unknown[]) => applyGroupGatingMock(...args),
+}));
+
+vi.mock("./last-route.js", () => ({
+  updateLastRouteInBackground: () => {},
+}));
+
+vi.mock("./peer.js", () => ({
+  resolvePeerId: (msg: { from: string }) => msg.from,
+}));
+
+vi.mock("../config.runtime.js", () => ({
+  getRuntimeConfig: () => ({
+    channels: {
+      whatsapp: {
+        ackReaction: { enabled: true },
+      },
+    },
+  }),
+}));
+
+vi.mock("../../group-session-key.js", () => ({
+  resolveWhatsAppGroupSessionRoute: (route: unknown) => route,
+}));
+
+vi.mock("../../identity.js", () => ({
+  getPrimaryIdentityId: () => undefined,
+  getSenderIdentity: () => ({ e164: "+15550000002", name: "Alice" }),
+}));
+
+vi.mock("../../text-runtime.js", () => ({
+  normalizeE164: (value: string) => value,
+}));
+
+vi.mock("openclaw/plugin-sdk/routing", () => ({
+  buildGroupHistoryKey: () => "group-key",
+  resolveAgentRoute: () => ({
+    agentId: "main",
+    accountId: "default",
+    sessionKey: "agent:main:whatsapp:+15550000002",
+    mainSessionKey: "agent:main:main",
+  }),
+}));
+
+import type { WebInboundMsg } from "../types.js";
+import { createWebOnMessageHandler } from "./on-message.js";
+
+function makeAudioMsg(): WebInboundMsg {
+  return {
+    id: "msg-1",
+    from: "+15550000002",
+    to: "+15550000001",
+    accessControlPassed: true,
+    body: "<media:audio>",
+    chatType: "direct",
+    mediaType: "audio/ogg; codecs=opus",
+    mediaPath: "/tmp/voice.ogg",
+    timestamp: 1700000000,
+    accountId: "default",
+  } as WebInboundMsg;
+}
+
+function makeGroupAudioMsg(): WebInboundMsg {
+  return {
+    ...makeAudioMsg(),
+    from: "1203630@g.us",
+    chatId: "1203630@g.us",
+    chatType: "group",
+    conversationId: "1203630@g.us",
+    wasMentioned: false,
+  } as WebInboundMsg;
+}
+
+function makeEchoTracker() {
+  return {
+    has: () => false,
+    forget: () => {},
+    rememberText: () => {},
+    buildCombinedKey: (p: { combinedBody: string }) => p.combinedBody,
+  };
+}
+
+describe("createWebOnMessageHandler audio preflight", () => {
+  beforeEach(() => {
+    events.length = 0;
+    maybeBroadcastMessageMock.mockReset();
+    maybeBroadcastMessageMock.mockImplementation(async () => false);
+    maybeSendAckReactionMock.mockReset();
+    maybeSendAckReactionMock.mockImplementation(async () => {
+      events.push("ack");
+      return ackReactionHandle;
+    });
+    transcribeFirstAudioMock.mockReset();
+    transcribeFirstAudioMock.mockImplementation(async () => {
+      events.push("stt");
+      return "transcribed voice note";
+    });
+    processMessageMock.mockReset();
+    processMessageMock.mockResolvedValue(true);
+    applyGroupGatingMock.mockReset();
+    applyGroupGatingMock.mockResolvedValue({ shouldProcess: true });
+  });
+
+  it("sends ack reaction before audio preflight for voice notes", async () => {
+    const handler = createWebOnMessageHandler({
+      cfg: {
+        channels: {
+          whatsapp: {
+            ackReaction: { enabled: true },
+          },
+        },
+      } as never,
+      verbose: false,
+      connectionId: "conn-1",
+      maxMediaBytes: 1024 * 1024,
+      groupHistoryLimit: 20,
+      groupHistories: new Map(),
+      groupMemberNames: new Map(),
+      echoTracker: makeEchoTracker() as never,
+      backgroundTasks: new Set(),
+      replyResolver: vi.fn() as never,
+      replyLogger: {
+        info: () => {},
+        warn: () => {},
+        debug: () => {},
+        error: () => {},
+      } as never,
+      baseMentionConfig: {} as never,
+      account: { authDir: "/tmp/auth", accountId: "default" },
+    });
+
+    await handler(makeAudioMsg());
+
+    expect(events).toEqual(["ack", "stt"]);
+    expect(processMessageMock).toHaveBeenCalledWith(
+      expect.objectContaining({
+        preflightAudioTranscript: "transcribed voice note",
+        ackAlreadySent: true,
+        ackReaction: ackReactionHandle,
+      }),
+    );
+  });
+
+  it("skips early DM ack/preflight when access-control was not explicitly passed through", async () => {
+    const handler = createWebOnMessageHandler({
+      cfg: {
+        channels: {
+          whatsapp: {
+            ackReaction: { enabled: true },
+          },
+        },
+      } as never,
+      verbose: false,
+      connectionId: "conn-1",
+      maxMediaBytes: 1024 * 1024,
+      groupHistoryLimit: 20,
+      groupHistories: new Map(),
+      groupMemberNames: new Map(),
+      echoTracker: makeEchoTracker() as never,
+      backgroundTasks: new Set(),
+      replyResolver: vi.fn() as never,
+      replyLogger: {
+        info: () => {},
+        warn: () => {},
+        debug: () => {},
+        error: () => {},
+      } as never,
+      baseMentionConfig: {} as never,
+      account: { authDir: "/tmp/auth", accountId: "default" },
+    });
+
+    await handler({ ...makeAudioMsg(), accessControlPassed: undefined });
+
+    expect(events).toEqual([]);
+    expect(transcribeFirstAudioMock).not.toHaveBeenCalled();
+    expect(maybeSendAckReactionMock).not.toHaveBeenCalled();
+    expect(processMessageMock).toHaveBeenCalledWith(
+      expect.not.objectContaining({
+        preflightAudioTranscript: expect.anything(),
+        ackAlreadySent: true,
+      }),
+    );
+  });
+
+  it("preserves per-agent ack checks for group broadcast voice notes", async () => {
+    maybeBroadcastMessageMock.mockImplementation(
+      async (params: {
+        ackAlreadySent?: boolean;
+        ackReaction?: unknown;
+        preflightAudioTranscript?: string | null;
+      }) => {
+        expect(params.preflightAudioTranscript).toBe("transcribed voice note");
+        expect(params.ackAlreadySent).toBeUndefined();
+        expect(params.ackReaction).toBeUndefined();
+        return true;
+      },
+    );
+    const handler = createWebOnMessageHandler({
+      cfg: {
+        channels: {
+          whatsapp: {
+            ackReaction: { enabled: true },
+          },
+        },
+        broadcast: {
+          "1203630@g.us": ["main", "backup"],
+        },
+      } as never,
+      verbose: false,
+      connectionId: "conn-1",
+      maxMediaBytes: 1024 * 1024,
+      groupHistoryLimit: 20,
+      groupHistories: new Map(),
+      groupMemberNames: new Map(),
+      echoTracker: makeEchoTracker() as never,
+      backgroundTasks: new Set(),
+      replyResolver: vi.fn() as never,
+      replyLogger: {
+        info: () => {},
+        warn: () => {},
+        debug: () => {},
+        error: () => {},
+      } as never,
+      baseMentionConfig: {} as never,
+      account: { authDir: "/tmp/auth", accountId: "default" },
+    });
+
+    await handler(makeGroupAudioMsg());
+
+    expect(events).toEqual(["ack", "stt"]);
+    expect(processMessageMock).not.toHaveBeenCalled();
+  });
+
+  it("uses group voice transcript for mention gating before dispatch", async () => {
+    applyGroupGatingMock
+      .mockResolvedValueOnce({ shouldProcess: false, needsMentionText: true })
+      .mockResolvedValueOnce({ shouldProcess: true });
+    const handler = createWebOnMessageHandler({
+      cfg: {
+        channels: {
+          whatsapp: {
+            ackReaction: { enabled: true },
+          },
+        },
+      } as never,
+      verbose: false,
+      connectionId: "conn-1",
+      maxMediaBytes: 1024 * 1024,
+      groupHistoryLimit: 20,
+      groupHistories: new Map(),
+      groupMemberNames: new Map(),
+      echoTracker: makeEchoTracker() as never,
+      backgroundTasks: new Set(),
+      replyResolver: vi.fn() as never,
+      replyLogger: {
+        info: () => {},
+        warn: () => {},
+        debug: () => {},
+        error: () => {},
+      } as never,
+      baseMentionConfig: {} as never,
+      account: { authDir: "/tmp/auth", accountId: "default" },
+    });
+
+    await handler(makeGroupAudioMsg());
+
+    expect(applyGroupGatingMock).toHaveBeenNthCalledWith(
+      1,
+      expect.objectContaining({
+        deferMissingMention: true,
+      }),
+    );
+    expect(events).toEqual(["ack", "stt"]);
+    expect(applyGroupGatingMock).toHaveBeenNthCalledWith(
+      2,
+      expect.objectContaining({
+        mentionText: "transcribed voice note",
+      }),
+    );
+    expect(processMessageMock).toHaveBeenCalledWith(
+      expect.objectContaining({
+        preflightAudioTranscript: "transcribed voice note",
+        ackAlreadySent: true,
+      }),
+    );
+  });
+
+  it("does not transcribe group voice when policy gating rejects before mention", async () => {
+    applyGroupGatingMock.mockResolvedValueOnce({ shouldProcess: false });
+    const handler = createWebOnMessageHandler({
+      cfg: {
+        channels: {
+          whatsapp: {
+            ackReaction: { enabled: true },
+          },
+        },
+      } as never,
+      verbose: false,
+      connectionId: "conn-1",
+      maxMediaBytes: 1024 * 1024,
+      groupHistoryLimit: 20,
+      groupHistories: new Map(),
+      groupMemberNames: new Map(),
+      echoTracker: makeEchoTracker() as never,
+      backgroundTasks: new Set(),
+      replyResolver: vi.fn() as never,
+      replyLogger: {
+        info: () => {},
+        warn: () => {},
+        debug: () => {},
+        error: () => {},
+      } as never,
+      baseMentionConfig: {} as never,
+      account: { authDir: "/tmp/auth", accountId: "default" },
+    });
+
+    await handler(makeGroupAudioMsg());
+
+    expect(transcribeFirstAudioMock).not.toHaveBeenCalled();
+    expect(maybeSendAckReactionMock).not.toHaveBeenCalled();
+    expect(processMessageMock).not.toHaveBeenCalled();
+  });
+});

package/src/auto-reply/monitor/on-message.ts

@@ -0,0 +1,291 @@
+import type { AckReactionHandle } from "openclaw/plugin-sdk/channel-feedback";
+import type { getReplyFromConfig } from "openclaw/plugin-sdk/reply-runtime";
+import type { MsgContext } from "openclaw/plugin-sdk/reply-runtime";
+import { resolveAgentRoute } from "openclaw/plugin-sdk/routing";
+import { buildGroupHistoryKey } from "openclaw/plugin-sdk/routing";
+import { logVerbose } from "openclaw/plugin-sdk/runtime-env";
+import { resolveWhatsAppGroupSessionRoute } from "../../group-session-key.js";
+import { getPrimaryIdentityId, getSenderIdentity } from "../../identity.js";
+import { normalizeE164 } from "../../text-runtime.js";
+import { getRuntimeConfig } from "../config.runtime.js";
+import type { MentionConfig } from "../mentions.js";
+import type { WebInboundMsg } from "../types.js";
+import { maybeSendAckReaction } from "./ack-reaction.js";
+import { maybeBroadcastMessage } from "./broadcast.js";
+import type { EchoTracker } from "./echo.js";
+import type { GroupHistoryEntry } from "./group-gating.js";
+import { applyGroupGating } from "./group-gating.js";
+import { updateLastRouteInBackground } from "./last-route.js";
+import { resolvePeerId } from "./peer.js";
+import { processMessage } from "./process-message.js";
+
+export function createWebOnMessageHandler(params: {
+  cfg: ReturnType<typeof getRuntimeConfig>;
+  verbose: boolean;
+  connectionId: string;
+  maxMediaBytes: number;
+  groupHistoryLimit: number;
+  groupHistories: Map<string, GroupHistoryEntry[]>;
+  groupMemberNames: Map<string, Map<string, string>>;
+  echoTracker: EchoTracker;
+  backgroundTasks: Set<Promise<unknown>>;
+  replyResolver: typeof getReplyFromConfig;
+  replyLogger: ReturnType<(typeof import("openclaw/plugin-sdk/runtime-env"))["getChildLogger"]>;
+  baseMentionConfig: MentionConfig;
+  account: { authDir?: string; accountId?: string; selfChatMode?: boolean };
+}) {
+  const processForRoute = async (
+    msg: WebInboundMsg,
+    route: ReturnType<typeof resolveAgentRoute>,
+    groupHistoryKey: string,
+    opts?: {
+      groupHistory?: GroupHistoryEntry[];
+      suppressGroupHistoryClear?: boolean;
+      preflightAudioTranscript?: string | null;
+      ackAlreadySent?: boolean;
+      ackReaction?: AckReactionHandle | null;
+    },
+  ) => {
+    const processParams: Parameters<typeof processMessage>[0] = {
+      cfg: params.cfg,
+      msg,
+      route,
+      groupHistoryKey,
+      groupHistories: params.groupHistories,
+      groupMemberNames: params.groupMemberNames,
+      connectionId: params.connectionId,
+      verbose: params.verbose,
+      maxMediaBytes: params.maxMediaBytes,
+      replyResolver: params.replyResolver,
+      replyLogger: params.replyLogger,
+      backgroundTasks: params.backgroundTasks,
+      rememberSentText: params.echoTracker.rememberText,
+      echoHas: params.echoTracker.has,
+      echoForget: params.echoTracker.forget,
+      buildCombinedEchoKey: params.echoTracker.buildCombinedKey,
+    };
+    if (opts?.groupHistory !== undefined) {
+      processParams.groupHistory = opts.groupHistory;
+    }
+    if (opts?.suppressGroupHistoryClear !== undefined) {
+      processParams.suppressGroupHistoryClear = opts.suppressGroupHistoryClear;
+    }
+    if (opts?.preflightAudioTranscript !== undefined) {
+      processParams.preflightAudioTranscript = opts.preflightAudioTranscript;
+    }
+    if (opts?.ackAlreadySent === true) {
+      processParams.ackAlreadySent = true;
+    }
+    if (opts?.ackReaction !== undefined) {
+      processParams.ackReaction = opts.ackReaction;
+    }
+    return processMessage(processParams);
+  };
+
+  return async (msg: WebInboundMsg) => {
+    const conversationId = msg.conversationId ?? msg.from;
+    const peerId = resolvePeerId(msg);
+    // Fresh config for bindings lookup; other routing inputs are payload-derived.
+    const baseRoute = resolveAgentRoute({
+      cfg: getRuntimeConfig(),
+      channel: "whatsapp",
+      accountId: msg.accountId,
+      peer: {
+        kind: msg.chatType === "group" ? "group" : "direct",
+        id: peerId,
+      },
+    });
+    const route =
+      msg.chatType === "group" ? resolveWhatsAppGroupSessionRoute(baseRoute) : baseRoute;
+    const groupHistoryKey =
+      msg.chatType === "group"
+        ? buildGroupHistoryKey({
+            channel: "whatsapp",
+            accountId: route.accountId,
+            peerKind: "group",
+            peerId,
+          })
+        : route.sessionKey;
+
+    // Same-phone mode logging retained
+    if (msg.from === msg.to) {
+      logVerbose(`📱 Same-phone mode detected (from === to: ${msg.from})`);
+    }
+
+    // Skip if this is a message we just sent (echo detection)
+    if (params.echoTracker.has(msg.body)) {
+      logVerbose("Skipping auto-reply: detected echo (message matches recently sent text)");
+      params.echoTracker.forget(msg.body);
+      return;
+    }
+
+    // Preflight audio transcription: run once before broadcast fan-out so all
+    // agents share the same transcript instead of each making a separate STT call.
+    // For DMs, only do this on the real inbound path after access-control/pairing
+    // checks have already passed in inbound/monitor.ts. For groups, the first
+    // gating pass must approve the group/sender before STT is attempted.
+    // null = preflight was attempted but produced no transcript (failed / disabled / no audio);
+    // undefined = preflight was not attempted (non-audio message).
+    let preflightAudioTranscript: string | null | undefined;
+    const hasAudioBody =
+      msg.mediaType?.startsWith("audio/") === true && msg.body === "<media:audio>";
+    const canRunEarlyAudioPreflight = msg.chatType === "group" || msg.accessControlPassed === true;
+    let ackAlreadySent = false;
+    let ackReaction: AckReactionHandle | null = null;
+    const runAudioPreflightOnce = async () => {
+      if (
+        preflightAudioTranscript !== undefined ||
+        !canRunEarlyAudioPreflight ||
+        !hasAudioBody ||
+        !msg.mediaPath
+      ) {
+        return;
+      }
+      ackReaction = await maybeSendAckReaction({
+        cfg: params.cfg,
+        msg,
+        agentId: route.agentId,
+        sessionKey: route.sessionKey,
+        conversationId,
+        verbose: params.verbose,
+        accountId: route.accountId,
+        info: params.replyLogger.info.bind(params.replyLogger),
+        warn: params.replyLogger.warn.bind(params.replyLogger),
+      });
+      ackAlreadySent = ackReaction !== null;
+      try {
+        const { transcribeFirstAudio } = await import("./audio-preflight.runtime.js");
+        // transcribeFirstAudio returns undefined on failure/disabled; store null so
+        // processMessage knows the attempt was already made and does not retry.
+        preflightAudioTranscript =
+          (await transcribeFirstAudio({
+            ctx: {
+              MediaPaths: [msg.mediaPath],
+              MediaTypes: msg.mediaType ? [msg.mediaType] : undefined,
+            },
+            cfg: params.cfg,
+          })) ?? null;
+      } catch {
+        // Non-fatal: store null so per-agent retries are suppressed.
+        preflightAudioTranscript = null;
+      }
+    };
+
+    if (msg.chatType === "group") {
+      const sender = getSenderIdentity(msg);
+      const metaCtx = {
+        From: msg.from,
+        To: msg.to,
+        SessionKey: route.sessionKey,
+        AccountId: route.accountId,
+        ChatType: msg.chatType,
+        ConversationLabel: conversationId,
+        GroupSubject: msg.groupSubject,
+        SenderName: sender.name ?? undefined,
+        SenderId: getPrimaryIdentityId(sender) ?? undefined,
+        SenderE164: sender.e164 ?? undefined,
+        Provider: "whatsapp",
+        Surface: "whatsapp",
+        OriginatingChannel: "whatsapp",
+        OriginatingTo: conversationId,
+      } satisfies MsgContext;
+      updateLastRouteInBackground({
+        cfg: params.cfg,
+        backgroundTasks: params.backgroundTasks,
+        storeAgentId: route.agentId,
+        sessionKey: route.sessionKey,
+        channel: "whatsapp",
+        to: conversationId,
+        accountId: route.accountId,
+        ctx: metaCtx,
+        warn: params.replyLogger.warn.bind(params.replyLogger),
+      });
+
+      let gating = await applyGroupGating({
+        cfg: params.cfg,
+        msg,
+        deferMissingMention: hasAudioBody && Boolean(msg.mediaPath),
+        conversationId,
+        groupHistoryKey,
+        agentId: route.agentId,
+        sessionKey: route.sessionKey,
+        baseMentionConfig: params.baseMentionConfig,
+        authDir: params.account.authDir,
+        selfChatMode: params.account.selfChatMode,
+        groupHistories: params.groupHistories,
+        groupHistoryLimit: params.groupHistoryLimit,
+        groupMemberNames: params.groupMemberNames,
+        logVerbose,
+        replyLogger: params.replyLogger,
+      });
+      if (
+        !gating.shouldProcess &&
+        "needsMentionText" in gating &&
+        gating.needsMentionText === true
+      ) {
+        await runAudioPreflightOnce();
+        gating = await applyGroupGating({
+          cfg: params.cfg,
+          msg,
+          ...(typeof preflightAudioTranscript === "string"
+            ? { mentionText: preflightAudioTranscript }
+            : {}),
+          conversationId,
+          groupHistoryKey,
+          agentId: route.agentId,
+          sessionKey: route.sessionKey,
+          baseMentionConfig: params.baseMentionConfig,
+          authDir: params.account.authDir,
+          selfChatMode: params.account.selfChatMode,
+          groupHistories: params.groupHistories,
+          groupHistoryLimit: params.groupHistoryLimit,
+          groupMemberNames: params.groupMemberNames,
+          logVerbose,
+          replyLogger: params.replyLogger,
+        });
+      }
+      if (!gating.shouldProcess) {
+        return;
+      }
+    } else {
+      // Ensure `peerId` for DMs is stable and stored as E.164 when possible.
+      if (!msg.sender?.e164 && !msg.senderE164 && peerId && peerId.startsWith("+")) {
+        const normalized = normalizeE164(peerId);
+        if (normalized) {
+          msg.sender = { ...msg.sender, e164: normalized };
+          msg.senderE164 = normalized;
+        }
+      }
+    }
+
+    await runAudioPreflightOnce();
+
+    // Broadcast groups: when we'd reply anyway, run multiple agents.
+    // Does not bypass group mention/activation gating above.
+    if (
+      await maybeBroadcastMessage({
+        cfg: params.cfg,
+        msg,
+        peerId,
+        route,
+        groupHistoryKey,
+        groupHistories: params.groupHistories,
+        ...(preflightAudioTranscript !== undefined ? { preflightAudioTranscript } : {}),
+        // Group ack eligibility depends on the target agent/session, so a
+        // preflight ack attempt on the base route must not suppress downstream
+        // per-agent checks during broadcast fan-out.
+        ...(ackAlreadySent && msg.chatType !== "group" ? { ackAlreadySent: true } : {}),
+        ...(ackReaction && msg.chatType !== "group" ? { ackReaction } : {}),
+        processMessage: (m, r, k, opts) => processForRoute(m, r, k, opts),
+      })
+    ) {
+      return;
+    }
+
+    await processForRoute(msg, route, groupHistoryKey, {
+      ...(preflightAudioTranscript !== undefined ? { preflightAudioTranscript } : {}),
+      ...(ackAlreadySent ? { ackAlreadySent: true } : {}),
+      ...(ackReaction ? { ackReaction } : {}),
+    });
+  };
+}

package/src/auto-reply/monitor/peer.ts

@@ -0,0 +1,17 @@
+import { getSenderIdentity } from "../../identity.js";
+import { jidToE164, normalizeE164 } from "../../text-runtime.js";
+import type { WebInboundMsg } from "../types.js";
+
+export function resolvePeerId(msg: WebInboundMsg) {
+  if (msg.chatType === "group") {
+    return msg.conversationId ?? msg.from;
+  }
+  const sender = getSenderIdentity(msg);
+  if (sender.e164) {
+    return normalizeE164(sender.e164) ?? sender.e164;
+  }
+  if (msg.from.includes("@")) {
+    return jidToE164(msg.from) ?? msg.from;
+  }
+  return normalizeE164(msg.from) ?? msg.from;
+}