@aspalagin/openclaw-max 0.7.3 → 0.8.1

package/src/group-mention.ts

@@ -0,0 +1,191 @@
+/**
+ * Group mention facts beyond the bot's native mention: core's configured
+ * mention patterns (messages.groupChat / agent groupChat mentionPatterns),
+ * also matched against the transcript of a captionless voice message — MAX's
+ * own transcript, else a core preflight transcription.
+ */
+
+import { resolveAgentConfig } from 'openclaw/plugin-sdk/agent-scope-runtime';
+import type { ChannelLogSink } from 'openclaw/plugin-sdk/channel-contract';
+import {
+  buildMentionRegexes,
+  type ChannelInboundMediaInput,
+  matchesMentionPatterns,
+  toInboundMediaFacts,
+} from 'openclaw/plugin-sdk/channel-inbound';
+import type { OpenClawConfig } from 'openclaw/plugin-sdk/core';
+import {
+  createChannelPreflightAudio,
+  formatAudioTranscriptForAgent,
+} from 'openclaw/plugin-sdk/media-understanding-runtime';
+
+import { admitMaxGroupSender } from './access-policy.js';
+import type { ResolvedMaxAccount } from './accounts.js';
+import type { MaxApi, MaxAttachment } from './api.js';
+import {
+  collectInboundAttachments,
+  type MaxInboundAttachments,
+  readAudioTranscription,
+} from './inbound-attachments.js';
+
+type PreflightRequest = Parameters<
+  ReturnType<typeof createChannelPreflightAudio>['resolve']
+>[0]['request'];
+
+const maxPreflightAudio = createChannelPreflightAudio<ChannelInboundMediaInput>({
+  channel: 'max',
+  isAudio: (media) => media.kind === 'audio',
+});
+
+/**
+ * Only explicitly configured patterns count: without them core derives
+ * patterns from the agent's identity name and emoji, which would start waking
+ * the bot in MAX groups that so far needed a real mention.
+ */
+function hasConfiguredMentionPatterns(cfg: OpenClawConfig, agentId: string): boolean {
+  const agentGroupChat = resolveAgentConfig(cfg, agentId)?.groupChat;
+  if (agentGroupChat && Object.hasOwn(agentGroupChat, 'mentionPatterns')) return true;
+  const groupChat = cfg.messages?.groupChat;
+  return Boolean(groupChat && Object.hasOwn(groupChat, 'mentionPatterns'));
+}
+
+/** Core mention regexes for this group, after the channel's mentionPatterns policy. */
+export function resolveMaxMentionRegexes(params: {
+  cfg: OpenClawConfig;
+  agentId: string;
+  account: ResolvedMaxAccount;
+  chatId: string;
+}): RegExp[] {
+  const { cfg, agentId, account, chatId } = params;
+  if (!hasConfiguredMentionPatterns(cfg, agentId)) return [];
+  return buildMentionRegexes(cfg, agentId, {
+    provider: 'max',
+    conversationId: chatId,
+    ...(account.config.mentionPatterns ? { providerPolicy: account.config.mentionPatterns } : {}),
+  });
+}
+
+/**
+ * Transcribe the first audio of the message with core media understanding
+ * (tools.media.audio). Returns the transcript and the index of the input core
+ * transcribed; undefined when there is no audio, STT is off or it fails.
+ */
+export async function transcribeMaxVoiceForMention(params: {
+  cfg: OpenClawConfig;
+  accountId: string;
+  chatId: string;
+  mediaInputs: ChannelInboundMediaInput[];
+}): Promise<{ transcript: string; index: number } | undefined> {
+  const ctx = {
+    Provider: 'max',
+    Surface: 'max',
+    OriginatingChannel: 'max',
+    OriginatingTo: `max:${params.chatId}`,
+    AccountId: params.accountId,
+    ChatType: 'group',
+    media: toInboundMediaFacts(params.mediaInputs),
+  } as PreflightRequest['ctx'];
+  const transcript = await maxPreflightAudio.resolve({ request: { ctx, cfg: params.cfg } });
+  if (!transcript) return undefined;
+  // Core marks the fact it transcribed on the request context.
+  const media = (ctx.media ?? []) as Array<{ transcribed?: boolean }>;
+  const marked = media.findIndex(
+    (fact, index) => fact?.transcribed === true && params.mediaInputs[index]?.transcribed !== true,
+  );
+  const index =
+    marked >= 0 ? marked : params.mediaInputs.findIndex((input) => input.kind === 'audio');
+  return { transcript, index };
+}
+
+export interface MaxVoiceMention {
+  /** Attachments already collected for the check, the transcribed audio marked. */
+  attachments?: MaxInboundAttachments;
+  /** Core preflight transcript (not set when MAX's own transcript matched). */
+  transcript?: string;
+}
+
+/**
+ * Mention check for a captionless voice message in a mention-gated group
+ * whose chat already passed the group policy. MAX's own transcript is matched
+ * as is; otherwise, for senders the group admits, the audio is downloaded
+ * (mediaMaxMb, mediaMaxCount) and transcribed once by core. The transcript
+ * then goes to the agent as core formats it, and the audio fact is marked
+ * transcribed so the reply pipeline does not transcribe it again. No match,
+ * no audio or any failure: undefined ("not mentioned").
+ */
+export async function resolveMaxVoiceMention(params: {
+  attachments: MaxAttachment[];
+  messageId: string;
+  chatId: number | undefined;
+  chatIdStr: string;
+  senderId: string;
+  mentionRegexes: RegExp[];
+  cfg: OpenClawConfig;
+  api: MaxApi;
+  account: ResolvedMaxAccount;
+  log?: ChannelLogSink;
+}): Promise<MaxVoiceMention | undefined> {
+  const { attachments, mentionRegexes, account, log } = params;
+  const audio = attachments.find((att) => att.type === 'audio');
+  if (!audio || mentionRegexes.length === 0) return undefined;
+
+  const platformTranscript = readAudioTranscription(audio);
+  if (platformTranscript) {
+    return matchesMentionPatterns(platformTranscript, mentionRegexes) ? {} : undefined;
+  }
+  // The inbound gate admitted the sender already; the shared rule again, so
+  // no caller transcribes for a sender outside the group's allowlist.
+  if (!admitMaxGroupSender(account, params.chatId, params.senderId).admitted) {
+    return undefined;
+  }
+
+  try {
+    const collected = await collectInboundAttachments({
+      attachments,
+      messageId: params.messageId,
+      chatId: params.chatId,
+      api: params.api,
+      account,
+      log,
+    });
+    if (!collected.mediaInputs.some((input) => input.kind === 'audio' && !input.transcribed)) {
+      return undefined;
+    }
+    const result = await transcribeMaxVoiceForMention({
+      cfg: params.cfg,
+      accountId: account.accountId,
+      chatId: params.chatIdStr,
+      mediaInputs: collected.mediaInputs,
+    });
+    if (!result || !matchesMentionPatterns(result.transcript, mentionRegexes)) return undefined;
+    const mediaInputs = collected.mediaInputs.map((input, index) =>
+      index === result.index ? { ...input, transcribed: true } : input,
+    );
+    return {
+      attachments: {
+        ...collected,
+        mediaInputs,
+        descriptions: [...collected.descriptions, formatAudioTranscriptForAgent(result.transcript)],
+      },
+      transcript: result.transcript,
+    };
+  } catch (err) {
+    log?.debug?.(`[${account.accountId}] Voice mention check failed: ${String(err)}`);
+    return undefined;
+  }
+}
+
+/** Core's deferred transcript echo (tools.media.audio.echoTranscript), after the gate. */
+export async function echoMaxVoiceTranscript(params: {
+  cfg: OpenClawConfig;
+  accountId: string;
+  chatId: string;
+  transcript: string;
+}): Promise<void> {
+  await maxPreflightAudio.send({
+    transcript: params.transcript,
+    cfg: params.cfg,
+    accountId: params.accountId,
+    originatingTo: `max:${params.chatId}`,
+  });
+}

package/src/inbound-attachments.ts

@@ -6,9 +6,9 @@
 import type { ChannelLogSink } from 'openclaw/plugin-sdk/channel-contract';
 import type { ChannelInboundMediaInput } from 'openclaw/plugin-sdk/channel-inbound';
 
-import type { ResolvedMaxAccount } from './accounts.js';
+import { type ResolvedMaxAccount, resolveMaxAccountNetwork } from './accounts.js';
 import type { MaxApi, MaxAttachment } from './api.js';
-import { sanitizeMaxFileName } from './media-temp.js';
+import { fetchMaxRemoteMedia, sanitizeMaxFileName } from './media-temp.js';
 import { getMaxRuntime } from './runtime.js';
 import { rememberStickerCode } from './sticker-cache.js';
 
@@ -17,8 +17,24 @@ export interface MaxInboundAttachments {
   descriptions: string[];
   /** One entry per downloaded attachment; array position is attachment identity. */
   mediaInputs: ChannelInboundMediaInput[];
+  /** Media attachments taken for download (the per-message count limit). */
+  mediaTaken: number;
 }
 
+/** Default most media attachments downloaded per inbound message: a full MAX album. */
+export const DEFAULT_INBOUND_MEDIA_MAX_COUNT = 12;
+
+/** Per-message media count limit of the account (`mediaMaxCount`). */
+export function resolveInboundMediaMaxCount(account: ResolvedMaxAccount): number {
+  return account.config.mediaMaxCount ?? DEFAULT_INBOUND_MEDIA_MAX_COUNT;
+}
+
+/** A voice message whose audio could not be loaded and has no MAX transcript. */
+const VOICE_UNAVAILABLE = '[Voice message: audio unavailable, no transcript]';
+
+/** Longest link-preview description handed to the agent. */
+const MAX_SHARE_DESCRIPTION_CHARS = 300;
+
 /** Process attachments: download media, build descriptions for non-downloadable types. */
 export async function collectInboundAttachments(params: {
   attachments: MaxAttachment[];
@@ -27,11 +43,16 @@ export async function collectInboundAttachments(params: {
   api: MaxApi;
   account: ResolvedMaxAccount;
   log?: ChannelLogSink;
+  /** Media downloads left for this message; default: the account's mediaMaxCount. */
+  mediaBudget?: number;
 }): Promise<MaxInboundAttachments> {
   const { attachments, messageId, chatId, api, account, log } = params;
   const core = getMaxRuntime();
   const attachmentDescriptions: string[] = [];
   const mediaInputs: ChannelInboundMediaInput[] = [];
+  const mediaBudget = params.mediaBudget ?? resolveInboundMediaMaxCount(account);
+  let mediaTaken = 0;
+  let mediaSkipped = 0;
 
   for (const att of attachments) {
     const attType = att.type ?? 'unknown';
@@ -49,6 +70,25 @@ export async function collectInboundAttachments(params: {
         }
       }
 
+      // MAX may transcribe voice messages itself (AudioAttachment.transcription,
+      // a sibling of payload). The platform transcript wins: the text goes to
+      // the agent and the audio fact is marked transcribed (the SDK contract
+      // for platform transcripts), so core media understanding skips STT.
+      // Without it the fact stays untranscribed and core STT
+      // (tools.media.audio) runs; a failed STT leaves core's marker.
+      const transcription = attType === 'audio' ? readAudioTranscription(att) : undefined;
+      if (transcription) {
+        attachmentDescriptions.push(`[Voice transcript: ${transcription}]`);
+      }
+
+      // Over the per-message count limit: no lookup, no download; the text
+      // parts above (sticker code, transcript) are cheap and stay.
+      if (mediaTaken >= mediaBudget) {
+        mediaSkipped += 1;
+        continue;
+      }
+      mediaTaken += 1;
+
       let url = (payload?.url ?? (att as Record<string, unknown>).url ?? '') as string;
 
       // Inbound video attachments often carry only a token — resolve playback
@@ -75,18 +115,11 @@ export async function collectInboundAttachments(params: {
         }
       }
 
-      // MAX may transcribe voice messages itself (AudioAttachment.transcription,
-      // a sibling of payload). The text goes to the agent and the audio fact is
-      // marked transcribed, so core media understanding does not run STT again.
-      const transcription = attType === 'audio' ? readAudioTranscription(att) : undefined;
-      if (transcription) {
-        attachmentDescriptions.push(`[Voice transcript: ${transcription}]`);
-      }
-
       if (url && typeof url === 'string' && url.startsWith('http')) {
         try {
           const maxBytes = (account.config.mediaMaxMb ?? 20) * 1024 * 1024;
-          const fetched = await core.channel.media.fetchRemoteMedia({ url, maxBytes });
+          const { proxyUrl } = resolveMaxAccountNetwork(account);
+          const fetched = await fetchMaxRemoteMedia(url, maxBytes, proxyUrl);
           const inboundFileName = fetched.fileName
             ? sanitizeMaxFileName(fetched.fileName, fetched.contentType)
             : undefined;
@@ -99,23 +132,31 @@ export async function collectInboundAttachments(params: {
           );
           // Only the local copy goes to the agent: MAX download URLs are signed
           // and short-lived, so they are not recorded as the media url.
+          // A MAX audio attachment is a voice message: the explicit kind keeps
+          // core STT selecting it even when the CDN answers with a generic
+          // content type.
           mediaInputs.push({
             path: saved.path,
             contentType: saved.contentType,
             fileName: inboundFileName,
             messageId,
+            ...(attType === 'audio' ? { kind: 'audio' as const } : {}),
             ...(transcription ? { transcribed: true } : {}),
           });
         } catch (err) {
           log?.error?.(`[${account.accountId}] Failed to download ${attType}: ${String(err)}`);
           // Fall back to text description (sticker code already added above)
-          if (attType !== 'sticker') {
+          if (attType === 'audio') {
+            if (!transcription) attachmentDescriptions.push(VOICE_UNAVAILABLE);
+          } else if (attType !== 'sticker') {
             attachmentDescriptions.push(`[${attType}: ${url}]`);
           }
         }
       } else {
         // No URL — text description
-        if (attType === 'sticker') {
+        if (attType === 'audio') {
+          if (!transcription) attachmentDescriptions.push(VOICE_UNAVAILABLE);
+        } else if (attType === 'sticker') {
           const code = payload?.code ?? '';
           attachmentDescriptions.push(`[Sticker${code ? `: ${code}` : ''}]`);
         } else if (attType === 'file') {
@@ -126,25 +167,135 @@ export async function collectInboundAttachments(params: {
         }
       }
     } else if (attType === 'share') {
-      const url = (payload?.url ?? (att as Record<string, unknown>).url ?? '') as string;
-      attachmentDescriptions.push(`[Share${url ? `: ${url}` : ''}]`);
+      attachmentDescriptions.push(describeShare(att));
     } else if (attType === 'location') {
-      const lat = (att as Record<string, unknown>).latitude ?? payload?.latitude ?? '';
-      const lon = (att as Record<string, unknown>).longitude ?? payload?.longitude ?? '';
-      attachmentDescriptions.push(`[Location: ${lat}, ${lon}]`);
+      attachmentDescriptions.push(describeLocation(att));
     } else if (attType === 'contact') {
-      const name = payload?.name ?? payload?.vcf_info ?? '';
-      attachmentDescriptions.push(`[Contact${name ? `: ${name}` : ''}]`);
+      attachmentDescriptions.push(describeContact(payload));
     } else if (attType !== 'inline_keyboard') {
       attachmentDescriptions.push(`[${attType}]`);
     }
   }
 
-  return { descriptions: attachmentDescriptions, mediaInputs };
+  if (mediaSkipped > 0) {
+    log?.warn?.(
+      `[${account.accountId}] ${mediaSkipped} media attachment(s) of ${messageId} not loaded: mediaMaxCount`,
+    );
+    attachmentDescriptions.push(
+      `[${mediaSkipped} more media attachment(s) not loaded: limit of ${resolveInboundMediaMaxCount(account)} per message]`,
+    );
+  }
+
+  return { descriptions: attachmentDescriptions, mediaInputs, mediaTaken };
+}
+
+/** One line of untrusted text: whitespace collapsed, clipped by code points. */
+function toInlineText(value: unknown, maxChars?: number): string {
+  if (typeof value !== 'string') return '';
+  const chars = Array.from(value.replace(/\s+/g, ' ').trim());
+  if (maxChars === undefined || chars.length <= maxChars) return chars.join('');
+  return `${chars.slice(0, maxChars - 1).join('')}…`;
+}
+
+/** Link preview card: title, description and link, whichever MAX sent. */
+function describeShare(att: MaxAttachment): string {
+  const raw = att as Record<string, unknown>;
+  const payload = raw.payload as Record<string, unknown> | undefined;
+  const url = toInlineText(payload?.url ?? raw.url);
+  const parts = [
+    toInlineText(raw.title),
+    toInlineText(raw.description, MAX_SHARE_DESCRIPTION_CHARS),
+    url,
+  ].filter(Boolean);
+  return parts.length ? `[Share: ${parts.join(' — ')}]` : '[Share]';
+}
+
+/** Coordinates with a map link (a Yandex Maps placemark). */
+function describeLocation(att: MaxAttachment): string {
+  const raw = att as Record<string, unknown>;
+  const payload = raw.payload as Record<string, unknown> | undefined;
+  const lat = Number(raw.latitude ?? payload?.latitude);
+  const lon = Number(raw.longitude ?? payload?.longitude);
+  if (!Number.isFinite(lat) || !Number.isFinite(lon) || Math.abs(lat) > 90 || Math.abs(lon) > 180)
+    return '[Location]';
+  return `[Location: ${lat}, ${lon} — https://yandex.ru/maps/?pt=${lon},${lat}&z=16&l=map]`;
+}
+
+/** Contact card: name and phones from the VCard, the MAX profile if linked; never the raw VCard. */
+function describeContact(payload: Record<string, unknown> | undefined): string {
+  const vcard = typeof payload?.vcf_info === 'string' ? parseMaxVcard(payload.vcf_info) : undefined;
+  const user = payload?.max_info as Record<string, unknown> | null | undefined;
+  const profileName = [user?.first_name, user?.last_name]
+    .map((part) => toInlineText(part))
+    .filter(Boolean)
+    .join(' ');
+  const name = vcard?.name || profileName || toInlineText(payload?.name);
+  const parts = name ? [name] : [];
+  if (vcard?.phones.length) parts.push(`phone: ${vcard.phones.join(', ')}`);
+  if (user && typeof user.user_id === 'number') {
+    const username = toInlineText(user.username);
+    parts.push(`MAX user: ${user.user_id}${username ? ` (@${username.replace(/^@/, '')})` : ''}`);
+  }
+  return parts.length ? `[Contact: ${parts.join('; ')}]` : '[Contact]';
+}
+
+/** Most phone numbers taken from one VCard. */
+const MAX_VCARD_PHONES = 5;
+
+/**
+ * Display name and phone numbers of a VCard (2.1/3.0/4.0): FN, else N; TEL
+ * values (a `tel:` URI prefix dropped). Folded lines, escapes and
+ * quoted-printable values are decoded; everything else is ignored.
+ * @internal exported for testing.
+ */
+export function parseMaxVcard(vcf: string): { name: string; phones: string[] } {
+  const lines = vcf.replace(/\r?\n[ \t]/g, '').split(/\r?\n/);
+  let fullName = '';
+  let structuredName = '';
+  const phones: string[] = [];
+  for (const line of lines) {
+    const colon = line.indexOf(':');
+    if (colon <= 0) continue;
+    const [rawProperty, ...params] = line.slice(0, colon).split(';');
+    const property = rawProperty.replace(/^[^.]*\./, '').toUpperCase();
+    let value = line.slice(colon + 1);
+    if (params.some((param) => /^ENCODING=QUOTED-PRINTABLE$/i.test(param))) {
+      value = decodeQuotedPrintable(value);
+    }
+    if (property === 'FN' && !fullName) {
+      fullName = toInlineText(unescapeVcardValue(value));
+    } else if (property === 'N' && !structuredName) {
+      // N: family;given;additional;prefix;suffix
+      const [family = '', given = ''] = value.split(';').map(unescapeVcardValue);
+      structuredName = toInlineText(`${given} ${family}`);
+    } else if (property === 'TEL') {
+      const phone = toInlineText(unescapeVcardValue(value).replace(/^tel:/i, ''));
+      if (phone && !phones.includes(phone) && phones.length < MAX_VCARD_PHONES) phones.push(phone);
+    }
+  }
+  return { name: fullName || structuredName, phones };
+}
+
+function unescapeVcardValue(value: string): string {
+  return value.replace(/\\([nN,;\\])/g, (_, ch: string) => (ch === 'n' || ch === 'N' ? ' ' : ch));
+}
+
+function decodeQuotedPrintable(value: string): string {
+  const bytes: number[] = [];
+  for (let i = 0; i < value.length; i += 1) {
+    const hex = value[i] === '=' ? value.slice(i + 1, i + 3) : '';
+    if (/^[0-9A-Fa-f]{2}$/.test(hex)) {
+      bytes.push(parseInt(hex, 16));
+      i += 2;
+    } else {
+      bytes.push(...Buffer.from(value[i], 'utf8'));
+    }
+  }
+  return Buffer.from(bytes).toString('utf8');
 }
 
 /** Non-empty MAX transcription of an audio attachment, if any. */
-function readAudioTranscription(att: MaxAttachment): string | undefined {
+export function readAudioTranscription(att: MaxAttachment): string | undefined {
   const value = att.transcription;
   if (typeof value !== 'string') return undefined;
   const trimmed = value.trim();