@openclaw/qqbot 2026.5.1-beta.1

package/src/engine/gateway/inbound-attachments.ts

@@ -0,0 +1,360 @@
+import type { AudioConvertPort } from "../adapter/audio.port.js";
+import { downloadFile } from "../utils/file-utils.js";
+import { getQQBotMediaDir } from "../utils/platform.js";
+import { normalizeOptionalString } from "../utils/string-normalize.js";
+import { transcribeAudio, resolveSTTConfig } from "../utils/stt.js";
+
+// Re-export the port type for convenience.
+export type { AudioConvertPort } from "../adapter/audio.port.js";
+
+interface RawAttachment {
+  content_type: string;
+  url: string;
+  filename?: string;
+  voice_wav_url?: string;
+  asr_refer_text?: string;
+}
+
+type TranscriptSource = "stt" | "asr" | "fallback";
+
+/** Normalized attachment output consumed by the gateway. */
+export interface ProcessedAttachments {
+  attachmentInfo: string;
+  imageUrls: string[];
+  imageMediaTypes: string[];
+  voiceAttachmentPaths: string[];
+  voiceAttachmentUrls: string[];
+  voiceAsrReferTexts: string[];
+  voiceTranscripts: string[];
+  voiceTranscriptSources: TranscriptSource[];
+  attachmentLocalPaths: Array<string | null>;
+}
+
+interface ProcessContext {
+  accountId: string;
+  cfg: unknown;
+  audioConvert: AudioConvertPort;
+  log?: {
+    info: (msg: string) => void;
+    error: (msg: string) => void;
+    debug?: (msg: string) => void;
+  };
+}
+
+const EMPTY_RESULT: ProcessedAttachments = {
+  attachmentInfo: "",
+  imageUrls: [],
+  imageMediaTypes: [],
+  voiceAttachmentPaths: [],
+  voiceAttachmentUrls: [],
+  voiceAsrReferTexts: [],
+  voiceTranscripts: [],
+  voiceTranscriptSources: [],
+  attachmentLocalPaths: [],
+};
+
+/** Download, convert, transcribe, and classify inbound attachments. */
+export async function processAttachments(
+  attachments: RawAttachment[] | undefined,
+  ctx: ProcessContext,
+): Promise<ProcessedAttachments> {
+  if (!attachments?.length) {
+    return EMPTY_RESULT;
+  }
+
+  const { accountId: _accountId, cfg, log, audioConvert } = ctx;
+  const downloadDir = getQQBotMediaDir("downloads");
+
+  const imageUrls: string[] = [];
+  const imageMediaTypes: string[] = [];
+  const voiceAttachmentPaths: string[] = [];
+  const voiceAttachmentUrls: string[] = [];
+  const voiceAsrReferTexts: string[] = [];
+  const voiceTranscripts: string[] = [];
+  const voiceTranscriptSources: TranscriptSource[] = [];
+  const attachmentLocalPaths: Array<string | null> = [];
+  const otherAttachments: string[] = [];
+
+  // Phase 1: download all attachments in parallel.
+  const downloadTasks = attachments.map(async (att) => {
+    const attUrl = att.url?.startsWith("//") ? `https:${att.url}` : att.url;
+    const isVoice = audioConvert.isVoiceAttachment(att);
+    const wavUrl =
+      isVoice && att.voice_wav_url
+        ? att.voice_wav_url.startsWith("//")
+          ? `https:${att.voice_wav_url}`
+          : att.voice_wav_url
+        : "";
+
+    let localPath: string | null = null;
+    let audioPath: string | null = null;
+
+    if (isVoice && wavUrl) {
+      const wavLocalPath = await downloadFile(wavUrl, downloadDir);
+      if (wavLocalPath) {
+        localPath = wavLocalPath;
+        audioPath = wavLocalPath;
+        log?.debug?.(`Voice attachment: ${att.filename}, downloaded WAV directly (skip SILK→WAV)`);
+      } else {
+        log?.error(`Failed to download voice_wav_url, falling back to original URL`);
+      }
+    }
+
+    if (!localPath) {
+      localPath = await downloadFile(attUrl, downloadDir, att.filename);
+    }
+
+    return { att, attUrl, isVoice, localPath, audioPath };
+  });
+
+  const downloadResults = await Promise.all(downloadTasks);
+
+  // Phase 2: convert/transcribe voice attachments and classify everything else.
+  const processTasks = downloadResults.map(
+    async ({ att, attUrl, isVoice, localPath, audioPath }) => {
+      const asrReferText = normalizeOptionalString(att.asr_refer_text) ?? "";
+      const wavUrl =
+        isVoice && att.voice_wav_url
+          ? att.voice_wav_url.startsWith("//")
+            ? `https:${att.voice_wav_url}`
+            : att.voice_wav_url
+          : "";
+      const voiceSourceUrl = wavUrl || attUrl;
+
+      const meta = {
+        voiceUrl: isVoice && voiceSourceUrl ? voiceSourceUrl : undefined,
+        asrReferText: isVoice && asrReferText ? asrReferText : undefined,
+      };
+
+      if (localPath) {
+        if (att.content_type?.startsWith("image/")) {
+          log?.debug?.(`Downloaded attachment to: ${localPath}`);
+          return { localPath, type: "image" as const, contentType: att.content_type, meta };
+        }
+        if (isVoice) {
+          log?.debug?.(`Downloaded attachment to: ${localPath}`);
+          return processVoiceAttachment(
+            localPath,
+            audioPath,
+            att,
+            asrReferText,
+            cfg,
+            downloadDir,
+            audioConvert,
+            log,
+          );
+        }
+        log?.debug?.(`Downloaded attachment to: ${localPath}`);
+        return { localPath, type: "other" as const, filename: att.filename, meta };
+      }
+      log?.error(`Failed to download: ${attUrl}`);
+      if (att.content_type?.startsWith("image/")) {
+        return {
+          localPath: null,
+          type: "image-fallback" as const,
+          attUrl,
+          contentType: att.content_type,
+          meta,
+        };
+      }
+      if (isVoice && asrReferText) {
+        log?.info(`Voice attachment download failed, using asr_refer_text fallback`);
+        return {
+          localPath: null,
+          type: "voice-fallback" as const,
+          transcript: asrReferText,
+          meta,
+        };
+      }
+      return {
+        localPath: null,
+        type: "other-fallback" as const,
+        filename: att.filename ?? att.content_type,
+        meta,
+      };
+    },
+  );
+
+  const processResults = await Promise.all(processTasks);
+
+  // Phase 3: collect results in the original attachment order.
+  for (const result of processResults) {
+    if (result.meta.voiceUrl) {
+      voiceAttachmentUrls.push(result.meta.voiceUrl);
+    }
+    if (result.meta.asrReferText) {
+      voiceAsrReferTexts.push(result.meta.asrReferText);
+    }
+
+    if (result.type === "image" && result.localPath) {
+      imageUrls.push(result.localPath);
+      imageMediaTypes.push(result.contentType);
+      attachmentLocalPaths.push(result.localPath);
+    } else if (result.type === "voice" && result.localPath) {
+      voiceAttachmentPaths.push(result.localPath);
+      voiceTranscripts.push(result.transcript);
+      voiceTranscriptSources.push(result.transcriptSource);
+      attachmentLocalPaths.push(result.localPath);
+    } else if (result.type === "other" && result.localPath) {
+      otherAttachments.push(`[Attachment: ${result.localPath}]`);
+      attachmentLocalPaths.push(result.localPath);
+    } else if (result.type === "image-fallback") {
+      imageUrls.push(result.attUrl);
+      imageMediaTypes.push(result.contentType);
+      attachmentLocalPaths.push(null);
+    } else if (result.type === "voice-fallback") {
+      voiceTranscripts.push(result.transcript);
+      voiceTranscriptSources.push("asr");
+      attachmentLocalPaths.push(null);
+    } else if (result.type === "other-fallback") {
+      otherAttachments.push(`[Attachment: ${result.filename}] (download failed)`);
+      attachmentLocalPaths.push(null);
+    }
+  }
+
+  const attachmentInfo = otherAttachments.length > 0 ? "\n" + otherAttachments.join("\n") : "";
+
+  return {
+    attachmentInfo,
+    imageUrls,
+    imageMediaTypes,
+    voiceAttachmentPaths,
+    voiceAttachmentUrls,
+    voiceAsrReferTexts,
+    voiceTranscripts,
+    voiceTranscriptSources,
+    attachmentLocalPaths,
+  };
+}
+
+// formatVoiceText is now in core/utils/voice-text.ts (re-exported above).
+
+// Internal helpers.
+
+type VoiceResult =
+  | {
+      localPath: string;
+      type: "voice";
+      transcript: string;
+      transcriptSource: TranscriptSource;
+      meta: { voiceUrl?: string; asrReferText?: string };
+    }
+  | {
+      localPath: string;
+      type: "voice";
+      transcript: string;
+      transcriptSource: TranscriptSource;
+      meta: { voiceUrl?: string; asrReferText?: string };
+    };
+
+async function processVoiceAttachment(
+  localPath: string,
+  audioPath: string | null,
+  att: RawAttachment,
+  asrReferText: string,
+  cfg: unknown,
+  downloadDir: string,
+  audioConvert: AudioConvertPort,
+  log: ProcessContext["log"],
+): Promise<VoiceResult> {
+  const wavUrl = att.voice_wav_url
+    ? att.voice_wav_url.startsWith("//")
+      ? `https:${att.voice_wav_url}`
+      : att.voice_wav_url
+    : "";
+  const attUrl = att.url?.startsWith("//") ? `https:${att.url}` : att.url;
+  const voiceSourceUrl = wavUrl || attUrl;
+  const meta = {
+    voiceUrl: voiceSourceUrl || undefined,
+    asrReferText: asrReferText || undefined,
+  };
+
+  const sttCfg = resolveSTTConfig(cfg as Record<string, unknown>);
+  if (!sttCfg) {
+    if (asrReferText) {
+      log?.debug?.(
+        `Voice attachment: ${att.filename} (STT not configured, using asr_refer_text fallback)`,
+      );
+      return { localPath, type: "voice", transcript: asrReferText, transcriptSource: "asr", meta };
+    }
+    log?.debug?.(`Voice attachment: ${att.filename} (STT not configured, skipping transcription)`);
+    return {
+      localPath,
+      type: "voice",
+      transcript: "[Voice message - transcription unavailable because STT is not configured]",
+      transcriptSource: "fallback",
+      meta,
+    };
+  }
+
+  // Convert SILK input to WAV before STT when necessary.
+  if (!audioPath) {
+    log?.debug?.(`Voice attachment: ${att.filename}, converting SILK→WAV...`);
+    try {
+      const wavResult = await audioConvert.convertSilkToWav(localPath, downloadDir);
+      if (wavResult) {
+        audioPath = wavResult.wavPath;
+        log?.debug?.(
+          `Voice converted: ${wavResult.wavPath} (${audioConvert.formatDuration(wavResult.duration)})`,
+        );
+      } else {
+        audioPath = localPath;
+      }
+    } catch (convertErr) {
+      log?.error(
+        `Voice conversion failed: ${
+          convertErr instanceof Error ? convertErr.message : JSON.stringify(convertErr)
+        }`,
+      );
+      if (asrReferText) {
+        return {
+          localPath,
+          type: "voice",
+          transcript: asrReferText,
+          transcriptSource: "asr",
+          meta,
+        };
+      }
+      return {
+        localPath,
+        type: "voice",
+        transcript: "[Voice message - format conversion failed]",
+        transcriptSource: "fallback",
+        meta,
+      };
+    }
+  }
+
+  // Run speech-to-text on the prepared audio file.
+  try {
+    const transcript = await transcribeAudio(audioPath, cfg as Record<string, unknown>);
+    if (transcript) {
+      log?.debug?.(`STT transcript: ${transcript.slice(0, 100)}...`);
+      return { localPath, type: "voice", transcript, transcriptSource: "stt", meta };
+    }
+    if (asrReferText) {
+      log?.debug?.(`STT returned empty result, using asr_refer_text fallback`);
+      return { localPath, type: "voice", transcript: asrReferText, transcriptSource: "asr", meta };
+    }
+    log?.debug?.(`STT returned empty result`);
+    return {
+      localPath,
+      type: "voice",
+      transcript: "[Voice message - transcription returned an empty result]",
+      transcriptSource: "fallback",
+      meta,
+    };
+  } catch (sttErr) {
+    log?.error(`STT failed: ${sttErr instanceof Error ? sttErr.message : JSON.stringify(sttErr)}`);
+    if (asrReferText) {
+      return { localPath, type: "voice", transcript: asrReferText, transcriptSource: "asr", meta };
+    }
+    return {
+      localPath,
+      type: "voice",
+      transcript: "[Voice message - transcription failed]",
+      transcriptSource: "fallback",
+      meta,
+    };
+  }
+}

package/src/engine/gateway/inbound-context.ts

@@ -0,0 +1,195 @@
+/**
+ * InboundContext — the structured result of the inbound pipeline.
+ *
+ * Connects the inbound stage (content building, attachment processing,
+ * quote resolution) with the outbound stage (AI dispatch, deliver callbacks).
+ *
+ * All fields are readonly after construction. The outbound dispatcher
+ * reads from this object but never mutates it.
+ */
+
+import type { QQBotAccessDecision, QQBotAccessReasonCode } from "../access/index.js";
+import type { EngineAdapters } from "../adapter/index.js";
+import type { GroupActivationMode, SessionStoreReader } from "../group/activation.js";
+import type { HistoryEntry } from "../group/history.js";
+import type { GroupMessageGateResult } from "../group/message-gating.js";
+import type { QueuedMessage } from "./message-queue.js";
+import type {
+  GatewayAccount,
+  EngineLogger,
+  GatewayPluginRuntime,
+  ProcessedAttachments,
+} from "./types.js";
+import type { TypingKeepAlive } from "./typing-keepalive.js";
+
+// ============ InboundContext ============
+
+/** Quote (reply-to) metadata resolved during inbound processing. */
+export interface ReplyToInfo {
+  id: string;
+  body?: string;
+  sender?: string;
+  isQuote: boolean;
+}
+
+/**
+ * Group-specific inbound metadata.
+ *
+ * Populated for group / guild events; left `undefined` for DMs. Keeping
+ * the group fields under a nested bag makes it obvious which fields are
+ * safe to read only when `isGroupChat === true`.
+ *
+ * The shape is kept small on purpose: everything derivable from `gate`
+ * (raw wasMentioned / explicit / implicit / hasAnyMention / bypass) is
+ * stored once on `gate`, not duplicated on the outer object.
+ */
+export interface InboundGroupInfo {
+  // ---- Gating decision ----
+  /** Full gate evaluation result (source of truth for mention state). */
+  gate: GroupMessageGateResult;
+  /** Effective activation mode after session-store / cfg merge. */
+  activation: GroupActivationMode;
+
+  // ---- Persistence-relevant ----
+  /** Per-group history buffer cap. Zero → disabled. */
+  historyLimit: number;
+  /** `true` if this message was built by merging several queued entries. */
+  isMerged: boolean;
+  /** The unfiltered list of queued messages when `isMerged`, else undefined. */
+  mergedMessages?: readonly QueuedMessage[];
+
+  // ---- Presentation / prompt inputs ----
+  /** Bundle of display-only strings; assembled by the envelope stage. */
+  display: {
+    /** Human-readable group name ("My Group" / first 8 chars of openid). */
+    groupName: string;
+    /** Sender label ("Nick (OPENID)" / "OPENID") for the UI. */
+    senderLabel: string;
+    /** Channel-level intro hint contributed by the platform adapter. */
+    introHint?: string;
+    /** Per-group behaviour prompt appended to the system prompt. */
+    behaviorPrompt?: string;
+  };
+}
+
+/** Fully resolved inbound context passed to the outbound dispatcher. */
+export interface InboundContext {
+  // ---- Original event ----
+  event: QueuedMessage;
+
+  // ---- Routing ----
+  route: { sessionKey: string; accountId: string; agentId?: string };
+  isGroupChat: boolean;
+  peerId: string;
+  /** Fully qualified target address: "qqbot:c2c:xxx" / "qqbot:group:xxx" etc. */
+  qualifiedTarget: string;
+  fromAddress: string;
+
+  // ---- Content ----
+  /** event.content after parseFaceTags. */
+  parsedContent: string;
+  /** parsedContent + voiceText + attachmentInfo — the user-visible text. */
+  userContent: string;
+  /** "[Quoted message begins]…[ends]" or empty. */
+  quotePart: string;
+  /** Per-message dynamic metadata lines (images, voice, ASR). */
+  dynamicCtx: string;
+  /** quotePart + userContent. */
+  userMessage: string;
+  /** dynamicCtx + userMessage (or raw content for slash commands). */
+  agentBody: string;
+  /** Formatted inbound envelope (Web UI body). */
+  body: string;
+
+  // ---- System prompts ----
+  systemPrompts: string[];
+  groupSystemPrompt?: string;
+
+  // ---- Attachments ----
+  attachments: ProcessedAttachments;
+  localMediaPaths: string[];
+  localMediaTypes: string[];
+  remoteMediaUrls: string[];
+  remoteMediaTypes: string[];
+
+  // ---- Voice ----
+  uniqueVoicePaths: string[];
+  uniqueVoiceUrls: string[];
+  uniqueVoiceAsrReferTexts: string[];
+  voiceMediaTypes: string[];
+  hasAsrReferFallback: boolean;
+  voiceTranscriptSources: string[];
+
+  // ---- Reply-to / Quote ----
+  replyTo?: ReplyToInfo;
+
+  // ---- Auth ----
+  commandAuthorized: boolean;
+
+  // ---- Group ----
+  /** Populated only for group / guild messages. */
+  group?: InboundGroupInfo;
+
+  // ---- Blocking / skipping ----
+  /**
+   * Whether the inbound message should be blocked outright (access policy
+   * refused the sender). Mutually exclusive with `skipped`.
+   */
+  blocked: boolean;
+  /** Human-readable reason for `blocked`, for logging only. */
+  blockReason?: string;
+  /** Structured reason code for `blocked`. */
+  blockReasonCode?: QQBotAccessReasonCode;
+  /** The raw access decision produced by the policy engine. */
+  accessDecision?: QQBotAccessDecision;
+  /**
+   * Whether the inbound was accepted by access control but stopped before
+   * AI dispatch by the group gate (e.g. "skip_no_mention"). The caller
+   * should NOT forward `skipped` messages to the outbound dispatcher, but
+   * history / activity side-effects may already have been applied.
+   */
+  skipped: boolean;
+  /** Structured reason code for `skipped`. */
+  skipReason?: "drop_other_mention" | "block_unauthorized_command" | "skip_no_mention";
+
+  // ---- Typing ----
+  typing: { keepAlive: TypingKeepAlive | null };
+  /** refIdx returned by the initial InputNotify call. */
+  inputNotifyRefIdx?: string;
+}
+
+// ============ Pipeline dependencies ============
+
+/** Dependencies injected into the inbound pipeline. */
+export interface InboundPipelineDeps {
+  account: GatewayAccount;
+  cfg: unknown;
+  log?: EngineLogger;
+  runtime: GatewayPluginRuntime;
+  /** Start typing indicator and return the refIdx from InputNotify. */
+  startTyping: (event: QueuedMessage) => Promise<{
+    refIdx?: string;
+    keepAlive: TypingKeepAlive | null;
+  }>;
+  // ---- Group dependencies (optional — omit when the caller doesn't need
+  // group support, e.g. a DM-only test harness). ----
+  /** Shared per-connection history buffer, created by the gateway. */
+  groupHistories?: Map<string, HistoryEntry[]>;
+  /** Session-store reader for activation-mode overrides. */
+  sessionStoreReader?: SessionStoreReader;
+  /** Whether text-based control commands are enabled globally. */
+  allowTextCommands?: boolean;
+  /**
+   * Framework probe that returns true when `content` is a known control
+   * command. Injected to avoid hard-coding a list of commands in engine.
+   */
+  isControlCommand?: (content: string) => boolean;
+  /** Optional platform hook that contributes a channel-level intro hint. */
+  resolveGroupIntroHint?: (params: {
+    cfg: unknown;
+    accountId: string;
+    groupId: string;
+  }) => string | undefined;
+  /** SDK adapter ports for delegating to shared implementations. */
+  adapters: EngineAdapters;
+}