@gakr-gakr/qqbot 0.1.0

package/src/engine/gateway/inbound-attachments.ts

@@ -0,0 +1,360 @@
+import type { AudioConvertPort } from "../adapter/audio.port.js";
+import { downloadFile } from "../utils/file-utils.js";
+import { getQQBotMediaDir } from "../utils/platform.js";
+import { normalizeOptionalString } from "../utils/string-normalize.js";
+import { transcribeAudio, resolveSTTConfig } from "../utils/stt.js";
+
+// Re-export the port type for convenience.
+export type { AudioConvertPort } from "../adapter/audio.port.js";
+
+interface RawAttachment {
+  content_type: string;
+  url: string;
+  filename?: string;
+  voice_wav_url?: string;
+  asr_refer_text?: string;
+}
+
+type TranscriptSource = "stt" | "asr" | "fallback";
+
+/** Normalized attachment output consumed by the gateway. */
+export interface ProcessedAttachments {
+  attachmentInfo: string;
+  imageUrls: string[];
+  imageMediaTypes: string[];
+  voiceAttachmentPaths: string[];
+  voiceAttachmentUrls: string[];
+  voiceAsrReferTexts: string[];
+  voiceTranscripts: string[];
+  voiceTranscriptSources: TranscriptSource[];
+  attachmentLocalPaths: Array<string | null>;
+}
+
+interface ProcessContext {
+  accountId: string;
+  cfg: unknown;
+  audioConvert: AudioConvertPort;
+  log?: {
+    info: (msg: string) => void;
+    error: (msg: string) => void;
+    debug?: (msg: string) => void;
+  };
+}
+
+const EMPTY_RESULT: ProcessedAttachments = {
+  attachmentInfo: "",
+  imageUrls: [],
+  imageMediaTypes: [],
+  voiceAttachmentPaths: [],
+  voiceAttachmentUrls: [],
+  voiceAsrReferTexts: [],
+  voiceTranscripts: [],
+  voiceTranscriptSources: [],
+  attachmentLocalPaths: [],
+};
+
+/** Download, convert, transcribe, and classify inbound attachments. */
+export async function processAttachments(
+  attachments: RawAttachment[] | undefined,
+  ctx: ProcessContext,
+): Promise<ProcessedAttachments> {
+  if (!attachments?.length) {
+    return EMPTY_RESULT;
+  }
+
+  const { accountId: _accountId, cfg, log, audioConvert } = ctx;
+  const downloadDir = getQQBotMediaDir("downloads");
+
+  const imageUrls: string[] = [];
+  const imageMediaTypes: string[] = [];
+  const voiceAttachmentPaths: string[] = [];
+  const voiceAttachmentUrls: string[] = [];
+  const voiceAsrReferTexts: string[] = [];
+  const voiceTranscripts: string[] = [];
+  const voiceTranscriptSources: TranscriptSource[] = [];
+  const attachmentLocalPaths: Array<string | null> = [];
+  const otherAttachments: string[] = [];
+
+  // Phase 1: download all attachments in parallel.
+  const downloadTasks = attachments.map(async (att) => {
+    const attUrl = att.url?.startsWith("//") ? `https:${att.url}` : att.url;
+    const isVoice = audioConvert.isVoiceAttachment(att);
+    const wavUrl =
+      isVoice && att.voice_wav_url
+        ? att.voice_wav_url.startsWith("//")
+          ? `https:${att.voice_wav_url}`
+          : att.voice_wav_url
+        : "";
+
+    let localPath: string | null = null;
+    let audioPath: string | null = null;
+
+    if (isVoice && wavUrl) {
+      const wavLocalPath = await downloadFile(wavUrl, downloadDir);
+      if (wavLocalPath) {
+        localPath = wavLocalPath;
+        audioPath = wavLocalPath;
+        log?.debug?.(`Voice attachment: ${att.filename}, downloaded WAV directly (skip SILK→WAV)`);
+      } else {
+        log?.error(`Failed to download voice_wav_url, falling back to original URL`);
+      }
+    }
+
+    if (!localPath) {
+      localPath = await downloadFile(attUrl, downloadDir, att.filename);
+    }
+
+    return { att, attUrl, isVoice, localPath, audioPath };
+  });
+
+  const downloadResults = await Promise.all(downloadTasks);
+
+  // Phase 2: convert/transcribe voice attachments and classify everything else.
+  const processTasks = downloadResults.map(
+    async ({ att, attUrl, isVoice, localPath, audioPath }) => {
+      const asrReferText = normalizeOptionalString(att.asr_refer_text) ?? "";
+      const wavUrl =
+        isVoice && att.voice_wav_url
+          ? att.voice_wav_url.startsWith("//")
+            ? `https:${att.voice_wav_url}`
+            : att.voice_wav_url
+          : "";
+      const voiceSourceUrl = wavUrl || attUrl;
+
+      const meta = {
+        voiceUrl: isVoice && voiceSourceUrl ? voiceSourceUrl : undefined,
+        asrReferText: isVoice && asrReferText ? asrReferText : undefined,
+      };
+
+      if (localPath) {
+        if (att.content_type?.startsWith("image/")) {
+          log?.debug?.(`Downloaded attachment to: ${localPath}`);
+          return { localPath, type: "image" as const, contentType: att.content_type, meta };
+        }
+        if (isVoice) {
+          log?.debug?.(`Downloaded attachment to: ${localPath}`);
+          return processVoiceAttachment(
+            localPath,
+            audioPath,
+            att,
+            asrReferText,
+            cfg,
+            downloadDir,
+            audioConvert,
+            log,
+          );
+        }
+        log?.debug?.(`Downloaded attachment to: ${localPath}`);
+        return { localPath, type: "other" as const, filename: att.filename, meta };
+      }
+      log?.error(`Failed to download: ${attUrl}`);
+      if (att.content_type?.startsWith("image/")) {
+        return {
+          localPath: null,
+          type: "image-fallback" as const,
+          attUrl,
+          contentType: att.content_type,
+          meta,
+        };
+      }
+      if (isVoice && asrReferText) {
+        log?.info(`Voice attachment download failed, using asr_refer_text fallback`);
+        return {
+          localPath: null,
+          type: "voice-fallback" as const,
+          transcript: asrReferText,
+          meta,
+        };
+      }
+      return {
+        localPath: null,
+        type: "other-fallback" as const,
+        filename: att.filename ?? att.content_type,
+        meta,
+      };
+    },
+  );
+
+  const processResults = await Promise.all(processTasks);
+
+  // Phase 3: collect results in the original attachment order.
+  for (const result of processResults) {
+    if (result.meta.voiceUrl) {
+      voiceAttachmentUrls.push(result.meta.voiceUrl);
+    }
+    if (result.meta.asrReferText) {
+      voiceAsrReferTexts.push(result.meta.asrReferText);
+    }
+
+    if (result.type === "image" && result.localPath) {
+      imageUrls.push(result.localPath);
+      imageMediaTypes.push(result.contentType);
+      attachmentLocalPaths.push(result.localPath);
+    } else if (result.type === "voice" && result.localPath) {
+      voiceAttachmentPaths.push(result.localPath);
+      voiceTranscripts.push(result.transcript);
+      voiceTranscriptSources.push(result.transcriptSource);
+      attachmentLocalPaths.push(result.localPath);
+    } else if (result.type === "other" && result.localPath) {
+      otherAttachments.push(`[Attachment: ${result.localPath}]`);
+      attachmentLocalPaths.push(result.localPath);
+    } else if (result.type === "image-fallback") {
+      imageUrls.push(result.attUrl);
+      imageMediaTypes.push(result.contentType);
+      attachmentLocalPaths.push(null);
+    } else if (result.type === "voice-fallback") {
+      voiceTranscripts.push(result.transcript);
+      voiceTranscriptSources.push("asr");
+      attachmentLocalPaths.push(null);
+    } else if (result.type === "other-fallback") {
+      otherAttachments.push(`[Attachment: ${result.filename}] (download failed)`);
+      attachmentLocalPaths.push(null);
+    }
+  }
+
+  const attachmentInfo = otherAttachments.length > 0 ? "\n" + otherAttachments.join("\n") : "";
+
+  return {
+    attachmentInfo,
+    imageUrls,
+    imageMediaTypes,
+    voiceAttachmentPaths,
+    voiceAttachmentUrls,
+    voiceAsrReferTexts,
+    voiceTranscripts,
+    voiceTranscriptSources,
+    attachmentLocalPaths,
+  };
+}
+
+// formatVoiceText is now in core/utils/voice-text.ts (re-exported above).
+
+// Internal helpers.
+
+type VoiceResult =
+  | {
+      localPath: string;
+      type: "voice";
+      transcript: string;
+      transcriptSource: TranscriptSource;
+      meta: { voiceUrl?: string; asrReferText?: string };
+    }
+  | {
+      localPath: string;
+      type: "voice";
+      transcript: string;
+      transcriptSource: TranscriptSource;
+      meta: { voiceUrl?: string; asrReferText?: string };
+    };
+
+async function processVoiceAttachment(
+  localPath: string,
+  audioPath: string | null,
+  att: RawAttachment,
+  asrReferText: string,
+  cfg: unknown,
+  downloadDir: string,
+  audioConvert: AudioConvertPort,
+  log: ProcessContext["log"],
+): Promise<VoiceResult> {
+  const wavUrl = att.voice_wav_url
+    ? att.voice_wav_url.startsWith("//")
+      ? `https:${att.voice_wav_url}`
+      : att.voice_wav_url
+    : "";
+  const attUrl = att.url?.startsWith("//") ? `https:${att.url}` : att.url;
+  const voiceSourceUrl = wavUrl || attUrl;
+  const meta = {
+    voiceUrl: voiceSourceUrl || undefined,
+    asrReferText: asrReferText || undefined,
+  };
+
+  const sttCfg = resolveSTTConfig(cfg as Record<string, unknown>);
+  if (!sttCfg) {
+    if (asrReferText) {
+      log?.debug?.(
+        `Voice attachment: ${att.filename} (STT not configured, using asr_refer_text fallback)`,
+      );
+      return { localPath, type: "voice", transcript: asrReferText, transcriptSource: "asr", meta };
+    }
+    log?.debug?.(`Voice attachment: ${att.filename} (STT not configured, skipping transcription)`);
+    return {
+      localPath,
+      type: "voice",
+      transcript: "[Voice message - transcription unavailable because STT is not configured]",
+      transcriptSource: "fallback",
+      meta,
+    };
+  }
+
+  // Convert SILK input to WAV before STT when necessary.
+  if (!audioPath) {
+    log?.debug?.(`Voice attachment: ${att.filename}, converting SILK→WAV...`);
+    try {
+      const wavResult = await audioConvert.convertSilkToWav(localPath, downloadDir);
+      if (wavResult) {
+        audioPath = wavResult.wavPath;
+        log?.debug?.(
+          `Voice converted: ${wavResult.wavPath} (${audioConvert.formatDuration(wavResult.duration)})`,
+        );
+      } else {
+        audioPath = localPath;
+      }
+    } catch (convertErr) {
+      log?.error(
+        `Voice conversion failed: ${
+          convertErr instanceof Error ? convertErr.message : JSON.stringify(convertErr)
+        }`,
+      );
+      if (asrReferText) {
+        return {
+          localPath,
+          type: "voice",
+          transcript: asrReferText,
+          transcriptSource: "asr",
+          meta,
+        };
+      }
+      return {
+        localPath,
+        type: "voice",
+        transcript: "[Voice message - format conversion failed]",
+        transcriptSource: "fallback",
+        meta,
+      };
+    }
+  }
+
+  // Run speech-to-text on the prepared audio file.
+  try {
+    const transcript = await transcribeAudio(audioPath, cfg as Record<string, unknown>);
+    if (transcript) {
+      log?.debug?.(`STT transcript: ${transcript.slice(0, 100)}...`);
+      return { localPath, type: "voice", transcript, transcriptSource: "stt", meta };
+    }
+    if (asrReferText) {
+      log?.debug?.(`STT returned empty result, using asr_refer_text fallback`);
+      return { localPath, type: "voice", transcript: asrReferText, transcriptSource: "asr", meta };
+    }
+    log?.debug?.(`STT returned empty result`);
+    return {
+      localPath,
+      type: "voice",
+      transcript: "[Voice message - transcription returned an empty result]",
+      transcriptSource: "fallback",
+      meta,
+    };
+  } catch (sttErr) {
+    log?.error(`STT failed: ${sttErr instanceof Error ? sttErr.message : JSON.stringify(sttErr)}`);
+    if (asrReferText) {
+      return { localPath, type: "voice", transcript: asrReferText, transcriptSource: "asr", meta };
+    }
+    return {
+      localPath,
+      type: "voice",
+      transcript: "[Voice message - transcription failed]",
+      transcriptSource: "fallback",
+      meta,
+    };
+  }
+}

package/src/engine/gateway/inbound-context.ts

@@ -0,0 +1,82 @@
+import type { ChannelIngressDecision } from "autobot/plugin-sdk/channel-ingress-runtime";
+import type { EngineAdapters } from "../adapter/index.js";
+import type { GroupActivationMode, SessionStoreReader } from "../group/activation.js";
+import type { HistoryEntry } from "../group/history.js";
+import type { GroupMessageGateResult } from "../group/message-gating.js";
+import type { QueuedMessage } from "./message-queue.js";
+import type { GatewayAccount, EngineLogger, GatewayPluginRuntime } from "./types.js";
+import type { TypingKeepAlive } from "./typing-keepalive.js";
+
+export interface ReplyToInfo {
+  id: string;
+  body?: string;
+  sender?: string;
+  isQuote: boolean;
+}
+
+export interface InboundGroupInfo {
+  gate: GroupMessageGateResult;
+  activation: GroupActivationMode;
+  historyLimit: number;
+  isMerged: boolean;
+  mergedMessages?: readonly QueuedMessage[];
+  display: {
+    groupName: string;
+    senderLabel: string;
+    introHint?: string;
+    behaviorPrompt?: string;
+  };
+}
+
+export interface InboundContext {
+  event: QueuedMessage;
+  route: { sessionKey: string; accountId: string; agentId?: string };
+  isGroupChat: boolean;
+  peerId: string;
+  qualifiedTarget: string;
+  fromAddress: string;
+  agentBody: string;
+  body: string;
+  groupSystemPrompt?: string;
+  localMediaPaths: string[];
+  localMediaTypes: string[];
+  remoteMediaUrls: string[];
+  uniqueVoicePaths: string[];
+  uniqueVoiceUrls: string[];
+  uniqueVoiceAsrReferTexts: string[];
+  voiceMediaTypes: string[];
+  hasAsrReferFallback: boolean;
+  voiceTranscriptSources: string[];
+  replyTo?: ReplyToInfo;
+  commandAuthorized: boolean;
+  group?: InboundGroupInfo;
+  blocked: boolean;
+  blockReason?: string;
+  blockReasonCode?: string;
+  accessDecision?: ChannelIngressDecision["decision"];
+  skipped: boolean;
+  skipReason?: "drop_other_mention" | "block_unauthorized_command" | "skip_no_mention";
+  typing: { keepAlive: TypingKeepAlive | null };
+  inputNotifyRefIdx?: string;
+}
+
+export interface InboundPipelineDeps {
+  account: GatewayAccount;
+  cfg: unknown;
+  log?: EngineLogger;
+  runtime: GatewayPluginRuntime;
+  startTyping: (event: QueuedMessage) => Promise<{
+    refIdx?: string;
+    keepAlive: TypingKeepAlive | null;
+  }>;
+  groupHistories?: Map<string, HistoryEntry[]>;
+  sessionStoreReader?: SessionStoreReader;
+  allowTextCommands?: boolean;
+  isControlCommand?: (content: string) => boolean;
+  resolveGroupIntroHint?: (params: {
+    cfg: unknown;
+    accountId: string;
+    groupId: string;
+  }) => string | undefined;
+  adapters: EngineAdapters;
+}

package/src/engine/gateway/inbound-pipeline.ts

@@ -0,0 +1,171 @@
+import type { HistoryPort } from "../adapter/history.port.js";
+import type { HistoryEntry } from "../group/history.js";
+import { processAttachments } from "./inbound-attachments.js";
+import type { InboundContext, InboundPipelineDeps } from "./inbound-context.js";
+import type { QueuedMessage } from "./message-queue.js";
+import {
+  buildAgentBody,
+  buildBody,
+  buildDynamicCtx,
+  buildGroupSystemPrompt,
+  buildQuotePart,
+  buildSkippedInboundContext,
+  buildUserContent,
+  buildUserMessage,
+  classifyMedia,
+  resolveQuote,
+  runAccessStage,
+  runGroupGateStage,
+  writeRefIndex,
+} from "./stages/index.js";
+
+export async function buildInboundContext(
+  event: QueuedMessage,
+  deps: InboundPipelineDeps,
+): Promise<InboundContext> {
+  const { account, log } = deps;
+
+  const accessResult = await runAccessStage(event, deps);
+  if (accessResult.kind === "block") {
+    return accessResult.context;
+  }
+  const { isGroupChat, peerId, qualifiedTarget, fromAddress, route, access } = accessResult;
+
+  const typingPromise = deps.startTyping(event);
+
+  const processed = await processAttachments(event.attachments, {
+    accountId: account.accountId,
+    cfg: deps.cfg,
+    audioConvert: deps.adapters.audioConvert,
+    log,
+  });
+
+  const { parsedContent, userContent } = buildUserContent({
+    event,
+    attachmentInfo: processed.attachmentInfo,
+    voiceTranscripts: processed.voiceTranscripts,
+  });
+
+  const replyTo = await resolveQuote(event, deps);
+
+  const typingResult = await typingPromise;
+  writeRefIndex({
+    event,
+    parsedContent,
+    processed,
+    inputNotifyRefIdx: typingResult.refIdx,
+  });
+
+  let groupInfo: InboundContext["group"];
+  if (event.type === "group" && event.groupOpenid) {
+    const gateOutcome = runGroupGateStage({
+      event,
+      deps,
+      accountId: account.accountId,
+      agentId: route.agentId,
+      sessionKey: route.sessionKey,
+      userContent,
+      processedAttachments: processed,
+      access,
+    });
+
+    if (gateOutcome.kind === "skip") {
+      typingResult.keepAlive?.stop();
+      return buildSkippedInboundContext({
+        event,
+        route,
+        isGroupChat: true,
+        peerId,
+        qualifiedTarget,
+        fromAddress,
+        group: gateOutcome.groupInfo,
+        skipReason: gateOutcome.skipReason,
+        access,
+        typing: { keepAlive: typingResult.keepAlive },
+        inputNotifyRefIdx: typingResult.refIdx,
+      });
+    }
+    groupInfo = gateOutcome.groupInfo;
+  }
+
+  const body = buildBody({
+    event,
+    deps,
+    userContent,
+    isGroupChat,
+    imageUrls: processed.imageUrls,
+  });
+  const quotePart = buildQuotePart(replyTo);
+  const media = classifyMedia(processed);
+  const dynamicCtx = buildDynamicCtx({
+    imageUrls: processed.imageUrls,
+    uniqueVoicePaths: media.uniqueVoicePaths,
+    uniqueVoiceUrls: media.uniqueVoiceUrls,
+    uniqueVoiceAsrReferTexts: media.uniqueVoiceAsrReferTexts,
+  });
+
+  const userMessage = buildUserMessage({
+    event,
+    userContent,
+    quotePart,
+    isGroupChat,
+    groupInfo,
+  });
+  const agentBody = buildAgentBody({
+    event,
+    userContent,
+    userMessage,
+    dynamicCtx,
+    isGroupChat,
+    groupInfo,
+    deps,
+  });
+
+  const accountSystemInstruction = account.systemPrompt ?? "";
+  const groupSystemPrompt = buildGroupSystemPrompt(accountSystemInstruction, groupInfo);
+
+  return {
+    event,
+    route,
+    isGroupChat,
+    peerId,
+    qualifiedTarget,
+    fromAddress,
+    agentBody,
+    body,
+    groupSystemPrompt,
+    localMediaPaths: media.localMediaPaths,
+    localMediaTypes: media.localMediaTypes,
+    remoteMediaUrls: media.remoteMediaUrls,
+    uniqueVoicePaths: media.uniqueVoicePaths,
+    uniqueVoiceUrls: media.uniqueVoiceUrls,
+    uniqueVoiceAsrReferTexts: media.uniqueVoiceAsrReferTexts,
+    voiceMediaTypes: media.voiceMediaTypes,
+    hasAsrReferFallback: media.hasAsrReferFallback,
+    voiceTranscriptSources: media.voiceTranscriptSources,
+    replyTo,
+    commandAuthorized: access.commandAccess.authorized,
+    group: groupInfo,
+    blocked: false,
+    skipped: false,
+    accessDecision: access.senderAccess.decision,
+    typing: { keepAlive: typingResult.keepAlive },
+    inputNotifyRefIdx: typingResult.refIdx,
+  };
+}
+
+export function clearGroupPendingHistory(params: {
+  historyMap: Map<string, HistoryEntry[]> | undefined;
+  groupOpenid: string | undefined;
+  historyLimit: number;
+  historyPort: HistoryPort;
+}): void {
+  if (!params.historyMap || !params.groupOpenid) {
+    return;
+  }
+  params.historyPort.clearPendingHistory({
+    historyMap: params.historyMap,
+    historyKey: params.groupOpenid,
+    limit: params.historyLimit,
+  });
+}