@openclaw/qqbot 2026.5.2-beta.2 → 2026.5.3-beta.1

package/src/engine/gateway/inbound-attachments.ts

@@ -1,360 +0,0 @@
-import type { AudioConvertPort } from "../adapter/audio.port.js";
-import { downloadFile } from "../utils/file-utils.js";
-import { getQQBotMediaDir } from "../utils/platform.js";
-import { normalizeOptionalString } from "../utils/string-normalize.js";
-import { transcribeAudio, resolveSTTConfig } from "../utils/stt.js";
-
-// Re-export the port type for convenience.
-export type { AudioConvertPort } from "../adapter/audio.port.js";
-
-interface RawAttachment {
-  content_type: string;
-  url: string;
-  filename?: string;
-  voice_wav_url?: string;
-  asr_refer_text?: string;
-}
-
-type TranscriptSource = "stt" | "asr" | "fallback";
-
-/** Normalized attachment output consumed by the gateway. */
-export interface ProcessedAttachments {
-  attachmentInfo: string;
-  imageUrls: string[];
-  imageMediaTypes: string[];
-  voiceAttachmentPaths: string[];
-  voiceAttachmentUrls: string[];
-  voiceAsrReferTexts: string[];
-  voiceTranscripts: string[];
-  voiceTranscriptSources: TranscriptSource[];
-  attachmentLocalPaths: Array<string | null>;
-}
-
-interface ProcessContext {
-  accountId: string;
-  cfg: unknown;
-  audioConvert: AudioConvertPort;
-  log?: {
-    info: (msg: string) => void;
-    error: (msg: string) => void;
-    debug?: (msg: string) => void;
-  };
-}
-
-const EMPTY_RESULT: ProcessedAttachments = {
-  attachmentInfo: "",
-  imageUrls: [],
-  imageMediaTypes: [],
-  voiceAttachmentPaths: [],
-  voiceAttachmentUrls: [],
-  voiceAsrReferTexts: [],
-  voiceTranscripts: [],
-  voiceTranscriptSources: [],
-  attachmentLocalPaths: [],
-};
-
-/** Download, convert, transcribe, and classify inbound attachments. */
-export async function processAttachments(
-  attachments: RawAttachment[] | undefined,
-  ctx: ProcessContext,
-): Promise<ProcessedAttachments> {
-  if (!attachments?.length) {
-    return EMPTY_RESULT;
-  }
-
-  const { accountId: _accountId, cfg, log, audioConvert } = ctx;
-  const downloadDir = getQQBotMediaDir("downloads");
-
-  const imageUrls: string[] = [];
-  const imageMediaTypes: string[] = [];
-  const voiceAttachmentPaths: string[] = [];
-  const voiceAttachmentUrls: string[] = [];
-  const voiceAsrReferTexts: string[] = [];
-  const voiceTranscripts: string[] = [];
-  const voiceTranscriptSources: TranscriptSource[] = [];
-  const attachmentLocalPaths: Array<string | null> = [];
-  const otherAttachments: string[] = [];
-
-  // Phase 1: download all attachments in parallel.
-  const downloadTasks = attachments.map(async (att) => {
-    const attUrl = att.url?.startsWith("//") ? `https:${att.url}` : att.url;
-    const isVoice = audioConvert.isVoiceAttachment(att);
-    const wavUrl =
-      isVoice && att.voice_wav_url
-        ? att.voice_wav_url.startsWith("//")
-          ? `https:${att.voice_wav_url}`
-          : att.voice_wav_url
-        : "";
-
-    let localPath: string | null = null;
-    let audioPath: string | null = null;
-
-    if (isVoice && wavUrl) {
-      const wavLocalPath = await downloadFile(wavUrl, downloadDir);
-      if (wavLocalPath) {
-        localPath = wavLocalPath;
-        audioPath = wavLocalPath;
-        log?.debug?.(`Voice attachment: ${att.filename}, downloaded WAV directly (skip SILK→WAV)`);
-      } else {
-        log?.error(`Failed to download voice_wav_url, falling back to original URL`);
-      }
-    }
-
-    if (!localPath) {
-      localPath = await downloadFile(attUrl, downloadDir, att.filename);
-    }
-
-    return { att, attUrl, isVoice, localPath, audioPath };
-  });
-
-  const downloadResults = await Promise.all(downloadTasks);
-
-  // Phase 2: convert/transcribe voice attachments and classify everything else.
-  const processTasks = downloadResults.map(
-    async ({ att, attUrl, isVoice, localPath, audioPath }) => {
-      const asrReferText = normalizeOptionalString(att.asr_refer_text) ?? "";
-      const wavUrl =
-        isVoice && att.voice_wav_url
-          ? att.voice_wav_url.startsWith("//")
-            ? `https:${att.voice_wav_url}`
-            : att.voice_wav_url
-          : "";
-      const voiceSourceUrl = wavUrl || attUrl;
-
-      const meta = {
-        voiceUrl: isVoice && voiceSourceUrl ? voiceSourceUrl : undefined,
-        asrReferText: isVoice && asrReferText ? asrReferText : undefined,
-      };
-
-      if (localPath) {
-        if (att.content_type?.startsWith("image/")) {
-          log?.debug?.(`Downloaded attachment to: ${localPath}`);
-          return { localPath, type: "image" as const, contentType: att.content_type, meta };
-        }
-        if (isVoice) {
-          log?.debug?.(`Downloaded attachment to: ${localPath}`);
-          return processVoiceAttachment(
-            localPath,
-            audioPath,
-            att,
-            asrReferText,
-            cfg,
-            downloadDir,
-            audioConvert,
-            log,
-          );
-        }
-        log?.debug?.(`Downloaded attachment to: ${localPath}`);
-        return { localPath, type: "other" as const, filename: att.filename, meta };
-      }
-      log?.error(`Failed to download: ${attUrl}`);
-      if (att.content_type?.startsWith("image/")) {
-        return {
-          localPath: null,
-          type: "image-fallback" as const,
-          attUrl,
-          contentType: att.content_type,
-          meta,
-        };
-      }
-      if (isVoice && asrReferText) {
-        log?.info(`Voice attachment download failed, using asr_refer_text fallback`);
-        return {
-          localPath: null,
-          type: "voice-fallback" as const,
-          transcript: asrReferText,
-          meta,
-        };
-      }
-      return {
-        localPath: null,
-        type: "other-fallback" as const,
-        filename: att.filename ?? att.content_type,
-        meta,
-      };
-    },
-  );
-
-  const processResults = await Promise.all(processTasks);
-
-  // Phase 3: collect results in the original attachment order.
-  for (const result of processResults) {
-    if (result.meta.voiceUrl) {
-      voiceAttachmentUrls.push(result.meta.voiceUrl);
-    }
-    if (result.meta.asrReferText) {
-      voiceAsrReferTexts.push(result.meta.asrReferText);
-    }
-
-    if (result.type === "image" && result.localPath) {
-      imageUrls.push(result.localPath);
-      imageMediaTypes.push(result.contentType);
-      attachmentLocalPaths.push(result.localPath);
-    } else if (result.type === "voice" && result.localPath) {
-      voiceAttachmentPaths.push(result.localPath);
-      voiceTranscripts.push(result.transcript);
-      voiceTranscriptSources.push(result.transcriptSource);
-      attachmentLocalPaths.push(result.localPath);
-    } else if (result.type === "other" && result.localPath) {
-      otherAttachments.push(`[Attachment: ${result.localPath}]`);
-      attachmentLocalPaths.push(result.localPath);
-    } else if (result.type === "image-fallback") {
-      imageUrls.push(result.attUrl);
-      imageMediaTypes.push(result.contentType);
-      attachmentLocalPaths.push(null);
-    } else if (result.type === "voice-fallback") {
-      voiceTranscripts.push(result.transcript);
-      voiceTranscriptSources.push("asr");
-      attachmentLocalPaths.push(null);
-    } else if (result.type === "other-fallback") {
-      otherAttachments.push(`[Attachment: ${result.filename}] (download failed)`);
-      attachmentLocalPaths.push(null);
-    }
-  }
-
-  const attachmentInfo = otherAttachments.length > 0 ? "\n" + otherAttachments.join("\n") : "";
-
-  return {
-    attachmentInfo,
-    imageUrls,
-    imageMediaTypes,
-    voiceAttachmentPaths,
-    voiceAttachmentUrls,
-    voiceAsrReferTexts,
-    voiceTranscripts,
-    voiceTranscriptSources,
-    attachmentLocalPaths,
-  };
-}
-
-// formatVoiceText is now in core/utils/voice-text.ts (re-exported above).
-
-// Internal helpers.
-
-type VoiceResult =
-  | {
-      localPath: string;
-      type: "voice";
-      transcript: string;
-      transcriptSource: TranscriptSource;
-      meta: { voiceUrl?: string; asrReferText?: string };
-    }
-  | {
-      localPath: string;
-      type: "voice";
-      transcript: string;
-      transcriptSource: TranscriptSource;
-      meta: { voiceUrl?: string; asrReferText?: string };
-    };
-
-async function processVoiceAttachment(
-  localPath: string,
-  audioPath: string | null,
-  att: RawAttachment,
-  asrReferText: string,
-  cfg: unknown,
-  downloadDir: string,
-  audioConvert: AudioConvertPort,
-  log: ProcessContext["log"],
-): Promise<VoiceResult> {
-  const wavUrl = att.voice_wav_url
-    ? att.voice_wav_url.startsWith("//")
-      ? `https:${att.voice_wav_url}`
-      : att.voice_wav_url
-    : "";
-  const attUrl = att.url?.startsWith("//") ? `https:${att.url}` : att.url;
-  const voiceSourceUrl = wavUrl || attUrl;
-  const meta = {
-    voiceUrl: voiceSourceUrl || undefined,
-    asrReferText: asrReferText || undefined,
-  };
-
-  const sttCfg = resolveSTTConfig(cfg as Record<string, unknown>);
-  if (!sttCfg) {
-    if (asrReferText) {
-      log?.debug?.(
-        `Voice attachment: ${att.filename} (STT not configured, using asr_refer_text fallback)`,
-      );
-      return { localPath, type: "voice", transcript: asrReferText, transcriptSource: "asr", meta };
-    }
-    log?.debug?.(`Voice attachment: ${att.filename} (STT not configured, skipping transcription)`);
-    return {
-      localPath,
-      type: "voice",
-      transcript: "[Voice message - transcription unavailable because STT is not configured]",
-      transcriptSource: "fallback",
-      meta,
-    };
-  }
-
-  // Convert SILK input to WAV before STT when necessary.
-  if (!audioPath) {
-    log?.debug?.(`Voice attachment: ${att.filename}, converting SILK→WAV...`);
-    try {
-      const wavResult = await audioConvert.convertSilkToWav(localPath, downloadDir);
-      if (wavResult) {
-        audioPath = wavResult.wavPath;
-        log?.debug?.(
-          `Voice converted: ${wavResult.wavPath} (${audioConvert.formatDuration(wavResult.duration)})`,
-        );
-      } else {
-        audioPath = localPath;
-      }
-    } catch (convertErr) {
-      log?.error(
-        `Voice conversion failed: ${
-          convertErr instanceof Error ? convertErr.message : JSON.stringify(convertErr)
-        }`,
-      );
-      if (asrReferText) {
-        return {
-          localPath,
-          type: "voice",
-          transcript: asrReferText,
-          transcriptSource: "asr",
-          meta,
-        };
-      }
-      return {
-        localPath,
-        type: "voice",
-        transcript: "[Voice message - format conversion failed]",
-        transcriptSource: "fallback",
-        meta,
-      };
-    }
-  }
-
-  // Run speech-to-text on the prepared audio file.
-  try {
-    const transcript = await transcribeAudio(audioPath, cfg as Record<string, unknown>);
-    if (transcript) {
-      log?.debug?.(`STT transcript: ${transcript.slice(0, 100)}...`);
-      return { localPath, type: "voice", transcript, transcriptSource: "stt", meta };
-    }
-    if (asrReferText) {
-      log?.debug?.(`STT returned empty result, using asr_refer_text fallback`);
-      return { localPath, type: "voice", transcript: asrReferText, transcriptSource: "asr", meta };
-    }
-    log?.debug?.(`STT returned empty result`);
-    return {
-      localPath,
-      type: "voice",
-      transcript: "[Voice message - transcription returned an empty result]",
-      transcriptSource: "fallback",
-      meta,
-    };
-  } catch (sttErr) {
-    log?.error(`STT failed: ${sttErr instanceof Error ? sttErr.message : JSON.stringify(sttErr)}`);
-    if (asrReferText) {
-      return { localPath, type: "voice", transcript: asrReferText, transcriptSource: "asr", meta };
-    }
-    return {
-      localPath,
-      type: "voice",
-      transcript: "[Voice message - transcription failed]",
-      transcriptSource: "fallback",
-      meta,
-    };
-  }
-}

package/src/engine/gateway/inbound-context.ts

@@ -1,195 +0,0 @@
-/**
- * InboundContext — the structured result of the inbound pipeline.
- *
- * Connects the inbound stage (content building, attachment processing,
- * quote resolution) with the outbound stage (AI dispatch, deliver callbacks).
- *
- * All fields are readonly after construction. The outbound dispatcher
- * reads from this object but never mutates it.
- */
-
-import type { QQBotAccessDecision, QQBotAccessReasonCode } from "../access/index.js";
-import type { EngineAdapters } from "../adapter/index.js";
-import type { GroupActivationMode, SessionStoreReader } from "../group/activation.js";
-import type { HistoryEntry } from "../group/history.js";
-import type { GroupMessageGateResult } from "../group/message-gating.js";
-import type { QueuedMessage } from "./message-queue.js";
-import type {
-  GatewayAccount,
-  EngineLogger,
-  GatewayPluginRuntime,
-  ProcessedAttachments,
-} from "./types.js";
-import type { TypingKeepAlive } from "./typing-keepalive.js";
-
-// ============ InboundContext ============
-
-/** Quote (reply-to) metadata resolved during inbound processing. */
-export interface ReplyToInfo {
-  id: string;
-  body?: string;
-  sender?: string;
-  isQuote: boolean;
-}
-
-/**
- * Group-specific inbound metadata.
- *
- * Populated for group / guild events; left `undefined` for DMs. Keeping
- * the group fields under a nested bag makes it obvious which fields are
- * safe to read only when `isGroupChat === true`.
- *
- * The shape is kept small on purpose: everything derivable from `gate`
- * (raw wasMentioned / explicit / implicit / hasAnyMention / bypass) is
- * stored once on `gate`, not duplicated on the outer object.
- */
-export interface InboundGroupInfo {
-  // ---- Gating decision ----
-  /** Full gate evaluation result (source of truth for mention state). */
-  gate: GroupMessageGateResult;
-  /** Effective activation mode after session-store / cfg merge. */
-  activation: GroupActivationMode;
-
-  // ---- Persistence-relevant ----
-  /** Per-group history buffer cap. Zero → disabled. */
-  historyLimit: number;
-  /** `true` if this message was built by merging several queued entries. */
-  isMerged: boolean;
-  /** The unfiltered list of queued messages when `isMerged`, else undefined. */
-  mergedMessages?: readonly QueuedMessage[];
-
-  // ---- Presentation / prompt inputs ----
-  /** Bundle of display-only strings; assembled by the envelope stage. */
-  display: {
-    /** Human-readable group name ("My Group" / first 8 chars of openid). */
-    groupName: string;
-    /** Sender label ("Nick (OPENID)" / "OPENID") for the UI. */
-    senderLabel: string;
-    /** Channel-level intro hint contributed by the platform adapter. */
-    introHint?: string;
-    /** Per-group behaviour prompt appended to the system prompt. */
-    behaviorPrompt?: string;
-  };
-}
-
-/** Fully resolved inbound context passed to the outbound dispatcher. */
-export interface InboundContext {
-  // ---- Original event ----
-  event: QueuedMessage;
-
-  // ---- Routing ----
-  route: { sessionKey: string; accountId: string; agentId?: string };
-  isGroupChat: boolean;
-  peerId: string;
-  /** Fully qualified target address: "qqbot:c2c:xxx" / "qqbot:group:xxx" etc. */
-  qualifiedTarget: string;
-  fromAddress: string;
-
-  // ---- Content ----
-  /** event.content after parseFaceTags. */
-  parsedContent: string;
-  /** parsedContent + voiceText + attachmentInfo — the user-visible text. */
-  userContent: string;
-  /** "[Quoted message begins]…[ends]" or empty. */
-  quotePart: string;
-  /** Per-message dynamic metadata lines (images, voice, ASR). */
-  dynamicCtx: string;
-  /** quotePart + userContent. */
-  userMessage: string;
-  /** dynamicCtx + userMessage (or raw content for slash commands). */
-  agentBody: string;
-  /** Formatted inbound envelope (Web UI body). */
-  body: string;
-
-  // ---- System prompts ----
-  systemPrompts: string[];
-  groupSystemPrompt?: string;
-
-  // ---- Attachments ----
-  attachments: ProcessedAttachments;
-  localMediaPaths: string[];
-  localMediaTypes: string[];
-  remoteMediaUrls: string[];
-  remoteMediaTypes: string[];
-
-  // ---- Voice ----
-  uniqueVoicePaths: string[];
-  uniqueVoiceUrls: string[];
-  uniqueVoiceAsrReferTexts: string[];
-  voiceMediaTypes: string[];
-  hasAsrReferFallback: boolean;
-  voiceTranscriptSources: string[];
-
-  // ---- Reply-to / Quote ----
-  replyTo?: ReplyToInfo;
-
-  // ---- Auth ----
-  commandAuthorized: boolean;
-
-  // ---- Group ----
-  /** Populated only for group / guild messages. */
-  group?: InboundGroupInfo;
-
-  // ---- Blocking / skipping ----
-  /**
-   * Whether the inbound message should be blocked outright (access policy
-   * refused the sender). Mutually exclusive with `skipped`.
-   */
-  blocked: boolean;
-  /** Human-readable reason for `blocked`, for logging only. */
-  blockReason?: string;
-  /** Structured reason code for `blocked`. */
-  blockReasonCode?: QQBotAccessReasonCode;
-  /** The raw access decision produced by the policy engine. */
-  accessDecision?: QQBotAccessDecision;
-  /**
-   * Whether the inbound was accepted by access control but stopped before
-   * AI dispatch by the group gate (e.g. "skip_no_mention"). The caller
-   * should NOT forward `skipped` messages to the outbound dispatcher, but
-   * history / activity side-effects may already have been applied.
-   */
-  skipped: boolean;
-  /** Structured reason code for `skipped`. */
-  skipReason?: "drop_other_mention" | "block_unauthorized_command" | "skip_no_mention";
-
-  // ---- Typing ----
-  typing: { keepAlive: TypingKeepAlive | null };
-  /** refIdx returned by the initial InputNotify call. */
-  inputNotifyRefIdx?: string;
-}
-
-// ============ Pipeline dependencies ============
-
-/** Dependencies injected into the inbound pipeline. */
-export interface InboundPipelineDeps {
-  account: GatewayAccount;
-  cfg: unknown;
-  log?: EngineLogger;
-  runtime: GatewayPluginRuntime;
-  /** Start typing indicator and return the refIdx from InputNotify. */
-  startTyping: (event: QueuedMessage) => Promise<{
-    refIdx?: string;
-    keepAlive: TypingKeepAlive | null;
-  }>;
-  // ---- Group dependencies (optional — omit when the caller doesn't need
-  // group support, e.g. a DM-only test harness). ----
-  /** Shared per-connection history buffer, created by the gateway. */
-  groupHistories?: Map<string, HistoryEntry[]>;
-  /** Session-store reader for activation-mode overrides. */
-  sessionStoreReader?: SessionStoreReader;
-  /** Whether text-based control commands are enabled globally. */
-  allowTextCommands?: boolean;
-  /**
-   * Framework probe that returns true when `content` is a known control
-   * command. Injected to avoid hard-coding a list of commands in engine.
-   */
-  isControlCommand?: (content: string) => boolean;
-  /** Optional platform hook that contributes a channel-level intro hint. */
-  resolveGroupIntroHint?: (params: {
-    cfg: unknown;
-    accountId: string;
-    groupId: string;
-  }) => string | undefined;
-  /** SDK adapter ports for delegating to shared implementations. */
-  adapters: EngineAdapters;
-}