mioku-plugin-chat 2.3.0 → 2.4.0

package/core/media/history-media.ts

@@ -12,14 +12,51 @@ const execFileAsync = promisify(execFile);
 const VIDEO_FRAME_COUNT = 5;
 const VIDEO_FRAME_EXTRACTION_FALLBACK =
   "用户发送了一个视频，但未能提取画面内容";
+// 体积阈值：超过此大小（10MB）的视频不再整体上传给多模态工作模型，改用抽帧。
+export const VIDEO_FULL_UPLOAD_MAX_BYTES = 10 * 1024 * 1024;
 
 interface SummaryResult {
   summary: string;
 }
 
+export interface MediaMessageSegment {
+  type?: string;
+  file?: string;
+  path?: string;
+  url?: string;
+  id?: string;
+  data?: {
+    file?: string;
+    path?: string;
+    url?: string;
+    id?: string;
+    data?: string;
+    xml?: string;
+  };
+}
+
+export function getSegmentSourceCandidates(seg: MediaMessageSegment): string[] {
+  return Array.from(
+    new Set(
+      [
+        seg?.file,
+        seg?.data?.file,
+        seg?.path,
+        seg?.data?.path,
+        seg?.url,
+        seg?.data?.url,
+      ]
+        .map((v) => (typeof v === "string" ? v.trim() : ""))
+        .filter(Boolean),
+    ),
+  );
+}
+
 export interface MediaSummaryStore {
   getMediaSummary(key: string): MediaSummaryRecord | null;
   saveMediaSummary(summary: MediaSummaryRecord): void;
+  getMediaSummaryBySource?(sourceKey: string): MediaSummaryRecord | null;
+  saveMediaSummarySource?(sourceKey: string, summaryKey: string): void;
 }
 
 export interface HistoryMediaProcessingOptions {
@@ -71,53 +108,8 @@ export async function summarizeHistoryVideo(
       videoFile.source,
       videoFile.contentHash,
       options,
-      async () => {
-        const frames = await extractVideoFrames(
-          videoFile.path,
-          VIDEO_FRAME_COUNT,
-          options,
-        );
-        if (frames.length === 0) {
-          getHistoryMediaLogger(options).warn(
-            "[history-media] Video frame extraction returned 0 frames",
-          );
-          return VIDEO_FRAME_EXTRACTION_FALLBACK;
-        }
-
-        const content: any[] = [
-          {
-            type: "text",
-            text: `These ${frames.length} frames were sampled evenly from a chat video. Summarize the video's likely content in Chinese for chat history context. Mention visible people/objects/actions/text, and keep it concise.`,
-          },
-          ...frames.map((frame) => ({
-            type: "image_url",
-            image_url: { url: frame, detail: "auto" },
-          })),
-        ];
-
-        const response = await runHistoryMediaAIRequest(options, () =>
-          options.ai!.complete({
-            model: options.multimodalWorkingModel,
-            messages: [
-              {
-                role: "system",
-                content:
-                  "You summarize video frames for a chat history. Be factual and concise. If frames are ambiguous, say so.",
-              },
-              {
-                role: "user",
-                content,
-              },
-            ],
-            temperature: 0.3,
-          }),
-        );
-        if (!response) {
-          return "";
-        }
-
-        return normalizeSummary(response.content);
-      },
+      () => summarizeVideoContent(videoFile.path, videoFile.byteSize, options),
+      sources,
     );
 
     logMediaSummary(options, "video", result.summary);
@@ -127,30 +119,128 @@ export async function summarizeHistoryVideo(
   }
 }
 
-export async function getCachedHistoryVideoTag(
-  videoSource: string | string[],
+export async function summarizeVideoContent(
+  videoPath: string,
+  byteSize: number,
   options: HistoryMediaProcessingOptions,
 ): Promise<string> {
-  const sources = normalizeVideoSources(videoSource);
-  if (sources.length === 0) return "[video]";
-  try {
-    const videoFile = await downloadVideoForAnalysis(sources, options);
+  // 小视频直接整体喂给多模态工作模型；过大则抽帧避免请求体爆炸。
+  if (byteSize <= VIDEO_FULL_UPLOAD_MAX_BYTES) {
     try {
-      const summary = getCachedSummaryByHash(
-        "video",
-        videoFile.contentHash,
-        options,
+      const summary = await summarizeVideoByFullVideo(videoPath, options);
+      if (summary) return summary;
+    } catch (err) {
+      getHistoryMediaLogger(options).warn(
+        `[history-media] Full-video summarization failed, falling back to frames: ${err}`,
       );
-      return summary ? `[video:${summary}]` : "[video]";
-    } finally {
-      await videoFile.cleanup();
     }
-  } catch (err) {
+  }
+  return summarizeVideoByFrames(videoPath, options);
+}
+
+async function summarizeVideoByFullVideo(
+  videoPath: string,
+  options: HistoryMediaProcessingOptions,
+): Promise<string> {
+  const mimeType = await probeVideoMimeType(videoPath);
+  const buffer = await fs.readFile(videoPath);
+  const dataUrl = `data:${mimeType};base64,${buffer.toString("base64")}`;
+
+  const content: any[] = [
+    {
+      type: "text",
+      text: "This is a video sent by someone in a chat. Summarize the video's content in Chinese for later use as chat history context: describe the visible people/objects/actions/scenes/on-screen text, and briefly explain what the video is about. Stay factual and concise; state uncertainty honestly when unclear.",
+    },
+    {
+      type: "video_url",
+      video_url: { url: dataUrl },
+    },
+  ];
+
+  const response = await runHistoryMediaAIRequest(options, () =>
+    options.ai!.complete({
+      model: options.multimodalWorkingModel,
+      messages: [
+        {
+          role: "system",
+          content:
+            "You summarize video content for a chat history. Describe only what the video actually shows, objectively and concisely. Do not invent information that is not present.",
+        },
+        {
+          role: "user",
+          content,
+        },
+      ],
+      temperature: 0.3,
+    }),
+  );
+  if (!response) return "";
+  return normalizeSummary(response.content);
+}
+
+async function summarizeVideoByFrames(
+  videoPath: string,
+  options: HistoryMediaProcessingOptions,
+): Promise<string> {
+  const frames = await extractVideoFrames(
+    videoPath,
+    VIDEO_FRAME_COUNT,
+    options,
+  );
+  if (frames.length === 0) {
     getHistoryMediaLogger(options).warn(
-      `[history-media] Failed to resolve cached video by hash: ${err}`,
+      "[history-media] Video frame extraction returned 0 frames",
     );
-    return "[video]";
+    return VIDEO_FRAME_EXTRACTION_FALLBACK;
   }
+
+  const content: any[] = [
+    {
+      type: "text",
+      text: `These ${frames.length} frames were sampled evenly from a video sent in a chat. Summarize the video's likely content in Chinese for later use as chat history context: note the visible people/objects/actions/on-screen text, and infer what the video is about. Stay factual and concise; if the frames are ambiguous or insufficient, say so honestly.`,
+    },
+    ...frames.map((frame) => ({
+      type: "image_url",
+      image_url: { url: frame, detail: "auto" },
+    })),
+  ];
+
+  const response = await runHistoryMediaAIRequest(options, () =>
+    options.ai!.complete({
+      model: options.multimodalWorkingModel,
+      messages: [
+        {
+          role: "system",
+          content:
+            "You summarize video content for a chat history from evenly sampled frames. Describe what the frames plausibly show, objectively and concisely. Do not invent information that is not present.",
+        },
+        {
+          role: "user",
+          content,
+        },
+      ],
+      temperature: 0.3,
+    }),
+  );
+  if (!response) return "";
+  return normalizeSummary(response.content);
+}
+
+export async function getCachedHistoryVideoTag(
+  videoSource: string | string[],
+  options: HistoryMediaProcessingOptions,
+): Promise<string> {
+  const sources = normalizeVideoSources(videoSource);
+  if (sources.length === 0) return "[video]";
+
+  for (const source of sources) {
+    const cached = options.db?.getMediaSummaryBySource?.(source);
+    if (cached?.summary && !isFallbackVideoSummary(cached.summary)) {
+      return `[video:${cached.summary}]`;
+    }
+  }
+
+  return "[video]";
 }
 
 export async function summarizeHistoryForward(
@@ -315,8 +405,16 @@ async function getOrCreateSummary(
   contentHash: string,
   options: HistoryMediaProcessingOptions,
   producer: () => Promise<string>,
+  sourceAliases: string[] = [],
 ): Promise<SummaryResult> {
   const cacheKey = `${kind}:${contentHash}`;
+  const aliases = Array.from(
+    new Set(
+      [source, ...sourceAliases]
+        .map((s) => String(s || "").trim())
+        .filter(Boolean),
+    ),
+  );
   const cached = options.db?.getMediaSummary(cacheKey);
   if (cached?.summary) {
     if (kind === "video") {
@@ -325,9 +423,11 @@ async function getOrCreateSummary(
         // Old versions cached this probe failure as a valid summary. Ignore it
         // so the new message can be downloaded and diagnosed again.
       } else {
+        writeMediaSummarySources(options, cacheKey, aliases);
         return { summary: cached.summary };
       }
     } else {
+      writeMediaSummarySources(options, cacheKey, aliases);
       return { summary: cached.summary };
     }
   }
@@ -345,6 +445,7 @@ async function getOrCreateSummary(
         summary,
         createdAt: Date.now(),
       });
+      writeMediaSummarySources(options, cacheKey, aliases);
       return { summary };
     }
   } catch (err) {
@@ -358,6 +459,17 @@ async function getOrCreateSummary(
   };
 }
 
+function writeMediaSummarySources(
+  options: HistoryMediaProcessingOptions,
+  summaryKey: string,
+  sources: string[],
+): void {
+  if (!options.db?.saveMediaSummarySource) return;
+  for (const source of sources) {
+    options.db.saveMediaSummarySource!(source, summaryKey);
+  }
+}
+
 function logMediaSummary(
   options: HistoryMediaProcessingOptions,
   kind: MediaSummaryKind,
@@ -478,6 +590,39 @@ async function probeVideoDuration(videoUrl: string): Promise<number> {
   return Number.isFinite(duration) && duration > 0 ? duration : 1;
 }
 
+export async function probeVideoMimeType(videoPath: string): Promise<string> {
+  try {
+    const { stdout } = await execFileAsync(
+      "ffprobe",
+      [
+        "-v",
+        "error",
+        "-show_entries",
+        "format=format_name",
+        "-of",
+        "default=noprint_wrappers=1:nokey=1",
+        videoPath,
+      ],
+      { timeout: 20_000 },
+    );
+    return mapFormatNameToMimeType(String(stdout).trim());
+  } catch {
+    return "video/mp4";
+  }
+}
+
+function mapFormatNameToMimeType(formatName: string): string {
+  const names = formatName.split(",").map((s) => s.trim().toLowerCase());
+  if (names.some((n) => n === "webm")) return "video/webm";
+  if (names.some((n) => n === "matroska" || n === "mkv"))
+    return "video/x-matroska";
+  if (names.some((n) => n === "avi")) return "video/x-msvideo";
+  if (names.some((n) => n === "flv")) return "video/x-flv";
+  if (names.some((n) => n === "mov" || n === "mp4" || n === "m4v"))
+    return "video/mp4";
+  return "video/mp4";
+}
+
 function buildFfmpegInputArgs(input: string, args: string[]): string[] {
   if (!/^https?:\/\//i.test(input)) return args;
   const inputIndex = args.findIndex((arg) => arg === input);
@@ -610,20 +755,23 @@ function truncateText(value: string, maxLength: number): string {
   return value.length > maxLength ? `${value.slice(0, maxLength)}...` : value;
 }
 
-async function downloadVideoForAnalysis(
+export async function downloadVideoForAnalysis(
   sources: string | string[],
   options: HistoryMediaProcessingOptions,
 ): Promise<{
   path: string;
   source: string;
   contentHash: string;
+  byteSize: number;
   cleanup: () => Promise<void>;
 }> {
   const candidates = normalizeVideoSources(sources);
   let lastError: unknown;
 
   for (const source of candidates) {
-    const tempDir = await fs.mkdtemp(path.join(os.tmpdir(), "mioku-video-src-"));
+    const tempDir = await fs.mkdtemp(
+      path.join(os.tmpdir(), "mioku-video-src-"),
+    );
     const filePath = path.join(tempDir, "video");
 
     try {
@@ -633,6 +781,7 @@ async function downloadVideoForAnalysis(
         path: filePath,
         source,
         contentHash: hashSource(buffer),
+        byteSize: buffer.length,
         cleanup: () => fs.rm(tempDir, { recursive: true, force: true }),
       };
     } catch (err) {
@@ -677,11 +826,7 @@ async function readVideoSource(source: string): Promise<Buffer> {
 function normalizeVideoSources(input: string | string[]): string[] {
   const raw = Array.isArray(input) ? input : [input];
   return Array.from(
-    new Set(
-      raw
-        .map((source) => String(source || "").trim())
-        .filter(Boolean),
-    ),
+    new Set(raw.map((source) => String(source || "").trim()).filter(Boolean)),
   );
 }
 

package/core/media/image-analyzer.ts

@@ -9,6 +9,7 @@ import { existsSync, mkdirSync, createWriteStream, unlink } from "fs";
 import * as https from "https";
 import * as http from "http";
 import { URL } from "url";
+import { prepareImageUrlsForModel } from "./image-compress";
 
 /**
  * 图片分析结果
@@ -74,27 +75,25 @@ const EMOTION_TAGS = [
 ];
 
 /**
- * 计算图片内容的哈希值
+ * 计算图片内容的哈希值（仅基于下载到的图片字节；URL 不参与哈希）
  */
-export async function calculateImageHash(url: string): Promise<string> {
+export async function calculateImageHash(url: string): Promise<string | null> {
   try {
-    // 下载图片内容
     const response = await fetch(url);
     if (!response.ok) {
-      // 如果下载失败，降级为 URL 哈希
-      return crypto.createHash("md5").update(url).digest("hex");
+      logger.warn(
+        `[image-analyzer] Failed to download image for hashing: ${response.status} ${response.statusText}`,
+      );
+      return null;
     }
 
     const buffer = Buffer.from(await response.arrayBuffer());
-
-    // 基于图片内容计算哈希
     return crypto.createHash("md5").update(buffer).digest("hex");
   } catch (err) {
     logger.warn(
-      `[image-analyzer] Failed to calculate content hash, using URL hash: ${err}`,
+      `[image-analyzer] Failed to download image for hashing: ${err}`,
     );
-    // 降级为 URL 哈希
-    return crypto.createHash("md5").update(url).digest("hex");
+    return null;
   }
 }
 
@@ -176,6 +175,8 @@ export async function analyzeImage(
       }
     }
 
+    imageUrls = await prepareImageUrlsForModel(imageUrls);
+
     const systemPrompt = `You are an image classification and analysis assistant. Analyze images and return structured information.
 
 Instructions:
@@ -336,6 +337,12 @@ export async function processImage(
   try {
     // 计算哈希（基于图片内容）
     const hash = await calculateImageHash(imageUrl);
+    if (!hash) {
+      logger.warn(
+        `[image-analyzer] Skipping analysis, image content unavailable: ${imageUrl}`,
+      );
+      return null;
+    }
 
     // 检查是否已存在
     const existing = db.getImageByHash(hash);
@@ -343,7 +350,13 @@ export async function processImage(
       logger.info(formatImageRecognitionLog(existing));
       return existing;
     }
-    const analysis = await analyzeImage(ai, imageUrl, model, undefined, options);
+    const analysis = await analyzeImage(
+      ai,
+      imageUrl,
+      model,
+      undefined,
+      options,
+    );
     if (!analysis.success || !analysis.type) {
       logger.warn(`[image-analyzer] Analysis failed: ${analysis.error}`);
       return null;
@@ -436,15 +449,11 @@ export async function processImage(
 
 /**
  * 从图片 URL 获取描述标签
- * 如果图片已在数据库中，返回 [meme:描述] 或 [image:描述]
- * 否则返回 [image]
+ * 命中数据库记录时返回 [meme:描述] 或 [image:描述]，否则返回 [image]。
+ * 收集聊天历史时用它，之前没识别到的图片不会再次触发请求。
  */
-export async function getImageTag(
-  imageUrl: string,
-  db: ChatDatabase,
-): Promise<string> {
-  const hash = await calculateImageHash(imageUrl);
-  const record = db.getImageByHash(hash);
+export function getImageTag(imageUrl: string, db: ChatDatabase): string {
+  const record = db.getImageByUrl(imageUrl);
 
   if (record) {
     return `[${record.type}:${record.description}]`;

package/core/media/image-compress.ts

@@ -0,0 +1,75 @@
+import sharp from "sharp";
+import { logger } from "mioki";
+
+const IMAGE_MAX_BYTES = 1 * 1024 * 1024;
+const COMPRESS_MAX_WIDTH = 1280;
+const COMPRESS_JPEG_QUALITY = 80;
+
+const FETCH_HEADERS = {
+  "User-Agent":
+    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
+  Referer: "https://qq.com/",
+};
+
+/**
+ * 准备发给模型的图片 URL：体积超过 1MB 时压缩为 JPEG data URL，否则原样返回。
+ * data URL（如 GIF 抽帧结果）与非 http(s) 链接直接放行。
+ */
+export async function prepareImageUrlForModel(url: string): Promise<string> {
+  if (url.startsWith("data:")) return url;
+  if (!/^https?:\/\//i.test(url)) return url;
+
+  const size = await probeImageSize(url);
+  if (size !== null && size <= IMAGE_MAX_BYTES) return url;
+
+  let buffer: Buffer;
+  try {
+    buffer = await downloadImageBuffer(url);
+  } catch (err) {
+    logger.warn(`[image-compress] download failed, using original: ${err}`);
+    return url;
+  }
+
+  if (buffer.length <= IMAGE_MAX_BYTES) return url;
+
+  try {
+    const compressed = await sharp(buffer)
+      .resize({ width: COMPRESS_MAX_WIDTH, withoutEnlargement: true })
+      .jpeg({ quality: COMPRESS_JPEG_QUALITY })
+      .toBuffer();
+    logger.info(
+      `[image-compress] compressed ${buffer.length} -> ${compressed.length} bytes`,
+    );
+    return `data:image/jpeg;base64,${compressed.toString("base64")}`;
+  } catch (err) {
+    logger.warn(`[image-compress] compress failed, using original: ${err}`);
+    return url;
+  }
+}
+
+export async function prepareImageUrlsForModel(
+  urls: string[],
+): Promise<string[]> {
+  return Promise.all(urls.map(prepareImageUrlForModel));
+}
+
+async function probeImageSize(url: string): Promise<number | null> {
+  try {
+    const head = await fetch(url, { method: "HEAD", headers: FETCH_HEADERS });
+    if (!head.ok) return null;
+    const len = head.headers.get("content-length");
+    if (!len) return null;
+    const n = Number(len);
+    return Number.isFinite(n) ? n : null;
+  } catch {
+    return null;
+  }
+}
+
+async function downloadImageBuffer(url: string): Promise<Buffer> {
+  const response = await fetch(url, { headers: FETCH_HEADERS });
+  if (!response.ok) {
+    throw new Error(`download failed: ${response.status} ${response.statusText}`);
+  }
+  return Buffer.from(await response.arrayBuffer());
+}

package/core/media/segment.ts

@@ -0,0 +1,91 @@
+import type { AIInstance } from "mioku";
+import type { ChatConfig, MediaSummaryRecord } from "../../types";
+import type { HistoryMediaOptions } from "../../manage/types";
+import {
+  getSegmentSourceCandidates,
+  type HistoryMediaProcessingOptions,
+  type MediaMessageSegment,
+} from "./history-media";
+
+export { getSegmentSourceCandidates };
+
+export function buildHistoryMediaOptions(
+  ai: AIInstance,
+  config: ChatConfig,
+): HistoryMediaOptions {
+  return {
+    ai,
+    workingModel: config.workingModel || config.model,
+    multimodalWorkingModel: config.multimodalWorkingModel || config.model,
+  };
+}
+
+export function buildHistoryMediaProcessingOptions(
+  ai: AIInstance,
+  config: ChatConfig,
+  db: {
+    getMediaSummary(key: string): MediaSummaryRecord | null;
+    saveMediaSummary(summary: MediaSummaryRecord): void;
+  },
+  bot: {
+    api<T = unknown>(action: string, params?: Record<string, unknown>): Promise<T>;
+  },
+  groupId: number,
+  log: HistoryMediaProcessingOptions["logger"],
+  runAIRequest?: <T>(request: () => Promise<T>) => Promise<T | null>,
+): HistoryMediaProcessingOptions {
+  return {
+    ...buildHistoryMediaOptions(ai, config),
+    db,
+    logger: log,
+    bot,
+    groupId,
+    runAIRequest,
+  };
+}
+
+export function getSegmentUrl(seg: {
+  url?: string;
+  data?: { url?: string };
+}): string | null {
+  const url = seg?.url || seg?.data?.url;
+  return typeof url === "string" && url.trim() ? url.trim() : null;
+}
+
+export async function getVideoSourceCandidatesFromMessage(
+  bot: {
+    api(action: string, params?: Record<string, unknown>): Promise<unknown>;
+  },
+  messageId: number | string | undefined,
+): Promise<string[]> {
+  if (messageId == null) return [];
+  const result = (await bot.api("get_msg", { message_id: messageId })) as {
+    message?: unknown[];
+    data?: { message?: unknown[] };
+  };
+  const segments = result?.message || result?.data?.message || [];
+  if (!Array.isArray(segments)) return [];
+  const candidates: string[] = [];
+  for (const seg of segments as MediaMessageSegment[]) {
+    if (seg?.type !== "video") continue;
+    candidates.push(...getSegmentSourceCandidates(seg));
+  }
+  return candidates;
+}
+
+export function getForwardId(seg: {
+  id?: unknown;
+  data?: { id?: unknown };
+}): string | null {
+  return String(seg?.id || seg?.data?.id || "");
+}
+
+export function getCardData(seg: { data?: unknown }): string | null {
+  const data = seg?.data;
+  if (!data) return null;
+  return typeof data === "string" ? data : JSON.stringify(data);
+}
+
+export function isMediaAnalysisBlocked(config: ChatConfig, userId: number): boolean {
+  return Boolean(config.mediaAnalysisBlacklistUsers?.includes(userId));
+}