mioku-plugin-chat 3.1.0 → 3.1.1

package/core/chat-engine.ts

@@ -29,6 +29,7 @@ import {
   type StructuredUserInput,
 } from "../manage/group-structured-history";
 import { prepareImageUrlsForModel } from "./media/image-compress";
+import { mainModelSupportsVision } from "../utils/model";
 
 interface StructuredHistoryRunContext {
   manager: GroupStructuredHistoryManager;
@@ -116,7 +117,10 @@ export async function runChat(
   const currentUserMessages = hasStructuredHistory
     ? buildStructuredUserMessages(structuredHistory!.currentUserInputs)
     : [];
-  const directImageUrls = toolCtx.config.isMultimodal
+  const directImageUrls = mainModelSupportsVision(
+    toolCtx.aiService,
+    toolCtx.config.model,
+  )
     ? toolCtx.pendingImageUrls
     : undefined;
   const webSearchState = { count: 0 };
@@ -762,7 +766,7 @@ function sanitizeBrackets(text: string): string {
   let working = text.replace(FUNCTIONAL_BRACKET_PREFIX, (match) => {
     const idx = placeholders.length;
     placeholders.push(match);
-    return `${idx}`;
+    return `\u0000${idx}\u0000`;
   });
 
   let result = "";
@@ -772,8 +776,8 @@ function sanitizeBrackets(text: string): string {
   while (i < working.length) {
     const ch = working[i];
 
-    if (ch === "") {
-      const endIdx = working.indexOf("", i + 1);
+    if (ch === "\u0000") {
+      const endIdx = working.indexOf("\u0000", i + 1);
       if (endIdx > i) {
         const idx = parseInt(working.slice(i + 1, endIdx), 10);
         if (!Number.isNaN(idx) && placeholders[idx] !== undefined) {

package/core/chat-turn.ts

@@ -2,7 +2,12 @@ import type { MiokuContext } from "mioku";
 import type { AITool, Bot, ChatRuntimePromptInjection } from "mioku";
 import type { ChatPluginContext, ChatRuntimeState } from "../context";
 import type { ChatConfig, ChatMessage, TargetMessage } from "../types";
-import { getGroupHistory, getBotRole, getQuotedContent } from "../utils";
+import {
+  getGroupHistory,
+  getBotRole,
+  getQuotedContent,
+  mainModelSupportsVision,
+} from "../utils";
 import { buildStructuredUserInputFromTarget } from "../manage/group-structured-history";
 
 export type RuntimeReplyContextType =
@@ -43,6 +48,53 @@ interface ResolvedRuntimeContext {
 const NO_NEW_MESSAGE =
   "[No new user message in this turn. Reply naturally based on the runtime instruction and recent context.]";
 
+/**
+ * 主模型不支持视觉时，引用消息里的图片交给视觉工作模型描述（结果按内容哈希入库复用）。
+ * 主模型支持视觉时返回 undefined：图片会直接附到请求里，不必重复描述。
+ */
+async function describeQuotedImage(
+  pluginCtx: ChatPluginContext,
+  cfg: ChatConfig,
+  imageUrl: string | undefined,
+  userId: string,
+  groupId: string | undefined,
+): Promise<string | undefined> {
+  if (
+    !imageUrl ||
+    mainModelSupportsVision(pluginCtx.aiService, cfg.model)
+  ) {
+    return undefined;
+  }
+  const visionAI =
+    pluginCtx.visionAIInstance ?? pluginCtx.aiService.getDefault();
+  if (!visionAI) return undefined;
+
+  try {
+    const { processImage } = await import("./media/image-analyzer");
+    const record = await processImage(
+      visionAI,
+      imageUrl,
+      cfg.multimodalWorkingModel,
+      pluginCtx.db,
+      {
+        runAIRequest: (request) =>
+          pluginCtx.runWithRateLimitGuard(request, {
+            userId,
+            groupId,
+            label: "quoted-image",
+            skipRetryOnRateLimit: true,
+          }),
+      },
+    );
+    return record?.description?.trim() || undefined;
+  } catch (err) {
+    pluginCtx.ctx.logger.error(
+      `[chat-turn] Failed to describe quoted image: ${err}`,
+    );
+    return undefined;
+  }
+}
+
 function buildRuntimeTargetMessageContent(
   ctx: MiokuContext,
   event: any,
@@ -289,16 +341,14 @@ export async function processChat(
     }
     if (quotedInfo?.imageUrl) imageUrls.push(quotedInfo.imageUrl);
 
-    let messageContent = ctx.text(e) || "";
-    let extraContext = "";
-    if (quotedInfo) {
-      const parts = [
-        `[Quoted message #${quotedInfo.messageId} from ${quotedInfo.senderName}: ${quotedInfo.content}]`,
-      ];
-      if (quotedInfo.imageUrl) parts.push("[Quoted message contains an image]");
-      extraContext = parts.join(" ");
-    }
-    if (extraContext) messageContent = extraContext + " " + messageContent;
+    // 与历史拉取并行，省掉视觉描述带来的额外等待
+    const quotedImageNotePromise = describeQuotedImage(
+      pluginCtx,
+      cfg,
+      quotedInfo?.imageUrl,
+      userId,
+      groupId,
+    );
 
     const rawHistory = groupId
       ? await getGroupHistory(
@@ -350,6 +400,24 @@ export async function processChat(
       userId,
     );
 
+    const quotedImageNote = await quotedImageNotePromise;
+    let messageContent = ctx.text(e) || "";
+    if (quotedInfo) {
+      const quotedText =
+        quotedInfo.content || (quotedInfo.imageUrl ? "[image]" : "");
+      const parts = [
+        `[Quoted message #${quotedInfo.messageId} from ${quotedInfo.senderName}: ${quotedText}]`,
+      ];
+      if (quotedInfo.imageUrl) {
+        parts.push(
+          quotedImageNote
+            ? `[Quoted message image: ${quotedImageNote}]`
+            : "[Quoted message contains an image]",
+        );
+      }
+      messageContent = `${parts.join(" ")} ${messageContent}`;
+    }
+
     const targetMessage: TargetMessage = {
       userName: senderName,
       userId,

package/core/media/history-media.ts

@@ -556,7 +556,7 @@ async function extractVideoFrames(
         { timeout: 30_000 },
       );
       const buffer = await fs.readFile(outputPath);
-      frames.push(`data:image/jpeg);base64,${buffer.toString("base64")}`);
+      frames.push(`data:image/jpeg;base64,${buffer.toString("base64")}`);
     }
 
     return frames;

package/core/media/image-compress.ts

@@ -5,6 +5,13 @@ const IMAGE_MAX_BYTES = 1 * 1024 * 1024;
 const COMPRESS_MAX_WIDTH = 1280;
 const COMPRESS_JPEG_QUALITY = 80;
 
+const INLINE_MIME_TYPES = new Set([
+  "image/jpeg",
+  "image/png",
+  "image/webp",
+  "image/gif",
+]);
+
 export const QQ_IMAGE_FETCH_HEADERS = {
   "User-Agent":
     "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
@@ -13,26 +20,24 @@ export const QQ_IMAGE_FETCH_HEADERS = {
 
 const FETCH_HEADERS = QQ_IMAGE_FETCH_HEADERS;
 
-/**
- * 准备发给模型的图片 URL：体积超过 1MB 时压缩为 JPEG data URL，否则原样返回。
- * data URL（如 GIF 抽帧结果）与非 http(s) 链接直接放行。
- */
 export async function prepareImageUrlForModel(url: string): Promise<string> {
   if (url.startsWith("data:")) return url;
   if (!/^https?:\/\//i.test(url)) return url;
 
-  const size = await probeImageSize(url);
-  if (size !== null && size <= IMAGE_MAX_BYTES) return url;
-
   let buffer: Buffer;
+  let mimeType: string;
   try {
-    buffer = await downloadImageBuffer(url);
+    const downloaded = await downloadImage(url);
+    buffer = downloaded.buffer;
+    mimeType = downloaded.mimeType;
   } catch (err) {
     logger.warn(`[image-compress] download failed, using original: ${err}`);
     return url;
   }
 
-  if (buffer.length <= IMAGE_MAX_BYTES) return url;
+  if (buffer.length <= IMAGE_MAX_BYTES && INLINE_MIME_TYPES.has(mimeType)) {
+    return `data:${mimeType};base64,${buffer.toString("base64")}`;
+  }
 
   try {
     const compressed = await sharp(buffer)
@@ -44,8 +49,8 @@ export async function prepareImageUrlForModel(url: string): Promise<string> {
     );
     return `data:image/jpeg;base64,${compressed.toString("base64")}`;
   } catch (err) {
-    logger.warn(`[image-compress] compress failed, using original: ${err}`);
-    return url;
+    logger.warn(`[image-compress] compress failed, inlining original: ${err}`);
+    return `data:${mimeType};base64,${buffer.toString("base64")}`;
   }
 }
 
@@ -55,23 +60,29 @@ export async function prepareImageUrlsForModel(
   return Promise.all(urls.map(prepareImageUrlForModel));
 }
 
-async function probeImageSize(url: string): Promise<number | null> {
-  try {
-    const head = await fetch(url, { method: "HEAD", headers: FETCH_HEADERS });
-    if (!head.ok) return null;
-    const len = head.headers.get("content-length");
-    if (!len) return null;
-    const n = Number(len);
-    return Number.isFinite(n) ? n : null;
-  } catch {
-    return null;
-  }
-}
-
-async function downloadImageBuffer(url: string): Promise<Buffer> {
+async function downloadImage(
+  url: string,
+): Promise<{ buffer: Buffer; mimeType: string }> {
   const response = await fetch(url, { headers: FETCH_HEADERS });
   if (!response.ok) {
-    throw new Error(`download failed: ${response.status} ${response.statusText}`);
+    throw new Error(
+      `download failed: ${response.status} ${response.statusText}`,
+    );
+  }
+
+  const mimeType = normalizeImageMimeType(
+    response.headers.get("content-type") || "",
+  );
+  if (!mimeType) {
+    throw new Error(
+      `not an image response: ${response.headers.get("content-type") || "unknown"}`,
+    );
   }
-  return Buffer.from(await response.arrayBuffer());
+
+  return { buffer: Buffer.from(await response.arrayBuffer()), mimeType };
+}
+
+function normalizeImageMimeType(contentType: string): string | null {
+  const mimeType = contentType.split(";")[0]?.trim().toLowerCase() || "";
+  return mimeType.startsWith("image/") ? mimeType : null;
 }

package/core/tools/info.ts

@@ -2,6 +2,7 @@ import {logger} from "mioku";
 import type { AITool } from "mioku";
 import { TOOL_RESULT_FOLLOWUP_KEY } from "mioku";
 import type { ToolContext } from "../../types";
+import { mainModelSupportsVision } from "../../utils/model";
 
 async function createImageFollowupResult(
   imageUrl: string,
@@ -158,7 +159,7 @@ export function createInfoTools(toolCtx: ToolContext): AITool[] {
         }
 
         if (media.kind === "image") {
-          if (toolCtx.config.isMultimodal) {
+          if (mainModelSupportsVision(toolCtx.aiService, toolCtx.config.model)) {
             return await createImageFollowupResult(
               media.url,
               `The image from message #${args.message_id} is attached. Inspect it directly and answer the user's question from the visual content.`,
@@ -205,7 +206,8 @@ export function createInfoTools(toolCtx: ToolContext): AITool[] {
 
         try {
           const attachFullVideo =
-            toolCtx.config.isMultimodal && videoFile.byteSize <= VIDEO_FULL_UPLOAD_MAX_BYTES;
+            mainModelSupportsVision(toolCtx.aiService, toolCtx.config.model) &&
+            videoFile.byteSize <= VIDEO_FULL_UPLOAD_MAX_BYTES;
 
           if (attachFullVideo) {
             const fs = await import("fs/promises");
@@ -266,7 +268,7 @@ export function createInfoTools(toolCtx: ToolContext): AITool[] {
         const avatarUrl = `https://q1.qlogo.cn/g?b=qq&nk=${args.user_id}&s=640`;
         logger.info(`[view_member_avatar] Analyzing avatar: ${avatarUrl}`);
 
-        if (toolCtx.config.isMultimodal) {
+        if (mainModelSupportsVision(toolCtx.aiService, toolCtx.config.model)) {
           return await createImageFollowupResult(
             avatarUrl,
             `User ${args.user_id}'s QQ avatar is attached. Inspect it directly and answer the user's question from the visual content.`,

package/index.ts

@@ -48,6 +48,7 @@ import { buildHistoryMediaOptions } from "./core/media/segment";
 import type { ChatConfigProvider } from "./humanize";
 import type { ChatPluginContext, ChatHandlerState } from "./context";
 import { mergeGroupOverrides } from "./utils/group-config";
+import { mainModelSupportsVision } from "./utils/model";
 import {
   mergeChatConfig,
   normalizeIdList,
@@ -95,9 +96,20 @@ function resolveRoleInstances(aiService: AIService): {
   const visionModel = findModel("vision", vision) || workingModel;
 
   const models = aiService.listModels?.() ?? [];
-  const visionDesc =
-    models.find((item) => item.id === bindings.vision) ||
-    models.find((item) => item.modelId === visionModel);
+  const findModelDesc = (role: AIModelRole, modelId: string) => {
+    const exact = models.find((item) => item.id === bindings[role]);
+    if (exact) return exact;
+    const providerId = instances.find(
+      (info) => info.role === role || info.name === role,
+    )?.providerId;
+    return (
+      models.find(
+        (item) => item.modelId === modelId && item.providerId === providerId,
+      ) || models.find((item) => item.modelId === modelId)
+    );
+  };
+
+  const visionDesc = findModelDesc("vision", visionModel);
   const isMultimodal =
     visionDesc?.capabilities?.includes("vision") ?? Boolean(visionModel);
 
@@ -341,7 +353,7 @@ export default definePlugin({
       return;
     }
 
-roleModels = resolved.models;
+    roleModels = resolved.models;
     roleIsMultimodal = resolved.isMultimodal;
     await refreshBaseCache();
 
@@ -567,7 +579,11 @@ roleModels = resolved.models;
       usage: ".tts <文本>",
       handler: async ({ event, match }) => {
         if (String(event.user_id ?? "") === String(event.self_id ?? "")) return;
-        await handleTtsCommand(pluginCtx, event, String(match?.[1] ?? "").trim());
+        await handleTtsCommand(
+          pluginCtx,
+          event,
+          String(match?.[1] ?? "").trim(),
+        );
       },
     });
 
@@ -575,7 +591,7 @@ roleModels = resolved.models;
     ctx.handle("notice.group.poke", createPokeHandler(pluginCtx, handlerState));
 
     ctx.logger.info(
-      `聊天插件加载成功 (main=${roleModels.main || "?"}, work=${roleModels.working || "?"}, vision=${roleModels.vision || "?"})`,
+      `聊天插件加载成功 (main=${roleModels.main || "?"}${mainModelSupportsVision(aiService, roleModels.main) ? "" : " 无视觉，图片交给视觉模型描述"}, work=${roleModels.working || "?"}, vision=${roleModels.vision || "?"})`,
     );
 
     return () => {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "mioku-plugin-chat",
-  "version": "3.1.0",
+  "version": "3.1.1",
   "description": "AI 智能聊天插件",
   "main": "index.ts",
   "type": "module",

package/utils/index.ts

@@ -9,3 +9,5 @@ export {
   buildChatMessageFromEvent,
   normalizeIdList,
 } from "./message";
+
+export { mainModelSupportsVision } from "./model";

package/utils/message.ts

@@ -283,10 +283,17 @@ function extractQuotedText(messageSegs: any): string {
   return parts.join(" ").trim();
 }
 
-function buildReplyAnnotation(
+/** 引用关系：id 必有，发送者与原文可能缺失（例如 onebot 历史只给 id） */
+interface ReplyRef {
+  id: string;
+  name?: string;
+  text?: string;
+}
+
+function extractReplyRef(
   source: any,
   ctx: HistoryFormatContext,
-): string | null {
+): ReplyRef | null {
   if (!source || typeof source !== "object") return null;
   const sourceId = source.id ?? source.message_id ?? source.message_seq;
   const sourceUserId = source.user_id;
@@ -294,18 +301,38 @@ function buildReplyAnnotation(
     source.sender?.card || source.sender?.nickname || source.nickname;
   const sourceText = extractQuotedText(source.message);
 
-  if (sourceUserId == null && !sourceText && !sourceNickname) return null;
+  if (
+    sourceId == null &&
+    sourceUserId == null &&
+    !sourceText &&
+    !sourceNickname
+  ) {
+    return null;
+  }
 
-  const idStr = sourceId != null ? String(sourceId) : "?";
-  let displayName: string | undefined;
+  let name: string | undefined;
   if (sourceUserId != null) {
-    displayName =
-      ctx.memberNameCache.get(String(sourceUserId)) || String(sourceUserId);
+    name = ctx.memberNameCache.get(String(sourceUserId)) || String(sourceUserId);
   }
-  if (!displayName) displayName = sourceNickname || "unknown";
+  if (!name) name = sourceNickname || undefined;
+
+  return {
+    id: sourceId != null ? String(sourceId) : "",
+    name,
+    text: sourceText || undefined,
+  };
+}
+
+function renderReplyRef(ref: ReplyRef): string {
+  const head = `↪ reply to #${ref.id || "?"}`;
+  if (!ref.name && !ref.text) return head;
+  if (!ref.text) return `${head} ${ref.name}`;
+  return ref.name ? `${head} ${ref.name}: "${ref.text}"` : `${head}: "${ref.text}"`;
+}
 
-  const text = sourceText || "(empty)";
-  return `↪ reply to #${idStr} ${displayName}: "${text}"`;
+function summarizeQuotedContent(content: string): string {
+  const flat = content.replace(/\s+/g, " ").trim();
+  return flat.length > 80 ? `${flat.slice(0, 80)}…` : flat;
 }
 
 async function getImageTagWithHashCache(
@@ -345,12 +372,14 @@ async function getImageTagWithHashCache(
 async function formatHistoryMessage(
   msg: HistoryMessage,
   ctx: HistoryFormatContext,
-): Promise<FormattedHistoryMessage | null> {
+): Promise<{ formatted: FormattedHistoryMessage; reply: ReplyRef | null } | null> {
   // 自己或本运行时其它 bot 的消息不作为用户历史，
   // 否则多 bot 同群时会把彼此的消息当作用户接话，形成循环触发
   if (String(msg.user_id) === String(ctx.botUin)) return null;
   if (ctx.botIds.has(String(msg.user_id))) return null;
 
+  const reply = extractReplyRef((msg as { source?: unknown }).source, ctx);
+
   let content = "";
   try {
     content = await buildMessageContent(msg, ctx);
@@ -359,19 +388,22 @@ async function formatHistoryMessage(
     return null;
   }
 
-  if (!content.trim()) return null;
+  if (!content.trim() && !reply) return null;
 
   return {
-    userId: String(msg.user_id ?? "").trim(),
-    userName: msg.nickname || String(msg.user_id ?? "unknown"),
-    userRole: "member",
-    content,
-    messageId:
-      msg.message_id != null && msg.message_id !== ""
-        ? String(msg.message_id)
-        : "",
-    timestamp: msg.time ?? Date.now(),
-    role: "user",
+    formatted: {
+      userId: String(msg.user_id ?? "").trim(),
+      userName: msg.nickname || String(msg.user_id ?? "unknown"),
+      userRole: "member",
+      content,
+      messageId:
+        msg.message_id != null && msg.message_id !== ""
+          ? String(msg.message_id)
+          : "",
+      timestamp: msg.time ?? Date.now(),
+      role: "user",
+    },
+    reply,
   };
 }
 
@@ -385,9 +417,6 @@ async function buildMessageContent(
   const { db, historyMediaOptions } = ctx;
   const parts: string[] = [];
 
-  const replyAnnotation = buildReplyAnnotation(msg.source, ctx);
-  if (replyAnnotation) parts.push(replyAnnotation);
-
   const textSegs = msg.message.filter((seg: any) => seg.type === "text");
   const textContent = textSegs
     .map((seg: any) => seg.data?.text || "")
@@ -623,15 +652,51 @@ export async function getGroupHistory(
       hashLookupCache,
     };
 
-    const formattedResults = await mapWithConcurrency(
+    const processed = await mapWithConcurrency(
       messages,
       HISTORY_MEDIA_CONCURRENCY,
       (msg: any) => formatHistoryMessage(msg, formatCtx),
     );
-    const formatted = formattedResults.filter(
-      (msg): msg is FormattedHistoryMessage => Boolean(msg),
+    const entries = processed.filter(
+      (
+        item,
+      ): item is {
+        formatted: FormattedHistoryMessage;
+        reply: ReplyRef | null;
+      } => Boolean(item),
     );
 
+    // 引用关系单独渲染：被引用的消息若也在本批历史里，直接补上发送者与原文
+    const byMessageId = new Map<string, FormattedHistoryMessage>();
+    for (const item of entries) {
+      if (item.formatted.messageId) {
+        byMessageId.set(item.formatted.messageId, item.formatted);
+      }
+    }
+    for (const botMessage of botMessages) {
+      if (botMessage.messageId && !byMessageId.has(botMessage.messageId)) {
+        byMessageId.set(botMessage.messageId, botMessage);
+      }
+    }
+
+    const formatted = entries.map(({ formatted: message, reply }) => {
+      if (!reply) return message;
+      const quoted = reply.id ? byMessageId.get(reply.id) : undefined;
+      const annotation = renderReplyRef({
+        ...reply,
+        name: reply.name ?? quoted?.userName,
+        text:
+          reply.text ??
+          (quoted ? summarizeQuotedContent(quoted.content) || undefined : undefined),
+      });
+      return {
+        ...message,
+        content: message.content
+          ? `${annotation} ${message.content}`
+          : annotation,
+      };
+    });
+
     // 合并 bot 消息
     const allMessages = [...botMessages, ...formatted];
     allMessages.sort((a, b) => a.timestamp - b.timestamp);

package/utils/model.ts

@@ -0,0 +1,31 @@
+import type { AIModelDescriptor, AIService } from "mioku";
+
+const supportsVision = (model?: AIModelDescriptor): boolean =>
+  Boolean(model?.capabilities?.includes("vision"));
+
+export function mainModelSupportsVision(
+  aiService: AIService | undefined,
+  fallbackModelId?: string,
+): boolean {
+  if (!aiService) return false;
+
+  const models = aiService.listModels?.() ?? [];
+  const boundId = aiService.getRoleBindings?.()?.main;
+  if (boundId) {
+    const bound = models.find((item) => item.id === boundId);
+    if (bound) return supportsVision(bound);
+  }
+
+  const instance = (aiService.listInstances?.() ?? []).find(
+    (item) => item.role === "main" || item.name === "main",
+  );
+  const modelId = fallbackModelId || instance?.modelId || "";
+  if (!modelId) return false;
+
+  return supportsVision(
+    models.find(
+      (item) =>
+        item.modelId === modelId && item.providerId === instance?.providerId,
+    ) ?? models.find((item) => item.modelId === modelId),
+  );
+}