mioku-plugin-chat 2.3.0 → 2.4.0

package/core/tools/info.ts

@@ -0,0 +1,284 @@
+import { logger } from "mioki";
+import type { AITool } from "mioku";
+import { TOOL_RESULT_FOLLOWUP_KEY } from "mioku";
+import type { ToolContext } from "../../types";
+
+async function createImageFollowupResult(
+  imageUrl: string,
+  text: string,
+  note: string,
+): Promise<Record<string, any>> {
+  let imageUrls = [imageUrl];
+  let gifFrameNote = "";
+
+  try {
+    const { isGifUrl, extractGifFrames } = await import("../media/gif-extractor");
+    if (await isGifUrl(imageUrl)) {
+      const result = await extractGifFrames(imageUrl);
+      if (result && result.frames.length > 0) {
+        imageUrls = result.frames;
+        gifFrameNote = ` The original image is an animated GIF; ${result.frames.length} extracted frame(s) are attached in order.`;
+      } else {
+        logger.warn("[view_media] Failed to extract GIF frames, attaching original image");
+      }
+    }
+  } catch (err) {
+    logger.warn(`[view_media] Failed to prepare image attachment: ${err}`);
+  }
+
+  const { prepareImageUrlsForModel } = await import("../media/image-compress");
+  const compressedImageUrls = await prepareImageUrlsForModel(imageUrls);
+
+  return {
+    success: true,
+    image_attached: true,
+    note: `${note}${gifFrameNote}`,
+    [TOOL_RESULT_FOLLOWUP_KEY]: {
+      text: `${text}${gifFrameNote}`,
+      images: compressedImageUrls.map((url) => ({ url, detail: "auto" })),
+    },
+  };
+}
+
+function createVideoFollowupResult(
+  videoUrl: string,
+  text: string,
+  note: string,
+): Record<string, any> {
+  return {
+    success: true,
+    video_attached: true,
+    note,
+    [TOOL_RESULT_FOLLOWUP_KEY]: {
+      text,
+      videos: [{ url: videoUrl, detail: "auto" }],
+    },
+  };
+}
+
+export function createInfoTools(toolCtx: ToolContext): AITool[] {
+  const tools: AITool[] = [];
+
+  if (toolCtx.groupId) {
+    tools.push({
+      name: "get_group_member_info",
+      description:
+        "Get detailed info about a group member,including gender, age, QQ rating, group level, group title, etc",
+      parameters: {
+        type: "object",
+        properties: {
+          user_id: { type: "number", description: "QQ number of the member" },
+        },
+        required: ["user_id"],
+      },
+      handler: async (args) => {
+        try {
+          const info = await toolCtx.ctx
+            .pickBot(toolCtx.event.self_id)
+            .getGroupMemberInfo(toolCtx.groupId!, args.user_id);
+          return {
+            nickname: info.nickname,
+            card: info.card,
+            sex: info.sex,
+            age: info.age,
+            area: info.area,
+            level: info.level,
+            qq_level: info.qq_level,
+            title: info.title,
+          };
+        } catch (err) {
+          return { error: `Failed to get member info: ${err}` };
+        }
+      },
+    });
+
+    tools.push({
+      name: "get_group_member_list",
+      description: "Get the list of group members (returns name and role only)",
+      parameters: { type: "object", properties: {} },
+      handler: async () => {
+        try {
+          const list = await toolCtx.ctx
+            .pickBot(toolCtx.event.self_id)
+            .getGroupMemberList(toolCtx.groupId!);
+          const members = (list as any[]).map((m) => ({
+            user_id: m.user_id,
+            nickname: m.card || m.nickname,
+            role: m.role,
+          }));
+          return { members: members.slice(0, 50), total: members.length };
+        } catch (err) {
+          return { error: `Failed to get member list: ${err}` };
+        }
+      },
+    });
+  }
+
+  tools.push({
+    name: "view_media",
+    description:
+      "View and analyze an image or video by its message ID. Use this when you need to see what's in an image or video to answer the user's question. The media will be analyzed and described to you.",
+    parameters: {
+      type: "object",
+      properties: {
+        message_id: {
+          type: "number",
+          description:
+            "The message ID (message_id) of the image or video. You can get this from the original message that contains the media.",
+        },
+      },
+      required: ["message_id"],
+    },
+    handler: async (args) => {
+      try {
+        const { getMediaByMessageId, describeImage } = await import("../multimodal");
+        const media = await getMediaByMessageId(toolCtx.ctx, args.message_id, toolCtx.event);
+
+        if (!media) {
+          return { error: "Image or video not found in the specified message" };
+        }
+
+        if (media.kind === "image") {
+          if (toolCtx.config.isMultimodal) {
+            return await createImageFollowupResult(
+              media.url,
+              `The image from message #${args.message_id} is attached. Inspect it directly and answer the user's question from the visual content.`,
+              "The image has been attached to the next main model request. Inspect it directly instead of relying on a worker-model description.",
+            );
+          }
+
+          const ai = toolCtx.aiService.getDefault();
+          if (!ai) return { error: "AI instance not available" };
+
+          const result = await describeImage(
+            ai,
+            media.url,
+            toolCtx.config.multimodalWorkingModel,
+            toolCtx.event?.raw_message || undefined,
+          );
+
+          if (!result.success) {
+            return { error: result.error || "Failed to analyze image" };
+          }
+
+          return {
+            success: true,
+            description: result.description,
+            note: "The image has been analyzed. Use the description above to answer the user's question.",
+          };
+        }
+
+        // 视频：超阈值一律交多模态工作模型概括（抽帧），否则按主模型是否多模态决定直接附加或概括。
+        const {
+          downloadVideoForAnalysis,
+          summarizeVideoContent,
+          probeVideoMimeType,
+          VIDEO_FULL_UPLOAD_MAX_BYTES,
+        } = await import("../media/history-media");
+
+        const videoFile = await downloadVideoForAnalysis(media.sources, {
+          logger: {
+            info: (m) => logger.info(m),
+            warn: (m) => logger.warn(m),
+            error: (m) => logger.error(m),
+          },
+        });
+
+        try {
+          const attachFullVideo =
+            toolCtx.config.isMultimodal && videoFile.byteSize <= VIDEO_FULL_UPLOAD_MAX_BYTES;
+
+          if (attachFullVideo) {
+            const fs = await import("fs/promises");
+            const mimeType = await probeVideoMimeType(videoFile.path);
+            const buffer = await fs.readFile(videoFile.path);
+            const dataUrl = `data:${mimeType};base64,${buffer.toString("base64")}`;
+            return createVideoFollowupResult(
+              dataUrl,
+              `The video from message #${args.message_id} is attached. Inspect it directly and answer the user's question from the visual content.`,
+              "The video has been attached to the next main model request. Inspect it directly instead of relying on a worker-model description.",
+            );
+          }
+
+          const ai = toolCtx.aiService.getDefault();
+          if (!ai) return { error: "AI instance not available" };
+
+          const summary = await summarizeVideoContent(videoFile.path, videoFile.byteSize, {
+            ai,
+            multimodalWorkingModel: toolCtx.config.multimodalWorkingModel,
+            logger: {
+              info: (m) => logger.info(m),
+              warn: (m) => logger.warn(m),
+              error: (m) => logger.error(m),
+            },
+          });
+
+          return {
+            success: true,
+            description: summary,
+            note: "The video has been summarized. Use the description above to answer the user's question.",
+          };
+        } finally {
+          await videoFile.cleanup();
+        }
+      } catch (err) {
+        return { error: `Failed to analyze media: ${err}` };
+      }
+    },
+  });
+
+  tools.push({
+    name: "view_member_avatar",
+    description:
+      "View and analyze a group member's QQ avatar. Use this when you need to see what someone's avatar looks like. The avatar will be analyzed and described to you.",
+    parameters: {
+      type: "object",
+      properties: {
+        user_id: {
+          type: "number",
+          description: "QQ number of the member whose avatar you want to view",
+        },
+      },
+      required: ["user_id"],
+    },
+    handler: async (args) => {
+      try {
+        const avatarUrl = `https://q1.qlogo.cn/g?b=qq&nk=${args.user_id}&s=640`;
+        logger.info(`[view_member_avatar] Analyzing avatar: ${avatarUrl}`);
+
+        if (toolCtx.config.isMultimodal) {
+          return await createImageFollowupResult(
+            avatarUrl,
+            `User ${args.user_id}'s QQ avatar is attached. Inspect it directly and answer the user's question from the visual content.`,
+            "The avatar has been attached to the next main model request. Inspect it directly instead of relying on a worker-model description.",
+          );
+        }
+
+        const { describeImage } = await import("../multimodal");
+        const ai = toolCtx.aiService.getDefault();
+        if (!ai) return { error: "AI instance not available" };
+
+        const result = await describeImage(
+          ai,
+          avatarUrl,
+          toolCtx.config.multimodalWorkingModel,
+          `User ${args.user_id}'s QQ avatar`,
+        );
+
+        if (!result.success) {
+          return { error: result.error || "Failed to analyze avatar" };
+        }
+
+        return {
+          success: true,
+          description: result.description,
+          note: "The avatar has been analyzed. Use the description above to answer the user's question.",
+        };
+      } catch (err) {
+        return { error: `Failed to analyze avatar: ${err}` };
+      }
+    },
+  });
+
+  return tools;
+}

package/core/tools/load-skill.ts

@@ -0,0 +1,149 @@
+import type { AITool } from "mioku";
+import type { ToolContext } from "../../types";
+import type { SkillSessionManager } from "../../manage/skill-session";
+import {
+  filterAllowedExternalSkills,
+  getSkillRequiredPermissionRole,
+  isExternalSkillAllowed,
+  hasSkillPermission,
+} from "../external-skills";
+import {
+  isFeatureEnabled,
+  isBuiltinFeature,
+  getFeatureMeta,
+  type FeatureName,
+} from "../feature-prompts";
+import {
+  buildWebSearchFeatureSection,
+  buildWebReadFeatureSection,
+  buildRecallMemoryFeatureSection,
+} from "../prompt";
+import { createWebSearchTool, createWebReadPageTool, createRecallMemoryTool } from "./web";
+
+type ConstraintStrength = "low" | "medium" | "high";
+
+function normalizeConstraintStrength(value: unknown): ConstraintStrength {
+  if (value === "low" || value === "high" || value === "medium") return value;
+  return "medium";
+}
+
+function createFeatureTools(
+  toolCtx: ToolContext,
+  featureName: FeatureName,
+): AITool[] {
+  switch (featureName) {
+    case "web_search":
+      return [createWebSearchTool(toolCtx)];
+    case "web_read_page":
+      return [createWebReadPageTool(toolCtx)];
+    case "recall_memory":
+      return [createRecallMemoryTool(toolCtx)];
+    default:
+      return [];
+  }
+}
+
+export function createLoadSkillTool(
+  toolCtx: ToolContext,
+  skillManager: SkillSessionManager,
+): AITool {
+  return {
+    name: "load_skill",
+    description:
+      "Load an external skill's tools into the current session. Tools will be available for 1 hour.",
+    parameters: {
+      type: "object",
+      properties: {
+        skill_name: { type: "string", description: "Skill name to load" },
+      },
+      required: ["skill_name"],
+    },
+    handler: async (args) => {
+      const skillName = String(args?.skill_name || "").trim();
+
+      if (isBuiltinFeature(skillName)) {
+        const feature = getFeatureMeta(skillName);
+        if (!feature) return { error: `Feature "${skillName}" not found` };
+        if (!isFeatureEnabled(toolCtx.config, feature)) {
+          return { error: `Feature "${skillName}" is not enabled in config` };
+        }
+
+        const featureTools: AITool[] = feature.hasTools
+          ? createFeatureTools(toolCtx, skillName)
+          : [];
+        if (featureTools.length > 0) {
+          skillManager.loadSkill(toolCtx.sessionId, skillName, featureTools);
+        } else {
+          skillManager.loadFeature(toolCtx.sessionId, skillName, 60 * 60 * 1000);
+        }
+
+        const toolStrength = normalizeConstraintStrength(toolCtx.config.toolCallConstraintStrength);
+        const sections: string[] = [];
+        if (skillName === "web_search") {
+          const s = buildWebSearchFeatureSection(toolCtx.config, toolStrength);
+          if (s) sections.push(s);
+        } else if (skillName === "web_read_page") {
+          const s = buildWebReadFeatureSection(toolCtx.config, toolStrength);
+          if (s) sections.push(s);
+        } else if (skillName === "recall_memory") {
+          const s = buildRecallMemoryFeatureSection(toolCtx.config);
+          if (s) sections.push(s);
+        }
+        const usageHint = sections.length > 0 ? sections.join("\n") : "";
+
+        return {
+          success: true,
+          skill_name: skillName,
+          feature: true,
+          expires_in: "1 hour",
+          tools: featureTools.map((t) => ({
+            name: `${skillName}.${t.name}`,
+            description: t.description,
+            parameters: t.parameters,
+          })),
+          ...(usageHint ? { usage: usageHint } : {}),
+        };
+      }
+
+      if (!isExternalSkillAllowed(toolCtx.config, skillName)) {
+        const allSkills = toolCtx.aiService.getAllSkills?.();
+        const allowedSkills = allSkills
+          ? filterAllowedExternalSkills(
+              toolCtx.config,
+              [...allSkills.values()],
+              toolCtx.triggerSkillRole,
+            )
+          : [];
+        const allowedNames = allowedSkills.map((skill) => skill.name);
+        return {
+          error:
+            allowedNames.length > 0
+              ? `Skill "${skillName}" is not allowed. Allowed skills: ${allowedNames.join(", ")}`
+              : "No external skills are allowed in current config",
+        };
+      }
+
+      const skill = toolCtx.aiService.getSkill(skillName);
+      if (!skill) return { error: `Skill "${skillName}" does not exist` };
+      const requiredRole = getSkillRequiredPermissionRole(skill);
+      if (!hasSkillPermission(toolCtx.triggerSkillRole, requiredRole)) {
+        return {
+          error: `Permission denied: loading skill "${skill.name}" requires role "${requiredRole}", current role is "${toolCtx.triggerSkillRole}"`,
+        };
+      }
+
+      skillManager.loadSkill(toolCtx.sessionId, skill.name, skill.tools);
+
+      return {
+        success: true,
+        skill_name: skill.name,
+        expires_in: "1 hour",
+        tools: skill.tools.map((t) => ({
+          name: `${skill.name}.${t.name}`,
+          description: t.description,
+          parameters: t.parameters,
+        })),
+      };
+    },
+  };
+}