koishi-plugin-message-dedup 0.0.1 → 0.0.3

package/README.md

@@ -2,7 +2,9 @@
 
 Koishi 消息去重插件，检测群内重复的图片、链接、聊天记录。
 
-![](assets/dup-1.jpg)
+<p align="center">
+  <img src="assets/dup-1.jpg" width="200">
+</p>
 
 ## 功能
 

package/lib/config.d.ts

@@ -6,6 +6,8 @@ export interface Config {
     imageSimilarityThreshold: number;
     linkExactMatch: boolean;
     forwardContentMaxLength: number;
+    forwardImageMatchMode: 'all' | 'majority';
+    forwardImageSimilarityThreshold: number;
     retentionDays: number;
     stickerDir: string;
     sendMethod: 'koishi' | 'onebot';

package/lib/config.js

@@ -4,8 +4,8 @@ exports.Config = void 0;
 const koishi_1 = require("koishi");
 exports.Config = koishi_1.Schema.object({
     enableImage: koishi_1.Schema.boolean()
-        .default(false)
-        .description('启用图片去重（默认关闭，可能误判表情包）'),
+        .default(true)
+        .description('启用图片去重（自动排除表情包）'),
     enableLink: koishi_1.Schema.boolean()
         .default(true)
         .description('启用链接去重'),
@@ -25,6 +25,17 @@ exports.Config = koishi_1.Schema.object({
         .min(100)
         .max(2000)
         .description('转发消息内容摘要最大长度'),
+    forwardImageMatchMode: koishi_1.Schema.union([
+        koishi_1.Schema.const('all').description('全部匹配'),
+        koishi_1.Schema.const('majority').description('过半匹配')
+    ])
+        .default('all')
+        .description('转发消息图片匹配模式：全部匹配要求所有图片相同，过半匹配只需超半数图片相同'),
+    forwardImageSimilarityThreshold: koishi_1.Schema.number()
+        .default(10)
+        .min(0)
+        .max(32)
+        .description('转发消息图片相似度阈值（百分比，0为完全相同）'),
     retentionDays: koishi_1.Schema.number()
         .default(7)
         .min(1)

package/lib/database.d.ts

@@ -16,7 +16,23 @@ export interface DedupRecord {
     originalContent: string;
     extraInfo: string;
 }
+/**
+ * 转发消息额外信息结构
+ */
+export interface ForwardExtraInfo {
+    forwardId: string;
+    preview: string;
+    textHash: string;
+    imageHashes: string[];
+    imageCount: number;
+    failedImages: number;
+}
 export declare function extendDatabase(ctx: Context): void;
 export declare function findDuplicate(ctx: Context, guildId: string, contentType: 'image' | 'link' | 'forward', contentHash: string, imageThreshold?: number): Promise<DedupRecord | null>;
 export declare function saveRecord(ctx: Context, record: DedupRecord): Promise<void>;
 export declare function cleanupOldRecords(ctx: Context, retentionDays: number): Promise<void>;
+/**
+ * 比较转发消息是否重复
+ * 使用文本哈希 + 图片哈希组合匹配
+ */
+export declare function compareForwardMessages(ctx: Context, guildId: string, newTextHash: string, newImageHashes: string[], imageMatchMode: 'all' | 'majority', imageThreshold: number): Promise<DedupRecord | null>;

package/lib/database.js

@@ -4,6 +4,7 @@ exports.extendDatabase = extendDatabase;
 exports.findDuplicate = findDuplicate;
 exports.saveRecord = saveRecord;
 exports.cleanupOldRecords = cleanupOldRecords;
+exports.compareForwardMessages = compareForwardMessages;
 const hash_1 = require("./hash");
 function extendDatabase(ctx) {
     ctx.model.extend('message_dedup', {
@@ -23,10 +24,12 @@ function extendDatabase(ctx) {
     });
 }
 async function findDuplicate(ctx, guildId, contentType, contentHash, imageThreshold) {
-    const records = await ctx.database.get('message_dedup', {
+    let records = await ctx.database.get('message_dedup', {
         guildId,
         contentType
     });
+    // 按时间戳升序排序，确保返回最早匹配的记录（原消息）
+    records = records.sort((a, b) => a.timestamp - b.timestamp);
     if (contentType === 'image' && imageThreshold !== undefined) {
         // 图片需要计算汉明距离
         // compareHashes返回0-1，0表示相同
@@ -60,3 +63,86 @@ async function cleanupOldRecords(ctx, retentionDays) {
         timestamp: { $lt: cutoffTime }
     });
 }
+/**
+ * 比较转发消息是否重复
+ * 使用文本哈希 + 图片哈希组合匹配
+ */
+async function compareForwardMessages(ctx, guildId, newTextHash, newImageHashes, imageMatchMode, imageThreshold) {
+    const records = await ctx.database.get('message_dedup', {
+        guildId,
+        contentType: 'forward'
+    });
+    // 按时间戳升序排序，确保返回最早匹配的记录（原消息）
+    const sortedRecords = records.sort((a, b) => a.timestamp - b.timestamp);
+    const thresholdRatio = imageThreshold / 100;
+    for (const record of sortedRecords) {
+        let extra;
+        try {
+            extra = JSON.parse(record.extraInfo);
+        }
+        catch {
+            // 兼容旧记录格式，跳过
+            continue;
+        }
+        // 1. 文本哈希必须匹配
+        if (extra.textHash !== newTextHash)
+            continue;
+        const newCount = newImageHashes.length;
+        const oldCount = extra.imageHashes?.length || 0;
+        // 2. 无图片情况：纯文本匹配
+        if (newCount === 0 && oldCount === 0) {
+            return record;
+        }
+        // 3. 单边无图片：不匹配
+        if (newCount === 0 || oldCount === 0) {
+            continue;
+        }
+        // 4. 全部匹配模式：数量必须相同
+        if (imageMatchMode === 'all' && newCount !== oldCount) {
+            continue;
+        }
+        // 5. 计算图片匹配数
+        const matchedCount = countImageMatches(newImageHashes, extra.imageHashes, thresholdRatio);
+        const totalCount = Math.max(newCount, oldCount);
+        // 6. 根据模式判断是否匹配
+        if (imageMatchMode === 'all') {
+            if (matchedCount === newCount && newCount === oldCount) {
+                return record;
+            }
+        }
+        else { // majority
+            if (matchedCount > totalCount / 2) {
+                return record;
+            }
+        }
+    }
+    return null;
+}
+/**
+ * 计算两个图片哈希列表的匹配数量
+ * 使用贪心算法：每个新图片找最相似的未匹配旧图片
+ */
+function countImageMatches(hashes1, hashes2, threshold // 0-1 之间的阈值
+) {
+    let matchCount = 0;
+    const used = new Set();
+    for (const h1 of hashes1) {
+        for (let i = 0; i < hashes2.length; i++) {
+            if (used.has(i))
+                continue;
+            try {
+                const distance = (0, hash_1.calculateHashDistance)(h1, hashes2[i]);
+                if (distance <= threshold) {
+                    matchCount++;
+                    used.add(i);
+                    break;
+                }
+            }
+            catch {
+                // 哈希格式错误，跳过
+                continue;
+            }
+        }
+    }
+    return matchCount;
+}

package/lib/index.js

@@ -94,10 +94,18 @@ async function processMessage(session, config, ctx, logger) {
         return null;
     const username = session.author?.nickname || session.author?.username || session.username || '未知用户';
     const originalContent = session.content || extractTextFromElements(elements) || '';
-    // 1. 检查图片
+    // 1. 检查图片（排除表情包：subType=1）
     if (config.enableImage) {
         for (const elem of elements) {
             if (elem.type === 'img' && elem.attrs?.src) {
+                // 表情包 subType 为 1，跳过
+                const subType = elem.attrs['sub-type'] ?? elem.attrs.subType;
+                if (subType === 1 || subType === '1') {
+                    if (config.debug) {
+                        logger.info('跳过表情包');
+                    }
+                    continue;
+                }
                 const duplicate = await processImage(elem.attrs.src, session, username, originalContent, config, ctx, logger);
                 if (duplicate)
                     return duplicate;
@@ -116,6 +124,9 @@ async function processMessage(session, config, ctx, logger) {
     }
     // 3. 检查转发消息
     if (config.enableForward) {
+        if (config.debug) {
+            logger.info(`检查转发消息, elements types: ${elements.map(e => e.type).join(', ')}`);
+        }
         for (const elem of elements) {
             if (elem.type === 'forward') {
                 const duplicate = await processForward(elem, session, username, originalContent, config, ctx, logger);
@@ -149,6 +160,13 @@ async function processImage(imageUrl, session, username, originalContent, config
             }
             return null;
         }
+        // 检测异常哈希（如全0或几乎全0），跳过处理避免误判
+        if (isAbnormalHash(hash)) {
+            if (config.debug) {
+                logger.warn(`异常图片哈希，跳过: ${hash}`);
+            }
+            return null;
+        }
         if (config.debug) {
             logger.info(`图片哈希: ${hash}`);
         }
@@ -217,33 +235,136 @@ async function processLink(url, session, username, originalContent, config, ctx,
 }
 async function processForward(forwardElem, session, username, originalContent, config, ctx, logger) {
     try {
-        const content = extractForwardContent(forwardElem);
-        if (!content || content.length < 10) {
+        if (config.debug) {
+            logger.info(`处理转发消息, elem: ${JSON.stringify(forwardElem.attrs, null, 2)}`);
+        }
+        // 获取转发消息 ID
+        const forwardId = forwardElem.attrs?.id;
+        if (!forwardId) {
             return null;
         }
-        const truncated = content.slice(0, config.forwardContentMaxLength);
-        const hash = (0, hash_1.calculateStringHash)(truncated);
+        // 转发消息内容
+        let textParts = [];
+        let images = [];
+        let apiSuccess = false;
+        // 先尝试通过 OneBot API 获取转发消息内容
+        if (session.platform === 'onebot' && session.bot?.internal) {
+            const internal = session.bot.internal;
+            // 尝试多种 payload 格式
+            const payloads = [
+                { message_id: forwardId },
+                { id: forwardId },
+            ];
+            // 尝试多种 API 调用方式
+            const callApi = async (action, params) => {
+                if (typeof internal._get === 'function') {
+                    return await internal._get(action, params);
+                }
+                if (typeof internal.request === 'function') {
+                    return await internal.request(action, params);
+                }
+                if (typeof internal.callAction === 'function') {
+                    return await internal.callAction(action, params);
+                }
+                return null;
+            };
+            for (const payload of payloads) {
+                try {
+                    const forwardData = await callApi('get_forward_msg', payload);
+                    if (forwardData) {
+                        const messages = extractMessagesArray(forwardData);
+                        if (messages && Array.isArray(messages) && messages.length > 0) {
+                            apiSuccess = true;
+                            if (config.debug) {
+                                logger.info(`转发消息节点数量: ${messages.length}`);
+                            }
+                            // 提取文本和图片
+                            for (const node of messages) {
+                                const msgArray = node.message || node.content || node.data;
+                                if (Array.isArray(msgArray)) {
+                                    for (const m of msgArray) {
+                                        if (m.type === 'text') {
+                                            textParts.push(m.data?.text || m.text || '');
+                                        }
+                                        if (m.type === 'image') {
+                                            const url = m.data?.url || m.data?.file || m.url || m.file || '';
+                                            if (url) {
+                                                // 下载图片并计算哈希
+                                                let hash = null;
+                                                try {
+                                                    hash = await (0, hash_1.downloadAndHashImage)(url, ctx);
+                                                    if (hash && isAbnormalHash(hash)) {
+                                                        if (config.debug) {
+                                                            logger.warn(`转发消息图片哈希异常，跳过: ${hash}`);
+                                                        }
+                                                        hash = null;
+                                                    }
+                                                }
+                                                catch (err) {
+                                                    if (config.debug) {
+                                                        logger.warn(`转发消息图片下载失败: ${url.slice(0, 50)}...`);
+                                                    }
+                                                }
+                                                images.push({ url, hash });
+                                            }
+                                        }
+                                    }
+                                }
+                                // 兼容字符串格式
+                                if (typeof node.content === 'string') {
+                                    textParts.push(node.content);
+                                }
+                                if (typeof node.text === 'string') {
+                                    textParts.push(node.text);
+                                }
+                            }
+                            break;
+                        }
+                    }
+                }
+                catch (err) {
+                    // API 调用失败，继续尝试
+                }
+            }
+        }
+        // 提取有效图片哈希列表
+        const imageHashes = images.filter(img => img.hash !== null).map(img => img.hash);
+        const failedImages = images.filter(img => img.hash === null).length;
+        // 计算文本哈希
+        const textContent = textParts.join('\n').trim();
+        const textToHash = textContent.slice(0, config.forwardContentMaxLength);
+        const textHash = textContent ? (0, hash_1.calculateStringHash)(textToHash) : (0, hash_1.calculateStringHash)(forwardId);
         if (config.debug) {
-            logger.info(`转发消息哈希: ${hash}, 内容长度: ${content.length}`);
+            logger.info(`转发消息: 文本长度=${textToHash.length}, 图片=${imageHashes.length}, 失败=${failedImages}, API成功=${apiSuccess}`);
         }
         const guildId = session.guildId;
-        const duplicate = await (0, database_1.findDuplicate)(ctx, guildId, 'forward', hash);
+        // 使用新的比较函数查询重复消息
+        const duplicate = await (0, database_1.compareForwardMessages)(ctx, guildId, textHash, imageHashes, config.forwardImageMatchMode, config.forwardImageSimilarityThreshold);
         if (duplicate) {
             if (config.debug) {
                 logger.info(`发现重复转发消息`);
             }
             return duplicate;
         }
+        // 保存记录
+        const extraInfo = {
+            forwardId,
+            preview: textToHash.slice(0, 100),
+            textHash,
+            imageHashes,
+            imageCount: images.length,
+            failedImages
+        };
         await (0, database_1.saveRecord)(ctx, {
             guildId: session.guildId,
             userId: session.userId,
             username,
             timestamp: Date.now(),
             contentType: 'forward',
-            contentHash: hash,
+            contentHash: textHash,
             originalMessageId: session.messageId,
-            originalContent: truncated.slice(0, 100),
-            extraInfo: JSON.stringify({ preview: truncated.slice(0, 100) })
+            originalContent: textToHash.slice(0, 100),
+            extraInfo: JSON.stringify(extraInfo)
         });
         return null;
     }
@@ -269,6 +390,28 @@ function extractForwardContent(elem) {
     }
     return content;
 }
+/**
+ * 从多种可能的数据结构中提取 messages 数组
+ * 参考 chatluna-forward-msg 的实现
+ */
+function extractMessagesArray(data) {
+    const candidates = [
+        data,
+        data?.messages,
+        data?.data,
+        data?.result,
+        data?.response,
+        data?.data?.messages,
+        data?.response?.messages,
+        data?.envelope?.result?.messages,
+    ];
+    for (const item of candidates) {
+        if (Array.isArray(item)) {
+            return item;
+        }
+    }
+    return null;
+}
 async function sendDuplicateWarning(session, duplicate, config, ctx) {
     const date = new Date(duplicate.timestamp);
     const dateStr = date.toLocaleString('zh-CN', {
@@ -343,3 +486,31 @@ function getRandomSticker(baseDir, stickerDir) {
     }
     return null;
 }
+/**
+ * 检测哈希是否异常（如几乎全0或全1）
+ * 异常哈希会导致误判，应跳过处理
+ */
+function isAbnormalHash(hash) {
+    // pHash 通常是64位十六进制或二进制
+    // 统计0和1的比例，如果比例极端则认为异常
+    // 如果是二进制格式（64位）
+    if (hash.length === 64 && /^[01]+$/.test(hash)) {
+        const zeros = hash.split('0').length - 1;
+        const ones = hash.split('1').length - 1;
+        // 如果超过90%是同一个值，认为异常
+        if (zeros >= 58 || ones >= 58) {
+            return true;
+        }
+    }
+    // 如果是十六进制格式
+    if (/^[0-9a-fA-F]+$/.test(hash)) {
+        // 检查是否几乎全是0或几乎全是f
+        const nonZeroCount = hash.replace(/0/gi, '').length;
+        const nonFCount = hash.replace(/f/gi, '').length;
+        // 如果超过90%是同一个值
+        if (nonZeroCount <= hash.length * 0.1 || nonFCount <= hash.length * 0.1) {
+            return true;
+        }
+    }
+    return false;
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "koishi-plugin-message-dedup",
-  "version": "0.0.1",
+  "version": "0.0.3",
   "description": "消息去重插件，检测群内重复的图片、链接、聊天记录",
   "main": "lib/index.js",
   "typings": "lib/index.d.ts",