koishi-plugin-message-dedup 0.0.2 → 0.0.3

package/lib/config.d.ts

@@ -6,6 +6,8 @@ export interface Config {
     imageSimilarityThreshold: number;
     linkExactMatch: boolean;
     forwardContentMaxLength: number;
+    forwardImageMatchMode: 'all' | 'majority';
+    forwardImageSimilarityThreshold: number;
     retentionDays: number;
     stickerDir: string;
     sendMethod: 'koishi' | 'onebot';

package/lib/config.js

@@ -25,6 +25,17 @@ exports.Config = koishi_1.Schema.object({
         .min(100)
         .max(2000)
         .description('转发消息内容摘要最大长度'),
+    forwardImageMatchMode: koishi_1.Schema.union([
+        koishi_1.Schema.const('all').description('全部匹配'),
+        koishi_1.Schema.const('majority').description('过半匹配')
+    ])
+        .default('all')
+        .description('转发消息图片匹配模式：全部匹配要求所有图片相同，过半匹配只需超半数图片相同'),
+    forwardImageSimilarityThreshold: koishi_1.Schema.number()
+        .default(10)
+        .min(0)
+        .max(32)
+        .description('转发消息图片相似度阈值（百分比，0为完全相同）'),
     retentionDays: koishi_1.Schema.number()
         .default(7)
         .min(1)

package/lib/database.d.ts

@@ -16,7 +16,23 @@ export interface DedupRecord {
     originalContent: string;
     extraInfo: string;
 }
+/**
+ * 转发消息额外信息结构
+ */
+export interface ForwardExtraInfo {
+    forwardId: string;
+    preview: string;
+    textHash: string;
+    imageHashes: string[];
+    imageCount: number;
+    failedImages: number;
+}
 export declare function extendDatabase(ctx: Context): void;
 export declare function findDuplicate(ctx: Context, guildId: string, contentType: 'image' | 'link' | 'forward', contentHash: string, imageThreshold?: number): Promise<DedupRecord | null>;
 export declare function saveRecord(ctx: Context, record: DedupRecord): Promise<void>;
 export declare function cleanupOldRecords(ctx: Context, retentionDays: number): Promise<void>;
+/**
+ * 比较转发消息是否重复
+ * 使用文本哈希 + 图片哈希组合匹配
+ */
+export declare function compareForwardMessages(ctx: Context, guildId: string, newTextHash: string, newImageHashes: string[], imageMatchMode: 'all' | 'majority', imageThreshold: number): Promise<DedupRecord | null>;

package/lib/database.js

@@ -4,6 +4,7 @@ exports.extendDatabase = extendDatabase;
 exports.findDuplicate = findDuplicate;
 exports.saveRecord = saveRecord;
 exports.cleanupOldRecords = cleanupOldRecords;
+exports.compareForwardMessages = compareForwardMessages;
 const hash_1 = require("./hash");
 function extendDatabase(ctx) {
     ctx.model.extend('message_dedup', {
@@ -23,10 +24,12 @@ function extendDatabase(ctx) {
     });
 }
 async function findDuplicate(ctx, guildId, contentType, contentHash, imageThreshold) {
-    const records = await ctx.database.get('message_dedup', {
+    let records = await ctx.database.get('message_dedup', {
         guildId,
         contentType
     });
+    // 按时间戳升序排序，确保返回最早匹配的记录（原消息）
+    records = records.sort((a, b) => a.timestamp - b.timestamp);
     if (contentType === 'image' && imageThreshold !== undefined) {
         // 图片需要计算汉明距离
         // compareHashes返回0-1，0表示相同
@@ -60,3 +63,86 @@ async function cleanupOldRecords(ctx, retentionDays) {
         timestamp: { $lt: cutoffTime }
     });
 }
+/**
+ * 比较转发消息是否重复
+ * 使用文本哈希 + 图片哈希组合匹配
+ */
+async function compareForwardMessages(ctx, guildId, newTextHash, newImageHashes, imageMatchMode, imageThreshold) {
+    const records = await ctx.database.get('message_dedup', {
+        guildId,
+        contentType: 'forward'
+    });
+    // 按时间戳升序排序，确保返回最早匹配的记录（原消息）
+    const sortedRecords = records.sort((a, b) => a.timestamp - b.timestamp);
+    const thresholdRatio = imageThreshold / 100;
+    for (const record of sortedRecords) {
+        let extra;
+        try {
+            extra = JSON.parse(record.extraInfo);
+        }
+        catch {
+            // 兼容旧记录格式，跳过
+            continue;
+        }
+        // 1. 文本哈希必须匹配
+        if (extra.textHash !== newTextHash)
+            continue;
+        const newCount = newImageHashes.length;
+        const oldCount = extra.imageHashes?.length || 0;
+        // 2. 无图片情况：纯文本匹配
+        if (newCount === 0 && oldCount === 0) {
+            return record;
+        }
+        // 3. 单边无图片：不匹配
+        if (newCount === 0 || oldCount === 0) {
+            continue;
+        }
+        // 4. 全部匹配模式：数量必须相同
+        if (imageMatchMode === 'all' && newCount !== oldCount) {
+            continue;
+        }
+        // 5. 计算图片匹配数
+        const matchedCount = countImageMatches(newImageHashes, extra.imageHashes, thresholdRatio);
+        const totalCount = Math.max(newCount, oldCount);
+        // 6. 根据模式判断是否匹配
+        if (imageMatchMode === 'all') {
+            if (matchedCount === newCount && newCount === oldCount) {
+                return record;
+            }
+        }
+        else { // majority
+            if (matchedCount > totalCount / 2) {
+                return record;
+            }
+        }
+    }
+    return null;
+}
+/**
+ * 计算两个图片哈希列表的匹配数量
+ * 使用贪心算法：每个新图片找最相似的未匹配旧图片
+ */
+function countImageMatches(hashes1, hashes2, threshold // 0-1 之间的阈值
+) {
+    let matchCount = 0;
+    const used = new Set();
+    for (const h1 of hashes1) {
+        for (let i = 0; i < hashes2.length; i++) {
+            if (used.has(i))
+                continue;
+            try {
+                const distance = (0, hash_1.calculateHashDistance)(h1, hashes2[i]);
+                if (distance <= threshold) {
+                    matchCount++;
+                    used.add(i);
+                    break;
+                }
+            }
+            catch {
+                // 哈希格式错误，跳过
+                continue;
+            }
+        }
+    }
+    return matchCount;
+}

package/lib/index.js

@@ -160,6 +160,13 @@ async function processImage(imageUrl, session, username, originalContent, config
             }
             return null;
         }
+        // 检测异常哈希（如全0或几乎全0），跳过处理避免误判
+        if (isAbnormalHash(hash)) {
+            if (config.debug) {
+                logger.warn(`异常图片哈希，跳过: ${hash}`);
+            }
+            return null;
+        }
         if (config.debug) {
             logger.info(`图片哈希: ${hash}`);
         }
@@ -229,15 +236,18 @@ async function processLink(url, session, username, originalContent, config, ctx,
 async function processForward(forwardElem, session, username, originalContent, config, ctx, logger) {
     try {
         if (config.debug) {
-            logger.info(`处理转发消息, elem: ${JSON.stringify(forwardElem, null, 2)}`);
+            logger.info(`处理转发消息, elem: ${JSON.stringify(forwardElem.attrs, null, 2)}`);
         }
         // 获取转发消息 ID
         const forwardId = forwardElem.attrs?.id;
         if (!forwardId) {
             return null;
         }
+        // 转发消息内容
+        let textParts = [];
+        let images = [];
+        let apiSuccess = false;
         // 先尝试通过 OneBot API 获取转发消息内容
-        let content = '';
         if (session.platform === 'onebot' && session.bot?.internal) {
             const internal = session.bot.internal;
             // 尝试多种 payload 格式
@@ -264,17 +274,49 @@ async function processForward(forwardElem, session, username, originalContent, c
                     if (forwardData) {
                         const messages = extractMessagesArray(forwardData);
                         if (messages && Array.isArray(messages) && messages.length > 0) {
-                            content = messages.map((node) => {
-                                if (node.message) {
-                                    return node.message
-                                        .filter((m) => m.type === 'text')
-                                        .map((m) => m.data?.text || '')
-                                        .join('');
-                                }
-                                return '';
-                            }).join('\n');
+                            apiSuccess = true;
                             if (config.debug) {
-                                logger.info(`get_forward_msg成功获取内容, 长度: ${content.length}`);
+                                logger.info(`转发消息节点数量: ${messages.length}`);
+                            }
+                            // 提取文本和图片
+                            for (const node of messages) {
+                                const msgArray = node.message || node.content || node.data;
+                                if (Array.isArray(msgArray)) {
+                                    for (const m of msgArray) {
+                                        if (m.type === 'text') {
+                                            textParts.push(m.data?.text || m.text || '');
+                                        }
+                                        if (m.type === 'image') {
+                                            const url = m.data?.url || m.data?.file || m.url || m.file || '';
+                                            if (url) {
+                                                // 下载图片并计算哈希
+                                                let hash = null;
+                                                try {
+                                                    hash = await (0, hash_1.downloadAndHashImage)(url, ctx);
+                                                    if (hash && isAbnormalHash(hash)) {
+                                                        if (config.debug) {
+                                                            logger.warn(`转发消息图片哈希异常，跳过: ${hash}`);
+                                                        }
+                                                        hash = null;
+                                                    }
+                                                }
+                                                catch (err) {
+                                                    if (config.debug) {
+                                                        logger.warn(`转发消息图片下载失败: ${url.slice(0, 50)}...`);
+                                                    }
+                                                }
+                                                images.push({ url, hash });
+                                            }
+                                        }
+                                    }
+                                }
+                                // 兼容字符串格式
+                                if (typeof node.content === 'string') {
+                                    textParts.push(node.content);
+                                }
+                                if (typeof node.text === 'string') {
+                                    textParts.push(node.text);
+                                }
                             }
                             break;
                         }
@@ -285,35 +327,44 @@ async function processForward(forwardElem, session, username, originalContent, c
                 }
             }
         }
-        // 如果 API 获取失败，直接用 forwardId 作为去重标识
-        // 同一内容的转发消息在不同用户发送时应该有相同的 ID
-        const hashSource = content || forwardId;
-        if (config.debug) {
-            logger.info(`转发消息去重标识: ${hashSource.slice(0, 50)} (来源: ${content ? 'API内容' : 'forwardId'})`);
-        }
-        const truncated = hashSource.slice(0, config.forwardContentMaxLength);
-        const hash = (0, hash_1.calculateStringHash)(truncated);
+        // 提取有效图片哈希列表
+        const imageHashes = images.filter(img => img.hash !== null).map(img => img.hash);
+        const failedImages = images.filter(img => img.hash === null).length;
+        // 计算文本哈希
+        const textContent = textParts.join('\n').trim();
+        const textToHash = textContent.slice(0, config.forwardContentMaxLength);
+        const textHash = textContent ? (0, hash_1.calculateStringHash)(textToHash) : (0, hash_1.calculateStringHash)(forwardId);
         if (config.debug) {
-            logger.info(`转发消息哈希: ${hash}`);
+            logger.info(`转发消息: 文本长度=${textToHash.length}, 图片=${imageHashes.length}, 失败=${failedImages}, API成功=${apiSuccess}`);
         }
         const guildId = session.guildId;
-        const duplicate = await (0, database_1.findDuplicate)(ctx, guildId, 'forward', hash);
+        // 使用新的比较函数查询重复消息
+        const duplicate = await (0, database_1.compareForwardMessages)(ctx, guildId, textHash, imageHashes, config.forwardImageMatchMode, config.forwardImageSimilarityThreshold);
         if (duplicate) {
             if (config.debug) {
                 logger.info(`发现重复转发消息`);
             }
             return duplicate;
         }
+        // 保存记录
+        const extraInfo = {
+            forwardId,
+            preview: textToHash.slice(0, 100),
+            textHash,
+            imageHashes,
+            imageCount: images.length,
+            failedImages
+        };
         await (0, database_1.saveRecord)(ctx, {
             guildId: session.guildId,
             userId: session.userId,
             username,
             timestamp: Date.now(),
             contentType: 'forward',
-            contentHash: hash,
+            contentHash: textHash,
             originalMessageId: session.messageId,
-            originalContent: truncated.slice(0, 100),
-            extraInfo: JSON.stringify({ forwardId, preview: truncated.slice(0, 100) })
+            originalContent: textToHash.slice(0, 100),
+            extraInfo: JSON.stringify(extraInfo)
         });
         return null;
     }
@@ -435,3 +486,31 @@ function getRandomSticker(baseDir, stickerDir) {
     }
     return null;
 }
+/**
+ * 检测哈希是否异常（如几乎全0或全1）
+ * 异常哈希会导致误判，应跳过处理
+ */
+function isAbnormalHash(hash) {
+    // pHash 通常是64位十六进制或二进制
+    // 统计0和1的比例，如果比例极端则认为异常
+    // 如果是二进制格式（64位）
+    if (hash.length === 64 && /^[01]+$/.test(hash)) {
+        const zeros = hash.split('0').length - 1;
+        const ones = hash.split('1').length - 1;
+        // 如果超过90%是同一个值，认为异常
+        if (zeros >= 58 || ones >= 58) {
+            return true;
+        }
+    }
+    // 如果是十六进制格式
+    if (/^[0-9a-fA-F]+$/.test(hash)) {
+        // 检查是否几乎全是0或几乎全是f
+        const nonZeroCount = hash.replace(/0/gi, '').length;
+        const nonFCount = hash.replace(/f/gi, '').length;
+        // 如果超过90%是同一个值
+        if (nonZeroCount <= hash.length * 0.1 || nonFCount <= hash.length * 0.1) {
+            return true;
+        }
+    }
+    return false;
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "koishi-plugin-message-dedup",
-  "version": "0.0.2",
+  "version": "0.0.3",
   "description": "消息去重插件，检测群内重复的图片、链接、聊天记录",
   "main": "lib/index.js",
   "typings": "lib/index.d.ts",