koishi-plugin-best-cave 2.2.6 → 2.2.8

package/lib/index.d.ts

@@ -39,7 +39,8 @@ export interface Config {
     caveFormat: string;
     enableSimilarity: boolean;
     textThreshold: number;
-    imageThreshold: number;
+    imageWholeThreshold: number;
+    imagePartThreshold: number;
     localPath?: string;
     enableS3: boolean;
     endpoint?: string;

package/lib/index.js

@@ -445,14 +445,14 @@ async function handleFileUploads(ctx, config, fileManager, logger2, reviewManage
         const similarityScores = /* @__PURE__ */ new Map();
         for (const existing of existingColorPHashes) {
           const similarity = hashManager.calculateSimilarity(colorPHash, existing.hash);
-          if (similarity >= config.imageThreshold) {
+          if (similarity >= config.imageWholeThreshold) {
             if (!similarityScores.has(existing.cave)) similarityScores.set(existing.cave, {});
             similarityScores.get(existing.cave).colorSim = similarity;
           }
         }
         for (const existing of existingDHashes) {
           const similarity = hashManager.calculateSimilarity(dHash, existing.hash);
-          if (similarity >= config.imageThreshold) {
+          if (similarity >= config.imageWholeThreshold) {
             if (!similarityScores.has(existing.cave)) similarityScores.set(existing.cave, {});
             similarityScores.get(existing.cave).dSim = similarity;
           }
@@ -475,7 +475,7 @@ async function handleFileUploads(ctx, config, fileManager, logger2, reviewManage
           for (const existing of existingSubHashObjects) {
             if (notifiedPartialCaves.has(existing.cave)) continue;
             const similarity = hashManager.calculateSimilarity(newSubHash, existing.hash);
-            if (similarity >= config.imageThreshold) {
+            if (similarity >= config.imagePartThreshold) {
               await session.send(`图片局部与回声洞（${existing.cave}）的相似度为 ${(similarity * 100).toFixed(2)}%`);
               notifiedPartialCaves.add(existing.cave);
             }
@@ -666,26 +666,51 @@ var HashManager = class {
    */
   async generateHashesForHistoricalCaves() {
     const allCaves = await this.ctx.database.get("cave", { status: "active" });
-    const existingHashedCaveIds = new Set((await this.ctx.database.get("cave_hash", {}, { fields: ["cave"] })).map((h4) => h4.cave));
+    const existingHashes = await this.ctx.database.get("cave_hash", {}, { fields: ["cave", "hash", "type"] });
+    const existingHashSet = new Set(existingHashes.map((h4) => `${h4.cave}-${h4.hash}-${h4.type}`));
+    const processedCaveIds = new Set(existingHashes.map((h4) => h4.cave));
+    const cavesToProcess = allCaves.filter((cave) => !processedCaveIds.has(cave.id));
+    const totalToProcessCount = cavesToProcess.length;
+    if (totalToProcessCount === 0) {
+      return "无需补全回声洞哈希";
+    }
+    this.logger.info(`开始补全 ${totalToProcessCount} 个回声洞的哈希...`);
     let hashesToInsert = [];
-    let historicalCount = 0;
+    const batchHashSet = /* @__PURE__ */ new Set();
+    let processedCaveCount = 0;
     let totalHashesGenerated = 0;
-    for (const cave of allCaves) {
-      if (existingHashedCaveIds.has(cave.id)) continue;
-      historicalCount++;
-      const newHashesForCave = await this.generateAllHashesForCave(cave);
-      hashesToInsert.push(...newHashesForCave);
+    let errorCount = 0;
+    const flushBatch = /* @__PURE__ */ __name(async () => {
+      const batchSize = hashesToInsert.length;
+      if (batchSize === 0) return;
+      await this.ctx.database.upsert("cave_hash", hashesToInsert);
+      totalHashesGenerated += batchSize;
+      this.logger.info(`正在导入 ${batchSize} 条回声洞哈希... (已处理 ${processedCaveCount}/${totalToProcessCount})`);
+      hashesToInsert = [];
+      batchHashSet.clear();
+    }, "flushBatch");
+    for (const cave of cavesToProcess) {
+      processedCaveCount++;
+      try {
+        const newHashesForCave = await this.generateAllHashesForCave(cave);
+        for (const hashObj of newHashesForCave) {
+          const uniqueKey = `${hashObj.cave}-${hashObj.hash}-${hashObj.type}`;
+          if (!existingHashSet.has(uniqueKey) && !batchHashSet.has(uniqueKey)) {
+            hashesToInsert.push(hashObj);
+            batchHashSet.add(uniqueKey);
+          }
+        }
+      } catch (error) {
+        errorCount++;
+        this.logger.warn(`补全回声洞（${cave.id}）时发生错误: ${error.message}`);
+        continue;
+      }
       if (hashesToInsert.length >= 100) {
-        await this.ctx.database.upsert("cave_hash", hashesToInsert);
-        totalHashesGenerated += hashesToInsert.length;
-        hashesToInsert = [];
+        await flushBatch();
       }
     }
-    if (hashesToInsert.length > 0) {
-      await this.ctx.database.upsert("cave_hash", hashesToInsert);
-      totalHashesGenerated += hashesToInsert.length;
-    }
-    return totalHashesGenerated > 0 ? `已补全 ${historicalCount} 个回声洞的 ${totalHashesGenerated} 条哈希` : "无需补全回声洞哈希";
+    await flushBatch();
+    return `已补全 ${totalToProcessCount} 个回声洞的 ${totalHashesGenerated} 条哈希（失败 ${errorCount} 条）`;
   }
   /**
    * @description 为单个回声洞对象生成所有类型的哈希。
@@ -743,20 +768,21 @@ var HashManager = class {
       phash_color: /* @__PURE__ */ new Map(),
       dhash_gray: /* @__PURE__ */ new Map()
     };
-    const subHashToCaves = /* @__PURE__ */ new Map();
+    const subHashGroups = /* @__PURE__ */ new Map();
     for (const hash of allHashes) {
       if (hashGroups[hash.type]) {
         if (!hashGroups[hash.type].has(hash.cave)) hashGroups[hash.type].set(hash.cave, []);
         hashGroups[hash.type].get(hash.cave).push(hash.hash);
       } else if (hash.type.startsWith("sub_phash_")) {
-        if (!subHashToCaves.has(hash.hash)) subHashToCaves.set(hash.hash, /* @__PURE__ */ new Set());
-        subHashToCaves.get(hash.hash).add(hash.cave);
+        if (!subHashGroups.has(hash.cave)) subHashGroups.set(hash.cave, []);
+        subHashGroups.get(hash.cave).push(hash.hash);
       }
     }
     const similarPairs = {
       text: /* @__PURE__ */ new Set(),
       image_color: /* @__PURE__ */ new Set(),
-      image_dhash: /* @__PURE__ */ new Set()
+      image_dhash: /* @__PURE__ */ new Set(),
+      image_part: /* @__PURE__ */ new Set()
     };
     for (let i = 0; i < allCaveIds.length; i++) {
       for (let j = i + 1; j < allCaveIds.length; j++) {
@@ -775,7 +801,7 @@ var HashManager = class {
         for (const h1 of colorHashes1) {
           for (const h22 of colorHashes2) {
             const sim = this.calculateSimilarity(h1, h22);
-            if (sim >= this.config.imageThreshold) {
+            if (sim >= this.config.imageWholeThreshold) {
               similarPairs.image_color.add(`${id1} & ${id2} = ${(sim * 100).toFixed(2)}%`);
             }
           }
@@ -785,27 +811,36 @@ var HashManager = class {
         for (const h1 of dHashes1) {
           for (const h22 of dHashes2) {
             const sim = this.calculateSimilarity(h1, h22);
-            if (sim >= this.config.imageThreshold) {
+            if (sim >= this.config.imageWholeThreshold) {
               similarPairs.image_dhash.add(`${id1} & ${id2} = ${(sim * 100).toFixed(2)}%`);
             }
           }
         }
+        const subHashes1 = subHashGroups.get(id1) || [];
+        const subHashes2 = subHashGroups.get(id2) || [];
+        if (subHashes1.length > 0 && subHashes2.length > 0) {
+          let maxPartSim = 0;
+          for (const h1 of subHashes1) {
+            for (const h22 of subHashes2) {
+              const sim = this.calculateSimilarity(h1, h22);
+              if (sim > maxPartSim) {
+                maxPartSim = sim;
+              }
+            }
+          }
+          if (maxPartSim >= this.config.imagePartThreshold) {
+            similarPairs.image_part.add(`${id1} & ${id2} = ${(maxPartSim * 100).toFixed(2)}%`);
+          }
+        }
       }
     }
-    const subHashDuplicates = [];
-    subHashToCaves.forEach((caves2) => {
-      if (caves2.size > 1) {
-        const sortedCaves = [...caves2].sort((a, b) => a - b).join(", ");
-        subHashDuplicates.push(`[${sortedCaves}]`);
-      }
-    });
-    const totalFindings = similarPairs.text.size + similarPairs.image_color.size + similarPairs.image_dhash.size + subHashDuplicates.length;
+    const totalFindings = similarPairs.text.size + similarPairs.image_color.size + similarPairs.image_dhash.size + similarPairs.image_part.size;
     if (totalFindings === 0) return "未发现高相似度的内容";
-    let report = `已发现 ${totalFindings} 组高相似度或重复的内容:`;
+    let report = `已发现 ${totalFindings} 组高相似度的内容:`;
     if (similarPairs.text.size > 0) report += "\n文本近似:\n" + [...similarPairs.text].join("\n");
-    if (similarPairs.image_color.size > 0) report += "\n图片整体相似:\n" + [...similarPairs.image_color].join("\n");
+    if (similarPairs.image_color.size > 0) report += "\n图片颜色相似:\n" + [...similarPairs.image_color].join("\n");
     if (similarPairs.image_dhash.size > 0) report += "\n图片结构相似:\n" + [...similarPairs.image_dhash].join("\n");
-    if (subHashDuplicates.length > 0) report += "\n图片局部重复:\n" + [...new Set(subHashDuplicates)].join("\n");
+    if (similarPairs.image_part.size > 0) report += "\n图片局部近似:\n" + [...similarPairs.image_part].join("\n");
     return report.trim();
   }
   /**
@@ -918,11 +953,25 @@ var HashManager = class {
    * @returns {string} 64位二进制 Simhash 对应的16位十六进制字符串。
    */
   generateTextSimhash(text) {
-    if (!text?.trim()) return "";
-    const tokens = text.toLowerCase().split(/[^a-z0-9\u4e00-\u9fa5]+/).filter(Boolean);
-    if (tokens.length === 0) return "";
+    const cleanText = (text || "").toLowerCase().replace(/\s+/g, "");
+    if (!cleanText) {
+      return "";
+    }
+    const n = 2;
+    const tokens = /* @__PURE__ */ new Set();
+    if (cleanText.length < n) {
+      tokens.add(cleanText);
+    } else {
+      for (let i = 0; i <= cleanText.length - n; i++) {
+        tokens.add(cleanText.substring(i, i + n));
+      }
+    }
+    const tokenArray = Array.from(tokens);
+    if (tokenArray.length === 0) {
+      return "";
+    }
     const vector = new Array(64).fill(0);
-    tokens.forEach((token) => {
+    tokenArray.forEach((token) => {
       const hash = crypto.createHash("md5").update(token).digest();
       for (let i = 0; i < 64; i++) {
         vector[i] += hash[Math.floor(i / 8)] >> i % 8 & 1 ? 1 : -1;
@@ -970,7 +1019,8 @@ var Config = import_koishi3.Schema.intersect([
     enableReview: import_koishi3.Schema.boolean().default(false).description("启用审核"),
     enableSimilarity: import_koishi3.Schema.boolean().default(false).description("启用查重"),
     textThreshold: import_koishi3.Schema.number().min(0).max(1).step(0.01).default(0.9).description("文本相似度阈值"),
-    imageThreshold: import_koishi3.Schema.number().min(0).max(1).step(0.01).default(0.9).description("图片相似度阈值")
+    imageWholeThreshold: import_koishi3.Schema.number().min(0).max(1).step(0.01).default(0.9).description("图片整体相似度阈值"),
+    imagePartThreshold: import_koishi3.Schema.number().min(0).max(1).step(0.01).default(0.95).description("图片局部相似度阈值")
   }).description("复核配置"),
   import_koishi3.Schema.object({
     localPath: import_koishi3.Schema.string().description("文件映射路径"),

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "koishi-plugin-best-cave",
   "description": "功能强大、高度可定制的回声洞。支持丰富的媒体类型、内容查重、人工审核、用户昵称、数据迁移以及本地/S3 双重文件存储后端。",
-  "version": "2.2.6",
+  "version": "2.2.8",
   "contributors": [
     "Yis_Rime <yis_rime@outlook.com>"
   ],