koishi-plugin-best-cave 2.2.5 → 2.2.7

package/lib/index.js

@@ -666,26 +666,51 @@ var HashManager = class {
    */
   async generateHashesForHistoricalCaves() {
     const allCaves = await this.ctx.database.get("cave", { status: "active" });
-    const existingHashedCaveIds = new Set((await this.ctx.database.get("cave_hash", {}, { fields: ["cave"] })).map((h4) => h4.cave));
+    const existingHashes = await this.ctx.database.get("cave_hash", {}, { fields: ["cave", "hash", "type"] });
+    const existingHashSet = new Set(existingHashes.map((h4) => `${h4.cave}-${h4.hash}-${h4.type}`));
+    const processedCaveIds = new Set(existingHashes.map((h4) => h4.cave));
+    const cavesToProcess = allCaves.filter((cave) => !processedCaveIds.has(cave.id));
+    const totalToProcessCount = cavesToProcess.length;
+    if (totalToProcessCount === 0) {
+      return "无需补全回声洞哈希";
+    }
+    this.logger.info(`开始补全 ${totalToProcessCount} 个回声洞的哈希...`);
     let hashesToInsert = [];
-    let historicalCount = 0;
+    const batchHashSet = /* @__PURE__ */ new Set();
+    let processedCaveCount = 0;
     let totalHashesGenerated = 0;
-    for (const cave of allCaves) {
-      if (existingHashedCaveIds.has(cave.id)) continue;
-      historicalCount++;
-      const newHashesForCave = await this.generateAllHashesForCave(cave);
-      hashesToInsert.push(...newHashesForCave);
+    let errorCount = 0;
+    const flushBatch = /* @__PURE__ */ __name(async () => {
+      const batchSize = hashesToInsert.length;
+      if (batchSize === 0) return;
+      await this.ctx.database.upsert("cave_hash", hashesToInsert);
+      totalHashesGenerated += batchSize;
+      this.logger.info(`正在导入 ${batchSize} 条回声洞哈希... (已处理 ${processedCaveCount}/${totalToProcessCount})`);
+      hashesToInsert = [];
+      batchHashSet.clear();
+    }, "flushBatch");
+    for (const cave of cavesToProcess) {
+      processedCaveCount++;
+      try {
+        const newHashesForCave = await this.generateAllHashesForCave(cave);
+        for (const hashObj of newHashesForCave) {
+          const uniqueKey = `${hashObj.cave}-${hashObj.hash}-${hashObj.type}`;
+          if (!existingHashSet.has(uniqueKey) && !batchHashSet.has(uniqueKey)) {
+            hashesToInsert.push(hashObj);
+            batchHashSet.add(uniqueKey);
+          }
+        }
+      } catch (error) {
+        errorCount++;
+        this.logger.warn(`补全回声洞（${cave.id}）时发生错误: ${error.message}`);
+        continue;
+      }
       if (hashesToInsert.length >= 100) {
-        await this.ctx.database.upsert("cave_hash", hashesToInsert);
-        totalHashesGenerated += hashesToInsert.length;
-        hashesToInsert = [];
+        await flushBatch();
       }
     }
-    if (hashesToInsert.length > 0) {
-      await this.ctx.database.upsert("cave_hash", hashesToInsert);
-      totalHashesGenerated += hashesToInsert.length;
-    }
-    return totalHashesGenerated > 0 ? `已补全 ${historicalCount} 个回声洞的 ${totalHashesGenerated} 条哈希` : "无需补全回声洞哈希";
+    await flushBatch();
+    return `已补全 ${totalToProcessCount} 个回声洞的 ${totalHashesGenerated} 条哈希（失败${errorCount} 条）`;
   }
   /**
    * @description 为单个回声洞对象生成所有类型的哈希。
@@ -697,7 +722,9 @@ var HashManager = class {
     const combinedText = cave.elements.filter((el) => el.type === "text" && el.content).map((el) => el.content).join(" ");
     if (combinedText) {
       const textHash = this.generateTextSimhash(combinedText);
-      allHashes.push({ cave: cave.id, hash: textHash, type: "simhash" });
+      if (textHash) {
+        allHashes.push({ cave: cave.id, hash: textHash, type: "simhash" });
+      }
     }
     for (const el of cave.elements.filter((el2) => el2.type === "image" && el2.file)) {
       try {
@@ -916,11 +943,25 @@ var HashManager = class {
    * @returns {string} 64位二进制 Simhash 对应的16位十六进制字符串。
    */
   generateTextSimhash(text) {
-    if (!text?.trim()) return "";
-    const tokens = text.toLowerCase().split(/[^a-z0-9\u4e00-\u9fa5]+/).filter(Boolean);
-    if (tokens.length === 0) return "";
+    const cleanText = (text || "").toLowerCase().replace(/\s+/g, "");
+    if (!cleanText) {
+      return "";
+    }
+    const n = 2;
+    const tokens = /* @__PURE__ */ new Set();
+    if (cleanText.length < n) {
+      tokens.add(cleanText);
+    } else {
+      for (let i = 0; i <= cleanText.length - n; i++) {
+        tokens.add(cleanText.substring(i, i + n));
+      }
+    }
+    const tokenArray = Array.from(tokens);
+    if (tokenArray.length === 0) {
+      return "";
+    }
     const vector = new Array(64).fill(0);
-    tokens.forEach((token) => {
+    tokenArray.forEach((token) => {
       const hash = crypto.createHash("md5").update(token).digest();
       for (let i = 0; i < 64; i++) {
         vector[i] += hash[Math.floor(i / 8)] >> i % 8 & 1 ? 1 : -1;
@@ -1040,14 +1081,16 @@ function apply(ctx, config) {
         const combinedText = finalElementsForDb.filter((el) => el.type === "text" && el.content).map((el) => el.content).join(" ");
         if (combinedText) {
           const newSimhash = hashManager.generateTextSimhash(combinedText);
-          const existingTextHashes = await ctx.database.get("cave_hash", { type: "simhash" });
-          for (const existing of existingTextHashes) {
-            const similarity = hashManager.calculateSimilarity(newSimhash, existing.hash);
-            if (similarity >= config.textThreshold) {
-              return `文本与回声洞（${existing.cave}）的相似度为 ${(similarity * 100).toFixed(2)}%，超过阈值`;
+          if (newSimhash) {
+            const existingTextHashes = await ctx.database.get("cave_hash", { type: "simhash" });
+            for (const existing of existingTextHashes) {
+              const similarity = hashManager.calculateSimilarity(newSimhash, existing.hash);
+              if (similarity >= config.textThreshold) {
+                return `文本与回声洞（${existing.cave}）的相似度为 ${(similarity * 100).toFixed(2)}%，超过阈值`;
+              }
             }
+            textHashesToStore.push({ hash: newSimhash, type: "simhash" });
           }
-          textHashesToStore.push({ hash: newSimhash, type: "simhash" });
         }
       }
       const userName = (config.enableProfile ? await profileManager.getNickname(session.userId) : null) || session.username;

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "koishi-plugin-best-cave",
   "description": "功能强大、高度可定制的回声洞。支持丰富的媒体类型、内容查重、人工审核、用户昵称、数据迁移以及本地/S3 双重文件存储后端。",
-  "version": "2.2.5",
+  "version": "2.2.7",
   "contributors": [
     "Yis_Rime <yis_rime@outlook.com>"
   ],