@dan-uni/dan-any-plugin-detaolu 0.7.3 → 0.9.5

package/dist/index.js

@@ -270,19 +270,26 @@ function detect_similarity(str, mode, index_l, S) {
     const dist = ret >>> 19 & 2047;
     const idx_diff = 524287 & ret;
     let reason_str;
-    if (0 === reason) {
-        S.combined_identical++;
-        reason_str = '==';
-    } else if (1 === reason) {
-        S.combined_edit_distance++;
-        reason_str = `\u{2264}${dist}`;
-    } else if (3 === reason) {
-        S.combined_cosine_distance++;
-        reason_str = `${dist}%`;
-    } else if (2 === reason) {
-        S.combined_pinyin_distance++;
-        reason_str = `P\u{2264}${dist}`;
-    } else throw new Error(`similarity wasm returned unknown reason: ${ret}`);
+    switch(reason){
+        case 0:
+            S.combined_identical++;
+            reason_str = '==';
+            break;
+        case 1:
+            S.combined_edit_distance++;
+            reason_str = `≤${dist}`;
+            break;
+        case 3:
+            S.combined_cosine_distance++;
+            reason_str = `${dist}%`;
+            break;
+        case 2:
+            S.combined_pinyin_distance++;
+            reason_str = `P≤${dist}`;
+            break;
+        default:
+            throw new Error(`similarity wasm returned unknown reason: ${ret}`);
+    }
     return {
         reason: reason_str,
         idx_diff
@@ -344,7 +351,7 @@ class Queue {
 }
 /**
  * @author: xmcp(代码主要逻辑来源)
- * @see: https://github.com/xmcp/pakku.js
+ * @see: https://github.com/xmcp/pakku.js/blob/master/pakkujs/core/combine_worker.ts
  * @license: GPL-3.0
  * 本文件内代码来源见上，经部分修改，并整合config注释
  */ const DEFAULT_CONFIG = {
@@ -372,51 +379,51 @@ class Queue {
     PROC_TYPE4: true,
     PROC_POOL1: false
 };
-const ENDING_CHARS = new Set(".\u3002,\uFF0C/?\uFF1F!\uFF01\u2026~\uFF5E@^\u3001+=-_\u2642\u2640 ");
+const ENDING_CHARS = new Set('.。,，/?？!！…~～@^、+=-_♂♀ ');
 const WIDTH_TABLE = new Map(Object.entries({
-    "\u3000": ' ',
-    "\uFF11": '1',
-    "\uFF12": '2',
-    "\uFF13": '3',
-    "\uFF14": '4',
-    "\uFF15": '5',
-    "\uFF16": '6',
-    "\uFF17": '7',
-    "\uFF18": '8',
-    "\uFF19": '9',
-    "\uFF10": '0',
-    '!': "\uFF01",
-    "\uFF20": '@',
-    "\uFF03": '#',
-    "\uFF04": '$',
-    "\uFF05": '%',
-    "\uFF3E": '^',
-    "\uFF06": '&',
-    "\uFF0A": '*',
-    "\uFF08": '(',
-    "\uFF09": ')',
-    "\uFF0D": '-',
-    "\uFF1D": '=',
-    "\uFF3F": '_',
-    "\uFF0B": '+',
-    "\uFF3B": '[',
-    "\uFF3D": ']',
-    "\uFF5B": '{',
-    "\uFF5D": '}',
-    ';': "\uFF1B",
-    "\uFF07": "'",
-    ':': "\uFF1A",
-    "\uFF02": '"',
-    ',': "\uFF0C",
-    "\uFF0E": '.',
-    "\uFF0F": '/',
-    "\uFF1C": '<',
-    "\uFF1E": '>',
-    '?': "\uFF1F",
-    "\uFF3C": '\\',
-    "\uFF5C": '|',
-    "\uFF40": '`',
-    "\uFF5E": '~',
+    '　': ' ',
+    '１': '1',
+    '２': '2',
+    '３': '3',
+    '４': '4',
+    '５': '5',
+    '６': '6',
+    '７': '7',
+    '８': '8',
+    '９': '9',
+    '０': '0',
+    '!': '！',
+    '＠': '@',
+    '＃': '#',
+    '＄': '$',
+    '％': '%',
+    '＾': '^',
+    '＆': '&',
+    '＊': '*',
+    '（': '(',
+    '）': ')',
+    '－': '-',
+    '＝': '=',
+    '＿': '_',
+    '＋': '+',
+    '［': '[',
+    '］': ']',
+    '｛': '{',
+    '｝': '}',
+    ';': '；',
+    '＇': "'",
+    ':': '：',
+    '＂': '"',
+    ',': '，',
+    '．': '.',
+    '／': '/',
+    '＜': '<',
+    '＞': '>',
+    '?': '？',
+    '＼': '\\',
+    '｜': '|',
+    '｀': '`',
+    '～': '~',
     ｑ: 'q',
     ｗ: 'w',
     ｅ: 'e',
@@ -530,19 +537,23 @@ function trim_dispstr(text) {
 }
 function select_median_length(strs) {
     if (1 === strs.length) return strs[0];
-    const sorted = strs.sort((a, b)=>a.length - b.length);
+    const sorted = strs.toSorted((a, b)=>a.length - b.length);
     const mid = Math.floor(sorted.length / 2);
     return sorted[mid];
 }
 async function load_wasm(wasm_mod) {
-    await similarity_stub_init(wasm_mod ?? await fs_extra.readFile(new URL('./similarity-gen.wasm', import.meta.url)));
+    await similarity_stub_init(wasm_mod ?? await fs_extra.readFile(new URL('similarity-gen.wasm', import.meta.url)));
+}
+function make_ptr_idx(idx, is_next_chunk) {
+    return is_next_chunk ? -1 - idx : idx;
 }
 async function merge(chunk, config = DEFAULT_CONFIG) {
     await load_wasm();
     begin_chunk(config);
     const ret = {
         clusters: [],
-        stats: new Stats()
+        stats: new Stats(),
+        deleted_chunk: []
     };
     function apply_single_cluster(idx, obj, desc) {
         ret.clusters.push({
@@ -565,7 +576,7 @@ async function merge(chunk, config = DEFAULT_CONFIG) {
     function apply_cluster(irs) {
         if (1 === irs.length) ret.clusters.push({
             peers_ptr: irs.map((ir)=>[
-                    ir.idx,
+                    ir.ptr_idx,
                     ir.sim_reason
                 ]),
             desc: [],
@@ -575,7 +586,8 @@ async function merge(chunk, config = DEFAULT_CONFIG) {
         });
         else {
             const text_cnts = new Map();
-            let most_texts = [], most_cnt = 0;
+            let most_texts = [];
+            let most_cnt = 0;
             for (const ir of irs){
                 const text = ir.str;
                 const cnt = 1 + (text_cnts.get(text) || 0);
@@ -590,11 +602,11 @@ async function merge(chunk, config = DEFAULT_CONFIG) {
             const most_text = select_median_length(most_texts);
             ret.clusters.push({
                 peers_ptr: irs.map((ir)=>[
-                        ir.idx,
+                        ir.ptr_idx,
                         ir.sim_reason
                     ]),
                 desc: most_cnt > 1 ? [
-                    `\u{91C7}\u{7528}\u{4E86}\u{51FA}\u{73B0} ${most_cnt} \u{6B21}\u{7684}\u{6587}\u{672C}`
+                    `采用了出现 ${most_cnt} 次的文本`
                 ] : [],
                 chosen_str: most_text,
                 danuni_count: most_cnt,
@@ -602,26 +614,26 @@ async function merge(chunk, config = DEFAULT_CONFIG) {
             });
         }
     }
-    function obj_to_ir(objs, s) {
+    function obj_to_ir(objs, s, is_next_chunk) {
         return objs.map((obj, idx)=>{
             if (!config.PROC_POOL1 && 1 === obj.pool) {
                 if (s) {
                     s.ignored_type++;
-                    apply_single_cluster(idx, obj, "\u5DF2\u5FFD\u7565\u5B57\u5E55\u5F39\u5E55\uFF0C\u53EF\u4EE5\u5728\u9009\u9879\u4E2D\u4FEE\u6539");
+                    apply_single_cluster(idx, obj, '已忽略字幕弹幕，可以在选项中修改');
                 }
                 return null;
             }
             if (!config.PROC_TYPE7 && 4 === obj.mode) {
                 if (s) {
                     s.ignored_type++;
-                    apply_single_cluster(idx, obj, "\u5DF2\u5FFD\u7565\u7279\u6B8A\u5F39\u5E55\uFF0C\u53EF\u4EE5\u5728\u9009\u9879\u4E2D\u4FEE\u6539");
+                    apply_single_cluster(idx, obj, '已忽略特殊弹幕，可以在选项中修改');
                 }
                 return null;
             }
             if (!config.PROC_TYPE4 && 1 === obj.mode) {
                 if (s) {
                     s.ignored_type++;
-                    apply_single_cluster(idx, obj, "\u5DF2\u5FFD\u7565\u5E95\u90E8\u5F39\u5E55\uFF0C\u53EF\u4EE5\u5728\u9009\u9879\u4E2D\u4FEE\u6539");
+                    apply_single_cluster(idx, obj, '已忽略底部弹幕，可以在选项中修改');
                 }
                 return null;
             }
@@ -632,6 +644,12 @@ async function merge(chunk, config = DEFAULT_CONFIG) {
                     if (s) {
                         s.deleted_blacklist++;
                         s.deleted_blacklist_each[matched] = (s.deleted_blacklist_each[matched] || 0) + 1;
+                        ret.deleted_chunk.push({
+                            ...obj,
+                            pakku: {
+                                deleted_reason: `命中黑名单：${matched}`
+                            }
+                        });
                     }
                     return null;
                 }
@@ -639,7 +657,7 @@ async function merge(chunk, config = DEFAULT_CONFIG) {
             if (whitelisted(disp_str, config)) {
                 if (s) {
                     s.ignored_whitelist++;
-                    apply_single_cluster(idx, obj, "\u547D\u4E2D\u767D\u540D\u5355");
+                    apply_single_cluster(idx, obj, '命中白名单');
                 }
                 return null;
             }
@@ -648,12 +666,12 @@ async function merge(chunk, config = DEFAULT_CONFIG) {
             return {
                 obj,
                 str: detaolued,
-                idx,
+                ptr_idx: make_ptr_idx(idx, is_next_chunk),
                 sim_reason: 'ORIG'
             };
         }).filter((obj)=>null !== obj);
     }
-    const danmus = obj_to_ir(chunk.objs, ret.stats);
+    const danmus = obj_to_ir(chunk.objs, ret.stats, false);
     const nearby_danmus = new Queue();
     const THRESHOLD_MS = (config?.THRESHOLD ?? DEFAULT_CONFIG.THRESHOLD) * 1000;
     for (const dm of danmus){
@@ -664,13 +682,14 @@ async function merge(chunk, config = DEFAULT_CONFIG) {
             nearby_danmus.pop();
         }
         const sim = detect_similarity(dm.str, dm.obj.mode, nearby_danmus.index_l, ret.stats);
-        if (null !== sim) {
+        if (null === sim) nearby_danmus.push([
+            dm
+        ]);
+        else {
             const candidate = nearby_danmus.storage[nearby_danmus.index_r - sim.idx_diff];
             dm.sim_reason = sim.reason;
             candidate.push(dm);
-        } else nearby_danmus.push([
-            dm
-        ]);
+        }
     }
     begin_index_lock();
     for (const candidate of nearby_danmus)apply_cluster(candidate);
@@ -690,7 +709,8 @@ async function src_detaolu(that, config) {
     const selected = p.clusters.map((p)=>{
         if (1 === p.danuni_dans.length) return p.danuni_dans[0].danuni_dan;
         {
-            const dans = p.danuni_dans, pool = new UniPool(dans.map((d)=>d.danuni_dan));
+            const dans = p.danuni_dans;
+            const pool = new UniPool(dans.map((d)=>d.danuni_dan));
             function isAllBottomMode(p) {
                 return p.dans.every((d)=>d.mode === UniDMTools.Modes.Bottom);
             }

package/dist/index.js.LICENSE.txt

@@ -1,6 +1,6 @@
 /**
  * @author: xmcp(代码主要逻辑来源)
- * @see: https://github.com/xmcp/pakku.js
+ * @see: https://github.com/xmcp/pakku.js/blob/master/pakkujs/core/combine_worker.ts
  * @license: GPL-3.0
  * 本文件内代码来源见上，经部分修改，并整合config注释
  */