@dan-uni/dan-any-plugin-detaolu 0.9.2 → 1.0.1

package/dist/index.js

@@ -50,7 +50,7 @@ var Module = function(moduleArg = {}) {
         return 'FS_createPath' === a || 'FS_createDataFile' === a || 'FS_createPreloadedFile' === a || 'FS_unlink' === a || 'addRunDependency' === a || 'FS_createLazyFile' === a || 'FS_createDevice' === a || 'removeRunDependency' === a;
     }
     function L(a, b) {
-        'undefined' == typeof globalThis || Object.getOwnPropertyDescriptor(globalThis, a) || Object.defineProperty(globalThis, a, {
+        "u" < typeof globalThis || Object.getOwnPropertyDescriptor(globalThis, a) || Object.defineProperty(globalThis, a, {
             configurable: !0,
             get () {
                 b();
@@ -77,7 +77,7 @@ var Module = function(moduleArg = {}) {
     var N = (a)=>{
         N.g || (N.g = {});
         N.g[a] || (N.g[a] = 1, t(a));
-    }, P = 'undefined' != typeof TextDecoder ? new TextDecoder() : void 0, R = [
+    }, P = "u" > typeof TextDecoder ? new TextDecoder() : void 0, R = [
         null,
         [],
         []
@@ -270,19 +270,26 @@ function detect_similarity(str, mode, index_l, S) {
     const dist = ret >>> 19 & 2047;
     const idx_diff = 524287 & ret;
     let reason_str;
-    if (0 === reason) {
-        S.combined_identical++;
-        reason_str = '==';
-    } else if (1 === reason) {
-        S.combined_edit_distance++;
-        reason_str = `\u{2264}${dist}`;
-    } else if (3 === reason) {
-        S.combined_cosine_distance++;
-        reason_str = `${dist}%`;
-    } else if (2 === reason) {
-        S.combined_pinyin_distance++;
-        reason_str = `P\u{2264}${dist}`;
-    } else throw new Error(`similarity wasm returned unknown reason: ${ret}`);
+    switch(reason){
+        case 0:
+            S.combined_identical++;
+            reason_str = '==';
+            break;
+        case 1:
+            S.combined_edit_distance++;
+            reason_str = `≤${dist}`;
+            break;
+        case 3:
+            S.combined_cosine_distance++;
+            reason_str = `${dist}%`;
+            break;
+        case 2:
+            S.combined_pinyin_distance++;
+            reason_str = `P≤${dist}`;
+            break;
+        default:
+            throw new Error(`similarity wasm returned unknown reason: ${ret}`);
+    }
     return {
         reason: reason_str,
         idx_diff
@@ -344,7 +351,10 @@ class Queue {
 }
 /**
  * @author: xmcp(代码主要逻辑来源)
- * @see: https://github.com/xmcp/pakku.js
+ * @see: https://github.com/xmcp/pakku.js/blob/master/pakkujs/core/combine_worker.ts
+ * @see: https://github.com/xmcp/pakku.js/blob/master/pakkujs/background/config.ts
+ * @see: https://github.com/xmcp/pakku.js/blob/master/pakkujs/page/options.html
+ * @see: https://github.com/xmcp/pakku.js/blob/master/pakkujs/page/options.ts
  * @license: GPL-3.0
  * 本文件内代码来源见上，经部分修改，并整合config注释
  */ const DEFAULT_CONFIG = {
@@ -365,6 +375,8 @@ class Queue {
             '66666'
         ]
     ],
+    FORCELIST_CONTINUE_ON_MATCH: true,
+    FORCELIST_APPLY_SINGULAR: false,
     WHITELIST: [],
     BLACKLIST: [],
     CROSS_MODE: true,
@@ -372,51 +384,51 @@ class Queue {
     PROC_TYPE4: true,
     PROC_POOL1: false
 };
-const ENDING_CHARS = new Set(".\u3002,\uFF0C/?\uFF1F!\uFF01\u2026~\uFF5E@^\u3001+=-_\u2642\u2640 ");
+const ENDING_CHARS = new Set('.。,，/?？!！…~～@^、+=-_♂♀ ');
 const WIDTH_TABLE = new Map(Object.entries({
-    "\u3000": ' ',
-    "\uFF11": '1',
-    "\uFF12": '2',
-    "\uFF13": '3',
-    "\uFF14": '4',
-    "\uFF15": '5',
-    "\uFF16": '6',
-    "\uFF17": '7',
-    "\uFF18": '8',
-    "\uFF19": '9',
-    "\uFF10": '0',
-    '!': "\uFF01",
-    "\uFF20": '@',
-    "\uFF03": '#',
-    "\uFF04": '$',
-    "\uFF05": '%',
-    "\uFF3E": '^',
-    "\uFF06": '&',
-    "\uFF0A": '*',
-    "\uFF08": '(',
-    "\uFF09": ')',
-    "\uFF0D": '-',
-    "\uFF1D": '=',
-    "\uFF3F": '_',
-    "\uFF0B": '+',
-    "\uFF3B": '[',
-    "\uFF3D": ']',
-    "\uFF5B": '{',
-    "\uFF5D": '}',
-    ';': "\uFF1B",
-    "\uFF07": "'",
-    ':': "\uFF1A",
-    "\uFF02": '"',
-    ',': "\uFF0C",
-    "\uFF0E": '.',
-    "\uFF0F": '/',
-    "\uFF1C": '<',
-    "\uFF1E": '>',
-    '?': "\uFF1F",
-    "\uFF3C": '\\',
-    "\uFF5C": '|',
-    "\uFF40": '`',
-    "\uFF5E": '~',
+    '　': ' ',
+    '１': '1',
+    '２': '2',
+    '３': '3',
+    '４': '4',
+    '５': '5',
+    '６': '6',
+    '７': '7',
+    '８': '8',
+    '９': '9',
+    '０': '0',
+    '!': '！',
+    '＠': '@',
+    '＃': '#',
+    '＄': '$',
+    '％': '%',
+    '＾': '^',
+    '＆': '&',
+    '＊': '*',
+    '（': '(',
+    '）': ')',
+    '－': '-',
+    '＝': '=',
+    '＿': '_',
+    '＋': '+',
+    '［': '[',
+    '］': ']',
+    '｛': '{',
+    '｝': '}',
+    ';': '；',
+    '＇': "'",
+    ':': '：',
+    '＂': '"',
+    ',': '，',
+    '．': '.',
+    '／': '/',
+    '＜': '<',
+    '＞': '>',
+    '?': '？',
+    '＼': '\\',
+    '｜': '|',
+    '｀': '`',
+    '～': '~',
     ｑ: 'q',
     ｗ: 'w',
     ｅ: 'e',
@@ -475,9 +487,10 @@ const detaolu = (inp, config)=>{
     const TRIM_SPACE = config.TRIM_SPACE;
     const TRIM_WIDTH = config.TRIM_WIDTH;
     const FORCELIST = (config?.FORCELIST ?? DEFAULT_CONFIG.FORCELIST).map(([pattern, repl])=>[
-            new RegExp(pattern, 'gi'),
+            new RegExp(pattern, 'giu'),
             repl
         ]);
+    const FORCELIST_BREAK_ON_MATCH = !config.FORCELIST_CONTINUE_ON_MATCH;
     let len = inp.length;
     let text = '';
     if (TRIM_ENDING) {
@@ -490,20 +503,19 @@ const detaolu = (inp, config)=>{
     }
     else text = inp.slice(0, len);
     if (TRIM_SPACE) text = text.replaceAll(/[ \u3000]+/g, ' ').replaceAll(/([\u3000-\u9FFF\uFF00-\uFFEF]) (?=[\u3000-\u9FFF\uFF00-\uFFEF])/g, '$1');
+    let taolu_matched = false;
     for (const taolu of FORCELIST)if (taolu[0].test(text)) {
         text = text.replace(taolu[0], taolu[1]);
-        return [
-            true,
-            text
-        ];
+        taolu_matched = true;
+        if (FORCELIST_BREAK_ON_MATCH) break;
     }
     return [
-        false,
+        taolu_matched,
         text
     ];
 };
 const whitelisted = (text, config)=>{
-    const WHITELIST = (config?.WHITELIST ?? DEFAULT_CONFIG.WHITELIST).map((x)=>new RegExp(x[0], 'i'));
+    const WHITELIST = (config?.WHITELIST ?? DEFAULT_CONFIG.WHITELIST).map((x)=>new RegExp(x[0], 'iu'));
     if (0 === WHITELIST.length) return false;
     return WHITELIST.some((re)=>re.test(text));
 };
@@ -530,12 +542,15 @@ function trim_dispstr(text) {
 }
 function select_median_length(strs) {
     if (1 === strs.length) return strs[0];
-    const sorted = strs.sort((a, b)=>a.length - b.length);
+    const sorted = strs.toSorted((a, b)=>a.length - b.length);
     const mid = Math.floor(sorted.length / 2);
     return sorted[mid];
 }
 async function load_wasm(wasm_mod) {
-    await similarity_stub_init(wasm_mod ?? await fs_extra.readFile(new URL('./similarity-gen.wasm', import.meta.url)));
+    await similarity_stub_init(wasm_mod ?? await fs_extra.readFile(new URL('similarity-gen.wasm', import.meta.url)));
+}
+function make_ptr_idx(idx, is_next_chunk) {
+    return is_next_chunk ? -1 - idx : idx;
 }
 async function merge(chunk, config = DEFAULT_CONFIG) {
     await load_wasm();
@@ -566,7 +581,7 @@ async function merge(chunk, config = DEFAULT_CONFIG) {
     function apply_cluster(irs) {
         if (1 === irs.length) ret.clusters.push({
             peers_ptr: irs.map((ir)=>[
-                    ir.idx,
+                    ir.ptr_idx,
                     ir.sim_reason
                 ]),
             desc: [],
@@ -576,7 +591,8 @@ async function merge(chunk, config = DEFAULT_CONFIG) {
         });
         else {
             const text_cnts = new Map();
-            let most_texts = [], most_cnt = 0;
+            let most_texts = [];
+            let most_cnt = 0;
             for (const ir of irs){
                 const text = ir.str;
                 const cnt = 1 + (text_cnts.get(text) || 0);
@@ -591,11 +607,11 @@ async function merge(chunk, config = DEFAULT_CONFIG) {
             const most_text = select_median_length(most_texts);
             ret.clusters.push({
                 peers_ptr: irs.map((ir)=>[
-                        ir.idx,
+                        ir.ptr_idx,
                         ir.sim_reason
                     ]),
                 desc: most_cnt > 1 ? [
-                    `\u{91C7}\u{7528}\u{4E86}\u{51FA}\u{73B0} ${most_cnt} \u{6B21}\u{7684}\u{6587}\u{672C}`
+                    `采用了出现 ${most_cnt} 次的文本`
                 ] : [],
                 chosen_str: most_text,
                 danuni_count: most_cnt,
@@ -603,26 +619,26 @@ async function merge(chunk, config = DEFAULT_CONFIG) {
             });
         }
     }
-    function obj_to_ir(objs, s) {
+    function obj_to_ir(objs, s, is_next_chunk) {
         return objs.map((obj, idx)=>{
             if (!config.PROC_POOL1 && 1 === obj.pool) {
                 if (s) {
                     s.ignored_type++;
-                    apply_single_cluster(idx, obj, "\u5DF2\u5FFD\u7565\u5B57\u5E55\u5F39\u5E55\uFF0C\u53EF\u4EE5\u5728\u9009\u9879\u4E2D\u4FEE\u6539");
+                    apply_single_cluster(idx, obj, '已忽略字幕弹幕，可以在选项中修改');
                 }
                 return null;
             }
             if (!config.PROC_TYPE7 && 4 === obj.mode) {
                 if (s) {
                     s.ignored_type++;
-                    apply_single_cluster(idx, obj, "\u5DF2\u5FFD\u7565\u7279\u6B8A\u5F39\u5E55\uFF0C\u53EF\u4EE5\u5728\u9009\u9879\u4E2D\u4FEE\u6539");
+                    apply_single_cluster(idx, obj, '已忽略特殊弹幕，可以在选项中修改');
                 }
                 return null;
             }
             if (!config.PROC_TYPE4 && 1 === obj.mode) {
                 if (s) {
                     s.ignored_type++;
-                    apply_single_cluster(idx, obj, "\u5DF2\u5FFD\u7565\u5E95\u90E8\u5F39\u5E55\uFF0C\u53EF\u4EE5\u5728\u9009\u9879\u4E2D\u4FEE\u6539");
+                    apply_single_cluster(idx, obj, '已忽略底部弹幕，可以在选项中修改');
                 }
                 return null;
             }
@@ -636,7 +652,7 @@ async function merge(chunk, config = DEFAULT_CONFIG) {
                         ret.deleted_chunk.push({
                             ...obj,
                             pakku: {
-                                deleted_reason: `\u{547D}\u{4E2D}\u{9ED1}\u{540D}\u{5355}\u{FF1A}${matched}`
+                                deleted_reason: `命中黑名单：${matched}`
                             }
                         });
                     }
@@ -646,21 +662,27 @@ async function merge(chunk, config = DEFAULT_CONFIG) {
             if (whitelisted(disp_str, config)) {
                 if (s) {
                     s.ignored_whitelist++;
-                    apply_single_cluster(idx, obj, "\u547D\u4E2D\u767D\u540D\u5355");
+                    apply_single_cluster(idx, obj, '命中白名单');
                 }
                 return null;
             }
             const [matched_taolu, detaolued] = detaolu(disp_str, config);
-            if (matched_taolu && s) s.num_taolu_matched++;
+            if (matched_taolu) {
+                if (s) s.num_taolu_matched++;
+                if (config.FORCELIST_APPLY_SINGULAR) obj = {
+                    ...obj,
+                    content: detaolued
+                };
+            }
             return {
                 obj,
                 str: detaolued,
-                idx,
+                ptr_idx: make_ptr_idx(idx, is_next_chunk),
                 sim_reason: 'ORIG'
             };
         }).filter((obj)=>null !== obj);
     }
-    const danmus = obj_to_ir(chunk.objs, ret.stats);
+    const danmus = obj_to_ir(chunk.objs, ret.stats, false);
     const nearby_danmus = new Queue();
     const THRESHOLD_MS = (config?.THRESHOLD ?? DEFAULT_CONFIG.THRESHOLD) * 1000;
     for (const dm of danmus){
@@ -671,13 +693,14 @@ async function merge(chunk, config = DEFAULT_CONFIG) {
             nearby_danmus.pop();
         }
         const sim = detect_similarity(dm.str, dm.obj.mode, nearby_danmus.index_l, ret.stats);
-        if (null !== sim) {
+        if (null === sim) nearby_danmus.push([
+            dm
+        ]);
+        else {
             const candidate = nearby_danmus.storage[nearby_danmus.index_r - sim.idx_diff];
             dm.sim_reason = sim.reason;
             candidate.push(dm);
-        } else nearby_danmus.push([
-            dm
-        ]);
+        }
     }
     begin_index_lock();
     for (const candidate of nearby_danmus)apply_cluster(candidate);
@@ -697,7 +720,8 @@ async function src_detaolu(that, config) {
     const selected = p.clusters.map((p)=>{
         if (1 === p.danuni_dans.length) return p.danuni_dans[0].danuni_dan;
         {
-            const dans = p.danuni_dans, pool = new UniPool(dans.map((d)=>d.danuni_dan));
+            const dans = p.danuni_dans;
+            const pool = new UniPool(dans.map((d)=>d.danuni_dan));
             function isAllBottomMode(p) {
                 return p.dans.every((d)=>d.mode === UniDMTools.Modes.Bottom);
             }
@@ -736,4 +760,4 @@ function detaolu_constructor(config) {
     return (that)=>src_detaolu(that, config);
 }
 const src = detaolu_constructor;
-export { src as default };
+export default src;

package/dist/index.js.LICENSE.txt

@@ -1,6 +1,9 @@
 /**
  * @author: xmcp(代码主要逻辑来源)
- * @see: https://github.com/xmcp/pakku.js
+ * @see: https://github.com/xmcp/pakku.js/blob/master/pakkujs/core/combine_worker.ts
+ * @see: https://github.com/xmcp/pakku.js/blob/master/pakkujs/background/config.ts
+ * @see: https://github.com/xmcp/pakku.js/blob/master/pakkujs/page/options.html
+ * @see: https://github.com/xmcp/pakku.js/blob/master/pakkujs/page/options.ts
  * @license: GPL-3.0
  * 本文件内代码来源见上，经部分修改，并整合config注释
  */