@dan-uni/dan-any-plugin-detaolu 0.9.2 → 1.0.1

package/dist/index.umd.min.js.LICENSE.txt

@@ -37,7 +37,10 @@ and limitations under the License.
 
 /**
  * @author: xmcp(代码主要逻辑来源)
- * @see: https://github.com/xmcp/pakku.js
+ * @see: https://github.com/xmcp/pakku.js/blob/master/pakkujs/core/combine_worker.ts
+ * @see: https://github.com/xmcp/pakku.js/blob/master/pakkujs/background/config.ts
+ * @see: https://github.com/xmcp/pakku.js/blob/master/pakkujs/page/options.html
+ * @see: https://github.com/xmcp/pakku.js/blob/master/pakkujs/page/options.ts
  * @license: GPL-3.0
  * 本文件内代码来源见上，经部分修改，并整合config注释
  */

package/dist/pakku.js/index.d.ts

@@ -1,42 +1,65 @@
 /**
  * @author: xmcp(代码主要逻辑来源)
- * @see: https://github.com/xmcp/pakku.js
+ * @see: https://github.com/xmcp/pakku.js/blob/master/pakkujs/core/combine_worker.ts
+ * @see: https://github.com/xmcp/pakku.js/blob/master/pakkujs/background/config.ts
+ * @see: https://github.com/xmcp/pakku.js/blob/master/pakkujs/page/options.html
+ * @see: https://github.com/xmcp/pakku.js/blob/master/pakkujs/page/options.ts
  * @license: GPL-3.0
  * 本文件内代码来源见上，经部分修改，并整合config注释
  */
 import type { DanmuChunk, DanmuClusterOutput, DanmuObject } from './types';
 export declare const DEFAULT_CONFIG: {
     /**
-     * 时间阈值(合并n秒内的弹幕)：
+     * 时间阈值：合并时间差在n秒之内的重复弹幕
      * 超长（大概 60 秒以上？）的阈值可能会导致程序运行缓慢
      */
     THRESHOLD: number;
     /**
      * 编辑距离合并阈值：
-     * 根据编辑距离判断不完全一致但内容相近（例如有错别字）的弹幕,
-     * 能有效击杀 "<code>你指尖跃动的电光</code>" 和 "<code>你<b>之间</b>跃动的电光</code>" 等
-     * @example 禁用(0),轻微(≤3),中等(≤5),强力(≤8)
+     * 根据编辑距离判断不完全一致但内容相近（例如有错别字）的弹幕
+     * 能有效击杀 "你指尖跃动的电光" 和 "你之间跃动的电光" 等
+     * @example 禁用(0), 轻微(≤3), 中等(≤5), 强力(≤8)
      */
     MAX_DIST: number;
     /**
      * 词频向量合并阈值：
-     * 根据 2-Gram 频率向量的夹角判断不完全一致但内容类似的弹幕,
-     * 能有效击杀 "<code>yeah!~</code>" 和 "<code>yeah!~yeah!~yeah!~yeah!~</code>" 等
-     * @example 禁用(1000),轻微(60%),中等(45%),强力(30%)
+     * 根据 2-Gram 频率向量的夹角判断不完全一致但内容类似的弹幕
+     * 能有效击杀 "yeah!~" 和 "yeah!~yeah!~yeah!~yeah!~" 等
+     * @example 禁用(1000), 轻微(60%), 中等(45%), 强力(30%)
      */
     MAX_COSINE: number;
     /**
      * 识别谐音弹幕：
-     * 将常用汉字转换为拼音再进行比较,
-     * 能有效击杀 "<code>布拉迪巴特福来</code>" 和 "<code>布拉迪·八德福莱</code>" 等
+     * 将常用汉字转换为拼音再进行比较
+     * 能有效击杀 "布拉迪巴特福来" 和 "布拉迪·八德福莱" 等
      */
     TRIM_PINYIN: boolean;
     TRIM_ENDING: boolean;
     TRIM_SPACE: boolean;
     TRIM_WIDTH: boolean;
+    /**
+     * 内容替换：符合这些规则的弹幕，判断是否合并前会先对内容进行替换
+     */
     FORCELIST: string[][];
+    /**
+     * 内容替换规则命中时：继续尝试匹配后续规则
+     */
+    FORCELIST_CONTINUE_ON_MATCH: boolean;
+    /**
+     * 内容替换规则命中时：即使未触发合并也使用替换后的文本
+     */
+    FORCELIST_APPLY_SINGULAR: boolean;
+    /**
+     * 强制忽略：符合这些规则的弹幕不会被合并，优先级高于内容替换规则
+     */
     WHITELIST: [string, string][];
+    /**
+     * 强制删除：符合这些规则的弹幕会直接被删除（未实现）
+     */
     BLACKLIST: [string, string][];
+    /**
+     * 合并不同类型的弹幕（取消勾选后，底部弹幕不会跟滚动弹幕合并到一起）
+     */
     CROSS_MODE: boolean;
     PROC_TYPE7: boolean;
     PROC_TYPE4: boolean;

package/package.json

@@ -1,21 +1,21 @@
 {
   "name": "@dan-uni/dan-any-plugin-detaolu",
-  "version": "0.9.2",
+  "version": "1.0.1",
   "description": "A filter, dedupe and anti-spam plugin of dan-any, a danmaku transformer lib, based on pakku.js.",
-  "keywords": [
-    "bangumi",
-    "danmaku"
-  ],
+  "author": "rinne",
   "license": "GPL-3.0-or-later",
   "homepage": "https://github.com/ani-uni/danuni/tree/master/packages/dan-any-plugin-detaolu#readme",
-  "bugs": {
-    "url": "https://github.com/ani-uni/danuni/issues"
-  },
   "repository": {
     "type": "git",
     "url": "git+https://github.com/ani-uni/danuni.git"
   },
-  "author": "rinne",
+  "bugs": {
+    "url": "https://github.com/ani-uni/danuni/issues"
+  },
+  "keywords": [
+    "bangumi",
+    "danmaku"
+  ],
   "main": "dist/index.js",
   "module": "src/index.ts",
   "types": "dist/index.d.ts",
@@ -31,7 +31,7 @@
   },
   "dependencies": {
     "@dan-uni/dan-any": "workspace:^",
-    "fs-extra": "^11.3.0"
+    "fs-extra": "^11.3.3"
   },
   "devDependencies": {
     "@types/fs-extra": "^11.0.4"

package/src/index.ts

@@ -22,8 +22,8 @@ async function detaolu(that: UniPool, config?: DeTaoLuConfig) {
     if (p.danuni_dans.length === 1) {
       return p.danuni_dans[0].danuni_dan
     } else {
-      const dans = p.danuni_dans,
-        pool = new UniPool(dans.map((d) => d.danuni_dan))
+      const dans = p.danuni_dans
+      const pool = new UniPool(dans.map((d) => d.danuni_dan))
       function isAllBottomMode(p: UniPool) {
         return p.dans.every((d) => d.mode === UniDMTools.Modes.Bottom)
       }

package/src/pakku.js/index.ts

@@ -1,6 +1,9 @@
 /**
  * @author: xmcp(代码主要逻辑来源)
- * @see: https://github.com/xmcp/pakku.js
+ * @see: https://github.com/xmcp/pakku.js/blob/master/pakkujs/core/combine_worker.ts
+ * @see: https://github.com/xmcp/pakku.js/blob/master/pakkujs/background/config.ts
+ * @see: https://github.com/xmcp/pakku.js/blob/master/pakkujs/page/options.html
+ * @see: https://github.com/xmcp/pakku.js/blob/master/pakkujs/page/options.ts
  * @license: GPL-3.0
  * 本文件内代码来源见上，经部分修改，并整合config注释
  */
@@ -19,28 +22,28 @@ import { Queue, Stats } from './types'
 export const DEFAULT_CONFIG = {
   // 弹幕合并
   /**
-   * 时间阈值(合并n秒内的弹幕)：
+   * 时间阈值：合并时间差在n秒之内的重复弹幕
    * 超长（大概 60 秒以上？）的阈值可能会导致程序运行缓慢
    */
   THRESHOLD: 30,
   /**
    * 编辑距离合并阈值：
-   * 根据编辑距离判断不完全一致但内容相近（例如有错别字）的弹幕,
-   * 能有效击杀 "<code>你指尖跃动的电光</code>" 和 "<code>你<b>之间</b>跃动的电光</code>" 等
-   * @example 禁用(0),轻微(≤3),中等(≤5),强力(≤8)
+   * 根据编辑距离判断不完全一致但内容相近（例如有错别字）的弹幕
+   * 能有效击杀 "你指尖跃动的电光" 和 "你之间跃动的电光" 等
+   * @example 禁用(0), 轻微(≤3), 中等(≤5), 强力(≤8)
    */
   MAX_DIST: 5,
   /**
    * 词频向量合并阈值：
-   * 根据 2-Gram 频率向量的夹角判断不完全一致但内容类似的弹幕,
-   * 能有效击杀 "<code>yeah!~</code>" 和 "<code>yeah!~yeah!~yeah!~yeah!~</code>" 等
-   * @example 禁用(1000),轻微(60%),中等(45%),强力(30%)
+   * 根据 2-Gram 频率向量的夹角判断不完全一致但内容类似的弹幕
+   * 能有效击杀 "yeah!~" 和 "yeah!~yeah!~yeah!~yeah!~" 等
+   * @example 禁用(1000), 轻微(60%), 中等(45%), 强力(30%)
    */
   MAX_COSINE: 45,
   /**
    * 识别谐音弹幕：
-   * 将常用汉字转换为拼音再进行比较,
-   * 能有效击杀 "<code>布拉迪巴特福来</code>" 和 "<code>布拉迪·八德福莱</code>" 等
+   * 将常用汉字转换为拼音再进行比较
+   * 能有效击杀 "布拉迪巴特福来" 和 "布拉迪·八德福莱" 等
    */
   TRIM_PINYIN: true,
   // 比较文本时：
@@ -49,15 +52,35 @@ export const DEFAULT_CONFIG = {
   TRIM_WIDTH: true, // 忽略全半角差异
 
   // 例外设置
+  /**
+   * 内容替换：符合这些规则的弹幕，判断是否合并前会先对内容进行替换
+   */
   FORCELIST: [
     ['^23{2,}$', '23333'],
     ['^6{3,}$', '66666'],
-  ], // 强制合并（符合这些规则的弹幕，在比较是否相同时会先进行替换）
-  WHITELIST: [] as [string, string][], // 强制忽略（符合这些规则的弹幕，即使内容相同也不会被合并）
-  BLACKLIST: [] as [string, string][], // 强制删除（符合这些规则的弹幕，会直接被删除）
-  CROSS_MODE: true, // 合并不同类型的弹幕(取消勾选后，底部弹幕不会跟滚动弹幕合并到一起)
+  ],
+  /**
+   * 内容替换规则命中时：继续尝试匹配后续规则
+   */
+  FORCELIST_CONTINUE_ON_MATCH: true,
+  /**
+   * 内容替换规则命中时：即使未触发合并也使用替换后的文本
+   */
+  FORCELIST_APPLY_SINGULAR: false,
+  /**
+   * 强制忽略：符合这些规则的弹幕不会被合并，优先级高于内容替换规则
+   */
+  WHITELIST: [] as [string, string][],
+  /**
+   * 强制删除：符合这些规则的弹幕会直接被删除（未实现）
+   */
+  BLACKLIST: [] as [string, string][],
+  /**
+   * 合并不同类型的弹幕（取消勾选后，底部弹幕不会跟滚动弹幕合并到一起）
+   */
+  CROSS_MODE: true,
   // 放过特定类型的弹幕：
-  PROC_TYPE7: true, // 高级弹幕
+  PROC_TYPE7: true, // 高级弹幕（特殊弹幕）
   PROC_TYPE4: true, // 底部弹幕
   PROC_POOL1: false, // 字幕弹幕(位于弹幕池1)
 
@@ -90,7 +113,7 @@ export type Config = Partial<typeof DEFAULT_CONFIG>
 interface DanmuIr {
   obj: DanmuObject
   str: string // for similarity algorithm
-  idx: int
+  ptr_idx: int
   sim_reason: string
 }
 
@@ -201,13 +224,14 @@ const WIDTH_TABLE = new Map(
 /**
  * 反套路
  */
-const detaolu = (inp: string, config: Config) => {
+const detaolu = (inp: string, config: Config): [boolean, string] => {
   const TRIM_ENDING = config.TRIM_ENDING
   const TRIM_SPACE = config.TRIM_SPACE
   const TRIM_WIDTH = config.TRIM_WIDTH
   const FORCELIST = (config?.FORCELIST ?? DEFAULT_CONFIG.FORCELIST).map(
-    ([pattern, repl]) => [new RegExp(pattern, 'gi'), repl] as [RegExp, string],
+    ([pattern, repl]) => [new RegExp(pattern, 'giu'), repl] as [RegExp, string],
   )
+  const FORCELIST_BREAK_ON_MATCH = !config.FORCELIST_CONTINUE_ON_MATCH
 
   let len = inp.length
   let text = ''
@@ -242,14 +266,16 @@ const detaolu = (inp: string, config: Config) => {
       )
   }
 
+  let taolu_matched = false
   for (const taolu of FORCELIST) {
     if (taolu[0].test(text)) {
       text = text.replace(taolu[0], taolu[1])
-      return [true, text]
+      taolu_matched = true
+      if (FORCELIST_BREAK_ON_MATCH) break
     }
   }
 
-  return [false, text]
+  return [taolu_matched, text]
 }
 
 /**
@@ -257,7 +283,7 @@ const detaolu = (inp: string, config: Config) => {
  */
 const whitelisted = (text: string, config: Config) => {
   const WHITELIST = (config?.WHITELIST ?? DEFAULT_CONFIG.WHITELIST).map(
-    (x) => new RegExp(x[0], 'i'),
+    (x) => new RegExp(x[0], 'iu'),
   )
   if (WHITELIST.length === 0) return false
   else return WHITELIST.some((re) => re.test(text))
@@ -308,7 +334,7 @@ function trim_dispstr(text: string): string {
 function select_median_length(strs: string[]): string {
   if (strs.length === 1) return strs[0]
 
-  const sorted = strs.sort((a, b) => a.length - b.length)
+  const sorted = strs.toSorted((a, b) => a.length - b.length)
   const mid = Math.floor(sorted.length / 2)
   return sorted[mid]
 }
@@ -316,10 +342,14 @@ function select_median_length(strs: string[]): string {
 async function load_wasm(wasm_mod?: ArrayBuffer) {
   await sim_init(
     wasm_mod ??
-      (await fs.readFile(new URL('./similarity-gen.wasm', import.meta.url))),
+      (await fs.readFile(new URL('similarity-gen.wasm', import.meta.url))),
   )
 }
 
+function make_ptr_idx(idx: int, is_next_chunk: boolean): int {
+  return is_next_chunk ? -1 - idx : idx
+}
+
 async function merge(
   chunk: DanmuChunk<DanmuObject>,
   // next_chunk: DanmuChunk<DanmuObject>,
@@ -349,9 +379,9 @@ async function merge(
   function apply_cluster(irs: DanmuIr[]) {
     if (irs.length === 1) {
       ret.clusters.push({
-        peers_ptr: irs.map((ir) => [ir.idx, ir.sim_reason]),
+        peers_ptr: irs.map((ir) => [ir.ptr_idx, ir.sim_reason]),
         desc: [],
-        chosen_str: irs[0].obj.content, // do not use detaolued str for single danmu
+        chosen_str: irs[0].obj.content,
         // danuni
         danuni_count: irs.length,
         // danuni_senders: irs.map((ir) => ir.obj.danuni_sender),
@@ -359,8 +389,8 @@ async function merge(
       })
     } else {
       const text_cnts = new Map()
-      let most_texts: string[] = [],
-        most_cnt = 0
+      let most_texts: string[] = []
+      let most_cnt = 0
 
       for (const ir of irs) {
         const text = ir.str
@@ -378,7 +408,7 @@ async function merge(
       const most_text = select_median_length(most_texts)
 
       ret.clusters.push({
-        peers_ptr: irs.map((ir) => [ir.idx, ir.sim_reason]),
+        peers_ptr: irs.map((ir) => [ir.ptr_idx, ir.sim_reason]),
         desc: most_cnt > 1 ? [`采用了出现 ${most_cnt} 次的文本`] : [],
         chosen_str: most_text,
         // danuni
@@ -389,7 +419,11 @@ async function merge(
     }
   }
 
-  function obj_to_ir(objs: DanmuObject[], s: Stats | null): DanmuIr[] {
+  function obj_to_ir(
+    objs: DanmuObject[],
+    s: Stats | null,
+    is_next_chunk: boolean,
+  ): DanmuIr[] {
     return objs
       .map((obj, idx) => {
         if (!config.PROC_POOL1 && obj.pool === 1) {
@@ -465,22 +499,27 @@ async function merge(
 
         const [matched_taolu, detaolued] = detaolu(disp_str, config)
 
-        if (matched_taolu && s) {
-          s.num_taolu_matched++
+        if (matched_taolu) {
+          if (s) s.num_taolu_matched++
+          if (config.FORCELIST_APPLY_SINGULAR)
+            obj = {
+              ...obj,
+              content: detaolued,
+            }
         }
 
         return {
           obj,
           str: detaolued,
-          idx,
+          ptr_idx: make_ptr_idx(idx, is_next_chunk),
           sim_reason: 'ORIG',
         }
       })
       .filter((obj) => obj !== null) as DanmuIr[]
   }
 
-  const danmus = obj_to_ir(chunk.objs, ret.stats)
-  // const next_chunk_danmus = obj_to_ir(next_chunk.objs, null)
+  const danmus = obj_to_ir(chunk.objs, ret.stats, false)
+  // const next_chunk_danmus = obj_to_ir(next_chunk.objs, null, true)
 
   const nearby_danmus: Queue<DanmuIr[]> = new Queue()
 
@@ -504,13 +543,13 @@ async function merge(
       nearby_danmus.index_l,
       ret.stats,
     )
-    if (sim !== null) {
+    if (sim === null) {
+      nearby_danmus.push([dm])
+    } else {
       const candidate =
         nearby_danmus.storage[nearby_danmus.index_r - sim.idx_diff]
       dm.sim_reason = sim.reason
       candidate.push(dm)
-    } else {
-      nearby_danmus.push([dm])
     }
   }
 

package/src/pakku.js/similarity-gen.js

@@ -1,18 +1,5 @@
-/* eslint-disable getter-return */
-/* eslint-disable unicorn/catch-error-name */
-/* eslint-disable unicorn/prefer-number-properties */
-/* eslint-disable block-scoped-var */
-/* eslint-disable object-shorthand */
-/* eslint-disable no-setter-return */
-/* eslint-disable prefer-template */
-/* eslint-disable unicorn/new-for-builtins */
-/* eslint-disable unicorn/throw-new-error */
-/* eslint-disable no-console */
-/* eslint-disable unicorn/consistent-function-scoping */
-/* eslint-disable no-unused-expressions */
-/* eslint-disable vars-on-top */
-/* eslint-disable import/no-mutable-exports */
-/* eslint-disable import/no-default-export */
+/* eslint-disable @eslint-community/eslint-comments/no-unlimited-disable */
+/* eslint-disable */
 /** @nocollapse */ var Module = function (moduleArg = {}) {
   var moduleRtn
 

package/src/pakku.js/similarity_stub.ts

@@ -75,20 +75,34 @@ export function detect_similarity(
   const idx_diff = ret & ((1 << 19) - 1)
 
   let reason_str
-  if (reason === CombinedReason.combined_identical) {
-    S.combined_identical++
-    reason_str = '=='
-  } else if (reason === CombinedReason.combined_edit_distance) {
-    S.combined_edit_distance++
-    reason_str = `≤${dist}`
-  } else if (reason === CombinedReason.combined_cosine_distance) {
-    S.combined_cosine_distance++
-    reason_str = `${dist}%`
-  } else if (reason === CombinedReason.combined_pinyin_distance) {
-    S.combined_pinyin_distance++
-    reason_str = `P≤${dist}`
-  } else {
-    throw new Error(`similarity wasm returned unknown reason: ${ret}`)
+  switch (reason) {
+    case CombinedReason.combined_identical: {
+      S.combined_identical++
+      reason_str = '=='
+
+      break
+    }
+    case CombinedReason.combined_edit_distance: {
+      S.combined_edit_distance++
+      reason_str = `≤${dist}`
+
+      break
+    }
+    case CombinedReason.combined_cosine_distance: {
+      S.combined_cosine_distance++
+      reason_str = `${dist}%`
+
+      break
+    }
+    case CombinedReason.combined_pinyin_distance: {
+      S.combined_pinyin_distance++
+      reason_str = `P≤${dist}`
+
+      break
+    }
+    default: {
+      throw new Error(`similarity wasm returned unknown reason: ${ret}`)
+    }
   }
 
   return { reason: reason_str, idx_diff }

package/tsconfig.json

@@ -2,14 +2,14 @@
   "compilerOptions": {
     /* Visit https://aka.ms/tsconfig to read more about this file */
     /* Projects */
-    // "incremental": true,                              /* Save .tsbuildinfo files to allow for incremental compilation of projects. */
+    "incremental": true /* Save .tsbuildinfo files to allow for incremental compilation of projects. */,
     // "composite": true,                                /* Enable constraints that allow a TypeScript project to be used with project references. */
     // "tsBuildInfoFile": "./.tsbuildinfo",              /* Specify the path to .tsbuildinfo incremental compilation file. */
     // "disableSourceOfProjectReferenceRedirect": true,  /* Disable preferring source files instead of declaration files when referencing composite projects. */
     // "disableSolutionSearching": true,                 /* Opt a project out of multi-project reference checking when editing. */
     // "disableReferencedProjectLoad": true,             /* Reduce the number of projects loaded automatically by TypeScript. */
     /* Language and Environment */
-    "target": "ES2021" /* Set the JavaScript language version for emitted JavaScript and include compatible library declarations. */,
+    "target": "ES2023" /* Set the JavaScript language version for emitted JavaScript and include compatible library declarations. */,
     // "lib": [],                                        /* Specify a set of bundled library declaration files that describe the target runtime environment. */
     // "jsx": "preserve",                                /* Specify what JSX code is generated. */
     // "experimentalDecorators": true,                   /* Enable experimental support for legacy experimental decorators. */

package/types/tsconfig.tsbuildinfo

@@ -0,0 +1 @@
+{"version":"5.9.3"}