@dan-uni/dan-any-plugin-detaolu 1.4.8 → 2.0.0

package/src/pakku.js/index.ts

@@ -1,615 +0,0 @@
-/**
- * @author: xmcp(代码主要逻辑来源)
- * @see: https://github.com/xmcp/pakku.js/blob/master/pakkujs/core/combine_worker.ts
- * @see: https://github.com/xmcp/pakku.js/blob/master/pakkujs/background/config.ts
- * @see: https://github.com/xmcp/pakku.js/blob/master/pakkujs/page/options.html
- * @see: https://github.com/xmcp/pakku.js/blob/master/pakkujs/page/options.ts
- * @license: GPL-3.0
- * 本文件内代码来源见上，经部分修改，并整合config注释
- */
-
-import fs from 'fs-extra'
-import type { DanmuChunk, DanmuClusterOutput, DanmuObject, int } from './types'
-
-import {
-  begin_chunk,
-  begin_index_lock,
-  detect_similarity,
-  init as sim_init,
-} from './similarity_stub'
-import { Queue, Stats } from './types'
-
-export const DEFAULT_CONFIG = {
-  // 弹幕合并
-  /**
-   * 时间阈值：合并时间差在n秒之内的重复弹幕
-   * 超长（大概 60 秒以上？）的阈值可能会导致程序运行缓慢
-   */
-  THRESHOLD: 30,
-  /**
-   * 编辑距离合并阈值：
-   * 根据编辑距离判断不完全一致但内容相近（例如有错别字）的弹幕
-   * 能有效击杀 "你指尖跃动的电光" 和 "你之间跃动的电光" 等
-   * @example 禁用(0), 轻微(≤3), 中等(≤5), 强力(≤8)
-   */
-  MAX_DIST: 5,
-  /**
-   * 词频向量合并阈值：
-   * 根据 2-Gram 频率向量的夹角判断不完全一致但内容类似的弹幕
-   * 能有效击杀 "yeah!~" 和 "yeah!~yeah!~yeah!~yeah!~" 等
-   * @example 禁用(1000), 轻微(60%), 中等(45%), 强力(30%)
-   */
-  MAX_COSINE: 45,
-  /**
-   * 识别谐音弹幕：
-   * 将常用汉字转换为拼音再进行比较
-   * 能有效击杀 "布拉迪巴特福来" 和 "布拉迪·八德福莱" 等
-   */
-  TRIM_PINYIN: true,
-  // 比较文本时：
-  TRIM_ENDING: true, // 忽略末尾标点
-  TRIM_SPACE: true, // 忽略多余空格
-  TRIM_WIDTH: true, // 忽略全半角差异
-
-  // 例外设置
-  /**
-   * 内容替换：符合这些规则的弹幕，判断是否合并前会先对内容进行替换
-   */
-  FORCELIST: [
-    ['^23{2,}$', '23333'],
-    ['^6{3,}$', '66666'],
-  ],
-  /**
-   * 内容替换规则命中时：继续尝试匹配后续规则
-   */
-  FORCELIST_CONTINUE_ON_MATCH: true,
-  /**
-   * 内容替换规则命中时：即使未触发合并也使用替换后的文本
-   */
-  FORCELIST_APPLY_SINGULAR: false,
-  /**
-   * 强制忽略：符合这些规则的弹幕不会被合并，优先级高于内容替换规则
-   */
-  WHITELIST: [] as [string, string][],
-  /**
-   * 强制删除：符合这些规则的弹幕会直接被删除（未实现）
-   */
-  BLACKLIST: [] as [string, string][],
-  /**
-   * 合并不同类型的弹幕（取消勾选后，底部弹幕不会跟滚动弹幕合并到一起）
-   */
-  CROSS_MODE: true,
-  // 放过特定类型的弹幕：
-  PROC_TYPE7: true, // 高级弹幕（特殊弹幕）
-  PROC_TYPE4: true, // 底部弹幕
-  PROC_POOL1: false, // 字幕弹幕(位于弹幕池1)
-
-  // // 显示设置
-  // DANMU_MARK: 'prefix' as 'prefix' | 'suffix' | 'off', // 弹幕数量标记(开头/结尾/关闭)
-  // MARK_THRESHOLD: 1, // 仅当数字大于n时显示
-  // DANMU_SUBSCRIPT: true, // 数量标记显示成下标(₍₂₎/[x2])
-  // // ENLARGE: true,
-  // // SHRINK_THRESHOLD: 0,
-  // /**
-  //  * 自动弹幕优选：
-  //  * 瞬时弹幕密度大于阈值时，按比例删除低权重弹幕，优先删除未合并弹幕
-  //  * @example 禁用(0),轻微(>120),中等(>75),强力(>50)
-  //  */
-  // DROP_THRESHOLD: 0,
-  // /**
-  //  * 合并后尽量显示为固定弹幕：
-  //  * 滚动弹幕和顶部 / 底部弹幕合并后显示在顶部 / 底部
-  //  */
-  // MODE_ELEVATION: true,
-  // /**
-  //  * 合并后的弹幕显示于n百分位弹幕的时间点
-  //  * @example 0%(0),20%(20),50%(50)
-  //  */
-  // REPRESENTATIVE_PERCENT: 20,
-}
-
-export type Config = Partial<typeof DEFAULT_CONFIG>
-type ResolvedConfig = typeof DEFAULT_CONFIG
-
-interface DanmuIr {
-  obj: DanmuObject
-  str: string // for similarity algorithm
-  ptr_idx: int
-  sim_reason: string
-}
-
-const ENDING_CHARS = new Set('.。,，/?？!！…~～@^、+=-_♂♀ ')
-// const TRIM_EXTRA_SPACE_RE = /[ \u3000]+/g
-// const TRIM_CJK_SPACE_RE =
-// /([\u3000-\u9FFF\uFF00-\uFFEF]) (?=[\u3000-\u9FFF\uFF00-\uFFEF])/g
-const WIDTH_TABLE = new Map(
-  Object.entries({
-    '　': ' ',
-    '１': '1',
-    '２': '2',
-    '３': '3',
-    '４': '4',
-    '５': '5',
-    '６': '6',
-    '７': '7',
-    '８': '8',
-    '９': '9',
-    '０': '0',
-    '!': '！',
-    '＠': '@',
-    '＃': '#',
-    '＄': '$',
-    '％': '%',
-    '＾': '^',
-    '＆': '&',
-    '＊': '*',
-    '（': '(',
-    '）': ')',
-    '－': '-',
-    '＝': '=',
-    '＿': '_',
-    '＋': '+',
-    '［': '[',
-    '］': ']',
-    '｛': '{',
-    '｝': '}',
-    ';': '；',
-    '＇': "'",
-    ':': '：',
-    '＂': '"',
-    ',': '，',
-    '．': '.',
-    '／': '/',
-    '＜': '<',
-    '＞': '>',
-    '?': '？',
-    '＼': '\\',
-    '｜': '|',
-    '｀': '`',
-    '～': '~',
-    ｑ: 'q',
-    ｗ: 'w',
-    ｅ: 'e',
-    ｒ: 'r',
-    ｔ: 't',
-    ｙ: 'y',
-    ｕ: 'u',
-    ｉ: 'i',
-    ｏ: 'o',
-    ｐ: 'p',
-    ａ: 'a',
-    ｓ: 's',
-    ｄ: 'd',
-    ｆ: 'f',
-    ｇ: 'g',
-    ｈ: 'h',
-    ｊ: 'j',
-    ｋ: 'k',
-    ｌ: 'l',
-    ｚ: 'z',
-    ｘ: 'x',
-    ｃ: 'c',
-    ｖ: 'v',
-    ｂ: 'b',
-    ｎ: 'n',
-    ｍ: 'm',
-    Ｑ: 'Q',
-    Ｗ: 'W',
-    Ｅ: 'E',
-    Ｒ: 'R',
-    Ｔ: 'T',
-    Ｙ: 'Y',
-    Ｕ: 'U',
-    Ｉ: 'I',
-    Ｏ: 'O',
-    Ｐ: 'P',
-    Ａ: 'A',
-    Ｓ: 'S',
-    Ｄ: 'D',
-    Ｆ: 'F',
-    Ｇ: 'G',
-    Ｈ: 'H',
-    Ｊ: 'J',
-    Ｋ: 'K',
-    Ｌ: 'L',
-    Ｚ: 'Z',
-    Ｘ: 'X',
-    Ｃ: 'C',
-    Ｖ: 'V',
-    Ｂ: 'B',
-    Ｎ: 'N',
-    Ｍ: 'M',
-  }),
-)
-
-/**
- * 反套路
- */
-function detaolu_meta(
-  config: ResolvedConfig,
-): (text: string) => [boolean, string] {
-  const TRIM_ENDING = config.TRIM_ENDING
-  const TRIM_SPACE = config.TRIM_SPACE
-  const TRIM_WIDTH = config.TRIM_WIDTH
-  const FORCELIST = config.FORCELIST.map(
-    ([pattern, repl]) => [new RegExp(pattern, 'giu'), repl] as [RegExp, string],
-  )
-  const FORCELIST_BREAK_ON_MATCH = !config.FORCELIST_CONTINUE_ON_MATCH
-
-  return (inp: string) => {
-    let len = inp.length
-    let text = ''
-
-    if (TRIM_ENDING) {
-      while (ENDING_CHARS.has(inp.charAt(len - 1)))
-        // assert str.charAt(-1)===''
-        len--
-      if (len === 0)
-        // all chars are ending chars, do nothing
-        len = inp.length
-    }
-
-    if (TRIM_WIDTH) {
-      for (let i = 0; i < len; i++) {
-        const c = inp.charAt(i)
-        text += WIDTH_TABLE.get(c) || c
-      }
-    } else {
-      text = inp.slice(0, len)
-    }
-
-    if (TRIM_SPACE) {
-      // text = text
-      //   .replace(TRIM_EXTRA_SPACE_RE, ' ')
-      //   .replace(TRIM_CJK_SPACE_RE, '$1')
-      text = text
-        .replaceAll(/[ \u3000]+/g, ' ')
-        .replaceAll(
-          /([\u3000-\u9FFF\uFF00-\uFFEF]) (?=[\u3000-\u9FFF\uFF00-\uFFEF])/g,
-          '$1',
-        )
-    }
-
-    let taolu_matched = false
-    for (const taolu of FORCELIST) {
-      if (taolu[0].test(text)) {
-        text = text.replace(taolu[0], taolu[1])
-        taolu_matched = true
-        if (FORCELIST_BREAK_ON_MATCH) break
-      }
-    }
-
-    return [taolu_matched, text]
-  }
-}
-
-/**
- * 白名单处理
- */
-function whitelisted_meta(config: ResolvedConfig): (text: string) => boolean {
-  const WHITELIST = config.WHITELIST.map((x) => new RegExp(x[0], 'iu'))
-
-  if (WHITELIST.length === 0) return () => false
-
-  return (text: string) => WHITELIST.some((re) => re.test(text))
-}
-
-/**
- * 黑名单处理
- */
-function blacklisted_meta(
-  config: ResolvedConfig,
-): (text: string) => string | null {
-  const BLACKLIST = config.BLACKLIST.map((x) =>
-    x[0] ? new RegExp(x[1]) : x[1].toLowerCase(),
-  )
-
-  if (BLACKLIST.length === 0) return () => null
-
-  return (text: string) => {
-    const lower = text.toLowerCase()
-    for (const pattern of BLACKLIST) {
-      const matched =
-        typeof pattern === 'string'
-          ? lower.includes(pattern)
-          : pattern.test(text)
-      if (matched) {
-        return typeof pattern === 'string'
-          ? ` ${pattern}`
-          : ` /${pattern.source}/`
-      }
-    }
-    return null
-  }
-}
-
-function extract_special_danmu(text: string): string {
-  try {
-    text = JSON.parse(text)[4]
-  } catch {}
-  return text
-}
-
-/**
- * 删除换行符/制表符
- */
-function trim_dispstr(text: string): string {
-  return text.replaceAll(/([\r\n\t])/g, '').trim()
-}
-
-/**
- * 选取中间值(最多出现的文字)
- */
-function select_median_length(strs: string[]): string {
-  if (strs.length === 1) return strs[0]
-
-  const sorted = strs.toSorted((a, b) => a.length - b.length)
-  const mid = Math.floor(sorted.length / 2)
-  return sorted[mid]
-}
-
-function u8array_to_arraybuffer(array: Uint8Array): ArrayBuffer {
-  return array.buffer.slice(
-    array.byteOffset,
-    array.byteOffset + array.byteLength,
-  ) as ArrayBuffer
-}
-
-async function load_wasm(wasm_mod?: ArrayBuffer) {
-  if (wasm_mod) {
-    await sim_init(wasm_mod)
-    return
-  }
-
-  const wasm_path = new URL('similarity-gen.wasm', import.meta.url).pathname
-  const wasm_u8 = await fs.readFile(wasm_path)
-  await sim_init(u8array_to_arraybuffer(wasm_u8))
-}
-
-function make_ptr_idx(idx: int, is_next_chunk: boolean): int {
-  return is_next_chunk ? -1 - idx : idx
-}
-
-async function merge(
-  chunk: DanmuChunk<DanmuObject>,
-  // next_chunk: DanmuChunk<DanmuObject>,
-  config: Config = DEFAULT_CONFIG,
-): Promise<DanmuClusterOutput> {
-  const local_config: ResolvedConfig = { ...DEFAULT_CONFIG, ...config }
-
-  await load_wasm()
-
-  begin_chunk(local_config)
-
-  const ret: DanmuClusterOutput = {
-    clusters: [],
-    stats: new Stats(),
-    deleted_chunk: [],
-  }
-
-  function apply_single_cluster(idx: int, obj: DanmuObject, desc: string) {
-    ret.clusters.push({
-      peers_ptr: [[idx, 'IGN']],
-      desc: [desc],
-      chosen_str: obj.content,
-      // danuni
-      danuni_count: 1,
-      // danuni_senders: [obj.danuni_sender],
-      danuni_dans: [obj],
-    })
-  }
-  function apply_cluster(irs: DanmuIr[]) {
-    if (irs.length === 1) {
-      ret.clusters.push({
-        peers_ptr: irs.map((ir) => [ir.ptr_idx, ir.sim_reason]),
-        desc: [],
-        chosen_str: irs[0].obj.content,
-        // danuni
-        danuni_count: irs.length,
-        // danuni_senders: irs.map((ir) => ir.obj.danuni_sender),
-        danuni_dans: irs.map((ir) => ir.obj),
-      })
-    } else {
-      const text_cnts = new Map()
-      let most_texts: string[] = []
-      let most_cnt = 0
-
-      for (const ir of irs) {
-        const text = ir.str
-        const cnt = 1 + (text_cnts.get(text) || 0)
-        text_cnts.set(text, cnt)
-
-        if (cnt > most_cnt) {
-          most_texts = [text]
-          most_cnt = cnt
-        } else if (cnt === most_cnt) {
-          most_texts.push(text)
-        }
-      }
-
-      const most_text = select_median_length(most_texts)
-
-      ret.clusters.push({
-        peers_ptr: irs.map((ir) => [ir.ptr_idx, ir.sim_reason]),
-        desc: most_cnt > 1 ? [`采用了出现 ${most_cnt} 次的文本`] : [],
-        chosen_str: most_text,
-        // danuni
-        danuni_count: most_cnt,
-        // danuni_senders: irs.map((ir) => ir.obj.danuni_sender),
-        danuni_dans: irs.map((ir) => ir.obj),
-      })
-    }
-  }
-
-  const detaolu = detaolu_meta(local_config)
-  const whitelisted = whitelisted_meta(local_config)
-  const blacklisted = blacklisted_meta(local_config)
-
-  function obj_to_ir(
-    objs: DanmuObject[],
-    s: Stats | null,
-    is_next_chunk: boolean,
-  ): DanmuIr[] {
-    return objs
-      .map((obj, idx) => {
-        if (!local_config.PROC_POOL1 && obj.pool === 1) {
-          if (s) {
-            s.ignored_type++
-            apply_single_cluster(idx, obj, '已忽略字幕弹幕，可以在选项中修改')
-          }
-          return null
-        }
-        // if (!config.PROC_TYPE7 && obj.mode === 7) {
-        if (!local_config.PROC_TYPE7 && obj.mode === 4) {
-          if (s) {
-            s.ignored_type++
-            apply_single_cluster(idx, obj, '已忽略特殊弹幕，可以在选项中修改')
-          }
-          return null
-        }
-        // if (!config.PROC_TYPE4 && obj.mode === 4) {
-        if (!local_config.PROC_TYPE4 && obj.mode === 1) {
-          if (s) {
-            s.ignored_type++
-            apply_single_cluster(idx, obj, '已忽略底部弹幕，可以在选项中修改')
-          }
-          return null
-        }
-        // if (obj.mode === 8) {
-        //   if (s) {
-        //     s.ignored_script++
-        //     apply_single_cluster(idx, obj, '代码弹幕')
-        //   }
-        //   return null
-        // }
-        // if (obj.mode === 9) {
-        //   if (s) {
-        //     s.ignored_script++
-        //     apply_single_cluster(idx, obj, 'BAS弹幕')
-        //   }
-        //   return null
-        // }
-
-        const disp_str = trim_dispstr(
-          // obj.mode === 7 && obj.content[0] === '['
-          obj.mode === 4 && obj.content[0] === '['
-            ? extract_special_danmu(obj.content)
-            : obj.content,
-        )
-
-        // if (obj.mode !== 8 && obj.mode !== 9) {
-        if (obj.mode !== 4) {
-          const matched = blacklisted(disp_str)
-          if (matched) {
-            if (s) {
-              s.deleted_blacklist++
-              s.deleted_blacklist_each[matched] =
-                (s.deleted_blacklist_each[matched] || 0) + 1
-              ret.deleted_chunk.push({
-                ...obj,
-                pakku: {
-                  deleted_reason: `命中黑名单：${matched}`,
-                },
-              })
-            }
-            return null
-          }
-        }
-        if (whitelisted(disp_str)) {
-          if (s) {
-            s.ignored_whitelist++
-            apply_single_cluster(idx, obj, '命中白名单')
-          }
-          return null
-        }
-
-        const [matched_taolu, detaolued] = detaolu(disp_str)
-
-        if (matched_taolu) {
-          if (s) s.num_taolu_matched++
-          if (local_config.FORCELIST_APPLY_SINGULAR)
-            obj = {
-              ...obj,
-              content: detaolued,
-            }
-        }
-
-        return {
-          obj,
-          str: detaolued,
-          ptr_idx: make_ptr_idx(idx, is_next_chunk),
-          sim_reason: 'ORIG',
-        }
-      })
-      .filter((obj) => obj !== null) as DanmuIr[]
-  }
-
-  const danmus = obj_to_ir(chunk.objs, ret.stats, false)
-  // const next_chunk_danmus = obj_to_ir(next_chunk.objs, null, true)
-
-  const nearby_danmus: Queue<DanmuIr[]> = new Queue()
-
-  const THRESHOLD_MS = local_config.THRESHOLD * 1000
-
-  for (const dm of danmus) {
-    while (true) {
-      const peeked = nearby_danmus.peek()
-      if (
-        peeked === null ||
-        dm.obj.time_ms - peeked[0].obj.time_ms <= THRESHOLD_MS
-      )
-        break
-      apply_cluster(peeked)
-      nearby_danmus.pop()
-    }
-
-    const sim = detect_similarity(
-      dm.str,
-      dm.obj.mode,
-      nearby_danmus.index_l,
-      ret.stats,
-    )
-    if (sim === null) {
-      nearby_danmus.push([dm])
-    } else {
-      const candidate =
-        nearby_danmus.storage[nearby_danmus.index_r - sim.idx_diff]
-      dm.sim_reason = sim.reason
-      candidate.push(dm)
-    }
-  }
-
-  // now process last few clusters with the next chunk
-  begin_index_lock()
-  // outer: for (const dm of next_chunk_danmus) {
-  //   while (true) {
-  //     const peeked = nearby_danmus.peek()
-  //     if (peeked === null) break outer
-  //     if (dm.obj.time_ms - peeked[0].obj.time_ms <= THRESHOLD_MS) break
-  //     apply_cluster(peeked)
-  //     nearby_danmus.pop()
-  //   }
-
-  //   const sim = detect_similarity(
-  //     dm.str,
-  //     dm.obj.mode,
-  //     nearby_danmus.index_l,
-  //     ret.stats,
-  //   )
-  //   if (sim !== null) {
-  //     const candidate =
-  //       nearby_danmus.storage[nearby_danmus.index_r - sim.idx_diff]
-  //     dm.sim_reason = sim.reason
-  //     candidate.push(dm)
-  //   }
-  // }
-
-  // finally apply remaining clusters
-  for (const candidate of nearby_danmus) {
-    apply_cluster(candidate)
-  }
-
-  return ret
-}
-
-export default merge