@dan-uni/dan-any-plugin-detaolu 0.7.3 → 0.9.5

package/dist/index.umd.min.js.LICENSE.txt

@@ -27,7 +27,7 @@ See the Apache Version 2.0 License for specific language governing permissions
 and limitations under the License.
 ***************************************************************************** */
 
-/*! For license information please see index.umd.min.js.LICENSE.txt */
+/*! For license information please see index.min.js.LICENSE.txt */
 
 /*! ieee754. BSD-3-Clause License. Feross Aboukhadijeh <https://feross.org/opensource> */
 
@@ -37,7 +37,7 @@ and limitations under the License.
 
 /**
  * @author: xmcp(代码主要逻辑来源)
- * @see: https://github.com/xmcp/pakku.js
+ * @see: https://github.com/xmcp/pakku.js/blob/master/pakkujs/core/combine_worker.ts
  * @license: GPL-3.0
  * 本文件内代码来源见上，经部分修改，并整合config注释
  */

package/dist/pakku.js/index.d.ts

@@ -1,6 +1,6 @@
 /**
  * @author: xmcp(代码主要逻辑来源)
- * @see: https://github.com/xmcp/pakku.js
+ * @see: https://github.com/xmcp/pakku.js/blob/master/pakkujs/core/combine_worker.ts
  * @license: GPL-3.0
  * 本文件内代码来源见上，经部分修改，并整合config注释
  */

package/dist/pakku.js/types.d.ts

@@ -23,6 +23,11 @@ export interface DanmuObjectRepresentative extends DanmuObject {
         disp_str: string;
     };
 }
+export interface DanmuObjectDeleted extends DanmuObject {
+    pakku: {
+        deleted_reason: string;
+    };
+}
 export interface DanmuChunk<ObjectType extends DanmuObject> {
     objs: ObjectType[];
 }
@@ -41,6 +46,7 @@ export interface DanmuClusterPtr {
 export interface DanmuClusterOutput {
     clusters: DanmuClusterPtr[];
     stats: Stats;
+    deleted_chunk: DanmuObjectDeleted[];
 }
 export declare class Stats {
     combined_identical: number;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@dan-uni/dan-any-plugin-detaolu",
-  "version": "0.7.3",
+  "version": "0.9.5",
   "description": "A filter, dedupe and anti-spam plugin of dan-any, a danmaku transformer lib, based on pakku.js.",
   "keywords": [
     "bangumi",
@@ -31,7 +31,7 @@
   },
   "dependencies": {
     "@dan-uni/dan-any": "workspace:^",
-    "fs-extra": "^11.3.0"
+    "fs-extra": "^11.3.2"
   },
   "devDependencies": {
     "@types/fs-extra": "^11.0.4"

package/src/index.ts

@@ -22,8 +22,8 @@ async function detaolu(that: UniPool, config?: DeTaoLuConfig) {
     if (p.danuni_dans.length === 1) {
       return p.danuni_dans[0].danuni_dan
     } else {
-      const dans = p.danuni_dans,
-        pool = new UniPool(dans.map((d) => d.danuni_dan))
+      const dans = p.danuni_dans
+      const pool = new UniPool(dans.map((d) => d.danuni_dan))
       function isAllBottomMode(p: UniPool) {
         return p.dans.every((d) => d.mode === UniDMTools.Modes.Bottom)
       }

package/src/pakku.js/index.ts

@@ -1,6 +1,6 @@
 /**
  * @author: xmcp(代码主要逻辑来源)
- * @see: https://github.com/xmcp/pakku.js
+ * @see: https://github.com/xmcp/pakku.js/blob/master/pakkujs/core/combine_worker.ts
  * @license: GPL-3.0
  * 本文件内代码来源见上，经部分修改，并整合config注释
  */
@@ -90,7 +90,7 @@ export type Config = Partial<typeof DEFAULT_CONFIG>
 interface DanmuIr {
   obj: DanmuObject
   str: string // for similarity algorithm
-  idx: int
+  ptr_idx: int
   sim_reason: string
 }
 
@@ -308,7 +308,7 @@ function trim_dispstr(text: string): string {
 function select_median_length(strs: string[]): string {
   if (strs.length === 1) return strs[0]
 
-  const sorted = strs.sort((a, b) => a.length - b.length)
+  const sorted = strs.toSorted((a, b) => a.length - b.length)
   const mid = Math.floor(sorted.length / 2)
   return sorted[mid]
 }
@@ -316,10 +316,14 @@ function select_median_length(strs: string[]): string {
 async function load_wasm(wasm_mod?: ArrayBuffer) {
   await sim_init(
     wasm_mod ??
-      (await fs.readFile(new URL('./similarity-gen.wasm', import.meta.url))),
+      (await fs.readFile(new URL('similarity-gen.wasm', import.meta.url))),
   )
 }
 
+function make_ptr_idx(idx: int, is_next_chunk: boolean): int {
+  return is_next_chunk ? -1 - idx : idx
+}
+
 async function merge(
   chunk: DanmuChunk<DanmuObject>,
   // next_chunk: DanmuChunk<DanmuObject>,
@@ -332,6 +336,7 @@ async function merge(
   const ret: DanmuClusterOutput = {
     clusters: [],
     stats: new Stats(),
+    deleted_chunk: [],
   }
 
   function apply_single_cluster(idx: int, obj: DanmuObject, desc: string) {
@@ -348,7 +353,7 @@ async function merge(
   function apply_cluster(irs: DanmuIr[]) {
     if (irs.length === 1) {
       ret.clusters.push({
-        peers_ptr: irs.map((ir) => [ir.idx, ir.sim_reason]),
+        peers_ptr: irs.map((ir) => [ir.ptr_idx, ir.sim_reason]),
         desc: [],
         chosen_str: irs[0].obj.content, // do not use detaolued str for single danmu
         // danuni
@@ -358,8 +363,8 @@ async function merge(
       })
     } else {
       const text_cnts = new Map()
-      let most_texts: string[] = [],
-        most_cnt = 0
+      let most_texts: string[] = []
+      let most_cnt = 0
 
       for (const ir of irs) {
         const text = ir.str
@@ -377,7 +382,7 @@ async function merge(
       const most_text = select_median_length(most_texts)
 
       ret.clusters.push({
-        peers_ptr: irs.map((ir) => [ir.idx, ir.sim_reason]),
+        peers_ptr: irs.map((ir) => [ir.ptr_idx, ir.sim_reason]),
         desc: most_cnt > 1 ? [`采用了出现 ${most_cnt} 次的文本`] : [],
         chosen_str: most_text,
         // danuni
@@ -388,7 +393,11 @@ async function merge(
     }
   }
 
-  function obj_to_ir(objs: DanmuObject[], s: Stats | null): DanmuIr[] {
+  function obj_to_ir(
+    objs: DanmuObject[],
+    s: Stats | null,
+    is_next_chunk: boolean,
+  ): DanmuIr[] {
     return objs
       .map((obj, idx) => {
         if (!config.PROC_POOL1 && obj.pool === 1) {
@@ -444,6 +453,12 @@ async function merge(
               s.deleted_blacklist++
               s.deleted_blacklist_each[matched] =
                 (s.deleted_blacklist_each[matched] || 0) + 1
+              ret.deleted_chunk.push({
+                ...obj,
+                pakku: {
+                  deleted_reason: `命中黑名单：${matched}`,
+                },
+              })
             }
             return null
           }
@@ -465,15 +480,15 @@ async function merge(
         return {
           obj,
           str: detaolued,
-          idx,
+          ptr_idx: make_ptr_idx(idx, is_next_chunk),
           sim_reason: 'ORIG',
         }
       })
       .filter((obj) => obj !== null) as DanmuIr[]
   }
 
-  const danmus = obj_to_ir(chunk.objs, ret.stats)
-  // const next_chunk_danmus = obj_to_ir(next_chunk.objs, null)
+  const danmus = obj_to_ir(chunk.objs, ret.stats, false)
+  // const next_chunk_danmus = obj_to_ir(next_chunk.objs, null, true)
 
   const nearby_danmus: Queue<DanmuIr[]> = new Queue()
 
@@ -497,13 +512,13 @@ async function merge(
       nearby_danmus.index_l,
       ret.stats,
     )
-    if (sim !== null) {
+    if (sim === null) {
+      nearby_danmus.push([dm])
+    } else {
       const candidate =
         nearby_danmus.storage[nearby_danmus.index_r - sim.idx_diff]
       dm.sim_reason = sim.reason
       candidate.push(dm)
-    } else {
-      nearby_danmus.push([dm])
     }
   }
 

package/src/pakku.js/similarity-gen.js

@@ -1,18 +1,5 @@
-/* eslint-disable getter-return */
-/* eslint-disable unicorn/catch-error-name */
-/* eslint-disable unicorn/prefer-number-properties */
-/* eslint-disable block-scoped-var */
-/* eslint-disable object-shorthand */
-/* eslint-disable no-setter-return */
-/* eslint-disable prefer-template */
-/* eslint-disable unicorn/new-for-builtins */
-/* eslint-disable unicorn/throw-new-error */
-/* eslint-disable no-console */
-/* eslint-disable unicorn/consistent-function-scoping */
-/* eslint-disable no-unused-expressions */
-/* eslint-disable vars-on-top */
-/* eslint-disable import/no-mutable-exports */
-/* eslint-disable import/no-default-export */
+/* eslint-disable @eslint-community/eslint-comments/no-unlimited-disable */
+/* eslint-disable */
 /** @nocollapse */ var Module = function (moduleArg = {}) {
   var moduleRtn
 

package/src/pakku.js/similarity_stub.ts

@@ -75,20 +75,34 @@ export function detect_similarity(
   const idx_diff = ret & ((1 << 19) - 1)
 
   let reason_str
-  if (reason === CombinedReason.combined_identical) {
-    S.combined_identical++
-    reason_str = '=='
-  } else if (reason === CombinedReason.combined_edit_distance) {
-    S.combined_edit_distance++
-    reason_str = `≤${dist}`
-  } else if (reason === CombinedReason.combined_cosine_distance) {
-    S.combined_cosine_distance++
-    reason_str = `${dist}%`
-  } else if (reason === CombinedReason.combined_pinyin_distance) {
-    S.combined_pinyin_distance++
-    reason_str = `P≤${dist}`
-  } else {
-    throw new Error(`similarity wasm returned unknown reason: ${ret}`)
+  switch (reason) {
+    case CombinedReason.combined_identical: {
+      S.combined_identical++
+      reason_str = '=='
+
+      break
+    }
+    case CombinedReason.combined_edit_distance: {
+      S.combined_edit_distance++
+      reason_str = `≤${dist}`
+
+      break
+    }
+    case CombinedReason.combined_cosine_distance: {
+      S.combined_cosine_distance++
+      reason_str = `${dist}%`
+
+      break
+    }
+    case CombinedReason.combined_pinyin_distance: {
+      S.combined_pinyin_distance++
+      reason_str = `P≤${dist}`
+
+      break
+    }
+    default: {
+      throw new Error(`similarity wasm returned unknown reason: ${ret}`)
+    }
   }
 
   return { reason: reason_str, idx_diff }

package/src/pakku.js/types.ts

@@ -46,6 +46,12 @@ export interface DanmuObjectRepresentative extends DanmuObject {
   }
 }
 
+export interface DanmuObjectDeleted extends DanmuObject {
+  pakku: {
+    deleted_reason: string
+  }
+}
+
 export interface DanmuChunk<ObjectType extends DanmuObject> {
   objs: ObjectType[]
 
@@ -79,6 +85,7 @@ export interface DanmuClusterPtr {
 export interface DanmuClusterOutput {
   clusters: DanmuClusterPtr[]
   stats: Stats
+  deleted_chunk: DanmuObjectDeleted[]
 }
 
 export class Stats {

package/tsconfig.json

@@ -2,14 +2,14 @@
   "compilerOptions": {
     /* Visit https://aka.ms/tsconfig to read more about this file */
     /* Projects */
-    // "incremental": true,                              /* Save .tsbuildinfo files to allow for incremental compilation of projects. */
+    "incremental": true /* Save .tsbuildinfo files to allow for incremental compilation of projects. */,
     // "composite": true,                                /* Enable constraints that allow a TypeScript project to be used with project references. */
     // "tsBuildInfoFile": "./.tsbuildinfo",              /* Specify the path to .tsbuildinfo incremental compilation file. */
     // "disableSourceOfProjectReferenceRedirect": true,  /* Disable preferring source files instead of declaration files when referencing composite projects. */
     // "disableSolutionSearching": true,                 /* Opt a project out of multi-project reference checking when editing. */
     // "disableReferencedProjectLoad": true,             /* Reduce the number of projects loaded automatically by TypeScript. */
     /* Language and Environment */
-    "target": "ES2021" /* Set the JavaScript language version for emitted JavaScript and include compatible library declarations. */,
+    "target": "ES2023" /* Set the JavaScript language version for emitted JavaScript and include compatible library declarations. */,
     // "lib": [],                                        /* Specify a set of bundled library declaration files that describe the target runtime environment. */
     // "jsx": "preserve",                                /* Specify what JSX code is generated. */
     // "experimentalDecorators": true,                   /* Enable experimental support for legacy experimental decorators. */

package/types/tsconfig.tsbuildinfo

@@ -0,0 +1 @@
+{"version":"5.9.3"}