extra-deduplicator 0.0.0-stage → 0.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/LICENSE ADDED
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2024 BlackGlory <woshenmedoubuzhidao@blackglory.me>
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
package/README.md CHANGED
@@ -1,3 +1,45 @@
1
- # Temporary Holding Version
1
+ # extra-deduplicator
2
+ A data deduplication library, which is designed for extracting new data.
2
3
 
3
- This version is a temporary placeholder for this package. An operational version to replace this has been submitted for review and is awaiting a staged release.
4
+ ## Install
5
+ ```sh
6
+ npm install --save extra-deduplicator
7
+ ```
8
+
9
+ ## Usage
10
+ ```ts
11
+ import { Deduplicator } from 'extra-deduplicator'
12
+
13
+ const deduplicator = await Deduplicator.create({ filename: 'dedup.db' })
14
+
15
+ const records = ['record-1', 'record-2']
16
+ const newRecords = await deduplicator.addAndDidff(records)
17
+ ```
18
+
19
+ ## API
20
+ ```ts
21
+ interface IOptions<T> {
22
+ filename?: string
23
+
24
+ hash?: (value: T) => JSONValue
25
+
26
+ shrink?: {
27
+ target?: number
28
+ threshold?: number
29
+ }
30
+ }
31
+
32
+ class Deduplicator<T> {
33
+ static create(options?: IOptions): Promise<Deduplicator<T>>
34
+
35
+ add(values: T[]): Promise<void>
36
+
37
+ diff(values: T[]): Promise<T[]>
38
+ diffLast(value: T): Promise<boolean>
39
+
40
+ addAndDiff(values: T[]): Promise<T[]>
41
+ addAndDiffLast(value: T): Promise<boolean>
42
+
43
+ close(): void
44
+ }
45
+ ```
@@ -0,0 +1,11 @@
1
+ import { Database as IDatabase } from 'better-sqlite3';
2
+ interface IShrinkOptions {
3
+ target: number;
4
+ threshold: number;
5
+ }
6
+ export declare function openDatabase(filename?: string): Promise<IDatabase>;
7
+ export declare function hasHashes(db: IDatabase, hashes: string[]): boolean[];
8
+ export declare function isLastHash(db: IDatabase, hash: string): boolean;
9
+ export declare function addHashes(db: IDatabase, hashes: string[], shrinkOptions: IShrinkOptions): string[];
10
+ export declare function addLastHash(db: IDatabase, hash: string, shrinkOptions: IShrinkOptions): boolean;
11
+ export {};
@@ -0,0 +1,106 @@
1
+ import Database from 'better-sqlite3';
2
+ import { migrate } from '@blackglory/better-sqlite3-migrations';
3
+ import { findMigrationFilenames, readMigrationFile } from 'migration-files';
4
+ import { map } from 'extra-promise';
5
+ import { assert, isFinite } from '@blackglory/prelude';
6
+ import { findUpPackageFilename } from 'extra-filesystem';
7
+ import path from 'path';
8
+ export async function openDatabase(filename = ':memory:') {
9
+ const packageFilename = await findUpPackageFilename(import.meta.dirname);
10
+ assert(packageFilename, 'package.json not found');
11
+ const packageRoot = path.dirname(packageFilename);
12
+ const migrationsPath = path.join(packageRoot, 'migrations');
13
+ const migrationFilenames = await findMigrationFilenames(migrationsPath);
14
+ const migrations = await map(migrationFilenames, readMigrationFile);
15
+ const db = new Database(filename);
16
+ migrate(db, migrations);
17
+ return db;
18
+ }
19
+ export function hasHashes(db, hashes) {
20
+ return hashes.map(hash => {
21
+ const row = db.prepare(`
22
+ SELECT EXISTS(
23
+ SELECT *
24
+ FROM hash
25
+ WHERE hash = $hash
26
+ ) AS hash_exists
27
+ `).get({ hash });
28
+ return !!(row === null || row === void 0 ? void 0 : row.hash_exists);
29
+ });
30
+ }
31
+ export function isLastHash(db, hash) {
32
+ const row = db.prepare(`
33
+ SELECT hash
34
+ FROM hash
35
+ ORDER BY id DESC
36
+ LIMIT 1
37
+ `).get();
38
+ return (row === null || row === void 0 ? void 0 : row.hash) === hash;
39
+ }
40
+ export function addHashes(db, hashes, shrinkOptions) {
41
+ return db.transaction(() => {
42
+ const newHashes = [];
43
+ for (const hash of hashes) {
44
+ const { changes } = db.prepare(`
45
+ INSERT INTO hash (hash)
46
+ VALUES ($hash)
47
+ ON CONFLICT DO NOTHING
48
+ `).run({ hash });
49
+ if (changes === 1) {
50
+ newHashes.push(hash);
51
+ }
52
+ else {
53
+ db.prepare(`
54
+ INSERT INTO hash (hash)
55
+ VALUES ($hash)
56
+ `).run({ hash });
57
+ }
58
+ }
59
+ shrink(db, shrinkOptions);
60
+ return newHashes;
61
+ })();
62
+ }
63
+ export function addLastHash(db, hash, shrinkOptions) {
64
+ return db.transaction(() => {
65
+ const row = db.prepare(`
66
+ SELECT hash
67
+ FROM hash
68
+ ORDER BY id DESC
69
+ LIMIT 1
70
+ `).get();
71
+ if ((row === null || row === void 0 ? void 0 : row.hash) === hash) {
72
+ return false;
73
+ }
74
+ else {
75
+ db.prepare(`
76
+ INSERT INTO hash (hash)
77
+ VALUES ($hash)
78
+ `).run({ hash });
79
+ shrink(db, shrinkOptions);
80
+ return true;
81
+ }
82
+ })();
83
+ }
84
+ function shrink(db, options) {
85
+ if (isFinite(options.threshold)) {
86
+ db.transaction(() => {
87
+ const count = countHashes(db);
88
+ if (count >= options.threshold) {
89
+ db.prepare(`
90
+ DELETE FROM hash
91
+ ORDER BY id ASC
92
+ LIMIT $limit
93
+ `).run({ limit: count - options.target });
94
+ }
95
+ })();
96
+ }
97
+ }
98
+ function countHashes(db) {
99
+ const row = db.prepare(`
100
+ SELECT COUNT(*) as count
101
+ FROM hash
102
+ `).get();
103
+ assert(row);
104
+ return row.count;
105
+ }
106
+ //# sourceMappingURL=database.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"database.js","sourceRoot":"","sources":["../src/database.ts"],"names":[],"mappings":"AAAA,OAAO,QAAmC,MAAM,gBAAgB,CAAA;AAChE,OAAO,EAAE,OAAO,EAAE,MAAM,uCAAuC,CAAA;AAC/D,OAAO,EAAE,sBAAsB,EAAE,iBAAiB,EAAE,MAAM,iBAAiB,CAAA;AAC3E,OAAO,EAAE,GAAG,EAAE,MAAM,eAAe,CAAA;AACnC,OAAO,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,qBAAqB,CAAA;AACtD,OAAO,EAAE,qBAAqB,EAAE,MAAM,kBAAkB,CAAA;AACxD,OAAO,IAAI,MAAM,MAAM,CAAA;AAQvB,MAAM,CAAC,KAAK,UAAU,YAAY,CAAC,QAAQ,GAAG,UAAU;IACtD,MAAM,eAAe,GAAG,MAAM,qBAAqB,CAAC,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAA;IACxE,MAAM,CAAC,eAAe,EAAE,wBAAwB,CAAC,CAAA;IAEjD,MAAM,WAAW,GAAG,IAAI,CAAC,OAAO,CAAC,eAAe,CAAC,CAAA;IACjD,MAAM,cAAc,GAAG,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,YAAY,CAAC,CAAA;IAC3D,MAAM,kBAAkB,GAAG,MAAM,sBAAsB,CAAC,cAAc,CAAC,CAAA;IACvE,MAAM,UAAU,GAAG,MAAM,GAAG,CAAC,kBAAkB,EAAE,iBAAiB,CAAC,CAAA;IAEnE,MAAM,EAAE,GAAG,IAAI,QAAQ,CAAC,QAAQ,CAAC,CAAA;IACjC,OAAO,CAAC,EAAE,EAAE,UAAU,CAAC,CAAA;IAEvB,OAAO,EAAE,CAAA;AACX,CAAC;AAED,MAAM,UAAU,SAAS,CAAC,EAAa,EAAE,MAAgB;IACvD,OAAO,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE;QACvB,MAAM,GAAG,GAAG,EAAE,CAAC,OAAO,CAGpB;;;;;;KAMD,CAAC,CAAC,GAAG,CAAC,EAAE,IAAI,EAAE,CAAC,CAAA;QAEhB,OAAO,CAAC,CAAC,CAAA,GAAG,aAAH,GAAG,uBAAH,GAAG,CAAE,WAAW,CAAA,CAAA;IAC3B,CAAC,CAAC,CAAA;AACJ,CAAC;AAED,MAAM,UAAU,UAAU,CAAC,EAAa,EAAE,IAAY;IACpD,MAAM,GAAG,GAAG,EAAE,CAAC,OAAO,CAGpB;;;;;GAKD,CAAC,CAAC,GAAG,EAAE,CAAA;IAER,OAAO,CAAA,GAAG,aAAH,GAAG,uBAAH,GAAG,CAAE,IAAI,MAAK,IAAI,CAAA;AAC3B,CAAC;AAED,MAAM,UAAU,SAAS,CACvB,EAAa,EACb,MAAgB,EAChB,aAA6B;IAE7B,OAAO,EAAE,CAAC,WAAW,CAAC,GAAG,EAAE;QACzB,MAAM,SAAS,GAAa,EAAE,CAAA;QAE9B,KAAK,MAAM,IAAI,IAAI,MAAM,EAAE,CAAC;YAC1B,MAAM,EAAE,OAAO,EAAE,GAAG,EAAE,CAAC,OAAO,CAAmB;;;;OAIhD,CAAC,CAAC,GAAG,CAAC,EAAE,IAAI,EAAE,CAAC,CAAA;YAEhB,IAAI,OAAO,KAAK,CAAC,EAAE,CAAC;gBAClB,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;YACtB,CAAC;iBAAM,CAAC;gBAEN,EAAE,CAAC,OAAO,CAAmB;;;SAG5B,CAAC,CAAC,GAAG,CAAC,EAAE,IAAI,EAAE,CAAC,CAAA;YAClB,CAAC;QACH,CAAC;QAED,MAAM,CAAC,EAAE,EAAE,aAAa,CAAC,CAAA;QAEzB,OAAO,SAAS,CAAA;IAClB,CAAC,CAAC,EAAE,CAAA;AACN,CAAC;AAED,MAAM,UAAU,WAAW,CACzB,EAAa,EACb,IAAY,EACZ,aAA6B;IAE7B,OAAO,EAAE,CAAC,WAAW,CAAC,GAAG,EAAE;QACzB,MAAM,GAAG,GAAG,EAAE,CAAC,OAAO,CAAuB;;;;;KAK5C,CAAC,CAAC,GAAG,EAAE,CAAA;QAER,IAAI,CAAA,GAAG,aAAH,GAAG,uBAAH,GAAG,CAAE,IAAI,MAAK,IAAI,EAAE,CAAC;YACvB,OAAO,KAAK,CAAA;QACd,CAAC;aAAM,CAAC;YACN,EAAE,CAAC,OAAO,CAAmB;;;OAG5B,CAAC,CAAC,GAAG,CAAC,EAAE,IAAI,EAAE,CAAC,CAAA;YAChB,MAAM,CAAC,EAAE,EAAE,aAAa,CAAC,CAAA;YAEzB,OAAO,IAAI,CAAA;QACb,CAAC;IACH,CAAC,CAAC,EAAE,CAAA;AACN,CAAC;AAED,SAAS,MAAM,CAAC,EAAa,EAAE,OAAuB;IACpD,IAAI,QAAQ,CAAC,OAAO,CAAC,SAAS,CAAC,EAAE,CAAC;QAChC,EAAE,CAAC,WAAW,CAAC,GAAG,EAAE;YAClB,MAAM,KAAK,GAAG,WAAW,CAAC,EAAE,CAAC,CAAA;YAE7B,IAAI,KAAK,IAAI,OAAO,CAAC,SAAS,EAAE,CAAC;gBAC/B,EAAE,CAAC,OAAO,CAAoB;;;;SAI7B,CAAC,CAAC,GAAG,CAAC,EAAE,KAAK,EAAE,KAAK,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,CAAA;YAC3C,CAAC;QACH,CAAC,CAAC,EAAE,CAAA;IACN,CAAC;AACH,CAAC;AAED,SAAS,WAAW,CAAC,EAAa;IAChC,MAAM,GAAG,GAAG,EAAE,CAAC,OAAO,CAAwB;;;GAG7C,CAAC,CAAC,GAAG,EAAE,CAAA;IACR,MAAM,CAAC,GAAG,CAAC,CAAA;IAEX,OAAO,GAAG,CAAC,KAAK,CAAA;AAClB,CAAC"}
@@ -0,0 +1,22 @@
1
+ import { type Awaitable } from '@blackglory/prelude';
2
+ export interface IOptions<T> {
3
+ filename?: string;
4
+ hash?: (value: T) => Awaitable<string>;
5
+ shrink?: {
6
+ target?: number;
7
+ threshold?: number;
8
+ };
9
+ }
10
+ export declare class Deduplicator<T> {
11
+ private db;
12
+ private hash;
13
+ private shrink;
14
+ private constructor();
15
+ static create<T>({ filename, hash, shrink: { threshold, target } }?: IOptions<T>): Promise<Deduplicator<T>>;
16
+ close(): void;
17
+ add(values: T[]): Promise<void>;
18
+ diff(values: T[]): Promise<T[]>;
19
+ diffLast(value: T): Promise<boolean>;
20
+ addAndDiff(values: T[]): Promise<T[]>;
21
+ addAndDiffLast(value: T): Promise<boolean>;
22
+ }
@@ -0,0 +1,57 @@
1
+ import { toArray, uniq, zip } from 'iterable-operator';
2
+ import { isntEmptyArray, assert, isPositiveInfinity, isFinite } from '@blackglory/prelude';
3
+ import { map } from 'extra-promise';
4
+ import { openDatabase, hasHashes, isLastHash, addHashes, addLastHash } from './database.js';
5
+ import { stringify } from 'extra-json-stable-stringify';
6
+ import { sha256 } from 'extra-compatible';
7
+ export class Deduplicator {
8
+ constructor(db, hash, shrink) {
9
+ this.db = db;
10
+ this.hash = hash;
11
+ this.shrink = shrink;
12
+ }
13
+ static async create({ filename, hash = value => sha256(stringify(value)), shrink: { threshold = Infinity, target = Math.floor(threshold / 2) } = {} } = {}) {
14
+ assert(isPositiveInfinity(threshold) || Number.isInteger(threshold), 'The shrink.threshold must be Infinity or an integer');
15
+ assert(isPositiveInfinity(target) || Number.isInteger(target), 'The shrink.target must be Infinity or an integer');
16
+ assert(threshold > 1, 'The shrink.threshold must be greater than 1');
17
+ if (isFinite(threshold)) {
18
+ assert(target < threshold, 'The shrink.target must be less than shrink.threshold');
19
+ }
20
+ const db = await openDatabase(filename);
21
+ return new Deduplicator(db, hash, { threshold, target });
22
+ }
23
+ close() {
24
+ this.db.close();
25
+ }
26
+ async add(values) {
27
+ const hashes = await map(values, value => this.hash(value));
28
+ addHashes(this.db, hashes, this.shrink);
29
+ }
30
+ async diff(values) {
31
+ values = toArray(uniq(values));
32
+ const hashes = await map(values, value => this.hash(value));
33
+ const results = hasHashes(this.db, hashes);
34
+ const newValues = values.filter((_, index) => !results[index]);
35
+ return newValues;
36
+ }
37
+ async diffLast(value) {
38
+ const hash = await this.hash(value);
39
+ return !isLastHash(this.db, hash);
40
+ }
41
+ async addAndDiff(values) {
42
+ const hashes = await map(values, this.hash);
43
+ const newHashes = addHashes(this.db, hashes, this.shrink);
44
+ if (isntEmptyArray(newHashes)) {
45
+ const hashToValue = Object.fromEntries(zip(hashes, values));
46
+ return newHashes.map(hash => hashToValue[hash]);
47
+ }
48
+ else {
49
+ return [];
50
+ }
51
+ }
52
+ async addAndDiffLast(value) {
53
+ const hash = await this.hash(value);
54
+ return addLastHash(this.db, hash, this.shrink);
55
+ }
56
+ }
57
+ //# sourceMappingURL=deduplicator.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"deduplicator.js","sourceRoot":"","sources":["../src/deduplicator.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,GAAG,EAAE,MAAM,mBAAmB,CAAA;AACtD,OAAO,EAAE,cAAc,EAAE,MAAM,EAAkB,kBAAkB,EAAE,QAAQ,EAAE,MAAM,qBAAqB,CAAA;AAC1G,OAAO,EAAE,GAAG,EAAE,MAAM,eAAe,CAAA;AACnC,OAAO,EAAE,YAAY,EAAE,SAAS,EAAE,UAAU,EAAE,SAAS,EAAE,WAAW,EAAE,MAAM,eAAe,CAAA;AAE3F,OAAO,EAAE,SAAS,EAAE,MAAM,6BAA6B,CAAA;AACvD,OAAO,EAAE,MAAM,EAAE,MAAM,kBAAkB,CAAA;AAazC,MAAM,OAAO,YAAY;IACvB,YACU,EAAY,EACZ,IAAqC,EACrC,MAGP;QALO,OAAE,GAAF,EAAE,CAAU;QACZ,SAAI,GAAJ,IAAI,CAAiC;QACrC,WAAM,GAAN,MAAM,CAGb;IACA,CAAC;IAEJ,MAAM,CAAC,KAAK,CAAC,MAAM,CACjB,EACE,QAAQ,EACR,IAAI,GAAG,KAAK,CAAC,EAAE,CAAC,MAAM,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,EACxC,MAAM,EAAE,EACN,SAAS,GAAG,QAAQ,EACpB,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,SAAS,GAAG,CAAC,CAAC,EACnC,GAAG,EAAE,KACS,EAAE;QAEnB,MAAM,CACJ,kBAAkB,CAAC,SAAS,CAAC,IAAI,MAAM,CAAC,SAAS,CAAC,SAAS,CAAC,EAC5D,qDAAqD,CACtD,CAAA;QACD,MAAM,CACJ,kBAAkB,CAAC,MAAM,CAAC,IAAI,MAAM,CAAC,SAAS,CAAC,MAAM,CAAC,EACtD,kDAAkD,CACnD,CAAA;QACD,MAAM,CAAC,SAAS,GAAG,CAAC,EAAE,6CAA6C,CAAC,CAAA;QACpE,IAAI,QAAQ,CAAC,SAAS,CAAC,EAAE,CAAC;YACxB,MAAM,CAAC,MAAM,GAAG,SAAS,EAAE,sDAAsD,CAAC,CAAA;QACpF,CAAC;QAED,MAAM,EAAE,GAAG,MAAM,YAAY,CAAC,QAAQ,CAAC,CAAA;QAEvC,OAAO,IAAI,YAAY,CAAI,EAAE,EAAE,IAAI,EAAE,EAAE,SAAS,EAAE,MAAM,EAAE,CAAC,CAAA;IAC7D,CAAC;IAED,KAAK;QACH,IAAI,CAAC,EAAE,CAAC,KAAK,EAAE,CAAA;IACjB,CAAC;IAED,KAAK,CAAC,GAAG,CAAC,MAAW;QACnB,MAAM,MAAM,GAAG,MAAM,GAAG,CAAC,MAAM,EAAE,KAAK,CAAC,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAA;QAE3D,SAAS,CAAC,IAAI,CAAC,EAAE,EAAE,MAAM,EAAE,IAAI,CAAC,MAAM,CAAC,CAAA;IACzC,CAAC;IAED,KAAK,CAAC,IAAI,CAAC,MAAW;QACpB,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAA;QAE9B,MAAM,MAAM,GAAG,MAAM,GAAG,CAAC,MAAM,EAAE,KAAK,CAAC,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAA;QAC3D,MAAM,OAAO,GAAG,SAAS,CAAC,IAAI,CAAC,EAAE,EAAE,MAAM,CAAC,CAAA;QAC1C,MAAM,SAAS,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAA;QAE9D,OAAO,SAAS,CAAA;IAClB,CAAC;IAED,KAAK,CAAC,QAAQ,CAAC,KAAQ;QACrB,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;QAEnC,OAAO,CAAC,UAAU,CAAC,IAAI,CAAC,EAAE,EAAE,IAAI,CAAC,CAAA;IACnC,CAAC;IAED,KAAK,CAAC,UAAU,CAAC,MAAW;QAC1B,MAAM,MAAM,GAAG,MAAM,GAAG,CAAC,MAAM,EAAE,IAAI,CAAC,IAAI,CAAC,CAAA;QAE3C,MAAM,SAAS,GAAG,SAAS,CAAC,IAAI,CAAC,EAAE,EAAE,MAAM,EAAE,IAAI,CAAC,MAAM,CAAC,CAAA;QAEzD,IAAI,cAAc,CAAC,SAAS,CAAC,EAAE,CAAC;YAC9B,MAAM,WAAW,GAAG,MAAM,CAAC,WAAW,CAAC,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAAA;YAE3D,OAAO,SAAS,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC,CAAA;QACjD,CAAC;aAAM,CAAC;YACN,OAAO,EAAE,CAAA;QACX,CAAC;IACH,CAAC;IAED,KAAK,CAAC,cAAc,CAAC,KAAQ;QAC3B,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;QAEnC,OAAO,WAAW,CAAC,IAAI,CAAC,EAAE,EAAE,IAAI,EAAE,IAAI,CAAC,MAAM,CAAC,CAAA;IAChD,CAAC;CACF"}
package/lib/index.d.ts ADDED
@@ -0,0 +1 @@
1
+ export * from './deduplicator.js';
package/lib/index.js ADDED
@@ -0,0 +1,2 @@
1
+ export * from './deduplicator.js';
2
+ //# sourceMappingURL=index.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,cAAc,mBAAmB,CAAA"}
@@ -0,0 +1,18 @@
1
+ --------------------------------------------------------------------------------
2
+ -- Up
3
+ --------------------------------------------------------------------------------
4
+
5
+ PRAGMA journal_mode = WAL;
6
+
7
+ CREATE TABLE hash (
8
+ id INTEGER PRIMARY KEY AUTOINCREMENT
9
+ , hash TEXT UNIQUE ON CONFLICT REPLACE
10
+ );
11
+
12
+ --------------------------------------------------------------------------------
13
+ -- Down
14
+ --------------------------------------------------------------------------------
15
+
16
+ PRAGMA journal_mode = DELETE;
17
+
18
+ DROP TABLE hash;
@@ -0,0 +1,18 @@
1
+ --------------------------------------------------------------------------------
2
+ -- Up
3
+ --------------------------------------------------------------------------------
4
+
5
+ PRAGMA journal_mode = WAL;
6
+
7
+ CREATE TABLE hash (
8
+ id INTEGER PRIMARY KEY AUTOINCREMENT
9
+ , hash TEXT UNIQUE ON CONFLICT REPLACE
10
+ );
11
+
12
+ --------------------------------------------------------------------------------
13
+ -- Down
14
+ --------------------------------------------------------------------------------
15
+
16
+ PRAGMA journal_mode = DELETE;
17
+
18
+ DROP TABLE hash;
package/package.json CHANGED
@@ -1,6 +1,63 @@
1
1
  {
2
2
  "name": "extra-deduplicator",
3
- "version": "0.0.0-stage",
4
- "stub": true,
5
- "description": "Temporary package placeholder for staged publishing"
6
- }
3
+ "version": "0.3.0",
4
+ "description": "",
5
+ "keywords": [],
6
+ "files": [
7
+ "lib",
8
+ "src",
9
+ "migrations"
10
+ ],
11
+ "type": "module",
12
+ "main": "lib/index.js",
13
+ "types": "lib/index.d.ts",
14
+ "repository": "git@github.com:BlackGlory/extra-deduplicator.git",
15
+ "author": "BlackGlory <woshenmedoubuzhidao@blackglory.me>",
16
+ "license": "MIT",
17
+ "sideEffects": false,
18
+ "engines": {
19
+ "node": ">=22"
20
+ },
21
+ "scripts": {
22
+ "lint": "eslint --ext .js,.jsx,.ts,.tsx --quiet src __tests__",
23
+ "test": "vitest --run",
24
+ "prepublishOnly": "run-s clean build",
25
+ "clean": "rimraf lib",
26
+ "build": "tsc --project tsconfig.build.json && tsc-alias --project tsconfig.build.json",
27
+ "release": "standard-version"
28
+ },
29
+ "husky": {
30
+ "hooks": {
31
+ "pre-commit": "run-s lint build test",
32
+ "commit-msg": "commitlint -E HUSKY_GIT_PARAMS"
33
+ }
34
+ },
35
+ "devDependencies": {
36
+ "@commitlint/cli": "^21.2.3",
37
+ "@commitlint/config-conventional": "^21.2.3",
38
+ "@eslint/js": "^10.0.1",
39
+ "@types/better-sqlite3": "^9.6.0",
40
+ "@types/node": "22",
41
+ "eslint": "^10.0.2",
42
+ "husky": "^4.3.0",
43
+ "npm-run-all": "^4.1.5",
44
+ "rimraf": "^6.1.3",
45
+ "standard-version": "^9.5.0",
46
+ "tsc-alias": "^1.9.7",
47
+ "typescript": "6",
48
+ "typescript-eslint": "^8.56.1",
49
+ "vite": "^8.3.4",
50
+ "vitest": "^5.0.3"
51
+ },
52
+ "dependencies": {
53
+ "@blackglory/better-sqlite3-migrations": "^0.2.3",
54
+ "@blackglory/prelude": "^0.5.0",
55
+ "better-sqlite3": "^13.0.3",
56
+ "extra-compatible": "^0.3.0",
57
+ "extra-filesystem": "^0.6.2",
58
+ "extra-json-stable-stringify": "^0.1.2",
59
+ "extra-promise": "^7.1.1",
60
+ "iterable-operator": "^6.0.0",
61
+ "migration-files": "^0.4.3"
62
+ }
63
+ }
@@ -0,0 +1,144 @@
1
+ import Database, { Database as IDatabase } from 'better-sqlite3'
2
+ import { migrate } from '@blackglory/better-sqlite3-migrations'
3
+ import { findMigrationFilenames, readMigrationFile } from 'migration-files'
4
+ import { map } from 'extra-promise'
5
+ import { assert, isFinite } from '@blackglory/prelude'
6
+ import { findUpPackageFilename } from 'extra-filesystem'
7
+ import path from 'path'
8
+
9
+ interface IShrinkOptions {
10
+ target: number
11
+ threshold: number
12
+ }
13
+
14
+
15
+ export async function openDatabase(filename = ':memory:'): Promise<IDatabase> {
16
+ const packageFilename = await findUpPackageFilename(import.meta.dirname)
17
+ assert(packageFilename, 'package.json not found')
18
+
19
+ const packageRoot = path.dirname(packageFilename)
20
+ const migrationsPath = path.join(packageRoot, 'migrations')
21
+ const migrationFilenames = await findMigrationFilenames(migrationsPath)
22
+ const migrations = await map(migrationFilenames, readMigrationFile)
23
+
24
+ const db = new Database(filename)
25
+ migrate(db, migrations)
26
+
27
+ return db
28
+ }
29
+
30
+ export function hasHashes(db: IDatabase, hashes: string[]): boolean[] {
31
+ return hashes.map(hash => {
32
+ const row = db.prepare<
33
+ { hash: string }
34
+ , { hash_exists: number }
35
+ >(`
36
+ SELECT EXISTS(
37
+ SELECT *
38
+ FROM hash
39
+ WHERE hash = $hash
40
+ ) AS hash_exists
41
+ `).get({ hash })
42
+
43
+ return !!row?.hash_exists
44
+ })
45
+ }
46
+
47
+ export function isLastHash(db: IDatabase, hash: string): boolean {
48
+ const row = db.prepare<
49
+ []
50
+ , { hash: string }
51
+ >(`
52
+ SELECT hash
53
+ FROM hash
54
+ ORDER BY id DESC
55
+ LIMIT 1
56
+ `).get()
57
+
58
+ return row?.hash === hash
59
+ }
60
+
61
+ export function addHashes(
62
+ db: IDatabase
63
+ , hashes: string[]
64
+ , shrinkOptions: IShrinkOptions
65
+ ): string[] {
66
+ return db.transaction(() => {
67
+ const newHashes: string[] = []
68
+
69
+ for (const hash of hashes) {
70
+ const { changes } = db.prepare<{ hash: string }>(`
71
+ INSERT INTO hash (hash)
72
+ VALUES ($hash)
73
+ ON CONFLICT DO NOTHING
74
+ `).run({ hash })
75
+
76
+ if (changes === 1) {
77
+ newHashes.push(hash)
78
+ } else {
79
+ // 如果没有行改变, 则说明记录已经存在了, 重新插入此记录以更新其id.
80
+ db.prepare<{ hash: string }>(`
81
+ INSERT INTO hash (hash)
82
+ VALUES ($hash)
83
+ `).run({ hash })
84
+ }
85
+ }
86
+
87
+ shrink(db, shrinkOptions)
88
+
89
+ return newHashes
90
+ })()
91
+ }
92
+
93
+ export function addLastHash(
94
+ db: IDatabase
95
+ , hash: string
96
+ , shrinkOptions: IShrinkOptions
97
+ ): boolean {
98
+ return db.transaction(() => {
99
+ const row = db.prepare<[], { hash: string }>(`
100
+ SELECT hash
101
+ FROM hash
102
+ ORDER BY id DESC
103
+ LIMIT 1
104
+ `).get()
105
+
106
+ if (row?.hash === hash) {
107
+ return false
108
+ } else {
109
+ db.prepare<{ hash: string }>(`
110
+ INSERT INTO hash (hash)
111
+ VALUES ($hash)
112
+ `).run({ hash })
113
+ shrink(db, shrinkOptions)
114
+
115
+ return true
116
+ }
117
+ })()
118
+ }
119
+
120
+ function shrink(db: IDatabase, options: IShrinkOptions): void {
121
+ if (isFinite(options.threshold)) {
122
+ db.transaction(() => {
123
+ const count = countHashes(db)
124
+
125
+ if (count >= options.threshold) {
126
+ db.prepare<{ limit: number }>(`
127
+ DELETE FROM hash
128
+ ORDER BY id ASC
129
+ LIMIT $limit
130
+ `).run({ limit: count - options.target })
131
+ }
132
+ })()
133
+ }
134
+ }
135
+
136
+ function countHashes(db: IDatabase): number {
137
+ const row = db.prepare<[], { count: number }>(`
138
+ SELECT COUNT(*) as count
139
+ FROM hash
140
+ `).get()
141
+ assert(row)
142
+
143
+ return row.count
144
+ }
@@ -0,0 +1,103 @@
1
+ import { toArray, uniq, zip } from 'iterable-operator'
2
+ import { isntEmptyArray, assert, type Awaitable, isPositiveInfinity, isFinite } from '@blackglory/prelude'
3
+ import { map } from 'extra-promise'
4
+ import { openDatabase, hasHashes, isLastHash, addHashes, addLastHash } from './database.js'
5
+ import type { Database } from 'better-sqlite3'
6
+ import { stringify } from 'extra-json-stable-stringify'
7
+ import { sha256 } from 'extra-compatible'
8
+
9
+ export interface IOptions<T> {
10
+ filename?: string
11
+
12
+ hash?: (value: T) => Awaitable<string>
13
+
14
+ shrink?: {
15
+ target?: number
16
+ threshold?: number
17
+ }
18
+ }
19
+
20
+ export class Deduplicator<T> {
21
+ private constructor(
22
+ private db: Database
23
+ , private hash: (value: T) => Awaitable<string>
24
+ , private shrink: {
25
+ threshold: number
26
+ target: number
27
+ }
28
+ ) {}
29
+
30
+ static async create<T>(
31
+ {
32
+ filename
33
+ , hash = value => sha256(stringify(value))
34
+ , shrink: {
35
+ threshold = Infinity
36
+ , target = Math.floor(threshold / 2)
37
+ } = {}
38
+ }: IOptions<T> = {}
39
+ ): Promise<Deduplicator<T>> {
40
+ assert(
41
+ isPositiveInfinity(threshold) || Number.isInteger(threshold)
42
+ , 'The shrink.threshold must be Infinity or an integer'
43
+ )
44
+ assert(
45
+ isPositiveInfinity(target) || Number.isInteger(target)
46
+ , 'The shrink.target must be Infinity or an integer'
47
+ )
48
+ assert(threshold > 1, 'The shrink.threshold must be greater than 1')
49
+ if (isFinite(threshold)) {
50
+ assert(target < threshold, 'The shrink.target must be less than shrink.threshold')
51
+ }
52
+
53
+ const db = await openDatabase(filename)
54
+
55
+ return new Deduplicator<T>(db, hash, { threshold, target })
56
+ }
57
+
58
+ close(): void {
59
+ this.db.close()
60
+ }
61
+
62
+ async add(values: T[]): Promise<void> {
63
+ const hashes = await map(values, value => this.hash(value))
64
+
65
+ addHashes(this.db, hashes, this.shrink)
66
+ }
67
+
68
+ async diff(values: T[]): Promise<T[]> {
69
+ values = toArray(uniq(values))
70
+
71
+ const hashes = await map(values, value => this.hash(value))
72
+ const results = hasHashes(this.db, hashes)
73
+ const newValues = values.filter((_, index) => !results[index])
74
+
75
+ return newValues
76
+ }
77
+
78
+ async diffLast(value: T): Promise<boolean> {
79
+ const hash = await this.hash(value)
80
+
81
+ return !isLastHash(this.db, hash)
82
+ }
83
+
84
+ async addAndDiff(values: T[]): Promise<T[]> {
85
+ const hashes = await map(values, this.hash)
86
+
87
+ const newHashes = addHashes(this.db, hashes, this.shrink)
88
+
89
+ if (isntEmptyArray(newHashes)) {
90
+ const hashToValue = Object.fromEntries(zip(hashes, values))
91
+
92
+ return newHashes.map(hash => hashToValue[hash])
93
+ } else {
94
+ return []
95
+ }
96
+ }
97
+
98
+ async addAndDiffLast(value: T): Promise<boolean> {
99
+ const hash = await this.hash(value)
100
+
101
+ return addLastHash(this.db, hash, this.shrink)
102
+ }
103
+ }
package/src/index.ts ADDED
@@ -0,0 +1 @@
1
+ export * from './deduplicator.js'