extra-deduplicator 0.0.0-stage → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +44 -2
- package/lib/database.d.ts +11 -0
- package/lib/database.js +106 -0
- package/lib/database.js.map +1 -0
- package/lib/deduplicator.d.ts +22 -0
- package/lib/deduplicator.js +57 -0
- package/lib/deduplicator.js.map +1 -0
- package/lib/index.d.ts +1 -0
- package/lib/index.js +2 -0
- package/lib/index.js.map +1 -0
- package/migrations/001-initial.sql +18 -0
- package/migrations/schema.sql +18 -0
- package/package.json +61 -4
- package/src/database.ts +144 -0
- package/src/deduplicator.ts +103 -0
- package/src/index.ts +1 -0
package/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2024 BlackGlory <woshenmedoubuzhidao@blackglory.me>
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
package/README.md
CHANGED
|
@@ -1,3 +1,45 @@
|
|
|
1
|
-
#
|
|
1
|
+
# extra-deduplicator
|
|
2
|
+
A data deduplication library, which is designed for extracting new data.
|
|
2
3
|
|
|
3
|
-
|
|
4
|
+
## Install
|
|
5
|
+
```sh
|
|
6
|
+
npm install --save extra-deduplicator
|
|
7
|
+
```
|
|
8
|
+
|
|
9
|
+
## Usage
|
|
10
|
+
```ts
|
|
11
|
+
import { Deduplicator } from 'extra-deduplicator'
|
|
12
|
+
|
|
13
|
+
const deduplicator = await Deduplicator.create({ filename: 'dedup.db' })
|
|
14
|
+
|
|
15
|
+
const records = ['record-1', 'record-2']
|
|
16
|
+
const newRecords = await deduplicator.addAndDidff(records)
|
|
17
|
+
```
|
|
18
|
+
|
|
19
|
+
## API
|
|
20
|
+
```ts
|
|
21
|
+
interface IOptions<T> {
|
|
22
|
+
filename?: string
|
|
23
|
+
|
|
24
|
+
hash?: (value: T) => JSONValue
|
|
25
|
+
|
|
26
|
+
shrink?: {
|
|
27
|
+
target?: number
|
|
28
|
+
threshold?: number
|
|
29
|
+
}
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
class Deduplicator<T> {
|
|
33
|
+
static create(options?: IOptions): Promise<Deduplicator<T>>
|
|
34
|
+
|
|
35
|
+
add(values: T[]): Promise<void>
|
|
36
|
+
|
|
37
|
+
diff(values: T[]): Promise<T[]>
|
|
38
|
+
diffLast(value: T): Promise<boolean>
|
|
39
|
+
|
|
40
|
+
addAndDiff(values: T[]): Promise<T[]>
|
|
41
|
+
addAndDiffLast(value: T): Promise<boolean>
|
|
42
|
+
|
|
43
|
+
close(): void
|
|
44
|
+
}
|
|
45
|
+
```
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
import { Database as IDatabase } from 'better-sqlite3';
|
|
2
|
+
interface IShrinkOptions {
|
|
3
|
+
target: number;
|
|
4
|
+
threshold: number;
|
|
5
|
+
}
|
|
6
|
+
export declare function openDatabase(filename?: string): Promise<IDatabase>;
|
|
7
|
+
export declare function hasHashes(db: IDatabase, hashes: string[]): boolean[];
|
|
8
|
+
export declare function isLastHash(db: IDatabase, hash: string): boolean;
|
|
9
|
+
export declare function addHashes(db: IDatabase, hashes: string[], shrinkOptions: IShrinkOptions): string[];
|
|
10
|
+
export declare function addLastHash(db: IDatabase, hash: string, shrinkOptions: IShrinkOptions): boolean;
|
|
11
|
+
export {};
|
package/lib/database.js
ADDED
|
@@ -0,0 +1,106 @@
|
|
|
1
|
+
import Database from 'better-sqlite3';
|
|
2
|
+
import { migrate } from '@blackglory/better-sqlite3-migrations';
|
|
3
|
+
import { findMigrationFilenames, readMigrationFile } from 'migration-files';
|
|
4
|
+
import { map } from 'extra-promise';
|
|
5
|
+
import { assert, isFinite } from '@blackglory/prelude';
|
|
6
|
+
import { findUpPackageFilename } from 'extra-filesystem';
|
|
7
|
+
import path from 'path';
|
|
8
|
+
export async function openDatabase(filename = ':memory:') {
|
|
9
|
+
const packageFilename = await findUpPackageFilename(import.meta.dirname);
|
|
10
|
+
assert(packageFilename, 'package.json not found');
|
|
11
|
+
const packageRoot = path.dirname(packageFilename);
|
|
12
|
+
const migrationsPath = path.join(packageRoot, 'migrations');
|
|
13
|
+
const migrationFilenames = await findMigrationFilenames(migrationsPath);
|
|
14
|
+
const migrations = await map(migrationFilenames, readMigrationFile);
|
|
15
|
+
const db = new Database(filename);
|
|
16
|
+
migrate(db, migrations);
|
|
17
|
+
return db;
|
|
18
|
+
}
|
|
19
|
+
export function hasHashes(db, hashes) {
|
|
20
|
+
return hashes.map(hash => {
|
|
21
|
+
const row = db.prepare(`
|
|
22
|
+
SELECT EXISTS(
|
|
23
|
+
SELECT *
|
|
24
|
+
FROM hash
|
|
25
|
+
WHERE hash = $hash
|
|
26
|
+
) AS hash_exists
|
|
27
|
+
`).get({ hash });
|
|
28
|
+
return !!(row === null || row === void 0 ? void 0 : row.hash_exists);
|
|
29
|
+
});
|
|
30
|
+
}
|
|
31
|
+
export function isLastHash(db, hash) {
|
|
32
|
+
const row = db.prepare(`
|
|
33
|
+
SELECT hash
|
|
34
|
+
FROM hash
|
|
35
|
+
ORDER BY id DESC
|
|
36
|
+
LIMIT 1
|
|
37
|
+
`).get();
|
|
38
|
+
return (row === null || row === void 0 ? void 0 : row.hash) === hash;
|
|
39
|
+
}
|
|
40
|
+
export function addHashes(db, hashes, shrinkOptions) {
|
|
41
|
+
return db.transaction(() => {
|
|
42
|
+
const newHashes = [];
|
|
43
|
+
for (const hash of hashes) {
|
|
44
|
+
const { changes } = db.prepare(`
|
|
45
|
+
INSERT INTO hash (hash)
|
|
46
|
+
VALUES ($hash)
|
|
47
|
+
ON CONFLICT DO NOTHING
|
|
48
|
+
`).run({ hash });
|
|
49
|
+
if (changes === 1) {
|
|
50
|
+
newHashes.push(hash);
|
|
51
|
+
}
|
|
52
|
+
else {
|
|
53
|
+
db.prepare(`
|
|
54
|
+
INSERT INTO hash (hash)
|
|
55
|
+
VALUES ($hash)
|
|
56
|
+
`).run({ hash });
|
|
57
|
+
}
|
|
58
|
+
}
|
|
59
|
+
shrink(db, shrinkOptions);
|
|
60
|
+
return newHashes;
|
|
61
|
+
})();
|
|
62
|
+
}
|
|
63
|
+
export function addLastHash(db, hash, shrinkOptions) {
|
|
64
|
+
return db.transaction(() => {
|
|
65
|
+
const row = db.prepare(`
|
|
66
|
+
SELECT hash
|
|
67
|
+
FROM hash
|
|
68
|
+
ORDER BY id DESC
|
|
69
|
+
LIMIT 1
|
|
70
|
+
`).get();
|
|
71
|
+
if ((row === null || row === void 0 ? void 0 : row.hash) === hash) {
|
|
72
|
+
return false;
|
|
73
|
+
}
|
|
74
|
+
else {
|
|
75
|
+
db.prepare(`
|
|
76
|
+
INSERT INTO hash (hash)
|
|
77
|
+
VALUES ($hash)
|
|
78
|
+
`).run({ hash });
|
|
79
|
+
shrink(db, shrinkOptions);
|
|
80
|
+
return true;
|
|
81
|
+
}
|
|
82
|
+
})();
|
|
83
|
+
}
|
|
84
|
+
function shrink(db, options) {
|
|
85
|
+
if (isFinite(options.threshold)) {
|
|
86
|
+
db.transaction(() => {
|
|
87
|
+
const count = countHashes(db);
|
|
88
|
+
if (count >= options.threshold) {
|
|
89
|
+
db.prepare(`
|
|
90
|
+
DELETE FROM hash
|
|
91
|
+
ORDER BY id ASC
|
|
92
|
+
LIMIT $limit
|
|
93
|
+
`).run({ limit: count - options.target });
|
|
94
|
+
}
|
|
95
|
+
})();
|
|
96
|
+
}
|
|
97
|
+
}
|
|
98
|
+
function countHashes(db) {
|
|
99
|
+
const row = db.prepare(`
|
|
100
|
+
SELECT COUNT(*) as count
|
|
101
|
+
FROM hash
|
|
102
|
+
`).get();
|
|
103
|
+
assert(row);
|
|
104
|
+
return row.count;
|
|
105
|
+
}
|
|
106
|
+
//# sourceMappingURL=database.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"database.js","sourceRoot":"","sources":["../src/database.ts"],"names":[],"mappings":"AAAA,OAAO,QAAmC,MAAM,gBAAgB,CAAA;AAChE,OAAO,EAAE,OAAO,EAAE,MAAM,uCAAuC,CAAA;AAC/D,OAAO,EAAE,sBAAsB,EAAE,iBAAiB,EAAE,MAAM,iBAAiB,CAAA;AAC3E,OAAO,EAAE,GAAG,EAAE,MAAM,eAAe,CAAA;AACnC,OAAO,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,qBAAqB,CAAA;AACtD,OAAO,EAAE,qBAAqB,EAAE,MAAM,kBAAkB,CAAA;AACxD,OAAO,IAAI,MAAM,MAAM,CAAA;AAQvB,MAAM,CAAC,KAAK,UAAU,YAAY,CAAC,QAAQ,GAAG,UAAU;IACtD,MAAM,eAAe,GAAG,MAAM,qBAAqB,CAAC,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAA;IACxE,MAAM,CAAC,eAAe,EAAE,wBAAwB,CAAC,CAAA;IAEjD,MAAM,WAAW,GAAG,IAAI,CAAC,OAAO,CAAC,eAAe,CAAC,CAAA;IACjD,MAAM,cAAc,GAAG,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,YAAY,CAAC,CAAA;IAC3D,MAAM,kBAAkB,GAAG,MAAM,sBAAsB,CAAC,cAAc,CAAC,CAAA;IACvE,MAAM,UAAU,GAAG,MAAM,GAAG,CAAC,kBAAkB,EAAE,iBAAiB,CAAC,CAAA;IAEnE,MAAM,EAAE,GAAG,IAAI,QAAQ,CAAC,QAAQ,CAAC,CAAA;IACjC,OAAO,CAAC,EAAE,EAAE,UAAU,CAAC,CAAA;IAEvB,OAAO,EAAE,CAAA;AACX,CAAC;AAED,MAAM,UAAU,SAAS,CAAC,EAAa,EAAE,MAAgB;IACvD,OAAO,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE;QACvB,MAAM,GAAG,GAAG,EAAE,CAAC,OAAO,CAGpB;;;;;;KAMD,CAAC,CAAC,GAAG,CAAC,EAAE,IAAI,EAAE,CAAC,CAAA;QAEhB,OAAO,CAAC,CAAC,CAAA,GAAG,aAAH,GAAG,uBAAH,GAAG,CAAE,WAAW,CAAA,CAAA;IAC3B,CAAC,CAAC,CAAA;AACJ,CAAC;AAED,MAAM,UAAU,UAAU,CAAC,EAAa,EAAE,IAAY;IACpD,MAAM,GAAG,GAAG,EAAE,CAAC,OAAO,CAGpB;;;;;GAKD,CAAC,CAAC,GAAG,EAAE,CAAA;IAER,OAAO,CAAA,GAAG,aAAH,GAAG,uBAAH,GAAG,CAAE,IAAI,MAAK,IAAI,CAAA;AAC3B,CAAC;AAED,MAAM,UAAU,SAAS,CACvB,EAAa,EACb,MAAgB,EAChB,aAA6B;IAE7B,OAAO,EAAE,CAAC,WAAW,CAAC,GAAG,EAAE;QACzB,MAAM,SAAS,GAAa,EAAE,CAAA;QAE9B,KAAK,MAAM,IAAI,IAAI,MAAM,EAAE,CAAC;YAC1B,MAAM,EAAE,OAAO,EAAE,GAAG,EAAE,CAAC,OAAO,CAAmB;;;;OAIhD,CAAC,CAAC,GAAG,CAAC,EAAE,IAAI,EAAE,CAAC,CAAA;YAEhB,IAAI,OAAO,KAAK,CAAC,EAAE,CAAC;gBAClB,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;YACtB,CAAC;iBAAM,CAAC;gBAEN,EAAE,CAAC,OAAO,CAAmB;;;SAG5B,CAAC,CAAC,GAAG,CAAC,EAAE,IAAI,EAAE,CAAC,CAAA;YAClB,CAAC;QACH,CAAC;QAED,MAAM,CAAC,EAAE,EAAE,aAAa,CAAC,CAAA;QAEzB,OAAO,SAAS,CAAA;IAClB,CAAC,CAAC,EAAE,CAAA;AACN,CAAC;AAED,MAAM,UAAU,WAAW,CACzB,EAAa,EACb,IAAY,EACZ,aAA6B;IAE7B,OAAO,EAAE,CAAC,WAAW,CAAC,GAAG,EAAE;QACzB,MAAM,GAAG,GAAG,EAAE,CAAC,OAAO,CAAuB;;;;;KAK5C,CAAC,CAAC,GAAG,EAAE,CAAA;QAER,IAAI,CAAA,GAAG,aAAH,GAAG,uBAAH,GAAG,CAAE,IAAI,MAAK,IAAI,EAAE,CAAC;YACvB,OAAO,KAAK,CAAA;QACd,CAAC;aAAM,CAAC;YACN,EAAE,CAAC,OAAO,CAAmB;;;OAG5B,CAAC,CAAC,GAAG,CAAC,EAAE,IAAI,EAAE,CAAC,CAAA;YAChB,MAAM,CAAC,EAAE,EAAE,aAAa,CAAC,CAAA;YAEzB,OAAO,IAAI,CAAA;QACb,CAAC;IACH,CAAC,CAAC,EAAE,CAAA;AACN,CAAC;AAED,SAAS,MAAM,CAAC,EAAa,EAAE,OAAuB;IACpD,IAAI,QAAQ,CAAC,OAAO,CAAC,SAAS,CAAC,EAAE,CAAC;QAChC,EAAE,CAAC,WAAW,CAAC,GAAG,EAAE;YAClB,MAAM,KAAK,GAAG,WAAW,CAAC,EAAE,CAAC,CAAA;YAE7B,IAAI,KAAK,IAAI,OAAO,CAAC,SAAS,EAAE,CAAC;gBAC/B,EAAE,CAAC,OAAO,CAAoB;;;;SAI7B,CAAC,CAAC,GAAG,CAAC,EAAE,KAAK,EAAE,KAAK,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,CAAA;YAC3C,CAAC;QACH,CAAC,CAAC,EAAE,CAAA;IACN,CAAC;AACH,CAAC;AAED,SAAS,WAAW,CAAC,EAAa;IAChC,MAAM,GAAG,GAAG,EAAE,CAAC,OAAO,CAAwB;;;GAG7C,CAAC,CAAC,GAAG,EAAE,CAAA;IACR,MAAM,CAAC,GAAG,CAAC,CAAA;IAEX,OAAO,GAAG,CAAC,KAAK,CAAA;AAClB,CAAC"}
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
import { type Awaitable } from '@blackglory/prelude';
|
|
2
|
+
export interface IOptions<T> {
|
|
3
|
+
filename?: string;
|
|
4
|
+
hash?: (value: T) => Awaitable<string>;
|
|
5
|
+
shrink?: {
|
|
6
|
+
target?: number;
|
|
7
|
+
threshold?: number;
|
|
8
|
+
};
|
|
9
|
+
}
|
|
10
|
+
export declare class Deduplicator<T> {
|
|
11
|
+
private db;
|
|
12
|
+
private hash;
|
|
13
|
+
private shrink;
|
|
14
|
+
private constructor();
|
|
15
|
+
static create<T>({ filename, hash, shrink: { threshold, target } }?: IOptions<T>): Promise<Deduplicator<T>>;
|
|
16
|
+
close(): void;
|
|
17
|
+
add(values: T[]): Promise<void>;
|
|
18
|
+
diff(values: T[]): Promise<T[]>;
|
|
19
|
+
diffLast(value: T): Promise<boolean>;
|
|
20
|
+
addAndDiff(values: T[]): Promise<T[]>;
|
|
21
|
+
addAndDiffLast(value: T): Promise<boolean>;
|
|
22
|
+
}
|
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
import { toArray, uniq, zip } from 'iterable-operator';
|
|
2
|
+
import { isntEmptyArray, assert, isPositiveInfinity, isFinite } from '@blackglory/prelude';
|
|
3
|
+
import { map } from 'extra-promise';
|
|
4
|
+
import { openDatabase, hasHashes, isLastHash, addHashes, addLastHash } from './database.js';
|
|
5
|
+
import { stringify } from 'extra-json-stable-stringify';
|
|
6
|
+
import { sha256 } from 'extra-compatible';
|
|
7
|
+
export class Deduplicator {
|
|
8
|
+
constructor(db, hash, shrink) {
|
|
9
|
+
this.db = db;
|
|
10
|
+
this.hash = hash;
|
|
11
|
+
this.shrink = shrink;
|
|
12
|
+
}
|
|
13
|
+
static async create({ filename, hash = value => sha256(stringify(value)), shrink: { threshold = Infinity, target = Math.floor(threshold / 2) } = {} } = {}) {
|
|
14
|
+
assert(isPositiveInfinity(threshold) || Number.isInteger(threshold), 'The shrink.threshold must be Infinity or an integer');
|
|
15
|
+
assert(isPositiveInfinity(target) || Number.isInteger(target), 'The shrink.target must be Infinity or an integer');
|
|
16
|
+
assert(threshold > 1, 'The shrink.threshold must be greater than 1');
|
|
17
|
+
if (isFinite(threshold)) {
|
|
18
|
+
assert(target < threshold, 'The shrink.target must be less than shrink.threshold');
|
|
19
|
+
}
|
|
20
|
+
const db = await openDatabase(filename);
|
|
21
|
+
return new Deduplicator(db, hash, { threshold, target });
|
|
22
|
+
}
|
|
23
|
+
close() {
|
|
24
|
+
this.db.close();
|
|
25
|
+
}
|
|
26
|
+
async add(values) {
|
|
27
|
+
const hashes = await map(values, value => this.hash(value));
|
|
28
|
+
addHashes(this.db, hashes, this.shrink);
|
|
29
|
+
}
|
|
30
|
+
async diff(values) {
|
|
31
|
+
values = toArray(uniq(values));
|
|
32
|
+
const hashes = await map(values, value => this.hash(value));
|
|
33
|
+
const results = hasHashes(this.db, hashes);
|
|
34
|
+
const newValues = values.filter((_, index) => !results[index]);
|
|
35
|
+
return newValues;
|
|
36
|
+
}
|
|
37
|
+
async diffLast(value) {
|
|
38
|
+
const hash = await this.hash(value);
|
|
39
|
+
return !isLastHash(this.db, hash);
|
|
40
|
+
}
|
|
41
|
+
async addAndDiff(values) {
|
|
42
|
+
const hashes = await map(values, this.hash);
|
|
43
|
+
const newHashes = addHashes(this.db, hashes, this.shrink);
|
|
44
|
+
if (isntEmptyArray(newHashes)) {
|
|
45
|
+
const hashToValue = Object.fromEntries(zip(hashes, values));
|
|
46
|
+
return newHashes.map(hash => hashToValue[hash]);
|
|
47
|
+
}
|
|
48
|
+
else {
|
|
49
|
+
return [];
|
|
50
|
+
}
|
|
51
|
+
}
|
|
52
|
+
async addAndDiffLast(value) {
|
|
53
|
+
const hash = await this.hash(value);
|
|
54
|
+
return addLastHash(this.db, hash, this.shrink);
|
|
55
|
+
}
|
|
56
|
+
}
|
|
57
|
+
//# sourceMappingURL=deduplicator.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"deduplicator.js","sourceRoot":"","sources":["../src/deduplicator.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,GAAG,EAAE,MAAM,mBAAmB,CAAA;AACtD,OAAO,EAAE,cAAc,EAAE,MAAM,EAAkB,kBAAkB,EAAE,QAAQ,EAAE,MAAM,qBAAqB,CAAA;AAC1G,OAAO,EAAE,GAAG,EAAE,MAAM,eAAe,CAAA;AACnC,OAAO,EAAE,YAAY,EAAE,SAAS,EAAE,UAAU,EAAE,SAAS,EAAE,WAAW,EAAE,MAAM,eAAe,CAAA;AAE3F,OAAO,EAAE,SAAS,EAAE,MAAM,6BAA6B,CAAA;AACvD,OAAO,EAAE,MAAM,EAAE,MAAM,kBAAkB,CAAA;AAazC,MAAM,OAAO,YAAY;IACvB,YACU,EAAY,EACZ,IAAqC,EACrC,MAGP;QALO,OAAE,GAAF,EAAE,CAAU;QACZ,SAAI,GAAJ,IAAI,CAAiC;QACrC,WAAM,GAAN,MAAM,CAGb;IACA,CAAC;IAEJ,MAAM,CAAC,KAAK,CAAC,MAAM,CACjB,EACE,QAAQ,EACR,IAAI,GAAG,KAAK,CAAC,EAAE,CAAC,MAAM,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,EACxC,MAAM,EAAE,EACN,SAAS,GAAG,QAAQ,EACpB,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,SAAS,GAAG,CAAC,CAAC,EACnC,GAAG,EAAE,KACS,EAAE;QAEnB,MAAM,CACJ,kBAAkB,CAAC,SAAS,CAAC,IAAI,MAAM,CAAC,SAAS,CAAC,SAAS,CAAC,EAC5D,qDAAqD,CACtD,CAAA;QACD,MAAM,CACJ,kBAAkB,CAAC,MAAM,CAAC,IAAI,MAAM,CAAC,SAAS,CAAC,MAAM,CAAC,EACtD,kDAAkD,CACnD,CAAA;QACD,MAAM,CAAC,SAAS,GAAG,CAAC,EAAE,6CAA6C,CAAC,CAAA;QACpE,IAAI,QAAQ,CAAC,SAAS,CAAC,EAAE,CAAC;YACxB,MAAM,CAAC,MAAM,GAAG,SAAS,EAAE,sDAAsD,CAAC,CAAA;QACpF,CAAC;QAED,MAAM,EAAE,GAAG,MAAM,YAAY,CAAC,QAAQ,CAAC,CAAA;QAEvC,OAAO,IAAI,YAAY,CAAI,EAAE,EAAE,IAAI,EAAE,EAAE,SAAS,EAAE,MAAM,EAAE,CAAC,CAAA;IAC7D,CAAC;IAED,KAAK;QACH,IAAI,CAAC,EAAE,CAAC,KAAK,EAAE,CAAA;IACjB,CAAC;IAED,KAAK,CAAC,GAAG,CAAC,MAAW;QACnB,MAAM,MAAM,GAAG,MAAM,GAAG,CAAC,MAAM,EAAE,KAAK,CAAC,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAA;QAE3D,SAAS,CAAC,IAAI,CAAC,EAAE,EAAE,MAAM,EAAE,IAAI,CAAC,MAAM,CAAC,CAAA;IACzC,CAAC;IAED,KAAK,CAAC,IAAI,CAAC,MAAW;QACpB,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAA;QAE9B,MAAM,MAAM,GAAG,MAAM,GAAG,CAAC,MAAM,EAAE,KAAK,CAAC,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAA;QAC3D,MAAM,OAAO,GAAG,SAAS,CAAC,IAAI,CAAC,EAAE,EAAE,MAAM,CAAC,CAAA;QAC1C,MAAM,SAAS,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAA;QAE9D,OAAO,SAAS,CAAA;IAClB,CAAC;IAED,KAAK,CAAC,QAAQ,CAAC,KAAQ;QACrB,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;QAEnC,OAAO,CAAC,UAAU,CAAC,IAAI,CAAC,EAAE,EAAE,IAAI,CAAC,CAAA;IACnC,CAAC;IAED,KAAK,CAAC,UAAU,CAAC,MAAW;QAC1B,MAAM,MAAM,GAAG,MAAM,GAAG,CAAC,MAAM,EAAE,IAAI,CAAC,IAAI,CAAC,CAAA;QAE3C,MAAM,SAAS,GAAG,SAAS,CAAC,IAAI,CAAC,EAAE,EAAE,MAAM,EAAE,IAAI,CAAC,MAAM,CAAC,CAAA;QAEzD,IAAI,cAAc,CAAC,SAAS,CAAC,EAAE,CAAC;YAC9B,MAAM,WAAW,GAAG,MAAM,CAAC,WAAW,CAAC,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAAA;YAE3D,OAAO,SAAS,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC,CAAA;QACjD,CAAC;aAAM,CAAC;YACN,OAAO,EAAE,CAAA;QACX,CAAC;IACH,CAAC;IAED,KAAK,CAAC,cAAc,CAAC,KAAQ;QAC3B,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;QAEnC,OAAO,WAAW,CAAC,IAAI,CAAC,EAAE,EAAE,IAAI,EAAE,IAAI,CAAC,MAAM,CAAC,CAAA;IAChD,CAAC;CACF"}
|
package/lib/index.d.ts
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export * from './deduplicator.js';
|
package/lib/index.js
ADDED
package/lib/index.js.map
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,cAAc,mBAAmB,CAAA"}
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
--------------------------------------------------------------------------------
|
|
2
|
+
-- Up
|
|
3
|
+
--------------------------------------------------------------------------------
|
|
4
|
+
|
|
5
|
+
PRAGMA journal_mode = WAL;
|
|
6
|
+
|
|
7
|
+
CREATE TABLE hash (
|
|
8
|
+
id INTEGER PRIMARY KEY AUTOINCREMENT
|
|
9
|
+
, hash TEXT UNIQUE ON CONFLICT REPLACE
|
|
10
|
+
);
|
|
11
|
+
|
|
12
|
+
--------------------------------------------------------------------------------
|
|
13
|
+
-- Down
|
|
14
|
+
--------------------------------------------------------------------------------
|
|
15
|
+
|
|
16
|
+
PRAGMA journal_mode = DELETE;
|
|
17
|
+
|
|
18
|
+
DROP TABLE hash;
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
--------------------------------------------------------------------------------
|
|
2
|
+
-- Up
|
|
3
|
+
--------------------------------------------------------------------------------
|
|
4
|
+
|
|
5
|
+
PRAGMA journal_mode = WAL;
|
|
6
|
+
|
|
7
|
+
CREATE TABLE hash (
|
|
8
|
+
id INTEGER PRIMARY KEY AUTOINCREMENT
|
|
9
|
+
, hash TEXT UNIQUE ON CONFLICT REPLACE
|
|
10
|
+
);
|
|
11
|
+
|
|
12
|
+
--------------------------------------------------------------------------------
|
|
13
|
+
-- Down
|
|
14
|
+
--------------------------------------------------------------------------------
|
|
15
|
+
|
|
16
|
+
PRAGMA journal_mode = DELETE;
|
|
17
|
+
|
|
18
|
+
DROP TABLE hash;
|
package/package.json
CHANGED
|
@@ -1,6 +1,63 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "extra-deduplicator",
|
|
3
|
-
"version": "0.
|
|
4
|
-
"
|
|
5
|
-
"
|
|
6
|
-
|
|
3
|
+
"version": "0.3.0",
|
|
4
|
+
"description": "",
|
|
5
|
+
"keywords": [],
|
|
6
|
+
"files": [
|
|
7
|
+
"lib",
|
|
8
|
+
"src",
|
|
9
|
+
"migrations"
|
|
10
|
+
],
|
|
11
|
+
"type": "module",
|
|
12
|
+
"main": "lib/index.js",
|
|
13
|
+
"types": "lib/index.d.ts",
|
|
14
|
+
"repository": "git@github.com:BlackGlory/extra-deduplicator.git",
|
|
15
|
+
"author": "BlackGlory <woshenmedoubuzhidao@blackglory.me>",
|
|
16
|
+
"license": "MIT",
|
|
17
|
+
"sideEffects": false,
|
|
18
|
+
"engines": {
|
|
19
|
+
"node": ">=22"
|
|
20
|
+
},
|
|
21
|
+
"scripts": {
|
|
22
|
+
"lint": "eslint --ext .js,.jsx,.ts,.tsx --quiet src __tests__",
|
|
23
|
+
"test": "vitest --run",
|
|
24
|
+
"prepublishOnly": "run-s clean build",
|
|
25
|
+
"clean": "rimraf lib",
|
|
26
|
+
"build": "tsc --project tsconfig.build.json && tsc-alias --project tsconfig.build.json",
|
|
27
|
+
"release": "standard-version"
|
|
28
|
+
},
|
|
29
|
+
"husky": {
|
|
30
|
+
"hooks": {
|
|
31
|
+
"pre-commit": "run-s lint build test",
|
|
32
|
+
"commit-msg": "commitlint -E HUSKY_GIT_PARAMS"
|
|
33
|
+
}
|
|
34
|
+
},
|
|
35
|
+
"devDependencies": {
|
|
36
|
+
"@commitlint/cli": "^21.2.3",
|
|
37
|
+
"@commitlint/config-conventional": "^21.2.3",
|
|
38
|
+
"@eslint/js": "^10.0.1",
|
|
39
|
+
"@types/better-sqlite3": "^9.6.0",
|
|
40
|
+
"@types/node": "22",
|
|
41
|
+
"eslint": "^10.0.2",
|
|
42
|
+
"husky": "^4.3.0",
|
|
43
|
+
"npm-run-all": "^4.1.5",
|
|
44
|
+
"rimraf": "^6.1.3",
|
|
45
|
+
"standard-version": "^9.5.0",
|
|
46
|
+
"tsc-alias": "^1.9.7",
|
|
47
|
+
"typescript": "6",
|
|
48
|
+
"typescript-eslint": "^8.56.1",
|
|
49
|
+
"vite": "^8.3.4",
|
|
50
|
+
"vitest": "^5.0.3"
|
|
51
|
+
},
|
|
52
|
+
"dependencies": {
|
|
53
|
+
"@blackglory/better-sqlite3-migrations": "^0.2.3",
|
|
54
|
+
"@blackglory/prelude": "^0.5.0",
|
|
55
|
+
"better-sqlite3": "^13.0.3",
|
|
56
|
+
"extra-compatible": "^0.3.0",
|
|
57
|
+
"extra-filesystem": "^0.6.2",
|
|
58
|
+
"extra-json-stable-stringify": "^0.1.2",
|
|
59
|
+
"extra-promise": "^7.1.1",
|
|
60
|
+
"iterable-operator": "^6.0.0",
|
|
61
|
+
"migration-files": "^0.4.3"
|
|
62
|
+
}
|
|
63
|
+
}
|
package/src/database.ts
ADDED
|
@@ -0,0 +1,144 @@
|
|
|
1
|
+
import Database, { Database as IDatabase } from 'better-sqlite3'
|
|
2
|
+
import { migrate } from '@blackglory/better-sqlite3-migrations'
|
|
3
|
+
import { findMigrationFilenames, readMigrationFile } from 'migration-files'
|
|
4
|
+
import { map } from 'extra-promise'
|
|
5
|
+
import { assert, isFinite } from '@blackglory/prelude'
|
|
6
|
+
import { findUpPackageFilename } from 'extra-filesystem'
|
|
7
|
+
import path from 'path'
|
|
8
|
+
|
|
9
|
+
interface IShrinkOptions {
|
|
10
|
+
target: number
|
|
11
|
+
threshold: number
|
|
12
|
+
}
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
export async function openDatabase(filename = ':memory:'): Promise<IDatabase> {
|
|
16
|
+
const packageFilename = await findUpPackageFilename(import.meta.dirname)
|
|
17
|
+
assert(packageFilename, 'package.json not found')
|
|
18
|
+
|
|
19
|
+
const packageRoot = path.dirname(packageFilename)
|
|
20
|
+
const migrationsPath = path.join(packageRoot, 'migrations')
|
|
21
|
+
const migrationFilenames = await findMigrationFilenames(migrationsPath)
|
|
22
|
+
const migrations = await map(migrationFilenames, readMigrationFile)
|
|
23
|
+
|
|
24
|
+
const db = new Database(filename)
|
|
25
|
+
migrate(db, migrations)
|
|
26
|
+
|
|
27
|
+
return db
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
export function hasHashes(db: IDatabase, hashes: string[]): boolean[] {
|
|
31
|
+
return hashes.map(hash => {
|
|
32
|
+
const row = db.prepare<
|
|
33
|
+
{ hash: string }
|
|
34
|
+
, { hash_exists: number }
|
|
35
|
+
>(`
|
|
36
|
+
SELECT EXISTS(
|
|
37
|
+
SELECT *
|
|
38
|
+
FROM hash
|
|
39
|
+
WHERE hash = $hash
|
|
40
|
+
) AS hash_exists
|
|
41
|
+
`).get({ hash })
|
|
42
|
+
|
|
43
|
+
return !!row?.hash_exists
|
|
44
|
+
})
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
export function isLastHash(db: IDatabase, hash: string): boolean {
|
|
48
|
+
const row = db.prepare<
|
|
49
|
+
[]
|
|
50
|
+
, { hash: string }
|
|
51
|
+
>(`
|
|
52
|
+
SELECT hash
|
|
53
|
+
FROM hash
|
|
54
|
+
ORDER BY id DESC
|
|
55
|
+
LIMIT 1
|
|
56
|
+
`).get()
|
|
57
|
+
|
|
58
|
+
return row?.hash === hash
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
export function addHashes(
|
|
62
|
+
db: IDatabase
|
|
63
|
+
, hashes: string[]
|
|
64
|
+
, shrinkOptions: IShrinkOptions
|
|
65
|
+
): string[] {
|
|
66
|
+
return db.transaction(() => {
|
|
67
|
+
const newHashes: string[] = []
|
|
68
|
+
|
|
69
|
+
for (const hash of hashes) {
|
|
70
|
+
const { changes } = db.prepare<{ hash: string }>(`
|
|
71
|
+
INSERT INTO hash (hash)
|
|
72
|
+
VALUES ($hash)
|
|
73
|
+
ON CONFLICT DO NOTHING
|
|
74
|
+
`).run({ hash })
|
|
75
|
+
|
|
76
|
+
if (changes === 1) {
|
|
77
|
+
newHashes.push(hash)
|
|
78
|
+
} else {
|
|
79
|
+
// 如果没有行改变, 则说明记录已经存在了, 重新插入此记录以更新其id.
|
|
80
|
+
db.prepare<{ hash: string }>(`
|
|
81
|
+
INSERT INTO hash (hash)
|
|
82
|
+
VALUES ($hash)
|
|
83
|
+
`).run({ hash })
|
|
84
|
+
}
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
shrink(db, shrinkOptions)
|
|
88
|
+
|
|
89
|
+
return newHashes
|
|
90
|
+
})()
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
export function addLastHash(
|
|
94
|
+
db: IDatabase
|
|
95
|
+
, hash: string
|
|
96
|
+
, shrinkOptions: IShrinkOptions
|
|
97
|
+
): boolean {
|
|
98
|
+
return db.transaction(() => {
|
|
99
|
+
const row = db.prepare<[], { hash: string }>(`
|
|
100
|
+
SELECT hash
|
|
101
|
+
FROM hash
|
|
102
|
+
ORDER BY id DESC
|
|
103
|
+
LIMIT 1
|
|
104
|
+
`).get()
|
|
105
|
+
|
|
106
|
+
if (row?.hash === hash) {
|
|
107
|
+
return false
|
|
108
|
+
} else {
|
|
109
|
+
db.prepare<{ hash: string }>(`
|
|
110
|
+
INSERT INTO hash (hash)
|
|
111
|
+
VALUES ($hash)
|
|
112
|
+
`).run({ hash })
|
|
113
|
+
shrink(db, shrinkOptions)
|
|
114
|
+
|
|
115
|
+
return true
|
|
116
|
+
}
|
|
117
|
+
})()
|
|
118
|
+
}
|
|
119
|
+
|
|
120
|
+
function shrink(db: IDatabase, options: IShrinkOptions): void {
|
|
121
|
+
if (isFinite(options.threshold)) {
|
|
122
|
+
db.transaction(() => {
|
|
123
|
+
const count = countHashes(db)
|
|
124
|
+
|
|
125
|
+
if (count >= options.threshold) {
|
|
126
|
+
db.prepare<{ limit: number }>(`
|
|
127
|
+
DELETE FROM hash
|
|
128
|
+
ORDER BY id ASC
|
|
129
|
+
LIMIT $limit
|
|
130
|
+
`).run({ limit: count - options.target })
|
|
131
|
+
}
|
|
132
|
+
})()
|
|
133
|
+
}
|
|
134
|
+
}
|
|
135
|
+
|
|
136
|
+
function countHashes(db: IDatabase): number {
|
|
137
|
+
const row = db.prepare<[], { count: number }>(`
|
|
138
|
+
SELECT COUNT(*) as count
|
|
139
|
+
FROM hash
|
|
140
|
+
`).get()
|
|
141
|
+
assert(row)
|
|
142
|
+
|
|
143
|
+
return row.count
|
|
144
|
+
}
|
|
@@ -0,0 +1,103 @@
|
|
|
1
|
+
import { toArray, uniq, zip } from 'iterable-operator'
|
|
2
|
+
import { isntEmptyArray, assert, type Awaitable, isPositiveInfinity, isFinite } from '@blackglory/prelude'
|
|
3
|
+
import { map } from 'extra-promise'
|
|
4
|
+
import { openDatabase, hasHashes, isLastHash, addHashes, addLastHash } from './database.js'
|
|
5
|
+
import type { Database } from 'better-sqlite3'
|
|
6
|
+
import { stringify } from 'extra-json-stable-stringify'
|
|
7
|
+
import { sha256 } from 'extra-compatible'
|
|
8
|
+
|
|
9
|
+
export interface IOptions<T> {
|
|
10
|
+
filename?: string
|
|
11
|
+
|
|
12
|
+
hash?: (value: T) => Awaitable<string>
|
|
13
|
+
|
|
14
|
+
shrink?: {
|
|
15
|
+
target?: number
|
|
16
|
+
threshold?: number
|
|
17
|
+
}
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
export class Deduplicator<T> {
|
|
21
|
+
private constructor(
|
|
22
|
+
private db: Database
|
|
23
|
+
, private hash: (value: T) => Awaitable<string>
|
|
24
|
+
, private shrink: {
|
|
25
|
+
threshold: number
|
|
26
|
+
target: number
|
|
27
|
+
}
|
|
28
|
+
) {}
|
|
29
|
+
|
|
30
|
+
static async create<T>(
|
|
31
|
+
{
|
|
32
|
+
filename
|
|
33
|
+
, hash = value => sha256(stringify(value))
|
|
34
|
+
, shrink: {
|
|
35
|
+
threshold = Infinity
|
|
36
|
+
, target = Math.floor(threshold / 2)
|
|
37
|
+
} = {}
|
|
38
|
+
}: IOptions<T> = {}
|
|
39
|
+
): Promise<Deduplicator<T>> {
|
|
40
|
+
assert(
|
|
41
|
+
isPositiveInfinity(threshold) || Number.isInteger(threshold)
|
|
42
|
+
, 'The shrink.threshold must be Infinity or an integer'
|
|
43
|
+
)
|
|
44
|
+
assert(
|
|
45
|
+
isPositiveInfinity(target) || Number.isInteger(target)
|
|
46
|
+
, 'The shrink.target must be Infinity or an integer'
|
|
47
|
+
)
|
|
48
|
+
assert(threshold > 1, 'The shrink.threshold must be greater than 1')
|
|
49
|
+
if (isFinite(threshold)) {
|
|
50
|
+
assert(target < threshold, 'The shrink.target must be less than shrink.threshold')
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
const db = await openDatabase(filename)
|
|
54
|
+
|
|
55
|
+
return new Deduplicator<T>(db, hash, { threshold, target })
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
close(): void {
|
|
59
|
+
this.db.close()
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
async add(values: T[]): Promise<void> {
|
|
63
|
+
const hashes = await map(values, value => this.hash(value))
|
|
64
|
+
|
|
65
|
+
addHashes(this.db, hashes, this.shrink)
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
async diff(values: T[]): Promise<T[]> {
|
|
69
|
+
values = toArray(uniq(values))
|
|
70
|
+
|
|
71
|
+
const hashes = await map(values, value => this.hash(value))
|
|
72
|
+
const results = hasHashes(this.db, hashes)
|
|
73
|
+
const newValues = values.filter((_, index) => !results[index])
|
|
74
|
+
|
|
75
|
+
return newValues
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
async diffLast(value: T): Promise<boolean> {
|
|
79
|
+
const hash = await this.hash(value)
|
|
80
|
+
|
|
81
|
+
return !isLastHash(this.db, hash)
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
async addAndDiff(values: T[]): Promise<T[]> {
|
|
85
|
+
const hashes = await map(values, this.hash)
|
|
86
|
+
|
|
87
|
+
const newHashes = addHashes(this.db, hashes, this.shrink)
|
|
88
|
+
|
|
89
|
+
if (isntEmptyArray(newHashes)) {
|
|
90
|
+
const hashToValue = Object.fromEntries(zip(hashes, values))
|
|
91
|
+
|
|
92
|
+
return newHashes.map(hash => hashToValue[hash])
|
|
93
|
+
} else {
|
|
94
|
+
return []
|
|
95
|
+
}
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
async addAndDiffLast(value: T): Promise<boolean> {
|
|
99
|
+
const hash = await this.hash(value)
|
|
100
|
+
|
|
101
|
+
return addLastHash(this.db, hash, this.shrink)
|
|
102
|
+
}
|
|
103
|
+
}
|
package/src/index.ts
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export * from './deduplicator.js'
|