extra-deduplicator 0.3.0 → 0.3.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -0
- package/lib/database.d.ts +1 -0
- package/lib/database.js +14 -4
- package/lib/database.js.map +1 -1
- package/lib/deduplicator.d.ts +1 -0
- package/lib/deduplicator.js +4 -1
- package/lib/deduplicator.js.map +1 -1
- package/package.json +4 -1
- package/src/database.ts +16 -4
- package/src/deduplicator.ts +5 -1
package/README.md
CHANGED
package/lib/database.d.ts
CHANGED
|
@@ -8,4 +8,5 @@ export declare function hasHashes(db: IDatabase, hashes: string[]): boolean[];
|
|
|
8
8
|
export declare function isLastHash(db: IDatabase, hash: string): boolean;
|
|
9
9
|
export declare function addHashes(db: IDatabase, hashes: string[], shrinkOptions: IShrinkOptions): string[];
|
|
10
10
|
export declare function addLastHash(db: IDatabase, hash: string, shrinkOptions: IShrinkOptions): boolean;
|
|
11
|
+
export declare function isEmpty(db: IDatabase): boolean;
|
|
11
12
|
export {};
|
package/lib/database.js
CHANGED
|
@@ -20,10 +20,10 @@ export function hasHashes(db, hashes) {
|
|
|
20
20
|
return hashes.map(hash => {
|
|
21
21
|
const row = db.prepare(`
|
|
22
22
|
SELECT EXISTS(
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
23
|
+
SELECT *
|
|
24
|
+
FROM hash
|
|
25
|
+
WHERE hash = $hash
|
|
26
|
+
) AS hash_exists
|
|
27
27
|
`).get({ hash });
|
|
28
28
|
return !!(row === null || row === void 0 ? void 0 : row.hash_exists);
|
|
29
29
|
});
|
|
@@ -95,6 +95,16 @@ function shrink(db, options) {
|
|
|
95
95
|
})();
|
|
96
96
|
}
|
|
97
97
|
}
|
|
98
|
+
export function isEmpty(db) {
|
|
99
|
+
const row = db.prepare(`
|
|
100
|
+
SELECT EXISTS (
|
|
101
|
+
SELECT 1
|
|
102
|
+
FROM hash
|
|
103
|
+
LIMIT 1
|
|
104
|
+
) AS matched
|
|
105
|
+
`).get();
|
|
106
|
+
return !(row === null || row === void 0 ? void 0 : row.matched);
|
|
107
|
+
}
|
|
98
108
|
function countHashes(db) {
|
|
99
109
|
const row = db.prepare(`
|
|
100
110
|
SELECT COUNT(*) as count
|
package/lib/database.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"database.js","sourceRoot":"","sources":["../src/database.ts"],"names":[],"mappings":"AAAA,OAAO,QAAmC,MAAM,gBAAgB,CAAA;AAChE,OAAO,EAAE,OAAO,EAAE,MAAM,uCAAuC,CAAA;AAC/D,OAAO,EAAE,sBAAsB,EAAE,iBAAiB,EAAE,MAAM,iBAAiB,CAAA;AAC3E,OAAO,EAAE,GAAG,EAAE,MAAM,eAAe,CAAA;AACnC,OAAO,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,qBAAqB,CAAA;AACtD,OAAO,EAAE,qBAAqB,EAAE,MAAM,kBAAkB,CAAA;AACxD,OAAO,IAAI,MAAM,MAAM,CAAA;AAQvB,MAAM,CAAC,KAAK,UAAU,YAAY,CAAC,QAAQ,GAAG,UAAU;IACtD,MAAM,eAAe,GAAG,MAAM,qBAAqB,CAAC,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAA;IACxE,MAAM,CAAC,eAAe,EAAE,wBAAwB,CAAC,CAAA;IAEjD,MAAM,WAAW,GAAG,IAAI,CAAC,OAAO,CAAC,eAAe,CAAC,CAAA;IACjD,MAAM,cAAc,GAAG,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,YAAY,CAAC,CAAA;IAC3D,MAAM,kBAAkB,GAAG,MAAM,sBAAsB,CAAC,cAAc,CAAC,CAAA;IACvE,MAAM,UAAU,GAAG,MAAM,GAAG,CAAC,kBAAkB,EAAE,iBAAiB,CAAC,CAAA;IAEnE,MAAM,EAAE,GAAG,IAAI,QAAQ,CAAC,QAAQ,CAAC,CAAA;IACjC,OAAO,CAAC,EAAE,EAAE,UAAU,CAAC,CAAA;IAEvB,OAAO,EAAE,CAAA;AACX,CAAC;AAED,MAAM,UAAU,SAAS,CAAC,EAAa,EAAE,MAAgB;IACvD,OAAO,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE;QACvB,MAAM,GAAG,GAAG,EAAE,CAAC,OAAO,CAGpB;;;;;;KAMD,CAAC,CAAC,GAAG,CAAC,EAAE,IAAI,EAAE,CAAC,CAAA;QAEhB,OAAO,CAAC,CAAC,CAAA,GAAG,aAAH,GAAG,uBAAH,GAAG,CAAE,WAAW,CAAA,CAAA;IAC3B,CAAC,CAAC,CAAA;AACJ,CAAC;AAED,MAAM,UAAU,UAAU,CAAC,EAAa,EAAE,IAAY;IACpD,MAAM,GAAG,GAAG,EAAE,CAAC,OAAO,CAGpB;;;;;GAKD,CAAC,CAAC,GAAG,EAAE,CAAA;IAER,OAAO,CAAA,GAAG,aAAH,GAAG,uBAAH,GAAG,CAAE,IAAI,MAAK,IAAI,CAAA;AAC3B,CAAC;AAED,MAAM,UAAU,SAAS,CACvB,EAAa,EACb,MAAgB,EAChB,aAA6B;IAE7B,OAAO,EAAE,CAAC,WAAW,CAAC,GAAG,EAAE;QACzB,MAAM,SAAS,GAAa,EAAE,CAAA;QAE9B,KAAK,MAAM,IAAI,IAAI,MAAM,EAAE,CAAC;YAC1B,MAAM,EAAE,OAAO,EAAE,GAAG,EAAE,CAAC,OAAO,CAAmB;;;;OAIhD,CAAC,CAAC,GAAG,CAAC,EAAE,IAAI,EAAE,CAAC,CAAA;YAEhB,IAAI,OAAO,KAAK,CAAC,EAAE,CAAC;gBAClB,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;YACtB,CAAC;iBAAM,CAAC;gBAEN,EAAE,CAAC,OAAO,CAAmB;;;SAG5B,CAAC,CAAC,GAAG,CAAC,EAAE,IAAI,EAAE,CAAC,CAAA;YAClB,CAAC;QACH,CAAC;QAED,MAAM,CAAC,EAAE,EAAE,aAAa,CAAC,CAAA;QAEzB,OAAO,SAAS,CAAA;IAClB,CAAC,CAAC,EAAE,CAAA;AACN,CAAC;AAED,MAAM,UAAU,WAAW,CACzB,EAAa,EACb,IAAY,EACZ,aAA6B;IAE7B,OAAO,EAAE,CAAC,WAAW,CAAC,GAAG,EAAE;QACzB,MAAM,GAAG,GAAG,EAAE,CAAC,OAAO,CAAuB;;;;;KAK5C,CAAC,CAAC,GAAG,EAAE,CAAA;QAER,IAAI,CAAA,GAAG,aAAH,GAAG,uBAAH,GAAG,CAAE,IAAI,MAAK,IAAI,EAAE,CAAC;YACvB,OAAO,KAAK,CAAA;QACd,CAAC;aAAM,CAAC;YACN,EAAE,CAAC,OAAO,CAAmB;;;OAG5B,CAAC,CAAC,GAAG,CAAC,EAAE,IAAI,EAAE,CAAC,CAAA;YAChB,MAAM,CAAC,EAAE,EAAE,aAAa,CAAC,CAAA;YAEzB,OAAO,IAAI,CAAA;QACb,CAAC;IACH,CAAC,CAAC,EAAE,CAAA;AACN,CAAC;AAED,SAAS,MAAM,CAAC,EAAa,EAAE,OAAuB;IACpD,IAAI,QAAQ,CAAC,OAAO,CAAC,SAAS,CAAC,EAAE,CAAC;QAChC,EAAE,CAAC,WAAW,CAAC,GAAG,EAAE;YAClB,MAAM,KAAK,GAAG,WAAW,CAAC,EAAE,CAAC,CAAA;YAE7B,IAAI,KAAK,IAAI,OAAO,CAAC,SAAS,EAAE,CAAC;gBAC/B,EAAE,CAAC,OAAO,CAAoB;;;;SAI7B,CAAC,CAAC,GAAG,CAAC,EAAE,KAAK,EAAE,KAAK,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,CAAA;YAC3C,CAAC;QACH,CAAC,CAAC,EAAE,CAAA;IACN,CAAC;AACH,CAAC;AAED,SAAS,WAAW,CAAC,EAAa;IAChC,MAAM,GAAG,GAAG,EAAE,CAAC,OAAO,CAAwB;;;GAG7C,CAAC,CAAC,GAAG,EAAE,CAAA;IACR,MAAM,CAAC,GAAG,CAAC,CAAA;IAEX,OAAO,GAAG,CAAC,KAAK,CAAA;AAClB,CAAC"}
|
|
1
|
+
{"version":3,"file":"database.js","sourceRoot":"","sources":["../src/database.ts"],"names":[],"mappings":"AAAA,OAAO,QAAmC,MAAM,gBAAgB,CAAA;AAChE,OAAO,EAAE,OAAO,EAAE,MAAM,uCAAuC,CAAA;AAC/D,OAAO,EAAE,sBAAsB,EAAE,iBAAiB,EAAE,MAAM,iBAAiB,CAAA;AAC3E,OAAO,EAAE,GAAG,EAAE,MAAM,eAAe,CAAA;AACnC,OAAO,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,qBAAqB,CAAA;AACtD,OAAO,EAAE,qBAAqB,EAAE,MAAM,kBAAkB,CAAA;AACxD,OAAO,IAAI,MAAM,MAAM,CAAA;AAQvB,MAAM,CAAC,KAAK,UAAU,YAAY,CAAC,QAAQ,GAAG,UAAU;IACtD,MAAM,eAAe,GAAG,MAAM,qBAAqB,CAAC,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAA;IACxE,MAAM,CAAC,eAAe,EAAE,wBAAwB,CAAC,CAAA;IAEjD,MAAM,WAAW,GAAG,IAAI,CAAC,OAAO,CAAC,eAAe,CAAC,CAAA;IACjD,MAAM,cAAc,GAAG,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,YAAY,CAAC,CAAA;IAC3D,MAAM,kBAAkB,GAAG,MAAM,sBAAsB,CAAC,cAAc,CAAC,CAAA;IACvE,MAAM,UAAU,GAAG,MAAM,GAAG,CAAC,kBAAkB,EAAE,iBAAiB,CAAC,CAAA;IAEnE,MAAM,EAAE,GAAG,IAAI,QAAQ,CAAC,QAAQ,CAAC,CAAA;IACjC,OAAO,CAAC,EAAE,EAAE,UAAU,CAAC,CAAA;IAEvB,OAAO,EAAE,CAAA;AACX,CAAC;AAED,MAAM,UAAU,SAAS,CAAC,EAAa,EAAE,MAAgB;IACvD,OAAO,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE;QACvB,MAAM,GAAG,GAAG,EAAE,CAAC,OAAO,CAGpB;;;;;;KAMD,CAAC,CAAC,GAAG,CAAC,EAAE,IAAI,EAAE,CAAC,CAAA;QAEhB,OAAO,CAAC,CAAC,CAAA,GAAG,aAAH,GAAG,uBAAH,GAAG,CAAE,WAAW,CAAA,CAAA;IAC3B,CAAC,CAAC,CAAA;AACJ,CAAC;AAED,MAAM,UAAU,UAAU,CAAC,EAAa,EAAE,IAAY;IACpD,MAAM,GAAG,GAAG,EAAE,CAAC,OAAO,CAGpB;;;;;GAKD,CAAC,CAAC,GAAG,EAAE,CAAA;IAER,OAAO,CAAA,GAAG,aAAH,GAAG,uBAAH,GAAG,CAAE,IAAI,MAAK,IAAI,CAAA;AAC3B,CAAC;AAED,MAAM,UAAU,SAAS,CACvB,EAAa,EACb,MAAgB,EAChB,aAA6B;IAE7B,OAAO,EAAE,CAAC,WAAW,CAAC,GAAG,EAAE;QACzB,MAAM,SAAS,GAAa,EAAE,CAAA;QAE9B,KAAK,MAAM,IAAI,IAAI,MAAM,EAAE,CAAC;YAC1B,MAAM,EAAE,OAAO,EAAE,GAAG,EAAE,CAAC,OAAO,CAAmB;;;;OAIhD,CAAC,CAAC,GAAG,CAAC,EAAE,IAAI,EAAE,CAAC,CAAA;YAEhB,IAAI,OAAO,KAAK,CAAC,EAAE,CAAC;gBAClB,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;YACtB,CAAC;iBAAM,CAAC;gBAEN,EAAE,CAAC,OAAO,CAAmB;;;SAG5B,CAAC,CAAC,GAAG,CAAC,EAAE,IAAI,EAAE,CAAC,CAAA;YAClB,CAAC;QACH,CAAC;QAED,MAAM,CAAC,EAAE,EAAE,aAAa,CAAC,CAAA;QAEzB,OAAO,SAAS,CAAA;IAClB,CAAC,CAAC,EAAE,CAAA;AACN,CAAC;AAED,MAAM,UAAU,WAAW,CACzB,EAAa,EACb,IAAY,EACZ,aAA6B;IAE7B,OAAO,EAAE,CAAC,WAAW,CAAC,GAAG,EAAE;QACzB,MAAM,GAAG,GAAG,EAAE,CAAC,OAAO,CAAuB;;;;;KAK5C,CAAC,CAAC,GAAG,EAAE,CAAA;QAER,IAAI,CAAA,GAAG,aAAH,GAAG,uBAAH,GAAG,CAAE,IAAI,MAAK,IAAI,EAAE,CAAC;YACvB,OAAO,KAAK,CAAA;QACd,CAAC;aAAM,CAAC;YACN,EAAE,CAAC,OAAO,CAAmB;;;OAG5B,CAAC,CAAC,GAAG,CAAC,EAAE,IAAI,EAAE,CAAC,CAAA;YAChB,MAAM,CAAC,EAAE,EAAE,aAAa,CAAC,CAAA;YAEzB,OAAO,IAAI,CAAA;QACb,CAAC;IACH,CAAC,CAAC,EAAE,CAAA;AACN,CAAC;AAED,SAAS,MAAM,CAAC,EAAa,EAAE,OAAuB;IACpD,IAAI,QAAQ,CAAC,OAAO,CAAC,SAAS,CAAC,EAAE,CAAC;QAChC,EAAE,CAAC,WAAW,CAAC,GAAG,EAAE;YAClB,MAAM,KAAK,GAAG,WAAW,CAAC,EAAE,CAAC,CAAA;YAE7B,IAAI,KAAK,IAAI,OAAO,CAAC,SAAS,EAAE,CAAC;gBAC/B,EAAE,CAAC,OAAO,CAAoB;;;;SAI7B,CAAC,CAAC,GAAG,CAAC,EAAE,KAAK,EAAE,KAAK,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,CAAA;YAC3C,CAAC;QACH,CAAC,CAAC,EAAE,CAAA;IACN,CAAC;AACH,CAAC;AAED,MAAM,UAAU,OAAO,CAAC,EAAa;IACnC,MAAM,GAAG,GAAG,EAAE,CAAC,OAAO,CAA0B;;;;;;GAM/C,CAAC,CAAC,GAAG,EAAE,CAAA;IAER,OAAO,CAAC,CAAA,GAAG,aAAH,GAAG,uBAAH,GAAG,CAAE,OAAO,CAAA,CAAA;AACtB,CAAC;AAED,SAAS,WAAW,CAAC,EAAa;IAChC,MAAM,GAAG,GAAG,EAAE,CAAC,OAAO,CAAwB;;;GAG7C,CAAC,CAAC,GAAG,EAAE,CAAA;IACR,MAAM,CAAC,GAAG,CAAC,CAAA;IAEX,OAAO,GAAG,CAAC,KAAK,CAAA;AAClB,CAAC"}
|
package/lib/deduplicator.d.ts
CHANGED
|
@@ -14,6 +14,7 @@ export declare class Deduplicator<T> {
|
|
|
14
14
|
private constructor();
|
|
15
15
|
static create<T>({ filename, hash, shrink: { threshold, target } }?: IOptions<T>): Promise<Deduplicator<T>>;
|
|
16
16
|
close(): void;
|
|
17
|
+
isEmpty(): boolean;
|
|
17
18
|
add(values: T[]): Promise<void>;
|
|
18
19
|
diff(values: T[]): Promise<T[]>;
|
|
19
20
|
diffLast(value: T): Promise<boolean>;
|
package/lib/deduplicator.js
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { toArray, uniq, zip } from 'iterable-operator';
|
|
2
2
|
import { isntEmptyArray, assert, isPositiveInfinity, isFinite } from '@blackglory/prelude';
|
|
3
3
|
import { map } from 'extra-promise';
|
|
4
|
-
import { openDatabase, hasHashes, isLastHash, addHashes, addLastHash } from './database.js';
|
|
4
|
+
import { openDatabase, hasHashes, isLastHash, addHashes, addLastHash, isEmpty } from './database.js';
|
|
5
5
|
import { stringify } from 'extra-json-stable-stringify';
|
|
6
6
|
import { sha256 } from 'extra-compatible';
|
|
7
7
|
export class Deduplicator {
|
|
@@ -23,6 +23,9 @@ export class Deduplicator {
|
|
|
23
23
|
close() {
|
|
24
24
|
this.db.close();
|
|
25
25
|
}
|
|
26
|
+
isEmpty() {
|
|
27
|
+
return isEmpty(this.db);
|
|
28
|
+
}
|
|
26
29
|
async add(values) {
|
|
27
30
|
const hashes = await map(values, value => this.hash(value));
|
|
28
31
|
addHashes(this.db, hashes, this.shrink);
|
package/lib/deduplicator.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"deduplicator.js","sourceRoot":"","sources":["../src/deduplicator.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,GAAG,EAAE,MAAM,mBAAmB,CAAA;AACtD,OAAO,EAAE,cAAc,EAAE,MAAM,EAAkB,kBAAkB,EAAE,QAAQ,EAAE,MAAM,qBAAqB,CAAA;AAC1G,OAAO,EAAE,GAAG,EAAE,MAAM,eAAe,CAAA;AACnC,OAAO,EAAE,YAAY,EAAE,SAAS,EAAE,UAAU,EAAE,SAAS,EAAE,WAAW,EAAE,MAAM,eAAe,CAAA;
|
|
1
|
+
{"version":3,"file":"deduplicator.js","sourceRoot":"","sources":["../src/deduplicator.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,GAAG,EAAE,MAAM,mBAAmB,CAAA;AACtD,OAAO,EAAE,cAAc,EAAE,MAAM,EAAkB,kBAAkB,EAAE,QAAQ,EAAE,MAAM,qBAAqB,CAAA;AAC1G,OAAO,EAAE,GAAG,EAAE,MAAM,eAAe,CAAA;AACnC,OAAO,EAAE,YAAY,EAAE,SAAS,EAAE,UAAU,EAAE,SAAS,EAAE,WAAW,EAAE,OAAO,EAAE,MAAM,eAAe,CAAA;AAEpG,OAAO,EAAE,SAAS,EAAE,MAAM,6BAA6B,CAAA;AACvD,OAAO,EAAE,MAAM,EAAE,MAAM,kBAAkB,CAAA;AAazC,MAAM,OAAO,YAAY;IACvB,YACU,EAAY,EACZ,IAAqC,EACrC,MAGP;QALO,OAAE,GAAF,EAAE,CAAU;QACZ,SAAI,GAAJ,IAAI,CAAiC;QACrC,WAAM,GAAN,MAAM,CAGb;IACA,CAAC;IAEJ,MAAM,CAAC,KAAK,CAAC,MAAM,CACjB,EACE,QAAQ,EACR,IAAI,GAAG,KAAK,CAAC,EAAE,CAAC,MAAM,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,EACxC,MAAM,EAAE,EACN,SAAS,GAAG,QAAQ,EACpB,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,SAAS,GAAG,CAAC,CAAC,EACnC,GAAG,EAAE,KACS,EAAE;QAEnB,MAAM,CACJ,kBAAkB,CAAC,SAAS,CAAC,IAAI,MAAM,CAAC,SAAS,CAAC,SAAS,CAAC,EAC5D,qDAAqD,CACtD,CAAA;QACD,MAAM,CACJ,kBAAkB,CAAC,MAAM,CAAC,IAAI,MAAM,CAAC,SAAS,CAAC,MAAM,CAAC,EACtD,kDAAkD,CACnD,CAAA;QACD,MAAM,CAAC,SAAS,GAAG,CAAC,EAAE,6CAA6C,CAAC,CAAA;QACpE,IAAI,QAAQ,CAAC,SAAS,CAAC,EAAE,CAAC;YACxB,MAAM,CAAC,MAAM,GAAG,SAAS,EAAE,sDAAsD,CAAC,CAAA;QACpF,CAAC;QAED,MAAM,EAAE,GAAG,MAAM,YAAY,CAAC,QAAQ,CAAC,CAAA;QAEvC,OAAO,IAAI,YAAY,CAAI,EAAE,EAAE,IAAI,EAAE,EAAE,SAAS,EAAE,MAAM,EAAE,CAAC,CAAA;IAC7D,CAAC;IAED,KAAK;QACH,IAAI,CAAC,EAAE,CAAC,KAAK,EAAE,CAAA;IACjB,CAAC;IAED,OAAO;QACL,OAAO,OAAO,CAAC,IAAI,CAAC,EAAE,CAAC,CAAA;IACzB,CAAC;IAED,KAAK,CAAC,GAAG,CAAC,MAAW;QACnB,MAAM,MAAM,GAAG,MAAM,GAAG,CAAC,MAAM,EAAE,KAAK,CAAC,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAA;QAE3D,SAAS,CAAC,IAAI,CAAC,EAAE,EAAE,MAAM,EAAE,IAAI,CAAC,MAAM,CAAC,CAAA;IACzC,CAAC;IAED,KAAK,CAAC,IAAI,CAAC,MAAW;QACpB,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAA;QAE9B,MAAM,MAAM,GAAG,MAAM,GAAG,CAAC,MAAM,EAAE,KAAK,CAAC,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAA;QAC3D,MAAM,OAAO,GAAG,SAAS,CAAC,IAAI,CAAC,EAAE,EAAE,MAAM,CAAC,CAAA;QAC1C,MAAM,SAAS,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAA;QAE9D,OAAO,SAAS,CAAA;IAClB,CAAC;IAED,KAAK,CAAC,QAAQ,CAAC,KAAQ;QACrB,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;QAEnC,OAAO,CAAC,UAAU,CAAC,IAAI,CAAC,EAAE,EAAE,IAAI,CAAC,CAAA;IACnC,CAAC;IAED,KAAK,CAAC,UAAU,CAAC,MAAW;QAC1B,MAAM,MAAM,GAAG,MAAM,GAAG,CAAC,MAAM,EAAE,IAAI,CAAC,IAAI,CAAC,CAAA;QAE3C,MAAM,SAAS,GAAG,SAAS,CAAC,IAAI,CAAC,EAAE,EAAE,MAAM,EAAE,IAAI,CAAC,MAAM,CAAC,CAAA;QAEzD,IAAI,cAAc,CAAC,SAAS,CAAC,EAAE,CAAC;YAC9B,MAAM,WAAW,GAAG,MAAM,CAAC,WAAW,CAAC,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAAA;YAE3D,OAAO,SAAS,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC,CAAA;QACjD,CAAC;aAAM,CAAC;YACN,OAAO,EAAE,CAAA;QACX,CAAC;IACH,CAAC;IAED,KAAK,CAAC,cAAc,CAAC,KAAQ;QAC3B,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;QAEnC,OAAO,WAAW,CAAC,IAAI,CAAC,EAAE,EAAE,IAAI,EAAE,IAAI,CAAC,MAAM,CAAC,CAAA;IAChD,CAAC;CACF"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "extra-deduplicator",
|
|
3
|
-
"version": "0.3.
|
|
3
|
+
"version": "0.3.1",
|
|
4
4
|
"description": "",
|
|
5
5
|
"keywords": [],
|
|
6
6
|
"files": [
|
|
@@ -59,5 +59,8 @@
|
|
|
59
59
|
"extra-promise": "^7.1.1",
|
|
60
60
|
"iterable-operator": "^6.0.0",
|
|
61
61
|
"migration-files": "^0.4.3"
|
|
62
|
+
},
|
|
63
|
+
"allowScripts": {
|
|
64
|
+
"husky": true
|
|
62
65
|
}
|
|
63
66
|
}
|
package/src/database.ts
CHANGED
|
@@ -34,10 +34,10 @@ export function hasHashes(db: IDatabase, hashes: string[]): boolean[] {
|
|
|
34
34
|
, { hash_exists: number }
|
|
35
35
|
>(`
|
|
36
36
|
SELECT EXISTS(
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
37
|
+
SELECT *
|
|
38
|
+
FROM hash
|
|
39
|
+
WHERE hash = $hash
|
|
40
|
+
) AS hash_exists
|
|
41
41
|
`).get({ hash })
|
|
42
42
|
|
|
43
43
|
return !!row?.hash_exists
|
|
@@ -133,6 +133,18 @@ function shrink(db: IDatabase, options: IShrinkOptions): void {
|
|
|
133
133
|
}
|
|
134
134
|
}
|
|
135
135
|
|
|
136
|
+
export function isEmpty(db: IDatabase): boolean {
|
|
137
|
+
const row = db.prepare<[], { matched: number }>(`
|
|
138
|
+
SELECT EXISTS (
|
|
139
|
+
SELECT 1
|
|
140
|
+
FROM hash
|
|
141
|
+
LIMIT 1
|
|
142
|
+
) AS matched
|
|
143
|
+
`).get()
|
|
144
|
+
|
|
145
|
+
return !row?.matched
|
|
146
|
+
}
|
|
147
|
+
|
|
136
148
|
function countHashes(db: IDatabase): number {
|
|
137
149
|
const row = db.prepare<[], { count: number }>(`
|
|
138
150
|
SELECT COUNT(*) as count
|
package/src/deduplicator.ts
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { toArray, uniq, zip } from 'iterable-operator'
|
|
2
2
|
import { isntEmptyArray, assert, type Awaitable, isPositiveInfinity, isFinite } from '@blackglory/prelude'
|
|
3
3
|
import { map } from 'extra-promise'
|
|
4
|
-
import { openDatabase, hasHashes, isLastHash, addHashes, addLastHash } from './database.js'
|
|
4
|
+
import { openDatabase, hasHashes, isLastHash, addHashes, addLastHash, isEmpty } from './database.js'
|
|
5
5
|
import type { Database } from 'better-sqlite3'
|
|
6
6
|
import { stringify } from 'extra-json-stable-stringify'
|
|
7
7
|
import { sha256 } from 'extra-compatible'
|
|
@@ -59,6 +59,10 @@ export class Deduplicator<T> {
|
|
|
59
59
|
this.db.close()
|
|
60
60
|
}
|
|
61
61
|
|
|
62
|
+
isEmpty(): boolean {
|
|
63
|
+
return isEmpty(this.db)
|
|
64
|
+
}
|
|
65
|
+
|
|
62
66
|
async add(values: T[]): Promise<void> {
|
|
63
67
|
const hashes = await map(values, value => this.hash(value))
|
|
64
68
|
|