extra-deduplicator 0.3.0 → 0.3.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md CHANGED
@@ -32,6 +32,8 @@ interface IOptions<T> {
32
32
  class Deduplicator<T> {
33
33
  static create(options?: IOptions): Promise<Deduplicator<T>>
34
34
 
35
+ isEmpty(): boolean
36
+
35
37
  add(values: T[]): Promise<void>
36
38
 
37
39
  diff(values: T[]): Promise<T[]>
package/lib/database.d.ts CHANGED
@@ -8,4 +8,5 @@ export declare function hasHashes(db: IDatabase, hashes: string[]): boolean[];
8
8
  export declare function isLastHash(db: IDatabase, hash: string): boolean;
9
9
  export declare function addHashes(db: IDatabase, hashes: string[], shrinkOptions: IShrinkOptions): string[];
10
10
  export declare function addLastHash(db: IDatabase, hash: string, shrinkOptions: IShrinkOptions): boolean;
11
+ export declare function isEmpty(db: IDatabase): boolean;
11
12
  export {};
package/lib/database.js CHANGED
@@ -20,10 +20,10 @@ export function hasHashes(db, hashes) {
20
20
  return hashes.map(hash => {
21
21
  const row = db.prepare(`
22
22
  SELECT EXISTS(
23
- SELECT *
24
- FROM hash
25
- WHERE hash = $hash
26
- ) AS hash_exists
23
+ SELECT *
24
+ FROM hash
25
+ WHERE hash = $hash
26
+ ) AS hash_exists
27
27
  `).get({ hash });
28
28
  return !!(row === null || row === void 0 ? void 0 : row.hash_exists);
29
29
  });
@@ -95,6 +95,16 @@ function shrink(db, options) {
95
95
  })();
96
96
  }
97
97
  }
98
+ export function isEmpty(db) {
99
+ const row = db.prepare(`
100
+ SELECT EXISTS (
101
+ SELECT 1
102
+ FROM hash
103
+ LIMIT 1
104
+ ) AS matched
105
+ `).get();
106
+ return !(row === null || row === void 0 ? void 0 : row.matched);
107
+ }
98
108
  function countHashes(db) {
99
109
  const row = db.prepare(`
100
110
  SELECT COUNT(*) as count
@@ -1 +1 @@
1
- {"version":3,"file":"database.js","sourceRoot":"","sources":["../src/database.ts"],"names":[],"mappings":"AAAA,OAAO,QAAmC,MAAM,gBAAgB,CAAA;AAChE,OAAO,EAAE,OAAO,EAAE,MAAM,uCAAuC,CAAA;AAC/D,OAAO,EAAE,sBAAsB,EAAE,iBAAiB,EAAE,MAAM,iBAAiB,CAAA;AAC3E,OAAO,EAAE,GAAG,EAAE,MAAM,eAAe,CAAA;AACnC,OAAO,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,qBAAqB,CAAA;AACtD,OAAO,EAAE,qBAAqB,EAAE,MAAM,kBAAkB,CAAA;AACxD,OAAO,IAAI,MAAM,MAAM,CAAA;AAQvB,MAAM,CAAC,KAAK,UAAU,YAAY,CAAC,QAAQ,GAAG,UAAU;IACtD,MAAM,eAAe,GAAG,MAAM,qBAAqB,CAAC,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAA;IACxE,MAAM,CAAC,eAAe,EAAE,wBAAwB,CAAC,CAAA;IAEjD,MAAM,WAAW,GAAG,IAAI,CAAC,OAAO,CAAC,eAAe,CAAC,CAAA;IACjD,MAAM,cAAc,GAAG,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,YAAY,CAAC,CAAA;IAC3D,MAAM,kBAAkB,GAAG,MAAM,sBAAsB,CAAC,cAAc,CAAC,CAAA;IACvE,MAAM,UAAU,GAAG,MAAM,GAAG,CAAC,kBAAkB,EAAE,iBAAiB,CAAC,CAAA;IAEnE,MAAM,EAAE,GAAG,IAAI,QAAQ,CAAC,QAAQ,CAAC,CAAA;IACjC,OAAO,CAAC,EAAE,EAAE,UAAU,CAAC,CAAA;IAEvB,OAAO,EAAE,CAAA;AACX,CAAC;AAED,MAAM,UAAU,SAAS,CAAC,EAAa,EAAE,MAAgB;IACvD,OAAO,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE;QACvB,MAAM,GAAG,GAAG,EAAE,CAAC,OAAO,CAGpB;;;;;;KAMD,CAAC,CAAC,GAAG,CAAC,EAAE,IAAI,EAAE,CAAC,CAAA;QAEhB,OAAO,CAAC,CAAC,CAAA,GAAG,aAAH,GAAG,uBAAH,GAAG,CAAE,WAAW,CAAA,CAAA;IAC3B,CAAC,CAAC,CAAA;AACJ,CAAC;AAED,MAAM,UAAU,UAAU,CAAC,EAAa,EAAE,IAAY;IACpD,MAAM,GAAG,GAAG,EAAE,CAAC,OAAO,CAGpB;;;;;GAKD,CAAC,CAAC,GAAG,EAAE,CAAA;IAER,OAAO,CAAA,GAAG,aAAH,GAAG,uBAAH,GAAG,CAAE,IAAI,MAAK,IAAI,CAAA;AAC3B,CAAC;AAED,MAAM,UAAU,SAAS,CACvB,EAAa,EACb,MAAgB,EAChB,aAA6B;IAE7B,OAAO,EAAE,CAAC,WAAW,CAAC,GAAG,EAAE;QACzB,MAAM,SAAS,GAAa,EAAE,CAAA;QAE9B,KAAK,MAAM,IAAI,IAAI,MAAM,EAAE,CAAC;YAC1B,MAAM,EAAE,OAAO,EAAE,GAAG,EAAE,CAAC,OAAO,CAAmB;;;;OAIhD,CAAC,CAAC,GAAG,CAAC,EAAE,IAAI,EAAE,CAAC,CAAA;YAEhB,IAAI,OAAO,KAAK,CAAC,EAAE,CAAC;gBAClB,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;YACtB,CAAC;iBAAM,CAAC;gBAEN,EAAE,CAAC,OAAO,CAAmB;;;SAG5B,CAAC,CAAC,GAAG,CAAC,EAAE,IAAI,EAAE,CAAC,CAAA;YAClB,CAAC;QACH,CAAC;QAED,MAAM,CAAC,EAAE,EAAE,aAAa,CAAC,CAAA;QAEzB,OAAO,SAAS,CAAA;IAClB,CAAC,CAAC,EAAE,CAAA;AACN,CAAC;AAED,MAAM,UAAU,WAAW,CACzB,EAAa,EACb,IAAY,EACZ,aAA6B;IAE7B,OAAO,EAAE,CAAC,WAAW,CAAC,GAAG,EAAE;QACzB,MAAM,GAAG,GAAG,EAAE,CAAC,OAAO,CAAuB;;;;;KAK5C,CAAC,CAAC,GAAG,EAAE,CAAA;QAER,IAAI,CAAA,GAAG,aAAH,GAAG,uBAAH,GAAG,CAAE,IAAI,MAAK,IAAI,EAAE,CAAC;YACvB,OAAO,KAAK,CAAA;QACd,CAAC;aAAM,CAAC;YACN,EAAE,CAAC,OAAO,CAAmB;;;OAG5B,CAAC,CAAC,GAAG,CAAC,EAAE,IAAI,EAAE,CAAC,CAAA;YAChB,MAAM,CAAC,EAAE,EAAE,aAAa,CAAC,CAAA;YAEzB,OAAO,IAAI,CAAA;QACb,CAAC;IACH,CAAC,CAAC,EAAE,CAAA;AACN,CAAC;AAED,SAAS,MAAM,CAAC,EAAa,EAAE,OAAuB;IACpD,IAAI,QAAQ,CAAC,OAAO,CAAC,SAAS,CAAC,EAAE,CAAC;QAChC,EAAE,CAAC,WAAW,CAAC,GAAG,EAAE;YAClB,MAAM,KAAK,GAAG,WAAW,CAAC,EAAE,CAAC,CAAA;YAE7B,IAAI,KAAK,IAAI,OAAO,CAAC,SAAS,EAAE,CAAC;gBAC/B,EAAE,CAAC,OAAO,CAAoB;;;;SAI7B,CAAC,CAAC,GAAG,CAAC,EAAE,KAAK,EAAE,KAAK,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,CAAA;YAC3C,CAAC;QACH,CAAC,CAAC,EAAE,CAAA;IACN,CAAC;AACH,CAAC;AAED,SAAS,WAAW,CAAC,EAAa;IAChC,MAAM,GAAG,GAAG,EAAE,CAAC,OAAO,CAAwB;;;GAG7C,CAAC,CAAC,GAAG,EAAE,CAAA;IACR,MAAM,CAAC,GAAG,CAAC,CAAA;IAEX,OAAO,GAAG,CAAC,KAAK,CAAA;AAClB,CAAC"}
1
+ {"version":3,"file":"database.js","sourceRoot":"","sources":["../src/database.ts"],"names":[],"mappings":"AAAA,OAAO,QAAmC,MAAM,gBAAgB,CAAA;AAChE,OAAO,EAAE,OAAO,EAAE,MAAM,uCAAuC,CAAA;AAC/D,OAAO,EAAE,sBAAsB,EAAE,iBAAiB,EAAE,MAAM,iBAAiB,CAAA;AAC3E,OAAO,EAAE,GAAG,EAAE,MAAM,eAAe,CAAA;AACnC,OAAO,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,qBAAqB,CAAA;AACtD,OAAO,EAAE,qBAAqB,EAAE,MAAM,kBAAkB,CAAA;AACxD,OAAO,IAAI,MAAM,MAAM,CAAA;AAQvB,MAAM,CAAC,KAAK,UAAU,YAAY,CAAC,QAAQ,GAAG,UAAU;IACtD,MAAM,eAAe,GAAG,MAAM,qBAAqB,CAAC,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAA;IACxE,MAAM,CAAC,eAAe,EAAE,wBAAwB,CAAC,CAAA;IAEjD,MAAM,WAAW,GAAG,IAAI,CAAC,OAAO,CAAC,eAAe,CAAC,CAAA;IACjD,MAAM,cAAc,GAAG,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,YAAY,CAAC,CAAA;IAC3D,MAAM,kBAAkB,GAAG,MAAM,sBAAsB,CAAC,cAAc,CAAC,CAAA;IACvE,MAAM,UAAU,GAAG,MAAM,GAAG,CAAC,kBAAkB,EAAE,iBAAiB,CAAC,CAAA;IAEnE,MAAM,EAAE,GAAG,IAAI,QAAQ,CAAC,QAAQ,CAAC,CAAA;IACjC,OAAO,CAAC,EAAE,EAAE,UAAU,CAAC,CAAA;IAEvB,OAAO,EAAE,CAAA;AACX,CAAC;AAED,MAAM,UAAU,SAAS,CAAC,EAAa,EAAE,MAAgB;IACvD,OAAO,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE;QACvB,MAAM,GAAG,GAAG,EAAE,CAAC,OAAO,CAGpB;;;;;;KAMD,CAAC,CAAC,GAAG,CAAC,EAAE,IAAI,EAAE,CAAC,CAAA;QAEhB,OAAO,CAAC,CAAC,CAAA,GAAG,aAAH,GAAG,uBAAH,GAAG,CAAE,WAAW,CAAA,CAAA;IAC3B,CAAC,CAAC,CAAA;AACJ,CAAC;AAED,MAAM,UAAU,UAAU,CAAC,EAAa,EAAE,IAAY;IACpD,MAAM,GAAG,GAAG,EAAE,CAAC,OAAO,CAGpB;;;;;GAKD,CAAC,CAAC,GAAG,EAAE,CAAA;IAER,OAAO,CAAA,GAAG,aAAH,GAAG,uBAAH,GAAG,CAAE,IAAI,MAAK,IAAI,CAAA;AAC3B,CAAC;AAED,MAAM,UAAU,SAAS,CACvB,EAAa,EACb,MAAgB,EAChB,aAA6B;IAE7B,OAAO,EAAE,CAAC,WAAW,CAAC,GAAG,EAAE;QACzB,MAAM,SAAS,GAAa,EAAE,CAAA;QAE9B,KAAK,MAAM,IAAI,IAAI,MAAM,EAAE,CAAC;YAC1B,MAAM,EAAE,OAAO,EAAE,GAAG,EAAE,CAAC,OAAO,CAAmB;;;;OAIhD,CAAC,CAAC,GAAG,CAAC,EAAE,IAAI,EAAE,CAAC,CAAA;YAEhB,IAAI,OAAO,KAAK,CAAC,EAAE,CAAC;gBAClB,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;YACtB,CAAC;iBAAM,CAAC;gBAEN,EAAE,CAAC,OAAO,CAAmB;;;SAG5B,CAAC,CAAC,GAAG,CAAC,EAAE,IAAI,EAAE,CAAC,CAAA;YAClB,CAAC;QACH,CAAC;QAED,MAAM,CAAC,EAAE,EAAE,aAAa,CAAC,CAAA;QAEzB,OAAO,SAAS,CAAA;IAClB,CAAC,CAAC,EAAE,CAAA;AACN,CAAC;AAED,MAAM,UAAU,WAAW,CACzB,EAAa,EACb,IAAY,EACZ,aAA6B;IAE7B,OAAO,EAAE,CAAC,WAAW,CAAC,GAAG,EAAE;QACzB,MAAM,GAAG,GAAG,EAAE,CAAC,OAAO,CAAuB;;;;;KAK5C,CAAC,CAAC,GAAG,EAAE,CAAA;QAER,IAAI,CAAA,GAAG,aAAH,GAAG,uBAAH,GAAG,CAAE,IAAI,MAAK,IAAI,EAAE,CAAC;YACvB,OAAO,KAAK,CAAA;QACd,CAAC;aAAM,CAAC;YACN,EAAE,CAAC,OAAO,CAAmB;;;OAG5B,CAAC,CAAC,GAAG,CAAC,EAAE,IAAI,EAAE,CAAC,CAAA;YAChB,MAAM,CAAC,EAAE,EAAE,aAAa,CAAC,CAAA;YAEzB,OAAO,IAAI,CAAA;QACb,CAAC;IACH,CAAC,CAAC,EAAE,CAAA;AACN,CAAC;AAED,SAAS,MAAM,CAAC,EAAa,EAAE,OAAuB;IACpD,IAAI,QAAQ,CAAC,OAAO,CAAC,SAAS,CAAC,EAAE,CAAC;QAChC,EAAE,CAAC,WAAW,CAAC,GAAG,EAAE;YAClB,MAAM,KAAK,GAAG,WAAW,CAAC,EAAE,CAAC,CAAA;YAE7B,IAAI,KAAK,IAAI,OAAO,CAAC,SAAS,EAAE,CAAC;gBAC/B,EAAE,CAAC,OAAO,CAAoB;;;;SAI7B,CAAC,CAAC,GAAG,CAAC,EAAE,KAAK,EAAE,KAAK,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,CAAA;YAC3C,CAAC;QACH,CAAC,CAAC,EAAE,CAAA;IACN,CAAC;AACH,CAAC;AAED,MAAM,UAAU,OAAO,CAAC,EAAa;IACnC,MAAM,GAAG,GAAG,EAAE,CAAC,OAAO,CAA0B;;;;;;GAM/C,CAAC,CAAC,GAAG,EAAE,CAAA;IAER,OAAO,CAAC,CAAA,GAAG,aAAH,GAAG,uBAAH,GAAG,CAAE,OAAO,CAAA,CAAA;AACtB,CAAC;AAED,SAAS,WAAW,CAAC,EAAa;IAChC,MAAM,GAAG,GAAG,EAAE,CAAC,OAAO,CAAwB;;;GAG7C,CAAC,CAAC,GAAG,EAAE,CAAA;IACR,MAAM,CAAC,GAAG,CAAC,CAAA;IAEX,OAAO,GAAG,CAAC,KAAK,CAAA;AAClB,CAAC"}
@@ -14,6 +14,7 @@ export declare class Deduplicator<T> {
14
14
  private constructor();
15
15
  static create<T>({ filename, hash, shrink: { threshold, target } }?: IOptions<T>): Promise<Deduplicator<T>>;
16
16
  close(): void;
17
+ isEmpty(): boolean;
17
18
  add(values: T[]): Promise<void>;
18
19
  diff(values: T[]): Promise<T[]>;
19
20
  diffLast(value: T): Promise<boolean>;
@@ -1,7 +1,7 @@
1
1
  import { toArray, uniq, zip } from 'iterable-operator';
2
2
  import { isntEmptyArray, assert, isPositiveInfinity, isFinite } from '@blackglory/prelude';
3
3
  import { map } from 'extra-promise';
4
- import { openDatabase, hasHashes, isLastHash, addHashes, addLastHash } from './database.js';
4
+ import { openDatabase, hasHashes, isLastHash, addHashes, addLastHash, isEmpty } from './database.js';
5
5
  import { stringify } from 'extra-json-stable-stringify';
6
6
  import { sha256 } from 'extra-compatible';
7
7
  export class Deduplicator {
@@ -23,6 +23,9 @@ export class Deduplicator {
23
23
  close() {
24
24
  this.db.close();
25
25
  }
26
+ isEmpty() {
27
+ return isEmpty(this.db);
28
+ }
26
29
  async add(values) {
27
30
  const hashes = await map(values, value => this.hash(value));
28
31
  addHashes(this.db, hashes, this.shrink);
@@ -1 +1 @@
1
- {"version":3,"file":"deduplicator.js","sourceRoot":"","sources":["../src/deduplicator.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,GAAG,EAAE,MAAM,mBAAmB,CAAA;AACtD,OAAO,EAAE,cAAc,EAAE,MAAM,EAAkB,kBAAkB,EAAE,QAAQ,EAAE,MAAM,qBAAqB,CAAA;AAC1G,OAAO,EAAE,GAAG,EAAE,MAAM,eAAe,CAAA;AACnC,OAAO,EAAE,YAAY,EAAE,SAAS,EAAE,UAAU,EAAE,SAAS,EAAE,WAAW,EAAE,MAAM,eAAe,CAAA;AAE3F,OAAO,EAAE,SAAS,EAAE,MAAM,6BAA6B,CAAA;AACvD,OAAO,EAAE,MAAM,EAAE,MAAM,kBAAkB,CAAA;AAazC,MAAM,OAAO,YAAY;IACvB,YACU,EAAY,EACZ,IAAqC,EACrC,MAGP;QALO,OAAE,GAAF,EAAE,CAAU;QACZ,SAAI,GAAJ,IAAI,CAAiC;QACrC,WAAM,GAAN,MAAM,CAGb;IACA,CAAC;IAEJ,MAAM,CAAC,KAAK,CAAC,MAAM,CACjB,EACE,QAAQ,EACR,IAAI,GAAG,KAAK,CAAC,EAAE,CAAC,MAAM,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,EACxC,MAAM,EAAE,EACN,SAAS,GAAG,QAAQ,EACpB,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,SAAS,GAAG,CAAC,CAAC,EACnC,GAAG,EAAE,KACS,EAAE;QAEnB,MAAM,CACJ,kBAAkB,CAAC,SAAS,CAAC,IAAI,MAAM,CAAC,SAAS,CAAC,SAAS,CAAC,EAC5D,qDAAqD,CACtD,CAAA;QACD,MAAM,CACJ,kBAAkB,CAAC,MAAM,CAAC,IAAI,MAAM,CAAC,SAAS,CAAC,MAAM,CAAC,EACtD,kDAAkD,CACnD,CAAA;QACD,MAAM,CAAC,SAAS,GAAG,CAAC,EAAE,6CAA6C,CAAC,CAAA;QACpE,IAAI,QAAQ,CAAC,SAAS,CAAC,EAAE,CAAC;YACxB,MAAM,CAAC,MAAM,GAAG,SAAS,EAAE,sDAAsD,CAAC,CAAA;QACpF,CAAC;QAED,MAAM,EAAE,GAAG,MAAM,YAAY,CAAC,QAAQ,CAAC,CAAA;QAEvC,OAAO,IAAI,YAAY,CAAI,EAAE,EAAE,IAAI,EAAE,EAAE,SAAS,EAAE,MAAM,EAAE,CAAC,CAAA;IAC7D,CAAC;IAED,KAAK;QACH,IAAI,CAAC,EAAE,CAAC,KAAK,EAAE,CAAA;IACjB,CAAC;IAED,KAAK,CAAC,GAAG,CAAC,MAAW;QACnB,MAAM,MAAM,GAAG,MAAM,GAAG,CAAC,MAAM,EAAE,KAAK,CAAC,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAA;QAE3D,SAAS,CAAC,IAAI,CAAC,EAAE,EAAE,MAAM,EAAE,IAAI,CAAC,MAAM,CAAC,CAAA;IACzC,CAAC;IAED,KAAK,CAAC,IAAI,CAAC,MAAW;QACpB,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAA;QAE9B,MAAM,MAAM,GAAG,MAAM,GAAG,CAAC,MAAM,EAAE,KAAK,CAAC,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAA;QAC3D,MAAM,OAAO,GAAG,SAAS,CAAC,IAAI,CAAC,EAAE,EAAE,MAAM,CAAC,CAAA;QAC1C,MAAM,SAAS,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAA;QAE9D,OAAO,SAAS,CAAA;IAClB,CAAC;IAED,KAAK,CAAC,QAAQ,CAAC,KAAQ;QACrB,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;QAEnC,OAAO,CAAC,UAAU,CAAC,IAAI,CAAC,EAAE,EAAE,IAAI,CAAC,CAAA;IACnC,CAAC;IAED,KAAK,CAAC,UAAU,CAAC,MAAW;QAC1B,MAAM,MAAM,GAAG,MAAM,GAAG,CAAC,MAAM,EAAE,IAAI,CAAC,IAAI,CAAC,CAAA;QAE3C,MAAM,SAAS,GAAG,SAAS,CAAC,IAAI,CAAC,EAAE,EAAE,MAAM,EAAE,IAAI,CAAC,MAAM,CAAC,CAAA;QAEzD,IAAI,cAAc,CAAC,SAAS,CAAC,EAAE,CAAC;YAC9B,MAAM,WAAW,GAAG,MAAM,CAAC,WAAW,CAAC,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAAA;YAE3D,OAAO,SAAS,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC,CAAA;QACjD,CAAC;aAAM,CAAC;YACN,OAAO,EAAE,CAAA;QACX,CAAC;IACH,CAAC;IAED,KAAK,CAAC,cAAc,CAAC,KAAQ;QAC3B,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;QAEnC,OAAO,WAAW,CAAC,IAAI,CAAC,EAAE,EAAE,IAAI,EAAE,IAAI,CAAC,MAAM,CAAC,CAAA;IAChD,CAAC;CACF"}
1
+ {"version":3,"file":"deduplicator.js","sourceRoot":"","sources":["../src/deduplicator.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,GAAG,EAAE,MAAM,mBAAmB,CAAA;AACtD,OAAO,EAAE,cAAc,EAAE,MAAM,EAAkB,kBAAkB,EAAE,QAAQ,EAAE,MAAM,qBAAqB,CAAA;AAC1G,OAAO,EAAE,GAAG,EAAE,MAAM,eAAe,CAAA;AACnC,OAAO,EAAE,YAAY,EAAE,SAAS,EAAE,UAAU,EAAE,SAAS,EAAE,WAAW,EAAE,OAAO,EAAE,MAAM,eAAe,CAAA;AAEpG,OAAO,EAAE,SAAS,EAAE,MAAM,6BAA6B,CAAA;AACvD,OAAO,EAAE,MAAM,EAAE,MAAM,kBAAkB,CAAA;AAazC,MAAM,OAAO,YAAY;IACvB,YACU,EAAY,EACZ,IAAqC,EACrC,MAGP;QALO,OAAE,GAAF,EAAE,CAAU;QACZ,SAAI,GAAJ,IAAI,CAAiC;QACrC,WAAM,GAAN,MAAM,CAGb;IACA,CAAC;IAEJ,MAAM,CAAC,KAAK,CAAC,MAAM,CACjB,EACE,QAAQ,EACR,IAAI,GAAG,KAAK,CAAC,EAAE,CAAC,MAAM,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,EACxC,MAAM,EAAE,EACN,SAAS,GAAG,QAAQ,EACpB,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,SAAS,GAAG,CAAC,CAAC,EACnC,GAAG,EAAE,KACS,EAAE;QAEnB,MAAM,CACJ,kBAAkB,CAAC,SAAS,CAAC,IAAI,MAAM,CAAC,SAAS,CAAC,SAAS,CAAC,EAC5D,qDAAqD,CACtD,CAAA;QACD,MAAM,CACJ,kBAAkB,CAAC,MAAM,CAAC,IAAI,MAAM,CAAC,SAAS,CAAC,MAAM,CAAC,EACtD,kDAAkD,CACnD,CAAA;QACD,MAAM,CAAC,SAAS,GAAG,CAAC,EAAE,6CAA6C,CAAC,CAAA;QACpE,IAAI,QAAQ,CAAC,SAAS,CAAC,EAAE,CAAC;YACxB,MAAM,CAAC,MAAM,GAAG,SAAS,EAAE,sDAAsD,CAAC,CAAA;QACpF,CAAC;QAED,MAAM,EAAE,GAAG,MAAM,YAAY,CAAC,QAAQ,CAAC,CAAA;QAEvC,OAAO,IAAI,YAAY,CAAI,EAAE,EAAE,IAAI,EAAE,EAAE,SAAS,EAAE,MAAM,EAAE,CAAC,CAAA;IAC7D,CAAC;IAED,KAAK;QACH,IAAI,CAAC,EAAE,CAAC,KAAK,EAAE,CAAA;IACjB,CAAC;IAED,OAAO;QACL,OAAO,OAAO,CAAC,IAAI,CAAC,EAAE,CAAC,CAAA;IACzB,CAAC;IAED,KAAK,CAAC,GAAG,CAAC,MAAW;QACnB,MAAM,MAAM,GAAG,MAAM,GAAG,CAAC,MAAM,EAAE,KAAK,CAAC,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAA;QAE3D,SAAS,CAAC,IAAI,CAAC,EAAE,EAAE,MAAM,EAAE,IAAI,CAAC,MAAM,CAAC,CAAA;IACzC,CAAC;IAED,KAAK,CAAC,IAAI,CAAC,MAAW;QACpB,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAA;QAE9B,MAAM,MAAM,GAAG,MAAM,GAAG,CAAC,MAAM,EAAE,KAAK,CAAC,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAA;QAC3D,MAAM,OAAO,GAAG,SAAS,CAAC,IAAI,CAAC,EAAE,EAAE,MAAM,CAAC,CAAA;QAC1C,MAAM,SAAS,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAA;QAE9D,OAAO,SAAS,CAAA;IAClB,CAAC;IAED,KAAK,CAAC,QAAQ,CAAC,KAAQ;QACrB,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;QAEnC,OAAO,CAAC,UAAU,CAAC,IAAI,CAAC,EAAE,EAAE,IAAI,CAAC,CAAA;IACnC,CAAC;IAED,KAAK,CAAC,UAAU,CAAC,MAAW;QAC1B,MAAM,MAAM,GAAG,MAAM,GAAG,CAAC,MAAM,EAAE,IAAI,CAAC,IAAI,CAAC,CAAA;QAE3C,MAAM,SAAS,GAAG,SAAS,CAAC,IAAI,CAAC,EAAE,EAAE,MAAM,EAAE,IAAI,CAAC,MAAM,CAAC,CAAA;QAEzD,IAAI,cAAc,CAAC,SAAS,CAAC,EAAE,CAAC;YAC9B,MAAM,WAAW,GAAG,MAAM,CAAC,WAAW,CAAC,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAAA;YAE3D,OAAO,SAAS,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC,CAAA;QACjD,CAAC;aAAM,CAAC;YACN,OAAO,EAAE,CAAA;QACX,CAAC;IACH,CAAC;IAED,KAAK,CAAC,cAAc,CAAC,KAAQ;QAC3B,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;QAEnC,OAAO,WAAW,CAAC,IAAI,CAAC,EAAE,EAAE,IAAI,EAAE,IAAI,CAAC,MAAM,CAAC,CAAA;IAChD,CAAC;CACF"}
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "extra-deduplicator",
3
- "version": "0.3.0",
3
+ "version": "0.3.1",
4
4
  "description": "",
5
5
  "keywords": [],
6
6
  "files": [
@@ -59,5 +59,8 @@
59
59
  "extra-promise": "^7.1.1",
60
60
  "iterable-operator": "^6.0.0",
61
61
  "migration-files": "^0.4.3"
62
+ },
63
+ "allowScripts": {
64
+ "husky": true
62
65
  }
63
66
  }
package/src/database.ts CHANGED
@@ -34,10 +34,10 @@ export function hasHashes(db: IDatabase, hashes: string[]): boolean[] {
34
34
  , { hash_exists: number }
35
35
  >(`
36
36
  SELECT EXISTS(
37
- SELECT *
38
- FROM hash
39
- WHERE hash = $hash
40
- ) AS hash_exists
37
+ SELECT *
38
+ FROM hash
39
+ WHERE hash = $hash
40
+ ) AS hash_exists
41
41
  `).get({ hash })
42
42
 
43
43
  return !!row?.hash_exists
@@ -133,6 +133,18 @@ function shrink(db: IDatabase, options: IShrinkOptions): void {
133
133
  }
134
134
  }
135
135
 
136
+ export function isEmpty(db: IDatabase): boolean {
137
+ const row = db.prepare<[], { matched: number }>(`
138
+ SELECT EXISTS (
139
+ SELECT 1
140
+ FROM hash
141
+ LIMIT 1
142
+ ) AS matched
143
+ `).get()
144
+
145
+ return !row?.matched
146
+ }
147
+
136
148
  function countHashes(db: IDatabase): number {
137
149
  const row = db.prepare<[], { count: number }>(`
138
150
  SELECT COUNT(*) as count
@@ -1,7 +1,7 @@
1
1
  import { toArray, uniq, zip } from 'iterable-operator'
2
2
  import { isntEmptyArray, assert, type Awaitable, isPositiveInfinity, isFinite } from '@blackglory/prelude'
3
3
  import { map } from 'extra-promise'
4
- import { openDatabase, hasHashes, isLastHash, addHashes, addLastHash } from './database.js'
4
+ import { openDatabase, hasHashes, isLastHash, addHashes, addLastHash, isEmpty } from './database.js'
5
5
  import type { Database } from 'better-sqlite3'
6
6
  import { stringify } from 'extra-json-stable-stringify'
7
7
  import { sha256 } from 'extra-compatible'
@@ -59,6 +59,10 @@ export class Deduplicator<T> {
59
59
  this.db.close()
60
60
  }
61
61
 
62
+ isEmpty(): boolean {
63
+ return isEmpty(this.db)
64
+ }
65
+
62
66
  async add(values: T[]): Promise<void> {
63
67
  const hashes = await map(values, value => this.hash(value))
64
68