@powersync/service-module-mongodb-storage 0.12.1 → 0.12.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +9 -0
- package/dist/migrations/db/migrations/1741697235857-bucket-state-index.js +1 -4
- package/dist/migrations/db/migrations/1741697235857-bucket-state-index.js.map +1 -1
- package/dist/storage/MongoBucketStorage.d.ts +3 -2
- package/dist/storage/MongoBucketStorage.js +4 -2
- package/dist/storage/MongoBucketStorage.js.map +1 -1
- package/dist/storage/implementation/MongoChecksums.d.ts +45 -13
- package/dist/storage/implementation/MongoChecksums.js +148 -135
- package/dist/storage/implementation/MongoChecksums.js.map +1 -1
- package/dist/storage/implementation/MongoCompactor.js +23 -21
- package/dist/storage/implementation/MongoCompactor.js.map +1 -1
- package/dist/storage/implementation/MongoSyncBucketStorage.d.ts +5 -2
- package/dist/storage/implementation/MongoSyncBucketStorage.js +3 -3
- package/dist/storage/implementation/MongoSyncBucketStorage.js.map +1 -1
- package/dist/storage/implementation/MongoTestStorageFactoryGenerator.d.ts +2 -0
- package/dist/storage/implementation/MongoTestStorageFactoryGenerator.js +4 -3
- package/dist/storage/implementation/MongoTestStorageFactoryGenerator.js.map +1 -1
- package/dist/storage/implementation/db.d.ts +4 -0
- package/dist/storage/implementation/db.js +10 -0
- package/dist/storage/implementation/db.js.map +1 -1
- package/dist/storage/implementation/models.d.ts +4 -0
- package/package.json +3 -3
- package/src/migrations/db/migrations/1741697235857-bucket-state-index.ts +1 -7
- package/src/storage/MongoBucketStorage.ts +4 -3
- package/src/storage/implementation/MongoChecksums.ts +172 -150
- package/src/storage/implementation/MongoCompactor.ts +23 -22
- package/src/storage/implementation/MongoSyncBucketStorage.ts +9 -4
- package/src/storage/implementation/MongoTestStorageFactoryGenerator.ts +7 -4
- package/src/storage/implementation/db.ts +14 -0
- package/src/storage/implementation/models.ts +4 -0
- package/test/src/__snapshots__/storage.test.ts.snap +17 -1
- package/test/src/storage.test.ts +38 -1
- package/test/src/storage_compacting.test.ts +120 -5
- package/tsconfig.tsbuildinfo +1 -1
package/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,14 @@
|
|
|
1
1
|
# @powersync/service-module-mongodb-storage
|
|
2
2
|
|
|
3
|
+
## 0.12.2
|
|
4
|
+
|
|
5
|
+
### Patch Changes
|
|
6
|
+
|
|
7
|
+
- bec7496: Fix "E11000 duplicate key error collection: powersync_demo.bucket_state" in some cases on sync rules deploy
|
|
8
|
+
- 725daa1: Fix rare issue of incorrect checksums on fallback after checksum query timed out.
|
|
9
|
+
- Updated dependencies [bec7496]
|
|
10
|
+
- @powersync/service-core@1.15.2
|
|
11
|
+
|
|
3
12
|
## 0.12.1
|
|
4
13
|
|
|
5
14
|
### Patch Changes
|
|
@@ -4,10 +4,7 @@ export const up = async (context) => {
|
|
|
4
4
|
const { service_context: { configuration } } = context;
|
|
5
5
|
const db = storage.createPowerSyncMongo(configuration.storage);
|
|
6
6
|
try {
|
|
7
|
-
await db.
|
|
8
|
-
'_id.g': 1,
|
|
9
|
-
last_op: 1
|
|
10
|
-
}, { name: INDEX_NAME, unique: true });
|
|
7
|
+
await db.createBucketStateIndex();
|
|
11
8
|
}
|
|
12
9
|
finally {
|
|
13
10
|
await db.client.close();
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"1741697235857-bucket-state-index.js","sourceRoot":"","sources":["../../../../src/migrations/db/migrations/1741697235857-bucket-state-index.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,OAAO,MAAM,mCAAmC,CAAC;AAG7D,MAAM,UAAU,GAAG,gBAAgB,CAAC;AAEpC,MAAM,CAAC,MAAM,EAAE,GAA0C,KAAK,EAAE,OAAO,EAAE,EAAE;IACzE,MAAM,EACJ,eAAe,EAAE,EAAE,aAAa,EAAE,EACnC,GAAG,OAAO,CAAC;IACZ,MAAM,EAAE,GAAG,OAAO,CAAC,oBAAoB,CAAC,aAAa,CAAC,OAA6B,CAAC,CAAC;IAErF,IAAI,CAAC;QACH,MAAM,EAAE,CAAC,
|
|
1
|
+
{"version":3,"file":"1741697235857-bucket-state-index.js","sourceRoot":"","sources":["../../../../src/migrations/db/migrations/1741697235857-bucket-state-index.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,OAAO,MAAM,mCAAmC,CAAC;AAG7D,MAAM,UAAU,GAAG,gBAAgB,CAAC;AAEpC,MAAM,CAAC,MAAM,EAAE,GAA0C,KAAK,EAAE,OAAO,EAAE,EAAE;IACzE,MAAM,EACJ,eAAe,EAAE,EAAE,aAAa,EAAE,EACnC,GAAG,OAAO,CAAC;IACZ,MAAM,EAAE,GAAG,OAAO,CAAC,oBAAoB,CAAC,aAAa,CAAC,OAA6B,CAAC,CAAC;IAErF,IAAI,CAAC;QACH,MAAM,EAAE,CAAC,sBAAsB,EAAE,CAAC;IACpC,CAAC;YAAS,CAAC;QACT,MAAM,EAAE,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC;IAC1B,CAAC;AACH,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,IAAI,GAA0C,KAAK,EAAE,OAAO,EAAE,EAAE;IAC3E,MAAM,EACJ,eAAe,EAAE,EAAE,aAAa,EAAE,EACnC,GAAG,OAAO,CAAC;IAEZ,MAAM,EAAE,GAAG,OAAO,CAAC,oBAAoB,CAAC,aAAa,CAAC,OAA6B,CAAC,CAAC;IAErF,IAAI,CAAC;QACH,IAAI,MAAM,EAAE,CAAC,YAAY,CAAC,WAAW,CAAC,UAAU,CAAC,EAAE,CAAC;YAClD,MAAM,EAAE,CAAC,YAAY,CAAC,SAAS,CAAC,UAAU,CAAC,CAAC;QAC9C,CAAC;IACH,CAAC;YAAS,CAAC;QACT,MAAM,EAAE,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC;IAC1B,CAAC;AACH,CAAC,CAAC"}
|
|
@@ -2,8 +2,9 @@ import { GetIntanceOptions, storage } from '@powersync/service-core';
|
|
|
2
2
|
import { BaseObserver } from '@powersync/lib-services-framework';
|
|
3
3
|
import { PowerSyncMongo } from './implementation/db.js';
|
|
4
4
|
import { MongoPersistedSyncRulesContent } from './implementation/MongoPersistedSyncRulesContent.js';
|
|
5
|
-
import { MongoSyncBucketStorage } from './implementation/MongoSyncBucketStorage.js';
|
|
5
|
+
import { MongoSyncBucketStorage, MongoSyncBucketStorageOptions } from './implementation/MongoSyncBucketStorage.js';
|
|
6
6
|
export declare class MongoBucketStorage extends BaseObserver<storage.BucketStorageFactoryListener> implements storage.BucketStorageFactory {
|
|
7
|
+
private internalOptions?;
|
|
7
8
|
private readonly client;
|
|
8
9
|
private readonly session;
|
|
9
10
|
readonly slot_name_prefix: string;
|
|
@@ -11,7 +12,7 @@ export declare class MongoBucketStorage extends BaseObserver<storage.BucketStora
|
|
|
11
12
|
readonly db: PowerSyncMongo;
|
|
12
13
|
constructor(db: PowerSyncMongo, options: {
|
|
13
14
|
slot_name_prefix: string;
|
|
14
|
-
});
|
|
15
|
+
}, internalOptions?: MongoSyncBucketStorageOptions | undefined);
|
|
15
16
|
[Symbol.asyncDispose](): Promise<void>;
|
|
16
17
|
getInstance(syncRules: storage.PersistedSyncRulesContent, options?: GetIntanceOptions): MongoSyncBucketStorage;
|
|
17
18
|
getSystemIdentifier(): Promise<storage.BucketStorageSystemIdentifier>;
|
|
@@ -7,14 +7,16 @@ import { MongoPersistedSyncRulesContent } from './implementation/MongoPersistedS
|
|
|
7
7
|
import { MongoSyncBucketStorage } from './implementation/MongoSyncBucketStorage.js';
|
|
8
8
|
import { generateSlotName } from './implementation/util.js';
|
|
9
9
|
export class MongoBucketStorage extends BaseObserver {
|
|
10
|
+
internalOptions;
|
|
10
11
|
client;
|
|
11
12
|
session;
|
|
12
13
|
// TODO: This is still Postgres specific and needs to be reworked
|
|
13
14
|
slot_name_prefix;
|
|
14
15
|
activeStorageCache;
|
|
15
16
|
db;
|
|
16
|
-
constructor(db, options) {
|
|
17
|
+
constructor(db, options, internalOptions) {
|
|
17
18
|
super();
|
|
19
|
+
this.internalOptions = internalOptions;
|
|
18
20
|
this.client = db.client;
|
|
19
21
|
this.db = db;
|
|
20
22
|
this.session = this.client.startSession();
|
|
@@ -28,7 +30,7 @@ export class MongoBucketStorage extends BaseObserver {
|
|
|
28
30
|
if (typeof id == 'bigint') {
|
|
29
31
|
id = Number(id);
|
|
30
32
|
}
|
|
31
|
-
const storage = new MongoSyncBucketStorage(this, id, syncRules, slot_name);
|
|
33
|
+
const storage = new MongoSyncBucketStorage(this, id, syncRules, slot_name, undefined, this.internalOptions);
|
|
32
34
|
if (!options?.skipLifecycleHooks) {
|
|
33
35
|
this.iterateListeners((cb) => cb.syncStorageCreated?.(storage));
|
|
34
36
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"MongoBucketStorage.js","sourceRoot":"","sources":["../../src/storage/MongoBucketStorage.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,+BAA+B,CAAC;AAE7D,OAAO,EAAqB,OAAO,EAAE,MAAM,yBAAyB,CAAC;AAErE,OAAO,EAAE,YAAY,EAAE,SAAS,EAAE,MAAM,EAAE,YAAY,EAAE,MAAM,mCAAmC,CAAC;AAClG,OAAO,EAAE,EAAE,IAAI,IAAI,EAAE,MAAM,MAAM,CAAC;AAElC,OAAO,KAAK,SAAS,MAAM,gCAAgC,CAAC;AAK5D,OAAO,EAAE,8BAA8B,EAAE,MAAM,oDAAoD,CAAC;AACpG,OAAO,EAAE,sBAAsB,EAAE,MAAM,4CAA4C,CAAC;AACpF,OAAO,EAAE,gBAAgB,EAAE,MAAM,0BAA0B,CAAC;AAE5D,MAAM,OAAO,kBACX,SAAQ,YAAkD;IAGzC,MAAM,CAAoB;IAC1B,OAAO,CAAsB;IAC9C,iEAAiE;IACjD,gBAAgB,CAAS;IAEjC,kBAAkB,CAAqC;IAE/C,EAAE,CAAiB;IAEnC,YACE,EAAkB,EAClB,OAEC;QAED,KAAK,EAAE,CAAC;QACR,IAAI,CAAC,MAAM,GAAG,EAAE,CAAC,MAAM,CAAC;QACxB,IAAI,CAAC,EAAE,GAAG,EAAE,CAAC;QACb,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC,MAAM,CAAC,YAAY,EAAE,CAAC;QAC1C,IAAI,CAAC,gBAAgB,GAAG,OAAO,CAAC,gBAAgB,CAAC;IACnD,CAAC;IAED,KAAK,CAAC,CAAC,MAAM,CAAC,YAAY,CAAC;QACzB,QAAQ;IACV,CAAC;IAED,WAAW,CAAC,SAA4C,EAAE,OAA2B;QACnF,IAAI,EAAE,EAAE,EAAE,SAAS,EAAE,GAAG,SAAS,CAAC;QAClC,IAAK,OAAO,EAAU,IAAI,QAAQ,EAAE,CAAC;YACnC,EAAE,GAAG,MAAM,CAAC,EAAE,CAAC,CAAC;QAClB,CAAC;QACD,MAAM,OAAO,GAAG,IAAI,sBAAsB,CAAC,IAAI,EAAE,EAAE,EAAE,SAAS,EAAE,SAAS,CAAC,CAAC;QAC3E,IAAI,CAAC,OAAO,EAAE,kBAAkB,EAAE,CAAC;YACjC,IAAI,CAAC,gBAAgB,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,kBAAkB,EAAE,CAAC,OAAO,CAAC,CAAC,CAAC;QAClE,CAAC;QACD,OAAO,CAAC,gBAAgB,CAAC;YACvB,YAAY,EAAE,CAAC,KAAK,EAAE,EAAE;gBACtB,KAAK,CAAC,gBAAgB,CAAC;oBACrB,gBAAgB,EAAE,CAAC,OAAO,EAAE,EAAE,CAAC,IAAI,CAAC,gBAAgB,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,gBAAgB,EAAE,CAAC,OAAO,CAAC,CAAC;iBAC7F,CAAC,CAAC;YACL,CAAC;SACF,CAAC,CAAC;QACH,OAAO,OAAO,CAAC;IACjB,CAAC;IAED,KAAK,CAAC,mBAAmB;QACvB,MAAM,EAAE,OAAO,EAAE,EAAE,EAAE,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,EAAE,CAAC,OAAO,CAAC;YAC/C,KAAK,EAAE,CAAC;SACT,CAAC,CAAC;QACH,IAAI,EAAE,IAAI,IAAI,EAAE,CAAC;YACf,MAAM,IAAI,YAAY,CACpB,SAAS,CAAC,WAAW,EACrB,oEAAoE,CACrE,CAAC;QACJ,CAAC;QAED,OAAO;YACL,EAAE;YACF,IAAI,EAAE,SAAS,CAAC,qBAAqB;SACtC,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,kBAAkB,CAAC,OAAuC;QAC9D,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,uBAAuB,EAAE,CAAC;QAClD,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,yBAAyB,EAAE,CAAC;QAEtD,IAAI,IAAI,EAAE,kBAAkB,IAAI,OAAO,CAAC,OAAO,EAAE,CAAC;YAChD,MAAM,CAAC,IAAI,CAAC,yCAAyC,CAAC,CAAC;YACvD,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC;QAC5B,CAAC;aAAM,IAAI,IAAI,IAAI,IAAI,IAAI,MAAM,EAAE,kBAAkB,IAAI,OAAO,CAAC,OAAO,EAAE,CAAC;YACzE,MAAM,CAAC,IAAI,CAAC,yCAAyC,CAAC,CAAC;YACvD,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC;QAC5B,CAAC;aAAM,CAAC;YACN,MAAM,CAAC,IAAI,CAAC,uCAAuC,CAAC,CAAC;YACrD,MAAM,oBAAoB,GAAG,MAAM,IAAI,CAAC,eAAe,CAAC,OAAO,CAAC,CAAC;YACjE,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,oBAAoB,EAAE,IAAI,EAAE,oBAAoB,CAAC,YAAY,IAAI,SAAS,EAAE,CAAC;QACvG,CAAC;IACH,CAAC;IAED,KAAK,CAAC,kBAAkB,CAAC,mBAA2B;QAClD,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,uBAAuB,EAAE,CAAC;QAClD,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,yBAAyB,EAAE,CAAC;QAEtD,IAAI,IAAI,IAAI,IAAI,IAAI,IAAI,CAAC,EAAE,IAAI,mBAAmB,EAAE,CAAC;YACnD,wCAAwC;YACxC,MAAM,IAAI,CAAC,eAAe,CAAC;gBACzB,OAAO,EAAE,IAAI,CAAC,kBAAkB;gBAChC,QAAQ,EAAE,KAAK;aAChB,CAAC,CAAC;YACH,gCAAgC;YAChC,MAAM,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,SAAS,CAChC;gBACE,GAAG,EAAE,IAAI,CAAC,EAAE;gBACZ,KAAK,EAAE,OAAO,CAAC,aAAa,CAAC,UAAU;aACxC,EACD;gBACE,IAAI,EAAE;oBACJ,KAAK,EAAE,OAAO,CAAC,aAAa,CAAC,IAAI;iBAClC;aACF,CACF,CAAC;YACF,MAAM,IAAI,CAAC,EAAE,CAAC,gBAAgB,EAAE,CAAC;QACnC,CAAC;aAAM,IAAI,IAAI,IAAI,IAAI,IAAI,MAAM,EAAE,EAAE,IAAI,mBAAmB,EAAE,CAAC;YAC7D,sEAAsE;YACtE,MAAM,IAAI,CAAC,eAAe,CAAC;gBACzB,OAAO,EAAE,MAAM,CAAC,kBAAkB;gBAClC,QAAQ,EAAE,KAAK;aAChB,CAAC,CAAC;YAEH,wFAAwF;YACxF,iEAAiE;YACjE,yCAAyC;YAEzC,MAAM,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,SAAS,CAChC;gBACE,GAAG,EAAE,MAAM,CAAC,EAAE;gBACd,KAAK,EAAE,OAAO,CAAC,aAAa,CAAC,MAAM;aACpC,EACD;gBACE,IAAI,EAAE;oBACJ,KAAK,EAAE,OAAO,CAAC,aAAa,CAAC,OAAO;iBACrC;aACF,CACF,CAAC;YACF,MAAM,IAAI,CAAC,EAAE,CAAC,gBAAgB,EAAE,CAAC;QACnC,CAAC;aAAM,IAAI,IAAI,IAAI,IAAI,IAAI,MAAM,EAAE,EAAE,IAAI,mBAAmB,EAAE,CAAC;YAC7D,6EAA6E;YAE7E,MAAM,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,SAAS,CAChC;gBACE,GAAG,EAAE,MAAM,CAAC,EAAE;gBACd,KAAK,EAAE,OAAO,CAAC,aAAa,CAAC,MAAM;aACpC,EACD;gBACE,IAAI,EAAE;oBACJ,KAAK,EAAE,OAAO,CAAC,aAAa,CAAC,OAAO;iBACrC;aACF,CACF,CAAC;YACF,MAAM,IAAI,CAAC,EAAE,CAAC,gBAAgB,EAAE,CAAC;QACnC,CAAC;IACH,CAAC;IAED,KAAK,CAAC,eAAe,CAAC,OAAuC;QAC3D,IAAI,OAAO,CAAC,QAAQ,EAAE,CAAC;YACrB,iDAAiD;YACjD,YAAY,CAAC,QAAQ,CAAC,OAAO,CAAC,OAAO,EAAE;gBACrC,2CAA2C;gBAC3C,MAAM,EAAE,SAAS;gBACjB,aAAa,EAAE,gBAAgB,EAAE,4BAA4B;gBAC7D,YAAY,EAAE,IAAI;aACnB,CAAC,CAAC;QACL,CAAC;aAAM,CAAC;YACN,+CAA+C;YAC/C,6FAA6F;QAC/F,CAAC;QAED,IAAI,KAAK,GAA+C,SAAS,CAAC;QAElE,MAAM,IAAI,CAAC,OAAO,CAAC,eAAe,CAAC,KAAK,IAAI,EAAE;YAC5C,wDAAwD;YACxD,MAAM,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,UAAU,CACjC;gBACE,KAAK,EAAE,OAAO,CAAC,aAAa,CAAC,UAAU;aACxC,EACD,EAAE,IAAI,EAAE,EAAE,KAAK,EAAE,OAAO,CAAC,aAAa,CAAC,IAAI,EAAE,EAAE,CAChD,CAAC;YAEF,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,cAAc,CAAC,gBAAgB,CAC1D;gBACE,GAAG,EAAE,YAAY;aAClB,EACD;gBACE,IAAI,EAAE;oBACJ,KAAK,EAAE,EAAE;iBACV;aACF,EACD;gBACE,MAAM,EAAE,IAAI;gBACZ,cAAc,EAAE,OAAO;aACxB,CACF,CAAC;YAEF,MAAM,EAAE,GAAG,MAAM,CAAC,MAAO,CAAC,KAAK,CAAC,CAAC;YACjC,MAAM,SAAS,GAAG,gBAAgB,CAAC,IAAI,CAAC,gBAAgB,EAAE,EAAE,CAAC,CAAC;YAE9D,MAAM,GAAG,GAAqB;gBAC5B,GAAG,EAAE,EAAE;gBACP,OAAO,EAAE,OAAO,CAAC,OAAO;gBACxB,eAAe,EAAE,IAAI;gBACrB,mBAAmB,EAAE,IAAI;gBACzB,oBAAoB,EAAE,IAAI;gBAC1B,YAAY,EAAE,IAAI;gBAClB,aAAa,EAAE,KAAK;gBACpB,YAAY,EAAE,SAAS;gBACvB,KAAK,EAAE,OAAO,CAAC,aAAa,CAAC,UAAU;gBACvC,SAAS,EAAE,SAAS;gBACpB,kBAAkB,EAAE,IAAI;gBACxB,gBAAgB,EAAE,IAAI;gBACtB,iBAAiB,EAAE,IAAI;aACxB,CAAC;YACF,MAAM,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC;YACxC,MAAM,IAAI,CAAC,EAAE,CAAC,gBAAgB,EAAE,CAAC;YACjC,KAAK,GAAG,IAAI,8BAA8B,CAAC,IAAI,CAAC,EAAE,EAAE,GAAG,CAAC,CAAC;YACzD,IAAI,OAAO,CAAC,IAAI,EAAE,CAAC;gBACjB,MAAM,IAAI,GAAG,MAAM,KAAK,CAAC,IAAI,EAAE,CAAC;YAClC,CAAC;QACH,CAAC,CAAC,CAAC;QAEH,OAAO,KAAM,CAAC;IAChB,CAAC;IAED,KAAK,CAAC,yBAAyB;QAC7B,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,OAAO,CAC1C;YACE,KAAK,EAAE,EAAE,GAAG,EAAE,CAAC,OAAO,CAAC,aAAa,CAAC,MAAM,EAAE,OAAO,CAAC,aAAa,CAAC,OAAO,CAAC,EAAE;SAC9E,EACD,EAAE,IAAI,EAAE,EAAE,GAAG,EAAE,CAAC,CAAC,EAAE,EAAE,KAAK,EAAE,CAAC,EAAE,CAChC,CAAC;QACF,IAAI,GAAG,IAAI,IAAI,EAAE,CAAC;YAChB,OAAO,IAAI,CAAC;QACd,CAAC;QAED,OAAO,IAAI,8BAA8B,CAAC,IAAI,CAAC,EAAE,EAAE,GAAG,CAAC,CAAC;IAC1D,CAAC;IAED,KAAK,CAAC,kBAAkB,CAAC,OAAsC;QAC7D,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,yBAAyB,EAAE,CAAC;QACvD,OAAO,OAAO,EAAE,MAAM,CAAC,OAAO,CAAC,IAAI,IAAI,CAAC;IAC1C,CAAC;IAED,KAAK,CAAC,uBAAuB;QAC3B,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,OAAO,CAC1C;YACE,KAAK,EAAE,OAAO,CAAC,aAAa,CAAC,UAAU;SACxC,EACD,EAAE,IAAI,EAAE,EAAE,GAAG,EAAE,CAAC,CAAC,EAAE,EAAE,KAAK,EAAE,CAAC,EAAE,CAChC,CAAC;QACF,IAAI,GAAG,IAAI,IAAI,EAAE,CAAC;YAChB,OAAO,IAAI,CAAC;QACd,CAAC;QAED,OAAO,IAAI,8BAA8B,CAAC,IAAI,CAAC,EAAE,EAAE,GAAG,CAAC,CAAC;IAC1D,CAAC;IAED,KAAK,CAAC,gBAAgB,CAAC,OAAsC;QAC3D,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,uBAAuB,EAAE,CAAC;QACrD,OAAO,OAAO,EAAE,MAAM,CAAC,OAAO,CAAC,IAAI,IAAI,CAAC;IAC1C,CAAC;IAED,KAAK,CAAC,uBAAuB;QAC3B,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,UAAU;aAClC,IAAI,CAAC;YACJ,KAAK,EAAE,EAAE,GAAG,EAAE,CAAC,OAAO,CAAC,aAAa,CAAC,UAAU,EAAE,OAAO,CAAC,aAAa,CAAC,MAAM,CAAC,EAAE;SACjF,CAAC;aACD,OAAO,EAAE,CAAC;QAEb,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE;YACtB,OAAO,IAAI,8BAA8B,CAAC,IAAI,CAAC,EAAE,EAAE,GAAG,CAAC,CAAC;QAC1D,CAAC,CAAC,CAAC;IACL,CAAC;IAED,KAAK,CAAC,mBAAmB;QACvB,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,UAAU;aAClC,IAAI,CAAC;YACJ,KAAK,EAAE,OAAO,CAAC,aAAa,CAAC,IAAI;SAClC,CAAC;aACD,OAAO,EAAE,CAAC;QAEb,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE;YACtB,OAAO,IAAI,8BAA8B,CAAC,IAAI,CAAC,EAAE,EAAE,GAAG,CAAC,CAAC;QAC1D,CAAC,CAAC,CAAC;IACL,CAAC;IAED,KAAK,CAAC,gBAAgB;QACpB,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,yBAAyB,EAAE,CAAC;QACvD,IAAI,OAAO,IAAI,IAAI,EAAE,CAAC;YACpB,OAAO,IAAI,CAAC;QACd,CAAC;QAED,gDAAgD;QAChD,+FAA+F;QAC/F,wBAAwB;QACxB,IAAI,IAAI,CAAC,kBAAkB,EAAE,QAAQ,IAAI,OAAO,CAAC,EAAE,EAAE,CAAC;YACpD,OAAO,IAAI,CAAC,kBAAkB,CAAC;QACjC,CAAC;aAAM,CAAC;YACN,MAAM,QAAQ,GAAG,IAAI,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC;YAC3C,IAAI,CAAC,kBAAkB,GAAG,QAAQ,CAAC;YACnC,OAAO,QAAQ,CAAC;QAClB,CAAC;IACH,CAAC;IAED,KAAK,CAAC,iBAAiB;QACrB,MAAM,iBAAiB,GAAG,CAAC,CAAU,EAAE,EAAE;YACvC,IAAI,SAAS,CAAC,kBAAkB,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,QAAQ,IAAI,mBAAmB,EAAE,CAAC;gBACzE,sCAAsC;gBACtC,OAAO,CAAC,EAAE,YAAY,EAAE,EAAE,IAAI,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC;YACzC,CAAC;iBAAM,CAAC;gBACN,OAAO,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;YAC3B,CAAC;QACH,CAAC,CAAC;QAEF,MAAM,iBAAiB,GAAG,MAAM,IAAI,CAAC,kBAAkB,CAAC,EAAE,aAAa,EAAE,QAAQ,EAAE,CAAC,CAAC;QACrF,IAAI,iBAAiB,IAAI,IAAI,EAAE,CAAC;YAC9B,OAAO;gBACL,qBAAqB,EAAE,CAAC;gBACxB,qBAAqB,EAAE,CAAC;gBACxB,sBAAsB,EAAE,CAAC;aAC1B,CAAC;QACJ,CAAC;QACD,MAAM,oBAAoB,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,WAAW;aAEnD,SAAS,CAAC;YACT;gBACE,UAAU,EAAE;oBACV,YAAY,EAAE,EAAE;iBACjB;aACF;SACF,CAAC;aACD,OAAO,EAAE;aACT,KAAK,CAAC,iBAAiB,CAAC,CAAC;QAE5B,MAAM,oBAAoB,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,iBAAiB;aACzD,SAAS,CAAC;YACT;gBACE,UAAU,EAAE;oBACV,YAAY,EAAE,EAAE;iBACjB;aACF;SACF,CAAC;aACD,OAAO,EAAE;aACT,KAAK,CAAC,iBAAiB,CAAC,CAAC;QAE5B,MAAM,qBAAqB,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,YAAY;aACrD,SAAS,CAAC;YACT;gBACE,UAAU,EAAE;oBACV,YAAY,EAAE,EAAE;iBACjB;aACF;SACF,CAAC;aACD,OAAO,EAAE;aACT,KAAK,CAAC,iBAAiB,CAAC,CAAC;QAE5B,OAAO;YACL,qBAAqB,EAAE,MAAM,CAAC,oBAAoB,CAAC,CAAC,CAAC,CAAC,YAAY,CAAC,IAAI,CAAC;YACxE,qBAAqB,EAAE,MAAM,CAAC,oBAAoB,CAAC,CAAC,CAAC,CAAC,YAAY,CAAC,IAAI,CAAC;YACxE,sBAAsB,EAAE,MAAM,CAAC,qBAAqB,CAAC,CAAC,CAAC,CAAC,YAAY,CAAC,IAAI,CAAC;SAC3E,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,sBAAsB;QAC1B,IAAI,QAAQ,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,QAAQ,CAAC,OAAO,CAAC;YAC5C,GAAG,EAAE,EAAE,OAAO,EAAE,IAAI,EAAE;SACvB,CAAC,CAAC;QAEH,IAAI,CAAC,QAAQ,EAAE,CAAC;YACd,MAAM,OAAO,GAAG,IAAI,SAAS,CAAC,KAAK,CAAC,gBAAgB,CAAC;gBACnD,UAAU,EAAE,IAAI,CAAC,EAAE,CAAC,KAAK;gBACzB,IAAI,EAAE,4BAA4B;aACnC,CAAC,CAAC;YAEH,MAAM,OAAO,CAAC,IAAI,CAAC,KAAK,IAAI,EAAE;gBAC5B,MAAM,IAAI,CAAC,EAAE,CAAC,QAAQ,CAAC,SAAS,CAAC;oBAC/B,GAAG,EAAE,IAAI,EAAE;iBACZ,CAAC,CAAC;YACL,CAAC,CAAC,CAAC;YAEH,QAAQ,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,QAAQ,CAAC,OAAO,CAAC;gBACxC,GAAG,EAAE,EAAE,OAAO,EAAE,IAAI,EAAE;aACvB,CAAC,CAAC;QACL,CAAC;QAED,OAAO,QAAS,CAAC,GAAG,CAAC;IACvB,CAAC;CACF"}
|
|
1
|
+
{"version":3,"file":"MongoBucketStorage.js","sourceRoot":"","sources":["../../src/storage/MongoBucketStorage.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,+BAA+B,CAAC;AAE7D,OAAO,EAAqB,OAAO,EAAE,MAAM,yBAAyB,CAAC;AAErE,OAAO,EAAE,YAAY,EAAE,SAAS,EAAE,MAAM,EAAE,YAAY,EAAE,MAAM,mCAAmC,CAAC;AAClG,OAAO,EAAE,EAAE,IAAI,IAAI,EAAE,MAAM,MAAM,CAAC;AAElC,OAAO,KAAK,SAAS,MAAM,gCAAgC,CAAC;AAK5D,OAAO,EAAE,8BAA8B,EAAE,MAAM,oDAAoD,CAAC;AACpG,OAAO,EAAE,sBAAsB,EAAiC,MAAM,4CAA4C,CAAC;AACnH,OAAO,EAAE,gBAAgB,EAAE,MAAM,0BAA0B,CAAC;AAE5D,MAAM,OAAO,kBACX,SAAQ,YAAkD;IAiBhD;IAdO,MAAM,CAAoB;IAC1B,OAAO,CAAsB;IAC9C,iEAAiE;IACjD,gBAAgB,CAAS;IAEjC,kBAAkB,CAAqC;IAE/C,EAAE,CAAiB;IAEnC,YACE,EAAkB,EAClB,OAEC,EACO,eAA+C;QAEvD,KAAK,EAAE,CAAC;QAFA,oBAAe,GAAf,eAAe,CAAgC;QAGvD,IAAI,CAAC,MAAM,GAAG,EAAE,CAAC,MAAM,CAAC;QACxB,IAAI,CAAC,EAAE,GAAG,EAAE,CAAC;QACb,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC,MAAM,CAAC,YAAY,EAAE,CAAC;QAC1C,IAAI,CAAC,gBAAgB,GAAG,OAAO,CAAC,gBAAgB,CAAC;IACnD,CAAC;IAED,KAAK,CAAC,CAAC,MAAM,CAAC,YAAY,CAAC;QACzB,QAAQ;IACV,CAAC;IAED,WAAW,CAAC,SAA4C,EAAE,OAA2B;QACnF,IAAI,EAAE,EAAE,EAAE,SAAS,EAAE,GAAG,SAAS,CAAC;QAClC,IAAK,OAAO,EAAU,IAAI,QAAQ,EAAE,CAAC;YACnC,EAAE,GAAG,MAAM,CAAC,EAAE,CAAC,CAAC;QAClB,CAAC;QACD,MAAM,OAAO,GAAG,IAAI,sBAAsB,CAAC,IAAI,EAAE,EAAE,EAAE,SAAS,EAAE,SAAS,EAAE,SAAS,EAAE,IAAI,CAAC,eAAe,CAAC,CAAC;QAC5G,IAAI,CAAC,OAAO,EAAE,kBAAkB,EAAE,CAAC;YACjC,IAAI,CAAC,gBAAgB,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,kBAAkB,EAAE,CAAC,OAAO,CAAC,CAAC,CAAC;QAClE,CAAC;QACD,OAAO,CAAC,gBAAgB,CAAC;YACvB,YAAY,EAAE,CAAC,KAAK,EAAE,EAAE;gBACtB,KAAK,CAAC,gBAAgB,CAAC;oBACrB,gBAAgB,EAAE,CAAC,OAAO,EAAE,EAAE,CAAC,IAAI,CAAC,gBAAgB,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,gBAAgB,EAAE,CAAC,OAAO,CAAC,CAAC;iBAC7F,CAAC,CAAC;YACL,CAAC;SACF,CAAC,CAAC;QACH,OAAO,OAAO,CAAC;IACjB,CAAC;IAED,KAAK,CAAC,mBAAmB;QACvB,MAAM,EAAE,OAAO,EAAE,EAAE,EAAE,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,EAAE,CAAC,OAAO,CAAC;YAC/C,KAAK,EAAE,CAAC;SACT,CAAC,CAAC;QACH,IAAI,EAAE,IAAI,IAAI,EAAE,CAAC;YACf,MAAM,IAAI,YAAY,CACpB,SAAS,CAAC,WAAW,EACrB,oEAAoE,CACrE,CAAC;QACJ,CAAC;QAED,OAAO;YACL,EAAE;YACF,IAAI,EAAE,SAAS,CAAC,qBAAqB;SACtC,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,kBAAkB,CAAC,OAAuC;QAC9D,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,uBAAuB,EAAE,CAAC;QAClD,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,yBAAyB,EAAE,CAAC;QAEtD,IAAI,IAAI,EAAE,kBAAkB,IAAI,OAAO,CAAC,OAAO,EAAE,CAAC;YAChD,MAAM,CAAC,IAAI,CAAC,yCAAyC,CAAC,CAAC;YACvD,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC;QAC5B,CAAC;aAAM,IAAI,IAAI,IAAI,IAAI,IAAI,MAAM,EAAE,kBAAkB,IAAI,OAAO,CAAC,OAAO,EAAE,CAAC;YACzE,MAAM,CAAC,IAAI,CAAC,yCAAyC,CAAC,CAAC;YACvD,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC;QAC5B,CAAC;aAAM,CAAC;YACN,MAAM,CAAC,IAAI,CAAC,uCAAuC,CAAC,CAAC;YACrD,MAAM,oBAAoB,GAAG,MAAM,IAAI,CAAC,eAAe,CAAC,OAAO,CAAC,CAAC;YACjE,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,oBAAoB,EAAE,IAAI,EAAE,oBAAoB,CAAC,YAAY,IAAI,SAAS,EAAE,CAAC;QACvG,CAAC;IACH,CAAC;IAED,KAAK,CAAC,kBAAkB,CAAC,mBAA2B;QAClD,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,uBAAuB,EAAE,CAAC;QAClD,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,yBAAyB,EAAE,CAAC;QAEtD,IAAI,IAAI,IAAI,IAAI,IAAI,IAAI,CAAC,EAAE,IAAI,mBAAmB,EAAE,CAAC;YACnD,wCAAwC;YACxC,MAAM,IAAI,CAAC,eAAe,CAAC;gBACzB,OAAO,EAAE,IAAI,CAAC,kBAAkB;gBAChC,QAAQ,EAAE,KAAK;aAChB,CAAC,CAAC;YACH,gCAAgC;YAChC,MAAM,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,SAAS,CAChC;gBACE,GAAG,EAAE,IAAI,CAAC,EAAE;gBACZ,KAAK,EAAE,OAAO,CAAC,aAAa,CAAC,UAAU;aACxC,EACD;gBACE,IAAI,EAAE;oBACJ,KAAK,EAAE,OAAO,CAAC,aAAa,CAAC,IAAI;iBAClC;aACF,CACF,CAAC;YACF,MAAM,IAAI,CAAC,EAAE,CAAC,gBAAgB,EAAE,CAAC;QACnC,CAAC;aAAM,IAAI,IAAI,IAAI,IAAI,IAAI,MAAM,EAAE,EAAE,IAAI,mBAAmB,EAAE,CAAC;YAC7D,sEAAsE;YACtE,MAAM,IAAI,CAAC,eAAe,CAAC;gBACzB,OAAO,EAAE,MAAM,CAAC,kBAAkB;gBAClC,QAAQ,EAAE,KAAK;aAChB,CAAC,CAAC;YAEH,wFAAwF;YACxF,iEAAiE;YACjE,yCAAyC;YAEzC,MAAM,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,SAAS,CAChC;gBACE,GAAG,EAAE,MAAM,CAAC,EAAE;gBACd,KAAK,EAAE,OAAO,CAAC,aAAa,CAAC,MAAM;aACpC,EACD;gBACE,IAAI,EAAE;oBACJ,KAAK,EAAE,OAAO,CAAC,aAAa,CAAC,OAAO;iBACrC;aACF,CACF,CAAC;YACF,MAAM,IAAI,CAAC,EAAE,CAAC,gBAAgB,EAAE,CAAC;QACnC,CAAC;aAAM,IAAI,IAAI,IAAI,IAAI,IAAI,MAAM,EAAE,EAAE,IAAI,mBAAmB,EAAE,CAAC;YAC7D,6EAA6E;YAE7E,MAAM,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,SAAS,CAChC;gBACE,GAAG,EAAE,MAAM,CAAC,EAAE;gBACd,KAAK,EAAE,OAAO,CAAC,aAAa,CAAC,MAAM;aACpC,EACD;gBACE,IAAI,EAAE;oBACJ,KAAK,EAAE,OAAO,CAAC,aAAa,CAAC,OAAO;iBACrC;aACF,CACF,CAAC;YACF,MAAM,IAAI,CAAC,EAAE,CAAC,gBAAgB,EAAE,CAAC;QACnC,CAAC;IACH,CAAC;IAED,KAAK,CAAC,eAAe,CAAC,OAAuC;QAC3D,IAAI,OAAO,CAAC,QAAQ,EAAE,CAAC;YACrB,iDAAiD;YACjD,YAAY,CAAC,QAAQ,CAAC,OAAO,CAAC,OAAO,EAAE;gBACrC,2CAA2C;gBAC3C,MAAM,EAAE,SAAS;gBACjB,aAAa,EAAE,gBAAgB,EAAE,4BAA4B;gBAC7D,YAAY,EAAE,IAAI;aACnB,CAAC,CAAC;QACL,CAAC;aAAM,CAAC;YACN,+CAA+C;YAC/C,6FAA6F;QAC/F,CAAC;QAED,IAAI,KAAK,GAA+C,SAAS,CAAC;QAElE,MAAM,IAAI,CAAC,OAAO,CAAC,eAAe,CAAC,KAAK,IAAI,EAAE;YAC5C,wDAAwD;YACxD,MAAM,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,UAAU,CACjC;gBACE,KAAK,EAAE,OAAO,CAAC,aAAa,CAAC,UAAU;aACxC,EACD,EAAE,IAAI,EAAE,EAAE,KAAK,EAAE,OAAO,CAAC,aAAa,CAAC,IAAI,EAAE,EAAE,CAChD,CAAC;YAEF,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,cAAc,CAAC,gBAAgB,CAC1D;gBACE,GAAG,EAAE,YAAY;aAClB,EACD;gBACE,IAAI,EAAE;oBACJ,KAAK,EAAE,EAAE;iBACV;aACF,EACD;gBACE,MAAM,EAAE,IAAI;gBACZ,cAAc,EAAE,OAAO;aACxB,CACF,CAAC;YAEF,MAAM,EAAE,GAAG,MAAM,CAAC,MAAO,CAAC,KAAK,CAAC,CAAC;YACjC,MAAM,SAAS,GAAG,gBAAgB,CAAC,IAAI,CAAC,gBAAgB,EAAE,EAAE,CAAC,CAAC;YAE9D,MAAM,GAAG,GAAqB;gBAC5B,GAAG,EAAE,EAAE;gBACP,OAAO,EAAE,OAAO,CAAC,OAAO;gBACxB,eAAe,EAAE,IAAI;gBACrB,mBAAmB,EAAE,IAAI;gBACzB,oBAAoB,EAAE,IAAI;gBAC1B,YAAY,EAAE,IAAI;gBAClB,aAAa,EAAE,KAAK;gBACpB,YAAY,EAAE,SAAS;gBACvB,KAAK,EAAE,OAAO,CAAC,aAAa,CAAC,UAAU;gBACvC,SAAS,EAAE,SAAS;gBACpB,kBAAkB,EAAE,IAAI;gBACxB,gBAAgB,EAAE,IAAI;gBACtB,iBAAiB,EAAE,IAAI;aACxB,CAAC;YACF,MAAM,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC;YACxC,MAAM,IAAI,CAAC,EAAE,CAAC,gBAAgB,EAAE,CAAC;YACjC,KAAK,GAAG,IAAI,8BAA8B,CAAC,IAAI,CAAC,EAAE,EAAE,GAAG,CAAC,CAAC;YACzD,IAAI,OAAO,CAAC,IAAI,EAAE,CAAC;gBACjB,MAAM,IAAI,GAAG,MAAM,KAAK,CAAC,IAAI,EAAE,CAAC;YAClC,CAAC;QACH,CAAC,CAAC,CAAC;QAEH,OAAO,KAAM,CAAC;IAChB,CAAC;IAED,KAAK,CAAC,yBAAyB;QAC7B,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,OAAO,CAC1C;YACE,KAAK,EAAE,EAAE,GAAG,EAAE,CAAC,OAAO,CAAC,aAAa,CAAC,MAAM,EAAE,OAAO,CAAC,aAAa,CAAC,OAAO,CAAC,EAAE;SAC9E,EACD,EAAE,IAAI,EAAE,EAAE,GAAG,EAAE,CAAC,CAAC,EAAE,EAAE,KAAK,EAAE,CAAC,EAAE,CAChC,CAAC;QACF,IAAI,GAAG,IAAI,IAAI,EAAE,CAAC;YAChB,OAAO,IAAI,CAAC;QACd,CAAC;QAED,OAAO,IAAI,8BAA8B,CAAC,IAAI,CAAC,EAAE,EAAE,GAAG,CAAC,CAAC;IAC1D,CAAC;IAED,KAAK,CAAC,kBAAkB,CAAC,OAAsC;QAC7D,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,yBAAyB,EAAE,CAAC;QACvD,OAAO,OAAO,EAAE,MAAM,CAAC,OAAO,CAAC,IAAI,IAAI,CAAC;IAC1C,CAAC;IAED,KAAK,CAAC,uBAAuB;QAC3B,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,OAAO,CAC1C;YACE,KAAK,EAAE,OAAO,CAAC,aAAa,CAAC,UAAU;SACxC,EACD,EAAE,IAAI,EAAE,EAAE,GAAG,EAAE,CAAC,CAAC,EAAE,EAAE,KAAK,EAAE,CAAC,EAAE,CAChC,CAAC;QACF,IAAI,GAAG,IAAI,IAAI,EAAE,CAAC;YAChB,OAAO,IAAI,CAAC;QACd,CAAC;QAED,OAAO,IAAI,8BAA8B,CAAC,IAAI,CAAC,EAAE,EAAE,GAAG,CAAC,CAAC;IAC1D,CAAC;IAED,KAAK,CAAC,gBAAgB,CAAC,OAAsC;QAC3D,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,uBAAuB,EAAE,CAAC;QACrD,OAAO,OAAO,EAAE,MAAM,CAAC,OAAO,CAAC,IAAI,IAAI,CAAC;IAC1C,CAAC;IAED,KAAK,CAAC,uBAAuB;QAC3B,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,UAAU;aAClC,IAAI,CAAC;YACJ,KAAK,EAAE,EAAE,GAAG,EAAE,CAAC,OAAO,CAAC,aAAa,CAAC,UAAU,EAAE,OAAO,CAAC,aAAa,CAAC,MAAM,CAAC,EAAE;SACjF,CAAC;aACD,OAAO,EAAE,CAAC;QAEb,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE;YACtB,OAAO,IAAI,8BAA8B,CAAC,IAAI,CAAC,EAAE,EAAE,GAAG,CAAC,CAAC;QAC1D,CAAC,CAAC,CAAC;IACL,CAAC;IAED,KAAK,CAAC,mBAAmB;QACvB,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,UAAU;aAClC,IAAI,CAAC;YACJ,KAAK,EAAE,OAAO,CAAC,aAAa,CAAC,IAAI;SAClC,CAAC;aACD,OAAO,EAAE,CAAC;QAEb,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE;YACtB,OAAO,IAAI,8BAA8B,CAAC,IAAI,CAAC,EAAE,EAAE,GAAG,CAAC,CAAC;QAC1D,CAAC,CAAC,CAAC;IACL,CAAC;IAED,KAAK,CAAC,gBAAgB;QACpB,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,yBAAyB,EAAE,CAAC;QACvD,IAAI,OAAO,IAAI,IAAI,EAAE,CAAC;YACpB,OAAO,IAAI,CAAC;QACd,CAAC;QAED,gDAAgD;QAChD,+FAA+F;QAC/F,wBAAwB;QACxB,IAAI,IAAI,CAAC,kBAAkB,EAAE,QAAQ,IAAI,OAAO,CAAC,EAAE,EAAE,CAAC;YACpD,OAAO,IAAI,CAAC,kBAAkB,CAAC;QACjC,CAAC;aAAM,CAAC;YACN,MAAM,QAAQ,GAAG,IAAI,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC;YAC3C,IAAI,CAAC,kBAAkB,GAAG,QAAQ,CAAC;YACnC,OAAO,QAAQ,CAAC;QAClB,CAAC;IACH,CAAC;IAED,KAAK,CAAC,iBAAiB;QACrB,MAAM,iBAAiB,GAAG,CAAC,CAAU,EAAE,EAAE;YACvC,IAAI,SAAS,CAAC,kBAAkB,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,QAAQ,IAAI,mBAAmB,EAAE,CAAC;gBACzE,sCAAsC;gBACtC,OAAO,CAAC,EAAE,YAAY,EAAE,EAAE,IAAI,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC;YACzC,CAAC;iBAAM,CAAC;gBACN,OAAO,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;YAC3B,CAAC;QACH,CAAC,CAAC;QAEF,MAAM,iBAAiB,GAAG,MAAM,IAAI,CAAC,kBAAkB,CAAC,EAAE,aAAa,EAAE,QAAQ,EAAE,CAAC,CAAC;QACrF,IAAI,iBAAiB,IAAI,IAAI,EAAE,CAAC;YAC9B,OAAO;gBACL,qBAAqB,EAAE,CAAC;gBACxB,qBAAqB,EAAE,CAAC;gBACxB,sBAAsB,EAAE,CAAC;aAC1B,CAAC;QACJ,CAAC;QACD,MAAM,oBAAoB,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,WAAW;aAEnD,SAAS,CAAC;YACT;gBACE,UAAU,EAAE;oBACV,YAAY,EAAE,EAAE;iBACjB;aACF;SACF,CAAC;aACD,OAAO,EAAE;aACT,KAAK,CAAC,iBAAiB,CAAC,CAAC;QAE5B,MAAM,oBAAoB,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,iBAAiB;aACzD,SAAS,CAAC;YACT;gBACE,UAAU,EAAE;oBACV,YAAY,EAAE,EAAE;iBACjB;aACF;SACF,CAAC;aACD,OAAO,EAAE;aACT,KAAK,CAAC,iBAAiB,CAAC,CAAC;QAE5B,MAAM,qBAAqB,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,YAAY;aACrD,SAAS,CAAC;YACT;gBACE,UAAU,EAAE;oBACV,YAAY,EAAE,EAAE;iBACjB;aACF;SACF,CAAC;aACD,OAAO,EAAE;aACT,KAAK,CAAC,iBAAiB,CAAC,CAAC;QAE5B,OAAO;YACL,qBAAqB,EAAE,MAAM,CAAC,oBAAoB,CAAC,CAAC,CAAC,CAAC,YAAY,CAAC,IAAI,CAAC;YACxE,qBAAqB,EAAE,MAAM,CAAC,oBAAoB,CAAC,CAAC,CAAC,CAAC,YAAY,CAAC,IAAI,CAAC;YACxE,sBAAsB,EAAE,MAAM,CAAC,qBAAqB,CAAC,CAAC,CAAC,CAAC,YAAY,CAAC,IAAI,CAAC;SAC3E,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,sBAAsB;QAC1B,IAAI,QAAQ,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,QAAQ,CAAC,OAAO,CAAC;YAC5C,GAAG,EAAE,EAAE,OAAO,EAAE,IAAI,EAAE;SACvB,CAAC,CAAC;QAEH,IAAI,CAAC,QAAQ,EAAE,CAAC;YACd,MAAM,OAAO,GAAG,IAAI,SAAS,CAAC,KAAK,CAAC,gBAAgB,CAAC;gBACnD,UAAU,EAAE,IAAI,CAAC,EAAE,CAAC,KAAK;gBACzB,IAAI,EAAE,4BAA4B;aACnC,CAAC,CAAC;YAEH,MAAM,OAAO,CAAC,IAAI,CAAC,KAAK,IAAI,EAAE;gBAC5B,MAAM,IAAI,CAAC,EAAE,CAAC,QAAQ,CAAC,SAAS,CAAC;oBAC/B,GAAG,EAAE,IAAI,EAAE;iBACZ,CAAC,CAAC;YACL,CAAC,CAAC,CAAC;YAEH,QAAQ,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,QAAQ,CAAC,OAAO,CAAC;gBACxC,GAAG,EAAE,EAAE,OAAO,EAAE,IAAI,EAAE;aACvB,CAAC,CAAC;QACL,CAAC;QAED,OAAO,QAAS,CAAC,GAAG,CAAC;IACvB,CAAC;CACF"}
|
|
@@ -1,34 +1,66 @@
|
|
|
1
1
|
import { ChecksumMap, FetchPartialBucketChecksum, InternalOpId, PartialChecksumMap } from '@powersync/service-core';
|
|
2
2
|
import { PowerSyncMongo } from './db.js';
|
|
3
|
+
/**
|
|
4
|
+
* Checksum calculation options, primarily for tests.
|
|
5
|
+
*/
|
|
6
|
+
export interface MongoChecksumOptions {
|
|
7
|
+
/**
|
|
8
|
+
* How many buckets to process in a batch when calculating checksums.
|
|
9
|
+
*/
|
|
10
|
+
bucketBatchLimit?: number;
|
|
11
|
+
/**
|
|
12
|
+
* Limit on the number of documents to calculate a checksum on at a time.
|
|
13
|
+
*/
|
|
14
|
+
operationBatchLimit?: number;
|
|
15
|
+
}
|
|
3
16
|
/**
|
|
4
17
|
* Checksum query implementation.
|
|
18
|
+
*
|
|
19
|
+
* General implementation flow is:
|
|
20
|
+
* 1. getChecksums() -> check cache for (partial) matches. If not found or partial match, query the remainder using computePartialChecksums().
|
|
21
|
+
* 2. computePartialChecksums() -> query bucket_state for partial matches. Query the remainder using computePartialChecksumsDirect().
|
|
22
|
+
* 3. computePartialChecksumsDirect() -> split into batches of 200 buckets at a time -> computePartialChecksumsInternal()
|
|
23
|
+
* 4. computePartialChecksumsInternal() -> aggregate over 50_000 operations in bucket_data at a time
|
|
5
24
|
*/
|
|
6
25
|
export declare class MongoChecksums {
|
|
7
26
|
private db;
|
|
8
27
|
private group_id;
|
|
28
|
+
private options?;
|
|
9
29
|
private cache;
|
|
10
|
-
constructor(db: PowerSyncMongo, group_id: number);
|
|
30
|
+
constructor(db: PowerSyncMongo, group_id: number, options?: MongoChecksumOptions | undefined);
|
|
11
31
|
/**
|
|
12
|
-
* Calculate checksums, utilizing the cache
|
|
32
|
+
* Calculate checksums, utilizing the cache for partial checkums, and querying the remainder from
|
|
33
|
+
* the database (bucket_state + bucket_data).
|
|
13
34
|
*/
|
|
14
35
|
getChecksums(checkpoint: InternalOpId, buckets: string[]): Promise<ChecksumMap>;
|
|
15
36
|
clearCache(): void;
|
|
16
37
|
/**
|
|
17
|
-
* Calculate (partial) checksums from bucket_state and
|
|
38
|
+
* Calculate (partial) checksums from bucket_state (pre-aggregated) and bucket_data (individual operations).
|
|
18
39
|
*
|
|
19
|
-
* Results are not cached.
|
|
40
|
+
* Results are not cached here. This method is only called by {@link ChecksumCache.getChecksumMap},
|
|
41
|
+
* which is responsible for caching its result.
|
|
42
|
+
*
|
|
43
|
+
* As long as data is compacted regularly, this should be fast. Large buckets without pre-compacted bucket_state
|
|
44
|
+
* can be slow.
|
|
20
45
|
*/
|
|
21
|
-
private
|
|
46
|
+
private computePartialChecksums;
|
|
22
47
|
/**
|
|
23
|
-
* Calculate (partial) checksums from the data collection directly.
|
|
48
|
+
* Calculate (partial) checksums from the data collection directly, bypassing the cache and bucket_state.
|
|
49
|
+
*
|
|
50
|
+
* Can be used directly in cases where the cache should be bypassed, such as from a compact job.
|
|
51
|
+
*
|
|
52
|
+
* Internally, we do calculations in smaller batches of buckets as appropriate.
|
|
53
|
+
*
|
|
54
|
+
* For large buckets, this can be slow, but should not time out as the underlying queries are performed in
|
|
55
|
+
* smaller batches.
|
|
24
56
|
*/
|
|
25
|
-
|
|
26
|
-
private queryPartialChecksumsInternal;
|
|
57
|
+
computePartialChecksumsDirect(batch: FetchPartialBucketChecksum[]): Promise<PartialChecksumMap>;
|
|
27
58
|
/**
|
|
28
|
-
*
|
|
29
|
-
*
|
|
30
|
-
*
|
|
59
|
+
* Query a batch of checksums.
|
|
60
|
+
*
|
|
61
|
+
* We limit the number of operations that the query aggregates in each sub-batch, to avoid potential query timeouts.
|
|
62
|
+
*
|
|
63
|
+
* `batch` must be limited to DEFAULT_BUCKET_BATCH_LIMIT buckets before calling this.
|
|
31
64
|
*/
|
|
32
|
-
|
|
33
|
-
private slowChecksum;
|
|
65
|
+
private computePartialChecksumsInternal;
|
|
34
66
|
}
|
|
@@ -1,23 +1,33 @@
|
|
|
1
|
-
import { addPartialChecksums, bson, ChecksumCache, isPartialChecksum } from '@powersync/service-core';
|
|
2
1
|
import * as lib_mongo from '@powersync/lib-service-mongodb';
|
|
3
|
-
import {
|
|
2
|
+
import { addPartialChecksums, bson, ChecksumCache, isPartialChecksum } from '@powersync/service-core';
|
|
3
|
+
const DEFAULT_BUCKET_BATCH_LIMIT = 200;
|
|
4
|
+
const DEFAULT_OPERATION_BATCH_LIMIT = 50_000;
|
|
4
5
|
/**
|
|
5
6
|
* Checksum query implementation.
|
|
7
|
+
*
|
|
8
|
+
* General implementation flow is:
|
|
9
|
+
* 1. getChecksums() -> check cache for (partial) matches. If not found or partial match, query the remainder using computePartialChecksums().
|
|
10
|
+
* 2. computePartialChecksums() -> query bucket_state for partial matches. Query the remainder using computePartialChecksumsDirect().
|
|
11
|
+
* 3. computePartialChecksumsDirect() -> split into batches of 200 buckets at a time -> computePartialChecksumsInternal()
|
|
12
|
+
* 4. computePartialChecksumsInternal() -> aggregate over 50_000 operations in bucket_data at a time
|
|
6
13
|
*/
|
|
7
14
|
export class MongoChecksums {
|
|
8
15
|
db;
|
|
9
16
|
group_id;
|
|
17
|
+
options;
|
|
10
18
|
cache = new ChecksumCache({
|
|
11
19
|
fetchChecksums: (batch) => {
|
|
12
|
-
return this.
|
|
20
|
+
return this.computePartialChecksums(batch);
|
|
13
21
|
}
|
|
14
22
|
});
|
|
15
|
-
constructor(db, group_id) {
|
|
23
|
+
constructor(db, group_id, options) {
|
|
16
24
|
this.db = db;
|
|
17
25
|
this.group_id = group_id;
|
|
26
|
+
this.options = options;
|
|
18
27
|
}
|
|
19
28
|
/**
|
|
20
|
-
* Calculate checksums, utilizing the cache
|
|
29
|
+
* Calculate checksums, utilizing the cache for partial checkums, and querying the remainder from
|
|
30
|
+
* the database (bucket_state + bucket_data).
|
|
21
31
|
*/
|
|
22
32
|
async getChecksums(checkpoint, buckets) {
|
|
23
33
|
return this.cache.getChecksumMap(checkpoint, buckets);
|
|
@@ -26,11 +36,15 @@ export class MongoChecksums {
|
|
|
26
36
|
this.cache.clear();
|
|
27
37
|
}
|
|
28
38
|
/**
|
|
29
|
-
* Calculate (partial) checksums from bucket_state and
|
|
39
|
+
* Calculate (partial) checksums from bucket_state (pre-aggregated) and bucket_data (individual operations).
|
|
40
|
+
*
|
|
41
|
+
* Results are not cached here. This method is only called by {@link ChecksumCache.getChecksumMap},
|
|
42
|
+
* which is responsible for caching its result.
|
|
30
43
|
*
|
|
31
|
-
*
|
|
44
|
+
* As long as data is compacted regularly, this should be fast. Large buckets without pre-compacted bucket_state
|
|
45
|
+
* can be slow.
|
|
32
46
|
*/
|
|
33
|
-
async
|
|
47
|
+
async computePartialChecksums(batch) {
|
|
34
48
|
if (batch.length == 0) {
|
|
35
49
|
return new Map();
|
|
36
50
|
}
|
|
@@ -79,7 +93,7 @@ export class MongoChecksums {
|
|
|
79
93
|
start
|
|
80
94
|
};
|
|
81
95
|
});
|
|
82
|
-
const queriedChecksums = await this.
|
|
96
|
+
const queriedChecksums = await this.computePartialChecksumsDirect(mappedRequests);
|
|
83
97
|
return new Map(batch.map((request) => {
|
|
84
98
|
const bucket = request.bucket;
|
|
85
99
|
// Could be null if this is either (1) a partial request, or (2) no compacted checksum was available
|
|
@@ -91,54 +105,140 @@ export class MongoChecksums {
|
|
|
91
105
|
}));
|
|
92
106
|
}
|
|
93
107
|
/**
|
|
94
|
-
* Calculate (partial) checksums from the data collection directly.
|
|
108
|
+
* Calculate (partial) checksums from the data collection directly, bypassing the cache and bucket_state.
|
|
109
|
+
*
|
|
110
|
+
* Can be used directly in cases where the cache should be bypassed, such as from a compact job.
|
|
111
|
+
*
|
|
112
|
+
* Internally, we do calculations in smaller batches of buckets as appropriate.
|
|
113
|
+
*
|
|
114
|
+
* For large buckets, this can be slow, but should not time out as the underlying queries are performed in
|
|
115
|
+
* smaller batches.
|
|
95
116
|
*/
|
|
96
|
-
async
|
|
97
|
-
|
|
98
|
-
|
|
117
|
+
async computePartialChecksumsDirect(batch) {
|
|
118
|
+
// Limit the number of buckets we query for at a time.
|
|
119
|
+
const bucketBatchLimit = this.options?.bucketBatchLimit ?? DEFAULT_BUCKET_BATCH_LIMIT;
|
|
120
|
+
if (batch.length < bucketBatchLimit) {
|
|
121
|
+
// Single batch - no need for splitting the batch and merging results
|
|
122
|
+
return await this.computePartialChecksumsInternal(batch);
|
|
99
123
|
}
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
124
|
+
// Split the batch and merge results
|
|
125
|
+
let results = new Map();
|
|
126
|
+
for (let i = 0; i < batch.length; i += bucketBatchLimit) {
|
|
127
|
+
const bucketBatch = batch.slice(i, i + bucketBatchLimit);
|
|
128
|
+
const batchResults = await this.computePartialChecksumsInternal(bucketBatch);
|
|
129
|
+
for (let r of batchResults.values()) {
|
|
130
|
+
results.set(r.bucket, r);
|
|
105
131
|
}
|
|
106
|
-
throw lib_mongo.mapQueryError(e, 'while reading checksums');
|
|
107
132
|
}
|
|
133
|
+
return results;
|
|
108
134
|
}
|
|
109
|
-
|
|
110
|
-
|
|
135
|
+
/**
|
|
136
|
+
* Query a batch of checksums.
|
|
137
|
+
*
|
|
138
|
+
* We limit the number of operations that the query aggregates in each sub-batch, to avoid potential query timeouts.
|
|
139
|
+
*
|
|
140
|
+
* `batch` must be limited to DEFAULT_BUCKET_BATCH_LIMIT buckets before calling this.
|
|
141
|
+
*/
|
|
142
|
+
async computePartialChecksumsInternal(batch) {
|
|
143
|
+
const batchLimit = this.options?.operationBatchLimit ?? DEFAULT_OPERATION_BATCH_LIMIT;
|
|
144
|
+
// Map requests by bucket. We adjust this as we get partial results.
|
|
145
|
+
let requests = new Map();
|
|
111
146
|
for (let request of batch) {
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
147
|
+
requests.set(request.bucket, request);
|
|
148
|
+
}
|
|
149
|
+
const partialChecksums = new Map();
|
|
150
|
+
while (requests.size > 0) {
|
|
151
|
+
const filters = [];
|
|
152
|
+
for (let request of requests.values()) {
|
|
153
|
+
filters.push({
|
|
154
|
+
_id: {
|
|
155
|
+
$gt: {
|
|
156
|
+
g: this.group_id,
|
|
157
|
+
b: request.bucket,
|
|
158
|
+
o: request.start ?? new bson.MinKey()
|
|
159
|
+
},
|
|
160
|
+
$lte: {
|
|
161
|
+
g: this.group_id,
|
|
162
|
+
b: request.bucket,
|
|
163
|
+
o: request.end
|
|
164
|
+
}
|
|
165
|
+
}
|
|
166
|
+
});
|
|
167
|
+
}
|
|
168
|
+
// Aggregate over a max of `batchLimit` operations at a time.
|
|
169
|
+
// Let's say we have 3 buckets (A, B, C), each with 10 operations, and our batch limit is 12.
|
|
170
|
+
// Then we'll do three batches:
|
|
171
|
+
// 1. Query: A[1-end], B[1-end], C[1-end]
|
|
172
|
+
// Returns: A[1-10], B[1-2]
|
|
173
|
+
// 2. Query: B[3-end], C[1-end]
|
|
174
|
+
// Returns: B[3-10], C[1-4]
|
|
175
|
+
// 3. Query: C[5-end]
|
|
176
|
+
// Returns: C[5-10]
|
|
177
|
+
const aggregate = await this.db.bucket_data
|
|
178
|
+
.aggregate([
|
|
179
|
+
{
|
|
180
|
+
$match: {
|
|
181
|
+
$or: filters
|
|
182
|
+
}
|
|
183
|
+
},
|
|
184
|
+
// sort and limit _before_ grouping
|
|
185
|
+
{ $sort: { _id: 1 } },
|
|
186
|
+
{ $limit: batchLimit },
|
|
187
|
+
{
|
|
188
|
+
$group: {
|
|
189
|
+
_id: '$_id.b',
|
|
190
|
+
// Historically, checksum may be stored as 'int' or 'double'.
|
|
191
|
+
// More recently, this should be a 'long'.
|
|
192
|
+
// $toLong ensures that we always sum it as a long, avoiding inaccuracies in the calculations.
|
|
193
|
+
checksum_total: { $sum: { $toLong: '$checksum' } },
|
|
194
|
+
count: { $sum: 1 },
|
|
195
|
+
has_clear_op: {
|
|
196
|
+
$max: {
|
|
197
|
+
$cond: [{ $eq: ['$op', 'CLEAR'] }, 1, 0]
|
|
198
|
+
}
|
|
199
|
+
},
|
|
200
|
+
last_op: { $max: '$_id.o' }
|
|
123
201
|
}
|
|
124
202
|
}
|
|
203
|
+
], { session: undefined, readConcern: 'snapshot', maxTimeMS: lib_mongo.MONGO_CHECKSUM_TIMEOUT_MS })
|
|
204
|
+
.toArray()
|
|
205
|
+
.catch((e) => {
|
|
206
|
+
throw lib_mongo.mapQueryError(e, 'while reading checksums');
|
|
125
207
|
});
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
208
|
+
let batchCount = 0;
|
|
209
|
+
let limitReached = false;
|
|
210
|
+
for (let doc of aggregate) {
|
|
211
|
+
const bucket = doc._id;
|
|
212
|
+
const checksum = checksumFromAggregate(doc);
|
|
213
|
+
const existing = partialChecksums.get(bucket);
|
|
214
|
+
if (existing != null) {
|
|
215
|
+
partialChecksums.set(bucket, addPartialChecksums(bucket, existing, checksum));
|
|
132
216
|
}
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
217
|
+
else {
|
|
218
|
+
partialChecksums.set(bucket, checksum);
|
|
219
|
+
}
|
|
220
|
+
batchCount += doc.count;
|
|
221
|
+
if (batchCount == batchLimit) {
|
|
222
|
+
// Limit reached. Request more in the next batch.
|
|
223
|
+
// Note that this only affects the _last_ bucket in a batch.
|
|
224
|
+
limitReached = true;
|
|
225
|
+
const req = requests.get(bucket);
|
|
226
|
+
requests.set(bucket, {
|
|
227
|
+
bucket,
|
|
228
|
+
start: doc.last_op,
|
|
229
|
+
end: req.end
|
|
230
|
+
});
|
|
231
|
+
}
|
|
232
|
+
else {
|
|
233
|
+
// All done for this bucket
|
|
234
|
+
requests.delete(bucket);
|
|
235
|
+
}
|
|
236
|
+
batchCount++;
|
|
237
|
+
}
|
|
238
|
+
if (!limitReached) {
|
|
239
|
+
break;
|
|
240
|
+
}
|
|
241
|
+
}
|
|
142
242
|
return new Map(batch.map((request) => {
|
|
143
243
|
const bucket = request.bucket;
|
|
144
244
|
// Could be null if we got no data
|
|
@@ -160,96 +260,9 @@ export class MongoChecksums {
|
|
|
160
260
|
return [bucket, partialChecksum];
|
|
161
261
|
}));
|
|
162
262
|
}
|
|
163
|
-
/**
|
|
164
|
-
* Checksums for large buckets can run over the query timeout.
|
|
165
|
-
* To avoid this, we query in batches.
|
|
166
|
-
* This version can handle larger amounts of data, but is slower, especially for many buckets.
|
|
167
|
-
*/
|
|
168
|
-
async queryPartialChecksumsFallback(batch) {
|
|
169
|
-
const partialChecksums = new Map();
|
|
170
|
-
for (let request of batch) {
|
|
171
|
-
const checksum = await this.slowChecksum(request);
|
|
172
|
-
partialChecksums.set(request.bucket, checksum);
|
|
173
|
-
}
|
|
174
|
-
return partialChecksums;
|
|
175
|
-
}
|
|
176
|
-
async slowChecksum(request) {
|
|
177
|
-
const batchLimit = 50_000;
|
|
178
|
-
let lowerBound = 0n;
|
|
179
|
-
const bucket = request.bucket;
|
|
180
|
-
let runningChecksum = {
|
|
181
|
-
bucket,
|
|
182
|
-
partialCount: 0,
|
|
183
|
-
partialChecksum: 0
|
|
184
|
-
};
|
|
185
|
-
if (request.start == null) {
|
|
186
|
-
runningChecksum = {
|
|
187
|
-
bucket,
|
|
188
|
-
count: 0,
|
|
189
|
-
checksum: 0
|
|
190
|
-
};
|
|
191
|
-
}
|
|
192
|
-
while (true) {
|
|
193
|
-
const filter = {
|
|
194
|
-
_id: {
|
|
195
|
-
$gt: {
|
|
196
|
-
g: this.group_id,
|
|
197
|
-
b: bucket,
|
|
198
|
-
o: lowerBound
|
|
199
|
-
},
|
|
200
|
-
$lte: {
|
|
201
|
-
g: this.group_id,
|
|
202
|
-
b: bucket,
|
|
203
|
-
o: request.end
|
|
204
|
-
}
|
|
205
|
-
}
|
|
206
|
-
};
|
|
207
|
-
const docs = await this.db.bucket_data
|
|
208
|
-
.aggregate([
|
|
209
|
-
{
|
|
210
|
-
$match: filter
|
|
211
|
-
},
|
|
212
|
-
// sort and limit _before_ grouping
|
|
213
|
-
{ $sort: { _id: 1 } },
|
|
214
|
-
{ $limit: batchLimit },
|
|
215
|
-
CHECKSUM_QUERY_GROUP_STAGE
|
|
216
|
-
], { session: undefined, readConcern: 'snapshot', maxTimeMS: lib_mongo.MONGO_CHECKSUM_TIMEOUT_MS })
|
|
217
|
-
.toArray();
|
|
218
|
-
const doc = docs[0];
|
|
219
|
-
if (doc == null) {
|
|
220
|
-
return runningChecksum;
|
|
221
|
-
}
|
|
222
|
-
const partial = checksumFromAggregate(doc);
|
|
223
|
-
runningChecksum = addPartialChecksums(bucket, runningChecksum, partial);
|
|
224
|
-
const isFinal = doc.count != batchLimit;
|
|
225
|
-
if (isFinal) {
|
|
226
|
-
break;
|
|
227
|
-
}
|
|
228
|
-
else {
|
|
229
|
-
lowerBound = doc.last_op;
|
|
230
|
-
}
|
|
231
|
-
}
|
|
232
|
-
return runningChecksum;
|
|
233
|
-
}
|
|
234
263
|
}
|
|
235
|
-
const CHECKSUM_QUERY_GROUP_STAGE = {
|
|
236
|
-
$group: {
|
|
237
|
-
_id: '$_id.b',
|
|
238
|
-
// Historically, checksum may be stored as 'int' or 'double'.
|
|
239
|
-
// More recently, this should be a 'long'.
|
|
240
|
-
// $toLong ensures that we always sum it as a long, avoiding inaccuracies in the calculations.
|
|
241
|
-
checksum_total: { $sum: { $toLong: '$checksum' } },
|
|
242
|
-
count: { $sum: 1 },
|
|
243
|
-
has_clear_op: {
|
|
244
|
-
$max: {
|
|
245
|
-
$cond: [{ $eq: ['$op', 'CLEAR'] }, 1, 0]
|
|
246
|
-
}
|
|
247
|
-
},
|
|
248
|
-
last_op: { $max: '$_id.o' }
|
|
249
|
-
}
|
|
250
|
-
};
|
|
251
264
|
/**
|
|
252
|
-
* Convert output of
|
|
265
|
+
* Convert output of the $group stage into a checksum.
|
|
253
266
|
*/
|
|
254
267
|
function checksumFromAggregate(doc) {
|
|
255
268
|
const partialChecksum = Number(BigInt(doc.checksum_total) & 0xffffffffn) & 0xffffffff;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"MongoChecksums.js","sourceRoot":"","sources":["../../../src/storage/implementation/MongoChecksums.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,mBAAmB,EACnB,IAAI,EAEJ,aAAa,EAIb,iBAAiB,EAIlB,MAAM,yBAAyB,CAAC;
|
|
1
|
+
{"version":3,"file":"MongoChecksums.js","sourceRoot":"","sources":["../../../src/storage/implementation/MongoChecksums.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,SAAS,MAAM,gCAAgC,CAAC;AAC5D,OAAO,EACL,mBAAmB,EACnB,IAAI,EAEJ,aAAa,EAIb,iBAAiB,EAIlB,MAAM,yBAAyB,CAAC;AAkBjC,MAAM,0BAA0B,GAAG,GAAG,CAAC;AACvC,MAAM,6BAA6B,GAAG,MAAM,CAAC;AAE7C;;;;;;;;GAQG;AACH,MAAM,OAAO,cAAc;IAQf;IACA;IACA;IATF,KAAK,GAAG,IAAI,aAAa,CAAC;QAChC,cAAc,EAAE,CAAC,KAAK,EAAE,EAAE;YACxB,OAAO,IAAI,CAAC,uBAAuB,CAAC,KAAK,CAAC,CAAC;QAC7C,CAAC;KACF,CAAC,CAAC;IAEH,YACU,EAAkB,EAClB,QAAgB,EAChB,OAA8B;QAF9B,OAAE,GAAF,EAAE,CAAgB;QAClB,aAAQ,GAAR,QAAQ,CAAQ;QAChB,YAAO,GAAP,OAAO,CAAuB;IACrC,CAAC;IAEJ;;;OAGG;IACH,KAAK,CAAC,YAAY,CAAC,UAAwB,EAAE,OAAiB;QAC5D,OAAO,IAAI,CAAC,KAAK,CAAC,cAAc,CAAC,UAAU,EAAE,OAAO,CAAC,CAAC;IACxD,CAAC;IAED,UAAU;QACR,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC;IACrB,CAAC;IAED;;;;;;;;OAQG;IACK,KAAK,CAAC,uBAAuB,CAAC,KAAmC;QACvE,IAAI,KAAK,CAAC,MAAM,IAAI,CAAC,EAAE,CAAC;YACtB,OAAO,IAAI,GAAG,EAAE,CAAC;QACnB,CAAC;QAED,MAAM,UAAU,GAAU,EAAE,CAAC;QAC7B,KAAK,IAAI,OAAO,IAAI,KAAK,EAAE,CAAC;YAC1B,IAAI,OAAO,CAAC,KAAK,IAAI,IAAI,EAAE,CAAC;gBAC1B,UAAU,CAAC,IAAI,CAAC;oBACd,GAAG,EAAE;wBACH,CAAC,EAAE,IAAI,CAAC,QAAQ;wBAChB,CAAC,EAAE,OAAO,CAAC,MAAM;qBAClB;oBACD,uBAAuB,EAAE,EAAE,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,OAAO,CAAC,GAAG,EAAE;iBAC9D,CAAC,CAAC;YACL,CAAC;QACH,CAAC;QAED,MAAM,SAAS,GAAG,IAAI,GAAG,EAA4D,CAAC;QAEtF,IAAI,UAAU,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC1B,4EAA4E;YAC5E,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,YAAY;iBACtC,IAAI,CAAC;gBACJ,GAAG,EAAE,UAAU;aAChB,CAAC;iBACD,OAAO,EAAE,CAAC;YACb,KAAK,IAAI,KAAK,IAAI,MAAM,EAAE,CAAC;gBACzB,MAAM,cAAc,GAAG,KAAK,CAAC,eAAgB,CAAC;gBAC9C,SAAS,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,EAAE;oBACzB,IAAI,EAAE,cAAc,CAAC,KAAK;oBAC1B,QAAQ,EAAE;wBACR,MAAM,EAAE,KAAK,CAAC,GAAG,CAAC,CAAC;wBACnB,QAAQ,EAAE,MAAM,CAAC,cAAc,CAAC,QAAQ,CAAC;wBACzC,KAAK,EAAE,cAAc,CAAC,KAAK;qBAC5B;iBACF,CAAC,CAAC;YACL,CAAC;QACH,CAAC;QAED,MAAM,cAAc,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE;YAC3C,IAAI,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC;YAC1B,IAAI,KAAK,IAAI,IAAI,EAAE,CAAC;gBAClB,MAAM,QAAQ,GAAG,SAAS,CAAC,GAAG,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;gBAC/C,IAAI,QAAQ,IAAI,IAAI,EAAE,CAAC;oBACrB,KAAK,GAAG,QAAQ,CAAC,IAAI,CAAC;gBACxB,CAAC;YACH,CAAC;YACD,OAAO;gBACL,GAAG,OAAO;gBACV,KAAK;aACN,CAAC;QACJ,CAAC,CAAC,CAAC;QAEH,MAAM,gBAAgB,GAAG,MAAM,IAAI,CAAC,6BAA6B,CAAC,cAAc,CAAC,CAAC;QAElF,OAAO,IAAI,GAAG,CACZ,KAAK,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE;YACpB,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;YAC9B,oGAAoG;YACpG,MAAM,QAAQ,GAAG,SAAS,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;YACvC,kCAAkC;YAClC,MAAM,eAAe,GAAG,gBAAgB,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;YACrD,MAAM,MAAM,GAAG,mBAAmB,CAAC,MAAM,EAAE,QAAQ,EAAE,QAAQ,IAAI,IAAI,EAAE,eAAe,IAAI,IAAI,CAAC,CAAC;YAEhG,OAAO,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;QAC1B,CAAC,CAAC,CACH,CAAC;IACJ,CAAC;IAED;;;;;;;;;OASG;IACI,KAAK,CAAC,6BAA6B,CAAC,KAAmC;QAC5E,sDAAsD;QACtD,MAAM,gBAAgB,GAAG,IAAI,CAAC,OAAO,EAAE,gBAAgB,IAAI,0BAA0B,CAAC;QAEtF,IAAI,KAAK,CAAC,MAAM,GAAG,gBAAgB,EAAE,CAAC;YACpC,qEAAqE;YACrE,OAAO,MAAM,IAAI,CAAC,+BAA+B,CAAC,KAAK,CAAC,CAAC;QAC3D,CAAC;QACD,oCAAoC;QACpC,IAAI,OAAO,GAAG,IAAI,GAAG,EAAiC,CAAC;QACvD,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,IAAI,gBAAgB,EAAE,CAAC;YACxD,MAAM,WAAW,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,gBAAgB,CAAC,CAAC;YACzD,MAAM,YAAY,GAAG,MAAM,IAAI,CAAC,+BAA+B,CAAC,WAAW,CAAC,CAAC;YAC7E,KAAK,IAAI,CAAC,IAAI,YAAY,CAAC,MAAM,EAAE,EAAE,CAAC;gBACpC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;YAC3B,CAAC;QACH,CAAC;QACD,OAAO,OAAO,CAAC;IACjB,CAAC;IAED;;;;;;OAMG;IACK,KAAK,CAAC,+BAA+B,CAAC,KAAmC;QAC/E,MAAM,UAAU,GAAG,IAAI,CAAC,OAAO,EAAE,mBAAmB,IAAI,6BAA6B,CAAC;QAEtF,oEAAoE;QACpE,IAAI,QAAQ,GAAG,IAAI,GAAG,EAAsC,CAAC;QAC7D,KAAK,IAAI,OAAO,IAAI,KAAK,EAAE,CAAC;YAC1B,QAAQ,CAAC,GAAG,CAAC,OAAO,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;QACxC,CAAC;QAED,MAAM,gBAAgB,GAAG,IAAI,GAAG,EAAiC,CAAC;QAElE,OAAO,QAAQ,CAAC,IAAI,GAAG,CAAC,EAAE,CAAC;YACzB,MAAM,OAAO,GAAU,EAAE,CAAC;YAC1B,KAAK,IAAI,OAAO,IAAI,QAAQ,CAAC,MAAM,EAAE,EAAE,CAAC;gBACtC,OAAO,CAAC,IAAI,CAAC;oBACX,GAAG,EAAE;wBACH,GAAG,EAAE;4BACH,CAAC,EAAE,IAAI,CAAC,QAAQ;4BAChB,CAAC,EAAE,OAAO,CAAC,MAAM;4BACjB,CAAC,EAAE,OAAO,CAAC,KAAK,IAAI,IAAI,IAAI,CAAC,MAAM,EAAE;yBACtC;wBACD,IAAI,EAAE;4BACJ,CAAC,EAAE,IAAI,CAAC,QAAQ;4BAChB,CAAC,EAAE,OAAO,CAAC,MAAM;4BACjB,CAAC,EAAE,OAAO,CAAC,GAAG;yBACf;qBACF;iBACF,CAAC,CAAC;YACL,CAAC;YAED,6DAA6D;YAC7D,6FAA6F;YAC7F,+BAA+B;YAC/B,yCAAyC;YACzC,8BAA8B;YAC9B,+BAA+B;YAC/B,8BAA8B;YAC9B,qBAAqB;YACrB,sBAAsB;YACtB,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,WAAW;iBACxC,SAAS,CACR;gBACE;oBACE,MAAM,EAAE;wBACN,GAAG,EAAE,OAAO;qBACb;iBACF;gBACD,mCAAmC;gBACnC,EAAE,KAAK,EAAE,EAAE,GAAG,EAAE,CAAC,EAAE,EAAE;gBACrB,EAAE,MAAM,EAAE,UAAU,EAAE;gBACtB;oBACE,MAAM,EAAE;wBACN,GAAG,EAAE,QAAQ;wBACb,6DAA6D;wBAC7D,0CAA0C;wBAC1C,8FAA8F;wBAC9F,cAAc,EAAE,EAAE,IAAI,EAAE,EAAE,OAAO,EAAE,WAAW,EAAE,EAAE;wBAClD,KAAK,EAAE,EAAE,IAAI,EAAE,CAAC,EAAE;wBAClB,YAAY,EAAE;4BACZ,IAAI,EAAE;gCACJ,KAAK,EAAE,CAAC,EAAE,GAAG,EAAE,CAAC,KAAK,EAAE,OAAO,CAAC,EAAE,EAAE,CAAC,EAAE,CAAC,CAAC;6BACzC;yBACF;wBACD,OAAO,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;qBAC5B;iBACF;aACF,EACD,EAAE,OAAO,EAAE,SAAS,EAAE,WAAW,EAAE,UAAU,EAAE,SAAS,EAAE,SAAS,CAAC,yBAAyB,EAAE,CAChG;iBACA,OAAO,EAAE;iBACT,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE;gBACX,MAAM,SAAS,CAAC,aAAa,CAAC,CAAC,EAAE,yBAAyB,CAAC,CAAC;YAC9D,CAAC,CAAC,CAAC;YAEL,IAAI,UAAU,GAAG,CAAC,CAAC;YACnB,IAAI,YAAY,GAAG,KAAK,CAAC;YACzB,KAAK,IAAI,GAAG,IAAI,SAAS,EAAE,CAAC;gBAC1B,MAAM,MAAM,GAAG,GAAG,CAAC,GAAG,CAAC;gBACvB,MAAM,QAAQ,GAAG,qBAAqB,CAAC,GAAG,CAAC,CAAC;gBAE5C,MAAM,QAAQ,GAAG,gBAAgB,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;gBAC9C,IAAI,QAAQ,IAAI,IAAI,EAAE,CAAC;oBACrB,gBAAgB,CAAC,GAAG,CAAC,MAAM,EAAE,mBAAmB,CAAC,MAAM,EAAE,QAAQ,EAAE,QAAQ,CAAC,CAAC,CAAC;gBAChF,CAAC;qBAAM,CAAC;oBACN,gBAAgB,CAAC,GAAG,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;gBACzC,CAAC;gBAED,UAAU,IAAI,GAAG,CAAC,KAAK,CAAC;gBACxB,IAAI,UAAU,IAAI,UAAU,EAAE,CAAC;oBAC7B,iDAAiD;oBACjD,4DAA4D;oBAC5D,YAAY,GAAG,IAAI,CAAC;oBACpB,MAAM,GAAG,GAAG,QAAQ,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;oBACjC,QAAQ,CAAC,GAAG,CAAC,MAAM,EAAE;wBACnB,MAAM;wBACN,KAAK,EAAE,GAAG,CAAC,OAAO;wBAClB,GAAG,EAAE,GAAI,CAAC,GAAG;qBACd,CAAC,CAAC;gBACL,CAAC;qBAAM,CAAC;oBACN,2BAA2B;oBAC3B,QAAQ,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;gBAC1B,CAAC;gBACD,UAAU,EAAE,CAAC;YACf,CAAC;YACD,IAAI,CAAC,YAAY,EAAE,CAAC;gBAClB,MAAM;YACR,CAAC;QACH,CAAC;QAED,OAAO,IAAI,GAAG,CACZ,KAAK,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE;YACpB,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;YAC9B,kCAAkC;YAClC,IAAI,eAAe,GAAG,gBAAgB,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;YACnD,IAAI,eAAe,IAAI,IAAI,EAAE,CAAC;gBAC5B,eAAe,GAAG;oBAChB,MAAM;oBACN,YAAY,EAAE,CAAC;oBACf,eAAe,EAAE,CAAC;iBACnB,CAAC;YACJ,CAAC;YACD,IAAI,OAAO,CAAC,KAAK,IAAI,IAAI,IAAI,iBAAiB,CAAC,eAAe,CAAC,EAAE,CAAC;gBAChE,eAAe,GAAG;oBAChB,MAAM;oBACN,KAAK,EAAE,eAAe,CAAC,YAAY;oBACnC,QAAQ,EAAE,eAAe,CAAC,eAAe;iBAC1C,CAAC;YACJ,CAAC;YAED,OAAO,CAAC,MAAM,EAAE,eAAe,CAAC,CAAC;QACnC,CAAC,CAAC,CACH,CAAC;IACJ,CAAC;CACF;AAED;;GAEG;AACH,SAAS,qBAAqB,CAAC,GAAkB;IAC/C,MAAM,eAAe,GAAG,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,cAAc,CAAC,GAAG,WAAW,CAAC,GAAG,UAAU,CAAC;IACtF,MAAM,MAAM,GAAG,GAAG,CAAC,GAAG,CAAC;IAEvB,IAAI,GAAG,CAAC,YAAY,IAAI,CAAC,EAAE,CAAC;QAC1B,OAAO;YACL,4CAA4C;YAC5C,MAAM;YACN,QAAQ,EAAE,eAAe;YACzB,KAAK,EAAE,GAAG,CAAC,KAAK;SACQ,CAAC;IAC7B,CAAC;SAAM,CAAC;QACN,OAAO;YACL,gDAAgD;YAChD,MAAM;YACN,YAAY,EAAE,GAAG,CAAC,KAAK;YACvB,eAAe;SACU,CAAC;IAC9B,CAAC;AACH,CAAC"}
|