@powersync/service-module-mongodb-storage 0.12.10 → 0.12.12
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +24 -0
- package/dist/storage/implementation/MongoChecksums.d.ts +7 -1
- package/dist/storage/implementation/MongoChecksums.js +15 -6
- package/dist/storage/implementation/MongoChecksums.js.map +1 -1
- package/dist/storage/implementation/MongoCompactor.d.ts +1 -0
- package/dist/storage/implementation/MongoCompactor.js +40 -10
- package/dist/storage/implementation/MongoCompactor.js.map +1 -1
- package/dist/storage/implementation/MongoSyncRulesLock.js +8 -1
- package/dist/storage/implementation/MongoSyncRulesLock.js.map +1 -1
- package/dist/storage/implementation/models.d.ts +4 -0
- package/package.json +6 -6
- package/src/storage/implementation/MongoChecksums.ts +16 -6
- package/src/storage/implementation/MongoCompactor.ts +45 -11
- package/src/storage/implementation/MongoSyncRulesLock.ts +13 -4
- package/src/storage/implementation/models.ts +5 -0
- package/test/src/storage_compacting.test.ts +1 -0
- package/tsconfig.tsbuildinfo +1 -1
package/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,29 @@
|
|
|
1
1
|
# @powersync/service-module-mongodb-storage
|
|
2
2
|
|
|
3
|
+
## 0.12.12
|
|
4
|
+
|
|
5
|
+
### Patch Changes
|
|
6
|
+
|
|
7
|
+
- Updated dependencies [fff0024]
|
|
8
|
+
- @powersync/service-sync-rules@0.29.7
|
|
9
|
+
- @powersync/service-core@1.16.3
|
|
10
|
+
- @powersync/lib-services-framework@0.7.10
|
|
11
|
+
- @powersync/lib-service-mongodb@0.6.12
|
|
12
|
+
|
|
13
|
+
## 0.12.11
|
|
14
|
+
|
|
15
|
+
### Patch Changes
|
|
16
|
+
|
|
17
|
+
- b364581: Avoid re-compacting recently compacted buckets.
|
|
18
|
+
- d889219: Fix memory leaks when retrying replication after errors.
|
|
19
|
+
- b364581: Reduce batch size for checksum pre-calculations to reduce timeouts.
|
|
20
|
+
- Updated dependencies [b364581]
|
|
21
|
+
- Updated dependencies [d889219]
|
|
22
|
+
- Updated dependencies [0ace0d3]
|
|
23
|
+
- Updated dependencies [7eb7957]
|
|
24
|
+
- Updated dependencies [b364581]
|
|
25
|
+
- @powersync/service-core@1.16.2
|
|
26
|
+
|
|
3
27
|
## 0.12.10
|
|
4
28
|
|
|
5
29
|
### Patch Changes
|
|
@@ -26,8 +26,14 @@ export declare class MongoChecksums {
|
|
|
26
26
|
private db;
|
|
27
27
|
private group_id;
|
|
28
28
|
private options?;
|
|
29
|
-
private
|
|
29
|
+
private _cache;
|
|
30
30
|
constructor(db: PowerSyncMongo, group_id: number, options?: MongoChecksumOptions | undefined);
|
|
31
|
+
/**
|
|
32
|
+
* Lazy-instantiated cache.
|
|
33
|
+
*
|
|
34
|
+
* This means the cache only allocates memory once it is used for the first time.
|
|
35
|
+
*/
|
|
36
|
+
private get cache();
|
|
31
37
|
/**
|
|
32
38
|
* Calculate checksums, utilizing the cache for partial checkums, and querying the remainder from
|
|
33
39
|
* the database (bucket_state + bucket_data).
|
|
@@ -15,16 +15,25 @@ export class MongoChecksums {
|
|
|
15
15
|
db;
|
|
16
16
|
group_id;
|
|
17
17
|
options;
|
|
18
|
-
|
|
19
|
-
fetchChecksums: (batch) => {
|
|
20
|
-
return this.computePartialChecksums(batch);
|
|
21
|
-
}
|
|
22
|
-
});
|
|
18
|
+
_cache;
|
|
23
19
|
constructor(db, group_id, options) {
|
|
24
20
|
this.db = db;
|
|
25
21
|
this.group_id = group_id;
|
|
26
22
|
this.options = options;
|
|
27
23
|
}
|
|
24
|
+
/**
|
|
25
|
+
* Lazy-instantiated cache.
|
|
26
|
+
*
|
|
27
|
+
* This means the cache only allocates memory once it is used for the first time.
|
|
28
|
+
*/
|
|
29
|
+
get cache() {
|
|
30
|
+
this._cache ??= new ChecksumCache({
|
|
31
|
+
fetchChecksums: (batch) => {
|
|
32
|
+
return this.computePartialChecksums(batch);
|
|
33
|
+
}
|
|
34
|
+
});
|
|
35
|
+
return this._cache;
|
|
36
|
+
}
|
|
28
37
|
/**
|
|
29
38
|
* Calculate checksums, utilizing the cache for partial checkums, and querying the remainder from
|
|
30
39
|
* the database (bucket_state + bucket_data).
|
|
@@ -117,7 +126,7 @@ export class MongoChecksums {
|
|
|
117
126
|
async computePartialChecksumsDirect(batch) {
|
|
118
127
|
// Limit the number of buckets we query for at a time.
|
|
119
128
|
const bucketBatchLimit = this.options?.bucketBatchLimit ?? DEFAULT_BUCKET_BATCH_LIMIT;
|
|
120
|
-
if (batch.length
|
|
129
|
+
if (batch.length <= bucketBatchLimit) {
|
|
121
130
|
// Single batch - no need for splitting the batch and merging results
|
|
122
131
|
return await this.computePartialChecksumsInternal(batch);
|
|
123
132
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"MongoChecksums.js","sourceRoot":"","sources":["../../../src/storage/implementation/MongoChecksums.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,SAAS,MAAM,gCAAgC,CAAC;AAC5D,OAAO,EACL,mBAAmB,EACnB,IAAI,EAEJ,aAAa,EAIb,iBAAiB,EAIlB,MAAM,yBAAyB,CAAC;AAkBjC,MAAM,0BAA0B,GAAG,GAAG,CAAC;AACvC,MAAM,6BAA6B,GAAG,MAAM,CAAC;AAE7C;;;;;;;;GAQG;AACH,MAAM,OAAO,cAAc;
|
|
1
|
+
{"version":3,"file":"MongoChecksums.js","sourceRoot":"","sources":["../../../src/storage/implementation/MongoChecksums.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,SAAS,MAAM,gCAAgC,CAAC;AAC5D,OAAO,EACL,mBAAmB,EACnB,IAAI,EAEJ,aAAa,EAIb,iBAAiB,EAIlB,MAAM,yBAAyB,CAAC;AAkBjC,MAAM,0BAA0B,GAAG,GAAG,CAAC;AACvC,MAAM,6BAA6B,GAAG,MAAM,CAAC;AAE7C;;;;;;;;GAQG;AACH,MAAM,OAAO,cAAc;IAIf;IACA;IACA;IALF,MAAM,CAA4B;IAE1C,YACU,EAAkB,EAClB,QAAgB,EAChB,OAA8B;QAF9B,OAAE,GAAF,EAAE,CAAgB;QAClB,aAAQ,GAAR,QAAQ,CAAQ;QAChB,YAAO,GAAP,OAAO,CAAuB;IACrC,CAAC;IAEJ;;;;OAIG;IACH,IAAY,KAAK;QACf,IAAI,CAAC,MAAM,KAAK,IAAI,aAAa,CAAC;YAChC,cAAc,EAAE,CAAC,KAAK,EAAE,EAAE;gBACxB,OAAO,IAAI,CAAC,uBAAuB,CAAC,KAAK,CAAC,CAAC;YAC7C,CAAC;SACF,CAAC,CAAC;QACH,OAAO,IAAI,CAAC,MAAM,CAAC;IACrB,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,YAAY,CAAC,UAAwB,EAAE,OAAiB;QAC5D,OAAO,IAAI,CAAC,KAAK,CAAC,cAAc,CAAC,UAAU,EAAE,OAAO,CAAC,CAAC;IACxD,CAAC;IAED,UAAU;QACR,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC;IACrB,CAAC;IAED;;;;;;;;OAQG;IACK,KAAK,CAAC,uBAAuB,CAAC,KAAmC;QACvE,IAAI,KAAK,CAAC,MAAM,IAAI,CAAC,EAAE,CAAC;YACtB,OAAO,IAAI,GAAG,EAAE,CAAC;QACnB,CAAC;QAED,MAAM,UAAU,GAAU,EAAE,CAAC;QAC7B,KAAK,IAAI,OAAO,IAAI,KAAK,EAAE,CAAC;YAC1B,IAAI,OAAO,CAAC,KAAK,IAAI,IAAI,EAAE,CAAC;gBAC1B,UAAU,CAAC,IAAI,CAAC;oBACd,GAAG,EAAE;wBACH,CAAC,EAAE,IAAI,CAAC,QAAQ;wBAChB,CAAC,EAAE,OAAO,CAAC,MAAM;qBAClB;oBACD,uBAAuB,EAAE,EAAE,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,OAAO,CAAC,GAAG,EAAE;iBAC9D,CAAC,CAAC;YACL,CAAC;QACH,CAAC;QAED,MAAM,SAAS,GAAG,IAAI,GAAG,EAA4D,CAAC;QAEtF,IAAI,UAAU,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC1B,4EAA4E;YAC5E,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,YAAY;iBACtC,IAAI,CAAC;gBACJ,GAAG,EAAE,UAAU;aAChB,CAAC;iBACD,OAAO,EAAE,CAAC;YACb,KAAK,IAAI,KAAK,IAAI,MAAM,EAAE,CAAC;gBACzB,MAAM,cAAc,GAAG,KAAK,CAAC,eAAgB,CAAC;gBAC9C,SAAS,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,EAAE;oBACzB,IAAI,EAAE,cAAc,CAAC,KAAK;oBAC1B,QAAQ,EAAE;wBACR,MAAM,EAAE,KAAK,CAAC,GAAG,CAAC,CAAC;wBACnB,QAAQ,EAAE,MAAM,CAAC,cAAc,CAAC,QAAQ,CAAC;wBACzC,KAAK,EAAE,cAAc,CAAC,KAAK;qBAC5B;iBACF,CAAC,CAAC;YACL,CAAC;QACH,CAAC;QAED,MAAM,cAAc,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE;YAC3C,IAAI,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC;YAC1B,IAAI,KAAK,IAAI,IAAI,EAAE,CAAC;gBAClB,MAAM,QAAQ,GAAG,SAAS,CAAC,GAAG,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;gBAC/C,IAAI,QAAQ,IAAI,IAAI,EAAE,CAAC;oBACrB,KAAK,GAAG,QAAQ,CAAC,IAAI,CAAC;gBACxB,CAAC;YACH,CAAC;YACD,OAAO;gBACL,GAAG,OAAO;gBACV,KAAK;aACN,CAAC;QACJ,CAAC,CAAC,CAAC;QAEH,MAAM,gBAAgB,GAAG,MAAM,IAAI,CAAC,6BAA6B,CAAC,cAAc,CAAC,CAAC;QAElF,OAAO,IAAI,GAAG,CACZ,KAAK,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE;YACpB,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;YAC9B,oGAAoG;YACpG,MAAM,QAAQ,GAAG,SAAS,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;YACvC,kCAAkC;YAClC,MAAM,eAAe,GAAG,gBAAgB,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;YACrD,MAAM,MAAM,GAAG,mBAAmB,CAAC,MAAM,EAAE,QAAQ,EAAE,QAAQ,IAAI,IAAI,EAAE,eAAe,IAAI,IAAI,CAAC,CAAC;YAEhG,OAAO,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;QAC1B,CAAC,CAAC,CACH,CAAC;IACJ,CAAC;IAED;;;;;;;;;OASG;IACI,KAAK,CAAC,6BAA6B,CAAC,KAAmC;QAC5E,sDAAsD;QACtD,MAAM,gBAAgB,GAAG,IAAI,CAAC,OAAO,EAAE,gBAAgB,IAAI,0BAA0B,CAAC;QAEtF,IAAI,KAAK,CAAC,MAAM,IAAI,gBAAgB,EAAE,CAAC;YACrC,qEAAqE;YACrE,OAAO,MAAM,IAAI,CAAC,+BAA+B,CAAC,KAAK,CAAC,CAAC;QAC3D,CAAC;QACD,oCAAoC;QACpC,IAAI,OAAO,GAAG,IAAI,GAAG,EAAiC,CAAC;QACvD,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,IAAI,gBAAgB,EAAE,CAAC;YACxD,MAAM,WAAW,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,gBAAgB,CAAC,CAAC;YACzD,MAAM,YAAY,GAAG,MAAM,IAAI,CAAC,+BAA+B,CAAC,WAAW,CAAC,CAAC;YAC7E,KAAK,IAAI,CAAC,IAAI,YAAY,CAAC,MAAM,EAAE,EAAE,CAAC;gBACpC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;YAC3B,CAAC;QACH,CAAC;QACD,OAAO,OAAO,CAAC;IACjB,CAAC;IAED;;;;;;OAMG;IACK,KAAK,CAAC,+BAA+B,CAAC,KAAmC;QAC/E,MAAM,UAAU,GAAG,IAAI,CAAC,OAAO,EAAE,mBAAmB,IAAI,6BAA6B,CAAC;QAEtF,oEAAoE;QACpE,IAAI,QAAQ,GAAG,IAAI,GAAG,EAAsC,CAAC;QAC7D,KAAK,IAAI,OAAO,IAAI,KAAK,EAAE,CAAC;YAC1B,QAAQ,CAAC,GAAG,CAAC,OAAO,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;QACxC,CAAC;QAED,MAAM,gBAAgB,GAAG,IAAI,GAAG,EAAiC,CAAC;QAElE,OAAO,QAAQ,CAAC,IAAI,GAAG,CAAC,EAAE,CAAC;YACzB,MAAM,OAAO,GAAU,EAAE,CAAC;YAC1B,KAAK,IAAI,OAAO,IAAI,QAAQ,CAAC,MAAM,EAAE,EAAE,CAAC;gBACtC,OAAO,CAAC,IAAI,CAAC;oBACX,GAAG,EAAE;wBACH,GAAG,EAAE;4BACH,CAAC,EAAE,IAAI,CAAC,QAAQ;4BAChB,CAAC,EAAE,OAAO,CAAC,MAAM;4BACjB,CAAC,EAAE,OAAO,CAAC,KAAK,IAAI,IAAI,IAAI,CAAC,MAAM,EAAE;yBACtC;wBACD,IAAI,EAAE;4BACJ,CAAC,EAAE,IAAI,CAAC,QAAQ;4BAChB,CAAC,EAAE,OAAO,CAAC,MAAM;4BACjB,CAAC,EAAE,OAAO,CAAC,GAAG;yBACf;qBACF;iBACF,CAAC,CAAC;YACL,CAAC;YAED,6DAA6D;YAC7D,6FAA6F;YAC7F,+BAA+B;YAC/B,yCAAyC;YACzC,8BAA8B;YAC9B,+BAA+B;YAC/B,8BAA8B;YAC9B,qBAAqB;YACrB,sBAAsB;YACtB,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,WAAW;iBACxC,SAAS,CACR;gBACE;oBACE,MAAM,EAAE;wBACN,GAAG,EAAE,OAAO;qBACb;iBACF;gBACD,mCAAmC;gBACnC,EAAE,KAAK,EAAE,EAAE,GAAG,EAAE,CAAC,EAAE,EAAE;gBACrB,EAAE,MAAM,EAAE,UAAU,EAAE;gBACtB;oBACE,MAAM,EAAE;wBACN,GAAG,EAAE,QAAQ;wBACb,6DAA6D;wBAC7D,0CAA0C;wBAC1C,8FAA8F;wBAC9F,cAAc,EAAE,EAAE,IAAI,EAAE,EAAE,OAAO,EAAE,WAAW,EAAE,EAAE;wBAClD,KAAK,EAAE,EAAE,IAAI,EAAE,CAAC,EAAE;wBAClB,YAAY,EAAE;4BACZ,IAAI,EAAE;gCACJ,KAAK,EAAE,CAAC,EAAE,GAAG,EAAE,CAAC,KAAK,EAAE,OAAO,CAAC,EAAE,EAAE,CAAC,EAAE,CAAC,CAAC;6BACzC;yBACF;wBACD,OAAO,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;qBAC5B;iBACF;gBACD,4DAA4D;gBAC5D,wEAAwE;gBACxE,EAAE,KAAK,EAAE,EAAE,GAAG,EAAE,CAAC,EAAE,EAAE;aACtB,EACD,EAAE,OAAO,EAAE,SAAS,EAAE,WAAW,EAAE,UAAU,EAAE,SAAS,EAAE,SAAS,CAAC,yBAAyB,EAAE,CAChG;iBACA,OAAO,EAAE;iBACT,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE;gBACX,MAAM,SAAS,CAAC,aAAa,CAAC,CAAC,EAAE,yBAAyB,CAAC,CAAC;YAC9D,CAAC,CAAC,CAAC;YAEL,IAAI,UAAU,GAAG,CAAC,CAAC;YACnB,IAAI,YAAY,GAAG,KAAK,CAAC;YACzB,KAAK,IAAI,GAAG,IAAI,SAAS,EAAE,CAAC;gBAC1B,MAAM,MAAM,GAAG,GAAG,CAAC,GAAG,CAAC;gBACvB,MAAM,QAAQ,GAAG,qBAAqB,CAAC,GAAG,CAAC,CAAC;gBAE5C,MAAM,QAAQ,GAAG,gBAAgB,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;gBAC9C,IAAI,QAAQ,IAAI,IAAI,EAAE,CAAC;oBACrB,gBAAgB,CAAC,GAAG,CAAC,MAAM,EAAE,mBAAmB,CAAC,MAAM,EAAE,QAAQ,EAAE,QAAQ,CAAC,CAAC,CAAC;gBAChF,CAAC;qBAAM,CAAC;oBACN,gBAAgB,CAAC,GAAG,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;gBACzC,CAAC;gBAED,UAAU,IAAI,GAAG,CAAC,KAAK,CAAC;gBACxB,IAAI,UAAU,IAAI,UAAU,EAAE,CAAC;oBAC7B,iDAAiD;oBACjD,4DAA4D;oBAC5D,YAAY,GAAG,IAAI,CAAC;oBACpB,MAAM,GAAG,GAAG,QAAQ,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;oBACjC,QAAQ,CAAC,GAAG,CAAC,MAAM,EAAE;wBACnB,MAAM;wBACN,KAAK,EAAE,GAAG,CAAC,OAAO;wBAClB,GAAG,EAAE,GAAI,CAAC,GAAG;qBACd,CAAC,CAAC;gBACL,CAAC;qBAAM,CAAC;oBACN,2BAA2B;oBAC3B,QAAQ,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;gBAC1B,CAAC;YACH,CAAC;YACD,IAAI,CAAC,YAAY,EAAE,CAAC;gBAClB,MAAM;YACR,CAAC;QACH,CAAC;QAED,OAAO,IAAI,GAAG,CACZ,KAAK,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE;YACpB,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;YAC9B,kCAAkC;YAClC,IAAI,eAAe,GAAG,gBAAgB,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;YACnD,IAAI,eAAe,IAAI,IAAI,EAAE,CAAC;gBAC5B,eAAe,GAAG;oBAChB,MAAM;oBACN,YAAY,EAAE,CAAC;oBACf,eAAe,EAAE,CAAC;iBACnB,CAAC;YACJ,CAAC;YACD,IAAI,OAAO,CAAC,KAAK,IAAI,IAAI,IAAI,iBAAiB,CAAC,eAAe,CAAC,EAAE,CAAC;gBAChE,eAAe,GAAG;oBAChB,MAAM;oBACN,KAAK,EAAE,eAAe,CAAC,YAAY;oBACnC,QAAQ,EAAE,eAAe,CAAC,eAAe;iBAC1C,CAAC;YACJ,CAAC;YAED,OAAO,CAAC,MAAM,EAAE,eAAe,CAAC,CAAC;QACnC,CAAC,CAAC,CACH,CAAC;IACJ,CAAC;CACF;AAED;;GAEG;AACH,SAAS,qBAAqB,CAAC,GAAkB;IAC/C,MAAM,eAAe,GAAG,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,cAAc,CAAC,GAAG,WAAW,CAAC,GAAG,UAAU,CAAC;IACtF,MAAM,MAAM,GAAG,GAAG,CAAC,GAAG,CAAC;IAEvB,IAAI,GAAG,CAAC,YAAY,IAAI,CAAC,EAAE,CAAC;QAC1B,OAAO;YACL,4CAA4C;YAC5C,MAAM;YACN,QAAQ,EAAE,eAAe;YACzB,KAAK,EAAE,GAAG,CAAC,KAAK;SACQ,CAAC;IAC7B,CAAC;SAAM,CAAC;QACN,OAAO;YACL,gDAAgD;YAChD,MAAM;YACN,YAAY,EAAE,GAAG,CAAC,KAAK;YACvB,eAAe;SACU,CAAC;IAC9B,CAAC;AACH,CAAC"}
|
|
@@ -5,6 +5,7 @@ import { cacheKey } from './OperationBatch.js';
|
|
|
5
5
|
const DEFAULT_CLEAR_BATCH_LIMIT = 5000;
|
|
6
6
|
const DEFAULT_MOVE_BATCH_LIMIT = 2000;
|
|
7
7
|
const DEFAULT_MOVE_BATCH_QUERY_LIMIT = 10_000;
|
|
8
|
+
const DEFAULT_MIN_BUCKET_CHANGES = 10;
|
|
8
9
|
/** This default is primarily for tests. */
|
|
9
10
|
const DEFAULT_MEMORY_LIMIT_MB = 64;
|
|
10
11
|
export class MongoCompactor {
|
|
@@ -16,6 +17,7 @@ export class MongoCompactor {
|
|
|
16
17
|
moveBatchLimit;
|
|
17
18
|
moveBatchQueryLimit;
|
|
18
19
|
clearBatchLimit;
|
|
20
|
+
minBucketChanges;
|
|
19
21
|
maxOpId;
|
|
20
22
|
buckets;
|
|
21
23
|
signal;
|
|
@@ -28,6 +30,7 @@ export class MongoCompactor {
|
|
|
28
30
|
this.moveBatchLimit = options?.moveBatchLimit ?? DEFAULT_MOVE_BATCH_LIMIT;
|
|
29
31
|
this.moveBatchQueryLimit = options?.moveBatchQueryLimit ?? DEFAULT_MOVE_BATCH_QUERY_LIMIT;
|
|
30
32
|
this.clearBatchLimit = options?.clearBatchLimit ?? DEFAULT_CLEAR_BATCH_LIMIT;
|
|
33
|
+
this.minBucketChanges = options?.minBucketChanges ?? DEFAULT_MIN_BUCKET_CHANGES;
|
|
31
34
|
this.maxOpId = options?.maxOpId ?? 0n;
|
|
32
35
|
this.buckets = options?.compactBuckets;
|
|
33
36
|
this.signal = options?.signal;
|
|
@@ -52,14 +55,25 @@ export class MongoCompactor {
|
|
|
52
55
|
}
|
|
53
56
|
async compactDirtyBuckets() {
|
|
54
57
|
while (!this.signal?.aborted) {
|
|
55
|
-
// Process all buckets with
|
|
56
|
-
|
|
58
|
+
// Process all buckets with 10 or more changes since last time.
|
|
59
|
+
// We exclude the last 100 compacted buckets, to avoid repeatedly re-compacting the same buckets over and over
|
|
60
|
+
// if they are modified while compacting.
|
|
61
|
+
const TRACK_RECENTLY_COMPACTED_NUMBER = 100;
|
|
62
|
+
let recentlyCompacted = [];
|
|
63
|
+
const buckets = await this.dirtyBucketBatch({
|
|
64
|
+
minBucketChanges: this.minBucketChanges,
|
|
65
|
+
exclude: recentlyCompacted
|
|
66
|
+
});
|
|
57
67
|
if (buckets.length == 0) {
|
|
58
68
|
// All done
|
|
59
69
|
break;
|
|
60
70
|
}
|
|
61
|
-
for (let bucket of buckets) {
|
|
71
|
+
for (let { bucket } of buckets) {
|
|
62
72
|
await this.compactSingleBucket(bucket);
|
|
73
|
+
recentlyCompacted.push(bucket);
|
|
74
|
+
}
|
|
75
|
+
if (recentlyCompacted.length > TRACK_RECENTLY_COMPACTED_NUMBER) {
|
|
76
|
+
recentlyCompacted = recentlyCompacted.slice(-TRACK_RECENTLY_COMPACTED_NUMBER);
|
|
63
77
|
}
|
|
64
78
|
}
|
|
65
79
|
}
|
|
@@ -379,9 +393,19 @@ export class MongoCompactor {
|
|
|
379
393
|
break;
|
|
380
394
|
}
|
|
381
395
|
const start = Date.now();
|
|
382
|
-
logger.info(`Calculating checksums for batch of ${buckets.length} buckets
|
|
383
|
-
|
|
384
|
-
|
|
396
|
+
logger.info(`Calculating checksums for batch of ${buckets.length} buckets`);
|
|
397
|
+
// Filter batch by estimated bucket size, to reduce possibility of timeouts
|
|
398
|
+
let checkBuckets = [];
|
|
399
|
+
let totalCountEstimate = 0;
|
|
400
|
+
for (let bucket of buckets) {
|
|
401
|
+
checkBuckets.push(bucket);
|
|
402
|
+
totalCountEstimate += bucket.estimatedCount;
|
|
403
|
+
if (totalCountEstimate > 50_000) {
|
|
404
|
+
break;
|
|
405
|
+
}
|
|
406
|
+
}
|
|
407
|
+
await this.updateChecksumsBatch(checkBuckets.map((b) => b.bucket));
|
|
408
|
+
logger.info(`Updated checksums for batch of ${checkBuckets.length} buckets in ${Date.now() - start}ms`);
|
|
385
409
|
count += buckets.length;
|
|
386
410
|
}
|
|
387
411
|
return { buckets: count };
|
|
@@ -400,19 +424,25 @@ export class MongoCompactor {
|
|
|
400
424
|
const dirtyBuckets = await this.db.bucket_state
|
|
401
425
|
.find({
|
|
402
426
|
'_id.g': this.group_id,
|
|
403
|
-
'estimate_since_compact.count': { $gte: options.minBucketChanges }
|
|
427
|
+
'estimate_since_compact.count': { $gte: options.minBucketChanges },
|
|
428
|
+
'_id.b': { $nin: options.exclude ?? [] }
|
|
404
429
|
}, {
|
|
405
430
|
projection: {
|
|
406
|
-
_id: 1
|
|
431
|
+
_id: 1,
|
|
432
|
+
estimate_since_compact: 1,
|
|
433
|
+
compacted_state: 1
|
|
407
434
|
},
|
|
408
435
|
sort: {
|
|
409
436
|
'estimate_since_compact.count': -1
|
|
410
437
|
},
|
|
411
|
-
limit:
|
|
438
|
+
limit: 200,
|
|
412
439
|
maxTimeMS: MONGO_OPERATION_TIMEOUT_MS
|
|
413
440
|
})
|
|
414
441
|
.toArray();
|
|
415
|
-
return dirtyBuckets.map((bucket) =>
|
|
442
|
+
return dirtyBuckets.map((bucket) => ({
|
|
443
|
+
bucket: bucket._id.b,
|
|
444
|
+
estimatedCount: bucket.estimate_since_compact.count + (bucket.compacted_state?.count ?? 0)
|
|
445
|
+
}));
|
|
416
446
|
}
|
|
417
447
|
async updateChecksumsBatch(buckets) {
|
|
418
448
|
const checksums = await this.storage.checksums.computePartialChecksumsDirect(buckets.map((bucket) => {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"MongoCompactor.js","sourceRoot":"","sources":["../../../src/storage/implementation/MongoCompactor.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,0BAA0B,EAAE,MAAM,gCAAgC,CAAC;AACnF,OAAO,EAAE,MAAM,EAAE,yBAAyB,EAAE,qBAAqB,EAAE,MAAM,mCAAmC,CAAC;AAC7G,OAAO,EACL,YAAY,EAEZ,iBAAiB,EAGjB,KAAK,EACN,MAAM,yBAAyB,CAAC;AAKjC,OAAO,EAAE,QAAQ,EAAE,MAAM,qBAAqB,CAAC;AA8C/C,MAAM,yBAAyB,GAAG,IAAI,CAAC;AACvC,MAAM,wBAAwB,GAAG,IAAI,CAAC;AACtC,MAAM,8BAA8B,GAAG,MAAM,CAAC;AAE9C,2CAA2C;AAC3C,MAAM,uBAAuB,GAAG,EAAE,CAAC;AAEnC,MAAM,OAAO,cAAc;IAcf;IACA;IAdF,OAAO,GAAsD,EAAE,CAAC;IAChE,kBAAkB,GAAuD,EAAE,CAAC;IAE5E,YAAY,CAAS;IACrB,cAAc,CAAS;IACvB,mBAAmB,CAAS;IAC5B,eAAe,CAAS;IACxB,OAAO,CAAS;IAChB,OAAO,CAAuB;IAC9B,MAAM,CAAe;IACrB,QAAQ,CAAS;IAEzB,YACU,OAA+B,EAC/B,EAAkB,EAC1B,OAA6B;QAFrB,YAAO,GAAP,OAAO,CAAwB;QAC/B,OAAE,GAAF,EAAE,CAAgB;QAG1B,IAAI,CAAC,QAAQ,GAAG,OAAO,CAAC,QAAQ,CAAC;QACjC,IAAI,CAAC,YAAY,GAAG,CAAC,OAAO,EAAE,aAAa,IAAI,uBAAuB,CAAC,GAAG,IAAI,GAAG,IAAI,CAAC;QACtF,IAAI,CAAC,cAAc,GAAG,OAAO,EAAE,cAAc,IAAI,wBAAwB,CAAC;QAC1E,IAAI,CAAC,mBAAmB,GAAG,OAAO,EAAE,mBAAmB,IAAI,8BAA8B,CAAC;QAC1F,IAAI,CAAC,eAAe,GAAG,OAAO,EAAE,eAAe,IAAI,yBAAyB,CAAC;QAC7E,IAAI,CAAC,OAAO,GAAG,OAAO,EAAE,OAAO,IAAI,EAAE,CAAC;QACtC,IAAI,CAAC,OAAO,GAAG,OAAO,EAAE,cAAc,CAAC;QACvC,IAAI,CAAC,MAAM,GAAG,OAAO,EAAE,MAAM,CAAC;IAChC,CAAC;IAED;;;;OAIG;IACH,KAAK,CAAC,OAAO;QACX,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;YACjB,KAAK,IAAI,MAAM,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;gBAChC,wDAAwD;gBACxD,yCAAyC;gBACzC,0DAA0D;gBAC1D,MAAM,IAAI,CAAC,mBAAmB,CAAC,MAAM,CAAC,CAAC;YACzC,CAAC;QACH,CAAC;aAAM,CAAC;YACN,MAAM,IAAI,CAAC,mBAAmB,EAAE,CAAC;QACnC,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,mBAAmB;QAC/B,OAAO,CAAC,IAAI,CAAC,MAAM,EAAE,OAAO,EAAE,CAAC;YAC7B,6DAA6D;YAC7D,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,gBAAgB,CAAC,EAAE,gBAAgB,EAAE,CAAC,EAAE,CAAC,CAAC;YACrE,IAAI,OAAO,CAAC,MAAM,IAAI,CAAC,EAAE,CAAC;gBACxB,WAAW;gBACX,MAAM;YACR,CAAC;YACD,KAAK,IAAI,MAAM,IAAI,OAAO,EAAE,CAAC;gBAC3B,MAAM,IAAI,CAAC,mBAAmB,CAAC,MAAM,CAAC,CAAC;YACzC,CAAC;QACH,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,mBAAmB,CAAC,MAAc;QAC9C,MAAM,YAAY,GAAG,IAAI,CAAC,YAAY,CAAC;QAEvC,IAAI,YAAY,GAAuB;YACrC,MAAM;YACN,IAAI,EAAE,IAAI,GAAG,EAAE;YACf,YAAY,EAAE,CAAC;YACf,UAAU,EAAE,IAAI;YAChB,WAAW,EAAE,CAAC;YAEd,QAAQ,EAAE,CAAC;YACX,OAAO,EAAE,CAAC;YACV,OAAO,EAAE,CAAC;SACX,CAAC;QAEF,uBAAuB;QACvB,MAAM,UAAU,GAAkB;YAChC,CAAC,EAAE,IAAI,CAAC,QAAQ;YAChB,CAAC,EAAE,MAAM;YACT,CAAC,EAAE,IAAI,KAAK,CAAC,MAAM,EAAS;SAC7B,CAAC;QAEF,yCAAyC;QACzC,IAAI,UAAU,GAAkB;YAC9B,CAAC,EAAE,IAAI,CAAC,QAAQ;YAChB,CAAC,EAAE,MAAM;YACT,CAAC,EAAE,IAAI,KAAK,CAAC,MAAM,EAAS;SAC7B,CAAC;QAEF,OAAO,CAAC,IAAI,CAAC,MAAM,EAAE,OAAO,EAAE,CAAC;YAC7B,sDAAsD;YACtD,MAAM,MAAM,GAAG,IAAI,CAAC,EAAE,CAAC,WAAW,CAAC,SAAS,CAC1C;gBACE;oBACE,MAAM,EAAE;wBACN,GAAG,EAAE;4BACH,IAAI,EAAE,UAAU;4BAChB,GAAG,EAAE,UAAU;yBAChB;qBACF;iBACF;gBACD,EAAE,KAAK,EAAE,EAAE,GAAG,EAAE,CAAC,CAAC,EAAE,EAAE;gBACtB,EAAE,MAAM,EAAE,IAAI,CAAC,mBAAmB,EAAE;gBACpC;oBACE,QAAQ,EAAE;wBACR,GAAG,EAAE,CAAC;wBACN,EAAE,EAAE,CAAC;wBACL,KAAK,EAAE,CAAC;wBACR,MAAM,EAAE,CAAC;wBACT,YAAY,EAAE,CAAC;wBACf,UAAU,EAAE,CAAC;wBACb,QAAQ,EAAE,CAAC;wBACX,IAAI,EAAE,EAAE,SAAS,EAAE,QAAQ,EAAE;qBAC9B;iBACF;aACF,EACD;gBACE,2DAA2D;gBAC3D,+DAA+D;gBAC/D,SAAS,EAAE,IAAI,CAAC,mBAAmB,GAAG,CAAC;aACxC,CACF,CAAC;YACF,+GAA+G;YAC/G,oCAAoC;YACpC,MAAM,KAAK,GAAG,MAAM,MAAM,CAAC,OAAO,EAAE,CAAC;YAErC,IAAI,KAAK,CAAC,MAAM,IAAI,CAAC,EAAE,CAAC;gBACtB,wBAAwB;gBACxB,MAAM;YACR,CAAC;YAED,oCAAoC;YACpC,UAAU,GAAG,KAAK,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC;YAEzC,KAAK,IAAI,GAAG,IAAI,KAAK,EAAE,CAAC;gBACtB,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC,GAAG,IAAI,CAAC,OAAO,EAAE,CAAC;oBAC7B,SAAS;gBACX,CAAC;gBAED,YAAY,CAAC,QAAQ,GAAG,YAAY,CAAC,YAAY,CAAC,QAAQ,EAAE,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC,CAAC;gBAClF,YAAY,CAAC,OAAO,IAAI,CAAC,CAAC;gBAE1B,IAAI,eAAe,GAAG,GAAG,CAAC,EAAE,IAAI,KAAK,CAAC;gBAEtC,YAAY,CAAC,OAAO,IAAI,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;gBACzC,IAAI,GAAG,CAAC,EAAE,IAAI,QAAQ,IAAI,GAAG,CAAC,EAAE,IAAI,KAAK,EAAE,CAAC;oBAC1C,MAAM,GAAG,GAAG,GAAG,GAAG,CAAC,KAAK,IAAI,GAAG,CAAC,MAAM,IAAI,QAAQ,CAAC,GAAG,CAAC,YAAa,EAAE,GAAG,CAAC,UAAW,CAAC,EAAE,CAAC;oBACzF,MAAM,QAAQ,GAAG,YAAY,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;oBAC5C,IAAI,QAAQ,EAAE,CAAC;wBACb,8CAA8C;wBAC9C,eAAe,GAAG,KAAK,CAAC;wBAExB,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC;4BAChB,SAAS,EAAE;gCACT,MAAM,EAAE;oCACN,GAAG,EAAE,GAAG,CAAC,GAAG;iCACb;gCACD,MAAM,EAAE;oCACN,IAAI,EAAE;wCACJ,EAAE,EAAE,MAAM;wCACV,SAAS,EAAE,QAAQ;qCACpB;oCACD,MAAM,EAAE;wCACN,YAAY,EAAE,CAAC;wCACf,UAAU,EAAE,CAAC;wCACb,KAAK,EAAE,CAAC;wCACR,MAAM,EAAE,CAAC;wCACT,IAAI,EAAE,CAAC;qCACR;iCACF;6BACF;yBACF,CAAC,CAAC;wBAEH,YAAY,CAAC,OAAO,IAAI,GAAG,GAAG,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,iCAAiC;oBACnF,CAAC;yBAAM,CAAC;wBACN,IAAI,YAAY,CAAC,YAAY,IAAI,YAAY,EAAE,CAAC;4BAC9C,wBAAwB;4BACxB,6CAA6C;wBAC/C,CAAC;6BAAM,CAAC;4BACN,4DAA4D;4BAC5D,YAAY,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;4BACrD,6BAA6B;4BAC7B,oBAAoB;4BACpB,sBAAsB;4BACtB,6BAA6B;4BAC7B,YAAY,CAAC,YAAY,IAAI,GAAG,CAAC,MAAM,GAAG,GAAG,CAAC;wBAChD,CAAC;oBACH,CAAC;gBACH,CAAC;gBAED,IAAI,eAAe,EAAE,CAAC;oBACpB,YAAY,CAAC,UAAU,GAAG,IAAI,CAAC;oBAC/B,YAAY,CAAC,WAAW,GAAG,CAAC,CAAC;gBAC/B,CAAC;qBAAM,IAAI,GAAG,CAAC,EAAE,IAAI,OAAO,EAAE,CAAC;oBAC7B,IAAI,YAAY,CAAC,UAAU,IAAI,IAAI,EAAE,CAAC;wBACpC,YAAY,CAAC,UAAU,GAAG,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC;oBACtC,CAAC;oBACD,YAAY,CAAC,WAAW,IAAI,CAAC,CAAC;gBAChC,CAAC;gBAED,IAAI,IAAI,CAAC,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,kBAAkB,CAAC,MAAM,IAAI,IAAI,CAAC,cAAc,EAAE,CAAC;oBAChF,MAAM,IAAI,CAAC,KAAK,EAAE,CAAC;gBACrB,CAAC;YACH,CAAC;YAED,MAAM,CAAC,IAAI,CAAC,6BAA6B,KAAK,CAAC,MAAM,oBAAoB,MAAM,EAAE,CAAC,CAAC;QACrF,CAAC;QAED,qCAAqC;QACrC,YAAY,CAAC,IAAI,CAAC,KAAK,EAAE,CAAC;QAC1B,IAAI,YAAY,CAAC,UAAU,IAAI,IAAI,IAAI,YAAY,CAAC,WAAW,IAAI,CAAC,EAAE,CAAC;YACrE,MAAM,CAAC,IAAI,CACT,sBAAsB,IAAI,CAAC,QAAQ,IAAI,MAAM,IAAI,YAAY,CAAC,UAAU,cAAc,YAAY,CAAC,WAAW,aAAa,CAC5H,CAAC;YACF,8BAA8B;YAC9B,MAAM,IAAI,CAAC,KAAK,EAAE,CAAC;YACnB,MAAM,IAAI,CAAC,WAAW,CAAC,YAAY,CAAC,CAAC;QACvC,CAAC;QAED,+DAA+D;QAC/D,IAAI,CAAC,qBAAqB,CAAC,YAAY,CAAC,CAAC;QAEzC,mDAAmD;QACnD,MAAM,IAAI,CAAC,KAAK,EAAE,CAAC;IACrB,CAAC;IAED;;OAEG;IACK,qBAAqB,CAAC,KAAyB;QACrD,IAAI,KAAK,CAAC,OAAO,GAAG,CAAC,EAAE,CAAC;YACtB,MAAM,IAAI,qBAAqB,CAC7B,oBAAoB,KAAK,CAAC,OAAO,aAAa,KAAK,CAAC,QAAQ,iBAAiB,KAAK,CAAC,WAAW,aAAa,IAAI,CAAC,OAAO,EAAE,CAC1H,CAAC;QACJ,CAAC;QACD,IAAI,CAAC,kBAAkB,CAAC,IAAI,CAAC;YAC3B,SAAS,EAAE;gBACT,MAAM,EAAE;oBACN,GAAG,EAAE;wBACH,CAAC,EAAE,IAAI,CAAC,QAAQ;wBAChB,CAAC,EAAE,KAAK,CAAC,MAAM;qBAChB;iBACF;gBACD,MAAM,EAAE;oBACN,IAAI,EAAE;wBACJ,eAAe,EAAE;4BACf,KAAK,EAAE,IAAI,CAAC,OAAO;4BACnB,KAAK,EAAE,KAAK,CAAC,OAAO;4BACpB,QAAQ,EAAE,MAAM,CAAC,KAAK,CAAC,QAAQ,CAAC;4BAChC,KAAK,EAAE,KAAK,CAAC,OAAO;yBACrB;wBACD,sBAAsB,EAAE;4BACtB,sGAAsG;4BACtG,sCAAsC;4BACtC,oEAAoE;4BACpE,KAAK,EAAE,CAAC;4BACR,KAAK,EAAE,CAAC;yBACT;qBACF;iBACF;gBACD,wDAAwD;gBACxD,0FAA0F;gBAC1F,MAAM,EAAE,KAAK;aACd;SACF,CAAC,CAAC;IACL,CAAC;IAEO,KAAK,CAAC,KAAK;QACjB,IAAI,IAAI,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC5B,MAAM,CAAC,IAAI,CAAC,cAAc,IAAI,CAAC,OAAO,CAAC,MAAM,MAAM,CAAC,CAAC;YACrD,MAAM,IAAI,CAAC,EAAE,CAAC,WAAW,CAAC,SAAS,CAAC,IAAI,CAAC,OAAO,EAAE;gBAChD,0BAA0B;gBAC1B,8EAA8E;gBAC9E,yDAAyD;gBACzD,uCAAuC;gBACvC,OAAO,EAAE,KAAK;aACf,CAAC,CAAC;YACH,IAAI,CAAC,OAAO,GAAG,EAAE,CAAC;QACpB,CAAC;QACD,IAAI,IAAI,CAAC,kBAAkB,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACvC,MAAM,CAAC,IAAI,CAAC,YAAY,IAAI,CAAC,kBAAkB,CAAC,MAAM,gBAAgB,CAAC,CAAC;YACxE,MAAM,IAAI,CAAC,EAAE,CAAC,YAAY,CAAC,SAAS,CAAC,IAAI,CAAC,kBAAkB,EAAE;gBAC5D,OAAO,EAAE,KAAK;aACf,CAAC,CAAC;YACH,IAAI,CAAC,kBAAkB,GAAG,EAAE,CAAC;QAC/B,CAAC;IACH,CAAC;IAED;;;;;;OAMG;IACK,KAAK,CAAC,WAAW,CAAC,YAAgC;QACxD,MAAM,MAAM,GAAG,YAAY,CAAC,MAAM,CAAC;QACnC,MAAM,OAAO,GAAG,YAAY,CAAC,UAAW,CAAC;QAEzC,MAAM,QAAQ,GAAG;YACf,GAAG,EAAE;gBACH,IAAI,EAAE;oBACJ,CAAC,EAAE,IAAI,CAAC,QAAQ;oBAChB,CAAC,EAAE,MAAM;oBACT,CAAC,EAAE,IAAI,KAAK,CAAC,MAAM,EAAS;iBAC7B;gBACD,IAAI,EAAE;oBACJ,CAAC,EAAE,IAAI,CAAC,QAAQ;oBAChB,CAAC,EAAE,MAAM;oBACT,CAAC,EAAE,OAAO;iBACX;aACF;SACF,CAAC;QAEF,MAAM,OAAO,GAAG,IAAI,CAAC,EAAE,CAAC,MAAM,CAAC,YAAY,EAAE,CAAC;QAC9C,IAAI,CAAC;YACH,IAAI,IAAI,GAAG,KAAK,CAAC;YACjB,OAAO,CAAC,IAAI,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,OAAO,EAAE,CAAC;gBACtC,IAAI,WAAW,GAAG,CAAC,CAAC;gBACpB,6EAA6E;gBAC7E,kDAAkD;gBAClD,0EAA0E;gBAC1E,MAAM,OAAO,CAAC,eAAe,CAC3B,KAAK,IAAI,EAAE;oBACT,MAAM,KAAK,GAAG,IAAI,CAAC,EAAE,CAAC,WAAW,CAAC,IAAI,CAAC,QAAQ,EAAE;wBAC/C,OAAO;wBACP,IAAI,EAAE,EAAE,GAAG,EAAE,CAAC,EAAE;wBAChB,UAAU,EAAE;4BACV,GAAG,EAAE,CAAC;4BACN,EAAE,EAAE,CAAC;4BACL,QAAQ,EAAE,CAAC;4BACX,SAAS,EAAE,CAAC;yBACb;wBACD,KAAK,EAAE,IAAI,CAAC,eAAe;qBAC5B,CAAC,CAAC;oBACH,IAAI,QAAQ,GAAG,CAAC,CAAC;oBACjB,IAAI,QAAQ,GAAyB,IAAI,CAAC;oBAC1C,IAAI,QAAQ,GAAkB,IAAI,CAAC;oBACnC,IAAI,OAAO,GAAG,KAAK,CAAC;oBACpB,IAAI,kBAAkB,GAAG,CAAC,CAAC;oBAC3B,IAAI,KAAK,EAAE,IAAI,EAAE,IAAI,KAAK,CAAC,MAAM,EAAE,EAAE,CAAC;wBACpC,IAAI,EAAE,CAAC,EAAE,IAAI,MAAM,IAAI,EAAE,CAAC,EAAE,IAAI,QAAQ,IAAI,EAAE,CAAC,EAAE,IAAI,OAAO,EAAE,CAAC;4BAC7D,QAAQ,GAAG,KAAK,CAAC,YAAY,CAAC,QAAQ,EAAE,MAAM,CAAC,EAAE,CAAC,QAAQ,CAAC,CAAC,CAAC;4BAC7D,QAAQ,GAAG,EAAE,CAAC,GAAG,CAAC;4BAClB,kBAAkB,IAAI,CAAC,CAAC;4BACxB,IAAI,EAAE,CAAC,EAAE,IAAI,OAAO,EAAE,CAAC;gCACrB,OAAO,GAAG,IAAI,CAAC;4BACjB,CAAC;4BACD,IAAI,EAAE,CAAC,SAAS,IAAI,IAAI,EAAE,CAAC;gCACzB,IAAI,QAAQ,IAAI,IAAI,IAAI,EAAE,CAAC,SAAS,GAAG,QAAQ,EAAE,CAAC;oCAChD,QAAQ,GAAG,EAAE,CAAC,SAAS,CAAC;gCAC1B,CAAC;4BACH,CAAC;wBACH,CAAC;6BAAM,CAAC;4BACN,MAAM,IAAI,yBAAyB,CACjC,cAAc,EAAE,CAAC,EAAE,iBAAiB,EAAE,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC,GAAG,CAAC,CAAC,EAAE,CACvE,CAAC;wBACJ,CAAC;oBACH,CAAC;oBACD,IAAI,CAAC,OAAO,EAAE,CAAC;wBACb,IAAI,GAAG,IAAI,CAAC;wBACZ,OAAO;oBACT,CAAC;oBAED,MAAM,CAAC,IAAI,CAAC,sBAAsB,kBAAkB,WAAW,QAAQ,EAAE,CAAC,EAAE,CAAC,CAAC;oBAC9E,MAAM,IAAI,CAAC,EAAE,CAAC,WAAW,CAAC,UAAU,CAClC;wBACE,GAAG,EAAE;4BACH,IAAI,EAAE;gCACJ,CAAC,EAAE,IAAI,CAAC,QAAQ;gCAChB,CAAC,EAAE,MAAM;gCACT,CAAC,EAAE,IAAI,KAAK,CAAC,MAAM,EAAS;6BAC7B;4BACD,IAAI,EAAE,QAAS;yBAChB;qBACF,EACD,EAAE,OAAO,EAAE,CACZ,CAAC;oBAEF,MAAM,IAAI,CAAC,EAAE,CAAC,WAAW,CAAC,SAAS,CACjC;wBACE,GAAG,EAAE,QAAS;wBACd,EAAE,EAAE,OAAO;wBACX,QAAQ,EAAE,MAAM,CAAC,QAAQ,CAAC;wBAC1B,IAAI,EAAE,IAAI;wBACV,SAAS,EAAE,QAAQ;qBACpB,EACD,EAAE,OAAO,EAAE,CACZ,CAAC;oBAEF,WAAW,GAAG,CAAC,kBAAkB,GAAG,CAAC,CAAC;gBACxC,CAAC,EACD;oBACE,YAAY,EAAE,EAAE,CAAC,EAAE,UAAU,EAAE;oBAC/B,WAAW,EAAE,EAAE,KAAK,EAAE,UAAU,EAAE;iBACnC,CACF,CAAC;gBACF,yFAAyF;gBACzF,YAAY,CAAC,OAAO,IAAI,WAAW,CAAC;YACtC,CAAC;QACH,CAAC;gBAAS,CAAC;YACT,MAAM,OAAO,CAAC,UAAU,EAAE,CAAC;QAC7B,CAAC;IACH,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,iBAAiB,CAAC,OAAqC;QAC3D,IAAI,KAAK,GAAG,CAAC,CAAC;QACd,OAAO,CAAC,IAAI,CAAC,MAAM,EAAE,OAAO,EAAE,CAAC;YAC7B,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,gBAAgB,CAAC,OAAO,CAAC,CAAC;YACrD,IAAI,OAAO,CAAC,MAAM,IAAI,CAAC,EAAE,CAAC;gBACxB,WAAW;gBACX,MAAM;YACR,CAAC;YACD,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;YACzB,MAAM,CAAC,IAAI,CAAC,sCAAsC,OAAO,CAAC,MAAM,yBAAyB,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;YAEvG,MAAM,IAAI,CAAC,oBAAoB,CAAC,OAAO,CAAC,CAAC;YACzC,MAAM,CAAC,IAAI,CAAC,kCAAkC,OAAO,CAAC,MAAM,eAAe,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK,IAAI,CAAC,CAAC;YACnG,KAAK,IAAI,OAAO,CAAC,MAAM,CAAC;QAC1B,CAAC;QACD,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC;IAC5B,CAAC;IAED;;;;;OAKG;IACK,KAAK,CAAC,gBAAgB,CAAC,OAAqC;QAClE,IAAI,OAAO,CAAC,gBAAgB,IAAI,CAAC,EAAE,CAAC;YAClC,MAAM,IAAI,yBAAyB,CAAC,+BAA+B,CAAC,CAAC;QACvE,CAAC;QACD,4EAA4E;QAC5E,MAAM,YAAY,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,YAAY;aAC5C,IAAI,CACH;YACE,OAAO,EAAE,IAAI,CAAC,QAAQ;YACtB,8BAA8B,EAAE,EAAE,IAAI,EAAE,OAAO,CAAC,gBAAgB,EAAE;SACnE,EACD;YACE,UAAU,EAAE;gBACV,GAAG,EAAE,CAAC;aACP;YACD,IAAI,EAAE;gBACJ,8BAA8B,EAAE,CAAC,CAAC;aACnC;YACD,KAAK,EAAE,KAAK;YACZ,SAAS,EAAE,0BAA0B;SACtC,CACF;aACA,OAAO,EAAE,CAAC;QAEb,OAAO,YAAY,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;IACpD,CAAC;IAEO,KAAK,CAAC,oBAAoB,CAAC,OAAiB;QAClD,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC,6BAA6B,CAC1E,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE;YACrB,OAAO;gBACL,MAAM;gBACN,GAAG,EAAE,IAAI,CAAC,OAAO;aAClB,CAAC;QACJ,CAAC,CAAC,CACH,CAAC;QAEF,KAAK,IAAI,cAAc,IAAI,SAAS,CAAC,MAAM,EAAE,EAAE,CAAC;YAC9C,IAAI,iBAAiB,CAAC,cAAc,CAAC,EAAE,CAAC;gBACtC,qDAAqD;gBACrD,MAAM,IAAI,qBAAqB,CAAC,+BAA+B,IAAI,CAAC,SAAS,CAAC,cAAc,CAAC,EAAE,CAAC,CAAC;YACnG,CAAC;YAED,IAAI,CAAC,kBAAkB,CAAC,IAAI,CAAC;gBAC3B,SAAS,EAAE;oBACT,MAAM,EAAE;wBACN,GAAG,EAAE;4BACH,CAAC,EAAE,IAAI,CAAC,QAAQ;4BAChB,CAAC,EAAE,cAAc,CAAC,MAAM;yBACzB;qBACF;oBACD,MAAM,EAAE;wBACN,IAAI,EAAE;4BACJ,eAAe,EAAE;gCACf,KAAK,EAAE,IAAI,CAAC,OAAO;gCACnB,KAAK,EAAE,cAAc,CAAC,KAAK;gCAC3B,QAAQ,EAAE,MAAM,CAAC,cAAc,CAAC,QAAQ,CAAC;gCACzC,KAAK,EAAE,IAAI;6BACZ;4BACD,sBAAsB,EAAE;gCACtB,KAAK,EAAE,CAAC;gCACR,KAAK,EAAE,CAAC;6BACT;yBACF;qBACF;oBACD,oGAAoG;oBACpG,4CAA4C;oBAC5C,MAAM,EAAE,KAAK;iBACd;aACF,CAAC,CAAC;QACL,CAAC;QAED,MAAM,IAAI,CAAC,KAAK,EAAE,CAAC;IACrB,CAAC;CACF"}
|
|
1
|
+
{"version":3,"file":"MongoCompactor.js","sourceRoot":"","sources":["../../../src/storage/implementation/MongoCompactor.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,0BAA0B,EAAE,MAAM,gCAAgC,CAAC;AACnF,OAAO,EAAE,MAAM,EAAE,yBAAyB,EAAE,qBAAqB,EAAE,MAAM,mCAAmC,CAAC;AAC7G,OAAO,EACL,YAAY,EAEZ,iBAAiB,EAGjB,KAAK,EACN,MAAM,yBAAyB,CAAC;AAKjC,OAAO,EAAE,QAAQ,EAAE,MAAM,qBAAqB,CAAC;AA8C/C,MAAM,yBAAyB,GAAG,IAAI,CAAC;AACvC,MAAM,wBAAwB,GAAG,IAAI,CAAC;AACtC,MAAM,8BAA8B,GAAG,MAAM,CAAC;AAC9C,MAAM,0BAA0B,GAAG,EAAE,CAAC;AAEtC,2CAA2C;AAC3C,MAAM,uBAAuB,GAAG,EAAE,CAAC;AAEnC,MAAM,OAAO,cAAc;IAef;IACA;IAfF,OAAO,GAAsD,EAAE,CAAC;IAChE,kBAAkB,GAAuD,EAAE,CAAC;IAE5E,YAAY,CAAS;IACrB,cAAc,CAAS;IACvB,mBAAmB,CAAS;IAC5B,eAAe,CAAS;IACxB,gBAAgB,CAAS;IACzB,OAAO,CAAS;IAChB,OAAO,CAAuB;IAC9B,MAAM,CAAe;IACrB,QAAQ,CAAS;IAEzB,YACU,OAA+B,EAC/B,EAAkB,EAC1B,OAA6B;QAFrB,YAAO,GAAP,OAAO,CAAwB;QAC/B,OAAE,GAAF,EAAE,CAAgB;QAG1B,IAAI,CAAC,QAAQ,GAAG,OAAO,CAAC,QAAQ,CAAC;QACjC,IAAI,CAAC,YAAY,GAAG,CAAC,OAAO,EAAE,aAAa,IAAI,uBAAuB,CAAC,GAAG,IAAI,GAAG,IAAI,CAAC;QACtF,IAAI,CAAC,cAAc,GAAG,OAAO,EAAE,cAAc,IAAI,wBAAwB,CAAC;QAC1E,IAAI,CAAC,mBAAmB,GAAG,OAAO,EAAE,mBAAmB,IAAI,8BAA8B,CAAC;QAC1F,IAAI,CAAC,eAAe,GAAG,OAAO,EAAE,eAAe,IAAI,yBAAyB,CAAC;QAC7E,IAAI,CAAC,gBAAgB,GAAG,OAAO,EAAE,gBAAgB,IAAI,0BAA0B,CAAC;QAChF,IAAI,CAAC,OAAO,GAAG,OAAO,EAAE,OAAO,IAAI,EAAE,CAAC;QACtC,IAAI,CAAC,OAAO,GAAG,OAAO,EAAE,cAAc,CAAC;QACvC,IAAI,CAAC,MAAM,GAAG,OAAO,EAAE,MAAM,CAAC;IAChC,CAAC;IAED;;;;OAIG;IACH,KAAK,CAAC,OAAO;QACX,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;YACjB,KAAK,IAAI,MAAM,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;gBAChC,wDAAwD;gBACxD,yCAAyC;gBACzC,0DAA0D;gBAC1D,MAAM,IAAI,CAAC,mBAAmB,CAAC,MAAM,CAAC,CAAC;YACzC,CAAC;QACH,CAAC;aAAM,CAAC;YACN,MAAM,IAAI,CAAC,mBAAmB,EAAE,CAAC;QACnC,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,mBAAmB;QAC/B,OAAO,CAAC,IAAI,CAAC,MAAM,EAAE,OAAO,EAAE,CAAC;YAC7B,+DAA+D;YAC/D,8GAA8G;YAC9G,yCAAyC;YACzC,MAAM,+BAA+B,GAAG,GAAG,CAAC;YAE5C,IAAI,iBAAiB,GAAa,EAAE,CAAC;YACrC,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,gBAAgB,CAAC;gBAC1C,gBAAgB,EAAE,IAAI,CAAC,gBAAgB;gBACvC,OAAO,EAAE,iBAAiB;aAC3B,CAAC,CAAC;YACH,IAAI,OAAO,CAAC,MAAM,IAAI,CAAC,EAAE,CAAC;gBACxB,WAAW;gBACX,MAAM;YACR,CAAC;YACD,KAAK,IAAI,EAAE,MAAM,EAAE,IAAI,OAAO,EAAE,CAAC;gBAC/B,MAAM,IAAI,CAAC,mBAAmB,CAAC,MAAM,CAAC,CAAC;gBACvC,iBAAiB,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YACjC,CAAC;YACD,IAAI,iBAAiB,CAAC,MAAM,GAAG,+BAA+B,EAAE,CAAC;gBAC/D,iBAAiB,GAAG,iBAAiB,CAAC,KAAK,CAAC,CAAC,+BAA+B,CAAC,CAAC;YAChF,CAAC;QACH,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,mBAAmB,CAAC,MAAc;QAC9C,MAAM,YAAY,GAAG,IAAI,CAAC,YAAY,CAAC;QAEvC,IAAI,YAAY,GAAuB;YACrC,MAAM;YACN,IAAI,EAAE,IAAI,GAAG,EAAE;YACf,YAAY,EAAE,CAAC;YACf,UAAU,EAAE,IAAI;YAChB,WAAW,EAAE,CAAC;YAEd,QAAQ,EAAE,CAAC;YACX,OAAO,EAAE,CAAC;YACV,OAAO,EAAE,CAAC;SACX,CAAC;QAEF,uBAAuB;QACvB,MAAM,UAAU,GAAkB;YAChC,CAAC,EAAE,IAAI,CAAC,QAAQ;YAChB,CAAC,EAAE,MAAM;YACT,CAAC,EAAE,IAAI,KAAK,CAAC,MAAM,EAAS;SAC7B,CAAC;QAEF,yCAAyC;QACzC,IAAI,UAAU,GAAkB;YAC9B,CAAC,EAAE,IAAI,CAAC,QAAQ;YAChB,CAAC,EAAE,MAAM;YACT,CAAC,EAAE,IAAI,KAAK,CAAC,MAAM,EAAS;SAC7B,CAAC;QAEF,OAAO,CAAC,IAAI,CAAC,MAAM,EAAE,OAAO,EAAE,CAAC;YAC7B,sDAAsD;YACtD,MAAM,MAAM,GAAG,IAAI,CAAC,EAAE,CAAC,WAAW,CAAC,SAAS,CAC1C;gBACE;oBACE,MAAM,EAAE;wBACN,GAAG,EAAE;4BACH,IAAI,EAAE,UAAU;4BAChB,GAAG,EAAE,UAAU;yBAChB;qBACF;iBACF;gBACD,EAAE,KAAK,EAAE,EAAE,GAAG,EAAE,CAAC,CAAC,EAAE,EAAE;gBACtB,EAAE,MAAM,EAAE,IAAI,CAAC,mBAAmB,EAAE;gBACpC;oBACE,QAAQ,EAAE;wBACR,GAAG,EAAE,CAAC;wBACN,EAAE,EAAE,CAAC;wBACL,KAAK,EAAE,CAAC;wBACR,MAAM,EAAE,CAAC;wBACT,YAAY,EAAE,CAAC;wBACf,UAAU,EAAE,CAAC;wBACb,QAAQ,EAAE,CAAC;wBACX,IAAI,EAAE,EAAE,SAAS,EAAE,QAAQ,EAAE;qBAC9B;iBACF;aACF,EACD;gBACE,2DAA2D;gBAC3D,+DAA+D;gBAC/D,SAAS,EAAE,IAAI,CAAC,mBAAmB,GAAG,CAAC;aACxC,CACF,CAAC;YACF,+GAA+G;YAC/G,oCAAoC;YACpC,MAAM,KAAK,GAAG,MAAM,MAAM,CAAC,OAAO,EAAE,CAAC;YAErC,IAAI,KAAK,CAAC,MAAM,IAAI,CAAC,EAAE,CAAC;gBACtB,wBAAwB;gBACxB,MAAM;YACR,CAAC;YAED,oCAAoC;YACpC,UAAU,GAAG,KAAK,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC;YAEzC,KAAK,IAAI,GAAG,IAAI,KAAK,EAAE,CAAC;gBACtB,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC,GAAG,IAAI,CAAC,OAAO,EAAE,CAAC;oBAC7B,SAAS;gBACX,CAAC;gBAED,YAAY,CAAC,QAAQ,GAAG,YAAY,CAAC,YAAY,CAAC,QAAQ,EAAE,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC,CAAC;gBAClF,YAAY,CAAC,OAAO,IAAI,CAAC,CAAC;gBAE1B,IAAI,eAAe,GAAG,GAAG,CAAC,EAAE,IAAI,KAAK,CAAC;gBAEtC,YAAY,CAAC,OAAO,IAAI,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;gBACzC,IAAI,GAAG,CAAC,EAAE,IAAI,QAAQ,IAAI,GAAG,CAAC,EAAE,IAAI,KAAK,EAAE,CAAC;oBAC1C,MAAM,GAAG,GAAG,GAAG,GAAG,CAAC,KAAK,IAAI,GAAG,CAAC,MAAM,IAAI,QAAQ,CAAC,GAAG,CAAC,YAAa,EAAE,GAAG,CAAC,UAAW,CAAC,EAAE,CAAC;oBACzF,MAAM,QAAQ,GAAG,YAAY,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;oBAC5C,IAAI,QAAQ,EAAE,CAAC;wBACb,8CAA8C;wBAC9C,eAAe,GAAG,KAAK,CAAC;wBAExB,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC;4BAChB,SAAS,EAAE;gCACT,MAAM,EAAE;oCACN,GAAG,EAAE,GAAG,CAAC,GAAG;iCACb;gCACD,MAAM,EAAE;oCACN,IAAI,EAAE;wCACJ,EAAE,EAAE,MAAM;wCACV,SAAS,EAAE,QAAQ;qCACpB;oCACD,MAAM,EAAE;wCACN,YAAY,EAAE,CAAC;wCACf,UAAU,EAAE,CAAC;wCACb,KAAK,EAAE,CAAC;wCACR,MAAM,EAAE,CAAC;wCACT,IAAI,EAAE,CAAC;qCACR;iCACF;6BACF;yBACF,CAAC,CAAC;wBAEH,YAAY,CAAC,OAAO,IAAI,GAAG,GAAG,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,iCAAiC;oBACnF,CAAC;yBAAM,CAAC;wBACN,IAAI,YAAY,CAAC,YAAY,IAAI,YAAY,EAAE,CAAC;4BAC9C,wBAAwB;4BACxB,6CAA6C;wBAC/C,CAAC;6BAAM,CAAC;4BACN,4DAA4D;4BAC5D,YAAY,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;4BACrD,6BAA6B;4BAC7B,oBAAoB;4BACpB,sBAAsB;4BACtB,6BAA6B;4BAC7B,YAAY,CAAC,YAAY,IAAI,GAAG,CAAC,MAAM,GAAG,GAAG,CAAC;wBAChD,CAAC;oBACH,CAAC;gBACH,CAAC;gBAED,IAAI,eAAe,EAAE,CAAC;oBACpB,YAAY,CAAC,UAAU,GAAG,IAAI,CAAC;oBAC/B,YAAY,CAAC,WAAW,GAAG,CAAC,CAAC;gBAC/B,CAAC;qBAAM,IAAI,GAAG,CAAC,EAAE,IAAI,OAAO,EAAE,CAAC;oBAC7B,IAAI,YAAY,CAAC,UAAU,IAAI,IAAI,EAAE,CAAC;wBACpC,YAAY,CAAC,UAAU,GAAG,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC;oBACtC,CAAC;oBACD,YAAY,CAAC,WAAW,IAAI,CAAC,CAAC;gBAChC,CAAC;gBAED,IAAI,IAAI,CAAC,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,kBAAkB,CAAC,MAAM,IAAI,IAAI,CAAC,cAAc,EAAE,CAAC;oBAChF,MAAM,IAAI,CAAC,KAAK,EAAE,CAAC;gBACrB,CAAC;YACH,CAAC;YAED,MAAM,CAAC,IAAI,CAAC,6BAA6B,KAAK,CAAC,MAAM,oBAAoB,MAAM,EAAE,CAAC,CAAC;QACrF,CAAC;QAED,qCAAqC;QACrC,YAAY,CAAC,IAAI,CAAC,KAAK,EAAE,CAAC;QAC1B,IAAI,YAAY,CAAC,UAAU,IAAI,IAAI,IAAI,YAAY,CAAC,WAAW,IAAI,CAAC,EAAE,CAAC;YACrE,MAAM,CAAC,IAAI,CACT,sBAAsB,IAAI,CAAC,QAAQ,IAAI,MAAM,IAAI,YAAY,CAAC,UAAU,cAAc,YAAY,CAAC,WAAW,aAAa,CAC5H,CAAC;YACF,8BAA8B;YAC9B,MAAM,IAAI,CAAC,KAAK,EAAE,CAAC;YACnB,MAAM,IAAI,CAAC,WAAW,CAAC,YAAY,CAAC,CAAC;QACvC,CAAC;QAED,+DAA+D;QAC/D,IAAI,CAAC,qBAAqB,CAAC,YAAY,CAAC,CAAC;QAEzC,mDAAmD;QACnD,MAAM,IAAI,CAAC,KAAK,EAAE,CAAC;IACrB,CAAC;IAED;;OAEG;IACK,qBAAqB,CAAC,KAAyB;QACrD,IAAI,KAAK,CAAC,OAAO,GAAG,CAAC,EAAE,CAAC;YACtB,MAAM,IAAI,qBAAqB,CAC7B,oBAAoB,KAAK,CAAC,OAAO,aAAa,KAAK,CAAC,QAAQ,iBAAiB,KAAK,CAAC,WAAW,aAAa,IAAI,CAAC,OAAO,EAAE,CAC1H,CAAC;QACJ,CAAC;QACD,IAAI,CAAC,kBAAkB,CAAC,IAAI,CAAC;YAC3B,SAAS,EAAE;gBACT,MAAM,EAAE;oBACN,GAAG,EAAE;wBACH,CAAC,EAAE,IAAI,CAAC,QAAQ;wBAChB,CAAC,EAAE,KAAK,CAAC,MAAM;qBAChB;iBACF;gBACD,MAAM,EAAE;oBACN,IAAI,EAAE;wBACJ,eAAe,EAAE;4BACf,KAAK,EAAE,IAAI,CAAC,OAAO;4BACnB,KAAK,EAAE,KAAK,CAAC,OAAO;4BACpB,QAAQ,EAAE,MAAM,CAAC,KAAK,CAAC,QAAQ,CAAC;4BAChC,KAAK,EAAE,KAAK,CAAC,OAAO;yBACrB;wBACD,sBAAsB,EAAE;4BACtB,sGAAsG;4BACtG,sCAAsC;4BACtC,oEAAoE;4BACpE,KAAK,EAAE,CAAC;4BACR,KAAK,EAAE,CAAC;yBACT;qBACF;iBACF;gBACD,wDAAwD;gBACxD,0FAA0F;gBAC1F,MAAM,EAAE,KAAK;aACd;SACF,CAAC,CAAC;IACL,CAAC;IAEO,KAAK,CAAC,KAAK;QACjB,IAAI,IAAI,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC5B,MAAM,CAAC,IAAI,CAAC,cAAc,IAAI,CAAC,OAAO,CAAC,MAAM,MAAM,CAAC,CAAC;YACrD,MAAM,IAAI,CAAC,EAAE,CAAC,WAAW,CAAC,SAAS,CAAC,IAAI,CAAC,OAAO,EAAE;gBAChD,0BAA0B;gBAC1B,8EAA8E;gBAC9E,yDAAyD;gBACzD,uCAAuC;gBACvC,OAAO,EAAE,KAAK;aACf,CAAC,CAAC;YACH,IAAI,CAAC,OAAO,GAAG,EAAE,CAAC;QACpB,CAAC;QACD,IAAI,IAAI,CAAC,kBAAkB,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACvC,MAAM,CAAC,IAAI,CAAC,YAAY,IAAI,CAAC,kBAAkB,CAAC,MAAM,gBAAgB,CAAC,CAAC;YACxE,MAAM,IAAI,CAAC,EAAE,CAAC,YAAY,CAAC,SAAS,CAAC,IAAI,CAAC,kBAAkB,EAAE;gBAC5D,OAAO,EAAE,KAAK;aACf,CAAC,CAAC;YACH,IAAI,CAAC,kBAAkB,GAAG,EAAE,CAAC;QAC/B,CAAC;IACH,CAAC;IAED;;;;;;OAMG;IACK,KAAK,CAAC,WAAW,CAAC,YAAgC;QACxD,MAAM,MAAM,GAAG,YAAY,CAAC,MAAM,CAAC;QACnC,MAAM,OAAO,GAAG,YAAY,CAAC,UAAW,CAAC;QAEzC,MAAM,QAAQ,GAAG;YACf,GAAG,EAAE;gBACH,IAAI,EAAE;oBACJ,CAAC,EAAE,IAAI,CAAC,QAAQ;oBAChB,CAAC,EAAE,MAAM;oBACT,CAAC,EAAE,IAAI,KAAK,CAAC,MAAM,EAAS;iBAC7B;gBACD,IAAI,EAAE;oBACJ,CAAC,EAAE,IAAI,CAAC,QAAQ;oBAChB,CAAC,EAAE,MAAM;oBACT,CAAC,EAAE,OAAO;iBACX;aACF;SACF,CAAC;QAEF,MAAM,OAAO,GAAG,IAAI,CAAC,EAAE,CAAC,MAAM,CAAC,YAAY,EAAE,CAAC;QAC9C,IAAI,CAAC;YACH,IAAI,IAAI,GAAG,KAAK,CAAC;YACjB,OAAO,CAAC,IAAI,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,OAAO,EAAE,CAAC;gBACtC,IAAI,WAAW,GAAG,CAAC,CAAC;gBACpB,6EAA6E;gBAC7E,kDAAkD;gBAClD,0EAA0E;gBAC1E,MAAM,OAAO,CAAC,eAAe,CAC3B,KAAK,IAAI,EAAE;oBACT,MAAM,KAAK,GAAG,IAAI,CAAC,EAAE,CAAC,WAAW,CAAC,IAAI,CAAC,QAAQ,EAAE;wBAC/C,OAAO;wBACP,IAAI,EAAE,EAAE,GAAG,EAAE,CAAC,EAAE;wBAChB,UAAU,EAAE;4BACV,GAAG,EAAE,CAAC;4BACN,EAAE,EAAE,CAAC;4BACL,QAAQ,EAAE,CAAC;4BACX,SAAS,EAAE,CAAC;yBACb;wBACD,KAAK,EAAE,IAAI,CAAC,eAAe;qBAC5B,CAAC,CAAC;oBACH,IAAI,QAAQ,GAAG,CAAC,CAAC;oBACjB,IAAI,QAAQ,GAAyB,IAAI,CAAC;oBAC1C,IAAI,QAAQ,GAAkB,IAAI,CAAC;oBACnC,IAAI,OAAO,GAAG,KAAK,CAAC;oBACpB,IAAI,kBAAkB,GAAG,CAAC,CAAC;oBAC3B,IAAI,KAAK,EAAE,IAAI,EAAE,IAAI,KAAK,CAAC,MAAM,EAAE,EAAE,CAAC;wBACpC,IAAI,EAAE,CAAC,EAAE,IAAI,MAAM,IAAI,EAAE,CAAC,EAAE,IAAI,QAAQ,IAAI,EAAE,CAAC,EAAE,IAAI,OAAO,EAAE,CAAC;4BAC7D,QAAQ,GAAG,KAAK,CAAC,YAAY,CAAC,QAAQ,EAAE,MAAM,CAAC,EAAE,CAAC,QAAQ,CAAC,CAAC,CAAC;4BAC7D,QAAQ,GAAG,EAAE,CAAC,GAAG,CAAC;4BAClB,kBAAkB,IAAI,CAAC,CAAC;4BACxB,IAAI,EAAE,CAAC,EAAE,IAAI,OAAO,EAAE,CAAC;gCACrB,OAAO,GAAG,IAAI,CAAC;4BACjB,CAAC;4BACD,IAAI,EAAE,CAAC,SAAS,IAAI,IAAI,EAAE,CAAC;gCACzB,IAAI,QAAQ,IAAI,IAAI,IAAI,EAAE,CAAC,SAAS,GAAG,QAAQ,EAAE,CAAC;oCAChD,QAAQ,GAAG,EAAE,CAAC,SAAS,CAAC;gCAC1B,CAAC;4BACH,CAAC;wBACH,CAAC;6BAAM,CAAC;4BACN,MAAM,IAAI,yBAAyB,CACjC,cAAc,EAAE,CAAC,EAAE,iBAAiB,EAAE,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC,GAAG,CAAC,CAAC,EAAE,CACvE,CAAC;wBACJ,CAAC;oBACH,CAAC;oBACD,IAAI,CAAC,OAAO,EAAE,CAAC;wBACb,IAAI,GAAG,IAAI,CAAC;wBACZ,OAAO;oBACT,CAAC;oBAED,MAAM,CAAC,IAAI,CAAC,sBAAsB,kBAAkB,WAAW,QAAQ,EAAE,CAAC,EAAE,CAAC,CAAC;oBAC9E,MAAM,IAAI,CAAC,EAAE,CAAC,WAAW,CAAC,UAAU,CAClC;wBACE,GAAG,EAAE;4BACH,IAAI,EAAE;gCACJ,CAAC,EAAE,IAAI,CAAC,QAAQ;gCAChB,CAAC,EAAE,MAAM;gCACT,CAAC,EAAE,IAAI,KAAK,CAAC,MAAM,EAAS;6BAC7B;4BACD,IAAI,EAAE,QAAS;yBAChB;qBACF,EACD,EAAE,OAAO,EAAE,CACZ,CAAC;oBAEF,MAAM,IAAI,CAAC,EAAE,CAAC,WAAW,CAAC,SAAS,CACjC;wBACE,GAAG,EAAE,QAAS;wBACd,EAAE,EAAE,OAAO;wBACX,QAAQ,EAAE,MAAM,CAAC,QAAQ,CAAC;wBAC1B,IAAI,EAAE,IAAI;wBACV,SAAS,EAAE,QAAQ;qBACpB,EACD,EAAE,OAAO,EAAE,CACZ,CAAC;oBAEF,WAAW,GAAG,CAAC,kBAAkB,GAAG,CAAC,CAAC;gBACxC,CAAC,EACD;oBACE,YAAY,EAAE,EAAE,CAAC,EAAE,UAAU,EAAE;oBAC/B,WAAW,EAAE,EAAE,KAAK,EAAE,UAAU,EAAE;iBACnC,CACF,CAAC;gBACF,yFAAyF;gBACzF,YAAY,CAAC,OAAO,IAAI,WAAW,CAAC;YACtC,CAAC;QACH,CAAC;gBAAS,CAAC;YACT,MAAM,OAAO,CAAC,UAAU,EAAE,CAAC;QAC7B,CAAC;IACH,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,iBAAiB,CAAC,OAAqC;QAC3D,IAAI,KAAK,GAAG,CAAC,CAAC;QACd,OAAO,CAAC,IAAI,CAAC,MAAM,EAAE,OAAO,EAAE,CAAC;YAC7B,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,gBAAgB,CAAC,OAAO,CAAC,CAAC;YACrD,IAAI,OAAO,CAAC,MAAM,IAAI,CAAC,EAAE,CAAC;gBACxB,WAAW;gBACX,MAAM;YACR,CAAC;YACD,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;YACzB,MAAM,CAAC,IAAI,CAAC,sCAAsC,OAAO,CAAC,MAAM,UAAU,CAAC,CAAC;YAE5E,2EAA2E;YAC3E,IAAI,YAAY,GAAmB,EAAE,CAAC;YACtC,IAAI,kBAAkB,GAAG,CAAC,CAAC;YAC3B,KAAK,IAAI,MAAM,IAAI,OAAO,EAAE,CAAC;gBAC3B,YAAY,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;gBAC1B,kBAAkB,IAAI,MAAM,CAAC,cAAc,CAAC;gBAC5C,IAAI,kBAAkB,GAAG,MAAM,EAAE,CAAC;oBAChC,MAAM;gBACR,CAAC;YACH,CAAC;YACD,MAAM,IAAI,CAAC,oBAAoB,CAAC,YAAY,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC;YACnE,MAAM,CAAC,IAAI,CAAC,kCAAkC,YAAY,CAAC,MAAM,eAAe,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK,IAAI,CAAC,CAAC;YACxG,KAAK,IAAI,OAAO,CAAC,MAAM,CAAC;QAC1B,CAAC;QACD,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC;IAC5B,CAAC;IAED;;;;;OAKG;IACK,KAAK,CAAC,gBAAgB,CAAC,OAG9B;QACC,IAAI,OAAO,CAAC,gBAAgB,IAAI,CAAC,EAAE,CAAC;YAClC,MAAM,IAAI,yBAAyB,CAAC,+BAA+B,CAAC,CAAC;QACvE,CAAC;QACD,4EAA4E;QAC5E,MAAM,YAAY,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,YAAY;aAC5C,IAAI,CACH;YACE,OAAO,EAAE,IAAI,CAAC,QAAQ;YACtB,8BAA8B,EAAE,EAAE,IAAI,EAAE,OAAO,CAAC,gBAAgB,EAAE;YAClE,OAAO,EAAE,EAAE,IAAI,EAAE,OAAO,CAAC,OAAO,IAAI,EAAE,EAAE;SACzC,EACD;YACE,UAAU,EAAE;gBACV,GAAG,EAAE,CAAC;gBACN,sBAAsB,EAAE,CAAC;gBACzB,eAAe,EAAE,CAAC;aACnB;YACD,IAAI,EAAE;gBACJ,8BAA8B,EAAE,CAAC,CAAC;aACnC;YACD,KAAK,EAAE,GAAG;YACV,SAAS,EAAE,0BAA0B;SACtC,CACF;aACA,OAAO,EAAE,CAAC;QAEb,OAAO,YAAY,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;YACnC,MAAM,EAAE,MAAM,CAAC,GAAG,CAAC,CAAC;YACpB,cAAc,EAAE,MAAM,CAAC,sBAAuB,CAAC,KAAK,GAAG,CAAC,MAAM,CAAC,eAAe,EAAE,KAAK,IAAI,CAAC,CAAC;SAC5F,CAAC,CAAC,CAAC;IACN,CAAC;IAEO,KAAK,CAAC,oBAAoB,CAAC,OAAiB;QAClD,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC,6BAA6B,CAC1E,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE;YACrB,OAAO;gBACL,MAAM;gBACN,GAAG,EAAE,IAAI,CAAC,OAAO;aAClB,CAAC;QACJ,CAAC,CAAC,CACH,CAAC;QAEF,KAAK,IAAI,cAAc,IAAI,SAAS,CAAC,MAAM,EAAE,EAAE,CAAC;YAC9C,IAAI,iBAAiB,CAAC,cAAc,CAAC,EAAE,CAAC;gBACtC,qDAAqD;gBACrD,MAAM,IAAI,qBAAqB,CAAC,+BAA+B,IAAI,CAAC,SAAS,CAAC,cAAc,CAAC,EAAE,CAAC,CAAC;YACnG,CAAC;YAED,IAAI,CAAC,kBAAkB,CAAC,IAAI,CAAC;gBAC3B,SAAS,EAAE;oBACT,MAAM,EAAE;wBACN,GAAG,EAAE;4BACH,CAAC,EAAE,IAAI,CAAC,QAAQ;4BAChB,CAAC,EAAE,cAAc,CAAC,MAAM;yBACzB;qBACF;oBACD,MAAM,EAAE;wBACN,IAAI,EAAE;4BACJ,eAAe,EAAE;gCACf,KAAK,EAAE,IAAI,CAAC,OAAO;gCACnB,KAAK,EAAE,cAAc,CAAC,KAAK;gCAC3B,QAAQ,EAAE,MAAM,CAAC,cAAc,CAAC,QAAQ,CAAC;gCACzC,KAAK,EAAE,IAAI;6BACZ;4BACD,sBAAsB,EAAE;gCACtB,KAAK,EAAE,CAAC;gCACR,KAAK,EAAE,CAAC;6BACT;yBACF;qBACF;oBACD,oGAAoG;oBACpG,4CAA4C;oBAC5C,MAAM,EAAE,KAAK;iBACd;aACF,CAAC,CAAC;QACL,CAAC;QAED,MAAM,IAAI,CAAC,KAAK,EAAE,CAAC;IACrB,CAAC;CACF"}
|
|
@@ -23,7 +23,14 @@ export class MongoSyncRulesLock {
|
|
|
23
23
|
returnDocument: 'before'
|
|
24
24
|
});
|
|
25
25
|
if (doc == null) {
|
|
26
|
-
|
|
26
|
+
// Query the existing lock to get the expiration time (best effort - it may have been released in the meantime).
|
|
27
|
+
const heldLock = await db.sync_rules.findOne({ _id: sync_rules.id }, { projection: { lock: 1 } });
|
|
28
|
+
if (heldLock?.lock?.expires_at) {
|
|
29
|
+
throw new ServiceError(ErrorCode.PSYNC_S1003, `Sync rules: ${sync_rules.id} have been locked by another process for replication, expiring at ${heldLock.lock.expires_at.toISOString()}.`);
|
|
30
|
+
}
|
|
31
|
+
else {
|
|
32
|
+
throw new ServiceError(ErrorCode.PSYNC_S1003, `Sync rules: ${sync_rules.id} have been locked by another process for replication.`);
|
|
33
|
+
}
|
|
27
34
|
}
|
|
28
35
|
return new MongoSyncRulesLock(db, sync_rules.id, lockId);
|
|
29
36
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"MongoSyncRulesLock.js","sourceRoot":"","sources":["../../../src/storage/implementation/MongoSyncRulesLock.ts"],"names":[],"mappings":"AAAA,OAAO,MAAM,MAAM,QAAQ,CAAC;AAE5B,OAAO,EAAE,SAAS,EAAE,MAAM,EAAE,YAAY,EAAE,MAAM,mCAAmC,CAAC;AAIpF;;;GAGG;AACH,MAAM,OAAO,kBAAkB;
|
|
1
|
+
{"version":3,"file":"MongoSyncRulesLock.js","sourceRoot":"","sources":["../../../src/storage/implementation/MongoSyncRulesLock.ts"],"names":[],"mappings":"AAAA,OAAO,MAAM,MAAM,QAAQ,CAAC;AAE5B,OAAO,EAAE,SAAS,EAAE,MAAM,EAAE,YAAY,EAAE,MAAM,mCAAmC,CAAC;AAIpF;;;GAGG;AACH,MAAM,OAAO,kBAAkB;IA2CnB;IACD;IACC;IA5CO,eAAe,CAAiB;IAEjD,MAAM,CAAC,KAAK,CAAC,UAAU,CACrB,EAAkB,EAClB,UAA6C;QAE7C,MAAM,MAAM,GAAG,MAAM,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;QACrD,MAAM,GAAG,GAAG,MAAM,EAAE,CAAC,UAAU,CAAC,gBAAgB,CAC9C,EAAE,GAAG,EAAE,UAAU,CAAC,EAAE,EAAE,GAAG,EAAE,CAAC,EAAE,IAAI,EAAE,IAAI,EAAE,EAAE,EAAE,iBAAiB,EAAE,EAAE,GAAG,EAAE,IAAI,IAAI,EAAE,EAAE,EAAE,CAAC,EAAE,EACzF;YACE,IAAI,EAAE;gBACJ,IAAI,EAAE;oBACJ,EAAE,EAAE,MAAM;oBACV,UAAU,EAAE,IAAI,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,IAAI,CAAC;iBAC7C;aACF;SACF,EACD;YACE,UAAU,EAAE,EAAE,IAAI,EAAE,CAAC,EAAE;YACvB,cAAc,EAAE,QAAQ;SACzB,CACF,CAAC;QAEF,IAAI,GAAG,IAAI,IAAI,EAAE,CAAC;YAChB,gHAAgH;YAChH,MAAM,QAAQ,GAAG,MAAM,EAAE,CAAC,UAAU,CAAC,OAAO,CAAC,EAAE,GAAG,EAAE,UAAU,CAAC,EAAE,EAAE,EAAE,EAAE,UAAU,EAAE,EAAE,IAAI,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC;YAClG,IAAI,QAAQ,EAAE,IAAI,EAAE,UAAU,EAAE,CAAC;gBAC/B,MAAM,IAAI,YAAY,CACpB,SAAS,CAAC,WAAW,EACrB,eAAe,UAAU,CAAC,EAAE,qEAAqE,QAAQ,CAAC,IAAI,CAAC,UAAU,CAAC,WAAW,EAAE,GAAG,CAC3I,CAAC;YACJ,CAAC;iBAAM,CAAC;gBACN,MAAM,IAAI,YAAY,CACpB,SAAS,CAAC,WAAW,EACrB,eAAe,UAAU,CAAC,EAAE,uDAAuD,CACpF,CAAC;YACJ,CAAC;QACH,CAAC;QACD,OAAO,IAAI,kBAAkB,CAAC,EAAE,EAAE,UAAU,CAAC,EAAE,EAAE,MAAM,CAAC,CAAC;IAC3D,CAAC;IAED,YACU,EAAkB,EACnB,aAAqB,EACpB,OAAe;QAFf,OAAE,GAAF,EAAE,CAAgB;QACnB,kBAAa,GAAb,aAAa,CAAQ;QACpB,YAAO,GAAP,OAAO,CAAQ;QAEvB,IAAI,CAAC,eAAe,GAAG,WAAW,CAAC,KAAK,IAAI,EAAE;YAC5C,IAAI,CAAC;gBACH,MAAM,IAAI,CAAC,OAAO,EAAE,CAAC;YACvB,CAAC;YAAC,OAAO,CAAC,EAAE,CAAC;gBACX,MAAM,CAAC,KAAK,CAAC,wBAAwB,EAAE,CAAC,CAAC,CAAC;gBAC1C,aAAa,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC;YACtC,CAAC;QACH,CAAC,EAAE,MAAM,CAAC,CAAC;IACb,CAAC;IAED,KAAK,CAAC,OAAO;QACX,aAAa,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC;QACpC,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,SAAS,CAC/C;YACE,GAAG,EAAE,IAAI,CAAC,aAAa;YACvB,SAAS,EAAE,IAAI,CAAC,OAAO;SACxB,EACD;YACE,MAAM,EAAE,EAAE,IAAI,EAAE,CAAC,EAAE;SACpB,CACF,CAAC;QACF,IAAI,MAAM,CAAC,aAAa,IAAI,CAAC,EAAE,CAAC;YAC9B,iBAAiB;YACjB,MAAM,CAAC,IAAI,CAAC,0BAA0B,IAAI,CAAC,aAAa,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC,CAAC;QAC9E,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,OAAO;QACnB,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,gBAAgB,CACtD;YACE,GAAG,EAAE,IAAI,CAAC,aAAa;YACvB,SAAS,EAAE,IAAI,CAAC,OAAO;SACxB,EACD;YACE,IAAI,EAAE,EAAE,iBAAiB,EAAE,IAAI,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,IAAI,CAAC,EAAE;SAC9D,EACD,EAAE,cAAc,EAAE,OAAO,EAAE,CAC5B,CAAC;QACF,IAAI,MAAM,IAAI,IAAI,EAAE,CAAC;YACnB,MAAM,IAAI,KAAK,CAAC,0BAA0B,IAAI,CAAC,aAAa,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC,CAAC;QAClF,CAAC;IACH,CAAC;CACF"}
|
package/package.json
CHANGED
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
"name": "@powersync/service-module-mongodb-storage",
|
|
3
3
|
"repository": "https://github.com/powersync-ja/powersync-service",
|
|
4
4
|
"types": "dist/index.d.ts",
|
|
5
|
-
"version": "0.12.
|
|
5
|
+
"version": "0.12.12",
|
|
6
6
|
"main": "dist/index.js",
|
|
7
7
|
"license": "FSL-1.1-ALv2",
|
|
8
8
|
"type": "module",
|
|
@@ -27,15 +27,15 @@
|
|
|
27
27
|
"lru-cache": "^10.2.2",
|
|
28
28
|
"ts-codec": "^1.3.0",
|
|
29
29
|
"uuid": "^11.1.0",
|
|
30
|
-
"@powersync/lib-service-mongodb": "0.6.
|
|
31
|
-
"@powersync/lib-services-framework": "0.7.
|
|
32
|
-
"@powersync/service-core": "1.16.
|
|
30
|
+
"@powersync/lib-service-mongodb": "0.6.12",
|
|
31
|
+
"@powersync/lib-services-framework": "0.7.10",
|
|
32
|
+
"@powersync/service-core": "1.16.3",
|
|
33
33
|
"@powersync/service-jsonbig": "0.17.12",
|
|
34
|
-
"@powersync/service-sync-rules": "0.29.
|
|
34
|
+
"@powersync/service-sync-rules": "0.29.7",
|
|
35
35
|
"@powersync/service-types": "0.13.1"
|
|
36
36
|
},
|
|
37
37
|
"devDependencies": {
|
|
38
|
-
"@powersync/service-core-tests": "0.12.
|
|
38
|
+
"@powersync/service-core-tests": "0.12.12"
|
|
39
39
|
},
|
|
40
40
|
"scripts": {
|
|
41
41
|
"build": "tsc -b",
|
|
@@ -42,11 +42,7 @@ const DEFAULT_OPERATION_BATCH_LIMIT = 50_000;
|
|
|
42
42
|
* 4. computePartialChecksumsInternal() -> aggregate over 50_000 operations in bucket_data at a time
|
|
43
43
|
*/
|
|
44
44
|
export class MongoChecksums {
|
|
45
|
-
private
|
|
46
|
-
fetchChecksums: (batch) => {
|
|
47
|
-
return this.computePartialChecksums(batch);
|
|
48
|
-
}
|
|
49
|
-
});
|
|
45
|
+
private _cache: ChecksumCache | undefined;
|
|
50
46
|
|
|
51
47
|
constructor(
|
|
52
48
|
private db: PowerSyncMongo,
|
|
@@ -54,6 +50,20 @@ export class MongoChecksums {
|
|
|
54
50
|
private options?: MongoChecksumOptions
|
|
55
51
|
) {}
|
|
56
52
|
|
|
53
|
+
/**
|
|
54
|
+
* Lazy-instantiated cache.
|
|
55
|
+
*
|
|
56
|
+
* This means the cache only allocates memory once it is used for the first time.
|
|
57
|
+
*/
|
|
58
|
+
private get cache(): ChecksumCache {
|
|
59
|
+
this._cache ??= new ChecksumCache({
|
|
60
|
+
fetchChecksums: (batch) => {
|
|
61
|
+
return this.computePartialChecksums(batch);
|
|
62
|
+
}
|
|
63
|
+
});
|
|
64
|
+
return this._cache;
|
|
65
|
+
}
|
|
66
|
+
|
|
57
67
|
/**
|
|
58
68
|
* Calculate checksums, utilizing the cache for partial checkums, and querying the remainder from
|
|
59
69
|
* the database (bucket_state + bucket_data).
|
|
@@ -159,7 +169,7 @@ export class MongoChecksums {
|
|
|
159
169
|
// Limit the number of buckets we query for at a time.
|
|
160
170
|
const bucketBatchLimit = this.options?.bucketBatchLimit ?? DEFAULT_BUCKET_BATCH_LIMIT;
|
|
161
171
|
|
|
162
|
-
if (batch.length
|
|
172
|
+
if (batch.length <= bucketBatchLimit) {
|
|
163
173
|
// Single batch - no need for splitting the batch and merging results
|
|
164
174
|
return await this.computePartialChecksumsInternal(batch);
|
|
165
175
|
}
|
|
@@ -61,6 +61,7 @@ export interface MongoCompactOptions extends storage.CompactOptions {}
|
|
|
61
61
|
const DEFAULT_CLEAR_BATCH_LIMIT = 5000;
|
|
62
62
|
const DEFAULT_MOVE_BATCH_LIMIT = 2000;
|
|
63
63
|
const DEFAULT_MOVE_BATCH_QUERY_LIMIT = 10_000;
|
|
64
|
+
const DEFAULT_MIN_BUCKET_CHANGES = 10;
|
|
64
65
|
|
|
65
66
|
/** This default is primarily for tests. */
|
|
66
67
|
const DEFAULT_MEMORY_LIMIT_MB = 64;
|
|
@@ -73,6 +74,7 @@ export class MongoCompactor {
|
|
|
73
74
|
private moveBatchLimit: number;
|
|
74
75
|
private moveBatchQueryLimit: number;
|
|
75
76
|
private clearBatchLimit: number;
|
|
77
|
+
private minBucketChanges: number;
|
|
76
78
|
private maxOpId: bigint;
|
|
77
79
|
private buckets: string[] | undefined;
|
|
78
80
|
private signal?: AbortSignal;
|
|
@@ -88,6 +90,7 @@ export class MongoCompactor {
|
|
|
88
90
|
this.moveBatchLimit = options?.moveBatchLimit ?? DEFAULT_MOVE_BATCH_LIMIT;
|
|
89
91
|
this.moveBatchQueryLimit = options?.moveBatchQueryLimit ?? DEFAULT_MOVE_BATCH_QUERY_LIMIT;
|
|
90
92
|
this.clearBatchLimit = options?.clearBatchLimit ?? DEFAULT_CLEAR_BATCH_LIMIT;
|
|
93
|
+
this.minBucketChanges = options?.minBucketChanges ?? DEFAULT_MIN_BUCKET_CHANGES;
|
|
91
94
|
this.maxOpId = options?.maxOpId ?? 0n;
|
|
92
95
|
this.buckets = options?.compactBuckets;
|
|
93
96
|
this.signal = options?.signal;
|
|
@@ -113,14 +116,26 @@ export class MongoCompactor {
|
|
|
113
116
|
|
|
114
117
|
private async compactDirtyBuckets() {
|
|
115
118
|
while (!this.signal?.aborted) {
|
|
116
|
-
// Process all buckets with
|
|
117
|
-
|
|
119
|
+
// Process all buckets with 10 or more changes since last time.
|
|
120
|
+
// We exclude the last 100 compacted buckets, to avoid repeatedly re-compacting the same buckets over and over
|
|
121
|
+
// if they are modified while compacting.
|
|
122
|
+
const TRACK_RECENTLY_COMPACTED_NUMBER = 100;
|
|
123
|
+
|
|
124
|
+
let recentlyCompacted: string[] = [];
|
|
125
|
+
const buckets = await this.dirtyBucketBatch({
|
|
126
|
+
minBucketChanges: this.minBucketChanges,
|
|
127
|
+
exclude: recentlyCompacted
|
|
128
|
+
});
|
|
118
129
|
if (buckets.length == 0) {
|
|
119
130
|
// All done
|
|
120
131
|
break;
|
|
121
132
|
}
|
|
122
|
-
for (let bucket of buckets) {
|
|
133
|
+
for (let { bucket } of buckets) {
|
|
123
134
|
await this.compactSingleBucket(bucket);
|
|
135
|
+
recentlyCompacted.push(bucket);
|
|
136
|
+
}
|
|
137
|
+
if (recentlyCompacted.length > TRACK_RECENTLY_COMPACTED_NUMBER) {
|
|
138
|
+
recentlyCompacted = recentlyCompacted.slice(-TRACK_RECENTLY_COMPACTED_NUMBER);
|
|
124
139
|
}
|
|
125
140
|
}
|
|
126
141
|
}
|
|
@@ -482,10 +497,20 @@ export class MongoCompactor {
|
|
|
482
497
|
break;
|
|
483
498
|
}
|
|
484
499
|
const start = Date.now();
|
|
485
|
-
logger.info(`Calculating checksums for batch of ${buckets.length} buckets
|
|
500
|
+
logger.info(`Calculating checksums for batch of ${buckets.length} buckets`);
|
|
486
501
|
|
|
487
|
-
|
|
488
|
-
|
|
502
|
+
// Filter batch by estimated bucket size, to reduce possibility of timeouts
|
|
503
|
+
let checkBuckets: typeof buckets = [];
|
|
504
|
+
let totalCountEstimate = 0;
|
|
505
|
+
for (let bucket of buckets) {
|
|
506
|
+
checkBuckets.push(bucket);
|
|
507
|
+
totalCountEstimate += bucket.estimatedCount;
|
|
508
|
+
if (totalCountEstimate > 50_000) {
|
|
509
|
+
break;
|
|
510
|
+
}
|
|
511
|
+
}
|
|
512
|
+
await this.updateChecksumsBatch(checkBuckets.map((b) => b.bucket));
|
|
513
|
+
logger.info(`Updated checksums for batch of ${checkBuckets.length} buckets in ${Date.now() - start}ms`);
|
|
489
514
|
count += buckets.length;
|
|
490
515
|
}
|
|
491
516
|
return { buckets: count };
|
|
@@ -497,7 +522,10 @@ export class MongoCompactor {
|
|
|
497
522
|
* This cannot be used to iterate on its own - the client is expected to process these buckets and
|
|
498
523
|
* set estimate_since_compact.count: 0 when done, before fetching the next batch.
|
|
499
524
|
*/
|
|
500
|
-
private async dirtyBucketBatch(options: {
|
|
525
|
+
private async dirtyBucketBatch(options: {
|
|
526
|
+
minBucketChanges: number;
|
|
527
|
+
exclude?: string[];
|
|
528
|
+
}): Promise<{ bucket: string; estimatedCount: number }[]> {
|
|
501
529
|
if (options.minBucketChanges <= 0) {
|
|
502
530
|
throw new ReplicationAssertionError('minBucketChanges must be >= 1');
|
|
503
531
|
}
|
|
@@ -506,22 +534,28 @@ export class MongoCompactor {
|
|
|
506
534
|
.find(
|
|
507
535
|
{
|
|
508
536
|
'_id.g': this.group_id,
|
|
509
|
-
'estimate_since_compact.count': { $gte: options.minBucketChanges }
|
|
537
|
+
'estimate_since_compact.count': { $gte: options.minBucketChanges },
|
|
538
|
+
'_id.b': { $nin: options.exclude ?? [] }
|
|
510
539
|
},
|
|
511
540
|
{
|
|
512
541
|
projection: {
|
|
513
|
-
_id: 1
|
|
542
|
+
_id: 1,
|
|
543
|
+
estimate_since_compact: 1,
|
|
544
|
+
compacted_state: 1
|
|
514
545
|
},
|
|
515
546
|
sort: {
|
|
516
547
|
'estimate_since_compact.count': -1
|
|
517
548
|
},
|
|
518
|
-
limit:
|
|
549
|
+
limit: 200,
|
|
519
550
|
maxTimeMS: MONGO_OPERATION_TIMEOUT_MS
|
|
520
551
|
}
|
|
521
552
|
)
|
|
522
553
|
.toArray();
|
|
523
554
|
|
|
524
|
-
return dirtyBuckets.map((bucket) =>
|
|
555
|
+
return dirtyBuckets.map((bucket) => ({
|
|
556
|
+
bucket: bucket._id.b,
|
|
557
|
+
estimatedCount: bucket.estimate_since_compact!.count + (bucket.compacted_state?.count ?? 0)
|
|
558
|
+
}));
|
|
525
559
|
}
|
|
526
560
|
|
|
527
561
|
private async updateChecksumsBatch(buckets: string[]) {
|
|
@@ -33,10 +33,19 @@ export class MongoSyncRulesLock implements storage.ReplicationLock {
|
|
|
33
33
|
);
|
|
34
34
|
|
|
35
35
|
if (doc == null) {
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
36
|
+
// Query the existing lock to get the expiration time (best effort - it may have been released in the meantime).
|
|
37
|
+
const heldLock = await db.sync_rules.findOne({ _id: sync_rules.id }, { projection: { lock: 1 } });
|
|
38
|
+
if (heldLock?.lock?.expires_at) {
|
|
39
|
+
throw new ServiceError(
|
|
40
|
+
ErrorCode.PSYNC_S1003,
|
|
41
|
+
`Sync rules: ${sync_rules.id} have been locked by another process for replication, expiring at ${heldLock.lock.expires_at.toISOString()}.`
|
|
42
|
+
);
|
|
43
|
+
} else {
|
|
44
|
+
throw new ServiceError(
|
|
45
|
+
ErrorCode.PSYNC_S1003,
|
|
46
|
+
`Sync rules: ${sync_rules.id} have been locked by another process for replication.`
|
|
47
|
+
);
|
|
48
|
+
}
|
|
40
49
|
}
|
|
41
50
|
return new MongoSyncRulesLock(db, sync_rules.id, lockId);
|
|
42
51
|
}
|