@powersync/service-core 0.0.0-dev-20241101083236 → 0.0.0-dev-20241107065634
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +21 -8
- package/dist/entry/commands/compact-action.js +11 -3
- package/dist/entry/commands/compact-action.js.map +1 -1
- package/dist/storage/BucketStorage.d.ts +2 -0
- package/dist/storage/mongo/MongoCompactor.js +18 -2
- package/dist/storage/mongo/MongoCompactor.js.map +1 -1
- package/package.json +6 -6
- package/src/entry/commands/compact-action.ts +10 -3
- package/src/storage/BucketStorage.ts +2 -0
- package/src/storage/mongo/MongoCompactor.ts +18 -2
- package/tsconfig.tsbuildinfo +1 -1
package/CHANGELOG.md
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
# @powersync/service-core
|
|
2
2
|
|
|
3
|
-
## 0.0.0-dev-
|
|
3
|
+
## 0.0.0-dev-20241107065634
|
|
4
4
|
|
|
5
5
|
### Minor Changes
|
|
6
6
|
|
|
@@ -17,9 +17,7 @@
|
|
|
17
17
|
|
|
18
18
|
### Patch Changes
|
|
19
19
|
|
|
20
|
-
-
|
|
21
|
-
- 17a6db0: Fix storageStats error in metrics endpoint when collections don't exist.
|
|
22
|
-
- 271050f: Improved sync rules storage cached parsed sync rules, accommodating different parsing options where necessary.
|
|
20
|
+
- ac03d8f: Improved sync rules storage cached parsed sync rules, accommodating different parsing options where necessary.
|
|
23
21
|
- e4d19b2: Moved tag variable initialization in diagnostics route to ensure it is initialized before usage
|
|
24
22
|
- Updated dependencies [d2ece1b]
|
|
25
23
|
- Updated dependencies [d2ece1b]
|
|
@@ -27,10 +25,25 @@
|
|
|
27
25
|
- Updated dependencies [d2ece1b]
|
|
28
26
|
- Updated dependencies [d51921f]
|
|
29
27
|
- Updated dependencies [4ecaee2]
|
|
30
|
-
- @powersync/lib-services-framework@0.0.0-dev-
|
|
31
|
-
- @powersync/service-sync-rules@0.0.0-dev-
|
|
32
|
-
- @powersync/service-rsocket-router@0.0.0-dev-
|
|
33
|
-
- @powersync/service-types@0.0.0-dev-
|
|
28
|
+
- @powersync/lib-services-framework@0.0.0-dev-20241107065634
|
|
29
|
+
- @powersync/service-sync-rules@0.0.0-dev-20241107065634
|
|
30
|
+
- @powersync/service-rsocket-router@0.0.0-dev-20241107065634
|
|
31
|
+
- @powersync/service-types@0.0.0-dev-20241107065634
|
|
32
|
+
|
|
33
|
+
## 0.8.7
|
|
34
|
+
|
|
35
|
+
### Patch Changes
|
|
36
|
+
|
|
37
|
+
- 6b72e6c: Improved Postgres connection port restrictions. Connections are now supported on ports >= 1024.
|
|
38
|
+
|
|
39
|
+
## 0.8.6
|
|
40
|
+
|
|
41
|
+
### Patch Changes
|
|
42
|
+
|
|
43
|
+
- 2d3bb6a: Fix "operation exceeded time limit" error
|
|
44
|
+
- 17a6db0: Fix storageStats error in metrics endpoint when collections don't exist.
|
|
45
|
+
- Updated dependencies [0f90b02]
|
|
46
|
+
- @powersync/service-sync-rules@0.20.0
|
|
34
47
|
|
|
35
48
|
## 0.8.5
|
|
36
49
|
|
|
@@ -67,10 +67,18 @@ const HEAP_LIMIT = v8.getHeapStatistics().heap_size_limit;
|
|
|
67
67
|
*/
|
|
68
68
|
const COMPACT_MEMORY_LIMIT_MB = Math.min(HEAP_LIMIT / 1024 / 1024 - 128, 1024);
|
|
69
69
|
export function registerCompactAction(program) {
|
|
70
|
-
const compactCommand = program
|
|
70
|
+
const compactCommand = program
|
|
71
|
+
.command(COMMAND_NAME)
|
|
72
|
+
.option(`-b, --buckets [buckets]`, 'Bucket or bucket definition name (optional, comma-separate multiple names)');
|
|
71
73
|
wrapConfigCommand(compactCommand);
|
|
72
74
|
return compactCommand.description('Compact storage').action(async (options) => {
|
|
73
|
-
|
|
75
|
+
const buckets = options.buckets?.split(',');
|
|
76
|
+
if (buckets != null) {
|
|
77
|
+
logger.info('Compacting storage for all buckets...');
|
|
78
|
+
}
|
|
79
|
+
else {
|
|
80
|
+
logger.info(`Compacting storage for ${buckets.join(', ')}...`);
|
|
81
|
+
}
|
|
74
82
|
const runnerConfig = extractRunnerOptions(options);
|
|
75
83
|
const configuration = await utils.loadConfig(runnerConfig);
|
|
76
84
|
logger.info('Successfully loaded configuration...');
|
|
@@ -92,7 +100,7 @@ export function registerCompactAction(program) {
|
|
|
92
100
|
}
|
|
93
101
|
const p = __addDisposableResource(env_1, bucketStorage.getInstance(active), false);
|
|
94
102
|
logger.info('Performing compaction...');
|
|
95
|
-
await p.compact({ memoryLimitMB: COMPACT_MEMORY_LIMIT_MB });
|
|
103
|
+
await p.compact({ memoryLimitMB: COMPACT_MEMORY_LIMIT_MB, compactBuckets: buckets });
|
|
96
104
|
logger.info('Successfully compacted storage.');
|
|
97
105
|
}
|
|
98
106
|
catch (e_1) {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"compact-action.js","sourceRoot":"","sources":["../../../src/entry/commands/compact-action.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAEA,OAAO,EAAE,MAAM,EAAE,MAAM,mCAAmC,CAAC;AAC3D,OAAO,KAAK,EAAE,MAAM,IAAI,CAAC;AACzB,OAAO,KAAK,OAAO,MAAM,gCAAgC,CAAC;AAC1D,OAAO,KAAK,KAAK,MAAM,0BAA0B,CAAC;AAClD,OAAO,EAAE,oBAAoB,EAAE,iBAAiB,EAAE,MAAM,qBAAqB,CAAC;AAE9E,MAAM,YAAY,GAAG,SAAS,CAAC;AAE/B;;GAEG;AACH,MAAM,UAAU,GAAG,EAAE,CAAC,iBAAiB,EAAE,CAAC,eAAe,CAAC;AAE1D;;;;GAIG;AACH,MAAM,uBAAuB,GAAG,IAAI,CAAC,GAAG,CAAC,UAAU,GAAG,IAAI,GAAG,IAAI,GAAG,GAAG,EAAE,IAAI,CAAC,CAAC;AAE/E,MAAM,UAAU,qBAAqB,CAAC,OAAgB;IACpD,MAAM,cAAc,GAAG,OAAO,
|
|
1
|
+
{"version":3,"file":"compact-action.js","sourceRoot":"","sources":["../../../src/entry/commands/compact-action.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAEA,OAAO,EAAE,MAAM,EAAE,MAAM,mCAAmC,CAAC;AAC3D,OAAO,KAAK,EAAE,MAAM,IAAI,CAAC;AACzB,OAAO,KAAK,OAAO,MAAM,gCAAgC,CAAC;AAC1D,OAAO,KAAK,KAAK,MAAM,0BAA0B,CAAC;AAClD,OAAO,EAAE,oBAAoB,EAAE,iBAAiB,EAAE,MAAM,qBAAqB,CAAC;AAE9E,MAAM,YAAY,GAAG,SAAS,CAAC;AAE/B;;GAEG;AACH,MAAM,UAAU,GAAG,EAAE,CAAC,iBAAiB,EAAE,CAAC,eAAe,CAAC;AAE1D;;;;GAIG;AACH,MAAM,uBAAuB,GAAG,IAAI,CAAC,GAAG,CAAC,UAAU,GAAG,IAAI,GAAG,IAAI,GAAG,GAAG,EAAE,IAAI,CAAC,CAAC;AAE/E,MAAM,UAAU,qBAAqB,CAAC,OAAgB;IACpD,MAAM,cAAc,GAAG,OAAO;SAC3B,OAAO,CAAC,YAAY,CAAC;SACrB,MAAM,CAAC,yBAAyB,EAAE,4EAA4E,CAAC,CAAC;IAEnH,iBAAiB,CAAC,cAAc,CAAC,CAAC;IAElC,OAAO,cAAc,CAAC,WAAW,CAAC,iBAAiB,CAAC,CAAC,MAAM,CAAC,KAAK,EAAE,OAAO,EAAE,EAAE;QAC5E,MAAM,OAAO,GAAG,OAAO,CAAC,OAAO,EAAE,KAAK,CAAC,GAAG,CAAC,CAAC;QAC5C,IAAI,OAAO,IAAI,IAAI,EAAE,CAAC;YACpB,MAAM,CAAC,IAAI,CAAC,uCAAuC,CAAC,CAAC;QACvD,CAAC;aAAM,CAAC;YACN,MAAM,CAAC,IAAI,CAAC,0BAA0B,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QACjE,CAAC;QACD,MAAM,YAAY,GAAG,oBAAoB,CAAC,OAAO,CAAC,CAAC;QACnD,MAAM,aAAa,GAAG,MAAM,KAAK,CAAC,UAAU,CAAC,YAAY,CAAC,CAAC;QAC3D,MAAM,CAAC,IAAI,CAAC,sCAAsC,CAAC,CAAC;QACpD,MAAM,EAAE,OAAO,EAAE,aAAa,EAAE,GAAG,aAAa,CAAC;QACjD,MAAM,CAAC,IAAI,CAAC,0BAA0B,CAAC,CAAC;QACxC,MAAM,IAAI,GAAG,OAAO,CAAC,oBAAoB,CAAC,aAAa,CAAC,CAAC;QACzD,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC;QAC3B,MAAM,MAAM,CAAC,OAAO,EAAE,CAAC;QACvB,IAAI,CAAC;;;gBACH,MAAM,aAAa,GAAG,IAAI,OAAO,CAAC,kBAAkB,CAAC,IAAI,EAAE;oBACzD,gBAAgB,EAAE,aAAa,CAAC,gBAAgB;iBACjD,CAAC,CAAC;gBACH,MAAM,MAAM,GAAG,MAAM,aAAa,CAAC,yBAAyB,EAAE,CAAC;gBAC/D,IAAI,MAAM,IAAI,IAAI,EAAE,CAAC;oBACnB,MAAM,CAAC,IAAI,CAAC,+BAA+B,CAAC,CAAC;oBAC7C,OAAO;gBACT,CAAC;gBACD,MAAM,CAAC,kCAAG,aAAa,CAAC,WAAW,CAAC,MAAM,CAAC,QAAA,CAAC;gBAC5C,MAAM,CAAC,IAAI,CAAC,0BAA0B,CAAC,CAAC;gBACxC,MAAM,CAAC,CAAC,OAAO,CAAC,EAAE,aAAa,EAAE,uBAAuB,EAAE,cAAc,EAAE,OAAO,EAAE,CAAC,CAAC;gBACrF,MAAM,CAAC,IAAI,CAAC,iCAAiC,CAAC,CAAC;;;;;;;;;SAChD;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,MAAM,CAAC,KAAK,CAAC,sBAAsB,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;YACnD,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAClB,CAAC;gBAAS,CAAC;YACT,MAAM,MAAM,CAAC,KAAK,EAAE,CAAC;YACrB,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAClB,CAAC;IACH,CAAC,CAAC,CAAC;AACL,CAAC"}
|
|
@@ -366,6 +366,8 @@ export interface CompactOptions {
|
|
|
366
366
|
* If specified, compact only the specific buckets.
|
|
367
367
|
*
|
|
368
368
|
* If not specified, compacts all buckets.
|
|
369
|
+
*
|
|
370
|
+
* These can be individual bucket names, or bucket definition names.
|
|
369
371
|
*/
|
|
370
372
|
compactBuckets?: string[];
|
|
371
373
|
}
|
|
@@ -40,16 +40,32 @@ export class MongoCompactor {
|
|
|
40
40
|
async compactInternal(bucket) {
|
|
41
41
|
const idLimitBytes = this.idLimitBytes;
|
|
42
42
|
let currentState = null;
|
|
43
|
+
let bucketLower;
|
|
44
|
+
let bucketUpper;
|
|
45
|
+
if (bucket == null) {
|
|
46
|
+
bucketLower = new MinKey();
|
|
47
|
+
bucketUpper = new MaxKey();
|
|
48
|
+
}
|
|
49
|
+
else if (bucket.includes('[')) {
|
|
50
|
+
// Exact bucket name
|
|
51
|
+
bucketLower = bucket;
|
|
52
|
+
bucketUpper = bucket;
|
|
53
|
+
}
|
|
54
|
+
else {
|
|
55
|
+
// Bucket definition name
|
|
56
|
+
bucketLower = `${bucket}[`;
|
|
57
|
+
bucketUpper = `${bucket}[\uFFFF`;
|
|
58
|
+
}
|
|
43
59
|
// Constant lower bound
|
|
44
60
|
const lowerBound = {
|
|
45
61
|
g: this.group_id,
|
|
46
|
-
b:
|
|
62
|
+
b: bucketLower,
|
|
47
63
|
o: new MinKey()
|
|
48
64
|
};
|
|
49
65
|
// Upper bound is adjusted for each batch
|
|
50
66
|
let upperBound = {
|
|
51
67
|
g: this.group_id,
|
|
52
|
-
b:
|
|
68
|
+
b: bucketUpper,
|
|
53
69
|
o: new MaxKey()
|
|
54
70
|
};
|
|
55
71
|
while (true) {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"MongoCompactor.js","sourceRoot":"","sources":["../../../src/storage/mongo/MongoCompactor.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,mCAAmC,CAAC;AAC3D,OAAO,EAAyB,MAAM,EAAE,MAAM,EAAE,MAAM,SAAS,CAAC;AAChE,OAAO,EAAE,YAAY,EAAE,MAAM,qBAAqB,CAAC;AAInD,OAAO,EAAE,QAAQ,EAAE,MAAM,qBAAqB,CAAC;AAqC/C,MAAM,yBAAyB,GAAG,IAAI,CAAC;AACvC,MAAM,wBAAwB,GAAG,IAAI,CAAC;AACtC,MAAM,8BAA8B,GAAG,KAAM,CAAC;AAE9C,2CAA2C;AAC3C,MAAM,uBAAuB,GAAG,EAAE,CAAC;AAEnC,MAAM,OAAO,cAAc;IAUzB,YACU,EAAkB,EAClB,QAAgB,EACxB,OAA6B;QAFrB,OAAE,GAAF,EAAE,CAAgB;QAClB,aAAQ,GAAR,QAAQ,CAAQ;QAXlB,YAAO,GAAgD,EAAE,CAAC;QAchE,IAAI,CAAC,YAAY,GAAG,CAAC,OAAO,EAAE,aAAa,IAAI,uBAAuB,CAAC,GAAG,IAAI,GAAG,IAAI,CAAC;QACtF,IAAI,CAAC,cAAc,GAAG,OAAO,EAAE,cAAc,IAAI,wBAAwB,CAAC;QAC1E,IAAI,CAAC,mBAAmB,GAAG,OAAO,EAAE,mBAAmB,IAAI,8BAA8B,CAAC;QAC1F,IAAI,CAAC,eAAe,GAAG,OAAO,EAAE,eAAe,IAAI,yBAAyB,CAAC;QAC7E,IAAI,CAAC,OAAO,GAAG,OAAO,EAAE,OAAO,CAAC;QAChC,IAAI,CAAC,OAAO,GAAG,OAAO,EAAE,cAAc,CAAC;IACzC,CAAC;IAED;;;;OAIG;IACH,KAAK,CAAC,OAAO;QACX,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;YACjB,KAAK,IAAI,MAAM,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;gBAChC,wDAAwD;gBACxD,yCAAyC;gBACzC,0DAA0D;gBAC1D,MAAM,IAAI,CAAC,eAAe,CAAC,MAAM,CAAC,CAAC;YACrC,CAAC;QACH,CAAC;aAAM,CAAC;YACN,MAAM,IAAI,CAAC,eAAe,CAAC,SAAS,CAAC,CAAC;QACxC,CAAC;IACH,CAAC;IAED,KAAK,CAAC,eAAe,CAAC,MAA0B;QAC9C,MAAM,YAAY,GAAG,IAAI,CAAC,YAAY,CAAC;QAEvC,IAAI,YAAY,GAA8B,IAAI,CAAC;QAEnD,uBAAuB;QACvB,MAAM,UAAU,GAAkB;YAChC,CAAC,EAAE,IAAI,CAAC,QAAQ;YAChB,CAAC,EAAE,
|
|
1
|
+
{"version":3,"file":"MongoCompactor.js","sourceRoot":"","sources":["../../../src/storage/mongo/MongoCompactor.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,mCAAmC,CAAC;AAC3D,OAAO,EAAyB,MAAM,EAAE,MAAM,EAAE,MAAM,SAAS,CAAC;AAChE,OAAO,EAAE,YAAY,EAAE,MAAM,qBAAqB,CAAC;AAInD,OAAO,EAAE,QAAQ,EAAE,MAAM,qBAAqB,CAAC;AAqC/C,MAAM,yBAAyB,GAAG,IAAI,CAAC;AACvC,MAAM,wBAAwB,GAAG,IAAI,CAAC;AACtC,MAAM,8BAA8B,GAAG,KAAM,CAAC;AAE9C,2CAA2C;AAC3C,MAAM,uBAAuB,GAAG,EAAE,CAAC;AAEnC,MAAM,OAAO,cAAc;IAUzB,YACU,EAAkB,EAClB,QAAgB,EACxB,OAA6B;QAFrB,OAAE,GAAF,EAAE,CAAgB;QAClB,aAAQ,GAAR,QAAQ,CAAQ;QAXlB,YAAO,GAAgD,EAAE,CAAC;QAchE,IAAI,CAAC,YAAY,GAAG,CAAC,OAAO,EAAE,aAAa,IAAI,uBAAuB,CAAC,GAAG,IAAI,GAAG,IAAI,CAAC;QACtF,IAAI,CAAC,cAAc,GAAG,OAAO,EAAE,cAAc,IAAI,wBAAwB,CAAC;QAC1E,IAAI,CAAC,mBAAmB,GAAG,OAAO,EAAE,mBAAmB,IAAI,8BAA8B,CAAC;QAC1F,IAAI,CAAC,eAAe,GAAG,OAAO,EAAE,eAAe,IAAI,yBAAyB,CAAC;QAC7E,IAAI,CAAC,OAAO,GAAG,OAAO,EAAE,OAAO,CAAC;QAChC,IAAI,CAAC,OAAO,GAAG,OAAO,EAAE,cAAc,CAAC;IACzC,CAAC;IAED;;;;OAIG;IACH,KAAK,CAAC,OAAO;QACX,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;YACjB,KAAK,IAAI,MAAM,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;gBAChC,wDAAwD;gBACxD,yCAAyC;gBACzC,0DAA0D;gBAC1D,MAAM,IAAI,CAAC,eAAe,CAAC,MAAM,CAAC,CAAC;YACrC,CAAC;QACH,CAAC;aAAM,CAAC;YACN,MAAM,IAAI,CAAC,eAAe,CAAC,SAAS,CAAC,CAAC;QACxC,CAAC;IACH,CAAC;IAED,KAAK,CAAC,eAAe,CAAC,MAA0B;QAC9C,MAAM,YAAY,GAAG,IAAI,CAAC,YAAY,CAAC;QAEvC,IAAI,YAAY,GAA8B,IAAI,CAAC;QAEnD,IAAI,WAA4B,CAAC;QACjC,IAAI,WAA4B,CAAC;QAEjC,IAAI,MAAM,IAAI,IAAI,EAAE,CAAC;YACnB,WAAW,GAAG,IAAI,MAAM,EAAE,CAAC;YAC3B,WAAW,GAAG,IAAI,MAAM,EAAE,CAAC;QAC7B,CAAC;aAAM,IAAI,MAAM,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;YAChC,oBAAoB;YACpB,WAAW,GAAG,MAAM,CAAC;YACrB,WAAW,GAAG,MAAM,CAAC;QACvB,CAAC;aAAM,CAAC;YACN,yBAAyB;YACzB,WAAW,GAAG,GAAG,MAAM,GAAG,CAAC;YAC3B,WAAW,GAAG,GAAG,MAAM,SAAS,CAAC;QACnC,CAAC;QAED,uBAAuB;QACvB,MAAM,UAAU,GAAkB;YAChC,CAAC,EAAE,IAAI,CAAC,QAAQ;YAChB,CAAC,EAAE,WAAqB;YACxB,CAAC,EAAE,IAAI,MAAM,EAAS;SACvB,CAAC;QAEF,yCAAyC;QACzC,IAAI,UAAU,GAAkB;YAC9B,CAAC,EAAE,IAAI,CAAC,QAAQ;YAChB,CAAC,EAAE,WAAqB;YACxB,CAAC,EAAE,IAAI,MAAM,EAAS;SACvB,CAAC;QAEF,OAAO,IAAI,EAAE,CAAC;YACZ,sDAAsD;YACtD,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,WAAW;iBACpC,IAAI,CACH;gBACE,GAAG,EAAE;oBACH,IAAI,EAAE,UAAU;oBAChB,GAAG,EAAE,UAAU;iBAChB;aACF,EACD;gBACE,UAAU,EAAE;oBACV,GAAG,EAAE,CAAC;oBACN,EAAE,EAAE,CAAC;oBACL,KAAK,EAAE,CAAC;oBACR,MAAM,EAAE,CAAC;oBACT,YAAY,EAAE,CAAC;oBACf,UAAU,EAAE,CAAC;iBACd;gBACD,KAAK,EAAE,IAAI,CAAC,mBAAmB;gBAC/B,IAAI,EAAE,EAAE,GAAG,EAAE,CAAC,CAAC,EAAE;gBACjB,WAAW,EAAE,IAAI;aAClB,CACF;iBACA,OAAO,EAAE,CAAC;YAEb,IAAI,KAAK,CAAC,MAAM,IAAI,CAAC,EAAE,CAAC;gBACtB,wBAAwB;gBACxB,MAAM;YACR,CAAC;YAED,oCAAoC;YACpC,UAAU,GAAG,KAAK,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC;YAEzC,KAAK,IAAI,GAAG,IAAI,KAAK,EAAE,CAAC;gBACtB,IAAI,YAAY,IAAI,IAAI,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC,IAAI,YAAY,CAAC,MAAM,EAAE,CAAC;oBAC7D,IAAI,YAAY,IAAI,IAAI,IAAI,YAAY,CAAC,UAAU,IAAI,IAAI,IAAI,YAAY,CAAC,WAAW,IAAI,CAAC,EAAE,CAAC;wBAC7F,0CAA0C;wBAC1C,MAAM,IAAI,CAAC,KAAK,EAAE,CAAC;wBACnB,MAAM,CAAC,IAAI,CACT,sBAAsB,IAAI,CAAC,QAAQ,IAAI,YAAY,CAAC,MAAM,IAAI,YAAY,CAAC,UAAU,cAAc,YAAY,CAAC,WAAW,aAAa,CACzI,CAAC;wBAEF,MAAM,MAAM,GAAG,YAAY,CAAC,MAAM,CAAC;wBACnC,MAAM,OAAO,GAAG,YAAY,CAAC,UAAU,CAAC;wBACxC,qCAAqC;wBACrC,YAAY,GAAG,IAAI,CAAC;wBACpB,MAAM,IAAI,CAAC,WAAW,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;oBAC1C,CAAC;oBACD,YAAY,GAAG;wBACb,MAAM,EAAE,GAAG,CAAC,GAAG,CAAC,CAAC;wBACjB,IAAI,EAAE,IAAI,GAAG,EAAE;wBACf,YAAY,EAAE,CAAC;wBACf,UAAU,EAAE,IAAI;wBAChB,WAAW,EAAE,CAAC;qBACf,CAAC;gBACJ,CAAC;gBAED,IAAI,IAAI,CAAC,OAAO,IAAI,IAAI,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC,GAAG,IAAI,CAAC,OAAO,EAAE,CAAC;oBACrD,SAAS;gBACX,CAAC;gBAED,IAAI,eAAe,GAAG,GAAG,CAAC,EAAE,IAAI,KAAK,CAAC;gBAEtC,IAAI,GAAG,CAAC,EAAE,IAAI,QAAQ,IAAI,GAAG,CAAC,EAAE,IAAI,KAAK,EAAE,CAAC;oBAC1C,MAAM,GAAG,GAAG,GAAG,GAAG,CAAC,KAAK,IAAI,GAAG,CAAC,MAAM,IAAI,QAAQ,CAAC,GAAG,CAAC,YAAa,EAAE,GAAG,CAAC,UAAW,CAAC,EAAE,CAAC;oBACzF,MAAM,QAAQ,GAAG,YAAY,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;oBAC5C,IAAI,QAAQ,EAAE,CAAC;wBACb,8CAA8C;wBAC9C,eAAe,GAAG,KAAK,CAAC;wBAExB,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC;4BAChB,SAAS,EAAE;gCACT,MAAM,EAAE;oCACN,GAAG,EAAE,GAAG,CAAC,GAAG;iCACb;gCACD,MAAM,EAAE;oCACN,IAAI,EAAE;wCACJ,EAAE,EAAE,MAAM;wCACV,SAAS,EAAE,QAAQ;qCACpB;oCACD,MAAM,EAAE;wCACN,YAAY,EAAE,CAAC;wCACf,UAAU,EAAE,CAAC;wCACb,KAAK,EAAE,CAAC;wCACR,MAAM,EAAE,CAAC;wCACT,IAAI,EAAE,CAAC;qCACR;iCACF;6BACF;yBACF,CAAC,CAAC;oBACL,CAAC;yBAAM,CAAC;wBACN,IAAI,YAAY,CAAC,YAAY,IAAI,YAAY,EAAE,CAAC;4BAC9C,wBAAwB;4BACxB,6CAA6C;wBAC/C,CAAC;6BAAM,CAAC;4BACN,4DAA4D;4BAC5D,YAAY,CAAC,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;4BAC/C,6BAA6B;4BAC7B,oBAAoB;4BACpB,sBAAsB;4BACtB,6BAA6B;4BAC7B,YAAY,CAAC,YAAY,IAAI,GAAG,CAAC,MAAM,GAAG,GAAG,CAAC;wBAChD,CAAC;oBACH,CAAC;gBACH,CAAC;gBAED,IAAI,eAAe,EAAE,CAAC;oBACpB,YAAY,CAAC,UAAU,GAAG,IAAI,CAAC;oBAC/B,YAAY,CAAC,WAAW,GAAG,CAAC,CAAC;gBAC/B,CAAC;qBAAM,IAAI,GAAG,CAAC,EAAE,IAAI,OAAO,EAAE,CAAC;oBAC7B,IAAI,YAAY,CAAC,UAAU,IAAI,IAAI,EAAE,CAAC;wBACpC,YAAY,CAAC,UAAU,GAAG,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC;oBACtC,CAAC;oBACD,YAAY,CAAC,WAAW,IAAI,CAAC,CAAC;gBAChC,CAAC;gBAED,IAAI,IAAI,CAAC,OAAO,CAAC,MAAM,IAAI,IAAI,CAAC,cAAc,EAAE,CAAC;oBAC/C,MAAM,IAAI,CAAC,KAAK,EAAE,CAAC;gBACrB,CAAC;YACH,CAAC;QACH,CAAC;QAED,MAAM,IAAI,CAAC,KAAK,EAAE,CAAC;QACnB,YAAY,EAAE,IAAI,CAAC,KAAK,EAAE,CAAC;QAC3B,IAAI,YAAY,EAAE,UAAU,IAAI,IAAI,IAAI,YAAY,EAAE,WAAW,GAAG,CAAC,EAAE,CAAC;YACtE,MAAM,CAAC,IAAI,CACT,sBAAsB,IAAI,CAAC,QAAQ,IAAI,YAAY,CAAC,MAAM,IAAI,YAAY,CAAC,UAAU,cAAc,YAAY,CAAC,WAAW,aAAa,CACzI,CAAC;YACF,MAAM,MAAM,GAAG,YAAY,CAAC,MAAM,CAAC;YACnC,MAAM,OAAO,GAAG,YAAY,CAAC,UAAU,CAAC;YACxC,qCAAqC;YACrC,YAAY,GAAG,IAAI,CAAC;YACpB,MAAM,IAAI,CAAC,WAAW,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;QAC1C,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,KAAK;QACjB,IAAI,IAAI,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC5B,MAAM,CAAC,IAAI,CAAC,cAAc,IAAI,CAAC,OAAO,CAAC,MAAM,MAAM,CAAC,CAAC;YACrD,MAAM,IAAI,CAAC,EAAE,CAAC,WAAW,CAAC,SAAS,CAAC,IAAI,CAAC,OAAO,EAAE;gBAChD,0BAA0B;gBAC1B,8EAA8E;gBAC9E,yDAAyD;gBACzD,uCAAuC;gBACvC,OAAO,EAAE,KAAK;aACf,CAAC,CAAC;YACH,IAAI,CAAC,OAAO,GAAG,EAAE,CAAC;QACpB,CAAC;IACH,CAAC;IAED;;;;;OAKG;IACK,KAAK,CAAC,WAAW,CAAC,MAAc,EAAE,EAAU;QAClD,MAAM,QAAQ,GAAG;YACf,GAAG,EAAE;gBACH,IAAI,EAAE;oBACJ,CAAC,EAAE,IAAI,CAAC,QAAQ;oBAChB,CAAC,EAAE,MAAM;oBACT,CAAC,EAAE,IAAI,MAAM,EAAS;iBACvB;gBACD,IAAI,EAAE;oBACJ,CAAC,EAAE,IAAI,CAAC,QAAQ;oBAChB,CAAC,EAAE,MAAM;oBACT,CAAC,EAAE,EAAE;iBACN;aACF;SACF,CAAC;QAEF,MAAM,OAAO,GAAG,IAAI,CAAC,EAAE,CAAC,MAAM,CAAC,YAAY,EAAE,CAAC;QAC9C,IAAI,CAAC;YACH,IAAI,IAAI,GAAG,KAAK,CAAC;YACjB,OAAO,CAAC,IAAI,EAAE,CAAC;gBACb,6EAA6E;gBAC7E,kDAAkD;gBAClD,0EAA0E;gBAC1E,MAAM,OAAO,CAAC,eAAe,CAC3B,KAAK,IAAI,EAAE;oBACT,MAAM,KAAK,GAAG,IAAI,CAAC,EAAE,CAAC,WAAW,CAAC,IAAI,CAAC,QAAQ,EAAE;wBAC/C,OAAO;wBACP,IAAI,EAAE,EAAE,GAAG,EAAE,CAAC,EAAE;wBAChB,UAAU,EAAE;4BACV,GAAG,EAAE,CAAC;4BACN,EAAE,EAAE,CAAC;4BACL,QAAQ,EAAE,CAAC;4BACX,SAAS,EAAE,CAAC;yBACb;wBACD,KAAK,EAAE,IAAI,CAAC,eAAe;qBAC5B,CAAC,CAAC;oBACH,IAAI,QAAQ,GAAG,CAAC,CAAC;oBACjB,IAAI,QAAQ,GAAyB,IAAI,CAAC;oBAC1C,IAAI,QAAQ,GAAkB,IAAI,CAAC;oBACnC,IAAI,OAAO,GAAG,KAAK,CAAC;oBACpB,IAAI,KAAK,EAAE,IAAI,EAAE,IAAI,KAAK,CAAC,MAAM,EAAE,EAAE,CAAC;wBACpC,IAAI,EAAE,CAAC,EAAE,IAAI,MAAM,IAAI,EAAE,CAAC,EAAE,IAAI,QAAQ,IAAI,EAAE,CAAC,EAAE,IAAI,OAAO,EAAE,CAAC;4BAC7D,QAAQ,GAAG,YAAY,CAAC,QAAQ,EAAE,EAAE,CAAC,QAAQ,CAAC,CAAC;4BAC/C,QAAQ,GAAG,EAAE,CAAC,GAAG,CAAC;4BAClB,IAAI,EAAE,CAAC,EAAE,IAAI,OAAO,EAAE,CAAC;gCACrB,OAAO,GAAG,IAAI,CAAC;4BACjB,CAAC;4BACD,IAAI,EAAE,CAAC,SAAS,IAAI,IAAI,EAAE,CAAC;gCACzB,IAAI,QAAQ,IAAI,IAAI,IAAI,EAAE,CAAC,SAAS,GAAG,QAAQ,EAAE,CAAC;oCAChD,QAAQ,GAAG,EAAE,CAAC,SAAS,CAAC;gCAC1B,CAAC;4BACH,CAAC;wBACH,CAAC;6BAAM,CAAC;4BACN,MAAM,IAAI,KAAK,CAAC,cAAc,EAAE,CAAC,EAAE,iBAAiB,EAAE,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC;wBAC1F,CAAC;oBACH,CAAC;oBACD,IAAI,CAAC,OAAO,EAAE,CAAC;wBACb,IAAI,GAAG,IAAI,CAAC;wBACZ,OAAO;oBACT,CAAC;oBAED,MAAM,CAAC,IAAI,CAAC,qBAAqB,QAAQ,EAAE,CAAC,EAAE,CAAC,CAAC;oBAChD,MAAM,IAAI,CAAC,EAAE,CAAC,WAAW,CAAC,UAAU,CAClC;wBACE,GAAG,EAAE;4BACH,IAAI,EAAE;gCACJ,CAAC,EAAE,IAAI,CAAC,QAAQ;gCAChB,CAAC,EAAE,MAAM;gCACT,CAAC,EAAE,IAAI,MAAM,EAAS;6BACvB;4BACD,IAAI,EAAE,QAAS;yBAChB;qBACF,EACD,EAAE,OAAO,EAAE,CACZ,CAAC;oBAEF,MAAM,IAAI,CAAC,EAAE,CAAC,WAAW,CAAC,SAAS,CACjC;wBACE,GAAG,EAAE,QAAS;wBACd,EAAE,EAAE,OAAO;wBACX,QAAQ,EAAE,QAAQ;wBAClB,IAAI,EAAE,IAAI;wBACV,SAAS,EAAE,QAAQ;qBACpB,EACD,EAAE,OAAO,EAAE,CACZ,CAAC;gBACJ,CAAC,EACD;oBACE,YAAY,EAAE,EAAE,CAAC,EAAE,UAAU,EAAE;oBAC/B,WAAW,EAAE,EAAE,KAAK,EAAE,UAAU,EAAE;iBACnC,CACF,CAAC;YACJ,CAAC;QACH,CAAC;gBAAS,CAAC;YACT,MAAM,OAAO,CAAC,UAAU,EAAE,CAAC;QAC7B,CAAC;IACH,CAAC;CACF;AAED;;;;;GAKG;AACH,SAAS,OAAO,CAAC,CAAS;IACxB,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;IACf,OAAO,CAAC,CAAC;AACX,CAAC"}
|
package/package.json
CHANGED
|
@@ -5,7 +5,7 @@
|
|
|
5
5
|
"publishConfig": {
|
|
6
6
|
"access": "public"
|
|
7
7
|
},
|
|
8
|
-
"version": "0.0.0-dev-
|
|
8
|
+
"version": "0.0.0-dev-20241107065634",
|
|
9
9
|
"main": "dist/index.js",
|
|
10
10
|
"license": "FSL-1.1-Apache-2.0",
|
|
11
11
|
"type": "module",
|
|
@@ -33,11 +33,11 @@
|
|
|
33
33
|
"uuid": "^9.0.1",
|
|
34
34
|
"winston": "^3.13.0",
|
|
35
35
|
"yaml": "^2.3.2",
|
|
36
|
-
"@powersync/lib-services-framework": "0.0.0-dev-
|
|
36
|
+
"@powersync/lib-services-framework": "0.0.0-dev-20241107065634",
|
|
37
37
|
"@powersync/service-jsonbig": "0.17.10",
|
|
38
|
-
"@powersync/service-
|
|
39
|
-
"@powersync/service-
|
|
40
|
-
"@powersync/service-types": "0.0.0-dev-
|
|
38
|
+
"@powersync/service-sync-rules": "0.0.0-dev-20241107065634",
|
|
39
|
+
"@powersync/service-rsocket-router": "0.0.0-dev-20241107065634",
|
|
40
|
+
"@powersync/service-types": "0.0.0-dev-20241107065634"
|
|
41
41
|
},
|
|
42
42
|
"devDependencies": {
|
|
43
43
|
"@types/async": "^3.2.24",
|
|
@@ -53,6 +53,6 @@
|
|
|
53
53
|
"build": "tsc -b",
|
|
54
54
|
"build:tests": "tsc -b test/tsconfig.json",
|
|
55
55
|
"test": "vitest",
|
|
56
|
-
"clean": "rm -rf ./
|
|
56
|
+
"clean": "rm -rf ./dist && tsc -b --clean"
|
|
57
57
|
}
|
|
58
58
|
}
|
|
@@ -21,12 +21,19 @@ const HEAP_LIMIT = v8.getHeapStatistics().heap_size_limit;
|
|
|
21
21
|
const COMPACT_MEMORY_LIMIT_MB = Math.min(HEAP_LIMIT / 1024 / 1024 - 128, 1024);
|
|
22
22
|
|
|
23
23
|
export function registerCompactAction(program: Command) {
|
|
24
|
-
const compactCommand = program
|
|
24
|
+
const compactCommand = program
|
|
25
|
+
.command(COMMAND_NAME)
|
|
26
|
+
.option(`-b, --buckets [buckets]`, 'Bucket or bucket definition name (optional, comma-separate multiple names)');
|
|
25
27
|
|
|
26
28
|
wrapConfigCommand(compactCommand);
|
|
27
29
|
|
|
28
30
|
return compactCommand.description('Compact storage').action(async (options) => {
|
|
29
|
-
|
|
31
|
+
const buckets = options.buckets?.split(',');
|
|
32
|
+
if (buckets != null) {
|
|
33
|
+
logger.info('Compacting storage for all buckets...');
|
|
34
|
+
} else {
|
|
35
|
+
logger.info(`Compacting storage for ${buckets.join(', ')}...`);
|
|
36
|
+
}
|
|
30
37
|
const runnerConfig = extractRunnerOptions(options);
|
|
31
38
|
const configuration = await utils.loadConfig(runnerConfig);
|
|
32
39
|
logger.info('Successfully loaded configuration...');
|
|
@@ -46,7 +53,7 @@ export function registerCompactAction(program: Command) {
|
|
|
46
53
|
}
|
|
47
54
|
using p = bucketStorage.getInstance(active);
|
|
48
55
|
logger.info('Performing compaction...');
|
|
49
|
-
await p.compact({ memoryLimitMB: COMPACT_MEMORY_LIMIT_MB });
|
|
56
|
+
await p.compact({ memoryLimitMB: COMPACT_MEMORY_LIMIT_MB, compactBuckets: buckets });
|
|
50
57
|
logger.info('Successfully compacted storage.');
|
|
51
58
|
} catch (e) {
|
|
52
59
|
logger.error(`Failed to compact: ${e.toString()}`);
|
|
@@ -472,6 +472,8 @@ export interface CompactOptions {
|
|
|
472
472
|
* If specified, compact only the specific buckets.
|
|
473
473
|
*
|
|
474
474
|
* If not specified, compacts all buckets.
|
|
475
|
+
*
|
|
476
|
+
* These can be individual bucket names, or bucket definition names.
|
|
475
477
|
*/
|
|
476
478
|
compactBuckets?: string[];
|
|
477
479
|
}
|
|
@@ -94,17 +94,33 @@ export class MongoCompactor {
|
|
|
94
94
|
|
|
95
95
|
let currentState: CurrentBucketState | null = null;
|
|
96
96
|
|
|
97
|
+
let bucketLower: string | MinKey;
|
|
98
|
+
let bucketUpper: string | MaxKey;
|
|
99
|
+
|
|
100
|
+
if (bucket == null) {
|
|
101
|
+
bucketLower = new MinKey();
|
|
102
|
+
bucketUpper = new MaxKey();
|
|
103
|
+
} else if (bucket.includes('[')) {
|
|
104
|
+
// Exact bucket name
|
|
105
|
+
bucketLower = bucket;
|
|
106
|
+
bucketUpper = bucket;
|
|
107
|
+
} else {
|
|
108
|
+
// Bucket definition name
|
|
109
|
+
bucketLower = `${bucket}[`;
|
|
110
|
+
bucketUpper = `${bucket}[\uFFFF`;
|
|
111
|
+
}
|
|
112
|
+
|
|
97
113
|
// Constant lower bound
|
|
98
114
|
const lowerBound: BucketDataKey = {
|
|
99
115
|
g: this.group_id,
|
|
100
|
-
b:
|
|
116
|
+
b: bucketLower as string,
|
|
101
117
|
o: new MinKey() as any
|
|
102
118
|
};
|
|
103
119
|
|
|
104
120
|
// Upper bound is adjusted for each batch
|
|
105
121
|
let upperBound: BucketDataKey = {
|
|
106
122
|
g: this.group_id,
|
|
107
|
-
b:
|
|
123
|
+
b: bucketUpper as string,
|
|
108
124
|
o: new MaxKey() as any
|
|
109
125
|
};
|
|
110
126
|
|