@powersync/service-core 0.5.0 → 0.5.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +6 -0
- package/dist/entry/commands/compact-action.js +3 -4
- package/dist/entry/commands/compact-action.js.map +1 -1
- package/dist/storage/BucketStorage.d.ts +6 -0
- package/dist/storage/mongo/MongoCompactor.d.ts +2 -0
- package/dist/storage/mongo/MongoCompactor.js +16 -2
- package/dist/storage/mongo/MongoCompactor.js.map +1 -1
- package/package.json +2 -2
- package/src/entry/commands/compact-action.ts +3 -4
- package/src/storage/BucketStorage.ts +7 -0
- package/src/storage/mongo/MongoCompactor.ts +17 -2
- package/tsconfig.tsbuildinfo +1 -1
package/CHANGELOG.md
CHANGED
|
@@ -1,7 +1,6 @@
|
|
|
1
1
|
import { logger } from '@powersync/lib-services-framework';
|
|
2
2
|
import * as v8 from 'v8';
|
|
3
|
-
import {
|
|
4
|
-
import { MongoBucketStorage, PowerSyncMongo } from '../../storage/storage-index.js';
|
|
3
|
+
import { createPowerSyncMongo, MongoBucketStorage } from '../../storage/storage-index.js';
|
|
5
4
|
import { loadConfig } from '../../util/config.js';
|
|
6
5
|
import { extractRunnerOptions, wrapConfigCommand } from './config-command.js';
|
|
7
6
|
const COMMAND_NAME = 'compact';
|
|
@@ -22,10 +21,10 @@ export function registerCompactAction(program) {
|
|
|
22
21
|
const runnerConfig = extractRunnerOptions(options);
|
|
23
22
|
const config = await loadConfig(runnerConfig);
|
|
24
23
|
const { storage } = config;
|
|
25
|
-
const
|
|
24
|
+
const psdb = createPowerSyncMongo(storage);
|
|
25
|
+
const client = psdb.client;
|
|
26
26
|
await client.connect();
|
|
27
27
|
try {
|
|
28
|
-
const psdb = new PowerSyncMongo(client);
|
|
29
28
|
const bucketStorage = new MongoBucketStorage(psdb, { slot_name_prefix: config.slot_name_prefix });
|
|
30
29
|
const active = await bucketStorage.getActiveSyncRules();
|
|
31
30
|
if (active == null) {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"compact-action.js","sourceRoot":"","sources":["../../../src/entry/commands/compact-action.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,MAAM,EAAE,MAAM,mCAAmC,CAAC;AAC3D,OAAO,KAAK,EAAE,MAAM,IAAI,CAAC;AACzB,OAAO,EAAE,
|
|
1
|
+
{"version":3,"file":"compact-action.js","sourceRoot":"","sources":["../../../src/entry/commands/compact-action.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,MAAM,EAAE,MAAM,mCAAmC,CAAC;AAC3D,OAAO,KAAK,EAAE,MAAM,IAAI,CAAC;AACzB,OAAO,EAAE,oBAAoB,EAAE,kBAAkB,EAAE,MAAM,gCAAgC,CAAC;AAC1F,OAAO,EAAE,UAAU,EAAE,MAAM,sBAAsB,CAAC;AAClD,OAAO,EAAE,oBAAoB,EAAE,iBAAiB,EAAE,MAAM,qBAAqB,CAAC;AAE9E,MAAM,YAAY,GAAG,SAAS,CAAC;AAE/B;;GAEG;AACH,MAAM,UAAU,GAAG,EAAE,CAAC,iBAAiB,EAAE,CAAC,eAAe,CAAC;AAE1D;;;;GAIG;AACH,MAAM,uBAAuB,GAAG,IAAI,CAAC,GAAG,CAAC,UAAU,GAAG,IAAI,GAAG,IAAI,GAAG,GAAG,EAAE,IAAI,CAAC,CAAC;AAE/E,MAAM,UAAU,qBAAqB,CAAC,OAAgB;IACpD,MAAM,cAAc,GAAG,OAAO,CAAC,OAAO,CAAC,YAAY,CAAC,CAAC;IAErD,iBAAiB,CAAC,cAAc,CAAC,CAAC;IAElC,OAAO,cAAc,CAAC,WAAW,CAAC,iBAAiB,CAAC,CAAC,MAAM,CAAC,KAAK,EAAE,OAAO,EAAE,EAAE;QAC5E,MAAM,YAAY,GAAG,oBAAoB,CAAC,OAAO,CAAC,CAAC;QAEnD,MAAM,MAAM,GAAG,MAAM,UAAU,CAAC,YAAY,CAAC,CAAC;QAC9C,MAAM,EAAE,OAAO,EAAE,GAAG,MAAM,CAAC;QAC3B,MAAM,IAAI,GAAG,oBAAoB,CAAC,OAAO,CAAC,CAAC;QAC3C,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC;QAC3B,MAAM,MAAM,CAAC,OAAO,EAAE,CAAC;QACvB,IAAI;YACF,MAAM,aAAa,GAAG,IAAI,kBAAkB,CAAC,IAAI,EAAE,EAAE,gBAAgB,EAAE,MAAM,CAAC,gBAAgB,EAAE,CAAC,CAAC;YAClG,MAAM,MAAM,GAAG,MAAM,aAAa,CAAC,kBAAkB,EAAE,CAAC;YACxD,IAAI,MAAM,IAAI,IAAI,EAAE;gBAClB,MAAM,CAAC,IAAI,CAAC,+BAA+B,CAAC,CAAC;gBAC7C,OAAO;aACR;YACD,MAAM,CAAC,GAAG,aAAa,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC;YAC5C,MAAM,CAAC,CAAC,OAAO,CAAC,EAAE,aAAa,EAAE,uBAAuB,EAAE,CAAC,CAAC;YAC5D,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;SACrB;QAAC,OAAO,CAAC,EAAE;YACV,MAAM,CAAC,KAAK,CAAC,sBAAsB,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;YACnD,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;SACjB;gBAAS;YACR,MAAM,MAAM,CAAC,KAAK,EAAE,CAAC;YACrB,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;SACjB;IACH,CAAC,CAAC,CAAC;AACL,CAAC"}
|
|
@@ -331,4 +331,10 @@ export interface CompactOptions {
|
|
|
331
331
|
* not be compacted, to avoid invalidating checkpoints in use.
|
|
332
332
|
*/
|
|
333
333
|
maxOpId?: bigint;
|
|
334
|
+
/**
|
|
335
|
+
* If specified, compact only the specific buckets.
|
|
336
|
+
*
|
|
337
|
+
* If not specified, compacts all buckets.
|
|
338
|
+
*/
|
|
339
|
+
compactBuckets?: string[];
|
|
334
340
|
}
|
|
@@ -20,6 +20,7 @@ export declare class MongoCompactor {
|
|
|
20
20
|
private moveBatchQueryLimit;
|
|
21
21
|
private clearBatchLimit;
|
|
22
22
|
private maxOpId;
|
|
23
|
+
private buckets;
|
|
23
24
|
constructor(db: PowerSyncMongo, group_id: number, options?: MongoCompactOptions);
|
|
24
25
|
/**
|
|
25
26
|
* Compact buckets by converting operations into MOVE and/or CLEAR operations.
|
|
@@ -27,6 +28,7 @@ export declare class MongoCompactor {
|
|
|
27
28
|
* See /docs/compacting-operations.md for details.
|
|
28
29
|
*/
|
|
29
30
|
compact(): Promise<void>;
|
|
31
|
+
compactInternal(bucket: string | undefined): Promise<void>;
|
|
30
32
|
private flush;
|
|
31
33
|
/**
|
|
32
34
|
* Perform a CLEAR compact for a bucket.
|
|
@@ -16,6 +16,7 @@ export class MongoCompactor {
|
|
|
16
16
|
this.moveBatchQueryLimit = options?.moveBatchQueryLimit ?? DEFAULT_MOVE_BATCH_QUERY_LIMIT;
|
|
17
17
|
this.clearBatchLimit = options?.clearBatchLimit ?? DEFAULT_CLEAR_BATCH_LIMIT;
|
|
18
18
|
this.maxOpId = options?.maxOpId;
|
|
19
|
+
this.buckets = options?.compactBuckets;
|
|
19
20
|
}
|
|
20
21
|
/**
|
|
21
22
|
* Compact buckets by converting operations into MOVE and/or CLEAR operations.
|
|
@@ -23,18 +24,31 @@ export class MongoCompactor {
|
|
|
23
24
|
* See /docs/compacting-operations.md for details.
|
|
24
25
|
*/
|
|
25
26
|
async compact() {
|
|
27
|
+
if (this.buckets) {
|
|
28
|
+
for (let bucket of this.buckets) {
|
|
29
|
+
// We can make this more efficient later on by iterating
|
|
30
|
+
// through the buckets in a single query.
|
|
31
|
+
// That makes batching more tricky, so we leave for later.
|
|
32
|
+
await this.compactInternal(bucket);
|
|
33
|
+
}
|
|
34
|
+
}
|
|
35
|
+
else {
|
|
36
|
+
await this.compactInternal(undefined);
|
|
37
|
+
}
|
|
38
|
+
}
|
|
39
|
+
async compactInternal(bucket) {
|
|
26
40
|
const idLimitBytes = this.idLimitBytes;
|
|
27
41
|
let currentState = null;
|
|
28
42
|
// Constant lower bound
|
|
29
43
|
const lowerBound = {
|
|
30
44
|
g: this.group_id,
|
|
31
|
-
b: new MinKey(),
|
|
45
|
+
b: bucket ?? new MinKey(),
|
|
32
46
|
o: new MinKey()
|
|
33
47
|
};
|
|
34
48
|
// Upper bound is adjusted for each batch
|
|
35
49
|
let upperBound = {
|
|
36
50
|
g: this.group_id,
|
|
37
|
-
b: new MaxKey(),
|
|
51
|
+
b: bucket ?? new MaxKey(),
|
|
38
52
|
o: new MaxKey()
|
|
39
53
|
};
|
|
40
54
|
while (true) {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"MongoCompactor.js","sourceRoot":"","sources":["../../../src/storage/mongo/MongoCompactor.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,mCAAmC,CAAC;AAC3D,OAAO,EAAyB,MAAM,EAAE,MAAM,EAAE,MAAM,SAAS,CAAC;AAChE,OAAO,EAAE,YAAY,EAAE,MAAM,qBAAqB,CAAC;AAwCnD,MAAM,yBAAyB,GAAG,IAAI,CAAC;AACvC,MAAM,wBAAwB,GAAG,IAAI,CAAC;AACtC,MAAM,8BAA8B,GAAG,KAAM,CAAC;AAE9C,2CAA2C;AAC3C,MAAM,uBAAuB,GAAG,EAAE,CAAC;AAEnC,MAAM,OAAO,cAAc;
|
|
1
|
+
{"version":3,"file":"MongoCompactor.js","sourceRoot":"","sources":["../../../src/storage/mongo/MongoCompactor.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,mCAAmC,CAAC;AAC3D,OAAO,EAAyB,MAAM,EAAE,MAAM,EAAE,MAAM,SAAS,CAAC;AAChE,OAAO,EAAE,YAAY,EAAE,MAAM,qBAAqB,CAAC;AAwCnD,MAAM,yBAAyB,GAAG,IAAI,CAAC;AACvC,MAAM,wBAAwB,GAAG,IAAI,CAAC;AACtC,MAAM,8BAA8B,GAAG,KAAM,CAAC;AAE9C,2CAA2C;AAC3C,MAAM,uBAAuB,GAAG,EAAE,CAAC;AAEnC,MAAM,OAAO,cAAc;IAUzB,YAAoB,EAAkB,EAAU,QAAgB,EAAE,OAA6B;QAA3E,OAAE,GAAF,EAAE,CAAgB;QAAU,aAAQ,GAAR,QAAQ,CAAQ;QATxD,YAAO,GAAgD,EAAE,CAAC;QAUhE,IAAI,CAAC,YAAY,GAAG,CAAC,OAAO,EAAE,aAAa,IAAI,uBAAuB,CAAC,GAAG,IAAI,GAAG,IAAI,CAAC;QACtF,IAAI,CAAC,cAAc,GAAG,OAAO,EAAE,cAAc,IAAI,wBAAwB,CAAC;QAC1E,IAAI,CAAC,mBAAmB,GAAG,OAAO,EAAE,mBAAmB,IAAI,8BAA8B,CAAC;QAC1F,IAAI,CAAC,eAAe,GAAG,OAAO,EAAE,eAAe,IAAI,yBAAyB,CAAC;QAC7E,IAAI,CAAC,OAAO,GAAG,OAAO,EAAE,OAAO,CAAC;QAChC,IAAI,CAAC,OAAO,GAAG,OAAO,EAAE,cAAc,CAAC;IACzC,CAAC;IAED;;;;OAIG;IACH,KAAK,CAAC,OAAO;QACX,IAAI,IAAI,CAAC,OAAO,EAAE;YAChB,KAAK,IAAI,MAAM,IAAI,IAAI,CAAC,OAAO,EAAE;gBAC/B,wDAAwD;gBACxD,yCAAyC;gBACzC,0DAA0D;gBAC1D,MAAM,IAAI,CAAC,eAAe,CAAC,MAAM,CAAC,CAAC;aACpC;SACF;aAAM;YACL,MAAM,IAAI,CAAC,eAAe,CAAC,SAAS,CAAC,CAAC;SACvC;IACH,CAAC;IAED,KAAK,CAAC,eAAe,CAAC,MAA0B;QAC9C,MAAM,YAAY,GAAG,IAAI,CAAC,YAAY,CAAC;QAEvC,IAAI,YAAY,GAA8B,IAAI,CAAC;QAEnD,uBAAuB;QACvB,MAAM,UAAU,GAAkB;YAChC,CAAC,EAAE,IAAI,CAAC,QAAQ;YAChB,CAAC,EAAE,MAAM,IAAK,IAAI,MAAM,EAAU;YAClC,CAAC,EAAE,IAAI,MAAM,EAAS;SACvB,CAAC;QAEF,yCAAyC;QACzC,IAAI,UAAU,GAAkB;YAC9B,CAAC,EAAE,IAAI,CAAC,QAAQ;YAChB,CAAC,EAAE,MAAM,IAAK,IAAI,MAAM,EAAU;YAClC,CAAC,EAAE,IAAI,MAAM,EAAS;SACvB,CAAC;QAEF,OAAO,IAAI,EAAE;YACX,sDAAsD;YACtD,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,WAAW;iBACpC,IAAI,CACH;gBACE,GAAG,EAAE;oBACH,IAAI,EAAE,UAAU;oBAChB,GAAG,EAAE,UAAU;iBAChB;aACF,EACD;gBACE,UAAU,EAAE;oBACV,GAAG,EAAE,CAAC;oBACN,EAAE,EAAE,CAAC;oBACL,KAAK,EAAE,CAAC;oBACR,MAAM,EAAE,CAAC;oBACT,YAAY,EAAE,CAAC;oBACf,UAAU,EAAE,CAAC;iBACd;gBACD,KAAK,EAAE,IAAI,CAAC,mBAAmB;gBAC/B,IAAI,EAAE,EAAE,GAAG,EAAE,CAAC,CAAC,EAAE;gBACjB,WAAW,EAAE,IAAI;aAClB,CACF;iBACA,OAAO,EAAE,CAAC;YAEb,IAAI,KAAK,CAAC,MAAM,IAAI,CAAC,EAAE;gBACrB,wBAAwB;gBACxB,MAAM;aACP;YAED,oCAAoC;YACpC,UAAU,GAAG,KAAK,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC;YAEzC,KAAK,IAAI,GAAG,IAAI,KAAK,EAAE;gBACrB,IAAI,YAAY,IAAI,IAAI,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC,IAAI,YAAY,CAAC,MAAM,EAAE;oBAC5D,IAAI,YAAY,IAAI,IAAI,IAAI,YAAY,CAAC,UAAU,IAAI,IAAI,IAAI,YAAY,CAAC,WAAW,IAAI,CAAC,EAAE;wBAC5F,0CAA0C;wBAC1C,MAAM,IAAI,CAAC,KAAK,EAAE,CAAC;wBACnB,MAAM,CAAC,IAAI,CACT,sBAAsB,IAAI,CAAC,QAAQ,IAAI,YAAY,CAAC,MAAM,IAAI,YAAY,CAAC,UAAU,cAAc,YAAY,CAAC,WAAW,aAAa,CACzI,CAAC;wBAEF,MAAM,MAAM,GAAG,YAAY,CAAC,MAAM,CAAC;wBACnC,MAAM,OAAO,GAAG,YAAY,CAAC,UAAU,CAAC;wBACxC,qCAAqC;wBACrC,YAAY,GAAG,IAAI,CAAC;wBACpB,MAAM,IAAI,CAAC,WAAW,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;qBACzC;oBACD,YAAY,GAAG;wBACb,MAAM,EAAE,GAAG,CAAC,GAAG,CAAC,CAAC;wBACjB,IAAI,EAAE,IAAI,GAAG,EAAE;wBACf,YAAY,EAAE,CAAC;wBACf,UAAU,EAAE,IAAI;wBAChB,WAAW,EAAE,CAAC;qBACf,CAAC;iBACH;gBAED,IAAI,IAAI,CAAC,OAAO,IAAI,IAAI,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC,GAAG,IAAI,CAAC,OAAO,EAAE;oBACpD,SAAS;iBACV;gBAED,IAAI,eAAe,GAAG,GAAG,CAAC,EAAE,IAAI,KAAK,CAAC;gBAEtC,IAAI,GAAG,CAAC,EAAE,IAAI,QAAQ,IAAI,GAAG,CAAC,EAAE,IAAI,KAAK,EAAE;oBACzC,MAAM,GAAG,GAAG,GAAG,GAAG,CAAC,KAAK,IAAI,GAAG,CAAC,MAAM,IAAI,GAAG,CAAC,YAAY,IAAI,GAAG,CAAC,UAAU,EAAE,WAAW,EAAE,EAAE,CAAC;oBAC9F,MAAM,QAAQ,GAAG,YAAY,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;oBAC5C,IAAI,QAAQ,EAAE;wBACZ,8CAA8C;wBAC9C,eAAe,GAAG,KAAK,CAAC;wBAExB,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC;4BAChB,SAAS,EAAE;gCACT,MAAM,EAAE;oCACN,GAAG,EAAE,GAAG,CAAC,GAAG;iCACb;gCACD,MAAM,EAAE;oCACN,IAAI,EAAE;wCACJ,EAAE,EAAE,MAAM;wCACV,SAAS,EAAE,QAAQ;qCACpB;oCACD,MAAM,EAAE;wCACN,YAAY,EAAE,CAAC;wCACf,UAAU,EAAE,CAAC;wCACb,KAAK,EAAE,CAAC;wCACR,MAAM,EAAE,CAAC;wCACT,IAAI,EAAE,CAAC;qCACR;iCACF;6BACF;yBACF,CAAC,CAAC;qBACJ;yBAAM;wBACL,IAAI,YAAY,CAAC,YAAY,IAAI,YAAY,EAAE;4BAC7C,wBAAwB;4BACxB,6CAA6C;yBAC9C;6BAAM;4BACL,4DAA4D;4BAC5D,YAAY,CAAC,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;4BAC/C,6BAA6B;4BAC7B,oBAAoB;4BACpB,sBAAsB;4BACtB,6BAA6B;4BAC7B,YAAY,CAAC,YAAY,IAAI,GAAG,CAAC,MAAM,GAAG,GAAG,CAAC;yBAC/C;qBACF;iBACF;gBAED,IAAI,eAAe,EAAE;oBACnB,YAAY,CAAC,UAAU,GAAG,IAAI,CAAC;oBAC/B,YAAY,CAAC,WAAW,GAAG,CAAC,CAAC;iBAC9B;qBAAM,IAAI,GAAG,CAAC,EAAE,IAAI,OAAO,EAAE;oBAC5B,IAAI,YAAY,CAAC,UAAU,IAAI,IAAI,EAAE;wBACnC,YAAY,CAAC,UAAU,GAAG,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC;qBACrC;oBACD,YAAY,CAAC,WAAW,IAAI,CAAC,CAAC;iBAC/B;gBAED,IAAI,IAAI,CAAC,OAAO,CAAC,MAAM,IAAI,IAAI,CAAC,cAAc,EAAE;oBAC9C,MAAM,IAAI,CAAC,KAAK,EAAE,CAAC;iBACpB;aACF;SACF;QAED,MAAM,IAAI,CAAC,KAAK,EAAE,CAAC;QACnB,YAAY,EAAE,IAAI,CAAC,KAAK,EAAE,CAAC;QAC3B,IAAI,YAAY,EAAE,UAAU,IAAI,IAAI,IAAI,YAAY,EAAE,WAAW,GAAG,CAAC,EAAE;YACrE,MAAM,CAAC,IAAI,CACT,sBAAsB,IAAI,CAAC,QAAQ,IAAI,YAAY,CAAC,MAAM,IAAI,YAAY,CAAC,UAAU,cAAc,YAAY,CAAC,WAAW,aAAa,CACzI,CAAC;YACF,MAAM,MAAM,GAAG,YAAY,CAAC,MAAM,CAAC;YACnC,MAAM,OAAO,GAAG,YAAY,CAAC,UAAU,CAAC;YACxC,qCAAqC;YACrC,YAAY,GAAG,IAAI,CAAC;YACpB,MAAM,IAAI,CAAC,WAAW,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;SACzC;IACH,CAAC;IAEO,KAAK,CAAC,KAAK;QACjB,IAAI,IAAI,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE;YAC3B,MAAM,CAAC,IAAI,CAAC,cAAc,IAAI,CAAC,OAAO,CAAC,MAAM,MAAM,CAAC,CAAC;YACrD,MAAM,IAAI,CAAC,EAAE,CAAC,WAAW,CAAC,SAAS,CAAC,IAAI,CAAC,OAAO,EAAE;gBAChD,0BAA0B;gBAC1B,8EAA8E;gBAC9E,yDAAyD;gBACzD,uCAAuC;gBACvC,OAAO,EAAE,KAAK;aACf,CAAC,CAAC;YACH,IAAI,CAAC,OAAO,GAAG,EAAE,CAAC;SACnB;IACH,CAAC;IAED;;;;;OAKG;IACK,KAAK,CAAC,WAAW,CAAC,MAAc,EAAE,EAAU;QAClD,MAAM,QAAQ,GAAG;YACf,GAAG,EAAE;gBACH,IAAI,EAAE;oBACJ,CAAC,EAAE,IAAI,CAAC,QAAQ;oBAChB,CAAC,EAAE,MAAM;oBACT,CAAC,EAAE,IAAI,MAAM,EAAS;iBACvB;gBACD,IAAI,EAAE;oBACJ,CAAC,EAAE,IAAI,CAAC,QAAQ;oBAChB,CAAC,EAAE,MAAM;oBACT,CAAC,EAAE,EAAE;iBACN;aACF;SACF,CAAC;QAEF,MAAM,OAAO,GAAG,IAAI,CAAC,EAAE,CAAC,MAAM,CAAC,YAAY,EAAE,CAAC;QAC9C,IAAI;YACF,IAAI,IAAI,GAAG,KAAK,CAAC;YACjB,OAAO,CAAC,IAAI,EAAE;gBACZ,6EAA6E;gBAC7E,kDAAkD;gBAClD,0EAA0E;gBAC1E,MAAM,OAAO,CAAC,eAAe,CAC3B,KAAK,IAAI,EAAE;oBACT,MAAM,KAAK,GAAG,IAAI,CAAC,EAAE,CAAC,WAAW,CAAC,IAAI,CAAC,QAAQ,EAAE;wBAC/C,OAAO;wBACP,IAAI,EAAE,EAAE,GAAG,EAAE,CAAC,EAAE;wBAChB,UAAU,EAAE;4BACV,GAAG,EAAE,CAAC;4BACN,EAAE,EAAE,CAAC;4BACL,QAAQ,EAAE,CAAC;4BACX,SAAS,EAAE,CAAC;yBACb;wBACD,KAAK,EAAE,IAAI,CAAC,eAAe;qBAC5B,CAAC,CAAC;oBACH,IAAI,QAAQ,GAAG,CAAC,CAAC;oBACjB,IAAI,QAAQ,GAAyB,IAAI,CAAC;oBAC1C,IAAI,QAAQ,GAAkB,IAAI,CAAC;oBACnC,IAAI,OAAO,GAAG,KAAK,CAAC;oBACpB,IAAI,KAAK,EAAE,IAAI,EAAE,IAAI,KAAK,CAAC,MAAM,EAAE,EAAE;wBACnC,IAAI,EAAE,CAAC,EAAE,IAAI,MAAM,IAAI,EAAE,CAAC,EAAE,IAAI,QAAQ,IAAI,EAAE,CAAC,EAAE,IAAI,OAAO,EAAE;4BAC5D,QAAQ,GAAG,YAAY,CAAC,QAAQ,EAAE,EAAE,CAAC,QAAQ,CAAC,CAAC;4BAC/C,QAAQ,GAAG,EAAE,CAAC,GAAG,CAAC;4BAClB,IAAI,EAAE,CAAC,EAAE,IAAI,OAAO,EAAE;gCACpB,OAAO,GAAG,IAAI,CAAC;6BAChB;4BACD,IAAI,EAAE,CAAC,SAAS,IAAI,IAAI,EAAE;gCACxB,IAAI,QAAQ,IAAI,IAAI,IAAI,EAAE,CAAC,SAAS,GAAG,QAAQ,EAAE;oCAC/C,QAAQ,GAAG,EAAE,CAAC,SAAS,CAAC;iCACzB;6BACF;yBACF;6BAAM;4BACL,MAAM,IAAI,KAAK,CAAC,cAAc,EAAE,CAAC,EAAE,iBAAiB,EAAE,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC;yBACzF;qBACF;oBACD,IAAI,CAAC,OAAO,EAAE;wBACZ,IAAI,GAAG,IAAI,CAAC;wBACZ,OAAO;qBACR;oBAED,MAAM,CAAC,IAAI,CAAC,qBAAqB,QAAQ,EAAE,CAAC,EAAE,CAAC,CAAC;oBAChD,MAAM,IAAI,CAAC,EAAE,CAAC,WAAW,CAAC,UAAU,CAClC;wBACE,GAAG,EAAE;4BACH,IAAI,EAAE;gCACJ,CAAC,EAAE,IAAI,CAAC,QAAQ;gCAChB,CAAC,EAAE,MAAM;gCACT,CAAC,EAAE,IAAI,MAAM,EAAS;6BACvB;4BACD,IAAI,EAAE,QAAS;yBAChB;qBACF,EACD,EAAE,OAAO,EAAE,CACZ,CAAC;oBAEF,MAAM,IAAI,CAAC,EAAE,CAAC,WAAW,CAAC,SAAS,CACjC;wBACE,GAAG,EAAE,QAAS;wBACd,EAAE,EAAE,OAAO;wBACX,QAAQ,EAAE,QAAQ;wBAClB,IAAI,EAAE,IAAI;wBACV,SAAS,EAAE,QAAQ;qBACpB,EACD,EAAE,OAAO,EAAE,CACZ,CAAC;gBACJ,CAAC,EACD;oBACE,YAAY,EAAE,EAAE,CAAC,EAAE,UAAU,EAAE;oBAC/B,WAAW,EAAE,EAAE,KAAK,EAAE,UAAU,EAAE;iBACnC,CACF,CAAC;aACH;SACF;gBAAS;YACR,MAAM,OAAO,CAAC,UAAU,EAAE,CAAC;SAC5B;IACH,CAAC;CACF;AAED;;;;;GAKG;AACH,SAAS,OAAO,CAAC,CAAS;IACxB,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;IACf,OAAO,CAAC,CAAC;AACX,CAAC"}
|
package/package.json
CHANGED
|
@@ -5,7 +5,7 @@
|
|
|
5
5
|
"publishConfig": {
|
|
6
6
|
"access": "public"
|
|
7
7
|
},
|
|
8
|
-
"version": "0.5.
|
|
8
|
+
"version": "0.5.1",
|
|
9
9
|
"main": "dist/index.js",
|
|
10
10
|
"license": "FSL-1.1-Apache-2.0",
|
|
11
11
|
"type": "module",
|
|
@@ -35,9 +35,9 @@
|
|
|
35
35
|
"yaml": "^2.3.2",
|
|
36
36
|
"@powersync/lib-services-framework": "0.1.1",
|
|
37
37
|
"@powersync/service-jpgwire": "0.17.13",
|
|
38
|
+
"@powersync/service-jsonbig": "0.17.10",
|
|
38
39
|
"@powersync/service-rsocket-router": "0.0.10",
|
|
39
40
|
"@powersync/service-sync-rules": "0.18.1",
|
|
40
|
-
"@powersync/service-jsonbig": "0.17.10",
|
|
41
41
|
"@powersync/service-types": "0.1.0"
|
|
42
42
|
},
|
|
43
43
|
"devDependencies": {
|
|
@@ -2,8 +2,7 @@ import { Command } from 'commander';
|
|
|
2
2
|
|
|
3
3
|
import { logger } from '@powersync/lib-services-framework';
|
|
4
4
|
import * as v8 from 'v8';
|
|
5
|
-
import {
|
|
6
|
-
import { MongoBucketStorage, PowerSyncMongo } from '../../storage/storage-index.js';
|
|
5
|
+
import { createPowerSyncMongo, MongoBucketStorage } from '../../storage/storage-index.js';
|
|
7
6
|
import { loadConfig } from '../../util/config.js';
|
|
8
7
|
import { extractRunnerOptions, wrapConfigCommand } from './config-command.js';
|
|
9
8
|
|
|
@@ -31,10 +30,10 @@ export function registerCompactAction(program: Command) {
|
|
|
31
30
|
|
|
32
31
|
const config = await loadConfig(runnerConfig);
|
|
33
32
|
const { storage } = config;
|
|
34
|
-
const
|
|
33
|
+
const psdb = createPowerSyncMongo(storage);
|
|
34
|
+
const client = psdb.client;
|
|
35
35
|
await client.connect();
|
|
36
36
|
try {
|
|
37
|
-
const psdb = new PowerSyncMongo(client);
|
|
38
37
|
const bucketStorage = new MongoBucketStorage(psdb, { slot_name_prefix: config.slot_name_prefix });
|
|
39
38
|
const active = await bucketStorage.getActiveSyncRules();
|
|
40
39
|
if (active == null) {
|
|
@@ -426,4 +426,11 @@ export interface CompactOptions {
|
|
|
426
426
|
* not be compacted, to avoid invalidating checkpoints in use.
|
|
427
427
|
*/
|
|
428
428
|
maxOpId?: bigint;
|
|
429
|
+
|
|
430
|
+
/**
|
|
431
|
+
* If specified, compact only the specific buckets.
|
|
432
|
+
*
|
|
433
|
+
* If not specified, compacts all buckets.
|
|
434
|
+
*/
|
|
435
|
+
compactBuckets?: string[];
|
|
429
436
|
}
|
|
@@ -55,6 +55,7 @@ export class MongoCompactor {
|
|
|
55
55
|
private moveBatchQueryLimit: number;
|
|
56
56
|
private clearBatchLimit: number;
|
|
57
57
|
private maxOpId: bigint | undefined;
|
|
58
|
+
private buckets: string[] | undefined;
|
|
58
59
|
|
|
59
60
|
constructor(private db: PowerSyncMongo, private group_id: number, options?: MongoCompactOptions) {
|
|
60
61
|
this.idLimitBytes = (options?.memoryLimitMB ?? DEFAULT_MEMORY_LIMIT_MB) * 1024 * 1024;
|
|
@@ -62,6 +63,7 @@ export class MongoCompactor {
|
|
|
62
63
|
this.moveBatchQueryLimit = options?.moveBatchQueryLimit ?? DEFAULT_MOVE_BATCH_QUERY_LIMIT;
|
|
63
64
|
this.clearBatchLimit = options?.clearBatchLimit ?? DEFAULT_CLEAR_BATCH_LIMIT;
|
|
64
65
|
this.maxOpId = options?.maxOpId;
|
|
66
|
+
this.buckets = options?.compactBuckets;
|
|
65
67
|
}
|
|
66
68
|
|
|
67
69
|
/**
|
|
@@ -70,6 +72,19 @@ export class MongoCompactor {
|
|
|
70
72
|
* See /docs/compacting-operations.md for details.
|
|
71
73
|
*/
|
|
72
74
|
async compact() {
|
|
75
|
+
if (this.buckets) {
|
|
76
|
+
for (let bucket of this.buckets) {
|
|
77
|
+
// We can make this more efficient later on by iterating
|
|
78
|
+
// through the buckets in a single query.
|
|
79
|
+
// That makes batching more tricky, so we leave for later.
|
|
80
|
+
await this.compactInternal(bucket);
|
|
81
|
+
}
|
|
82
|
+
} else {
|
|
83
|
+
await this.compactInternal(undefined);
|
|
84
|
+
}
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
async compactInternal(bucket: string | undefined) {
|
|
73
88
|
const idLimitBytes = this.idLimitBytes;
|
|
74
89
|
|
|
75
90
|
let currentState: CurrentBucketState | null = null;
|
|
@@ -77,14 +92,14 @@ export class MongoCompactor {
|
|
|
77
92
|
// Constant lower bound
|
|
78
93
|
const lowerBound: BucketDataKey = {
|
|
79
94
|
g: this.group_id,
|
|
80
|
-
b: new MinKey() as any,
|
|
95
|
+
b: bucket ?? (new MinKey() as any),
|
|
81
96
|
o: new MinKey() as any
|
|
82
97
|
};
|
|
83
98
|
|
|
84
99
|
// Upper bound is adjusted for each batch
|
|
85
100
|
let upperBound: BucketDataKey = {
|
|
86
101
|
g: this.group_id,
|
|
87
|
-
b: new MaxKey() as any,
|
|
102
|
+
b: bucket ?? (new MaxKey() as any),
|
|
88
103
|
o: new MaxKey() as any
|
|
89
104
|
};
|
|
90
105
|
|