@powersync/service-core 0.0.0-dev-20241016143203 → 0.0.0-dev-20241016162519
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +8 -16
- package/dist/db/mongo.d.ts +0 -6
- package/dist/db/mongo.js +0 -6
- package/dist/db/mongo.js.map +1 -1
- package/dist/storage/BucketStorage.d.ts +6 -0
- package/dist/storage/BucketStorage.js.map +1 -1
- package/dist/storage/MongoBucketStorage.d.ts +10 -1
- package/dist/storage/MongoBucketStorage.js +33 -32
- package/dist/storage/MongoBucketStorage.js.map +1 -1
- package/dist/storage/mongo/MongoSyncBucketStorage.d.ts +1 -1
- package/dist/storage/mongo/MongoSyncBucketStorage.js +17 -26
- package/dist/storage/mongo/MongoSyncBucketStorage.js.map +1 -1
- package/dist/storage/mongo/MongoWriteCheckpointAPI.d.ts +3 -1
- package/dist/storage/mongo/MongoWriteCheckpointAPI.js +15 -7
- package/dist/storage/mongo/MongoWriteCheckpointAPI.js.map +1 -1
- package/dist/storage/mongo/db.d.ts +0 -9
- package/dist/storage/mongo/db.js +0 -11
- package/dist/storage/mongo/db.js.map +1 -1
- package/dist/storage/write-checkpoint.d.ts +2 -0
- package/dist/storage/write-checkpoint.js.map +1 -1
- package/dist/util/config/compound-config-collector.js +2 -1
- package/dist/util/config/compound-config-collector.js.map +1 -1
- package/dist/util/config/types.d.ts +1 -0
- package/package.json +5 -5
- package/src/db/mongo.ts +0 -7
- package/src/storage/BucketStorage.ts +9 -0
- package/src/storage/MongoBucketStorage.ts +41 -32
- package/src/storage/mongo/MongoSyncBucketStorage.ts +21 -26
- package/src/storage/mongo/MongoWriteCheckpointAPI.ts +18 -11
- package/src/storage/mongo/db.ts +0 -12
- package/src/storage/write-checkpoint.ts +4 -0
- package/src/util/config/compound-config-collector.ts +2 -1
- package/src/util/config/types.ts +1 -0
- package/test/src/data_storage.test.ts +0 -22
- package/test/src/sync.test.ts +16 -2
- package/test/src/util.ts +3 -14
- package/tsconfig.tsbuildinfo +1 -1
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"db.js","sourceRoot":"","sources":["../../../src/storage/mongo/db.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,MAAM,sBAAsB,CAAC;AAa3C,OAAO,EAAE,wBAAwB,EAAE,MAAM,WAAW,CAAC;AASrD,MAAM,UAAU,oBAAoB,CAAC,MAA6C;IAChF,OAAO,IAAI,cAAc,CAAC,EAAE,CAAC,KAAK,CAAC,iBAAiB,CAAC,MAAM,CAAC,EAAE,EAAE,QAAQ,EAAE,MAAM,CAAC,QAAQ,EAAE,CAAC,CAAC;AAC/F,CAAC;AAED,MAAM,OAAO,cAAc;IAezB,YAAY,MAAyB,EAAE,OAA+B;QACpE,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QAErB,MAAM,EAAE,GAAG,MAAM,CAAC,EAAE,CAAC,OAAO,EAAE,QAAQ,EAAE;YACtC,GAAG,wBAAwB;SAC5B,CAAC,CAAC;QACH,IAAI,CAAC,EAAE,GAAG,EAAE,CAAC;QAEb,IAAI,CAAC,YAAY,GAAG,EAAE,CAAC,UAAU,CAAsB,cAAc,CAAC,CAAC;QACvE,IAAI,CAAC,WAAW,GAAG,EAAE,CAAC,UAAU,CAAC,aAAa,CAAC,CAAC;QAChD,IAAI,CAAC,iBAAiB,GAAG,EAAE,CAAC,UAAU,CAAC,mBAAmB,CAAC,CAAC;QAC5D,IAAI,CAAC,cAAc,GAAG,EAAE,CAAC,UAAU,CAAC,gBAAgB,CAAC,CAAC;QACtD,IAAI,CAAC,UAAU,GAAG,EAAE,CAAC,UAAU,CAAC,YAAY,CAAC,CAAC;QAC9C,IAAI,CAAC,aAAa,GAAG,EAAE,CAAC,UAAU,CAAC,eAAe,CAAC,CAAC;QACpD,IAAI,CAAC,wBAAwB,GAAG,EAAE,CAAC,UAAU,CAAC,0BAA0B,CAAC,CAAC;QAC1E,IAAI,CAAC,iBAAiB,GAAG,EAAE,CAAC,UAAU,CAAC,mBAAmB,CAAC,CAAC;QAC5D,IAAI,CAAC,QAAQ,GAAG,EAAE,CAAC,UAAU,CAAC,UAAU,CAAC,CAAC;QAC1C,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC;IAC3C,CAAC;IAED
|
|
1
|
+
{"version":3,"file":"db.js","sourceRoot":"","sources":["../../../src/storage/mongo/db.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,MAAM,sBAAsB,CAAC;AAa3C,OAAO,EAAE,wBAAwB,EAAE,MAAM,WAAW,CAAC;AASrD,MAAM,UAAU,oBAAoB,CAAC,MAA6C;IAChF,OAAO,IAAI,cAAc,CAAC,EAAE,CAAC,KAAK,CAAC,iBAAiB,CAAC,MAAM,CAAC,EAAE,EAAE,QAAQ,EAAE,MAAM,CAAC,QAAQ,EAAE,CAAC,CAAC;AAC/F,CAAC;AAED,MAAM,OAAO,cAAc;IAezB,YAAY,MAAyB,EAAE,OAA+B;QACpE,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QAErB,MAAM,EAAE,GAAG,MAAM,CAAC,EAAE,CAAC,OAAO,EAAE,QAAQ,EAAE;YACtC,GAAG,wBAAwB;SAC5B,CAAC,CAAC;QACH,IAAI,CAAC,EAAE,GAAG,EAAE,CAAC;QAEb,IAAI,CAAC,YAAY,GAAG,EAAE,CAAC,UAAU,CAAsB,cAAc,CAAC,CAAC;QACvE,IAAI,CAAC,WAAW,GAAG,EAAE,CAAC,UAAU,CAAC,aAAa,CAAC,CAAC;QAChD,IAAI,CAAC,iBAAiB,GAAG,EAAE,CAAC,UAAU,CAAC,mBAAmB,CAAC,CAAC;QAC5D,IAAI,CAAC,cAAc,GAAG,EAAE,CAAC,UAAU,CAAC,gBAAgB,CAAC,CAAC;QACtD,IAAI,CAAC,UAAU,GAAG,EAAE,CAAC,UAAU,CAAC,YAAY,CAAC,CAAC;QAC9C,IAAI,CAAC,aAAa,GAAG,EAAE,CAAC,UAAU,CAAC,eAAe,CAAC,CAAC;QACpD,IAAI,CAAC,wBAAwB,GAAG,EAAE,CAAC,UAAU,CAAC,0BAA0B,CAAC,CAAC;QAC1E,IAAI,CAAC,iBAAiB,GAAG,EAAE,CAAC,UAAU,CAAC,mBAAmB,CAAC,CAAC;QAC5D,IAAI,CAAC,QAAQ,GAAG,EAAE,CAAC,UAAU,CAAC,UAAU,CAAC,CAAC;QAC1C,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC;IAC3C,CAAC;IAED,KAAK,CAAC,KAAK;QACT,MAAM,IAAI,CAAC,YAAY,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC;QACvC,MAAM,IAAI,CAAC,WAAW,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC;QACtC,MAAM,IAAI,CAAC,iBAAiB,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC;QAC5C,MAAM,IAAI,CAAC,cAAc,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC;QACzC,MAAM,IAAI,CAAC,UAAU,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC;QACrC,MAAM,IAAI,CAAC,aAAa,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC;QACxC,MAAM,IAAI,CAAC,iBAAiB,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC;QAC5C,MAAM,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC,EAAE,CAAC,CAAC;QAClC,MAAM,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC;IAClC,CAAC;CACF"}
|
|
@@ -47,6 +47,8 @@ export interface ManagedWriteCheckpointFilters extends BaseWriteCheckpointIdenti
|
|
|
47
47
|
export type ManagedWriteCheckpointOptions = ManagedWriteCheckpointFilters;
|
|
48
48
|
export type LastWriteCheckpointFilters = CustomWriteCheckpointFilters | ManagedWriteCheckpointFilters;
|
|
49
49
|
export interface WriteCheckpointAPI {
|
|
50
|
+
readonly writeCheckpointMode: WriteCheckpointMode;
|
|
51
|
+
setWriteCheckpointMode(mode: WriteCheckpointMode): void;
|
|
50
52
|
batchCreateCustomWriteCheckpoints(checkpoints: CustomWriteCheckpointOptions[]): Promise<void>;
|
|
51
53
|
createCustomWriteCheckpoint(checkpoint: CustomWriteCheckpointOptions): Promise<bigint>;
|
|
52
54
|
createManagedWriteCheckpoint(checkpoint: ManagedWriteCheckpointOptions): Promise<bigint>;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"write-checkpoint.js","sourceRoot":"","sources":["../../src/storage/write-checkpoint.ts"],"names":[],"mappings":"AAAA,MAAM,CAAN,IAAY,mBAYX;AAZD,WAAY,mBAAmB;IAC7B;;;OAGG;IACH,wCAAiB,CAAA;IACjB;;;;OAIG;IACH,0CAAmB,CAAA;AACrB,CAAC,EAZW,mBAAmB,KAAnB,mBAAmB,QAY9B;
|
|
1
|
+
{"version":3,"file":"write-checkpoint.js","sourceRoot":"","sources":["../../src/storage/write-checkpoint.ts"],"names":[],"mappings":"AAAA,MAAM,CAAN,IAAY,mBAYX;AAZD,WAAY,mBAAmB;IAC7B;;;OAGG;IACH,wCAAiB,CAAA;IACjB;;;;OAIG;IACH,0CAAmB,CAAA;AACrB,CAAC,EAZW,mBAAmB,KAAnB,mBAAmB,QAY9B;AA0DD,MAAM,CAAC,MAAM,6BAA6B,GAAG,mBAAmB,CAAC,OAAO,CAAC"}
|
|
@@ -75,7 +75,8 @@ export class CompoundConfigCollector {
|
|
|
75
75
|
},
|
|
76
76
|
// TODO maybe move this out of the connection or something
|
|
77
77
|
// slot_name_prefix: connections[0]?.slot_name_prefix ?? 'powersync_'
|
|
78
|
-
slot_name_prefix: 'powersync_'
|
|
78
|
+
slot_name_prefix: 'powersync_',
|
|
79
|
+
parameters: baseConfig.parameters ?? {}
|
|
79
80
|
};
|
|
80
81
|
return config;
|
|
81
82
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"compound-config-collector.js","sourceRoot":"","sources":["../../../src/util/config/compound-config-collector.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,mCAAmC,CAAC;AAE3D,OAAO,KAAK,IAAI,MAAM,0BAA0B,CAAC;AAEjD,OAAO,EAAE,qBAAqB,EAAE,MAAM,8CAA8C,CAAC;AACrF,OAAO,EAAE,uBAAuB,EAAE,MAAM,gDAAgD,CAAC;AACzF,OAAO,EAAE,yBAAyB,EAAE,MAAM,kDAAkD,CAAC;AAC7F,OAAO,EAAE,wBAAwB,EAAE,MAAM,kDAAkD,CAAC;AAC5F,OAAO,EAAE,4BAA4B,EAAE,MAAM,sDAAsD,CAAC;AACpG,OAAO,EAAE,wBAAwB,EAAE,MAAM,kDAAkD,CAAC;AA4B5F,MAAM,iBAAiB,GAAG,eAAe,CAAC;AAE1C,MAAM,yBAAyB,GAAmC;IAChE,gBAAgB,EAAE,CAAC,IAAI,qBAAqB,EAAE,EAAE,IAAI,yBAAyB,EAAE,EAAE,IAAI,uBAAuB,EAAE,CAAC;IAC/G,mBAAmB,EAAE;QACnB,IAAI,wBAAwB,EAAE;QAC9B,IAAI,4BAA4B,EAAE;QAClC,IAAI,wBAAwB,EAAE;KAC/B;CACF,CAAC;AAEF,MAAM,OAAO,uBAAuB;IAClC,YAAsB,UAA0C,yBAAyB;QAAnE,YAAO,GAAP,OAAO,CAA4D;IAAG,CAAC;IAE7F;;OAEG;IACH,KAAK,CAAC,aAAa,CAAC,eAA6B,EAAE;QACjD,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,iBAAiB,CAAC,YAAY,CAAC,CAAC;QAE9D,MAAM,WAAW,GAAG,UAAU,CAAC,WAAW,EAAE,WAAW,IAAI,EAAE,CAAC;QAC9D,IAAI,WAAW,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC3B,MAAM,IAAI,KAAK,CAAC,8DAA8D,CAAC,CAAC;QAClF,CAAC;QAED,MAAM,UAAU,GAAG,IAAI,IAAI,CAAC,oBAAoB,EAAE,CAAC;QACnD,MAAM,QAAQ,GAAG,IAAI,IAAI,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAC;QAE/C,MAAM,SAAS,GAAG,UAAU,CAAC,WAAW,EAAE,IAAI,EAAE,IAAI,IAAI,EAAE,CAAC;QAC3D,MAAM,eAAe,GAAG,MAAM,IAAI,CAAC,kBAAkB,CAAC,UAAU,CAAC,SAAS,CAAC,CAAC;QAE5E,UAAU,CAAC,GAAG,CAAC,eAAe,CAAC,CAAC;QAEhC,IAAI,SAAS,GAAG,UAAU,CAAC,WAAW,EAAE,QAAQ,IAAI,EAAE,CAAC;QACvD,IAAI,OAAO,SAAS,IAAI,QAAQ,EAAE,CAAC;YACjC,SAAS,GAAG,CAAC,SAAS,CAAC,CAAC;QAC1B,CAAC;QAED,KAAK,IAAI,GAAG,IAAI,SAAS,EAAE,CAAC;YAC1B,UAAU,CAAC,GAAG,CACZ,IAAI,IAAI,CAAC,kBAAkB,CACzB,IAAI,IAAI,CAAC,mBAAmB,CAAC,GAAG,EAAE,EAAE,cAAc,EAAE,CAAC,CAAC,UAAU,CAAC,WAAW,EAAE,gBAAgB,EAAE,CAAC,CAClG,CACF,CAAC;QACJ,CAAC;QAED,MAAM,UAAU,GAAG,CAAC,UAAU,CAAC,WAAW,EAAE,IAAI,EAAE,IAAI,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,GAAG,IAAI,iBAAiB,CAAC,CAAC;QAE1G,IAAI,MAAgC,CAAC;QACrC,IAAI,UAAU,CAAC,GAAG,EAAE,SAAS,IAAI,UAAU,IAAI,IAAI,IAAI,UAAU,CAAC,GAAG,IAAI,KAAK,EAAE,CAAC;YAC/E,MAAM,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC,UAAU,CAAC,CAAC;QACpD,CAAC;QAED,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,gBAAgB,CAAC,UAAU,EAAE,YAAY,CAAC,CAAC;QAEzE,IAAI,aAAa,GAAa,UAAU,CAAC,WAAW,EAAE,QAAQ,IAAI,EAAE,CAAC;QAErE,IAAI,MAAM,GAA4B;YACpC,WAAW,EAAE,UAAU;YACvB,WAAW,EAAE,UAAU,CAAC,WAAW,EAAE,WAAW,IAAI,EAAE;YACtD,OAAO,EAAE,UAAU,CAAC,OAAO;YAC3B,eAAe,EAAE,QAAQ;YACzB,+DAA+D;YAC/D,mEAAmE;YACnE,mBAAmB,EAAE,IAAI,IAAI,CAAC,QAAQ,CAAC,eAAe,CAAC;YACvD,UAAU,EAAE,UAAU,CAAC,GAAG,EAAE,MAAM,IAAI,EAAE;YACxC,GAAG,EAAE;gBACH,SAAS,EAAE,UAAU,CAAC,GAAG,EAAE,SAAS,IAAI,KAAK;gBAC7C,WAAW,EAAE,UAAU,CAAC,GAAG,EAAE,WAAW,IAAI,KAAK;gBACjD,aAAa,EAAE,UAAU,CAAC,GAAG,EAAE,aAAa;gBAC5C,QAAQ,EAAE,UAAU,CAAC,GAAG,EAAE,QAAQ,IAAI,KAAK;gBAC3C,OAAO,EAAE,MAAM;aAChB;YACD,IAAI,EAAE,UAAU,CAAC,IAAI,IAAI,IAAI;YAC7B,UAAU;YACV,aAAa;YAEb,oBAAoB,EAAE,IAAI,EAAE,QAAQ;YACpC,QAAQ,EAAE,UAAU,CAAC,QAAQ,IAAI,EAAE;YACnC,UAAU,EAAE,UAAU,CAAC,UAAU;YACjC,SAAS,EAAE;gBACT,yBAAyB,EAAE,UAAU,CAAC,SAAS,EAAE,yBAAyB,IAAI,KAAK;gBACnF,yBAAyB,EACvB,UAAU,CAAC,SAAS,EAAE,yBAAyB,IAAI,0CAA0C;aAChG;YACD,0DAA0D;YAC1D,qEAAqE;YACrE,gBAAgB,EAAE,YAAY;
|
|
1
|
+
{"version":3,"file":"compound-config-collector.js","sourceRoot":"","sources":["../../../src/util/config/compound-config-collector.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,mCAAmC,CAAC;AAE3D,OAAO,KAAK,IAAI,MAAM,0BAA0B,CAAC;AAEjD,OAAO,EAAE,qBAAqB,EAAE,MAAM,8CAA8C,CAAC;AACrF,OAAO,EAAE,uBAAuB,EAAE,MAAM,gDAAgD,CAAC;AACzF,OAAO,EAAE,yBAAyB,EAAE,MAAM,kDAAkD,CAAC;AAC7F,OAAO,EAAE,wBAAwB,EAAE,MAAM,kDAAkD,CAAC;AAC5F,OAAO,EAAE,4BAA4B,EAAE,MAAM,sDAAsD,CAAC;AACpG,OAAO,EAAE,wBAAwB,EAAE,MAAM,kDAAkD,CAAC;AA4B5F,MAAM,iBAAiB,GAAG,eAAe,CAAC;AAE1C,MAAM,yBAAyB,GAAmC;IAChE,gBAAgB,EAAE,CAAC,IAAI,qBAAqB,EAAE,EAAE,IAAI,yBAAyB,EAAE,EAAE,IAAI,uBAAuB,EAAE,CAAC;IAC/G,mBAAmB,EAAE;QACnB,IAAI,wBAAwB,EAAE;QAC9B,IAAI,4BAA4B,EAAE;QAClC,IAAI,wBAAwB,EAAE;KAC/B;CACF,CAAC;AAEF,MAAM,OAAO,uBAAuB;IAClC,YAAsB,UAA0C,yBAAyB;QAAnE,YAAO,GAAP,OAAO,CAA4D;IAAG,CAAC;IAE7F;;OAEG;IACH,KAAK,CAAC,aAAa,CAAC,eAA6B,EAAE;QACjD,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,iBAAiB,CAAC,YAAY,CAAC,CAAC;QAE9D,MAAM,WAAW,GAAG,UAAU,CAAC,WAAW,EAAE,WAAW,IAAI,EAAE,CAAC;QAC9D,IAAI,WAAW,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC3B,MAAM,IAAI,KAAK,CAAC,8DAA8D,CAAC,CAAC;QAClF,CAAC;QAED,MAAM,UAAU,GAAG,IAAI,IAAI,CAAC,oBAAoB,EAAE,CAAC;QACnD,MAAM,QAAQ,GAAG,IAAI,IAAI,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAC;QAE/C,MAAM,SAAS,GAAG,UAAU,CAAC,WAAW,EAAE,IAAI,EAAE,IAAI,IAAI,EAAE,CAAC;QAC3D,MAAM,eAAe,GAAG,MAAM,IAAI,CAAC,kBAAkB,CAAC,UAAU,CAAC,SAAS,CAAC,CAAC;QAE5E,UAAU,CAAC,GAAG,CAAC,eAAe,CAAC,CAAC;QAEhC,IAAI,SAAS,GAAG,UAAU,CAAC,WAAW,EAAE,QAAQ,IAAI,EAAE,CAAC;QACvD,IAAI,OAAO,SAAS,IAAI,QAAQ,EAAE,CAAC;YACjC,SAAS,GAAG,CAAC,SAAS,CAAC,CAAC;QAC1B,CAAC;QAED,KAAK,IAAI,GAAG,IAAI,SAAS,EAAE,CAAC;YAC1B,UAAU,CAAC,GAAG,CACZ,IAAI,IAAI,CAAC,kBAAkB,CACzB,IAAI,IAAI,CAAC,mBAAmB,CAAC,GAAG,EAAE,EAAE,cAAc,EAAE,CAAC,CAAC,UAAU,CAAC,WAAW,EAAE,gBAAgB,EAAE,CAAC,CAClG,CACF,CAAC;QACJ,CAAC;QAED,MAAM,UAAU,GAAG,CAAC,UAAU,CAAC,WAAW,EAAE,IAAI,EAAE,IAAI,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,GAAG,IAAI,iBAAiB,CAAC,CAAC;QAE1G,IAAI,MAAgC,CAAC;QACrC,IAAI,UAAU,CAAC,GAAG,EAAE,SAAS,IAAI,UAAU,IAAI,IAAI,IAAI,UAAU,CAAC,GAAG,IAAI,KAAK,EAAE,CAAC;YAC/E,MAAM,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC,UAAU,CAAC,CAAC;QACpD,CAAC;QAED,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,gBAAgB,CAAC,UAAU,EAAE,YAAY,CAAC,CAAC;QAEzE,IAAI,aAAa,GAAa,UAAU,CAAC,WAAW,EAAE,QAAQ,IAAI,EAAE,CAAC;QAErE,IAAI,MAAM,GAA4B;YACpC,WAAW,EAAE,UAAU;YACvB,WAAW,EAAE,UAAU,CAAC,WAAW,EAAE,WAAW,IAAI,EAAE;YACtD,OAAO,EAAE,UAAU,CAAC,OAAO;YAC3B,eAAe,EAAE,QAAQ;YACzB,+DAA+D;YAC/D,mEAAmE;YACnE,mBAAmB,EAAE,IAAI,IAAI,CAAC,QAAQ,CAAC,eAAe,CAAC;YACvD,UAAU,EAAE,UAAU,CAAC,GAAG,EAAE,MAAM,IAAI,EAAE;YACxC,GAAG,EAAE;gBACH,SAAS,EAAE,UAAU,CAAC,GAAG,EAAE,SAAS,IAAI,KAAK;gBAC7C,WAAW,EAAE,UAAU,CAAC,GAAG,EAAE,WAAW,IAAI,KAAK;gBACjD,aAAa,EAAE,UAAU,CAAC,GAAG,EAAE,aAAa;gBAC5C,QAAQ,EAAE,UAAU,CAAC,GAAG,EAAE,QAAQ,IAAI,KAAK;gBAC3C,OAAO,EAAE,MAAM;aAChB;YACD,IAAI,EAAE,UAAU,CAAC,IAAI,IAAI,IAAI;YAC7B,UAAU;YACV,aAAa;YAEb,oBAAoB,EAAE,IAAI,EAAE,QAAQ;YACpC,QAAQ,EAAE,UAAU,CAAC,QAAQ,IAAI,EAAE;YACnC,UAAU,EAAE,UAAU,CAAC,UAAU;YACjC,SAAS,EAAE;gBACT,yBAAyB,EAAE,UAAU,CAAC,SAAS,EAAE,yBAAyB,IAAI,KAAK;gBACnF,yBAAyB,EACvB,UAAU,CAAC,SAAS,EAAE,yBAAyB,IAAI,0CAA0C;aAChG;YACD,0DAA0D;YAC1D,qEAAqE;YACrE,gBAAgB,EAAE,YAAY;YAC9B,UAAU,EAAE,UAAU,CAAC,UAAU,IAAI,EAAE;SACxC,CAAC;QAEF,OAAO,MAAM,CAAC;IAChB,CAAC;IAED;;;OAGG;IACO,KAAK,CAAC,iBAAiB,CAAC,aAA2B;QAC3D,KAAK,MAAM,SAAS,IAAI,IAAI,CAAC,OAAO,CAAC,gBAAgB,EAAE,CAAC;YACtD,IAAI,CAAC;gBACH,MAAM,UAAU,GAAG,MAAM,SAAS,CAAC,OAAO,CAAC,aAAa,CAAC,CAAC;gBAC1D,IAAI,UAAU,EAAE,CAAC;oBACf,OAAO,UAAU,CAAC;gBACpB,CAAC;gBACD,MAAM,CAAC,KAAK,CACV,2CAA2C,SAAS,CAAC,IAAI,iDAAiD,CAC3G,CAAC;YACJ,CAAC;YAAC,OAAO,EAAE,EAAE,CAAC;gBACZ,yCAAyC;gBACzC,MAAM,IAAI,KAAK,CAAC,kCAAkC,SAAS,CAAC,IAAI,8BAA8B,EAAE,EAAE,CAAC,CAAC;YACtG,CAAC;QACH,CAAC;QACD,MAAM,IAAI,KAAK,CAAC,uFAAuF,CAAC,CAAC;IAC3G,CAAC;IAES,KAAK,CAAC,gBAAgB,CAC9B,UAAsC,EACtC,YAA0B;QAE1B,KAAK,MAAM,SAAS,IAAI,IAAI,CAAC,OAAO,CAAC,mBAAmB,EAAE,CAAC;YACzD,IAAI,CAAC;gBACH,MAAM,MAAM,GAAG,MAAM,SAAS,CAAC,OAAO,CAAC,UAAU,EAAE,YAAY,CAAC,CAAC;gBACjE,IAAI,MAAM,EAAE,CAAC;oBACX,OAAO,MAAM,CAAC;gBAChB,CAAC;gBACD,MAAM,CAAC,KAAK,CACV,qCAAqC,SAAS,CAAC,IAAI,iDAAiD,CACrG,CAAC;YACJ,CAAC;YAAC,OAAO,EAAE,EAAE,CAAC;gBACZ,yCAAyC;gBACzC,MAAM,IAAI,KAAK,CAAC,sCAAsC,SAAS,CAAC,IAAI,8BAA8B,EAAE,EAAE,CAAC,CAAC;YAC1G,CAAC;QACH,CAAC;QACD,OAAO;YACL,OAAO,EAAE,KAAK;SACf,CAAC;IACJ,CAAC;CACF"}
|
package/package.json
CHANGED
|
@@ -5,7 +5,7 @@
|
|
|
5
5
|
"publishConfig": {
|
|
6
6
|
"access": "public"
|
|
7
7
|
},
|
|
8
|
-
"version": "0.0.0-dev-
|
|
8
|
+
"version": "0.0.0-dev-20241016162519",
|
|
9
9
|
"main": "dist/index.js",
|
|
10
10
|
"license": "FSL-1.1-Apache-2.0",
|
|
11
11
|
"type": "module",
|
|
@@ -33,11 +33,11 @@
|
|
|
33
33
|
"uuid": "^9.0.1",
|
|
34
34
|
"winston": "^3.13.0",
|
|
35
35
|
"yaml": "^2.3.2",
|
|
36
|
-
"@powersync/lib-services-framework": "0.0.0-dev-
|
|
36
|
+
"@powersync/lib-services-framework": "0.0.0-dev-20241016162519",
|
|
37
37
|
"@powersync/service-jsonbig": "0.17.10",
|
|
38
|
-
"@powersync/service-rsocket-router": "0.0.0-dev-
|
|
39
|
-
"@powersync/service-sync-rules": "0.0.0-dev-
|
|
40
|
-
"@powersync/service-types": "0.0.0-dev-
|
|
38
|
+
"@powersync/service-rsocket-router": "0.0.0-dev-20241016162519",
|
|
39
|
+
"@powersync/service-sync-rules": "0.0.0-dev-20241016162519",
|
|
40
|
+
"@powersync/service-types": "0.0.0-dev-20241016162519"
|
|
41
41
|
},
|
|
42
42
|
"devDependencies": {
|
|
43
43
|
"@types/async": "^3.2.24",
|
package/src/db/mongo.ts
CHANGED
|
@@ -23,13 +23,6 @@ export const MONGO_SOCKET_TIMEOUT_MS = 60_000;
|
|
|
23
23
|
*/
|
|
24
24
|
export const MONGO_OPERATION_TIMEOUT_MS = 30_000;
|
|
25
25
|
|
|
26
|
-
/**
|
|
27
|
-
* Same as above, but specifically for clear operations.
|
|
28
|
-
*
|
|
29
|
-
* These are retried when reaching the timeout.
|
|
30
|
-
*/
|
|
31
|
-
export const MONGO_CLEAR_OPERATION_TIMEOUT_MS = 5_000;
|
|
32
|
-
|
|
33
26
|
export function createMongoClient(config: configFile.PowerSyncConfig['storage']) {
|
|
34
27
|
const normalized = normalizeMongoConfig(config);
|
|
35
28
|
return new mongo.MongoClient(normalized.uri, {
|
|
@@ -90,6 +90,11 @@ export interface BucketStorageFactory
|
|
|
90
90
|
*/
|
|
91
91
|
getActiveCheckpoint(): Promise<ActiveCheckpoint>;
|
|
92
92
|
|
|
93
|
+
/**
|
|
94
|
+
* Yields the latest sync checkpoint.
|
|
95
|
+
*/
|
|
96
|
+
watchActiveCheckpoint(signal: AbortSignal): AsyncIterable<ActiveCheckpoint>;
|
|
97
|
+
|
|
93
98
|
/**
|
|
94
99
|
* Yields the latest user write checkpoint whenever the sync checkpoint updates.
|
|
95
100
|
*/
|
|
@@ -118,6 +123,8 @@ export interface ActiveCheckpoint {
|
|
|
118
123
|
hasSyncRules(): boolean;
|
|
119
124
|
|
|
120
125
|
getBucketStorage(): Promise<SyncRulesBucketStorage | null>;
|
|
126
|
+
|
|
127
|
+
syncRules: PersistedSyncRules | null;
|
|
121
128
|
}
|
|
122
129
|
|
|
123
130
|
export interface StorageMetrics {
|
|
@@ -265,6 +272,8 @@ export interface SyncRulesBucketStorage extends DisposableObserverClient<SyncRul
|
|
|
265
272
|
*/
|
|
266
273
|
clear(): Promise<void>;
|
|
267
274
|
|
|
275
|
+
setSnapshotDone(lsn: string): Promise<void>;
|
|
276
|
+
|
|
268
277
|
autoActivate(): Promise<void>;
|
|
269
278
|
|
|
270
279
|
/**
|
|
@@ -78,6 +78,9 @@ export class MongoBucketStorage
|
|
|
78
78
|
db: PowerSyncMongo,
|
|
79
79
|
options: {
|
|
80
80
|
slot_name_prefix: string;
|
|
81
|
+
/**
|
|
82
|
+
* Initial Write Checkpoint Mode
|
|
83
|
+
*/
|
|
81
84
|
write_checkpoint_mode?: WriteCheckpointMode;
|
|
82
85
|
}
|
|
83
86
|
) {
|
|
@@ -93,6 +96,10 @@ export class MongoBucketStorage
|
|
|
93
96
|
});
|
|
94
97
|
}
|
|
95
98
|
|
|
99
|
+
get writeCheckpointMode() {
|
|
100
|
+
return this.writeCheckpointAPI.writeCheckpointMode;
|
|
101
|
+
}
|
|
102
|
+
|
|
96
103
|
getInstance(options: PersistedSyncRulesContent): MongoSyncBucketStorage {
|
|
97
104
|
let { id, slot_name } = options;
|
|
98
105
|
if ((typeof id as any) == 'bigint') {
|
|
@@ -303,6 +310,10 @@ export class MongoBucketStorage
|
|
|
303
310
|
return this.writeCheckpointAPI.batchCreateCustomWriteCheckpoints(checkpoints);
|
|
304
311
|
}
|
|
305
312
|
|
|
313
|
+
setWriteCheckpointMode(mode: WriteCheckpointMode): void {
|
|
314
|
+
return this.writeCheckpointAPI.setWriteCheckpointMode(mode);
|
|
315
|
+
}
|
|
316
|
+
|
|
306
317
|
async createCustomWriteCheckpoint(options: CustomWriteCheckpointOptions): Promise<bigint> {
|
|
307
318
|
return this.writeCheckpointAPI.createCustomWriteCheckpoint(options);
|
|
308
319
|
}
|
|
@@ -331,56 +342,39 @@ export class MongoBucketStorage
|
|
|
331
342
|
}
|
|
332
343
|
|
|
333
344
|
async getStorageMetrics(): Promise<StorageMetrics> {
|
|
334
|
-
const ignoreNotExiting = (e: unknown) => {
|
|
335
|
-
if (e instanceof mongo.MongoServerError && e.codeName == 'NamespaceNotFound') {
|
|
336
|
-
// Collection doesn't exist - return 0
|
|
337
|
-
return [{ storageStats: { size: 0 } }];
|
|
338
|
-
} else {
|
|
339
|
-
return Promise.reject(e);
|
|
340
|
-
}
|
|
341
|
-
};
|
|
342
|
-
|
|
343
|
-
const active_sync_rules = await this.getActiveSyncRules({ defaultSchema: 'public' });
|
|
344
|
-
if (active_sync_rules == null) {
|
|
345
|
-
return {
|
|
346
|
-
operations_size_bytes: 0,
|
|
347
|
-
parameters_size_bytes: 0,
|
|
348
|
-
replication_size_bytes: 0
|
|
349
|
-
};
|
|
350
|
-
}
|
|
351
345
|
const operations_aggregate = await this.db.bucket_data
|
|
352
346
|
|
|
353
347
|
.aggregate([
|
|
354
348
|
{
|
|
355
349
|
$collStats: {
|
|
356
|
-
storageStats: {}
|
|
350
|
+
storageStats: {},
|
|
351
|
+
count: {}
|
|
357
352
|
}
|
|
358
353
|
}
|
|
359
354
|
])
|
|
360
|
-
.toArray()
|
|
361
|
-
.catch(ignoreNotExiting);
|
|
355
|
+
.toArray();
|
|
362
356
|
|
|
363
357
|
const parameters_aggregate = await this.db.bucket_parameters
|
|
364
358
|
.aggregate([
|
|
365
359
|
{
|
|
366
360
|
$collStats: {
|
|
367
|
-
storageStats: {}
|
|
361
|
+
storageStats: {},
|
|
362
|
+
count: {}
|
|
368
363
|
}
|
|
369
364
|
}
|
|
370
365
|
])
|
|
371
|
-
.toArray()
|
|
372
|
-
.catch(ignoreNotExiting);
|
|
366
|
+
.toArray();
|
|
373
367
|
|
|
374
368
|
const replication_aggregate = await this.db.current_data
|
|
375
369
|
.aggregate([
|
|
376
370
|
{
|
|
377
371
|
$collStats: {
|
|
378
|
-
storageStats: {}
|
|
372
|
+
storageStats: {},
|
|
373
|
+
count: {}
|
|
379
374
|
}
|
|
380
375
|
}
|
|
381
376
|
])
|
|
382
|
-
.toArray()
|
|
383
|
-
.catch(ignoreNotExiting);
|
|
377
|
+
.toArray();
|
|
384
378
|
|
|
385
379
|
return {
|
|
386
380
|
operations_size_bytes: operations_aggregate[0].storageStats.size,
|
|
@@ -425,14 +419,19 @@ export class MongoBucketStorage
|
|
|
425
419
|
return null;
|
|
426
420
|
}
|
|
427
421
|
return (await this.storageCache.fetch(doc._id)) ?? null;
|
|
428
|
-
}
|
|
429
|
-
|
|
422
|
+
},
|
|
423
|
+
syncRules: doc?.content
|
|
424
|
+
? new MongoPersistedSyncRulesContent(this.db, doc).parsed({
|
|
425
|
+
defaultSchema: ''
|
|
426
|
+
})
|
|
427
|
+
: null
|
|
428
|
+
} satisfies ActiveCheckpoint;
|
|
430
429
|
}
|
|
431
430
|
|
|
432
431
|
/**
|
|
433
432
|
* Instance-wide watch on the latest available checkpoint (op_id + lsn).
|
|
434
433
|
*/
|
|
435
|
-
private async *
|
|
434
|
+
private async *_watchActiveCheckpoint(signal: AbortSignal): AsyncIterable<ActiveCheckpoint> {
|
|
436
435
|
const pipeline: mongo.Document[] = [
|
|
437
436
|
{
|
|
438
437
|
$match: {
|
|
@@ -445,7 +444,8 @@ export class MongoBucketStorage
|
|
|
445
444
|
operationType: 1,
|
|
446
445
|
'fullDocument._id': 1,
|
|
447
446
|
'fullDocument.last_checkpoint': 1,
|
|
448
|
-
'fullDocument.last_checkpoint_lsn': 1
|
|
447
|
+
'fullDocument.last_checkpoint_lsn': 1,
|
|
448
|
+
'fullDocument.content': 1
|
|
449
449
|
}
|
|
450
450
|
}
|
|
451
451
|
];
|
|
@@ -467,7 +467,8 @@ export class MongoBucketStorage
|
|
|
467
467
|
projection: {
|
|
468
468
|
_id: 1,
|
|
469
469
|
last_checkpoint: 1,
|
|
470
|
-
last_checkpoint_lsn: 1
|
|
470
|
+
last_checkpoint_lsn: 1,
|
|
471
|
+
content: 1
|
|
471
472
|
}
|
|
472
473
|
}
|
|
473
474
|
);
|
|
@@ -516,6 +517,7 @@ export class MongoBucketStorage
|
|
|
516
517
|
if (doc == null) {
|
|
517
518
|
continue;
|
|
518
519
|
}
|
|
520
|
+
|
|
519
521
|
const op = this.makeActiveCheckpoint(doc);
|
|
520
522
|
// Check for LSN / checkpoint changes - ignore other metadata changes
|
|
521
523
|
if (lastOp == null || op.lsn != lastOp.lsn || op.checkpoint != lastOp.checkpoint) {
|
|
@@ -527,9 +529,16 @@ export class MongoBucketStorage
|
|
|
527
529
|
|
|
528
530
|
// Nothing is done here until a subscriber starts to iterate
|
|
529
531
|
private readonly sharedIter = new sync.BroadcastIterable((signal) => {
|
|
530
|
-
return this.
|
|
532
|
+
return this._watchActiveCheckpoint(signal);
|
|
531
533
|
});
|
|
532
534
|
|
|
535
|
+
/**
|
|
536
|
+
* Watch changes to the active sync rules and checkpoint.
|
|
537
|
+
*/
|
|
538
|
+
watchActiveCheckpoint(signal: AbortSignal): AsyncIterable<ActiveCheckpoint> {
|
|
539
|
+
return wrapWithAbort(this.sharedIter, signal);
|
|
540
|
+
}
|
|
541
|
+
|
|
533
542
|
/**
|
|
534
543
|
* User-specific watch on the latest checkpoint and/or write checkpoint.
|
|
535
544
|
*/
|
|
@@ -31,8 +31,6 @@ import { BucketDataDocument, BucketDataKey, SourceKey, SyncRuleState } from './m
|
|
|
31
31
|
import { MongoBucketBatch } from './MongoBucketBatch.js';
|
|
32
32
|
import { MongoCompactor } from './MongoCompactor.js';
|
|
33
33
|
import { BSON_DESERIALIZE_OPTIONS, idPrefixFilter, mapOpEntry, readSingleBatch, serializeLookup } from './util.js';
|
|
34
|
-
import { logger } from '@powersync/lib-services-framework';
|
|
35
|
-
import * as timers from 'timers/promises';
|
|
36
34
|
|
|
37
35
|
export class MongoSyncBucketStorage
|
|
38
36
|
extends DisposableObserver<SyncRulesBucketStorageListener>
|
|
@@ -461,28 +459,10 @@ export class MongoSyncBucketStorage
|
|
|
461
459
|
}
|
|
462
460
|
|
|
463
461
|
async clear(): Promise<void> {
|
|
464
|
-
while (true) {
|
|
465
|
-
try {
|
|
466
|
-
await this.clearIteration();
|
|
467
|
-
return;
|
|
468
|
-
} catch (e: unknown) {
|
|
469
|
-
if (e instanceof mongo.MongoServerError && e.codeName == 'MaxTimeMSExpired') {
|
|
470
|
-
logger.info(
|
|
471
|
-
`Clearing took longer than ${db.mongo.MONGO_CLEAR_OPERATION_TIMEOUT_MS}ms, waiting and triggering another iteration.`
|
|
472
|
-
);
|
|
473
|
-
await timers.setTimeout(db.mongo.MONGO_CLEAR_OPERATION_TIMEOUT_MS / 5);
|
|
474
|
-
continue;
|
|
475
|
-
} else {
|
|
476
|
-
throw e;
|
|
477
|
-
}
|
|
478
|
-
}
|
|
479
|
-
}
|
|
480
|
-
}
|
|
481
|
-
|
|
482
|
-
private async clearIteration(): Promise<void> {
|
|
483
462
|
// Individual operations here may time out with the maxTimeMS option.
|
|
484
463
|
// It is expected to still make progress, and continue on the next try.
|
|
485
464
|
|
|
465
|
+
// TODO: Transactional?
|
|
486
466
|
await this.db.sync_rules.updateOne(
|
|
487
467
|
{
|
|
488
468
|
_id: this.group_id
|
|
@@ -496,33 +476,48 @@ export class MongoSyncBucketStorage
|
|
|
496
476
|
no_checkpoint_before: null
|
|
497
477
|
}
|
|
498
478
|
},
|
|
499
|
-
{ maxTimeMS: db.mongo.
|
|
479
|
+
{ maxTimeMS: db.mongo.MONGO_OPERATION_TIMEOUT_MS }
|
|
500
480
|
);
|
|
501
481
|
await this.db.bucket_data.deleteMany(
|
|
502
482
|
{
|
|
503
483
|
_id: idPrefixFilter<BucketDataKey>({ g: this.group_id }, ['b', 'o'])
|
|
504
484
|
},
|
|
505
|
-
{ maxTimeMS: db.mongo.
|
|
485
|
+
{ maxTimeMS: db.mongo.MONGO_OPERATION_TIMEOUT_MS }
|
|
506
486
|
);
|
|
507
487
|
await this.db.bucket_parameters.deleteMany(
|
|
508
488
|
{
|
|
509
489
|
key: idPrefixFilter<SourceKey>({ g: this.group_id }, ['t', 'k'])
|
|
510
490
|
},
|
|
511
|
-
{ maxTimeMS: db.mongo.
|
|
491
|
+
{ maxTimeMS: db.mongo.MONGO_OPERATION_TIMEOUT_MS }
|
|
512
492
|
);
|
|
513
493
|
|
|
514
494
|
await this.db.current_data.deleteMany(
|
|
515
495
|
{
|
|
516
496
|
_id: idPrefixFilter<SourceKey>({ g: this.group_id }, ['t', 'k'])
|
|
517
497
|
},
|
|
518
|
-
{ maxTimeMS: db.mongo.
|
|
498
|
+
{ maxTimeMS: db.mongo.MONGO_OPERATION_TIMEOUT_MS }
|
|
519
499
|
);
|
|
520
500
|
|
|
521
501
|
await this.db.source_tables.deleteMany(
|
|
522
502
|
{
|
|
523
503
|
group_id: this.group_id
|
|
524
504
|
},
|
|
525
|
-
{ maxTimeMS: db.mongo.
|
|
505
|
+
{ maxTimeMS: db.mongo.MONGO_OPERATION_TIMEOUT_MS }
|
|
506
|
+
);
|
|
507
|
+
}
|
|
508
|
+
|
|
509
|
+
async setSnapshotDone(lsn: string): Promise<void> {
|
|
510
|
+
await this.db.sync_rules.updateOne(
|
|
511
|
+
{
|
|
512
|
+
_id: this.group_id
|
|
513
|
+
},
|
|
514
|
+
{
|
|
515
|
+
$set: {
|
|
516
|
+
snapshot_done: true,
|
|
517
|
+
persisted_lsn: lsn,
|
|
518
|
+
last_checkpoint_ts: new Date()
|
|
519
|
+
}
|
|
520
|
+
}
|
|
526
521
|
);
|
|
527
522
|
}
|
|
528
523
|
|
|
@@ -17,11 +17,19 @@ export type MongoCheckpointAPIOptions = {
|
|
|
17
17
|
|
|
18
18
|
export class MongoWriteCheckpointAPI implements WriteCheckpointAPI {
|
|
19
19
|
readonly db: PowerSyncMongo;
|
|
20
|
-
|
|
20
|
+
private _mode: WriteCheckpointMode;
|
|
21
21
|
|
|
22
22
|
constructor(options: MongoCheckpointAPIOptions) {
|
|
23
23
|
this.db = options.db;
|
|
24
|
-
this.
|
|
24
|
+
this._mode = options.mode;
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
get writeCheckpointMode() {
|
|
28
|
+
return this._mode;
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
setWriteCheckpointMode(mode: WriteCheckpointMode): void {
|
|
32
|
+
this._mode = mode;
|
|
25
33
|
}
|
|
26
34
|
|
|
27
35
|
async batchCreateCustomWriteCheckpoints(checkpoints: CustomWriteCheckpointOptions[]): Promise<void> {
|
|
@@ -29,12 +37,11 @@ export class MongoWriteCheckpointAPI implements WriteCheckpointAPI {
|
|
|
29
37
|
}
|
|
30
38
|
|
|
31
39
|
async createCustomWriteCheckpoint(options: CustomWriteCheckpointOptions): Promise<bigint> {
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
40
|
+
/**
|
|
41
|
+
* Allow creating custom checkpoints even if the current mode is not `custom`.
|
|
42
|
+
* There might be a state where the next sync rules rely on replicating custom
|
|
43
|
+
* write checkpoints, but the current active sync rules uses managed checkpoints.
|
|
44
|
+
*/
|
|
38
45
|
const { checkpoint, user_id, sync_rules_id } = options;
|
|
39
46
|
const doc = await this.db.custom_write_checkpoints.findOneAndUpdate(
|
|
40
47
|
{
|
|
@@ -52,9 +59,9 @@ export class MongoWriteCheckpointAPI implements WriteCheckpointAPI {
|
|
|
52
59
|
}
|
|
53
60
|
|
|
54
61
|
async createManagedWriteCheckpoint(checkpoint: ManagedWriteCheckpointOptions): Promise<bigint> {
|
|
55
|
-
if (this.
|
|
62
|
+
if (this.writeCheckpointMode !== WriteCheckpointMode.MANAGED) {
|
|
56
63
|
throw new framework.errors.ValidationError(
|
|
57
|
-
`
|
|
64
|
+
`Attempting to create a managed Write Checkpoint when the current Write Checkpoint mode is set to "${this.writeCheckpointMode}"`
|
|
58
65
|
);
|
|
59
66
|
}
|
|
60
67
|
|
|
@@ -77,7 +84,7 @@ export class MongoWriteCheckpointAPI implements WriteCheckpointAPI {
|
|
|
77
84
|
}
|
|
78
85
|
|
|
79
86
|
async lastWriteCheckpoint(filters: LastWriteCheckpointFilters): Promise<bigint | null> {
|
|
80
|
-
switch (this.
|
|
87
|
+
switch (this.writeCheckpointMode) {
|
|
81
88
|
case WriteCheckpointMode.CUSTOM:
|
|
82
89
|
if (false == 'sync_rules_id' in filters) {
|
|
83
90
|
throw new framework.errors.ValidationError(`Sync rules ID is required for custom Write Checkpoint filtering`);
|
package/src/storage/mongo/db.ts
CHANGED
|
@@ -62,9 +62,6 @@ export class PowerSyncMongo {
|
|
|
62
62
|
this.locks = this.db.collection('locks');
|
|
63
63
|
}
|
|
64
64
|
|
|
65
|
-
/**
|
|
66
|
-
* Clear all collections.
|
|
67
|
-
*/
|
|
68
65
|
async clear() {
|
|
69
66
|
await this.current_data.deleteMany({});
|
|
70
67
|
await this.bucket_data.deleteMany({});
|
|
@@ -76,13 +73,4 @@ export class PowerSyncMongo {
|
|
|
76
73
|
await this.instance.deleteOne({});
|
|
77
74
|
await this.locks.deleteMany({});
|
|
78
75
|
}
|
|
79
|
-
|
|
80
|
-
/**
|
|
81
|
-
* Drop the entire database.
|
|
82
|
-
*
|
|
83
|
-
* Primarily for tests.
|
|
84
|
-
*/
|
|
85
|
-
async drop() {
|
|
86
|
-
await this.db.dropDatabase();
|
|
87
|
-
}
|
|
88
76
|
}
|
|
@@ -55,6 +55,10 @@ export type ManagedWriteCheckpointOptions = ManagedWriteCheckpointFilters;
|
|
|
55
55
|
export type LastWriteCheckpointFilters = CustomWriteCheckpointFilters | ManagedWriteCheckpointFilters;
|
|
56
56
|
|
|
57
57
|
export interface WriteCheckpointAPI {
|
|
58
|
+
readonly writeCheckpointMode: WriteCheckpointMode;
|
|
59
|
+
|
|
60
|
+
setWriteCheckpointMode(mode: WriteCheckpointMode): void;
|
|
61
|
+
|
|
58
62
|
batchCreateCustomWriteCheckpoints(checkpoints: CustomWriteCheckpointOptions[]): Promise<void>;
|
|
59
63
|
|
|
60
64
|
createCustomWriteCheckpoint(checkpoint: CustomWriteCheckpointOptions): Promise<bigint>;
|
|
@@ -122,7 +122,8 @@ export class CompoundConfigCollector {
|
|
|
122
122
|
},
|
|
123
123
|
// TODO maybe move this out of the connection or something
|
|
124
124
|
// slot_name_prefix: connections[0]?.slot_name_prefix ?? 'powersync_'
|
|
125
|
-
slot_name_prefix: 'powersync_'
|
|
125
|
+
slot_name_prefix: 'powersync_',
|
|
126
|
+
parameters: baseConfig.parameters ?? {}
|
|
126
127
|
};
|
|
127
128
|
|
|
128
129
|
return config;
|
package/src/util/config/types.ts
CHANGED
|
@@ -1438,26 +1438,4 @@ bucket_definitions:
|
|
|
1438
1438
|
expect(errorCaught).true;
|
|
1439
1439
|
expect(isDisposed).true;
|
|
1440
1440
|
});
|
|
1441
|
-
|
|
1442
|
-
test('empty storage metrics', async () => {
|
|
1443
|
-
const f = await factory({ dropAll: true });
|
|
1444
|
-
|
|
1445
|
-
const metrics = await f.getStorageMetrics();
|
|
1446
|
-
expect(metrics).toEqual({
|
|
1447
|
-
operations_size_bytes: 0,
|
|
1448
|
-
parameters_size_bytes: 0,
|
|
1449
|
-
replication_size_bytes: 0
|
|
1450
|
-
});
|
|
1451
|
-
|
|
1452
|
-
const r = await f.configureSyncRules('bucket_definitions: {}');
|
|
1453
|
-
const storage = f.getInstance(r.persisted_sync_rules!);
|
|
1454
|
-
await storage.autoActivate();
|
|
1455
|
-
|
|
1456
|
-
const metrics2 = await f.getStorageMetrics();
|
|
1457
|
-
expect(metrics2).toEqual({
|
|
1458
|
-
operations_size_bytes: 0,
|
|
1459
|
-
parameters_size_bytes: 0,
|
|
1460
|
-
replication_size_bytes: 0
|
|
1461
|
-
});
|
|
1462
|
-
});
|
|
1463
1441
|
}
|
package/test/src/sync.test.ts
CHANGED
|
@@ -5,7 +5,15 @@ import { JSONBig } from '@powersync/service-jsonbig';
|
|
|
5
5
|
import { RequestParameters } from '@powersync/service-sync-rules';
|
|
6
6
|
import * as timers from 'timers/promises';
|
|
7
7
|
import { describe, expect, test } from 'vitest';
|
|
8
|
-
import {
|
|
8
|
+
import {
|
|
9
|
+
BATCH_OPTIONS,
|
|
10
|
+
makeTestTable,
|
|
11
|
+
MONGO_STORAGE_FACTORY,
|
|
12
|
+
PARSE_OPTIONS,
|
|
13
|
+
StorageFactory,
|
|
14
|
+
ZERO_LSN
|
|
15
|
+
} from './util.js';
|
|
16
|
+
import { ParseSyncRulesOptions, StartBatchOptions } from '@/storage/BucketStorage.js';
|
|
9
17
|
|
|
10
18
|
describe('sync - mongodb', function () {
|
|
11
19
|
defineTests(MONGO_STORAGE_FACTORY);
|
|
@@ -30,7 +38,8 @@ function defineTests(factory: StorageFactory) {
|
|
|
30
38
|
content: BASIC_SYNC_RULES
|
|
31
39
|
});
|
|
32
40
|
|
|
33
|
-
const storage = f.getInstance(syncRules);
|
|
41
|
+
const storage = await f.getInstance(syncRules);
|
|
42
|
+
await storage.setSnapshotDone(ZERO_LSN);
|
|
34
43
|
await storage.autoActivate();
|
|
35
44
|
|
|
36
45
|
const result = await storage.startBatch(BATCH_OPTIONS, async (batch) => {
|
|
@@ -82,6 +91,7 @@ function defineTests(factory: StorageFactory) {
|
|
|
82
91
|
});
|
|
83
92
|
|
|
84
93
|
const storage = await f.getInstance(syncRules);
|
|
94
|
+
await storage.setSnapshotDone(ZERO_LSN);
|
|
85
95
|
await storage.autoActivate();
|
|
86
96
|
|
|
87
97
|
const result = await storage.startBatch(BATCH_OPTIONS, async (batch) => {
|
|
@@ -126,6 +136,7 @@ function defineTests(factory: StorageFactory) {
|
|
|
126
136
|
});
|
|
127
137
|
|
|
128
138
|
const storage = await f.getInstance(syncRules);
|
|
139
|
+
await storage.setSnapshotDone(ZERO_LSN);
|
|
129
140
|
await storage.autoActivate();
|
|
130
141
|
|
|
131
142
|
const stream = streamResponse({
|
|
@@ -153,6 +164,7 @@ function defineTests(factory: StorageFactory) {
|
|
|
153
164
|
});
|
|
154
165
|
|
|
155
166
|
const storage = await f.getInstance(syncRules);
|
|
167
|
+
await storage.setSnapshotDone(ZERO_LSN);
|
|
156
168
|
await storage.autoActivate();
|
|
157
169
|
|
|
158
170
|
const stream = streamResponse({
|
|
@@ -214,6 +226,7 @@ function defineTests(factory: StorageFactory) {
|
|
|
214
226
|
});
|
|
215
227
|
|
|
216
228
|
const storage = await f.getInstance(syncRules);
|
|
229
|
+
await storage.setSnapshotDone(ZERO_LSN);
|
|
217
230
|
await storage.autoActivate();
|
|
218
231
|
|
|
219
232
|
const exp = Date.now() / 1000 + 0.1;
|
|
@@ -252,6 +265,7 @@ function defineTests(factory: StorageFactory) {
|
|
|
252
265
|
});
|
|
253
266
|
|
|
254
267
|
const storage = await f.getInstance(syncRules);
|
|
268
|
+
await storage.setSnapshotDone(ZERO_LSN);
|
|
255
269
|
await storage.autoActivate();
|
|
256
270
|
|
|
257
271
|
await storage.startBatch(BATCH_OPTIONS, async (batch) => {
|
package/test/src/util.ts
CHANGED
|
@@ -24,22 +24,11 @@ await Metrics.initialise({
|
|
|
24
24
|
});
|
|
25
25
|
Metrics.getInstance().resetCounters();
|
|
26
26
|
|
|
27
|
-
export
|
|
28
|
-
/**
|
|
29
|
-
* By default, collections are only cleared/
|
|
30
|
-
* Setting this to true will drop the collections completely.
|
|
31
|
-
*/
|
|
32
|
-
dropAll?: boolean;
|
|
33
|
-
}
|
|
34
|
-
export type StorageFactory = (options?: StorageOptions) => Promise<BucketStorageFactory>;
|
|
27
|
+
export type StorageFactory = () => Promise<BucketStorageFactory>;
|
|
35
28
|
|
|
36
|
-
export const MONGO_STORAGE_FACTORY: StorageFactory = async (
|
|
29
|
+
export const MONGO_STORAGE_FACTORY: StorageFactory = async () => {
|
|
37
30
|
const db = await connectMongo();
|
|
38
|
-
|
|
39
|
-
await db.drop();
|
|
40
|
-
} else {
|
|
41
|
-
await db.clear();
|
|
42
|
-
}
|
|
31
|
+
await db.clear();
|
|
43
32
|
return new MongoBucketStorage(db, { slot_name_prefix: 'test_' });
|
|
44
33
|
};
|
|
45
34
|
|