@powersync/service-core 1.23.3 → 1.25.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +45 -0
- package/dist/api/RouteAPI.d.ts +17 -3
- package/dist/entry/commands/compact-action.js +4 -0
- package/dist/entry/commands/compact-action.js.map +1 -1
- package/dist/replication/AbstractReplicator.d.ts +9 -4
- package/dist/replication/AbstractReplicator.js +35 -11
- package/dist/replication/AbstractReplicator.js.map +1 -1
- package/dist/routes/configure-fastify.d.ts +31 -0
- package/dist/routes/endpoints/checkpointing.d.ts +62 -0
- package/dist/routes/endpoints/checkpointing.js +63 -3
- package/dist/routes/endpoints/checkpointing.js.map +1 -1
- package/dist/storage/BucketStorageBatch.d.ts +4 -1
- package/dist/storage/BucketStorageBatch.js.map +1 -1
- package/dist/storage/BucketStorageFactory.d.ts +2 -2
- package/dist/storage/CheckpointChecksumInvalidatedError.d.ts +12 -0
- package/dist/storage/CheckpointChecksumInvalidatedError.js +17 -0
- package/dist/storage/CheckpointChecksumInvalidatedError.js.map +1 -0
- package/dist/storage/SourceEntity.d.ts +7 -2
- package/dist/storage/SourceTable.d.ts +24 -1
- package/dist/storage/SourceTable.js +34 -6
- package/dist/storage/SourceTable.js.map +1 -1
- package/dist/storage/SourceTableReconciler.d.ts +83 -0
- package/dist/storage/SourceTableReconciler.js +95 -0
- package/dist/storage/SourceTableReconciler.js.map +1 -0
- package/dist/storage/SyncRulesBucketStorage.d.ts +32 -1
- package/dist/storage/SyncRulesBucketStorage.js +3 -0
- package/dist/storage/SyncRulesBucketStorage.js.map +1 -1
- package/dist/storage/WriteCheckpointAPI.d.ts +60 -3
- package/dist/storage/WriteCheckpointAPI.js +33 -0
- package/dist/storage/WriteCheckpointAPI.js.map +1 -1
- package/dist/storage/implementation/BucketDefinitionMapping.d.ts +4 -4
- package/dist/storage/implementation/BucketDefinitionMapping.js.map +1 -1
- package/dist/storage/storage-index.d.ts +2 -0
- package/dist/storage/storage-index.js +2 -0
- package/dist/storage/storage-index.js.map +1 -1
- package/dist/sync/BucketChecksumState.d.ts +7 -0
- package/dist/sync/BucketChecksumState.js +33 -13
- package/dist/sync/BucketChecksumState.js.map +1 -1
- package/dist/sync/sync.js +34 -6
- package/dist/sync/sync.js.map +1 -1
- package/dist/sync/util.d.ts +5 -0
- package/dist/sync/util.js +9 -0
- package/dist/sync/util.js.map +1 -1
- package/dist/util/checkpointing.d.ts +1 -0
- package/dist/util/checkpointing.js +1 -1
- package/dist/util/checkpointing.js.map +1 -1
- package/dist/util/config/compound-config-collector.js +9 -1
- package/dist/util/config/compound-config-collector.js.map +1 -1
- package/dist/util/config/defaults.d.ts +1 -0
- package/dist/util/config/defaults.js +1 -0
- package/dist/util/config/defaults.js.map +1 -1
- package/dist/util/config/types.d.ts +1 -0
- package/dist/util/utils.d.ts +1 -1
- package/dist/util/utils.js +1 -1
- package/dist/util/utils.js.map +1 -1
- package/dist/util/write-checkpoint-batcher.d.ts +1 -1
- package/dist/util/write-checkpoint-batcher.js +18 -8
- package/dist/util/write-checkpoint-batcher.js.map +1 -1
- package/package.json +5 -5
- package/src/api/RouteAPI.ts +18 -3
- package/src/entry/commands/compact-action.ts +6 -0
- package/src/replication/AbstractReplicator.ts +42 -12
- package/src/routes/endpoints/checkpointing.ts +77 -3
- package/src/storage/BucketStorageBatch.ts +4 -1
- package/src/storage/BucketStorageFactory.ts +2 -2
- package/src/storage/CheckpointChecksumInvalidatedError.ts +17 -0
- package/src/storage/SourceEntity.ts +6 -2
- package/src/storage/SourceTable.ts +51 -7
- package/src/storage/SourceTableReconciler.ts +188 -0
- package/src/storage/SyncRulesBucketStorage.ts +38 -1
- package/src/storage/WriteCheckpointAPI.ts +108 -3
- package/src/storage/implementation/BucketDefinitionMapping.ts +4 -4
- package/src/storage/storage-index.ts +2 -0
- package/src/sync/BucketChecksumState.ts +38 -13
- package/src/sync/sync.ts +33 -7
- package/src/sync/util.ts +12 -0
- package/src/util/checkpointing.ts +2 -1
- package/src/util/config/compound-config-collector.ts +12 -0
- package/src/util/config/defaults.ts +1 -0
- package/src/util/config/types.ts +1 -0
- package/src/util/utils.ts +1 -1
- package/src/util/write-checkpoint-batcher.ts +30 -10
- package/test/src/AbstractReplicator.test.ts +147 -0
- package/test/src/config.test.ts +60 -0
- package/test/src/routes/checkpointing.test.ts +54 -0
- package/test/src/source-table-reconciler.test.ts +244 -0
- package/test/src/sync/BucketChecksumState.test.ts +57 -11
- package/test/src/sync/util.test.ts +13 -1
- package/test/src/util/checkpointing.test.ts +99 -20
- package/tsconfig.tsbuildinfo +1 -1
package/dist/util/utils.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"utils.js","sourceRoot":"","sources":["../../src/util/utils.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,IAAI,MAAM,MAAM,CAAC;AAC7B,OAAO,MAAM,MAAM,QAAQ,CAAC;AAC5B,OAAO,KAAK,IAAI,MAAM,MAAM,CAAC;AAK7B,OAAO,EAAE,qBAAqB,EAAE,MAAM,mCAAmC,CAAC;AA2B1E,MAAM,CAAC,MAAM,YAAY,GAAG,sCAAsC,CAAC;
|
|
1
|
+
{"version":3,"file":"utils.js","sourceRoot":"","sources":["../../src/util/utils.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,IAAI,MAAM,MAAM,CAAC;AAC7B,OAAO,MAAM,MAAM,QAAQ,CAAC;AAC5B,OAAO,KAAK,IAAI,MAAM,MAAM,CAAC;AAK7B,OAAO,EAAE,qBAAqB,EAAE,MAAM,mCAAmC,CAAC;AA2B1E,MAAM,CAAC,MAAM,YAAY,GAAG,IAAI,CAAC,KAAK,CAAC,sCAAsC,CAAC,CAAC;AAE/E,MAAM,UAAU,gBAAgB,CAAC,UAAkB;IACjD,OAAO,IAAI,UAAU,CAAC,OAAO,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC,OAAO,CAAC,KAAK,EAAE,KAAK,CAAC,GAAG,CAAC;AACrE,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,aAAa,CAAC,GAA8B;IAC1D,OAAO,GAAG,gBAAgB,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,gBAAgB,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC;AACzE,CAAC;AAED,MAAM,UAAU,QAAQ,CAAC,IAAY,EAAE,EAAU,EAAE,IAAY;IAC7D,MAAM,IAAI,GAAG,MAAM,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC;IACzC,IAAI,CAAC,MAAM,CAAC,OAAO,IAAI,IAAI,EAAE,IAAI,IAAI,EAAE,CAAC,CAAC;IACzC,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC;IAC7B,OAAO,MAAM,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;AAChC,CAAC;AAED,MAAM,UAAU,UAAU,CAAC,SAAiB;IAC1C,MAAM,IAAI,GAAG,MAAM,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC;IACzC,IAAI,CAAC,MAAM,CAAC,UAAU,SAAS,EAAE,CAAC,CAAC;IACnC,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC;IAC7B,OAAO,MAAM,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;AAChC,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,sBAAsB,CAAC,EAAgB;IACrD,6EAA6E;IAC7E,6CAA6C;IAC7C,IAAI,OAAO,EAAE,IAAI,QAAQ,EAAE,CAAC;QAC1B,MAAM,IAAI,qBAAqB,CAAC,yBAAyB,EAAE,KAAK,OAAO,EAAE,GAAG,CAAC,CAAC;IAChF,CAAC;IACD,OAAO,EAAE,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC;AACzB,CAAC;AAED,MAAM,UAAU,aAAa,CAAC,QAAqB,EAAE,OAAoB;IACvE,mBAAmB;IACnB,MAAM,cAAc,GAAG,IAAI,GAAG,EAA0B,CAAC;IAEzD,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAS,QAAQ,CAAC,IAAI,EAAE,CAAC,CAAC;IAElD,KAAK,IAAI,QAAQ,IAAI,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC;QACtC,MAAM,CAAC,GAAG,QAAQ,CAAC,GAAG,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;QACxC,IAAI,CAAC,IAAI,IAAI,EAAE,CAAC;YACd,QAAQ;YACR,cAAc,CAAC,GAAG,CAAC,QAAQ,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;QAChD,CAAC;aAAM,CAAC;YACN,QAAQ,CAAC,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;YACjC,IAAI,QAAQ,CAAC,QAAQ,IAAI,CAAC,CAAC,QAAQ,IAAI,QAAQ,CAAC,KAAK,IAAI,CAAC,CAAC,KAAK,EAAE,CAAC;gBACjE,UAAU;gBACV,cAAc,CAAC,GAAG,CAAC,QAAQ,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;YAChD,CAAC;iBAAM,CAAC;gBACN,YAAY;YACd,CAAC;QACH,CAAC;IACH,CAAC;IAED,OAAO;QACL,cAAc,EAAE,CAAC,GAAG,cAAc,CAAC,MAAM,EAAE,CAAC;QAC5C,cAAc,EAAE,CAAC,GAAG,QAAQ,CAAC;KAC9B,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,YAAY,CAAC,CAAS,EAAE,CAAS;IAC/C,OAAO,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,UAAU,CAAC;AAC9B,CAAC;AAED,MAAM,UAAU,iBAAiB,CAAC,CAAmC;IACnE,OAAO,iBAAiB,IAAI,CAAC,CAAC;AAChC,CAAC;AAED,MAAM,UAAU,kBAAkB,CAAC,CAAiB,EAAE,CAA0C;IAC9F,MAAM,QAAQ,GAAG,mBAAmB,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC;IACrD,IAAI,iBAAiB,CAAC,QAAQ,CAAC,EAAE,CAAC;QAChC,oCAAoC;QACpC,MAAM,IAAI,qBAAqB,CAAC,wBAAwB,CAAC,CAAC;IAC5D,CAAC;IACD,OAAO,QAAQ,CAAC;AAClB,CAAC;AAED,MAAM,UAAU,mBAAmB,CACjC,MAAc,EACd,CAA0C,EAC1C,CAA0C;IAE1C,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,IAAI,IAAI,EAAE,CAAC;QAC3B,IAAI,CAAC,iBAAiB,CAAC,CAAC,CAAC,EAAE,CAAC;YAC1B,aAAa;YACb,OAAO,CAAC,CAAC;QACX,CAAC;QACD,QAAQ;QACR,IAAI,CAAC,iBAAiB,CAAC,CAAC,CAAC,EAAE,CAAC;YAC1B,OAAO;gBACL,MAAM;gBACN,QAAQ,EAAE,YAAY,CAAC,CAAC,CAAC,QAAQ,EAAE,CAAC,CAAC,eAAe,CAAC;gBACrD,KAAK,EAAE,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,YAAY;aAChC,CAAC;QACJ,CAAC;aAAM,CAAC;YACN,OAAO;gBACL,MAAM;gBACN,eAAe,EAAE,YAAY,CAAC,CAAC,CAAC,eAAe,EAAE,CAAC,CAAC,eAAe,CAAC;gBACnE,YAAY,EAAE,CAAC,CAAC,YAAY,GAAG,CAAC,CAAC,YAAY;aAC9C,CAAC;QACJ,CAAC;IACH,CAAC;SAAM,IAAI,CAAC,IAAI,IAAI,EAAE,CAAC;QACrB,OAAO,CAAC,CAAC;IACX,CAAC;SAAM,IAAI,CAAC,IAAI,IAAI,EAAE,CAAC;QACrB,OAAO,CAAC,CAAC;IACX,CAAC;SAAM,CAAC;QACN,+DAA+D;QAC/D,OAAO;YACL,MAAM;YACN,eAAe,EAAE,CAAC;YAClB,YAAY,EAAE,CAAC;SAChB,CAAC;IACJ,CAAC;AACH,CAAC;AAED,SAAS,qBAAqB,CAC5B,KAAoC,EACpC,OAAmC;IAEnC,IAAI,MAAM,GAAwB,EAAE,CAAC;IACrC,KAAK,IAAI,MAAM,IAAI,OAAO,EAAE,CAAC;QAC3B,MAAM,IAAI,GAAG,MAAM,CAAC,IAAI,CAAC;QACzB,MAAM,CAAC,IAAI,CAAC,GAAG,KAAK,CAAC,IAAI,CAAC,CAAC;IAC7B,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,MAAM,UAAU,0BAA0B,CACxC,KAAoC,EACpC,OAAmC;IAEnC,IAAI,OAAO,CAAC,MAAM,IAAI,CAAC,EAAE,CAAC;QACxB,gDAAgD;QAChD,OAAO,IAAI,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,EAAE,CAAC,CAAC;IAClC,CAAC;IACD,MAAM,WAAW,GAAG,qBAAqB,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC;IAE1D,OAAO,cAAc,CAAC,WAAW,CAAC,CAAC;AACrC,CAAC;AAED,MAAM,UAAU,cAAc,CAAC,GAAyB;IACtD,+EAA+E;IAC/E,4EAA4E;IAC5E,MAAM,IAAI,GAAG,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC;IACjC,MAAM,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;IAChC,OAAO,IAAI,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,EAAE,YAAY,EAAE,MAAM,CAAC,CAAC,CAAC;AAC5D,CAAC;AAED,MAAM,UAAU,gBAAgB,CAAI,GAAqC;IACvE,KAAK,IAAI,GAAG,IAAI,GAAG,EAAE,CAAC;QACpB,IAAI,OAAO,GAAG,CAAC,GAAG,CAAC,IAAI,WAAW,EAAE,CAAC;YACnC,OAAO,IAAI,CAAC;QACd,CAAC;IACH,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,aAAa,CAC3B,SAAkB,EAClB,GAAqC;IAErC,IAAI,CAAC,SAAS,EAAE,CAAC;QACf,gDAAgD;QAChD,OAAO,IAAI,CAAC;IACd,CAAC;IACD,OAAO,CAAC,gBAAgB,CAAC,GAAG,CAAC,CAAC;AAChC,CAAC;AAED;;;;;;;;;;;GAWG;AACH,MAAM,UAAU,YAAY,CAAC,UAAwB;IACnD,IAAI,QAAQ,GAAG,IAAI,GAAG,EAAsB,CAAC;IAC7C,IAAI,aAAa,GAAG,CAAC,CAAC;IAEtB,KAAK,IAAI,EAAE,IAAI,UAAU,EAAE,CAAC;QAC1B,MAAM,GAAG,GAAG,MAAM,CAAC,EAAE,CAAC,CAAC;QACvB,IAAI,EAAE,CAAC,EAAE,IAAI,KAAK,EAAE,CAAC;YACnB,MAAM,QAAQ,GAAG,QAAQ,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;YACnC,IAAI,QAAQ,EAAE,CAAC;gBACb,aAAa,GAAG,YAAY,CAAC,aAAa,EAAE,QAAQ,CAAC,QAAkB,CAAC,CAAC;YAC3E,CAAC;YACD,QAAQ,CAAC,GAAG,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC;QACxB,CAAC;aAAM,IAAI,EAAE,CAAC,EAAE,IAAI,QAAQ,EAAE,CAAC;YAC7B,MAAM,QAAQ,GAAG,QAAQ,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;YACnC,IAAI,QAAQ,EAAE,CAAC;gBACb,aAAa,GAAG,YAAY,CAAC,aAAa,EAAE,QAAQ,CAAC,QAAkB,CAAC,CAAC;YAC3E,CAAC;YACD,QAAQ,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;YACrB,aAAa,GAAG,YAAY,CAAC,aAAa,EAAE,EAAE,CAAC,QAAkB,CAAC,CAAC;QACrE,CAAC;aAAM,IAAI,EAAE,CAAC,EAAE,IAAI,OAAO,EAAE,CAAC;YAC5B,QAAQ,CAAC,KAAK,EAAE,CAAC;YACjB,aAAa,GAAG,EAAE,CAAC,QAAkB,CAAC;QACxC,CAAC;aAAM,IAAI,EAAE,CAAC,EAAE,IAAI,MAAM,EAAE,CAAC;YAC3B,aAAa,GAAG,YAAY,CAAC,aAAa,EAAE,EAAE,CAAC,QAAkB,CAAC,CAAC;QACrE,CAAC;aAAM,CAAC;YACN,MAAM,IAAI,KAAK,CAAC,qBAAqB,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC;QAChD,CAAC;IACH,CAAC;IAED,MAAM,IAAI,GAAG,CAAC,GAAG,QAAQ,CAAC,MAAM,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE;QAChD,OAAO,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC;IACnD,CAAC,CAAC,CAAC;IAEH,IAAI,UAAU,GAAiB;QAC7B,wDAAwD;QACxD,EAAE,KAAK,EAAE,GAAG,EAAE,EAAE,EAAE,OAAO,EAAE,QAAQ,EAAE,aAAa,EAAE;QACpD,GAAG,IAAI;KACR,CAAC;IAEF,OAAO,UAAU,CAAC;AACpB,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,OAAO,CAAC,CAAS;IAC/B,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;IACf,OAAO,CAAC,CAAC;AACX,CAAC;AAED,SAAS,MAAM,CAAC,KAAiB;IAC/B,OAAO,GAAG,KAAK,CAAC,WAAW,IAAI,KAAK,CAAC,SAAS,IAAI,KAAK,CAAC,MAAM,EAAE,CAAC;AACnE,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,eAAe,CAAC,MAAiD;IAC/E,IAAI,MAAM,IAAI,IAAI,EAAE,CAAC;QACnB,OAAO,EAAE,CAAC;IACZ,CAAC;IACD,IAAI,IAAI,GAAG,CAAC,CAAC;IACb,KAAK,IAAI,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC;QAChD,IAAI,IAAI,EAAE,GAAG,GAAG,CAAC,MAAM,CAAC;QACxB,+CAA+C;QAC/C,IAAI,KAAK,IAAI,IAAI,EAAE,CAAC;YAClB,IAAI,IAAI,CAAC,CAAC;QACZ,CAAC;aAAM,IAAI,OAAO,KAAK,IAAI,QAAQ,EAAE,CAAC;YACpC,IAAI,IAAI,CAAC,CAAC;QACZ,CAAC;aAAM,IAAI,OAAO,KAAK,IAAI,QAAQ,EAAE,CAAC;YACpC,IAAI,IAAI,CAAC,CAAC;QACZ,CAAC;aAAM,IAAI,OAAO,KAAK,IAAI,QAAQ,EAAE,CAAC;YACpC,IAAI,IAAI,KAAK,CAAC,MAAM,CAAC;QACvB,CAAC;aAAM,IAAI,KAAK,YAAY,UAAU,EAAE,CAAC;YACvC,IAAI,IAAI,KAAK,CAAC,UAAU,CAAC;QAC3B,CAAC;IACH,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC"}
|
|
@@ -11,7 +11,7 @@ export declare class WriteCheckpointBatcher {
|
|
|
11
11
|
private inFlight;
|
|
12
12
|
private scheduledPump;
|
|
13
13
|
constructor(getAPI: () => RouteAPI, getStorage: () => BucketStorageFactory);
|
|
14
|
-
enqueue(userId: string): Promise<CreateWriteCheckpointResult>;
|
|
14
|
+
enqueue(userId: string, checkpointRequestId?: bigint): Promise<CreateWriteCheckpointResult>;
|
|
15
15
|
private schedulePump;
|
|
16
16
|
private pump;
|
|
17
17
|
private executeBatch;
|
|
@@ -15,9 +15,9 @@ export class WriteCheckpointBatcher {
|
|
|
15
15
|
this.getAPI = getAPI;
|
|
16
16
|
this.getStorage = getStorage;
|
|
17
17
|
}
|
|
18
|
-
enqueue(userId) {
|
|
18
|
+
enqueue(userId, checkpointRequestId) {
|
|
19
19
|
const resolvers = Promise.withResolvers();
|
|
20
|
-
this.pending.push({ userId, resolvers });
|
|
20
|
+
this.pending.push({ userId, checkpointRequestId, resolvers });
|
|
21
21
|
this.schedulePump();
|
|
22
22
|
return resolvers.promise;
|
|
23
23
|
}
|
|
@@ -54,9 +54,13 @@ export class WriteCheckpointBatcher {
|
|
|
54
54
|
if (!syncBucketStorage) {
|
|
55
55
|
throw new ServiceError(ErrorCode.PSYNC_S2302, `Cannot create Write Checkpoint since no sync config is active.`);
|
|
56
56
|
}
|
|
57
|
+
// The source adapter reads the head, hands it to this callback to persist the
|
|
58
|
+
// write-checkpoint mapping, then forces a source marker only when storage
|
|
59
|
+
// reports an advance. Keeping the marker inside the callback means it is
|
|
60
|
+
// causally ordered after the head within a single source session.
|
|
57
61
|
const { writeCheckpoints, currentCheckpoint } = await this.getAPI().createReplicationHead(async (currentCheckpoint) => {
|
|
58
|
-
const writeCheckpoints = await this.createBatchWriteCheckpoints(syncBucketStorage, batch, currentCheckpoint);
|
|
59
|
-
return { writeCheckpoints, currentCheckpoint };
|
|
62
|
+
const { writeCheckpoints, shouldAdvance } = await this.createBatchWriteCheckpoints(syncBucketStorage, batch, currentCheckpoint);
|
|
63
|
+
return { response: { writeCheckpoints, currentCheckpoint }, shouldAdvance };
|
|
60
64
|
});
|
|
61
65
|
const results = batch.map((request) => {
|
|
62
66
|
const writeCheckpoint = writeCheckpoints.get(request.userId);
|
|
@@ -81,10 +85,16 @@ export class WriteCheckpointBatcher {
|
|
|
81
85
|
}
|
|
82
86
|
}
|
|
83
87
|
async createBatchWriteCheckpoints(syncBucketStorage, batch, currentCheckpoint) {
|
|
84
|
-
return syncBucketStorage.createManagedWriteCheckpoints(batch.map((request) =>
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
+
return syncBucketStorage.createManagedWriteCheckpoints(batch.map((request) => {
|
|
89
|
+
const checkpoint = {
|
|
90
|
+
user_id: request.userId,
|
|
91
|
+
heads: { '1': currentCheckpoint }
|
|
92
|
+
};
|
|
93
|
+
if (request.checkpointRequestId != null) {
|
|
94
|
+
checkpoint.checkpoint_request_id = request.checkpointRequestId;
|
|
95
|
+
}
|
|
96
|
+
return checkpoint;
|
|
97
|
+
}));
|
|
88
98
|
}
|
|
89
99
|
}
|
|
90
100
|
//# sourceMappingURL=write-checkpoint-batcher.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"write-checkpoint-batcher.js","sourceRoot":"","sources":["../../src/util/write-checkpoint-batcher.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,qBAAqB,EAAE,YAAY,EAAE,MAAM,mCAAmC,CAAC;
|
|
1
|
+
{"version":3,"file":"write-checkpoint-batcher.js","sourceRoot":"","sources":["../../src/util/write-checkpoint-batcher.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,qBAAqB,EAAE,YAAY,EAAE,MAAM,mCAAmC,CAAC;AASnG,kFAAkF;AAClF,iFAAiF;AACjF,mEAAmE;AACnE,iHAAiH;AACjH,qHAAqH;AACrH,MAAM,sCAAsC,GAAG,CAAC,CAAC;AAajD,MAAM,OAAO,sBAAsB;IAMd;IACA;IANX,OAAO,GAA4B,EAAE,CAAC;IACtC,QAAQ,GAAG,CAAC,CAAC;IACb,aAAa,CAA6B;IAElD,YACmB,MAAsB,EACtB,UAAsC;QADtC,WAAM,GAAN,MAAM,CAAgB;QACtB,eAAU,GAAV,UAAU,CAA4B;IACtD,CAAC;IAEJ,OAAO,CAAC,MAAc,EAAE,mBAA4B;QAClD,MAAM,SAAS,GAAG,OAAO,CAAC,aAAa,EAA+B,CAAC;QACvE,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,mBAAmB,EAAE,SAAS,EAAE,CAAC,CAAC;QAC9D,IAAI,CAAC,YAAY,EAAE,CAAC;QACpB,OAAO,SAAS,CAAC,OAAO,CAAC;IAC3B,CAAC;IAEO,YAAY;QAClB,IAAI,IAAI,CAAC,aAAa,IAAI,IAAI,EAAE,CAAC;YAC/B,OAAO;QACT,CAAC;QAED,2EAA2E;QAC3E,+CAA+C;QAC/C,uGAAuG;QACvG,yGAAyG;QACzG,gHAAgH;QAChH,+GAA+G;QAC/G,wEAAwE;QACxE,IAAI,CAAC,aAAa,GAAG,UAAU,CAAC,GAAG,EAAE;YACnC,IAAI,CAAC,aAAa,GAAG,SAAS,CAAC;YAC/B,IAAI,CAAC,IAAI,EAAE,CAAC;QACd,CAAC,EAAE,CAAC,CAAC,CAAC;IACR,CAAC;IAEO,IAAI;QACV,OAAO,IAAI,CAAC,QAAQ,GAAG,sCAAsC,IAAI,IAAI,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACzF,MAAM,KAAK,GAAG,IAAI,CAAC,OAAO,CAAC;YAC3B,IAAI,CAAC,OAAO,GAAG,EAAE,CAAC;YAClB,IAAI,CAAC,QAAQ,EAAE,CAAC;YAEhB,KAAK,IAAI,CAAC,YAAY,CAAC,KAAK,CAAC,CAAC,OAAO,CAAC,GAAG,EAAE;gBACzC,IAAI,CAAC,QAAQ,EAAE,CAAC;gBAChB,IAAI,CAAC,YAAY,EAAE,CAAC;YACtB,CAAC,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,YAAY,CAAC,KAA8B;QACvD,IAAI,CAAC;YACH,MAAM,iBAAiB,GAAG,CAAC,MAAM,IAAI,CAAC,UAAU,EAAE,CAAC,mBAAmB,EAAE,CAAC,EAAE,OAAO,CAAC;YACnF,IAAI,CAAC,iBAAiB,EAAE,CAAC;gBACvB,MAAM,IAAI,YAAY,CAAC,SAAS,CAAC,WAAW,EAAE,gEAAgE,CAAC,CAAC;YAClH,CAAC;YAED,8EAA8E;YAC9E,0EAA0E;YAC1E,yEAAyE;YACzE,kEAAkE;YAClE,MAAM,EAAE,gBAAgB,EAAE,iBAAiB,EAAE,GAAG,MAAM,IAAI,CAAC,MAAM,EAAE,CAAC,qBAAqB,CACvF,KAAK,EAAE,iBAAiB,EAAE,EAAE;gBAC1B,MAAM,EAAE,gBAAgB,EAAE,aAAa,EAAE,GAAG,MAAM,IAAI,CAAC,2BAA2B,CAChF,iBAAiB,EACjB,KAAK,EACL,iBAAiB,CAClB,CAAC;gBACF,OAAO,EAAE,QAAQ,EAAE,EAAE,gBAAgB,EAAE,iBAAiB,EAAE,EAAE,aAAa,EAAE,CAAC;YAC9E,CAAC,CACF,CAAC;YAEF,MAAM,OAAO,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE;gBACpC,MAAM,eAAe,GAAG,gBAAgB,CAAC,GAAG,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;gBAC7D,IAAI,eAAe,IAAI,IAAI,EAAE,CAAC;oBAC5B,MAAM,IAAI,qBAAqB,CAC7B,iEAAiE,OAAO,CAAC,MAAM,EAAE,CAClF,CAAC;gBACJ,CAAC;gBAED,OAAO,EAAE,OAAO,EAAE,eAAe,EAAE,CAAC;YACtC,CAAC,CAAC,CAAC;YAEH,KAAK,MAAM,EAAE,OAAO,EAAE,eAAe,EAAE,IAAI,OAAO,EAAE,CAAC;gBACnD,OAAO,CAAC,SAAS,CAAC,OAAO,CAAC;oBACxB,eAAe,EAAE,MAAM,CAAC,eAAe,CAAC;oBACxC,eAAe,EAAE,iBAAiB;iBACnC,CAAC,CAAC;YACL,CAAC;QACH,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,2EAA2E;YAC3E,oDAAoD;YACpD,KAAK,MAAM,OAAO,IAAI,KAAK,EAAE,CAAC;gBAC5B,OAAO,CAAC,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;YAClC,CAAC;QACH,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,2BAA2B,CACvC,iBAAyC,EACzC,KAA8B,EAC9B,iBAAyB;QAEzB,OAAO,iBAAiB,CAAC,6BAA6B,CACpD,KAAK,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE;YACpB,MAAM,UAAU,GAAkC;gBAChD,OAAO,EAAE,OAAO,CAAC,MAAM;gBACvB,KAAK,EAAE,EAAE,GAAG,EAAE,iBAAiB,EAAE;aAClC,CAAC;YACF,IAAI,OAAO,CAAC,mBAAmB,IAAI,IAAI,EAAE,CAAC;gBACxC,UAAU,CAAC,qBAAqB,GAAG,OAAO,CAAC,mBAAmB,CAAC;YACjE,CAAC;YACD,OAAO,UAAU,CAAC;QACpB,CAAC,CAAC,CACH,CAAC;IACJ,CAAC;CACF"}
|
package/package.json
CHANGED
|
@@ -4,7 +4,7 @@
|
|
|
4
4
|
"publishConfig": {
|
|
5
5
|
"access": "public"
|
|
6
6
|
},
|
|
7
|
-
"version": "1.
|
|
7
|
+
"version": "1.25.0",
|
|
8
8
|
"license": "FSL-1.1-ALv2",
|
|
9
9
|
"type": "module",
|
|
10
10
|
"exports": "./dist/index.js",
|
|
@@ -32,11 +32,11 @@
|
|
|
32
32
|
"uuid": "^14.0.0",
|
|
33
33
|
"winston": "^3.13.0",
|
|
34
34
|
"yaml": "^2.8.3",
|
|
35
|
-
"@powersync/lib-services-framework": "0.
|
|
35
|
+
"@powersync/lib-services-framework": "0.10.0",
|
|
36
36
|
"@powersync/service-jsonbig": "0.17.13",
|
|
37
|
-
"@powersync/service-rsocket-router": "0.2.
|
|
38
|
-
"@powersync/service-sync-rules": "0.
|
|
39
|
-
"@powersync/service-types": "0.
|
|
37
|
+
"@powersync/service-rsocket-router": "0.2.25",
|
|
38
|
+
"@powersync/service-sync-rules": "0.40.0",
|
|
39
|
+
"@powersync/service-types": "0.17.0"
|
|
40
40
|
},
|
|
41
41
|
"devDependencies": {
|
|
42
42
|
"@types/async": "^3.2.24",
|
package/src/api/RouteAPI.ts
CHANGED
|
@@ -73,9 +73,15 @@ export interface RouteAPI {
|
|
|
73
73
|
*
|
|
74
74
|
* The position is provided to the callback so the caller can persist its
|
|
75
75
|
* write-checkpoint mapping before the source adapter forces any required
|
|
76
|
-
* source-side marker or keepalive.
|
|
76
|
+
* source-side marker or keepalive. The callback returns whether storage
|
|
77
|
+
* actually needs a source marker: when `shouldAdvance` is true the adapter
|
|
77
78
|
* must ensure that the replication stream will observe this position or a
|
|
78
|
-
* greater one, even when the source is otherwise idle.
|
|
79
|
+
* greater one, even when the source is otherwise idle. When it is false, the
|
|
80
|
+
* adapter may skip the source marker.
|
|
81
|
+
*
|
|
82
|
+
* Reading the head and forcing the marker happen within a single source
|
|
83
|
+
* session/connection where applicable, so the marker is causally ordered
|
|
84
|
+
* after the head that was handed to the callback.
|
|
79
85
|
*/
|
|
80
86
|
createReplicationHead<T>(callback: ReplicationHeadCallback<T>): Promise<T>;
|
|
81
87
|
|
|
@@ -111,4 +117,13 @@ export interface RouteAPI {
|
|
|
111
117
|
getParseSyncRulesOptions(): ParseSyncConfigOptions;
|
|
112
118
|
}
|
|
113
119
|
|
|
114
|
-
export
|
|
120
|
+
export interface ReplicationHeadResult<T> {
|
|
121
|
+
response: T;
|
|
122
|
+
/**
|
|
123
|
+
* True when storage needs the source to force a later observable position.
|
|
124
|
+
* False when storage can prove no marker is needed.
|
|
125
|
+
*/
|
|
126
|
+
shouldAdvance: boolean;
|
|
127
|
+
}
|
|
128
|
+
|
|
129
|
+
export type ReplicationHeadCallback<T> = (head: string) => Promise<ReplicationHeadResult<T>>;
|
|
@@ -21,6 +21,7 @@ const HEAP_LIMIT = v8.getHeapStatistics().heap_size_limit;
|
|
|
21
21
|
* Limit to 1024MB overall.
|
|
22
22
|
*/
|
|
23
23
|
const COMPACT_MEMORY_LIMIT_MB = Math.min(HEAP_LIMIT / 1024 / 1024 - 128, 1024);
|
|
24
|
+
const MINUTE_MS = 60 * 1000;
|
|
24
25
|
|
|
25
26
|
export function registerCompactAction(program: Command) {
|
|
26
27
|
const compactCommand = program
|
|
@@ -82,6 +83,9 @@ export function registerCompactAction(program: Command) {
|
|
|
82
83
|
// Start the storage engine in order to create the appropriate BucketStorage
|
|
83
84
|
await serviceContext.lifeCycleEngine.start();
|
|
84
85
|
const bucketStorage = serviceContext.storageEngine.activeBucketStorage;
|
|
86
|
+
const deleteCheckpointRequestsBefore = new Date(
|
|
87
|
+
Date.now() - config.api_parameters.checkpoint_request_retention_minutes * MINUTE_MS
|
|
88
|
+
);
|
|
85
89
|
|
|
86
90
|
const active = (await bucketStorage.getActiveSyncConfig())?.storage;
|
|
87
91
|
if (active == null) {
|
|
@@ -94,12 +98,14 @@ export function registerCompactAction(program: Command) {
|
|
|
94
98
|
memoryLimitMB: COMPACT_MEMORY_LIMIT_MB,
|
|
95
99
|
compactBuckets: buckets,
|
|
96
100
|
compactParameterData: compactParameters ?? false,
|
|
101
|
+
deleteCheckpointRequestsBefore,
|
|
97
102
|
signal: abortController.signal
|
|
98
103
|
});
|
|
99
104
|
} else {
|
|
100
105
|
await active.compact({
|
|
101
106
|
memoryLimitMB: COMPACT_MEMORY_LIMIT_MB,
|
|
102
107
|
compactParameterData: compactParameters ?? true,
|
|
108
|
+
deleteCheckpointRequestsBefore,
|
|
103
109
|
signal: abortController.signal
|
|
104
110
|
});
|
|
105
111
|
}
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { container, ErrorCode, logger } from '@powersync/lib-services-framework';
|
|
1
|
+
import { container, ErrorCode, logger, ReplicationAbortedError } from '@powersync/lib-services-framework';
|
|
2
2
|
import { ReplicationMetric } from '@powersync/service-types';
|
|
3
3
|
import { hrtime } from 'node:process';
|
|
4
4
|
import { setTimeout as sleep } from 'node:timers/promises';
|
|
@@ -11,8 +11,8 @@ import { AbstractReplicationJob } from './AbstractReplicationJob.js';
|
|
|
11
11
|
import { ErrorRateLimiter } from './ErrorRateLimiter.js';
|
|
12
12
|
import { ConnectionTestResult } from './ReplicationModule.js';
|
|
13
13
|
|
|
14
|
-
// 1 minute
|
|
15
|
-
const
|
|
14
|
+
// Default to 1 minute when no source-specific interval is configured.
|
|
15
|
+
const DEFAULT_HEARTBEAT_INTERVAL_SECONDS = 60;
|
|
16
16
|
|
|
17
17
|
// In the initial startup period, we use a short refresh interval. This helps to take over replication quickly in the case
|
|
18
18
|
// of rolling deploys. After the initial period, we switch to a longer refresh interval to reduce load.
|
|
@@ -34,6 +34,10 @@ export interface AbstractReplicatorOptions {
|
|
|
34
34
|
* This limits the effect of retries when there is a persistent issue.
|
|
35
35
|
*/
|
|
36
36
|
rateLimiter: ErrorRateLimiter;
|
|
37
|
+
/**
|
|
38
|
+
* Interval in seconds between source connection pings. Null or undefined uses the default; 0 disables pings.
|
|
39
|
+
*/
|
|
40
|
+
heartbeatIntervalSeconds?: number | null;
|
|
37
41
|
}
|
|
38
42
|
|
|
39
43
|
/**
|
|
@@ -51,25 +55,29 @@ export abstract class AbstractReplicator<T extends AbstractReplicationJob = Abst
|
|
|
51
55
|
private replicationJobs = new Map<string, T>();
|
|
52
56
|
|
|
53
57
|
/**
|
|
54
|
-
* Map of
|
|
58
|
+
* Map of replication stream ids to promises that are clearing the replication stream.
|
|
55
59
|
*
|
|
56
|
-
* We primarily do this to keep track of what we're currently clearing
|
|
57
|
-
* use the Promise value.
|
|
60
|
+
* We primarily do this to keep track of what we're currently clearing.
|
|
58
61
|
*/
|
|
59
|
-
|
|
62
|
+
protected readonly clearingJobs = new Map<number, Promise<void>>();
|
|
60
63
|
|
|
61
64
|
/**
|
|
62
65
|
* Used for replication lag computation.
|
|
63
66
|
*/
|
|
64
67
|
private activeReplicationJob: T | undefined = undefined;
|
|
65
68
|
|
|
66
|
-
//
|
|
69
|
+
// The first ping is only after the configured interval, not when starting.
|
|
67
70
|
private lastPing = hrtime.bigint();
|
|
68
71
|
|
|
72
|
+
private readonly heartbeatIntervalNanos: bigint | null;
|
|
73
|
+
|
|
69
74
|
private abortController: AbortController | undefined;
|
|
70
75
|
|
|
71
76
|
protected constructor(private options: AbstractReplicatorOptions) {
|
|
72
77
|
this.logger = logger.child({ name: `Replicator:${options.id}` });
|
|
78
|
+
const heartbeatIntervalSeconds = options.heartbeatIntervalSeconds ?? DEFAULT_HEARTBEAT_INTERVAL_SECONDS;
|
|
79
|
+
this.heartbeatIntervalNanos =
|
|
80
|
+
heartbeatIntervalSeconds === 0 ? null : BigInt(Math.round(heartbeatIntervalSeconds * 1_000_000_000));
|
|
73
81
|
}
|
|
74
82
|
|
|
75
83
|
/**
|
|
@@ -139,6 +147,7 @@ export abstract class AbstractReplicator<T extends AbstractReplicationJob = Abst
|
|
|
139
147
|
for (const job of this.replicationJobs.values()) {
|
|
140
148
|
promises.push(job.stop());
|
|
141
149
|
}
|
|
150
|
+
promises.push(...this.clearingJobs.values());
|
|
142
151
|
await Promise.all(promises);
|
|
143
152
|
}
|
|
144
153
|
|
|
@@ -184,9 +193,9 @@ export abstract class AbstractReplicator<T extends AbstractReplicationJob = Abst
|
|
|
184
193
|
|
|
185
194
|
// Ensure that the replication jobs' connections are kept alive.
|
|
186
195
|
// We don't ping while in error retry back-off, to avoid having too failures.
|
|
187
|
-
if (this.rateLimiter.mayPing()) {
|
|
196
|
+
if (this.heartbeatIntervalNanos != null && this.rateLimiter.mayPing()) {
|
|
188
197
|
const now = hrtime.bigint();
|
|
189
|
-
if (now - this.lastPing >=
|
|
198
|
+
if (now - this.lastPing >= this.heartbeatIntervalNanos) {
|
|
190
199
|
for (const activeJob of this.replicationJobs.values()) {
|
|
191
200
|
await activeJob.keepAlive();
|
|
192
201
|
}
|
|
@@ -312,9 +321,17 @@ export abstract class AbstractReplicator<T extends AbstractReplicationJob = Abst
|
|
|
312
321
|
// It is important to be able to continue running the refresh loop, otherwise we cannot
|
|
313
322
|
// retry locked replication stream, for example.
|
|
314
323
|
const syncRuleStorage = this.storage.getInstance(replicationStream, { skipLifecycleHooks: true });
|
|
315
|
-
const promise = this.
|
|
324
|
+
const promise = this.terminateStoppedReplicationStream(replicationStream, syncRuleStorage)
|
|
316
325
|
.catch((e) => {
|
|
317
|
-
|
|
326
|
+
if (e instanceof ReplicationAbortedError) {
|
|
327
|
+
// Expected when shutdown aborts an in-progress cleanup.
|
|
328
|
+
} else if (e?.errorData?.code === ErrorCode.PSYNC_S1003) {
|
|
329
|
+
this.logReplicationStreamInfoOnce(replicationStream, 'replication-stream-cleanup-locked', () => {
|
|
330
|
+
replicationStream.logger.info(`[${e.errorData.code}] ${e.errorData.description}`);
|
|
331
|
+
});
|
|
332
|
+
} else {
|
|
333
|
+
syncRuleStorage.logger.warn(`Failed clean up replication config`, e);
|
|
334
|
+
}
|
|
318
335
|
})
|
|
319
336
|
.finally(() => {
|
|
320
337
|
this.clearingJobs.delete(replicationStream.replicationStreamId);
|
|
@@ -370,6 +387,19 @@ export abstract class AbstractReplicator<T extends AbstractReplicationJob = Abst
|
|
|
370
387
|
return `${this.id}-${syncRuleId}`;
|
|
371
388
|
}
|
|
372
389
|
|
|
390
|
+
protected async terminateStoppedReplicationStream(
|
|
391
|
+
replicationStream: storage.PersistedReplicationStream,
|
|
392
|
+
syncRuleStorage: storage.SyncRulesBucketStorage
|
|
393
|
+
) {
|
|
394
|
+
const lock = await replicationStream.lock();
|
|
395
|
+
this.clearReplicationStreamInfoLog(replicationStream, 'replication-stream-cleanup-locked');
|
|
396
|
+
try {
|
|
397
|
+
await this.terminateSyncRules(syncRuleStorage);
|
|
398
|
+
} finally {
|
|
399
|
+
await lock.release();
|
|
400
|
+
}
|
|
401
|
+
}
|
|
402
|
+
|
|
373
403
|
protected async terminateSyncRules(syncRuleStorage: storage.SyncRulesBucketStorage) {
|
|
374
404
|
syncRuleStorage.logger.info(`Terminating replication stream...`);
|
|
375
405
|
// This deletes postgres replication slots - should complete quickly.
|
|
@@ -1,10 +1,49 @@
|
|
|
1
|
-
import { logger, router, schema } from '@powersync/lib-services-framework';
|
|
1
|
+
import { codecs, logger, router, schema } from '@powersync/lib-services-framework';
|
|
2
2
|
import * as t from 'ts-codec';
|
|
3
3
|
|
|
4
4
|
import * as util from '../../util/util-index.js';
|
|
5
5
|
import { authUser } from '../auth.js';
|
|
6
6
|
import { routeDefinition } from '../router.js';
|
|
7
7
|
|
|
8
|
+
const CHECKPOINT_REQUEST_ID_MAX = 9_223_372_036_854_775_807n;
|
|
9
|
+
|
|
10
|
+
function isValidCheckpointRequestId(value: bigint) {
|
|
11
|
+
return value > 0n && value <= CHECKPOINT_REQUEST_ID_MAX;
|
|
12
|
+
}
|
|
13
|
+
|
|
14
|
+
export const CheckpointRequestPayload = t.object({
|
|
15
|
+
client_id: t.string,
|
|
16
|
+
// Positive int64, matching the managed write checkpoint id. Clients should
|
|
17
|
+
// send values larger than Number.MAX_SAFE_INTEGER as strings, since JSON
|
|
18
|
+
// numbers are only validated up to the safe-integer range.
|
|
19
|
+
checkpoint_request_id: codecs.bigint
|
|
20
|
+
});
|
|
21
|
+
|
|
22
|
+
const CheckpointRequestPayloadShapeValidator = schema.createTsCodecValidator(CheckpointRequestPayload, {
|
|
23
|
+
allowAdditional: true
|
|
24
|
+
});
|
|
25
|
+
|
|
26
|
+
const CheckpointRequestPayloadValidator = {
|
|
27
|
+
validate(params: t.Encoded<typeof CheckpointRequestPayload>) {
|
|
28
|
+
const shapeValidation = CheckpointRequestPayloadShapeValidator.validate(params);
|
|
29
|
+
if (!shapeValidation.valid) {
|
|
30
|
+
return shapeValidation;
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
const decodedParams = CheckpointRequestPayload.decode(params);
|
|
34
|
+
if (!isValidCheckpointRequestId(decodedParams.checkpoint_request_id)) {
|
|
35
|
+
return {
|
|
36
|
+
valid: false as const,
|
|
37
|
+
errors: [`Expected checkpoint_request_id between 1 and ${CHECKPOINT_REQUEST_ID_MAX}`]
|
|
38
|
+
};
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
return {
|
|
42
|
+
valid: true as const
|
|
43
|
+
};
|
|
44
|
+
}
|
|
45
|
+
};
|
|
46
|
+
|
|
8
47
|
const WriteCheckpointRequest = t.object({
|
|
9
48
|
client_id: t.string.optional()
|
|
10
49
|
});
|
|
@@ -24,7 +63,7 @@ export const writeCheckpoint = routeDefinition({
|
|
|
24
63
|
// Since we don't use LSNs anymore, the only way to get that is to wait.
|
|
25
64
|
const start = Date.now();
|
|
26
65
|
|
|
27
|
-
const head = await apiHandler.createReplicationHead(async (head) => head);
|
|
66
|
+
const head = await apiHandler.createReplicationHead(async (head) => ({ response: head, shouldAdvance: true }));
|
|
28
67
|
|
|
29
68
|
const timeout = 50_000;
|
|
30
69
|
|
|
@@ -70,4 +109,39 @@ export const writeCheckpoint2 = routeDefinition({
|
|
|
70
109
|
}
|
|
71
110
|
});
|
|
72
111
|
|
|
73
|
-
export const
|
|
112
|
+
export const checkpointRequest = routeDefinition({
|
|
113
|
+
path: '/sync/checkpoint-request',
|
|
114
|
+
method: router.HTTPMethod.POST,
|
|
115
|
+
authorize: authUser,
|
|
116
|
+
validator: CheckpointRequestPayloadValidator,
|
|
117
|
+
handler: async (request) => {
|
|
118
|
+
const { token_payload, service_context } = request.context;
|
|
119
|
+
const { params } = request;
|
|
120
|
+
|
|
121
|
+
const decodedParams = CheckpointRequestPayload.decode(params);
|
|
122
|
+
|
|
123
|
+
// Storage only applies supplied request ids that advance the stored managed checkpoint.
|
|
124
|
+
// Stale or duplicate ids return the stored checkpoint; storage decides whether
|
|
125
|
+
// the matched checkpoint still needs a source marker.
|
|
126
|
+
const { replicationHead, writeCheckpoint } = await util.createWriteCheckpoint({
|
|
127
|
+
userId: token_payload!.userIdString,
|
|
128
|
+
clientId: decodedParams.client_id,
|
|
129
|
+
batcher: service_context.writeCheckpointBatcher,
|
|
130
|
+
checkpointRequestId: decodedParams.checkpoint_request_id
|
|
131
|
+
});
|
|
132
|
+
|
|
133
|
+
logger.info(
|
|
134
|
+
`Requested checkpoint for ${token_payload!.userIdString}/${decodedParams.client_id}: ${writeCheckpoint} | ${replicationHead}`
|
|
135
|
+
);
|
|
136
|
+
|
|
137
|
+
// Return the checkpoint request id storage is actually at after this request.
|
|
138
|
+
// When an earlier request has already advanced the stored value beyond the
|
|
139
|
+
// supplied checkpoint_request_id, this returns that larger previous value so
|
|
140
|
+
// the client can treat the response as a stale-request acknowledgement.
|
|
141
|
+
return {
|
|
142
|
+
checkpoint_request_id: String(writeCheckpoint)
|
|
143
|
+
};
|
|
144
|
+
}
|
|
145
|
+
});
|
|
146
|
+
|
|
147
|
+
export const CHECKPOINT_ROUTES = [checkpointRequest, writeCheckpoint, writeCheckpoint2];
|
|
@@ -173,7 +173,10 @@ export interface BucketStorageBatch extends ObserverClient<BucketBatchStorageLis
|
|
|
173
173
|
resolveTables(options: ResolveTablesOptions): Promise<ResolveTablesResult>;
|
|
174
174
|
|
|
175
175
|
/**
|
|
176
|
-
* Queue a custom
|
|
176
|
+
* Queue a custom checkpoint request to be persisted after operations are
|
|
177
|
+
* flushed. Set `checkpoint_requested_at` when the custom checkpoint came from
|
|
178
|
+
* a client checkpoint request and should be cleaned up by request-retention
|
|
179
|
+
* compaction; omit it for persistent source-owned checkpoints.
|
|
177
180
|
*/
|
|
178
181
|
addCustomWriteCheckpoint(checkpoint: BatchedCustomWriteCheckpointOptions): void;
|
|
179
182
|
}
|
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
import { BaseObserver, logger } from '@powersync/lib-services-framework';
|
|
2
2
|
import {
|
|
3
3
|
PrecompiledSyncConfig,
|
|
4
|
+
SerializedSyncPlan as RawSerializedSyncPlan,
|
|
4
5
|
SerializedCompatibilityContext,
|
|
5
|
-
SerializedSyncPlanV1,
|
|
6
6
|
serializeSyncPlan,
|
|
7
7
|
SqlSyncRules,
|
|
8
8
|
SyncConfigWithErrors
|
|
@@ -171,7 +171,7 @@ export interface SerializedSyncPlan {
|
|
|
171
171
|
/**
|
|
172
172
|
* The serialized plan, from {@link serializeSyncPlan}.
|
|
173
173
|
*/
|
|
174
|
-
plan:
|
|
174
|
+
plan: RawSerializedSyncPlan;
|
|
175
175
|
compatibility: SerializedCompatibilityContext;
|
|
176
176
|
/**
|
|
177
177
|
* Event descriptors are not currently represented in the sync plan because they don't use the sync streams compiler
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
import { InternalOpId } from '../util/util-index.js';
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* A checkpoint cannot be served because compaction rewrote a bucket-data
|
|
5
|
+
* document across its end boundary.
|
|
6
|
+
*
|
|
7
|
+
* The sync loop must skip this checkpoint before it sends its checkpoint line.
|
|
8
|
+
*/
|
|
9
|
+
export class CheckpointChecksumInvalidatedError extends Error {
|
|
10
|
+
constructor(
|
|
11
|
+
public readonly checkpoint: InternalOpId,
|
|
12
|
+
public readonly bucket: string
|
|
13
|
+
) {
|
|
14
|
+
super(`Checkpoint ${checkpoint} was invalidated by compaction in bucket ${bucket}`);
|
|
15
|
+
this.name = 'CheckpointChecksumInvalidatedError';
|
|
16
|
+
}
|
|
17
|
+
}
|
|
@@ -1,5 +1,11 @@
|
|
|
1
1
|
import { SourceTableRef } from '@powersync/service-sync-rules';
|
|
2
2
|
|
|
3
|
+
/**
|
|
4
|
+
* Source-specific JSON metadata. Storage does not interpret it. Source-table APIs use null to
|
|
5
|
+
* represent the absence of metadata.
|
|
6
|
+
*/
|
|
7
|
+
export type JsonValue = string | number | boolean | null | JsonValue[] | { [key: string]: JsonValue };
|
|
8
|
+
|
|
3
9
|
export interface ColumnDescriptor {
|
|
4
10
|
name: string;
|
|
5
11
|
/**
|
|
@@ -19,8 +25,6 @@ export interface SourceEntityDescriptor extends SourceTableRef {
|
|
|
19
25
|
* If specified, this is specifically used to detect renames.
|
|
20
26
|
*/
|
|
21
27
|
objectId: number | string | undefined;
|
|
22
|
-
schema: string;
|
|
23
|
-
name: string;
|
|
24
28
|
/**
|
|
25
29
|
* The columns that are used to uniquely identify a record in the source entity.
|
|
26
30
|
*/
|
|
@@ -8,13 +8,23 @@ import {
|
|
|
8
8
|
} from '@powersync/service-sync-rules';
|
|
9
9
|
import { bson } from '../index.js';
|
|
10
10
|
import * as util from '../util/util-index.js';
|
|
11
|
-
import { ColumnDescriptor } from './SourceEntity.js';
|
|
11
|
+
import { ColumnDescriptor, JsonValue } from './SourceEntity.js';
|
|
12
12
|
|
|
13
13
|
/**
|
|
14
14
|
* Format of the id depends on the bucket storage module. It should be consistent within the module.
|
|
15
15
|
*/
|
|
16
16
|
export type SourceTableId = string | bson.ObjectId;
|
|
17
17
|
|
|
18
|
+
/**
|
|
19
|
+
* Compare source-table ids without coercing between storage-specific types.
|
|
20
|
+
*/
|
|
21
|
+
export function sourceTableIdEquals(left: SourceTableId, right: SourceTableId): boolean {
|
|
22
|
+
if (typeof left === 'string' || typeof right === 'string') {
|
|
23
|
+
return typeof left === 'string' && typeof right === 'string' && left === right;
|
|
24
|
+
}
|
|
25
|
+
return left.equals(right);
|
|
26
|
+
}
|
|
27
|
+
|
|
18
28
|
export interface SourceTableOptions {
|
|
19
29
|
id: SourceTableId;
|
|
20
30
|
ref: SourceTableRef;
|
|
@@ -25,6 +35,10 @@ export interface SourceTableOptions {
|
|
|
25
35
|
parameterLookupSources: ParameterIndexLookupCreator[];
|
|
26
36
|
bucketDataSourceIds?: Set<BucketDefinitionId>;
|
|
27
37
|
parameterLookupSourceIds?: Set<ParameterIndexId>;
|
|
38
|
+
/**
|
|
39
|
+
* Source-specific metadata. Null when no metadata has been recorded.
|
|
40
|
+
*/
|
|
41
|
+
sourceMetadata?: JsonValue;
|
|
28
42
|
}
|
|
29
43
|
|
|
30
44
|
export interface TableSnapshotStatus {
|
|
@@ -136,6 +150,10 @@ export class SourceTable {
|
|
|
136
150
|
return this.options.parameterLookupSourceIds;
|
|
137
151
|
}
|
|
138
152
|
|
|
153
|
+
get sourceMetadata() {
|
|
154
|
+
return this.options.sourceMetadata ?? null;
|
|
155
|
+
}
|
|
156
|
+
|
|
139
157
|
/**
|
|
140
158
|
* Sanitized name of the entity in the format of "{schema}.{entity name}".
|
|
141
159
|
* Suitable for safe use in Postgres queries.
|
|
@@ -152,23 +170,41 @@ export class SourceTable {
|
|
|
152
170
|
* In-memory clone of the table status.
|
|
153
171
|
*/
|
|
154
172
|
clone() {
|
|
173
|
+
return this.copyWithSourceMetadata(this.sourceMetadata);
|
|
174
|
+
}
|
|
175
|
+
|
|
176
|
+
/**
|
|
177
|
+
* Copy this table with different source metadata, preserving its resolved state.
|
|
178
|
+
*/
|
|
179
|
+
withSourceMetadata(sourceMetadata: JsonValue) {
|
|
180
|
+
return this.copyWithSourceMetadata(sourceMetadata);
|
|
181
|
+
}
|
|
182
|
+
|
|
183
|
+
private copyWithSourceMetadata(sourceMetadata: JsonValue) {
|
|
155
184
|
const copy = new SourceTable({
|
|
156
185
|
id: this.id,
|
|
157
|
-
ref: this.options.ref,
|
|
186
|
+
ref: { ...this.options.ref },
|
|
158
187
|
objectId: this.objectId,
|
|
159
|
-
replicaIdColumns: this.replicaIdColumns,
|
|
188
|
+
replicaIdColumns: this.replicaIdColumns.map((column) => ({ ...column })),
|
|
160
189
|
snapshotComplete: this.snapshotComplete,
|
|
161
|
-
bucketDataSources: this.bucketDataSources,
|
|
162
|
-
parameterLookupSources: this.parameterLookupSources,
|
|
190
|
+
bucketDataSources: [...this.bucketDataSources],
|
|
191
|
+
parameterLookupSources: [...this.parameterLookupSources],
|
|
163
192
|
bucketDataSourceIds: this.bucketDataSourceIds == null ? undefined : new Set(this.bucketDataSourceIds),
|
|
164
193
|
parameterLookupSourceIds:
|
|
165
|
-
this.parameterLookupSourceIds == null ? undefined : new Set(this.parameterLookupSourceIds)
|
|
194
|
+
this.parameterLookupSourceIds == null ? undefined : new Set(this.parameterLookupSourceIds),
|
|
195
|
+
sourceMetadata: structuredClone(sourceMetadata)
|
|
166
196
|
});
|
|
167
197
|
copy.syncData = this.syncData;
|
|
168
198
|
copy.syncParameters = this.syncParameters;
|
|
169
199
|
copy.syncEvent = this.syncEvent;
|
|
170
200
|
copy.storeCurrentData = this.storeCurrentData;
|
|
171
|
-
copy.snapshotStatus =
|
|
201
|
+
copy.snapshotStatus =
|
|
202
|
+
this.snapshotStatus == null
|
|
203
|
+
? undefined
|
|
204
|
+
: {
|
|
205
|
+
...this.snapshotStatus,
|
|
206
|
+
lastKey: this.snapshotStatus.lastKey?.slice() ?? null
|
|
207
|
+
};
|
|
172
208
|
return copy;
|
|
173
209
|
}
|
|
174
210
|
|
|
@@ -183,3 +219,11 @@ export class SourceTable {
|
|
|
183
219
|
}
|
|
184
220
|
}
|
|
185
221
|
}
|
|
222
|
+
|
|
223
|
+
/**
|
|
224
|
+
* A cloned SourceTable exposed to reconciliation with public fields typed as read-only.
|
|
225
|
+
* `options` is omitted so callers cannot mutate the underlying option bag without an explicit cast.
|
|
226
|
+
*/
|
|
227
|
+
export type SourceTableCandidate = Omit<Readonly<SourceTable>, 'options' | 'withSourceMetadata'> & {
|
|
228
|
+
withSourceMetadata(sourceMetadata: JsonValue): SourceTableCandidate;
|
|
229
|
+
};
|