@powersync/service-core 0.18.0 → 1.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +22 -0
- package/dist/api/RouteAPI.d.ts +9 -1
- package/dist/api/diagnostics.js +107 -169
- package/dist/api/diagnostics.js.map +1 -1
- package/dist/entry/commands/compact-action.js +10 -73
- package/dist/entry/commands/compact-action.js.map +1 -1
- package/dist/modules/AbstractModule.d.ts +1 -1
- package/dist/replication/AbstractReplicator.js +2 -65
- package/dist/replication/AbstractReplicator.js.map +1 -1
- package/dist/routes/configure-fastify.d.ts +3 -3
- package/dist/routes/endpoints/admin.d.ts +6 -6
- package/dist/routes/endpoints/checkpointing.js +16 -85
- package/dist/routes/endpoints/checkpointing.js.map +1 -1
- package/dist/routes/endpoints/socket-route.js +5 -5
- package/dist/routes/endpoints/socket-route.js.map +1 -1
- package/dist/routes/endpoints/sync-stream.js +5 -5
- package/dist/routes/endpoints/sync-stream.js.map +1 -1
- package/dist/runner/teardown.js +3 -65
- package/dist/runner/teardown.js.map +1 -1
- package/dist/storage/BucketStorage.d.ts +1 -442
- package/dist/storage/BucketStorage.js +0 -9
- package/dist/storage/BucketStorage.js.map +1 -1
- package/dist/storage/BucketStorageBatch.d.ts +130 -0
- package/dist/storage/BucketStorageBatch.js +10 -0
- package/dist/storage/BucketStorageBatch.js.map +1 -0
- package/dist/storage/BucketStorageFactory.d.ts +136 -0
- package/dist/storage/BucketStorageFactory.js +2 -0
- package/dist/storage/BucketStorageFactory.js.map +1 -0
- package/dist/storage/ChecksumCache.js.map +1 -1
- package/dist/storage/PersistedSyncRulesContent.d.ts +20 -0
- package/dist/storage/PersistedSyncRulesContent.js +2 -0
- package/dist/storage/PersistedSyncRulesContent.js.map +1 -0
- package/dist/storage/ReplicationEventPayload.d.ts +1 -1
- package/dist/storage/ReplicationLock.d.ts +4 -0
- package/dist/storage/ReplicationLock.js +2 -0
- package/dist/storage/ReplicationLock.js.map +1 -0
- package/dist/storage/StorageEngine.d.ts +4 -4
- package/dist/storage/StorageEngine.js +2 -2
- package/dist/storage/StorageEngine.js.map +1 -1
- package/dist/storage/StorageProvider.d.ts +4 -1
- package/dist/storage/SyncRulesBucketStorage.d.ts +201 -0
- package/dist/storage/SyncRulesBucketStorage.js +7 -0
- package/dist/storage/SyncRulesBucketStorage.js.map +1 -0
- package/dist/storage/bson.d.ts +11 -3
- package/dist/storage/bson.js +24 -2
- package/dist/storage/bson.js.map +1 -1
- package/dist/storage/storage-index.d.ts +5 -0
- package/dist/storage/storage-index.js +5 -0
- package/dist/storage/storage-index.js.map +1 -1
- package/dist/sync/BucketChecksumState.d.ts +91 -0
- package/dist/sync/BucketChecksumState.js +313 -0
- package/dist/sync/BucketChecksumState.js.map +1 -0
- package/dist/sync/sync-index.d.ts +1 -0
- package/dist/sync/sync-index.js +1 -0
- package/dist/sync/sync-index.js.map +1 -1
- package/dist/sync/sync.d.ts +7 -3
- package/dist/sync/sync.js +131 -135
- package/dist/sync/sync.js.map +1 -1
- package/dist/sync/util.d.ts +9 -0
- package/dist/sync/util.js +44 -0
- package/dist/sync/util.js.map +1 -1
- package/dist/util/checkpointing.d.ts +13 -0
- package/dist/util/checkpointing.js +29 -0
- package/dist/util/checkpointing.js.map +1 -0
- package/dist/util/protocol-types.d.ts +13 -4
- package/dist/util/util-index.d.ts +1 -0
- package/dist/util/util-index.js +1 -0
- package/dist/util/util-index.js.map +1 -1
- package/dist/util/utils.d.ts +0 -1
- package/dist/util/utils.js +0 -9
- package/dist/util/utils.js.map +1 -1
- package/package.json +4 -4
- package/src/api/RouteAPI.ts +11 -1
- package/src/api/diagnostics.ts +1 -1
- package/src/entry/commands/compact-action.ts +2 -3
- package/src/modules/AbstractModule.ts +1 -1
- package/src/replication/AbstractReplicator.ts +1 -2
- package/src/routes/endpoints/checkpointing.ts +11 -22
- package/src/routes/endpoints/socket-route.ts +7 -5
- package/src/routes/endpoints/sync-stream.ts +8 -5
- package/src/runner/teardown.ts +1 -1
- package/src/storage/BucketStorage.ts +1 -552
- package/src/storage/BucketStorageBatch.ts +158 -0
- package/src/storage/BucketStorageFactory.ts +156 -0
- package/src/storage/ChecksumCache.ts +1 -0
- package/src/storage/PersistedSyncRulesContent.ts +26 -0
- package/src/storage/ReplicationEventPayload.ts +1 -1
- package/src/storage/ReplicationLock.ts +5 -0
- package/src/storage/StorageEngine.ts +4 -4
- package/src/storage/StorageProvider.ts +4 -1
- package/src/storage/SyncRulesBucketStorage.ts +256 -0
- package/src/storage/bson.ts +28 -4
- package/src/storage/storage-index.ts +5 -0
- package/src/sync/BucketChecksumState.ts +392 -0
- package/src/sync/sync-index.ts +1 -0
- package/src/sync/sync.ts +173 -157
- package/src/sync/util.ts +54 -0
- package/src/util/checkpointing.ts +41 -0
- package/src/util/protocol-types.ts +16 -4
- package/src/util/util-index.ts +1 -0
- package/src/util/utils.ts +0 -10
- package/test/src/auth.test.ts +5 -5
- package/test/src/sync/BucketChecksumState.test.ts +565 -0
- package/test/src/sync/util.test.ts +34 -0
- package/tsconfig.tsbuildinfo +1 -1
package/dist/util/utils.js
CHANGED
|
@@ -117,15 +117,6 @@ export function isCompleteRow(storeData, row) {
|
|
|
117
117
|
}
|
|
118
118
|
return !hasToastedValues(row);
|
|
119
119
|
}
|
|
120
|
-
export function checkpointUserId(user_id, client_id) {
|
|
121
|
-
if (user_id == null) {
|
|
122
|
-
throw new Error('user_id is required');
|
|
123
|
-
}
|
|
124
|
-
if (client_id == null) {
|
|
125
|
-
return user_id;
|
|
126
|
-
}
|
|
127
|
-
return `${user_id}/${client_id}`;
|
|
128
|
-
}
|
|
129
120
|
/**
|
|
130
121
|
* Reduce a bucket to the final state as stored on the client.
|
|
131
122
|
*
|
package/dist/util/utils.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"utils.js","sourceRoot":"","sources":["../../src/util/utils.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,IAAI,MAAM,MAAM,CAAC;AAC7B,OAAO,MAAM,MAAM,QAAQ,CAAC;AAC5B,OAAO,KAAK,IAAI,MAAM,MAAM,CAAC;AAM7B,OAAO,EAAE,qBAAqB,EAAE,MAAM,mCAAmC,CAAC;AAI1E,MAAM,CAAC,MAAM,YAAY,GAAG,sCAAsC,CAAC;AAEnE,MAAM,UAAU,gBAAgB,CAAC,UAAkB;IACjD,OAAO,IAAI,UAAU,CAAC,OAAO,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC,OAAO,CAAC,KAAK,EAAE,KAAK,CAAC,GAAG,CAAC;AACrE,CAAC;AAED,MAAM,UAAU,QAAQ,CAAC,IAAY,EAAE,EAAU,EAAE,IAAY;IAC7D,MAAM,IAAI,GAAG,MAAM,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC;IACzC,IAAI,CAAC,MAAM,CAAC,OAAO,IAAI,IAAI,EAAE,IAAI,IAAI,EAAE,CAAC,CAAC;IACzC,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC;IAC7B,OAAO,MAAM,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;AAChC,CAAC;AAED,MAAM,UAAU,UAAU,CAAC,SAAiB;IAC1C,MAAM,IAAI,GAAG,MAAM,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC;IACzC,IAAI,CAAC,MAAM,CAAC,UAAU,SAAS,EAAE,CAAC,CAAC;IACnC,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC;IAC7B,OAAO,MAAM,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;AAChC,CAAC;AAED,MAAM,UAAU,eAAe,CAAC,EAAU;IACxC,6EAA6E;IAC7E,6CAA6C;IAC7C,IAAI,OAAO,EAAE,IAAI,QAAQ,EAAE,CAAC;QAC1B,MAAM,IAAI,qBAAqB,CAAC,yBAAyB,EAAE,KAAK,OAAO,EAAE,GAAG,CAAC,CAAC;IAChF,CAAC;IACD,OAAO,EAAE,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC;AACzB,CAAC;AAED,MAAM,UAAU,aAAa,CAAC,QAAqB,EAAE,OAAoB;IACvE,mBAAmB;IACnB,MAAM,cAAc,GAAG,IAAI,GAAG,EAA0B,CAAC;IAEzD,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAS,QAAQ,CAAC,IAAI,EAAE,CAAC,CAAC;IAElD,KAAK,IAAI,QAAQ,IAAI,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC;QACtC,MAAM,CAAC,GAAG,QAAQ,CAAC,GAAG,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;QACxC,IAAI,CAAC,IAAI,IAAI,EAAE,CAAC;YACd,QAAQ;YACR,cAAc,CAAC,GAAG,CAAC,QAAQ,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;QAChD,CAAC;aAAM,CAAC;YACN,QAAQ,CAAC,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;YACjC,IAAI,QAAQ,CAAC,QAAQ,IAAI,CAAC,CAAC,QAAQ,IAAI,QAAQ,CAAC,KAAK,IAAI,CAAC,CAAC,KAAK,EAAE,CAAC;gBACjE,UAAU;gBACV,cAAc,CAAC,GAAG,CAAC,QAAQ,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;YAChD,CAAC;iBAAM,CAAC;gBACN,YAAY;YACd,CAAC;QACH,CAAC;IACH,CAAC;IAED,OAAO;QACL,cAAc,EAAE,CAAC,GAAG,cAAc,CAAC,MAAM,EAAE,CAAC;QAC5C,cAAc,EAAE,CAAC,GAAG,QAAQ,CAAC;KAC9B,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,YAAY,CAAC,CAAS,EAAE,CAAS;IAC/C,OAAO,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,UAAU,CAAC;AAC9B,CAAC;AAED,MAAM,UAAU,kBAAkB,CAAC,CAAiB,EAAE,CAAyB;IAC7E,IAAI,CAAC,IAAI,IAAI,EAAE,CAAC;QACd,OAAO,CAAC,CAAC;IACX,CAAC;SAAM,IAAI,CAAC,CAAC,cAAc,EAAE,CAAC;QAC5B,OAAO;YACL,MAAM,EAAE,CAAC,CAAC,MAAM;YAChB,KAAK,EAAE,CAAC,CAAC,YAAY;YACrB,QAAQ,EAAE,CAAC,CAAC,eAAe;SAC5B,CAAC;IACJ,CAAC;SAAM,CAAC;QACN,OAAO;YACL,MAAM,EAAE,CAAC,CAAC,MAAM;YAChB,KAAK,EAAE,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,YAAY;YAC/B,QAAQ,EAAE,YAAY,CAAC,CAAC,CAAC,QAAQ,EAAE,CAAC,CAAC,eAAe,CAAC;SACtD,CAAC;IACJ,CAAC;AACH,CAAC;AAED,SAAS,qBAAqB,CAC5B,KAAoC,EACpC,OAAmC;IAEnC,IAAI,MAAM,GAAwB,EAAE,CAAC;IACrC,KAAK,IAAI,MAAM,IAAI,OAAO,EAAE,CAAC;QAC3B,MAAM,IAAI,GAAG,MAAM,CAAC,IAAI,CAAC;QACzB,MAAM,CAAC,IAAI,CAAC,GAAG,KAAK,CAAC,IAAI,CAAC,CAAC;IAC7B,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,MAAM,UAAU,0BAA0B,CACxC,KAAoC,EACpC,OAAmC;IAEnC,IAAI,OAAO,CAAC,MAAM,IAAI,CAAC,EAAE,CAAC;QACxB,gDAAgD;QAChD,OAAO,IAAI,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,EAAE,CAAC,CAAC;IAClC,CAAC;IACD,MAAM,WAAW,GAAG,qBAAqB,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC;IAE1D,OAAO,cAAc,CAAC,WAAW,CAAC,CAAC;AACrC,CAAC;AAED,MAAM,UAAU,cAAc,CAAC,GAAyB;IACtD,+EAA+E;IAC/E,4EAA4E;IAC5E,MAAM,IAAI,GAAG,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC;IACjC,MAAM,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;IAChC,OAAO,IAAI,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,EAAE,YAAY,EAAE,MAAM,CAAC,CAAC,CAAC;AAC5D,CAAC;AAED,MAAM,UAAU,gBAAgB,CAAC,GAAkC;IACjE,KAAK,IAAI,GAAG,IAAI,GAAG,EAAE,CAAC;QACpB,IAAI,OAAO,GAAG,CAAC,GAAG,CAAC,IAAI,WAAW,EAAE,CAAC;YACnC,OAAO,IAAI,CAAC;QACd,CAAC;IACH,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,aAAa,CAAC,SAAkB,EAAE,GAAkC;IAClF,IAAI,CAAC,SAAS,EAAE,CAAC;QACf,gDAAgD;QAChD,OAAO,IAAI,CAAC;IACd,CAAC;IACD,OAAO,CAAC,gBAAgB,CAAC,GAAG,CAAC,CAAC;AAChC,CAAC;AAED
|
|
1
|
+
{"version":3,"file":"utils.js","sourceRoot":"","sources":["../../src/util/utils.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,IAAI,MAAM,MAAM,CAAC;AAC7B,OAAO,MAAM,MAAM,QAAQ,CAAC;AAC5B,OAAO,KAAK,IAAI,MAAM,MAAM,CAAC;AAM7B,OAAO,EAAE,qBAAqB,EAAE,MAAM,mCAAmC,CAAC;AAI1E,MAAM,CAAC,MAAM,YAAY,GAAG,sCAAsC,CAAC;AAEnE,MAAM,UAAU,gBAAgB,CAAC,UAAkB;IACjD,OAAO,IAAI,UAAU,CAAC,OAAO,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC,OAAO,CAAC,KAAK,EAAE,KAAK,CAAC,GAAG,CAAC;AACrE,CAAC;AAED,MAAM,UAAU,QAAQ,CAAC,IAAY,EAAE,EAAU,EAAE,IAAY;IAC7D,MAAM,IAAI,GAAG,MAAM,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC;IACzC,IAAI,CAAC,MAAM,CAAC,OAAO,IAAI,IAAI,EAAE,IAAI,IAAI,EAAE,CAAC,CAAC;IACzC,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC;IAC7B,OAAO,MAAM,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;AAChC,CAAC;AAED,MAAM,UAAU,UAAU,CAAC,SAAiB;IAC1C,MAAM,IAAI,GAAG,MAAM,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC;IACzC,IAAI,CAAC,MAAM,CAAC,UAAU,SAAS,EAAE,CAAC,CAAC;IACnC,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC;IAC7B,OAAO,MAAM,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;AAChC,CAAC;AAED,MAAM,UAAU,eAAe,CAAC,EAAU;IACxC,6EAA6E;IAC7E,6CAA6C;IAC7C,IAAI,OAAO,EAAE,IAAI,QAAQ,EAAE,CAAC;QAC1B,MAAM,IAAI,qBAAqB,CAAC,yBAAyB,EAAE,KAAK,OAAO,EAAE,GAAG,CAAC,CAAC;IAChF,CAAC;IACD,OAAO,EAAE,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC;AACzB,CAAC;AAED,MAAM,UAAU,aAAa,CAAC,QAAqB,EAAE,OAAoB;IACvE,mBAAmB;IACnB,MAAM,cAAc,GAAG,IAAI,GAAG,EAA0B,CAAC;IAEzD,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAS,QAAQ,CAAC,IAAI,EAAE,CAAC,CAAC;IAElD,KAAK,IAAI,QAAQ,IAAI,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC;QACtC,MAAM,CAAC,GAAG,QAAQ,CAAC,GAAG,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;QACxC,IAAI,CAAC,IAAI,IAAI,EAAE,CAAC;YACd,QAAQ;YACR,cAAc,CAAC,GAAG,CAAC,QAAQ,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;QAChD,CAAC;aAAM,CAAC;YACN,QAAQ,CAAC,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;YACjC,IAAI,QAAQ,CAAC,QAAQ,IAAI,CAAC,CAAC,QAAQ,IAAI,QAAQ,CAAC,KAAK,IAAI,CAAC,CAAC,KAAK,EAAE,CAAC;gBACjE,UAAU;gBACV,cAAc,CAAC,GAAG,CAAC,QAAQ,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;YAChD,CAAC;iBAAM,CAAC;gBACN,YAAY;YACd,CAAC;QACH,CAAC;IACH,CAAC;IAED,OAAO;QACL,cAAc,EAAE,CAAC,GAAG,cAAc,CAAC,MAAM,EAAE,CAAC;QAC5C,cAAc,EAAE,CAAC,GAAG,QAAQ,CAAC;KAC9B,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,YAAY,CAAC,CAAS,EAAE,CAAS;IAC/C,OAAO,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,UAAU,CAAC;AAC9B,CAAC;AAED,MAAM,UAAU,kBAAkB,CAAC,CAAiB,EAAE,CAAyB;IAC7E,IAAI,CAAC,IAAI,IAAI,EAAE,CAAC;QACd,OAAO,CAAC,CAAC;IACX,CAAC;SAAM,IAAI,CAAC,CAAC,cAAc,EAAE,CAAC;QAC5B,OAAO;YACL,MAAM,EAAE,CAAC,CAAC,MAAM;YAChB,KAAK,EAAE,CAAC,CAAC,YAAY;YACrB,QAAQ,EAAE,CAAC,CAAC,eAAe;SAC5B,CAAC;IACJ,CAAC;SAAM,CAAC;QACN,OAAO;YACL,MAAM,EAAE,CAAC,CAAC,MAAM;YAChB,KAAK,EAAE,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,YAAY;YAC/B,QAAQ,EAAE,YAAY,CAAC,CAAC,CAAC,QAAQ,EAAE,CAAC,CAAC,eAAe,CAAC;SACtD,CAAC;IACJ,CAAC;AACH,CAAC;AAED,SAAS,qBAAqB,CAC5B,KAAoC,EACpC,OAAmC;IAEnC,IAAI,MAAM,GAAwB,EAAE,CAAC;IACrC,KAAK,IAAI,MAAM,IAAI,OAAO,EAAE,CAAC;QAC3B,MAAM,IAAI,GAAG,MAAM,CAAC,IAAI,CAAC;QACzB,MAAM,CAAC,IAAI,CAAC,GAAG,KAAK,CAAC,IAAI,CAAC,CAAC;IAC7B,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,MAAM,UAAU,0BAA0B,CACxC,KAAoC,EACpC,OAAmC;IAEnC,IAAI,OAAO,CAAC,MAAM,IAAI,CAAC,EAAE,CAAC;QACxB,gDAAgD;QAChD,OAAO,IAAI,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,EAAE,CAAC,CAAC;IAClC,CAAC;IACD,MAAM,WAAW,GAAG,qBAAqB,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC;IAE1D,OAAO,cAAc,CAAC,WAAW,CAAC,CAAC;AACrC,CAAC;AAED,MAAM,UAAU,cAAc,CAAC,GAAyB;IACtD,+EAA+E;IAC/E,4EAA4E;IAC5E,MAAM,IAAI,GAAG,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC;IACjC,MAAM,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;IAChC,OAAO,IAAI,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,EAAE,YAAY,EAAE,MAAM,CAAC,CAAC,CAAC;AAC5D,CAAC;AAED,MAAM,UAAU,gBAAgB,CAAC,GAAkC;IACjE,KAAK,IAAI,GAAG,IAAI,GAAG,EAAE,CAAC;QACpB,IAAI,OAAO,GAAG,CAAC,GAAG,CAAC,IAAI,WAAW,EAAE,CAAC;YACnC,OAAO,IAAI,CAAC;QACd,CAAC;IACH,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,aAAa,CAAC,SAAkB,EAAE,GAAkC;IAClF,IAAI,CAAC,SAAS,EAAE,CAAC;QACf,gDAAgD;QAChD,OAAO,IAAI,CAAC;IACd,CAAC;IACD,OAAO,CAAC,gBAAgB,CAAC,GAAG,CAAC,CAAC;AAChC,CAAC;AAED;;;;;;;;;;;GAWG;AACH,MAAM,UAAU,YAAY,CAAC,UAAwB;IACnD,IAAI,QAAQ,GAAG,IAAI,GAAG,EAAsB,CAAC;IAC7C,IAAI,aAAa,GAAG,CAAC,CAAC;IAEtB,KAAK,IAAI,EAAE,IAAI,UAAU,EAAE,CAAC;QAC1B,MAAM,GAAG,GAAG,MAAM,CAAC,EAAE,CAAC,CAAC;QACvB,IAAI,EAAE,CAAC,EAAE,IAAI,KAAK,EAAE,CAAC;YACnB,MAAM,QAAQ,GAAG,QAAQ,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;YACnC,IAAI,QAAQ,EAAE,CAAC;gBACb,aAAa,GAAG,YAAY,CAAC,aAAa,EAAE,QAAQ,CAAC,QAAkB,CAAC,CAAC;YAC3E,CAAC;YACD,QAAQ,CAAC,GAAG,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC;QACxB,CAAC;aAAM,IAAI,EAAE,CAAC,EAAE,IAAI,QAAQ,EAAE,CAAC;YAC7B,MAAM,QAAQ,GAAG,QAAQ,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;YACnC,IAAI,QAAQ,EAAE,CAAC;gBACb,aAAa,GAAG,YAAY,CAAC,aAAa,EAAE,QAAQ,CAAC,QAAkB,CAAC,CAAC;YAC3E,CAAC;YACD,QAAQ,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;YACrB,aAAa,GAAG,YAAY,CAAC,aAAa,EAAE,EAAE,CAAC,QAAkB,CAAC,CAAC;QACrE,CAAC;aAAM,IAAI,EAAE,CAAC,EAAE,IAAI,OAAO,EAAE,CAAC;YAC5B,QAAQ,CAAC,KAAK,EAAE,CAAC;YACjB,aAAa,GAAG,EAAE,CAAC,QAAkB,CAAC;QACxC,CAAC;aAAM,IAAI,EAAE,CAAC,EAAE,IAAI,MAAM,EAAE,CAAC;YAC3B,aAAa,GAAG,YAAY,CAAC,aAAa,EAAE,EAAE,CAAC,QAAkB,CAAC,CAAC;QACrE,CAAC;aAAM,CAAC;YACN,MAAM,IAAI,KAAK,CAAC,qBAAqB,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC;QAChD,CAAC;IACH,CAAC;IAED,MAAM,IAAI,GAAG,CAAC,GAAG,QAAQ,CAAC,MAAM,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE;QAChD,OAAO,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC;IACnD,CAAC,CAAC,CAAC;IAEH,IAAI,UAAU,GAAiB;QAC7B,wDAAwD;QACxD,EAAE,KAAK,EAAE,GAAG,EAAE,EAAE,EAAE,OAAO,EAAE,QAAQ,EAAE,aAAa,EAAE;QACpD,GAAG,IAAI;KACR,CAAC;IAEF,OAAO,UAAU,CAAC;AACpB,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,OAAO,CAAC,CAAS;IAC/B,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;IACf,OAAO,CAAC,CAAC;AACX,CAAC;AAED,SAAS,MAAM,CAAC,KAAiB;IAC/B,OAAO,GAAG,KAAK,CAAC,WAAW,IAAI,KAAK,CAAC,SAAS,IAAI,KAAK,CAAC,MAAM,EAAE,CAAC;AACnE,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,eAAe,CAAC,MAAiD;IAC/E,IAAI,MAAM,IAAI,IAAI,EAAE,CAAC;QACnB,OAAO,EAAE,CAAC;IACZ,CAAC;IACD,IAAI,IAAI,GAAG,CAAC,CAAC;IACb,KAAK,IAAI,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC;QAChD,IAAI,IAAI,EAAE,GAAG,GAAG,CAAC,MAAM,CAAC;QACxB,+CAA+C;QAC/C,IAAI,KAAK,IAAI,IAAI,EAAE,CAAC;YAClB,IAAI,IAAI,CAAC,CAAC;QACZ,CAAC;aAAM,IAAI,OAAO,KAAK,IAAI,QAAQ,EAAE,CAAC;YACpC,IAAI,IAAI,CAAC,CAAC;QACZ,CAAC;aAAM,IAAI,OAAO,KAAK,IAAI,QAAQ,EAAE,CAAC;YACpC,IAAI,IAAI,CAAC,CAAC;QACZ,CAAC;aAAM,IAAI,OAAO,KAAK,IAAI,QAAQ,EAAE,CAAC;YACpC,IAAI,IAAI,KAAK,CAAC,MAAM,CAAC;QACvB,CAAC;aAAM,IAAI,KAAK,YAAY,UAAU,EAAE,CAAC;YACvC,IAAI,IAAI,KAAK,CAAC,UAAU,CAAC;QAC3B,CAAC;IACH,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC"}
|
package/package.json
CHANGED
|
@@ -5,7 +5,7 @@
|
|
|
5
5
|
"publishConfig": {
|
|
6
6
|
"access": "public"
|
|
7
7
|
},
|
|
8
|
-
"version": "
|
|
8
|
+
"version": "1.7.0",
|
|
9
9
|
"main": "dist/index.js",
|
|
10
10
|
"license": "FSL-1.1-Apache-2.0",
|
|
11
11
|
"type": "module",
|
|
@@ -32,10 +32,10 @@
|
|
|
32
32
|
"uuid": "^9.0.1",
|
|
33
33
|
"winston": "^3.13.0",
|
|
34
34
|
"yaml": "^2.3.2",
|
|
35
|
-
"@powersync/lib-services-framework": "0.5.
|
|
35
|
+
"@powersync/lib-services-framework": "0.5.2",
|
|
36
36
|
"@powersync/service-jsonbig": "0.17.10",
|
|
37
|
-
"@powersync/service-rsocket-router": "0.0.
|
|
38
|
-
"@powersync/service-sync-rules": "0.
|
|
37
|
+
"@powersync/service-rsocket-router": "0.0.19",
|
|
38
|
+
"@powersync/service-sync-rules": "0.24.0",
|
|
39
39
|
"@powersync/service-types": "0.8.0"
|
|
40
40
|
},
|
|
41
41
|
"devDependencies": {
|
package/src/api/RouteAPI.ts
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { SqlSyncRules, TablePattern } from '@powersync/service-sync-rules';
|
|
2
2
|
import * as types from '@powersync/service-types';
|
|
3
|
-
import { ParseSyncRulesOptions, SyncRulesBucketStorage } from '../storage/
|
|
3
|
+
import { ParseSyncRulesOptions, SyncRulesBucketStorage } from '../storage/storage-index.js';
|
|
4
4
|
|
|
5
5
|
export interface PatternResult {
|
|
6
6
|
schema: string;
|
|
@@ -54,6 +54,14 @@ export interface RouteAPI {
|
|
|
54
54
|
*/
|
|
55
55
|
getReplicationHead(): Promise<string>;
|
|
56
56
|
|
|
57
|
+
/**
|
|
58
|
+
* Get the current LSN or equivalent replication HEAD position identifier.
|
|
59
|
+
*
|
|
60
|
+
* The position is provided to the callback. After the callback returns,
|
|
61
|
+
* the replication head or a greater one will be streamed on the replication stream.
|
|
62
|
+
*/
|
|
63
|
+
createReplicationHead<T>(callback: ReplicationHeadCallback<T>): Promise<T>;
|
|
64
|
+
|
|
57
65
|
/**
|
|
58
66
|
* @returns The schema for tables inside the connected database. This is typically
|
|
59
67
|
* used to validate sync rules.
|
|
@@ -76,3 +84,5 @@ export interface RouteAPI {
|
|
|
76
84
|
*/
|
|
77
85
|
getParseSyncRulesOptions(): ParseSyncRulesOptions;
|
|
78
86
|
}
|
|
87
|
+
|
|
88
|
+
export type ReplicationHeadCallback<T> = (head: string) => Promise<T>;
|
package/src/api/diagnostics.ts
CHANGED
|
@@ -57,7 +57,7 @@ export async function getSyncRulesStatus(
|
|
|
57
57
|
// This method can run under some situations if no connection is configured yet.
|
|
58
58
|
// It will return a default tag in such a case. This default tag is not module specific.
|
|
59
59
|
const tag = sourceConfig.tag ?? DEFAULT_TAG;
|
|
60
|
-
|
|
60
|
+
const systemStorage = live_status ? bucketStorage.getInstance(sync_rules) : undefined;
|
|
61
61
|
const status = await systemStorage?.getStatus();
|
|
62
62
|
let replication_lag_bytes: number | undefined = undefined;
|
|
63
63
|
|
|
@@ -50,14 +50,13 @@ export function registerCompactAction(program: Command) {
|
|
|
50
50
|
await serviceContext.lifeCycleEngine.start();
|
|
51
51
|
const bucketStorage = serviceContext.storageEngine.activeBucketStorage;
|
|
52
52
|
|
|
53
|
-
const active = await bucketStorage.
|
|
53
|
+
const active = await bucketStorage.getActiveStorage();
|
|
54
54
|
if (active == null) {
|
|
55
55
|
logger.info('No active instance to compact');
|
|
56
56
|
return;
|
|
57
57
|
}
|
|
58
|
-
using p = bucketStorage.getInstance(active);
|
|
59
58
|
logger.info('Performing compaction...');
|
|
60
|
-
await
|
|
59
|
+
await active.compact({ memoryLimitMB: COMPACT_MEMORY_LIMIT_MB, compactBuckets: buckets });
|
|
61
60
|
logger.info('Successfully compacted storage.');
|
|
62
61
|
} catch (e) {
|
|
63
62
|
logger.error(`Failed to compact: ${e.toString()}`);
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { ServiceContextContainer } from '../system/ServiceContext.js';
|
|
2
2
|
import { logger } from '@powersync/lib-services-framework';
|
|
3
3
|
import winston from 'winston';
|
|
4
|
-
import { PersistedSyncRulesContent } from '../storage/
|
|
4
|
+
import { PersistedSyncRulesContent } from '../storage/storage-index.js';
|
|
5
5
|
|
|
6
6
|
export interface TearDownOptions {
|
|
7
7
|
/**
|
|
@@ -199,7 +199,6 @@ export abstract class AbstractReplicator<T extends AbstractReplicationJob = Abst
|
|
|
199
199
|
try {
|
|
200
200
|
await job.stop();
|
|
201
201
|
await this.terminateSyncRules(job.storage);
|
|
202
|
-
job.storage[Symbol.dispose]();
|
|
203
202
|
} catch (e) {
|
|
204
203
|
// This will be retried
|
|
205
204
|
this.logger.warn('Failed to terminate old replication job}', e);
|
|
@@ -210,7 +209,7 @@ export abstract class AbstractReplicator<T extends AbstractReplicationJob = Abst
|
|
|
210
209
|
const stopped = await this.storage.getStoppedSyncRules();
|
|
211
210
|
for (let syncRules of stopped) {
|
|
212
211
|
try {
|
|
213
|
-
|
|
212
|
+
const syncRuleStorage = this.storage.getInstance(syncRules);
|
|
214
213
|
await this.terminateSyncRules(syncRuleStorage);
|
|
215
214
|
} catch (e) {
|
|
216
215
|
this.logger.warn(`Failed clean up replication config for sync rule: ${syncRules.id}`, e);
|
|
@@ -1,7 +1,6 @@
|
|
|
1
1
|
import { logger, router, schema } from '@powersync/lib-services-framework';
|
|
2
2
|
import * as t from 'ts-codec';
|
|
3
3
|
|
|
4
|
-
import * as framework from '@powersync/lib-services-framework';
|
|
5
4
|
import * as util from '../../util/util-index.js';
|
|
6
5
|
import { authUser } from '../auth.js';
|
|
7
6
|
import { routeDefinition } from '../router.js';
|
|
@@ -25,14 +24,15 @@ export const writeCheckpoint = routeDefinition({
|
|
|
25
24
|
// Since we don't use LSNs anymore, the only way to get that is to wait.
|
|
26
25
|
const start = Date.now();
|
|
27
26
|
|
|
28
|
-
const head = await apiHandler.
|
|
27
|
+
const head = await apiHandler.createReplicationHead(async (head) => head);
|
|
29
28
|
|
|
30
29
|
const timeout = 50_000;
|
|
31
30
|
|
|
32
31
|
logger.info(`Waiting for LSN checkpoint: ${head}`);
|
|
33
32
|
while (Date.now() - start < timeout) {
|
|
34
|
-
const
|
|
35
|
-
|
|
33
|
+
const bucketStorage = await service_context.storageEngine.activeBucketStorage.getActiveStorage();
|
|
34
|
+
const cp = await bucketStorage?.getCheckpoint();
|
|
35
|
+
if (cp == null) {
|
|
36
36
|
throw new Error('No sync rules available');
|
|
37
37
|
}
|
|
38
38
|
if (cp.lsn && cp.lsn >= head) {
|
|
@@ -56,25 +56,14 @@ export const writeCheckpoint2 = routeDefinition({
|
|
|
56
56
|
|
|
57
57
|
const apiHandler = service_context.routerEngine!.getAPI();
|
|
58
58
|
|
|
59
|
-
const
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
storageEngine: { activeBucketStorage }
|
|
65
|
-
} = service_context;
|
|
66
|
-
|
|
67
|
-
const activeSyncRules = await activeBucketStorage.getActiveSyncRulesContent();
|
|
68
|
-
if (!activeSyncRules) {
|
|
69
|
-
throw new framework.errors.ValidationError(`Cannot create Write Checkpoint since no sync rules are active.`);
|
|
70
|
-
}
|
|
71
|
-
|
|
72
|
-
using syncBucketStorage = activeBucketStorage.getInstance(activeSyncRules);
|
|
73
|
-
const writeCheckpoint = await syncBucketStorage.createManagedWriteCheckpoint({
|
|
74
|
-
user_id: full_user_id,
|
|
75
|
-
heads: { '1': currentCheckpoint }
|
|
59
|
+
const { replicationHead, writeCheckpoint } = await util.createWriteCheckpoint({
|
|
60
|
+
userId: user_id,
|
|
61
|
+
clientId: payload.params.client_id,
|
|
62
|
+
api: apiHandler,
|
|
63
|
+
storage: service_context.storageEngine.activeBucketStorage
|
|
76
64
|
});
|
|
77
|
-
|
|
65
|
+
|
|
66
|
+
logger.info(`Write checkpoint for ${user_id}/${payload.params.client_id}: ${writeCheckpoint} | ${replicationHead}`);
|
|
78
67
|
|
|
79
68
|
return {
|
|
80
69
|
write_checkpoint: String(writeCheckpoint)
|
|
@@ -49,9 +49,9 @@ export const syncStreamReactive: SocketRouteGenerator = (router) =>
|
|
|
49
49
|
const {
|
|
50
50
|
storageEngine: { activeBucketStorage }
|
|
51
51
|
} = service_context;
|
|
52
|
-
|
|
53
|
-
const
|
|
54
|
-
if (
|
|
52
|
+
|
|
53
|
+
const bucketStorage = await activeBucketStorage.getActiveStorage();
|
|
54
|
+
if (bucketStorage == null) {
|
|
55
55
|
responder.onError(
|
|
56
56
|
new errors.ServiceError({
|
|
57
57
|
status: 500,
|
|
@@ -63,6 +63,8 @@ export const syncStreamReactive: SocketRouteGenerator = (router) =>
|
|
|
63
63
|
return;
|
|
64
64
|
}
|
|
65
65
|
|
|
66
|
+
const syncRules = bucketStorage.getParsedSyncRules(routerEngine!.getAPI().getParseSyncRulesOptions());
|
|
67
|
+
|
|
66
68
|
const removeStopHandler = routerEngine!.addStopHandler(() => {
|
|
67
69
|
controller.abort();
|
|
68
70
|
});
|
|
@@ -71,8 +73,8 @@ export const syncStreamReactive: SocketRouteGenerator = (router) =>
|
|
|
71
73
|
const tracker = new sync.RequestTracker();
|
|
72
74
|
try {
|
|
73
75
|
for await (const data of sync.streamResponse({
|
|
74
|
-
|
|
75
|
-
|
|
76
|
+
bucketStorage: bucketStorage,
|
|
77
|
+
syncRules: syncRules,
|
|
76
78
|
params: {
|
|
77
79
|
...params,
|
|
78
80
|
binary_data: true // always true for web sockets
|
|
@@ -36,15 +36,18 @@ export const syncStreamed = routeDefinition({
|
|
|
36
36
|
const params: util.StreamingSyncRequest = payload.params;
|
|
37
37
|
const syncParams = new RequestParameters(payload.context.token_payload!, payload.params.parameters ?? {});
|
|
38
38
|
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
if (
|
|
39
|
+
const bucketStorage = await storageEngine.activeBucketStorage.getActiveStorage();
|
|
40
|
+
|
|
41
|
+
if (bucketStorage == null) {
|
|
42
42
|
throw new errors.ServiceError({
|
|
43
43
|
status: 500,
|
|
44
44
|
code: ErrorCode.PSYNC_S2302,
|
|
45
45
|
description: 'No sync rules available'
|
|
46
46
|
});
|
|
47
47
|
}
|
|
48
|
+
|
|
49
|
+
const syncRules = bucketStorage.getParsedSyncRules(routerEngine!.getAPI().getParseSyncRulesOptions());
|
|
50
|
+
|
|
48
51
|
const controller = new AbortController();
|
|
49
52
|
const tracker = new sync.RequestTracker();
|
|
50
53
|
try {
|
|
@@ -53,8 +56,8 @@ export const syncStreamed = routeDefinition({
|
|
|
53
56
|
sync.transformToBytesTracked(
|
|
54
57
|
sync.ndjson(
|
|
55
58
|
sync.streamResponse({
|
|
56
|
-
|
|
57
|
-
|
|
59
|
+
bucketStorage,
|
|
60
|
+
syncRules: syncRules,
|
|
58
61
|
params,
|
|
59
62
|
syncParams,
|
|
60
63
|
token: payload.context.token_payload!,
|
package/src/runner/teardown.ts
CHANGED
|
@@ -51,7 +51,7 @@ async function terminateSyncRules(storageFactory: storage.BucketStorageFactory,
|
|
|
51
51
|
|
|
52
52
|
// Mark the sync rules as terminated
|
|
53
53
|
for (let syncRules of combinedSyncRules) {
|
|
54
|
-
|
|
54
|
+
const syncRulesStorage = storageFactory.getInstance(syncRules);
|
|
55
55
|
// The storage will be dropped at the end of the teardown, so we don't need to clear it here
|
|
56
56
|
await syncRulesStorage.terminate({ clearStorage: false });
|
|
57
57
|
}
|