@powersync/service-core 0.0.0-dev-20240718134716 → 0.0.0-dev-20240918082156
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +89 -6
- package/dist/api/RouteAPI.d.ts +68 -0
- package/dist/api/RouteAPI.js +2 -0
- package/dist/api/RouteAPI.js.map +1 -0
- package/dist/api/api-index.d.ts +1 -0
- package/dist/api/api-index.js +1 -0
- package/dist/api/api-index.js.map +1 -1
- package/dist/api/diagnostics.d.ts +4 -4
- package/dist/api/diagnostics.js +11 -65
- package/dist/api/diagnostics.js.map +1 -1
- package/dist/api/schema.d.ts +3 -5
- package/dist/api/schema.js +9 -79
- package/dist/api/schema.js.map +1 -1
- package/dist/auth/KeyStore.d.ts +7 -4
- package/dist/auth/KeyStore.js +1 -1
- package/dist/auth/KeyStore.js.map +1 -1
- package/dist/auth/auth-index.d.ts +0 -1
- package/dist/auth/auth-index.js +0 -1
- package/dist/auth/auth-index.js.map +1 -1
- package/dist/entry/cli-entry.js +4 -2
- package/dist/entry/cli-entry.js.map +1 -1
- package/dist/entry/commands/compact-action.d.ts +2 -0
- package/dist/entry/commands/compact-action.js +52 -0
- package/dist/entry/commands/compact-action.js.map +1 -0
- package/dist/entry/commands/migrate-action.js +4 -5
- package/dist/entry/commands/migrate-action.js.map +1 -1
- package/dist/entry/commands/teardown-action.js +2 -2
- package/dist/entry/commands/teardown-action.js.map +1 -1
- package/dist/entry/entry-index.d.ts +1 -0
- package/dist/entry/entry-index.js +1 -0
- package/dist/entry/entry-index.js.map +1 -1
- package/dist/index.d.ts +4 -2
- package/dist/index.js +4 -2
- package/dist/index.js.map +1 -1
- package/dist/metrics/Metrics.d.ts +6 -5
- package/dist/metrics/Metrics.js +53 -10
- package/dist/metrics/Metrics.js.map +1 -1
- package/dist/migrations/db/migrations/1684951997326-init.d.ts +2 -2
- package/dist/migrations/db/migrations/1684951997326-init.js +4 -2
- package/dist/migrations/db/migrations/1684951997326-init.js.map +1 -1
- package/dist/migrations/db/migrations/1702295701188-sync-rule-state.d.ts +2 -2
- package/dist/migrations/db/migrations/1702295701188-sync-rule-state.js +4 -2
- package/dist/migrations/db/migrations/1702295701188-sync-rule-state.js.map +1 -1
- package/dist/migrations/db/migrations/1711543888062-write-checkpoint-index.d.ts +2 -2
- package/dist/migrations/db/migrations/1711543888062-write-checkpoint-index.js +4 -2
- package/dist/migrations/db/migrations/1711543888062-write-checkpoint-index.js.map +1 -1
- package/dist/migrations/migrations.d.ts +8 -0
- package/dist/migrations/migrations.js +19 -7
- package/dist/migrations/migrations.js.map +1 -1
- package/dist/modules/AbstractModule.d.ts +26 -0
- package/dist/modules/AbstractModule.js +11 -0
- package/dist/modules/AbstractModule.js.map +1 -0
- package/dist/modules/ModuleManager.d.ts +11 -0
- package/dist/modules/ModuleManager.js +32 -0
- package/dist/modules/ModuleManager.js.map +1 -0
- package/dist/modules/modules-index.d.ts +2 -0
- package/dist/modules/modules-index.js +3 -0
- package/dist/modules/modules-index.js.map +1 -0
- package/dist/replication/AbstractReplicationJob.d.ts +38 -0
- package/dist/replication/AbstractReplicationJob.js +51 -0
- package/dist/replication/AbstractReplicationJob.js.map +1 -0
- package/dist/replication/AbstractReplicator.d.ts +53 -0
- package/dist/replication/AbstractReplicator.js +187 -0
- package/dist/replication/AbstractReplicator.js.map +1 -0
- package/dist/replication/ErrorRateLimiter.d.ts +0 -9
- package/dist/replication/ErrorRateLimiter.js +1 -42
- package/dist/replication/ErrorRateLimiter.js.map +1 -1
- package/dist/replication/ReplicationEngine.d.ts +18 -0
- package/dist/replication/ReplicationEngine.js +41 -0
- package/dist/replication/ReplicationEngine.js.map +1 -0
- package/dist/replication/ReplicationModule.d.ts +39 -0
- package/dist/replication/ReplicationModule.js +65 -0
- package/dist/replication/ReplicationModule.js.map +1 -0
- package/dist/replication/replication-index.d.ts +4 -6
- package/dist/replication/replication-index.js +4 -6
- package/dist/replication/replication-index.js.map +1 -1
- package/dist/routes/RouterEngine.d.ts +42 -0
- package/dist/routes/RouterEngine.js +80 -0
- package/dist/routes/RouterEngine.js.map +1 -0
- package/dist/routes/auth.d.ts +2 -2
- package/dist/routes/auth.js +11 -11
- package/dist/routes/auth.js.map +1 -1
- package/dist/routes/configure-fastify.d.ts +737 -0
- package/dist/routes/configure-fastify.js +57 -0
- package/dist/routes/configure-fastify.js.map +1 -0
- package/dist/routes/configure-rsocket.d.ts +13 -0
- package/dist/routes/configure-rsocket.js +47 -0
- package/dist/routes/configure-rsocket.js.map +1 -0
- package/dist/routes/endpoints/admin.d.ts +0 -34
- package/dist/routes/endpoints/admin.js +48 -89
- package/dist/routes/endpoints/admin.js.map +1 -1
- package/dist/routes/endpoints/checkpointing.d.ts +56 -16
- package/dist/routes/endpoints/checkpointing.js +33 -12
- package/dist/routes/endpoints/checkpointing.js.map +1 -1
- package/dist/routes/endpoints/route-endpoints-index.d.ts +0 -1
- package/dist/routes/endpoints/route-endpoints-index.js +0 -1
- package/dist/routes/endpoints/route-endpoints-index.js.map +1 -1
- package/dist/routes/endpoints/socket-route.js +46 -39
- package/dist/routes/endpoints/socket-route.js.map +1 -1
- package/dist/routes/endpoints/sync-rules.d.ts +1 -1
- package/dist/routes/endpoints/sync-rules.js +32 -23
- package/dist/routes/endpoints/sync-rules.js.map +1 -1
- package/dist/routes/endpoints/sync-stream.d.ts +10 -0
- package/dist/routes/endpoints/sync-stream.js +17 -13
- package/dist/routes/endpoints/sync-stream.js.map +1 -1
- package/dist/routes/route-register.d.ts +1 -1
- package/dist/routes/route-register.js +1 -1
- package/dist/routes/route-register.js.map +1 -1
- package/dist/routes/router-socket.d.ts +5 -4
- package/dist/routes/router-socket.js +2 -1
- package/dist/routes/router-socket.js.map +1 -1
- package/dist/routes/router.d.ts +7 -2
- package/dist/routes/router.js.map +1 -1
- package/dist/routes/routes-index.d.ts +3 -0
- package/dist/routes/routes-index.js +3 -0
- package/dist/routes/routes-index.js.map +1 -1
- package/dist/runner/teardown.js +47 -76
- package/dist/runner/teardown.js.map +1 -1
- package/dist/storage/BucketStorage.d.ts +61 -20
- package/dist/storage/BucketStorage.js +0 -10
- package/dist/storage/BucketStorage.js.map +1 -1
- package/dist/storage/MongoBucketStorage.d.ts +4 -4
- package/dist/storage/MongoBucketStorage.js +19 -24
- package/dist/storage/MongoBucketStorage.js.map +1 -1
- package/dist/storage/SourceEntity.d.ts +20 -0
- package/dist/storage/SourceEntity.js +2 -0
- package/dist/storage/SourceEntity.js.map +1 -0
- package/dist/storage/SourceTable.d.ts +4 -5
- package/dist/storage/SourceTable.js +3 -4
- package/dist/storage/SourceTable.js.map +1 -1
- package/dist/storage/StorageEngine.d.ts +24 -0
- package/dist/storage/StorageEngine.js +43 -0
- package/dist/storage/StorageEngine.js.map +1 -0
- package/dist/storage/StorageProvider.d.ts +21 -0
- package/dist/storage/StorageProvider.js +2 -0
- package/dist/storage/StorageProvider.js.map +1 -0
- package/dist/storage/mongo/MongoBucketBatch.d.ts +1 -1
- package/dist/storage/mongo/MongoBucketBatch.js +6 -7
- package/dist/storage/mongo/MongoBucketBatch.js.map +1 -1
- package/dist/storage/mongo/MongoCompactor.d.ts +40 -0
- package/dist/storage/mongo/MongoCompactor.js +293 -0
- package/dist/storage/mongo/MongoCompactor.js.map +1 -0
- package/dist/storage/mongo/MongoPersistedSyncRulesContent.d.ts +2 -2
- package/dist/storage/mongo/MongoPersistedSyncRulesContent.js +2 -2
- package/dist/storage/mongo/MongoPersistedSyncRulesContent.js.map +1 -1
- package/dist/storage/mongo/MongoStorageProvider.d.ts +5 -0
- package/dist/storage/mongo/MongoStorageProvider.js +26 -0
- package/dist/storage/mongo/MongoStorageProvider.js.map +1 -0
- package/dist/storage/mongo/MongoSyncBucketStorage.d.ts +9 -7
- package/dist/storage/mongo/MongoSyncBucketStorage.js +43 -28
- package/dist/storage/mongo/MongoSyncBucketStorage.js.map +1 -1
- package/dist/storage/mongo/MongoSyncRulesLock.js +1 -1
- package/dist/storage/mongo/MongoSyncRulesLock.js.map +1 -1
- package/dist/storage/mongo/OperationBatch.d.ts +7 -3
- package/dist/storage/mongo/OperationBatch.js +16 -7
- package/dist/storage/mongo/OperationBatch.js.map +1 -1
- package/dist/storage/mongo/PersistedBatch.d.ts +3 -3
- package/dist/storage/mongo/PersistedBatch.js +2 -2
- package/dist/storage/mongo/PersistedBatch.js.map +1 -1
- package/dist/storage/mongo/models.d.ts +17 -7
- package/dist/storage/mongo/models.js.map +1 -1
- package/dist/storage/mongo/util.d.ts +14 -0
- package/dist/storage/mongo/util.js +70 -0
- package/dist/storage/mongo/util.js.map +1 -1
- package/dist/storage/storage-index.d.ts +5 -2
- package/dist/storage/storage-index.js +5 -2
- package/dist/storage/storage-index.js.map +1 -1
- package/dist/sync/RequestTracker.js +2 -3
- package/dist/sync/RequestTracker.js.map +1 -1
- package/dist/sync/sync-index.d.ts +1 -0
- package/dist/sync/sync-index.js +1 -0
- package/dist/sync/sync-index.js.map +1 -1
- package/dist/sync/sync.d.ts +2 -1
- package/dist/sync/sync.js +56 -17
- package/dist/sync/sync.js.map +1 -1
- package/dist/system/ServiceContext.d.ts +37 -0
- package/dist/system/ServiceContext.js +48 -0
- package/dist/system/ServiceContext.js.map +1 -0
- package/dist/system/system-index.d.ts +1 -1
- package/dist/system/system-index.js +1 -1
- package/dist/system/system-index.js.map +1 -1
- package/dist/util/config/collectors/config-collector.d.ts +12 -0
- package/dist/util/config/collectors/config-collector.js +43 -0
- package/dist/util/config/collectors/config-collector.js.map +1 -1
- package/dist/util/config/compound-config-collector.d.ts +10 -29
- package/dist/util/config/compound-config-collector.js +28 -84
- package/dist/util/config/compound-config-collector.js.map +1 -1
- package/dist/util/config/sync-rules/sync-rules-provider.d.ts +9 -0
- package/dist/util/config/sync-rules/sync-rules-provider.js +15 -0
- package/dist/util/config/sync-rules/sync-rules-provider.js.map +1 -0
- package/dist/util/config/types.d.ts +6 -4
- package/dist/util/config/types.js.map +1 -1
- package/dist/util/config.d.ts +3 -4
- package/dist/util/config.js +5 -20
- package/dist/util/config.js.map +1 -1
- package/dist/util/protocol-types.d.ts +4 -0
- package/dist/util/protocol-types.js +5 -1
- package/dist/util/protocol-types.js.map +1 -1
- package/dist/util/util-index.d.ts +3 -6
- package/dist/util/util-index.js +3 -6
- package/dist/util/util-index.js.map +1 -1
- package/dist/util/utils.d.ts +10 -6
- package/dist/util/utils.js +45 -25
- package/dist/util/utils.js.map +1 -1
- package/package.json +7 -7
- package/src/api/RouteAPI.ts +78 -0
- package/src/api/api-index.ts +1 -0
- package/src/api/diagnostics.ts +16 -71
- package/src/api/schema.ts +13 -89
- package/src/auth/KeyStore.ts +9 -6
- package/src/auth/auth-index.ts +0 -1
- package/src/entry/cli-entry.ts +4 -2
- package/src/entry/commands/compact-action.ts +57 -0
- package/src/entry/commands/migrate-action.ts +5 -8
- package/src/entry/commands/teardown-action.ts +2 -2
- package/src/entry/entry-index.ts +1 -0
- package/src/index.ts +5 -2
- package/src/metrics/Metrics.ts +70 -15
- package/src/migrations/db/migrations/1684951997326-init.ts +9 -4
- package/src/migrations/db/migrations/1702295701188-sync-rule-state.ts +7 -4
- package/src/migrations/db/migrations/1711543888062-write-checkpoint-index.ts +6 -4
- package/src/migrations/migrations.ts +24 -8
- package/src/modules/AbstractModule.ts +37 -0
- package/src/modules/ModuleManager.ts +34 -0
- package/src/modules/modules-index.ts +2 -0
- package/src/replication/AbstractReplicationJob.ts +79 -0
- package/src/replication/AbstractReplicator.ts +227 -0
- package/src/replication/ErrorRateLimiter.ts +0 -44
- package/src/replication/ReplicationEngine.ts +43 -0
- package/src/replication/ReplicationModule.ts +101 -0
- package/src/replication/replication-index.ts +4 -6
- package/src/routes/RouterEngine.ts +120 -0
- package/src/routes/auth.ts +21 -12
- package/src/routes/configure-fastify.ts +101 -0
- package/src/routes/configure-rsocket.ts +60 -0
- package/src/routes/endpoints/admin.ts +74 -100
- package/src/routes/endpoints/checkpointing.ts +46 -12
- package/src/routes/endpoints/route-endpoints-index.ts +0 -1
- package/src/routes/endpoints/socket-route.ts +50 -42
- package/src/routes/endpoints/sync-rules.ts +41 -25
- package/src/routes/endpoints/sync-stream.ts +17 -13
- package/src/routes/route-register.ts +2 -2
- package/src/routes/router-socket.ts +6 -5
- package/src/routes/router.ts +7 -2
- package/src/routes/routes-index.ts +3 -0
- package/src/runner/teardown.ts +50 -88
- package/src/storage/BucketStorage.ts +74 -26
- package/src/storage/MongoBucketStorage.ts +23 -26
- package/src/storage/SourceEntity.ts +22 -0
- package/src/storage/SourceTable.ts +4 -6
- package/src/storage/StorageEngine.ts +55 -0
- package/src/storage/StorageProvider.ts +27 -0
- package/src/storage/mongo/MongoBucketBatch.ts +8 -8
- package/src/storage/mongo/MongoCompactor.ts +372 -0
- package/src/storage/mongo/MongoPersistedSyncRulesContent.ts +3 -3
- package/src/storage/mongo/MongoStorageProvider.ts +31 -0
- package/src/storage/mongo/MongoSyncBucketStorage.ts +64 -34
- package/src/storage/mongo/MongoSyncRulesLock.ts +1 -1
- package/src/storage/mongo/OperationBatch.ts +18 -11
- package/src/storage/mongo/PersistedBatch.ts +6 -5
- package/src/storage/mongo/models.ts +17 -7
- package/src/storage/mongo/util.ts +71 -1
- package/src/storage/storage-index.ts +5 -2
- package/src/sync/RequestTracker.ts +3 -3
- package/src/sync/sync-index.ts +1 -0
- package/src/sync/sync.ts +66 -17
- package/src/system/ServiceContext.ts +68 -0
- package/src/system/system-index.ts +1 -1
- package/src/util/config/collectors/config-collector.ts +48 -0
- package/src/util/config/compound-config-collector.ts +45 -110
- package/src/util/config/sync-rules/sync-rules-provider.ts +18 -0
- package/src/util/config/types.ts +6 -5
- package/src/util/config.ts +6 -23
- package/src/util/protocol-types.ts +6 -1
- package/src/util/util-index.ts +3 -6
- package/src/util/utils.ts +55 -39
- package/test/src/__snapshots__/sync.test.ts.snap +90 -5
- package/test/src/auth.test.ts +7 -7
- package/test/src/broadcast_iterable.test.ts +1 -1
- package/test/src/bucket_validation.test.ts +142 -0
- package/test/src/bucket_validation.ts +116 -0
- package/test/src/checksum_cache.test.ts +3 -3
- package/test/src/compacting.test.ts +216 -0
- package/test/src/data_storage.test.ts +275 -204
- package/test/src/env.ts +1 -3
- package/test/src/merge_iterable.test.ts +1 -6
- package/test/src/setup.ts +1 -1
- package/test/src/stream_utils.ts +42 -0
- package/test/src/sync.test.ts +209 -48
- package/test/src/util.ts +110 -55
- package/test/tsconfig.json +1 -1
- package/tsconfig.tsbuildinfo +1 -1
- package/dist/auth/SupabaseKeyCollector.d.ts +0 -22
- package/dist/auth/SupabaseKeyCollector.js +0 -61
- package/dist/auth/SupabaseKeyCollector.js.map +0 -1
- package/dist/replication/PgRelation.d.ts +0 -16
- package/dist/replication/PgRelation.js +0 -26
- package/dist/replication/PgRelation.js.map +0 -1
- package/dist/replication/WalConnection.d.ts +0 -34
- package/dist/replication/WalConnection.js +0 -190
- package/dist/replication/WalConnection.js.map +0 -1
- package/dist/replication/WalStream.d.ts +0 -57
- package/dist/replication/WalStream.js +0 -517
- package/dist/replication/WalStream.js.map +0 -1
- package/dist/replication/WalStreamManager.d.ts +0 -30
- package/dist/replication/WalStreamManager.js +0 -198
- package/dist/replication/WalStreamManager.js.map +0 -1
- package/dist/replication/WalStreamRunner.d.ts +0 -38
- package/dist/replication/WalStreamRunner.js +0 -155
- package/dist/replication/WalStreamRunner.js.map +0 -1
- package/dist/replication/util.d.ts +0 -9
- package/dist/replication/util.js +0 -62
- package/dist/replication/util.js.map +0 -1
- package/dist/routes/endpoints/dev.d.ts +0 -312
- package/dist/routes/endpoints/dev.js +0 -172
- package/dist/routes/endpoints/dev.js.map +0 -1
- package/dist/system/CorePowerSyncSystem.d.ts +0 -23
- package/dist/system/CorePowerSyncSystem.js +0 -52
- package/dist/system/CorePowerSyncSystem.js.map +0 -1
- package/dist/util/PgManager.d.ts +0 -24
- package/dist/util/PgManager.js +0 -55
- package/dist/util/PgManager.js.map +0 -1
- package/dist/util/migration_lib.d.ts +0 -11
- package/dist/util/migration_lib.js +0 -64
- package/dist/util/migration_lib.js.map +0 -1
- package/dist/util/pgwire_utils.d.ts +0 -24
- package/dist/util/pgwire_utils.js +0 -117
- package/dist/util/pgwire_utils.js.map +0 -1
- package/dist/util/populate_test_data.d.ts +0 -8
- package/dist/util/populate_test_data.js +0 -65
- package/dist/util/populate_test_data.js.map +0 -1
- package/src/auth/SupabaseKeyCollector.ts +0 -67
- package/src/replication/PgRelation.ts +0 -42
- package/src/replication/WalConnection.ts +0 -227
- package/src/replication/WalStream.ts +0 -628
- package/src/replication/WalStreamManager.ts +0 -213
- package/src/replication/WalStreamRunner.ts +0 -180
- package/src/replication/util.ts +0 -76
- package/src/routes/endpoints/dev.ts +0 -199
- package/src/system/CorePowerSyncSystem.ts +0 -64
- package/src/util/PgManager.ts +0 -64
- package/src/util/migration_lib.ts +0 -79
- package/src/util/pgwire_utils.ts +0 -139
- package/src/util/populate_test_data.ts +0 -78
- package/test/src/__snapshots__/pg_test.test.ts.snap +0 -256
- package/test/src/large_batch.test.ts +0 -194
- package/test/src/pg_test.test.ts +0 -450
- package/test/src/schema_changes.test.ts +0 -545
- package/test/src/slow_tests.test.ts +0 -296
- package/test/src/validation.test.ts +0 -63
- package/test/src/wal_stream.test.ts +0 -314
- package/test/src/wal_stream_utils.ts +0 -147
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"checkpointing.js","sourceRoot":"","sources":["../../../src/routes/endpoints/checkpointing.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,CAAC,MAAM,UAAU,CAAC;AAC9B,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,mCAAmC,CAAC;
|
|
1
|
+
{"version":3,"file":"checkpointing.js","sourceRoot":"","sources":["../../../src/routes/endpoints/checkpointing.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,CAAC,MAAM,UAAU,CAAC;AAC9B,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,mCAAmC,CAAC;AAE3E,OAAO,KAAK,IAAI,MAAM,0BAA0B,CAAC;AACjD,OAAO,EAAE,QAAQ,EAAE,MAAM,YAAY,CAAC;AACtC,OAAO,EAAE,eAAe,EAAE,MAAM,cAAc,CAAC;AAE/C,MAAM,sBAAsB,GAAG,CAAC,CAAC,MAAM,CAAC;IACtC,SAAS,EAAE,CAAC,CAAC,MAAM,CAAC,QAAQ,EAAE;CAC/B,CAAC,CAAC;AAEH,MAAM,CAAC,MAAM,eAAe,GAAG,eAAe,CAAC;IAC7C,IAAI,EAAE,wBAAwB;IAC9B,MAAM,EAAE,MAAM,CAAC,UAAU,CAAC,GAAG;IAC7B,SAAS,EAAE,QAAQ;IACnB,SAAS,EAAE,MAAM,CAAC,sBAAsB,CAAC,sBAAsB,EAAE,EAAE,eAAe,EAAE,IAAI,EAAE,CAAC;IAC3F,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,EAAE;QACzB,MAAM,EACJ,OAAO,EAAE,EAAE,eAAe,EAAE,EAC7B,GAAG,OAAO,CAAC;QACZ,MAAM,UAAU,GAAG,eAAe,CAAC,YAAa,CAAC,MAAM,EAAE,CAAC;QAE1D,gDAAgD;QAChD,wEAAwE;QACxE,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAEzB,MAAM,IAAI,GAAG,MAAM,UAAU,CAAC,kBAAkB,EAAE,CAAC;QAEnD,MAAM,OAAO,GAAG,KAAM,CAAC;QAEvB,MAAM,CAAC,IAAI,CAAC,+BAA+B,IAAI,EAAE,CAAC,CAAC;QACnD,OAAO,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK,GAAG,OAAO,EAAE;YACnC,MAAM,EAAE,GAAG,MAAM,eAAe,CAAC,aAAa,CAAC,mBAAmB,CAAC,mBAAmB,EAAE,CAAC;YACzF,IAAI,CAAC,EAAE,CAAC,YAAY,EAAE,EAAE;gBACtB,MAAM,IAAI,KAAK,CAAC,yBAAyB,CAAC,CAAC;aAC5C;YACD,IAAI,EAAE,CAAC,GAAG,IAAI,EAAE,CAAC,GAAG,IAAI,IAAI,EAAE;gBAC5B,MAAM,CAAC,IAAI,CAAC,yBAAyB,IAAI,MAAM,EAAE,CAAC,UAAU,EAAE,CAAC,CAAC;gBAChE,OAAO,EAAE,UAAU,EAAE,EAAE,CAAC,UAAU,EAAE,CAAC;aACtC;YAED,MAAM,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,CAAC;SACzD;QACD,MAAM,IAAI,KAAK,CAAC,sCAAsC,CAAC,CAAC;IAC1D,CAAC;CACF,CAAC,CAAC;AAEH,MAAM,CAAC,MAAM,gBAAgB,GAAG,eAAe,CAAC;IAC9C,IAAI,EAAE,yBAAyB;IAC/B,MAAM,EAAE,MAAM,CAAC,UAAU,CAAC,GAAG;IAC7B,SAAS,EAAE,QAAQ;IACnB,SAAS,EAAE,MAAM,CAAC,sBAAsB,CAAC,sBAAsB,EAAE,EAAE,eAAe,EAAE,IAAI,EAAE,CAAC;IAC3F,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,EAAE;QACzB,MAAM,EAAE,OAAO,EAAE,eAAe,EAAE,GAAG,OAAO,CAAC,OAAO,CAAC;QAErD,MAAM,UAAU,GAAG,eAAe,CAAC,YAAa,CAAC,MAAM,EAAE,CAAC;QAE1D,MAAM,SAAS,GAAG,OAAO,CAAC,MAAM,CAAC,SAAS,CAAC;QAC3C,MAAM,YAAY,GAAG,IAAI,CAAC,gBAAgB,CAAC,OAAO,EAAE,SAAS,CAAC,CAAC;QAE/D,MAAM,iBAAiB,GAAG,MAAM,UAAU,CAAC,kBAAkB,EAAE,CAAC;QAChE,MAAM,EACJ,aAAa,EAAE,EAAE,mBAAmB,EAAE,EACvC,GAAG,eAAe,CAAC;QAEpB,MAAM,eAAe,GAAG,MAAM,mBAAmB,CAAC,qBAAqB,CAAC,YAAY,EAAE,EAAE,GAAG,EAAE,iBAAiB,EAAE,CAAC,CAAC;QAClH,MAAM,CAAC,IAAI,CAAC,uBAAuB,IAAI,CAAC,SAAS,CAAC,EAAE,iBAAiB,EAAE,EAAE,EAAE,MAAM,CAAC,YAAY,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC;QAEtG,OAAO;YACL,gBAAgB,EAAE,MAAM,CAAC,eAAe,CAAC;SAC1C,CAAC;IACJ,CAAC;CACF,CAAC,CAAC;AAEH,MAAM,CAAC,MAAM,iBAAiB,GAAG,CAAC,eAAe,EAAE,gBAAgB,CAAC,CAAC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"route-endpoints-index.js","sourceRoot":"","sources":["../../../src/routes/endpoints/route-endpoints-index.ts"],"names":[],"mappings":"AAAA,cAAc,YAAY,CAAC;AAC3B,cAAc,oBAAoB,CAAC;AACnC,cAAc,
|
|
1
|
+
{"version":3,"file":"route-endpoints-index.js","sourceRoot":"","sources":["../../../src/routes/endpoints/route-endpoints-index.ts"],"names":[],"mappings":"AAAA,cAAc,YAAY,CAAC;AAC3B,cAAc,oBAAoB,CAAC;AACnC,cAAc,mBAAmB,CAAC;AAClC,cAAc,iBAAiB,CAAC;AAChC,cAAc,kBAAkB,CAAC"}
|
|
@@ -1,22 +1,31 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { errors, logger, schema } from '@powersync/lib-services-framework';
|
|
2
2
|
import { RequestParameters } from '@powersync/service-sync-rules';
|
|
3
3
|
import { serialize } from 'bson';
|
|
4
4
|
import { Metrics } from '../../metrics/Metrics.js';
|
|
5
|
-
import
|
|
5
|
+
import * as sync from '../../sync/sync-index.js';
|
|
6
6
|
import * as util from '../../util/util-index.js';
|
|
7
7
|
import { SyncRoutes } from './sync-stream.js';
|
|
8
|
-
import { RequestTracker } from '../../sync/RequestTracker.js';
|
|
9
8
|
export const syncStreamReactive = (router) => router.reactiveStream(SyncRoutes.STREAM, {
|
|
10
|
-
authorize: ({ context }) => {
|
|
11
|
-
return {
|
|
12
|
-
authorized: !!context.token_payload,
|
|
13
|
-
errors: ['Authentication required'].concat(context.token_errors ?? [])
|
|
14
|
-
};
|
|
15
|
-
},
|
|
16
9
|
validator: schema.createTsCodecValidator(util.StreamingSyncRequest, { allowAdditional: true }),
|
|
17
|
-
handler: async ({ context, params, responder, observer, initialN }) => {
|
|
18
|
-
const {
|
|
19
|
-
|
|
10
|
+
handler: async ({ context, params, responder, observer, initialN, signal: upstreamSignal }) => {
|
|
11
|
+
const { service_context } = context;
|
|
12
|
+
const { routerEngine } = service_context;
|
|
13
|
+
// Create our own controller that we can abort directly
|
|
14
|
+
const controller = new AbortController();
|
|
15
|
+
upstreamSignal.addEventListener('abort', () => {
|
|
16
|
+
controller.abort();
|
|
17
|
+
});
|
|
18
|
+
if (upstreamSignal.aborted) {
|
|
19
|
+
controller.abort();
|
|
20
|
+
}
|
|
21
|
+
const signal = controller.signal;
|
|
22
|
+
let requestedN = initialN;
|
|
23
|
+
const disposer = observer.registerListener({
|
|
24
|
+
request(n) {
|
|
25
|
+
requestedN += n;
|
|
26
|
+
}
|
|
27
|
+
});
|
|
28
|
+
if (routerEngine.closed) {
|
|
20
29
|
responder.onError(new errors.JourneyError({
|
|
21
30
|
status: 503,
|
|
22
31
|
code: 'SERVICE_UNAVAILABLE',
|
|
@@ -25,11 +34,10 @@ export const syncStreamReactive = (router) => router.reactiveStream(SyncRoutes.S
|
|
|
25
34
|
responder.onComplete();
|
|
26
35
|
return;
|
|
27
36
|
}
|
|
28
|
-
const controller = new AbortController();
|
|
29
37
|
const syncParams = new RequestParameters(context.token_payload, params.parameters ?? {});
|
|
30
|
-
const
|
|
38
|
+
const { storageEngine: { activeBucketStorage } } = service_context;
|
|
31
39
|
// Sanity check before we start the stream
|
|
32
|
-
const cp = await
|
|
40
|
+
const cp = await activeBucketStorage.getActiveCheckpoint();
|
|
33
41
|
if (!cp.hasSyncRules()) {
|
|
34
42
|
responder.onError(new errors.JourneyError({
|
|
35
43
|
status: 500,
|
|
@@ -39,24 +47,15 @@ export const syncStreamReactive = (router) => router.reactiveStream(SyncRoutes.S
|
|
|
39
47
|
responder.onComplete();
|
|
40
48
|
return;
|
|
41
49
|
}
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
request(n) {
|
|
45
|
-
requestedN += n;
|
|
46
|
-
},
|
|
47
|
-
cancel: () => {
|
|
48
|
-
controller.abort();
|
|
49
|
-
}
|
|
50
|
+
const removeStopHandler = routerEngine.addStopHandler(() => {
|
|
51
|
+
controller.abort();
|
|
50
52
|
});
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
});
|
|
54
|
-
const metrics = container.getImplementation(Metrics);
|
|
55
|
-
metrics.concurrent_connections.add(1);
|
|
56
|
-
const tracker = new RequestTracker();
|
|
53
|
+
Metrics.getInstance().concurrent_connections.add(1);
|
|
54
|
+
const tracker = new sync.RequestTracker();
|
|
57
55
|
try {
|
|
58
|
-
for await (const data of streamResponse({
|
|
59
|
-
storage,
|
|
56
|
+
for await (const data of sync.streamResponse({
|
|
57
|
+
storage: activeBucketStorage,
|
|
58
|
+
parseOptions: routerEngine.getAPI().getParseSyncRulesOptions(),
|
|
60
59
|
params: {
|
|
61
60
|
...params,
|
|
62
61
|
binary_data: true // always true for web sockets
|
|
@@ -68,8 +67,11 @@ export const syncStreamReactive = (router) => router.reactiveStream(SyncRoutes.S
|
|
|
68
67
|
keep_alive: false
|
|
69
68
|
},
|
|
70
69
|
tracker,
|
|
71
|
-
signal
|
|
70
|
+
signal
|
|
72
71
|
})) {
|
|
72
|
+
if (signal.aborted) {
|
|
73
|
+
break;
|
|
74
|
+
}
|
|
73
75
|
if (data == null) {
|
|
74
76
|
// Empty value just to flush iterator memory
|
|
75
77
|
continue;
|
|
@@ -85,7 +87,7 @@ export const syncStreamReactive = (router) => router.reactiveStream(SyncRoutes.S
|
|
|
85
87
|
requestedN--;
|
|
86
88
|
tracker.addDataSynced(serialized.length);
|
|
87
89
|
}
|
|
88
|
-
if (requestedN <= 0) {
|
|
90
|
+
if (requestedN <= 0 && !signal.aborted) {
|
|
89
91
|
await new Promise((resolve) => {
|
|
90
92
|
const l = observer.registerListener({
|
|
91
93
|
request() {
|
|
@@ -93,14 +95,17 @@ export const syncStreamReactive = (router) => router.reactiveStream(SyncRoutes.S
|
|
|
93
95
|
// Management of updating the total requested items is done above
|
|
94
96
|
resolve();
|
|
95
97
|
l();
|
|
98
|
+
signal.removeEventListener('abort', onAbort);
|
|
96
99
|
}
|
|
97
|
-
},
|
|
98
|
-
cancel: () => {
|
|
99
|
-
// Don't wait here if the request is cancelled
|
|
100
|
-
resolve();
|
|
101
|
-
l();
|
|
102
100
|
}
|
|
103
101
|
});
|
|
102
|
+
const onAbort = () => {
|
|
103
|
+
// Don't wait here if the request is cancelled
|
|
104
|
+
resolve();
|
|
105
|
+
l();
|
|
106
|
+
signal.removeEventListener('abort', onAbort);
|
|
107
|
+
};
|
|
108
|
+
signal.addEventListener('abort', onAbort);
|
|
104
109
|
});
|
|
105
110
|
}
|
|
106
111
|
}
|
|
@@ -118,10 +123,12 @@ export const syncStreamReactive = (router) => router.reactiveStream(SyncRoutes.S
|
|
|
118
123
|
disposer();
|
|
119
124
|
logger.info(`Sync stream complete`, {
|
|
120
125
|
user_id: syncParams.user_id,
|
|
126
|
+
client_id: params.client_id,
|
|
127
|
+
user_agent: context.user_agent,
|
|
121
128
|
operations_synced: tracker.operationsSynced,
|
|
122
129
|
data_synced_bytes: tracker.dataSyncedBytes
|
|
123
130
|
});
|
|
124
|
-
|
|
131
|
+
Metrics.getInstance().concurrent_connections.add(-1);
|
|
125
132
|
}
|
|
126
133
|
}
|
|
127
134
|
});
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"socket-route.js","sourceRoot":"","sources":["../../../src/routes/endpoints/socket-route.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,
|
|
1
|
+
{"version":3,"file":"socket-route.js","sourceRoot":"","sources":["../../../src/routes/endpoints/socket-route.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,mCAAmC,CAAC;AAC3E,OAAO,EAAE,iBAAiB,EAAE,MAAM,+BAA+B,CAAC;AAClE,OAAO,EAAE,SAAS,EAAE,MAAM,MAAM,CAAC;AAEjC,OAAO,EAAE,OAAO,EAAE,MAAM,0BAA0B,CAAC;AACnD,OAAO,KAAK,IAAI,MAAM,0BAA0B,CAAC;AACjD,OAAO,KAAK,IAAI,MAAM,0BAA0B,CAAC;AAEjD,OAAO,EAAE,UAAU,EAAE,MAAM,kBAAkB,CAAC;AAE9C,MAAM,CAAC,MAAM,kBAAkB,GAAyB,CAAC,MAAM,EAAE,EAAE,CACjE,MAAM,CAAC,cAAc,CAAiC,UAAU,CAAC,MAAM,EAAE;IACvE,SAAS,EAAE,MAAM,CAAC,sBAAsB,CAAC,IAAI,CAAC,oBAAoB,EAAE,EAAE,eAAe,EAAE,IAAI,EAAE,CAAC;IAC9F,OAAO,EAAE,KAAK,EAAE,EAAE,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,QAAQ,EAAE,QAAQ,EAAE,MAAM,EAAE,cAAc,EAAE,EAAE,EAAE;QAC5F,MAAM,EAAE,eAAe,EAAE,GAAG,OAAO,CAAC;QACpC,MAAM,EAAE,YAAY,EAAE,GAAG,eAAe,CAAC;QAEzC,uDAAuD;QACvD,MAAM,UAAU,GAAG,IAAI,eAAe,EAAE,CAAC;QACzC,cAAc,CAAC,gBAAgB,CAAC,OAAO,EAAE,GAAG,EAAE;YAC5C,UAAU,CAAC,KAAK,EAAE,CAAC;QACrB,CAAC,CAAC,CAAC;QACH,IAAI,cAAc,CAAC,OAAO,EAAE;YAC1B,UAAU,CAAC,KAAK,EAAE,CAAC;SACpB;QACD,MAAM,MAAM,GAAG,UAAU,CAAC,MAAM,CAAC;QAEjC,IAAI,UAAU,GAAG,QAAQ,CAAC;QAC1B,MAAM,QAAQ,GAAG,QAAQ,CAAC,gBAAgB,CAAC;YACzC,OAAO,CAAC,CAAC;gBACP,UAAU,IAAI,CAAC,CAAC;YAClB,CAAC;SACF,CAAC,CAAC;QAEH,IAAI,YAAa,CAAC,MAAM,EAAE;YACxB,SAAS,CAAC,OAAO,CACf,IAAI,MAAM,CAAC,YAAY,CAAC;gBACtB,MAAM,EAAE,GAAG;gBACX,IAAI,EAAE,qBAAqB;gBAC3B,WAAW,EAAE,iCAAiC;aAC/C,CAAC,CACH,CAAC;YACF,SAAS,CAAC,UAAU,EAAE,CAAC;YACvB,OAAO;SACR;QAED,MAAM,UAAU,GAAG,IAAI,iBAAiB,CAAC,OAAO,CAAC,aAAc,EAAE,MAAM,CAAC,UAAU,IAAI,EAAE,CAAC,CAAC;QAE1F,MAAM,EACJ,aAAa,EAAE,EAAE,mBAAmB,EAAE,EACvC,GAAG,eAAe,CAAC;QACpB,0CAA0C;QAC1C,MAAM,EAAE,GAAG,MAAM,mBAAmB,CAAC,mBAAmB,EAAE,CAAC;QAC3D,IAAI,CAAC,EAAE,CAAC,YAAY,EAAE,EAAE;YACtB,SAAS,CAAC,OAAO,CACf,IAAI,MAAM,CAAC,YAAY,CAAC;gBACtB,MAAM,EAAE,GAAG;gBACX,IAAI,EAAE,eAAe;gBACrB,WAAW,EAAE,yBAAyB;aACvC,CAAC,CACH,CAAC;YACF,SAAS,CAAC,UAAU,EAAE,CAAC;YACvB,OAAO;SACR;QAED,MAAM,iBAAiB,GAAG,YAAa,CAAC,cAAc,CAAC,GAAG,EAAE;YAC1D,UAAU,CAAC,KAAK,EAAE,CAAC;QACrB,CAAC,CAAC,CAAC;QAEH,OAAO,CAAC,WAAW,EAAE,CAAC,sBAAsB,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;QACpD,MAAM,OAAO,GAAG,IAAI,IAAI,CAAC,cAAc,EAAE,CAAC;QAC1C,IAAI;YACF,IAAI,KAAK,EAAE,MAAM,IAAI,IAAI,IAAI,CAAC,cAAc,CAAC;gBAC3C,OAAO,EAAE,mBAAmB;gBAC5B,YAAY,EAAE,YAAa,CAAC,MAAM,EAAE,CAAC,wBAAwB,EAAE;gBAC/D,MAAM,EAAE;oBACN,GAAG,MAAM;oBACT,WAAW,EAAE,IAAI,CAAC,8BAA8B;iBACjD;gBACD,UAAU;gBACV,KAAK,EAAE,OAAQ,CAAC,aAAc;gBAC9B,kBAAkB,EAAE;oBAClB,8CAA8C;oBAC9C,UAAU,EAAE,KAAK;iBAClB;gBACD,OAAO;gBACP,MAAM;aACP,CAAC,EAAE;gBACF,IAAI,MAAM,CAAC,OAAO,EAAE;oBAClB,MAAM;iBACP;gBACD,IAAI,IAAI,IAAI,IAAI,EAAE;oBAChB,4CAA4C;oBAC5C,SAAS;iBACV;qBAAM,IAAI,OAAO,IAAI,IAAI,QAAQ,EAAE;oBAClC,2CAA2C;oBAC3C,MAAM,IAAI,KAAK,CAAC,2BAA2B,IAAI,EAAE,CAAC,CAAC;iBACpD;gBAED;oBACE,+CAA+C;oBAC/C,MAAM,UAAU,GAAG,SAAS,CAAC,IAAI,CAAW,CAAC;oBAC7C,SAAS,CAAC,MAAM,CAAC,EAAE,IAAI,EAAE,UAAU,EAAE,EAAE,KAAK,CAAC,CAAC;oBAC9C,UAAU,EAAE,CAAC;oBACb,OAAO,CAAC,aAAa,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC;iBAC1C;gBAED,IAAI,UAAU,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE;oBACtC,MAAM,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,EAAE;wBAClC,MAAM,CAAC,GAAG,QAAQ,CAAC,gBAAgB,CAAC;4BAClC,OAAO;gCACL,IAAI,UAAU,GAAG,CAAC,EAAE;oCAClB,iEAAiE;oCACjE,OAAO,EAAE,CAAC;oCACV,CAAC,EAAE,CAAC;oCACJ,MAAM,CAAC,mBAAmB,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;iCAC9C;4BACH,CAAC;yBACF,CAAC,CAAC;wBACH,MAAM,OAAO,GAAG,GAAG,EAAE;4BACnB,8CAA8C;4BAC9C,OAAO,EAAE,CAAC;4BACV,CAAC,EAAE,CAAC;4BACJ,MAAM,CAAC,mBAAmB,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;wBAC/C,CAAC,CAAC;wBACF,MAAM,CAAC,gBAAgB,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;oBAC5C,CAAC,CAAC,CAAC;iBACJ;aACF;SACF;QAAC,OAAO,EAAE,EAAE;YACX,kDAAkD;YAClD,4CAA4C;YAC5C,MAAM,KAAK,GAAG,IAAI,MAAM,CAAC,mBAAmB,CAAC,EAAE,CAAC,CAAC;YACjD,MAAM,CAAC,KAAK,CAAC,mBAAmB,EAAE,KAAK,CAAC,CAAC;YACzC,SAAS,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;SAC1B;gBAAS;YACR,SAAS,CAAC,UAAU,EAAE,CAAC;YACvB,iBAAiB,EAAE,CAAC;YACpB,QAAQ,EAAE,CAAC;YACX,MAAM,CAAC,IAAI,CAAC,sBAAsB,EAAE;gBAClC,OAAO,EAAE,UAAU,CAAC,OAAO;gBAC3B,SAAS,EAAE,MAAM,CAAC,SAAS;gBAC3B,UAAU,EAAE,OAAO,CAAC,UAAU;gBAC9B,iBAAiB,EAAE,OAAO,CAAC,gBAAgB;gBAC3C,iBAAiB,EAAE,OAAO,CAAC,eAAe;aAC3C,CAAC,CAAC;YACH,OAAO,CAAC,WAAW,EAAE,CAAC,sBAAsB,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;SACtD;IACH,CAAC;CACF,CAAC,CAAC"}
|
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import type { FastifyPluginAsync } from 'fastify';
|
|
2
1
|
import { router, schema } from '@powersync/lib-services-framework';
|
|
2
|
+
import type { FastifyPluginAsync } from 'fastify';
|
|
3
3
|
export declare const yamlPlugin: FastifyPluginAsync;
|
|
4
4
|
export declare const deploySyncRules: router.Endpoint<{
|
|
5
5
|
content: string;
|
|
@@ -1,7 +1,6 @@
|
|
|
1
|
-
import * as t from 'ts-codec';
|
|
2
1
|
import { errors, router, schema } from '@powersync/lib-services-framework';
|
|
3
2
|
import { SqlSyncRules, SyncRulesErrors } from '@powersync/service-sync-rules';
|
|
4
|
-
import * as
|
|
3
|
+
import * as t from 'ts-codec';
|
|
5
4
|
import { authApi } from '../auth.js';
|
|
6
5
|
import { routeDefinition } from '../router.js';
|
|
7
6
|
const DeploySyncRulesRequest = t.object({
|
|
@@ -31,7 +30,9 @@ export const deploySyncRules = routeDefinition({
|
|
|
31
30
|
plugins: [yamlPlugin],
|
|
32
31
|
validator: schema.createTsCodecValidator(DeploySyncRulesRequest, { allowAdditional: true }),
|
|
33
32
|
handler: async (payload) => {
|
|
34
|
-
|
|
33
|
+
const { service_context } = payload.context;
|
|
34
|
+
const { storageEngine } = service_context;
|
|
35
|
+
if (service_context.configuration.sync_rules.present) {
|
|
35
36
|
// If sync rules are configured via the config, disable deploy via the API.
|
|
36
37
|
throw new errors.JourneyError({
|
|
37
38
|
status: 422,
|
|
@@ -42,7 +43,12 @@ export const deploySyncRules = routeDefinition({
|
|
|
42
43
|
}
|
|
43
44
|
const content = payload.params.content;
|
|
44
45
|
try {
|
|
45
|
-
|
|
46
|
+
const apiHandler = service_context.routerEngine.getAPI();
|
|
47
|
+
SqlSyncRules.fromYaml(payload.params.content, {
|
|
48
|
+
...apiHandler.getParseSyncRulesOptions(),
|
|
49
|
+
// We don't do any schema-level validation at this point
|
|
50
|
+
schema: undefined
|
|
51
|
+
});
|
|
46
52
|
}
|
|
47
53
|
catch (e) {
|
|
48
54
|
throw new errors.JourneyError({
|
|
@@ -52,7 +58,7 @@ export const deploySyncRules = routeDefinition({
|
|
|
52
58
|
details: e.message
|
|
53
59
|
});
|
|
54
60
|
}
|
|
55
|
-
const sync_rules = await
|
|
61
|
+
const sync_rules = await storageEngine.activeBucketStorage.updateSyncRules({
|
|
56
62
|
content: content
|
|
57
63
|
});
|
|
58
64
|
return {
|
|
@@ -72,7 +78,9 @@ export const validateSyncRules = routeDefinition({
|
|
|
72
78
|
validator: schema.createTsCodecValidator(ValidateSyncRulesRequest, { allowAdditional: true }),
|
|
73
79
|
handler: async (payload) => {
|
|
74
80
|
const content = payload.params.content;
|
|
75
|
-
const
|
|
81
|
+
const { service_context } = payload.context;
|
|
82
|
+
const apiHandler = service_context.routerEngine.getAPI();
|
|
83
|
+
const info = await debugSyncRules(apiHandler, content);
|
|
76
84
|
return replyPrettyJson(info);
|
|
77
85
|
}
|
|
78
86
|
});
|
|
@@ -81,8 +89,9 @@ export const currentSyncRules = routeDefinition({
|
|
|
81
89
|
method: router.HTTPMethod.GET,
|
|
82
90
|
authorize: authApi,
|
|
83
91
|
handler: async (payload) => {
|
|
84
|
-
const
|
|
85
|
-
const
|
|
92
|
+
const { service_context } = payload.context;
|
|
93
|
+
const { storageEngine: { activeBucketStorage } } = service_context;
|
|
94
|
+
const sync_rules = await activeBucketStorage.getActiveSyncRulesContent();
|
|
86
95
|
if (!sync_rules) {
|
|
87
96
|
throw new errors.JourneyError({
|
|
88
97
|
status: 422,
|
|
@@ -90,11 +99,10 @@ export const currentSyncRules = routeDefinition({
|
|
|
90
99
|
description: 'No active sync rules'
|
|
91
100
|
});
|
|
92
101
|
}
|
|
93
|
-
const
|
|
94
|
-
const
|
|
95
|
-
const
|
|
96
|
-
|
|
97
|
-
: null;
|
|
102
|
+
const apiHandler = service_context.routerEngine.getAPI();
|
|
103
|
+
const info = await debugSyncRules(apiHandler, sync_rules.sync_rules_content);
|
|
104
|
+
const next = await activeBucketStorage.getNextSyncRulesContent();
|
|
105
|
+
const next_info = next ? await debugSyncRules(apiHandler, next.sync_rules_content) : null;
|
|
98
106
|
const response = {
|
|
99
107
|
current: {
|
|
100
108
|
slot_name: sync_rules.slot_name,
|
|
@@ -119,8 +127,9 @@ export const reprocessSyncRules = routeDefinition({
|
|
|
119
127
|
authorize: authApi,
|
|
120
128
|
validator: schema.createTsCodecValidator(ReprocessSyncRulesRequest),
|
|
121
129
|
handler: async (payload) => {
|
|
122
|
-
const
|
|
123
|
-
const
|
|
130
|
+
const { storageEngine: { activeBucketStorage } } = payload.context.service_context;
|
|
131
|
+
const apiHandler = payload.context.service_context.routerEngine.getAPI();
|
|
132
|
+
const sync_rules = await activeBucketStorage.getActiveSyncRules(apiHandler.getParseSyncRulesOptions());
|
|
124
133
|
if (sync_rules == null) {
|
|
125
134
|
throw new errors.JourneyError({
|
|
126
135
|
status: 422,
|
|
@@ -128,7 +137,7 @@ export const reprocessSyncRules = routeDefinition({
|
|
|
128
137
|
description: 'No active sync rules'
|
|
129
138
|
});
|
|
130
139
|
}
|
|
131
|
-
const new_rules = await
|
|
140
|
+
const new_rules = await activeBucketStorage.updateSyncRules({
|
|
132
141
|
content: sync_rules.sync_rules.content
|
|
133
142
|
});
|
|
134
143
|
return {
|
|
@@ -144,15 +153,15 @@ function replyPrettyJson(payload) {
|
|
|
144
153
|
headers: { 'Content-Type': 'application/json' }
|
|
145
154
|
});
|
|
146
155
|
}
|
|
147
|
-
async function debugSyncRules(
|
|
156
|
+
async function debugSyncRules(apiHandler, sync_rules) {
|
|
148
157
|
try {
|
|
149
|
-
const rules = SqlSyncRules.fromYaml(sync_rules
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
sync_rules: rules
|
|
158
|
+
const rules = SqlSyncRules.fromYaml(sync_rules, {
|
|
159
|
+
...apiHandler.getParseSyncRulesOptions(),
|
|
160
|
+
// No schema-based validation at this point
|
|
161
|
+
schema: undefined
|
|
154
162
|
});
|
|
155
|
-
const
|
|
163
|
+
const source_table_patterns = rules.getSourceTables();
|
|
164
|
+
const resolved_tables = await apiHandler.getDebugTablesInfo(source_table_patterns, rules);
|
|
156
165
|
return {
|
|
157
166
|
valid: true,
|
|
158
167
|
bucket_definitions: rules.bucket_descriptors.map((d) => {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"sync-rules.js","sourceRoot":"","sources":["../../../src/routes/endpoints/sync-rules.ts"],"names":[],"mappings":"AAAA,OAAO,
|
|
1
|
+
{"version":3,"file":"sync-rules.js","sourceRoot":"","sources":["../../../src/routes/endpoints/sync-rules.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,mCAAmC,CAAC;AAC3E,OAAO,EAAE,YAAY,EAAE,eAAe,EAAE,MAAM,+BAA+B,CAAC;AAE9E,OAAO,KAAK,CAAC,MAAM,UAAU,CAAC;AAE9B,OAAO,EAAE,OAAO,EAAE,MAAM,YAAY,CAAC;AACrC,OAAO,EAAE,eAAe,EAAE,MAAM,cAAc,CAAC;AAG/C,MAAM,sBAAsB,GAAG,CAAC,CAAC,MAAM,CAAC;IACtC,OAAO,EAAE,CAAC,CAAC,MAAM;CAClB,CAAC,CAAC;AAEH,MAAM,CAAC,MAAM,UAAU,GAAuB,KAAK,EAAE,OAAO,EAAE,EAAE;IAC9D,OAAO,CAAC,oBAAoB,CAAC,kBAAkB,EAAE,KAAK,EAAE,OAAO,EAAE,OAAO,EAAE,EAAE,EAAE,EAAE;QAC9E,MAAM,IAAI,GAAU,EAAE,CAAC;QACvB,IAAI,KAAK,EAAE,MAAM,KAAK,IAAI,OAAO,EAAE;YACjC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;SAClB;QAED,OAAO,CAAC,MAAM,GAAG,EAAE,OAAO,EAAE,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC;IACrE,CAAC,CAAC,CAAC;AACL,CAAC,CAAC;AAEF;;;;GAIG;AACH,kBAAkB;AAClB,UAAU,CAAC,MAAM,CAAC,GAAG,CAAC,eAAe,CAAC,CAAC,GAAG,IAAI,CAAC;AAE/C,MAAM,CAAC,MAAM,eAAe,GAAG,eAAe,CAAC;IAC7C,IAAI,EAAE,2BAA2B;IACjC,MAAM,EAAE,MAAM,CAAC,UAAU,CAAC,IAAI;IAC9B,SAAS,EAAE,OAAO;IAClB,KAAK,EAAE,IAAI;IACX,OAAO,EAAE,CAAC,UAAU,CAAC;IACrB,SAAS,EAAE,MAAM,CAAC,sBAAsB,CAAC,sBAAsB,EAAE,EAAE,eAAe,EAAE,IAAI,EAAE,CAAC;IAC3F,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,EAAE;QACzB,MAAM,EAAE,eAAe,EAAE,GAAG,OAAO,CAAC,OAAO,CAAC;QAC5C,MAAM,EAAE,aAAa,EAAE,GAAG,eAAe,CAAC;QAE1C,IAAI,eAAe,CAAC,aAAa,CAAC,UAAU,CAAC,OAAO,EAAE;YACpD,2EAA2E;YAC3E,MAAM,IAAI,MAAM,CAAC,YAAY,CAAC;gBAC5B,MAAM,EAAE,GAAG;gBACX,IAAI,EAAE,cAAc;gBACpB,WAAW,EAAE,yBAAyB;gBACtC,OAAO,EAAE,6CAA6C;aACvD,CAAC,CAAC;SACJ;QACD,MAAM,OAAO,GAAG,OAAO,CAAC,MAAM,CAAC,OAAO,CAAC;QAEvC,IAAI;YACF,MAAM,UAAU,GAAG,eAAe,CAAC,YAAa,CAAC,MAAM,EAAE,CAAC;YAC1D,YAAY,CAAC,QAAQ,CAAC,OAAO,CAAC,MAAM,CAAC,OAAO,EAAE;gBAC5C,GAAG,UAAU,CAAC,wBAAwB,EAAE;gBACxC,wDAAwD;gBACxD,MAAM,EAAE,SAAS;aAClB,CAAC,CAAC;SACJ;QAAC,OAAO,CAAC,EAAE;YACV,MAAM,IAAI,MAAM,CAAC,YAAY,CAAC;gBAC5B,MAAM,EAAE,GAAG;gBACX,IAAI,EAAE,oBAAoB;gBAC1B,WAAW,EAAE,2BAA2B;gBACxC,OAAO,EAAE,CAAC,CAAC,OAAO;aACnB,CAAC,CAAC;SACJ;QAED,MAAM,UAAU,GAAG,MAAM,aAAa,CAAC,mBAAmB,CAAC,eAAe,CAAC;YACzE,OAAO,EAAE,OAAO;SACjB,CAAC,CAAC;QAEH,OAAO;YACL,SAAS,EAAE,UAAU,CAAC,SAAS;SAChC,CAAC;IACJ,CAAC;CACF,CAAC,CAAC;AAEH,MAAM,wBAAwB,GAAG,CAAC,CAAC,MAAM,CAAC;IACxC,OAAO,EAAE,CAAC,CAAC,MAAM;CAClB,CAAC,CAAC;AAEH,MAAM,CAAC,MAAM,iBAAiB,GAAG,eAAe,CAAC;IAC/C,IAAI,EAAE,6BAA6B;IACnC,MAAM,EAAE,MAAM,CAAC,UAAU,CAAC,IAAI;IAC9B,SAAS,EAAE,OAAO;IAClB,KAAK,EAAE,IAAI;IACX,OAAO,EAAE,CAAC,UAAU,CAAC;IACrB,SAAS,EAAE,MAAM,CAAC,sBAAsB,CAAC,wBAAwB,EAAE,EAAE,eAAe,EAAE,IAAI,EAAE,CAAC;IAC7F,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,EAAE;QACzB,MAAM,OAAO,GAAG,OAAO,CAAC,MAAM,CAAC,OAAO,CAAC;QACvC,MAAM,EAAE,eAAe,EAAE,GAAG,OAAO,CAAC,OAAO,CAAC;QAC5C,MAAM,UAAU,GAAG,eAAe,CAAC,YAAa,CAAC,MAAM,EAAE,CAAC;QAE1D,MAAM,IAAI,GAAG,MAAM,cAAc,CAAC,UAAU,EAAE,OAAO,CAAC,CAAC;QAEvD,OAAO,eAAe,CAAC,IAAI,CAAC,CAAC;IAC/B,CAAC;CACF,CAAC,CAAC;AAEH,MAAM,CAAC,MAAM,gBAAgB,GAAG,eAAe,CAAC;IAC9C,IAAI,EAAE,4BAA4B;IAClC,MAAM,EAAE,MAAM,CAAC,UAAU,CAAC,GAAG;IAC7B,SAAS,EAAE,OAAO;IAClB,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,EAAE;QACzB,MAAM,EAAE,eAAe,EAAE,GAAG,OAAO,CAAC,OAAO,CAAC;QAC5C,MAAM,EACJ,aAAa,EAAE,EAAE,mBAAmB,EAAE,EACvC,GAAG,eAAe,CAAC;QAEpB,MAAM,UAAU,GAAG,MAAM,mBAAmB,CAAC,yBAAyB,EAAE,CAAC;QACzE,IAAI,CAAC,UAAU,EAAE;YACf,MAAM,IAAI,MAAM,CAAC,YAAY,CAAC;gBAC5B,MAAM,EAAE,GAAG;gBACX,IAAI,EAAE,eAAe;gBACrB,WAAW,EAAE,sBAAsB;aACpC,CAAC,CAAC;SACJ;QAED,MAAM,UAAU,GAAG,eAAe,CAAC,YAAa,CAAC,MAAM,EAAE,CAAC;QAC1D,MAAM,IAAI,GAAG,MAAM,cAAc,CAAC,UAAU,EAAE,UAAU,CAAC,kBAAkB,CAAC,CAAC;QAC7E,MAAM,IAAI,GAAG,MAAM,mBAAmB,CAAC,uBAAuB,EAAE,CAAC;QAEjE,MAAM,SAAS,GAAG,IAAI,CAAC,CAAC,CAAC,MAAM,cAAc,CAAC,UAAU,EAAE,IAAI,CAAC,kBAAkB,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;QAE1F,MAAM,QAAQ,GAAG;YACf,OAAO,EAAE;gBACP,SAAS,EAAE,UAAU,CAAC,SAAS;gBAC/B,OAAO,EAAE,UAAU,CAAC,kBAAkB;gBACtC,GAAG,IAAI;aACR;YACD,IAAI,EACF,IAAI,IAAI,IAAI;gBACV,CAAC,CAAC,IAAI;gBACN,CAAC,CAAC;oBACE,SAAS,EAAE,IAAI,CAAC,SAAS;oBACzB,OAAO,EAAE,IAAI,CAAC,kBAAkB;oBAChC,GAAG,SAAS;iBACb;SACR,CAAC;QAEF,OAAO,eAAe,CAAC,EAAE,IAAI,EAAE,QAAQ,EAAE,CAAC,CAAC;IAC7C,CAAC;CACF,CAAC,CAAC;AAEH,MAAM,yBAAyB,GAAG,CAAC,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;AAE/C,MAAM,CAAC,MAAM,kBAAkB,GAAG,eAAe,CAAC;IAChD,IAAI,EAAE,8BAA8B;IACpC,MAAM,EAAE,MAAM,CAAC,UAAU,CAAC,IAAI;IAC9B,SAAS,EAAE,OAAO;IAClB,SAAS,EAAE,MAAM,CAAC,sBAAsB,CAAC,yBAAyB,CAAC;IACnE,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,EAAE;QACzB,MAAM,EACJ,aAAa,EAAE,EAAE,mBAAmB,EAAE,EACvC,GAAG,OAAO,CAAC,OAAO,CAAC,eAAe,CAAC;QACpC,MAAM,UAAU,GAAG,OAAO,CAAC,OAAO,CAAC,eAAe,CAAC,YAAa,CAAC,MAAM,EAAE,CAAC;QAC1E,MAAM,UAAU,GAAG,MAAM,mBAAmB,CAAC,kBAAkB,CAAC,UAAU,CAAC,wBAAwB,EAAE,CAAC,CAAC;QACvG,IAAI,UAAU,IAAI,IAAI,EAAE;YACtB,MAAM,IAAI,MAAM,CAAC,YAAY,CAAC;gBAC5B,MAAM,EAAE,GAAG;gBACX,IAAI,EAAE,eAAe;gBACrB,WAAW,EAAE,sBAAsB;aACpC,CAAC,CAAC;SACJ;QAED,MAAM,SAAS,GAAG,MAAM,mBAAmB,CAAC,eAAe,CAAC;YAC1D,OAAO,EAAE,UAAU,CAAC,UAAU,CAAC,OAAO;SACvC,CAAC,CAAC;QACH,OAAO;YACL,SAAS,EAAE,SAAS,CAAC,SAAS;SAC/B,CAAC;IACJ,CAAC;CACF,CAAC,CAAC;AAEH,MAAM,CAAC,MAAM,iBAAiB,GAAG,CAAC,iBAAiB,EAAE,eAAe,EAAE,kBAAkB,EAAE,gBAAgB,CAAC,CAAC;AAE5G,SAAS,eAAe,CAAC,OAAY;IACnC,OAAO,IAAI,MAAM,CAAC,cAAc,CAAC;QAC/B,MAAM,EAAE,GAAG;QACX,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC,GAAG,IAAI;QAC7C,OAAO,EAAE,EAAE,cAAc,EAAE,kBAAkB,EAAE;KAChD,CAAC,CAAC;AACL,CAAC;AAED,KAAK,UAAU,cAAc,CAAC,UAAoB,EAAE,UAAkB;IACpE,IAAI;QACF,MAAM,KAAK,GAAG,YAAY,CAAC,QAAQ,CAAC,UAAU,EAAE;YAC9C,GAAG,UAAU,CAAC,wBAAwB,EAAE;YACxC,2CAA2C;YAC3C,MAAM,EAAE,SAAS;SAClB,CAAC,CAAC;QACH,MAAM,qBAAqB,GAAG,KAAK,CAAC,eAAe,EAAE,CAAC;QACtD,MAAM,eAAe,GAAG,MAAM,UAAU,CAAC,kBAAkB,CAAC,qBAAqB,EAAE,KAAK,CAAC,CAAC;QAE1F,OAAO;YACL,KAAK,EAAE,IAAI;YACX,kBAAkB,EAAE,KAAK,CAAC,kBAAkB,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE;gBACrD,IAAI,qBAAqB,GAAG,CAAC,GAAG,CAAC,CAAC,iBAAiB,CAAC,MAAM,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;gBACrE,IAAI,gBAAgB,GAAG,CAAC,GAAG,CAAC,CAAC,YAAY,CAAC,MAAM,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;gBAC3D,OAAO;oBACL,IAAI,EAAE,CAAC,CAAC,IAAI;oBACZ,iBAAiB,EAAE,CAAC,CAAC,iBAAiB;oBACtC,wBAAwB,EAAE,CAAC,CAAC,wBAAwB,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE;wBAC7D,OAAO;4BACL,GAAG,EAAE,CAAC,CAAC,GAAG;yBACX,CAAC;oBACJ,CAAC,CAAC;oBACF,iBAAiB,EAAE,qBAAqB,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE;wBACjD,OAAO;4BACL,GAAG,EAAE,CAAC,CAAC,GAAG;4BACV,KAAK,EAAE,CAAC,CAAC,WAAW;4BACpB,gBAAgB,EAAE,CAAC,CAAC,gBAAgB;yBACrC,CAAC;oBACJ,CAAC,CAAC;oBAEF,YAAY,EAAE,gBAAgB,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE;wBACvC,OAAO;4BACL,GAAG,EAAE,CAAC,CAAC,GAAG;4BACV,KAAK,EAAE,CAAC,CAAC,WAAW;4BACpB,OAAO,EAAE,CAAC,CAAC,iBAAiB,EAAE;yBAC/B,CAAC;oBACJ,CAAC,CAAC;iBACH,CAAC;YACJ,CAAC,CAAC;YACF,aAAa,EAAE,eAAe;YAC9B,WAAW,EAAE,KAAK,CAAC,oBAAoB,EAAE;SAC1C,CAAC;KACH;IAAC,OAAO,CAAC,EAAE;QACV,IAAI,CAAC,YAAY,eAAe,EAAE;YAChC,OAAO;gBACL,KAAK,EAAE,KAAK;gBACZ,MAAM,EAAE,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC;aACvC,CAAC;SACH;QACD,OAAO;YACL,KAAK,EAAE,KAAK;YACZ,MAAM,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC;SACpB,CAAC;KACH;AACH,CAAC"}
|
|
@@ -14,6 +14,7 @@ export declare const syncStreamed: router.Endpoint<{
|
|
|
14
14
|
include_checksum?: boolean | undefined;
|
|
15
15
|
raw_data?: boolean | undefined;
|
|
16
16
|
binary_data?: boolean | undefined;
|
|
17
|
+
client_id?: string | undefined;
|
|
17
18
|
}, router.RouterResponse<Readable> | undefined, import("../router.js").Context, import("../router.js").RequestEndpointHandlerPayload<{
|
|
18
19
|
parameters?: Record<string, any> | undefined;
|
|
19
20
|
buckets?: {
|
|
@@ -24,6 +25,7 @@ export declare const syncStreamed: router.Endpoint<{
|
|
|
24
25
|
include_checksum?: boolean | undefined;
|
|
25
26
|
raw_data?: boolean | undefined;
|
|
26
27
|
binary_data?: boolean | undefined;
|
|
28
|
+
client_id?: string | undefined;
|
|
27
29
|
}, import("../router.js").Context, import("../router.js").BasicRouterRequest>, router.EndpointHandler<import("../router.js").RequestEndpointHandlerPayload<{
|
|
28
30
|
parameters?: Record<string, any> | undefined;
|
|
29
31
|
buckets?: {
|
|
@@ -34,6 +36,7 @@ export declare const syncStreamed: router.Endpoint<{
|
|
|
34
36
|
include_checksum?: boolean | undefined;
|
|
35
37
|
raw_data?: boolean | undefined;
|
|
36
38
|
binary_data?: boolean | undefined;
|
|
39
|
+
client_id?: string | undefined;
|
|
37
40
|
}, import("../router.js").Context, import("../router.js").BasicRouterRequest>, router.RouterResponse<Readable> | undefined>> & {
|
|
38
41
|
path: SyncRoutes;
|
|
39
42
|
method: router.HTTPMethod.POST;
|
|
@@ -54,6 +57,7 @@ export declare const syncStreamed: router.Endpoint<{
|
|
|
54
57
|
include_checksum?: boolean | undefined;
|
|
55
58
|
raw_data?: boolean | undefined;
|
|
56
59
|
binary_data?: boolean | undefined;
|
|
60
|
+
client_id?: string | undefined;
|
|
57
61
|
}, string[]>;
|
|
58
62
|
handler: (payload: import("../router.js").RequestEndpointHandlerPayload<{
|
|
59
63
|
parameters?: Record<string, any> | undefined;
|
|
@@ -65,6 +69,7 @@ export declare const syncStreamed: router.Endpoint<{
|
|
|
65
69
|
include_checksum?: boolean | undefined;
|
|
66
70
|
raw_data?: boolean | undefined;
|
|
67
71
|
binary_data?: boolean | undefined;
|
|
72
|
+
client_id?: string | undefined;
|
|
68
73
|
}, import("../router.js").Context, import("../router.js").BasicRouterRequest>) => Promise<router.RouterResponse<Readable> | undefined>;
|
|
69
74
|
};
|
|
70
75
|
export declare const SYNC_STREAM_ROUTES: (router.Endpoint<{
|
|
@@ -77,6 +82,7 @@ export declare const SYNC_STREAM_ROUTES: (router.Endpoint<{
|
|
|
77
82
|
include_checksum?: boolean | undefined;
|
|
78
83
|
raw_data?: boolean | undefined;
|
|
79
84
|
binary_data?: boolean | undefined;
|
|
85
|
+
client_id?: string | undefined;
|
|
80
86
|
}, router.RouterResponse<Readable> | undefined, import("../router.js").Context, import("../router.js").RequestEndpointHandlerPayload<{
|
|
81
87
|
parameters?: Record<string, any> | undefined;
|
|
82
88
|
buckets?: {
|
|
@@ -87,6 +93,7 @@ export declare const SYNC_STREAM_ROUTES: (router.Endpoint<{
|
|
|
87
93
|
include_checksum?: boolean | undefined;
|
|
88
94
|
raw_data?: boolean | undefined;
|
|
89
95
|
binary_data?: boolean | undefined;
|
|
96
|
+
client_id?: string | undefined;
|
|
90
97
|
}, import("../router.js").Context, import("../router.js").BasicRouterRequest>, router.EndpointHandler<import("../router.js").RequestEndpointHandlerPayload<{
|
|
91
98
|
parameters?: Record<string, any> | undefined;
|
|
92
99
|
buckets?: {
|
|
@@ -97,6 +104,7 @@ export declare const SYNC_STREAM_ROUTES: (router.Endpoint<{
|
|
|
97
104
|
include_checksum?: boolean | undefined;
|
|
98
105
|
raw_data?: boolean | undefined;
|
|
99
106
|
binary_data?: boolean | undefined;
|
|
107
|
+
client_id?: string | undefined;
|
|
100
108
|
}, import("../router.js").Context, import("../router.js").BasicRouterRequest>, router.RouterResponse<Readable> | undefined>> & {
|
|
101
109
|
path: SyncRoutes;
|
|
102
110
|
method: router.HTTPMethod.POST;
|
|
@@ -117,6 +125,7 @@ export declare const SYNC_STREAM_ROUTES: (router.Endpoint<{
|
|
|
117
125
|
include_checksum?: boolean | undefined;
|
|
118
126
|
raw_data?: boolean | undefined;
|
|
119
127
|
binary_data?: boolean | undefined;
|
|
128
|
+
client_id?: string | undefined;
|
|
120
129
|
}, string[]>;
|
|
121
130
|
handler: (payload: import("../router.js").RequestEndpointHandlerPayload<{
|
|
122
131
|
parameters?: Record<string, any> | undefined;
|
|
@@ -128,5 +137,6 @@ export declare const SYNC_STREAM_ROUTES: (router.Endpoint<{
|
|
|
128
137
|
include_checksum?: boolean | undefined;
|
|
129
138
|
raw_data?: boolean | undefined;
|
|
130
139
|
binary_data?: boolean | undefined;
|
|
140
|
+
client_id?: string | undefined;
|
|
131
141
|
}, import("../router.js").Context, import("../router.js").BasicRouterRequest>) => Promise<router.RouterResponse<Readable> | undefined>;
|
|
132
142
|
})[];
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { errors, logger, router, schema } from '@powersync/lib-services-framework';
|
|
2
2
|
import { RequestParameters } from '@powersync/service-sync-rules';
|
|
3
3
|
import { Readable } from 'stream';
|
|
4
4
|
import * as sync from '../../sync/sync-index.js';
|
|
@@ -6,7 +6,6 @@ import * as util from '../../util/util-index.js';
|
|
|
6
6
|
import { Metrics } from '../../metrics/Metrics.js';
|
|
7
7
|
import { authUser } from '../auth.js';
|
|
8
8
|
import { routeDefinition } from '../router.js';
|
|
9
|
-
import { RequestTracker } from '../../sync/RequestTracker.js';
|
|
10
9
|
export var SyncRoutes;
|
|
11
10
|
(function (SyncRoutes) {
|
|
12
11
|
SyncRoutes["STREAM"] = "/sync/stream";
|
|
@@ -17,8 +16,12 @@ export const syncStreamed = routeDefinition({
|
|
|
17
16
|
authorize: authUser,
|
|
18
17
|
validator: schema.createTsCodecValidator(util.StreamingSyncRequest, { allowAdditional: true }),
|
|
19
18
|
handler: async (payload) => {
|
|
20
|
-
const
|
|
21
|
-
|
|
19
|
+
const { service_context } = payload.context;
|
|
20
|
+
const { routerEngine, storageEngine } = service_context;
|
|
21
|
+
const headers = payload.request.headers;
|
|
22
|
+
const userAgent = headers['x-user-agent'] ?? headers['user-agent'];
|
|
23
|
+
const clientId = payload.params.client_id;
|
|
24
|
+
if (routerEngine.closed) {
|
|
22
25
|
throw new errors.JourneyError({
|
|
23
26
|
status: 503,
|
|
24
27
|
code: 'SERVICE_UNAVAILABLE',
|
|
@@ -27,9 +30,8 @@ export const syncStreamed = routeDefinition({
|
|
|
27
30
|
}
|
|
28
31
|
const params = payload.params;
|
|
29
32
|
const syncParams = new RequestParameters(payload.context.token_payload, payload.params.parameters ?? {});
|
|
30
|
-
const storage = system.storage;
|
|
31
33
|
// Sanity check before we start the stream
|
|
32
|
-
const cp = await
|
|
34
|
+
const cp = await storageEngine.activeBucketStorage.getActiveCheckpoint();
|
|
33
35
|
if (!cp.hasSyncRules()) {
|
|
34
36
|
throw new errors.JourneyError({
|
|
35
37
|
status: 500,
|
|
@@ -37,20 +39,20 @@ export const syncStreamed = routeDefinition({
|
|
|
37
39
|
description: 'No sync rules available'
|
|
38
40
|
});
|
|
39
41
|
}
|
|
40
|
-
const metrics = container.getImplementation(Metrics);
|
|
41
42
|
const controller = new AbortController();
|
|
42
|
-
const tracker = new RequestTracker();
|
|
43
|
+
const tracker = new sync.RequestTracker();
|
|
43
44
|
try {
|
|
44
|
-
|
|
45
|
+
Metrics.getInstance().concurrent_connections.add(1);
|
|
45
46
|
const stream = Readable.from(sync.transformToBytesTracked(sync.ndjson(sync.streamResponse({
|
|
46
|
-
storage,
|
|
47
|
+
storage: storageEngine.activeBucketStorage,
|
|
48
|
+
parseOptions: routerEngine.getAPI().getParseSyncRulesOptions(),
|
|
47
49
|
params,
|
|
48
50
|
syncParams,
|
|
49
51
|
token: payload.context.token_payload,
|
|
50
52
|
tracker,
|
|
51
53
|
signal: controller.signal
|
|
52
54
|
})), tracker), { objectMode: false, highWaterMark: 16 * 1024 });
|
|
53
|
-
const deregister =
|
|
55
|
+
const deregister = routerEngine.addStopHandler(() => {
|
|
54
56
|
// This error is not currently propagated to the client
|
|
55
57
|
controller.abort();
|
|
56
58
|
stream.destroy(new Error('Shutting down system'));
|
|
@@ -73,9 +75,11 @@ export const syncStreamed = routeDefinition({
|
|
|
73
75
|
data: stream,
|
|
74
76
|
afterSend: async () => {
|
|
75
77
|
controller.abort();
|
|
76
|
-
|
|
78
|
+
Metrics.getInstance().concurrent_connections.add(-1);
|
|
77
79
|
logger.info(`Sync stream complete`, {
|
|
78
80
|
user_id: syncParams.user_id,
|
|
81
|
+
client_id: clientId,
|
|
82
|
+
user_agent: userAgent,
|
|
79
83
|
operations_synced: tracker.operationsSynced,
|
|
80
84
|
data_synced_bytes: tracker.dataSyncedBytes
|
|
81
85
|
});
|
|
@@ -84,7 +88,7 @@ export const syncStreamed = routeDefinition({
|
|
|
84
88
|
}
|
|
85
89
|
catch (ex) {
|
|
86
90
|
controller.abort();
|
|
87
|
-
|
|
91
|
+
Metrics.getInstance().concurrent_connections.add(-1);
|
|
88
92
|
}
|
|
89
93
|
}
|
|
90
94
|
});
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"sync-stream.js","sourceRoot":"","sources":["../../../src/routes/endpoints/sync-stream.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,
|
|
1
|
+
{"version":3,"file":"sync-stream.js","sourceRoot":"","sources":["../../../src/routes/endpoints/sync-stream.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,mCAAmC,CAAC;AACnF,OAAO,EAAE,iBAAiB,EAAE,MAAM,+BAA+B,CAAC;AAClE,OAAO,EAAE,QAAQ,EAAE,MAAM,QAAQ,CAAC;AAElC,OAAO,KAAK,IAAI,MAAM,0BAA0B,CAAC;AACjD,OAAO,KAAK,IAAI,MAAM,0BAA0B,CAAC;AAEjD,OAAO,EAAE,OAAO,EAAE,MAAM,0BAA0B,CAAC;AACnD,OAAO,EAAE,QAAQ,EAAE,MAAM,YAAY,CAAC;AACtC,OAAO,EAAE,eAAe,EAAE,MAAM,cAAc,CAAC;AAE/C,MAAM,CAAN,IAAY,UAEX;AAFD,WAAY,UAAU;IACpB,qCAAuB,CAAA;AACzB,CAAC,EAFW,UAAU,KAAV,UAAU,QAErB;AAED,MAAM,CAAC,MAAM,YAAY,GAAG,eAAe,CAAC;IAC1C,IAAI,EAAE,UAAU,CAAC,MAAM;IACvB,MAAM,EAAE,MAAM,CAAC,UAAU,CAAC,IAAI;IAC9B,SAAS,EAAE,QAAQ;IACnB,SAAS,EAAE,MAAM,CAAC,sBAAsB,CAAC,IAAI,CAAC,oBAAoB,EAAE,EAAE,eAAe,EAAE,IAAI,EAAE,CAAC;IAC9F,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,EAAE;QACzB,MAAM,EAAE,eAAe,EAAE,GAAG,OAAO,CAAC,OAAO,CAAC;QAC5C,MAAM,EAAE,YAAY,EAAE,aAAa,EAAE,GAAG,eAAe,CAAC;QACxD,MAAM,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC,OAAO,CAAC;QACxC,MAAM,SAAS,GAAG,OAAO,CAAC,cAAc,CAAC,IAAI,OAAO,CAAC,YAAY,CAAC,CAAC;QACnE,MAAM,QAAQ,GAAG,OAAO,CAAC,MAAM,CAAC,SAAS,CAAC;QAE1C,IAAI,YAAa,CAAC,MAAM,EAAE;YACxB,MAAM,IAAI,MAAM,CAAC,YAAY,CAAC;gBAC5B,MAAM,EAAE,GAAG;gBACX,IAAI,EAAE,qBAAqB;gBAC3B,WAAW,EAAE,iCAAiC;aAC/C,CAAC,CAAC;SACJ;QAED,MAAM,MAAM,GAA8B,OAAO,CAAC,MAAM,CAAC;QACzD,MAAM,UAAU,GAAG,IAAI,iBAAiB,CAAC,OAAO,CAAC,OAAO,CAAC,aAAc,EAAE,OAAO,CAAC,MAAM,CAAC,UAAU,IAAI,EAAE,CAAC,CAAC;QAE1G,0CAA0C;QAC1C,MAAM,EAAE,GAAG,MAAM,aAAa,CAAC,mBAAmB,CAAC,mBAAmB,EAAE,CAAC;QACzE,IAAI,CAAC,EAAE,CAAC,YAAY,EAAE,EAAE;YACtB,MAAM,IAAI,MAAM,CAAC,YAAY,CAAC;gBAC5B,MAAM,EAAE,GAAG;gBACX,IAAI,EAAE,eAAe;gBACrB,WAAW,EAAE,yBAAyB;aACvC,CAAC,CAAC;SACJ;QACD,MAAM,UAAU,GAAG,IAAI,eAAe,EAAE,CAAC;QACzC,MAAM,OAAO,GAAG,IAAI,IAAI,CAAC,cAAc,EAAE,CAAC;QAC1C,IAAI;YACF,OAAO,CAAC,WAAW,EAAE,CAAC,sBAAsB,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;YACpD,MAAM,MAAM,GAAG,QAAQ,CAAC,IAAI,CAC1B,IAAI,CAAC,uBAAuB,CAC1B,IAAI,CAAC,MAAM,CACT,IAAI,CAAC,cAAc,CAAC;gBAClB,OAAO,EAAE,aAAa,CAAC,mBAAmB;gBAC1C,YAAY,EAAE,YAAa,CAAC,MAAM,EAAE,CAAC,wBAAwB,EAAE;gBAC/D,MAAM;gBACN,UAAU;gBACV,KAAK,EAAE,OAAO,CAAC,OAAO,CAAC,aAAc;gBACrC,OAAO;gBACP,MAAM,EAAE,UAAU,CAAC,MAAM;aAC1B,CAAC,CACH,EACD,OAAO,CACR,EACD,EAAE,UAAU,EAAE,KAAK,EAAE,aAAa,EAAE,EAAE,GAAG,IAAI,EAAE,CAChD,CAAC;YAEF,MAAM,UAAU,GAAG,YAAa,CAAC,cAAc,CAAC,GAAG,EAAE;gBACnD,uDAAuD;gBACvD,UAAU,CAAC,KAAK,EAAE,CAAC;gBACnB,MAAM,CAAC,OAAO,CAAC,IAAI,KAAK,CAAC,sBAAsB,CAAC,CAAC,CAAC;YACpD,CAAC,CAAC,CAAC;YACH,MAAM,CAAC,EAAE,CAAC,OAAO,EAAE,GAAG,EAAE;gBACtB,UAAU,EAAE,CAAC;YACf,CAAC,CAAC,CAAC;YAEH,MAAM,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,KAAK,EAAE,EAAE;gBAC3B,UAAU,CAAC,KAAK,EAAE,CAAC;gBACnB,oDAAoD;gBACpD,IAAI,KAAK,CAAC,OAAO,IAAI,sBAAsB,EAAE;oBAC3C,MAAM,CAAC,KAAK,CAAC,+BAA+B,EAAE,KAAK,CAAC,CAAC;iBACtD;YACH,CAAC,CAAC,CAAC;YAEH,OAAO,IAAI,MAAM,CAAC,cAAc,CAAC;gBAC/B,MAAM,EAAE,GAAG;gBACX,OAAO,EAAE;oBACP,cAAc,EAAE,sBAAsB;iBACvC;gBACD,IAAI,EAAE,MAAM;gBACZ,SAAS,EAAE,KAAK,IAAI,EAAE;oBACpB,UAAU,CAAC,KAAK,EAAE,CAAC;oBACnB,OAAO,CAAC,WAAW,EAAE,CAAC,sBAAsB,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;oBACrD,MAAM,CAAC,IAAI,CAAC,sBAAsB,EAAE;wBAClC,OAAO,EAAE,UAAU,CAAC,OAAO;wBAC3B,SAAS,EAAE,QAAQ;wBACnB,UAAU,EAAE,SAAS;wBACrB,iBAAiB,EAAE,OAAO,CAAC,gBAAgB;wBAC3C,iBAAiB,EAAE,OAAO,CAAC,eAAe;qBAC3C,CAAC,CAAC;gBACL,CAAC;aACF,CAAC,CAAC;SACJ;QAAC,OAAO,EAAE,EAAE;YACX,UAAU,CAAC,KAAK,EAAE,CAAC;YACnB,OAAO,CAAC,WAAW,EAAE,CAAC,sBAAsB,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;SACtD;IACH,CAAC;CACF,CAAC,CAAC;AAEH,MAAM,CAAC,MAAM,kBAAkB,GAAG,CAAC,YAAY,CAAC,CAAC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"route-register.js","sourceRoot":"","sources":["../../src/routes/route-register.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,MAAM,
|
|
1
|
+
{"version":3,"file":"route-register.js","sourceRoot":"","sources":["../../src/routes/route-register.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,MAAM,EAAc,MAAM,EAAE,MAAM,EAAE,MAAM,mCAAmC,CAAC;AAQvF;;GAEG;AACH,MAAM,UAAU,qBAAqB,CACnC,GAA4B,EAC5B,eAAgC,EAChC,SAA+C;IAE/C,KAAK,MAAM,CAAC,IAAI,SAAS,EAAE;QACzB,sCAAsC;QACtC,GAAG,CAAC,QAAQ,CAAC,KAAK,WAAW,OAAO;YAClC,OAAO,CAAC,KAAK,CAAC;gBACZ,GAAG,EAAE,CAAC,CAAC,IAAI;gBACX,MAAM,EAAE,CAAC,CAAC,MAAoB;gBAC9B,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,KAAK,EAAE,EAAE;oBAChC,MAAM,SAAS,GAAG,IAAI,IAAI,EAAE,CAAC;oBAC7B,IAAI,QAA+B,CAAC;oBACpC,IAAI;wBACF,MAAM,OAAO,GAAG,MAAM,eAAe,CAAC,OAAO,CAAC,CAAC;wBAE/C,IAAI,QAAQ,GAAG;4BACb,GAAI,OAAO,CAAC,MAAc;4BAC1B,GAAI,OAAO,CAAC,KAAa;yBAC1B,CAAC;wBAEF,IAAI,OAAO,OAAO,CAAC,IAAI,KAAK,QAAQ,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE;4BACtG,QAAQ,GAAG;gCACT,GAAG,QAAQ;gCACX,GAAG,OAAO,CAAC,IAAI;6BAChB,CAAC;yBACH;wBAED,MAAM,OAAO,GAAkC;4BAC7C,OAAO,EAAE,OAAO;4BAChB,MAAM,EAAE,QAAQ;4BAChB,OAAO;yBACR,CAAC;wBAEF,MAAM,gBAAgB,GAAG,MAAM,MAAM,CAAC,eAAe,CAAC,CAAC,EAAE,OAAO,CAAC,CAAC;wBAElE,IAAI,MAAM,CAAC,cAAc,CAAC,gBAAgB,CAAC,gBAAgB,CAAC,EAAE;4BAC5D,QAAQ,GAAG,gBAAgB,CAAC;yBAC7B;6BAAM,IAAI,MAAM,CAAC,eAAe,CAAC,gBAAgB,CAAC,IAAI,MAAM,CAAC,QAAQ,CAAC,gBAAgB,CAAC,EAAE;4BACxF,QAAQ,GAAG,IAAI,MAAM,CAAC,cAAc,CAAC;gCACnC,MAAM,EAAE,GAAG;gCACX,IAAI,EAAE,gBAAgB;6BACvB,CAAC,CAAC;yBACJ;6BAAM;4BACL,QAAQ,GAAG,IAAI,MAAM,CAAC,cAAc,CAAC;gCACnC,MAAM,EAAE,GAAG;gCACX,IAAI,EAAE,EAAE,IAAI,EAAE,gBAAgB,EAAE;6BACjC,CAAC,CAAC;yBACJ;qBACF;oBAAC,OAAO,EAAE,EAAE;wBACX,MAAM,YAAY,GAAG,MAAM,CAAC,YAAY,CAAC,cAAc,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,MAAM,CAAC,mBAAmB,CAAC,EAAE,CAAC,CAAC;wBACtG,MAAM,CAAC,KAAK,CAAC,gBAAgB,EAAE,YAAY,CAAC,CAAC;wBAE7C,QAAQ,GAAG,IAAI,MAAM,CAAC,cAAc,CAAC;4BACnC,MAAM,EAAE,YAAY,CAAC,SAAS,CAAC,MAAM,IAAI,GAAG;4BAC5C,OAAO,EAAE;gCACP,cAAc,EAAE,kBAAkB;6BACnC;4BACD,IAAI,EAAE;gCACJ,KAAK,EAAE,YAAY,CAAC,SAAS;6BAC9B;yBACF,CAAC,CAAC;qBACJ;oBAED,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,OAAO,CAAC,CAAC,GAAG,EAAE,EAAE;wBAC5C,KAAK,CAAC,MAAM,CAAC,GAAG,EAAE,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC;oBAC3C,CAAC,CAAC,CAAC;oBACH,KAAK,CAAC,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;oBAC9B,IAAI;wBACF,MAAM,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;qBACjC;4BAAS;wBACR,MAAM,QAAQ,CAAC,SAAS,EAAE,EAAE,CAAC;wBAC7B,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,MAAM,IAAI,OAAO,CAAC,GAAG,EAAE,EAAE;4BACxC,WAAW,EAAE,IAAI,CAAC,KAAK,CAAC,IAAI,IAAI,EAAE,CAAC,OAAO,EAAE,GAAG,SAAS,CAAC,OAAO,EAAE,GAAG,MAAM,CAAC,OAAO,CAAC;4BACpF,MAAM,EAAE,QAAQ,CAAC,MAAM;4BACvB,MAAM,EAAE,CAAC,CAAC,MAAM;4BAChB,IAAI,EAAE,OAAO,CAAC,GAAG;4BACjB,KAAK,EAAE,CAAC,CAAC,IAAI;yBACd,CAAC,CAAC;qBACJ;gBACH,CAAC;aACF,CAAC,CAAC;YAEH,IAAI,CAAC,CAAC,CAAC,CAAC,KAAK,IAAI,IAAI,CAAC,EAAE;gBACtB,OAAO,CAAC,2BAA2B,EAAE,CAAC;aACvC;YAED,CAAC,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC;QAC3D,CAAC,CAAC,CAAC;KACJ;AACH,CAAC"}
|
|
@@ -1,10 +1,11 @@
|
|
|
1
|
+
import { IReactiveStream, ReactiveSocketRouter } from '@powersync/service-rsocket-router';
|
|
1
2
|
import * as t from 'ts-codec';
|
|
2
|
-
import { ReactiveSocketRouter, IReactiveStream } from '@powersync/service-rsocket-router';
|
|
3
3
|
import { Context } from './router.js';
|
|
4
|
-
export declare const RSocketContextMeta: t.ObjectCodec<{
|
|
5
|
-
token: t.IdentityCodec<t.CodecType.String>;
|
|
6
|
-
}>;
|
|
7
4
|
/**
|
|
8
5
|
* Creates a socket route handler given a router instance
|
|
9
6
|
*/
|
|
10
7
|
export type SocketRouteGenerator = (router: ReactiveSocketRouter<Context>) => IReactiveStream;
|
|
8
|
+
export declare const RSocketContextMeta: t.ObjectCodec<{
|
|
9
|
+
token: t.IdentityCodec<t.CodecType.String>;
|
|
10
|
+
user_agent: t.OptionalCodec<t.Codec<string, string, string, t.CodecProps>>;
|
|
11
|
+
}>;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"router-socket.js","sourceRoot":"","sources":["../../src/routes/router-socket.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"router-socket.js","sourceRoot":"","sources":["../../src/routes/router-socket.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,CAAC,MAAM,UAAU,CAAC;AAS9B,MAAM,CAAC,MAAM,kBAAkB,GAAG,CAAC,CAAC,MAAM,CAAC;IACzC,KAAK,EAAE,CAAC,CAAC,MAAM;IACf,UAAU,EAAE,CAAC,CAAC,MAAM,CAAC,QAAQ,EAAE;CAChC,CAAC,CAAC"}
|