@powersync/service-core 0.0.0-dev-20240718134716 → 0.0.0-dev-20240918082156
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +89 -6
- package/dist/api/RouteAPI.d.ts +68 -0
- package/dist/api/RouteAPI.js +2 -0
- package/dist/api/RouteAPI.js.map +1 -0
- package/dist/api/api-index.d.ts +1 -0
- package/dist/api/api-index.js +1 -0
- package/dist/api/api-index.js.map +1 -1
- package/dist/api/diagnostics.d.ts +4 -4
- package/dist/api/diagnostics.js +11 -65
- package/dist/api/diagnostics.js.map +1 -1
- package/dist/api/schema.d.ts +3 -5
- package/dist/api/schema.js +9 -79
- package/dist/api/schema.js.map +1 -1
- package/dist/auth/KeyStore.d.ts +7 -4
- package/dist/auth/KeyStore.js +1 -1
- package/dist/auth/KeyStore.js.map +1 -1
- package/dist/auth/auth-index.d.ts +0 -1
- package/dist/auth/auth-index.js +0 -1
- package/dist/auth/auth-index.js.map +1 -1
- package/dist/entry/cli-entry.js +4 -2
- package/dist/entry/cli-entry.js.map +1 -1
- package/dist/entry/commands/compact-action.d.ts +2 -0
- package/dist/entry/commands/compact-action.js +52 -0
- package/dist/entry/commands/compact-action.js.map +1 -0
- package/dist/entry/commands/migrate-action.js +4 -5
- package/dist/entry/commands/migrate-action.js.map +1 -1
- package/dist/entry/commands/teardown-action.js +2 -2
- package/dist/entry/commands/teardown-action.js.map +1 -1
- package/dist/entry/entry-index.d.ts +1 -0
- package/dist/entry/entry-index.js +1 -0
- package/dist/entry/entry-index.js.map +1 -1
- package/dist/index.d.ts +4 -2
- package/dist/index.js +4 -2
- package/dist/index.js.map +1 -1
- package/dist/metrics/Metrics.d.ts +6 -5
- package/dist/metrics/Metrics.js +53 -10
- package/dist/metrics/Metrics.js.map +1 -1
- package/dist/migrations/db/migrations/1684951997326-init.d.ts +2 -2
- package/dist/migrations/db/migrations/1684951997326-init.js +4 -2
- package/dist/migrations/db/migrations/1684951997326-init.js.map +1 -1
- package/dist/migrations/db/migrations/1702295701188-sync-rule-state.d.ts +2 -2
- package/dist/migrations/db/migrations/1702295701188-sync-rule-state.js +4 -2
- package/dist/migrations/db/migrations/1702295701188-sync-rule-state.js.map +1 -1
- package/dist/migrations/db/migrations/1711543888062-write-checkpoint-index.d.ts +2 -2
- package/dist/migrations/db/migrations/1711543888062-write-checkpoint-index.js +4 -2
- package/dist/migrations/db/migrations/1711543888062-write-checkpoint-index.js.map +1 -1
- package/dist/migrations/migrations.d.ts +8 -0
- package/dist/migrations/migrations.js +19 -7
- package/dist/migrations/migrations.js.map +1 -1
- package/dist/modules/AbstractModule.d.ts +26 -0
- package/dist/modules/AbstractModule.js +11 -0
- package/dist/modules/AbstractModule.js.map +1 -0
- package/dist/modules/ModuleManager.d.ts +11 -0
- package/dist/modules/ModuleManager.js +32 -0
- package/dist/modules/ModuleManager.js.map +1 -0
- package/dist/modules/modules-index.d.ts +2 -0
- package/dist/modules/modules-index.js +3 -0
- package/dist/modules/modules-index.js.map +1 -0
- package/dist/replication/AbstractReplicationJob.d.ts +38 -0
- package/dist/replication/AbstractReplicationJob.js +51 -0
- package/dist/replication/AbstractReplicationJob.js.map +1 -0
- package/dist/replication/AbstractReplicator.d.ts +53 -0
- package/dist/replication/AbstractReplicator.js +187 -0
- package/dist/replication/AbstractReplicator.js.map +1 -0
- package/dist/replication/ErrorRateLimiter.d.ts +0 -9
- package/dist/replication/ErrorRateLimiter.js +1 -42
- package/dist/replication/ErrorRateLimiter.js.map +1 -1
- package/dist/replication/ReplicationEngine.d.ts +18 -0
- package/dist/replication/ReplicationEngine.js +41 -0
- package/dist/replication/ReplicationEngine.js.map +1 -0
- package/dist/replication/ReplicationModule.d.ts +39 -0
- package/dist/replication/ReplicationModule.js +65 -0
- package/dist/replication/ReplicationModule.js.map +1 -0
- package/dist/replication/replication-index.d.ts +4 -6
- package/dist/replication/replication-index.js +4 -6
- package/dist/replication/replication-index.js.map +1 -1
- package/dist/routes/RouterEngine.d.ts +42 -0
- package/dist/routes/RouterEngine.js +80 -0
- package/dist/routes/RouterEngine.js.map +1 -0
- package/dist/routes/auth.d.ts +2 -2
- package/dist/routes/auth.js +11 -11
- package/dist/routes/auth.js.map +1 -1
- package/dist/routes/configure-fastify.d.ts +737 -0
- package/dist/routes/configure-fastify.js +57 -0
- package/dist/routes/configure-fastify.js.map +1 -0
- package/dist/routes/configure-rsocket.d.ts +13 -0
- package/dist/routes/configure-rsocket.js +47 -0
- package/dist/routes/configure-rsocket.js.map +1 -0
- package/dist/routes/endpoints/admin.d.ts +0 -34
- package/dist/routes/endpoints/admin.js +48 -89
- package/dist/routes/endpoints/admin.js.map +1 -1
- package/dist/routes/endpoints/checkpointing.d.ts +56 -16
- package/dist/routes/endpoints/checkpointing.js +33 -12
- package/dist/routes/endpoints/checkpointing.js.map +1 -1
- package/dist/routes/endpoints/route-endpoints-index.d.ts +0 -1
- package/dist/routes/endpoints/route-endpoints-index.js +0 -1
- package/dist/routes/endpoints/route-endpoints-index.js.map +1 -1
- package/dist/routes/endpoints/socket-route.js +46 -39
- package/dist/routes/endpoints/socket-route.js.map +1 -1
- package/dist/routes/endpoints/sync-rules.d.ts +1 -1
- package/dist/routes/endpoints/sync-rules.js +32 -23
- package/dist/routes/endpoints/sync-rules.js.map +1 -1
- package/dist/routes/endpoints/sync-stream.d.ts +10 -0
- package/dist/routes/endpoints/sync-stream.js +17 -13
- package/dist/routes/endpoints/sync-stream.js.map +1 -1
- package/dist/routes/route-register.d.ts +1 -1
- package/dist/routes/route-register.js +1 -1
- package/dist/routes/route-register.js.map +1 -1
- package/dist/routes/router-socket.d.ts +5 -4
- package/dist/routes/router-socket.js +2 -1
- package/dist/routes/router-socket.js.map +1 -1
- package/dist/routes/router.d.ts +7 -2
- package/dist/routes/router.js.map +1 -1
- package/dist/routes/routes-index.d.ts +3 -0
- package/dist/routes/routes-index.js +3 -0
- package/dist/routes/routes-index.js.map +1 -1
- package/dist/runner/teardown.js +47 -76
- package/dist/runner/teardown.js.map +1 -1
- package/dist/storage/BucketStorage.d.ts +61 -20
- package/dist/storage/BucketStorage.js +0 -10
- package/dist/storage/BucketStorage.js.map +1 -1
- package/dist/storage/MongoBucketStorage.d.ts +4 -4
- package/dist/storage/MongoBucketStorage.js +19 -24
- package/dist/storage/MongoBucketStorage.js.map +1 -1
- package/dist/storage/SourceEntity.d.ts +20 -0
- package/dist/storage/SourceEntity.js +2 -0
- package/dist/storage/SourceEntity.js.map +1 -0
- package/dist/storage/SourceTable.d.ts +4 -5
- package/dist/storage/SourceTable.js +3 -4
- package/dist/storage/SourceTable.js.map +1 -1
- package/dist/storage/StorageEngine.d.ts +24 -0
- package/dist/storage/StorageEngine.js +43 -0
- package/dist/storage/StorageEngine.js.map +1 -0
- package/dist/storage/StorageProvider.d.ts +21 -0
- package/dist/storage/StorageProvider.js +2 -0
- package/dist/storage/StorageProvider.js.map +1 -0
- package/dist/storage/mongo/MongoBucketBatch.d.ts +1 -1
- package/dist/storage/mongo/MongoBucketBatch.js +6 -7
- package/dist/storage/mongo/MongoBucketBatch.js.map +1 -1
- package/dist/storage/mongo/MongoCompactor.d.ts +40 -0
- package/dist/storage/mongo/MongoCompactor.js +293 -0
- package/dist/storage/mongo/MongoCompactor.js.map +1 -0
- package/dist/storage/mongo/MongoPersistedSyncRulesContent.d.ts +2 -2
- package/dist/storage/mongo/MongoPersistedSyncRulesContent.js +2 -2
- package/dist/storage/mongo/MongoPersistedSyncRulesContent.js.map +1 -1
- package/dist/storage/mongo/MongoStorageProvider.d.ts +5 -0
- package/dist/storage/mongo/MongoStorageProvider.js +26 -0
- package/dist/storage/mongo/MongoStorageProvider.js.map +1 -0
- package/dist/storage/mongo/MongoSyncBucketStorage.d.ts +9 -7
- package/dist/storage/mongo/MongoSyncBucketStorage.js +43 -28
- package/dist/storage/mongo/MongoSyncBucketStorage.js.map +1 -1
- package/dist/storage/mongo/MongoSyncRulesLock.js +1 -1
- package/dist/storage/mongo/MongoSyncRulesLock.js.map +1 -1
- package/dist/storage/mongo/OperationBatch.d.ts +7 -3
- package/dist/storage/mongo/OperationBatch.js +16 -7
- package/dist/storage/mongo/OperationBatch.js.map +1 -1
- package/dist/storage/mongo/PersistedBatch.d.ts +3 -3
- package/dist/storage/mongo/PersistedBatch.js +2 -2
- package/dist/storage/mongo/PersistedBatch.js.map +1 -1
- package/dist/storage/mongo/models.d.ts +17 -7
- package/dist/storage/mongo/models.js.map +1 -1
- package/dist/storage/mongo/util.d.ts +14 -0
- package/dist/storage/mongo/util.js +70 -0
- package/dist/storage/mongo/util.js.map +1 -1
- package/dist/storage/storage-index.d.ts +5 -2
- package/dist/storage/storage-index.js +5 -2
- package/dist/storage/storage-index.js.map +1 -1
- package/dist/sync/RequestTracker.js +2 -3
- package/dist/sync/RequestTracker.js.map +1 -1
- package/dist/sync/sync-index.d.ts +1 -0
- package/dist/sync/sync-index.js +1 -0
- package/dist/sync/sync-index.js.map +1 -1
- package/dist/sync/sync.d.ts +2 -1
- package/dist/sync/sync.js +56 -17
- package/dist/sync/sync.js.map +1 -1
- package/dist/system/ServiceContext.d.ts +37 -0
- package/dist/system/ServiceContext.js +48 -0
- package/dist/system/ServiceContext.js.map +1 -0
- package/dist/system/system-index.d.ts +1 -1
- package/dist/system/system-index.js +1 -1
- package/dist/system/system-index.js.map +1 -1
- package/dist/util/config/collectors/config-collector.d.ts +12 -0
- package/dist/util/config/collectors/config-collector.js +43 -0
- package/dist/util/config/collectors/config-collector.js.map +1 -1
- package/dist/util/config/compound-config-collector.d.ts +10 -29
- package/dist/util/config/compound-config-collector.js +28 -84
- package/dist/util/config/compound-config-collector.js.map +1 -1
- package/dist/util/config/sync-rules/sync-rules-provider.d.ts +9 -0
- package/dist/util/config/sync-rules/sync-rules-provider.js +15 -0
- package/dist/util/config/sync-rules/sync-rules-provider.js.map +1 -0
- package/dist/util/config/types.d.ts +6 -4
- package/dist/util/config/types.js.map +1 -1
- package/dist/util/config.d.ts +3 -4
- package/dist/util/config.js +5 -20
- package/dist/util/config.js.map +1 -1
- package/dist/util/protocol-types.d.ts +4 -0
- package/dist/util/protocol-types.js +5 -1
- package/dist/util/protocol-types.js.map +1 -1
- package/dist/util/util-index.d.ts +3 -6
- package/dist/util/util-index.js +3 -6
- package/dist/util/util-index.js.map +1 -1
- package/dist/util/utils.d.ts +10 -6
- package/dist/util/utils.js +45 -25
- package/dist/util/utils.js.map +1 -1
- package/package.json +7 -7
- package/src/api/RouteAPI.ts +78 -0
- package/src/api/api-index.ts +1 -0
- package/src/api/diagnostics.ts +16 -71
- package/src/api/schema.ts +13 -89
- package/src/auth/KeyStore.ts +9 -6
- package/src/auth/auth-index.ts +0 -1
- package/src/entry/cli-entry.ts +4 -2
- package/src/entry/commands/compact-action.ts +57 -0
- package/src/entry/commands/migrate-action.ts +5 -8
- package/src/entry/commands/teardown-action.ts +2 -2
- package/src/entry/entry-index.ts +1 -0
- package/src/index.ts +5 -2
- package/src/metrics/Metrics.ts +70 -15
- package/src/migrations/db/migrations/1684951997326-init.ts +9 -4
- package/src/migrations/db/migrations/1702295701188-sync-rule-state.ts +7 -4
- package/src/migrations/db/migrations/1711543888062-write-checkpoint-index.ts +6 -4
- package/src/migrations/migrations.ts +24 -8
- package/src/modules/AbstractModule.ts +37 -0
- package/src/modules/ModuleManager.ts +34 -0
- package/src/modules/modules-index.ts +2 -0
- package/src/replication/AbstractReplicationJob.ts +79 -0
- package/src/replication/AbstractReplicator.ts +227 -0
- package/src/replication/ErrorRateLimiter.ts +0 -44
- package/src/replication/ReplicationEngine.ts +43 -0
- package/src/replication/ReplicationModule.ts +101 -0
- package/src/replication/replication-index.ts +4 -6
- package/src/routes/RouterEngine.ts +120 -0
- package/src/routes/auth.ts +21 -12
- package/src/routes/configure-fastify.ts +101 -0
- package/src/routes/configure-rsocket.ts +60 -0
- package/src/routes/endpoints/admin.ts +74 -100
- package/src/routes/endpoints/checkpointing.ts +46 -12
- package/src/routes/endpoints/route-endpoints-index.ts +0 -1
- package/src/routes/endpoints/socket-route.ts +50 -42
- package/src/routes/endpoints/sync-rules.ts +41 -25
- package/src/routes/endpoints/sync-stream.ts +17 -13
- package/src/routes/route-register.ts +2 -2
- package/src/routes/router-socket.ts +6 -5
- package/src/routes/router.ts +7 -2
- package/src/routes/routes-index.ts +3 -0
- package/src/runner/teardown.ts +50 -88
- package/src/storage/BucketStorage.ts +74 -26
- package/src/storage/MongoBucketStorage.ts +23 -26
- package/src/storage/SourceEntity.ts +22 -0
- package/src/storage/SourceTable.ts +4 -6
- package/src/storage/StorageEngine.ts +55 -0
- package/src/storage/StorageProvider.ts +27 -0
- package/src/storage/mongo/MongoBucketBatch.ts +8 -8
- package/src/storage/mongo/MongoCompactor.ts +372 -0
- package/src/storage/mongo/MongoPersistedSyncRulesContent.ts +3 -3
- package/src/storage/mongo/MongoStorageProvider.ts +31 -0
- package/src/storage/mongo/MongoSyncBucketStorage.ts +64 -34
- package/src/storage/mongo/MongoSyncRulesLock.ts +1 -1
- package/src/storage/mongo/OperationBatch.ts +18 -11
- package/src/storage/mongo/PersistedBatch.ts +6 -5
- package/src/storage/mongo/models.ts +17 -7
- package/src/storage/mongo/util.ts +71 -1
- package/src/storage/storage-index.ts +5 -2
- package/src/sync/RequestTracker.ts +3 -3
- package/src/sync/sync-index.ts +1 -0
- package/src/sync/sync.ts +66 -17
- package/src/system/ServiceContext.ts +68 -0
- package/src/system/system-index.ts +1 -1
- package/src/util/config/collectors/config-collector.ts +48 -0
- package/src/util/config/compound-config-collector.ts +45 -110
- package/src/util/config/sync-rules/sync-rules-provider.ts +18 -0
- package/src/util/config/types.ts +6 -5
- package/src/util/config.ts +6 -23
- package/src/util/protocol-types.ts +6 -1
- package/src/util/util-index.ts +3 -6
- package/src/util/utils.ts +55 -39
- package/test/src/__snapshots__/sync.test.ts.snap +90 -5
- package/test/src/auth.test.ts +7 -7
- package/test/src/broadcast_iterable.test.ts +1 -1
- package/test/src/bucket_validation.test.ts +142 -0
- package/test/src/bucket_validation.ts +116 -0
- package/test/src/checksum_cache.test.ts +3 -3
- package/test/src/compacting.test.ts +216 -0
- package/test/src/data_storage.test.ts +275 -204
- package/test/src/env.ts +1 -3
- package/test/src/merge_iterable.test.ts +1 -6
- package/test/src/setup.ts +1 -1
- package/test/src/stream_utils.ts +42 -0
- package/test/src/sync.test.ts +209 -48
- package/test/src/util.ts +110 -55
- package/test/tsconfig.json +1 -1
- package/tsconfig.tsbuildinfo +1 -1
- package/dist/auth/SupabaseKeyCollector.d.ts +0 -22
- package/dist/auth/SupabaseKeyCollector.js +0 -61
- package/dist/auth/SupabaseKeyCollector.js.map +0 -1
- package/dist/replication/PgRelation.d.ts +0 -16
- package/dist/replication/PgRelation.js +0 -26
- package/dist/replication/PgRelation.js.map +0 -1
- package/dist/replication/WalConnection.d.ts +0 -34
- package/dist/replication/WalConnection.js +0 -190
- package/dist/replication/WalConnection.js.map +0 -1
- package/dist/replication/WalStream.d.ts +0 -57
- package/dist/replication/WalStream.js +0 -517
- package/dist/replication/WalStream.js.map +0 -1
- package/dist/replication/WalStreamManager.d.ts +0 -30
- package/dist/replication/WalStreamManager.js +0 -198
- package/dist/replication/WalStreamManager.js.map +0 -1
- package/dist/replication/WalStreamRunner.d.ts +0 -38
- package/dist/replication/WalStreamRunner.js +0 -155
- package/dist/replication/WalStreamRunner.js.map +0 -1
- package/dist/replication/util.d.ts +0 -9
- package/dist/replication/util.js +0 -62
- package/dist/replication/util.js.map +0 -1
- package/dist/routes/endpoints/dev.d.ts +0 -312
- package/dist/routes/endpoints/dev.js +0 -172
- package/dist/routes/endpoints/dev.js.map +0 -1
- package/dist/system/CorePowerSyncSystem.d.ts +0 -23
- package/dist/system/CorePowerSyncSystem.js +0 -52
- package/dist/system/CorePowerSyncSystem.js.map +0 -1
- package/dist/util/PgManager.d.ts +0 -24
- package/dist/util/PgManager.js +0 -55
- package/dist/util/PgManager.js.map +0 -1
- package/dist/util/migration_lib.d.ts +0 -11
- package/dist/util/migration_lib.js +0 -64
- package/dist/util/migration_lib.js.map +0 -1
- package/dist/util/pgwire_utils.d.ts +0 -24
- package/dist/util/pgwire_utils.js +0 -117
- package/dist/util/pgwire_utils.js.map +0 -1
- package/dist/util/populate_test_data.d.ts +0 -8
- package/dist/util/populate_test_data.js +0 -65
- package/dist/util/populate_test_data.js.map +0 -1
- package/src/auth/SupabaseKeyCollector.ts +0 -67
- package/src/replication/PgRelation.ts +0 -42
- package/src/replication/WalConnection.ts +0 -227
- package/src/replication/WalStream.ts +0 -628
- package/src/replication/WalStreamManager.ts +0 -213
- package/src/replication/WalStreamRunner.ts +0 -180
- package/src/replication/util.ts +0 -76
- package/src/routes/endpoints/dev.ts +0 -199
- package/src/system/CorePowerSyncSystem.ts +0 -64
- package/src/util/PgManager.ts +0 -64
- package/src/util/migration_lib.ts +0 -79
- package/src/util/pgwire_utils.ts +0 -139
- package/src/util/populate_test_data.ts +0 -78
- package/test/src/__snapshots__/pg_test.test.ts.snap +0 -256
- package/test/src/large_batch.test.ts +0 -194
- package/test/src/pg_test.test.ts +0 -450
- package/test/src/schema_changes.test.ts +0 -545
- package/test/src/slow_tests.test.ts +0 -296
- package/test/src/validation.test.ts +0 -63
- package/test/src/wal_stream.test.ts +0 -314
- package/test/src/wal_stream_utils.ts +0 -147
package/dist/sync/sync.js
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { JSONBig, JsonContainer } from '@powersync/service-jsonbig';
|
|
2
|
-
import { Semaphore } from 'async-mutex';
|
|
2
|
+
import { Semaphore, withTimeout } from 'async-mutex';
|
|
3
3
|
import { AbortError } from 'ix/aborterror.js';
|
|
4
4
|
import * as util from '../util/util-index.js';
|
|
5
5
|
import { logger } from '@powersync/lib-services-framework';
|
|
@@ -9,9 +9,15 @@ import { tokenStream } from './util.js';
|
|
|
9
9
|
* Maximum number of connections actively fetching data.
|
|
10
10
|
*/
|
|
11
11
|
const MAX_ACTIVE_CONNECTIONS = 10;
|
|
12
|
-
|
|
12
|
+
/**
|
|
13
|
+
* Maximum duration to wait for the mutex to become available.
|
|
14
|
+
*
|
|
15
|
+
* This gives an explicit error if there are mutex issues, rather than just hanging.
|
|
16
|
+
*/
|
|
17
|
+
const MUTEX_ACQUIRE_TIMEOUT = 30000;
|
|
18
|
+
const syncSemaphore = withTimeout(new Semaphore(MAX_ACTIVE_CONNECTIONS), MUTEX_ACQUIRE_TIMEOUT, new Error(`Timeout while waiting for data`));
|
|
13
19
|
export async function* streamResponse(options) {
|
|
14
|
-
const { storage, params, syncParams, token, tokenStreamOptions, tracker, signal } = options;
|
|
20
|
+
const { storage, params, syncParams, token, tokenStreamOptions, tracker, signal, parseOptions } = options;
|
|
15
21
|
// We also need to be able to abort, so we create our own controller.
|
|
16
22
|
const controller = new AbortController();
|
|
17
23
|
if (signal) {
|
|
@@ -23,7 +29,7 @@ export async function* streamResponse(options) {
|
|
|
23
29
|
}
|
|
24
30
|
}
|
|
25
31
|
const ki = tokenStream(token, controller.signal, tokenStreamOptions);
|
|
26
|
-
const stream = streamResponseInner(storage, params, syncParams, tracker, controller.signal);
|
|
32
|
+
const stream = streamResponseInner(storage, params, syncParams, tracker, parseOptions, controller.signal);
|
|
27
33
|
// Merge the two streams, and abort as soon as one of the streams end.
|
|
28
34
|
const merged = mergeAsyncIterables([stream, ki], controller.signal);
|
|
29
35
|
try {
|
|
@@ -43,7 +49,7 @@ export async function* streamResponse(options) {
|
|
|
43
49
|
controller.abort();
|
|
44
50
|
}
|
|
45
51
|
}
|
|
46
|
-
async function* streamResponseInner(storage, params, syncParams, tracker, signal) {
|
|
52
|
+
async function* streamResponseInner(storage, params, syncParams, tracker, parseOptions, signal) {
|
|
47
53
|
// Bucket state of bucket id -> op_id.
|
|
48
54
|
// This starts with the state from the client. May contain buckets that the user do not have access to (anymore).
|
|
49
55
|
let dataBuckets = new Map();
|
|
@@ -55,7 +61,8 @@ async function* streamResponseInner(storage, params, syncParams, tracker, signal
|
|
|
55
61
|
dataBuckets.set(name, start);
|
|
56
62
|
}
|
|
57
63
|
}
|
|
58
|
-
const
|
|
64
|
+
const checkpointUserId = util.checkpointUserId(syncParams.token_parameters.user_id, params.client_id);
|
|
65
|
+
const stream = storage.watchWriteCheckpoint(checkpointUserId, signal);
|
|
59
66
|
for await (const next of stream) {
|
|
60
67
|
const { base, writeCheckpoint } = next;
|
|
61
68
|
const checkpoint = base.checkpoint;
|
|
@@ -64,8 +71,8 @@ async function* streamResponseInner(storage, params, syncParams, tracker, signal
|
|
|
64
71
|
// Sync rules deleted in the meantime - try again with the next checkpoint.
|
|
65
72
|
continue;
|
|
66
73
|
}
|
|
67
|
-
const
|
|
68
|
-
const allBuckets = await
|
|
74
|
+
const syncRules = storage.getParsedSyncRules(parseOptions);
|
|
75
|
+
const allBuckets = await syncRules.queryBucketIds({
|
|
69
76
|
getParameterSets(lookups) {
|
|
70
77
|
return storage.getParameterSets(checkpoint, lookups);
|
|
71
78
|
},
|
|
@@ -146,7 +153,8 @@ async function* streamResponseInner(storage, params, syncParams, tracker, signal
|
|
|
146
153
|
raw_data,
|
|
147
154
|
binary_data,
|
|
148
155
|
signal,
|
|
149
|
-
tracker
|
|
156
|
+
tracker,
|
|
157
|
+
user_id: syncParams.user_id
|
|
150
158
|
});
|
|
151
159
|
await new Promise((resolve) => setTimeout(resolve, 10));
|
|
152
160
|
}
|
|
@@ -189,20 +197,36 @@ async function* bucketDataInBatches(request) {
|
|
|
189
197
|
*/
|
|
190
198
|
async function* bucketDataBatch(request) {
|
|
191
199
|
const { storage, checkpoint, bucketsToFetch, dataBuckets, raw_data, binary_data, tracker, signal } = request;
|
|
192
|
-
const
|
|
200
|
+
const checkpointOp = BigInt(checkpoint);
|
|
201
|
+
let checkpointInvalidated = false;
|
|
202
|
+
if (syncSemaphore.isLocked()) {
|
|
203
|
+
logger.info('Sync concurrency limit reached, waiting for lock', { user_id: request.user_id });
|
|
204
|
+
}
|
|
205
|
+
const [value, release] = await syncSemaphore.acquire();
|
|
193
206
|
try {
|
|
207
|
+
if (value <= 3) {
|
|
208
|
+
// This can be noisy, so we only log when we get close to the
|
|
209
|
+
// concurrency limit.
|
|
210
|
+
logger.info(`Got sync lock. Slots available: ${value - 1}`, {
|
|
211
|
+
user_id: request.user_id,
|
|
212
|
+
sync_data_slots: value - 1
|
|
213
|
+
});
|
|
214
|
+
}
|
|
194
215
|
// Optimization: Only fetch buckets for which the checksums have changed since the last checkpoint
|
|
195
216
|
// For the first batch, this will be all buckets.
|
|
196
217
|
const filteredBuckets = new Map(bucketsToFetch.map((bucket) => [bucket, dataBuckets.get(bucket)]));
|
|
197
218
|
const data = storage.getBucketDataBatch(checkpoint, filteredBuckets);
|
|
198
219
|
let has_more = false;
|
|
199
|
-
for await (let r of data) {
|
|
220
|
+
for await (let { batch: r, targetOp } of data) {
|
|
200
221
|
if (signal.aborted) {
|
|
201
222
|
return;
|
|
202
223
|
}
|
|
203
224
|
if (r.has_more) {
|
|
204
225
|
has_more = true;
|
|
205
226
|
}
|
|
227
|
+
if (targetOp != null && targetOp > checkpointOp) {
|
|
228
|
+
checkpointInvalidated = true;
|
|
229
|
+
}
|
|
206
230
|
if (r.data.length == 0) {
|
|
207
231
|
continue;
|
|
208
232
|
}
|
|
@@ -237,15 +261,30 @@ async function* bucketDataBatch(request) {
|
|
|
237
261
|
dataBuckets.set(r.bucket, r.next_after);
|
|
238
262
|
}
|
|
239
263
|
if (!has_more) {
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
264
|
+
if (checkpointInvalidated) {
|
|
265
|
+
// Checkpoint invalidated by a CLEAR or MOVE op.
|
|
266
|
+
// Don't send the checkpoint_complete line in this case.
|
|
267
|
+
// More data should be available immediately for a new checkpoint.
|
|
268
|
+
yield { data: null, done: true };
|
|
269
|
+
}
|
|
270
|
+
else {
|
|
271
|
+
const line = {
|
|
272
|
+
checkpoint_complete: {
|
|
273
|
+
last_op_id: checkpoint
|
|
274
|
+
}
|
|
275
|
+
};
|
|
276
|
+
yield { data: line, done: true };
|
|
277
|
+
}
|
|
246
278
|
}
|
|
247
279
|
}
|
|
248
280
|
finally {
|
|
281
|
+
if (value <= 3) {
|
|
282
|
+
// This can be noisy, so we only log when we get close to the
|
|
283
|
+
// concurrency limit.
|
|
284
|
+
logger.info(`Releasing sync lock`, {
|
|
285
|
+
user_id: request.user_id
|
|
286
|
+
});
|
|
287
|
+
}
|
|
249
288
|
release();
|
|
250
289
|
}
|
|
251
290
|
}
|
package/dist/sync/sync.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"sync.js","sourceRoot":"","sources":["../../src/sync/sync.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,aAAa,EAAE,MAAM,4BAA4B,CAAC;AAEpE,OAAO,EAAE,SAAS,EAAE,MAAM,aAAa,CAAC;
|
|
1
|
+
{"version":3,"file":"sync.js","sourceRoot":"","sources":["../../src/sync/sync.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,aAAa,EAAE,MAAM,4BAA4B,CAAC;AAEpE,OAAO,EAAE,SAAS,EAAE,WAAW,EAAE,MAAM,aAAa,CAAC;AAErD,OAAO,EAAE,UAAU,EAAE,MAAM,kBAAkB,CAAC;AAI9C,OAAO,KAAK,IAAI,MAAM,uBAAuB,CAAC;AAE9C,OAAO,EAAE,MAAM,EAAE,MAAM,mCAAmC,CAAC;AAC3D,OAAO,EAAE,mBAAmB,EAAE,MAAM,YAAY,CAAC;AAEjD,OAAO,EAAsB,WAAW,EAAE,MAAM,WAAW,CAAC;AAE5D;;GAEG;AACH,MAAM,sBAAsB,GAAG,EAAE,CAAC;AAElC;;;;GAIG;AACH,MAAM,qBAAqB,GAAG,KAAM,CAAC;AAErC,MAAM,aAAa,GAAG,WAAW,CAC/B,IAAI,SAAS,CAAC,sBAAsB,CAAC,EACrC,qBAAqB,EACrB,IAAI,KAAK,CAAC,gCAAgC,CAAC,CAC5C,CAAC;AAiBF,MAAM,CAAC,KAAK,SAAS,CAAC,CAAC,cAAc,CACnC,OAA6B;IAE7B,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,UAAU,EAAE,KAAK,EAAE,kBAAkB,EAAE,OAAO,EAAE,MAAM,EAAE,YAAY,EAAE,GAAG,OAAO,CAAC;IAC1G,qEAAqE;IACrE,MAAM,UAAU,GAAG,IAAI,eAAe,EAAE,CAAC;IACzC,IAAI,MAAM,EAAE;QACV,MAAM,CAAC,gBAAgB,CACrB,OAAO,EACP,GAAG,EAAE;YACH,UAAU,CAAC,KAAK,EAAE,CAAC;QACrB,CAAC,EACD,EAAE,IAAI,EAAE,IAAI,EAAE,CACf,CAAC;QACF,IAAI,MAAM,CAAC,OAAO,EAAE;YAClB,UAAU,CAAC,KAAK,EAAE,CAAC;SACpB;KACF;IACD,MAAM,EAAE,GAAG,WAAW,CAAC,KAAK,EAAE,UAAU,CAAC,MAAM,EAAE,kBAAkB,CAAC,CAAC;IACrE,MAAM,MAAM,GAAG,mBAAmB,CAAC,OAAO,EAAE,MAAM,EAAE,UAAU,EAAE,OAAO,EAAE,YAAY,EAAE,UAAU,CAAC,MAAM,CAAC,CAAC;IAC1G,sEAAsE;IACtE,MAAM,MAAM,GAAG,mBAAmB,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,EAAE,UAAU,CAAC,MAAM,CAAC,CAAC;IAEpE,IAAI;QACF,KAAK,CAAC,CAAC,MAAM,CAAC;KACf;IAAC,OAAO,CAAC,EAAE;QACV,IAAI,CAAC,YAAY,UAAU,EAAE;YAC3B,OAAO;SACR;aAAM;YACL,MAAM,CAAC,CAAC;SACT;KACF;YAAS;QACR,iFAAiF;QACjF,qBAAqB;QACrB,UAAU,CAAC,KAAK,EAAE,CAAC;KACpB;AACH,CAAC;AAED,KAAK,SAAS,CAAC,CAAC,mBAAmB,CACjC,OAAqC,EACrC,MAAiC,EACjC,UAA6B,EAC7B,OAAuB,EACvB,YAA2C,EAC3C,MAAmB;IAEnB,sCAAsC;IACtC,iHAAiH;IACjH,IAAI,WAAW,GAAG,IAAI,GAAG,EAAkB,CAAC;IAE5C,IAAI,aAAa,GAA4B,IAAI,CAAC;IAClD,IAAI,mBAAmB,GAAkB,IAAI,CAAC;IAE9C,MAAM,EAAE,QAAQ,EAAE,WAAW,EAAE,GAAG,MAAM,CAAC;IAEzC,IAAI,MAAM,CAAC,OAAO,EAAE;QAClB,KAAK,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,KAAK,EAAE,IAAI,MAAM,CAAC,OAAO,EAAE;YACjD,WAAW,CAAC,GAAG,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;SAC9B;KACF;IAED,MAAM,gBAAgB,GAAG,IAAI,CAAC,gBAAgB,CAAC,UAAU,CAAC,gBAAgB,CAAC,OAAiB,EAAE,MAAM,CAAC,SAAS,CAAC,CAAC;IAChH,MAAM,MAAM,GAAG,OAAO,CAAC,oBAAoB,CAAC,gBAAgB,EAAE,MAAM,CAAC,CAAC;IACtE,IAAI,KAAK,EAAE,MAAM,IAAI,IAAI,MAAM,EAAE;QAC/B,MAAM,EAAE,IAAI,EAAE,eAAe,EAAE,GAAG,IAAI,CAAC;QACvC,MAAM,UAAU,GAAG,IAAI,CAAC,UAAU,CAAC;QAEnC,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,gBAAgB,EAAE,CAAC;QAC9C,IAAI,OAAO,IAAI,IAAI,EAAE;YACnB,2EAA2E;YAC3E,SAAS;SACV;QACD,MAAM,SAAS,GAAG,OAAO,CAAC,kBAAkB,CAAC,YAAY,CAAC,CAAC;QAE3D,MAAM,UAAU,GAAG,MAAM,SAAS,CAAC,cAAc,CAAC;YAChD,gBAAgB,CAAC,OAAO;gBACtB,OAAO,OAAO,CAAC,gBAAgB,CAAC,UAAU,EAAE,OAAO,CAAC,CAAC;YACvD,CAAC;YACD,UAAU,EAAE,UAAU;SACvB,CAAC,CAAC;QAEH,IAAI,UAAU,CAAC,MAAM,GAAG,IAAI,EAAE;YAC5B,MAAM,CAAC,KAAK,CAAC,kBAAkB,EAAE;gBAC/B,UAAU;gBACV,OAAO,EAAE,UAAU,CAAC,OAAO;gBAC3B,OAAO,EAAE,UAAU,CAAC,MAAM;aAC3B,CAAC,CAAC;YACH,iEAAiE;YACjE,MAAM,IAAI,KAAK,CAAC,qBAAqB,UAAU,CAAC,MAAM,EAAE,CAAC,CAAC;SAC3D;QAED,IAAI,cAAc,GAAG,IAAI,GAAG,EAAkB,CAAC;QAC/C,KAAK,IAAI,MAAM,IAAI,UAAU,EAAE;YAC7B,cAAc,CAAC,GAAG,CAAC,MAAM,EAAE,WAAW,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,GAAG,CAAC,CAAC;SAC5D;QACD,WAAW,GAAG,cAAc,CAAC;QAE7B,MAAM,UAAU,GAAG,CAAC,GAAG,WAAW,CAAC,IAAI,EAAE,CAAC,CAAC;QAC3C,MAAM,WAAW,GAAG,MAAM,OAAO,CAAC,YAAY,CAAC,UAAU,EAAE,UAAU,CAAC,CAAC;QACvE,mEAAmE;QACnE,IAAI,cAAwB,CAAC;QAE7B,IAAI,aAAa,EAAE;YACjB,MAAM,IAAI,GAAG,IAAI,CAAC,aAAa,CAAC,aAAa,EAAE,WAAW,CAAC,CAAC;YAE5D,IACE,mBAAmB,IAAI,eAAe;gBACtC,IAAI,CAAC,cAAc,CAAC,MAAM,IAAI,CAAC;gBAC/B,IAAI,CAAC,cAAc,CAAC,MAAM,IAAI,CAAC,EAC/B;gBACA,iDAAiD;gBACjD,SAAS;aACV;YACD,cAAc,GAAG,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC;YAE1D,IAAI,OAAO,GAAG,uBAAuB,UAAU,KAAK,CAAC;YACrD,OAAO,IAAI,UAAU,eAAe,KAAK,CAAC;YAC1C,OAAO,IAAI,YAAY,UAAU,CAAC,MAAM,KAAK,CAAC;YAC9C,OAAO,IAAI,YAAY,cAAc,CAAC,IAAI,CAAC,cAAc,EAAE,EAAE,CAAC,KAAK,CAAC;YACpE,OAAO,IAAI,YAAY,cAAc,CAAC,IAAI,CAAC,cAAc,EAAE,EAAE,CAAC,EAAE,CAAC;YACjE,MAAM,CAAC,IAAI,CAAC,OAAO,EAAE;gBACnB,UAAU;gBACV,OAAO,EAAE,UAAU,CAAC,OAAO;gBAC3B,OAAO,EAAE,UAAU,CAAC,MAAM;gBAC1B,OAAO,EAAE,IAAI,CAAC,cAAc,CAAC,MAAM;gBACnC,OAAO,EAAE,IAAI,CAAC,cAAc,CAAC,MAAM;aACpC,CAAC,CAAC;YAEH,MAAM,aAAa,GAAqC;gBACtD,eAAe,EAAE;oBACf,UAAU,EAAE,UAAU;oBACtB,gBAAgB,EAAE,eAAe,CAAC,CAAC,CAAC,MAAM,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC,SAAS;oBACvE,eAAe,EAAE,IAAI,CAAC,cAAc;oBACpC,eAAe,EAAE,IAAI,CAAC,cAAc;iBACrC;aACF,CAAC;YAEF,MAAM,aAAa,CAAC;SACrB;aAAM;YACL,IAAI,OAAO,GAAG,mBAAmB,UAAU,aAAa,eAAe,KAAK,CAAC;YAC7E,OAAO,IAAI,YAAY,UAAU,CAAC,MAAM,IAAI,cAAc,CAAC,UAAU,EAAE,EAAE,CAAC,EAAE,CAAC;YAC7E,MAAM,CAAC,IAAI,CAAC,OAAO,EAAE,EAAE,UAAU,EAAE,OAAO,EAAE,UAAU,CAAC,OAAO,EAAE,OAAO,EAAE,UAAU,CAAC,MAAM,EAAE,CAAC,CAAC;YAC9F,cAAc,GAAG,UAAU,CAAC;YAC5B,MAAM,aAAa,GAAiC;gBAClD,UAAU,EAAE;oBACV,UAAU,EAAE,UAAU;oBACtB,gBAAgB,EAAE,eAAe,CAAC,CAAC,CAAC,MAAM,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC,SAAS;oBACvE,OAAO,EAAE,CAAC,GAAG,WAAW,CAAC,MAAM,EAAE,CAAC;iBACnC;aACF,CAAC;YACF,MAAM,aAAa,CAAC;SACrB;QACD,aAAa,GAAG,WAAW,CAAC;QAC5B,mBAAmB,GAAG,eAAe,CAAC;QAEtC,+EAA+E;QAC/E,sFAAsF;QACtF,KAAK,CAAC,CAAC,mBAAmB,CAAC;YACzB,OAAO;YACP,UAAU;YACV,cAAc;YACd,WAAW;YACX,QAAQ;YACR,WAAW;YACX,MAAM;YACN,OAAO;YACP,OAAO,EAAE,UAAU,CAAC,OAAO;SAC5B,CAAC,CAAC;QAEH,MAAM,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,CAAC;KACzD;AACH,CAAC;AAeD,KAAK,SAAS,CAAC,CAAC,mBAAmB,CAAC,OAA0B;IAC5D,IAAI,MAAM,GAAG,KAAK,CAAC;IACnB,OAAO,CAAC,OAAO,CAAC,MAAM,CAAC,OAAO,IAAI,CAAC,MAAM,EAAE;QACzC,wEAAwE;QACxE,6EAA6E;QAC7E,4GAA4G;QAC5G,gBAAgB;QAChB,gBAAgB;QAChB,qBAAqB;QACrB,MAAM;QACN,WAAW;QACX,IAAI;QACJ,MAAM,IAAI,GAAG,eAAe,CAAC,OAAO,CAAC,CAAC;QACtC,IAAI;YACF,OAAO,IAAI,EAAE;gBACX,MAAM,EAAE,KAAK,EAAE,IAAI,EAAE,QAAQ,EAAE,GAAG,MAAM,IAAI,CAAC,IAAI,EAAE,CAAC;gBACpD,IAAI,QAAQ,EAAE;oBACZ,MAAM;iBACP;qBAAM;oBACL,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,GAAG,KAAK,CAAC;oBAC7B,MAAM,IAAI,CAAC;oBACX,IAAI,IAAI,EAAE;wBACR,MAAM,GAAG,IAAI,CAAC;qBACf;iBACF;aACF;SACF;gBAAS;YACR,MAAM,IAAI,CAAC,MAAM,EAAE,CAAC;SACrB;KACF;AACH,CAAC;AAOD;;GAEG;AACH,KAAK,SAAS,CAAC,CAAC,eAAe,CAAC,OAA0B;IACxD,MAAM,EAAE,OAAO,EAAE,UAAU,EAAE,cAAc,EAAE,WAAW,EAAE,QAAQ,EAAE,WAAW,EAAE,OAAO,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC;IAE7G,MAAM,YAAY,GAAG,MAAM,CAAC,UAAU,CAAC,CAAC;IACxC,IAAI,qBAAqB,GAAG,KAAK,CAAC;IAElC,IAAI,aAAa,CAAC,QAAQ,EAAE,EAAE;QAC5B,MAAM,CAAC,IAAI,CAAC,kDAAkD,EAAE,EAAE,OAAO,EAAE,OAAO,CAAC,OAAO,EAAE,CAAC,CAAC;KAC/F;IACD,MAAM,CAAC,KAAK,EAAE,OAAO,CAAC,GAAG,MAAM,aAAa,CAAC,OAAO,EAAE,CAAC;IACvD,IAAI;QACF,IAAI,KAAK,IAAI,CAAC,EAAE;YACd,6DAA6D;YAC7D,qBAAqB;YACrB,MAAM,CAAC,IAAI,CAAC,mCAAmC,KAAK,GAAG,CAAC,EAAE,EAAE;gBAC1D,OAAO,EAAE,OAAO,CAAC,OAAO;gBACxB,eAAe,EAAE,KAAK,GAAG,CAAC;aAC3B,CAAC,CAAC;SACJ;QACD,kGAAkG;QAClG,iDAAiD;QACjD,MAAM,eAAe,GAAG,IAAI,GAAG,CAAC,cAAc,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,MAAM,EAAE,WAAW,CAAC,GAAG,CAAC,MAAM,CAAE,CAAC,CAAC,CAAC,CAAC;QACpG,MAAM,IAAI,GAAG,OAAO,CAAC,kBAAkB,CAAC,UAAU,EAAE,eAAe,CAAC,CAAC;QAErE,IAAI,QAAQ,GAAG,KAAK,CAAC;QAErB,IAAI,KAAK,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,EAAE,QAAQ,EAAE,IAAI,IAAI,EAAE;YAC7C,IAAI,MAAM,CAAC,OAAO,EAAE;gBAClB,OAAO;aACR;YACD,IAAI,CAAC,CAAC,QAAQ,EAAE;gBACd,QAAQ,GAAG,IAAI,CAAC;aACjB;YACD,IAAI,QAAQ,IAAI,IAAI,IAAI,QAAQ,GAAG,YAAY,EAAE;gBAC/C,qBAAqB,GAAG,IAAI,CAAC;aAC9B;YACD,IAAI,CAAC,CAAC,IAAI,CAAC,MAAM,IAAI,CAAC,EAAE;gBACtB,SAAS;aACV;YACD,MAAM,CAAC,KAAK,CAAC,oBAAoB,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC;YAE7C,IAAI,SAAc,CAAC;YACnB,IAAI,WAAW,EAAE;gBACf,wEAAwE;gBACxE,SAAS,GAAG,EAAE,IAAI,EAAE,CAAC,EAAE,CAAC;aACzB;iBAAM,IAAI,QAAQ,EAAE;gBACnB,uEAAuE;gBACvE,MAAM,QAAQ,GAA2B;oBACvC,IAAI,EAAE,CAAC;iBACR,CAAC;gBACF,SAAS,GAAG,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,CAAC;aACtC;iBAAM;gBACL,8EAA8E;gBAC9E,4BAA4B;gBAC5B,MAAM,QAAQ,GAA2B;oBACvC,IAAI,EAAE,uBAAuB,CAAC,CAAC,CAAC;iBACjC,CAAC;gBACF,SAAS,GAAG,OAAO,CAAC,SAAS,CAAC,QAAQ,CAAC,CAAC;aACzC;YACD,MAAM,EAAE,IAAI,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC;YACvC,IAAI,SAAS,CAAC,MAAM,GAAG,KAAM,EAAE;gBAC7B,0EAA0E;gBAC1E,8CAA8C;gBAC9C,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC;aACnC;YACD,OAAO,CAAC,mBAAmB,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YAE3C,WAAW,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,UAAU,CAAC,CAAC;SACzC;QAED,IAAI,CAAC,QAAQ,EAAE;YACb,IAAI,qBAAqB,EAAE;gBACzB,gDAAgD;gBAChD,wDAAwD;gBACxD,kEAAkE;gBAClE,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC;aAClC;iBAAM;gBACL,MAAM,IAAI,GAAyC;oBACjD,mBAAmB,EAAE;wBACnB,UAAU,EAAE,UAAU;qBACvB;iBACF,CAAC;gBACF,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC;aAClC;SACF;KACF;YAAS;QACR,IAAI,KAAK,IAAI,CAAC,EAAE;YACd,6DAA6D;YAC7D,qBAAqB;YACrB,MAAM,CAAC,IAAI,CAAC,qBAAqB,EAAE;gBACjC,OAAO,EAAE,OAAO,CAAC,OAAO;aACzB,CAAC,CAAC;SACJ;QACD,OAAO,EAAE,CAAC;KACX;AACH,CAAC;AAED,SAAS,uBAAuB,CAAC,UAA+B;IAC9D,OAAO;QACL,GAAG,UAAU;QACb,IAAI,EAAE,UAAU,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE;YAClC,OAAO;gBACL,GAAG,KAAK;gBACR,IAAI,EAAE,KAAK,CAAC,IAAI,IAAI,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,aAAa,CAAC,KAAK,CAAC,IAAc,CAAC;gBACzE,QAAQ,EAAE,MAAM,CAAC,KAAK,CAAC,QAAQ,CAAC;aACjC,CAAC;QACJ,CAAC,CAAC;KACH,CAAC;AACJ,CAAC;AAED,SAAS,cAAc,CAAC,OAAyC,EAAE,KAAa;IAC9E,OAAO,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE;QAC1B,IAAI,OAAO,CAAC,IAAI,QAAQ,EAAE;YACxB,OAAO,CAAC,CAAC,MAAM,CAAC;SACjB;aAAM;YACL,OAAO,CAAC,CAAC;SACV;IACH,CAAC,CAAC,CAAC;IACH,IAAI,OAAO,CAAC,MAAM,IAAI,KAAK,EAAE;QAC3B,OAAO,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC;KAChC;IACD,MAAM,OAAO,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC;IACxC,OAAO,GAAG,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,KAAK,CAAC;AACzC,CAAC"}
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
import { LifeCycledSystem, ServiceIdentifier } from '@powersync/lib-services-framework';
|
|
2
|
+
import * as metrics from '../metrics/Metrics.js';
|
|
3
|
+
import * as replication from '../replication/replication-index.js';
|
|
4
|
+
import * as routes from '../routes/routes-index.js';
|
|
5
|
+
import * as storage from '../storage/storage-index.js';
|
|
6
|
+
import * as utils from '../util/util-index.js';
|
|
7
|
+
export interface ServiceContext {
|
|
8
|
+
configuration: utils.ResolvedPowerSyncConfig;
|
|
9
|
+
lifeCycleEngine: LifeCycledSystem;
|
|
10
|
+
metrics: metrics.Metrics | null;
|
|
11
|
+
replicationEngine: replication.ReplicationEngine | null;
|
|
12
|
+
routerEngine: routes.RouterEngine | null;
|
|
13
|
+
storageEngine: storage.StorageEngine;
|
|
14
|
+
}
|
|
15
|
+
/**
|
|
16
|
+
* Context which allows for registering and getting implementations
|
|
17
|
+
* of various service engines.
|
|
18
|
+
* This controls registering, initializing and the lifecycle of various services.
|
|
19
|
+
*/
|
|
20
|
+
export declare class ServiceContextContainer implements ServiceContext {
|
|
21
|
+
configuration: utils.ResolvedPowerSyncConfig;
|
|
22
|
+
lifeCycleEngine: LifeCycledSystem;
|
|
23
|
+
storageEngine: storage.StorageEngine;
|
|
24
|
+
constructor(configuration: utils.ResolvedPowerSyncConfig);
|
|
25
|
+
get replicationEngine(): replication.ReplicationEngine | null;
|
|
26
|
+
get routerEngine(): routes.RouterEngine | null;
|
|
27
|
+
get metrics(): metrics.Metrics | null;
|
|
28
|
+
/**
|
|
29
|
+
* Allows for registering core and generic implementations of services/helpers.
|
|
30
|
+
* This uses the framework container under the hood.
|
|
31
|
+
*/
|
|
32
|
+
register<T>(identifier: ServiceIdentifier<T>, implementation: T): void;
|
|
33
|
+
/**
|
|
34
|
+
* Gets the implementation of an identifiable service.
|
|
35
|
+
*/
|
|
36
|
+
get<T>(identifier: ServiceIdentifier<T>): T;
|
|
37
|
+
}
|
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
import { LifeCycledSystem, container } from '@powersync/lib-services-framework';
|
|
2
|
+
import * as metrics from '../metrics/Metrics.js';
|
|
3
|
+
import * as replication from '../replication/replication-index.js';
|
|
4
|
+
import * as routes from '../routes/routes-index.js';
|
|
5
|
+
import * as storage from '../storage/storage-index.js';
|
|
6
|
+
/**
|
|
7
|
+
* Context which allows for registering and getting implementations
|
|
8
|
+
* of various service engines.
|
|
9
|
+
* This controls registering, initializing and the lifecycle of various services.
|
|
10
|
+
*/
|
|
11
|
+
export class ServiceContextContainer {
|
|
12
|
+
constructor(configuration) {
|
|
13
|
+
this.configuration = configuration;
|
|
14
|
+
this.lifeCycleEngine = new LifeCycledSystem();
|
|
15
|
+
this.storageEngine = new storage.StorageEngine({
|
|
16
|
+
configuration
|
|
17
|
+
});
|
|
18
|
+
this.lifeCycleEngine.withLifecycle(this.storageEngine, {
|
|
19
|
+
start: (storageEngine) => storageEngine.start(),
|
|
20
|
+
stop: (storageEngine) => storageEngine.shutDown()
|
|
21
|
+
});
|
|
22
|
+
// Mongo storage is available as an option by default TODO: Consider moving this to a Mongo Storage Module
|
|
23
|
+
this.storageEngine.registerProvider(new storage.MongoStorageProvider());
|
|
24
|
+
}
|
|
25
|
+
get replicationEngine() {
|
|
26
|
+
return container.getOptional(replication.ReplicationEngine);
|
|
27
|
+
}
|
|
28
|
+
get routerEngine() {
|
|
29
|
+
return container.getOptional(routes.RouterEngine);
|
|
30
|
+
}
|
|
31
|
+
get metrics() {
|
|
32
|
+
return container.getOptional(metrics.Metrics);
|
|
33
|
+
}
|
|
34
|
+
/**
|
|
35
|
+
* Allows for registering core and generic implementations of services/helpers.
|
|
36
|
+
* This uses the framework container under the hood.
|
|
37
|
+
*/
|
|
38
|
+
register(identifier, implementation) {
|
|
39
|
+
container.register(identifier, implementation);
|
|
40
|
+
}
|
|
41
|
+
/**
|
|
42
|
+
* Gets the implementation of an identifiable service.
|
|
43
|
+
*/
|
|
44
|
+
get(identifier) {
|
|
45
|
+
return container.getImplementation(identifier);
|
|
46
|
+
}
|
|
47
|
+
}
|
|
48
|
+
//# sourceMappingURL=ServiceContext.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"ServiceContext.js","sourceRoot":"","sources":["../../src/system/ServiceContext.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,gBAAgB,EAAqB,SAAS,EAAE,MAAM,mCAAmC,CAAC;AAEnG,OAAO,KAAK,OAAO,MAAM,uBAAuB,CAAC;AACjD,OAAO,KAAK,WAAW,MAAM,qCAAqC,CAAC;AACnE,OAAO,KAAK,MAAM,MAAM,2BAA2B,CAAC;AACpD,OAAO,KAAK,OAAO,MAAM,6BAA6B,CAAC;AAYvD;;;;GAIG;AACH,MAAM,OAAO,uBAAuB;IAIlC,YAAmB,aAA4C;QAA5C,kBAAa,GAAb,aAAa,CAA+B;QAC7D,IAAI,CAAC,eAAe,GAAG,IAAI,gBAAgB,EAAE,CAAC;QAE9C,IAAI,CAAC,aAAa,GAAG,IAAI,OAAO,CAAC,aAAa,CAAC;YAC7C,aAAa;SACd,CAAC,CAAC;QACH,IAAI,CAAC,eAAe,CAAC,aAAa,CAAC,IAAI,CAAC,aAAa,EAAE;YACrD,KAAK,EAAE,CAAC,aAAa,EAAE,EAAE,CAAC,aAAa,CAAC,KAAK,EAAE;YAC/C,IAAI,EAAE,CAAC,aAAa,EAAE,EAAE,CAAC,aAAa,CAAC,QAAQ,EAAE;SAClD,CAAC,CAAC;QAEH,0GAA0G;QAC1G,IAAI,CAAC,aAAa,CAAC,gBAAgB,CAAC,IAAI,OAAO,CAAC,oBAAoB,EAAE,CAAC,CAAC;IAC1E,CAAC;IAED,IAAI,iBAAiB;QACnB,OAAO,SAAS,CAAC,WAAW,CAAC,WAAW,CAAC,iBAAiB,CAAC,CAAC;IAC9D,CAAC;IAED,IAAI,YAAY;QACd,OAAO,SAAS,CAAC,WAAW,CAAC,MAAM,CAAC,YAAY,CAAC,CAAC;IACpD,CAAC;IAED,IAAI,OAAO;QACT,OAAO,SAAS,CAAC,WAAW,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;IAChD,CAAC;IAED;;;OAGG;IACH,QAAQ,CAAI,UAAgC,EAAE,cAAiB;QAC7D,SAAS,CAAC,QAAQ,CAAC,UAAU,EAAE,cAAc,CAAC,CAAC;IACjD,CAAC;IAED;;OAEG;IACH,GAAG,CAAI,UAAgC;QACrC,OAAO,SAAS,CAAC,iBAAiB,CAAC,UAAU,CAAC,CAAC;IACjD,CAAC;CACF"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
export * from './
|
|
1
|
+
export * from './ServiceContext.js';
|
|
@@ -1,2 +1,2 @@
|
|
|
1
|
-
export * from './
|
|
1
|
+
export * from './ServiceContext.js';
|
|
2
2
|
//# sourceMappingURL=system-index.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"system-index.js","sourceRoot":"","sources":["../../src/system/system-index.ts"],"names":[],"mappings":"AAAA,cAAc,
|
|
1
|
+
{"version":3,"file":"system-index.js","sourceRoot":"","sources":["../../src/system/system-index.ts"],"names":[],"mappings":"AAAA,cAAc,qBAAqB,CAAC"}
|
|
@@ -11,6 +11,18 @@ export declare abstract class ConfigCollector {
|
|
|
11
11
|
* @returns null if this collector cannot provide a config
|
|
12
12
|
*/
|
|
13
13
|
abstract collectSerialized(runnerConfig: RunnerConfig): Promise<configFile.SerializedPowerSyncConfig | null>;
|
|
14
|
+
/**
|
|
15
|
+
* Collects the PowerSyncConfig settings.
|
|
16
|
+
* Validates and decodes the config.
|
|
17
|
+
* @returns null if this collector cannot provide a config
|
|
18
|
+
*/
|
|
19
|
+
collect(runner_config: RunnerConfig): Promise<configFile.PowerSyncConfig | null>;
|
|
20
|
+
/**
|
|
21
|
+
* Validates input config
|
|
22
|
+
* ts-codec itself doesn't give great validation errors, so we use json schema for that
|
|
23
|
+
*/
|
|
24
|
+
validate(config: configFile.PowerSyncConfig): void;
|
|
25
|
+
decode(encoded: configFile.SerializedPowerSyncConfig): configFile.PowerSyncConfig;
|
|
14
26
|
protected parseContent(content: string, contentType?: ConfigFileFormat): any;
|
|
15
27
|
protected parseYaml(content: string): any;
|
|
16
28
|
protected parseJSON(content: string): any;
|
|
@@ -1,4 +1,7 @@
|
|
|
1
|
+
import * as t from 'ts-codec';
|
|
1
2
|
import * as yaml from 'yaml';
|
|
3
|
+
import { configFile } from '@powersync/service-types';
|
|
4
|
+
import { schema } from '@powersync/lib-services-framework';
|
|
2
5
|
export var ConfigFileFormat;
|
|
3
6
|
(function (ConfigFileFormat) {
|
|
4
7
|
ConfigFileFormat["YAML"] = "yaml";
|
|
@@ -15,7 +18,47 @@ export var ConfigFileFormat;
|
|
|
15
18
|
* uri: !env PS_MONGO_URI
|
|
16
19
|
*/
|
|
17
20
|
const YAML_ENV_PREFIX = 'PS_';
|
|
21
|
+
// ts-codec itself doesn't give great validation errors, so we use json schema for that
|
|
22
|
+
const configSchemaValidator = schema
|
|
23
|
+
.parseJSONSchema(t.generateJSONSchema(configFile.powerSyncConfig, { allowAdditional: true, parsers: [configFile.portParser] }))
|
|
24
|
+
.validator();
|
|
18
25
|
export class ConfigCollector {
|
|
26
|
+
/**
|
|
27
|
+
* Collects the PowerSyncConfig settings.
|
|
28
|
+
* Validates and decodes the config.
|
|
29
|
+
* @returns null if this collector cannot provide a config
|
|
30
|
+
*/
|
|
31
|
+
async collect(runner_config) {
|
|
32
|
+
const serialized = await this.collectSerialized(runner_config);
|
|
33
|
+
if (!serialized) {
|
|
34
|
+
return null;
|
|
35
|
+
}
|
|
36
|
+
/**
|
|
37
|
+
* After this point a serialized config has been found. Any failures to decode or validate
|
|
38
|
+
* will result in a hard stop.
|
|
39
|
+
*/
|
|
40
|
+
const decoded = this.decode(serialized);
|
|
41
|
+
this.validate(decoded);
|
|
42
|
+
return decoded;
|
|
43
|
+
}
|
|
44
|
+
/**
|
|
45
|
+
* Validates input config
|
|
46
|
+
* ts-codec itself doesn't give great validation errors, so we use json schema for that
|
|
47
|
+
*/
|
|
48
|
+
validate(config) {
|
|
49
|
+
const valid = configSchemaValidator.validate(config);
|
|
50
|
+
if (!valid.valid) {
|
|
51
|
+
throw new Error(`Failed to validate PowerSync config: ${valid.errors.join(', ')}`);
|
|
52
|
+
}
|
|
53
|
+
}
|
|
54
|
+
decode(encoded) {
|
|
55
|
+
try {
|
|
56
|
+
return configFile.powerSyncConfig.decode(encoded);
|
|
57
|
+
}
|
|
58
|
+
catch (ex) {
|
|
59
|
+
throw new Error(`Failed to decode PowerSync config: ${ex}`);
|
|
60
|
+
}
|
|
61
|
+
}
|
|
19
62
|
parseContent(content, contentType) {
|
|
20
63
|
switch (contentType) {
|
|
21
64
|
case ConfigFileFormat.YAML:
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"config-collector.js","sourceRoot":"","sources":["../../../../src/util/config/collectors/config-collector.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,IAAI,MAAM,MAAM,CAAC;
|
|
1
|
+
{"version":3,"file":"config-collector.js","sourceRoot":"","sources":["../../../../src/util/config/collectors/config-collector.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,CAAC,MAAM,UAAU,CAAC;AAC9B,OAAO,KAAK,IAAI,MAAM,MAAM,CAAC;AAE7B,OAAO,EAAE,UAAU,EAAE,MAAM,0BAA0B,CAAC;AACtD,OAAO,EAAE,MAAM,EAAE,MAAM,mCAAmC,CAAC;AAI3D,MAAM,CAAN,IAAY,gBAGX;AAHD,WAAY,gBAAgB;IAC1B,iCAAa,CAAA;IACb,iCAAa,CAAA;AACf,CAAC,EAHW,gBAAgB,KAAhB,gBAAgB,QAG3B;AAED;;;;;;;;;GASG;AACH,MAAM,eAAe,GAAG,KAAK,CAAC;AAE9B,uFAAuF;AACvF,MAAM,qBAAqB,GAAG,MAAM;KACjC,eAAe,CACd,CAAC,CAAC,kBAAkB,CAAC,UAAU,CAAC,eAAe,EAAE,EAAE,eAAe,EAAE,IAAI,EAAE,OAAO,EAAE,CAAC,UAAU,CAAC,UAAU,CAAC,EAAE,CAAC,CAC9G;KACA,SAAS,EAAE,CAAC;AAEf,MAAM,OAAgB,eAAe;IASnC;;;;OAIG;IACH,KAAK,CAAC,OAAO,CAAC,aAA2B;QACvC,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,iBAAiB,CAAC,aAAa,CAAC,CAAC;QAC/D,IAAI,CAAC,UAAU,EAAE;YACf,OAAO,IAAI,CAAC;SACb;QAED;;;WAGG;QACH,MAAM,OAAO,GAAG,IAAI,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC;QACxC,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC;QACvB,OAAO,OAAO,CAAC;IACjB,CAAC;IAED;;;OAGG;IACH,QAAQ,CAAC,MAAkC;QACzC,MAAM,KAAK,GAAG,qBAAqB,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;QACrD,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE;YAChB,MAAM,IAAI,KAAK,CAAC,wCAAwC,KAAK,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;SACpF;IACH,CAAC;IAED,MAAM,CAAC,OAA6C;QAClD,IAAI;YACF,OAAO,UAAU,CAAC,eAAe,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;SACnD;QAAC,OAAO,EAAE,EAAE;YACX,MAAM,IAAI,KAAK,CAAC,sCAAsC,EAAE,EAAE,CAAC,CAAC;SAC7D;IACH,CAAC;IAES,YAAY,CAAC,OAAe,EAAE,WAA8B;QACpE,QAAQ,WAAW,EAAE;YACnB,KAAK,gBAAgB,CAAC,IAAI;gBACxB,OAAO,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC;YACjC,KAAK,gBAAgB,CAAC,IAAI;gBACxB,OAAO,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC;YACjC,OAAO,CAAC,CAAC;gBACP,6CAA6C;gBAC7C,IAAI;oBACF,OAAO,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC;iBAChC;gBAAC,OAAO,EAAE,EAAE,GAAE;gBACf,IAAI;oBACF,OAAO,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC;iBAChC;gBAAC,OAAO,EAAE,EAAE;oBACX,MAAM,IAAI,KAAK,CAAC,kEAAkE,EAAE,EAAE,CAAC,CAAC;iBACzF;aACF;SACF;IACH,CAAC;IAES,SAAS,CAAC,OAAe;QACjC,MAAM,WAAW,GAAG,IAAI,IAAI,CAAC,WAAW,EAAE,CAAC;QAE3C,MAAM,MAAM,GAAG,IAAI,CAAC,aAAa,CAAC,OAAO,EAAE;YACzC,MAAM,EAAE,MAAM;YACd,gBAAgB,EAAE,IAAI;YACtB,WAAW;YACX,UAAU,EAAE;gBACV;oBACE,GAAG,EAAE,MAAM;oBACX,OAAO,CAAC,OAAe,EAAE,OAAgC;wBACvD,IAAI,CAAC,OAAO,CAAC,UAAU,CAAC,eAAe,CAAC,EAAE;4BACxC,OAAO,CACL,iDAAiD,OAAO,+CAA+C,eAAe,GAAG,CAC1H,CAAC;4BACF,OAAO,OAAO,CAAC;yBAChB;wBACD,MAAM,KAAK,GAAG,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;wBACnC,IAAI,OAAO,KAAK,IAAI,WAAW,EAAE;4BAC/B,OAAO,CACL,iDAAiD,OAAO,6DAA6D,CACtH,CAAC;4BACF,OAAO,OAAO,CAAC;yBAChB;wBACD,OAAO,KAAK,CAAC;oBACf,CAAC;iBACF;aACF;SACF,CAAC,CAAC;QAEH,IAAI,MAAM,CAAC,MAAM,CAAC,MAAM,EAAE;YACxB,MAAM,IAAI,KAAK,CACb,gEAAgE,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CACjH,CAAC;SACH;QAED,OAAO,MAAM,CAAC,IAAI,EAAE,CAAC;IACvB,CAAC;IAES,SAAS,CAAC,OAAe;QACjC,OAAO,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;IAC7B,CAAC;CACF"}
|
|
@@ -1,8 +1,7 @@
|
|
|
1
|
-
import * as t from 'ts-codec';
|
|
2
1
|
import { configFile } from '@powersync/service-types';
|
|
3
2
|
import { ConfigCollector } from './collectors/config-collector.js';
|
|
4
|
-
import { ResolvedPowerSyncConfig, RunnerConfig, SyncRulesConfig } from './types.js';
|
|
5
3
|
import { SyncRulesCollector } from './sync-rules/sync-collector.js';
|
|
4
|
+
import { ResolvedPowerSyncConfig, RunnerConfig, SyncRulesConfig } from './types.js';
|
|
6
5
|
export type CompoundConfigCollectorOptions = {
|
|
7
6
|
/**
|
|
8
7
|
* Collectors for PowerSync configuration content.
|
|
@@ -17,42 +16,24 @@ export type CompoundConfigCollectorOptions = {
|
|
|
17
16
|
*/
|
|
18
17
|
syncRulesCollectors: SyncRulesCollector[];
|
|
19
18
|
};
|
|
20
|
-
export type
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
19
|
+
export type ConfigCollectedEvent = {
|
|
20
|
+
base_config: configFile.PowerSyncConfig;
|
|
21
|
+
resolved_config: ResolvedPowerSyncConfig;
|
|
22
|
+
};
|
|
23
|
+
export type ConfigCollectorListener = {
|
|
24
|
+
configCollected?: (event: ConfigCollectedEvent) => Promise<void>;
|
|
24
25
|
};
|
|
25
|
-
export declare class CompoundConfigCollector
|
|
26
|
+
export declare class CompoundConfigCollector {
|
|
26
27
|
protected options: CompoundConfigCollectorOptions;
|
|
27
28
|
constructor(options?: CompoundConfigCollectorOptions);
|
|
28
|
-
/**
|
|
29
|
-
* The default ts-codec for validations and decoding
|
|
30
|
-
*/
|
|
31
|
-
get codec(): t.AnyCodec;
|
|
32
29
|
/**
|
|
33
30
|
* Collects and resolves base config
|
|
34
31
|
*/
|
|
35
|
-
collectConfig(runnerConfig?: RunnerConfig): Promise<
|
|
32
|
+
collectConfig(runnerConfig?: RunnerConfig): Promise<ResolvedPowerSyncConfig>;
|
|
36
33
|
/**
|
|
37
34
|
* Collects the base PowerSyncConfig from various registered collectors.
|
|
38
35
|
* @throws if no collector could return a configuration.
|
|
39
36
|
*/
|
|
40
|
-
protected collectBaseConfig(runner_config: RunnerConfig): Promise<
|
|
41
|
-
/**
|
|
42
|
-
* Performs the resolving of the common (shared) base configuration
|
|
43
|
-
*/
|
|
44
|
-
protected resolveBaseConfig(baseConfig: Generics['DESERIALIZED'], runnerConfig?: RunnerConfig): Promise<ResolvedPowerSyncConfig>;
|
|
45
|
-
/**
|
|
46
|
-
* Perform any additional resolving from {@link ResolvedPowerSyncConfig}
|
|
47
|
-
* to the extended {@link Generics['RESOLVED']}
|
|
48
|
-
*
|
|
49
|
-
*/
|
|
50
|
-
protected resolveConfig(baseConfig: Generics['DESERIALIZED'], resolvedBaseConfig: ResolvedPowerSyncConfig, runnerConfig?: RunnerConfig): Promise<Generics['RESOLVED']>;
|
|
37
|
+
protected collectBaseConfig(runner_config: RunnerConfig): Promise<configFile.PowerSyncConfig>;
|
|
51
38
|
protected collectSyncRules(baseConfig: configFile.PowerSyncConfig, runnerConfig: RunnerConfig): Promise<SyncRulesConfig>;
|
|
52
|
-
/**
|
|
53
|
-
* Validates input config
|
|
54
|
-
* ts-codec itself doesn't give great validation errors, so we use json schema for that
|
|
55
|
-
*/
|
|
56
|
-
protected validate(config: Generics['DESERIALIZED']): void;
|
|
57
|
-
protected decode(encoded: Generics['SERIALIZED']): Generics['DESERIALIZED'];
|
|
58
39
|
}
|
|
@@ -1,13 +1,11 @@
|
|
|
1
|
-
import
|
|
2
|
-
import { configFile, normalizeConnection } from '@powersync/service-types';
|
|
1
|
+
import { logger } from '@powersync/lib-services-framework';
|
|
3
2
|
import * as auth from '../../auth/auth-index.js';
|
|
4
3
|
import { Base64ConfigCollector } from './collectors/impl/base64-config-collector.js';
|
|
4
|
+
import { FallbackConfigCollector } from './collectors/impl/fallback-config-collector.js';
|
|
5
5
|
import { FileSystemConfigCollector } from './collectors/impl/filesystem-config-collector.js';
|
|
6
6
|
import { Base64SyncRulesCollector } from './sync-rules/impl/base64-sync-rules-collector.js';
|
|
7
|
-
import { InlineSyncRulesCollector } from './sync-rules/impl/inline-sync-rules-collector.js';
|
|
8
7
|
import { FileSystemSyncRulesCollector } from './sync-rules/impl/filesystem-sync-rules-collector.js';
|
|
9
|
-
import {
|
|
10
|
-
import { logger, schema } from '@powersync/lib-services-framework';
|
|
8
|
+
import { InlineSyncRulesCollector } from './sync-rules/impl/inline-sync-rules-collector.js';
|
|
11
9
|
const POWERSYNC_DEV_KID = 'powersync-dev';
|
|
12
10
|
const DEFAULT_COLLECTOR_OPTIONS = {
|
|
13
11
|
configCollectors: [new Base64ConfigCollector(), new FileSystemConfigCollector(), new FallbackConfigCollector()],
|
|
@@ -21,66 +19,20 @@ export class CompoundConfigCollector {
|
|
|
21
19
|
constructor(options = DEFAULT_COLLECTOR_OPTIONS) {
|
|
22
20
|
this.options = options;
|
|
23
21
|
}
|
|
24
|
-
/**
|
|
25
|
-
* The default ts-codec for validations and decoding
|
|
26
|
-
*/
|
|
27
|
-
get codec() {
|
|
28
|
-
return configFile.powerSyncConfig;
|
|
29
|
-
}
|
|
30
22
|
/**
|
|
31
23
|
* Collects and resolves base config
|
|
32
24
|
*/
|
|
33
25
|
async collectConfig(runnerConfig = {}) {
|
|
34
26
|
const baseConfig = await this.collectBaseConfig(runnerConfig);
|
|
35
|
-
const
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
/**
|
|
39
|
-
* Collects the base PowerSyncConfig from various registered collectors.
|
|
40
|
-
* @throws if no collector could return a configuration.
|
|
41
|
-
*/
|
|
42
|
-
async collectBaseConfig(runner_config) {
|
|
43
|
-
for (const collector of this.options.configCollectors) {
|
|
44
|
-
try {
|
|
45
|
-
const baseConfig = await collector.collectSerialized(runner_config);
|
|
46
|
-
if (baseConfig) {
|
|
47
|
-
const decoded = this.decode(baseConfig);
|
|
48
|
-
this.validate(decoded);
|
|
49
|
-
return decoded;
|
|
50
|
-
}
|
|
51
|
-
logger.debug(`Could not collect PowerSync config with ${collector.name} method. Moving on to next method if available.`);
|
|
52
|
-
}
|
|
53
|
-
catch (ex) {
|
|
54
|
-
// An error in a collector is a hard stop
|
|
55
|
-
throw new Error(`Could not collect config using ${collector.name} method. Caught exception: ${ex}`);
|
|
56
|
-
}
|
|
27
|
+
const dataSources = baseConfig.replication?.connections ?? [];
|
|
28
|
+
if (dataSources.length > 1) {
|
|
29
|
+
throw new Error('Only a single replication data source is supported currently');
|
|
57
30
|
}
|
|
58
|
-
throw new Error('PowerSyncConfig could not be collected using any of the registered config collectors.');
|
|
59
|
-
}
|
|
60
|
-
/**
|
|
61
|
-
* Performs the resolving of the common (shared) base configuration
|
|
62
|
-
*/
|
|
63
|
-
async resolveBaseConfig(baseConfig, runnerConfig = {}) {
|
|
64
|
-
const connections = baseConfig.replication?.connections ?? [];
|
|
65
|
-
if (connections.length > 1) {
|
|
66
|
-
throw new Error('Only a single replication connection is supported currently');
|
|
67
|
-
}
|
|
68
|
-
const mapped = connections.map((c) => {
|
|
69
|
-
const conf = {
|
|
70
|
-
type: 'postgresql',
|
|
71
|
-
...normalizeConnection(c),
|
|
72
|
-
debug_api: c.debug_api ?? false
|
|
73
|
-
};
|
|
74
|
-
return conf;
|
|
75
|
-
});
|
|
76
31
|
const collectors = new auth.CompoundKeyCollector();
|
|
77
32
|
const keyStore = new auth.KeyStore(collectors);
|
|
78
33
|
const inputKeys = baseConfig.client_auth?.jwks?.keys ?? [];
|
|
79
34
|
const staticCollector = await auth.StaticKeyCollector.importKeys(inputKeys);
|
|
80
35
|
collectors.add(staticCollector);
|
|
81
|
-
if (baseConfig.client_auth?.supabase && mapped.length > 0) {
|
|
82
|
-
collectors.add(new auth.CachedKeyCollector(new auth.SupabaseKeyCollector(mapped[0])));
|
|
83
|
-
}
|
|
84
36
|
let jwks_uris = baseConfig.client_auth?.jwks_uri ?? [];
|
|
85
37
|
if (typeof jwks_uris == 'string') {
|
|
86
38
|
jwks_uris = [jwks_uris];
|
|
@@ -96,7 +48,8 @@ export class CompoundConfigCollector {
|
|
|
96
48
|
const sync_rules = await this.collectSyncRules(baseConfig, runnerConfig);
|
|
97
49
|
let jwt_audiences = baseConfig.client_auth?.audience ?? [];
|
|
98
50
|
let config = {
|
|
99
|
-
|
|
51
|
+
base_config: baseConfig,
|
|
52
|
+
connections: baseConfig.replication?.connections || [],
|
|
100
53
|
storage: baseConfig.storage,
|
|
101
54
|
client_keystore: keyStore,
|
|
102
55
|
// Dev tokens only use the static keys, no external key sources
|
|
@@ -120,18 +73,31 @@ export class CompoundConfigCollector {
|
|
|
120
73
|
disable_telemetry_sharing: baseConfig.telemetry?.disable_telemetry_sharing ?? false,
|
|
121
74
|
internal_service_endpoint: baseConfig.telemetry?.internal_service_endpoint ?? 'https://pulse.journeyapps.com/v1/metrics'
|
|
122
75
|
},
|
|
123
|
-
|
|
76
|
+
// TODO maybe move this out of the connection or something
|
|
77
|
+
// slot_name_prefix: connections[0]?.slot_name_prefix ?? 'powersync_'
|
|
78
|
+
slot_name_prefix: 'powersync_'
|
|
124
79
|
};
|
|
125
80
|
return config;
|
|
126
81
|
}
|
|
127
82
|
/**
|
|
128
|
-
*
|
|
129
|
-
*
|
|
130
|
-
*
|
|
83
|
+
* Collects the base PowerSyncConfig from various registered collectors.
|
|
84
|
+
* @throws if no collector could return a configuration.
|
|
131
85
|
*/
|
|
132
|
-
async
|
|
133
|
-
|
|
134
|
-
|
|
86
|
+
async collectBaseConfig(runner_config) {
|
|
87
|
+
for (const collector of this.options.configCollectors) {
|
|
88
|
+
try {
|
|
89
|
+
const baseConfig = await collector.collect(runner_config);
|
|
90
|
+
if (baseConfig) {
|
|
91
|
+
return baseConfig;
|
|
92
|
+
}
|
|
93
|
+
logger.debug(`Could not collect PowerSync config with ${collector.name} method. Moving on to next method if available.`);
|
|
94
|
+
}
|
|
95
|
+
catch (ex) {
|
|
96
|
+
// An error in a collector is a hard stop
|
|
97
|
+
throw new Error(`Could not collect config using ${collector.name} method. Caught exception: ${ex}`);
|
|
98
|
+
}
|
|
99
|
+
}
|
|
100
|
+
throw new Error('PowerSyncConfig could not be collected using any of the registered config collectors.');
|
|
135
101
|
}
|
|
136
102
|
async collectSyncRules(baseConfig, runnerConfig) {
|
|
137
103
|
for (const collector of this.options.syncRulesCollectors) {
|
|
@@ -151,27 +117,5 @@ export class CompoundConfigCollector {
|
|
|
151
117
|
present: false
|
|
152
118
|
};
|
|
153
119
|
}
|
|
154
|
-
/**
|
|
155
|
-
* Validates input config
|
|
156
|
-
* ts-codec itself doesn't give great validation errors, so we use json schema for that
|
|
157
|
-
*/
|
|
158
|
-
validate(config) {
|
|
159
|
-
// ts-codec itself doesn't give great validation errors, so we use json schema for that
|
|
160
|
-
const validator = schema
|
|
161
|
-
.parseJSONSchema(t.generateJSONSchema(this.codec, { allowAdditional: true, parsers: [configFile.portParser] }))
|
|
162
|
-
.validator();
|
|
163
|
-
const valid = validator.validate(config);
|
|
164
|
-
if (!valid.valid) {
|
|
165
|
-
throw new Error(`Failed to validate PowerSync config: ${valid.errors.join(', ')}`);
|
|
166
|
-
}
|
|
167
|
-
}
|
|
168
|
-
decode(encoded) {
|
|
169
|
-
try {
|
|
170
|
-
return this.codec.decode(encoded);
|
|
171
|
-
}
|
|
172
|
-
catch (ex) {
|
|
173
|
-
throw new Error(`Failed to decode PowerSync config: ${ex}`);
|
|
174
|
-
}
|
|
175
|
-
}
|
|
176
120
|
}
|
|
177
121
|
//# sourceMappingURL=compound-config-collector.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"compound-config-collector.js","sourceRoot":"","sources":["../../../src/util/config/compound-config-collector.ts"],"names":[],"mappings":"AAAA,OAAO,
|
|
1
|
+
{"version":3,"file":"compound-config-collector.js","sourceRoot":"","sources":["../../../src/util/config/compound-config-collector.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,mCAAmC,CAAC;AAE3D,OAAO,KAAK,IAAI,MAAM,0BAA0B,CAAC;AAEjD,OAAO,EAAE,qBAAqB,EAAE,MAAM,8CAA8C,CAAC;AACrF,OAAO,EAAE,uBAAuB,EAAE,MAAM,gDAAgD,CAAC;AACzF,OAAO,EAAE,yBAAyB,EAAE,MAAM,kDAAkD,CAAC;AAC7F,OAAO,EAAE,wBAAwB,EAAE,MAAM,kDAAkD,CAAC;AAC5F,OAAO,EAAE,4BAA4B,EAAE,MAAM,sDAAsD,CAAC;AACpG,OAAO,EAAE,wBAAwB,EAAE,MAAM,kDAAkD,CAAC;AA4B5F,MAAM,iBAAiB,GAAG,eAAe,CAAC;AAE1C,MAAM,yBAAyB,GAAmC;IAChE,gBAAgB,EAAE,CAAC,IAAI,qBAAqB,EAAE,EAAE,IAAI,yBAAyB,EAAE,EAAE,IAAI,uBAAuB,EAAE,CAAC;IAC/G,mBAAmB,EAAE;QACnB,IAAI,wBAAwB,EAAE;QAC9B,IAAI,4BAA4B,EAAE;QAClC,IAAI,wBAAwB,EAAE;KAC/B;CACF,CAAC;AAEF,MAAM,OAAO,uBAAuB;IAClC,YAAsB,UAA0C,yBAAyB;QAAnE,YAAO,GAAP,OAAO,CAA4D;IAAG,CAAC;IAE7F;;OAEG;IACH,KAAK,CAAC,aAAa,CAAC,eAA6B,EAAE;QACjD,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,iBAAiB,CAAC,YAAY,CAAC,CAAC;QAE9D,MAAM,WAAW,GAAG,UAAU,CAAC,WAAW,EAAE,WAAW,IAAI,EAAE,CAAC;QAC9D,IAAI,WAAW,CAAC,MAAM,GAAG,CAAC,EAAE;YAC1B,MAAM,IAAI,KAAK,CAAC,8DAA8D,CAAC,CAAC;SACjF;QAED,MAAM,UAAU,GAAG,IAAI,IAAI,CAAC,oBAAoB,EAAE,CAAC;QACnD,MAAM,QAAQ,GAAG,IAAI,IAAI,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAC;QAE/C,MAAM,SAAS,GAAG,UAAU,CAAC,WAAW,EAAE,IAAI,EAAE,IAAI,IAAI,EAAE,CAAC;QAC3D,MAAM,eAAe,GAAG,MAAM,IAAI,CAAC,kBAAkB,CAAC,UAAU,CAAC,SAAS,CAAC,CAAC;QAE5E,UAAU,CAAC,GAAG,CAAC,eAAe,CAAC,CAAC;QAEhC,IAAI,SAAS,GAAG,UAAU,CAAC,WAAW,EAAE,QAAQ,IAAI,EAAE,CAAC;QACvD,IAAI,OAAO,SAAS,IAAI,QAAQ,EAAE;YAChC,SAAS,GAAG,CAAC,SAAS,CAAC,CAAC;SACzB;QAED,KAAK,IAAI,GAAG,IAAI,SAAS,EAAE;YACzB,UAAU,CAAC,GAAG,CACZ,IAAI,IAAI,CAAC,kBAAkB,CACzB,IAAI,IAAI,CAAC,mBAAmB,CAAC,GAAG,EAAE,EAAE,cAAc,EAAE,CAAC,CAAC,UAAU,CAAC,WAAW,EAAE,gBAAgB,EAAE,CAAC,CAClG,CACF,CAAC;SACH;QAED,MAAM,UAAU,GAAG,CAAC,UAAU,CAAC,WAAW,EAAE,IAAI,EAAE,IAAI,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,GAAG,IAAI,iBAAiB,CAAC,CAAC;QAE1G,IAAI,MAAgC,CAAC;QACrC,IAAI,UAAU,CAAC,GAAG,EAAE,SAAS,IAAI,UAAU,IAAI,IAAI,IAAI,UAAU,CAAC,GAAG,IAAI,KAAK,EAAE;YAC9E,MAAM,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC,UAAU,CAAC,CAAC;SACnD;QAED,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,gBAAgB,CAAC,UAAU,EAAE,YAAY,CAAC,CAAC;QAEzE,IAAI,aAAa,GAAa,UAAU,CAAC,WAAW,EAAE,QAAQ,IAAI,EAAE,CAAC;QAErE,IAAI,MAAM,GAA4B;YACpC,WAAW,EAAE,UAAU;YACvB,WAAW,EAAE,UAAU,CAAC,WAAW,EAAE,WAAW,IAAI,EAAE;YACtD,OAAO,EAAE,UAAU,CAAC,OAAO;YAC3B,eAAe,EAAE,QAAQ;YACzB,+DAA+D;YAC/D,mEAAmE;YACnE,mBAAmB,EAAE,IAAI,IAAI,CAAC,QAAQ,CAAC,eAAe,CAAC;YACvD,UAAU,EAAE,UAAU,CAAC,GAAG,EAAE,MAAM,IAAI,EAAE;YACxC,GAAG,EAAE;gBACH,SAAS,EAAE,UAAU,CAAC,GAAG,EAAE,SAAS,IAAI,KAAK;gBAC7C,WAAW,EAAE,UAAU,CAAC,GAAG,EAAE,WAAW,IAAI,KAAK;gBACjD,aAAa,EAAE,UAAU,CAAC,GAAG,EAAE,aAAa;gBAC5C,QAAQ,EAAE,UAAU,CAAC,GAAG,EAAE,QAAQ,IAAI,KAAK;gBAC3C,OAAO,EAAE,MAAM;aAChB;YACD,IAAI,EAAE,UAAU,CAAC,IAAI,IAAI,IAAI;YAC7B,UAAU;YACV,aAAa;YAEb,oBAAoB,EAAE,IAAI;YAC1B,QAAQ,EAAE,UAAU,CAAC,QAAQ,IAAI,EAAE;YACnC,UAAU,EAAE,UAAU,CAAC,UAAU;YACjC,SAAS,EAAE;gBACT,yBAAyB,EAAE,UAAU,CAAC,SAAS,EAAE,yBAAyB,IAAI,KAAK;gBACnF,yBAAyB,EACvB,UAAU,CAAC,SAAS,EAAE,yBAAyB,IAAI,0CAA0C;aAChG;YACD,0DAA0D;YAC1D,qEAAqE;YACrE,gBAAgB,EAAE,YAAY;SAC/B,CAAC;QAEF,OAAO,MAAM,CAAC;IAChB,CAAC;IAED;;;OAGG;IACO,KAAK,CAAC,iBAAiB,CAAC,aAA2B;QAC3D,KAAK,MAAM,SAAS,IAAI,IAAI,CAAC,OAAO,CAAC,gBAAgB,EAAE;YACrD,IAAI;gBACF,MAAM,UAAU,GAAG,MAAM,SAAS,CAAC,OAAO,CAAC,aAAa,CAAC,CAAC;gBAC1D,IAAI,UAAU,EAAE;oBACd,OAAO,UAAU,CAAC;iBACnB;gBACD,MAAM,CAAC,KAAK,CACV,2CAA2C,SAAS,CAAC,IAAI,iDAAiD,CAC3G,CAAC;aACH;YAAC,OAAO,EAAE,EAAE;gBACX,yCAAyC;gBACzC,MAAM,IAAI,KAAK,CAAC,kCAAkC,SAAS,CAAC,IAAI,8BAA8B,EAAE,EAAE,CAAC,CAAC;aACrG;SACF;QACD,MAAM,IAAI,KAAK,CAAC,uFAAuF,CAAC,CAAC;IAC3G,CAAC;IAES,KAAK,CAAC,gBAAgB,CAC9B,UAAsC,EACtC,YAA0B;QAE1B,KAAK,MAAM,SAAS,IAAI,IAAI,CAAC,OAAO,CAAC,mBAAmB,EAAE;YACxD,IAAI;gBACF,MAAM,MAAM,GAAG,MAAM,SAAS,CAAC,OAAO,CAAC,UAAU,EAAE,YAAY,CAAC,CAAC;gBACjE,IAAI,MAAM,EAAE;oBACV,OAAO,MAAM,CAAC;iBACf;gBACD,MAAM,CAAC,KAAK,CACV,qCAAqC,SAAS,CAAC,IAAI,iDAAiD,CACrG,CAAC;aACH;YAAC,OAAO,EAAE,EAAE;gBACX,yCAAyC;gBACzC,MAAM,IAAI,KAAK,CAAC,sCAAsC,SAAS,CAAC,IAAI,8BAA8B,EAAE,EAAE,CAAC,CAAC;aACzG;SACF;QACD,OAAO;YACL,OAAO,EAAE,KAAK;SACf,CAAC;IACJ,CAAC;CACF"}
|