@powersync/service-core 0.0.0-dev-20240725112650 → 0.0.0-dev-20240918082156
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +80 -2
- package/dist/api/RouteAPI.d.ts +68 -0
- package/dist/api/RouteAPI.js +2 -0
- package/dist/api/RouteAPI.js.map +1 -0
- package/dist/api/api-index.d.ts +1 -0
- package/dist/api/api-index.js +1 -0
- package/dist/api/api-index.js.map +1 -1
- package/dist/api/diagnostics.d.ts +4 -4
- package/dist/api/diagnostics.js +11 -65
- package/dist/api/diagnostics.js.map +1 -1
- package/dist/api/schema.d.ts +3 -5
- package/dist/api/schema.js +9 -79
- package/dist/api/schema.js.map +1 -1
- package/dist/auth/KeyStore.d.ts +7 -4
- package/dist/auth/KeyStore.js +1 -1
- package/dist/auth/KeyStore.js.map +1 -1
- package/dist/auth/auth-index.d.ts +0 -1
- package/dist/auth/auth-index.js +0 -1
- package/dist/auth/auth-index.js.map +1 -1
- package/dist/entry/cli-entry.js +3 -2
- package/dist/entry/cli-entry.js.map +1 -1
- package/dist/entry/commands/compact-action.js +12 -8
- package/dist/entry/commands/compact-action.js.map +1 -1
- package/dist/entry/commands/migrate-action.js +4 -5
- package/dist/entry/commands/migrate-action.js.map +1 -1
- package/dist/entry/commands/teardown-action.js +2 -2
- package/dist/entry/commands/teardown-action.js.map +1 -1
- package/dist/index.d.ts +4 -2
- package/dist/index.js +4 -2
- package/dist/index.js.map +1 -1
- package/dist/metrics/Metrics.d.ts +2 -2
- package/dist/metrics/Metrics.js +5 -13
- package/dist/metrics/Metrics.js.map +1 -1
- package/dist/migrations/db/migrations/1684951997326-init.d.ts +2 -2
- package/dist/migrations/db/migrations/1684951997326-init.js +4 -2
- package/dist/migrations/db/migrations/1684951997326-init.js.map +1 -1
- package/dist/migrations/db/migrations/1702295701188-sync-rule-state.d.ts +2 -2
- package/dist/migrations/db/migrations/1702295701188-sync-rule-state.js +4 -2
- package/dist/migrations/db/migrations/1702295701188-sync-rule-state.js.map +1 -1
- package/dist/migrations/db/migrations/1711543888062-write-checkpoint-index.d.ts +2 -2
- package/dist/migrations/db/migrations/1711543888062-write-checkpoint-index.js +4 -2
- package/dist/migrations/db/migrations/1711543888062-write-checkpoint-index.js.map +1 -1
- package/dist/migrations/migrations.d.ts +8 -0
- package/dist/migrations/migrations.js +19 -7
- package/dist/migrations/migrations.js.map +1 -1
- package/dist/modules/AbstractModule.d.ts +26 -0
- package/dist/modules/AbstractModule.js +11 -0
- package/dist/modules/AbstractModule.js.map +1 -0
- package/dist/modules/ModuleManager.d.ts +11 -0
- package/dist/modules/ModuleManager.js +32 -0
- package/dist/modules/ModuleManager.js.map +1 -0
- package/dist/modules/modules-index.d.ts +2 -0
- package/dist/modules/modules-index.js +3 -0
- package/dist/modules/modules-index.js.map +1 -0
- package/dist/replication/AbstractReplicationJob.d.ts +38 -0
- package/dist/replication/AbstractReplicationJob.js +51 -0
- package/dist/replication/AbstractReplicationJob.js.map +1 -0
- package/dist/replication/AbstractReplicator.d.ts +53 -0
- package/dist/replication/AbstractReplicator.js +187 -0
- package/dist/replication/AbstractReplicator.js.map +1 -0
- package/dist/replication/ErrorRateLimiter.d.ts +0 -9
- package/dist/replication/ErrorRateLimiter.js +1 -42
- package/dist/replication/ErrorRateLimiter.js.map +1 -1
- package/dist/replication/ReplicationEngine.d.ts +18 -0
- package/dist/replication/ReplicationEngine.js +41 -0
- package/dist/replication/ReplicationEngine.js.map +1 -0
- package/dist/replication/ReplicationModule.d.ts +39 -0
- package/dist/replication/ReplicationModule.js +65 -0
- package/dist/replication/ReplicationModule.js.map +1 -0
- package/dist/replication/replication-index.d.ts +4 -6
- package/dist/replication/replication-index.js +4 -6
- package/dist/replication/replication-index.js.map +1 -1
- package/dist/routes/RouterEngine.d.ts +42 -0
- package/dist/routes/RouterEngine.js +80 -0
- package/dist/routes/RouterEngine.js.map +1 -0
- package/dist/routes/auth.d.ts +2 -2
- package/dist/routes/auth.js +11 -11
- package/dist/routes/auth.js.map +1 -1
- package/dist/routes/configure-fastify.d.ts +30 -176
- package/dist/routes/configure-fastify.js +10 -11
- package/dist/routes/configure-fastify.js.map +1 -1
- package/dist/routes/configure-rsocket.d.ts +3 -3
- package/dist/routes/configure-rsocket.js +6 -5
- package/dist/routes/configure-rsocket.js.map +1 -1
- package/dist/routes/endpoints/admin.d.ts +0 -34
- package/dist/routes/endpoints/admin.js +48 -89
- package/dist/routes/endpoints/admin.js.map +1 -1
- package/dist/routes/endpoints/checkpointing.d.ts +56 -16
- package/dist/routes/endpoints/checkpointing.js +33 -12
- package/dist/routes/endpoints/checkpointing.js.map +1 -1
- package/dist/routes/endpoints/route-endpoints-index.d.ts +0 -1
- package/dist/routes/endpoints/route-endpoints-index.js +0 -1
- package/dist/routes/endpoints/route-endpoints-index.js.map +1 -1
- package/dist/routes/endpoints/socket-route.js +40 -25
- package/dist/routes/endpoints/socket-route.js.map +1 -1
- package/dist/routes/endpoints/sync-rules.d.ts +1 -1
- package/dist/routes/endpoints/sync-rules.js +32 -23
- package/dist/routes/endpoints/sync-rules.js.map +1 -1
- package/dist/routes/endpoints/sync-stream.d.ts +10 -0
- package/dist/routes/endpoints/sync-stream.js +13 -8
- package/dist/routes/endpoints/sync-stream.js.map +1 -1
- package/dist/routes/router-socket.d.ts +1 -0
- package/dist/routes/router-socket.js +2 -1
- package/dist/routes/router-socket.js.map +1 -1
- package/dist/routes/router.d.ts +6 -2
- package/dist/routes/router.js.map +1 -1
- package/dist/routes/routes-index.d.ts +1 -0
- package/dist/routes/routes-index.js +1 -0
- package/dist/routes/routes-index.js.map +1 -1
- package/dist/runner/teardown.js +47 -76
- package/dist/runner/teardown.js.map +1 -1
- package/dist/storage/BucketStorage.d.ts +30 -19
- package/dist/storage/BucketStorage.js +0 -10
- package/dist/storage/BucketStorage.js.map +1 -1
- package/dist/storage/MongoBucketStorage.d.ts +4 -4
- package/dist/storage/MongoBucketStorage.js +19 -24
- package/dist/storage/MongoBucketStorage.js.map +1 -1
- package/dist/storage/SourceEntity.d.ts +20 -0
- package/dist/storage/SourceEntity.js +2 -0
- package/dist/storage/SourceEntity.js.map +1 -0
- package/dist/storage/SourceTable.d.ts +4 -5
- package/dist/storage/SourceTable.js +3 -4
- package/dist/storage/SourceTable.js.map +1 -1
- package/dist/storage/StorageEngine.d.ts +24 -0
- package/dist/storage/StorageEngine.js +43 -0
- package/dist/storage/StorageEngine.js.map +1 -0
- package/dist/storage/StorageProvider.d.ts +21 -0
- package/dist/storage/StorageProvider.js +2 -0
- package/dist/storage/StorageProvider.js.map +1 -0
- package/dist/storage/mongo/MongoBucketBatch.d.ts +1 -1
- package/dist/storage/mongo/MongoBucketBatch.js +6 -7
- package/dist/storage/mongo/MongoBucketBatch.js.map +1 -1
- package/dist/storage/mongo/MongoCompactor.js +2 -1
- package/dist/storage/mongo/MongoCompactor.js.map +1 -1
- package/dist/storage/mongo/MongoPersistedSyncRulesContent.d.ts +2 -2
- package/dist/storage/mongo/MongoPersistedSyncRulesContent.js +2 -2
- package/dist/storage/mongo/MongoPersistedSyncRulesContent.js.map +1 -1
- package/dist/storage/mongo/MongoStorageProvider.d.ts +5 -0
- package/dist/storage/mongo/MongoStorageProvider.js +26 -0
- package/dist/storage/mongo/MongoStorageProvider.js.map +1 -0
- package/dist/storage/mongo/MongoSyncBucketStorage.d.ts +7 -6
- package/dist/storage/mongo/MongoSyncBucketStorage.js +24 -15
- package/dist/storage/mongo/MongoSyncBucketStorage.js.map +1 -1
- package/dist/storage/mongo/MongoSyncRulesLock.js +1 -1
- package/dist/storage/mongo/MongoSyncRulesLock.js.map +1 -1
- package/dist/storage/mongo/OperationBatch.d.ts +7 -3
- package/dist/storage/mongo/OperationBatch.js +16 -7
- package/dist/storage/mongo/OperationBatch.js.map +1 -1
- package/dist/storage/mongo/PersistedBatch.d.ts +3 -3
- package/dist/storage/mongo/PersistedBatch.js +2 -2
- package/dist/storage/mongo/PersistedBatch.js.map +1 -1
- package/dist/storage/mongo/models.d.ts +13 -4
- package/dist/storage/mongo/models.js.map +1 -1
- package/dist/storage/mongo/util.d.ts +12 -1
- package/dist/storage/mongo/util.js +50 -2
- package/dist/storage/mongo/util.js.map +1 -1
- package/dist/storage/storage-index.d.ts +5 -2
- package/dist/storage/storage-index.js +5 -2
- package/dist/storage/storage-index.js.map +1 -1
- package/dist/sync/sync.d.ts +2 -1
- package/dist/sync/sync.js +36 -10
- package/dist/sync/sync.js.map +1 -1
- package/dist/sync/util.js.map +1 -1
- package/dist/system/ServiceContext.d.ts +37 -0
- package/dist/system/ServiceContext.js +48 -0
- package/dist/system/ServiceContext.js.map +1 -0
- package/dist/system/system-index.d.ts +1 -1
- package/dist/system/system-index.js +1 -1
- package/dist/system/system-index.js.map +1 -1
- package/dist/util/config/compound-config-collector.d.ts +9 -2
- package/dist/util/config/compound-config-collector.js +14 -23
- package/dist/util/config/compound-config-collector.js.map +1 -1
- package/dist/util/config/sync-rules/sync-rules-provider.d.ts +9 -0
- package/dist/util/config/sync-rules/sync-rules-provider.js +15 -0
- package/dist/util/config/sync-rules/sync-rules-provider.js.map +1 -0
- package/dist/util/config/types.d.ts +6 -4
- package/dist/util/config/types.js.map +1 -1
- package/dist/util/config.d.ts +3 -4
- package/dist/util/config.js +5 -20
- package/dist/util/config.js.map +1 -1
- package/dist/util/protocol-types.d.ts +4 -0
- package/dist/util/protocol-types.js +5 -1
- package/dist/util/protocol-types.js.map +1 -1
- package/dist/util/util-index.d.ts +3 -6
- package/dist/util/util-index.js +3 -6
- package/dist/util/util-index.js.map +1 -1
- package/dist/util/utils.d.ts +10 -6
- package/dist/util/utils.js +45 -25
- package/dist/util/utils.js.map +1 -1
- package/package.json +5 -7
- package/src/api/RouteAPI.ts +78 -0
- package/src/api/api-index.ts +1 -0
- package/src/api/diagnostics.ts +16 -71
- package/src/api/schema.ts +13 -89
- package/src/auth/KeyStore.ts +9 -6
- package/src/auth/auth-index.ts +0 -1
- package/src/entry/cli-entry.ts +3 -2
- package/src/entry/commands/compact-action.ts +12 -9
- package/src/entry/commands/migrate-action.ts +5 -8
- package/src/entry/commands/teardown-action.ts +2 -2
- package/src/index.ts +5 -2
- package/src/metrics/Metrics.ts +6 -16
- package/src/migrations/db/migrations/1684951997326-init.ts +9 -4
- package/src/migrations/db/migrations/1702295701188-sync-rule-state.ts +7 -4
- package/src/migrations/db/migrations/1711543888062-write-checkpoint-index.ts +6 -4
- package/src/migrations/migrations.ts +24 -8
- package/src/modules/AbstractModule.ts +37 -0
- package/src/modules/ModuleManager.ts +34 -0
- package/src/modules/modules-index.ts +2 -0
- package/src/replication/AbstractReplicationJob.ts +79 -0
- package/src/replication/AbstractReplicator.ts +227 -0
- package/src/replication/ErrorRateLimiter.ts +0 -44
- package/src/replication/ReplicationEngine.ts +43 -0
- package/src/replication/ReplicationModule.ts +101 -0
- package/src/replication/replication-index.ts +4 -6
- package/src/routes/RouterEngine.ts +120 -0
- package/src/routes/auth.ts +21 -12
- package/src/routes/configure-fastify.ts +13 -14
- package/src/routes/configure-rsocket.ts +9 -8
- package/src/routes/endpoints/admin.ts +74 -100
- package/src/routes/endpoints/checkpointing.ts +46 -12
- package/src/routes/endpoints/route-endpoints-index.ts +0 -1
- package/src/routes/endpoints/socket-route.ts +44 -27
- package/src/routes/endpoints/sync-rules.ts +41 -25
- package/src/routes/endpoints/sync-stream.ts +13 -8
- package/src/routes/router-socket.ts +2 -1
- package/src/routes/router.ts +6 -3
- package/src/routes/routes-index.ts +1 -0
- package/src/runner/teardown.ts +50 -88
- package/src/storage/BucketStorage.ts +38 -25
- package/src/storage/MongoBucketStorage.ts +23 -26
- package/src/storage/SourceEntity.ts +22 -0
- package/src/storage/SourceTable.ts +4 -6
- package/src/storage/StorageEngine.ts +55 -0
- package/src/storage/StorageProvider.ts +27 -0
- package/src/storage/mongo/MongoBucketBatch.ts +8 -8
- package/src/storage/mongo/MongoCompactor.ts +2 -1
- package/src/storage/mongo/MongoPersistedSyncRulesContent.ts +3 -3
- package/src/storage/mongo/MongoStorageProvider.ts +31 -0
- package/src/storage/mongo/MongoSyncBucketStorage.ts +39 -20
- package/src/storage/mongo/MongoSyncRulesLock.ts +1 -1
- package/src/storage/mongo/OperationBatch.ts +18 -11
- package/src/storage/mongo/PersistedBatch.ts +6 -5
- package/src/storage/mongo/models.ts +13 -4
- package/src/storage/mongo/util.ts +49 -4
- package/src/storage/storage-index.ts +5 -2
- package/src/sync/sync.ts +46 -11
- package/src/sync/util.ts +0 -1
- package/src/system/ServiceContext.ts +68 -0
- package/src/system/system-index.ts +1 -1
- package/src/util/config/compound-config-collector.ts +30 -31
- package/src/util/config/sync-rules/sync-rules-provider.ts +18 -0
- package/src/util/config/types.ts +6 -5
- package/src/util/config.ts +6 -23
- package/src/util/protocol-types.ts +6 -1
- package/src/util/util-index.ts +3 -6
- package/src/util/utils.ts +55 -39
- package/test/src/__snapshots__/sync.test.ts.snap +7 -7
- package/test/src/auth.test.ts +7 -7
- package/test/src/broadcast_iterable.test.ts +1 -1
- package/test/src/checksum_cache.test.ts +3 -3
- package/test/src/compacting.test.ts +26 -17
- package/test/src/data_storage.test.ts +258 -146
- package/test/src/env.ts +1 -3
- package/test/src/merge_iterable.test.ts +1 -6
- package/test/src/setup.ts +1 -1
- package/test/src/stream_utils.ts +42 -0
- package/test/src/sync.test.ts +52 -31
- package/test/src/util.ts +48 -51
- package/test/tsconfig.json +1 -1
- package/tsconfig.tsbuildinfo +1 -1
- package/dist/auth/SupabaseKeyCollector.d.ts +0 -22
- package/dist/auth/SupabaseKeyCollector.js +0 -61
- package/dist/auth/SupabaseKeyCollector.js.map +0 -1
- package/dist/replication/PgRelation.d.ts +0 -16
- package/dist/replication/PgRelation.js +0 -26
- package/dist/replication/PgRelation.js.map +0 -1
- package/dist/replication/WalConnection.d.ts +0 -34
- package/dist/replication/WalConnection.js +0 -190
- package/dist/replication/WalConnection.js.map +0 -1
- package/dist/replication/WalStream.d.ts +0 -57
- package/dist/replication/WalStream.js +0 -515
- package/dist/replication/WalStream.js.map +0 -1
- package/dist/replication/WalStreamManager.d.ts +0 -30
- package/dist/replication/WalStreamManager.js +0 -198
- package/dist/replication/WalStreamManager.js.map +0 -1
- package/dist/replication/WalStreamRunner.d.ts +0 -38
- package/dist/replication/WalStreamRunner.js +0 -155
- package/dist/replication/WalStreamRunner.js.map +0 -1
- package/dist/replication/util.d.ts +0 -9
- package/dist/replication/util.js +0 -62
- package/dist/replication/util.js.map +0 -1
- package/dist/routes/endpoints/dev.d.ts +0 -312
- package/dist/routes/endpoints/dev.js +0 -172
- package/dist/routes/endpoints/dev.js.map +0 -1
- package/dist/system/CorePowerSyncSystem.d.ts +0 -23
- package/dist/system/CorePowerSyncSystem.js +0 -52
- package/dist/system/CorePowerSyncSystem.js.map +0 -1
- package/dist/util/PgManager.d.ts +0 -24
- package/dist/util/PgManager.js +0 -55
- package/dist/util/PgManager.js.map +0 -1
- package/dist/util/migration_lib.d.ts +0 -11
- package/dist/util/migration_lib.js +0 -64
- package/dist/util/migration_lib.js.map +0 -1
- package/dist/util/pgwire_utils.d.ts +0 -24
- package/dist/util/pgwire_utils.js +0 -117
- package/dist/util/pgwire_utils.js.map +0 -1
- package/dist/util/populate_test_data.d.ts +0 -8
- package/dist/util/populate_test_data.js +0 -65
- package/dist/util/populate_test_data.js.map +0 -1
- package/src/auth/SupabaseKeyCollector.ts +0 -67
- package/src/replication/PgRelation.ts +0 -42
- package/src/replication/WalConnection.ts +0 -227
- package/src/replication/WalStream.ts +0 -624
- package/src/replication/WalStreamManager.ts +0 -213
- package/src/replication/WalStreamRunner.ts +0 -180
- package/src/replication/util.ts +0 -76
- package/src/routes/endpoints/dev.ts +0 -199
- package/src/system/CorePowerSyncSystem.ts +0 -64
- package/src/util/PgManager.ts +0 -64
- package/src/util/migration_lib.ts +0 -79
- package/src/util/pgwire_utils.ts +0 -139
- package/src/util/populate_test_data.ts +0 -78
- package/test/src/__snapshots__/pg_test.test.ts.snap +0 -256
- package/test/src/large_batch.test.ts +0 -194
- package/test/src/pg_test.test.ts +0 -450
- package/test/src/schema_changes.test.ts +0 -545
- package/test/src/slow_tests.test.ts +0 -338
- package/test/src/validation.test.ts +0 -63
- package/test/src/wal_stream.test.ts +0 -319
- package/test/src/wal_stream_utils.ts +0 -156
|
@@ -1,13 +1,22 @@
|
|
|
1
1
|
import * as bson from 'bson';
|
|
2
2
|
import { SqliteJsonValue } from '@powersync/service-sync-rules';
|
|
3
3
|
|
|
4
|
+
/**
|
|
5
|
+
* Replica id uniquely identifying a row on the source database.
|
|
6
|
+
*
|
|
7
|
+
* Can be any value serializable to BSON.
|
|
8
|
+
*
|
|
9
|
+
* If the value is an entire document, the data serialized to a v5 UUID may be a good choice here.
|
|
10
|
+
*/
|
|
11
|
+
export type ReplicaId = bson.UUID | bson.Document | any;
|
|
12
|
+
|
|
4
13
|
export interface SourceKey {
|
|
5
14
|
/** group_id */
|
|
6
15
|
g: number;
|
|
7
16
|
/** source table id */
|
|
8
17
|
t: bson.ObjectId;
|
|
9
18
|
/** source key */
|
|
10
|
-
k:
|
|
19
|
+
k: ReplicaId;
|
|
11
20
|
}
|
|
12
21
|
|
|
13
22
|
export interface BucketDataKey {
|
|
@@ -43,7 +52,7 @@ export interface BucketDataDocument {
|
|
|
43
52
|
_id: BucketDataKey;
|
|
44
53
|
op: OpType;
|
|
45
54
|
source_table?: bson.ObjectId;
|
|
46
|
-
source_key?:
|
|
55
|
+
source_key?: ReplicaId;
|
|
47
56
|
table?: string;
|
|
48
57
|
row_id?: string;
|
|
49
58
|
checksum: number;
|
|
@@ -57,11 +66,11 @@ export interface SourceTableDocument {
|
|
|
57
66
|
_id: bson.ObjectId;
|
|
58
67
|
group_id: number;
|
|
59
68
|
connection_id: number;
|
|
60
|
-
relation_id: number | undefined;
|
|
69
|
+
relation_id: number | string | undefined;
|
|
61
70
|
schema_name: string;
|
|
62
71
|
table_name: string;
|
|
63
72
|
replica_id_columns: string[] | null;
|
|
64
|
-
replica_id_columns2: { name: string; type_oid
|
|
73
|
+
replica_id_columns2: { name: string; type_oid?: number; type?: string }[] | undefined;
|
|
65
74
|
snapshot_done: boolean | undefined;
|
|
66
75
|
}
|
|
67
76
|
|
|
@@ -1,10 +1,11 @@
|
|
|
1
1
|
import { SqliteJsonValue } from '@powersync/service-sync-rules';
|
|
2
2
|
import * as bson from 'bson';
|
|
3
|
-
import * as mongo from 'mongodb';
|
|
4
3
|
import * as crypto from 'crypto';
|
|
5
|
-
import
|
|
6
|
-
import
|
|
4
|
+
import * as mongo from 'mongodb';
|
|
5
|
+
import * as uuid from 'uuid';
|
|
7
6
|
import { OplogEntry } from '../../util/protocol-types.js';
|
|
7
|
+
import { ID_NAMESPACE, timestampToOpId } from '../../util/utils.js';
|
|
8
|
+
import { BucketDataDocument, ReplicaId } from './models.js';
|
|
8
9
|
|
|
9
10
|
/**
|
|
10
11
|
* Lookup serialization must be number-agnostic. I.e. normalize numbers, instead of preserving numbers.
|
|
@@ -98,7 +99,7 @@ export function mapOpEntry(row: BucketDataDocument): OplogEntry {
|
|
|
98
99
|
object_type: row.table,
|
|
99
100
|
object_id: row.row_id,
|
|
100
101
|
checksum: Number(row.checksum),
|
|
101
|
-
subkey:
|
|
102
|
+
subkey: replicaIdToSubkey(row.source_table!, row.source_key!),
|
|
102
103
|
data: row.data
|
|
103
104
|
};
|
|
104
105
|
} else {
|
|
@@ -111,3 +112,47 @@ export function mapOpEntry(row: BucketDataDocument): OplogEntry {
|
|
|
111
112
|
};
|
|
112
113
|
}
|
|
113
114
|
}
|
|
115
|
+
|
|
116
|
+
/**
|
|
117
|
+
* Returns true if two ReplicaId values are the same (serializes to the same BSON value).
|
|
118
|
+
*/
|
|
119
|
+
export function replicaIdEquals(a: ReplicaId, b: ReplicaId) {
|
|
120
|
+
if (a === b) {
|
|
121
|
+
return true;
|
|
122
|
+
} else if (typeof a == 'string' && typeof b == 'string') {
|
|
123
|
+
return a == b;
|
|
124
|
+
} else if (isUUID(a) && isUUID(b)) {
|
|
125
|
+
return a.equals(b);
|
|
126
|
+
} else if (a == null && b == null) {
|
|
127
|
+
return true;
|
|
128
|
+
} else if (a != null || b != null) {
|
|
129
|
+
return false;
|
|
130
|
+
} else {
|
|
131
|
+
// There are many possible primitive values, this covers them all
|
|
132
|
+
return (bson.serialize({ id: a }) as Buffer).equals(bson.serialize({ id: b }));
|
|
133
|
+
}
|
|
134
|
+
}
|
|
135
|
+
|
|
136
|
+
export function replicaIdToSubkey(table: bson.ObjectId, id: ReplicaId): string {
|
|
137
|
+
if (isUUID(id)) {
|
|
138
|
+
// Special case for UUID for backwards-compatiblity
|
|
139
|
+
return `${table.toHexString()}/${id.toHexString()}`;
|
|
140
|
+
} else {
|
|
141
|
+
// Hashed UUID from the table and id
|
|
142
|
+
const repr = bson.serialize({ table, id });
|
|
143
|
+
return uuid.v5(repr, ID_NAMESPACE);
|
|
144
|
+
}
|
|
145
|
+
}
|
|
146
|
+
|
|
147
|
+
/**
|
|
148
|
+
* True if this is a bson.UUID.
|
|
149
|
+
*
|
|
150
|
+
* Works even with multiple copies of the bson package.
|
|
151
|
+
*/
|
|
152
|
+
export function isUUID(value: any): value is bson.UUID {
|
|
153
|
+
if (value == null || typeof value != 'object') {
|
|
154
|
+
return false;
|
|
155
|
+
}
|
|
156
|
+
const uuid = value as bson.UUID;
|
|
157
|
+
return uuid._bsontype == 'Binary' && uuid.sub_type == bson.Binary.SUBTYPE_UUID;
|
|
158
|
+
}
|
|
@@ -1,6 +1,8 @@
|
|
|
1
|
-
export * from './SourceTable.js';
|
|
2
|
-
export * from './MongoBucketStorage.js';
|
|
3
1
|
export * from './BucketStorage.js';
|
|
2
|
+
export * from './MongoBucketStorage.js';
|
|
3
|
+
export * from './SourceEntity.js';
|
|
4
|
+
export * from './SourceTable.js';
|
|
5
|
+
export * from './StorageEngine.js';
|
|
4
6
|
|
|
5
7
|
export * from './mongo/db.js';
|
|
6
8
|
export * from './mongo/models.js';
|
|
@@ -8,6 +10,7 @@ export * from './mongo/MongoBucketBatch.js';
|
|
|
8
10
|
export * from './mongo/MongoIdSequence.js';
|
|
9
11
|
export * from './mongo/MongoPersistedSyncRules.js';
|
|
10
12
|
export * from './mongo/MongoPersistedSyncRulesContent.js';
|
|
13
|
+
export * from './mongo/MongoStorageProvider.js';
|
|
11
14
|
export * from './mongo/MongoSyncBucketStorage.js';
|
|
12
15
|
export * from './mongo/MongoSyncRulesLock.js';
|
|
13
16
|
export * from './mongo/OperationBatch.js';
|
package/src/sync/sync.ts
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import { JSONBig, JsonContainer } from '@powersync/service-jsonbig';
|
|
2
2
|
import { RequestParameters } from '@powersync/service-sync-rules';
|
|
3
|
-
import { Semaphore } from 'async-mutex';
|
|
3
|
+
import { Semaphore, withTimeout } from 'async-mutex';
|
|
4
|
+
|
|
4
5
|
import { AbortError } from 'ix/aborterror.js';
|
|
5
6
|
|
|
6
7
|
import * as auth from '../auth/auth-index.js';
|
|
@@ -8,22 +9,34 @@ import * as storage from '../storage/storage-index.js';
|
|
|
8
9
|
import * as util from '../util/util-index.js';
|
|
9
10
|
|
|
10
11
|
import { logger } from '@powersync/lib-services-framework';
|
|
11
|
-
import { Metrics } from '../metrics/Metrics.js';
|
|
12
12
|
import { mergeAsyncIterables } from './merge.js';
|
|
13
|
-
import { TokenStreamOptions, tokenStream } from './util.js';
|
|
14
13
|
import { RequestTracker } from './RequestTracker.js';
|
|
14
|
+
import { TokenStreamOptions, tokenStream } from './util.js';
|
|
15
15
|
|
|
16
16
|
/**
|
|
17
17
|
* Maximum number of connections actively fetching data.
|
|
18
18
|
*/
|
|
19
19
|
const MAX_ACTIVE_CONNECTIONS = 10;
|
|
20
|
-
|
|
20
|
+
|
|
21
|
+
/**
|
|
22
|
+
* Maximum duration to wait for the mutex to become available.
|
|
23
|
+
*
|
|
24
|
+
* This gives an explicit error if there are mutex issues, rather than just hanging.
|
|
25
|
+
*/
|
|
26
|
+
const MUTEX_ACQUIRE_TIMEOUT = 30_000;
|
|
27
|
+
|
|
28
|
+
const syncSemaphore = withTimeout(
|
|
29
|
+
new Semaphore(MAX_ACTIVE_CONNECTIONS),
|
|
30
|
+
MUTEX_ACQUIRE_TIMEOUT,
|
|
31
|
+
new Error(`Timeout while waiting for data`)
|
|
32
|
+
);
|
|
21
33
|
|
|
22
34
|
export interface SyncStreamParameters {
|
|
23
35
|
storage: storage.BucketStorageFactory;
|
|
24
36
|
params: util.StreamingSyncRequest;
|
|
25
37
|
syncParams: RequestParameters;
|
|
26
38
|
token: auth.JwtPayload;
|
|
39
|
+
parseOptions: storage.ParseSyncRulesOptions;
|
|
27
40
|
/**
|
|
28
41
|
* If this signal is aborted, the stream response ends as soon as possible, without error.
|
|
29
42
|
*/
|
|
@@ -36,7 +49,7 @@ export interface SyncStreamParameters {
|
|
|
36
49
|
export async function* streamResponse(
|
|
37
50
|
options: SyncStreamParameters
|
|
38
51
|
): AsyncIterable<util.StreamingSyncLine | string | null> {
|
|
39
|
-
const { storage, params, syncParams, token, tokenStreamOptions, tracker, signal } = options;
|
|
52
|
+
const { storage, params, syncParams, token, tokenStreamOptions, tracker, signal, parseOptions } = options;
|
|
40
53
|
// We also need to be able to abort, so we create our own controller.
|
|
41
54
|
const controller = new AbortController();
|
|
42
55
|
if (signal) {
|
|
@@ -52,7 +65,7 @@ export async function* streamResponse(
|
|
|
52
65
|
}
|
|
53
66
|
}
|
|
54
67
|
const ki = tokenStream(token, controller.signal, tokenStreamOptions);
|
|
55
|
-
const stream = streamResponseInner(storage, params, syncParams, tracker, controller.signal);
|
|
68
|
+
const stream = streamResponseInner(storage, params, syncParams, tracker, parseOptions, controller.signal);
|
|
56
69
|
// Merge the two streams, and abort as soon as one of the streams end.
|
|
57
70
|
const merged = mergeAsyncIterables([stream, ki], controller.signal);
|
|
58
71
|
|
|
@@ -76,6 +89,7 @@ async function* streamResponseInner(
|
|
|
76
89
|
params: util.StreamingSyncRequest,
|
|
77
90
|
syncParams: RequestParameters,
|
|
78
91
|
tracker: RequestTracker,
|
|
92
|
+
parseOptions: storage.ParseSyncRulesOptions,
|
|
79
93
|
signal: AbortSignal
|
|
80
94
|
): AsyncGenerator<util.StreamingSyncLine | string | null> {
|
|
81
95
|
// Bucket state of bucket id -> op_id.
|
|
@@ -93,7 +107,8 @@ async function* streamResponseInner(
|
|
|
93
107
|
}
|
|
94
108
|
}
|
|
95
109
|
|
|
96
|
-
const
|
|
110
|
+
const checkpointUserId = util.checkpointUserId(syncParams.token_parameters.user_id as string, params.client_id);
|
|
111
|
+
const stream = storage.watchWriteCheckpoint(checkpointUserId, signal);
|
|
97
112
|
for await (const next of stream) {
|
|
98
113
|
const { base, writeCheckpoint } = next;
|
|
99
114
|
const checkpoint = base.checkpoint;
|
|
@@ -103,9 +118,9 @@ async function* streamResponseInner(
|
|
|
103
118
|
// Sync rules deleted in the meantime - try again with the next checkpoint.
|
|
104
119
|
continue;
|
|
105
120
|
}
|
|
106
|
-
const
|
|
121
|
+
const syncRules = storage.getParsedSyncRules(parseOptions);
|
|
107
122
|
|
|
108
|
-
const allBuckets = await
|
|
123
|
+
const allBuckets = await syncRules.queryBucketIds({
|
|
109
124
|
getParameterSets(lookups) {
|
|
110
125
|
return storage.getParameterSets(checkpoint, lookups);
|
|
111
126
|
},
|
|
@@ -196,7 +211,8 @@ async function* streamResponseInner(
|
|
|
196
211
|
raw_data,
|
|
197
212
|
binary_data,
|
|
198
213
|
signal,
|
|
199
|
-
tracker
|
|
214
|
+
tracker,
|
|
215
|
+
user_id: syncParams.user_id
|
|
200
216
|
});
|
|
201
217
|
|
|
202
218
|
await new Promise((resolve) => setTimeout(resolve, 10));
|
|
@@ -213,6 +229,7 @@ interface BucketDataRequest {
|
|
|
213
229
|
binary_data: boolean | undefined;
|
|
214
230
|
tracker: RequestTracker;
|
|
215
231
|
signal: AbortSignal;
|
|
232
|
+
user_id?: string;
|
|
216
233
|
}
|
|
217
234
|
|
|
218
235
|
async function* bucketDataInBatches(request: BucketDataRequest) {
|
|
@@ -261,8 +278,19 @@ async function* bucketDataBatch(request: BucketDataRequest): AsyncGenerator<Buck
|
|
|
261
278
|
const checkpointOp = BigInt(checkpoint);
|
|
262
279
|
let checkpointInvalidated = false;
|
|
263
280
|
|
|
264
|
-
|
|
281
|
+
if (syncSemaphore.isLocked()) {
|
|
282
|
+
logger.info('Sync concurrency limit reached, waiting for lock', { user_id: request.user_id });
|
|
283
|
+
}
|
|
284
|
+
const [value, release] = await syncSemaphore.acquire();
|
|
265
285
|
try {
|
|
286
|
+
if (value <= 3) {
|
|
287
|
+
// This can be noisy, so we only log when we get close to the
|
|
288
|
+
// concurrency limit.
|
|
289
|
+
logger.info(`Got sync lock. Slots available: ${value - 1}`, {
|
|
290
|
+
user_id: request.user_id,
|
|
291
|
+
sync_data_slots: value - 1
|
|
292
|
+
});
|
|
293
|
+
}
|
|
266
294
|
// Optimization: Only fetch buckets for which the checksums have changed since the last checkpoint
|
|
267
295
|
// For the first batch, this will be all buckets.
|
|
268
296
|
const filteredBuckets = new Map(bucketsToFetch.map((bucket) => [bucket, dataBuckets.get(bucket)!]));
|
|
@@ -330,6 +358,13 @@ async function* bucketDataBatch(request: BucketDataRequest): AsyncGenerator<Buck
|
|
|
330
358
|
}
|
|
331
359
|
}
|
|
332
360
|
} finally {
|
|
361
|
+
if (value <= 3) {
|
|
362
|
+
// This can be noisy, so we only log when we get close to the
|
|
363
|
+
// concurrency limit.
|
|
364
|
+
logger.info(`Releasing sync lock`, {
|
|
365
|
+
user_id: request.user_id
|
|
366
|
+
});
|
|
367
|
+
}
|
|
333
368
|
release();
|
|
334
369
|
}
|
|
335
370
|
}
|
package/src/sync/util.ts
CHANGED
|
@@ -0,0 +1,68 @@
|
|
|
1
|
+
import { LifeCycledSystem, ServiceIdentifier, container } from '@powersync/lib-services-framework';
|
|
2
|
+
|
|
3
|
+
import * as metrics from '../metrics/Metrics.js';
|
|
4
|
+
import * as replication from '../replication/replication-index.js';
|
|
5
|
+
import * as routes from '../routes/routes-index.js';
|
|
6
|
+
import * as storage from '../storage/storage-index.js';
|
|
7
|
+
import * as utils from '../util/util-index.js';
|
|
8
|
+
|
|
9
|
+
export interface ServiceContext {
|
|
10
|
+
configuration: utils.ResolvedPowerSyncConfig;
|
|
11
|
+
lifeCycleEngine: LifeCycledSystem;
|
|
12
|
+
metrics: metrics.Metrics | null;
|
|
13
|
+
replicationEngine: replication.ReplicationEngine | null;
|
|
14
|
+
routerEngine: routes.RouterEngine | null;
|
|
15
|
+
storageEngine: storage.StorageEngine;
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
/**
|
|
19
|
+
* Context which allows for registering and getting implementations
|
|
20
|
+
* of various service engines.
|
|
21
|
+
* This controls registering, initializing and the lifecycle of various services.
|
|
22
|
+
*/
|
|
23
|
+
export class ServiceContextContainer implements ServiceContext {
|
|
24
|
+
lifeCycleEngine: LifeCycledSystem;
|
|
25
|
+
storageEngine: storage.StorageEngine;
|
|
26
|
+
|
|
27
|
+
constructor(public configuration: utils.ResolvedPowerSyncConfig) {
|
|
28
|
+
this.lifeCycleEngine = new LifeCycledSystem();
|
|
29
|
+
|
|
30
|
+
this.storageEngine = new storage.StorageEngine({
|
|
31
|
+
configuration
|
|
32
|
+
});
|
|
33
|
+
this.lifeCycleEngine.withLifecycle(this.storageEngine, {
|
|
34
|
+
start: (storageEngine) => storageEngine.start(),
|
|
35
|
+
stop: (storageEngine) => storageEngine.shutDown()
|
|
36
|
+
});
|
|
37
|
+
|
|
38
|
+
// Mongo storage is available as an option by default TODO: Consider moving this to a Mongo Storage Module
|
|
39
|
+
this.storageEngine.registerProvider(new storage.MongoStorageProvider());
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
get replicationEngine(): replication.ReplicationEngine | null {
|
|
43
|
+
return container.getOptional(replication.ReplicationEngine);
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
get routerEngine(): routes.RouterEngine | null {
|
|
47
|
+
return container.getOptional(routes.RouterEngine);
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
get metrics(): metrics.Metrics | null {
|
|
51
|
+
return container.getOptional(metrics.Metrics);
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
/**
|
|
55
|
+
* Allows for registering core and generic implementations of services/helpers.
|
|
56
|
+
* This uses the framework container under the hood.
|
|
57
|
+
*/
|
|
58
|
+
register<T>(identifier: ServiceIdentifier<T>, implementation: T) {
|
|
59
|
+
container.register(identifier, implementation);
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
/**
|
|
63
|
+
* Gets the implementation of an identifiable service.
|
|
64
|
+
*/
|
|
65
|
+
get<T>(identifier: ServiceIdentifier<T>) {
|
|
66
|
+
return container.getImplementation(identifier);
|
|
67
|
+
}
|
|
68
|
+
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
export * from './
|
|
1
|
+
export * from './ServiceContext.js';
|
|
@@ -1,17 +1,15 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import {
|
|
3
|
-
import { ResolvedConnection, ResolvedPowerSyncConfig, RunnerConfig, SyncRulesConfig } from './types.js';
|
|
1
|
+
import { logger } from '@powersync/lib-services-framework';
|
|
2
|
+
import { configFile } from '@powersync/service-types';
|
|
4
3
|
import * as auth from '../../auth/auth-index.js';
|
|
5
|
-
import {
|
|
4
|
+
import { ConfigCollector } from './collectors/config-collector.js';
|
|
6
5
|
import { Base64ConfigCollector } from './collectors/impl/base64-config-collector.js';
|
|
6
|
+
import { FallbackConfigCollector } from './collectors/impl/fallback-config-collector.js';
|
|
7
7
|
import { FileSystemConfigCollector } from './collectors/impl/filesystem-config-collector.js';
|
|
8
8
|
import { Base64SyncRulesCollector } from './sync-rules/impl/base64-sync-rules-collector.js';
|
|
9
|
-
import { InlineSyncRulesCollector } from './sync-rules/impl/inline-sync-rules-collector.js';
|
|
10
9
|
import { FileSystemSyncRulesCollector } from './sync-rules/impl/filesystem-sync-rules-collector.js';
|
|
11
|
-
import {
|
|
12
|
-
import {
|
|
13
|
-
|
|
14
|
-
const POWERSYNC_DEV_KID = 'powersync-dev';
|
|
10
|
+
import { InlineSyncRulesCollector } from './sync-rules/impl/inline-sync-rules-collector.js';
|
|
11
|
+
import { SyncRulesCollector } from './sync-rules/sync-collector.js';
|
|
12
|
+
import { ResolvedPowerSyncConfig, RunnerConfig, SyncRulesConfig } from './types.js';
|
|
15
13
|
|
|
16
14
|
export type CompoundConfigCollectorOptions = {
|
|
17
15
|
/**
|
|
@@ -28,6 +26,17 @@ export type CompoundConfigCollectorOptions = {
|
|
|
28
26
|
syncRulesCollectors: SyncRulesCollector[];
|
|
29
27
|
};
|
|
30
28
|
|
|
29
|
+
export type ConfigCollectedEvent = {
|
|
30
|
+
base_config: configFile.PowerSyncConfig;
|
|
31
|
+
resolved_config: ResolvedPowerSyncConfig;
|
|
32
|
+
};
|
|
33
|
+
|
|
34
|
+
export type ConfigCollectorListener = {
|
|
35
|
+
configCollected?: (event: ConfigCollectedEvent) => Promise<void>;
|
|
36
|
+
};
|
|
37
|
+
|
|
38
|
+
const POWERSYNC_DEV_KID = 'powersync-dev';
|
|
39
|
+
|
|
31
40
|
const DEFAULT_COLLECTOR_OPTIONS: CompoundConfigCollectorOptions = {
|
|
32
41
|
configCollectors: [new Base64ConfigCollector(), new FileSystemConfigCollector(), new FallbackConfigCollector()],
|
|
33
42
|
syncRulesCollectors: [
|
|
@@ -43,24 +52,14 @@ export class CompoundConfigCollector {
|
|
|
43
52
|
/**
|
|
44
53
|
* Collects and resolves base config
|
|
45
54
|
*/
|
|
46
|
-
async collectConfig(
|
|
47
|
-
const baseConfig = await this.collectBaseConfig(
|
|
55
|
+
async collectConfig(runnerConfig: RunnerConfig = {}): Promise<ResolvedPowerSyncConfig> {
|
|
56
|
+
const baseConfig = await this.collectBaseConfig(runnerConfig);
|
|
48
57
|
|
|
49
|
-
const
|
|
50
|
-
if (
|
|
51
|
-
throw new Error('Only a single replication
|
|
58
|
+
const dataSources = baseConfig.replication?.connections ?? [];
|
|
59
|
+
if (dataSources.length > 1) {
|
|
60
|
+
throw new Error('Only a single replication data source is supported currently');
|
|
52
61
|
}
|
|
53
62
|
|
|
54
|
-
const mapped = connections.map((c) => {
|
|
55
|
-
const conf: ResolvedConnection = {
|
|
56
|
-
type: 'postgresql' as const,
|
|
57
|
-
...normalizeConnection(c),
|
|
58
|
-
debug_api: c.debug_api ?? false
|
|
59
|
-
};
|
|
60
|
-
|
|
61
|
-
return conf;
|
|
62
|
-
});
|
|
63
|
-
|
|
64
63
|
const collectors = new auth.CompoundKeyCollector();
|
|
65
64
|
const keyStore = new auth.KeyStore(collectors);
|
|
66
65
|
|
|
@@ -69,10 +68,6 @@ export class CompoundConfigCollector {
|
|
|
69
68
|
|
|
70
69
|
collectors.add(staticCollector);
|
|
71
70
|
|
|
72
|
-
if (baseConfig.client_auth?.supabase && mapped.length > 0) {
|
|
73
|
-
collectors.add(new auth.CachedKeyCollector(new auth.SupabaseKeyCollector(mapped[0])));
|
|
74
|
-
}
|
|
75
|
-
|
|
76
71
|
let jwks_uris = baseConfig.client_auth?.jwks_uri ?? [];
|
|
77
72
|
if (typeof jwks_uris == 'string') {
|
|
78
73
|
jwks_uris = [jwks_uris];
|
|
@@ -93,12 +88,13 @@ export class CompoundConfigCollector {
|
|
|
93
88
|
devKey = await auth.KeySpec.importKey(baseDevKey);
|
|
94
89
|
}
|
|
95
90
|
|
|
96
|
-
const sync_rules = await this.collectSyncRules(baseConfig,
|
|
91
|
+
const sync_rules = await this.collectSyncRules(baseConfig, runnerConfig);
|
|
97
92
|
|
|
98
93
|
let jwt_audiences: string[] = baseConfig.client_auth?.audience ?? [];
|
|
99
94
|
|
|
100
95
|
let config: ResolvedPowerSyncConfig = {
|
|
101
|
-
|
|
96
|
+
base_config: baseConfig,
|
|
97
|
+
connections: baseConfig.replication?.connections || [],
|
|
102
98
|
storage: baseConfig.storage,
|
|
103
99
|
client_keystore: keyStore,
|
|
104
100
|
// Dev tokens only use the static keys, no external key sources
|
|
@@ -124,8 +120,11 @@ export class CompoundConfigCollector {
|
|
|
124
120
|
internal_service_endpoint:
|
|
125
121
|
baseConfig.telemetry?.internal_service_endpoint ?? 'https://pulse.journeyapps.com/v1/metrics'
|
|
126
122
|
},
|
|
127
|
-
|
|
123
|
+
// TODO maybe move this out of the connection or something
|
|
124
|
+
// slot_name_prefix: connections[0]?.slot_name_prefix ?? 'powersync_'
|
|
125
|
+
slot_name_prefix: 'powersync_'
|
|
128
126
|
};
|
|
127
|
+
|
|
129
128
|
return config;
|
|
130
129
|
}
|
|
131
130
|
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
import { SyncRulesConfig } from '../types.js';
|
|
2
|
+
import fs from 'fs/promises';
|
|
3
|
+
|
|
4
|
+
export interface SyncRulesProvider {
|
|
5
|
+
get(): Promise<string | undefined>;
|
|
6
|
+
}
|
|
7
|
+
|
|
8
|
+
export class ConfigurationFileSyncRulesProvider implements SyncRulesProvider {
|
|
9
|
+
constructor(private config: SyncRulesConfig) {}
|
|
10
|
+
|
|
11
|
+
async get(): Promise<string | undefined> {
|
|
12
|
+
if (this.config.content) {
|
|
13
|
+
return this.config.content;
|
|
14
|
+
} else if (this.config.path) {
|
|
15
|
+
return await fs.readFile(this.config.path, 'utf-8');
|
|
16
|
+
}
|
|
17
|
+
}
|
|
18
|
+
}
|
package/src/util/config/types.ts
CHANGED
|
@@ -1,4 +1,6 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { configFile } from '@powersync/service-types';
|
|
2
|
+
import { PowerSyncConfig } from '@powersync/service-types/src/config/PowerSyncConfig.js';
|
|
3
|
+
import { CompoundKeyCollector } from '../../auth/CompoundKeyCollector.js';
|
|
2
4
|
import { KeySpec } from '../../auth/KeySpec.js';
|
|
3
5
|
import { KeyStore } from '../../auth/KeyStore.js';
|
|
4
6
|
|
|
@@ -20,8 +22,6 @@ export type MigrationContext = {
|
|
|
20
22
|
|
|
21
23
|
export type Runner = (config: RunnerConfig) => Promise<void>;
|
|
22
24
|
|
|
23
|
-
export type ResolvedConnection = configFile.PostgresConnection & NormalizedPostgresConnection;
|
|
24
|
-
|
|
25
25
|
export type SyncRulesConfig = {
|
|
26
26
|
present: boolean;
|
|
27
27
|
content?: string;
|
|
@@ -29,7 +29,8 @@ export type SyncRulesConfig = {
|
|
|
29
29
|
};
|
|
30
30
|
|
|
31
31
|
export type ResolvedPowerSyncConfig = {
|
|
32
|
-
|
|
32
|
+
base_config: PowerSyncConfig;
|
|
33
|
+
connections?: configFile.DataSourceConfig[];
|
|
33
34
|
storage: configFile.StorageConfig;
|
|
34
35
|
dev: {
|
|
35
36
|
demo_auth: boolean;
|
|
@@ -41,7 +42,7 @@ export type ResolvedPowerSyncConfig = {
|
|
|
41
42
|
*/
|
|
42
43
|
dev_key?: KeySpec;
|
|
43
44
|
};
|
|
44
|
-
client_keystore: KeyStore
|
|
45
|
+
client_keystore: KeyStore<CompoundKeyCollector>;
|
|
45
46
|
/**
|
|
46
47
|
* Keystore for development tokens.
|
|
47
48
|
*/
|
package/src/util/config.ts
CHANGED
|
@@ -1,31 +1,14 @@
|
|
|
1
1
|
import * as fs from 'fs/promises';
|
|
2
|
-
import { baseUri } from '@powersync/service-types';
|
|
3
2
|
|
|
4
|
-
import {
|
|
5
|
-
import {
|
|
3
|
+
import { container } from '@powersync/lib-services-framework';
|
|
4
|
+
import { ResolvedPowerSyncConfig, RunnerConfig } from './config/types.js';
|
|
5
|
+
import { CompoundConfigCollector } from './util-index.js';
|
|
6
6
|
|
|
7
7
|
/**
|
|
8
|
-
*
|
|
8
|
+
* Loads the resolved config using the registered config collector
|
|
9
9
|
*/
|
|
10
|
-
export function
|
|
11
|
-
|
|
12
|
-
throw new Error('Not supported');
|
|
13
|
-
}
|
|
14
|
-
|
|
15
|
-
const uri = new URL(baseUri(options));
|
|
16
|
-
uri.username = options.username;
|
|
17
|
-
uri.password = options.password;
|
|
18
|
-
if (options.sslmode != 'disable') {
|
|
19
|
-
// verify-full is tricky to actually use on a client, since they won't have the cert
|
|
20
|
-
// Just use "require" by default
|
|
21
|
-
// uri.searchParams.set('sslmode', options.sslmode);
|
|
22
|
-
uri.searchParams.set('sslmode', 'require');
|
|
23
|
-
}
|
|
24
|
-
return uri.toString();
|
|
25
|
-
}
|
|
26
|
-
|
|
27
|
-
export function loadConfig(runnerConfig: RunnerConfig = {}) {
|
|
28
|
-
const collector = new CompoundConfigCollector();
|
|
10
|
+
export async function loadConfig(runnerConfig: RunnerConfig) {
|
|
11
|
+
const collector = container.getImplementation(CompoundConfigCollector);
|
|
29
12
|
return collector.collectConfig(runnerConfig);
|
|
30
13
|
}
|
|
31
14
|
|
|
@@ -94,7 +94,12 @@ export const StreamingSyncRequest = t.object({
|
|
|
94
94
|
/**
|
|
95
95
|
* Client parameters to be passed to the sync rules.
|
|
96
96
|
*/
|
|
97
|
-
parameters: t.record(t.any).optional()
|
|
97
|
+
parameters: t.record(t.any).optional(),
|
|
98
|
+
|
|
99
|
+
/**
|
|
100
|
+
* Unique client id.
|
|
101
|
+
*/
|
|
102
|
+
client_id: t.string.optional()
|
|
98
103
|
});
|
|
99
104
|
|
|
100
105
|
export type StreamingSyncRequest = t.Decoded<typeof StreamingSyncRequest>;
|
package/src/util/util-index.ts
CHANGED
|
@@ -1,25 +1,22 @@
|
|
|
1
1
|
export * from './alerting.js';
|
|
2
2
|
export * from './env.js';
|
|
3
3
|
export * from './memory-tracking.js';
|
|
4
|
-
export * from './migration_lib.js';
|
|
5
4
|
export * from './Mutex.js';
|
|
6
|
-
export * from './PgManager.js';
|
|
7
|
-
export * from './pgwire_utils.js';
|
|
8
|
-
export * from './populate_test_data.js';
|
|
9
5
|
export * from './protocol-types.js';
|
|
10
6
|
export * from './secs.js';
|
|
11
7
|
export * from './utils.js';
|
|
12
8
|
|
|
13
9
|
export * from './config.js';
|
|
14
|
-
export * from './config/types.js';
|
|
15
10
|
export * from './config/compound-config-collector.js';
|
|
11
|
+
export * from './config/types.js';
|
|
16
12
|
|
|
17
13
|
export * from './config/collectors/config-collector.js';
|
|
18
14
|
export * from './config/collectors/impl/base64-config-collector.js';
|
|
19
15
|
export * from './config/collectors/impl/fallback-config-collector.js';
|
|
20
16
|
export * from './config/collectors/impl/filesystem-config-collector.js';
|
|
21
17
|
|
|
22
|
-
export * from './config/sync-rules/sync-collector.js';
|
|
23
18
|
export * from './config/sync-rules/impl/base64-sync-rules-collector.js';
|
|
24
19
|
export * from './config/sync-rules/impl/filesystem-sync-rules-collector.js';
|
|
25
20
|
export * from './config/sync-rules/impl/inline-sync-rules-collector.js';
|
|
21
|
+
export * from './config/sync-rules/sync-collector.js';
|
|
22
|
+
export * from './config/sync-rules/sync-rules-provider.js';
|