@powersync/service-core 0.0.0-dev-20240718134716 → 0.0.0-dev-20240918082156
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +89 -6
- package/dist/api/RouteAPI.d.ts +68 -0
- package/dist/api/RouteAPI.js +2 -0
- package/dist/api/RouteAPI.js.map +1 -0
- package/dist/api/api-index.d.ts +1 -0
- package/dist/api/api-index.js +1 -0
- package/dist/api/api-index.js.map +1 -1
- package/dist/api/diagnostics.d.ts +4 -4
- package/dist/api/diagnostics.js +11 -65
- package/dist/api/diagnostics.js.map +1 -1
- package/dist/api/schema.d.ts +3 -5
- package/dist/api/schema.js +9 -79
- package/dist/api/schema.js.map +1 -1
- package/dist/auth/KeyStore.d.ts +7 -4
- package/dist/auth/KeyStore.js +1 -1
- package/dist/auth/KeyStore.js.map +1 -1
- package/dist/auth/auth-index.d.ts +0 -1
- package/dist/auth/auth-index.js +0 -1
- package/dist/auth/auth-index.js.map +1 -1
- package/dist/entry/cli-entry.js +4 -2
- package/dist/entry/cli-entry.js.map +1 -1
- package/dist/entry/commands/compact-action.d.ts +2 -0
- package/dist/entry/commands/compact-action.js +52 -0
- package/dist/entry/commands/compact-action.js.map +1 -0
- package/dist/entry/commands/migrate-action.js +4 -5
- package/dist/entry/commands/migrate-action.js.map +1 -1
- package/dist/entry/commands/teardown-action.js +2 -2
- package/dist/entry/commands/teardown-action.js.map +1 -1
- package/dist/entry/entry-index.d.ts +1 -0
- package/dist/entry/entry-index.js +1 -0
- package/dist/entry/entry-index.js.map +1 -1
- package/dist/index.d.ts +4 -2
- package/dist/index.js +4 -2
- package/dist/index.js.map +1 -1
- package/dist/metrics/Metrics.d.ts +6 -5
- package/dist/metrics/Metrics.js +53 -10
- package/dist/metrics/Metrics.js.map +1 -1
- package/dist/migrations/db/migrations/1684951997326-init.d.ts +2 -2
- package/dist/migrations/db/migrations/1684951997326-init.js +4 -2
- package/dist/migrations/db/migrations/1684951997326-init.js.map +1 -1
- package/dist/migrations/db/migrations/1702295701188-sync-rule-state.d.ts +2 -2
- package/dist/migrations/db/migrations/1702295701188-sync-rule-state.js +4 -2
- package/dist/migrations/db/migrations/1702295701188-sync-rule-state.js.map +1 -1
- package/dist/migrations/db/migrations/1711543888062-write-checkpoint-index.d.ts +2 -2
- package/dist/migrations/db/migrations/1711543888062-write-checkpoint-index.js +4 -2
- package/dist/migrations/db/migrations/1711543888062-write-checkpoint-index.js.map +1 -1
- package/dist/migrations/migrations.d.ts +8 -0
- package/dist/migrations/migrations.js +19 -7
- package/dist/migrations/migrations.js.map +1 -1
- package/dist/modules/AbstractModule.d.ts +26 -0
- package/dist/modules/AbstractModule.js +11 -0
- package/dist/modules/AbstractModule.js.map +1 -0
- package/dist/modules/ModuleManager.d.ts +11 -0
- package/dist/modules/ModuleManager.js +32 -0
- package/dist/modules/ModuleManager.js.map +1 -0
- package/dist/modules/modules-index.d.ts +2 -0
- package/dist/modules/modules-index.js +3 -0
- package/dist/modules/modules-index.js.map +1 -0
- package/dist/replication/AbstractReplicationJob.d.ts +38 -0
- package/dist/replication/AbstractReplicationJob.js +51 -0
- package/dist/replication/AbstractReplicationJob.js.map +1 -0
- package/dist/replication/AbstractReplicator.d.ts +53 -0
- package/dist/replication/AbstractReplicator.js +187 -0
- package/dist/replication/AbstractReplicator.js.map +1 -0
- package/dist/replication/ErrorRateLimiter.d.ts +0 -9
- package/dist/replication/ErrorRateLimiter.js +1 -42
- package/dist/replication/ErrorRateLimiter.js.map +1 -1
- package/dist/replication/ReplicationEngine.d.ts +18 -0
- package/dist/replication/ReplicationEngine.js +41 -0
- package/dist/replication/ReplicationEngine.js.map +1 -0
- package/dist/replication/ReplicationModule.d.ts +39 -0
- package/dist/replication/ReplicationModule.js +65 -0
- package/dist/replication/ReplicationModule.js.map +1 -0
- package/dist/replication/replication-index.d.ts +4 -6
- package/dist/replication/replication-index.js +4 -6
- package/dist/replication/replication-index.js.map +1 -1
- package/dist/routes/RouterEngine.d.ts +42 -0
- package/dist/routes/RouterEngine.js +80 -0
- package/dist/routes/RouterEngine.js.map +1 -0
- package/dist/routes/auth.d.ts +2 -2
- package/dist/routes/auth.js +11 -11
- package/dist/routes/auth.js.map +1 -1
- package/dist/routes/configure-fastify.d.ts +737 -0
- package/dist/routes/configure-fastify.js +57 -0
- package/dist/routes/configure-fastify.js.map +1 -0
- package/dist/routes/configure-rsocket.d.ts +13 -0
- package/dist/routes/configure-rsocket.js +47 -0
- package/dist/routes/configure-rsocket.js.map +1 -0
- package/dist/routes/endpoints/admin.d.ts +0 -34
- package/dist/routes/endpoints/admin.js +48 -89
- package/dist/routes/endpoints/admin.js.map +1 -1
- package/dist/routes/endpoints/checkpointing.d.ts +56 -16
- package/dist/routes/endpoints/checkpointing.js +33 -12
- package/dist/routes/endpoints/checkpointing.js.map +1 -1
- package/dist/routes/endpoints/route-endpoints-index.d.ts +0 -1
- package/dist/routes/endpoints/route-endpoints-index.js +0 -1
- package/dist/routes/endpoints/route-endpoints-index.js.map +1 -1
- package/dist/routes/endpoints/socket-route.js +46 -39
- package/dist/routes/endpoints/socket-route.js.map +1 -1
- package/dist/routes/endpoints/sync-rules.d.ts +1 -1
- package/dist/routes/endpoints/sync-rules.js +32 -23
- package/dist/routes/endpoints/sync-rules.js.map +1 -1
- package/dist/routes/endpoints/sync-stream.d.ts +10 -0
- package/dist/routes/endpoints/sync-stream.js +17 -13
- package/dist/routes/endpoints/sync-stream.js.map +1 -1
- package/dist/routes/route-register.d.ts +1 -1
- package/dist/routes/route-register.js +1 -1
- package/dist/routes/route-register.js.map +1 -1
- package/dist/routes/router-socket.d.ts +5 -4
- package/dist/routes/router-socket.js +2 -1
- package/dist/routes/router-socket.js.map +1 -1
- package/dist/routes/router.d.ts +7 -2
- package/dist/routes/router.js.map +1 -1
- package/dist/routes/routes-index.d.ts +3 -0
- package/dist/routes/routes-index.js +3 -0
- package/dist/routes/routes-index.js.map +1 -1
- package/dist/runner/teardown.js +47 -76
- package/dist/runner/teardown.js.map +1 -1
- package/dist/storage/BucketStorage.d.ts +61 -20
- package/dist/storage/BucketStorage.js +0 -10
- package/dist/storage/BucketStorage.js.map +1 -1
- package/dist/storage/MongoBucketStorage.d.ts +4 -4
- package/dist/storage/MongoBucketStorage.js +19 -24
- package/dist/storage/MongoBucketStorage.js.map +1 -1
- package/dist/storage/SourceEntity.d.ts +20 -0
- package/dist/storage/SourceEntity.js +2 -0
- package/dist/storage/SourceEntity.js.map +1 -0
- package/dist/storage/SourceTable.d.ts +4 -5
- package/dist/storage/SourceTable.js +3 -4
- package/dist/storage/SourceTable.js.map +1 -1
- package/dist/storage/StorageEngine.d.ts +24 -0
- package/dist/storage/StorageEngine.js +43 -0
- package/dist/storage/StorageEngine.js.map +1 -0
- package/dist/storage/StorageProvider.d.ts +21 -0
- package/dist/storage/StorageProvider.js +2 -0
- package/dist/storage/StorageProvider.js.map +1 -0
- package/dist/storage/mongo/MongoBucketBatch.d.ts +1 -1
- package/dist/storage/mongo/MongoBucketBatch.js +6 -7
- package/dist/storage/mongo/MongoBucketBatch.js.map +1 -1
- package/dist/storage/mongo/MongoCompactor.d.ts +40 -0
- package/dist/storage/mongo/MongoCompactor.js +293 -0
- package/dist/storage/mongo/MongoCompactor.js.map +1 -0
- package/dist/storage/mongo/MongoPersistedSyncRulesContent.d.ts +2 -2
- package/dist/storage/mongo/MongoPersistedSyncRulesContent.js +2 -2
- package/dist/storage/mongo/MongoPersistedSyncRulesContent.js.map +1 -1
- package/dist/storage/mongo/MongoStorageProvider.d.ts +5 -0
- package/dist/storage/mongo/MongoStorageProvider.js +26 -0
- package/dist/storage/mongo/MongoStorageProvider.js.map +1 -0
- package/dist/storage/mongo/MongoSyncBucketStorage.d.ts +9 -7
- package/dist/storage/mongo/MongoSyncBucketStorage.js +43 -28
- package/dist/storage/mongo/MongoSyncBucketStorage.js.map +1 -1
- package/dist/storage/mongo/MongoSyncRulesLock.js +1 -1
- package/dist/storage/mongo/MongoSyncRulesLock.js.map +1 -1
- package/dist/storage/mongo/OperationBatch.d.ts +7 -3
- package/dist/storage/mongo/OperationBatch.js +16 -7
- package/dist/storage/mongo/OperationBatch.js.map +1 -1
- package/dist/storage/mongo/PersistedBatch.d.ts +3 -3
- package/dist/storage/mongo/PersistedBatch.js +2 -2
- package/dist/storage/mongo/PersistedBatch.js.map +1 -1
- package/dist/storage/mongo/models.d.ts +17 -7
- package/dist/storage/mongo/models.js.map +1 -1
- package/dist/storage/mongo/util.d.ts +14 -0
- package/dist/storage/mongo/util.js +70 -0
- package/dist/storage/mongo/util.js.map +1 -1
- package/dist/storage/storage-index.d.ts +5 -2
- package/dist/storage/storage-index.js +5 -2
- package/dist/storage/storage-index.js.map +1 -1
- package/dist/sync/RequestTracker.js +2 -3
- package/dist/sync/RequestTracker.js.map +1 -1
- package/dist/sync/sync-index.d.ts +1 -0
- package/dist/sync/sync-index.js +1 -0
- package/dist/sync/sync-index.js.map +1 -1
- package/dist/sync/sync.d.ts +2 -1
- package/dist/sync/sync.js +56 -17
- package/dist/sync/sync.js.map +1 -1
- package/dist/system/ServiceContext.d.ts +37 -0
- package/dist/system/ServiceContext.js +48 -0
- package/dist/system/ServiceContext.js.map +1 -0
- package/dist/system/system-index.d.ts +1 -1
- package/dist/system/system-index.js +1 -1
- package/dist/system/system-index.js.map +1 -1
- package/dist/util/config/collectors/config-collector.d.ts +12 -0
- package/dist/util/config/collectors/config-collector.js +43 -0
- package/dist/util/config/collectors/config-collector.js.map +1 -1
- package/dist/util/config/compound-config-collector.d.ts +10 -29
- package/dist/util/config/compound-config-collector.js +28 -84
- package/dist/util/config/compound-config-collector.js.map +1 -1
- package/dist/util/config/sync-rules/sync-rules-provider.d.ts +9 -0
- package/dist/util/config/sync-rules/sync-rules-provider.js +15 -0
- package/dist/util/config/sync-rules/sync-rules-provider.js.map +1 -0
- package/dist/util/config/types.d.ts +6 -4
- package/dist/util/config/types.js.map +1 -1
- package/dist/util/config.d.ts +3 -4
- package/dist/util/config.js +5 -20
- package/dist/util/config.js.map +1 -1
- package/dist/util/protocol-types.d.ts +4 -0
- package/dist/util/protocol-types.js +5 -1
- package/dist/util/protocol-types.js.map +1 -1
- package/dist/util/util-index.d.ts +3 -6
- package/dist/util/util-index.js +3 -6
- package/dist/util/util-index.js.map +1 -1
- package/dist/util/utils.d.ts +10 -6
- package/dist/util/utils.js +45 -25
- package/dist/util/utils.js.map +1 -1
- package/package.json +7 -7
- package/src/api/RouteAPI.ts +78 -0
- package/src/api/api-index.ts +1 -0
- package/src/api/diagnostics.ts +16 -71
- package/src/api/schema.ts +13 -89
- package/src/auth/KeyStore.ts +9 -6
- package/src/auth/auth-index.ts +0 -1
- package/src/entry/cli-entry.ts +4 -2
- package/src/entry/commands/compact-action.ts +57 -0
- package/src/entry/commands/migrate-action.ts +5 -8
- package/src/entry/commands/teardown-action.ts +2 -2
- package/src/entry/entry-index.ts +1 -0
- package/src/index.ts +5 -2
- package/src/metrics/Metrics.ts +70 -15
- package/src/migrations/db/migrations/1684951997326-init.ts +9 -4
- package/src/migrations/db/migrations/1702295701188-sync-rule-state.ts +7 -4
- package/src/migrations/db/migrations/1711543888062-write-checkpoint-index.ts +6 -4
- package/src/migrations/migrations.ts +24 -8
- package/src/modules/AbstractModule.ts +37 -0
- package/src/modules/ModuleManager.ts +34 -0
- package/src/modules/modules-index.ts +2 -0
- package/src/replication/AbstractReplicationJob.ts +79 -0
- package/src/replication/AbstractReplicator.ts +227 -0
- package/src/replication/ErrorRateLimiter.ts +0 -44
- package/src/replication/ReplicationEngine.ts +43 -0
- package/src/replication/ReplicationModule.ts +101 -0
- package/src/replication/replication-index.ts +4 -6
- package/src/routes/RouterEngine.ts +120 -0
- package/src/routes/auth.ts +21 -12
- package/src/routes/configure-fastify.ts +101 -0
- package/src/routes/configure-rsocket.ts +60 -0
- package/src/routes/endpoints/admin.ts +74 -100
- package/src/routes/endpoints/checkpointing.ts +46 -12
- package/src/routes/endpoints/route-endpoints-index.ts +0 -1
- package/src/routes/endpoints/socket-route.ts +50 -42
- package/src/routes/endpoints/sync-rules.ts +41 -25
- package/src/routes/endpoints/sync-stream.ts +17 -13
- package/src/routes/route-register.ts +2 -2
- package/src/routes/router-socket.ts +6 -5
- package/src/routes/router.ts +7 -2
- package/src/routes/routes-index.ts +3 -0
- package/src/runner/teardown.ts +50 -88
- package/src/storage/BucketStorage.ts +74 -26
- package/src/storage/MongoBucketStorage.ts +23 -26
- package/src/storage/SourceEntity.ts +22 -0
- package/src/storage/SourceTable.ts +4 -6
- package/src/storage/StorageEngine.ts +55 -0
- package/src/storage/StorageProvider.ts +27 -0
- package/src/storage/mongo/MongoBucketBatch.ts +8 -8
- package/src/storage/mongo/MongoCompactor.ts +372 -0
- package/src/storage/mongo/MongoPersistedSyncRulesContent.ts +3 -3
- package/src/storage/mongo/MongoStorageProvider.ts +31 -0
- package/src/storage/mongo/MongoSyncBucketStorage.ts +64 -34
- package/src/storage/mongo/MongoSyncRulesLock.ts +1 -1
- package/src/storage/mongo/OperationBatch.ts +18 -11
- package/src/storage/mongo/PersistedBatch.ts +6 -5
- package/src/storage/mongo/models.ts +17 -7
- package/src/storage/mongo/util.ts +71 -1
- package/src/storage/storage-index.ts +5 -2
- package/src/sync/RequestTracker.ts +3 -3
- package/src/sync/sync-index.ts +1 -0
- package/src/sync/sync.ts +66 -17
- package/src/system/ServiceContext.ts +68 -0
- package/src/system/system-index.ts +1 -1
- package/src/util/config/collectors/config-collector.ts +48 -0
- package/src/util/config/compound-config-collector.ts +45 -110
- package/src/util/config/sync-rules/sync-rules-provider.ts +18 -0
- package/src/util/config/types.ts +6 -5
- package/src/util/config.ts +6 -23
- package/src/util/protocol-types.ts +6 -1
- package/src/util/util-index.ts +3 -6
- package/src/util/utils.ts +55 -39
- package/test/src/__snapshots__/sync.test.ts.snap +90 -5
- package/test/src/auth.test.ts +7 -7
- package/test/src/broadcast_iterable.test.ts +1 -1
- package/test/src/bucket_validation.test.ts +142 -0
- package/test/src/bucket_validation.ts +116 -0
- package/test/src/checksum_cache.test.ts +3 -3
- package/test/src/compacting.test.ts +216 -0
- package/test/src/data_storage.test.ts +275 -204
- package/test/src/env.ts +1 -3
- package/test/src/merge_iterable.test.ts +1 -6
- package/test/src/setup.ts +1 -1
- package/test/src/stream_utils.ts +42 -0
- package/test/src/sync.test.ts +209 -48
- package/test/src/util.ts +110 -55
- package/test/tsconfig.json +1 -1
- package/tsconfig.tsbuildinfo +1 -1
- package/dist/auth/SupabaseKeyCollector.d.ts +0 -22
- package/dist/auth/SupabaseKeyCollector.js +0 -61
- package/dist/auth/SupabaseKeyCollector.js.map +0 -1
- package/dist/replication/PgRelation.d.ts +0 -16
- package/dist/replication/PgRelation.js +0 -26
- package/dist/replication/PgRelation.js.map +0 -1
- package/dist/replication/WalConnection.d.ts +0 -34
- package/dist/replication/WalConnection.js +0 -190
- package/dist/replication/WalConnection.js.map +0 -1
- package/dist/replication/WalStream.d.ts +0 -57
- package/dist/replication/WalStream.js +0 -517
- package/dist/replication/WalStream.js.map +0 -1
- package/dist/replication/WalStreamManager.d.ts +0 -30
- package/dist/replication/WalStreamManager.js +0 -198
- package/dist/replication/WalStreamManager.js.map +0 -1
- package/dist/replication/WalStreamRunner.d.ts +0 -38
- package/dist/replication/WalStreamRunner.js +0 -155
- package/dist/replication/WalStreamRunner.js.map +0 -1
- package/dist/replication/util.d.ts +0 -9
- package/dist/replication/util.js +0 -62
- package/dist/replication/util.js.map +0 -1
- package/dist/routes/endpoints/dev.d.ts +0 -312
- package/dist/routes/endpoints/dev.js +0 -172
- package/dist/routes/endpoints/dev.js.map +0 -1
- package/dist/system/CorePowerSyncSystem.d.ts +0 -23
- package/dist/system/CorePowerSyncSystem.js +0 -52
- package/dist/system/CorePowerSyncSystem.js.map +0 -1
- package/dist/util/PgManager.d.ts +0 -24
- package/dist/util/PgManager.js +0 -55
- package/dist/util/PgManager.js.map +0 -1
- package/dist/util/migration_lib.d.ts +0 -11
- package/dist/util/migration_lib.js +0 -64
- package/dist/util/migration_lib.js.map +0 -1
- package/dist/util/pgwire_utils.d.ts +0 -24
- package/dist/util/pgwire_utils.js +0 -117
- package/dist/util/pgwire_utils.js.map +0 -1
- package/dist/util/populate_test_data.d.ts +0 -8
- package/dist/util/populate_test_data.js +0 -65
- package/dist/util/populate_test_data.js.map +0 -1
- package/src/auth/SupabaseKeyCollector.ts +0 -67
- package/src/replication/PgRelation.ts +0 -42
- package/src/replication/WalConnection.ts +0 -227
- package/src/replication/WalStream.ts +0 -628
- package/src/replication/WalStreamManager.ts +0 -213
- package/src/replication/WalStreamRunner.ts +0 -180
- package/src/replication/util.ts +0 -76
- package/src/routes/endpoints/dev.ts +0 -199
- package/src/system/CorePowerSyncSystem.ts +0 -64
- package/src/util/PgManager.ts +0 -64
- package/src/util/migration_lib.ts +0 -79
- package/src/util/pgwire_utils.ts +0 -139
- package/src/util/populate_test_data.ts +0 -78
- package/test/src/__snapshots__/pg_test.test.ts.snap +0 -256
- package/test/src/large_batch.test.ts +0 -194
- package/test/src/pg_test.test.ts +0 -450
- package/test/src/schema_changes.test.ts +0 -545
- package/test/src/slow_tests.test.ts +0 -296
- package/test/src/validation.test.ts +0 -63
- package/test/src/wal_stream.test.ts +0 -314
- package/test/src/wal_stream_utils.ts +0 -147
package/src/api/schema.ts
CHANGED
|
@@ -1,99 +1,23 @@
|
|
|
1
|
-
import
|
|
2
|
-
import { pgwireRows } from '@powersync/service-jpgwire';
|
|
3
|
-
import { DatabaseSchema, internal_routes } from '@powersync/service-types';
|
|
1
|
+
import { internal_routes } from '@powersync/service-types';
|
|
4
2
|
|
|
5
|
-
import * as
|
|
6
|
-
import { CorePowerSyncSystem } from '../system/CorePowerSyncSystem.js';
|
|
3
|
+
import * as api from '../api/api-index.js';
|
|
7
4
|
|
|
8
|
-
export async function getConnectionsSchema(
|
|
9
|
-
if (
|
|
10
|
-
return {
|
|
5
|
+
export async function getConnectionsSchema(api: api.RouteAPI): Promise<internal_routes.GetSchemaResponse> {
|
|
6
|
+
if (!api) {
|
|
7
|
+
return {
|
|
8
|
+
connections: []
|
|
9
|
+
};
|
|
11
10
|
}
|
|
12
|
-
|
|
11
|
+
|
|
12
|
+
const baseConfig = await api.getSourceConfig();
|
|
13
|
+
|
|
13
14
|
return {
|
|
14
15
|
connections: [
|
|
15
16
|
{
|
|
16
|
-
schemas,
|
|
17
|
-
tag:
|
|
18
|
-
id:
|
|
17
|
+
schemas: await api.getConnectionSchema(),
|
|
18
|
+
tag: baseConfig.tag!,
|
|
19
|
+
id: baseConfig.id
|
|
19
20
|
}
|
|
20
21
|
]
|
|
21
22
|
};
|
|
22
23
|
}
|
|
23
|
-
|
|
24
|
-
export async function getConnectionSchema(db: pgwire.PgClient): Promise<DatabaseSchema[]> {
|
|
25
|
-
// https://github.com/Borvik/vscode-postgres/blob/88ec5ed061a0c9bced6c5d4ec122d0759c3f3247/src/language/server.ts
|
|
26
|
-
const results = await util.retriedQuery(
|
|
27
|
-
db,
|
|
28
|
-
`SELECT
|
|
29
|
-
tbl.schemaname,
|
|
30
|
-
tbl.tablename,
|
|
31
|
-
tbl.quoted_name,
|
|
32
|
-
json_agg(a ORDER BY attnum) as columns
|
|
33
|
-
FROM
|
|
34
|
-
(
|
|
35
|
-
SELECT
|
|
36
|
-
n.nspname as schemaname,
|
|
37
|
-
c.relname as tablename,
|
|
38
|
-
(quote_ident(n.nspname) || '.' || quote_ident(c.relname)) as quoted_name
|
|
39
|
-
FROM
|
|
40
|
-
pg_catalog.pg_class c
|
|
41
|
-
JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace
|
|
42
|
-
WHERE
|
|
43
|
-
c.relkind = 'r'
|
|
44
|
-
AND n.nspname not in ('information_schema', 'pg_catalog', 'pg_toast')
|
|
45
|
-
AND n.nspname not like 'pg_temp_%'
|
|
46
|
-
AND n.nspname not like 'pg_toast_temp_%'
|
|
47
|
-
AND c.relnatts > 0
|
|
48
|
-
AND has_schema_privilege(n.oid, 'USAGE') = true
|
|
49
|
-
AND has_table_privilege(quote_ident(n.nspname) || '.' || quote_ident(c.relname), 'SELECT, INSERT, UPDATE, DELETE, TRUNCATE, REFERENCES, TRIGGER') = true
|
|
50
|
-
) as tbl
|
|
51
|
-
LEFT JOIN (
|
|
52
|
-
SELECT
|
|
53
|
-
attrelid,
|
|
54
|
-
attname,
|
|
55
|
-
format_type(atttypid, atttypmod) as data_type,
|
|
56
|
-
(SELECT typname FROM pg_catalog.pg_type WHERE oid = atttypid) as pg_type,
|
|
57
|
-
attnum,
|
|
58
|
-
attisdropped
|
|
59
|
-
FROM
|
|
60
|
-
pg_attribute
|
|
61
|
-
) as a ON (
|
|
62
|
-
a.attrelid = tbl.quoted_name::regclass
|
|
63
|
-
AND a.attnum > 0
|
|
64
|
-
AND NOT a.attisdropped
|
|
65
|
-
AND has_column_privilege(tbl.quoted_name, a.attname, 'SELECT, INSERT, UPDATE, REFERENCES')
|
|
66
|
-
)
|
|
67
|
-
GROUP BY schemaname, tablename, quoted_name`
|
|
68
|
-
);
|
|
69
|
-
const rows = pgwireRows(results);
|
|
70
|
-
|
|
71
|
-
let schemas: Record<string, any> = {};
|
|
72
|
-
|
|
73
|
-
for (let row of rows) {
|
|
74
|
-
const schema = (schemas[row.schemaname] ??= {
|
|
75
|
-
name: row.schemaname,
|
|
76
|
-
tables: []
|
|
77
|
-
});
|
|
78
|
-
const table = {
|
|
79
|
-
name: row.tablename,
|
|
80
|
-
columns: [] as any[]
|
|
81
|
-
};
|
|
82
|
-
schema.tables.push(table);
|
|
83
|
-
|
|
84
|
-
const columnInfo = JSON.parse(row.columns);
|
|
85
|
-
for (let column of columnInfo) {
|
|
86
|
-
let pg_type = column.pg_type as string;
|
|
87
|
-
if (pg_type.startsWith('_')) {
|
|
88
|
-
pg_type = `${pg_type.substring(1)}[]`;
|
|
89
|
-
}
|
|
90
|
-
table.columns.push({
|
|
91
|
-
name: column.attname,
|
|
92
|
-
type: column.data_type,
|
|
93
|
-
pg_type: pg_type
|
|
94
|
-
});
|
|
95
|
-
}
|
|
96
|
-
}
|
|
97
|
-
|
|
98
|
-
return Object.values(schemas);
|
|
99
|
-
}
|
package/src/auth/KeyStore.ts
CHANGED
|
@@ -1,9 +1,9 @@
|
|
|
1
|
+
import { logger } from '@powersync/lib-services-framework';
|
|
1
2
|
import * as jose from 'jose';
|
|
2
3
|
import secs from '../util/secs.js';
|
|
3
|
-
import { KeyOptions, KeySpec, SUPPORTED_ALGORITHMS } from './KeySpec.js';
|
|
4
|
-
import { KeyCollector } from './KeyCollector.js';
|
|
5
4
|
import { JwtPayload } from './JwtPayload.js';
|
|
6
|
-
import {
|
|
5
|
+
import { KeyCollector } from './KeyCollector.js';
|
|
6
|
+
import { KeyOptions, KeySpec, SUPPORTED_ALGORITHMS } from './KeySpec.js';
|
|
7
7
|
|
|
8
8
|
/**
|
|
9
9
|
* KeyStore to get keys and verify tokens.
|
|
@@ -32,10 +32,13 @@ import { logger } from '@powersync/lib-services-framework';
|
|
|
32
32
|
* If we have a matching kid, we can generally get a detailed error (e.g. signature verification failed, invalid algorithm, etc).
|
|
33
33
|
* If we don't have a matching kid, we'll generally just get an error "Could not find an appropriate key...".
|
|
34
34
|
*/
|
|
35
|
-
export class KeyStore {
|
|
36
|
-
|
|
35
|
+
export class KeyStore<Collector extends KeyCollector = KeyCollector> {
|
|
36
|
+
/**
|
|
37
|
+
* @internal
|
|
38
|
+
*/
|
|
39
|
+
collector: Collector;
|
|
37
40
|
|
|
38
|
-
constructor(collector:
|
|
41
|
+
constructor(collector: Collector) {
|
|
39
42
|
this.collector = collector;
|
|
40
43
|
}
|
|
41
44
|
|
package/src/auth/auth-index.ts
CHANGED
package/src/entry/cli-entry.ts
CHANGED
|
@@ -1,10 +1,11 @@
|
|
|
1
1
|
import { Command } from 'commander';
|
|
2
2
|
|
|
3
|
+
import { logger } from '@powersync/lib-services-framework';
|
|
3
4
|
import * as utils from '../util/util-index.js';
|
|
5
|
+
import { registerCompactAction } from './commands/compact-action.js';
|
|
4
6
|
import { registerMigrationAction } from './commands/migrate-action.js';
|
|
7
|
+
import { registerStartAction } from './commands/start-action.js';
|
|
5
8
|
import { registerTearDownAction } from './commands/teardown-action.js';
|
|
6
|
-
import { registerStartAction } from './entry-index.js';
|
|
7
|
-
import { logger } from '@powersync/lib-services-framework';
|
|
8
9
|
|
|
9
10
|
/**
|
|
10
11
|
* Generates a Commander program which serves as the entry point
|
|
@@ -18,6 +19,7 @@ export function generateEntryProgram(startHandlers?: Record<utils.ServiceRunner,
|
|
|
18
19
|
|
|
19
20
|
registerTearDownAction(entryProgram);
|
|
20
21
|
registerMigrationAction(entryProgram);
|
|
22
|
+
registerCompactAction(entryProgram);
|
|
21
23
|
|
|
22
24
|
if (startHandlers) {
|
|
23
25
|
registerStartAction(entryProgram, startHandlers);
|
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
import { Command } from 'commander';
|
|
2
|
+
|
|
3
|
+
import { logger } from '@powersync/lib-services-framework';
|
|
4
|
+
import * as v8 from 'v8';
|
|
5
|
+
import * as storage from '../../storage/storage-index.js';
|
|
6
|
+
import * as utils from '../../util/util-index.js';
|
|
7
|
+
import { extractRunnerOptions, wrapConfigCommand } from './config-command.js';
|
|
8
|
+
|
|
9
|
+
const COMMAND_NAME = 'compact';
|
|
10
|
+
|
|
11
|
+
/**
|
|
12
|
+
* Approximately max-old-space-size + 64MB.
|
|
13
|
+
*/
|
|
14
|
+
const HEAP_LIMIT = v8.getHeapStatistics().heap_size_limit;
|
|
15
|
+
|
|
16
|
+
/**
|
|
17
|
+
* Subtract 128MB for process overhead.
|
|
18
|
+
*
|
|
19
|
+
* Limit to 1024MB overall.
|
|
20
|
+
*/
|
|
21
|
+
const COMPACT_MEMORY_LIMIT_MB = Math.min(HEAP_LIMIT / 1024 / 1024 - 128, 1024);
|
|
22
|
+
|
|
23
|
+
export function registerCompactAction(program: Command) {
|
|
24
|
+
const compactCommand = program.command(COMMAND_NAME);
|
|
25
|
+
|
|
26
|
+
wrapConfigCommand(compactCommand);
|
|
27
|
+
|
|
28
|
+
return compactCommand.description('Compact storage').action(async (options) => {
|
|
29
|
+
logger.info('Compacting storage...');
|
|
30
|
+
const runnerConfig = extractRunnerOptions(options);
|
|
31
|
+
const configuration = await utils.loadConfig(runnerConfig);
|
|
32
|
+
logger.info('Successfully loaded configuration...');
|
|
33
|
+
const { storage: storageConfig } = configuration;
|
|
34
|
+
logger.info('Connecting to storage...');
|
|
35
|
+
const psdb = storage.createPowerSyncMongo(storageConfig);
|
|
36
|
+
const client = psdb.client;
|
|
37
|
+
await client.connect();
|
|
38
|
+
try {
|
|
39
|
+
const bucketStorage = new storage.MongoBucketStorage(psdb, { slot_name_prefix: configuration.slot_name_prefix });
|
|
40
|
+
const active = await bucketStorage.getActiveSyncRulesContent();
|
|
41
|
+
if (active == null) {
|
|
42
|
+
logger.info('No active instance to compact');
|
|
43
|
+
return;
|
|
44
|
+
}
|
|
45
|
+
const p = bucketStorage.getInstance(active);
|
|
46
|
+
logger.info('Performing compaction...');
|
|
47
|
+
await p.compact({ memoryLimitMB: COMPACT_MEMORY_LIMIT_MB });
|
|
48
|
+
logger.info('Successfully compacted storage.');
|
|
49
|
+
} catch (e) {
|
|
50
|
+
logger.error(`Failed to compact: ${e.toString()}`);
|
|
51
|
+
process.exit(1);
|
|
52
|
+
} finally {
|
|
53
|
+
await client.close();
|
|
54
|
+
process.exit(0);
|
|
55
|
+
}
|
|
56
|
+
});
|
|
57
|
+
}
|
|
@@ -1,9 +1,8 @@
|
|
|
1
|
+
import { logger } from '@powersync/lib-services-framework';
|
|
1
2
|
import { Command } from 'commander';
|
|
2
3
|
|
|
4
|
+
import * as migrations from '../../migrations/migrations-index.js';
|
|
3
5
|
import { extractRunnerOptions, wrapConfigCommand } from './config-command.js';
|
|
4
|
-
import { migrate } from '../../migrations/migrations.js';
|
|
5
|
-
import { Direction } from '../../migrations/definitions.js';
|
|
6
|
-
import { logger } from '@powersync/lib-services-framework';
|
|
7
6
|
|
|
8
7
|
const COMMAND_NAME = 'migrate';
|
|
9
8
|
|
|
@@ -15,13 +14,11 @@ export function registerMigrationAction(program: Command) {
|
|
|
15
14
|
return migrationCommand
|
|
16
15
|
.description('Run migrations')
|
|
17
16
|
.argument('<direction>', 'Migration direction. `up` or `down`')
|
|
18
|
-
.action(async (direction: Direction, options) => {
|
|
19
|
-
const runnerConfig = extractRunnerOptions(options);
|
|
20
|
-
|
|
17
|
+
.action(async (direction: migrations.Direction, options) => {
|
|
21
18
|
try {
|
|
22
|
-
await migrate({
|
|
19
|
+
await migrations.migrate({
|
|
23
20
|
direction,
|
|
24
|
-
runner_config:
|
|
21
|
+
runner_config: extractRunnerOptions(options)
|
|
25
22
|
});
|
|
26
23
|
|
|
27
24
|
process.exit(0);
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { Command } from 'commander';
|
|
2
2
|
|
|
3
|
-
import { extractRunnerOptions, wrapConfigCommand } from './config-command.js';
|
|
4
3
|
import { teardown } from '../../runner/teardown.js';
|
|
4
|
+
import { extractRunnerOptions, wrapConfigCommand } from './config-command.js';
|
|
5
5
|
|
|
6
6
|
const COMMAND_NAME = 'teardown';
|
|
7
7
|
|
|
@@ -12,7 +12,7 @@ export function registerTearDownAction(program: Command) {
|
|
|
12
12
|
|
|
13
13
|
return teardownCommand
|
|
14
14
|
.argument('[ack]', 'Type `TEARDOWN` to confirm teardown should occur')
|
|
15
|
-
.description('Terminate all replicating sync rules,
|
|
15
|
+
.description('Terminate all replicating sync rules, clear remote configuration and remove all data')
|
|
16
16
|
.action(async (ack, options) => {
|
|
17
17
|
if (ack !== 'TEARDOWN') {
|
|
18
18
|
throw new Error('TEARDOWN was not acknowledged.');
|
package/src/entry/entry-index.ts
CHANGED
package/src/index.ts
CHANGED
|
@@ -18,8 +18,11 @@ export * as framework from '@powersync/lib-services-framework';
|
|
|
18
18
|
export * from './metrics/Metrics.js';
|
|
19
19
|
export * as metrics from './metrics/Metrics.js';
|
|
20
20
|
|
|
21
|
-
export * from './migrations/migrations.js';
|
|
22
21
|
export * as migrations from './migrations/migrations-index.js';
|
|
22
|
+
export * from './migrations/migrations.js';
|
|
23
|
+
|
|
24
|
+
export * from './modules/modules-index.js';
|
|
25
|
+
export * as modules from './modules/modules-index.js';
|
|
23
26
|
|
|
24
27
|
export * from './replication/replication-index.js';
|
|
25
28
|
export * as replication from './replication/replication-index.js';
|
|
@@ -33,7 +36,7 @@ export * as storage from './storage/storage-index.js';
|
|
|
33
36
|
export * from './sync/sync-index.js';
|
|
34
37
|
export * as sync from './sync/sync-index.js';
|
|
35
38
|
|
|
36
|
-
export * from './system/
|
|
39
|
+
export * from './system/system-index.js';
|
|
37
40
|
export * as system from './system/system-index.js';
|
|
38
41
|
|
|
39
42
|
export * from './util/util-index.js';
|
package/src/metrics/Metrics.ts
CHANGED
|
@@ -1,10 +1,11 @@
|
|
|
1
1
|
import { Attributes, Counter, ObservableGauge, UpDownCounter, ValueType } from '@opentelemetry/api';
|
|
2
|
+
import { OTLPMetricExporter } from '@opentelemetry/exporter-metrics-otlp-http';
|
|
2
3
|
import { PrometheusExporter } from '@opentelemetry/exporter-prometheus';
|
|
3
|
-
import {
|
|
4
|
-
import
|
|
5
|
-
import * as storage from '../storage/storage-index.js';
|
|
6
|
-
import { CorePowerSyncSystem } from '../system/CorePowerSyncSystem.js';
|
|
4
|
+
import { Resource } from '@opentelemetry/resources';
|
|
5
|
+
import { MeterProvider, MetricReader, PeriodicExportingMetricReader } from '@opentelemetry/sdk-metrics';
|
|
7
6
|
import { logger } from '@powersync/lib-services-framework';
|
|
7
|
+
import * as storage from '../storage/storage-index.js';
|
|
8
|
+
import * as util from '../util/util-index.js';
|
|
8
9
|
|
|
9
10
|
export interface MetricsOptions {
|
|
10
11
|
disable_telemetry_sharing: boolean;
|
|
@@ -13,6 +14,8 @@ export interface MetricsOptions {
|
|
|
13
14
|
}
|
|
14
15
|
|
|
15
16
|
export class Metrics {
|
|
17
|
+
private static instance: Metrics;
|
|
18
|
+
|
|
16
19
|
private prometheusExporter: PrometheusExporter;
|
|
17
20
|
private meterProvider: MeterProvider;
|
|
18
21
|
|
|
@@ -55,7 +58,7 @@ export class Metrics {
|
|
|
55
58
|
// Record on API pod
|
|
56
59
|
public concurrent_connections: UpDownCounter<Attributes>;
|
|
57
60
|
|
|
58
|
-
constructor(meterProvider: MeterProvider, prometheusExporter: PrometheusExporter) {
|
|
61
|
+
private constructor(meterProvider: MeterProvider, prometheusExporter: PrometheusExporter) {
|
|
59
62
|
this.meterProvider = meterProvider;
|
|
60
63
|
this.prometheusExporter = prometheusExporter;
|
|
61
64
|
const meter = meterProvider.getMeter('powersync');
|
|
@@ -127,6 +130,66 @@ export class Metrics {
|
|
|
127
130
|
this.concurrent_connections.add(0);
|
|
128
131
|
}
|
|
129
132
|
|
|
133
|
+
public static getInstance(): Metrics {
|
|
134
|
+
if (!Metrics.instance) {
|
|
135
|
+
throw new Error('Metrics have not been initialised');
|
|
136
|
+
}
|
|
137
|
+
|
|
138
|
+
return Metrics.instance;
|
|
139
|
+
}
|
|
140
|
+
|
|
141
|
+
public static async initialise(options: MetricsOptions): Promise<void> {
|
|
142
|
+
if (Metrics.instance) {
|
|
143
|
+
return;
|
|
144
|
+
}
|
|
145
|
+
logger.info('Configuring telemetry.');
|
|
146
|
+
|
|
147
|
+
logger.info(
|
|
148
|
+
`
|
|
149
|
+
Attention:
|
|
150
|
+
PowerSync collects completely anonymous telemetry regarding usage.
|
|
151
|
+
This information is used to shape our roadmap to better serve our customers.
|
|
152
|
+
You can learn more, including how to opt-out if you'd not like to participate in this anonymous program, by visiting the following URL:
|
|
153
|
+
https://docs.powersync.com/self-hosting/telemetry
|
|
154
|
+
Anonymous telemetry is currently: ${options.disable_telemetry_sharing ? 'disabled' : 'enabled'}
|
|
155
|
+
`.trim()
|
|
156
|
+
);
|
|
157
|
+
|
|
158
|
+
const configuredExporters: MetricReader[] = [];
|
|
159
|
+
|
|
160
|
+
const port: number = util.env.METRICS_PORT ?? 0;
|
|
161
|
+
const prometheusExporter = new PrometheusExporter({ port: port, preventServerStart: true });
|
|
162
|
+
configuredExporters.push(prometheusExporter);
|
|
163
|
+
|
|
164
|
+
if (!options.disable_telemetry_sharing) {
|
|
165
|
+
logger.info('Sharing anonymous telemetry');
|
|
166
|
+
const periodicExporter = new PeriodicExportingMetricReader({
|
|
167
|
+
exporter: new OTLPMetricExporter({
|
|
168
|
+
url: options.internal_metrics_endpoint
|
|
169
|
+
}),
|
|
170
|
+
exportIntervalMillis: 1000 * 60 * 5 // 5 minutes
|
|
171
|
+
});
|
|
172
|
+
|
|
173
|
+
configuredExporters.push(periodicExporter);
|
|
174
|
+
}
|
|
175
|
+
|
|
176
|
+
const meterProvider = new MeterProvider({
|
|
177
|
+
resource: new Resource({
|
|
178
|
+
['service']: 'PowerSync',
|
|
179
|
+
['instance_id']: options.powersync_instance_id
|
|
180
|
+
}),
|
|
181
|
+
readers: configuredExporters
|
|
182
|
+
});
|
|
183
|
+
|
|
184
|
+
if (port > 0) {
|
|
185
|
+
await prometheusExporter.startServer();
|
|
186
|
+
}
|
|
187
|
+
|
|
188
|
+
Metrics.instance = new Metrics(meterProvider, prometheusExporter);
|
|
189
|
+
|
|
190
|
+
logger.info('Telemetry configuration complete.');
|
|
191
|
+
}
|
|
192
|
+
|
|
130
193
|
public async shutdown(): Promise<void> {
|
|
131
194
|
await this.meterProvider.shutdown();
|
|
132
195
|
}
|
|
@@ -137,7 +200,7 @@ export class Metrics {
|
|
|
137
200
|
this.concurrent_connections.add(0);
|
|
138
201
|
}
|
|
139
202
|
|
|
140
|
-
public configureReplicationMetrics(
|
|
203
|
+
public configureReplicationMetrics(bucketStorage: storage.BucketStorageFactory) {
|
|
141
204
|
// Rate limit collection of these stats, since it may be an expensive query
|
|
142
205
|
const MINIMUM_INTERVAL = 60_000;
|
|
143
206
|
|
|
@@ -146,7 +209,7 @@ export class Metrics {
|
|
|
146
209
|
|
|
147
210
|
function getMetrics() {
|
|
148
211
|
if (cachedRequest == null || Date.now() - cacheTimestamp > MINIMUM_INTERVAL) {
|
|
149
|
-
cachedRequest =
|
|
212
|
+
cachedRequest = bucketStorage.getStorageMetrics().catch((e) => {
|
|
150
213
|
logger.error(`Failed to get storage metrics`, e);
|
|
151
214
|
return null;
|
|
152
215
|
});
|
|
@@ -175,14 +238,6 @@ export class Metrics {
|
|
|
175
238
|
result.observe(metrics.replication_size_bytes);
|
|
176
239
|
}
|
|
177
240
|
});
|
|
178
|
-
|
|
179
|
-
const class_scoped_data_replicated_bytes = this.data_replicated_bytes;
|
|
180
|
-
// Record replicated bytes using global jpgwire metrics.
|
|
181
|
-
jpgwire.setMetricsRecorder({
|
|
182
|
-
addBytesRead(bytes) {
|
|
183
|
-
class_scoped_data_replicated_bytes.add(bytes);
|
|
184
|
-
}
|
|
185
|
-
});
|
|
186
241
|
}
|
|
187
242
|
|
|
188
243
|
public async getMetricValueForTests(name: string): Promise<number | undefined> {
|
|
@@ -2,8 +2,11 @@ import * as mongo from '../../../db/mongo.js';
|
|
|
2
2
|
import * as storage from '../../../storage/storage-index.js';
|
|
3
3
|
import * as utils from '../../../util/util-index.js';
|
|
4
4
|
|
|
5
|
-
export const up = async (context
|
|
6
|
-
const
|
|
5
|
+
export const up = async (context: utils.MigrationContext) => {
|
|
6
|
+
const { runner_config } = context;
|
|
7
|
+
|
|
8
|
+
const config = await utils.loadConfig(runner_config);
|
|
9
|
+
|
|
7
10
|
const database = storage.createPowerSyncMongo(config.storage);
|
|
8
11
|
await mongo.waitForAuth(database.db);
|
|
9
12
|
try {
|
|
@@ -20,8 +23,10 @@ export const up = async (context?: utils.MigrationContext) => {
|
|
|
20
23
|
}
|
|
21
24
|
};
|
|
22
25
|
|
|
23
|
-
export const down = async (context
|
|
24
|
-
const
|
|
26
|
+
export const down = async (context: utils.MigrationContext) => {
|
|
27
|
+
const { runner_config } = context;
|
|
28
|
+
const config = await utils.loadConfig(runner_config);
|
|
29
|
+
|
|
25
30
|
const database = storage.createPowerSyncMongo(config.storage);
|
|
26
31
|
try {
|
|
27
32
|
if (await database.bucket_parameters.indexExists('lookup')) {
|
|
@@ -23,9 +23,11 @@ interface LegacySyncRulesDocument extends storage.SyncRuleDocument {
|
|
|
23
23
|
auto_activate?: boolean;
|
|
24
24
|
}
|
|
25
25
|
|
|
26
|
-
export const up = async (context
|
|
27
|
-
const
|
|
26
|
+
export const up = async (context: utils.MigrationContext) => {
|
|
27
|
+
const { runner_config } = context;
|
|
28
|
+
const config = await utils.loadConfig(runner_config);
|
|
28
29
|
const db = storage.createPowerSyncMongo(config.storage);
|
|
30
|
+
|
|
29
31
|
await mongo.waitForAuth(db.db);
|
|
30
32
|
try {
|
|
31
33
|
// We keep the old flags for existing deployments still shutting down.
|
|
@@ -68,8 +70,9 @@ export const up = async (context?: utils.MigrationContext) => {
|
|
|
68
70
|
}
|
|
69
71
|
};
|
|
70
72
|
|
|
71
|
-
export const down = async (context
|
|
72
|
-
const
|
|
73
|
+
export const down = async (context: utils.MigrationContext) => {
|
|
74
|
+
const { runner_config } = context;
|
|
75
|
+
const config = await utils.loadConfig(runner_config);
|
|
73
76
|
|
|
74
77
|
const db = storage.createPowerSyncMongo(config.storage);
|
|
75
78
|
try {
|
|
@@ -1,8 +1,9 @@
|
|
|
1
1
|
import * as storage from '../../../storage/storage-index.js';
|
|
2
2
|
import * as utils from '../../../util/util-index.js';
|
|
3
3
|
|
|
4
|
-
export const up = async (context
|
|
5
|
-
const
|
|
4
|
+
export const up = async (context: utils.MigrationContext) => {
|
|
5
|
+
const { runner_config } = context;
|
|
6
|
+
const config = await utils.loadConfig(runner_config);
|
|
6
7
|
const db = storage.createPowerSyncMongo(config.storage);
|
|
7
8
|
|
|
8
9
|
try {
|
|
@@ -17,8 +18,9 @@ export const up = async (context?: utils.MigrationContext) => {
|
|
|
17
18
|
}
|
|
18
19
|
};
|
|
19
20
|
|
|
20
|
-
export const down = async (context
|
|
21
|
-
const
|
|
21
|
+
export const down = async (context: utils.MigrationContext) => {
|
|
22
|
+
const { runner_config } = context;
|
|
23
|
+
const config = await utils.loadConfig(runner_config);
|
|
22
24
|
|
|
23
25
|
const db = storage.createPowerSyncMongo(config.storage);
|
|
24
26
|
|
|
@@ -2,13 +2,13 @@ import * as fs from 'fs/promises';
|
|
|
2
2
|
import * as path from 'path';
|
|
3
3
|
import { fileURLToPath } from 'url';
|
|
4
4
|
|
|
5
|
+
import { logger } from '@powersync/lib-services-framework';
|
|
5
6
|
import * as db from '../db/db-index.js';
|
|
6
|
-
import * as util from '../util/util-index.js';
|
|
7
7
|
import * as locks from '../locks/locks-index.js';
|
|
8
|
+
import * as util from '../util/util-index.js';
|
|
8
9
|
import { Direction } from './definitions.js';
|
|
9
|
-
import { createMongoMigrationStore } from './store/migration-store.js';
|
|
10
10
|
import { execute, writeLogsToStore } from './executor.js';
|
|
11
|
-
import {
|
|
11
|
+
import { createMongoMigrationStore } from './store/migration-store.js';
|
|
12
12
|
|
|
13
13
|
const DEFAULT_MONGO_LOCK_COLLECTION = 'locks';
|
|
14
14
|
const MONGO_LOCK_PROCESS = 'migrations';
|
|
@@ -23,18 +23,23 @@ export type MigrationOptions = {
|
|
|
23
23
|
runner_config: util.RunnerConfig;
|
|
24
24
|
};
|
|
25
25
|
|
|
26
|
+
export type AutomaticMigrationParams = {
|
|
27
|
+
config: util.ResolvedPowerSyncConfig;
|
|
28
|
+
runner_config: util.RunnerConfig;
|
|
29
|
+
};
|
|
30
|
+
|
|
26
31
|
/**
|
|
27
32
|
* Loads migrations and injects a custom context for loading the specified
|
|
28
33
|
* runner configuration.
|
|
29
34
|
*/
|
|
30
|
-
const loadMigrations = async (dir: string,
|
|
35
|
+
const loadMigrations = async (dir: string, runnerConfig: util.RunnerConfig) => {
|
|
31
36
|
const files = await fs.readdir(dir);
|
|
32
37
|
const migrations = files.filter((file) => {
|
|
33
38
|
return path.extname(file) === '.js';
|
|
34
39
|
});
|
|
35
40
|
|
|
36
41
|
const context: util.MigrationContext = {
|
|
37
|
-
runner_config
|
|
42
|
+
runner_config: runnerConfig
|
|
38
43
|
};
|
|
39
44
|
|
|
40
45
|
return await Promise.all(
|
|
@@ -55,14 +60,13 @@ const loadMigrations = async (dir: string, runner_config: util.RunnerConfig) =>
|
|
|
55
60
|
export const migrate = async (options: MigrationOptions) => {
|
|
56
61
|
const { direction, runner_config } = options;
|
|
57
62
|
|
|
63
|
+
const config = await util.loadConfig(runner_config);
|
|
64
|
+
const { storage } = config;
|
|
58
65
|
/**
|
|
59
66
|
* Try and get Mongo from config file.
|
|
60
67
|
* But this might not be available in Journey Micro as we use the standard Mongo.
|
|
61
68
|
*/
|
|
62
69
|
|
|
63
|
-
const config = await util.loadConfig(runner_config);
|
|
64
|
-
const { storage } = config;
|
|
65
|
-
|
|
66
70
|
const client = db.mongo.createMongoClient(storage);
|
|
67
71
|
logger.info('Connecting to MongoDB');
|
|
68
72
|
await client.connect();
|
|
@@ -124,3 +128,15 @@ export const migrate = async (options: MigrationOptions) => {
|
|
|
124
128
|
logger.info('Done with migrations');
|
|
125
129
|
}
|
|
126
130
|
};
|
|
131
|
+
|
|
132
|
+
/**
|
|
133
|
+
* Ensures automatic migrations are executed
|
|
134
|
+
*/
|
|
135
|
+
export const ensureAutomaticMigrations = async (params: AutomaticMigrationParams) => {
|
|
136
|
+
if (!params.config.migrations?.disable_auto_migration) {
|
|
137
|
+
await migrate({
|
|
138
|
+
direction: Direction.Up,
|
|
139
|
+
runner_config: params.runner_config
|
|
140
|
+
});
|
|
141
|
+
}
|
|
142
|
+
};
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
import { ServiceContextContainer } from '../system/ServiceContext.js';
|
|
2
|
+
import { logger } from '@powersync/lib-services-framework';
|
|
3
|
+
import winston from 'winston';
|
|
4
|
+
import { PersistedSyncRulesContent } from '../storage/BucketStorage.js';
|
|
5
|
+
|
|
6
|
+
export interface TearDownOptions {
|
|
7
|
+
/**
|
|
8
|
+
* If required, tear down any configuration/state for the specific sync rules
|
|
9
|
+
*/
|
|
10
|
+
syncRules?: PersistedSyncRulesContent[];
|
|
11
|
+
}
|
|
12
|
+
|
|
13
|
+
export interface AbstractModuleOptions {
|
|
14
|
+
name: string;
|
|
15
|
+
}
|
|
16
|
+
|
|
17
|
+
export abstract class AbstractModule {
|
|
18
|
+
protected logger: winston.Logger;
|
|
19
|
+
|
|
20
|
+
protected constructor(protected options: AbstractModuleOptions) {
|
|
21
|
+
this.logger = logger.child({ name: `Module:${options.name}` });
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
/**
|
|
25
|
+
* Initialize the module using any required services from the ServiceContext
|
|
26
|
+
*/
|
|
27
|
+
public abstract initialize(context: ServiceContextContainer): Promise<void>;
|
|
28
|
+
|
|
29
|
+
/**
|
|
30
|
+
* Permanently clean up and dispose of any configuration or state for this module.
|
|
31
|
+
*/
|
|
32
|
+
public abstract teardown(options: TearDownOptions): Promise<void>;
|
|
33
|
+
|
|
34
|
+
public get name() {
|
|
35
|
+
return this.options.name;
|
|
36
|
+
}
|
|
37
|
+
}
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
import { logger } from '@powersync/lib-services-framework';
|
|
2
|
+
import * as system from '../system/system-index.js';
|
|
3
|
+
import { AbstractModule, TearDownOptions } from './AbstractModule.js';
|
|
4
|
+
/**
|
|
5
|
+
* The module manager keeps track of activated modules
|
|
6
|
+
*/
|
|
7
|
+
export class ModuleManager {
|
|
8
|
+
private readonly modules: Map<string, AbstractModule> = new Map();
|
|
9
|
+
|
|
10
|
+
public register(modules: AbstractModule[]) {
|
|
11
|
+
for (const module of modules) {
|
|
12
|
+
if (this.modules.has(module.name)) {
|
|
13
|
+
logger.warn(`Module ${module.name} already registered, skipping.`);
|
|
14
|
+
continue;
|
|
15
|
+
}
|
|
16
|
+
this.modules.set(module.name, module);
|
|
17
|
+
logger.info(`Successfully registered Module ${module.name}.`);
|
|
18
|
+
}
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
async initialize(serviceContext: system.ServiceContextContainer) {
|
|
22
|
+
logger.info(`Initializing modules...`);
|
|
23
|
+
for (const module of this.modules.values()) {
|
|
24
|
+
await module.initialize(serviceContext);
|
|
25
|
+
}
|
|
26
|
+
logger.info(`Successfully Initialized modules.`);
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
async tearDown(options: TearDownOptions) {
|
|
30
|
+
for (const module of this.modules.values()) {
|
|
31
|
+
await module.teardown(options);
|
|
32
|
+
}
|
|
33
|
+
}
|
|
34
|
+
}
|