@powersync/service-core 0.8.8 → 0.9.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +31 -0
- package/dist/api/RouteAPI.d.ts +67 -0
- package/dist/api/RouteAPI.js +2 -0
- package/dist/api/RouteAPI.js.map +1 -0
- package/dist/api/api-index.d.ts +1 -0
- package/dist/api/api-index.js +1 -0
- package/dist/api/api-index.js.map +1 -1
- package/dist/api/diagnostics.d.ts +4 -4
- package/dist/api/diagnostics.js +170 -158
- package/dist/api/diagnostics.js.map +1 -1
- package/dist/api/schema.d.ts +3 -5
- package/dist/api/schema.js +14 -80
- package/dist/api/schema.js.map +1 -1
- package/dist/auth/CachedKeyCollector.js.map +1 -1
- package/dist/auth/KeySpec.js.map +1 -1
- package/dist/auth/KeyStore.d.ts +7 -4
- package/dist/auth/KeyStore.js +1 -1
- package/dist/auth/KeyStore.js.map +1 -1
- package/dist/auth/LeakyBucket.js.map +1 -1
- package/dist/auth/RemoteJWKSCollector.d.ts +0 -2
- package/dist/auth/RemoteJWKSCollector.js.map +1 -1
- package/dist/auth/auth-index.d.ts +0 -1
- package/dist/auth/auth-index.js +0 -1
- package/dist/auth/auth-index.js.map +1 -1
- package/dist/db/mongo.js +5 -3
- package/dist/db/mongo.js.map +1 -1
- package/dist/entry/cli-entry.js +3 -2
- package/dist/entry/cli-entry.js.map +1 -1
- package/dist/entry/commands/compact-action.js +90 -14
- package/dist/entry/commands/compact-action.js.map +1 -1
- package/dist/entry/commands/migrate-action.js +4 -5
- package/dist/entry/commands/migrate-action.js.map +1 -1
- package/dist/entry/commands/teardown-action.js +2 -2
- package/dist/entry/commands/teardown-action.js.map +1 -1
- package/dist/index.d.ts +4 -2
- package/dist/index.js +4 -2
- package/dist/index.js.map +1 -1
- package/dist/locks/MongoLocks.js.map +1 -1
- package/dist/metrics/Metrics.d.ts +2 -2
- package/dist/metrics/Metrics.js +5 -13
- package/dist/metrics/Metrics.js.map +1 -1
- package/dist/migrations/db/migrations/1684951997326-init.d.ts +2 -2
- package/dist/migrations/db/migrations/1684951997326-init.js +4 -2
- package/dist/migrations/db/migrations/1684951997326-init.js.map +1 -1
- package/dist/migrations/db/migrations/1702295701188-sync-rule-state.d.ts +2 -2
- package/dist/migrations/db/migrations/1702295701188-sync-rule-state.js +4 -2
- package/dist/migrations/db/migrations/1702295701188-sync-rule-state.js.map +1 -1
- package/dist/migrations/db/migrations/1711543888062-write-checkpoint-index.d.ts +2 -2
- package/dist/migrations/db/migrations/1711543888062-write-checkpoint-index.js +4 -2
- package/dist/migrations/db/migrations/1711543888062-write-checkpoint-index.js.map +1 -1
- package/dist/migrations/db/migrations/1727099539247-custom-write-checkpoint-index.d.ts +3 -0
- package/dist/migrations/db/migrations/1727099539247-custom-write-checkpoint-index.js +31 -0
- package/dist/migrations/db/migrations/1727099539247-custom-write-checkpoint-index.js.map +1 -0
- package/dist/migrations/executor.js.map +1 -1
- package/dist/migrations/migrations.d.ts +8 -0
- package/dist/migrations/migrations.js +19 -7
- package/dist/migrations/migrations.js.map +1 -1
- package/dist/migrations/store/migration-store.js.map +1 -1
- package/dist/modules/AbstractModule.d.ts +26 -0
- package/dist/modules/AbstractModule.js +11 -0
- package/dist/modules/AbstractModule.js.map +1 -0
- package/dist/modules/ModuleManager.d.ts +11 -0
- package/dist/modules/ModuleManager.js +32 -0
- package/dist/modules/ModuleManager.js.map +1 -0
- package/dist/modules/modules-index.d.ts +2 -0
- package/dist/modules/modules-index.js +3 -0
- package/dist/modules/modules-index.js.map +1 -0
- package/dist/replication/AbstractReplicationJob.d.ts +37 -0
- package/dist/replication/AbstractReplicationJob.js +51 -0
- package/dist/replication/AbstractReplicationJob.js.map +1 -0
- package/dist/replication/AbstractReplicator.d.ts +53 -0
- package/dist/replication/AbstractReplicator.js +250 -0
- package/dist/replication/AbstractReplicator.js.map +1 -0
- package/dist/replication/ErrorRateLimiter.d.ts +0 -10
- package/dist/replication/ErrorRateLimiter.js +1 -42
- package/dist/replication/ErrorRateLimiter.js.map +1 -1
- package/dist/replication/ReplicationEngine.d.ts +18 -0
- package/dist/replication/ReplicationEngine.js +41 -0
- package/dist/replication/ReplicationEngine.js.map +1 -0
- package/dist/replication/ReplicationModule.d.ts +51 -0
- package/dist/replication/ReplicationModule.js +68 -0
- package/dist/replication/ReplicationModule.js.map +1 -0
- package/dist/replication/replication-index.d.ts +4 -6
- package/dist/replication/replication-index.js +4 -6
- package/dist/replication/replication-index.js.map +1 -1
- package/dist/routes/RouterEngine.d.ts +42 -0
- package/dist/routes/RouterEngine.js +80 -0
- package/dist/routes/RouterEngine.js.map +1 -0
- package/dist/routes/auth.d.ts +2 -2
- package/dist/routes/auth.js +11 -11
- package/dist/routes/auth.js.map +1 -1
- package/dist/routes/configure-fastify.d.ts +37 -23
- package/dist/routes/configure-fastify.js +18 -18
- package/dist/routes/configure-fastify.js.map +1 -1
- package/dist/routes/configure-rsocket.d.ts +3 -4
- package/dist/routes/configure-rsocket.js +7 -4
- package/dist/routes/configure-rsocket.js.map +1 -1
- package/dist/routes/endpoints/admin.d.ts +30 -0
- package/dist/routes/endpoints/admin.js +46 -67
- package/dist/routes/endpoints/admin.js.map +1 -1
- package/dist/routes/endpoints/checkpointing.js +103 -15
- package/dist/routes/endpoints/checkpointing.js.map +1 -1
- package/dist/routes/endpoints/socket-route.js +8 -6
- package/dist/routes/endpoints/socket-route.js.map +1 -1
- package/dist/routes/endpoints/sync-rules.d.ts +1 -1
- package/dist/routes/endpoints/sync-rules.js +32 -23
- package/dist/routes/endpoints/sync-rules.js.map +1 -1
- package/dist/routes/endpoints/sync-stream.d.ts +0 -1
- package/dist/routes/endpoints/sync-stream.js +8 -8
- package/dist/routes/endpoints/sync-stream.js.map +1 -1
- package/dist/routes/hooks.js.map +1 -1
- package/dist/routes/route-register.js.map +1 -1
- package/dist/routes/router.d.ts +9 -2
- package/dist/routes/router.js.map +1 -1
- package/dist/routes/routes-index.d.ts +1 -0
- package/dist/routes/routes-index.js +1 -0
- package/dist/routes/routes-index.js.map +1 -1
- package/dist/runner/teardown.js +109 -76
- package/dist/runner/teardown.js.map +1 -1
- package/dist/storage/BucketStorage.d.ts +86 -36
- package/dist/storage/BucketStorage.js +6 -10
- package/dist/storage/BucketStorage.js.map +1 -1
- package/dist/storage/ChecksumCache.js.map +1 -1
- package/dist/storage/MongoBucketStorage.d.ts +7 -11
- package/dist/storage/MongoBucketStorage.js +48 -41
- package/dist/storage/MongoBucketStorage.js.map +1 -1
- package/dist/storage/ReplicationEventPayload.d.ts +14 -0
- package/dist/storage/ReplicationEventPayload.js +2 -0
- package/dist/storage/ReplicationEventPayload.js.map +1 -0
- package/dist/storage/SourceEntity.d.ts +20 -0
- package/dist/storage/SourceEntity.js +2 -0
- package/dist/storage/SourceEntity.js.map +1 -0
- package/dist/storage/SourceTable.d.ts +12 -5
- package/dist/storage/SourceTable.js +12 -5
- package/dist/storage/SourceTable.js.map +1 -1
- package/dist/storage/StorageEngine.d.ts +28 -0
- package/dist/storage/StorageEngine.js +45 -0
- package/dist/storage/StorageEngine.js.map +1 -0
- package/dist/storage/StorageProvider.d.ts +21 -0
- package/dist/storage/StorageProvider.js +2 -0
- package/dist/storage/StorageProvider.js.map +1 -0
- package/dist/storage/WriteCheckpointAPI.d.ts +74 -0
- package/dist/storage/WriteCheckpointAPI.js +16 -0
- package/dist/storage/WriteCheckpointAPI.js.map +1 -0
- package/dist/storage/mongo/MongoBucketBatch.d.ts +24 -5
- package/dist/storage/mongo/MongoBucketBatch.js +119 -62
- package/dist/storage/mongo/MongoBucketBatch.js.map +1 -1
- package/dist/storage/mongo/MongoCompactor.js +20 -3
- package/dist/storage/mongo/MongoCompactor.js.map +1 -1
- package/dist/storage/mongo/MongoIdSequence.js.map +1 -1
- package/dist/storage/mongo/MongoPersistedSyncRulesContent.d.ts +2 -2
- package/dist/storage/mongo/MongoPersistedSyncRulesContent.js +2 -2
- package/dist/storage/mongo/MongoPersistedSyncRulesContent.js.map +1 -1
- package/dist/storage/mongo/MongoStorageProvider.d.ts +5 -0
- package/dist/storage/mongo/MongoStorageProvider.js +26 -0
- package/dist/storage/mongo/MongoStorageProvider.js.map +1 -0
- package/dist/storage/mongo/MongoSyncBucketStorage.d.ts +18 -10
- package/dist/storage/mongo/MongoSyncBucketStorage.js +140 -25
- package/dist/storage/mongo/MongoSyncBucketStorage.js.map +1 -1
- package/dist/storage/mongo/MongoSyncRulesLock.js +1 -1
- package/dist/storage/mongo/MongoSyncRulesLock.js.map +1 -1
- package/dist/storage/mongo/MongoWriteCheckpointAPI.d.ts +20 -0
- package/dist/storage/mongo/MongoWriteCheckpointAPI.js +103 -0
- package/dist/storage/mongo/MongoWriteCheckpointAPI.js.map +1 -0
- package/dist/storage/mongo/OperationBatch.d.ts +13 -4
- package/dist/storage/mongo/OperationBatch.js +25 -7
- package/dist/storage/mongo/OperationBatch.js.map +1 -1
- package/dist/storage/mongo/PersistedBatch.d.ts +3 -3
- package/dist/storage/mongo/PersistedBatch.js +2 -2
- package/dist/storage/mongo/PersistedBatch.js.map +1 -1
- package/dist/storage/mongo/config.d.ts +19 -0
- package/dist/storage/mongo/config.js +26 -0
- package/dist/storage/mongo/config.js.map +1 -0
- package/dist/storage/mongo/db.d.ts +3 -2
- package/dist/storage/mongo/db.js +1 -0
- package/dist/storage/mongo/db.js.map +1 -1
- package/dist/storage/mongo/models.d.ts +20 -5
- package/dist/storage/mongo/models.js.map +1 -1
- package/dist/storage/mongo/util.d.ts +12 -1
- package/dist/storage/mongo/util.js +50 -2
- package/dist/storage/mongo/util.js.map +1 -1
- package/dist/storage/storage-index.d.ts +8 -2
- package/dist/storage/storage-index.js +8 -2
- package/dist/storage/storage-index.js.map +1 -1
- package/dist/sync/BroadcastIterable.d.ts +0 -1
- package/dist/sync/BroadcastIterable.js.map +1 -1
- package/dist/sync/LastValueSink.d.ts +0 -1
- package/dist/sync/LastValueSink.js.map +1 -1
- package/dist/sync/merge.d.ts +0 -1
- package/dist/sync/merge.js.map +1 -1
- package/dist/sync/safeRace.js.map +1 -1
- package/dist/sync/sync.d.ts +1 -1
- package/dist/sync/sync.js +5 -5
- package/dist/sync/sync.js.map +1 -1
- package/dist/sync/util.d.ts +0 -2
- package/dist/sync/util.js.map +1 -1
- package/dist/system/ServiceContext.d.ts +37 -0
- package/dist/system/ServiceContext.js +48 -0
- package/dist/system/ServiceContext.js.map +1 -0
- package/dist/system/system-index.d.ts +1 -1
- package/dist/system/system-index.js +1 -1
- package/dist/system/system-index.js.map +1 -1
- package/dist/util/Mutex.js.map +1 -1
- package/dist/util/config/collectors/config-collector.js.map +1 -1
- package/dist/util/config/collectors/impl/base64-config-collector.js.map +1 -1
- package/dist/util/config/collectors/impl/filesystem-config-collector.js.map +1 -1
- package/dist/util/config/compound-config-collector.d.ts +9 -2
- package/dist/util/config/compound-config-collector.js +16 -24
- package/dist/util/config/compound-config-collector.js.map +1 -1
- package/dist/util/config/sync-rules/impl/base64-sync-rules-collector.js.map +1 -1
- package/dist/util/config/sync-rules/impl/filesystem-sync-rules-collector.js.map +1 -1
- package/dist/util/config/sync-rules/impl/inline-sync-rules-collector.js.map +1 -1
- package/dist/util/config/sync-rules/sync-rules-provider.d.ts +9 -0
- package/dist/util/config/sync-rules/sync-rules-provider.js +15 -0
- package/dist/util/config/sync-rules/sync-rules-provider.js.map +1 -0
- package/dist/util/config/types.d.ts +7 -4
- package/dist/util/config/types.js.map +1 -1
- package/dist/util/config.d.ts +3 -4
- package/dist/util/config.js +5 -20
- package/dist/util/config.js.map +1 -1
- package/dist/util/memory-tracking.js.map +1 -1
- package/dist/util/secs.js.map +1 -1
- package/dist/util/util-index.d.ts +3 -6
- package/dist/util/util-index.js +3 -6
- package/dist/util/util-index.js.map +1 -1
- package/dist/util/utils.d.ts +10 -7
- package/dist/util/utils.js +36 -25
- package/dist/util/utils.js.map +1 -1
- package/package.json +8 -12
- package/src/api/RouteAPI.ts +78 -0
- package/src/api/api-index.ts +1 -0
- package/src/api/diagnostics.ts +18 -70
- package/src/api/schema.ts +18 -90
- package/src/auth/KeyStore.ts +9 -6
- package/src/auth/RemoteJWKSCollector.ts +4 -1
- package/src/auth/auth-index.ts +0 -1
- package/src/db/mongo.ts +5 -3
- package/src/entry/cli-entry.ts +3 -2
- package/src/entry/commands/compact-action.ts +24 -12
- package/src/entry/commands/migrate-action.ts +5 -8
- package/src/entry/commands/teardown-action.ts +2 -2
- package/src/index.ts +5 -2
- package/src/metrics/Metrics.ts +6 -16
- package/src/migrations/db/migrations/1684951997326-init.ts +9 -4
- package/src/migrations/db/migrations/1702295701188-sync-rule-state.ts +7 -4
- package/src/migrations/db/migrations/1711543888062-write-checkpoint-index.ts +6 -4
- package/src/migrations/db/migrations/1727099539247-custom-write-checkpoint-index.ts +37 -0
- package/src/migrations/migrations.ts +24 -8
- package/src/modules/AbstractModule.ts +37 -0
- package/src/modules/ModuleManager.ts +34 -0
- package/src/modules/modules-index.ts +2 -0
- package/src/replication/AbstractReplicationJob.ts +79 -0
- package/src/replication/AbstractReplicator.ts +228 -0
- package/src/replication/ErrorRateLimiter.ts +0 -44
- package/src/replication/ReplicationEngine.ts +43 -0
- package/src/replication/ReplicationModule.ts +122 -0
- package/src/replication/replication-index.ts +4 -6
- package/src/routes/RouterEngine.ts +120 -0
- package/src/routes/auth.ts +21 -12
- package/src/routes/configure-fastify.ts +26 -27
- package/src/routes/configure-rsocket.ts +13 -8
- package/src/routes/endpoints/admin.ts +72 -76
- package/src/routes/endpoints/checkpointing.ts +51 -11
- package/src/routes/endpoints/socket-route.ts +10 -6
- package/src/routes/endpoints/sync-rules.ts +41 -25
- package/src/routes/endpoints/sync-stream.ts +8 -8
- package/src/routes/router.ts +8 -3
- package/src/routes/routes-index.ts +1 -0
- package/src/runner/teardown.ts +50 -88
- package/src/storage/BucketStorage.ts +103 -41
- package/src/storage/MongoBucketStorage.ts +65 -53
- package/src/storage/ReplicationEventPayload.ts +16 -0
- package/src/storage/SourceEntity.ts +22 -0
- package/src/storage/SourceTable.ts +14 -7
- package/src/storage/StorageEngine.ts +62 -0
- package/src/storage/StorageProvider.ts +27 -0
- package/src/storage/WriteCheckpointAPI.ts +85 -0
- package/src/storage/mongo/MongoBucketBatch.ts +164 -84
- package/src/storage/mongo/MongoCompactor.ts +25 -4
- package/src/storage/mongo/MongoPersistedSyncRulesContent.ts +7 -4
- package/src/storage/mongo/MongoStorageProvider.ts +31 -0
- package/src/storage/mongo/MongoSyncBucketStorage.ts +118 -41
- package/src/storage/mongo/MongoSyncRulesLock.ts +7 -3
- package/src/storage/mongo/MongoWriteCheckpointAPI.ts +151 -0
- package/src/storage/mongo/OperationBatch.ts +28 -12
- package/src/storage/mongo/PersistedBatch.ts +10 -6
- package/src/storage/mongo/config.ts +40 -0
- package/src/storage/mongo/db.ts +4 -1
- package/src/storage/mongo/models.ts +21 -5
- package/src/storage/mongo/util.ts +48 -3
- package/src/storage/storage-index.ts +8 -2
- package/src/sync/sync.ts +7 -4
- package/src/sync/util.ts +0 -1
- package/src/system/ServiceContext.ts +68 -0
- package/src/system/system-index.ts +1 -1
- package/src/util/config/compound-config-collector.ts +31 -31
- package/src/util/config/sync-rules/sync-rules-provider.ts +18 -0
- package/src/util/config/types.ts +7 -5
- package/src/util/config.ts +6 -23
- package/src/util/util-index.ts +3 -6
- package/src/util/utils.ts +48 -41
- package/test/src/__snapshots__/sync.test.ts.snap +14 -14
- package/test/src/auth.test.ts +7 -7
- package/test/src/broadcast_iterable.test.ts +1 -1
- package/test/src/compacting.test.ts +50 -40
- package/test/src/data_storage.test.ts +382 -202
- package/test/src/env.ts +1 -3
- package/test/src/merge_iterable.test.ts +1 -6
- package/test/src/routes/probes.integration.test.ts +34 -30
- package/test/src/setup.ts +1 -1
- package/test/src/stream_utils.ts +42 -0
- package/test/src/sync.test.ts +115 -39
- package/test/src/util.ts +48 -51
- package/test/tsconfig.json +1 -1
- package/tsconfig.tsbuildinfo +1 -1
- package/vitest.config.ts +7 -1
- package/dist/auth/SupabaseKeyCollector.d.ts +0 -22
- package/dist/auth/SupabaseKeyCollector.js +0 -61
- package/dist/auth/SupabaseKeyCollector.js.map +0 -1
- package/dist/replication/PgRelation.d.ts +0 -16
- package/dist/replication/PgRelation.js +0 -26
- package/dist/replication/PgRelation.js.map +0 -1
- package/dist/replication/WalConnection.d.ts +0 -34
- package/dist/replication/WalConnection.js +0 -190
- package/dist/replication/WalConnection.js.map +0 -1
- package/dist/replication/WalStream.d.ts +0 -57
- package/dist/replication/WalStream.js +0 -519
- package/dist/replication/WalStream.js.map +0 -1
- package/dist/replication/WalStreamManager.d.ts +0 -30
- package/dist/replication/WalStreamManager.js +0 -198
- package/dist/replication/WalStreamManager.js.map +0 -1
- package/dist/replication/WalStreamRunner.d.ts +0 -38
- package/dist/replication/WalStreamRunner.js +0 -155
- package/dist/replication/WalStreamRunner.js.map +0 -1
- package/dist/replication/util.d.ts +0 -9
- package/dist/replication/util.js +0 -62
- package/dist/replication/util.js.map +0 -1
- package/dist/system/CorePowerSyncSystem.d.ts +0 -23
- package/dist/system/CorePowerSyncSystem.js +0 -52
- package/dist/system/CorePowerSyncSystem.js.map +0 -1
- package/dist/util/PgManager.d.ts +0 -24
- package/dist/util/PgManager.js +0 -55
- package/dist/util/PgManager.js.map +0 -1
- package/dist/util/migration_lib.d.ts +0 -11
- package/dist/util/migration_lib.js +0 -64
- package/dist/util/migration_lib.js.map +0 -1
- package/dist/util/pgwire_utils.d.ts +0 -24
- package/dist/util/pgwire_utils.js +0 -117
- package/dist/util/pgwire_utils.js.map +0 -1
- package/dist/util/populate_test_data.d.ts +0 -8
- package/dist/util/populate_test_data.js +0 -65
- package/dist/util/populate_test_data.js.map +0 -1
- package/src/auth/SupabaseKeyCollector.ts +0 -67
- package/src/replication/PgRelation.ts +0 -42
- package/src/replication/WalConnection.ts +0 -227
- package/src/replication/WalStream.ts +0 -631
- package/src/replication/WalStreamManager.ts +0 -213
- package/src/replication/WalStreamRunner.ts +0 -180
- package/src/replication/util.ts +0 -76
- package/src/system/CorePowerSyncSystem.ts +0 -64
- package/src/util/PgManager.ts +0 -64
- package/src/util/migration_lib.ts +0 -79
- package/src/util/pgwire_utils.ts +0 -139
- package/src/util/populate_test_data.ts +0 -78
- package/test/src/__snapshots__/pg_test.test.ts.snap +0 -256
- package/test/src/large_batch.test.ts +0 -194
- package/test/src/pg_test.test.ts +0 -450
- package/test/src/schema_changes.test.ts +0 -545
- package/test/src/slow_tests.test.ts +0 -338
- package/test/src/validation.test.ts +0 -63
- package/test/src/wal_stream.test.ts +0 -319
- package/test/src/wal_stream_utils.ts +0 -156
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
import * as sync_rules from '@powersync/service-sync-rules';
|
|
2
|
+
import { BucketStorageBatch, SaveOp } from './BucketStorage.js';
|
|
3
|
+
import { SourceTable } from './SourceTable.js';
|
|
4
|
+
|
|
5
|
+
export type EventData = {
|
|
6
|
+
op: SaveOp;
|
|
7
|
+
before?: sync_rules.SqliteRow;
|
|
8
|
+
after?: sync_rules.SqliteRow;
|
|
9
|
+
};
|
|
10
|
+
|
|
11
|
+
export type ReplicationEventPayload = {
|
|
12
|
+
batch: BucketStorageBatch;
|
|
13
|
+
data: EventData;
|
|
14
|
+
event: sync_rules.SqlEventDescriptor;
|
|
15
|
+
table: SourceTable;
|
|
16
|
+
};
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
export interface ColumnDescriptor {
|
|
2
|
+
name: string;
|
|
3
|
+
/**
|
|
4
|
+
* The type of the column ie VARCHAR, INT, etc
|
|
5
|
+
*/
|
|
6
|
+
type?: string;
|
|
7
|
+
/**
|
|
8
|
+
* Some data sources have a type id that can be used to identify the type of the column
|
|
9
|
+
*/
|
|
10
|
+
typeId?: number;
|
|
11
|
+
}
|
|
12
|
+
|
|
13
|
+
// TODO: This needs to be consolidated with SourceTable into something new.
|
|
14
|
+
export interface SourceEntityDescriptor {
|
|
15
|
+
/**
|
|
16
|
+
* The internal id of the data source structure in the database
|
|
17
|
+
*/
|
|
18
|
+
objectId: number | string;
|
|
19
|
+
schema: string;
|
|
20
|
+
name: string;
|
|
21
|
+
replicationColumns: ColumnDescriptor[];
|
|
22
|
+
}
|
|
@@ -1,10 +1,8 @@
|
|
|
1
|
-
import {
|
|
2
|
-
|
|
3
|
-
import * as replication from '../replication/replication-index.js';
|
|
1
|
+
import { DEFAULT_TAG } from '@powersync/service-sync-rules';
|
|
4
2
|
import * as util from '../util/util-index.js';
|
|
3
|
+
import { ColumnDescriptor } from './SourceEntity.js';
|
|
5
4
|
|
|
6
5
|
export class SourceTable {
|
|
7
|
-
static readonly DEFAULT_SCHEMA = DEFAULT_SCHEMA;
|
|
8
6
|
static readonly DEFAULT_TAG = DEFAULT_TAG;
|
|
9
7
|
|
|
10
8
|
/**
|
|
@@ -25,14 +23,23 @@ export class SourceTable {
|
|
|
25
23
|
*/
|
|
26
24
|
public syncParameters = true;
|
|
27
25
|
|
|
26
|
+
/**
|
|
27
|
+
* True if the table is used in sync rules for events.
|
|
28
|
+
*
|
|
29
|
+
* This value is resolved externally, and cached here.
|
|
30
|
+
*
|
|
31
|
+
* Defaults to true for tests.
|
|
32
|
+
*/
|
|
33
|
+
public syncEvent = true;
|
|
34
|
+
|
|
28
35
|
constructor(
|
|
29
36
|
public readonly id: any,
|
|
30
37
|
public readonly connectionTag: string,
|
|
31
|
-
public readonly
|
|
38
|
+
public readonly objectId: number | string,
|
|
32
39
|
public readonly schema: string,
|
|
33
40
|
public readonly table: string,
|
|
34
41
|
|
|
35
|
-
public readonly replicaIdColumns:
|
|
42
|
+
public readonly replicaIdColumns: ColumnDescriptor[],
|
|
36
43
|
public readonly snapshotComplete: boolean
|
|
37
44
|
) {}
|
|
38
45
|
|
|
@@ -55,6 +62,6 @@ export class SourceTable {
|
|
|
55
62
|
}
|
|
56
63
|
|
|
57
64
|
get syncAny() {
|
|
58
|
-
return this.syncData || this.syncParameters;
|
|
65
|
+
return this.syncData || this.syncParameters || this.syncEvent;
|
|
59
66
|
}
|
|
60
67
|
}
|
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
import { DisposableListener, DisposableObserver, logger } from '@powersync/lib-services-framework';
|
|
2
|
+
import { ResolvedPowerSyncConfig } from '../util/util-index.js';
|
|
3
|
+
import { BucketStorageFactory } from './BucketStorage.js';
|
|
4
|
+
import { ActiveStorage, BucketStorageProvider } from './StorageProvider.js';
|
|
5
|
+
|
|
6
|
+
export type StorageEngineOptions = {
|
|
7
|
+
configuration: ResolvedPowerSyncConfig;
|
|
8
|
+
};
|
|
9
|
+
|
|
10
|
+
export interface StorageEngineListener extends DisposableListener {
|
|
11
|
+
storageActivated: (storage: BucketStorageFactory) => void;
|
|
12
|
+
}
|
|
13
|
+
|
|
14
|
+
export class StorageEngine extends DisposableObserver<StorageEngineListener> {
|
|
15
|
+
// TODO: This will need to revisited when we actually support multiple storage providers.
|
|
16
|
+
private storageProviders: Map<string, BucketStorageProvider> = new Map();
|
|
17
|
+
private currentActiveStorage: ActiveStorage | null = null;
|
|
18
|
+
|
|
19
|
+
constructor(private options: StorageEngineOptions) {
|
|
20
|
+
super();
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
get activeBucketStorage(): BucketStorageFactory {
|
|
24
|
+
return this.activeStorage.storage;
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
get activeStorage(): ActiveStorage {
|
|
28
|
+
if (!this.currentActiveStorage) {
|
|
29
|
+
throw new Error(`No storage provider has been initialized yet.`);
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
return this.currentActiveStorage;
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
/**
|
|
36
|
+
* Register a provider which generates a {@link BucketStorageFactory}
|
|
37
|
+
* given the matching config specified in the loaded {@link ResolvedPowerSyncConfig}
|
|
38
|
+
*/
|
|
39
|
+
registerProvider(provider: BucketStorageProvider) {
|
|
40
|
+
this.storageProviders.set(provider.type, provider);
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
public async start(): Promise<void> {
|
|
44
|
+
logger.info('Starting Storage Engine...');
|
|
45
|
+
const { configuration } = this.options;
|
|
46
|
+
this.currentActiveStorage = await this.storageProviders.get(configuration.storage.type)!.getStorage({
|
|
47
|
+
resolvedConfig: configuration
|
|
48
|
+
});
|
|
49
|
+
this.iterateListeners((cb) => cb.storageActivated?.(this.activeBucketStorage));
|
|
50
|
+
logger.info(`Successfully activated storage: ${configuration.storage.type}.`);
|
|
51
|
+
logger.info('Successfully started Storage Engine.');
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
/**
|
|
55
|
+
* Shutdown the storage engine, safely shutting down any activated storage providers.
|
|
56
|
+
*/
|
|
57
|
+
public async shutDown(): Promise<void> {
|
|
58
|
+
logger.info('Shutting down Storage Engine...');
|
|
59
|
+
await this.currentActiveStorage?.shutDown();
|
|
60
|
+
logger.info('Successfully shut down Storage Engine.');
|
|
61
|
+
}
|
|
62
|
+
}
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
import * as util from '../util/util-index.js';
|
|
2
|
+
import { BucketStorageFactory } from './BucketStorage.js';
|
|
3
|
+
|
|
4
|
+
export interface ActiveStorage {
|
|
5
|
+
storage: BucketStorageFactory;
|
|
6
|
+
shutDown(): Promise<void>;
|
|
7
|
+
|
|
8
|
+
/**
|
|
9
|
+
* Tear down / drop the storage permanently
|
|
10
|
+
*/
|
|
11
|
+
tearDown(): Promise<boolean>;
|
|
12
|
+
}
|
|
13
|
+
|
|
14
|
+
export interface GetStorageOptions {
|
|
15
|
+
// TODO: This should just be the storage config. Update once the slot name prefix coupling has been removed from the storage
|
|
16
|
+
resolvedConfig: util.ResolvedPowerSyncConfig;
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
export interface BucketStorageProvider {
|
|
20
|
+
/**
|
|
21
|
+
* The storage type that this provider provides.
|
|
22
|
+
* The type should match the `type` field in the config.
|
|
23
|
+
*/
|
|
24
|
+
type: string;
|
|
25
|
+
|
|
26
|
+
getStorage(options: GetStorageOptions): Promise<ActiveStorage>;
|
|
27
|
+
}
|
|
@@ -0,0 +1,85 @@
|
|
|
1
|
+
export enum WriteCheckpointMode {
|
|
2
|
+
/**
|
|
3
|
+
* Raw mappings of `user_id` to `write_checkpoint`s should
|
|
4
|
+
* be supplied for each set of sync rules.
|
|
5
|
+
*/
|
|
6
|
+
CUSTOM = 'custom',
|
|
7
|
+
/**
|
|
8
|
+
* Write checkpoints are stored as a mapping of `user_id` plus
|
|
9
|
+
* replication HEAD (lsn in Postgres) to an automatically generated
|
|
10
|
+
* incrementing `write_checkpoint` (stored as`client_id`).
|
|
11
|
+
*/
|
|
12
|
+
MANAGED = 'managed'
|
|
13
|
+
}
|
|
14
|
+
|
|
15
|
+
export interface BaseWriteCheckpointIdentifier {
|
|
16
|
+
/**
|
|
17
|
+
* Identifier for User's account.
|
|
18
|
+
*/
|
|
19
|
+
user_id: string;
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
export interface CustomWriteCheckpointFilters extends BaseWriteCheckpointIdentifier {
|
|
23
|
+
/**
|
|
24
|
+
* Sync rules which were active when this checkpoint was created.
|
|
25
|
+
*/
|
|
26
|
+
sync_rules_id: number;
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
export interface BatchedCustomWriteCheckpointOptions extends BaseWriteCheckpointIdentifier {
|
|
30
|
+
/**
|
|
31
|
+
* A supplied incrementing Write Checkpoint number
|
|
32
|
+
*/
|
|
33
|
+
checkpoint: bigint;
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
export interface CustomWriteCheckpointOptions extends BatchedCustomWriteCheckpointOptions {
|
|
37
|
+
/**
|
|
38
|
+
* Sync rules which were active when this checkpoint was created.
|
|
39
|
+
*/
|
|
40
|
+
sync_rules_id: number;
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
/**
|
|
44
|
+
* Managed Write Checkpoints are a mapping of User ID to replication HEAD
|
|
45
|
+
*/
|
|
46
|
+
export interface ManagedWriteCheckpointFilters extends BaseWriteCheckpointIdentifier {
|
|
47
|
+
/**
|
|
48
|
+
* Replication HEAD(s) at the creation of the checkpoint.
|
|
49
|
+
*/
|
|
50
|
+
heads: Record<string, string>;
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
export type ManagedWriteCheckpointOptions = ManagedWriteCheckpointFilters;
|
|
54
|
+
|
|
55
|
+
export type SyncStorageLastWriteCheckpointFilters = BaseWriteCheckpointIdentifier | ManagedWriteCheckpointFilters;
|
|
56
|
+
export type LastWriteCheckpointFilters = CustomWriteCheckpointFilters | ManagedWriteCheckpointFilters;
|
|
57
|
+
|
|
58
|
+
export interface BaseWriteCheckpointAPI {
|
|
59
|
+
readonly writeCheckpointMode: WriteCheckpointMode;
|
|
60
|
+
setWriteCheckpointMode(mode: WriteCheckpointMode): void;
|
|
61
|
+
createManagedWriteCheckpoint(checkpoint: ManagedWriteCheckpointOptions): Promise<bigint>;
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
/**
|
|
65
|
+
* Write Checkpoint API to be used in conjunction with a {@link SyncRulesBucketStorage}.
|
|
66
|
+
* This storage corresponds with a set of sync rules. These APIs don't require specifying a
|
|
67
|
+
* sync rules id.
|
|
68
|
+
*/
|
|
69
|
+
export interface SyncStorageWriteCheckpointAPI extends BaseWriteCheckpointAPI {
|
|
70
|
+
batchCreateCustomWriteCheckpoints(checkpoints: BatchedCustomWriteCheckpointOptions[]): Promise<void>;
|
|
71
|
+
createCustomWriteCheckpoint(checkpoint: BatchedCustomWriteCheckpointOptions): Promise<bigint>;
|
|
72
|
+
lastWriteCheckpoint(filters: SyncStorageLastWriteCheckpointFilters): Promise<bigint | null>;
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
/**
|
|
76
|
+
* Write Checkpoint API which is interfaced directly with the storage layer. This requires
|
|
77
|
+
* sync rules identifiers for custom write checkpoints.
|
|
78
|
+
*/
|
|
79
|
+
export interface WriteCheckpointAPI extends BaseWriteCheckpointAPI {
|
|
80
|
+
batchCreateCustomWriteCheckpoints(checkpoints: CustomWriteCheckpointOptions[]): Promise<void>;
|
|
81
|
+
createCustomWriteCheckpoint(checkpoint: CustomWriteCheckpointOptions): Promise<bigint>;
|
|
82
|
+
lastWriteCheckpoint(filters: LastWriteCheckpointFilters): Promise<bigint | null>;
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
export const DEFAULT_WRITE_CHECKPOINT_MODE = WriteCheckpointMode.MANAGED;
|
|
@@ -1,18 +1,25 @@
|
|
|
1
|
-
import { SqliteRow, SqlSyncRules } from '@powersync/service-sync-rules';
|
|
1
|
+
import { SqlEventDescriptor, SqliteRow, SqlSyncRules } from '@powersync/service-sync-rules';
|
|
2
2
|
import * as bson from 'bson';
|
|
3
3
|
import * as mongo from 'mongodb';
|
|
4
4
|
|
|
5
|
+
import { container, DisposableObserver, errors, logger } from '@powersync/lib-services-framework';
|
|
5
6
|
import * as util from '../../util/util-index.js';
|
|
6
|
-
import
|
|
7
|
-
|
|
8
|
-
|
|
7
|
+
import {
|
|
8
|
+
BucketBatchStorageListener,
|
|
9
|
+
BucketStorageBatch,
|
|
10
|
+
FlushedResult,
|
|
11
|
+
mergeToast,
|
|
12
|
+
SaveOptions
|
|
13
|
+
} from '../BucketStorage.js';
|
|
9
14
|
import { SourceTable } from '../SourceTable.js';
|
|
15
|
+
import { BatchedCustomWriteCheckpointOptions, CustomWriteCheckpointOptions } from '../WriteCheckpointAPI.js';
|
|
10
16
|
import { PowerSyncMongo } from './db.js';
|
|
11
|
-
import { CurrentBucket, CurrentDataDocument, SourceKey } from './models.js';
|
|
17
|
+
import { CurrentBucket, CurrentDataDocument, SourceKey, SyncRuleDocument } from './models.js';
|
|
12
18
|
import { MongoIdSequence } from './MongoIdSequence.js';
|
|
19
|
+
import { batchCreateCustomWriteCheckpoints } from './MongoWriteCheckpointAPI.js';
|
|
13
20
|
import { cacheKey, OperationBatch, RecordOperation } from './OperationBatch.js';
|
|
14
21
|
import { PersistedBatch } from './PersistedBatch.js';
|
|
15
|
-
import { BSON_DESERIALIZE_OPTIONS, idPrefixFilter, serializeLookup } from './util.js';
|
|
22
|
+
import { BSON_DESERIALIZE_OPTIONS, idPrefixFilter, replicaIdEquals, serializeLookup } from './util.js';
|
|
16
23
|
|
|
17
24
|
/**
|
|
18
25
|
* 15MB
|
|
@@ -25,7 +32,18 @@ const MAX_ROW_SIZE = 15 * 1024 * 1024;
|
|
|
25
32
|
//
|
|
26
33
|
// In the future, we can investigate allowing multiple replication streams operating independently.
|
|
27
34
|
const replicationMutex = new util.Mutex();
|
|
28
|
-
|
|
35
|
+
|
|
36
|
+
export interface MongoBucketBatchOptions {
|
|
37
|
+
db: PowerSyncMongo;
|
|
38
|
+
syncRules: SqlSyncRules;
|
|
39
|
+
groupId: number;
|
|
40
|
+
slotName: string;
|
|
41
|
+
lastCheckpointLsn: string | null;
|
|
42
|
+
noCheckpointBeforeLsn: string;
|
|
43
|
+
storeCurrentData: boolean;
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
export class MongoBucketBatch extends DisposableObserver<BucketBatchStorageListener> implements BucketStorageBatch {
|
|
29
47
|
private readonly client: mongo.MongoClient;
|
|
30
48
|
public readonly db: PowerSyncMongo;
|
|
31
49
|
public readonly session: mongo.ClientSession;
|
|
@@ -34,8 +52,10 @@ export class MongoBucketBatch implements BucketStorageBatch {
|
|
|
34
52
|
private readonly group_id: number;
|
|
35
53
|
|
|
36
54
|
private readonly slot_name: string;
|
|
55
|
+
private readonly storeCurrentData: boolean;
|
|
37
56
|
|
|
38
57
|
private batch: OperationBatch | null = null;
|
|
58
|
+
private write_checkpoint_batch: CustomWriteCheckpointOptions[] = [];
|
|
39
59
|
|
|
40
60
|
/**
|
|
41
61
|
* Last LSN received associated with a checkpoint.
|
|
@@ -55,22 +75,29 @@ export class MongoBucketBatch implements BucketStorageBatch {
|
|
|
55
75
|
*/
|
|
56
76
|
public last_flushed_op: bigint | null = null;
|
|
57
77
|
|
|
58
|
-
constructor(
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
last_checkpoint_lsn
|
|
64
|
-
no_checkpoint_before_lsn
|
|
65
|
-
) {
|
|
66
|
-
this.db = db;
|
|
67
|
-
this.client = db.client;
|
|
68
|
-
this.sync_rules = sync_rules;
|
|
69
|
-
this.group_id = group_id;
|
|
70
|
-
this.slot_name = slot_name;
|
|
78
|
+
constructor(options: MongoBucketBatchOptions) {
|
|
79
|
+
super();
|
|
80
|
+
this.client = options.db.client;
|
|
81
|
+
this.db = options.db;
|
|
82
|
+
this.group_id = options.groupId;
|
|
83
|
+
this.last_checkpoint_lsn = options.lastCheckpointLsn;
|
|
84
|
+
this.no_checkpoint_before_lsn = options.noCheckpointBeforeLsn;
|
|
71
85
|
this.session = this.client.startSession();
|
|
72
|
-
this.
|
|
73
|
-
this.
|
|
86
|
+
this.slot_name = options.slotName;
|
|
87
|
+
this.sync_rules = options.syncRules;
|
|
88
|
+
this.storeCurrentData = options.storeCurrentData;
|
|
89
|
+
this.batch = new OperationBatch();
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
addCustomWriteCheckpoint(checkpoint: BatchedCustomWriteCheckpointOptions): void {
|
|
93
|
+
this.write_checkpoint_batch.push({
|
|
94
|
+
...checkpoint,
|
|
95
|
+
sync_rules_id: this.group_id
|
|
96
|
+
});
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
get lastCheckpointLsn() {
|
|
100
|
+
return this.last_checkpoint_lsn;
|
|
74
101
|
}
|
|
75
102
|
|
|
76
103
|
async flush(): Promise<FlushedResult | null> {
|
|
@@ -83,6 +110,8 @@ export class MongoBucketBatch implements BucketStorageBatch {
|
|
|
83
110
|
result = r;
|
|
84
111
|
}
|
|
85
112
|
}
|
|
113
|
+
await batchCreateCustomWriteCheckpoints(this.db, this.write_checkpoint_batch);
|
|
114
|
+
this.write_checkpoint_batch = [];
|
|
86
115
|
return result;
|
|
87
116
|
}
|
|
88
117
|
|
|
@@ -118,38 +147,44 @@ export class MongoBucketBatch implements BucketStorageBatch {
|
|
|
118
147
|
batch: OperationBatch,
|
|
119
148
|
op_seq: MongoIdSequence
|
|
120
149
|
): Promise<OperationBatch | null> {
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
150
|
+
let sizes: Map<string, number> | undefined = undefined;
|
|
151
|
+
if (this.storeCurrentData) {
|
|
152
|
+
// We skip this step if we don't store current_data, since the sizes will
|
|
153
|
+
// always be small in that case.
|
|
154
|
+
|
|
155
|
+
// Find sizes of current_data documents, to assist in intelligent batching without
|
|
156
|
+
// exceeding memory limits.
|
|
157
|
+
//
|
|
158
|
+
// A previous attempt tried to do batching by the results of the current_data query
|
|
159
|
+
// (automatically limited to 48MB(?) per batch by MongoDB). The issue is that it changes
|
|
160
|
+
// the order of processing, which then becomes really tricky to manage.
|
|
161
|
+
// This now takes 2+ queries, but doesn't have any issues with order of operations.
|
|
162
|
+
const sizeLookups: SourceKey[] = batch.batch.map((r) => {
|
|
163
|
+
return { g: this.group_id, t: r.record.sourceTable.id, k: r.beforeId };
|
|
164
|
+
});
|
|
131
165
|
|
|
132
|
-
|
|
166
|
+
sizes = new Map<string, number>();
|
|
133
167
|
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
168
|
+
const sizeCursor: mongo.AggregationCursor<{ _id: SourceKey; size: number }> = this.db.current_data.aggregate(
|
|
169
|
+
[
|
|
170
|
+
{
|
|
171
|
+
$match: {
|
|
172
|
+
_id: { $in: sizeLookups }
|
|
173
|
+
}
|
|
174
|
+
},
|
|
175
|
+
{
|
|
176
|
+
$project: {
|
|
177
|
+
_id: 1,
|
|
178
|
+
size: { $bsonSize: '$$ROOT' }
|
|
179
|
+
}
|
|
145
180
|
}
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
181
|
+
],
|
|
182
|
+
{ session }
|
|
183
|
+
);
|
|
184
|
+
for await (let doc of sizeCursor.stream()) {
|
|
185
|
+
const key = cacheKey(doc._id.t, doc._id.k);
|
|
186
|
+
sizes.set(key, doc.size);
|
|
187
|
+
}
|
|
153
188
|
}
|
|
154
189
|
|
|
155
190
|
// If set, we need to start a new transaction with this batch.
|
|
@@ -157,6 +192,7 @@ export class MongoBucketBatch implements BucketStorageBatch {
|
|
|
157
192
|
let transactionSize = 0;
|
|
158
193
|
|
|
159
194
|
// Now batch according to the sizes
|
|
195
|
+
// This is a single batch if storeCurrentData == false
|
|
160
196
|
for await (let b of batch.batched(sizes)) {
|
|
161
197
|
if (resumeBatch) {
|
|
162
198
|
for (let op of b) {
|
|
@@ -194,7 +230,7 @@ export class MongoBucketBatch implements BucketStorageBatch {
|
|
|
194
230
|
if (nextData != null) {
|
|
195
231
|
// Update our current_data and size cache
|
|
196
232
|
current_data_lookup.set(op.internalAfterKey!, nextData);
|
|
197
|
-
sizes
|
|
233
|
+
sizes?.set(op.internalAfterKey!, nextData.data.length());
|
|
198
234
|
}
|
|
199
235
|
|
|
200
236
|
if (persistedBatch!.shouldFlushTransaction()) {
|
|
@@ -244,14 +280,18 @@ export class MongoBucketBatch implements BucketStorageBatch {
|
|
|
244
280
|
existing_buckets = [];
|
|
245
281
|
existing_lookups = [];
|
|
246
282
|
// Log to help with debugging if there was a consistency issue
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
283
|
+
if (this.storeCurrentData) {
|
|
284
|
+
logger.warn(
|
|
285
|
+
`Cannot find previous record for update on ${record.sourceTable.qualifiedName}: ${beforeId} / ${record.before?.id}`
|
|
286
|
+
);
|
|
287
|
+
}
|
|
250
288
|
} else {
|
|
251
|
-
const data = bson.deserialize((result.data as mongo.Binary).buffer, BSON_DESERIALIZE_OPTIONS) as SqliteRow;
|
|
252
289
|
existing_buckets = result.buckets;
|
|
253
290
|
existing_lookups = result.lookups;
|
|
254
|
-
|
|
291
|
+
if (this.storeCurrentData) {
|
|
292
|
+
const data = bson.deserialize((result.data as mongo.Binary).buffer, BSON_DESERIALIZE_OPTIONS) as SqliteRow;
|
|
293
|
+
after = mergeToast(after!, data);
|
|
294
|
+
}
|
|
255
295
|
}
|
|
256
296
|
} else if (record.tag == 'delete') {
|
|
257
297
|
const result = current_data;
|
|
@@ -260,9 +300,11 @@ export class MongoBucketBatch implements BucketStorageBatch {
|
|
|
260
300
|
existing_buckets = [];
|
|
261
301
|
existing_lookups = [];
|
|
262
302
|
// Log to help with debugging if there was a consistency issue
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
303
|
+
if (this.storeCurrentData) {
|
|
304
|
+
logger.warn(
|
|
305
|
+
`Cannot find previous record for delete on ${record.sourceTable.qualifiedName}: ${beforeId} / ${record.before?.id}`
|
|
306
|
+
);
|
|
307
|
+
}
|
|
266
308
|
} else {
|
|
267
309
|
existing_buckets = result.buckets;
|
|
268
310
|
existing_lookups = result.lookups;
|
|
@@ -270,7 +312,9 @@ export class MongoBucketBatch implements BucketStorageBatch {
|
|
|
270
312
|
}
|
|
271
313
|
|
|
272
314
|
let afterData: bson.Binary | undefined;
|
|
273
|
-
if (afterId) {
|
|
315
|
+
if (afterId != null && !this.storeCurrentData) {
|
|
316
|
+
afterData = new bson.Binary(bson.serialize({}));
|
|
317
|
+
} else if (afterId != null) {
|
|
274
318
|
try {
|
|
275
319
|
// This will fail immediately if the record is > 16MB.
|
|
276
320
|
afterData = new bson.Binary(bson.serialize(after!));
|
|
@@ -301,7 +345,7 @@ export class MongoBucketBatch implements BucketStorageBatch {
|
|
|
301
345
|
}
|
|
302
346
|
|
|
303
347
|
// 2. Save bucket data
|
|
304
|
-
if (beforeId != null && (afterId == null || !beforeId
|
|
348
|
+
if (beforeId != null && (afterId == null || !replicaIdEquals(beforeId, afterId))) {
|
|
305
349
|
// Source ID updated
|
|
306
350
|
if (sourceTable.syncData) {
|
|
307
351
|
// Delete old record
|
|
@@ -431,7 +475,7 @@ export class MongoBucketBatch implements BucketStorageBatch {
|
|
|
431
475
|
};
|
|
432
476
|
}
|
|
433
477
|
|
|
434
|
-
if (afterId == null || !beforeId
|
|
478
|
+
if (afterId == null || !replicaIdEquals(beforeId, afterId)) {
|
|
435
479
|
// Either a delete (afterId == null), or replaced the old replication id
|
|
436
480
|
batch.deleteCurrentData(before_key);
|
|
437
481
|
}
|
|
@@ -528,14 +572,15 @@ export class MongoBucketBatch implements BucketStorageBatch {
|
|
|
528
572
|
});
|
|
529
573
|
}
|
|
530
574
|
|
|
531
|
-
async
|
|
575
|
+
async [Symbol.asyncDispose]() {
|
|
532
576
|
await this.session.endSession();
|
|
577
|
+
super[Symbol.dispose]();
|
|
533
578
|
}
|
|
534
579
|
|
|
535
580
|
async commit(lsn: string): Promise<boolean> {
|
|
536
581
|
await this.flush();
|
|
537
582
|
|
|
538
|
-
if (this.last_checkpoint_lsn != null && lsn
|
|
583
|
+
if (this.last_checkpoint_lsn != null && lsn < this.last_checkpoint_lsn) {
|
|
539
584
|
// When re-applying transactions, don't create a new checkpoint until
|
|
540
585
|
// we are past the last transaction.
|
|
541
586
|
logger.info(`Re-applied transaction ${lsn} - skipping checkpoint`);
|
|
@@ -546,26 +591,29 @@ export class MongoBucketBatch implements BucketStorageBatch {
|
|
|
546
591
|
return false;
|
|
547
592
|
}
|
|
548
593
|
|
|
594
|
+
const now = new Date();
|
|
595
|
+
const update: Partial<SyncRuleDocument> = {
|
|
596
|
+
last_checkpoint_lsn: lsn,
|
|
597
|
+
last_checkpoint_ts: now,
|
|
598
|
+
last_keepalive_ts: now,
|
|
599
|
+
snapshot_done: true,
|
|
600
|
+
last_fatal_error: null
|
|
601
|
+
};
|
|
602
|
+
|
|
549
603
|
if (this.persisted_op != null) {
|
|
550
|
-
|
|
551
|
-
await this.db.sync_rules.updateOne(
|
|
552
|
-
{
|
|
553
|
-
_id: this.group_id
|
|
554
|
-
},
|
|
555
|
-
{
|
|
556
|
-
$set: {
|
|
557
|
-
last_checkpoint: this.persisted_op,
|
|
558
|
-
last_checkpoint_lsn: lsn,
|
|
559
|
-
last_checkpoint_ts: now,
|
|
560
|
-
last_keepalive_ts: now,
|
|
561
|
-
snapshot_done: true,
|
|
562
|
-
last_fatal_error: null
|
|
563
|
-
}
|
|
564
|
-
},
|
|
565
|
-
{ session: this.session }
|
|
566
|
-
);
|
|
567
|
-
this.persisted_op = null;
|
|
604
|
+
update.last_checkpoint = this.persisted_op;
|
|
568
605
|
}
|
|
606
|
+
|
|
607
|
+
await this.db.sync_rules.updateOne(
|
|
608
|
+
{
|
|
609
|
+
_id: this.group_id
|
|
610
|
+
},
|
|
611
|
+
{
|
|
612
|
+
$set: update
|
|
613
|
+
},
|
|
614
|
+
{ session: this.session }
|
|
615
|
+
);
|
|
616
|
+
this.persisted_op = null;
|
|
569
617
|
this.last_checkpoint_lsn = lsn;
|
|
570
618
|
return true;
|
|
571
619
|
}
|
|
@@ -606,6 +654,29 @@ export class MongoBucketBatch implements BucketStorageBatch {
|
|
|
606
654
|
}
|
|
607
655
|
|
|
608
656
|
async save(record: SaveOptions): Promise<FlushedResult | null> {
|
|
657
|
+
const { after, before, sourceTable, tag } = record;
|
|
658
|
+
for (const event of this.getTableEvents(sourceTable)) {
|
|
659
|
+
this.iterateListeners((cb) =>
|
|
660
|
+
cb.replicationEvent?.({
|
|
661
|
+
batch: this,
|
|
662
|
+
table: sourceTable,
|
|
663
|
+
data: {
|
|
664
|
+
op: tag,
|
|
665
|
+
after: after && util.isCompleteRow(after) ? after : undefined,
|
|
666
|
+
before: before && util.isCompleteRow(before) ? before : undefined
|
|
667
|
+
},
|
|
668
|
+
event
|
|
669
|
+
})
|
|
670
|
+
);
|
|
671
|
+
}
|
|
672
|
+
|
|
673
|
+
/**
|
|
674
|
+
* Return if the table is just an event table
|
|
675
|
+
*/
|
|
676
|
+
if (!sourceTable.syncData && !sourceTable.syncParameters) {
|
|
677
|
+
return null;
|
|
678
|
+
}
|
|
679
|
+
|
|
609
680
|
logger.debug(`Saving ${record.tag}:${record.before?.id}/${record.after?.id}`);
|
|
610
681
|
|
|
611
682
|
this.batch ??= new OperationBatch();
|
|
@@ -743,7 +814,7 @@ export class MongoBucketBatch implements BucketStorageBatch {
|
|
|
743
814
|
const copy = new SourceTable(
|
|
744
815
|
table.id,
|
|
745
816
|
table.connectionTag,
|
|
746
|
-
table.
|
|
817
|
+
table.objectId,
|
|
747
818
|
table.schema,
|
|
748
819
|
table.table,
|
|
749
820
|
table.replicaIdColumns,
|
|
@@ -754,6 +825,15 @@ export class MongoBucketBatch implements BucketStorageBatch {
|
|
|
754
825
|
return copy;
|
|
755
826
|
});
|
|
756
827
|
}
|
|
828
|
+
|
|
829
|
+
/**
|
|
830
|
+
* Gets relevant {@link SqlEventDescriptor}s for the given {@link SourceTable}
|
|
831
|
+
*/
|
|
832
|
+
protected getTableEvents(table: SourceTable): SqlEventDescriptor[] {
|
|
833
|
+
return this.sync_rules.event_descriptors.filter((evt) =>
|
|
834
|
+
[...evt.getSourceTables()].some((sourceTable) => sourceTable.matches(table))
|
|
835
|
+
);
|
|
836
|
+
}
|
|
757
837
|
}
|
|
758
838
|
|
|
759
839
|
export function currentBucketKey(b: CurrentBucket) {
|