@powersync/service-core 0.0.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.probes/.gitkeep +0 -0
- package/CHANGELOG.md +13 -0
- package/LICENSE +67 -0
- package/README.md +3 -0
- package/dist/api/api-index.d.ts +2 -0
- package/dist/api/api-index.js +3 -0
- package/dist/api/api-index.js.map +1 -0
- package/dist/api/diagnostics.d.ts +21 -0
- package/dist/api/diagnostics.js +183 -0
- package/dist/api/diagnostics.js.map +1 -0
- package/dist/api/schema.d.ts +5 -0
- package/dist/api/schema.js +88 -0
- package/dist/api/schema.js.map +1 -0
- package/dist/auth/CachedKeyCollector.d.ts +46 -0
- package/dist/auth/CachedKeyCollector.js +116 -0
- package/dist/auth/CachedKeyCollector.js.map +1 -0
- package/dist/auth/CompoundKeyCollector.d.ts +8 -0
- package/dist/auth/CompoundKeyCollector.js +23 -0
- package/dist/auth/CompoundKeyCollector.js.map +1 -0
- package/dist/auth/JwtPayload.d.ts +10 -0
- package/dist/auth/JwtPayload.js +2 -0
- package/dist/auth/JwtPayload.js.map +1 -0
- package/dist/auth/KeyCollector.d.ts +24 -0
- package/dist/auth/KeyCollector.js +2 -0
- package/dist/auth/KeyCollector.js.map +1 -0
- package/dist/auth/KeySpec.d.ts +26 -0
- package/dist/auth/KeySpec.js +49 -0
- package/dist/auth/KeySpec.js.map +1 -0
- package/dist/auth/KeyStore.d.ts +39 -0
- package/dist/auth/KeyStore.js +131 -0
- package/dist/auth/KeyStore.js.map +1 -0
- package/dist/auth/LeakyBucket.d.ts +39 -0
- package/dist/auth/LeakyBucket.js +57 -0
- package/dist/auth/LeakyBucket.js.map +1 -0
- package/dist/auth/RemoteJWKSCollector.d.ts +24 -0
- package/dist/auth/RemoteJWKSCollector.js +106 -0
- package/dist/auth/RemoteJWKSCollector.js.map +1 -0
- package/dist/auth/StaticKeyCollector.d.ts +14 -0
- package/dist/auth/StaticKeyCollector.js +19 -0
- package/dist/auth/StaticKeyCollector.js.map +1 -0
- package/dist/auth/SupabaseKeyCollector.d.ts +22 -0
- package/dist/auth/SupabaseKeyCollector.js +61 -0
- package/dist/auth/SupabaseKeyCollector.js.map +1 -0
- package/dist/auth/auth-index.d.ts +10 -0
- package/dist/auth/auth-index.js +11 -0
- package/dist/auth/auth-index.js.map +1 -0
- package/dist/db/db-index.d.ts +1 -0
- package/dist/db/db-index.js +2 -0
- package/dist/db/db-index.js.map +1 -0
- package/dist/db/mongo.d.ts +29 -0
- package/dist/db/mongo.js +65 -0
- package/dist/db/mongo.js.map +1 -0
- package/dist/entry/cli-entry.d.ts +15 -0
- package/dist/entry/cli-entry.js +36 -0
- package/dist/entry/cli-entry.js.map +1 -0
- package/dist/entry/commands/config-command.d.ts +10 -0
- package/dist/entry/commands/config-command.js +21 -0
- package/dist/entry/commands/config-command.js.map +1 -0
- package/dist/entry/commands/migrate-action.d.ts +2 -0
- package/dist/entry/commands/migrate-action.js +18 -0
- package/dist/entry/commands/migrate-action.js.map +1 -0
- package/dist/entry/commands/start-action.d.ts +3 -0
- package/dist/entry/commands/start-action.js +15 -0
- package/dist/entry/commands/start-action.js.map +1 -0
- package/dist/entry/commands/teardown-action.d.ts +2 -0
- package/dist/entry/commands/teardown-action.js +17 -0
- package/dist/entry/commands/teardown-action.js.map +1 -0
- package/dist/entry/entry-index.d.ts +5 -0
- package/dist/entry/entry-index.js +6 -0
- package/dist/entry/entry-index.js.map +1 -0
- package/dist/index.d.ts +24 -0
- package/dist/index.js +26 -0
- package/dist/index.js.map +1 -0
- package/dist/metrics/metrics.d.ts +16 -0
- package/dist/metrics/metrics.js +139 -0
- package/dist/metrics/metrics.js.map +1 -0
- package/dist/migrations/db/migrations/1684951997326-init.d.ts +3 -0
- package/dist/migrations/db/migrations/1684951997326-init.js +31 -0
- package/dist/migrations/db/migrations/1684951997326-init.js.map +1 -0
- package/dist/migrations/db/migrations/1688556755264-initial-sync-rules.d.ts +2 -0
- package/dist/migrations/db/migrations/1688556755264-initial-sync-rules.js +5 -0
- package/dist/migrations/db/migrations/1688556755264-initial-sync-rules.js.map +1 -0
- package/dist/migrations/db/migrations/1702295701188-sync-rule-state.d.ts +3 -0
- package/dist/migrations/db/migrations/1702295701188-sync-rule-state.js +54 -0
- package/dist/migrations/db/migrations/1702295701188-sync-rule-state.js.map +1 -0
- package/dist/migrations/db/migrations/1711543888062-write-checkpoint-index.d.ts +3 -0
- package/dist/migrations/db/migrations/1711543888062-write-checkpoint-index.js +27 -0
- package/dist/migrations/db/migrations/1711543888062-write-checkpoint-index.js.map +1 -0
- package/dist/migrations/db/store.d.ts +3 -0
- package/dist/migrations/db/store.js +10 -0
- package/dist/migrations/db/store.js.map +1 -0
- package/dist/migrations/migrations.d.ts +10 -0
- package/dist/migrations/migrations.js +94 -0
- package/dist/migrations/migrations.js.map +1 -0
- package/dist/replication/ErrorRateLimiter.d.ts +17 -0
- package/dist/replication/ErrorRateLimiter.js +42 -0
- package/dist/replication/ErrorRateLimiter.js.map +1 -0
- package/dist/replication/PgRelation.d.ts +16 -0
- package/dist/replication/PgRelation.js +26 -0
- package/dist/replication/PgRelation.js.map +1 -0
- package/dist/replication/WalConnection.d.ts +34 -0
- package/dist/replication/WalConnection.js +190 -0
- package/dist/replication/WalConnection.js.map +1 -0
- package/dist/replication/WalStream.d.ts +58 -0
- package/dist/replication/WalStream.js +517 -0
- package/dist/replication/WalStream.js.map +1 -0
- package/dist/replication/WalStreamManager.d.ts +30 -0
- package/dist/replication/WalStreamManager.js +199 -0
- package/dist/replication/WalStreamManager.js.map +1 -0
- package/dist/replication/WalStreamRunner.d.ts +38 -0
- package/dist/replication/WalStreamRunner.js +155 -0
- package/dist/replication/WalStreamRunner.js.map +1 -0
- package/dist/replication/replication-index.d.ts +7 -0
- package/dist/replication/replication-index.js +8 -0
- package/dist/replication/replication-index.js.map +1 -0
- package/dist/replication/util.d.ts +9 -0
- package/dist/replication/util.js +62 -0
- package/dist/replication/util.js.map +1 -0
- package/dist/routes/admin.d.ts +7 -0
- package/dist/routes/admin.js +192 -0
- package/dist/routes/admin.js.map +1 -0
- package/dist/routes/auth.d.ts +58 -0
- package/dist/routes/auth.js +182 -0
- package/dist/routes/auth.js.map +1 -0
- package/dist/routes/checkpointing.d.ts +3 -0
- package/dist/routes/checkpointing.js +30 -0
- package/dist/routes/checkpointing.js.map +1 -0
- package/dist/routes/dev.d.ts +6 -0
- package/dist/routes/dev.js +163 -0
- package/dist/routes/dev.js.map +1 -0
- package/dist/routes/route-generators.d.ts +15 -0
- package/dist/routes/route-generators.js +32 -0
- package/dist/routes/route-generators.js.map +1 -0
- package/dist/routes/router-socket.d.ts +10 -0
- package/dist/routes/router-socket.js +5 -0
- package/dist/routes/router-socket.js.map +1 -0
- package/dist/routes/router.d.ts +13 -0
- package/dist/routes/router.js +2 -0
- package/dist/routes/router.js.map +1 -0
- package/dist/routes/routes-index.d.ts +4 -0
- package/dist/routes/routes-index.js +5 -0
- package/dist/routes/routes-index.js.map +1 -0
- package/dist/routes/socket-route.d.ts +2 -0
- package/dist/routes/socket-route.js +119 -0
- package/dist/routes/socket-route.js.map +1 -0
- package/dist/routes/sync-rules.d.ts +6 -0
- package/dist/routes/sync-rules.js +182 -0
- package/dist/routes/sync-rules.js.map +1 -0
- package/dist/routes/sync-stream.d.ts +5 -0
- package/dist/routes/sync-stream.js +74 -0
- package/dist/routes/sync-stream.js.map +1 -0
- package/dist/runner/teardown.d.ts +2 -0
- package/dist/runner/teardown.js +79 -0
- package/dist/runner/teardown.js.map +1 -0
- package/dist/storage/BucketStorage.d.ts +298 -0
- package/dist/storage/BucketStorage.js +25 -0
- package/dist/storage/BucketStorage.js.map +1 -0
- package/dist/storage/MongoBucketStorage.d.ts +51 -0
- package/dist/storage/MongoBucketStorage.js +388 -0
- package/dist/storage/MongoBucketStorage.js.map +1 -0
- package/dist/storage/SourceTable.d.ts +39 -0
- package/dist/storage/SourceTable.js +50 -0
- package/dist/storage/SourceTable.js.map +1 -0
- package/dist/storage/mongo/MongoBucketBatch.d.ts +48 -0
- package/dist/storage/mongo/MongoBucketBatch.js +584 -0
- package/dist/storage/mongo/MongoBucketBatch.js.map +1 -0
- package/dist/storage/mongo/MongoIdSequence.d.ts +12 -0
- package/dist/storage/mongo/MongoIdSequence.js +21 -0
- package/dist/storage/mongo/MongoIdSequence.js.map +1 -0
- package/dist/storage/mongo/MongoPersistedSyncRules.d.ts +9 -0
- package/dist/storage/mongo/MongoPersistedSyncRules.js +9 -0
- package/dist/storage/mongo/MongoPersistedSyncRules.js.map +1 -0
- package/dist/storage/mongo/MongoPersistedSyncRulesContent.d.ts +20 -0
- package/dist/storage/mongo/MongoPersistedSyncRulesContent.js +26 -0
- package/dist/storage/mongo/MongoPersistedSyncRulesContent.js.map +1 -0
- package/dist/storage/mongo/MongoSyncBucketStorage.d.ts +27 -0
- package/dist/storage/mongo/MongoSyncBucketStorage.js +379 -0
- package/dist/storage/mongo/MongoSyncBucketStorage.js.map +1 -0
- package/dist/storage/mongo/MongoSyncRulesLock.d.ts +16 -0
- package/dist/storage/mongo/MongoSyncRulesLock.js +65 -0
- package/dist/storage/mongo/MongoSyncRulesLock.js.map +1 -0
- package/dist/storage/mongo/OperationBatch.d.ts +26 -0
- package/dist/storage/mongo/OperationBatch.js +101 -0
- package/dist/storage/mongo/OperationBatch.js.map +1 -0
- package/dist/storage/mongo/PersistedBatch.d.ts +42 -0
- package/dist/storage/mongo/PersistedBatch.js +200 -0
- package/dist/storage/mongo/PersistedBatch.js.map +1 -0
- package/dist/storage/mongo/db.d.ts +23 -0
- package/dist/storage/mongo/db.js +34 -0
- package/dist/storage/mongo/db.js.map +1 -0
- package/dist/storage/mongo/models.d.ts +137 -0
- package/dist/storage/mongo/models.js +27 -0
- package/dist/storage/mongo/models.js.map +1 -0
- package/dist/storage/mongo/util.d.ts +26 -0
- package/dist/storage/mongo/util.js +81 -0
- package/dist/storage/mongo/util.js.map +1 -0
- package/dist/storage/storage-index.d.ts +14 -0
- package/dist/storage/storage-index.js +15 -0
- package/dist/storage/storage-index.js.map +1 -0
- package/dist/sync/BroadcastIterable.d.ts +38 -0
- package/dist/sync/BroadcastIterable.js +153 -0
- package/dist/sync/BroadcastIterable.js.map +1 -0
- package/dist/sync/LastValueSink.d.ts +25 -0
- package/dist/sync/LastValueSink.js +84 -0
- package/dist/sync/LastValueSink.js.map +1 -0
- package/dist/sync/merge.d.ts +39 -0
- package/dist/sync/merge.js +175 -0
- package/dist/sync/merge.js.map +1 -0
- package/dist/sync/safeRace.d.ts +1 -0
- package/dist/sync/safeRace.js +91 -0
- package/dist/sync/safeRace.js.map +1 -0
- package/dist/sync/sync-index.d.ts +6 -0
- package/dist/sync/sync-index.js +7 -0
- package/dist/sync/sync-index.js.map +1 -0
- package/dist/sync/sync.d.ts +18 -0
- package/dist/sync/sync.js +248 -0
- package/dist/sync/sync.js.map +1 -0
- package/dist/sync/util.d.ts +26 -0
- package/dist/sync/util.js +73 -0
- package/dist/sync/util.js.map +1 -0
- package/dist/system/CorePowerSyncSystem.d.ts +18 -0
- package/dist/system/CorePowerSyncSystem.js +28 -0
- package/dist/system/CorePowerSyncSystem.js.map +1 -0
- package/dist/util/Mutex.d.ts +47 -0
- package/dist/util/Mutex.js +132 -0
- package/dist/util/Mutex.js.map +1 -0
- package/dist/util/PgManager.d.ts +24 -0
- package/dist/util/PgManager.js +55 -0
- package/dist/util/PgManager.js.map +1 -0
- package/dist/util/alerting.d.ts +4 -0
- package/dist/util/alerting.js +14 -0
- package/dist/util/alerting.js.map +1 -0
- package/dist/util/config/collectors/config-collector.d.ts +29 -0
- package/dist/util/config/collectors/config-collector.js +116 -0
- package/dist/util/config/collectors/config-collector.js.map +1 -0
- package/dist/util/config/collectors/impl/base64-config-collector.d.ts +6 -0
- package/dist/util/config/collectors/impl/base64-config-collector.js +15 -0
- package/dist/util/config/collectors/impl/base64-config-collector.js.map +1 -0
- package/dist/util/config/collectors/impl/fallback-config-collector.d.ts +11 -0
- package/dist/util/config/collectors/impl/fallback-config-collector.js +19 -0
- package/dist/util/config/collectors/impl/fallback-config-collector.js.map +1 -0
- package/dist/util/config/collectors/impl/filesystem-config-collector.d.ts +6 -0
- package/dist/util/config/collectors/impl/filesystem-config-collector.js +35 -0
- package/dist/util/config/collectors/impl/filesystem-config-collector.js.map +1 -0
- package/dist/util/config/compound-config-collector.d.ts +32 -0
- package/dist/util/config/compound-config-collector.js +126 -0
- package/dist/util/config/compound-config-collector.js.map +1 -0
- package/dist/util/config/sync-rules/impl/base64-sync-rules-collector.d.ts +7 -0
- package/dist/util/config/sync-rules/impl/base64-sync-rules-collector.js +17 -0
- package/dist/util/config/sync-rules/impl/base64-sync-rules-collector.js.map +1 -0
- package/dist/util/config/sync-rules/impl/filesystem-sync-rules-collector.d.ts +7 -0
- package/dist/util/config/sync-rules/impl/filesystem-sync-rules-collector.js +21 -0
- package/dist/util/config/sync-rules/impl/filesystem-sync-rules-collector.js.map +1 -0
- package/dist/util/config/sync-rules/impl/inline-sync-rules-collector.d.ts +7 -0
- package/dist/util/config/sync-rules/impl/inline-sync-rules-collector.js +17 -0
- package/dist/util/config/sync-rules/impl/inline-sync-rules-collector.js.map +1 -0
- package/dist/util/config/sync-rules/sync-collector.d.ts +6 -0
- package/dist/util/config/sync-rules/sync-collector.js +3 -0
- package/dist/util/config/sync-rules/sync-collector.js.map +1 -0
- package/dist/util/config/types.d.ts +53 -0
- package/dist/util/config/types.js +7 -0
- package/dist/util/config/types.js.map +1 -0
- package/dist/util/config.d.ts +7 -0
- package/dist/util/config.js +35 -0
- package/dist/util/config.js.map +1 -0
- package/dist/util/env.d.ts +10 -0
- package/dist/util/env.js +25 -0
- package/dist/util/env.js.map +1 -0
- package/dist/util/memory-tracking.d.ts +7 -0
- package/dist/util/memory-tracking.js +58 -0
- package/dist/util/memory-tracking.js.map +1 -0
- package/dist/util/migration_lib.d.ts +11 -0
- package/dist/util/migration_lib.js +64 -0
- package/dist/util/migration_lib.js.map +1 -0
- package/dist/util/pgwire_utils.d.ts +24 -0
- package/dist/util/pgwire_utils.js +117 -0
- package/dist/util/pgwire_utils.js.map +1 -0
- package/dist/util/populate_test_data.d.ts +8 -0
- package/dist/util/populate_test_data.js +65 -0
- package/dist/util/populate_test_data.js.map +1 -0
- package/dist/util/protocol-types.d.ts +178 -0
- package/dist/util/protocol-types.js +38 -0
- package/dist/util/protocol-types.js.map +1 -0
- package/dist/util/secs.d.ts +2 -0
- package/dist/util/secs.js +49 -0
- package/dist/util/secs.js.map +1 -0
- package/dist/util/util-index.d.ts +22 -0
- package/dist/util/util-index.js +23 -0
- package/dist/util/util-index.js.map +1 -0
- package/dist/util/utils.d.ts +14 -0
- package/dist/util/utils.js +75 -0
- package/dist/util/utils.js.map +1 -0
- package/package.json +55 -0
- package/src/api/api-index.ts +2 -0
- package/src/api/diagnostics.ts +221 -0
- package/src/api/schema.ts +99 -0
- package/src/auth/CachedKeyCollector.ts +132 -0
- package/src/auth/CompoundKeyCollector.ts +33 -0
- package/src/auth/JwtPayload.ts +11 -0
- package/src/auth/KeyCollector.ts +27 -0
- package/src/auth/KeySpec.ts +67 -0
- package/src/auth/KeyStore.ts +156 -0
- package/src/auth/LeakyBucket.ts +66 -0
- package/src/auth/RemoteJWKSCollector.ts +130 -0
- package/src/auth/StaticKeyCollector.ts +21 -0
- package/src/auth/SupabaseKeyCollector.ts +67 -0
- package/src/auth/auth-index.ts +10 -0
- package/src/db/db-index.ts +1 -0
- package/src/db/mongo.ts +72 -0
- package/src/entry/cli-entry.ts +41 -0
- package/src/entry/commands/config-command.ts +36 -0
- package/src/entry/commands/migrate-action.ts +25 -0
- package/src/entry/commands/start-action.ts +24 -0
- package/src/entry/commands/teardown-action.ts +23 -0
- package/src/entry/entry-index.ts +5 -0
- package/src/index.ts +37 -0
- package/src/metrics/metrics.ts +169 -0
- package/src/migrations/db/migrations/1684951997326-init.ts +33 -0
- package/src/migrations/db/migrations/1688556755264-initial-sync-rules.ts +5 -0
- package/src/migrations/db/migrations/1702295701188-sync-rule-state.ts +99 -0
- package/src/migrations/db/migrations/1711543888062-write-checkpoint-index.ts +32 -0
- package/src/migrations/db/store.ts +11 -0
- package/src/migrations/migrations.ts +122 -0
- package/src/replication/ErrorRateLimiter.ts +49 -0
- package/src/replication/PgRelation.ts +42 -0
- package/src/replication/WalConnection.ts +227 -0
- package/src/replication/WalStream.ts +626 -0
- package/src/replication/WalStreamManager.ts +214 -0
- package/src/replication/WalStreamRunner.ts +180 -0
- package/src/replication/replication-index.ts +7 -0
- package/src/replication/util.ts +76 -0
- package/src/routes/admin.ts +229 -0
- package/src/routes/auth.ts +209 -0
- package/src/routes/checkpointing.ts +38 -0
- package/src/routes/dev.ts +194 -0
- package/src/routes/route-generators.ts +39 -0
- package/src/routes/router-socket.ts +13 -0
- package/src/routes/router.ts +17 -0
- package/src/routes/routes-index.ts +5 -0
- package/src/routes/socket-route.ts +131 -0
- package/src/routes/sync-rules.ts +210 -0
- package/src/routes/sync-stream.ts +92 -0
- package/src/runner/teardown.ts +91 -0
- package/src/storage/BucketStorage.ts +386 -0
- package/src/storage/MongoBucketStorage.ts +493 -0
- package/src/storage/SourceTable.ts +60 -0
- package/src/storage/mongo/MongoBucketBatch.ts +756 -0
- package/src/storage/mongo/MongoIdSequence.ts +24 -0
- package/src/storage/mongo/MongoPersistedSyncRules.ts +16 -0
- package/src/storage/mongo/MongoPersistedSyncRulesContent.ts +47 -0
- package/src/storage/mongo/MongoSyncBucketStorage.ts +517 -0
- package/src/storage/mongo/MongoSyncRulesLock.ts +81 -0
- package/src/storage/mongo/OperationBatch.ts +115 -0
- package/src/storage/mongo/PersistedBatch.ts +245 -0
- package/src/storage/mongo/db.ts +69 -0
- package/src/storage/mongo/models.ts +157 -0
- package/src/storage/mongo/util.ts +88 -0
- package/src/storage/storage-index.ts +15 -0
- package/src/sync/BroadcastIterable.ts +161 -0
- package/src/sync/LastValueSink.ts +100 -0
- package/src/sync/merge.ts +200 -0
- package/src/sync/safeRace.ts +99 -0
- package/src/sync/sync-index.ts +6 -0
- package/src/sync/sync.ts +312 -0
- package/src/sync/util.ts +98 -0
- package/src/system/CorePowerSyncSystem.ts +43 -0
- package/src/util/Mutex.ts +159 -0
- package/src/util/PgManager.ts +64 -0
- package/src/util/alerting.ts +17 -0
- package/src/util/config/collectors/config-collector.ts +141 -0
- package/src/util/config/collectors/impl/base64-config-collector.ts +18 -0
- package/src/util/config/collectors/impl/fallback-config-collector.ts +22 -0
- package/src/util/config/collectors/impl/filesystem-config-collector.ts +41 -0
- package/src/util/config/compound-config-collector.ts +171 -0
- package/src/util/config/sync-rules/impl/base64-sync-rules-collector.ts +21 -0
- package/src/util/config/sync-rules/impl/filesystem-sync-rules-collector.ts +26 -0
- package/src/util/config/sync-rules/impl/inline-sync-rules-collector.ts +21 -0
- package/src/util/config/sync-rules/sync-collector.ts +8 -0
- package/src/util/config/types.ts +60 -0
- package/src/util/config.ts +39 -0
- package/src/util/env.ts +28 -0
- package/src/util/memory-tracking.ts +67 -0
- package/src/util/migration_lib.ts +79 -0
- package/src/util/pgwire_utils.ts +139 -0
- package/src/util/populate_test_data.ts +78 -0
- package/src/util/protocol-types.ts +223 -0
- package/src/util/secs.ts +54 -0
- package/src/util/util-index.ts +25 -0
- package/src/util/utils.ts +102 -0
- package/test/src/__snapshots__/pg_test.test.ts.snap +256 -0
- package/test/src/__snapshots__/sync.test.ts.snap +235 -0
- package/test/src/auth.test.ts +340 -0
- package/test/src/broadcast_iterable.test.ts +156 -0
- package/test/src/data_storage.test.ts +1176 -0
- package/test/src/env.ts +8 -0
- package/test/src/large_batch.test.ts +194 -0
- package/test/src/merge_iterable.test.ts +355 -0
- package/test/src/pg_test.test.ts +432 -0
- package/test/src/schema_changes.test.ts +545 -0
- package/test/src/slow_tests.test.ts +257 -0
- package/test/src/sql_functions.test.ts +254 -0
- package/test/src/sql_operators.test.ts +132 -0
- package/test/src/sync.test.ts +293 -0
- package/test/src/sync_rules.test.ts +1051 -0
- package/test/src/util.ts +67 -0
- package/test/src/validation.test.ts +63 -0
- package/test/src/wal_stream.test.ts +310 -0
- package/test/src/wal_stream_utils.ts +147 -0
- package/test/tsconfig.json +20 -0
- package/tsconfig.json +20 -0
- package/tsconfig.tsbuildinfo +1 -0
- package/vitest.config.ts +11 -0
|
@@ -0,0 +1,117 @@
|
|
|
1
|
+
// Adapted from https://github.com/kagis/pgwire/blob/0dc927f9f8990a903f238737326e53ba1c8d094f/mod.js#L2218
|
|
2
|
+
import * as bson from 'bson';
|
|
3
|
+
import * as uuid from 'uuid';
|
|
4
|
+
import * as pgwire from '@powersync/service-jpgwire';
|
|
5
|
+
import { toSyncRulesRow } from '@powersync/service-sync-rules';
|
|
6
|
+
import * as micro from '@journeyapps-platform/micro';
|
|
7
|
+
/**
|
|
8
|
+
* pgwire message -> SQLite row.
|
|
9
|
+
* @param message
|
|
10
|
+
*/
|
|
11
|
+
export function constructAfterRecord(message) {
|
|
12
|
+
const rawData = message.afterRaw;
|
|
13
|
+
const record = pgwire.decodeTuple(message.relation, rawData);
|
|
14
|
+
return toSyncRulesRow(record);
|
|
15
|
+
}
|
|
16
|
+
export function hasToastedValues(row) {
|
|
17
|
+
for (let key in row) {
|
|
18
|
+
if (typeof row[key] == 'undefined') {
|
|
19
|
+
return true;
|
|
20
|
+
}
|
|
21
|
+
}
|
|
22
|
+
return false;
|
|
23
|
+
}
|
|
24
|
+
export function isCompleteRow(row) {
|
|
25
|
+
return !hasToastedValues(row);
|
|
26
|
+
}
|
|
27
|
+
/**
|
|
28
|
+
* pgwire message -> SQLite row.
|
|
29
|
+
* @param message
|
|
30
|
+
*/
|
|
31
|
+
export function constructBeforeRecord(message) {
|
|
32
|
+
const rawData = message.beforeRaw;
|
|
33
|
+
if (rawData == null) {
|
|
34
|
+
return undefined;
|
|
35
|
+
}
|
|
36
|
+
const record = pgwire.decodeTuple(message.relation, rawData);
|
|
37
|
+
return toSyncRulesRow(record);
|
|
38
|
+
}
|
|
39
|
+
function getRawReplicaIdentity(tuple, columns) {
|
|
40
|
+
let result = {};
|
|
41
|
+
for (let column of columns) {
|
|
42
|
+
const name = column.name;
|
|
43
|
+
result[name] = tuple[name];
|
|
44
|
+
}
|
|
45
|
+
return result;
|
|
46
|
+
}
|
|
47
|
+
const ID_NAMESPACE = 'a396dd91-09fc-4017-a28d-3df722f651e9';
|
|
48
|
+
export function getUuidReplicaIdentityString(tuple, columns) {
|
|
49
|
+
const rawIdentity = getRawReplicaIdentity(tuple, columns);
|
|
50
|
+
return uuidForRow(rawIdentity);
|
|
51
|
+
}
|
|
52
|
+
export function uuidForRow(row) {
|
|
53
|
+
// Important: This must not change, since it will affect how ids are generated.
|
|
54
|
+
// Use BSON so that it's a well-defined format without encoding ambiguities.
|
|
55
|
+
const repr = bson.serialize(row);
|
|
56
|
+
return uuid.v5(repr, ID_NAMESPACE);
|
|
57
|
+
}
|
|
58
|
+
export function getUuidReplicaIdentityBson(tuple, columns) {
|
|
59
|
+
if (columns.length == 0) {
|
|
60
|
+
// REPLICA IDENTITY NOTHING - generate random id
|
|
61
|
+
return new bson.UUID(uuid.v4());
|
|
62
|
+
}
|
|
63
|
+
const rawIdentity = getRawReplicaIdentity(tuple, columns);
|
|
64
|
+
return uuidForRowBson(rawIdentity);
|
|
65
|
+
}
|
|
66
|
+
export function uuidForRowBson(row) {
|
|
67
|
+
// Important: This must not change, since it will affect how ids are generated.
|
|
68
|
+
// Use BSON so that it's a well-defined format without encoding ambiguities.
|
|
69
|
+
const repr = bson.serialize(row);
|
|
70
|
+
const buffer = Buffer.alloc(16);
|
|
71
|
+
return new bson.UUID(uuid.v5(repr, ID_NAMESPACE, buffer));
|
|
72
|
+
}
|
|
73
|
+
export function escapeIdentifier(identifier) {
|
|
74
|
+
return `"${identifier.replace(/"/g, '""').replace(/\./g, '"."')}"`;
|
|
75
|
+
}
|
|
76
|
+
export function autoParameter(arg) {
|
|
77
|
+
if (arg == null) {
|
|
78
|
+
return { type: 'varchar', value: null };
|
|
79
|
+
}
|
|
80
|
+
else if (typeof arg == 'string') {
|
|
81
|
+
return { type: 'varchar', value: arg };
|
|
82
|
+
}
|
|
83
|
+
else if (typeof arg == 'number') {
|
|
84
|
+
if (Number.isInteger(arg)) {
|
|
85
|
+
return { type: 'int8', value: arg };
|
|
86
|
+
}
|
|
87
|
+
else {
|
|
88
|
+
return { type: 'float8', value: arg };
|
|
89
|
+
}
|
|
90
|
+
}
|
|
91
|
+
else if (typeof arg == 'boolean') {
|
|
92
|
+
return { type: 'bool', value: arg };
|
|
93
|
+
}
|
|
94
|
+
else if (typeof arg == 'bigint') {
|
|
95
|
+
return { type: 'int8', value: arg };
|
|
96
|
+
}
|
|
97
|
+
else {
|
|
98
|
+
throw new Error(`Unsupported query parameter: ${typeof arg}`);
|
|
99
|
+
}
|
|
100
|
+
}
|
|
101
|
+
/**
|
|
102
|
+
* Retry a simple query - up to 2 attempts total.
|
|
103
|
+
*/
|
|
104
|
+
export async function retriedQuery(db, ...args) {
|
|
105
|
+
for (let tries = 2;; tries--) {
|
|
106
|
+
try {
|
|
107
|
+
return await db.query(...args);
|
|
108
|
+
}
|
|
109
|
+
catch (e) {
|
|
110
|
+
if (tries == 1) {
|
|
111
|
+
throw e;
|
|
112
|
+
}
|
|
113
|
+
micro.logger.warn('Query error, retrying', e);
|
|
114
|
+
}
|
|
115
|
+
}
|
|
116
|
+
}
|
|
117
|
+
//# sourceMappingURL=pgwire_utils.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"pgwire_utils.js","sourceRoot":"","sources":["../../src/util/pgwire_utils.ts"],"names":[],"mappings":"AAAA,0GAA0G;AAE1G,OAAO,KAAK,IAAI,MAAM,MAAM,CAAC;AAC7B,OAAO,KAAK,IAAI,MAAM,MAAM,CAAC;AAC7B,OAAO,KAAK,MAAM,MAAM,4BAA4B,CAAC;AACrD,OAAO,EAAkD,cAAc,EAAE,MAAM,+BAA+B,CAAC;AAC/G,OAAO,KAAK,KAAK,MAAM,6BAA6B,CAAC;AAIrD;;;GAGG;AACH,MAAM,UAAU,oBAAoB,CAAC,OAAsD;IACzF,MAAM,OAAO,GAAI,OAAe,CAAC,QAAQ,CAAC;IAE1C,MAAM,MAAM,GAAG,MAAM,CAAC,WAAW,CAAC,OAAO,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;IAC7D,OAAO,cAAc,CAAC,MAAM,CAAC,CAAC;AAChC,CAAC;AAED,MAAM,UAAU,gBAAgB,CAAC,GAAuB;IACtD,KAAK,IAAI,GAAG,IAAI,GAAG,EAAE,CAAC;QACpB,IAAI,OAAO,GAAG,CAAC,GAAG,CAAC,IAAI,WAAW,EAAE,CAAC;YACnC,OAAO,IAAI,CAAC;QACd,CAAC;IACH,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED,MAAM,UAAU,aAAa,CAAC,GAAuB;IACnD,OAAO,CAAC,gBAAgB,CAAC,GAAG,CAAC,CAAC;AAChC,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,qBAAqB,CAAC,OAAsD;IAC1F,MAAM,OAAO,GAAI,OAAe,CAAC,SAAS,CAAC;IAC3C,IAAI,OAAO,IAAI,IAAI,EAAE,CAAC;QACpB,OAAO,SAAS,CAAC;IACnB,CAAC;IACD,MAAM,MAAM,GAAG,MAAM,CAAC,WAAW,CAAC,OAAO,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;IAC7D,OAAO,cAAc,CAAC,MAAM,CAAC,CAAC;AAChC,CAAC;AAED,SAAS,qBAAqB,CAC5B,KAAyB,EACzB,OAAwC;IAExC,IAAI,MAAM,GAAwB,EAAE,CAAC;IACrC,KAAK,IAAI,MAAM,IAAI,OAAO,EAAE,CAAC;QAC3B,MAAM,IAAI,GAAG,MAAM,CAAC,IAAI,CAAC;QACzB,MAAM,CAAC,IAAI,CAAC,GAAG,KAAK,CAAC,IAAI,CAAC,CAAC;IAC7B,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC;AACD,MAAM,YAAY,GAAG,sCAAsC,CAAC;AAE5D,MAAM,UAAU,4BAA4B,CAC1C,KAAyB,EACzB,OAAwC;IAExC,MAAM,WAAW,GAAG,qBAAqB,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC;IAE1D,OAAO,UAAU,CAAC,WAAW,CAAC,CAAC;AACjC,CAAC;AAED,MAAM,UAAU,UAAU,CAAC,GAAc;IACvC,+EAA+E;IAC/E,4EAA4E;IAC5E,MAAM,IAAI,GAAG,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC;IACjC,OAAO,IAAI,CAAC,EAAE,CAAC,IAAI,EAAE,YAAY,CAAC,CAAC;AACrC,CAAC;AAED,MAAM,UAAU,0BAA0B,CACxC,KAAyB,EACzB,OAAwC;IAExC,IAAI,OAAO,CAAC,MAAM,IAAI,CAAC,EAAE,CAAC;QACxB,gDAAgD;QAChD,OAAO,IAAI,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,EAAE,CAAC,CAAC;IAClC,CAAC;IACD,MAAM,WAAW,GAAG,qBAAqB,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC;IAE1D,OAAO,cAAc,CAAC,WAAW,CAAC,CAAC;AACrC,CAAC;AAED,MAAM,UAAU,cAAc,CAAC,GAAc;IAC3C,+EAA+E;IAC/E,4EAA4E;IAC5E,MAAM,IAAI,GAAG,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC;IACjC,MAAM,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;IAChC,OAAO,IAAI,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,EAAE,YAAY,EAAE,MAAM,CAAC,CAAC,CAAC;AAC5D,CAAC;AAED,MAAM,UAAU,gBAAgB,CAAC,UAAkB;IACjD,OAAO,IAAI,UAAU,CAAC,OAAO,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC,OAAO,CAAC,KAAK,EAAE,KAAK,CAAC,GAAG,CAAC;AACrE,CAAC;AAED,MAAM,UAAU,aAAa,CAAC,GAA8B;IAC1D,IAAI,GAAG,IAAI,IAAI,EAAE,CAAC;QAChB,OAAO,EAAE,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC;IAC1C,CAAC;SAAM,IAAI,OAAO,GAAG,IAAI,QAAQ,EAAE,CAAC;QAClC,OAAO,EAAE,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,GAAG,EAAE,CAAC;IACzC,CAAC;SAAM,IAAI,OAAO,GAAG,IAAI,QAAQ,EAAE,CAAC;QAClC,IAAI,MAAM,CAAC,SAAS,CAAC,GAAG,CAAC,EAAE,CAAC;YAC1B,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,GAAG,EAAE,CAAC;QACtC,CAAC;aAAM,CAAC;YACN,OAAO,EAAE,IAAI,EAAE,QAAQ,EAAE,KAAK,EAAE,GAAG,EAAE,CAAC;QACxC,CAAC;IACH,CAAC;SAAM,IAAI,OAAO,GAAG,IAAI,SAAS,EAAE,CAAC;QACnC,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,GAAG,EAAE,CAAC;IACtC,CAAC;SAAM,IAAI,OAAO,GAAG,IAAI,QAAQ,EAAE,CAAC;QAClC,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,GAAG,EAAE,CAAC;IACtC,CAAC;SAAM,CAAC;QACN,MAAM,IAAI,KAAK,CAAC,gCAAgC,OAAO,GAAG,EAAE,CAAC,CAAC;IAChE,CAAC;AACH,CAAC;AAKD;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,YAAY,CAAC,EAAmB,EAAE,GAAG,IAAW;IACpE,KAAK,IAAI,KAAK,GAAG,CAAC,GAAI,KAAK,EAAE,EAAE,CAAC;QAC9B,IAAI,CAAC;YACH,OAAO,MAAM,EAAE,CAAC,KAAK,CAAC,GAAG,IAAI,CAAC,CAAC;QACjC,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,IAAI,KAAK,IAAI,CAAC,EAAE,CAAC;gBACf,MAAM,CAAC,CAAC;YACV,CAAC;YACD,KAAK,CAAC,MAAM,CAAC,IAAI,CAAC,uBAAuB,EAAE,CAAC,CAAC,CAAC;QAChD,CAAC;IACH,CAAC;AACH,CAAC"}
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
import { NormalizedPostgresConnection } from '@powersync/service-types';
|
|
2
|
+
export interface PopulateDataOptions {
|
|
3
|
+
connection: NormalizedPostgresConnection;
|
|
4
|
+
num_transactions: number;
|
|
5
|
+
per_transaction: number;
|
|
6
|
+
size: number;
|
|
7
|
+
}
|
|
8
|
+
export declare function populateData(options: PopulateDataOptions): Promise<number>;
|
|
@@ -0,0 +1,65 @@
|
|
|
1
|
+
import * as crypto from 'crypto';
|
|
2
|
+
import { Worker, isMainThread, parentPort, workerData } from 'node:worker_threads';
|
|
3
|
+
import { connectPgWire } from '@powersync/service-jpgwire';
|
|
4
|
+
if (isMainThread || parentPort == null) {
|
|
5
|
+
// Not a worker - ignore
|
|
6
|
+
}
|
|
7
|
+
else {
|
|
8
|
+
try {
|
|
9
|
+
const options = workerData;
|
|
10
|
+
const result = await populateDataInner(options);
|
|
11
|
+
parentPort.postMessage(result);
|
|
12
|
+
process.exit(0);
|
|
13
|
+
}
|
|
14
|
+
catch (e) {
|
|
15
|
+
// This is a bug, not a connection issue
|
|
16
|
+
console.error(e);
|
|
17
|
+
// Only closes the Worker thread
|
|
18
|
+
process.exit(2);
|
|
19
|
+
}
|
|
20
|
+
}
|
|
21
|
+
async function populateDataInner(options) {
|
|
22
|
+
// Dedicated connection so we can release the memory easily
|
|
23
|
+
const initialDb = await connectPgWire(options.connection, { type: 'standard' });
|
|
24
|
+
const largeDescription = crypto.randomBytes(options.size / 2).toString('hex');
|
|
25
|
+
let operation_count = 0;
|
|
26
|
+
for (let i = 0; i < options.num_transactions; i++) {
|
|
27
|
+
const prefix = `test${i}K`;
|
|
28
|
+
await initialDb.query({
|
|
29
|
+
statement: `INSERT INTO test_data(id, description, other) SELECT $1 || i, $2, 'foo' FROM generate_series(1, $3) i`,
|
|
30
|
+
params: [
|
|
31
|
+
{ type: 'varchar', value: prefix },
|
|
32
|
+
{ type: 'varchar', value: largeDescription },
|
|
33
|
+
{ type: 'int4', value: options.per_transaction }
|
|
34
|
+
]
|
|
35
|
+
});
|
|
36
|
+
operation_count += options.per_transaction;
|
|
37
|
+
}
|
|
38
|
+
await initialDb.end();
|
|
39
|
+
return operation_count;
|
|
40
|
+
}
|
|
41
|
+
export async function populateData(options) {
|
|
42
|
+
const WORKER_TIMEOUT = 30000;
|
|
43
|
+
const worker = new Worker(new URL('./populate_test_data.js', import.meta.url), {
|
|
44
|
+
workerData: options
|
|
45
|
+
});
|
|
46
|
+
const timeout = setTimeout(() => {
|
|
47
|
+
// Exits with code 1 below
|
|
48
|
+
worker.terminate();
|
|
49
|
+
}, WORKER_TIMEOUT);
|
|
50
|
+
try {
|
|
51
|
+
return await new Promise((resolve, reject) => {
|
|
52
|
+
worker.on('message', resolve);
|
|
53
|
+
worker.on('error', reject);
|
|
54
|
+
worker.on('exit', (code) => {
|
|
55
|
+
if (code !== 0) {
|
|
56
|
+
reject(new Error(`Populating data failed with exit code ${code}`));
|
|
57
|
+
}
|
|
58
|
+
});
|
|
59
|
+
});
|
|
60
|
+
}
|
|
61
|
+
finally {
|
|
62
|
+
clearTimeout(timeout);
|
|
63
|
+
}
|
|
64
|
+
}
|
|
65
|
+
//# sourceMappingURL=populate_test_data.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"populate_test_data.js","sourceRoot":"","sources":["../../src/util/populate_test_data.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,MAAM,MAAM,QAAQ,CAAC;AACjC,OAAO,EAAE,MAAM,EAAE,YAAY,EAAE,UAAU,EAAE,UAAU,EAAE,MAAM,qBAAqB,CAAC;AAEnF,OAAO,EAAE,aAAa,EAAE,MAAM,4BAA4B,CAAC;AAY3D,IAAI,YAAY,IAAI,UAAU,IAAI,IAAI,EAAE,CAAC;IACvC,wBAAwB;AAC1B,CAAC;KAAM,CAAC;IACN,IAAI,CAAC;QACH,MAAM,OAAO,GAAG,UAAiC,CAAC;QAElD,MAAM,MAAM,GAAG,MAAM,iBAAiB,CAAC,OAAO,CAAC,CAAC;QAChD,UAAU,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC;QAC/B,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACX,wCAAwC;QACxC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;QACjB,gCAAgC;QAChC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;AACH,CAAC;AAED,KAAK,UAAU,iBAAiB,CAAC,OAA4B;IAC3D,2DAA2D;IAC3D,MAAM,SAAS,GAAG,MAAM,aAAa,CAAC,OAAO,CAAC,UAAU,EAAE,EAAE,IAAI,EAAE,UAAU,EAAE,CAAC,CAAC;IAChF,MAAM,gBAAgB,GAAG,MAAM,CAAC,WAAW,CAAC,OAAO,CAAC,IAAI,GAAG,CAAC,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;IAC9E,IAAI,eAAe,GAAG,CAAC,CAAC;IACxB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,OAAO,CAAC,gBAAgB,EAAE,CAAC,EAAE,EAAE,CAAC;QAClD,MAAM,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC;QAE3B,MAAM,SAAS,CAAC,KAAK,CAAC;YACpB,SAAS,EAAE,uGAAuG;YAClH,MAAM,EAAE;gBACN,EAAE,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,MAAM,EAAE;gBAClC,EAAE,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,gBAAgB,EAAE;gBAC5C,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,OAAO,CAAC,eAAe,EAAE;aACjD;SACF,CAAC,CAAC;QACH,eAAe,IAAI,OAAO,CAAC,eAAe,CAAC;IAC7C,CAAC;IACD,MAAM,SAAS,CAAC,GAAG,EAAE,CAAC;IACtB,OAAO,eAAe,CAAC;AACzB,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,YAAY,CAAC,OAA4B;IAC7D,MAAM,cAAc,GAAG,KAAM,CAAC;IAE9B,MAAM,MAAM,GAAG,IAAI,MAAM,CAAC,IAAI,GAAG,CAAC,yBAAyB,EAAE,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE;QAC7E,UAAU,EAAE,OAAO;KACpB,CAAC,CAAC;IACH,MAAM,OAAO,GAAG,UAAU,CAAC,GAAG,EAAE;QAC9B,0BAA0B;QAC1B,MAAM,CAAC,SAAS,EAAE,CAAC;IACrB,CAAC,EAAE,cAAc,CAAC,CAAC;IACnB,IAAI,CAAC;QACH,OAAO,MAAM,IAAI,OAAO,CAAS,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;YACnD,MAAM,CAAC,EAAE,CAAC,SAAS,EAAE,OAAO,CAAC,CAAC;YAC9B,MAAM,CAAC,EAAE,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;YAC3B,MAAM,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC,IAAI,EAAE,EAAE;gBACzB,IAAI,IAAI,KAAK,CAAC,EAAE,CAAC;oBACf,MAAM,CAAC,IAAI,KAAK,CAAC,yCAAyC,IAAI,EAAE,CAAC,CAAC,CAAC;gBACrE,CAAC;YACH,CAAC,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;IACL,CAAC;YAAS,CAAC;QACT,YAAY,CAAC,OAAO,CAAC,CAAC;IACxB,CAAC;AACH,CAAC"}
|
|
@@ -0,0 +1,178 @@
|
|
|
1
|
+
import * as t from 'ts-codec';
|
|
2
|
+
import { SqliteJsonValue } from '@powersync/service-sync-rules';
|
|
3
|
+
/**
|
|
4
|
+
* For sync2.json
|
|
5
|
+
*/
|
|
6
|
+
export interface ContinueCheckpointRequest {
|
|
7
|
+
/**
|
|
8
|
+
* Existing bucket states. Only these buckets are synchronized.
|
|
9
|
+
*/
|
|
10
|
+
buckets: BucketRequest[];
|
|
11
|
+
checkpoint_token: string;
|
|
12
|
+
limit?: number;
|
|
13
|
+
}
|
|
14
|
+
export interface SyncNewCheckpointRequest {
|
|
15
|
+
/**
|
|
16
|
+
* Existing bucket states. Used if include_data is specified.
|
|
17
|
+
*/
|
|
18
|
+
buckets?: BucketRequest[];
|
|
19
|
+
request_checkpoint: {
|
|
20
|
+
/**
|
|
21
|
+
* Whether or not to include an initial data request.
|
|
22
|
+
*/
|
|
23
|
+
include_data: boolean;
|
|
24
|
+
/**
|
|
25
|
+
* Whether or not to compute a checksum.
|
|
26
|
+
*/
|
|
27
|
+
include_checksum: boolean;
|
|
28
|
+
};
|
|
29
|
+
limit?: number;
|
|
30
|
+
}
|
|
31
|
+
export type SyncRequest = ContinueCheckpointRequest | SyncNewCheckpointRequest;
|
|
32
|
+
export interface SyncResponse {
|
|
33
|
+
/**
|
|
34
|
+
* Data for the buckets returned. May not have an an entry for each bucket in the request.
|
|
35
|
+
*/
|
|
36
|
+
data?: SyncBucketData[];
|
|
37
|
+
/**
|
|
38
|
+
* True if the response limit has been reached, and another request must be made.
|
|
39
|
+
*/
|
|
40
|
+
has_more: boolean;
|
|
41
|
+
checkpoint_token?: string;
|
|
42
|
+
checkpoint?: Checkpoint;
|
|
43
|
+
}
|
|
44
|
+
export declare const BucketRequest: t.ObjectCodec<{
|
|
45
|
+
name: t.IdentityCodec<t.CodecType.String>;
|
|
46
|
+
/**
|
|
47
|
+
* Base-10 number. Sync all data from this bucket with op_id > after.
|
|
48
|
+
*/
|
|
49
|
+
after: t.IdentityCodec<t.CodecType.String>;
|
|
50
|
+
}>;
|
|
51
|
+
export type BucketRequest = t.Decoded<typeof BucketRequest>;
|
|
52
|
+
export declare const StreamingSyncRequest: t.ObjectCodec<{
|
|
53
|
+
/**
|
|
54
|
+
* Existing bucket states.
|
|
55
|
+
*/
|
|
56
|
+
buckets: t.OptionalCodec<t.Codec<{
|
|
57
|
+
name: string;
|
|
58
|
+
after: string;
|
|
59
|
+
}[], {
|
|
60
|
+
name: string;
|
|
61
|
+
after: string;
|
|
62
|
+
}[], string, t.CodecProps>>;
|
|
63
|
+
/**
|
|
64
|
+
* If specified, limit the response to only include these buckets.
|
|
65
|
+
*/
|
|
66
|
+
only: t.OptionalCodec<t.Codec<string[], string[], string, t.CodecProps>>;
|
|
67
|
+
/**
|
|
68
|
+
* Whether or not to compute a checksum for each checkpoint
|
|
69
|
+
*/
|
|
70
|
+
include_checksum: t.OptionalCodec<t.Codec<boolean, boolean, string, t.CodecProps>>;
|
|
71
|
+
/**
|
|
72
|
+
* True to keep `data` as a string, instead of nested JSON.
|
|
73
|
+
*/
|
|
74
|
+
raw_data: t.OptionalCodec<t.Codec<boolean, boolean, string, t.CodecProps>>;
|
|
75
|
+
/**
|
|
76
|
+
* Data is received in a serialized BSON Buffer
|
|
77
|
+
*/
|
|
78
|
+
binary_data: t.OptionalCodec<t.Codec<boolean, boolean, string, t.CodecProps>>;
|
|
79
|
+
}>;
|
|
80
|
+
export type StreamingSyncRequest = t.Decoded<typeof StreamingSyncRequest>;
|
|
81
|
+
export interface StreamingSyncCheckpoint {
|
|
82
|
+
checkpoint: Checkpoint;
|
|
83
|
+
}
|
|
84
|
+
export interface StreamingSyncCheckpointDiff {
|
|
85
|
+
checkpoint_diff: {
|
|
86
|
+
last_op_id: OpId;
|
|
87
|
+
write_checkpoint?: OpId;
|
|
88
|
+
updated_buckets: BucketChecksum[];
|
|
89
|
+
removed_buckets: string[];
|
|
90
|
+
};
|
|
91
|
+
}
|
|
92
|
+
export interface StreamingSyncData {
|
|
93
|
+
data: SyncBucketData;
|
|
94
|
+
}
|
|
95
|
+
export interface StreamingSyncCheckpointComplete {
|
|
96
|
+
checkpoint_complete: {
|
|
97
|
+
last_op_id: OpId;
|
|
98
|
+
};
|
|
99
|
+
}
|
|
100
|
+
export interface StreamingSyncKeepalive {
|
|
101
|
+
}
|
|
102
|
+
export type StreamingSyncLine = StreamingSyncData | StreamingSyncCheckpoint | StreamingSyncCheckpointDiff | StreamingSyncCheckpointComplete | StreamingSyncKeepalive;
|
|
103
|
+
/**
|
|
104
|
+
* 64-bit unsigned number, as a base-10 string.
|
|
105
|
+
*/
|
|
106
|
+
export type OpId = string;
|
|
107
|
+
export interface Checkpoint {
|
|
108
|
+
last_op_id: OpId;
|
|
109
|
+
write_checkpoint?: OpId;
|
|
110
|
+
buckets: BucketChecksum[];
|
|
111
|
+
}
|
|
112
|
+
export interface BucketState {
|
|
113
|
+
bucket: string;
|
|
114
|
+
op_id: string;
|
|
115
|
+
}
|
|
116
|
+
export interface SyncDataBatch {
|
|
117
|
+
buckets: SyncBucketData[];
|
|
118
|
+
}
|
|
119
|
+
export interface SyncBucketData {
|
|
120
|
+
bucket: string;
|
|
121
|
+
data: OplogEntry[];
|
|
122
|
+
/**
|
|
123
|
+
* True if the response does not contain all the data for this bucket, and another request must be made.
|
|
124
|
+
*/
|
|
125
|
+
has_more: boolean;
|
|
126
|
+
/**
|
|
127
|
+
* The `after` specified in the request.
|
|
128
|
+
*/
|
|
129
|
+
after: OpId;
|
|
130
|
+
/**
|
|
131
|
+
* Use this for the next request.
|
|
132
|
+
*/
|
|
133
|
+
next_after: OpId;
|
|
134
|
+
}
|
|
135
|
+
export interface OplogEntry {
|
|
136
|
+
op_id: OpId;
|
|
137
|
+
op: 'PUT' | 'REMOVE' | 'MOVE' | 'CLEAR';
|
|
138
|
+
object_type?: string;
|
|
139
|
+
object_id?: string;
|
|
140
|
+
data?: Record<string, SqliteJsonValue> | string | null;
|
|
141
|
+
checksum: number | bigint;
|
|
142
|
+
subkey?: string;
|
|
143
|
+
}
|
|
144
|
+
export interface BucketChecksum {
|
|
145
|
+
bucket: string;
|
|
146
|
+
/**
|
|
147
|
+
* 32-bit unsigned hash.
|
|
148
|
+
*/
|
|
149
|
+
checksum: number;
|
|
150
|
+
/**
|
|
151
|
+
* Count of operations - informational only.
|
|
152
|
+
*/
|
|
153
|
+
count: number;
|
|
154
|
+
}
|
|
155
|
+
export declare function isContinueCheckpointRequest(request: SyncRequest): request is ContinueCheckpointRequest;
|
|
156
|
+
export declare function isSyncNewCheckpointRequest(request: SyncRequest): request is SyncNewCheckpointRequest;
|
|
157
|
+
/**
|
|
158
|
+
* For crud.json
|
|
159
|
+
*/
|
|
160
|
+
export interface CrudRequest {
|
|
161
|
+
data: CrudEntry[];
|
|
162
|
+
}
|
|
163
|
+
export interface CrudEntry {
|
|
164
|
+
op: 'PUT' | 'PATCH' | 'DELETE';
|
|
165
|
+
type: string;
|
|
166
|
+
id: string;
|
|
167
|
+
data: string;
|
|
168
|
+
}
|
|
169
|
+
export interface CrudResponse {
|
|
170
|
+
/**
|
|
171
|
+
* A sync response with a checkpoint >= this checkpoint would contain all the changes in this request.
|
|
172
|
+
*
|
|
173
|
+
* Any earlier checkpoint may or may not contain these changes.
|
|
174
|
+
*
|
|
175
|
+
* May be empty when the request contains no ops.
|
|
176
|
+
*/
|
|
177
|
+
checkpoint?: OpId;
|
|
178
|
+
}
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
import * as t from 'ts-codec';
|
|
2
|
+
export const BucketRequest = t.object({
|
|
3
|
+
name: t.string,
|
|
4
|
+
/**
|
|
5
|
+
* Base-10 number. Sync all data from this bucket with op_id > after.
|
|
6
|
+
*/
|
|
7
|
+
after: t.string
|
|
8
|
+
});
|
|
9
|
+
export const StreamingSyncRequest = t.object({
|
|
10
|
+
/**
|
|
11
|
+
* Existing bucket states.
|
|
12
|
+
*/
|
|
13
|
+
buckets: t.array(BucketRequest).optional(),
|
|
14
|
+
/**
|
|
15
|
+
* If specified, limit the response to only include these buckets.
|
|
16
|
+
*/
|
|
17
|
+
only: t.array(t.string).optional(),
|
|
18
|
+
/**
|
|
19
|
+
* Whether or not to compute a checksum for each checkpoint
|
|
20
|
+
*/
|
|
21
|
+
include_checksum: t.boolean.optional(),
|
|
22
|
+
/**
|
|
23
|
+
* True to keep `data` as a string, instead of nested JSON.
|
|
24
|
+
*/
|
|
25
|
+
raw_data: t.boolean.optional(),
|
|
26
|
+
/**
|
|
27
|
+
* Data is received in a serialized BSON Buffer
|
|
28
|
+
*/
|
|
29
|
+
binary_data: t.boolean.optional()
|
|
30
|
+
});
|
|
31
|
+
export function isContinueCheckpointRequest(request) {
|
|
32
|
+
return (Array.isArray(request.buckets) &&
|
|
33
|
+
typeof request.checkpoint_token == 'string');
|
|
34
|
+
}
|
|
35
|
+
export function isSyncNewCheckpointRequest(request) {
|
|
36
|
+
return typeof request.request_checkpoint == 'object';
|
|
37
|
+
}
|
|
38
|
+
//# sourceMappingURL=protocol-types.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"protocol-types.js","sourceRoot":"","sources":["../../src/util/protocol-types.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,CAAC,MAAM,UAAU,CAAC;AAwD9B,MAAM,CAAC,MAAM,aAAa,GAAG,CAAC,CAAC,MAAM,CAAC;IACpC,IAAI,EAAE,CAAC,CAAC,MAAM;IAEd;;OAEG;IACH,KAAK,EAAE,CAAC,CAAC,MAAM;CAChB,CAAC,CAAC;AAIH,MAAM,CAAC,MAAM,oBAAoB,GAAG,CAAC,CAAC,MAAM,CAAC;IAC3C;;OAEG;IACH,OAAO,EAAE,CAAC,CAAC,KAAK,CAAC,aAAa,CAAC,CAAC,QAAQ,EAAE;IAE1C;;OAEG;IACH,IAAI,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,QAAQ,EAAE;IAElC;;OAEG;IACH,gBAAgB,EAAE,CAAC,CAAC,OAAO,CAAC,QAAQ,EAAE;IAEtC;;OAEG;IACH,QAAQ,EAAE,CAAC,CAAC,OAAO,CAAC,QAAQ,EAAE;IAE9B;;OAEG;IACH,WAAW,EAAE,CAAC,CAAC,OAAO,CAAC,QAAQ,EAAE;CAClC,CAAC,CAAC;AAgGH,MAAM,UAAU,2BAA2B,CAAC,OAAoB;IAC9D,OAAO,CACL,KAAK,CAAC,OAAO,CAAE,OAAqC,CAAC,OAAO,CAAC;QAC7D,OAAQ,OAAqC,CAAC,gBAAgB,IAAI,QAAQ,CAC3E,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,0BAA0B,CAAC,OAAoB;IAC7D,OAAO,OAAQ,OAAoC,CAAC,kBAAkB,IAAI,QAAQ,CAAC;AACrF,CAAC"}
|
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
// From https://github.com/panva/jose/blob/1c2ae17a15838757ae2bfda9d1d8389d435f74b5/src/lib/secs.ts#L8
|
|
2
|
+
// MIT license.
|
|
3
|
+
// Copied here since it's not exported by jose.
|
|
4
|
+
const minute = 60;
|
|
5
|
+
const hour = minute * 60;
|
|
6
|
+
const day = hour * 24;
|
|
7
|
+
const week = day * 7;
|
|
8
|
+
const year = day * 365.25;
|
|
9
|
+
const REGEX = /^(\d+|\d+\.\d+) ?(seconds?|secs?|s|minutes?|mins?|m|hours?|hrs?|h|days?|d|weeks?|w|years?|yrs?|y)$/i;
|
|
10
|
+
export default (str) => {
|
|
11
|
+
const matched = REGEX.exec(str);
|
|
12
|
+
if (!matched) {
|
|
13
|
+
throw new TypeError('Invalid time period format');
|
|
14
|
+
}
|
|
15
|
+
const value = parseFloat(matched[1]);
|
|
16
|
+
const unit = matched[2].toLowerCase();
|
|
17
|
+
switch (unit) {
|
|
18
|
+
case 'sec':
|
|
19
|
+
case 'secs':
|
|
20
|
+
case 'second':
|
|
21
|
+
case 'seconds':
|
|
22
|
+
case 's':
|
|
23
|
+
return Math.round(value);
|
|
24
|
+
case 'minute':
|
|
25
|
+
case 'minutes':
|
|
26
|
+
case 'min':
|
|
27
|
+
case 'mins':
|
|
28
|
+
case 'm':
|
|
29
|
+
return Math.round(value * minute);
|
|
30
|
+
case 'hour':
|
|
31
|
+
case 'hours':
|
|
32
|
+
case 'hr':
|
|
33
|
+
case 'hrs':
|
|
34
|
+
case 'h':
|
|
35
|
+
return Math.round(value * hour);
|
|
36
|
+
case 'day':
|
|
37
|
+
case 'days':
|
|
38
|
+
case 'd':
|
|
39
|
+
return Math.round(value * day);
|
|
40
|
+
case 'week':
|
|
41
|
+
case 'weeks':
|
|
42
|
+
case 'w':
|
|
43
|
+
return Math.round(value * week);
|
|
44
|
+
// years matched
|
|
45
|
+
default:
|
|
46
|
+
return Math.round(value * year);
|
|
47
|
+
}
|
|
48
|
+
};
|
|
49
|
+
//# sourceMappingURL=secs.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"secs.js","sourceRoot":"","sources":["../../src/util/secs.ts"],"names":[],"mappings":"AAAA,sGAAsG;AACtG,eAAe;AACf,+CAA+C;AAE/C,MAAM,MAAM,GAAG,EAAE,CAAC;AAClB,MAAM,IAAI,GAAG,MAAM,GAAG,EAAE,CAAC;AACzB,MAAM,GAAG,GAAG,IAAI,GAAG,EAAE,CAAC;AACtB,MAAM,IAAI,GAAG,GAAG,GAAG,CAAC,CAAC;AACrB,MAAM,IAAI,GAAG,GAAG,GAAG,MAAM,CAAC;AAE1B,MAAM,KAAK,GAAG,qGAAqG,CAAC;AAEpH,eAAe,CAAC,GAAW,EAAU,EAAE;IACrC,MAAM,OAAO,GAAG,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IAEhC,IAAI,CAAC,OAAO,EAAE,CAAC;QACb,MAAM,IAAI,SAAS,CAAC,4BAA4B,CAAC,CAAC;IACpD,CAAC;IAED,MAAM,KAAK,GAAG,UAAU,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC;IACrC,MAAM,IAAI,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC;IAEtC,QAAQ,IAAI,EAAE,CAAC;QACb,KAAK,KAAK,CAAC;QACX,KAAK,MAAM,CAAC;QACZ,KAAK,QAAQ,CAAC;QACd,KAAK,SAAS,CAAC;QACf,KAAK,GAAG;YACN,OAAO,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;QAC3B,KAAK,QAAQ,CAAC;QACd,KAAK,SAAS,CAAC;QACf,KAAK,KAAK,CAAC;QACX,KAAK,MAAM,CAAC;QACZ,KAAK,GAAG;YACN,OAAO,IAAI,CAAC,KAAK,CAAC,KAAK,GAAG,MAAM,CAAC,CAAC;QACpC,KAAK,MAAM,CAAC;QACZ,KAAK,OAAO,CAAC;QACb,KAAK,IAAI,CAAC;QACV,KAAK,KAAK,CAAC;QACX,KAAK,GAAG;YACN,OAAO,IAAI,CAAC,KAAK,CAAC,KAAK,GAAG,IAAI,CAAC,CAAC;QAClC,KAAK,KAAK,CAAC;QACX,KAAK,MAAM,CAAC;QACZ,KAAK,GAAG;YACN,OAAO,IAAI,CAAC,KAAK,CAAC,KAAK,GAAG,GAAG,CAAC,CAAC;QACjC,KAAK,MAAM,CAAC;QACZ,KAAK,OAAO,CAAC;QACb,KAAK,GAAG;YACN,OAAO,IAAI,CAAC,KAAK,CAAC,KAAK,GAAG,IAAI,CAAC,CAAC;QAClC,gBAAgB;QAChB;YACE,OAAO,IAAI,CAAC,KAAK,CAAC,KAAK,GAAG,IAAI,CAAC,CAAC;IACpC,CAAC;AACH,CAAC,CAAC"}
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
export * from './alerting.js';
|
|
2
|
+
export * from './env.js';
|
|
3
|
+
export * from './memory-tracking.js';
|
|
4
|
+
export * from './migration_lib.js';
|
|
5
|
+
export * from './Mutex.js';
|
|
6
|
+
export * from './PgManager.js';
|
|
7
|
+
export * from './pgwire_utils.js';
|
|
8
|
+
export * from './populate_test_data.js';
|
|
9
|
+
export * from './protocol-types.js';
|
|
10
|
+
export * from './secs.js';
|
|
11
|
+
export * from './utils.js';
|
|
12
|
+
export * from './config.js';
|
|
13
|
+
export * from './config/types.js';
|
|
14
|
+
export * from './config/compound-config-collector.js';
|
|
15
|
+
export * from './config/collectors/config-collector.js';
|
|
16
|
+
export * from './config/collectors/impl/base64-config-collector.js';
|
|
17
|
+
export * from './config/collectors/impl/fallback-config-collector.js';
|
|
18
|
+
export * from './config/collectors/impl/filesystem-config-collector.js';
|
|
19
|
+
export * from './config/sync-rules/sync-collector.js';
|
|
20
|
+
export * from './config/sync-rules/impl/base64-sync-rules-collector.js';
|
|
21
|
+
export * from './config/sync-rules/impl/filesystem-sync-rules-collector.js';
|
|
22
|
+
export * from './config/sync-rules/impl/inline-sync-rules-collector.js';
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
export * from './alerting.js';
|
|
2
|
+
export * from './env.js';
|
|
3
|
+
export * from './memory-tracking.js';
|
|
4
|
+
export * from './migration_lib.js';
|
|
5
|
+
export * from './Mutex.js';
|
|
6
|
+
export * from './PgManager.js';
|
|
7
|
+
export * from './pgwire_utils.js';
|
|
8
|
+
export * from './populate_test_data.js';
|
|
9
|
+
export * from './protocol-types.js';
|
|
10
|
+
export * from './secs.js';
|
|
11
|
+
export * from './utils.js';
|
|
12
|
+
export * from './config.js';
|
|
13
|
+
export * from './config/types.js';
|
|
14
|
+
export * from './config/compound-config-collector.js';
|
|
15
|
+
export * from './config/collectors/config-collector.js';
|
|
16
|
+
export * from './config/collectors/impl/base64-config-collector.js';
|
|
17
|
+
export * from './config/collectors/impl/fallback-config-collector.js';
|
|
18
|
+
export * from './config/collectors/impl/filesystem-config-collector.js';
|
|
19
|
+
export * from './config/sync-rules/sync-collector.js';
|
|
20
|
+
export * from './config/sync-rules/impl/base64-sync-rules-collector.js';
|
|
21
|
+
export * from './config/sync-rules/impl/filesystem-sync-rules-collector.js';
|
|
22
|
+
export * from './config/sync-rules/impl/inline-sync-rules-collector.js';
|
|
23
|
+
//# sourceMappingURL=util-index.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"util-index.js","sourceRoot":"","sources":["../../src/util/util-index.ts"],"names":[],"mappings":"AAAA,cAAc,eAAe,CAAC;AAC9B,cAAc,UAAU,CAAC;AACzB,cAAc,sBAAsB,CAAC;AACrC,cAAc,oBAAoB,CAAC;AACnC,cAAc,YAAY,CAAC;AAC3B,cAAc,gBAAgB,CAAC;AAC/B,cAAc,mBAAmB,CAAC;AAClC,cAAc,yBAAyB,CAAC;AACxC,cAAc,qBAAqB,CAAC;AACpC,cAAc,WAAW,CAAC;AAC1B,cAAc,YAAY,CAAC;AAE3B,cAAc,aAAa,CAAC;AAC5B,cAAc,mBAAmB,CAAC;AAClC,cAAc,uCAAuC,CAAC;AAEtD,cAAc,yCAAyC,CAAC;AACxD,cAAc,qDAAqD,CAAC;AACpE,cAAc,uDAAuD,CAAC;AACtE,cAAc,yDAAyD,CAAC;AAExE,cAAc,uCAAuC,CAAC;AACtD,cAAc,yDAAyD,CAAC;AACxE,cAAc,6DAA6D,CAAC;AAC5E,cAAc,yDAAyD,CAAC"}
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
import * as pgwire from '@powersync/service-jpgwire';
|
|
2
|
+
import * as storage from '../storage/storage-index.js';
|
|
3
|
+
import { BucketChecksum, OpId } from './protocol-types.js';
|
|
4
|
+
export declare function hashData(type: string, id: string, data: string): number;
|
|
5
|
+
export declare function hashDelete(sourceKey: string): number;
|
|
6
|
+
export declare function timestampToOpId(ts: bigint): OpId;
|
|
7
|
+
export declare function checksumsDiff(previous: BucketChecksum[], current: BucketChecksum[]): {
|
|
8
|
+
updated_buckets: BucketChecksum[];
|
|
9
|
+
removed_buckets: string[];
|
|
10
|
+
};
|
|
11
|
+
export declare function getClientCheckpoint(db: pgwire.PgClient, bucketStorage: storage.BucketStorageFactory, options?: {
|
|
12
|
+
timeout?: number;
|
|
13
|
+
}): Promise<OpId>;
|
|
14
|
+
export declare function createWriteCheckpoint(db: pgwire.PgClient, bucketStorage: storage.BucketStorageFactory, user_id: string): Promise<bigint>;
|
|
@@ -0,0 +1,75 @@
|
|
|
1
|
+
import crypto from 'crypto';
|
|
2
|
+
import { pgwireRows } from '@powersync/service-jpgwire';
|
|
3
|
+
import * as micro from '@journeyapps-platform/micro';
|
|
4
|
+
import { retriedQuery } from './pgwire_utils.js';
|
|
5
|
+
export function hashData(type, id, data) {
|
|
6
|
+
const hash = crypto.createHash('sha256');
|
|
7
|
+
hash.update(`put.${type}.${id}.${data}`);
|
|
8
|
+
const buffer = hash.digest();
|
|
9
|
+
return buffer.readUInt32LE(0);
|
|
10
|
+
}
|
|
11
|
+
export function hashDelete(sourceKey) {
|
|
12
|
+
const hash = crypto.createHash('sha256');
|
|
13
|
+
hash.update(`delete.${sourceKey}`);
|
|
14
|
+
const buffer = hash.digest();
|
|
15
|
+
return buffer.readUInt32LE(0);
|
|
16
|
+
}
|
|
17
|
+
export function timestampToOpId(ts) {
|
|
18
|
+
// Dynamic values are passed in in some cases, so we make extra sure that the
|
|
19
|
+
// number is a bigint and not number or Long.
|
|
20
|
+
if (typeof ts != 'bigint') {
|
|
21
|
+
throw new Error(`bigint expected, got: ${ts} (${typeof ts})`);
|
|
22
|
+
}
|
|
23
|
+
return ts.toString(10);
|
|
24
|
+
}
|
|
25
|
+
export function checksumsDiff(previous, current) {
|
|
26
|
+
const updated_buckets = [];
|
|
27
|
+
const previousBuckets = new Map();
|
|
28
|
+
for (let checksum of previous) {
|
|
29
|
+
previousBuckets.set(checksum.bucket, checksum);
|
|
30
|
+
}
|
|
31
|
+
for (let checksum of current) {
|
|
32
|
+
if (!previousBuckets.has(checksum.bucket)) {
|
|
33
|
+
updated_buckets.push(checksum);
|
|
34
|
+
}
|
|
35
|
+
else {
|
|
36
|
+
const p = previousBuckets.get(checksum.bucket);
|
|
37
|
+
if (p?.checksum != checksum.checksum || p?.count != checksum.count) {
|
|
38
|
+
updated_buckets.push(checksum);
|
|
39
|
+
}
|
|
40
|
+
previousBuckets.delete(checksum.bucket);
|
|
41
|
+
}
|
|
42
|
+
}
|
|
43
|
+
const removed_buckets = [...previousBuckets.keys()];
|
|
44
|
+
return {
|
|
45
|
+
updated_buckets,
|
|
46
|
+
removed_buckets
|
|
47
|
+
};
|
|
48
|
+
}
|
|
49
|
+
export async function getClientCheckpoint(db, bucketStorage, options) {
|
|
50
|
+
const start = Date.now();
|
|
51
|
+
const [{ lsn }] = pgwireRows(await db.query(`SELECT pg_logical_emit_message(false, 'powersync', 'ping') as lsn`));
|
|
52
|
+
// This old API needs a persisted checkpoint id.
|
|
53
|
+
// Since we don't use LSNs anymore, the only way to get that is to wait.
|
|
54
|
+
const timeout = options?.timeout ?? 50000;
|
|
55
|
+
micro.logger.info(`Waiting for LSN checkpoint: ${lsn}`);
|
|
56
|
+
while (Date.now() - start < timeout) {
|
|
57
|
+
const cp = await bucketStorage.getActiveCheckpoint();
|
|
58
|
+
if (!cp.hasSyncRules()) {
|
|
59
|
+
throw new Error('No sync rules available');
|
|
60
|
+
}
|
|
61
|
+
if (cp.lsn >= lsn) {
|
|
62
|
+
micro.logger.info(`Got write checkpoint: ${lsn} : ${cp.checkpoint}`);
|
|
63
|
+
return cp.checkpoint;
|
|
64
|
+
}
|
|
65
|
+
await new Promise((resolve) => setTimeout(resolve, 30));
|
|
66
|
+
}
|
|
67
|
+
throw new Error('Timeout while waiting for checkpoint');
|
|
68
|
+
}
|
|
69
|
+
export async function createWriteCheckpoint(db, bucketStorage, user_id) {
|
|
70
|
+
const [{ lsn }] = pgwireRows(await retriedQuery(db, `SELECT pg_logical_emit_message(false, 'powersync', 'ping') as lsn`));
|
|
71
|
+
const id = await bucketStorage.createWriteCheckpoint(user_id, { '1': lsn });
|
|
72
|
+
micro.logger.info(`Write checkpoint 2: ${JSON.stringify({ lsn, id: String(id) })}`);
|
|
73
|
+
return id;
|
|
74
|
+
}
|
|
75
|
+
//# sourceMappingURL=utils.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"utils.js","sourceRoot":"","sources":["../../src/util/utils.ts"],"names":[],"mappings":"AAAA,OAAO,MAAM,MAAM,QAAQ,CAAC;AAE5B,OAAO,EAAE,UAAU,EAAE,MAAM,4BAA4B,CAAC;AACxD,OAAO,KAAK,KAAK,MAAM,6BAA6B,CAAC;AAIrD,OAAO,EAAE,YAAY,EAAE,MAAM,mBAAmB,CAAC;AAEjD,MAAM,UAAU,QAAQ,CAAC,IAAY,EAAE,EAAU,EAAE,IAAY;IAC7D,MAAM,IAAI,GAAG,MAAM,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC;IACzC,IAAI,CAAC,MAAM,CAAC,OAAO,IAAI,IAAI,EAAE,IAAI,IAAI,EAAE,CAAC,CAAC;IACzC,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC;IAC7B,OAAO,MAAM,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;AAChC,CAAC;AAED,MAAM,UAAU,UAAU,CAAC,SAAiB;IAC1C,MAAM,IAAI,GAAG,MAAM,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC;IACzC,IAAI,CAAC,MAAM,CAAC,UAAU,SAAS,EAAE,CAAC,CAAC;IACnC,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC;IAC7B,OAAO,MAAM,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;AAChC,CAAC;AAED,MAAM,UAAU,eAAe,CAAC,EAAU;IACxC,6EAA6E;IAC7E,6CAA6C;IAC7C,IAAI,OAAO,EAAE,IAAI,QAAQ,EAAE,CAAC;QAC1B,MAAM,IAAI,KAAK,CAAC,yBAAyB,EAAE,KAAK,OAAO,EAAE,GAAG,CAAC,CAAC;IAChE,CAAC;IACD,OAAO,EAAE,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC;AACzB,CAAC;AAED,MAAM,UAAU,aAAa,CAAC,QAA0B,EAAE,OAAyB;IACjF,MAAM,eAAe,GAAqB,EAAE,CAAC;IAE7C,MAAM,eAAe,GAAG,IAAI,GAAG,EAA0B,CAAC;IAC1D,KAAK,IAAI,QAAQ,IAAI,QAAQ,EAAE,CAAC;QAC9B,eAAe,CAAC,GAAG,CAAC,QAAQ,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;IACjD,CAAC;IACD,KAAK,IAAI,QAAQ,IAAI,OAAO,EAAE,CAAC;QAC7B,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC;YAC1C,eAAe,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QACjC,CAAC;aAAM,CAAC;YACN,MAAM,CAAC,GAAG,eAAe,CAAC,GAAG,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;YAC/C,IAAI,CAAC,EAAE,QAAQ,IAAI,QAAQ,CAAC,QAAQ,IAAI,CAAC,EAAE,KAAK,IAAI,QAAQ,CAAC,KAAK,EAAE,CAAC;gBACnE,eAAe,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;YACjC,CAAC;YACD,eAAe,CAAC,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;QAC1C,CAAC;IACH,CAAC;IAED,MAAM,eAAe,GAAa,CAAC,GAAG,eAAe,CAAC,IAAI,EAAE,CAAC,CAAC;IAC9D,OAAO;QACL,eAAe;QACf,eAAe;KAChB,CAAC;AACJ,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,mBAAmB,CACvC,EAAmB,EACnB,aAA2C,EAC3C,OAA8B;IAE9B,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAEzB,MAAM,CAAC,EAAE,GAAG,EAAE,CAAC,GAAG,UAAU,CAAC,MAAM,EAAE,CAAC,KAAK,CAAC,mEAAmE,CAAC,CAAC,CAAC;IAElH,gDAAgD;IAChD,wEAAwE;IAExE,MAAM,OAAO,GAAG,OAAO,EAAE,OAAO,IAAI,KAAM,CAAC;IAE3C,KAAK,CAAC,MAAM,CAAC,IAAI,CAAC,+BAA+B,GAAG,EAAE,CAAC,CAAC;IACxD,OAAO,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK,GAAG,OAAO,EAAE,CAAC;QACpC,MAAM,EAAE,GAAG,MAAM,aAAa,CAAC,mBAAmB,EAAE,CAAC;QACrD,IAAI,CAAC,EAAE,CAAC,YAAY,EAAE,EAAE,CAAC;YACvB,MAAM,IAAI,KAAK,CAAC,yBAAyB,CAAC,CAAC;QAC7C,CAAC;QACD,IAAI,EAAE,CAAC,GAAG,IAAI,GAAG,EAAE,CAAC;YAClB,KAAK,CAAC,MAAM,CAAC,IAAI,CAAC,yBAAyB,GAAG,MAAM,EAAE,CAAC,UAAU,EAAE,CAAC,CAAC;YACrE,OAAO,EAAE,CAAC,UAAU,CAAC;QACvB,CAAC;QAED,MAAM,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,CAAC;IAC1D,CAAC;IAED,MAAM,IAAI,KAAK,CAAC,sCAAsC,CAAC,CAAC;AAC1D,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,qBAAqB,CACzC,EAAmB,EACnB,aAA2C,EAC3C,OAAe;IAEf,MAAM,CAAC,EAAE,GAAG,EAAE,CAAC,GAAG,UAAU,CAC1B,MAAM,YAAY,CAAC,EAAE,EAAE,mEAAmE,CAAC,CAC5F,CAAC;IAEF,MAAM,EAAE,GAAG,MAAM,aAAa,CAAC,qBAAqB,CAAC,OAAO,EAAE,EAAE,GAAG,EAAE,GAAG,EAAE,CAAC,CAAC;IAC5E,KAAK,CAAC,MAAM,CAAC,IAAI,CAAC,uBAAuB,IAAI,CAAC,SAAS,CAAC,EAAE,GAAG,EAAE,EAAE,EAAE,MAAM,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC;IACpF,OAAO,EAAE,CAAC;AACZ,CAAC"}
|