@rocicorp/zero 1.11.0-canary.0 → 1.11.0-canary.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/out/analyze-query/src/bin-analyze.js +1 -1
- package/out/analyze-query/src/bin-analyze.js.map +1 -1
- package/out/replicache/src/btree/read.d.ts +12 -1
- package/out/replicache/src/btree/read.d.ts.map +1 -1
- package/out/replicache/src/btree/read.js +24 -12
- package/out/replicache/src/btree/read.js.map +1 -1
- package/out/replicache/src/dag/gc.d.ts.map +1 -1
- package/out/replicache/src/dag/gc.js +2 -1
- package/out/replicache/src/dag/gc.js.map +1 -1
- package/out/replicache/src/dag/invalid-ref-count-error.d.ts +12 -0
- package/out/replicache/src/dag/invalid-ref-count-error.d.ts.map +1 -0
- package/out/replicache/src/dag/invalid-ref-count-error.js +19 -0
- package/out/replicache/src/dag/invalid-ref-count-error.js.map +1 -0
- package/out/replicache/src/dag/store-impl.d.ts +12 -2
- package/out/replicache/src/dag/store-impl.d.ts.map +1 -1
- package/out/replicache/src/dag/store-impl.js +22 -8
- package/out/replicache/src/dag/store-impl.js.map +1 -1
- package/out/replicache/src/db/rebase.d.ts +14 -2
- package/out/replicache/src/db/rebase.d.ts.map +1 -1
- package/out/replicache/src/db/rebase.js +27 -5
- package/out/replicache/src/db/rebase.js.map +1 -1
- package/out/replicache/src/db/write.js.map +1 -1
- package/out/replicache/src/kv/expo-sqlite/store.d.ts.map +1 -1
- package/out/replicache/src/kv/expo-sqlite/store.js +28 -5
- package/out/replicache/src/kv/expo-sqlite/store.js.map +1 -1
- package/out/replicache/src/kv/idb-store-with-mem-fallback.d.ts +15 -5
- package/out/replicache/src/kv/idb-store-with-mem-fallback.d.ts.map +1 -1
- package/out/replicache/src/kv/idb-store-with-mem-fallback.js +25 -25
- package/out/replicache/src/kv/idb-store-with-mem-fallback.js.map +1 -1
- package/out/replicache/src/kv/idb-store.d.ts +9 -0
- package/out/replicache/src/kv/idb-store.d.ts.map +1 -1
- package/out/replicache/src/kv/idb-store.js +14 -2
- package/out/replicache/src/kv/idb-store.js.map +1 -1
- package/out/replicache/src/kv/mem-store.d.ts +1 -0
- package/out/replicache/src/kv/mem-store.d.ts.map +1 -1
- package/out/replicache/src/kv/mem-store.js +3 -0
- package/out/replicache/src/kv/mem-store.js.map +1 -1
- package/out/replicache/src/kv/op-sqlite/store.d.ts.map +1 -1
- package/out/replicache/src/kv/op-sqlite/store.js +1 -1
- package/out/replicache/src/kv/op-sqlite/store.js.map +1 -1
- package/out/replicache/src/kv/sqlite-store.d.ts +13 -1
- package/out/replicache/src/kv/sqlite-store.d.ts.map +1 -1
- package/out/replicache/src/kv/sqlite-store.js +49 -4
- package/out/replicache/src/kv/sqlite-store.js.map +1 -1
- package/out/replicache/src/kv/store.d.ts +6 -0
- package/out/replicache/src/kv/store.d.ts.map +1 -1
- package/out/replicache/src/persist/collect-idb-databases.d.ts +10 -5
- package/out/replicache/src/persist/collect-idb-databases.d.ts.map +1 -1
- package/out/replicache/src/persist/collect-idb-databases.js +16 -9
- package/out/replicache/src/persist/collect-idb-databases.js.map +1 -1
- package/out/replicache/src/persist/persist.d.ts.map +1 -1
- package/out/replicache/src/persist/persist.js +12 -5
- package/out/replicache/src/persist/persist.js.map +1 -1
- package/out/replicache/src/persist/refresh.js +6 -2
- package/out/replicache/src/persist/refresh.js.map +1 -1
- package/out/replicache/src/replicache-impl.d.ts +14 -3
- package/out/replicache/src/replicache-impl.d.ts.map +1 -1
- package/out/replicache/src/replicache-impl.js +118 -11
- package/out/replicache/src/replicache-impl.js.map +1 -1
- package/out/replicache/src/replicache-options.d.ts +9 -0
- package/out/replicache/src/replicache-options.d.ts.map +1 -1
- package/out/replicache/src/transactions.d.ts.map +1 -1
- package/out/replicache/src/transactions.js +1 -1
- package/out/replicache/src/transactions.js.map +1 -1
- package/out/shared/src/error.d.ts +11 -0
- package/out/shared/src/error.d.ts.map +1 -1
- package/out/shared/src/error.js +35 -1
- package/out/shared/src/error.js.map +1 -1
- package/out/shared/src/iterables.d.ts +2 -0
- package/out/shared/src/iterables.d.ts.map +1 -1
- package/out/shared/src/iterables.js +18 -1
- package/out/shared/src/iterables.js.map +1 -1
- package/out/shared/src/map.d.ts +7 -2
- package/out/shared/src/map.d.ts.map +1 -1
- package/out/shared/src/map.js.map +1 -1
- package/out/shared/src/ring-buffer.d.ts +8 -0
- package/out/shared/src/ring-buffer.d.ts.map +1 -1
- package/out/shared/src/ring-buffer.js +22 -1
- package/out/shared/src/ring-buffer.js.map +1 -1
- package/out/zero/package.js +3 -4
- package/out/zero/package.js.map +1 -1
- package/out/zero-cache/src/config/normalize.d.ts.map +1 -1
- package/out/zero-cache/src/config/normalize.js +25 -1
- package/out/zero-cache/src/config/normalize.js.map +1 -1
- package/out/zero-cache/src/config/zero-config.d.ts +48 -2
- package/out/zero-cache/src/config/zero-config.d.ts.map +1 -1
- package/out/zero-cache/src/config/zero-config.js +171 -29
- package/out/zero-cache/src/config/zero-config.js.map +1 -1
- package/out/zero-cache/src/server/change-streamer.d.ts.map +1 -1
- package/out/zero-cache/src/server/change-streamer.js +25 -9
- package/out/zero-cache/src/server/change-streamer.js.map +1 -1
- package/out/zero-cache/src/server/inspector-delegate.d.ts +18 -2
- package/out/zero-cache/src/server/inspector-delegate.d.ts.map +1 -1
- package/out/zero-cache/src/server/inspector-delegate.js +20 -7
- package/out/zero-cache/src/server/inspector-delegate.js.map +1 -1
- package/out/zero-cache/src/server/main.d.ts.map +1 -1
- package/out/zero-cache/src/server/main.js +8 -1
- package/out/zero-cache/src/server/main.js.map +1 -1
- package/out/zero-cache/src/server/mutator.js +2 -0
- package/out/zero-cache/src/server/mutator.js.map +1 -1
- package/out/zero-cache/src/server/reaper.d.ts.map +1 -1
- package/out/zero-cache/src/server/reaper.js +2 -0
- package/out/zero-cache/src/server/reaper.js.map +1 -1
- package/out/zero-cache/src/server/replicator.d.ts.map +1 -1
- package/out/zero-cache/src/server/replicator.js +8 -2
- package/out/zero-cache/src/server/replicator.js.map +1 -1
- package/out/zero-cache/src/server/runner/run-worker.js +1 -1
- package/out/zero-cache/src/server/runner/zero-dispatcher.d.ts.map +1 -1
- package/out/zero-cache/src/server/runner/zero-dispatcher.js +3 -0
- package/out/zero-cache/src/server/runner/zero-dispatcher.js.map +1 -1
- package/out/zero-cache/src/server/syncer.d.ts.map +1 -1
- package/out/zero-cache/src/server/syncer.js +2 -0
- package/out/zero-cache/src/server/syncer.js.map +1 -1
- package/out/zero-cache/src/services/analyze.d.ts.map +1 -1
- package/out/zero-cache/src/services/analyze.js +3 -2
- package/out/zero-cache/src/services/analyze.js.map +1 -1
- package/out/zero-cache/src/services/change-source/common/backfill-manager.d.ts.map +1 -1
- package/out/zero-cache/src/services/change-source/common/backfill-manager.js +14 -1
- package/out/zero-cache/src/services/change-source/common/backfill-manager.js.map +1 -1
- package/out/zero-cache/src/services/change-source/common/change-stream-multiplexer.d.ts.map +1 -1
- package/out/zero-cache/src/services/change-source/common/change-stream-multiplexer.js +15 -3
- package/out/zero-cache/src/services/change-source/common/change-stream-multiplexer.js.map +1 -1
- package/out/zero-cache/src/services/change-source/common/replica-schema.d.ts.map +1 -1
- package/out/zero-cache/src/services/change-source/common/replica-schema.js +2 -39
- package/out/zero-cache/src/services/change-source/common/replica-schema.js.map +1 -1
- package/out/zero-cache/src/services/change-source/pg/backfill-resume.d.ts +156 -0
- package/out/zero-cache/src/services/change-source/pg/backfill-resume.d.ts.map +1 -0
- package/out/zero-cache/src/services/change-source/pg/backfill-resume.js +15 -0
- package/out/zero-cache/src/services/change-source/pg/backfill-resume.js.map +1 -0
- package/out/zero-cache/src/services/change-source/pg/change-source-init.d.ts +25 -0
- package/out/zero-cache/src/services/change-source/pg/change-source-init.d.ts.map +1 -0
- package/out/zero-cache/src/services/change-source/pg/change-source-init.js +202 -0
- package/out/zero-cache/src/services/change-source/pg/change-source-init.js.map +1 -0
- package/out/zero-cache/src/services/change-source/pg/change-source.d.ts +14 -16
- package/out/zero-cache/src/services/change-source/pg/change-source.d.ts.map +1 -1
- package/out/zero-cache/src/services/change-source/pg/change-source.js +64 -143
- package/out/zero-cache/src/services/change-source/pg/change-source.js.map +1 -1
- package/out/zero-cache/src/services/change-source/pg/initial-sync.d.ts +28 -2
- package/out/zero-cache/src/services/change-source/pg/initial-sync.d.ts.map +1 -1
- package/out/zero-cache/src/services/change-source/pg/initial-sync.js +44 -21
- package/out/zero-cache/src/services/change-source/pg/initial-sync.js.map +1 -1
- package/out/zero-cache/src/services/change-source/pg/replication-slot-cleanup-monitor.d.ts +16 -0
- package/out/zero-cache/src/services/change-source/pg/replication-slot-cleanup-monitor.d.ts.map +1 -0
- package/out/zero-cache/src/services/change-source/pg/replication-slot-cleanup-monitor.js +113 -0
- package/out/zero-cache/src/services/change-source/pg/replication-slot-cleanup-monitor.js.map +1 -0
- package/out/zero-cache/src/services/change-source/pg/replication-slots.d.ts +21 -15
- package/out/zero-cache/src/services/change-source/pg/replication-slots.d.ts.map +1 -1
- package/out/zero-cache/src/services/change-source/pg/replication-slots.js +86 -23
- package/out/zero-cache/src/services/change-source/pg/replication-slots.js.map +1 -1
- package/out/zero-cache/src/services/change-source/pg/schema/index-predicate.d.ts.map +1 -1
- package/out/zero-cache/src/services/change-source/pg/schema/index-predicate.js +68 -8
- package/out/zero-cache/src/services/change-source/pg/schema/index-predicate.js.map +1 -1
- package/out/zero-cache/src/services/change-source/pg/schema/init.d.ts.map +1 -1
- package/out/zero-cache/src/services/change-source/pg/schema/init.js +32 -1
- package/out/zero-cache/src/services/change-source/pg/schema/init.js.map +1 -1
- package/out/zero-cache/src/services/change-source/pg/schema/replica-stage-enum.d.ts +10 -0
- package/out/zero-cache/src/services/change-source/pg/schema/replica-stage-enum.d.ts.map +1 -0
- package/out/zero-cache/src/services/change-source/pg/schema/replica-stage-enum.js +6 -0
- package/out/zero-cache/src/services/change-source/pg/schema/replica-stage-enum.js.map +1 -0
- package/out/zero-cache/src/services/change-source/pg/schema/shard.d.ts +50 -27
- package/out/zero-cache/src/services/change-source/pg/schema/shard.d.ts.map +1 -1
- package/out/zero-cache/src/services/change-source/pg/schema/shard.js +100 -34
- package/out/zero-cache/src/services/change-source/pg/schema/shard.js.map +1 -1
- package/out/zero-cache/src/services/change-streamer/backup-monitor.d.ts.map +1 -1
- package/out/zero-cache/src/services/change-streamer/backup-monitor.js +58 -7
- package/out/zero-cache/src/services/change-streamer/backup-monitor.js.map +1 -1
- package/out/zero-cache/src/services/change-streamer/broadcast.d.ts +8 -2
- package/out/zero-cache/src/services/change-streamer/broadcast.d.ts.map +1 -1
- package/out/zero-cache/src/services/change-streamer/broadcast.js +24 -4
- package/out/zero-cache/src/services/change-streamer/broadcast.js.map +1 -1
- package/out/zero-cache/src/services/change-streamer/change-log-codec.d.ts +19 -4
- package/out/zero-cache/src/services/change-streamer/change-log-codec.d.ts.map +1 -1
- package/out/zero-cache/src/services/change-streamer/change-log-codec.js +28 -8
- package/out/zero-cache/src/services/change-streamer/change-log-codec.js.map +1 -1
- package/out/zero-cache/src/services/change-streamer/change-streamer-http.d.ts +5 -2
- package/out/zero-cache/src/services/change-streamer/change-streamer-http.d.ts.map +1 -1
- package/out/zero-cache/src/services/change-streamer/change-streamer-http.js +21 -14
- package/out/zero-cache/src/services/change-streamer/change-streamer-http.js.map +1 -1
- package/out/zero-cache/src/services/change-streamer/change-streamer-service.d.ts +6 -0
- package/out/zero-cache/src/services/change-streamer/change-streamer-service.d.ts.map +1 -1
- package/out/zero-cache/src/services/change-streamer/change-streamer-service.js +97 -63
- package/out/zero-cache/src/services/change-streamer/change-streamer-service.js.map +1 -1
- package/out/zero-cache/src/services/change-streamer/change-streamer.d.ts +12 -8
- package/out/zero-cache/src/services/change-streamer/change-streamer.d.ts.map +1 -1
- package/out/zero-cache/src/services/change-streamer/change-streamer.js.map +1 -1
- package/out/zero-cache/src/services/change-streamer/forwarder.d.ts +3 -3
- package/out/zero-cache/src/services/change-streamer/forwarder.d.ts.map +1 -1
- package/out/zero-cache/src/services/change-streamer/forwarder.js +36 -5
- package/out/zero-cache/src/services/change-streamer/forwarder.js.map +1 -1
- package/out/zero-cache/src/services/change-streamer/schema/tables.d.ts +1 -1
- package/out/zero-cache/src/services/change-streamer/schema/tables.d.ts.map +1 -1
- package/out/zero-cache/src/services/change-streamer/schema/tables.js +4 -4
- package/out/zero-cache/src/services/change-streamer/schema/tables.js.map +1 -1
- package/out/zero-cache/src/services/change-streamer/snapshot.d.ts +3 -1
- package/out/zero-cache/src/services/change-streamer/snapshot.d.ts.map +1 -1
- package/out/zero-cache/src/services/change-streamer/snapshot.js +54 -20
- package/out/zero-cache/src/services/change-streamer/snapshot.js.map +1 -1
- package/out/zero-cache/src/services/change-streamer/sqlite-change-log-writer.d.ts +6 -1
- package/out/zero-cache/src/services/change-streamer/sqlite-change-log-writer.d.ts.map +1 -1
- package/out/zero-cache/src/services/change-streamer/sqlite-change-log-writer.js +27 -9
- package/out/zero-cache/src/services/change-streamer/sqlite-change-log-writer.js.map +1 -1
- package/out/zero-cache/src/services/change-streamer/storer.d.ts +2 -2
- package/out/zero-cache/src/services/change-streamer/storer.d.ts.map +1 -1
- package/out/zero-cache/src/services/change-streamer/storer.js +8 -5
- package/out/zero-cache/src/services/change-streamer/storer.js.map +1 -1
- package/out/zero-cache/src/services/change-streamer/subscriber.d.ts +4 -1
- package/out/zero-cache/src/services/change-streamer/subscriber.d.ts.map +1 -1
- package/out/zero-cache/src/services/change-streamer/subscriber.js +42 -0
- package/out/zero-cache/src/services/change-streamer/subscriber.js.map +1 -1
- package/out/zero-cache/src/services/change-streamer/upstream-acker.d.ts +17 -1
- package/out/zero-cache/src/services/change-streamer/upstream-acker.d.ts.map +1 -1
- package/out/zero-cache/src/services/change-streamer/upstream-acker.js +19 -3
- package/out/zero-cache/src/services/change-streamer/upstream-acker.js.map +1 -1
- package/out/zero-cache/src/services/change-streamer/vfs-watermark-poller.d.ts.map +1 -1
- package/out/zero-cache/src/services/change-streamer/vfs-watermark-poller.js +1 -0
- package/out/zero-cache/src/services/change-streamer/vfs-watermark-poller.js.map +1 -1
- package/out/zero-cache/src/services/life-cycle.d.ts.map +1 -1
- package/out/zero-cache/src/services/life-cycle.js +3 -1
- package/out/zero-cache/src/services/life-cycle.js.map +1 -1
- package/out/zero-cache/src/services/litestream/commands.d.ts.map +1 -1
- package/out/zero-cache/src/services/litestream/commands.js +8 -3
- package/out/zero-cache/src/services/litestream/commands.js.map +1 -1
- package/out/zero-cache/src/services/litestream/config-v5.yml +15 -1
- package/out/zero-cache/src/services/litestream/litestream-checkpointer.js +131 -0
- package/out/zero-cache/src/services/litestream/litestream-checkpointer.js.map +1 -0
- package/out/zero-cache/src/services/litestream/litestream-controller.d.ts +70 -0
- package/out/zero-cache/src/services/litestream/litestream-controller.d.ts.map +1 -0
- package/out/zero-cache/src/services/litestream/litestream-controller.js +101 -0
- package/out/zero-cache/src/services/litestream/litestream-controller.js.map +1 -0
- package/out/zero-cache/src/services/mutagen/mutagen.d.ts +5 -0
- package/out/zero-cache/src/services/mutagen/mutagen.d.ts.map +1 -1
- package/out/zero-cache/src/services/mutagen/mutagen.js +7 -1
- package/out/zero-cache/src/services/mutagen/mutagen.js.map +1 -1
- package/out/zero-cache/src/services/mutagen/pusher.js +2 -1
- package/out/zero-cache/src/services/mutagen/pusher.js.map +1 -1
- package/out/zero-cache/src/services/profz.d.ts +7 -0
- package/out/zero-cache/src/services/profz.d.ts.map +1 -0
- package/out/zero-cache/src/services/profz.js +105 -0
- package/out/zero-cache/src/services/profz.js.map +1 -0
- package/out/zero-cache/src/services/replicator/change-log-db.js.map +1 -1
- package/out/zero-cache/src/services/replicator/change-log-stream-writer.d.ts +5 -4
- package/out/zero-cache/src/services/replicator/change-log-stream-writer.d.ts.map +1 -1
- package/out/zero-cache/src/services/replicator/change-log-stream-writer.js +20 -22
- package/out/zero-cache/src/services/replicator/change-log-stream-writer.js.map +1 -1
- package/out/zero-cache/src/services/replicator/change-processor.js +1 -0
- package/out/zero-cache/src/services/replicator/change-processor.js.map +1 -1
- package/out/zero-cache/src/services/replicator/incremental-sync.d.ts +1 -1
- package/out/zero-cache/src/services/replicator/incremental-sync.d.ts.map +1 -1
- package/out/zero-cache/src/services/replicator/incremental-sync.js +58 -17
- package/out/zero-cache/src/services/replicator/incremental-sync.js.map +1 -1
- package/out/zero-cache/src/services/replicator/replication-status.d.ts +35 -1
- package/out/zero-cache/src/services/replicator/replication-status.d.ts.map +1 -1
- package/out/zero-cache/src/services/replicator/replication-status.js +96 -6
- package/out/zero-cache/src/services/replicator/replication-status.js.map +1 -1
- package/out/zero-cache/src/services/replicator/replicator.js.map +1 -1
- package/out/zero-cache/src/services/replicator/sqlite-change-log-observability.d.ts +6 -6
- package/out/zero-cache/src/services/replicator/sqlite-change-log-observability.d.ts.map +1 -1
- package/out/zero-cache/src/services/replicator/sqlite-change-log-observability.js +19 -14
- package/out/zero-cache/src/services/replicator/sqlite-change-log-observability.js.map +1 -1
- package/out/zero-cache/src/services/replicator/write-worker-client.d.ts +31 -6
- package/out/zero-cache/src/services/replicator/write-worker-client.d.ts.map +1 -1
- package/out/zero-cache/src/services/replicator/write-worker-client.js +5 -4
- package/out/zero-cache/src/services/replicator/write-worker-client.js.map +1 -1
- package/out/zero-cache/src/services/replicator/write-worker.js +30 -5
- package/out/zero-cache/src/services/replicator/write-worker.js.map +1 -1
- package/out/zero-cache/src/services/run-ast.d.ts +12 -0
- package/out/zero-cache/src/services/run-ast.d.ts.map +1 -1
- package/out/zero-cache/src/services/run-ast.js +26 -14
- package/out/zero-cache/src/services/run-ast.js.map +1 -1
- package/out/zero-cache/src/services/runner.d.ts.map +1 -1
- package/out/zero-cache/src/services/runner.js +1 -1
- package/out/zero-cache/src/services/runner.js.map +1 -1
- package/out/zero-cache/src/services/view-syncer/client-handler.d.ts +1 -1
- package/out/zero-cache/src/services/view-syncer/client-handler.d.ts.map +1 -1
- package/out/zero-cache/src/services/view-syncer/client-handler.js +36 -31
- package/out/zero-cache/src/services/view-syncer/client-handler.js.map +1 -1
- package/out/zero-cache/src/services/view-syncer/cvr.d.ts +45 -0
- package/out/zero-cache/src/services/view-syncer/cvr.d.ts.map +1 -1
- package/out/zero-cache/src/services/view-syncer/cvr.js +133 -1
- package/out/zero-cache/src/services/view-syncer/cvr.js.map +1 -1
- package/out/zero-cache/src/services/view-syncer/drain-coordinator.d.ts +14 -1
- package/out/zero-cache/src/services/view-syncer/drain-coordinator.d.ts.map +1 -1
- package/out/zero-cache/src/services/view-syncer/drain-coordinator.js +30 -4
- package/out/zero-cache/src/services/view-syncer/drain-coordinator.js.map +1 -1
- package/out/zero-cache/src/services/view-syncer/hydration-budget.d.ts +33 -0
- package/out/zero-cache/src/services/view-syncer/hydration-budget.d.ts.map +1 -0
- package/out/zero-cache/src/services/view-syncer/hydration-budget.js +62 -0
- package/out/zero-cache/src/services/view-syncer/hydration-budget.js.map +1 -0
- package/out/zero-cache/src/services/view-syncer/hydration-circuit-breaker.d.ts +51 -0
- package/out/zero-cache/src/services/view-syncer/hydration-circuit-breaker.d.ts.map +1 -0
- package/out/zero-cache/src/services/view-syncer/hydration-circuit-breaker.js +78 -0
- package/out/zero-cache/src/services/view-syncer/hydration-circuit-breaker.js.map +1 -0
- package/out/zero-cache/src/services/view-syncer/inspect-handler.d.ts +1 -1
- package/out/zero-cache/src/services/view-syncer/inspect-handler.d.ts.map +1 -1
- package/out/zero-cache/src/services/view-syncer/inspect-handler.js +2 -2
- package/out/zero-cache/src/services/view-syncer/inspect-handler.js.map +1 -1
- package/out/zero-cache/src/services/view-syncer/pipeline-driver.d.ts +2 -1
- package/out/zero-cache/src/services/view-syncer/pipeline-driver.d.ts.map +1 -1
- package/out/zero-cache/src/services/view-syncer/pipeline-driver.js +33 -18
- package/out/zero-cache/src/services/view-syncer/pipeline-driver.js.map +1 -1
- package/out/zero-cache/src/services/view-syncer/snapshotter.d.ts +4 -1
- package/out/zero-cache/src/services/view-syncer/snapshotter.d.ts.map +1 -1
- package/out/zero-cache/src/services/view-syncer/snapshotter.js +9 -3
- package/out/zero-cache/src/services/view-syncer/snapshotter.js.map +1 -1
- package/out/zero-cache/src/services/view-syncer/view-syncer.d.ts +11 -1
- package/out/zero-cache/src/services/view-syncer/view-syncer.d.ts.map +1 -1
- package/out/zero-cache/src/services/view-syncer/view-syncer.js +507 -93
- package/out/zero-cache/src/services/view-syncer/view-syncer.js.map +1 -1
- package/out/zero-cache/src/types/processes.d.ts +32 -0
- package/out/zero-cache/src/types/processes.d.ts.map +1 -1
- package/out/zero-cache/src/types/processes.js +43 -4
- package/out/zero-cache/src/types/processes.js.map +1 -1
- package/out/zero-cache/src/types/profiler.d.ts +12 -0
- package/out/zero-cache/src/types/profiler.d.ts.map +1 -1
- package/out/zero-cache/src/types/profiler.js +46 -1
- package/out/zero-cache/src/types/profiler.js.map +1 -1
- package/out/zero-cache/src/types/streams.d.ts +25 -9
- package/out/zero-cache/src/types/streams.d.ts.map +1 -1
- package/out/zero-cache/src/types/streams.js +118 -34
- package/out/zero-cache/src/types/streams.js.map +1 -1
- package/out/zero-cache/src/types/subscription.d.ts +4 -0
- package/out/zero-cache/src/types/subscription.d.ts.map +1 -1
- package/out/zero-cache/src/types/subscription.js +71 -0
- package/out/zero-cache/src/types/subscription.js.map +1 -1
- package/out/zero-cache/src/types/ws.d.ts.map +1 -1
- package/out/zero-cache/src/types/ws.js +23 -8
- package/out/zero-cache/src/types/ws.js.map +1 -1
- package/out/zero-cache/src/workers/connection.js +2 -2
- package/out/zero-cache/src/workers/replicator.d.ts +1 -0
- package/out/zero-cache/src/workers/replicator.d.ts.map +1 -1
- package/out/zero-cache/src/workers/replicator.js +4 -2
- package/out/zero-cache/src/workers/replicator.js.map +1 -1
- package/out/zero-cache/src/workers/syncer-ws-message-handler.d.ts.map +1 -1
- package/out/zero-cache/src/workers/syncer-ws-message-handler.js +21 -16
- package/out/zero-cache/src/workers/syncer-ws-message-handler.js.map +1 -1
- package/out/zero-cache/src/workers/syncer.d.ts.map +1 -1
- package/out/zero-cache/src/workers/syncer.js +10 -1
- package/out/zero-cache/src/workers/syncer.js.map +1 -1
- package/out/zero-client/src/client/connection-manager.d.ts +20 -0
- package/out/zero-client/src/client/connection-manager.d.ts.map +1 -1
- package/out/zero-client/src/client/connection-manager.js +77 -13
- package/out/zero-client/src/client/connection-manager.js.map +1 -1
- package/out/zero-client/src/client/connection-status-enum.d.ts +2 -0
- package/out/zero-client/src/client/connection-status-enum.d.ts.map +1 -1
- package/out/zero-client/src/client/connection-status-enum.js +3 -1
- package/out/zero-client/src/client/connection-status-enum.js.map +1 -1
- package/out/zero-client/src/client/connection.d.ts +6 -0
- package/out/zero-client/src/client/connection.d.ts.map +1 -1
- package/out/zero-client/src/client/connection.js +2 -1
- package/out/zero-client/src/client/connection.js.map +1 -1
- package/out/zero-client/src/client/context.d.ts +27 -2
- package/out/zero-client/src/client/context.d.ts.map +1 -1
- package/out/zero-client/src/client/context.js +112 -1
- package/out/zero-client/src/client/context.js.map +1 -1
- package/out/zero-client/src/client/custom.js +2 -2
- package/out/zero-client/src/client/custom.js.map +1 -1
- package/out/zero-client/src/client/error.js +1 -1
- package/out/zero-client/src/client/inspector/lazy-inspector.js +2 -2
- package/out/zero-client/src/client/inspector/lazy-inspector.js.map +1 -1
- package/out/zero-client/src/client/ivm-branch.d.ts.map +1 -1
- package/out/zero-client/src/client/ivm-branch.js +20 -3
- package/out/zero-client/src/client/ivm-branch.js.map +1 -1
- package/out/zero-client/src/client/mutator-proxy.d.ts.map +1 -1
- package/out/zero-client/src/client/mutator-proxy.js +2 -1
- package/out/zero-client/src/client/mutator-proxy.js.map +1 -1
- package/out/zero-client/src/client/options.d.ts +3 -0
- package/out/zero-client/src/client/options.d.ts.map +1 -1
- package/out/zero-client/src/client/options.js.map +1 -1
- package/out/zero-client/src/client/query-manager.d.ts.map +1 -1
- package/out/zero-client/src/client/query-manager.js +42 -6
- package/out/zero-client/src/client/query-manager.js.map +1 -1
- package/out/zero-client/src/client/version.js +1 -1
- package/out/zero-client/src/client/zero-rep.d.ts +1 -0
- package/out/zero-client/src/client/zero-rep.d.ts.map +1 -1
- package/out/zero-client/src/client/zero-rep.js +9 -6
- package/out/zero-client/src/client/zero-rep.js.map +1 -1
- package/out/zero-client/src/client/zero.d.ts +18 -3
- package/out/zero-client/src/client/zero.d.ts.map +1 -1
- package/out/zero-client/src/client/zero.js +122 -34
- package/out/zero-client/src/client/zero.js.map +1 -1
- package/out/zero-client/src/types/query-result.d.ts +2 -0
- package/out/zero-client/src/types/query-result.d.ts.map +1 -1
- package/out/zero-events/src/status.d.ts +26 -0
- package/out/zero-events/src/status.d.ts.map +1 -1
- package/out/zero-protocol/src/analyze-query-result.d.ts +5 -5
- package/out/zero-protocol/src/analyze-query-result.d.ts.map +1 -1
- package/out/zero-protocol/src/analyze-query-result.js +1 -1
- package/out/zero-protocol/src/analyze-query-result.js.map +1 -1
- package/out/zero-protocol/src/down.d.ts +4 -4
- package/out/zero-protocol/src/inspect-down.d.ts +6 -6
- package/out/zero-protocol/src/protocol-version.d.ts +1 -1
- package/out/zero-protocol/src/protocol-version.d.ts.map +1 -1
- package/out/zero-protocol/src/protocol-version.js.map +1 -1
- package/out/zero-react/src/use-query.d.ts +5 -4
- package/out/zero-react/src/use-query.d.ts.map +1 -1
- package/out/zero-react/src/use-query.js +86 -9
- package/out/zero-react/src/use-query.js.map +1 -1
- package/out/zero-server/src/adapters/drizzle.d.ts +2 -1
- package/out/zero-server/src/adapters/drizzle.d.ts.map +1 -1
- package/out/zero-server/src/adapters/drizzle.js +12 -7
- package/out/zero-server/src/adapters/drizzle.js.map +1 -1
- package/out/zero-server/src/adapters/kysely.d.ts +2 -1
- package/out/zero-server/src/adapters/kysely.d.ts.map +1 -1
- package/out/zero-server/src/adapters/kysely.js +8 -2
- package/out/zero-server/src/adapters/kysely.js.map +1 -1
- package/out/zero-server/src/adapters/pg.d.ts +1 -0
- package/out/zero-server/src/adapters/pg.d.ts.map +1 -1
- package/out/zero-server/src/adapters/pg.js +8 -2
- package/out/zero-server/src/adapters/pg.js.map +1 -1
- package/out/zero-server/src/adapters/postgresjs.d.ts +1 -0
- package/out/zero-server/src/adapters/postgresjs.d.ts.map +1 -1
- package/out/zero-server/src/adapters/postgresjs.js +7 -1
- package/out/zero-server/src/adapters/postgresjs.js.map +1 -1
- package/out/zero-server/src/adapters/prisma.d.ts +3 -2
- package/out/zero-server/src/adapters/prisma.d.ts.map +1 -1
- package/out/zero-server/src/adapters/prisma.js +8 -2
- package/out/zero-server/src/adapters/prisma.js.map +1 -1
- package/out/zero-server/src/custom.d.ts +5 -1
- package/out/zero-server/src/custom.d.ts.map +1 -1
- package/out/zero-server/src/custom.js +6 -3
- package/out/zero-server/src/custom.js.map +1 -1
- package/out/zero-server/src/mod.d.ts +1 -1
- package/out/zero-server/src/mod.d.ts.map +1 -1
- package/out/zero-server/src/pg-query-executor.d.ts +2 -2
- package/out/zero-server/src/pg-query-executor.d.ts.map +1 -1
- package/out/zero-server/src/pg-query-executor.js +2 -2
- package/out/zero-server/src/pg-query-executor.js.map +1 -1
- package/out/zero-server/src/schema.d.ts +2 -2
- package/out/zero-server/src/schema.d.ts.map +1 -1
- package/out/zero-server/src/schema.js +2 -2
- package/out/zero-server/src/schema.js.map +1 -1
- package/out/zero-server/src/zql-database.d.ts +21 -1
- package/out/zero-server/src/zql-database.d.ts.map +1 -1
- package/out/zero-server/src/zql-database.js +33 -2
- package/out/zero-server/src/zql-database.js.map +1 -1
- package/out/zero-solid/src/solid-view.d.ts +17 -0
- package/out/zero-solid/src/solid-view.d.ts.map +1 -1
- package/out/zero-solid/src/solid-view.js +39 -1
- package/out/zero-solid/src/solid-view.js.map +1 -1
- package/out/zero-solid/src/use-query.js +5 -3
- package/out/zero-solid/src/use-query.js.map +1 -1
- package/out/zql/src/builder/builder.d.ts.map +1 -1
- package/out/zql/src/builder/builder.js +16 -10
- package/out/zql/src/builder/builder.js.map +1 -1
- package/out/zql/src/builder/debug-delegate.d.ts +6 -1
- package/out/zql/src/builder/debug-delegate.d.ts.map +1 -1
- package/out/zql/src/builder/debug-delegate.js +14 -2
- package/out/zql/src/builder/debug-delegate.js.map +1 -1
- package/out/zql/src/ivm/array-view.d.ts +29 -0
- package/out/zql/src/ivm/array-view.d.ts.map +1 -1
- package/out/zql/src/ivm/array-view.js +63 -12
- package/out/zql/src/ivm/array-view.js.map +1 -1
- package/out/zql/src/ivm/cap.d.ts.map +1 -1
- package/out/zql/src/ivm/cap.js +5 -3
- package/out/zql/src/ivm/cap.js.map +1 -1
- package/out/zql/src/ivm/data.d.ts.map +1 -1
- package/out/zql/src/ivm/data.js +8 -3
- package/out/zql/src/ivm/data.js.map +1 -1
- package/out/zql/src/ivm/deferred-input.d.ts +44 -0
- package/out/zql/src/ivm/deferred-input.d.ts.map +1 -0
- package/out/zql/src/ivm/deferred-input.js +89 -0
- package/out/zql/src/ivm/deferred-input.js.map +1 -0
- package/out/zql/src/ivm/exists.d.ts +1 -1
- package/out/zql/src/ivm/exists.d.ts.map +1 -1
- package/out/zql/src/ivm/exists.js.map +1 -1
- package/out/zql/src/ivm/fan-in.d.ts +2 -2
- package/out/zql/src/ivm/fan-in.d.ts.map +1 -1
- package/out/zql/src/ivm/fan-in.js +6 -5
- package/out/zql/src/ivm/fan-in.js.map +1 -1
- package/out/zql/src/ivm/fan-out.d.ts +2 -2
- package/out/zql/src/ivm/fan-out.d.ts.map +1 -1
- package/out/zql/src/ivm/fan-out.js.map +1 -1
- package/out/zql/src/ivm/filter-operators.d.ts +7 -6
- package/out/zql/src/ivm/filter-operators.d.ts.map +1 -1
- package/out/zql/src/ivm/filter-operators.js +29 -12
- package/out/zql/src/ivm/filter-operators.js.map +1 -1
- package/out/zql/src/ivm/filter.d.ts +2 -2
- package/out/zql/src/ivm/filter.d.ts.map +1 -1
- package/out/zql/src/ivm/filter.js +7 -4
- package/out/zql/src/ivm/filter.js.map +1 -1
- package/out/zql/src/ivm/flipped-join.js +2 -2
- package/out/zql/src/ivm/flipped-join.js.map +1 -1
- package/out/zql/src/ivm/maybe-split-and-push-edit-change.d.ts +1 -1
- package/out/zql/src/ivm/maybe-split-and-push-edit-change.d.ts.map +1 -1
- package/out/zql/src/ivm/maybe-split-and-push-edit-change.js +6 -4
- package/out/zql/src/ivm/maybe-split-and-push-edit-change.js.map +1 -1
- package/out/zql/src/ivm/memory-source.d.ts +15 -3
- package/out/zql/src/ivm/memory-source.d.ts.map +1 -1
- package/out/zql/src/ivm/memory-source.js +57 -23
- package/out/zql/src/ivm/memory-source.js.map +1 -1
- package/out/zql/src/ivm/operator.d.ts +40 -0
- package/out/zql/src/ivm/operator.d.ts.map +1 -1
- package/out/zql/src/ivm/operator.js.map +1 -1
- package/out/zql/src/ivm/take.d.ts.map +1 -1
- package/out/zql/src/ivm/take.js +3 -4
- package/out/zql/src/ivm/take.js.map +1 -1
- package/out/zql/src/ivm/union-fan-in.d.ts.map +1 -1
- package/out/zql/src/ivm/union-fan-in.js +9 -7
- package/out/zql/src/ivm/union-fan-in.js.map +1 -1
- package/out/zql/src/ivm/view-apply-change.d.ts.map +1 -1
- package/out/zql/src/ivm/view-apply-change.js +9 -4
- package/out/zql/src/ivm/view-apply-change.js.map +1 -1
- package/out/zql/src/ivm/view.d.ts +15 -0
- package/out/zql/src/ivm/view.d.ts.map +1 -1
- package/out/zql/src/mutate/custom.d.ts +24 -1
- package/out/zql/src/mutate/custom.d.ts.map +1 -1
- package/out/zql/src/mutate/custom.js.map +1 -1
- package/out/zql/src/mutate/mutator-registry.d.ts.map +1 -1
- package/out/zql/src/mutate/mutator-registry.js +2 -2
- package/out/zql/src/mutate/mutator-registry.js.map +1 -1
- package/out/zql/src/planner/planner-builder.d.ts.map +1 -1
- package/out/zql/src/planner/planner-builder.js +14 -4
- package/out/zql/src/planner/planner-builder.js.map +1 -1
- package/out/zql/src/planner/planner-connection.d.ts +13 -0
- package/out/zql/src/planner/planner-connection.d.ts.map +1 -1
- package/out/zql/src/planner/planner-connection.js +42 -7
- package/out/zql/src/planner/planner-connection.js.map +1 -1
- package/out/zql/src/planner/planner-debug.d.ts +1 -1
- package/out/zql/src/planner/planner-debug.d.ts.map +1 -1
- package/out/zql/src/planner/planner-debug.js.map +1 -1
- package/out/zql/src/planner/planner-fan-out.d.ts +1 -0
- package/out/zql/src/planner/planner-fan-out.d.ts.map +1 -1
- package/out/zql/src/planner/planner-fan-out.js +3 -0
- package/out/zql/src/planner/planner-fan-out.js.map +1 -1
- package/out/zql/src/planner/planner-filter.d.ts +59 -0
- package/out/zql/src/planner/planner-filter.d.ts.map +1 -0
- package/out/zql/src/planner/planner-filter.js +116 -0
- package/out/zql/src/planner/planner-filter.js.map +1 -0
- package/out/zql/src/planner/planner-graph.d.ts +2 -0
- package/out/zql/src/planner/planner-graph.d.ts.map +1 -1
- package/out/zql/src/planner/planner-graph.js +2 -0
- package/out/zql/src/planner/planner-graph.js.map +1 -1
- package/out/zql/src/planner/planner-join.d.ts +2 -0
- package/out/zql/src/planner/planner-join.d.ts.map +1 -1
- package/out/zql/src/planner/planner-join.js +7 -0
- package/out/zql/src/planner/planner-join.js.map +1 -1
- package/out/zql/src/planner/planner-node.d.ts +2 -1
- package/out/zql/src/planner/planner-node.d.ts.map +1 -1
- package/out/zql/src/planner/planner-node.js.map +1 -1
- package/out/zql/src/query/query-delegate-base.d.ts +9 -1
- package/out/zql/src/query/query-delegate-base.d.ts.map +1 -1
- package/out/zql/src/query/query-delegate-base.js +129 -33
- package/out/zql/src/query/query-delegate-base.js.map +1 -1
- package/out/zql/src/query/query-delegate.d.ts +50 -1
- package/out/zql/src/query/query-delegate.d.ts.map +1 -1
- package/out/zql/src/query/query-impl.d.ts +13 -11
- package/out/zql/src/query/query-impl.d.ts.map +1 -1
- package/out/zql/src/query/query-impl.js +383 -60
- package/out/zql/src/query/query-impl.js.map +1 -1
- package/out/zql/src/query/query-transitions.d.ts +171 -0
- package/out/zql/src/query/query-transitions.d.ts.map +1 -0
- package/out/zql/src/query/query-transitions.js +207 -0
- package/out/zql/src/query/query-transitions.js.map +1 -0
- package/out/zql/src/query/query.d.ts +11 -2
- package/out/zql/src/query/query.d.ts.map +1 -1
- package/out/zql/src/query/runnable-query-impl.d.ts +1 -0
- package/out/zql/src/query/runnable-query-impl.d.ts.map +1 -1
- package/out/zql/src/query/runnable-query-impl.js +21 -1
- package/out/zql/src/query/runnable-query-impl.js.map +1 -1
- package/out/zql/src/query/typed-view.d.ts +1 -1
- package/out/zql/src/query/typed-view.d.ts.map +1 -1
- package/out/zqlite/src/db.d.ts +9 -1
- package/out/zqlite/src/db.d.ts.map +1 -1
- package/out/zqlite/src/db.js +8 -2
- package/out/zqlite/src/db.js.map +1 -1
- package/out/zqlite/src/query-builder.d.ts +1 -1
- package/out/zqlite/src/query-builder.d.ts.map +1 -1
- package/out/zqlite/src/query-builder.js +2 -1
- package/out/zqlite/src/query-builder.js.map +1 -1
- package/out/zqlite/src/sqlite-cost-model.d.ts.map +1 -1
- package/out/zqlite/src/sqlite-cost-model.js +2 -30
- package/out/zqlite/src/sqlite-cost-model.js.map +1 -1
- package/out/zqlite/src/table-source.d.ts +1 -0
- package/out/zqlite/src/table-source.d.ts.map +1 -1
- package/out/zqlite/src/table-source.js +20 -8
- package/out/zqlite/src/table-source.js.map +1 -1
- package/package.json +3 -4
|
@@ -0,0 +1,156 @@
|
|
|
1
|
+
import type { PublishedTableSpec } from '../../../db/specs.ts';
|
|
2
|
+
import type { PostgresDB, PostgresTransaction } from '../../../types/pg.ts';
|
|
3
|
+
/**
|
|
4
|
+
* The subset of a column spec needed to decide whether the column can
|
|
5
|
+
* participate in a resumable row key, and to render its values as SQL
|
|
6
|
+
* literals.
|
|
7
|
+
*
|
|
8
|
+
* `collationIsDeterministic` is not part of {@link PublishedTableSpec} (it is
|
|
9
|
+
* stripped when the published schema is canonicalized); it is fetched
|
|
10
|
+
* separately with {@link getKeyCollations}.
|
|
11
|
+
*/
|
|
12
|
+
export type ResumeColumnSpec = {
|
|
13
|
+
readonly typeOID: number;
|
|
14
|
+
readonly pgTypeClass?: string | null | undefined;
|
|
15
|
+
readonly elemPgTypeClass?: string | null | undefined;
|
|
16
|
+
readonly collationIsDeterministic?: boolean | null | undefined;
|
|
17
|
+
};
|
|
18
|
+
/**
|
|
19
|
+
* A **mark** is the Postgres text form of a row key's values, in
|
|
20
|
+
* `relation.rowKey.columns` order. It is computed by the change source (which
|
|
21
|
+
* knows the column types), stored and returned opaquely by subscribers, and
|
|
22
|
+
* compared only for equality outside of Postgres.
|
|
23
|
+
*/
|
|
24
|
+
export type Mark = readonly string[];
|
|
25
|
+
/**
|
|
26
|
+
* Whether a single column's values have an exact text form whose literal is
|
|
27
|
+
* trivially safe to inline into a `COPY (query) TO STDOUT` (which accepts no
|
|
28
|
+
* bind parameters).
|
|
29
|
+
*
|
|
30
|
+
* Types whose text form is lossy (`numeric`, the timestamp family, floats) and
|
|
31
|
+
* columns with a non-deterministic collation are excluded. Excluding the latter
|
|
32
|
+
* is conservatism rather than necessity: a unique index under a
|
|
33
|
+
* non-deterministic collation enforces uniqueness by the same equality that
|
|
34
|
+
* orders it.
|
|
35
|
+
*/
|
|
36
|
+
export declare function isResumableColumn(spec: ResumeColumnSpec): boolean;
|
|
37
|
+
/**
|
|
38
|
+
* Whether every column of the row key is resumable. When this is false the
|
|
39
|
+
* change source never attaches a `lastKey` to its `backfill` messages, so no
|
|
40
|
+
* subscriber ever holds a mark for the table and every run starts from the
|
|
41
|
+
* beginning.
|
|
42
|
+
*/
|
|
43
|
+
export declare function isResumableKey(specs: readonly ResumeColumnSpec[]): boolean;
|
|
44
|
+
/**
|
|
45
|
+
* Renders one mark value as a SQL literal.
|
|
46
|
+
*
|
|
47
|
+
* The literal must be a *constant* expression so that Postgres can use it as
|
|
48
|
+
* an index condition. (This rules out, e.g.,
|
|
49
|
+
* `convert_from(decode(...), 'UTF8')`, which is `STABLE` and therefore
|
|
50
|
+
* degrades a resume seek into a filtered full index scan.)
|
|
51
|
+
*
|
|
52
|
+
* @throws if `text` is not a valid value for the column's type, which
|
|
53
|
+
* indicates a corrupt or forged mark.
|
|
54
|
+
*/
|
|
55
|
+
export declare function keyLiteral(spec: ResumeColumnSpec, text: string): string;
|
|
56
|
+
export declare class InvalidMarkError extends Error {
|
|
57
|
+
readonly name = "InvalidMarkError";
|
|
58
|
+
}
|
|
59
|
+
/**
|
|
60
|
+
* The `ORDER BY` clause for an ordered backfill: the row key columns in
|
|
61
|
+
* `relation.rowKey.columns` order, each in its own (native) collation.
|
|
62
|
+
*
|
|
63
|
+
* Ordering in the column's native collation — rather than `COLLATE "C"` —
|
|
64
|
+
* keeps the key's btree usable for both the ordering and the resume seek. Any
|
|
65
|
+
* total, snapshot-stable order suffices, because keys are only ever compared
|
|
66
|
+
* by Postgres.
|
|
67
|
+
*/
|
|
68
|
+
export declare function orderByRowKey(rowKeyColumns: readonly string[]): string;
|
|
69
|
+
/**
|
|
70
|
+
* A boolean SQL expression selecting rows whose key sorts strictly after
|
|
71
|
+
* `mark`, using a row-constructor comparison so that it is index-optimizable
|
|
72
|
+
* on the key's btree.
|
|
73
|
+
*/
|
|
74
|
+
export declare function resumeWhere(rowKeyColumns: readonly string[], specs: readonly ResumeColumnSpec[], mark: Mark): string;
|
|
75
|
+
/**
|
|
76
|
+
* The row filter of the table's publications, as a parenthesized boolean
|
|
77
|
+
* expression, or `null` if the table is published without a filter.
|
|
78
|
+
*/
|
|
79
|
+
export declare function publicationRowFilter(table: PublishedTableSpec): string | null;
|
|
80
|
+
/**
|
|
81
|
+
* Returns whether any row of `table` has a key in `(from, to]`, under the
|
|
82
|
+
* table's publication row filter. `from === null` means "from the beginning",
|
|
83
|
+
* in which case the answer is whether any row exists up to and including `to`.
|
|
84
|
+
*
|
|
85
|
+
* This is the only key comparison the system performs outside of the backfill
|
|
86
|
+
* COPY itself, and it is what the {@link BackfillManager} asks in order to
|
|
87
|
+
* decide whether a declaring subscriber needs rows that the running run has
|
|
88
|
+
* already passed.
|
|
89
|
+
*/
|
|
90
|
+
export declare function rowsExist(sql: PostgresDB | PostgresTransaction, table: PublishedTableSpec, rowKeyColumns: readonly string[], specs: readonly ResumeColumnSpec[], from: Mark | null, to: Mark): Promise<boolean>;
|
|
91
|
+
/**
|
|
92
|
+
* Converts a decoded row key value to its mark (Postgres text) form.
|
|
93
|
+
*
|
|
94
|
+
* Accepts the shapes produced by both the binary COPY decoders
|
|
95
|
+
* (`pg-copy-binary.ts`) and the text COPY type parsers.
|
|
96
|
+
*/
|
|
97
|
+
export declare function textKey(spec: ResumeColumnSpec, value: unknown): string;
|
|
98
|
+
/**
|
|
99
|
+
* Computes the mark of the last row of a batch of `rowValues`, whose leading
|
|
100
|
+
* values are the row key values (see the `backfill` message).
|
|
101
|
+
*/
|
|
102
|
+
export declare function markOfLastRow(specs: readonly ResumeColumnSpec[], rowValues: readonly (readonly unknown[])[]): string[];
|
|
103
|
+
/**
|
|
104
|
+
* Reads the collation determinism of the specified columns.
|
|
105
|
+
*
|
|
106
|
+
* `PublishedTableSpec` does not carry this (the published schema strips it
|
|
107
|
+
* during canonicalization), so it is read directly. `null` means the column
|
|
108
|
+
* has no collation, e.g. an integer or uuid column.
|
|
109
|
+
*/
|
|
110
|
+
export declare function getKeyCollations(sql: PostgresDB | PostgresTransaction, relationOID: number, columns: readonly string[]): Promise<Map<string, boolean | null>>;
|
|
111
|
+
/**
|
|
112
|
+
* The minimum `pg_stats.correlation` of a row key's leading column for its
|
|
113
|
+
* table's backfill to be ordered (and therefore resumable).
|
|
114
|
+
*
|
|
115
|
+
* Ordering a backfill COPY by the row key means walking the key's index in
|
|
116
|
+
* order and fetching each row from the heap. When the heap's physical order
|
|
117
|
+
* matches the key order the fetches are sequential and ordering is nearly
|
|
118
|
+
* free; when it does not, each of N rows is a scattered visit into the same
|
|
119
|
+
* pages, and throughput collapses.
|
|
120
|
+
*
|
|
121
|
+
* Measured on 1M rows (~155MB, warm cache, PG 17) with
|
|
122
|
+
* `backfill-resume.bench.pg.ts`, ordered COPY as a fraction of unordered:
|
|
123
|
+
*
|
|
124
|
+
* | correlation | ordered / unordered |
|
|
125
|
+
* | ----------- | ------------------- |
|
|
126
|
+
* | 1.0 | 82% |
|
|
127
|
+
* | 0.99995 | 51% |
|
|
128
|
+
* | 0.995 | 32% |
|
|
129
|
+
* | 0.70 | 22% |
|
|
130
|
+
* | 0.0 (uuid) | 18% |
|
|
131
|
+
* | 0.0 (text) | 17% |
|
|
132
|
+
*
|
|
133
|
+
* Correlation saturates near 1, so the threshold has to be tight to hold
|
|
134
|
+
* ordering within ~2x of an unordered COPY. In practice this admits
|
|
135
|
+
* append-mostly tables with a monotonic key, and `CLUSTER`ed tables; a table
|
|
136
|
+
* with a random key (uuid, nanoid) is not ordered, and so is not resumable,
|
|
137
|
+
* and keeps exactly today's behavior.
|
|
138
|
+
*/
|
|
139
|
+
export declare const MIN_KEY_CORRELATION = 0.9999;
|
|
140
|
+
/**
|
|
141
|
+
* Reads `pg_stats.correlation` for the leading column of the row key: the
|
|
142
|
+
* statistical correlation between the column's value order and the physical
|
|
143
|
+
* row order.
|
|
144
|
+
*
|
|
145
|
+
* Returns `null` if the table has never been analyzed, or if the column has
|
|
146
|
+
* no correlation statistic. Both are treated as "do not order" by
|
|
147
|
+
* {@link isCheaplyOrderable}, which keeps today's behavior for a table whose
|
|
148
|
+
* cost is unknown.
|
|
149
|
+
*/
|
|
150
|
+
export declare function getKeyCorrelation(sql: PostgresDB | PostgresTransaction, relationOID: number, leadingKeyColumn: string): Promise<number | null>;
|
|
151
|
+
/**
|
|
152
|
+
* Whether ordering a backfill of the table by its row key is cheap enough to
|
|
153
|
+
* be worth doing, based on {@link getKeyCorrelation}.
|
|
154
|
+
*/
|
|
155
|
+
export declare function isCheaplyOrderable(correlation: number | null, minCorrelation?: number): boolean;
|
|
156
|
+
//# sourceMappingURL=backfill-resume.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"backfill-resume.d.ts","sourceRoot":"","sources":["../../../../../../../zero-cache/src/services/change-source/pg/backfill-resume.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAC,kBAAkB,EAAC,MAAM,sBAAsB,CAAC;AAU7D,OAAO,KAAK,EAAC,UAAU,EAAE,mBAAmB,EAAC,MAAM,sBAAsB,CAAC;AAG1E;;;;;;;;GAQG;AACH,MAAM,MAAM,gBAAgB,GAAG;IAC7B,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,WAAW,CAAC,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,CAAC;IACjD,QAAQ,CAAC,eAAe,CAAC,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,CAAC;IACrD,QAAQ,CAAC,wBAAwB,CAAC,EAAE,OAAO,GAAG,IAAI,GAAG,SAAS,CAAC;CAChE,CAAC;AAEF;;;;;GAKG;AACH,MAAM,MAAM,IAAI,GAAG,SAAS,MAAM,EAAE,CAAC;AASrC;;;;;;;;;;GAUG;AACH,wBAAgB,iBAAiB,CAAC,IAAI,EAAE,gBAAgB,GAAG,OAAO,CAejE;AAED;;;;;GAKG;AACH,wBAAgB,cAAc,CAAC,KAAK,EAAE,SAAS,gBAAgB,EAAE,GAAG,OAAO,CAE1E;AAcD;;;;;;;;;;GAUG;AACH,wBAAgB,UAAU,CAAC,IAAI,EAAE,gBAAgB,EAAE,IAAI,EAAE,MAAM,GAAG,MAAM,CAwBvE;AAED,qBAAa,gBAAiB,SAAQ,KAAK;IACzC,QAAQ,CAAC,IAAI,sBAAsB;CACpC;AAED;;;;;;;;GAQG;AACH,wBAAgB,aAAa,CAAC,aAAa,EAAE,SAAS,MAAM,EAAE,GAAG,MAAM,CAEtE;AAED;;;;GAIG;AACH,wBAAgB,WAAW,CACzB,aAAa,EAAE,SAAS,MAAM,EAAE,EAChC,KAAK,EAAE,SAAS,gBAAgB,EAAE,EAClC,IAAI,EAAE,IAAI,GACT,MAAM,CAER;AAkBD;;;GAGG;AACH,wBAAgB,oBAAoB,CAAC,KAAK,EAAE,kBAAkB,GAAG,MAAM,GAAG,IAAI,CAY7E;AAED;;;;;;;;;GASG;AACH,wBAAsB,SAAS,CAC7B,GAAG,EAAE,UAAU,GAAG,mBAAmB,EACrC,KAAK,EAAE,kBAAkB,EACzB,aAAa,EAAE,SAAS,MAAM,EAAE,EAChC,KAAK,EAAE,SAAS,gBAAgB,EAAE,EAClC,IAAI,EAAE,IAAI,GAAG,IAAI,EACjB,EAAE,EAAE,IAAI,GACP,OAAO,CAAC,OAAO,CAAC,CAWlB;AAED;;;;;GAKG;AACH,wBAAgB,OAAO,CAAC,IAAI,EAAE,gBAAgB,EAAE,KAAK,EAAE,OAAO,GAAG,MAAM,CAuBtE;AAED;;;GAGG;AACH,wBAAgB,aAAa,CAC3B,KAAK,EAAE,SAAS,gBAAgB,EAAE,EAClC,SAAS,EAAE,SAAS,CAAC,SAAS,OAAO,EAAE,CAAC,EAAE,GACzC,MAAM,EAAE,CAGV;AAED;;;;;;GAMG;AACH,wBAAsB,gBAAgB,CACpC,GAAG,EAAE,UAAU,GAAG,mBAAmB,EACrC,WAAW,EAAE,MAAM,EACnB,OAAO,EAAE,SAAS,MAAM,EAAE,GACzB,OAAO,CAAC,GAAG,CAAC,MAAM,EAAE,OAAO,GAAG,IAAI,CAAC,CAAC,CAOtC;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;GA2BG;AACH,eAAO,MAAM,mBAAmB,SAAS,CAAC;AAE1C;;;;;;;;;GASG;AACH,wBAAsB,iBAAiB,CACrC,GAAG,EAAE,UAAU,GAAG,mBAAmB,EACrC,WAAW,EAAE,MAAM,EACnB,gBAAgB,EAAE,MAAM,GACvB,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,CAUxB;AAED;;;GAGG;AACH,wBAAgB,kBAAkB,CAChC,WAAW,EAAE,MAAM,GAAG,IAAI,EAC1B,cAAc,GAAE,MAA4B,GAC3C,OAAO,CAET"}
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
//#region ../zero-cache/src/services/change-source/pg/backfill-resume.ts
|
|
2
|
+
/**
|
|
3
|
+
* The row filter of the table's publications, as a parenthesized boolean
|
|
4
|
+
* expression, or `null` if the table is published without a filter.
|
|
5
|
+
*/
|
|
6
|
+
function publicationRowFilter(table) {
|
|
7
|
+
const rowFilters = Object.values(table.publications).map(({ rowFilter }) => rowFilter);
|
|
8
|
+
if (rowFilters.some((filter) => filter === null)) return null;
|
|
9
|
+
const filters = rowFilters.filter((filter) => !!filter);
|
|
10
|
+
return filters.length === 0 ? null : `(${filters.join(" OR ")})`;
|
|
11
|
+
}
|
|
12
|
+
//#endregion
|
|
13
|
+
export { publicationRowFilter };
|
|
14
|
+
|
|
15
|
+
//# sourceMappingURL=backfill-resume.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"backfill-resume.js","names":[],"sources":["../../../../../../../zero-cache/src/services/change-source/pg/backfill-resume.ts"],"sourcesContent":["import {must} from '../../../../../shared/src/must.ts';\nimport * as PostgresTypeClass from '../../../db/postgres-type-class-enum.ts';\nimport type {PublishedTableSpec} from '../../../db/specs.ts';\nimport {\n BOOL,\n INT2,\n INT4,\n INT8,\n TEXT,\n UUID,\n VARCHAR,\n} from '../../../types/pg-types.ts';\nimport type {PostgresDB, PostgresTransaction} from '../../../types/pg.ts';\nimport {id} from '../../../types/sql.ts';\n\n/**\n * The subset of a column spec needed to decide whether the column can\n * participate in a resumable row key, and to render its values as SQL\n * literals.\n *\n * `collationIsDeterministic` is not part of {@link PublishedTableSpec} (it is\n * stripped when the published schema is canonicalized); it is fetched\n * separately with {@link getKeyCollations}.\n */\nexport type ResumeColumnSpec = {\n readonly typeOID: number;\n readonly pgTypeClass?: string | null | undefined;\n readonly elemPgTypeClass?: string | null | undefined;\n readonly collationIsDeterministic?: boolean | null | undefined;\n};\n\n/**\n * A **mark** is the Postgres text form of a row key's values, in\n * `relation.rowKey.columns` order. It is computed by the change source (which\n * knows the column types), stored and returned opaquely by subscribers, and\n * compared only for equality outside of Postgres.\n */\nexport type Mark = readonly string[];\n\nconst INT_TYPES: ReadonlySet<number> = new Set([INT2, INT4, INT8]);\nconst TEXT_TYPES: ReadonlySet<number> = new Set([TEXT, VARCHAR]);\n\nconst INT_RE = /^-?[0-9]+$/;\nconst UUID_RE =\n /^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}$/;\n\n/**\n * Whether a single column's values have an exact text form whose literal is\n * trivially safe to inline into a `COPY (query) TO STDOUT` (which accepts no\n * bind parameters).\n *\n * Types whose text form is lossy (`numeric`, the timestamp family, floats) and\n * columns with a non-deterministic collation are excluded. Excluding the latter\n * is conservatism rather than necessity: a unique index under a\n * non-deterministic collation enforces uniqueness by the same equality that\n * orders it.\n */\nexport function isResumableColumn(spec: ResumeColumnSpec): boolean {\n if (spec.elemPgTypeClass !== null && spec.elemPgTypeClass !== undefined) {\n return false; // array types\n }\n if (spec.pgTypeClass === PostgresTypeClass.Enum) {\n return false; // enum sort order is by attnum, not by text\n }\n const {typeOID} = spec;\n if (TEXT_TYPES.has(typeOID)) {\n // A non-deterministic collation is excluded; no collation at all\n // (`null`/`undefined`) is not expected for a text column, but is treated\n // conservatively as unknown.\n return spec.collationIsDeterministic === true;\n }\n return INT_TYPES.has(typeOID) || typeOID === UUID || typeOID === BOOL;\n}\n\n/**\n * Whether every column of the row key is resumable. When this is false the\n * change source never attaches a `lastKey` to its `backfill` messages, so no\n * subscriber ever holds a mark for the table and every run starts from the\n * beginning.\n */\nexport function isResumableKey(specs: readonly ResumeColumnSpec[]): boolean {\n return specs.length > 0 && specs.every(isResumableColumn);\n}\n\n/**\n * Escapes a string as a Postgres `E'...'` literal, which has the same meaning\n * whether or not `standard_conforming_strings` is on.\n *\n * A plain `'...'` literal would suffice under the default (and, since PG 9.1,\n * only supported) setting, but the `E` form makes the escaping explicit and\n * independent of server configuration.\n */\nfunction escapeString(value: string): string {\n return `E'${value.replace(/\\\\/g, '\\\\\\\\').replace(/'/g, \"\\\\'\")}'`;\n}\n\n/**\n * Renders one mark value as a SQL literal.\n *\n * The literal must be a *constant* expression so that Postgres can use it as\n * an index condition. (This rules out, e.g.,\n * `convert_from(decode(...), 'UTF8')`, which is `STABLE` and therefore\n * degrades a resume seek into a filtered full index scan.)\n *\n * @throws if `text` is not a valid value for the column's type, which\n * indicates a corrupt or forged mark.\n */\nexport function keyLiteral(spec: ResumeColumnSpec, text: string): string {\n const {typeOID} = spec;\n if (INT_TYPES.has(typeOID)) {\n if (!INT_RE.test(text)) {\n throw new InvalidMarkError(`${text} is not an integer`);\n }\n return text;\n }\n if (typeOID === UUID) {\n if (!UUID_RE.test(text)) {\n throw new InvalidMarkError(`${text} is not a uuid`);\n }\n return `'${text.toLowerCase()}'::uuid`;\n }\n if (typeOID === BOOL) {\n if (text !== 'true' && text !== 'false') {\n throw new InvalidMarkError(`${text} is not a boolean`);\n }\n return text;\n }\n if (TEXT_TYPES.has(typeOID)) {\n return escapeString(text);\n }\n throw new InvalidMarkError(`type OID ${typeOID} is not resumable`);\n}\n\nexport class InvalidMarkError extends Error {\n readonly name = 'InvalidMarkError';\n}\n\n/**\n * The `ORDER BY` clause for an ordered backfill: the row key columns in\n * `relation.rowKey.columns` order, each in its own (native) collation.\n *\n * Ordering in the column's native collation — rather than `COLLATE \"C\"` —\n * keeps the key's btree usable for both the ordering and the resume seek. Any\n * total, snapshot-stable order suffices, because keys are only ever compared\n * by Postgres.\n */\nexport function orderByRowKey(rowKeyColumns: readonly string[]): string {\n return rowKeyColumns.map(id).join(',');\n}\n\n/**\n * A boolean SQL expression selecting rows whose key sorts strictly after\n * `mark`, using a row-constructor comparison so that it is index-optimizable\n * on the key's btree.\n */\nexport function resumeWhere(\n rowKeyColumns: readonly string[],\n specs: readonly ResumeColumnSpec[],\n mark: Mark,\n): string {\n return rowComparison(rowKeyColumns, specs, mark, '>');\n}\n\nfunction rowComparison(\n rowKeyColumns: readonly string[],\n specs: readonly ResumeColumnSpec[],\n mark: Mark,\n op: '>' | '<=',\n): string {\n if (rowKeyColumns.length !== mark.length) {\n throw new InvalidMarkError(\n `mark has ${mark.length} values for a ${rowKeyColumns.length} column key`,\n );\n }\n const cols = rowKeyColumns.map(id).join(',');\n const vals = mark.map((text, i) => keyLiteral(specs[i], text)).join(',');\n return `(${cols}) ${op} (${vals})`;\n}\n\n/**\n * The row filter of the table's publications, as a parenthesized boolean\n * expression, or `null` if the table is published without a filter.\n */\nexport function publicationRowFilter(table: PublishedTableSpec): string | null {\n const rowFilters = Object.values(table.publications).map(\n ({rowFilter}) => rowFilter,\n );\n // PostgreSQL publishes the union of all publications. An unfiltered\n // publication therefore makes every row eligible, regardless of filters on\n // any other publication containing the table.\n if (rowFilters.some(filter => filter === null)) {\n return null;\n }\n const filters = rowFilters.filter((filter): filter is string => !!filter);\n return filters.length === 0 ? null : `(${filters.join(' OR ')})`;\n}\n\n/**\n * Returns whether any row of `table` has a key in `(from, to]`, under the\n * table's publication row filter. `from === null` means \"from the beginning\",\n * in which case the answer is whether any row exists up to and including `to`.\n *\n * This is the only key comparison the system performs outside of the backfill\n * COPY itself, and it is what the {@link BackfillManager} asks in order to\n * decide whether a declaring subscriber needs rows that the running run has\n * already passed.\n */\nexport async function rowsExist(\n sql: PostgresDB | PostgresTransaction,\n table: PublishedTableSpec,\n rowKeyColumns: readonly string[],\n specs: readonly ResumeColumnSpec[],\n from: Mark | null,\n to: Mark,\n): Promise<boolean> {\n const conditions = [\n publicationRowFilter(table),\n from === null ? null : rowComparison(rowKeyColumns, specs, from, '>'),\n rowComparison(rowKeyColumns, specs, to, '<='),\n ].filter(c => c !== null);\n const rows = await sql.unsafe(\n /*sql*/ `SELECT 1 FROM ${id(table.schema)}.${id(table.name)} ` +\n `WHERE ${conditions.join(' AND ')} LIMIT 1`,\n );\n return rows.length > 0;\n}\n\n/**\n * Converts a decoded row key value to its mark (Postgres text) form.\n *\n * Accepts the shapes produced by both the binary COPY decoders\n * (`pg-copy-binary.ts`) and the text COPY type parsers.\n */\nexport function textKey(spec: ResumeColumnSpec, value: unknown): string {\n const {typeOID} = spec;\n if (value === null || value === undefined) {\n // Row key columns are `NOT NULL` (see `db/lite-tables.ts`), so this\n // indicates a bug rather than data.\n throw new InvalidMarkError(`null value in a row key`);\n }\n if (typeOID === BOOL) {\n switch (typeof value) {\n case 'boolean':\n return value ? 'true' : 'false';\n case 'number':\n case 'bigint':\n return value ? 'true' : 'false';\n case 'string':\n return value === 't' || value === 'true' ? 'true' : 'false';\n }\n }\n const text = typeof value === 'string' ? value : String(value);\n if (typeOID === UUID) {\n return text.toLowerCase();\n }\n return text;\n}\n\n/**\n * Computes the mark of the last row of a batch of `rowValues`, whose leading\n * values are the row key values (see the `backfill` message).\n */\nexport function markOfLastRow(\n specs: readonly ResumeColumnSpec[],\n rowValues: readonly (readonly unknown[])[],\n): string[] {\n const last = must(rowValues.at(-1), `no rows`);\n return specs.map((spec, i) => textKey(spec, last[i]));\n}\n\n/**\n * Reads the collation determinism of the specified columns.\n *\n * `PublishedTableSpec` does not carry this (the published schema strips it\n * during canonicalization), so it is read directly. `null` means the column\n * has no collation, e.g. an integer or uuid column.\n */\nexport async function getKeyCollations(\n sql: PostgresDB | PostgresTransaction,\n relationOID: number,\n columns: readonly string[],\n): Promise<Map<string, boolean | null>> {\n const rows = await sql<{col: string; deterministic: boolean | null}[]>`\n SELECT attname AS \"col\", coll.collisdeterministic AS \"deterministic\"\n FROM pg_attribute\n LEFT JOIN pg_collation coll ON coll.oid = attcollation\n WHERE attrelid = ${relationOID} AND attname IN ${sql(columns as string[])}`;\n return new Map(rows.map(({col, deterministic}) => [col, deterministic]));\n}\n\n/**\n * The minimum `pg_stats.correlation` of a row key's leading column for its\n * table's backfill to be ordered (and therefore resumable).\n *\n * Ordering a backfill COPY by the row key means walking the key's index in\n * order and fetching each row from the heap. When the heap's physical order\n * matches the key order the fetches are sequential and ordering is nearly\n * free; when it does not, each of N rows is a scattered visit into the same\n * pages, and throughput collapses.\n *\n * Measured on 1M rows (~155MB, warm cache, PG 17) with\n * `backfill-resume.bench.pg.ts`, ordered COPY as a fraction of unordered:\n *\n * | correlation | ordered / unordered |\n * | ----------- | ------------------- |\n * | 1.0 | 82% |\n * | 0.99995 | 51% |\n * | 0.995 | 32% |\n * | 0.70 | 22% |\n * | 0.0 (uuid) | 18% |\n * | 0.0 (text) | 17% |\n *\n * Correlation saturates near 1, so the threshold has to be tight to hold\n * ordering within ~2x of an unordered COPY. In practice this admits\n * append-mostly tables with a monotonic key, and `CLUSTER`ed tables; a table\n * with a random key (uuid, nanoid) is not ordered, and so is not resumable,\n * and keeps exactly today's behavior.\n */\nexport const MIN_KEY_CORRELATION = 0.9999;\n\n/**\n * Reads `pg_stats.correlation` for the leading column of the row key: the\n * statistical correlation between the column's value order and the physical\n * row order.\n *\n * Returns `null` if the table has never been analyzed, or if the column has\n * no correlation statistic. Both are treated as \"do not order\" by\n * {@link isCheaplyOrderable}, which keeps today's behavior for a table whose\n * cost is unknown.\n */\nexport async function getKeyCorrelation(\n sql: PostgresDB | PostgresTransaction,\n relationOID: number,\n leadingKeyColumn: string,\n): Promise<number | null> {\n const rows = await sql<{correlation: number | null}[]>`\n SELECT s.correlation\n FROM pg_stats s, pg_class c, pg_namespace n\n WHERE c.oid = ${relationOID}\n AND n.oid = c.relnamespace\n AND s.schemaname = n.nspname\n AND s.tablename = c.relname\n AND s.attname = ${leadingKeyColumn}`;\n return rows[0]?.correlation ?? null;\n}\n\n/**\n * Whether ordering a backfill of the table by its row key is cheap enough to\n * be worth doing, based on {@link getKeyCorrelation}.\n */\nexport function isCheaplyOrderable(\n correlation: number | null,\n minCorrelation: number = MIN_KEY_CORRELATION,\n): boolean {\n return correlation !== null && Math.abs(correlation) >= minCorrelation;\n}\n"],"mappings":";;;;;AAuLA,SAAgB,qBAAqB,OAA0C;CAC7E,MAAM,aAAa,OAAO,OAAO,MAAM,YAAY,EAAE,KAClD,EAAC,gBAAe,SACnB;CAIA,IAAI,WAAW,MAAK,WAAU,WAAW,IAAI,GAC3C,OAAO;CAET,MAAM,UAAU,WAAW,QAAQ,WAA6B,CAAC,CAAC,MAAM;CACxE,OAAO,QAAQ,WAAW,IAAI,OAAO,IAAI,QAAQ,KAAK,MAAM,EAAE;AAChE"}
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
import type { LogContext } from '@rocicorp/logger';
|
|
2
|
+
import { type PostgresDB } from '../../../types/pg.ts';
|
|
3
|
+
import { type ShardConfig, type ShardID } from '../../../types/shards.ts';
|
|
4
|
+
import { type InitializeResult, type RestoreOptions } from '../common/replica-restore.ts';
|
|
5
|
+
import { type InitialSyncOptions, type ReplicaOptions, type ServerContext } from './initial-sync.ts';
|
|
6
|
+
import { type ReplicaState } from './schema/shard.ts';
|
|
7
|
+
interface PurgeLock {
|
|
8
|
+
release(): Promise<void>;
|
|
9
|
+
}
|
|
10
|
+
export type InitializeOptions = ReplicaOptions & {
|
|
11
|
+
slotPerReplica: boolean | undefined;
|
|
12
|
+
inactiveReplicaGracePeriodMs: number;
|
|
13
|
+
};
|
|
14
|
+
/**
|
|
15
|
+
* Initializes a Postgres change source, including the initial sync of the
|
|
16
|
+
* replica, before streaming changes from the corresponding logical replication
|
|
17
|
+
* stream.
|
|
18
|
+
*/
|
|
19
|
+
export declare function initializePostgresChangeSource(lc: LogContext, upstreamURI: string, shard: ShardConfig, replicaDbFile: string, syncOptions: InitialSyncOptions, context: ServerContext, lagReportIntervalMs?: number, restoreOptions?: RestoreOptions, { epoch, slotPerReplica, backupV5, inactiveReplicaGracePeriodMs, }?: InitializeOptions, purgeLock?: PurgeLock | null, streamInboundTimeoutMs?: number | undefined): Promise<InitializeResult>;
|
|
20
|
+
export declare function getSourceAndDestinationReplicas(lc: LogContext, sql: PostgresDB, shard: ShardID, epoch: number, slotFailover: boolean, gracePeriodMs: number, pollIntervalMs?: number): Promise<{
|
|
21
|
+
restoreFrom: ReplicaState;
|
|
22
|
+
replicateTo: ReplicaState;
|
|
23
|
+
} | undefined>;
|
|
24
|
+
export {};
|
|
25
|
+
//# sourceMappingURL=change-source-init.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"change-source-init.d.ts","sourceRoot":"","sources":["../../../../../../../zero-cache/src/services/change-source/pg/change-source-init.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAQjD,OAAO,EAAkB,KAAK,UAAU,EAAC,MAAM,sBAAsB,CAAC;AACtE,OAAO,EAAC,KAAK,WAAW,EAAE,KAAK,OAAO,EAAC,MAAM,0BAA0B,CAAC;AAMxE,OAAO,EAEL,KAAK,gBAAgB,EACrB,KAAK,cAAc,EACpB,MAAM,8BAA8B,CAAC;AAGtC,OAAO,EAEL,KAAK,kBAAkB,EACvB,KAAK,cAAc,EACnB,KAAK,aAAa,EACnB,MAAM,mBAAmB,CAAC;AAU3B,OAAO,EAOL,KAAK,YAAY,EAClB,MAAM,mBAAmB,CAAC;AAE3B,UAAU,SAAS;IACjB,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;CAC1B;AAED,MAAM,MAAM,iBAAiB,GAAG,cAAc,GAAG;IAG/C,cAAc,EAAE,OAAO,GAAG,SAAS,CAAC;IAEpC,4BAA4B,EAAE,MAAM,CAAC;CACtC,CAAC;AAEF;;;;GAIG;AACH,wBAAsB,8BAA8B,CAClD,EAAE,EAAE,UAAU,EACd,WAAW,EAAE,MAAM,EACnB,KAAK,EAAE,WAAW,EAClB,aAAa,EAAE,MAAM,EACrB,WAAW,EAAE,kBAAkB,EAC/B,OAAO,EAAE,aAAa,EACtB,mBAAmB,SAAI,EACvB,cAAc,GAAE,cAAmB,EACnC,EACE,KAAK,EACL,cAAc,EACd,QAAQ,EACR,4BAA4B,GAC7B,GAAE,iBAKF,EACD,SAAS,CAAC,EAAE,SAAS,GAAG,IAAI,EAC5B,sBAAsB,CAAC,EAAE,MAAM,GAAG,SAAS,GAC1C,OAAO,CAAC,gBAAgB,CAAC,CAwH3B;AAyFD,wBAAsB,+BAA+B,CACnD,EAAE,EAAE,UAAU,EACd,GAAG,EAAE,UAAU,EACf,KAAK,EAAE,OAAO,EACd,KAAK,EAAE,MAAM,EACb,YAAY,EAAE,OAAO,EACrB,aAAa,EAAE,MAAM,EACrB,cAAc,SAA2B,GACxC,OAAO,CAAC;IAAC,WAAW,EAAE,YAAY,CAAC;IAAC,WAAW,EAAE,YAAY,CAAA;CAAC,GAAG,SAAS,CAAC,CAoI7E"}
|
|
@@ -0,0 +1,202 @@
|
|
|
1
|
+
import { assert } from "../../../../../shared/src/asserts.js";
|
|
2
|
+
import { deepEqual } from "../../../../../shared/src/json.js";
|
|
3
|
+
import { promiseVoid } from "../../../../../shared/src/resolved-promises.js";
|
|
4
|
+
import { sleep } from "../../../../../shared/src/sleep.js";
|
|
5
|
+
import { must } from "../../../../../shared/src/must.js";
|
|
6
|
+
import { Database } from "../../../../../zqlite/src/db.js";
|
|
7
|
+
import { getSubscriptionStateAndContext } from "../../replicator/schema/replication-state.js";
|
|
8
|
+
import { StatementRunner } from "../../../db/statements.js";
|
|
9
|
+
import { connectPgClient } from "../../../types/pg.js";
|
|
10
|
+
import { toBigInt } from "./lsn.js";
|
|
11
|
+
import "./schema/replica-stage-enum.js";
|
|
12
|
+
import { dropShard, getActiveReplicas, getReplicaAtVersion, getRestoreCandidates, initRestoreReplica, internalPublicationPrefix } from "./schema/shard.js";
|
|
13
|
+
import { AutoResetSignal } from "../../change-streamer/schema/tables.js";
|
|
14
|
+
import { restoreReplica } from "../common/replica-restore.js";
|
|
15
|
+
import { initReplica } from "../common/replica-schema.js";
|
|
16
|
+
import { claimSlotForResumption, createReplicaAndSlot } from "./replication-slots.js";
|
|
17
|
+
import { ensureShardSchema } from "./schema/init.js";
|
|
18
|
+
import { initialSync } from "./initial-sync.js";
|
|
19
|
+
import { PostgresChangeSource } from "./change-source.js";
|
|
20
|
+
//#region ../zero-cache/src/services/change-source/pg/change-source-init.ts
|
|
21
|
+
/**
|
|
22
|
+
* Initializes a Postgres change source, including the initial sync of the
|
|
23
|
+
* replica, before streaming changes from the corresponding logical replication
|
|
24
|
+
* stream.
|
|
25
|
+
*/
|
|
26
|
+
async function initializePostgresChangeSource(lc, upstreamURI, shard, replicaDbFile, syncOptions, context, lagReportIntervalMs = 0, restoreOptions = {}, { epoch, slotPerReplica, backupV5, inactiveReplicaGracePeriodMs } = {
|
|
27
|
+
epoch: 0,
|
|
28
|
+
slotPerReplica: false,
|
|
29
|
+
backupV5: true,
|
|
30
|
+
inactiveReplicaGracePeriodMs: DEFAULT_INACTIVE_REPLICA_GRACE_PERIOD_MS
|
|
31
|
+
}, purgeLock, streamInboundTimeoutMs) {
|
|
32
|
+
const db = await connectPgClient(lc, upstreamURI, "change-source-init");
|
|
33
|
+
try {
|
|
34
|
+
await ensureShardSchema(lc, db, shard, syncOptions.installPartialIndexTriggers);
|
|
35
|
+
if (slotPerReplica) assert(purgeLock === null, `There should be no purgeLock for RMv2`);
|
|
36
|
+
const restoredReplica = slotPerReplica ? await forkOrResumeReplica(lc, db, shard, epoch, syncOptions.replicationSlotFailover ?? false, replicaDbFile, restoreOptions, inactiveReplicaGracePeriodMs) : await selectAndRestoreReplica(lc, db, shard, replicaDbFile, restoreOptions);
|
|
37
|
+
let initialSyncedReplica;
|
|
38
|
+
await initReplica(lc, `replica-${shard.appID}-${shard.shardNum}`, replicaDbFile, async (log, tx) => {
|
|
39
|
+
purgeLock?.release();
|
|
40
|
+
initialSyncedReplica = await initialSync(log, shard, tx, upstreamURI, syncOptions, context, {
|
|
41
|
+
epoch,
|
|
42
|
+
backupV5
|
|
43
|
+
});
|
|
44
|
+
});
|
|
45
|
+
const replica = new Database(lc, replicaDbFile);
|
|
46
|
+
const subscriptionState = getSubscriptionStateAndContext(new StatementRunner(replica));
|
|
47
|
+
replica.close();
|
|
48
|
+
const { upstreamReplica, pgVersion } = await checkAndUpdateUpstream(lc, db, shard, subscriptionState, (initialSyncedReplica ?? restoredReplica)?.id);
|
|
49
|
+
const backupPath = initialSyncedReplica ? initialSyncedReplica.backupPath : backupV5 ? String(Date.now()) : restoredReplica?.backupPath ?? null;
|
|
50
|
+
return {
|
|
51
|
+
subscriptionState,
|
|
52
|
+
changeSource: new PostgresChangeSource(lc, upstreamURI, shard, upstreamReplica, pgVersion, {
|
|
53
|
+
backupPath,
|
|
54
|
+
backupV5
|
|
55
|
+
}, context, lagReportIntervalMs, syncOptions.textCopy, streamInboundTimeoutMs),
|
|
56
|
+
destinationBackupURL: backupPath && restoreOptions.litestream?.backupURL ? new URL(backupPath, restoreOptions.litestream.backupURL).toString() : restoreOptions.litestream?.backupURL,
|
|
57
|
+
replicaID: upstreamReplica.id,
|
|
58
|
+
waitForBackupBeforeServing: initialSyncedReplica !== void 0 || backupPath !== (restoredReplica?.backupPath ?? null)
|
|
59
|
+
};
|
|
60
|
+
} finally {
|
|
61
|
+
await db.end();
|
|
62
|
+
}
|
|
63
|
+
}
|
|
64
|
+
async function selectAndRestoreReplica(lc, sql, shard, replicaFile, { litestream, constraints }) {
|
|
65
|
+
const replicas = (await getActiveReplicas(lc, sql, shard)).filter(({ generation }) => generation === (constraints?.replicaVersion ?? generation));
|
|
66
|
+
if (replicas.length === 0) {
|
|
67
|
+
lc.info?.(`no suitable replicas to restore from`, { replicas });
|
|
68
|
+
return;
|
|
69
|
+
}
|
|
70
|
+
const [replica] = replicas;
|
|
71
|
+
if (litestream?.backupURL) {
|
|
72
|
+
const { backupURL: backupBaseURL } = litestream;
|
|
73
|
+
const { slot, backupPath, confirmedFlushLsn } = replica;
|
|
74
|
+
const backupURL = new URL(backupPath ?? "", backupBaseURL).toString();
|
|
75
|
+
lc.info?.(`restoring replica from ${backupURL} (${slot}@${confirmedFlushLsn})`, { replicas });
|
|
76
|
+
await restoreReplica(lc, {
|
|
77
|
+
...litestream,
|
|
78
|
+
backupURL
|
|
79
|
+
}, replicaFile, constraints);
|
|
80
|
+
}
|
|
81
|
+
return replica;
|
|
82
|
+
}
|
|
83
|
+
async function forkOrResumeReplica(lc, sql, shard, epoch, slotFailover, replicaFile, { litestream, constraints }, gracePeriodMs) {
|
|
84
|
+
const result = await getSourceAndDestinationReplicas(lc, sql, shard, epoch, slotFailover, gracePeriodMs);
|
|
85
|
+
if (!result) return;
|
|
86
|
+
const { restoreFrom, replicateTo } = result;
|
|
87
|
+
if (litestream?.backupURL) {
|
|
88
|
+
const { backupURL: backupBaseURL } = litestream;
|
|
89
|
+
const { slot, backupPath, confirmedFlushLsn } = restoreFrom;
|
|
90
|
+
const backupURL = new URL(backupPath ?? "", backupBaseURL).toString();
|
|
91
|
+
lc.info?.(`restoring replica from ${backupURL} (${slot}@${confirmedFlushLsn})`, {
|
|
92
|
+
restoreFrom,
|
|
93
|
+
replicateTo
|
|
94
|
+
});
|
|
95
|
+
await restoreReplica(lc, {
|
|
96
|
+
...litestream,
|
|
97
|
+
backupURL
|
|
98
|
+
}, replicaFile, constraints);
|
|
99
|
+
}
|
|
100
|
+
return replicateTo;
|
|
101
|
+
}
|
|
102
|
+
var REPLICA_POLL_INTERVAL_MS = 5e3;
|
|
103
|
+
var DEFAULT_INACTIVE_REPLICA_GRACE_PERIOD_MS = 2e4;
|
|
104
|
+
async function getSourceAndDestinationReplicas(lc, sql, shard, epoch, slotFailover, gracePeriodMs, pollIntervalMs = REPLICA_POLL_INTERVAL_MS) {
|
|
105
|
+
const inactiveSince = /* @__PURE__ */ new Map();
|
|
106
|
+
let destination;
|
|
107
|
+
try {
|
|
108
|
+
for (let i = 0;; i++) {
|
|
109
|
+
if (i > 0) await sleep(pollIntervalMs);
|
|
110
|
+
const replicas = await getRestoreCandidates(lc, sql, shard, epoch);
|
|
111
|
+
if (replicas.length === 0) {
|
|
112
|
+
lc.info?.(`no suitable replicas to restore from`, { replicas });
|
|
113
|
+
destination?.initialSession.destroy();
|
|
114
|
+
return;
|
|
115
|
+
}
|
|
116
|
+
for (const replica of replicas) {
|
|
117
|
+
if (replica.active) inactiveSince.delete(replica.id);
|
|
118
|
+
else if (!inactiveSince.has(replica.id)) inactiveSince.set(replica.id, Date.now());
|
|
119
|
+
if (replica.stage === 0) {
|
|
120
|
+
if (i % 12 === 0) lc.info?.(`waiting for initial sync of ${replica.id}`, { replica });
|
|
121
|
+
break;
|
|
122
|
+
}
|
|
123
|
+
if (replica.stage === 1) {
|
|
124
|
+
const { active, confirmedFlushLsn } = replica;
|
|
125
|
+
if (active) {
|
|
126
|
+
destination ??= await createReplicaAndSlot(lc, sql, "fork-replica-session", shard, epoch, Date.now().toString(), slotFailover, {
|
|
127
|
+
backupPath: null,
|
|
128
|
+
backupV5: true
|
|
129
|
+
}, () => promiseVoid, 2);
|
|
130
|
+
if (toBigInt(confirmedFlushLsn) < toBigInt(destination.slot.consistent_point)) {
|
|
131
|
+
lc.info?.(`waiting for ${replica.id}@${confirmedFlushLsn} to reach ${destination.slot.slot_name}@${destination.slot.consistent_point}`, { replica });
|
|
132
|
+
break;
|
|
133
|
+
}
|
|
134
|
+
}
|
|
135
|
+
if (destination && toBigInt(confirmedFlushLsn) >= toBigInt(destination.slot.consistent_point)) {
|
|
136
|
+
lc.info?.(`forking replica ${replica.id}@${replica.slot}`, { replica });
|
|
137
|
+
return {
|
|
138
|
+
restoreFrom: replica,
|
|
139
|
+
replicateTo: must(await initRestoreReplica(sql, shard, {
|
|
140
|
+
sourceID: replica.id,
|
|
141
|
+
destID: destination.replica.id
|
|
142
|
+
}), `replica ${destination.replica.id} was deleted`)
|
|
143
|
+
};
|
|
144
|
+
}
|
|
145
|
+
}
|
|
146
|
+
if (replica.backupPath && !replica.active) {
|
|
147
|
+
const now = Date.now();
|
|
148
|
+
const inactiveMs = now - (inactiveSince.get(replica.id) ?? now);
|
|
149
|
+
lc.info?.(`replica ${replica.id}@${replica.slot} as been inactive for ${inactiveMs}ms`, { replica });
|
|
150
|
+
if (inactiveMs >= gracePeriodMs) {
|
|
151
|
+
const reserved = await claimSlotForResumption(lc, sql, shard, "resume-replica", replica.slot);
|
|
152
|
+
if (!reserved) lc.warn?.(`unable to resume replica ${replica.id}@${replica.slot}`);
|
|
153
|
+
else {
|
|
154
|
+
lc.info?.(`resuming replica ${replica.id}@${replica.slot}`);
|
|
155
|
+
destination?.initialSession.destroy();
|
|
156
|
+
return {
|
|
157
|
+
restoreFrom: replica,
|
|
158
|
+
replicateTo: reserved.replica
|
|
159
|
+
};
|
|
160
|
+
}
|
|
161
|
+
}
|
|
162
|
+
}
|
|
163
|
+
}
|
|
164
|
+
}
|
|
165
|
+
} catch (e) {
|
|
166
|
+
destination?.initialSession.destroy();
|
|
167
|
+
throw e;
|
|
168
|
+
}
|
|
169
|
+
}
|
|
170
|
+
async function checkAndUpdateUpstream(lc, sql, shard, { replicaVersion, publications: subscribed, initialSyncContext }, replicaID) {
|
|
171
|
+
const upstreamReplica = await getReplicaAtVersion(lc, sql, shard, replicaVersion, replicaID, initialSyncContext);
|
|
172
|
+
if (!upstreamReplica) throw new AutoResetSignal(`No replication slot for replica at version ${replicaVersion} and id ${replicaID}}`);
|
|
173
|
+
const requested = shard.publications.toSorted();
|
|
174
|
+
const replicated = upstreamReplica.publications.filter((p) => !p.startsWith(internalPublicationPrefix(shard))).sort();
|
|
175
|
+
if (!deepEqual(requested, replicated)) {
|
|
176
|
+
lc.warn?.(`Dropping shard to change publications to: [${requested}]`);
|
|
177
|
+
await sql.unsafe(dropShard(shard.appID, shard.shardNum));
|
|
178
|
+
throw new AutoResetSignal(`Requested publications [${requested}] do not match configured publications: [${replicated}]`);
|
|
179
|
+
}
|
|
180
|
+
if (!deepEqual(upstreamReplica.publications, subscribed)) throw new AutoResetSignal(`Upstream publications [${upstreamReplica.publications}] do not match subscribed publications [${subscribed}]`);
|
|
181
|
+
const exists = await sql`
|
|
182
|
+
SELECT pubname FROM pg_publication WHERE pubname IN ${sql(subscribed)};
|
|
183
|
+
`.values();
|
|
184
|
+
if (exists.length !== subscribed.length) throw new AutoResetSignal(`Upstream publications [${exists.flat()}] do not contain all subscribed publications [${subscribed}]`);
|
|
185
|
+
const { slot } = upstreamReplica;
|
|
186
|
+
const result = await sql`
|
|
187
|
+
SELECT restart_lsn as "restartLSN", wal_status as "walStatus" FROM pg_replication_slots
|
|
188
|
+
WHERE slot_name = ${slot}`;
|
|
189
|
+
if (result.length === 0) throw new AutoResetSignal(`replication slot ${slot} is missing`);
|
|
190
|
+
const [{ restartLSN, walStatus }] = result;
|
|
191
|
+
if (restartLSN === null || walStatus === "lost") throw new AutoResetSignal(`replication slot ${slot} has been invalidated for exceeding the max_slot_wal_keep_size`);
|
|
192
|
+
const [{ pgVersion }] = await sql`
|
|
193
|
+
SELECT current_setting('server_version_num')::int as "pgVersion"`;
|
|
194
|
+
return {
|
|
195
|
+
upstreamReplica,
|
|
196
|
+
pgVersion
|
|
197
|
+
};
|
|
198
|
+
}
|
|
199
|
+
//#endregion
|
|
200
|
+
export { initializePostgresChangeSource };
|
|
201
|
+
|
|
202
|
+
//# sourceMappingURL=change-source-init.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"change-source-init.js","names":[],"sources":["../../../../../../../zero-cache/src/services/change-source/pg/change-source-init.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {assert} from '../../../../../shared/src/asserts.ts';\nimport {deepEqual} from '../../../../../shared/src/json.ts';\nimport {must} from '../../../../../shared/src/must.ts';\nimport {promiseVoid} from '../../../../../shared/src/resolved-promises.ts';\nimport {sleep} from '../../../../../shared/src/sleep.ts';\nimport {Database} from '../../../../../zqlite/src/db.ts';\nimport {StatementRunner} from '../../../db/statements.ts';\nimport {connectPgClient, type PostgresDB} from '../../../types/pg.ts';\nimport {type ShardConfig, type ShardID} from '../../../types/shards.ts';\nimport {AutoResetSignal} from '../../change-streamer/schema/tables.ts';\nimport {\n getSubscriptionStateAndContext,\n type SubscriptionStateAndContext,\n} from '../../replicator/schema/replication-state.ts';\nimport {\n restoreReplica,\n type InitializeResult,\n type RestoreOptions,\n} from '../common/replica-restore.ts';\nimport {initReplica} from '../common/replica-schema.ts';\nimport {PostgresChangeSource} from './change-source.ts';\nimport {\n initialSync,\n type InitialSyncOptions,\n type ReplicaOptions,\n type ServerContext,\n} from './initial-sync.ts';\nimport {toBigInt, type LSN} from './lsn.ts';\nimport {\n claimSlotForResumption,\n createReplicaAndSlot,\n type ReplicationSlotResult,\n} from './replication-slots.ts';\nimport {ensureShardSchema} from './schema/init.ts';\nimport * as ReplicaStage from './schema/replica-stage-enum.ts';\nimport {InitialSync, Replicate} from './schema/replica-stage-enum.ts';\nimport {\n dropShard,\n getActiveReplicas,\n getReplicaAtVersion,\n getRestoreCandidates,\n initRestoreReplica,\n internalPublicationPrefix,\n type ReplicaState,\n} from './schema/shard.ts';\n\ninterface PurgeLock {\n release(): Promise<void>;\n}\n\nexport type InitializeOptions = ReplicaOptions & {\n // Create a new slot for the replica rather than taking over\n // an existing one (i.e. high-availability mode).\n slotPerReplica: boolean | undefined;\n\n inactiveReplicaGracePeriodMs: number;\n};\n\n/**\n * Initializes a Postgres change source, including the initial sync of the\n * replica, before streaming changes from the corresponding logical replication\n * stream.\n */\nexport async function initializePostgresChangeSource(\n lc: LogContext,\n upstreamURI: string,\n shard: ShardConfig,\n replicaDbFile: string,\n syncOptions: InitialSyncOptions,\n context: ServerContext,\n lagReportIntervalMs = 0,\n restoreOptions: RestoreOptions = {},\n {\n epoch,\n slotPerReplica,\n backupV5,\n inactiveReplicaGracePeriodMs,\n }: InitializeOptions = {\n epoch: 0,\n slotPerReplica: false,\n backupV5: true,\n inactiveReplicaGracePeriodMs: DEFAULT_INACTIVE_REPLICA_GRACE_PERIOD_MS,\n },\n purgeLock?: PurgeLock | null,\n streamInboundTimeoutMs?: number | undefined,\n): Promise<InitializeResult> {\n const db = await connectPgClient(lc, upstreamURI, 'change-source-init');\n try {\n await ensureShardSchema(\n lc,\n db,\n shard,\n syncOptions.installPartialIndexTriggers,\n );\n\n if (slotPerReplica) {\n // Sanity check: This should be disabled via pgChangeLogEnabled=false.\n assert(purgeLock === null, `There should be no purgeLock for RMv2`);\n }\n\n const restoredReplica = slotPerReplica\n ? await forkOrResumeReplica(\n lc,\n db,\n shard,\n epoch,\n syncOptions.replicationSlotFailover ?? false,\n replicaDbFile,\n restoreOptions,\n inactiveReplicaGracePeriodMs,\n )\n : await selectAndRestoreReplica(\n lc,\n db,\n shard,\n replicaDbFile,\n restoreOptions,\n );\n\n let initialSyncedReplica: ReplicaState | undefined;\n await initReplica(\n lc,\n `replica-${shard.appID}-${shard.shardNum}`,\n replicaDbFile,\n async (log, tx) => {\n // In RMv1, the purge lock on the change-db must be released before performing\n // initial sync; if the change-db and upstream are the same db, a lock-holding\n // transaction will prevent a replication slot from being created. This awkward\n // dependency can go away with RMv2.\n void purgeLock?.release();\n initialSyncedReplica = await initialSync(\n log,\n shard,\n tx,\n upstreamURI,\n syncOptions,\n context,\n {epoch, backupV5},\n );\n },\n );\n\n const replica = new Database(lc, replicaDbFile);\n const subscriptionState = getSubscriptionStateAndContext(\n new StatementRunner(replica),\n );\n replica.close();\n\n // Check that upstream is properly setup, and throw an AutoReset to re-run\n // initial sync if not.\n const {upstreamReplica, pgVersion} = await checkAndUpdateUpstream(\n lc,\n db,\n shard,\n subscriptionState,\n (initialSyncedReplica ?? restoredReplica)?.id,\n );\n\n const backupPath = initialSyncedReplica\n ? // If initial sync was performed, use that initial backupPath.\n initialSyncedReplica.backupPath\n : // Otherwise, use a new, unique path when backing up with litestream v5. This will be\n // recorded in the replicas table by the PostgresChangeSource.\n backupV5\n ? String(Date.now())\n : (restoredReplica?.backupPath ?? null);\n\n const changeSource = new PostgresChangeSource(\n lc,\n upstreamURI,\n shard,\n upstreamReplica,\n pgVersion,\n {backupPath, backupV5},\n context,\n lagReportIntervalMs,\n syncOptions.textCopy,\n streamInboundTimeoutMs,\n );\n\n const destinationBackupURL =\n backupPath && restoreOptions.litestream?.backupURL\n ? new URL(backupPath, restoreOptions.litestream.backupURL).toString()\n : // For legacy RMv1 replicas (on litestream-v3), backup to the same location\n restoreOptions.litestream?.backupURL;\n\n return {\n subscriptionState,\n changeSource,\n destinationBackupURL,\n // The replica this change stream belongs to. It is part of the identity\n // the SQLite change log records, because a generation (i.e.\n // `replicaVersion`) is shared by every sibling of a forked replica and so\n // cannot distinguish two siblings' logs.\n replicaID: upstreamReplica.id,\n waitForBackupBeforeServing:\n // Wait for the first backup if there was an initial sync,\n initialSyncedReplica !== undefined ||\n // or if the destination differs from where it was restored\n // (i.e. backupV5).\n backupPath !== (restoredReplica?.backupPath ?? null),\n };\n } finally {\n await db.end();\n }\n}\n\n// RMv1: Selects a replica to restore from and returns it, with the\n// intention of taking over the slot (in the ChangeSource).\nasync function selectAndRestoreReplica(\n lc: LogContext,\n sql: PostgresDB,\n shard: ShardID,\n replicaFile: string,\n {litestream, constraints}: RestoreOptions,\n): Promise<ReplicaState | undefined> {\n const replicas = (await getActiveReplicas(lc, sql, shard)).filter(\n // filter to the generation specified by the constraints, if present\n ({generation}) =>\n generation === (constraints?.replicaVersion ?? generation),\n );\n if (replicas.length === 0) {\n lc.info?.(`no suitable replicas to restore from`, {replicas});\n return undefined;\n }\n const [replica] = replicas;\n\n if (litestream?.backupURL) {\n const {backupURL: backupBaseURL} = litestream;\n const {slot, backupPath, confirmedFlushLsn} = replica;\n const backupURL = new URL(backupPath ?? '', backupBaseURL).toString();\n lc.info?.(\n `restoring replica from ${backupURL} (${slot}@${confirmedFlushLsn})`,\n {replicas},\n );\n await restoreReplica(\n lc,\n {...litestream, backupURL}, // includes the replica's backup sub-path\n replicaFile,\n constraints,\n );\n }\n return replica;\n}\n\n// RMv2: Restores from an active replica and creates a new replica / slot\n// to continue replication (i.e. \"fork\"). If only orphaned replicas remain,\n// claims a slot, restores the backup, and \"resumes\" replication for that\n// slot.\nasync function forkOrResumeReplica(\n lc: LogContext,\n sql: PostgresDB,\n shard: ShardID,\n epoch: number,\n slotFailover: boolean,\n replicaFile: string,\n {litestream, constraints}: RestoreOptions,\n gracePeriodMs: number,\n): Promise<ReplicaState | undefined> {\n const result = await getSourceAndDestinationReplicas(\n lc,\n sql,\n shard,\n epoch,\n slotFailover,\n gracePeriodMs,\n );\n if (!result) {\n return undefined; // can't restore, must initial-sync\n }\n const {restoreFrom, replicateTo} = result;\n\n if (litestream?.backupURL) {\n const {backupURL: backupBaseURL} = litestream;\n const {slot, backupPath, confirmedFlushLsn} = restoreFrom;\n const backupURL = new URL(backupPath ?? '', backupBaseURL).toString();\n lc.info?.(\n `restoring replica from ${backupURL} (${slot}@${confirmedFlushLsn})`,\n {restoreFrom, replicateTo},\n );\n await restoreReplica(\n lc,\n {...litestream, backupURL}, // includes the replica's backup sub-path\n replicaFile,\n constraints,\n );\n }\n return replicateTo;\n}\n\nconst REPLICA_POLL_INTERVAL_MS = 5_000;\nconst DEFAULT_INACTIVE_REPLICA_GRACE_PERIOD_MS = 20_000;\n\n// Exported for testing.\nexport async function getSourceAndDestinationReplicas(\n lc: LogContext,\n sql: PostgresDB,\n shard: ShardID,\n epoch: number,\n slotFailover: boolean,\n gracePeriodMs: number,\n pollIntervalMs = REPLICA_POLL_INTERVAL_MS,\n): Promise<{restoreFrom: ReplicaState; replicateTo: ReplicaState} | undefined> {\n const inactiveSince = new Map<string, number>(); // tracks replica inactivity\n let destination: ReplicationSlotResult<void> | undefined;\n\n try {\n for (let i = 0; ; i++) {\n if (i > 0) {\n await sleep(pollIntervalMs);\n }\n const replicas = await getRestoreCandidates(lc, sql, shard, epoch);\n if (replicas.length === 0) {\n lc.info?.(`no suitable replicas to restore from`, {replicas});\n destination?.initialSession.destroy();\n return undefined;\n }\n for (const replica of replicas) {\n // Track inactivity to resume replicas after a grace period.\n if (replica.active) {\n inactiveSince.delete(replica.id);\n } else if (!inactiveSince.has(replica.id)) {\n inactiveSince.set(replica.id, Date.now());\n }\n\n // Note: Only `active` InitialSync replicas are returned from\n // getRestoreCandidates().\n if (replica.stage === InitialSync) {\n // Log periodically; initial-sync can be long\n if (i % 12 === 0) {\n lc.info?.(`waiting for initial sync of ${replica.id}`, {replica});\n }\n break;\n }\n\n if (replica.stage === Replicate) {\n const {active, confirmedFlushLsn} = replica;\n if (active) {\n // Create a replication slot to fork the active replica.\n destination ??= await createReplicaAndSlot(\n lc,\n sql,\n 'fork-replica-session',\n shard,\n epoch,\n Date.now().toString(), // replicaID\n slotFailover,\n {\n backupPath: null, // set only after the backup has been confirmed\n backupV5: true, // RMv2 requires backupV5\n },\n () => promiseVoid,\n ReplicaStage.Restore,\n );\n if (\n toBigInt(confirmedFlushLsn) <\n toBigInt(destination.slot.consistent_point)\n ) {\n lc.info?.(\n `waiting for ${replica.id}@${confirmedFlushLsn} to reach ${destination.slot.slot_name}@${destination.slot.consistent_point}`,\n {replica},\n );\n break;\n }\n }\n\n // If the replica is past the destination LSN, fork it, regardless of\n // whether its slot is still active. Forking is preferable to resuming,\n // as resuming removes the replica from being a candidate for\n // subsequent forks, and carries the risk of stealing the slot from a\n // task attempting to reconnect.\n if (\n destination &&\n toBigInt(confirmedFlushLsn) >=\n toBigInt(destination.slot.consistent_point)\n ) {\n lc.info?.(`forking replica ${replica.id}@${replica.slot}`, {\n replica,\n });\n const replicateTo = must(\n await initRestoreReplica(sql, shard, {\n sourceID: replica.id,\n destID: destination.replica.id,\n }),\n `replica ${destination.replica.id} was deleted`,\n );\n return {restoreFrom: replica, replicateTo};\n }\n }\n\n if (replica.backupPath && !replica.active) {\n // An inactive replica that has a backupPath may be orphaned, or it\n // may be an active task that was temporarily disconnected and\n // attempting to reestablish a session.\n //\n // Resume the replica after a grace period to avoid stealing the slot\n // from an active task that was temporarily disconnected. Resumption\n // is a last resort only taken in the absence of alternatives.\n const now = Date.now();\n const inactiveMs = now - (inactiveSince.get(replica.id) ?? now);\n lc.info?.(\n `replica ${replica.id}@${replica.slot} as been inactive for ${inactiveMs}ms`,\n {replica},\n );\n if (inactiveMs >= gracePeriodMs) {\n const reserved = await claimSlotForResumption(\n lc,\n sql,\n shard,\n 'resume-replica',\n replica.slot,\n );\n if (!reserved) {\n lc.warn?.(\n `unable to resume replica ${replica.id}@${replica.slot}`,\n );\n } else {\n lc.info?.(`resuming replica ${replica.id}@${replica.slot}`);\n // If a new replication slot was created in anticipation of forking,\n // cancel it\n destination?.initialSession.destroy();\n return {\n restoreFrom: replica,\n replicateTo: reserved.replica,\n };\n }\n }\n }\n }\n }\n } catch (e) {\n destination?.initialSession.destroy();\n throw e;\n }\n}\n\nasync function checkAndUpdateUpstream(\n lc: LogContext,\n sql: PostgresDB,\n shard: ShardConfig,\n {\n replicaVersion,\n publications: subscribed,\n initialSyncContext,\n }: SubscriptionStateAndContext,\n replicaID: string | undefined,\n) {\n const upstreamReplica = await getReplicaAtVersion(\n lc,\n sql,\n shard,\n replicaVersion,\n replicaID,\n initialSyncContext,\n );\n if (!upstreamReplica) {\n throw new AutoResetSignal(\n `No replication slot for replica at version ${replicaVersion} and id ${replicaID}}`,\n );\n }\n\n // Verify that the publications match what is being replicated.\n const requested = shard.publications.toSorted();\n const replicated = upstreamReplica.publications\n .filter(p => !p.startsWith(internalPublicationPrefix(shard)))\n .sort();\n if (!deepEqual(requested, replicated)) {\n lc.warn?.(`Dropping shard to change publications to: [${requested}]`);\n await sql.unsafe(dropShard(shard.appID, shard.shardNum));\n throw new AutoResetSignal(\n `Requested publications [${requested}] do not match configured ` +\n `publications: [${replicated}]`,\n );\n }\n\n // Sanity check: The subscription state on the replica should have the\n // same publications. This should be guaranteed by the equivalence of the\n // replicaVersion, but it doesn't hurt to verify.\n if (!deepEqual(upstreamReplica.publications, subscribed)) {\n throw new AutoResetSignal(\n `Upstream publications [${upstreamReplica.publications}] do not ` +\n `match subscribed publications [${subscribed}]`,\n );\n }\n\n // Verify that the publications exist.\n const exists = await sql`\n SELECT pubname FROM pg_publication WHERE pubname IN ${sql(subscribed)};\n `.values();\n if (exists.length !== subscribed.length) {\n throw new AutoResetSignal(\n `Upstream publications [${exists.flat()}] do not contain ` +\n `all subscribed publications [${subscribed}]`,\n );\n }\n\n const {slot} = upstreamReplica;\n const result = await sql<{restartLSN: LSN | null; walStatus: string | null}[]>\n /*sql*/ `\n SELECT restart_lsn as \"restartLSN\", wal_status as \"walStatus\" FROM pg_replication_slots\n WHERE slot_name = ${slot}`;\n if (result.length === 0) {\n throw new AutoResetSignal(`replication slot ${slot} is missing`);\n }\n const [{restartLSN, walStatus}] = result;\n if (restartLSN === null || walStatus === 'lost') {\n throw new AutoResetSignal(\n `replication slot ${slot} has been invalidated for exceeding the max_slot_wal_keep_size`,\n );\n }\n const [{pgVersion}] = await sql<{pgVersion: number}[]> /*sql*/ `\n SELECT current_setting('server_version_num')::int as \"pgVersion\"`;\n return {upstreamReplica, pgVersion};\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;AAgEA,eAAsB,+BACpB,IACA,aACA,OACA,eACA,aACA,SACA,sBAAsB,GACtB,iBAAiC,CAAC,GAClC,EACE,OACA,gBACA,UACA,iCACqB;CACrB,OAAO;CACP,gBAAgB;CAChB,UAAU;CACV,8BAA8B;AAChC,GACA,WACA,wBAC2B;CAC3B,MAAM,KAAK,MAAM,gBAAgB,IAAI,aAAa,oBAAoB;CACtE,IAAI;EACF,MAAM,kBACJ,IACA,IACA,OACA,YAAY,2BACd;EAEA,IAAI,gBAEF,OAAO,cAAc,MAAM,uCAAuC;EAGpE,MAAM,kBAAkB,iBACpB,MAAM,oBACJ,IACA,IACA,OACA,OACA,YAAY,2BAA2B,OACvC,eACA,gBACA,4BACF,IACA,MAAM,wBACJ,IACA,IACA,OACA,eACA,cACF;EAEJ,IAAI;EACJ,MAAM,YACJ,IACA,WAAW,MAAM,MAAM,GAAG,MAAM,YAChC,eACA,OAAO,KAAK,OAAO;GAKjB,WAAgB,QAAQ;GACxB,uBAAuB,MAAM,YAC3B,KACA,OACA,IACA,aACA,aACA,SACA;IAAC;IAAO;GAAQ,CAClB;EACF,CACF;EAEA,MAAM,UAAU,IAAI,SAAS,IAAI,aAAa;EAC9C,MAAM,oBAAoB,+BACxB,IAAI,gBAAgB,OAAO,CAC7B;EACA,QAAQ,MAAM;EAId,MAAM,EAAC,iBAAiB,cAAa,MAAM,uBACzC,IACA,IACA,OACA,oBACC,wBAAwB,kBAAkB,EAC7C;EAEA,MAAM,aAAa,uBAEf,qBAAqB,aAGrB,WACE,OAAO,KAAK,IAAI,CAAC,IAChB,iBAAiB,cAAc;EAqBtC,OAAO;GACL;GACA,cAAA,IArBuB,qBACvB,IACA,aACA,OACA,iBACA,WACA;IAAC;IAAY;GAAQ,GACrB,SACA,qBACA,YAAY,UACZ,sBAWA;GACA,sBARA,cAAc,eAAe,YAAY,YACrC,IAAI,IAAI,YAAY,eAAe,WAAW,SAAS,EAAE,SAAS,IAElE,eAAe,YAAY;GAU/B,WAAW,gBAAgB;GAC3B,4BAEE,yBAAyB,KAAA,KAGzB,gBAAgB,iBAAiB,cAAc;EACnD;CACF,UAAU;EACR,MAAM,GAAG,IAAI;CACf;AACF;AAIA,eAAe,wBACb,IACA,KACA,OACA,aACA,EAAC,YAAY,eACsB;CACnC,MAAM,YAAY,MAAM,kBAAkB,IAAI,KAAK,KAAK,GAAG,QAExD,EAAC,iBACA,gBAAgB,aAAa,kBAAkB,WACnD;CACA,IAAI,SAAS,WAAW,GAAG;EACzB,GAAG,OAAO,wCAAwC,EAAC,SAAQ,CAAC;EAC5D;CACF;CACA,MAAM,CAAC,WAAW;CAElB,IAAI,YAAY,WAAW;EACzB,MAAM,EAAC,WAAW,kBAAiB;EACnC,MAAM,EAAC,MAAM,YAAY,sBAAqB;EAC9C,MAAM,YAAY,IAAI,IAAI,cAAc,IAAI,aAAa,EAAE,SAAS;EACpE,GAAG,OACD,0BAA0B,UAAU,IAAI,KAAK,GAAG,kBAAkB,IAClE,EAAC,SAAQ,CACX;EACA,MAAM,eACJ,IACA;GAAC,GAAG;GAAY;EAAS,GACzB,aACA,WACF;CACF;CACA,OAAO;AACT;AAMA,eAAe,oBACb,IACA,KACA,OACA,OACA,cACA,aACA,EAAC,YAAY,eACb,eACmC;CACnC,MAAM,SAAS,MAAM,gCACnB,IACA,KACA,OACA,OACA,cACA,aACF;CACA,IAAI,CAAC,QACH;CAEF,MAAM,EAAC,aAAa,gBAAe;CAEnC,IAAI,YAAY,WAAW;EACzB,MAAM,EAAC,WAAW,kBAAiB;EACnC,MAAM,EAAC,MAAM,YAAY,sBAAqB;EAC9C,MAAM,YAAY,IAAI,IAAI,cAAc,IAAI,aAAa,EAAE,SAAS;EACpE,GAAG,OACD,0BAA0B,UAAU,IAAI,KAAK,GAAG,kBAAkB,IAClE;GAAC;GAAa;EAAW,CAC3B;EACA,MAAM,eACJ,IACA;GAAC,GAAG;GAAY;EAAS,GACzB,aACA,WACF;CACF;CACA,OAAO;AACT;AAEA,IAAM,2BAA2B;AACjC,IAAM,2CAA2C;AAGjD,eAAsB,gCACpB,IACA,KACA,OACA,OACA,cACA,eACA,iBAAiB,0BAC4D;CAC7E,MAAM,gCAAgB,IAAI,IAAoB;CAC9C,IAAI;CAEJ,IAAI;EACF,KAAK,IAAI,IAAI,IAAK,KAAK;GACrB,IAAI,IAAI,GACN,MAAM,MAAM,cAAc;GAE5B,MAAM,WAAW,MAAM,qBAAqB,IAAI,KAAK,OAAO,KAAK;GACjE,IAAI,SAAS,WAAW,GAAG;IACzB,GAAG,OAAO,wCAAwC,EAAC,SAAQ,CAAC;IAC5D,aAAa,eAAe,QAAQ;IACpC;GACF;GACA,KAAK,MAAM,WAAW,UAAU;IAE9B,IAAI,QAAQ,QACV,cAAc,OAAO,QAAQ,EAAE;SAC1B,IAAI,CAAC,cAAc,IAAI,QAAQ,EAAE,GACtC,cAAc,IAAI,QAAQ,IAAI,KAAK,IAAI,CAAC;IAK1C,IAAI,QAAQ,UAAA,GAAuB;KAEjC,IAAI,IAAI,OAAO,GACb,GAAG,OAAO,+BAA+B,QAAQ,MAAM,EAAC,QAAO,CAAC;KAElE;IACF;IAEA,IAAI,QAAQ,UAAA,GAAqB;KAC/B,MAAM,EAAC,QAAQ,sBAAqB;KACpC,IAAI,QAAQ;MAEV,gBAAgB,MAAM,qBACpB,IACA,KACA,wBACA,OACA,OACA,KAAK,IAAI,EAAE,SAAS,GACpB,cACA;OACE,YAAY;OACZ,UAAU;MACZ,SACM,aACN,CACF;MACA,IACE,SAAS,iBAAiB,IAC1B,SAAS,YAAY,KAAK,gBAAgB,GAC1C;OACA,GAAG,OACD,eAAe,QAAQ,GAAG,GAAG,kBAAkB,YAAY,YAAY,KAAK,UAAU,GAAG,YAAY,KAAK,oBAC1G,EAAC,QAAO,CACV;OACA;MACF;KACF;KAOA,IACE,eACA,SAAS,iBAAiB,KACxB,SAAS,YAAY,KAAK,gBAAgB,GAC5C;MACA,GAAG,OAAO,mBAAmB,QAAQ,GAAG,GAAG,QAAQ,QAAQ,EACzD,QACF,CAAC;MAQD,OAAO;OAAC,aAAa;OAAS,aAPV,KAClB,MAAM,mBAAmB,KAAK,OAAO;QACnC,UAAU,QAAQ;QAClB,QAAQ,YAAY,QAAQ;OAC9B,CAAC,GACD,WAAW,YAAY,QAAQ,GAAG,aAEN;MAAW;KAC3C;IACF;IAEA,IAAI,QAAQ,cAAc,CAAC,QAAQ,QAAQ;KAQzC,MAAM,MAAM,KAAK,IAAI;KACrB,MAAM,aAAa,OAAO,cAAc,IAAI,QAAQ,EAAE,KAAK;KAC3D,GAAG,OACD,WAAW,QAAQ,GAAG,GAAG,QAAQ,KAAK,wBAAwB,WAAW,KACzE,EAAC,QAAO,CACV;KACA,IAAI,cAAc,eAAe;MAC/B,MAAM,WAAW,MAAM,uBACrB,IACA,KACA,OACA,kBACA,QAAQ,IACV;MACA,IAAI,CAAC,UACH,GAAG,OACD,4BAA4B,QAAQ,GAAG,GAAG,QAAQ,MACpD;WACK;OACL,GAAG,OAAO,oBAAoB,QAAQ,GAAG,GAAG,QAAQ,MAAM;OAG1D,aAAa,eAAe,QAAQ;OACpC,OAAO;QACL,aAAa;QACb,aAAa,SAAS;OACxB;MACF;KACF;IACF;GACF;EACF;CACF,SAAS,GAAG;EACV,aAAa,eAAe,QAAQ;EACpC,MAAM;CACR;AACF;AAEA,eAAe,uBACb,IACA,KACA,OACA,EACE,gBACA,cAAc,YACd,sBAEF,WACA;CACA,MAAM,kBAAkB,MAAM,oBAC5B,IACA,KACA,OACA,gBACA,WACA,kBACF;CACA,IAAI,CAAC,iBACH,MAAM,IAAI,gBACR,8CAA8C,eAAe,UAAU,UAAU,EACnF;CAIF,MAAM,YAAY,MAAM,aAAa,SAAS;CAC9C,MAAM,aAAa,gBAAgB,aAChC,QAAO,MAAK,CAAC,EAAE,WAAW,0BAA0B,KAAK,CAAC,CAAC,EAC3D,KAAK;CACR,IAAI,CAAC,UAAU,WAAW,UAAU,GAAG;EACrC,GAAG,OAAO,8CAA8C,UAAU,EAAE;EACpE,MAAM,IAAI,OAAO,UAAU,MAAM,OAAO,MAAM,QAAQ,CAAC;EACvD,MAAM,IAAI,gBACR,2BAA2B,UAAU,2CACjB,WAAW,EACjC;CACF;CAKA,IAAI,CAAC,UAAU,gBAAgB,cAAc,UAAU,GACrD,MAAM,IAAI,gBACR,0BAA0B,gBAAgB,aAAa,0CACnB,WAAW,EACjD;CAIF,MAAM,SAAS,MAAM,GAAG;0DACgC,IAAI,UAAU,EAAE;IACtE,OAAO;CACT,IAAI,OAAO,WAAW,WAAW,QAC/B,MAAM,IAAI,gBACR,0BAA0B,OAAO,KAAK,EAAE,gDACN,WAAW,EAC/C;CAGF,MAAM,EAAC,SAAQ;CACf,MAAM,SAAS,MAAM,GACb;;0BAEgB;CACxB,IAAI,OAAO,WAAW,GACpB,MAAM,IAAI,gBAAgB,oBAAoB,KAAK,YAAY;CAEjE,MAAM,CAAC,EAAC,YAAY,eAAc;CAClC,IAAI,eAAe,QAAQ,cAAc,QACvC,MAAM,IAAI,gBACR,oBAAoB,KAAK,+DAC3B;CAEF,MAAM,CAAC,EAAC,eAAc,MAAM,GAAmC;;CAE/D,OAAO;EAAC;EAAiB;CAAS;AACpC"}
|
|
@@ -3,35 +3,25 @@ import * as v from '../../../../../shared/src/valita.ts';
|
|
|
3
3
|
import type { PublishedIndexSpec, PublishedTableSpec } from '../../../db/specs.ts';
|
|
4
4
|
import { type LexiVersion } from '../../../types/lexi-version.ts';
|
|
5
5
|
import { type PostgresDB } from '../../../types/pg.ts';
|
|
6
|
-
import { type
|
|
6
|
+
import { type ShardID } from '../../../types/shards.ts';
|
|
7
7
|
import type { Sink } from '../../../types/streams.ts';
|
|
8
8
|
import type { ChangeSource, ChangeStream } from '../change-source.ts';
|
|
9
9
|
import { type Listener } from '../common/change-stream-multiplexer.ts';
|
|
10
|
-
import { type InitializeResult, type RestoreOptions } from '../common/replica-restore.ts';
|
|
11
10
|
import type { BackfillRequest, DownstreamStatusMessage } from '../protocol/current.ts';
|
|
12
11
|
import type { ChangeStreamMessage } from '../protocol/current/downstream.ts';
|
|
13
12
|
import { streamBackfill } from './backfill-stream.ts';
|
|
14
|
-
import { type
|
|
13
|
+
import { type ServerContext } from './initial-sync.ts';
|
|
15
14
|
import type { MessageMessage, MessageRelation as PostgresRelation } from './logical-replication/pgoutput.types.ts';
|
|
16
15
|
import { subscribe } from './logical-replication/stream.ts';
|
|
17
16
|
import { type PublishedTableWithReplicaIdentity } from './schema/published.ts';
|
|
18
17
|
import { type BackupOptions, type InternalShardConfig, type Replica } from './schema/shard.ts';
|
|
19
|
-
interface PurgeLock {
|
|
20
|
-
release(): Promise<void>;
|
|
21
|
-
}
|
|
22
|
-
/**
|
|
23
|
-
* Initializes a Postgres change source, including the initial sync of the
|
|
24
|
-
* replica, before streaming changes from the corresponding logical replication
|
|
25
|
-
* stream.
|
|
26
|
-
*/
|
|
27
|
-
export declare function initializePostgresChangeSource(lc: LogContext, upstreamURI: string, shard: ShardConfig, replicaDbFile: string, syncOptions: InitialSyncOptions, context: ServerContext, lagReportIntervalMs?: number, restoreOptions?: RestoreOptions, { backupV5 }?: ReplicaOptions, purgeLock?: PurgeLock | null, streamInboundTimeoutMs?: number | undefined): Promise<InitializeResult>;
|
|
28
18
|
/**
|
|
29
19
|
* Postgres implementation of a {@link ChangeSource} backed by a logical
|
|
30
20
|
* replication stream.
|
|
31
21
|
*/
|
|
32
22
|
export declare class PostgresChangeSource implements ChangeSource {
|
|
33
23
|
#private;
|
|
34
|
-
constructor(lc: LogContext, upstreamUri: string, shard: ShardID, replica: Replica, backupOptions: BackupOptions, context: ServerContext, lagReportIntervalMs: number, textCopy?: boolean | undefined, streamInboundTimeoutMs?: number | undefined, deps?: {
|
|
24
|
+
constructor(lc: LogContext, upstreamUri: string, shard: ShardID, replica: Replica, pgVersion: number, backupOptions: BackupOptions, context: ServerContext, lagReportIntervalMs: number, textCopy?: boolean | undefined, streamInboundTimeoutMs?: number | undefined, deps?: {
|
|
35
25
|
subscribe?: typeof subscribe;
|
|
36
26
|
streamBackfill?: typeof streamBackfill;
|
|
37
27
|
});
|
|
@@ -45,7 +35,15 @@ export declare class PostgresChangeSource implements ChangeSource {
|
|
|
45
35
|
}
|
|
46
36
|
export declare class Acker implements Listener {
|
|
47
37
|
#private;
|
|
48
|
-
|
|
38
|
+
/**
|
|
39
|
+
* @param resumeWatermark the watermark the stream resumes after. What came
|
|
40
|
+
* before it was received on an earlier connection, and only the
|
|
41
|
+
* change-streamer knows whether it has persisted it, so keepalives are
|
|
42
|
+
* not acked until the change-streamer has acked it. A change-streamer
|
|
43
|
+
* whose SQLite change log is ahead of its backup would otherwise have
|
|
44
|
+
* the slot moved past transactions that only that log holds.
|
|
45
|
+
*/
|
|
46
|
+
constructor(acks: Sink<bigint>, resumeWatermark: string | null);
|
|
49
47
|
onChange(change: ChangeStreamMessage): void;
|
|
50
48
|
ack(watermark: LexiVersion): void;
|
|
51
49
|
}
|
|
@@ -59,8 +57,8 @@ export declare class LagReporter {
|
|
|
59
57
|
#private;
|
|
60
58
|
static readonly MESSAGE_SUFFIX = "/lag-report/v1";
|
|
61
59
|
readonly messagePrefix: string;
|
|
62
|
-
constructor(lc: LogContext, shard: ShardID, db: PostgresDB, lagIntervalMs: number);
|
|
63
|
-
get pgVersion(): number
|
|
60
|
+
constructor(lc: LogContext, shard: ShardID, db: PostgresDB, pgVersion: number, lagIntervalMs: number);
|
|
61
|
+
get pgVersion(): number;
|
|
64
62
|
initiateLagReport(log?: boolean): Promise<{
|
|
65
63
|
firstCommitTimeMs: number;
|
|
66
64
|
nextSendTimeMs: number;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"change-source.d.ts","sourceRoot":"","sources":["../../../../../../../zero-cache/src/services/change-source/pg/change-source.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAcjD,OAAO,KAAK,CAAC,MAAM,qCAAqC,CAAC;
|
|
1
|
+
{"version":3,"file":"change-source.d.ts","sourceRoot":"","sources":["../../../../../../../zero-cache/src/services/change-source/pg/change-source.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAcjD,OAAO,KAAK,CAAC,MAAM,qCAAqC,CAAC;AAMzD,OAAO,KAAK,EAEV,kBAAkB,EAClB,kBAAkB,EACnB,MAAM,sBAAsB,CAAC;AAE9B,OAAO,EAAC,KAAK,WAAW,EAAC,MAAM,gCAAgC,CAAC;AAEhE,OAAO,EAA4B,KAAK,UAAU,EAAC,MAAM,sBAAsB,CAAC;AAChF,OAAO,EAAiB,KAAK,OAAO,EAAC,MAAM,0BAA0B,CAAC;AAKtE,OAAO,KAAK,EAAC,IAAI,EAAC,MAAM,2BAA2B,CAAC;AACpD,OAAO,KAAK,EAAC,YAAY,EAAE,YAAY,EAAC,MAAM,qBAAqB,CAAC;AAEpE,OAAO,EAEL,KAAK,QAAQ,EACd,MAAM,wCAAwC,CAAC;AAChD,OAAO,KAAK,EACV,eAAe,EACf,uBAAuB,EAExB,MAAM,wBAAwB,CAAC;AAQhC,OAAO,KAAK,EAEV,mBAAmB,EAEpB,MAAM,mCAAmC,CAAC;AAE3C,OAAO,EAAC,cAAc,EAAC,MAAM,sBAAsB,CAAC;AACpD,OAAO,EAAC,KAAK,aAAa,EAAC,MAAM,mBAAmB,CAAC;AACrD,OAAO,KAAK,EAEV,cAAc,EACd,eAAe,IAAI,gBAAgB,EACpC,MAAM,yCAAyC,CAAC;AACjD,OAAO,EAAC,SAAS,EAAqB,MAAM,iCAAiC,CAAC;AAK9E,OAAO,EAIL,KAAK,iCAAiC,EACvC,MAAM,uBAAuB,CAAC;AAE/B,OAAO,EAKL,KAAK,aAAa,EAClB,KAAK,mBAAmB,EACxB,KAAK,OAAO,EACb,MAAM,mBAAmB,CAAC;AAU3B;;;GAGG;AACH,qBAAa,oBAAqB,YAAW,YAAY;;IAgBvD,YACE,EAAE,EAAE,UAAU,EACd,WAAW,EAAE,MAAM,EACnB,KAAK,EAAE,OAAO,EACd,OAAO,EAAE,OAAO,EAChB,SAAS,EAAE,MAAM,EACjB,aAAa,EAAE,aAAa,EAC5B,OAAO,EAAE,aAAa,EACtB,mBAAmB,EAAE,MAAM,EAC3B,QAAQ,CAAC,EAAE,OAAO,GAAG,SAAS,EAC9B,sBAAsB,CAAC,EAAE,MAAM,GAAG,SAAS,EAG3C,IAAI,GAAE;QACJ,SAAS,CAAC,EAAE,OAAO,SAAS,CAAC;QAC7B,cAAc,CAAC,EAAE,OAAO,cAAc,CAAC;KACnC,EAwCP;IAEK,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC,CAI1B;IAEK,gBAAgB;;;cAyBrB;IAEK,WAAW,CACf,eAAe,EAAE,MAAM,EACvB,gBAAgB,GAAE,eAAe,EAAO,GACvC,OAAO,CAAC,YAAY,CAAC,CAqBvB;IAGK,mBAAmB,CACvB,IAAI,EAAE,MAAM,EACZ,eAAe,EAAE,MAAM,EACvB,WAAW,EAAE,mBAAmB,EAChC,gBAAgB,EAAE,eAAe,EAAE,GAClC,OAAO,CAAC,YAAY,CAAC,CAgKvB;CAiFF;AAGD,qBAAa,KAAM,YAAW,QAAQ;;IAIpC;;;;;;;OAOG;IACH,YAAY,IAAI,EAAE,IAAI,CAAC,MAAM,CAAC,EAAE,eAAe,EAAE,MAAM,GAAG,IAAI,EAG7D;IAED,QAAQ,CAAC,MAAM,EAAE,mBAAmB,GAAG,IAAI,CA0B1C;IAMD,GAAG,CAAC,SAAS,EAAE,WAAW,QAQzB;CAYF;AAED,QAAA,MAAM,eAAe;;;;aAInB,CAAC;AAEH,MAAM,MAAM,SAAS,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,eAAe,CAAC,CAAC;AAIxD,qBAAa,WAAW;;IACtB,MAAM,CAAC,QAAQ,CAAC,cAAc,oBAAoB;IAGlD,QAAQ,CAAC,aAAa,EAAE,MAAM,CAAC;IAoB/B,YACE,EAAE,EAAE,UAAU,EACd,KAAK,EAAE,OAAO,EACd,EAAE,EAAE,UAAU,EACd,SAAS,EAAE,MAAM,EACjB,aAAa,EAAE,MAAM,EAOtB;IAED,IAAI,SAAS,WAEZ;IAEK,iBAAiB,CAAC,GAAG,UAAQ;;;OAuElC;IAED;;;;;;;;;;;;;;;;;;OAkBG;IACH,eAAe,CAAC,GAAG,EAAE,MAAM,GAAG,uBAAuB,GAAG,SAAS,CAYhE;IAED,IAAI,SAGH;IAwCD,uBAAuB,CAAC,GAAG,EAAE,cAAc,GAAG,uBAAuB,CAUpE;CAsCF;AA2rBD,wBAAgB,iBAAiB,CAAC,CAAC,EAAE,kBAAkB,EAAE,CAAC,EAAE,gBAAgB,WAwB3E;AAsBD;;;;;;;GAOG;AACH,wBAAgB,0BAA0B,CACxC,IAAI,EAAE,kBAAkB,EACxB,IAAI,EAAE,kBAAkB,EACxB,UAAU,EAAE,GAAG,CAAC,MAAM,EAAE,iCAAiC,CAAC,EAC1D,UAAU,EAAE,GAAG,CAAC,MAAM,EAAE,iCAAiC,CAAC,GACzD,OAAO,CAwET"}
|