@rocicorp/zero 1.11.0-canary.1 → 1.11.0-canary.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/out/analyze-query/src/bin-analyze.js +1 -1
- package/out/analyze-query/src/bin-analyze.js.map +1 -1
- package/out/replicache/src/btree/read.d.ts +12 -1
- package/out/replicache/src/btree/read.d.ts.map +1 -1
- package/out/replicache/src/btree/read.js +24 -12
- package/out/replicache/src/btree/read.js.map +1 -1
- package/out/replicache/src/dag/gc.d.ts.map +1 -1
- package/out/replicache/src/dag/gc.js +2 -1
- package/out/replicache/src/dag/gc.js.map +1 -1
- package/out/replicache/src/dag/invalid-ref-count-error.d.ts +12 -0
- package/out/replicache/src/dag/invalid-ref-count-error.d.ts.map +1 -0
- package/out/replicache/src/dag/invalid-ref-count-error.js +19 -0
- package/out/replicache/src/dag/invalid-ref-count-error.js.map +1 -0
- package/out/replicache/src/dag/store-impl.d.ts +12 -2
- package/out/replicache/src/dag/store-impl.d.ts.map +1 -1
- package/out/replicache/src/dag/store-impl.js +22 -8
- package/out/replicache/src/dag/store-impl.js.map +1 -1
- package/out/replicache/src/db/rebase.d.ts +14 -2
- package/out/replicache/src/db/rebase.d.ts.map +1 -1
- package/out/replicache/src/db/rebase.js +27 -5
- package/out/replicache/src/db/rebase.js.map +1 -1
- package/out/replicache/src/db/write.js.map +1 -1
- package/out/replicache/src/kv/expo-sqlite/store.d.ts.map +1 -1
- package/out/replicache/src/kv/expo-sqlite/store.js +28 -5
- package/out/replicache/src/kv/expo-sqlite/store.js.map +1 -1
- package/out/replicache/src/kv/idb-store-with-mem-fallback.d.ts +15 -5
- package/out/replicache/src/kv/idb-store-with-mem-fallback.d.ts.map +1 -1
- package/out/replicache/src/kv/idb-store-with-mem-fallback.js +25 -25
- package/out/replicache/src/kv/idb-store-with-mem-fallback.js.map +1 -1
- package/out/replicache/src/kv/idb-store.d.ts +9 -0
- package/out/replicache/src/kv/idb-store.d.ts.map +1 -1
- package/out/replicache/src/kv/idb-store.js +14 -2
- package/out/replicache/src/kv/idb-store.js.map +1 -1
- package/out/replicache/src/kv/mem-store.d.ts +1 -0
- package/out/replicache/src/kv/mem-store.d.ts.map +1 -1
- package/out/replicache/src/kv/mem-store.js +3 -0
- package/out/replicache/src/kv/mem-store.js.map +1 -1
- package/out/replicache/src/kv/op-sqlite/store.d.ts.map +1 -1
- package/out/replicache/src/kv/op-sqlite/store.js +1 -1
- package/out/replicache/src/kv/op-sqlite/store.js.map +1 -1
- package/out/replicache/src/kv/sqlite-store.d.ts +13 -1
- package/out/replicache/src/kv/sqlite-store.d.ts.map +1 -1
- package/out/replicache/src/kv/sqlite-store.js +49 -4
- package/out/replicache/src/kv/sqlite-store.js.map +1 -1
- package/out/replicache/src/kv/store.d.ts +6 -0
- package/out/replicache/src/kv/store.d.ts.map +1 -1
- package/out/replicache/src/persist/collect-idb-databases.d.ts +10 -5
- package/out/replicache/src/persist/collect-idb-databases.d.ts.map +1 -1
- package/out/replicache/src/persist/collect-idb-databases.js +16 -9
- package/out/replicache/src/persist/collect-idb-databases.js.map +1 -1
- package/out/replicache/src/persist/persist.d.ts.map +1 -1
- package/out/replicache/src/persist/persist.js +12 -5
- package/out/replicache/src/persist/persist.js.map +1 -1
- package/out/replicache/src/persist/refresh.js +6 -2
- package/out/replicache/src/persist/refresh.js.map +1 -1
- package/out/replicache/src/replicache-impl.d.ts +14 -3
- package/out/replicache/src/replicache-impl.d.ts.map +1 -1
- package/out/replicache/src/replicache-impl.js +118 -11
- package/out/replicache/src/replicache-impl.js.map +1 -1
- package/out/replicache/src/replicache-options.d.ts +9 -0
- package/out/replicache/src/replicache-options.d.ts.map +1 -1
- package/out/replicache/src/transactions.d.ts.map +1 -1
- package/out/replicache/src/transactions.js +1 -1
- package/out/replicache/src/transactions.js.map +1 -1
- package/out/shared/src/error.d.ts +11 -0
- package/out/shared/src/error.d.ts.map +1 -1
- package/out/shared/src/error.js +35 -1
- package/out/shared/src/error.js.map +1 -1
- package/out/shared/src/iterables.d.ts +2 -0
- package/out/shared/src/iterables.d.ts.map +1 -1
- package/out/shared/src/iterables.js +18 -1
- package/out/shared/src/iterables.js.map +1 -1
- package/out/shared/src/map.d.ts +7 -2
- package/out/shared/src/map.d.ts.map +1 -1
- package/out/shared/src/map.js.map +1 -1
- package/out/shared/src/ring-buffer.d.ts +8 -0
- package/out/shared/src/ring-buffer.d.ts.map +1 -1
- package/out/shared/src/ring-buffer.js +22 -1
- package/out/shared/src/ring-buffer.js.map +1 -1
- package/out/zero/package.js +3 -4
- package/out/zero/package.js.map +1 -1
- package/out/zero-cache/src/config/normalize.d.ts.map +1 -1
- package/out/zero-cache/src/config/normalize.js +25 -1
- package/out/zero-cache/src/config/normalize.js.map +1 -1
- package/out/zero-cache/src/config/zero-config.d.ts +33 -0
- package/out/zero-cache/src/config/zero-config.d.ts.map +1 -1
- package/out/zero-cache/src/config/zero-config.js +95 -5
- package/out/zero-cache/src/config/zero-config.js.map +1 -1
- package/out/zero-cache/src/server/change-streamer.d.ts.map +1 -1
- package/out/zero-cache/src/server/change-streamer.js +25 -9
- package/out/zero-cache/src/server/change-streamer.js.map +1 -1
- package/out/zero-cache/src/server/inspector-delegate.d.ts +18 -2
- package/out/zero-cache/src/server/inspector-delegate.d.ts.map +1 -1
- package/out/zero-cache/src/server/inspector-delegate.js +20 -7
- package/out/zero-cache/src/server/inspector-delegate.js.map +1 -1
- package/out/zero-cache/src/server/main.d.ts.map +1 -1
- package/out/zero-cache/src/server/main.js +8 -1
- package/out/zero-cache/src/server/main.js.map +1 -1
- package/out/zero-cache/src/server/mutator.js +2 -0
- package/out/zero-cache/src/server/mutator.js.map +1 -1
- package/out/zero-cache/src/server/reaper.d.ts.map +1 -1
- package/out/zero-cache/src/server/reaper.js +2 -0
- package/out/zero-cache/src/server/reaper.js.map +1 -1
- package/out/zero-cache/src/server/replicator.d.ts.map +1 -1
- package/out/zero-cache/src/server/replicator.js +3 -1
- package/out/zero-cache/src/server/replicator.js.map +1 -1
- package/out/zero-cache/src/server/runner/zero-dispatcher.d.ts.map +1 -1
- package/out/zero-cache/src/server/runner/zero-dispatcher.js +3 -0
- package/out/zero-cache/src/server/runner/zero-dispatcher.js.map +1 -1
- package/out/zero-cache/src/server/syncer.d.ts.map +1 -1
- package/out/zero-cache/src/server/syncer.js +2 -0
- package/out/zero-cache/src/server/syncer.js.map +1 -1
- package/out/zero-cache/src/services/analyze.d.ts.map +1 -1
- package/out/zero-cache/src/services/analyze.js +3 -2
- package/out/zero-cache/src/services/analyze.js.map +1 -1
- package/out/zero-cache/src/services/change-source/common/backfill-manager.d.ts.map +1 -1
- package/out/zero-cache/src/services/change-source/common/backfill-manager.js +14 -1
- package/out/zero-cache/src/services/change-source/common/backfill-manager.js.map +1 -1
- package/out/zero-cache/src/services/change-source/common/change-stream-multiplexer.d.ts.map +1 -1
- package/out/zero-cache/src/services/change-source/common/change-stream-multiplexer.js +15 -3
- package/out/zero-cache/src/services/change-source/common/change-stream-multiplexer.js.map +1 -1
- package/out/zero-cache/src/services/change-source/common/replica-schema.d.ts.map +1 -1
- package/out/zero-cache/src/services/change-source/common/replica-schema.js +2 -39
- package/out/zero-cache/src/services/change-source/common/replica-schema.js.map +1 -1
- package/out/zero-cache/src/services/change-source/pg/backfill-resume.d.ts +156 -0
- package/out/zero-cache/src/services/change-source/pg/backfill-resume.d.ts.map +1 -0
- package/out/zero-cache/src/services/change-source/pg/backfill-resume.js +15 -0
- package/out/zero-cache/src/services/change-source/pg/backfill-resume.js.map +1 -0
- package/out/zero-cache/src/services/change-source/pg/change-source-init.d.ts +25 -0
- package/out/zero-cache/src/services/change-source/pg/change-source-init.d.ts.map +1 -0
- package/out/zero-cache/src/services/change-source/pg/change-source-init.js +202 -0
- package/out/zero-cache/src/services/change-source/pg/change-source-init.js.map +1 -0
- package/out/zero-cache/src/services/change-source/pg/change-source.d.ts +14 -16
- package/out/zero-cache/src/services/change-source/pg/change-source.d.ts.map +1 -1
- package/out/zero-cache/src/services/change-source/pg/change-source.js +64 -143
- package/out/zero-cache/src/services/change-source/pg/change-source.js.map +1 -1
- package/out/zero-cache/src/services/change-source/pg/initial-sync.d.ts +28 -2
- package/out/zero-cache/src/services/change-source/pg/initial-sync.d.ts.map +1 -1
- package/out/zero-cache/src/services/change-source/pg/initial-sync.js +44 -21
- package/out/zero-cache/src/services/change-source/pg/initial-sync.js.map +1 -1
- package/out/zero-cache/src/services/change-source/pg/replication-slot-cleanup-monitor.d.ts +16 -0
- package/out/zero-cache/src/services/change-source/pg/replication-slot-cleanup-monitor.d.ts.map +1 -0
- package/out/zero-cache/src/services/change-source/pg/replication-slot-cleanup-monitor.js +113 -0
- package/out/zero-cache/src/services/change-source/pg/replication-slot-cleanup-monitor.js.map +1 -0
- package/out/zero-cache/src/services/change-source/pg/replication-slots.d.ts +21 -15
- package/out/zero-cache/src/services/change-source/pg/replication-slots.d.ts.map +1 -1
- package/out/zero-cache/src/services/change-source/pg/replication-slots.js +86 -23
- package/out/zero-cache/src/services/change-source/pg/replication-slots.js.map +1 -1
- package/out/zero-cache/src/services/change-source/pg/schema/index-predicate.d.ts.map +1 -1
- package/out/zero-cache/src/services/change-source/pg/schema/index-predicate.js +68 -8
- package/out/zero-cache/src/services/change-source/pg/schema/index-predicate.js.map +1 -1
- package/out/zero-cache/src/services/change-source/pg/schema/init.d.ts.map +1 -1
- package/out/zero-cache/src/services/change-source/pg/schema/init.js +32 -1
- package/out/zero-cache/src/services/change-source/pg/schema/init.js.map +1 -1
- package/out/zero-cache/src/services/change-source/pg/schema/replica-stage-enum.d.ts +10 -0
- package/out/zero-cache/src/services/change-source/pg/schema/replica-stage-enum.d.ts.map +1 -0
- package/out/zero-cache/src/services/change-source/pg/schema/replica-stage-enum.js +6 -0
- package/out/zero-cache/src/services/change-source/pg/schema/replica-stage-enum.js.map +1 -0
- package/out/zero-cache/src/services/change-source/pg/schema/shard.d.ts +50 -27
- package/out/zero-cache/src/services/change-source/pg/schema/shard.d.ts.map +1 -1
- package/out/zero-cache/src/services/change-source/pg/schema/shard.js +100 -34
- package/out/zero-cache/src/services/change-source/pg/schema/shard.js.map +1 -1
- package/out/zero-cache/src/services/change-streamer/backup-monitor.d.ts.map +1 -1
- package/out/zero-cache/src/services/change-streamer/backup-monitor.js +58 -7
- package/out/zero-cache/src/services/change-streamer/backup-monitor.js.map +1 -1
- package/out/zero-cache/src/services/change-streamer/broadcast.d.ts +8 -2
- package/out/zero-cache/src/services/change-streamer/broadcast.d.ts.map +1 -1
- package/out/zero-cache/src/services/change-streamer/broadcast.js +24 -4
- package/out/zero-cache/src/services/change-streamer/broadcast.js.map +1 -1
- package/out/zero-cache/src/services/change-streamer/change-log-codec.d.ts +19 -4
- package/out/zero-cache/src/services/change-streamer/change-log-codec.d.ts.map +1 -1
- package/out/zero-cache/src/services/change-streamer/change-log-codec.js +28 -8
- package/out/zero-cache/src/services/change-streamer/change-log-codec.js.map +1 -1
- package/out/zero-cache/src/services/change-streamer/change-streamer-http.d.ts +5 -2
- package/out/zero-cache/src/services/change-streamer/change-streamer-http.d.ts.map +1 -1
- package/out/zero-cache/src/services/change-streamer/change-streamer-http.js +21 -14
- package/out/zero-cache/src/services/change-streamer/change-streamer-http.js.map +1 -1
- package/out/zero-cache/src/services/change-streamer/change-streamer-service.d.ts +6 -0
- package/out/zero-cache/src/services/change-streamer/change-streamer-service.d.ts.map +1 -1
- package/out/zero-cache/src/services/change-streamer/change-streamer-service.js +97 -63
- package/out/zero-cache/src/services/change-streamer/change-streamer-service.js.map +1 -1
- package/out/zero-cache/src/services/change-streamer/change-streamer.d.ts +12 -8
- package/out/zero-cache/src/services/change-streamer/change-streamer.d.ts.map +1 -1
- package/out/zero-cache/src/services/change-streamer/change-streamer.js.map +1 -1
- package/out/zero-cache/src/services/change-streamer/forwarder.d.ts +3 -3
- package/out/zero-cache/src/services/change-streamer/forwarder.d.ts.map +1 -1
- package/out/zero-cache/src/services/change-streamer/forwarder.js +36 -5
- package/out/zero-cache/src/services/change-streamer/forwarder.js.map +1 -1
- package/out/zero-cache/src/services/change-streamer/schema/tables.d.ts +1 -1
- package/out/zero-cache/src/services/change-streamer/schema/tables.d.ts.map +1 -1
- package/out/zero-cache/src/services/change-streamer/schema/tables.js +4 -4
- package/out/zero-cache/src/services/change-streamer/schema/tables.js.map +1 -1
- package/out/zero-cache/src/services/change-streamer/snapshot.d.ts +3 -1
- package/out/zero-cache/src/services/change-streamer/snapshot.d.ts.map +1 -1
- package/out/zero-cache/src/services/change-streamer/snapshot.js +54 -20
- package/out/zero-cache/src/services/change-streamer/snapshot.js.map +1 -1
- package/out/zero-cache/src/services/change-streamer/sqlite-change-log-writer.d.ts +6 -1
- package/out/zero-cache/src/services/change-streamer/sqlite-change-log-writer.d.ts.map +1 -1
- package/out/zero-cache/src/services/change-streamer/sqlite-change-log-writer.js +27 -9
- package/out/zero-cache/src/services/change-streamer/sqlite-change-log-writer.js.map +1 -1
- package/out/zero-cache/src/services/change-streamer/storer.d.ts +2 -2
- package/out/zero-cache/src/services/change-streamer/storer.d.ts.map +1 -1
- package/out/zero-cache/src/services/change-streamer/storer.js +8 -5
- package/out/zero-cache/src/services/change-streamer/storer.js.map +1 -1
- package/out/zero-cache/src/services/change-streamer/subscriber.d.ts +4 -1
- package/out/zero-cache/src/services/change-streamer/subscriber.d.ts.map +1 -1
- package/out/zero-cache/src/services/change-streamer/subscriber.js +42 -0
- package/out/zero-cache/src/services/change-streamer/subscriber.js.map +1 -1
- package/out/zero-cache/src/services/change-streamer/upstream-acker.d.ts +17 -1
- package/out/zero-cache/src/services/change-streamer/upstream-acker.d.ts.map +1 -1
- package/out/zero-cache/src/services/change-streamer/upstream-acker.js +19 -3
- package/out/zero-cache/src/services/change-streamer/upstream-acker.js.map +1 -1
- package/out/zero-cache/src/services/change-streamer/vfs-watermark-poller.d.ts.map +1 -1
- package/out/zero-cache/src/services/change-streamer/vfs-watermark-poller.js +1 -0
- package/out/zero-cache/src/services/change-streamer/vfs-watermark-poller.js.map +1 -1
- package/out/zero-cache/src/services/life-cycle.d.ts.map +1 -1
- package/out/zero-cache/src/services/life-cycle.js +3 -1
- package/out/zero-cache/src/services/life-cycle.js.map +1 -1
- package/out/zero-cache/src/services/litestream/commands.d.ts.map +1 -1
- package/out/zero-cache/src/services/litestream/commands.js +3 -1
- package/out/zero-cache/src/services/litestream/commands.js.map +1 -1
- package/out/zero-cache/src/services/litestream/config-v5.yml +3 -4
- package/out/zero-cache/src/services/litestream/litestream-checkpointer.js +51 -24
- package/out/zero-cache/src/services/litestream/litestream-checkpointer.js.map +1 -1
- package/out/zero-cache/src/services/mutagen/mutagen.d.ts +5 -0
- package/out/zero-cache/src/services/mutagen/mutagen.d.ts.map +1 -1
- package/out/zero-cache/src/services/mutagen/mutagen.js +7 -1
- package/out/zero-cache/src/services/mutagen/mutagen.js.map +1 -1
- package/out/zero-cache/src/services/mutagen/pusher.js +2 -1
- package/out/zero-cache/src/services/mutagen/pusher.js.map +1 -1
- package/out/zero-cache/src/services/profz.d.ts +7 -0
- package/out/zero-cache/src/services/profz.d.ts.map +1 -0
- package/out/zero-cache/src/services/profz.js +105 -0
- package/out/zero-cache/src/services/profz.js.map +1 -0
- package/out/zero-cache/src/services/replicator/change-log-db.js.map +1 -1
- package/out/zero-cache/src/services/replicator/change-log-stream-writer.d.ts +5 -4
- package/out/zero-cache/src/services/replicator/change-log-stream-writer.d.ts.map +1 -1
- package/out/zero-cache/src/services/replicator/change-log-stream-writer.js +20 -22
- package/out/zero-cache/src/services/replicator/change-log-stream-writer.js.map +1 -1
- package/out/zero-cache/src/services/replicator/change-processor.js +1 -0
- package/out/zero-cache/src/services/replicator/change-processor.js.map +1 -1
- package/out/zero-cache/src/services/replicator/incremental-sync.d.ts +1 -1
- package/out/zero-cache/src/services/replicator/incremental-sync.d.ts.map +1 -1
- package/out/zero-cache/src/services/replicator/incremental-sync.js +57 -17
- package/out/zero-cache/src/services/replicator/incremental-sync.js.map +1 -1
- package/out/zero-cache/src/services/replicator/replication-status.d.ts +35 -1
- package/out/zero-cache/src/services/replicator/replication-status.d.ts.map +1 -1
- package/out/zero-cache/src/services/replicator/replication-status.js +96 -6
- package/out/zero-cache/src/services/replicator/replication-status.js.map +1 -1
- package/out/zero-cache/src/services/replicator/replicator.js.map +1 -1
- package/out/zero-cache/src/services/replicator/sqlite-change-log-observability.d.ts +6 -6
- package/out/zero-cache/src/services/replicator/sqlite-change-log-observability.d.ts.map +1 -1
- package/out/zero-cache/src/services/replicator/sqlite-change-log-observability.js +19 -14
- package/out/zero-cache/src/services/replicator/sqlite-change-log-observability.js.map +1 -1
- package/out/zero-cache/src/services/replicator/write-worker-client.d.ts +4 -4
- package/out/zero-cache/src/services/replicator/write-worker-client.d.ts.map +1 -1
- package/out/zero-cache/src/services/replicator/write-worker-client.js +2 -2
- package/out/zero-cache/src/services/replicator/write-worker-client.js.map +1 -1
- package/out/zero-cache/src/services/replicator/write-worker.js +10 -4
- package/out/zero-cache/src/services/replicator/write-worker.js.map +1 -1
- package/out/zero-cache/src/services/run-ast.d.ts +12 -0
- package/out/zero-cache/src/services/run-ast.d.ts.map +1 -1
- package/out/zero-cache/src/services/run-ast.js +26 -14
- package/out/zero-cache/src/services/run-ast.js.map +1 -1
- package/out/zero-cache/src/services/runner.d.ts.map +1 -1
- package/out/zero-cache/src/services/runner.js +1 -1
- package/out/zero-cache/src/services/runner.js.map +1 -1
- package/out/zero-cache/src/services/view-syncer/client-handler.d.ts +1 -1
- package/out/zero-cache/src/services/view-syncer/client-handler.d.ts.map +1 -1
- package/out/zero-cache/src/services/view-syncer/client-handler.js +36 -31
- package/out/zero-cache/src/services/view-syncer/client-handler.js.map +1 -1
- package/out/zero-cache/src/services/view-syncer/cvr.d.ts +45 -0
- package/out/zero-cache/src/services/view-syncer/cvr.d.ts.map +1 -1
- package/out/zero-cache/src/services/view-syncer/cvr.js +133 -1
- package/out/zero-cache/src/services/view-syncer/cvr.js.map +1 -1
- package/out/zero-cache/src/services/view-syncer/drain-coordinator.d.ts +14 -1
- package/out/zero-cache/src/services/view-syncer/drain-coordinator.d.ts.map +1 -1
- package/out/zero-cache/src/services/view-syncer/drain-coordinator.js +30 -4
- package/out/zero-cache/src/services/view-syncer/drain-coordinator.js.map +1 -1
- package/out/zero-cache/src/services/view-syncer/hydration-budget.d.ts +33 -0
- package/out/zero-cache/src/services/view-syncer/hydration-budget.d.ts.map +1 -0
- package/out/zero-cache/src/services/view-syncer/hydration-budget.js +62 -0
- package/out/zero-cache/src/services/view-syncer/hydration-budget.js.map +1 -0
- package/out/zero-cache/src/services/view-syncer/hydration-circuit-breaker.d.ts +51 -0
- package/out/zero-cache/src/services/view-syncer/hydration-circuit-breaker.d.ts.map +1 -0
- package/out/zero-cache/src/services/view-syncer/hydration-circuit-breaker.js +78 -0
- package/out/zero-cache/src/services/view-syncer/hydration-circuit-breaker.js.map +1 -0
- package/out/zero-cache/src/services/view-syncer/inspect-handler.d.ts +1 -1
- package/out/zero-cache/src/services/view-syncer/inspect-handler.d.ts.map +1 -1
- package/out/zero-cache/src/services/view-syncer/inspect-handler.js +2 -2
- package/out/zero-cache/src/services/view-syncer/inspect-handler.js.map +1 -1
- package/out/zero-cache/src/services/view-syncer/pipeline-driver.d.ts +2 -1
- package/out/zero-cache/src/services/view-syncer/pipeline-driver.d.ts.map +1 -1
- package/out/zero-cache/src/services/view-syncer/pipeline-driver.js +33 -18
- package/out/zero-cache/src/services/view-syncer/pipeline-driver.js.map +1 -1
- package/out/zero-cache/src/services/view-syncer/snapshotter.d.ts +4 -1
- package/out/zero-cache/src/services/view-syncer/snapshotter.d.ts.map +1 -1
- package/out/zero-cache/src/services/view-syncer/snapshotter.js +9 -3
- package/out/zero-cache/src/services/view-syncer/snapshotter.js.map +1 -1
- package/out/zero-cache/src/services/view-syncer/view-syncer.d.ts +11 -1
- package/out/zero-cache/src/services/view-syncer/view-syncer.d.ts.map +1 -1
- package/out/zero-cache/src/services/view-syncer/view-syncer.js +507 -93
- package/out/zero-cache/src/services/view-syncer/view-syncer.js.map +1 -1
- package/out/zero-cache/src/types/processes.d.ts +32 -0
- package/out/zero-cache/src/types/processes.d.ts.map +1 -1
- package/out/zero-cache/src/types/processes.js +43 -4
- package/out/zero-cache/src/types/processes.js.map +1 -1
- package/out/zero-cache/src/types/profiler.d.ts +12 -0
- package/out/zero-cache/src/types/profiler.d.ts.map +1 -1
- package/out/zero-cache/src/types/profiler.js +46 -1
- package/out/zero-cache/src/types/profiler.js.map +1 -1
- package/out/zero-cache/src/types/streams.d.ts +25 -9
- package/out/zero-cache/src/types/streams.d.ts.map +1 -1
- package/out/zero-cache/src/types/streams.js +118 -34
- package/out/zero-cache/src/types/streams.js.map +1 -1
- package/out/zero-cache/src/types/subscription.d.ts +4 -0
- package/out/zero-cache/src/types/subscription.d.ts.map +1 -1
- package/out/zero-cache/src/types/subscription.js +71 -0
- package/out/zero-cache/src/types/subscription.js.map +1 -1
- package/out/zero-cache/src/types/ws.d.ts.map +1 -1
- package/out/zero-cache/src/types/ws.js +23 -8
- package/out/zero-cache/src/types/ws.js.map +1 -1
- package/out/zero-cache/src/workers/syncer-ws-message-handler.d.ts.map +1 -1
- package/out/zero-cache/src/workers/syncer-ws-message-handler.js +21 -16
- package/out/zero-cache/src/workers/syncer-ws-message-handler.js.map +1 -1
- package/out/zero-cache/src/workers/syncer.d.ts.map +1 -1
- package/out/zero-cache/src/workers/syncer.js +10 -1
- package/out/zero-cache/src/workers/syncer.js.map +1 -1
- package/out/zero-client/src/client/connection-manager.d.ts +20 -0
- package/out/zero-client/src/client/connection-manager.d.ts.map +1 -1
- package/out/zero-client/src/client/connection-manager.js +77 -13
- package/out/zero-client/src/client/connection-manager.js.map +1 -1
- package/out/zero-client/src/client/connection-status-enum.d.ts +2 -0
- package/out/zero-client/src/client/connection-status-enum.d.ts.map +1 -1
- package/out/zero-client/src/client/connection-status-enum.js +3 -1
- package/out/zero-client/src/client/connection-status-enum.js.map +1 -1
- package/out/zero-client/src/client/connection.d.ts +6 -0
- package/out/zero-client/src/client/connection.d.ts.map +1 -1
- package/out/zero-client/src/client/connection.js +2 -1
- package/out/zero-client/src/client/connection.js.map +1 -1
- package/out/zero-client/src/client/context.d.ts +27 -2
- package/out/zero-client/src/client/context.d.ts.map +1 -1
- package/out/zero-client/src/client/context.js +112 -1
- package/out/zero-client/src/client/context.js.map +1 -1
- package/out/zero-client/src/client/custom.js +2 -2
- package/out/zero-client/src/client/custom.js.map +1 -1
- package/out/zero-client/src/client/error.js +1 -1
- package/out/zero-client/src/client/inspector/lazy-inspector.js +2 -2
- package/out/zero-client/src/client/inspector/lazy-inspector.js.map +1 -1
- package/out/zero-client/src/client/ivm-branch.d.ts.map +1 -1
- package/out/zero-client/src/client/ivm-branch.js +20 -3
- package/out/zero-client/src/client/ivm-branch.js.map +1 -1
- package/out/zero-client/src/client/mutator-proxy.d.ts.map +1 -1
- package/out/zero-client/src/client/mutator-proxy.js +2 -1
- package/out/zero-client/src/client/mutator-proxy.js.map +1 -1
- package/out/zero-client/src/client/options.d.ts +3 -0
- package/out/zero-client/src/client/options.d.ts.map +1 -1
- package/out/zero-client/src/client/options.js.map +1 -1
- package/out/zero-client/src/client/query-manager.d.ts.map +1 -1
- package/out/zero-client/src/client/query-manager.js +42 -6
- package/out/zero-client/src/client/query-manager.js.map +1 -1
- package/out/zero-client/src/client/version.js +1 -1
- package/out/zero-client/src/client/zero-rep.d.ts +1 -0
- package/out/zero-client/src/client/zero-rep.d.ts.map +1 -1
- package/out/zero-client/src/client/zero-rep.js +9 -6
- package/out/zero-client/src/client/zero-rep.js.map +1 -1
- package/out/zero-client/src/client/zero.d.ts +18 -3
- package/out/zero-client/src/client/zero.d.ts.map +1 -1
- package/out/zero-client/src/client/zero.js +120 -32
- package/out/zero-client/src/client/zero.js.map +1 -1
- package/out/zero-client/src/types/query-result.d.ts +2 -0
- package/out/zero-client/src/types/query-result.d.ts.map +1 -1
- package/out/zero-events/src/status.d.ts +26 -0
- package/out/zero-events/src/status.d.ts.map +1 -1
- package/out/zero-protocol/src/down.d.ts +2 -2
- package/out/zero-protocol/src/inspect-down.d.ts +81 -81
- package/out/zero-protocol/src/poke.d.ts +6 -6
- package/out/zero-react/src/use-query.d.ts +5 -4
- package/out/zero-react/src/use-query.d.ts.map +1 -1
- package/out/zero-react/src/use-query.js +86 -9
- package/out/zero-react/src/use-query.js.map +1 -1
- package/out/zero-server/src/adapters/drizzle.d.ts +2 -1
- package/out/zero-server/src/adapters/drizzle.d.ts.map +1 -1
- package/out/zero-server/src/adapters/drizzle.js +12 -7
- package/out/zero-server/src/adapters/drizzle.js.map +1 -1
- package/out/zero-server/src/adapters/kysely.d.ts +2 -1
- package/out/zero-server/src/adapters/kysely.d.ts.map +1 -1
- package/out/zero-server/src/adapters/kysely.js +8 -2
- package/out/zero-server/src/adapters/kysely.js.map +1 -1
- package/out/zero-server/src/adapters/pg.d.ts +1 -0
- package/out/zero-server/src/adapters/pg.d.ts.map +1 -1
- package/out/zero-server/src/adapters/pg.js +8 -2
- package/out/zero-server/src/adapters/pg.js.map +1 -1
- package/out/zero-server/src/adapters/postgresjs.d.ts +1 -0
- package/out/zero-server/src/adapters/postgresjs.d.ts.map +1 -1
- package/out/zero-server/src/adapters/postgresjs.js +7 -1
- package/out/zero-server/src/adapters/postgresjs.js.map +1 -1
- package/out/zero-server/src/adapters/prisma.d.ts +3 -2
- package/out/zero-server/src/adapters/prisma.d.ts.map +1 -1
- package/out/zero-server/src/adapters/prisma.js +8 -2
- package/out/zero-server/src/adapters/prisma.js.map +1 -1
- package/out/zero-server/src/custom.d.ts +5 -1
- package/out/zero-server/src/custom.d.ts.map +1 -1
- package/out/zero-server/src/custom.js +6 -3
- package/out/zero-server/src/custom.js.map +1 -1
- package/out/zero-server/src/mod.d.ts +1 -1
- package/out/zero-server/src/mod.d.ts.map +1 -1
- package/out/zero-server/src/pg-query-executor.d.ts +2 -2
- package/out/zero-server/src/pg-query-executor.d.ts.map +1 -1
- package/out/zero-server/src/pg-query-executor.js +2 -2
- package/out/zero-server/src/pg-query-executor.js.map +1 -1
- package/out/zero-server/src/schema.d.ts +2 -2
- package/out/zero-server/src/schema.d.ts.map +1 -1
- package/out/zero-server/src/schema.js +2 -2
- package/out/zero-server/src/schema.js.map +1 -1
- package/out/zero-server/src/zql-database.d.ts +21 -1
- package/out/zero-server/src/zql-database.d.ts.map +1 -1
- package/out/zero-server/src/zql-database.js +33 -2
- package/out/zero-server/src/zql-database.js.map +1 -1
- package/out/zero-solid/src/solid-view.d.ts +17 -0
- package/out/zero-solid/src/solid-view.d.ts.map +1 -1
- package/out/zero-solid/src/solid-view.js +39 -1
- package/out/zero-solid/src/solid-view.js.map +1 -1
- package/out/zero-solid/src/use-query.js +5 -3
- package/out/zero-solid/src/use-query.js.map +1 -1
- package/out/zql/src/builder/debug-delegate.d.ts +6 -1
- package/out/zql/src/builder/debug-delegate.d.ts.map +1 -1
- package/out/zql/src/builder/debug-delegate.js +14 -2
- package/out/zql/src/builder/debug-delegate.js.map +1 -1
- package/out/zql/src/ivm/array-view.d.ts +29 -0
- package/out/zql/src/ivm/array-view.d.ts.map +1 -1
- package/out/zql/src/ivm/array-view.js +63 -12
- package/out/zql/src/ivm/array-view.js.map +1 -1
- package/out/zql/src/ivm/cap.d.ts.map +1 -1
- package/out/zql/src/ivm/cap.js +5 -3
- package/out/zql/src/ivm/cap.js.map +1 -1
- package/out/zql/src/ivm/data.d.ts.map +1 -1
- package/out/zql/src/ivm/data.js +8 -3
- package/out/zql/src/ivm/data.js.map +1 -1
- package/out/zql/src/ivm/deferred-input.d.ts +44 -0
- package/out/zql/src/ivm/deferred-input.d.ts.map +1 -0
- package/out/zql/src/ivm/deferred-input.js +89 -0
- package/out/zql/src/ivm/deferred-input.js.map +1 -0
- package/out/zql/src/ivm/exists.d.ts +1 -1
- package/out/zql/src/ivm/exists.d.ts.map +1 -1
- package/out/zql/src/ivm/exists.js.map +1 -1
- package/out/zql/src/ivm/fan-in.d.ts +2 -2
- package/out/zql/src/ivm/fan-in.d.ts.map +1 -1
- package/out/zql/src/ivm/fan-in.js +6 -5
- package/out/zql/src/ivm/fan-in.js.map +1 -1
- package/out/zql/src/ivm/fan-out.d.ts +2 -2
- package/out/zql/src/ivm/fan-out.d.ts.map +1 -1
- package/out/zql/src/ivm/fan-out.js.map +1 -1
- package/out/zql/src/ivm/filter-operators.d.ts +6 -6
- package/out/zql/src/ivm/filter-operators.d.ts.map +1 -1
- package/out/zql/src/ivm/filter-operators.js +10 -8
- package/out/zql/src/ivm/filter-operators.js.map +1 -1
- package/out/zql/src/ivm/filter.d.ts +2 -2
- package/out/zql/src/ivm/filter.d.ts.map +1 -1
- package/out/zql/src/ivm/filter.js +7 -4
- package/out/zql/src/ivm/filter.js.map +1 -1
- package/out/zql/src/ivm/flipped-join.js +2 -2
- package/out/zql/src/ivm/flipped-join.js.map +1 -1
- package/out/zql/src/ivm/maybe-split-and-push-edit-change.d.ts +1 -1
- package/out/zql/src/ivm/maybe-split-and-push-edit-change.d.ts.map +1 -1
- package/out/zql/src/ivm/maybe-split-and-push-edit-change.js +6 -4
- package/out/zql/src/ivm/maybe-split-and-push-edit-change.js.map +1 -1
- package/out/zql/src/ivm/memory-source.d.ts +15 -3
- package/out/zql/src/ivm/memory-source.d.ts.map +1 -1
- package/out/zql/src/ivm/memory-source.js +40 -19
- package/out/zql/src/ivm/memory-source.js.map +1 -1
- package/out/zql/src/ivm/take.d.ts.map +1 -1
- package/out/zql/src/ivm/take.js +3 -4
- package/out/zql/src/ivm/take.js.map +1 -1
- package/out/zql/src/ivm/union-fan-in.d.ts.map +1 -1
- package/out/zql/src/ivm/union-fan-in.js +9 -7
- package/out/zql/src/ivm/union-fan-in.js.map +1 -1
- package/out/zql/src/ivm/view-apply-change.d.ts.map +1 -1
- package/out/zql/src/ivm/view-apply-change.js +9 -4
- package/out/zql/src/ivm/view-apply-change.js.map +1 -1
- package/out/zql/src/ivm/view.d.ts +15 -0
- package/out/zql/src/ivm/view.d.ts.map +1 -1
- package/out/zql/src/mutate/custom.d.ts +24 -1
- package/out/zql/src/mutate/custom.d.ts.map +1 -1
- package/out/zql/src/mutate/custom.js.map +1 -1
- package/out/zql/src/mutate/mutator-registry.d.ts.map +1 -1
- package/out/zql/src/mutate/mutator-registry.js +2 -2
- package/out/zql/src/mutate/mutator-registry.js.map +1 -1
- package/out/zql/src/query/query-delegate-base.d.ts +9 -1
- package/out/zql/src/query/query-delegate-base.d.ts.map +1 -1
- package/out/zql/src/query/query-delegate-base.js +129 -33
- package/out/zql/src/query/query-delegate-base.js.map +1 -1
- package/out/zql/src/query/query-delegate.d.ts +50 -1
- package/out/zql/src/query/query-delegate.d.ts.map +1 -1
- package/out/zql/src/query/query-impl.d.ts +13 -11
- package/out/zql/src/query/query-impl.d.ts.map +1 -1
- package/out/zql/src/query/query-impl.js +383 -60
- package/out/zql/src/query/query-impl.js.map +1 -1
- package/out/zql/src/query/query-transitions.d.ts +171 -0
- package/out/zql/src/query/query-transitions.d.ts.map +1 -0
- package/out/zql/src/query/query-transitions.js +207 -0
- package/out/zql/src/query/query-transitions.js.map +1 -0
- package/out/zql/src/query/query.d.ts +11 -2
- package/out/zql/src/query/query.d.ts.map +1 -1
- package/out/zql/src/query/runnable-query-impl.d.ts +1 -0
- package/out/zql/src/query/runnable-query-impl.d.ts.map +1 -1
- package/out/zql/src/query/runnable-query-impl.js +21 -1
- package/out/zql/src/query/runnable-query-impl.js.map +1 -1
- package/out/zql/src/query/typed-view.d.ts +1 -1
- package/out/zql/src/query/typed-view.d.ts.map +1 -1
- package/out/zqlite/src/db.d.ts +9 -1
- package/out/zqlite/src/db.d.ts.map +1 -1
- package/out/zqlite/src/db.js +8 -2
- package/out/zqlite/src/db.js.map +1 -1
- package/out/zqlite/src/table-source.d.ts +1 -0
- package/out/zqlite/src/table-source.d.ts.map +1 -1
- package/out/zqlite/src/table-source.js +10 -5
- package/out/zqlite/src/table-source.js.map +1 -1
- package/package.json +3 -4
|
@@ -5,6 +5,7 @@ import { appSchema, check, upstreamSchema } from "../../../../types/shards.js";
|
|
|
5
5
|
import { id } from "../../../../types/sql.js";
|
|
6
6
|
import { getPublicationInfo, publishedSchema } from "./published.js";
|
|
7
7
|
import { createEventFunctionStatements, createEventTriggerStatements } from "./ddl.js";
|
|
8
|
+
import { replicaStageSchema } from "./replica-stage-enum.js";
|
|
8
9
|
import { validate } from "./validation.js";
|
|
9
10
|
import postgres from "postgres";
|
|
10
11
|
import { PG_INSUFFICIENT_PRIVILEGE } from "@drdgvhbh/postgres-error-codes";
|
|
@@ -121,10 +122,6 @@ function shardSetup(shardConfig, metadataPublication) {
|
|
|
121
122
|
${getClientsTableDefinition(shard)}
|
|
122
123
|
${getMutationsTableDefinition(shard)}
|
|
123
124
|
|
|
124
|
-
DROP PUBLICATION IF EXISTS ${id(metadataPublication)};
|
|
125
|
-
CREATE PUBLICATION ${id(metadataPublication)}
|
|
126
|
-
FOR TABLE ${app}."permissions", TABLE ${shard}."clients", ${shard}."mutations";
|
|
127
|
-
|
|
128
125
|
CREATE TABLE ${shard}."${SHARD_CONFIG_TABLE}" (
|
|
129
126
|
"publications" TEXT[] NOT NULL,
|
|
130
127
|
"ddlDetection" BOOL NOT NULL,
|
|
@@ -147,14 +144,23 @@ function shardSetup(shardConfig, metadataPublication) {
|
|
|
147
144
|
"id" TEXT PRIMARY KEY DEFAULT replace(gen_random_uuid()::text, '-', ''),
|
|
148
145
|
"rank" BIGSERIAL,
|
|
149
146
|
"slot" TEXT NOT NULL,
|
|
150
|
-
"version" TEXT
|
|
151
|
-
"
|
|
147
|
+
"version" TEXT, -- replaced by generation, slated for deletion
|
|
148
|
+
"epoch" INT4 DEFAULT 0,
|
|
149
|
+
"generation" TEXT NOT NULL,
|
|
150
|
+
"stage" INT4 DEFAULT 0,
|
|
152
151
|
"backupPath" TEXT, -- subpath within the litestream backup URL
|
|
153
152
|
"backupV5" BOOL DEFAULT false,
|
|
154
153
|
"initialSchema" JSON, -- set after initial sync
|
|
155
154
|
"initialSyncContext" JSON,
|
|
156
155
|
"subscriberContext" JSON
|
|
157
156
|
);
|
|
157
|
+
|
|
158
|
+
DROP PUBLICATION IF EXISTS ${id(metadataPublication)};
|
|
159
|
+
CREATE PUBLICATION ${id(metadataPublication)}
|
|
160
|
+
FOR TABLE ${app}."permissions",
|
|
161
|
+
TABLE ${shard}."clients",
|
|
162
|
+
${shard}."mutations",
|
|
163
|
+
${shard}."replicas" (id); -- advance the LSN/version with new replicas
|
|
158
164
|
`;
|
|
159
165
|
}
|
|
160
166
|
function dropShard(appID, shardID) {
|
|
@@ -174,8 +180,9 @@ var replicaInfoSchema = valita_exports.object({
|
|
|
174
180
|
id: valita_exports.string(),
|
|
175
181
|
rank: valita_exports.bigint(),
|
|
176
182
|
slot: valita_exports.string(),
|
|
177
|
-
|
|
178
|
-
generation: valita_exports.string()
|
|
183
|
+
epoch: valita_exports.number(),
|
|
184
|
+
generation: valita_exports.string(),
|
|
185
|
+
stage: replicaStageSchema,
|
|
179
186
|
backupPath: valita_exports.string().nullable(),
|
|
180
187
|
backupV5: valita_exports.boolean()
|
|
181
188
|
});
|
|
@@ -184,24 +191,12 @@ var fullReplicaRowSchema = replicaInfoSchema.extend({
|
|
|
184
191
|
initialSyncContext: jsonObjectSchema.nullable(),
|
|
185
192
|
subscriberContext: jsonObjectSchema.nullable()
|
|
186
193
|
});
|
|
187
|
-
var replicaSchema = internalShardConfigSchema.extend(fullReplicaRowSchema.shape)
|
|
188
|
-
const { version, generation, ...r } = replica;
|
|
189
|
-
return {
|
|
190
|
-
...r,
|
|
191
|
-
generation: generation ?? version
|
|
192
|
-
};
|
|
193
|
-
});
|
|
194
|
+
var replicaSchema = internalShardConfigSchema.extend(fullReplicaRowSchema.shape);
|
|
194
195
|
var replicationSlotStateSchema = valita_exports.object({
|
|
195
196
|
active: valita_exports.boolean(),
|
|
196
|
-
confirmedFlushLsn: valita_exports.string()
|
|
197
|
-
});
|
|
198
|
-
var replicaStateSchema = replicaInfoSchema.extend(replicationSlotStateSchema.shape).map((replica) => {
|
|
199
|
-
const { version, generation, ...r } = replica;
|
|
200
|
-
return {
|
|
201
|
-
...r,
|
|
202
|
-
generation: generation ?? version
|
|
203
|
-
};
|
|
197
|
+
confirmedFlushLsn: valita_exports.string()
|
|
204
198
|
});
|
|
199
|
+
var replicaStateSchema = replicaInfoSchema.extend(replicationSlotStateSchema.shape);
|
|
205
200
|
function triggerSetup(shard) {
|
|
206
201
|
const schema = id(upstreamSchema(shard));
|
|
207
202
|
return createEventTriggerStatements(shard) + `UPDATE ${schema}."shardConfig" SET "ddlDetection" = true;`;
|
|
@@ -212,22 +207,23 @@ function triggerSetup(shard) {
|
|
|
212
207
|
* to ensure that the slot does not get dropped by concurrent cleanup
|
|
213
208
|
* logic.
|
|
214
209
|
*
|
|
215
|
-
* Once initial sync is complete, {@link
|
|
210
|
+
* Once initial sync is complete, {@link initInitialSyncReplica} should be called to
|
|
216
211
|
* make the replica usable for incremental sync.
|
|
217
212
|
*/
|
|
218
|
-
async function createReplica(sql, shard, id, slot, replicaVersion, { backupPath, backupV5 }) {
|
|
213
|
+
async function createReplica(sql, shard, id, slot, epoch, replicaVersion, { backupPath, backupV5 }, stage) {
|
|
219
214
|
const schema = upstreamSchema(shard);
|
|
220
215
|
const values = {
|
|
221
216
|
id,
|
|
222
217
|
slot,
|
|
223
|
-
|
|
218
|
+
epoch,
|
|
224
219
|
generation: replicaVersion,
|
|
225
220
|
backupPath,
|
|
226
|
-
backupV5
|
|
221
|
+
backupV5,
|
|
222
|
+
stage
|
|
227
223
|
};
|
|
228
224
|
await sql`INSERT INTO ${sql(schema)}.replicas ${sql(values)}`;
|
|
229
225
|
}
|
|
230
|
-
async function
|
|
226
|
+
async function initInitialSyncReplica(sql, shard, id, { tables, indexes }, initialSyncContext) {
|
|
231
227
|
const schema = upstreamSchema(shard);
|
|
232
228
|
const values = {
|
|
233
229
|
initialSchema: {
|
|
@@ -238,6 +234,18 @@ async function initReplica(sql, shard, id, { tables, indexes }, initialSyncConte
|
|
|
238
234
|
};
|
|
239
235
|
await sql`UPDATE ${sql(schema)}.replicas SET ${sql(values)} WHERE id = ${id}`;
|
|
240
236
|
}
|
|
237
|
+
async function initRestoreReplica(sql, shard, { sourceID, destID }) {
|
|
238
|
+
const schema = upstreamSchema(shard);
|
|
239
|
+
await sql`
|
|
240
|
+
UPDATE ${sql(schema)}.replicas dest
|
|
241
|
+
SET "generation" = source."generation",
|
|
242
|
+
"initialSchema" = source."initialSchema",
|
|
243
|
+
"initialSyncContext" = source."initialSyncContext"
|
|
244
|
+
FROM ${sql(schema)}.replicas source
|
|
245
|
+
WHERE source.id = ${sourceID} AND dest.id = ${destID};
|
|
246
|
+
`;
|
|
247
|
+
return getReplicaState(sql, shard, destID);
|
|
248
|
+
}
|
|
241
249
|
/**
|
|
242
250
|
* Gets at the given version and optional ID. If no ID is specified, gets
|
|
243
251
|
* the latest (i.e. highest rank) replica with that version.
|
|
@@ -249,8 +257,9 @@ async function getReplicaAtVersion(lc, sql, shard, replicaVersion, id = null, co
|
|
|
249
257
|
replicas."id",
|
|
250
258
|
replicas."rank",
|
|
251
259
|
replicas."slot",
|
|
252
|
-
replicas."
|
|
260
|
+
replicas."epoch",
|
|
253
261
|
replicas."generation",
|
|
262
|
+
replicas."stage",
|
|
254
263
|
replicas."backupPath",
|
|
255
264
|
replicas."backupV5",
|
|
256
265
|
replicas."initialSchema",
|
|
@@ -259,13 +268,13 @@ async function getReplicaAtVersion(lc, sql, shard, replicaVersion, id = null, co
|
|
|
259
268
|
"shardConfig"."publications",
|
|
260
269
|
"shardConfig"."ddlDetection"
|
|
261
270
|
FROM ${schema}.replicas JOIN ${schema}."shardConfig" ON true
|
|
262
|
-
WHERE
|
|
271
|
+
WHERE generation = ${replicaVersion} AND "initialSyncContext" IS NOT NULL
|
|
263
272
|
AND id = COALESCE(${id}, id)
|
|
264
273
|
ORDER BY rank DESC LIMIT 1;
|
|
265
274
|
`;
|
|
266
275
|
if (result.length === 0) {
|
|
267
276
|
const allReplicas = await sql`
|
|
268
|
-
SELECT id, slot,
|
|
277
|
+
SELECT id, slot, generation, "initialSyncContext", "subscriberContext"
|
|
269
278
|
FROM ${schema}.replicas`;
|
|
270
279
|
lc.info?.(`Replica ${id}@${replicaVersion}` + (context ? `(context: ${stringify(context)}) ` : "") + `not found in: ${stringify(allReplicas)}`);
|
|
271
280
|
return null;
|
|
@@ -278,27 +287,84 @@ async function getActiveReplicas(lc, sql, shard) {
|
|
|
278
287
|
replicas."id",
|
|
279
288
|
replicas."rank",
|
|
280
289
|
replicas."slot",
|
|
281
|
-
replicas."
|
|
290
|
+
replicas."epoch",
|
|
282
291
|
replicas."generation",
|
|
292
|
+
replicas."stage",
|
|
283
293
|
replicas."backupPath",
|
|
284
294
|
replicas."backupV5",
|
|
285
295
|
slots."active",
|
|
286
296
|
slots."confirmed_flush_lsn" as "confirmedFlushLsn"
|
|
287
297
|
FROM ${sql(upstreamSchema(shard))}.replicas JOIN pg_replication_slots slots ON slot = slot_name
|
|
288
298
|
WHERE "initialSyncContext" IS NOT NULL
|
|
299
|
+
-- Exclude replicas whose slot has been invalidated (e.g. for exceeding
|
|
300
|
+
-- max_slot_wal_keep_size). Such a replica can never be resumed, and
|
|
301
|
+
-- restoring it just results in an AutoResetSignal on every attempt.
|
|
302
|
+
AND slots.restart_lsn IS NOT NULL
|
|
303
|
+
AND slots.wal_status IS DISTINCT FROM 'lost'
|
|
289
304
|
ORDER BY generation DESC, confirmed_flush_lsn DESC;
|
|
290
305
|
`, valita_exports.array(replicaStateSchema), "passthrough");
|
|
291
306
|
lc.info?.(`current replicas`, { replicas });
|
|
292
307
|
return replicas;
|
|
293
308
|
}
|
|
309
|
+
/**
|
|
310
|
+
* Returns candidates for restoring a new replica from. These include
|
|
311
|
+
* all replicas in the newest `generation` of the given `epoch`, in order
|
|
312
|
+
* of earliest available backup:
|
|
313
|
+
* - An active InitialSync replica, which will be the only replica in
|
|
314
|
+
* its generation by definition. Orphaned/inactive initial syncs are skipped.
|
|
315
|
+
* - Replicas in the Replicate stage, active first.
|
|
316
|
+
* - Replicas in the Restore stage, potentially available for resumption
|
|
317
|
+
* as a last resort.
|
|
318
|
+
*/
|
|
319
|
+
async function getRestoreCandidates(lc, sql, shard, epoch) {
|
|
320
|
+
const schema = sql(upstreamSchema(shard));
|
|
321
|
+
const replicas = parse(await sql`
|
|
322
|
+
SELECT
|
|
323
|
+
replicas."id",
|
|
324
|
+
replicas."rank",
|
|
325
|
+
replicas."slot",
|
|
326
|
+
replicas."epoch",
|
|
327
|
+
replicas."generation",
|
|
328
|
+
replicas."stage",
|
|
329
|
+
replicas."backupPath",
|
|
330
|
+
replicas."backupV5",
|
|
331
|
+
slots."active",
|
|
332
|
+
slots."confirmed_flush_lsn" as "confirmedFlushLsn"
|
|
333
|
+
FROM ${schema}.replicas JOIN pg_replication_slots slots ON slot = slot_name
|
|
334
|
+
|
|
335
|
+
-- coalesce on the newest generation in the epoch
|
|
336
|
+
WHERE generation = (
|
|
337
|
+
SELECT MAX(r.generation)
|
|
338
|
+
FROM ${schema}.replicas r
|
|
339
|
+
JOIN pg_replication_slots s ON r.slot = s.slot_name
|
|
340
|
+
WHERE r."epoch" = ${epoch} AND r."backupV5"
|
|
341
|
+
-- ignore orphaned InitialSyncs, which cannot be restored from
|
|
342
|
+
AND (s."active" OR r."stage" != ${0})
|
|
343
|
+
AND s.restart_lsn IS NOT NULL
|
|
344
|
+
AND s.wal_status IS DISTINCT FROM 'lost'
|
|
345
|
+
)
|
|
346
|
+
AND "epoch" = ${epoch} AND "backupV5"
|
|
347
|
+
AND (slots.active OR "stage" != ${0})
|
|
348
|
+
AND slots.restart_lsn IS NOT NULL
|
|
349
|
+
AND slots.wal_status IS DISTINCT FROM 'lost'
|
|
350
|
+
|
|
351
|
+
ORDER BY
|
|
352
|
+
stage ASC, -- ordered by backup availalibility
|
|
353
|
+
active DESC, -- prefer active slots (fork over resume)
|
|
354
|
+
confirmed_flush_lsn DESC; -- prefer replicas further ahead
|
|
355
|
+
`, valita_exports.array(replicaStateSchema), "passthrough");
|
|
356
|
+
lc.info?.(`current replicas at epoch ${epoch}`, { replicas });
|
|
357
|
+
return replicas;
|
|
358
|
+
}
|
|
294
359
|
async function getReplicaState(sql, shard, id) {
|
|
295
360
|
const results = await sql`
|
|
296
361
|
SELECT
|
|
297
362
|
replicas."id",
|
|
298
363
|
replicas."rank",
|
|
299
364
|
replicas."slot",
|
|
300
|
-
replicas."
|
|
365
|
+
replicas."epoch",
|
|
301
366
|
replicas."generation",
|
|
367
|
+
replicas."stage",
|
|
302
368
|
replicas."backupPath",
|
|
303
369
|
replicas."backupV5",
|
|
304
370
|
slots."active",
|
|
@@ -395,6 +461,6 @@ function replicaIdentitiesForTablesWithoutPrimaryKeys(pubs) {
|
|
|
395
461
|
} };
|
|
396
462
|
}
|
|
397
463
|
//#endregion
|
|
398
|
-
export { SHARD_CONFIG_TABLE, createReplica, dropShard, ensureGlobalTables, getActiveReplicas, getInternalShardConfig, getMutationsTableDefinition, getReplicaAtVersion, getReplicaState,
|
|
464
|
+
export { SHARD_CONFIG_TABLE, createReplica, dropShard, ensureGlobalTables, getActiveReplicas, getInternalShardConfig, getMutationsTableDefinition, getReplicaAtVersion, getReplicaState, getRestoreCandidates, initInitialSyncReplica, initRestoreReplica, internalPublicationPrefix, legacyReplicationSlot, metadataPublicationName, replicaIdentitiesForTablesWithoutPrimaryKeys, replicationSlotExpression, replicationSlotPrefix, setupTablesAndReplication, setupTriggers, validatePublications };
|
|
399
465
|
|
|
400
466
|
//# sourceMappingURL=shard.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"shard.js","names":[],"sources":["../../../../../../../../zero-cache/src/services/change-source/pg/schema/shard.ts"],"sourcesContent":["import {PG_INSUFFICIENT_PRIVILEGE} from '@drdgvhbh/postgres-error-codes';\nimport type {LogContext} from '@rocicorp/logger';\nimport {literal} from 'pg-format';\nimport postgres from 'postgres';\nimport {assert} from '../../../../../../shared/src/asserts.ts';\nimport {\n jsonObjectSchema,\n stringify,\n type JSONObject,\n} from '../../../../../../shared/src/bigint-json.ts';\nimport * as v from '../../../../../../shared/src/valita.ts';\nimport {Default} from '../../../../db/postgres-replica-identity-enum.ts';\nimport type {PostgresDB, PostgresTransaction} from '../../../../types/pg.ts';\nimport type {AppID, ShardConfig, ShardID} from '../../../../types/shards.ts';\nimport {appSchema, check, upstreamSchema} from '../../../../types/shards.ts';\nimport {id} from '../../../../types/sql.ts';\nimport {\n createEventFunctionStatements,\n createEventTriggerStatements,\n} from './ddl.ts';\nimport {\n getPublicationInfo,\n publishedSchema,\n type PublicationInfo,\n type PublishedSchema,\n} from './published.ts';\nimport {validate} from './validation.ts';\n\n/**\n * PostgreSQL unquoted identifiers must start with a letter or underscore\n * and contain only letters, digits, and underscores.\n */\nconst VALID_PUBLICATION_NAME = /^[a-zA-Z_][a-zA-Z0-9_]*$/;\n\n/**\n * Validates that a publication name is a valid PostgreSQL identifier.\n * This provides defense-in-depth against SQL injection when publication\n * names are used in replication commands.\n */\nexport function validatePublicationName(name: string): void {\n if (!VALID_PUBLICATION_NAME.test(name)) {\n throw new Error(\n `Invalid publication name \"${name}\". Publication names must start with a letter or underscore ` +\n `and contain only letters, digits, and underscores.`,\n );\n }\n if (name.length > 63) {\n throw new Error(\n `Publication name \"${name}\" exceeds PostgreSQL's 63-character identifier limit.`,\n );\n }\n}\n\nexport function internalPublicationPrefix({appID}: AppID) {\n return `_${appID}_`;\n}\n\nexport function legacyReplicationSlot({appID, shardNum}: ShardID) {\n return `${appID}_${shardNum}`;\n}\n\nexport function replicationSlotPrefix(shard: ShardID) {\n const {appID, shardNum} = check(shard);\n return `${appID}_${shardNum}_`;\n}\n\n/**\n * An expression used to match replication slots in the shard\n * in a Postgres `LIKE` operator.\n */\nexport function replicationSlotExpression(shard: ShardID) {\n // Underscores have a special meaning in LIKE values\n // so they have to be escaped.\n return `${replicationSlotPrefix(shard)}%`.replaceAll('_', '\\\\_');\n}\n\nfunction defaultPublicationName(appID: string, shardID: string | number) {\n return `_${appID}_public_${shardID}`;\n}\n\nexport function metadataPublicationName(\n appID: string,\n shardID: string | number,\n) {\n return `_${appID}_metadata_${shardID}`;\n}\n\n// The GLOBAL_SETUP must be idempotent as it can be run multiple times for different shards.\nfunction globalSetup(appID: AppID): string {\n const app = id(appSchema(appID));\n\n return /*sql*/ `\n CREATE SCHEMA IF NOT EXISTS ${app};\n\n CREATE TABLE IF NOT EXISTS ${app}.permissions (\n \"permissions\" JSONB,\n \"hash\" TEXT,\n\n -- Ensure that there is only a single row in the table.\n -- Application code can be agnostic to this column, and\n -- simply invoke UPDATE statements on the version columns.\n \"lock\" BOOL PRIMARY KEY DEFAULT true CHECK (lock)\n );\n\n CREATE OR REPLACE FUNCTION ${app}.set_permissions_hash()\n RETURNS TRIGGER AS $$\n BEGIN\n NEW.hash = md5(NEW.permissions::text);\n RETURN NEW;\n END;\n $$ LANGUAGE plpgsql;\n\n CREATE OR REPLACE TRIGGER on_set_permissions \n BEFORE INSERT OR UPDATE ON ${app}.permissions\n FOR EACH ROW\n EXECUTE FUNCTION ${app}.set_permissions_hash();\n\n INSERT INTO ${app}.permissions (permissions) VALUES (NULL) ON CONFLICT DO NOTHING;\n`;\n}\n\nexport async function ensureGlobalTables(db: PostgresDB, appID: AppID) {\n await db.unsafe(globalSetup(appID));\n}\n\nexport function getClientsTableDefinition(schema: string) {\n return /*sql*/ `\n CREATE TABLE ${schema}.\"clients\" (\n \"clientGroupID\" TEXT NOT NULL,\n \"clientID\" TEXT NOT NULL,\n \"lastMutationID\" BIGINT NOT NULL,\n \"userID\" TEXT,\n PRIMARY KEY(\"clientGroupID\", \"clientID\")\n );`;\n}\n\n/**\n * Tracks the results of mutations.\n * 1. It is an error for the same mutation ID to be used twice.\n * 2. The result is JSONB to allow for arbitrary results.\n *\n * The tables must be cleaned up as the clients\n * receive the mutation responses and as clients are removed.\n */\nexport function getMutationsTableDefinition(schema: string) {\n return /*sql*/ `\n CREATE TABLE ${schema}.\"mutations\" (\n \"clientGroupID\" TEXT NOT NULL,\n \"clientID\" TEXT NOT NULL,\n \"mutationID\" BIGINT NOT NULL,\n \"result\" JSON NOT NULL,\n PRIMARY KEY(\"clientGroupID\", \"clientID\", \"mutationID\")\n );`;\n}\n\nexport const SHARD_CONFIG_TABLE = 'shardConfig';\n\nexport function shardSetup(\n shardConfig: ShardConfig,\n metadataPublication: string,\n): string {\n const app = id(appSchema(shardConfig));\n const shard = id(upstreamSchema(shardConfig));\n\n const pubs = shardConfig.publications.toSorted();\n assert(\n pubs.includes(metadataPublication),\n () => `Publications must include ${metadataPublication}`,\n );\n\n return /*sql*/ `\n CREATE SCHEMA IF NOT EXISTS ${shard};\n\n ${getClientsTableDefinition(shard)}\n ${getMutationsTableDefinition(shard)}\n\n DROP PUBLICATION IF EXISTS ${id(metadataPublication)};\n CREATE PUBLICATION ${id(metadataPublication)}\n FOR TABLE ${app}.\"permissions\", TABLE ${shard}.\"clients\", ${shard}.\"mutations\";\n\n CREATE TABLE ${shard}.\"${SHARD_CONFIG_TABLE}\" (\n \"publications\" TEXT[] NOT NULL,\n \"ddlDetection\" BOOL NOT NULL,\n\n -- Ensure that there is only a single row in the table.\n \"lock\" BOOL PRIMARY KEY DEFAULT true CHECK (lock)\n );\n\n INSERT INTO ${shard}.\"${SHARD_CONFIG_TABLE}\" (\n \"publications\",\n \"ddlDetection\" \n ) VALUES (\n ARRAY[${literal(pubs)}], \n false -- set in SAVEPOINT with triggerSetup() statements\n );\n\n CREATE TABLE ${shard}.replicas (\n -- The DEFAULT exists purely for backwards compatibility support.\n -- New code always specifies a value based on Date.now().\n \"id\" TEXT PRIMARY KEY DEFAULT replace(gen_random_uuid()::text, '-', ''),\n \"rank\" BIGSERIAL,\n \"slot\" TEXT NOT NULL,\n \"version\" TEXT NOT NULL,\n \"generation\" TEXT, -- to replace version, NULL-able in the interim\n \"backupPath\" TEXT, -- subpath within the litestream backup URL\n \"backupV5\" BOOL DEFAULT false,\n \"initialSchema\" JSON, -- set after initial sync\n \"initialSyncContext\" JSON,\n \"subscriberContext\" JSON\n );\n `;\n}\n\nexport function dropShard(appID: string, shardID: string | number): string {\n const schema = `${appID}_${shardID}`;\n const metadataPublication = metadataPublicationName(appID, shardID);\n const defaultPublication = defaultPublicationName(appID, shardID);\n\n // DROP SCHEMA ... CASCADE does not drop dependent PUBLICATIONS,\n // so PUBLICATIONs must be dropped explicitly.\n return /*sql*/ `\n DROP PUBLICATION IF EXISTS ${id(defaultPublication)};\n DROP PUBLICATION IF EXISTS ${id(metadataPublication)};\n DROP SCHEMA IF EXISTS ${id(schema)} CASCADE;\n `;\n}\n\nconst internalShardConfigSchema = v.object({\n publications: v.array(v.string()),\n ddlDetection: v.boolean(),\n});\n\nexport type InternalShardConfig = v.Infer<typeof internalShardConfigSchema>;\n\nconst replicaInfoSchema = v.object({\n id: v.string(),\n rank: v.bigint(),\n slot: v.string(),\n version: v.string(),\n generation: v.string().nullable(),\n backupPath: v.string().nullable(),\n backupV5: v.boolean(),\n});\n\nconst fullReplicaRowSchema = replicaInfoSchema.extend({\n initialSchema: publishedSchema,\n initialSyncContext: jsonObjectSchema.nullable(),\n subscriberContext: jsonObjectSchema.nullable(),\n});\n\nconst replicaSchema = internalShardConfigSchema\n .extend(fullReplicaRowSchema.shape)\n .map(replica => {\n const {version, generation, ...r} = replica;\n return {...r, generation: generation ?? version};\n });\n\nexport type Replica = v.Infer<typeof replicaSchema>;\n\nconst replicationSlotStateSchema = v.object({\n active: v.boolean(),\n confirmedFlushLsn: v.string().nullable(),\n});\n\nconst replicaStateSchema = replicaInfoSchema\n .extend(replicationSlotStateSchema.shape)\n .map(replica => {\n const {version, generation, ...r} = replica;\n return {...r, generation: generation ?? version};\n });\n\nexport type ReplicaState = v.Infer<typeof replicaStateSchema>;\n\n// triggerSetup is run separately in a sub-transaction (i.e. SAVEPOINT) so\n// that a failure (e.g. due to lack of superuser permissions) can be handled\n// by continuing in a degraded mode (ddlDetection = false).\nfunction triggerSetup(shard: ShardConfig): string {\n const schema = id(upstreamSchema(shard));\n return (\n createEventTriggerStatements(shard) +\n /*sql*/ `UPDATE ${schema}.\"shardConfig\" SET \"ddlDetection\" = true;`\n );\n}\n\nexport type BackupOptions = {\n backupPath: string | null;\n backupV5: boolean;\n};\n\n/**\n * Creates a new replica to mark it as the owner of a specified `slot`.\n * This should be done with an advisory lock for replica/slot management\n * to ensure that the slot does not get dropped by concurrent cleanup\n * logic.\n *\n * Once initial sync is complete, {@link initReplica} should be called to\n * make the replica usable for incremental sync.\n */\nexport async function createReplica(\n sql: PostgresDB,\n shard: ShardID,\n id: string,\n slot: string,\n replicaVersion: string,\n {backupPath, backupV5}: BackupOptions,\n) {\n const schema = upstreamSchema(shard);\n const values: Partial<v.Infer<typeof fullReplicaRowSchema>> = {\n id,\n slot,\n version: replicaVersion,\n generation: replicaVersion,\n backupPath,\n backupV5,\n };\n await sql`INSERT INTO ${sql(schema)}.replicas ${sql(values)}`;\n}\n\n// Called in initial-sync to store the exact schema that was initially synced.\nexport async function initReplica(\n sql: PostgresDB,\n shard: ShardID,\n id: string,\n {tables, indexes}: PublishedSchema,\n initialSyncContext: JSONObject,\n) {\n const schema = upstreamSchema(shard);\n const synced: PublishedSchema = {tables, indexes};\n const values = {initialSchema: synced, initialSyncContext};\n await sql`UPDATE ${sql(schema)}.replicas SET ${sql(values)} WHERE id = ${id}`;\n}\n\n/**\n * Gets at the given version and optional ID. If no ID is specified, gets\n * the latest (i.e. highest rank) replica with that version.\n */\nexport async function getReplicaAtVersion(\n lc: LogContext,\n sql: PostgresDB,\n shard: ShardID,\n replicaVersion: string,\n id: string | null = null,\n context?: JSONObject,\n): Promise<Replica | null> {\n const schema = sql(upstreamSchema(shard));\n const result = await sql`\n SELECT\n replicas.\"id\",\n replicas.\"rank\",\n replicas.\"slot\",\n replicas.\"version\",\n replicas.\"generation\",\n replicas.\"backupPath\",\n replicas.\"backupV5\",\n replicas.\"initialSchema\",\n replicas.\"initialSyncContext\",\n replicas.\"subscriberContext\",\n \"shardConfig\".\"publications\",\n \"shardConfig\".\"ddlDetection\"\n FROM ${schema}.replicas JOIN ${schema}.\"shardConfig\" ON true\n WHERE version = ${replicaVersion} AND \"initialSyncContext\" IS NOT NULL\n AND id = COALESCE(${id}, id)\n ORDER BY rank DESC LIMIT 1;\n `;\n if (result.length === 0) {\n // log out all the replicas and the joined shardConfig\n const allReplicas = await sql`\n SELECT id, slot, version, \"initialSyncContext\", \"subscriberContext\" \n FROM ${schema}.replicas`;\n lc.info?.(\n `Replica ${id}@${replicaVersion}` +\n (context ? `(context: ${stringify(context)}) ` : '') +\n `not found in: ${stringify(allReplicas)}`,\n );\n return null;\n }\n return v.parse(result[0], replicaSchema, 'passthrough');\n}\n\nexport async function getActiveReplicas(\n lc: LogContext,\n sql: PostgresDB,\n shard: ShardID,\n): Promise<ReplicaState[]> {\n const schema = sql(upstreamSchema(shard));\n const results = await sql`\n SELECT\n replicas.\"id\",\n replicas.\"rank\",\n replicas.\"slot\",\n replicas.\"version\",\n replicas.\"generation\",\n replicas.\"backupPath\",\n replicas.\"backupV5\",\n slots.\"active\",\n slots.\"confirmed_flush_lsn\" as \"confirmedFlushLsn\"\n FROM ${schema}.replicas JOIN pg_replication_slots slots ON slot = slot_name\n WHERE \"initialSyncContext\" IS NOT NULL\n ORDER BY generation DESC, confirmed_flush_lsn DESC;\n `;\n const replicas = v.parse(results, v.array(replicaStateSchema), 'passthrough');\n lc.info?.(`current replicas`, {replicas});\n return replicas;\n}\n\nexport async function getReplicaState(\n sql: PostgresDB,\n shard: ShardID,\n id: string,\n): Promise<ReplicaState | null> {\n const schema = sql(upstreamSchema(shard));\n const results = await sql`\n SELECT\n replicas.\"id\",\n replicas.\"rank\",\n replicas.\"slot\",\n replicas.\"version\",\n replicas.\"generation\",\n replicas.\"backupPath\",\n replicas.\"backupV5\",\n slots.\"active\",\n slots.\"confirmed_flush_lsn\" as \"confirmedFlushLsn\"\n FROM ${schema}.replicas JOIN pg_replication_slots slots ON slot = slot_name\n WHERE \"id\" = ${id};\n `;\n return results.length\n ? v.parse(results[0], replicaStateSchema, 'passthrough')\n : null;\n}\n\nexport async function getInternalShardConfig(\n sql: PostgresDB,\n shard: ShardID,\n): Promise<InternalShardConfig> {\n const result = await sql`\n SELECT \"publications\", \"ddlDetection\"\n FROM ${sql(upstreamSchema(shard))}.\"shardConfig\";\n `;\n assert(\n result.length === 1,\n () => `Expected exactly one shardConfig row, got ${result.length}`,\n );\n return v.parse(result[0], internalShardConfigSchema, 'passthrough');\n}\n\n/**\n * Sets up and returns all publications (including internal ones) for\n * the given shard.\n */\nexport async function setupTablesAndReplication(\n lc: LogContext,\n sql: PostgresTransaction,\n requested: ShardConfig,\n includePartialIndexes = true,\n) {\n const {publications} = requested;\n // Validate requested publications.\n for (const pub of publications) {\n validatePublicationName(pub);\n if (pub.startsWith('_')) {\n throw new Error(\n `Publication names starting with \"_\" are reserved for internal use.\\n` +\n `Please use a different name for publication \"${pub}\".`,\n );\n }\n }\n const allPublications: string[] = [];\n\n // Setup application publications.\n if (publications.length) {\n const results = await sql<{pubname: string}[]>`\n SELECT pubname from pg_publication WHERE pubname IN ${sql(\n publications,\n )}`.values();\n\n if (results.length !== publications.length) {\n throw new Error(\n `Unknown or invalid publications. Specified: [${publications}]. Found: [${results.flat()}]`,\n );\n }\n allPublications.push(...publications);\n } else {\n const defaultPublication = defaultPublicationName(\n requested.appID,\n requested.shardNum,\n );\n await sql`\n DROP PUBLICATION IF EXISTS ${sql(defaultPublication)}`;\n await sql`\n CREATE PUBLICATION ${sql(defaultPublication)} \n FOR TABLES IN SCHEMA public\n WITH (publish_via_partition_root = true)`;\n allPublications.push(defaultPublication);\n }\n\n const metadataPublication = metadataPublicationName(\n requested.appID,\n requested.shardNum,\n );\n allPublications.push(metadataPublication);\n\n const shard = {...requested, publications: allPublications};\n\n // Setup the global tables and shard tables / publications.\n await sql.unsafe(globalSetup(shard) + shardSetup(shard, metadataPublication));\n\n const pubs = await getPublicationInfo(sql, allPublications);\n await replicaIdentitiesForTablesWithoutPrimaryKeys(pubs)?.apply(lc, sql);\n\n await setupTriggers(lc, sql, shard, includePartialIndexes);\n}\n\nexport async function setupTriggers(\n lc: LogContext,\n tx: PostgresTransaction,\n shard: ShardConfig,\n includePartialIndexes = true,\n) {\n const schema = upstreamSchema(shard);\n const [{ddlDetection}] = await tx<InternalShardConfig[]> /*sql*/ `\n SELECT \"ddlDetection\" FROM ${tx(schema)}.\"shardConfig\"`;\n\n // The functions invoked by event triggers are installed even if\n // event triggers are not supported/allowed by the db provider.\n // This allows users to manually invoke the update_schemas() function\n // as a workaround.\n await tx.unsafe(createEventFunctionStatements(shard, includePartialIndexes));\n try {\n await tx.savepoint(sub => sub.unsafe(triggerSetup(shard)));\n } catch (e) {\n if (ddlDetection) {\n // If ddlDetection has already been enabled, subsequent failures to\n // upgrade the trigger should be propagated rather than swallowed.\n throw e;\n }\n if (\n !(\n e instanceof postgres.PostgresError &&\n e.code === PG_INSUFFICIENT_PRIVILEGE\n )\n ) {\n throw e;\n }\n // If triggerSetup() fails, replication continues in ddlDetection=false mode.\n lc.warn?.(\n `Unable to create event triggers for schema change detection:\\n\\n` +\n `\"${e.hint ?? e.message}\"\\n\\n` +\n `Proceeding in degraded mode: schema changes will halt replication,\\n` +\n `requiring the replica to be reset (manually or with --auto-reset).`,\n );\n }\n}\n\nexport function validatePublications(\n lc: LogContext,\n published: PublicationInfo,\n) {\n // Verify that all publications export the proper events.\n published.publications.forEach(pub => {\n if (\n !pub.pubinsert ||\n !pub.pubupdate ||\n !pub.pubdelete ||\n !pub.pubtruncate\n ) {\n // TODO: Make APIError?\n throw new Error(\n `PUBLICATION ${pub.pubname} must publish insert, update, delete, and truncate`,\n );\n }\n });\n\n published.tables.forEach(table => validate(lc, table));\n}\n\ntype ReplicaIdentities = {\n apply(lc: LogContext, db: PostgresDB): Promise<void>;\n};\n\nexport function replicaIdentitiesForTablesWithoutPrimaryKeys(\n pubs: PublishedSchema,\n): ReplicaIdentities | undefined {\n const replicaIdentities: {\n schema: string;\n tableName: string;\n indexName: string;\n }[] = [];\n for (const table of pubs.tables) {\n if (!table.primaryKey?.length && table.replicaIdentity === Default) {\n // Look for an index that can serve as the REPLICA IDENTITY USING INDEX. It must be:\n // - UNIQUE\n // - NOT NULL columns\n // - not deferrable (i.e. isImmediate)\n // - not partial\n //\n // https://www.postgresql.org/docs/current/sql-altertable.html#SQL-ALTERTABLE-REPLICA-IDENTITY\n const {schema, name: tableName} = table;\n for (const {columns, name: indexName} of pubs.indexes.filter(\n idx =>\n idx.schema === schema &&\n idx.tableName === tableName &&\n idx.unique &&\n idx.isImmediate &&\n idx.predicate === undefined,\n )) {\n if (Object.keys(columns).some(col => !table.columns[col].notNull)) {\n continue; // Only indexes with all NOT NULL columns are suitable.\n }\n replicaIdentities.push({schema, tableName, indexName});\n break;\n }\n }\n }\n\n if (replicaIdentities.length === 0) {\n return undefined;\n }\n return {\n apply: async (lc: LogContext, sql: PostgresDB) => {\n for (const {schema, tableName, indexName} of replicaIdentities) {\n lc.info?.(\n `setting \"${indexName}\" as the REPLICA IDENTITY for \"${tableName}\"`,\n );\n await sql`\n ALTER TABLE ${sql(schema)}.${sql(tableName)} \n REPLICA IDENTITY USING INDEX ${sql(indexName)}`;\n }\n },\n };\n}\n"],"mappings":";;;;;;;;;;;;;;;;AAgCA,IAAM,yBAAyB;;;;;;AAO/B,SAAgB,wBAAwB,MAAoB;CAC1D,IAAI,CAAC,uBAAuB,KAAK,IAAI,GACnC,MAAM,IAAI,MACR,6BAA6B,KAAK,+GAEpC;CAEF,IAAI,KAAK,SAAS,IAChB,MAAM,IAAI,MACR,qBAAqB,KAAK,sDAC5B;AAEJ;AAEA,SAAgB,0BAA0B,EAAC,SAAe;CACxD,OAAO,IAAI,MAAM;AACnB;AAEA,SAAgB,sBAAsB,EAAC,OAAO,YAAoB;CAChE,OAAO,GAAG,MAAM,GAAG;AACrB;AAEA,SAAgB,sBAAsB,OAAgB;CACpD,MAAM,EAAC,OAAO,aAAY,MAAM,KAAK;CACrC,OAAO,GAAG,MAAM,GAAG,SAAS;AAC9B;;;;;AAMA,SAAgB,0BAA0B,OAAgB;CAGxD,OAAO,GAAG,sBAAsB,KAAK,EAAE,GAAG,WAAW,KAAK,KAAK;AACjE;AAEA,SAAS,uBAAuB,OAAe,SAA0B;CACvE,OAAO,IAAI,MAAM,UAAU;AAC7B;AAEA,SAAgB,wBACd,OACA,SACA;CACA,OAAO,IAAI,MAAM,YAAY;AAC/B;AAGA,SAAS,YAAY,OAAsB;CACzC,MAAM,MAAM,GAAG,UAAU,KAAK,CAAC;CAE/B,OAAe;gCACe,IAAI;;+BAEL,IAAI;;;;;;;;;;+BAUJ,IAAI;;;;;;;;;iCASF,IAAI;;uBAEd,IAAI;;gBAEX,IAAI;;AAEpB;AAEA,eAAsB,mBAAmB,IAAgB,OAAc;CACrE,MAAM,GAAG,OAAO,YAAY,KAAK,CAAC;AACpC;AAEA,SAAgB,0BAA0B,QAAgB;CACxD,OAAe;iBACA,OAAO;;;;;;;AAOxB;;;;;;;;;AAUA,SAAgB,4BAA4B,QAAgB;CAC1D,OAAe;iBACA,OAAO;;;;;;;AAOxB;AAEA,IAAa,qBAAqB;AAElC,SAAgB,WACd,aACA,qBACQ;CACR,MAAM,MAAM,GAAG,UAAU,WAAW,CAAC;CACrC,MAAM,QAAQ,GAAG,eAAe,WAAW,CAAC;CAE5C,MAAM,OAAO,YAAY,aAAa,SAAS;CAC/C,OACE,KAAK,SAAS,mBAAmB,SAC3B,6BAA6B,qBACrC;CAEA,OAAe;gCACe,MAAM;;IAElC,0BAA0B,KAAK,EAAE;IACjC,4BAA4B,KAAK,EAAE;;+BAER,GAAG,mBAAmB,EAAE;uBAChC,GAAG,mBAAmB,EAAE;gBAC/B,IAAI,wBAAwB,MAAM,cAAc,MAAM;;iBAErD,MAAM,IAAI,mBAAmB;;;;;;;;gBAQ9B,MAAM,IAAI,mBAAmB;;;;cAI/B,QAAQ,IAAI,EAAE;;;;iBAIX,MAAM;;;;;;;;;;;;;;;AAevB;AAEA,SAAgB,UAAU,OAAe,SAAkC;CACzE,MAAM,SAAS,GAAG,MAAM,GAAG;CAC3B,MAAM,sBAAsB,wBAAwB,OAAO,OAAO;CAKlE,OAAe;iCACgB,GALJ,uBAAuB,OAAO,OAKvB,CAAkB,EAAE;iCACvB,GAAG,mBAAmB,EAAE;4BAC7B,GAAG,MAAM,EAAE;;AAEvC;AAEA,IAAM,4BAA4B,eAAE,OAAO;CACzC,cAAc,eAAE,MAAM,eAAE,OAAO,CAAC;CAChC,cAAc,eAAE,QAAQ;AAC1B,CAAC;AAID,IAAM,oBAAoB,eAAE,OAAO;CACjC,IAAI,eAAE,OAAO;CACb,MAAM,eAAE,OAAO;CACf,MAAM,eAAE,OAAO;CACf,SAAS,eAAE,OAAO;CAClB,YAAY,eAAE,OAAO,EAAE,SAAS;CAChC,YAAY,eAAE,OAAO,EAAE,SAAS;CAChC,UAAU,eAAE,QAAQ;AACtB,CAAC;AAED,IAAM,uBAAuB,kBAAkB,OAAO;CACpD,eAAe;CACf,oBAAoB,iBAAiB,SAAS;CAC9C,mBAAmB,iBAAiB,SAAS;AAC/C,CAAC;AAED,IAAM,gBAAgB,0BACnB,OAAO,qBAAqB,KAAK,EACjC,KAAI,YAAW;CACd,MAAM,EAAC,SAAS,YAAY,GAAG,MAAK;CACpC,OAAO;EAAC,GAAG;EAAG,YAAY,cAAc;CAAO;AACjD,CAAC;AAIH,IAAM,6BAA6B,eAAE,OAAO;CAC1C,QAAQ,eAAE,QAAQ;CAClB,mBAAmB,eAAE,OAAO,EAAE,SAAS;AACzC,CAAC;AAED,IAAM,qBAAqB,kBACxB,OAAO,2BAA2B,KAAK,EACvC,KAAI,YAAW;CACd,MAAM,EAAC,SAAS,YAAY,GAAG,MAAK;CACpC,OAAO;EAAC,GAAG;EAAG,YAAY,cAAc;CAAO;AACjD,CAAC;AAOH,SAAS,aAAa,OAA4B;CAChD,MAAM,SAAS,GAAG,eAAe,KAAK,CAAC;CACvC,OACE,6BAA6B,KAAK,IAC1B,UAAU,OAAO;AAE7B;;;;;;;;;;AAgBA,eAAsB,cACpB,KACA,OACA,IACA,MACA,gBACA,EAAC,YAAY,YACb;CACA,MAAM,SAAS,eAAe,KAAK;CACnC,MAAM,SAAwD;EAC5D;EACA;EACA,SAAS;EACT,YAAY;EACZ;EACA;CACF;CACA,MAAM,GAAG,eAAe,IAAI,MAAM,EAAE,YAAY,IAAI,MAAM;AAC5D;AAGA,eAAsB,YACpB,KACA,OACA,IACA,EAAC,QAAQ,WACT,oBACA;CACA,MAAM,SAAS,eAAe,KAAK;CAEnC,MAAM,SAAS;EAAC,eAAe;GADE;GAAQ;EACV;EAAQ;CAAkB;CACzD,MAAM,GAAG,UAAU,IAAI,MAAM,EAAE,gBAAgB,IAAI,MAAM,EAAE,cAAc;AAC3E;;;;;AAMA,eAAsB,oBACpB,IACA,KACA,OACA,gBACA,KAAoB,MACpB,SACyB;CACzB,MAAM,SAAS,IAAI,eAAe,KAAK,CAAC;CACxC,MAAM,SAAS,MAAM,GAAG;;;;;;;;;;;;;;WAcf,OAAO,iBAAiB,OAAO;wBAClB,eAAe;0BACb,GAAG;;;CAG3B,IAAI,OAAO,WAAW,GAAG;EAEvB,MAAM,cAAc,MAAM,GAAG;;eAElB,OAAO;EAClB,GAAG,OACD,WAAW,GAAG,GAAG,oBACd,UAAU,aAAa,UAAU,OAAO,EAAE,MAAM,MACjD,iBAAiB,UAAU,WAAW,GAC1C;EACA,OAAO;CACT;CACA,OAAO,MAAQ,OAAO,IAAI,eAAe,aAAa;AACxD;AAEA,eAAsB,kBACpB,IACA,KACA,OACyB;CAiBzB,MAAM,WAAW,MAAQ,MAfH,GAAG;;;;;;;;;;;WADV,IAAI,eAAe,KAAK,CAY9B,EAAO;;;KAIkB,eAAE,MAAM,kBAAkB,GAAG,aAAa;CAC5E,GAAG,OAAO,oBAAoB,EAAC,SAAQ,CAAC;CACxC,OAAO;AACT;AAEA,eAAsB,gBACpB,KACA,OACA,IAC8B;CAE9B,MAAM,UAAU,MAAM,GAAG;;;;;;;;;;;WADV,IAAI,eAAe,KAAK,CAY9B,EAAO;qBACG,GAAG;;CAEtB,OAAO,QAAQ,SACX,MAAQ,QAAQ,IAAI,oBAAoB,aAAa,IACrD;AACN;AAEA,eAAsB,uBACpB,KACA,OAC8B;CAC9B,MAAM,SAAS,MAAM,GAAG;;aAEb,IAAI,eAAe,KAAK,CAAC,EAAE;;CAEtC,OACE,OAAO,WAAW,SACZ,6CAA6C,OAAO,QAC5D;CACA,OAAO,MAAQ,OAAO,IAAI,2BAA2B,aAAa;AACpE;;;;;AAMA,eAAsB,0BACpB,IACA,KACA,WACA,wBAAwB,MACxB;CACA,MAAM,EAAC,iBAAgB;CAEvB,KAAK,MAAM,OAAO,cAAc;EAC9B,wBAAwB,GAAG;EAC3B,IAAI,IAAI,WAAW,GAAG,GACpB,MAAM,IAAI,MACR,oHACkD,IAAI,GACxD;CAEJ;CACA,MAAM,kBAA4B,CAAC;CAGnC,IAAI,aAAa,QAAQ;EACvB,MAAM,UAAU,MAAM,GAAwB;0DACQ,IACpD,YACF,IAAI,OAAO;EAEX,IAAI,QAAQ,WAAW,aAAa,QAClC,MAAM,IAAI,MACR,gDAAgD,aAAa,aAAa,QAAQ,KAAK,EAAE,EAC3F;EAEF,gBAAgB,KAAK,GAAG,YAAY;CACtC,OAAO;EACL,MAAM,qBAAqB,uBACzB,UAAU,OACV,UAAU,QACZ;EACA,MAAM,GAAG;mCACsB,IAAI,kBAAkB;EACrD,MAAM,GAAG;2BACc,IAAI,kBAAkB,EAAE;;;EAG/C,gBAAgB,KAAK,kBAAkB;CACzC;CAEA,MAAM,sBAAsB,wBAC1B,UAAU,OACV,UAAU,QACZ;CACA,gBAAgB,KAAK,mBAAmB;CAExC,MAAM,QAAQ;EAAC,GAAG;EAAW,cAAc;CAAe;CAG1D,MAAM,IAAI,OAAO,YAAY,KAAK,IAAI,WAAW,OAAO,mBAAmB,CAAC;CAG5E,MAAM,6CAA6C,MADhC,mBAAmB,KAAK,eAAe,CACH,GAAG,MAAM,IAAI,GAAG;CAEvE,MAAM,cAAc,IAAI,KAAK,OAAO,qBAAqB;AAC3D;AAEA,eAAsB,cACpB,IACA,IACA,OACA,wBAAwB,MACxB;CAEA,MAAM,CAAC,EAAC,kBAAiB,MAAM,EAAkC;iCAClC,GAFhB,eAAe,KAEI,CAAM,EAAE;CAM1C,MAAM,GAAG,OAAO,8BAA8B,OAAO,qBAAqB,CAAC;CAC3E,IAAI;EACF,MAAM,GAAG,WAAU,QAAO,IAAI,OAAO,aAAa,KAAK,CAAC,CAAC;CAC3D,SAAS,GAAG;EACV,IAAI,cAGF,MAAM;EAER,IACE,EACE,aAAa,SAAS,iBACtB,EAAE,SAAS,4BAGb,MAAM;EAGR,GAAG,OACD,oEACM,EAAE,QAAQ,EAAE,QAAQ,4IAG5B;CACF;AACF;AAEA,SAAgB,qBACd,IACA,WACA;CAEA,UAAU,aAAa,SAAQ,QAAO;EACpC,IACE,CAAC,IAAI,aACL,CAAC,IAAI,aACL,CAAC,IAAI,aACL,CAAC,IAAI,aAGL,MAAM,IAAI,MACR,eAAe,IAAI,QAAQ,mDAC7B;CAEJ,CAAC;CAED,UAAU,OAAO,SAAQ,UAAS,SAAS,IAAI,KAAK,CAAC;AACvD;AAMA,SAAgB,6CACd,MAC+B;CAC/B,MAAM,oBAIA,CAAC;CACP,KAAK,MAAM,SAAS,KAAK,QACvB,IAAI,CAAC,MAAM,YAAY,UAAU,MAAM,oBAAA,KAA6B;EAQlE,MAAM,EAAC,QAAQ,MAAM,cAAa;EAClC,KAAK,MAAM,EAAC,SAAS,MAAM,eAAc,KAAK,QAAQ,QACpD,QACE,IAAI,WAAW,UACf,IAAI,cAAc,aAClB,IAAI,UACJ,IAAI,eACJ,IAAI,cAAc,KAAA,CACtB,GAAG;GACD,IAAI,OAAO,KAAK,OAAO,EAAE,MAAK,QAAO,CAAC,MAAM,QAAQ,KAAK,OAAO,GAC9D;GAEF,kBAAkB,KAAK;IAAC;IAAQ;IAAW;GAAS,CAAC;GACrD;EACF;CACF;CAGF,IAAI,kBAAkB,WAAW,GAC/B;CAEF,OAAO,EACL,OAAO,OAAO,IAAgB,QAAoB;EAChD,KAAK,MAAM,EAAC,QAAQ,WAAW,eAAc,mBAAmB;GAC9D,GAAG,OACD,YAAY,UAAU,iCAAiC,UAAU,EACnE;GACA,MAAM,GAAG;sBACK,IAAI,MAAM,EAAE,GAAG,IAAI,SAAS,EAAE;yCACX,IAAI,SAAS;EAChD;CACF,EACF;AACF"}
|
|
1
|
+
{"version":3,"file":"shard.js","names":[],"sources":["../../../../../../../../zero-cache/src/services/change-source/pg/schema/shard.ts"],"sourcesContent":["import {PG_INSUFFICIENT_PRIVILEGE} from '@drdgvhbh/postgres-error-codes';\nimport type {LogContext} from '@rocicorp/logger';\nimport {literal} from 'pg-format';\nimport postgres from 'postgres';\nimport {assert} from '../../../../../../shared/src/asserts.ts';\nimport {\n jsonObjectSchema,\n stringify,\n type JSONObject,\n} from '../../../../../../shared/src/bigint-json.ts';\nimport * as v from '../../../../../../shared/src/valita.ts';\nimport {Default} from '../../../../db/postgres-replica-identity-enum.ts';\nimport type {PostgresDB, PostgresTransaction} from '../../../../types/pg.ts';\nimport type {AppID, ShardConfig, ShardID} from '../../../../types/shards.ts';\nimport {appSchema, check, upstreamSchema} from '../../../../types/shards.ts';\nimport {id} from '../../../../types/sql.ts';\nimport {\n createEventFunctionStatements,\n createEventTriggerStatements,\n} from './ddl.ts';\nimport {\n getPublicationInfo,\n publishedSchema,\n type PublicationInfo,\n type PublishedSchema,\n} from './published.ts';\nimport {\n InitialSync,\n replicaStageSchema,\n type ReplicaStage,\n} from './replica-stage-enum.ts';\nimport {validate} from './validation.ts';\n\n/**\n * PostgreSQL unquoted identifiers must start with a letter or underscore\n * and contain only letters, digits, and underscores.\n */\nconst VALID_PUBLICATION_NAME = /^[a-zA-Z_][a-zA-Z0-9_]*$/;\n\n/**\n * Validates that a publication name is a valid PostgreSQL identifier.\n * This provides defense-in-depth against SQL injection when publication\n * names are used in replication commands.\n */\nexport function validatePublicationName(name: string): void {\n if (!VALID_PUBLICATION_NAME.test(name)) {\n throw new Error(\n `Invalid publication name \"${name}\". Publication names must start with a letter or underscore ` +\n `and contain only letters, digits, and underscores.`,\n );\n }\n if (name.length > 63) {\n throw new Error(\n `Publication name \"${name}\" exceeds PostgreSQL's 63-character identifier limit.`,\n );\n }\n}\n\nexport function internalPublicationPrefix({appID}: AppID) {\n return `_${appID}_`;\n}\n\nexport function legacyReplicationSlot({appID, shardNum}: ShardID) {\n return `${appID}_${shardNum}`;\n}\n\nexport function replicationSlotPrefix(shard: ShardID) {\n const {appID, shardNum} = check(shard);\n return `${appID}_${shardNum}_`;\n}\n\n/**\n * An expression used to match replication slots in the shard\n * in a Postgres `LIKE` operator.\n */\nexport function replicationSlotExpression(shard: ShardID) {\n // Underscores have a special meaning in LIKE values\n // so they have to be escaped.\n return `${replicationSlotPrefix(shard)}%`.replaceAll('_', '\\\\_');\n}\n\nfunction defaultPublicationName(appID: string, shardID: string | number) {\n return `_${appID}_public_${shardID}`;\n}\n\nexport function metadataPublicationName(\n appID: string,\n shardID: string | number,\n) {\n return `_${appID}_metadata_${shardID}`;\n}\n\n// The GLOBAL_SETUP must be idempotent as it can be run multiple times for different shards.\nfunction globalSetup(appID: AppID): string {\n const app = id(appSchema(appID));\n\n return /*sql*/ `\n CREATE SCHEMA IF NOT EXISTS ${app};\n\n CREATE TABLE IF NOT EXISTS ${app}.permissions (\n \"permissions\" JSONB,\n \"hash\" TEXT,\n\n -- Ensure that there is only a single row in the table.\n -- Application code can be agnostic to this column, and\n -- simply invoke UPDATE statements on the version columns.\n \"lock\" BOOL PRIMARY KEY DEFAULT true CHECK (lock)\n );\n\n CREATE OR REPLACE FUNCTION ${app}.set_permissions_hash()\n RETURNS TRIGGER AS $$\n BEGIN\n NEW.hash = md5(NEW.permissions::text);\n RETURN NEW;\n END;\n $$ LANGUAGE plpgsql;\n\n CREATE OR REPLACE TRIGGER on_set_permissions \n BEFORE INSERT OR UPDATE ON ${app}.permissions\n FOR EACH ROW\n EXECUTE FUNCTION ${app}.set_permissions_hash();\n\n INSERT INTO ${app}.permissions (permissions) VALUES (NULL) ON CONFLICT DO NOTHING;\n`;\n}\n\nexport async function ensureGlobalTables(db: PostgresDB, appID: AppID) {\n await db.unsafe(globalSetup(appID));\n}\n\nexport function getClientsTableDefinition(schema: string) {\n return /*sql*/ `\n CREATE TABLE ${schema}.\"clients\" (\n \"clientGroupID\" TEXT NOT NULL,\n \"clientID\" TEXT NOT NULL,\n \"lastMutationID\" BIGINT NOT NULL,\n \"userID\" TEXT,\n PRIMARY KEY(\"clientGroupID\", \"clientID\")\n );`;\n}\n\n/**\n * Tracks the results of mutations.\n * 1. It is an error for the same mutation ID to be used twice.\n * 2. The result is JSONB to allow for arbitrary results.\n *\n * The tables must be cleaned up as the clients\n * receive the mutation responses and as clients are removed.\n */\nexport function getMutationsTableDefinition(schema: string) {\n return /*sql*/ `\n CREATE TABLE ${schema}.\"mutations\" (\n \"clientGroupID\" TEXT NOT NULL,\n \"clientID\" TEXT NOT NULL,\n \"mutationID\" BIGINT NOT NULL,\n \"result\" JSON NOT NULL,\n PRIMARY KEY(\"clientGroupID\", \"clientID\", \"mutationID\")\n );`;\n}\n\nexport const SHARD_CONFIG_TABLE = 'shardConfig';\n\nexport function shardSetup(\n shardConfig: ShardConfig,\n metadataPublication: string,\n): string {\n const app = id(appSchema(shardConfig));\n const shard = id(upstreamSchema(shardConfig));\n\n const pubs = shardConfig.publications.toSorted();\n assert(\n pubs.includes(metadataPublication),\n () => `Publications must include ${metadataPublication}`,\n );\n\n return /*sql*/ `\n CREATE SCHEMA IF NOT EXISTS ${shard};\n\n ${getClientsTableDefinition(shard)}\n ${getMutationsTableDefinition(shard)}\n\n CREATE TABLE ${shard}.\"${SHARD_CONFIG_TABLE}\" (\n \"publications\" TEXT[] NOT NULL,\n \"ddlDetection\" BOOL NOT NULL,\n\n -- Ensure that there is only a single row in the table.\n \"lock\" BOOL PRIMARY KEY DEFAULT true CHECK (lock)\n );\n\n INSERT INTO ${shard}.\"${SHARD_CONFIG_TABLE}\" (\n \"publications\",\n \"ddlDetection\" \n ) VALUES (\n ARRAY[${literal(pubs)}], \n false -- set in SAVEPOINT with triggerSetup() statements\n );\n\n CREATE TABLE ${shard}.replicas (\n -- The DEFAULT exists purely for backwards compatibility support.\n -- New code always specifies a value based on Date.now().\n \"id\" TEXT PRIMARY KEY DEFAULT replace(gen_random_uuid()::text, '-', ''),\n \"rank\" BIGSERIAL,\n \"slot\" TEXT NOT NULL,\n \"version\" TEXT, -- replaced by generation, slated for deletion\n \"epoch\" INT4 DEFAULT 0,\n \"generation\" TEXT NOT NULL,\n \"stage\" INT4 DEFAULT ${InitialSync},\n \"backupPath\" TEXT, -- subpath within the litestream backup URL\n \"backupV5\" BOOL DEFAULT false,\n \"initialSchema\" JSON, -- set after initial sync\n \"initialSyncContext\" JSON,\n \"subscriberContext\" JSON\n );\n\n DROP PUBLICATION IF EXISTS ${id(metadataPublication)};\n CREATE PUBLICATION ${id(metadataPublication)}\n FOR TABLE ${app}.\"permissions\", \n TABLE ${shard}.\"clients\", \n ${shard}.\"mutations\", \n ${shard}.\"replicas\" (id); -- advance the LSN/version with new replicas\n `;\n}\n\nexport function dropShard(appID: string, shardID: string | number): string {\n const schema = `${appID}_${shardID}`;\n const metadataPublication = metadataPublicationName(appID, shardID);\n const defaultPublication = defaultPublicationName(appID, shardID);\n\n // DROP SCHEMA ... CASCADE does not drop dependent PUBLICATIONS,\n // so PUBLICATIONs must be dropped explicitly.\n return /*sql*/ `\n DROP PUBLICATION IF EXISTS ${id(defaultPublication)};\n DROP PUBLICATION IF EXISTS ${id(metadataPublication)};\n DROP SCHEMA IF EXISTS ${id(schema)} CASCADE;\n `;\n}\n\nconst internalShardConfigSchema = v.object({\n publications: v.array(v.string()),\n ddlDetection: v.boolean(),\n});\n\nexport type InternalShardConfig = v.Infer<typeof internalShardConfigSchema>;\n\nconst replicaInfoSchema = v.object({\n id: v.string(),\n rank: v.bigint(),\n slot: v.string(),\n epoch: v.number(),\n generation: v.string(),\n stage: replicaStageSchema,\n backupPath: v.string().nullable(),\n backupV5: v.boolean(),\n});\n\nconst fullReplicaRowSchema = replicaInfoSchema.extend({\n initialSchema: publishedSchema,\n initialSyncContext: jsonObjectSchema.nullable(),\n subscriberContext: jsonObjectSchema.nullable(),\n});\n\nconst replicaSchema = internalShardConfigSchema.extend(\n fullReplicaRowSchema.shape,\n);\n\nexport type Replica = v.Infer<typeof replicaSchema>;\n\nconst replicationSlotStateSchema = v.object({\n active: v.boolean(),\n confirmedFlushLsn: v.string(),\n});\n\nconst replicaStateSchema = replicaInfoSchema.extend(\n replicationSlotStateSchema.shape,\n);\n\nexport type ReplicaState = v.Infer<typeof replicaStateSchema>;\n\n// triggerSetup is run separately in a sub-transaction (i.e. SAVEPOINT) so\n// that a failure (e.g. due to lack of superuser permissions) can be handled\n// by continuing in a degraded mode (ddlDetection = false).\nfunction triggerSetup(shard: ShardConfig): string {\n const schema = id(upstreamSchema(shard));\n return (\n createEventTriggerStatements(shard) +\n /*sql*/ `UPDATE ${schema}.\"shardConfig\" SET \"ddlDetection\" = true;`\n );\n}\n\nexport type BackupOptions = {\n backupPath: string | null;\n backupV5: boolean;\n};\n\n/**\n * Creates a new replica to mark it as the owner of a specified `slot`.\n * This should be done with an advisory lock for replica/slot management\n * to ensure that the slot does not get dropped by concurrent cleanup\n * logic.\n *\n * Once initial sync is complete, {@link initInitialSyncReplica} should be called to\n * make the replica usable for incremental sync.\n */\nexport async function createReplica(\n sql: PostgresDB,\n shard: ShardID,\n id: string,\n slot: string,\n epoch: number,\n replicaVersion: string,\n {backupPath, backupV5}: BackupOptions,\n stage: ReplicaStage,\n) {\n const schema = upstreamSchema(shard);\n const values: Partial<v.Infer<typeof fullReplicaRowSchema>> = {\n id,\n slot,\n epoch,\n generation: replicaVersion,\n backupPath,\n backupV5,\n stage,\n };\n await sql`INSERT INTO ${sql(schema)}.replicas ${sql(values)}`;\n}\n\n// Called in initial-sync to store the exact schema that was initially synced.\nexport async function initInitialSyncReplica(\n sql: PostgresDB,\n shard: ShardID,\n id: string,\n {tables, indexes}: PublishedSchema,\n initialSyncContext: JSONObject,\n) {\n const schema = upstreamSchema(shard);\n const synced: PublishedSchema = {tables, indexes};\n const values = {initialSchema: synced, initialSyncContext};\n await sql`UPDATE ${sql(schema)}.replicas SET ${sql(values)} WHERE id = ${id}`;\n}\n\n// Called to initialize a Restore replica (`destID`) with the generation\n// and initial sync context of the `sourceID` replica.\nexport async function initRestoreReplica(\n sql: PostgresDB,\n shard: ShardID,\n {sourceID, destID}: {sourceID: string; destID: string},\n): Promise<ReplicaState | null> {\n const schema = upstreamSchema(shard);\n await sql`\n UPDATE ${sql(schema)}.replicas dest \n SET \"generation\" = source.\"generation\",\n \"initialSchema\" = source.\"initialSchema\",\n \"initialSyncContext\" = source.\"initialSyncContext\"\n FROM ${sql(schema)}.replicas source\n WHERE source.id = ${sourceID} AND dest.id = ${destID};\n `;\n return getReplicaState(sql, shard, destID);\n}\n\n/**\n * Gets at the given version and optional ID. If no ID is specified, gets\n * the latest (i.e. highest rank) replica with that version.\n */\nexport async function getReplicaAtVersion(\n lc: LogContext,\n sql: PostgresDB,\n shard: ShardID,\n replicaVersion: string,\n id: string | null = null,\n context?: JSONObject,\n): Promise<Replica | null> {\n const schema = sql(upstreamSchema(shard));\n const result = await sql`\n SELECT\n replicas.\"id\",\n replicas.\"rank\",\n replicas.\"slot\",\n replicas.\"epoch\",\n replicas.\"generation\",\n replicas.\"stage\",\n replicas.\"backupPath\",\n replicas.\"backupV5\",\n replicas.\"initialSchema\",\n replicas.\"initialSyncContext\",\n replicas.\"subscriberContext\",\n \"shardConfig\".\"publications\",\n \"shardConfig\".\"ddlDetection\"\n FROM ${schema}.replicas JOIN ${schema}.\"shardConfig\" ON true\n WHERE generation = ${replicaVersion} AND \"initialSyncContext\" IS NOT NULL\n AND id = COALESCE(${id}, id)\n ORDER BY rank DESC LIMIT 1;\n `;\n if (result.length === 0) {\n // log out all the replicas and the joined shardConfig\n const allReplicas = await sql`\n SELECT id, slot, generation, \"initialSyncContext\", \"subscriberContext\" \n FROM ${schema}.replicas`;\n lc.info?.(\n `Replica ${id}@${replicaVersion}` +\n (context ? `(context: ${stringify(context)}) ` : '') +\n `not found in: ${stringify(allReplicas)}`,\n );\n return null;\n }\n return v.parse(result[0], replicaSchema, 'passthrough');\n}\n\nexport async function getActiveReplicas(\n lc: LogContext,\n sql: PostgresDB,\n shard: ShardID,\n): Promise<ReplicaState[]> {\n const schema = sql(upstreamSchema(shard));\n const results = await sql`\n SELECT\n replicas.\"id\",\n replicas.\"rank\",\n replicas.\"slot\",\n replicas.\"epoch\",\n replicas.\"generation\",\n replicas.\"stage\",\n replicas.\"backupPath\",\n replicas.\"backupV5\",\n slots.\"active\",\n slots.\"confirmed_flush_lsn\" as \"confirmedFlushLsn\"\n FROM ${schema}.replicas JOIN pg_replication_slots slots ON slot = slot_name\n WHERE \"initialSyncContext\" IS NOT NULL\n -- Exclude replicas whose slot has been invalidated (e.g. for exceeding\n -- max_slot_wal_keep_size). Such a replica can never be resumed, and\n -- restoring it just results in an AutoResetSignal on every attempt.\n AND slots.restart_lsn IS NOT NULL\n AND slots.wal_status IS DISTINCT FROM 'lost'\n ORDER BY generation DESC, confirmed_flush_lsn DESC;\n `;\n const replicas = v.parse(results, v.array(replicaStateSchema), 'passthrough');\n lc.info?.(`current replicas`, {replicas});\n return replicas;\n}\n\n/**\n * Returns candidates for restoring a new replica from. These include\n * all replicas in the newest `generation` of the given `epoch`, in order\n * of earliest available backup:\n * - An active InitialSync replica, which will be the only replica in\n * its generation by definition. Orphaned/inactive initial syncs are skipped.\n * - Replicas in the Replicate stage, active first.\n * - Replicas in the Restore stage, potentially available for resumption\n * as a last resort.\n */\nexport async function getRestoreCandidates(\n lc: LogContext,\n sql: PostgresDB,\n shard: ShardID,\n epoch: number,\n): Promise<ReplicaState[]> {\n const schema = sql(upstreamSchema(shard));\n const results = await sql`\n SELECT\n replicas.\"id\",\n replicas.\"rank\",\n replicas.\"slot\",\n replicas.\"epoch\",\n replicas.\"generation\",\n replicas.\"stage\",\n replicas.\"backupPath\",\n replicas.\"backupV5\",\n slots.\"active\",\n slots.\"confirmed_flush_lsn\" as \"confirmedFlushLsn\"\n FROM ${schema}.replicas JOIN pg_replication_slots slots ON slot = slot_name\n\n -- coalesce on the newest generation in the epoch\n WHERE generation = (\n SELECT MAX(r.generation)\n FROM ${schema}.replicas r\n JOIN pg_replication_slots s ON r.slot = s.slot_name\n WHERE r.\"epoch\" = ${epoch} AND r.\"backupV5\"\n -- ignore orphaned InitialSyncs, which cannot be restored from\n AND (s.\"active\" OR r.\"stage\" != ${InitialSync})\n AND s.restart_lsn IS NOT NULL\n AND s.wal_status IS DISTINCT FROM 'lost'\n )\n AND \"epoch\" = ${epoch} AND \"backupV5\"\n AND (slots.active OR \"stage\" != ${InitialSync})\n AND slots.restart_lsn IS NOT NULL\n AND slots.wal_status IS DISTINCT FROM 'lost'\n\n ORDER BY \n stage ASC, -- ordered by backup availalibility\n active DESC, -- prefer active slots (fork over resume)\n confirmed_flush_lsn DESC; -- prefer replicas further ahead\n `;\n const replicas = v.parse(results, v.array(replicaStateSchema), 'passthrough');\n lc.info?.(`current replicas at epoch ${epoch}`, {replicas});\n return replicas;\n}\n\nexport async function getReplicaState(\n sql: PostgresDB,\n shard: ShardID,\n id: string,\n): Promise<ReplicaState | null> {\n const schema = sql(upstreamSchema(shard));\n const results = await sql`\n SELECT\n replicas.\"id\",\n replicas.\"rank\",\n replicas.\"slot\",\n replicas.\"epoch\",\n replicas.\"generation\",\n replicas.\"stage\",\n replicas.\"backupPath\",\n replicas.\"backupV5\",\n slots.\"active\",\n slots.\"confirmed_flush_lsn\" as \"confirmedFlushLsn\"\n FROM ${schema}.replicas JOIN pg_replication_slots slots ON slot = slot_name\n WHERE \"id\" = ${id};\n `;\n return results.length\n ? v.parse(results[0], replicaStateSchema, 'passthrough')\n : null;\n}\n\nexport async function getInternalShardConfig(\n sql: PostgresDB,\n shard: ShardID,\n): Promise<InternalShardConfig> {\n const result = await sql`\n SELECT \"publications\", \"ddlDetection\"\n FROM ${sql(upstreamSchema(shard))}.\"shardConfig\";\n `;\n assert(\n result.length === 1,\n () => `Expected exactly one shardConfig row, got ${result.length}`,\n );\n return v.parse(result[0], internalShardConfigSchema, 'passthrough');\n}\n\n/**\n * Sets up and returns all publications (including internal ones) for\n * the given shard.\n */\nexport async function setupTablesAndReplication(\n lc: LogContext,\n sql: PostgresTransaction,\n requested: ShardConfig,\n includePartialIndexes = true,\n) {\n const {publications} = requested;\n // Validate requested publications.\n for (const pub of publications) {\n validatePublicationName(pub);\n if (pub.startsWith('_')) {\n throw new Error(\n `Publication names starting with \"_\" are reserved for internal use.\\n` +\n `Please use a different name for publication \"${pub}\".`,\n );\n }\n }\n const allPublications: string[] = [];\n\n // Setup application publications.\n if (publications.length) {\n const results = await sql<{pubname: string}[]>`\n SELECT pubname from pg_publication WHERE pubname IN ${sql(\n publications,\n )}`.values();\n\n if (results.length !== publications.length) {\n throw new Error(\n `Unknown or invalid publications. Specified: [${publications}]. Found: [${results.flat()}]`,\n );\n }\n allPublications.push(...publications);\n } else {\n const defaultPublication = defaultPublicationName(\n requested.appID,\n requested.shardNum,\n );\n await sql`\n DROP PUBLICATION IF EXISTS ${sql(defaultPublication)}`;\n await sql`\n CREATE PUBLICATION ${sql(defaultPublication)} \n FOR TABLES IN SCHEMA public\n WITH (publish_via_partition_root = true)`;\n allPublications.push(defaultPublication);\n }\n\n const metadataPublication = metadataPublicationName(\n requested.appID,\n requested.shardNum,\n );\n allPublications.push(metadataPublication);\n\n const shard = {...requested, publications: allPublications};\n\n // Setup the global tables and shard tables / publications.\n await sql.unsafe(globalSetup(shard) + shardSetup(shard, metadataPublication));\n\n const pubs = await getPublicationInfo(sql, allPublications);\n await replicaIdentitiesForTablesWithoutPrimaryKeys(pubs)?.apply(lc, sql);\n\n await setupTriggers(lc, sql, shard, includePartialIndexes);\n}\n\nexport async function setupTriggers(\n lc: LogContext,\n tx: PostgresTransaction,\n shard: ShardConfig,\n includePartialIndexes = true,\n) {\n const schema = upstreamSchema(shard);\n const [{ddlDetection}] = await tx<InternalShardConfig[]> /*sql*/ `\n SELECT \"ddlDetection\" FROM ${tx(schema)}.\"shardConfig\"`;\n\n // The functions invoked by event triggers are installed even if\n // event triggers are not supported/allowed by the db provider.\n // This allows users to manually invoke the update_schemas() function\n // as a workaround.\n await tx.unsafe(createEventFunctionStatements(shard, includePartialIndexes));\n try {\n await tx.savepoint(sub => sub.unsafe(triggerSetup(shard)));\n } catch (e) {\n if (ddlDetection) {\n // If ddlDetection has already been enabled, subsequent failures to\n // upgrade the trigger should be propagated rather than swallowed.\n throw e;\n }\n if (\n !(\n e instanceof postgres.PostgresError &&\n e.code === PG_INSUFFICIENT_PRIVILEGE\n )\n ) {\n throw e;\n }\n // If triggerSetup() fails, replication continues in ddlDetection=false mode.\n lc.warn?.(\n `Unable to create event triggers for schema change detection:\\n\\n` +\n `\"${e.hint ?? e.message}\"\\n\\n` +\n `Proceeding in degraded mode: schema changes will halt replication,\\n` +\n `requiring the replica to be reset (manually or with --auto-reset).`,\n );\n }\n}\n\nexport function validatePublications(\n lc: LogContext,\n published: PublicationInfo,\n) {\n // Verify that all publications export the proper events.\n published.publications.forEach(pub => {\n if (\n !pub.pubinsert ||\n !pub.pubupdate ||\n !pub.pubdelete ||\n !pub.pubtruncate\n ) {\n // TODO: Make APIError?\n throw new Error(\n `PUBLICATION ${pub.pubname} must publish insert, update, delete, and truncate`,\n );\n }\n });\n\n published.tables.forEach(table => validate(lc, table));\n}\n\ntype ReplicaIdentities = {\n apply(lc: LogContext, db: PostgresDB): Promise<void>;\n};\n\nexport function replicaIdentitiesForTablesWithoutPrimaryKeys(\n pubs: PublishedSchema,\n): ReplicaIdentities | undefined {\n const replicaIdentities: {\n schema: string;\n tableName: string;\n indexName: string;\n }[] = [];\n for (const table of pubs.tables) {\n if (!table.primaryKey?.length && table.replicaIdentity === Default) {\n // Look for an index that can serve as the REPLICA IDENTITY USING INDEX. It must be:\n // - UNIQUE\n // - NOT NULL columns\n // - not deferrable (i.e. isImmediate)\n // - not partial\n //\n // https://www.postgresql.org/docs/current/sql-altertable.html#SQL-ALTERTABLE-REPLICA-IDENTITY\n const {schema, name: tableName} = table;\n for (const {columns, name: indexName} of pubs.indexes.filter(\n idx =>\n idx.schema === schema &&\n idx.tableName === tableName &&\n idx.unique &&\n idx.isImmediate &&\n idx.predicate === undefined,\n )) {\n if (Object.keys(columns).some(col => !table.columns[col].notNull)) {\n continue; // Only indexes with all NOT NULL columns are suitable.\n }\n replicaIdentities.push({schema, tableName, indexName});\n break;\n }\n }\n }\n\n if (replicaIdentities.length === 0) {\n return undefined;\n }\n return {\n apply: async (lc: LogContext, sql: PostgresDB) => {\n for (const {schema, tableName, indexName} of replicaIdentities) {\n lc.info?.(\n `setting \"${indexName}\" as the REPLICA IDENTITY for \"${tableName}\"`,\n );\n await sql`\n ALTER TABLE ${sql(schema)}.${sql(tableName)} \n REPLICA IDENTITY USING INDEX ${sql(indexName)}`;\n }\n },\n };\n}\n"],"mappings":";;;;;;;;;;;;;;;;;AAqCA,IAAM,yBAAyB;;;;;;AAO/B,SAAgB,wBAAwB,MAAoB;CAC1D,IAAI,CAAC,uBAAuB,KAAK,IAAI,GACnC,MAAM,IAAI,MACR,6BAA6B,KAAK,+GAEpC;CAEF,IAAI,KAAK,SAAS,IAChB,MAAM,IAAI,MACR,qBAAqB,KAAK,sDAC5B;AAEJ;AAEA,SAAgB,0BAA0B,EAAC,SAAe;CACxD,OAAO,IAAI,MAAM;AACnB;AAEA,SAAgB,sBAAsB,EAAC,OAAO,YAAoB;CAChE,OAAO,GAAG,MAAM,GAAG;AACrB;AAEA,SAAgB,sBAAsB,OAAgB;CACpD,MAAM,EAAC,OAAO,aAAY,MAAM,KAAK;CACrC,OAAO,GAAG,MAAM,GAAG,SAAS;AAC9B;;;;;AAMA,SAAgB,0BAA0B,OAAgB;CAGxD,OAAO,GAAG,sBAAsB,KAAK,EAAE,GAAG,WAAW,KAAK,KAAK;AACjE;AAEA,SAAS,uBAAuB,OAAe,SAA0B;CACvE,OAAO,IAAI,MAAM,UAAU;AAC7B;AAEA,SAAgB,wBACd,OACA,SACA;CACA,OAAO,IAAI,MAAM,YAAY;AAC/B;AAGA,SAAS,YAAY,OAAsB;CACzC,MAAM,MAAM,GAAG,UAAU,KAAK,CAAC;CAE/B,OAAe;gCACe,IAAI;;+BAEL,IAAI;;;;;;;;;;+BAUJ,IAAI;;;;;;;;;iCASF,IAAI;;uBAEd,IAAI;;gBAEX,IAAI;;AAEpB;AAEA,eAAsB,mBAAmB,IAAgB,OAAc;CACrE,MAAM,GAAG,OAAO,YAAY,KAAK,CAAC;AACpC;AAEA,SAAgB,0BAA0B,QAAgB;CACxD,OAAe;iBACA,OAAO;;;;;;;AAOxB;;;;;;;;;AAUA,SAAgB,4BAA4B,QAAgB;CAC1D,OAAe;iBACA,OAAO;;;;;;;AAOxB;AAEA,IAAa,qBAAqB;AAElC,SAAgB,WACd,aACA,qBACQ;CACR,MAAM,MAAM,GAAG,UAAU,WAAW,CAAC;CACrC,MAAM,QAAQ,GAAG,eAAe,WAAW,CAAC;CAE5C,MAAM,OAAO,YAAY,aAAa,SAAS;CAC/C,OACE,KAAK,SAAS,mBAAmB,SAC3B,6BAA6B,qBACrC;CAEA,OAAe;gCACe,MAAM;;IAElC,0BAA0B,KAAK,EAAE;IACjC,4BAA4B,KAAK,EAAE;;iBAEtB,MAAM,IAAI,mBAAmB;;;;;;;;gBAQ9B,MAAM,IAAI,mBAAmB;;;;cAI/B,QAAQ,IAAI,EAAE;;;;iBAIX,MAAM;;;;;;;;;;;;;;;;;+BAiBQ,GAAG,mBAAmB,EAAE;uBAChC,GAAG,mBAAmB,EAAE;gBAC/B,IAAI;gBACJ,MAAM;gBACN,MAAM;gBACN,MAAM;;AAEtB;AAEA,SAAgB,UAAU,OAAe,SAAkC;CACzE,MAAM,SAAS,GAAG,MAAM,GAAG;CAC3B,MAAM,sBAAsB,wBAAwB,OAAO,OAAO;CAKlE,OAAe;iCACgB,GALJ,uBAAuB,OAAO,OAKvB,CAAkB,EAAE;iCACvB,GAAG,mBAAmB,EAAE;4BAC7B,GAAG,MAAM,EAAE;;AAEvC;AAEA,IAAM,4BAA4B,eAAE,OAAO;CACzC,cAAc,eAAE,MAAM,eAAE,OAAO,CAAC;CAChC,cAAc,eAAE,QAAQ;AAC1B,CAAC;AAID,IAAM,oBAAoB,eAAE,OAAO;CACjC,IAAI,eAAE,OAAO;CACb,MAAM,eAAE,OAAO;CACf,MAAM,eAAE,OAAO;CACf,OAAO,eAAE,OAAO;CAChB,YAAY,eAAE,OAAO;CACrB,OAAO;CACP,YAAY,eAAE,OAAO,EAAE,SAAS;CAChC,UAAU,eAAE,QAAQ;AACtB,CAAC;AAED,IAAM,uBAAuB,kBAAkB,OAAO;CACpD,eAAe;CACf,oBAAoB,iBAAiB,SAAS;CAC9C,mBAAmB,iBAAiB,SAAS;AAC/C,CAAC;AAED,IAAM,gBAAgB,0BAA0B,OAC9C,qBAAqB,KACvB;AAIA,IAAM,6BAA6B,eAAE,OAAO;CAC1C,QAAQ,eAAE,QAAQ;CAClB,mBAAmB,eAAE,OAAO;AAC9B,CAAC;AAED,IAAM,qBAAqB,kBAAkB,OAC3C,2BAA2B,KAC7B;AAOA,SAAS,aAAa,OAA4B;CAChD,MAAM,SAAS,GAAG,eAAe,KAAK,CAAC;CACvC,OACE,6BAA6B,KAAK,IAC1B,UAAU,OAAO;AAE7B;;;;;;;;;;AAgBA,eAAsB,cACpB,KACA,OACA,IACA,MACA,OACA,gBACA,EAAC,YAAY,YACb,OACA;CACA,MAAM,SAAS,eAAe,KAAK;CACnC,MAAM,SAAwD;EAC5D;EACA;EACA;EACA,YAAY;EACZ;EACA;EACA;CACF;CACA,MAAM,GAAG,eAAe,IAAI,MAAM,EAAE,YAAY,IAAI,MAAM;AAC5D;AAGA,eAAsB,uBACpB,KACA,OACA,IACA,EAAC,QAAQ,WACT,oBACA;CACA,MAAM,SAAS,eAAe,KAAK;CAEnC,MAAM,SAAS;EAAC,eAAe;GADE;GAAQ;EACV;EAAQ;CAAkB;CACzD,MAAM,GAAG,UAAU,IAAI,MAAM,EAAE,gBAAgB,IAAI,MAAM,EAAE,cAAc;AAC3E;AAIA,eAAsB,mBACpB,KACA,OACA,EAAC,UAAU,UACmB;CAC9B,MAAM,SAAS,eAAe,KAAK;CACnC,MAAM,GAAG;aACE,IAAI,MAAM,EAAE;;;;aAIZ,IAAI,MAAM,EAAE;0BACC,SAAS,iBAAiB,OAAO;;CAEzD,OAAO,gBAAgB,KAAK,OAAO,MAAM;AAC3C;;;;;AAMA,eAAsB,oBACpB,IACA,KACA,OACA,gBACA,KAAoB,MACpB,SACyB;CACzB,MAAM,SAAS,IAAI,eAAe,KAAK,CAAC;CACxC,MAAM,SAAS,MAAM,GAAG;;;;;;;;;;;;;;;WAef,OAAO,iBAAiB,OAAO;2BACf,eAAe;0BAChB,GAAG;;;CAG3B,IAAI,OAAO,WAAW,GAAG;EAEvB,MAAM,cAAc,MAAM,GAAG;;eAElB,OAAO;EAClB,GAAG,OACD,WAAW,GAAG,GAAG,oBACd,UAAU,aAAa,UAAU,OAAO,EAAE,MAAM,MACjD,iBAAiB,UAAU,WAAW,GAC1C;EACA,OAAO;CACT;CACA,OAAO,MAAQ,OAAO,IAAI,eAAe,aAAa;AACxD;AAEA,eAAsB,kBACpB,IACA,KACA,OACyB;CAuBzB,MAAM,WAAW,MAAQ,MArBH,GAAG;;;;;;;;;;;;WADV,IAAI,eAAe,KAAK,CAa9B,EAAO;;;;;;;;KASkB,eAAE,MAAM,kBAAkB,GAAG,aAAa;CAC5E,GAAG,OAAO,oBAAoB,EAAC,SAAQ,CAAC;CACxC,OAAO;AACT;;;;;;;;;;;AAYA,eAAsB,qBACpB,IACA,KACA,OACA,OACyB;CACzB,MAAM,SAAS,IAAI,eAAe,KAAK,CAAC;CAoCxC,MAAM,WAAW,MAAQ,MAnCH,GAAG;;;;;;;;;;;;WAYhB,OAAO;;;;;iBAKD,OAAO;;6BAEK,MAAM;;+CAEsB;;;;2BAI9B,MAAM;+CACwB;;;;;;;;KASrB,eAAE,MAAM,kBAAkB,GAAG,aAAa;CAC5E,GAAG,OAAO,6BAA6B,SAAS,EAAC,SAAQ,CAAC;CAC1D,OAAO;AACT;AAEA,eAAsB,gBACpB,KACA,OACA,IAC8B;CAE9B,MAAM,UAAU,MAAM,GAAG;;;;;;;;;;;;WADV,IAAI,eAAe,KAAK,CAa9B,EAAO;qBACG,GAAG;;CAEtB,OAAO,QAAQ,SACX,MAAQ,QAAQ,IAAI,oBAAoB,aAAa,IACrD;AACN;AAEA,eAAsB,uBACpB,KACA,OAC8B;CAC9B,MAAM,SAAS,MAAM,GAAG;;aAEb,IAAI,eAAe,KAAK,CAAC,EAAE;;CAEtC,OACE,OAAO,WAAW,SACZ,6CAA6C,OAAO,QAC5D;CACA,OAAO,MAAQ,OAAO,IAAI,2BAA2B,aAAa;AACpE;;;;;AAMA,eAAsB,0BACpB,IACA,KACA,WACA,wBAAwB,MACxB;CACA,MAAM,EAAC,iBAAgB;CAEvB,KAAK,MAAM,OAAO,cAAc;EAC9B,wBAAwB,GAAG;EAC3B,IAAI,IAAI,WAAW,GAAG,GACpB,MAAM,IAAI,MACR,oHACkD,IAAI,GACxD;CAEJ;CACA,MAAM,kBAA4B,CAAC;CAGnC,IAAI,aAAa,QAAQ;EACvB,MAAM,UAAU,MAAM,GAAwB;0DACQ,IACpD,YACF,IAAI,OAAO;EAEX,IAAI,QAAQ,WAAW,aAAa,QAClC,MAAM,IAAI,MACR,gDAAgD,aAAa,aAAa,QAAQ,KAAK,EAAE,EAC3F;EAEF,gBAAgB,KAAK,GAAG,YAAY;CACtC,OAAO;EACL,MAAM,qBAAqB,uBACzB,UAAU,OACV,UAAU,QACZ;EACA,MAAM,GAAG;mCACsB,IAAI,kBAAkB;EACrD,MAAM,GAAG;2BACc,IAAI,kBAAkB,EAAE;;;EAG/C,gBAAgB,KAAK,kBAAkB;CACzC;CAEA,MAAM,sBAAsB,wBAC1B,UAAU,OACV,UAAU,QACZ;CACA,gBAAgB,KAAK,mBAAmB;CAExC,MAAM,QAAQ;EAAC,GAAG;EAAW,cAAc;CAAe;CAG1D,MAAM,IAAI,OAAO,YAAY,KAAK,IAAI,WAAW,OAAO,mBAAmB,CAAC;CAG5E,MAAM,6CAA6C,MADhC,mBAAmB,KAAK,eAAe,CACH,GAAG,MAAM,IAAI,GAAG;CAEvE,MAAM,cAAc,IAAI,KAAK,OAAO,qBAAqB;AAC3D;AAEA,eAAsB,cACpB,IACA,IACA,OACA,wBAAwB,MACxB;CAEA,MAAM,CAAC,EAAC,kBAAiB,MAAM,EAAkC;iCAClC,GAFhB,eAAe,KAEI,CAAM,EAAE;CAM1C,MAAM,GAAG,OAAO,8BAA8B,OAAO,qBAAqB,CAAC;CAC3E,IAAI;EACF,MAAM,GAAG,WAAU,QAAO,IAAI,OAAO,aAAa,KAAK,CAAC,CAAC;CAC3D,SAAS,GAAG;EACV,IAAI,cAGF,MAAM;EAER,IACE,EACE,aAAa,SAAS,iBACtB,EAAE,SAAS,4BAGb,MAAM;EAGR,GAAG,OACD,oEACM,EAAE,QAAQ,EAAE,QAAQ,4IAG5B;CACF;AACF;AAEA,SAAgB,qBACd,IACA,WACA;CAEA,UAAU,aAAa,SAAQ,QAAO;EACpC,IACE,CAAC,IAAI,aACL,CAAC,IAAI,aACL,CAAC,IAAI,aACL,CAAC,IAAI,aAGL,MAAM,IAAI,MACR,eAAe,IAAI,QAAQ,mDAC7B;CAEJ,CAAC;CAED,UAAU,OAAO,SAAQ,UAAS,SAAS,IAAI,KAAK,CAAC;AACvD;AAMA,SAAgB,6CACd,MAC+B;CAC/B,MAAM,oBAIA,CAAC;CACP,KAAK,MAAM,SAAS,KAAK,QACvB,IAAI,CAAC,MAAM,YAAY,UAAU,MAAM,oBAAA,KAA6B;EAQlE,MAAM,EAAC,QAAQ,MAAM,cAAa;EAClC,KAAK,MAAM,EAAC,SAAS,MAAM,eAAc,KAAK,QAAQ,QACpD,QACE,IAAI,WAAW,UACf,IAAI,cAAc,aAClB,IAAI,UACJ,IAAI,eACJ,IAAI,cAAc,KAAA,CACtB,GAAG;GACD,IAAI,OAAO,KAAK,OAAO,EAAE,MAAK,QAAO,CAAC,MAAM,QAAQ,KAAK,OAAO,GAC9D;GAEF,kBAAkB,KAAK;IAAC;IAAQ;IAAW;GAAS,CAAC;GACrD;EACF;CACF;CAGF,IAAI,kBAAkB,WAAW,GAC/B;CAEF,OAAO,EACL,OAAO,OAAO,IAAgB,QAAoB;EAChD,KAAK,MAAM,EAAC,QAAQ,WAAW,eAAc,mBAAmB;GAC9D,GAAG,OACD,YAAY,UAAU,iCAAiC,UAAU,EACnE;GACA,MAAM,GAAG;sBACK,IAAI,MAAM,EAAE,GAAG,IAAI,SAAS,EAAE;yCACX,IAAI,SAAS;EAChD;CACF,EACF;AACF"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"backup-monitor.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/change-streamer/backup-monitor.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAKjD,OAAO,KAAK,EAAC,MAAM,EAAC,MAAM,wBAAwB,CAAC;AACnD,OAAO,KAAK,EAAC,gBAAgB,EAAC,MAAM,eAAe,CAAC;AACpD,OAAO,KAAK,EAAC,qBAAqB,EAAC,MAAM,sBAAsB,CAAC;AAEhE,MAAM,MAAM,iBAAiB,GAAG;IAC9B,SAAS,EAAE,MAAM,CAAC;IAClB,WAAW,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IACjC,YAAY,EAAE,MAAM,CAAC;CACtB,CAAC;AAEF,qBAAa,aAAc,YAAW,gBAAgB;;IACpD,QAAQ,CAAC,EAAE,oBAAoB;
|
|
1
|
+
{"version":3,"file":"backup-monitor.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/change-streamer/backup-monitor.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAKjD,OAAO,KAAK,EAAC,MAAM,EAAC,MAAM,wBAAwB,CAAC;AACnD,OAAO,KAAK,EAAC,gBAAgB,EAAC,MAAM,eAAe,CAAC;AACpD,OAAO,KAAK,EAAC,qBAAqB,EAAC,MAAM,sBAAsB,CAAC;AAEhE,MAAM,MAAM,iBAAiB,GAAG;IAC9B,SAAS,EAAE,MAAM,CAAC;IAClB,WAAW,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IACjC,YAAY,EAAE,MAAM,CAAC;CACtB,CAAC;AAEF,qBAAa,aAAc,YAAW,gBAAgB;;IACpD,QAAQ,CAAC,EAAE,oBAAoB;IAiB/B,YACE,EAAE,EAAE,UAAU,EACd,UAAU,EAAE,MAAM,CAAC,iBAAiB,CAAC,EACrC,cAAc,EAAE,qBAAqB,EACrC,WAAW,EAAE,MAAM,EAMpB;IAED,mBAAmB,kBAElB;IAEK,GAAG,kBAsBR;IAqDD,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC,CAGpB;CA+BF"}
|
|
@@ -11,6 +11,14 @@ var BackupMonitor = class {
|
|
|
11
11
|
#replicaFile;
|
|
12
12
|
#firstBackupReceived = resolver();
|
|
13
13
|
#latestBackup;
|
|
14
|
+
#firstBackupResolved = false;
|
|
15
|
+
/**
|
|
16
|
+
* The replica's `stateVersion` when the monitor started, i.e. the point the
|
|
17
|
+
* backup has to reach before it covers what this task is about to serve.
|
|
18
|
+
* `undefined` when the replica could not be read, which degrades the gate to
|
|
19
|
+
* "any watermark" rather than hanging startup forever.
|
|
20
|
+
*/
|
|
21
|
+
#coverageTarget;
|
|
14
22
|
constructor(lc, watermarks, changeStreamer, replicaFile) {
|
|
15
23
|
this.#lc = lc;
|
|
16
24
|
this.#watermarks = watermarks;
|
|
@@ -23,21 +31,64 @@ var BackupMonitor = class {
|
|
|
23
31
|
async run() {
|
|
24
32
|
this.#lc.info?.("starting backup monitor");
|
|
25
33
|
this.#initBackupLagMetric();
|
|
34
|
+
this.#coverageTarget = this.#readReplicaStateVersion();
|
|
26
35
|
for await (const backedUp of this.#watermarks) {
|
|
27
|
-
if (
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
this.#
|
|
33
|
-
|
|
36
|
+
if (this.#latestBackup) {
|
|
37
|
+
if (backedUp.watermark < this.#latestBackup.watermark) {
|
|
38
|
+
this.#lc.warn?.(`ignoring earlier backup watermark`, { backedUp });
|
|
39
|
+
continue;
|
|
40
|
+
}
|
|
41
|
+
if (backedUp.watermark === this.#latestBackup.watermark) {
|
|
42
|
+
this.#lc.debug?.(`ignoring redundant backup watermark`, { backedUp });
|
|
43
|
+
continue;
|
|
44
|
+
}
|
|
34
45
|
}
|
|
35
46
|
this.#lc.info?.(`received backup watermark`, { backedUp });
|
|
36
47
|
this.#latestBackup = backedUp;
|
|
37
48
|
this.#changeStreamer.trackBackupWatermark(backedUp.watermark);
|
|
49
|
+
this.#checkFirstBackupCovers(backedUp);
|
|
38
50
|
}
|
|
39
51
|
this.#lc.info?.("watermark stream closed. BackupMonitor stopped.");
|
|
40
52
|
}
|
|
53
|
+
/**
|
|
54
|
+
* Resolves {@link firstBackupReceived} once the backup actually covers the
|
|
55
|
+
* replica, rather than on the first watermark of any value.
|
|
56
|
+
*
|
|
57
|
+
* A new backup destination starts empty and is filled by re-uploading the
|
|
58
|
+
* local LTX chain, so the first watermarks read back out of it are real but
|
|
59
|
+
* far behind the replica. Releasing the readiness gate on one of those lets
|
|
60
|
+
* the task serve against a backup that does not yet cover it, which is what
|
|
61
|
+
* demotes a restoring view-syncer to Postgres catchup.
|
|
62
|
+
*
|
|
63
|
+
* The target is pinned at startup rather than compared against the replica's
|
|
64
|
+
* current position, which keeps the gate satisfiable under sustained writes.
|
|
65
|
+
*/
|
|
66
|
+
#checkFirstBackupCovers(backedUp) {
|
|
67
|
+
if (this.#firstBackupResolved) return;
|
|
68
|
+
const target = this.#coverageTarget;
|
|
69
|
+
if (target !== void 0 && backedUp.watermark < target) {
|
|
70
|
+
this.#lc.info?.(`backup does not yet cover the replica`, {
|
|
71
|
+
backedUp,
|
|
72
|
+
coverageTarget: target
|
|
73
|
+
});
|
|
74
|
+
return;
|
|
75
|
+
}
|
|
76
|
+
this.#firstBackupResolved = true;
|
|
77
|
+
this.#firstBackupReceived.resolve();
|
|
78
|
+
}
|
|
79
|
+
#readReplicaStateVersion() {
|
|
80
|
+
let db;
|
|
81
|
+
try {
|
|
82
|
+
db = new Database(this.#lc, this.#replicaFile, { readonly: true });
|
|
83
|
+
const { stateVersion } = db.prepare(`SELECT stateVersion FROM "_zero.replicationState"`).get();
|
|
84
|
+
return stateVersion;
|
|
85
|
+
} catch (e) {
|
|
86
|
+
this.#lc.warn?.("unable to read the replica's stateVersion; the initial backup gate will accept the first watermark", e);
|
|
87
|
+
return;
|
|
88
|
+
} finally {
|
|
89
|
+
db?.close();
|
|
90
|
+
}
|
|
91
|
+
}
|
|
41
92
|
stop() {
|
|
42
93
|
this.#watermarks.cancel();
|
|
43
94
|
return promiseVoid;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"backup-monitor.js","names":["#lc","#watermarks","#changeStreamer","#replicaFile","#firstBackupReceived","#initBackupLagMetric","#latestBackup"],"sources":["../../../../../../zero-cache/src/services/change-streamer/backup-monitor.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {resolver} from '@rocicorp/resolver';\nimport {promiseVoid} from '../../../../shared/src/resolved-promises.ts';\nimport {Database} from '../../../../zqlite/src/db.ts';\nimport {getOrCreateGauge} from '../../observability/metrics.ts';\nimport type {Source} from '../../types/streams.ts';\nimport type {SingletonService} from '../service.ts';\nimport type {ChangeStreamerService} from './change-streamer.ts';\n\nexport type BackedUpWatermark = {\n watermark: string;\n writeTimeMs?: number | undefined; // optional for debuggin; available on v5\n backupTimeMs: number;\n};\n\nexport class BackupMonitor implements SingletonService {\n readonly id = 'backup-monitor';\n readonly #lc: LogContext;\n readonly #watermarks: Source<BackedUpWatermark>;\n readonly #changeStreamer: ChangeStreamerService;\n readonly #replicaFile: string;\n readonly #firstBackupReceived = resolver();\n\n #latestBackup: BackedUpWatermark | undefined;\n\n constructor(\n lc: LogContext,\n watermarks: Source<BackedUpWatermark>,\n changeStreamer: ChangeStreamerService,\n replicaFile: string,\n ) {\n this.#lc = lc;\n this.#watermarks = watermarks;\n this.#changeStreamer = changeStreamer;\n this.#replicaFile = replicaFile;\n }\n\n firstBackupReceived() {\n return this.#firstBackupReceived.promise;\n }\n\n async run() {\n this.#lc.info?.('starting backup monitor');\n this.#initBackupLagMetric();\n\n for await (const backedUp of this.#watermarks) {\n if (
|
|
1
|
+
{"version":3,"file":"backup-monitor.js","names":["#lc","#watermarks","#changeStreamer","#replicaFile","#firstBackupReceived","#initBackupLagMetric","#coverageTarget","#readReplicaStateVersion","#latestBackup","#checkFirstBackupCovers","#firstBackupResolved"],"sources":["../../../../../../zero-cache/src/services/change-streamer/backup-monitor.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {resolver} from '@rocicorp/resolver';\nimport {promiseVoid} from '../../../../shared/src/resolved-promises.ts';\nimport {Database} from '../../../../zqlite/src/db.ts';\nimport {getOrCreateGauge} from '../../observability/metrics.ts';\nimport type {Source} from '../../types/streams.ts';\nimport type {SingletonService} from '../service.ts';\nimport type {ChangeStreamerService} from './change-streamer.ts';\n\nexport type BackedUpWatermark = {\n watermark: string;\n writeTimeMs?: number | undefined; // optional for debuggin; available on v5\n backupTimeMs: number;\n};\n\nexport class BackupMonitor implements SingletonService {\n readonly id = 'backup-monitor';\n readonly #lc: LogContext;\n readonly #watermarks: Source<BackedUpWatermark>;\n readonly #changeStreamer: ChangeStreamerService;\n readonly #replicaFile: string;\n readonly #firstBackupReceived = resolver();\n\n #latestBackup: BackedUpWatermark | undefined;\n #firstBackupResolved = false;\n /**\n * The replica's `stateVersion` when the monitor started, i.e. the point the\n * backup has to reach before it covers what this task is about to serve.\n * `undefined` when the replica could not be read, which degrades the gate to\n * \"any watermark\" rather than hanging startup forever.\n */\n #coverageTarget: string | undefined;\n\n constructor(\n lc: LogContext,\n watermarks: Source<BackedUpWatermark>,\n changeStreamer: ChangeStreamerService,\n replicaFile: string,\n ) {\n this.#lc = lc;\n this.#watermarks = watermarks;\n this.#changeStreamer = changeStreamer;\n this.#replicaFile = replicaFile;\n }\n\n firstBackupReceived() {\n return this.#firstBackupReceived.promise;\n }\n\n async run() {\n this.#lc.info?.('starting backup monitor');\n this.#initBackupLagMetric();\n this.#coverageTarget = this.#readReplicaStateVersion();\n\n for await (const backedUp of this.#watermarks) {\n if (this.#latestBackup) {\n if (backedUp.watermark < this.#latestBackup.watermark) {\n this.#lc.warn?.(`ignoring earlier backup watermark`, {backedUp});\n continue;\n }\n if (backedUp.watermark === this.#latestBackup.watermark) {\n this.#lc.debug?.(`ignoring redundant backup watermark`, {backedUp});\n continue;\n }\n }\n this.#lc.info?.(`received backup watermark`, {backedUp});\n this.#latestBackup = backedUp;\n this.#changeStreamer.trackBackupWatermark(backedUp.watermark);\n this.#checkFirstBackupCovers(backedUp);\n }\n this.#lc.info?.('watermark stream closed. BackupMonitor stopped.');\n }\n\n /**\n * Resolves {@link firstBackupReceived} once the backup actually covers the\n * replica, rather than on the first watermark of any value.\n *\n * A new backup destination starts empty and is filled by re-uploading the\n * local LTX chain, so the first watermarks read back out of it are real but\n * far behind the replica. Releasing the readiness gate on one of those lets\n * the task serve against a backup that does not yet cover it, which is what\n * demotes a restoring view-syncer to Postgres catchup.\n *\n * The target is pinned at startup rather than compared against the replica's\n * current position, which keeps the gate satisfiable under sustained writes.\n */\n #checkFirstBackupCovers(backedUp: BackedUpWatermark) {\n if (this.#firstBackupResolved) {\n return;\n }\n const target = this.#coverageTarget;\n if (target !== undefined && backedUp.watermark < target) {\n this.#lc.info?.(`backup does not yet cover the replica`, {\n backedUp,\n coverageTarget: target,\n });\n return;\n }\n this.#firstBackupResolved = true;\n this.#firstBackupReceived.resolve();\n }\n\n #readReplicaStateVersion(): string | undefined {\n let db;\n try {\n db = new Database(this.#lc, this.#replicaFile, {readonly: true});\n const {stateVersion} = db\n .prepare(/*sql*/ `SELECT stateVersion FROM \"_zero.replicationState\"`)\n .get<{stateVersion: string}>();\n return stateVersion;\n } catch (e) {\n // Without a target the gate degrades to its previous behavior. That is\n // strictly better than blocking startup on a replica we cannot read.\n this.#lc.warn?.(\n `unable to read the replica's stateVersion; ` +\n `the initial backup gate will accept the first watermark`,\n e,\n );\n return undefined;\n } finally {\n db?.close();\n }\n }\n\n stop(): Promise<void> {\n this.#watermarks.cancel();\n return promiseVoid;\n }\n\n #initBackupLagMetric() {\n getOrCreateGauge('replica', 'backup_lag', {\n description:\n 'Latency from when a change is written to the replica ' +\n 'to when it is backed up to litestream. It is expected to create a saw ' +\n 'pattern from 0 to the configured ZERO_LITESTREAM_INCREMENTAL_BACKUP_INTERVAL_MINUTES.',\n unit: 'millisecond',\n }).addCallback(o => {\n const latestBackup = this.#latestBackup;\n if (!latestBackup) {\n this.#lc.warn?.(\n `no backed up watermarks. unable to report replica.backup_lag`,\n );\n return;\n }\n const db = new Database(this.#lc, this.#replicaFile, {readonly: true});\n try {\n const {writeTimeMs} = db\n .prepare(/*sql*/ `SELECT writeTimeMs FROM \"_zero.replicationState\"`)\n .get<{writeTimeMs: number}>();\n const backupLag = Math.max(0, writeTimeMs - latestBackup.backupTimeMs);\n o.observe(backupLag);\n } catch (e) {\n this.#lc.warn?.(`error measuring replica.backup_lag metric`, e);\n } finally {\n db.close();\n }\n });\n }\n}\n"],"mappings":";;;;;AAeA,IAAa,gBAAb,MAAuD;CACrD,KAAc;CACd;CACA;CACA;CACA;CACA,uBAAgC,SAAS;CAEzC;CACA,uBAAuB;;;;;;;CAOvB;CAEA,YACE,IACA,YACA,gBACA,aACA;EACA,KAAKA,MAAM;EACX,KAAKC,cAAc;EACnB,KAAKC,kBAAkB;EACvB,KAAKC,eAAe;CACtB;CAEA,sBAAsB;EACpB,OAAO,KAAKC,qBAAqB;CACnC;CAEA,MAAM,MAAM;EACV,KAAKJ,IAAI,OAAO,yBAAyB;EACzC,KAAKK,qBAAqB;EAC1B,KAAKC,kBAAkB,KAAKC,yBAAyB;EAErD,WAAW,MAAM,YAAY,KAAKN,aAAa;GAC7C,IAAI,KAAKO,eAAe;IACtB,IAAI,SAAS,YAAY,KAAKA,cAAc,WAAW;KACrD,KAAKR,IAAI,OAAO,qCAAqC,EAAC,SAAQ,CAAC;KAC/D;IACF;IACA,IAAI,SAAS,cAAc,KAAKQ,cAAc,WAAW;KACvD,KAAKR,IAAI,QAAQ,uCAAuC,EAAC,SAAQ,CAAC;KAClE;IACF;GACF;GACA,KAAKA,IAAI,OAAO,6BAA6B,EAAC,SAAQ,CAAC;GACvD,KAAKQ,gBAAgB;GACrB,KAAKN,gBAAgB,qBAAqB,SAAS,SAAS;GAC5D,KAAKO,wBAAwB,QAAQ;EACvC;EACA,KAAKT,IAAI,OAAO,iDAAiD;CACnE;;;;;;;;;;;;;;CAeA,wBAAwB,UAA6B;EACnD,IAAI,KAAKU,sBACP;EAEF,MAAM,SAAS,KAAKJ;EACpB,IAAI,WAAW,KAAA,KAAa,SAAS,YAAY,QAAQ;GACvD,KAAKN,IAAI,OAAO,yCAAyC;IACvD;IACA,gBAAgB;GAClB,CAAC;GACD;EACF;EACA,KAAKU,uBAAuB;EAC5B,KAAKN,qBAAqB,QAAQ;CACpC;CAEA,2BAA+C;EAC7C,IAAI;EACJ,IAAI;GACF,KAAK,IAAI,SAAS,KAAKJ,KAAK,KAAKG,cAAc,EAAC,UAAU,KAAI,CAAC;GAC/D,MAAM,EAAC,iBAAgB,GACpB,QAAgB,mDAAmD,EACnE,IAA4B;GAC/B,OAAO;EACT,SAAS,GAAG;GAGV,KAAKH,IAAI,OACP,sGAEA,CACF;GACA;EACF,UAAU;GACR,IAAI,MAAM;EACZ;CACF;CAEA,OAAsB;EACpB,KAAKC,YAAY,OAAO;EACxB,OAAO;CACT;CAEA,uBAAuB;EACrB,iBAAiB,WAAW,cAAc;GACxC,aACE;GAGF,MAAM;EACR,CAAC,EAAE,aAAY,MAAK;GAClB,MAAM,eAAe,KAAKO;GAC1B,IAAI,CAAC,cAAc;IACjB,KAAKR,IAAI,OACP,8DACF;IACA;GACF;GACA,MAAM,KAAK,IAAI,SAAS,KAAKA,KAAK,KAAKG,cAAc,EAAC,UAAU,KAAI,CAAC;GACrE,IAAI;IACF,MAAM,EAAC,gBAAe,GACnB,QAAgB,kDAAkD,EAClE,IAA2B;IAC9B,MAAM,YAAY,KAAK,IAAI,GAAG,cAAc,aAAa,YAAY;IACrE,EAAE,QAAQ,SAAS;GACrB,SAAS,GAAG;IACV,KAAKH,IAAI,OAAO,6CAA6C,CAAC;GAChE,UAAU;IACR,GAAG,MAAM;GACX;EACF,CAAC;CACH;AACF"}
|
|
@@ -1,7 +1,13 @@
|
|
|
1
1
|
import type { LogContext } from '@rocicorp/logger';
|
|
2
|
+
import { type PreSerialized } from '../../types/streams.ts';
|
|
2
3
|
import type { WatermarkedChange } from './change-streamer.ts';
|
|
3
4
|
import type { Subscriber } from './subscriber.ts';
|
|
4
5
|
export type BroadcastReleaseMode = 'all-subscribers' | 'consensus-timeout';
|
|
6
|
+
export type PreSerializedBatch = PreSerialized & {
|
|
7
|
+
readonly changes: readonly WatermarkedChange[];
|
|
8
|
+
};
|
|
9
|
+
export declare function isPreSerializedBatch(val: unknown): val is PreSerializedBatch;
|
|
10
|
+
export declare function preSerializeBatch(changes: readonly WatermarkedChange[]): PreSerializedBatch;
|
|
5
11
|
/**
|
|
6
12
|
* Enables event-driven early release of a {@link Broadcast}. When provided and a
|
|
7
13
|
* majority has acked, the broadcast schedules its own consensus-timeout release
|
|
@@ -78,12 +84,12 @@ export declare class Broadcast {
|
|
|
78
84
|
* Sends the change to the subscribers without the tracking machinery.
|
|
79
85
|
* This is suitable for fire-and-forget (i.e. pipelined) sends.
|
|
80
86
|
*/
|
|
81
|
-
static withoutTracking(subscribers: Iterable<Subscriber>, change: WatermarkedChange): void;
|
|
87
|
+
static withoutTracking(subscribers: Iterable<Subscriber>, change: WatermarkedChange | readonly WatermarkedChange[]): void;
|
|
82
88
|
/**
|
|
83
89
|
* Broadcasts the `change` to the `subscribers` and tracks their
|
|
84
90
|
* completion.
|
|
85
91
|
*/
|
|
86
|
-
constructor(lc: LogContext, subscribers: Iterable<Subscriber>, change: WatermarkedChange, earlyRelease?: EarlyReleaseOptions);
|
|
92
|
+
constructor(lc: LogContext, subscribers: Iterable<Subscriber>, change: WatermarkedChange | readonly WatermarkedChange[], earlyRelease?: EarlyReleaseOptions);
|
|
87
93
|
get isDone(): boolean;
|
|
88
94
|
get done(): Promise<void>;
|
|
89
95
|
get releaseMode(): BroadcastReleaseMode;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"broadcast.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/change-streamer/broadcast.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;
|
|
1
|
+
{"version":3,"file":"broadcast.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/change-streamer/broadcast.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAGjD,OAAO,EAAkB,KAAK,aAAa,EAAC,MAAM,wBAAwB,CAAC;AAC3E,OAAO,KAAK,EAAC,iBAAiB,EAAC,MAAM,sBAAsB,CAAC;AAC5D,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,iBAAiB,CAAC;AAEhD,MAAM,MAAM,oBAAoB,GAAG,iBAAiB,GAAG,mBAAmB,CAAC;AAE3E,MAAM,MAAM,kBAAkB,GAAG,aAAa,GAAG;IAC/C,QAAQ,CAAC,OAAO,EAAE,SAAS,iBAAiB,EAAE,CAAC;CAChD,CAAC;AAEF,wBAAgB,oBAAoB,CAAC,GAAG,EAAE,OAAO,GAAG,GAAG,IAAI,kBAAkB,CAE5E;AAED,wBAAgB,iBAAiB,CAC/B,OAAO,EAAE,SAAS,iBAAiB,EAAE,GACpC,kBAAkB,CAYpB;AAED;;;;;GAKG;AACH,MAAM,MAAM,mBAAmB,GAAG;IAChC,2EAA2E;IAC3E,0BAA0B,EAAE,MAAM,CAAC;IACnC,mDAAmD;IACnD,YAAY,CAAC,EAAE,OAAO,UAAU,CAAC;IACjC,gEAAgE;IAChE,cAAc,CAAC,EAAE,OAAO,YAAY,CAAC;CACtC,CAAC;AAEF;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAuDG;AACH,qBAAa,SAAS;;IACpB;;;OAGG;IACH,MAAM,CAAC,eAAe,CACpB,WAAW,EAAE,QAAQ,CAAC,UAAU,CAAC,EACjC,MAAM,EAAE,iBAAiB,GAAG,SAAS,iBAAiB,EAAE,QAOzD;IA0BD;;;OAGG;IACH,YACE,EAAE,EAAE,UAAU,EACd,WAAW,EAAE,QAAQ,CAAC,UAAU,CAAC,EACjC,MAAM,EAAE,iBAAiB,GAAG,SAAS,iBAAiB,EAAE,EACxD,YAAY,CAAC,EAAE,mBAAmB,EA+BnC;IAwDD,IAAI,MAAM,IAAI,OAAO,CAEpB;IAED,IAAI,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC,CAExB;IAED,IAAI,WAAW,IAAI,oBAAoB,CAEtC;IAED,eAAe,CAAC,GAAG,EAAE,MAAM,QAa1B;CAiBF"}
|
|
@@ -1,5 +1,17 @@
|
|
|
1
|
+
import { must } from "../../../../shared/src/must.js";
|
|
2
|
+
import "../../types/streams.js";
|
|
1
3
|
import { resolver } from "@rocicorp/resolver";
|
|
2
4
|
//#region ../zero-cache/src/services/change-streamer/broadcast.ts
|
|
5
|
+
function preSerializeBatch(changes) {
|
|
6
|
+
const jsonList = changes.map((c) => c[2]);
|
|
7
|
+
const joined = jsonList.join(",");
|
|
8
|
+
const payload = Buffer.from(changes.length === 1 ? `,"msg":${jsonList[0]}}` : `,"batch":[${joined}]}`, "utf8");
|
|
9
|
+
return {
|
|
10
|
+
changes,
|
|
11
|
+
payload,
|
|
12
|
+
byteLength: payload.length
|
|
13
|
+
};
|
|
14
|
+
}
|
|
3
15
|
/**
|
|
4
16
|
* Initiates and tracks the progress of a change broadcasted to
|
|
5
17
|
* a set of subscribers, implementing **consensus-based timeouts**
|
|
@@ -62,7 +74,9 @@ var Broadcast = class {
|
|
|
62
74
|
* This is suitable for fire-and-forget (i.e. pipelined) sends.
|
|
63
75
|
*/
|
|
64
76
|
static withoutTracking(subscribers, change) {
|
|
65
|
-
|
|
77
|
+
const changes = toBatch(change);
|
|
78
|
+
const preSerialized = preSerializeBatch(changes);
|
|
79
|
+
for (const sub of subscribers) sub.sendBatch(changes, preSerialized);
|
|
66
80
|
}
|
|
67
81
|
#lc;
|
|
68
82
|
#pending;
|
|
@@ -83,18 +97,20 @@ var Broadcast = class {
|
|
|
83
97
|
* completion.
|
|
84
98
|
*/
|
|
85
99
|
constructor(lc, subscribers, change, earlyRelease) {
|
|
100
|
+
const changes = toBatch(change);
|
|
101
|
+
const preSerialized = preSerializeBatch(changes);
|
|
86
102
|
this.#lc = lc;
|
|
87
103
|
this.#pending = new Set(subscribers);
|
|
88
104
|
this.#completed = [];
|
|
89
|
-
this.#watermark =
|
|
105
|
+
this.#watermark = must(changes.at(-1))[0];
|
|
90
106
|
this.#majority = Math.floor(this.#pending.size / 2) + 1;
|
|
91
107
|
this.#earlyReleaseTimeoutProportion = earlyRelease?.consensusTimeoutProportion;
|
|
92
108
|
this.#setTimeout = earlyRelease?.setTimeoutFn ?? setTimeout;
|
|
93
109
|
this.#clearTimeout = earlyRelease?.clearTimeoutFn ?? clearTimeout;
|
|
94
110
|
for (const sub of this.#pending) {
|
|
95
|
-
const
|
|
111
|
+
const totalChanges = sub.numPending + changes.length;
|
|
96
112
|
if (sub.mode === "backup") this.#needsBackupResponse ||= !sub.isBacklogged();
|
|
97
|
-
sub.
|
|
113
|
+
sub.sendBatch(changes, preSerialized).catch(() => {}).finally(() => this.#markCompleted(sub, totalChanges));
|
|
98
114
|
}
|
|
99
115
|
if (this.#pending.size === 0) this.#setDone("all-subscribers");
|
|
100
116
|
}
|
|
@@ -161,6 +177,10 @@ var Broadcast = class {
|
|
|
161
177
|
});
|
|
162
178
|
}
|
|
163
179
|
};
|
|
180
|
+
function toBatch(change) {
|
|
181
|
+
if (change.length === 0) return [];
|
|
182
|
+
return Array.isArray(change[0]) ? change : [change];
|
|
183
|
+
}
|
|
164
184
|
//#endregion
|
|
165
185
|
export { Broadcast };
|
|
166
186
|
|