@rocicorp/zero 1.10.0-canary.0 → 1.10.0-canary.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/out/replicache/src/db/commit.js +1 -4
- package/out/replicache/src/db/commit.js.map +1 -1
- package/out/replicache/src/db/read.d.ts +1 -0
- package/out/replicache/src/db/read.d.ts.map +1 -1
- package/out/replicache/src/db/read.js +1 -4
- package/out/replicache/src/db/read.js.map +1 -1
- package/out/replicache/src/db/write.d.ts +2 -1
- package/out/replicache/src/db/write.d.ts.map +1 -1
- package/out/replicache/src/db/write.js +4 -51
- package/out/replicache/src/db/write.js.map +1 -1
- package/out/replicache/src/persist/clients.d.ts +1 -0
- package/out/replicache/src/persist/clients.d.ts.map +1 -1
- package/out/replicache/src/persist/clients.js +4 -31
- package/out/replicache/src/persist/clients.js.map +1 -1
- package/out/replicache/src/process-env.d.ts +28 -0
- package/out/replicache/src/process-env.d.ts.map +1 -0
- package/out/replicache/src/replicache-impl.d.ts.map +1 -1
- package/out/replicache/src/replicache-impl.js.map +1 -1
- package/out/replicache/src/scan-iterator.js +2 -14
- package/out/replicache/src/scan-iterator.js.map +1 -1
- package/out/replicache/src/subscriptions.d.ts +1 -0
- package/out/replicache/src/subscriptions.d.ts.map +1 -1
- package/out/replicache/src/subscriptions.js +2 -10
- package/out/replicache/src/subscriptions.js.map +1 -1
- package/out/replicache/src/transactions.d.ts +1 -0
- package/out/replicache/src/transactions.d.ts.map +1 -1
- package/out/replicache/src/transactions.js +2 -4
- package/out/replicache/src/transactions.js.map +1 -1
- package/out/z2s/src/compiler.d.ts.map +1 -1
- package/out/z2s/src/compiler.js +1 -22
- package/out/z2s/src/compiler.js.map +1 -1
- package/out/zero/package.js +3 -3
- package/out/zero/package.js.map +1 -1
- package/out/zero-cache/src/config/normalize.d.ts +1 -0
- package/out/zero-cache/src/config/normalize.d.ts.map +1 -1
- package/out/zero-cache/src/config/normalize.js +10 -1
- package/out/zero-cache/src/config/normalize.js.map +1 -1
- package/out/zero-cache/src/config/zero-config.d.ts +52 -1
- package/out/zero-cache/src/config/zero-config.d.ts.map +1 -1
- package/out/zero-cache/src/config/zero-config.js +171 -22
- package/out/zero-cache/src/config/zero-config.js.map +1 -1
- package/out/zero-cache/src/custom/fetch.js +1 -1
- package/out/zero-cache/src/custom/fetch.js.map +1 -1
- package/out/zero-cache/src/db/delete-lite-db.js +2 -1
- package/out/zero-cache/src/db/delete-lite-db.js.map +1 -1
- package/out/zero-cache/src/db/migration-lite.d.ts.map +1 -1
- package/out/zero-cache/src/db/migration-lite.js +5 -2
- package/out/zero-cache/src/db/migration-lite.js.map +1 -1
- package/out/zero-cache/src/db/migration.d.ts.map +1 -1
- package/out/zero-cache/src/db/migration.js +4 -2
- package/out/zero-cache/src/db/migration.js.map +1 -1
- package/out/zero-cache/src/db/sqlite-corruption.d.ts +3 -2
- package/out/zero-cache/src/db/sqlite-corruption.d.ts.map +1 -1
- package/out/zero-cache/src/db/sqlite-corruption.js +13 -8
- package/out/zero-cache/src/db/sqlite-corruption.js.map +1 -1
- package/out/zero-cache/src/observability/metrics.d.ts +6 -1
- package/out/zero-cache/src/observability/metrics.d.ts.map +1 -1
- package/out/zero-cache/src/observability/metrics.js +20 -2
- package/out/zero-cache/src/observability/metrics.js.map +1 -1
- package/out/zero-cache/src/server/backup-watermark-poller.js +31 -0
- package/out/zero-cache/src/server/backup-watermark-poller.js.map +1 -0
- package/out/zero-cache/src/server/change-streamer.d.ts.map +1 -1
- package/out/zero-cache/src/server/change-streamer.js +91 -31
- package/out/zero-cache/src/server/change-streamer.js.map +1 -1
- package/out/zero-cache/src/server/main.d.ts.map +1 -1
- package/out/zero-cache/src/server/main.js +6 -16
- package/out/zero-cache/src/server/main.js.map +1 -1
- package/out/zero-cache/src/server/replicator.d.ts.map +1 -1
- package/out/zero-cache/src/server/replicator.js +48 -5
- package/out/zero-cache/src/server/replicator.js.map +1 -1
- package/out/zero-cache/src/server/runner/run-worker.js +2 -1
- package/out/zero-cache/src/server/runner/run-worker.js.map +1 -1
- package/out/zero-cache/src/server/syncer.d.ts.map +1 -1
- package/out/zero-cache/src/server/syncer.js +1 -1
- package/out/zero-cache/src/server/syncer.js.map +1 -1
- package/out/zero-cache/src/server/worker-urls.d.ts +1 -1
- package/out/zero-cache/src/server/worker-urls.js +2 -2
- package/out/zero-cache/src/server/worker-urls.js.map +1 -1
- package/out/zero-cache/src/services/change-source/change-source.d.ts +4 -2
- package/out/zero-cache/src/services/change-source/change-source.d.ts.map +1 -1
- package/out/zero-cache/src/services/change-source/common/replica-restore.d.ts +35 -0
- package/out/zero-cache/src/services/change-source/common/replica-restore.d.ts.map +1 -0
- package/out/zero-cache/src/services/change-source/common/replica-restore.js +34 -0
- package/out/zero-cache/src/services/change-source/common/replica-restore.js.map +1 -0
- package/out/zero-cache/src/services/change-source/common/replica-schema.d.ts +2 -0
- package/out/zero-cache/src/services/change-source/common/replica-schema.d.ts.map +1 -1
- package/out/zero-cache/src/services/change-source/common/replica-schema.js +73 -1
- package/out/zero-cache/src/services/change-source/common/replica-schema.js.map +1 -1
- package/out/zero-cache/src/services/change-source/custom/change-source.d.ts +2 -6
- package/out/zero-cache/src/services/change-source/custom/change-source.d.ts.map +1 -1
- package/out/zero-cache/src/services/change-source/custom/change-source.js +14 -5
- package/out/zero-cache/src/services/change-source/custom/change-source.js.map +1 -1
- package/out/zero-cache/src/services/change-source/pg/backfill-stream.js +4 -7
- package/out/zero-cache/src/services/change-source/pg/backfill-stream.js.map +1 -1
- package/out/zero-cache/src/services/change-source/pg/change-source.d.ts +7 -7
- package/out/zero-cache/src/services/change-source/pg/change-source.d.ts.map +1 -1
- package/out/zero-cache/src/services/change-source/pg/change-source.js +82 -25
- package/out/zero-cache/src/services/change-source/pg/change-source.js.map +1 -1
- package/out/zero-cache/src/services/change-source/pg/initial-sync.d.ts +9 -1
- package/out/zero-cache/src/services/change-source/pg/initial-sync.d.ts.map +1 -1
- package/out/zero-cache/src/services/change-source/pg/initial-sync.js +12 -4
- package/out/zero-cache/src/services/change-source/pg/initial-sync.js.map +1 -1
- package/out/zero-cache/src/services/change-source/pg/logical-replication/stream.d.ts +45 -2
- package/out/zero-cache/src/services/change-source/pg/logical-replication/stream.d.ts.map +1 -1
- package/out/zero-cache/src/services/change-source/pg/logical-replication/stream.js +59 -8
- package/out/zero-cache/src/services/change-source/pg/logical-replication/stream.js.map +1 -1
- package/out/zero-cache/src/services/change-source/pg/replication-slots.d.ts +4 -2
- package/out/zero-cache/src/services/change-source/pg/replication-slots.d.ts.map +1 -1
- package/out/zero-cache/src/services/change-source/pg/replication-slots.js +7 -4
- package/out/zero-cache/src/services/change-source/pg/replication-slots.js.map +1 -1
- package/out/zero-cache/src/services/change-source/pg/schema/init.d.ts +0 -5
- package/out/zero-cache/src/services/change-source/pg/schema/init.d.ts.map +1 -1
- package/out/zero-cache/src/services/change-source/pg/schema/init.js +29 -41
- package/out/zero-cache/src/services/change-source/pg/schema/init.js.map +1 -1
- package/out/zero-cache/src/services/change-source/pg/schema/shard.d.ts +35 -16
- package/out/zero-cache/src/services/change-source/pg/schema/shard.d.ts.map +1 -1
- package/out/zero-cache/src/services/change-source/pg/schema/shard.js +77 -7
- package/out/zero-cache/src/services/change-source/pg/schema/shard.js.map +1 -1
- package/out/zero-cache/src/services/change-source/protocol/current/data.d.ts +2 -1
- package/out/zero-cache/src/services/change-source/protocol/current/data.d.ts.map +1 -1
- package/out/zero-cache/src/services/change-source/protocol/current/data.js +5 -13
- package/out/zero-cache/src/services/change-source/protocol/current/data.js.map +1 -1
- package/out/zero-cache/src/services/change-source/protocol/current/downstream.d.ts +6 -3
- package/out/zero-cache/src/services/change-source/protocol/current/downstream.d.ts.map +1 -1
- package/out/zero-cache/src/services/change-source/protocol/current/schema-change-tags.d.ts +6 -0
- package/out/zero-cache/src/services/change-source/protocol/current/schema-change-tags.d.ts.map +1 -0
- package/out/zero-cache/src/services/change-source/protocol/current/schema-change-tags.js +21 -0
- package/out/zero-cache/src/services/change-source/protocol/current/schema-change-tags.js.map +1 -0
- package/out/zero-cache/src/services/change-source/protocol/current/status.d.ts +1 -0
- package/out/zero-cache/src/services/change-source/protocol/current/status.d.ts.map +1 -1
- package/out/zero-cache/src/services/change-source/protocol/current/upstream.d.ts +1 -0
- package/out/zero-cache/src/services/change-source/protocol/current/upstream.d.ts.map +1 -1
- package/out/zero-cache/src/services/change-streamer/backup-cleanup-monitor-factory.d.ts +5 -6
- package/out/zero-cache/src/services/change-streamer/backup-cleanup-monitor-factory.d.ts.map +1 -1
- package/out/zero-cache/src/services/change-streamer/backup-cleanup-monitor-factory.js +32 -9
- package/out/zero-cache/src/services/change-streamer/backup-cleanup-monitor-factory.js.map +1 -1
- package/out/zero-cache/src/services/change-streamer/backup-monitor.d.ts +15 -5
- package/out/zero-cache/src/services/change-streamer/backup-monitor.d.ts.map +1 -1
- package/out/zero-cache/src/services/change-streamer/backup-monitor.js +66 -0
- package/out/zero-cache/src/services/change-streamer/backup-monitor.js.map +1 -0
- package/out/zero-cache/src/services/change-streamer/broadcast.d.ts +68 -78
- package/out/zero-cache/src/services/change-streamer/broadcast.d.ts.map +1 -1
- package/out/zero-cache/src/services/change-streamer/broadcast.js +84 -92
- package/out/zero-cache/src/services/change-streamer/broadcast.js.map +1 -1
- package/out/zero-cache/src/services/change-streamer/change-log-codec.d.ts +19 -0
- package/out/zero-cache/src/services/change-streamer/change-log-codec.d.ts.map +1 -0
- package/out/zero-cache/src/services/change-streamer/change-log-codec.js +50 -0
- package/out/zero-cache/src/services/change-streamer/change-log-codec.js.map +1 -0
- package/out/zero-cache/src/services/change-streamer/change-log-initializer.d.ts +141 -0
- package/out/zero-cache/src/services/change-streamer/change-log-initializer.d.ts.map +1 -0
- package/out/zero-cache/src/services/change-streamer/change-log-initializer.js +270 -0
- package/out/zero-cache/src/services/change-streamer/change-log-initializer.js.map +1 -0
- package/out/zero-cache/src/services/change-streamer/change-log-transaction-hash.d.ts +19 -0
- package/out/zero-cache/src/services/change-streamer/change-log-transaction-hash.d.ts.map +1 -0
- package/out/zero-cache/src/services/change-streamer/change-log-transaction-hash.js +39 -0
- package/out/zero-cache/src/services/change-streamer/change-log-transaction-hash.js.map +1 -0
- package/out/zero-cache/src/services/change-streamer/change-streamer-http.d.ts +3 -4
- package/out/zero-cache/src/services/change-streamer/change-streamer-http.d.ts.map +1 -1
- package/out/zero-cache/src/services/change-streamer/change-streamer-http.js +12 -17
- package/out/zero-cache/src/services/change-streamer/change-streamer-http.js.map +1 -1
- package/out/zero-cache/src/services/change-streamer/change-streamer-service.d.ts +69 -24
- package/out/zero-cache/src/services/change-streamer/change-streamer-service.d.ts.map +1 -1
- package/out/zero-cache/src/services/change-streamer/change-streamer-service.js +333 -66
- package/out/zero-cache/src/services/change-streamer/change-streamer-service.js.map +1 -1
- package/out/zero-cache/src/services/change-streamer/change-streamer.d.ts +56 -13
- package/out/zero-cache/src/services/change-streamer/change-streamer.d.ts.map +1 -1
- package/out/zero-cache/src/services/change-streamer/change-streamer.js.map +1 -1
- package/out/zero-cache/src/services/change-streamer/error-type-enum.d.ts +8 -0
- package/out/zero-cache/src/services/change-streamer/error-type-enum.d.ts.map +1 -1
- package/out/zero-cache/src/services/change-streamer/forwarder.d.ts +14 -3
- package/out/zero-cache/src/services/change-streamer/forwarder.d.ts.map +1 -1
- package/out/zero-cache/src/services/change-streamer/forwarder.js +41 -11
- package/out/zero-cache/src/services/change-streamer/forwarder.js.map +1 -1
- package/out/zero-cache/src/services/change-streamer/{litestream3-backup-monitor.d.ts → litestream3-prometheus-poller.d.ts} +19 -45
- package/out/zero-cache/src/services/change-streamer/litestream3-prometheus-poller.d.ts.map +1 -0
- package/out/zero-cache/src/services/change-streamer/{litestream3-backup-monitor.js → litestream3-prometheus-poller.js} +57 -171
- package/out/zero-cache/src/services/change-streamer/litestream3-prometheus-poller.js.map +1 -0
- package/out/zero-cache/src/services/change-streamer/replica-poller.d.ts +15 -0
- package/out/zero-cache/src/services/change-streamer/replica-poller.d.ts.map +1 -0
- package/out/zero-cache/src/services/change-streamer/replica-poller.js +55 -0
- package/out/zero-cache/src/services/change-streamer/replica-poller.js.map +1 -0
- package/out/zero-cache/src/services/change-streamer/schema/tables.d.ts +5 -1
- package/out/zero-cache/src/services/change-streamer/schema/tables.d.ts.map +1 -1
- package/out/zero-cache/src/services/change-streamer/schema/tables.js +2 -1
- package/out/zero-cache/src/services/change-streamer/schema/tables.js.map +1 -1
- package/out/zero-cache/src/services/change-streamer/snapshot-reservations.d.ts +14 -0
- package/out/zero-cache/src/services/change-streamer/snapshot-reservations.d.ts.map +1 -0
- package/out/zero-cache/src/services/change-streamer/snapshot-reservations.js +86 -0
- package/out/zero-cache/src/services/change-streamer/snapshot-reservations.js.map +1 -0
- package/out/zero-cache/src/services/change-streamer/snapshot.d.ts +4 -1
- package/out/zero-cache/src/services/change-streamer/snapshot.d.ts.map +1 -1
- package/out/zero-cache/src/services/change-streamer/snapshot.js +39 -14
- package/out/zero-cache/src/services/change-streamer/snapshot.js.map +1 -1
- package/out/zero-cache/src/services/change-streamer/sqlite-change-log-catchup.d.ts +75 -0
- package/out/zero-cache/src/services/change-streamer/sqlite-change-log-catchup.d.ts.map +1 -0
- package/out/zero-cache/src/services/change-streamer/sqlite-change-log-catchup.js +261 -0
- package/out/zero-cache/src/services/change-streamer/sqlite-change-log-catchup.js.map +1 -0
- package/out/zero-cache/src/services/change-streamer/sqlite-change-log-purge-scheduler.d.ts +121 -0
- package/out/zero-cache/src/services/change-streamer/sqlite-change-log-purge-scheduler.d.ts.map +1 -0
- package/out/zero-cache/src/services/change-streamer/sqlite-change-log-purge-scheduler.js +391 -0
- package/out/zero-cache/src/services/change-streamer/sqlite-change-log-purge-scheduler.js.map +1 -0
- package/out/zero-cache/src/services/change-streamer/sqlite-change-log-reader.d.ts +62 -0
- package/out/zero-cache/src/services/change-streamer/sqlite-change-log-reader.d.ts.map +1 -0
- package/out/zero-cache/src/services/change-streamer/sqlite-change-log-reader.js +159 -0
- package/out/zero-cache/src/services/change-streamer/sqlite-change-log-reader.js.map +1 -0
- package/out/zero-cache/src/services/change-streamer/sqlite-change-log-writer.d.ts +109 -0
- package/out/zero-cache/src/services/change-streamer/sqlite-change-log-writer.d.ts.map +1 -0
- package/out/zero-cache/src/services/change-streamer/sqlite-change-log-writer.js +258 -0
- package/out/zero-cache/src/services/change-streamer/sqlite-change-log-writer.js.map +1 -0
- package/out/zero-cache/src/services/change-streamer/storer.d.ts +28 -12
- package/out/zero-cache/src/services/change-streamer/storer.d.ts.map +1 -1
- package/out/zero-cache/src/services/change-streamer/storer.js +116 -121
- package/out/zero-cache/src/services/change-streamer/storer.js.map +1 -1
- package/out/zero-cache/src/services/change-streamer/subscriber.d.ts +61 -10
- package/out/zero-cache/src/services/change-streamer/subscriber.d.ts.map +1 -1
- package/out/zero-cache/src/services/change-streamer/subscriber.js +101 -8
- package/out/zero-cache/src/services/change-streamer/subscriber.js.map +1 -1
- package/out/zero-cache/src/services/change-streamer/upstream-acker.d.ts +34 -0
- package/out/zero-cache/src/services/change-streamer/upstream-acker.d.ts.map +1 -0
- package/out/zero-cache/src/services/change-streamer/upstream-acker.js +86 -0
- package/out/zero-cache/src/services/change-streamer/upstream-acker.js.map +1 -0
- package/out/zero-cache/src/services/life-cycle.d.ts +1 -1
- package/out/zero-cache/src/services/life-cycle.d.ts.map +1 -1
- package/out/zero-cache/src/services/life-cycle.js +4 -3
- package/out/zero-cache/src/services/life-cycle.js.map +1 -1
- package/out/zero-cache/src/services/litestream/commands.d.ts +12 -11
- package/out/zero-cache/src/services/litestream/commands.d.ts.map +1 -1
- package/out/zero-cache/src/services/litestream/commands.js +107 -148
- package/out/zero-cache/src/services/litestream/commands.js.map +1 -1
- package/out/zero-cache/src/services/litestream/config-v5.yml +42 -0
- package/out/zero-cache/src/services/litestream/config.yml +3 -2
- package/out/zero-cache/src/services/litestream/metrics.d.ts +4 -4
- package/out/zero-cache/src/services/litestream/metrics.d.ts.map +1 -1
- package/out/zero-cache/src/services/litestream/metrics.js +7 -10
- package/out/zero-cache/src/services/litestream/metrics.js.map +1 -1
- package/out/zero-cache/src/services/litestream/vfs-watermark-poller.d.ts +57 -0
- package/out/zero-cache/src/services/litestream/vfs-watermark-poller.d.ts.map +1 -0
- package/out/zero-cache/src/services/litestream/vfs-watermark-poller.js +137 -0
- package/out/zero-cache/src/services/litestream/vfs-watermark-poller.js.map +1 -0
- package/out/zero-cache/src/services/mutagen/mutagen.d.ts.map +1 -1
- package/out/zero-cache/src/services/mutagen/mutagen.js +4 -1
- package/out/zero-cache/src/services/mutagen/mutagen.js.map +1 -1
- package/out/zero-cache/src/services/replicator/change-log-cookies.d.ts +169 -0
- package/out/zero-cache/src/services/replicator/change-log-cookies.d.ts.map +1 -0
- package/out/zero-cache/src/services/replicator/change-log-cookies.js +434 -0
- package/out/zero-cache/src/services/replicator/change-log-cookies.js.map +1 -0
- package/out/zero-cache/src/services/replicator/change-log-db.d.ts +358 -0
- package/out/zero-cache/src/services/replicator/change-log-db.d.ts.map +1 -0
- package/out/zero-cache/src/services/replicator/change-log-db.js +477 -0
- package/out/zero-cache/src/services/replicator/change-log-db.js.map +1 -0
- package/out/zero-cache/src/services/replicator/change-log-stream-writer.d.ts +55 -0
- package/out/zero-cache/src/services/replicator/change-log-stream-writer.d.ts.map +1 -0
- package/out/zero-cache/src/services/replicator/change-log-stream-writer.js +150 -0
- package/out/zero-cache/src/services/replicator/change-log-stream-writer.js.map +1 -0
- package/out/zero-cache/src/services/replicator/change-processor.d.ts +6 -0
- package/out/zero-cache/src/services/replicator/change-processor.d.ts.map +1 -1
- package/out/zero-cache/src/services/replicator/change-processor.js +70 -25
- package/out/zero-cache/src/services/replicator/change-processor.js.map +1 -1
- package/out/zero-cache/src/services/replicator/incremental-sync.d.ts +1 -1
- package/out/zero-cache/src/services/replicator/incremental-sync.d.ts.map +1 -1
- package/out/zero-cache/src/services/replicator/incremental-sync.js +19 -6
- package/out/zero-cache/src/services/replicator/incremental-sync.js.map +1 -1
- package/out/zero-cache/src/services/replicator/notifier.d.ts +0 -22
- package/out/zero-cache/src/services/replicator/notifier.d.ts.map +1 -1
- package/out/zero-cache/src/services/replicator/notifier.js +7 -1
- package/out/zero-cache/src/services/replicator/notifier.js.map +1 -1
- package/out/zero-cache/src/services/replicator/replicator.d.ts +17 -2
- package/out/zero-cache/src/services/replicator/replicator.d.ts.map +1 -1
- package/out/zero-cache/src/services/replicator/replicator.js +9 -3
- package/out/zero-cache/src/services/replicator/replicator.js.map +1 -1
- package/out/zero-cache/src/services/replicator/reporter/recorder.d.ts +27 -0
- package/out/zero-cache/src/services/replicator/reporter/recorder.d.ts.map +1 -1
- package/out/zero-cache/src/services/replicator/reporter/recorder.js +32 -0
- package/out/zero-cache/src/services/replicator/reporter/recorder.js.map +1 -1
- package/out/zero-cache/src/services/replicator/reporter/report-schema.d.ts +1 -1
- package/out/zero-cache/src/services/replicator/schema/backfilling.d.ts +119 -0
- package/out/zero-cache/src/services/replicator/schema/backfilling.d.ts.map +1 -0
- package/out/zero-cache/src/services/replicator/schema/backfilling.js +214 -0
- package/out/zero-cache/src/services/replicator/schema/backfilling.js.map +1 -0
- package/out/zero-cache/src/services/replicator/schema/change-log.d.ts +3 -3
- package/out/zero-cache/src/services/replicator/schema/column-metadata.js +1 -1
- package/out/zero-cache/src/services/replicator/schema/column-metadata.js.map +1 -1
- package/out/zero-cache/src/services/replicator/schema/replication-state.d.ts +1 -1
- package/out/zero-cache/src/services/replicator/schema/replication-state.d.ts.map +1 -1
- package/out/zero-cache/src/services/replicator/schema/replication-state.js +9 -5
- package/out/zero-cache/src/services/replicator/schema/replication-state.js.map +1 -1
- package/out/zero-cache/src/services/replicator/sqlite-change-log-observability.d.ts +131 -0
- package/out/zero-cache/src/services/replicator/sqlite-change-log-observability.d.ts.map +1 -0
- package/out/zero-cache/src/services/replicator/sqlite-change-log-observability.js +413 -0
- package/out/zero-cache/src/services/replicator/sqlite-change-log-observability.js.map +1 -0
- package/out/zero-cache/src/services/replicator/sqlite-change-log-purger.d.ts +120 -0
- package/out/zero-cache/src/services/replicator/sqlite-change-log-purger.d.ts.map +1 -0
- package/out/zero-cache/src/services/replicator/sqlite-change-log-purger.js +283 -0
- package/out/zero-cache/src/services/replicator/sqlite-change-log-purger.js.map +1 -0
- package/out/zero-cache/src/services/replicator/write-worker.js +23 -12
- package/out/zero-cache/src/services/replicator/write-worker.js.map +1 -1
- package/out/zero-cache/src/services/run-ast.d.ts.map +1 -1
- package/out/zero-cache/src/services/run-ast.js +2 -1
- package/out/zero-cache/src/services/run-ast.js.map +1 -1
- package/out/zero-cache/src/services/running-state.d.ts +1 -1
- package/out/zero-cache/src/services/running-state.d.ts.map +1 -1
- package/out/zero-cache/src/services/running-state.js +1 -0
- package/out/zero-cache/src/services/running-state.js.map +1 -1
- package/out/zero-cache/src/services/view-syncer/e2e-serving-lag.d.ts +53 -0
- package/out/zero-cache/src/services/view-syncer/e2e-serving-lag.d.ts.map +1 -0
- package/out/zero-cache/src/services/view-syncer/e2e-serving-lag.js +68 -0
- package/out/zero-cache/src/services/view-syncer/e2e-serving-lag.js.map +1 -0
- package/out/zero-cache/src/services/view-syncer/pipeline-driver.d.ts.map +1 -1
- package/out/zero-cache/src/services/view-syncer/pipeline-driver.js +18 -3
- package/out/zero-cache/src/services/view-syncer/pipeline-driver.js.map +1 -1
- package/out/zero-cache/src/services/view-syncer/snapshotter.d.ts.map +1 -1
- package/out/zero-cache/src/services/view-syncer/snapshotter.js.map +1 -1
- package/out/zero-cache/src/services/view-syncer/view-syncer.d.ts +11 -1
- package/out/zero-cache/src/services/view-syncer/view-syncer.d.ts.map +1 -1
- package/out/zero-cache/src/services/view-syncer/view-syncer.js +64 -6
- package/out/zero-cache/src/services/view-syncer/view-syncer.js.map +1 -1
- package/out/zero-cache/src/types/processes.d.ts +1 -2
- package/out/zero-cache/src/types/processes.d.ts.map +1 -1
- package/out/zero-cache/src/types/processes.js +1 -25
- package/out/zero-cache/src/types/processes.js.map +1 -1
- package/out/zero-cache/src/types/streams.d.ts +21 -0
- package/out/zero-cache/src/types/streams.d.ts.map +1 -1
- package/out/zero-cache/src/types/streams.js +25 -4
- package/out/zero-cache/src/types/streams.js.map +1 -1
- package/out/zero-cache/src/types/subscription.d.ts +11 -0
- package/out/zero-cache/src/types/subscription.d.ts.map +1 -1
- package/out/zero-cache/src/types/subscription.js +27 -0
- package/out/zero-cache/src/types/subscription.js.map +1 -1
- package/out/zero-cache/src/workers/connection.d.ts +1 -0
- package/out/zero-cache/src/workers/connection.d.ts.map +1 -1
- package/out/zero-cache/src/workers/connection.js +24 -2
- package/out/zero-cache/src/workers/connection.js.map +1 -1
- package/out/zero-cache/src/workers/replicator.d.ts +25 -0
- package/out/zero-cache/src/workers/replicator.d.ts.map +1 -1
- package/out/zero-cache/src/workers/replicator.js +33 -1
- package/out/zero-cache/src/workers/replicator.js.map +1 -1
- package/out/zero-cache/src/workers/syncer.d.ts +12 -0
- package/out/zero-cache/src/workers/syncer.d.ts.map +1 -1
- package/out/zero-cache/src/workers/syncer.js +75 -0
- package/out/zero-cache/src/workers/syncer.js.map +1 -1
- package/out/zero-client/src/client/version.js +1 -1
- package/out/zero-client/src/client/zero.d.ts +6 -7
- package/out/zero-client/src/client/zero.d.ts.map +1 -1
- package/out/zero-client/src/client/zero.js +156 -101
- package/out/zero-client/src/client/zero.js.map +1 -1
- package/out/zero-schema/src/builder/table-builder.d.ts +23 -0
- package/out/zero-schema/src/builder/table-builder.d.ts.map +1 -1
- package/out/zero-schema/src/builder/table-builder.js +18 -0
- package/out/zero-schema/src/builder/table-builder.js.map +1 -1
- package/out/zero-server/src/custom.js +10 -3
- package/out/zero-server/src/custom.js.map +1 -1
- package/out/zero-server/src/schema.d.ts +8 -0
- package/out/zero-server/src/schema.d.ts.map +1 -1
- package/out/zero-server/src/schema.js +63 -24
- package/out/zero-server/src/schema.js.map +1 -1
- package/out/zero-types/src/schema.d.ts +11 -0
- package/out/zero-types/src/schema.d.ts.map +1 -1
- package/out/zql/src/mutate/crud.d.ts +10 -8
- package/out/zql/src/mutate/crud.d.ts.map +1 -1
- package/out/zql/src/mutate/crud.js.map +1 -1
- package/out/zql/src/query/query-delegate-base.d.ts.map +1 -1
- package/out/zql/src/query/query-delegate-base.js +5 -1
- package/out/zql/src/query/query-delegate-base.js.map +1 -1
- package/out/zql/src/query/query-impl.d.ts +2 -2
- package/out/zql/src/query/query-impl.d.ts.map +1 -1
- package/out/zql/src/query/query-impl.js.map +1 -1
- package/out/zql/src/query/query.d.ts +83 -14
- package/out/zql/src/query/query.d.ts.map +1 -1
- package/out/zql/src/query/static-query.d.ts +1 -1
- package/out/zql/src/query/static-query.d.ts.map +1 -1
- package/out/zqlite/src/db.js +2 -2
- package/out/zqlite/src/resolve-scalar-subqueries.d.ts +15 -0
- package/out/zqlite/src/resolve-scalar-subqueries.d.ts.map +1 -1
- package/out/zqlite/src/resolve-scalar-subqueries.js +33 -14
- package/out/zqlite/src/resolve-scalar-subqueries.js.map +1 -1
- package/out/zqlite/src/sqlite-cost-model.js +5 -5
- package/out/zqlite/src/table-source.js +3 -3
- package/package.json +3 -3
- package/out/replicache/src/db/index.js +0 -91
- package/out/replicache/src/db/index.js.map +0 -1
- package/out/replicache/src/lazy.d.ts +0 -2
- package/out/replicache/src/lazy.d.ts.map +0 -1
- package/out/replicache/src/lazy.js +0 -12
- package/out/replicache/src/lazy.js.map +0 -1
- package/out/zero-cache/src/server/backup-watermark-reader.js +0 -35
- package/out/zero-cache/src/server/backup-watermark-reader.js.map +0 -1
- package/out/zero-cache/src/services/change-streamer/litestream3-backup-monitor.d.ts.map +0 -1
- package/out/zero-cache/src/services/change-streamer/litestream3-backup-monitor.js.map +0 -1
- package/out/zero-cache/src/services/change-streamer/replica-monitor.d.ts +0 -16
- package/out/zero-cache/src/services/change-streamer/replica-monitor.d.ts.map +0 -1
- package/out/zero-cache/src/services/change-streamer/replica-monitor.js +0 -54
- package/out/zero-cache/src/services/change-streamer/replica-monitor.js.map +0 -1
- package/out/zero-cache/src/services/change-streamer/vfs-backup-monitor.d.ts +0 -25
- package/out/zero-cache/src/services/change-streamer/vfs-backup-monitor.d.ts.map +0 -1
- package/out/zero-cache/src/services/change-streamer/vfs-backup-monitor.js +0 -159
- package/out/zero-cache/src/services/change-streamer/vfs-backup-monitor.js.map +0 -1
- package/out/zero-cache/src/services/litestream/vfs-watermark-reader.d.ts +0 -44
- package/out/zero-cache/src/services/litestream/vfs-watermark-reader.d.ts.map +0 -1
- package/out/zero-cache/src/services/litestream/vfs-watermark-reader.js +0 -65
- package/out/zero-cache/src/services/litestream/vfs-watermark-reader.js.map +0 -1
- package/out/zero-cache/src/services/litestream/vfs-watermark-worker-source.d.ts +0 -11
- package/out/zero-cache/src/services/litestream/vfs-watermark-worker-source.d.ts.map +0 -1
- package/out/zero-cache/src/services/litestream/vfs-watermark-worker-source.js +0 -74
- package/out/zero-cache/src/services/litestream/vfs-watermark-worker-source.js.map +0 -1
- package/out/zero-cache/src/services/litestream/vfs-watermark-worker.d.ts +0 -42
- package/out/zero-cache/src/services/litestream/vfs-watermark-worker.d.ts.map +0 -1
- package/out/zero-cache/src/services/litestream/vfs-watermark-worker.js +0 -117
- package/out/zero-cache/src/services/litestream/vfs-watermark-worker.js.map +0 -1
|
@@ -0,0 +1,434 @@
|
|
|
1
|
+
import { unreachable } from "../../../../shared/src/asserts.js";
|
|
2
|
+
import { parse, valita_exports } from "../../../../shared/src/valita.js";
|
|
3
|
+
import { BigIntJSON } from "../../../../shared/src/bigint-json.js";
|
|
4
|
+
import { backfillRequestSchema } from "../change-source/protocol/current/upstream.js";
|
|
5
|
+
//#region ../zero-cache/src/services/replicator/change-log-cookies.ts
|
|
6
|
+
/**
|
|
7
|
+
* The change log's *second* job.
|
|
8
|
+
*
|
|
9
|
+
* A change log is a buffer **and** a cookie jar. The buffer catches a
|
|
10
|
+
* reconnecting subscriber up to the head; the cookie jar holds the backfill
|
|
11
|
+
* progress state that the change-streamer hands the change-source on every
|
|
12
|
+
* `startStream`, so that the change source never has to persist backfill state
|
|
13
|
+
* upstream. See `change-log-db.ts` for the buffer.
|
|
14
|
+
*
|
|
15
|
+
* There are two cookies, mirroring the two `cdc` tables the Postgres change log
|
|
16
|
+
* has always carried (`change-streamer/schema/tables.ts`):
|
|
17
|
+
*
|
|
18
|
+
* - **table metadata**, the most current {@link TableMetadata} of a table, which
|
|
19
|
+
* a {@link BackfillRequest} has to carry; and
|
|
20
|
+
* - **backfilling columns**, the opaque {@link BackfillID} of every column whose
|
|
21
|
+
* backfill has not completed.
|
|
22
|
+
*
|
|
23
|
+
* Both are a *fold over the change stream*: the cookie set is only meaningful
|
|
24
|
+
* paired with the watermark it was folded up to. {@link cookieOps} is that fold,
|
|
25
|
+
* expressed transport-free so that the Postgres change log and the SQLite change
|
|
26
|
+
* log cannot drift from each other — they must agree on all eight transitions
|
|
27
|
+
* forever, and the failure mode if they disagree is a backfill that is silently
|
|
28
|
+
* never re-requested. The two interpreters are {@link ChangeLogCookieWriter}
|
|
29
|
+
* here and `Storer.#trackBackfillMetadata()` in
|
|
30
|
+
* `change-streamer/storer.ts`.
|
|
31
|
+
*
|
|
32
|
+
* The tables live in the change-log database rather than in the replica: the
|
|
33
|
+
* replicator is a subscriber *downstream* of the change-streamer's write loop,
|
|
34
|
+
* so its state version trails the log's head, and pairing its cookies with the
|
|
35
|
+
* log's resume watermark would lose every backfill that completed in between.
|
|
36
|
+
* The replica's own copies (`_zero.tableMetadata`, `_zero.column_metadata`) are
|
|
37
|
+
* the *seed* source; these are the *working* set.
|
|
38
|
+
*/
|
|
39
|
+
var CHANGE_LOG_TABLE_METADATA_TABLE = "_zero.changeLogTableMetadata";
|
|
40
|
+
var CHANGE_LOG_BACKFILLING_TABLE = "_zero.changeLogBackfilling";
|
|
41
|
+
var CREATE_CHANGE_LOG_COOKIE_SCHEMA = `
|
|
42
|
+
CREATE TABLE "${CHANGE_LOG_TABLE_METADATA_TABLE}" (
|
|
43
|
+
"schema" TEXT NOT NULL,
|
|
44
|
+
"table" TEXT NOT NULL,
|
|
45
|
+
"metadata" TEXT NOT NULL,
|
|
46
|
+
PRIMARY KEY ("schema", "table")
|
|
47
|
+
);
|
|
48
|
+
|
|
49
|
+
CREATE TABLE "${CHANGE_LOG_BACKFILLING_TABLE}" (
|
|
50
|
+
"schema" TEXT NOT NULL,
|
|
51
|
+
"table" TEXT NOT NULL,
|
|
52
|
+
"column" TEXT NOT NULL,
|
|
53
|
+
"backfill" TEXT NOT NULL,
|
|
54
|
+
PRIMARY KEY ("schema", "table", "column")
|
|
55
|
+
);
|
|
56
|
+
`;
|
|
57
|
+
var DROP_CHANGE_LOG_COOKIE_TABLES = `
|
|
58
|
+
DROP TABLE IF EXISTS "${CHANGE_LOG_TABLE_METADATA_TABLE}";
|
|
59
|
+
DROP TABLE IF EXISTS "${CHANGE_LOG_BACKFILLING_TABLE}";
|
|
60
|
+
`;
|
|
61
|
+
/**
|
|
62
|
+
* The fold, and the only place either store decides what a schema change means
|
|
63
|
+
* for the cookie set. Both interpreters are mechanical translations of what this
|
|
64
|
+
* returns, in the order it returns them.
|
|
65
|
+
*
|
|
66
|
+
* Changes that carry no cookie state — index changes, and the `create-table` /
|
|
67
|
+
* `add-column` variants whose optional `metadata` and `backfill` fields are
|
|
68
|
+
* absent — produce no ops.
|
|
69
|
+
*/
|
|
70
|
+
function cookieOps(change) {
|
|
71
|
+
switch (change.tag) {
|
|
72
|
+
case "update-table-metadata": return [{
|
|
73
|
+
op: "upsert-metadata",
|
|
74
|
+
table: change.table,
|
|
75
|
+
metadata: change.new
|
|
76
|
+
}];
|
|
77
|
+
case "create-table": {
|
|
78
|
+
const { spec, metadata, backfill } = change;
|
|
79
|
+
const table = {
|
|
80
|
+
schema: spec.schema,
|
|
81
|
+
name: spec.name
|
|
82
|
+
};
|
|
83
|
+
const ops = [];
|
|
84
|
+
if (metadata) ops.push({
|
|
85
|
+
op: "upsert-metadata",
|
|
86
|
+
table,
|
|
87
|
+
metadata
|
|
88
|
+
});
|
|
89
|
+
if (backfill) for (const [column, id] of Object.entries(backfill)) ops.push({
|
|
90
|
+
op: "upsert-backfill",
|
|
91
|
+
table,
|
|
92
|
+
column,
|
|
93
|
+
backfill: id
|
|
94
|
+
});
|
|
95
|
+
return ops;
|
|
96
|
+
}
|
|
97
|
+
case "rename-table": return [{
|
|
98
|
+
op: "rename-table",
|
|
99
|
+
old: change.old,
|
|
100
|
+
new: change.new
|
|
101
|
+
}];
|
|
102
|
+
case "drop-table": return [{
|
|
103
|
+
op: "drop-table",
|
|
104
|
+
table: change.id
|
|
105
|
+
}];
|
|
106
|
+
case "add-column": {
|
|
107
|
+
const { table, tableMetadata, column, backfill } = change;
|
|
108
|
+
const ops = [];
|
|
109
|
+
if (tableMetadata) ops.push({
|
|
110
|
+
op: "upsert-metadata",
|
|
111
|
+
table,
|
|
112
|
+
metadata: tableMetadata
|
|
113
|
+
});
|
|
114
|
+
if (backfill) ops.push({
|
|
115
|
+
op: "upsert-backfill",
|
|
116
|
+
table,
|
|
117
|
+
column: column.name,
|
|
118
|
+
backfill
|
|
119
|
+
});
|
|
120
|
+
return ops;
|
|
121
|
+
}
|
|
122
|
+
case "update-column": {
|
|
123
|
+
const { table, old, new: updated } = change;
|
|
124
|
+
return old.name === updated.name ? [] : [{
|
|
125
|
+
op: "rename-column",
|
|
126
|
+
table,
|
|
127
|
+
old: old.name,
|
|
128
|
+
new: updated.name
|
|
129
|
+
}];
|
|
130
|
+
}
|
|
131
|
+
case "drop-column": return [{
|
|
132
|
+
op: "drop-column",
|
|
133
|
+
table: change.table,
|
|
134
|
+
column: change.column
|
|
135
|
+
}];
|
|
136
|
+
case "backfill-completed": {
|
|
137
|
+
const { relation, columns } = change;
|
|
138
|
+
const cleared = [...relation.rowKey.columns, ...columns];
|
|
139
|
+
return cleared.length === 0 ? [] : [{
|
|
140
|
+
op: "complete-backfill",
|
|
141
|
+
table: {
|
|
142
|
+
schema: relation.schema,
|
|
143
|
+
name: relation.name
|
|
144
|
+
},
|
|
145
|
+
columns: cleared
|
|
146
|
+
}];
|
|
147
|
+
}
|
|
148
|
+
case "create-index":
|
|
149
|
+
case "drop-index": return [];
|
|
150
|
+
default: unreachable(change);
|
|
151
|
+
}
|
|
152
|
+
}
|
|
153
|
+
/**
|
|
154
|
+
* The SQLite interpreter of {@link cookieOps}, over prepared statements on the
|
|
155
|
+
* change-log database.
|
|
156
|
+
*
|
|
157
|
+
* The statements run inside the log's transaction, in stream order, at the point
|
|
158
|
+
* the schema change is appended (see `ChangeLogStreamWriter`). That is what
|
|
159
|
+
* makes the cookie set atomic with the head — a crash cannot leave cookies ahead
|
|
160
|
+
* of or behind the rows they were folded from — and what makes an aborted
|
|
161
|
+
* transaction discard its cookies with its rows, for free.
|
|
162
|
+
*
|
|
163
|
+
* Statements are prepared against the tables as they exist now, so an instance
|
|
164
|
+
* does not survive a reconcile that drops and recreates them.
|
|
165
|
+
*/
|
|
166
|
+
var ChangeLogCookieWriter = class {
|
|
167
|
+
#upsertMetadata;
|
|
168
|
+
#upsertBackfill;
|
|
169
|
+
#renameMetadataTable;
|
|
170
|
+
#renameBackfillTable;
|
|
171
|
+
#deleteMetadataTable;
|
|
172
|
+
#deleteBackfillTable;
|
|
173
|
+
#renameBackfillColumn;
|
|
174
|
+
#deleteBackfillColumn;
|
|
175
|
+
constructor(db) {
|
|
176
|
+
this.#upsertMetadata = db.prepare(`
|
|
177
|
+
INSERT INTO "${CHANGE_LOG_TABLE_METADATA_TABLE}"
|
|
178
|
+
("schema", "table", "metadata") VALUES (?, ?, ?)
|
|
179
|
+
ON CONFLICT ("schema", "table")
|
|
180
|
+
DO UPDATE SET "metadata" = excluded."metadata"
|
|
181
|
+
`);
|
|
182
|
+
this.#upsertBackfill = db.prepare(`
|
|
183
|
+
INSERT INTO "${CHANGE_LOG_BACKFILLING_TABLE}"
|
|
184
|
+
("schema", "table", "column", "backfill") VALUES (?, ?, ?, ?)
|
|
185
|
+
ON CONFLICT ("schema", "table", "column")
|
|
186
|
+
DO UPDATE SET "backfill" = excluded."backfill"
|
|
187
|
+
`);
|
|
188
|
+
this.#renameMetadataTable = db.prepare(`
|
|
189
|
+
UPDATE "${CHANGE_LOG_TABLE_METADATA_TABLE}"
|
|
190
|
+
SET "schema" = ?, "table" = ? WHERE "schema" = ? AND "table" = ?
|
|
191
|
+
`);
|
|
192
|
+
this.#renameBackfillTable = db.prepare(`
|
|
193
|
+
UPDATE "${CHANGE_LOG_BACKFILLING_TABLE}"
|
|
194
|
+
SET "schema" = ?, "table" = ? WHERE "schema" = ? AND "table" = ?
|
|
195
|
+
`);
|
|
196
|
+
this.#deleteMetadataTable = db.prepare(`
|
|
197
|
+
DELETE FROM "${CHANGE_LOG_TABLE_METADATA_TABLE}"
|
|
198
|
+
WHERE "schema" = ? AND "table" = ?
|
|
199
|
+
`);
|
|
200
|
+
this.#deleteBackfillTable = db.prepare(`
|
|
201
|
+
DELETE FROM "${CHANGE_LOG_BACKFILLING_TABLE}"
|
|
202
|
+
WHERE "schema" = ? AND "table" = ?
|
|
203
|
+
`);
|
|
204
|
+
this.#renameBackfillColumn = db.prepare(`
|
|
205
|
+
UPDATE "${CHANGE_LOG_BACKFILLING_TABLE}" SET "column" = ?
|
|
206
|
+
WHERE "schema" = ? AND "table" = ? AND "column" = ?
|
|
207
|
+
`);
|
|
208
|
+
this.#deleteBackfillColumn = db.prepare(`
|
|
209
|
+
DELETE FROM "${CHANGE_LOG_BACKFILLING_TABLE}"
|
|
210
|
+
WHERE "schema" = ? AND "table" = ? AND "column" = ?
|
|
211
|
+
`);
|
|
212
|
+
}
|
|
213
|
+
/** Applies the change's cookie ops, returning the ops that were applied. */
|
|
214
|
+
apply(change) {
|
|
215
|
+
const ops = cookieOps(change);
|
|
216
|
+
for (const op of ops) this.#run(op);
|
|
217
|
+
return ops;
|
|
218
|
+
}
|
|
219
|
+
#run(op) {
|
|
220
|
+
switch (op.op) {
|
|
221
|
+
case "upsert-metadata":
|
|
222
|
+
this.#upsertMetadata.run(op.table.schema, op.table.name, BigIntJSON.stringify(op.metadata));
|
|
223
|
+
break;
|
|
224
|
+
case "upsert-backfill":
|
|
225
|
+
this.#upsertBackfill.run(op.table.schema, op.table.name, op.column, BigIntJSON.stringify(op.backfill));
|
|
226
|
+
break;
|
|
227
|
+
case "rename-table":
|
|
228
|
+
this.#renameMetadataTable.run(op.new.schema, op.new.name, op.old.schema, op.old.name);
|
|
229
|
+
this.#renameBackfillTable.run(op.new.schema, op.new.name, op.old.schema, op.old.name);
|
|
230
|
+
break;
|
|
231
|
+
case "drop-table":
|
|
232
|
+
this.#deleteMetadataTable.run(op.table.schema, op.table.name);
|
|
233
|
+
this.#deleteBackfillTable.run(op.table.schema, op.table.name);
|
|
234
|
+
break;
|
|
235
|
+
case "rename-column":
|
|
236
|
+
this.#renameBackfillColumn.run(op.new, op.table.schema, op.table.name, op.old);
|
|
237
|
+
break;
|
|
238
|
+
case "drop-column":
|
|
239
|
+
this.#deleteBackfillColumn.run(op.table.schema, op.table.name, op.column);
|
|
240
|
+
break;
|
|
241
|
+
case "complete-backfill":
|
|
242
|
+
for (const column of op.columns) this.#deleteBackfillColumn.run(op.table.schema, op.table.name, column);
|
|
243
|
+
break;
|
|
244
|
+
default: unreachable(op);
|
|
245
|
+
}
|
|
246
|
+
}
|
|
247
|
+
};
|
|
248
|
+
var tableKey = (schema, table) => JSON.stringify([schema, table]);
|
|
249
|
+
var columnKey = (schema, table, column) => JSON.stringify([
|
|
250
|
+
schema,
|
|
251
|
+
table,
|
|
252
|
+
column
|
|
253
|
+
]);
|
|
254
|
+
var cmp = (a, b) => a < b ? -1 : a > b ? 1 : 0;
|
|
255
|
+
/**
|
|
256
|
+
* The in-memory interpreter of {@link cookieOps}: the third one, and the only
|
|
257
|
+
* one that is not a store.
|
|
258
|
+
*
|
|
259
|
+
* A replica-derived cookie set is a snapshot at the replica's state version,
|
|
260
|
+
* which trails the change log's head, so it cannot be compared with a
|
|
261
|
+
* Postgres-derived set directly. This carries it forward over the schema changes
|
|
262
|
+
* the log holds in between, which turns "the two stores disagree, but they are
|
|
263
|
+
* also at different positions" into an exact equality — and makes a
|
|
264
|
+
* disagreement between any two of the three interpreters visible as an
|
|
265
|
+
* inequality that neither store would have noticed on its own.
|
|
266
|
+
*
|
|
267
|
+
* `changes` must be the log's schema changes over `(from, to]` in stream order.
|
|
268
|
+
* Anything else produces a set that belongs to no watermark.
|
|
269
|
+
*/
|
|
270
|
+
function foldCookies(cookies, changes) {
|
|
271
|
+
const metadata = new Map(cookies.tableMetadata.map((c) => [tableKey(c.schema, c.table), c]));
|
|
272
|
+
const backfilling = new Map(cookies.backfilling.map((c) => [columnKey(c.schema, c.table, c.column), c]));
|
|
273
|
+
const dropColumn = ({ schema, name }, column) => backfilling.delete(columnKey(schema, name, column));
|
|
274
|
+
for (const change of changes) for (const op of cookieOps(change)) switch (op.op) {
|
|
275
|
+
case "upsert-metadata":
|
|
276
|
+
metadata.set(tableKey(op.table.schema, op.table.name), {
|
|
277
|
+
schema: op.table.schema,
|
|
278
|
+
table: op.table.name,
|
|
279
|
+
metadata: op.metadata
|
|
280
|
+
});
|
|
281
|
+
break;
|
|
282
|
+
case "upsert-backfill":
|
|
283
|
+
backfilling.set(columnKey(op.table.schema, op.table.name, op.column), {
|
|
284
|
+
schema: op.table.schema,
|
|
285
|
+
table: op.table.name,
|
|
286
|
+
column: op.column,
|
|
287
|
+
backfill: op.backfill
|
|
288
|
+
});
|
|
289
|
+
break;
|
|
290
|
+
case "rename-table": {
|
|
291
|
+
const { old, new: renamed } = op;
|
|
292
|
+
const moved = metadata.get(tableKey(old.schema, old.name));
|
|
293
|
+
if (moved) {
|
|
294
|
+
metadata.delete(tableKey(old.schema, old.name));
|
|
295
|
+
metadata.set(tableKey(renamed.schema, renamed.name), {
|
|
296
|
+
...moved,
|
|
297
|
+
schema: renamed.schema,
|
|
298
|
+
table: renamed.name
|
|
299
|
+
});
|
|
300
|
+
}
|
|
301
|
+
for (const [key, cookie] of [...backfilling]) if (cookie.schema === old.schema && cookie.table === old.name) {
|
|
302
|
+
backfilling.delete(key);
|
|
303
|
+
backfilling.set(columnKey(renamed.schema, renamed.name, cookie.column), {
|
|
304
|
+
...cookie,
|
|
305
|
+
schema: renamed.schema,
|
|
306
|
+
table: renamed.name
|
|
307
|
+
});
|
|
308
|
+
}
|
|
309
|
+
break;
|
|
310
|
+
}
|
|
311
|
+
case "drop-table": {
|
|
312
|
+
const { schema, name } = op.table;
|
|
313
|
+
metadata.delete(tableKey(schema, name));
|
|
314
|
+
for (const [key, cookie] of [...backfilling]) if (cookie.schema === schema && cookie.table === name) backfilling.delete(key);
|
|
315
|
+
break;
|
|
316
|
+
}
|
|
317
|
+
case "rename-column": {
|
|
318
|
+
const { schema, name } = op.table;
|
|
319
|
+
const moved = backfilling.get(columnKey(schema, name, op.old));
|
|
320
|
+
if (moved) {
|
|
321
|
+
backfilling.delete(columnKey(schema, name, op.old));
|
|
322
|
+
backfilling.set(columnKey(schema, name, op.new), {
|
|
323
|
+
...moved,
|
|
324
|
+
column: op.new
|
|
325
|
+
});
|
|
326
|
+
}
|
|
327
|
+
break;
|
|
328
|
+
}
|
|
329
|
+
case "drop-column":
|
|
330
|
+
dropColumn(op.table, op.column);
|
|
331
|
+
break;
|
|
332
|
+
case "complete-backfill":
|
|
333
|
+
for (const column of op.columns) dropColumn(op.table, column);
|
|
334
|
+
break;
|
|
335
|
+
default: unreachable(op);
|
|
336
|
+
}
|
|
337
|
+
return {
|
|
338
|
+
tableMetadata: [...metadata.values()].toSorted((a, b) => cmp(a.schema, b.schema) || cmp(a.table, b.table)),
|
|
339
|
+
backfilling: [...backfilling.values()].toSorted((a, b) => cmp(a.schema, b.schema) || cmp(a.table, b.table) || cmp(a.column, b.column))
|
|
340
|
+
};
|
|
341
|
+
}
|
|
342
|
+
var backfillRequestsSchema = valita_exports.array(backfillRequestSchema);
|
|
343
|
+
/**
|
|
344
|
+
* Returns one {@link BackfillRequest} for each table with an active backfill.
|
|
345
|
+
* Each request contains the stored table metadata, if available.
|
|
346
|
+
*/
|
|
347
|
+
function backfillRequestsFrom(cookies) {
|
|
348
|
+
const metadata = new Map(cookies.tableMetadata.map((c) => [tableKey(c.schema, c.table), c.metadata]));
|
|
349
|
+
const requests = [];
|
|
350
|
+
let curr;
|
|
351
|
+
for (const { schema, table, column, backfill } of cookies.backfilling) {
|
|
352
|
+
if (curr?.table.schema !== schema || curr.table.name !== table) {
|
|
353
|
+
curr = {
|
|
354
|
+
table: {
|
|
355
|
+
schema,
|
|
356
|
+
name: table,
|
|
357
|
+
metadata: metadata.get(tableKey(schema, table)) ?? null
|
|
358
|
+
},
|
|
359
|
+
columns: {}
|
|
360
|
+
};
|
|
361
|
+
requests.push(curr);
|
|
362
|
+
}
|
|
363
|
+
curr.columns[column] = backfill;
|
|
364
|
+
}
|
|
365
|
+
return parse(requests, backfillRequestsSchema);
|
|
366
|
+
}
|
|
367
|
+
/**
|
|
368
|
+
* Reads the whole cookie set, ordered by primary key. This is what a
|
|
369
|
+
* change-streamer would hand `startStream`, and what the Postgres-derived and
|
|
370
|
+
* replica-derived sets are compared against.
|
|
371
|
+
*/
|
|
372
|
+
function readCookies(db) {
|
|
373
|
+
return {
|
|
374
|
+
tableMetadata: db.prepare(`
|
|
375
|
+
SELECT "schema", "table", "metadata"
|
|
376
|
+
FROM "${CHANGE_LOG_TABLE_METADATA_TABLE}"
|
|
377
|
+
ORDER BY "schema", "table"
|
|
378
|
+
`).all().map(({ schema, table, metadata }) => ({
|
|
379
|
+
schema,
|
|
380
|
+
table,
|
|
381
|
+
metadata: BigIntJSON.parse(metadata)
|
|
382
|
+
})),
|
|
383
|
+
backfilling: db.prepare(`
|
|
384
|
+
SELECT "schema", "table", "column", "backfill"
|
|
385
|
+
FROM "${CHANGE_LOG_BACKFILLING_TABLE}"
|
|
386
|
+
ORDER BY "schema", "table", "column"
|
|
387
|
+
`).all().map(({ schema, table, column, backfill }) => ({
|
|
388
|
+
schema,
|
|
389
|
+
table,
|
|
390
|
+
column,
|
|
391
|
+
backfill: BigIntJSON.parse(backfill)
|
|
392
|
+
}))
|
|
393
|
+
};
|
|
394
|
+
}
|
|
395
|
+
/**
|
|
396
|
+
* Replaces the cookie set wholesale.
|
|
397
|
+
*
|
|
398
|
+
* The cookies are unversioned point-in-time state, so a reconciliation that
|
|
399
|
+
* moves the head invalidates them: truncating the stream above `W` does not roll
|
|
400
|
+
* the cookie set back to `W`, and the truncated transactions can contain
|
|
401
|
+
* `create-table`, `add-column`, and `backfill-completed`. The caller must run
|
|
402
|
+
* this in the same transaction as the reconciliation that invalidated them.
|
|
403
|
+
*/
|
|
404
|
+
function replaceCookies(db, cookies) {
|
|
405
|
+
db.prepare(`DELETE FROM "${CHANGE_LOG_TABLE_METADATA_TABLE}"`).run();
|
|
406
|
+
db.prepare(`DELETE FROM "${CHANGE_LOG_BACKFILLING_TABLE}"`).run();
|
|
407
|
+
if (cookies.tableMetadata.length > 0) {
|
|
408
|
+
const insert = db.prepare(`
|
|
409
|
+
INSERT INTO "${CHANGE_LOG_TABLE_METADATA_TABLE}"
|
|
410
|
+
("schema", "table", "metadata") VALUES (?, ?, ?)
|
|
411
|
+
`);
|
|
412
|
+
for (const { schema, table, metadata } of cookies.tableMetadata) insert.run(schema, table, BigIntJSON.stringify(metadata));
|
|
413
|
+
}
|
|
414
|
+
if (cookies.backfilling.length > 0) {
|
|
415
|
+
const insert = db.prepare(`
|
|
416
|
+
INSERT INTO "${CHANGE_LOG_BACKFILLING_TABLE}"
|
|
417
|
+
("schema", "table", "column", "backfill") VALUES (?, ?, ?, ?)
|
|
418
|
+
`);
|
|
419
|
+
for (const { schema, table, column, backfill } of cookies.backfilling) insert.run(schema, table, column, BigIntJSON.stringify(backfill));
|
|
420
|
+
}
|
|
421
|
+
}
|
|
422
|
+
function readCookieRowCounts(db) {
|
|
423
|
+
return db.prepare(`
|
|
424
|
+
SELECT
|
|
425
|
+
(SELECT count(*) FROM "${CHANGE_LOG_TABLE_METADATA_TABLE}")
|
|
426
|
+
AS "tableMetadata",
|
|
427
|
+
(SELECT count(*) FROM "${CHANGE_LOG_BACKFILLING_TABLE}")
|
|
428
|
+
AS "backfilling"
|
|
429
|
+
`).get();
|
|
430
|
+
}
|
|
431
|
+
//#endregion
|
|
432
|
+
export { CHANGE_LOG_BACKFILLING_TABLE, CHANGE_LOG_TABLE_METADATA_TABLE, CREATE_CHANGE_LOG_COOKIE_SCHEMA, ChangeLogCookieWriter, DROP_CHANGE_LOG_COOKIE_TABLES, backfillRequestsFrom, cookieOps, foldCookies, readCookieRowCounts, readCookies, replaceCookies };
|
|
433
|
+
|
|
434
|
+
//# sourceMappingURL=change-log-cookies.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"change-log-cookies.js","names":["#upsertMetadata","#upsertBackfill","#renameMetadataTable","#renameBackfillTable","#deleteMetadataTable","#deleteBackfillTable","#renameBackfillColumn","#deleteBackfillColumn","#run"],"sources":["../../../../../../zero-cache/src/services/replicator/change-log-cookies.ts"],"sourcesContent":["/**\n * The change log's *second* job.\n *\n * A change log is a buffer **and** a cookie jar. The buffer catches a\n * reconnecting subscriber up to the head; the cookie jar holds the backfill\n * progress state that the change-streamer hands the change-source on every\n * `startStream`, so that the change source never has to persist backfill state\n * upstream. See `change-log-db.ts` for the buffer.\n *\n * There are two cookies, mirroring the two `cdc` tables the Postgres change log\n * has always carried (`change-streamer/schema/tables.ts`):\n *\n * - **table metadata**, the most current {@link TableMetadata} of a table, which\n * a {@link BackfillRequest} has to carry; and\n * - **backfilling columns**, the opaque {@link BackfillID} of every column whose\n * backfill has not completed.\n *\n * Both are a *fold over the change stream*: the cookie set is only meaningful\n * paired with the watermark it was folded up to. {@link cookieOps} is that fold,\n * expressed transport-free so that the Postgres change log and the SQLite change\n * log cannot drift from each other — they must agree on all eight transitions\n * forever, and the failure mode if they disagree is a backfill that is silently\n * never re-requested. The two interpreters are {@link ChangeLogCookieWriter}\n * here and `Storer.#trackBackfillMetadata()` in\n * `change-streamer/storer.ts`.\n *\n * The tables live in the change-log database rather than in the replica: the\n * replicator is a subscriber *downstream* of the change-streamer's write loop,\n * so its state version trails the log's head, and pairing its cookies with the\n * log's resume watermark would lose every backfill that completed in between.\n * The replica's own copies (`_zero.tableMetadata`, `_zero.column_metadata`) are\n * the *seed* source; these are the *working* set.\n */\n\nimport {unreachable} from '../../../../shared/src/asserts.ts';\nimport {BigIntJSON} from '../../../../shared/src/bigint-json.ts';\nimport * as v from '../../../../shared/src/valita.ts';\nimport type {Database, Statement} from '../../../../zqlite/src/db.ts';\nimport type {\n BackfillID,\n Identifier,\n SchemaChange,\n TableMetadata,\n} from '../change-source/protocol/current/data.ts';\nimport {\n backfillRequestSchema,\n type BackfillRequest,\n} from '../change-source/protocol/current/upstream.ts';\n\nexport const CHANGE_LOG_TABLE_METADATA_TABLE = '_zero.changeLogTableMetadata';\nexport const CHANGE_LOG_BACKFILLING_TABLE = '_zero.changeLogBackfilling';\n\n// The names follow the file's existing convention, beside\n// \"_zero.changeLogStream\" and \"_zero.changeLogMeta\".\n//\n// `metadata` and `backfill` hold the JSON that the `cdc` tables hold as JSONB.\n// SQLite has no JSONB, and nothing here queries into the documents: they are\n// stored to be handed back to the change source verbatim.\nexport const CREATE_CHANGE_LOG_COOKIE_SCHEMA = /*sql*/ `\n CREATE TABLE \"${CHANGE_LOG_TABLE_METADATA_TABLE}\" (\n \"schema\" TEXT NOT NULL,\n \"table\" TEXT NOT NULL,\n \"metadata\" TEXT NOT NULL,\n PRIMARY KEY (\"schema\", \"table\")\n );\n\n CREATE TABLE \"${CHANGE_LOG_BACKFILLING_TABLE}\" (\n \"schema\" TEXT NOT NULL,\n \"table\" TEXT NOT NULL,\n \"column\" TEXT NOT NULL,\n \"backfill\" TEXT NOT NULL,\n PRIMARY KEY (\"schema\", \"table\", \"column\")\n );\n`;\n\nexport const DROP_CHANGE_LOG_COOKIE_TABLES = /*sql*/ `\n DROP TABLE IF EXISTS \"${CHANGE_LOG_TABLE_METADATA_TABLE}\";\n DROP TABLE IF EXISTS \"${CHANGE_LOG_BACKFILLING_TABLE}\";\n`;\n\nexport type TableMetadataCookie = {\n readonly schema: string;\n readonly table: string;\n readonly metadata: TableMetadata;\n};\n\nexport type BackfillCookie = {\n readonly schema: string;\n readonly table: string;\n readonly column: string;\n readonly backfill: BackfillID;\n};\n\n/**\n * The whole cookie set, as of the log's head. Both lists are ordered by their\n * primary key so that two stores' sets can be compared directly.\n */\nexport type CookieSet = {\n readonly tableMetadata: readonly TableMetadataCookie[];\n readonly backfilling: readonly BackfillCookie[];\n};\n\nexport const EMPTY_COOKIE_SET: CookieSet = {\n tableMetadata: [],\n backfilling: [],\n};\n\n/**\n * A transport-free description of a cookie mutation. This is the whole of what\n * a {@link SchemaChange} means to the cookie jar; everything else about the\n * change belongs to the buffer.\n */\nexport type CookieOp =\n | {op: 'upsert-metadata'; table: Identifier; metadata: TableMetadata}\n | {\n op: 'upsert-backfill';\n table: Identifier;\n column: string;\n backfill: BackfillID;\n }\n | {op: 'rename-table'; old: Identifier; new: Identifier}\n | {op: 'drop-table'; table: Identifier}\n | {op: 'rename-column'; table: Identifier; old: string; new: string}\n | {op: 'drop-column'; table: Identifier; column: string}\n | {op: 'complete-backfill'; table: Identifier; columns: readonly string[]};\n\nexport type CookieOpTag = CookieOp['op'];\n\n/**\n * The fold, and the only place either store decides what a schema change means\n * for the cookie set. Both interpreters are mechanical translations of what this\n * returns, in the order it returns them.\n *\n * Changes that carry no cookie state — index changes, and the `create-table` /\n * `add-column` variants whose optional `metadata` and `backfill` fields are\n * absent — produce no ops.\n */\nexport function cookieOps(change: SchemaChange): CookieOp[] {\n switch (change.tag) {\n case 'update-table-metadata':\n return [\n {op: 'upsert-metadata', table: change.table, metadata: change.new},\n ];\n\n case 'create-table': {\n const {spec, metadata, backfill} = change;\n const table = {schema: spec.schema, name: spec.name};\n const ops: CookieOp[] = [];\n if (metadata) {\n ops.push({op: 'upsert-metadata', table, metadata});\n }\n if (backfill) {\n for (const [column, id] of Object.entries(backfill)) {\n ops.push({op: 'upsert-backfill', table, column, backfill: id});\n }\n }\n return ops;\n }\n\n case 'rename-table':\n return [{op: 'rename-table', old: change.old, new: change.new}];\n\n case 'drop-table':\n return [{op: 'drop-table', table: change.id}];\n\n case 'add-column': {\n const {table, tableMetadata, column, backfill} = change;\n const ops: CookieOp[] = [];\n if (tableMetadata) {\n ops.push({op: 'upsert-metadata', table, metadata: tableMetadata});\n }\n if (backfill) {\n ops.push({\n op: 'upsert-backfill',\n table,\n column: column.name,\n backfill,\n });\n }\n return ops;\n }\n\n case 'update-column': {\n // Only a rename moves the cookie; the rest of a column update does not\n // affect whether or how it is being backfilled.\n const {table, old, new: updated} = change;\n return old.name === updated.name\n ? []\n : [{op: 'rename-column', table, old: old.name, new: updated.name}];\n }\n\n case 'drop-column':\n return [{op: 'drop-column', table: change.table, column: change.column}];\n\n case 'backfill-completed': {\n const {relation, columns} = change;\n // The rowKey columns are excluded from `columns` but are backfilled with\n // them, so both are cleared.\n const cleared = [...relation.rowKey.columns, ...columns];\n return cleared.length === 0\n ? []\n : [\n {\n op: 'complete-backfill',\n table: {schema: relation.schema, name: relation.name},\n columns: cleared,\n },\n ];\n }\n\n case 'create-index':\n case 'drop-index':\n return [];\n\n default:\n // Unreachable: `isSchemaChange()` gates on the same tag set this switch\n // covers. It is exhaustive so that a new schema-change tag is a compile\n // error here rather than a silently dropped backfill.\n unreachable(change);\n }\n}\n\n/**\n * The SQLite interpreter of {@link cookieOps}, over prepared statements on the\n * change-log database.\n *\n * The statements run inside the log's transaction, in stream order, at the point\n * the schema change is appended (see `ChangeLogStreamWriter`). That is what\n * makes the cookie set atomic with the head — a crash cannot leave cookies ahead\n * of or behind the rows they were folded from — and what makes an aborted\n * transaction discard its cookies with its rows, for free.\n *\n * Statements are prepared against the tables as they exist now, so an instance\n * does not survive a reconcile that drops and recreates them.\n */\nexport class ChangeLogCookieWriter {\n readonly #upsertMetadata: Statement;\n readonly #upsertBackfill: Statement;\n readonly #renameMetadataTable: Statement;\n readonly #renameBackfillTable: Statement;\n readonly #deleteMetadataTable: Statement;\n readonly #deleteBackfillTable: Statement;\n readonly #renameBackfillColumn: Statement;\n readonly #deleteBackfillColumn: Statement;\n\n constructor(db: Database) {\n this.#upsertMetadata = db.prepare(/*sql*/ `\n INSERT INTO \"${CHANGE_LOG_TABLE_METADATA_TABLE}\"\n (\"schema\", \"table\", \"metadata\") VALUES (?, ?, ?)\n ON CONFLICT (\"schema\", \"table\")\n DO UPDATE SET \"metadata\" = excluded.\"metadata\"\n `);\n this.#upsertBackfill = db.prepare(/*sql*/ `\n INSERT INTO \"${CHANGE_LOG_BACKFILLING_TABLE}\"\n (\"schema\", \"table\", \"column\", \"backfill\") VALUES (?, ?, ?, ?)\n ON CONFLICT (\"schema\", \"table\", \"column\")\n DO UPDATE SET \"backfill\" = excluded.\"backfill\"\n `);\n this.#renameMetadataTable = db.prepare(/*sql*/ `\n UPDATE \"${CHANGE_LOG_TABLE_METADATA_TABLE}\"\n SET \"schema\" = ?, \"table\" = ? WHERE \"schema\" = ? AND \"table\" = ?\n `);\n this.#renameBackfillTable = db.prepare(/*sql*/ `\n UPDATE \"${CHANGE_LOG_BACKFILLING_TABLE}\"\n SET \"schema\" = ?, \"table\" = ? WHERE \"schema\" = ? AND \"table\" = ?\n `);\n this.#deleteMetadataTable = db.prepare(/*sql*/ `\n DELETE FROM \"${CHANGE_LOG_TABLE_METADATA_TABLE}\"\n WHERE \"schema\" = ? AND \"table\" = ?\n `);\n this.#deleteBackfillTable = db.prepare(/*sql*/ `\n DELETE FROM \"${CHANGE_LOG_BACKFILLING_TABLE}\"\n WHERE \"schema\" = ? AND \"table\" = ?\n `);\n this.#renameBackfillColumn = db.prepare(/*sql*/ `\n UPDATE \"${CHANGE_LOG_BACKFILLING_TABLE}\" SET \"column\" = ?\n WHERE \"schema\" = ? AND \"table\" = ? AND \"column\" = ?\n `);\n this.#deleteBackfillColumn = db.prepare(/*sql*/ `\n DELETE FROM \"${CHANGE_LOG_BACKFILLING_TABLE}\"\n WHERE \"schema\" = ? AND \"table\" = ? AND \"column\" = ?\n `);\n }\n\n /** Applies the change's cookie ops, returning the ops that were applied. */\n apply(change: SchemaChange): CookieOp[] {\n const ops = cookieOps(change);\n for (const op of ops) {\n this.#run(op);\n }\n return ops;\n }\n\n #run(op: CookieOp): void {\n switch (op.op) {\n case 'upsert-metadata':\n this.#upsertMetadata.run(\n op.table.schema,\n op.table.name,\n BigIntJSON.stringify(op.metadata),\n );\n break;\n\n case 'upsert-backfill':\n this.#upsertBackfill.run(\n op.table.schema,\n op.table.name,\n op.column,\n BigIntJSON.stringify(op.backfill),\n );\n break;\n\n case 'rename-table':\n this.#renameMetadataTable.run(\n op.new.schema,\n op.new.name,\n op.old.schema,\n op.old.name,\n );\n this.#renameBackfillTable.run(\n op.new.schema,\n op.new.name,\n op.old.schema,\n op.old.name,\n );\n break;\n\n case 'drop-table':\n this.#deleteMetadataTable.run(op.table.schema, op.table.name);\n this.#deleteBackfillTable.run(op.table.schema, op.table.name);\n break;\n\n case 'rename-column':\n this.#renameBackfillColumn.run(\n op.new,\n op.table.schema,\n op.table.name,\n op.old,\n );\n break;\n\n case 'drop-column':\n this.#deleteBackfillColumn.run(\n op.table.schema,\n op.table.name,\n op.column,\n );\n break;\n\n case 'complete-backfill':\n // A per-column delete rather than an `IN` list, which cannot be a\n // single prepared statement across arities. The columns of one\n // completed backfill are few, and this runs only on schema changes.\n for (const column of op.columns) {\n this.#deleteBackfillColumn.run(\n op.table.schema,\n op.table.name,\n column,\n );\n }\n break;\n\n default:\n unreachable(op);\n }\n }\n}\n\nconst tableKey = (schema: string, table: string) =>\n JSON.stringify([schema, table]);\n\nconst columnKey = (schema: string, table: string, column: string) =>\n JSON.stringify([schema, table, column]);\n\nconst cmp = (a: string, b: string) => (a < b ? -1 : a > b ? 1 : 0);\n\n/**\n * The in-memory interpreter of {@link cookieOps}: the third one, and the only\n * one that is not a store.\n *\n * A replica-derived cookie set is a snapshot at the replica's state version,\n * which trails the change log's head, so it cannot be compared with a\n * Postgres-derived set directly. This carries it forward over the schema changes\n * the log holds in between, which turns \"the two stores disagree, but they are\n * also at different positions\" into an exact equality — and makes a\n * disagreement between any two of the three interpreters visible as an\n * inequality that neither store would have noticed on its own.\n *\n * `changes` must be the log's schema changes over `(from, to]` in stream order.\n * Anything else produces a set that belongs to no watermark.\n */\nexport function foldCookies(\n cookies: CookieSet,\n changes: Iterable<SchemaChange>,\n): CookieSet {\n const metadata = new Map<string, TableMetadataCookie>(\n cookies.tableMetadata.map(c => [tableKey(c.schema, c.table), c]),\n );\n const backfilling = new Map<string, BackfillCookie>(\n cookies.backfilling.map(c => [columnKey(c.schema, c.table, c.column), c]),\n );\n const dropColumn = ({schema, name}: Identifier, column: string) =>\n backfilling.delete(columnKey(schema, name, column));\n\n for (const change of changes) {\n for (const op of cookieOps(change)) {\n switch (op.op) {\n case 'upsert-metadata':\n metadata.set(tableKey(op.table.schema, op.table.name), {\n schema: op.table.schema,\n table: op.table.name,\n metadata: op.metadata,\n });\n break;\n\n case 'upsert-backfill':\n backfilling.set(\n columnKey(op.table.schema, op.table.name, op.column),\n {\n schema: op.table.schema,\n table: op.table.name,\n column: op.column,\n backfill: op.backfill,\n },\n );\n break;\n\n case 'rename-table': {\n const {old, new: renamed} = op;\n const moved = metadata.get(tableKey(old.schema, old.name));\n if (moved) {\n metadata.delete(tableKey(old.schema, old.name));\n metadata.set(tableKey(renamed.schema, renamed.name), {\n ...moved,\n schema: renamed.schema,\n table: renamed.name,\n });\n }\n // Snapshotted before mutating, since the rename both deletes and\n // inserts into the map being scanned.\n for (const [key, cookie] of [...backfilling]) {\n if (cookie.schema === old.schema && cookie.table === old.name) {\n backfilling.delete(key);\n backfilling.set(\n columnKey(renamed.schema, renamed.name, cookie.column),\n {...cookie, schema: renamed.schema, table: renamed.name},\n );\n }\n }\n break;\n }\n\n case 'drop-table': {\n const {schema, name} = op.table;\n metadata.delete(tableKey(schema, name));\n for (const [key, cookie] of [...backfilling]) {\n if (cookie.schema === schema && cookie.table === name) {\n backfilling.delete(key);\n }\n }\n break;\n }\n\n case 'rename-column': {\n const {schema, name} = op.table;\n const moved = backfilling.get(columnKey(schema, name, op.old));\n if (moved) {\n backfilling.delete(columnKey(schema, name, op.old));\n backfilling.set(columnKey(schema, name, op.new), {\n ...moved,\n column: op.new,\n });\n }\n break;\n }\n\n case 'drop-column':\n dropColumn(op.table, op.column);\n break;\n\n case 'complete-backfill':\n for (const column of op.columns) {\n dropColumn(op.table, column);\n }\n break;\n\n default:\n unreachable(op);\n }\n }\n }\n\n return {\n tableMetadata: [...metadata.values()].toSorted(\n (a, b) => cmp(a.schema, b.schema) || cmp(a.table, b.table),\n ),\n backfilling: [...backfilling.values()].toSorted(\n (a, b) =>\n cmp(a.schema, b.schema) ||\n cmp(a.table, b.table) ||\n cmp(a.column, b.column),\n ),\n };\n}\n\nconst backfillRequestsSchema = v.array(backfillRequestSchema);\n\n/**\n * Returns one {@link BackfillRequest} for each table with an active backfill.\n * Each request contains the stored table metadata, if available.\n */\nexport function backfillRequestsFrom(cookies: CookieSet): BackfillRequest[] {\n const metadata = new Map(\n cookies.tableMetadata.map(c => [tableKey(c.schema, c.table), c.metadata]),\n );\n const requests: BackfillRequest[] = [];\n let curr: BackfillRequest | undefined;\n for (const {schema, table, column, backfill} of cookies.backfilling) {\n if (curr?.table.schema !== schema || curr.table.name !== table) {\n curr = {\n table: {\n schema,\n name: table,\n // Use `null` when a backfilling table has no metadata. This matches\n // the result of the Postgres left join.\n metadata: metadata.get(tableKey(schema, table)) ?? null,\n },\n columns: {},\n };\n requests.push(curr);\n }\n curr.columns[column] = backfill;\n }\n return v.parse(requests, backfillRequestsSchema);\n}\n\n/**\n * Reads the whole cookie set, ordered by primary key. This is what a\n * change-streamer would hand `startStream`, and what the Postgres-derived and\n * replica-derived sets are compared against.\n */\nexport function readCookies(db: Database): CookieSet {\n const tableMetadata = db\n .prepare(/*sql*/ `\n SELECT \"schema\", \"table\", \"metadata\"\n FROM \"${CHANGE_LOG_TABLE_METADATA_TABLE}\"\n ORDER BY \"schema\", \"table\"\n `)\n .all<{schema: string; table: string; metadata: string}>()\n .map(({schema, table, metadata}) => ({\n schema,\n table,\n metadata: BigIntJSON.parse(metadata) as TableMetadata,\n }));\n\n const backfilling = db\n .prepare(/*sql*/ `\n SELECT \"schema\", \"table\", \"column\", \"backfill\"\n FROM \"${CHANGE_LOG_BACKFILLING_TABLE}\"\n ORDER BY \"schema\", \"table\", \"column\"\n `)\n .all<{schema: string; table: string; column: string; backfill: string}>()\n .map(({schema, table, column, backfill}) => ({\n schema,\n table,\n column,\n backfill: BigIntJSON.parse(backfill) as BackfillID,\n }));\n\n return {tableMetadata, backfilling};\n}\n\n/**\n * Replaces the cookie set wholesale.\n *\n * The cookies are unversioned point-in-time state, so a reconciliation that\n * moves the head invalidates them: truncating the stream above `W` does not roll\n * the cookie set back to `W`, and the truncated transactions can contain\n * `create-table`, `add-column`, and `backfill-completed`. The caller must run\n * this in the same transaction as the reconciliation that invalidated them.\n */\nexport function replaceCookies(db: Database, cookies: CookieSet): void {\n db.prepare(/*sql*/ `DELETE FROM \"${CHANGE_LOG_TABLE_METADATA_TABLE}\"`).run();\n db.prepare(/*sql*/ `DELETE FROM \"${CHANGE_LOG_BACKFILLING_TABLE}\"`).run();\n\n if (cookies.tableMetadata.length > 0) {\n const insert = db.prepare(/*sql*/ `\n INSERT INTO \"${CHANGE_LOG_TABLE_METADATA_TABLE}\"\n (\"schema\", \"table\", \"metadata\") VALUES (?, ?, ?)\n `);\n for (const {schema, table, metadata} of cookies.tableMetadata) {\n insert.run(schema, table, BigIntJSON.stringify(metadata));\n }\n }\n if (cookies.backfilling.length > 0) {\n const insert = db.prepare(/*sql*/ `\n INSERT INTO \"${CHANGE_LOG_BACKFILLING_TABLE}\"\n (\"schema\", \"table\", \"column\", \"backfill\") VALUES (?, ?, ?, ?)\n `);\n for (const {schema, table, column, backfill} of cookies.backfilling) {\n insert.run(schema, table, column, BigIntJSON.stringify(backfill));\n }\n }\n}\n\n/** Retained cookie rows, by table. Both are normally zero. */\nexport type CookieRowCounts = {\n readonly tableMetadata: number;\n readonly backfilling: number;\n};\n\nexport function readCookieRowCounts(db: Database): CookieRowCounts {\n return db\n .prepare(/*sql*/ `\n SELECT\n (SELECT count(*) FROM \"${CHANGE_LOG_TABLE_METADATA_TABLE}\")\n AS \"tableMetadata\",\n (SELECT count(*) FROM \"${CHANGE_LOG_BACKFILLING_TABLE}\")\n AS \"backfilling\"\n `)\n .get<CookieRowCounts>();\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAiDA,IAAa,kCAAkC;AAC/C,IAAa,+BAA+B;AAQ5C,IAAa,kCAA0C;kBACrC,gCAAgC;;;;;;;kBAOhC,6BAA6B;;;;;;;;AAS/C,IAAa,gCAAwC;0BAC3B,gCAAgC;0BAChC,6BAA6B;;;;;;;;;;;AA4DvD,SAAgB,UAAU,QAAkC;CAC1D,QAAQ,OAAO,KAAf;EACE,KAAK,yBACH,OAAO,CACL;GAAC,IAAI;GAAmB,OAAO,OAAO;GAAO,UAAU,OAAO;EAAG,CACnE;EAEF,KAAK,gBAAgB;GACnB,MAAM,EAAC,MAAM,UAAU,aAAY;GACnC,MAAM,QAAQ;IAAC,QAAQ,KAAK;IAAQ,MAAM,KAAK;GAAI;GACnD,MAAM,MAAkB,CAAC;GACzB,IAAI,UACF,IAAI,KAAK;IAAC,IAAI;IAAmB;IAAO;GAAQ,CAAC;GAEnD,IAAI,UACF,KAAK,MAAM,CAAC,QAAQ,OAAO,OAAO,QAAQ,QAAQ,GAChD,IAAI,KAAK;IAAC,IAAI;IAAmB;IAAO;IAAQ,UAAU;GAAE,CAAC;GAGjE,OAAO;EACT;EAEA,KAAK,gBACH,OAAO,CAAC;GAAC,IAAI;GAAgB,KAAK,OAAO;GAAK,KAAK,OAAO;EAAG,CAAC;EAEhE,KAAK,cACH,OAAO,CAAC;GAAC,IAAI;GAAc,OAAO,OAAO;EAAE,CAAC;EAE9C,KAAK,cAAc;GACjB,MAAM,EAAC,OAAO,eAAe,QAAQ,aAAY;GACjD,MAAM,MAAkB,CAAC;GACzB,IAAI,eACF,IAAI,KAAK;IAAC,IAAI;IAAmB;IAAO,UAAU;GAAa,CAAC;GAElE,IAAI,UACF,IAAI,KAAK;IACP,IAAI;IACJ;IACA,QAAQ,OAAO;IACf;GACF,CAAC;GAEH,OAAO;EACT;EAEA,KAAK,iBAAiB;GAGpB,MAAM,EAAC,OAAO,KAAK,KAAK,YAAW;GACnC,OAAO,IAAI,SAAS,QAAQ,OACxB,CAAC,IACD,CAAC;IAAC,IAAI;IAAiB;IAAO,KAAK,IAAI;IAAM,KAAK,QAAQ;GAAI,CAAC;EACrE;EAEA,KAAK,eACH,OAAO,CAAC;GAAC,IAAI;GAAe,OAAO,OAAO;GAAO,QAAQ,OAAO;EAAM,CAAC;EAEzE,KAAK,sBAAsB;GACzB,MAAM,EAAC,UAAU,YAAW;GAG5B,MAAM,UAAU,CAAC,GAAG,SAAS,OAAO,SAAS,GAAG,OAAO;GACvD,OAAO,QAAQ,WAAW,IACtB,CAAC,IACD,CACE;IACE,IAAI;IACJ,OAAO;KAAC,QAAQ,SAAS;KAAQ,MAAM,SAAS;IAAI;IACpD,SAAS;GACX,CACF;EACN;EAEA,KAAK;EACL,KAAK,cACH,OAAO,CAAC;EAEV,SAIE,YAAY,MAAM;CACtB;AACF;;;;;;;;;;;;;;AAeA,IAAa,wBAAb,MAAmC;CACjC;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CAEA,YAAY,IAAc;EACxB,KAAKA,kBAAkB,GAAG,QAAgB;qBACzB,gCAAgC;;;;KAIhD;EACD,KAAKC,kBAAkB,GAAG,QAAgB;qBACzB,6BAA6B;;;;KAI7C;EACD,KAAKC,uBAAuB,GAAG,QAAgB;gBACnC,gCAAgC;;KAE3C;EACD,KAAKC,uBAAuB,GAAG,QAAgB;gBACnC,6BAA6B;;KAExC;EACD,KAAKC,uBAAuB,GAAG,QAAgB;qBAC9B,gCAAgC;;KAEhD;EACD,KAAKC,uBAAuB,GAAG,QAAgB;qBAC9B,6BAA6B;;KAE7C;EACD,KAAKC,wBAAwB,GAAG,QAAgB;gBACpC,6BAA6B;;KAExC;EACD,KAAKC,wBAAwB,GAAG,QAAgB;qBAC/B,6BAA6B;;KAE7C;CACH;;CAGA,MAAM,QAAkC;EACtC,MAAM,MAAM,UAAU,MAAM;EAC5B,KAAK,MAAM,MAAM,KACf,KAAKC,KAAK,EAAE;EAEd,OAAO;CACT;CAEA,KAAK,IAAoB;EACvB,QAAQ,GAAG,IAAX;GACE,KAAK;IACH,KAAKR,gBAAgB,IACnB,GAAG,MAAM,QACT,GAAG,MAAM,MACT,WAAW,UAAU,GAAG,QAAQ,CAClC;IACA;GAEF,KAAK;IACH,KAAKC,gBAAgB,IACnB,GAAG,MAAM,QACT,GAAG,MAAM,MACT,GAAG,QACH,WAAW,UAAU,GAAG,QAAQ,CAClC;IACA;GAEF,KAAK;IACH,KAAKC,qBAAqB,IACxB,GAAG,IAAI,QACP,GAAG,IAAI,MACP,GAAG,IAAI,QACP,GAAG,IAAI,IACT;IACA,KAAKC,qBAAqB,IACxB,GAAG,IAAI,QACP,GAAG,IAAI,MACP,GAAG,IAAI,QACP,GAAG,IAAI,IACT;IACA;GAEF,KAAK;IACH,KAAKC,qBAAqB,IAAI,GAAG,MAAM,QAAQ,GAAG,MAAM,IAAI;IAC5D,KAAKC,qBAAqB,IAAI,GAAG,MAAM,QAAQ,GAAG,MAAM,IAAI;IAC5D;GAEF,KAAK;IACH,KAAKC,sBAAsB,IACzB,GAAG,KACH,GAAG,MAAM,QACT,GAAG,MAAM,MACT,GAAG,GACL;IACA;GAEF,KAAK;IACH,KAAKC,sBAAsB,IACzB,GAAG,MAAM,QACT,GAAG,MAAM,MACT,GAAG,MACL;IACA;GAEF,KAAK;IAIH,KAAK,MAAM,UAAU,GAAG,SACtB,KAAKA,sBAAsB,IACzB,GAAG,MAAM,QACT,GAAG,MAAM,MACT,MACF;IAEF;GAEF,SACE,YAAY,EAAE;EAClB;CACF;AACF;AAEA,IAAM,YAAY,QAAgB,UAChC,KAAK,UAAU,CAAC,QAAQ,KAAK,CAAC;AAEhC,IAAM,aAAa,QAAgB,OAAe,WAChD,KAAK,UAAU;CAAC;CAAQ;CAAO;AAAM,CAAC;AAExC,IAAM,OAAO,GAAW,MAAe,IAAI,IAAI,KAAK,IAAI,IAAI,IAAI;;;;;;;;;;;;;;;;AAiBhE,SAAgB,YACd,SACA,SACW;CACX,MAAM,WAAW,IAAI,IACnB,QAAQ,cAAc,KAAI,MAAK,CAAC,SAAS,EAAE,QAAQ,EAAE,KAAK,GAAG,CAAC,CAAC,CACjE;CACA,MAAM,cAAc,IAAI,IACtB,QAAQ,YAAY,KAAI,MAAK,CAAC,UAAU,EAAE,QAAQ,EAAE,OAAO,EAAE,MAAM,GAAG,CAAC,CAAC,CAC1E;CACA,MAAM,cAAc,EAAC,QAAQ,QAAmB,WAC9C,YAAY,OAAO,UAAU,QAAQ,MAAM,MAAM,CAAC;CAEpD,KAAK,MAAM,UAAU,SACnB,KAAK,MAAM,MAAM,UAAU,MAAM,GAC/B,QAAQ,GAAG,IAAX;EACE,KAAK;GACH,SAAS,IAAI,SAAS,GAAG,MAAM,QAAQ,GAAG,MAAM,IAAI,GAAG;IACrD,QAAQ,GAAG,MAAM;IACjB,OAAO,GAAG,MAAM;IAChB,UAAU,GAAG;GACf,CAAC;GACD;EAEF,KAAK;GACH,YAAY,IACV,UAAU,GAAG,MAAM,QAAQ,GAAG,MAAM,MAAM,GAAG,MAAM,GACnD;IACE,QAAQ,GAAG,MAAM;IACjB,OAAO,GAAG,MAAM;IAChB,QAAQ,GAAG;IACX,UAAU,GAAG;GACf,CACF;GACA;EAEF,KAAK,gBAAgB;GACnB,MAAM,EAAC,KAAK,KAAK,YAAW;GAC5B,MAAM,QAAQ,SAAS,IAAI,SAAS,IAAI,QAAQ,IAAI,IAAI,CAAC;GACzD,IAAI,OAAO;IACT,SAAS,OAAO,SAAS,IAAI,QAAQ,IAAI,IAAI,CAAC;IAC9C,SAAS,IAAI,SAAS,QAAQ,QAAQ,QAAQ,IAAI,GAAG;KACnD,GAAG;KACH,QAAQ,QAAQ;KAChB,OAAO,QAAQ;IACjB,CAAC;GACH;GAGA,KAAK,MAAM,CAAC,KAAK,WAAW,CAAC,GAAG,WAAW,GACzC,IAAI,OAAO,WAAW,IAAI,UAAU,OAAO,UAAU,IAAI,MAAM;IAC7D,YAAY,OAAO,GAAG;IACtB,YAAY,IACV,UAAU,QAAQ,QAAQ,QAAQ,MAAM,OAAO,MAAM,GACrD;KAAC,GAAG;KAAQ,QAAQ,QAAQ;KAAQ,OAAO,QAAQ;IAAI,CACzD;GACF;GAEF;EACF;EAEA,KAAK,cAAc;GACjB,MAAM,EAAC,QAAQ,SAAQ,GAAG;GAC1B,SAAS,OAAO,SAAS,QAAQ,IAAI,CAAC;GACtC,KAAK,MAAM,CAAC,KAAK,WAAW,CAAC,GAAG,WAAW,GACzC,IAAI,OAAO,WAAW,UAAU,OAAO,UAAU,MAC/C,YAAY,OAAO,GAAG;GAG1B;EACF;EAEA,KAAK,iBAAiB;GACpB,MAAM,EAAC,QAAQ,SAAQ,GAAG;GAC1B,MAAM,QAAQ,YAAY,IAAI,UAAU,QAAQ,MAAM,GAAG,GAAG,CAAC;GAC7D,IAAI,OAAO;IACT,YAAY,OAAO,UAAU,QAAQ,MAAM,GAAG,GAAG,CAAC;IAClD,YAAY,IAAI,UAAU,QAAQ,MAAM,GAAG,GAAG,GAAG;KAC/C,GAAG;KACH,QAAQ,GAAG;IACb,CAAC;GACH;GACA;EACF;EAEA,KAAK;GACH,WAAW,GAAG,OAAO,GAAG,MAAM;GAC9B;EAEF,KAAK;GACH,KAAK,MAAM,UAAU,GAAG,SACtB,WAAW,GAAG,OAAO,MAAM;GAE7B;EAEF,SACE,YAAY,EAAE;CAClB;CAIJ,OAAO;EACL,eAAe,CAAC,GAAG,SAAS,OAAO,CAAC,EAAE,UACnC,GAAG,MAAM,IAAI,EAAE,QAAQ,EAAE,MAAM,KAAK,IAAI,EAAE,OAAO,EAAE,KAAK,CAC3D;EACA,aAAa,CAAC,GAAG,YAAY,OAAO,CAAC,EAAE,UACpC,GAAG,MACF,IAAI,EAAE,QAAQ,EAAE,MAAM,KACtB,IAAI,EAAE,OAAO,EAAE,KAAK,KACpB,IAAI,EAAE,QAAQ,EAAE,MAAM,CAC1B;CACF;AACF;AAEA,IAAM,yBAAyB,eAAE,MAAM,qBAAqB;;;;;AAM5D,SAAgB,qBAAqB,SAAuC;CAC1E,MAAM,WAAW,IAAI,IACnB,QAAQ,cAAc,KAAI,MAAK,CAAC,SAAS,EAAE,QAAQ,EAAE,KAAK,GAAG,EAAE,QAAQ,CAAC,CAC1E;CACA,MAAM,WAA8B,CAAC;CACrC,IAAI;CACJ,KAAK,MAAM,EAAC,QAAQ,OAAO,QAAQ,cAAa,QAAQ,aAAa;EACnE,IAAI,MAAM,MAAM,WAAW,UAAU,KAAK,MAAM,SAAS,OAAO;GAC9D,OAAO;IACL,OAAO;KACL;KACA,MAAM;KAGN,UAAU,SAAS,IAAI,SAAS,QAAQ,KAAK,CAAC,KAAK;IACrD;IACA,SAAS,CAAC;GACZ;GACA,SAAS,KAAK,IAAI;EACpB;EACA,KAAK,QAAQ,UAAU;CACzB;CACA,OAAO,MAAQ,UAAU,sBAAsB;AACjD;;;;;;AAOA,SAAgB,YAAY,IAAyB;CA4BnD,OAAO;EAAC,eA3Bc,GACnB,QAAgB;;gBAEL,gCAAgC;;KAE3C,EACA,IAAuD,EACvD,KAAK,EAAC,QAAQ,OAAO,gBAAe;GACnC;GACA;GACA,UAAU,WAAW,MAAM,QAAQ;EACrC,EAgBM;EAAe,aAdH,GACjB,QAAgB;;gBAEL,6BAA6B;;KAExC,EACA,IAAuE,EACvE,KAAK,EAAC,QAAQ,OAAO,QAAQ,gBAAe;GAC3C;GACA;GACA;GACA,UAAU,WAAW,MAAM,QAAQ;EACrC,EAEqB;CAAW;AACpC;;;;;;;;;;AAWA,SAAgB,eAAe,IAAc,SAA0B;CACrE,GAAG,QAAgB,gBAAgB,gCAAgC,EAAE,EAAE,IAAI;CAC3E,GAAG,QAAgB,gBAAgB,6BAA6B,EAAE,EAAE,IAAI;CAExE,IAAI,QAAQ,cAAc,SAAS,GAAG;EACpC,MAAM,SAAS,GAAG,QAAgB;qBACjB,gCAAgC;;KAEhD;EACD,KAAK,MAAM,EAAC,QAAQ,OAAO,cAAa,QAAQ,eAC9C,OAAO,IAAI,QAAQ,OAAO,WAAW,UAAU,QAAQ,CAAC;CAE5D;CACA,IAAI,QAAQ,YAAY,SAAS,GAAG;EAClC,MAAM,SAAS,GAAG,QAAgB;qBACjB,6BAA6B;;KAE7C;EACD,KAAK,MAAM,EAAC,QAAQ,OAAO,QAAQ,cAAa,QAAQ,aACtD,OAAO,IAAI,QAAQ,OAAO,QAAQ,WAAW,UAAU,QAAQ,CAAC;CAEpE;AACF;AAQA,SAAgB,oBAAoB,IAA+B;CACjE,OAAO,GACJ,QAAgB;;iCAEY,gCAAgC;;iCAEhC,6BAA6B;;KAEzD,EACA,IAAqB;AAC1B"}
|