@rocicorp/zero 1.11.0-canary.0 → 1.11.0-canary.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/out/analyze-query/src/bin-analyze.js +1 -1
- package/out/analyze-query/src/bin-analyze.js.map +1 -1
- package/out/replicache/src/btree/read.d.ts +12 -1
- package/out/replicache/src/btree/read.d.ts.map +1 -1
- package/out/replicache/src/btree/read.js +24 -12
- package/out/replicache/src/btree/read.js.map +1 -1
- package/out/replicache/src/dag/gc.d.ts.map +1 -1
- package/out/replicache/src/dag/gc.js +2 -1
- package/out/replicache/src/dag/gc.js.map +1 -1
- package/out/replicache/src/dag/invalid-ref-count-error.d.ts +12 -0
- package/out/replicache/src/dag/invalid-ref-count-error.d.ts.map +1 -0
- package/out/replicache/src/dag/invalid-ref-count-error.js +19 -0
- package/out/replicache/src/dag/invalid-ref-count-error.js.map +1 -0
- package/out/replicache/src/dag/store-impl.d.ts +12 -2
- package/out/replicache/src/dag/store-impl.d.ts.map +1 -1
- package/out/replicache/src/dag/store-impl.js +22 -8
- package/out/replicache/src/dag/store-impl.js.map +1 -1
- package/out/replicache/src/db/rebase.d.ts +14 -2
- package/out/replicache/src/db/rebase.d.ts.map +1 -1
- package/out/replicache/src/db/rebase.js +27 -5
- package/out/replicache/src/db/rebase.js.map +1 -1
- package/out/replicache/src/db/write.js.map +1 -1
- package/out/replicache/src/kv/expo-sqlite/store.d.ts.map +1 -1
- package/out/replicache/src/kv/expo-sqlite/store.js +28 -5
- package/out/replicache/src/kv/expo-sqlite/store.js.map +1 -1
- package/out/replicache/src/kv/idb-store-with-mem-fallback.d.ts +15 -5
- package/out/replicache/src/kv/idb-store-with-mem-fallback.d.ts.map +1 -1
- package/out/replicache/src/kv/idb-store-with-mem-fallback.js +25 -25
- package/out/replicache/src/kv/idb-store-with-mem-fallback.js.map +1 -1
- package/out/replicache/src/kv/idb-store.d.ts +9 -0
- package/out/replicache/src/kv/idb-store.d.ts.map +1 -1
- package/out/replicache/src/kv/idb-store.js +14 -2
- package/out/replicache/src/kv/idb-store.js.map +1 -1
- package/out/replicache/src/kv/mem-store.d.ts +1 -0
- package/out/replicache/src/kv/mem-store.d.ts.map +1 -1
- package/out/replicache/src/kv/mem-store.js +3 -0
- package/out/replicache/src/kv/mem-store.js.map +1 -1
- package/out/replicache/src/kv/op-sqlite/store.d.ts.map +1 -1
- package/out/replicache/src/kv/op-sqlite/store.js +1 -1
- package/out/replicache/src/kv/op-sqlite/store.js.map +1 -1
- package/out/replicache/src/kv/sqlite-store.d.ts +13 -1
- package/out/replicache/src/kv/sqlite-store.d.ts.map +1 -1
- package/out/replicache/src/kv/sqlite-store.js +49 -4
- package/out/replicache/src/kv/sqlite-store.js.map +1 -1
- package/out/replicache/src/kv/store.d.ts +6 -0
- package/out/replicache/src/kv/store.d.ts.map +1 -1
- package/out/replicache/src/persist/collect-idb-databases.d.ts +10 -5
- package/out/replicache/src/persist/collect-idb-databases.d.ts.map +1 -1
- package/out/replicache/src/persist/collect-idb-databases.js +16 -9
- package/out/replicache/src/persist/collect-idb-databases.js.map +1 -1
- package/out/replicache/src/persist/persist.d.ts.map +1 -1
- package/out/replicache/src/persist/persist.js +12 -5
- package/out/replicache/src/persist/persist.js.map +1 -1
- package/out/replicache/src/persist/refresh.js +6 -2
- package/out/replicache/src/persist/refresh.js.map +1 -1
- package/out/replicache/src/replicache-impl.d.ts +14 -3
- package/out/replicache/src/replicache-impl.d.ts.map +1 -1
- package/out/replicache/src/replicache-impl.js +118 -11
- package/out/replicache/src/replicache-impl.js.map +1 -1
- package/out/replicache/src/replicache-options.d.ts +9 -0
- package/out/replicache/src/replicache-options.d.ts.map +1 -1
- package/out/replicache/src/transactions.d.ts.map +1 -1
- package/out/replicache/src/transactions.js +1 -1
- package/out/replicache/src/transactions.js.map +1 -1
- package/out/shared/src/error.d.ts +11 -0
- package/out/shared/src/error.d.ts.map +1 -1
- package/out/shared/src/error.js +35 -1
- package/out/shared/src/error.js.map +1 -1
- package/out/shared/src/iterables.d.ts +2 -0
- package/out/shared/src/iterables.d.ts.map +1 -1
- package/out/shared/src/iterables.js +18 -1
- package/out/shared/src/iterables.js.map +1 -1
- package/out/shared/src/map.d.ts +7 -2
- package/out/shared/src/map.d.ts.map +1 -1
- package/out/shared/src/map.js.map +1 -1
- package/out/shared/src/ring-buffer.d.ts +8 -0
- package/out/shared/src/ring-buffer.d.ts.map +1 -1
- package/out/shared/src/ring-buffer.js +22 -1
- package/out/shared/src/ring-buffer.js.map +1 -1
- package/out/zero/package.js +3 -4
- package/out/zero/package.js.map +1 -1
- package/out/zero-cache/src/config/normalize.d.ts.map +1 -1
- package/out/zero-cache/src/config/normalize.js +25 -1
- package/out/zero-cache/src/config/normalize.js.map +1 -1
- package/out/zero-cache/src/config/zero-config.d.ts +48 -2
- package/out/zero-cache/src/config/zero-config.d.ts.map +1 -1
- package/out/zero-cache/src/config/zero-config.js +171 -29
- package/out/zero-cache/src/config/zero-config.js.map +1 -1
- package/out/zero-cache/src/server/change-streamer.d.ts.map +1 -1
- package/out/zero-cache/src/server/change-streamer.js +25 -9
- package/out/zero-cache/src/server/change-streamer.js.map +1 -1
- package/out/zero-cache/src/server/inspector-delegate.d.ts +18 -2
- package/out/zero-cache/src/server/inspector-delegate.d.ts.map +1 -1
- package/out/zero-cache/src/server/inspector-delegate.js +20 -7
- package/out/zero-cache/src/server/inspector-delegate.js.map +1 -1
- package/out/zero-cache/src/server/main.d.ts.map +1 -1
- package/out/zero-cache/src/server/main.js +8 -1
- package/out/zero-cache/src/server/main.js.map +1 -1
- package/out/zero-cache/src/server/mutator.js +2 -0
- package/out/zero-cache/src/server/mutator.js.map +1 -1
- package/out/zero-cache/src/server/reaper.d.ts.map +1 -1
- package/out/zero-cache/src/server/reaper.js +2 -0
- package/out/zero-cache/src/server/reaper.js.map +1 -1
- package/out/zero-cache/src/server/replicator.d.ts.map +1 -1
- package/out/zero-cache/src/server/replicator.js +8 -2
- package/out/zero-cache/src/server/replicator.js.map +1 -1
- package/out/zero-cache/src/server/runner/run-worker.js +1 -1
- package/out/zero-cache/src/server/runner/zero-dispatcher.d.ts.map +1 -1
- package/out/zero-cache/src/server/runner/zero-dispatcher.js +3 -0
- package/out/zero-cache/src/server/runner/zero-dispatcher.js.map +1 -1
- package/out/zero-cache/src/server/syncer.d.ts.map +1 -1
- package/out/zero-cache/src/server/syncer.js +2 -0
- package/out/zero-cache/src/server/syncer.js.map +1 -1
- package/out/zero-cache/src/services/analyze.d.ts.map +1 -1
- package/out/zero-cache/src/services/analyze.js +3 -2
- package/out/zero-cache/src/services/analyze.js.map +1 -1
- package/out/zero-cache/src/services/change-source/common/backfill-manager.d.ts.map +1 -1
- package/out/zero-cache/src/services/change-source/common/backfill-manager.js +14 -1
- package/out/zero-cache/src/services/change-source/common/backfill-manager.js.map +1 -1
- package/out/zero-cache/src/services/change-source/common/change-stream-multiplexer.d.ts.map +1 -1
- package/out/zero-cache/src/services/change-source/common/change-stream-multiplexer.js +15 -3
- package/out/zero-cache/src/services/change-source/common/change-stream-multiplexer.js.map +1 -1
- package/out/zero-cache/src/services/change-source/common/replica-schema.d.ts.map +1 -1
- package/out/zero-cache/src/services/change-source/common/replica-schema.js +2 -39
- package/out/zero-cache/src/services/change-source/common/replica-schema.js.map +1 -1
- package/out/zero-cache/src/services/change-source/pg/backfill-resume.d.ts +156 -0
- package/out/zero-cache/src/services/change-source/pg/backfill-resume.d.ts.map +1 -0
- package/out/zero-cache/src/services/change-source/pg/backfill-resume.js +15 -0
- package/out/zero-cache/src/services/change-source/pg/backfill-resume.js.map +1 -0
- package/out/zero-cache/src/services/change-source/pg/change-source-init.d.ts +25 -0
- package/out/zero-cache/src/services/change-source/pg/change-source-init.d.ts.map +1 -0
- package/out/zero-cache/src/services/change-source/pg/change-source-init.js +202 -0
- package/out/zero-cache/src/services/change-source/pg/change-source-init.js.map +1 -0
- package/out/zero-cache/src/services/change-source/pg/change-source.d.ts +14 -16
- package/out/zero-cache/src/services/change-source/pg/change-source.d.ts.map +1 -1
- package/out/zero-cache/src/services/change-source/pg/change-source.js +64 -143
- package/out/zero-cache/src/services/change-source/pg/change-source.js.map +1 -1
- package/out/zero-cache/src/services/change-source/pg/initial-sync.d.ts +28 -2
- package/out/zero-cache/src/services/change-source/pg/initial-sync.d.ts.map +1 -1
- package/out/zero-cache/src/services/change-source/pg/initial-sync.js +44 -21
- package/out/zero-cache/src/services/change-source/pg/initial-sync.js.map +1 -1
- package/out/zero-cache/src/services/change-source/pg/replication-slot-cleanup-monitor.d.ts +16 -0
- package/out/zero-cache/src/services/change-source/pg/replication-slot-cleanup-monitor.d.ts.map +1 -0
- package/out/zero-cache/src/services/change-source/pg/replication-slot-cleanup-monitor.js +113 -0
- package/out/zero-cache/src/services/change-source/pg/replication-slot-cleanup-monitor.js.map +1 -0
- package/out/zero-cache/src/services/change-source/pg/replication-slots.d.ts +21 -15
- package/out/zero-cache/src/services/change-source/pg/replication-slots.d.ts.map +1 -1
- package/out/zero-cache/src/services/change-source/pg/replication-slots.js +86 -23
- package/out/zero-cache/src/services/change-source/pg/replication-slots.js.map +1 -1
- package/out/zero-cache/src/services/change-source/pg/schema/index-predicate.d.ts.map +1 -1
- package/out/zero-cache/src/services/change-source/pg/schema/index-predicate.js +68 -8
- package/out/zero-cache/src/services/change-source/pg/schema/index-predicate.js.map +1 -1
- package/out/zero-cache/src/services/change-source/pg/schema/init.d.ts.map +1 -1
- package/out/zero-cache/src/services/change-source/pg/schema/init.js +32 -1
- package/out/zero-cache/src/services/change-source/pg/schema/init.js.map +1 -1
- package/out/zero-cache/src/services/change-source/pg/schema/replica-stage-enum.d.ts +10 -0
- package/out/zero-cache/src/services/change-source/pg/schema/replica-stage-enum.d.ts.map +1 -0
- package/out/zero-cache/src/services/change-source/pg/schema/replica-stage-enum.js +6 -0
- package/out/zero-cache/src/services/change-source/pg/schema/replica-stage-enum.js.map +1 -0
- package/out/zero-cache/src/services/change-source/pg/schema/shard.d.ts +50 -27
- package/out/zero-cache/src/services/change-source/pg/schema/shard.d.ts.map +1 -1
- package/out/zero-cache/src/services/change-source/pg/schema/shard.js +100 -34
- package/out/zero-cache/src/services/change-source/pg/schema/shard.js.map +1 -1
- package/out/zero-cache/src/services/change-streamer/backup-monitor.d.ts.map +1 -1
- package/out/zero-cache/src/services/change-streamer/backup-monitor.js +58 -7
- package/out/zero-cache/src/services/change-streamer/backup-monitor.js.map +1 -1
- package/out/zero-cache/src/services/change-streamer/broadcast.d.ts +8 -2
- package/out/zero-cache/src/services/change-streamer/broadcast.d.ts.map +1 -1
- package/out/zero-cache/src/services/change-streamer/broadcast.js +24 -4
- package/out/zero-cache/src/services/change-streamer/broadcast.js.map +1 -1
- package/out/zero-cache/src/services/change-streamer/change-log-codec.d.ts +19 -4
- package/out/zero-cache/src/services/change-streamer/change-log-codec.d.ts.map +1 -1
- package/out/zero-cache/src/services/change-streamer/change-log-codec.js +28 -8
- package/out/zero-cache/src/services/change-streamer/change-log-codec.js.map +1 -1
- package/out/zero-cache/src/services/change-streamer/change-streamer-http.d.ts +5 -2
- package/out/zero-cache/src/services/change-streamer/change-streamer-http.d.ts.map +1 -1
- package/out/zero-cache/src/services/change-streamer/change-streamer-http.js +21 -14
- package/out/zero-cache/src/services/change-streamer/change-streamer-http.js.map +1 -1
- package/out/zero-cache/src/services/change-streamer/change-streamer-service.d.ts +6 -0
- package/out/zero-cache/src/services/change-streamer/change-streamer-service.d.ts.map +1 -1
- package/out/zero-cache/src/services/change-streamer/change-streamer-service.js +97 -63
- package/out/zero-cache/src/services/change-streamer/change-streamer-service.js.map +1 -1
- package/out/zero-cache/src/services/change-streamer/change-streamer.d.ts +12 -8
- package/out/zero-cache/src/services/change-streamer/change-streamer.d.ts.map +1 -1
- package/out/zero-cache/src/services/change-streamer/change-streamer.js.map +1 -1
- package/out/zero-cache/src/services/change-streamer/forwarder.d.ts +3 -3
- package/out/zero-cache/src/services/change-streamer/forwarder.d.ts.map +1 -1
- package/out/zero-cache/src/services/change-streamer/forwarder.js +36 -5
- package/out/zero-cache/src/services/change-streamer/forwarder.js.map +1 -1
- package/out/zero-cache/src/services/change-streamer/schema/tables.d.ts +1 -1
- package/out/zero-cache/src/services/change-streamer/schema/tables.d.ts.map +1 -1
- package/out/zero-cache/src/services/change-streamer/schema/tables.js +4 -4
- package/out/zero-cache/src/services/change-streamer/schema/tables.js.map +1 -1
- package/out/zero-cache/src/services/change-streamer/snapshot.d.ts +3 -1
- package/out/zero-cache/src/services/change-streamer/snapshot.d.ts.map +1 -1
- package/out/zero-cache/src/services/change-streamer/snapshot.js +54 -20
- package/out/zero-cache/src/services/change-streamer/snapshot.js.map +1 -1
- package/out/zero-cache/src/services/change-streamer/sqlite-change-log-writer.d.ts +6 -1
- package/out/zero-cache/src/services/change-streamer/sqlite-change-log-writer.d.ts.map +1 -1
- package/out/zero-cache/src/services/change-streamer/sqlite-change-log-writer.js +27 -9
- package/out/zero-cache/src/services/change-streamer/sqlite-change-log-writer.js.map +1 -1
- package/out/zero-cache/src/services/change-streamer/storer.d.ts +2 -2
- package/out/zero-cache/src/services/change-streamer/storer.d.ts.map +1 -1
- package/out/zero-cache/src/services/change-streamer/storer.js +8 -5
- package/out/zero-cache/src/services/change-streamer/storer.js.map +1 -1
- package/out/zero-cache/src/services/change-streamer/subscriber.d.ts +4 -1
- package/out/zero-cache/src/services/change-streamer/subscriber.d.ts.map +1 -1
- package/out/zero-cache/src/services/change-streamer/subscriber.js +42 -0
- package/out/zero-cache/src/services/change-streamer/subscriber.js.map +1 -1
- package/out/zero-cache/src/services/change-streamer/upstream-acker.d.ts +17 -1
- package/out/zero-cache/src/services/change-streamer/upstream-acker.d.ts.map +1 -1
- package/out/zero-cache/src/services/change-streamer/upstream-acker.js +19 -3
- package/out/zero-cache/src/services/change-streamer/upstream-acker.js.map +1 -1
- package/out/zero-cache/src/services/change-streamer/vfs-watermark-poller.d.ts.map +1 -1
- package/out/zero-cache/src/services/change-streamer/vfs-watermark-poller.js +1 -0
- package/out/zero-cache/src/services/change-streamer/vfs-watermark-poller.js.map +1 -1
- package/out/zero-cache/src/services/life-cycle.d.ts.map +1 -1
- package/out/zero-cache/src/services/life-cycle.js +3 -1
- package/out/zero-cache/src/services/life-cycle.js.map +1 -1
- package/out/zero-cache/src/services/litestream/commands.d.ts.map +1 -1
- package/out/zero-cache/src/services/litestream/commands.js +8 -3
- package/out/zero-cache/src/services/litestream/commands.js.map +1 -1
- package/out/zero-cache/src/services/litestream/config-v5.yml +15 -1
- package/out/zero-cache/src/services/litestream/litestream-checkpointer.js +131 -0
- package/out/zero-cache/src/services/litestream/litestream-checkpointer.js.map +1 -0
- package/out/zero-cache/src/services/litestream/litestream-controller.d.ts +70 -0
- package/out/zero-cache/src/services/litestream/litestream-controller.d.ts.map +1 -0
- package/out/zero-cache/src/services/litestream/litestream-controller.js +101 -0
- package/out/zero-cache/src/services/litestream/litestream-controller.js.map +1 -0
- package/out/zero-cache/src/services/mutagen/mutagen.d.ts +5 -0
- package/out/zero-cache/src/services/mutagen/mutagen.d.ts.map +1 -1
- package/out/zero-cache/src/services/mutagen/mutagen.js +7 -1
- package/out/zero-cache/src/services/mutagen/mutagen.js.map +1 -1
- package/out/zero-cache/src/services/mutagen/pusher.js +2 -1
- package/out/zero-cache/src/services/mutagen/pusher.js.map +1 -1
- package/out/zero-cache/src/services/profz.d.ts +7 -0
- package/out/zero-cache/src/services/profz.d.ts.map +1 -0
- package/out/zero-cache/src/services/profz.js +105 -0
- package/out/zero-cache/src/services/profz.js.map +1 -0
- package/out/zero-cache/src/services/replicator/change-log-db.js.map +1 -1
- package/out/zero-cache/src/services/replicator/change-log-stream-writer.d.ts +5 -4
- package/out/zero-cache/src/services/replicator/change-log-stream-writer.d.ts.map +1 -1
- package/out/zero-cache/src/services/replicator/change-log-stream-writer.js +20 -22
- package/out/zero-cache/src/services/replicator/change-log-stream-writer.js.map +1 -1
- package/out/zero-cache/src/services/replicator/change-processor.js +1 -0
- package/out/zero-cache/src/services/replicator/change-processor.js.map +1 -1
- package/out/zero-cache/src/services/replicator/incremental-sync.d.ts +1 -1
- package/out/zero-cache/src/services/replicator/incremental-sync.d.ts.map +1 -1
- package/out/zero-cache/src/services/replicator/incremental-sync.js +58 -17
- package/out/zero-cache/src/services/replicator/incremental-sync.js.map +1 -1
- package/out/zero-cache/src/services/replicator/replication-status.d.ts +35 -1
- package/out/zero-cache/src/services/replicator/replication-status.d.ts.map +1 -1
- package/out/zero-cache/src/services/replicator/replication-status.js +96 -6
- package/out/zero-cache/src/services/replicator/replication-status.js.map +1 -1
- package/out/zero-cache/src/services/replicator/replicator.js.map +1 -1
- package/out/zero-cache/src/services/replicator/sqlite-change-log-observability.d.ts +6 -6
- package/out/zero-cache/src/services/replicator/sqlite-change-log-observability.d.ts.map +1 -1
- package/out/zero-cache/src/services/replicator/sqlite-change-log-observability.js +19 -14
- package/out/zero-cache/src/services/replicator/sqlite-change-log-observability.js.map +1 -1
- package/out/zero-cache/src/services/replicator/write-worker-client.d.ts +31 -6
- package/out/zero-cache/src/services/replicator/write-worker-client.d.ts.map +1 -1
- package/out/zero-cache/src/services/replicator/write-worker-client.js +5 -4
- package/out/zero-cache/src/services/replicator/write-worker-client.js.map +1 -1
- package/out/zero-cache/src/services/replicator/write-worker.js +30 -5
- package/out/zero-cache/src/services/replicator/write-worker.js.map +1 -1
- package/out/zero-cache/src/services/run-ast.d.ts +12 -0
- package/out/zero-cache/src/services/run-ast.d.ts.map +1 -1
- package/out/zero-cache/src/services/run-ast.js +26 -14
- package/out/zero-cache/src/services/run-ast.js.map +1 -1
- package/out/zero-cache/src/services/runner.d.ts.map +1 -1
- package/out/zero-cache/src/services/runner.js +1 -1
- package/out/zero-cache/src/services/runner.js.map +1 -1
- package/out/zero-cache/src/services/view-syncer/client-handler.d.ts +1 -1
- package/out/zero-cache/src/services/view-syncer/client-handler.d.ts.map +1 -1
- package/out/zero-cache/src/services/view-syncer/client-handler.js +36 -31
- package/out/zero-cache/src/services/view-syncer/client-handler.js.map +1 -1
- package/out/zero-cache/src/services/view-syncer/cvr.d.ts +45 -0
- package/out/zero-cache/src/services/view-syncer/cvr.d.ts.map +1 -1
- package/out/zero-cache/src/services/view-syncer/cvr.js +133 -1
- package/out/zero-cache/src/services/view-syncer/cvr.js.map +1 -1
- package/out/zero-cache/src/services/view-syncer/drain-coordinator.d.ts +14 -1
- package/out/zero-cache/src/services/view-syncer/drain-coordinator.d.ts.map +1 -1
- package/out/zero-cache/src/services/view-syncer/drain-coordinator.js +30 -4
- package/out/zero-cache/src/services/view-syncer/drain-coordinator.js.map +1 -1
- package/out/zero-cache/src/services/view-syncer/hydration-budget.d.ts +33 -0
- package/out/zero-cache/src/services/view-syncer/hydration-budget.d.ts.map +1 -0
- package/out/zero-cache/src/services/view-syncer/hydration-budget.js +62 -0
- package/out/zero-cache/src/services/view-syncer/hydration-budget.js.map +1 -0
- package/out/zero-cache/src/services/view-syncer/hydration-circuit-breaker.d.ts +51 -0
- package/out/zero-cache/src/services/view-syncer/hydration-circuit-breaker.d.ts.map +1 -0
- package/out/zero-cache/src/services/view-syncer/hydration-circuit-breaker.js +78 -0
- package/out/zero-cache/src/services/view-syncer/hydration-circuit-breaker.js.map +1 -0
- package/out/zero-cache/src/services/view-syncer/inspect-handler.d.ts +1 -1
- package/out/zero-cache/src/services/view-syncer/inspect-handler.d.ts.map +1 -1
- package/out/zero-cache/src/services/view-syncer/inspect-handler.js +2 -2
- package/out/zero-cache/src/services/view-syncer/inspect-handler.js.map +1 -1
- package/out/zero-cache/src/services/view-syncer/pipeline-driver.d.ts +2 -1
- package/out/zero-cache/src/services/view-syncer/pipeline-driver.d.ts.map +1 -1
- package/out/zero-cache/src/services/view-syncer/pipeline-driver.js +33 -18
- package/out/zero-cache/src/services/view-syncer/pipeline-driver.js.map +1 -1
- package/out/zero-cache/src/services/view-syncer/snapshotter.d.ts +4 -1
- package/out/zero-cache/src/services/view-syncer/snapshotter.d.ts.map +1 -1
- package/out/zero-cache/src/services/view-syncer/snapshotter.js +9 -3
- package/out/zero-cache/src/services/view-syncer/snapshotter.js.map +1 -1
- package/out/zero-cache/src/services/view-syncer/view-syncer.d.ts +11 -1
- package/out/zero-cache/src/services/view-syncer/view-syncer.d.ts.map +1 -1
- package/out/zero-cache/src/services/view-syncer/view-syncer.js +507 -93
- package/out/zero-cache/src/services/view-syncer/view-syncer.js.map +1 -1
- package/out/zero-cache/src/types/processes.d.ts +32 -0
- package/out/zero-cache/src/types/processes.d.ts.map +1 -1
- package/out/zero-cache/src/types/processes.js +43 -4
- package/out/zero-cache/src/types/processes.js.map +1 -1
- package/out/zero-cache/src/types/profiler.d.ts +12 -0
- package/out/zero-cache/src/types/profiler.d.ts.map +1 -1
- package/out/zero-cache/src/types/profiler.js +46 -1
- package/out/zero-cache/src/types/profiler.js.map +1 -1
- package/out/zero-cache/src/types/streams.d.ts +25 -9
- package/out/zero-cache/src/types/streams.d.ts.map +1 -1
- package/out/zero-cache/src/types/streams.js +118 -34
- package/out/zero-cache/src/types/streams.js.map +1 -1
- package/out/zero-cache/src/types/subscription.d.ts +4 -0
- package/out/zero-cache/src/types/subscription.d.ts.map +1 -1
- package/out/zero-cache/src/types/subscription.js +71 -0
- package/out/zero-cache/src/types/subscription.js.map +1 -1
- package/out/zero-cache/src/types/ws.d.ts.map +1 -1
- package/out/zero-cache/src/types/ws.js +23 -8
- package/out/zero-cache/src/types/ws.js.map +1 -1
- package/out/zero-cache/src/workers/connection.js +2 -2
- package/out/zero-cache/src/workers/replicator.d.ts +1 -0
- package/out/zero-cache/src/workers/replicator.d.ts.map +1 -1
- package/out/zero-cache/src/workers/replicator.js +4 -2
- package/out/zero-cache/src/workers/replicator.js.map +1 -1
- package/out/zero-cache/src/workers/syncer-ws-message-handler.d.ts.map +1 -1
- package/out/zero-cache/src/workers/syncer-ws-message-handler.js +21 -16
- package/out/zero-cache/src/workers/syncer-ws-message-handler.js.map +1 -1
- package/out/zero-cache/src/workers/syncer.d.ts.map +1 -1
- package/out/zero-cache/src/workers/syncer.js +10 -1
- package/out/zero-cache/src/workers/syncer.js.map +1 -1
- package/out/zero-client/src/client/connection-manager.d.ts +20 -0
- package/out/zero-client/src/client/connection-manager.d.ts.map +1 -1
- package/out/zero-client/src/client/connection-manager.js +77 -13
- package/out/zero-client/src/client/connection-manager.js.map +1 -1
- package/out/zero-client/src/client/connection-status-enum.d.ts +2 -0
- package/out/zero-client/src/client/connection-status-enum.d.ts.map +1 -1
- package/out/zero-client/src/client/connection-status-enum.js +3 -1
- package/out/zero-client/src/client/connection-status-enum.js.map +1 -1
- package/out/zero-client/src/client/connection.d.ts +6 -0
- package/out/zero-client/src/client/connection.d.ts.map +1 -1
- package/out/zero-client/src/client/connection.js +2 -1
- package/out/zero-client/src/client/connection.js.map +1 -1
- package/out/zero-client/src/client/context.d.ts +27 -2
- package/out/zero-client/src/client/context.d.ts.map +1 -1
- package/out/zero-client/src/client/context.js +112 -1
- package/out/zero-client/src/client/context.js.map +1 -1
- package/out/zero-client/src/client/custom.js +2 -2
- package/out/zero-client/src/client/custom.js.map +1 -1
- package/out/zero-client/src/client/error.js +1 -1
- package/out/zero-client/src/client/inspector/lazy-inspector.js +2 -2
- package/out/zero-client/src/client/inspector/lazy-inspector.js.map +1 -1
- package/out/zero-client/src/client/ivm-branch.d.ts.map +1 -1
- package/out/zero-client/src/client/ivm-branch.js +20 -3
- package/out/zero-client/src/client/ivm-branch.js.map +1 -1
- package/out/zero-client/src/client/mutator-proxy.d.ts.map +1 -1
- package/out/zero-client/src/client/mutator-proxy.js +2 -1
- package/out/zero-client/src/client/mutator-proxy.js.map +1 -1
- package/out/zero-client/src/client/options.d.ts +3 -0
- package/out/zero-client/src/client/options.d.ts.map +1 -1
- package/out/zero-client/src/client/options.js.map +1 -1
- package/out/zero-client/src/client/query-manager.d.ts.map +1 -1
- package/out/zero-client/src/client/query-manager.js +42 -6
- package/out/zero-client/src/client/query-manager.js.map +1 -1
- package/out/zero-client/src/client/version.js +1 -1
- package/out/zero-client/src/client/zero-rep.d.ts +1 -0
- package/out/zero-client/src/client/zero-rep.d.ts.map +1 -1
- package/out/zero-client/src/client/zero-rep.js +9 -6
- package/out/zero-client/src/client/zero-rep.js.map +1 -1
- package/out/zero-client/src/client/zero.d.ts +18 -3
- package/out/zero-client/src/client/zero.d.ts.map +1 -1
- package/out/zero-client/src/client/zero.js +122 -34
- package/out/zero-client/src/client/zero.js.map +1 -1
- package/out/zero-client/src/types/query-result.d.ts +2 -0
- package/out/zero-client/src/types/query-result.d.ts.map +1 -1
- package/out/zero-events/src/status.d.ts +26 -0
- package/out/zero-events/src/status.d.ts.map +1 -1
- package/out/zero-protocol/src/analyze-query-result.d.ts +5 -5
- package/out/zero-protocol/src/analyze-query-result.d.ts.map +1 -1
- package/out/zero-protocol/src/analyze-query-result.js +1 -1
- package/out/zero-protocol/src/analyze-query-result.js.map +1 -1
- package/out/zero-protocol/src/down.d.ts +4 -4
- package/out/zero-protocol/src/inspect-down.d.ts +6 -6
- package/out/zero-protocol/src/protocol-version.d.ts +1 -1
- package/out/zero-protocol/src/protocol-version.d.ts.map +1 -1
- package/out/zero-protocol/src/protocol-version.js.map +1 -1
- package/out/zero-react/src/use-query.d.ts +5 -4
- package/out/zero-react/src/use-query.d.ts.map +1 -1
- package/out/zero-react/src/use-query.js +86 -9
- package/out/zero-react/src/use-query.js.map +1 -1
- package/out/zero-server/src/adapters/drizzle.d.ts +2 -1
- package/out/zero-server/src/adapters/drizzle.d.ts.map +1 -1
- package/out/zero-server/src/adapters/drizzle.js +12 -7
- package/out/zero-server/src/adapters/drizzle.js.map +1 -1
- package/out/zero-server/src/adapters/kysely.d.ts +2 -1
- package/out/zero-server/src/adapters/kysely.d.ts.map +1 -1
- package/out/zero-server/src/adapters/kysely.js +8 -2
- package/out/zero-server/src/adapters/kysely.js.map +1 -1
- package/out/zero-server/src/adapters/pg.d.ts +1 -0
- package/out/zero-server/src/adapters/pg.d.ts.map +1 -1
- package/out/zero-server/src/adapters/pg.js +8 -2
- package/out/zero-server/src/adapters/pg.js.map +1 -1
- package/out/zero-server/src/adapters/postgresjs.d.ts +1 -0
- package/out/zero-server/src/adapters/postgresjs.d.ts.map +1 -1
- package/out/zero-server/src/adapters/postgresjs.js +7 -1
- package/out/zero-server/src/adapters/postgresjs.js.map +1 -1
- package/out/zero-server/src/adapters/prisma.d.ts +3 -2
- package/out/zero-server/src/adapters/prisma.d.ts.map +1 -1
- package/out/zero-server/src/adapters/prisma.js +8 -2
- package/out/zero-server/src/adapters/prisma.js.map +1 -1
- package/out/zero-server/src/custom.d.ts +5 -1
- package/out/zero-server/src/custom.d.ts.map +1 -1
- package/out/zero-server/src/custom.js +6 -3
- package/out/zero-server/src/custom.js.map +1 -1
- package/out/zero-server/src/mod.d.ts +1 -1
- package/out/zero-server/src/mod.d.ts.map +1 -1
- package/out/zero-server/src/pg-query-executor.d.ts +2 -2
- package/out/zero-server/src/pg-query-executor.d.ts.map +1 -1
- package/out/zero-server/src/pg-query-executor.js +2 -2
- package/out/zero-server/src/pg-query-executor.js.map +1 -1
- package/out/zero-server/src/schema.d.ts +2 -2
- package/out/zero-server/src/schema.d.ts.map +1 -1
- package/out/zero-server/src/schema.js +2 -2
- package/out/zero-server/src/schema.js.map +1 -1
- package/out/zero-server/src/zql-database.d.ts +21 -1
- package/out/zero-server/src/zql-database.d.ts.map +1 -1
- package/out/zero-server/src/zql-database.js +33 -2
- package/out/zero-server/src/zql-database.js.map +1 -1
- package/out/zero-solid/src/solid-view.d.ts +17 -0
- package/out/zero-solid/src/solid-view.d.ts.map +1 -1
- package/out/zero-solid/src/solid-view.js +39 -1
- package/out/zero-solid/src/solid-view.js.map +1 -1
- package/out/zero-solid/src/use-query.js +5 -3
- package/out/zero-solid/src/use-query.js.map +1 -1
- package/out/zql/src/builder/builder.d.ts.map +1 -1
- package/out/zql/src/builder/builder.js +16 -10
- package/out/zql/src/builder/builder.js.map +1 -1
- package/out/zql/src/builder/debug-delegate.d.ts +6 -1
- package/out/zql/src/builder/debug-delegate.d.ts.map +1 -1
- package/out/zql/src/builder/debug-delegate.js +14 -2
- package/out/zql/src/builder/debug-delegate.js.map +1 -1
- package/out/zql/src/ivm/array-view.d.ts +29 -0
- package/out/zql/src/ivm/array-view.d.ts.map +1 -1
- package/out/zql/src/ivm/array-view.js +63 -12
- package/out/zql/src/ivm/array-view.js.map +1 -1
- package/out/zql/src/ivm/cap.d.ts.map +1 -1
- package/out/zql/src/ivm/cap.js +5 -3
- package/out/zql/src/ivm/cap.js.map +1 -1
- package/out/zql/src/ivm/data.d.ts.map +1 -1
- package/out/zql/src/ivm/data.js +8 -3
- package/out/zql/src/ivm/data.js.map +1 -1
- package/out/zql/src/ivm/deferred-input.d.ts +44 -0
- package/out/zql/src/ivm/deferred-input.d.ts.map +1 -0
- package/out/zql/src/ivm/deferred-input.js +89 -0
- package/out/zql/src/ivm/deferred-input.js.map +1 -0
- package/out/zql/src/ivm/exists.d.ts +1 -1
- package/out/zql/src/ivm/exists.d.ts.map +1 -1
- package/out/zql/src/ivm/exists.js.map +1 -1
- package/out/zql/src/ivm/fan-in.d.ts +2 -2
- package/out/zql/src/ivm/fan-in.d.ts.map +1 -1
- package/out/zql/src/ivm/fan-in.js +6 -5
- package/out/zql/src/ivm/fan-in.js.map +1 -1
- package/out/zql/src/ivm/fan-out.d.ts +2 -2
- package/out/zql/src/ivm/fan-out.d.ts.map +1 -1
- package/out/zql/src/ivm/fan-out.js.map +1 -1
- package/out/zql/src/ivm/filter-operators.d.ts +7 -6
- package/out/zql/src/ivm/filter-operators.d.ts.map +1 -1
- package/out/zql/src/ivm/filter-operators.js +29 -12
- package/out/zql/src/ivm/filter-operators.js.map +1 -1
- package/out/zql/src/ivm/filter.d.ts +2 -2
- package/out/zql/src/ivm/filter.d.ts.map +1 -1
- package/out/zql/src/ivm/filter.js +7 -4
- package/out/zql/src/ivm/filter.js.map +1 -1
- package/out/zql/src/ivm/flipped-join.js +2 -2
- package/out/zql/src/ivm/flipped-join.js.map +1 -1
- package/out/zql/src/ivm/maybe-split-and-push-edit-change.d.ts +1 -1
- package/out/zql/src/ivm/maybe-split-and-push-edit-change.d.ts.map +1 -1
- package/out/zql/src/ivm/maybe-split-and-push-edit-change.js +6 -4
- package/out/zql/src/ivm/maybe-split-and-push-edit-change.js.map +1 -1
- package/out/zql/src/ivm/memory-source.d.ts +15 -3
- package/out/zql/src/ivm/memory-source.d.ts.map +1 -1
- package/out/zql/src/ivm/memory-source.js +57 -23
- package/out/zql/src/ivm/memory-source.js.map +1 -1
- package/out/zql/src/ivm/operator.d.ts +40 -0
- package/out/zql/src/ivm/operator.d.ts.map +1 -1
- package/out/zql/src/ivm/operator.js.map +1 -1
- package/out/zql/src/ivm/take.d.ts.map +1 -1
- package/out/zql/src/ivm/take.js +3 -4
- package/out/zql/src/ivm/take.js.map +1 -1
- package/out/zql/src/ivm/union-fan-in.d.ts.map +1 -1
- package/out/zql/src/ivm/union-fan-in.js +9 -7
- package/out/zql/src/ivm/union-fan-in.js.map +1 -1
- package/out/zql/src/ivm/view-apply-change.d.ts.map +1 -1
- package/out/zql/src/ivm/view-apply-change.js +9 -4
- package/out/zql/src/ivm/view-apply-change.js.map +1 -1
- package/out/zql/src/ivm/view.d.ts +15 -0
- package/out/zql/src/ivm/view.d.ts.map +1 -1
- package/out/zql/src/mutate/custom.d.ts +24 -1
- package/out/zql/src/mutate/custom.d.ts.map +1 -1
- package/out/zql/src/mutate/custom.js.map +1 -1
- package/out/zql/src/mutate/mutator-registry.d.ts.map +1 -1
- package/out/zql/src/mutate/mutator-registry.js +2 -2
- package/out/zql/src/mutate/mutator-registry.js.map +1 -1
- package/out/zql/src/planner/planner-builder.d.ts.map +1 -1
- package/out/zql/src/planner/planner-builder.js +14 -4
- package/out/zql/src/planner/planner-builder.js.map +1 -1
- package/out/zql/src/planner/planner-connection.d.ts +13 -0
- package/out/zql/src/planner/planner-connection.d.ts.map +1 -1
- package/out/zql/src/planner/planner-connection.js +42 -7
- package/out/zql/src/planner/planner-connection.js.map +1 -1
- package/out/zql/src/planner/planner-debug.d.ts +1 -1
- package/out/zql/src/planner/planner-debug.d.ts.map +1 -1
- package/out/zql/src/planner/planner-debug.js.map +1 -1
- package/out/zql/src/planner/planner-fan-out.d.ts +1 -0
- package/out/zql/src/planner/planner-fan-out.d.ts.map +1 -1
- package/out/zql/src/planner/planner-fan-out.js +3 -0
- package/out/zql/src/planner/planner-fan-out.js.map +1 -1
- package/out/zql/src/planner/planner-filter.d.ts +59 -0
- package/out/zql/src/planner/planner-filter.d.ts.map +1 -0
- package/out/zql/src/planner/planner-filter.js +116 -0
- package/out/zql/src/planner/planner-filter.js.map +1 -0
- package/out/zql/src/planner/planner-graph.d.ts +2 -0
- package/out/zql/src/planner/planner-graph.d.ts.map +1 -1
- package/out/zql/src/planner/planner-graph.js +2 -0
- package/out/zql/src/planner/planner-graph.js.map +1 -1
- package/out/zql/src/planner/planner-join.d.ts +2 -0
- package/out/zql/src/planner/planner-join.d.ts.map +1 -1
- package/out/zql/src/planner/planner-join.js +7 -0
- package/out/zql/src/planner/planner-join.js.map +1 -1
- package/out/zql/src/planner/planner-node.d.ts +2 -1
- package/out/zql/src/planner/planner-node.d.ts.map +1 -1
- package/out/zql/src/planner/planner-node.js.map +1 -1
- package/out/zql/src/query/query-delegate-base.d.ts +9 -1
- package/out/zql/src/query/query-delegate-base.d.ts.map +1 -1
- package/out/zql/src/query/query-delegate-base.js +129 -33
- package/out/zql/src/query/query-delegate-base.js.map +1 -1
- package/out/zql/src/query/query-delegate.d.ts +50 -1
- package/out/zql/src/query/query-delegate.d.ts.map +1 -1
- package/out/zql/src/query/query-impl.d.ts +13 -11
- package/out/zql/src/query/query-impl.d.ts.map +1 -1
- package/out/zql/src/query/query-impl.js +383 -60
- package/out/zql/src/query/query-impl.js.map +1 -1
- package/out/zql/src/query/query-transitions.d.ts +171 -0
- package/out/zql/src/query/query-transitions.d.ts.map +1 -0
- package/out/zql/src/query/query-transitions.js +207 -0
- package/out/zql/src/query/query-transitions.js.map +1 -0
- package/out/zql/src/query/query.d.ts +11 -2
- package/out/zql/src/query/query.d.ts.map +1 -1
- package/out/zql/src/query/runnable-query-impl.d.ts +1 -0
- package/out/zql/src/query/runnable-query-impl.d.ts.map +1 -1
- package/out/zql/src/query/runnable-query-impl.js +21 -1
- package/out/zql/src/query/runnable-query-impl.js.map +1 -1
- package/out/zql/src/query/typed-view.d.ts +1 -1
- package/out/zql/src/query/typed-view.d.ts.map +1 -1
- package/out/zqlite/src/db.d.ts +9 -1
- package/out/zqlite/src/db.d.ts.map +1 -1
- package/out/zqlite/src/db.js +8 -2
- package/out/zqlite/src/db.js.map +1 -1
- package/out/zqlite/src/query-builder.d.ts +1 -1
- package/out/zqlite/src/query-builder.d.ts.map +1 -1
- package/out/zqlite/src/query-builder.js +2 -1
- package/out/zqlite/src/query-builder.js.map +1 -1
- package/out/zqlite/src/sqlite-cost-model.d.ts.map +1 -1
- package/out/zqlite/src/sqlite-cost-model.js +2 -30
- package/out/zqlite/src/sqlite-cost-model.js.map +1 -1
- package/out/zqlite/src/table-source.d.ts +1 -0
- package/out/zqlite/src/table-source.d.ts.map +1 -1
- package/out/zqlite/src/table-source.js +20 -8
- package/out/zqlite/src/table-source.js.map +1 -1
- package/package.json +3 -4
|
@@ -50,22 +50,24 @@ var ChangeLogStreamWriter = class {
|
|
|
50
50
|
`);
|
|
51
51
|
this.#cookies = new ChangeLogCookieWriter(db.db);
|
|
52
52
|
}
|
|
53
|
-
begin(watermark, json) {
|
|
53
|
+
begin(watermark, json, storedChange = extractChangeSubstring(json, "begin"), hash = true) {
|
|
54
54
|
assertInvariant(this.#watermark === void 0, `change-log stream transaction already open at ${this.#watermark}`);
|
|
55
55
|
this.#db.beginImmediate();
|
|
56
56
|
this.#watermark = watermark;
|
|
57
57
|
this.#pos = 0;
|
|
58
|
-
const change =
|
|
58
|
+
const change = storedChange;
|
|
59
59
|
const estimatedBytes = estimateChangeLogStreamRowBytes(watermark, "begin", change);
|
|
60
60
|
this.#insertChange.run(watermark, this.#pos, "begin", estimatedBytes, change);
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
61
|
+
if (hash) {
|
|
62
|
+
this.#hasher = new ChangeLogTransactionHasher();
|
|
63
|
+
this.#hasher.add({
|
|
64
|
+
watermark,
|
|
65
|
+
pos: this.#pos,
|
|
66
|
+
tag: "begin",
|
|
67
|
+
change,
|
|
68
|
+
precommit: null
|
|
69
|
+
});
|
|
70
|
+
}
|
|
69
71
|
this.#estimatedBytes = estimatedBytes;
|
|
70
72
|
}
|
|
71
73
|
/**
|
|
@@ -78,13 +80,13 @@ var ChangeLogStreamWriter = class {
|
|
|
78
80
|
* has to flush its buffered batch to get the same ordering
|
|
79
81
|
* (`Storer.#processQueue()`); SQLite gets it from statement order.
|
|
80
82
|
*/
|
|
81
|
-
append(json, change) {
|
|
83
|
+
append(json, change, storedChange = extractChangeSubstring(json, change.tag)) {
|
|
82
84
|
const watermark = this.#requireWatermark();
|
|
83
85
|
const { tag } = change;
|
|
84
|
-
const stored =
|
|
86
|
+
const stored = storedChange;
|
|
85
87
|
const estimatedBytes = estimateChangeLogStreamRowBytes(watermark, tag, stored);
|
|
86
88
|
this.#insertChange.run(watermark, ++this.#pos, tag, estimatedBytes, stored);
|
|
87
|
-
this.#
|
|
89
|
+
this.#hasher?.add({
|
|
88
90
|
watermark,
|
|
89
91
|
pos: this.#pos,
|
|
90
92
|
tag,
|
|
@@ -94,14 +96,14 @@ var ChangeLogStreamWriter = class {
|
|
|
94
96
|
this.#estimatedBytes += estimatedBytes;
|
|
95
97
|
if (isSchemaChange(change)) for (const op of this.#cookies.apply(change)) this.#cookieMutations.push(op.op);
|
|
96
98
|
}
|
|
97
|
-
commit(watermark, json, writeTimeMs) {
|
|
99
|
+
commit(watermark, json, writeTimeMs, storedChange = extractChangeSubstring(json, "commit")) {
|
|
98
100
|
const precommit = this.#requireWatermark();
|
|
99
101
|
assertInvariant(watermark === precommit, `change-log stream commit ${watermark} does not match begin ${precommit}`);
|
|
100
|
-
const change =
|
|
102
|
+
const change = storedChange;
|
|
101
103
|
const estimatedBytes = estimateChangeLogStreamRowBytes(watermark, "commit", change, precommit, true);
|
|
102
104
|
this.#insertCommit.run(watermark, ++this.#pos, estimatedBytes, change, precommit, writeTimeMs);
|
|
103
|
-
const hasher = this.#
|
|
104
|
-
hasher
|
|
105
|
+
const hasher = this.#hasher;
|
|
106
|
+
hasher?.add({
|
|
105
107
|
watermark,
|
|
106
108
|
pos: this.#pos,
|
|
107
109
|
tag: "commit",
|
|
@@ -111,7 +113,7 @@ var ChangeLogStreamWriter = class {
|
|
|
111
113
|
const stats = {
|
|
112
114
|
rows: this.#pos + 1,
|
|
113
115
|
estimatedBytes: this.#estimatedBytes + estimatedBytes,
|
|
114
|
-
hash: hasher
|
|
116
|
+
hash: hasher?.digest(),
|
|
115
117
|
cookieMutations: this.#cookieMutations
|
|
116
118
|
};
|
|
117
119
|
this.#db.commit();
|
|
@@ -139,10 +141,6 @@ var ChangeLogStreamWriter = class {
|
|
|
139
141
|
assertInvariant(this.#watermark !== void 0, "change-log stream message received outside of a transaction");
|
|
140
142
|
return this.#watermark;
|
|
141
143
|
}
|
|
142
|
-
#requireHasher() {
|
|
143
|
-
assertInvariant(this.#hasher !== void 0, "change-log stream hash received outside of a transaction");
|
|
144
|
-
return this.#hasher;
|
|
145
|
-
}
|
|
146
144
|
};
|
|
147
145
|
//#endregion
|
|
148
146
|
export { ChangeLogStreamWriter };
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"change-log-stream-writer.js","names":["#db","#insertChange","#insertCommit","#cookies","#watermark","#pos","#hasher","#estimatedBytes","#requireWatermark","#requireHasher","#cookieMutations","#reset"],"sources":["../../../../../../zero-cache/src/services/replicator/change-log-stream-writer.ts"],"sourcesContent":["import type {Statement} from '../../../../zqlite/src/db.ts';\nimport type {StatementRunner} from '../../db/statements.ts';\nimport {\n isSchemaChange,\n type DataOrSchemaChange,\n} from '../change-source/protocol/current/data.ts';\nimport {extractChangeSubstring} from '../change-streamer/change-log-codec.ts';\nimport {ChangeLogTransactionHasher} from '../change-streamer/change-log-transaction-hash.ts';\nimport {ChangeLogCookieWriter, type CookieOpTag} from './change-log-cookies.ts';\nimport {\n CHANGE_LOG_STREAM_TABLE,\n estimateChangeLogStreamRowBytes,\n} from './change-log-db.ts';\n\nexport type ChangeLogStreamTransactionStats = {\n readonly rows: number;\n readonly estimatedBytes: number;\n readonly hash: string;\n /**\n * The cookie mutations this transaction's schema changes folded in, in stream\n * order. Empty for the overwhelming majority of transactions.\n */\n readonly cookieMutations: readonly CookieOpTag[];\n};\n\nexport class ChangeLogStreamInvariantError extends Error {\n override readonly name = 'ChangeLogStreamInvariantError';\n}\n\nfunction assertInvariant(\n condition: unknown,\n message: string,\n): asserts condition {\n if (!condition) {\n throw new ChangeLogStreamInvariantError(message);\n }\n}\n\n/**\n * Appends the canonical downstream stream to the change-log database, which is\n * a separate file from the replica, and folds its schema changes into the log's\n * cookie jar (`change-log-cookies.ts`) as it goes.\n *\n * This class owns the change log's transaction, and its transaction alone. It is\n * driven from the change-streamer's stream loop, which commits it before\n * forwarding a transaction's `commit` message; see `SQLiteChangeLogWriter` in\n * `change-streamer/sqlite-change-log-writer.ts` for the ordering that makes a\n * hole unreachable, and for the fail-soft policy that wraps every call here.\n *\n * Because the cookies are written in that same transaction, they are atomic with\n * the head and {@link rollback} discards them with the rows they were folded\n * from. Neither needs any handling of its own.\n */\nexport class ChangeLogStreamWriter {\n readonly #db: StatementRunner;\n readonly #insertChange: Statement;\n readonly #insertCommit: Statement;\n readonly #cookies: ChangeLogCookieWriter;\n\n #watermark: string | undefined;\n #pos = 0;\n #estimatedBytes = 0;\n #hasher: ChangeLogTransactionHasher | undefined;\n #cookieMutations: CookieOpTag[] = [];\n\n constructor(db: StatementRunner) {\n this.#db = db;\n this.#insertChange = db.db.prepare(/*sql*/ `\n INSERT INTO \"${CHANGE_LOG_STREAM_TABLE}\"\n (\"watermark\", \"pos\", \"tag\", \"estimatedBytes\", \"change\")\n VALUES (?, ?, ?, ?, ?)\n `);\n this.#insertCommit = db.db.prepare(/*sql*/ `\n INSERT INTO \"${CHANGE_LOG_STREAM_TABLE}\"\n (\"watermark\", \"pos\", \"tag\", \"estimatedBytes\", \"change\", \"precommit\",\n \"writeTimeMs\")\n VALUES (?, ?, 'commit', ?, ?, ?, ?)\n `);\n this.#cookies = new ChangeLogCookieWriter(db.db);\n }\n\n begin(watermark: string, json: string): void {\n assertInvariant(\n this.#watermark === undefined,\n `change-log stream transaction already open at ${this.#watermark}`,\n );\n // This process is the sole writer of the change-log database, so the write\n // lock is taken up front rather than risking a failed upgrade from a\n // deferred transaction partway through the append.\n this.#db.beginImmediate();\n this.#watermark = watermark;\n this.#pos = 0;\n const change = extractChangeSubstring(json, 'begin');\n const estimatedBytes = estimateChangeLogStreamRowBytes(\n watermark,\n 'begin',\n change,\n );\n this.#insertChange.run(\n watermark,\n this.#pos,\n 'begin',\n estimatedBytes,\n change,\n );\n this.#hasher = new ChangeLogTransactionHasher();\n this.#hasher.add({\n watermark,\n pos: this.#pos,\n tag: 'begin',\n change,\n precommit: null,\n });\n this.#estimatedBytes = estimatedBytes;\n }\n\n /**\n * Appends one data or schema change. The parsed message is taken rather than\n * just its tag because a schema change also mutates the cookie jar, and the\n * caller already holds it — so the fold costs no re-parse.\n *\n * The cookie statements run here, i.e. inside this transaction and in stream\n * order relative to the rows they were folded from. The Postgres change log\n * has to flush its buffered batch to get the same ordering\n * (`Storer.#processQueue()`); SQLite gets it from statement order.\n */\n append(json: string, change: DataOrSchemaChange): void {\n const watermark = this.#requireWatermark();\n const {tag} = change;\n const stored = extractChangeSubstring(json, tag);\n const estimatedBytes = estimateChangeLogStreamRowBytes(\n watermark,\n tag,\n stored,\n );\n this.#insertChange.run(watermark, ++this.#pos, tag, estimatedBytes, stored);\n this.#requireHasher().add({\n watermark,\n pos: this.#pos,\n tag,\n change: stored,\n precommit: null,\n });\n this.#estimatedBytes += estimatedBytes;\n\n if (isSchemaChange(change)) {\n for (const op of this.#cookies.apply(change)) {\n this.#cookieMutations.push(op.op);\n }\n }\n }\n\n commit(\n watermark: string,\n json: string,\n writeTimeMs: number,\n ): ChangeLogStreamTransactionStats {\n const precommit = this.#requireWatermark();\n assertInvariant(\n watermark === precommit,\n `change-log stream commit ${watermark} does not match begin ${precommit}`,\n );\n const change = extractChangeSubstring(json, 'commit');\n const estimatedBytes = estimateChangeLogStreamRowBytes(\n watermark,\n 'commit',\n change,\n precommit,\n true,\n );\n this.#insertCommit.run(\n watermark,\n ++this.#pos,\n estimatedBytes,\n change,\n precommit,\n writeTimeMs,\n );\n const hasher = this.#requireHasher();\n hasher.add({\n watermark,\n pos: this.#pos,\n tag: 'commit',\n change,\n precommit,\n });\n const stats = {\n rows: this.#pos + 1,\n estimatedBytes: this.#estimatedBytes + estimatedBytes,\n hash: hasher.digest(),\n cookieMutations: this.#cookieMutations,\n };\n // The log is durable at `watermark` from here on, and nothing that can\n // advance the watermark the stream would resume from has run yet. A crash in\n // that window leaves a transaction the resumed stream re-delivers, which\n // reconciliation truncates.\n this.#db.commit();\n this.#reset();\n return stats;\n }\n\n /**\n * Discards the open transaction, if any. Safe to call when none is open — the\n * caller rolls back on an interrupted change stream, which can arrive between\n * transactions as well as inside one.\n */\n rollback(): void {\n const {inTransaction} = this.#db.db;\n this.#reset();\n if (inTransaction) {\n this.#db.rollback();\n }\n }\n\n #reset(): void {\n this.#watermark = undefined;\n this.#pos = 0;\n this.#estimatedBytes = 0;\n this.#hasher = undefined;\n // A fresh array rather than a truncation: the committed stats hold a\n // reference to the previous one.\n this.#cookieMutations = [];\n }\n\n #requireWatermark(): string {\n assertInvariant(\n this.#watermark !== undefined,\n 'change-log stream message received outside of a transaction',\n );\n return this.#watermark;\n }\n\n #requireHasher(): ChangeLogTransactionHasher {\n assertInvariant(\n this.#hasher !== undefined,\n 'change-log stream hash received outside of a transaction',\n );\n return this.#hasher;\n }\n}\n"],"mappings":";;;;;;AAyBA,IAAa,gCAAb,cAAmD,MAAM;CACvD,OAAyB;AAC3B;AAEA,SAAS,gBACP,WACA,SACmB;CACnB,IAAI,CAAC,WACH,MAAM,IAAI,8BAA8B,OAAO;AAEnD;;;;;;;;;;;;;;;;AAiBA,IAAa,wBAAb,MAAmC;CACjC;CACA;CACA;CACA;CAEA;CACA,OAAO;CACP,kBAAkB;CAClB;CACA,mBAAkC,CAAC;CAEnC,YAAY,IAAqB;EAC/B,KAAKA,MAAM;EACX,KAAKC,gBAAgB,GAAG,GAAG,QAAgB;qBAC1B,wBAAwB;;;KAGxC;EACD,KAAKC,gBAAgB,GAAG,GAAG,QAAgB;qBAC1B,wBAAwB;;;;KAIxC;EACD,KAAKC,WAAW,IAAI,sBAAsB,GAAG,EAAE;CACjD;CAEA,MAAM,WAAmB,MAAoB;EAC3C,gBACE,KAAKC,eAAe,KAAA,GACpB,iDAAiD,KAAKA,YACxD;EAIA,KAAKJ,IAAI,eAAe;EACxB,KAAKI,aAAa;EAClB,KAAKC,OAAO;EACZ,MAAM,SAAS,uBAAuB,MAAM,OAAO;EACnD,MAAM,iBAAiB,gCACrB,WACA,SACA,MACF;EACA,KAAKJ,cAAc,IACjB,WACA,KAAKI,MACL,SACA,gBACA,MACF;EACA,KAAKC,UAAU,IAAI,2BAA2B;EAC9C,KAAKA,QAAQ,IAAI;GACf;GACA,KAAK,KAAKD;GACV,KAAK;GACL;GACA,WAAW;EACb,CAAC;EACD,KAAKE,kBAAkB;CACzB;;;;;;;;;;;CAYA,OAAO,MAAc,QAAkC;EACrD,MAAM,YAAY,KAAKC,kBAAkB;EACzC,MAAM,EAAC,QAAO;EACd,MAAM,SAAS,uBAAuB,MAAM,GAAG;EAC/C,MAAM,iBAAiB,gCACrB,WACA,KACA,MACF;EACA,KAAKP,cAAc,IAAI,WAAW,EAAE,KAAKI,MAAM,KAAK,gBAAgB,MAAM;EAC1E,KAAKI,eAAe,EAAE,IAAI;GACxB;GACA,KAAK,KAAKJ;GACV;GACA,QAAQ;GACR,WAAW;EACb,CAAC;EACD,KAAKE,mBAAmB;EAExB,IAAI,eAAe,MAAM,GACvB,KAAK,MAAM,MAAM,KAAKJ,SAAS,MAAM,MAAM,GACzC,KAAKO,iBAAiB,KAAK,GAAG,EAAE;CAGtC;CAEA,OACE,WACA,MACA,aACiC;EACjC,MAAM,YAAY,KAAKF,kBAAkB;EACzC,gBACE,cAAc,WACd,4BAA4B,UAAU,wBAAwB,WAChE;EACA,MAAM,SAAS,uBAAuB,MAAM,QAAQ;EACpD,MAAM,iBAAiB,gCACrB,WACA,UACA,QACA,WACA,IACF;EACA,KAAKN,cAAc,IACjB,WACA,EAAE,KAAKG,MACP,gBACA,QACA,WACA,WACF;EACA,MAAM,SAAS,KAAKI,eAAe;EACnC,OAAO,IAAI;GACT;GACA,KAAK,KAAKJ;GACV,KAAK;GACL;GACA;EACF,CAAC;EACD,MAAM,QAAQ;GACZ,MAAM,KAAKA,OAAO;GAClB,gBAAgB,KAAKE,kBAAkB;GACvC,MAAM,OAAO,OAAO;GACpB,iBAAiB,KAAKG;EACxB;EAKA,KAAKV,IAAI,OAAO;EAChB,KAAKW,OAAO;EACZ,OAAO;CACT;;;;;;CAOA,WAAiB;EACf,MAAM,EAAC,kBAAiB,KAAKX,IAAI;EACjC,KAAKW,OAAO;EACZ,IAAI,eACF,KAAKX,IAAI,SAAS;CAEtB;CAEA,SAAe;EACb,KAAKI,aAAa,KAAA;EAClB,KAAKC,OAAO;EACZ,KAAKE,kBAAkB;EACvB,KAAKD,UAAU,KAAA;EAGf,KAAKI,mBAAmB,CAAC;CAC3B;CAEA,oBAA4B;EAC1B,gBACE,KAAKN,eAAe,KAAA,GACpB,6DACF;EACA,OAAO,KAAKA;CACd;CAEA,iBAA6C;EAC3C,gBACE,KAAKE,YAAY,KAAA,GACjB,0DACF;EACA,OAAO,KAAKA;CACd;AACF"}
|
|
1
|
+
{"version":3,"file":"change-log-stream-writer.js","names":["#db","#insertChange","#insertCommit","#cookies","#watermark","#pos","#hasher","#estimatedBytes","#requireWatermark","#cookieMutations","#reset"],"sources":["../../../../../../zero-cache/src/services/replicator/change-log-stream-writer.ts"],"sourcesContent":["import type {Statement} from '../../../../zqlite/src/db.ts';\nimport type {StatementRunner} from '../../db/statements.ts';\nimport {\n isSchemaChange,\n type DataOrSchemaChange,\n} from '../change-source/protocol/current/data.ts';\nimport {extractChangeSubstring} from '../change-streamer/change-log-codec.ts';\nimport {ChangeLogTransactionHasher} from '../change-streamer/change-log-transaction-hash.ts';\nimport {ChangeLogCookieWriter, type CookieOpTag} from './change-log-cookies.ts';\nimport {\n CHANGE_LOG_STREAM_TABLE,\n estimateChangeLogStreamRowBytes,\n} from './change-log-db.ts';\n\nexport type ChangeLogStreamTransactionStats = {\n readonly rows: number;\n readonly estimatedBytes: number;\n /** Present only for transactions selected for shadow-write validation. */\n readonly hash?: string | undefined;\n /**\n * The cookie mutations this transaction's schema changes folded in, in stream\n * order. Empty for the overwhelming majority of transactions.\n */\n readonly cookieMutations: readonly CookieOpTag[];\n};\n\nexport class ChangeLogStreamInvariantError extends Error {\n override readonly name = 'ChangeLogStreamInvariantError';\n}\n\nfunction assertInvariant(\n condition: unknown,\n message: string,\n): asserts condition {\n if (!condition) {\n throw new ChangeLogStreamInvariantError(message);\n }\n}\n\n/**\n * Appends the canonical downstream stream to the change-log database, which is\n * a separate file from the replica, and folds its schema changes into the log's\n * cookie jar (`change-log-cookies.ts`) as it goes.\n *\n * This class owns the change log's transaction, and its transaction alone. It is\n * driven from the change-streamer's stream loop, which commits it before\n * forwarding a transaction's `commit` message; see `SQLiteChangeLogWriter` in\n * `change-streamer/sqlite-change-log-writer.ts` for the ordering that makes a\n * hole unreachable, and for the fail-soft policy that wraps every call here.\n *\n * Because the cookies are written in that same transaction, they are atomic with\n * the head and {@link rollback} discards them with the rows they were folded\n * from. Neither needs any handling of its own.\n */\nexport class ChangeLogStreamWriter {\n readonly #db: StatementRunner;\n readonly #insertChange: Statement;\n readonly #insertCommit: Statement;\n readonly #cookies: ChangeLogCookieWriter;\n\n #watermark: string | undefined;\n #pos = 0;\n #estimatedBytes = 0;\n #hasher: ChangeLogTransactionHasher | undefined;\n #cookieMutations: CookieOpTag[] = [];\n\n constructor(db: StatementRunner) {\n this.#db = db;\n this.#insertChange = db.db.prepare(/*sql*/ `\n INSERT INTO \"${CHANGE_LOG_STREAM_TABLE}\"\n (\"watermark\", \"pos\", \"tag\", \"estimatedBytes\", \"change\")\n VALUES (?, ?, ?, ?, ?)\n `);\n this.#insertCommit = db.db.prepare(/*sql*/ `\n INSERT INTO \"${CHANGE_LOG_STREAM_TABLE}\"\n (\"watermark\", \"pos\", \"tag\", \"estimatedBytes\", \"change\", \"precommit\",\n \"writeTimeMs\")\n VALUES (?, ?, 'commit', ?, ?, ?, ?)\n `);\n this.#cookies = new ChangeLogCookieWriter(db.db);\n }\n\n begin(\n watermark: string,\n json: string,\n storedChange = extractChangeSubstring(json, 'begin'),\n hash = true,\n ): void {\n assertInvariant(\n this.#watermark === undefined,\n `change-log stream transaction already open at ${this.#watermark}`,\n );\n // This process is the sole writer of the change-log database, so the write\n // lock is taken up front rather than risking a failed upgrade from a\n // deferred transaction partway through the append.\n this.#db.beginImmediate();\n this.#watermark = watermark;\n this.#pos = 0;\n const change = storedChange;\n const estimatedBytes = estimateChangeLogStreamRowBytes(\n watermark,\n 'begin',\n change,\n );\n this.#insertChange.run(\n watermark,\n this.#pos,\n 'begin',\n estimatedBytes,\n change,\n );\n if (hash) {\n this.#hasher = new ChangeLogTransactionHasher();\n this.#hasher.add({\n watermark,\n pos: this.#pos,\n tag: 'begin',\n change,\n precommit: null,\n });\n }\n this.#estimatedBytes = estimatedBytes;\n }\n\n /**\n * Appends one data or schema change. The parsed message is taken rather than\n * just its tag because a schema change also mutates the cookie jar, and the\n * caller already holds it — so the fold costs no re-parse.\n *\n * The cookie statements run here, i.e. inside this transaction and in stream\n * order relative to the rows they were folded from. The Postgres change log\n * has to flush its buffered batch to get the same ordering\n * (`Storer.#processQueue()`); SQLite gets it from statement order.\n */\n append(\n json: string,\n change: DataOrSchemaChange,\n storedChange = extractChangeSubstring(json, change.tag),\n ): void {\n const watermark = this.#requireWatermark();\n const {tag} = change;\n const stored = storedChange;\n const estimatedBytes = estimateChangeLogStreamRowBytes(\n watermark,\n tag,\n stored,\n );\n this.#insertChange.run(watermark, ++this.#pos, tag, estimatedBytes, stored);\n this.#hasher?.add({\n watermark,\n pos: this.#pos,\n tag,\n change: stored,\n precommit: null,\n });\n this.#estimatedBytes += estimatedBytes;\n\n if (isSchemaChange(change)) {\n for (const op of this.#cookies.apply(change)) {\n this.#cookieMutations.push(op.op);\n }\n }\n }\n\n commit(\n watermark: string,\n json: string,\n writeTimeMs: number,\n storedChange = extractChangeSubstring(json, 'commit'),\n ): ChangeLogStreamTransactionStats {\n const precommit = this.#requireWatermark();\n assertInvariant(\n watermark === precommit,\n `change-log stream commit ${watermark} does not match begin ${precommit}`,\n );\n const change = storedChange;\n const estimatedBytes = estimateChangeLogStreamRowBytes(\n watermark,\n 'commit',\n change,\n precommit,\n true,\n );\n this.#insertCommit.run(\n watermark,\n ++this.#pos,\n estimatedBytes,\n change,\n precommit,\n writeTimeMs,\n );\n const hasher = this.#hasher;\n hasher?.add({\n watermark,\n pos: this.#pos,\n tag: 'commit',\n change,\n precommit,\n });\n const stats = {\n rows: this.#pos + 1,\n estimatedBytes: this.#estimatedBytes + estimatedBytes,\n hash: hasher?.digest(),\n cookieMutations: this.#cookieMutations,\n };\n // The log is durable at `watermark` from here on, and nothing that can\n // advance the watermark the stream would resume from has run yet. A crash in\n // that window leaves a transaction the resumed stream re-delivers, which\n // reconciliation truncates.\n this.#db.commit();\n this.#reset();\n return stats;\n }\n\n /**\n * Discards the open transaction, if any. Safe to call when none is open — the\n * caller rolls back on an interrupted change stream, which can arrive between\n * transactions as well as inside one.\n */\n rollback(): void {\n const {inTransaction} = this.#db.db;\n this.#reset();\n if (inTransaction) {\n this.#db.rollback();\n }\n }\n\n #reset(): void {\n this.#watermark = undefined;\n this.#pos = 0;\n this.#estimatedBytes = 0;\n this.#hasher = undefined;\n // A fresh array rather than a truncation: the committed stats hold a\n // reference to the previous one.\n this.#cookieMutations = [];\n }\n\n #requireWatermark(): string {\n assertInvariant(\n this.#watermark !== undefined,\n 'change-log stream message received outside of a transaction',\n );\n return this.#watermark;\n }\n}\n"],"mappings":";;;;;;AA0BA,IAAa,gCAAb,cAAmD,MAAM;CACvD,OAAyB;AAC3B;AAEA,SAAS,gBACP,WACA,SACmB;CACnB,IAAI,CAAC,WACH,MAAM,IAAI,8BAA8B,OAAO;AAEnD;;;;;;;;;;;;;;;;AAiBA,IAAa,wBAAb,MAAmC;CACjC;CACA;CACA;CACA;CAEA;CACA,OAAO;CACP,kBAAkB;CAClB;CACA,mBAAkC,CAAC;CAEnC,YAAY,IAAqB;EAC/B,KAAKA,MAAM;EACX,KAAKC,gBAAgB,GAAG,GAAG,QAAgB;qBAC1B,wBAAwB;;;KAGxC;EACD,KAAKC,gBAAgB,GAAG,GAAG,QAAgB;qBAC1B,wBAAwB;;;;KAIxC;EACD,KAAKC,WAAW,IAAI,sBAAsB,GAAG,EAAE;CACjD;CAEA,MACE,WACA,MACA,eAAe,uBAAuB,MAAM,OAAO,GACnD,OAAO,MACD;EACN,gBACE,KAAKC,eAAe,KAAA,GACpB,iDAAiD,KAAKA,YACxD;EAIA,KAAKJ,IAAI,eAAe;EACxB,KAAKI,aAAa;EAClB,KAAKC,OAAO;EACZ,MAAM,SAAS;EACf,MAAM,iBAAiB,gCACrB,WACA,SACA,MACF;EACA,KAAKJ,cAAc,IACjB,WACA,KAAKI,MACL,SACA,gBACA,MACF;EACA,IAAI,MAAM;GACR,KAAKC,UAAU,IAAI,2BAA2B;GAC9C,KAAKA,QAAQ,IAAI;IACf;IACA,KAAK,KAAKD;IACV,KAAK;IACL;IACA,WAAW;GACb,CAAC;EACH;EACA,KAAKE,kBAAkB;CACzB;;;;;;;;;;;CAYA,OACE,MACA,QACA,eAAe,uBAAuB,MAAM,OAAO,GAAG,GAChD;EACN,MAAM,YAAY,KAAKC,kBAAkB;EACzC,MAAM,EAAC,QAAO;EACd,MAAM,SAAS;EACf,MAAM,iBAAiB,gCACrB,WACA,KACA,MACF;EACA,KAAKP,cAAc,IAAI,WAAW,EAAE,KAAKI,MAAM,KAAK,gBAAgB,MAAM;EAC1E,KAAKC,SAAS,IAAI;GAChB;GACA,KAAK,KAAKD;GACV;GACA,QAAQ;GACR,WAAW;EACb,CAAC;EACD,KAAKE,mBAAmB;EAExB,IAAI,eAAe,MAAM,GACvB,KAAK,MAAM,MAAM,KAAKJ,SAAS,MAAM,MAAM,GACzC,KAAKM,iBAAiB,KAAK,GAAG,EAAE;CAGtC;CAEA,OACE,WACA,MACA,aACA,eAAe,uBAAuB,MAAM,QAAQ,GACnB;EACjC,MAAM,YAAY,KAAKD,kBAAkB;EACzC,gBACE,cAAc,WACd,4BAA4B,UAAU,wBAAwB,WAChE;EACA,MAAM,SAAS;EACf,MAAM,iBAAiB,gCACrB,WACA,UACA,QACA,WACA,IACF;EACA,KAAKN,cAAc,IACjB,WACA,EAAE,KAAKG,MACP,gBACA,QACA,WACA,WACF;EACA,MAAM,SAAS,KAAKC;EACpB,QAAQ,IAAI;GACV;GACA,KAAK,KAAKD;GACV,KAAK;GACL;GACA;EACF,CAAC;EACD,MAAM,QAAQ;GACZ,MAAM,KAAKA,OAAO;GAClB,gBAAgB,KAAKE,kBAAkB;GACvC,MAAM,QAAQ,OAAO;GACrB,iBAAiB,KAAKE;EACxB;EAKA,KAAKT,IAAI,OAAO;EAChB,KAAKU,OAAO;EACZ,OAAO;CACT;;;;;;CAOA,WAAiB;EACf,MAAM,EAAC,kBAAiB,KAAKV,IAAI;EACjC,KAAKU,OAAO;EACZ,IAAI,eACF,KAAKV,IAAI,SAAS;CAEtB;CAEA,SAAe;EACb,KAAKI,aAAa,KAAA;EAClB,KAAKC,OAAO;EACZ,KAAKE,kBAAkB;EACvB,KAAKD,UAAU,KAAA;EAGf,KAAKG,mBAAmB,CAAC;CAC3B;CAEA,oBAA4B;EAC1B,gBACE,KAAKL,eAAe,KAAA,GACpB,6DACF;EACA,OAAO,KAAKA;CACd;AACF"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"change-processor.js","names":["#db","#changeLog","#tableMetadata","#backfilling","#mode","#failService","#tableSpecs","#failure","#currentTx","#fail","#processMessage","#beginTransaction","#lc","#startMs","#version","#jsonFormat","#columnMetadata","#reloadTableSpecs","#tableSpec","#upsert","#getKey","#logSetOp","#logDeleteOp","#delete","#logTruncateOp","#logResetOp","#bumpVersions","#pos","#numChangeLogEntries","#schemaChanged","#completedBackfill"],"sources":["../../../../../../zero-cache/src/services/replicator/change-processor.ts"],"sourcesContent":["import {constants as bufferConstants} from 'node:buffer';\nimport type {LogContext} from '@rocicorp/logger';\nimport {SqliteError} from '@rocicorp/zero-sqlite3';\nimport {AbortError} from '../../../../shared/src/abort-error.ts';\nimport {assert, unreachable} from '../../../../shared/src/asserts.ts';\nimport {stringify} from '../../../../shared/src/bigint-json.ts';\nimport {must} from '../../../../shared/src/must.ts';\nimport {mapEntries} from '../../../../shared/src/objects.ts';\nimport type {DownloadStatus} from '../../../../zero-events/src/status.ts';\nimport {\n createLiteIndexStatement,\n createLiteTableStatement,\n liteColumnDef,\n} from '../../db/create.ts';\nimport {\n computeZqlSpecs,\n listTables,\n type LiteTableSpecWithReplicationStatus,\n} from '../../db/lite-tables.ts';\nimport {\n isArrayColumn,\n isEnumColumn,\n mapPostgresToLite,\n mapPostgresToLiteColumn,\n mapPostgresToLiteIndex,\n} from '../../db/pg-to-lite.ts';\nimport type {ColumnSpec} from '../../db/specs.ts';\nimport type {StatementRunner} from '../../db/statements.ts';\nimport type {LexiVersion} from '../../types/lexi-version.ts';\nimport {\n JSON_PARSED,\n liteRow,\n type JSONFormat,\n type LiteRow,\n type LiteRowKey,\n type LiteValueType,\n} from '../../types/lite.ts';\nimport {liteTableName} from '../../types/names.ts';\nimport {id} from '../../types/sql.ts';\nimport type {\n BackfillCompleted,\n Change,\n ColumnAdd,\n ColumnDrop,\n ColumnUpdate,\n Identifier,\n IndexCreate,\n IndexDrop,\n MessageBackfill,\n MessageCommit,\n MessageDelete,\n MessageInsert,\n MessageRelation,\n MessageTruncate,\n MessageUpdate,\n TableCreate,\n TableDrop,\n TableRename,\n TableUpdateMetadata,\n} from '../change-source/protocol/current/data.ts';\nimport type {ChangeStreamData} from '../change-source/protocol/current/downstream.ts';\nimport type {ReplicatorMode} from './replicator.ts';\nimport {BackfillingTracker} from './schema/backfilling.ts';\nimport {ChangeLog, DEL_OP, SET_OP} from './schema/change-log.ts';\nimport {ColumnMetadataStore} from './schema/column-metadata.ts';\nimport {\n ZERO_VERSION_COLUMN_NAME,\n updateReplicationWatermark,\n} from './schema/replication-state.ts';\nimport {TableMetadataTracker} from './schema/table-metadata.ts';\n\nexport type ChangeProcessorMode = ReplicatorMode | 'initial-sync';\n\nconst BIND_VALUE_TOO_BIG_ERROR =\n 'The bound string, buffer, or bigint is too big';\nconst MAX_SQLITE_BIND_BYTES = Math.min(\n bufferConstants.MAX_LENGTH,\n bufferConstants.MAX_STRING_LENGTH,\n);\n\nexport type CommitResult = {\n watermark: string;\n completedBackfill: DownloadStatus | undefined;\n schemaUpdated: boolean;\n changeLogUpdated: boolean;\n /**\n * Millisecond epoch at which the transaction committed upstream, if the\n * ChangeSource reported one. Propagated to ViewSyncers as the origin\n * timestamp of the end-to-end serving lag measurement.\n */\n upstreamCommitTimeMs?: number | undefined;\n};\n\n/**\n * The ChangeProcessor partitions the stream of messages into transactions\n * by creating a {@link TransactionProcessor} when a transaction begins, and dispatching\n * messages to it until the commit is received.\n *\n * From https://www.postgresql.org/docs/current/protocol-logical-replication.html#PROTOCOL-LOGICAL-MESSAGES-FLOW :\n *\n * \"The logical replication protocol sends individual transactions one by one.\n * This means that all messages between a pair of Begin and Commit messages\n * belong to the same transaction.\"\n */\nexport class ChangeProcessor {\n readonly #db: StatementRunner;\n readonly #changeLog: ChangeLog;\n readonly #tableMetadata: TableMetadataTracker;\n readonly #backfilling: BackfillingTracker;\n readonly #mode: ChangeProcessorMode;\n readonly #failService: (lc: LogContext, err: unknown) => void;\n\n // The TransactionProcessor lazily loads table specs into this Map,\n // and reloads them after a schema change. It is cached here to avoid\n // reading them from the DB on every transaction.\n readonly #tableSpecs = new Map<string, LiteTableSpecWithReplicationStatus>();\n\n #currentTx: TransactionProcessor | null = null;\n\n #failure: Error | undefined;\n\n constructor(\n db: StatementRunner,\n mode: ChangeProcessorMode,\n failService: (lc: LogContext, err: unknown) => void,\n ) {\n this.#db = db;\n this.#changeLog = new ChangeLog(db.db);\n this.#tableMetadata = new TableMetadataTracker(db.db);\n this.#backfilling = new BackfillingTracker(db.db);\n this.#mode = mode;\n this.#failService = failService;\n }\n\n #fail(lc: LogContext, err: unknown) {\n if (!this.#failure) {\n let failureError = err;\n const rollbackErrors = [attempt(() => this.#currentTx?.abort(lc))].filter(\n e => e !== undefined,\n );\n\n if (rollbackErrors.length) {\n const combinedError = new Error(\n `Message processing failed and rollback also failed: operation error = ${String(err)}; rollback error = ${rollbackErrors.map(String).join('; ')}`,\n );\n combinedError.cause = err;\n failureError = combinedError;\n }\n\n this.#failure = ensureError(failureError);\n\n if (!(this.#failure instanceof AbortError)) {\n // Propagate the failure up to the service.\n lc.error?.('Message Processing failed:', this.#failure);\n this.#failService(lc, this.#failure);\n }\n }\n }\n\n abort(lc: LogContext) {\n this.#fail(lc, new AbortError());\n }\n\n /** @return If a transaction was committed. */\n processMessage(\n lc: LogContext,\n downstream: ChangeStreamData,\n ): CommitResult | null {\n const [type, message] = downstream;\n if (this.#failure) {\n lc.debug?.(`Dropping ${message.tag}`);\n return null;\n }\n try {\n const watermark =\n type === 'begin'\n ? downstream[2].commitWatermark\n : type === 'commit'\n ? downstream[2].watermark\n : undefined;\n return this.#processMessage(lc, message, watermark);\n } catch (e) {\n this.#fail(lc, e);\n }\n return null;\n }\n\n #beginTransaction(\n lc: LogContext,\n commitVersion: string,\n jsonFormat: JSONFormat,\n ): TransactionProcessor {\n const start = Date.now();\n\n // litestream can technically hold the lock for an arbitrary amount of time\n // when checkpointing a large commit. Crashing on the busy-timeout in this\n // scenario will either produce a corrupt backup or otherwise prevent\n // replication from proceeding.\n //\n // Instead, retry the lock acquisition indefinitely. If this masks\n // an unknown deadlock situation, manual intervention will be necessary.\n for (let i = 0; ; i++) {\n try {\n return new TransactionProcessor(\n lc,\n this.#db,\n this.#mode,\n this.#changeLog,\n this.#tableMetadata,\n this.#backfilling,\n this.#tableSpecs,\n commitVersion,\n jsonFormat,\n );\n } catch (e) {\n if (e instanceof SqliteError && e.code === 'SQLITE_BUSY') {\n lc.warn?.(\n `SQLITE_BUSY for ${Date.now() - start} ms (attempt ${i + 1}). ` +\n `This is only expected if litestream is performing a large ` +\n `checkpoint.`,\n e,\n );\n continue;\n }\n throw e;\n }\n }\n }\n\n /** @return If a transaction was committed. */\n #processMessage(\n lc: LogContext,\n msg: Change,\n watermark: string | undefined,\n ): CommitResult | null {\n if (msg.tag === 'begin') {\n if (this.#currentTx) {\n throw new Error(`Already in a transaction ${stringify(msg)}`);\n }\n this.#currentTx = this.#beginTransaction(\n lc,\n must(watermark),\n msg.json ?? JSON_PARSED,\n );\n return null;\n }\n\n // For non-begin messages, there should be a #currentTx set.\n const tx = this.#currentTx;\n if (!tx) {\n throw new Error(\n `Received message outside of transaction: ${stringify(msg)}`,\n );\n }\n\n if (msg.tag === 'commit') {\n assert(watermark, 'watermark is required for commit messages');\n const result = tx.processCommit(msg, watermark);\n // Clear only after a successful commit so #fail can roll back a commit\n // path that throws before SQLite has committed.\n this.#currentTx = null;\n return result;\n }\n\n if (msg.tag === 'rollback') {\n tx.abort(lc);\n this.#currentTx = null;\n return null;\n }\n\n switch (msg.tag) {\n case 'insert':\n tx.processInsert(msg);\n break;\n case 'update':\n tx.processUpdate(msg);\n break;\n case 'delete':\n tx.processDelete(msg);\n break;\n case 'truncate':\n tx.processTruncate(msg);\n break;\n case 'create-table':\n tx.processCreateTable(msg);\n break;\n case 'rename-table':\n tx.processRenameTable(msg);\n break;\n case 'update-table-metadata':\n tx.processTableMetadata(msg);\n break;\n case 'add-column':\n tx.processAddColumn(msg);\n break;\n case 'update-column':\n tx.processUpdateColumn(msg);\n break;\n case 'drop-column':\n tx.processDropColumn(msg);\n break;\n case 'drop-table':\n tx.processDropTable(msg);\n break;\n case 'create-index':\n tx.processCreateIndex(msg);\n break;\n case 'drop-index':\n tx.processDropIndex(msg);\n break;\n case 'backfill':\n tx.processBackfill(msg);\n break;\n case 'backfill-completed':\n tx.processBackfillCompleted(msg);\n break;\n default:\n unreachable(msg);\n }\n\n return null;\n }\n}\n\n/**\n * The {@link TransactionProcessor} handles the sequence of messages from\n * upstream, from `BEGIN` to `COMMIT` and executes the corresponding mutations\n * on the {@link postgres.TransactionSql} on the replica.\n *\n * When applying row contents to the replica, the `_0_version` column is added / updated,\n * and a corresponding entry in the `ChangeLog` is added. The version value is derived\n * from the watermark of the preceding transaction (stored as the `nextStateVersion` in the\n * `ReplicationState` table).\n *\n * Side note: For non-streaming Postgres transactions, the commitEndLsn (and thus\n * commit watermark) is available in the `begin` message, so it could theoretically\n * be used for the row version of changes within the transaction. However, the\n * commitEndLsn is not available in the streaming (in-progress) transaction\n * protocol, and may not be available for CDC streams of other upstream types.\n * Therefore, the zero replication protocol is designed to not require the commit\n * watermark when a transaction begins.\n *\n * Also of interest is the fact that all INSERT Messages are logically applied as\n * UPSERTs. See {@link processInsert} for the underlying motivation.\n */\nclass TransactionProcessor {\n readonly #lc: LogContext;\n readonly #startMs: number;\n readonly #db: StatementRunner;\n readonly #mode: ChangeProcessorMode;\n readonly #version: LexiVersion;\n readonly #changeLog: ChangeLog;\n readonly #tableMetadata: TableMetadataTracker;\n readonly #backfilling: BackfillingTracker;\n readonly #tableSpecs: Map<string, LiteTableSpecWithReplicationStatus>;\n readonly #jsonFormat: JSONFormat;\n readonly #columnMetadata: ColumnMetadataStore;\n\n #pos = 0;\n #schemaChanged = false;\n #numChangeLogEntries = 0;\n\n constructor(\n lc: LogContext,\n db: StatementRunner,\n mode: ChangeProcessorMode,\n changeLog: ChangeLog,\n tableMetadata: TableMetadataTracker,\n backfilling: BackfillingTracker,\n tableSpecs: Map<string, LiteTableSpecWithReplicationStatus>,\n commitVersion: LexiVersion,\n jsonFormat: JSONFormat,\n ) {\n this.#startMs = Date.now();\n this.#mode = mode;\n this.#jsonFormat = jsonFormat;\n\n switch (mode) {\n case 'serving':\n // This is the only transaction that commits to the serving replica.\n // Snapshotters use BEGIN CONCURRENT for private changes that are\n // always rolled back, while BEGIN IMMEDIATE lets this writer spill\n // dirty pages during large transactions.\n db.beginImmediate();\n break;\n case 'backup':\n // For the backup-replicator (i.e. replication-manager), there are no View Syncers\n // and thus BEGIN CONCURRENT is not necessary. In fact, BEGIN CONCURRENT can cause\n // deadlocks with forced wal-checkpoints (which `litestream replicate` performs),\n // so it is important to use vanilla transactions in this configuration.\n db.beginImmediate();\n break;\n case 'initial-sync':\n // When the ChangeProcessor is used for initial-sync, the calling code\n // handles the transaction boundaries.\n break;\n default:\n unreachable();\n }\n this.#db = db;\n this.#version = commitVersion;\n this.#lc = lc.withContext('version', commitVersion);\n this.#changeLog = changeLog;\n this.#tableMetadata = tableMetadata;\n this.#backfilling = backfilling;\n this.#tableSpecs = tableSpecs;\n // The column_metadata table is guaranteed to exist since the\n // replica-schema.ts migration to v8.\n this.#columnMetadata = must(ColumnMetadataStore.getInstance(db.db));\n\n if (this.#tableSpecs.size === 0) {\n this.#reloadTableSpecs();\n }\n }\n\n #reloadTableSpecs() {\n this.#tableSpecs.clear();\n // zqlSpecs include the primary key derived from unique indexes\n const zqlSpecs = computeZqlSpecs(this.#lc, this.#db.db, {\n includeBackfillingColumns: true,\n });\n for (let spec of listTables(this.#db.db)) {\n if (!spec.primaryKey) {\n spec = {\n ...spec,\n primaryKey: [\n ...(zqlSpecs.get(spec.name)?.tableSpec.primaryKey ?? []),\n ],\n };\n }\n this.#tableSpecs.set(spec.name, spec);\n }\n }\n\n #tableSpec(name: string) {\n return must(this.#tableSpecs.get(name), `Unknown table ${name}`);\n }\n\n #getKey(\n {row, numCols}: {row: LiteRow; numCols: number},\n {relation}: {relation: MessageRelation},\n ): LiteRowKey {\n const keyColumns =\n relation.rowKey.type !== 'full'\n ? relation.rowKey.columns // already a suitable key\n : this.#tableSpec(liteTableName(relation)).primaryKey;\n if (!keyColumns?.length) {\n throw new Error(\n `Cannot replicate table \"${relation.name}\" without a PRIMARY KEY or UNIQUE INDEX`,\n );\n }\n // For the common case (replica identity default), the row is already the\n // key for deletes and updates, in which case a new object can be avoided.\n if (numCols === keyColumns.length) {\n return row;\n }\n const key: Record<string, LiteValueType> = {};\n for (const col of keyColumns) {\n key[col] = row[col];\n }\n return key;\n }\n\n processInsert(insert: MessageInsert) {\n const table = liteTableName(insert.relation);\n const tableSpec = this.#tableSpec(table);\n const newRow = liteRow(insert.new, tableSpec, this.#jsonFormat);\n\n this.#upsert(table, {\n ...newRow.row,\n [ZERO_VERSION_COLUMN_NAME]: this.#version,\n });\n\n if (insert.relation.rowKey.columns.length === 0) {\n // INSERTs can be replicated for rows without a PRIMARY KEY or a\n // UNIQUE INDEX. These are written to the replica but not recorded\n // in the changeLog, because these rows cannot participate in IVM.\n //\n // (Once the table schema has been corrected to include a key, the\n // associated schema change will reset pipelines and data can be\n // loaded via hydration.)\n return;\n }\n const key = this.#getKey(newRow, insert);\n this.#logSetOp(table, key, getBackfilledColumns(newRow.row, tableSpec));\n }\n\n #upsert(table: string, row: LiteRow) {\n const columns = Object.keys(row).map(c => id(c));\n this.#db.run(\n `\n INSERT OR REPLACE INTO ${id(table)} (${columns.join(',')})\n VALUES (${Array.from({length: columns.length}).fill('?').join(',')})\n `,\n Object.values(row),\n );\n }\n\n // Updates by default are applied as UPDATE commands to support partial\n // row specifications from the change source. In particular, this is needed\n // to handle updates for which unchanged TOASTed values are not sent:\n //\n // https://www.postgresql.org/docs/current/protocol-logicalrep-message-formats.html#PROTOCOL-LOGICALREP-MESSAGE-FORMATS-TUPLEDATA\n //\n // However, in certain cases an UPDATE may be received for a row that\n // was not initially synced, such as when, an existing table is added\n // to the app's publication.\n //\n // In order to facilitate \"resumptive\" replication, the logic falls back to\n // an INSERT if the update did not change any rows.\n processUpdate(update: MessageUpdate) {\n const table = liteTableName(update.relation);\n const tableSpec = this.#tableSpec(table);\n const newRow = liteRow(update.new, tableSpec, this.#jsonFormat);\n const row = {...newRow.row, [ZERO_VERSION_COLUMN_NAME]: this.#version};\n\n // update.key is set with the old values if the key has changed.\n const oldKey = update.key\n ? this.#getKey(\n liteRow(update.key, this.#tableSpec(table), this.#jsonFormat),\n update,\n )\n : null;\n const newKey = this.#getKey(newRow, update);\n\n if (oldKey) {\n this.#logDeleteOp(table, oldKey, tableSpec.backfilling);\n }\n this.#logSetOp(table, newKey, getBackfilledColumns(newRow.row, tableSpec));\n\n const currKey = oldKey ?? newKey;\n const conds = Object.keys(currKey).map(col => `${id(col)}=?`);\n const setExprs = Object.keys(row).map(col => `${id(col)}=?`);\n\n try {\n const {changes} = this.#db.run(\n `\n UPDATE ${id(table)}\n SET ${setExprs.join(',')}\n WHERE ${conds.join(' AND ')}\n `,\n [...Object.values(row), ...Object.values(currKey)],\n );\n\n // If the UPDATE did not affect any rows, perform an UPSERT of the\n // new row for resumptive replication.\n if (changes === 0) {\n this.#upsert(table, row);\n }\n } catch (e) {\n if (\n !(e instanceof RangeError) ||\n e.message !== BIND_VALUE_TOO_BIG_ERROR\n ) {\n throw e;\n }\n\n const binding = [...Object.entries(row), ...Object.entries(currKey)]\n .map(\n ([column, value]) =>\n [column, oversizedValueDescription(value)] as const,\n )\n .find(([, description]) => description);\n const relationOid =\n 'relationOid' in update.relation &&\n typeof update.relation.relationOid === 'number'\n ? ` relationOid=${update.relation.relationOid}`\n : '';\n const error = new Error(\n `Oversized SQLite update binding: tx=${this.#version}${relationOid} table=${update.relation.schema}.${update.relation.name} column=${binding?.[0] ?? 'unknown'}${binding?.[1] ? ` ${binding[1]}` : ''}`,\n {cause: e},\n );\n error.name = 'OversizedUpdateBindingError';\n throw error;\n }\n }\n\n processDelete(del: MessageDelete) {\n const table = liteTableName(del.relation);\n const tableSpec = this.#tableSpec(table);\n const rowKey = this.#getKey(\n liteRow(del.key, tableSpec, this.#jsonFormat),\n del,\n );\n\n this.#delete(table, rowKey);\n this.#logDeleteOp(table, rowKey, tableSpec.backfilling);\n }\n\n #delete(table: string, rowKey: LiteRowKey) {\n const conds = Object.keys(rowKey).map(col => `${id(col)}=?`);\n this.#db.run(\n `DELETE FROM ${id(table)} WHERE ${conds.join(' AND ')}`,\n Object.values(rowKey),\n );\n }\n\n processTruncate(truncate: MessageTruncate) {\n for (const relation of truncate.relations) {\n const table = liteTableName(relation);\n // Update replica data.\n this.#db.run(`DELETE FROM ${id(table)}`);\n\n // Update change log.\n this.#logTruncateOp(table);\n }\n }\n\n processCreateTable(create: TableCreate) {\n if (create.metadata) {\n this.#tableMetadata.setUpstreamMetadata(create.spec, create.metadata);\n }\n this.#backfilling.apply(create);\n const table = mapPostgresToLite(create.spec);\n this.#db.db.exec(createLiteTableStatement(table));\n\n // Write to metadata table\n for (const [colName, colSpec] of Object.entries(create.spec.columns)) {\n this.#columnMetadata.insert(\n table.name,\n colName,\n colSpec,\n create.backfill?.[colName],\n );\n }\n\n this.#logResetOp(table.name);\n this.#lc.info?.(create.tag, table.name);\n }\n\n processTableMetadata(msg: TableUpdateMetadata) {\n this.#tableMetadata.setUpstreamMetadata(msg.table, msg.new);\n // Inert today: the fold's only op for this tag is `upsert-metadata`, which\n // the line above is the replica's interpreter of. Called anyway so that\n // every site that hands a schema change to one cookie tracker hands it to\n // both -- an exception here would be a schema change the shared fold has an\n // opinion about and this store never sees, which is the drift the fold\n // exists to make impossible.\n this.#backfilling.apply(msg);\n }\n\n processRenameTable(rename: TableRename) {\n this.#tableMetadata.rename(rename.old, rename.new);\n this.#backfilling.apply(rename);\n\n const oldName = liteTableName(rename.old);\n const newName = liteTableName(rename.new);\n this.#db.db.exec(`ALTER TABLE ${id(oldName)} RENAME TO ${id(newName)}`);\n\n // Rename in metadata table\n this.#columnMetadata.renameTable(oldName, newName);\n\n this.#bumpVersions(rename.new);\n this.#logResetOp(oldName);\n this.#lc.info?.(rename.tag, oldName, newName);\n }\n\n processAddColumn(msg: ColumnAdd) {\n if (msg.tableMetadata) {\n this.#tableMetadata.setUpstreamMetadata(msg.table, msg.tableMetadata);\n }\n this.#backfilling.apply(msg);\n const table = liteTableName(msg.table);\n const {name} = msg.column;\n const spec = mapPostgresToLiteColumn(table, msg.column);\n this.#db.db.exec(\n `ALTER TABLE ${id(table)} ADD ${id(name)} ${liteColumnDef(spec)}`,\n );\n\n // Write to metadata table\n this.#columnMetadata.insert(table, name, msg.column.spec, msg.backfill);\n\n if (msg.backfill) {\n this.#reloadTableSpecs();\n } else {\n // Make the new column visible immediately if it's not being backfilled.\n // Otherwise, the version bump will happen with the backfill is complete.\n this.#bumpVersions(msg.table);\n }\n this.#lc.info?.(msg.tag, table, msg.column);\n }\n\n processUpdateColumn(msg: ColumnUpdate) {\n // A no-op unless this is a rename, which is the only part of a column\n // update that moves the backfill cookie.\n this.#backfilling.apply(msg);\n\n const table = liteTableName(msg.table);\n let oldName = msg.old.name;\n const newName = msg.new.name;\n\n const storageTypesDiffer = differentStorageTypes(\n msg.old.spec,\n msg.new.spec,\n );\n\n // If neither the column name nor the SQLite data type changes, only the\n // upstream metadata needs to be updated. This includes changes such as a\n // varchar character limit and nullability, which SQLite does not enforce\n // but a freshly built replica still records.\n if (oldName === newName && !storageTypesDiffer) {\n this.#columnMetadata.update(\n table,\n msg.old.name,\n msg.new.name,\n msg.new.spec,\n );\n if (Boolean(msg.old.spec.notNull) !== Boolean(msg.new.spec.notNull)) {\n this.#bumpVersions(msg.table);\n }\n this.#lc.info?.(msg.tag, 'updated metadata only', msg.old, msg.new);\n return;\n }\n // Storage type changes require a new column so that SQLite uses the new\n // representation for existing and future values.\n if (storageTypesDiffer) {\n // update-column can ignore defaults because it does not change the values\n // in existing rows.\n //\n // https://www.postgresql.org/docs/current/sql-altertable.html#SQL-ALTERTABLE-DESC-SET-DROP-DEFAULT\n //\n // \"The new default value will only apply in subsequent INSERT or UPDATE\n // commands; it does not cause rows already in the table to change.\"\n //\n // This allows support for _changing_ column defaults to any expression,\n // since it does not affect what the replica needs to do.\n const newLiteSpec = mapPostgresToLiteColumn(\n table,\n msg.new,\n 'ignore-default',\n );\n if (oldName !== newName) {\n // Rename the column in place first so that SQLite rewrites every\n // reference to it in the table's index definitions, including the\n // WHERE clauses of partial indexes.\n this.#db.db.exec(\n `ALTER TABLE ${id(table)} RENAME ${id(oldName)} TO ${id(newName)}`,\n );\n oldName = newName;\n }\n const tableSpec = must(\n listTables(this.#db.db, false, false).find(\n tableSpec => tableSpec.name === table,\n ),\n );\n // Recreating the table drops its indexes, so capture their DDL to\n // replay afterwards. Auto-indexes (PRIMARY KEY) have no DDL and are\n // recreated by the CREATE TABLE statement itself.\n const indexDDL = this.#db.db\n .prepare(\n `SELECT sql FROM sqlite_master\n WHERE type = 'index' AND tbl_name = ? AND sql IS NOT NULL\n ORDER BY rowid`,\n )\n .all<{sql: string}>(table)\n .map(({sql}) => sql);\n const tmpTable = `tmp.${table}`;\n const columns = Object.keys(tableSpec.columns);\n const stmts = [\n createLiteTableStatement({\n ...tableSpec,\n name: tmpTable,\n columns: mapEntries(tableSpec.columns, (column, spec) => [\n column,\n column === newName ? {...newLiteSpec, pos: spec.pos} : spec,\n ]),\n }),\n `INSERT INTO ${id(tmpTable)} (${columns.map(id).join(',')})\n SELECT ${columns.map(id).join(',')} FROM ${id(table)};`,\n `DROP TABLE ${id(table)};`,\n `ALTER TABLE ${id(tmpTable)} RENAME TO ${id(table)};`,\n ...indexDDL.map(sql => `${sql};`),\n ];\n this.#db.db.exec(stmts.join(''));\n }\n if (oldName !== newName) {\n this.#db.db.exec(\n `ALTER TABLE ${id(table)} RENAME ${id(oldName)} TO ${id(newName)}`,\n );\n }\n\n // Update metadata table\n this.#columnMetadata.update(\n table,\n msg.old.name,\n msg.new.name,\n msg.new.spec,\n );\n\n this.#bumpVersions(msg.table);\n this.#lc.info?.(msg.tag, table, msg.new);\n }\n\n processDropColumn(msg: ColumnDrop) {\n const table = liteTableName(msg.table);\n const {column} = msg;\n this.#db.db.exec(`ALTER TABLE ${id(table)} DROP ${id(column)}`);\n\n // Delete from metadata table\n this.#columnMetadata.deleteColumn(table, column);\n this.#backfilling.apply(msg);\n\n this.#bumpVersions(msg.table);\n this.#lc.info?.(msg.tag, table, column);\n }\n\n processDropTable(drop: TableDrop) {\n this.#tableMetadata.drop(drop.id);\n this.#backfilling.apply(drop);\n\n const name = liteTableName(drop.id);\n this.#db.db.exec(`DROP TABLE IF EXISTS ${id(name)}`);\n\n // Delete from metadata table\n this.#columnMetadata.deleteTable(name);\n\n this.#logResetOp(name);\n this.#lc.info?.(drop.tag, name);\n }\n\n processCreateIndex(create: IndexCreate) {\n const index = mapPostgresToLiteIndex(create.spec);\n this.#db.db.exec(createLiteIndexStatement(index));\n\n // indexes affect tables visibility (e.g. sync-ability is gated on\n // having a unique index), so reset pipelines to refresh table schemas.\n this.#logResetOp(index.tableName);\n this.#lc.info?.(create.tag, index.name);\n }\n\n processDropIndex(drop: IndexDrop) {\n const name = liteTableName(drop.id);\n this.#db.db.exec(`DROP INDEX IF EXISTS ${id(name)}`);\n this.#lc.info?.(drop.tag, name);\n }\n\n #bumpVersions(table: Identifier) {\n this.#tableMetadata.setMinRowVersion(table, this.#version);\n this.#logResetOp(liteTableName(table));\n }\n\n /**\n * @param backfilledColumns `backfilling` columns for which values were set\n */\n #logSetOp(\n table: string,\n key: LiteRowKey,\n backfilledColumns: string[] | undefined,\n ) {\n // The \"serving\" replicator always writes to the change-log (for IVM).\n // The \"backup\" replicator only needs to write to the change log\n // when writing columns that are being backfilled.\n if (this.#mode === 'serving' || backfilledColumns !== undefined) {\n this.#changeLog.logSetOp(\n this.#version,\n this.#pos++,\n table,\n key,\n backfilledColumns,\n );\n this.#numChangeLogEntries++;\n }\n }\n\n #logDeleteOp(table: string, key: LiteRowKey, backfilling?: string[]) {\n // The \"serving\" replicator always writes to the change-log (for IVM).\n // The \"backup\" replicator only needs to write to the change log\n // when writing columns that are being backfilled.\n if (this.#mode === 'serving' || backfilling?.length) {\n this.#changeLog.logDeleteOp(this.#version, this.#pos++, table, key);\n this.#numChangeLogEntries++;\n }\n }\n\n #logTruncateOp(table: string) {\n if (this.#mode === 'serving') {\n this.#changeLog.logTruncateOp(this.#version, table);\n this.#numChangeLogEntries++;\n }\n }\n\n #logResetOp(table: string) {\n this.#schemaChanged = true;\n if (this.#mode === 'serving') {\n this.#changeLog.logResetOp(this.#version, table);\n this.#numChangeLogEntries++;\n }\n this.#reloadTableSpecs();\n }\n\n processBackfill({relation, watermark, columns, rowValues}: MessageBackfill) {\n const tableName = liteTableName(relation);\n const tableSpec = must(this.#tableSpecs.get(tableName));\n const rowKeyCols = relation.rowKey.columns;\n const cols = [...rowKeyCols, ...columns];\n\n // Common parts of the INSERT sql statement.\n const insertColsStr = [...cols, ZERO_VERSION_COLUMN_NAME].map(id).join(',');\n const qMarks = Array.from({length: cols.length + 1})\n .fill('?')\n .join(',');\n const rowKeyColsStr = rowKeyCols.map(id).join(',');\n\n let backfilled = 0;\n let skipped = 0;\n for (const v of rowValues) {\n const row = liteRow(\n Object.fromEntries(cols.map((c, i) => [c, v[i]])),\n tableSpec,\n this.#jsonFormat,\n );\n const rowKey = this.#getKey(row, {relation});\n const rowOp = this.#changeLog.getLatestRowOp(tableName, rowKey);\n if (rowOp?.op === DEL_OP && rowOp.stateVersion > watermark) {\n skipped++;\n continue; // the row was deleted after the backfill snapshot\n }\n const updates =\n rowOp?.op === SET_OP\n ? cols.filter(\n c => (rowOp.backfillingColumnVersions[c] ?? '') <= watermark,\n )\n : cols;\n if (updates.length === 0) {\n // row already has newer values for all backfilling columns.\n skipped++;\n continue;\n }\n const updateStmts = updates.map(col => `${id(col)}=excluded.${id(col)}`);\n this.#db.run(\n /*sql*/ `\n INSERT INTO ${id(tableName)} (${insertColsStr}) VALUES (${qMarks})\n ON CONFLICT (${rowKeyColsStr})\n DO UPDATE SET ${updateStmts.join(',')};\n `,\n ...Object.values(row.row),\n watermark, // the _0_version for new rows (i.e. table backfill)\n );\n backfilled++;\n }\n\n this.#lc.debug?.(\n `backfilled ${backfilled} rows (skipped ${skipped}) into ${tableName}`,\n );\n }\n\n #completedBackfill: DownloadStatus | undefined;\n\n processBackfillCompleted(msg: BackfillCompleted) {\n const {relation, columns, status} = msg;\n const tableName = liteTableName(relation);\n const rowKeyCols = relation.rowKey.columns;\n const cols = [...rowKeyCols, ...columns];\n\n const columnMetadata = must(ColumnMetadataStore.getInstance(this.#db.db));\n for (const col of cols) {\n columnMetadata.clearBackfilling(tableName, col);\n }\n this.#backfilling.apply(msg);\n // Given that new columns are being exposed for every row in the table, bump the\n // row version for all rows.\n this.#bumpVersions(relation);\n if (status) {\n this.#completedBackfill = {table: tableName, columns: cols, ...status};\n }\n this.#lc.info?.(`finished backfilling ${tableName}`);\n\n // Note that there is no need to clear the backfillingColumnVersions values\n // in the changeLog. It could theoretically be done for clarity but:\n // (1) it could be non-trivial in terms of latency introduced and\n // (2) the data must be preserved if _other_ columns are in the process\n // of being backfilled\n //\n // Thus, for speed and simplicity, the values are left as is. (Note that\n // subsequent replicated changes to those rows will clear the values if\n // no backfills are in progress).\n }\n\n processCommit(commit: MessageCommit, watermark: string): CommitResult {\n if (watermark !== this.#version) {\n throw new Error(\n `'commit' version ${watermark} does not match 'begin' version ${\n this.#version\n }: ${stringify(commit)}`,\n );\n }\n updateReplicationWatermark(this.#db, watermark);\n\n if (this.#schemaChanged) {\n const start = Date.now();\n this.#db.db.pragma('optimize');\n this.#lc.info?.(\n `PRAGMA optimized after schema change (${Date.now() - start} ms)`,\n );\n }\n\n if (this.#mode !== 'initial-sync') {\n this.#db.commit();\n }\n\n const elapsedMs = Date.now() - this.#startMs;\n this.#lc.debug?.(`Committed tx@${this.#version} (${elapsedMs} ms)`);\n\n return {\n watermark,\n completedBackfill: this.#completedBackfill,\n schemaUpdated: this.#schemaChanged,\n changeLogUpdated: this.#numChangeLogEntries > 0,\n upstreamCommitTimeMs: commit.commitTimeMs,\n };\n }\n\n abort(lc: LogContext) {\n lc.info?.(`aborting transaction ${this.#version}`);\n this.#db.rollback();\n }\n}\n\nfunction differentStorageTypes(\n oldSpec: ColumnSpec,\n newSpec: ColumnSpec,\n): boolean {\n return (\n oldSpec.dataType !== newSpec.dataType ||\n isEnumColumn(oldSpec) !== isEnumColumn(newSpec) ||\n isArrayColumn(oldSpec) !== isArrayColumn(newSpec)\n );\n}\n\nfunction getBackfilledColumns(\n row: LiteRow,\n {backfilling}: LiteTableSpecWithReplicationStatus,\n): string[] | undefined {\n if (!backfilling?.length) {\n return undefined; // common case\n }\n return backfilling.filter(col => col in row);\n}\n\nfunction oversizedValueDescription(value: LiteValueType): string | undefined {\n if (typeof value === 'bigint') {\n return BigInt.asIntN(64, value) !== value\n ? 'valueType=bigint fitsInt64=false'\n : undefined;\n }\n\n const sizeBytes =\n typeof value === 'string'\n ? Buffer.byteLength(value)\n : value instanceof Uint8Array\n ? value.byteLength\n : undefined;\n if (sizeBytes !== undefined && sizeBytes > MAX_SQLITE_BIND_BYTES) {\n const valueType = typeof value === 'string' ? 'string' : 'buffer';\n return `valueType=${valueType} sizeBytes=${sizeBytes} limitBytes=${MAX_SQLITE_BIND_BYTES}`;\n }\n\n return undefined;\n}\n\n/** Runs `fn`, returning its error rather than throwing it. */\nfunction attempt(fn: () => void): unknown {\n try {\n fn();\n return undefined;\n } catch (e) {\n return e ?? new Error('rollback failed');\n }\n}\n\nfunction ensureError(err: unknown): Error {\n if (err instanceof Error) {\n return err;\n }\n const error = new Error();\n error.cause = err;\n return error;\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;AAyEA,IAAM,2BACJ;AACF,IAAM,wBAAwB,KAAK,IACjC,UAAgB,YAChB,UAAgB,iBAClB;;;;;;;;;;;;AA0BA,IAAa,kBAAb,MAA6B;CAC3B;CACA;CACA;CACA;CACA;CACA;CAKA,8BAAuB,IAAI,IAAgD;CAE3E,aAA0C;CAE1C;CAEA,YACE,IACA,MACA,aACA;EACA,KAAKA,MAAM;EACX,KAAKC,aAAa,IAAI,UAAU,GAAG,EAAE;EACrC,KAAKC,iBAAiB,IAAI,qBAAqB,GAAG,EAAE;EACpD,KAAKC,eAAe,IAAI,mBAAmB,GAAG,EAAE;EAChD,KAAKC,QAAQ;EACb,KAAKC,eAAe;CACtB;CAEA,MAAM,IAAgB,KAAc;EAClC,IAAI,CAAC,KAAKE,UAAU;GAClB,IAAI,eAAe;GACnB,MAAM,iBAAiB,CAAC,cAAc,KAAKC,YAAY,MAAM,EAAE,CAAC,CAAC,EAAE,QACjE,MAAK,MAAM,KAAA,CACb;GAEA,IAAI,eAAe,QAAQ;IACzB,MAAM,gCAAgB,IAAI,MACxB,yEAAyE,OAAO,GAAG,EAAE,qBAAqB,eAAe,IAAI,MAAM,EAAE,KAAK,IAAI,GAChJ;IACA,cAAc,QAAQ;IACtB,eAAe;GACjB;GAEA,KAAKD,WAAW,YAAY,YAAY;GAExC,IAAI,EAAE,KAAKA,oBAAoB,aAAa;IAE1C,GAAG,QAAQ,8BAA8B,KAAKA,QAAQ;IACtD,KAAKF,aAAa,IAAI,KAAKE,QAAQ;GACrC;EACF;CACF;CAEA,MAAM,IAAgB;EACpB,KAAKE,MAAM,IAAI,IAAI,WAAW,CAAC;CACjC;;CAGA,eACE,IACA,YACqB;EACrB,MAAM,CAAC,MAAM,WAAW;EACxB,IAAI,KAAKF,UAAU;GACjB,GAAG,QAAQ,YAAY,QAAQ,KAAK;GACpC,OAAO;EACT;EACA,IAAI;GACF,MAAM,YACJ,SAAS,UACL,WAAW,GAAG,kBACd,SAAS,WACP,WAAW,GAAG,YACd,KAAA;GACR,OAAO,KAAKG,gBAAgB,IAAI,SAAS,SAAS;EACpD,SAAS,GAAG;GACV,KAAKD,MAAM,IAAI,CAAC;EAClB;EACA,OAAO;CACT;CAEA,kBACE,IACA,eACA,YACsB;EACtB,MAAM,QAAQ,KAAK,IAAI;EASvB,KAAK,IAAI,IAAI,IAAK,KAChB,IAAI;GACF,OAAO,IAAI,qBACT,IACA,KAAKT,KACL,KAAKI,OACL,KAAKH,YACL,KAAKC,gBACL,KAAKC,cACL,KAAKG,aACL,eACA,UACF;EACF,SAAS,GAAG;GACV,IAAI,aAAa,eAAe,EAAE,SAAS,eAAe;IACxD,GAAG,OACD,mBAAmB,KAAK,IAAI,IAAI,MAAM,eAAe,IAAI,EAAE,2EAG3D,CACF;IACA;GACF;GACA,MAAM;EACR;CAEJ;;CAGA,gBACE,IACA,KACA,WACqB;EACrB,IAAI,IAAI,QAAQ,SAAS;GACvB,IAAI,KAAKE,YACP,MAAM,IAAI,MAAM,4BAA4B,UAAU,GAAG,GAAG;GAE9D,KAAKA,aAAa,KAAKG,kBACrB,IACA,KAAK,SAAS,GACd,IAAI,QAAA,GACN;GACA,OAAO;EACT;EAGA,MAAM,KAAK,KAAKH;EAChB,IAAI,CAAC,IACH,MAAM,IAAI,MACR,4CAA4C,UAAU,GAAG,GAC3D;EAGF,IAAI,IAAI,QAAQ,UAAU;GACxB,OAAO,WAAW,2CAA2C;GAC7D,MAAM,SAAS,GAAG,cAAc,KAAK,SAAS;GAG9C,KAAKA,aAAa;GAClB,OAAO;EACT;EAEA,IAAI,IAAI,QAAQ,YAAY;GAC1B,GAAG,MAAM,EAAE;GACX,KAAKA,aAAa;GAClB,OAAO;EACT;EAEA,QAAQ,IAAI,KAAZ;GACE,KAAK;IACH,GAAG,cAAc,GAAG;IACpB;GACF,KAAK;IACH,GAAG,cAAc,GAAG;IACpB;GACF,KAAK;IACH,GAAG,cAAc,GAAG;IACpB;GACF,KAAK;IACH,GAAG,gBAAgB,GAAG;IACtB;GACF,KAAK;IACH,GAAG,mBAAmB,GAAG;IACzB;GACF,KAAK;IACH,GAAG,mBAAmB,GAAG;IACzB;GACF,KAAK;IACH,GAAG,qBAAqB,GAAG;IAC3B;GACF,KAAK;IACH,GAAG,iBAAiB,GAAG;IACvB;GACF,KAAK;IACH,GAAG,oBAAoB,GAAG;IAC1B;GACF,KAAK;IACH,GAAG,kBAAkB,GAAG;IACxB;GACF,KAAK;IACH,GAAG,iBAAiB,GAAG;IACvB;GACF,KAAK;IACH,GAAG,mBAAmB,GAAG;IACzB;GACF,KAAK;IACH,GAAG,iBAAiB,GAAG;IACvB;GACF,KAAK;IACH,GAAG,gBAAgB,GAAG;IACtB;GACF,KAAK;IACH,GAAG,yBAAyB,GAAG;IAC/B;GACF,SACE,YAAY,GAAG;EACnB;EAEA,OAAO;CACT;AACF;;;;;;;;;;;;;;;;;;;;;;AAuBA,IAAM,uBAAN,MAA2B;CACzB;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CAEA,OAAO;CACP,iBAAiB;CACjB,uBAAuB;CAEvB,YACE,IACA,IACA,MACA,WACA,eACA,aACA,YACA,eACA,YACA;EACA,KAAKK,WAAW,KAAK,IAAI;EACzB,KAAKT,QAAQ;EACb,KAAKW,cAAc;EAEnB,QAAQ,MAAR;GACE,KAAK;IAKH,GAAG,eAAe;IAClB;GACF,KAAK;IAKH,GAAG,eAAe;IAClB;GACF,KAAK,gBAGH;GACF,SACE,YAAY;EAChB;EACA,KAAKf,MAAM;EACX,KAAKc,WAAW;EAChB,KAAKF,MAAM,GAAG,YAAY,WAAW,aAAa;EAClD,KAAKX,aAAa;EAClB,KAAKC,iBAAiB;EACtB,KAAKC,eAAe;EACpB,KAAKG,cAAc;EAGnB,KAAKU,kBAAkB,KAAK,oBAAoB,YAAY,GAAG,EAAE,CAAC;EAElE,IAAI,KAAKV,YAAY,SAAS,GAC5B,KAAKW,kBAAkB;CAE3B;CAEA,oBAAoB;EAClB,KAAKX,YAAY,MAAM;EAEvB,MAAM,WAAW,gBAAgB,KAAKM,KAAK,KAAKZ,IAAI,IAAI,EACtD,2BAA2B,KAC7B,CAAC;EACD,KAAK,IAAI,QAAQ,WAAW,KAAKA,IAAI,EAAE,GAAG;GACxC,IAAI,CAAC,KAAK,YACR,OAAO;IACL,GAAG;IACH,YAAY,CACV,GAAI,SAAS,IAAI,KAAK,IAAI,GAAG,UAAU,cAAc,CAAC,CACxD;GACF;GAEF,KAAKM,YAAY,IAAI,KAAK,MAAM,IAAI;EACtC;CACF;CAEA,WAAW,MAAc;EACvB,OAAO,KAAK,KAAKA,YAAY,IAAI,IAAI,GAAG,iBAAiB,MAAM;CACjE;CAEA,QACE,EAAC,KAAK,WACN,EAAC,YACW;EACZ,MAAM,aACJ,SAAS,OAAO,SAAS,SACrB,SAAS,OAAO,UAChB,KAAKY,WAAW,cAAc,QAAQ,CAAC,EAAE;EAC/C,IAAI,CAAC,YAAY,QACf,MAAM,IAAI,MACR,2BAA2B,SAAS,KAAK,wCAC3C;EAIF,IAAI,YAAY,WAAW,QACzB,OAAO;EAET,MAAM,MAAqC,CAAC;EAC5C,KAAK,MAAM,OAAO,YAChB,IAAI,OAAO,IAAI;EAEjB,OAAO;CACT;CAEA,cAAc,QAAuB;EACnC,MAAM,QAAQ,cAAc,OAAO,QAAQ;EAC3C,MAAM,YAAY,KAAKA,WAAW,KAAK;EACvC,MAAM,SAAS,QAAQ,OAAO,KAAK,WAAW,KAAKH,WAAW;EAE9D,KAAKI,QAAQ,OAAO;GAClB,GAAG,OAAO;IACT,2BAA2B,KAAKL;EACnC,CAAC;EAED,IAAI,OAAO,SAAS,OAAO,QAAQ,WAAW,GAQ5C;EAEF,MAAM,MAAM,KAAKM,QAAQ,QAAQ,MAAM;EACvC,KAAKC,UAAU,OAAO,KAAK,qBAAqB,OAAO,KAAK,SAAS,CAAC;CACxE;CAEA,QAAQ,OAAe,KAAc;EACnC,MAAM,UAAU,OAAO,KAAK,GAAG,EAAE,KAAI,MAAK,GAAG,CAAC,CAAC;EAC/C,KAAKrB,IAAI,IACP;+BACyB,GAAG,KAAK,EAAE,IAAI,QAAQ,KAAK,GAAG,EAAE;kBAC7C,MAAM,KAAK,EAAC,QAAQ,QAAQ,OAAM,CAAC,EAAE,KAAK,GAAG,EAAE,KAAK,GAAG,EAAE;SAErE,OAAO,OAAO,GAAG,CACnB;CACF;CAcA,cAAc,QAAuB;EACnC,MAAM,QAAQ,cAAc,OAAO,QAAQ;EAC3C,MAAM,YAAY,KAAKkB,WAAW,KAAK;EACvC,MAAM,SAAS,QAAQ,OAAO,KAAK,WAAW,KAAKH,WAAW;EAC9D,MAAM,MAAM;GAAC,GAAG,OAAO;IAAM,2BAA2B,KAAKD;EAAQ;EAGrE,MAAM,SAAS,OAAO,MAClB,KAAKM,QACH,QAAQ,OAAO,KAAK,KAAKF,WAAW,KAAK,GAAG,KAAKH,WAAW,GAC5D,MACF,IACA;EACJ,MAAM,SAAS,KAAKK,QAAQ,QAAQ,MAAM;EAE1C,IAAI,QACF,KAAKE,aAAa,OAAO,QAAQ,UAAU,WAAW;EAExD,KAAKD,UAAU,OAAO,QAAQ,qBAAqB,OAAO,KAAK,SAAS,CAAC;EAEzE,MAAM,UAAU,UAAU;EAC1B,MAAM,QAAQ,OAAO,KAAK,OAAO,EAAE,KAAI,QAAO,GAAG,GAAG,GAAG,EAAE,GAAG;EAC5D,MAAM,WAAW,OAAO,KAAK,GAAG,EAAE,KAAI,QAAO,GAAG,GAAG,GAAG,EAAE,GAAG;EAE3D,IAAI;GACF,MAAM,EAAC,YAAW,KAAKrB,IAAI,IACzB;iBACS,GAAG,KAAK,EAAE;gBACX,SAAS,KAAK,GAAG,EAAE;kBACjB,MAAM,KAAK,OAAO,EAAE;WAE9B,CAAC,GAAG,OAAO,OAAO,GAAG,GAAG,GAAG,OAAO,OAAO,OAAO,CAAC,CACnD;GAIA,IAAI,YAAY,GACd,KAAKmB,QAAQ,OAAO,GAAG;EAE3B,SAAS,GAAG;GACV,IACE,EAAE,aAAa,eACf,EAAE,YAAY,0BAEd,MAAM;GAGR,MAAM,UAAU,CAAC,GAAG,OAAO,QAAQ,GAAG,GAAG,GAAG,OAAO,QAAQ,OAAO,CAAC,EAChE,KACE,CAAC,QAAQ,WACR,CAAC,QAAQ,0BAA0B,KAAK,CAAC,CAC7C,EACC,MAAM,GAAG,iBAAiB,WAAW;GACxC,MAAM,cACJ,iBAAiB,OAAO,YACxB,OAAO,OAAO,SAAS,gBAAgB,WACnC,gBAAgB,OAAO,SAAS,gBAChC;GACN,MAAM,QAAQ,IAAI,MAChB,uCAAuC,KAAKL,WAAW,YAAY,SAAS,OAAO,SAAS,OAAO,GAAG,OAAO,SAAS,KAAK,UAAU,UAAU,MAAM,YAAY,UAAU,KAAK,IAAI,QAAQ,OAAO,MACnM,EAAC,OAAO,EAAC,CACX;GACA,MAAM,OAAO;GACb,MAAM;EACR;CACF;CAEA,cAAc,KAAoB;EAChC,MAAM,QAAQ,cAAc,IAAI,QAAQ;EACxC,MAAM,YAAY,KAAKI,WAAW,KAAK;EACvC,MAAM,SAAS,KAAKE,QAClB,QAAQ,IAAI,KAAK,WAAW,KAAKL,WAAW,GAC5C,GACF;EAEA,KAAKQ,QAAQ,OAAO,MAAM;EAC1B,KAAKD,aAAa,OAAO,QAAQ,UAAU,WAAW;CACxD;CAEA,QAAQ,OAAe,QAAoB;EACzC,MAAM,QAAQ,OAAO,KAAK,MAAM,EAAE,KAAI,QAAO,GAAG,GAAG,GAAG,EAAE,GAAG;EAC3D,KAAKtB,IAAI,IACP,eAAe,GAAG,KAAK,EAAE,SAAS,MAAM,KAAK,OAAO,KACpD,OAAO,OAAO,MAAM,CACtB;CACF;CAEA,gBAAgB,UAA2B;EACzC,KAAK,MAAM,YAAY,SAAS,WAAW;GACzC,MAAM,QAAQ,cAAc,QAAQ;GAEpC,KAAKA,IAAI,IAAI,eAAe,GAAG,KAAK,GAAG;GAGvC,KAAKwB,eAAe,KAAK;EAC3B;CACF;CAEA,mBAAmB,QAAqB;EACtC,IAAI,OAAO,UACT,KAAKtB,eAAe,oBAAoB,OAAO,MAAM,OAAO,QAAQ;EAEtE,KAAKC,aAAa,MAAM,MAAM;EAC9B,MAAM,QAAQ,kBAAkB,OAAO,IAAI;EAC3C,KAAKH,IAAI,GAAG,KAAK,yBAAyB,KAAK,CAAC;EAGhD,KAAK,MAAM,CAAC,SAAS,YAAY,OAAO,QAAQ,OAAO,KAAK,OAAO,GACjE,KAAKgB,gBAAgB,OACnB,MAAM,MACN,SACA,SACA,OAAO,WAAW,QACpB;EAGF,KAAKS,YAAY,MAAM,IAAI;EAC3B,KAAKb,IAAI,OAAO,OAAO,KAAK,MAAM,IAAI;CACxC;CAEA,qBAAqB,KAA0B;EAC7C,KAAKV,eAAe,oBAAoB,IAAI,OAAO,IAAI,GAAG;EAO1D,KAAKC,aAAa,MAAM,GAAG;CAC7B;CAEA,mBAAmB,QAAqB;EACtC,KAAKD,eAAe,OAAO,OAAO,KAAK,OAAO,GAAG;EACjD,KAAKC,aAAa,MAAM,MAAM;EAE9B,MAAM,UAAU,cAAc,OAAO,GAAG;EACxC,MAAM,UAAU,cAAc,OAAO,GAAG;EACxC,KAAKH,IAAI,GAAG,KAAK,eAAe,GAAG,OAAO,EAAE,aAAa,GAAG,OAAO,GAAG;EAGtE,KAAKgB,gBAAgB,YAAY,SAAS,OAAO;EAEjD,KAAKU,cAAc,OAAO,GAAG;EAC7B,KAAKD,YAAY,OAAO;EACxB,KAAKb,IAAI,OAAO,OAAO,KAAK,SAAS,OAAO;CAC9C;CAEA,iBAAiB,KAAgB;EAC/B,IAAI,IAAI,eACN,KAAKV,eAAe,oBAAoB,IAAI,OAAO,IAAI,aAAa;EAEtE,KAAKC,aAAa,MAAM,GAAG;EAC3B,MAAM,QAAQ,cAAc,IAAI,KAAK;EACrC,MAAM,EAAC,SAAQ,IAAI;EACnB,MAAM,OAAO,wBAAwB,OAAO,IAAI,MAAM;EACtD,KAAKH,IAAI,GAAG,KACV,eAAe,GAAG,KAAK,EAAE,OAAO,GAAG,IAAI,EAAE,GAAG,cAAc,IAAI,GAChE;EAGA,KAAKgB,gBAAgB,OAAO,OAAO,MAAM,IAAI,OAAO,MAAM,IAAI,QAAQ;EAEtE,IAAI,IAAI,UACN,KAAKC,kBAAkB;OAIvB,KAAKS,cAAc,IAAI,KAAK;EAE9B,KAAKd,IAAI,OAAO,IAAI,KAAK,OAAO,IAAI,MAAM;CAC5C;CAEA,oBAAoB,KAAmB;EAGrC,KAAKT,aAAa,MAAM,GAAG;EAE3B,MAAM,QAAQ,cAAc,IAAI,KAAK;EACrC,IAAI,UAAU,IAAI,IAAI;EACtB,MAAM,UAAU,IAAI,IAAI;EAExB,MAAM,qBAAqB,sBACzB,IAAI,IAAI,MACR,IAAI,IAAI,IACV;EAMA,IAAI,YAAY,WAAW,CAAC,oBAAoB;GAC9C,KAAKa,gBAAgB,OACnB,OACA,IAAI,IAAI,MACR,IAAI,IAAI,MACR,IAAI,IAAI,IACV;GACA,IAAI,QAAQ,IAAI,IAAI,KAAK,OAAO,MAAM,QAAQ,IAAI,IAAI,KAAK,OAAO,GAChE,KAAKU,cAAc,IAAI,KAAK;GAE9B,KAAKd,IAAI,OAAO,IAAI,KAAK,yBAAyB,IAAI,KAAK,IAAI,GAAG;GAClE;EACF;EAGA,IAAI,oBAAoB;GAWtB,MAAM,cAAc,wBAClB,OACA,IAAI,KACJ,gBACF;GACA,IAAI,YAAY,SAAS;IAIvB,KAAKZ,IAAI,GAAG,KACV,eAAe,GAAG,KAAK,EAAE,UAAU,GAAG,OAAO,EAAE,MAAM,GAAG,OAAO,GACjE;IACA,UAAU;GACZ;GACA,MAAM,YAAY,KAChB,WAAW,KAAKA,IAAI,IAAI,OAAO,KAAK,EAAE,MACpC,cAAa,UAAU,SAAS,KAClC,CACF;GAIA,MAAM,WAAW,KAAKA,IAAI,GACvB,QACC;;4BAGF,EACC,IAAmB,KAAK,EACxB,KAAK,EAAC,UAAS,GAAG;GACrB,MAAM,WAAW,OAAO;GACxB,MAAM,UAAU,OAAO,KAAK,UAAU,OAAO;GAC7C,MAAM,QAAQ;IACZ,yBAAyB;KACvB,GAAG;KACH,MAAM;KACN,SAAS,WAAW,UAAU,UAAU,QAAQ,SAAS,CACvD,QACA,WAAW,UAAU;MAAC,GAAG;MAAa,KAAK,KAAK;KAAG,IAAI,IACzD,CAAC;IACH,CAAC;IACD,eAAe,GAAG,QAAQ,EAAE,IAAI,QAAQ,IAAI,EAAE,EAAE,KAAK,GAAG,EAAE;kBAChD,QAAQ,IAAI,EAAE,EAAE,KAAK,GAAG,EAAE,QAAQ,GAAG,KAAK,EAAE;IACtD,cAAc,GAAG,KAAK,EAAE;IACxB,eAAe,GAAG,QAAQ,EAAE,aAAa,GAAG,KAAK,EAAE;IACnD,GAAG,SAAS,KAAI,QAAO,GAAG,IAAI,EAAE;GAClC;GACA,KAAKA,IAAI,GAAG,KAAK,MAAM,KAAK,EAAE,CAAC;EACjC;EACA,IAAI,YAAY,SACd,KAAKA,IAAI,GAAG,KACV,eAAe,GAAG,KAAK,EAAE,UAAU,GAAG,OAAO,EAAE,MAAM,GAAG,OAAO,GACjE;EAIF,KAAKgB,gBAAgB,OACnB,OACA,IAAI,IAAI,MACR,IAAI,IAAI,MACR,IAAI,IAAI,IACV;EAEA,KAAKU,cAAc,IAAI,KAAK;EAC5B,KAAKd,IAAI,OAAO,IAAI,KAAK,OAAO,IAAI,GAAG;CACzC;CAEA,kBAAkB,KAAiB;EACjC,MAAM,QAAQ,cAAc,IAAI,KAAK;EACrC,MAAM,EAAC,WAAU;EACjB,KAAKZ,IAAI,GAAG,KAAK,eAAe,GAAG,KAAK,EAAE,QAAQ,GAAG,MAAM,GAAG;EAG9D,KAAKgB,gBAAgB,aAAa,OAAO,MAAM;EAC/C,KAAKb,aAAa,MAAM,GAAG;EAE3B,KAAKuB,cAAc,IAAI,KAAK;EAC5B,KAAKd,IAAI,OAAO,IAAI,KAAK,OAAO,MAAM;CACxC;CAEA,iBAAiB,MAAiB;EAChC,KAAKV,eAAe,KAAK,KAAK,EAAE;EAChC,KAAKC,aAAa,MAAM,IAAI;EAE5B,MAAM,OAAO,cAAc,KAAK,EAAE;EAClC,KAAKH,IAAI,GAAG,KAAK,wBAAwB,GAAG,IAAI,GAAG;EAGnD,KAAKgB,gBAAgB,YAAY,IAAI;EAErC,KAAKS,YAAY,IAAI;EACrB,KAAKb,IAAI,OAAO,KAAK,KAAK,IAAI;CAChC;CAEA,mBAAmB,QAAqB;EACtC,MAAM,QAAQ,uBAAuB,OAAO,IAAI;EAChD,KAAKZ,IAAI,GAAG,KAAK,yBAAyB,KAAK,CAAC;EAIhD,KAAKyB,YAAY,MAAM,SAAS;EAChC,KAAKb,IAAI,OAAO,OAAO,KAAK,MAAM,IAAI;CACxC;CAEA,iBAAiB,MAAiB;EAChC,MAAM,OAAO,cAAc,KAAK,EAAE;EAClC,KAAKZ,IAAI,GAAG,KAAK,wBAAwB,GAAG,IAAI,GAAG;EACnD,KAAKY,IAAI,OAAO,KAAK,KAAK,IAAI;CAChC;CAEA,cAAc,OAAmB;EAC/B,KAAKV,eAAe,iBAAiB,OAAO,KAAKY,QAAQ;EACzD,KAAKW,YAAY,cAAc,KAAK,CAAC;CACvC;;;;CAKA,UACE,OACA,KACA,mBACA;EAIA,IAAI,KAAKrB,UAAU,aAAa,sBAAsB,KAAA,GAAW;GAC/D,KAAKH,WAAW,SACd,KAAKa,UACL,KAAKa,QACL,OACA,KACA,iBACF;GACA,KAAKC;EACP;CACF;CAEA,aAAa,OAAe,KAAiB,aAAwB;EAInE,IAAI,KAAKxB,UAAU,aAAa,aAAa,QAAQ;GACnD,KAAKH,WAAW,YAAY,KAAKa,UAAU,KAAKa,QAAQ,OAAO,GAAG;GAClE,KAAKC;EACP;CACF;CAEA,eAAe,OAAe;EAC5B,IAAI,KAAKxB,UAAU,WAAW;GAC5B,KAAKH,WAAW,cAAc,KAAKa,UAAU,KAAK;GAClD,KAAKc;EACP;CACF;CAEA,YAAY,OAAe;EACzB,KAAKC,iBAAiB;EACtB,IAAI,KAAKzB,UAAU,WAAW;GAC5B,KAAKH,WAAW,WAAW,KAAKa,UAAU,KAAK;GAC/C,KAAKc;EACP;EACA,KAAKX,kBAAkB;CACzB;CAEA,gBAAgB,EAAC,UAAU,WAAW,SAAS,aAA6B;EAC1E,MAAM,YAAY,cAAc,QAAQ;EACxC,MAAM,YAAY,KAAK,KAAKX,YAAY,IAAI,SAAS,CAAC;EACtD,MAAM,aAAa,SAAS,OAAO;EACnC,MAAM,OAAO,CAAC,GAAG,YAAY,GAAG,OAAO;EAGvC,MAAM,gBAAgB,CAAC,GAAG,MAAM,wBAAwB,EAAE,IAAI,EAAE,EAAE,KAAK,GAAG;EAC1E,MAAM,SAAS,MAAM,KAAK,EAAC,QAAQ,KAAK,SAAS,EAAC,CAAC,EAChD,KAAK,GAAG,EACR,KAAK,GAAG;EACX,MAAM,gBAAgB,WAAW,IAAI,EAAE,EAAE,KAAK,GAAG;EAEjD,IAAI,aAAa;EACjB,IAAI,UAAU;EACd,KAAK,MAAM,KAAK,WAAW;GACzB,MAAM,MAAM,QACV,OAAO,YAAY,KAAK,KAAK,GAAG,MAAM,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,GAChD,WACA,KAAKS,WACP;GACA,MAAM,SAAS,KAAKK,QAAQ,KAAK,EAAC,SAAQ,CAAC;GAC3C,MAAM,QAAQ,KAAKnB,WAAW,eAAe,WAAW,MAAM;GAC9D,IAAI,OAAO,OAAA,OAAiB,MAAM,eAAe,WAAW;IAC1D;IACA;GACF;GACA,MAAM,UACJ,OAAO,OAAA,MACH,KAAK,QACH,OAAM,MAAM,0BAA0B,MAAM,OAAO,SACrD,IACA;GACN,IAAI,QAAQ,WAAW,GAAG;IAExB;IACA;GACF;GACA,MAAM,cAAc,QAAQ,KAAI,QAAO,GAAG,GAAG,GAAG,EAAE,YAAY,GAAG,GAAG,GAAG;GACvE,KAAKD,IAAI,IACC;sBACM,GAAG,SAAS,EAAE,IAAI,cAAc,YAAY,OAAO;yBAChD,cAAc;0BACb,YAAY,KAAK,GAAG,EAAE;SAExC,GAAG,OAAO,OAAO,IAAI,GAAG,GACxB,SACF;GACA;EACF;EAEA,KAAKY,IAAI,QACP,cAAc,WAAW,iBAAiB,QAAQ,SAAS,WAC7D;CACF;CAEA;CAEA,yBAAyB,KAAwB;EAC/C,MAAM,EAAC,UAAU,SAAS,WAAU;EACpC,MAAM,YAAY,cAAc,QAAQ;EAExC,MAAM,OAAO,CAAC,GADK,SAAS,OAAO,SACN,GAAG,OAAO;EAEvC,MAAM,iBAAiB,KAAK,oBAAoB,YAAY,KAAKZ,IAAI,EAAE,CAAC;EACxE,KAAK,MAAM,OAAO,MAChB,eAAe,iBAAiB,WAAW,GAAG;EAEhD,KAAKG,aAAa,MAAM,GAAG;EAG3B,KAAKuB,cAAc,QAAQ;EAC3B,IAAI,QACF,KAAKI,qBAAqB;GAAC,OAAO;GAAW,SAAS;GAAM,GAAG;EAAM;EAEvE,KAAKlB,IAAI,OAAO,wBAAwB,WAAW;CAWrD;CAEA,cAAc,QAAuB,WAAiC;EACpE,IAAI,cAAc,KAAKE,UACrB,MAAM,IAAI,MACR,oBAAoB,UAAU,kCAC5B,KAAKA,SACN,IAAI,UAAU,MAAM,GACvB;EAEF,2BAA2B,KAAKd,KAAK,SAAS;EAE9C,IAAI,KAAK6B,gBAAgB;GACvB,MAAM,QAAQ,KAAK,IAAI;GACvB,KAAK7B,IAAI,GAAG,OAAO,UAAU;GAC7B,KAAKY,IAAI,OACP,yCAAyC,KAAK,IAAI,IAAI,MAAM,KAC9D;EACF;EAEA,IAAI,KAAKR,UAAU,gBACjB,KAAKJ,IAAI,OAAO;EAGlB,MAAM,YAAY,KAAK,IAAI,IAAI,KAAKa;EACpC,KAAKD,IAAI,QAAQ,gBAAgB,KAAKE,SAAS,IAAI,UAAU,KAAK;EAElE,OAAO;GACL;GACA,mBAAmB,KAAKgB;GACxB,eAAe,KAAKD;GACpB,kBAAkB,KAAKD,uBAAuB;GAC9C,sBAAsB,OAAO;EAC/B;CACF;CAEA,MAAM,IAAgB;EACpB,GAAG,OAAO,wBAAwB,KAAKd,UAAU;EACjD,KAAKd,IAAI,SAAS;CACpB;AACF;AAEA,SAAS,sBACP,SACA,SACS;CACT,OACE,QAAQ,aAAa,QAAQ,YAC7B,aAAa,OAAO,MAAM,aAAa,OAAO,KAC9C,cAAc,OAAO,MAAM,cAAc,OAAO;AAEpD;AAEA,SAAS,qBACP,KACA,EAAC,eACqB;CACtB,IAAI,CAAC,aAAa,QAChB;CAEF,OAAO,YAAY,QAAO,QAAO,OAAO,GAAG;AAC7C;AAEA,SAAS,0BAA0B,OAA0C;CAC3E,IAAI,OAAO,UAAU,UACnB,OAAO,OAAO,OAAO,IAAI,KAAK,MAAM,QAChC,qCACA,KAAA;CAGN,MAAM,YACJ,OAAO,UAAU,WACb,OAAO,WAAW,KAAK,IACvB,iBAAiB,aACf,MAAM,aACN,KAAA;CACR,IAAI,cAAc,KAAA,KAAa,YAAY,uBAEzC,OAAO,aADW,OAAO,UAAU,WAAW,WAAW,SAC3B,aAAa,UAAU,cAAc;AAIvE;;AAGA,SAAS,QAAQ,IAAyB;CACxC,IAAI;EACF,GAAG;EACH;CACF,SAAS,GAAG;EACV,OAAO,qBAAK,IAAI,MAAM,iBAAiB;CACzC;AACF;AAEA,SAAS,YAAY,KAAqB;CACxC,IAAI,eAAe,OACjB,OAAO;CAET,MAAM,wBAAQ,IAAI,MAAM;CACxB,MAAM,QAAQ;CACd,OAAO;AACT"}
|
|
1
|
+
{"version":3,"file":"change-processor.js","names":["#db","#changeLog","#tableMetadata","#backfilling","#mode","#failService","#tableSpecs","#failure","#currentTx","#fail","#processMessage","#beginTransaction","#lc","#startMs","#version","#jsonFormat","#columnMetadata","#reloadTableSpecs","#tableSpec","#upsert","#getKey","#logSetOp","#logDeleteOp","#delete","#logTruncateOp","#logResetOp","#bumpVersions","#pos","#numChangeLogEntries","#schemaChanged","#completedBackfill"],"sources":["../../../../../../zero-cache/src/services/replicator/change-processor.ts"],"sourcesContent":["import {constants as bufferConstants} from 'node:buffer';\nimport type {LogContext} from '@rocicorp/logger';\nimport {SqliteError} from '@rocicorp/zero-sqlite3';\nimport {AbortError} from '../../../../shared/src/abort-error.ts';\nimport {assert, unreachable} from '../../../../shared/src/asserts.ts';\nimport {stringify} from '../../../../shared/src/bigint-json.ts';\nimport {must} from '../../../../shared/src/must.ts';\nimport {mapEntries} from '../../../../shared/src/objects.ts';\nimport type {DownloadStatus} from '../../../../zero-events/src/status.ts';\nimport {\n createLiteIndexStatement,\n createLiteTableStatement,\n liteColumnDef,\n} from '../../db/create.ts';\nimport {\n computeZqlSpecs,\n listTables,\n type LiteTableSpecWithReplicationStatus,\n} from '../../db/lite-tables.ts';\nimport {\n isArrayColumn,\n isEnumColumn,\n mapPostgresToLite,\n mapPostgresToLiteColumn,\n mapPostgresToLiteIndex,\n} from '../../db/pg-to-lite.ts';\nimport type {ColumnSpec} from '../../db/specs.ts';\nimport type {StatementRunner} from '../../db/statements.ts';\nimport type {LexiVersion} from '../../types/lexi-version.ts';\nimport {\n JSON_PARSED,\n liteRow,\n type JSONFormat,\n type LiteRow,\n type LiteRowKey,\n type LiteValueType,\n} from '../../types/lite.ts';\nimport {liteTableName} from '../../types/names.ts';\nimport {id} from '../../types/sql.ts';\nimport type {\n BackfillCompleted,\n Change,\n ColumnAdd,\n ColumnDrop,\n ColumnUpdate,\n Identifier,\n IndexCreate,\n IndexDrop,\n MessageBackfill,\n MessageCommit,\n MessageDelete,\n MessageInsert,\n MessageRelation,\n MessageTruncate,\n MessageUpdate,\n TableCreate,\n TableDrop,\n TableRename,\n TableUpdateMetadata,\n} from '../change-source/protocol/current/data.ts';\nimport type {ChangeStreamData} from '../change-source/protocol/current/downstream.ts';\nimport type {ReplicatorMode} from './replicator.ts';\nimport {BackfillingTracker} from './schema/backfilling.ts';\nimport {ChangeLog, DEL_OP, SET_OP} from './schema/change-log.ts';\nimport {ColumnMetadataStore} from './schema/column-metadata.ts';\nimport {\n ZERO_VERSION_COLUMN_NAME,\n updateReplicationWatermark,\n} from './schema/replication-state.ts';\nimport {TableMetadataTracker} from './schema/table-metadata.ts';\n\nexport type ChangeProcessorMode = ReplicatorMode | 'initial-sync';\n\nconst BIND_VALUE_TOO_BIG_ERROR =\n 'The bound string, buffer, or bigint is too big';\nconst MAX_SQLITE_BIND_BYTES = Math.min(\n bufferConstants.MAX_LENGTH,\n bufferConstants.MAX_STRING_LENGTH,\n);\n\nexport type CommitResult = {\n watermark: string;\n completedBackfill: DownloadStatus | undefined;\n schemaUpdated: boolean;\n changeLogUpdated: boolean;\n /**\n * Millisecond epoch at which the transaction committed upstream, if the\n * ChangeSource reported one. Propagated to ViewSyncers as the origin\n * timestamp of the end-to-end serving lag measurement.\n */\n upstreamCommitTimeMs?: number | undefined;\n};\n\n/**\n * The ChangeProcessor partitions the stream of messages into transactions\n * by creating a {@link TransactionProcessor} when a transaction begins, and dispatching\n * messages to it until the commit is received.\n *\n * From https://www.postgresql.org/docs/current/protocol-logical-replication.html#PROTOCOL-LOGICAL-MESSAGES-FLOW :\n *\n * \"The logical replication protocol sends individual transactions one by one.\n * This means that all messages between a pair of Begin and Commit messages\n * belong to the same transaction.\"\n */\nexport class ChangeProcessor {\n readonly #db: StatementRunner;\n readonly #changeLog: ChangeLog;\n readonly #tableMetadata: TableMetadataTracker;\n readonly #backfilling: BackfillingTracker;\n readonly #mode: ChangeProcessorMode;\n readonly #failService: (lc: LogContext, err: unknown) => void;\n\n // The TransactionProcessor lazily loads table specs into this Map,\n // and reloads them after a schema change. It is cached here to avoid\n // reading them from the DB on every transaction.\n readonly #tableSpecs = new Map<string, LiteTableSpecWithReplicationStatus>();\n\n #currentTx: TransactionProcessor | null = null;\n\n #failure: Error | undefined;\n\n constructor(\n db: StatementRunner,\n mode: ChangeProcessorMode,\n failService: (lc: LogContext, err: unknown) => void,\n ) {\n this.#db = db;\n this.#changeLog = new ChangeLog(db.db);\n this.#tableMetadata = new TableMetadataTracker(db.db);\n this.#backfilling = new BackfillingTracker(db.db);\n this.#mode = mode;\n this.#failService = failService;\n }\n\n #fail(lc: LogContext, err: unknown) {\n if (!this.#failure) {\n let failureError = err;\n const rollbackErrors = [attempt(() => this.#currentTx?.abort(lc))].filter(\n e => e !== undefined,\n );\n\n if (rollbackErrors.length) {\n const combinedError = new Error(\n `Message processing failed and rollback also failed: operation error = ${String(err)}; rollback error = ${rollbackErrors.map(String).join('; ')}`,\n );\n combinedError.cause = err;\n failureError = combinedError;\n }\n\n this.#failure = ensureError(failureError);\n\n if (!(this.#failure instanceof AbortError)) {\n // Propagate the failure up to the service.\n lc.error?.('Message Processing failed:', this.#failure);\n this.#failService(lc, this.#failure);\n }\n }\n }\n\n abort(lc: LogContext) {\n this.#fail(lc, new AbortError());\n }\n\n /** @return If a transaction was committed. */\n processMessage(\n lc: LogContext,\n downstream: ChangeStreamData,\n ): CommitResult | null {\n const [type, message] = downstream;\n if (this.#failure) {\n lc.debug?.(`Dropping ${message.tag}`);\n return null;\n }\n try {\n const watermark =\n type === 'begin'\n ? downstream[2].commitWatermark\n : type === 'commit'\n ? downstream[2].watermark\n : undefined;\n return this.#processMessage(lc, message, watermark);\n } catch (e) {\n this.#fail(lc, e);\n }\n return null;\n }\n\n #beginTransaction(\n lc: LogContext,\n commitVersion: string,\n jsonFormat: JSONFormat,\n ): TransactionProcessor {\n const start = Date.now();\n\n // litestream can technically hold the lock for an arbitrary amount of time\n // when checkpointing a large commit. Crashing on the busy-timeout in this\n // scenario will either produce a corrupt backup or otherwise prevent\n // replication from proceeding.\n //\n // Instead, retry the lock acquisition indefinitely. If this masks\n // an unknown deadlock situation, manual intervention will be necessary.\n for (let i = 0; ; i++) {\n try {\n return new TransactionProcessor(\n lc,\n this.#db,\n this.#mode,\n this.#changeLog,\n this.#tableMetadata,\n this.#backfilling,\n this.#tableSpecs,\n commitVersion,\n jsonFormat,\n );\n } catch (e) {\n if (e instanceof SqliteError && e.code === 'SQLITE_BUSY') {\n lc.warn?.(\n `SQLITE_BUSY for ${Date.now() - start} ms (attempt ${i + 1}). ` +\n `This is only expected if litestream is performing a large ` +\n `checkpoint.`,\n e,\n );\n continue;\n }\n throw e;\n }\n }\n }\n\n /** @return If a transaction was committed. */\n #processMessage(\n lc: LogContext,\n msg: Change,\n watermark: string | undefined,\n ): CommitResult | null {\n if (msg.tag === 'begin') {\n if (this.#currentTx) {\n throw new Error(`Already in a transaction ${stringify(msg)}`);\n }\n this.#currentTx = this.#beginTransaction(\n lc,\n must(watermark),\n msg.json ?? JSON_PARSED,\n );\n return null;\n }\n\n // For non-begin messages, there should be a #currentTx set.\n const tx = this.#currentTx;\n if (!tx) {\n throw new Error(\n `Received message outside of transaction: ${stringify(msg)}`,\n );\n }\n\n if (msg.tag === 'commit') {\n assert(watermark, 'watermark is required for commit messages');\n const result = tx.processCommit(msg, watermark);\n // Clear only after a successful commit so #fail can roll back a commit\n // path that throws before SQLite has committed.\n this.#currentTx = null;\n return result;\n }\n\n if (msg.tag === 'rollback') {\n tx.abort(lc);\n this.#currentTx = null;\n return null;\n }\n\n switch (msg.tag) {\n case 'insert':\n tx.processInsert(msg);\n break;\n case 'update':\n tx.processUpdate(msg);\n break;\n case 'delete':\n tx.processDelete(msg);\n break;\n case 'truncate':\n tx.processTruncate(msg);\n break;\n case 'create-table':\n tx.processCreateTable(msg);\n break;\n case 'rename-table':\n tx.processRenameTable(msg);\n break;\n case 'update-table-metadata':\n tx.processTableMetadata(msg);\n break;\n case 'add-column':\n tx.processAddColumn(msg);\n break;\n case 'update-column':\n tx.processUpdateColumn(msg);\n break;\n case 'drop-column':\n tx.processDropColumn(msg);\n break;\n case 'drop-table':\n tx.processDropTable(msg);\n break;\n case 'create-index':\n tx.processCreateIndex(msg);\n break;\n case 'drop-index':\n tx.processDropIndex(msg);\n break;\n case 'backfill':\n tx.processBackfill(msg);\n break;\n case 'backfill-completed':\n tx.processBackfillCompleted(msg);\n break;\n default:\n unreachable(msg);\n }\n\n return null;\n }\n}\n\n/**\n * The {@link TransactionProcessor} handles the sequence of messages from\n * upstream, from `BEGIN` to `COMMIT` and executes the corresponding mutations\n * on the {@link postgres.TransactionSql} on the replica.\n *\n * When applying row contents to the replica, the `_0_version` column is added / updated,\n * and a corresponding entry in the `ChangeLog` is added. The version value is derived\n * from the watermark of the preceding transaction (stored as the `nextStateVersion` in the\n * `ReplicationState` table).\n *\n * Side note: For non-streaming Postgres transactions, the commitEndLsn (and thus\n * commit watermark) is available in the `begin` message, so it could theoretically\n * be used for the row version of changes within the transaction. However, the\n * commitEndLsn is not available in the streaming (in-progress) transaction\n * protocol, and may not be available for CDC streams of other upstream types.\n * Therefore, the zero replication protocol is designed to not require the commit\n * watermark when a transaction begins.\n *\n * Also of interest is the fact that all INSERT Messages are logically applied as\n * UPSERTs. See {@link processInsert} for the underlying motivation.\n */\nclass TransactionProcessor {\n readonly #lc: LogContext;\n readonly #startMs: number;\n readonly #db: StatementRunner;\n readonly #mode: ChangeProcessorMode;\n readonly #version: LexiVersion;\n readonly #changeLog: ChangeLog;\n readonly #tableMetadata: TableMetadataTracker;\n readonly #backfilling: BackfillingTracker;\n readonly #tableSpecs: Map<string, LiteTableSpecWithReplicationStatus>;\n readonly #jsonFormat: JSONFormat;\n readonly #columnMetadata: ColumnMetadataStore;\n\n #pos = 0;\n #schemaChanged = false;\n #numChangeLogEntries = 0;\n\n constructor(\n lc: LogContext,\n db: StatementRunner,\n mode: ChangeProcessorMode,\n changeLog: ChangeLog,\n tableMetadata: TableMetadataTracker,\n backfilling: BackfillingTracker,\n tableSpecs: Map<string, LiteTableSpecWithReplicationStatus>,\n commitVersion: LexiVersion,\n jsonFormat: JSONFormat,\n ) {\n this.#startMs = Date.now();\n this.#mode = mode;\n this.#jsonFormat = jsonFormat;\n\n switch (mode) {\n case 'serving':\n // This is the only transaction that commits to the serving replica.\n // Snapshotters use BEGIN CONCURRENT for private changes that are\n // always rolled back, while BEGIN IMMEDIATE lets this writer spill\n // dirty pages during large transactions.\n db.beginImmediate();\n break;\n case 'backup':\n // For the backup-replicator (i.e. replication-manager), there are no View Syncers\n // and thus BEGIN CONCURRENT is not necessary. In fact, BEGIN CONCURRENT can cause\n // deadlocks with forced wal-checkpoints (which `litestream replicate` performs),\n // so it is important to use vanilla transactions in this configuration.\n db.beginImmediate();\n break;\n case 'initial-sync':\n // When the ChangeProcessor is used for initial-sync, the calling code\n // handles the transaction boundaries.\n break;\n default:\n unreachable();\n }\n this.#db = db;\n this.#version = commitVersion;\n this.#lc = lc.withContext('version', commitVersion);\n this.#changeLog = changeLog;\n this.#tableMetadata = tableMetadata;\n this.#backfilling = backfilling;\n this.#tableSpecs = tableSpecs;\n // The column_metadata table is guaranteed to exist since the\n // replica-schema.ts migration to v8.\n this.#columnMetadata = must(ColumnMetadataStore.getInstance(db.db));\n\n if (this.#tableSpecs.size === 0) {\n this.#reloadTableSpecs();\n }\n }\n\n #reloadTableSpecs() {\n this.#tableSpecs.clear();\n // zqlSpecs include the primary key derived from unique indexes\n const zqlSpecs = computeZqlSpecs(this.#lc, this.#db.db, {\n includeBackfillingColumns: true,\n });\n for (let spec of listTables(this.#db.db)) {\n if (!spec.primaryKey) {\n spec = {\n ...spec,\n primaryKey: [\n ...(zqlSpecs.get(spec.name)?.tableSpec.primaryKey ?? []),\n ],\n };\n }\n this.#tableSpecs.set(spec.name, spec);\n }\n }\n\n #tableSpec(name: string) {\n return must(this.#tableSpecs.get(name), `Unknown table ${name}`);\n }\n\n #getKey(\n {row, numCols}: {row: LiteRow; numCols: number},\n {relation}: {relation: MessageRelation},\n ): LiteRowKey {\n const keyColumns =\n relation.rowKey.type !== 'full'\n ? relation.rowKey.columns // already a suitable key\n : this.#tableSpec(liteTableName(relation)).primaryKey;\n if (!keyColumns?.length) {\n throw new Error(\n `Cannot replicate table \"${relation.name}\" without a PRIMARY KEY or UNIQUE INDEX`,\n );\n }\n // For the common case (replica identity default), the row is already the\n // key for deletes and updates, in which case a new object can be avoided.\n if (numCols === keyColumns.length) {\n return row;\n }\n const key: Record<string, LiteValueType> = {};\n for (const col of keyColumns) {\n key[col] = row[col];\n }\n return key;\n }\n\n processInsert(insert: MessageInsert) {\n const table = liteTableName(insert.relation);\n const tableSpec = this.#tableSpec(table);\n const newRow = liteRow(insert.new, tableSpec, this.#jsonFormat);\n\n this.#upsert(table, {\n ...newRow.row,\n [ZERO_VERSION_COLUMN_NAME]: this.#version,\n });\n\n if (insert.relation.rowKey.columns.length === 0) {\n // INSERTs can be replicated for rows without a PRIMARY KEY or a\n // UNIQUE INDEX. These are written to the replica but not recorded\n // in the changeLog, because these rows cannot participate in IVM.\n //\n // (Once the table schema has been corrected to include a key, the\n // associated schema change will reset pipelines and data can be\n // loaded via hydration.)\n return;\n }\n const key = this.#getKey(newRow, insert);\n this.#logSetOp(table, key, getBackfilledColumns(newRow.row, tableSpec));\n }\n\n #upsert(table: string, row: LiteRow) {\n const columns = Object.keys(row).map(c => id(c));\n this.#db.run(\n `\n INSERT OR REPLACE INTO ${id(table)} (${columns.join(',')})\n VALUES (${Array.from({length: columns.length}).fill('?').join(',')})\n `,\n Object.values(row),\n );\n }\n\n // Updates by default are applied as UPDATE commands to support partial\n // row specifications from the change source. In particular, this is needed\n // to handle updates for which unchanged TOASTed values are not sent:\n //\n // https://www.postgresql.org/docs/current/protocol-logicalrep-message-formats.html#PROTOCOL-LOGICALREP-MESSAGE-FORMATS-TUPLEDATA\n //\n // However, in certain cases an UPDATE may be received for a row that\n // was not initially synced, such as when, an existing table is added\n // to the app's publication.\n //\n // In order to facilitate \"resumptive\" replication, the logic falls back to\n // an INSERT if the update did not change any rows.\n processUpdate(update: MessageUpdate) {\n const table = liteTableName(update.relation);\n const tableSpec = this.#tableSpec(table);\n const newRow = liteRow(update.new, tableSpec, this.#jsonFormat);\n const row = {...newRow.row, [ZERO_VERSION_COLUMN_NAME]: this.#version};\n\n // update.key is set with the old values if the key has changed.\n const oldKey = update.key\n ? this.#getKey(\n liteRow(update.key, this.#tableSpec(table), this.#jsonFormat),\n update,\n )\n : null;\n const newKey = this.#getKey(newRow, update);\n\n if (oldKey) {\n this.#logDeleteOp(table, oldKey, tableSpec.backfilling);\n }\n this.#logSetOp(table, newKey, getBackfilledColumns(newRow.row, tableSpec));\n\n const currKey = oldKey ?? newKey;\n const conds = Object.keys(currKey).map(col => `${id(col)}=?`);\n const setExprs = Object.keys(row).map(col => `${id(col)}=?`);\n\n try {\n const {changes} = this.#db.run(\n `\n UPDATE ${id(table)}\n SET ${setExprs.join(',')}\n WHERE ${conds.join(' AND ')}\n `,\n [...Object.values(row), ...Object.values(currKey)],\n );\n\n // If the UPDATE did not affect any rows, perform an UPSERT of the\n // new row for resumptive replication.\n if (changes === 0) {\n this.#upsert(table, row);\n }\n } catch (e) {\n if (\n !(e instanceof RangeError) ||\n e.message !== BIND_VALUE_TOO_BIG_ERROR\n ) {\n throw e;\n }\n\n const binding = [...Object.entries(row), ...Object.entries(currKey)]\n .map(\n ([column, value]) =>\n [column, oversizedValueDescription(value)] as const,\n )\n .find(([, description]) => description);\n const relationOid =\n 'relationOid' in update.relation &&\n typeof update.relation.relationOid === 'number'\n ? ` relationOid=${update.relation.relationOid}`\n : '';\n const error = new Error(\n `Oversized SQLite update binding: tx=${this.#version}${relationOid} table=${update.relation.schema}.${update.relation.name} column=${binding?.[0] ?? 'unknown'}${binding?.[1] ? ` ${binding[1]}` : ''}`,\n {cause: e},\n );\n error.name = 'OversizedUpdateBindingError';\n throw error;\n }\n }\n\n processDelete(del: MessageDelete) {\n const table = liteTableName(del.relation);\n const tableSpec = this.#tableSpec(table);\n const rowKey = this.#getKey(\n liteRow(del.key, tableSpec, this.#jsonFormat),\n del,\n );\n\n this.#delete(table, rowKey);\n this.#logDeleteOp(table, rowKey, tableSpec.backfilling);\n }\n\n #delete(table: string, rowKey: LiteRowKey) {\n const conds = Object.keys(rowKey).map(col => `${id(col)}=?`);\n this.#db.run(\n `DELETE FROM ${id(table)} WHERE ${conds.join(' AND ')}`,\n Object.values(rowKey),\n );\n }\n\n processTruncate(truncate: MessageTruncate) {\n for (const relation of truncate.relations) {\n const table = liteTableName(relation);\n // Update replica data.\n this.#db.run(`DELETE FROM ${id(table)}`);\n\n // Update change log.\n this.#logTruncateOp(table);\n }\n }\n\n processCreateTable(create: TableCreate) {\n if (create.metadata) {\n this.#tableMetadata.setUpstreamMetadata(create.spec, create.metadata);\n }\n this.#backfilling.apply(create);\n const table = mapPostgresToLite(create.spec);\n this.#db.db.exec(createLiteTableStatement(table));\n\n // Write to metadata table\n for (const [colName, colSpec] of Object.entries(create.spec.columns)) {\n this.#columnMetadata.insert(\n table.name,\n colName,\n colSpec,\n create.backfill?.[colName],\n );\n }\n\n this.#logResetOp(table.name);\n this.#lc.info?.(create.tag, table.name);\n }\n\n processTableMetadata(msg: TableUpdateMetadata) {\n this.#tableMetadata.setUpstreamMetadata(msg.table, msg.new);\n // Inert today: the fold's only op for this tag is `upsert-metadata`, which\n // the line above is the replica's interpreter of. Called anyway so that\n // every site that hands a schema change to one cookie tracker hands it to\n // both -- an exception here would be a schema change the shared fold has an\n // opinion about and this store never sees, which is the drift the fold\n // exists to make impossible.\n this.#backfilling.apply(msg);\n }\n\n processRenameTable(rename: TableRename) {\n this.#tableMetadata.rename(rename.old, rename.new);\n this.#backfilling.apply(rename);\n\n const oldName = liteTableName(rename.old);\n const newName = liteTableName(rename.new);\n this.#db.db.exec(`ALTER TABLE ${id(oldName)} RENAME TO ${id(newName)}`);\n\n // Rename in metadata table\n this.#columnMetadata.renameTable(oldName, newName);\n\n this.#bumpVersions(rename.new);\n this.#logResetOp(oldName);\n this.#lc.info?.(rename.tag, oldName, newName);\n }\n\n processAddColumn(msg: ColumnAdd) {\n if (msg.tableMetadata) {\n this.#tableMetadata.setUpstreamMetadata(msg.table, msg.tableMetadata);\n }\n this.#backfilling.apply(msg);\n const table = liteTableName(msg.table);\n const {name} = msg.column;\n const spec = mapPostgresToLiteColumn(table, msg.column);\n this.#db.db.exec(\n `ALTER TABLE ${id(table)} ADD ${id(name)} ${liteColumnDef(spec)}`,\n );\n\n // Write to metadata table\n this.#columnMetadata.insert(table, name, msg.column.spec, msg.backfill);\n\n if (msg.backfill) {\n this.#reloadTableSpecs();\n } else {\n // Make the new column visible immediately if it's not being backfilled.\n // Otherwise, the version bump will happen with the backfill is complete.\n this.#bumpVersions(msg.table);\n }\n this.#lc.info?.(msg.tag, table, msg.column);\n }\n\n processUpdateColumn(msg: ColumnUpdate) {\n // A no-op unless this is a rename, which is the only part of a column\n // update that moves the backfill cookie.\n this.#backfilling.apply(msg);\n\n const table = liteTableName(msg.table);\n let oldName = msg.old.name;\n const newName = msg.new.name;\n\n const storageTypesDiffer = differentStorageTypes(\n msg.old.spec,\n msg.new.spec,\n );\n\n // If neither the column name nor the SQLite data type changes, only the\n // upstream metadata needs to be updated. This includes changes such as a\n // varchar character limit and nullability, which SQLite does not enforce\n // but a freshly built replica still records.\n if (oldName === newName && !storageTypesDiffer) {\n this.#columnMetadata.update(\n table,\n msg.old.name,\n msg.new.name,\n msg.new.spec,\n );\n if (Boolean(msg.old.spec.notNull) !== Boolean(msg.new.spec.notNull)) {\n this.#bumpVersions(msg.table);\n }\n this.#lc.info?.(msg.tag, 'updated metadata only', msg.old, msg.new);\n return;\n }\n // Storage type changes require a new column so that SQLite uses the new\n // representation for existing and future values.\n if (storageTypesDiffer) {\n // update-column can ignore defaults because it does not change the values\n // in existing rows.\n //\n // https://www.postgresql.org/docs/current/sql-altertable.html#SQL-ALTERTABLE-DESC-SET-DROP-DEFAULT\n //\n // \"The new default value will only apply in subsequent INSERT or UPDATE\n // commands; it does not cause rows already in the table to change.\"\n //\n // This allows support for _changing_ column defaults to any expression,\n // since it does not affect what the replica needs to do.\n const newLiteSpec = mapPostgresToLiteColumn(\n table,\n msg.new,\n 'ignore-default',\n );\n if (oldName !== newName) {\n // Rename the column in place first so that SQLite rewrites every\n // reference to it in the table's index definitions, including the\n // WHERE clauses of partial indexes.\n this.#db.db.exec(\n `ALTER TABLE ${id(table)} RENAME ${id(oldName)} TO ${id(newName)}`,\n );\n oldName = newName;\n }\n const tableSpec = must(\n listTables(this.#db.db, false, false).find(\n tableSpec => tableSpec.name === table,\n ),\n );\n // Recreating the table drops its indexes, so capture their DDL to\n // replay afterwards. Auto-indexes (PRIMARY KEY) have no DDL and are\n // recreated by the CREATE TABLE statement itself.\n const indexDDL = this.#db.db\n .prepare(\n `SELECT sql FROM sqlite_master\n WHERE type = 'index' AND tbl_name = ? AND sql IS NOT NULL\n ORDER BY rowid`,\n )\n .all<{sql: string}>(table)\n .map(({sql}) => sql);\n const tmpTable = `tmp.${table}`;\n const columns = Object.keys(tableSpec.columns);\n const stmts = [\n createLiteTableStatement({\n ...tableSpec,\n name: tmpTable,\n columns: mapEntries(tableSpec.columns, (column, spec) => [\n column,\n column === newName ? {...newLiteSpec, pos: spec.pos} : spec,\n ]),\n }),\n `INSERT INTO ${id(tmpTable)} (${columns.map(id).join(',')})\n SELECT ${columns.map(id).join(',')} FROM ${id(table)};`,\n `DROP TABLE ${id(table)};`,\n `ALTER TABLE ${id(tmpTable)} RENAME TO ${id(table)};`,\n ...indexDDL.map(sql => `${sql};`),\n ];\n this.#db.db.exec(stmts.join(''));\n }\n if (oldName !== newName) {\n this.#db.db.exec(\n `ALTER TABLE ${id(table)} RENAME ${id(oldName)} TO ${id(newName)}`,\n );\n }\n\n // Update metadata table\n this.#columnMetadata.update(\n table,\n msg.old.name,\n msg.new.name,\n msg.new.spec,\n );\n\n this.#bumpVersions(msg.table);\n this.#lc.info?.(msg.tag, table, msg.new);\n }\n\n processDropColumn(msg: ColumnDrop) {\n const table = liteTableName(msg.table);\n const {column} = msg;\n this.#db.db.exec(`ALTER TABLE ${id(table)} DROP ${id(column)}`);\n\n // Delete from metadata table\n this.#columnMetadata.deleteColumn(table, column);\n this.#backfilling.apply(msg);\n\n this.#bumpVersions(msg.table);\n this.#lc.info?.(msg.tag, table, column);\n }\n\n processDropTable(drop: TableDrop) {\n this.#tableMetadata.drop(drop.id);\n this.#backfilling.apply(drop);\n\n const name = liteTableName(drop.id);\n this.#db.db.exec(`DROP TABLE IF EXISTS ${id(name)}`);\n\n // Delete from metadata table\n this.#columnMetadata.deleteTable(name);\n\n this.#logResetOp(name);\n this.#lc.info?.(drop.tag, name);\n }\n\n processCreateIndex(create: IndexCreate) {\n const index = mapPostgresToLiteIndex(create.spec);\n this.#db.db.exec(createLiteIndexStatement(index));\n\n // indexes affect tables visibility (e.g. sync-ability is gated on\n // having a unique index), so reset pipelines to refresh table schemas.\n this.#logResetOp(index.tableName);\n this.#lc.info?.(create.tag, index.name);\n }\n\n processDropIndex(drop: IndexDrop) {\n const name = liteTableName(drop.id);\n this.#db.db.exec(`DROP INDEX IF EXISTS ${id(name)}`);\n this.#lc.info?.(drop.tag, name);\n }\n\n #bumpVersions(table: Identifier) {\n this.#tableMetadata.setMinRowVersion(table, this.#version);\n this.#logResetOp(liteTableName(table));\n }\n\n /**\n * @param backfilledColumns `backfilling` columns for which values were set\n */\n #logSetOp(\n table: string,\n key: LiteRowKey,\n backfilledColumns: string[] | undefined,\n ) {\n // The \"serving\" replicator always writes to the change-log (for IVM).\n // The \"backup\" replicator only needs to write to the change log\n // when writing columns that are being backfilled.\n if (this.#mode === 'serving' || backfilledColumns !== undefined) {\n this.#changeLog.logSetOp(\n this.#version,\n this.#pos++,\n table,\n key,\n backfilledColumns,\n );\n this.#numChangeLogEntries++;\n }\n }\n\n #logDeleteOp(table: string, key: LiteRowKey, backfilling?: string[]) {\n // The \"serving\" replicator always writes to the change-log (for IVM).\n // The \"backup\" replicator only needs to write to the change log\n // when writing columns that are being backfilled.\n if (this.#mode === 'serving' || backfilling?.length) {\n this.#changeLog.logDeleteOp(this.#version, this.#pos++, table, key);\n this.#numChangeLogEntries++;\n }\n }\n\n #logTruncateOp(table: string) {\n if (this.#mode === 'serving') {\n this.#changeLog.logTruncateOp(this.#version, table);\n this.#numChangeLogEntries++;\n }\n }\n\n #logResetOp(table: string) {\n this.#schemaChanged = true;\n if (this.#mode === 'serving') {\n this.#changeLog.logResetOp(this.#version, table);\n this.#numChangeLogEntries++;\n }\n this.#reloadTableSpecs();\n }\n\n processBackfill({relation, watermark, columns, rowValues}: MessageBackfill) {\n const tableName = liteTableName(relation);\n const tableSpec = must(this.#tableSpecs.get(tableName));\n const rowKeyCols = relation.rowKey.columns;\n const cols = [...rowKeyCols, ...columns];\n\n // Common parts of the INSERT sql statement.\n const insertColsStr = [...cols, ZERO_VERSION_COLUMN_NAME].map(id).join(',');\n const qMarks = Array.from({length: cols.length + 1})\n .fill('?')\n .join(',');\n const rowKeyColsStr = rowKeyCols.map(id).join(',');\n\n let backfilled = 0;\n let skipped = 0;\n for (const v of rowValues) {\n const row = liteRow(\n Object.fromEntries(cols.map((c, i) => [c, v[i]])),\n tableSpec,\n this.#jsonFormat,\n );\n const rowKey = this.#getKey(row, {relation});\n const rowOp = this.#changeLog.getLatestRowOp(tableName, rowKey);\n if (rowOp?.op === DEL_OP && rowOp.stateVersion > watermark) {\n skipped++;\n continue; // the row was deleted after the backfill snapshot\n }\n const updates =\n rowOp?.op === SET_OP\n ? cols.filter(\n c => (rowOp.backfillingColumnVersions[c] ?? '') <= watermark,\n )\n : cols;\n if (updates.length === 0) {\n // row already has newer values for all backfilling columns.\n skipped++;\n continue;\n }\n const updateStmts = updates.map(col => `${id(col)}=excluded.${id(col)}`);\n this.#db.run(\n /*sql*/ `\n INSERT INTO ${id(tableName)} (${insertColsStr}) VALUES (${qMarks})\n ON CONFLICT (${rowKeyColsStr})\n DO UPDATE SET ${updateStmts.join(',')};\n `,\n ...Object.values(row.row),\n watermark, // the _0_version for new rows (i.e. table backfill)\n );\n backfilled++;\n }\n\n this.#lc.debug?.(\n `backfilled ${backfilled} rows (skipped ${skipped}) into ${tableName}`,\n );\n }\n\n #completedBackfill: DownloadStatus | undefined;\n\n processBackfillCompleted(msg: BackfillCompleted) {\n const {relation, columns, status} = msg;\n const tableName = liteTableName(relation);\n const rowKeyCols = relation.rowKey.columns;\n const cols = [...rowKeyCols, ...columns];\n\n const columnMetadata = must(ColumnMetadataStore.getInstance(this.#db.db));\n for (const col of cols) {\n columnMetadata.clearBackfilling(tableName, col);\n }\n this.#backfilling.apply(msg);\n // Given that new columns are being exposed for every row in the table, bump the\n // row version for all rows.\n this.#bumpVersions(relation);\n if (status) {\n this.#completedBackfill = {table: tableName, columns: cols, ...status};\n }\n this.#lc.info?.(`finished backfilling ${tableName}`);\n\n // Note that there is no need to clear the backfillingColumnVersions values\n // in the changeLog. It could theoretically be done for clarity but:\n // (1) it could be non-trivial in terms of latency introduced and\n // (2) the data must be preserved if _other_ columns are in the process\n // of being backfilled\n //\n // Thus, for speed and simplicity, the values are left as is. (Note that\n // subsequent replicated changes to those rows will clear the values if\n // no backfills are in progress).\n }\n\n processCommit(commit: MessageCommit, watermark: string): CommitResult {\n if (watermark !== this.#version) {\n throw new Error(\n `'commit' version ${watermark} does not match 'begin' version ${\n this.#version\n }: ${stringify(commit)}`,\n );\n }\n updateReplicationWatermark(this.#db, watermark);\n\n if (this.#schemaChanged) {\n const start = Date.now();\n this.#db.db.pragma('optimize');\n this.#lc.info?.(\n `PRAGMA optimized after schema change (${Date.now() - start} ms)`,\n );\n }\n\n if (this.#mode !== 'initial-sync') {\n this.#db.commit();\n }\n\n const elapsedMs = Date.now() - this.#startMs;\n this.#lc.debug?.(`Committed tx@${this.#version} (${elapsedMs} ms)`);\n\n return {\n watermark,\n completedBackfill: this.#completedBackfill,\n schemaUpdated: this.#schemaChanged,\n changeLogUpdated: this.#numChangeLogEntries > 0,\n upstreamCommitTimeMs: commit.commitTimeMs,\n };\n }\n\n abort(lc: LogContext) {\n lc.info?.(`aborting transaction ${this.#version}`);\n this.#db.rollback();\n // A schema change in the transaction reloaded the table specs, which now\n // describe the schema that the rollback undid. Clearing them makes the\n // next transaction reload them from the database.\n this.#tableSpecs.clear();\n }\n}\n\nfunction differentStorageTypes(\n oldSpec: ColumnSpec,\n newSpec: ColumnSpec,\n): boolean {\n return (\n oldSpec.dataType !== newSpec.dataType ||\n isEnumColumn(oldSpec) !== isEnumColumn(newSpec) ||\n isArrayColumn(oldSpec) !== isArrayColumn(newSpec)\n );\n}\n\nfunction getBackfilledColumns(\n row: LiteRow,\n {backfilling}: LiteTableSpecWithReplicationStatus,\n): string[] | undefined {\n if (!backfilling?.length) {\n return undefined; // common case\n }\n return backfilling.filter(col => col in row);\n}\n\nfunction oversizedValueDescription(value: LiteValueType): string | undefined {\n if (typeof value === 'bigint') {\n return BigInt.asIntN(64, value) !== value\n ? 'valueType=bigint fitsInt64=false'\n : undefined;\n }\n\n const sizeBytes =\n typeof value === 'string'\n ? Buffer.byteLength(value)\n : value instanceof Uint8Array\n ? value.byteLength\n : undefined;\n if (sizeBytes !== undefined && sizeBytes > MAX_SQLITE_BIND_BYTES) {\n const valueType = typeof value === 'string' ? 'string' : 'buffer';\n return `valueType=${valueType} sizeBytes=${sizeBytes} limitBytes=${MAX_SQLITE_BIND_BYTES}`;\n }\n\n return undefined;\n}\n\n/** Runs `fn`, returning its error rather than throwing it. */\nfunction attempt(fn: () => void): unknown {\n try {\n fn();\n return undefined;\n } catch (e) {\n return e ?? new Error('rollback failed');\n }\n}\n\nfunction ensureError(err: unknown): Error {\n if (err instanceof Error) {\n return err;\n }\n const error = new Error();\n error.cause = err;\n return error;\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;AAyEA,IAAM,2BACJ;AACF,IAAM,wBAAwB,KAAK,IACjC,UAAgB,YAChB,UAAgB,iBAClB;;;;;;;;;;;;AA0BA,IAAa,kBAAb,MAA6B;CAC3B;CACA;CACA;CACA;CACA;CACA;CAKA,8BAAuB,IAAI,IAAgD;CAE3E,aAA0C;CAE1C;CAEA,YACE,IACA,MACA,aACA;EACA,KAAKA,MAAM;EACX,KAAKC,aAAa,IAAI,UAAU,GAAG,EAAE;EACrC,KAAKC,iBAAiB,IAAI,qBAAqB,GAAG,EAAE;EACpD,KAAKC,eAAe,IAAI,mBAAmB,GAAG,EAAE;EAChD,KAAKC,QAAQ;EACb,KAAKC,eAAe;CACtB;CAEA,MAAM,IAAgB,KAAc;EAClC,IAAI,CAAC,KAAKE,UAAU;GAClB,IAAI,eAAe;GACnB,MAAM,iBAAiB,CAAC,cAAc,KAAKC,YAAY,MAAM,EAAE,CAAC,CAAC,EAAE,QACjE,MAAK,MAAM,KAAA,CACb;GAEA,IAAI,eAAe,QAAQ;IACzB,MAAM,gCAAgB,IAAI,MACxB,yEAAyE,OAAO,GAAG,EAAE,qBAAqB,eAAe,IAAI,MAAM,EAAE,KAAK,IAAI,GAChJ;IACA,cAAc,QAAQ;IACtB,eAAe;GACjB;GAEA,KAAKD,WAAW,YAAY,YAAY;GAExC,IAAI,EAAE,KAAKA,oBAAoB,aAAa;IAE1C,GAAG,QAAQ,8BAA8B,KAAKA,QAAQ;IACtD,KAAKF,aAAa,IAAI,KAAKE,QAAQ;GACrC;EACF;CACF;CAEA,MAAM,IAAgB;EACpB,KAAKE,MAAM,IAAI,IAAI,WAAW,CAAC;CACjC;;CAGA,eACE,IACA,YACqB;EACrB,MAAM,CAAC,MAAM,WAAW;EACxB,IAAI,KAAKF,UAAU;GACjB,GAAG,QAAQ,YAAY,QAAQ,KAAK;GACpC,OAAO;EACT;EACA,IAAI;GACF,MAAM,YACJ,SAAS,UACL,WAAW,GAAG,kBACd,SAAS,WACP,WAAW,GAAG,YACd,KAAA;GACR,OAAO,KAAKG,gBAAgB,IAAI,SAAS,SAAS;EACpD,SAAS,GAAG;GACV,KAAKD,MAAM,IAAI,CAAC;EAClB;EACA,OAAO;CACT;CAEA,kBACE,IACA,eACA,YACsB;EACtB,MAAM,QAAQ,KAAK,IAAI;EASvB,KAAK,IAAI,IAAI,IAAK,KAChB,IAAI;GACF,OAAO,IAAI,qBACT,IACA,KAAKT,KACL,KAAKI,OACL,KAAKH,YACL,KAAKC,gBACL,KAAKC,cACL,KAAKG,aACL,eACA,UACF;EACF,SAAS,GAAG;GACV,IAAI,aAAa,eAAe,EAAE,SAAS,eAAe;IACxD,GAAG,OACD,mBAAmB,KAAK,IAAI,IAAI,MAAM,eAAe,IAAI,EAAE,2EAG3D,CACF;IACA;GACF;GACA,MAAM;EACR;CAEJ;;CAGA,gBACE,IACA,KACA,WACqB;EACrB,IAAI,IAAI,QAAQ,SAAS;GACvB,IAAI,KAAKE,YACP,MAAM,IAAI,MAAM,4BAA4B,UAAU,GAAG,GAAG;GAE9D,KAAKA,aAAa,KAAKG,kBACrB,IACA,KAAK,SAAS,GACd,IAAI,QAAA,GACN;GACA,OAAO;EACT;EAGA,MAAM,KAAK,KAAKH;EAChB,IAAI,CAAC,IACH,MAAM,IAAI,MACR,4CAA4C,UAAU,GAAG,GAC3D;EAGF,IAAI,IAAI,QAAQ,UAAU;GACxB,OAAO,WAAW,2CAA2C;GAC7D,MAAM,SAAS,GAAG,cAAc,KAAK,SAAS;GAG9C,KAAKA,aAAa;GAClB,OAAO;EACT;EAEA,IAAI,IAAI,QAAQ,YAAY;GAC1B,GAAG,MAAM,EAAE;GACX,KAAKA,aAAa;GAClB,OAAO;EACT;EAEA,QAAQ,IAAI,KAAZ;GACE,KAAK;IACH,GAAG,cAAc,GAAG;IACpB;GACF,KAAK;IACH,GAAG,cAAc,GAAG;IACpB;GACF,KAAK;IACH,GAAG,cAAc,GAAG;IACpB;GACF,KAAK;IACH,GAAG,gBAAgB,GAAG;IACtB;GACF,KAAK;IACH,GAAG,mBAAmB,GAAG;IACzB;GACF,KAAK;IACH,GAAG,mBAAmB,GAAG;IACzB;GACF,KAAK;IACH,GAAG,qBAAqB,GAAG;IAC3B;GACF,KAAK;IACH,GAAG,iBAAiB,GAAG;IACvB;GACF,KAAK;IACH,GAAG,oBAAoB,GAAG;IAC1B;GACF,KAAK;IACH,GAAG,kBAAkB,GAAG;IACxB;GACF,KAAK;IACH,GAAG,iBAAiB,GAAG;IACvB;GACF,KAAK;IACH,GAAG,mBAAmB,GAAG;IACzB;GACF,KAAK;IACH,GAAG,iBAAiB,GAAG;IACvB;GACF,KAAK;IACH,GAAG,gBAAgB,GAAG;IACtB;GACF,KAAK;IACH,GAAG,yBAAyB,GAAG;IAC/B;GACF,SACE,YAAY,GAAG;EACnB;EAEA,OAAO;CACT;AACF;;;;;;;;;;;;;;;;;;;;;;AAuBA,IAAM,uBAAN,MAA2B;CACzB;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CAEA,OAAO;CACP,iBAAiB;CACjB,uBAAuB;CAEvB,YACE,IACA,IACA,MACA,WACA,eACA,aACA,YACA,eACA,YACA;EACA,KAAKK,WAAW,KAAK,IAAI;EACzB,KAAKT,QAAQ;EACb,KAAKW,cAAc;EAEnB,QAAQ,MAAR;GACE,KAAK;IAKH,GAAG,eAAe;IAClB;GACF,KAAK;IAKH,GAAG,eAAe;IAClB;GACF,KAAK,gBAGH;GACF,SACE,YAAY;EAChB;EACA,KAAKf,MAAM;EACX,KAAKc,WAAW;EAChB,KAAKF,MAAM,GAAG,YAAY,WAAW,aAAa;EAClD,KAAKX,aAAa;EAClB,KAAKC,iBAAiB;EACtB,KAAKC,eAAe;EACpB,KAAKG,cAAc;EAGnB,KAAKU,kBAAkB,KAAK,oBAAoB,YAAY,GAAG,EAAE,CAAC;EAElE,IAAI,KAAKV,YAAY,SAAS,GAC5B,KAAKW,kBAAkB;CAE3B;CAEA,oBAAoB;EAClB,KAAKX,YAAY,MAAM;EAEvB,MAAM,WAAW,gBAAgB,KAAKM,KAAK,KAAKZ,IAAI,IAAI,EACtD,2BAA2B,KAC7B,CAAC;EACD,KAAK,IAAI,QAAQ,WAAW,KAAKA,IAAI,EAAE,GAAG;GACxC,IAAI,CAAC,KAAK,YACR,OAAO;IACL,GAAG;IACH,YAAY,CACV,GAAI,SAAS,IAAI,KAAK,IAAI,GAAG,UAAU,cAAc,CAAC,CACxD;GACF;GAEF,KAAKM,YAAY,IAAI,KAAK,MAAM,IAAI;EACtC;CACF;CAEA,WAAW,MAAc;EACvB,OAAO,KAAK,KAAKA,YAAY,IAAI,IAAI,GAAG,iBAAiB,MAAM;CACjE;CAEA,QACE,EAAC,KAAK,WACN,EAAC,YACW;EACZ,MAAM,aACJ,SAAS,OAAO,SAAS,SACrB,SAAS,OAAO,UAChB,KAAKY,WAAW,cAAc,QAAQ,CAAC,EAAE;EAC/C,IAAI,CAAC,YAAY,QACf,MAAM,IAAI,MACR,2BAA2B,SAAS,KAAK,wCAC3C;EAIF,IAAI,YAAY,WAAW,QACzB,OAAO;EAET,MAAM,MAAqC,CAAC;EAC5C,KAAK,MAAM,OAAO,YAChB,IAAI,OAAO,IAAI;EAEjB,OAAO;CACT;CAEA,cAAc,QAAuB;EACnC,MAAM,QAAQ,cAAc,OAAO,QAAQ;EAC3C,MAAM,YAAY,KAAKA,WAAW,KAAK;EACvC,MAAM,SAAS,QAAQ,OAAO,KAAK,WAAW,KAAKH,WAAW;EAE9D,KAAKI,QAAQ,OAAO;GAClB,GAAG,OAAO;IACT,2BAA2B,KAAKL;EACnC,CAAC;EAED,IAAI,OAAO,SAAS,OAAO,QAAQ,WAAW,GAQ5C;EAEF,MAAM,MAAM,KAAKM,QAAQ,QAAQ,MAAM;EACvC,KAAKC,UAAU,OAAO,KAAK,qBAAqB,OAAO,KAAK,SAAS,CAAC;CACxE;CAEA,QAAQ,OAAe,KAAc;EACnC,MAAM,UAAU,OAAO,KAAK,GAAG,EAAE,KAAI,MAAK,GAAG,CAAC,CAAC;EAC/C,KAAKrB,IAAI,IACP;+BACyB,GAAG,KAAK,EAAE,IAAI,QAAQ,KAAK,GAAG,EAAE;kBAC7C,MAAM,KAAK,EAAC,QAAQ,QAAQ,OAAM,CAAC,EAAE,KAAK,GAAG,EAAE,KAAK,GAAG,EAAE;SAErE,OAAO,OAAO,GAAG,CACnB;CACF;CAcA,cAAc,QAAuB;EACnC,MAAM,QAAQ,cAAc,OAAO,QAAQ;EAC3C,MAAM,YAAY,KAAKkB,WAAW,KAAK;EACvC,MAAM,SAAS,QAAQ,OAAO,KAAK,WAAW,KAAKH,WAAW;EAC9D,MAAM,MAAM;GAAC,GAAG,OAAO;IAAM,2BAA2B,KAAKD;EAAQ;EAGrE,MAAM,SAAS,OAAO,MAClB,KAAKM,QACH,QAAQ,OAAO,KAAK,KAAKF,WAAW,KAAK,GAAG,KAAKH,WAAW,GAC5D,MACF,IACA;EACJ,MAAM,SAAS,KAAKK,QAAQ,QAAQ,MAAM;EAE1C,IAAI,QACF,KAAKE,aAAa,OAAO,QAAQ,UAAU,WAAW;EAExD,KAAKD,UAAU,OAAO,QAAQ,qBAAqB,OAAO,KAAK,SAAS,CAAC;EAEzE,MAAM,UAAU,UAAU;EAC1B,MAAM,QAAQ,OAAO,KAAK,OAAO,EAAE,KAAI,QAAO,GAAG,GAAG,GAAG,EAAE,GAAG;EAC5D,MAAM,WAAW,OAAO,KAAK,GAAG,EAAE,KAAI,QAAO,GAAG,GAAG,GAAG,EAAE,GAAG;EAE3D,IAAI;GACF,MAAM,EAAC,YAAW,KAAKrB,IAAI,IACzB;iBACS,GAAG,KAAK,EAAE;gBACX,SAAS,KAAK,GAAG,EAAE;kBACjB,MAAM,KAAK,OAAO,EAAE;WAE9B,CAAC,GAAG,OAAO,OAAO,GAAG,GAAG,GAAG,OAAO,OAAO,OAAO,CAAC,CACnD;GAIA,IAAI,YAAY,GACd,KAAKmB,QAAQ,OAAO,GAAG;EAE3B,SAAS,GAAG;GACV,IACE,EAAE,aAAa,eACf,EAAE,YAAY,0BAEd,MAAM;GAGR,MAAM,UAAU,CAAC,GAAG,OAAO,QAAQ,GAAG,GAAG,GAAG,OAAO,QAAQ,OAAO,CAAC,EAChE,KACE,CAAC,QAAQ,WACR,CAAC,QAAQ,0BAA0B,KAAK,CAAC,CAC7C,EACC,MAAM,GAAG,iBAAiB,WAAW;GACxC,MAAM,cACJ,iBAAiB,OAAO,YACxB,OAAO,OAAO,SAAS,gBAAgB,WACnC,gBAAgB,OAAO,SAAS,gBAChC;GACN,MAAM,QAAQ,IAAI,MAChB,uCAAuC,KAAKL,WAAW,YAAY,SAAS,OAAO,SAAS,OAAO,GAAG,OAAO,SAAS,KAAK,UAAU,UAAU,MAAM,YAAY,UAAU,KAAK,IAAI,QAAQ,OAAO,MACnM,EAAC,OAAO,EAAC,CACX;GACA,MAAM,OAAO;GACb,MAAM;EACR;CACF;CAEA,cAAc,KAAoB;EAChC,MAAM,QAAQ,cAAc,IAAI,QAAQ;EACxC,MAAM,YAAY,KAAKI,WAAW,KAAK;EACvC,MAAM,SAAS,KAAKE,QAClB,QAAQ,IAAI,KAAK,WAAW,KAAKL,WAAW,GAC5C,GACF;EAEA,KAAKQ,QAAQ,OAAO,MAAM;EAC1B,KAAKD,aAAa,OAAO,QAAQ,UAAU,WAAW;CACxD;CAEA,QAAQ,OAAe,QAAoB;EACzC,MAAM,QAAQ,OAAO,KAAK,MAAM,EAAE,KAAI,QAAO,GAAG,GAAG,GAAG,EAAE,GAAG;EAC3D,KAAKtB,IAAI,IACP,eAAe,GAAG,KAAK,EAAE,SAAS,MAAM,KAAK,OAAO,KACpD,OAAO,OAAO,MAAM,CACtB;CACF;CAEA,gBAAgB,UAA2B;EACzC,KAAK,MAAM,YAAY,SAAS,WAAW;GACzC,MAAM,QAAQ,cAAc,QAAQ;GAEpC,KAAKA,IAAI,IAAI,eAAe,GAAG,KAAK,GAAG;GAGvC,KAAKwB,eAAe,KAAK;EAC3B;CACF;CAEA,mBAAmB,QAAqB;EACtC,IAAI,OAAO,UACT,KAAKtB,eAAe,oBAAoB,OAAO,MAAM,OAAO,QAAQ;EAEtE,KAAKC,aAAa,MAAM,MAAM;EAC9B,MAAM,QAAQ,kBAAkB,OAAO,IAAI;EAC3C,KAAKH,IAAI,GAAG,KAAK,yBAAyB,KAAK,CAAC;EAGhD,KAAK,MAAM,CAAC,SAAS,YAAY,OAAO,QAAQ,OAAO,KAAK,OAAO,GACjE,KAAKgB,gBAAgB,OACnB,MAAM,MACN,SACA,SACA,OAAO,WAAW,QACpB;EAGF,KAAKS,YAAY,MAAM,IAAI;EAC3B,KAAKb,IAAI,OAAO,OAAO,KAAK,MAAM,IAAI;CACxC;CAEA,qBAAqB,KAA0B;EAC7C,KAAKV,eAAe,oBAAoB,IAAI,OAAO,IAAI,GAAG;EAO1D,KAAKC,aAAa,MAAM,GAAG;CAC7B;CAEA,mBAAmB,QAAqB;EACtC,KAAKD,eAAe,OAAO,OAAO,KAAK,OAAO,GAAG;EACjD,KAAKC,aAAa,MAAM,MAAM;EAE9B,MAAM,UAAU,cAAc,OAAO,GAAG;EACxC,MAAM,UAAU,cAAc,OAAO,GAAG;EACxC,KAAKH,IAAI,GAAG,KAAK,eAAe,GAAG,OAAO,EAAE,aAAa,GAAG,OAAO,GAAG;EAGtE,KAAKgB,gBAAgB,YAAY,SAAS,OAAO;EAEjD,KAAKU,cAAc,OAAO,GAAG;EAC7B,KAAKD,YAAY,OAAO;EACxB,KAAKb,IAAI,OAAO,OAAO,KAAK,SAAS,OAAO;CAC9C;CAEA,iBAAiB,KAAgB;EAC/B,IAAI,IAAI,eACN,KAAKV,eAAe,oBAAoB,IAAI,OAAO,IAAI,aAAa;EAEtE,KAAKC,aAAa,MAAM,GAAG;EAC3B,MAAM,QAAQ,cAAc,IAAI,KAAK;EACrC,MAAM,EAAC,SAAQ,IAAI;EACnB,MAAM,OAAO,wBAAwB,OAAO,IAAI,MAAM;EACtD,KAAKH,IAAI,GAAG,KACV,eAAe,GAAG,KAAK,EAAE,OAAO,GAAG,IAAI,EAAE,GAAG,cAAc,IAAI,GAChE;EAGA,KAAKgB,gBAAgB,OAAO,OAAO,MAAM,IAAI,OAAO,MAAM,IAAI,QAAQ;EAEtE,IAAI,IAAI,UACN,KAAKC,kBAAkB;OAIvB,KAAKS,cAAc,IAAI,KAAK;EAE9B,KAAKd,IAAI,OAAO,IAAI,KAAK,OAAO,IAAI,MAAM;CAC5C;CAEA,oBAAoB,KAAmB;EAGrC,KAAKT,aAAa,MAAM,GAAG;EAE3B,MAAM,QAAQ,cAAc,IAAI,KAAK;EACrC,IAAI,UAAU,IAAI,IAAI;EACtB,MAAM,UAAU,IAAI,IAAI;EAExB,MAAM,qBAAqB,sBACzB,IAAI,IAAI,MACR,IAAI,IAAI,IACV;EAMA,IAAI,YAAY,WAAW,CAAC,oBAAoB;GAC9C,KAAKa,gBAAgB,OACnB,OACA,IAAI,IAAI,MACR,IAAI,IAAI,MACR,IAAI,IAAI,IACV;GACA,IAAI,QAAQ,IAAI,IAAI,KAAK,OAAO,MAAM,QAAQ,IAAI,IAAI,KAAK,OAAO,GAChE,KAAKU,cAAc,IAAI,KAAK;GAE9B,KAAKd,IAAI,OAAO,IAAI,KAAK,yBAAyB,IAAI,KAAK,IAAI,GAAG;GAClE;EACF;EAGA,IAAI,oBAAoB;GAWtB,MAAM,cAAc,wBAClB,OACA,IAAI,KACJ,gBACF;GACA,IAAI,YAAY,SAAS;IAIvB,KAAKZ,IAAI,GAAG,KACV,eAAe,GAAG,KAAK,EAAE,UAAU,GAAG,OAAO,EAAE,MAAM,GAAG,OAAO,GACjE;IACA,UAAU;GACZ;GACA,MAAM,YAAY,KAChB,WAAW,KAAKA,IAAI,IAAI,OAAO,KAAK,EAAE,MACpC,cAAa,UAAU,SAAS,KAClC,CACF;GAIA,MAAM,WAAW,KAAKA,IAAI,GACvB,QACC;;4BAGF,EACC,IAAmB,KAAK,EACxB,KAAK,EAAC,UAAS,GAAG;GACrB,MAAM,WAAW,OAAO;GACxB,MAAM,UAAU,OAAO,KAAK,UAAU,OAAO;GAC7C,MAAM,QAAQ;IACZ,yBAAyB;KACvB,GAAG;KACH,MAAM;KACN,SAAS,WAAW,UAAU,UAAU,QAAQ,SAAS,CACvD,QACA,WAAW,UAAU;MAAC,GAAG;MAAa,KAAK,KAAK;KAAG,IAAI,IACzD,CAAC;IACH,CAAC;IACD,eAAe,GAAG,QAAQ,EAAE,IAAI,QAAQ,IAAI,EAAE,EAAE,KAAK,GAAG,EAAE;kBAChD,QAAQ,IAAI,EAAE,EAAE,KAAK,GAAG,EAAE,QAAQ,GAAG,KAAK,EAAE;IACtD,cAAc,GAAG,KAAK,EAAE;IACxB,eAAe,GAAG,QAAQ,EAAE,aAAa,GAAG,KAAK,EAAE;IACnD,GAAG,SAAS,KAAI,QAAO,GAAG,IAAI,EAAE;GAClC;GACA,KAAKA,IAAI,GAAG,KAAK,MAAM,KAAK,EAAE,CAAC;EACjC;EACA,IAAI,YAAY,SACd,KAAKA,IAAI,GAAG,KACV,eAAe,GAAG,KAAK,EAAE,UAAU,GAAG,OAAO,EAAE,MAAM,GAAG,OAAO,GACjE;EAIF,KAAKgB,gBAAgB,OACnB,OACA,IAAI,IAAI,MACR,IAAI,IAAI,MACR,IAAI,IAAI,IACV;EAEA,KAAKU,cAAc,IAAI,KAAK;EAC5B,KAAKd,IAAI,OAAO,IAAI,KAAK,OAAO,IAAI,GAAG;CACzC;CAEA,kBAAkB,KAAiB;EACjC,MAAM,QAAQ,cAAc,IAAI,KAAK;EACrC,MAAM,EAAC,WAAU;EACjB,KAAKZ,IAAI,GAAG,KAAK,eAAe,GAAG,KAAK,EAAE,QAAQ,GAAG,MAAM,GAAG;EAG9D,KAAKgB,gBAAgB,aAAa,OAAO,MAAM;EAC/C,KAAKb,aAAa,MAAM,GAAG;EAE3B,KAAKuB,cAAc,IAAI,KAAK;EAC5B,KAAKd,IAAI,OAAO,IAAI,KAAK,OAAO,MAAM;CACxC;CAEA,iBAAiB,MAAiB;EAChC,KAAKV,eAAe,KAAK,KAAK,EAAE;EAChC,KAAKC,aAAa,MAAM,IAAI;EAE5B,MAAM,OAAO,cAAc,KAAK,EAAE;EAClC,KAAKH,IAAI,GAAG,KAAK,wBAAwB,GAAG,IAAI,GAAG;EAGnD,KAAKgB,gBAAgB,YAAY,IAAI;EAErC,KAAKS,YAAY,IAAI;EACrB,KAAKb,IAAI,OAAO,KAAK,KAAK,IAAI;CAChC;CAEA,mBAAmB,QAAqB;EACtC,MAAM,QAAQ,uBAAuB,OAAO,IAAI;EAChD,KAAKZ,IAAI,GAAG,KAAK,yBAAyB,KAAK,CAAC;EAIhD,KAAKyB,YAAY,MAAM,SAAS;EAChC,KAAKb,IAAI,OAAO,OAAO,KAAK,MAAM,IAAI;CACxC;CAEA,iBAAiB,MAAiB;EAChC,MAAM,OAAO,cAAc,KAAK,EAAE;EAClC,KAAKZ,IAAI,GAAG,KAAK,wBAAwB,GAAG,IAAI,GAAG;EACnD,KAAKY,IAAI,OAAO,KAAK,KAAK,IAAI;CAChC;CAEA,cAAc,OAAmB;EAC/B,KAAKV,eAAe,iBAAiB,OAAO,KAAKY,QAAQ;EACzD,KAAKW,YAAY,cAAc,KAAK,CAAC;CACvC;;;;CAKA,UACE,OACA,KACA,mBACA;EAIA,IAAI,KAAKrB,UAAU,aAAa,sBAAsB,KAAA,GAAW;GAC/D,KAAKH,WAAW,SACd,KAAKa,UACL,KAAKa,QACL,OACA,KACA,iBACF;GACA,KAAKC;EACP;CACF;CAEA,aAAa,OAAe,KAAiB,aAAwB;EAInE,IAAI,KAAKxB,UAAU,aAAa,aAAa,QAAQ;GACnD,KAAKH,WAAW,YAAY,KAAKa,UAAU,KAAKa,QAAQ,OAAO,GAAG;GAClE,KAAKC;EACP;CACF;CAEA,eAAe,OAAe;EAC5B,IAAI,KAAKxB,UAAU,WAAW;GAC5B,KAAKH,WAAW,cAAc,KAAKa,UAAU,KAAK;GAClD,KAAKc;EACP;CACF;CAEA,YAAY,OAAe;EACzB,KAAKC,iBAAiB;EACtB,IAAI,KAAKzB,UAAU,WAAW;GAC5B,KAAKH,WAAW,WAAW,KAAKa,UAAU,KAAK;GAC/C,KAAKc;EACP;EACA,KAAKX,kBAAkB;CACzB;CAEA,gBAAgB,EAAC,UAAU,WAAW,SAAS,aAA6B;EAC1E,MAAM,YAAY,cAAc,QAAQ;EACxC,MAAM,YAAY,KAAK,KAAKX,YAAY,IAAI,SAAS,CAAC;EACtD,MAAM,aAAa,SAAS,OAAO;EACnC,MAAM,OAAO,CAAC,GAAG,YAAY,GAAG,OAAO;EAGvC,MAAM,gBAAgB,CAAC,GAAG,MAAM,wBAAwB,EAAE,IAAI,EAAE,EAAE,KAAK,GAAG;EAC1E,MAAM,SAAS,MAAM,KAAK,EAAC,QAAQ,KAAK,SAAS,EAAC,CAAC,EAChD,KAAK,GAAG,EACR,KAAK,GAAG;EACX,MAAM,gBAAgB,WAAW,IAAI,EAAE,EAAE,KAAK,GAAG;EAEjD,IAAI,aAAa;EACjB,IAAI,UAAU;EACd,KAAK,MAAM,KAAK,WAAW;GACzB,MAAM,MAAM,QACV,OAAO,YAAY,KAAK,KAAK,GAAG,MAAM,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,GAChD,WACA,KAAKS,WACP;GACA,MAAM,SAAS,KAAKK,QAAQ,KAAK,EAAC,SAAQ,CAAC;GAC3C,MAAM,QAAQ,KAAKnB,WAAW,eAAe,WAAW,MAAM;GAC9D,IAAI,OAAO,OAAA,OAAiB,MAAM,eAAe,WAAW;IAC1D;IACA;GACF;GACA,MAAM,UACJ,OAAO,OAAA,MACH,KAAK,QACH,OAAM,MAAM,0BAA0B,MAAM,OAAO,SACrD,IACA;GACN,IAAI,QAAQ,WAAW,GAAG;IAExB;IACA;GACF;GACA,MAAM,cAAc,QAAQ,KAAI,QAAO,GAAG,GAAG,GAAG,EAAE,YAAY,GAAG,GAAG,GAAG;GACvE,KAAKD,IAAI,IACC;sBACM,GAAG,SAAS,EAAE,IAAI,cAAc,YAAY,OAAO;yBAChD,cAAc;0BACb,YAAY,KAAK,GAAG,EAAE;SAExC,GAAG,OAAO,OAAO,IAAI,GAAG,GACxB,SACF;GACA;EACF;EAEA,KAAKY,IAAI,QACP,cAAc,WAAW,iBAAiB,QAAQ,SAAS,WAC7D;CACF;CAEA;CAEA,yBAAyB,KAAwB;EAC/C,MAAM,EAAC,UAAU,SAAS,WAAU;EACpC,MAAM,YAAY,cAAc,QAAQ;EAExC,MAAM,OAAO,CAAC,GADK,SAAS,OAAO,SACN,GAAG,OAAO;EAEvC,MAAM,iBAAiB,KAAK,oBAAoB,YAAY,KAAKZ,IAAI,EAAE,CAAC;EACxE,KAAK,MAAM,OAAO,MAChB,eAAe,iBAAiB,WAAW,GAAG;EAEhD,KAAKG,aAAa,MAAM,GAAG;EAG3B,KAAKuB,cAAc,QAAQ;EAC3B,IAAI,QACF,KAAKI,qBAAqB;GAAC,OAAO;GAAW,SAAS;GAAM,GAAG;EAAM;EAEvE,KAAKlB,IAAI,OAAO,wBAAwB,WAAW;CAWrD;CAEA,cAAc,QAAuB,WAAiC;EACpE,IAAI,cAAc,KAAKE,UACrB,MAAM,IAAI,MACR,oBAAoB,UAAU,kCAC5B,KAAKA,SACN,IAAI,UAAU,MAAM,GACvB;EAEF,2BAA2B,KAAKd,KAAK,SAAS;EAE9C,IAAI,KAAK6B,gBAAgB;GACvB,MAAM,QAAQ,KAAK,IAAI;GACvB,KAAK7B,IAAI,GAAG,OAAO,UAAU;GAC7B,KAAKY,IAAI,OACP,yCAAyC,KAAK,IAAI,IAAI,MAAM,KAC9D;EACF;EAEA,IAAI,KAAKR,UAAU,gBACjB,KAAKJ,IAAI,OAAO;EAGlB,MAAM,YAAY,KAAK,IAAI,IAAI,KAAKa;EACpC,KAAKD,IAAI,QAAQ,gBAAgB,KAAKE,SAAS,IAAI,UAAU,KAAK;EAElE,OAAO;GACL;GACA,mBAAmB,KAAKgB;GACxB,eAAe,KAAKD;GACpB,kBAAkB,KAAKD,uBAAuB;GAC9C,sBAAsB,OAAO;EAC/B;CACF;CAEA,MAAM,IAAgB;EACpB,GAAG,OAAO,wBAAwB,KAAKd,UAAU;EACjD,KAAKd,IAAI,SAAS;EAIlB,KAAKM,YAAY,MAAM;CACzB;AACF;AAEA,SAAS,sBACP,SACA,SACS;CACT,OACE,QAAQ,aAAa,QAAQ,YAC7B,aAAa,OAAO,MAAM,aAAa,OAAO,KAC9C,cAAc,OAAO,MAAM,cAAc,OAAO;AAEpD;AAEA,SAAS,qBACP,KACA,EAAC,eACqB;CACtB,IAAI,CAAC,aAAa,QAChB;CAEF,OAAO,YAAY,QAAO,QAAO,OAAO,GAAG;AAC7C;AAEA,SAAS,0BAA0B,OAA0C;CAC3E,IAAI,OAAO,UAAU,UACnB,OAAO,OAAO,OAAO,IAAI,KAAK,MAAM,QAChC,qCACA,KAAA;CAGN,MAAM,YACJ,OAAO,UAAU,WACb,OAAO,WAAW,KAAK,IACvB,iBAAiB,aACf,MAAM,aACN,KAAA;CACR,IAAI,cAAc,KAAA,KAAa,YAAY,uBAEzC,OAAO,aADW,OAAO,UAAU,WAAW,WAAW,SAC3B,aAAa,UAAU,cAAc;AAIvE;;AAGA,SAAS,QAAQ,IAAyB;CACxC,IAAI;EACF,GAAG;EACH;CACF,SAAS,GAAG;EACV,OAAO,qBAAK,IAAI,MAAM,iBAAiB;CACzC;AACF;AAEA,SAAS,YAAY,KAAqB;CACxC,IAAI,eAAe,OACjB,OAAO;CAET,MAAM,wBAAQ,IAAI,MAAM;CACxB,MAAM,QAAQ;CACd,OAAO;AACT"}
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import type { LogContext } from '@rocicorp/logger';
|
|
2
2
|
import type { Source } from '../../types/streams.ts';
|
|
3
3
|
import { type ChangeStreamer } from '../change-streamer/change-streamer.ts';
|
|
4
|
-
import type
|
|
4
|
+
import { type ReplicationStatusPublisher } from './replication-status.ts';
|
|
5
5
|
import type { ReplicaState, ReplicatorMode } from './replicator.ts';
|
|
6
6
|
import type { WriteWorkerClient } from './write-worker-client.ts';
|
|
7
7
|
/**
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"incremental-sync.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/replicator/incremental-sync.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;
|
|
1
|
+
{"version":3,"file":"incremental-sync.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/replicator/incremental-sync.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAKjD,OAAO,KAAK,EAAC,MAAM,EAAC,MAAM,wBAAwB,CAAC;AAGnD,OAAO,EAGL,KAAK,cAAc,EAEpB,MAAM,uCAAuC,CAAC;AAK/C,OAAO,EAEL,KAAK,0BAA0B,EAChC,MAAM,yBAAyB,CAAC;AACjC,OAAO,KAAK,EAAC,YAAY,EAAE,cAAc,EAAC,MAAM,iBAAiB,CAAC;AAGlE,OAAO,KAAK,EAAC,iBAAiB,EAAC,MAAM,0BAA0B,CAAC;AAOhE;;;;;;GAMG;AACH,qBAAa,iBAAiB;;IAmB5B,YACE,EAAE,EAAE,UAAU,EACd,MAAM,EAAE,MAAM,EACd,EAAE,EAAE,MAAM,EACV,cAAc,EAAE,cAAc,EAC9B,MAAM,EAAE,iBAAiB,EACzB,IAAI,EAAE,cAAc,EACpB,eAAe,EAAE,0BAA0B,GAAG,IAAI,EAWnD;IAEK,GAAG,kBAkOR;IA6BD,SAAS,IAAI,MAAM,CAAC,YAAY,CAAC,CAEhC;IAED,IAAI,CAAC,EAAE,EAAE,UAAU,EAAE,GAAG,CAAC,EAAE,OAAO,QAIjC;CACF"}
|
|
@@ -1,10 +1,14 @@
|
|
|
1
1
|
import { AbortError } from "../../../../shared/src/abort-error.js";
|
|
2
|
+
import { mapPostgresToLiteIndex } from "../../db/pg-to-lite.js";
|
|
2
3
|
import { getOrCreateCounter } from "../../observability/metrics.js";
|
|
3
4
|
import { RunningState } from "../running-state.js";
|
|
5
|
+
import { IndexingProgress } from "./replication-status.js";
|
|
4
6
|
import { errorTypeToReadableName } from "../change-streamer/change-streamer.js";
|
|
5
7
|
import { Notifier } from "./notifier.js";
|
|
6
8
|
import { ReplicationReportRecorder } from "./reporter/recorder.js";
|
|
7
9
|
//#region ../zero-cache/src/services/replicator/incremental-sync.ts
|
|
10
|
+
var MAX_WORKER_BATCH_MESSAGES = 256;
|
|
11
|
+
var MAX_WORKER_BATCH_SIZE = 1024 * 1024;
|
|
8
12
|
/**
|
|
9
13
|
* The {@link IncrementalSyncer} manages a logical replication stream from upstream,
|
|
10
14
|
* handling application lifecycle events (start, stop) and retrying the
|
|
@@ -64,11 +68,29 @@ var IncrementalSyncer = class {
|
|
|
64
68
|
initial: watermark === initialWatermark,
|
|
65
69
|
logsChangeStream: false
|
|
66
70
|
});
|
|
67
|
-
this.#state.resetBackoff();
|
|
68
71
|
unregister = this.#state.cancelOnStop(downstream);
|
|
69
72
|
this.#statusPublisher?.publish(lc, "Replicating", `Replicating from ${watermark}`);
|
|
70
|
-
let
|
|
71
|
-
|
|
73
|
+
let backfill;
|
|
74
|
+
let writeBatch = [];
|
|
75
|
+
let writeBatchSize = 0;
|
|
76
|
+
let inTransaction = false;
|
|
77
|
+
let indexing;
|
|
78
|
+
const publishBackfillStatus = (table) => this.#statusPublisher?.publish(lc, "Replicating", `Backfilling ${table} table`, 3e3, () => backfill ? { downloadStatus: [{
|
|
79
|
+
...backfill.status,
|
|
80
|
+
table: backfill.table,
|
|
81
|
+
columns: backfill.columns
|
|
82
|
+
}] } : {});
|
|
83
|
+
const flushWrites = async () => {
|
|
84
|
+
if (writeBatch.length === 0) return;
|
|
85
|
+
const batch = writeBatch;
|
|
86
|
+
writeBatch = [];
|
|
87
|
+
writeBatchSize = 0;
|
|
88
|
+
const result = await this.#worker.processMessages(batch);
|
|
89
|
+
this.#handleResult(lc, result);
|
|
90
|
+
if (result?.completedBackfill) backfill = void 0;
|
|
91
|
+
};
|
|
92
|
+
for await (const { data: message, size } of downstream) {
|
|
93
|
+
this.#state.resetBackoff();
|
|
72
94
|
this.#replicationEvents.add(1);
|
|
73
95
|
switch (message[0]) {
|
|
74
96
|
case "status": {
|
|
@@ -91,21 +113,39 @@ var IncrementalSyncer = class {
|
|
|
91
113
|
default: {
|
|
92
114
|
const msg = message[1];
|
|
93
115
|
if (msg.tag === "backfill" && msg.status) {
|
|
94
|
-
const { status } = msg;
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
116
|
+
const { status, relation } = msg;
|
|
117
|
+
const first = !backfill;
|
|
118
|
+
backfill = {
|
|
119
|
+
status,
|
|
120
|
+
table: relation.name,
|
|
121
|
+
columns: [...relation.rowKey.columns, ...msg.columns]
|
|
122
|
+
};
|
|
123
|
+
if (first) publishBackfillStatus(relation.name);
|
|
124
|
+
}
|
|
125
|
+
if (msg.tag === "create-index" && this.#statusPublisher) {
|
|
126
|
+
await flushWrites();
|
|
127
|
+
const index = mapPostgresToLiteIndex(msg.spec);
|
|
128
|
+
indexing ??= new IndexingProgress();
|
|
129
|
+
indexing.start(index);
|
|
130
|
+
this.#statusPublisher.publish(lc, "Replicating", `Creating index ${index.name} on ${index.tableName}`, 3e3, indexing.state);
|
|
131
|
+
indexing.restartTimer();
|
|
132
|
+
writeBatch.push(message);
|
|
133
|
+
writeBatchSize += size;
|
|
134
|
+
await flushWrites();
|
|
135
|
+
const elapsed = indexing.finish();
|
|
136
|
+
lc.info?.(`Created index ${index.name} (${elapsed.toFixed(3)} ms)`);
|
|
137
|
+
this.#statusPublisher.publish(lc, "Replicating", `Created index ${index.name} on ${index.tableName}`, 0, indexing.state);
|
|
138
|
+
if (backfill) publishBackfillStatus(backfill.table);
|
|
139
|
+
break;
|
|
104
140
|
}
|
|
105
|
-
|
|
106
|
-
const
|
|
107
|
-
|
|
108
|
-
if (
|
|
141
|
+
indexing = void 0;
|
|
142
|
+
const type = message[0];
|
|
143
|
+
const invalidTransactionSequence = type === "begin" ? inTransaction : !inTransaction;
|
|
144
|
+
if (type === "begin") inTransaction = true;
|
|
145
|
+
else if (type === "commit" || type === "rollback") inTransaction = false;
|
|
146
|
+
writeBatch.push(message);
|
|
147
|
+
writeBatchSize += size;
|
|
148
|
+
if (type === "commit" || type === "rollback" || invalidTransactionSequence || writeBatch.length >= MAX_WORKER_BATCH_MESSAGES || writeBatchSize >= MAX_WORKER_BATCH_SIZE) await flushWrites();
|
|
109
149
|
break;
|
|
110
150
|
}
|
|
111
151
|
}
|
|
@@ -142,6 +182,7 @@ var IncrementalSyncer = class {
|
|
|
142
182
|
}
|
|
143
183
|
stop(lc, err) {
|
|
144
184
|
this.#state.stop(lc, err);
|
|
185
|
+
this.#worker.abort();
|
|
145
186
|
}
|
|
146
187
|
};
|
|
147
188
|
//#endregion
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"incremental-sync.js","names":["#lc","#taskID","#id","#changeStreamer","#worker","#mode","#statusPublisher","#notifier","#reporter","#state","#replicationEvents","#handleResult"],"sources":["../../../../../../zero-cache/src/services/replicator/incremental-sync.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {AbortError} from '../../../../shared/src/abort-error.ts';\nimport type {Enum} from '../../../../shared/src/enum.ts';\nimport {getOrCreateCounter} from '../../observability/metrics.ts';\nimport type {Source} from '../../types/streams.ts';\nimport type {DownloadStatus} from '../change-source/protocol/current.ts';\nimport type {ChangeStreamData} from '../change-source/protocol/current/downstream.ts';\nimport {\n errorTypeToReadableName,\n PROTOCOL_VERSION,\n type ChangeStreamer,\n type SerializedDownstream,\n} from '../change-streamer/change-streamer.ts';\nimport type * as ErrorType from '../change-streamer/error-type-enum.ts';\nimport {RunningState} from '../running-state.ts';\nimport type {CommitResult} from './change-processor.ts';\nimport {Notifier} from './notifier.ts';\nimport type {ReplicationStatusPublisher} from './replication-status.ts';\nimport type {ReplicaState, ReplicatorMode} from './replicator.ts';\nimport {ReplicationReportRecorder} from './reporter/recorder.ts';\nimport type {ReplicationReport} from './reporter/report-schema.ts';\nimport type {WriteWorkerClient} from './write-worker-client.ts';\n\ntype ErrorType = Enum<typeof ErrorType>;\n\n/**\n * The {@link IncrementalSyncer} manages a logical replication stream from upstream,\n * handling application lifecycle events (start, stop) and retrying the\n * connection with exponential backoff. The actual handling of the logical\n * replication messages is done by the {@link ChangeProcessor}, which runs\n * in a worker thread via the {@link WriteWorkerClient}.\n */\nexport class IncrementalSyncer {\n readonly #lc: LogContext;\n readonly #taskID: string;\n readonly #id: string;\n readonly #changeStreamer: ChangeStreamer;\n readonly #worker: WriteWorkerClient;\n readonly #mode: ReplicatorMode;\n readonly #statusPublisher: ReplicationStatusPublisher | null;\n readonly #notifier: Notifier;\n readonly #reporter: ReplicationReportRecorder;\n\n readonly #state = new RunningState('IncrementalSyncer');\n\n readonly #replicationEvents = getOrCreateCounter(\n 'replication',\n 'events',\n 'Number of replication events processed',\n );\n\n constructor(\n lc: LogContext,\n taskID: string,\n id: string,\n changeStreamer: ChangeStreamer,\n worker: WriteWorkerClient,\n mode: ReplicatorMode,\n statusPublisher: ReplicationStatusPublisher | null,\n ) {\n this.#lc = lc;\n this.#taskID = taskID;\n this.#id = id;\n this.#changeStreamer = changeStreamer;\n this.#worker = worker;\n this.#mode = mode;\n this.#statusPublisher = statusPublisher;\n this.#notifier = new Notifier();\n this.#reporter = new ReplicationReportRecorder(lc);\n }\n\n async run() {\n const lc = this.#lc;\n let workerError: Error | undefined;\n this.#worker.onError(err => {\n workerError ??= err;\n this.#state.stop(lc, err);\n });\n lc.info?.(`Starting IncrementalSyncer`);\n const {watermark: initialWatermark} =\n await this.#worker.getSubscriptionState();\n\n // Notify any waiting subscribers that the replica is ready to be read.\n // This initial notification intentionally omits replicaReadyTimeMs because\n // it represents already-current state, not newly-unserved work.\n void this.#notifier.notifySubscribers({\n state: 'version-ready',\n watermark: initialWatermark,\n });\n\n while (this.#state.shouldRun()) {\n const {replicaVersion, watermark} =\n await this.#worker.getSubscriptionState();\n\n let downstream: Source<SerializedDownstream> | undefined;\n let unregister = () => {};\n let err: unknown | undefined;\n\n try {\n downstream = await this.#changeStreamer.subscribe({\n protocolVersion: PROTOCOL_VERSION,\n taskID: this.#taskID,\n id: this.#id,\n mode: this.#mode,\n watermark,\n replicaVersion,\n initial: watermark === initialWatermark,\n // The SQLite change log is written by the change-streamer itself, so\n // no replicator logs the change stream any more. The parameter stays\n // on the wire for change-streamers that still exclude a writer from\n // SQLite catchup.\n logsChangeStream: false,\n });\n this.#state.resetBackoff();\n unregister = this.#state.cancelOnStop(downstream);\n this.#statusPublisher?.publish(\n lc,\n 'Replicating',\n `Replicating from ${watermark}`,\n );\n\n let backfillStatus: DownloadStatus | undefined;\n\n for await (const {data: message} of downstream) {\n this.#replicationEvents.add(1);\n switch (message[0]) {\n case 'status': {\n const {lagReport} = message[1];\n if (lagReport) {\n const report: ReplicationReport = {\n nextSendTimeMs: lagReport.nextSendTimeMs,\n };\n if (lagReport.lastTimings) {\n report.lastTimings = {\n ...lagReport.lastTimings,\n replicateTimeMs: Date.now(),\n };\n }\n this.#reporter.record(report);\n }\n break;\n }\n case 'error': {\n // Signal from the replication-manager that the view-syncer must\n // shut down and restore a new backup from litestream.\n const {type, message: msg} = message[1];\n this.stop(\n lc,\n // Note: The AbortError indicates a clean / intentional shutdown.\n new AbortError(\n `${errorTypeToReadableName(type as ErrorType)}: ${msg}`,\n ),\n );\n break;\n }\n default: {\n const msg = message[1];\n if (msg.tag === 'backfill' && msg.status) {\n const {status} = msg;\n if (!backfillStatus) {\n // Start publishing the status every 3 seconds.\n backfillStatus = status;\n this.#statusPublisher?.publish(\n lc,\n 'Replicating',\n `Backfilling ${msg.relation.name} table`,\n 3000,\n () =>\n backfillStatus\n ? {\n downloadStatus: [\n {\n ...backfillStatus,\n table: msg.relation.name,\n columns: [\n ...msg.relation.rowKey.columns,\n ...msg.columns,\n ],\n },\n ],\n }\n : {},\n );\n }\n backfillStatus = status; // Update the current status\n }\n\n const data = message as ChangeStreamData;\n const result = await this.#worker.processMessage(data);\n\n this.#handleResult(lc, result);\n if (result?.completedBackfill) {\n backfillStatus = undefined;\n }\n break;\n }\n }\n }\n this.#worker.abort();\n } catch (e) {\n err = e;\n this.#worker.abort();\n } finally {\n downstream?.cancel();\n unregister();\n this.#statusPublisher?.stop();\n }\n await this.#state.backoff(lc, err);\n }\n lc.info?.('IncrementalSyncer stopped');\n if (workerError) {\n throw workerError;\n }\n }\n\n #handleResult(lc: LogContext, result: CommitResult | null) {\n if (!result) {\n return;\n }\n if (result.completedBackfill) {\n // Publish the final status\n const status = result.completedBackfill;\n this.#statusPublisher?.publish(\n lc,\n 'Replicating',\n `Backfilled ${status.table} table`,\n 0,\n () => ({downloadStatus: [status]}),\n );\n } else if (result.schemaUpdated) {\n this.#statusPublisher?.publish(lc, 'Replicating', 'Schema updated');\n }\n if (result.watermark && result.changeLogUpdated) {\n void this.#notifier.notifySubscribers({\n state: 'version-ready',\n watermark: result.watermark,\n replicaReadyTimeMs: Date.now(),\n upstreamCommitTimeMs: result.upstreamCommitTimeMs,\n });\n }\n }\n\n subscribe(): Source<ReplicaState> {\n return this.#notifier.subscribe();\n }\n\n stop(lc: LogContext, err?: unknown) {\n this.#state.stop(lc, err);\n }\n}\n"],"mappings":";;;;;;;;;;;;;;AAgCA,IAAa,oBAAb,MAA+B;CAC7B;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CAEA,SAAkB,IAAI,aAAa,mBAAmB;CAEtD,qBAA8B,mBAC5B,eACA,UACA,wCACF;CAEA,YACE,IACA,QACA,IACA,gBACA,QACA,MACA,iBACA;EACA,KAAKA,MAAM;EACX,KAAKC,UAAU;EACf,KAAKC,MAAM;EACX,KAAKC,kBAAkB;EACvB,KAAKC,UAAU;EACf,KAAKC,QAAQ;EACb,KAAKC,mBAAmB;EACxB,KAAKC,YAAY,IAAI,SAAS;EAC9B,KAAKC,YAAY,IAAI,0BAA0B,EAAE;CACnD;CAEA,MAAM,MAAM;EACV,MAAM,KAAK,KAAKR;EAChB,IAAI;EACJ,KAAKI,QAAQ,SAAQ,QAAO;GAC1B,gBAAgB;GAChB,KAAKK,OAAO,KAAK,IAAI,GAAG;EAC1B,CAAC;EACD,GAAG,OAAO,4BAA4B;EACtC,MAAM,EAAC,WAAW,qBAChB,MAAM,KAAKL,QAAQ,qBAAqB;EAK1C,KAAUG,UAAU,kBAAkB;GACpC,OAAO;GACP,WAAW;EACb,CAAC;EAED,OAAO,KAAKE,OAAO,UAAU,GAAG;GAC9B,MAAM,EAAC,gBAAgB,cACrB,MAAM,KAAKL,QAAQ,qBAAqB;GAE1C,IAAI;GACJ,IAAI,mBAAmB,CAAC;GACxB,IAAI;GAEJ,IAAI;IACF,aAAa,MAAM,KAAKD,gBAAgB,UAAU;KAChD,iBAAA;KACA,QAAQ,KAAKF;KACb,IAAI,KAAKC;KACT,MAAM,KAAKG;KACX;KACA;KACA,SAAS,cAAc;KAKvB,kBAAkB;IACpB,CAAC;IACD,KAAKI,OAAO,aAAa;IACzB,aAAa,KAAKA,OAAO,aAAa,UAAU;IAChD,KAAKH,kBAAkB,QACrB,IACA,eACA,oBAAoB,WACtB;IAEA,IAAI;IAEJ,WAAW,MAAM,EAAC,MAAM,aAAY,YAAY;KAC9C,KAAKI,mBAAmB,IAAI,CAAC;KAC7B,QAAQ,QAAQ,IAAhB;MACE,KAAK,UAAU;OACb,MAAM,EAAC,cAAa,QAAQ;OAC5B,IAAI,WAAW;QACb,MAAM,SAA4B,EAChC,gBAAgB,UAAU,eAC5B;QACA,IAAI,UAAU,aACZ,OAAO,cAAc;SACnB,GAAG,UAAU;SACb,iBAAiB,KAAK,IAAI;QAC5B;QAEF,KAAKF,UAAU,OAAO,MAAM;OAC9B;OACA;MACF;MACA,KAAK,SAAS;OAGZ,MAAM,EAAC,MAAM,SAAS,QAAO,QAAQ;OACrC,KAAK,KACH,IAEA,IAAI,WACF,GAAG,wBAAwB,IAAiB,EAAE,IAAI,KACpD,CACF;OACA;MACF;MACA,SAAS;OACP,MAAM,MAAM,QAAQ;OACpB,IAAI,IAAI,QAAQ,cAAc,IAAI,QAAQ;QACxC,MAAM,EAAC,WAAU;QACjB,IAAI,CAAC,gBAAgB;SAEnB,iBAAiB;SACjB,KAAKF,kBAAkB,QACrB,IACA,eACA,eAAe,IAAI,SAAS,KAAK,SACjC,WAEE,iBACI,EACE,gBAAgB,CACd;UACE,GAAG;UACH,OAAO,IAAI,SAAS;UACpB,SAAS,CACP,GAAG,IAAI,SAAS,OAAO,SACvB,GAAG,IAAI,OACT;SACF,CACF,EACF,IACA,CAAC,CACT;QACF;QACA,iBAAiB;OACnB;OAEA,MAAM,OAAO;OACb,MAAM,SAAS,MAAM,KAAKF,QAAQ,eAAe,IAAI;OAErD,KAAKO,cAAc,IAAI,MAAM;OAC7B,IAAI,QAAQ,mBACV,iBAAiB,KAAA;OAEnB;MACF;KACF;IACF;IACA,KAAKP,QAAQ,MAAM;GACrB,SAAS,GAAG;IACV,MAAM;IACN,KAAKA,QAAQ,MAAM;GACrB,UAAU;IACR,YAAY,OAAO;IACnB,WAAW;IACX,KAAKE,kBAAkB,KAAK;GAC9B;GACA,MAAM,KAAKG,OAAO,QAAQ,IAAI,GAAG;EACnC;EACA,GAAG,OAAO,2BAA2B;EACrC,IAAI,aACF,MAAM;CAEV;CAEA,cAAc,IAAgB,QAA6B;EACzD,IAAI,CAAC,QACH;EAEF,IAAI,OAAO,mBAAmB;GAE5B,MAAM,SAAS,OAAO;GACtB,KAAKH,kBAAkB,QACrB,IACA,eACA,cAAc,OAAO,MAAM,SAC3B,UACO,EAAC,gBAAgB,CAAC,MAAM,EAAC,EAClC;EACF,OAAO,IAAI,OAAO,eAChB,KAAKA,kBAAkB,QAAQ,IAAI,eAAe,gBAAgB;EAEpE,IAAI,OAAO,aAAa,OAAO,kBAC7B,KAAUC,UAAU,kBAAkB;GACpC,OAAO;GACP,WAAW,OAAO;GAClB,oBAAoB,KAAK,IAAI;GAC7B,sBAAsB,OAAO;EAC/B,CAAC;CAEL;CAEA,YAAkC;EAChC,OAAO,KAAKA,UAAU,UAAU;CAClC;CAEA,KAAK,IAAgB,KAAe;EAClC,KAAKE,OAAO,KAAK,IAAI,GAAG;CAC1B;AACF"}
|
|
1
|
+
{"version":3,"file":"incremental-sync.js","names":["#lc","#taskID","#id","#changeStreamer","#worker","#mode","#statusPublisher","#notifier","#reporter","#state","#replicationEvents","#handleResult"],"sources":["../../../../../../zero-cache/src/services/replicator/incremental-sync.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {AbortError} from '../../../../shared/src/abort-error.ts';\nimport type {Enum} from '../../../../shared/src/enum.ts';\nimport {mapPostgresToLiteIndex} from '../../db/pg-to-lite.ts';\nimport {getOrCreateCounter} from '../../observability/metrics.ts';\nimport type {Source} from '../../types/streams.ts';\nimport type {DownloadStatus} from '../change-source/protocol/current.ts';\nimport type {ChangeStreamData} from '../change-source/protocol/current/downstream.ts';\nimport {\n errorTypeToReadableName,\n PROTOCOL_VERSION,\n type ChangeStreamer,\n type SizedDownstream,\n} from '../change-streamer/change-streamer.ts';\nimport type * as ErrorType from '../change-streamer/error-type-enum.ts';\nimport {RunningState} from '../running-state.ts';\nimport type {CommitResult} from './change-processor.ts';\nimport {Notifier} from './notifier.ts';\nimport {\n IndexingProgress,\n type ReplicationStatusPublisher,\n} from './replication-status.ts';\nimport type {ReplicaState, ReplicatorMode} from './replicator.ts';\nimport {ReplicationReportRecorder} from './reporter/recorder.ts';\nimport type {ReplicationReport} from './reporter/report-schema.ts';\nimport type {WriteWorkerClient} from './write-worker-client.ts';\n\ntype ErrorType = Enum<typeof ErrorType>;\n\nconst MAX_WORKER_BATCH_MESSAGES = 256;\nconst MAX_WORKER_BATCH_SIZE = 1024 * 1024;\n\n/**\n * The {@link IncrementalSyncer} manages a logical replication stream from upstream,\n * handling application lifecycle events (start, stop) and retrying the\n * connection with exponential backoff. The actual handling of the logical\n * replication messages is done by the {@link ChangeProcessor}, which runs\n * in a worker thread via the {@link WriteWorkerClient}.\n */\nexport class IncrementalSyncer {\n readonly #lc: LogContext;\n readonly #taskID: string;\n readonly #id: string;\n readonly #changeStreamer: ChangeStreamer;\n readonly #worker: WriteWorkerClient;\n readonly #mode: ReplicatorMode;\n readonly #statusPublisher: ReplicationStatusPublisher | null;\n readonly #notifier: Notifier;\n readonly #reporter: ReplicationReportRecorder;\n\n readonly #state = new RunningState('IncrementalSyncer');\n\n readonly #replicationEvents = getOrCreateCounter(\n 'replication',\n 'events',\n 'Number of replication events processed',\n );\n\n constructor(\n lc: LogContext,\n taskID: string,\n id: string,\n changeStreamer: ChangeStreamer,\n worker: WriteWorkerClient,\n mode: ReplicatorMode,\n statusPublisher: ReplicationStatusPublisher | null,\n ) {\n this.#lc = lc;\n this.#taskID = taskID;\n this.#id = id;\n this.#changeStreamer = changeStreamer;\n this.#worker = worker;\n this.#mode = mode;\n this.#statusPublisher = statusPublisher;\n this.#notifier = new Notifier();\n this.#reporter = new ReplicationReportRecorder(lc);\n }\n\n async run() {\n const lc = this.#lc;\n let workerError: Error | undefined;\n this.#worker.onError(err => {\n workerError ??= err;\n this.#state.stop(lc, err);\n });\n lc.info?.(`Starting IncrementalSyncer`);\n const {watermark: initialWatermark} =\n await this.#worker.getSubscriptionState();\n\n // Notify any waiting subscribers that the replica is ready to be read.\n // This initial notification intentionally omits replicaReadyTimeMs because\n // it represents already-current state, not newly-unserved work.\n void this.#notifier.notifySubscribers({\n state: 'version-ready',\n watermark: initialWatermark,\n });\n\n while (this.#state.shouldRun()) {\n const {replicaVersion, watermark} =\n await this.#worker.getSubscriptionState();\n\n let downstream: Source<SizedDownstream> | undefined;\n let unregister = () => {};\n let err: unknown | undefined;\n\n try {\n downstream = await this.#changeStreamer.subscribe({\n protocolVersion: PROTOCOL_VERSION,\n taskID: this.#taskID,\n id: this.#id,\n mode: this.#mode,\n watermark,\n replicaVersion,\n initial: watermark === initialWatermark,\n // The SQLite change log is written by the change-streamer itself, so\n // no replicator logs the change stream any more. The parameter stays\n // on the wire for change-streamers that still exclude a writer from\n // SQLite catchup.\n logsChangeStream: false,\n });\n unregister = this.#state.cancelOnStop(downstream);\n this.#statusPublisher?.publish(\n lc,\n 'Replicating',\n `Replicating from ${watermark}`,\n );\n\n let backfill:\n | {status: DownloadStatus; table: string; columns: string[]}\n | undefined;\n let writeBatch: ChangeStreamData[] = [];\n let writeBatchSize = 0;\n let inTransaction = false;\n // Indexes created in the current run of consecutive index creations.\n let indexing: IndexingProgress | undefined;\n\n const publishBackfillStatus = (table: string) =>\n this.#statusPublisher?.publish(\n lc,\n 'Replicating',\n `Backfilling ${table} table`,\n 3000,\n () =>\n backfill\n ? {\n downloadStatus: [\n {\n ...backfill.status,\n table: backfill.table,\n columns: backfill.columns,\n },\n ],\n }\n : {},\n );\n\n const flushWrites = async () => {\n if (writeBatch.length === 0) {\n return;\n }\n const batch = writeBatch;\n writeBatch = [];\n writeBatchSize = 0;\n\n const result = await this.#worker.processMessages(batch);\n this.#handleResult(lc, result);\n if (result?.completedBackfill) {\n backfill = undefined;\n }\n };\n\n for await (const {data: message, size} of downstream) {\n // Only reset backoff after receiving an actual message.\n this.#state.resetBackoff();\n\n this.#replicationEvents.add(1);\n switch (message[0]) {\n case 'status': {\n const {lagReport} = message[1];\n if (lagReport) {\n const report: ReplicationReport = {\n nextSendTimeMs: lagReport.nextSendTimeMs,\n };\n if (lagReport.lastTimings) {\n report.lastTimings = {\n ...lagReport.lastTimings,\n replicateTimeMs: Date.now(),\n };\n }\n this.#reporter.record(report);\n }\n break;\n }\n case 'error': {\n // Signal from the replication-manager that the view-syncer must\n // shut down and restore a new backup from litestream.\n const {type, message: msg} = message[1];\n this.stop(\n lc,\n // Note: The AbortError indicates a clean / intentional shutdown.\n new AbortError(\n `${errorTypeToReadableName(type as ErrorType)}: ${msg}`,\n ),\n );\n break;\n }\n default: {\n const msg = message[1];\n if (msg.tag === 'backfill' && msg.status) {\n const {status, relation} = msg;\n const first = !backfill;\n backfill = {\n status, // Update the current status\n table: relation.name,\n columns: [...relation.rowKey.columns, ...msg.columns],\n };\n if (first) {\n // Start publishing the status every 3 seconds.\n publishBackfillStatus(relation.name);\n }\n }\n\n if (msg.tag === 'create-index' && this.#statusPublisher) {\n // Creating an index on an existing table can take a long\n // time. Flush preceding changes so that the index creation\n // is processed (and timed) on its own.\n await flushWrites();\n const index = mapPostgresToLiteIndex(msg.spec);\n indexing ??= new IndexingProgress();\n indexing.start(index);\n this.#statusPublisher.publish(\n lc,\n 'Replicating',\n `Creating index ${index.name} on ${index.tableName}`,\n 3000,\n indexing.state,\n );\n // Exclude the time spent publishing the status.\n indexing.restartTimer();\n writeBatch.push(message as ChangeStreamData);\n writeBatchSize += size;\n await flushWrites();\n const elapsed = indexing.finish();\n lc.info?.(\n `Created index ${index.name} (${elapsed.toFixed(3)} ms)`,\n );\n this.#statusPublisher.publish(\n lc,\n 'Replicating',\n `Created index ${index.name} on ${index.tableName}`,\n 0,\n indexing.state,\n );\n if (backfill) {\n // Resume reporting the progress of the ongoing backfill.\n publishBackfillStatus(backfill.table);\n }\n break;\n }\n indexing = undefined;\n\n const type = message[0];\n const invalidTransactionSequence =\n type === 'begin' ? inTransaction : !inTransaction;\n if (type === 'begin') {\n inTransaction = true;\n } else if (type === 'commit' || type === 'rollback') {\n inTransaction = false;\n }\n\n writeBatch.push(message as ChangeStreamData);\n writeBatchSize += size;\n if (\n // Waiting here ensures that consuming the commit, which ACKs\n // it upstream, only happens after the SQLite commit finishes.\n type === 'commit' ||\n type === 'rollback' ||\n // Promptly surface malformed transaction sequences instead\n // of leaving a partial batch waiting for another message.\n invalidTransactionSequence ||\n writeBatch.length >= MAX_WORKER_BATCH_MESSAGES ||\n writeBatchSize >= MAX_WORKER_BATCH_SIZE\n ) {\n await flushWrites();\n }\n break;\n }\n }\n }\n this.#worker.abort();\n } catch (e) {\n err = e;\n this.#worker.abort();\n } finally {\n downstream?.cancel();\n unregister();\n this.#statusPublisher?.stop();\n }\n await this.#state.backoff(lc, err);\n }\n lc.info?.('IncrementalSyncer stopped');\n if (workerError) {\n throw workerError;\n }\n }\n\n #handleResult(lc: LogContext, result: CommitResult | null) {\n if (!result) {\n return;\n }\n if (result.completedBackfill) {\n // Publish the final status\n const status = result.completedBackfill;\n this.#statusPublisher?.publish(\n lc,\n 'Replicating',\n `Backfilled ${status.table} table`,\n 0,\n () => ({downloadStatus: [status]}),\n );\n } else if (result.schemaUpdated) {\n this.#statusPublisher?.publish(lc, 'Replicating', 'Schema updated');\n }\n if (result.watermark && result.changeLogUpdated) {\n void this.#notifier.notifySubscribers({\n state: 'version-ready',\n watermark: result.watermark,\n replicaReadyTimeMs: Date.now(),\n upstreamCommitTimeMs: result.upstreamCommitTimeMs,\n });\n }\n }\n\n subscribe(): Source<ReplicaState> {\n return this.#notifier.subscribe();\n }\n\n stop(lc: LogContext, err?: unknown) {\n this.#state.stop(lc, err);\n // Abort any polling loop that the write worker may be in.\n this.#worker.abort();\n }\n}\n"],"mappings":";;;;;;;;;AA6BA,IAAM,4BAA4B;AAClC,IAAM,wBAAwB,OAAO;;;;;;;;AASrC,IAAa,oBAAb,MAA+B;CAC7B;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CAEA,SAAkB,IAAI,aAAa,mBAAmB;CAEtD,qBAA8B,mBAC5B,eACA,UACA,wCACF;CAEA,YACE,IACA,QACA,IACA,gBACA,QACA,MACA,iBACA;EACA,KAAKA,MAAM;EACX,KAAKC,UAAU;EACf,KAAKC,MAAM;EACX,KAAKC,kBAAkB;EACvB,KAAKC,UAAU;EACf,KAAKC,QAAQ;EACb,KAAKC,mBAAmB;EACxB,KAAKC,YAAY,IAAI,SAAS;EAC9B,KAAKC,YAAY,IAAI,0BAA0B,EAAE;CACnD;CAEA,MAAM,MAAM;EACV,MAAM,KAAK,KAAKR;EAChB,IAAI;EACJ,KAAKI,QAAQ,SAAQ,QAAO;GAC1B,gBAAgB;GAChB,KAAKK,OAAO,KAAK,IAAI,GAAG;EAC1B,CAAC;EACD,GAAG,OAAO,4BAA4B;EACtC,MAAM,EAAC,WAAW,qBAChB,MAAM,KAAKL,QAAQ,qBAAqB;EAK1C,KAAUG,UAAU,kBAAkB;GACpC,OAAO;GACP,WAAW;EACb,CAAC;EAED,OAAO,KAAKE,OAAO,UAAU,GAAG;GAC9B,MAAM,EAAC,gBAAgB,cACrB,MAAM,KAAKL,QAAQ,qBAAqB;GAE1C,IAAI;GACJ,IAAI,mBAAmB,CAAC;GACxB,IAAI;GAEJ,IAAI;IACF,aAAa,MAAM,KAAKD,gBAAgB,UAAU;KAChD,iBAAA;KACA,QAAQ,KAAKF;KACb,IAAI,KAAKC;KACT,MAAM,KAAKG;KACX;KACA;KACA,SAAS,cAAc;KAKvB,kBAAkB;IACpB,CAAC;IACD,aAAa,KAAKI,OAAO,aAAa,UAAU;IAChD,KAAKH,kBAAkB,QACrB,IACA,eACA,oBAAoB,WACtB;IAEA,IAAI;IAGJ,IAAI,aAAiC,CAAC;IACtC,IAAI,iBAAiB;IACrB,IAAI,gBAAgB;IAEpB,IAAI;IAEJ,MAAM,yBAAyB,UAC7B,KAAKA,kBAAkB,QACrB,IACA,eACA,eAAe,MAAM,SACrB,WAEE,WACI,EACE,gBAAgB,CACd;KACE,GAAG,SAAS;KACZ,OAAO,SAAS;KAChB,SAAS,SAAS;IACpB,CACF,EACF,IACA,CAAC,CACT;IAEF,MAAM,cAAc,YAAY;KAC9B,IAAI,WAAW,WAAW,GACxB;KAEF,MAAM,QAAQ;KACd,aAAa,CAAC;KACd,iBAAiB;KAEjB,MAAM,SAAS,MAAM,KAAKF,QAAQ,gBAAgB,KAAK;KACvD,KAAKO,cAAc,IAAI,MAAM;KAC7B,IAAI,QAAQ,mBACV,WAAW,KAAA;IAEf;IAEA,WAAW,MAAM,EAAC,MAAM,SAAS,UAAS,YAAY;KAEpD,KAAKF,OAAO,aAAa;KAEzB,KAAKC,mBAAmB,IAAI,CAAC;KAC7B,QAAQ,QAAQ,IAAhB;MACE,KAAK,UAAU;OACb,MAAM,EAAC,cAAa,QAAQ;OAC5B,IAAI,WAAW;QACb,MAAM,SAA4B,EAChC,gBAAgB,UAAU,eAC5B;QACA,IAAI,UAAU,aACZ,OAAO,cAAc;SACnB,GAAG,UAAU;SACb,iBAAiB,KAAK,IAAI;QAC5B;QAEF,KAAKF,UAAU,OAAO,MAAM;OAC9B;OACA;MACF;MACA,KAAK,SAAS;OAGZ,MAAM,EAAC,MAAM,SAAS,QAAO,QAAQ;OACrC,KAAK,KACH,IAEA,IAAI,WACF,GAAG,wBAAwB,IAAiB,EAAE,IAAI,KACpD,CACF;OACA;MACF;MACA,SAAS;OACP,MAAM,MAAM,QAAQ;OACpB,IAAI,IAAI,QAAQ,cAAc,IAAI,QAAQ;QACxC,MAAM,EAAC,QAAQ,aAAY;QAC3B,MAAM,QAAQ,CAAC;QACf,WAAW;SACT;SACA,OAAO,SAAS;SAChB,SAAS,CAAC,GAAG,SAAS,OAAO,SAAS,GAAG,IAAI,OAAO;QACtD;QACA,IAAI,OAEF,sBAAsB,SAAS,IAAI;OAEvC;OAEA,IAAI,IAAI,QAAQ,kBAAkB,KAAKF,kBAAkB;QAIvD,MAAM,YAAY;QAClB,MAAM,QAAQ,uBAAuB,IAAI,IAAI;QAC7C,aAAa,IAAI,iBAAiB;QAClC,SAAS,MAAM,KAAK;QACpB,KAAKA,iBAAiB,QACpB,IACA,eACA,kBAAkB,MAAM,KAAK,MAAM,MAAM,aACzC,KACA,SAAS,KACX;QAEA,SAAS,aAAa;QACtB,WAAW,KAAK,OAA2B;QAC3C,kBAAkB;QAClB,MAAM,YAAY;QAClB,MAAM,UAAU,SAAS,OAAO;QAChC,GAAG,OACD,iBAAiB,MAAM,KAAK,IAAI,QAAQ,QAAQ,CAAC,EAAE,KACrD;QACA,KAAKA,iBAAiB,QACpB,IACA,eACA,iBAAiB,MAAM,KAAK,MAAM,MAAM,aACxC,GACA,SAAS,KACX;QACA,IAAI,UAEF,sBAAsB,SAAS,KAAK;QAEtC;OACF;OACA,WAAW,KAAA;OAEX,MAAM,OAAO,QAAQ;OACrB,MAAM,6BACJ,SAAS,UAAU,gBAAgB,CAAC;OACtC,IAAI,SAAS,SACX,gBAAgB;YACX,IAAI,SAAS,YAAY,SAAS,YACvC,gBAAgB;OAGlB,WAAW,KAAK,OAA2B;OAC3C,kBAAkB;OAClB,IAGE,SAAS,YACT,SAAS,cAGT,8BACA,WAAW,UAAU,6BACrB,kBAAkB,uBAElB,MAAM,YAAY;OAEpB;MACF;KACF;IACF;IACA,KAAKF,QAAQ,MAAM;GACrB,SAAS,GAAG;IACV,MAAM;IACN,KAAKA,QAAQ,MAAM;GACrB,UAAU;IACR,YAAY,OAAO;IACnB,WAAW;IACX,KAAKE,kBAAkB,KAAK;GAC9B;GACA,MAAM,KAAKG,OAAO,QAAQ,IAAI,GAAG;EACnC;EACA,GAAG,OAAO,2BAA2B;EACrC,IAAI,aACF,MAAM;CAEV;CAEA,cAAc,IAAgB,QAA6B;EACzD,IAAI,CAAC,QACH;EAEF,IAAI,OAAO,mBAAmB;GAE5B,MAAM,SAAS,OAAO;GACtB,KAAKH,kBAAkB,QACrB,IACA,eACA,cAAc,OAAO,MAAM,SAC3B,UACO,EAAC,gBAAgB,CAAC,MAAM,EAAC,EAClC;EACF,OAAO,IAAI,OAAO,eAChB,KAAKA,kBAAkB,QAAQ,IAAI,eAAe,gBAAgB;EAEpE,IAAI,OAAO,aAAa,OAAO,kBAC7B,KAAUC,UAAU,kBAAkB;GACpC,OAAO;GACP,WAAW,OAAO;GAClB,oBAAoB,KAAK,IAAI;GAC7B,sBAAsB,OAAO;EAC/B,CAAC;CAEL;CAEA,YAAkC;EAChC,OAAO,KAAKA,UAAU,UAAU;CAClC;CAEA,KAAK,IAAgB,KAAe;EAClC,KAAKE,OAAO,KAAK,IAAI,GAAG;EAExB,KAAKL,QAAQ,MAAM;CACrB;AACF"}
|