@rocicorp/zero 1.11.0-canary.1 → 1.11.0-canary.11
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/out/analyze-query/src/bin-analyze.js +9 -8
- package/out/analyze-query/src/bin-analyze.js.map +1 -1
- package/out/replicache/src/btree/node.d.ts.map +1 -1
- package/out/replicache/src/btree/node.js +3 -6
- package/out/replicache/src/btree/node.js.map +1 -1
- package/out/replicache/src/btree/read.d.ts +12 -1
- package/out/replicache/src/btree/read.d.ts.map +1 -1
- package/out/replicache/src/btree/read.js +24 -12
- package/out/replicache/src/btree/read.js.map +1 -1
- package/out/replicache/src/dag/gc.d.ts.map +1 -1
- package/out/replicache/src/dag/gc.js +8 -11
- package/out/replicache/src/dag/gc.js.map +1 -1
- package/out/replicache/src/dag/invalid-ref-count-error.d.ts +12 -0
- package/out/replicache/src/dag/invalid-ref-count-error.d.ts.map +1 -0
- package/out/replicache/src/dag/invalid-ref-count-error.js +19 -0
- package/out/replicache/src/dag/invalid-ref-count-error.js.map +1 -0
- package/out/replicache/src/dag/store-impl.d.ts +12 -2
- package/out/replicache/src/dag/store-impl.d.ts.map +1 -1
- package/out/replicache/src/dag/store-impl.js +22 -8
- package/out/replicache/src/dag/store-impl.js.map +1 -1
- package/out/replicache/src/db/rebase.d.ts +14 -2
- package/out/replicache/src/db/rebase.d.ts.map +1 -1
- package/out/replicache/src/db/rebase.js +27 -5
- package/out/replicache/src/db/rebase.js.map +1 -1
- package/out/replicache/src/db/write.js.map +1 -1
- package/out/replicache/src/kv/expo-sqlite/store.d.ts.map +1 -1
- package/out/replicache/src/kv/expo-sqlite/store.js +28 -5
- package/out/replicache/src/kv/expo-sqlite/store.js.map +1 -1
- package/out/replicache/src/kv/idb-store-with-mem-fallback.d.ts +15 -5
- package/out/replicache/src/kv/idb-store-with-mem-fallback.d.ts.map +1 -1
- package/out/replicache/src/kv/idb-store-with-mem-fallback.js +25 -25
- package/out/replicache/src/kv/idb-store-with-mem-fallback.js.map +1 -1
- package/out/replicache/src/kv/idb-store.d.ts +9 -0
- package/out/replicache/src/kv/idb-store.d.ts.map +1 -1
- package/out/replicache/src/kv/idb-store.js +14 -2
- package/out/replicache/src/kv/idb-store.js.map +1 -1
- package/out/replicache/src/kv/mem-store.d.ts +1 -0
- package/out/replicache/src/kv/mem-store.d.ts.map +1 -1
- package/out/replicache/src/kv/mem-store.js +3 -0
- package/out/replicache/src/kv/mem-store.js.map +1 -1
- package/out/replicache/src/kv/op-sqlite/store.d.ts.map +1 -1
- package/out/replicache/src/kv/op-sqlite/store.js +1 -1
- package/out/replicache/src/kv/op-sqlite/store.js.map +1 -1
- package/out/replicache/src/kv/sqlite-store.d.ts +13 -1
- package/out/replicache/src/kv/sqlite-store.d.ts.map +1 -1
- package/out/replicache/src/kv/sqlite-store.js +73 -6
- package/out/replicache/src/kv/sqlite-store.js.map +1 -1
- package/out/replicache/src/kv/store.d.ts +6 -0
- package/out/replicache/src/kv/store.d.ts.map +1 -1
- package/out/replicache/src/persist/collect-idb-databases.d.ts +10 -5
- package/out/replicache/src/persist/collect-idb-databases.d.ts.map +1 -1
- package/out/replicache/src/persist/collect-idb-databases.js +16 -9
- package/out/replicache/src/persist/collect-idb-databases.js.map +1 -1
- package/out/replicache/src/persist/persist.d.ts.map +1 -1
- package/out/replicache/src/persist/persist.js +13 -6
- package/out/replicache/src/persist/persist.js.map +1 -1
- package/out/replicache/src/persist/refresh.js +6 -2
- package/out/replicache/src/persist/refresh.js.map +1 -1
- package/out/replicache/src/replicache-impl.d.ts +14 -3
- package/out/replicache/src/replicache-impl.d.ts.map +1 -1
- package/out/replicache/src/replicache-impl.js +118 -11
- package/out/replicache/src/replicache-impl.js.map +1 -1
- package/out/replicache/src/replicache-options.d.ts +9 -0
- package/out/replicache/src/replicache-options.d.ts.map +1 -1
- package/out/replicache/src/transactions.d.ts.map +1 -1
- package/out/replicache/src/transactions.js +1 -1
- package/out/replicache/src/transactions.js.map +1 -1
- package/out/shared/src/arrays.d.ts +5 -0
- package/out/shared/src/arrays.d.ts.map +1 -1
- package/out/shared/src/arrays.js +8 -10
- package/out/shared/src/arrays.js.map +1 -1
- package/out/shared/src/error.d.ts +11 -0
- package/out/shared/src/error.d.ts.map +1 -1
- package/out/shared/src/error.js +35 -1
- package/out/shared/src/error.js.map +1 -1
- package/out/shared/src/iterables.d.ts +2 -0
- package/out/shared/src/iterables.d.ts.map +1 -1
- package/out/shared/src/iterables.js +18 -1
- package/out/shared/src/iterables.js.map +1 -1
- package/out/shared/src/map.d.ts +24 -2
- package/out/shared/src/map.d.ts.map +1 -1
- package/out/shared/src/map.js +18 -4
- package/out/shared/src/map.js.map +1 -1
- package/out/shared/src/ring-buffer.d.ts +8 -0
- package/out/shared/src/ring-buffer.d.ts.map +1 -1
- package/out/shared/src/ring-buffer.js +22 -1
- package/out/shared/src/ring-buffer.js.map +1 -1
- package/out/z2s/src/compiler.js +1 -1
- package/out/z2s/src/sql.d.ts.map +1 -1
- package/out/z2s/src/sql.js +2 -2
- package/out/z2s/src/sql.js.map +1 -1
- package/out/zero/package.js +3 -4
- package/out/zero/package.js.map +1 -1
- package/out/zero-cache/src/auth/write-authorizer.d.ts.map +1 -1
- package/out/zero-cache/src/auth/write-authorizer.js +1 -0
- package/out/zero-cache/src/auth/write-authorizer.js.map +1 -1
- package/out/zero-cache/src/config/normalize.d.ts.map +1 -1
- package/out/zero-cache/src/config/normalize.js +25 -1
- package/out/zero-cache/src/config/normalize.js.map +1 -1
- package/out/zero-cache/src/config/zero-config.d.ts +48 -0
- package/out/zero-cache/src/config/zero-config.d.ts.map +1 -1
- package/out/zero-cache/src/config/zero-config.js +130 -5
- package/out/zero-cache/src/config/zero-config.js.map +1 -1
- package/out/zero-cache/src/db/lite-tables.d.ts.map +1 -1
- package/out/zero-cache/src/db/lite-tables.js +3 -4
- package/out/zero-cache/src/db/lite-tables.js.map +1 -1
- package/out/zero-cache/src/observability/events.js +4 -1
- package/out/zero-cache/src/observability/events.js.map +1 -1
- package/out/zero-cache/src/observability/metrics.d.ts.map +1 -1
- package/out/zero-cache/src/observability/metrics.js +2 -7
- package/out/zero-cache/src/observability/metrics.js.map +1 -1
- package/out/zero-cache/src/server/change-streamer.d.ts.map +1 -1
- package/out/zero-cache/src/server/change-streamer.js +27 -10
- package/out/zero-cache/src/server/change-streamer.js.map +1 -1
- package/out/zero-cache/src/server/inspector-delegate.d.ts +18 -2
- package/out/zero-cache/src/server/inspector-delegate.d.ts.map +1 -1
- package/out/zero-cache/src/server/inspector-delegate.js +22 -15
- package/out/zero-cache/src/server/inspector-delegate.js.map +1 -1
- package/out/zero-cache/src/server/main.d.ts.map +1 -1
- package/out/zero-cache/src/server/main.js +8 -1
- package/out/zero-cache/src/server/main.js.map +1 -1
- package/out/zero-cache/src/server/mutator.js +2 -0
- package/out/zero-cache/src/server/mutator.js.map +1 -1
- package/out/zero-cache/src/server/reaper.d.ts.map +1 -1
- package/out/zero-cache/src/server/reaper.js +2 -0
- package/out/zero-cache/src/server/reaper.js.map +1 -1
- package/out/zero-cache/src/server/replicator.d.ts.map +1 -1
- package/out/zero-cache/src/server/replicator.js +12 -3
- package/out/zero-cache/src/server/replicator.js.map +1 -1
- package/out/zero-cache/src/server/runner/zero-dispatcher.d.ts.map +1 -1
- package/out/zero-cache/src/server/runner/zero-dispatcher.js +3 -0
- package/out/zero-cache/src/server/runner/zero-dispatcher.js.map +1 -1
- package/out/zero-cache/src/server/syncer-assigner.d.ts +14 -0
- package/out/zero-cache/src/server/syncer-assigner.d.ts.map +1 -0
- package/out/zero-cache/src/server/syncer-assigner.js +72 -0
- package/out/zero-cache/src/server/syncer-assigner.js.map +1 -0
- package/out/zero-cache/src/server/syncer.d.ts.map +1 -1
- package/out/zero-cache/src/server/syncer.js +5 -1
- package/out/zero-cache/src/server/syncer.js.map +1 -1
- package/out/zero-cache/src/server/worker-dispatcher.d.ts.map +1 -1
- package/out/zero-cache/src/server/worker-dispatcher.js +21 -4
- package/out/zero-cache/src/server/worker-dispatcher.js.map +1 -1
- package/out/zero-cache/src/services/analyze.d.ts.map +1 -1
- package/out/zero-cache/src/services/analyze.js +11 -9
- package/out/zero-cache/src/services/analyze.js.map +1 -1
- package/out/zero-cache/src/services/change-source/common/backfill-manager.d.ts.map +1 -1
- package/out/zero-cache/src/services/change-source/common/backfill-manager.js +15 -3
- package/out/zero-cache/src/services/change-source/common/backfill-manager.js.map +1 -1
- package/out/zero-cache/src/services/change-source/common/change-stream-multiplexer.d.ts.map +1 -1
- package/out/zero-cache/src/services/change-source/common/change-stream-multiplexer.js +15 -3
- package/out/zero-cache/src/services/change-source/common/change-stream-multiplexer.js.map +1 -1
- package/out/zero-cache/src/services/change-source/common/replica-schema.d.ts.map +1 -1
- package/out/zero-cache/src/services/change-source/common/replica-schema.js +2 -39
- package/out/zero-cache/src/services/change-source/common/replica-schema.js.map +1 -1
- package/out/zero-cache/src/services/change-source/pg/backfill-resume.d.ts +156 -0
- package/out/zero-cache/src/services/change-source/pg/backfill-resume.d.ts.map +1 -0
- package/out/zero-cache/src/services/change-source/pg/backfill-resume.js +15 -0
- package/out/zero-cache/src/services/change-source/pg/backfill-resume.js.map +1 -0
- package/out/zero-cache/src/services/change-source/pg/change-source-init.d.ts +25 -0
- package/out/zero-cache/src/services/change-source/pg/change-source-init.d.ts.map +1 -0
- package/out/zero-cache/src/services/change-source/pg/change-source-init.js +202 -0
- package/out/zero-cache/src/services/change-source/pg/change-source-init.js.map +1 -0
- package/out/zero-cache/src/services/change-source/pg/change-source.d.ts +14 -16
- package/out/zero-cache/src/services/change-source/pg/change-source.d.ts.map +1 -1
- package/out/zero-cache/src/services/change-source/pg/change-source.js +64 -143
- package/out/zero-cache/src/services/change-source/pg/change-source.js.map +1 -1
- package/out/zero-cache/src/services/change-source/pg/initial-sync.d.ts +28 -2
- package/out/zero-cache/src/services/change-source/pg/initial-sync.d.ts.map +1 -1
- package/out/zero-cache/src/services/change-source/pg/initial-sync.js +44 -21
- package/out/zero-cache/src/services/change-source/pg/initial-sync.js.map +1 -1
- package/out/zero-cache/src/services/change-source/pg/replication-slot-cleanup-monitor.d.ts +16 -0
- package/out/zero-cache/src/services/change-source/pg/replication-slot-cleanup-monitor.d.ts.map +1 -0
- package/out/zero-cache/src/services/change-source/pg/replication-slot-cleanup-monitor.js +113 -0
- package/out/zero-cache/src/services/change-source/pg/replication-slot-cleanup-monitor.js.map +1 -0
- package/out/zero-cache/src/services/change-source/pg/replication-slots.d.ts +21 -15
- package/out/zero-cache/src/services/change-source/pg/replication-slots.d.ts.map +1 -1
- package/out/zero-cache/src/services/change-source/pg/replication-slots.js +86 -23
- package/out/zero-cache/src/services/change-source/pg/replication-slots.js.map +1 -1
- package/out/zero-cache/src/services/change-source/pg/schema/index-predicate.d.ts.map +1 -1
- package/out/zero-cache/src/services/change-source/pg/schema/index-predicate.js +68 -8
- package/out/zero-cache/src/services/change-source/pg/schema/index-predicate.js.map +1 -1
- package/out/zero-cache/src/services/change-source/pg/schema/init.d.ts.map +1 -1
- package/out/zero-cache/src/services/change-source/pg/schema/init.js +32 -1
- package/out/zero-cache/src/services/change-source/pg/schema/init.js.map +1 -1
- package/out/zero-cache/src/services/change-source/pg/schema/replica-stage-enum.d.ts +10 -0
- package/out/zero-cache/src/services/change-source/pg/schema/replica-stage-enum.d.ts.map +1 -0
- package/out/zero-cache/src/services/change-source/pg/schema/replica-stage-enum.js +6 -0
- package/out/zero-cache/src/services/change-source/pg/schema/replica-stage-enum.js.map +1 -0
- package/out/zero-cache/src/services/change-source/pg/schema/shard.d.ts +50 -27
- package/out/zero-cache/src/services/change-source/pg/schema/shard.d.ts.map +1 -1
- package/out/zero-cache/src/services/change-source/pg/schema/shard.js +100 -34
- package/out/zero-cache/src/services/change-source/pg/schema/shard.js.map +1 -1
- package/out/zero-cache/src/services/change-streamer/backup-monitor.d.ts.map +1 -1
- package/out/zero-cache/src/services/change-streamer/backup-monitor.js +58 -7
- package/out/zero-cache/src/services/change-streamer/backup-monitor.js.map +1 -1
- package/out/zero-cache/src/services/change-streamer/broadcast.d.ts +8 -2
- package/out/zero-cache/src/services/change-streamer/broadcast.d.ts.map +1 -1
- package/out/zero-cache/src/services/change-streamer/broadcast.js +24 -4
- package/out/zero-cache/src/services/change-streamer/broadcast.js.map +1 -1
- package/out/zero-cache/src/services/change-streamer/change-log-codec.d.ts +19 -4
- package/out/zero-cache/src/services/change-streamer/change-log-codec.d.ts.map +1 -1
- package/out/zero-cache/src/services/change-streamer/change-log-codec.js +28 -8
- package/out/zero-cache/src/services/change-streamer/change-log-codec.js.map +1 -1
- package/out/zero-cache/src/services/change-streamer/change-streamer-http.d.ts +5 -2
- package/out/zero-cache/src/services/change-streamer/change-streamer-http.d.ts.map +1 -1
- package/out/zero-cache/src/services/change-streamer/change-streamer-http.js +21 -14
- package/out/zero-cache/src/services/change-streamer/change-streamer-http.js.map +1 -1
- package/out/zero-cache/src/services/change-streamer/change-streamer-service.d.ts +11 -0
- package/out/zero-cache/src/services/change-streamer/change-streamer-service.d.ts.map +1 -1
- package/out/zero-cache/src/services/change-streamer/change-streamer-service.js +97 -63
- package/out/zero-cache/src/services/change-streamer/change-streamer-service.js.map +1 -1
- package/out/zero-cache/src/services/change-streamer/change-streamer.d.ts +12 -8
- package/out/zero-cache/src/services/change-streamer/change-streamer.d.ts.map +1 -1
- package/out/zero-cache/src/services/change-streamer/change-streamer.js.map +1 -1
- package/out/zero-cache/src/services/change-streamer/forwarder.d.ts +3 -3
- package/out/zero-cache/src/services/change-streamer/forwarder.d.ts.map +1 -1
- package/out/zero-cache/src/services/change-streamer/forwarder.js +36 -5
- package/out/zero-cache/src/services/change-streamer/forwarder.js.map +1 -1
- package/out/zero-cache/src/services/change-streamer/schema/tables.d.ts +1 -1
- package/out/zero-cache/src/services/change-streamer/schema/tables.d.ts.map +1 -1
- package/out/zero-cache/src/services/change-streamer/schema/tables.js +4 -4
- package/out/zero-cache/src/services/change-streamer/schema/tables.js.map +1 -1
- package/out/zero-cache/src/services/change-streamer/snapshot-reservations.d.ts.map +1 -1
- package/out/zero-cache/src/services/change-streamer/snapshot-reservations.js +15 -3
- package/out/zero-cache/src/services/change-streamer/snapshot-reservations.js.map +1 -1
- package/out/zero-cache/src/services/change-streamer/snapshot.d.ts +21 -1
- package/out/zero-cache/src/services/change-streamer/snapshot.d.ts.map +1 -1
- package/out/zero-cache/src/services/change-streamer/snapshot.js +61 -21
- package/out/zero-cache/src/services/change-streamer/snapshot.js.map +1 -1
- package/out/zero-cache/src/services/change-streamer/sqlite-change-log-writer.d.ts +6 -1
- package/out/zero-cache/src/services/change-streamer/sqlite-change-log-writer.d.ts.map +1 -1
- package/out/zero-cache/src/services/change-streamer/sqlite-change-log-writer.js +27 -9
- package/out/zero-cache/src/services/change-streamer/sqlite-change-log-writer.js.map +1 -1
- package/out/zero-cache/src/services/change-streamer/storer.d.ts +2 -2
- package/out/zero-cache/src/services/change-streamer/storer.d.ts.map +1 -1
- package/out/zero-cache/src/services/change-streamer/storer.js +8 -5
- package/out/zero-cache/src/services/change-streamer/storer.js.map +1 -1
- package/out/zero-cache/src/services/change-streamer/subscriber.d.ts +4 -1
- package/out/zero-cache/src/services/change-streamer/subscriber.d.ts.map +1 -1
- package/out/zero-cache/src/services/change-streamer/subscriber.js +42 -0
- package/out/zero-cache/src/services/change-streamer/subscriber.js.map +1 -1
- package/out/zero-cache/src/services/change-streamer/upstream-acker.d.ts +17 -1
- package/out/zero-cache/src/services/change-streamer/upstream-acker.d.ts.map +1 -1
- package/out/zero-cache/src/services/change-streamer/upstream-acker.js +19 -3
- package/out/zero-cache/src/services/change-streamer/upstream-acker.js.map +1 -1
- package/out/zero-cache/src/services/change-streamer/vfs-watermark-poller.d.ts.map +1 -1
- package/out/zero-cache/src/services/change-streamer/vfs-watermark-poller.js +1 -0
- package/out/zero-cache/src/services/change-streamer/vfs-watermark-poller.js.map +1 -1
- package/out/zero-cache/src/services/life-cycle.d.ts.map +1 -1
- package/out/zero-cache/src/services/life-cycle.js +3 -1
- package/out/zero-cache/src/services/life-cycle.js.map +1 -1
- package/out/zero-cache/src/services/litestream/commands.d.ts.map +1 -1
- package/out/zero-cache/src/services/litestream/commands.js +3 -1
- package/out/zero-cache/src/services/litestream/commands.js.map +1 -1
- package/out/zero-cache/src/services/litestream/config-v5.yml +3 -4
- package/out/zero-cache/src/services/litestream/litestream-checkpointer.js +51 -24
- package/out/zero-cache/src/services/litestream/litestream-checkpointer.js.map +1 -1
- package/out/zero-cache/src/services/litestream/restore-progress.d.ts +24 -0
- package/out/zero-cache/src/services/litestream/restore-progress.d.ts.map +1 -0
- package/out/zero-cache/src/services/litestream/restore-progress.js +91 -0
- package/out/zero-cache/src/services/litestream/restore-progress.js.map +1 -0
- package/out/zero-cache/src/services/mutagen/mutagen.d.ts +5 -0
- package/out/zero-cache/src/services/mutagen/mutagen.d.ts.map +1 -1
- package/out/zero-cache/src/services/mutagen/mutagen.js +7 -1
- package/out/zero-cache/src/services/mutagen/mutagen.js.map +1 -1
- package/out/zero-cache/src/services/mutagen/pusher.d.ts.map +1 -1
- package/out/zero-cache/src/services/mutagen/pusher.js +5 -6
- package/out/zero-cache/src/services/mutagen/pusher.js.map +1 -1
- package/out/zero-cache/src/services/profz.d.ts +7 -0
- package/out/zero-cache/src/services/profz.d.ts.map +1 -0
- package/out/zero-cache/src/services/profz.js +105 -0
- package/out/zero-cache/src/services/profz.js.map +1 -0
- package/out/zero-cache/src/services/replicator/change-log-db.js.map +1 -1
- package/out/zero-cache/src/services/replicator/change-log-stream-writer.d.ts +5 -4
- package/out/zero-cache/src/services/replicator/change-log-stream-writer.d.ts.map +1 -1
- package/out/zero-cache/src/services/replicator/change-log-stream-writer.js +20 -22
- package/out/zero-cache/src/services/replicator/change-log-stream-writer.js.map +1 -1
- package/out/zero-cache/src/services/replicator/change-processor.js +1 -0
- package/out/zero-cache/src/services/replicator/change-processor.js.map +1 -1
- package/out/zero-cache/src/services/replicator/incremental-sync.d.ts +1 -1
- package/out/zero-cache/src/services/replicator/incremental-sync.d.ts.map +1 -1
- package/out/zero-cache/src/services/replicator/incremental-sync.js +57 -17
- package/out/zero-cache/src/services/replicator/incremental-sync.js.map +1 -1
- package/out/zero-cache/src/services/replicator/replication-status.d.ts +35 -1
- package/out/zero-cache/src/services/replicator/replication-status.d.ts.map +1 -1
- package/out/zero-cache/src/services/replicator/replication-status.js +96 -6
- package/out/zero-cache/src/services/replicator/replication-status.js.map +1 -1
- package/out/zero-cache/src/services/replicator/replicator.js.map +1 -1
- package/out/zero-cache/src/services/replicator/sqlite-change-log-observability.d.ts +6 -6
- package/out/zero-cache/src/services/replicator/sqlite-change-log-observability.d.ts.map +1 -1
- package/out/zero-cache/src/services/replicator/sqlite-change-log-observability.js +19 -14
- package/out/zero-cache/src/services/replicator/sqlite-change-log-observability.js.map +1 -1
- package/out/zero-cache/src/services/replicator/write-worker-client.d.ts +4 -4
- package/out/zero-cache/src/services/replicator/write-worker-client.d.ts.map +1 -1
- package/out/zero-cache/src/services/replicator/write-worker-client.js +2 -2
- package/out/zero-cache/src/services/replicator/write-worker-client.js.map +1 -1
- package/out/zero-cache/src/services/replicator/write-worker.js +10 -4
- package/out/zero-cache/src/services/replicator/write-worker.js.map +1 -1
- package/out/zero-cache/src/services/run-ast.d.ts +12 -0
- package/out/zero-cache/src/services/run-ast.d.ts.map +1 -1
- package/out/zero-cache/src/services/run-ast.js +26 -14
- package/out/zero-cache/src/services/run-ast.js.map +1 -1
- package/out/zero-cache/src/services/runner.d.ts +2 -1
- package/out/zero-cache/src/services/runner.d.ts.map +1 -1
- package/out/zero-cache/src/services/runner.js +10 -2
- package/out/zero-cache/src/services/runner.js.map +1 -1
- package/out/zero-cache/src/services/view-syncer/client-handler.d.ts +1 -1
- package/out/zero-cache/src/services/view-syncer/client-handler.d.ts.map +1 -1
- package/out/zero-cache/src/services/view-syncer/client-handler.js +36 -31
- package/out/zero-cache/src/services/view-syncer/client-handler.js.map +1 -1
- package/out/zero-cache/src/services/view-syncer/cvr.d.ts +45 -0
- package/out/zero-cache/src/services/view-syncer/cvr.d.ts.map +1 -1
- package/out/zero-cache/src/services/view-syncer/cvr.js +133 -1
- package/out/zero-cache/src/services/view-syncer/cvr.js.map +1 -1
- package/out/zero-cache/src/services/view-syncer/drain-coordinator.d.ts +14 -1
- package/out/zero-cache/src/services/view-syncer/drain-coordinator.d.ts.map +1 -1
- package/out/zero-cache/src/services/view-syncer/drain-coordinator.js +30 -4
- package/out/zero-cache/src/services/view-syncer/drain-coordinator.js.map +1 -1
- package/out/zero-cache/src/services/view-syncer/hydration-budget.d.ts +33 -0
- package/out/zero-cache/src/services/view-syncer/hydration-budget.d.ts.map +1 -0
- package/out/zero-cache/src/services/view-syncer/hydration-budget.js +62 -0
- package/out/zero-cache/src/services/view-syncer/hydration-budget.js.map +1 -0
- package/out/zero-cache/src/services/view-syncer/hydration-circuit-breaker.d.ts +51 -0
- package/out/zero-cache/src/services/view-syncer/hydration-circuit-breaker.d.ts.map +1 -0
- package/out/zero-cache/src/services/view-syncer/hydration-circuit-breaker.js +78 -0
- package/out/zero-cache/src/services/view-syncer/hydration-circuit-breaker.js.map +1 -0
- package/out/zero-cache/src/services/view-syncer/inspect-handler.d.ts +1 -1
- package/out/zero-cache/src/services/view-syncer/inspect-handler.d.ts.map +1 -1
- package/out/zero-cache/src/services/view-syncer/inspect-handler.js +2 -2
- package/out/zero-cache/src/services/view-syncer/inspect-handler.js.map +1 -1
- package/out/zero-cache/src/services/view-syncer/pipeline-driver.d.ts +14 -1
- package/out/zero-cache/src/services/view-syncer/pipeline-driver.d.ts.map +1 -1
- package/out/zero-cache/src/services/view-syncer/pipeline-driver.js +64 -30
- package/out/zero-cache/src/services/view-syncer/pipeline-driver.js.map +1 -1
- package/out/zero-cache/src/services/view-syncer/query-covering.d.ts.map +1 -1
- package/out/zero-cache/src/services/view-syncer/query-covering.js +2 -6
- package/out/zero-cache/src/services/view-syncer/query-covering.js.map +1 -1
- package/out/zero-cache/src/services/view-syncer/snapshot-row-cache.d.ts +95 -0
- package/out/zero-cache/src/services/view-syncer/snapshot-row-cache.d.ts.map +1 -0
- package/out/zero-cache/src/services/view-syncer/snapshot-row-cache.js +155 -0
- package/out/zero-cache/src/services/view-syncer/snapshot-row-cache.js.map +1 -0
- package/out/zero-cache/src/services/view-syncer/snapshotter.d.ts +48 -4
- package/out/zero-cache/src/services/view-syncer/snapshotter.d.ts.map +1 -1
- package/out/zero-cache/src/services/view-syncer/snapshotter.js +125 -28
- package/out/zero-cache/src/services/view-syncer/snapshotter.js.map +1 -1
- package/out/zero-cache/src/services/view-syncer/view-syncer.d.ts +11 -1
- package/out/zero-cache/src/services/view-syncer/view-syncer.d.ts.map +1 -1
- package/out/zero-cache/src/services/view-syncer/view-syncer.js +512 -98
- package/out/zero-cache/src/services/view-syncer/view-syncer.js.map +1 -1
- package/out/zero-cache/src/types/processes.d.ts +39 -0
- package/out/zero-cache/src/types/processes.d.ts.map +1 -1
- package/out/zero-cache/src/types/processes.js +43 -4
- package/out/zero-cache/src/types/processes.js.map +1 -1
- package/out/zero-cache/src/types/profiler.d.ts +12 -0
- package/out/zero-cache/src/types/profiler.d.ts.map +1 -1
- package/out/zero-cache/src/types/profiler.js +46 -1
- package/out/zero-cache/src/types/profiler.js.map +1 -1
- package/out/zero-cache/src/types/row-key.d.ts.map +1 -1
- package/out/zero-cache/src/types/row-key.js +5 -5
- package/out/zero-cache/src/types/row-key.js.map +1 -1
- package/out/zero-cache/src/types/streams.d.ts +25 -9
- package/out/zero-cache/src/types/streams.d.ts.map +1 -1
- package/out/zero-cache/src/types/streams.js +118 -34
- package/out/zero-cache/src/types/streams.js.map +1 -1
- package/out/zero-cache/src/types/subscription.d.ts +4 -0
- package/out/zero-cache/src/types/subscription.d.ts.map +1 -1
- package/out/zero-cache/src/types/subscription.js +71 -0
- package/out/zero-cache/src/types/subscription.js.map +1 -1
- package/out/zero-cache/src/types/websocket-handoff.d.ts +1 -1
- package/out/zero-cache/src/types/websocket-handoff.d.ts.map +1 -1
- package/out/zero-cache/src/types/websocket-handoff.js +18 -9
- package/out/zero-cache/src/types/websocket-handoff.js.map +1 -1
- package/out/zero-cache/src/types/ws.d.ts.map +1 -1
- package/out/zero-cache/src/types/ws.js +23 -8
- package/out/zero-cache/src/types/ws.js.map +1 -1
- package/out/zero-cache/src/workers/connect-params.d.ts +1 -0
- package/out/zero-cache/src/workers/connect-params.d.ts.map +1 -1
- package/out/zero-cache/src/workers/connect-params.js.map +1 -1
- package/out/zero-cache/src/workers/syncer-ws-message-handler.d.ts.map +1 -1
- package/out/zero-cache/src/workers/syncer-ws-message-handler.js +21 -16
- package/out/zero-cache/src/workers/syncer-ws-message-handler.js.map +1 -1
- package/out/zero-cache/src/workers/syncer.d.ts.map +1 -1
- package/out/zero-cache/src/workers/syncer.js +55 -4
- package/out/zero-cache/src/workers/syncer.js.map +1 -1
- package/out/zero-client/src/client/connection-manager.d.ts +20 -0
- package/out/zero-client/src/client/connection-manager.d.ts.map +1 -1
- package/out/zero-client/src/client/connection-manager.js +77 -13
- package/out/zero-client/src/client/connection-manager.js.map +1 -1
- package/out/zero-client/src/client/connection-status-enum.d.ts +2 -0
- package/out/zero-client/src/client/connection-status-enum.d.ts.map +1 -1
- package/out/zero-client/src/client/connection-status-enum.js +3 -1
- package/out/zero-client/src/client/connection-status-enum.js.map +1 -1
- package/out/zero-client/src/client/connection.d.ts +6 -0
- package/out/zero-client/src/client/connection.d.ts.map +1 -1
- package/out/zero-client/src/client/connection.js +2 -1
- package/out/zero-client/src/client/connection.js.map +1 -1
- package/out/zero-client/src/client/context.d.ts +27 -2
- package/out/zero-client/src/client/context.d.ts.map +1 -1
- package/out/zero-client/src/client/context.js +112 -1
- package/out/zero-client/src/client/context.js.map +1 -1
- package/out/zero-client/src/client/custom.js +2 -2
- package/out/zero-client/src/client/custom.js.map +1 -1
- package/out/zero-client/src/client/error.js +1 -1
- package/out/zero-client/src/client/inspector/lazy-inspector.js +2 -2
- package/out/zero-client/src/client/inspector/lazy-inspector.js.map +1 -1
- package/out/zero-client/src/client/ivm-branch.d.ts.map +1 -1
- package/out/zero-client/src/client/ivm-branch.js +20 -3
- package/out/zero-client/src/client/ivm-branch.js.map +1 -1
- package/out/zero-client/src/client/mutator-proxy.d.ts.map +1 -1
- package/out/zero-client/src/client/mutator-proxy.js +2 -1
- package/out/zero-client/src/client/mutator-proxy.js.map +1 -1
- package/out/zero-client/src/client/options.d.ts +3 -0
- package/out/zero-client/src/client/options.d.ts.map +1 -1
- package/out/zero-client/src/client/options.js.map +1 -1
- package/out/zero-client/src/client/query-manager.d.ts.map +1 -1
- package/out/zero-client/src/client/query-manager.js +44 -11
- package/out/zero-client/src/client/query-manager.js.map +1 -1
- package/out/zero-client/src/client/version.js +1 -1
- package/out/zero-client/src/client/zero-rep.d.ts +1 -0
- package/out/zero-client/src/client/zero-rep.d.ts.map +1 -1
- package/out/zero-client/src/client/zero-rep.js +9 -6
- package/out/zero-client/src/client/zero-rep.js.map +1 -1
- package/out/zero-client/src/client/zero.d.ts +18 -3
- package/out/zero-client/src/client/zero.d.ts.map +1 -1
- package/out/zero-client/src/client/zero.js +120 -32
- package/out/zero-client/src/client/zero.js.map +1 -1
- package/out/zero-client/src/types/query-result.d.ts +2 -0
- package/out/zero-client/src/types/query-result.d.ts.map +1 -1
- package/out/zero-events/src/status.d.ts +58 -0
- package/out/zero-events/src/status.d.ts.map +1 -1
- package/out/zero-protocol/src/ast.d.ts.map +1 -1
- package/out/zero-protocol/src/ast.js +13 -14
- package/out/zero-protocol/src/ast.js.map +1 -1
- package/out/zero-protocol/src/poke.d.ts +6 -6
- package/out/zero-protocol/src/query-hash.d.ts.map +1 -1
- package/out/zero-protocol/src/query-hash.js +2 -5
- package/out/zero-protocol/src/query-hash.js.map +1 -1
- package/out/zero-react/src/use-query.d.ts +5 -4
- package/out/zero-react/src/use-query.d.ts.map +1 -1
- package/out/zero-react/src/use-query.js +86 -9
- package/out/zero-react/src/use-query.js.map +1 -1
- package/out/zero-server/src/adapters/drizzle.d.ts +2 -1
- package/out/zero-server/src/adapters/drizzle.d.ts.map +1 -1
- package/out/zero-server/src/adapters/drizzle.js +12 -7
- package/out/zero-server/src/adapters/drizzle.js.map +1 -1
- package/out/zero-server/src/adapters/kysely.d.ts +2 -1
- package/out/zero-server/src/adapters/kysely.d.ts.map +1 -1
- package/out/zero-server/src/adapters/kysely.js +8 -2
- package/out/zero-server/src/adapters/kysely.js.map +1 -1
- package/out/zero-server/src/adapters/pg.d.ts +1 -0
- package/out/zero-server/src/adapters/pg.d.ts.map +1 -1
- package/out/zero-server/src/adapters/pg.js +8 -2
- package/out/zero-server/src/adapters/pg.js.map +1 -1
- package/out/zero-server/src/adapters/postgresjs.d.ts +1 -0
- package/out/zero-server/src/adapters/postgresjs.d.ts.map +1 -1
- package/out/zero-server/src/adapters/postgresjs.js +7 -1
- package/out/zero-server/src/adapters/postgresjs.js.map +1 -1
- package/out/zero-server/src/adapters/prisma.d.ts +3 -2
- package/out/zero-server/src/adapters/prisma.d.ts.map +1 -1
- package/out/zero-server/src/adapters/prisma.js +8 -2
- package/out/zero-server/src/adapters/prisma.js.map +1 -1
- package/out/zero-server/src/custom.d.ts +5 -1
- package/out/zero-server/src/custom.d.ts.map +1 -1
- package/out/zero-server/src/custom.js +6 -3
- package/out/zero-server/src/custom.js.map +1 -1
- package/out/zero-server/src/mod.d.ts +1 -1
- package/out/zero-server/src/mod.d.ts.map +1 -1
- package/out/zero-server/src/pg-query-executor.d.ts +2 -2
- package/out/zero-server/src/pg-query-executor.d.ts.map +1 -1
- package/out/zero-server/src/pg-query-executor.js +2 -2
- package/out/zero-server/src/pg-query-executor.js.map +1 -1
- package/out/zero-server/src/schema.d.ts +2 -2
- package/out/zero-server/src/schema.d.ts.map +1 -1
- package/out/zero-server/src/schema.js +2 -2
- package/out/zero-server/src/schema.js.map +1 -1
- package/out/zero-server/src/zql-database.d.ts +21 -1
- package/out/zero-server/src/zql-database.d.ts.map +1 -1
- package/out/zero-server/src/zql-database.js +33 -2
- package/out/zero-server/src/zql-database.js.map +1 -1
- package/out/zero-solid/src/solid-view.d.ts +17 -0
- package/out/zero-solid/src/solid-view.d.ts.map +1 -1
- package/out/zero-solid/src/solid-view.js +39 -1
- package/out/zero-solid/src/solid-view.js.map +1 -1
- package/out/zero-solid/src/use-query.js +5 -3
- package/out/zero-solid/src/use-query.js.map +1 -1
- package/out/zql/src/builder/builder.d.ts +19 -0
- package/out/zql/src/builder/builder.d.ts.map +1 -1
- package/out/zql/src/builder/builder.js +38 -12
- package/out/zql/src/builder/builder.js.map +1 -1
- package/out/zql/src/builder/correlated-predicate-pushdown.d.ts +43 -0
- package/out/zql/src/builder/correlated-predicate-pushdown.d.ts.map +1 -0
- package/out/zql/src/builder/correlated-predicate-pushdown.js +161 -0
- package/out/zql/src/builder/correlated-predicate-pushdown.js.map +1 -0
- package/out/zql/src/builder/debug-delegate.d.ts +6 -1
- package/out/zql/src/builder/debug-delegate.d.ts.map +1 -1
- package/out/zql/src/builder/debug-delegate.js +14 -2
- package/out/zql/src/builder/debug-delegate.js.map +1 -1
- package/out/zql/src/ivm/array-view.d.ts +29 -0
- package/out/zql/src/ivm/array-view.d.ts.map +1 -1
- package/out/zql/src/ivm/array-view.js +63 -12
- package/out/zql/src/ivm/array-view.js.map +1 -1
- package/out/zql/src/ivm/cap.d.ts.map +1 -1
- package/out/zql/src/ivm/cap.js +10 -4
- package/out/zql/src/ivm/cap.js.map +1 -1
- package/out/zql/src/ivm/connection-index.d.ts +68 -0
- package/out/zql/src/ivm/connection-index.d.ts.map +1 -0
- package/out/zql/src/ivm/connection-index.js +149 -0
- package/out/zql/src/ivm/connection-index.js.map +1 -0
- package/out/zql/src/ivm/data.d.ts.map +1 -1
- package/out/zql/src/ivm/data.js +8 -3
- package/out/zql/src/ivm/data.js.map +1 -1
- package/out/zql/src/ivm/deferred-input.d.ts +44 -0
- package/out/zql/src/ivm/deferred-input.d.ts.map +1 -0
- package/out/zql/src/ivm/deferred-input.js +89 -0
- package/out/zql/src/ivm/deferred-input.js.map +1 -0
- package/out/zql/src/ivm/exists.d.ts +1 -1
- package/out/zql/src/ivm/exists.d.ts.map +1 -1
- package/out/zql/src/ivm/exists.js +1 -1
- package/out/zql/src/ivm/exists.js.map +1 -1
- package/out/zql/src/ivm/fan-in.d.ts +2 -2
- package/out/zql/src/ivm/fan-in.d.ts.map +1 -1
- package/out/zql/src/ivm/fan-in.js +6 -5
- package/out/zql/src/ivm/fan-in.js.map +1 -1
- package/out/zql/src/ivm/fan-out.d.ts +2 -2
- package/out/zql/src/ivm/fan-out.d.ts.map +1 -1
- package/out/zql/src/ivm/fan-out.js.map +1 -1
- package/out/zql/src/ivm/filter-operators.d.ts +6 -6
- package/out/zql/src/ivm/filter-operators.d.ts.map +1 -1
- package/out/zql/src/ivm/filter-operators.js +10 -8
- package/out/zql/src/ivm/filter-operators.js.map +1 -1
- package/out/zql/src/ivm/filter.d.ts +2 -2
- package/out/zql/src/ivm/filter.d.ts.map +1 -1
- package/out/zql/src/ivm/filter.js +7 -4
- package/out/zql/src/ivm/filter.js.map +1 -1
- package/out/zql/src/ivm/flipped-join.d.ts +6 -4
- package/out/zql/src/ivm/flipped-join.d.ts.map +1 -1
- package/out/zql/src/ivm/flipped-join.js +20 -33
- package/out/zql/src/ivm/flipped-join.js.map +1 -1
- package/out/zql/src/ivm/join-utils.d.ts +10 -0
- package/out/zql/src/ivm/join-utils.d.ts.map +1 -1
- package/out/zql/src/ivm/join-utils.js +23 -1
- package/out/zql/src/ivm/join-utils.js.map +1 -1
- package/out/zql/src/ivm/join.d.ts +5 -1
- package/out/zql/src/ivm/join.d.ts.map +1 -1
- package/out/zql/src/ivm/join.js +91 -13
- package/out/zql/src/ivm/join.js.map +1 -1
- package/out/zql/src/ivm/maybe-split-and-push-edit-change.d.ts +1 -1
- package/out/zql/src/ivm/maybe-split-and-push-edit-change.d.ts.map +1 -1
- package/out/zql/src/ivm/maybe-split-and-push-edit-change.js +6 -4
- package/out/zql/src/ivm/maybe-split-and-push-edit-change.js.map +1 -1
- package/out/zql/src/ivm/memory-source.d.ts +15 -3
- package/out/zql/src/ivm/memory-source.d.ts.map +1 -1
- package/out/zql/src/ivm/memory-source.js +40 -19
- package/out/zql/src/ivm/memory-source.js.map +1 -1
- package/out/zql/src/ivm/take-gate.d.ts +35 -0
- package/out/zql/src/ivm/take-gate.d.ts.map +1 -0
- package/out/zql/src/ivm/take-gate.js +81 -0
- package/out/zql/src/ivm/take-gate.js.map +1 -0
- package/out/zql/src/ivm/take.d.ts +6 -1
- package/out/zql/src/ivm/take.d.ts.map +1 -1
- package/out/zql/src/ivm/take.js +104 -88
- package/out/zql/src/ivm/take.js.map +1 -1
- package/out/zql/src/ivm/union-fan-in.d.ts.map +1 -1
- package/out/zql/src/ivm/union-fan-in.js +9 -7
- package/out/zql/src/ivm/union-fan-in.js.map +1 -1
- package/out/zql/src/ivm/view-apply-change.d.ts.map +1 -1
- package/out/zql/src/ivm/view-apply-change.js +9 -4
- package/out/zql/src/ivm/view-apply-change.js.map +1 -1
- package/out/zql/src/ivm/view.d.ts +15 -0
- package/out/zql/src/ivm/view.d.ts.map +1 -1
- package/out/zql/src/mutate/custom.d.ts +24 -1
- package/out/zql/src/mutate/custom.d.ts.map +1 -1
- package/out/zql/src/mutate/custom.js.map +1 -1
- package/out/zql/src/mutate/mutator-registry.d.ts.map +1 -1
- package/out/zql/src/mutate/mutator-registry.js +2 -2
- package/out/zql/src/mutate/mutator-registry.js.map +1 -1
- package/out/zql/src/planner/planner-builder.d.ts +12 -3
- package/out/zql/src/planner/planner-builder.d.ts.map +1 -1
- package/out/zql/src/planner/planner-builder.js +27 -17
- package/out/zql/src/planner/planner-builder.js.map +1 -1
- package/out/zql/src/planner/planner-connection.d.ts +15 -2
- package/out/zql/src/planner/planner-connection.d.ts.map +1 -1
- package/out/zql/src/planner/planner-connection.js +73 -8
- package/out/zql/src/planner/planner-connection.js.map +1 -1
- package/out/zql/src/planner/planner-debug.d.ts.map +1 -1
- package/out/zql/src/planner/planner-debug.js +3 -8
- package/out/zql/src/planner/planner-debug.js.map +1 -1
- package/out/zql/src/planner/planner-source.d.ts +2 -2
- package/out/zql/src/planner/planner-source.d.ts.map +1 -1
- package/out/zql/src/planner/planner-source.js +2 -2
- package/out/zql/src/planner/planner-source.js.map +1 -1
- package/out/zql/src/query/query-delegate-base.d.ts +9 -1
- package/out/zql/src/query/query-delegate-base.d.ts.map +1 -1
- package/out/zql/src/query/query-delegate-base.js +129 -33
- package/out/zql/src/query/query-delegate-base.js.map +1 -1
- package/out/zql/src/query/query-delegate.d.ts +50 -1
- package/out/zql/src/query/query-delegate.d.ts.map +1 -1
- package/out/zql/src/query/query-impl.d.ts +13 -11
- package/out/zql/src/query/query-impl.d.ts.map +1 -1
- package/out/zql/src/query/query-impl.js +366 -60
- package/out/zql/src/query/query-impl.js.map +1 -1
- package/out/zql/src/query/query-transitions.d.ts +171 -0
- package/out/zql/src/query/query-transitions.d.ts.map +1 -0
- package/out/zql/src/query/query-transitions.js +204 -0
- package/out/zql/src/query/query-transitions.js.map +1 -0
- package/out/zql/src/query/query.d.ts +11 -2
- package/out/zql/src/query/query.d.ts.map +1 -1
- package/out/zql/src/query/runnable-query-impl.d.ts +1 -0
- package/out/zql/src/query/runnable-query-impl.d.ts.map +1 -1
- package/out/zql/src/query/runnable-query-impl.js +13 -1
- package/out/zql/src/query/runnable-query-impl.js.map +1 -1
- package/out/zql/src/query/typed-view.d.ts +1 -1
- package/out/zql/src/query/typed-view.d.ts.map +1 -1
- package/out/zqlite/src/db.d.ts +9 -1
- package/out/zqlite/src/db.d.ts.map +1 -1
- package/out/zqlite/src/db.js +8 -2
- package/out/zqlite/src/db.js.map +1 -1
- package/out/zqlite/src/query-delegate.d.ts.map +1 -1
- package/out/zqlite/src/query-delegate.js +5 -6
- package/out/zqlite/src/query-delegate.js.map +1 -1
- package/out/zqlite/src/sqlite-stat-fanout.d.ts.map +1 -1
- package/out/zqlite/src/sqlite-stat-fanout.js +3 -5
- package/out/zqlite/src/sqlite-stat-fanout.js.map +1 -1
- package/out/zqlite/src/table-source.d.ts +15 -2
- package/out/zqlite/src/table-source.d.ts.map +1 -1
- package/out/zqlite/src/table-source.js +21 -10
- package/out/zqlite/src/table-source.js.map +1 -1
- package/package.json +3 -4
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"lite-tables.js","names":[],"sources":["../../../../../zero-cache/src/db/lite-tables.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {must} from '../../../shared/src/must.ts';\nimport {difference} from '../../../shared/src/set-utils.ts';\nimport * as v from '../../../shared/src/valita.ts';\nimport {primaryKeySchema} from '../../../zero-protocol/src/primary-key.ts';\nimport type {Database} from '../../../zqlite/src/db.ts';\nimport {\n ColumnMetadataStore,\n metadataToLiteTypeString,\n} from '../services/replicator/schema/column-metadata.ts';\nimport {TableMetadataTracker} from '../services/replicator/schema/table-metadata.ts';\nimport {\n isArray,\n isEnum,\n liteTypeToZqlValueType,\n mapLiteDataTypeToZqlSchemaValue,\n nullableUpstream,\n} from '../types/lite.ts';\nimport * as PostgresTypeClass from './postgres-type-class-enum.ts';\nimport type {\n LiteAndZqlSpec,\n LiteIndexSpec,\n LiteTableSpec,\n MutableLiteIndexSpec,\n MutableLiteTableSpec,\n} from './specs.ts';\n\ntype ColumnInfo = {\n table: string;\n name: string;\n type: string;\n notNull: number;\n dflt: string | null;\n keyPos: number;\n};\n\nexport type LiteTableSpecWithReplicationStatus = LiteTableSpec & {\n readonly backfilling?: string[];\n readonly minRowVersion?: string | null;\n};\n\ntype MutableLiteTableSpecWithReplicationStatus = MutableLiteTableSpec & {\n backfilling: string[];\n minRowVersion: string | null;\n};\n\nexport function listTables(\n db: Database,\n useColumnMetadata = true,\n useTableMetadata = true,\n): LiteTableSpecWithReplicationStatus[] {\n const columns = db\n .prepare(\n `\n SELECT\n m.name as \"table\",\n p.name as name,\n p.type as type,\n p.\"notnull\" as \"notNull\",\n p.dflt_value as \"dflt\",\n p.pk as keyPos\n FROM sqlite_master as m\n LEFT JOIN pragma_table_info(m.name) as p\n WHERE m.type = 'table'\n AND m.name NOT LIKE 'sqlite_%'\n AND m.name NOT LIKE '_zero.%'\n AND m.name NOT LIKE '_litestream_%'\n `,\n )\n .all() as ColumnInfo[];\n\n const minRowVersions = useTableMetadata\n ? new TableMetadataTracker(db).getMinRowVersions()\n : new Map();\n const tables: LiteTableSpecWithReplicationStatus[] = [];\n let table: MutableLiteTableSpecWithReplicationStatus | undefined;\n\n columns.forEach(col => {\n if (col.table !== table?.name) {\n // New table\n table = {\n name: col.table,\n columns: {},\n backfilling: [],\n minRowVersion: minRowVersions.get(col.table) ?? null,\n };\n tables.push(table);\n }\n\n // Try to read from metadata table first, fall back to SQLite column type\n let dataType: string;\n let elemPgTypeClass:\n | typeof PostgresTypeClass.Base\n | typeof PostgresTypeClass.Enum\n | null;\n\n const metadata = useColumnMetadata\n ? ColumnMetadataStore.getInstance(db)?.getColumn(col.table, col.name)\n : undefined;\n if (metadata) {\n // Read from metadata table and convert to pipe notation\n dataType = metadataToLiteTypeString(metadata);\n elemPgTypeClass = metadata.isArray\n ? metadata.isEnum\n ? PostgresTypeClass.Enum\n : PostgresTypeClass.Base\n : null;\n } else {\n // Fall back to reading from SQLite column type (pipe notation)\n dataType = col.type;\n elemPgTypeClass = isArray(col.type)\n ? isEnum(col.type)\n ? PostgresTypeClass.Enum\n : PostgresTypeClass.Base\n : null;\n }\n\n table.columns[col.name] = {\n pos: Object.keys(table.columns).length + 1,\n dataType,\n characterMaximumLength: null,\n notNull: col.notNull !== 0,\n dflt: col.dflt,\n elemPgTypeClass,\n };\n if (metadata?.isBackfilling) {\n table.backfilling.push(col.name);\n }\n if (col.keyPos) {\n table.primaryKey ??= [];\n while (table.primaryKey.length < col.keyPos) {\n table.primaryKey.push('');\n }\n table.primaryKey[col.keyPos - 1] = col.name;\n }\n });\n\n return tables;\n}\n\n/**\n * An index as it exists on the replica. SQLite retains only the DDL text of a\n * partial index, not a structured predicate, so replica indexes report a\n * `partial` flag in place of the {@link LiteIndexSpec.predicate}.\n */\nexport type ReplicaIndexSpec = Omit<LiteIndexSpec, 'predicate'> & {\n readonly partial?: boolean | undefined;\n};\n\ntype MutableReplicaIndexSpec = Omit<MutableLiteIndexSpec, 'predicate'> & {\n partial?: boolean | undefined;\n};\n\nexport function listIndexes(db: Database): ReplicaIndexSpec[] {\n const indexes = db\n .prepare(\n `SELECT \n idx.name as indexName, \n idx.tbl_name as tableName, \n info.\"unique\" as \"unique\",\n info.partial as partial,\n col.name as column,\n CASE WHEN col.desc = 0 THEN 'ASC' ELSE 'DESC' END as dir\n FROM sqlite_master as idx\n JOIN pragma_index_list(idx.tbl_name) AS info ON info.name = idx.name\n JOIN pragma_index_xinfo(idx.name) as col\n WHERE idx.type = 'index' AND \n col.key = 1 AND\n idx.tbl_name NOT LIKE '_zero.%'\n ORDER BY idx.name, col.seqno ASC`,\n )\n .all() as {\n indexName: string;\n tableName: string;\n unique: number;\n partial: number;\n column: string;\n dir: 'ASC' | 'DESC';\n }[];\n\n const ret: MutableReplicaIndexSpec[] = [];\n for (const {\n indexName: name,\n tableName,\n unique,\n partial,\n column,\n dir,\n } of indexes) {\n if (ret.at(-1)?.name === name) {\n // Aggregate multiple column names into the array.\n must(ret.at(-1)).columns[column] = dir;\n } else {\n ret.push({\n tableName,\n name,\n columns: {[column]: dir},\n unique: unique !== 0,\n ...(partial === 0 ? {} : {partial: true}),\n });\n }\n }\n\n return ret;\n}\n\nexport type ZqlSpecOptions = {\n /**\n * Controls whether to include backfilling columns in the computed\n * LiteAndZqlSpec. In general, backfilling columns should be include\n * in \"replication\" logic (copying data from upstream to the replica),\n * and excluded from \"sync\" logic (sending data from the replica to\n * clients).\n */\n includeBackfillingColumns: boolean;\n};\n\n/**\n * Computes a TableSpec \"view\" of the replicated data that is\n * suitable for processing / consumption for the client. This\n * includes:\n * * excluding tables without a PRIMARY KEY or UNIQUE INDEX\n * * excluding columns with types that are not supported by ZQL\n * * choosing columns to use as the primary key amongst those\n * in unique indexes\n *\n * @param tableSpecs an optional map to reset and populate\n * @param fullTables an optional map to receive the full table specs,\n * which may include tables and columns that are not synced to\n * the client because they lack a primary key or are of unsupported\n * data types.\n */\nexport function computeZqlSpecs(\n lc: LogContext,\n replica: Database,\n opts: ZqlSpecOptions,\n tableSpecs: Map<string, LiteAndZqlSpec> = new Map(),\n fullTables?: Map<string, LiteTableSpec>,\n): Map<string, LiteAndZqlSpec> {\n return computeZqlSpecsFromLiteSpecs(\n listTables(replica),\n listIndexes(replica),\n opts,\n tableSpecs,\n fullTables,\n lc,\n );\n}\n\nfunction isPartialIndex(idx: LiteIndexSpec | ReplicaIndexSpec): boolean {\n return (\n ('predicate' in idx && idx.predicate !== undefined) ||\n ('partial' in idx && idx.partial === true)\n );\n}\n\nexport function computeZqlSpecsFromLiteSpecs(\n tables: LiteTableSpecWithReplicationStatus[],\n indexes: readonly (LiteIndexSpec | ReplicaIndexSpec)[],\n {includeBackfillingColumns}: ZqlSpecOptions,\n tableSpecs: Map<string, LiteAndZqlSpec> = new Map(),\n fullTables?: Map<string, LiteTableSpec>,\n lc?: LogContext,\n): Map<string, LiteAndZqlSpec> {\n tableSpecs.clear();\n fullTables?.clear();\n\n const uniqueIndexColumns = new Map<string, string[][]>();\n // A partial unique index only constrains the rows matching its predicate,\n // so it cannot serve as a row key.\n for (const {tableName, columns} of indexes.filter(\n idx => idx.unique && !isPartialIndex(idx),\n )) {\n if (!uniqueIndexColumns.has(tableName)) {\n uniqueIndexColumns.set(tableName, []);\n }\n uniqueIndexColumns.get(tableName)?.push(Object.keys(columns));\n }\n\n tables.forEach(fullTable => {\n fullTables?.set(fullTable.name, fullTable);\n\n const backfilling = new Set(fullTable.backfilling);\n // Only include columns that:\n // - have a defined ZQL Value\n // - aren't backfilling if `includeBackfillingColumns` is false\n const visibleColumns = Object.entries(fullTable.columns).filter(\n ([col, {dataType}]) =>\n liteTypeToZqlValueType(dataType) &&\n (includeBackfillingColumns || !backfilling.has(col)),\n );\n const notNullColumns = new Set(\n visibleColumns\n .filter(\n ([col, {dataType}]) =>\n !nullableUpstream(dataType) || fullTable.primaryKey?.includes(col),\n )\n .map(([col]) => col),\n );\n\n const uniqueKeys = uniqueIndexColumns.get(fullTable.name) ?? [];\n // Examine all column combinations that can serve as a primary key,\n // i.e. excluding indexes over nullable or unsynced columns.\n const keys = uniqueKeys.filter(\n key => difference(new Set(key), notNullColumns).size === 0,\n );\n if (keys.length === 0) {\n // Only include tables with a row key.\n //\n // Note that this will automatically exclude tables that are being\n // backfilled (when includeBackfillingColumns is `false`) since candidate\n // keys only include visible columns.\n lc?.debug?.(\n `not syncing table ${fullTable.name} because it ` +\n (backfilling.size ? 'is being backfilled' : 'has no primary key'),\n );\n return;\n }\n // Pick the \"best\" (i.e. shortest) key for default IVM operations.\n const primaryKey = keys.sort(keyCmp)[0];\n\n const tableSpec = {\n ...fullTable,\n columns: Object.fromEntries(visibleColumns),\n primaryKey: v.parse(primaryKey, primaryKeySchema),\n uniqueKeys: uniqueKeys.map(key => v.parse(key, primaryKeySchema)),\n allPotentialPrimaryKeys: keys.map(key => v.parse(key, primaryKeySchema)),\n minRowVersion: fullTable.minRowVersion ?? null,\n };\n\n tableSpecs.set(tableSpec.name, {\n tableSpec,\n zqlSpec: Object.fromEntries(\n Object.entries(tableSpec.columns).map(([name, {dataType}]) => [\n name,\n mapLiteDataTypeToZqlSchemaValue(dataType),\n ]),\n ),\n });\n });\n return tableSpecs;\n}\n\nexport function mustGetTableSpec(\n tableSpecs: Map<string, LiteAndZqlSpec>,\n tableName: string,\n): LiteAndZqlSpec {\n const tableSpec = tableSpecs.get(tableName);\n if (!tableSpec) {\n throw new Error(\n `table '${tableName}' is not one of: ${[...tableSpecs.keys()]\n .filter(t => !t.includes('.') && !t.startsWith('_litestream_'))\n .sort()}. ` +\n `Check the spelling and ensure that the table has a primary key.`,\n );\n }\n return tableSpec;\n}\n\n// Deterministic comparator for favoring shorter row keys.\nfunction keyCmp(a: string[], b: string[]) {\n if (a.length !== b.length) {\n return a.length - b.length; // Fewer columns are better.\n }\n for (let i = 0; i < a.length; i++) {\n if (a[i] < b[i]) {\n return -1;\n }\n if (a[i] > b[i]) {\n return 1;\n }\n }\n return 0;\n}\n"],"mappings":";;;;;;;;AA8CA,SAAgB,WACd,IACA,oBAAoB,MACpB,mBAAmB,MACmB;CACtC,MAAM,UAAU,GACb,QACC;;;;;;;;;;;;;;OAeF,EACC,IAAI;CAEP,MAAM,iBAAiB,mBACnB,IAAI,qBAAqB,EAAE,EAAE,kBAAkB,oBAC/C,IAAI,IAAI;CACZ,MAAM,SAA+C,CAAC;CACtD,IAAI;CAEJ,QAAQ,SAAQ,QAAO;EACrB,IAAI,IAAI,UAAU,OAAO,MAAM;GAE7B,QAAQ;IACN,MAAM,IAAI;IACV,SAAS,CAAC;IACV,aAAa,CAAC;IACd,eAAe,eAAe,IAAI,IAAI,KAAK,KAAK;GAClD;GACA,OAAO,KAAK,KAAK;EACnB;EAGA,IAAI;EACJ,IAAI;EAKJ,MAAM,WAAW,oBACb,oBAAoB,YAAY,EAAE,GAAG,UAAU,IAAI,OAAO,IAAI,IAAI,IAClE,KAAA;EACJ,IAAI,UAAU;GAEZ,WAAW,yBAAyB,QAAQ;GAC5C,kBAAkB,SAAS,UACvB,SAAS,SACP,MACA,MACF;EACN,OAAO;GAEL,WAAW,IAAI;GACf,kBAAkB,QAAQ,IAAI,IAAI,IAC9B,OAAO,IAAI,IAAI,IACb,MACA,MACF;EACN;EAEA,MAAM,QAAQ,IAAI,QAAQ;GACxB,KAAK,OAAO,KAAK,MAAM,OAAO,EAAE,SAAS;GACzC;GACA,wBAAwB;GACxB,SAAS,IAAI,YAAY;GACzB,MAAM,IAAI;GACV;EACF;EACA,IAAI,UAAU,eACZ,MAAM,YAAY,KAAK,IAAI,IAAI;EAEjC,IAAI,IAAI,QAAQ;GACd,MAAM,eAAe,CAAC;GACtB,OAAO,MAAM,WAAW,SAAS,IAAI,QACnC,MAAM,WAAW,KAAK,EAAE;GAE1B,MAAM,WAAW,IAAI,SAAS,KAAK,IAAI;EACzC;CACF,CAAC;CAED,OAAO;AACT;AAeA,SAAgB,YAAY,IAAkC;CAC5D,MAAM,UAAU,GACb,QACC;;;;;;;;;;;;;wCAcF,EACC,IAAI;CASP,MAAM,MAAiC,CAAC;CACxC,KAAK,MAAM,EACT,WAAW,MACX,WACA,QACA,SACA,QACA,SACG,SACH,IAAI,IAAI,GAAG,EAAE,GAAG,SAAS,MAEvB,KAAK,IAAI,GAAG,EAAE,CAAC,EAAE,QAAQ,UAAU;MAEnC,IAAI,KAAK;EACP;EACA;EACA,SAAS,GAAE,SAAS,IAAG;EACvB,QAAQ,WAAW;EACnB,GAAI,YAAY,IAAI,CAAC,IAAI,EAAC,SAAS,KAAI;CACzC,CAAC;CAIL,OAAO;AACT;;;;;;;;;;;;;;;;AA4BA,SAAgB,gBACd,IACA,SACA,MACA,6BAA0C,IAAI,IAAI,GAClD,YAC6B;CAC7B,OAAO,6BACL,WAAW,OAAO,GAClB,YAAY,OAAO,GACnB,MACA,YACA,YACA,EACF;AACF;AAEA,SAAS,eAAe,KAAgD;CACtE,OACG,eAAe,OAAO,IAAI,cAAc,KAAA,KACxC,aAAa,OAAO,IAAI,YAAY;AAEzC;AAEA,SAAgB,6BACd,QACA,SACA,EAAC,6BACD,6BAA0C,IAAI,IAAI,GAClD,YACA,IAC6B;CAC7B,WAAW,MAAM;CACjB,YAAY,MAAM;CAElB,MAAM,qCAAqB,IAAI,IAAwB;CAGvD,KAAK,MAAM,EAAC,WAAW,aAAY,QAAQ,QACzC,QAAO,IAAI,UAAU,CAAC,eAAe,GAAG,CAC1C,GAAG;EACD,IAAI,CAAC,mBAAmB,IAAI,SAAS,GACnC,mBAAmB,IAAI,WAAW,CAAC,CAAC;EAEtC,mBAAmB,IAAI,SAAS,GAAG,KAAK,OAAO,KAAK,OAAO,CAAC;CAC9D;CAEA,OAAO,SAAQ,cAAa;EAC1B,YAAY,IAAI,UAAU,MAAM,SAAS;EAEzC,MAAM,cAAc,IAAI,IAAI,UAAU,WAAW;EAIjD,MAAM,iBAAiB,OAAO,QAAQ,UAAU,OAAO,EAAE,QACtD,CAAC,KAAK,EAAC,gBACN,uBAAuB,QAAQ,MAC9B,6BAA6B,CAAC,YAAY,IAAI,GAAG,EACtD;EACA,MAAM,iBAAiB,IAAI,IACzB,eACG,QACE,CAAC,KAAK,EAAC,gBACN,CAAC,iBAAiB,QAAQ,KAAK,UAAU,YAAY,SAAS,GAAG,CACrE,EACC,KAAK,CAAC,SAAS,GAAG,CACvB;EAEA,MAAM,aAAa,mBAAmB,IAAI,UAAU,IAAI,KAAK,CAAC;EAG9D,MAAM,OAAO,WAAW,QACtB,QAAO,WAAW,IAAI,IAAI,GAAG,GAAG,cAAc,EAAE,SAAS,CAC3D;EACA,IAAI,KAAK,WAAW,GAAG;GAMrB,IAAI,QACF,qBAAqB,UAAU,KAAK,iBACjC,YAAY,OAAO,wBAAwB,qBAChD;GACA;EACF;EAEA,MAAM,aAAa,KAAK,KAAK,MAAM,EAAE;EAErC,MAAM,YAAY;GAChB,GAAG;GACH,SAAS,OAAO,YAAY,cAAc;GAC1C,YAAY,MAAQ,YAAY,gBAAgB;GAChD,YAAY,WAAW,KAAI,QAAO,MAAQ,KAAK,gBAAgB,CAAC;GAChE,yBAAyB,KAAK,KAAI,QAAO,MAAQ,KAAK,gBAAgB,CAAC;GACvE,eAAe,UAAU,iBAAiB;EAC5C;EAEA,WAAW,IAAI,UAAU,MAAM;GAC7B;GACA,SAAS,OAAO,YACd,OAAO,QAAQ,UAAU,OAAO,EAAE,KAAK,CAAC,MAAM,EAAC,gBAAe,CAC5D,MACA,gCAAgC,QAAQ,CAC1C,CAAC,CACH;EACF,CAAC;CACH,CAAC;CACD,OAAO;AACT;AAEA,SAAgB,iBACd,YACA,WACgB;CAChB,MAAM,YAAY,WAAW,IAAI,SAAS;CAC1C,IAAI,CAAC,WACH,MAAM,IAAI,MACR,UAAU,UAAU,mBAAmB,CAAC,GAAG,WAAW,KAAK,CAAC,EACzD,QAAO,MAAK,CAAC,EAAE,SAAS,GAAG,KAAK,CAAC,EAAE,WAAW,cAAc,CAAC,EAC7D,KAAK,EAAE,kEAEZ;CAEF,OAAO;AACT;AAGA,SAAS,OAAO,GAAa,GAAa;CACxC,IAAI,EAAE,WAAW,EAAE,QACjB,OAAO,EAAE,SAAS,EAAE;CAEtB,KAAK,IAAI,IAAI,GAAG,IAAI,EAAE,QAAQ,KAAK;EACjC,IAAI,EAAE,KAAK,EAAE,IACX,OAAO;EAET,IAAI,EAAE,KAAK,EAAE,IACX,OAAO;CAEX;CACA,OAAO;AACT"}
|
|
1
|
+
{"version":3,"file":"lite-tables.js","names":[],"sources":["../../../../../zero-cache/src/db/lite-tables.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {newArray} from '../../../shared/src/arrays.ts';\nimport {getOrInsertComputed} from '../../../shared/src/map.ts';\nimport {must} from '../../../shared/src/must.ts';\nimport {difference} from '../../../shared/src/set-utils.ts';\nimport * as v from '../../../shared/src/valita.ts';\nimport {primaryKeySchema} from '../../../zero-protocol/src/primary-key.ts';\nimport type {Database} from '../../../zqlite/src/db.ts';\nimport {\n ColumnMetadataStore,\n metadataToLiteTypeString,\n} from '../services/replicator/schema/column-metadata.ts';\nimport {TableMetadataTracker} from '../services/replicator/schema/table-metadata.ts';\nimport {\n isArray,\n isEnum,\n liteTypeToZqlValueType,\n mapLiteDataTypeToZqlSchemaValue,\n nullableUpstream,\n} from '../types/lite.ts';\nimport * as PostgresTypeClass from './postgres-type-class-enum.ts';\nimport type {\n LiteAndZqlSpec,\n LiteIndexSpec,\n LiteTableSpec,\n MutableLiteIndexSpec,\n MutableLiteTableSpec,\n} from './specs.ts';\n\ntype ColumnInfo = {\n table: string;\n name: string;\n type: string;\n notNull: number;\n dflt: string | null;\n keyPos: number;\n};\n\nexport type LiteTableSpecWithReplicationStatus = LiteTableSpec & {\n readonly backfilling?: string[];\n readonly minRowVersion?: string | null;\n};\n\ntype MutableLiteTableSpecWithReplicationStatus = MutableLiteTableSpec & {\n backfilling: string[];\n minRowVersion: string | null;\n};\n\nexport function listTables(\n db: Database,\n useColumnMetadata = true,\n useTableMetadata = true,\n): LiteTableSpecWithReplicationStatus[] {\n const columns = db\n .prepare(\n `\n SELECT\n m.name as \"table\",\n p.name as name,\n p.type as type,\n p.\"notnull\" as \"notNull\",\n p.dflt_value as \"dflt\",\n p.pk as keyPos\n FROM sqlite_master as m\n LEFT JOIN pragma_table_info(m.name) as p\n WHERE m.type = 'table'\n AND m.name NOT LIKE 'sqlite_%'\n AND m.name NOT LIKE '_zero.%'\n AND m.name NOT LIKE '_litestream_%'\n `,\n )\n .all() as ColumnInfo[];\n\n const minRowVersions = useTableMetadata\n ? new TableMetadataTracker(db).getMinRowVersions()\n : new Map();\n const tables: LiteTableSpecWithReplicationStatus[] = [];\n let table: MutableLiteTableSpecWithReplicationStatus | undefined;\n\n columns.forEach(col => {\n if (col.table !== table?.name) {\n // New table\n table = {\n name: col.table,\n columns: {},\n backfilling: [],\n minRowVersion: minRowVersions.get(col.table) ?? null,\n };\n tables.push(table);\n }\n\n // Try to read from metadata table first, fall back to SQLite column type\n let dataType: string;\n let elemPgTypeClass:\n | typeof PostgresTypeClass.Base\n | typeof PostgresTypeClass.Enum\n | null;\n\n const metadata = useColumnMetadata\n ? ColumnMetadataStore.getInstance(db)?.getColumn(col.table, col.name)\n : undefined;\n if (metadata) {\n // Read from metadata table and convert to pipe notation\n dataType = metadataToLiteTypeString(metadata);\n elemPgTypeClass = metadata.isArray\n ? metadata.isEnum\n ? PostgresTypeClass.Enum\n : PostgresTypeClass.Base\n : null;\n } else {\n // Fall back to reading from SQLite column type (pipe notation)\n dataType = col.type;\n elemPgTypeClass = isArray(col.type)\n ? isEnum(col.type)\n ? PostgresTypeClass.Enum\n : PostgresTypeClass.Base\n : null;\n }\n\n table.columns[col.name] = {\n pos: Object.keys(table.columns).length + 1,\n dataType,\n characterMaximumLength: null,\n notNull: col.notNull !== 0,\n dflt: col.dflt,\n elemPgTypeClass,\n };\n if (metadata?.isBackfilling) {\n table.backfilling.push(col.name);\n }\n if (col.keyPos) {\n table.primaryKey ??= [];\n while (table.primaryKey.length < col.keyPos) {\n table.primaryKey.push('');\n }\n table.primaryKey[col.keyPos - 1] = col.name;\n }\n });\n\n return tables;\n}\n\n/**\n * An index as it exists on the replica. SQLite retains only the DDL text of a\n * partial index, not a structured predicate, so replica indexes report a\n * `partial` flag in place of the {@link LiteIndexSpec.predicate}.\n */\nexport type ReplicaIndexSpec = Omit<LiteIndexSpec, 'predicate'> & {\n readonly partial?: boolean | undefined;\n};\n\ntype MutableReplicaIndexSpec = Omit<MutableLiteIndexSpec, 'predicate'> & {\n partial?: boolean | undefined;\n};\n\nexport function listIndexes(db: Database): ReplicaIndexSpec[] {\n const indexes = db\n .prepare(\n `SELECT \n idx.name as indexName, \n idx.tbl_name as tableName, \n info.\"unique\" as \"unique\",\n info.partial as partial,\n col.name as column,\n CASE WHEN col.desc = 0 THEN 'ASC' ELSE 'DESC' END as dir\n FROM sqlite_master as idx\n JOIN pragma_index_list(idx.tbl_name) AS info ON info.name = idx.name\n JOIN pragma_index_xinfo(idx.name) as col\n WHERE idx.type = 'index' AND \n col.key = 1 AND\n idx.tbl_name NOT LIKE '_zero.%'\n ORDER BY idx.name, col.seqno ASC`,\n )\n .all() as {\n indexName: string;\n tableName: string;\n unique: number;\n partial: number;\n column: string;\n dir: 'ASC' | 'DESC';\n }[];\n\n const ret: MutableReplicaIndexSpec[] = [];\n for (const {\n indexName: name,\n tableName,\n unique,\n partial,\n column,\n dir,\n } of indexes) {\n if (ret.at(-1)?.name === name) {\n // Aggregate multiple column names into the array.\n must(ret.at(-1)).columns[column] = dir;\n } else {\n ret.push({\n tableName,\n name,\n columns: {[column]: dir},\n unique: unique !== 0,\n ...(partial === 0 ? {} : {partial: true}),\n });\n }\n }\n\n return ret;\n}\n\nexport type ZqlSpecOptions = {\n /**\n * Controls whether to include backfilling columns in the computed\n * LiteAndZqlSpec. In general, backfilling columns should be include\n * in \"replication\" logic (copying data from upstream to the replica),\n * and excluded from \"sync\" logic (sending data from the replica to\n * clients).\n */\n includeBackfillingColumns: boolean;\n};\n\n/**\n * Computes a TableSpec \"view\" of the replicated data that is\n * suitable for processing / consumption for the client. This\n * includes:\n * * excluding tables without a PRIMARY KEY or UNIQUE INDEX\n * * excluding columns with types that are not supported by ZQL\n * * choosing columns to use as the primary key amongst those\n * in unique indexes\n *\n * @param tableSpecs an optional map to reset and populate\n * @param fullTables an optional map to receive the full table specs,\n * which may include tables and columns that are not synced to\n * the client because they lack a primary key or are of unsupported\n * data types.\n */\nexport function computeZqlSpecs(\n lc: LogContext,\n replica: Database,\n opts: ZqlSpecOptions,\n tableSpecs: Map<string, LiteAndZqlSpec> = new Map(),\n fullTables?: Map<string, LiteTableSpec>,\n): Map<string, LiteAndZqlSpec> {\n return computeZqlSpecsFromLiteSpecs(\n listTables(replica),\n listIndexes(replica),\n opts,\n tableSpecs,\n fullTables,\n lc,\n );\n}\n\nfunction isPartialIndex(idx: LiteIndexSpec | ReplicaIndexSpec): boolean {\n return (\n ('predicate' in idx && idx.predicate !== undefined) ||\n ('partial' in idx && idx.partial === true)\n );\n}\n\nexport function computeZqlSpecsFromLiteSpecs(\n tables: LiteTableSpecWithReplicationStatus[],\n indexes: readonly (LiteIndexSpec | ReplicaIndexSpec)[],\n {includeBackfillingColumns}: ZqlSpecOptions,\n tableSpecs: Map<string, LiteAndZqlSpec> = new Map(),\n fullTables?: Map<string, LiteTableSpec>,\n lc?: LogContext,\n): Map<string, LiteAndZqlSpec> {\n tableSpecs.clear();\n fullTables?.clear();\n\n const uniqueIndexColumns = new Map<string, string[][]>();\n // A partial unique index only constrains the rows matching its predicate,\n // so it cannot serve as a row key.\n for (const {tableName, columns} of indexes.filter(\n idx => idx.unique && !isPartialIndex(idx),\n )) {\n getOrInsertComputed(uniqueIndexColumns, tableName, newArray).push(\n Object.keys(columns),\n );\n }\n\n tables.forEach(fullTable => {\n fullTables?.set(fullTable.name, fullTable);\n\n const backfilling = new Set(fullTable.backfilling);\n // Only include columns that:\n // - have a defined ZQL Value\n // - aren't backfilling if `includeBackfillingColumns` is false\n const visibleColumns = Object.entries(fullTable.columns).filter(\n ([col, {dataType}]) =>\n liteTypeToZqlValueType(dataType) &&\n (includeBackfillingColumns || !backfilling.has(col)),\n );\n const notNullColumns = new Set(\n visibleColumns\n .filter(\n ([col, {dataType}]) =>\n !nullableUpstream(dataType) || fullTable.primaryKey?.includes(col),\n )\n .map(([col]) => col),\n );\n\n const uniqueKeys = uniqueIndexColumns.get(fullTable.name) ?? [];\n // Examine all column combinations that can serve as a primary key,\n // i.e. excluding indexes over nullable or unsynced columns.\n const keys = uniqueKeys.filter(\n key => difference(new Set(key), notNullColumns).size === 0,\n );\n if (keys.length === 0) {\n // Only include tables with a row key.\n //\n // Note that this will automatically exclude tables that are being\n // backfilled (when includeBackfillingColumns is `false`) since candidate\n // keys only include visible columns.\n lc?.debug?.(\n `not syncing table ${fullTable.name} because it ` +\n (backfilling.size ? 'is being backfilled' : 'has no primary key'),\n );\n return;\n }\n // Pick the \"best\" (i.e. shortest) key for default IVM operations.\n const primaryKey = keys.sort(keyCmp)[0];\n\n const tableSpec = {\n ...fullTable,\n columns: Object.fromEntries(visibleColumns),\n primaryKey: v.parse(primaryKey, primaryKeySchema),\n uniqueKeys: uniqueKeys.map(key => v.parse(key, primaryKeySchema)),\n allPotentialPrimaryKeys: keys.map(key => v.parse(key, primaryKeySchema)),\n minRowVersion: fullTable.minRowVersion ?? null,\n };\n\n tableSpecs.set(tableSpec.name, {\n tableSpec,\n zqlSpec: Object.fromEntries(\n Object.entries(tableSpec.columns).map(([name, {dataType}]) => [\n name,\n mapLiteDataTypeToZqlSchemaValue(dataType),\n ]),\n ),\n });\n });\n return tableSpecs;\n}\n\nexport function mustGetTableSpec(\n tableSpecs: Map<string, LiteAndZqlSpec>,\n tableName: string,\n): LiteAndZqlSpec {\n const tableSpec = tableSpecs.get(tableName);\n if (!tableSpec) {\n throw new Error(\n `table '${tableName}' is not one of: ${[...tableSpecs.keys()]\n .filter(t => !t.includes('.') && !t.startsWith('_litestream_'))\n .sort()}. ` +\n `Check the spelling and ensure that the table has a primary key.`,\n );\n }\n return tableSpec;\n}\n\n// Deterministic comparator for favoring shorter row keys.\nfunction keyCmp(a: string[], b: string[]) {\n if (a.length !== b.length) {\n return a.length - b.length; // Fewer columns are better.\n }\n for (let i = 0; i < a.length; i++) {\n if (a[i] < b[i]) {\n return -1;\n }\n if (a[i] > b[i]) {\n return 1;\n }\n }\n return 0;\n}\n"],"mappings":";;;;;;;;;;AAgDA,SAAgB,WACd,IACA,oBAAoB,MACpB,mBAAmB,MACmB;CACtC,MAAM,UAAU,GACb,QACC;;;;;;;;;;;;;;OAeF,EACC,IAAI;CAEP,MAAM,iBAAiB,mBACnB,IAAI,qBAAqB,EAAE,EAAE,kBAAkB,oBAC/C,IAAI,IAAI;CACZ,MAAM,SAA+C,CAAC;CACtD,IAAI;CAEJ,QAAQ,SAAQ,QAAO;EACrB,IAAI,IAAI,UAAU,OAAO,MAAM;GAE7B,QAAQ;IACN,MAAM,IAAI;IACV,SAAS,CAAC;IACV,aAAa,CAAC;IACd,eAAe,eAAe,IAAI,IAAI,KAAK,KAAK;GAClD;GACA,OAAO,KAAK,KAAK;EACnB;EAGA,IAAI;EACJ,IAAI;EAKJ,MAAM,WAAW,oBACb,oBAAoB,YAAY,EAAE,GAAG,UAAU,IAAI,OAAO,IAAI,IAAI,IAClE,KAAA;EACJ,IAAI,UAAU;GAEZ,WAAW,yBAAyB,QAAQ;GAC5C,kBAAkB,SAAS,UACvB,SAAS,SACP,MACA,MACF;EACN,OAAO;GAEL,WAAW,IAAI;GACf,kBAAkB,QAAQ,IAAI,IAAI,IAC9B,OAAO,IAAI,IAAI,IACb,MACA,MACF;EACN;EAEA,MAAM,QAAQ,IAAI,QAAQ;GACxB,KAAK,OAAO,KAAK,MAAM,OAAO,EAAE,SAAS;GACzC;GACA,wBAAwB;GACxB,SAAS,IAAI,YAAY;GACzB,MAAM,IAAI;GACV;EACF;EACA,IAAI,UAAU,eACZ,MAAM,YAAY,KAAK,IAAI,IAAI;EAEjC,IAAI,IAAI,QAAQ;GACd,MAAM,eAAe,CAAC;GACtB,OAAO,MAAM,WAAW,SAAS,IAAI,QACnC,MAAM,WAAW,KAAK,EAAE;GAE1B,MAAM,WAAW,IAAI,SAAS,KAAK,IAAI;EACzC;CACF,CAAC;CAED,OAAO;AACT;AAeA,SAAgB,YAAY,IAAkC;CAC5D,MAAM,UAAU,GACb,QACC;;;;;;;;;;;;;wCAcF,EACC,IAAI;CASP,MAAM,MAAiC,CAAC;CACxC,KAAK,MAAM,EACT,WAAW,MACX,WACA,QACA,SACA,QACA,SACG,SACH,IAAI,IAAI,GAAG,EAAE,GAAG,SAAS,MAEvB,KAAK,IAAI,GAAG,EAAE,CAAC,EAAE,QAAQ,UAAU;MAEnC,IAAI,KAAK;EACP;EACA;EACA,SAAS,GAAE,SAAS,IAAG;EACvB,QAAQ,WAAW;EACnB,GAAI,YAAY,IAAI,CAAC,IAAI,EAAC,SAAS,KAAI;CACzC,CAAC;CAIL,OAAO;AACT;;;;;;;;;;;;;;;;AA4BA,SAAgB,gBACd,IACA,SACA,MACA,6BAA0C,IAAI,IAAI,GAClD,YAC6B;CAC7B,OAAO,6BACL,WAAW,OAAO,GAClB,YAAY,OAAO,GACnB,MACA,YACA,YACA,EACF;AACF;AAEA,SAAS,eAAe,KAAgD;CACtE,OACG,eAAe,OAAO,IAAI,cAAc,KAAA,KACxC,aAAa,OAAO,IAAI,YAAY;AAEzC;AAEA,SAAgB,6BACd,QACA,SACA,EAAC,6BACD,6BAA0C,IAAI,IAAI,GAClD,YACA,IAC6B;CAC7B,WAAW,MAAM;CACjB,YAAY,MAAM;CAElB,MAAM,qCAAqB,IAAI,IAAwB;CAGvD,KAAK,MAAM,EAAC,WAAW,aAAY,QAAQ,QACzC,QAAO,IAAI,UAAU,CAAC,eAAe,GAAG,CAC1C,GACE,oBAAoB,oBAAoB,WAAW,QAAQ,EAAE,KAC3D,OAAO,KAAK,OAAO,CACrB;CAGF,OAAO,SAAQ,cAAa;EAC1B,YAAY,IAAI,UAAU,MAAM,SAAS;EAEzC,MAAM,cAAc,IAAI,IAAI,UAAU,WAAW;EAIjD,MAAM,iBAAiB,OAAO,QAAQ,UAAU,OAAO,EAAE,QACtD,CAAC,KAAK,EAAC,gBACN,uBAAuB,QAAQ,MAC9B,6BAA6B,CAAC,YAAY,IAAI,GAAG,EACtD;EACA,MAAM,iBAAiB,IAAI,IACzB,eACG,QACE,CAAC,KAAK,EAAC,gBACN,CAAC,iBAAiB,QAAQ,KAAK,UAAU,YAAY,SAAS,GAAG,CACrE,EACC,KAAK,CAAC,SAAS,GAAG,CACvB;EAEA,MAAM,aAAa,mBAAmB,IAAI,UAAU,IAAI,KAAK,CAAC;EAG9D,MAAM,OAAO,WAAW,QACtB,QAAO,WAAW,IAAI,IAAI,GAAG,GAAG,cAAc,EAAE,SAAS,CAC3D;EACA,IAAI,KAAK,WAAW,GAAG;GAMrB,IAAI,QACF,qBAAqB,UAAU,KAAK,iBACjC,YAAY,OAAO,wBAAwB,qBAChD;GACA;EACF;EAEA,MAAM,aAAa,KAAK,KAAK,MAAM,EAAE;EAErC,MAAM,YAAY;GAChB,GAAG;GACH,SAAS,OAAO,YAAY,cAAc;GAC1C,YAAY,MAAQ,YAAY,gBAAgB;GAChD,YAAY,WAAW,KAAI,QAAO,MAAQ,KAAK,gBAAgB,CAAC;GAChE,yBAAyB,KAAK,KAAI,QAAO,MAAQ,KAAK,gBAAgB,CAAC;GACvE,eAAe,UAAU,iBAAiB;EAC5C;EAEA,WAAW,IAAI,UAAU,MAAM;GAC7B;GACA,SAAS,OAAO,YACd,OAAO,QAAQ,UAAU,OAAO,EAAE,KAAK,CAAC,MAAM,EAAC,gBAAe,CAC5D,MACA,gCAAgC,QAAQ,CAC1C,CAAC,CACH;EACF,CAAC;CACH,CAAC;CACD,OAAO;AACT;AAEA,SAAgB,iBACd,YACA,WACgB;CAChB,MAAM,YAAY,WAAW,IAAI,SAAS;CAC1C,IAAI,CAAC,WACH,MAAM,IAAI,MACR,UAAU,UAAU,mBAAmB,CAAC,GAAG,WAAW,KAAK,CAAC,EACzD,QAAO,MAAK,CAAC,EAAE,SAAS,GAAG,KAAK,CAAC,EAAE,WAAW,cAAc,CAAC,EAC7D,KAAK,EAAE,kEAEZ;CAEF,OAAO;AACT;AAGA,SAAS,OAAO,GAAa,GAAa;CACxC,IAAI,EAAE,WAAW,EAAE,QACjB,OAAO,EAAE,SAAS,EAAE;CAEtB,KAAK,IAAI,IAAI,GAAG,IAAI,EAAE,QAAQ,KAAK;EACjC,IAAI,EAAE,KAAK,EAAE,IACX,OAAO;EAET,IAAI,EAAE,KAAK,EAAE,IACX,OAAO;CAEX;CACA,OAAO;AACT"}
|
|
@@ -81,6 +81,9 @@ function initEventSink(lc, { taskID, cloudEvent }) {
|
|
|
81
81
|
}
|
|
82
82
|
};
|
|
83
83
|
}
|
|
84
|
+
function publishEvent(lc, event) {
|
|
85
|
+
publishFn(lc, event);
|
|
86
|
+
}
|
|
84
87
|
async function publishCriticalEvent(lc, event) {
|
|
85
88
|
await publishFn(lc, event);
|
|
86
89
|
}
|
|
@@ -96,6 +99,6 @@ function makeErrorDetails(e) {
|
|
|
96
99
|
return errorDetails;
|
|
97
100
|
}
|
|
98
101
|
//#endregion
|
|
99
|
-
export { initEventSink, makeErrorDetails, publishCriticalEvent };
|
|
102
|
+
export { initEventSink, makeErrorDetails, publishCriticalEvent, publishEvent };
|
|
100
103
|
|
|
101
104
|
//# sourceMappingURL=events.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"events.js","names":[],"sources":["../../../../../zero-cache/src/observability/events.ts"],"sourcesContent":["import {gzip} from 'node:zlib';\nimport type {LogContext} from '@rocicorp/logger';\nimport {resolver} from '@rocicorp/resolver';\nimport {CloudEvent, emitterFor, httpTransport} from 'cloudevents';\nimport {nanoid} from 'nanoid';\nimport {stringify} from '../../../shared/src/bigint-json.ts';\nimport {isJSONValue, type JSONObject} from '../../../shared/src/json.ts';\nimport {must} from '../../../shared/src/must.ts';\nimport {promiseVoid} from '../../../shared/src/resolved-promises.ts';\nimport {sleep} from '../../../shared/src/sleep.ts';\nimport * as v from '../../../shared/src/valita.ts';\nimport {type ZeroEvent} from '../../../zero-events/src/index.ts';\nimport type {NormalizedZeroConfig} from '../config/normalize.ts';\n\nconst MAX_PUBLISH_ATTEMPTS = 6;\nconst INITIAL_PUBLISH_BACKOFF_MS = 500;\n\ntype PublisherFn = (lc: LogContext, event: ZeroEvent) => Promise<void>;\n\nlet publishFn: PublisherFn = (lc, {type}) => {\n lc.warn?.(\n `Cannot publish \"${type}\" event before initEventSink(). ` +\n `This is only expected in unit tests.`,\n );\n return promiseVoid;\n};\n\nconst attributeValueSchema = v.union(v.string(), v.number(), v.boolean());\n\nconst eventSchema = v.record(attributeValueSchema);\n\ntype PartialEvent = v.Infer<typeof eventSchema>;\n\n// Note: This conforms to the format of the knative K_CE_OVERRIDES binding:\n// https://github.com/knative/eventing/blob/main/docs/spec/sources.md#sinkbinding\nconst extensionsObjectSchema = v.object({extensions: eventSchema});\n\nasync function base64gzip(str: string): Promise<string> {\n const {promise: gzipped, resolve, reject} = resolver<Buffer>();\n gzip(Buffer.from(str), (err, buf) => (err ? reject(err) : resolve(buf)));\n return (await gzipped).toString('base64');\n}\n\n/**\n * Initializes a per-process event sink according to the cloud event\n * parameters in the ZeroConfig. This must be called at the beginning\n * of the process, before any ZeroEvents are generated / published.\n */\nexport function initEventSink(\n lc: LogContext,\n {taskID, cloudEvent}: Pick<NormalizedZeroConfig, 'taskID' | 'cloudEvent'>,\n) {\n if (!cloudEvent.sinkEnv) {\n // The default implementation just outputs the events to logs.\n publishFn = (lc, event) => {\n lc.info?.(`ZeroEvent: ${event.type}`, event);\n return promiseVoid;\n };\n return;\n }\n\n let overrides: PartialEvent = {};\n\n if (cloudEvent.extensionOverridesEnv) {\n const strVal = must(process.env[cloudEvent.extensionOverridesEnv]);\n const {extensions} = v.parse(JSON.parse(strVal), extensionsObjectSchema);\n overrides = extensions;\n }\n\n async function createCloudEvent(event: ZeroEvent) {\n const {type, time} = event;\n const json = stringify(event);\n const data = await base64gzip(json);\n\n return new CloudEvent({\n id: nanoid(),\n source: taskID,\n type,\n time,\n // Pass `data` as text/plain to prevent intermediaries from\n // base64-decoding it. It is the responsibility of the final processor\n // to recognize that datacontentencoding === \"gzip\" and unpack the\n // `data` accordingly before parsing it.\n datacontenttype: 'text/plain',\n datacontentencoding: 'gzip',\n data,\n ...overrides,\n });\n }\n\n const sinkURI = must(process.env[cloudEvent.sinkEnv]);\n const emit = emitterFor(httpTransport(sinkURI));\n lc.debug?.(`Publishing ZeroEvents to ${sinkURI}`);\n\n publishFn = async (lc, event) => {\n let cloudEvent: CloudEvent<string>;\n try {\n cloudEvent = await createCloudEvent(event);\n } catch (e) {\n lc.error?.(`Error creating CloudEvent ${event.type}`, e);\n return;\n }\n lc.debug?.(`Publishing CloudEvent: ${cloudEvent.type}`);\n\n for (let i = 0; i < MAX_PUBLISH_ATTEMPTS; i++) {\n if (i > 0) {\n // exponential backoff on retries\n await sleep(INITIAL_PUBLISH_BACKOFF_MS * 2 ** (i - 1));\n }\n try {\n await emit(cloudEvent);\n // Avoid logging the (possibly large and) unreadable data field.\n const {data: _, ...event} = cloudEvent;\n lc.info?.(`Published CloudEvent: ${cloudEvent.type}`, {event});\n return;\n } catch (e) {\n lc.warn?.(`Error publishing ${cloudEvent.type} (attempt ${i + 1})`, e);\n }\n }\n };\n}\n\nexport function initEventSinkForTesting(sink: ZeroEvent[], now = new Date()) {\n publishFn = (lc, event) => {\n lc.info?.(`Testing event sink received ${event.type} event`, event);\n // Replace the default Date.now() with the test instance for determinism.\n sink.push({...event, time: now.toISOString()});\n return promiseVoid;\n };\n}\n\nexport function publishEvent<E extends ZeroEvent>(lc: LogContext, event: E) {\n void publishFn(lc, event);\n}\n\nexport async function publishCriticalEvent<E extends ZeroEvent>(\n lc: LogContext,\n event: E,\n) {\n await publishFn(lc, event);\n}\n\nexport function makeErrorDetails(e: unknown): JSONObject {\n const err = e instanceof Error ? e : new Error(String(e));\n const errorDetails: JSONObject = {\n name: err.name,\n message: err.message,\n stack: err.stack,\n cause: err.cause ? makeErrorDetails(err.cause) : undefined,\n };\n // Include any enumerable properties (e.g. of Error subtypes).\n for (const [field, value] of Object.entries(err)) {\n if (isJSONValue(value, [])) {\n errorDetails[field] = value;\n }\n }\n return errorDetails;\n}\n"],"mappings":";;;;;;;;;;;AAcA,IAAM,uBAAuB;AAC7B,IAAM,6BAA6B;AAInC,IAAI,aAA0B,IAAI,EAAC,WAAU;CAC3C,GAAG,OACD,mBAAmB,KAAK,qEAE1B;CACA,OAAO;AACT;AAEA,IAAM,uBAAuB,eAAE,MAAM,eAAE,OAAO,GAAG,eAAE,OAAO,GAAG,eAAE,QAAQ,CAAC;AAExE,IAAM,cAAc,eAAE,OAAO,oBAAoB;AAMjD,IAAM,yBAAyB,eAAE,OAAO,EAAC,YAAY,YAAW,CAAC;AAEjE,eAAe,WAAW,KAA8B;CACtD,MAAM,EAAC,SAAS,SAAS,SAAS,WAAU,SAAiB;CAC7D,KAAK,OAAO,KAAK,GAAG,IAAI,KAAK,QAAS,MAAM,OAAO,GAAG,IAAI,QAAQ,GAAG,CAAE;CACvE,QAAQ,MAAM,SAAS,SAAS,QAAQ;AAC1C;;;;;;AAOA,SAAgB,cACd,IACA,EAAC,QAAQ,cACT;CACA,IAAI,CAAC,WAAW,SAAS;EAEvB,aAAa,IAAI,UAAU;GACzB,GAAG,OAAO,cAAc,MAAM,QAAQ,KAAK;GAC3C,OAAO;EACT;EACA;CACF;CAEA,IAAI,YAA0B,CAAC;CAE/B,IAAI,WAAW,uBAAuB;EACpC,MAAM,SAAS,KAAK,QAAQ,IAAI,WAAW,sBAAsB;EACjE,MAAM,EAAC,eAAc,MAAQ,KAAK,MAAM,MAAM,GAAG,sBAAsB;EACvE,YAAY;CACd;CAEA,eAAe,iBAAiB,OAAkB;EAChD,MAAM,EAAC,MAAM,SAAQ;EAErB,MAAM,OAAO,MAAM,WADN,UAAU,KACO,CAAI;EAElC,OAAO,IAAI,WAAW;GACpB,IAAI,OAAO;GACX,QAAQ;GACR;GACA;GAKA,iBAAiB;GACjB,qBAAqB;GACrB;GACA,GAAG;EACL,CAAC;CACH;CAEA,MAAM,UAAU,KAAK,QAAQ,IAAI,WAAW,QAAQ;CACpD,MAAM,OAAO,WAAW,cAAc,OAAO,CAAC;CAC9C,GAAG,QAAQ,4BAA4B,SAAS;CAEhD,YAAY,OAAO,IAAI,UAAU;EAC/B,IAAI;EACJ,IAAI;GACF,aAAa,MAAM,iBAAiB,KAAK;EAC3C,SAAS,GAAG;GACV,GAAG,QAAQ,6BAA6B,MAAM,QAAQ,CAAC;GACvD;EACF;EACA,GAAG,QAAQ,0BAA0B,WAAW,MAAM;EAEtD,KAAK,IAAI,IAAI,GAAG,IAAI,sBAAsB,KAAK;GAC7C,IAAI,IAAI,GAEN,MAAM,MAAM,6BAA6B,MAAM,IAAI,EAAE;GAEvD,IAAI;IACF,MAAM,KAAK,UAAU;IAErB,MAAM,EAAC,MAAM,GAAG,GAAG,UAAS;IAC5B,GAAG,OAAO,yBAAyB,WAAW,QAAQ,EAAC,MAAK,CAAC;IAC7D;GACF,SAAS,GAAG;IACV,GAAG,OAAO,oBAAoB,WAAW,KAAK,YAAY,IAAI,EAAE,IAAI,CAAC;GACvE;EACF;CACF;AACF;
|
|
1
|
+
{"version":3,"file":"events.js","names":[],"sources":["../../../../../zero-cache/src/observability/events.ts"],"sourcesContent":["import {gzip} from 'node:zlib';\nimport type {LogContext} from '@rocicorp/logger';\nimport {resolver} from '@rocicorp/resolver';\nimport {CloudEvent, emitterFor, httpTransport} from 'cloudevents';\nimport {nanoid} from 'nanoid';\nimport {stringify} from '../../../shared/src/bigint-json.ts';\nimport {isJSONValue, type JSONObject} from '../../../shared/src/json.ts';\nimport {must} from '../../../shared/src/must.ts';\nimport {promiseVoid} from '../../../shared/src/resolved-promises.ts';\nimport {sleep} from '../../../shared/src/sleep.ts';\nimport * as v from '../../../shared/src/valita.ts';\nimport {type ZeroEvent} from '../../../zero-events/src/index.ts';\nimport type {NormalizedZeroConfig} from '../config/normalize.ts';\n\nconst MAX_PUBLISH_ATTEMPTS = 6;\nconst INITIAL_PUBLISH_BACKOFF_MS = 500;\n\ntype PublisherFn = (lc: LogContext, event: ZeroEvent) => Promise<void>;\n\nlet publishFn: PublisherFn = (lc, {type}) => {\n lc.warn?.(\n `Cannot publish \"${type}\" event before initEventSink(). ` +\n `This is only expected in unit tests.`,\n );\n return promiseVoid;\n};\n\nconst attributeValueSchema = v.union(v.string(), v.number(), v.boolean());\n\nconst eventSchema = v.record(attributeValueSchema);\n\ntype PartialEvent = v.Infer<typeof eventSchema>;\n\n// Note: This conforms to the format of the knative K_CE_OVERRIDES binding:\n// https://github.com/knative/eventing/blob/main/docs/spec/sources.md#sinkbinding\nconst extensionsObjectSchema = v.object({extensions: eventSchema});\n\nasync function base64gzip(str: string): Promise<string> {\n const {promise: gzipped, resolve, reject} = resolver<Buffer>();\n gzip(Buffer.from(str), (err, buf) => (err ? reject(err) : resolve(buf)));\n return (await gzipped).toString('base64');\n}\n\n/**\n * Initializes a per-process event sink according to the cloud event\n * parameters in the ZeroConfig. This must be called at the beginning\n * of the process, before any ZeroEvents are generated / published.\n */\nexport function initEventSink(\n lc: LogContext,\n {taskID, cloudEvent}: Pick<NormalizedZeroConfig, 'taskID' | 'cloudEvent'>,\n) {\n if (!cloudEvent.sinkEnv) {\n // The default implementation just outputs the events to logs.\n publishFn = (lc, event) => {\n lc.info?.(`ZeroEvent: ${event.type}`, event);\n return promiseVoid;\n };\n return;\n }\n\n let overrides: PartialEvent = {};\n\n if (cloudEvent.extensionOverridesEnv) {\n const strVal = must(process.env[cloudEvent.extensionOverridesEnv]);\n const {extensions} = v.parse(JSON.parse(strVal), extensionsObjectSchema);\n overrides = extensions;\n }\n\n async function createCloudEvent(event: ZeroEvent) {\n const {type, time} = event;\n const json = stringify(event);\n const data = await base64gzip(json);\n\n return new CloudEvent({\n id: nanoid(),\n source: taskID,\n type,\n time,\n // Pass `data` as text/plain to prevent intermediaries from\n // base64-decoding it. It is the responsibility of the final processor\n // to recognize that datacontentencoding === \"gzip\" and unpack the\n // `data` accordingly before parsing it.\n datacontenttype: 'text/plain',\n datacontentencoding: 'gzip',\n data,\n ...overrides,\n });\n }\n\n const sinkURI = must(process.env[cloudEvent.sinkEnv]);\n const emit = emitterFor(httpTransport(sinkURI));\n lc.debug?.(`Publishing ZeroEvents to ${sinkURI}`);\n\n publishFn = async (lc, event) => {\n let cloudEvent: CloudEvent<string>;\n try {\n cloudEvent = await createCloudEvent(event);\n } catch (e) {\n lc.error?.(`Error creating CloudEvent ${event.type}`, e);\n return;\n }\n lc.debug?.(`Publishing CloudEvent: ${cloudEvent.type}`);\n\n for (let i = 0; i < MAX_PUBLISH_ATTEMPTS; i++) {\n if (i > 0) {\n // exponential backoff on retries\n await sleep(INITIAL_PUBLISH_BACKOFF_MS * 2 ** (i - 1));\n }\n try {\n await emit(cloudEvent);\n // Avoid logging the (possibly large and) unreadable data field.\n const {data: _, ...event} = cloudEvent;\n lc.info?.(`Published CloudEvent: ${cloudEvent.type}`, {event});\n return;\n } catch (e) {\n lc.warn?.(`Error publishing ${cloudEvent.type} (attempt ${i + 1})`, e);\n }\n }\n };\n}\n\nexport function initEventSinkForTesting(sink: ZeroEvent[], now = new Date()) {\n publishFn = (lc, event) => {\n lc.info?.(`Testing event sink received ${event.type} event`, event);\n // Replace the default Date.now() with the test instance for determinism.\n sink.push({...event, time: now.toISOString()});\n return promiseVoid;\n };\n}\n\nexport function publishEvent<E extends ZeroEvent>(lc: LogContext, event: E) {\n void publishFn(lc, event);\n}\n\nexport async function publishCriticalEvent<E extends ZeroEvent>(\n lc: LogContext,\n event: E,\n) {\n await publishFn(lc, event);\n}\n\nexport function makeErrorDetails(e: unknown): JSONObject {\n const err = e instanceof Error ? e : new Error(String(e));\n const errorDetails: JSONObject = {\n name: err.name,\n message: err.message,\n stack: err.stack,\n cause: err.cause ? makeErrorDetails(err.cause) : undefined,\n };\n // Include any enumerable properties (e.g. of Error subtypes).\n for (const [field, value] of Object.entries(err)) {\n if (isJSONValue(value, [])) {\n errorDetails[field] = value;\n }\n }\n return errorDetails;\n}\n"],"mappings":";;;;;;;;;;;AAcA,IAAM,uBAAuB;AAC7B,IAAM,6BAA6B;AAInC,IAAI,aAA0B,IAAI,EAAC,WAAU;CAC3C,GAAG,OACD,mBAAmB,KAAK,qEAE1B;CACA,OAAO;AACT;AAEA,IAAM,uBAAuB,eAAE,MAAM,eAAE,OAAO,GAAG,eAAE,OAAO,GAAG,eAAE,QAAQ,CAAC;AAExE,IAAM,cAAc,eAAE,OAAO,oBAAoB;AAMjD,IAAM,yBAAyB,eAAE,OAAO,EAAC,YAAY,YAAW,CAAC;AAEjE,eAAe,WAAW,KAA8B;CACtD,MAAM,EAAC,SAAS,SAAS,SAAS,WAAU,SAAiB;CAC7D,KAAK,OAAO,KAAK,GAAG,IAAI,KAAK,QAAS,MAAM,OAAO,GAAG,IAAI,QAAQ,GAAG,CAAE;CACvE,QAAQ,MAAM,SAAS,SAAS,QAAQ;AAC1C;;;;;;AAOA,SAAgB,cACd,IACA,EAAC,QAAQ,cACT;CACA,IAAI,CAAC,WAAW,SAAS;EAEvB,aAAa,IAAI,UAAU;GACzB,GAAG,OAAO,cAAc,MAAM,QAAQ,KAAK;GAC3C,OAAO;EACT;EACA;CACF;CAEA,IAAI,YAA0B,CAAC;CAE/B,IAAI,WAAW,uBAAuB;EACpC,MAAM,SAAS,KAAK,QAAQ,IAAI,WAAW,sBAAsB;EACjE,MAAM,EAAC,eAAc,MAAQ,KAAK,MAAM,MAAM,GAAG,sBAAsB;EACvE,YAAY;CACd;CAEA,eAAe,iBAAiB,OAAkB;EAChD,MAAM,EAAC,MAAM,SAAQ;EAErB,MAAM,OAAO,MAAM,WADN,UAAU,KACO,CAAI;EAElC,OAAO,IAAI,WAAW;GACpB,IAAI,OAAO;GACX,QAAQ;GACR;GACA;GAKA,iBAAiB;GACjB,qBAAqB;GACrB;GACA,GAAG;EACL,CAAC;CACH;CAEA,MAAM,UAAU,KAAK,QAAQ,IAAI,WAAW,QAAQ;CACpD,MAAM,OAAO,WAAW,cAAc,OAAO,CAAC;CAC9C,GAAG,QAAQ,4BAA4B,SAAS;CAEhD,YAAY,OAAO,IAAI,UAAU;EAC/B,IAAI;EACJ,IAAI;GACF,aAAa,MAAM,iBAAiB,KAAK;EAC3C,SAAS,GAAG;GACV,GAAG,QAAQ,6BAA6B,MAAM,QAAQ,CAAC;GACvD;EACF;EACA,GAAG,QAAQ,0BAA0B,WAAW,MAAM;EAEtD,KAAK,IAAI,IAAI,GAAG,IAAI,sBAAsB,KAAK;GAC7C,IAAI,IAAI,GAEN,MAAM,MAAM,6BAA6B,MAAM,IAAI,EAAE;GAEvD,IAAI;IACF,MAAM,KAAK,UAAU;IAErB,MAAM,EAAC,MAAM,GAAG,GAAG,UAAS;IAC5B,GAAG,OAAO,yBAAyB,WAAW,QAAQ,EAAC,MAAK,CAAC;IAC7D;GACF,SAAS,GAAG;IACV,GAAG,OAAO,oBAAoB,WAAW,KAAK,YAAY,IAAI,EAAE,IAAI,CAAC;GACvE;EACF;CACF;AACF;AAWA,SAAgB,aAAkC,IAAgB,OAAU;CAC1E,UAAe,IAAI,KAAK;AAC1B;AAEA,eAAsB,qBACpB,IACA,OACA;CACA,MAAM,UAAU,IAAI,KAAK;AAC3B;AAEA,SAAgB,iBAAiB,GAAwB;CACvD,MAAM,MAAM,aAAa,QAAQ,IAAI,IAAI,MAAM,OAAO,CAAC,CAAC;CACxD,MAAM,eAA2B;EAC/B,MAAM,IAAI;EACV,SAAS,IAAI;EACb,OAAO,IAAI;EACX,OAAO,IAAI,QAAQ,iBAAiB,IAAI,KAAK,IAAI,KAAA;CACnD;CAEA,KAAK,MAAM,CAAC,OAAO,UAAU,OAAO,QAAQ,GAAG,GAC7C,IAAI,YAAY,OAAO,CAAC,CAAC,GACvB,aAAa,SAAS;CAG1B,OAAO;AACT"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"metrics.d.ts","sourceRoot":"","sources":["../../../../../zero-cache/src/observability/metrics.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACV,OAAO,EACP,SAAS,EAET,aAAa,EACb,eAAe,EACf,aAAa,EACd,MAAM,oBAAoB,CAAC;
|
|
1
|
+
{"version":3,"file":"metrics.d.ts","sourceRoot":"","sources":["../../../../../zero-cache/src/observability/metrics.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACV,OAAO,EACP,SAAS,EAET,aAAa,EACb,eAAe,EACf,aAAa,EACd,MAAM,oBAAoB,CAAC;AAM5B,MAAM,MAAM,QAAQ,GAChB,aAAa,GACb,SAAS,GACT,MAAM,GACN,UAAU,GACV,QAAQ,CAAC;AAIb,KAAK,OAAO,GAAG,aAAa,GAAG;IAAC,WAAW,EAAE,MAAM,CAAA;CAAC,CAAC;AACrD,KAAK,gBAAgB,GAAG,OAAO,GAAG;IAAC,gBAAgB,EAAE,MAAM,EAAE,CAAA;CAAC,CAAC;AAE/D,eAAO,MAAM,iCAAiC,YAC5C,2BAA2B,EAC3B,iCAAiC,EACjC,2BAA2B,CACnB,CAAC;AAEX,eAAO,MAAM,oCAAoC,UAEhD,CAAC;AAoBF,wBAAgB,wBAAwB,CACtC,QAAQ,EAAE,QAAQ,EAClB,IAAI,EAAE,MAAM,EACZ,WAAW,EAAE,MAAM,GAClB,aAAa,CAAC;AACjB,wBAAgB,wBAAwB,CACtC,QAAQ,EAAE,QAAQ,EAClB,IAAI,EAAE,MAAM,EACZ,IAAI,EAAE,OAAO,GACZ,aAAa,CAAC;AAcjB;;;;;;GAMG;AACH,MAAM,MAAM,gBAAgB,GAAG;IAC7B;;;;;;OAMG;IACH,QAAQ,CACN,UAAU,EAAE,MAAM,EAClB,UAAU,CAAC,EAAE,UAAU,CAAC,SAAS,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,GAC9C,IAAI,CAAC;CACT,CAAC;AAoBF;;;GAGG;AACH,wBAAgB,yBAAyB,CACvC,QAAQ,EAAE,QAAQ,EAClB,IAAI,EAAE,MAAM,EACZ,IAAI,EAAE,gBAAgB,GACrB,SAAS,CAWX;AAED,wBAAgB,oBAAoB,CAClC,QAAQ,EAAE,QAAQ,EAClB,IAAI,EAAE,MAAM,EACZ,IAAI,EAAE,gBAAgB,GACrB,gBAAgB,CAelB;AAKD,wBAAgB,0BAA0B,CACxC,QAAQ,EAAE,QAAQ,EAClB,IAAI,EAAE,MAAM,EACZ,IAAI,EAAE,OAAO,GACZ,gBAAgB,CAQlB;AAED;;;;;;;;;;;;;;;;;GAiBG;AACH,wBAAgB,2BAA2B,CACzC,QAAQ,EAAE,QAAQ,EAClB,IAAI,EAAE,MAAM,EACZ,WAAW,EAAE,MAAM,GAClB,gBAAgB,CAMlB;AAID,wBAAgB,kBAAkB,CAChC,QAAQ,EAAE,QAAQ,EAClB,IAAI,EAAE,MAAM,EACZ,WAAW,EAAE,MAAM,GAClB,OAAO,CAAC;AACX,wBAAgB,kBAAkB,CAChC,QAAQ,EAAE,QAAQ,EAClB,IAAI,EAAE,MAAM,EACZ,IAAI,EAAE,OAAO,GACZ,OAAO,CAAC;AAgBX,wBAAgB,gBAAgB,CAC9B,QAAQ,EAAE,QAAQ,EAClB,IAAI,EAAE,MAAM,EACZ,WAAW,EAAE,MAAM,GAClB,eAAe,CAAC;AACnB,wBAAgB,gBAAgB,CAC9B,QAAQ,EAAE,QAAQ,EAClB,IAAI,EAAE,MAAM,EACZ,IAAI,EAAE,OAAO,GACZ,eAAe,CAAC"}
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { getOrInsertComputed } from "../../../shared/src/map.js";
|
|
1
2
|
import { metrics } from "@opentelemetry/api";
|
|
2
3
|
//#region ../zero-cache/src/observability/metrics.ts
|
|
3
4
|
var meter;
|
|
@@ -27,13 +28,7 @@ function getMeter() {
|
|
|
27
28
|
}
|
|
28
29
|
function cache() {
|
|
29
30
|
const instruments = /* @__PURE__ */ new Map();
|
|
30
|
-
return (name, creator) =>
|
|
31
|
-
const existing = instruments.get(name);
|
|
32
|
-
if (existing) return existing;
|
|
33
|
-
const ret = creator(name);
|
|
34
|
-
instruments.set(name, ret);
|
|
35
|
-
return ret;
|
|
36
|
-
};
|
|
31
|
+
return (name, creator) => getOrInsertComputed(instruments, name, creator);
|
|
37
32
|
}
|
|
38
33
|
var upDownCounters = cache();
|
|
39
34
|
function getOrCreateUpDownCounter(category, name, opts) {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"metrics.js","names":[],"sources":["../../../../../zero-cache/src/observability/metrics.ts"],"sourcesContent":["import type {\n Counter,\n Histogram,\n Meter,\n MetricOptions,\n ObservableGauge,\n UpDownCounter,\n} from '@opentelemetry/api';\nimport {metrics} from '@opentelemetry/api';\n\n// intentional lazy initialization so it is not started before the SDK is started.\n\nexport type Category =\n | 'replication' // postgres to replica\n | 'replica' // health of replica and litestream backup\n | 'sync' // replica to client\n | 'mutation'\n | 'server';\n\nlet meter: Meter | undefined;\n\ntype Options = MetricOptions & {description: string};\ntype HistogramOptions = Options & {bucketBoundaries: number[]};\n\nexport const NATIVE_HISTOGRAM_INSTRUMENT_NAMES = [\n 'zero.sync.view_syncer_lag',\n 'zero.sync.view_syncer_hydration',\n 'zero.sync.e2e_serving_lag',\n] as const;\n\nexport const LONG_DURATION_HISTOGRAM_BOUNDARIES_S = [\n 1, 2, 5, 10, 30, 60, 120, 300, 600, 1200, 2400, 3600, 7200,\n];\n\nfunction getMeter() {\n if (!meter) {\n meter = metrics.getMeter('zero');\n }\n return meter;\n}\n\nfunction cache<TRet>(): (\n name: string,\n creator: (name: string) => TRet,\n) => TRet {\n const instruments = new Map<string, TRet>();\n return (name: string, creator: (name: string) => TRet)
|
|
1
|
+
{"version":3,"file":"metrics.js","names":[],"sources":["../../../../../zero-cache/src/observability/metrics.ts"],"sourcesContent":["import type {\n Counter,\n Histogram,\n Meter,\n MetricOptions,\n ObservableGauge,\n UpDownCounter,\n} from '@opentelemetry/api';\nimport {metrics} from '@opentelemetry/api';\nimport {getOrInsertComputed} from '../../../shared/src/map.ts';\n\n// intentional lazy initialization so it is not started before the SDK is started.\n\nexport type Category =\n | 'replication' // postgres to replica\n | 'replica' // health of replica and litestream backup\n | 'sync' // replica to client\n | 'mutation'\n | 'server';\n\nlet meter: Meter | undefined;\n\ntype Options = MetricOptions & {description: string};\ntype HistogramOptions = Options & {bucketBoundaries: number[]};\n\nexport const NATIVE_HISTOGRAM_INSTRUMENT_NAMES = [\n 'zero.sync.view_syncer_lag',\n 'zero.sync.view_syncer_hydration',\n 'zero.sync.e2e_serving_lag',\n] as const;\n\nexport const LONG_DURATION_HISTOGRAM_BOUNDARIES_S = [\n 1, 2, 5, 10, 30, 60, 120, 300, 600, 1200, 2400, 3600, 7200,\n];\n\nfunction getMeter() {\n if (!meter) {\n meter = metrics.getMeter('zero');\n }\n return meter;\n}\n\nfunction cache<TRet extends object>(): (\n name: string,\n creator: (name: string) => TRet,\n) => TRet {\n const instruments = new Map<string, TRet>();\n return (name: string, creator: (name: string) => TRet) =>\n getOrInsertComputed(instruments, name, creator);\n}\n\nconst upDownCounters = cache<UpDownCounter>();\n\nexport function getOrCreateUpDownCounter(\n category: Category,\n name: string,\n description: string,\n): UpDownCounter;\nexport function getOrCreateUpDownCounter(\n category: Category,\n name: string,\n opts: Options,\n): UpDownCounter;\nexport function getOrCreateUpDownCounter(\n category: Category,\n name: string,\n opts: string | Options,\n): UpDownCounter {\n return upDownCounters(name, name =>\n getMeter().createUpDownCounter(\n `zero.${category}.${name}`,\n typeof opts === 'string' ? {description: opts} : opts,\n ),\n );\n}\n\n/**\n * A latency histogram whose {@link recordMs} method accepts raw millisecond\n * durations and converts them to seconds internally.\n *\n * Use {@link getOrCreateLatencyHistogram} to create one — the unit (`'s'`),\n * bucket boundaries, and ms→s conversion are all baked in\n */\nexport type LatencyHistogram = {\n /**\n * Record a duration. Pass the raw elapsed milliseconds — the conversion to\n * seconds (required by the `unit: 's'` OTel histogram) is handled internally.\n *\n * @param durationMs Elapsed time in **milliseconds** (do NOT pre-divide).\n * @param attributes Optional OTel attributes to attach to the observation.\n */\n recordMs(\n durationMs: number,\n attributes?: Parameters<Histogram['record']>[1],\n ): void;\n};\n\n/**\n * Bucket boundaries (in seconds) for zero's latency histograms.\n *\n * The operational range is 1 ms – 5,000 ms (including customers actively\n * tuning queries). ~2× logarithmic steps give proportionally consistent\n * `histogram_quantile` accuracy regardless of where values cluster within\n * that range. 10,000 ms and 30,000 ms are overflow catchers for truly broken\n * states.\n *\n * 1 ms, 2 ms, 5 ms, 10 ms, 20 ms, 50 ms, 100 ms, 200 ms, 500 ms,\n * 1 s, 2 s, 5 s, 10 s, 30 s\n */\nconst LATENCY_HISTOGRAM_BOUNDARIES_S = [\n 0.001, 0.002, 0.005, 0.01, 0.02, 0.05, 0.1, 0.2, 0.5, 1, 2, 5, 10, 30,\n];\n\nconst histograms = cache<Histogram>();\n\n/**\n * Creates a histogram for dimensionless values without applying the\n * millisecond-to-second conversion used by the latency helpers below.\n */\nexport function getOrCreateValueHistogram(\n category: Category,\n name: string,\n opts: HistogramOptions,\n): Histogram {\n const {bucketBoundaries, ...metricOptions} = opts;\n return histograms(name, name =>\n getMeter().createHistogram(`zero.${category}.${name}`, {\n ...metricOptions,\n advice: {\n ...metricOptions.advice,\n explicitBucketBoundaries: bucketBoundaries,\n },\n }),\n );\n}\n\nexport function getOrCreateHistogram(\n category: Category,\n name: string,\n opts: HistogramOptions,\n): LatencyHistogram {\n const {bucketBoundaries, ...metricOptions} = opts;\n const h = histograms(name, name =>\n getMeter().createHistogram(`zero.${category}.${name}`, {\n ...metricOptions,\n advice: {\n ...metricOptions.advice,\n explicitBucketBoundaries: bucketBoundaries,\n },\n }),\n );\n return {\n recordMs: (durationMs, attributes) =>\n h.record(durationMs / 1000, attributes),\n };\n}\n\n// Add instruments created through this helper to\n// NATIVE_HISTOGRAM_INSTRUMENT_NAMES so the SDK exports them as exponential\n// histograms.\nexport function getOrCreateNativeHistogram(\n category: Category,\n name: string,\n opts: Options,\n): LatencyHistogram {\n const h = histograms(name, name =>\n getMeter().createHistogram(`zero.${category}.${name}`, opts),\n );\n return {\n recordMs: (durationMs, attributes) =>\n h.record(durationMs / 1000, attributes),\n };\n}\n\n/**\n * Creates (or retrieves) a latency histogram for the given metric.\n *\n * - `unit` is always `'s'` (seconds), matching the OTel convention.\n * - Bucket boundaries are pre-set for zero's typical operation range\n * (1 ms – 5 s); see {@link LATENCY_HISTOGRAM_BOUNDARIES_S}.\n * - The returned {@link LatencyHistogram} accepts **milliseconds** via\n * `recordMs()`, so callers never need to divide by 1000.\n *\n * @example\n * ```ts\n * readonly #hydrationTime = getOrCreateLatencyHistogram(\n * 'sync', 'hydration-time', 'Time to hydrate a query.',\n * );\n * // ...\n * this.#hydrationTime.recordMs(performance.now() - start);\n * ```\n */\nexport function getOrCreateLatencyHistogram(\n category: Category,\n name: string,\n description: string,\n): LatencyHistogram {\n return getOrCreateHistogram(category, name, {\n description,\n unit: 's',\n bucketBoundaries: LATENCY_HISTOGRAM_BOUNDARIES_S,\n });\n}\n\nconst counters = cache<Counter>();\n\nexport function getOrCreateCounter(\n category: Category,\n name: string,\n description: string,\n): Counter;\nexport function getOrCreateCounter(\n category: Category,\n name: string,\n opts: Options,\n): Counter;\nexport function getOrCreateCounter(\n category: Category,\n name: string,\n opts: string | Options,\n): Counter {\n return counters(name, name =>\n getMeter().createCounter(\n `zero.${category}.${name}`,\n typeof opts === 'string' ? {description: opts} : opts,\n ),\n );\n}\n\nconst gauges = cache<ObservableGauge>();\n\nexport function getOrCreateGauge(\n category: Category,\n name: string,\n description: string,\n): ObservableGauge;\nexport function getOrCreateGauge(\n category: Category,\n name: string,\n opts: Options,\n): ObservableGauge;\nexport function getOrCreateGauge(\n category: Category,\n name: string,\n opts: string | Options,\n): ObservableGauge {\n return gauges(name, name =>\n getMeter().createObservableGauge(\n `zero.${category}.${name}`,\n typeof opts === 'string' ? {description: opts} : opts,\n ),\n );\n}\n"],"mappings":";;;AAoBA,IAAI;AAKJ,IAAa,oCAAoC;CAC/C;CACA;CACA;AACF;AAEA,IAAa,uCAAuC;CAClD;CAAG;CAAG;CAAG;CAAI;CAAI;CAAI;CAAK;CAAK;CAAK;CAAM;CAAM;CAAM;AACxD;AAEA,SAAS,WAAW;CAClB,IAAI,CAAC,OACH,QAAQ,QAAQ,SAAS,MAAM;CAEjC,OAAO;AACT;AAEA,SAAS,QAGC;CACR,MAAM,8BAAc,IAAI,IAAkB;CAC1C,QAAQ,MAAc,YACpB,oBAAoB,aAAa,MAAM,OAAO;AAClD;AAEA,IAAM,iBAAiB,MAAqB;AAY5C,SAAgB,yBACd,UACA,MACA,MACe;CACf,OAAO,eAAe,OAAM,SAC1B,SAAS,EAAE,oBACT,QAAQ,SAAS,GAAG,QACpB,OAAO,SAAS,WAAW,EAAC,aAAa,KAAI,IAAI,IACnD,CACF;AACF;;;;;;;;;;;;;AAmCA,IAAM,iCAAiC;CACrC;CAAO;CAAO;CAAO;CAAM;CAAM;CAAM;CAAK;CAAK;CAAK;CAAG;CAAG;CAAG;CAAI;AACrE;AAEA,IAAM,aAAa,MAAiB;;;;;AAMpC,SAAgB,0BACd,UACA,MACA,MACW;CACX,MAAM,EAAC,kBAAkB,GAAG,kBAAiB;CAC7C,OAAO,WAAW,OAAM,SACtB,SAAS,EAAE,gBAAgB,QAAQ,SAAS,GAAG,QAAQ;EACrD,GAAG;EACH,QAAQ;GACN,GAAG,cAAc;GACjB,0BAA0B;EAC5B;CACF,CAAC,CACH;AACF;AAEA,SAAgB,qBACd,UACA,MACA,MACkB;CAClB,MAAM,EAAC,kBAAkB,GAAG,kBAAiB;CAC7C,MAAM,IAAI,WAAW,OAAM,SACzB,SAAS,EAAE,gBAAgB,QAAQ,SAAS,GAAG,QAAQ;EACrD,GAAG;EACH,QAAQ;GACN,GAAG,cAAc;GACjB,0BAA0B;EAC5B;CACF,CAAC,CACH;CACA,OAAO,EACL,WAAW,YAAY,eACrB,EAAE,OAAO,aAAa,KAAM,UAAU,EAC1C;AACF;AAKA,SAAgB,2BACd,UACA,MACA,MACkB;CAClB,MAAM,IAAI,WAAW,OAAM,SACzB,SAAS,EAAE,gBAAgB,QAAQ,SAAS,GAAG,QAAQ,IAAI,CAC7D;CACA,OAAO,EACL,WAAW,YAAY,eACrB,EAAE,OAAO,aAAa,KAAM,UAAU,EAC1C;AACF;;;;;;;;;;;;;;;;;;;AAoBA,SAAgB,4BACd,UACA,MACA,aACkB;CAClB,OAAO,qBAAqB,UAAU,MAAM;EAC1C;EACA,MAAM;EACN,kBAAkB;CACpB,CAAC;AACH;AAEA,IAAM,WAAW,MAAe;AAYhC,SAAgB,mBACd,UACA,MACA,MACS;CACT,OAAO,SAAS,OAAM,SACpB,SAAS,EAAE,cACT,QAAQ,SAAS,GAAG,QACpB,OAAO,SAAS,WAAW,EAAC,aAAa,KAAI,IAAI,IACnD,CACF;AACF;AAEA,IAAM,SAAS,MAAuB;AAYtC,SAAgB,iBACd,UACA,MACA,MACiB;CACjB,OAAO,OAAO,OAAM,SAClB,SAAS,EAAE,sBACT,QAAQ,SAAS,GAAG,QACpB,OAAO,SAAS,WAAW,EAAC,aAAa,KAAI,IAAI,IACnD,CACF;AACF"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"change-streamer.d.ts","sourceRoot":"","sources":["../../../../../zero-cache/src/server/change-streamer.ts"],"names":[],"mappings":"AAsCA,OAAO,
|
|
1
|
+
{"version":3,"file":"change-streamer.d.ts","sourceRoot":"","sources":["../../../../../zero-cache/src/server/change-streamer.ts"],"names":[],"mappings":"AAsCA,OAAO,EAML,KAAK,MAAM,EACZ,MAAM,uBAAuB,CAAC;AAW/B,wBAA8B,SAAS,CACrC,MAAM,EAAE,MAAM,EACd,GAAG,EAAE,MAAM,CAAC,UAAU,EACtB,GAAG,IAAI,EAAE,MAAM,EAAE,GAChB,OAAO,CAAC,IAAI,CAAC,CAwXf"}
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { assert } from "../../../shared/src/asserts.js";
|
|
2
2
|
import { promiseVoid } from "../../../shared/src/resolved-promises.js";
|
|
3
3
|
import { must } from "../../../shared/src/must.js";
|
|
4
|
-
import { childWorker, parentWorker, singleProcessMode } from "../types/processes.js";
|
|
4
|
+
import { broadcastWorker, childWorker, parentWorker, singleProcessMode } from "../types/processes.js";
|
|
5
5
|
import { getShardConfig } from "../types/shards.js";
|
|
6
6
|
import { getNormalizedZeroConfig } from "../config/zero-config.js";
|
|
7
7
|
import { DatabaseInitError } from "../../../zqlite/src/db.js";
|
|
@@ -11,6 +11,7 @@ import { registerSQLiteCorruptionDiagnosticTarget } from "../db/sqlite-corruptio
|
|
|
11
11
|
import { ProcessManager, exitAfter, runUntilKilled } from "../services/life-cycle.js";
|
|
12
12
|
import { createLogContext } from "./logging.js";
|
|
13
13
|
import { REPLICATOR_URL } from "./worker-urls.js";
|
|
14
|
+
import { installProfileHandler } from "../types/profiler.js";
|
|
14
15
|
import { getServerContext } from "../config/server-context.js";
|
|
15
16
|
import { deleteLiteDB } from "../db/delete-lite-db.js";
|
|
16
17
|
import { warmupConnections } from "../db/warmup.js";
|
|
@@ -20,7 +21,7 @@ import { ReplicationStatusPublisher, replicationStatusError } from "../services/
|
|
|
20
21
|
import { changeLogFileName, deleteChangeLogDB } from "../services/replicator/change-log-db.js";
|
|
21
22
|
import { startReplicaBackupProcess } from "../services/litestream/commands.js";
|
|
22
23
|
import { initializeCustomChangeSource } from "../services/change-source/custom/change-source.js";
|
|
23
|
-
import { initializePostgresChangeSource } from "../services/change-source/pg/change-source.js";
|
|
24
|
+
import { initializePostgresChangeSource } from "../services/change-source/pg/change-source-init.js";
|
|
24
25
|
import { createBackupCleanupMonitor } from "../services/change-streamer/backup-cleanup-monitor-factory.js";
|
|
25
26
|
import { ChangeStreamerHttpServer } from "../services/change-streamer/change-streamer-http.js";
|
|
26
27
|
import { sqliteFileBytes } from "../services/replicator/sqlite-change-log-observability.js";
|
|
@@ -33,11 +34,12 @@ import { LogContext, consoleLogSink } from "@rocicorp/logger";
|
|
|
33
34
|
//#region ../zero-cache/src/server/change-streamer.ts
|
|
34
35
|
var lc = new LogContext("info", {}, consoleLogSink);
|
|
35
36
|
async function runWorker(parent, env, ...argv) {
|
|
37
|
+
installProfileHandler(parent, "change-streamer");
|
|
36
38
|
const config = getNormalizedZeroConfig({
|
|
37
39
|
env,
|
|
38
40
|
argv
|
|
39
41
|
});
|
|
40
|
-
const { taskID, changeStreamer: { port, address, protocol, startupDelayMs, backPressureLimitHeapProportion, flowControlConsensusTimeoutProportion, flowControlSlowSubscriberGracePeriodSeconds, sqliteChangeLogMode, sqliteChangeLogReadPercent, sqliteChangeLogColdReadPercent, sqliteChangeLogComparePercent, sqliteChangeLogRetentionMs, sqliteChangeLogReadBatchRows, sqliteChangeLogPurgeBatchRows, sqliteChangeLogBarrierTimeoutMs }, autoReset, replicationLag, litestream, upstream, change, replica, initialSync, keepaliveTimeoutMs, sqliteCorruptionChecks } = config;
|
|
42
|
+
const { taskID, changeStreamer: { port, address, protocol, startupDelayMs, backPressureLimitHeapProportion, flowControlConsensusTimeoutProportion, flowControlSlowSubscriberGracePeriodSeconds, pgChangeLogEnabled, sqliteChangeLogMode, sqliteChangeLogReadPercent, sqliteChangeLogColdReadPercent, sqliteChangeLogComparePercent, sqliteChangeLogRetentionMs, sqliteChangeLogReadBatchRows, sqliteChangeLogPurgeBatchRows, sqliteChangeLogBarrierTimeoutMs }, autoReset, replicationLag, litestream, upstream, change, replica, initialSync, keepaliveTimeoutMs, sqliteCorruptionChecks } = config;
|
|
41
43
|
startOtelAuto(createLogContext(config, "change-streamer", 0, false), "change-streamer", 0);
|
|
42
44
|
lc = createLogContext(config, "change-streamer");
|
|
43
45
|
registerSQLiteCorruptionDiagnosticTarget({
|
|
@@ -49,7 +51,7 @@ async function runWorker(parent, env, ...argv) {
|
|
|
49
51
|
warmupConnections(lc, changeDB, "change").catch(() => {});
|
|
50
52
|
const shard = getShardConfig(config);
|
|
51
53
|
await initChangeStreamerSchema(lc, changeDB, shard);
|
|
52
|
-
let purgeLock = litestream.backupURL && litestream.executable ? await new PurgeLocker(lc, shard, changeDB).acquire() : null;
|
|
54
|
+
let purgeLock = pgChangeLogEnabled && litestream.backupURL && litestream.executable ? await new PurgeLocker(lc, shard, changeDB).acquire() : null;
|
|
53
55
|
const restoreOptions = {
|
|
54
56
|
litestream,
|
|
55
57
|
constraints: purgeLock ?? void 0
|
|
@@ -72,16 +74,24 @@ async function runWorker(parent, env, ...argv) {
|
|
|
72
74
|
}
|
|
73
75
|
let waitForFirstBackupBeforeServing = false;
|
|
74
76
|
for (const first of [true, false]) try {
|
|
77
|
+
const { pgReplicationEpoch: epoch, pgReplicationSlotPerReplica: slotPerReplica, pgResumeOrphanedSlotGracePeriodMs: inactiveReplicaGracePeriodMs } = upstream;
|
|
75
78
|
const { changeSource, subscriptionState, destinationBackupURL, replicaID, waitForBackupBeforeServing } = upstream.type === "pg" ? await initializePostgresChangeSource(lc, upstream.db, shard, replica.file, {
|
|
76
79
|
...initialSync,
|
|
77
80
|
replicationSlotFailover: upstream.pgReplicationSlotFailover,
|
|
78
81
|
installPartialIndexTriggers: upstream.pgPartialIndexTriggers
|
|
79
|
-
}, context, replicationLag.reportIntervalMs, restoreOptions, {
|
|
82
|
+
}, context, replicationLag.reportIntervalMs, restoreOptions, {
|
|
83
|
+
epoch,
|
|
84
|
+
slotPerReplica,
|
|
85
|
+
inactiveReplicaGracePeriodMs,
|
|
86
|
+
backupV5: litestream.backupUsingV5
|
|
87
|
+
}, purgeLock, upstream.pgStreamInboundTimeoutMs) : await initializeCustomChangeSource(lc, upstream.db, shard, replica.file, context, restoreOptions);
|
|
80
88
|
const replicationStatusPublisher = ReplicationStatusPublisher.forReplicaFile(replica.file);
|
|
81
89
|
changeStreamer = await initializeStreamer(lc, shard, taskID, address, protocol, changeDB, changeSource, replicationStatusPublisher, subscriptionState, destinationBackupURL ? {
|
|
82
90
|
backupURL: destinationBackupURL,
|
|
83
|
-
litestreamVersion: litestream.backupUsingV5 ? "v5" : "legacy"
|
|
91
|
+
litestreamVersion: litestream.backupUsingV5 ? "v5" : "legacy",
|
|
92
|
+
replicaFile: replica.file
|
|
84
93
|
} : null, purgeLock, autoReset ?? false, {
|
|
94
|
+
pgChangeLogEnabled,
|
|
85
95
|
backPressureLimitHeapProportion,
|
|
86
96
|
flowControlConsensusTimeoutProportion,
|
|
87
97
|
flowControlSlowSubscriberGracePeriodMs: flowControlSlowSubscriberGracePeriodSeconds > 0 ? flowControlSlowSubscriberGracePeriodSeconds * 1e3 : void 0,
|
|
@@ -104,7 +114,7 @@ async function runWorker(parent, env, ...argv) {
|
|
|
104
114
|
retentionMs: sqliteChangeLogRetentionMs,
|
|
105
115
|
batchRows: sqliteChangeLogPurgeBatchRows
|
|
106
116
|
} : void 0,
|
|
107
|
-
sqliteChangeLogCompare: sqliteChangeLogComparing ? {
|
|
117
|
+
sqliteChangeLogCompare: pgChangeLogEnabled && sqliteChangeLogComparing ? {
|
|
108
118
|
replicaFile: replica.file,
|
|
109
119
|
comparePercent: sqliteChangeLogComparePercent,
|
|
110
120
|
retentionMs: sqliteChangeLogRetentionMs,
|
|
@@ -133,14 +143,17 @@ async function runWorker(parent, env, ...argv) {
|
|
|
133
143
|
}
|
|
134
144
|
assert(changeStreamer, `resetting replica did not advance replicaVersion`);
|
|
135
145
|
const processes = new ProcessManager(lc, parent);
|
|
146
|
+
const profileSubWorkers = [];
|
|
136
147
|
if (backupURL) {
|
|
137
148
|
lc.info?.("setting up backup to", backupURL);
|
|
138
149
|
litestream.backupURL = backupURL;
|
|
139
150
|
const { promise: backupStarted, resolve } = resolver();
|
|
140
|
-
processes.addWorker(childWorker(REPLICATOR_URL, env, "backup"), "supporting", "backup-replicator").once("message", () => {
|
|
151
|
+
const backupReplicator = processes.addWorker(childWorker(REPLICATOR_URL, env, "backup"), "supporting", "backup-replicator").once("message", () => {
|
|
141
152
|
processes.addSubprocess(startReplicaBackupProcess(lc, litestream, replica.file), "supporting", "litestream");
|
|
142
153
|
resolve();
|
|
143
154
|
});
|
|
155
|
+
profileSubWorkers.push(backupReplicator);
|
|
156
|
+
backupReplicator.onMessageType("profileResponse", (res) => parent.send(["profileResponse", res]));
|
|
144
157
|
await backupStarted;
|
|
145
158
|
}
|
|
146
159
|
const backupMonitor = createBackupCleanupMonitor({
|
|
@@ -158,11 +171,15 @@ async function runWorker(parent, env, ...argv) {
|
|
|
158
171
|
lc.info?.(`initial backup confirmed after ${elapsed.toFixed(2)}ms`);
|
|
159
172
|
});
|
|
160
173
|
}
|
|
174
|
+
const profileWorker = profileSubWorkers.length > 0 ? broadcastWorker(profileSubWorkers) : void 0;
|
|
175
|
+
const getProfileWorker = profileWorker ? () => Promise.resolve(profileWorker) : void 0;
|
|
161
176
|
const changeStreamerWebServer = new ChangeStreamerHttpServer(lc, {
|
|
162
177
|
port,
|
|
163
178
|
keepaliveTimeoutMs,
|
|
164
|
-
startupDelayMs,
|
|
165
|
-
readinessGate
|
|
179
|
+
startupDelayMs: upstream.pgReplicationSlotPerReplica ? 0 : startupDelayMs,
|
|
180
|
+
readinessGate,
|
|
181
|
+
config,
|
|
182
|
+
getProfileWorker
|
|
166
183
|
}, parent, changeStreamer);
|
|
167
184
|
readinessGate.then(() => parent.send(["ready", { ready: true }]));
|
|
168
185
|
try {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"change-streamer.js","names":[],"sources":["../../../../../zero-cache/src/server/change-streamer.ts"],"sourcesContent":["import {consoleLogSink, LogContext} from '@rocicorp/logger';\nimport {resolver} from '@rocicorp/resolver';\nimport {assert} from '../../../shared/src/asserts.ts';\nimport {must} from '../../../shared/src/must.ts';\nimport {promiseVoid} from '../../../shared/src/resolved-promises.ts';\nimport {DatabaseInitError} from '../../../zqlite/src/db.ts';\nimport {getServerContext} from '../config/server-context.ts';\nimport {getNormalizedZeroConfig} from '../config/zero-config.ts';\nimport {deleteLiteDB} from '../db/delete-lite-db.ts';\nimport {registerSQLiteCorruptionDiagnosticTarget} from '../db/sqlite-corruption.ts';\nimport {warmupConnections} from '../db/warmup.ts';\nimport {initEventSink, publishCriticalEvent} from '../observability/events.ts';\nimport {getOrCreateGauge} from '../observability/metrics.ts';\nimport {initializeCustomChangeSource} from '../services/change-source/custom/change-source.ts';\nimport {initializePostgresChangeSource} from '../services/change-source/pg/change-source.ts';\nimport {createBackupCleanupMonitor} from '../services/change-streamer/backup-cleanup-monitor-factory.ts';\nimport {ChangeStreamerHttpServer} from '../services/change-streamer/change-streamer-http.ts';\nimport {initializeStreamer} from '../services/change-streamer/change-streamer-service.ts';\nimport type {ChangeStreamerService} from '../services/change-streamer/change-streamer.ts';\nimport {initChangeStreamerSchema} from '../services/change-streamer/schema/init.ts';\nimport {AutoResetSignal} from '../services/change-streamer/schema/tables.ts';\nimport {PurgeLocker} from '../services/change-streamer/storer.ts';\nimport {\n exitAfter,\n ProcessManager,\n runUntilKilled,\n} from '../services/life-cycle.ts';\nimport {startReplicaBackupProcess} from '../services/litestream/commands.ts';\nimport {\n changeLogFileName,\n deleteChangeLogDB,\n} from '../services/replicator/change-log-db.ts';\nimport {\n replicationStatusError,\n ReplicationStatusPublisher,\n} from '../services/replicator/replication-status.ts';\nimport {sqliteFileBytes} from '../services/replicator/sqlite-change-log-observability.ts';\nimport {connectPgClient} from '../types/pg.ts';\nimport {\n childWorker,\n parentWorker,\n singleProcessMode,\n type Worker,\n} from '../types/processes.ts';\nimport {getShardConfig} from '../types/shards.ts';\nimport type {ReplicaFileMode} from '../workers/replicator.ts';\nimport {createLogContext} from './logging.ts';\nimport {startOtelAuto} from './otel-start.ts';\nimport {REPLICATOR_URL} from './worker-urls.ts';\n\n// Default LogContext, overridden in runWorker\nlet lc = new LogContext('info', {}, consoleLogSink);\n\nexport default async function runWorker(\n parent: Worker,\n env: NodeJS.ProcessEnv,\n ...argv: string[]\n): Promise<void> {\n const config = getNormalizedZeroConfig({env, argv});\n const {\n taskID,\n changeStreamer: {\n port,\n address,\n protocol,\n startupDelayMs,\n backPressureLimitHeapProportion,\n flowControlConsensusTimeoutProportion,\n flowControlSlowSubscriberGracePeriodSeconds,\n sqliteChangeLogMode,\n sqliteChangeLogReadPercent,\n sqliteChangeLogColdReadPercent,\n sqliteChangeLogComparePercent,\n sqliteChangeLogRetentionMs,\n sqliteChangeLogReadBatchRows,\n sqliteChangeLogPurgeBatchRows,\n sqliteChangeLogBarrierTimeoutMs,\n },\n autoReset,\n replicationLag,\n litestream,\n upstream,\n change,\n replica,\n initialSync,\n keepaliveTimeoutMs,\n sqliteCorruptionChecks,\n } = config;\n\n startOtelAuto(\n createLogContext(config, 'change-streamer', 0, false),\n 'change-streamer',\n 0,\n );\n lc = createLogContext(config, 'change-streamer');\n registerSQLiteCorruptionDiagnosticTarget(\n {\n debugName: 'change-streamer replica',\n dbPath: replica.file,\n },\n sqliteCorruptionChecks,\n );\n initEventSink(lc, config);\n\n // Startup-time client used for for change-streamer initialization\n // and handoff / takeover. Steady-state clients are managed by the\n // change-streamer (Storer) implementation.\n const changeDB = await connectPgClient(\n lc,\n change.db,\n 'change-streamer-init',\n {max: 5},\n {sendStringAsJson: true},\n );\n void warmupConnections(lc, changeDB, 'change').catch(() => {});\n\n const shard = getShardConfig(config);\n\n // Ensure the change DB schema is initialized/up-to-date.\n await initChangeStreamerSchema(lc, changeDB, shard);\n\n // When restoring from litestream, acquire a lock to prevent change-log\n // purges. This ensures that (this) change-streamer will be able to resume\n // from the backup.\n let purgeLock =\n litestream.backupURL && litestream.executable\n ? await new PurgeLocker(lc, shard, changeDB).acquire()\n : null;\n const restoreOptions = {litestream, constraints: purgeLock ?? undefined};\n\n let changeStreamer: ChangeStreamerService | undefined;\n let backupURL: string | undefined;\n\n const context = getServerContext(config);\n const sqliteChangeLogEnabled = sqliteChangeLogMode !== 'off';\n // The modes are cumulative, so `serve` implies `compare`.\n const sqliteChangeLogComparing =\n sqliteChangeLogMode === 'compare' || sqliteChangeLogMode === 'serve';\n if (sqliteChangeLogEnabled) {\n const changeLogFile = changeLogFileName(replica.file);\n registerSQLiteCorruptionDiagnosticTarget(\n {\n debugName: 'change-streamer change-log',\n dbPath: changeLogFile,\n },\n sqliteCorruptionChecks,\n );\n // The log's bytes are accounted for in the process that writes them. This\n // is local disk only — the log is excluded from the litestream backup — and\n // it is a whole-database total, because a table that left the replica left\n // through its wal: the replica's footprint is what shrank and this is where\n // it went.\n getOrCreateGauge('replica', 'sqlite_change_log.file_bytes', {\n description:\n `The SQLite change log's total on-disk footprint: its main db file ` +\n `plus its wal sidecars.`,\n unit: 'By',\n }).addCallback(async o =>\n o.observe(await sqliteFileBytes(lc, changeLogFile)),\n );\n }\n\n let waitForFirstBackupBeforeServing = false;\n for (const first of [true, false]) {\n try {\n // Note: This performs initial sync of the replica if necessary.\n const {\n changeSource,\n subscriptionState,\n destinationBackupURL,\n replicaID,\n waitForBackupBeforeServing,\n } =\n upstream.type === 'pg'\n ? await initializePostgresChangeSource(\n lc,\n upstream.db,\n shard,\n replica.file,\n {\n ...initialSync,\n replicationSlotFailover: upstream.pgReplicationSlotFailover,\n installPartialIndexTriggers: upstream.pgPartialIndexTriggers,\n },\n context,\n replicationLag.reportIntervalMs,\n restoreOptions,\n {backupV5: litestream.backupUsingV5},\n purgeLock,\n upstream.pgStreamInboundTimeoutMs,\n )\n : await initializeCustomChangeSource(\n lc,\n upstream.db,\n shard,\n replica.file,\n context,\n restoreOptions,\n );\n\n const replicationStatusPublisher =\n ReplicationStatusPublisher.forReplicaFile(replica.file);\n\n changeStreamer = await initializeStreamer(\n lc,\n shard,\n taskID,\n address,\n protocol,\n changeDB,\n changeSource,\n replicationStatusPublisher,\n subscriptionState,\n destinationBackupURL\n ? {\n backupURL: destinationBackupURL,\n litestreamVersion: litestream.backupUsingV5 ? 'v5' : 'legacy',\n }\n : null,\n purgeLock,\n autoReset ?? false,\n {\n backPressureLimitHeapProportion,\n flowControlConsensusTimeoutProportion,\n flowControlSlowSubscriberGracePeriodMs:\n flowControlSlowSubscriberGracePeriodSeconds > 0\n ? flowControlSlowSubscriberGracePeriodSeconds * 1000\n : undefined,\n statementTimeoutMs: change.statementTimeoutMs,\n changeLogBatchSize: change.logBatchSize,\n sqliteCatchup: {\n changeLogFile: changeLogFileName(replica.file),\n readBatchRows: sqliteChangeLogReadBatchRows,\n barrierTimeoutMs: sqliteChangeLogBarrierTimeoutMs,\n },\n // The presence of these options is the writer's gate. This process\n // performs the restore and the initial sync, so the replica's\n // identity is in hand at the moment the log is opened.\n sqliteChangeLogWriter: sqliteChangeLogEnabled\n ? {\n replicaFile: replica.file,\n identity: {\n // RMv2 supplies the epoch; until then the generation and the\n // replica ID are the whole of the identity.\n epoch: null,\n generation: subscriptionState.replicaVersion,\n replicaID,\n },\n }\n : undefined,\n // The purge scheduler shares the writer's gate -- mode, not the\n // read path -- because `write` mode, with no read selector at all,\n // is the configuration it ships in.\n sqliteChangeLogPurge: sqliteChangeLogEnabled\n ? {\n retentionMs: sqliteChangeLogRetentionMs,\n batchRows: sqliteChangeLogPurgeBatchRows,\n }\n : undefined,\n // Compare mode runs both advisory checks. Postgres remains authoritative.\n sqliteChangeLogCompare: sqliteChangeLogComparing\n ? {\n replicaFile: replica.file,\n comparePercent: sqliteChangeLogComparePercent,\n retentionMs: sqliteChangeLogRetentionMs,\n readBatchRows: sqliteChangeLogReadBatchRows,\n }\n : undefined,\n // Slice 11 lands dark by default: serve mode constructs the stable\n // router, while readPercent=0 keeps every catchup on PG and emits\n // eligibility metrics before any canary traffic is enabled.\n sqliteChangeLogServe:\n sqliteChangeLogMode === 'serve'\n ? {\n readPercent: sqliteChangeLogReadPercent,\n coldReadPercent: sqliteChangeLogColdReadPercent,\n retentionMs: sqliteChangeLogRetentionMs,\n }\n : undefined,\n },\n setTimeout,\n );\n backupURL = destinationBackupURL;\n waitForFirstBackupBeforeServing = waitForBackupBeforeServing;\n break;\n } catch (e) {\n if (first && e instanceof AutoResetSignal) {\n lc.warn?.(`resetting replica ${replica.file}`, e);\n // TODO: Make deleteLiteDB work with litestream. It will probably have to be\n // a semantic wipe instead of a file delete.\n deleteLiteDB(replica.file);\n // The change log carries the identity of the replica it was written\n // beside, and the retry performs a fresh initial sync with a new\n // replicaVersion. Reconciliation would catch that on its own (as\n // 'identity-mismatch') but only after the writer opened a file that is\n // known here to be garbage.\n deleteChangeLogDB(replica.file);\n // Release the purge lock before retrying. This is safe because the\n // purge lock exists to preserve change-log entries so the new\n // change-streamer can resume from the backup replica's watermark.\n // An AutoResetSignal means we cant resume from the backup replica\n // (e.g. its replication slot is gone), so the change-log entries the lock\n // was protecting are no longer needed. The retry performs a fresh\n // initial sync with a new replication slot, independent of the old\n // change-log. Releasing is also necessary to avoid a\n // self-deadlock when CHANGE_DB == UPSTREAM_DB:\n // CREATE_REPLICATION_SLOT waits for all older transactions to\n // finish, including this lock's open transaction.\n await purgeLock?.release();\n purgeLock = null;\n continue; // execute again with a fresh initial-sync\n }\n if (e instanceof DatabaseInitError) {\n throw new Error(\n `Cannot open ZERO_REPLICA_FILE at \"${replica.file}\". Please check that the path is valid.`,\n {cause: e},\n );\n }\n throw e;\n }\n }\n // impossible: upstream must have advanced in order for replication to be stuck.\n assert(changeStreamer, `resetting replica did not advance replicaVersion`);\n\n const processes = new ProcessManager(lc, parent);\n if (backupURL) {\n lc.info?.('setting up backup to', backupURL);\n litestream.backupURL = backupURL;\n const {promise: backupStarted, resolve} = resolver();\n\n // Start a backup replicator and corresponding litestream backup process.\n processes\n .addWorker(\n childWorker(REPLICATOR_URL, env, 'backup' satisfies ReplicaFileMode),\n 'supporting',\n 'backup-replicator',\n )\n // Wait for the replicator's first message (i.e. \"ready\") before starting\n // litestream backup in order to avoid contending on the sqlite lock\n // when the replicator first prepares the db file.\n .once('message', () => {\n processes.addSubprocess(\n startReplicaBackupProcess(lc, litestream, replica.file),\n 'supporting',\n 'litestream',\n );\n resolve();\n });\n await backupStarted;\n }\n\n const backupMonitor = createBackupCleanupMonitor({\n lc,\n config,\n replicaFile: replica.file,\n changeStreamer,\n });\n\n let readinessGate = promiseVoid;\n if (waitForFirstBackupBeforeServing) {\n const start = performance.now();\n lc.info?.(`awaiting initial backup ...`);\n\n readinessGate = backupMonitor.firstBackupReceived().then(() => {\n const elapsed = performance.now() - start;\n lc.info?.(`initial backup confirmed after ${elapsed.toFixed(2)}ms`);\n });\n }\n\n const changeStreamerWebServer = new ChangeStreamerHttpServer(\n lc,\n {port, keepaliveTimeoutMs, startupDelayMs, readinessGate},\n parent,\n changeStreamer,\n );\n\n void readinessGate.then(() => parent.send(['ready', {ready: true}]));\n\n // Note: The changeStreamer itself is not started here; it is started by the\n // changeStreamerWebServer after a delay to ensure that routing\n // routing elements have registered the server before the\n // change-streamer takes over the replication slot.\n // TODO: Remove this delay and start the changeStreamer normally here once\n // transitioned to RMv2, since changeStreamer startup will no longer\n // disrupt an existing task.\n try {\n await runUntilKilled(lc, parent, changeStreamerWebServer, backupMonitor);\n } catch (err) {\n processes.logErrorAndExit(err, 'change-streamer');\n } finally {\n await processes.shutdown();\n }\n}\n\n// fork()\nif (!singleProcessMode()) {\n void exitAfter(\n () => lc,\n () =>\n runWorker(\n must(parentWorker),\n process.env,\n ...process.argv.slice(2),\n ).catch(async e => {\n await publishCriticalEvent(\n lc,\n replicationStatusError(lc, 'Initializing', e),\n );\n throw e;\n }),\n );\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAmDA,IAAI,KAAK,IAAI,WAAW,QAAQ,CAAC,GAAG,cAAc;AAElD,eAA8B,UAC5B,QACA,KACA,GAAG,MACY;CACf,MAAM,SAAS,wBAAwB;EAAC;EAAK;CAAI,CAAC;CAClD,MAAM,EACJ,QACA,gBAAgB,EACd,MACA,SACA,UACA,gBACA,iCACA,uCACA,6CACA,qBACA,4BACA,gCACA,+BACA,4BACA,8BACA,+BACA,mCAEF,WACA,gBACA,YACA,UACA,QACA,SACA,aACA,oBACA,2BACE;CAEJ,cACE,iBAAiB,QAAQ,mBAAmB,GAAG,KAAK,GACpD,mBACA,CACF;CACA,KAAK,iBAAiB,QAAQ,iBAAiB;CAC/C,yCACE;EACE,WAAW;EACX,QAAQ,QAAQ;CAClB,GACA,sBACF;CACA,cAAc,IAAI,MAAM;CAKxB,MAAM,WAAW,MAAM,gBACrB,IACA,OAAO,IACP,wBACA,EAAC,KAAK,EAAC,GACP,EAAC,kBAAkB,KAAI,CACzB;CACA,kBAAuB,IAAI,UAAU,QAAQ,EAAE,YAAY,CAAC,CAAC;CAE7D,MAAM,QAAQ,eAAe,MAAM;CAGnC,MAAM,yBAAyB,IAAI,UAAU,KAAK;CAKlD,IAAI,YACF,WAAW,aAAa,WAAW,aAC/B,MAAM,IAAI,YAAY,IAAI,OAAO,QAAQ,EAAE,QAAQ,IACnD;CACN,MAAM,iBAAiB;EAAC;EAAY,aAAa,aAAa,KAAA;CAAS;CAEvE,IAAI;CACJ,IAAI;CAEJ,MAAM,UAAU,iBAAiB,MAAM;CACvC,MAAM,yBAAyB,wBAAwB;CAEvD,MAAM,2BACJ,wBAAwB,aAAa,wBAAwB;CAC/D,IAAI,wBAAwB;EAC1B,MAAM,gBAAgB,kBAAkB,QAAQ,IAAI;EACpD,yCACE;GACE,WAAW;GACX,QAAQ;EACV,GACA,sBACF;EAMA,iBAAiB,WAAW,gCAAgC;GAC1D,aACE;GAEF,MAAM;EACR,CAAC,EAAE,YAAY,OAAM,MACnB,EAAE,QAAQ,MAAM,gBAAgB,IAAI,aAAa,CAAC,CACpD;CACF;CAEA,IAAI,kCAAkC;CACtC,KAAK,MAAM,SAAS,CAAC,MAAM,KAAK,GAC9B,IAAI;EAEF,MAAM,EACJ,cACA,mBACA,sBACA,WACA,+BAEA,SAAS,SAAS,OACd,MAAM,+BACJ,IACA,SAAS,IACT,OACA,QAAQ,MACR;GACE,GAAG;GACH,yBAAyB,SAAS;GAClC,6BAA6B,SAAS;EACxC,GACA,SACA,eAAe,kBACf,gBACA,EAAC,UAAU,WAAW,cAAa,GACnC,WACA,SAAS,wBACX,IACA,MAAM,6BACJ,IACA,SAAS,IACT,OACA,QAAQ,MACR,SACA,cACF;EAEN,MAAM,6BACJ,2BAA2B,eAAe,QAAQ,IAAI;EAExD,iBAAiB,MAAM,mBACrB,IACA,OACA,QACA,SACA,UACA,UACA,cACA,4BACA,mBACA,uBACI;GACE,WAAW;GACX,mBAAmB,WAAW,gBAAgB,OAAO;EACvD,IACA,MACJ,WACA,aAAa,OACb;GACE;GACA;GACA,wCACE,8CAA8C,IAC1C,8CAA8C,MAC9C,KAAA;GACN,oBAAoB,OAAO;GAC3B,oBAAoB,OAAO;GAC3B,eAAe;IACb,eAAe,kBAAkB,QAAQ,IAAI;IAC7C,eAAe;IACf,kBAAkB;GACpB;GAIA,uBAAuB,yBACnB;IACE,aAAa,QAAQ;IACrB,UAAU;KAGR,OAAO;KACP,YAAY,kBAAkB;KAC9B;IACF;GACF,IACA,KAAA;GAIJ,sBAAsB,yBAClB;IACE,aAAa;IACb,WAAW;GACb,IACA,KAAA;GAEJ,wBAAwB,2BACpB;IACE,aAAa,QAAQ;IACrB,gBAAgB;IAChB,aAAa;IACb,eAAe;GACjB,IACA,KAAA;GAIJ,sBACE,wBAAwB,UACpB;IACE,aAAa;IACb,iBAAiB;IACjB,aAAa;GACf,IACA,KAAA;EACR,GACA,UACF;EACA,YAAY;EACZ,kCAAkC;EAClC;CACF,SAAS,GAAG;EACV,IAAI,SAAS,aAAa,iBAAiB;GACzC,GAAG,OAAO,qBAAqB,QAAQ,QAAQ,CAAC;GAGhD,aAAa,QAAQ,IAAI;GAMzB,kBAAkB,QAAQ,IAAI;GAY9B,MAAM,WAAW,QAAQ;GACzB,YAAY;GACZ;EACF;EACA,IAAI,aAAa,mBACf,MAAM,IAAI,MACR,qCAAqC,QAAQ,KAAK,0CAClD,EAAC,OAAO,EAAC,CACX;EAEF,MAAM;CACR;CAGF,OAAO,gBAAgB,kDAAkD;CAEzE,MAAM,YAAY,IAAI,eAAe,IAAI,MAAM;CAC/C,IAAI,WAAW;EACb,GAAG,OAAO,wBAAwB,SAAS;EAC3C,WAAW,YAAY;EACvB,MAAM,EAAC,SAAS,eAAe,YAAW,SAAS;EAGnD,UACG,UACC,YAAY,gBAAgB,KAAK,QAAkC,GACnE,cACA,mBACF,EAIC,KAAK,iBAAiB;GACrB,UAAU,cACR,0BAA0B,IAAI,YAAY,QAAQ,IAAI,GACtD,cACA,YACF;GACA,QAAQ;EACV,CAAC;EACH,MAAM;CACR;CAEA,MAAM,gBAAgB,2BAA2B;EAC/C;EACA;EACA,aAAa,QAAQ;EACrB;CACF,CAAC;CAED,IAAI,gBAAgB;CACpB,IAAI,iCAAiC;EACnC,MAAM,QAAQ,YAAY,IAAI;EAC9B,GAAG,OAAO,6BAA6B;EAEvC,gBAAgB,cAAc,oBAAoB,EAAE,WAAW;GAC7D,MAAM,UAAU,YAAY,IAAI,IAAI;GACpC,GAAG,OAAO,kCAAkC,QAAQ,QAAQ,CAAC,EAAE,GAAG;EACpE,CAAC;CACH;CAEA,MAAM,0BAA0B,IAAI,yBAClC,IACA;EAAC;EAAM;EAAoB;EAAgB;CAAa,GACxD,QACA,cACF;CAEA,cAAmB,WAAW,OAAO,KAAK,CAAC,SAAS,EAAC,OAAO,KAAI,CAAC,CAAC,CAAC;CASnE,IAAI;EACF,MAAM,eAAe,IAAI,QAAQ,yBAAyB,aAAa;CACzE,SAAS,KAAK;EACZ,UAAU,gBAAgB,KAAK,iBAAiB;CAClD,UAAU;EACR,MAAM,UAAU,SAAS;CAC3B;AACF;AAGA,IAAI,CAAC,kBAAkB,GACrB,gBACQ,UAEJ,UACE,KAAK,YAAY,GACjB,QAAQ,KACR,GAAG,QAAQ,KAAK,MAAM,CAAC,CACzB,EAAE,MAAM,OAAM,MAAK;CACjB,MAAM,qBACJ,IACA,uBAAuB,IAAI,gBAAgB,CAAC,CAC9C;CACA,MAAM;AACR,CAAC,CACL"}
|
|
1
|
+
{"version":3,"file":"change-streamer.js","names":[],"sources":["../../../../../zero-cache/src/server/change-streamer.ts"],"sourcesContent":["import {consoleLogSink, LogContext} from '@rocicorp/logger';\nimport {resolver} from '@rocicorp/resolver';\nimport {assert} from '../../../shared/src/asserts.ts';\nimport {must} from '../../../shared/src/must.ts';\nimport {promiseVoid} from '../../../shared/src/resolved-promises.ts';\nimport {DatabaseInitError} from '../../../zqlite/src/db.ts';\nimport {getServerContext} from '../config/server-context.ts';\nimport {getNormalizedZeroConfig} from '../config/zero-config.ts';\nimport {deleteLiteDB} from '../db/delete-lite-db.ts';\nimport {registerSQLiteCorruptionDiagnosticTarget} from '../db/sqlite-corruption.ts';\nimport {warmupConnections} from '../db/warmup.ts';\nimport {initEventSink, publishCriticalEvent} from '../observability/events.ts';\nimport {getOrCreateGauge} from '../observability/metrics.ts';\nimport {initializeCustomChangeSource} from '../services/change-source/custom/change-source.ts';\nimport {initializePostgresChangeSource} from '../services/change-source/pg/change-source-init.ts';\nimport {createBackupCleanupMonitor} from '../services/change-streamer/backup-cleanup-monitor-factory.ts';\nimport {ChangeStreamerHttpServer} from '../services/change-streamer/change-streamer-http.ts';\nimport {initializeStreamer} from '../services/change-streamer/change-streamer-service.ts';\nimport type {ChangeStreamerService} from '../services/change-streamer/change-streamer.ts';\nimport {initChangeStreamerSchema} from '../services/change-streamer/schema/init.ts';\nimport {AutoResetSignal} from '../services/change-streamer/schema/tables.ts';\nimport {PurgeLocker} from '../services/change-streamer/storer.ts';\nimport {\n exitAfter,\n ProcessManager,\n runUntilKilled,\n} from '../services/life-cycle.ts';\nimport {startReplicaBackupProcess} from '../services/litestream/commands.ts';\nimport {\n changeLogFileName,\n deleteChangeLogDB,\n} from '../services/replicator/change-log-db.ts';\nimport {\n replicationStatusError,\n ReplicationStatusPublisher,\n} from '../services/replicator/replication-status.ts';\nimport {sqliteFileBytes} from '../services/replicator/sqlite-change-log-observability.ts';\nimport {connectPgClient} from '../types/pg.ts';\nimport {\n broadcastWorker,\n childWorker,\n parentWorker,\n singleProcessMode,\n type ProfileResponseMessage,\n type Worker,\n} from '../types/processes.ts';\nimport {installProfileHandler} from '../types/profiler.ts';\nimport {getShardConfig} from '../types/shards.ts';\nimport type {ReplicaFileMode} from '../workers/replicator.ts';\nimport {createLogContext} from './logging.ts';\nimport {startOtelAuto} from './otel-start.ts';\nimport {REPLICATOR_URL} from './worker-urls.ts';\n\n// Default LogContext, overridden in runWorker\nlet lc = new LogContext('info', {}, consoleLogSink);\n\nexport default async function runWorker(\n parent: Worker,\n env: NodeJS.ProcessEnv,\n ...argv: string[]\n): Promise<void> {\n installProfileHandler(parent, 'change-streamer');\n const config = getNormalizedZeroConfig({env, argv});\n const {\n taskID,\n changeStreamer: {\n port,\n address,\n protocol,\n startupDelayMs,\n backPressureLimitHeapProportion,\n flowControlConsensusTimeoutProportion,\n flowControlSlowSubscriberGracePeriodSeconds,\n pgChangeLogEnabled,\n sqliteChangeLogMode,\n sqliteChangeLogReadPercent,\n sqliteChangeLogColdReadPercent,\n sqliteChangeLogComparePercent,\n sqliteChangeLogRetentionMs,\n sqliteChangeLogReadBatchRows,\n sqliteChangeLogPurgeBatchRows,\n sqliteChangeLogBarrierTimeoutMs,\n },\n autoReset,\n replicationLag,\n litestream,\n upstream,\n change,\n replica,\n initialSync,\n keepaliveTimeoutMs,\n sqliteCorruptionChecks,\n } = config;\n\n startOtelAuto(\n createLogContext(config, 'change-streamer', 0, false),\n 'change-streamer',\n 0,\n );\n lc = createLogContext(config, 'change-streamer');\n registerSQLiteCorruptionDiagnosticTarget(\n {\n debugName: 'change-streamer replica',\n dbPath: replica.file,\n },\n sqliteCorruptionChecks,\n );\n initEventSink(lc, config);\n\n // Startup-time client used for for change-streamer initialization\n // and handoff / takeover. Steady-state clients are managed by the\n // change-streamer (Storer) implementation.\n const changeDB = await connectPgClient(\n lc,\n change.db,\n 'change-streamer-init',\n {max: 5},\n {sendStringAsJson: true},\n );\n void warmupConnections(lc, changeDB, 'change').catch(() => {});\n\n const shard = getShardConfig(config);\n\n // Ensure the change DB schema is initialized/up-to-date.\n await initChangeStreamerSchema(lc, changeDB, shard);\n\n // When restoring from litestream, acquire a lock to prevent change-log\n // purges. This ensures that (this) change-streamer will be able to resume\n // from the backup.\n let purgeLock =\n pgChangeLogEnabled && litestream.backupURL && litestream.executable\n ? await new PurgeLocker(lc, shard, changeDB).acquire()\n : null;\n const restoreOptions = {litestream, constraints: purgeLock ?? undefined};\n\n let changeStreamer: ChangeStreamerService | undefined;\n let backupURL: string | undefined;\n\n const context = getServerContext(config);\n const sqliteChangeLogEnabled = sqliteChangeLogMode !== 'off';\n // The modes are cumulative, so `serve` implies `compare`.\n const sqliteChangeLogComparing =\n sqliteChangeLogMode === 'compare' || sqliteChangeLogMode === 'serve';\n if (sqliteChangeLogEnabled) {\n const changeLogFile = changeLogFileName(replica.file);\n registerSQLiteCorruptionDiagnosticTarget(\n {\n debugName: 'change-streamer change-log',\n dbPath: changeLogFile,\n },\n sqliteCorruptionChecks,\n );\n // The log's bytes are accounted for in the process that writes them. This\n // is local disk only — the log is excluded from the litestream backup — and\n // it is a whole-database total, because a table that left the replica left\n // through its wal: the replica's footprint is what shrank and this is where\n // it went.\n getOrCreateGauge('replica', 'sqlite_change_log.file_bytes', {\n description:\n `The SQLite change log's total on-disk footprint: its main db file ` +\n `plus its wal sidecars.`,\n unit: 'By',\n }).addCallback(async o =>\n o.observe(await sqliteFileBytes(lc, changeLogFile)),\n );\n }\n\n let waitForFirstBackupBeforeServing = false;\n for (const first of [true, false]) {\n try {\n // Note: This performs initial sync of the replica if necessary.\n const {\n pgReplicationEpoch: epoch,\n pgReplicationSlotPerReplica: slotPerReplica,\n pgResumeOrphanedSlotGracePeriodMs: inactiveReplicaGracePeriodMs,\n } = upstream;\n const {\n changeSource,\n subscriptionState,\n destinationBackupURL,\n replicaID,\n waitForBackupBeforeServing,\n } =\n upstream.type === 'pg'\n ? await initializePostgresChangeSource(\n lc,\n upstream.db,\n shard,\n replica.file,\n {\n ...initialSync,\n replicationSlotFailover: upstream.pgReplicationSlotFailover,\n installPartialIndexTriggers: upstream.pgPartialIndexTriggers,\n },\n context,\n replicationLag.reportIntervalMs,\n restoreOptions,\n {\n epoch,\n slotPerReplica,\n inactiveReplicaGracePeriodMs,\n backupV5: litestream.backupUsingV5,\n },\n purgeLock,\n upstream.pgStreamInboundTimeoutMs,\n )\n : await initializeCustomChangeSource(\n lc,\n upstream.db,\n shard,\n replica.file,\n context,\n restoreOptions,\n );\n\n const replicationStatusPublisher =\n ReplicationStatusPublisher.forReplicaFile(replica.file);\n\n changeStreamer = await initializeStreamer(\n lc,\n shard,\n taskID,\n address,\n protocol,\n changeDB,\n changeSource,\n replicationStatusPublisher,\n subscriptionState,\n destinationBackupURL\n ? {\n backupURL: destinationBackupURL,\n litestreamVersion: litestream.backupUsingV5 ? 'v5' : 'legacy',\n replicaFile: replica.file,\n }\n : null,\n purgeLock,\n autoReset ?? false,\n {\n pgChangeLogEnabled,\n backPressureLimitHeapProportion,\n flowControlConsensusTimeoutProportion,\n flowControlSlowSubscriberGracePeriodMs:\n flowControlSlowSubscriberGracePeriodSeconds > 0\n ? flowControlSlowSubscriberGracePeriodSeconds * 1000\n : undefined,\n statementTimeoutMs: change.statementTimeoutMs,\n changeLogBatchSize: change.logBatchSize,\n sqliteCatchup: {\n changeLogFile: changeLogFileName(replica.file),\n readBatchRows: sqliteChangeLogReadBatchRows,\n barrierTimeoutMs: sqliteChangeLogBarrierTimeoutMs,\n },\n // The presence of these options is the writer's gate. This process\n // performs the restore and the initial sync, so the replica's\n // identity is in hand at the moment the log is opened.\n sqliteChangeLogWriter: sqliteChangeLogEnabled\n ? {\n replicaFile: replica.file,\n identity: {\n // RMv2 supplies the epoch; until then the generation and the\n // replica ID are the whole of the identity.\n epoch: null,\n generation: subscriptionState.replicaVersion,\n replicaID,\n },\n }\n : undefined,\n // The purge scheduler shares the writer's gate -- mode, not the\n // read path -- because `write` mode, with no read selector at all,\n // is the configuration it ships in.\n sqliteChangeLogPurge: sqliteChangeLogEnabled\n ? {\n retentionMs: sqliteChangeLogRetentionMs,\n batchRows: sqliteChangeLogPurgeBatchRows,\n }\n : undefined,\n // Compare mode runs both advisory checks. Postgres remains authoritative.\n sqliteChangeLogCompare:\n pgChangeLogEnabled && sqliteChangeLogComparing\n ? {\n replicaFile: replica.file,\n comparePercent: sqliteChangeLogComparePercent,\n retentionMs: sqliteChangeLogRetentionMs,\n readBatchRows: sqliteChangeLogReadBatchRows,\n }\n : undefined,\n // Slice 11 lands dark by default: serve mode constructs the stable\n // router, while readPercent=0 keeps every catchup on PG and emits\n // eligibility metrics before any canary traffic is enabled.\n sqliteChangeLogServe:\n sqliteChangeLogMode === 'serve'\n ? {\n readPercent: sqliteChangeLogReadPercent,\n coldReadPercent: sqliteChangeLogColdReadPercent,\n retentionMs: sqliteChangeLogRetentionMs,\n }\n : undefined,\n },\n setTimeout,\n );\n backupURL = destinationBackupURL;\n waitForFirstBackupBeforeServing = waitForBackupBeforeServing;\n break;\n } catch (e) {\n if (first && e instanceof AutoResetSignal) {\n lc.warn?.(`resetting replica ${replica.file}`, e);\n // TODO: Make deleteLiteDB work with litestream. It will probably have to be\n // a semantic wipe instead of a file delete.\n deleteLiteDB(replica.file);\n // The change log carries the identity of the replica it was written\n // beside, and the retry performs a fresh initial sync with a new\n // replicaVersion. Reconciliation would catch that on its own (as\n // 'identity-mismatch') but only after the writer opened a file that is\n // known here to be garbage.\n deleteChangeLogDB(replica.file);\n // Release the purge lock before retrying. This is safe because the\n // purge lock exists to preserve change-log entries so the new\n // change-streamer can resume from the backup replica's watermark.\n // An AutoResetSignal means we cant resume from the backup replica\n // (e.g. its replication slot is gone), so the change-log entries the lock\n // was protecting are no longer needed. The retry performs a fresh\n // initial sync with a new replication slot, independent of the old\n // change-log. Releasing is also necessary to avoid a\n // self-deadlock when CHANGE_DB == UPSTREAM_DB:\n // CREATE_REPLICATION_SLOT waits for all older transactions to\n // finish, including this lock's open transaction.\n await purgeLock?.release();\n purgeLock = null;\n continue; // execute again with a fresh initial-sync\n }\n if (e instanceof DatabaseInitError) {\n throw new Error(\n `Cannot open ZERO_REPLICA_FILE at \"${replica.file}\". Please check that the path is valid.`,\n {cause: e},\n );\n }\n throw e;\n }\n }\n // impossible: upstream must have advanced in order for replication to be stuck.\n assert(changeStreamer, `resetting replica did not advance replicaVersion`);\n\n const processes = new ProcessManager(lc, parent);\n const profileSubWorkers: Worker[] = [];\n if (backupURL) {\n lc.info?.('setting up backup to', backupURL);\n litestream.backupURL = backupURL;\n const {promise: backupStarted, resolve} = resolver();\n\n // Start a backup replicator and corresponding litestream backup process.\n const backupReplicator = processes\n .addWorker(\n childWorker(REPLICATOR_URL, env, 'backup' satisfies ReplicaFileMode),\n 'supporting',\n 'backup-replicator',\n )\n // Wait for the replicator's first message (i.e. \"ready\") before starting\n // litestream backup in order to avoid contending on the sqlite lock\n // when the replicator first prepares the db file.\n .once('message', () => {\n processes.addSubprocess(\n startReplicaBackupProcess(lc, litestream, replica.file),\n 'supporting',\n 'litestream',\n );\n resolve();\n });\n profileSubWorkers.push(backupReplicator);\n // Relay profileResponse messages from backup-replicator up to parent\n backupReplicator.onMessageType<ProfileResponseMessage>(\n 'profileResponse',\n res => parent.send(['profileResponse', res]),\n );\n await backupStarted;\n }\n\n const backupMonitor = createBackupCleanupMonitor({\n lc,\n config,\n replicaFile: replica.file,\n changeStreamer,\n });\n\n let readinessGate = promiseVoid;\n if (waitForFirstBackupBeforeServing) {\n const start = performance.now();\n lc.info?.(`awaiting initial backup ...`);\n\n readinessGate = backupMonitor.firstBackupReceived().then(() => {\n const elapsed = performance.now() - start;\n lc.info?.(`initial backup confirmed after ${elapsed.toFixed(2)}ms`);\n });\n }\n\n // Create the broadcast facade once: each broadcastWorker() adds permanent\n // 'message' forwarders to every sub worker, so creating one per /profz\n // request would leak a forwarder per request.\n const profileWorker =\n profileSubWorkers.length > 0\n ? broadcastWorker(profileSubWorkers)\n : undefined;\n const getProfileWorker = profileWorker\n ? () => Promise.resolve(profileWorker)\n : undefined;\n\n const changeStreamerWebServer = new ChangeStreamerHttpServer(\n lc,\n {\n port,\n keepaliveTimeoutMs,\n // The startup delay is only relevant for RMv1, and is disabled for RMv2.\n startupDelayMs: upstream.pgReplicationSlotPerReplica ? 0 : startupDelayMs,\n readinessGate,\n config,\n getProfileWorker,\n },\n parent,\n changeStreamer,\n );\n\n void readinessGate.then(() => parent.send(['ready', {ready: true}]));\n\n // Note: The changeStreamer itself is not started here; it is started by the\n // changeStreamerWebServer after a delay to ensure that routing\n // routing elements have registered the server before the\n // change-streamer takes over the replication slot.\n // TODO: Remove this delay and start the changeStreamer normally here once\n // transitioned to RMv2, since changeStreamer startup will no longer\n // disrupt an existing task.\n try {\n await runUntilKilled(lc, parent, changeStreamerWebServer, backupMonitor);\n } catch (err) {\n processes.logErrorAndExit(err, 'change-streamer');\n } finally {\n await processes.shutdown();\n }\n}\n\n// fork()\nif (!singleProcessMode()) {\n void exitAfter(\n () => lc,\n () =>\n runWorker(\n must(parentWorker),\n process.env,\n ...process.argv.slice(2),\n ).catch(async e => {\n await publishCriticalEvent(\n lc,\n replicationStatusError(lc, 'Initializing', e),\n );\n throw e;\n }),\n );\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAsDA,IAAI,KAAK,IAAI,WAAW,QAAQ,CAAC,GAAG,cAAc;AAElD,eAA8B,UAC5B,QACA,KACA,GAAG,MACY;CACf,sBAAsB,QAAQ,iBAAiB;CAC/C,MAAM,SAAS,wBAAwB;EAAC;EAAK;CAAI,CAAC;CAClD,MAAM,EACJ,QACA,gBAAgB,EACd,MACA,SACA,UACA,gBACA,iCACA,uCACA,6CACA,oBACA,qBACA,4BACA,gCACA,+BACA,4BACA,8BACA,+BACA,mCAEF,WACA,gBACA,YACA,UACA,QACA,SACA,aACA,oBACA,2BACE;CAEJ,cACE,iBAAiB,QAAQ,mBAAmB,GAAG,KAAK,GACpD,mBACA,CACF;CACA,KAAK,iBAAiB,QAAQ,iBAAiB;CAC/C,yCACE;EACE,WAAW;EACX,QAAQ,QAAQ;CAClB,GACA,sBACF;CACA,cAAc,IAAI,MAAM;CAKxB,MAAM,WAAW,MAAM,gBACrB,IACA,OAAO,IACP,wBACA,EAAC,KAAK,EAAC,GACP,EAAC,kBAAkB,KAAI,CACzB;CACA,kBAAuB,IAAI,UAAU,QAAQ,EAAE,YAAY,CAAC,CAAC;CAE7D,MAAM,QAAQ,eAAe,MAAM;CAGnC,MAAM,yBAAyB,IAAI,UAAU,KAAK;CAKlD,IAAI,YACF,sBAAsB,WAAW,aAAa,WAAW,aACrD,MAAM,IAAI,YAAY,IAAI,OAAO,QAAQ,EAAE,QAAQ,IACnD;CACN,MAAM,iBAAiB;EAAC;EAAY,aAAa,aAAa,KAAA;CAAS;CAEvE,IAAI;CACJ,IAAI;CAEJ,MAAM,UAAU,iBAAiB,MAAM;CACvC,MAAM,yBAAyB,wBAAwB;CAEvD,MAAM,2BACJ,wBAAwB,aAAa,wBAAwB;CAC/D,IAAI,wBAAwB;EAC1B,MAAM,gBAAgB,kBAAkB,QAAQ,IAAI;EACpD,yCACE;GACE,WAAW;GACX,QAAQ;EACV,GACA,sBACF;EAMA,iBAAiB,WAAW,gCAAgC;GAC1D,aACE;GAEF,MAAM;EACR,CAAC,EAAE,YAAY,OAAM,MACnB,EAAE,QAAQ,MAAM,gBAAgB,IAAI,aAAa,CAAC,CACpD;CACF;CAEA,IAAI,kCAAkC;CACtC,KAAK,MAAM,SAAS,CAAC,MAAM,KAAK,GAC9B,IAAI;EAEF,MAAM,EACJ,oBAAoB,OACpB,6BAA6B,gBAC7B,mCAAmC,iCACjC;EACJ,MAAM,EACJ,cACA,mBACA,sBACA,WACA,+BAEA,SAAS,SAAS,OACd,MAAM,+BACJ,IACA,SAAS,IACT,OACA,QAAQ,MACR;GACE,GAAG;GACH,yBAAyB,SAAS;GAClC,6BAA6B,SAAS;EACxC,GACA,SACA,eAAe,kBACf,gBACA;GACE;GACA;GACA;GACA,UAAU,WAAW;EACvB,GACA,WACA,SAAS,wBACX,IACA,MAAM,6BACJ,IACA,SAAS,IACT,OACA,QAAQ,MACR,SACA,cACF;EAEN,MAAM,6BACJ,2BAA2B,eAAe,QAAQ,IAAI;EAExD,iBAAiB,MAAM,mBACrB,IACA,OACA,QACA,SACA,UACA,UACA,cACA,4BACA,mBACA,uBACI;GACE,WAAW;GACX,mBAAmB,WAAW,gBAAgB,OAAO;GACrD,aAAa,QAAQ;EACvB,IACA,MACJ,WACA,aAAa,OACb;GACE;GACA;GACA;GACA,wCACE,8CAA8C,IAC1C,8CAA8C,MAC9C,KAAA;GACN,oBAAoB,OAAO;GAC3B,oBAAoB,OAAO;GAC3B,eAAe;IACb,eAAe,kBAAkB,QAAQ,IAAI;IAC7C,eAAe;IACf,kBAAkB;GACpB;GAIA,uBAAuB,yBACnB;IACE,aAAa,QAAQ;IACrB,UAAU;KAGR,OAAO;KACP,YAAY,kBAAkB;KAC9B;IACF;GACF,IACA,KAAA;GAIJ,sBAAsB,yBAClB;IACE,aAAa;IACb,WAAW;GACb,IACA,KAAA;GAEJ,wBACE,sBAAsB,2BAClB;IACE,aAAa,QAAQ;IACrB,gBAAgB;IAChB,aAAa;IACb,eAAe;GACjB,IACA,KAAA;GAIN,sBACE,wBAAwB,UACpB;IACE,aAAa;IACb,iBAAiB;IACjB,aAAa;GACf,IACA,KAAA;EACR,GACA,UACF;EACA,YAAY;EACZ,kCAAkC;EAClC;CACF,SAAS,GAAG;EACV,IAAI,SAAS,aAAa,iBAAiB;GACzC,GAAG,OAAO,qBAAqB,QAAQ,QAAQ,CAAC;GAGhD,aAAa,QAAQ,IAAI;GAMzB,kBAAkB,QAAQ,IAAI;GAY9B,MAAM,WAAW,QAAQ;GACzB,YAAY;GACZ;EACF;EACA,IAAI,aAAa,mBACf,MAAM,IAAI,MACR,qCAAqC,QAAQ,KAAK,0CAClD,EAAC,OAAO,EAAC,CACX;EAEF,MAAM;CACR;CAGF,OAAO,gBAAgB,kDAAkD;CAEzE,MAAM,YAAY,IAAI,eAAe,IAAI,MAAM;CAC/C,MAAM,oBAA8B,CAAC;CACrC,IAAI,WAAW;EACb,GAAG,OAAO,wBAAwB,SAAS;EAC3C,WAAW,YAAY;EACvB,MAAM,EAAC,SAAS,eAAe,YAAW,SAAS;EAGnD,MAAM,mBAAmB,UACtB,UACC,YAAY,gBAAgB,KAAK,QAAkC,GACnE,cACA,mBACF,EAIC,KAAK,iBAAiB;GACrB,UAAU,cACR,0BAA0B,IAAI,YAAY,QAAQ,IAAI,GACtD,cACA,YACF;GACA,QAAQ;EACV,CAAC;EACH,kBAAkB,KAAK,gBAAgB;EAEvC,iBAAiB,cACf,oBACA,QAAO,OAAO,KAAK,CAAC,mBAAmB,GAAG,CAAC,CAC7C;EACA,MAAM;CACR;CAEA,MAAM,gBAAgB,2BAA2B;EAC/C;EACA;EACA,aAAa,QAAQ;EACrB;CACF,CAAC;CAED,IAAI,gBAAgB;CACpB,IAAI,iCAAiC;EACnC,MAAM,QAAQ,YAAY,IAAI;EAC9B,GAAG,OAAO,6BAA6B;EAEvC,gBAAgB,cAAc,oBAAoB,EAAE,WAAW;GAC7D,MAAM,UAAU,YAAY,IAAI,IAAI;GACpC,GAAG,OAAO,kCAAkC,QAAQ,QAAQ,CAAC,EAAE,GAAG;EACpE,CAAC;CACH;CAKA,MAAM,gBACJ,kBAAkB,SAAS,IACvB,gBAAgB,iBAAiB,IACjC,KAAA;CACN,MAAM,mBAAmB,sBACf,QAAQ,QAAQ,aAAa,IACnC,KAAA;CAEJ,MAAM,0BAA0B,IAAI,yBAClC,IACA;EACE;EACA;EAEA,gBAAgB,SAAS,8BAA8B,IAAI;EAC3D;EACA;EACA;CACF,GACA,QACA,cACF;CAEA,cAAmB,WAAW,OAAO,KAAK,CAAC,SAAS,EAAC,OAAO,KAAI,CAAC,CAAC,CAAC;CASnE,IAAI;EACF,MAAM,eAAe,IAAI,QAAQ,yBAAyB,aAAa;CACzE,SAAS,KAAK;EACZ,UAAU,gBAAgB,KAAK,iBAAiB;CAClD,UAAU;EACR,MAAM,UAAU,SAAS;CAC3B;AACF;AAGA,IAAI,CAAC,kBAAkB,GACrB,gBACQ,UAEJ,UACE,KAAK,YAAY,GACjB,QAAQ,KACR,GAAG,QAAQ,KAAK,MAAM,CAAC,CACzB,EAAE,MAAM,OAAM,MAAK;CACjB,MAAM,qBACJ,IACA,uBAAuB,IAAI,gBAAgB,CAAC,CAC9C;CACA,MAAM;AACR,CAAC,CACL"}
|
|
@@ -15,6 +15,11 @@ export type ServerMetrics = {
|
|
|
15
15
|
'query-update-server': TDigest;
|
|
16
16
|
};
|
|
17
17
|
type ClientGroupID = string;
|
|
18
|
+
/**
|
|
19
|
+
* The service that established a client group's authentication, i.e. the
|
|
20
|
+
* ViewSyncer serving it. See {@link InspectorDelegate.clearAuthenticated}.
|
|
21
|
+
*/
|
|
22
|
+
type AuthOwner = object;
|
|
18
23
|
export declare class InspectorDelegate implements MetricsDelegate {
|
|
19
24
|
#private;
|
|
20
25
|
constructor(customQueryTransformer: CustomQueryTransformer | undefined);
|
|
@@ -32,8 +37,19 @@ export declare class InspectorDelegate implements MetricsDelegate {
|
|
|
32
37
|
* per "worker".
|
|
33
38
|
*/
|
|
34
39
|
isAuthenticated(clientGroupID: ClientGroupID): boolean;
|
|
35
|
-
|
|
36
|
-
|
|
40
|
+
/**
|
|
41
|
+
* Marks the client group as authenticated on behalf of `owner`, the service
|
|
42
|
+
* (ViewSyncer) that handled the authentication.
|
|
43
|
+
*/
|
|
44
|
+
setAuthenticated(clientGroupID: ClientGroupID, owner?: AuthOwner): void;
|
|
45
|
+
/**
|
|
46
|
+
* Clears the client group's authentication. When `owner` is given, the
|
|
47
|
+
* authentication is only cleared if it was established by that owner: a
|
|
48
|
+
* replacement ViewSyncer can start for the same client group while the
|
|
49
|
+
* previous one is still shutting down, and the previous one's cleanup must
|
|
50
|
+
* not revoke what the replacement has since authenticated.
|
|
51
|
+
*/
|
|
52
|
+
clearAuthenticated(clientGroupID: ClientGroupID, owner?: AuthOwner): void;
|
|
37
53
|
/**
|
|
38
54
|
* Transforms a single custom query by name and args using the configured
|
|
39
55
|
* CustomQueryTransformer. This is primarily used by the inspector to transform
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"inspector-delegate.d.ts","sourceRoot":"","sources":["../../../../../zero-cache/src/server/inspector-delegate.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAC,iBAAiB,EAAC,MAAM,6BAA6B,CAAC;
|
|
1
|
+
{"version":3,"file":"inspector-delegate.d.ts","sourceRoot":"","sources":["../../../../../zero-cache/src/server/inspector-delegate.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAC,iBAAiB,EAAC,MAAM,6BAA6B,CAAC;AAGnE,OAAO,EAAC,OAAO,EAAC,MAAM,gCAAgC,CAAC;AACvD,OAAO,KAAK,EAAC,GAAG,EAAC,MAAM,mCAAmC,CAAC;AAC3D,OAAO,KAAK,EAAC,kBAAkB,IAAI,sBAAsB,EAAC,MAAM,4CAA4C,CAAC;AAE7G,OAAO,EAEL,KAAK,SAAS,EACd,KAAK,eAAe,EACrB,MAAM,4CAA4C,CAAC;AAEpD,OAAO,KAAK,EAAC,sBAAsB,EAAC,MAAM,sCAAsC,CAAC;AACjF,OAAO,KAAK,EAAC,iBAAiB,EAAC,MAAM,uDAAuD,CAAC;AAI7F;;;;GAIG;AACH,MAAM,MAAM,aAAa,GAAG;IAC1B,8BAA8B,EAAE,OAAO,CAAC;IACxC,qBAAqB,EAAE,OAAO,CAAC;CAChC,CAAC;AAEF,KAAK,aAAa,GAAG,MAAM,CAAC;AAE5B;;;GAGG;AACH,KAAK,SAAS,GAAG,MAAM,CAAC;AAWxB,qBAAa,iBAAkB,YAAW,eAAe;;IAOvD,YAAY,sBAAsB,EAAE,sBAAsB,GAAG,SAAS,EAErE;IAED,SAAS,CAAC,CAAC,SAAS,MAAM,SAAS,EACjC,MAAM,EAAE,CAAC,EACT,KAAK,EAAE,MAAM,EACb,GAAG,IAAI,EAAE,SAAS,CAAC,CAAC,CAAC,GACpB,IAAI,CAaN;IAED,sBAAsB,CAAC,OAAO,EAAE,MAAM,GAAG,sBAAsB,GAAG,IAAI,CAUrE;IAED,cAAc;;;MAEb;IAED,cAAc,CAAC,OAAO,EAAE,MAAM,GAAG,GAAG,GAAG,SAAS,CAE/C;IAED,WAAW,CAAC,OAAO,EAAE,MAAM,GAAG,IAAI,CAIjC;IAED,QAAQ,CAAC,OAAO,EAAE,MAAM,EAAE,GAAG,EAAE,GAAG,GAAG,IAAI,CAExC;IAED;;;OAGG;IACH,eAAe,CAAC,aAAa,EAAE,aAAa,GAAG,OAAO,CAIrD;IAED;;;OAGG;IACH,gBAAgB,CACd,aAAa,EAAE,aAAa,EAC5B,KAAK,GAAE,SAAoB,GAC1B,IAAI,CAEN;IAED;;;;;;OAMG;IACH,kBAAkB,CAAC,aAAa,EAAE,aAAa,EAAE,KAAK,CAAC,EAAE,SAAS,QAOjE;IAED;;;;OAIG;IACG,oBAAoB,CACxB,IAAI,EAAE,MAAM,EACZ,IAAI,EAAE,SAAS,iBAAiB,EAAE,EAClC,GAAG,EAAE,iBAAiB,GACrB,OAAO,CAAC,GAAG,CAAC,CAsCd;CACF"}
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { assert } from "../../../shared/src/asserts.js";
|
|
2
|
+
import { getOrInsertComputed } from "../../../shared/src/map.js";
|
|
2
3
|
import { hashNameAndArgs } from "../../../zero-protocol/src/query-hash-visitor.js";
|
|
3
4
|
import { mapValues } from "../../../shared/src/objects.js";
|
|
4
5
|
import { isServerMetric } from "../../../zql/src/query/metrics-delegate.js";
|
|
@@ -7,11 +8,13 @@ import { TDigest } from "../../../shared/src/tdigest.js";
|
|
|
7
8
|
import { isDevelopmentMode } from "../config/normalize.js";
|
|
8
9
|
import { ProtocolErrorWithLevel } from "../types/error-with-level.js";
|
|
9
10
|
//#region ../zero-cache/src/server/inspector-delegate.ts
|
|
11
|
+
var NO_OWNER = {};
|
|
10
12
|
/**
|
|
11
|
-
*
|
|
12
|
-
* share this state across all
|
|
13
|
+
* Authenticated client group IDs, each mapped to the owner that authenticated
|
|
14
|
+
* it. We keep this outside of the class to share this state across all
|
|
15
|
+
* instances of the InspectorDelegate.
|
|
13
16
|
*/
|
|
14
|
-
var authenticatedClientGroupIDs = /* @__PURE__ */ new
|
|
17
|
+
var authenticatedClientGroupIDs = /* @__PURE__ */ new Map();
|
|
15
18
|
var InspectorDelegate = class {
|
|
16
19
|
#globalMetrics = newMetrics();
|
|
17
20
|
#perQueryHydrateMs = /* @__PURE__ */ new Map();
|
|
@@ -25,14 +28,7 @@ var InspectorDelegate = class {
|
|
|
25
28
|
assert(isServerMetric(metric), `Invalid server metric: ${metric}`);
|
|
26
29
|
const queryID = args[0];
|
|
27
30
|
if (metric === "query-materialization-server") this.#perQueryHydrateMs.set(queryID, value);
|
|
28
|
-
else
|
|
29
|
-
let digest = this.#perQueryUpdateMetrics.get(queryID);
|
|
30
|
-
if (!digest) {
|
|
31
|
-
digest = new TDigest();
|
|
32
|
-
this.#perQueryUpdateMetrics.set(queryID, digest);
|
|
33
|
-
}
|
|
34
|
-
digest.add(value);
|
|
35
|
-
}
|
|
31
|
+
else getOrInsertComputed(this.#perQueryUpdateMetrics, queryID, () => new TDigest()).add(value);
|
|
36
32
|
this.#globalMetrics[metric].add(value);
|
|
37
33
|
}
|
|
38
34
|
getMetricsJSONForQuery(queryID) {
|
|
@@ -65,11 +61,22 @@ var InspectorDelegate = class {
|
|
|
65
61
|
isAuthenticated(clientGroupID) {
|
|
66
62
|
return isDevelopmentMode() || authenticatedClientGroupIDs.has(clientGroupID);
|
|
67
63
|
}
|
|
68
|
-
|
|
69
|
-
|
|
64
|
+
/**
|
|
65
|
+
* Marks the client group as authenticated on behalf of `owner`, the service
|
|
66
|
+
* (ViewSyncer) that handled the authentication.
|
|
67
|
+
*/
|
|
68
|
+
setAuthenticated(clientGroupID, owner = NO_OWNER) {
|
|
69
|
+
authenticatedClientGroupIDs.set(clientGroupID, owner);
|
|
70
70
|
}
|
|
71
|
-
|
|
72
|
-
|
|
71
|
+
/**
|
|
72
|
+
* Clears the client group's authentication. When `owner` is given, the
|
|
73
|
+
* authentication is only cleared if it was established by that owner: a
|
|
74
|
+
* replacement ViewSyncer can start for the same client group while the
|
|
75
|
+
* previous one is still shutting down, and the previous one's cleanup must
|
|
76
|
+
* not revoke what the replacement has since authenticated.
|
|
77
|
+
*/
|
|
78
|
+
clearAuthenticated(clientGroupID, owner) {
|
|
79
|
+
if (owner === void 0 || authenticatedClientGroupIDs.get(clientGroupID) === owner) authenticatedClientGroupIDs.delete(clientGroupID);
|
|
73
80
|
}
|
|
74
81
|
/**
|
|
75
82
|
* Transforms a single custom query by name and args using the configured
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"inspector-delegate.js","names":["#globalMetrics","#perQueryHydrateMs","#perQueryUpdateMetrics","#queryIDToAST","#customQueryTransformer"],"sources":["../../../../../zero-cache/src/server/inspector-delegate.ts"],"sourcesContent":["import {assert} from '../../../shared/src/asserts.ts';\nimport type {ReadonlyJSONValue} from '../../../shared/src/json.ts';\nimport {mapValues} from '../../../shared/src/objects.ts';\nimport {TDigest} from '../../../shared/src/tdigest.ts';\nimport type {AST} from '../../../zero-protocol/src/ast.ts';\nimport type {QueryServerMetrics as QueryServerMetricsJSON} from '../../../zero-protocol/src/inspect-down.ts';\nimport {hashOfNameAndArgs} from '../../../zero-protocol/src/query-hash.ts';\nimport {\n isServerMetric,\n type MetricMap,\n type MetricsDelegate,\n} from '../../../zql/src/query/metrics-delegate.ts';\nimport {isDevelopmentMode} from '../config/normalize.ts';\nimport type {CustomQueryTransformer} from '../custom-queries/transform-query.ts';\nimport type {ConnectionContext} from '../services/view-syncer/connection-context-manager.ts';\nimport type {CustomQueryRecord} from '../services/view-syncer/schema/types.ts';\nimport {ProtocolErrorWithLevel} from '../types/error-with-level.ts';\n\n/**\n * Server-side metrics collected for queries during materialization and update.\n * These metrics are reported via the inspector and complement client-side metrics.\n * Used for the global aggregate (all queries combined).\n */\nexport type ServerMetrics = {\n 'query-materialization-server': TDigest;\n 'query-update-server': TDigest;\n};\n\ntype ClientGroupID = string;\n\n/**\n *
|
|
1
|
+
{"version":3,"file":"inspector-delegate.js","names":["#globalMetrics","#perQueryHydrateMs","#perQueryUpdateMetrics","#queryIDToAST","#customQueryTransformer"],"sources":["../../../../../zero-cache/src/server/inspector-delegate.ts"],"sourcesContent":["import {assert} from '../../../shared/src/asserts.ts';\nimport type {ReadonlyJSONValue} from '../../../shared/src/json.ts';\nimport {getOrInsertComputed} from '../../../shared/src/map.ts';\nimport {mapValues} from '../../../shared/src/objects.ts';\nimport {TDigest} from '../../../shared/src/tdigest.ts';\nimport type {AST} from '../../../zero-protocol/src/ast.ts';\nimport type {QueryServerMetrics as QueryServerMetricsJSON} from '../../../zero-protocol/src/inspect-down.ts';\nimport {hashOfNameAndArgs} from '../../../zero-protocol/src/query-hash.ts';\nimport {\n isServerMetric,\n type MetricMap,\n type MetricsDelegate,\n} from '../../../zql/src/query/metrics-delegate.ts';\nimport {isDevelopmentMode} from '../config/normalize.ts';\nimport type {CustomQueryTransformer} from '../custom-queries/transform-query.ts';\nimport type {ConnectionContext} from '../services/view-syncer/connection-context-manager.ts';\nimport type {CustomQueryRecord} from '../services/view-syncer/schema/types.ts';\nimport {ProtocolErrorWithLevel} from '../types/error-with-level.ts';\n\n/**\n * Server-side metrics collected for queries during materialization and update.\n * These metrics are reported via the inspector and complement client-side metrics.\n * Used for the global aggregate (all queries combined).\n */\nexport type ServerMetrics = {\n 'query-materialization-server': TDigest;\n 'query-update-server': TDigest;\n};\n\ntype ClientGroupID = string;\n\n/**\n * The service that established a client group's authentication, i.e. the\n * ViewSyncer serving it. See {@link InspectorDelegate.clearAuthenticated}.\n */\ntype AuthOwner = object;\n\nconst NO_OWNER: AuthOwner = {};\n\n/**\n * Authenticated client group IDs, each mapped to the owner that authenticated\n * it. We keep this outside of the class to share this state across all\n * instances of the InspectorDelegate.\n */\nconst authenticatedClientGroupIDs = new Map<ClientGroupID, AuthOwner>();\n\nexport class InspectorDelegate implements MetricsDelegate {\n readonly #globalMetrics: ServerMetrics = newMetrics();\n readonly #perQueryHydrateMs = new Map<string, number>();\n readonly #perQueryUpdateMetrics = new Map<string, TDigest>();\n readonly #queryIDToAST: Map<string, AST> = new Map();\n readonly #customQueryTransformer: CustomQueryTransformer | undefined;\n\n constructor(customQueryTransformer: CustomQueryTransformer | undefined) {\n this.#customQueryTransformer = customQueryTransformer;\n }\n\n addMetric<K extends keyof MetricMap>(\n metric: K,\n value: number,\n ...args: MetricMap[K]\n ): void {\n assert(isServerMetric(metric), `Invalid server metric: ${metric}`);\n const queryID = args[0];\n if (metric === 'query-materialization-server') {\n this.#perQueryHydrateMs.set(queryID, value);\n } else {\n getOrInsertComputed(\n this.#perQueryUpdateMetrics,\n queryID,\n () => new TDigest(),\n ).add(value);\n }\n this.#globalMetrics[metric].add(value);\n }\n\n getMetricsJSONForQuery(queryID: string): QueryServerMetricsJSON | null {\n const hydrateMs = this.#perQueryHydrateMs.get(queryID);\n const updateMetrics = this.#perQueryUpdateMetrics.get(queryID);\n if (hydrateMs === undefined && updateMetrics === undefined) {\n return null;\n }\n return {\n 'query-hydration-server-ms': hydrateMs,\n 'query-update-server': (updateMetrics ?? new TDigest()).toJSON(),\n };\n }\n\n getMetricsJSON() {\n return mapValues(this.#globalMetrics, v => v.toJSON());\n }\n\n getASTForQuery(queryID: string): AST | undefined {\n return this.#queryIDToAST.get(queryID);\n }\n\n removeQuery(queryID: string): void {\n this.#perQueryHydrateMs.delete(queryID);\n this.#perQueryUpdateMetrics.delete(queryID);\n this.#queryIDToAST.delete(queryID);\n }\n\n addQuery(queryID: string, ast: AST): void {\n this.#queryIDToAST.set(queryID, ast);\n }\n\n /**\n * Check if the client is authenticated. We only require authentication once\n * per \"worker\".\n */\n isAuthenticated(clientGroupID: ClientGroupID): boolean {\n return (\n isDevelopmentMode() || authenticatedClientGroupIDs.has(clientGroupID)\n );\n }\n\n /**\n * Marks the client group as authenticated on behalf of `owner`, the service\n * (ViewSyncer) that handled the authentication.\n */\n setAuthenticated(\n clientGroupID: ClientGroupID,\n owner: AuthOwner = NO_OWNER,\n ): void {\n authenticatedClientGroupIDs.set(clientGroupID, owner);\n }\n\n /**\n * Clears the client group's authentication. When `owner` is given, the\n * authentication is only cleared if it was established by that owner: a\n * replacement ViewSyncer can start for the same client group while the\n * previous one is still shutting down, and the previous one's cleanup must\n * not revoke what the replacement has since authenticated.\n */\n clearAuthenticated(clientGroupID: ClientGroupID, owner?: AuthOwner) {\n if (\n owner === undefined ||\n authenticatedClientGroupIDs.get(clientGroupID) === owner\n ) {\n authenticatedClientGroupIDs.delete(clientGroupID);\n }\n }\n\n /**\n * Transforms a single custom query by name and args using the configured\n * CustomQueryTransformer. This is primarily used by the inspector to transform\n * queries for analysis.\n */\n async transformCustomQuery(\n name: string,\n args: readonly ReadonlyJSONValue[],\n ctx: ConnectionContext,\n ): Promise<AST> {\n assert(\n this.#customQueryTransformer,\n 'Custom query transformation requested but no CustomQueryTransformer is configured',\n );\n\n // Create a fake CustomQueryRecord for the single query\n const queryID = hashOfNameAndArgs(name, args);\n const queries: CustomQueryRecord[] = [\n {\n id: queryID,\n type: 'custom',\n name,\n args,\n clientState: {},\n },\n ];\n\n const results = await this.#customQueryTransformer.transform(ctx, queries);\n\n if ('kind' in results.result) {\n throw new ProtocolErrorWithLevel(results.result, 'warn');\n }\n\n const result = results.result[0];\n if (!result) {\n throw new Error('No transformation result returned');\n }\n\n if ('error' in result) {\n const message =\n result.message ?? 'Unknown application error from custom query';\n throw new Error(\n `Error transforming custom query ${name} (${result.error}): ${message} ${JSON.stringify(result.details)}`,\n );\n }\n\n return result.transformedAst;\n }\n}\n\nfunction newMetrics(): ServerMetrics {\n return {\n 'query-materialization-server': new TDigest(),\n 'query-update-server': new TDigest(),\n };\n}\n"],"mappings":";;;;;;;;;;AAqCA,IAAM,WAAsB,CAAC;;;;;;AAO7B,IAAM,8CAA8B,IAAI,IAA8B;AAEtE,IAAa,oBAAb,MAA0D;CACxD,iBAAyC,WAAW;CACpD,qCAA8B,IAAI,IAAoB;CACtD,yCAAkC,IAAI,IAAqB;CAC3D,gCAA2C,IAAI,IAAI;CACnD;CAEA,YAAY,wBAA4D;EACtE,KAAKI,0BAA0B;CACjC;CAEA,UACE,QACA,OACA,GAAG,MACG;EACN,OAAO,eAAe,MAAM,GAAG,0BAA0B,QAAQ;EACjE,MAAM,UAAU,KAAK;EACrB,IAAI,WAAW,gCACb,KAAKH,mBAAmB,IAAI,SAAS,KAAK;OAE1C,oBACE,KAAKC,wBACL,eACM,IAAI,QAAQ,CACpB,EAAE,IAAI,KAAK;EAEb,KAAKF,eAAe,QAAQ,IAAI,KAAK;CACvC;CAEA,uBAAuB,SAAgD;EACrE,MAAM,YAAY,KAAKC,mBAAmB,IAAI,OAAO;EACrD,MAAM,gBAAgB,KAAKC,uBAAuB,IAAI,OAAO;EAC7D,IAAI,cAAc,KAAA,KAAa,kBAAkB,KAAA,GAC/C,OAAO;EAET,OAAO;GACL,6BAA6B;GAC7B,wBAAwB,iBAAiB,IAAI,QAAQ,GAAG,OAAO;EACjE;CACF;CAEA,iBAAiB;EACf,OAAO,UAAU,KAAKF,iBAAgB,MAAK,EAAE,OAAO,CAAC;CACvD;CAEA,eAAe,SAAkC;EAC/C,OAAO,KAAKG,cAAc,IAAI,OAAO;CACvC;CAEA,YAAY,SAAuB;EACjC,KAAKF,mBAAmB,OAAO,OAAO;EACtC,KAAKC,uBAAuB,OAAO,OAAO;EAC1C,KAAKC,cAAc,OAAO,OAAO;CACnC;CAEA,SAAS,SAAiB,KAAgB;EACxC,KAAKA,cAAc,IAAI,SAAS,GAAG;CACrC;;;;;CAMA,gBAAgB,eAAuC;EACrD,OACE,kBAAkB,KAAK,4BAA4B,IAAI,aAAa;CAExE;;;;;CAMA,iBACE,eACA,QAAmB,UACb;EACN,4BAA4B,IAAI,eAAe,KAAK;CACtD;;;;;;;;CASA,mBAAmB,eAA8B,OAAmB;EAClE,IACE,UAAU,KAAA,KACV,4BAA4B,IAAI,aAAa,MAAM,OAEnD,4BAA4B,OAAO,aAAa;CAEpD;;;;;;CAOA,MAAM,qBACJ,MACA,MACA,KACc;EACd,OACE,KAAKC,yBACL,mFACF;EAIA,MAAM,UAA+B,CACnC;GACE,IAHY,gBAAkB,MAAM,IAGhC;GACJ,MAAM;GACN;GACA;GACA,aAAa,CAAC;EAChB,CACF;EAEA,MAAM,UAAU,MAAM,KAAKA,wBAAwB,UAAU,KAAK,OAAO;EAEzE,IAAI,UAAU,QAAQ,QACpB,MAAM,IAAI,uBAAuB,QAAQ,QAAQ,MAAM;EAGzD,MAAM,SAAS,QAAQ,OAAO;EAC9B,IAAI,CAAC,QACH,MAAM,IAAI,MAAM,mCAAmC;EAGrD,IAAI,WAAW,QAAQ;GACrB,MAAM,UACJ,OAAO,WAAW;GACpB,MAAM,IAAI,MACR,mCAAmC,KAAK,IAAI,OAAO,MAAM,KAAK,QAAQ,GAAG,KAAK,UAAU,OAAO,OAAO,GACxG;EACF;EAEA,OAAO,OAAO;CAChB;AACF;AAEA,SAAS,aAA4B;CACnC,OAAO;EACL,gCAAgC,IAAI,QAAQ;EAC5C,uBAAuB,IAAI,QAAQ;CACrC;AACF"}
|