@rocicorp/zero 1.11.0-canary.0 → 1.11.0-canary.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/out/analyze-query/src/bin-analyze.js +1 -1
- package/out/analyze-query/src/bin-analyze.js.map +1 -1
- package/out/replicache/src/btree/read.d.ts +12 -1
- package/out/replicache/src/btree/read.d.ts.map +1 -1
- package/out/replicache/src/btree/read.js +24 -12
- package/out/replicache/src/btree/read.js.map +1 -1
- package/out/replicache/src/dag/gc.d.ts.map +1 -1
- package/out/replicache/src/dag/gc.js +2 -1
- package/out/replicache/src/dag/gc.js.map +1 -1
- package/out/replicache/src/dag/invalid-ref-count-error.d.ts +12 -0
- package/out/replicache/src/dag/invalid-ref-count-error.d.ts.map +1 -0
- package/out/replicache/src/dag/invalid-ref-count-error.js +19 -0
- package/out/replicache/src/dag/invalid-ref-count-error.js.map +1 -0
- package/out/replicache/src/dag/store-impl.d.ts +12 -2
- package/out/replicache/src/dag/store-impl.d.ts.map +1 -1
- package/out/replicache/src/dag/store-impl.js +22 -8
- package/out/replicache/src/dag/store-impl.js.map +1 -1
- package/out/replicache/src/db/rebase.d.ts +14 -2
- package/out/replicache/src/db/rebase.d.ts.map +1 -1
- package/out/replicache/src/db/rebase.js +27 -5
- package/out/replicache/src/db/rebase.js.map +1 -1
- package/out/replicache/src/db/write.js.map +1 -1
- package/out/replicache/src/kv/expo-sqlite/store.d.ts.map +1 -1
- package/out/replicache/src/kv/expo-sqlite/store.js +28 -5
- package/out/replicache/src/kv/expo-sqlite/store.js.map +1 -1
- package/out/replicache/src/kv/idb-store-with-mem-fallback.d.ts +15 -5
- package/out/replicache/src/kv/idb-store-with-mem-fallback.d.ts.map +1 -1
- package/out/replicache/src/kv/idb-store-with-mem-fallback.js +25 -25
- package/out/replicache/src/kv/idb-store-with-mem-fallback.js.map +1 -1
- package/out/replicache/src/kv/idb-store.d.ts +9 -0
- package/out/replicache/src/kv/idb-store.d.ts.map +1 -1
- package/out/replicache/src/kv/idb-store.js +14 -2
- package/out/replicache/src/kv/idb-store.js.map +1 -1
- package/out/replicache/src/kv/mem-store.d.ts +1 -0
- package/out/replicache/src/kv/mem-store.d.ts.map +1 -1
- package/out/replicache/src/kv/mem-store.js +3 -0
- package/out/replicache/src/kv/mem-store.js.map +1 -1
- package/out/replicache/src/kv/op-sqlite/store.d.ts.map +1 -1
- package/out/replicache/src/kv/op-sqlite/store.js +1 -1
- package/out/replicache/src/kv/op-sqlite/store.js.map +1 -1
- package/out/replicache/src/kv/sqlite-store.d.ts +13 -1
- package/out/replicache/src/kv/sqlite-store.d.ts.map +1 -1
- package/out/replicache/src/kv/sqlite-store.js +49 -4
- package/out/replicache/src/kv/sqlite-store.js.map +1 -1
- package/out/replicache/src/kv/store.d.ts +6 -0
- package/out/replicache/src/kv/store.d.ts.map +1 -1
- package/out/replicache/src/persist/collect-idb-databases.d.ts +10 -5
- package/out/replicache/src/persist/collect-idb-databases.d.ts.map +1 -1
- package/out/replicache/src/persist/collect-idb-databases.js +16 -9
- package/out/replicache/src/persist/collect-idb-databases.js.map +1 -1
- package/out/replicache/src/persist/persist.d.ts.map +1 -1
- package/out/replicache/src/persist/persist.js +12 -5
- package/out/replicache/src/persist/persist.js.map +1 -1
- package/out/replicache/src/persist/refresh.js +6 -2
- package/out/replicache/src/persist/refresh.js.map +1 -1
- package/out/replicache/src/replicache-impl.d.ts +14 -3
- package/out/replicache/src/replicache-impl.d.ts.map +1 -1
- package/out/replicache/src/replicache-impl.js +118 -11
- package/out/replicache/src/replicache-impl.js.map +1 -1
- package/out/replicache/src/replicache-options.d.ts +9 -0
- package/out/replicache/src/replicache-options.d.ts.map +1 -1
- package/out/replicache/src/transactions.d.ts.map +1 -1
- package/out/replicache/src/transactions.js +1 -1
- package/out/replicache/src/transactions.js.map +1 -1
- package/out/shared/src/error.d.ts +11 -0
- package/out/shared/src/error.d.ts.map +1 -1
- package/out/shared/src/error.js +35 -1
- package/out/shared/src/error.js.map +1 -1
- package/out/shared/src/iterables.d.ts +2 -0
- package/out/shared/src/iterables.d.ts.map +1 -1
- package/out/shared/src/iterables.js +18 -1
- package/out/shared/src/iterables.js.map +1 -1
- package/out/shared/src/map.d.ts +7 -2
- package/out/shared/src/map.d.ts.map +1 -1
- package/out/shared/src/map.js.map +1 -1
- package/out/shared/src/ring-buffer.d.ts +8 -0
- package/out/shared/src/ring-buffer.d.ts.map +1 -1
- package/out/shared/src/ring-buffer.js +22 -1
- package/out/shared/src/ring-buffer.js.map +1 -1
- package/out/zero/package.js +3 -4
- package/out/zero/package.js.map +1 -1
- package/out/zero-cache/src/config/normalize.d.ts.map +1 -1
- package/out/zero-cache/src/config/normalize.js +25 -1
- package/out/zero-cache/src/config/normalize.js.map +1 -1
- package/out/zero-cache/src/config/zero-config.d.ts +48 -2
- package/out/zero-cache/src/config/zero-config.d.ts.map +1 -1
- package/out/zero-cache/src/config/zero-config.js +171 -29
- package/out/zero-cache/src/config/zero-config.js.map +1 -1
- package/out/zero-cache/src/server/change-streamer.d.ts.map +1 -1
- package/out/zero-cache/src/server/change-streamer.js +25 -9
- package/out/zero-cache/src/server/change-streamer.js.map +1 -1
- package/out/zero-cache/src/server/inspector-delegate.d.ts +18 -2
- package/out/zero-cache/src/server/inspector-delegate.d.ts.map +1 -1
- package/out/zero-cache/src/server/inspector-delegate.js +20 -7
- package/out/zero-cache/src/server/inspector-delegate.js.map +1 -1
- package/out/zero-cache/src/server/main.d.ts.map +1 -1
- package/out/zero-cache/src/server/main.js +8 -1
- package/out/zero-cache/src/server/main.js.map +1 -1
- package/out/zero-cache/src/server/mutator.js +2 -0
- package/out/zero-cache/src/server/mutator.js.map +1 -1
- package/out/zero-cache/src/server/reaper.d.ts.map +1 -1
- package/out/zero-cache/src/server/reaper.js +2 -0
- package/out/zero-cache/src/server/reaper.js.map +1 -1
- package/out/zero-cache/src/server/replicator.d.ts.map +1 -1
- package/out/zero-cache/src/server/replicator.js +8 -2
- package/out/zero-cache/src/server/replicator.js.map +1 -1
- package/out/zero-cache/src/server/runner/run-worker.js +1 -1
- package/out/zero-cache/src/server/runner/zero-dispatcher.d.ts.map +1 -1
- package/out/zero-cache/src/server/runner/zero-dispatcher.js +3 -0
- package/out/zero-cache/src/server/runner/zero-dispatcher.js.map +1 -1
- package/out/zero-cache/src/server/syncer.d.ts.map +1 -1
- package/out/zero-cache/src/server/syncer.js +2 -0
- package/out/zero-cache/src/server/syncer.js.map +1 -1
- package/out/zero-cache/src/services/analyze.d.ts.map +1 -1
- package/out/zero-cache/src/services/analyze.js +3 -2
- package/out/zero-cache/src/services/analyze.js.map +1 -1
- package/out/zero-cache/src/services/change-source/common/backfill-manager.d.ts.map +1 -1
- package/out/zero-cache/src/services/change-source/common/backfill-manager.js +14 -1
- package/out/zero-cache/src/services/change-source/common/backfill-manager.js.map +1 -1
- package/out/zero-cache/src/services/change-source/common/change-stream-multiplexer.d.ts.map +1 -1
- package/out/zero-cache/src/services/change-source/common/change-stream-multiplexer.js +15 -3
- package/out/zero-cache/src/services/change-source/common/change-stream-multiplexer.js.map +1 -1
- package/out/zero-cache/src/services/change-source/common/replica-schema.d.ts.map +1 -1
- package/out/zero-cache/src/services/change-source/common/replica-schema.js +2 -39
- package/out/zero-cache/src/services/change-source/common/replica-schema.js.map +1 -1
- package/out/zero-cache/src/services/change-source/pg/backfill-resume.d.ts +156 -0
- package/out/zero-cache/src/services/change-source/pg/backfill-resume.d.ts.map +1 -0
- package/out/zero-cache/src/services/change-source/pg/backfill-resume.js +15 -0
- package/out/zero-cache/src/services/change-source/pg/backfill-resume.js.map +1 -0
- package/out/zero-cache/src/services/change-source/pg/change-source-init.d.ts +25 -0
- package/out/zero-cache/src/services/change-source/pg/change-source-init.d.ts.map +1 -0
- package/out/zero-cache/src/services/change-source/pg/change-source-init.js +202 -0
- package/out/zero-cache/src/services/change-source/pg/change-source-init.js.map +1 -0
- package/out/zero-cache/src/services/change-source/pg/change-source.d.ts +14 -16
- package/out/zero-cache/src/services/change-source/pg/change-source.d.ts.map +1 -1
- package/out/zero-cache/src/services/change-source/pg/change-source.js +64 -143
- package/out/zero-cache/src/services/change-source/pg/change-source.js.map +1 -1
- package/out/zero-cache/src/services/change-source/pg/initial-sync.d.ts +28 -2
- package/out/zero-cache/src/services/change-source/pg/initial-sync.d.ts.map +1 -1
- package/out/zero-cache/src/services/change-source/pg/initial-sync.js +44 -21
- package/out/zero-cache/src/services/change-source/pg/initial-sync.js.map +1 -1
- package/out/zero-cache/src/services/change-source/pg/replication-slot-cleanup-monitor.d.ts +16 -0
- package/out/zero-cache/src/services/change-source/pg/replication-slot-cleanup-monitor.d.ts.map +1 -0
- package/out/zero-cache/src/services/change-source/pg/replication-slot-cleanup-monitor.js +113 -0
- package/out/zero-cache/src/services/change-source/pg/replication-slot-cleanup-monitor.js.map +1 -0
- package/out/zero-cache/src/services/change-source/pg/replication-slots.d.ts +21 -15
- package/out/zero-cache/src/services/change-source/pg/replication-slots.d.ts.map +1 -1
- package/out/zero-cache/src/services/change-source/pg/replication-slots.js +86 -23
- package/out/zero-cache/src/services/change-source/pg/replication-slots.js.map +1 -1
- package/out/zero-cache/src/services/change-source/pg/schema/index-predicate.d.ts.map +1 -1
- package/out/zero-cache/src/services/change-source/pg/schema/index-predicate.js +68 -8
- package/out/zero-cache/src/services/change-source/pg/schema/index-predicate.js.map +1 -1
- package/out/zero-cache/src/services/change-source/pg/schema/init.d.ts.map +1 -1
- package/out/zero-cache/src/services/change-source/pg/schema/init.js +32 -1
- package/out/zero-cache/src/services/change-source/pg/schema/init.js.map +1 -1
- package/out/zero-cache/src/services/change-source/pg/schema/replica-stage-enum.d.ts +10 -0
- package/out/zero-cache/src/services/change-source/pg/schema/replica-stage-enum.d.ts.map +1 -0
- package/out/zero-cache/src/services/change-source/pg/schema/replica-stage-enum.js +6 -0
- package/out/zero-cache/src/services/change-source/pg/schema/replica-stage-enum.js.map +1 -0
- package/out/zero-cache/src/services/change-source/pg/schema/shard.d.ts +50 -27
- package/out/zero-cache/src/services/change-source/pg/schema/shard.d.ts.map +1 -1
- package/out/zero-cache/src/services/change-source/pg/schema/shard.js +100 -34
- package/out/zero-cache/src/services/change-source/pg/schema/shard.js.map +1 -1
- package/out/zero-cache/src/services/change-streamer/backup-monitor.d.ts.map +1 -1
- package/out/zero-cache/src/services/change-streamer/backup-monitor.js +58 -7
- package/out/zero-cache/src/services/change-streamer/backup-monitor.js.map +1 -1
- package/out/zero-cache/src/services/change-streamer/broadcast.d.ts +8 -2
- package/out/zero-cache/src/services/change-streamer/broadcast.d.ts.map +1 -1
- package/out/zero-cache/src/services/change-streamer/broadcast.js +24 -4
- package/out/zero-cache/src/services/change-streamer/broadcast.js.map +1 -1
- package/out/zero-cache/src/services/change-streamer/change-log-codec.d.ts +19 -4
- package/out/zero-cache/src/services/change-streamer/change-log-codec.d.ts.map +1 -1
- package/out/zero-cache/src/services/change-streamer/change-log-codec.js +28 -8
- package/out/zero-cache/src/services/change-streamer/change-log-codec.js.map +1 -1
- package/out/zero-cache/src/services/change-streamer/change-streamer-http.d.ts +5 -2
- package/out/zero-cache/src/services/change-streamer/change-streamer-http.d.ts.map +1 -1
- package/out/zero-cache/src/services/change-streamer/change-streamer-http.js +21 -14
- package/out/zero-cache/src/services/change-streamer/change-streamer-http.js.map +1 -1
- package/out/zero-cache/src/services/change-streamer/change-streamer-service.d.ts +6 -0
- package/out/zero-cache/src/services/change-streamer/change-streamer-service.d.ts.map +1 -1
- package/out/zero-cache/src/services/change-streamer/change-streamer-service.js +97 -63
- package/out/zero-cache/src/services/change-streamer/change-streamer-service.js.map +1 -1
- package/out/zero-cache/src/services/change-streamer/change-streamer.d.ts +12 -8
- package/out/zero-cache/src/services/change-streamer/change-streamer.d.ts.map +1 -1
- package/out/zero-cache/src/services/change-streamer/change-streamer.js.map +1 -1
- package/out/zero-cache/src/services/change-streamer/forwarder.d.ts +3 -3
- package/out/zero-cache/src/services/change-streamer/forwarder.d.ts.map +1 -1
- package/out/zero-cache/src/services/change-streamer/forwarder.js +36 -5
- package/out/zero-cache/src/services/change-streamer/forwarder.js.map +1 -1
- package/out/zero-cache/src/services/change-streamer/schema/tables.d.ts +1 -1
- package/out/zero-cache/src/services/change-streamer/schema/tables.d.ts.map +1 -1
- package/out/zero-cache/src/services/change-streamer/schema/tables.js +4 -4
- package/out/zero-cache/src/services/change-streamer/schema/tables.js.map +1 -1
- package/out/zero-cache/src/services/change-streamer/snapshot.d.ts +3 -1
- package/out/zero-cache/src/services/change-streamer/snapshot.d.ts.map +1 -1
- package/out/zero-cache/src/services/change-streamer/snapshot.js +54 -20
- package/out/zero-cache/src/services/change-streamer/snapshot.js.map +1 -1
- package/out/zero-cache/src/services/change-streamer/sqlite-change-log-writer.d.ts +6 -1
- package/out/zero-cache/src/services/change-streamer/sqlite-change-log-writer.d.ts.map +1 -1
- package/out/zero-cache/src/services/change-streamer/sqlite-change-log-writer.js +27 -9
- package/out/zero-cache/src/services/change-streamer/sqlite-change-log-writer.js.map +1 -1
- package/out/zero-cache/src/services/change-streamer/storer.d.ts +2 -2
- package/out/zero-cache/src/services/change-streamer/storer.d.ts.map +1 -1
- package/out/zero-cache/src/services/change-streamer/storer.js +8 -5
- package/out/zero-cache/src/services/change-streamer/storer.js.map +1 -1
- package/out/zero-cache/src/services/change-streamer/subscriber.d.ts +4 -1
- package/out/zero-cache/src/services/change-streamer/subscriber.d.ts.map +1 -1
- package/out/zero-cache/src/services/change-streamer/subscriber.js +42 -0
- package/out/zero-cache/src/services/change-streamer/subscriber.js.map +1 -1
- package/out/zero-cache/src/services/change-streamer/upstream-acker.d.ts +17 -1
- package/out/zero-cache/src/services/change-streamer/upstream-acker.d.ts.map +1 -1
- package/out/zero-cache/src/services/change-streamer/upstream-acker.js +19 -3
- package/out/zero-cache/src/services/change-streamer/upstream-acker.js.map +1 -1
- package/out/zero-cache/src/services/change-streamer/vfs-watermark-poller.d.ts.map +1 -1
- package/out/zero-cache/src/services/change-streamer/vfs-watermark-poller.js +1 -0
- package/out/zero-cache/src/services/change-streamer/vfs-watermark-poller.js.map +1 -1
- package/out/zero-cache/src/services/life-cycle.d.ts.map +1 -1
- package/out/zero-cache/src/services/life-cycle.js +3 -1
- package/out/zero-cache/src/services/life-cycle.js.map +1 -1
- package/out/zero-cache/src/services/litestream/commands.d.ts.map +1 -1
- package/out/zero-cache/src/services/litestream/commands.js +8 -3
- package/out/zero-cache/src/services/litestream/commands.js.map +1 -1
- package/out/zero-cache/src/services/litestream/config-v5.yml +15 -1
- package/out/zero-cache/src/services/litestream/litestream-checkpointer.js +131 -0
- package/out/zero-cache/src/services/litestream/litestream-checkpointer.js.map +1 -0
- package/out/zero-cache/src/services/litestream/litestream-controller.d.ts +70 -0
- package/out/zero-cache/src/services/litestream/litestream-controller.d.ts.map +1 -0
- package/out/zero-cache/src/services/litestream/litestream-controller.js +101 -0
- package/out/zero-cache/src/services/litestream/litestream-controller.js.map +1 -0
- package/out/zero-cache/src/services/mutagen/mutagen.d.ts +5 -0
- package/out/zero-cache/src/services/mutagen/mutagen.d.ts.map +1 -1
- package/out/zero-cache/src/services/mutagen/mutagen.js +7 -1
- package/out/zero-cache/src/services/mutagen/mutagen.js.map +1 -1
- package/out/zero-cache/src/services/mutagen/pusher.js +2 -1
- package/out/zero-cache/src/services/mutagen/pusher.js.map +1 -1
- package/out/zero-cache/src/services/profz.d.ts +7 -0
- package/out/zero-cache/src/services/profz.d.ts.map +1 -0
- package/out/zero-cache/src/services/profz.js +105 -0
- package/out/zero-cache/src/services/profz.js.map +1 -0
- package/out/zero-cache/src/services/replicator/change-log-db.js.map +1 -1
- package/out/zero-cache/src/services/replicator/change-log-stream-writer.d.ts +5 -4
- package/out/zero-cache/src/services/replicator/change-log-stream-writer.d.ts.map +1 -1
- package/out/zero-cache/src/services/replicator/change-log-stream-writer.js +20 -22
- package/out/zero-cache/src/services/replicator/change-log-stream-writer.js.map +1 -1
- package/out/zero-cache/src/services/replicator/change-processor.js +1 -0
- package/out/zero-cache/src/services/replicator/change-processor.js.map +1 -1
- package/out/zero-cache/src/services/replicator/incremental-sync.d.ts +1 -1
- package/out/zero-cache/src/services/replicator/incremental-sync.d.ts.map +1 -1
- package/out/zero-cache/src/services/replicator/incremental-sync.js +58 -17
- package/out/zero-cache/src/services/replicator/incremental-sync.js.map +1 -1
- package/out/zero-cache/src/services/replicator/replication-status.d.ts +35 -1
- package/out/zero-cache/src/services/replicator/replication-status.d.ts.map +1 -1
- package/out/zero-cache/src/services/replicator/replication-status.js +96 -6
- package/out/zero-cache/src/services/replicator/replication-status.js.map +1 -1
- package/out/zero-cache/src/services/replicator/replicator.js.map +1 -1
- package/out/zero-cache/src/services/replicator/sqlite-change-log-observability.d.ts +6 -6
- package/out/zero-cache/src/services/replicator/sqlite-change-log-observability.d.ts.map +1 -1
- package/out/zero-cache/src/services/replicator/sqlite-change-log-observability.js +19 -14
- package/out/zero-cache/src/services/replicator/sqlite-change-log-observability.js.map +1 -1
- package/out/zero-cache/src/services/replicator/write-worker-client.d.ts +31 -6
- package/out/zero-cache/src/services/replicator/write-worker-client.d.ts.map +1 -1
- package/out/zero-cache/src/services/replicator/write-worker-client.js +5 -4
- package/out/zero-cache/src/services/replicator/write-worker-client.js.map +1 -1
- package/out/zero-cache/src/services/replicator/write-worker.js +30 -5
- package/out/zero-cache/src/services/replicator/write-worker.js.map +1 -1
- package/out/zero-cache/src/services/run-ast.d.ts +12 -0
- package/out/zero-cache/src/services/run-ast.d.ts.map +1 -1
- package/out/zero-cache/src/services/run-ast.js +26 -14
- package/out/zero-cache/src/services/run-ast.js.map +1 -1
- package/out/zero-cache/src/services/runner.d.ts.map +1 -1
- package/out/zero-cache/src/services/runner.js +1 -1
- package/out/zero-cache/src/services/runner.js.map +1 -1
- package/out/zero-cache/src/services/view-syncer/client-handler.d.ts +1 -1
- package/out/zero-cache/src/services/view-syncer/client-handler.d.ts.map +1 -1
- package/out/zero-cache/src/services/view-syncer/client-handler.js +36 -31
- package/out/zero-cache/src/services/view-syncer/client-handler.js.map +1 -1
- package/out/zero-cache/src/services/view-syncer/cvr.d.ts +45 -0
- package/out/zero-cache/src/services/view-syncer/cvr.d.ts.map +1 -1
- package/out/zero-cache/src/services/view-syncer/cvr.js +133 -1
- package/out/zero-cache/src/services/view-syncer/cvr.js.map +1 -1
- package/out/zero-cache/src/services/view-syncer/drain-coordinator.d.ts +14 -1
- package/out/zero-cache/src/services/view-syncer/drain-coordinator.d.ts.map +1 -1
- package/out/zero-cache/src/services/view-syncer/drain-coordinator.js +30 -4
- package/out/zero-cache/src/services/view-syncer/drain-coordinator.js.map +1 -1
- package/out/zero-cache/src/services/view-syncer/hydration-budget.d.ts +33 -0
- package/out/zero-cache/src/services/view-syncer/hydration-budget.d.ts.map +1 -0
- package/out/zero-cache/src/services/view-syncer/hydration-budget.js +62 -0
- package/out/zero-cache/src/services/view-syncer/hydration-budget.js.map +1 -0
- package/out/zero-cache/src/services/view-syncer/hydration-circuit-breaker.d.ts +51 -0
- package/out/zero-cache/src/services/view-syncer/hydration-circuit-breaker.d.ts.map +1 -0
- package/out/zero-cache/src/services/view-syncer/hydration-circuit-breaker.js +78 -0
- package/out/zero-cache/src/services/view-syncer/hydration-circuit-breaker.js.map +1 -0
- package/out/zero-cache/src/services/view-syncer/inspect-handler.d.ts +1 -1
- package/out/zero-cache/src/services/view-syncer/inspect-handler.d.ts.map +1 -1
- package/out/zero-cache/src/services/view-syncer/inspect-handler.js +2 -2
- package/out/zero-cache/src/services/view-syncer/inspect-handler.js.map +1 -1
- package/out/zero-cache/src/services/view-syncer/pipeline-driver.d.ts +2 -1
- package/out/zero-cache/src/services/view-syncer/pipeline-driver.d.ts.map +1 -1
- package/out/zero-cache/src/services/view-syncer/pipeline-driver.js +33 -18
- package/out/zero-cache/src/services/view-syncer/pipeline-driver.js.map +1 -1
- package/out/zero-cache/src/services/view-syncer/snapshotter.d.ts +4 -1
- package/out/zero-cache/src/services/view-syncer/snapshotter.d.ts.map +1 -1
- package/out/zero-cache/src/services/view-syncer/snapshotter.js +9 -3
- package/out/zero-cache/src/services/view-syncer/snapshotter.js.map +1 -1
- package/out/zero-cache/src/services/view-syncer/view-syncer.d.ts +11 -1
- package/out/zero-cache/src/services/view-syncer/view-syncer.d.ts.map +1 -1
- package/out/zero-cache/src/services/view-syncer/view-syncer.js +507 -93
- package/out/zero-cache/src/services/view-syncer/view-syncer.js.map +1 -1
- package/out/zero-cache/src/types/processes.d.ts +32 -0
- package/out/zero-cache/src/types/processes.d.ts.map +1 -1
- package/out/zero-cache/src/types/processes.js +43 -4
- package/out/zero-cache/src/types/processes.js.map +1 -1
- package/out/zero-cache/src/types/profiler.d.ts +12 -0
- package/out/zero-cache/src/types/profiler.d.ts.map +1 -1
- package/out/zero-cache/src/types/profiler.js +46 -1
- package/out/zero-cache/src/types/profiler.js.map +1 -1
- package/out/zero-cache/src/types/streams.d.ts +25 -9
- package/out/zero-cache/src/types/streams.d.ts.map +1 -1
- package/out/zero-cache/src/types/streams.js +118 -34
- package/out/zero-cache/src/types/streams.js.map +1 -1
- package/out/zero-cache/src/types/subscription.d.ts +4 -0
- package/out/zero-cache/src/types/subscription.d.ts.map +1 -1
- package/out/zero-cache/src/types/subscription.js +71 -0
- package/out/zero-cache/src/types/subscription.js.map +1 -1
- package/out/zero-cache/src/types/ws.d.ts.map +1 -1
- package/out/zero-cache/src/types/ws.js +23 -8
- package/out/zero-cache/src/types/ws.js.map +1 -1
- package/out/zero-cache/src/workers/connection.js +2 -2
- package/out/zero-cache/src/workers/replicator.d.ts +1 -0
- package/out/zero-cache/src/workers/replicator.d.ts.map +1 -1
- package/out/zero-cache/src/workers/replicator.js +4 -2
- package/out/zero-cache/src/workers/replicator.js.map +1 -1
- package/out/zero-cache/src/workers/syncer-ws-message-handler.d.ts.map +1 -1
- package/out/zero-cache/src/workers/syncer-ws-message-handler.js +21 -16
- package/out/zero-cache/src/workers/syncer-ws-message-handler.js.map +1 -1
- package/out/zero-cache/src/workers/syncer.d.ts.map +1 -1
- package/out/zero-cache/src/workers/syncer.js +10 -1
- package/out/zero-cache/src/workers/syncer.js.map +1 -1
- package/out/zero-client/src/client/connection-manager.d.ts +20 -0
- package/out/zero-client/src/client/connection-manager.d.ts.map +1 -1
- package/out/zero-client/src/client/connection-manager.js +77 -13
- package/out/zero-client/src/client/connection-manager.js.map +1 -1
- package/out/zero-client/src/client/connection-status-enum.d.ts +2 -0
- package/out/zero-client/src/client/connection-status-enum.d.ts.map +1 -1
- package/out/zero-client/src/client/connection-status-enum.js +3 -1
- package/out/zero-client/src/client/connection-status-enum.js.map +1 -1
- package/out/zero-client/src/client/connection.d.ts +6 -0
- package/out/zero-client/src/client/connection.d.ts.map +1 -1
- package/out/zero-client/src/client/connection.js +2 -1
- package/out/zero-client/src/client/connection.js.map +1 -1
- package/out/zero-client/src/client/context.d.ts +27 -2
- package/out/zero-client/src/client/context.d.ts.map +1 -1
- package/out/zero-client/src/client/context.js +112 -1
- package/out/zero-client/src/client/context.js.map +1 -1
- package/out/zero-client/src/client/custom.js +2 -2
- package/out/zero-client/src/client/custom.js.map +1 -1
- package/out/zero-client/src/client/error.js +1 -1
- package/out/zero-client/src/client/inspector/lazy-inspector.js +2 -2
- package/out/zero-client/src/client/inspector/lazy-inspector.js.map +1 -1
- package/out/zero-client/src/client/ivm-branch.d.ts.map +1 -1
- package/out/zero-client/src/client/ivm-branch.js +20 -3
- package/out/zero-client/src/client/ivm-branch.js.map +1 -1
- package/out/zero-client/src/client/mutator-proxy.d.ts.map +1 -1
- package/out/zero-client/src/client/mutator-proxy.js +2 -1
- package/out/zero-client/src/client/mutator-proxy.js.map +1 -1
- package/out/zero-client/src/client/options.d.ts +3 -0
- package/out/zero-client/src/client/options.d.ts.map +1 -1
- package/out/zero-client/src/client/options.js.map +1 -1
- package/out/zero-client/src/client/query-manager.d.ts.map +1 -1
- package/out/zero-client/src/client/query-manager.js +42 -6
- package/out/zero-client/src/client/query-manager.js.map +1 -1
- package/out/zero-client/src/client/version.js +1 -1
- package/out/zero-client/src/client/zero-rep.d.ts +1 -0
- package/out/zero-client/src/client/zero-rep.d.ts.map +1 -1
- package/out/zero-client/src/client/zero-rep.js +9 -6
- package/out/zero-client/src/client/zero-rep.js.map +1 -1
- package/out/zero-client/src/client/zero.d.ts +18 -3
- package/out/zero-client/src/client/zero.d.ts.map +1 -1
- package/out/zero-client/src/client/zero.js +122 -34
- package/out/zero-client/src/client/zero.js.map +1 -1
- package/out/zero-client/src/types/query-result.d.ts +2 -0
- package/out/zero-client/src/types/query-result.d.ts.map +1 -1
- package/out/zero-events/src/status.d.ts +26 -0
- package/out/zero-events/src/status.d.ts.map +1 -1
- package/out/zero-protocol/src/analyze-query-result.d.ts +5 -5
- package/out/zero-protocol/src/analyze-query-result.d.ts.map +1 -1
- package/out/zero-protocol/src/analyze-query-result.js +1 -1
- package/out/zero-protocol/src/analyze-query-result.js.map +1 -1
- package/out/zero-protocol/src/down.d.ts +4 -4
- package/out/zero-protocol/src/inspect-down.d.ts +6 -6
- package/out/zero-protocol/src/protocol-version.d.ts +1 -1
- package/out/zero-protocol/src/protocol-version.d.ts.map +1 -1
- package/out/zero-protocol/src/protocol-version.js.map +1 -1
- package/out/zero-react/src/use-query.d.ts +5 -4
- package/out/zero-react/src/use-query.d.ts.map +1 -1
- package/out/zero-react/src/use-query.js +86 -9
- package/out/zero-react/src/use-query.js.map +1 -1
- package/out/zero-server/src/adapters/drizzle.d.ts +2 -1
- package/out/zero-server/src/adapters/drizzle.d.ts.map +1 -1
- package/out/zero-server/src/adapters/drizzle.js +12 -7
- package/out/zero-server/src/adapters/drizzle.js.map +1 -1
- package/out/zero-server/src/adapters/kysely.d.ts +2 -1
- package/out/zero-server/src/adapters/kysely.d.ts.map +1 -1
- package/out/zero-server/src/adapters/kysely.js +8 -2
- package/out/zero-server/src/adapters/kysely.js.map +1 -1
- package/out/zero-server/src/adapters/pg.d.ts +1 -0
- package/out/zero-server/src/adapters/pg.d.ts.map +1 -1
- package/out/zero-server/src/adapters/pg.js +8 -2
- package/out/zero-server/src/adapters/pg.js.map +1 -1
- package/out/zero-server/src/adapters/postgresjs.d.ts +1 -0
- package/out/zero-server/src/adapters/postgresjs.d.ts.map +1 -1
- package/out/zero-server/src/adapters/postgresjs.js +7 -1
- package/out/zero-server/src/adapters/postgresjs.js.map +1 -1
- package/out/zero-server/src/adapters/prisma.d.ts +3 -2
- package/out/zero-server/src/adapters/prisma.d.ts.map +1 -1
- package/out/zero-server/src/adapters/prisma.js +8 -2
- package/out/zero-server/src/adapters/prisma.js.map +1 -1
- package/out/zero-server/src/custom.d.ts +5 -1
- package/out/zero-server/src/custom.d.ts.map +1 -1
- package/out/zero-server/src/custom.js +6 -3
- package/out/zero-server/src/custom.js.map +1 -1
- package/out/zero-server/src/mod.d.ts +1 -1
- package/out/zero-server/src/mod.d.ts.map +1 -1
- package/out/zero-server/src/pg-query-executor.d.ts +2 -2
- package/out/zero-server/src/pg-query-executor.d.ts.map +1 -1
- package/out/zero-server/src/pg-query-executor.js +2 -2
- package/out/zero-server/src/pg-query-executor.js.map +1 -1
- package/out/zero-server/src/schema.d.ts +2 -2
- package/out/zero-server/src/schema.d.ts.map +1 -1
- package/out/zero-server/src/schema.js +2 -2
- package/out/zero-server/src/schema.js.map +1 -1
- package/out/zero-server/src/zql-database.d.ts +21 -1
- package/out/zero-server/src/zql-database.d.ts.map +1 -1
- package/out/zero-server/src/zql-database.js +33 -2
- package/out/zero-server/src/zql-database.js.map +1 -1
- package/out/zero-solid/src/solid-view.d.ts +17 -0
- package/out/zero-solid/src/solid-view.d.ts.map +1 -1
- package/out/zero-solid/src/solid-view.js +39 -1
- package/out/zero-solid/src/solid-view.js.map +1 -1
- package/out/zero-solid/src/use-query.js +5 -3
- package/out/zero-solid/src/use-query.js.map +1 -1
- package/out/zql/src/builder/builder.d.ts.map +1 -1
- package/out/zql/src/builder/builder.js +16 -10
- package/out/zql/src/builder/builder.js.map +1 -1
- package/out/zql/src/builder/debug-delegate.d.ts +6 -1
- package/out/zql/src/builder/debug-delegate.d.ts.map +1 -1
- package/out/zql/src/builder/debug-delegate.js +14 -2
- package/out/zql/src/builder/debug-delegate.js.map +1 -1
- package/out/zql/src/ivm/array-view.d.ts +29 -0
- package/out/zql/src/ivm/array-view.d.ts.map +1 -1
- package/out/zql/src/ivm/array-view.js +63 -12
- package/out/zql/src/ivm/array-view.js.map +1 -1
- package/out/zql/src/ivm/cap.d.ts.map +1 -1
- package/out/zql/src/ivm/cap.js +5 -3
- package/out/zql/src/ivm/cap.js.map +1 -1
- package/out/zql/src/ivm/data.d.ts.map +1 -1
- package/out/zql/src/ivm/data.js +8 -3
- package/out/zql/src/ivm/data.js.map +1 -1
- package/out/zql/src/ivm/deferred-input.d.ts +44 -0
- package/out/zql/src/ivm/deferred-input.d.ts.map +1 -0
- package/out/zql/src/ivm/deferred-input.js +89 -0
- package/out/zql/src/ivm/deferred-input.js.map +1 -0
- package/out/zql/src/ivm/exists.d.ts +1 -1
- package/out/zql/src/ivm/exists.d.ts.map +1 -1
- package/out/zql/src/ivm/exists.js.map +1 -1
- package/out/zql/src/ivm/fan-in.d.ts +2 -2
- package/out/zql/src/ivm/fan-in.d.ts.map +1 -1
- package/out/zql/src/ivm/fan-in.js +6 -5
- package/out/zql/src/ivm/fan-in.js.map +1 -1
- package/out/zql/src/ivm/fan-out.d.ts +2 -2
- package/out/zql/src/ivm/fan-out.d.ts.map +1 -1
- package/out/zql/src/ivm/fan-out.js.map +1 -1
- package/out/zql/src/ivm/filter-operators.d.ts +7 -6
- package/out/zql/src/ivm/filter-operators.d.ts.map +1 -1
- package/out/zql/src/ivm/filter-operators.js +29 -12
- package/out/zql/src/ivm/filter-operators.js.map +1 -1
- package/out/zql/src/ivm/filter.d.ts +2 -2
- package/out/zql/src/ivm/filter.d.ts.map +1 -1
- package/out/zql/src/ivm/filter.js +7 -4
- package/out/zql/src/ivm/filter.js.map +1 -1
- package/out/zql/src/ivm/flipped-join.js +2 -2
- package/out/zql/src/ivm/flipped-join.js.map +1 -1
- package/out/zql/src/ivm/maybe-split-and-push-edit-change.d.ts +1 -1
- package/out/zql/src/ivm/maybe-split-and-push-edit-change.d.ts.map +1 -1
- package/out/zql/src/ivm/maybe-split-and-push-edit-change.js +6 -4
- package/out/zql/src/ivm/maybe-split-and-push-edit-change.js.map +1 -1
- package/out/zql/src/ivm/memory-source.d.ts +15 -3
- package/out/zql/src/ivm/memory-source.d.ts.map +1 -1
- package/out/zql/src/ivm/memory-source.js +57 -23
- package/out/zql/src/ivm/memory-source.js.map +1 -1
- package/out/zql/src/ivm/operator.d.ts +40 -0
- package/out/zql/src/ivm/operator.d.ts.map +1 -1
- package/out/zql/src/ivm/operator.js.map +1 -1
- package/out/zql/src/ivm/take.d.ts.map +1 -1
- package/out/zql/src/ivm/take.js +3 -4
- package/out/zql/src/ivm/take.js.map +1 -1
- package/out/zql/src/ivm/union-fan-in.d.ts.map +1 -1
- package/out/zql/src/ivm/union-fan-in.js +9 -7
- package/out/zql/src/ivm/union-fan-in.js.map +1 -1
- package/out/zql/src/ivm/view-apply-change.d.ts.map +1 -1
- package/out/zql/src/ivm/view-apply-change.js +9 -4
- package/out/zql/src/ivm/view-apply-change.js.map +1 -1
- package/out/zql/src/ivm/view.d.ts +15 -0
- package/out/zql/src/ivm/view.d.ts.map +1 -1
- package/out/zql/src/mutate/custom.d.ts +24 -1
- package/out/zql/src/mutate/custom.d.ts.map +1 -1
- package/out/zql/src/mutate/custom.js.map +1 -1
- package/out/zql/src/mutate/mutator-registry.d.ts.map +1 -1
- package/out/zql/src/mutate/mutator-registry.js +2 -2
- package/out/zql/src/mutate/mutator-registry.js.map +1 -1
- package/out/zql/src/planner/planner-builder.d.ts.map +1 -1
- package/out/zql/src/planner/planner-builder.js +14 -4
- package/out/zql/src/planner/planner-builder.js.map +1 -1
- package/out/zql/src/planner/planner-connection.d.ts +13 -0
- package/out/zql/src/planner/planner-connection.d.ts.map +1 -1
- package/out/zql/src/planner/planner-connection.js +42 -7
- package/out/zql/src/planner/planner-connection.js.map +1 -1
- package/out/zql/src/planner/planner-debug.d.ts +1 -1
- package/out/zql/src/planner/planner-debug.d.ts.map +1 -1
- package/out/zql/src/planner/planner-debug.js.map +1 -1
- package/out/zql/src/planner/planner-fan-out.d.ts +1 -0
- package/out/zql/src/planner/planner-fan-out.d.ts.map +1 -1
- package/out/zql/src/planner/planner-fan-out.js +3 -0
- package/out/zql/src/planner/planner-fan-out.js.map +1 -1
- package/out/zql/src/planner/planner-filter.d.ts +59 -0
- package/out/zql/src/planner/planner-filter.d.ts.map +1 -0
- package/out/zql/src/planner/planner-filter.js +116 -0
- package/out/zql/src/planner/planner-filter.js.map +1 -0
- package/out/zql/src/planner/planner-graph.d.ts +2 -0
- package/out/zql/src/planner/planner-graph.d.ts.map +1 -1
- package/out/zql/src/planner/planner-graph.js +2 -0
- package/out/zql/src/planner/planner-graph.js.map +1 -1
- package/out/zql/src/planner/planner-join.d.ts +2 -0
- package/out/zql/src/planner/planner-join.d.ts.map +1 -1
- package/out/zql/src/planner/planner-join.js +7 -0
- package/out/zql/src/planner/planner-join.js.map +1 -1
- package/out/zql/src/planner/planner-node.d.ts +2 -1
- package/out/zql/src/planner/planner-node.d.ts.map +1 -1
- package/out/zql/src/planner/planner-node.js.map +1 -1
- package/out/zql/src/query/query-delegate-base.d.ts +9 -1
- package/out/zql/src/query/query-delegate-base.d.ts.map +1 -1
- package/out/zql/src/query/query-delegate-base.js +129 -33
- package/out/zql/src/query/query-delegate-base.js.map +1 -1
- package/out/zql/src/query/query-delegate.d.ts +50 -1
- package/out/zql/src/query/query-delegate.d.ts.map +1 -1
- package/out/zql/src/query/query-impl.d.ts +13 -11
- package/out/zql/src/query/query-impl.d.ts.map +1 -1
- package/out/zql/src/query/query-impl.js +383 -60
- package/out/zql/src/query/query-impl.js.map +1 -1
- package/out/zql/src/query/query-transitions.d.ts +171 -0
- package/out/zql/src/query/query-transitions.d.ts.map +1 -0
- package/out/zql/src/query/query-transitions.js +207 -0
- package/out/zql/src/query/query-transitions.js.map +1 -0
- package/out/zql/src/query/query.d.ts +11 -2
- package/out/zql/src/query/query.d.ts.map +1 -1
- package/out/zql/src/query/runnable-query-impl.d.ts +1 -0
- package/out/zql/src/query/runnable-query-impl.d.ts.map +1 -1
- package/out/zql/src/query/runnable-query-impl.js +21 -1
- package/out/zql/src/query/runnable-query-impl.js.map +1 -1
- package/out/zql/src/query/typed-view.d.ts +1 -1
- package/out/zql/src/query/typed-view.d.ts.map +1 -1
- package/out/zqlite/src/db.d.ts +9 -1
- package/out/zqlite/src/db.d.ts.map +1 -1
- package/out/zqlite/src/db.js +8 -2
- package/out/zqlite/src/db.js.map +1 -1
- package/out/zqlite/src/query-builder.d.ts +1 -1
- package/out/zqlite/src/query-builder.d.ts.map +1 -1
- package/out/zqlite/src/query-builder.js +2 -1
- package/out/zqlite/src/query-builder.js.map +1 -1
- package/out/zqlite/src/sqlite-cost-model.d.ts.map +1 -1
- package/out/zqlite/src/sqlite-cost-model.js +2 -30
- package/out/zqlite/src/sqlite-cost-model.js.map +1 -1
- package/out/zqlite/src/table-source.d.ts +1 -0
- package/out/zqlite/src/table-source.d.ts.map +1 -1
- package/out/zqlite/src/table-source.js +20 -8
- package/out/zqlite/src/table-source.js.map +1 -1
- package/package.json +3 -4
|
@@ -33,9 +33,25 @@ var UpstreamAcker = class {
|
|
|
33
33
|
this.#trackPgChangeLog = trackPgChangeLog;
|
|
34
34
|
this.#trackBackup = trackBackup;
|
|
35
35
|
}
|
|
36
|
-
|
|
36
|
+
/**
|
|
37
|
+
* Starts tracking a new upstream connection, which resumes the change stream
|
|
38
|
+
* after `resumeWatermark`.
|
|
39
|
+
*
|
|
40
|
+
* Everything at or before `resumeWatermark` was committed on an earlier
|
|
41
|
+
* connection, so it counts as committed on this one: a status watermark past
|
|
42
|
+
* it waits for the tracked stores to reach it, as it would for a commit on
|
|
43
|
+
* this connection. Otherwise the first keepalive of a stream that resumes
|
|
44
|
+
* ahead of the stores -- a SQLite change log ahead of its backup, which is
|
|
45
|
+
* the normal state -- would move the replication slot past transactions that
|
|
46
|
+
* no store has persisted, and a task restored from the backup would then
|
|
47
|
+
* resume below the slot, which the upstream moves forward without a word.
|
|
48
|
+
*
|
|
49
|
+
* Pass `''` when the stream resumes from what a tracked store has itself
|
|
50
|
+
* persisted, which leaves nothing before it outstanding.
|
|
51
|
+
*/
|
|
52
|
+
reset(upstream, resumeWatermark) {
|
|
37
53
|
this.#upstream = upstream;
|
|
38
|
-
this.#lastTx =
|
|
54
|
+
this.#lastTx = resumeWatermark;
|
|
39
55
|
this.#lastStatus = "";
|
|
40
56
|
this.#lastAck = "";
|
|
41
57
|
}
|
|
@@ -47,7 +63,7 @@ var UpstreamAcker = class {
|
|
|
47
63
|
this.#maybeAck();
|
|
48
64
|
break;
|
|
49
65
|
case "commit":
|
|
50
|
-
this.#lastTx = downstream[2].watermark;
|
|
66
|
+
this.#lastTx = max(this.#lastTx, downstream[2].watermark);
|
|
51
67
|
this.#maybeAck();
|
|
52
68
|
break;
|
|
53
69
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"upstream-acker.js","names":["#trackPgChangeLog","#trackBackup","#upstream","#lastTx","#lastStatus","#lastAck","#maybeAck","#pgChangeLogWatermark","#backupWatermark"],"sources":["../../../../../../zero-cache/src/services/change-streamer/upstream-acker.ts"],"sourcesContent":["import {assert} from '../../../../shared/src/asserts.ts';\nimport {max, min} from '../../types/lexi-version.ts';\nimport type {Sink} from '../../types/streams.ts';\nimport type {\n ChangeSourceUpstream,\n ChangeStreamMessage,\n} from '../change-source/protocol/current.ts';\n\ntype Opts = {\n trackPgChangeLog: boolean;\n trackBackup: boolean;\n};\n\n/**\n * Tracks the progress (watermark) of multiple streams:\n * - downstream transactions and status messages (the latter\n * indicating LSNs that are not relevant to the publication\n * but nevertheless need to be ACKed)\n * - PG change-log commits\n * - backup watermarks\n *\n * and sends upstream ACKs accordingly. When both the PG change-log\n * and backup watermarks are being considered for upstream ACKs\n * (i.e. RMv1.5), only watermarks that have been reached by both\n * stores are acked.\n *\n * The UpstreamAcker also takes into account that an upstream\n * connection can be disconnected and {@link reset()}. In this case,\n * the progress of the persistent stores is retained and the acks\n * are resent on the new upstream connection as necessary.\n */\nexport class UpstreamAcker {\n readonly #trackPgChangeLog: boolean;\n readonly #trackBackup: boolean;\n\n #pgChangeLogWatermark = '';\n #backupWatermark = '';\n\n #upstream: Sink<ChangeSourceUpstream> | undefined;\n #lastTx = '';\n #lastStatus = '';\n #lastAck = '';\n\n constructor({trackPgChangeLog, trackBackup}: Opts) {\n assert(\n trackPgChangeLog || trackBackup,\n `At least one of trackPgChangeLog or trackBackup must be true`,\n );\n this.#trackPgChangeLog = trackPgChangeLog;\n this.#trackBackup = trackBackup;\n }\n\n reset(upstream: Sink<ChangeSourceUpstream
|
|
1
|
+
{"version":3,"file":"upstream-acker.js","names":["#trackPgChangeLog","#trackBackup","#upstream","#lastTx","#lastStatus","#lastAck","#maybeAck","#pgChangeLogWatermark","#backupWatermark"],"sources":["../../../../../../zero-cache/src/services/change-streamer/upstream-acker.ts"],"sourcesContent":["import {assert} from '../../../../shared/src/asserts.ts';\nimport {max, min} from '../../types/lexi-version.ts';\nimport type {Sink} from '../../types/streams.ts';\nimport type {\n ChangeSourceUpstream,\n ChangeStreamMessage,\n} from '../change-source/protocol/current.ts';\n\ntype Opts = {\n trackPgChangeLog: boolean;\n trackBackup: boolean;\n};\n\n/**\n * Tracks the progress (watermark) of multiple streams:\n * - downstream transactions and status messages (the latter\n * indicating LSNs that are not relevant to the publication\n * but nevertheless need to be ACKed)\n * - PG change-log commits\n * - backup watermarks\n *\n * and sends upstream ACKs accordingly. When both the PG change-log\n * and backup watermarks are being considered for upstream ACKs\n * (i.e. RMv1.5), only watermarks that have been reached by both\n * stores are acked.\n *\n * The UpstreamAcker also takes into account that an upstream\n * connection can be disconnected and {@link reset()}. In this case,\n * the progress of the persistent stores is retained and the acks\n * are resent on the new upstream connection as necessary.\n */\nexport class UpstreamAcker {\n readonly #trackPgChangeLog: boolean;\n readonly #trackBackup: boolean;\n\n #pgChangeLogWatermark = '';\n #backupWatermark = '';\n\n #upstream: Sink<ChangeSourceUpstream> | undefined;\n #lastTx = '';\n #lastStatus = '';\n #lastAck = '';\n\n constructor({trackPgChangeLog, trackBackup}: Opts) {\n assert(\n trackPgChangeLog || trackBackup,\n `At least one of trackPgChangeLog or trackBackup must be true`,\n );\n this.#trackPgChangeLog = trackPgChangeLog;\n this.#trackBackup = trackBackup;\n }\n\n /**\n * Starts tracking a new upstream connection, which resumes the change stream\n * after `resumeWatermark`.\n *\n * Everything at or before `resumeWatermark` was committed on an earlier\n * connection, so it counts as committed on this one: a status watermark past\n * it waits for the tracked stores to reach it, as it would for a commit on\n * this connection. Otherwise the first keepalive of a stream that resumes\n * ahead of the stores -- a SQLite change log ahead of its backup, which is\n * the normal state -- would move the replication slot past transactions that\n * no store has persisted, and a task restored from the backup would then\n * resume below the slot, which the upstream moves forward without a word.\n *\n * Pass `''` when the stream resumes from what a tracked store has itself\n * persisted, which leaves nothing before it outstanding.\n */\n reset(upstream: Sink<ChangeSourceUpstream>, resumeWatermark: string) {\n this.#upstream = upstream;\n this.#lastTx = resumeWatermark;\n this.#lastStatus = '';\n this.#lastAck = '';\n }\n\n trackDownstream(downstream: ChangeStreamMessage) {\n const [tag, msg] = downstream;\n switch (tag) {\n case 'status':\n if (msg.ack) {\n this.#lastStatus = downstream[2].watermark;\n }\n this.#maybeAck();\n break;\n case 'commit':\n // A commit replayed from below the resume watermark does not lower it.\n this.#lastTx = max(this.#lastTx, downstream[2].watermark);\n this.#maybeAck();\n break;\n }\n }\n\n trackPgChangeLog(committedWatermark: string) {\n // Watermarks should never move backwards, but use max() defensively.\n this.#pgChangeLogWatermark = max(\n committedWatermark,\n this.#pgChangeLogWatermark,\n );\n this.#maybeAck();\n }\n\n trackBackup(backedUpWatermark: string) {\n // Watermarks should never move backwards, but use max() defensively.\n this.#backupWatermark = max(backedUpWatermark, this.#backupWatermark);\n this.#maybeAck();\n }\n\n #maybeAck() {\n const currentWatermark = !this.#trackBackup\n ? this.#pgChangeLogWatermark\n : !this.#trackPgChangeLog\n ? this.#backupWatermark\n : min(this.#pgChangeLogWatermark, this.#backupWatermark);\n if (currentWatermark > this.#lastAck) {\n this.#upstream?.push([\n 'status',\n {tag: 'commit'},\n {watermark: currentWatermark},\n ]);\n this.#lastAck = currentWatermark;\n }\n // If all committed transactions have been acked, ack any outstanding\n // status LSN's thereafter (i.e. non-publication upstream changes).\n if (this.#lastAck >= this.#lastTx && this.#lastStatus > this.#lastAck) {\n this.#upstream?.push([\n 'status',\n {ack: true},\n {watermark: this.#lastStatus},\n ]);\n this.#lastAck = this.#lastStatus;\n }\n }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;AA+BA,IAAa,gBAAb,MAA2B;CACzB;CACA;CAEA,wBAAwB;CACxB,mBAAmB;CAEnB;CACA,UAAU;CACV,cAAc;CACd,WAAW;CAEX,YAAY,EAAC,kBAAkB,eAAoB;EACjD,OACE,oBAAoB,aACpB,8DACF;EACA,KAAKA,oBAAoB;EACzB,KAAKC,eAAe;CACtB;;;;;;;;;;;;;;;;;CAkBA,MAAM,UAAsC,iBAAyB;EACnE,KAAKC,YAAY;EACjB,KAAKC,UAAU;EACf,KAAKC,cAAc;EACnB,KAAKC,WAAW;CAClB;CAEA,gBAAgB,YAAiC;EAC/C,MAAM,CAAC,KAAK,OAAO;EACnB,QAAQ,KAAR;GACE,KAAK;IACH,IAAI,IAAI,KACN,KAAKD,cAAc,WAAW,GAAG;IAEnC,KAAKE,UAAU;IACf;GACF,KAAK;IAEH,KAAKH,UAAU,IAAI,KAAKA,SAAS,WAAW,GAAG,SAAS;IACxD,KAAKG,UAAU;IACf;EACJ;CACF;CAEA,iBAAiB,oBAA4B;EAE3C,KAAKC,wBAAwB,IAC3B,oBACA,KAAKA,qBACP;EACA,KAAKD,UAAU;CACjB;CAEA,YAAY,mBAA2B;EAErC,KAAKE,mBAAmB,IAAI,mBAAmB,KAAKA,gBAAgB;EACpE,KAAKF,UAAU;CACjB;CAEA,YAAY;EACV,MAAM,mBAAmB,CAAC,KAAKL,eAC3B,KAAKM,wBACL,CAAC,KAAKP,oBACJ,KAAKQ,mBACL,IAAI,KAAKD,uBAAuB,KAAKC,gBAAgB;EAC3D,IAAI,mBAAmB,KAAKH,UAAU;GACpC,KAAKH,WAAW,KAAK;IACnB;IACA,EAAC,KAAK,SAAQ;IACd,EAAC,WAAW,iBAAgB;GAC9B,CAAC;GACD,KAAKG,WAAW;EAClB;EAGA,IAAI,KAAKA,YAAY,KAAKF,WAAW,KAAKC,cAAc,KAAKC,UAAU;GACrE,KAAKH,WAAW,KAAK;IACnB;IACA,EAAC,KAAK,KAAI;IACV,EAAC,WAAW,KAAKE,YAAW;GAC9B,CAAC;GACD,KAAKC,WAAW,KAAKD;EACvB;CACF;AACF"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"vfs-watermark-poller.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/change-streamer/vfs-watermark-poller.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAIjD,OAAO,KAAK,EAAC,MAAM,EAAC,MAAM,wBAAwB,CAAC;AAGnD,OAAO,KAAK,EAAC,iBAAiB,EAAC,MAAM,qBAAqB,CAAC;AAe3D,MAAM,MAAM,eAAe,GAAG;IAC5B,UAAU,EAAE,MAAM,CAAC;IACnB,oBAAoB,EAAE,MAAM,CAAC;IAC7B,oBAAoB,CAAC,EAAE,MAAM,CAAC;IAC9B,SAAS,EAAE,MAAM,CAAC;IAClB,MAAM,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAC5B,QAAQ,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAC9B,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB,CAAC;AAyBF,qBAAa,kBAAkB;;IAkB7B,YACE,EAAE,EAAE,UAAU,EACd,WAAW,EAAE,MAAM,EACnB,YAAY,EAAE,eAAe,EAuB9B;IAED;;;OAGG;IACH,KAAK,IAAI,MAAM,CAAC,iBAAiB,CAAC,CASjC;IAGD,QAAQ,CAAC,mBAAmB,aAiB1B;
|
|
1
|
+
{"version":3,"file":"vfs-watermark-poller.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/change-streamer/vfs-watermark-poller.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAIjD,OAAO,KAAK,EAAC,MAAM,EAAC,MAAM,wBAAwB,CAAC;AAGnD,OAAO,KAAK,EAAC,iBAAiB,EAAC,MAAM,qBAAqB,CAAC;AAe3D,MAAM,MAAM,eAAe,GAAG;IAC5B,UAAU,EAAE,MAAM,CAAC;IACnB,oBAAoB,EAAE,MAAM,CAAC;IAC7B,oBAAoB,CAAC,EAAE,MAAM,CAAC;IAC9B,SAAS,EAAE,MAAM,CAAC;IAClB,MAAM,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAC5B,QAAQ,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAC9B,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB,CAAC;AAyBF,qBAAa,kBAAkB;;IAkB7B,YACE,EAAE,EAAE,UAAU,EACd,WAAW,EAAE,MAAM,EACnB,YAAY,EAAE,eAAe,EAuB9B;IAED;;;OAGG;IACH,KAAK,IAAI,MAAM,CAAC,iBAAiB,CAAC,CASjC;IAGD,QAAQ,CAAC,mBAAmB,aAiB1B;CAqHH;AAED,wBAAgB,4BAA4B,CAAC,EAC3C,SAAS,EACT,QAAQ,EACR,MAAM,GACP,EAAE;IACD,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,QAAQ,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IACvC,QAAQ,CAAC,MAAM,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;CACtC,GAAG,MAAM,CAiBT"}
|
|
@@ -90,6 +90,7 @@ var VfsWatermarkPoller = class {
|
|
|
90
90
|
] }).on("error", (err) => {
|
|
91
91
|
this.#lc.error?.(`received error from vfs-query process`, err);
|
|
92
92
|
}).on("close", (code, signal) => {
|
|
93
|
+
if (this.#remotePoller !== child) return;
|
|
93
94
|
this.#remotePoller = void 0;
|
|
94
95
|
if (this.#shouldPollRemote) {
|
|
95
96
|
if (code !== 0) this.#lc.warn?.(`vfs-query exited with ${code}, ${signal}. retrying in ${this.#remotePollerBackoffMs}ms`);
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"vfs-watermark-poller.js","names":["#lc","#state","#pollerConfig","#readLocalWatermark","#stream","#localWatermark","#disableRemotePoller","#localPollTimer","#remotePollerBackoffTimer","#remoteWatermark","#enableRemotePoller","#remotePoller","#shouldPollRemote","#remotePollerBackoffMs","#readRemotePollerLine"],"sources":["../../../../../../zero-cache/src/services/change-streamer/vfs-watermark-poller.ts"],"sourcesContent":["import {spawn, type ChildProcess} from 'node:child_process';\nimport {createInterface} from 'node:readline';\nimport type {LogContext} from '@rocicorp/logger';\nimport {assert} from '../../../../shared/src/asserts.ts';\nimport * as v from '../../../../shared/src/valita.ts';\nimport {Database, type Statement} from '../../../../zqlite/src/db.ts';\nimport type {Source} from '../../types/streams.ts';\nimport {Subscription} from '../../types/subscription.ts';\nimport {RunningState} from '../running-state.ts';\nimport type {BackedUpWatermark} from './backup-monitor.ts';\n\nconst LOCAL_CHECK_INTERVAL_MS = 1_000;\n\ntype LocalWatermark = {\n stateVersion: string;\n writeTimeMs: number;\n};\n\ntype RemoteWatermark = {\n watermark: string;\n writeTimeMs: number;\n backupTimeMs: number;\n};\n\nexport type VfsPollerConfig = {\n executable: string;\n remotePollIntervalMs: number;\n remoteQueryTimeoutMs?: number;\n backupURL: string;\n region?: string | undefined;\n endpoint?: string | undefined;\n logLevel?: string;\n logFormat?: string;\n};\n\nconst queryResultRowSchema = v.object({\n stateVersion: v.string(),\n writeTimeMs: v.number(),\n});\n\nconst remoteQueryMetadataSchema = v.object({\n litestream_time: v\n .string()\n .map(str => new Date(str).getTime())\n .optional(),\n});\n\n// Expected format of each line of stdout from the vfs-query, e.g.\n// {\"query_result\":[{\"stateVersion\":\"51y50ko\",\"writeTimeMs\":1787259951121}],\"metadata\":{\"litestream_time\":\"2026-08-20T21:05:57.36439Z\"}}\n// {\"query_result\":[{\"stateVersion\":\"51y7360\",\"writeTimeMs\":1787259966137}],\"metadata\":{\"litestream_time\":\"2026-08-20T21:06:12.384531Z\"}}\nconst remoteQueryLineSchema = v.object({\n query_result: v.tuple([queryResultRowSchema]),\n metadata: remoteQueryMetadataSchema,\n});\n\nconst INITIAL_BACKOFF_MS = 2_000;\nconst MAX_BACKOFF_MS = 30_000;\n\nexport class VfsWatermarkPoller {\n readonly #lc: LogContext;\n readonly #state = new RunningState('vfs-watermark-poller');\n readonly #pollerConfig: VfsPollerConfig;\n readonly #readLocalWatermark: Statement;\n\n readonly #stream: Subscription<BackedUpWatermark>;\n\n #localWatermark: LocalWatermark;\n #remoteWatermark: RemoteWatermark | undefined;\n #localPollTimer: NodeJS.Timeout | undefined;\n\n #shouldPollRemote = false;\n\n #remotePoller: ChildProcess | undefined;\n #remotePollerBackoffMs = INITIAL_BACKOFF_MS;\n #remotePollerBackoffTimer: NodeJS.Timeout | undefined;\n\n constructor(\n lc: LogContext,\n replicaFile: string,\n pollerConfig: VfsPollerConfig,\n ) {\n this.#lc = lc.withContext('component', 'vfs-watermark-poller');\n this.#pollerConfig = pollerConfig;\n\n const db = new Database(this.#lc, replicaFile, {readonly: true});\n this.#readLocalWatermark = db.prepare(\n `SELECT stateVersion, writeTimeMs FROM \"_zero.replicationState\"`,\n );\n // Purely to type the member variable as always defined\n this.#localWatermark = v.parse(\n this.#readLocalWatermark.get(),\n queryResultRowSchema,\n );\n this.#stream = Subscription.create<BackedUpWatermark>({\n cleanup: () => {\n this.#disableRemotePoller();\n clearInterval(this.#localPollTimer);\n clearTimeout(this.#remotePollerBackoffTimer);\n this.#state.stop(this.#lc);\n db.close();\n },\n });\n }\n\n /**\n * Starts polling for watermarks and pushes new ones to returned stream.\n * Must only be called once.\n */\n start(): Source<BackedUpWatermark> {\n assert(this.#localPollTimer === undefined, `Already called start()`);\n this.#lc.info?.(`monitoring backups at ${this.#pollerConfig.backupURL}`);\n this.#localPollTimer = setInterval(\n this.checkLocalWatermark,\n LOCAL_CHECK_INTERVAL_MS,\n );\n this.checkLocalWatermark();\n return this.#stream;\n }\n\n // Exported for testing\n readonly checkLocalWatermark = () => {\n try {\n this.#localWatermark = v.parse(\n this.#readLocalWatermark.get(),\n queryResultRowSchema,\n );\n } catch (e) {\n this.#lc.warn?.(`unable to read local watermark`, e);\n return;\n }\n if (\n this.#localWatermark.stateVersion !== this.#remoteWatermark?.watermark\n ) {\n this.#enableRemotePoller();\n } else {\n this.#disableRemotePoller();\n }\n };\n\n #enableRemotePoller() {\n if (this.#remotePoller || this.#remotePollerBackoffTimer) {\n return; // already polling or backing off\n }\n this.#shouldPollRemote = true;\n if (this.#remoteWatermark) {\n this.#lc.debug?.(\n `local and remote watermarks differ. resuming backup polling`,\n {local: this.#localWatermark, remote: this.#remoteWatermark},\n );\n }\n const child = spawn(\n this.#pollerConfig.executable,\n [\n '--replica-url',\n buildLitestreamVfsReplicaURL(this.#pollerConfig),\n '--query',\n `SELECT stateVersion, writeTimeMs FROM \"_zero.replicationState\"`,\n '--remote-poll-interval',\n `${this.#pollerConfig.remotePollIntervalMs}ms`,\n ...(this.#pollerConfig.logLevel\n ? ['--log-level', this.#pollerConfig.logLevel]\n : []),\n ...(this.#pollerConfig.logFormat\n ? ['--log-format', this.#pollerConfig.logFormat]\n : []),\n ...(this.#pollerConfig.remoteQueryTimeoutMs\n ? ['--query-timeout', `${this.#pollerConfig.remoteQueryTimeoutMs}ms`]\n : []),\n ],\n {stdio: ['pipe', 'pipe', 'inherit']},\n )\n .on('error', err => {\n this.#lc.error?.(`received error from vfs-query process`, err);\n })\n .on('close', (code, signal) => {\n this.#remotePoller = undefined;\n if (this.#shouldPollRemote) {\n if (code !== 0) {\n this.#lc.warn?.(\n `vfs-query exited with ${code}, ${signal}. retrying in ${this.#remotePollerBackoffMs}ms`,\n );\n }\n if (!this.#remotePollerBackoffTimer) {\n this.#remotePollerBackoffTimer = setTimeout(() => {\n this.#remotePollerBackoffTimer = undefined;\n if (this.#shouldPollRemote) {\n this.#enableRemotePoller();\n }\n }, this.#remotePollerBackoffMs);\n this.#remotePollerBackoffMs = Math.min(\n this.#remotePollerBackoffMs * 2,\n MAX_BACKOFF_MS,\n );\n }\n }\n });\n createInterface({input: child.stdout}).on(\n 'line',\n this.#readRemotePollerLine,\n );\n this.#remotePoller = child;\n }\n\n readonly #readRemotePollerLine = (line: string) => {\n try {\n const update = v.parse(\n JSON.parse(line),\n remoteQueryLineSchema,\n 'passthrough',\n );\n const [{stateVersion: watermark, writeTimeMs}] = update.query_result;\n const remoteWatermark: RemoteWatermark = {\n watermark,\n writeTimeMs,\n backupTimeMs: update.metadata.litestream_time ?? Date.now(),\n };\n if (remoteWatermark.watermark !== this.#remoteWatermark?.watermark) {\n this.#lc.debug?.(\n `backup watermark advanced from ${this.#remoteWatermark?.watermark} to ${remoteWatermark.watermark}`,\n {local: this.#localWatermark, remote: remoteWatermark},\n );\n this.#stream.push(remoteWatermark);\n }\n this.#remoteWatermark = remoteWatermark;\n\n // Reset backoff on successful output from the poller\n this.#remotePollerBackoffMs = INITIAL_BACKOFF_MS;\n } catch (e) {\n this.#lc.error?.(`unable to parse vfs-query output: ${line}`, e);\n }\n };\n\n #disableRemotePoller() {\n if (!this.#shouldPollRemote) {\n return;\n }\n this.#shouldPollRemote = false;\n if (this.#remotePoller || this.#remotePollerBackoffTimer) {\n this.#lc.debug?.(\n `local and remote watermarks match. pausing backup polling`,\n {local: this.#localWatermark, remote: this.#remoteWatermark},\n );\n }\n this.#remotePoller?.kill('SIGTERM');\n this.#remotePoller = undefined;\n clearTimeout(this.#remotePollerBackoffTimer);\n this.#remotePollerBackoffTimer = undefined;\n }\n}\n\nexport function buildLitestreamVfsReplicaURL({\n backupURL,\n endpoint,\n region,\n}: {\n readonly backupURL: string;\n readonly endpoint?: string | undefined;\n readonly region?: string | undefined;\n}): string {\n if (endpoint === undefined && region === undefined) {\n return backupURL;\n }\n\n const url = new URL(backupURL);\n if (url.protocol !== 's3:') {\n return backupURL;\n }\n\n if (endpoint !== undefined && !url.searchParams.has('endpoint')) {\n url.searchParams.set('endpoint', endpoint);\n }\n if (region !== undefined && !url.searchParams.has('region')) {\n url.searchParams.set('region', region);\n }\n return url.toString();\n}\n"],"mappings":";;;;;;;;AAWA,IAAM,0BAA0B;AAwBhC,IAAM,uBAAuB,eAAE,OAAO;CACpC,cAAc,eAAE,OAAO;CACvB,aAAa,eAAE,OAAO;AACxB,CAAC;AAED,IAAM,4BAA4B,eAAE,OAAO,EACzC,iBAAiB,eACd,OAAO,EACP,KAAI,QAAO,IAAI,KAAK,GAAG,EAAE,QAAQ,CAAC,EAClC,SAAS,EACd,CAAC;AAKD,IAAM,wBAAwB,eAAE,OAAO;CACrC,cAAc,eAAE,MAAM,CAAC,oBAAoB,CAAC;CAC5C,UAAU;AACZ,CAAC;AAED,IAAM,qBAAqB;AAC3B,IAAM,iBAAiB;AAEvB,IAAa,qBAAb,MAAgC;CAC9B;CACA,SAAkB,IAAI,aAAa,sBAAsB;CACzD;CACA;CAEA;CAEA;CACA;CACA;CAEA,oBAAoB;CAEpB;CACA,yBAAyB;CACzB;CAEA,YACE,IACA,aACA,cACA;EACA,KAAKA,MAAM,GAAG,YAAY,aAAa,sBAAsB;EAC7D,KAAKE,gBAAgB;EAErB,MAAM,KAAK,IAAI,SAAS,KAAKF,KAAK,aAAa,EAAC,UAAU,KAAI,CAAC;EAC/D,KAAKG,sBAAsB,GAAG,QAC5B,gEACF;EAEA,KAAKE,kBAAkB,MACrB,KAAKF,oBAAoB,IAAI,GAC7B,oBACF;EACA,KAAKC,UAAU,aAAa,OAA0B,EACpD,eAAe;GACb,KAAKE,qBAAqB;GAC1B,cAAc,KAAKC,eAAe;GAClC,aAAa,KAAKC,yBAAyB;GAC3C,KAAKP,OAAO,KAAK,KAAKD,GAAG;GACzB,GAAG,MAAM;EACX,EACF,CAAC;CACH;;;;;CAMA,QAAmC;EACjC,OAAO,KAAKO,oBAAoB,KAAA,GAAW,wBAAwB;EACnE,KAAKP,IAAI,OAAO,yBAAyB,KAAKE,cAAc,WAAW;EACvE,KAAKK,kBAAkB,YACrB,KAAK,qBACL,uBACF;EACA,KAAK,oBAAoB;EACzB,OAAO,KAAKH;CACd;CAGA,4BAAqC;EACnC,IAAI;GACF,KAAKC,kBAAkB,MACrB,KAAKF,oBAAoB,IAAI,GAC7B,oBACF;EACF,SAAS,GAAG;GACV,KAAKH,IAAI,OAAO,kCAAkC,CAAC;GACnD;EACF;EACA,IACE,KAAKK,gBAAgB,iBAAiB,KAAKI,kBAAkB,WAE7D,KAAKC,oBAAoB;OAEzB,KAAKJ,qBAAqB;CAE9B;CAEA,sBAAsB;EACpB,IAAI,KAAKK,iBAAiB,KAAKH,2BAC7B;EAEF,KAAKI,oBAAoB;EACzB,IAAI,KAAKH,kBACP,KAAKT,IAAI,QACP,+DACA;GAAC,OAAO,KAAKK;GAAiB,QAAQ,KAAKI;EAAgB,CAC7D;EAEF,MAAM,QAAQ,MACZ,KAAKP,cAAc,YACnB;GACE;GACA,6BAA6B,KAAKA,aAAa;GAC/C;GACA;GACA;GACA,GAAG,KAAKA,cAAc,qBAAqB;GAC3C,GAAI,KAAKA,cAAc,WACnB,CAAC,eAAe,KAAKA,cAAc,QAAQ,IAC3C,CAAC;GACL,GAAI,KAAKA,cAAc,YACnB,CAAC,gBAAgB,KAAKA,cAAc,SAAS,IAC7C,CAAC;GACL,GAAI,KAAKA,cAAc,uBACnB,CAAC,mBAAmB,GAAG,KAAKA,cAAc,qBAAqB,GAAG,IAClE,CAAC;EACP,GACA,EAAC,OAAO;GAAC;GAAQ;GAAQ;EAAS,EAAC,CACrC,EACG,GAAG,UAAS,QAAO;GAClB,KAAKF,IAAI,QAAQ,yCAAyC,GAAG;EAC/D,CAAC,EACA,GAAG,UAAU,MAAM,WAAW;GAC7B,KAAKW,gBAAgB,KAAA;GACrB,IAAI,KAAKC,mBAAmB;IAC1B,IAAI,SAAS,GACX,KAAKZ,IAAI,OACP,yBAAyB,KAAK,IAAI,OAAO,gBAAgB,KAAKa,uBAAuB,GACvF;IAEF,IAAI,CAAC,KAAKL,2BAA2B;KACnC,KAAKA,4BAA4B,iBAAiB;MAChD,KAAKA,4BAA4B,KAAA;MACjC,IAAI,KAAKI,mBACP,KAAKF,oBAAoB;KAE7B,GAAG,KAAKG,sBAAsB;KAC9B,KAAKA,yBAAyB,KAAK,IACjC,KAAKA,yBAAyB,GAC9B,cACF;IACF;GACF;EACF,CAAC;EACH,gBAAgB,EAAC,OAAO,MAAM,OAAM,CAAC,EAAE,GACrC,QACA,KAAKC,qBACP;EACA,KAAKH,gBAAgB;CACvB;CAEA,yBAAkC,SAAiB;EACjD,IAAI;GACF,MAAM,SAAS,MACb,KAAK,MAAM,IAAI,GACf,uBACA,aACF;GACA,MAAM,CAAC,EAAC,cAAc,WAAW,iBAAgB,OAAO;GACxD,MAAM,kBAAmC;IACvC;IACA;IACA,cAAc,OAAO,SAAS,mBAAmB,KAAK,IAAI;GAC5D;GACA,IAAI,gBAAgB,cAAc,KAAKF,kBAAkB,WAAW;IAClE,KAAKT,IAAI,QACP,kCAAkC,KAAKS,kBAAkB,UAAU,MAAM,gBAAgB,aACzF;KAAC,OAAO,KAAKJ;KAAiB,QAAQ;IAAe,CACvD;IACA,KAAKD,QAAQ,KAAK,eAAe;GACnC;GACA,KAAKK,mBAAmB;GAGxB,KAAKI,yBAAyB;EAChC,SAAS,GAAG;GACV,KAAKb,IAAI,QAAQ,qCAAqC,QAAQ,CAAC;EACjE;CACF;CAEA,uBAAuB;EACrB,IAAI,CAAC,KAAKY,mBACR;EAEF,KAAKA,oBAAoB;EACzB,IAAI,KAAKD,iBAAiB,KAAKH,2BAC7B,KAAKR,IAAI,QACP,6DACA;GAAC,OAAO,KAAKK;GAAiB,QAAQ,KAAKI;EAAgB,CAC7D;EAEF,KAAKE,eAAe,KAAK,SAAS;EAClC,KAAKA,gBAAgB,KAAA;EACrB,aAAa,KAAKH,yBAAyB;EAC3C,KAAKA,4BAA4B,KAAA;CACnC;AACF;AAEA,SAAgB,6BAA6B,EAC3C,WACA,UACA,UAKS;CACT,IAAI,aAAa,KAAA,KAAa,WAAW,KAAA,GACvC,OAAO;CAGT,MAAM,MAAM,IAAI,IAAI,SAAS;CAC7B,IAAI,IAAI,aAAa,OACnB,OAAO;CAGT,IAAI,aAAa,KAAA,KAAa,CAAC,IAAI,aAAa,IAAI,UAAU,GAC5D,IAAI,aAAa,IAAI,YAAY,QAAQ;CAE3C,IAAI,WAAW,KAAA,KAAa,CAAC,IAAI,aAAa,IAAI,QAAQ,GACxD,IAAI,aAAa,IAAI,UAAU,MAAM;CAEvC,OAAO,IAAI,SAAS;AACtB"}
|
|
1
|
+
{"version":3,"file":"vfs-watermark-poller.js","names":["#lc","#state","#pollerConfig","#readLocalWatermark","#stream","#localWatermark","#disableRemotePoller","#localPollTimer","#remotePollerBackoffTimer","#remoteWatermark","#enableRemotePoller","#remotePoller","#shouldPollRemote","#remotePollerBackoffMs","#readRemotePollerLine"],"sources":["../../../../../../zero-cache/src/services/change-streamer/vfs-watermark-poller.ts"],"sourcesContent":["import {spawn, type ChildProcess} from 'node:child_process';\nimport {createInterface} from 'node:readline';\nimport type {LogContext} from '@rocicorp/logger';\nimport {assert} from '../../../../shared/src/asserts.ts';\nimport * as v from '../../../../shared/src/valita.ts';\nimport {Database, type Statement} from '../../../../zqlite/src/db.ts';\nimport type {Source} from '../../types/streams.ts';\nimport {Subscription} from '../../types/subscription.ts';\nimport {RunningState} from '../running-state.ts';\nimport type {BackedUpWatermark} from './backup-monitor.ts';\n\nconst LOCAL_CHECK_INTERVAL_MS = 1_000;\n\ntype LocalWatermark = {\n stateVersion: string;\n writeTimeMs: number;\n};\n\ntype RemoteWatermark = {\n watermark: string;\n writeTimeMs: number;\n backupTimeMs: number;\n};\n\nexport type VfsPollerConfig = {\n executable: string;\n remotePollIntervalMs: number;\n remoteQueryTimeoutMs?: number;\n backupURL: string;\n region?: string | undefined;\n endpoint?: string | undefined;\n logLevel?: string;\n logFormat?: string;\n};\n\nconst queryResultRowSchema = v.object({\n stateVersion: v.string(),\n writeTimeMs: v.number(),\n});\n\nconst remoteQueryMetadataSchema = v.object({\n litestream_time: v\n .string()\n .map(str => new Date(str).getTime())\n .optional(),\n});\n\n// Expected format of each line of stdout from the vfs-query, e.g.\n// {\"query_result\":[{\"stateVersion\":\"51y50ko\",\"writeTimeMs\":1787259951121}],\"metadata\":{\"litestream_time\":\"2026-08-20T21:05:57.36439Z\"}}\n// {\"query_result\":[{\"stateVersion\":\"51y7360\",\"writeTimeMs\":1787259966137}],\"metadata\":{\"litestream_time\":\"2026-08-20T21:06:12.384531Z\"}}\nconst remoteQueryLineSchema = v.object({\n query_result: v.tuple([queryResultRowSchema]),\n metadata: remoteQueryMetadataSchema,\n});\n\nconst INITIAL_BACKOFF_MS = 2_000;\nconst MAX_BACKOFF_MS = 30_000;\n\nexport class VfsWatermarkPoller {\n readonly #lc: LogContext;\n readonly #state = new RunningState('vfs-watermark-poller');\n readonly #pollerConfig: VfsPollerConfig;\n readonly #readLocalWatermark: Statement;\n\n readonly #stream: Subscription<BackedUpWatermark>;\n\n #localWatermark: LocalWatermark;\n #remoteWatermark: RemoteWatermark | undefined;\n #localPollTimer: NodeJS.Timeout | undefined;\n\n #shouldPollRemote = false;\n\n #remotePoller: ChildProcess | undefined;\n #remotePollerBackoffMs = INITIAL_BACKOFF_MS;\n #remotePollerBackoffTimer: NodeJS.Timeout | undefined;\n\n constructor(\n lc: LogContext,\n replicaFile: string,\n pollerConfig: VfsPollerConfig,\n ) {\n this.#lc = lc.withContext('component', 'vfs-watermark-poller');\n this.#pollerConfig = pollerConfig;\n\n const db = new Database(this.#lc, replicaFile, {readonly: true});\n this.#readLocalWatermark = db.prepare(\n `SELECT stateVersion, writeTimeMs FROM \"_zero.replicationState\"`,\n );\n // Purely to type the member variable as always defined\n this.#localWatermark = v.parse(\n this.#readLocalWatermark.get(),\n queryResultRowSchema,\n );\n this.#stream = Subscription.create<BackedUpWatermark>({\n cleanup: () => {\n this.#disableRemotePoller();\n clearInterval(this.#localPollTimer);\n clearTimeout(this.#remotePollerBackoffTimer);\n this.#state.stop(this.#lc);\n db.close();\n },\n });\n }\n\n /**\n * Starts polling for watermarks and pushes new ones to returned stream.\n * Must only be called once.\n */\n start(): Source<BackedUpWatermark> {\n assert(this.#localPollTimer === undefined, `Already called start()`);\n this.#lc.info?.(`monitoring backups at ${this.#pollerConfig.backupURL}`);\n this.#localPollTimer = setInterval(\n this.checkLocalWatermark,\n LOCAL_CHECK_INTERVAL_MS,\n );\n this.checkLocalWatermark();\n return this.#stream;\n }\n\n // Exported for testing\n readonly checkLocalWatermark = () => {\n try {\n this.#localWatermark = v.parse(\n this.#readLocalWatermark.get(),\n queryResultRowSchema,\n );\n } catch (e) {\n this.#lc.warn?.(`unable to read local watermark`, e);\n return;\n }\n if (\n this.#localWatermark.stateVersion !== this.#remoteWatermark?.watermark\n ) {\n this.#enableRemotePoller();\n } else {\n this.#disableRemotePoller();\n }\n };\n\n #enableRemotePoller() {\n if (this.#remotePoller || this.#remotePollerBackoffTimer) {\n return; // already polling or backing off\n }\n this.#shouldPollRemote = true;\n if (this.#remoteWatermark) {\n this.#lc.debug?.(\n `local and remote watermarks differ. resuming backup polling`,\n {local: this.#localWatermark, remote: this.#remoteWatermark},\n );\n }\n const child = spawn(\n this.#pollerConfig.executable,\n [\n '--replica-url',\n buildLitestreamVfsReplicaURL(this.#pollerConfig),\n '--query',\n `SELECT stateVersion, writeTimeMs FROM \"_zero.replicationState\"`,\n '--remote-poll-interval',\n `${this.#pollerConfig.remotePollIntervalMs}ms`,\n ...(this.#pollerConfig.logLevel\n ? ['--log-level', this.#pollerConfig.logLevel]\n : []),\n ...(this.#pollerConfig.logFormat\n ? ['--log-format', this.#pollerConfig.logFormat]\n : []),\n ...(this.#pollerConfig.remoteQueryTimeoutMs\n ? ['--query-timeout', `${this.#pollerConfig.remoteQueryTimeoutMs}ms`]\n : []),\n ],\n {stdio: ['pipe', 'pipe', 'inherit']},\n )\n .on('error', err => {\n this.#lc.error?.(`received error from vfs-query process`, err);\n })\n .on('close', (code, signal) => {\n if (this.#remotePoller !== child) {\n // This child was already stopped by #disableRemotePoller(), and a\n // replacement may have been spawned since. Its exit must neither\n // untrack that replacement nor schedule a respawn next to it.\n return;\n }\n this.#remotePoller = undefined;\n if (this.#shouldPollRemote) {\n if (code !== 0) {\n this.#lc.warn?.(\n `vfs-query exited with ${code}, ${signal}. retrying in ${this.#remotePollerBackoffMs}ms`,\n );\n }\n if (!this.#remotePollerBackoffTimer) {\n this.#remotePollerBackoffTimer = setTimeout(() => {\n this.#remotePollerBackoffTimer = undefined;\n if (this.#shouldPollRemote) {\n this.#enableRemotePoller();\n }\n }, this.#remotePollerBackoffMs);\n this.#remotePollerBackoffMs = Math.min(\n this.#remotePollerBackoffMs * 2,\n MAX_BACKOFF_MS,\n );\n }\n }\n });\n createInterface({input: child.stdout}).on(\n 'line',\n this.#readRemotePollerLine,\n );\n this.#remotePoller = child;\n }\n\n readonly #readRemotePollerLine = (line: string) => {\n try {\n const update = v.parse(\n JSON.parse(line),\n remoteQueryLineSchema,\n 'passthrough',\n );\n const [{stateVersion: watermark, writeTimeMs}] = update.query_result;\n const remoteWatermark: RemoteWatermark = {\n watermark,\n writeTimeMs,\n backupTimeMs: update.metadata.litestream_time ?? Date.now(),\n };\n if (remoteWatermark.watermark !== this.#remoteWatermark?.watermark) {\n this.#lc.debug?.(\n `backup watermark advanced from ${this.#remoteWatermark?.watermark} to ${remoteWatermark.watermark}`,\n {local: this.#localWatermark, remote: remoteWatermark},\n );\n this.#stream.push(remoteWatermark);\n }\n this.#remoteWatermark = remoteWatermark;\n\n // Reset backoff on successful output from the poller\n this.#remotePollerBackoffMs = INITIAL_BACKOFF_MS;\n } catch (e) {\n this.#lc.error?.(`unable to parse vfs-query output: ${line}`, e);\n }\n };\n\n #disableRemotePoller() {\n if (!this.#shouldPollRemote) {\n return;\n }\n this.#shouldPollRemote = false;\n if (this.#remotePoller || this.#remotePollerBackoffTimer) {\n this.#lc.debug?.(\n `local and remote watermarks match. pausing backup polling`,\n {local: this.#localWatermark, remote: this.#remoteWatermark},\n );\n }\n this.#remotePoller?.kill('SIGTERM');\n this.#remotePoller = undefined;\n clearTimeout(this.#remotePollerBackoffTimer);\n this.#remotePollerBackoffTimer = undefined;\n }\n}\n\nexport function buildLitestreamVfsReplicaURL({\n backupURL,\n endpoint,\n region,\n}: {\n readonly backupURL: string;\n readonly endpoint?: string | undefined;\n readonly region?: string | undefined;\n}): string {\n if (endpoint === undefined && region === undefined) {\n return backupURL;\n }\n\n const url = new URL(backupURL);\n if (url.protocol !== 's3:') {\n return backupURL;\n }\n\n if (endpoint !== undefined && !url.searchParams.has('endpoint')) {\n url.searchParams.set('endpoint', endpoint);\n }\n if (region !== undefined && !url.searchParams.has('region')) {\n url.searchParams.set('region', region);\n }\n return url.toString();\n}\n"],"mappings":";;;;;;;;AAWA,IAAM,0BAA0B;AAwBhC,IAAM,uBAAuB,eAAE,OAAO;CACpC,cAAc,eAAE,OAAO;CACvB,aAAa,eAAE,OAAO;AACxB,CAAC;AAED,IAAM,4BAA4B,eAAE,OAAO,EACzC,iBAAiB,eACd,OAAO,EACP,KAAI,QAAO,IAAI,KAAK,GAAG,EAAE,QAAQ,CAAC,EAClC,SAAS,EACd,CAAC;AAKD,IAAM,wBAAwB,eAAE,OAAO;CACrC,cAAc,eAAE,MAAM,CAAC,oBAAoB,CAAC;CAC5C,UAAU;AACZ,CAAC;AAED,IAAM,qBAAqB;AAC3B,IAAM,iBAAiB;AAEvB,IAAa,qBAAb,MAAgC;CAC9B;CACA,SAAkB,IAAI,aAAa,sBAAsB;CACzD;CACA;CAEA;CAEA;CACA;CACA;CAEA,oBAAoB;CAEpB;CACA,yBAAyB;CACzB;CAEA,YACE,IACA,aACA,cACA;EACA,KAAKA,MAAM,GAAG,YAAY,aAAa,sBAAsB;EAC7D,KAAKE,gBAAgB;EAErB,MAAM,KAAK,IAAI,SAAS,KAAKF,KAAK,aAAa,EAAC,UAAU,KAAI,CAAC;EAC/D,KAAKG,sBAAsB,GAAG,QAC5B,gEACF;EAEA,KAAKE,kBAAkB,MACrB,KAAKF,oBAAoB,IAAI,GAC7B,oBACF;EACA,KAAKC,UAAU,aAAa,OAA0B,EACpD,eAAe;GACb,KAAKE,qBAAqB;GAC1B,cAAc,KAAKC,eAAe;GAClC,aAAa,KAAKC,yBAAyB;GAC3C,KAAKP,OAAO,KAAK,KAAKD,GAAG;GACzB,GAAG,MAAM;EACX,EACF,CAAC;CACH;;;;;CAMA,QAAmC;EACjC,OAAO,KAAKO,oBAAoB,KAAA,GAAW,wBAAwB;EACnE,KAAKP,IAAI,OAAO,yBAAyB,KAAKE,cAAc,WAAW;EACvE,KAAKK,kBAAkB,YACrB,KAAK,qBACL,uBACF;EACA,KAAK,oBAAoB;EACzB,OAAO,KAAKH;CACd;CAGA,4BAAqC;EACnC,IAAI;GACF,KAAKC,kBAAkB,MACrB,KAAKF,oBAAoB,IAAI,GAC7B,oBACF;EACF,SAAS,GAAG;GACV,KAAKH,IAAI,OAAO,kCAAkC,CAAC;GACnD;EACF;EACA,IACE,KAAKK,gBAAgB,iBAAiB,KAAKI,kBAAkB,WAE7D,KAAKC,oBAAoB;OAEzB,KAAKJ,qBAAqB;CAE9B;CAEA,sBAAsB;EACpB,IAAI,KAAKK,iBAAiB,KAAKH,2BAC7B;EAEF,KAAKI,oBAAoB;EACzB,IAAI,KAAKH,kBACP,KAAKT,IAAI,QACP,+DACA;GAAC,OAAO,KAAKK;GAAiB,QAAQ,KAAKI;EAAgB,CAC7D;EAEF,MAAM,QAAQ,MACZ,KAAKP,cAAc,YACnB;GACE;GACA,6BAA6B,KAAKA,aAAa;GAC/C;GACA;GACA;GACA,GAAG,KAAKA,cAAc,qBAAqB;GAC3C,GAAI,KAAKA,cAAc,WACnB,CAAC,eAAe,KAAKA,cAAc,QAAQ,IAC3C,CAAC;GACL,GAAI,KAAKA,cAAc,YACnB,CAAC,gBAAgB,KAAKA,cAAc,SAAS,IAC7C,CAAC;GACL,GAAI,KAAKA,cAAc,uBACnB,CAAC,mBAAmB,GAAG,KAAKA,cAAc,qBAAqB,GAAG,IAClE,CAAC;EACP,GACA,EAAC,OAAO;GAAC;GAAQ;GAAQ;EAAS,EAAC,CACrC,EACG,GAAG,UAAS,QAAO;GAClB,KAAKF,IAAI,QAAQ,yCAAyC,GAAG;EAC/D,CAAC,EACA,GAAG,UAAU,MAAM,WAAW;GAC7B,IAAI,KAAKW,kBAAkB,OAIzB;GAEF,KAAKA,gBAAgB,KAAA;GACrB,IAAI,KAAKC,mBAAmB;IAC1B,IAAI,SAAS,GACX,KAAKZ,IAAI,OACP,yBAAyB,KAAK,IAAI,OAAO,gBAAgB,KAAKa,uBAAuB,GACvF;IAEF,IAAI,CAAC,KAAKL,2BAA2B;KACnC,KAAKA,4BAA4B,iBAAiB;MAChD,KAAKA,4BAA4B,KAAA;MACjC,IAAI,KAAKI,mBACP,KAAKF,oBAAoB;KAE7B,GAAG,KAAKG,sBAAsB;KAC9B,KAAKA,yBAAyB,KAAK,IACjC,KAAKA,yBAAyB,GAC9B,cACF;IACF;GACF;EACF,CAAC;EACH,gBAAgB,EAAC,OAAO,MAAM,OAAM,CAAC,EAAE,GACrC,QACA,KAAKC,qBACP;EACA,KAAKH,gBAAgB;CACvB;CAEA,yBAAkC,SAAiB;EACjD,IAAI;GACF,MAAM,SAAS,MACb,KAAK,MAAM,IAAI,GACf,uBACA,aACF;GACA,MAAM,CAAC,EAAC,cAAc,WAAW,iBAAgB,OAAO;GACxD,MAAM,kBAAmC;IACvC;IACA;IACA,cAAc,OAAO,SAAS,mBAAmB,KAAK,IAAI;GAC5D;GACA,IAAI,gBAAgB,cAAc,KAAKF,kBAAkB,WAAW;IAClE,KAAKT,IAAI,QACP,kCAAkC,KAAKS,kBAAkB,UAAU,MAAM,gBAAgB,aACzF;KAAC,OAAO,KAAKJ;KAAiB,QAAQ;IAAe,CACvD;IACA,KAAKD,QAAQ,KAAK,eAAe;GACnC;GACA,KAAKK,mBAAmB;GAGxB,KAAKI,yBAAyB;EAChC,SAAS,GAAG;GACV,KAAKb,IAAI,QAAQ,qCAAqC,QAAQ,CAAC;EACjE;CACF;CAEA,uBAAuB;EACrB,IAAI,CAAC,KAAKY,mBACR;EAEF,KAAKA,oBAAoB;EACzB,IAAI,KAAKD,iBAAiB,KAAKH,2BAC7B,KAAKR,IAAI,QACP,6DACA;GAAC,OAAO,KAAKK;GAAiB,QAAQ,KAAKI;EAAgB,CAC7D;EAEF,KAAKE,eAAe,KAAK,SAAS;EAClC,KAAKA,gBAAgB,KAAA;EACrB,aAAa,KAAKH,yBAAyB;EAC3C,KAAKA,4BAA4B,KAAA;CACnC;AACF;AAEA,SAAgB,6BAA6B,EAC3C,WACA,UACA,UAKS;CACT,IAAI,aAAa,KAAA,KAAa,WAAW,KAAA,GACvC,OAAO;CAGT,MAAM,MAAM,IAAI,IAAI,SAAS;CAC7B,IAAI,IAAI,aAAa,OACnB,OAAO;CAGT,IAAI,aAAa,KAAA,KAAa,CAAC,IAAI,aAAa,IAAI,UAAU,GAC5D,IAAI,aAAa,IAAI,YAAY,QAAQ;CAE3C,IAAI,WAAW,KAAA,KAAa,CAAC,IAAI,aAAa,IAAI,QAAQ,GACxD,IAAI,aAAa,IAAI,UAAU,MAAM;CAEvC,OAAO,IAAI,SAAS;AACtB"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"life-cycle.d.ts","sourceRoot":"","sources":["../../../../../zero-cache/src/services/life-cycle.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,mBAAmB,EAAC,MAAM,WAAW,CAAC;AAEnD,OAAO,KAAK,EAAC,YAAY,EAAC,MAAM,QAAQ,CAAC;AACzC,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;
|
|
1
|
+
{"version":3,"file":"life-cycle.d.ts","sourceRoot":"","sources":["../../../../../zero-cache/src/services/life-cycle.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,mBAAmB,EAAC,MAAM,WAAW,CAAC;AAEnD,OAAO,KAAK,EAAC,YAAY,EAAC,MAAM,QAAQ,CAAC;AACzC,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AASjD,OAAO,EAEL,KAAK,UAAU,EACf,KAAK,MAAM,EACZ,MAAM,uBAAuB,CAAC;AAE/B,OAAO,KAAK,EAAC,gBAAgB,EAAC,MAAM,cAAc,CAAC;AAEnD;;;;;;;;;;;;GAYG;AACH,MAAM,MAAM,UAAU,GAAG,aAAa,GAAG,YAAY,CAAC;AAEtD,eAAO,MAAM,iBAAiB,YAAI,SAAS,EAAE,QAAQ,CAAU,CAAC;AAChE,eAAO,MAAM,iBAAiB,YAAI,SAAS,EAAE,SAAS,CAAU,CAAC;AA8CjE,wBAAgB,uBAAuB,CAAC,UAAU,EAAE,MAAM,QAEzD;AAaD,eAAO,MAAM,8BAA8B,KAAK,CAAC;AAKjD,eAAO,MAAM,+BAA+B,KAAK,CAAC;AAElD;;;GAGG;AACH,qBAAa,cAAc;;IAWzB,YAAY,EAAE,EAAE,UAAU,EAAE,IAAI,EAAE,YAAY,EAyC7C;IAEK,QAAQ,kBAKb;IAuBD,aAAa,CAAC,IAAI,EAAE,UAAU,EAAE,IAAI,EAAE,UAAU,EAAE,IAAI,EAAE,MAAM,QAsB7D;IAKD,SAAS,CAAC,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,UAAU,EAAE,IAAI,EAAE,MAAM,GAAG,MAAM,CAwBhE;IAED,YAAY,IAAI,MAAM,EAAE,CAEvB;IAEK,eAAe,kBAEpB;IAED,eAAe,CAAC,GAAG,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,QAGzC;CA8EF;AAED;;;;;;GAMG;AAEH,wBAAsB,cAAc,CAClC,EAAE,EAAE,UAAU,EACd,MAAM,EAAE,YAAY,EACpB,GAAG,QAAQ,EAAE,gBAAgB,EAAE,GAC9B,OAAO,CAAC,IAAI,CAAC,CA6Bf;AAED,wBAAsB,SAAS,CAC7B,UAAU,EAAE,UAAU,GAAG,CAAC,MAAM,UAAU,CAAC,EAC3C,GAAG,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,iBAoCzB;AAED;;;;;;;;;;GAUG;AACH,qBAAa,gBAAgB;;IAQ3B,YAAY,EAAE,EAAE,UAAU,EAAE,YAAY,EAAE,MAAM,EAG/C;IAED,WAAW,CAAC,UAAU,EAAE,mBAAmB,QAgB1C;IAsCD,IAAI,SAKH;CACF"}
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { AbortError } from "../../../shared/src/abort-error.js";
|
|
1
2
|
import { singleProcessMode } from "../types/processes.js";
|
|
2
3
|
import { LONG_DURATION_HISTOGRAM_BOUNDARIES_S, getOrCreateHistogram } from "../observability/metrics.js";
|
|
3
4
|
import { ConfigurationError } from "../types/configuration-error.js";
|
|
@@ -191,7 +192,8 @@ async function exitAfter(logContext, run) {
|
|
|
191
192
|
lc.error?.(`exiting with configuration error: ${String(e)}`, e);
|
|
192
193
|
process.exit(0);
|
|
193
194
|
}
|
|
194
|
-
lc.
|
|
195
|
+
if (e instanceof AbortError) lc.warn?.(`exiting after abort: ${String(e)}`, e);
|
|
196
|
+
else lc.error?.(`exiting with error: ${String(e)}`, e);
|
|
195
197
|
try {
|
|
196
198
|
logLastChanceSQLiteCorruptionDiagnostics(lc, e);
|
|
197
199
|
} catch (diagnosticError) {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"life-cycle.js","names":["#lc","#userFacing","#all","#exitImpl","#start","#ready","#startDrain","#kill","#exit","#drainStart","#runningState","#onExit","#initializing","#nextID","#stopInterval","#lastHeartbeat","#checkIntervalTimer","#checkStopInterval","#checkImmediateTimer"],"sources":["../../../../../zero-cache/src/services/life-cycle.ts"],"sourcesContent":["import type {IncomingHttpHeaders} from 'node:http';\nimport {pid} from 'node:process';\nimport type {EventEmitter} from 'stream';\nimport type {LogContext} from '@rocicorp/logger';\nimport {resolver} from '@rocicorp/resolver';\nimport {logLastChanceSQLiteCorruptionDiagnostics} from '../db/sqlite-corruption.ts';\nimport {\n getOrCreateHistogram,\n LONG_DURATION_HISTOGRAM_BOUNDARIES_S,\n} from '../observability/metrics.ts';\nimport {ConfigurationError} from '../types/configuration-error.ts';\nimport {\n singleProcessMode,\n type Subprocess,\n type Worker,\n} from '../types/processes.ts';\nimport {RunningState} from './running-state.ts';\nimport type {SingletonService} from './service.ts';\n\n/**\n * * `user-facing` workers serve external requests and are the first to\n * receive a `SIGTERM` or `SIGINT` signal for graceful shutdown.\n *\n * * `supporting` workers support `user-facing` workers and are sent\n * the `SIGTERM` signal only after all `user-facing` workers have\n * exited.\n *\n * For other kill signals, such as `SIGQUIT` and `SIGABRT`, all workers\n * are stopped without draining. `SIGQUIT` is used to represent an\n * intentional shutdown (for which draining is not beneficial), whereas\n * `SIGABRT` is used for unexpected process exits.\n */\nexport type WorkerType = 'user-facing' | 'supporting';\n\nexport const GRACEFUL_SHUTDOWN = ['SIGTERM', 'SIGINT'] as const;\nexport const FORCEFUL_SHUTDOWN = ['SIGQUIT', 'SIGABRT'] as const;\n\ntype GracefulShutdownSignal = (typeof GRACEFUL_SHUTDOWN)[number];\n\nfunction workerStartupMetricName(name: string) {\n if (name === 'zero-cache') {\n return undefined;\n }\n if (name.startsWith('change-streamer')) {\n return 'change_streamer';\n }\n if (name.startsWith('replicator')) {\n if (name.includes('(backup)')) {\n return 'backup_replicator';\n }\n if (name.includes('(serving-copy)')) {\n return 'serving_copy_replicator';\n }\n if (name.includes('(serving)')) {\n return 'serving_replicator';\n }\n return 'other';\n }\n if (name.startsWith('syncer')) {\n return 'syncer';\n }\n if (name.startsWith('reaper')) {\n return 'reaper';\n }\n if (name.startsWith('shadow-syncer')) {\n return 'shadow_syncer';\n }\n if (name.startsWith('mutator')) {\n return 'mutator';\n }\n return 'other';\n}\n\nfunction startupDuration() {\n return getOrCreateHistogram('server', 'startup_duration', {\n description: 'Duration from starting zero-cache to its ready signal.',\n unit: 's',\n bucketBoundaries: LONG_DURATION_HISTOGRAM_BOUNDARIES_S,\n });\n}\n\nexport function recordStartupDurationMs(durationMs: number) {\n startupDuration().recordMs(durationMs, {component: 'dispatcher'});\n}\n\nfunction workerStartupDuration() {\n return getOrCreateHistogram('server', 'worker_startup_duration', {\n description: 'Duration from starting a worker to its ready signal.',\n unit: 's',\n bucketBoundaries: LONG_DURATION_HISTOGRAM_BOUNDARIES_S,\n });\n}\n\n// An internal error code used to indicate that a message has already been\n// logged at level ERRROR. When a process exits with this error code, the\n// parent process logs the exit at level WARN instead of ERROR.\nexport const UNHANDLED_EXCEPTION_ERROR_CODE = 13;\n\n// An internal error code used to indicate that the server should exit\n// without draining (e.g. due to a supporting worker get a signal to shut\n// down), but the exit is otherwise intentional.\nexport const INTENTIONAL_SHUTDOWN_ERROR_CODE = 14;\n\n/**\n * Handles readiness, termination signals, and coordination of graceful\n * shutdown.\n */\nexport class ProcessManager {\n readonly #lc: LogContext;\n readonly #userFacing = new Set<Subprocess>();\n readonly #all = new Set<Subprocess>();\n readonly #exitImpl: (code: number) => never;\n readonly #start = Date.now();\n readonly #ready: Promise<void>[] = [];\n\n #runningState = new RunningState('process-manager');\n #drainStart = 0;\n\n constructor(lc: LogContext, proc: EventEmitter) {\n this.#lc = lc.withContext('component', 'process-manager');\n\n // Propagate `SIGTERM` and `SIGINT` to all user-facing workers,\n // initiating a graceful shutdown. The parent process will\n // exit once all user-facing workers have exited ...\n for (const signal of GRACEFUL_SHUTDOWN) {\n proc.on(signal, () => this.#startDrain(signal));\n }\n\n // ... which will result in sending `SIGTERM` to the remaining workers.\n proc.on('exit', code =>\n this.#kill(\n this.#all,\n code === 0\n ? 'SIGTERM' // graceful, drained shutdown\n : code === INTENTIONAL_SHUTDOWN_ERROR_CODE\n ? 'SIGQUIT' // intentional abort without drain\n : 'SIGABRT', // unintentional shutdown, alertable error\n ),\n );\n\n // For other (catchable) kill signals, exit with a non-zero error code\n // to send a `SIGQUIT` (intentional shutdown) or `SIGABRT` (unexpected\n // shutdown) to all workers. For these signals, workers are stopped\n // immediately without draining, since there is no merit to slowly draining\n // when supporting workers have stopped.\n //\n // The logic for handling these signals is in `runUntilKilled()`.\n for (const signal of FORCEFUL_SHUTDOWN) {\n proc.on(signal, () =>\n this.#exit(signal === 'SIGQUIT' ? INTENTIONAL_SHUTDOWN_ERROR_CODE : -1),\n );\n }\n\n this.#exitImpl = (code: number) => {\n if (singleProcessMode()) {\n return proc.emit('exit', code) as never; // For unit / integration tests.\n }\n process.exit(code);\n };\n }\n\n async shutdown() {\n if (this.#drainStart === 0) {\n this.#startDrain(GRACEFUL_SHUTDOWN[0]);\n }\n await this.#runningState.stopped();\n }\n\n #exit(code: number) {\n this.#lc.info?.('exiting with code', code);\n this.#runningState.stop(this.#lc);\n void this.#lc.flush().finally(() => this.#exitImpl(code));\n }\n\n #startDrain(signal: GracefulShutdownSignal) {\n if (this.#all.size === 0) {\n // Shutdown if a signal is received before any subprocesses are added.\n this.#lc.info?.(`exiting on ${signal}`);\n this.#exit(0);\n }\n this.#lc.info?.(`initiating drain (${signal})`);\n this.#drainStart = Date.now();\n if (this.#userFacing.size) {\n this.#kill(this.#userFacing, signal);\n } else {\n this.#kill(this.#all, signal);\n }\n }\n\n addSubprocess(proc: Subprocess, type: WorkerType, name: string) {\n if (type === 'user-facing') {\n this.#userFacing.add(proc);\n }\n this.#all.add(proc);\n\n let isOpen = true;\n proc.on('close', (code, signal) => {\n isOpen = false;\n this.#onExit(code, signal, null, type, name, proc);\n });\n\n // As per https://nodejs.org/api/child_process.html#event-error\n // 'error' events can happen when sending a message to a child process\n // fails. This is not really an error when the server is shutting down,\n // so log any post-close errors at 'warn'.\n proc.on('error', err =>\n this.#lc[!isOpen || this.#drainStart > 0 ? 'warn' : 'error']?.(\n `error from ${name} ${proc.pid}`,\n err,\n ),\n );\n }\n\n readonly #initializing = new Map<number, string>();\n #nextID = 0;\n\n addWorker(worker: Worker, type: WorkerType, name: string): Worker {\n this.addSubprocess(worker, type, name);\n\n const start = performance.now();\n const id = ++this.#nextID;\n this.#initializing.set(id, name);\n const {promise, resolve} = resolver();\n this.#ready.push(promise);\n\n worker.onceMessageType('ready', () => {\n const elapsed = performance.now() - start;\n const workerMetricName = workerStartupMetricName(name);\n if (workerMetricName) {\n workerStartupDuration().recordMs(elapsed, {\n worker: workerMetricName,\n type,\n });\n }\n this.#lc.debug?.(`${name} ready (${Date.now() - this.#start} ms)`);\n this.#initializing.delete(id);\n resolve();\n });\n\n return worker;\n }\n\n initializing(): string[] {\n return [...this.#initializing.values()];\n }\n\n async allWorkersReady() {\n await Promise.all(this.#ready);\n }\n\n logErrorAndExit(err: unknown, name: string) {\n // only accessible by the main (i.e. user-facing) process.\n this.#onExit(-1, null, err, 'user-facing', name, undefined);\n }\n\n #onExit(\n code: number,\n sig: NodeJS.Signals | null,\n err: unknown,\n type: WorkerType,\n name: string,\n worker: Subprocess | undefined,\n ) {\n // Remove the worker from maps to avoid attempting to send more signals to it.\n if (worker) {\n this.#userFacing.delete(worker);\n this.#all.delete(worker);\n }\n\n const pid = worker?.pid ?? process.pid;\n\n if (type === 'supporting') {\n // Supporting workers like the replication-manager shut down without a\n // drain signal when receiving protocol-specific instructions (like auto\n // reset). In this case, a special error code is used to signal that the\n // server should be shut down without draining, but it is otherwise not\n // considered an unexpected/alertable error.\n if (code === 0 && (this.#drainStart === 0 || this.#userFacing.size > 0)) {\n code = INTENTIONAL_SHUTDOWN_ERROR_CODE;\n }\n const log =\n code === 0 || code === INTENTIONAL_SHUTDOWN_ERROR_CODE\n ? 'info'\n : 'warn';\n this.#lc[log]?.(`${name} (${pid}) exited with code (${code})`, err ?? '');\n return this.#exit(code);\n }\n\n const log =\n code === 0 || code === INTENTIONAL_SHUTDOWN_ERROR_CODE\n ? 'info'\n : this.#drainStart > 0 || code === UNHANDLED_EXCEPTION_ERROR_CODE\n ? 'warn'\n : 'error';\n this.#lc[log]?.(\n sig\n ? `${name} (${pid}) killed with (${sig})`\n : `${name} (${pid}) exited with code (${code})`,\n err ?? '',\n );\n\n // user-facing workers exited or finished draining.\n if (this.#userFacing.size === 0) {\n this.#lc.info?.(\n this.#drainStart\n ? `all user-facing workers drained (${\n Date.now() - this.#drainStart\n } ms)`\n : `all user-facing workers exited`,\n );\n return this.#exit(this.#drainStart ? 0 : code || -1);\n }\n\n if (this.#drainStart === 0) {\n // If a user-facing worker exits without receiving a drain signal,\n // shutdown the server.\n return this.#exit(code || -1);\n }\n\n return undefined;\n }\n\n #kill(workers: Iterable<Subprocess>, signal: NodeJS.Signals) {\n for (const worker of workers) {\n try {\n worker.kill(signal);\n } catch (e) {\n this.#lc.error?.(e);\n }\n }\n }\n}\n\n/**\n * Runs the specified services, stopping them on `SIGTERM` or `SIGINT` with\n * an optional {@link SingletonService.drain drain()}, or stopping them\n * without draining for `SIGQUIT`.\n *\n * @returns a Promise that resolves/rejects when any of the services stops/throws.\n */\n\nexport async function runUntilKilled(\n lc: LogContext,\n parent: EventEmitter,\n ...services: SingletonService[]\n): Promise<void> {\n if (services.length === 0) {\n return;\n }\n for (const signal of [...GRACEFUL_SHUTDOWN, ...FORCEFUL_SHUTDOWN]) {\n parent.once(signal, () => {\n const GRACEFUL_SIGNALS = GRACEFUL_SHUTDOWN as readonly NodeJS.Signals[];\n\n services.forEach(async svc => {\n if (GRACEFUL_SIGNALS.includes(signal) && svc.drain) {\n lc.info?.(`draining ${svc.constructor.name} ${svc.id} (${signal})`);\n await svc.drain();\n }\n lc.info?.(`stopping ${svc.constructor.name} ${svc.id} (${signal})`);\n await svc.stop();\n });\n });\n }\n\n try {\n // Run all services and resolve when any of them stops.\n const svc = await Promise.race(\n services.map(svc => svc.run().then(() => svc)),\n );\n lc.info?.(`${svc.constructor.name} (${svc.id}) stopped`);\n } catch (e) {\n lc.error?.(`exiting on error`, e);\n throw e;\n }\n}\n\nexport async function exitAfter(\n logContext: LogContext | (() => LogContext),\n run: () => Promise<void>,\n) {\n const getLogContext =\n typeof logContext === 'function' ? logContext : () => logContext;\n try {\n await run();\n const lc = getLogContext();\n lc.info?.(`pid ${pid} exiting normally`);\n process.exit(0);\n } catch (e) {\n const lc = getLogContext();\n if (e instanceof ConfigurationError) {\n lc.error?.(`exiting with configuration error: ${String(e)}`, e);\n process.exit(0);\n }\n lc.error?.(`exiting with error: ${String(e)}`, e);\n try {\n logLastChanceSQLiteCorruptionDiagnostics(lc, e);\n } catch (diagnosticError) {\n lc.error?.('SQLite corruption last-chance diagnostic failed', {\n error: diagnosticError,\n });\n }\n try {\n await lc.flush();\n } finally {\n process.exit(-1);\n }\n }\n}\n\n/**\n * The HeartbeatMonitor monitors the cadence heartbeats (e.g. \"/keepalive\"\n * health checks made to HttpServices) that signal that the server\n * should continue processing requests. When a configurable `stopInterval`\n * elapses without receiving these heartbeats, the monitor initiates a\n * graceful shutdown of the server. This works with common load balancing\n * frameworks such as AWS Elastic Load Balancing.\n *\n * The HeartbeatMonitor is **opt-in** in that it only kicks in after it\n * starts receiving keepalives.\n */\nexport class HeartbeatMonitor {\n readonly #stopInterval: number;\n\n #lc: LogContext;\n #checkIntervalTimer: NodeJS.Timeout | undefined;\n #checkImmediateTimer: NodeJS.Immediate | undefined;\n #lastHeartbeat = 0;\n\n constructor(lc: LogContext, stopInterval: number) {\n this.#lc = lc;\n this.#stopInterval = stopInterval;\n }\n\n onHeartbeat(reqHeaders: IncomingHttpHeaders) {\n this.#lastHeartbeat = Date.now();\n if (this.#checkIntervalTimer === undefined) {\n this.#lc.info?.(\n `starting heartbeat monitor at ${\n this.#stopInterval / 1000\n } second interval`,\n reqHeaders,\n );\n // e.g. check every 5 seconds to see if it's been over 20 seconds\n // since the last heartbeat.\n this.#checkIntervalTimer = setInterval(\n this.#checkStopInterval,\n this.#stopInterval / 4,\n );\n }\n }\n\n #checkStopInterval = () => {\n // In the Node.js event loop, timers like setInterval and setTimeout\n // run *before* I/O events coming from network sockets or file reads/writes.\n // When this process gets starved of CPU resources for long periods of time,\n // for example when other processes are monopolizing all available cores,\n // pathological behavior can emerge:\n // - keepalive network request comes in, but is queued in Node internals waiting\n // for time on the event loop\n // - CPU is starved/monopolized by other processes for longer than the time\n // configured via this.#stopInterval\n // - When CPU becomes available and the event loop wakes up, this stop interval\n // check is run *before* the keepalive request is processed. The value of\n // this.#lastHeartbeat is now very stale, and erroneously triggers a shutdown\n // even though keepalive requests were about to be processed and update\n // this.#lastHeartbeat. Downtime ensues.\n //\n // To avoid this, we push the check out to a phase of the event loop *after*\n // I/O events are processed, using setImmediate():\n // https://nodejs.org/en/learn/asynchronous-work/event-loop-timers-and-nexttick#setimmediate-vs-settimeout\n //\n // This ensures we see a value for this.#lastHeartbeat that reflects\n // any keepalive requests that came in during the current event loop turn.\n this.#checkImmediateTimer = setImmediate(() => {\n this.#checkImmediateTimer = undefined;\n const timeSinceLastHeartbeat = Date.now() - this.#lastHeartbeat;\n if (timeSinceLastHeartbeat >= this.#stopInterval) {\n this.#lc.info?.(\n `last heartbeat received ${\n timeSinceLastHeartbeat / 1000\n } seconds ago. draining.`,\n );\n process.kill(process.pid, GRACEFUL_SHUTDOWN[0]);\n }\n });\n };\n\n stop() {\n clearTimeout(this.#checkIntervalTimer);\n if (this.#checkImmediateTimer) {\n clearImmediate(this.#checkImmediateTimer);\n }\n }\n}\n"],"mappings":";;;;;;;;AAkCA,IAAa,oBAAoB,CAAC,WAAW,QAAQ;AACrD,IAAa,oBAAoB,CAAC,WAAW,SAAS;AAItD,SAAS,wBAAwB,MAAc;CAC7C,IAAI,SAAS,cACX;CAEF,IAAI,KAAK,WAAW,iBAAiB,GACnC,OAAO;CAET,IAAI,KAAK,WAAW,YAAY,GAAG;EACjC,IAAI,KAAK,SAAS,UAAU,GAC1B,OAAO;EAET,IAAI,KAAK,SAAS,gBAAgB,GAChC,OAAO;EAET,IAAI,KAAK,SAAS,WAAW,GAC3B,OAAO;EAET,OAAO;CACT;CACA,IAAI,KAAK,WAAW,QAAQ,GAC1B,OAAO;CAET,IAAI,KAAK,WAAW,QAAQ,GAC1B,OAAO;CAET,IAAI,KAAK,WAAW,eAAe,GACjC,OAAO;CAET,IAAI,KAAK,WAAW,SAAS,GAC3B,OAAO;CAET,OAAO;AACT;AAEA,SAAS,kBAAkB;CACzB,OAAO,qBAAqB,UAAU,oBAAoB;EACxD,aAAa;EACb,MAAM;EACN,kBAAkB;CACpB,CAAC;AACH;AAEA,SAAgB,wBAAwB,YAAoB;CAC1D,gBAAgB,EAAE,SAAS,YAAY,EAAC,WAAW,aAAY,CAAC;AAClE;AAEA,SAAS,wBAAwB;CAC/B,OAAO,qBAAqB,UAAU,2BAA2B;EAC/D,aAAa;EACb,MAAM;EACN,kBAAkB;CACpB,CAAC;AACH;;;;;AAgBA,IAAa,iBAAb,MAA4B;CAC1B;CACA,8BAAuB,IAAI,IAAgB;CAC3C,uBAAgB,IAAI,IAAgB;CACpC;CACA,SAAkB,KAAK,IAAI;CAC3B,SAAmC,CAAC;CAEpC,gBAAgB,IAAI,aAAa,iBAAiB;CAClD,cAAc;CAEd,YAAY,IAAgB,MAAoB;EAC9C,KAAKA,MAAM,GAAG,YAAY,aAAa,iBAAiB;EAKxD,KAAK,MAAM,UAAU,mBACnB,KAAK,GAAG,cAAc,KAAKM,YAAY,MAAM,CAAC;EAIhD,KAAK,GAAG,SAAQ,SACd,KAAKC,MACH,KAAKL,MACL,SAAS,IACL,YACA,SAAA,KACE,YACA,SACR,CACF;EASA,KAAK,MAAM,UAAU,mBACnB,KAAK,GAAG,cACN,KAAKM,MAAM,WAAW,YAAA,KAA8C,EAAE,CACxE;EAGF,KAAKL,aAAa,SAAiB;GACjC,IAAI,kBAAkB,GACpB,OAAO,KAAK,KAAK,QAAQ,IAAI;GAE/B,QAAQ,KAAK,IAAI;EACnB;CACF;CAEA,MAAM,WAAW;EACf,IAAI,KAAKM,gBAAgB,GACvB,KAAKH,YAAY,kBAAkB,EAAE;EAEvC,MAAM,KAAKI,cAAc,QAAQ;CACnC;CAEA,MAAM,MAAc;EAClB,KAAKV,IAAI,OAAO,qBAAqB,IAAI;EACzC,KAAKU,cAAc,KAAK,KAAKV,GAAG;EAChC,KAAUA,IAAI,MAAM,EAAE,cAAc,KAAKG,UAAU,IAAI,CAAC;CAC1D;CAEA,YAAY,QAAgC;EAC1C,IAAI,KAAKD,KAAK,SAAS,GAAG;GAExB,KAAKF,IAAI,OAAO,cAAc,QAAQ;GACtC,KAAKQ,MAAM,CAAC;EACd;EACA,KAAKR,IAAI,OAAO,qBAAqB,OAAO,EAAE;EAC9C,KAAKS,cAAc,KAAK,IAAI;EAC5B,IAAI,KAAKR,YAAY,MACnB,KAAKM,MAAM,KAAKN,aAAa,MAAM;OAEnC,KAAKM,MAAM,KAAKL,MAAM,MAAM;CAEhC;CAEA,cAAc,MAAkB,MAAkB,MAAc;EAC9D,IAAI,SAAS,eACX,KAAKD,YAAY,IAAI,IAAI;EAE3B,KAAKC,KAAK,IAAI,IAAI;EAElB,IAAI,SAAS;EACb,KAAK,GAAG,UAAU,MAAM,WAAW;GACjC,SAAS;GACT,KAAKS,QAAQ,MAAM,QAAQ,MAAM,MAAM,MAAM,IAAI;EACnD,CAAC;EAMD,KAAK,GAAG,UAAS,QACf,KAAKX,IAAI,CAAC,UAAU,KAAKS,cAAc,IAAI,SAAS,WAClD,cAAc,KAAK,GAAG,KAAK,OAC3B,GACF,CACF;CACF;CAEA,gCAAyB,IAAI,IAAoB;CACjD,UAAU;CAEV,UAAU,QAAgB,MAAkB,MAAsB;EAChE,KAAK,cAAc,QAAQ,MAAM,IAAI;EAErC,MAAM,QAAQ,YAAY,IAAI;EAC9B,MAAM,KAAK,EAAE,KAAKI;EAClB,KAAKD,cAAc,IAAI,IAAI,IAAI;EAC/B,MAAM,EAAC,SAAS,YAAW,SAAS;EACpC,KAAKP,OAAO,KAAK,OAAO;EAExB,OAAO,gBAAgB,eAAe;GACpC,MAAM,UAAU,YAAY,IAAI,IAAI;GACpC,MAAM,mBAAmB,wBAAwB,IAAI;GACrD,IAAI,kBACF,sBAAsB,EAAE,SAAS,SAAS;IACxC,QAAQ;IACR;GACF,CAAC;GAEH,KAAKL,IAAI,QAAQ,GAAG,KAAK,UAAU,KAAK,IAAI,IAAI,KAAKI,OAAO,KAAK;GACjE,KAAKQ,cAAc,OAAO,EAAE;GAC5B,QAAQ;EACV,CAAC;EAED,OAAO;CACT;CAEA,eAAyB;EACvB,OAAO,CAAC,GAAG,KAAKA,cAAc,OAAO,CAAC;CACxC;CAEA,MAAM,kBAAkB;EACtB,MAAM,QAAQ,IAAI,KAAKP,MAAM;CAC/B;CAEA,gBAAgB,KAAc,MAAc;EAE1C,KAAKM,QAAQ,IAAI,MAAM,KAAK,eAAe,MAAM,KAAA,CAAS;CAC5D;CAEA,QACE,MACA,KACA,KACA,MACA,MACA,QACA;EAEA,IAAI,QAAQ;GACV,KAAKV,YAAY,OAAO,MAAM;GAC9B,KAAKC,KAAK,OAAO,MAAM;EACzB;EAEA,MAAM,MAAM,QAAQ,OAAO,QAAQ;EAEnC,IAAI,SAAS,cAAc;GAMzB,IAAI,SAAS,MAAM,KAAKO,gBAAgB,KAAK,KAAKR,YAAY,OAAO,IACnE,OAAA;GAEF,MAAM,MACJ,SAAS,KAAK,SAAA,KACV,SACA;GACN,KAAKD,IAAI,OAAO,GAAG,KAAK,IAAI,IAAI,sBAAsB,KAAK,IAAI,OAAO,EAAE;GACxE,OAAO,KAAKQ,MAAM,IAAI;EACxB;EAEA,MAAM,MACJ,SAAS,KAAK,SAAA,KACV,SACA,KAAKC,cAAc,KAAK,SAAA,KACtB,SACA;EACR,KAAKT,IAAI,OACP,MACI,GAAG,KAAK,IAAI,IAAI,iBAAiB,IAAI,KACrC,GAAG,KAAK,IAAI,IAAI,sBAAsB,KAAK,IAC/C,OAAO,EACT;EAGA,IAAI,KAAKC,YAAY,SAAS,GAAG;GAC/B,KAAKD,IAAI,OACP,KAAKS,cACD,oCACE,KAAK,IAAI,IAAI,KAAKA,YACnB,QACD,gCACN;GACA,OAAO,KAAKD,MAAM,KAAKC,cAAc,IAAI,QAAQ,EAAE;EACrD;EAEA,IAAI,KAAKA,gBAAgB,GAGvB,OAAO,KAAKD,MAAM,QAAQ,EAAE;CAIhC;CAEA,MAAM,SAA+B,QAAwB;EAC3D,KAAK,MAAM,UAAU,SACnB,IAAI;GACF,OAAO,KAAK,MAAM;EACpB,SAAS,GAAG;GACV,KAAKR,IAAI,QAAQ,CAAC;EACpB;CAEJ;AACF;;;;;;;;AAUA,eAAsB,eACpB,IACA,QACA,GAAG,UACY;CACf,IAAI,SAAS,WAAW,GACtB;CAEF,KAAK,MAAM,UAAU,CAAC,GAAG,mBAAmB,GAAG,iBAAiB,GAC9D,OAAO,KAAK,cAAc;EACxB,MAAM,mBAAmB;EAEzB,SAAS,QAAQ,OAAM,QAAO;GAC5B,IAAI,iBAAiB,SAAS,MAAM,KAAK,IAAI,OAAO;IAClD,GAAG,OAAO,YAAY,IAAI,YAAY,KAAK,GAAG,IAAI,GAAG,IAAI,OAAO,EAAE;IAClE,MAAM,IAAI,MAAM;GAClB;GACA,GAAG,OAAO,YAAY,IAAI,YAAY,KAAK,GAAG,IAAI,GAAG,IAAI,OAAO,EAAE;GAClE,MAAM,IAAI,KAAK;EACjB,CAAC;CACH,CAAC;CAGH,IAAI;EAEF,MAAM,MAAM,MAAM,QAAQ,KACxB,SAAS,KAAI,QAAO,IAAI,IAAI,EAAE,WAAW,GAAG,CAAC,CAC/C;EACA,GAAG,OAAO,GAAG,IAAI,YAAY,KAAK,IAAI,IAAI,GAAG,UAAU;CACzD,SAAS,GAAG;EACV,GAAG,QAAQ,oBAAoB,CAAC;EAChC,MAAM;CACR;AACF;AAEA,eAAsB,UACpB,YACA,KACA;CACA,MAAM,gBACJ,OAAO,eAAe,aAAa,mBAAmB;CACxD,IAAI;EACF,MAAM,IAAI;EAEV,cAAA,EAAG,OAAO,OAAO,IAAI,kBAAkB;EACvC,QAAQ,KAAK,CAAC;CAChB,SAAS,GAAG;EACV,MAAM,KAAK,cAAc;EACzB,IAAI,aAAa,oBAAoB;GACnC,GAAG,QAAQ,qCAAqC,OAAO,CAAC,KAAK,CAAC;GAC9D,QAAQ,KAAK,CAAC;EAChB;EACA,GAAG,QAAQ,uBAAuB,OAAO,CAAC,KAAK,CAAC;EAChD,IAAI;GACF,yCAAyC,IAAI,CAAC;EAChD,SAAS,iBAAiB;GACxB,GAAG,QAAQ,mDAAmD,EAC5D,OAAO,gBACT,CAAC;EACH;EACA,IAAI;GACF,MAAM,GAAG,MAAM;EACjB,UAAU;GACR,QAAQ,KAAK,EAAE;EACjB;CACF;AACF;;;;;;;;;;;;AAaA,IAAa,mBAAb,MAA8B;CAC5B;CAEA;CACA;CACA;CACA,iBAAiB;CAEjB,YAAY,IAAgB,cAAsB;EAChD,KAAKA,MAAM;EACX,KAAKc,gBAAgB;CACvB;CAEA,YAAY,YAAiC;EAC3C,KAAKC,iBAAiB,KAAK,IAAI;EAC/B,IAAI,KAAKC,wBAAwB,KAAA,GAAW;GAC1C,KAAKhB,IAAI,OACP,iCACE,KAAKc,gBAAgB,IACtB,mBACD,UACF;GAGA,KAAKE,sBAAsB,YACzB,KAAKC,oBACL,KAAKH,gBAAgB,CACvB;EACF;CACF;CAEA,2BAA2B;EAsBzB,KAAKI,uBAAuB,mBAAmB;GAC7C,KAAKA,uBAAuB,KAAA;GAC5B,MAAM,yBAAyB,KAAK,IAAI,IAAI,KAAKH;GACjD,IAAI,0BAA0B,KAAKD,eAAe;IAChD,KAAKd,IAAI,OACP,2BACE,yBAAyB,IAC1B,wBACH;IACA,QAAQ,KAAK,QAAQ,KAAK,kBAAkB,EAAE;GAChD;EACF,CAAC;CACH;CAEA,OAAO;EACL,aAAa,KAAKgB,mBAAmB;EACrC,IAAI,KAAKE,sBACP,eAAe,KAAKA,oBAAoB;CAE5C;AACF"}
|
|
1
|
+
{"version":3,"file":"life-cycle.js","names":["#lc","#userFacing","#all","#exitImpl","#start","#ready","#startDrain","#kill","#exit","#drainStart","#runningState","#onExit","#initializing","#nextID","#stopInterval","#lastHeartbeat","#checkIntervalTimer","#checkStopInterval","#checkImmediateTimer"],"sources":["../../../../../zero-cache/src/services/life-cycle.ts"],"sourcesContent":["import type {IncomingHttpHeaders} from 'node:http';\nimport {pid} from 'node:process';\nimport type {EventEmitter} from 'stream';\nimport type {LogContext} from '@rocicorp/logger';\nimport {resolver} from '@rocicorp/resolver';\nimport {AbortError} from '../../../shared/src/abort-error.ts';\nimport {logLastChanceSQLiteCorruptionDiagnostics} from '../db/sqlite-corruption.ts';\nimport {\n getOrCreateHistogram,\n LONG_DURATION_HISTOGRAM_BOUNDARIES_S,\n} from '../observability/metrics.ts';\nimport {ConfigurationError} from '../types/configuration-error.ts';\nimport {\n singleProcessMode,\n type Subprocess,\n type Worker,\n} from '../types/processes.ts';\nimport {RunningState} from './running-state.ts';\nimport type {SingletonService} from './service.ts';\n\n/**\n * * `user-facing` workers serve external requests and are the first to\n * receive a `SIGTERM` or `SIGINT` signal for graceful shutdown.\n *\n * * `supporting` workers support `user-facing` workers and are sent\n * the `SIGTERM` signal only after all `user-facing` workers have\n * exited.\n *\n * For other kill signals, such as `SIGQUIT` and `SIGABRT`, all workers\n * are stopped without draining. `SIGQUIT` is used to represent an\n * intentional shutdown (for which draining is not beneficial), whereas\n * `SIGABRT` is used for unexpected process exits.\n */\nexport type WorkerType = 'user-facing' | 'supporting';\n\nexport const GRACEFUL_SHUTDOWN = ['SIGTERM', 'SIGINT'] as const;\nexport const FORCEFUL_SHUTDOWN = ['SIGQUIT', 'SIGABRT'] as const;\n\ntype GracefulShutdownSignal = (typeof GRACEFUL_SHUTDOWN)[number];\n\nfunction workerStartupMetricName(name: string) {\n if (name === 'zero-cache') {\n return undefined;\n }\n if (name.startsWith('change-streamer')) {\n return 'change_streamer';\n }\n if (name.startsWith('replicator')) {\n if (name.includes('(backup)')) {\n return 'backup_replicator';\n }\n if (name.includes('(serving-copy)')) {\n return 'serving_copy_replicator';\n }\n if (name.includes('(serving)')) {\n return 'serving_replicator';\n }\n return 'other';\n }\n if (name.startsWith('syncer')) {\n return 'syncer';\n }\n if (name.startsWith('reaper')) {\n return 'reaper';\n }\n if (name.startsWith('shadow-syncer')) {\n return 'shadow_syncer';\n }\n if (name.startsWith('mutator')) {\n return 'mutator';\n }\n return 'other';\n}\n\nfunction startupDuration() {\n return getOrCreateHistogram('server', 'startup_duration', {\n description: 'Duration from starting zero-cache to its ready signal.',\n unit: 's',\n bucketBoundaries: LONG_DURATION_HISTOGRAM_BOUNDARIES_S,\n });\n}\n\nexport function recordStartupDurationMs(durationMs: number) {\n startupDuration().recordMs(durationMs, {component: 'dispatcher'});\n}\n\nfunction workerStartupDuration() {\n return getOrCreateHistogram('server', 'worker_startup_duration', {\n description: 'Duration from starting a worker to its ready signal.',\n unit: 's',\n bucketBoundaries: LONG_DURATION_HISTOGRAM_BOUNDARIES_S,\n });\n}\n\n// An internal error code used to indicate that a message has already been\n// logged at level ERRROR. When a process exits with this error code, the\n// parent process logs the exit at level WARN instead of ERROR.\nexport const UNHANDLED_EXCEPTION_ERROR_CODE = 13;\n\n// An internal error code used to indicate that the server should exit\n// without draining (e.g. due to a supporting worker get a signal to shut\n// down), but the exit is otherwise intentional.\nexport const INTENTIONAL_SHUTDOWN_ERROR_CODE = 14;\n\n/**\n * Handles readiness, termination signals, and coordination of graceful\n * shutdown.\n */\nexport class ProcessManager {\n readonly #lc: LogContext;\n readonly #userFacing = new Set<Subprocess>();\n readonly #all = new Set<Subprocess>();\n readonly #exitImpl: (code: number) => never;\n readonly #start = Date.now();\n readonly #ready: Promise<void>[] = [];\n\n #runningState = new RunningState('process-manager');\n #drainStart = 0;\n\n constructor(lc: LogContext, proc: EventEmitter) {\n this.#lc = lc.withContext('component', 'process-manager');\n\n // Propagate `SIGTERM` and `SIGINT` to all user-facing workers,\n // initiating a graceful shutdown. The parent process will\n // exit once all user-facing workers have exited ...\n for (const signal of GRACEFUL_SHUTDOWN) {\n proc.on(signal, () => this.#startDrain(signal));\n }\n\n // ... which will result in sending `SIGTERM` to the remaining workers.\n proc.on('exit', code =>\n this.#kill(\n this.#all,\n code === 0\n ? 'SIGTERM' // graceful, drained shutdown\n : code === INTENTIONAL_SHUTDOWN_ERROR_CODE\n ? 'SIGQUIT' // intentional abort without drain\n : 'SIGABRT', // unintentional shutdown, alertable error\n ),\n );\n\n // For other (catchable) kill signals, exit with a non-zero error code\n // to send a `SIGQUIT` (intentional shutdown) or `SIGABRT` (unexpected\n // shutdown) to all workers. For these signals, workers are stopped\n // immediately without draining, since there is no merit to slowly draining\n // when supporting workers have stopped.\n //\n // The logic for handling these signals is in `runUntilKilled()`.\n for (const signal of FORCEFUL_SHUTDOWN) {\n proc.on(signal, () =>\n this.#exit(signal === 'SIGQUIT' ? INTENTIONAL_SHUTDOWN_ERROR_CODE : -1),\n );\n }\n\n this.#exitImpl = (code: number) => {\n if (singleProcessMode()) {\n return proc.emit('exit', code) as never; // For unit / integration tests.\n }\n process.exit(code);\n };\n }\n\n async shutdown() {\n if (this.#drainStart === 0) {\n this.#startDrain(GRACEFUL_SHUTDOWN[0]);\n }\n await this.#runningState.stopped();\n }\n\n #exit(code: number) {\n this.#lc.info?.('exiting with code', code);\n this.#runningState.stop(this.#lc);\n void this.#lc.flush().finally(() => this.#exitImpl(code));\n }\n\n #startDrain(signal: GracefulShutdownSignal) {\n if (this.#all.size === 0) {\n // Shutdown if a signal is received before any subprocesses are added.\n this.#lc.info?.(`exiting on ${signal}`);\n this.#exit(0);\n }\n this.#lc.info?.(`initiating drain (${signal})`);\n this.#drainStart = Date.now();\n if (this.#userFacing.size) {\n this.#kill(this.#userFacing, signal);\n } else {\n this.#kill(this.#all, signal);\n }\n }\n\n addSubprocess(proc: Subprocess, type: WorkerType, name: string) {\n if (type === 'user-facing') {\n this.#userFacing.add(proc);\n }\n this.#all.add(proc);\n\n let isOpen = true;\n proc.on('close', (code, signal) => {\n isOpen = false;\n this.#onExit(code, signal, null, type, name, proc);\n });\n\n // As per https://nodejs.org/api/child_process.html#event-error\n // 'error' events can happen when sending a message to a child process\n // fails. This is not really an error when the server is shutting down,\n // so log any post-close errors at 'warn'.\n proc.on('error', err =>\n this.#lc[!isOpen || this.#drainStart > 0 ? 'warn' : 'error']?.(\n `error from ${name} ${proc.pid}`,\n err,\n ),\n );\n }\n\n readonly #initializing = new Map<number, string>();\n #nextID = 0;\n\n addWorker(worker: Worker, type: WorkerType, name: string): Worker {\n this.addSubprocess(worker, type, name);\n\n const start = performance.now();\n const id = ++this.#nextID;\n this.#initializing.set(id, name);\n const {promise, resolve} = resolver();\n this.#ready.push(promise);\n\n worker.onceMessageType('ready', () => {\n const elapsed = performance.now() - start;\n const workerMetricName = workerStartupMetricName(name);\n if (workerMetricName) {\n workerStartupDuration().recordMs(elapsed, {\n worker: workerMetricName,\n type,\n });\n }\n this.#lc.debug?.(`${name} ready (${Date.now() - this.#start} ms)`);\n this.#initializing.delete(id);\n resolve();\n });\n\n return worker;\n }\n\n initializing(): string[] {\n return [...this.#initializing.values()];\n }\n\n async allWorkersReady() {\n await Promise.all(this.#ready);\n }\n\n logErrorAndExit(err: unknown, name: string) {\n // only accessible by the main (i.e. user-facing) process.\n this.#onExit(-1, null, err, 'user-facing', name, undefined);\n }\n\n #onExit(\n code: number,\n sig: NodeJS.Signals | null,\n err: unknown,\n type: WorkerType,\n name: string,\n worker: Subprocess | undefined,\n ) {\n // Remove the worker from maps to avoid attempting to send more signals to it.\n if (worker) {\n this.#userFacing.delete(worker);\n this.#all.delete(worker);\n }\n\n const pid = worker?.pid ?? process.pid;\n\n if (type === 'supporting') {\n // Supporting workers like the replication-manager shut down without a\n // drain signal when receiving protocol-specific instructions (like auto\n // reset). In this case, a special error code is used to signal that the\n // server should be shut down without draining, but it is otherwise not\n // considered an unexpected/alertable error.\n if (code === 0 && (this.#drainStart === 0 || this.#userFacing.size > 0)) {\n code = INTENTIONAL_SHUTDOWN_ERROR_CODE;\n }\n const log =\n code === 0 || code === INTENTIONAL_SHUTDOWN_ERROR_CODE\n ? 'info'\n : 'warn';\n this.#lc[log]?.(`${name} (${pid}) exited with code (${code})`, err ?? '');\n return this.#exit(code);\n }\n\n const log =\n code === 0 || code === INTENTIONAL_SHUTDOWN_ERROR_CODE\n ? 'info'\n : this.#drainStart > 0 || code === UNHANDLED_EXCEPTION_ERROR_CODE\n ? 'warn'\n : 'error';\n this.#lc[log]?.(\n sig\n ? `${name} (${pid}) killed with (${sig})`\n : `${name} (${pid}) exited with code (${code})`,\n err ?? '',\n );\n\n // user-facing workers exited or finished draining.\n if (this.#userFacing.size === 0) {\n this.#lc.info?.(\n this.#drainStart\n ? `all user-facing workers drained (${\n Date.now() - this.#drainStart\n } ms)`\n : `all user-facing workers exited`,\n );\n return this.#exit(this.#drainStart ? 0 : code || -1);\n }\n\n if (this.#drainStart === 0) {\n // If a user-facing worker exits without receiving a drain signal,\n // shutdown the server.\n return this.#exit(code || -1);\n }\n\n return undefined;\n }\n\n #kill(workers: Iterable<Subprocess>, signal: NodeJS.Signals) {\n for (const worker of workers) {\n try {\n worker.kill(signal);\n } catch (e) {\n this.#lc.error?.(e);\n }\n }\n }\n}\n\n/**\n * Runs the specified services, stopping them on `SIGTERM` or `SIGINT` with\n * an optional {@link SingletonService.drain drain()}, or stopping them\n * without draining for `SIGQUIT`.\n *\n * @returns a Promise that resolves/rejects when any of the services stops/throws.\n */\n\nexport async function runUntilKilled(\n lc: LogContext,\n parent: EventEmitter,\n ...services: SingletonService[]\n): Promise<void> {\n if (services.length === 0) {\n return;\n }\n for (const signal of [...GRACEFUL_SHUTDOWN, ...FORCEFUL_SHUTDOWN]) {\n parent.once(signal, () => {\n const GRACEFUL_SIGNALS = GRACEFUL_SHUTDOWN as readonly NodeJS.Signals[];\n\n services.forEach(async svc => {\n if (GRACEFUL_SIGNALS.includes(signal) && svc.drain) {\n lc.info?.(`draining ${svc.constructor.name} ${svc.id} (${signal})`);\n await svc.drain();\n }\n lc.info?.(`stopping ${svc.constructor.name} ${svc.id} (${signal})`);\n await svc.stop();\n });\n });\n }\n\n try {\n // Run all services and resolve when any of them stops.\n const svc = await Promise.race(\n services.map(svc => svc.run().then(() => svc)),\n );\n lc.info?.(`${svc.constructor.name} (${svc.id}) stopped`);\n } catch (e) {\n lc.error?.(`exiting on error`, e);\n throw e;\n }\n}\n\nexport async function exitAfter(\n logContext: LogContext | (() => LogContext),\n run: () => Promise<void>,\n) {\n const getLogContext =\n typeof logContext === 'function' ? logContext : () => logContext;\n try {\n await run();\n const lc = getLogContext();\n lc.info?.(`pid ${pid} exiting normally`);\n process.exit(0);\n } catch (e) {\n const lc = getLogContext();\n if (e instanceof ConfigurationError) {\n lc.error?.(`exiting with configuration error: ${String(e)}`, e);\n process.exit(0);\n }\n if (e instanceof AbortError) {\n // AbortErrors signal an intentional shutdown or restart (e.g. SIGTERM\n // during a rollout, AutoResetSignal) and should not raise alerts. The\n // non-zero exit code is kept so that restart semantics are unchanged.\n lc.warn?.(`exiting after abort: ${String(e)}`, e);\n } else {\n lc.error?.(`exiting with error: ${String(e)}`, e);\n }\n try {\n logLastChanceSQLiteCorruptionDiagnostics(lc, e);\n } catch (diagnosticError) {\n lc.error?.('SQLite corruption last-chance diagnostic failed', {\n error: diagnosticError,\n });\n }\n try {\n await lc.flush();\n } finally {\n process.exit(-1);\n }\n }\n}\n\n/**\n * The HeartbeatMonitor monitors the cadence heartbeats (e.g. \"/keepalive\"\n * health checks made to HttpServices) that signal that the server\n * should continue processing requests. When a configurable `stopInterval`\n * elapses without receiving these heartbeats, the monitor initiates a\n * graceful shutdown of the server. This works with common load balancing\n * frameworks such as AWS Elastic Load Balancing.\n *\n * The HeartbeatMonitor is **opt-in** in that it only kicks in after it\n * starts receiving keepalives.\n */\nexport class HeartbeatMonitor {\n readonly #stopInterval: number;\n\n #lc: LogContext;\n #checkIntervalTimer: NodeJS.Timeout | undefined;\n #checkImmediateTimer: NodeJS.Immediate | undefined;\n #lastHeartbeat = 0;\n\n constructor(lc: LogContext, stopInterval: number) {\n this.#lc = lc;\n this.#stopInterval = stopInterval;\n }\n\n onHeartbeat(reqHeaders: IncomingHttpHeaders) {\n this.#lastHeartbeat = Date.now();\n if (this.#checkIntervalTimer === undefined) {\n this.#lc.info?.(\n `starting heartbeat monitor at ${\n this.#stopInterval / 1000\n } second interval`,\n reqHeaders,\n );\n // e.g. check every 5 seconds to see if it's been over 20 seconds\n // since the last heartbeat.\n this.#checkIntervalTimer = setInterval(\n this.#checkStopInterval,\n this.#stopInterval / 4,\n );\n }\n }\n\n #checkStopInterval = () => {\n // In the Node.js event loop, timers like setInterval and setTimeout\n // run *before* I/O events coming from network sockets or file reads/writes.\n // When this process gets starved of CPU resources for long periods of time,\n // for example when other processes are monopolizing all available cores,\n // pathological behavior can emerge:\n // - keepalive network request comes in, but is queued in Node internals waiting\n // for time on the event loop\n // - CPU is starved/monopolized by other processes for longer than the time\n // configured via this.#stopInterval\n // - When CPU becomes available and the event loop wakes up, this stop interval\n // check is run *before* the keepalive request is processed. The value of\n // this.#lastHeartbeat is now very stale, and erroneously triggers a shutdown\n // even though keepalive requests were about to be processed and update\n // this.#lastHeartbeat. Downtime ensues.\n //\n // To avoid this, we push the check out to a phase of the event loop *after*\n // I/O events are processed, using setImmediate():\n // https://nodejs.org/en/learn/asynchronous-work/event-loop-timers-and-nexttick#setimmediate-vs-settimeout\n //\n // This ensures we see a value for this.#lastHeartbeat that reflects\n // any keepalive requests that came in during the current event loop turn.\n this.#checkImmediateTimer = setImmediate(() => {\n this.#checkImmediateTimer = undefined;\n const timeSinceLastHeartbeat = Date.now() - this.#lastHeartbeat;\n if (timeSinceLastHeartbeat >= this.#stopInterval) {\n this.#lc.info?.(\n `last heartbeat received ${\n timeSinceLastHeartbeat / 1000\n } seconds ago. draining.`,\n );\n process.kill(process.pid, GRACEFUL_SHUTDOWN[0]);\n }\n });\n };\n\n stop() {\n clearTimeout(this.#checkIntervalTimer);\n if (this.#checkImmediateTimer) {\n clearImmediate(this.#checkImmediateTimer);\n }\n }\n}\n"],"mappings":";;;;;;;;;AAmCA,IAAa,oBAAoB,CAAC,WAAW,QAAQ;AACrD,IAAa,oBAAoB,CAAC,WAAW,SAAS;AAItD,SAAS,wBAAwB,MAAc;CAC7C,IAAI,SAAS,cACX;CAEF,IAAI,KAAK,WAAW,iBAAiB,GACnC,OAAO;CAET,IAAI,KAAK,WAAW,YAAY,GAAG;EACjC,IAAI,KAAK,SAAS,UAAU,GAC1B,OAAO;EAET,IAAI,KAAK,SAAS,gBAAgB,GAChC,OAAO;EAET,IAAI,KAAK,SAAS,WAAW,GAC3B,OAAO;EAET,OAAO;CACT;CACA,IAAI,KAAK,WAAW,QAAQ,GAC1B,OAAO;CAET,IAAI,KAAK,WAAW,QAAQ,GAC1B,OAAO;CAET,IAAI,KAAK,WAAW,eAAe,GACjC,OAAO;CAET,IAAI,KAAK,WAAW,SAAS,GAC3B,OAAO;CAET,OAAO;AACT;AAEA,SAAS,kBAAkB;CACzB,OAAO,qBAAqB,UAAU,oBAAoB;EACxD,aAAa;EACb,MAAM;EACN,kBAAkB;CACpB,CAAC;AACH;AAEA,SAAgB,wBAAwB,YAAoB;CAC1D,gBAAgB,EAAE,SAAS,YAAY,EAAC,WAAW,aAAY,CAAC;AAClE;AAEA,SAAS,wBAAwB;CAC/B,OAAO,qBAAqB,UAAU,2BAA2B;EAC/D,aAAa;EACb,MAAM;EACN,kBAAkB;CACpB,CAAC;AACH;;;;;AAgBA,IAAa,iBAAb,MAA4B;CAC1B;CACA,8BAAuB,IAAI,IAAgB;CAC3C,uBAAgB,IAAI,IAAgB;CACpC;CACA,SAAkB,KAAK,IAAI;CAC3B,SAAmC,CAAC;CAEpC,gBAAgB,IAAI,aAAa,iBAAiB;CAClD,cAAc;CAEd,YAAY,IAAgB,MAAoB;EAC9C,KAAKA,MAAM,GAAG,YAAY,aAAa,iBAAiB;EAKxD,KAAK,MAAM,UAAU,mBACnB,KAAK,GAAG,cAAc,KAAKM,YAAY,MAAM,CAAC;EAIhD,KAAK,GAAG,SAAQ,SACd,KAAKC,MACH,KAAKL,MACL,SAAS,IACL,YACA,SAAA,KACE,YACA,SACR,CACF;EASA,KAAK,MAAM,UAAU,mBACnB,KAAK,GAAG,cACN,KAAKM,MAAM,WAAW,YAAA,KAA8C,EAAE,CACxE;EAGF,KAAKL,aAAa,SAAiB;GACjC,IAAI,kBAAkB,GACpB,OAAO,KAAK,KAAK,QAAQ,IAAI;GAE/B,QAAQ,KAAK,IAAI;EACnB;CACF;CAEA,MAAM,WAAW;EACf,IAAI,KAAKM,gBAAgB,GACvB,KAAKH,YAAY,kBAAkB,EAAE;EAEvC,MAAM,KAAKI,cAAc,QAAQ;CACnC;CAEA,MAAM,MAAc;EAClB,KAAKV,IAAI,OAAO,qBAAqB,IAAI;EACzC,KAAKU,cAAc,KAAK,KAAKV,GAAG;EAChC,KAAUA,IAAI,MAAM,EAAE,cAAc,KAAKG,UAAU,IAAI,CAAC;CAC1D;CAEA,YAAY,QAAgC;EAC1C,IAAI,KAAKD,KAAK,SAAS,GAAG;GAExB,KAAKF,IAAI,OAAO,cAAc,QAAQ;GACtC,KAAKQ,MAAM,CAAC;EACd;EACA,KAAKR,IAAI,OAAO,qBAAqB,OAAO,EAAE;EAC9C,KAAKS,cAAc,KAAK,IAAI;EAC5B,IAAI,KAAKR,YAAY,MACnB,KAAKM,MAAM,KAAKN,aAAa,MAAM;OAEnC,KAAKM,MAAM,KAAKL,MAAM,MAAM;CAEhC;CAEA,cAAc,MAAkB,MAAkB,MAAc;EAC9D,IAAI,SAAS,eACX,KAAKD,YAAY,IAAI,IAAI;EAE3B,KAAKC,KAAK,IAAI,IAAI;EAElB,IAAI,SAAS;EACb,KAAK,GAAG,UAAU,MAAM,WAAW;GACjC,SAAS;GACT,KAAKS,QAAQ,MAAM,QAAQ,MAAM,MAAM,MAAM,IAAI;EACnD,CAAC;EAMD,KAAK,GAAG,UAAS,QACf,KAAKX,IAAI,CAAC,UAAU,KAAKS,cAAc,IAAI,SAAS,WAClD,cAAc,KAAK,GAAG,KAAK,OAC3B,GACF,CACF;CACF;CAEA,gCAAyB,IAAI,IAAoB;CACjD,UAAU;CAEV,UAAU,QAAgB,MAAkB,MAAsB;EAChE,KAAK,cAAc,QAAQ,MAAM,IAAI;EAErC,MAAM,QAAQ,YAAY,IAAI;EAC9B,MAAM,KAAK,EAAE,KAAKI;EAClB,KAAKD,cAAc,IAAI,IAAI,IAAI;EAC/B,MAAM,EAAC,SAAS,YAAW,SAAS;EACpC,KAAKP,OAAO,KAAK,OAAO;EAExB,OAAO,gBAAgB,eAAe;GACpC,MAAM,UAAU,YAAY,IAAI,IAAI;GACpC,MAAM,mBAAmB,wBAAwB,IAAI;GACrD,IAAI,kBACF,sBAAsB,EAAE,SAAS,SAAS;IACxC,QAAQ;IACR;GACF,CAAC;GAEH,KAAKL,IAAI,QAAQ,GAAG,KAAK,UAAU,KAAK,IAAI,IAAI,KAAKI,OAAO,KAAK;GACjE,KAAKQ,cAAc,OAAO,EAAE;GAC5B,QAAQ;EACV,CAAC;EAED,OAAO;CACT;CAEA,eAAyB;EACvB,OAAO,CAAC,GAAG,KAAKA,cAAc,OAAO,CAAC;CACxC;CAEA,MAAM,kBAAkB;EACtB,MAAM,QAAQ,IAAI,KAAKP,MAAM;CAC/B;CAEA,gBAAgB,KAAc,MAAc;EAE1C,KAAKM,QAAQ,IAAI,MAAM,KAAK,eAAe,MAAM,KAAA,CAAS;CAC5D;CAEA,QACE,MACA,KACA,KACA,MACA,MACA,QACA;EAEA,IAAI,QAAQ;GACV,KAAKV,YAAY,OAAO,MAAM;GAC9B,KAAKC,KAAK,OAAO,MAAM;EACzB;EAEA,MAAM,MAAM,QAAQ,OAAO,QAAQ;EAEnC,IAAI,SAAS,cAAc;GAMzB,IAAI,SAAS,MAAM,KAAKO,gBAAgB,KAAK,KAAKR,YAAY,OAAO,IACnE,OAAA;GAEF,MAAM,MACJ,SAAS,KAAK,SAAA,KACV,SACA;GACN,KAAKD,IAAI,OAAO,GAAG,KAAK,IAAI,IAAI,sBAAsB,KAAK,IAAI,OAAO,EAAE;GACxE,OAAO,KAAKQ,MAAM,IAAI;EACxB;EAEA,MAAM,MACJ,SAAS,KAAK,SAAA,KACV,SACA,KAAKC,cAAc,KAAK,SAAA,KACtB,SACA;EACR,KAAKT,IAAI,OACP,MACI,GAAG,KAAK,IAAI,IAAI,iBAAiB,IAAI,KACrC,GAAG,KAAK,IAAI,IAAI,sBAAsB,KAAK,IAC/C,OAAO,EACT;EAGA,IAAI,KAAKC,YAAY,SAAS,GAAG;GAC/B,KAAKD,IAAI,OACP,KAAKS,cACD,oCACE,KAAK,IAAI,IAAI,KAAKA,YACnB,QACD,gCACN;GACA,OAAO,KAAKD,MAAM,KAAKC,cAAc,IAAI,QAAQ,EAAE;EACrD;EAEA,IAAI,KAAKA,gBAAgB,GAGvB,OAAO,KAAKD,MAAM,QAAQ,EAAE;CAIhC;CAEA,MAAM,SAA+B,QAAwB;EAC3D,KAAK,MAAM,UAAU,SACnB,IAAI;GACF,OAAO,KAAK,MAAM;EACpB,SAAS,GAAG;GACV,KAAKR,IAAI,QAAQ,CAAC;EACpB;CAEJ;AACF;;;;;;;;AAUA,eAAsB,eACpB,IACA,QACA,GAAG,UACY;CACf,IAAI,SAAS,WAAW,GACtB;CAEF,KAAK,MAAM,UAAU,CAAC,GAAG,mBAAmB,GAAG,iBAAiB,GAC9D,OAAO,KAAK,cAAc;EACxB,MAAM,mBAAmB;EAEzB,SAAS,QAAQ,OAAM,QAAO;GAC5B,IAAI,iBAAiB,SAAS,MAAM,KAAK,IAAI,OAAO;IAClD,GAAG,OAAO,YAAY,IAAI,YAAY,KAAK,GAAG,IAAI,GAAG,IAAI,OAAO,EAAE;IAClE,MAAM,IAAI,MAAM;GAClB;GACA,GAAG,OAAO,YAAY,IAAI,YAAY,KAAK,GAAG,IAAI,GAAG,IAAI,OAAO,EAAE;GAClE,MAAM,IAAI,KAAK;EACjB,CAAC;CACH,CAAC;CAGH,IAAI;EAEF,MAAM,MAAM,MAAM,QAAQ,KACxB,SAAS,KAAI,QAAO,IAAI,IAAI,EAAE,WAAW,GAAG,CAAC,CAC/C;EACA,GAAG,OAAO,GAAG,IAAI,YAAY,KAAK,IAAI,IAAI,GAAG,UAAU;CACzD,SAAS,GAAG;EACV,GAAG,QAAQ,oBAAoB,CAAC;EAChC,MAAM;CACR;AACF;AAEA,eAAsB,UACpB,YACA,KACA;CACA,MAAM,gBACJ,OAAO,eAAe,aAAa,mBAAmB;CACxD,IAAI;EACF,MAAM,IAAI;EAEV,cAAA,EAAG,OAAO,OAAO,IAAI,kBAAkB;EACvC,QAAQ,KAAK,CAAC;CAChB,SAAS,GAAG;EACV,MAAM,KAAK,cAAc;EACzB,IAAI,aAAa,oBAAoB;GACnC,GAAG,QAAQ,qCAAqC,OAAO,CAAC,KAAK,CAAC;GAC9D,QAAQ,KAAK,CAAC;EAChB;EACA,IAAI,aAAa,YAIf,GAAG,OAAO,wBAAwB,OAAO,CAAC,KAAK,CAAC;OAEhD,GAAG,QAAQ,uBAAuB,OAAO,CAAC,KAAK,CAAC;EAElD,IAAI;GACF,yCAAyC,IAAI,CAAC;EAChD,SAAS,iBAAiB;GACxB,GAAG,QAAQ,mDAAmD,EAC5D,OAAO,gBACT,CAAC;EACH;EACA,IAAI;GACF,MAAM,GAAG,MAAM;EACjB,UAAU;GACR,QAAQ,KAAK,EAAE;EACjB;CACF;AACF;;;;;;;;;;;;AAaA,IAAa,mBAAb,MAA8B;CAC5B;CAEA;CACA;CACA;CACA,iBAAiB;CAEjB,YAAY,IAAgB,cAAsB;EAChD,KAAKA,MAAM;EACX,KAAKc,gBAAgB;CACvB;CAEA,YAAY,YAAiC;EAC3C,KAAKC,iBAAiB,KAAK,IAAI;EAC/B,IAAI,KAAKC,wBAAwB,KAAA,GAAW;GAC1C,KAAKhB,IAAI,OACP,iCACE,KAAKc,gBAAgB,IACtB,mBACD,UACF;GAGA,KAAKE,sBAAsB,YACzB,KAAKC,oBACL,KAAKH,gBAAgB,CACvB;EACF;CACF;CAEA,2BAA2B;EAsBzB,KAAKI,uBAAuB,mBAAmB;GAC7C,KAAKA,uBAAuB,KAAA;GAC5B,MAAM,yBAAyB,KAAK,IAAI,IAAI,KAAKH;GACjD,IAAI,0BAA0B,KAAKD,eAAe;IAChD,KAAKd,IAAI,OACP,2BACE,yBAAyB,IAC1B,wBACH;IACA,QAAQ,KAAK,QAAQ,KAAK,kBAAkB,EAAE;GAChD;EACF,CAAC;CACH;CAEA,OAAO;EACL,aAAa,KAAKgB,mBAAmB;EACrC,IAAI,KAAKE,sBACP,eAAe,KAAKA,oBAAoB;CAE5C;AACF"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"commands.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/litestream/commands.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,YAAY,EAAC,MAAM,oBAAoB,CAAC;AAIrD,OAAO,KAAK,EAAC,UAAU,EAAW,MAAM,kBAAkB,CAAC;AAI3D,OAAO,EAAC,KAAK,gBAAgB,EAAC,MAAM,2BAA2B,CAAC;
|
|
1
|
+
{"version":3,"file":"commands.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/litestream/commands.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,YAAY,EAAC,MAAM,oBAAoB,CAAC;AAIrD,OAAO,KAAK,EAAC,UAAU,EAAW,MAAM,kBAAkB,CAAC;AAI3D,OAAO,EAAC,KAAK,gBAAgB,EAAC,MAAM,2BAA2B,CAAC;AAUhE,OAAO,EAWL,KAAK,cAAc,EACpB,MAAM,cAAc,CAAC;AAEtB,MAAM,MAAM,kBAAkB,GAAG;IAC/B,cAAc,EAAE,MAAM,CAAC;IACvB,YAAY,EAAE,MAAM,CAAC;CACtB,CAAC;AAEF,MAAM,MAAM,aAAa,GACrB,SAAS,GACT,WAAW,GACX,iBAAiB,GACjB,OAAO,CAAC;AAEZ,KAAK,cAAc,GAAG;IACpB,QAAQ,EAAE,OAAO,CAAC;IAClB,SAAS,EAAE,MAAM,GAAG,SAAS,CAAC;IAC9B,MAAM,EAAE,aAAa,CAAC;CACvB,CAAC;AAKF,qBAAa,uBAAwB,SAAQ,KAAK;IAChD,MAAM,CAAC,QAAQ,CAAC,IAAI,6BAA6B;IAEjD,YAAY,SAAS,EAAE,MAAM,GAAG,SAAS,EAExC;CACF;AAyFD,wBAAsB,UAAU,CAC9B,EAAE,EAAE,UAAU,EACd,MAAM,EAAE,gBAAgB,EACxB,WAAW,EAAE,MAAM,EACnB,kBAAkB,EAAE,kBAAkB,GAAG,SAAS,EAClD,IAAI,EAAE,cAAc,GACnB,OAAO,CAAC,cAAc,CAAC,CAwIzB;AAoGD,wBAAgB,yBAAyB,CACvC,EAAE,EAAE,UAAU,EACd,MAAM,EAAE,gBAAgB,EACxB,WAAW,EAAE,MAAM,GAClB,YAAY,CAuCd;AAQD;;;;;;;;;;;;;GAaG;AACH,wBAAsB,iBAAiB,CACrC,EAAE,EAAE,UAAU,EACd,MAAM,EAAE,gBAAgB,EACxB,WAAW,EAAE,MAAM,GAClB,OAAO,CAAC,IAAI,CAAC,CAgBf;AA8DD;;;;;;GAMG;AACH,wBAAgB,uBAAuB,CACrC,EAAE,EAAE,UAAU,EACd,OAAO,EAAE,WAAW,GAAG,KAAK,EAC5B,MAAM,EAAE,MAAM,GACb,IAAI,EAAE,CAoBR"}
|
|
@@ -6,6 +6,7 @@ import { StatementRunner } from "../../db/statements.js";
|
|
|
6
6
|
import { isSQLiteCorruption, logSQLiteCorruptionDiagnostics } from "../../db/sqlite-corruption.js";
|
|
7
7
|
import { deleteLiteDB } from "../../db/delete-lite-db.js";
|
|
8
8
|
import { deleteChangeLogDB } from "../replicator/change-log-db.js";
|
|
9
|
+
import { litestreamSocketPath } from "./litestream-controller.js";
|
|
9
10
|
import { litestreamBackupListDuration, litestreamBackupMetricAttrs, litestreamBackupProcessDuration, litestreamBackupProcessMetricAttrs, litestreamBackupProcessRuns, litestreamRestoreAttempts, litestreamRestoreMetricAttrs, litestreamRestoreProcessDuration, litestreamRestoreValidationDuration, litestreamRestoredDbBytes } from "./metrics.js";
|
|
10
11
|
import { resolver } from "@rocicorp/resolver";
|
|
11
12
|
import { spawn } from "node:child_process";
|
|
@@ -15,10 +16,12 @@ import { basename, dirname, join } from "node:path";
|
|
|
15
16
|
var MAX_LOGGED_RESTORE_DIRECTORY_ENTRIES = 100;
|
|
16
17
|
var LEGACY_RESTORE_WAL_INDEX = /^[0-9a-f]{8}$/;
|
|
17
18
|
function getLitestream(mode, config, replicaFile, logLevelOverride) {
|
|
18
|
-
const { executable, executableV5, restoreUsingV5, backupUsingV5, backupURL, logLevel, configPath, configPathV5, endpoint, region, port, checkpointThresholdMB, minCheckpointPageCount = checkpointThresholdMB *
|
|
19
|
+
const { executable, executableV5, restoreUsingV5, backupUsingV5, backupURL, logLevel, configPath, configPathV5, endpoint, region, port, checkpointThresholdMB, minCheckpointPageCount = checkpointThresholdMB * 256, maxCheckpointPageCount = minCheckpointPageCount * 10, forceCheckpointThresholdMB, incrementalBackupIntervalMinutes, incrementalBackupIntervalSeconds, snapshotBackupIntervalHours, snapshotBackupIntervalHoursV5, multipartConcurrency, multipartSize } = config;
|
|
19
20
|
const v5 = mode === "restore" && restoreUsingV5 || mode === "replicate" && backupUsingV5;
|
|
20
21
|
const litestream = (v5 ? executableV5 : executable) ?? must(executable, `Missing --litestream-executable`);
|
|
21
22
|
const litestreamConfig = v5 ? configPathV5 : configPath;
|
|
23
|
+
const snapshotIntervalHours = v5 ? snapshotBackupIntervalHoursV5 : snapshotBackupIntervalHours;
|
|
24
|
+
const truncatePageN = forceCheckpointThresholdMB ? -1 : 121359;
|
|
22
25
|
return {
|
|
23
26
|
litestream,
|
|
24
27
|
env: {
|
|
@@ -29,12 +32,14 @@ function getLitestream(mode, config, replicaFile, logLevelOverride) {
|
|
|
29
32
|
["ZERO_LITESTREAM_MAX_CHECKPOINT_PAGE_COUNT"]: String(maxCheckpointPageCount),
|
|
30
33
|
["ZERO_LITESTREAM_INCREMENTAL_BACKUP_INTERVAL_MINUTES"]: String(incrementalBackupIntervalMinutes),
|
|
31
34
|
["ZERO_LITESTREAM_INCREMENTAL_BACKUP_INTERVAL_SECONDS"]: String(incrementalBackupIntervalSeconds),
|
|
35
|
+
["ZERO_LITESTREAM_TRUNCATE_PAGE_N"]: String(truncatePageN),
|
|
32
36
|
["ZERO_LITESTREAM_LOG_LEVEL"]: logLevelOverride ?? logLevel,
|
|
33
|
-
["ZERO_LITESTREAM_SNAPSHOT_BACKUP_INTERVAL_HOURS"]: String(
|
|
34
|
-
["ZERO_LITESTREAM_SNAPSHOT_RETENTION_INTERVAL_HOURS"]: String(
|
|
37
|
+
["ZERO_LITESTREAM_SNAPSHOT_BACKUP_INTERVAL_HOURS"]: String(snapshotIntervalHours),
|
|
38
|
+
["ZERO_LITESTREAM_SNAPSHOT_RETENTION_INTERVAL_HOURS"]: String(snapshotIntervalHours + 6),
|
|
35
39
|
["ZERO_LITESTREAM_MULTIPART_CONCURRENCY"]: String(multipartConcurrency),
|
|
36
40
|
["ZERO_LITESTREAM_MULTIPART_SIZE"]: String(multipartSize),
|
|
37
41
|
["ZERO_LOG_FORMAT"]: "json",
|
|
42
|
+
["ZERO_LITESTREAM_SOCKET_PATH"]: litestreamSocketPath(replicaFile),
|
|
38
43
|
["LITESTREAM_CONFIG"]: litestreamConfig,
|
|
39
44
|
["LITESTREAM_PORT"]: String(port),
|
|
40
45
|
...endpoint ? { ["ZERO_LITESTREAM_ENDPOINT"]: endpoint } : {},
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"commands.js","names":[],"sources":["../../../../../../zero-cache/src/services/litestream/commands.ts"],"sourcesContent":["import type {ChildProcess} from 'node:child_process';\nimport {spawn} from 'node:child_process';\nimport {existsSync, readdirSync, rmSync, statSync} from 'node:fs';\nimport {basename, dirname, join} from 'node:path';\nimport type {LogContext, LogLevel} from '@rocicorp/logger';\nimport {resolver} from '@rocicorp/resolver';\nimport {must} from '../../../../shared/src/must.ts';\nimport {Database} from '../../../../zqlite/src/db.ts';\nimport {type LitestreamConfig} from '../../config/normalize.ts';\nimport {deleteLiteDB} from '../../db/delete-lite-db.ts';\nimport {\n isSQLiteCorruption,\n logSQLiteCorruptionDiagnostics,\n} from '../../db/sqlite-corruption.ts';\nimport {StatementRunner} from '../../db/statements.ts';\nimport {deleteChangeLogDB} from '../replicator/change-log-db.ts';\nimport {getSubscriptionState} from '../replicator/schema/replication-state.ts';\nimport {\n litestreamBackupListDuration,\n litestreamBackupMetricAttrs,\n litestreamBackupProcessDuration,\n litestreamBackupProcessMetricAttrs,\n litestreamBackupProcessRuns,\n litestreamRestoreAttempts,\n litestreamRestoredDbBytes,\n litestreamRestoreMetricAttrs,\n litestreamRestoreProcessDuration,\n litestreamRestoreValidationDuration,\n type LitestreamRole,\n} from './metrics.ts';\n\nexport type ReplicaConstraints = {\n replicaVersion: string;\n minWatermark: string;\n};\n\nexport type RestoreResult =\n | 'success'\n | 'no_backup'\n | 'invalid_replica'\n | 'error';\n\ntype RestoreAttempt = {\n restored: boolean;\n backupURL: string | undefined;\n result: RestoreResult;\n};\n\nconst MAX_LOGGED_RESTORE_DIRECTORY_ENTRIES = 100;\nconst LEGACY_RESTORE_WAL_INDEX = /^[0-9a-f]{8}$/;\n\nexport class BackupNotFoundException extends Error {\n static readonly name = 'BackupNotFoundException';\n\n constructor(backupURL: string | undefined) {\n super(`backup not found at ${backupURL}`);\n }\n}\n\nfunction getLitestream(\n mode: 'restore' | 'replicate',\n config: LitestreamConfig,\n replicaFile: string,\n logLevelOverride?: LogLevel,\n): {\n litestream: string;\n env: NodeJS.ProcessEnv;\n} {\n const {\n executable,\n executableV5,\n restoreUsingV5,\n backupUsingV5,\n backupURL,\n logLevel,\n configPath,\n configPathV5,\n endpoint,\n region,\n port,\n checkpointThresholdMB,\n minCheckpointPageCount = checkpointThresholdMB * 250, // SQLite page size is 4KB\n maxCheckpointPageCount = minCheckpointPageCount * 10,\n incrementalBackupIntervalMinutes,\n incrementalBackupIntervalSeconds,\n snapshotBackupIntervalHours,\n multipartConcurrency,\n multipartSize,\n } = config;\n\n const v5 =\n (mode === 'restore' && restoreUsingV5) ||\n (mode === 'replicate' && backupUsingV5);\n const litestream =\n (v5 ? executableV5 : executable) ??\n must(executable, `Missing --litestream-executable`);\n const litestreamConfig = v5 ? configPathV5 : configPath;\n\n return {\n litestream,\n env: {\n ...process.env,\n ['ZERO_REPLICA_FILE']: replicaFile,\n ['ZERO_LITESTREAM_BACKUP_URL']: must(backupURL),\n ['ZERO_LITESTREAM_MIN_CHECKPOINT_PAGE_COUNT']: String(\n minCheckpointPageCount,\n ),\n ['ZERO_LITESTREAM_MAX_CHECKPOINT_PAGE_COUNT']: String(\n maxCheckpointPageCount,\n ),\n ['ZERO_LITESTREAM_INCREMENTAL_BACKUP_INTERVAL_MINUTES']: String(\n incrementalBackupIntervalMinutes, // v3 only\n ),\n ['ZERO_LITESTREAM_INCREMENTAL_BACKUP_INTERVAL_SECONDS']: String(\n incrementalBackupIntervalSeconds, // v5 only\n ),\n ['ZERO_LITESTREAM_LOG_LEVEL']: logLevelOverride ?? logLevel,\n ['ZERO_LITESTREAM_SNAPSHOT_BACKUP_INTERVAL_HOURS']: String(\n snapshotBackupIntervalHours,\n ),\n ['ZERO_LITESTREAM_SNAPSHOT_RETENTION_INTERVAL_HOURS']: String(\n snapshotBackupIntervalHours + 6, // delete old snapshots after 6 hours\n ),\n ['ZERO_LITESTREAM_MULTIPART_CONCURRENCY']: String(multipartConcurrency),\n ['ZERO_LITESTREAM_MULTIPART_SIZE']: String(multipartSize),\n ['ZERO_LOG_FORMAT']: 'json',\n ['LITESTREAM_CONFIG']: litestreamConfig,\n ['LITESTREAM_PORT']: String(port),\n ...(endpoint ? {['ZERO_LITESTREAM_ENDPOINT']: endpoint} : {}),\n ...(region ? {['ZERO_LITESTREAM_REGION']: region} : {}),\n },\n };\n}\n\nexport async function tryRestore(\n lc: LogContext,\n config: LitestreamConfig,\n replicaFile: string,\n replicaConstraints: ReplicaConstraints | undefined,\n role: LitestreamRole,\n): Promise<RestoreAttempt> {\n const {backupURL} = config;\n const attrs = litestreamRestoreMetricAttrs(config, role, backupURL);\n let result: RestoreResult = 'error';\n try {\n logRestoreDirectoryContents(lc, replicaFile, 'before-temp-cleanup');\n deleteRestoreTempFiles(replicaFile);\n const replicaExistedBeforeRestore = existsSync(replicaFile);\n const {litestream, env} = getLitestream(\n 'restore',\n config,\n replicaFile,\n 'debug', // Include all output from `litestream restore`, as it's minimal.\n );\n const {\n restoreParallelism: parallelism,\n multipartConcurrency,\n multipartSize,\n } = config;\n lc.info?.(`starting litestream restore`, {\n restoreParallelism: parallelism,\n multipartConcurrency,\n multipartSize,\n });\n // Pipe (rather than inherit) litestream's stdout/stderr so that its own\n // `\"level\":\"ERROR\"` output on a failed restore does not go straight to the\n // pod's stdout — where a log-scraper alert would page on it — before our\n // code has decided how to handle the failure. The captured output is\n // included in the thrown error so the caller can log the final outcome.\n // See INC-961.\n const proc = spawn(\n litestream,\n [\n 'restore',\n '-if-db-not-exists',\n '-if-replica-exists',\n '-parallelism',\n String(parallelism),\n replicaFile,\n ],\n {env, stdio: ['ignore', 'pipe', 'pipe'], windowsHide: true},\n );\n let stdout = '';\n let stderr = '';\n proc.stdout.setEncoding('utf-8');\n proc.stderr.setEncoding('utf-8');\n // Output AND capture the stdout and stderr, the former for observability\n // and the latter for error handling.\n proc.stdout.on('data', chunk => {\n process.stdout.write(chunk);\n stdout += chunk;\n });\n proc.stderr.on('data', chunk => {\n process.stderr.write(chunk);\n stderr += chunk;\n });\n const {promise, resolve, reject} = resolver();\n proc.on('error', reject);\n proc.on('close', (code, signal) => {\n if (signal) {\n reject(`litestream killed with ${signal}`);\n } else if (code !== 0) {\n reject(`litestream exited with code ${code}`);\n } else {\n resolve();\n }\n });\n const processStart = performance.now();\n try {\n await promise;\n litestreamRestoreProcessDuration().recordMs(\n performance.now() - processStart,\n {...attrs, result: 'success'},\n );\n } catch (e) {\n litestreamRestoreProcessDuration().recordMs(\n performance.now() - processStart,\n {...attrs, result: 'error'},\n );\n const output = [stdout, stderr]\n .map(value => value.trim())\n .filter(Boolean)\n .join('\\n');\n throw new Error(output ? `${String(e)}\\n${output}` : String(e), {\n cause: e,\n });\n }\n if (!existsSync(replicaFile)) {\n result = 'no_backup';\n return {restored: false, backupURL, result};\n }\n const validationStart = performance.now();\n const valid = replicaIsValid(lc, replicaFile, replicaConstraints);\n litestreamRestoreValidationDuration().recordMs(\n performance.now() - validationStart,\n {...attrs, result: valid ? 'success' : 'invalid_replica'},\n );\n if (!valid) {\n result = 'invalid_replica';\n lc.info?.(`Deleting local replica and retrying restore`);\n deleteLiteDB(replicaFile);\n deleteChangeLogDB(replicaFile);\n return {restored: false, backupURL, result};\n }\n result = 'success';\n if (!replicaExistedBeforeRestore) {\n // This restore materialized the replica file, so any change log beside it\n // was written against a replica that is no longer there. Reconciliation\n // is the safety net — a restored replica keeps its replicaVersion, so the\n // rule falls through to truncate-or-reseed on head mismatch — but the log\n // is a cache and deleting it here is the explicit statement that nothing\n // in it survives the restore.\n //\n // A restore that reuses an existing replica (`-if-db-not-exists` made it\n // a no-op) deliberately keeps the log: that is the process-restart path,\n // where the log is still the one written beside this exact replica and\n // discarding it would cost every reconnecting subscriber a `too-old`.\n deleteChangeLogDB(replicaFile);\n litestreamRestoredDbBytes().add(statSync(replicaFile).size, {\n ...attrs,\n result: 'success',\n });\n }\n return {restored: true, backupURL, result};\n } finally {\n try {\n deleteRestoreTempFiles(replicaFile);\n } catch (e) {\n lc.warn?.('Unable to clean up Litestream restore temporary files', {\n replicaFile,\n error: String(e),\n });\n }\n logRestoreDirectoryContents(lc, replicaFile, 'after-temp-cleanup');\n litestreamRestoreAttempts().add(1, {...attrs, result});\n }\n}\n\nfunction deleteRestoreTempFiles(replicaFile: string) {\n const temporaryReplicaFile = `${replicaFile}.tmp`;\n deleteLiteDB(temporaryReplicaFile);\n\n // rocicorp/litestream zero@v0.0.9 downloads WAL indexes in parallel to\n // `<output>.tmp-<8 lowercase hex digits>-wal` before applying them.\n const directory = dirname(temporaryReplicaFile);\n if (!existsSync(directory)) {\n return;\n }\n const prefix = `${basename(temporaryReplicaFile)}-`;\n for (const name of readdirSync(directory)) {\n if (!name.startsWith(prefix) || !name.endsWith('-wal')) {\n continue;\n }\n const index = name.slice(prefix.length, -'-wal'.length);\n if (LEGACY_RESTORE_WAL_INDEX.test(index)) {\n rmSync(join(directory, name), {force: true});\n }\n }\n}\n\nfunction logRestoreDirectoryContents(\n lc: LogContext,\n replicaFile: string,\n phase: 'before-temp-cleanup' | 'after-temp-cleanup',\n) {\n const directory = dirname(replicaFile);\n try {\n const names = readdirSync(directory).sort();\n const entries = names\n .slice(0, MAX_LOGGED_RESTORE_DIRECTORY_ENTRIES)\n .map(name => ({name, sizeBytes: statSync(join(directory, name)).size}));\n lc.info?.('Litestream restore directory contents', {\n phase,\n directory,\n replicaFile,\n entryCount: names.length,\n omittedEntryCount: Math.max(\n 0,\n names.length - MAX_LOGGED_RESTORE_DIRECTORY_ENTRIES,\n ),\n entries,\n });\n } catch (e) {\n lc.warn?.('Unable to inspect Litestream restore directory', {\n phase,\n directory,\n replicaFile,\n error: String(e),\n });\n }\n}\n\nfunction replicaIsValid(\n lc: LogContext,\n replica: string,\n constraints: ReplicaConstraints | undefined,\n) {\n let db: Database | undefined;\n try {\n // Note: Open the database and read the subscription state as a\n // sanity / corruption check, even if there are no constraints.\n db = new Database(lc, replica);\n const {replicaVersion, watermark} = getSubscriptionState(\n new StatementRunner(db),\n );\n if (constraints) {\n if (replicaVersion !== constraints.replicaVersion) {\n lc.warn?.(\n `Local replica version ${replicaVersion} does not match expected replicaVersion ${constraints.replicaVersion}`,\n constraints,\n );\n return false;\n }\n if (watermark < constraints.minWatermark) {\n lc.warn?.(\n `Local replica watermark ${watermark} is earlier than minWatermark ${constraints.minWatermark}`,\n );\n return false;\n }\n lc.info?.(\n `Local replica at version ${replicaVersion} and watermark ${watermark} is compatible`,\n constraints,\n );\n }\n return true;\n } catch (e) {\n if (isSQLiteCorruption(e)) {\n logSQLiteCorruptionDiagnostics(lc, 'restored replica', replica, e);\n }\n lc.error?.('Error while validating restored replica', e);\n return false;\n } finally {\n db?.close();\n }\n}\n\nexport function startReplicaBackupProcess(\n lc: LogContext,\n config: LitestreamConfig,\n replicaFile: string,\n): ChildProcess {\n const {litestream, env} = getLitestream('replicate', config, replicaFile);\n const attrs = litestreamBackupProcessMetricAttrs(config);\n lc.info?.(`starting litestream backup to ${config.backupURL}`);\n const start = performance.now();\n const proc = spawn(litestream, ['replicate'], {\n env,\n stdio: 'inherit',\n windowsHide: true,\n });\n let recorded = false;\n const record = (result: 'success' | 'error' | 'stopped') => {\n if (recorded) {\n return;\n }\n recorded = true;\n const labels = {...attrs, result};\n litestreamBackupProcessRuns().add(1, labels);\n litestreamBackupProcessDuration().recordMs(\n performance.now() - start,\n labels,\n );\n };\n proc.on('error', e => {\n lc.warn?.(`litestream backup process error`, e);\n record('error');\n });\n proc.on('close', (code, signal) => {\n if (signal) {\n lc.info?.(`litestream backup process stopped`, {signal});\n record('stopped');\n } else if (code === 0) {\n record('success');\n } else {\n lc.warn?.(`litestream backup process exited with code ${code}`);\n record('error');\n }\n });\n return proc;\n}\n\n// Listing the backup state requires a few S3 LIST requests, which should\n// normally complete well within this timeout.\nconst LIST_BACKUP_TIMEOUT_MS = 30_000;\n\nconst wsRe = /\\s+/;\n\n/**\n * Returns the time of the most recent object (snapshot or WAL segment)\n * actually uploaded to the backup replica destination, as listed by the\n * bundled litestream CLI (`litestream snapshots` / `litestream wal`).\n *\n * This queries the replica destination (e.g. S3) directly, and thus serves\n * as a source of truth for backup durability. This is in contrast to the\n * `litestream_replica_progress` metric, which is exported when litestream\n * *believes* an upload has succeeded, and has been observed to advance even\n * when nothing is actually written to the destination.\n *\n * Rejects if the backup state cannot be determined (spawn error, non-zero\n * exit, timeout, or empty/unparseable listing).\n */\nexport async function getLastBackupTime(\n lc: LogContext,\n config: LitestreamConfig,\n replicaFile: string,\n): Promise<Date> {\n const [snapshots, wal] = await Promise.all([\n listBackupCreatedTimes(lc, config, replicaFile, 'snapshots'),\n listBackupCreatedTimes(lc, config, replicaFile, 'wal'),\n ]);\n const times = [...snapshots, ...wal];\n if (times.length === 0) {\n // Note: the litestream CLI exits with code 0 and logs listing errors\n // (e.g. S3 failures) to stderr, so an empty listing cannot be\n // distinguished from a failed one. Since a valid backup always contains\n // at least one snapshot, an empty listing is treated as a failure.\n throw new Error(\n `no snapshots or WAL segments listed at ${config.backupURL}`,\n );\n }\n return new Date(Math.max(...times.map(time => time.getTime())));\n}\n\n/**\n * Runs `litestream <snapshots|wal> <replica-file>` with the same config /\n * environment used by the `litestream replicate` process (so that the\n * backupURL, endpoint, region, and credentials are identical), and parses\n * the `created` column (RFC3339) of the tab-formatted output, e.g.:\n *\n * ```\n * replica generation index size created\n * s3 1862f44967b3863f 0 4546445 2026-06-10T01:11:32Z\n * ```\n */\nasync function listBackupCreatedTimes(\n lc: LogContext,\n config: LitestreamConfig,\n replicaFile: string,\n command: 'snapshots' | 'wal',\n): Promise<Date[]> {\n const start = performance.now();\n let result: 'success' | 'empty' | 'timeout' | 'error' = 'error';\n const {litestream, env} = getLitestream('replicate', config, replicaFile);\n const proc = spawn(litestream, [command, replicaFile], {\n env,\n stdio: ['ignore', 'pipe', 'inherit'],\n windowsHide: true,\n });\n const {promise, resolve, reject} = resolver<string>();\n let stdout = '';\n proc.stdout.setEncoding('utf-8');\n proc.stdout.on('data', chunk => (stdout += chunk));\n proc.on('error', reject);\n proc.on('close', (code, signal) => {\n if (signal) {\n reject(new Error(`litestream ${command} killed with ${signal}`));\n } else if (code !== 0) {\n reject(new Error(`litestream ${command} exited with code ${code}`));\n } else {\n resolve(stdout);\n }\n });\n const timeout = setTimeout(() => {\n result = 'timeout';\n reject(new Error(`timed out listing backup state (litestream ${command})`));\n proc.kill('SIGKILL');\n }, LIST_BACKUP_TIMEOUT_MS);\n\n try {\n const output = await promise;\n const times = parseBackupCreatedTimes(lc, command, output);\n result = times.length ? 'success' : 'empty';\n return times;\n } finally {\n clearTimeout(timeout);\n litestreamBackupListDuration().recordMs(performance.now() - start, {\n ...litestreamBackupMetricAttrs(config),\n command,\n result,\n });\n }\n}\n\n/**\n * Parses the `created` column (the last, RFC3339-formatted column) from the\n * tab-formatted output of `litestream snapshots` / `litestream wal`. The\n * header row and any unparseable lines are skipped.\n *\n * Exported for testing.\n */\nexport function parseBackupCreatedTimes(\n lc: LogContext,\n command: 'snapshots' | 'wal',\n output: string,\n): Date[] {\n const times: Date[] = [];\n for (const line of output.split('\\n')) {\n const cols = line.trim().split(wsRe);\n const created = cols.at(-1);\n if (\n cols.length < 2 ||\n created === undefined ||\n created === 'created' /* header row */\n ) {\n continue;\n }\n const time = new Date(created);\n if (Number.isNaN(time.getTime())) {\n lc.warn?.(`unexpected line in litestream ${command} output: ${line}`);\n continue;\n }\n times.push(time);\n }\n return times;\n}\n"],"mappings":";;;;;;;;;;;;;;AAgDA,IAAM,uCAAuC;AAC7C,IAAM,2BAA2B;AAUjC,SAAS,cACP,MACA,QACA,aACA,kBAIA;CACA,MAAM,EACJ,YACA,cACA,gBACA,eACA,WACA,UACA,YACA,cACA,UACA,QACA,MACA,uBACA,yBAAyB,wBAAwB,KACjD,yBAAyB,yBAAyB,IAClD,kCACA,kCACA,6BACA,sBACA,kBACE;CAEJ,MAAM,KACH,SAAS,aAAa,kBACtB,SAAS,eAAe;CAC3B,MAAM,cACH,KAAK,eAAe,eACrB,KAAK,YAAY,iCAAiC;CACpD,MAAM,mBAAmB,KAAK,eAAe;CAE7C,OAAO;EACL;EACA,KAAK;GACH,GAAG,QAAQ;IACV,sBAAsB;IACtB,+BAA+B,KAAK,SAAS;IAC7C,8CAA8C,OAC7C,sBACF;IACC,8CAA8C,OAC7C,sBACF;IACC,wDAAwD,OACvD,gCACF;IACC,wDAAwD,OACvD,gCACF;IACC,8BAA8B,oBAAoB;IAClD,mDAAmD,OAClD,2BACF;IACC,sDAAsD,OACrD,8BAA8B,CAChC;IACC,0CAA0C,OAAO,oBAAoB;IACrE,mCAAmC,OAAO,aAAa;IACvD,oBAAoB;IACpB,sBAAsB;IACtB,oBAAoB,OAAO,IAAI;GAChC,GAAI,WAAW,GAAE,6BAA6B,SAAQ,IAAI,CAAC;GAC3D,GAAI,SAAS,GAAE,2BAA2B,OAAM,IAAI,CAAC;EACvD;CACF;AACF;AAEA,eAAsB,WACpB,IACA,QACA,aACA,oBACA,MACyB;CACzB,MAAM,EAAC,cAAa;CACpB,MAAM,QAAQ,6BAA6B,QAAQ,MAAM,SAAS;CAClE,IAAI,SAAwB;CAC5B,IAAI;EACF,4BAA4B,IAAI,aAAa,qBAAqB;EAClE,uBAAuB,WAAW;EAClC,MAAM,8BAA8B,WAAW,WAAW;EAC1D,MAAM,EAAC,YAAY,QAAO,cACxB,WACA,QACA,aACA,OACF;EACA,MAAM,EACJ,oBAAoB,aACpB,sBACA,kBACE;EACJ,GAAG,OAAO,+BAA+B;GACvC,oBAAoB;GACpB;GACA;EACF,CAAC;EAOD,MAAM,OAAO,MACX,YACA;GACE;GACA;GACA;GACA;GACA,OAAO,WAAW;GAClB;EACF,GACA;GAAC;GAAK,OAAO;IAAC;IAAU;IAAQ;GAAM;GAAG,aAAa;EAAI,CAC5D;EACA,IAAI,SAAS;EACb,IAAI,SAAS;EACb,KAAK,OAAO,YAAY,OAAO;EAC/B,KAAK,OAAO,YAAY,OAAO;EAG/B,KAAK,OAAO,GAAG,SAAQ,UAAS;GAC9B,QAAQ,OAAO,MAAM,KAAK;GAC1B,UAAU;EACZ,CAAC;EACD,KAAK,OAAO,GAAG,SAAQ,UAAS;GAC9B,QAAQ,OAAO,MAAM,KAAK;GAC1B,UAAU;EACZ,CAAC;EACD,MAAM,EAAC,SAAS,SAAS,WAAU,SAAS;EAC5C,KAAK,GAAG,SAAS,MAAM;EACvB,KAAK,GAAG,UAAU,MAAM,WAAW;GACjC,IAAI,QACF,OAAO,0BAA0B,QAAQ;QACpC,IAAI,SAAS,GAClB,OAAO,+BAA+B,MAAM;QAE5C,QAAQ;EAEZ,CAAC;EACD,MAAM,eAAe,YAAY,IAAI;EACrC,IAAI;GACF,MAAM;GACN,iCAAiC,EAAE,SACjC,YAAY,IAAI,IAAI,cACpB;IAAC,GAAG;IAAO,QAAQ;GAAS,CAC9B;EACF,SAAS,GAAG;GACV,iCAAiC,EAAE,SACjC,YAAY,IAAI,IAAI,cACpB;IAAC,GAAG;IAAO,QAAQ;GAAO,CAC5B;GACA,MAAM,SAAS,CAAC,QAAQ,MAAM,EAC3B,KAAI,UAAS,MAAM,KAAK,CAAC,EACzB,OAAO,OAAO,EACd,KAAK,IAAI;GACZ,MAAM,IAAI,MAAM,SAAS,GAAG,OAAO,CAAC,EAAE,IAAI,WAAW,OAAO,CAAC,GAAG,EAC9D,OAAO,EACT,CAAC;EACH;EACA,IAAI,CAAC,WAAW,WAAW,GAAG;GAC5B,SAAS;GACT,OAAO;IAAC,UAAU;IAAO;IAAW;GAAM;EAC5C;EACA,MAAM,kBAAkB,YAAY,IAAI;EACxC,MAAM,QAAQ,eAAe,IAAI,aAAa,kBAAkB;EAChE,oCAAoC,EAAE,SACpC,YAAY,IAAI,IAAI,iBACpB;GAAC,GAAG;GAAO,QAAQ,QAAQ,YAAY;EAAiB,CAC1D;EACA,IAAI,CAAC,OAAO;GACV,SAAS;GACT,GAAG,OAAO,6CAA6C;GACvD,aAAa,WAAW;GACxB,kBAAkB,WAAW;GAC7B,OAAO;IAAC,UAAU;IAAO;IAAW;GAAM;EAC5C;EACA,SAAS;EACT,IAAI,CAAC,6BAA6B;GAYhC,kBAAkB,WAAW;GAC7B,0BAA0B,EAAE,IAAI,SAAS,WAAW,EAAE,MAAM;IAC1D,GAAG;IACH,QAAQ;GACV,CAAC;EACH;EACA,OAAO;GAAC,UAAU;GAAM;GAAW;EAAM;CAC3C,UAAU;EACR,IAAI;GACF,uBAAuB,WAAW;EACpC,SAAS,GAAG;GACV,GAAG,OAAO,yDAAyD;IACjE;IACA,OAAO,OAAO,CAAC;GACjB,CAAC;EACH;EACA,4BAA4B,IAAI,aAAa,oBAAoB;EACjE,0BAA0B,EAAE,IAAI,GAAG;GAAC,GAAG;GAAO;EAAM,CAAC;CACvD;AACF;AAEA,SAAS,uBAAuB,aAAqB;CACnD,MAAM,uBAAuB,GAAG,YAAY;CAC5C,aAAa,oBAAoB;CAIjC,MAAM,YAAY,QAAQ,oBAAoB;CAC9C,IAAI,CAAC,WAAW,SAAS,GACvB;CAEF,MAAM,SAAS,GAAG,SAAS,oBAAoB,EAAE;CACjD,KAAK,MAAM,QAAQ,YAAY,SAAS,GAAG;EACzC,IAAI,CAAC,KAAK,WAAW,MAAM,KAAK,CAAC,KAAK,SAAS,MAAM,GACnD;EAEF,MAAM,QAAQ,KAAK,MAAM,OAAO,QAAQ,EAAc;EACtD,IAAI,yBAAyB,KAAK,KAAK,GACrC,OAAO,KAAK,WAAW,IAAI,GAAG,EAAC,OAAO,KAAI,CAAC;CAE/C;AACF;AAEA,SAAS,4BACP,IACA,aACA,OACA;CACA,MAAM,YAAY,QAAQ,WAAW;CACrC,IAAI;EACF,MAAM,QAAQ,YAAY,SAAS,EAAE,KAAK;EAC1C,MAAM,UAAU,MACb,MAAM,GAAG,oCAAoC,EAC7C,KAAI,UAAS;GAAC;GAAM,WAAW,SAAS,KAAK,WAAW,IAAI,CAAC,EAAE;EAAI,EAAE;EACxE,GAAG,OAAO,yCAAyC;GACjD;GACA;GACA;GACA,YAAY,MAAM;GAClB,mBAAmB,KAAK,IACtB,GACA,MAAM,SAAS,oCACjB;GACA;EACF,CAAC;CACH,SAAS,GAAG;EACV,GAAG,OAAO,kDAAkD;GAC1D;GACA;GACA;GACA,OAAO,OAAO,CAAC;EACjB,CAAC;CACH;AACF;AAEA,SAAS,eACP,IACA,SACA,aACA;CACA,IAAI;CACJ,IAAI;EAGF,KAAK,IAAI,SAAS,IAAI,OAAO;EAC7B,MAAM,EAAC,gBAAgB,cAAa,qBAClC,IAAI,gBAAgB,EAAE,CACxB;EACA,IAAI,aAAa;GACf,IAAI,mBAAmB,YAAY,gBAAgB;IACjD,GAAG,OACD,yBAAyB,eAAe,0CAA0C,YAAY,kBAC9F,WACF;IACA,OAAO;GACT;GACA,IAAI,YAAY,YAAY,cAAc;IACxC,GAAG,OACD,2BAA2B,UAAU,gCAAgC,YAAY,cACnF;IACA,OAAO;GACT;GACA,GAAG,OACD,4BAA4B,eAAe,iBAAiB,UAAU,iBACtE,WACF;EACF;EACA,OAAO;CACT,SAAS,GAAG;EACV,IAAI,mBAAmB,CAAC,GACtB,+BAA+B,IAAI,oBAAoB,SAAS,CAAC;EAEnE,GAAG,QAAQ,2CAA2C,CAAC;EACvD,OAAO;CACT,UAAU;EACR,IAAI,MAAM;CACZ;AACF;AAEA,SAAgB,0BACd,IACA,QACA,aACc;CACd,MAAM,EAAC,YAAY,QAAO,cAAc,aAAa,QAAQ,WAAW;CACxE,MAAM,QAAQ,mCAAmC,MAAM;CACvD,GAAG,OAAO,iCAAiC,OAAO,WAAW;CAC7D,MAAM,QAAQ,YAAY,IAAI;CAC9B,MAAM,OAAO,MAAM,YAAY,CAAC,WAAW,GAAG;EAC5C;EACA,OAAO;EACP,aAAa;CACf,CAAC;CACD,IAAI,WAAW;CACf,MAAM,UAAU,WAA4C;EAC1D,IAAI,UACF;EAEF,WAAW;EACX,MAAM,SAAS;GAAC,GAAG;GAAO;EAAM;EAChC,4BAA4B,EAAE,IAAI,GAAG,MAAM;EAC3C,gCAAgC,EAAE,SAChC,YAAY,IAAI,IAAI,OACpB,MACF;CACF;CACA,KAAK,GAAG,UAAS,MAAK;EACpB,GAAG,OAAO,mCAAmC,CAAC;EAC9C,OAAO,OAAO;CAChB,CAAC;CACD,KAAK,GAAG,UAAU,MAAM,WAAW;EACjC,IAAI,QAAQ;GACV,GAAG,OAAO,qCAAqC,EAAC,OAAM,CAAC;GACvD,OAAO,SAAS;EAClB,OAAO,IAAI,SAAS,GAClB,OAAO,SAAS;OACX;GACL,GAAG,OAAO,8CAA8C,MAAM;GAC9D,OAAO,OAAO;EAChB;CACF,CAAC;CACD,OAAO;AACT;AAIA,IAAM,yBAAyB;AAE/B,IAAM,OAAO;;;;;;;;;;;;;;;AAgBb,eAAsB,kBACpB,IACA,QACA,aACe;CACf,MAAM,CAAC,WAAW,OAAO,MAAM,QAAQ,IAAI,CACzC,uBAAuB,IAAI,QAAQ,aAAa,WAAW,GAC3D,uBAAuB,IAAI,QAAQ,aAAa,KAAK,CACvD,CAAC;CACD,MAAM,QAAQ,CAAC,GAAG,WAAW,GAAG,GAAG;CACnC,IAAI,MAAM,WAAW,GAKnB,MAAM,IAAI,MACR,0CAA0C,OAAO,WACnD;CAEF,OAAO,IAAI,KAAK,KAAK,IAAI,GAAG,MAAM,KAAI,SAAQ,KAAK,QAAQ,CAAC,CAAC,CAAC;AAChE;;;;;;;;;;;;AAaA,eAAe,uBACb,IACA,QACA,aACA,SACiB;CACjB,MAAM,QAAQ,YAAY,IAAI;CAC9B,IAAI,SAAoD;CACxD,MAAM,EAAC,YAAY,QAAO,cAAc,aAAa,QAAQ,WAAW;CACxE,MAAM,OAAO,MAAM,YAAY,CAAC,SAAS,WAAW,GAAG;EACrD;EACA,OAAO;GAAC;GAAU;GAAQ;EAAS;EACnC,aAAa;CACf,CAAC;CACD,MAAM,EAAC,SAAS,SAAS,WAAU,SAAiB;CACpD,IAAI,SAAS;CACb,KAAK,OAAO,YAAY,OAAO;CAC/B,KAAK,OAAO,GAAG,SAAQ,UAAU,UAAU,KAAM;CACjD,KAAK,GAAG,SAAS,MAAM;CACvB,KAAK,GAAG,UAAU,MAAM,WAAW;EACjC,IAAI,QACF,uBAAO,IAAI,MAAM,cAAc,QAAQ,eAAe,QAAQ,CAAC;OAC1D,IAAI,SAAS,GAClB,uBAAO,IAAI,MAAM,cAAc,QAAQ,oBAAoB,MAAM,CAAC;OAElE,QAAQ,MAAM;CAElB,CAAC;CACD,MAAM,UAAU,iBAAiB;EAC/B,SAAS;EACT,uBAAO,IAAI,MAAM,8CAA8C,QAAQ,EAAE,CAAC;EAC1E,KAAK,KAAK,SAAS;CACrB,GAAG,sBAAsB;CAEzB,IAAI;EAEF,MAAM,QAAQ,wBAAwB,IAAI,SAAS,MAD9B,OACoC;EACzD,SAAS,MAAM,SAAS,YAAY;EACpC,OAAO;CACT,UAAU;EACR,aAAa,OAAO;EACpB,6BAA6B,EAAE,SAAS,YAAY,IAAI,IAAI,OAAO;GACjE,GAAG,4BAA4B,MAAM;GACrC;GACA;EACF,CAAC;CACH;AACF;;;;;;;;AASA,SAAgB,wBACd,IACA,SACA,QACQ;CACR,MAAM,QAAgB,CAAC;CACvB,KAAK,MAAM,QAAQ,OAAO,MAAM,IAAI,GAAG;EACrC,MAAM,OAAO,KAAK,KAAK,EAAE,MAAM,IAAI;EACnC,MAAM,UAAU,KAAK,GAAG,EAAE;EAC1B,IACE,KAAK,SAAS,KACd,YAAY,KAAA,KACZ,YAAY,WAEZ;EAEF,MAAM,OAAO,IAAI,KAAK,OAAO;EAC7B,IAAI,OAAO,MAAM,KAAK,QAAQ,CAAC,GAAG;GAChC,GAAG,OAAO,iCAAiC,QAAQ,WAAW,MAAM;GACpE;EACF;EACA,MAAM,KAAK,IAAI;CACjB;CACA,OAAO;AACT"}
|
|
1
|
+
{"version":3,"file":"commands.js","names":[],"sources":["../../../../../../zero-cache/src/services/litestream/commands.ts"],"sourcesContent":["import type {ChildProcess} from 'node:child_process';\nimport {spawn} from 'node:child_process';\nimport {existsSync, readdirSync, rmSync, statSync} from 'node:fs';\nimport {basename, dirname, join} from 'node:path';\nimport type {LogContext, LogLevel} from '@rocicorp/logger';\nimport {resolver} from '@rocicorp/resolver';\nimport {must} from '../../../../shared/src/must.ts';\nimport {Database} from '../../../../zqlite/src/db.ts';\nimport {type LitestreamConfig} from '../../config/normalize.ts';\nimport {deleteLiteDB} from '../../db/delete-lite-db.ts';\nimport {\n isSQLiteCorruption,\n logSQLiteCorruptionDiagnostics,\n} from '../../db/sqlite-corruption.ts';\nimport {StatementRunner} from '../../db/statements.ts';\nimport {deleteChangeLogDB} from '../replicator/change-log-db.ts';\nimport {getSubscriptionState} from '../replicator/schema/replication-state.ts';\nimport {litestreamSocketPath} from './litestream-controller.ts';\nimport {\n litestreamBackupListDuration,\n litestreamBackupMetricAttrs,\n litestreamBackupProcessDuration,\n litestreamBackupProcessMetricAttrs,\n litestreamBackupProcessRuns,\n litestreamRestoreAttempts,\n litestreamRestoredDbBytes,\n litestreamRestoreMetricAttrs,\n litestreamRestoreProcessDuration,\n litestreamRestoreValidationDuration,\n type LitestreamRole,\n} from './metrics.ts';\n\nexport type ReplicaConstraints = {\n replicaVersion: string;\n minWatermark: string;\n};\n\nexport type RestoreResult =\n | 'success'\n | 'no_backup'\n | 'invalid_replica'\n | 'error';\n\ntype RestoreAttempt = {\n restored: boolean;\n backupURL: string | undefined;\n result: RestoreResult;\n};\n\nconst MAX_LOGGED_RESTORE_DIRECTORY_ENTRIES = 100;\nconst LEGACY_RESTORE_WAL_INDEX = /^[0-9a-f]{8}$/;\n\nexport class BackupNotFoundException extends Error {\n static readonly name = 'BackupNotFoundException';\n\n constructor(backupURL: string | undefined) {\n super(`backup not found at ${backupURL}`);\n }\n}\n\nfunction getLitestream(\n mode: 'restore' | 'replicate',\n config: LitestreamConfig,\n replicaFile: string,\n logLevelOverride?: LogLevel,\n): {\n litestream: string;\n env: NodeJS.ProcessEnv;\n} {\n const {\n executable,\n executableV5,\n restoreUsingV5,\n backupUsingV5,\n backupURL,\n logLevel,\n configPath,\n configPathV5,\n endpoint,\n region,\n port,\n checkpointThresholdMB,\n // Note: This assumes the default page size. If page size configuration\n // is added, this computation must be adjusted accordingly.\n minCheckpointPageCount = checkpointThresholdMB * 256, // 1 MiB / 4 KiB page size\n maxCheckpointPageCount = minCheckpointPageCount * 10,\n forceCheckpointThresholdMB,\n incrementalBackupIntervalMinutes,\n incrementalBackupIntervalSeconds,\n snapshotBackupIntervalHours,\n snapshotBackupIntervalHoursV5,\n multipartConcurrency,\n multipartSize,\n } = config;\n\n const v5 =\n (mode === 'restore' && restoreUsingV5) ||\n (mode === 'replicate' && backupUsingV5);\n const litestream =\n (v5 ? executableV5 : executable) ??\n must(executable, `Missing --litestream-executable`);\n const litestreamConfig = v5 ? configPathV5 : configPath;\n const snapshotIntervalHours = v5\n ? snapshotBackupIntervalHoursV5\n : snapshotBackupIntervalHours;\n // Disable truncate-page-n if forced checkpoints are enabled,\n // and otherwise use litestream's default.\n const truncatePageN = forceCheckpointThresholdMB ? -1 : 121359;\n\n return {\n litestream,\n env: {\n ...process.env,\n ['ZERO_REPLICA_FILE']: replicaFile,\n ['ZERO_LITESTREAM_BACKUP_URL']: must(backupURL),\n ['ZERO_LITESTREAM_MIN_CHECKPOINT_PAGE_COUNT']: String(\n minCheckpointPageCount,\n ),\n ['ZERO_LITESTREAM_MAX_CHECKPOINT_PAGE_COUNT']: String(\n maxCheckpointPageCount,\n ),\n ['ZERO_LITESTREAM_INCREMENTAL_BACKUP_INTERVAL_MINUTES']: String(\n incrementalBackupIntervalMinutes, // v3 only\n ),\n ['ZERO_LITESTREAM_INCREMENTAL_BACKUP_INTERVAL_SECONDS']: String(\n incrementalBackupIntervalSeconds, // v5 only\n ),\n ['ZERO_LITESTREAM_TRUNCATE_PAGE_N']: String(truncatePageN),\n ['ZERO_LITESTREAM_LOG_LEVEL']: logLevelOverride ?? logLevel,\n ['ZERO_LITESTREAM_SNAPSHOT_BACKUP_INTERVAL_HOURS']: String(\n snapshotIntervalHours,\n ),\n ['ZERO_LITESTREAM_SNAPSHOT_RETENTION_INTERVAL_HOURS']: String(\n snapshotIntervalHours + 6, // delete old snapshots after 6 hours\n ),\n ['ZERO_LITESTREAM_MULTIPART_CONCURRENCY']: String(multipartConcurrency),\n ['ZERO_LITESTREAM_MULTIPART_SIZE']: String(multipartSize),\n ['ZERO_LOG_FORMAT']: 'json',\n ['ZERO_LITESTREAM_SOCKET_PATH']: litestreamSocketPath(replicaFile),\n ['LITESTREAM_CONFIG']: litestreamConfig,\n ['LITESTREAM_PORT']: String(port),\n ...(endpoint ? {['ZERO_LITESTREAM_ENDPOINT']: endpoint} : {}),\n ...(region ? {['ZERO_LITESTREAM_REGION']: region} : {}),\n },\n };\n}\n\nexport async function tryRestore(\n lc: LogContext,\n config: LitestreamConfig,\n replicaFile: string,\n replicaConstraints: ReplicaConstraints | undefined,\n role: LitestreamRole,\n): Promise<RestoreAttempt> {\n const {backupURL} = config;\n const attrs = litestreamRestoreMetricAttrs(config, role, backupURL);\n let result: RestoreResult = 'error';\n try {\n logRestoreDirectoryContents(lc, replicaFile, 'before-temp-cleanup');\n deleteRestoreTempFiles(replicaFile);\n const replicaExistedBeforeRestore = existsSync(replicaFile);\n const {litestream, env} = getLitestream(\n 'restore',\n config,\n replicaFile,\n 'debug', // Include all output from `litestream restore`, as it's minimal.\n );\n const {\n restoreParallelism: parallelism,\n multipartConcurrency,\n multipartSize,\n } = config;\n lc.info?.(`starting litestream restore`, {\n restoreParallelism: parallelism,\n multipartConcurrency,\n multipartSize,\n });\n // Pipe (rather than inherit) litestream's stdout/stderr so that its own\n // `\"level\":\"ERROR\"` output on a failed restore does not go straight to the\n // pod's stdout — where a log-scraper alert would page on it — before our\n // code has decided how to handle the failure. The captured output is\n // included in the thrown error so the caller can log the final outcome.\n // See INC-961.\n const proc = spawn(\n litestream,\n [\n 'restore',\n '-if-db-not-exists',\n '-if-replica-exists',\n '-parallelism',\n String(parallelism),\n replicaFile,\n ],\n {env, stdio: ['ignore', 'pipe', 'pipe'], windowsHide: true},\n );\n let stdout = '';\n let stderr = '';\n proc.stdout.setEncoding('utf-8');\n proc.stderr.setEncoding('utf-8');\n // Output AND capture the stdout and stderr, the former for observability\n // and the latter for error handling.\n proc.stdout.on('data', chunk => {\n process.stdout.write(chunk);\n stdout += chunk;\n });\n proc.stderr.on('data', chunk => {\n process.stderr.write(chunk);\n stderr += chunk;\n });\n const {promise, resolve, reject} = resolver();\n proc.on('error', reject);\n proc.on('close', (code, signal) => {\n if (signal) {\n reject(`litestream killed with ${signal}`);\n } else if (code !== 0) {\n reject(`litestream exited with code ${code}`);\n } else {\n resolve();\n }\n });\n const processStart = performance.now();\n try {\n await promise;\n litestreamRestoreProcessDuration().recordMs(\n performance.now() - processStart,\n {...attrs, result: 'success'},\n );\n } catch (e) {\n litestreamRestoreProcessDuration().recordMs(\n performance.now() - processStart,\n {...attrs, result: 'error'},\n );\n const output = [stdout, stderr]\n .map(value => value.trim())\n .filter(Boolean)\n .join('\\n');\n throw new Error(output ? `${String(e)}\\n${output}` : String(e), {\n cause: e,\n });\n }\n if (!existsSync(replicaFile)) {\n result = 'no_backup';\n return {restored: false, backupURL, result};\n }\n const validationStart = performance.now();\n const valid = replicaIsValid(lc, replicaFile, replicaConstraints);\n litestreamRestoreValidationDuration().recordMs(\n performance.now() - validationStart,\n {...attrs, result: valid ? 'success' : 'invalid_replica'},\n );\n if (!valid) {\n result = 'invalid_replica';\n lc.info?.(`Deleting local replica and retrying restore`);\n deleteLiteDB(replicaFile);\n deleteChangeLogDB(replicaFile);\n return {restored: false, backupURL, result};\n }\n result = 'success';\n if (!replicaExistedBeforeRestore) {\n // This restore materialized the replica file, so any change log beside it\n // was written against a replica that is no longer there. Reconciliation\n // is the safety net — a restored replica keeps its replicaVersion, so the\n // rule falls through to truncate-or-reseed on head mismatch — but the log\n // is a cache and deleting it here is the explicit statement that nothing\n // in it survives the restore.\n //\n // A restore that reuses an existing replica (`-if-db-not-exists` made it\n // a no-op) deliberately keeps the log: that is the process-restart path,\n // where the log is still the one written beside this exact replica and\n // discarding it would cost every reconnecting subscriber a `too-old`.\n deleteChangeLogDB(replicaFile);\n litestreamRestoredDbBytes().add(statSync(replicaFile).size, {\n ...attrs,\n result: 'success',\n });\n }\n return {restored: true, backupURL, result};\n } finally {\n try {\n deleteRestoreTempFiles(replicaFile);\n } catch (e) {\n lc.warn?.('Unable to clean up Litestream restore temporary files', {\n replicaFile,\n error: String(e),\n });\n }\n logRestoreDirectoryContents(lc, replicaFile, 'after-temp-cleanup');\n litestreamRestoreAttempts().add(1, {...attrs, result});\n }\n}\n\nfunction deleteRestoreTempFiles(replicaFile: string) {\n const temporaryReplicaFile = `${replicaFile}.tmp`;\n deleteLiteDB(temporaryReplicaFile);\n\n // rocicorp/litestream zero@v0.0.9 downloads WAL indexes in parallel to\n // `<output>.tmp-<8 lowercase hex digits>-wal` before applying them.\n const directory = dirname(temporaryReplicaFile);\n if (!existsSync(directory)) {\n return;\n }\n const prefix = `${basename(temporaryReplicaFile)}-`;\n for (const name of readdirSync(directory)) {\n if (!name.startsWith(prefix) || !name.endsWith('-wal')) {\n continue;\n }\n const index = name.slice(prefix.length, -'-wal'.length);\n if (LEGACY_RESTORE_WAL_INDEX.test(index)) {\n rmSync(join(directory, name), {force: true});\n }\n }\n}\n\nfunction logRestoreDirectoryContents(\n lc: LogContext,\n replicaFile: string,\n phase: 'before-temp-cleanup' | 'after-temp-cleanup',\n) {\n const directory = dirname(replicaFile);\n try {\n const names = readdirSync(directory).sort();\n const entries = names\n .slice(0, MAX_LOGGED_RESTORE_DIRECTORY_ENTRIES)\n .map(name => ({name, sizeBytes: statSync(join(directory, name)).size}));\n lc.info?.('Litestream restore directory contents', {\n phase,\n directory,\n replicaFile,\n entryCount: names.length,\n omittedEntryCount: Math.max(\n 0,\n names.length - MAX_LOGGED_RESTORE_DIRECTORY_ENTRIES,\n ),\n entries,\n });\n } catch (e) {\n lc.warn?.('Unable to inspect Litestream restore directory', {\n phase,\n directory,\n replicaFile,\n error: String(e),\n });\n }\n}\n\nfunction replicaIsValid(\n lc: LogContext,\n replica: string,\n constraints: ReplicaConstraints | undefined,\n) {\n let db: Database | undefined;\n try {\n // Note: Open the database and read the subscription state as a\n // sanity / corruption check, even if there are no constraints.\n db = new Database(lc, replica);\n const {replicaVersion, watermark} = getSubscriptionState(\n new StatementRunner(db),\n );\n if (constraints) {\n if (replicaVersion !== constraints.replicaVersion) {\n lc.warn?.(\n `Local replica version ${replicaVersion} does not match expected replicaVersion ${constraints.replicaVersion}`,\n constraints,\n );\n return false;\n }\n if (watermark < constraints.minWatermark) {\n lc.warn?.(\n `Local replica watermark ${watermark} is earlier than minWatermark ${constraints.minWatermark}`,\n );\n return false;\n }\n lc.info?.(\n `Local replica at version ${replicaVersion} and watermark ${watermark} is compatible`,\n constraints,\n );\n }\n return true;\n } catch (e) {\n if (isSQLiteCorruption(e)) {\n logSQLiteCorruptionDiagnostics(lc, 'restored replica', replica, e);\n }\n lc.error?.('Error while validating restored replica', e);\n return false;\n } finally {\n db?.close();\n }\n}\n\nexport function startReplicaBackupProcess(\n lc: LogContext,\n config: LitestreamConfig,\n replicaFile: string,\n): ChildProcess {\n const {litestream, env} = getLitestream('replicate', config, replicaFile);\n const attrs = litestreamBackupProcessMetricAttrs(config);\n lc.info?.(`starting litestream backup to ${config.backupURL}`);\n const start = performance.now();\n const proc = spawn(litestream, ['replicate'], {\n env,\n stdio: 'inherit',\n windowsHide: true,\n });\n let recorded = false;\n const record = (result: 'success' | 'error' | 'stopped') => {\n if (recorded) {\n return;\n }\n recorded = true;\n const labels = {...attrs, result};\n litestreamBackupProcessRuns().add(1, labels);\n litestreamBackupProcessDuration().recordMs(\n performance.now() - start,\n labels,\n );\n };\n proc.on('error', e => {\n lc.warn?.(`litestream backup process error`, e);\n record('error');\n });\n proc.on('close', (code, signal) => {\n if (signal) {\n lc.info?.(`litestream backup process stopped`, {signal});\n record('stopped');\n } else if (code === 0) {\n record('success');\n } else {\n lc.warn?.(`litestream backup process exited with code ${code}`);\n record('error');\n }\n });\n return proc;\n}\n\n// Listing the backup state requires a few S3 LIST requests, which should\n// normally complete well within this timeout.\nconst LIST_BACKUP_TIMEOUT_MS = 30_000;\n\nconst wsRe = /\\s+/;\n\n/**\n * Returns the time of the most recent object (snapshot or WAL segment)\n * actually uploaded to the backup replica destination, as listed by the\n * bundled litestream CLI (`litestream snapshots` / `litestream wal`).\n *\n * This queries the replica destination (e.g. S3) directly, and thus serves\n * as a source of truth for backup durability. This is in contrast to the\n * `litestream_replica_progress` metric, which is exported when litestream\n * *believes* an upload has succeeded, and has been observed to advance even\n * when nothing is actually written to the destination.\n *\n * Rejects if the backup state cannot be determined (spawn error, non-zero\n * exit, timeout, or empty/unparseable listing).\n */\nexport async function getLastBackupTime(\n lc: LogContext,\n config: LitestreamConfig,\n replicaFile: string,\n): Promise<Date> {\n const [snapshots, wal] = await Promise.all([\n listBackupCreatedTimes(lc, config, replicaFile, 'snapshots'),\n listBackupCreatedTimes(lc, config, replicaFile, 'wal'),\n ]);\n const times = [...snapshots, ...wal];\n if (times.length === 0) {\n // Note: the litestream CLI exits with code 0 and logs listing errors\n // (e.g. S3 failures) to stderr, so an empty listing cannot be\n // distinguished from a failed one. Since a valid backup always contains\n // at least one snapshot, an empty listing is treated as a failure.\n throw new Error(\n `no snapshots or WAL segments listed at ${config.backupURL}`,\n );\n }\n return new Date(Math.max(...times.map(time => time.getTime())));\n}\n\n/**\n * Runs `litestream <snapshots|wal> <replica-file>` with the same config /\n * environment used by the `litestream replicate` process (so that the\n * backupURL, endpoint, region, and credentials are identical), and parses\n * the `created` column (RFC3339) of the tab-formatted output, e.g.:\n *\n * ```\n * replica generation index size created\n * s3 1862f44967b3863f 0 4546445 2026-06-10T01:11:32Z\n * ```\n */\nasync function listBackupCreatedTimes(\n lc: LogContext,\n config: LitestreamConfig,\n replicaFile: string,\n command: 'snapshots' | 'wal',\n): Promise<Date[]> {\n const start = performance.now();\n let result: 'success' | 'empty' | 'timeout' | 'error' = 'error';\n const {litestream, env} = getLitestream('replicate', config, replicaFile);\n const proc = spawn(litestream, [command, replicaFile], {\n env,\n stdio: ['ignore', 'pipe', 'inherit'],\n windowsHide: true,\n });\n const {promise, resolve, reject} = resolver<string>();\n let stdout = '';\n proc.stdout.setEncoding('utf-8');\n proc.stdout.on('data', chunk => (stdout += chunk));\n proc.on('error', reject);\n proc.on('close', (code, signal) => {\n if (signal) {\n reject(new Error(`litestream ${command} killed with ${signal}`));\n } else if (code !== 0) {\n reject(new Error(`litestream ${command} exited with code ${code}`));\n } else {\n resolve(stdout);\n }\n });\n const timeout = setTimeout(() => {\n result = 'timeout';\n reject(new Error(`timed out listing backup state (litestream ${command})`));\n proc.kill('SIGKILL');\n }, LIST_BACKUP_TIMEOUT_MS);\n\n try {\n const output = await promise;\n const times = parseBackupCreatedTimes(lc, command, output);\n result = times.length ? 'success' : 'empty';\n return times;\n } finally {\n clearTimeout(timeout);\n litestreamBackupListDuration().recordMs(performance.now() - start, {\n ...litestreamBackupMetricAttrs(config),\n command,\n result,\n });\n }\n}\n\n/**\n * Parses the `created` column (the last, RFC3339-formatted column) from the\n * tab-formatted output of `litestream snapshots` / `litestream wal`. The\n * header row and any unparseable lines are skipped.\n *\n * Exported for testing.\n */\nexport function parseBackupCreatedTimes(\n lc: LogContext,\n command: 'snapshots' | 'wal',\n output: string,\n): Date[] {\n const times: Date[] = [];\n for (const line of output.split('\\n')) {\n const cols = line.trim().split(wsRe);\n const created = cols.at(-1);\n if (\n cols.length < 2 ||\n created === undefined ||\n created === 'created' /* header row */\n ) {\n continue;\n }\n const time = new Date(created);\n if (Number.isNaN(time.getTime())) {\n lc.warn?.(`unexpected line in litestream ${command} output: ${line}`);\n continue;\n }\n times.push(time);\n }\n return times;\n}\n"],"mappings":";;;;;;;;;;;;;;;AAiDA,IAAM,uCAAuC;AAC7C,IAAM,2BAA2B;AAUjC,SAAS,cACP,MACA,QACA,aACA,kBAIA;CACA,MAAM,EACJ,YACA,cACA,gBACA,eACA,WACA,UACA,YACA,cACA,UACA,QACA,MACA,uBAGA,yBAAyB,wBAAwB,KACjD,yBAAyB,yBAAyB,IAClD,4BACA,kCACA,kCACA,6BACA,+BACA,sBACA,kBACE;CAEJ,MAAM,KACH,SAAS,aAAa,kBACtB,SAAS,eAAe;CAC3B,MAAM,cACH,KAAK,eAAe,eACrB,KAAK,YAAY,iCAAiC;CACpD,MAAM,mBAAmB,KAAK,eAAe;CAC7C,MAAM,wBAAwB,KAC1B,gCACA;CAGJ,MAAM,gBAAgB,6BAA6B,KAAK;CAExD,OAAO;EACL;EACA,KAAK;GACH,GAAG,QAAQ;IACV,sBAAsB;IACtB,+BAA+B,KAAK,SAAS;IAC7C,8CAA8C,OAC7C,sBACF;IACC,8CAA8C,OAC7C,sBACF;IACC,wDAAwD,OACvD,gCACF;IACC,wDAAwD,OACvD,gCACF;IACC,oCAAoC,OAAO,aAAa;IACxD,8BAA8B,oBAAoB;IAClD,mDAAmD,OAClD,qBACF;IACC,sDAAsD,OACrD,wBAAwB,CAC1B;IACC,0CAA0C,OAAO,oBAAoB;IACrE,mCAAmC,OAAO,aAAa;IACvD,oBAAoB;IACpB,gCAAgC,qBAAqB,WAAW;IAChE,sBAAsB;IACtB,oBAAoB,OAAO,IAAI;GAChC,GAAI,WAAW,GAAE,6BAA6B,SAAQ,IAAI,CAAC;GAC3D,GAAI,SAAS,GAAE,2BAA2B,OAAM,IAAI,CAAC;EACvD;CACF;AACF;AAEA,eAAsB,WACpB,IACA,QACA,aACA,oBACA,MACyB;CACzB,MAAM,EAAC,cAAa;CACpB,MAAM,QAAQ,6BAA6B,QAAQ,MAAM,SAAS;CAClE,IAAI,SAAwB;CAC5B,IAAI;EACF,4BAA4B,IAAI,aAAa,qBAAqB;EAClE,uBAAuB,WAAW;EAClC,MAAM,8BAA8B,WAAW,WAAW;EAC1D,MAAM,EAAC,YAAY,QAAO,cACxB,WACA,QACA,aACA,OACF;EACA,MAAM,EACJ,oBAAoB,aACpB,sBACA,kBACE;EACJ,GAAG,OAAO,+BAA+B;GACvC,oBAAoB;GACpB;GACA;EACF,CAAC;EAOD,MAAM,OAAO,MACX,YACA;GACE;GACA;GACA;GACA;GACA,OAAO,WAAW;GAClB;EACF,GACA;GAAC;GAAK,OAAO;IAAC;IAAU;IAAQ;GAAM;GAAG,aAAa;EAAI,CAC5D;EACA,IAAI,SAAS;EACb,IAAI,SAAS;EACb,KAAK,OAAO,YAAY,OAAO;EAC/B,KAAK,OAAO,YAAY,OAAO;EAG/B,KAAK,OAAO,GAAG,SAAQ,UAAS;GAC9B,QAAQ,OAAO,MAAM,KAAK;GAC1B,UAAU;EACZ,CAAC;EACD,KAAK,OAAO,GAAG,SAAQ,UAAS;GAC9B,QAAQ,OAAO,MAAM,KAAK;GAC1B,UAAU;EACZ,CAAC;EACD,MAAM,EAAC,SAAS,SAAS,WAAU,SAAS;EAC5C,KAAK,GAAG,SAAS,MAAM;EACvB,KAAK,GAAG,UAAU,MAAM,WAAW;GACjC,IAAI,QACF,OAAO,0BAA0B,QAAQ;QACpC,IAAI,SAAS,GAClB,OAAO,+BAA+B,MAAM;QAE5C,QAAQ;EAEZ,CAAC;EACD,MAAM,eAAe,YAAY,IAAI;EACrC,IAAI;GACF,MAAM;GACN,iCAAiC,EAAE,SACjC,YAAY,IAAI,IAAI,cACpB;IAAC,GAAG;IAAO,QAAQ;GAAS,CAC9B;EACF,SAAS,GAAG;GACV,iCAAiC,EAAE,SACjC,YAAY,IAAI,IAAI,cACpB;IAAC,GAAG;IAAO,QAAQ;GAAO,CAC5B;GACA,MAAM,SAAS,CAAC,QAAQ,MAAM,EAC3B,KAAI,UAAS,MAAM,KAAK,CAAC,EACzB,OAAO,OAAO,EACd,KAAK,IAAI;GACZ,MAAM,IAAI,MAAM,SAAS,GAAG,OAAO,CAAC,EAAE,IAAI,WAAW,OAAO,CAAC,GAAG,EAC9D,OAAO,EACT,CAAC;EACH;EACA,IAAI,CAAC,WAAW,WAAW,GAAG;GAC5B,SAAS;GACT,OAAO;IAAC,UAAU;IAAO;IAAW;GAAM;EAC5C;EACA,MAAM,kBAAkB,YAAY,IAAI;EACxC,MAAM,QAAQ,eAAe,IAAI,aAAa,kBAAkB;EAChE,oCAAoC,EAAE,SACpC,YAAY,IAAI,IAAI,iBACpB;GAAC,GAAG;GAAO,QAAQ,QAAQ,YAAY;EAAiB,CAC1D;EACA,IAAI,CAAC,OAAO;GACV,SAAS;GACT,GAAG,OAAO,6CAA6C;GACvD,aAAa,WAAW;GACxB,kBAAkB,WAAW;GAC7B,OAAO;IAAC,UAAU;IAAO;IAAW;GAAM;EAC5C;EACA,SAAS;EACT,IAAI,CAAC,6BAA6B;GAYhC,kBAAkB,WAAW;GAC7B,0BAA0B,EAAE,IAAI,SAAS,WAAW,EAAE,MAAM;IAC1D,GAAG;IACH,QAAQ;GACV,CAAC;EACH;EACA,OAAO;GAAC,UAAU;GAAM;GAAW;EAAM;CAC3C,UAAU;EACR,IAAI;GACF,uBAAuB,WAAW;EACpC,SAAS,GAAG;GACV,GAAG,OAAO,yDAAyD;IACjE;IACA,OAAO,OAAO,CAAC;GACjB,CAAC;EACH;EACA,4BAA4B,IAAI,aAAa,oBAAoB;EACjE,0BAA0B,EAAE,IAAI,GAAG;GAAC,GAAG;GAAO;EAAM,CAAC;CACvD;AACF;AAEA,SAAS,uBAAuB,aAAqB;CACnD,MAAM,uBAAuB,GAAG,YAAY;CAC5C,aAAa,oBAAoB;CAIjC,MAAM,YAAY,QAAQ,oBAAoB;CAC9C,IAAI,CAAC,WAAW,SAAS,GACvB;CAEF,MAAM,SAAS,GAAG,SAAS,oBAAoB,EAAE;CACjD,KAAK,MAAM,QAAQ,YAAY,SAAS,GAAG;EACzC,IAAI,CAAC,KAAK,WAAW,MAAM,KAAK,CAAC,KAAK,SAAS,MAAM,GACnD;EAEF,MAAM,QAAQ,KAAK,MAAM,OAAO,QAAQ,EAAc;EACtD,IAAI,yBAAyB,KAAK,KAAK,GACrC,OAAO,KAAK,WAAW,IAAI,GAAG,EAAC,OAAO,KAAI,CAAC;CAE/C;AACF;AAEA,SAAS,4BACP,IACA,aACA,OACA;CACA,MAAM,YAAY,QAAQ,WAAW;CACrC,IAAI;EACF,MAAM,QAAQ,YAAY,SAAS,EAAE,KAAK;EAC1C,MAAM,UAAU,MACb,MAAM,GAAG,oCAAoC,EAC7C,KAAI,UAAS;GAAC;GAAM,WAAW,SAAS,KAAK,WAAW,IAAI,CAAC,EAAE;EAAI,EAAE;EACxE,GAAG,OAAO,yCAAyC;GACjD;GACA;GACA;GACA,YAAY,MAAM;GAClB,mBAAmB,KAAK,IACtB,GACA,MAAM,SAAS,oCACjB;GACA;EACF,CAAC;CACH,SAAS,GAAG;EACV,GAAG,OAAO,kDAAkD;GAC1D;GACA;GACA;GACA,OAAO,OAAO,CAAC;EACjB,CAAC;CACH;AACF;AAEA,SAAS,eACP,IACA,SACA,aACA;CACA,IAAI;CACJ,IAAI;EAGF,KAAK,IAAI,SAAS,IAAI,OAAO;EAC7B,MAAM,EAAC,gBAAgB,cAAa,qBAClC,IAAI,gBAAgB,EAAE,CACxB;EACA,IAAI,aAAa;GACf,IAAI,mBAAmB,YAAY,gBAAgB;IACjD,GAAG,OACD,yBAAyB,eAAe,0CAA0C,YAAY,kBAC9F,WACF;IACA,OAAO;GACT;GACA,IAAI,YAAY,YAAY,cAAc;IACxC,GAAG,OACD,2BAA2B,UAAU,gCAAgC,YAAY,cACnF;IACA,OAAO;GACT;GACA,GAAG,OACD,4BAA4B,eAAe,iBAAiB,UAAU,iBACtE,WACF;EACF;EACA,OAAO;CACT,SAAS,GAAG;EACV,IAAI,mBAAmB,CAAC,GACtB,+BAA+B,IAAI,oBAAoB,SAAS,CAAC;EAEnE,GAAG,QAAQ,2CAA2C,CAAC;EACvD,OAAO;CACT,UAAU;EACR,IAAI,MAAM;CACZ;AACF;AAEA,SAAgB,0BACd,IACA,QACA,aACc;CACd,MAAM,EAAC,YAAY,QAAO,cAAc,aAAa,QAAQ,WAAW;CACxE,MAAM,QAAQ,mCAAmC,MAAM;CACvD,GAAG,OAAO,iCAAiC,OAAO,WAAW;CAC7D,MAAM,QAAQ,YAAY,IAAI;CAC9B,MAAM,OAAO,MAAM,YAAY,CAAC,WAAW,GAAG;EAC5C;EACA,OAAO;EACP,aAAa;CACf,CAAC;CACD,IAAI,WAAW;CACf,MAAM,UAAU,WAA4C;EAC1D,IAAI,UACF;EAEF,WAAW;EACX,MAAM,SAAS;GAAC,GAAG;GAAO;EAAM;EAChC,4BAA4B,EAAE,IAAI,GAAG,MAAM;EAC3C,gCAAgC,EAAE,SAChC,YAAY,IAAI,IAAI,OACpB,MACF;CACF;CACA,KAAK,GAAG,UAAS,MAAK;EACpB,GAAG,OAAO,mCAAmC,CAAC;EAC9C,OAAO,OAAO;CAChB,CAAC;CACD,KAAK,GAAG,UAAU,MAAM,WAAW;EACjC,IAAI,QAAQ;GACV,GAAG,OAAO,qCAAqC,EAAC,OAAM,CAAC;GACvD,OAAO,SAAS;EAClB,OAAO,IAAI,SAAS,GAClB,OAAO,SAAS;OACX;GACL,GAAG,OAAO,8CAA8C,MAAM;GAC9D,OAAO,OAAO;EAChB;CACF,CAAC;CACD,OAAO;AACT;AAIA,IAAM,yBAAyB;AAE/B,IAAM,OAAO;;;;;;;;;;;;;;;AAgBb,eAAsB,kBACpB,IACA,QACA,aACe;CACf,MAAM,CAAC,WAAW,OAAO,MAAM,QAAQ,IAAI,CACzC,uBAAuB,IAAI,QAAQ,aAAa,WAAW,GAC3D,uBAAuB,IAAI,QAAQ,aAAa,KAAK,CACvD,CAAC;CACD,MAAM,QAAQ,CAAC,GAAG,WAAW,GAAG,GAAG;CACnC,IAAI,MAAM,WAAW,GAKnB,MAAM,IAAI,MACR,0CAA0C,OAAO,WACnD;CAEF,OAAO,IAAI,KAAK,KAAK,IAAI,GAAG,MAAM,KAAI,SAAQ,KAAK,QAAQ,CAAC,CAAC,CAAC;AAChE;;;;;;;;;;;;AAaA,eAAe,uBACb,IACA,QACA,aACA,SACiB;CACjB,MAAM,QAAQ,YAAY,IAAI;CAC9B,IAAI,SAAoD;CACxD,MAAM,EAAC,YAAY,QAAO,cAAc,aAAa,QAAQ,WAAW;CACxE,MAAM,OAAO,MAAM,YAAY,CAAC,SAAS,WAAW,GAAG;EACrD;EACA,OAAO;GAAC;GAAU;GAAQ;EAAS;EACnC,aAAa;CACf,CAAC;CACD,MAAM,EAAC,SAAS,SAAS,WAAU,SAAiB;CACpD,IAAI,SAAS;CACb,KAAK,OAAO,YAAY,OAAO;CAC/B,KAAK,OAAO,GAAG,SAAQ,UAAU,UAAU,KAAM;CACjD,KAAK,GAAG,SAAS,MAAM;CACvB,KAAK,GAAG,UAAU,MAAM,WAAW;EACjC,IAAI,QACF,uBAAO,IAAI,MAAM,cAAc,QAAQ,eAAe,QAAQ,CAAC;OAC1D,IAAI,SAAS,GAClB,uBAAO,IAAI,MAAM,cAAc,QAAQ,oBAAoB,MAAM,CAAC;OAElE,QAAQ,MAAM;CAElB,CAAC;CACD,MAAM,UAAU,iBAAiB;EAC/B,SAAS;EACT,uBAAO,IAAI,MAAM,8CAA8C,QAAQ,EAAE,CAAC;EAC1E,KAAK,KAAK,SAAS;CACrB,GAAG,sBAAsB;CAEzB,IAAI;EAEF,MAAM,QAAQ,wBAAwB,IAAI,SAAS,MAD9B,OACoC;EACzD,SAAS,MAAM,SAAS,YAAY;EACpC,OAAO;CACT,UAAU;EACR,aAAa,OAAO;EACpB,6BAA6B,EAAE,SAAS,YAAY,IAAI,IAAI,OAAO;GACjE,GAAG,4BAA4B,MAAM;GACrC;GACA;EACF,CAAC;CACH;AACF;;;;;;;;AASA,SAAgB,wBACd,IACA,SACA,QACQ;CACR,MAAM,QAAgB,CAAC;CACvB,KAAK,MAAM,QAAQ,OAAO,MAAM,IAAI,GAAG;EACrC,MAAM,OAAO,KAAK,KAAK,EAAE,MAAM,IAAI;EACnC,MAAM,UAAU,KAAK,GAAG,EAAE;EAC1B,IACE,KAAK,SAAS,KACd,YAAY,KAAA,KACZ,YAAY,WAEZ;EAEF,MAAM,OAAO,IAAI,KAAK,OAAO;EAC7B,IAAI,OAAO,MAAM,KAAK,QAAQ,CAAC,GAAG;GAChC,GAAG,OAAO,iCAAiC,QAAQ,WAAW,MAAM;GACpE;EACF;EACA,MAAM,KAAK,IAAI;CACjB;CACA,OAAO;AACT"}
|
|
@@ -25,8 +25,13 @@ validation:
|
|
|
25
25
|
|
|
26
26
|
dbs:
|
|
27
27
|
- path: ${ZERO_REPLICA_FILE}
|
|
28
|
-
# The monitor-interval is the interval at which
|
|
28
|
+
# The monitor-interval is the steady-state interval at which the db / wal
|
|
29
|
+
# is checked to create new l0 files and/or checkpoint the wal. The
|
|
30
|
+
# application will also explicitly trigger a "/sync" if the wal reaches
|
|
31
|
+
# ZERO_LITESTREAM_FORCE_CHECKPOINT_THRESHOLD_MB.
|
|
29
32
|
monitor-interval: ${ZERO_LITESTREAM_INCREMENTAL_BACKUP_INTERVAL_SECONDS}s
|
|
33
|
+
min-checkpoint-page-count: ${ZERO_LITESTREAM_MIN_CHECKPOINT_PAGE_COUNT}
|
|
34
|
+
truncate-page-n: ${ZERO_LITESTREAM_TRUNCATE_PAGE_N}
|
|
30
35
|
replica:
|
|
31
36
|
url: ${ZERO_LITESTREAM_BACKUP_URL}
|
|
32
37
|
endpoint: ${ZERO_LITESTREAM_ENDPOINT}
|
|
@@ -34,9 +39,18 @@ dbs:
|
|
|
34
39
|
auto-recover: true
|
|
35
40
|
concurrency: ${ZERO_LITESTREAM_MULTIPART_CONCURRENCY}
|
|
36
41
|
part-size: ${ZERO_LITESTREAM_MULTIPART_SIZE}
|
|
42
|
+
download-concurrency: ${ZERO_LITESTREAM_MULTIPART_CONCURRENCY}
|
|
43
|
+
download-part-size: ${ZERO_LITESTREAM_MULTIPART_SIZE}
|
|
37
44
|
|
|
38
45
|
logging:
|
|
39
46
|
level: ${ZERO_LITESTREAM_LOG_LEVEL}
|
|
40
47
|
type: ${ZERO_LOG_FORMAT}
|
|
41
48
|
|
|
42
49
|
addr: localhost:${LITESTREAM_PORT}
|
|
50
|
+
|
|
51
|
+
# Control API served as HTTP over a Unix domain socket, used for in-process IPC
|
|
52
|
+
# from zero-cache (e.g. forcing an immediate sync/checkpoint via POST /sync).
|
|
53
|
+
socket:
|
|
54
|
+
enabled: true
|
|
55
|
+
path: ${ZERO_LITESTREAM_SOCKET_PATH}
|
|
56
|
+
permissions: 0600
|