@rocicorp/zero 1.10.0-canary.0 → 1.10.0-canary.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/out/replicache/src/db/commit.js +1 -4
- package/out/replicache/src/db/commit.js.map +1 -1
- package/out/replicache/src/db/read.d.ts +1 -0
- package/out/replicache/src/db/read.d.ts.map +1 -1
- package/out/replicache/src/db/read.js +1 -4
- package/out/replicache/src/db/read.js.map +1 -1
- package/out/replicache/src/db/write.d.ts +2 -1
- package/out/replicache/src/db/write.d.ts.map +1 -1
- package/out/replicache/src/db/write.js +4 -51
- package/out/replicache/src/db/write.js.map +1 -1
- package/out/replicache/src/persist/clients.d.ts +1 -0
- package/out/replicache/src/persist/clients.d.ts.map +1 -1
- package/out/replicache/src/persist/clients.js +4 -31
- package/out/replicache/src/persist/clients.js.map +1 -1
- package/out/replicache/src/process-env.d.ts +28 -0
- package/out/replicache/src/process-env.d.ts.map +1 -0
- package/out/replicache/src/replicache-impl.d.ts.map +1 -1
- package/out/replicache/src/replicache-impl.js.map +1 -1
- package/out/replicache/src/scan-iterator.js +2 -14
- package/out/replicache/src/scan-iterator.js.map +1 -1
- package/out/replicache/src/subscriptions.d.ts +1 -0
- package/out/replicache/src/subscriptions.d.ts.map +1 -1
- package/out/replicache/src/subscriptions.js +2 -10
- package/out/replicache/src/subscriptions.js.map +1 -1
- package/out/replicache/src/transactions.d.ts +1 -0
- package/out/replicache/src/transactions.d.ts.map +1 -1
- package/out/replicache/src/transactions.js +2 -4
- package/out/replicache/src/transactions.js.map +1 -1
- package/out/z2s/src/compiler.d.ts.map +1 -1
- package/out/z2s/src/compiler.js +1 -22
- package/out/z2s/src/compiler.js.map +1 -1
- package/out/zero/package.js +3 -3
- package/out/zero/package.js.map +1 -1
- package/out/zero-cache/src/config/normalize.d.ts +1 -0
- package/out/zero-cache/src/config/normalize.d.ts.map +1 -1
- package/out/zero-cache/src/config/normalize.js +10 -1
- package/out/zero-cache/src/config/normalize.js.map +1 -1
- package/out/zero-cache/src/config/zero-config.d.ts +52 -1
- package/out/zero-cache/src/config/zero-config.d.ts.map +1 -1
- package/out/zero-cache/src/config/zero-config.js +171 -22
- package/out/zero-cache/src/config/zero-config.js.map +1 -1
- package/out/zero-cache/src/custom/fetch.js +1 -1
- package/out/zero-cache/src/custom/fetch.js.map +1 -1
- package/out/zero-cache/src/db/delete-lite-db.js +2 -1
- package/out/zero-cache/src/db/delete-lite-db.js.map +1 -1
- package/out/zero-cache/src/db/migration-lite.d.ts.map +1 -1
- package/out/zero-cache/src/db/migration-lite.js +5 -2
- package/out/zero-cache/src/db/migration-lite.js.map +1 -1
- package/out/zero-cache/src/db/migration.d.ts.map +1 -1
- package/out/zero-cache/src/db/migration.js +4 -2
- package/out/zero-cache/src/db/migration.js.map +1 -1
- package/out/zero-cache/src/db/sqlite-corruption.d.ts +3 -2
- package/out/zero-cache/src/db/sqlite-corruption.d.ts.map +1 -1
- package/out/zero-cache/src/db/sqlite-corruption.js +13 -8
- package/out/zero-cache/src/db/sqlite-corruption.js.map +1 -1
- package/out/zero-cache/src/observability/metrics.d.ts +6 -1
- package/out/zero-cache/src/observability/metrics.d.ts.map +1 -1
- package/out/zero-cache/src/observability/metrics.js +20 -2
- package/out/zero-cache/src/observability/metrics.js.map +1 -1
- package/out/zero-cache/src/server/backup-watermark-poller.js +31 -0
- package/out/zero-cache/src/server/backup-watermark-poller.js.map +1 -0
- package/out/zero-cache/src/server/change-streamer.d.ts.map +1 -1
- package/out/zero-cache/src/server/change-streamer.js +91 -31
- package/out/zero-cache/src/server/change-streamer.js.map +1 -1
- package/out/zero-cache/src/server/main.d.ts.map +1 -1
- package/out/zero-cache/src/server/main.js +6 -16
- package/out/zero-cache/src/server/main.js.map +1 -1
- package/out/zero-cache/src/server/replicator.d.ts.map +1 -1
- package/out/zero-cache/src/server/replicator.js +48 -5
- package/out/zero-cache/src/server/replicator.js.map +1 -1
- package/out/zero-cache/src/server/runner/run-worker.js +2 -1
- package/out/zero-cache/src/server/runner/run-worker.js.map +1 -1
- package/out/zero-cache/src/server/syncer.d.ts.map +1 -1
- package/out/zero-cache/src/server/syncer.js +1 -1
- package/out/zero-cache/src/server/syncer.js.map +1 -1
- package/out/zero-cache/src/server/worker-urls.d.ts +1 -1
- package/out/zero-cache/src/server/worker-urls.js +2 -2
- package/out/zero-cache/src/server/worker-urls.js.map +1 -1
- package/out/zero-cache/src/services/change-source/change-source.d.ts +4 -2
- package/out/zero-cache/src/services/change-source/change-source.d.ts.map +1 -1
- package/out/zero-cache/src/services/change-source/common/replica-restore.d.ts +35 -0
- package/out/zero-cache/src/services/change-source/common/replica-restore.d.ts.map +1 -0
- package/out/zero-cache/src/services/change-source/common/replica-restore.js +34 -0
- package/out/zero-cache/src/services/change-source/common/replica-restore.js.map +1 -0
- package/out/zero-cache/src/services/change-source/common/replica-schema.d.ts +2 -0
- package/out/zero-cache/src/services/change-source/common/replica-schema.d.ts.map +1 -1
- package/out/zero-cache/src/services/change-source/common/replica-schema.js +73 -1
- package/out/zero-cache/src/services/change-source/common/replica-schema.js.map +1 -1
- package/out/zero-cache/src/services/change-source/custom/change-source.d.ts +2 -6
- package/out/zero-cache/src/services/change-source/custom/change-source.d.ts.map +1 -1
- package/out/zero-cache/src/services/change-source/custom/change-source.js +14 -5
- package/out/zero-cache/src/services/change-source/custom/change-source.js.map +1 -1
- package/out/zero-cache/src/services/change-source/pg/backfill-stream.js +4 -7
- package/out/zero-cache/src/services/change-source/pg/backfill-stream.js.map +1 -1
- package/out/zero-cache/src/services/change-source/pg/change-source.d.ts +7 -7
- package/out/zero-cache/src/services/change-source/pg/change-source.d.ts.map +1 -1
- package/out/zero-cache/src/services/change-source/pg/change-source.js +82 -25
- package/out/zero-cache/src/services/change-source/pg/change-source.js.map +1 -1
- package/out/zero-cache/src/services/change-source/pg/initial-sync.d.ts +9 -1
- package/out/zero-cache/src/services/change-source/pg/initial-sync.d.ts.map +1 -1
- package/out/zero-cache/src/services/change-source/pg/initial-sync.js +12 -4
- package/out/zero-cache/src/services/change-source/pg/initial-sync.js.map +1 -1
- package/out/zero-cache/src/services/change-source/pg/logical-replication/stream.d.ts +45 -2
- package/out/zero-cache/src/services/change-source/pg/logical-replication/stream.d.ts.map +1 -1
- package/out/zero-cache/src/services/change-source/pg/logical-replication/stream.js +59 -8
- package/out/zero-cache/src/services/change-source/pg/logical-replication/stream.js.map +1 -1
- package/out/zero-cache/src/services/change-source/pg/replication-slots.d.ts +4 -2
- package/out/zero-cache/src/services/change-source/pg/replication-slots.d.ts.map +1 -1
- package/out/zero-cache/src/services/change-source/pg/replication-slots.js +7 -4
- package/out/zero-cache/src/services/change-source/pg/replication-slots.js.map +1 -1
- package/out/zero-cache/src/services/change-source/pg/schema/init.d.ts +0 -5
- package/out/zero-cache/src/services/change-source/pg/schema/init.d.ts.map +1 -1
- package/out/zero-cache/src/services/change-source/pg/schema/init.js +29 -41
- package/out/zero-cache/src/services/change-source/pg/schema/init.js.map +1 -1
- package/out/zero-cache/src/services/change-source/pg/schema/shard.d.ts +35 -16
- package/out/zero-cache/src/services/change-source/pg/schema/shard.d.ts.map +1 -1
- package/out/zero-cache/src/services/change-source/pg/schema/shard.js +77 -7
- package/out/zero-cache/src/services/change-source/pg/schema/shard.js.map +1 -1
- package/out/zero-cache/src/services/change-source/protocol/current/data.d.ts +2 -1
- package/out/zero-cache/src/services/change-source/protocol/current/data.d.ts.map +1 -1
- package/out/zero-cache/src/services/change-source/protocol/current/data.js +5 -13
- package/out/zero-cache/src/services/change-source/protocol/current/data.js.map +1 -1
- package/out/zero-cache/src/services/change-source/protocol/current/downstream.d.ts +6 -3
- package/out/zero-cache/src/services/change-source/protocol/current/downstream.d.ts.map +1 -1
- package/out/zero-cache/src/services/change-source/protocol/current/schema-change-tags.d.ts +6 -0
- package/out/zero-cache/src/services/change-source/protocol/current/schema-change-tags.d.ts.map +1 -0
- package/out/zero-cache/src/services/change-source/protocol/current/schema-change-tags.js +21 -0
- package/out/zero-cache/src/services/change-source/protocol/current/schema-change-tags.js.map +1 -0
- package/out/zero-cache/src/services/change-source/protocol/current/status.d.ts +1 -0
- package/out/zero-cache/src/services/change-source/protocol/current/status.d.ts.map +1 -1
- package/out/zero-cache/src/services/change-source/protocol/current/upstream.d.ts +1 -0
- package/out/zero-cache/src/services/change-source/protocol/current/upstream.d.ts.map +1 -1
- package/out/zero-cache/src/services/change-streamer/backup-cleanup-monitor-factory.d.ts +5 -6
- package/out/zero-cache/src/services/change-streamer/backup-cleanup-monitor-factory.d.ts.map +1 -1
- package/out/zero-cache/src/services/change-streamer/backup-cleanup-monitor-factory.js +32 -9
- package/out/zero-cache/src/services/change-streamer/backup-cleanup-monitor-factory.js.map +1 -1
- package/out/zero-cache/src/services/change-streamer/backup-monitor.d.ts +15 -5
- package/out/zero-cache/src/services/change-streamer/backup-monitor.d.ts.map +1 -1
- package/out/zero-cache/src/services/change-streamer/backup-monitor.js +66 -0
- package/out/zero-cache/src/services/change-streamer/backup-monitor.js.map +1 -0
- package/out/zero-cache/src/services/change-streamer/broadcast.d.ts +68 -78
- package/out/zero-cache/src/services/change-streamer/broadcast.d.ts.map +1 -1
- package/out/zero-cache/src/services/change-streamer/broadcast.js +84 -92
- package/out/zero-cache/src/services/change-streamer/broadcast.js.map +1 -1
- package/out/zero-cache/src/services/change-streamer/change-log-codec.d.ts +19 -0
- package/out/zero-cache/src/services/change-streamer/change-log-codec.d.ts.map +1 -0
- package/out/zero-cache/src/services/change-streamer/change-log-codec.js +50 -0
- package/out/zero-cache/src/services/change-streamer/change-log-codec.js.map +1 -0
- package/out/zero-cache/src/services/change-streamer/change-log-initializer.d.ts +141 -0
- package/out/zero-cache/src/services/change-streamer/change-log-initializer.d.ts.map +1 -0
- package/out/zero-cache/src/services/change-streamer/change-log-initializer.js +270 -0
- package/out/zero-cache/src/services/change-streamer/change-log-initializer.js.map +1 -0
- package/out/zero-cache/src/services/change-streamer/change-log-transaction-hash.d.ts +19 -0
- package/out/zero-cache/src/services/change-streamer/change-log-transaction-hash.d.ts.map +1 -0
- package/out/zero-cache/src/services/change-streamer/change-log-transaction-hash.js +39 -0
- package/out/zero-cache/src/services/change-streamer/change-log-transaction-hash.js.map +1 -0
- package/out/zero-cache/src/services/change-streamer/change-streamer-http.d.ts +3 -4
- package/out/zero-cache/src/services/change-streamer/change-streamer-http.d.ts.map +1 -1
- package/out/zero-cache/src/services/change-streamer/change-streamer-http.js +12 -17
- package/out/zero-cache/src/services/change-streamer/change-streamer-http.js.map +1 -1
- package/out/zero-cache/src/services/change-streamer/change-streamer-service.d.ts +69 -24
- package/out/zero-cache/src/services/change-streamer/change-streamer-service.d.ts.map +1 -1
- package/out/zero-cache/src/services/change-streamer/change-streamer-service.js +333 -66
- package/out/zero-cache/src/services/change-streamer/change-streamer-service.js.map +1 -1
- package/out/zero-cache/src/services/change-streamer/change-streamer.d.ts +56 -13
- package/out/zero-cache/src/services/change-streamer/change-streamer.d.ts.map +1 -1
- package/out/zero-cache/src/services/change-streamer/change-streamer.js.map +1 -1
- package/out/zero-cache/src/services/change-streamer/error-type-enum.d.ts +8 -0
- package/out/zero-cache/src/services/change-streamer/error-type-enum.d.ts.map +1 -1
- package/out/zero-cache/src/services/change-streamer/forwarder.d.ts +14 -3
- package/out/zero-cache/src/services/change-streamer/forwarder.d.ts.map +1 -1
- package/out/zero-cache/src/services/change-streamer/forwarder.js +41 -11
- package/out/zero-cache/src/services/change-streamer/forwarder.js.map +1 -1
- package/out/zero-cache/src/services/change-streamer/{litestream3-backup-monitor.d.ts → litestream3-prometheus-poller.d.ts} +19 -45
- package/out/zero-cache/src/services/change-streamer/litestream3-prometheus-poller.d.ts.map +1 -0
- package/out/zero-cache/src/services/change-streamer/{litestream3-backup-monitor.js → litestream3-prometheus-poller.js} +57 -171
- package/out/zero-cache/src/services/change-streamer/litestream3-prometheus-poller.js.map +1 -0
- package/out/zero-cache/src/services/change-streamer/replica-poller.d.ts +15 -0
- package/out/zero-cache/src/services/change-streamer/replica-poller.d.ts.map +1 -0
- package/out/zero-cache/src/services/change-streamer/replica-poller.js +55 -0
- package/out/zero-cache/src/services/change-streamer/replica-poller.js.map +1 -0
- package/out/zero-cache/src/services/change-streamer/schema/tables.d.ts +5 -1
- package/out/zero-cache/src/services/change-streamer/schema/tables.d.ts.map +1 -1
- package/out/zero-cache/src/services/change-streamer/schema/tables.js +2 -1
- package/out/zero-cache/src/services/change-streamer/schema/tables.js.map +1 -1
- package/out/zero-cache/src/services/change-streamer/snapshot-reservations.d.ts +14 -0
- package/out/zero-cache/src/services/change-streamer/snapshot-reservations.d.ts.map +1 -0
- package/out/zero-cache/src/services/change-streamer/snapshot-reservations.js +86 -0
- package/out/zero-cache/src/services/change-streamer/snapshot-reservations.js.map +1 -0
- package/out/zero-cache/src/services/change-streamer/snapshot.d.ts +4 -1
- package/out/zero-cache/src/services/change-streamer/snapshot.d.ts.map +1 -1
- package/out/zero-cache/src/services/change-streamer/snapshot.js +39 -14
- package/out/zero-cache/src/services/change-streamer/snapshot.js.map +1 -1
- package/out/zero-cache/src/services/change-streamer/sqlite-change-log-catchup.d.ts +75 -0
- package/out/zero-cache/src/services/change-streamer/sqlite-change-log-catchup.d.ts.map +1 -0
- package/out/zero-cache/src/services/change-streamer/sqlite-change-log-catchup.js +261 -0
- package/out/zero-cache/src/services/change-streamer/sqlite-change-log-catchup.js.map +1 -0
- package/out/zero-cache/src/services/change-streamer/sqlite-change-log-purge-scheduler.d.ts +121 -0
- package/out/zero-cache/src/services/change-streamer/sqlite-change-log-purge-scheduler.d.ts.map +1 -0
- package/out/zero-cache/src/services/change-streamer/sqlite-change-log-purge-scheduler.js +391 -0
- package/out/zero-cache/src/services/change-streamer/sqlite-change-log-purge-scheduler.js.map +1 -0
- package/out/zero-cache/src/services/change-streamer/sqlite-change-log-reader.d.ts +62 -0
- package/out/zero-cache/src/services/change-streamer/sqlite-change-log-reader.d.ts.map +1 -0
- package/out/zero-cache/src/services/change-streamer/sqlite-change-log-reader.js +159 -0
- package/out/zero-cache/src/services/change-streamer/sqlite-change-log-reader.js.map +1 -0
- package/out/zero-cache/src/services/change-streamer/sqlite-change-log-writer.d.ts +109 -0
- package/out/zero-cache/src/services/change-streamer/sqlite-change-log-writer.d.ts.map +1 -0
- package/out/zero-cache/src/services/change-streamer/sqlite-change-log-writer.js +258 -0
- package/out/zero-cache/src/services/change-streamer/sqlite-change-log-writer.js.map +1 -0
- package/out/zero-cache/src/services/change-streamer/storer.d.ts +28 -12
- package/out/zero-cache/src/services/change-streamer/storer.d.ts.map +1 -1
- package/out/zero-cache/src/services/change-streamer/storer.js +116 -121
- package/out/zero-cache/src/services/change-streamer/storer.js.map +1 -1
- package/out/zero-cache/src/services/change-streamer/subscriber.d.ts +61 -10
- package/out/zero-cache/src/services/change-streamer/subscriber.d.ts.map +1 -1
- package/out/zero-cache/src/services/change-streamer/subscriber.js +101 -8
- package/out/zero-cache/src/services/change-streamer/subscriber.js.map +1 -1
- package/out/zero-cache/src/services/change-streamer/upstream-acker.d.ts +34 -0
- package/out/zero-cache/src/services/change-streamer/upstream-acker.d.ts.map +1 -0
- package/out/zero-cache/src/services/change-streamer/upstream-acker.js +86 -0
- package/out/zero-cache/src/services/change-streamer/upstream-acker.js.map +1 -0
- package/out/zero-cache/src/services/life-cycle.d.ts +1 -1
- package/out/zero-cache/src/services/life-cycle.d.ts.map +1 -1
- package/out/zero-cache/src/services/life-cycle.js +4 -3
- package/out/zero-cache/src/services/life-cycle.js.map +1 -1
- package/out/zero-cache/src/services/litestream/commands.d.ts +12 -11
- package/out/zero-cache/src/services/litestream/commands.d.ts.map +1 -1
- package/out/zero-cache/src/services/litestream/commands.js +107 -148
- package/out/zero-cache/src/services/litestream/commands.js.map +1 -1
- package/out/zero-cache/src/services/litestream/config-v5.yml +42 -0
- package/out/zero-cache/src/services/litestream/config.yml +3 -2
- package/out/zero-cache/src/services/litestream/metrics.d.ts +4 -4
- package/out/zero-cache/src/services/litestream/metrics.d.ts.map +1 -1
- package/out/zero-cache/src/services/litestream/metrics.js +7 -10
- package/out/zero-cache/src/services/litestream/metrics.js.map +1 -1
- package/out/zero-cache/src/services/litestream/vfs-watermark-poller.d.ts +57 -0
- package/out/zero-cache/src/services/litestream/vfs-watermark-poller.d.ts.map +1 -0
- package/out/zero-cache/src/services/litestream/vfs-watermark-poller.js +137 -0
- package/out/zero-cache/src/services/litestream/vfs-watermark-poller.js.map +1 -0
- package/out/zero-cache/src/services/mutagen/mutagen.d.ts.map +1 -1
- package/out/zero-cache/src/services/mutagen/mutagen.js +4 -1
- package/out/zero-cache/src/services/mutagen/mutagen.js.map +1 -1
- package/out/zero-cache/src/services/replicator/change-log-cookies.d.ts +169 -0
- package/out/zero-cache/src/services/replicator/change-log-cookies.d.ts.map +1 -0
- package/out/zero-cache/src/services/replicator/change-log-cookies.js +434 -0
- package/out/zero-cache/src/services/replicator/change-log-cookies.js.map +1 -0
- package/out/zero-cache/src/services/replicator/change-log-db.d.ts +358 -0
- package/out/zero-cache/src/services/replicator/change-log-db.d.ts.map +1 -0
- package/out/zero-cache/src/services/replicator/change-log-db.js +477 -0
- package/out/zero-cache/src/services/replicator/change-log-db.js.map +1 -0
- package/out/zero-cache/src/services/replicator/change-log-stream-writer.d.ts +55 -0
- package/out/zero-cache/src/services/replicator/change-log-stream-writer.d.ts.map +1 -0
- package/out/zero-cache/src/services/replicator/change-log-stream-writer.js +150 -0
- package/out/zero-cache/src/services/replicator/change-log-stream-writer.js.map +1 -0
- package/out/zero-cache/src/services/replicator/change-processor.d.ts +6 -0
- package/out/zero-cache/src/services/replicator/change-processor.d.ts.map +1 -1
- package/out/zero-cache/src/services/replicator/change-processor.js +70 -25
- package/out/zero-cache/src/services/replicator/change-processor.js.map +1 -1
- package/out/zero-cache/src/services/replicator/incremental-sync.d.ts +1 -1
- package/out/zero-cache/src/services/replicator/incremental-sync.d.ts.map +1 -1
- package/out/zero-cache/src/services/replicator/incremental-sync.js +19 -6
- package/out/zero-cache/src/services/replicator/incremental-sync.js.map +1 -1
- package/out/zero-cache/src/services/replicator/notifier.d.ts +0 -22
- package/out/zero-cache/src/services/replicator/notifier.d.ts.map +1 -1
- package/out/zero-cache/src/services/replicator/notifier.js +7 -1
- package/out/zero-cache/src/services/replicator/notifier.js.map +1 -1
- package/out/zero-cache/src/services/replicator/replicator.d.ts +17 -2
- package/out/zero-cache/src/services/replicator/replicator.d.ts.map +1 -1
- package/out/zero-cache/src/services/replicator/replicator.js +9 -3
- package/out/zero-cache/src/services/replicator/replicator.js.map +1 -1
- package/out/zero-cache/src/services/replicator/reporter/recorder.d.ts +27 -0
- package/out/zero-cache/src/services/replicator/reporter/recorder.d.ts.map +1 -1
- package/out/zero-cache/src/services/replicator/reporter/recorder.js +32 -0
- package/out/zero-cache/src/services/replicator/reporter/recorder.js.map +1 -1
- package/out/zero-cache/src/services/replicator/reporter/report-schema.d.ts +1 -1
- package/out/zero-cache/src/services/replicator/schema/backfilling.d.ts +119 -0
- package/out/zero-cache/src/services/replicator/schema/backfilling.d.ts.map +1 -0
- package/out/zero-cache/src/services/replicator/schema/backfilling.js +214 -0
- package/out/zero-cache/src/services/replicator/schema/backfilling.js.map +1 -0
- package/out/zero-cache/src/services/replicator/schema/change-log.d.ts +3 -3
- package/out/zero-cache/src/services/replicator/schema/column-metadata.js +1 -1
- package/out/zero-cache/src/services/replicator/schema/column-metadata.js.map +1 -1
- package/out/zero-cache/src/services/replicator/schema/replication-state.d.ts +1 -1
- package/out/zero-cache/src/services/replicator/schema/replication-state.d.ts.map +1 -1
- package/out/zero-cache/src/services/replicator/schema/replication-state.js +9 -5
- package/out/zero-cache/src/services/replicator/schema/replication-state.js.map +1 -1
- package/out/zero-cache/src/services/replicator/sqlite-change-log-observability.d.ts +131 -0
- package/out/zero-cache/src/services/replicator/sqlite-change-log-observability.d.ts.map +1 -0
- package/out/zero-cache/src/services/replicator/sqlite-change-log-observability.js +413 -0
- package/out/zero-cache/src/services/replicator/sqlite-change-log-observability.js.map +1 -0
- package/out/zero-cache/src/services/replicator/sqlite-change-log-purger.d.ts +120 -0
- package/out/zero-cache/src/services/replicator/sqlite-change-log-purger.d.ts.map +1 -0
- package/out/zero-cache/src/services/replicator/sqlite-change-log-purger.js +283 -0
- package/out/zero-cache/src/services/replicator/sqlite-change-log-purger.js.map +1 -0
- package/out/zero-cache/src/services/replicator/write-worker.js +23 -12
- package/out/zero-cache/src/services/replicator/write-worker.js.map +1 -1
- package/out/zero-cache/src/services/run-ast.d.ts.map +1 -1
- package/out/zero-cache/src/services/run-ast.js +2 -1
- package/out/zero-cache/src/services/run-ast.js.map +1 -1
- package/out/zero-cache/src/services/running-state.d.ts +1 -1
- package/out/zero-cache/src/services/running-state.d.ts.map +1 -1
- package/out/zero-cache/src/services/running-state.js +1 -0
- package/out/zero-cache/src/services/running-state.js.map +1 -1
- package/out/zero-cache/src/services/view-syncer/e2e-serving-lag.d.ts +53 -0
- package/out/zero-cache/src/services/view-syncer/e2e-serving-lag.d.ts.map +1 -0
- package/out/zero-cache/src/services/view-syncer/e2e-serving-lag.js +68 -0
- package/out/zero-cache/src/services/view-syncer/e2e-serving-lag.js.map +1 -0
- package/out/zero-cache/src/services/view-syncer/pipeline-driver.d.ts.map +1 -1
- package/out/zero-cache/src/services/view-syncer/pipeline-driver.js +18 -3
- package/out/zero-cache/src/services/view-syncer/pipeline-driver.js.map +1 -1
- package/out/zero-cache/src/services/view-syncer/snapshotter.d.ts.map +1 -1
- package/out/zero-cache/src/services/view-syncer/snapshotter.js.map +1 -1
- package/out/zero-cache/src/services/view-syncer/view-syncer.d.ts +11 -1
- package/out/zero-cache/src/services/view-syncer/view-syncer.d.ts.map +1 -1
- package/out/zero-cache/src/services/view-syncer/view-syncer.js +64 -6
- package/out/zero-cache/src/services/view-syncer/view-syncer.js.map +1 -1
- package/out/zero-cache/src/types/processes.d.ts +1 -2
- package/out/zero-cache/src/types/processes.d.ts.map +1 -1
- package/out/zero-cache/src/types/processes.js +1 -25
- package/out/zero-cache/src/types/processes.js.map +1 -1
- package/out/zero-cache/src/types/streams.d.ts +21 -0
- package/out/zero-cache/src/types/streams.d.ts.map +1 -1
- package/out/zero-cache/src/types/streams.js +25 -4
- package/out/zero-cache/src/types/streams.js.map +1 -1
- package/out/zero-cache/src/types/subscription.d.ts +11 -0
- package/out/zero-cache/src/types/subscription.d.ts.map +1 -1
- package/out/zero-cache/src/types/subscription.js +27 -0
- package/out/zero-cache/src/types/subscription.js.map +1 -1
- package/out/zero-cache/src/workers/connection.d.ts +1 -0
- package/out/zero-cache/src/workers/connection.d.ts.map +1 -1
- package/out/zero-cache/src/workers/connection.js +24 -2
- package/out/zero-cache/src/workers/connection.js.map +1 -1
- package/out/zero-cache/src/workers/replicator.d.ts +25 -0
- package/out/zero-cache/src/workers/replicator.d.ts.map +1 -1
- package/out/zero-cache/src/workers/replicator.js +33 -1
- package/out/zero-cache/src/workers/replicator.js.map +1 -1
- package/out/zero-cache/src/workers/syncer.d.ts +12 -0
- package/out/zero-cache/src/workers/syncer.d.ts.map +1 -1
- package/out/zero-cache/src/workers/syncer.js +75 -0
- package/out/zero-cache/src/workers/syncer.js.map +1 -1
- package/out/zero-client/src/client/version.js +1 -1
- package/out/zero-client/src/client/zero.d.ts +6 -7
- package/out/zero-client/src/client/zero.d.ts.map +1 -1
- package/out/zero-client/src/client/zero.js +156 -101
- package/out/zero-client/src/client/zero.js.map +1 -1
- package/out/zero-schema/src/builder/table-builder.d.ts +23 -0
- package/out/zero-schema/src/builder/table-builder.d.ts.map +1 -1
- package/out/zero-schema/src/builder/table-builder.js +18 -0
- package/out/zero-schema/src/builder/table-builder.js.map +1 -1
- package/out/zero-server/src/custom.js +10 -3
- package/out/zero-server/src/custom.js.map +1 -1
- package/out/zero-server/src/schema.d.ts +8 -0
- package/out/zero-server/src/schema.d.ts.map +1 -1
- package/out/zero-server/src/schema.js +63 -24
- package/out/zero-server/src/schema.js.map +1 -1
- package/out/zero-types/src/schema.d.ts +11 -0
- package/out/zero-types/src/schema.d.ts.map +1 -1
- package/out/zql/src/mutate/crud.d.ts +10 -8
- package/out/zql/src/mutate/crud.d.ts.map +1 -1
- package/out/zql/src/mutate/crud.js.map +1 -1
- package/out/zql/src/query/query-delegate-base.d.ts.map +1 -1
- package/out/zql/src/query/query-delegate-base.js +5 -1
- package/out/zql/src/query/query-delegate-base.js.map +1 -1
- package/out/zql/src/query/query-impl.d.ts +2 -2
- package/out/zql/src/query/query-impl.d.ts.map +1 -1
- package/out/zql/src/query/query-impl.js.map +1 -1
- package/out/zql/src/query/query.d.ts +83 -14
- package/out/zql/src/query/query.d.ts.map +1 -1
- package/out/zql/src/query/static-query.d.ts +1 -1
- package/out/zql/src/query/static-query.d.ts.map +1 -1
- package/out/zqlite/src/resolve-scalar-subqueries.d.ts +15 -0
- package/out/zqlite/src/resolve-scalar-subqueries.d.ts.map +1 -1
- package/out/zqlite/src/resolve-scalar-subqueries.js +33 -14
- package/out/zqlite/src/resolve-scalar-subqueries.js.map +1 -1
- package/package.json +3 -3
- package/out/replicache/src/db/index.js +0 -91
- package/out/replicache/src/db/index.js.map +0 -1
- package/out/replicache/src/lazy.d.ts +0 -2
- package/out/replicache/src/lazy.d.ts.map +0 -1
- package/out/replicache/src/lazy.js +0 -12
- package/out/replicache/src/lazy.js.map +0 -1
- package/out/zero-cache/src/server/backup-watermark-reader.js +0 -35
- package/out/zero-cache/src/server/backup-watermark-reader.js.map +0 -1
- package/out/zero-cache/src/services/change-streamer/litestream3-backup-monitor.d.ts.map +0 -1
- package/out/zero-cache/src/services/change-streamer/litestream3-backup-monitor.js.map +0 -1
- package/out/zero-cache/src/services/change-streamer/replica-monitor.d.ts +0 -16
- package/out/zero-cache/src/services/change-streamer/replica-monitor.d.ts.map +0 -1
- package/out/zero-cache/src/services/change-streamer/replica-monitor.js +0 -54
- package/out/zero-cache/src/services/change-streamer/replica-monitor.js.map +0 -1
- package/out/zero-cache/src/services/change-streamer/vfs-backup-monitor.d.ts +0 -25
- package/out/zero-cache/src/services/change-streamer/vfs-backup-monitor.d.ts.map +0 -1
- package/out/zero-cache/src/services/change-streamer/vfs-backup-monitor.js +0 -159
- package/out/zero-cache/src/services/change-streamer/vfs-backup-monitor.js.map +0 -1
- package/out/zero-cache/src/services/litestream/vfs-watermark-reader.d.ts +0 -44
- package/out/zero-cache/src/services/litestream/vfs-watermark-reader.d.ts.map +0 -1
- package/out/zero-cache/src/services/litestream/vfs-watermark-reader.js +0 -65
- package/out/zero-cache/src/services/litestream/vfs-watermark-reader.js.map +0 -1
- package/out/zero-cache/src/services/litestream/vfs-watermark-worker-source.d.ts +0 -11
- package/out/zero-cache/src/services/litestream/vfs-watermark-worker-source.d.ts.map +0 -1
- package/out/zero-cache/src/services/litestream/vfs-watermark-worker-source.js +0 -74
- package/out/zero-cache/src/services/litestream/vfs-watermark-worker-source.js.map +0 -1
- package/out/zero-cache/src/services/litestream/vfs-watermark-worker.d.ts +0 -42
- package/out/zero-cache/src/services/litestream/vfs-watermark-worker.d.ts.map +0 -1
- package/out/zero-cache/src/services/litestream/vfs-watermark-worker.js +0 -117
- package/out/zero-cache/src/services/litestream/vfs-watermark-worker.js.map +0 -1
|
@@ -0,0 +1,261 @@
|
|
|
1
|
+
import { AbortError } from "../../../../shared/src/abort-error.js";
|
|
2
|
+
import { sleep } from "../../../../shared/src/sleep.js";
|
|
3
|
+
import { getOrCreateCounter } from "../../observability/metrics.js";
|
|
4
|
+
import { resolver } from "@rocicorp/resolver";
|
|
5
|
+
//#region ../zero-cache/src/services/change-streamer/sqlite-change-log-catchup.ts
|
|
6
|
+
var DEFAULT_BARRIER_POLL_INTERVAL_MS = 1e3;
|
|
7
|
+
var NOOP_CLEANUP_GUARD = { runWhilePurgeBlocked: (register) => Promise.resolve(register()) };
|
|
8
|
+
/**
|
|
9
|
+
* Coordinates a serving subscriber's gap-free transition from replica-local
|
|
10
|
+
* SQLite catchup to the Forwarder's live stream.
|
|
11
|
+
*
|
|
12
|
+
* Registration and required-head capture happen in one synchronous callback.
|
|
13
|
+
* The subscriber therefore either sees a transaction in SQLite catchup or in
|
|
14
|
+
* its live backlog (duplicates across that boundary are filtered by
|
|
15
|
+
* Subscriber), never in neither place.
|
|
16
|
+
*/
|
|
17
|
+
var SQLiteChangeLogCatchup = class {
|
|
18
|
+
#lc;
|
|
19
|
+
#forwarder;
|
|
20
|
+
#reader;
|
|
21
|
+
#batchSize;
|
|
22
|
+
#barrierTimeoutMs;
|
|
23
|
+
#barrierPollIntervalMs;
|
|
24
|
+
#cleanupGuard;
|
|
25
|
+
#sleep;
|
|
26
|
+
#now;
|
|
27
|
+
#barrierTimeouts = getOrCreateCounter("replication", "sqlite_change_log.barrier_timeouts", "SQLite change-log catchups that timed out waiting for the required head.");
|
|
28
|
+
#barrierBacklogOverflows = getOrCreateCounter("replication", "sqlite_change_log.barrier_backlog_overflows", "SQLite change-log catchups abandoned because the subscriber backlog reached its high water mark while waiting for the required head.");
|
|
29
|
+
#barrierWakeups = getOrCreateCounter("replication", "sqlite_change_log.barrier_wakeups", "SQLite catchup barrier waits, labeled by what ended the wait. A population dominated by \"poll\" means the change-log writer's commit notification is not reaching the barrier.");
|
|
30
|
+
#catchups = /* @__PURE__ */ new Map();
|
|
31
|
+
#commitWaiters = /* @__PURE__ */ new Set();
|
|
32
|
+
#closed = false;
|
|
33
|
+
constructor(lc, forwarder, reader, opts) {
|
|
34
|
+
this.#lc = lc.withContext("component", "sqlite-change-log-catchup");
|
|
35
|
+
this.#forwarder = forwarder;
|
|
36
|
+
this.#reader = reader;
|
|
37
|
+
this.#batchSize = opts.batchSize;
|
|
38
|
+
this.#barrierTimeoutMs = opts.barrierTimeoutMs;
|
|
39
|
+
this.#barrierPollIntervalMs = opts.barrierPollIntervalMs ?? DEFAULT_BARRIER_POLL_INTERVAL_MS;
|
|
40
|
+
this.#cleanupGuard = opts.cleanupGuard ?? NOOP_CLEANUP_GUARD;
|
|
41
|
+
this.#sleep = opts.sleep ?? sleep;
|
|
42
|
+
this.#now = opts.now ?? Date.now;
|
|
43
|
+
}
|
|
44
|
+
/**
|
|
45
|
+
* Registers the subscriber before resolving. Catchup itself continues in
|
|
46
|
+
* the background so subscribe() does not wait for SQLite to reach the
|
|
47
|
+
* required head.
|
|
48
|
+
*/
|
|
49
|
+
async catchup(subscriber, captureRequiredHead) {
|
|
50
|
+
if (this.#closed) {
|
|
51
|
+
subscriber.fail(new AbortError("SQLite change-log catchup is closed"));
|
|
52
|
+
return;
|
|
53
|
+
}
|
|
54
|
+
const abort = new AbortController();
|
|
55
|
+
this.#catchups.set(subscriber, abort);
|
|
56
|
+
let requiredHead;
|
|
57
|
+
try {
|
|
58
|
+
await this.#cleanupGuard.runWhilePurgeBlocked(() => {
|
|
59
|
+
this.#throwIfAborted(abort.signal);
|
|
60
|
+
requiredHead = captureRequiredHead();
|
|
61
|
+
this.#forwarder.add(subscriber);
|
|
62
|
+
});
|
|
63
|
+
} catch (error) {
|
|
64
|
+
this.#catchups.delete(subscriber);
|
|
65
|
+
if (!abort.signal.aborted) {
|
|
66
|
+
this.#lc.error?.(`error while registering SQLite catchup subscriber ${subscriber.id}`, error);
|
|
67
|
+
subscriber.fail(error);
|
|
68
|
+
}
|
|
69
|
+
return;
|
|
70
|
+
}
|
|
71
|
+
this.#run(subscriber, requiredHead, abort);
|
|
72
|
+
}
|
|
73
|
+
/**
|
|
74
|
+
* Notes that the change-log writer has committed `watermark`, i.e. that the
|
|
75
|
+
* log's head has advanced to it. The writer runs in this process, so this is
|
|
76
|
+
* the event itself rather than a subscriber's ACK of it.
|
|
77
|
+
*
|
|
78
|
+
* It is a wakeup, not an authority: `plan()` still decides what can be read.
|
|
79
|
+
* A notification that never arrives costs the barrier a poll interval, and one
|
|
80
|
+
* that arrives early costs an extra `plan()` call.
|
|
81
|
+
*/
|
|
82
|
+
onChangeLogCommit(watermark) {
|
|
83
|
+
for (const waiter of this.#commitWaiters) if (watermark >= waiter.watermark) {
|
|
84
|
+
this.#commitWaiters.delete(waiter);
|
|
85
|
+
waiter.resolve();
|
|
86
|
+
}
|
|
87
|
+
}
|
|
88
|
+
remove(subscriber) {
|
|
89
|
+
this.#catchups.get(subscriber)?.abort();
|
|
90
|
+
this.#catchups.delete(subscriber);
|
|
91
|
+
this.#forwarder.remove(subscriber);
|
|
92
|
+
}
|
|
93
|
+
close() {
|
|
94
|
+
if (this.#closed) return;
|
|
95
|
+
this.#closed = true;
|
|
96
|
+
for (const [subscriber, abort] of this.#catchups) {
|
|
97
|
+
abort.abort();
|
|
98
|
+
this.#forwarder.remove(subscriber);
|
|
99
|
+
}
|
|
100
|
+
this.#catchups.clear();
|
|
101
|
+
this.#reader.close();
|
|
102
|
+
}
|
|
103
|
+
[Symbol.dispose]() {
|
|
104
|
+
this.close();
|
|
105
|
+
}
|
|
106
|
+
async #run(subscriber, requiredHead, abort) {
|
|
107
|
+
const { signal } = abort;
|
|
108
|
+
try {
|
|
109
|
+
const requiredHeadStart = this.#now();
|
|
110
|
+
const required = await this.#awaitRequiredHead(requiredHead, signal);
|
|
111
|
+
const requiredHeadWaitMs = this.#now() - requiredHeadStart;
|
|
112
|
+
if (requiredHeadWaitMs > this.#barrierTimeoutMs) this.#lc.info?.(`waited ${requiredHeadWaitMs} ms for the forwarded transaction to commit before starting the SQLite barrier for ${subscriber.id}`);
|
|
113
|
+
const deadline = this.#now() + this.#barrierTimeoutMs;
|
|
114
|
+
const plan = await this.#waitForPlan(subscriber, required, deadline, signal);
|
|
115
|
+
this.#throwIfAborted(signal);
|
|
116
|
+
if (plan.kind === "too-old") {
|
|
117
|
+
const message = `earliest supported watermark is ${plan.minWatermark} (requested ${subscriber.watermark})`;
|
|
118
|
+
this.#lc.warn?.(`rejecting subscriber at watermark ${subscriber.watermark} (earliest watermark: ${plan.minWatermark})`);
|
|
119
|
+
subscriber.close(2, message);
|
|
120
|
+
return;
|
|
121
|
+
}
|
|
122
|
+
let count = 0;
|
|
123
|
+
const start = this.#now();
|
|
124
|
+
if (plan.kind === "range") {
|
|
125
|
+
let lastBatchConsumed;
|
|
126
|
+
for await (const changes of this.#reader.read(subscriber.watermark, plan.headWatermark, this.#batchSize, signal)) {
|
|
127
|
+
const waitStart = this.#now();
|
|
128
|
+
await lastBatchConsumed;
|
|
129
|
+
const elapsed = this.#now() - waitStart;
|
|
130
|
+
if (lastBatchConsumed) this.#lc[elapsed > 100 ? "info" : "debug"]?.(`waited ${elapsed.toFixed(3)} ms for ${subscriber.id} to consume the previous SQLite catchup batch`);
|
|
131
|
+
this.#throwIfAborted(signal);
|
|
132
|
+
for (const change of changes) {
|
|
133
|
+
lastBatchConsumed = subscriber.catchup(change);
|
|
134
|
+
count++;
|
|
135
|
+
}
|
|
136
|
+
}
|
|
137
|
+
await lastBatchConsumed;
|
|
138
|
+
} else this.#lc.warn?.(`subscriber ${subscriber.id} at watermark ${subscriber.watermark} is ahead of the SQLite change-log head ${plan.headWatermark}; waiting for the replica to catch up`);
|
|
139
|
+
this.#throwIfAborted(signal);
|
|
140
|
+
this.#lc.info?.(`caught up ${subscriber.id} from SQLite with ${count} changes (${this.#now() - start} ms)`);
|
|
141
|
+
subscriber.setCaughtUp();
|
|
142
|
+
} catch (error) {
|
|
143
|
+
if (signal.aborted) return;
|
|
144
|
+
if (error instanceof SQLiteChangeLogBarrierError) {
|
|
145
|
+
if (error instanceof SQLiteChangeLogBarrierTimeoutError) this.#barrierTimeouts.add(1);
|
|
146
|
+
this.#lc.warn?.(`ending subscription for ${subscriber.id} to retry SQLite catchup`, error);
|
|
147
|
+
subscriber.close();
|
|
148
|
+
return;
|
|
149
|
+
}
|
|
150
|
+
this.#lc.error?.(`error while catching up subscriber ${subscriber.id} from SQLite`, error);
|
|
151
|
+
subscriber.fail(error);
|
|
152
|
+
} finally {
|
|
153
|
+
if (this.#catchups.get(subscriber) === abort) this.#catchups.delete(subscriber);
|
|
154
|
+
}
|
|
155
|
+
}
|
|
156
|
+
/**
|
|
157
|
+
* Waits for the transaction that was in flight at registration to finish.
|
|
158
|
+
*
|
|
159
|
+
* This wait is bounded by the upstream transaction rather than by
|
|
160
|
+
* `barrierTimeoutMs`: the completion settles on both commit and rollback,
|
|
161
|
+
* and an interrupted change stream forwards a synthetic rollback, so it
|
|
162
|
+
* always settles. Aborting -- the subscriber disconnecting or the service
|
|
163
|
+
* shutting down -- is what releases it early.
|
|
164
|
+
*/
|
|
165
|
+
async #awaitRequiredHead(requiredHead, signal) {
|
|
166
|
+
if (typeof requiredHead === "string") return requiredHead;
|
|
167
|
+
this.#throwIfAborted(signal);
|
|
168
|
+
let onAbort;
|
|
169
|
+
try {
|
|
170
|
+
return await Promise.race([requiredHead, new Promise((_, reject) => {
|
|
171
|
+
onAbort = () => reject(new AbortError("SQLite change-log catchup aborted"));
|
|
172
|
+
signal.addEventListener("abort", onAbort, { once: true });
|
|
173
|
+
})]);
|
|
174
|
+
} finally {
|
|
175
|
+
if (onAbort) signal.removeEventListener("abort", onAbort);
|
|
176
|
+
}
|
|
177
|
+
}
|
|
178
|
+
async #waitForPlan(subscriber, requiredHead, deadline, signal) {
|
|
179
|
+
const backlogFull = subscriber.whenBacklogFull();
|
|
180
|
+
const barrier = new AbortController();
|
|
181
|
+
const abortBarrier = () => barrier.abort();
|
|
182
|
+
signal.addEventListener("abort", abortBarrier, { once: true });
|
|
183
|
+
if (signal.aborted) barrier.abort();
|
|
184
|
+
let observedHead = subscriber.watermark;
|
|
185
|
+
try {
|
|
186
|
+
const planReady = (async () => {
|
|
187
|
+
while (true) {
|
|
188
|
+
this.#throwIfAborted(barrier.signal);
|
|
189
|
+
const plan = this.#reader.plan(subscriber.watermark);
|
|
190
|
+
if (plan.kind !== "not-ready") {
|
|
191
|
+
observedHead = plan.headWatermark;
|
|
192
|
+
if (plan.headWatermark >= requiredHead) return plan;
|
|
193
|
+
}
|
|
194
|
+
const remaining = deadline - this.#now();
|
|
195
|
+
if (remaining <= 0) throw new SQLiteChangeLogBarrierTimeoutError(`timed out waiting for SQLite head ${observedHead} to reach required head ${requiredHead}`);
|
|
196
|
+
await this.#awaitChangeLogProgress(requiredHead, remaining, barrier.signal);
|
|
197
|
+
}
|
|
198
|
+
})();
|
|
199
|
+
const backlogOverflow = backlogFull.promise.then(() => {
|
|
200
|
+
this.#throwIfAborted(signal);
|
|
201
|
+
if (!subscriber.backlogFull) return new Promise(() => {});
|
|
202
|
+
this.#barrierWakeups.add(1, { "wakeup.source": "backlog" });
|
|
203
|
+
this.#barrierBacklogOverflows.add(1);
|
|
204
|
+
throw new SQLiteChangeLogBarrierBacklogError(`backlog for subscriber ${subscriber.id} reached its high water mark while waiting for SQLite head ${observedHead} to reach required head ${requiredHead}`);
|
|
205
|
+
});
|
|
206
|
+
return await Promise.race([planReady, backlogOverflow]);
|
|
207
|
+
} finally {
|
|
208
|
+
signal.removeEventListener("abort", abortBarrier);
|
|
209
|
+
barrier.abort();
|
|
210
|
+
backlogFull.cancel();
|
|
211
|
+
}
|
|
212
|
+
}
|
|
213
|
+
/**
|
|
214
|
+
* Waits for the change-log writer to commit `requiredHead`, or the poll
|
|
215
|
+
* interval as a backstop.
|
|
216
|
+
*
|
|
217
|
+
* No notification can be missed by registering after `plan()` was read: the
|
|
218
|
+
* notification follows the commit it reports, so if the writer had already
|
|
219
|
+
* committed `requiredHead`, `plan()` would have seen it.
|
|
220
|
+
*/
|
|
221
|
+
async #awaitChangeLogProgress(requiredHead, remainingMs, signal) {
|
|
222
|
+
const committed = resolver();
|
|
223
|
+
const waiter = {
|
|
224
|
+
watermark: requiredHead,
|
|
225
|
+
resolve: committed.resolve
|
|
226
|
+
};
|
|
227
|
+
this.#commitWaiters.add(waiter);
|
|
228
|
+
const poll = new AbortController();
|
|
229
|
+
const abortPoll = () => poll.abort();
|
|
230
|
+
signal.addEventListener("abort", abortPoll, { once: true });
|
|
231
|
+
try {
|
|
232
|
+
const wokenBy = await Promise.race([committed.promise.then(() => "commit"), this.#sleep(Math.min(this.#barrierPollIntervalMs, remainingMs), poll.signal).then(() => "poll")]);
|
|
233
|
+
this.#barrierWakeups.add(1, { "wakeup.source": wokenBy });
|
|
234
|
+
} finally {
|
|
235
|
+
this.#commitWaiters.delete(waiter);
|
|
236
|
+
signal.removeEventListener("abort", abortPoll);
|
|
237
|
+
poll.abort();
|
|
238
|
+
}
|
|
239
|
+
}
|
|
240
|
+
#throwIfAborted(signal) {
|
|
241
|
+
if (this.#closed || signal.aborted) throw new AbortError("SQLite change-log catchup aborted");
|
|
242
|
+
}
|
|
243
|
+
};
|
|
244
|
+
/**
|
|
245
|
+
* A barrier that gave up on the replica reaching the required head. Both
|
|
246
|
+
* reasons mean "not yet", not "never", so both end the subscription cleanly
|
|
247
|
+
* for a retry rather than failing it.
|
|
248
|
+
*/
|
|
249
|
+
var SQLiteChangeLogBarrierError = class extends Error {};
|
|
250
|
+
/** The replica did not reach the required head before the deadline. */
|
|
251
|
+
var SQLiteChangeLogBarrierTimeoutError = class extends SQLiteChangeLogBarrierError {
|
|
252
|
+
name = "SQLiteChangeLogBarrierTimeoutError";
|
|
253
|
+
};
|
|
254
|
+
/** Waiting any longer would have cost more than reconnecting. */
|
|
255
|
+
var SQLiteChangeLogBarrierBacklogError = class extends SQLiteChangeLogBarrierError {
|
|
256
|
+
name = "SQLiteChangeLogBarrierBacklogError";
|
|
257
|
+
};
|
|
258
|
+
//#endregion
|
|
259
|
+
export { SQLiteChangeLogCatchup };
|
|
260
|
+
|
|
261
|
+
//# sourceMappingURL=sqlite-change-log-catchup.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"sqlite-change-log-catchup.js","names":["#lc","#forwarder","#reader","#batchSize","#barrierTimeoutMs","#barrierPollIntervalMs","#cleanupGuard","#sleep","#now","#barrierTimeouts","#barrierBacklogOverflows","#barrierWakeups","#catchups","#commitWaiters","#closed","#throwIfAborted","#run","#awaitRequiredHead","#waitForPlan","#awaitChangeLogProgress"],"sources":["../../../../../../zero-cache/src/services/change-streamer/sqlite-change-log-catchup.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {resolver} from '@rocicorp/resolver';\nimport {AbortError} from '../../../../shared/src/abort-error.ts';\nimport {sleep} from '../../../../shared/src/sleep.ts';\nimport {getOrCreateCounter} from '../../observability/metrics.ts';\nimport type {WatermarkedChange} from './change-streamer.ts';\nimport * as ErrorType from './error-type-enum.ts';\nimport type {Forwarder} from './forwarder.ts';\nimport type {CatchupPlan} from './sqlite-change-log-reader.ts';\nimport type {Subscriber} from './subscriber.ts';\n\nexport interface SQLiteChangeLogCatchupReader {\n plan(fromWatermark: string): CatchupPlan;\n read(\n fromWatermark: string,\n throughWatermark: string,\n batchSize: number,\n signal?: AbortSignal,\n ): AsyncIterable<readonly WatermarkedChange[]>;\n close(): void;\n}\n\n/**\n * Slice 9 replaces this no-op implementation with a guard that waits for an\n * in-flight purge and blocks new purge dispatch while the subscriber's ACK is\n * made visible to the Forwarder.\n */\nexport interface SQLiteChangeLogCleanupGuard {\n runWhilePurgeBlocked<T>(register: () => T): Promise<T>;\n}\n\n// How long the barrier waits for the change-log writer's commit notification\n// before re-reading the log itself. The notification normally arrives first, so\n// this is a backstop for the windows in which none is coming: the writer is\n// disabled, or it is between stream connections.\nconst DEFAULT_BARRIER_POLL_INTERVAL_MS = 1000;\n\nexport type SQLiteChangeLogCatchupOptions = {\n batchSize: number;\n barrierTimeoutMs: number;\n barrierPollIntervalMs?: number | undefined;\n cleanupGuard?: SQLiteChangeLogCleanupGuard | undefined;\n sleep?: typeof sleep | undefined;\n now?: (() => number) | undefined;\n};\n\nconst NOOP_CLEANUP_GUARD: SQLiteChangeLogCleanupGuard = {\n runWhilePurgeBlocked: register => Promise.resolve(register()),\n};\n\n/**\n * Coordinates a serving subscriber's gap-free transition from replica-local\n * SQLite catchup to the Forwarder's live stream.\n *\n * Registration and required-head capture happen in one synchronous callback.\n * The subscriber therefore either sees a transaction in SQLite catchup or in\n * its live backlog (duplicates across that boundary are filtered by\n * Subscriber), never in neither place.\n */\nexport class SQLiteChangeLogCatchup implements Disposable {\n readonly #lc: LogContext;\n readonly #forwarder: Forwarder;\n readonly #reader: SQLiteChangeLogCatchupReader;\n readonly #batchSize: number;\n readonly #barrierTimeoutMs: number;\n readonly #barrierPollIntervalMs: number;\n readonly #cleanupGuard: SQLiteChangeLogCleanupGuard;\n readonly #sleep: typeof sleep;\n readonly #now: () => number;\n readonly #barrierTimeouts = getOrCreateCounter(\n 'replication',\n 'sqlite_change_log.barrier_timeouts',\n 'SQLite change-log catchups that timed out waiting for the required head.',\n );\n readonly #barrierBacklogOverflows = getOrCreateCounter(\n 'replication',\n 'sqlite_change_log.barrier_backlog_overflows',\n 'SQLite change-log catchups abandoned because the subscriber backlog ' +\n 'reached its high water mark while waiting for the required head.',\n );\n readonly #barrierWakeups = getOrCreateCounter(\n 'replication',\n 'sqlite_change_log.barrier_wakeups',\n 'SQLite catchup barrier waits, labeled by what ended the wait. A ' +\n 'population dominated by \"poll\" means the change-log writer\\'s commit ' +\n 'notification is not reaching the barrier.',\n );\n readonly #catchups = new Map<Subscriber, AbortController>();\n readonly #commitWaiters = new Set<CommitWaiter>();\n #closed = false;\n\n constructor(\n lc: LogContext,\n forwarder: Forwarder,\n reader: SQLiteChangeLogCatchupReader,\n opts: SQLiteChangeLogCatchupOptions,\n ) {\n this.#lc = lc.withContext('component', 'sqlite-change-log-catchup');\n this.#forwarder = forwarder;\n this.#reader = reader;\n this.#batchSize = opts.batchSize;\n this.#barrierTimeoutMs = opts.barrierTimeoutMs;\n this.#barrierPollIntervalMs =\n opts.barrierPollIntervalMs ?? DEFAULT_BARRIER_POLL_INTERVAL_MS;\n this.#cleanupGuard = opts.cleanupGuard ?? NOOP_CLEANUP_GUARD;\n this.#sleep = opts.sleep ?? sleep;\n this.#now = opts.now ?? Date.now;\n }\n\n /**\n * Registers the subscriber before resolving. Catchup itself continues in\n * the background so subscribe() does not wait for SQLite to reach the\n * required head.\n */\n async catchup(\n subscriber: Subscriber,\n captureRequiredHead: () => string | Promise<string>,\n ): Promise<void> {\n if (this.#closed) {\n subscriber.fail(new AbortError('SQLite change-log catchup is closed'));\n return;\n }\n\n const abort = new AbortController();\n this.#catchups.set(subscriber, abort);\n let requiredHead: string | Promise<string> | undefined;\n try {\n await this.#cleanupGuard.runWhilePurgeBlocked(() => {\n this.#throwIfAborted(abort.signal);\n requiredHead = captureRequiredHead();\n this.#forwarder.add(subscriber);\n });\n } catch (error) {\n this.#catchups.delete(subscriber);\n if (!abort.signal.aborted) {\n this.#lc.error?.(\n `error while registering SQLite catchup subscriber ${subscriber.id}`,\n error,\n );\n subscriber.fail(error);\n }\n return;\n }\n\n void this.#run(subscriber, requiredHead as string | Promise<string>, abort);\n }\n\n /**\n * Notes that the change-log writer has committed `watermark`, i.e. that the\n * log's head has advanced to it. The writer runs in this process, so this is\n * the event itself rather than a subscriber's ACK of it.\n *\n * It is a wakeup, not an authority: `plan()` still decides what can be read.\n * A notification that never arrives costs the barrier a poll interval, and one\n * that arrives early costs an extra `plan()` call.\n */\n onChangeLogCommit(watermark: string): void {\n for (const waiter of this.#commitWaiters) {\n if (watermark >= waiter.watermark) {\n this.#commitWaiters.delete(waiter);\n waiter.resolve();\n }\n }\n }\n\n remove(subscriber: Subscriber): void {\n this.#catchups.get(subscriber)?.abort();\n this.#catchups.delete(subscriber);\n this.#forwarder.remove(subscriber);\n }\n\n close(): void {\n if (this.#closed) {\n return;\n }\n this.#closed = true;\n for (const [subscriber, abort] of this.#catchups) {\n abort.abort();\n this.#forwarder.remove(subscriber);\n }\n this.#catchups.clear();\n this.#reader.close();\n }\n\n [Symbol.dispose](): void {\n this.close();\n }\n\n async #run(\n subscriber: Subscriber,\n requiredHead: string | Promise<string>,\n abort: AbortController,\n ): Promise<void> {\n const {signal} = abort;\n try {\n const requiredHeadStart = this.#now();\n const required = await this.#awaitRequiredHead(requiredHead, signal);\n const requiredHeadWaitMs = this.#now() - requiredHeadStart;\n if (requiredHeadWaitMs > this.#barrierTimeoutMs) {\n this.#lc.info?.(\n `waited ${requiredHeadWaitMs} ms for the forwarded transaction to ` +\n `commit before starting the SQLite barrier for ${subscriber.id}`,\n );\n }\n // The barrier bounds replica lag, not upstream transaction duration, so\n // its deadline starts once the required head is known. Sharing one\n // deadline with the wait above would fail subscribers that registered\n // during a transaction longer than barrierTimeoutMs, however current\n // the replica is.\n const deadline = this.#now() + this.#barrierTimeoutMs;\n const plan = await this.#waitForPlan(\n subscriber,\n required,\n deadline,\n signal,\n );\n this.#throwIfAborted(signal);\n\n if (plan.kind === 'too-old') {\n const message =\n `earliest supported watermark is ${plan.minWatermark} ` +\n `(requested ${subscriber.watermark})`;\n this.#lc.warn?.(\n `rejecting subscriber at watermark ${subscriber.watermark} ` +\n `(earliest watermark: ${plan.minWatermark})`,\n );\n subscriber.close(ErrorType.WatermarkTooOld, message);\n return;\n }\n\n let count = 0;\n const start = this.#now();\n if (plan.kind === 'range') {\n let lastBatchConsumed: Promise<unknown> | undefined;\n for await (const changes of this.#reader.read(\n subscriber.watermark,\n plan.headWatermark,\n this.#batchSize,\n signal,\n )) {\n const waitStart = this.#now();\n await lastBatchConsumed;\n const elapsed = this.#now() - waitStart;\n if (lastBatchConsumed) {\n this.#lc[elapsed > 100 ? 'info' : 'debug']?.(\n `waited ${elapsed.toFixed(3)} ms for ${subscriber.id} to consume ` +\n `the previous SQLite catchup batch`,\n );\n }\n this.#throwIfAborted(signal);\n for (const change of changes) {\n lastBatchConsumed = subscriber.catchup(change);\n count++;\n }\n }\n await lastBatchConsumed;\n } else {\n this.#lc.warn?.(\n `subscriber ${subscriber.id} at watermark ${subscriber.watermark} ` +\n `is ahead of the SQLite change-log head ${plan.headWatermark}; ` +\n `waiting for the replica to catch up`,\n );\n }\n\n this.#throwIfAborted(signal);\n this.#lc.info?.(\n `caught up ${subscriber.id} from SQLite with ${count} changes ` +\n `(${this.#now() - start} ms)`,\n );\n // Keep buffering live sends until the asynchronous backlog drain has\n // established its ordering boundary.\n void subscriber.setCaughtUp();\n } catch (error) {\n if (signal.aborted) {\n return;\n }\n if (error instanceof SQLiteChangeLogBarrierError) {\n if (error instanceof SQLiteChangeLogBarrierTimeoutError) {\n this.#barrierTimeouts.add(1);\n }\n // Giving up on the barrier means the replica has not caught up *yet*,\n // not that it cannot serve this subscriber. End the subscription\n // cleanly so the client reconnects and retries: IncrementalSyncer\n // treats any ['error', ...] message as terminal and restores a fresh\n // replica from litestream, whereas a clean end backs off and\n // re-subscribes. If retries keep failing until the change log is purged\n // past the subscriber's watermark, plan() returns 'too-old', which is\n // terminal by design.\n this.#lc.warn?.(\n `ending subscription for ${subscriber.id} to retry SQLite catchup`,\n error,\n );\n subscriber.close();\n return;\n }\n this.#lc.error?.(\n `error while catching up subscriber ${subscriber.id} from SQLite`,\n error,\n );\n subscriber.fail(error);\n } finally {\n if (this.#catchups.get(subscriber) === abort) {\n this.#catchups.delete(subscriber);\n }\n }\n }\n\n /**\n * Waits for the transaction that was in flight at registration to finish.\n *\n * This wait is bounded by the upstream transaction rather than by\n * `barrierTimeoutMs`: the completion settles on both commit and rollback,\n * and an interrupted change stream forwards a synthetic rollback, so it\n * always settles. Aborting -- the subscriber disconnecting or the service\n * shutting down -- is what releases it early.\n */\n async #awaitRequiredHead(\n requiredHead: string | Promise<string>,\n signal: AbortSignal,\n ): Promise<string> {\n if (typeof requiredHead === 'string') {\n return requiredHead;\n }\n this.#throwIfAborted(signal);\n let onAbort: (() => void) | undefined;\n try {\n return await Promise.race([\n requiredHead,\n new Promise<never>((_, reject) => {\n onAbort = () =>\n reject(new AbortError('SQLite change-log catchup aborted'));\n signal.addEventListener('abort', onAbort, {once: true});\n }),\n ]);\n } finally {\n if (onAbort) {\n signal.removeEventListener('abort', onAbort);\n }\n }\n }\n\n async #waitForPlan(\n subscriber: Subscriber,\n requiredHead: string,\n deadline: number,\n signal: AbortSignal,\n ): Promise<Exclude<CatchupPlan, {kind: 'not-ready'}>> {\n // Race backlog growth once around the whole polling loop. Attaching the\n // same unresolved promise to every per-poll race would retain one reaction\n // per iteration until the subscriber's backlog eventually filled.\n const backlogFull = subscriber.whenBacklogFull();\n const barrier = new AbortController();\n const abortBarrier = () => barrier.abort();\n signal.addEventListener('abort', abortBarrier, {once: true});\n if (signal.aborted) {\n barrier.abort();\n }\n\n let observedHead = subscriber.watermark;\n try {\n const planReady = (async () => {\n while (true) {\n this.#throwIfAborted(barrier.signal);\n const plan = this.#reader.plan(subscriber.watermark);\n // A not-ready log has no head to compare, so it can never satisfy\n // the required head: keep waiting for the writer to create and\n // reconcile it, and let the barrier deadline end the subscription\n // cleanly for a retry. Selection declines a not-ready log before a\n // subscriber gets here, and neither purging nor reconciliation can\n // empty a log that had content, so this is a belt-and-braces path.\n if (plan.kind !== 'not-ready') {\n observedHead = plan.headWatermark;\n if (plan.headWatermark >= requiredHead) {\n return plan;\n }\n }\n const remaining = deadline - this.#now();\n if (remaining <= 0) {\n throw new SQLiteChangeLogBarrierTimeoutError(\n `timed out waiting for SQLite head ${observedHead} to ` +\n `reach required head ${requiredHead}`,\n );\n }\n await this.#awaitChangeLogProgress(\n requiredHead,\n remaining,\n barrier.signal,\n );\n }\n })();\n\n const backlogOverflow = backlogFull.promise.then(() => {\n this.#throwIfAborted(signal);\n // close() also resolves backlog waiters so none are stranded. The\n // backlog is cleared first, which distinguishes close from overflow.\n if (!subscriber.backlogFull) {\n return new Promise<never>(() => {});\n }\n this.#barrierWakeups.add(1, {'wakeup.source': 'backlog'});\n // Bytes, not time, are what the barrier actually costs: past the high\n // water mark this subscriber's send() no longer resolves, so it holds\n // up every flush and can stall the replica this barrier waits on.\n this.#barrierBacklogOverflows.add(1);\n throw new SQLiteChangeLogBarrierBacklogError(\n `backlog for subscriber ${subscriber.id} reached its high water ` +\n `mark while waiting for SQLite head ${observedHead} to ` +\n `reach required head ${requiredHead}`,\n );\n });\n\n return await Promise.race([planReady, backlogOverflow]);\n } finally {\n signal.removeEventListener('abort', abortBarrier);\n barrier.abort();\n backlogFull.cancel();\n }\n }\n\n /**\n * Waits for the change-log writer to commit `requiredHead`, or the poll\n * interval as a backstop.\n *\n * No notification can be missed by registering after `plan()` was read: the\n * notification follows the commit it reports, so if the writer had already\n * committed `requiredHead`, `plan()` would have seen it.\n */\n async #awaitChangeLogProgress(\n requiredHead: string,\n remainingMs: number,\n signal: AbortSignal,\n ): Promise<void> {\n const committed = resolver<void>();\n const waiter = {watermark: requiredHead, resolve: committed.resolve};\n this.#commitWaiters.add(waiter);\n // Scopes the backstop timer to this wait so that a commit cancels it rather\n // than leaving it to fire against a barrier that has already moved on.\n const poll = new AbortController();\n const abortPoll = () => poll.abort();\n signal.addEventListener('abort', abortPoll, {once: true});\n try {\n const wokenBy = await Promise.race([\n committed.promise.then(() => 'commit' as const),\n this.#sleep(\n Math.min(this.#barrierPollIntervalMs, remainingMs),\n poll.signal,\n ).then(() => 'poll' as const),\n ]);\n this.#barrierWakeups.add(1, {'wakeup.source': wokenBy});\n } finally {\n this.#commitWaiters.delete(waiter);\n signal.removeEventListener('abort', abortPoll);\n poll.abort();\n }\n }\n\n #throwIfAborted(signal: AbortSignal): void {\n if (this.#closed || signal.aborted) {\n throw new AbortError('SQLite change-log catchup aborted');\n }\n }\n}\n\ntype CommitWaiter = {watermark: string; resolve: () => void};\n\n/**\n * A barrier that gave up on the replica reaching the required head. Both\n * reasons mean \"not yet\", not \"never\", so both end the subscription cleanly\n * for a retry rather than failing it.\n */\nexport class SQLiteChangeLogBarrierError extends Error {}\n\n/** The replica did not reach the required head before the deadline. */\nexport class SQLiteChangeLogBarrierTimeoutError extends SQLiteChangeLogBarrierError {\n readonly name = 'SQLiteChangeLogBarrierTimeoutError';\n}\n\n/** Waiting any longer would have cost more than reconnecting. */\nexport class SQLiteChangeLogBarrierBacklogError extends SQLiteChangeLogBarrierError {\n readonly name = 'SQLiteChangeLogBarrierBacklogError';\n}\n"],"mappings":";;;;;AAmCA,IAAM,mCAAmC;AAWzC,IAAM,qBAAkD,EACtD,uBAAsB,aAAY,QAAQ,QAAQ,SAAS,CAAC,EAC9D;;;;;;;;;;AAWA,IAAa,yBAAb,MAA0D;CACxD;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA,mBAA4B,mBAC1B,eACA,sCACA,0EACF;CACA,2BAAoC,mBAClC,eACA,+CACA,sIAEF;CACA,kBAA2B,mBACzB,eACA,qCACA,iLAGF;CACA,4BAAqB,IAAI,IAAiC;CAC1D,iCAA0B,IAAI,IAAkB;CAChD,UAAU;CAEV,YACE,IACA,WACA,QACA,MACA;EACA,KAAKA,MAAM,GAAG,YAAY,aAAa,2BAA2B;EAClE,KAAKC,aAAa;EAClB,KAAKC,UAAU;EACf,KAAKC,aAAa,KAAK;EACvB,KAAKC,oBAAoB,KAAK;EAC9B,KAAKC,yBACH,KAAK,yBAAyB;EAChC,KAAKC,gBAAgB,KAAK,gBAAgB;EAC1C,KAAKC,SAAS,KAAK,SAAS;EAC5B,KAAKC,OAAO,KAAK,OAAO,KAAK;CAC/B;;;;;;CAOA,MAAM,QACJ,YACA,qBACe;EACf,IAAI,KAAKM,SAAS;GAChB,WAAW,KAAK,IAAI,WAAW,qCAAqC,CAAC;GACrE;EACF;EAEA,MAAM,QAAQ,IAAI,gBAAgB;EAClC,KAAKF,UAAU,IAAI,YAAY,KAAK;EACpC,IAAI;EACJ,IAAI;GACF,MAAM,KAAKN,cAAc,2BAA2B;IAClD,KAAKS,gBAAgB,MAAM,MAAM;IACjC,eAAe,oBAAoB;IACnC,KAAKd,WAAW,IAAI,UAAU;GAChC,CAAC;EACH,SAAS,OAAO;GACd,KAAKW,UAAU,OAAO,UAAU;GAChC,IAAI,CAAC,MAAM,OAAO,SAAS;IACzB,KAAKZ,IAAI,QACP,qDAAqD,WAAW,MAChE,KACF;IACA,WAAW,KAAK,KAAK;GACvB;GACA;EACF;EAEA,KAAUgB,KAAK,YAAY,cAA0C,KAAK;CAC5E;;;;;;;;;;CAWA,kBAAkB,WAAyB;EACzC,KAAK,MAAM,UAAU,KAAKH,gBACxB,IAAI,aAAa,OAAO,WAAW;GACjC,KAAKA,eAAe,OAAO,MAAM;GACjC,OAAO,QAAQ;EACjB;CAEJ;CAEA,OAAO,YAA8B;EACnC,KAAKD,UAAU,IAAI,UAAU,GAAG,MAAM;EACtC,KAAKA,UAAU,OAAO,UAAU;EAChC,KAAKX,WAAW,OAAO,UAAU;CACnC;CAEA,QAAc;EACZ,IAAI,KAAKa,SACP;EAEF,KAAKA,UAAU;EACf,KAAK,MAAM,CAAC,YAAY,UAAU,KAAKF,WAAW;GAChD,MAAM,MAAM;GACZ,KAAKX,WAAW,OAAO,UAAU;EACnC;EACA,KAAKW,UAAU,MAAM;EACrB,KAAKV,QAAQ,MAAM;CACrB;CAEA,CAAC,OAAO,WAAiB;EACvB,KAAK,MAAM;CACb;CAEA,MAAMc,KACJ,YACA,cACA,OACe;EACf,MAAM,EAAC,WAAU;EACjB,IAAI;GACF,MAAM,oBAAoB,KAAKR,KAAK;GACpC,MAAM,WAAW,MAAM,KAAKS,mBAAmB,cAAc,MAAM;GACnE,MAAM,qBAAqB,KAAKT,KAAK,IAAI;GACzC,IAAI,qBAAqB,KAAKJ,mBAC5B,KAAKJ,IAAI,OACP,UAAU,mBAAmB,qFACsB,WAAW,IAChE;GAOF,MAAM,WAAW,KAAKQ,KAAK,IAAI,KAAKJ;GACpC,MAAM,OAAO,MAAM,KAAKc,aACtB,YACA,UACA,UACA,MACF;GACA,KAAKH,gBAAgB,MAAM;GAE3B,IAAI,KAAK,SAAS,WAAW;IAC3B,MAAM,UACJ,mCAAmC,KAAK,aAAa,cACvC,WAAW,UAAU;IACrC,KAAKf,IAAI,OACP,qCAAqC,WAAW,UAAU,wBAChC,KAAK,aAAa,EAC9C;IACA,WAAW,MAAM,GAA2B,OAAO;IACnD;GACF;GAEA,IAAI,QAAQ;GACZ,MAAM,QAAQ,KAAKQ,KAAK;GACxB,IAAI,KAAK,SAAS,SAAS;IACzB,IAAI;IACJ,WAAW,MAAM,WAAW,KAAKN,QAAQ,KACvC,WAAW,WACX,KAAK,eACL,KAAKC,YACL,MACF,GAAG;KACD,MAAM,YAAY,KAAKK,KAAK;KAC5B,MAAM;KACN,MAAM,UAAU,KAAKA,KAAK,IAAI;KAC9B,IAAI,mBACF,KAAKR,IAAI,UAAU,MAAM,SAAS,WAChC,UAAU,QAAQ,QAAQ,CAAC,EAAE,UAAU,WAAW,GAAG,8CAEvD;KAEF,KAAKe,gBAAgB,MAAM;KAC3B,KAAK,MAAM,UAAU,SAAS;MAC5B,oBAAoB,WAAW,QAAQ,MAAM;MAC7C;KACF;IACF;IACA,MAAM;GACR,OACE,KAAKf,IAAI,OACP,cAAc,WAAW,GAAG,gBAAgB,WAAW,UAAU,0CACrB,KAAK,cAAc,sCAEjE;GAGF,KAAKe,gBAAgB,MAAM;GAC3B,KAAKf,IAAI,OACP,aAAa,WAAW,GAAG,oBAAoB,MAAM,YAC/C,KAAKQ,KAAK,IAAI,MAAM,KAC5B;GAGA,WAAgB,YAAY;EAC9B,SAAS,OAAO;GACd,IAAI,OAAO,SACT;GAEF,IAAI,iBAAiB,6BAA6B;IAChD,IAAI,iBAAiB,oCACnB,KAAKC,iBAAiB,IAAI,CAAC;IAU7B,KAAKT,IAAI,OACP,2BAA2B,WAAW,GAAG,2BACzC,KACF;IACA,WAAW,MAAM;IACjB;GACF;GACA,KAAKA,IAAI,QACP,sCAAsC,WAAW,GAAG,eACpD,KACF;GACA,WAAW,KAAK,KAAK;EACvB,UAAU;GACR,IAAI,KAAKY,UAAU,IAAI,UAAU,MAAM,OACrC,KAAKA,UAAU,OAAO,UAAU;EAEpC;CACF;;;;;;;;;;CAWA,MAAMK,mBACJ,cACA,QACiB;EACjB,IAAI,OAAO,iBAAiB,UAC1B,OAAO;EAET,KAAKF,gBAAgB,MAAM;EAC3B,IAAI;EACJ,IAAI;GACF,OAAO,MAAM,QAAQ,KAAK,CACxB,cACA,IAAI,SAAgB,GAAG,WAAW;IAChC,gBACE,OAAO,IAAI,WAAW,mCAAmC,CAAC;IAC5D,OAAO,iBAAiB,SAAS,SAAS,EAAC,MAAM,KAAI,CAAC;GACxD,CAAC,CACH,CAAC;EACH,UAAU;GACR,IAAI,SACF,OAAO,oBAAoB,SAAS,OAAO;EAE/C;CACF;CAEA,MAAMG,aACJ,YACA,cACA,UACA,QACoD;EAIpD,MAAM,cAAc,WAAW,gBAAgB;EAC/C,MAAM,UAAU,IAAI,gBAAgB;EACpC,MAAM,qBAAqB,QAAQ,MAAM;EACzC,OAAO,iBAAiB,SAAS,cAAc,EAAC,MAAM,KAAI,CAAC;EAC3D,IAAI,OAAO,SACT,QAAQ,MAAM;EAGhB,IAAI,eAAe,WAAW;EAC9B,IAAI;GACF,MAAM,aAAa,YAAY;IAC7B,OAAO,MAAM;KACX,KAAKH,gBAAgB,QAAQ,MAAM;KACnC,MAAM,OAAO,KAAKb,QAAQ,KAAK,WAAW,SAAS;KAOnD,IAAI,KAAK,SAAS,aAAa;MAC7B,eAAe,KAAK;MACpB,IAAI,KAAK,iBAAiB,cACxB,OAAO;KAEX;KACA,MAAM,YAAY,WAAW,KAAKM,KAAK;KACvC,IAAI,aAAa,GACf,MAAM,IAAI,mCACR,qCAAqC,aAAa,0BACzB,cAC3B;KAEF,MAAM,KAAKW,wBACT,cACA,WACA,QAAQ,MACV;IACF;GACF,GAAG;GAEH,MAAM,kBAAkB,YAAY,QAAQ,WAAW;IACrD,KAAKJ,gBAAgB,MAAM;IAG3B,IAAI,CAAC,WAAW,aACd,OAAO,IAAI,cAAqB,CAAC,CAAC;IAEpC,KAAKJ,gBAAgB,IAAI,GAAG,EAAC,iBAAiB,UAAS,CAAC;IAIxD,KAAKD,yBAAyB,IAAI,CAAC;IACnC,MAAM,IAAI,mCACR,0BAA0B,WAAW,GAAG,6DACA,aAAa,0BAC5B,cAC3B;GACF,CAAC;GAED,OAAO,MAAM,QAAQ,KAAK,CAAC,WAAW,eAAe,CAAC;EACxD,UAAU;GACR,OAAO,oBAAoB,SAAS,YAAY;GAChD,QAAQ,MAAM;GACd,YAAY,OAAO;EACrB;CACF;;;;;;;;;CAUA,MAAMS,wBACJ,cACA,aACA,QACe;EACf,MAAM,YAAY,SAAe;EACjC,MAAM,SAAS;GAAC,WAAW;GAAc,SAAS,UAAU;EAAO;EACnE,KAAKN,eAAe,IAAI,MAAM;EAG9B,MAAM,OAAO,IAAI,gBAAgB;EACjC,MAAM,kBAAkB,KAAK,MAAM;EACnC,OAAO,iBAAiB,SAAS,WAAW,EAAC,MAAM,KAAI,CAAC;EACxD,IAAI;GACF,MAAM,UAAU,MAAM,QAAQ,KAAK,CACjC,UAAU,QAAQ,WAAW,QAAiB,GAC9C,KAAKN,OACH,KAAK,IAAI,KAAKF,wBAAwB,WAAW,GACjD,KAAK,MACP,EAAE,WAAW,MAAe,CAC9B,CAAC;GACD,KAAKM,gBAAgB,IAAI,GAAG,EAAC,iBAAiB,QAAO,CAAC;EACxD,UAAU;GACR,KAAKE,eAAe,OAAO,MAAM;GACjC,OAAO,oBAAoB,SAAS,SAAS;GAC7C,KAAK,MAAM;EACb;CACF;CAEA,gBAAgB,QAA2B;EACzC,IAAI,KAAKC,WAAW,OAAO,SACzB,MAAM,IAAI,WAAW,mCAAmC;CAE5D;AACF;;;;;;AASA,IAAa,8BAAb,cAAiD,MAAM,CAAC;;AAGxD,IAAa,qCAAb,cAAwD,4BAA4B;CAClF,OAAgB;AAClB;;AAGA,IAAa,qCAAb,cAAwD,4BAA4B;CAClF,OAAgB;AAClB"}
|
|
@@ -0,0 +1,121 @@
|
|
|
1
|
+
import type { LogContext } from '@rocicorp/logger';
|
|
2
|
+
import type { Database } from '../../../../zqlite/src/db.ts';
|
|
3
|
+
import type { SQLiteChangeLogCleanupGuard } from './sqlite-change-log-catchup.ts';
|
|
4
|
+
/** Why a pass stopped before draining the eligible set. */
|
|
5
|
+
export type PurgeStopReason = 'stopped' | 'paused' | 'writer-unavailable' | 'subscriber-behind' | 'batch-limit' | 'error';
|
|
6
|
+
/**
|
|
7
|
+
* The invariant-14 probe's classification of `plan(floor)` after a pass.
|
|
8
|
+
*
|
|
9
|
+
* `too-old` at the floor is not by itself a violation: a log that never held
|
|
10
|
+
* that history — routine at every fork, restore, and rolling restart — returns
|
|
11
|
+
* `too-old` without having purged anything. Only the seed point distinguishes
|
|
12
|
+
* the two, which is why `seedWatermark` is in the v2 meta row:
|
|
13
|
+
*
|
|
14
|
+
* ```
|
|
15
|
+
* violation ⇔ plan(floor) is `too-old` AND seedWatermark <= floor
|
|
16
|
+
* ```
|
|
17
|
+
*/
|
|
18
|
+
export type FloorProbeOutcome = 'retained' | 'ahead' | 'cold-log' | 'violation';
|
|
19
|
+
/** When the cleanup coordinator should run another pass. */
|
|
20
|
+
export type PurgeContinuation = 'immediate' | 'deferred';
|
|
21
|
+
export type PurgePassResult = {
|
|
22
|
+
/** The external floor the pass used. */
|
|
23
|
+
readonly floor: string;
|
|
24
|
+
readonly batches: number;
|
|
25
|
+
readonly deletedRows: number;
|
|
26
|
+
/** Why the pass ended before draining, if it did. */
|
|
27
|
+
readonly stopped: PurgeStopReason | undefined;
|
|
28
|
+
/** The invariant-14 probe's outcome, when the log was available to probe. */
|
|
29
|
+
readonly probe: FloorProbeOutcome | undefined;
|
|
30
|
+
/** Whether, and how soon, the cleanup coordinator should run another pass. */
|
|
31
|
+
readonly continuation: PurgeContinuation | undefined;
|
|
32
|
+
};
|
|
33
|
+
export type SQLiteChangeLogPurgeSchedulerOptions = {
|
|
34
|
+
/**
|
|
35
|
+
* `sqliteChangeLogRetentionMs`: the minimum retention window, ANDed on top
|
|
36
|
+
* of the external floor. It can only make purge more conservative — the
|
|
37
|
+
* purger never accepts it as a standalone bound.
|
|
38
|
+
*/
|
|
39
|
+
readonly retentionMs: number;
|
|
40
|
+
/**
|
|
41
|
+
* `sqliteChangeLogPurgeBatchRows`: the starting chunk size per batch. Not
|
|
42
|
+
* the bound — the statement duration target is.
|
|
43
|
+
*/
|
|
44
|
+
readonly batchRows: number;
|
|
45
|
+
readonly statementTargetMs?: number | undefined;
|
|
46
|
+
readonly maxBatchesPerPass?: number | undefined;
|
|
47
|
+
readonly inTransactionPollMs?: number | undefined;
|
|
48
|
+
/** The clock the retention cutoff is measured against. Defaults to `Date.now`. */
|
|
49
|
+
readonly now?: (() => number) | undefined;
|
|
50
|
+
readonly setTimeoutFn?: typeof setTimeout | undefined;
|
|
51
|
+
readonly clearTimeoutFn?: typeof clearTimeout | undefined;
|
|
52
|
+
/**
|
|
53
|
+
* Awaited between batches so fan-out and catchup make progress. Defaults to
|
|
54
|
+
* a macrotask (`setImmediate`), so pending I/O runs before the next batch.
|
|
55
|
+
*/
|
|
56
|
+
readonly yieldFn?: (() => Promise<void>) | undefined;
|
|
57
|
+
};
|
|
58
|
+
/**
|
|
59
|
+
* Runs bounded SQLite change-log purge passes on the writer's own connection.
|
|
60
|
+
*
|
|
61
|
+
* The caller supplies the current durable cleanup floor independently of any
|
|
62
|
+
* other change-log implementation and owns the timing of subsequent passes.
|
|
63
|
+
* Within a pass this class drains bounded batches, re-checking live subscriber
|
|
64
|
+
* ACKs before each one and releasing the cleanup guard between them, so a
|
|
65
|
+
* catchup registration or snapshot reservation never waits longer than one
|
|
66
|
+
* batch.
|
|
67
|
+
*
|
|
68
|
+
* Purge windows exist only between the writer's commits: purge shares the
|
|
69
|
+
* writer's connection, so it cannot run inside the writer's open transaction,
|
|
70
|
+
* and a batch that finds one waits for the commit notification instead.
|
|
71
|
+
*
|
|
72
|
+
* The result tells the cleanup coordinator whether the next pass should be
|
|
73
|
+
* immediate or deferred. This class deliberately owns no cleanup timer: the
|
|
74
|
+
* same level-triggered coordinator continues a large drain, advances the
|
|
75
|
+
* retention clock on a quiet source, and retries changing external state.
|
|
76
|
+
*/
|
|
77
|
+
export declare class SQLiteChangeLogPurgeScheduler {
|
|
78
|
+
#private;
|
|
79
|
+
/**
|
|
80
|
+
* The guard that brackets catchup registration: required-head capture and
|
|
81
|
+
* `Forwarder.add()` run under the same mutex as purge batches, so the
|
|
82
|
+
* registered ACK is visible to `Forwarder.getAcks()` before the next batch
|
|
83
|
+
* evaluates its gate.
|
|
84
|
+
*/
|
|
85
|
+
readonly cleanupGuard: SQLiteChangeLogCleanupGuard;
|
|
86
|
+
/**
|
|
87
|
+
* @param connection the writer's own connection (§3.3). `undefined` until
|
|
88
|
+
* the writer's first reconcile creates the file, and again after the
|
|
89
|
+
* writer fails soft and deletes it — both of which defer the pass rather
|
|
90
|
+
* than fail it, so a later coordinator pass picks up a late-appearing
|
|
91
|
+
* file without a restart.
|
|
92
|
+
* @param subscriberAcks live subscriber ACKs, re-read before every batch so
|
|
93
|
+
* the supplied floor remains safe throughout an asynchronous drain.
|
|
94
|
+
*/
|
|
95
|
+
constructor(lc: LogContext, connection: () => Database | undefined, subscriberAcks: () => Iterable<string>, opts: SQLiteChangeLogPurgeSchedulerOptions);
|
|
96
|
+
/**
|
|
97
|
+
* Runs one purge pass for the current durable cleanup `floor`.
|
|
98
|
+
* Never rejects; a failed pass is logged and reported as `stopped: 'error'`.
|
|
99
|
+
*/
|
|
100
|
+
purge(floor: string): Promise<PurgePassResult>;
|
|
101
|
+
/**
|
|
102
|
+
* Pauses purging while `taskID` holds a snapshot reservation. The returned
|
|
103
|
+
* promise resolves once no purge batch is in flight, i.e. once the bounds a
|
|
104
|
+
* backup monitor is about to advertise cannot be invalidated by a batch
|
|
105
|
+
* that had already been dispatched.
|
|
106
|
+
*/
|
|
107
|
+
pause(taskID: string): Promise<void>;
|
|
108
|
+
/**
|
|
109
|
+
* Ends one of `taskID`'s pauses. A later coordinator pass can purge once
|
|
110
|
+
* every reservation has ended. Unknown `taskID`s are ignored.
|
|
111
|
+
*/
|
|
112
|
+
resume(taskID: string): void;
|
|
113
|
+
/**
|
|
114
|
+
* Notes that the writer's open transaction has ended, waking any batch
|
|
115
|
+
* waiting for a purge window. A missed notification costs one poll
|
|
116
|
+
* interval, never correctness.
|
|
117
|
+
*/
|
|
118
|
+
onWriterIdle(): void;
|
|
119
|
+
stop(): void;
|
|
120
|
+
}
|
|
121
|
+
//# sourceMappingURL=sqlite-change-log-purge-scheduler.d.ts.map
|
package/out/zero-cache/src/services/change-streamer/sqlite-change-log-purge-scheduler.d.ts.map
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"sqlite-change-log-purge-scheduler.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/change-streamer/sqlite-change-log-purge-scheduler.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAEjD,OAAO,KAAK,EAAC,QAAQ,EAAY,MAAM,8BAA8B,CAAC;AAWtE,OAAO,KAAK,EAAC,2BAA2B,EAAC,MAAM,gCAAgC,CAAC;AA2BhF,2DAA2D;AAC3D,MAAM,MAAM,eAAe,GAEvB,SAAS,GAET,QAAQ,GAGR,oBAAoB,GAIpB,mBAAmB,GAEnB,aAAa,GAEb,OAAO,CAAC;AAEZ;;;;;;;;;;;GAWG;AACH,MAAM,MAAM,iBAAiB,GAGzB,UAAU,GAGV,OAAO,GAIP,UAAU,GAIV,WAAW,CAAC;AAEhB,4DAA4D;AAC5D,MAAM,MAAM,iBAAiB,GAEzB,WAAW,GAGX,UAAU,CAAC;AAEf,MAAM,MAAM,eAAe,GAAG;IAC5B,wCAAwC;IACxC,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,qDAAqD;IACrD,QAAQ,CAAC,OAAO,EAAE,eAAe,GAAG,SAAS,CAAC;IAC9C,6EAA6E;IAC7E,QAAQ,CAAC,KAAK,EAAE,iBAAiB,GAAG,SAAS,CAAC;IAC9C,8EAA8E;IAC9E,QAAQ,CAAC,YAAY,EAAE,iBAAiB,GAAG,SAAS,CAAC;CACtD,CAAC;AAEF,MAAM,MAAM,oCAAoC,GAAG;IACjD;;;;OAIG;IACH,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B;;;OAGG;IACH,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,iBAAiB,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAChD,QAAQ,CAAC,iBAAiB,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAChD,QAAQ,CAAC,mBAAmB,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAClD,kFAAkF;IAClF,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC,MAAM,MAAM,CAAC,GAAG,SAAS,CAAC;IAC1C,QAAQ,CAAC,YAAY,CAAC,EAAE,OAAO,UAAU,GAAG,SAAS,CAAC;IACtD,QAAQ,CAAC,cAAc,CAAC,EAAE,OAAO,YAAY,GAAG,SAAS,CAAC;IAC1D;;;OAGG;IACH,QAAQ,CAAC,OAAO,CAAC,EAAE,CAAC,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC,GAAG,SAAS,CAAC;CACtD,CAAC;AA0CF;;;;;;;;;;;;;;;;;;GAkBG;AACH,qBAAa,6BAA6B;;IAkExC;;;;;OAKG;IACH,QAAQ,CAAC,YAAY,EAAE,2BAA2B,CAEhD;IAEF;;;;;;;;OAQG;gBAED,EAAE,EAAE,UAAU,EACd,UAAU,EAAE,MAAM,QAAQ,GAAG,SAAS,EACtC,cAAc,EAAE,MAAM,QAAQ,CAAC,MAAM,CAAC,EACtC,IAAI,EAAE,oCAAoC;IAsB5C;;;OAGG;IACG,KAAK,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,eAAe,CAAC;IAapD;;;;;OAKG;IACH,KAAK,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAKpC;;;OAGG;IACH,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,IAAI;IAY5B;;;;OAIG;IACH,YAAY,IAAI,IAAI;IAOpB,IAAI,IAAI,IAAI;CAiRb"}
|