@rocicorp/zero 1.11.0-canary.10 → 1.11.0-canary.12
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/out/analyze-query/src/bin-analyze.js +8 -7
- package/out/analyze-query/src/bin-analyze.js.map +1 -1
- package/out/replicache/src/btree/node.d.ts.map +1 -1
- package/out/replicache/src/btree/node.js +3 -6
- package/out/replicache/src/btree/node.js.map +1 -1
- package/out/replicache/src/dag/gc.d.ts.map +1 -1
- package/out/replicache/src/dag/gc.js +6 -10
- package/out/replicache/src/dag/gc.js.map +1 -1
- package/out/replicache/src/kv/sqlite-store.d.ts.map +1 -1
- package/out/replicache/src/kv/sqlite-store.js +24 -2
- package/out/replicache/src/kv/sqlite-store.js.map +1 -1
- package/out/replicache/src/persist/persist.js +1 -1
- package/out/shared/src/arrays.d.ts +5 -0
- package/out/shared/src/arrays.d.ts.map +1 -1
- package/out/shared/src/arrays.js +8 -10
- package/out/shared/src/arrays.js.map +1 -1
- package/out/shared/src/map.d.ts +19 -2
- package/out/shared/src/map.d.ts.map +1 -1
- package/out/shared/src/map.js +18 -4
- package/out/shared/src/map.js.map +1 -1
- package/out/shared/src/ring-buffer.d.ts +2 -0
- package/out/shared/src/ring-buffer.d.ts.map +1 -1
- package/out/shared/src/ring-buffer.js +4 -0
- package/out/shared/src/ring-buffer.js.map +1 -1
- package/out/z2s/src/compiler.js +1 -1
- package/out/z2s/src/sql.d.ts.map +1 -1
- package/out/z2s/src/sql.js +2 -2
- package/out/z2s/src/sql.js.map +1 -1
- package/out/zero/package.js +1 -1
- package/out/zero/package.js.map +1 -1
- package/out/zero-cache/src/auth/write-authorizer.d.ts.map +1 -1
- package/out/zero-cache/src/auth/write-authorizer.js +1 -0
- package/out/zero-cache/src/auth/write-authorizer.js.map +1 -1
- package/out/zero-cache/src/config/zero-config.d.ts +15 -0
- package/out/zero-cache/src/config/zero-config.d.ts.map +1 -1
- package/out/zero-cache/src/config/zero-config.js +35 -0
- package/out/zero-cache/src/config/zero-config.js.map +1 -1
- package/out/zero-cache/src/db/lite-tables.d.ts.map +1 -1
- package/out/zero-cache/src/db/lite-tables.js +3 -4
- package/out/zero-cache/src/db/lite-tables.js.map +1 -1
- package/out/zero-cache/src/observability/events.js +4 -1
- package/out/zero-cache/src/observability/events.js.map +1 -1
- package/out/zero-cache/src/observability/metrics.d.ts.map +1 -1
- package/out/zero-cache/src/observability/metrics.js +2 -7
- package/out/zero-cache/src/observability/metrics.js.map +1 -1
- package/out/zero-cache/src/server/change-streamer.d.ts.map +1 -1
- package/out/zero-cache/src/server/change-streamer.js +2 -1
- package/out/zero-cache/src/server/change-streamer.js.map +1 -1
- package/out/zero-cache/src/server/inspector-delegate.d.ts.map +1 -1
- package/out/zero-cache/src/server/inspector-delegate.js +2 -8
- package/out/zero-cache/src/server/inspector-delegate.js.map +1 -1
- package/out/zero-cache/src/server/replicator.d.ts.map +1 -1
- package/out/zero-cache/src/server/replicator.js +32 -12
- package/out/zero-cache/src/server/replicator.js.map +1 -1
- package/out/zero-cache/src/server/syncer-assigner.d.ts +14 -0
- package/out/zero-cache/src/server/syncer-assigner.d.ts.map +1 -0
- package/out/zero-cache/src/server/syncer-assigner.js +72 -0
- package/out/zero-cache/src/server/syncer-assigner.js.map +1 -0
- package/out/zero-cache/src/server/syncer.d.ts.map +1 -1
- package/out/zero-cache/src/server/syncer.js +3 -1
- package/out/zero-cache/src/server/syncer.js.map +1 -1
- package/out/zero-cache/src/server/worker-dispatcher.d.ts.map +1 -1
- package/out/zero-cache/src/server/worker-dispatcher.js +21 -4
- package/out/zero-cache/src/server/worker-dispatcher.js.map +1 -1
- package/out/zero-cache/src/services/analyze.d.ts.map +1 -1
- package/out/zero-cache/src/services/analyze.js +8 -7
- package/out/zero-cache/src/services/analyze.js.map +1 -1
- package/out/zero-cache/src/services/change-source/common/backfill-manager.d.ts.map +1 -1
- package/out/zero-cache/src/services/change-source/common/backfill-manager.js +1 -2
- package/out/zero-cache/src/services/change-source/common/backfill-manager.js.map +1 -1
- package/out/zero-cache/src/services/change-streamer/change-streamer-http.d.ts +45 -6
- package/out/zero-cache/src/services/change-streamer/change-streamer-http.d.ts.map +1 -1
- package/out/zero-cache/src/services/change-streamer/change-streamer-http.js +117 -32
- package/out/zero-cache/src/services/change-streamer/change-streamer-http.js.map +1 -1
- package/out/zero-cache/src/services/change-streamer/change-streamer-service.d.ts +5 -0
- package/out/zero-cache/src/services/change-streamer/change-streamer-service.d.ts.map +1 -1
- package/out/zero-cache/src/services/change-streamer/change-streamer-service.js +2 -2
- package/out/zero-cache/src/services/change-streamer/change-streamer-service.js.map +1 -1
- package/out/zero-cache/src/services/change-streamer/change-streamer.d.ts +10 -17
- package/out/zero-cache/src/services/change-streamer/change-streamer.d.ts.map +1 -1
- package/out/zero-cache/src/services/change-streamer/change-streamer.js.map +1 -1
- package/out/zero-cache/src/services/change-streamer/schema/tables.d.ts.map +1 -1
- package/out/zero-cache/src/services/change-streamer/schema/tables.js +15 -4
- package/out/zero-cache/src/services/change-streamer/schema/tables.js.map +1 -1
- package/out/zero-cache/src/services/change-streamer/snapshot-message.d.ts +92 -0
- package/out/zero-cache/src/services/change-streamer/snapshot-message.d.ts.map +1 -0
- package/out/zero-cache/src/services/change-streamer/snapshot-message.js +42 -0
- package/out/zero-cache/src/services/change-streamer/snapshot-message.js.map +1 -0
- package/out/zero-cache/src/services/change-streamer/snapshot-reservations.d.ts +1 -1
- package/out/zero-cache/src/services/change-streamer/snapshot-reservations.d.ts.map +1 -1
- package/out/zero-cache/src/services/change-streamer/snapshot-reservations.js +16 -4
- package/out/zero-cache/src/services/change-streamer/snapshot-reservations.js.map +1 -1
- package/out/zero-cache/src/services/change-streamer/snapshot.d.ts +7 -71
- package/out/zero-cache/src/services/change-streamer/snapshot.d.ts.map +1 -1
- package/out/zero-cache/src/services/change-streamer/snapshot.js +26 -92
- package/out/zero-cache/src/services/change-streamer/snapshot.js.map +1 -1
- package/out/zero-cache/src/services/change-streamer/subscribe.d.ts +358 -0
- package/out/zero-cache/src/services/change-streamer/subscribe.d.ts.map +1 -0
- package/out/zero-cache/src/services/change-streamer/subscribe.js +68 -0
- package/out/zero-cache/src/services/change-streamer/subscribe.js.map +1 -0
- package/out/zero-cache/src/services/litestream/commands.d.ts +2 -3
- package/out/zero-cache/src/services/litestream/commands.d.ts.map +1 -1
- package/out/zero-cache/src/services/litestream/commands.js +3 -2
- package/out/zero-cache/src/services/litestream/commands.js.map +1 -1
- package/out/zero-cache/src/services/litestream/restore-progress.d.ts +24 -0
- package/out/zero-cache/src/services/litestream/restore-progress.d.ts.map +1 -0
- package/out/zero-cache/src/services/litestream/restore-progress.js +91 -0
- package/out/zero-cache/src/services/litestream/restore-progress.js.map +1 -0
- package/out/zero-cache/src/services/mutagen/pusher.d.ts.map +1 -1
- package/out/zero-cache/src/services/mutagen/pusher.js +3 -5
- package/out/zero-cache/src/services/mutagen/pusher.js.map +1 -1
- package/out/zero-cache/src/services/replicator/incremental-sync.d.ts +2 -1
- package/out/zero-cache/src/services/replicator/incremental-sync.d.ts.map +1 -1
- package/out/zero-cache/src/services/replicator/incremental-sync.js +13 -7
- package/out/zero-cache/src/services/replicator/incremental-sync.js.map +1 -1
- package/out/zero-cache/src/services/replicator/replicator.d.ts +2 -1
- package/out/zero-cache/src/services/replicator/replicator.d.ts.map +1 -1
- package/out/zero-cache/src/services/replicator/replicator.js +2 -2
- package/out/zero-cache/src/services/replicator/replicator.js.map +1 -1
- package/out/zero-cache/src/services/runner.d.ts +2 -1
- package/out/zero-cache/src/services/runner.d.ts.map +1 -1
- package/out/zero-cache/src/services/runner.js +10 -2
- package/out/zero-cache/src/services/runner.js.map +1 -1
- package/out/zero-cache/src/services/view-syncer/client-handler.d.ts +19 -2
- package/out/zero-cache/src/services/view-syncer/client-handler.d.ts.map +1 -1
- package/out/zero-cache/src/services/view-syncer/client-handler.js +8 -1
- package/out/zero-cache/src/services/view-syncer/client-handler.js.map +1 -1
- package/out/zero-cache/src/services/view-syncer/cvr-store.d.ts +8 -1
- package/out/zero-cache/src/services/view-syncer/cvr-store.d.ts.map +1 -1
- package/out/zero-cache/src/services/view-syncer/cvr-store.js +14 -4
- package/out/zero-cache/src/services/view-syncer/cvr-store.js.map +1 -1
- package/out/zero-cache/src/services/view-syncer/cvr.d.ts +6 -2
- package/out/zero-cache/src/services/view-syncer/cvr.d.ts.map +1 -1
- package/out/zero-cache/src/services/view-syncer/cvr.js +8 -4
- package/out/zero-cache/src/services/view-syncer/cvr.js.map +1 -1
- package/out/zero-cache/src/services/view-syncer/pipeline-driver.d.ts +12 -0
- package/out/zero-cache/src/services/view-syncer/pipeline-driver.d.ts.map +1 -1
- package/out/zero-cache/src/services/view-syncer/pipeline-driver.js +44 -13
- package/out/zero-cache/src/services/view-syncer/pipeline-driver.js.map +1 -1
- package/out/zero-cache/src/services/view-syncer/query-covering.d.ts.map +1 -1
- package/out/zero-cache/src/services/view-syncer/query-covering.js +2 -6
- package/out/zero-cache/src/services/view-syncer/query-covering.js.map +1 -1
- package/out/zero-cache/src/services/view-syncer/snapshot-row-cache.d.ts +95 -0
- package/out/zero-cache/src/services/view-syncer/snapshot-row-cache.d.ts.map +1 -0
- package/out/zero-cache/src/services/view-syncer/snapshot-row-cache.js +155 -0
- package/out/zero-cache/src/services/view-syncer/snapshot-row-cache.js.map +1 -0
- package/out/zero-cache/src/services/view-syncer/snapshotter.d.ts +65 -4
- package/out/zero-cache/src/services/view-syncer/snapshotter.d.ts.map +1 -1
- package/out/zero-cache/src/services/view-syncer/snapshotter.js +127 -28
- package/out/zero-cache/src/services/view-syncer/snapshotter.js.map +1 -1
- package/out/zero-cache/src/services/view-syncer/view-syncer.d.ts.map +1 -1
- package/out/zero-cache/src/services/view-syncer/view-syncer.js +41 -18
- package/out/zero-cache/src/services/view-syncer/view-syncer.js.map +1 -1
- package/out/zero-cache/src/types/processes.d.ts +7 -0
- package/out/zero-cache/src/types/processes.d.ts.map +1 -1
- package/out/zero-cache/src/types/processes.js.map +1 -1
- package/out/zero-cache/src/types/row-key.d.ts.map +1 -1
- package/out/zero-cache/src/types/row-key.js +5 -5
- package/out/zero-cache/src/types/row-key.js.map +1 -1
- package/out/zero-cache/src/types/streams.d.ts +33 -7
- package/out/zero-cache/src/types/streams.d.ts.map +1 -1
- package/out/zero-cache/src/types/streams.js +93 -50
- package/out/zero-cache/src/types/streams.js.map +1 -1
- package/out/zero-cache/src/types/subscription.d.ts +11 -0
- package/out/zero-cache/src/types/subscription.d.ts.map +1 -1
- package/out/zero-cache/src/types/subscription.js +56 -35
- package/out/zero-cache/src/types/subscription.js.map +1 -1
- package/out/zero-cache/src/types/websocket-handoff.d.ts +1 -1
- package/out/zero-cache/src/types/websocket-handoff.d.ts.map +1 -1
- package/out/zero-cache/src/types/websocket-handoff.js +18 -9
- package/out/zero-cache/src/types/websocket-handoff.js.map +1 -1
- package/out/zero-cache/src/types/ws.js +1 -31
- package/out/zero-cache/src/types/ws.js.map +1 -1
- package/out/zero-cache/src/workers/connect-params.d.ts +1 -0
- package/out/zero-cache/src/workers/connect-params.d.ts.map +1 -1
- package/out/zero-cache/src/workers/connect-params.js.map +1 -1
- package/out/zero-cache/src/workers/syncer.d.ts.map +1 -1
- package/out/zero-cache/src/workers/syncer.js +45 -3
- package/out/zero-cache/src/workers/syncer.js.map +1 -1
- package/out/zero-client/src/client/query-manager.d.ts.map +1 -1
- package/out/zero-client/src/client/query-manager.js +2 -5
- package/out/zero-client/src/client/query-manager.js.map +1 -1
- package/out/zero-client/src/client/version.js +1 -1
- package/out/zero-client/src/client/zero.d.ts.map +1 -1
- package/out/zero-client/src/client/zero.js +5 -5
- package/out/zero-client/src/client/zero.js.map +1 -1
- package/out/zero-events/src/status.d.ts +32 -0
- package/out/zero-events/src/status.d.ts.map +1 -1
- package/out/zero-protocol/src/ast.d.ts.map +1 -1
- package/out/zero-protocol/src/ast.js +13 -14
- package/out/zero-protocol/src/ast.js.map +1 -1
- package/out/zero-protocol/src/down.d.ts +2 -2
- package/out/zero-protocol/src/query-hash.d.ts.map +1 -1
- package/out/zero-protocol/src/query-hash.js +2 -5
- package/out/zero-protocol/src/query-hash.js.map +1 -1
- package/out/zql/src/builder/builder.d.ts +19 -0
- package/out/zql/src/builder/builder.d.ts.map +1 -1
- package/out/zql/src/builder/builder.js +41 -13
- package/out/zql/src/builder/builder.js.map +1 -1
- package/out/zql/src/builder/correlated-predicate-pushdown.d.ts +43 -0
- package/out/zql/src/builder/correlated-predicate-pushdown.d.ts.map +1 -0
- package/out/zql/src/builder/correlated-predicate-pushdown.js +161 -0
- package/out/zql/src/builder/correlated-predicate-pushdown.js.map +1 -0
- package/out/zql/src/ivm/cap.d.ts +2 -1
- package/out/zql/src/ivm/cap.d.ts.map +1 -1
- package/out/zql/src/ivm/cap.js +8 -1
- package/out/zql/src/ivm/cap.js.map +1 -1
- package/out/zql/src/ivm/connection-index.d.ts +68 -0
- package/out/zql/src/ivm/connection-index.d.ts.map +1 -0
- package/out/zql/src/ivm/connection-index.js +149 -0
- package/out/zql/src/ivm/connection-index.js.map +1 -0
- package/out/zql/src/ivm/exists.d.ts +2 -0
- package/out/zql/src/ivm/exists.d.ts.map +1 -1
- package/out/zql/src/ivm/exists.js +4 -1
- package/out/zql/src/ivm/exists.js.map +1 -1
- package/out/zql/src/ivm/fan-in.d.ts +5 -1
- package/out/zql/src/ivm/fan-in.d.ts.map +1 -1
- package/out/zql/src/ivm/fan-in.js +7 -0
- package/out/zql/src/ivm/fan-in.js.map +1 -1
- package/out/zql/src/ivm/fan-out.d.ts +3 -0
- package/out/zql/src/ivm/fan-out.d.ts.map +1 -1
- package/out/zql/src/ivm/fan-out.js +4 -0
- package/out/zql/src/ivm/fan-out.js.map +1 -1
- package/out/zql/src/ivm/filter-operators.d.ts +2 -0
- package/out/zql/src/ivm/filter-operators.d.ts.map +1 -1
- package/out/zql/src/ivm/filter-operators.js +9 -0
- package/out/zql/src/ivm/filter-operators.js.map +1 -1
- package/out/zql/src/ivm/filter.d.ts +4 -1
- package/out/zql/src/ivm/filter.d.ts.map +1 -1
- package/out/zql/src/ivm/filter.js +3 -0
- package/out/zql/src/ivm/filter.js.map +1 -1
- package/out/zql/src/ivm/flipped-join.d.ts +8 -5
- package/out/zql/src/ivm/flipped-join.d.ts.map +1 -1
- package/out/zql/src/ivm/flipped-join.js +85 -44
- package/out/zql/src/ivm/flipped-join.js.map +1 -1
- package/out/zql/src/ivm/join-utils.d.ts +27 -0
- package/out/zql/src/ivm/join-utils.d.ts.map +1 -1
- package/out/zql/src/ivm/join-utils.js +106 -1
- package/out/zql/src/ivm/join-utils.js.map +1 -1
- package/out/zql/src/ivm/join.d.ts +6 -2
- package/out/zql/src/ivm/join.d.ts.map +1 -1
- package/out/zql/src/ivm/join.js +68 -15
- package/out/zql/src/ivm/join.js.map +1 -1
- package/out/zql/src/ivm/memory-source.d.ts.map +1 -1
- package/out/zql/src/ivm/memory-source.js +33 -23
- package/out/zql/src/ivm/memory-source.js.map +1 -1
- package/out/zql/src/ivm/operator.d.ts +39 -10
- package/out/zql/src/ivm/operator.d.ts.map +1 -1
- package/out/zql/src/ivm/operator.js +9 -4
- package/out/zql/src/ivm/operator.js.map +1 -1
- package/out/zql/src/ivm/push-accumulated.d.ts +6 -1
- package/out/zql/src/ivm/push-accumulated.d.ts.map +1 -1
- package/out/zql/src/ivm/push-accumulated.js +47 -2
- package/out/zql/src/ivm/push-accumulated.js.map +1 -1
- package/out/zql/src/ivm/skip.d.ts +2 -1
- package/out/zql/src/ivm/skip.d.ts.map +1 -1
- package/out/zql/src/ivm/skip.js +3 -0
- package/out/zql/src/ivm/skip.js.map +1 -1
- package/out/zql/src/ivm/take-gate.d.ts +36 -0
- package/out/zql/src/ivm/take-gate.d.ts.map +1 -0
- package/out/zql/src/ivm/take-gate.js +84 -0
- package/out/zql/src/ivm/take-gate.js.map +1 -0
- package/out/zql/src/ivm/take.d.ts +8 -2
- package/out/zql/src/ivm/take.d.ts.map +1 -1
- package/out/zql/src/ivm/take.js +136 -100
- package/out/zql/src/ivm/take.js.map +1 -1
- package/out/zql/src/ivm/union-fan-in.d.ts +3 -0
- package/out/zql/src/ivm/union-fan-in.d.ts.map +1 -1
- package/out/zql/src/ivm/union-fan-in.js +56 -5
- package/out/zql/src/ivm/union-fan-in.js.map +1 -1
- package/out/zql/src/ivm/union-fan-out.d.ts +2 -1
- package/out/zql/src/ivm/union-fan-out.d.ts.map +1 -1
- package/out/zql/src/ivm/union-fan-out.js +5 -0
- package/out/zql/src/ivm/union-fan-out.js.map +1 -1
- package/out/zql/src/planner/planner-builder.d.ts +12 -3
- package/out/zql/src/planner/planner-builder.d.ts.map +1 -1
- package/out/zql/src/planner/planner-builder.js +27 -17
- package/out/zql/src/planner/planner-builder.js.map +1 -1
- package/out/zql/src/planner/planner-connection.d.ts +15 -2
- package/out/zql/src/planner/planner-connection.d.ts.map +1 -1
- package/out/zql/src/planner/planner-connection.js +73 -8
- package/out/zql/src/planner/planner-connection.js.map +1 -1
- package/out/zql/src/planner/planner-debug.d.ts.map +1 -1
- package/out/zql/src/planner/planner-debug.js +3 -8
- package/out/zql/src/planner/planner-debug.js.map +1 -1
- package/out/zql/src/planner/planner-source.d.ts +2 -2
- package/out/zql/src/planner/planner-source.d.ts.map +1 -1
- package/out/zql/src/planner/planner-source.js +2 -2
- package/out/zql/src/planner/planner-source.js.map +1 -1
- package/out/zql/src/query/measure-push-operator.d.ts +2 -1
- package/out/zql/src/query/measure-push-operator.d.ts.map +1 -1
- package/out/zql/src/query/measure-push-operator.js +7 -0
- package/out/zql/src/query/measure-push-operator.js.map +1 -1
- package/out/zql/src/query/query-impl.d.ts.map +1 -1
- package/out/zql/src/query/query-impl.js +6 -23
- package/out/zql/src/query/query-impl.js.map +1 -1
- package/out/zql/src/query/query-transitions.d.ts.map +1 -1
- package/out/zql/src/query/query-transitions.js +2 -5
- package/out/zql/src/query/query-transitions.js.map +1 -1
- package/out/zql/src/query/runnable-query-impl.d.ts.map +1 -1
- package/out/zql/src/query/runnable-query-impl.js +2 -10
- package/out/zql/src/query/runnable-query-impl.js.map +1 -1
- package/out/zqlite/src/query-delegate.d.ts.map +1 -1
- package/out/zqlite/src/query-delegate.js +5 -6
- package/out/zqlite/src/query-delegate.js.map +1 -1
- package/out/zqlite/src/sqlite-stat-fanout.d.ts.map +1 -1
- package/out/zqlite/src/sqlite-stat-fanout.js +3 -5
- package/out/zqlite/src/sqlite-stat-fanout.js.map +1 -1
- package/out/zqlite/src/table-source.d.ts +14 -2
- package/out/zqlite/src/table-source.d.ts.map +1 -1
- package/out/zqlite/src/table-source.js +15 -9
- package/out/zqlite/src/table-source.js.map +1 -1
- package/package.json +1 -1
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"metrics.js","names":[],"sources":["../../../../../zero-cache/src/observability/metrics.ts"],"sourcesContent":["import type {\n Counter,\n Histogram,\n Meter,\n MetricOptions,\n ObservableGauge,\n UpDownCounter,\n} from '@opentelemetry/api';\nimport {metrics} from '@opentelemetry/api';\n\n// intentional lazy initialization so it is not started before the SDK is started.\n\nexport type Category =\n | 'replication' // postgres to replica\n | 'replica' // health of replica and litestream backup\n | 'sync' // replica to client\n | 'mutation'\n | 'server';\n\nlet meter: Meter | undefined;\n\ntype Options = MetricOptions & {description: string};\ntype HistogramOptions = Options & {bucketBoundaries: number[]};\n\nexport const NATIVE_HISTOGRAM_INSTRUMENT_NAMES = [\n 'zero.sync.view_syncer_lag',\n 'zero.sync.view_syncer_hydration',\n 'zero.sync.e2e_serving_lag',\n] as const;\n\nexport const LONG_DURATION_HISTOGRAM_BOUNDARIES_S = [\n 1, 2, 5, 10, 30, 60, 120, 300, 600, 1200, 2400, 3600, 7200,\n];\n\nfunction getMeter() {\n if (!meter) {\n meter = metrics.getMeter('zero');\n }\n return meter;\n}\n\nfunction cache<TRet>(): (\n name: string,\n creator: (name: string) => TRet,\n) => TRet {\n const instruments = new Map<string, TRet>();\n return (name: string, creator: (name: string) => TRet)
|
|
1
|
+
{"version":3,"file":"metrics.js","names":[],"sources":["../../../../../zero-cache/src/observability/metrics.ts"],"sourcesContent":["import type {\n Counter,\n Histogram,\n Meter,\n MetricOptions,\n ObservableGauge,\n UpDownCounter,\n} from '@opentelemetry/api';\nimport {metrics} from '@opentelemetry/api';\nimport {getOrInsertComputed} from '../../../shared/src/map.ts';\n\n// intentional lazy initialization so it is not started before the SDK is started.\n\nexport type Category =\n | 'replication' // postgres to replica\n | 'replica' // health of replica and litestream backup\n | 'sync' // replica to client\n | 'mutation'\n | 'server';\n\nlet meter: Meter | undefined;\n\ntype Options = MetricOptions & {description: string};\ntype HistogramOptions = Options & {bucketBoundaries: number[]};\n\nexport const NATIVE_HISTOGRAM_INSTRUMENT_NAMES = [\n 'zero.sync.view_syncer_lag',\n 'zero.sync.view_syncer_hydration',\n 'zero.sync.e2e_serving_lag',\n] as const;\n\nexport const LONG_DURATION_HISTOGRAM_BOUNDARIES_S = [\n 1, 2, 5, 10, 30, 60, 120, 300, 600, 1200, 2400, 3600, 7200,\n];\n\nfunction getMeter() {\n if (!meter) {\n meter = metrics.getMeter('zero');\n }\n return meter;\n}\n\nfunction cache<TRet extends object>(): (\n name: string,\n creator: (name: string) => TRet,\n) => TRet {\n const instruments = new Map<string, TRet>();\n return (name: string, creator: (name: string) => TRet) =>\n getOrInsertComputed(instruments, name, creator);\n}\n\nconst upDownCounters = cache<UpDownCounter>();\n\nexport function getOrCreateUpDownCounter(\n category: Category,\n name: string,\n description: string,\n): UpDownCounter;\nexport function getOrCreateUpDownCounter(\n category: Category,\n name: string,\n opts: Options,\n): UpDownCounter;\nexport function getOrCreateUpDownCounter(\n category: Category,\n name: string,\n opts: string | Options,\n): UpDownCounter {\n return upDownCounters(name, name =>\n getMeter().createUpDownCounter(\n `zero.${category}.${name}`,\n typeof opts === 'string' ? {description: opts} : opts,\n ),\n );\n}\n\n/**\n * A latency histogram whose {@link recordMs} method accepts raw millisecond\n * durations and converts them to seconds internally.\n *\n * Use {@link getOrCreateLatencyHistogram} to create one — the unit (`'s'`),\n * bucket boundaries, and ms→s conversion are all baked in\n */\nexport type LatencyHistogram = {\n /**\n * Record a duration. Pass the raw elapsed milliseconds — the conversion to\n * seconds (required by the `unit: 's'` OTel histogram) is handled internally.\n *\n * @param durationMs Elapsed time in **milliseconds** (do NOT pre-divide).\n * @param attributes Optional OTel attributes to attach to the observation.\n */\n recordMs(\n durationMs: number,\n attributes?: Parameters<Histogram['record']>[1],\n ): void;\n};\n\n/**\n * Bucket boundaries (in seconds) for zero's latency histograms.\n *\n * The operational range is 1 ms – 5,000 ms (including customers actively\n * tuning queries). ~2× logarithmic steps give proportionally consistent\n * `histogram_quantile` accuracy regardless of where values cluster within\n * that range. 10,000 ms and 30,000 ms are overflow catchers for truly broken\n * states.\n *\n * 1 ms, 2 ms, 5 ms, 10 ms, 20 ms, 50 ms, 100 ms, 200 ms, 500 ms,\n * 1 s, 2 s, 5 s, 10 s, 30 s\n */\nconst LATENCY_HISTOGRAM_BOUNDARIES_S = [\n 0.001, 0.002, 0.005, 0.01, 0.02, 0.05, 0.1, 0.2, 0.5, 1, 2, 5, 10, 30,\n];\n\nconst histograms = cache<Histogram>();\n\n/**\n * Creates a histogram for dimensionless values without applying the\n * millisecond-to-second conversion used by the latency helpers below.\n */\nexport function getOrCreateValueHistogram(\n category: Category,\n name: string,\n opts: HistogramOptions,\n): Histogram {\n const {bucketBoundaries, ...metricOptions} = opts;\n return histograms(name, name =>\n getMeter().createHistogram(`zero.${category}.${name}`, {\n ...metricOptions,\n advice: {\n ...metricOptions.advice,\n explicitBucketBoundaries: bucketBoundaries,\n },\n }),\n );\n}\n\nexport function getOrCreateHistogram(\n category: Category,\n name: string,\n opts: HistogramOptions,\n): LatencyHistogram {\n const {bucketBoundaries, ...metricOptions} = opts;\n const h = histograms(name, name =>\n getMeter().createHistogram(`zero.${category}.${name}`, {\n ...metricOptions,\n advice: {\n ...metricOptions.advice,\n explicitBucketBoundaries: bucketBoundaries,\n },\n }),\n );\n return {\n recordMs: (durationMs, attributes) =>\n h.record(durationMs / 1000, attributes),\n };\n}\n\n// Add instruments created through this helper to\n// NATIVE_HISTOGRAM_INSTRUMENT_NAMES so the SDK exports them as exponential\n// histograms.\nexport function getOrCreateNativeHistogram(\n category: Category,\n name: string,\n opts: Options,\n): LatencyHistogram {\n const h = histograms(name, name =>\n getMeter().createHistogram(`zero.${category}.${name}`, opts),\n );\n return {\n recordMs: (durationMs, attributes) =>\n h.record(durationMs / 1000, attributes),\n };\n}\n\n/**\n * Creates (or retrieves) a latency histogram for the given metric.\n *\n * - `unit` is always `'s'` (seconds), matching the OTel convention.\n * - Bucket boundaries are pre-set for zero's typical operation range\n * (1 ms – 5 s); see {@link LATENCY_HISTOGRAM_BOUNDARIES_S}.\n * - The returned {@link LatencyHistogram} accepts **milliseconds** via\n * `recordMs()`, so callers never need to divide by 1000.\n *\n * @example\n * ```ts\n * readonly #hydrationTime = getOrCreateLatencyHistogram(\n * 'sync', 'hydration-time', 'Time to hydrate a query.',\n * );\n * // ...\n * this.#hydrationTime.recordMs(performance.now() - start);\n * ```\n */\nexport function getOrCreateLatencyHistogram(\n category: Category,\n name: string,\n description: string,\n): LatencyHistogram {\n return getOrCreateHistogram(category, name, {\n description,\n unit: 's',\n bucketBoundaries: LATENCY_HISTOGRAM_BOUNDARIES_S,\n });\n}\n\nconst counters = cache<Counter>();\n\nexport function getOrCreateCounter(\n category: Category,\n name: string,\n description: string,\n): Counter;\nexport function getOrCreateCounter(\n category: Category,\n name: string,\n opts: Options,\n): Counter;\nexport function getOrCreateCounter(\n category: Category,\n name: string,\n opts: string | Options,\n): Counter {\n return counters(name, name =>\n getMeter().createCounter(\n `zero.${category}.${name}`,\n typeof opts === 'string' ? {description: opts} : opts,\n ),\n );\n}\n\nconst gauges = cache<ObservableGauge>();\n\nexport function getOrCreateGauge(\n category: Category,\n name: string,\n description: string,\n): ObservableGauge;\nexport function getOrCreateGauge(\n category: Category,\n name: string,\n opts: Options,\n): ObservableGauge;\nexport function getOrCreateGauge(\n category: Category,\n name: string,\n opts: string | Options,\n): ObservableGauge {\n return gauges(name, name =>\n getMeter().createObservableGauge(\n `zero.${category}.${name}`,\n typeof opts === 'string' ? {description: opts} : opts,\n ),\n );\n}\n"],"mappings":";;;AAoBA,IAAI;AAKJ,IAAa,oCAAoC;CAC/C;CACA;CACA;AACF;AAEA,IAAa,uCAAuC;CAClD;CAAG;CAAG;CAAG;CAAI;CAAI;CAAI;CAAK;CAAK;CAAK;CAAM;CAAM;CAAM;AACxD;AAEA,SAAS,WAAW;CAClB,IAAI,CAAC,OACH,QAAQ,QAAQ,SAAS,MAAM;CAEjC,OAAO;AACT;AAEA,SAAS,QAGC;CACR,MAAM,8BAAc,IAAI,IAAkB;CAC1C,QAAQ,MAAc,YACpB,oBAAoB,aAAa,MAAM,OAAO;AAClD;AAEA,IAAM,iBAAiB,MAAqB;AAY5C,SAAgB,yBACd,UACA,MACA,MACe;CACf,OAAO,eAAe,OAAM,SAC1B,SAAS,EAAE,oBACT,QAAQ,SAAS,GAAG,QACpB,OAAO,SAAS,WAAW,EAAC,aAAa,KAAI,IAAI,IACnD,CACF;AACF;;;;;;;;;;;;;AAmCA,IAAM,iCAAiC;CACrC;CAAO;CAAO;CAAO;CAAM;CAAM;CAAM;CAAK;CAAK;CAAK;CAAG;CAAG;CAAG;CAAI;AACrE;AAEA,IAAM,aAAa,MAAiB;;;;;AAMpC,SAAgB,0BACd,UACA,MACA,MACW;CACX,MAAM,EAAC,kBAAkB,GAAG,kBAAiB;CAC7C,OAAO,WAAW,OAAM,SACtB,SAAS,EAAE,gBAAgB,QAAQ,SAAS,GAAG,QAAQ;EACrD,GAAG;EACH,QAAQ;GACN,GAAG,cAAc;GACjB,0BAA0B;EAC5B;CACF,CAAC,CACH;AACF;AAEA,SAAgB,qBACd,UACA,MACA,MACkB;CAClB,MAAM,EAAC,kBAAkB,GAAG,kBAAiB;CAC7C,MAAM,IAAI,WAAW,OAAM,SACzB,SAAS,EAAE,gBAAgB,QAAQ,SAAS,GAAG,QAAQ;EACrD,GAAG;EACH,QAAQ;GACN,GAAG,cAAc;GACjB,0BAA0B;EAC5B;CACF,CAAC,CACH;CACA,OAAO,EACL,WAAW,YAAY,eACrB,EAAE,OAAO,aAAa,KAAM,UAAU,EAC1C;AACF;AAKA,SAAgB,2BACd,UACA,MACA,MACkB;CAClB,MAAM,IAAI,WAAW,OAAM,SACzB,SAAS,EAAE,gBAAgB,QAAQ,SAAS,GAAG,QAAQ,IAAI,CAC7D;CACA,OAAO,EACL,WAAW,YAAY,eACrB,EAAE,OAAO,aAAa,KAAM,UAAU,EAC1C;AACF;;;;;;;;;;;;;;;;;;;AAoBA,SAAgB,4BACd,UACA,MACA,aACkB;CAClB,OAAO,qBAAqB,UAAU,MAAM;EAC1C;EACA,MAAM;EACN,kBAAkB;CACpB,CAAC;AACH;AAEA,IAAM,WAAW,MAAe;AAYhC,SAAgB,mBACd,UACA,MACA,MACS;CACT,OAAO,SAAS,OAAM,SACpB,SAAS,EAAE,cACT,QAAQ,SAAS,GAAG,QACpB,OAAO,SAAS,WAAW,EAAC,aAAa,KAAI,IAAI,IACnD,CACF;AACF;AAEA,IAAM,SAAS,MAAuB;AAYtC,SAAgB,iBACd,UACA,MACA,MACiB;CACjB,OAAO,OAAO,OAAM,SAClB,SAAS,EAAE,sBACT,QAAQ,SAAS,GAAG,QACpB,OAAO,SAAS,WAAW,EAAC,aAAa,KAAI,IAAI,IACnD,CACF;AACF"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"change-streamer.d.ts","sourceRoot":"","sources":["../../../../../zero-cache/src/server/change-streamer.ts"],"names":[],"mappings":"AAsCA,OAAO,EAML,KAAK,MAAM,EACZ,MAAM,uBAAuB,CAAC;AAW/B,wBAA8B,SAAS,CACrC,MAAM,EAAE,MAAM,EACd,GAAG,EAAE,MAAM,CAAC,UAAU,EACtB,GAAG,IAAI,EAAE,MAAM,EAAE,GAChB,OAAO,CAAC,IAAI,CAAC,
|
|
1
|
+
{"version":3,"file":"change-streamer.d.ts","sourceRoot":"","sources":["../../../../../zero-cache/src/server/change-streamer.ts"],"names":[],"mappings":"AAsCA,OAAO,EAML,KAAK,MAAM,EACZ,MAAM,uBAAuB,CAAC;AAW/B,wBAA8B,SAAS,CACrC,MAAM,EAAE,MAAM,EACd,GAAG,EAAE,MAAM,CAAC,UAAU,EACtB,GAAG,IAAI,EAAE,MAAM,EAAE,GAChB,OAAO,CAAC,IAAI,CAAC,CAwXf"}
|
|
@@ -88,7 +88,8 @@ async function runWorker(parent, env, ...argv) {
|
|
|
88
88
|
const replicationStatusPublisher = ReplicationStatusPublisher.forReplicaFile(replica.file);
|
|
89
89
|
changeStreamer = await initializeStreamer(lc, shard, taskID, address, protocol, changeDB, changeSource, replicationStatusPublisher, subscriptionState, destinationBackupURL ? {
|
|
90
90
|
backupURL: destinationBackupURL,
|
|
91
|
-
litestreamVersion: litestream.backupUsingV5 ? "v5" : "legacy"
|
|
91
|
+
litestreamVersion: litestream.backupUsingV5 ? "v5" : "legacy",
|
|
92
|
+
replicaFile: replica.file
|
|
92
93
|
} : null, purgeLock, autoReset ?? false, {
|
|
93
94
|
pgChangeLogEnabled,
|
|
94
95
|
backPressureLimitHeapProportion,
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"change-streamer.js","names":[],"sources":["../../../../../zero-cache/src/server/change-streamer.ts"],"sourcesContent":["import {consoleLogSink, LogContext} from '@rocicorp/logger';\nimport {resolver} from '@rocicorp/resolver';\nimport {assert} from '../../../shared/src/asserts.ts';\nimport {must} from '../../../shared/src/must.ts';\nimport {promiseVoid} from '../../../shared/src/resolved-promises.ts';\nimport {DatabaseInitError} from '../../../zqlite/src/db.ts';\nimport {getServerContext} from '../config/server-context.ts';\nimport {getNormalizedZeroConfig} from '../config/zero-config.ts';\nimport {deleteLiteDB} from '../db/delete-lite-db.ts';\nimport {registerSQLiteCorruptionDiagnosticTarget} from '../db/sqlite-corruption.ts';\nimport {warmupConnections} from '../db/warmup.ts';\nimport {initEventSink, publishCriticalEvent} from '../observability/events.ts';\nimport {getOrCreateGauge} from '../observability/metrics.ts';\nimport {initializeCustomChangeSource} from '../services/change-source/custom/change-source.ts';\nimport {initializePostgresChangeSource} from '../services/change-source/pg/change-source-init.ts';\nimport {createBackupCleanupMonitor} from '../services/change-streamer/backup-cleanup-monitor-factory.ts';\nimport {ChangeStreamerHttpServer} from '../services/change-streamer/change-streamer-http.ts';\nimport {initializeStreamer} from '../services/change-streamer/change-streamer-service.ts';\nimport type {ChangeStreamerService} from '../services/change-streamer/change-streamer.ts';\nimport {initChangeStreamerSchema} from '../services/change-streamer/schema/init.ts';\nimport {AutoResetSignal} from '../services/change-streamer/schema/tables.ts';\nimport {PurgeLocker} from '../services/change-streamer/storer.ts';\nimport {\n exitAfter,\n ProcessManager,\n runUntilKilled,\n} from '../services/life-cycle.ts';\nimport {startReplicaBackupProcess} from '../services/litestream/commands.ts';\nimport {\n changeLogFileName,\n deleteChangeLogDB,\n} from '../services/replicator/change-log-db.ts';\nimport {\n replicationStatusError,\n ReplicationStatusPublisher,\n} from '../services/replicator/replication-status.ts';\nimport {sqliteFileBytes} from '../services/replicator/sqlite-change-log-observability.ts';\nimport {connectPgClient} from '../types/pg.ts';\nimport {\n broadcastWorker,\n childWorker,\n parentWorker,\n singleProcessMode,\n type ProfileResponseMessage,\n type Worker,\n} from '../types/processes.ts';\nimport {installProfileHandler} from '../types/profiler.ts';\nimport {getShardConfig} from '../types/shards.ts';\nimport type {ReplicaFileMode} from '../workers/replicator.ts';\nimport {createLogContext} from './logging.ts';\nimport {startOtelAuto} from './otel-start.ts';\nimport {REPLICATOR_URL} from './worker-urls.ts';\n\n// Default LogContext, overridden in runWorker\nlet lc = new LogContext('info', {}, consoleLogSink);\n\nexport default async function runWorker(\n parent: Worker,\n env: NodeJS.ProcessEnv,\n ...argv: string[]\n): Promise<void> {\n installProfileHandler(parent, 'change-streamer');\n const config = getNormalizedZeroConfig({env, argv});\n const {\n taskID,\n changeStreamer: {\n port,\n address,\n protocol,\n startupDelayMs,\n backPressureLimitHeapProportion,\n flowControlConsensusTimeoutProportion,\n flowControlSlowSubscriberGracePeriodSeconds,\n pgChangeLogEnabled,\n sqliteChangeLogMode,\n sqliteChangeLogReadPercent,\n sqliteChangeLogColdReadPercent,\n sqliteChangeLogComparePercent,\n sqliteChangeLogRetentionMs,\n sqliteChangeLogReadBatchRows,\n sqliteChangeLogPurgeBatchRows,\n sqliteChangeLogBarrierTimeoutMs,\n },\n autoReset,\n replicationLag,\n litestream,\n upstream,\n change,\n replica,\n initialSync,\n keepaliveTimeoutMs,\n sqliteCorruptionChecks,\n } = config;\n\n startOtelAuto(\n createLogContext(config, 'change-streamer', 0, false),\n 'change-streamer',\n 0,\n );\n lc = createLogContext(config, 'change-streamer');\n registerSQLiteCorruptionDiagnosticTarget(\n {\n debugName: 'change-streamer replica',\n dbPath: replica.file,\n },\n sqliteCorruptionChecks,\n );\n initEventSink(lc, config);\n\n // Startup-time client used for for change-streamer initialization\n // and handoff / takeover. Steady-state clients are managed by the\n // change-streamer (Storer) implementation.\n const changeDB = await connectPgClient(\n lc,\n change.db,\n 'change-streamer-init',\n {max: 5},\n {sendStringAsJson: true},\n );\n void warmupConnections(lc, changeDB, 'change').catch(() => {});\n\n const shard = getShardConfig(config);\n\n // Ensure the change DB schema is initialized/up-to-date.\n await initChangeStreamerSchema(lc, changeDB, shard);\n\n // When restoring from litestream, acquire a lock to prevent change-log\n // purges. This ensures that (this) change-streamer will be able to resume\n // from the backup.\n let purgeLock =\n pgChangeLogEnabled && litestream.backupURL && litestream.executable\n ? await new PurgeLocker(lc, shard, changeDB).acquire()\n : null;\n const restoreOptions = {litestream, constraints: purgeLock ?? undefined};\n\n let changeStreamer: ChangeStreamerService | undefined;\n let backupURL: string | undefined;\n\n const context = getServerContext(config);\n const sqliteChangeLogEnabled = sqliteChangeLogMode !== 'off';\n // The modes are cumulative, so `serve` implies `compare`.\n const sqliteChangeLogComparing =\n sqliteChangeLogMode === 'compare' || sqliteChangeLogMode === 'serve';\n if (sqliteChangeLogEnabled) {\n const changeLogFile = changeLogFileName(replica.file);\n registerSQLiteCorruptionDiagnosticTarget(\n {\n debugName: 'change-streamer change-log',\n dbPath: changeLogFile,\n },\n sqliteCorruptionChecks,\n );\n // The log's bytes are accounted for in the process that writes them. This\n // is local disk only — the log is excluded from the litestream backup — and\n // it is a whole-database total, because a table that left the replica left\n // through its wal: the replica's footprint is what shrank and this is where\n // it went.\n getOrCreateGauge('replica', 'sqlite_change_log.file_bytes', {\n description:\n `The SQLite change log's total on-disk footprint: its main db file ` +\n `plus its wal sidecars.`,\n unit: 'By',\n }).addCallback(async o =>\n o.observe(await sqliteFileBytes(lc, changeLogFile)),\n );\n }\n\n let waitForFirstBackupBeforeServing = false;\n for (const first of [true, false]) {\n try {\n // Note: This performs initial sync of the replica if necessary.\n const {\n pgReplicationEpoch: epoch,\n pgReplicationSlotPerReplica: slotPerReplica,\n pgResumeOrphanedSlotGracePeriodMs: inactiveReplicaGracePeriodMs,\n } = upstream;\n const {\n changeSource,\n subscriptionState,\n destinationBackupURL,\n replicaID,\n waitForBackupBeforeServing,\n } =\n upstream.type === 'pg'\n ? await initializePostgresChangeSource(\n lc,\n upstream.db,\n shard,\n replica.file,\n {\n ...initialSync,\n replicationSlotFailover: upstream.pgReplicationSlotFailover,\n installPartialIndexTriggers: upstream.pgPartialIndexTriggers,\n },\n context,\n replicationLag.reportIntervalMs,\n restoreOptions,\n {\n epoch,\n slotPerReplica,\n inactiveReplicaGracePeriodMs,\n backupV5: litestream.backupUsingV5,\n },\n purgeLock,\n upstream.pgStreamInboundTimeoutMs,\n )\n : await initializeCustomChangeSource(\n lc,\n upstream.db,\n shard,\n replica.file,\n context,\n restoreOptions,\n );\n\n const replicationStatusPublisher =\n ReplicationStatusPublisher.forReplicaFile(replica.file);\n\n changeStreamer = await initializeStreamer(\n lc,\n shard,\n taskID,\n address,\n protocol,\n changeDB,\n changeSource,\n replicationStatusPublisher,\n subscriptionState,\n destinationBackupURL\n ? {\n backupURL: destinationBackupURL,\n litestreamVersion: litestream.backupUsingV5 ? 'v5' : 'legacy',\n }\n : null,\n purgeLock,\n autoReset ?? false,\n {\n pgChangeLogEnabled,\n backPressureLimitHeapProportion,\n flowControlConsensusTimeoutProportion,\n flowControlSlowSubscriberGracePeriodMs:\n flowControlSlowSubscriberGracePeriodSeconds > 0\n ? flowControlSlowSubscriberGracePeriodSeconds * 1000\n : undefined,\n statementTimeoutMs: change.statementTimeoutMs,\n changeLogBatchSize: change.logBatchSize,\n sqliteCatchup: {\n changeLogFile: changeLogFileName(replica.file),\n readBatchRows: sqliteChangeLogReadBatchRows,\n barrierTimeoutMs: sqliteChangeLogBarrierTimeoutMs,\n },\n // The presence of these options is the writer's gate. This process\n // performs the restore and the initial sync, so the replica's\n // identity is in hand at the moment the log is opened.\n sqliteChangeLogWriter: sqliteChangeLogEnabled\n ? {\n replicaFile: replica.file,\n identity: {\n // RMv2 supplies the epoch; until then the generation and the\n // replica ID are the whole of the identity.\n epoch: null,\n generation: subscriptionState.replicaVersion,\n replicaID,\n },\n }\n : undefined,\n // The purge scheduler shares the writer's gate -- mode, not the\n // read path -- because `write` mode, with no read selector at all,\n // is the configuration it ships in.\n sqliteChangeLogPurge: sqliteChangeLogEnabled\n ? {\n retentionMs: sqliteChangeLogRetentionMs,\n batchRows: sqliteChangeLogPurgeBatchRows,\n }\n : undefined,\n // Compare mode runs both advisory checks. Postgres remains authoritative.\n sqliteChangeLogCompare:\n pgChangeLogEnabled && sqliteChangeLogComparing\n ? {\n replicaFile: replica.file,\n comparePercent: sqliteChangeLogComparePercent,\n retentionMs: sqliteChangeLogRetentionMs,\n readBatchRows: sqliteChangeLogReadBatchRows,\n }\n : undefined,\n // Slice 11 lands dark by default: serve mode constructs the stable\n // router, while readPercent=0 keeps every catchup on PG and emits\n // eligibility metrics before any canary traffic is enabled.\n sqliteChangeLogServe:\n sqliteChangeLogMode === 'serve'\n ? {\n readPercent: sqliteChangeLogReadPercent,\n coldReadPercent: sqliteChangeLogColdReadPercent,\n retentionMs: sqliteChangeLogRetentionMs,\n }\n : undefined,\n },\n setTimeout,\n );\n backupURL = destinationBackupURL;\n waitForFirstBackupBeforeServing = waitForBackupBeforeServing;\n break;\n } catch (e) {\n if (first && e instanceof AutoResetSignal) {\n lc.warn?.(`resetting replica ${replica.file}`, e);\n // TODO: Make deleteLiteDB work with litestream. It will probably have to be\n // a semantic wipe instead of a file delete.\n deleteLiteDB(replica.file);\n // The change log carries the identity of the replica it was written\n // beside, and the retry performs a fresh initial sync with a new\n // replicaVersion. Reconciliation would catch that on its own (as\n // 'identity-mismatch') but only after the writer opened a file that is\n // known here to be garbage.\n deleteChangeLogDB(replica.file);\n // Release the purge lock before retrying. This is safe because the\n // purge lock exists to preserve change-log entries so the new\n // change-streamer can resume from the backup replica's watermark.\n // An AutoResetSignal means we cant resume from the backup replica\n // (e.g. its replication slot is gone), so the change-log entries the lock\n // was protecting are no longer needed. The retry performs a fresh\n // initial sync with a new replication slot, independent of the old\n // change-log. Releasing is also necessary to avoid a\n // self-deadlock when CHANGE_DB == UPSTREAM_DB:\n // CREATE_REPLICATION_SLOT waits for all older transactions to\n // finish, including this lock's open transaction.\n await purgeLock?.release();\n purgeLock = null;\n continue; // execute again with a fresh initial-sync\n }\n if (e instanceof DatabaseInitError) {\n throw new Error(\n `Cannot open ZERO_REPLICA_FILE at \"${replica.file}\". Please check that the path is valid.`,\n {cause: e},\n );\n }\n throw e;\n }\n }\n // impossible: upstream must have advanced in order for replication to be stuck.\n assert(changeStreamer, `resetting replica did not advance replicaVersion`);\n\n const processes = new ProcessManager(lc, parent);\n const profileSubWorkers: Worker[] = [];\n if (backupURL) {\n lc.info?.('setting up backup to', backupURL);\n litestream.backupURL = backupURL;\n const {promise: backupStarted, resolve} = resolver();\n\n // Start a backup replicator and corresponding litestream backup process.\n const backupReplicator = processes\n .addWorker(\n childWorker(REPLICATOR_URL, env, 'backup' satisfies ReplicaFileMode),\n 'supporting',\n 'backup-replicator',\n )\n // Wait for the replicator's first message (i.e. \"ready\") before starting\n // litestream backup in order to avoid contending on the sqlite lock\n // when the replicator first prepares the db file.\n .once('message', () => {\n processes.addSubprocess(\n startReplicaBackupProcess(lc, litestream, replica.file),\n 'supporting',\n 'litestream',\n );\n resolve();\n });\n profileSubWorkers.push(backupReplicator);\n // Relay profileResponse messages from backup-replicator up to parent\n backupReplicator.onMessageType<ProfileResponseMessage>(\n 'profileResponse',\n res => parent.send(['profileResponse', res]),\n );\n await backupStarted;\n }\n\n const backupMonitor = createBackupCleanupMonitor({\n lc,\n config,\n replicaFile: replica.file,\n changeStreamer,\n });\n\n let readinessGate = promiseVoid;\n if (waitForFirstBackupBeforeServing) {\n const start = performance.now();\n lc.info?.(`awaiting initial backup ...`);\n\n readinessGate = backupMonitor.firstBackupReceived().then(() => {\n const elapsed = performance.now() - start;\n lc.info?.(`initial backup confirmed after ${elapsed.toFixed(2)}ms`);\n });\n }\n\n // Create the broadcast facade once: each broadcastWorker() adds permanent\n // 'message' forwarders to every sub worker, so creating one per /profz\n // request would leak a forwarder per request.\n const profileWorker =\n profileSubWorkers.length > 0\n ? broadcastWorker(profileSubWorkers)\n : undefined;\n const getProfileWorker = profileWorker\n ? () => Promise.resolve(profileWorker)\n : undefined;\n\n const changeStreamerWebServer = new ChangeStreamerHttpServer(\n lc,\n {\n port,\n keepaliveTimeoutMs,\n // The startup delay is only relevant for RMv1, and is disabled for RMv2.\n startupDelayMs: upstream.pgReplicationSlotPerReplica ? 0 : startupDelayMs,\n readinessGate,\n config,\n getProfileWorker,\n },\n parent,\n changeStreamer,\n );\n\n void readinessGate.then(() => parent.send(['ready', {ready: true}]));\n\n // Note: The changeStreamer itself is not started here; it is started by the\n // changeStreamerWebServer after a delay to ensure that routing\n // routing elements have registered the server before the\n // change-streamer takes over the replication slot.\n // TODO: Remove this delay and start the changeStreamer normally here once\n // transitioned to RMv2, since changeStreamer startup will no longer\n // disrupt an existing task.\n try {\n await runUntilKilled(lc, parent, changeStreamerWebServer, backupMonitor);\n } catch (err) {\n processes.logErrorAndExit(err, 'change-streamer');\n } finally {\n await processes.shutdown();\n }\n}\n\n// fork()\nif (!singleProcessMode()) {\n void exitAfter(\n () => lc,\n () =>\n runWorker(\n must(parentWorker),\n process.env,\n ...process.argv.slice(2),\n ).catch(async e => {\n await publishCriticalEvent(\n lc,\n replicationStatusError(lc, 'Initializing', e),\n );\n throw e;\n }),\n );\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAsDA,IAAI,KAAK,IAAI,WAAW,QAAQ,CAAC,GAAG,cAAc;AAElD,eAA8B,UAC5B,QACA,KACA,GAAG,MACY;CACf,sBAAsB,QAAQ,iBAAiB;CAC/C,MAAM,SAAS,wBAAwB;EAAC;EAAK;CAAI,CAAC;CAClD,MAAM,EACJ,QACA,gBAAgB,EACd,MACA,SACA,UACA,gBACA,iCACA,uCACA,6CACA,oBACA,qBACA,4BACA,gCACA,+BACA,4BACA,8BACA,+BACA,mCAEF,WACA,gBACA,YACA,UACA,QACA,SACA,aACA,oBACA,2BACE;CAEJ,cACE,iBAAiB,QAAQ,mBAAmB,GAAG,KAAK,GACpD,mBACA,CACF;CACA,KAAK,iBAAiB,QAAQ,iBAAiB;CAC/C,yCACE;EACE,WAAW;EACX,QAAQ,QAAQ;CAClB,GACA,sBACF;CACA,cAAc,IAAI,MAAM;CAKxB,MAAM,WAAW,MAAM,gBACrB,IACA,OAAO,IACP,wBACA,EAAC,KAAK,EAAC,GACP,EAAC,kBAAkB,KAAI,CACzB;CACA,kBAAuB,IAAI,UAAU,QAAQ,EAAE,YAAY,CAAC,CAAC;CAE7D,MAAM,QAAQ,eAAe,MAAM;CAGnC,MAAM,yBAAyB,IAAI,UAAU,KAAK;CAKlD,IAAI,YACF,sBAAsB,WAAW,aAAa,WAAW,aACrD,MAAM,IAAI,YAAY,IAAI,OAAO,QAAQ,EAAE,QAAQ,IACnD;CACN,MAAM,iBAAiB;EAAC;EAAY,aAAa,aAAa,KAAA;CAAS;CAEvE,IAAI;CACJ,IAAI;CAEJ,MAAM,UAAU,iBAAiB,MAAM;CACvC,MAAM,yBAAyB,wBAAwB;CAEvD,MAAM,2BACJ,wBAAwB,aAAa,wBAAwB;CAC/D,IAAI,wBAAwB;EAC1B,MAAM,gBAAgB,kBAAkB,QAAQ,IAAI;EACpD,yCACE;GACE,WAAW;GACX,QAAQ;EACV,GACA,sBACF;EAMA,iBAAiB,WAAW,gCAAgC;GAC1D,aACE;GAEF,MAAM;EACR,CAAC,EAAE,YAAY,OAAM,MACnB,EAAE,QAAQ,MAAM,gBAAgB,IAAI,aAAa,CAAC,CACpD;CACF;CAEA,IAAI,kCAAkC;CACtC,KAAK,MAAM,SAAS,CAAC,MAAM,KAAK,GAC9B,IAAI;EAEF,MAAM,EACJ,oBAAoB,OACpB,6BAA6B,gBAC7B,mCAAmC,iCACjC;EACJ,MAAM,EACJ,cACA,mBACA,sBACA,WACA,+BAEA,SAAS,SAAS,OACd,MAAM,+BACJ,IACA,SAAS,IACT,OACA,QAAQ,MACR;GACE,GAAG;GACH,yBAAyB,SAAS;GAClC,6BAA6B,SAAS;EACxC,GACA,SACA,eAAe,kBACf,gBACA;GACE;GACA;GACA;GACA,UAAU,WAAW;EACvB,GACA,WACA,SAAS,wBACX,IACA,MAAM,6BACJ,IACA,SAAS,IACT,OACA,QAAQ,MACR,SACA,cACF;EAEN,MAAM,6BACJ,2BAA2B,eAAe,QAAQ,IAAI;EAExD,iBAAiB,MAAM,mBACrB,IACA,OACA,QACA,SACA,UACA,UACA,cACA,4BACA,mBACA,uBACI;GACE,WAAW;GACX,mBAAmB,WAAW,gBAAgB,OAAO;EACvD,IACA,MACJ,WACA,aAAa,OACb;GACE;GACA;GACA;GACA,wCACE,8CAA8C,IAC1C,8CAA8C,MAC9C,KAAA;GACN,oBAAoB,OAAO;GAC3B,oBAAoB,OAAO;GAC3B,eAAe;IACb,eAAe,kBAAkB,QAAQ,IAAI;IAC7C,eAAe;IACf,kBAAkB;GACpB;GAIA,uBAAuB,yBACnB;IACE,aAAa,QAAQ;IACrB,UAAU;KAGR,OAAO;KACP,YAAY,kBAAkB;KAC9B;IACF;GACF,IACA,KAAA;GAIJ,sBAAsB,yBAClB;IACE,aAAa;IACb,WAAW;GACb,IACA,KAAA;GAEJ,wBACE,sBAAsB,2BAClB;IACE,aAAa,QAAQ;IACrB,gBAAgB;IAChB,aAAa;IACb,eAAe;GACjB,IACA,KAAA;GAIN,sBACE,wBAAwB,UACpB;IACE,aAAa;IACb,iBAAiB;IACjB,aAAa;GACf,IACA,KAAA;EACR,GACA,UACF;EACA,YAAY;EACZ,kCAAkC;EAClC;CACF,SAAS,GAAG;EACV,IAAI,SAAS,aAAa,iBAAiB;GACzC,GAAG,OAAO,qBAAqB,QAAQ,QAAQ,CAAC;GAGhD,aAAa,QAAQ,IAAI;GAMzB,kBAAkB,QAAQ,IAAI;GAY9B,MAAM,WAAW,QAAQ;GACzB,YAAY;GACZ;EACF;EACA,IAAI,aAAa,mBACf,MAAM,IAAI,MACR,qCAAqC,QAAQ,KAAK,0CAClD,EAAC,OAAO,EAAC,CACX;EAEF,MAAM;CACR;CAGF,OAAO,gBAAgB,kDAAkD;CAEzE,MAAM,YAAY,IAAI,eAAe,IAAI,MAAM;CAC/C,MAAM,oBAA8B,CAAC;CACrC,IAAI,WAAW;EACb,GAAG,OAAO,wBAAwB,SAAS;EAC3C,WAAW,YAAY;EACvB,MAAM,EAAC,SAAS,eAAe,YAAW,SAAS;EAGnD,MAAM,mBAAmB,UACtB,UACC,YAAY,gBAAgB,KAAK,QAAkC,GACnE,cACA,mBACF,EAIC,KAAK,iBAAiB;GACrB,UAAU,cACR,0BAA0B,IAAI,YAAY,QAAQ,IAAI,GACtD,cACA,YACF;GACA,QAAQ;EACV,CAAC;EACH,kBAAkB,KAAK,gBAAgB;EAEvC,iBAAiB,cACf,oBACA,QAAO,OAAO,KAAK,CAAC,mBAAmB,GAAG,CAAC,CAC7C;EACA,MAAM;CACR;CAEA,MAAM,gBAAgB,2BAA2B;EAC/C;EACA;EACA,aAAa,QAAQ;EACrB;CACF,CAAC;CAED,IAAI,gBAAgB;CACpB,IAAI,iCAAiC;EACnC,MAAM,QAAQ,YAAY,IAAI;EAC9B,GAAG,OAAO,6BAA6B;EAEvC,gBAAgB,cAAc,oBAAoB,EAAE,WAAW;GAC7D,MAAM,UAAU,YAAY,IAAI,IAAI;GACpC,GAAG,OAAO,kCAAkC,QAAQ,QAAQ,CAAC,EAAE,GAAG;EACpE,CAAC;CACH;CAKA,MAAM,gBACJ,kBAAkB,SAAS,IACvB,gBAAgB,iBAAiB,IACjC,KAAA;CACN,MAAM,mBAAmB,sBACf,QAAQ,QAAQ,aAAa,IACnC,KAAA;CAEJ,MAAM,0BAA0B,IAAI,yBAClC,IACA;EACE;EACA;EAEA,gBAAgB,SAAS,8BAA8B,IAAI;EAC3D;EACA;EACA;CACF,GACA,QACA,cACF;CAEA,cAAmB,WAAW,OAAO,KAAK,CAAC,SAAS,EAAC,OAAO,KAAI,CAAC,CAAC,CAAC;CASnE,IAAI;EACF,MAAM,eAAe,IAAI,QAAQ,yBAAyB,aAAa;CACzE,SAAS,KAAK;EACZ,UAAU,gBAAgB,KAAK,iBAAiB;CAClD,UAAU;EACR,MAAM,UAAU,SAAS;CAC3B;AACF;AAGA,IAAI,CAAC,kBAAkB,GACrB,gBACQ,UAEJ,UACE,KAAK,YAAY,GACjB,QAAQ,KACR,GAAG,QAAQ,KAAK,MAAM,CAAC,CACzB,EAAE,MAAM,OAAM,MAAK;CACjB,MAAM,qBACJ,IACA,uBAAuB,IAAI,gBAAgB,CAAC,CAC9C;CACA,MAAM;AACR,CAAC,CACL"}
|
|
1
|
+
{"version":3,"file":"change-streamer.js","names":[],"sources":["../../../../../zero-cache/src/server/change-streamer.ts"],"sourcesContent":["import {consoleLogSink, LogContext} from '@rocicorp/logger';\nimport {resolver} from '@rocicorp/resolver';\nimport {assert} from '../../../shared/src/asserts.ts';\nimport {must} from '../../../shared/src/must.ts';\nimport {promiseVoid} from '../../../shared/src/resolved-promises.ts';\nimport {DatabaseInitError} from '../../../zqlite/src/db.ts';\nimport {getServerContext} from '../config/server-context.ts';\nimport {getNormalizedZeroConfig} from '../config/zero-config.ts';\nimport {deleteLiteDB} from '../db/delete-lite-db.ts';\nimport {registerSQLiteCorruptionDiagnosticTarget} from '../db/sqlite-corruption.ts';\nimport {warmupConnections} from '../db/warmup.ts';\nimport {initEventSink, publishCriticalEvent} from '../observability/events.ts';\nimport {getOrCreateGauge} from '../observability/metrics.ts';\nimport {initializeCustomChangeSource} from '../services/change-source/custom/change-source.ts';\nimport {initializePostgresChangeSource} from '../services/change-source/pg/change-source-init.ts';\nimport {createBackupCleanupMonitor} from '../services/change-streamer/backup-cleanup-monitor-factory.ts';\nimport {ChangeStreamerHttpServer} from '../services/change-streamer/change-streamer-http.ts';\nimport {initializeStreamer} from '../services/change-streamer/change-streamer-service.ts';\nimport type {ChangeStreamerService} from '../services/change-streamer/change-streamer.ts';\nimport {initChangeStreamerSchema} from '../services/change-streamer/schema/init.ts';\nimport {AutoResetSignal} from '../services/change-streamer/schema/tables.ts';\nimport {PurgeLocker} from '../services/change-streamer/storer.ts';\nimport {\n exitAfter,\n ProcessManager,\n runUntilKilled,\n} from '../services/life-cycle.ts';\nimport {startReplicaBackupProcess} from '../services/litestream/commands.ts';\nimport {\n changeLogFileName,\n deleteChangeLogDB,\n} from '../services/replicator/change-log-db.ts';\nimport {\n replicationStatusError,\n ReplicationStatusPublisher,\n} from '../services/replicator/replication-status.ts';\nimport {sqliteFileBytes} from '../services/replicator/sqlite-change-log-observability.ts';\nimport {connectPgClient} from '../types/pg.ts';\nimport {\n broadcastWorker,\n childWorker,\n parentWorker,\n singleProcessMode,\n type ProfileResponseMessage,\n type Worker,\n} from '../types/processes.ts';\nimport {installProfileHandler} from '../types/profiler.ts';\nimport {getShardConfig} from '../types/shards.ts';\nimport type {ReplicaFileMode} from '../workers/replicator.ts';\nimport {createLogContext} from './logging.ts';\nimport {startOtelAuto} from './otel-start.ts';\nimport {REPLICATOR_URL} from './worker-urls.ts';\n\n// Default LogContext, overridden in runWorker\nlet lc = new LogContext('info', {}, consoleLogSink);\n\nexport default async function runWorker(\n parent: Worker,\n env: NodeJS.ProcessEnv,\n ...argv: string[]\n): Promise<void> {\n installProfileHandler(parent, 'change-streamer');\n const config = getNormalizedZeroConfig({env, argv});\n const {\n taskID,\n changeStreamer: {\n port,\n address,\n protocol,\n startupDelayMs,\n backPressureLimitHeapProportion,\n flowControlConsensusTimeoutProportion,\n flowControlSlowSubscriberGracePeriodSeconds,\n pgChangeLogEnabled,\n sqliteChangeLogMode,\n sqliteChangeLogReadPercent,\n sqliteChangeLogColdReadPercent,\n sqliteChangeLogComparePercent,\n sqliteChangeLogRetentionMs,\n sqliteChangeLogReadBatchRows,\n sqliteChangeLogPurgeBatchRows,\n sqliteChangeLogBarrierTimeoutMs,\n },\n autoReset,\n replicationLag,\n litestream,\n upstream,\n change,\n replica,\n initialSync,\n keepaliveTimeoutMs,\n sqliteCorruptionChecks,\n } = config;\n\n startOtelAuto(\n createLogContext(config, 'change-streamer', 0, false),\n 'change-streamer',\n 0,\n );\n lc = createLogContext(config, 'change-streamer');\n registerSQLiteCorruptionDiagnosticTarget(\n {\n debugName: 'change-streamer replica',\n dbPath: replica.file,\n },\n sqliteCorruptionChecks,\n );\n initEventSink(lc, config);\n\n // Startup-time client used for for change-streamer initialization\n // and handoff / takeover. Steady-state clients are managed by the\n // change-streamer (Storer) implementation.\n const changeDB = await connectPgClient(\n lc,\n change.db,\n 'change-streamer-init',\n {max: 5},\n {sendStringAsJson: true},\n );\n void warmupConnections(lc, changeDB, 'change').catch(() => {});\n\n const shard = getShardConfig(config);\n\n // Ensure the change DB schema is initialized/up-to-date.\n await initChangeStreamerSchema(lc, changeDB, shard);\n\n // When restoring from litestream, acquire a lock to prevent change-log\n // purges. This ensures that (this) change-streamer will be able to resume\n // from the backup.\n let purgeLock =\n pgChangeLogEnabled && litestream.backupURL && litestream.executable\n ? await new PurgeLocker(lc, shard, changeDB).acquire()\n : null;\n const restoreOptions = {litestream, constraints: purgeLock ?? undefined};\n\n let changeStreamer: ChangeStreamerService | undefined;\n let backupURL: string | undefined;\n\n const context = getServerContext(config);\n const sqliteChangeLogEnabled = sqliteChangeLogMode !== 'off';\n // The modes are cumulative, so `serve` implies `compare`.\n const sqliteChangeLogComparing =\n sqliteChangeLogMode === 'compare' || sqliteChangeLogMode === 'serve';\n if (sqliteChangeLogEnabled) {\n const changeLogFile = changeLogFileName(replica.file);\n registerSQLiteCorruptionDiagnosticTarget(\n {\n debugName: 'change-streamer change-log',\n dbPath: changeLogFile,\n },\n sqliteCorruptionChecks,\n );\n // The log's bytes are accounted for in the process that writes them. This\n // is local disk only — the log is excluded from the litestream backup — and\n // it is a whole-database total, because a table that left the replica left\n // through its wal: the replica's footprint is what shrank and this is where\n // it went.\n getOrCreateGauge('replica', 'sqlite_change_log.file_bytes', {\n description:\n `The SQLite change log's total on-disk footprint: its main db file ` +\n `plus its wal sidecars.`,\n unit: 'By',\n }).addCallback(async o =>\n o.observe(await sqliteFileBytes(lc, changeLogFile)),\n );\n }\n\n let waitForFirstBackupBeforeServing = false;\n for (const first of [true, false]) {\n try {\n // Note: This performs initial sync of the replica if necessary.\n const {\n pgReplicationEpoch: epoch,\n pgReplicationSlotPerReplica: slotPerReplica,\n pgResumeOrphanedSlotGracePeriodMs: inactiveReplicaGracePeriodMs,\n } = upstream;\n const {\n changeSource,\n subscriptionState,\n destinationBackupURL,\n replicaID,\n waitForBackupBeforeServing,\n } =\n upstream.type === 'pg'\n ? await initializePostgresChangeSource(\n lc,\n upstream.db,\n shard,\n replica.file,\n {\n ...initialSync,\n replicationSlotFailover: upstream.pgReplicationSlotFailover,\n installPartialIndexTriggers: upstream.pgPartialIndexTriggers,\n },\n context,\n replicationLag.reportIntervalMs,\n restoreOptions,\n {\n epoch,\n slotPerReplica,\n inactiveReplicaGracePeriodMs,\n backupV5: litestream.backupUsingV5,\n },\n purgeLock,\n upstream.pgStreamInboundTimeoutMs,\n )\n : await initializeCustomChangeSource(\n lc,\n upstream.db,\n shard,\n replica.file,\n context,\n restoreOptions,\n );\n\n const replicationStatusPublisher =\n ReplicationStatusPublisher.forReplicaFile(replica.file);\n\n changeStreamer = await initializeStreamer(\n lc,\n shard,\n taskID,\n address,\n protocol,\n changeDB,\n changeSource,\n replicationStatusPublisher,\n subscriptionState,\n destinationBackupURL\n ? {\n backupURL: destinationBackupURL,\n litestreamVersion: litestream.backupUsingV5 ? 'v5' : 'legacy',\n replicaFile: replica.file,\n }\n : null,\n purgeLock,\n autoReset ?? false,\n {\n pgChangeLogEnabled,\n backPressureLimitHeapProportion,\n flowControlConsensusTimeoutProportion,\n flowControlSlowSubscriberGracePeriodMs:\n flowControlSlowSubscriberGracePeriodSeconds > 0\n ? flowControlSlowSubscriberGracePeriodSeconds * 1000\n : undefined,\n statementTimeoutMs: change.statementTimeoutMs,\n changeLogBatchSize: change.logBatchSize,\n sqliteCatchup: {\n changeLogFile: changeLogFileName(replica.file),\n readBatchRows: sqliteChangeLogReadBatchRows,\n barrierTimeoutMs: sqliteChangeLogBarrierTimeoutMs,\n },\n // The presence of these options is the writer's gate. This process\n // performs the restore and the initial sync, so the replica's\n // identity is in hand at the moment the log is opened.\n sqliteChangeLogWriter: sqliteChangeLogEnabled\n ? {\n replicaFile: replica.file,\n identity: {\n // RMv2 supplies the epoch; until then the generation and the\n // replica ID are the whole of the identity.\n epoch: null,\n generation: subscriptionState.replicaVersion,\n replicaID,\n },\n }\n : undefined,\n // The purge scheduler shares the writer's gate -- mode, not the\n // read path -- because `write` mode, with no read selector at all,\n // is the configuration it ships in.\n sqliteChangeLogPurge: sqliteChangeLogEnabled\n ? {\n retentionMs: sqliteChangeLogRetentionMs,\n batchRows: sqliteChangeLogPurgeBatchRows,\n }\n : undefined,\n // Compare mode runs both advisory checks. Postgres remains authoritative.\n sqliteChangeLogCompare:\n pgChangeLogEnabled && sqliteChangeLogComparing\n ? {\n replicaFile: replica.file,\n comparePercent: sqliteChangeLogComparePercent,\n retentionMs: sqliteChangeLogRetentionMs,\n readBatchRows: sqliteChangeLogReadBatchRows,\n }\n : undefined,\n // Slice 11 lands dark by default: serve mode constructs the stable\n // router, while readPercent=0 keeps every catchup on PG and emits\n // eligibility metrics before any canary traffic is enabled.\n sqliteChangeLogServe:\n sqliteChangeLogMode === 'serve'\n ? {\n readPercent: sqliteChangeLogReadPercent,\n coldReadPercent: sqliteChangeLogColdReadPercent,\n retentionMs: sqliteChangeLogRetentionMs,\n }\n : undefined,\n },\n setTimeout,\n );\n backupURL = destinationBackupURL;\n waitForFirstBackupBeforeServing = waitForBackupBeforeServing;\n break;\n } catch (e) {\n if (first && e instanceof AutoResetSignal) {\n lc.warn?.(`resetting replica ${replica.file}`, e);\n // TODO: Make deleteLiteDB work with litestream. It will probably have to be\n // a semantic wipe instead of a file delete.\n deleteLiteDB(replica.file);\n // The change log carries the identity of the replica it was written\n // beside, and the retry performs a fresh initial sync with a new\n // replicaVersion. Reconciliation would catch that on its own (as\n // 'identity-mismatch') but only after the writer opened a file that is\n // known here to be garbage.\n deleteChangeLogDB(replica.file);\n // Release the purge lock before retrying. This is safe because the\n // purge lock exists to preserve change-log entries so the new\n // change-streamer can resume from the backup replica's watermark.\n // An AutoResetSignal means we cant resume from the backup replica\n // (e.g. its replication slot is gone), so the change-log entries the lock\n // was protecting are no longer needed. The retry performs a fresh\n // initial sync with a new replication slot, independent of the old\n // change-log. Releasing is also necessary to avoid a\n // self-deadlock when CHANGE_DB == UPSTREAM_DB:\n // CREATE_REPLICATION_SLOT waits for all older transactions to\n // finish, including this lock's open transaction.\n await purgeLock?.release();\n purgeLock = null;\n continue; // execute again with a fresh initial-sync\n }\n if (e instanceof DatabaseInitError) {\n throw new Error(\n `Cannot open ZERO_REPLICA_FILE at \"${replica.file}\". Please check that the path is valid.`,\n {cause: e},\n );\n }\n throw e;\n }\n }\n // impossible: upstream must have advanced in order for replication to be stuck.\n assert(changeStreamer, `resetting replica did not advance replicaVersion`);\n\n const processes = new ProcessManager(lc, parent);\n const profileSubWorkers: Worker[] = [];\n if (backupURL) {\n lc.info?.('setting up backup to', backupURL);\n litestream.backupURL = backupURL;\n const {promise: backupStarted, resolve} = resolver();\n\n // Start a backup replicator and corresponding litestream backup process.\n const backupReplicator = processes\n .addWorker(\n childWorker(REPLICATOR_URL, env, 'backup' satisfies ReplicaFileMode),\n 'supporting',\n 'backup-replicator',\n )\n // Wait for the replicator's first message (i.e. \"ready\") before starting\n // litestream backup in order to avoid contending on the sqlite lock\n // when the replicator first prepares the db file.\n .once('message', () => {\n processes.addSubprocess(\n startReplicaBackupProcess(lc, litestream, replica.file),\n 'supporting',\n 'litestream',\n );\n resolve();\n });\n profileSubWorkers.push(backupReplicator);\n // Relay profileResponse messages from backup-replicator up to parent\n backupReplicator.onMessageType<ProfileResponseMessage>(\n 'profileResponse',\n res => parent.send(['profileResponse', res]),\n );\n await backupStarted;\n }\n\n const backupMonitor = createBackupCleanupMonitor({\n lc,\n config,\n replicaFile: replica.file,\n changeStreamer,\n });\n\n let readinessGate = promiseVoid;\n if (waitForFirstBackupBeforeServing) {\n const start = performance.now();\n lc.info?.(`awaiting initial backup ...`);\n\n readinessGate = backupMonitor.firstBackupReceived().then(() => {\n const elapsed = performance.now() - start;\n lc.info?.(`initial backup confirmed after ${elapsed.toFixed(2)}ms`);\n });\n }\n\n // Create the broadcast facade once: each broadcastWorker() adds permanent\n // 'message' forwarders to every sub worker, so creating one per /profz\n // request would leak a forwarder per request.\n const profileWorker =\n profileSubWorkers.length > 0\n ? broadcastWorker(profileSubWorkers)\n : undefined;\n const getProfileWorker = profileWorker\n ? () => Promise.resolve(profileWorker)\n : undefined;\n\n const changeStreamerWebServer = new ChangeStreamerHttpServer(\n lc,\n {\n port,\n keepaliveTimeoutMs,\n // The startup delay is only relevant for RMv1, and is disabled for RMv2.\n startupDelayMs: upstream.pgReplicationSlotPerReplica ? 0 : startupDelayMs,\n readinessGate,\n config,\n getProfileWorker,\n },\n parent,\n changeStreamer,\n );\n\n void readinessGate.then(() => parent.send(['ready', {ready: true}]));\n\n // Note: The changeStreamer itself is not started here; it is started by the\n // changeStreamerWebServer after a delay to ensure that routing\n // routing elements have registered the server before the\n // change-streamer takes over the replication slot.\n // TODO: Remove this delay and start the changeStreamer normally here once\n // transitioned to RMv2, since changeStreamer startup will no longer\n // disrupt an existing task.\n try {\n await runUntilKilled(lc, parent, changeStreamerWebServer, backupMonitor);\n } catch (err) {\n processes.logErrorAndExit(err, 'change-streamer');\n } finally {\n await processes.shutdown();\n }\n}\n\n// fork()\nif (!singleProcessMode()) {\n void exitAfter(\n () => lc,\n () =>\n runWorker(\n must(parentWorker),\n process.env,\n ...process.argv.slice(2),\n ).catch(async e => {\n await publishCriticalEvent(\n lc,\n replicationStatusError(lc, 'Initializing', e),\n );\n throw e;\n }),\n );\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAsDA,IAAI,KAAK,IAAI,WAAW,QAAQ,CAAC,GAAG,cAAc;AAElD,eAA8B,UAC5B,QACA,KACA,GAAG,MACY;CACf,sBAAsB,QAAQ,iBAAiB;CAC/C,MAAM,SAAS,wBAAwB;EAAC;EAAK;CAAI,CAAC;CAClD,MAAM,EACJ,QACA,gBAAgB,EACd,MACA,SACA,UACA,gBACA,iCACA,uCACA,6CACA,oBACA,qBACA,4BACA,gCACA,+BACA,4BACA,8BACA,+BACA,mCAEF,WACA,gBACA,YACA,UACA,QACA,SACA,aACA,oBACA,2BACE;CAEJ,cACE,iBAAiB,QAAQ,mBAAmB,GAAG,KAAK,GACpD,mBACA,CACF;CACA,KAAK,iBAAiB,QAAQ,iBAAiB;CAC/C,yCACE;EACE,WAAW;EACX,QAAQ,QAAQ;CAClB,GACA,sBACF;CACA,cAAc,IAAI,MAAM;CAKxB,MAAM,WAAW,MAAM,gBACrB,IACA,OAAO,IACP,wBACA,EAAC,KAAK,EAAC,GACP,EAAC,kBAAkB,KAAI,CACzB;CACA,kBAAuB,IAAI,UAAU,QAAQ,EAAE,YAAY,CAAC,CAAC;CAE7D,MAAM,QAAQ,eAAe,MAAM;CAGnC,MAAM,yBAAyB,IAAI,UAAU,KAAK;CAKlD,IAAI,YACF,sBAAsB,WAAW,aAAa,WAAW,aACrD,MAAM,IAAI,YAAY,IAAI,OAAO,QAAQ,EAAE,QAAQ,IACnD;CACN,MAAM,iBAAiB;EAAC;EAAY,aAAa,aAAa,KAAA;CAAS;CAEvE,IAAI;CACJ,IAAI;CAEJ,MAAM,UAAU,iBAAiB,MAAM;CACvC,MAAM,yBAAyB,wBAAwB;CAEvD,MAAM,2BACJ,wBAAwB,aAAa,wBAAwB;CAC/D,IAAI,wBAAwB;EAC1B,MAAM,gBAAgB,kBAAkB,QAAQ,IAAI;EACpD,yCACE;GACE,WAAW;GACX,QAAQ;EACV,GACA,sBACF;EAMA,iBAAiB,WAAW,gCAAgC;GAC1D,aACE;GAEF,MAAM;EACR,CAAC,EAAE,YAAY,OAAM,MACnB,EAAE,QAAQ,MAAM,gBAAgB,IAAI,aAAa,CAAC,CACpD;CACF;CAEA,IAAI,kCAAkC;CACtC,KAAK,MAAM,SAAS,CAAC,MAAM,KAAK,GAC9B,IAAI;EAEF,MAAM,EACJ,oBAAoB,OACpB,6BAA6B,gBAC7B,mCAAmC,iCACjC;EACJ,MAAM,EACJ,cACA,mBACA,sBACA,WACA,+BAEA,SAAS,SAAS,OACd,MAAM,+BACJ,IACA,SAAS,IACT,OACA,QAAQ,MACR;GACE,GAAG;GACH,yBAAyB,SAAS;GAClC,6BAA6B,SAAS;EACxC,GACA,SACA,eAAe,kBACf,gBACA;GACE;GACA;GACA;GACA,UAAU,WAAW;EACvB,GACA,WACA,SAAS,wBACX,IACA,MAAM,6BACJ,IACA,SAAS,IACT,OACA,QAAQ,MACR,SACA,cACF;EAEN,MAAM,6BACJ,2BAA2B,eAAe,QAAQ,IAAI;EAExD,iBAAiB,MAAM,mBACrB,IACA,OACA,QACA,SACA,UACA,UACA,cACA,4BACA,mBACA,uBACI;GACE,WAAW;GACX,mBAAmB,WAAW,gBAAgB,OAAO;GACrD,aAAa,QAAQ;EACvB,IACA,MACJ,WACA,aAAa,OACb;GACE;GACA;GACA;GACA,wCACE,8CAA8C,IAC1C,8CAA8C,MAC9C,KAAA;GACN,oBAAoB,OAAO;GAC3B,oBAAoB,OAAO;GAC3B,eAAe;IACb,eAAe,kBAAkB,QAAQ,IAAI;IAC7C,eAAe;IACf,kBAAkB;GACpB;GAIA,uBAAuB,yBACnB;IACE,aAAa,QAAQ;IACrB,UAAU;KAGR,OAAO;KACP,YAAY,kBAAkB;KAC9B;IACF;GACF,IACA,KAAA;GAIJ,sBAAsB,yBAClB;IACE,aAAa;IACb,WAAW;GACb,IACA,KAAA;GAEJ,wBACE,sBAAsB,2BAClB;IACE,aAAa,QAAQ;IACrB,gBAAgB;IAChB,aAAa;IACb,eAAe;GACjB,IACA,KAAA;GAIN,sBACE,wBAAwB,UACpB;IACE,aAAa;IACb,iBAAiB;IACjB,aAAa;GACf,IACA,KAAA;EACR,GACA,UACF;EACA,YAAY;EACZ,kCAAkC;EAClC;CACF,SAAS,GAAG;EACV,IAAI,SAAS,aAAa,iBAAiB;GACzC,GAAG,OAAO,qBAAqB,QAAQ,QAAQ,CAAC;GAGhD,aAAa,QAAQ,IAAI;GAMzB,kBAAkB,QAAQ,IAAI;GAY9B,MAAM,WAAW,QAAQ;GACzB,YAAY;GACZ;EACF;EACA,IAAI,aAAa,mBACf,MAAM,IAAI,MACR,qCAAqC,QAAQ,KAAK,0CAClD,EAAC,OAAO,EAAC,CACX;EAEF,MAAM;CACR;CAGF,OAAO,gBAAgB,kDAAkD;CAEzE,MAAM,YAAY,IAAI,eAAe,IAAI,MAAM;CAC/C,MAAM,oBAA8B,CAAC;CACrC,IAAI,WAAW;EACb,GAAG,OAAO,wBAAwB,SAAS;EAC3C,WAAW,YAAY;EACvB,MAAM,EAAC,SAAS,eAAe,YAAW,SAAS;EAGnD,MAAM,mBAAmB,UACtB,UACC,YAAY,gBAAgB,KAAK,QAAkC,GACnE,cACA,mBACF,EAIC,KAAK,iBAAiB;GACrB,UAAU,cACR,0BAA0B,IAAI,YAAY,QAAQ,IAAI,GACtD,cACA,YACF;GACA,QAAQ;EACV,CAAC;EACH,kBAAkB,KAAK,gBAAgB;EAEvC,iBAAiB,cACf,oBACA,QAAO,OAAO,KAAK,CAAC,mBAAmB,GAAG,CAAC,CAC7C;EACA,MAAM;CACR;CAEA,MAAM,gBAAgB,2BAA2B;EAC/C;EACA;EACA,aAAa,QAAQ;EACrB;CACF,CAAC;CAED,IAAI,gBAAgB;CACpB,IAAI,iCAAiC;EACnC,MAAM,QAAQ,YAAY,IAAI;EAC9B,GAAG,OAAO,6BAA6B;EAEvC,gBAAgB,cAAc,oBAAoB,EAAE,WAAW;GAC7D,MAAM,UAAU,YAAY,IAAI,IAAI;GACpC,GAAG,OAAO,kCAAkC,QAAQ,QAAQ,CAAC,EAAE,GAAG;EACpE,CAAC;CACH;CAKA,MAAM,gBACJ,kBAAkB,SAAS,IACvB,gBAAgB,iBAAiB,IACjC,KAAA;CACN,MAAM,mBAAmB,sBACf,QAAQ,QAAQ,aAAa,IACnC,KAAA;CAEJ,MAAM,0BAA0B,IAAI,yBAClC,IACA;EACE;EACA;EAEA,gBAAgB,SAAS,8BAA8B,IAAI;EAC3D;EACA;EACA;CACF,GACA,QACA,cACF;CAEA,cAAmB,WAAW,OAAO,KAAK,CAAC,SAAS,EAAC,OAAO,KAAI,CAAC,CAAC,CAAC;CASnE,IAAI;EACF,MAAM,eAAe,IAAI,QAAQ,yBAAyB,aAAa;CACzE,SAAS,KAAK;EACZ,UAAU,gBAAgB,KAAK,iBAAiB;CAClD,UAAU;EACR,MAAM,UAAU,SAAS;CAC3B;AACF;AAGA,IAAI,CAAC,kBAAkB,GACrB,gBACQ,UAEJ,UACE,KAAK,YAAY,GACjB,QAAQ,KACR,GAAG,QAAQ,KAAK,MAAM,CAAC,CACzB,EAAE,MAAM,OAAM,MAAK;CACjB,MAAM,qBACJ,IACA,uBAAuB,IAAI,gBAAgB,CAAC,CAC9C;CACA,MAAM;AACR,CAAC,CACL"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"inspector-delegate.d.ts","sourceRoot":"","sources":["../../../../../zero-cache/src/server/inspector-delegate.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAC,iBAAiB,EAAC,MAAM,6BAA6B,CAAC;
|
|
1
|
+
{"version":3,"file":"inspector-delegate.d.ts","sourceRoot":"","sources":["../../../../../zero-cache/src/server/inspector-delegate.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAC,iBAAiB,EAAC,MAAM,6BAA6B,CAAC;AAGnE,OAAO,EAAC,OAAO,EAAC,MAAM,gCAAgC,CAAC;AACvD,OAAO,KAAK,EAAC,GAAG,EAAC,MAAM,mCAAmC,CAAC;AAC3D,OAAO,KAAK,EAAC,kBAAkB,IAAI,sBAAsB,EAAC,MAAM,4CAA4C,CAAC;AAE7G,OAAO,EAEL,KAAK,SAAS,EACd,KAAK,eAAe,EACrB,MAAM,4CAA4C,CAAC;AAEpD,OAAO,KAAK,EAAC,sBAAsB,EAAC,MAAM,sCAAsC,CAAC;AACjF,OAAO,KAAK,EAAC,iBAAiB,EAAC,MAAM,uDAAuD,CAAC;AAI7F;;;;GAIG;AACH,MAAM,MAAM,aAAa,GAAG;IAC1B,8BAA8B,EAAE,OAAO,CAAC;IACxC,qBAAqB,EAAE,OAAO,CAAC;CAChC,CAAC;AAEF,KAAK,aAAa,GAAG,MAAM,CAAC;AAE5B;;;GAGG;AACH,KAAK,SAAS,GAAG,MAAM,CAAC;AAWxB,qBAAa,iBAAkB,YAAW,eAAe;;IAOvD,YAAY,sBAAsB,EAAE,sBAAsB,GAAG,SAAS,EAErE;IAED,SAAS,CAAC,CAAC,SAAS,MAAM,SAAS,EACjC,MAAM,EAAE,CAAC,EACT,KAAK,EAAE,MAAM,EACb,GAAG,IAAI,EAAE,SAAS,CAAC,CAAC,CAAC,GACpB,IAAI,CAaN;IAED,sBAAsB,CAAC,OAAO,EAAE,MAAM,GAAG,sBAAsB,GAAG,IAAI,CAUrE;IAED,cAAc;;;MAEb;IAED,cAAc,CAAC,OAAO,EAAE,MAAM,GAAG,GAAG,GAAG,SAAS,CAE/C;IAED,WAAW,CAAC,OAAO,EAAE,MAAM,GAAG,IAAI,CAIjC;IAED,QAAQ,CAAC,OAAO,EAAE,MAAM,EAAE,GAAG,EAAE,GAAG,GAAG,IAAI,CAExC;IAED;;;OAGG;IACH,eAAe,CAAC,aAAa,EAAE,aAAa,GAAG,OAAO,CAIrD;IAED;;;OAGG;IACH,gBAAgB,CACd,aAAa,EAAE,aAAa,EAC5B,KAAK,GAAE,SAAoB,GAC1B,IAAI,CAEN;IAED;;;;;;OAMG;IACH,kBAAkB,CAAC,aAAa,EAAE,aAAa,EAAE,KAAK,CAAC,EAAE,SAAS,QAOjE;IAED;;;;OAIG;IACG,oBAAoB,CACxB,IAAI,EAAE,MAAM,EACZ,IAAI,EAAE,SAAS,iBAAiB,EAAE,EAClC,GAAG,EAAE,iBAAiB,GACrB,OAAO,CAAC,GAAG,CAAC,CAsCd;CACF"}
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { assert } from "../../../shared/src/asserts.js";
|
|
2
|
+
import { getOrInsertComputed } from "../../../shared/src/map.js";
|
|
2
3
|
import { hashNameAndArgs } from "../../../zero-protocol/src/query-hash-visitor.js";
|
|
3
4
|
import { mapValues } from "../../../shared/src/objects.js";
|
|
4
5
|
import { isServerMetric } from "../../../zql/src/query/metrics-delegate.js";
|
|
@@ -27,14 +28,7 @@ var InspectorDelegate = class {
|
|
|
27
28
|
assert(isServerMetric(metric), `Invalid server metric: ${metric}`);
|
|
28
29
|
const queryID = args[0];
|
|
29
30
|
if (metric === "query-materialization-server") this.#perQueryHydrateMs.set(queryID, value);
|
|
30
|
-
else
|
|
31
|
-
let digest = this.#perQueryUpdateMetrics.get(queryID);
|
|
32
|
-
if (!digest) {
|
|
33
|
-
digest = new TDigest();
|
|
34
|
-
this.#perQueryUpdateMetrics.set(queryID, digest);
|
|
35
|
-
}
|
|
36
|
-
digest.add(value);
|
|
37
|
-
}
|
|
31
|
+
else getOrInsertComputed(this.#perQueryUpdateMetrics, queryID, () => new TDigest()).add(value);
|
|
38
32
|
this.#globalMetrics[metric].add(value);
|
|
39
33
|
}
|
|
40
34
|
getMetricsJSONForQuery(queryID) {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"inspector-delegate.js","names":["#globalMetrics","#perQueryHydrateMs","#perQueryUpdateMetrics","#queryIDToAST","#customQueryTransformer"],"sources":["../../../../../zero-cache/src/server/inspector-delegate.ts"],"sourcesContent":["import {assert} from '../../../shared/src/asserts.ts';\nimport type {ReadonlyJSONValue} from '../../../shared/src/json.ts';\nimport {mapValues} from '../../../shared/src/objects.ts';\nimport {TDigest} from '../../../shared/src/tdigest.ts';\nimport type {AST} from '../../../zero-protocol/src/ast.ts';\nimport type {QueryServerMetrics as QueryServerMetricsJSON} from '../../../zero-protocol/src/inspect-down.ts';\nimport {hashOfNameAndArgs} from '../../../zero-protocol/src/query-hash.ts';\nimport {\n isServerMetric,\n type MetricMap,\n type MetricsDelegate,\n} from '../../../zql/src/query/metrics-delegate.ts';\nimport {isDevelopmentMode} from '../config/normalize.ts';\nimport type {CustomQueryTransformer} from '../custom-queries/transform-query.ts';\nimport type {ConnectionContext} from '../services/view-syncer/connection-context-manager.ts';\nimport type {CustomQueryRecord} from '../services/view-syncer/schema/types.ts';\nimport {ProtocolErrorWithLevel} from '../types/error-with-level.ts';\n\n/**\n * Server-side metrics collected for queries during materialization and update.\n * These metrics are reported via the inspector and complement client-side metrics.\n * Used for the global aggregate (all queries combined).\n */\nexport type ServerMetrics = {\n 'query-materialization-server': TDigest;\n 'query-update-server': TDigest;\n};\n\ntype ClientGroupID = string;\n\n/**\n * The service that established a client group's authentication, i.e. the\n * ViewSyncer serving it. See {@link InspectorDelegate.clearAuthenticated}.\n */\ntype AuthOwner = object;\n\nconst NO_OWNER: AuthOwner = {};\n\n/**\n * Authenticated client group IDs, each mapped to the owner that authenticated\n * it. We keep this outside of the class to share this state across all\n * instances of the InspectorDelegate.\n */\nconst authenticatedClientGroupIDs = new Map<ClientGroupID, AuthOwner>();\n\nexport class InspectorDelegate implements MetricsDelegate {\n readonly #globalMetrics: ServerMetrics = newMetrics();\n readonly #perQueryHydrateMs = new Map<string, number>();\n readonly #perQueryUpdateMetrics = new Map<string, TDigest>();\n readonly #queryIDToAST: Map<string, AST> = new Map();\n readonly #customQueryTransformer: CustomQueryTransformer | undefined;\n\n constructor(customQueryTransformer: CustomQueryTransformer | undefined) {\n this.#customQueryTransformer = customQueryTransformer;\n }\n\n addMetric<K extends keyof MetricMap>(\n metric: K,\n value: number,\n ...args: MetricMap[K]\n ): void {\n assert(isServerMetric(metric), `Invalid server metric: ${metric}`);\n const queryID = args[0];\n if (metric === 'query-materialization-server') {\n this.#perQueryHydrateMs.set(queryID, value);\n } else {\n
|
|
1
|
+
{"version":3,"file":"inspector-delegate.js","names":["#globalMetrics","#perQueryHydrateMs","#perQueryUpdateMetrics","#queryIDToAST","#customQueryTransformer"],"sources":["../../../../../zero-cache/src/server/inspector-delegate.ts"],"sourcesContent":["import {assert} from '../../../shared/src/asserts.ts';\nimport type {ReadonlyJSONValue} from '../../../shared/src/json.ts';\nimport {getOrInsertComputed} from '../../../shared/src/map.ts';\nimport {mapValues} from '../../../shared/src/objects.ts';\nimport {TDigest} from '../../../shared/src/tdigest.ts';\nimport type {AST} from '../../../zero-protocol/src/ast.ts';\nimport type {QueryServerMetrics as QueryServerMetricsJSON} from '../../../zero-protocol/src/inspect-down.ts';\nimport {hashOfNameAndArgs} from '../../../zero-protocol/src/query-hash.ts';\nimport {\n isServerMetric,\n type MetricMap,\n type MetricsDelegate,\n} from '../../../zql/src/query/metrics-delegate.ts';\nimport {isDevelopmentMode} from '../config/normalize.ts';\nimport type {CustomQueryTransformer} from '../custom-queries/transform-query.ts';\nimport type {ConnectionContext} from '../services/view-syncer/connection-context-manager.ts';\nimport type {CustomQueryRecord} from '../services/view-syncer/schema/types.ts';\nimport {ProtocolErrorWithLevel} from '../types/error-with-level.ts';\n\n/**\n * Server-side metrics collected for queries during materialization and update.\n * These metrics are reported via the inspector and complement client-side metrics.\n * Used for the global aggregate (all queries combined).\n */\nexport type ServerMetrics = {\n 'query-materialization-server': TDigest;\n 'query-update-server': TDigest;\n};\n\ntype ClientGroupID = string;\n\n/**\n * The service that established a client group's authentication, i.e. the\n * ViewSyncer serving it. See {@link InspectorDelegate.clearAuthenticated}.\n */\ntype AuthOwner = object;\n\nconst NO_OWNER: AuthOwner = {};\n\n/**\n * Authenticated client group IDs, each mapped to the owner that authenticated\n * it. We keep this outside of the class to share this state across all\n * instances of the InspectorDelegate.\n */\nconst authenticatedClientGroupIDs = new Map<ClientGroupID, AuthOwner>();\n\nexport class InspectorDelegate implements MetricsDelegate {\n readonly #globalMetrics: ServerMetrics = newMetrics();\n readonly #perQueryHydrateMs = new Map<string, number>();\n readonly #perQueryUpdateMetrics = new Map<string, TDigest>();\n readonly #queryIDToAST: Map<string, AST> = new Map();\n readonly #customQueryTransformer: CustomQueryTransformer | undefined;\n\n constructor(customQueryTransformer: CustomQueryTransformer | undefined) {\n this.#customQueryTransformer = customQueryTransformer;\n }\n\n addMetric<K extends keyof MetricMap>(\n metric: K,\n value: number,\n ...args: MetricMap[K]\n ): void {\n assert(isServerMetric(metric), `Invalid server metric: ${metric}`);\n const queryID = args[0];\n if (metric === 'query-materialization-server') {\n this.#perQueryHydrateMs.set(queryID, value);\n } else {\n getOrInsertComputed(\n this.#perQueryUpdateMetrics,\n queryID,\n () => new TDigest(),\n ).add(value);\n }\n this.#globalMetrics[metric].add(value);\n }\n\n getMetricsJSONForQuery(queryID: string): QueryServerMetricsJSON | null {\n const hydrateMs = this.#perQueryHydrateMs.get(queryID);\n const updateMetrics = this.#perQueryUpdateMetrics.get(queryID);\n if (hydrateMs === undefined && updateMetrics === undefined) {\n return null;\n }\n return {\n 'query-hydration-server-ms': hydrateMs,\n 'query-update-server': (updateMetrics ?? new TDigest()).toJSON(),\n };\n }\n\n getMetricsJSON() {\n return mapValues(this.#globalMetrics, v => v.toJSON());\n }\n\n getASTForQuery(queryID: string): AST | undefined {\n return this.#queryIDToAST.get(queryID);\n }\n\n removeQuery(queryID: string): void {\n this.#perQueryHydrateMs.delete(queryID);\n this.#perQueryUpdateMetrics.delete(queryID);\n this.#queryIDToAST.delete(queryID);\n }\n\n addQuery(queryID: string, ast: AST): void {\n this.#queryIDToAST.set(queryID, ast);\n }\n\n /**\n * Check if the client is authenticated. We only require authentication once\n * per \"worker\".\n */\n isAuthenticated(clientGroupID: ClientGroupID): boolean {\n return (\n isDevelopmentMode() || authenticatedClientGroupIDs.has(clientGroupID)\n );\n }\n\n /**\n * Marks the client group as authenticated on behalf of `owner`, the service\n * (ViewSyncer) that handled the authentication.\n */\n setAuthenticated(\n clientGroupID: ClientGroupID,\n owner: AuthOwner = NO_OWNER,\n ): void {\n authenticatedClientGroupIDs.set(clientGroupID, owner);\n }\n\n /**\n * Clears the client group's authentication. When `owner` is given, the\n * authentication is only cleared if it was established by that owner: a\n * replacement ViewSyncer can start for the same client group while the\n * previous one is still shutting down, and the previous one's cleanup must\n * not revoke what the replacement has since authenticated.\n */\n clearAuthenticated(clientGroupID: ClientGroupID, owner?: AuthOwner) {\n if (\n owner === undefined ||\n authenticatedClientGroupIDs.get(clientGroupID) === owner\n ) {\n authenticatedClientGroupIDs.delete(clientGroupID);\n }\n }\n\n /**\n * Transforms a single custom query by name and args using the configured\n * CustomQueryTransformer. This is primarily used by the inspector to transform\n * queries for analysis.\n */\n async transformCustomQuery(\n name: string,\n args: readonly ReadonlyJSONValue[],\n ctx: ConnectionContext,\n ): Promise<AST> {\n assert(\n this.#customQueryTransformer,\n 'Custom query transformation requested but no CustomQueryTransformer is configured',\n );\n\n // Create a fake CustomQueryRecord for the single query\n const queryID = hashOfNameAndArgs(name, args);\n const queries: CustomQueryRecord[] = [\n {\n id: queryID,\n type: 'custom',\n name,\n args,\n clientState: {},\n },\n ];\n\n const results = await this.#customQueryTransformer.transform(ctx, queries);\n\n if ('kind' in results.result) {\n throw new ProtocolErrorWithLevel(results.result, 'warn');\n }\n\n const result = results.result[0];\n if (!result) {\n throw new Error('No transformation result returned');\n }\n\n if ('error' in result) {\n const message =\n result.message ?? 'Unknown application error from custom query';\n throw new Error(\n `Error transforming custom query ${name} (${result.error}): ${message} ${JSON.stringify(result.details)}`,\n );\n }\n\n return result.transformedAst;\n }\n}\n\nfunction newMetrics(): ServerMetrics {\n return {\n 'query-materialization-server': new TDigest(),\n 'query-update-server': new TDigest(),\n };\n}\n"],"mappings":";;;;;;;;;;AAqCA,IAAM,WAAsB,CAAC;;;;;;AAO7B,IAAM,8CAA8B,IAAI,IAA8B;AAEtE,IAAa,oBAAb,MAA0D;CACxD,iBAAyC,WAAW;CACpD,qCAA8B,IAAI,IAAoB;CACtD,yCAAkC,IAAI,IAAqB;CAC3D,gCAA2C,IAAI,IAAI;CACnD;CAEA,YAAY,wBAA4D;EACtE,KAAKI,0BAA0B;CACjC;CAEA,UACE,QACA,OACA,GAAG,MACG;EACN,OAAO,eAAe,MAAM,GAAG,0BAA0B,QAAQ;EACjE,MAAM,UAAU,KAAK;EACrB,IAAI,WAAW,gCACb,KAAKH,mBAAmB,IAAI,SAAS,KAAK;OAE1C,oBACE,KAAKC,wBACL,eACM,IAAI,QAAQ,CACpB,EAAE,IAAI,KAAK;EAEb,KAAKF,eAAe,QAAQ,IAAI,KAAK;CACvC;CAEA,uBAAuB,SAAgD;EACrE,MAAM,YAAY,KAAKC,mBAAmB,IAAI,OAAO;EACrD,MAAM,gBAAgB,KAAKC,uBAAuB,IAAI,OAAO;EAC7D,IAAI,cAAc,KAAA,KAAa,kBAAkB,KAAA,GAC/C,OAAO;EAET,OAAO;GACL,6BAA6B;GAC7B,wBAAwB,iBAAiB,IAAI,QAAQ,GAAG,OAAO;EACjE;CACF;CAEA,iBAAiB;EACf,OAAO,UAAU,KAAKF,iBAAgB,MAAK,EAAE,OAAO,CAAC;CACvD;CAEA,eAAe,SAAkC;EAC/C,OAAO,KAAKG,cAAc,IAAI,OAAO;CACvC;CAEA,YAAY,SAAuB;EACjC,KAAKF,mBAAmB,OAAO,OAAO;EACtC,KAAKC,uBAAuB,OAAO,OAAO;EAC1C,KAAKC,cAAc,OAAO,OAAO;CACnC;CAEA,SAAS,SAAiB,KAAgB;EACxC,KAAKA,cAAc,IAAI,SAAS,GAAG;CACrC;;;;;CAMA,gBAAgB,eAAuC;EACrD,OACE,kBAAkB,KAAK,4BAA4B,IAAI,aAAa;CAExE;;;;;CAMA,iBACE,eACA,QAAmB,UACb;EACN,4BAA4B,IAAI,eAAe,KAAK;CACtD;;;;;;;;CASA,mBAAmB,eAA8B,OAAmB;EAClE,IACE,UAAU,KAAA,KACV,4BAA4B,IAAI,aAAa,MAAM,OAEnD,4BAA4B,OAAO,aAAa;CAEpD;;;;;;CAOA,MAAM,qBACJ,MACA,MACA,KACc;EACd,OACE,KAAKC,yBACL,mFACF;EAIA,MAAM,UAA+B,CACnC;GACE,IAHY,gBAAkB,MAAM,IAGhC;GACJ,MAAM;GACN;GACA;GACA,aAAa,CAAC;EAChB,CACF;EAEA,MAAM,UAAU,MAAM,KAAKA,wBAAwB,UAAU,KAAK,OAAO;EAEzE,IAAI,UAAU,QAAQ,QACpB,MAAM,IAAI,uBAAuB,QAAQ,QAAQ,MAAM;EAGzD,MAAM,SAAS,QAAQ,OAAO;EAC9B,IAAI,CAAC,QACH,MAAM,IAAI,MAAM,mCAAmC;EAGrD,IAAI,WAAW,QAAQ;GACrB,MAAM,UACJ,OAAO,WAAW;GACpB,MAAM,IAAI,MACR,mCAAmC,KAAK,IAAI,OAAO,MAAM,KAAK,QAAQ,GAAG,KAAK,UAAU,OAAO,OAAO,GACxG;EACF;EAEA,OAAO,OAAO;CAChB;AACF;AAEA,SAAS,aAA4B;CACnC,OAAO;EACL,gCAAgC,IAAI,QAAQ;EAC5C,uBAAuB,IAAI,QAAQ;CACrC;AACF"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"replicator.d.ts","sourceRoot":"","sources":["../../../../../zero-cache/src/server/replicator.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"replicator.d.ts","sourceRoot":"","sources":["../../../../../zero-cache/src/server/replicator.ts"],"names":[],"mappings":"AA4CA,OAAO,EAGL,KAAK,MAAM,EACZ,MAAM,uBAAuB,CAAC;AAiB/B,wBAA8B,SAAS,CACrC,MAAM,EAAE,MAAM,EACd,GAAG,EAAE,MAAM,CAAC,UAAU,EACtB,GAAG,IAAI,EAAE,MAAM,EAAE,GAChB,OAAO,CAAC,IAAI,CAAC,CA4Hf"}
|
|
@@ -14,12 +14,13 @@ import { initEventSink } from "../observability/events.js";
|
|
|
14
14
|
import { ReplicationStatusPublisher } from "../services/replicator/replication-status.js";
|
|
15
15
|
import { litestreamRestoreDuration, litestreamRestoreMetricAttrs, litestreamRestoreRuns } from "../services/litestream/metrics.js";
|
|
16
16
|
import { tryRestore } from "../services/litestream/commands.js";
|
|
17
|
-
import { reserveAndGetSnapshotStatus } from "../services/change-streamer/snapshot.js";
|
|
18
17
|
import { ChangeStreamerHttpClient } from "../services/change-streamer/change-streamer-http.js";
|
|
19
18
|
import { sqliteFileBytes } from "../services/replicator/sqlite-change-log-observability.js";
|
|
20
19
|
import { startOtelAuto } from "./otel-start.js";
|
|
21
20
|
import { ThreadWriteWorkerClient } from "../services/replicator/write-worker-client.js";
|
|
22
21
|
import { deleteStaleChangeLog, getPragmaConfig, replicaFileModeSchema, setUpMessageHandlers, setupReplica } from "../workers/replicator.js";
|
|
22
|
+
import { reserveAndGetSnapshotStatus } from "../services/change-streamer/snapshot.js";
|
|
23
|
+
import { RestoreProgressReporter } from "../services/litestream/restore-progress.js";
|
|
23
24
|
import { ReplicatorService } from "../services/replicator/replicator.js";
|
|
24
25
|
import { LogContext, consoleLogSink } from "@rocicorp/logger";
|
|
25
26
|
import { pid } from "node:process";
|
|
@@ -44,7 +45,11 @@ async function runWorker(parent, env, ...args) {
|
|
|
44
45
|
dbPath: config.replica.file
|
|
45
46
|
}, config.sqliteCorruptionChecks);
|
|
46
47
|
initEventSink(lc, config);
|
|
47
|
-
|
|
48
|
+
const shard = getShardConfig(config);
|
|
49
|
+
const { taskID, change, changeStreamer: { port, uri: changeStreamerURI = runningLocalChangeStreamer ? `http://localhost:${port}/` : void 0 } } = config;
|
|
50
|
+
const changeStreamer = new ChangeStreamerHttpClient(lc, shard, change.db, changeStreamerURI);
|
|
51
|
+
let initialConnection;
|
|
52
|
+
if (fileMode === "serving" && !runningLocalChangeStreamer && (config.litestream.executable || config.litestream.executableV5)) initialConnection = await restoreReplica(lc, config, changeStreamer);
|
|
48
53
|
const { file: dbPath, walMode, pageSize } = await setupReplica(lc, fileMode, config.replica);
|
|
49
54
|
unregisterInitialCorruptionDiagnosticTarget();
|
|
50
55
|
registerSQLiteCorruptionDiagnosticTarget({
|
|
@@ -60,10 +65,7 @@ async function runWorker(parent, env, ...args) {
|
|
|
60
65
|
} : null;
|
|
61
66
|
const workerClient = new ThreadWriteWorkerClient();
|
|
62
67
|
await workerClient.init(dbPath, mode, pragmas, config.log, checkpoint);
|
|
63
|
-
const
|
|
64
|
-
const { taskID, change, changeStreamer: { port, uri: changeStreamerURI = runningLocalChangeStreamer ? `http://localhost:${port}/` : void 0 } } = config;
|
|
65
|
-
const changeStreamer = new ChangeStreamerHttpClient(lc, shard, change.db, changeStreamerURI);
|
|
66
|
-
const replicator = new ReplicatorService(lc, taskID, `${workerName}-${pid}`, mode, changeStreamer, workerClient, runningLocalChangeStreamer ? ReplicationStatusPublisher.forReplicaFile(dbPath) : null);
|
|
68
|
+
const replicator = new ReplicatorService(lc, taskID, `${workerName}-${pid}`, mode, changeStreamer, workerClient, runningLocalChangeStreamer ? ReplicationStatusPublisher.forReplicaFile(dbPath) : null, initialConnection);
|
|
67
69
|
setUpMessageHandlers(lc, replicator, parent);
|
|
68
70
|
const running = runUntilKilled(lc, parent, replicator);
|
|
69
71
|
for await (const _ of replicator.subscribe()) {
|
|
@@ -104,26 +106,44 @@ function observeSQLiteFileBytes(lc, file) {
|
|
|
104
106
|
return async (o) => o.observe(await sqliteFileBytes(lc, file));
|
|
105
107
|
}
|
|
106
108
|
var RETRY_INTERVAL_MS = 3e3;
|
|
107
|
-
async function restoreReplica(lc, config) {
|
|
109
|
+
async function restoreReplica(lc, config, changeStreamer) {
|
|
108
110
|
const start = performance.now();
|
|
109
111
|
let backupURL;
|
|
110
112
|
let result;
|
|
113
|
+
const progress = new RestoreProgressReporter(lc, config.replica.file);
|
|
111
114
|
try {
|
|
112
115
|
for (;;) {
|
|
113
|
-
const
|
|
114
|
-
({backupURL} =
|
|
115
|
-
const
|
|
116
|
+
const { reserved, followup } = await reserveAndGetSnapshotStatus(lc, config.taskID, changeStreamer);
|
|
117
|
+
({backupURL} = reserved);
|
|
118
|
+
const litestream = {
|
|
116
119
|
...config.litestream,
|
|
117
120
|
backupURL
|
|
118
|
-
}
|
|
121
|
+
};
|
|
122
|
+
progress.start(reserved.replicaSize);
|
|
123
|
+
let attempt;
|
|
124
|
+
try {
|
|
125
|
+
attempt = await tryRestore(lc, litestream, config.replica.file, reserved, "view_syncer", followup.signal);
|
|
126
|
+
} catch (e) {
|
|
127
|
+
if (followup.signal.aborted) {
|
|
128
|
+
lc.info?.(`reservation disconnected during restore. retrying`, String(e));
|
|
129
|
+
continue;
|
|
130
|
+
}
|
|
131
|
+
followup.cancel();
|
|
132
|
+
throw e;
|
|
133
|
+
} finally {
|
|
134
|
+
progress.stop();
|
|
135
|
+
}
|
|
119
136
|
if (attempt.restored) {
|
|
120
137
|
result = attempt.result;
|
|
121
|
-
|
|
138
|
+
progress.done();
|
|
139
|
+
return followup;
|
|
122
140
|
}
|
|
141
|
+
followup.cancel();
|
|
123
142
|
lc.info?.(`replica not found. retrying in ${RETRY_INTERVAL_MS / 1e3} seconds`);
|
|
124
143
|
await sleep(RETRY_INTERVAL_MS);
|
|
125
144
|
}
|
|
126
145
|
} finally {
|
|
146
|
+
progress.stop();
|
|
127
147
|
const labels = {
|
|
128
148
|
...litestreamRestoreMetricAttrs(config.litestream, "view_syncer", backupURL),
|
|
129
149
|
result: result ?? "error"
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"replicator.js","names":[],"sources":["../../../../../zero-cache/src/server/replicator.ts"],"sourcesContent":["import {stat} from 'node:fs/promises';\nimport {pid} from 'node:process';\nimport type {ObservableCallback} from '@opentelemetry/api';\nimport {consoleLogSink, LogContext} from '@rocicorp/logger';\nimport {assert} from '../../../shared/src/asserts.ts';\nimport {must} from '../../../shared/src/must.ts';\nimport {sleep} from '../../../shared/src/sleep.ts';\nimport * as v from '../../../shared/src/valita.ts';\nimport type {\n LitestreamConfig,\n NormalizedZeroConfig,\n} from '../config/normalize.ts';\nimport {getNormalizedZeroConfig} from '../config/zero-config.ts';\nimport {registerSQLiteCorruptionDiagnosticTarget} from '../db/sqlite-corruption.ts';\nimport {initEventSink} from '../observability/events.ts';\nimport {getOrCreateGauge} from '../observability/metrics.ts';\nimport {ChangeStreamerHttpClient} from '../services/change-streamer/change-streamer-http.ts';\nimport {reserveAndGetSnapshotStatus} from '../services/change-streamer/snapshot.ts';\nimport {exitAfter, runUntilKilled} from '../services/life-cycle.ts';\nimport {\n tryRestore,\n type RestoreResult,\n} from '../services/litestream/commands.ts';\nimport {\n litestreamRestoreDuration,\n litestreamRestoreMetricAttrs,\n litestreamRestoreRuns,\n} from '../services/litestream/metrics.ts';\nimport {ReplicationStatusPublisher} from '../services/replicator/replication-status.ts';\nimport {\n ReplicatorService,\n type ReplicatorMode,\n} from '../services/replicator/replicator.ts';\nimport {sqliteFileBytes} from '../services/replicator/sqlite-change-log-observability.ts';\nimport {\n ThreadWriteWorkerClient,\n type ForceCheckpointConfig,\n} from '../services/replicator/write-worker-client.ts';\nimport {\n parentWorker,\n singleProcessMode,\n type Worker,\n} from '../types/processes.ts';\nimport {installProfileHandler} from '../types/profiler.ts';\nimport {getShardConfig} from '../types/shards.ts';\nimport {\n deleteStaleChangeLog,\n getPragmaConfig,\n replicaFileModeSchema,\n setUpMessageHandlers,\n setupReplica,\n type WalMode,\n} from '../workers/replicator.ts';\nimport {createLogContext} from './logging.ts';\nimport {startOtelAuto} from './otel-start.ts';\n\n// Default LogContext, overridden in runWorker\nlet lc = new LogContext('info', {}, consoleLogSink);\n\nexport default async function runWorker(\n parent: Worker,\n env: NodeJS.ProcessEnv,\n ...args: string[]\n): Promise<void> {\n assert(args.length > 0, `replicator mode not specified`);\n const fileMode = v.parse(args[0], replicaFileModeSchema);\n\n const config = getNormalizedZeroConfig({env, argv: args.slice(1)});\n const mode: ReplicatorMode = fileMode === 'backup' ? 'backup' : 'serving';\n const runningLocalChangeStreamer =\n config.changeStreamer.mode === 'dedicated' && !config.changeStreamer.uri;\n const workerName = `${mode}-replicator`;\n installProfileHandler(parent, workerName);\n startOtelAuto(createLogContext(config, workerName, 0, false), workerName, 0);\n lc = createLogContext(config, workerName);\n const unregisterInitialCorruptionDiagnosticTarget =\n registerSQLiteCorruptionDiagnosticTarget(\n {\n debugName: `${workerName} replica`,\n dbPath: config.replica.file,\n },\n config.sqliteCorruptionChecks,\n );\n initEventSink(lc, config);\n\n // Remote view-syncers restore from the replication-manager, which supplies\n // the backup URL. A local change-streamer owns the canonical replica and\n // must not infer backup configuration from bundled executable paths.\n if (\n fileMode === 'serving' &&\n !runningLocalChangeStreamer &&\n (config.litestream.executable || config.litestream.executableV5)\n ) {\n await restoreReplica(lc, config);\n }\n\n const {\n file: dbPath,\n walMode,\n pageSize,\n } = await setupReplica(lc, fileMode, config.replica);\n unregisterInitialCorruptionDiagnosticTarget();\n registerSQLiteCorruptionDiagnosticTarget(\n {\n debugName: `${workerName} replica`,\n dbPath,\n },\n config.sqliteCorruptionChecks,\n );\n\n // A file left behind by a run with the writer enabled would otherwise hold\n // local disk indefinitely. See `replicatorDeletesStaleChangeLog` for why\n // this is keyed on the config flag and nothing else.\n deleteStaleChangeLog(config.changeStreamer.sqliteChangeLogMode, dbPath);\n\n setupMetrics(lc, dbPath, walMode);\n\n // Create the write worker for async SQLite writes. Enable write-path\n // checkpoint backpressure only for the backup replicator on litestream v5,\n // which owns the replica litestream backs up.\n const pragmas = getPragmaConfig(fileMode);\n const checkpoint: ForceCheckpointConfig | null =\n mode === 'backup' &&\n config.litestream.backupURL &&\n config.litestream.backupUsingV5 &&\n config.litestream.forceCheckpointThresholdMB\n ? {\n checkpointThresholdPages:\n (config.litestream.forceCheckpointThresholdMB * 1024 ** 2) /\n pageSize,\n maxWalPages: !config.litestream.maxWalSizeMB\n ? undefined\n : (config.litestream.maxWalSizeMB * 1024 ** 2) / pageSize,\n }\n : null;\n const workerClient = new ThreadWriteWorkerClient();\n await workerClient.init(dbPath, mode, pragmas, config.log, checkpoint);\n\n const shard = getShardConfig(config);\n const {\n taskID,\n change,\n changeStreamer: {\n port,\n uri: changeStreamerURI = runningLocalChangeStreamer\n ? `http://localhost:${port}/`\n : undefined,\n },\n } = config;\n const changeStreamer = new ChangeStreamerHttpClient(\n lc,\n shard,\n change.db,\n changeStreamerURI,\n );\n\n const replicator = new ReplicatorService(\n lc,\n taskID,\n `${workerName}-${pid}`,\n mode,\n changeStreamer,\n workerClient,\n runningLocalChangeStreamer\n ? // publish ReplicationStatusEvents from backup-replicator only\n ReplicationStatusPublisher.forReplicaFile(dbPath)\n : null,\n );\n\n setUpMessageHandlers(lc, replicator, parent);\n\n const running = runUntilKilled(lc, parent, replicator);\n\n // Signal readiness once the first ReplicaVersionReady notification is received.\n for await (const _ of replicator.subscribe()) {\n parent.send(['ready', {ready: true}]);\n break;\n }\n\n return running;\n}\n\nfunction setupMetrics(lc: LogContext, file: string, walMode: WalMode) {\n getOrCreateGauge('replica', 'db_size', {\n description:\n `The size of the replica's main db file, ` +\n `which does not include the wal file(s)`,\n unit: 'bytes',\n }).addCallback(observeFileSize(lc, file));\n\n getOrCreateGauge('replica', 'wal_size', {\n description: `The size of the replica's wal file`,\n unit: 'bytes',\n }).addCallback(observeFileSize(lc, `${file}-wal`));\n\n if (walMode === 'wal2') {\n getOrCreateGauge('replica', 'wal2_size', {\n description: `The size of the replica's wal2 file`,\n unit: 'bytes',\n }).addCallback(observeFileSize(lc, `${file}-wal2`));\n }\n\n // A whole-database total rather than a per-file one. Its counterpart,\n // `sqlite_change_log.file_bytes`, is emitted by the change-streamer, which is\n // the process that writes the log.\n getOrCreateGauge('replica', 'file_bytes', {\n description:\n `The replica's total on-disk footprint: its main db file plus its ` +\n `wal sidecars.`,\n unit: 'By',\n }).addCallback(observeSQLiteFileBytes(lc, file));\n}\n\nfunction observeFileSize(lc: LogContext, file: string): ObservableCallback {\n return async o => {\n try {\n const stats = await stat(file);\n o.observe(stats.size);\n } catch (e) {\n lc.warn?.(`unable to stat ${file} for size metrics`, e);\n }\n };\n}\n\nfunction observeSQLiteFileBytes(\n lc: LogContext,\n file: string,\n): ObservableCallback {\n return async o => o.observe(await sqliteFileBytes(lc, file));\n}\n\nconst RETRY_INTERVAL_MS = 3000;\n\n// View-syncers (no replicaConstraints) wait indefinitely for the\n// replication-manager to publish a restorable backup. On a fresh stack the\n// first backup is not durable until the initial sync completes and litestream\n// uploads the initial snapshot, which can take many minutes for a large\n// replica. The platform's startup probe budget (which scales with replica\n// size) is the backstop, so restoreReplica must not impose its own shorter\n// cap and self-terminate while the backup is still being produced.\nasync function restoreReplica(lc: LogContext, config: NormalizedZeroConfig) {\n const start = performance.now();\n let backupURL: string | undefined;\n let result: RestoreResult | undefined;\n try {\n for (;;) {\n const snapshotStatus = await reserveAndGetSnapshotStatus(lc, config);\n // The backupURL comes from the replication-manager's snapshot response.\n ({backupURL} = snapshotStatus);\n const litestream: LitestreamConfig = {...config.litestream, backupURL};\n const attempt = await tryRestore(\n lc,\n litestream,\n config.replica.file,\n snapshotStatus,\n 'view_syncer',\n );\n if (attempt.restored) {\n result = attempt.result;\n return;\n }\n lc.info?.(\n `replica not found. retrying in ${RETRY_INTERVAL_MS / 1000} seconds`,\n );\n await sleep(RETRY_INTERVAL_MS);\n }\n } finally {\n const attrs = litestreamRestoreMetricAttrs(\n config.litestream,\n 'view_syncer',\n backupURL,\n );\n const labels = {...attrs, result: result ?? 'error'};\n litestreamRestoreRuns().add(1, labels);\n litestreamRestoreDuration().recordMs(performance.now() - start, labels);\n }\n}\n\n// fork()\nif (!singleProcessMode()) {\n void exitAfter(\n () => lc,\n () => runWorker(must(parentWorker), process.env, ...process.argv.slice(2)),\n );\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;AAyDA,IAAI,KAAK,IAAI,WAAW,QAAQ,CAAC,GAAG,cAAc;AAElD,eAA8B,UAC5B,QACA,KACA,GAAG,MACY;CACf,OAAO,KAAK,SAAS,GAAG,+BAA+B;CACvD,MAAM,WAAW,MAAQ,KAAK,IAAI,qBAAqB;CAEvD,MAAM,SAAS,wBAAwB;EAAC;EAAK,MAAM,KAAK,MAAM,CAAC;CAAC,CAAC;CACjE,MAAM,OAAuB,aAAa,WAAW,WAAW;CAChE,MAAM,6BACJ,OAAO,eAAe,SAAS,eAAe,CAAC,OAAO,eAAe;CACvE,MAAM,aAAa,GAAG,KAAK;CAC3B,sBAAsB,QAAQ,UAAU;CACxC,cAAc,iBAAiB,QAAQ,YAAY,GAAG,KAAK,GAAG,YAAY,CAAC;CAC3E,KAAK,iBAAiB,QAAQ,UAAU;CACxC,MAAM,8CACJ,yCACE;EACE,WAAW,GAAG,WAAW;EACzB,QAAQ,OAAO,QAAQ;CACzB,GACA,OAAO,sBACT;CACF,cAAc,IAAI,MAAM;CAKxB,IACE,aAAa,aACb,CAAC,+BACA,OAAO,WAAW,cAAc,OAAO,WAAW,eAEnD,MAAM,eAAe,IAAI,MAAM;CAGjC,MAAM,EACJ,MAAM,QACN,SACA,aACE,MAAM,aAAa,IAAI,UAAU,OAAO,OAAO;CACnD,4CAA4C;CAC5C,yCACE;EACE,WAAW,GAAG,WAAW;EACzB;CACF,GACA,OAAO,sBACT;CAKA,qBAAqB,OAAO,eAAe,qBAAqB,MAAM;CAEtE,aAAa,IAAI,QAAQ,OAAO;CAKhC,MAAM,UAAU,gBAAgB,QAAQ;CACxC,MAAM,aACJ,SAAS,YACT,OAAO,WAAW,aAClB,OAAO,WAAW,iBAClB,OAAO,WAAW,6BACd;EACE,0BACG,OAAO,WAAW,6BAA6B,QAAQ,IACxD;EACF,aAAa,CAAC,OAAO,WAAW,eAC5B,KAAA,IACC,OAAO,WAAW,eAAe,QAAQ,IAAK;CACrD,IACA;CACN,MAAM,eAAe,IAAI,wBAAwB;CACjD,MAAM,aAAa,KAAK,QAAQ,MAAM,SAAS,OAAO,KAAK,UAAU;CAErE,MAAM,QAAQ,eAAe,MAAM;CACnC,MAAM,EACJ,QACA,QACA,gBAAgB,EACd,MACA,KAAK,oBAAoB,6BACrB,oBAAoB,KAAK,KACzB,KAAA,QAEJ;CACJ,MAAM,iBAAiB,IAAI,yBACzB,IACA,OACA,OAAO,IACP,iBACF;CAEA,MAAM,aAAa,IAAI,kBACrB,IACA,QACA,GAAG,WAAW,GAAG,OACjB,MACA,gBACA,cACA,6BAEI,2BAA2B,eAAe,MAAM,IAChD,IACN;CAEA,qBAAqB,IAAI,YAAY,MAAM;CAE3C,MAAM,UAAU,eAAe,IAAI,QAAQ,UAAU;CAGrD,WAAW,MAAM,KAAK,WAAW,UAAU,GAAG;EAC5C,OAAO,KAAK,CAAC,SAAS,EAAC,OAAO,KAAI,CAAC,CAAC;EACpC;CACF;CAEA,OAAO;AACT;AAEA,SAAS,aAAa,IAAgB,MAAc,SAAkB;CACpE,iBAAiB,WAAW,WAAW;EACrC,aACE;EAEF,MAAM;CACR,CAAC,EAAE,YAAY,gBAAgB,IAAI,IAAI,CAAC;CAExC,iBAAiB,WAAW,YAAY;EACtC,aAAa;EACb,MAAM;CACR,CAAC,EAAE,YAAY,gBAAgB,IAAI,GAAG,KAAK,KAAK,CAAC;CAEjD,IAAI,YAAY,QACd,iBAAiB,WAAW,aAAa;EACvC,aAAa;EACb,MAAM;CACR,CAAC,EAAE,YAAY,gBAAgB,IAAI,GAAG,KAAK,MAAM,CAAC;CAMpD,iBAAiB,WAAW,cAAc;EACxC,aACE;EAEF,MAAM;CACR,CAAC,EAAE,YAAY,uBAAuB,IAAI,IAAI,CAAC;AACjD;AAEA,SAAS,gBAAgB,IAAgB,MAAkC;CACzE,OAAO,OAAM,MAAK;EAChB,IAAI;GACF,MAAM,QAAQ,MAAM,KAAK,IAAI;GAC7B,EAAE,QAAQ,MAAM,IAAI;EACtB,SAAS,GAAG;GACV,GAAG,OAAO,kBAAkB,KAAK,oBAAoB,CAAC;EACxD;CACF;AACF;AAEA,SAAS,uBACP,IACA,MACoB;CACpB,OAAO,OAAM,MAAK,EAAE,QAAQ,MAAM,gBAAgB,IAAI,IAAI,CAAC;AAC7D;AAEA,IAAM,oBAAoB;AAS1B,eAAe,eAAe,IAAgB,QAA8B;CAC1E,MAAM,QAAQ,YAAY,IAAI;CAC9B,IAAI;CACJ,IAAI;CACJ,IAAI;EACF,SAAS;GACP,MAAM,iBAAiB,MAAM,4BAA4B,IAAI,MAAM;GAEnE,CAAC,CAAC,aAAa;GAEf,MAAM,UAAU,MAAM,WACpB,IACA;IAHoC,GAAG,OAAO;IAAY;GAG1D,GACA,OAAO,QAAQ,MACf,gBACA,aACF;GACA,IAAI,QAAQ,UAAU;IACpB,SAAS,QAAQ;IACjB;GACF;GACA,GAAG,OACD,kCAAkC,oBAAoB,IAAK,SAC7D;GACA,MAAM,MAAM,iBAAiB;EAC/B;CACF,UAAU;EAMR,MAAM,SAAS;GAAC,GALF,6BACZ,OAAO,YACP,eACA,SAEiB;GAAO,QAAQ,UAAU;EAAO;EACnD,sBAAsB,EAAE,IAAI,GAAG,MAAM;EACrC,0BAA0B,EAAE,SAAS,YAAY,IAAI,IAAI,OAAO,MAAM;CACxE;AACF;AAGA,IAAI,CAAC,kBAAkB,GACrB,gBACQ,UACA,UAAU,KAAK,YAAY,GAAG,QAAQ,KAAK,GAAG,QAAQ,KAAK,MAAM,CAAC,CAAC,CAC3E"}
|
|
1
|
+
{"version":3,"file":"replicator.js","names":[],"sources":["../../../../../zero-cache/src/server/replicator.ts"],"sourcesContent":["import {stat} from 'node:fs/promises';\nimport {pid} from 'node:process';\nimport type {ObservableCallback} from '@opentelemetry/api';\nimport {consoleLogSink, LogContext} from '@rocicorp/logger';\nimport {assert} from '../../../shared/src/asserts.ts';\nimport {must} from '../../../shared/src/must.ts';\nimport {sleep} from '../../../shared/src/sleep.ts';\nimport * as v from '../../../shared/src/valita.ts';\nimport type {\n LitestreamConfig,\n NormalizedZeroConfig,\n} from '../config/normalize.ts';\nimport {getNormalizedZeroConfig} from '../config/zero-config.ts';\nimport {registerSQLiteCorruptionDiagnosticTarget} from '../db/sqlite-corruption.ts';\nimport {initEventSink} from '../observability/events.ts';\nimport {getOrCreateGauge} from '../observability/metrics.ts';\nimport {\n ChangeStreamerHttpClient,\n type ReservationFollowup,\n type SnapshotReserver,\n} from '../services/change-streamer/change-streamer-http.ts';\nimport {reserveAndGetSnapshotStatus} from '../services/change-streamer/snapshot.ts';\nimport {exitAfter, runUntilKilled} from '../services/life-cycle.ts';\nimport {\n tryRestore,\n type RestoreAttempt,\n type RestoreResult,\n} from '../services/litestream/commands.ts';\nimport {\n litestreamRestoreDuration,\n litestreamRestoreMetricAttrs,\n litestreamRestoreRuns,\n} from '../services/litestream/metrics.ts';\nimport {RestoreProgressReporter} from '../services/litestream/restore-progress.ts';\nimport {ReplicationStatusPublisher} from '../services/replicator/replication-status.ts';\nimport {\n ReplicatorService,\n type ReplicatorMode,\n} from '../services/replicator/replicator.ts';\nimport {sqliteFileBytes} from '../services/replicator/sqlite-change-log-observability.ts';\nimport {\n ThreadWriteWorkerClient,\n type ForceCheckpointConfig,\n} from '../services/replicator/write-worker-client.ts';\nimport {\n parentWorker,\n singleProcessMode,\n type Worker,\n} from '../types/processes.ts';\nimport {installProfileHandler} from '../types/profiler.ts';\nimport {getShardConfig} from '../types/shards.ts';\nimport {\n deleteStaleChangeLog,\n getPragmaConfig,\n replicaFileModeSchema,\n setUpMessageHandlers,\n setupReplica,\n type WalMode,\n} from '../workers/replicator.ts';\nimport {createLogContext} from './logging.ts';\nimport {startOtelAuto} from './otel-start.ts';\n\n// Default LogContext, overridden in runWorker\nlet lc = new LogContext('info', {}, consoleLogSink);\n\nexport default async function runWorker(\n parent: Worker,\n env: NodeJS.ProcessEnv,\n ...args: string[]\n): Promise<void> {\n assert(args.length > 0, `replicator mode not specified`);\n const fileMode = v.parse(args[0], replicaFileModeSchema);\n\n const config = getNormalizedZeroConfig({env, argv: args.slice(1)});\n const mode: ReplicatorMode = fileMode === 'backup' ? 'backup' : 'serving';\n const runningLocalChangeStreamer =\n config.changeStreamer.mode === 'dedicated' && !config.changeStreamer.uri;\n const workerName = `${mode}-replicator`;\n installProfileHandler(parent, workerName);\n startOtelAuto(createLogContext(config, workerName, 0, false), workerName, 0);\n lc = createLogContext(config, workerName);\n const unregisterInitialCorruptionDiagnosticTarget =\n registerSQLiteCorruptionDiagnosticTarget(\n {\n debugName: `${workerName} replica`,\n dbPath: config.replica.file,\n },\n config.sqliteCorruptionChecks,\n );\n initEventSink(lc, config);\n\n const shard = getShardConfig(config);\n const {\n taskID,\n change,\n changeStreamer: {\n port,\n uri: changeStreamerURI = runningLocalChangeStreamer\n ? `http://localhost:${port}/`\n : undefined,\n },\n } = config;\n const changeStreamer = new ChangeStreamerHttpClient(\n lc,\n shard,\n change.db,\n changeStreamerURI,\n );\n\n // Remote view-syncers restore from the replication-manager, which supplies\n // the backup URL. A local change-streamer owns the canonical replica and\n // must not infer backup configuration from bundled executable paths.\n //\n // The reservation's connection (`initialConnection`) is threaded through to\n // IncrementalSyncer's first subscribe(), so the replication-manager that\n // reserved the change log for this restore is the one that serves the\n // subscription (single-connection handoff).\n let initialConnection: ReservationFollowup | undefined;\n if (\n fileMode === 'serving' &&\n !runningLocalChangeStreamer &&\n (config.litestream.executable || config.litestream.executableV5)\n ) {\n initialConnection = await restoreReplica(lc, config, changeStreamer);\n }\n\n const {\n file: dbPath,\n walMode,\n pageSize,\n } = await setupReplica(lc, fileMode, config.replica);\n unregisterInitialCorruptionDiagnosticTarget();\n registerSQLiteCorruptionDiagnosticTarget(\n {\n debugName: `${workerName} replica`,\n dbPath,\n },\n config.sqliteCorruptionChecks,\n );\n\n // A file left behind by a run with the writer enabled would otherwise hold\n // local disk indefinitely. See `replicatorDeletesStaleChangeLog` for why\n // this is keyed on the config flag and nothing else.\n deleteStaleChangeLog(config.changeStreamer.sqliteChangeLogMode, dbPath);\n\n setupMetrics(lc, dbPath, walMode);\n\n // Create the write worker for async SQLite writes. Enable write-path\n // checkpoint backpressure only for the backup replicator on litestream v5,\n // which owns the replica litestream backs up.\n const pragmas = getPragmaConfig(fileMode);\n const checkpoint: ForceCheckpointConfig | null =\n mode === 'backup' &&\n config.litestream.backupURL &&\n config.litestream.backupUsingV5 &&\n config.litestream.forceCheckpointThresholdMB\n ? {\n checkpointThresholdPages:\n (config.litestream.forceCheckpointThresholdMB * 1024 ** 2) /\n pageSize,\n maxWalPages: !config.litestream.maxWalSizeMB\n ? undefined\n : (config.litestream.maxWalSizeMB * 1024 ** 2) / pageSize,\n }\n : null;\n const workerClient = new ThreadWriteWorkerClient();\n await workerClient.init(dbPath, mode, pragmas, config.log, checkpoint);\n\n const replicator = new ReplicatorService(\n lc,\n taskID,\n `${workerName}-${pid}`,\n mode,\n changeStreamer,\n workerClient,\n runningLocalChangeStreamer\n ? // publish ReplicationStatusEvents from backup-replicator only\n ReplicationStatusPublisher.forReplicaFile(dbPath)\n : null,\n initialConnection,\n );\n\n setUpMessageHandlers(lc, replicator, parent);\n\n const running = runUntilKilled(lc, parent, replicator);\n\n // Signal readiness once the first ReplicaVersionReady notification is received.\n for await (const _ of replicator.subscribe()) {\n parent.send(['ready', {ready: true}]);\n break;\n }\n\n return running;\n}\n\nfunction setupMetrics(lc: LogContext, file: string, walMode: WalMode) {\n getOrCreateGauge('replica', 'db_size', {\n description:\n `The size of the replica's main db file, ` +\n `which does not include the wal file(s)`,\n unit: 'bytes',\n }).addCallback(observeFileSize(lc, file));\n\n getOrCreateGauge('replica', 'wal_size', {\n description: `The size of the replica's wal file`,\n unit: 'bytes',\n }).addCallback(observeFileSize(lc, `${file}-wal`));\n\n if (walMode === 'wal2') {\n getOrCreateGauge('replica', 'wal2_size', {\n description: `The size of the replica's wal2 file`,\n unit: 'bytes',\n }).addCallback(observeFileSize(lc, `${file}-wal2`));\n }\n\n // A whole-database total rather than a per-file one. Its counterpart,\n // `sqlite_change_log.file_bytes`, is emitted by the change-streamer, which is\n // the process that writes the log.\n getOrCreateGauge('replica', 'file_bytes', {\n description:\n `The replica's total on-disk footprint: its main db file plus its ` +\n `wal sidecars.`,\n unit: 'By',\n }).addCallback(observeSQLiteFileBytes(lc, file));\n}\n\nfunction observeFileSize(lc: LogContext, file: string): ObservableCallback {\n return async o => {\n try {\n const stats = await stat(file);\n o.observe(stats.size);\n } catch (e) {\n lc.warn?.(`unable to stat ${file} for size metrics`, e);\n }\n };\n}\n\nfunction observeSQLiteFileBytes(\n lc: LogContext,\n file: string,\n): ObservableCallback {\n return async o => o.observe(await sqliteFileBytes(lc, file));\n}\n\nconst RETRY_INTERVAL_MS = 3000;\n\n// View-syncers wait indefinitely for the replication-manager to publish a\n// restorable backup. On a fresh stack the first backup is not durable until\n// the initial sync completes and litestream uploads the initial snapshot,\n// which can take many minutes for a large replica. The platform's startup\n// probe budget (which scales with replica size) is the backstop, so\n// restoreReplica must not impose its own shorter cap and self-terminate\n// while the backup is still being produced.\n//\n// Returns the `followup` of the reservation whose backup was actually\n// restored, still open, so the caller can hand it to IncrementalSyncer's\n// first subscribe() via `followup.subscribe()`: the same replication-manager\n// that reserved the change log is the one that serves the subscription.\nasync function restoreReplica(\n lc: LogContext,\n config: NormalizedZeroConfig,\n changeStreamer: SnapshotReserver,\n): Promise<ReservationFollowup> {\n const start = performance.now();\n let backupURL: string | undefined;\n let result: RestoreResult | undefined;\n const progress = new RestoreProgressReporter(lc, config.replica.file);\n try {\n for (;;) {\n const {reserved, followup} = await reserveAndGetSnapshotStatus(\n lc,\n config.taskID,\n changeStreamer,\n );\n // The backupURL comes from the replication-manager's snapshot response.\n ({backupURL} = reserved);\n const litestream: LitestreamConfig = {...config.litestream, backupURL};\n progress.start(reserved.replicaSize);\n let attempt: RestoreAttempt;\n try {\n attempt = await tryRestore(\n lc,\n litestream,\n config.replica.file,\n reserved,\n 'view_syncer',\n followup.signal, // abort the restore if the reservation disconnects\n );\n } catch (e) {\n if (followup.signal.aborted) {\n lc.info?.(\n `reservation disconnected during restore. retrying`,\n String(e),\n );\n continue;\n }\n followup.cancel();\n throw e;\n } finally {\n progress.stop();\n }\n if (attempt.restored) {\n result = attempt.result;\n progress.done();\n return followup; // connection to be converted to a subscription\n }\n // This attempt's reservation is stale; a fresh one is made on retry.\n followup.cancel();\n lc.info?.(\n `replica not found. retrying in ${RETRY_INTERVAL_MS / 1000} seconds`,\n );\n await sleep(RETRY_INTERVAL_MS);\n }\n } finally {\n progress.stop();\n const attrs = litestreamRestoreMetricAttrs(\n config.litestream,\n 'view_syncer',\n backupURL,\n );\n const labels = {...attrs, result: result ?? 'error'};\n litestreamRestoreRuns().add(1, labels);\n litestreamRestoreDuration().recordMs(performance.now() - start, labels);\n }\n}\n\n// fork()\nif (!singleProcessMode()) {\n void exitAfter(\n () => lc,\n () => runWorker(must(parentWorker), process.env, ...process.argv.slice(2)),\n );\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;AA+DA,IAAI,KAAK,IAAI,WAAW,QAAQ,CAAC,GAAG,cAAc;AAElD,eAA8B,UAC5B,QACA,KACA,GAAG,MACY;CACf,OAAO,KAAK,SAAS,GAAG,+BAA+B;CACvD,MAAM,WAAW,MAAQ,KAAK,IAAI,qBAAqB;CAEvD,MAAM,SAAS,wBAAwB;EAAC;EAAK,MAAM,KAAK,MAAM,CAAC;CAAC,CAAC;CACjE,MAAM,OAAuB,aAAa,WAAW,WAAW;CAChE,MAAM,6BACJ,OAAO,eAAe,SAAS,eAAe,CAAC,OAAO,eAAe;CACvE,MAAM,aAAa,GAAG,KAAK;CAC3B,sBAAsB,QAAQ,UAAU;CACxC,cAAc,iBAAiB,QAAQ,YAAY,GAAG,KAAK,GAAG,YAAY,CAAC;CAC3E,KAAK,iBAAiB,QAAQ,UAAU;CACxC,MAAM,8CACJ,yCACE;EACE,WAAW,GAAG,WAAW;EACzB,QAAQ,OAAO,QAAQ;CACzB,GACA,OAAO,sBACT;CACF,cAAc,IAAI,MAAM;CAExB,MAAM,QAAQ,eAAe,MAAM;CACnC,MAAM,EACJ,QACA,QACA,gBAAgB,EACd,MACA,KAAK,oBAAoB,6BACrB,oBAAoB,KAAK,KACzB,KAAA,QAEJ;CACJ,MAAM,iBAAiB,IAAI,yBACzB,IACA,OACA,OAAO,IACP,iBACF;CAUA,IAAI;CACJ,IACE,aAAa,aACb,CAAC,+BACA,OAAO,WAAW,cAAc,OAAO,WAAW,eAEnD,oBAAoB,MAAM,eAAe,IAAI,QAAQ,cAAc;CAGrE,MAAM,EACJ,MAAM,QACN,SACA,aACE,MAAM,aAAa,IAAI,UAAU,OAAO,OAAO;CACnD,4CAA4C;CAC5C,yCACE;EACE,WAAW,GAAG,WAAW;EACzB;CACF,GACA,OAAO,sBACT;CAKA,qBAAqB,OAAO,eAAe,qBAAqB,MAAM;CAEtE,aAAa,IAAI,QAAQ,OAAO;CAKhC,MAAM,UAAU,gBAAgB,QAAQ;CACxC,MAAM,aACJ,SAAS,YACT,OAAO,WAAW,aAClB,OAAO,WAAW,iBAClB,OAAO,WAAW,6BACd;EACE,0BACG,OAAO,WAAW,6BAA6B,QAAQ,IACxD;EACF,aAAa,CAAC,OAAO,WAAW,eAC5B,KAAA,IACC,OAAO,WAAW,eAAe,QAAQ,IAAK;CACrD,IACA;CACN,MAAM,eAAe,IAAI,wBAAwB;CACjD,MAAM,aAAa,KAAK,QAAQ,MAAM,SAAS,OAAO,KAAK,UAAU;CAErE,MAAM,aAAa,IAAI,kBACrB,IACA,QACA,GAAG,WAAW,GAAG,OACjB,MACA,gBACA,cACA,6BAEI,2BAA2B,eAAe,MAAM,IAChD,MACJ,iBACF;CAEA,qBAAqB,IAAI,YAAY,MAAM;CAE3C,MAAM,UAAU,eAAe,IAAI,QAAQ,UAAU;CAGrD,WAAW,MAAM,KAAK,WAAW,UAAU,GAAG;EAC5C,OAAO,KAAK,CAAC,SAAS,EAAC,OAAO,KAAI,CAAC,CAAC;EACpC;CACF;CAEA,OAAO;AACT;AAEA,SAAS,aAAa,IAAgB,MAAc,SAAkB;CACpE,iBAAiB,WAAW,WAAW;EACrC,aACE;EAEF,MAAM;CACR,CAAC,EAAE,YAAY,gBAAgB,IAAI,IAAI,CAAC;CAExC,iBAAiB,WAAW,YAAY;EACtC,aAAa;EACb,MAAM;CACR,CAAC,EAAE,YAAY,gBAAgB,IAAI,GAAG,KAAK,KAAK,CAAC;CAEjD,IAAI,YAAY,QACd,iBAAiB,WAAW,aAAa;EACvC,aAAa;EACb,MAAM;CACR,CAAC,EAAE,YAAY,gBAAgB,IAAI,GAAG,KAAK,MAAM,CAAC;CAMpD,iBAAiB,WAAW,cAAc;EACxC,aACE;EAEF,MAAM;CACR,CAAC,EAAE,YAAY,uBAAuB,IAAI,IAAI,CAAC;AACjD;AAEA,SAAS,gBAAgB,IAAgB,MAAkC;CACzE,OAAO,OAAM,MAAK;EAChB,IAAI;GACF,MAAM,QAAQ,MAAM,KAAK,IAAI;GAC7B,EAAE,QAAQ,MAAM,IAAI;EACtB,SAAS,GAAG;GACV,GAAG,OAAO,kBAAkB,KAAK,oBAAoB,CAAC;EACxD;CACF;AACF;AAEA,SAAS,uBACP,IACA,MACoB;CACpB,OAAO,OAAM,MAAK,EAAE,QAAQ,MAAM,gBAAgB,IAAI,IAAI,CAAC;AAC7D;AAEA,IAAM,oBAAoB;AAc1B,eAAe,eACb,IACA,QACA,gBAC8B;CAC9B,MAAM,QAAQ,YAAY,IAAI;CAC9B,IAAI;CACJ,IAAI;CACJ,MAAM,WAAW,IAAI,wBAAwB,IAAI,OAAO,QAAQ,IAAI;CACpE,IAAI;EACF,SAAS;GACP,MAAM,EAAC,UAAU,aAAY,MAAM,4BACjC,IACA,OAAO,QACP,cACF;GAEA,CAAC,CAAC,aAAa;GACf,MAAM,aAA+B;IAAC,GAAG,OAAO;IAAY;GAAS;GACrE,SAAS,MAAM,SAAS,WAAW;GACnC,IAAI;GACJ,IAAI;IACF,UAAU,MAAM,WACd,IACA,YACA,OAAO,QAAQ,MACf,UACA,eACA,SAAS,MACX;GACF,SAAS,GAAG;IACV,IAAI,SAAS,OAAO,SAAS;KAC3B,GAAG,OACD,qDACA,OAAO,CAAC,CACV;KACA;IACF;IACA,SAAS,OAAO;IAChB,MAAM;GACR,UAAU;IACR,SAAS,KAAK;GAChB;GACA,IAAI,QAAQ,UAAU;IACpB,SAAS,QAAQ;IACjB,SAAS,KAAK;IACd,OAAO;GACT;GAEA,SAAS,OAAO;GAChB,GAAG,OACD,kCAAkC,oBAAoB,IAAK,SAC7D;GACA,MAAM,MAAM,iBAAiB;EAC/B;CACF,UAAU;EACR,SAAS,KAAK;EAMd,MAAM,SAAS;GAAC,GALF,6BACZ,OAAO,YACP,eACA,SAEiB;GAAO,QAAQ,UAAU;EAAO;EACnD,sBAAsB,EAAE,IAAI,GAAG,MAAM;EACrC,0BAA0B,EAAE,SAAS,YAAY,IAAI,IAAI,OAAO,MAAM;CACxE;AACF;AAGA,IAAI,CAAC,kBAAkB,GACrB,gBACQ,UACA,UAAU,KAAK,YAAY,GAAG,QAAQ,KAAK,GAAG,QAAQ,KAAK,MAAM,CAAC,CAAC,CAC3E"}
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
export type Assignment = {
|
|
2
|
+
readonly worker: number;
|
|
3
|
+
readonly generation: number;
|
|
4
|
+
};
|
|
5
|
+
export declare class SyncerAssigner {
|
|
6
|
+
#private;
|
|
7
|
+
constructor(taskID: string, numWorkers: number);
|
|
8
|
+
assign(clientGroupID: string): Assignment;
|
|
9
|
+
release(clientGroupID: string, workerIndex: number, generation?: number): void;
|
|
10
|
+
getWorkerLoad(workerIndex: number): number;
|
|
11
|
+
getAssignment(clientGroupID: string): number | undefined;
|
|
12
|
+
destroy(): void;
|
|
13
|
+
}
|
|
14
|
+
//# sourceMappingURL=syncer-assigner.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"syncer-assigner.d.ts","sourceRoot":"","sources":["../../../../../zero-cache/src/server/syncer-assigner.ts"],"names":[],"mappings":"AAEA,MAAM,MAAM,UAAU,GAAG;IACvB,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;CAC7B,CAAC;AAEF,qBAAa,cAAc;;IAOzB,YAAY,MAAM,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,EAI7C;IAED,MAAM,CAAC,aAAa,EAAE,MAAM,GAAG,UAAU,CAoCxC;IAED,OAAO,CACL,aAAa,EAAE,MAAM,EACrB,WAAW,EAAE,MAAM,EACnB,UAAU,CAAC,EAAE,MAAM,GAClB,IAAI,CAUN;IAED,aAAa,CAAC,WAAW,EAAE,MAAM,GAAG,MAAM,CAEzC;IAED,aAAa,CAAC,aAAa,EAAE,MAAM,GAAG,MAAM,GAAG,SAAS,CAEvD;IAQD,OAAO,IAAI,IAAI,CAGd;CACF"}
|
|
@@ -0,0 +1,72 @@
|
|
|
1
|
+
import { h32 } from "../../../shared/src/hash.js";
|
|
2
|
+
//#region ../zero-cache/src/server/syncer-assigner.ts
|
|
3
|
+
var SyncerAssigner = class {
|
|
4
|
+
#taskID;
|
|
5
|
+
#numWorkers;
|
|
6
|
+
#assignments = /* @__PURE__ */ new Map();
|
|
7
|
+
#workerLoads;
|
|
8
|
+
#nextGeneration = 0;
|
|
9
|
+
constructor(taskID, numWorkers) {
|
|
10
|
+
this.#taskID = taskID;
|
|
11
|
+
this.#numWorkers = numWorkers;
|
|
12
|
+
this.#workerLoads = new Array(numWorkers).fill(0);
|
|
13
|
+
}
|
|
14
|
+
assign(clientGroupID) {
|
|
15
|
+
if (this.#numWorkers <= 0) return {
|
|
16
|
+
worker: 0,
|
|
17
|
+
generation: 0
|
|
18
|
+
};
|
|
19
|
+
const generation = ++this.#nextGeneration;
|
|
20
|
+
const existing = this.#assignments.get(clientGroupID);
|
|
21
|
+
if (existing !== void 0) {
|
|
22
|
+
const updated = {
|
|
23
|
+
worker: existing.worker,
|
|
24
|
+
generation
|
|
25
|
+
};
|
|
26
|
+
this.#assignments.set(clientGroupID, updated);
|
|
27
|
+
return updated;
|
|
28
|
+
}
|
|
29
|
+
let minLoad = Infinity;
|
|
30
|
+
const tied = [];
|
|
31
|
+
for (let i = 0; i < this.#numWorkers; i++) {
|
|
32
|
+
const load = this.#workerLoads[i];
|
|
33
|
+
if (load < minLoad) {
|
|
34
|
+
minLoad = load;
|
|
35
|
+
tied.length = 0;
|
|
36
|
+
tied.push(i);
|
|
37
|
+
} else if (load === minLoad) tied.push(i);
|
|
38
|
+
}
|
|
39
|
+
const chosen = tied.length === 1 ? tied[0] : tied[h32(this.#taskID + "/" + clientGroupID) % tied.length];
|
|
40
|
+
this.#workerLoads[chosen]++;
|
|
41
|
+
const assignment = {
|
|
42
|
+
worker: chosen,
|
|
43
|
+
generation
|
|
44
|
+
};
|
|
45
|
+
this.#assignments.set(clientGroupID, assignment);
|
|
46
|
+
return assignment;
|
|
47
|
+
}
|
|
48
|
+
release(clientGroupID, workerIndex, generation) {
|
|
49
|
+
const existing = this.#assignments.get(clientGroupID);
|
|
50
|
+
if (existing === void 0 || existing.worker !== workerIndex) return;
|
|
51
|
+
if (generation !== void 0 && existing.generation !== generation) return;
|
|
52
|
+
this.#decrementLoad(workerIndex);
|
|
53
|
+
this.#assignments.delete(clientGroupID);
|
|
54
|
+
}
|
|
55
|
+
getWorkerLoad(workerIndex) {
|
|
56
|
+
return this.#workerLoads[workerIndex] ?? 0;
|
|
57
|
+
}
|
|
58
|
+
getAssignment(clientGroupID) {
|
|
59
|
+
return this.#assignments.get(clientGroupID)?.worker;
|
|
60
|
+
}
|
|
61
|
+
#decrementLoad(worker) {
|
|
62
|
+
if (this.#workerLoads[worker] > 0) this.#workerLoads[worker]--;
|
|
63
|
+
}
|
|
64
|
+
destroy() {
|
|
65
|
+
this.#assignments.clear();
|
|
66
|
+
this.#workerLoads.fill(0);
|
|
67
|
+
}
|
|
68
|
+
};
|
|
69
|
+
//#endregion
|
|
70
|
+
export { SyncerAssigner };
|
|
71
|
+
|
|
72
|
+
//# sourceMappingURL=syncer-assigner.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"syncer-assigner.js","names":["#taskID","#numWorkers","#assignments","#workerLoads","#nextGeneration","#decrementLoad"],"sources":["../../../../../zero-cache/src/server/syncer-assigner.ts"],"sourcesContent":["import {h32} from '../../../shared/src/hash.ts';\n\nexport type Assignment = {\n readonly worker: number;\n readonly generation: number;\n};\n\nexport class SyncerAssigner {\n readonly #taskID: string;\n readonly #numWorkers: number;\n readonly #assignments = new Map<string, Assignment>();\n readonly #workerLoads: number[];\n #nextGeneration = 0;\n\n constructor(taskID: string, numWorkers: number) {\n this.#taskID = taskID;\n this.#numWorkers = numWorkers;\n this.#workerLoads = new Array<number>(numWorkers).fill(0);\n }\n\n assign(clientGroupID: string): Assignment {\n if (this.#numWorkers <= 0) {\n return {worker: 0, generation: 0};\n }\n\n const generation = ++this.#nextGeneration;\n const existing = this.#assignments.get(clientGroupID);\n if (existing !== undefined) {\n const updated: Assignment = {worker: existing.worker, generation};\n this.#assignments.set(clientGroupID, updated);\n return updated;\n }\n\n // Find worker(s) with minimum active load\n let minLoad = Infinity;\n const tied: number[] = [];\n for (let i = 0; i < this.#numWorkers; i++) {\n const load = this.#workerLoads[i];\n if (load < minLoad) {\n minLoad = load;\n tied.length = 0;\n tied.push(i);\n } else if (load === minLoad) {\n tied.push(i);\n }\n }\n\n const chosen =\n tied.length === 1\n ? tied[0]\n : tied[h32(this.#taskID + '/' + clientGroupID) % tied.length];\n\n this.#workerLoads[chosen]++;\n const assignment: Assignment = {worker: chosen, generation};\n this.#assignments.set(clientGroupID, assignment);\n return assignment;\n }\n\n release(\n clientGroupID: string,\n workerIndex: number,\n generation?: number,\n ): void {\n const existing = this.#assignments.get(clientGroupID);\n if (existing === undefined || existing.worker !== workerIndex) {\n return;\n }\n if (generation !== undefined && existing.generation !== generation) {\n return;\n }\n this.#decrementLoad(workerIndex);\n this.#assignments.delete(clientGroupID);\n }\n\n getWorkerLoad(workerIndex: number): number {\n return this.#workerLoads[workerIndex] ?? 0;\n }\n\n getAssignment(clientGroupID: string): number | undefined {\n return this.#assignments.get(clientGroupID)?.worker;\n }\n\n #decrementLoad(worker: number): void {\n if (this.#workerLoads[worker] > 0) {\n this.#workerLoads[worker]--;\n }\n }\n\n destroy(): void {\n this.#assignments.clear();\n this.#workerLoads.fill(0);\n }\n}\n"],"mappings":";;AAOA,IAAa,iBAAb,MAA4B;CAC1B;CACA;CACA,+BAAwB,IAAI,IAAwB;CACpD;CACA,kBAAkB;CAElB,YAAY,QAAgB,YAAoB;EAC9C,KAAKA,UAAU;EACf,KAAKC,cAAc;EACnB,KAAKE,eAAe,IAAI,MAAc,UAAU,EAAE,KAAK,CAAC;CAC1D;CAEA,OAAO,eAAmC;EACxC,IAAI,KAAKF,eAAe,GACtB,OAAO;GAAC,QAAQ;GAAG,YAAY;EAAC;EAGlC,MAAM,aAAa,EAAE,KAAKG;EAC1B,MAAM,WAAW,KAAKF,aAAa,IAAI,aAAa;EACpD,IAAI,aAAa,KAAA,GAAW;GAC1B,MAAM,UAAsB;IAAC,QAAQ,SAAS;IAAQ;GAAU;GAChE,KAAKA,aAAa,IAAI,eAAe,OAAO;GAC5C,OAAO;EACT;EAGA,IAAI,UAAU;EACd,MAAM,OAAiB,CAAC;EACxB,KAAK,IAAI,IAAI,GAAG,IAAI,KAAKD,aAAa,KAAK;GACzC,MAAM,OAAO,KAAKE,aAAa;GAC/B,IAAI,OAAO,SAAS;IAClB,UAAU;IACV,KAAK,SAAS;IACd,KAAK,KAAK,CAAC;GACb,OAAO,IAAI,SAAS,SAClB,KAAK,KAAK,CAAC;EAEf;EAEA,MAAM,SACJ,KAAK,WAAW,IACZ,KAAK,KACL,KAAK,IAAI,KAAKH,UAAU,MAAM,aAAa,IAAI,KAAK;EAE1D,KAAKG,aAAa;EAClB,MAAM,aAAyB;GAAC,QAAQ;GAAQ;EAAU;EAC1D,KAAKD,aAAa,IAAI,eAAe,UAAU;EAC/C,OAAO;CACT;CAEA,QACE,eACA,aACA,YACM;EACN,MAAM,WAAW,KAAKA,aAAa,IAAI,aAAa;EACpD,IAAI,aAAa,KAAA,KAAa,SAAS,WAAW,aAChD;EAEF,IAAI,eAAe,KAAA,KAAa,SAAS,eAAe,YACtD;EAEF,KAAKG,eAAe,WAAW;EAC/B,KAAKH,aAAa,OAAO,aAAa;CACxC;CAEA,cAAc,aAA6B;EACzC,OAAO,KAAKC,aAAa,gBAAgB;CAC3C;CAEA,cAAc,eAA2C;EACvD,OAAO,KAAKD,aAAa,IAAI,aAAa,GAAG;CAC/C;CAEA,eAAe,QAAsB;EACnC,IAAI,KAAKC,aAAa,UAAU,GAC9B,KAAKA,aAAa;CAEtB;CAEA,UAAgB;EACd,KAAKD,aAAa,MAAM;EACxB,KAAKC,aAAa,KAAK,CAAC;CAC1B;AACF"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"syncer.d.ts","sourceRoot":"","sources":["../../../../../zero-cache/src/server/syncer.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"syncer.d.ts","sourceRoot":"","sources":["../../../../../zero-cache/src/server/syncer.ts"],"names":[],"mappings":"AAkCA,OAAO,EAGL,KAAK,MAAM,EACZ,MAAM,uBAAuB,CAAC;AAqC/B,wBAA8B,SAAS,CACrC,MAAM,EAAE,MAAM,EACd,GAAG,EAAE,MAAM,CAAC,UAAU,EACtB,GAAG,IAAI,EAAE,MAAM,EAAE,GAChB,OAAO,CAAC,IAAI,CAAC,CAgOf"}
|
|
@@ -25,6 +25,7 @@ import { CustomQueryTransformer } from "../custom-queries/transform-query.js";
|
|
|
25
25
|
import { MutagenService } from "../services/mutagen/mutagen.js";
|
|
26
26
|
import { PusherService } from "../services/mutagen/pusher.js";
|
|
27
27
|
import { ConnectionContextManagerImpl } from "../services/view-syncer/connection-context-manager.js";
|
|
28
|
+
import { SnapshotRowCache } from "../services/view-syncer/snapshot-row-cache.js";
|
|
28
29
|
import { ViewSyncerService } from "../services/view-syncer/view-syncer.js";
|
|
29
30
|
import { Syncer } from "../workers/syncer.js";
|
|
30
31
|
import { InspectorDelegate } from "./inspector-delegate.js";
|
|
@@ -100,6 +101,7 @@ async function runWorker(parent, env, ...args) {
|
|
|
100
101
|
})
|
|
101
102
|
};
|
|
102
103
|
};
|
|
104
|
+
const snapshotRowCache = config.snapshotRowCacheSize > 0 ? new SnapshotRowCache(config.snapshotRowCacheSize) : void 0;
|
|
103
105
|
const viewSyncerFactory = (id, sub, drainCoordinator) => {
|
|
104
106
|
const logger = lc.withContext("taskID", config.taskID).withContext("component", "view-syncer").withContext("appID", shard.appID).withContext("shardNum", shard.shardNum).withContext("clientGroupID", id).withContext("instance", randomID());
|
|
105
107
|
const customQueryTransformer = customQueryConfig && new CustomQueryTransformer(logger, shard);
|
|
@@ -108,7 +110,7 @@ async function runWorker(parent, env, ...args) {
|
|
|
108
110
|
const inspectorDelegate = new InspectorDelegate(customQueryTransformer);
|
|
109
111
|
const priorityOpRunningYieldThresholdMs = Math.max(config.yieldThresholdMs / 4, 2);
|
|
110
112
|
const normalYieldThresholdMs = Math.max(config.yieldThresholdMs, 2);
|
|
111
|
-
return new ViewSyncerService(config, logger, shard, config.taskID, id, cvrDB, new PipelineDriver(logger, config.log, new Snapshotter(logger, replicaFile, shard), shard, operatorStorage.createClientGroupStorage(id), id, inspectorDelegate, () => isPriorityOpRunning() ? priorityOpRunningYieldThresholdMs : normalYieldThresholdMs, config.enableQueryPlanner, config), sub, drainCoordinator, config.log.slowHydrateThreshold, inspectorDelegate, connContextManager, customQueryTransformer, runPriorityOp);
|
|
113
|
+
return new ViewSyncerService(config, logger, shard, config.taskID, id, cvrDB, new PipelineDriver(logger, config.log, new Snapshotter(logger, replicaFile, shard, void 0, snapshotRowCache), shard, operatorStorage.createClientGroupStorage(id), id, inspectorDelegate, () => isPriorityOpRunning() ? priorityOpRunningYieldThresholdMs : normalYieldThresholdMs, config.enableQueryPlanner, config), sub, drainCoordinator, config.log.slowHydrateThreshold, inspectorDelegate, connContextManager, customQueryTransformer, runPriorityOp);
|
|
112
114
|
};
|
|
113
115
|
const syncer = new Syncer(lc, config, viewSyncerFactory, upstreamDB ? (id) => new MutagenService(lc.withContext("component", "mutagen").withContext("clientGroupID", id), shard, id, upstreamDB, config, writeAuthzStorage) : void 0, pushConfig === void 0 ? void 0 : (id, connContextManager) => new PusherService(config, lc.withContext("clientGroupID", id), id, connContextManager), parent, validateLegacyJWT);
|
|
114
116
|
startAnonymousTelemetry(lc, config);
|