@rocicorp/zero 1.10.0-canary.22 → 1.10.0-canary.24
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/out/analyze-query/src/analyze-cli.js +2 -2
- package/out/analyze-query/src/analyze-cli.js.map +1 -1
- package/out/analyze-query/src/bin-analyze.js +9 -8
- package/out/analyze-query/src/bin-analyze.js.map +1 -1
- package/out/replicache/src/btree/node.d.ts.map +1 -1
- package/out/replicache/src/btree/node.js +3 -6
- package/out/replicache/src/btree/node.js.map +1 -1
- package/out/replicache/src/dag/gc.d.ts.map +1 -1
- package/out/replicache/src/dag/gc.js +6 -10
- package/out/replicache/src/dag/gc.js.map +1 -1
- package/out/replicache/src/kv/sqlite-store.d.ts.map +1 -1
- package/out/replicache/src/kv/sqlite-store.js +24 -2
- package/out/replicache/src/kv/sqlite-store.js.map +1 -1
- package/out/replicache/src/persist/persist.js +1 -1
- package/out/shared/src/arrays.d.ts +5 -0
- package/out/shared/src/arrays.d.ts.map +1 -1
- package/out/shared/src/arrays.js +8 -10
- package/out/shared/src/arrays.js.map +1 -1
- package/out/shared/src/map.d.ts +19 -2
- package/out/shared/src/map.d.ts.map +1 -1
- package/out/shared/src/map.js +18 -4
- package/out/shared/src/map.js.map +1 -1
- package/out/z2s/src/compiler.js +1 -1
- package/out/z2s/src/sql.d.ts.map +1 -1
- package/out/z2s/src/sql.js +2 -2
- package/out/z2s/src/sql.js.map +1 -1
- package/out/zero/package.js +1 -1
- package/out/zero/package.js.map +1 -1
- package/out/zero-cache/src/auth/write-authorizer.d.ts.map +1 -1
- package/out/zero-cache/src/auth/write-authorizer.js +1 -0
- package/out/zero-cache/src/auth/write-authorizer.js.map +1 -1
- package/out/zero-cache/src/config/zero-config.d.ts +23 -0
- package/out/zero-cache/src/config/zero-config.d.ts.map +1 -1
- package/out/zero-cache/src/config/zero-config.js +63 -1
- package/out/zero-cache/src/config/zero-config.js.map +1 -1
- package/out/zero-cache/src/db/lite-tables.d.ts.map +1 -1
- package/out/zero-cache/src/db/lite-tables.js +3 -4
- package/out/zero-cache/src/db/lite-tables.js.map +1 -1
- package/out/zero-cache/src/observability/metrics.d.ts.map +1 -1
- package/out/zero-cache/src/observability/metrics.js +2 -7
- package/out/zero-cache/src/observability/metrics.js.map +1 -1
- package/out/zero-cache/src/server/change-streamer.d.ts.map +1 -1
- package/out/zero-cache/src/server/change-streamer.js +11 -3
- package/out/zero-cache/src/server/change-streamer.js.map +1 -1
- package/out/zero-cache/src/server/inspector-delegate.d.ts +18 -2
- package/out/zero-cache/src/server/inspector-delegate.d.ts.map +1 -1
- package/out/zero-cache/src/server/inspector-delegate.js +22 -15
- package/out/zero-cache/src/server/inspector-delegate.js.map +1 -1
- package/out/zero-cache/src/server/main.d.ts.map +1 -1
- package/out/zero-cache/src/server/main.js +8 -1
- package/out/zero-cache/src/server/main.js.map +1 -1
- package/out/zero-cache/src/server/mutator.js +2 -0
- package/out/zero-cache/src/server/mutator.js.map +1 -1
- package/out/zero-cache/src/server/reaper.d.ts.map +1 -1
- package/out/zero-cache/src/server/reaper.js +2 -0
- package/out/zero-cache/src/server/reaper.js.map +1 -1
- package/out/zero-cache/src/server/replicator.d.ts.map +1 -1
- package/out/zero-cache/src/server/replicator.js +2 -0
- package/out/zero-cache/src/server/replicator.js.map +1 -1
- package/out/zero-cache/src/server/runner/zero-dispatcher.d.ts.map +1 -1
- package/out/zero-cache/src/server/runner/zero-dispatcher.js +3 -0
- package/out/zero-cache/src/server/runner/zero-dispatcher.js.map +1 -1
- package/out/zero-cache/src/server/syncer.d.ts.map +1 -1
- package/out/zero-cache/src/server/syncer.js +5 -1
- package/out/zero-cache/src/server/syncer.js.map +1 -1
- package/out/zero-cache/src/services/analyze.d.ts.map +1 -1
- package/out/zero-cache/src/services/analyze.js +11 -9
- package/out/zero-cache/src/services/analyze.js.map +1 -1
- package/out/zero-cache/src/services/change-source/common/backfill-manager.d.ts.map +1 -1
- package/out/zero-cache/src/services/change-source/common/backfill-manager.js +14 -1
- package/out/zero-cache/src/services/change-source/common/backfill-manager.js.map +1 -1
- package/out/zero-cache/src/services/change-source/common/change-stream-multiplexer.d.ts.map +1 -1
- package/out/zero-cache/src/services/change-source/common/change-stream-multiplexer.js +15 -3
- package/out/zero-cache/src/services/change-source/common/change-stream-multiplexer.js.map +1 -1
- package/out/zero-cache/src/services/change-source/custom/change-source.js +2 -2
- package/out/zero-cache/src/services/change-source/custom/change-source.js.map +1 -1
- package/out/zero-cache/src/services/change-source/pg/initial-sync.js +1 -1
- package/out/zero-cache/src/services/change-streamer/change-streamer-http.d.ts +3 -0
- package/out/zero-cache/src/services/change-streamer/change-streamer-http.d.ts.map +1 -1
- package/out/zero-cache/src/services/change-streamer/change-streamer-http.js +6 -4
- package/out/zero-cache/src/services/change-streamer/change-streamer-http.js.map +1 -1
- package/out/zero-cache/src/services/mutagen/mutagen.d.ts +5 -0
- package/out/zero-cache/src/services/mutagen/mutagen.d.ts.map +1 -1
- package/out/zero-cache/src/services/mutagen/mutagen.js +7 -1
- package/out/zero-cache/src/services/mutagen/mutagen.js.map +1 -1
- package/out/zero-cache/src/services/mutagen/pusher.d.ts +32 -32
- package/out/zero-cache/src/services/mutagen/pusher.d.ts.map +1 -1
- package/out/zero-cache/src/services/mutagen/pusher.js +5 -6
- package/out/zero-cache/src/services/mutagen/pusher.js.map +1 -1
- package/out/zero-cache/src/services/profz.d.ts +7 -0
- package/out/zero-cache/src/services/profz.d.ts.map +1 -0
- package/out/zero-cache/src/services/profz.js +103 -0
- package/out/zero-cache/src/services/profz.js.map +1 -0
- package/out/zero-cache/src/services/run-ast.d.ts +12 -0
- package/out/zero-cache/src/services/run-ast.d.ts.map +1 -1
- package/out/zero-cache/src/services/run-ast.js +26 -14
- package/out/zero-cache/src/services/run-ast.js.map +1 -1
- package/out/zero-cache/src/services/runner.d.ts.map +1 -1
- package/out/zero-cache/src/services/runner.js +1 -1
- package/out/zero-cache/src/services/runner.js.map +1 -1
- package/out/zero-cache/src/services/view-syncer/cvr.d.ts +45 -0
- package/out/zero-cache/src/services/view-syncer/cvr.d.ts.map +1 -1
- package/out/zero-cache/src/services/view-syncer/cvr.js +133 -1
- package/out/zero-cache/src/services/view-syncer/cvr.js.map +1 -1
- package/out/zero-cache/src/services/view-syncer/drain-coordinator.d.ts +14 -1
- package/out/zero-cache/src/services/view-syncer/drain-coordinator.d.ts.map +1 -1
- package/out/zero-cache/src/services/view-syncer/drain-coordinator.js +30 -4
- package/out/zero-cache/src/services/view-syncer/drain-coordinator.js.map +1 -1
- package/out/zero-cache/src/services/view-syncer/hydration-budget.d.ts +33 -0
- package/out/zero-cache/src/services/view-syncer/hydration-budget.d.ts.map +1 -0
- package/out/zero-cache/src/services/view-syncer/hydration-budget.js +62 -0
- package/out/zero-cache/src/services/view-syncer/hydration-budget.js.map +1 -0
- package/out/zero-cache/src/services/view-syncer/hydration-circuit-breaker.d.ts +51 -0
- package/out/zero-cache/src/services/view-syncer/hydration-circuit-breaker.d.ts.map +1 -0
- package/out/zero-cache/src/services/view-syncer/hydration-circuit-breaker.js +78 -0
- package/out/zero-cache/src/services/view-syncer/hydration-circuit-breaker.js.map +1 -0
- package/out/zero-cache/src/services/view-syncer/inspect-handler.d.ts +1 -1
- package/out/zero-cache/src/services/view-syncer/inspect-handler.d.ts.map +1 -1
- package/out/zero-cache/src/services/view-syncer/inspect-handler.js +2 -2
- package/out/zero-cache/src/services/view-syncer/inspect-handler.js.map +1 -1
- package/out/zero-cache/src/services/view-syncer/pipeline-driver.d.ts +2 -1
- package/out/zero-cache/src/services/view-syncer/pipeline-driver.d.ts.map +1 -1
- package/out/zero-cache/src/services/view-syncer/pipeline-driver.js +42 -17
- package/out/zero-cache/src/services/view-syncer/pipeline-driver.js.map +1 -1
- package/out/zero-cache/src/services/view-syncer/query-covering.d.ts.map +1 -1
- package/out/zero-cache/src/services/view-syncer/query-covering.js +2 -6
- package/out/zero-cache/src/services/view-syncer/query-covering.js.map +1 -1
- package/out/zero-cache/src/services/view-syncer/snapshot-row-cache.d.ts +95 -0
- package/out/zero-cache/src/services/view-syncer/snapshot-row-cache.d.ts.map +1 -0
- package/out/zero-cache/src/services/view-syncer/snapshot-row-cache.js +155 -0
- package/out/zero-cache/src/services/view-syncer/snapshot-row-cache.js.map +1 -0
- package/out/zero-cache/src/services/view-syncer/snapshotter.d.ts +42 -4
- package/out/zero-cache/src/services/view-syncer/snapshotter.d.ts.map +1 -1
- package/out/zero-cache/src/services/view-syncer/snapshotter.js +117 -28
- package/out/zero-cache/src/services/view-syncer/snapshotter.js.map +1 -1
- package/out/zero-cache/src/services/view-syncer/view-syncer.d.ts +11 -1
- package/out/zero-cache/src/services/view-syncer/view-syncer.d.ts.map +1 -1
- package/out/zero-cache/src/services/view-syncer/view-syncer.js +466 -95
- package/out/zero-cache/src/services/view-syncer/view-syncer.js.map +1 -1
- package/out/zero-cache/src/types/processes.d.ts +32 -0
- package/out/zero-cache/src/types/processes.d.ts.map +1 -1
- package/out/zero-cache/src/types/processes.js +43 -4
- package/out/zero-cache/src/types/processes.js.map +1 -1
- package/out/zero-cache/src/types/profiler.d.ts +12 -0
- package/out/zero-cache/src/types/profiler.d.ts.map +1 -1
- package/out/zero-cache/src/types/profiler.js +46 -1
- package/out/zero-cache/src/types/profiler.js.map +1 -1
- package/out/zero-cache/src/types/row-key.d.ts.map +1 -1
- package/out/zero-cache/src/types/row-key.js +5 -5
- package/out/zero-cache/src/types/row-key.js.map +1 -1
- package/out/zero-cache/src/workers/connection.js +3 -3
- package/out/zero-cache/src/workers/connection.js.map +1 -1
- package/out/zero-cache/src/workers/syncer-ws-message-handler.d.ts.map +1 -1
- package/out/zero-cache/src/workers/syncer-ws-message-handler.js +21 -16
- package/out/zero-cache/src/workers/syncer-ws-message-handler.js.map +1 -1
- package/out/zero-cache/src/workers/syncer.d.ts.map +1 -1
- package/out/zero-cache/src/workers/syncer.js +10 -1
- package/out/zero-cache/src/workers/syncer.js.map +1 -1
- package/out/zero-client/src/client/context.d.ts +16 -3
- package/out/zero-client/src/client/context.d.ts.map +1 -1
- package/out/zero-client/src/client/context.js +85 -14
- package/out/zero-client/src/client/context.js.map +1 -1
- package/out/zero-client/src/client/ivm-branch.d.ts.map +1 -1
- package/out/zero-client/src/client/ivm-branch.js +20 -3
- package/out/zero-client/src/client/ivm-branch.js.map +1 -1
- package/out/zero-client/src/client/query-manager.d.ts.map +1 -1
- package/out/zero-client/src/client/query-manager.js +2 -5
- package/out/zero-client/src/client/query-manager.js.map +1 -1
- package/out/zero-client/src/client/version.js +1 -1
- package/out/zero-client/src/client/zero-rep.d.ts.map +1 -1
- package/out/zero-client/src/client/zero-rep.js +1 -1
- package/out/zero-client/src/client/zero-rep.js.map +1 -1
- package/out/zero-client/src/client/zero.d.ts.map +1 -1
- package/out/zero-client/src/client/zero.js +2 -2
- package/out/zero-client/src/client/zero.js.map +1 -1
- package/out/zero-protocol/src/analyze-query-result.d.ts +5 -5
- package/out/zero-protocol/src/analyze-query-result.d.ts.map +1 -1
- package/out/zero-protocol/src/analyze-query-result.js +1 -1
- package/out/zero-protocol/src/analyze-query-result.js.map +1 -1
- package/out/zero-protocol/src/ast.d.ts.map +1 -1
- package/out/zero-protocol/src/ast.js +13 -14
- package/out/zero-protocol/src/ast.js.map +1 -1
- package/out/zero-protocol/src/down.d.ts +2 -2
- package/out/zero-protocol/src/error.d.ts +1 -1
- package/out/zero-protocol/src/inspect-down.d.ts +6 -6
- package/out/zero-protocol/src/protocol-version.d.ts.map +1 -1
- package/out/zero-protocol/src/protocol-version.js.map +1 -1
- package/out/zero-protocol/src/query-hash.d.ts.map +1 -1
- package/out/zero-protocol/src/query-hash.js +2 -5
- package/out/zero-protocol/src/query-hash.js.map +1 -1
- package/out/zero-solid/src/solid-view.d.ts +7 -0
- package/out/zero-solid/src/solid-view.d.ts.map +1 -1
- package/out/zero-solid/src/solid-view.js +13 -2
- package/out/zero-solid/src/solid-view.js.map +1 -1
- package/out/zql/src/builder/builder.d.ts +19 -0
- package/out/zql/src/builder/builder.d.ts.map +1 -1
- package/out/zql/src/builder/builder.js +28 -11
- package/out/zql/src/builder/builder.js.map +1 -1
- package/out/zql/src/builder/correlated-predicate-pushdown.d.ts +43 -0
- package/out/zql/src/builder/correlated-predicate-pushdown.d.ts.map +1 -0
- package/out/zql/src/builder/correlated-predicate-pushdown.js +161 -0
- package/out/zql/src/builder/correlated-predicate-pushdown.js.map +1 -0
- package/out/zql/src/builder/debug-delegate.d.ts +6 -1
- package/out/zql/src/builder/debug-delegate.d.ts.map +1 -1
- package/out/zql/src/builder/debug-delegate.js +14 -2
- package/out/zql/src/builder/debug-delegate.js.map +1 -1
- package/out/zql/src/ivm/array-view.d.ts +19 -0
- package/out/zql/src/ivm/array-view.d.ts.map +1 -1
- package/out/zql/src/ivm/array-view.js +39 -2
- package/out/zql/src/ivm/array-view.js.map +1 -1
- package/out/zql/src/ivm/connection-index.d.ts +68 -0
- package/out/zql/src/ivm/connection-index.d.ts.map +1 -0
- package/out/zql/src/ivm/connection-index.js +149 -0
- package/out/zql/src/ivm/connection-index.js.map +1 -0
- package/out/zql/src/ivm/exists.js +1 -1
- package/out/zql/src/ivm/filter-operators.d.ts +3 -2
- package/out/zql/src/ivm/filter-operators.d.ts.map +1 -1
- package/out/zql/src/ivm/filter-operators.js +19 -4
- package/out/zql/src/ivm/filter-operators.js.map +1 -1
- package/out/zql/src/ivm/memory-source.d.ts +13 -1
- package/out/zql/src/ivm/memory-source.d.ts.map +1 -1
- package/out/zql/src/ivm/memory-source.js +43 -4
- package/out/zql/src/ivm/memory-source.js.map +1 -1
- package/out/zql/src/ivm/operator.d.ts +40 -0
- package/out/zql/src/ivm/operator.d.ts.map +1 -1
- package/out/zql/src/ivm/operator.js.map +1 -1
- package/out/zql/src/ivm/view.d.ts +15 -0
- package/out/zql/src/ivm/view.d.ts.map +1 -1
- package/out/zql/src/planner/planner-builder.d.ts +12 -3
- package/out/zql/src/planner/planner-builder.d.ts.map +1 -1
- package/out/zql/src/planner/planner-builder.js +40 -20
- package/out/zql/src/planner/planner-builder.js.map +1 -1
- package/out/zql/src/planner/planner-connection.d.ts +28 -2
- package/out/zql/src/planner/planner-connection.d.ts.map +1 -1
- package/out/zql/src/planner/planner-connection.js +108 -8
- package/out/zql/src/planner/planner-connection.js.map +1 -1
- package/out/zql/src/planner/planner-debug.d.ts +1 -1
- package/out/zql/src/planner/planner-debug.d.ts.map +1 -1
- package/out/zql/src/planner/planner-debug.js +3 -8
- package/out/zql/src/planner/planner-debug.js.map +1 -1
- package/out/zql/src/planner/planner-fan-out.d.ts +1 -0
- package/out/zql/src/planner/planner-fan-out.d.ts.map +1 -1
- package/out/zql/src/planner/planner-fan-out.js +3 -0
- package/out/zql/src/planner/planner-fan-out.js.map +1 -1
- package/out/zql/src/planner/planner-filter.d.ts +59 -0
- package/out/zql/src/planner/planner-filter.d.ts.map +1 -0
- package/out/zql/src/planner/planner-filter.js +116 -0
- package/out/zql/src/planner/planner-filter.js.map +1 -0
- package/out/zql/src/planner/planner-graph.d.ts +2 -0
- package/out/zql/src/planner/planner-graph.d.ts.map +1 -1
- package/out/zql/src/planner/planner-graph.js +2 -0
- package/out/zql/src/planner/planner-graph.js.map +1 -1
- package/out/zql/src/planner/planner-join.d.ts +2 -0
- package/out/zql/src/planner/planner-join.d.ts.map +1 -1
- package/out/zql/src/planner/planner-join.js +7 -0
- package/out/zql/src/planner/planner-join.js.map +1 -1
- package/out/zql/src/planner/planner-node.d.ts +2 -1
- package/out/zql/src/planner/planner-node.d.ts.map +1 -1
- package/out/zql/src/planner/planner-node.js.map +1 -1
- package/out/zql/src/planner/planner-source.d.ts +2 -2
- package/out/zql/src/planner/planner-source.d.ts.map +1 -1
- package/out/zql/src/planner/planner-source.js +2 -2
- package/out/zql/src/planner/planner-source.js.map +1 -1
- package/out/zql/src/query/query-delegate-base.d.ts +2 -2
- package/out/zql/src/query/query-delegate-base.d.ts.map +1 -1
- package/out/zql/src/query/query-delegate-base.js +16 -10
- package/out/zql/src/query/query-delegate-base.js.map +1 -1
- package/out/zql/src/query/query-delegate.d.ts +22 -5
- package/out/zql/src/query/query-delegate.d.ts.map +1 -1
- package/out/zql/src/query/query-impl.d.ts.map +1 -1
- package/out/zql/src/query/query-impl.js +6 -23
- package/out/zql/src/query/query-impl.js.map +1 -1
- package/out/zql/src/query/query-transitions.d.ts.map +1 -1
- package/out/zql/src/query/query-transitions.js +2 -5
- package/out/zql/src/query/query-transitions.js.map +1 -1
- package/out/zql/src/query/query.d.ts +4 -1
- package/out/zql/src/query/query.d.ts.map +1 -1
- package/out/zql/src/query/runnable-query-impl.d.ts.map +1 -1
- package/out/zql/src/query/runnable-query-impl.js +2 -10
- package/out/zql/src/query/runnable-query-impl.js.map +1 -1
- package/out/zqlite/src/db.d.ts +9 -1
- package/out/zqlite/src/db.d.ts.map +1 -1
- package/out/zqlite/src/db.js +8 -2
- package/out/zqlite/src/db.js.map +1 -1
- package/out/zqlite/src/query-builder.d.ts +1 -1
- package/out/zqlite/src/query-builder.d.ts.map +1 -1
- package/out/zqlite/src/query-builder.js +2 -1
- package/out/zqlite/src/query-builder.js.map +1 -1
- package/out/zqlite/src/query-delegate.d.ts.map +1 -1
- package/out/zqlite/src/query-delegate.js +5 -6
- package/out/zqlite/src/query-delegate.js.map +1 -1
- package/out/zqlite/src/sqlite-cost-model.d.ts.map +1 -1
- package/out/zqlite/src/sqlite-cost-model.js +2 -30
- package/out/zqlite/src/sqlite-cost-model.js.map +1 -1
- package/out/zqlite/src/sqlite-stat-fanout.d.ts.map +1 -1
- package/out/zqlite/src/sqlite-stat-fanout.js +3 -5
- package/out/zqlite/src/sqlite-stat-fanout.js.map +1 -1
- package/out/zqlite/src/table-source.d.ts +15 -2
- package/out/zqlite/src/table-source.d.ts.map +1 -1
- package/out/zqlite/src/table-source.js +28 -12
- package/out/zqlite/src/table-source.js.map +1 -1
- package/package.json +1 -1
|
@@ -1,4 +1,6 @@
|
|
|
1
1
|
import { assert, unreachable } from "../../../../shared/src/asserts.js";
|
|
2
|
+
import { getOrInsertComputed } from "../../../../shared/src/map.js";
|
|
3
|
+
import { newArray } from "../../../../shared/src/arrays.js";
|
|
2
4
|
import { must } from "../../../../shared/src/must.js";
|
|
3
5
|
import { h64 } from "../../../../shared/src/hash.js";
|
|
4
6
|
import { Internal, InvalidConnectionRequest, InvalidConnectionRequestBaseCookie, Rehome } from "../../../../zero-protocol/src/error-kind-enum.js";
|
|
@@ -11,12 +13,11 @@ import { manualSpan, startAsyncSpan, startSpan } from "../../../../otel/src/span
|
|
|
11
13
|
import { ZERO_VERSION_COLUMN_NAME } from "../replicator/schema/constants.js";
|
|
12
14
|
import { transformAndHashQuery } from "../../auth/read-authorizer.js";
|
|
13
15
|
import { randInt } from "../../../../shared/src/rand.js";
|
|
14
|
-
import { getOrCreateCounter, getOrCreateLatencyHistogram, getOrCreateNativeHistogram, getOrCreateUpDownCounter } from "../../observability/metrics.js";
|
|
16
|
+
import { getOrCreateCounter, getOrCreateLatencyHistogram, getOrCreateNativeHistogram, getOrCreateUpDownCounter, getOrCreateValueHistogram } from "../../observability/metrics.js";
|
|
15
17
|
import { rowIDString } from "../../types/row-key.js";
|
|
16
18
|
import "../replicator/schema/replication-state.js";
|
|
17
19
|
import { parseSignature } from "./row-set-signature.js";
|
|
18
20
|
import { ResetPipelinesSignal } from "./snapshotter.js";
|
|
19
|
-
import "./pipeline-driver.js";
|
|
20
21
|
import { Subscription } from "../../types/subscription.js";
|
|
21
22
|
import { CustomKeyMap } from "../../../../shared/src/custom-key-map.js";
|
|
22
23
|
import { ttlClockAsNumber, ttlClockFromNumber } from "./ttl-clock.js";
|
|
@@ -26,8 +27,10 @@ import { isAuthErrorBody } from "../../auth/auth.js";
|
|
|
26
27
|
import { ClientHandler, startPoke } from "./client-handler.js";
|
|
27
28
|
import { tracer } from "./tracer.js";
|
|
28
29
|
import { CVRStore, ClientNotFoundError } from "./cvr-store.js";
|
|
29
|
-
import { CVRConfigDrivenUpdater, CVRQueryDrivenUpdater, nextEvictionTime } from "./cvr.js";
|
|
30
|
+
import { CVRConfigDrivenUpdater, CVRQueryDrivenUpdater, classifyQueriesForHydration, nextEvictionTime } from "./cvr.js";
|
|
30
31
|
import { E2EServingLagTracker } from "./e2e-serving-lag.js";
|
|
32
|
+
import { HydrationBudget } from "./hydration-budget.js";
|
|
33
|
+
import { HydrationCircuitBreaker } from "./hydration-circuit-breaker.js";
|
|
31
34
|
import { handleInspect } from "./inspect-handler.js";
|
|
32
35
|
import { QueryCoveringIndex } from "./query-covering.js";
|
|
33
36
|
import { resolver } from "@rocicorp/resolver";
|
|
@@ -62,6 +65,7 @@ var ViewSyncerService = class {
|
|
|
62
65
|
#drainCoordinator;
|
|
63
66
|
#keepaliveMs;
|
|
64
67
|
#slowHydrateThreshold;
|
|
68
|
+
#hydrationCircuitBreaker;
|
|
65
69
|
#lastConnectTime = Date.now();
|
|
66
70
|
/**
|
|
67
71
|
* The TTL clock is used to determine the time at which queries are considered
|
|
@@ -89,14 +93,36 @@ var ViewSyncerService = class {
|
|
|
89
93
|
#lock = new Lock();
|
|
90
94
|
#cvrStore;
|
|
91
95
|
#stopped = resolver();
|
|
96
|
+
/**
|
|
97
|
+
* Set when {@link #cleanup} begins. Lock tasks that were in flight when the
|
|
98
|
+
* view-syncer was stopped may still complete after the timers have been
|
|
99
|
+
* cleared; this flag prevents them from scheduling new timers, which would
|
|
100
|
+
* otherwise outlive the service (and retain everything it references).
|
|
101
|
+
*/
|
|
102
|
+
#shuttingDown = false;
|
|
92
103
|
#initialized = resolver();
|
|
93
104
|
#cvr;
|
|
94
|
-
|
|
105
|
+
/**
|
|
106
|
+
* Indicates whether the query pipelines have completed initial catch-up and
|
|
107
|
+
* hydration with the CVR snapshot, and are ready for steady-state operation.
|
|
108
|
+
*
|
|
109
|
+
* - When `false`: The syncer is in its initial catch-up phase. The replica
|
|
110
|
+
* advances without diffs until it reaches `cvr.version.stateVersion`,
|
|
111
|
+
* after which `#maybeHydratePipelines` hydrates queries.
|
|
112
|
+
* - When `true`: Pipelines are fully populated and in steady-state; replica
|
|
113
|
+
* changes advance incrementally via `#advancePipelines`, and client query
|
|
114
|
+
* updates are processed via `#syncQueryPipelineSet`.
|
|
115
|
+
*
|
|
116
|
+
* Resets to `false` if `#advancePipelines` returns a `ResetPipelinesSignal`
|
|
117
|
+
* and pipelines must be reset and rehydrated.
|
|
118
|
+
*/
|
|
119
|
+
#pipelinesHydrated = false;
|
|
95
120
|
#servedVersion = null;
|
|
96
121
|
#e2eServingLagTracker = new E2EServingLagTracker();
|
|
97
122
|
#expiredQueriesTimer = 0;
|
|
98
123
|
#authMaintenanceTimer = 0;
|
|
99
124
|
#setTimeout;
|
|
125
|
+
#now;
|
|
100
126
|
#customQueryTransformer;
|
|
101
127
|
#queryReplacements = /* @__PURE__ */ new Map();
|
|
102
128
|
#activeClients = getOrCreateUpDownCounter("sync", "active-clients", "Number of active sync clients");
|
|
@@ -123,10 +149,64 @@ var ViewSyncerService = class {
|
|
|
123
149
|
#pipelineResets = getOrCreateCounter("sync", "pipeline-resets", "Number of pipeline resets");
|
|
124
150
|
#rowSetSignatureDrifts = getOrCreateCounter("sync", "query.row-set-signature-drifts", "Number of times re-hydration of an unchanged query produced a different row-set signature than what is stored in the CVR (forcing a configVersion bump and full re-execution). Expected to be near-zero in steady state; persistent non-zero values indicate non-deterministic query execution (e.g. Cap operator picking different N-row subsets).");
|
|
125
151
|
#sameHashRehydrationVersionBumps = getOrCreateCounter("sync", "query.same-hash-rehydrations-forced-bump", "Number of times query-set reconciliation forced a configVersion bump for already-gotten same-transformation-hash query rehydration because trackQueries would not otherwise bump. Expected to be near-zero; non-zero values indicate pipeline/CVR row-set drift reached query-set reconciliation.");
|
|
152
|
+
#hydrationBudgetExhaustions = getOrCreateCounter("sync", "hydration_budget_exhaustions", {
|
|
153
|
+
description: "Number of hydration passes that exhaust their budget.",
|
|
154
|
+
unit: "{pass}"
|
|
155
|
+
});
|
|
156
|
+
#hydrationBudgetEvictions = getOrCreateCounter("sync", "hydration_budget_evictions", {
|
|
157
|
+
description: "Number of inactive queries removed after hydration budget exhaustion.",
|
|
158
|
+
unit: "{query}"
|
|
159
|
+
});
|
|
160
|
+
#hydrationBudgetElapsed = getOrCreateValueHistogram("sync", "hydration_budget_elapsed", {
|
|
161
|
+
description: "Elapsed milliseconds when optional query hydration stopped.",
|
|
162
|
+
unit: "ms",
|
|
163
|
+
bucketBoundaries: [
|
|
164
|
+
1,
|
|
165
|
+
2,
|
|
166
|
+
5,
|
|
167
|
+
10,
|
|
168
|
+
20,
|
|
169
|
+
50,
|
|
170
|
+
100,
|
|
171
|
+
200,
|
|
172
|
+
500,
|
|
173
|
+
1e3,
|
|
174
|
+
5e3
|
|
175
|
+
]
|
|
176
|
+
});
|
|
177
|
+
#hydrationBudgetOvershoot = getOrCreateValueHistogram("sync", "hydration_budget_overshoot", {
|
|
178
|
+
description: "Milliseconds elapsed beyond the configured hydration budget.",
|
|
179
|
+
unit: "ms",
|
|
180
|
+
bucketBoundaries: [
|
|
181
|
+
0,
|
|
182
|
+
1,
|
|
183
|
+
2,
|
|
184
|
+
5,
|
|
185
|
+
10,
|
|
186
|
+
20,
|
|
187
|
+
50,
|
|
188
|
+
100,
|
|
189
|
+
200,
|
|
190
|
+
500,
|
|
191
|
+
1e3
|
|
192
|
+
]
|
|
193
|
+
});
|
|
194
|
+
#queryEvictions = getOrCreateCounter("sync", "query_evictions", {
|
|
195
|
+
description: "Number of queries evicted from the CVR ahead of their removal by the client, grouped by reason. Inactive queries are evicted by ttl and hydration-budget; hydration-timeout and hydration-circuit-breaker evict active queries as well.",
|
|
196
|
+
unit: "{query}"
|
|
197
|
+
});
|
|
198
|
+
#hydrationTimeouts = getOrCreateCounter("sync", "hydration_timeouts", {
|
|
199
|
+
description: "Number of query hydrations aborted for exceeding the query hydration timeout.",
|
|
200
|
+
unit: "{query}"
|
|
201
|
+
});
|
|
202
|
+
#hydrationCircuitBreakerRejections = getOrCreateCounter("sync", "hydration_circuit_breaker_rejections", {
|
|
203
|
+
description: "Number of queries rejected without hydration because their hydration circuit breaker was open.",
|
|
204
|
+
unit: "{query}"
|
|
205
|
+
});
|
|
126
206
|
#inspectorDelegate;
|
|
127
207
|
#config;
|
|
128
208
|
#runPriorityOp;
|
|
129
|
-
constructor(config, lc, shard, taskID, clientGroupID, cvrDb, pipelineDriver, versionChanges, drainCoordinator, slowHydrateThreshold, inspectorDelegate, connContextManager, customQueryTransformer, runPriorityOp, keepaliveMs = DEFAULT_KEEPALIVE_MS, setTimeoutFn = setTimeout.bind(globalThis)) {
|
|
209
|
+
constructor(config, lc, shard, taskID, clientGroupID, cvrDb, pipelineDriver, versionChanges, drainCoordinator, slowHydrateThreshold, inspectorDelegate, connContextManager, customQueryTransformer, runPriorityOp, keepaliveMs = DEFAULT_KEEPALIVE_MS, setTimeoutFn = setTimeout.bind(globalThis), now = performance.now.bind(performance)) {
|
|
130
210
|
this.#config = config;
|
|
131
211
|
this.id = clientGroupID;
|
|
132
212
|
this.connContextManager = connContextManager;
|
|
@@ -141,6 +221,8 @@ var ViewSyncerService = class {
|
|
|
141
221
|
this.#customQueryTransformer = customQueryTransformer;
|
|
142
222
|
this.#cvrStore = new CVRStore(lc, cvrDb, shard, taskID, clientGroupID, () => this.#stateChanges.cancel());
|
|
143
223
|
this.#setTimeout = setTimeoutFn;
|
|
224
|
+
this.#now = now;
|
|
225
|
+
this.#hydrationCircuitBreaker = new HydrationCircuitBreaker(config.viewSyncerQueryHydrationTimeoutMs ?? 0, void 0, now);
|
|
144
226
|
this.#runPriorityOp = runPriorityOp;
|
|
145
227
|
this.keepalive();
|
|
146
228
|
}
|
|
@@ -190,10 +272,20 @@ var ViewSyncerService = class {
|
|
|
190
272
|
});
|
|
191
273
|
}
|
|
192
274
|
readyState() {
|
|
193
|
-
return Promise
|
|
275
|
+
return new Promise((resolve, reject) => {
|
|
276
|
+
const unsubscribe = this.#drainCoordinator.onDraining(() => resolve("draining"));
|
|
277
|
+
this.#initialized.promise.then((state) => {
|
|
278
|
+
unsubscribe();
|
|
279
|
+
resolve(state);
|
|
280
|
+
}, (err) => {
|
|
281
|
+
unsubscribe();
|
|
282
|
+
reject(err);
|
|
283
|
+
});
|
|
284
|
+
});
|
|
194
285
|
}
|
|
195
286
|
async run() {
|
|
196
287
|
try {
|
|
288
|
+
this.#scheduleShutdown(this.#keepaliveMs);
|
|
197
289
|
if (await this.readyState() === "draining") {
|
|
198
290
|
this.#lc.debug?.(`draining view-syncer ${this.id} before running`);
|
|
199
291
|
this.stop();
|
|
@@ -214,26 +306,29 @@ var ViewSyncerService = class {
|
|
|
214
306
|
lc.info?.(`resetting CVR: ${message}`);
|
|
215
307
|
throw new ClientNotFoundError(message);
|
|
216
308
|
}
|
|
217
|
-
|
|
309
|
+
let previousQueries;
|
|
310
|
+
if (this.#pipelinesHydrated) {
|
|
218
311
|
const result = await this.#advancePipelines(lc, cvr);
|
|
219
312
|
if (result === "success") return;
|
|
220
313
|
lc.info?.(`resetting pipelines: ${result.message}`);
|
|
221
314
|
this.#pipelineResets.add(1, { reason: result.reason });
|
|
315
|
+
switch (result.reason) {
|
|
316
|
+
case "advancement-timeout":
|
|
317
|
+
case "scalar-subquery":
|
|
318
|
+
case "truncation":
|
|
319
|
+
case "schema-change":
|
|
320
|
+
if (!Object.values(cvr.queries).some((q) => q.type === "client")) previousQueries = new Map(this.#pipelines.queries());
|
|
321
|
+
break;
|
|
322
|
+
case "permissions-change": break;
|
|
323
|
+
default: unreachable(result.reason);
|
|
324
|
+
}
|
|
222
325
|
this.#pipelines.reset(clientSchema);
|
|
223
|
-
this.#
|
|
326
|
+
this.#pipelinesHydrated = false;
|
|
224
327
|
this.connContextManager.setSharedRetransformReady(false);
|
|
225
328
|
}
|
|
226
|
-
const
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
lc.debug?.(`replica@${version} is behind cvr@${cvrVer}`);
|
|
230
|
-
return;
|
|
231
|
-
}
|
|
232
|
-
lc.info?.(`init pipelines@${version} (cvr@${cvrVer})`);
|
|
233
|
-
const driftedQueryIDs = await this.#hydrateUnchangedQueries(lc, cvr);
|
|
234
|
-
await this.#syncQueryPipelineSet(lc, cvr, "missing", void 0, driftedQueryIDs);
|
|
235
|
-
this.#pipelinesSynced = true;
|
|
236
|
-
this.connContextManager.setSharedRetransformReady(true);
|
|
329
|
+
const backgroundConnCtx = this.connContextManager.getBackgroundConnectionContext();
|
|
330
|
+
if (backgroundConnCtx) await this.#maybeHydratePipelines(lc, cvr, backgroundConnCtx, previousQueries);
|
|
331
|
+
else lc.info?.("No validated background connection; deferring pipeline init");
|
|
237
332
|
});
|
|
238
333
|
}
|
|
239
334
|
if (this.#drainCoordinator.shouldDrain()) this.#drainCoordinator.drainNextIn(this.#totalHydrationTimeMs());
|
|
@@ -247,11 +342,57 @@ var ViewSyncerService = class {
|
|
|
247
342
|
this.#stopped.resolve();
|
|
248
343
|
}
|
|
249
344
|
}
|
|
345
|
+
/**
|
|
346
|
+
* Ensures that query pipelines are hydrated and ready for steady-state
|
|
347
|
+
* operation.
|
|
348
|
+
*
|
|
349
|
+
* Preconditions:
|
|
350
|
+
* 1. Must be called from within the `#lock`.
|
|
351
|
+
* 2. The caller must provide an active, validated `ConnectionContext` for
|
|
352
|
+
* evaluating custom query transforms and permissions.
|
|
353
|
+
*
|
|
354
|
+
* Readiness prerequisites:
|
|
355
|
+
* - The underlying `QueryPipelineDriver` must be initialized with the schema
|
|
356
|
+
* (i.e. `this.#pipelines.initialized()` is true).
|
|
357
|
+
* - The SQLite replica must have caught up to at least the CVR's
|
|
358
|
+
* `stateVersion` (`version >= cvr.version.stateVersion`).
|
|
359
|
+
*
|
|
360
|
+
* If either prerequisite is not met, this method returns early as a no-op,
|
|
361
|
+
* waiting for the replica to catch up or the driver to initialize.
|
|
362
|
+
*
|
|
363
|
+
* Once ready:
|
|
364
|
+
* - Hydrates unchanged queries from the CVR snapshot without full row diffing.
|
|
365
|
+
* - Syncs missing, errored, or drifted queries via `#syncQueryPipelineSet`.
|
|
366
|
+
* - Sets `#pipelinesHydrated = true` and enables shared retransformations.
|
|
367
|
+
*/
|
|
368
|
+
async #maybeHydratePipelines(lc, cvr, connCtx, previousQueries) {
|
|
369
|
+
if (!this.#pipelines.initialized()) return;
|
|
370
|
+
const version = this.#pipelines.advanceWithoutDiff();
|
|
371
|
+
const cvrVer = versionString(cvr.version);
|
|
372
|
+
if (version < cvr.version.stateVersion) {
|
|
373
|
+
lc.debug?.(`replica@${version} is behind cvr@${cvrVer}`);
|
|
374
|
+
return;
|
|
375
|
+
}
|
|
376
|
+
lc.info?.(`init pipelines@${version} (cvr@${cvrVer})`);
|
|
377
|
+
const hydrationBudget = new HydrationBudget(this.#config.viewSyncerHydrationBudgetMs ?? 0, this.#now);
|
|
378
|
+
const hydrationPassStats = {
|
|
379
|
+
activeHydratedQueries: 0,
|
|
380
|
+
inactiveHydratedQueries: 0
|
|
381
|
+
};
|
|
382
|
+
const driftedQueryIDs = await this.#hydrateUnchangedQueries(lc, cvr, connCtx, hydrationPassStats, hydrationBudget, previousQueries);
|
|
383
|
+
await this.#syncQueryPipelineSet(lc, cvr, "missing", connCtx, driftedQueryIDs, hydrationBudget, hydrationPassStats, previousQueries);
|
|
384
|
+
this.#pipelinesHydrated = true;
|
|
385
|
+
this.connContextManager.setSharedRetransformReady(true);
|
|
386
|
+
}
|
|
250
387
|
#removeExpiredQueries = async (lc, cvr) => {
|
|
251
388
|
if (hasExpiredQueries(cvr)) {
|
|
252
389
|
lc = lc.withContext("method", "#removeExpiredQueries");
|
|
253
390
|
lc.debug?.("Queries have expired");
|
|
254
|
-
if (this.#
|
|
391
|
+
if (this.#pipelinesHydrated) {
|
|
392
|
+
const connCtx = this.connContextManager.getBackgroundConnectionContext();
|
|
393
|
+
if (connCtx) await this.#syncQueryPipelineSet(lc, cvr, "missing", connCtx);
|
|
394
|
+
else lc.info?.("No validated background connection to remove expired queries; deferring");
|
|
395
|
+
}
|
|
255
396
|
}
|
|
256
397
|
this.#scheduleExpireEviction(lc, cvr);
|
|
257
398
|
};
|
|
@@ -333,7 +474,14 @@ var ViewSyncerService = class {
|
|
|
333
474
|
return true;
|
|
334
475
|
}
|
|
335
476
|
#shutdownTimer = null;
|
|
477
|
+
#stopShutdownTimer() {
|
|
478
|
+
if (this.#shutdownTimer !== null) {
|
|
479
|
+
clearTimeout(this.#shutdownTimer);
|
|
480
|
+
this.#shutdownTimer = null;
|
|
481
|
+
}
|
|
482
|
+
}
|
|
336
483
|
#scheduleShutdown(delayMs = 0) {
|
|
484
|
+
if (this.#shuttingDown) return;
|
|
337
485
|
this.#shutdownTimer ??= this.#setTimeout(() => {
|
|
338
486
|
this.#shutdownTimer = null;
|
|
339
487
|
this.#runInLockWithCVR(() => {}).catch((e) => this.#stateChanges.fail(e));
|
|
@@ -380,6 +528,7 @@ var ViewSyncerService = class {
|
|
|
380
528
|
*/
|
|
381
529
|
#scheduleAuthMaintenance(lc) {
|
|
382
530
|
this.#stopAuthMaintenanceTimer();
|
|
531
|
+
if (this.#shuttingDown) return;
|
|
383
532
|
const plan = this.connContextManager.planMaintenance();
|
|
384
533
|
if (plan.earliestDeadlineAt === void 0) {
|
|
385
534
|
lc.debug?.("No auth maintenance wakeup scheduled");
|
|
@@ -467,7 +616,7 @@ var ViewSyncerService = class {
|
|
|
467
616
|
}
|
|
468
617
|
lc.debug?.("Auth changed, re-validating and re-transforming queries");
|
|
469
618
|
const connCtx = this.connContextManager.mustGetConnectionContext(selector);
|
|
470
|
-
if (!this.#
|
|
619
|
+
if (!this.#pipelinesHydrated) {
|
|
471
620
|
if (!await this.#validateConnection(connCtx)) return;
|
|
472
621
|
}
|
|
473
622
|
return await this.#handleConfigUpdate(lc, clientID, {}, cvr, "all", void 0, connCtx);
|
|
@@ -496,6 +645,7 @@ var ViewSyncerService = class {
|
|
|
496
645
|
}
|
|
497
646
|
#startTTLClockInterval(lc) {
|
|
498
647
|
this.#stopTTLClockInterval();
|
|
648
|
+
if (this.#shuttingDown) return;
|
|
499
649
|
this.#ttlClockInterval = this.#setTimeout(() => {
|
|
500
650
|
this.#updateTTLClockInCVRWithoutLock(lc);
|
|
501
651
|
this.#startTTLClockInterval(lc);
|
|
@@ -529,7 +679,8 @@ var ViewSyncerService = class {
|
|
|
529
679
|
await pokers.end(newCVR.version);
|
|
530
680
|
});
|
|
531
681
|
}
|
|
532
|
-
if (this.#
|
|
682
|
+
if (!this.#pipelinesHydrated) await this.#maybeHydratePipelines(lc, this.#cvr, connCtx);
|
|
683
|
+
else await this.#syncQueryPipelineSet(lc, this.#cvr, customQueryTransformMode, connCtx);
|
|
533
684
|
return this.#cvr;
|
|
534
685
|
}
|
|
535
686
|
/**
|
|
@@ -625,6 +776,7 @@ var ViewSyncerService = class {
|
|
|
625
776
|
#scheduleExpireEviction(lc, cvr) {
|
|
626
777
|
const { ttlClock } = cvr;
|
|
627
778
|
this.#stopExpireTimer();
|
|
779
|
+
if (this.#shuttingDown) return;
|
|
628
780
|
const next = nextEvictionTime(cvr);
|
|
629
781
|
if (next === void 0) {
|
|
630
782
|
lc.debug?.("no inactive queries with ttl");
|
|
@@ -652,7 +804,7 @@ var ViewSyncerService = class {
|
|
|
652
804
|
*
|
|
653
805
|
* This must be called from within the #lock.
|
|
654
806
|
*/
|
|
655
|
-
async #hydrateUnchangedQueries(lc, cvr) {
|
|
807
|
+
async #hydrateUnchangedQueries(lc, cvr, connCtx, hydrationPassStats, hydrationBudget, previousQueries) {
|
|
656
808
|
assert(this.#pipelines.initialized(), "pipelines must be initialized");
|
|
657
809
|
const dbVersion = this.#pipelines.currentVersion();
|
|
658
810
|
const cvrVersion = cvr.version;
|
|
@@ -661,33 +813,34 @@ var ViewSyncerService = class {
|
|
|
661
813
|
return /* @__PURE__ */ new Set();
|
|
662
814
|
}
|
|
663
815
|
const gotQueries = Object.entries(cvr.queries).filter(([_, state]) => state.transformationHash !== void 0);
|
|
816
|
+
const { required: requiredGotQueries } = classifyQueriesForHydration(gotQueries.map(([, query]) => query));
|
|
664
817
|
const customQueries = /* @__PURE__ */ new Map();
|
|
665
818
|
const otherQueries = [];
|
|
666
|
-
|
|
667
|
-
for (const [, query] of gotQueries) {
|
|
668
|
-
if (query.type !== "internal" && Object.values(query.clientState).every(({ inactivatedAt }) => inactivatedAt !== void 0)) {
|
|
669
|
-
inactivatedCount++;
|
|
670
|
-
continue;
|
|
671
|
-
}
|
|
672
|
-
if (query.type === "custom") customQueries.set(query.id, query);
|
|
673
|
-
else otherQueries.push(query);
|
|
674
|
-
}
|
|
819
|
+
const inactivatedCount = gotQueries.length - requiredGotQueries.length;
|
|
675
820
|
const transformedQueries = [];
|
|
821
|
+
for (const query of requiredGotQueries) if (query.type === "custom") {
|
|
822
|
+
const previous = previousQueries?.get(query.id);
|
|
823
|
+
if (previous && previous.transformationHash === query.transformationHash) transformedQueries.push({
|
|
824
|
+
id: query.id,
|
|
825
|
+
transformationHash: previous.transformationHash,
|
|
826
|
+
transformedAst: previous.originalAst ?? previous.transformedAst
|
|
827
|
+
});
|
|
828
|
+
else customQueries.set(query.id, query);
|
|
829
|
+
} else otherQueries.push(query);
|
|
676
830
|
let customErrorCount = 0;
|
|
677
831
|
let customHashMismatchCount = 0;
|
|
678
832
|
let otherHashMismatchCount = 0;
|
|
679
833
|
if (customQueries.size > 0 && !this.#customQueryTransformer) lc.warn?.("Custom/named queries were requested but no `ZERO_QUERY_URL` is configured for Zero Cache.");
|
|
680
|
-
const backgroundContext = this.connContextManager.mustGetBackgroundConnectionContext();
|
|
681
834
|
const customQueryTransformer = this.#customQueryTransformer;
|
|
682
835
|
if (customQueryTransformer && customQueries.size > 0) {
|
|
683
|
-
const transformedCustomQueries = await this.#runPriorityOp(lc, "#hydrateUnchangedQueries transforming custom queries", () => customQueryTransformer.transform(
|
|
684
|
-
if (transformedCustomQueries.kind === "success" && !transformedCustomQueries.cached) this.connContextManager.validateConnection(
|
|
836
|
+
const transformedCustomQueries = await hydrationBudget.excluding(() => this.#runPriorityOp(lc, "#hydrateUnchangedQueries transforming custom queries", () => customQueryTransformer.transform(connCtx, customQueries.values())));
|
|
837
|
+
if (transformedCustomQueries.kind === "success" && !transformedCustomQueries.cached) this.connContextManager.validateConnection(connCtx, connCtx.revision, transformedCustomQueries.validation);
|
|
685
838
|
if (Array.isArray(transformedCustomQueries.result)) for (const q of transformedCustomQueries.result) if ("error" in q) customErrorCount++;
|
|
686
839
|
else if (q.transformationHash !== customQueries.get(q.id)?.transformationHash) customHashMismatchCount++;
|
|
687
840
|
else transformedQueries.push(q);
|
|
688
841
|
}
|
|
689
842
|
for (const q of otherQueries) {
|
|
690
|
-
const transformed = transformAndHashQuery(lc, q.id, q.ast, must(this.#pipelines.currentPermissions()).permissions ?? { tables: {} },
|
|
843
|
+
const transformed = transformAndHashQuery(lc, q.id, q.ast, must(this.#pipelines.currentPermissions()).permissions ?? { tables: {} }, connCtx.auth?.type === "jwt" ? connCtx.auth : void 0, q.type === "internal");
|
|
691
844
|
if (transformed.transformationHash === q.transformationHash) transformedQueries.push(transformed);
|
|
692
845
|
else otherHashMismatchCount++;
|
|
693
846
|
}
|
|
@@ -700,6 +853,10 @@ var ViewSyncerService = class {
|
|
|
700
853
|
for (const { id: queryID, transformationHash, transformedAst } of transformedQueries) {
|
|
701
854
|
const query = cvr.queries[queryID];
|
|
702
855
|
const queryName = query.type === "custom" ? query.name : void 0;
|
|
856
|
+
if (query.type !== "internal" && this.#hydrationCircuitBreaker.isOpen(transformationHash)) {
|
|
857
|
+
lc.info?.(`skipping hydration of ${queryID}: hydration circuit breaker open`);
|
|
858
|
+
continue;
|
|
859
|
+
}
|
|
703
860
|
const covered = queryCoveringIndex ? this.#findQueryCoverageShadowHit(queryCoveringIndex, queryID, transformationHash, transformedAst, queryName) : void 0;
|
|
704
861
|
totalHydratedQueries++;
|
|
705
862
|
if (covered) {
|
|
@@ -708,19 +865,35 @@ var ViewSyncerService = class {
|
|
|
708
865
|
}
|
|
709
866
|
const timer = new TimeSliceTimer(lc);
|
|
710
867
|
let count = 0;
|
|
868
|
+
let timedOut = false;
|
|
711
869
|
await startAsyncSpan(tracer, "vs.#hydrateUnchangedQueries.addQuery", async (span) => {
|
|
712
870
|
span.setAttribute("queryHash", queryID);
|
|
713
871
|
span.setAttribute("transformationHash", transformationHash);
|
|
714
872
|
span.setAttribute("table", transformedAst.table);
|
|
715
873
|
if (queryName !== void 0) span.setAttribute("queryName", queryName);
|
|
716
|
-
for (const change of this.#pipelines.addQuery(transformationHash, queryID, transformedAst, await timer.start(), queryName, "unchanged-query-rehydrate")) if (change === "yield")
|
|
717
|
-
|
|
874
|
+
for (const change of this.#pipelines.addQuery(transformationHash, queryID, transformedAst, await timer.start(), queryName, "unchanged-query-rehydrate")) if (change === "yield") {
|
|
875
|
+
if (query.type !== "internal" && this.#hydrationCircuitBreaker.exceeded(timer.totalElapsed())) {
|
|
876
|
+
timedOut = true;
|
|
877
|
+
await timer.yieldProcess("yield in hydrateUnchangedQueries");
|
|
878
|
+
break;
|
|
879
|
+
}
|
|
880
|
+
await timer.yieldProcess("yield in hydrateUnchangedQueries");
|
|
881
|
+
} else count++;
|
|
718
882
|
});
|
|
719
883
|
const elapsed = timer.totalElapsed();
|
|
884
|
+
if (timedOut) {
|
|
885
|
+
this.#recordHydrationTimeout(lc, {
|
|
886
|
+
id: queryID,
|
|
887
|
+
transformationHash,
|
|
888
|
+
name: queryName
|
|
889
|
+
}, elapsed, count);
|
|
890
|
+
continue;
|
|
891
|
+
}
|
|
892
|
+
hydrationPassStats.activeHydratedQueries++;
|
|
720
893
|
this.#hydrations.add(1);
|
|
721
894
|
this.#hydrationTime.recordMs(elapsed);
|
|
722
|
-
this.#addQueryMaterializationServerMetric(
|
|
723
|
-
this.#inspectorDelegate.addQuery(
|
|
895
|
+
this.#addQueryMaterializationServerMetric(queryID, elapsed);
|
|
896
|
+
this.#inspectorDelegate.addQuery(queryID, transformedAst);
|
|
724
897
|
lc.debug?.(`hydrated ${count} rows for ${queryID} (${elapsed} ms)`);
|
|
725
898
|
let drifted = false;
|
|
726
899
|
const storedSigHex = cvr.queries[queryID]?.rowSetSignature;
|
|
@@ -781,11 +954,7 @@ var ViewSyncerService = class {
|
|
|
781
954
|
return;
|
|
782
955
|
}
|
|
783
956
|
const appErrorGroups = /* @__PURE__ */ new Map();
|
|
784
|
-
for (const err of errorOrErrors) for (const clientId of getAffectedClientIDs([err.id]))
|
|
785
|
-
const group = appErrorGroups.get(clientId) ?? [];
|
|
786
|
-
group.push(err);
|
|
787
|
-
appErrorGroups.set(clientId, group);
|
|
788
|
-
}
|
|
957
|
+
for (const err of errorOrErrors) for (const clientId of getAffectedClientIDs([err.id])) getOrInsertComputed(appErrorGroups, clientId, newArray).push(err);
|
|
789
958
|
for (const [clientId, errors] of appErrorGroups) this.#clients.get(clientId)?.sendQueryTransformApplicationErrors(errors);
|
|
790
959
|
}
|
|
791
960
|
#addQueryMaterializationServerMetric(queryID, elapsed) {
|
|
@@ -821,7 +990,10 @@ var ViewSyncerService = class {
|
|
|
821
990
|
*
|
|
822
991
|
* This must be called from within the #lock.
|
|
823
992
|
*/
|
|
824
|
-
#syncQueryPipelineSet(lc, cvr, customQueryTransformMode, connCtx, driftedQueryIDs = /* @__PURE__ */ new Set()) {
|
|
993
|
+
#syncQueryPipelineSet(lc, cvr, customQueryTransformMode, connCtx, driftedQueryIDs = /* @__PURE__ */ new Set(), hydrationBudget = new HydrationBudget(this.#config.viewSyncerHydrationBudgetMs ?? 0, this.#now), hydrationPassStats = {
|
|
994
|
+
activeHydratedQueries: 0,
|
|
995
|
+
inactiveHydratedQueries: 0
|
|
996
|
+
}, previousQueries) {
|
|
825
997
|
return startAsyncSpan(tracer, "vs.#syncQueryPipelineSet", async (span) => {
|
|
826
998
|
const start = performance.now();
|
|
827
999
|
span.setAttribute("clientGroupID", this.id);
|
|
@@ -830,38 +1002,45 @@ var ViewSyncerService = class {
|
|
|
830
1002
|
const now = Date.now();
|
|
831
1003
|
const ttlClock = this.#getTTLClock(now);
|
|
832
1004
|
const cvrQueryEntires = Object.entries(cvr.queries);
|
|
833
|
-
const customQueries = /* @__PURE__ */ new Map();
|
|
834
|
-
const otherQueries = [];
|
|
835
1005
|
const transformedQueries = [];
|
|
836
|
-
const
|
|
837
|
-
|
|
838
|
-
|
|
839
|
-
|
|
840
|
-
|
|
841
|
-
|
|
842
|
-
|
|
843
|
-
});
|
|
844
|
-
for (const { id, query: origQuery } of otherQueries) {
|
|
845
|
-
assert(id === origQuery.id, "query id mismatch");
|
|
846
|
-
const transformed = transformAndHashQuery(lc, origQuery.id, origQuery.ast, must(this.#pipelines.currentPermissions()).permissions ?? { tables: {} }, resolvedConnCtx.auth?.type === "jwt" ? resolvedConnCtx.auth : void 0, origQuery.type === "internal");
|
|
847
|
-
transformedQueries.push({
|
|
848
|
-
id,
|
|
849
|
-
origQuery,
|
|
850
|
-
transformed
|
|
851
|
-
});
|
|
852
|
-
}
|
|
853
|
-
if (customQueries.size > 0 && !this.#customQueryTransformer) lc.warn?.("Custom/named queries were requested but no `ZERO_QUERY_URL` is configured for Zero Cache.");
|
|
854
|
-
let erroredQueryIDs;
|
|
855
|
-
const customQueriesToTransform = customQueryTransformMode === "all" ? [...customQueries.values()] : customQueryTransformMode && [...customQueries.values()].filter((q) => !this.#pipelines.queries().has(q.id));
|
|
1006
|
+
const naturallyExpiredQueryIDs = new Set(cvrQueryEntires.map(([, query]) => query).filter((query) => expired(ttlClock, query)).map((query) => query.id));
|
|
1007
|
+
if (naturallyExpiredQueryIDs.size > 0) this.#queryEvictions.add(naturallyExpiredQueryIDs.size, { reason: "ttl" });
|
|
1008
|
+
const hydrationCandidates = cvrQueryEntires.map(([, query]) => query).filter((query) => !naturallyExpiredQueryIDs.has(query.id));
|
|
1009
|
+
const { required, optional } = classifyQueriesForHydration(hydrationCandidates);
|
|
1010
|
+
const requiredQueryIDs = new Set(required.map((query) => query.id));
|
|
1011
|
+
const optionalToHydrate = hydrationBudget.limitMs === 0 ? optional : optional.filter((query) => query.patchVersion !== void 0);
|
|
1012
|
+
if (!this.#customQueryTransformer && hydrationCandidates.some((query) => query.type === "custom")) lc.warn?.("Custom/named queries were requested but no `ZERO_QUERY_URL` is configured for Zero Cache.");
|
|
856
1013
|
const customQueryTransformer = this.#customQueryTransformer;
|
|
857
|
-
|
|
1014
|
+
const erroredQueryIDs = [];
|
|
1015
|
+
let transformedCustomQueryCount = 0;
|
|
1016
|
+
const transformOtherQueries = (queries) => {
|
|
1017
|
+
for (const origQuery of queries) {
|
|
1018
|
+
if (origQuery.type === "custom") continue;
|
|
1019
|
+
const transformed = transformAndHashQuery(lc, origQuery.id, origQuery.ast, must(this.#pipelines.currentPermissions()).permissions ?? { tables: {} }, connCtx.auth?.type === "jwt" ? connCtx.auth : void 0, origQuery.type === "internal");
|
|
1020
|
+
transformedQueries.push({
|
|
1021
|
+
id: origQuery.id,
|
|
1022
|
+
origQuery,
|
|
1023
|
+
transformed
|
|
1024
|
+
});
|
|
1025
|
+
}
|
|
1026
|
+
};
|
|
1027
|
+
const shouldTransformCustomQuery = (query) => {
|
|
1028
|
+
if (customQueryTransformMode === "all") return true;
|
|
1029
|
+
if (this.#pipelines.queries().has(query.id)) return false;
|
|
1030
|
+
const previous = previousQueries?.get(query.id);
|
|
1031
|
+
if (previous && previous.transformationHash === query.transformationHash) return false;
|
|
1032
|
+
return true;
|
|
1033
|
+
};
|
|
1034
|
+
const transformCustomQueries = async (queries) => {
|
|
1035
|
+
if (!customQueryTransformer || queries.length === 0) return;
|
|
1036
|
+
transformedCustomQueryCount += queries.length;
|
|
858
1037
|
const transformStart = performance.now();
|
|
859
1038
|
let transformedCustomQueries;
|
|
860
1039
|
try {
|
|
861
|
-
transformedCustomQueries = await this.#runPriorityOp(lc, "#syncQueryPipelineSet transforming custom queries", () => customQueryTransformer.transform(
|
|
1040
|
+
transformedCustomQueries = await hydrationBudget.excluding(() => this.#runPriorityOp(lc, "#syncQueryPipelineSet transforming custom queries", () => customQueryTransformer.transform(connCtx, queries)));
|
|
862
1041
|
if (transformedCustomQueries.kind === "failed") throw new ProtocolErrorWithLevel(transformedCustomQueries.result, "warn");
|
|
863
1042
|
else {
|
|
864
|
-
if (!transformedCustomQueries.cached) this.connContextManager.validateConnection(
|
|
1043
|
+
if (!transformedCustomQueries.cached) this.connContextManager.validateConnection(connCtx, connCtx.revision, transformedCustomQueries.validation);
|
|
865
1044
|
this.#queryTransformations.add(1, { result: "success" });
|
|
866
1045
|
}
|
|
867
1046
|
} catch (e) {
|
|
@@ -872,8 +1051,9 @@ var ViewSyncerService = class {
|
|
|
872
1051
|
this.#queryTransformationTime.recordMs(transformDuration);
|
|
873
1052
|
}
|
|
874
1053
|
const successfullyTransformedCustomQueries = /* @__PURE__ */ new Map();
|
|
875
|
-
|
|
876
|
-
|
|
1054
|
+
const customQueryMap = new Map(queries.map((query) => [query.id, query]));
|
|
1055
|
+
erroredQueryIDs.push(...this.#processTransformedCustomQueries(lc, transformedCustomQueries.result, (q) => {
|
|
1056
|
+
const origQuery = customQueryMap.get(q.id);
|
|
877
1057
|
if (origQuery) {
|
|
878
1058
|
successfullyTransformedCustomQueries.set(q.id, q);
|
|
879
1059
|
transformedQueries.push({
|
|
@@ -882,7 +1062,7 @@ var ViewSyncerService = class {
|
|
|
882
1062
|
transformed: q
|
|
883
1063
|
});
|
|
884
1064
|
}
|
|
885
|
-
},
|
|
1065
|
+
}, customQueryMap));
|
|
886
1066
|
for (const [queryID, newTransform] of successfullyTransformedCustomQueries) {
|
|
887
1067
|
const existingTransformHash = cvr.queries[queryID]?.transformationHash;
|
|
888
1068
|
if (existingTransformHash) {
|
|
@@ -895,21 +1075,55 @@ var ViewSyncerService = class {
|
|
|
895
1075
|
} else this.#queryTransformationNoOps.add(1);
|
|
896
1076
|
}
|
|
897
1077
|
}
|
|
1078
|
+
};
|
|
1079
|
+
const queriesToTransform = [...required, ...optionalToHydrate];
|
|
1080
|
+
transformOtherQueries(queriesToTransform);
|
|
1081
|
+
if (customQueryTransformMode === "missing" && previousQueries) {
|
|
1082
|
+
for (const query of queriesToTransform) if (query.type === "custom" && !this.#pipelines.queries().has(query.id)) {
|
|
1083
|
+
const previous = previousQueries.get(query.id);
|
|
1084
|
+
if (previous && previous.transformationHash === query.transformationHash) {
|
|
1085
|
+
this.#queryTransformationNoOps.add(1);
|
|
1086
|
+
transformedQueries.push({
|
|
1087
|
+
id: query.id,
|
|
1088
|
+
origQuery: query,
|
|
1089
|
+
transformed: {
|
|
1090
|
+
id: query.id,
|
|
1091
|
+
transformationHash: previous.transformationHash,
|
|
1092
|
+
transformedAst: previous.originalAst ?? previous.transformedAst
|
|
1093
|
+
}
|
|
1094
|
+
});
|
|
1095
|
+
}
|
|
1096
|
+
}
|
|
898
1097
|
}
|
|
899
|
-
|
|
1098
|
+
await transformCustomQueries(queriesToTransform.filter((query) => query.type === "custom" && shouldTransformCustomQuery(query)));
|
|
1099
|
+
const circuitBrokenQueries = transformedQueries.filter(({ id, origQuery, transformed }) => origQuery.type !== "internal" && this.#pipelines.queries().get(id)?.transformationHash !== transformed.transformationHash && this.#hydrationCircuitBreaker.isOpen(transformed.transformationHash)).map(({ id, origQuery, transformed }) => ({
|
|
1100
|
+
id,
|
|
1101
|
+
transformationHash: transformed.transformationHash,
|
|
1102
|
+
name: origQuery.type === "custom" ? origQuery.name : void 0
|
|
1103
|
+
}));
|
|
1104
|
+
if (circuitBrokenQueries.length > 0) this.#rejectCircuitBrokenQueries(lc, cvr, circuitBrokenQueries);
|
|
1105
|
+
const removeQueriesQueryIds = new Set([
|
|
1106
|
+
...naturallyExpiredQueryIDs,
|
|
1107
|
+
...erroredQueryIDs,
|
|
1108
|
+
...circuitBrokenQueries.map(({ id }) => id)
|
|
1109
|
+
]);
|
|
900
1110
|
const addQueries = transformedQueries.map(({ id, origQuery, transformed }) => ({
|
|
901
1111
|
id,
|
|
902
1112
|
ast: transformed.transformedAst,
|
|
903
1113
|
transformationHash: transformed.transformationHash,
|
|
904
1114
|
name: origQuery.type === "custom" ? origQuery.name : void 0
|
|
905
1115
|
})).filter((q) => !removeQueriesQueryIds.has(q.id) && this.#pipelines.queries().get(q.id)?.transformationHash !== q.transformationHash);
|
|
906
|
-
|
|
1116
|
+
const hydrationOrder = new Map([...required, ...optional].map((query, index) => [query.id, index]));
|
|
1117
|
+
addQueries.sort((a, b) => must(hydrationOrder.get(a.id)) - must(hydrationOrder.get(b.id)));
|
|
1118
|
+
const requiredAddQueries = addQueries.filter((query) => requiredQueryIDs.has(query.id));
|
|
1119
|
+
const optionalAddQueries = addQueries.filter((query) => !requiredQueryIDs.has(query.id));
|
|
1120
|
+
lc.info?.(`syncQueryPipelineSet: ${cvrQueryEntires.length} CVR queries, ${transformedCustomQueryCount} custom re-transformed, ${erroredQueryIDs.length} errored, ${circuitBrokenQueries.length} circuit-broken, ${removeQueriesQueryIds.size} to remove, ${addQueries.length} to add`);
|
|
907
1121
|
for (const q of addQueries) {
|
|
908
1122
|
const orig = cvr.queries[q.id];
|
|
909
1123
|
lc.debug?.("ViewSyncer adding query", q.ast, "transformed from", orig.type === "custom" ? orig.name : orig.ast);
|
|
910
1124
|
}
|
|
911
1125
|
if (addQueries.length > 0 || removeQueriesQueryIds.size > 0) {
|
|
912
|
-
await this.#addAndRemoveQueries(lc, cvr,
|
|
1126
|
+
await this.#addAndRemoveQueries(lc, cvr, requiredAddQueries, optionalAddQueries, Array.from(removeQueriesQueryIds, (id) => ({ id })), hydrationBudget, hydrationPassStats, driftedQueryIDs);
|
|
913
1127
|
if (addQueries.length > 0) this.#viewSyncerHydration.recordMs(performance.now() - start);
|
|
914
1128
|
} else await this.#catchupClients(lc, cvr);
|
|
915
1129
|
});
|
|
@@ -942,8 +1156,93 @@ var ViewSyncerService = class {
|
|
|
942
1156
|
record.count++;
|
|
943
1157
|
if (record.count >= THRASH_THRESHOLD) this.#lc.warn?.(`Query thrashing detected for query ${queryID}. ${record.count} replacements in 60s. This may indicate clients with different auth contexts connecting to the same client group.`);
|
|
944
1158
|
}
|
|
945
|
-
|
|
1159
|
+
/**
|
|
1160
|
+
* Records that hydrating `query` was aborted for exceeding the query
|
|
1161
|
+
* hydration timeout, and opens its circuit breaker.
|
|
1162
|
+
*/
|
|
1163
|
+
#recordHydrationTimeout(lc, query, elapsedMs, rowCount) {
|
|
1164
|
+
this.#hydrationCircuitBreaker.trip(query.transformationHash);
|
|
1165
|
+
this.#hydrationTimeouts.add(1);
|
|
1166
|
+
lc.warn?.("Query hydration aborted for exceeding the hydration timeout", {
|
|
1167
|
+
clientGroupID: this.id,
|
|
1168
|
+
queryHash: query.id,
|
|
1169
|
+
transformationHash: query.transformationHash,
|
|
1170
|
+
...query.name !== void 0 && { queryName: query.name },
|
|
1171
|
+
hydrationTimeoutMs: this.#hydrationCircuitBreaker.timeoutMs,
|
|
1172
|
+
hydrationElapsedMs: elapsedMs,
|
|
1173
|
+
...rowCount !== void 0 && { hydrationRowCount: rowCount },
|
|
1174
|
+
circuitBreakerOpenMs: this.#hydrationCircuitBreaker.openMs
|
|
1175
|
+
});
|
|
1176
|
+
}
|
|
1177
|
+
/** Records that `query` was rejected without hydration by its open breaker. */
|
|
1178
|
+
#recordCircuitBreakerRejection(lc, query) {
|
|
1179
|
+
this.#hydrationCircuitBreakerRejections.add(1);
|
|
1180
|
+
lc.warn?.("Query rejected by its open hydration circuit breaker", {
|
|
1181
|
+
clientGroupID: this.id,
|
|
1182
|
+
queryHash: query.id,
|
|
1183
|
+
transformationHash: query.transformationHash,
|
|
1184
|
+
...query.name !== void 0 && { queryName: query.name },
|
|
1185
|
+
hydrationTimeoutMs: this.#hydrationCircuitBreaker.timeoutMs,
|
|
1186
|
+
circuitBreakerOpenMs: this.#hydrationCircuitBreaker.openMs
|
|
1187
|
+
});
|
|
1188
|
+
}
|
|
1189
|
+
/**
|
|
1190
|
+
* Handles queries whose hydration circuit breaker is open: they are counted,
|
|
1191
|
+
* logged, and errored to the affected clients. The caller removes them from
|
|
1192
|
+
* the CVR.
|
|
1193
|
+
*/
|
|
1194
|
+
#rejectCircuitBrokenQueries(lc, cvr, queries) {
|
|
1195
|
+
for (const query of queries) this.#recordCircuitBreakerRejection(lc, query);
|
|
1196
|
+
this.#queryEvictions.add(queries.length, { reason: "hydration-circuit-breaker" });
|
|
1197
|
+
this.#sendHydrationTimeoutErrors(cvr, queries);
|
|
1198
|
+
}
|
|
1199
|
+
/**
|
|
1200
|
+
* Sends a per-query error to every client that desires one of `queries`.
|
|
1201
|
+
* The error goes out as a `transformError` application error, which every
|
|
1202
|
+
* client understands as "this query errored" without affecting the
|
|
1203
|
+
* connection or the client's other queries.
|
|
1204
|
+
*/
|
|
1205
|
+
#sendHydrationTimeoutErrors(cvr, queries) {
|
|
1206
|
+
const timeoutMs = this.#hydrationCircuitBreaker.timeoutMs;
|
|
1207
|
+
const errorsByClient = /* @__PURE__ */ new Map();
|
|
1208
|
+
for (const { id, name } of queries) {
|
|
1209
|
+
const query = cvr.queries[id];
|
|
1210
|
+
if (query === void 0 || query.type === "internal") continue;
|
|
1211
|
+
const error = {
|
|
1212
|
+
error: "app",
|
|
1213
|
+
id,
|
|
1214
|
+
name: name ?? (query.type === "custom" ? query.name : "legacy"),
|
|
1215
|
+
message: `Query hydration exceeded the ${timeoutMs}ms limit (ZERO_VIEW_SYNCER_QUERY_HYDRATION_TIMEOUT_MS) and was aborted`,
|
|
1216
|
+
details: {
|
|
1217
|
+
kind: "HydrationTimeout",
|
|
1218
|
+
timeoutMs
|
|
1219
|
+
}
|
|
1220
|
+
};
|
|
1221
|
+
for (const clientID of Object.keys(query.clientState)) getOrInsertComputed(errorsByClient, clientID, newArray).push(error);
|
|
1222
|
+
}
|
|
1223
|
+
for (const [clientID, errors] of errorsByClient) this.#clients.get(clientID)?.sendQueryTransformApplicationErrors(errors);
|
|
1224
|
+
}
|
|
1225
|
+
#recordHydrationBudgetExhaustion(lc, hydrationBudget, activeHydratedQueries, inactiveHydratedQueries, inactiveEvictedQueryIDs) {
|
|
1226
|
+
const elapsedMs = hydrationBudget.exhaustedAtMs;
|
|
1227
|
+
if (elapsedMs === void 0) return;
|
|
1228
|
+
this.#hydrationBudgetExhaustions.add(1);
|
|
1229
|
+
this.#hydrationBudgetEvictions.add(inactiveEvictedQueryIDs.length);
|
|
1230
|
+
this.#hydrationBudgetElapsed.record(elapsedMs);
|
|
1231
|
+
this.#hydrationBudgetOvershoot.record(Math.max(0, elapsedMs - hydrationBudget.limitMs));
|
|
1232
|
+
if (inactiveEvictedQueryIDs.length > 0) this.#queryEvictions.add(inactiveEvictedQueryIDs.length, { reason: "hydration-budget" });
|
|
1233
|
+
lc.info?.("view-syncer hydration budget exhausted", {
|
|
1234
|
+
clientGroupID: this.id,
|
|
1235
|
+
hydrationBudgetMs: hydrationBudget.limitMs,
|
|
1236
|
+
hydrationElapsedMs: elapsedMs,
|
|
1237
|
+
activeHydratedQueries,
|
|
1238
|
+
inactiveHydratedQueries,
|
|
1239
|
+
inactiveEvictedQueries: inactiveEvictedQueryIDs.length,
|
|
1240
|
+
firstEvictedQueryHash: inactiveEvictedQueryIDs[0] ?? null
|
|
1241
|
+
});
|
|
1242
|
+
}
|
|
1243
|
+
#addAndRemoveQueries(lc, cvr, requiredQueries, optionalQueries, removeQueries, hydrationBudget, hydrationPassStats, driftedQueryIDs = /* @__PURE__ */ new Set()) {
|
|
946
1244
|
return startAsyncSpan(tracer, "vs.#addAndRemoveQueries", async () => {
|
|
1245
|
+
const addQueries = [...requiredQueries, ...optionalQueries];
|
|
947
1246
|
assert(addQueries.length > 0 || removeQueries.length > 0, "Must have queries to add or remove");
|
|
948
1247
|
const start = performance.now();
|
|
949
1248
|
const stateVersion = this.#pipelines.currentVersion();
|
|
@@ -960,6 +1259,7 @@ var ViewSyncerService = class {
|
|
|
960
1259
|
updater.ensureNewVersion();
|
|
961
1260
|
}
|
|
962
1261
|
const { queryPatches, newVersion } = updater.trackQueries(lc, addQueries, removeQueries);
|
|
1262
|
+
if (hydrationBudget.limitMs !== 0 && optionalQueries.length > 0) assert(cmpVersions(cvr.version, newVersion) < 0, "Optional hydration requires a final poke version before row processing so a removal is actually poked");
|
|
963
1263
|
const pokers = startPoke(this.#getClients(), newVersion);
|
|
964
1264
|
for (const patch of queryPatches) await pokers.addPatch(patch);
|
|
965
1265
|
for (const q of removeQueries) {
|
|
@@ -976,22 +1276,54 @@ var ViewSyncerService = class {
|
|
|
976
1276
|
let totalHydratedQueries = 0;
|
|
977
1277
|
let coveredHydratedQueries = 0;
|
|
978
1278
|
let firstCoveredQuery;
|
|
1279
|
+
const hydratedQueryIDs = [];
|
|
1280
|
+
const budgetEvictedQueryIDs = [];
|
|
1281
|
+
const timedOutQueries = [];
|
|
1282
|
+
const rejectedQueries = [];
|
|
1283
|
+
const circuitBreaker = this.#hydrationCircuitBreaker;
|
|
979
1284
|
const self = this;
|
|
980
1285
|
await yieldProcess(lc);
|
|
981
|
-
function*
|
|
982
|
-
for (
|
|
1286
|
+
function* hydrateQueries(queries, optional, slowHydrateThreshold) {
|
|
1287
|
+
for (let i = 0; i < queries.length; i++) {
|
|
1288
|
+
if (optional && hydrationBudget.exhausted()) {
|
|
1289
|
+
budgetEvictedQueryIDs.push(...queries.slice(i).map((query) => query.id));
|
|
1290
|
+
return;
|
|
1291
|
+
}
|
|
1292
|
+
const q = must(queries[i]);
|
|
983
1293
|
let queryLC = lc.withContext("hash", q.id).withContext("queryHash", q.id).withContext("transformationHash", q.transformationHash);
|
|
984
1294
|
if (q.name !== void 0) queryLC = queryLC.withContext("queryName", q.name);
|
|
985
1295
|
queryLC.debug?.(`adding pipeline for query`, q.ast);
|
|
1296
|
+
const breakable = cvr.queries[q.id]?.type !== "internal";
|
|
1297
|
+
if (breakable && circuitBreaker.isOpen(q.transformationHash)) {
|
|
1298
|
+
rejectedQueries.push(q);
|
|
1299
|
+
self.#recordCircuitBreakerRejection(queryLC, q);
|
|
1300
|
+
continue;
|
|
1301
|
+
}
|
|
986
1302
|
const covered = queryCoveringIndex ? self.#findQueryCoverageShadowHit(queryCoveringIndex, q.id, q.transformationHash, q.ast, q.name) : void 0;
|
|
987
1303
|
totalHydratedQueries++;
|
|
988
1304
|
if (covered) {
|
|
989
1305
|
coveredHydratedQueries++;
|
|
990
1306
|
firstCoveredQuery ??= covered;
|
|
991
1307
|
}
|
|
992
|
-
|
|
1308
|
+
let timedOut = false;
|
|
1309
|
+
for (const change of pipelines.addQuery(q.transformationHash, q.id, q.ast, timer.startWithoutYielding(), q.name, "query-set-sync")) {
|
|
1310
|
+
if (change === "yield" && breakable && circuitBreaker.exceeded(timer.totalElapsed())) {
|
|
1311
|
+
timedOut = true;
|
|
1312
|
+
yield change;
|
|
1313
|
+
break;
|
|
1314
|
+
}
|
|
1315
|
+
yield change;
|
|
1316
|
+
}
|
|
993
1317
|
const elapsed = timer.stop();
|
|
994
1318
|
totalProcessTime += elapsed;
|
|
1319
|
+
if (timedOut) {
|
|
1320
|
+
timedOutQueries.push(q);
|
|
1321
|
+
self.#recordHydrationTimeout(queryLC, q, elapsed);
|
|
1322
|
+
continue;
|
|
1323
|
+
}
|
|
1324
|
+
hydratedQueryIDs.push(q.id);
|
|
1325
|
+
if (optional) hydrationPassStats.inactiveHydratedQueries++;
|
|
1326
|
+
else hydrationPassStats.activeHydratedQueries++;
|
|
995
1327
|
self.#addQueryMaterializationServerMetric(q.id, elapsed);
|
|
996
1328
|
self.#inspectorDelegate.addQuery(q.id, q.ast);
|
|
997
1329
|
queryCoveringIndex?.add(q.id, {
|
|
@@ -1005,18 +1337,43 @@ var ViewSyncerService = class {
|
|
|
1005
1337
|
transformationHash: q.transformationHash,
|
|
1006
1338
|
...q.name !== void 0 && { name: q.name }
|
|
1007
1339
|
});
|
|
1340
|
+
hydrations.add(1);
|
|
1341
|
+
hydrationTime.recordMs(elapsed);
|
|
1008
1342
|
}
|
|
1009
|
-
|
|
1010
|
-
|
|
1343
|
+
}
|
|
1344
|
+
function* generateRowChanges(slowHydrateThreshold) {
|
|
1345
|
+
yield* hydrateQueries(requiredQueries, false, slowHydrateThreshold);
|
|
1346
|
+
yield* hydrateQueries(optionalQueries, true, slowHydrateThreshold);
|
|
1011
1347
|
}
|
|
1012
1348
|
await this.#processChanges(lc, timer, generateRowChanges(this.#slowHydrateThreshold), updater, pokers);
|
|
1349
|
+
const abortedQueries = [...timedOutQueries, ...rejectedQueries];
|
|
1350
|
+
if (abortedQueries.length > 0) {
|
|
1351
|
+
const abortedQueryIDs = abortedQueries.map(({ id }) => id);
|
|
1352
|
+
for (const patch of await updater.abortExecutedQueries(lc, abortedQueryIDs)) await pokers.addPatch(patch);
|
|
1353
|
+
for (const queryID of abortedQueryIDs) {
|
|
1354
|
+
this.#pipelines.removeQuery(queryID);
|
|
1355
|
+
this.#inspectorDelegate.removeQuery(queryID);
|
|
1356
|
+
this.#queryReplacements.delete(queryID);
|
|
1357
|
+
}
|
|
1358
|
+
if (timedOutQueries.length > 0) this.#queryEvictions.add(timedOutQueries.length, { reason: "hydration-timeout" });
|
|
1359
|
+
if (rejectedQueries.length > 0) this.#queryEvictions.add(rejectedQueries.length, { reason: "hydration-circuit-breaker" });
|
|
1360
|
+
this.#sendHydrationTimeoutErrors(cvr, abortedQueries);
|
|
1361
|
+
}
|
|
1362
|
+
for (const patch of updater.removeTrackedQueries(budgetEvictedQueryIDs)) await pokers.addPatch(patch);
|
|
1363
|
+
for (const queryID of budgetEvictedQueryIDs) {
|
|
1364
|
+
this.#pipelines.removeQuery(queryID);
|
|
1365
|
+
this.#inspectorDelegate.removeQuery(queryID);
|
|
1366
|
+
this.#queryReplacements.delete(queryID);
|
|
1367
|
+
}
|
|
1368
|
+
this.#recordHydrationBudgetExhaustion(lc, hydrationBudget, hydrationPassStats.activeHydratedQueries, hydrationPassStats.inactiveHydratedQueries, budgetEvictedQueryIDs);
|
|
1013
1369
|
this.#logQueryCoverageShadowSummary(lc, "add", totalHydratedQueries, coveredHydratedQueries, firstCoveredQuery);
|
|
1014
1370
|
await startAsyncSpan(tracer, "vs.#syncQueryPipelineSet.deleteUnreferencedRows", async () => {
|
|
1015
1371
|
for (const patch of await updater.deleteUnreferencedRows(lc)) await pokers.addPatch(patch);
|
|
1016
1372
|
});
|
|
1017
1373
|
this.#cvr = await this.#flushUpdater(lc, updater);
|
|
1374
|
+
if (budgetEvictedQueryIDs.length > 0) this.#scheduleExpireEviction(lc, this.#cvr);
|
|
1018
1375
|
const finalVersion = this.#cvr.version;
|
|
1019
|
-
await this.#catchupClients(lc, cvr, finalVersion,
|
|
1376
|
+
await this.#catchupClients(lc, cvr, finalVersion, hydratedQueryIDs, pokers);
|
|
1020
1377
|
await startAsyncSpan(tracer, "vs.#syncQueryPipelineSet.pokeEnd", () => pokers.end(finalVersion));
|
|
1021
1378
|
this.#markVersionServed(stateVersion);
|
|
1022
1379
|
const wallTime = performance.now() - start;
|
|
@@ -1157,7 +1514,9 @@ var ViewSyncerService = class {
|
|
|
1157
1514
|
*
|
|
1158
1515
|
* Must be called from within the #lock.
|
|
1159
1516
|
*
|
|
1160
|
-
* Returns
|
|
1517
|
+
* Returns 'success' if changes were successfully processed and poked,
|
|
1518
|
+
* or a `ResetPipelinesSignal` if advancement aborted (e.g. schema change,
|
|
1519
|
+
* timeout) and pipelines need to be reset and rehydrated.
|
|
1161
1520
|
*/
|
|
1162
1521
|
#advancePipelines(lc, cvr) {
|
|
1163
1522
|
return startAsyncSpan(tracer, "vs.#advancePipelines", async (span) => {
|
|
@@ -1165,20 +1524,27 @@ var ViewSyncerService = class {
|
|
|
1165
1524
|
assert(this.#pipelines.initialized(), "pipelines must be initialized (advancePipelines");
|
|
1166
1525
|
const start = performance.now();
|
|
1167
1526
|
const timer = new TimeSliceTimer(lc);
|
|
1168
|
-
|
|
1169
|
-
|
|
1170
|
-
|
|
1171
|
-
|
|
1172
|
-
lc.debug?.(`applying ${numChanges} to advance to ${version}`);
|
|
1527
|
+
let pokers;
|
|
1528
|
+
let updater;
|
|
1529
|
+
let version;
|
|
1530
|
+
let numChanges = 0;
|
|
1173
1531
|
try {
|
|
1174
|
-
|
|
1532
|
+
const advancement = this.#pipelines.advance(timer);
|
|
1533
|
+
version = advancement.version;
|
|
1534
|
+
numChanges = advancement.numChanges;
|
|
1535
|
+
lc = lc.withContext("newVersion", version);
|
|
1536
|
+
updater = new CVRQueryDrivenUpdater(this.#cvrStore, cvr, version, this.#pipelines.replicaVersion, (queryID) => this.#pipelines.rowSetSignature(queryID));
|
|
1537
|
+
pokers = startPoke(this.#getClients(cvr.version), updater.updatedVersion());
|
|
1538
|
+
lc.debug?.(`applying ${numChanges} to advance to ${version}`);
|
|
1539
|
+
await this.#processChanges(lc, await timer.start(), advancement.changes, updater, pokers);
|
|
1175
1540
|
} catch (e) {
|
|
1176
1541
|
if (e instanceof ResetPipelinesSignal) {
|
|
1177
|
-
await pokers
|
|
1542
|
+
await pokers?.cancel();
|
|
1178
1543
|
return e;
|
|
1179
1544
|
}
|
|
1180
1545
|
throw e;
|
|
1181
1546
|
}
|
|
1547
|
+
assert(updater && pokers && version !== void 0, "advancement state missing");
|
|
1182
1548
|
this.#cvr = await this.#flushUpdater(lc, updater);
|
|
1183
1549
|
const finalVersion = this.#cvr.version;
|
|
1184
1550
|
await startAsyncSpan(tracer, "vs.#advancePipelines.pokeEnd", () => pokers.end(finalVersion));
|
|
@@ -1199,7 +1565,7 @@ var ViewSyncerService = class {
|
|
|
1199
1565
|
clientID,
|
|
1200
1566
|
wsID: client.wsID
|
|
1201
1567
|
});
|
|
1202
|
-
return handleInspect(lc, body, cvr, client, this.#inspectorDelegate, this.id, this.#cvrStore, this.#config, connCtx);
|
|
1568
|
+
return handleInspect(lc, body, cvr, client, this.#inspectorDelegate, this.id, this, this.#cvrStore, this.#config, connCtx);
|
|
1203
1569
|
};
|
|
1204
1570
|
async #runBackgroundRetransform(lc) {
|
|
1205
1571
|
const attemptRetransform = async (connCtx) => {
|
|
@@ -1286,15 +1652,18 @@ var ViewSyncerService = class {
|
|
|
1286
1652
|
return this.#stopped.promise;
|
|
1287
1653
|
}
|
|
1288
1654
|
async #cleanup(err) {
|
|
1655
|
+
this.#shuttingDown = true;
|
|
1289
1656
|
this.connContextManager.setSharedRetransformReady(false);
|
|
1290
1657
|
this.#stopTTLClockInterval();
|
|
1291
1658
|
this.#stopExpireTimer();
|
|
1292
1659
|
this.#stopAuthMaintenanceTimer();
|
|
1660
|
+
this.#stopShutdownTimer();
|
|
1293
1661
|
this.#customQueryTransformer?.destroy();
|
|
1294
1662
|
for (const client of this.#clients.values()) if (err) client.fail(err);
|
|
1295
1663
|
else client.close(`closed clientGroupID=${this.id}`);
|
|
1296
1664
|
await this.#lock.withLock(() => {});
|
|
1297
1665
|
this.#pipelines.destroy();
|
|
1666
|
+
this.#inspectorDelegate.clearAuthenticated(this.id, this);
|
|
1298
1667
|
}
|
|
1299
1668
|
/**
|
|
1300
1669
|
* Test helper: Manually mark initialization as complete.
|
|
@@ -1330,8 +1699,10 @@ function isTransformFailedError(error) {
|
|
|
1330
1699
|
return error.errorBody.kind === "TransformFailed" && !isAuthErrorBody(error.errorBody);
|
|
1331
1700
|
}
|
|
1332
1701
|
/**
|
|
1333
|
-
* A query must be expired for all clients
|
|
1334
|
-
* expired.
|
|
1702
|
+
* Whether a query's TTL has elapsed. A query must be expired for all clients
|
|
1703
|
+
* in order to be considered expired.
|
|
1704
|
+
*
|
|
1705
|
+
* Exported for testing.
|
|
1335
1706
|
*/
|
|
1336
1707
|
function expired(ttlClock, q) {
|
|
1337
1708
|
if (q.type === "internal") return false;
|