@rocicorp/zero 1.10.0-canary.22 → 1.10.0-canary.24
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/out/analyze-query/src/analyze-cli.js +2 -2
- package/out/analyze-query/src/analyze-cli.js.map +1 -1
- package/out/analyze-query/src/bin-analyze.js +9 -8
- package/out/analyze-query/src/bin-analyze.js.map +1 -1
- package/out/replicache/src/btree/node.d.ts.map +1 -1
- package/out/replicache/src/btree/node.js +3 -6
- package/out/replicache/src/btree/node.js.map +1 -1
- package/out/replicache/src/dag/gc.d.ts.map +1 -1
- package/out/replicache/src/dag/gc.js +6 -10
- package/out/replicache/src/dag/gc.js.map +1 -1
- package/out/replicache/src/kv/sqlite-store.d.ts.map +1 -1
- package/out/replicache/src/kv/sqlite-store.js +24 -2
- package/out/replicache/src/kv/sqlite-store.js.map +1 -1
- package/out/replicache/src/persist/persist.js +1 -1
- package/out/shared/src/arrays.d.ts +5 -0
- package/out/shared/src/arrays.d.ts.map +1 -1
- package/out/shared/src/arrays.js +8 -10
- package/out/shared/src/arrays.js.map +1 -1
- package/out/shared/src/map.d.ts +19 -2
- package/out/shared/src/map.d.ts.map +1 -1
- package/out/shared/src/map.js +18 -4
- package/out/shared/src/map.js.map +1 -1
- package/out/z2s/src/compiler.js +1 -1
- package/out/z2s/src/sql.d.ts.map +1 -1
- package/out/z2s/src/sql.js +2 -2
- package/out/z2s/src/sql.js.map +1 -1
- package/out/zero/package.js +1 -1
- package/out/zero/package.js.map +1 -1
- package/out/zero-cache/src/auth/write-authorizer.d.ts.map +1 -1
- package/out/zero-cache/src/auth/write-authorizer.js +1 -0
- package/out/zero-cache/src/auth/write-authorizer.js.map +1 -1
- package/out/zero-cache/src/config/zero-config.d.ts +23 -0
- package/out/zero-cache/src/config/zero-config.d.ts.map +1 -1
- package/out/zero-cache/src/config/zero-config.js +63 -1
- package/out/zero-cache/src/config/zero-config.js.map +1 -1
- package/out/zero-cache/src/db/lite-tables.d.ts.map +1 -1
- package/out/zero-cache/src/db/lite-tables.js +3 -4
- package/out/zero-cache/src/db/lite-tables.js.map +1 -1
- package/out/zero-cache/src/observability/metrics.d.ts.map +1 -1
- package/out/zero-cache/src/observability/metrics.js +2 -7
- package/out/zero-cache/src/observability/metrics.js.map +1 -1
- package/out/zero-cache/src/server/change-streamer.d.ts.map +1 -1
- package/out/zero-cache/src/server/change-streamer.js +11 -3
- package/out/zero-cache/src/server/change-streamer.js.map +1 -1
- package/out/zero-cache/src/server/inspector-delegate.d.ts +18 -2
- package/out/zero-cache/src/server/inspector-delegate.d.ts.map +1 -1
- package/out/zero-cache/src/server/inspector-delegate.js +22 -15
- package/out/zero-cache/src/server/inspector-delegate.js.map +1 -1
- package/out/zero-cache/src/server/main.d.ts.map +1 -1
- package/out/zero-cache/src/server/main.js +8 -1
- package/out/zero-cache/src/server/main.js.map +1 -1
- package/out/zero-cache/src/server/mutator.js +2 -0
- package/out/zero-cache/src/server/mutator.js.map +1 -1
- package/out/zero-cache/src/server/reaper.d.ts.map +1 -1
- package/out/zero-cache/src/server/reaper.js +2 -0
- package/out/zero-cache/src/server/reaper.js.map +1 -1
- package/out/zero-cache/src/server/replicator.d.ts.map +1 -1
- package/out/zero-cache/src/server/replicator.js +2 -0
- package/out/zero-cache/src/server/replicator.js.map +1 -1
- package/out/zero-cache/src/server/runner/zero-dispatcher.d.ts.map +1 -1
- package/out/zero-cache/src/server/runner/zero-dispatcher.js +3 -0
- package/out/zero-cache/src/server/runner/zero-dispatcher.js.map +1 -1
- package/out/zero-cache/src/server/syncer.d.ts.map +1 -1
- package/out/zero-cache/src/server/syncer.js +5 -1
- package/out/zero-cache/src/server/syncer.js.map +1 -1
- package/out/zero-cache/src/services/analyze.d.ts.map +1 -1
- package/out/zero-cache/src/services/analyze.js +11 -9
- package/out/zero-cache/src/services/analyze.js.map +1 -1
- package/out/zero-cache/src/services/change-source/common/backfill-manager.d.ts.map +1 -1
- package/out/zero-cache/src/services/change-source/common/backfill-manager.js +14 -1
- package/out/zero-cache/src/services/change-source/common/backfill-manager.js.map +1 -1
- package/out/zero-cache/src/services/change-source/common/change-stream-multiplexer.d.ts.map +1 -1
- package/out/zero-cache/src/services/change-source/common/change-stream-multiplexer.js +15 -3
- package/out/zero-cache/src/services/change-source/common/change-stream-multiplexer.js.map +1 -1
- package/out/zero-cache/src/services/change-source/custom/change-source.js +2 -2
- package/out/zero-cache/src/services/change-source/custom/change-source.js.map +1 -1
- package/out/zero-cache/src/services/change-source/pg/initial-sync.js +1 -1
- package/out/zero-cache/src/services/change-streamer/change-streamer-http.d.ts +3 -0
- package/out/zero-cache/src/services/change-streamer/change-streamer-http.d.ts.map +1 -1
- package/out/zero-cache/src/services/change-streamer/change-streamer-http.js +6 -4
- package/out/zero-cache/src/services/change-streamer/change-streamer-http.js.map +1 -1
- package/out/zero-cache/src/services/mutagen/mutagen.d.ts +5 -0
- package/out/zero-cache/src/services/mutagen/mutagen.d.ts.map +1 -1
- package/out/zero-cache/src/services/mutagen/mutagen.js +7 -1
- package/out/zero-cache/src/services/mutagen/mutagen.js.map +1 -1
- package/out/zero-cache/src/services/mutagen/pusher.d.ts +32 -32
- package/out/zero-cache/src/services/mutagen/pusher.d.ts.map +1 -1
- package/out/zero-cache/src/services/mutagen/pusher.js +5 -6
- package/out/zero-cache/src/services/mutagen/pusher.js.map +1 -1
- package/out/zero-cache/src/services/profz.d.ts +7 -0
- package/out/zero-cache/src/services/profz.d.ts.map +1 -0
- package/out/zero-cache/src/services/profz.js +103 -0
- package/out/zero-cache/src/services/profz.js.map +1 -0
- package/out/zero-cache/src/services/run-ast.d.ts +12 -0
- package/out/zero-cache/src/services/run-ast.d.ts.map +1 -1
- package/out/zero-cache/src/services/run-ast.js +26 -14
- package/out/zero-cache/src/services/run-ast.js.map +1 -1
- package/out/zero-cache/src/services/runner.d.ts.map +1 -1
- package/out/zero-cache/src/services/runner.js +1 -1
- package/out/zero-cache/src/services/runner.js.map +1 -1
- package/out/zero-cache/src/services/view-syncer/cvr.d.ts +45 -0
- package/out/zero-cache/src/services/view-syncer/cvr.d.ts.map +1 -1
- package/out/zero-cache/src/services/view-syncer/cvr.js +133 -1
- package/out/zero-cache/src/services/view-syncer/cvr.js.map +1 -1
- package/out/zero-cache/src/services/view-syncer/drain-coordinator.d.ts +14 -1
- package/out/zero-cache/src/services/view-syncer/drain-coordinator.d.ts.map +1 -1
- package/out/zero-cache/src/services/view-syncer/drain-coordinator.js +30 -4
- package/out/zero-cache/src/services/view-syncer/drain-coordinator.js.map +1 -1
- package/out/zero-cache/src/services/view-syncer/hydration-budget.d.ts +33 -0
- package/out/zero-cache/src/services/view-syncer/hydration-budget.d.ts.map +1 -0
- package/out/zero-cache/src/services/view-syncer/hydration-budget.js +62 -0
- package/out/zero-cache/src/services/view-syncer/hydration-budget.js.map +1 -0
- package/out/zero-cache/src/services/view-syncer/hydration-circuit-breaker.d.ts +51 -0
- package/out/zero-cache/src/services/view-syncer/hydration-circuit-breaker.d.ts.map +1 -0
- package/out/zero-cache/src/services/view-syncer/hydration-circuit-breaker.js +78 -0
- package/out/zero-cache/src/services/view-syncer/hydration-circuit-breaker.js.map +1 -0
- package/out/zero-cache/src/services/view-syncer/inspect-handler.d.ts +1 -1
- package/out/zero-cache/src/services/view-syncer/inspect-handler.d.ts.map +1 -1
- package/out/zero-cache/src/services/view-syncer/inspect-handler.js +2 -2
- package/out/zero-cache/src/services/view-syncer/inspect-handler.js.map +1 -1
- package/out/zero-cache/src/services/view-syncer/pipeline-driver.d.ts +2 -1
- package/out/zero-cache/src/services/view-syncer/pipeline-driver.d.ts.map +1 -1
- package/out/zero-cache/src/services/view-syncer/pipeline-driver.js +42 -17
- package/out/zero-cache/src/services/view-syncer/pipeline-driver.js.map +1 -1
- package/out/zero-cache/src/services/view-syncer/query-covering.d.ts.map +1 -1
- package/out/zero-cache/src/services/view-syncer/query-covering.js +2 -6
- package/out/zero-cache/src/services/view-syncer/query-covering.js.map +1 -1
- package/out/zero-cache/src/services/view-syncer/snapshot-row-cache.d.ts +95 -0
- package/out/zero-cache/src/services/view-syncer/snapshot-row-cache.d.ts.map +1 -0
- package/out/zero-cache/src/services/view-syncer/snapshot-row-cache.js +155 -0
- package/out/zero-cache/src/services/view-syncer/snapshot-row-cache.js.map +1 -0
- package/out/zero-cache/src/services/view-syncer/snapshotter.d.ts +42 -4
- package/out/zero-cache/src/services/view-syncer/snapshotter.d.ts.map +1 -1
- package/out/zero-cache/src/services/view-syncer/snapshotter.js +117 -28
- package/out/zero-cache/src/services/view-syncer/snapshotter.js.map +1 -1
- package/out/zero-cache/src/services/view-syncer/view-syncer.d.ts +11 -1
- package/out/zero-cache/src/services/view-syncer/view-syncer.d.ts.map +1 -1
- package/out/zero-cache/src/services/view-syncer/view-syncer.js +466 -95
- package/out/zero-cache/src/services/view-syncer/view-syncer.js.map +1 -1
- package/out/zero-cache/src/types/processes.d.ts +32 -0
- package/out/zero-cache/src/types/processes.d.ts.map +1 -1
- package/out/zero-cache/src/types/processes.js +43 -4
- package/out/zero-cache/src/types/processes.js.map +1 -1
- package/out/zero-cache/src/types/profiler.d.ts +12 -0
- package/out/zero-cache/src/types/profiler.d.ts.map +1 -1
- package/out/zero-cache/src/types/profiler.js +46 -1
- package/out/zero-cache/src/types/profiler.js.map +1 -1
- package/out/zero-cache/src/types/row-key.d.ts.map +1 -1
- package/out/zero-cache/src/types/row-key.js +5 -5
- package/out/zero-cache/src/types/row-key.js.map +1 -1
- package/out/zero-cache/src/workers/connection.js +3 -3
- package/out/zero-cache/src/workers/connection.js.map +1 -1
- package/out/zero-cache/src/workers/syncer-ws-message-handler.d.ts.map +1 -1
- package/out/zero-cache/src/workers/syncer-ws-message-handler.js +21 -16
- package/out/zero-cache/src/workers/syncer-ws-message-handler.js.map +1 -1
- package/out/zero-cache/src/workers/syncer.d.ts.map +1 -1
- package/out/zero-cache/src/workers/syncer.js +10 -1
- package/out/zero-cache/src/workers/syncer.js.map +1 -1
- package/out/zero-client/src/client/context.d.ts +16 -3
- package/out/zero-client/src/client/context.d.ts.map +1 -1
- package/out/zero-client/src/client/context.js +85 -14
- package/out/zero-client/src/client/context.js.map +1 -1
- package/out/zero-client/src/client/ivm-branch.d.ts.map +1 -1
- package/out/zero-client/src/client/ivm-branch.js +20 -3
- package/out/zero-client/src/client/ivm-branch.js.map +1 -1
- package/out/zero-client/src/client/query-manager.d.ts.map +1 -1
- package/out/zero-client/src/client/query-manager.js +2 -5
- package/out/zero-client/src/client/query-manager.js.map +1 -1
- package/out/zero-client/src/client/version.js +1 -1
- package/out/zero-client/src/client/zero-rep.d.ts.map +1 -1
- package/out/zero-client/src/client/zero-rep.js +1 -1
- package/out/zero-client/src/client/zero-rep.js.map +1 -1
- package/out/zero-client/src/client/zero.d.ts.map +1 -1
- package/out/zero-client/src/client/zero.js +2 -2
- package/out/zero-client/src/client/zero.js.map +1 -1
- package/out/zero-protocol/src/analyze-query-result.d.ts +5 -5
- package/out/zero-protocol/src/analyze-query-result.d.ts.map +1 -1
- package/out/zero-protocol/src/analyze-query-result.js +1 -1
- package/out/zero-protocol/src/analyze-query-result.js.map +1 -1
- package/out/zero-protocol/src/ast.d.ts.map +1 -1
- package/out/zero-protocol/src/ast.js +13 -14
- package/out/zero-protocol/src/ast.js.map +1 -1
- package/out/zero-protocol/src/down.d.ts +2 -2
- package/out/zero-protocol/src/error.d.ts +1 -1
- package/out/zero-protocol/src/inspect-down.d.ts +6 -6
- package/out/zero-protocol/src/protocol-version.d.ts.map +1 -1
- package/out/zero-protocol/src/protocol-version.js.map +1 -1
- package/out/zero-protocol/src/query-hash.d.ts.map +1 -1
- package/out/zero-protocol/src/query-hash.js +2 -5
- package/out/zero-protocol/src/query-hash.js.map +1 -1
- package/out/zero-solid/src/solid-view.d.ts +7 -0
- package/out/zero-solid/src/solid-view.d.ts.map +1 -1
- package/out/zero-solid/src/solid-view.js +13 -2
- package/out/zero-solid/src/solid-view.js.map +1 -1
- package/out/zql/src/builder/builder.d.ts +19 -0
- package/out/zql/src/builder/builder.d.ts.map +1 -1
- package/out/zql/src/builder/builder.js +28 -11
- package/out/zql/src/builder/builder.js.map +1 -1
- package/out/zql/src/builder/correlated-predicate-pushdown.d.ts +43 -0
- package/out/zql/src/builder/correlated-predicate-pushdown.d.ts.map +1 -0
- package/out/zql/src/builder/correlated-predicate-pushdown.js +161 -0
- package/out/zql/src/builder/correlated-predicate-pushdown.js.map +1 -0
- package/out/zql/src/builder/debug-delegate.d.ts +6 -1
- package/out/zql/src/builder/debug-delegate.d.ts.map +1 -1
- package/out/zql/src/builder/debug-delegate.js +14 -2
- package/out/zql/src/builder/debug-delegate.js.map +1 -1
- package/out/zql/src/ivm/array-view.d.ts +19 -0
- package/out/zql/src/ivm/array-view.d.ts.map +1 -1
- package/out/zql/src/ivm/array-view.js +39 -2
- package/out/zql/src/ivm/array-view.js.map +1 -1
- package/out/zql/src/ivm/connection-index.d.ts +68 -0
- package/out/zql/src/ivm/connection-index.d.ts.map +1 -0
- package/out/zql/src/ivm/connection-index.js +149 -0
- package/out/zql/src/ivm/connection-index.js.map +1 -0
- package/out/zql/src/ivm/exists.js +1 -1
- package/out/zql/src/ivm/filter-operators.d.ts +3 -2
- package/out/zql/src/ivm/filter-operators.d.ts.map +1 -1
- package/out/zql/src/ivm/filter-operators.js +19 -4
- package/out/zql/src/ivm/filter-operators.js.map +1 -1
- package/out/zql/src/ivm/memory-source.d.ts +13 -1
- package/out/zql/src/ivm/memory-source.d.ts.map +1 -1
- package/out/zql/src/ivm/memory-source.js +43 -4
- package/out/zql/src/ivm/memory-source.js.map +1 -1
- package/out/zql/src/ivm/operator.d.ts +40 -0
- package/out/zql/src/ivm/operator.d.ts.map +1 -1
- package/out/zql/src/ivm/operator.js.map +1 -1
- package/out/zql/src/ivm/view.d.ts +15 -0
- package/out/zql/src/ivm/view.d.ts.map +1 -1
- package/out/zql/src/planner/planner-builder.d.ts +12 -3
- package/out/zql/src/planner/planner-builder.d.ts.map +1 -1
- package/out/zql/src/planner/planner-builder.js +40 -20
- package/out/zql/src/planner/planner-builder.js.map +1 -1
- package/out/zql/src/planner/planner-connection.d.ts +28 -2
- package/out/zql/src/planner/planner-connection.d.ts.map +1 -1
- package/out/zql/src/planner/planner-connection.js +108 -8
- package/out/zql/src/planner/planner-connection.js.map +1 -1
- package/out/zql/src/planner/planner-debug.d.ts +1 -1
- package/out/zql/src/planner/planner-debug.d.ts.map +1 -1
- package/out/zql/src/planner/planner-debug.js +3 -8
- package/out/zql/src/planner/planner-debug.js.map +1 -1
- package/out/zql/src/planner/planner-fan-out.d.ts +1 -0
- package/out/zql/src/planner/planner-fan-out.d.ts.map +1 -1
- package/out/zql/src/planner/planner-fan-out.js +3 -0
- package/out/zql/src/planner/planner-fan-out.js.map +1 -1
- package/out/zql/src/planner/planner-filter.d.ts +59 -0
- package/out/zql/src/planner/planner-filter.d.ts.map +1 -0
- package/out/zql/src/planner/planner-filter.js +116 -0
- package/out/zql/src/planner/planner-filter.js.map +1 -0
- package/out/zql/src/planner/planner-graph.d.ts +2 -0
- package/out/zql/src/planner/planner-graph.d.ts.map +1 -1
- package/out/zql/src/planner/planner-graph.js +2 -0
- package/out/zql/src/planner/planner-graph.js.map +1 -1
- package/out/zql/src/planner/planner-join.d.ts +2 -0
- package/out/zql/src/planner/planner-join.d.ts.map +1 -1
- package/out/zql/src/planner/planner-join.js +7 -0
- package/out/zql/src/planner/planner-join.js.map +1 -1
- package/out/zql/src/planner/planner-node.d.ts +2 -1
- package/out/zql/src/planner/planner-node.d.ts.map +1 -1
- package/out/zql/src/planner/planner-node.js.map +1 -1
- package/out/zql/src/planner/planner-source.d.ts +2 -2
- package/out/zql/src/planner/planner-source.d.ts.map +1 -1
- package/out/zql/src/planner/planner-source.js +2 -2
- package/out/zql/src/planner/planner-source.js.map +1 -1
- package/out/zql/src/query/query-delegate-base.d.ts +2 -2
- package/out/zql/src/query/query-delegate-base.d.ts.map +1 -1
- package/out/zql/src/query/query-delegate-base.js +16 -10
- package/out/zql/src/query/query-delegate-base.js.map +1 -1
- package/out/zql/src/query/query-delegate.d.ts +22 -5
- package/out/zql/src/query/query-delegate.d.ts.map +1 -1
- package/out/zql/src/query/query-impl.d.ts.map +1 -1
- package/out/zql/src/query/query-impl.js +6 -23
- package/out/zql/src/query/query-impl.js.map +1 -1
- package/out/zql/src/query/query-transitions.d.ts.map +1 -1
- package/out/zql/src/query/query-transitions.js +2 -5
- package/out/zql/src/query/query-transitions.js.map +1 -1
- package/out/zql/src/query/query.d.ts +4 -1
- package/out/zql/src/query/query.d.ts.map +1 -1
- package/out/zql/src/query/runnable-query-impl.d.ts.map +1 -1
- package/out/zql/src/query/runnable-query-impl.js +2 -10
- package/out/zql/src/query/runnable-query-impl.js.map +1 -1
- package/out/zqlite/src/db.d.ts +9 -1
- package/out/zqlite/src/db.d.ts.map +1 -1
- package/out/zqlite/src/db.js +8 -2
- package/out/zqlite/src/db.js.map +1 -1
- package/out/zqlite/src/query-builder.d.ts +1 -1
- package/out/zqlite/src/query-builder.d.ts.map +1 -1
- package/out/zqlite/src/query-builder.js +2 -1
- package/out/zqlite/src/query-builder.js.map +1 -1
- package/out/zqlite/src/query-delegate.d.ts.map +1 -1
- package/out/zqlite/src/query-delegate.js +5 -6
- package/out/zqlite/src/query-delegate.js.map +1 -1
- package/out/zqlite/src/sqlite-cost-model.d.ts.map +1 -1
- package/out/zqlite/src/sqlite-cost-model.js +2 -30
- package/out/zqlite/src/sqlite-cost-model.js.map +1 -1
- package/out/zqlite/src/sqlite-stat-fanout.d.ts.map +1 -1
- package/out/zqlite/src/sqlite-stat-fanout.js +3 -5
- package/out/zqlite/src/sqlite-stat-fanout.js.map +1 -1
- package/out/zqlite/src/table-source.d.ts +15 -2
- package/out/zqlite/src/table-source.d.ts.map +1 -1
- package/out/zqlite/src/table-source.js +28 -12
- package/out/zqlite/src/table-source.js.map +1 -1
- package/package.json +1 -1
|
@@ -0,0 +1,155 @@
|
|
|
1
|
+
import { getOrInsert } from "../../../../shared/src/map.js";
|
|
2
|
+
import { must } from "../../../../shared/src/must.js";
|
|
3
|
+
import { stringify } from "../../../../shared/src/bigint-json.js";
|
|
4
|
+
import { getOrCreateCounter } from "../../observability/metrics.js";
|
|
5
|
+
import { RingBuffer } from "../../../../shared/src/ring-buffer.js";
|
|
6
|
+
//#region ../zero-cache/src/services/view-syncer/snapshot-row-cache.ts
|
|
7
|
+
/**
|
|
8
|
+
* The default maximum number of entries retained by a {@link SnapshotRowCache}.
|
|
9
|
+
*
|
|
10
|
+
* Each entry is a raw SQLite row (or the list of rows conflicting with
|
|
11
|
+
* a row on its unique keys) read while computing a snapshot diff. A
|
|
12
|
+
* replicated transaction of N rows produces up to 2N entries (the new value
|
|
13
|
+
* and the previous value(s) of each row), so the cache should be large enough
|
|
14
|
+
* to hold the largest transactions that are commonly replicated in order for
|
|
15
|
+
* all of the client groups on a worker to share the reads.
|
|
16
|
+
*/
|
|
17
|
+
var DEFAULT_MAX_SNAPSHOT_ROW_CACHE_ENTRIES = 5e4;
|
|
18
|
+
/**
|
|
19
|
+
* Serializes a lookup key value (i.e. a SQLite-bound parameter). Numbers and
|
|
20
|
+
* bigints serialize identically, as row keys parsed from the change log are
|
|
21
|
+
* JSON numbers whereas the values read from SQLite are bigints; both address
|
|
22
|
+
* the same row. Strings are prefixed to distinguish them from numbers.
|
|
23
|
+
*/
|
|
24
|
+
function serialize(v) {
|
|
25
|
+
switch (typeof v) {
|
|
26
|
+
case "string": return "s" + v;
|
|
27
|
+
case "number":
|
|
28
|
+
case "bigint": return String(v);
|
|
29
|
+
case "boolean": return v ? "true" : "false";
|
|
30
|
+
case "object": return v === null ? "null" : "o" + stringify(v);
|
|
31
|
+
default: return "u" + String(v);
|
|
32
|
+
}
|
|
33
|
+
}
|
|
34
|
+
var HIT = { result: "hit" };
|
|
35
|
+
var MISS = { result: "miss" };
|
|
36
|
+
/**
|
|
37
|
+
* A worker-wide, bounded cache of the rows that a `Snapshotter` reads from
|
|
38
|
+
* replica snapshots when computing the diff between two versions of the
|
|
39
|
+
* replica.
|
|
40
|
+
*
|
|
41
|
+
* Every client group (i.e. view-syncer) on a sync worker computes its own
|
|
42
|
+
* diff from the shared `_zero.changeLog2`, and each change in the diff costs
|
|
43
|
+
* two SQLite reads: the new value of the row (from the `curr` snapshot) and
|
|
44
|
+
* the previous value(s) that it replaces (from the `prev` snapshot). The
|
|
45
|
+
* reads for a given change are identical for every client group that
|
|
46
|
+
* traverses it, so a replicated transaction of N rows costs N × 2 × (client
|
|
47
|
+
* groups) reads per worker, most of which are redundant. This cache lets the
|
|
48
|
+
* client groups on a worker share those reads.
|
|
49
|
+
*
|
|
50
|
+
* ### Keys
|
|
51
|
+
*
|
|
52
|
+
* Entries are keyed by the exact SQL statement and arguments of the read
|
|
53
|
+
* (which encodes the table, the selected columns and the lookup keys), plus
|
|
54
|
+
* a `tag` supplied by the caller that pins the result to a version of the
|
|
55
|
+
* replica:
|
|
56
|
+
*
|
|
57
|
+
* - The new value of a row is the row as of the `stateVersion` recorded in
|
|
58
|
+
* its change log entry (its `_0_version` is asserted to equal that
|
|
59
|
+
* version), and is tagged with that `stateVersion`. It is thus shareable
|
|
60
|
+
* by all client groups regardless of the snapshots they are advancing
|
|
61
|
+
* between.
|
|
62
|
+
* - Previous values are read from the `prev` snapshot at its version, and
|
|
63
|
+
* are tagged with that version (and, when the result can depend on the
|
|
64
|
+
* changes applied to the snapshot before the read, the `curr` version
|
|
65
|
+
* as well). See `Diff` in `snapshotter.ts` for the details.
|
|
66
|
+
*
|
|
67
|
+
* Because every key includes the version(s) that the result depends on,
|
|
68
|
+
* entries never go stale and are never invalidated; they are only evicted.
|
|
69
|
+
*
|
|
70
|
+
* ### Eviction
|
|
71
|
+
*
|
|
72
|
+
* The cache is bounded by entry count and evicts in insertion (FIFO) order
|
|
73
|
+
* rather than by recency. Diffs are traversed in change log order by every
|
|
74
|
+
* client group, so the entries that will not be needed again are those that
|
|
75
|
+
* were inserted the earliest (all groups have moved past them), while the
|
|
76
|
+
* most recently inserted entries are the ones that the trailing groups have
|
|
77
|
+
* yet to reach. Refreshing an entry on a hit would evict exactly the wrong
|
|
78
|
+
* entries when a trailing group catches up.
|
|
79
|
+
*
|
|
80
|
+
* ### Sharing
|
|
81
|
+
*
|
|
82
|
+
* Cached values are shared by reference across client groups and must
|
|
83
|
+
* therefore never be mutated by callers.
|
|
84
|
+
*/
|
|
85
|
+
var SnapshotRowCache = class {
|
|
86
|
+
#maxEntries;
|
|
87
|
+
#entries = /* @__PURE__ */ new Map();
|
|
88
|
+
#keys = new RingBuffer();
|
|
89
|
+
#sqlIDs = /* @__PURE__ */ new Map();
|
|
90
|
+
#reads = getOrCreateCounter("sync", "ivm.snapshot-row-reads", "Row reads performed while computing snapshot diffs for IVM, labeled by whether the read was served by the worker-wide snapshot row cache.");
|
|
91
|
+
#hits = 0;
|
|
92
|
+
#misses = 0;
|
|
93
|
+
constructor(maxEntries = DEFAULT_MAX_SNAPSHOT_ROW_CACHE_ENTRIES) {
|
|
94
|
+
this.#maxEntries = Math.max(0, maxEntries);
|
|
95
|
+
}
|
|
96
|
+
get size() {
|
|
97
|
+
return this.#entries.size;
|
|
98
|
+
}
|
|
99
|
+
get maxEntries() {
|
|
100
|
+
return this.#maxEntries;
|
|
101
|
+
}
|
|
102
|
+
stats() {
|
|
103
|
+
return {
|
|
104
|
+
hits: this.#hits,
|
|
105
|
+
misses: this.#misses,
|
|
106
|
+
size: this.#entries.size
|
|
107
|
+
};
|
|
108
|
+
}
|
|
109
|
+
/**
|
|
110
|
+
* Returns the cached result of the read identified by `tag`, `sql`, `mode`
|
|
111
|
+
* and `args`, or performs the read via `read()`, caching and returning its
|
|
112
|
+
* result. `undefined` results (i.e. a missing row) are not cached.
|
|
113
|
+
*
|
|
114
|
+
* `mode` is part of the key because the same statement can be read both
|
|
115
|
+
* ways: for a table whose only unique key is its primary key, `getRow()`
|
|
116
|
+
* and `getRows()` produce identical SQL, args and tags, but the former
|
|
117
|
+
* caches a row and the latter an array of rows.
|
|
118
|
+
*/
|
|
119
|
+
getOrRead(tag, sql, mode, args, read) {
|
|
120
|
+
const key = this.#key(tag, sql, mode, args);
|
|
121
|
+
const cached = this.#entries.get(key);
|
|
122
|
+
if (cached !== void 0) {
|
|
123
|
+
this.#hits++;
|
|
124
|
+
this.#reads.add(1, HIT);
|
|
125
|
+
return cached;
|
|
126
|
+
}
|
|
127
|
+
this.#misses++;
|
|
128
|
+
this.#reads.add(1, MISS);
|
|
129
|
+
const value = read();
|
|
130
|
+
if (value !== void 0 && this.#maxEntries > 0) {
|
|
131
|
+
this.#entries.set(key, value);
|
|
132
|
+
this.#keys.push(key);
|
|
133
|
+
if (this.#keys.size > this.#maxEntries) this.#entries.delete(must(this.#keys.shift()));
|
|
134
|
+
}
|
|
135
|
+
return value;
|
|
136
|
+
}
|
|
137
|
+
clear() {
|
|
138
|
+
this.#entries.clear();
|
|
139
|
+
this.#keys.clear();
|
|
140
|
+
this.#sqlIDs.clear();
|
|
141
|
+
}
|
|
142
|
+
#key(tag, sql, mode, args) {
|
|
143
|
+
if (this.#sqlIDs.size >= 1e4 && !this.#sqlIDs.has(sql)) this.clear();
|
|
144
|
+
let key = `${tag}\0${mode}${getOrInsert(this.#sqlIDs, sql, this.#sqlIDs.size)}`;
|
|
145
|
+
for (const arg of args) {
|
|
146
|
+
const serialized = serialize(arg);
|
|
147
|
+
key += `\0${serialized.length}:${serialized}`;
|
|
148
|
+
}
|
|
149
|
+
return key;
|
|
150
|
+
}
|
|
151
|
+
};
|
|
152
|
+
//#endregion
|
|
153
|
+
export { SnapshotRowCache };
|
|
154
|
+
|
|
155
|
+
//# sourceMappingURL=snapshot-row-cache.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"snapshot-row-cache.js","names":["#maxEntries","#entries","#keys","#sqlIDs","#reads","#hits","#misses","#key"],"sources":["../../../../../../zero-cache/src/services/view-syncer/snapshot-row-cache.ts"],"sourcesContent":["import {stringify} from '../../../../shared/src/bigint-json.ts';\nimport {getOrInsert} from '../../../../shared/src/map.ts';\nimport {must} from '../../../../shared/src/must.ts';\nimport {RingBuffer} from '../../../../shared/src/ring-buffer.ts';\nimport {getOrCreateCounter} from '../../observability/metrics.ts';\n\n/**\n * The default maximum number of entries retained by a {@link SnapshotRowCache}.\n *\n * Each entry is a raw SQLite row (or the list of rows conflicting with\n * a row on its unique keys) read while computing a snapshot diff. A\n * replicated transaction of N rows produces up to 2N entries (the new value\n * and the previous value(s) of each row), so the cache should be large enough\n * to hold the largest transactions that are commonly replicated in order for\n * all of the client groups on a worker to share the reads.\n */\nexport const DEFAULT_MAX_SNAPSHOT_ROW_CACHE_ENTRIES = 50_000;\n\n/**\n * Serializes a lookup key value (i.e. a SQLite-bound parameter). Numbers and\n * bigints serialize identically, as row keys parsed from the change log are\n * JSON numbers whereas the values read from SQLite are bigints; both address\n * the same row. Strings are prefixed to distinguish them from numbers.\n */\nfunction serialize(v: unknown): string {\n switch (typeof v) {\n case 'string':\n return 's' + v;\n case 'number':\n case 'bigint':\n return String(v);\n case 'boolean':\n return v ? 'true' : 'false';\n case 'object':\n return v === null ? 'null' : 'o' + stringify(v);\n default:\n return 'u' + String(v);\n }\n}\n\n/**\n * How a statement's results are read: a single row (`Statement.get()`) or\n * all rows (`Statement.all()`).\n */\nexport type ReadMode = 'get' | 'all';\n\n/**\n * The maximum number of distinct SQL statements interned by a\n * {@link SnapshotRowCache} before it is cleared. There is one per table and\n * lookup shape, so this is only reached after many schema changes.\n */\nexport const MAX_INTERNED_STATEMENTS = 10_000;\n\nconst HIT = {result: 'hit'} as const;\nconst MISS = {result: 'miss'} as const;\n\n/**\n * A worker-wide, bounded cache of the rows that a `Snapshotter` reads from\n * replica snapshots when computing the diff between two versions of the\n * replica.\n *\n * Every client group (i.e. view-syncer) on a sync worker computes its own\n * diff from the shared `_zero.changeLog2`, and each change in the diff costs\n * two SQLite reads: the new value of the row (from the `curr` snapshot) and\n * the previous value(s) that it replaces (from the `prev` snapshot). The\n * reads for a given change are identical for every client group that\n * traverses it, so a replicated transaction of N rows costs N × 2 × (client\n * groups) reads per worker, most of which are redundant. This cache lets the\n * client groups on a worker share those reads.\n *\n * ### Keys\n *\n * Entries are keyed by the exact SQL statement and arguments of the read\n * (which encodes the table, the selected columns and the lookup keys), plus\n * a `tag` supplied by the caller that pins the result to a version of the\n * replica:\n *\n * - The new value of a row is the row as of the `stateVersion` recorded in\n * its change log entry (its `_0_version` is asserted to equal that\n * version), and is tagged with that `stateVersion`. It is thus shareable\n * by all client groups regardless of the snapshots they are advancing\n * between.\n * - Previous values are read from the `prev` snapshot at its version, and\n * are tagged with that version (and, when the result can depend on the\n * changes applied to the snapshot before the read, the `curr` version\n * as well). See `Diff` in `snapshotter.ts` for the details.\n *\n * Because every key includes the version(s) that the result depends on,\n * entries never go stale and are never invalidated; they are only evicted.\n *\n * ### Eviction\n *\n * The cache is bounded by entry count and evicts in insertion (FIFO) order\n * rather than by recency. Diffs are traversed in change log order by every\n * client group, so the entries that will not be needed again are those that\n * were inserted the earliest (all groups have moved past them), while the\n * most recently inserted entries are the ones that the trailing groups have\n * yet to reach. Refreshing an entry on a hit would evict exactly the wrong\n * entries when a trailing group catches up.\n *\n * ### Sharing\n *\n * Cached values are shared by reference across client groups and must\n * therefore never be mutated by callers.\n */\nexport class SnapshotRowCache {\n readonly #maxEntries: number;\n readonly #entries = new Map<string, unknown>();\n // The keys of #entries in insertion order, so that the oldest can be\n // evicted in O(1). (Finding the oldest key with a fresh Map iterator\n // instead costs O(evictions) per insert, as V8 leaves deleted entries in\n // the table until it is compacted and every new iterator walks over them.)\n readonly #keys = new RingBuffer<string>();\n // Interns the SQL text of each distinct read so that entry keys\n // do not repeat the (long) column list of every statement.\n readonly #sqlIDs = new Map<string, number>();\n readonly #reads = getOrCreateCounter(\n 'sync',\n 'ivm.snapshot-row-reads',\n 'Row reads performed while computing snapshot diffs for IVM, labeled by whether the read was served by the worker-wide snapshot row cache.',\n );\n #hits = 0;\n #misses = 0;\n\n constructor(maxEntries = DEFAULT_MAX_SNAPSHOT_ROW_CACHE_ENTRIES) {\n this.#maxEntries = Math.max(0, maxEntries);\n }\n\n get size(): number {\n return this.#entries.size;\n }\n\n get maxEntries(): number {\n return this.#maxEntries;\n }\n\n stats(): {hits: number; misses: number; size: number} {\n return {hits: this.#hits, misses: this.#misses, size: this.#entries.size};\n }\n\n /**\n * Returns the cached result of the read identified by `tag`, `sql`, `mode`\n * and `args`, or performs the read via `read()`, caching and returning its\n * result. `undefined` results (i.e. a missing row) are not cached.\n *\n * `mode` is part of the key because the same statement can be read both\n * ways: for a table whose only unique key is its primary key, `getRow()`\n * and `getRows()` produce identical SQL, args and tags, but the former\n * caches a row and the latter an array of rows.\n */\n getOrRead<T>(\n tag: string,\n sql: string,\n mode: ReadMode,\n args: unknown[],\n read: () => T,\n ): T {\n const key = this.#key(tag, sql, mode, args);\n const cached = this.#entries.get(key);\n if (cached !== undefined) {\n this.#hits++;\n this.#reads.add(1, HIT);\n return cached as T;\n }\n this.#misses++;\n this.#reads.add(1, MISS);\n const value = read();\n if (value !== undefined && this.#maxEntries > 0) {\n // `key` is not in #entries (this is a miss), so it is always added.\n this.#entries.set(key, value);\n this.#keys.push(key);\n if (this.#keys.size > this.#maxEntries) {\n this.#entries.delete(must(this.#keys.shift()));\n }\n }\n return value;\n }\n\n clear(): void {\n this.#entries.clear();\n this.#keys.clear();\n this.#sqlIDs.clear();\n }\n\n #key(tag: string, sql: string, mode: ReadMode, args: unknown[]): string {\n if (\n this.#sqlIDs.size >= MAX_INTERNED_STATEMENTS &&\n !this.#sqlIDs.has(sql)\n ) {\n // Statements are only added (e.g. when schema changes produce new\n // column lists), so bound them by starting over. The entries refer to\n // statements by ID and are cleared with them.\n this.clear();\n }\n const sqlID = getOrInsert(this.#sqlIDs, sql, this.#sqlIDs.size);\n let key = `${tag}\\0${mode}${sqlID}`;\n // Include each value's length so separators inside string values cannot\n // make different argument arrays produce the same key.\n for (const arg of args) {\n const serialized = serialize(arg);\n key += `\\0${serialized.length}:${serialized}`;\n }\n return key;\n }\n}\n"],"mappings":";;;;;;;;;;;;;;;;AAgBA,IAAa,yCAAyC;;;;;;;AAQtD,SAAS,UAAU,GAAoB;CACrC,QAAQ,OAAO,GAAf;EACE,KAAK,UACH,OAAO,MAAM;EACf,KAAK;EACL,KAAK,UACH,OAAO,OAAO,CAAC;EACjB,KAAK,WACH,OAAO,IAAI,SAAS;EACtB,KAAK,UACH,OAAO,MAAM,OAAO,SAAS,MAAM,UAAU,CAAC;EAChD,SACE,OAAO,MAAM,OAAO,CAAC;CACzB;AACF;AAeA,IAAM,MAAM,EAAC,QAAQ,MAAK;AAC1B,IAAM,OAAO,EAAC,QAAQ,OAAM;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAmD5B,IAAa,mBAAb,MAA8B;CAC5B;CACA,2BAAoB,IAAI,IAAqB;CAK7C,QAAiB,IAAI,WAAmB;CAGxC,0BAAmB,IAAI,IAAoB;CAC3C,SAAkB,mBAChB,QACA,0BACA,2IACF;CACA,QAAQ;CACR,UAAU;CAEV,YAAY,aAAa,wCAAwC;EAC/D,KAAKA,cAAc,KAAK,IAAI,GAAG,UAAU;CAC3C;CAEA,IAAI,OAAe;EACjB,OAAO,KAAKC,SAAS;CACvB;CAEA,IAAI,aAAqB;EACvB,OAAO,KAAKD;CACd;CAEA,QAAsD;EACpD,OAAO;GAAC,MAAM,KAAKK;GAAO,QAAQ,KAAKC;GAAS,MAAM,KAAKL,SAAS;EAAI;CAC1E;;;;;;;;;;;CAYA,UACE,KACA,KACA,MACA,MACA,MACG;EACH,MAAM,MAAM,KAAKM,KAAK,KAAK,KAAK,MAAM,IAAI;EAC1C,MAAM,SAAS,KAAKN,SAAS,IAAI,GAAG;EACpC,IAAI,WAAW,KAAA,GAAW;GACxB,KAAKI;GACL,KAAKD,OAAO,IAAI,GAAG,GAAG;GACtB,OAAO;EACT;EACA,KAAKE;EACL,KAAKF,OAAO,IAAI,GAAG,IAAI;EACvB,MAAM,QAAQ,KAAK;EACnB,IAAI,UAAU,KAAA,KAAa,KAAKJ,cAAc,GAAG;GAE/C,KAAKC,SAAS,IAAI,KAAK,KAAK;GAC5B,KAAKC,MAAM,KAAK,GAAG;GACnB,IAAI,KAAKA,MAAM,OAAO,KAAKF,aACzB,KAAKC,SAAS,OAAO,KAAK,KAAKC,MAAM,MAAM,CAAC,CAAC;EAEjD;EACA,OAAO;CACT;CAEA,QAAc;EACZ,KAAKD,SAAS,MAAM;EACpB,KAAKC,MAAM,MAAM;EACjB,KAAKC,QAAQ,MAAM;CACrB;CAEA,KAAK,KAAa,KAAa,MAAgB,MAAyB;EACtE,IACE,KAAKA,QAAQ,QAAA,OACb,CAAC,KAAKA,QAAQ,IAAI,GAAG,GAKrB,KAAK,MAAM;EAGb,IAAI,MAAM,GAAG,IAAI,IAAI,OADP,YAAY,KAAKA,SAAS,KAAK,KAAKA,QAAQ,IAC9B;EAG5B,KAAK,MAAM,OAAO,MAAM;GACtB,MAAM,aAAa,UAAU,GAAG;GAChC,OAAO,KAAK,WAAW,OAAO,GAAG;EACnC;EACA,OAAO;CACT;AACF"}
|
|
@@ -6,6 +6,7 @@ import type { LiteAndZqlSpec, LiteTableSpecWithKeysAndVersion } from '../../db/s
|
|
|
6
6
|
import { StatementRunner } from '../../db/statements.ts';
|
|
7
7
|
import { type RowKey, type RowValue } from '../../types/row-key.ts';
|
|
8
8
|
import type { AppID } from '../../types/shards.ts';
|
|
9
|
+
import type { SnapshotRowCache } from './snapshot-row-cache.ts';
|
|
9
10
|
/**
|
|
10
11
|
* A `Snapshotter` manages the progression of database snapshots for a
|
|
11
12
|
* ViewSyncer.
|
|
@@ -67,7 +68,13 @@ import type { AppID } from '../../types/shards.ts';
|
|
|
67
68
|
*/
|
|
68
69
|
export declare class Snapshotter {
|
|
69
70
|
#private;
|
|
70
|
-
|
|
71
|
+
/**
|
|
72
|
+
* @param rowCache An optional worker-wide {@link SnapshotRowCache} through
|
|
73
|
+
* which the row reads performed when iterating over a
|
|
74
|
+
* {@link SnapshotDiff} are shared with the other Snapshotters
|
|
75
|
+
* (i.e. client groups) on the worker.
|
|
76
|
+
*/
|
|
77
|
+
constructor(lc: LogContext, dbFile: string, { appID }: AppID, pageCacheSizeKib?: number, rowCache?: SnapshotRowCache);
|
|
71
78
|
/**
|
|
72
79
|
* Initializes the snapshot to the current head of the database. This must be
|
|
73
80
|
* only be called once. The state of whether a Snapshotter has been initialized
|
|
@@ -114,7 +121,7 @@ export declare class Snapshotter {
|
|
|
114
121
|
* on `prev` before each iteration, and (2) rollback to the save point after
|
|
115
122
|
* the iteration.
|
|
116
123
|
*/
|
|
117
|
-
advance(syncableTables: Map<string, LiteAndZqlSpec>, allTableNames: Set<string
|
|
124
|
+
advance(syncableTables: Map<string, LiteAndZqlSpec>, allTableNames: Set<string>, observedTables?: TableFilter | undefined): SnapshotDiff;
|
|
118
125
|
advanceWithoutDiff(): {
|
|
119
126
|
prev: Snapshot;
|
|
120
127
|
curr: Snapshot;
|
|
@@ -137,6 +144,9 @@ export type Change = {
|
|
|
137
144
|
readonly nextValue: Readonly<Row> | null;
|
|
138
145
|
readonly rowKey: RowKey;
|
|
139
146
|
};
|
|
147
|
+
export interface TableFilter {
|
|
148
|
+
has(table: string): boolean;
|
|
149
|
+
}
|
|
140
150
|
/**
|
|
141
151
|
* Represents the difference between two database Snapshots.
|
|
142
152
|
* Iterating over the object will produce a sequence of {@link Change}s
|
|
@@ -184,12 +194,40 @@ declare class Snapshot {
|
|
|
184
194
|
readonly version: string;
|
|
185
195
|
constructor(db: StatementRunner, appID: string);
|
|
186
196
|
numChangesSince(prevVersion: string): any;
|
|
197
|
+
/**
|
|
198
|
+
* Whether any table-wide op (i.e. a RESET or TRUNCATE) was logged after
|
|
199
|
+
* `prevVersion`. Iterating over a diff ending at this snapshot will
|
|
200
|
+
* abort with a {@link ResetPipelinesSignal} if so.
|
|
201
|
+
*/
|
|
202
|
+
hasTableWideOpSince(prevVersion: string): boolean;
|
|
187
203
|
changesSince(prevVersion: string): {
|
|
188
204
|
changes: IterableIterator<unknown>;
|
|
189
205
|
cleanup: () => void;
|
|
190
206
|
};
|
|
191
|
-
|
|
192
|
-
|
|
207
|
+
/**
|
|
208
|
+
* Reads the row with the given `rowKey`, or `undefined` if it does not
|
|
209
|
+
* exist.
|
|
210
|
+
*
|
|
211
|
+
* @param cache An optional {@link SnapshotRowCache} through which to
|
|
212
|
+
* perform the read, along with the `tag` that identifies the
|
|
213
|
+
* version(s) of the replica that the result depends on.
|
|
214
|
+
*/
|
|
215
|
+
getRow(table: LiteTableSpecWithKeysAndVersion, rowKey: JSONValue, cache?: SnapshotRowCache, tag?: string): any;
|
|
216
|
+
/**
|
|
217
|
+
* Reads the rows that conflict with `row` on any of the given unique
|
|
218
|
+
* `keys` (which includes the row with the same primary key, if any).
|
|
219
|
+
*
|
|
220
|
+
* @param cache An optional {@link SnapshotRowCache} through which to
|
|
221
|
+
* perform the read, along with the `tag` that identifies the
|
|
222
|
+
* version(s) of the replica that the result depends on.
|
|
223
|
+
*/
|
|
224
|
+
getRows(table: LiteTableSpecWithKeysAndVersion, keys: PrimaryKey[], row: RowValue, cache?: SnapshotRowCache, tag?: string): any[];
|
|
225
|
+
/**
|
|
226
|
+
* Whether the connection has been reset to a newer snapshot (i.e. via
|
|
227
|
+
* {@link resetToHead()}), after which reads no longer reflect
|
|
228
|
+
* {@link version}.
|
|
229
|
+
*/
|
|
230
|
+
get reset(): boolean;
|
|
193
231
|
resetToHead(): Snapshot;
|
|
194
232
|
}
|
|
195
233
|
export declare class InvalidDiffError extends Error {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"snapshotter.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/view-syncer/snapshotter.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAEjD,OAAO,EAAY,KAAK,SAAS,EAAC,MAAM,uCAAuC,CAAC;
|
|
1
|
+
{"version":3,"file":"snapshotter.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/view-syncer/snapshotter.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAEjD,OAAO,EAAY,KAAK,SAAS,EAAC,MAAM,uCAAuC,CAAC;AAGhF,OAAO,KAAK,EAAC,GAAG,EAAC,MAAM,uCAAuC,CAAC;AAC/D,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,sCAAsC,CAAC;AAGrE,OAAO,KAAK,EACV,cAAc,EACd,+BAA+B,EAChC,MAAM,mBAAmB,CAAC;AAC3B,OAAO,EAAC,eAAe,EAAC,MAAM,wBAAwB,CAAC;AACvD,OAAO,EAEL,KAAK,MAAM,EACX,KAAK,QAAQ,EACd,MAAM,wBAAwB,CAAC;AAChC,OAAO,KAAK,EAAC,KAAK,EAAC,MAAM,uBAAuB,CAAC;AAYjD,OAAO,KAAK,EAAC,gBAAgB,EAAC,MAAM,yBAAyB,CAAC;AAE9D;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA0DG;AACH,qBAAa,WAAW;;IAStB;;;;;OAKG;gBAED,EAAE,EAAE,UAAU,EACd,MAAM,EAAE,MAAM,EACd,EAAC,KAAK,EAAC,EAAE,KAAK,EACd,gBAAgB,CAAC,EAAE,MAAM,EACzB,QAAQ,CAAC,EAAE,gBAAgB;IAS7B;;;;OAIG;IACH,IAAI,IAAI,IAAI;IAYZ,WAAW,IAAI,OAAO;IAItB,+EAA+E;IAC/E,OAAO,IAAI,QAAQ;IAKnB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OAoCG;IACH,OAAO,CACL,cAAc,EAAE,GAAG,CAAC,MAAM,EAAE,cAAc,CAAC,EAC3C,aAAa,EAAE,GAAG,CAAC,MAAM,CAAC,EAC1B,cAAc,CAAC,EAAE,WAAW,GAAG,SAAS,GACvC,YAAY;IAaf,kBAAkB;;;;IAelB;;;OAGG;IACH,OAAO;CAKR;AAED,MAAM,MAAM,MAAM,GAAG;IACnB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB;;;;;OAKG;IACH,QAAQ,CAAC,UAAU,EAAE,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;IACrC,QAAQ,CAAC,SAAS,EAAE,QAAQ,CAAC,GAAG,CAAC,GAAG,IAAI,CAAC;IACzC,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;CACzB,CAAC;AAEF,MAAM,WAAW,WAAW;IAC1B,GAAG,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC;CAC7B;AAED;;;;;;GAMG;AACH,MAAM,WAAW,YAAa,SAAQ,QAAQ,CAAC,MAAM,CAAC;IACpD,QAAQ,CAAC,IAAI,EAAE;QACb,QAAQ,CAAC,EAAE,EAAE,eAAe,CAAC;QAC7B,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;KAC1B,CAAC;IACF,QAAQ,CAAC,IAAI,EAAE;QACb,QAAQ,CAAC,EAAE,EAAE,eAAe,CAAC;QAC7B,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;KAC1B,CAAC;IAEF;;;;;;;;;;OAUG;IACH,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;CAC1B;AAED;;;;GAIG;AACH,MAAM,MAAM,oBAAoB,GAC5B,qBAAqB,GACrB,iBAAiB,GACjB,eAAe,GACf,YAAY,GACZ,oBAAoB,CAAC;AAEzB,qBAAa,oBAAqB,SAAQ,KAAK;IAC7C,QAAQ,CAAC,IAAI,0BAA0B;IACvC,QAAQ,CAAC,MAAM,EAAE,oBAAoB,CAAC;gBAE1B,GAAG,EAAE,MAAM,EAAE,MAAM,EAAE,oBAAoB;CAItD;AAmCD,cAAM,QAAQ;;IACZ,MAAM,CAAC,MAAM,CACX,EAAE,EAAE,UAAU,EACd,MAAM,EAAE,MAAM,EACd,KAAK,EAAE,MAAM,EACb,gBAAgB,EAAE,MAAM,GAAG,SAAS;IAuBtC,QAAQ,CAAC,EAAE,EAAE,eAAe,CAAC;IAE7B,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;gBAGb,EAAE,EAAE,eAAe,EAAE,KAAK,EAAE,MAAM;IAY9C,eAAe,CAAC,WAAW,EAAE,MAAM;IAQnC;;;;OAIG;IACH,mBAAmB,CAAC,WAAW,EAAE,MAAM,GAAG,OAAO;IAUjD,YAAY,CAAC,WAAW,EAAE,MAAM;;;;IAahC;;;;;;;OAOG;IACH,MAAM,CACJ,KAAK,EAAE,+BAA+B,EACtC,MAAM,EAAE,SAAS,EACjB,KAAK,CAAC,EAAE,gBAAgB,EACxB,GAAG,CAAC,EAAE,MAAM;IA4Bd;;;;;;;OAOG;IACH,OAAO,CACL,KAAK,EAAE,+BAA+B,EACtC,IAAI,EAAE,UAAU,EAAE,EAClB,GAAG,EAAE,QAAQ,EACb,KAAK,CAAC,EAAE,gBAAgB,EACxB,GAAG,CAAC,EAAE,MAAM;IAiEd;;;;OAIG;IACH,IAAI,KAAK,IAAI,OAAO,CAEnB;IAED,WAAW,IAAI,QAAQ;CAKxB;AA0QD,qBAAa,gBAAiB,SAAQ,KAAK;gBAC7B,GAAG,EAAE,MAAM;CAGxB"}
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { assert } from "../../../../shared/src/asserts.js";
|
|
2
2
|
import { parse } from "../../../../shared/src/valita.js";
|
|
3
|
+
import { getOrInsertComputed } from "../../../../shared/src/map.js";
|
|
3
4
|
import { stringify } from "../../../../shared/src/bigint-json.js";
|
|
4
5
|
import { Database } from "../../../../zqlite/src/db.js";
|
|
5
6
|
import { fromSQLiteTypes } from "../../../../zqlite/src/table-source.js";
|
|
@@ -74,13 +75,21 @@ var Snapshotter = class {
|
|
|
74
75
|
#dbFile;
|
|
75
76
|
#appID;
|
|
76
77
|
#pageCacheSizeKib;
|
|
78
|
+
#rowCache;
|
|
77
79
|
#curr;
|
|
78
80
|
#prev;
|
|
79
|
-
|
|
81
|
+
/**
|
|
82
|
+
* @param rowCache An optional worker-wide {@link SnapshotRowCache} through
|
|
83
|
+
* which the row reads performed when iterating over a
|
|
84
|
+
* {@link SnapshotDiff} are shared with the other Snapshotters
|
|
85
|
+
* (i.e. client groups) on the worker.
|
|
86
|
+
*/
|
|
87
|
+
constructor(lc, dbFile, { appID }, pageCacheSizeKib, rowCache) {
|
|
80
88
|
this.#lc = lc;
|
|
81
89
|
this.#dbFile = dbFile;
|
|
82
90
|
this.#appID = appID;
|
|
83
91
|
this.#pageCacheSizeKib = pageCacheSizeKib;
|
|
92
|
+
this.#rowCache = rowCache;
|
|
84
93
|
}
|
|
85
94
|
/**
|
|
86
95
|
* Initializes the snapshot to the current head of the database. This must be
|
|
@@ -138,9 +147,9 @@ var Snapshotter = class {
|
|
|
138
147
|
* on `prev` before each iteration, and (2) rollback to the save point after
|
|
139
148
|
* the iteration.
|
|
140
149
|
*/
|
|
141
|
-
advance(syncableTables, allTableNames) {
|
|
150
|
+
advance(syncableTables, allTableNames, observedTables) {
|
|
142
151
|
const { prev, curr } = this.advanceWithoutDiff();
|
|
143
|
-
return new Diff(this.#appID, syncableTables, allTableNames, prev, curr);
|
|
152
|
+
return new Diff(this.#appID, syncableTables, allTableNames, prev, curr, observedTables, this.#rowCache);
|
|
144
153
|
}
|
|
145
154
|
advanceWithoutDiff() {
|
|
146
155
|
assert(this.#curr !== void 0, "Snapshotter has not been initialized");
|
|
@@ -170,6 +179,15 @@ var ResetPipelinesSignal = class extends Error {
|
|
|
170
179
|
this.reason = reason;
|
|
171
180
|
}
|
|
172
181
|
};
|
|
182
|
+
var tableReads = /* @__PURE__ */ new WeakMap();
|
|
183
|
+
function readsFor(table) {
|
|
184
|
+
return getOrInsertComputed(tableReads, table, () => ({
|
|
185
|
+
prefix: `SELECT ${Object.keys(table.columns).map((c) => id(c)).join(",")} FROM ${id(table.name)} WHERE `,
|
|
186
|
+
byKeyColumns: /* @__PURE__ */ new Map(),
|
|
187
|
+
byUniqueKeys: /* @__PURE__ */ new Map()
|
|
188
|
+
}));
|
|
189
|
+
}
|
|
190
|
+
var MAX_MEMOIZED_UNIQUE_KEYS = 30;
|
|
173
191
|
var Snapshot = class Snapshot {
|
|
174
192
|
static create(lc, dbFile, appID, pageCacheSizeKib) {
|
|
175
193
|
const conn = new Database(lc, dbFile);
|
|
@@ -182,6 +200,7 @@ var Snapshot = class Snapshot {
|
|
|
182
200
|
db;
|
|
183
201
|
#appID;
|
|
184
202
|
version;
|
|
203
|
+
#reset = false;
|
|
185
204
|
constructor(db, appID) {
|
|
186
205
|
db.beginConcurrent();
|
|
187
206
|
const { stateVersion } = getReplicationState(db);
|
|
@@ -193,6 +212,14 @@ var Snapshot = class Snapshot {
|
|
|
193
212
|
const { count } = this.db.get("SELECT COUNT(*) AS count FROM \"_zero.changeLog2\" WHERE stateVersion > ?", prevVersion);
|
|
194
213
|
return count;
|
|
195
214
|
}
|
|
215
|
+
/**
|
|
216
|
+
* Whether any table-wide op (i.e. a RESET or TRUNCATE) was logged after
|
|
217
|
+
* `prevVersion`. Iterating over a diff ending at this snapshot will
|
|
218
|
+
* abort with a {@link ResetPipelinesSignal} if so.
|
|
219
|
+
*/
|
|
220
|
+
hasTableWideOpSince(prevVersion) {
|
|
221
|
+
return this.db.get("SELECT 1 FROM \"_zero.changeLog2\" WHERE stateVersion > ? AND op IN (?, ?) LIMIT 1", prevVersion, "r", "t") !== void 0;
|
|
222
|
+
}
|
|
196
223
|
changesSince(prevVersion) {
|
|
197
224
|
const cached = this.db.statementCache.get(`SELECT "stateVersion", "table", "rowKey", "op" FROM "_zero.changeLog2"
|
|
198
225
|
WHERE "stateVersion" > ? ORDER BY "stateVersion" ASC, "pos" ASC`);
|
|
@@ -201,32 +228,83 @@ var Snapshot = class Snapshot {
|
|
|
201
228
|
cleanup: () => this.db.statementCache.return(cached)
|
|
202
229
|
};
|
|
203
230
|
}
|
|
204
|
-
|
|
231
|
+
/**
|
|
232
|
+
* Reads the row with the given `rowKey`, or `undefined` if it does not
|
|
233
|
+
* exist.
|
|
234
|
+
*
|
|
235
|
+
* @param cache An optional {@link SnapshotRowCache} through which to
|
|
236
|
+
* perform the read, along with the `tag` that identifies the
|
|
237
|
+
* version(s) of the replica that the result depends on.
|
|
238
|
+
*/
|
|
239
|
+
getRow(table, rowKey, cache, tag) {
|
|
205
240
|
const key = normalizedKeyOrder(rowKey);
|
|
206
|
-
const
|
|
207
|
-
const
|
|
208
|
-
const
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
241
|
+
const cols = Object.keys(key);
|
|
242
|
+
const reads = readsFor(table);
|
|
243
|
+
const sql = getOrInsertComputed(reads.byKeyColumns, cols.join("\0"), () => reads.prefix + cols.map((c) => `${id(c)}=?`).join(" AND "));
|
|
244
|
+
const args = Object.values(key);
|
|
245
|
+
const read = () => this.db.statementCache.use(sql, (cached) => {
|
|
246
|
+
cached.statement.safeIntegers(true);
|
|
247
|
+
return cached.statement.get(args);
|
|
248
|
+
});
|
|
249
|
+
return cache ? cache.getOrRead(tag ?? "", sql, "get", args, () => {
|
|
250
|
+
const row = read();
|
|
251
|
+
if (row !== void 0) this.#assertNotAdvanced(row);
|
|
252
|
+
return row;
|
|
253
|
+
}) : read();
|
|
215
254
|
}
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
255
|
+
/**
|
|
256
|
+
* Reads the rows that conflict with `row` on any of the given unique
|
|
257
|
+
* `keys` (which includes the row with the same primary key, if any).
|
|
258
|
+
*
|
|
259
|
+
* @param cache An optional {@link SnapshotRowCache} through which to
|
|
260
|
+
* perform the read, along with the `tag` that identifies the
|
|
261
|
+
* version(s) of the replica that the result depends on.
|
|
262
|
+
*/
|
|
263
|
+
getRows(table, keys, row, cache, tag) {
|
|
264
|
+
const validKeys = [];
|
|
265
|
+
let mask = 0;
|
|
266
|
+
for (let i = 0; i < keys.length; i++) {
|
|
267
|
+
const key = keys[i];
|
|
268
|
+
if (key.every((column) => row[column] !== null && row[column] !== void 0)) {
|
|
269
|
+
validKeys.push(key);
|
|
270
|
+
mask |= 1 << i;
|
|
271
|
+
}
|
|
227
272
|
}
|
|
273
|
+
if (validKeys.length === 0) return [];
|
|
274
|
+
const reads = readsFor(table);
|
|
275
|
+
const buildSQL = () => reads.prefix + validKeys.map((key) => key.map((c) => `${id(c)}=?`).join(" AND ")).join(" OR ");
|
|
276
|
+
const sql = keys === table.uniqueKeys && keys.length <= MAX_MEMOIZED_UNIQUE_KEYS ? getOrInsertComputed(reads.byUniqueKeys, mask, buildSQL) : buildSQL();
|
|
277
|
+
const args = validKeys.flatMap((key) => key.map((column) => row[column]));
|
|
278
|
+
const read = () => this.db.statementCache.use(sql, (cached) => {
|
|
279
|
+
cached.statement.safeIntegers(true);
|
|
280
|
+
return cached.statement.all(args);
|
|
281
|
+
});
|
|
282
|
+
return cache ? cache.getOrRead(tag ?? "", sql, "all", args, () => {
|
|
283
|
+
const rows = read();
|
|
284
|
+
for (const row of rows) this.#assertNotAdvanced(row);
|
|
285
|
+
return rows;
|
|
286
|
+
}) : read();
|
|
287
|
+
}
|
|
288
|
+
/**
|
|
289
|
+
* Rows are shared with other client groups via the SnapshotRowCache, so
|
|
290
|
+
* one read from a connection that has advanced past this snapshot (i.e. a
|
|
291
|
+
* {@link SnapshotDiff} used after its Snapshotter advanced) must fail
|
|
292
|
+
* before it is cached, rather than be served to the valid Diffs of other
|
|
293
|
+
* client groups (whose own validity checks would then fail).
|
|
294
|
+
*/
|
|
295
|
+
#assertNotAdvanced(row) {
|
|
296
|
+
if ((row["_0_version"] ?? "~") > this.version) throw new InvalidDiffError(`Diff is no longer valid. db has advanced past ${this.version}.`);
|
|
297
|
+
}
|
|
298
|
+
/**
|
|
299
|
+
* Whether the connection has been reset to a newer snapshot (i.e. via
|
|
300
|
+
* {@link resetToHead()}), after which reads no longer reflect
|
|
301
|
+
* {@link version}.
|
|
302
|
+
*/
|
|
303
|
+
get reset() {
|
|
304
|
+
return this.#reset;
|
|
228
305
|
}
|
|
229
306
|
resetToHead() {
|
|
307
|
+
this.#reset = true;
|
|
230
308
|
this.db.rollback();
|
|
231
309
|
return new Snapshot(this.db, this.#appID);
|
|
232
310
|
}
|
|
@@ -235,16 +313,20 @@ var Diff = class {
|
|
|
235
313
|
#permissionsTable;
|
|
236
314
|
#syncableTables;
|
|
237
315
|
#allTableNames;
|
|
316
|
+
#observedTables;
|
|
317
|
+
#rowCache;
|
|
238
318
|
prev;
|
|
239
319
|
curr;
|
|
240
320
|
changes;
|
|
241
|
-
constructor(appID, syncableTables, allTableNames, prev, curr) {
|
|
321
|
+
constructor(appID, syncableTables, allTableNames, prev, curr, observedTables, rowCache) {
|
|
242
322
|
this.#permissionsTable = `${appID}.permissions`;
|
|
243
323
|
this.#syncableTables = syncableTables;
|
|
244
324
|
this.#allTableNames = allTableNames;
|
|
325
|
+
this.#observedTables = observedTables;
|
|
245
326
|
this.prev = prev;
|
|
246
327
|
this.curr = curr;
|
|
247
328
|
this.changes = curr.numChangesSince(prev.version);
|
|
329
|
+
this.#rowCache = rowCache && !curr.hasTableWideOpSince(prev.version) ? rowCache : void 0;
|
|
248
330
|
}
|
|
249
331
|
[Symbol.iterator]() {
|
|
250
332
|
const { changes, cleanup: done } = this.curr.changesSince(this.prev.version);
|
|
@@ -270,6 +352,10 @@ var Diff = class {
|
|
|
270
352
|
const { table, rowKey, op, stateVersion } = parse(value, changeLogEntrySchema);
|
|
271
353
|
if (op === "r") throw new ResetPipelinesSignal(`schema for table ${table} has changed`, "schema-change");
|
|
272
354
|
if (op === "t") throw new ResetPipelinesSignal(`table ${table} has been truncated`, "truncation");
|
|
355
|
+
if (this.#observedTables && !this.#observedTables.has(table) && table !== this.#permissionsTable) {
|
|
356
|
+
if (this.#allTableNames.has(table)) continue;
|
|
357
|
+
throw new Error(`change for unknown table ${table}`);
|
|
358
|
+
}
|
|
273
359
|
const specs = this.#syncableTables.get(table);
|
|
274
360
|
if (!specs) {
|
|
275
361
|
if (this.#allTableNames.has(table)) continue;
|
|
@@ -278,11 +364,14 @@ var Diff = class {
|
|
|
278
364
|
const { tableSpec, zqlSpec } = specs;
|
|
279
365
|
assert((tableSpec.minRowVersion ?? "") < stateVersion, () => `unexpected change @${stateVersion} for table ${table} with minRowVersion ${tableSpec.minRowVersion}: ${op}(${rowKey})`);
|
|
280
366
|
assert(rowKey !== null, "rowKey must be present for row changes");
|
|
281
|
-
|
|
367
|
+
if (this.prev.reset) throw new InvalidDiffError(`Diff is no longer valid. prev db has advanced past ${this.prev.version}.`);
|
|
368
|
+
const cache = this.#rowCache;
|
|
369
|
+
const prevTag = tableSpec.uniqueKeys.length <= 1 ? `p:${this.prev.version}` : `p:${this.prev.version}:${this.curr.version}`;
|
|
370
|
+
const nextValue = op === "s" ? this.curr.getRow(tableSpec, rowKey, cache, `n:${stateVersion}`) : null;
|
|
282
371
|
let prevValues;
|
|
283
|
-
if (nextValue) prevValues = this.prev.getRows(tableSpec, tableSpec.uniqueKeys, nextValue);
|
|
372
|
+
if (nextValue) prevValues = this.prev.getRows(tableSpec, tableSpec.uniqueKeys, nextValue, cache, prevTag);
|
|
284
373
|
else {
|
|
285
|
-
const prevValue = this.prev.getRow(tableSpec, rowKey);
|
|
374
|
+
const prevValue = this.prev.getRow(tableSpec, rowKey, cache, prevTag);
|
|
286
375
|
prevValues = prevValue ? [prevValue] : [];
|
|
287
376
|
}
|
|
288
377
|
if (nextValue === void 0) throw new Error(`Missing value for ${table} ${stringify(rowKey)}`);
|