@rocicorp/zero 1.10.0-canary.22 → 1.10.0-canary.24

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (302) hide show
  1. package/out/analyze-query/src/analyze-cli.js +2 -2
  2. package/out/analyze-query/src/analyze-cli.js.map +1 -1
  3. package/out/analyze-query/src/bin-analyze.js +9 -8
  4. package/out/analyze-query/src/bin-analyze.js.map +1 -1
  5. package/out/replicache/src/btree/node.d.ts.map +1 -1
  6. package/out/replicache/src/btree/node.js +3 -6
  7. package/out/replicache/src/btree/node.js.map +1 -1
  8. package/out/replicache/src/dag/gc.d.ts.map +1 -1
  9. package/out/replicache/src/dag/gc.js +6 -10
  10. package/out/replicache/src/dag/gc.js.map +1 -1
  11. package/out/replicache/src/kv/sqlite-store.d.ts.map +1 -1
  12. package/out/replicache/src/kv/sqlite-store.js +24 -2
  13. package/out/replicache/src/kv/sqlite-store.js.map +1 -1
  14. package/out/replicache/src/persist/persist.js +1 -1
  15. package/out/shared/src/arrays.d.ts +5 -0
  16. package/out/shared/src/arrays.d.ts.map +1 -1
  17. package/out/shared/src/arrays.js +8 -10
  18. package/out/shared/src/arrays.js.map +1 -1
  19. package/out/shared/src/map.d.ts +19 -2
  20. package/out/shared/src/map.d.ts.map +1 -1
  21. package/out/shared/src/map.js +18 -4
  22. package/out/shared/src/map.js.map +1 -1
  23. package/out/z2s/src/compiler.js +1 -1
  24. package/out/z2s/src/sql.d.ts.map +1 -1
  25. package/out/z2s/src/sql.js +2 -2
  26. package/out/z2s/src/sql.js.map +1 -1
  27. package/out/zero/package.js +1 -1
  28. package/out/zero/package.js.map +1 -1
  29. package/out/zero-cache/src/auth/write-authorizer.d.ts.map +1 -1
  30. package/out/zero-cache/src/auth/write-authorizer.js +1 -0
  31. package/out/zero-cache/src/auth/write-authorizer.js.map +1 -1
  32. package/out/zero-cache/src/config/zero-config.d.ts +23 -0
  33. package/out/zero-cache/src/config/zero-config.d.ts.map +1 -1
  34. package/out/zero-cache/src/config/zero-config.js +63 -1
  35. package/out/zero-cache/src/config/zero-config.js.map +1 -1
  36. package/out/zero-cache/src/db/lite-tables.d.ts.map +1 -1
  37. package/out/zero-cache/src/db/lite-tables.js +3 -4
  38. package/out/zero-cache/src/db/lite-tables.js.map +1 -1
  39. package/out/zero-cache/src/observability/metrics.d.ts.map +1 -1
  40. package/out/zero-cache/src/observability/metrics.js +2 -7
  41. package/out/zero-cache/src/observability/metrics.js.map +1 -1
  42. package/out/zero-cache/src/server/change-streamer.d.ts.map +1 -1
  43. package/out/zero-cache/src/server/change-streamer.js +11 -3
  44. package/out/zero-cache/src/server/change-streamer.js.map +1 -1
  45. package/out/zero-cache/src/server/inspector-delegate.d.ts +18 -2
  46. package/out/zero-cache/src/server/inspector-delegate.d.ts.map +1 -1
  47. package/out/zero-cache/src/server/inspector-delegate.js +22 -15
  48. package/out/zero-cache/src/server/inspector-delegate.js.map +1 -1
  49. package/out/zero-cache/src/server/main.d.ts.map +1 -1
  50. package/out/zero-cache/src/server/main.js +8 -1
  51. package/out/zero-cache/src/server/main.js.map +1 -1
  52. package/out/zero-cache/src/server/mutator.js +2 -0
  53. package/out/zero-cache/src/server/mutator.js.map +1 -1
  54. package/out/zero-cache/src/server/reaper.d.ts.map +1 -1
  55. package/out/zero-cache/src/server/reaper.js +2 -0
  56. package/out/zero-cache/src/server/reaper.js.map +1 -1
  57. package/out/zero-cache/src/server/replicator.d.ts.map +1 -1
  58. package/out/zero-cache/src/server/replicator.js +2 -0
  59. package/out/zero-cache/src/server/replicator.js.map +1 -1
  60. package/out/zero-cache/src/server/runner/zero-dispatcher.d.ts.map +1 -1
  61. package/out/zero-cache/src/server/runner/zero-dispatcher.js +3 -0
  62. package/out/zero-cache/src/server/runner/zero-dispatcher.js.map +1 -1
  63. package/out/zero-cache/src/server/syncer.d.ts.map +1 -1
  64. package/out/zero-cache/src/server/syncer.js +5 -1
  65. package/out/zero-cache/src/server/syncer.js.map +1 -1
  66. package/out/zero-cache/src/services/analyze.d.ts.map +1 -1
  67. package/out/zero-cache/src/services/analyze.js +11 -9
  68. package/out/zero-cache/src/services/analyze.js.map +1 -1
  69. package/out/zero-cache/src/services/change-source/common/backfill-manager.d.ts.map +1 -1
  70. package/out/zero-cache/src/services/change-source/common/backfill-manager.js +14 -1
  71. package/out/zero-cache/src/services/change-source/common/backfill-manager.js.map +1 -1
  72. package/out/zero-cache/src/services/change-source/common/change-stream-multiplexer.d.ts.map +1 -1
  73. package/out/zero-cache/src/services/change-source/common/change-stream-multiplexer.js +15 -3
  74. package/out/zero-cache/src/services/change-source/common/change-stream-multiplexer.js.map +1 -1
  75. package/out/zero-cache/src/services/change-source/custom/change-source.js +2 -2
  76. package/out/zero-cache/src/services/change-source/custom/change-source.js.map +1 -1
  77. package/out/zero-cache/src/services/change-source/pg/initial-sync.js +1 -1
  78. package/out/zero-cache/src/services/change-streamer/change-streamer-http.d.ts +3 -0
  79. package/out/zero-cache/src/services/change-streamer/change-streamer-http.d.ts.map +1 -1
  80. package/out/zero-cache/src/services/change-streamer/change-streamer-http.js +6 -4
  81. package/out/zero-cache/src/services/change-streamer/change-streamer-http.js.map +1 -1
  82. package/out/zero-cache/src/services/mutagen/mutagen.d.ts +5 -0
  83. package/out/zero-cache/src/services/mutagen/mutagen.d.ts.map +1 -1
  84. package/out/zero-cache/src/services/mutagen/mutagen.js +7 -1
  85. package/out/zero-cache/src/services/mutagen/mutagen.js.map +1 -1
  86. package/out/zero-cache/src/services/mutagen/pusher.d.ts +32 -32
  87. package/out/zero-cache/src/services/mutagen/pusher.d.ts.map +1 -1
  88. package/out/zero-cache/src/services/mutagen/pusher.js +5 -6
  89. package/out/zero-cache/src/services/mutagen/pusher.js.map +1 -1
  90. package/out/zero-cache/src/services/profz.d.ts +7 -0
  91. package/out/zero-cache/src/services/profz.d.ts.map +1 -0
  92. package/out/zero-cache/src/services/profz.js +103 -0
  93. package/out/zero-cache/src/services/profz.js.map +1 -0
  94. package/out/zero-cache/src/services/run-ast.d.ts +12 -0
  95. package/out/zero-cache/src/services/run-ast.d.ts.map +1 -1
  96. package/out/zero-cache/src/services/run-ast.js +26 -14
  97. package/out/zero-cache/src/services/run-ast.js.map +1 -1
  98. package/out/zero-cache/src/services/runner.d.ts.map +1 -1
  99. package/out/zero-cache/src/services/runner.js +1 -1
  100. package/out/zero-cache/src/services/runner.js.map +1 -1
  101. package/out/zero-cache/src/services/view-syncer/cvr.d.ts +45 -0
  102. package/out/zero-cache/src/services/view-syncer/cvr.d.ts.map +1 -1
  103. package/out/zero-cache/src/services/view-syncer/cvr.js +133 -1
  104. package/out/zero-cache/src/services/view-syncer/cvr.js.map +1 -1
  105. package/out/zero-cache/src/services/view-syncer/drain-coordinator.d.ts +14 -1
  106. package/out/zero-cache/src/services/view-syncer/drain-coordinator.d.ts.map +1 -1
  107. package/out/zero-cache/src/services/view-syncer/drain-coordinator.js +30 -4
  108. package/out/zero-cache/src/services/view-syncer/drain-coordinator.js.map +1 -1
  109. package/out/zero-cache/src/services/view-syncer/hydration-budget.d.ts +33 -0
  110. package/out/zero-cache/src/services/view-syncer/hydration-budget.d.ts.map +1 -0
  111. package/out/zero-cache/src/services/view-syncer/hydration-budget.js +62 -0
  112. package/out/zero-cache/src/services/view-syncer/hydration-budget.js.map +1 -0
  113. package/out/zero-cache/src/services/view-syncer/hydration-circuit-breaker.d.ts +51 -0
  114. package/out/zero-cache/src/services/view-syncer/hydration-circuit-breaker.d.ts.map +1 -0
  115. package/out/zero-cache/src/services/view-syncer/hydration-circuit-breaker.js +78 -0
  116. package/out/zero-cache/src/services/view-syncer/hydration-circuit-breaker.js.map +1 -0
  117. package/out/zero-cache/src/services/view-syncer/inspect-handler.d.ts +1 -1
  118. package/out/zero-cache/src/services/view-syncer/inspect-handler.d.ts.map +1 -1
  119. package/out/zero-cache/src/services/view-syncer/inspect-handler.js +2 -2
  120. package/out/zero-cache/src/services/view-syncer/inspect-handler.js.map +1 -1
  121. package/out/zero-cache/src/services/view-syncer/pipeline-driver.d.ts +2 -1
  122. package/out/zero-cache/src/services/view-syncer/pipeline-driver.d.ts.map +1 -1
  123. package/out/zero-cache/src/services/view-syncer/pipeline-driver.js +42 -17
  124. package/out/zero-cache/src/services/view-syncer/pipeline-driver.js.map +1 -1
  125. package/out/zero-cache/src/services/view-syncer/query-covering.d.ts.map +1 -1
  126. package/out/zero-cache/src/services/view-syncer/query-covering.js +2 -6
  127. package/out/zero-cache/src/services/view-syncer/query-covering.js.map +1 -1
  128. package/out/zero-cache/src/services/view-syncer/snapshot-row-cache.d.ts +95 -0
  129. package/out/zero-cache/src/services/view-syncer/snapshot-row-cache.d.ts.map +1 -0
  130. package/out/zero-cache/src/services/view-syncer/snapshot-row-cache.js +155 -0
  131. package/out/zero-cache/src/services/view-syncer/snapshot-row-cache.js.map +1 -0
  132. package/out/zero-cache/src/services/view-syncer/snapshotter.d.ts +42 -4
  133. package/out/zero-cache/src/services/view-syncer/snapshotter.d.ts.map +1 -1
  134. package/out/zero-cache/src/services/view-syncer/snapshotter.js +117 -28
  135. package/out/zero-cache/src/services/view-syncer/snapshotter.js.map +1 -1
  136. package/out/zero-cache/src/services/view-syncer/view-syncer.d.ts +11 -1
  137. package/out/zero-cache/src/services/view-syncer/view-syncer.d.ts.map +1 -1
  138. package/out/zero-cache/src/services/view-syncer/view-syncer.js +466 -95
  139. package/out/zero-cache/src/services/view-syncer/view-syncer.js.map +1 -1
  140. package/out/zero-cache/src/types/processes.d.ts +32 -0
  141. package/out/zero-cache/src/types/processes.d.ts.map +1 -1
  142. package/out/zero-cache/src/types/processes.js +43 -4
  143. package/out/zero-cache/src/types/processes.js.map +1 -1
  144. package/out/zero-cache/src/types/profiler.d.ts +12 -0
  145. package/out/zero-cache/src/types/profiler.d.ts.map +1 -1
  146. package/out/zero-cache/src/types/profiler.js +46 -1
  147. package/out/zero-cache/src/types/profiler.js.map +1 -1
  148. package/out/zero-cache/src/types/row-key.d.ts.map +1 -1
  149. package/out/zero-cache/src/types/row-key.js +5 -5
  150. package/out/zero-cache/src/types/row-key.js.map +1 -1
  151. package/out/zero-cache/src/workers/connection.js +3 -3
  152. package/out/zero-cache/src/workers/connection.js.map +1 -1
  153. package/out/zero-cache/src/workers/syncer-ws-message-handler.d.ts.map +1 -1
  154. package/out/zero-cache/src/workers/syncer-ws-message-handler.js +21 -16
  155. package/out/zero-cache/src/workers/syncer-ws-message-handler.js.map +1 -1
  156. package/out/zero-cache/src/workers/syncer.d.ts.map +1 -1
  157. package/out/zero-cache/src/workers/syncer.js +10 -1
  158. package/out/zero-cache/src/workers/syncer.js.map +1 -1
  159. package/out/zero-client/src/client/context.d.ts +16 -3
  160. package/out/zero-client/src/client/context.d.ts.map +1 -1
  161. package/out/zero-client/src/client/context.js +85 -14
  162. package/out/zero-client/src/client/context.js.map +1 -1
  163. package/out/zero-client/src/client/ivm-branch.d.ts.map +1 -1
  164. package/out/zero-client/src/client/ivm-branch.js +20 -3
  165. package/out/zero-client/src/client/ivm-branch.js.map +1 -1
  166. package/out/zero-client/src/client/query-manager.d.ts.map +1 -1
  167. package/out/zero-client/src/client/query-manager.js +2 -5
  168. package/out/zero-client/src/client/query-manager.js.map +1 -1
  169. package/out/zero-client/src/client/version.js +1 -1
  170. package/out/zero-client/src/client/zero-rep.d.ts.map +1 -1
  171. package/out/zero-client/src/client/zero-rep.js +1 -1
  172. package/out/zero-client/src/client/zero-rep.js.map +1 -1
  173. package/out/zero-client/src/client/zero.d.ts.map +1 -1
  174. package/out/zero-client/src/client/zero.js +2 -2
  175. package/out/zero-client/src/client/zero.js.map +1 -1
  176. package/out/zero-protocol/src/analyze-query-result.d.ts +5 -5
  177. package/out/zero-protocol/src/analyze-query-result.d.ts.map +1 -1
  178. package/out/zero-protocol/src/analyze-query-result.js +1 -1
  179. package/out/zero-protocol/src/analyze-query-result.js.map +1 -1
  180. package/out/zero-protocol/src/ast.d.ts.map +1 -1
  181. package/out/zero-protocol/src/ast.js +13 -14
  182. package/out/zero-protocol/src/ast.js.map +1 -1
  183. package/out/zero-protocol/src/down.d.ts +2 -2
  184. package/out/zero-protocol/src/error.d.ts +1 -1
  185. package/out/zero-protocol/src/inspect-down.d.ts +6 -6
  186. package/out/zero-protocol/src/protocol-version.d.ts.map +1 -1
  187. package/out/zero-protocol/src/protocol-version.js.map +1 -1
  188. package/out/zero-protocol/src/query-hash.d.ts.map +1 -1
  189. package/out/zero-protocol/src/query-hash.js +2 -5
  190. package/out/zero-protocol/src/query-hash.js.map +1 -1
  191. package/out/zero-solid/src/solid-view.d.ts +7 -0
  192. package/out/zero-solid/src/solid-view.d.ts.map +1 -1
  193. package/out/zero-solid/src/solid-view.js +13 -2
  194. package/out/zero-solid/src/solid-view.js.map +1 -1
  195. package/out/zql/src/builder/builder.d.ts +19 -0
  196. package/out/zql/src/builder/builder.d.ts.map +1 -1
  197. package/out/zql/src/builder/builder.js +28 -11
  198. package/out/zql/src/builder/builder.js.map +1 -1
  199. package/out/zql/src/builder/correlated-predicate-pushdown.d.ts +43 -0
  200. package/out/zql/src/builder/correlated-predicate-pushdown.d.ts.map +1 -0
  201. package/out/zql/src/builder/correlated-predicate-pushdown.js +161 -0
  202. package/out/zql/src/builder/correlated-predicate-pushdown.js.map +1 -0
  203. package/out/zql/src/builder/debug-delegate.d.ts +6 -1
  204. package/out/zql/src/builder/debug-delegate.d.ts.map +1 -1
  205. package/out/zql/src/builder/debug-delegate.js +14 -2
  206. package/out/zql/src/builder/debug-delegate.js.map +1 -1
  207. package/out/zql/src/ivm/array-view.d.ts +19 -0
  208. package/out/zql/src/ivm/array-view.d.ts.map +1 -1
  209. package/out/zql/src/ivm/array-view.js +39 -2
  210. package/out/zql/src/ivm/array-view.js.map +1 -1
  211. package/out/zql/src/ivm/connection-index.d.ts +68 -0
  212. package/out/zql/src/ivm/connection-index.d.ts.map +1 -0
  213. package/out/zql/src/ivm/connection-index.js +149 -0
  214. package/out/zql/src/ivm/connection-index.js.map +1 -0
  215. package/out/zql/src/ivm/exists.js +1 -1
  216. package/out/zql/src/ivm/filter-operators.d.ts +3 -2
  217. package/out/zql/src/ivm/filter-operators.d.ts.map +1 -1
  218. package/out/zql/src/ivm/filter-operators.js +19 -4
  219. package/out/zql/src/ivm/filter-operators.js.map +1 -1
  220. package/out/zql/src/ivm/memory-source.d.ts +13 -1
  221. package/out/zql/src/ivm/memory-source.d.ts.map +1 -1
  222. package/out/zql/src/ivm/memory-source.js +43 -4
  223. package/out/zql/src/ivm/memory-source.js.map +1 -1
  224. package/out/zql/src/ivm/operator.d.ts +40 -0
  225. package/out/zql/src/ivm/operator.d.ts.map +1 -1
  226. package/out/zql/src/ivm/operator.js.map +1 -1
  227. package/out/zql/src/ivm/view.d.ts +15 -0
  228. package/out/zql/src/ivm/view.d.ts.map +1 -1
  229. package/out/zql/src/planner/planner-builder.d.ts +12 -3
  230. package/out/zql/src/planner/planner-builder.d.ts.map +1 -1
  231. package/out/zql/src/planner/planner-builder.js +40 -20
  232. package/out/zql/src/planner/planner-builder.js.map +1 -1
  233. package/out/zql/src/planner/planner-connection.d.ts +28 -2
  234. package/out/zql/src/planner/planner-connection.d.ts.map +1 -1
  235. package/out/zql/src/planner/planner-connection.js +108 -8
  236. package/out/zql/src/planner/planner-connection.js.map +1 -1
  237. package/out/zql/src/planner/planner-debug.d.ts +1 -1
  238. package/out/zql/src/planner/planner-debug.d.ts.map +1 -1
  239. package/out/zql/src/planner/planner-debug.js +3 -8
  240. package/out/zql/src/planner/planner-debug.js.map +1 -1
  241. package/out/zql/src/planner/planner-fan-out.d.ts +1 -0
  242. package/out/zql/src/planner/planner-fan-out.d.ts.map +1 -1
  243. package/out/zql/src/planner/planner-fan-out.js +3 -0
  244. package/out/zql/src/planner/planner-fan-out.js.map +1 -1
  245. package/out/zql/src/planner/planner-filter.d.ts +59 -0
  246. package/out/zql/src/planner/planner-filter.d.ts.map +1 -0
  247. package/out/zql/src/planner/planner-filter.js +116 -0
  248. package/out/zql/src/planner/planner-filter.js.map +1 -0
  249. package/out/zql/src/planner/planner-graph.d.ts +2 -0
  250. package/out/zql/src/planner/planner-graph.d.ts.map +1 -1
  251. package/out/zql/src/planner/planner-graph.js +2 -0
  252. package/out/zql/src/planner/planner-graph.js.map +1 -1
  253. package/out/zql/src/planner/planner-join.d.ts +2 -0
  254. package/out/zql/src/planner/planner-join.d.ts.map +1 -1
  255. package/out/zql/src/planner/planner-join.js +7 -0
  256. package/out/zql/src/planner/planner-join.js.map +1 -1
  257. package/out/zql/src/planner/planner-node.d.ts +2 -1
  258. package/out/zql/src/planner/planner-node.d.ts.map +1 -1
  259. package/out/zql/src/planner/planner-node.js.map +1 -1
  260. package/out/zql/src/planner/planner-source.d.ts +2 -2
  261. package/out/zql/src/planner/planner-source.d.ts.map +1 -1
  262. package/out/zql/src/planner/planner-source.js +2 -2
  263. package/out/zql/src/planner/planner-source.js.map +1 -1
  264. package/out/zql/src/query/query-delegate-base.d.ts +2 -2
  265. package/out/zql/src/query/query-delegate-base.d.ts.map +1 -1
  266. package/out/zql/src/query/query-delegate-base.js +16 -10
  267. package/out/zql/src/query/query-delegate-base.js.map +1 -1
  268. package/out/zql/src/query/query-delegate.d.ts +22 -5
  269. package/out/zql/src/query/query-delegate.d.ts.map +1 -1
  270. package/out/zql/src/query/query-impl.d.ts.map +1 -1
  271. package/out/zql/src/query/query-impl.js +6 -23
  272. package/out/zql/src/query/query-impl.js.map +1 -1
  273. package/out/zql/src/query/query-transitions.d.ts.map +1 -1
  274. package/out/zql/src/query/query-transitions.js +2 -5
  275. package/out/zql/src/query/query-transitions.js.map +1 -1
  276. package/out/zql/src/query/query.d.ts +4 -1
  277. package/out/zql/src/query/query.d.ts.map +1 -1
  278. package/out/zql/src/query/runnable-query-impl.d.ts.map +1 -1
  279. package/out/zql/src/query/runnable-query-impl.js +2 -10
  280. package/out/zql/src/query/runnable-query-impl.js.map +1 -1
  281. package/out/zqlite/src/db.d.ts +9 -1
  282. package/out/zqlite/src/db.d.ts.map +1 -1
  283. package/out/zqlite/src/db.js +8 -2
  284. package/out/zqlite/src/db.js.map +1 -1
  285. package/out/zqlite/src/query-builder.d.ts +1 -1
  286. package/out/zqlite/src/query-builder.d.ts.map +1 -1
  287. package/out/zqlite/src/query-builder.js +2 -1
  288. package/out/zqlite/src/query-builder.js.map +1 -1
  289. package/out/zqlite/src/query-delegate.d.ts.map +1 -1
  290. package/out/zqlite/src/query-delegate.js +5 -6
  291. package/out/zqlite/src/query-delegate.js.map +1 -1
  292. package/out/zqlite/src/sqlite-cost-model.d.ts.map +1 -1
  293. package/out/zqlite/src/sqlite-cost-model.js +2 -30
  294. package/out/zqlite/src/sqlite-cost-model.js.map +1 -1
  295. package/out/zqlite/src/sqlite-stat-fanout.d.ts.map +1 -1
  296. package/out/zqlite/src/sqlite-stat-fanout.js +3 -5
  297. package/out/zqlite/src/sqlite-stat-fanout.js.map +1 -1
  298. package/out/zqlite/src/table-source.d.ts +15 -2
  299. package/out/zqlite/src/table-source.d.ts.map +1 -1
  300. package/out/zqlite/src/table-source.js +28 -12
  301. package/out/zqlite/src/table-source.js.map +1 -1
  302. package/package.json +1 -1
@@ -1,4 +1,6 @@
1
1
  import { assert, unreachable } from "../../../../shared/src/asserts.js";
2
+ import { getOrInsertComputed } from "../../../../shared/src/map.js";
3
+ import { newArray } from "../../../../shared/src/arrays.js";
2
4
  import { must } from "../../../../shared/src/must.js";
3
5
  import { h64 } from "../../../../shared/src/hash.js";
4
6
  import { Internal, InvalidConnectionRequest, InvalidConnectionRequestBaseCookie, Rehome } from "../../../../zero-protocol/src/error-kind-enum.js";
@@ -11,12 +13,11 @@ import { manualSpan, startAsyncSpan, startSpan } from "../../../../otel/src/span
11
13
  import { ZERO_VERSION_COLUMN_NAME } from "../replicator/schema/constants.js";
12
14
  import { transformAndHashQuery } from "../../auth/read-authorizer.js";
13
15
  import { randInt } from "../../../../shared/src/rand.js";
14
- import { getOrCreateCounter, getOrCreateLatencyHistogram, getOrCreateNativeHistogram, getOrCreateUpDownCounter } from "../../observability/metrics.js";
16
+ import { getOrCreateCounter, getOrCreateLatencyHistogram, getOrCreateNativeHistogram, getOrCreateUpDownCounter, getOrCreateValueHistogram } from "../../observability/metrics.js";
15
17
  import { rowIDString } from "../../types/row-key.js";
16
18
  import "../replicator/schema/replication-state.js";
17
19
  import { parseSignature } from "./row-set-signature.js";
18
20
  import { ResetPipelinesSignal } from "./snapshotter.js";
19
- import "./pipeline-driver.js";
20
21
  import { Subscription } from "../../types/subscription.js";
21
22
  import { CustomKeyMap } from "../../../../shared/src/custom-key-map.js";
22
23
  import { ttlClockAsNumber, ttlClockFromNumber } from "./ttl-clock.js";
@@ -26,8 +27,10 @@ import { isAuthErrorBody } from "../../auth/auth.js";
26
27
  import { ClientHandler, startPoke } from "./client-handler.js";
27
28
  import { tracer } from "./tracer.js";
28
29
  import { CVRStore, ClientNotFoundError } from "./cvr-store.js";
29
- import { CVRConfigDrivenUpdater, CVRQueryDrivenUpdater, nextEvictionTime } from "./cvr.js";
30
+ import { CVRConfigDrivenUpdater, CVRQueryDrivenUpdater, classifyQueriesForHydration, nextEvictionTime } from "./cvr.js";
30
31
  import { E2EServingLagTracker } from "./e2e-serving-lag.js";
32
+ import { HydrationBudget } from "./hydration-budget.js";
33
+ import { HydrationCircuitBreaker } from "./hydration-circuit-breaker.js";
31
34
  import { handleInspect } from "./inspect-handler.js";
32
35
  import { QueryCoveringIndex } from "./query-covering.js";
33
36
  import { resolver } from "@rocicorp/resolver";
@@ -62,6 +65,7 @@ var ViewSyncerService = class {
62
65
  #drainCoordinator;
63
66
  #keepaliveMs;
64
67
  #slowHydrateThreshold;
68
+ #hydrationCircuitBreaker;
65
69
  #lastConnectTime = Date.now();
66
70
  /**
67
71
  * The TTL clock is used to determine the time at which queries are considered
@@ -89,14 +93,36 @@ var ViewSyncerService = class {
89
93
  #lock = new Lock();
90
94
  #cvrStore;
91
95
  #stopped = resolver();
96
+ /**
97
+ * Set when {@link #cleanup} begins. Lock tasks that were in flight when the
98
+ * view-syncer was stopped may still complete after the timers have been
99
+ * cleared; this flag prevents them from scheduling new timers, which would
100
+ * otherwise outlive the service (and retain everything it references).
101
+ */
102
+ #shuttingDown = false;
92
103
  #initialized = resolver();
93
104
  #cvr;
94
- #pipelinesSynced = false;
105
+ /**
106
+ * Indicates whether the query pipelines have completed initial catch-up and
107
+ * hydration with the CVR snapshot, and are ready for steady-state operation.
108
+ *
109
+ * - When `false`: The syncer is in its initial catch-up phase. The replica
110
+ * advances without diffs until it reaches `cvr.version.stateVersion`,
111
+ * after which `#maybeHydratePipelines` hydrates queries.
112
+ * - When `true`: Pipelines are fully populated and in steady-state; replica
113
+ * changes advance incrementally via `#advancePipelines`, and client query
114
+ * updates are processed via `#syncQueryPipelineSet`.
115
+ *
116
+ * Resets to `false` if `#advancePipelines` returns a `ResetPipelinesSignal`
117
+ * and pipelines must be reset and rehydrated.
118
+ */
119
+ #pipelinesHydrated = false;
95
120
  #servedVersion = null;
96
121
  #e2eServingLagTracker = new E2EServingLagTracker();
97
122
  #expiredQueriesTimer = 0;
98
123
  #authMaintenanceTimer = 0;
99
124
  #setTimeout;
125
+ #now;
100
126
  #customQueryTransformer;
101
127
  #queryReplacements = /* @__PURE__ */ new Map();
102
128
  #activeClients = getOrCreateUpDownCounter("sync", "active-clients", "Number of active sync clients");
@@ -123,10 +149,64 @@ var ViewSyncerService = class {
123
149
  #pipelineResets = getOrCreateCounter("sync", "pipeline-resets", "Number of pipeline resets");
124
150
  #rowSetSignatureDrifts = getOrCreateCounter("sync", "query.row-set-signature-drifts", "Number of times re-hydration of an unchanged query produced a different row-set signature than what is stored in the CVR (forcing a configVersion bump and full re-execution). Expected to be near-zero in steady state; persistent non-zero values indicate non-deterministic query execution (e.g. Cap operator picking different N-row subsets).");
125
151
  #sameHashRehydrationVersionBumps = getOrCreateCounter("sync", "query.same-hash-rehydrations-forced-bump", "Number of times query-set reconciliation forced a configVersion bump for already-gotten same-transformation-hash query rehydration because trackQueries would not otherwise bump. Expected to be near-zero; non-zero values indicate pipeline/CVR row-set drift reached query-set reconciliation.");
152
+ #hydrationBudgetExhaustions = getOrCreateCounter("sync", "hydration_budget_exhaustions", {
153
+ description: "Number of hydration passes that exhaust their budget.",
154
+ unit: "{pass}"
155
+ });
156
+ #hydrationBudgetEvictions = getOrCreateCounter("sync", "hydration_budget_evictions", {
157
+ description: "Number of inactive queries removed after hydration budget exhaustion.",
158
+ unit: "{query}"
159
+ });
160
+ #hydrationBudgetElapsed = getOrCreateValueHistogram("sync", "hydration_budget_elapsed", {
161
+ description: "Elapsed milliseconds when optional query hydration stopped.",
162
+ unit: "ms",
163
+ bucketBoundaries: [
164
+ 1,
165
+ 2,
166
+ 5,
167
+ 10,
168
+ 20,
169
+ 50,
170
+ 100,
171
+ 200,
172
+ 500,
173
+ 1e3,
174
+ 5e3
175
+ ]
176
+ });
177
+ #hydrationBudgetOvershoot = getOrCreateValueHistogram("sync", "hydration_budget_overshoot", {
178
+ description: "Milliseconds elapsed beyond the configured hydration budget.",
179
+ unit: "ms",
180
+ bucketBoundaries: [
181
+ 0,
182
+ 1,
183
+ 2,
184
+ 5,
185
+ 10,
186
+ 20,
187
+ 50,
188
+ 100,
189
+ 200,
190
+ 500,
191
+ 1e3
192
+ ]
193
+ });
194
+ #queryEvictions = getOrCreateCounter("sync", "query_evictions", {
195
+ description: "Number of queries evicted from the CVR ahead of their removal by the client, grouped by reason. Inactive queries are evicted by ttl and hydration-budget; hydration-timeout and hydration-circuit-breaker evict active queries as well.",
196
+ unit: "{query}"
197
+ });
198
+ #hydrationTimeouts = getOrCreateCounter("sync", "hydration_timeouts", {
199
+ description: "Number of query hydrations aborted for exceeding the query hydration timeout.",
200
+ unit: "{query}"
201
+ });
202
+ #hydrationCircuitBreakerRejections = getOrCreateCounter("sync", "hydration_circuit_breaker_rejections", {
203
+ description: "Number of queries rejected without hydration because their hydration circuit breaker was open.",
204
+ unit: "{query}"
205
+ });
126
206
  #inspectorDelegate;
127
207
  #config;
128
208
  #runPriorityOp;
129
- constructor(config, lc, shard, taskID, clientGroupID, cvrDb, pipelineDriver, versionChanges, drainCoordinator, slowHydrateThreshold, inspectorDelegate, connContextManager, customQueryTransformer, runPriorityOp, keepaliveMs = DEFAULT_KEEPALIVE_MS, setTimeoutFn = setTimeout.bind(globalThis)) {
209
+ constructor(config, lc, shard, taskID, clientGroupID, cvrDb, pipelineDriver, versionChanges, drainCoordinator, slowHydrateThreshold, inspectorDelegate, connContextManager, customQueryTransformer, runPriorityOp, keepaliveMs = DEFAULT_KEEPALIVE_MS, setTimeoutFn = setTimeout.bind(globalThis), now = performance.now.bind(performance)) {
130
210
  this.#config = config;
131
211
  this.id = clientGroupID;
132
212
  this.connContextManager = connContextManager;
@@ -141,6 +221,8 @@ var ViewSyncerService = class {
141
221
  this.#customQueryTransformer = customQueryTransformer;
142
222
  this.#cvrStore = new CVRStore(lc, cvrDb, shard, taskID, clientGroupID, () => this.#stateChanges.cancel());
143
223
  this.#setTimeout = setTimeoutFn;
224
+ this.#now = now;
225
+ this.#hydrationCircuitBreaker = new HydrationCircuitBreaker(config.viewSyncerQueryHydrationTimeoutMs ?? 0, void 0, now);
144
226
  this.#runPriorityOp = runPriorityOp;
145
227
  this.keepalive();
146
228
  }
@@ -190,10 +272,20 @@ var ViewSyncerService = class {
190
272
  });
191
273
  }
192
274
  readyState() {
193
- return Promise.race([this.#initialized.promise, this.#drainCoordinator.draining]);
275
+ return new Promise((resolve, reject) => {
276
+ const unsubscribe = this.#drainCoordinator.onDraining(() => resolve("draining"));
277
+ this.#initialized.promise.then((state) => {
278
+ unsubscribe();
279
+ resolve(state);
280
+ }, (err) => {
281
+ unsubscribe();
282
+ reject(err);
283
+ });
284
+ });
194
285
  }
195
286
  async run() {
196
287
  try {
288
+ this.#scheduleShutdown(this.#keepaliveMs);
197
289
  if (await this.readyState() === "draining") {
198
290
  this.#lc.debug?.(`draining view-syncer ${this.id} before running`);
199
291
  this.stop();
@@ -214,26 +306,29 @@ var ViewSyncerService = class {
214
306
  lc.info?.(`resetting CVR: ${message}`);
215
307
  throw new ClientNotFoundError(message);
216
308
  }
217
- if (this.#pipelinesSynced) {
309
+ let previousQueries;
310
+ if (this.#pipelinesHydrated) {
218
311
  const result = await this.#advancePipelines(lc, cvr);
219
312
  if (result === "success") return;
220
313
  lc.info?.(`resetting pipelines: ${result.message}`);
221
314
  this.#pipelineResets.add(1, { reason: result.reason });
315
+ switch (result.reason) {
316
+ case "advancement-timeout":
317
+ case "scalar-subquery":
318
+ case "truncation":
319
+ case "schema-change":
320
+ if (!Object.values(cvr.queries).some((q) => q.type === "client")) previousQueries = new Map(this.#pipelines.queries());
321
+ break;
322
+ case "permissions-change": break;
323
+ default: unreachable(result.reason);
324
+ }
222
325
  this.#pipelines.reset(clientSchema);
223
- this.#pipelinesSynced = false;
326
+ this.#pipelinesHydrated = false;
224
327
  this.connContextManager.setSharedRetransformReady(false);
225
328
  }
226
- const version = this.#pipelines.advanceWithoutDiff();
227
- const cvrVer = versionString(cvr.version);
228
- if (version < cvr.version.stateVersion) {
229
- lc.debug?.(`replica@${version} is behind cvr@${cvrVer}`);
230
- return;
231
- }
232
- lc.info?.(`init pipelines@${version} (cvr@${cvrVer})`);
233
- const driftedQueryIDs = await this.#hydrateUnchangedQueries(lc, cvr);
234
- await this.#syncQueryPipelineSet(lc, cvr, "missing", void 0, driftedQueryIDs);
235
- this.#pipelinesSynced = true;
236
- this.connContextManager.setSharedRetransformReady(true);
329
+ const backgroundConnCtx = this.connContextManager.getBackgroundConnectionContext();
330
+ if (backgroundConnCtx) await this.#maybeHydratePipelines(lc, cvr, backgroundConnCtx, previousQueries);
331
+ else lc.info?.("No validated background connection; deferring pipeline init");
237
332
  });
238
333
  }
239
334
  if (this.#drainCoordinator.shouldDrain()) this.#drainCoordinator.drainNextIn(this.#totalHydrationTimeMs());
@@ -247,11 +342,57 @@ var ViewSyncerService = class {
247
342
  this.#stopped.resolve();
248
343
  }
249
344
  }
345
+ /**
346
+ * Ensures that query pipelines are hydrated and ready for steady-state
347
+ * operation.
348
+ *
349
+ * Preconditions:
350
+ * 1. Must be called from within the `#lock`.
351
+ * 2. The caller must provide an active, validated `ConnectionContext` for
352
+ * evaluating custom query transforms and permissions.
353
+ *
354
+ * Readiness prerequisites:
355
+ * - The underlying `QueryPipelineDriver` must be initialized with the schema
356
+ * (i.e. `this.#pipelines.initialized()` is true).
357
+ * - The SQLite replica must have caught up to at least the CVR's
358
+ * `stateVersion` (`version >= cvr.version.stateVersion`).
359
+ *
360
+ * If either prerequisite is not met, this method returns early as a no-op,
361
+ * waiting for the replica to catch up or the driver to initialize.
362
+ *
363
+ * Once ready:
364
+ * - Hydrates unchanged queries from the CVR snapshot without full row diffing.
365
+ * - Syncs missing, errored, or drifted queries via `#syncQueryPipelineSet`.
366
+ * - Sets `#pipelinesHydrated = true` and enables shared retransformations.
367
+ */
368
+ async #maybeHydratePipelines(lc, cvr, connCtx, previousQueries) {
369
+ if (!this.#pipelines.initialized()) return;
370
+ const version = this.#pipelines.advanceWithoutDiff();
371
+ const cvrVer = versionString(cvr.version);
372
+ if (version < cvr.version.stateVersion) {
373
+ lc.debug?.(`replica@${version} is behind cvr@${cvrVer}`);
374
+ return;
375
+ }
376
+ lc.info?.(`init pipelines@${version} (cvr@${cvrVer})`);
377
+ const hydrationBudget = new HydrationBudget(this.#config.viewSyncerHydrationBudgetMs ?? 0, this.#now);
378
+ const hydrationPassStats = {
379
+ activeHydratedQueries: 0,
380
+ inactiveHydratedQueries: 0
381
+ };
382
+ const driftedQueryIDs = await this.#hydrateUnchangedQueries(lc, cvr, connCtx, hydrationPassStats, hydrationBudget, previousQueries);
383
+ await this.#syncQueryPipelineSet(lc, cvr, "missing", connCtx, driftedQueryIDs, hydrationBudget, hydrationPassStats, previousQueries);
384
+ this.#pipelinesHydrated = true;
385
+ this.connContextManager.setSharedRetransformReady(true);
386
+ }
250
387
  #removeExpiredQueries = async (lc, cvr) => {
251
388
  if (hasExpiredQueries(cvr)) {
252
389
  lc = lc.withContext("method", "#removeExpiredQueries");
253
390
  lc.debug?.("Queries have expired");
254
- if (this.#pipelinesSynced) await this.#syncQueryPipelineSet(lc, cvr, "missing", void 0);
391
+ if (this.#pipelinesHydrated) {
392
+ const connCtx = this.connContextManager.getBackgroundConnectionContext();
393
+ if (connCtx) await this.#syncQueryPipelineSet(lc, cvr, "missing", connCtx);
394
+ else lc.info?.("No validated background connection to remove expired queries; deferring");
395
+ }
255
396
  }
256
397
  this.#scheduleExpireEviction(lc, cvr);
257
398
  };
@@ -333,7 +474,14 @@ var ViewSyncerService = class {
333
474
  return true;
334
475
  }
335
476
  #shutdownTimer = null;
477
+ #stopShutdownTimer() {
478
+ if (this.#shutdownTimer !== null) {
479
+ clearTimeout(this.#shutdownTimer);
480
+ this.#shutdownTimer = null;
481
+ }
482
+ }
336
483
  #scheduleShutdown(delayMs = 0) {
484
+ if (this.#shuttingDown) return;
337
485
  this.#shutdownTimer ??= this.#setTimeout(() => {
338
486
  this.#shutdownTimer = null;
339
487
  this.#runInLockWithCVR(() => {}).catch((e) => this.#stateChanges.fail(e));
@@ -380,6 +528,7 @@ var ViewSyncerService = class {
380
528
  */
381
529
  #scheduleAuthMaintenance(lc) {
382
530
  this.#stopAuthMaintenanceTimer();
531
+ if (this.#shuttingDown) return;
383
532
  const plan = this.connContextManager.planMaintenance();
384
533
  if (plan.earliestDeadlineAt === void 0) {
385
534
  lc.debug?.("No auth maintenance wakeup scheduled");
@@ -467,7 +616,7 @@ var ViewSyncerService = class {
467
616
  }
468
617
  lc.debug?.("Auth changed, re-validating and re-transforming queries");
469
618
  const connCtx = this.connContextManager.mustGetConnectionContext(selector);
470
- if (!this.#pipelinesSynced) {
619
+ if (!this.#pipelinesHydrated) {
471
620
  if (!await this.#validateConnection(connCtx)) return;
472
621
  }
473
622
  return await this.#handleConfigUpdate(lc, clientID, {}, cvr, "all", void 0, connCtx);
@@ -496,6 +645,7 @@ var ViewSyncerService = class {
496
645
  }
497
646
  #startTTLClockInterval(lc) {
498
647
  this.#stopTTLClockInterval();
648
+ if (this.#shuttingDown) return;
499
649
  this.#ttlClockInterval = this.#setTimeout(() => {
500
650
  this.#updateTTLClockInCVRWithoutLock(lc);
501
651
  this.#startTTLClockInterval(lc);
@@ -529,7 +679,8 @@ var ViewSyncerService = class {
529
679
  await pokers.end(newCVR.version);
530
680
  });
531
681
  }
532
- if (this.#pipelinesSynced) await this.#syncQueryPipelineSet(lc, this.#cvr, customQueryTransformMode, connCtx);
682
+ if (!this.#pipelinesHydrated) await this.#maybeHydratePipelines(lc, this.#cvr, connCtx);
683
+ else await this.#syncQueryPipelineSet(lc, this.#cvr, customQueryTransformMode, connCtx);
533
684
  return this.#cvr;
534
685
  }
535
686
  /**
@@ -625,6 +776,7 @@ var ViewSyncerService = class {
625
776
  #scheduleExpireEviction(lc, cvr) {
626
777
  const { ttlClock } = cvr;
627
778
  this.#stopExpireTimer();
779
+ if (this.#shuttingDown) return;
628
780
  const next = nextEvictionTime(cvr);
629
781
  if (next === void 0) {
630
782
  lc.debug?.("no inactive queries with ttl");
@@ -652,7 +804,7 @@ var ViewSyncerService = class {
652
804
  *
653
805
  * This must be called from within the #lock.
654
806
  */
655
- async #hydrateUnchangedQueries(lc, cvr) {
807
+ async #hydrateUnchangedQueries(lc, cvr, connCtx, hydrationPassStats, hydrationBudget, previousQueries) {
656
808
  assert(this.#pipelines.initialized(), "pipelines must be initialized");
657
809
  const dbVersion = this.#pipelines.currentVersion();
658
810
  const cvrVersion = cvr.version;
@@ -661,33 +813,34 @@ var ViewSyncerService = class {
661
813
  return /* @__PURE__ */ new Set();
662
814
  }
663
815
  const gotQueries = Object.entries(cvr.queries).filter(([_, state]) => state.transformationHash !== void 0);
816
+ const { required: requiredGotQueries } = classifyQueriesForHydration(gotQueries.map(([, query]) => query));
664
817
  const customQueries = /* @__PURE__ */ new Map();
665
818
  const otherQueries = [];
666
- let inactivatedCount = 0;
667
- for (const [, query] of gotQueries) {
668
- if (query.type !== "internal" && Object.values(query.clientState).every(({ inactivatedAt }) => inactivatedAt !== void 0)) {
669
- inactivatedCount++;
670
- continue;
671
- }
672
- if (query.type === "custom") customQueries.set(query.id, query);
673
- else otherQueries.push(query);
674
- }
819
+ const inactivatedCount = gotQueries.length - requiredGotQueries.length;
675
820
  const transformedQueries = [];
821
+ for (const query of requiredGotQueries) if (query.type === "custom") {
822
+ const previous = previousQueries?.get(query.id);
823
+ if (previous && previous.transformationHash === query.transformationHash) transformedQueries.push({
824
+ id: query.id,
825
+ transformationHash: previous.transformationHash,
826
+ transformedAst: previous.originalAst ?? previous.transformedAst
827
+ });
828
+ else customQueries.set(query.id, query);
829
+ } else otherQueries.push(query);
676
830
  let customErrorCount = 0;
677
831
  let customHashMismatchCount = 0;
678
832
  let otherHashMismatchCount = 0;
679
833
  if (customQueries.size > 0 && !this.#customQueryTransformer) lc.warn?.("Custom/named queries were requested but no `ZERO_QUERY_URL` is configured for Zero Cache.");
680
- const backgroundContext = this.connContextManager.mustGetBackgroundConnectionContext();
681
834
  const customQueryTransformer = this.#customQueryTransformer;
682
835
  if (customQueryTransformer && customQueries.size > 0) {
683
- const transformedCustomQueries = await this.#runPriorityOp(lc, "#hydrateUnchangedQueries transforming custom queries", () => customQueryTransformer.transform(backgroundContext, customQueries.values()));
684
- if (transformedCustomQueries.kind === "success" && !transformedCustomQueries.cached) this.connContextManager.validateConnection(backgroundContext, backgroundContext.revision, transformedCustomQueries.validation);
836
+ const transformedCustomQueries = await hydrationBudget.excluding(() => this.#runPriorityOp(lc, "#hydrateUnchangedQueries transforming custom queries", () => customQueryTransformer.transform(connCtx, customQueries.values())));
837
+ if (transformedCustomQueries.kind === "success" && !transformedCustomQueries.cached) this.connContextManager.validateConnection(connCtx, connCtx.revision, transformedCustomQueries.validation);
685
838
  if (Array.isArray(transformedCustomQueries.result)) for (const q of transformedCustomQueries.result) if ("error" in q) customErrorCount++;
686
839
  else if (q.transformationHash !== customQueries.get(q.id)?.transformationHash) customHashMismatchCount++;
687
840
  else transformedQueries.push(q);
688
841
  }
689
842
  for (const q of otherQueries) {
690
- const transformed = transformAndHashQuery(lc, q.id, q.ast, must(this.#pipelines.currentPermissions()).permissions ?? { tables: {} }, backgroundContext.auth?.type === "jwt" ? backgroundContext.auth : void 0, q.type === "internal");
843
+ const transformed = transformAndHashQuery(lc, q.id, q.ast, must(this.#pipelines.currentPermissions()).permissions ?? { tables: {} }, connCtx.auth?.type === "jwt" ? connCtx.auth : void 0, q.type === "internal");
691
844
  if (transformed.transformationHash === q.transformationHash) transformedQueries.push(transformed);
692
845
  else otherHashMismatchCount++;
693
846
  }
@@ -700,6 +853,10 @@ var ViewSyncerService = class {
700
853
  for (const { id: queryID, transformationHash, transformedAst } of transformedQueries) {
701
854
  const query = cvr.queries[queryID];
702
855
  const queryName = query.type === "custom" ? query.name : void 0;
856
+ if (query.type !== "internal" && this.#hydrationCircuitBreaker.isOpen(transformationHash)) {
857
+ lc.info?.(`skipping hydration of ${queryID}: hydration circuit breaker open`);
858
+ continue;
859
+ }
703
860
  const covered = queryCoveringIndex ? this.#findQueryCoverageShadowHit(queryCoveringIndex, queryID, transformationHash, transformedAst, queryName) : void 0;
704
861
  totalHydratedQueries++;
705
862
  if (covered) {
@@ -708,19 +865,35 @@ var ViewSyncerService = class {
708
865
  }
709
866
  const timer = new TimeSliceTimer(lc);
710
867
  let count = 0;
868
+ let timedOut = false;
711
869
  await startAsyncSpan(tracer, "vs.#hydrateUnchangedQueries.addQuery", async (span) => {
712
870
  span.setAttribute("queryHash", queryID);
713
871
  span.setAttribute("transformationHash", transformationHash);
714
872
  span.setAttribute("table", transformedAst.table);
715
873
  if (queryName !== void 0) span.setAttribute("queryName", queryName);
716
- for (const change of this.#pipelines.addQuery(transformationHash, queryID, transformedAst, await timer.start(), queryName, "unchanged-query-rehydrate")) if (change === "yield") await timer.yieldProcess("yield in hydrateUnchangedQueries");
717
- else count++;
874
+ for (const change of this.#pipelines.addQuery(transformationHash, queryID, transformedAst, await timer.start(), queryName, "unchanged-query-rehydrate")) if (change === "yield") {
875
+ if (query.type !== "internal" && this.#hydrationCircuitBreaker.exceeded(timer.totalElapsed())) {
876
+ timedOut = true;
877
+ await timer.yieldProcess("yield in hydrateUnchangedQueries");
878
+ break;
879
+ }
880
+ await timer.yieldProcess("yield in hydrateUnchangedQueries");
881
+ } else count++;
718
882
  });
719
883
  const elapsed = timer.totalElapsed();
884
+ if (timedOut) {
885
+ this.#recordHydrationTimeout(lc, {
886
+ id: queryID,
887
+ transformationHash,
888
+ name: queryName
889
+ }, elapsed, count);
890
+ continue;
891
+ }
892
+ hydrationPassStats.activeHydratedQueries++;
720
893
  this.#hydrations.add(1);
721
894
  this.#hydrationTime.recordMs(elapsed);
722
- this.#addQueryMaterializationServerMetric(transformationHash, elapsed);
723
- this.#inspectorDelegate.addQuery(transformationHash, transformedAst);
895
+ this.#addQueryMaterializationServerMetric(queryID, elapsed);
896
+ this.#inspectorDelegate.addQuery(queryID, transformedAst);
724
897
  lc.debug?.(`hydrated ${count} rows for ${queryID} (${elapsed} ms)`);
725
898
  let drifted = false;
726
899
  const storedSigHex = cvr.queries[queryID]?.rowSetSignature;
@@ -781,11 +954,7 @@ var ViewSyncerService = class {
781
954
  return;
782
955
  }
783
956
  const appErrorGroups = /* @__PURE__ */ new Map();
784
- for (const err of errorOrErrors) for (const clientId of getAffectedClientIDs([err.id])) {
785
- const group = appErrorGroups.get(clientId) ?? [];
786
- group.push(err);
787
- appErrorGroups.set(clientId, group);
788
- }
957
+ for (const err of errorOrErrors) for (const clientId of getAffectedClientIDs([err.id])) getOrInsertComputed(appErrorGroups, clientId, newArray).push(err);
789
958
  for (const [clientId, errors] of appErrorGroups) this.#clients.get(clientId)?.sendQueryTransformApplicationErrors(errors);
790
959
  }
791
960
  #addQueryMaterializationServerMetric(queryID, elapsed) {
@@ -821,7 +990,10 @@ var ViewSyncerService = class {
821
990
  *
822
991
  * This must be called from within the #lock.
823
992
  */
824
- #syncQueryPipelineSet(lc, cvr, customQueryTransformMode, connCtx, driftedQueryIDs = /* @__PURE__ */ new Set()) {
993
+ #syncQueryPipelineSet(lc, cvr, customQueryTransformMode, connCtx, driftedQueryIDs = /* @__PURE__ */ new Set(), hydrationBudget = new HydrationBudget(this.#config.viewSyncerHydrationBudgetMs ?? 0, this.#now), hydrationPassStats = {
994
+ activeHydratedQueries: 0,
995
+ inactiveHydratedQueries: 0
996
+ }, previousQueries) {
825
997
  return startAsyncSpan(tracer, "vs.#syncQueryPipelineSet", async (span) => {
826
998
  const start = performance.now();
827
999
  span.setAttribute("clientGroupID", this.id);
@@ -830,38 +1002,45 @@ var ViewSyncerService = class {
830
1002
  const now = Date.now();
831
1003
  const ttlClock = this.#getTTLClock(now);
832
1004
  const cvrQueryEntires = Object.entries(cvr.queries);
833
- const customQueries = /* @__PURE__ */ new Map();
834
- const otherQueries = [];
835
1005
  const transformedQueries = [];
836
- const resolvedConnCtx = connCtx ?? this.connContextManager.mustGetBackgroundConnectionContext();
837
- for (const [id, query] of cvrQueryEntires) if (query.type === "custom") {
838
- assert(id === query.id, "custom query id mismatch");
839
- customQueries.set(id, query);
840
- } else otherQueries.push({
841
- id,
842
- query
843
- });
844
- for (const { id, query: origQuery } of otherQueries) {
845
- assert(id === origQuery.id, "query id mismatch");
846
- const transformed = transformAndHashQuery(lc, origQuery.id, origQuery.ast, must(this.#pipelines.currentPermissions()).permissions ?? { tables: {} }, resolvedConnCtx.auth?.type === "jwt" ? resolvedConnCtx.auth : void 0, origQuery.type === "internal");
847
- transformedQueries.push({
848
- id,
849
- origQuery,
850
- transformed
851
- });
852
- }
853
- if (customQueries.size > 0 && !this.#customQueryTransformer) lc.warn?.("Custom/named queries were requested but no `ZERO_QUERY_URL` is configured for Zero Cache.");
854
- let erroredQueryIDs;
855
- const customQueriesToTransform = customQueryTransformMode === "all" ? [...customQueries.values()] : customQueryTransformMode && [...customQueries.values()].filter((q) => !this.#pipelines.queries().has(q.id));
1006
+ const naturallyExpiredQueryIDs = new Set(cvrQueryEntires.map(([, query]) => query).filter((query) => expired(ttlClock, query)).map((query) => query.id));
1007
+ if (naturallyExpiredQueryIDs.size > 0) this.#queryEvictions.add(naturallyExpiredQueryIDs.size, { reason: "ttl" });
1008
+ const hydrationCandidates = cvrQueryEntires.map(([, query]) => query).filter((query) => !naturallyExpiredQueryIDs.has(query.id));
1009
+ const { required, optional } = classifyQueriesForHydration(hydrationCandidates);
1010
+ const requiredQueryIDs = new Set(required.map((query) => query.id));
1011
+ const optionalToHydrate = hydrationBudget.limitMs === 0 ? optional : optional.filter((query) => query.patchVersion !== void 0);
1012
+ if (!this.#customQueryTransformer && hydrationCandidates.some((query) => query.type === "custom")) lc.warn?.("Custom/named queries were requested but no `ZERO_QUERY_URL` is configured for Zero Cache.");
856
1013
  const customQueryTransformer = this.#customQueryTransformer;
857
- if (customQueryTransformer && customQueriesToTransform.length > 0) {
1014
+ const erroredQueryIDs = [];
1015
+ let transformedCustomQueryCount = 0;
1016
+ const transformOtherQueries = (queries) => {
1017
+ for (const origQuery of queries) {
1018
+ if (origQuery.type === "custom") continue;
1019
+ const transformed = transformAndHashQuery(lc, origQuery.id, origQuery.ast, must(this.#pipelines.currentPermissions()).permissions ?? { tables: {} }, connCtx.auth?.type === "jwt" ? connCtx.auth : void 0, origQuery.type === "internal");
1020
+ transformedQueries.push({
1021
+ id: origQuery.id,
1022
+ origQuery,
1023
+ transformed
1024
+ });
1025
+ }
1026
+ };
1027
+ const shouldTransformCustomQuery = (query) => {
1028
+ if (customQueryTransformMode === "all") return true;
1029
+ if (this.#pipelines.queries().has(query.id)) return false;
1030
+ const previous = previousQueries?.get(query.id);
1031
+ if (previous && previous.transformationHash === query.transformationHash) return false;
1032
+ return true;
1033
+ };
1034
+ const transformCustomQueries = async (queries) => {
1035
+ if (!customQueryTransformer || queries.length === 0) return;
1036
+ transformedCustomQueryCount += queries.length;
858
1037
  const transformStart = performance.now();
859
1038
  let transformedCustomQueries;
860
1039
  try {
861
- transformedCustomQueries = await this.#runPriorityOp(lc, "#syncQueryPipelineSet transforming custom queries", () => customQueryTransformer.transform(resolvedConnCtx, customQueriesToTransform));
1040
+ transformedCustomQueries = await hydrationBudget.excluding(() => this.#runPriorityOp(lc, "#syncQueryPipelineSet transforming custom queries", () => customQueryTransformer.transform(connCtx, queries)));
862
1041
  if (transformedCustomQueries.kind === "failed") throw new ProtocolErrorWithLevel(transformedCustomQueries.result, "warn");
863
1042
  else {
864
- if (!transformedCustomQueries.cached) this.connContextManager.validateConnection(resolvedConnCtx, resolvedConnCtx.revision, transformedCustomQueries.validation);
1043
+ if (!transformedCustomQueries.cached) this.connContextManager.validateConnection(connCtx, connCtx.revision, transformedCustomQueries.validation);
865
1044
  this.#queryTransformations.add(1, { result: "success" });
866
1045
  }
867
1046
  } catch (e) {
@@ -872,8 +1051,9 @@ var ViewSyncerService = class {
872
1051
  this.#queryTransformationTime.recordMs(transformDuration);
873
1052
  }
874
1053
  const successfullyTransformedCustomQueries = /* @__PURE__ */ new Map();
875
- erroredQueryIDs = this.#processTransformedCustomQueries(lc, transformedCustomQueries.result, (q) => {
876
- const origQuery = customQueries.get(q.id);
1054
+ const customQueryMap = new Map(queries.map((query) => [query.id, query]));
1055
+ erroredQueryIDs.push(...this.#processTransformedCustomQueries(lc, transformedCustomQueries.result, (q) => {
1056
+ const origQuery = customQueryMap.get(q.id);
877
1057
  if (origQuery) {
878
1058
  successfullyTransformedCustomQueries.set(q.id, q);
879
1059
  transformedQueries.push({
@@ -882,7 +1062,7 @@ var ViewSyncerService = class {
882
1062
  transformed: q
883
1063
  });
884
1064
  }
885
- }, customQueries);
1065
+ }, customQueryMap));
886
1066
  for (const [queryID, newTransform] of successfullyTransformedCustomQueries) {
887
1067
  const existingTransformHash = cvr.queries[queryID]?.transformationHash;
888
1068
  if (existingTransformHash) {
@@ -895,21 +1075,55 @@ var ViewSyncerService = class {
895
1075
  } else this.#queryTransformationNoOps.add(1);
896
1076
  }
897
1077
  }
1078
+ };
1079
+ const queriesToTransform = [...required, ...optionalToHydrate];
1080
+ transformOtherQueries(queriesToTransform);
1081
+ if (customQueryTransformMode === "missing" && previousQueries) {
1082
+ for (const query of queriesToTransform) if (query.type === "custom" && !this.#pipelines.queries().has(query.id)) {
1083
+ const previous = previousQueries.get(query.id);
1084
+ if (previous && previous.transformationHash === query.transformationHash) {
1085
+ this.#queryTransformationNoOps.add(1);
1086
+ transformedQueries.push({
1087
+ id: query.id,
1088
+ origQuery: query,
1089
+ transformed: {
1090
+ id: query.id,
1091
+ transformationHash: previous.transformationHash,
1092
+ transformedAst: previous.originalAst ?? previous.transformedAst
1093
+ }
1094
+ });
1095
+ }
1096
+ }
898
1097
  }
899
- const removeQueriesQueryIds = new Set([...Object.values(cvr.queries).filter((q) => expired(ttlClock, q)).map((q) => q.id), ...erroredQueryIDs || []]);
1098
+ await transformCustomQueries(queriesToTransform.filter((query) => query.type === "custom" && shouldTransformCustomQuery(query)));
1099
+ const circuitBrokenQueries = transformedQueries.filter(({ id, origQuery, transformed }) => origQuery.type !== "internal" && this.#pipelines.queries().get(id)?.transformationHash !== transformed.transformationHash && this.#hydrationCircuitBreaker.isOpen(transformed.transformationHash)).map(({ id, origQuery, transformed }) => ({
1100
+ id,
1101
+ transformationHash: transformed.transformationHash,
1102
+ name: origQuery.type === "custom" ? origQuery.name : void 0
1103
+ }));
1104
+ if (circuitBrokenQueries.length > 0) this.#rejectCircuitBrokenQueries(lc, cvr, circuitBrokenQueries);
1105
+ const removeQueriesQueryIds = new Set([
1106
+ ...naturallyExpiredQueryIDs,
1107
+ ...erroredQueryIDs,
1108
+ ...circuitBrokenQueries.map(({ id }) => id)
1109
+ ]);
900
1110
  const addQueries = transformedQueries.map(({ id, origQuery, transformed }) => ({
901
1111
  id,
902
1112
  ast: transformed.transformedAst,
903
1113
  transformationHash: transformed.transformationHash,
904
1114
  name: origQuery.type === "custom" ? origQuery.name : void 0
905
1115
  })).filter((q) => !removeQueriesQueryIds.has(q.id) && this.#pipelines.queries().get(q.id)?.transformationHash !== q.transformationHash);
906
- lc.info?.(`syncQueryPipelineSet: ${cvrQueryEntires.length} CVR queries, ${customQueriesToTransform.length} custom re-transformed, ${erroredQueryIDs?.length ?? 0} errored, ${removeQueriesQueryIds.size} to remove, ${addQueries.length} to add`);
1116
+ const hydrationOrder = new Map([...required, ...optional].map((query, index) => [query.id, index]));
1117
+ addQueries.sort((a, b) => must(hydrationOrder.get(a.id)) - must(hydrationOrder.get(b.id)));
1118
+ const requiredAddQueries = addQueries.filter((query) => requiredQueryIDs.has(query.id));
1119
+ const optionalAddQueries = addQueries.filter((query) => !requiredQueryIDs.has(query.id));
1120
+ lc.info?.(`syncQueryPipelineSet: ${cvrQueryEntires.length} CVR queries, ${transformedCustomQueryCount} custom re-transformed, ${erroredQueryIDs.length} errored, ${circuitBrokenQueries.length} circuit-broken, ${removeQueriesQueryIds.size} to remove, ${addQueries.length} to add`);
907
1121
  for (const q of addQueries) {
908
1122
  const orig = cvr.queries[q.id];
909
1123
  lc.debug?.("ViewSyncer adding query", q.ast, "transformed from", orig.type === "custom" ? orig.name : orig.ast);
910
1124
  }
911
1125
  if (addQueries.length > 0 || removeQueriesQueryIds.size > 0) {
912
- await this.#addAndRemoveQueries(lc, cvr, addQueries, Array.from(removeQueriesQueryIds, (id) => ({ id })), driftedQueryIDs);
1126
+ await this.#addAndRemoveQueries(lc, cvr, requiredAddQueries, optionalAddQueries, Array.from(removeQueriesQueryIds, (id) => ({ id })), hydrationBudget, hydrationPassStats, driftedQueryIDs);
913
1127
  if (addQueries.length > 0) this.#viewSyncerHydration.recordMs(performance.now() - start);
914
1128
  } else await this.#catchupClients(lc, cvr);
915
1129
  });
@@ -942,8 +1156,93 @@ var ViewSyncerService = class {
942
1156
  record.count++;
943
1157
  if (record.count >= THRASH_THRESHOLD) this.#lc.warn?.(`Query thrashing detected for query ${queryID}. ${record.count} replacements in 60s. This may indicate clients with different auth contexts connecting to the same client group.`);
944
1158
  }
945
- #addAndRemoveQueries(lc, cvr, addQueries, removeQueries, driftedQueryIDs = /* @__PURE__ */ new Set()) {
1159
+ /**
1160
+ * Records that hydrating `query` was aborted for exceeding the query
1161
+ * hydration timeout, and opens its circuit breaker.
1162
+ */
1163
+ #recordHydrationTimeout(lc, query, elapsedMs, rowCount) {
1164
+ this.#hydrationCircuitBreaker.trip(query.transformationHash);
1165
+ this.#hydrationTimeouts.add(1);
1166
+ lc.warn?.("Query hydration aborted for exceeding the hydration timeout", {
1167
+ clientGroupID: this.id,
1168
+ queryHash: query.id,
1169
+ transformationHash: query.transformationHash,
1170
+ ...query.name !== void 0 && { queryName: query.name },
1171
+ hydrationTimeoutMs: this.#hydrationCircuitBreaker.timeoutMs,
1172
+ hydrationElapsedMs: elapsedMs,
1173
+ ...rowCount !== void 0 && { hydrationRowCount: rowCount },
1174
+ circuitBreakerOpenMs: this.#hydrationCircuitBreaker.openMs
1175
+ });
1176
+ }
1177
+ /** Records that `query` was rejected without hydration by its open breaker. */
1178
+ #recordCircuitBreakerRejection(lc, query) {
1179
+ this.#hydrationCircuitBreakerRejections.add(1);
1180
+ lc.warn?.("Query rejected by its open hydration circuit breaker", {
1181
+ clientGroupID: this.id,
1182
+ queryHash: query.id,
1183
+ transformationHash: query.transformationHash,
1184
+ ...query.name !== void 0 && { queryName: query.name },
1185
+ hydrationTimeoutMs: this.#hydrationCircuitBreaker.timeoutMs,
1186
+ circuitBreakerOpenMs: this.#hydrationCircuitBreaker.openMs
1187
+ });
1188
+ }
1189
+ /**
1190
+ * Handles queries whose hydration circuit breaker is open: they are counted,
1191
+ * logged, and errored to the affected clients. The caller removes them from
1192
+ * the CVR.
1193
+ */
1194
+ #rejectCircuitBrokenQueries(lc, cvr, queries) {
1195
+ for (const query of queries) this.#recordCircuitBreakerRejection(lc, query);
1196
+ this.#queryEvictions.add(queries.length, { reason: "hydration-circuit-breaker" });
1197
+ this.#sendHydrationTimeoutErrors(cvr, queries);
1198
+ }
1199
+ /**
1200
+ * Sends a per-query error to every client that desires one of `queries`.
1201
+ * The error goes out as a `transformError` application error, which every
1202
+ * client understands as "this query errored" without affecting the
1203
+ * connection or the client's other queries.
1204
+ */
1205
+ #sendHydrationTimeoutErrors(cvr, queries) {
1206
+ const timeoutMs = this.#hydrationCircuitBreaker.timeoutMs;
1207
+ const errorsByClient = /* @__PURE__ */ new Map();
1208
+ for (const { id, name } of queries) {
1209
+ const query = cvr.queries[id];
1210
+ if (query === void 0 || query.type === "internal") continue;
1211
+ const error = {
1212
+ error: "app",
1213
+ id,
1214
+ name: name ?? (query.type === "custom" ? query.name : "legacy"),
1215
+ message: `Query hydration exceeded the ${timeoutMs}ms limit (ZERO_VIEW_SYNCER_QUERY_HYDRATION_TIMEOUT_MS) and was aborted`,
1216
+ details: {
1217
+ kind: "HydrationTimeout",
1218
+ timeoutMs
1219
+ }
1220
+ };
1221
+ for (const clientID of Object.keys(query.clientState)) getOrInsertComputed(errorsByClient, clientID, newArray).push(error);
1222
+ }
1223
+ for (const [clientID, errors] of errorsByClient) this.#clients.get(clientID)?.sendQueryTransformApplicationErrors(errors);
1224
+ }
1225
+ #recordHydrationBudgetExhaustion(lc, hydrationBudget, activeHydratedQueries, inactiveHydratedQueries, inactiveEvictedQueryIDs) {
1226
+ const elapsedMs = hydrationBudget.exhaustedAtMs;
1227
+ if (elapsedMs === void 0) return;
1228
+ this.#hydrationBudgetExhaustions.add(1);
1229
+ this.#hydrationBudgetEvictions.add(inactiveEvictedQueryIDs.length);
1230
+ this.#hydrationBudgetElapsed.record(elapsedMs);
1231
+ this.#hydrationBudgetOvershoot.record(Math.max(0, elapsedMs - hydrationBudget.limitMs));
1232
+ if (inactiveEvictedQueryIDs.length > 0) this.#queryEvictions.add(inactiveEvictedQueryIDs.length, { reason: "hydration-budget" });
1233
+ lc.info?.("view-syncer hydration budget exhausted", {
1234
+ clientGroupID: this.id,
1235
+ hydrationBudgetMs: hydrationBudget.limitMs,
1236
+ hydrationElapsedMs: elapsedMs,
1237
+ activeHydratedQueries,
1238
+ inactiveHydratedQueries,
1239
+ inactiveEvictedQueries: inactiveEvictedQueryIDs.length,
1240
+ firstEvictedQueryHash: inactiveEvictedQueryIDs[0] ?? null
1241
+ });
1242
+ }
1243
+ #addAndRemoveQueries(lc, cvr, requiredQueries, optionalQueries, removeQueries, hydrationBudget, hydrationPassStats, driftedQueryIDs = /* @__PURE__ */ new Set()) {
946
1244
  return startAsyncSpan(tracer, "vs.#addAndRemoveQueries", async () => {
1245
+ const addQueries = [...requiredQueries, ...optionalQueries];
947
1246
  assert(addQueries.length > 0 || removeQueries.length > 0, "Must have queries to add or remove");
948
1247
  const start = performance.now();
949
1248
  const stateVersion = this.#pipelines.currentVersion();
@@ -960,6 +1259,7 @@ var ViewSyncerService = class {
960
1259
  updater.ensureNewVersion();
961
1260
  }
962
1261
  const { queryPatches, newVersion } = updater.trackQueries(lc, addQueries, removeQueries);
1262
+ if (hydrationBudget.limitMs !== 0 && optionalQueries.length > 0) assert(cmpVersions(cvr.version, newVersion) < 0, "Optional hydration requires a final poke version before row processing so a removal is actually poked");
963
1263
  const pokers = startPoke(this.#getClients(), newVersion);
964
1264
  for (const patch of queryPatches) await pokers.addPatch(patch);
965
1265
  for (const q of removeQueries) {
@@ -976,22 +1276,54 @@ var ViewSyncerService = class {
976
1276
  let totalHydratedQueries = 0;
977
1277
  let coveredHydratedQueries = 0;
978
1278
  let firstCoveredQuery;
1279
+ const hydratedQueryIDs = [];
1280
+ const budgetEvictedQueryIDs = [];
1281
+ const timedOutQueries = [];
1282
+ const rejectedQueries = [];
1283
+ const circuitBreaker = this.#hydrationCircuitBreaker;
979
1284
  const self = this;
980
1285
  await yieldProcess(lc);
981
- function* generateRowChanges(slowHydrateThreshold) {
982
- for (const q of addQueries) {
1286
+ function* hydrateQueries(queries, optional, slowHydrateThreshold) {
1287
+ for (let i = 0; i < queries.length; i++) {
1288
+ if (optional && hydrationBudget.exhausted()) {
1289
+ budgetEvictedQueryIDs.push(...queries.slice(i).map((query) => query.id));
1290
+ return;
1291
+ }
1292
+ const q = must(queries[i]);
983
1293
  let queryLC = lc.withContext("hash", q.id).withContext("queryHash", q.id).withContext("transformationHash", q.transformationHash);
984
1294
  if (q.name !== void 0) queryLC = queryLC.withContext("queryName", q.name);
985
1295
  queryLC.debug?.(`adding pipeline for query`, q.ast);
1296
+ const breakable = cvr.queries[q.id]?.type !== "internal";
1297
+ if (breakable && circuitBreaker.isOpen(q.transformationHash)) {
1298
+ rejectedQueries.push(q);
1299
+ self.#recordCircuitBreakerRejection(queryLC, q);
1300
+ continue;
1301
+ }
986
1302
  const covered = queryCoveringIndex ? self.#findQueryCoverageShadowHit(queryCoveringIndex, q.id, q.transformationHash, q.ast, q.name) : void 0;
987
1303
  totalHydratedQueries++;
988
1304
  if (covered) {
989
1305
  coveredHydratedQueries++;
990
1306
  firstCoveredQuery ??= covered;
991
1307
  }
992
- yield* pipelines.addQuery(q.transformationHash, q.id, q.ast, timer.startWithoutYielding(), q.name, "query-set-sync");
1308
+ let timedOut = false;
1309
+ for (const change of pipelines.addQuery(q.transformationHash, q.id, q.ast, timer.startWithoutYielding(), q.name, "query-set-sync")) {
1310
+ if (change === "yield" && breakable && circuitBreaker.exceeded(timer.totalElapsed())) {
1311
+ timedOut = true;
1312
+ yield change;
1313
+ break;
1314
+ }
1315
+ yield change;
1316
+ }
993
1317
  const elapsed = timer.stop();
994
1318
  totalProcessTime += elapsed;
1319
+ if (timedOut) {
1320
+ timedOutQueries.push(q);
1321
+ self.#recordHydrationTimeout(queryLC, q, elapsed);
1322
+ continue;
1323
+ }
1324
+ hydratedQueryIDs.push(q.id);
1325
+ if (optional) hydrationPassStats.inactiveHydratedQueries++;
1326
+ else hydrationPassStats.activeHydratedQueries++;
995
1327
  self.#addQueryMaterializationServerMetric(q.id, elapsed);
996
1328
  self.#inspectorDelegate.addQuery(q.id, q.ast);
997
1329
  queryCoveringIndex?.add(q.id, {
@@ -1005,18 +1337,43 @@ var ViewSyncerService = class {
1005
1337
  transformationHash: q.transformationHash,
1006
1338
  ...q.name !== void 0 && { name: q.name }
1007
1339
  });
1340
+ hydrations.add(1);
1341
+ hydrationTime.recordMs(elapsed);
1008
1342
  }
1009
- hydrations.add(1);
1010
- hydrationTime.recordMs(totalProcessTime);
1343
+ }
1344
+ function* generateRowChanges(slowHydrateThreshold) {
1345
+ yield* hydrateQueries(requiredQueries, false, slowHydrateThreshold);
1346
+ yield* hydrateQueries(optionalQueries, true, slowHydrateThreshold);
1011
1347
  }
1012
1348
  await this.#processChanges(lc, timer, generateRowChanges(this.#slowHydrateThreshold), updater, pokers);
1349
+ const abortedQueries = [...timedOutQueries, ...rejectedQueries];
1350
+ if (abortedQueries.length > 0) {
1351
+ const abortedQueryIDs = abortedQueries.map(({ id }) => id);
1352
+ for (const patch of await updater.abortExecutedQueries(lc, abortedQueryIDs)) await pokers.addPatch(patch);
1353
+ for (const queryID of abortedQueryIDs) {
1354
+ this.#pipelines.removeQuery(queryID);
1355
+ this.#inspectorDelegate.removeQuery(queryID);
1356
+ this.#queryReplacements.delete(queryID);
1357
+ }
1358
+ if (timedOutQueries.length > 0) this.#queryEvictions.add(timedOutQueries.length, { reason: "hydration-timeout" });
1359
+ if (rejectedQueries.length > 0) this.#queryEvictions.add(rejectedQueries.length, { reason: "hydration-circuit-breaker" });
1360
+ this.#sendHydrationTimeoutErrors(cvr, abortedQueries);
1361
+ }
1362
+ for (const patch of updater.removeTrackedQueries(budgetEvictedQueryIDs)) await pokers.addPatch(patch);
1363
+ for (const queryID of budgetEvictedQueryIDs) {
1364
+ this.#pipelines.removeQuery(queryID);
1365
+ this.#inspectorDelegate.removeQuery(queryID);
1366
+ this.#queryReplacements.delete(queryID);
1367
+ }
1368
+ this.#recordHydrationBudgetExhaustion(lc, hydrationBudget, hydrationPassStats.activeHydratedQueries, hydrationPassStats.inactiveHydratedQueries, budgetEvictedQueryIDs);
1013
1369
  this.#logQueryCoverageShadowSummary(lc, "add", totalHydratedQueries, coveredHydratedQueries, firstCoveredQuery);
1014
1370
  await startAsyncSpan(tracer, "vs.#syncQueryPipelineSet.deleteUnreferencedRows", async () => {
1015
1371
  for (const patch of await updater.deleteUnreferencedRows(lc)) await pokers.addPatch(patch);
1016
1372
  });
1017
1373
  this.#cvr = await this.#flushUpdater(lc, updater);
1374
+ if (budgetEvictedQueryIDs.length > 0) this.#scheduleExpireEviction(lc, this.#cvr);
1018
1375
  const finalVersion = this.#cvr.version;
1019
- await this.#catchupClients(lc, cvr, finalVersion, addQueries.map((q) => q.id), pokers);
1376
+ await this.#catchupClients(lc, cvr, finalVersion, hydratedQueryIDs, pokers);
1020
1377
  await startAsyncSpan(tracer, "vs.#syncQueryPipelineSet.pokeEnd", () => pokers.end(finalVersion));
1021
1378
  this.#markVersionServed(stateVersion);
1022
1379
  const wallTime = performance.now() - start;
@@ -1157,7 +1514,9 @@ var ViewSyncerService = class {
1157
1514
  *
1158
1515
  * Must be called from within the #lock.
1159
1516
  *
1160
- * Returns false if the advancement failed due to a schema change.
1517
+ * Returns 'success' if changes were successfully processed and poked,
1518
+ * or a `ResetPipelinesSignal` if advancement aborted (e.g. schema change,
1519
+ * timeout) and pipelines need to be reset and rehydrated.
1161
1520
  */
1162
1521
  #advancePipelines(lc, cvr) {
1163
1522
  return startAsyncSpan(tracer, "vs.#advancePipelines", async (span) => {
@@ -1165,20 +1524,27 @@ var ViewSyncerService = class {
1165
1524
  assert(this.#pipelines.initialized(), "pipelines must be initialized (advancePipelines");
1166
1525
  const start = performance.now();
1167
1526
  const timer = new TimeSliceTimer(lc);
1168
- const { version, numChanges, changes } = this.#pipelines.advance(timer);
1169
- lc = lc.withContext("newVersion", version);
1170
- const updater = new CVRQueryDrivenUpdater(this.#cvrStore, cvr, version, this.#pipelines.replicaVersion, (queryID) => this.#pipelines.rowSetSignature(queryID));
1171
- const pokers = startPoke(this.#getClients(cvr.version), updater.updatedVersion());
1172
- lc.debug?.(`applying ${numChanges} to advance to ${version}`);
1527
+ let pokers;
1528
+ let updater;
1529
+ let version;
1530
+ let numChanges = 0;
1173
1531
  try {
1174
- await this.#processChanges(lc, await timer.start(), changes, updater, pokers);
1532
+ const advancement = this.#pipelines.advance(timer);
1533
+ version = advancement.version;
1534
+ numChanges = advancement.numChanges;
1535
+ lc = lc.withContext("newVersion", version);
1536
+ updater = new CVRQueryDrivenUpdater(this.#cvrStore, cvr, version, this.#pipelines.replicaVersion, (queryID) => this.#pipelines.rowSetSignature(queryID));
1537
+ pokers = startPoke(this.#getClients(cvr.version), updater.updatedVersion());
1538
+ lc.debug?.(`applying ${numChanges} to advance to ${version}`);
1539
+ await this.#processChanges(lc, await timer.start(), advancement.changes, updater, pokers);
1175
1540
  } catch (e) {
1176
1541
  if (e instanceof ResetPipelinesSignal) {
1177
- await pokers.cancel();
1542
+ await pokers?.cancel();
1178
1543
  return e;
1179
1544
  }
1180
1545
  throw e;
1181
1546
  }
1547
+ assert(updater && pokers && version !== void 0, "advancement state missing");
1182
1548
  this.#cvr = await this.#flushUpdater(lc, updater);
1183
1549
  const finalVersion = this.#cvr.version;
1184
1550
  await startAsyncSpan(tracer, "vs.#advancePipelines.pokeEnd", () => pokers.end(finalVersion));
@@ -1199,7 +1565,7 @@ var ViewSyncerService = class {
1199
1565
  clientID,
1200
1566
  wsID: client.wsID
1201
1567
  });
1202
- return handleInspect(lc, body, cvr, client, this.#inspectorDelegate, this.id, this.#cvrStore, this.#config, connCtx);
1568
+ return handleInspect(lc, body, cvr, client, this.#inspectorDelegate, this.id, this, this.#cvrStore, this.#config, connCtx);
1203
1569
  };
1204
1570
  async #runBackgroundRetransform(lc) {
1205
1571
  const attemptRetransform = async (connCtx) => {
@@ -1286,15 +1652,18 @@ var ViewSyncerService = class {
1286
1652
  return this.#stopped.promise;
1287
1653
  }
1288
1654
  async #cleanup(err) {
1655
+ this.#shuttingDown = true;
1289
1656
  this.connContextManager.setSharedRetransformReady(false);
1290
1657
  this.#stopTTLClockInterval();
1291
1658
  this.#stopExpireTimer();
1292
1659
  this.#stopAuthMaintenanceTimer();
1660
+ this.#stopShutdownTimer();
1293
1661
  this.#customQueryTransformer?.destroy();
1294
1662
  for (const client of this.#clients.values()) if (err) client.fail(err);
1295
1663
  else client.close(`closed clientGroupID=${this.id}`);
1296
1664
  await this.#lock.withLock(() => {});
1297
1665
  this.#pipelines.destroy();
1666
+ this.#inspectorDelegate.clearAuthenticated(this.id, this);
1298
1667
  }
1299
1668
  /**
1300
1669
  * Test helper: Manually mark initialization as complete.
@@ -1330,8 +1699,10 @@ function isTransformFailedError(error) {
1330
1699
  return error.errorBody.kind === "TransformFailed" && !isAuthErrorBody(error.errorBody);
1331
1700
  }
1332
1701
  /**
1333
- * A query must be expired for all clients in order to be considered
1334
- * expired.
1702
+ * Whether a query's TTL has elapsed. A query must be expired for all clients
1703
+ * in order to be considered expired.
1704
+ *
1705
+ * Exported for testing.
1335
1706
  */
1336
1707
  function expired(ttlClock, q) {
1337
1708
  if (q.type === "internal") return false;