@syncular/server 0.1.3 → 0.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (758) hide show
  1. package/README.md +773 -13
  2. package/dist/admin.d.ts +136 -0
  3. package/dist/admin.js +168 -0
  4. package/dist/blob-handlers.d.ts +69 -0
  5. package/dist/blob-handlers.js +245 -0
  6. package/dist/blob-store.d.ts +111 -0
  7. package/dist/blob-store.js +0 -0
  8. package/dist/content-encoding.d.ts +27 -0
  9. package/dist/content-encoding.js +69 -0
  10. package/dist/context.d.ts +151 -0
  11. package/dist/context.js +21 -0
  12. package/dist/crdt-merger.d.ts +28 -0
  13. package/dist/crdt-merger.js +11 -0
  14. package/dist/d1-storage.d.ts +50 -0
  15. package/dist/d1-storage.js +0 -0
  16. package/dist/errors.d.ts +35 -9
  17. package/dist/errors.js +220 -11
  18. package/dist/events-ring.d.ts +55 -0
  19. package/dist/events-ring.js +96 -0
  20. package/dist/events.d.ts +231 -0
  21. package/dist/events.js +22 -0
  22. package/dist/frame-bytes.d.ts +20 -0
  23. package/dist/frame-bytes.js +75 -0
  24. package/dist/handler.d.ts +9 -0
  25. package/dist/handler.js +490 -0
  26. package/dist/index.d.ts +42 -33
  27. package/dist/index.js +45 -27
  28. package/dist/lease-store.d.ts +50 -0
  29. package/dist/lease-store.js +0 -0
  30. package/dist/pg-executor-pglite.d.ts +27 -0
  31. package/dist/pg-executor-pglite.js +33 -0
  32. package/dist/pg-executor.d.ts +67 -0
  33. package/dist/pg-executor.js +56 -0
  34. package/dist/postgres-fanout.d.ts +83 -0
  35. package/dist/postgres-fanout.js +71 -0
  36. package/dist/postgres-storage.d.ts +69 -0
  37. package/dist/postgres-storage.js +766 -0
  38. package/dist/prune.d.ts +24 -39
  39. package/dist/prune.js +44 -146
  40. package/dist/pull.d.ts +42 -67
  41. package/dist/pull.js +387 -1187
  42. package/dist/push.d.ts +31 -66
  43. package/dist/push.js +406 -655
  44. package/dist/realtime.d.ts +197 -0
  45. package/dist/realtime.js +848 -0
  46. package/dist/relational-rows.d.ts +229 -0
  47. package/dist/relational-rows.js +514 -0
  48. package/dist/s3-blob-store.d.ts +142 -0
  49. package/dist/s3-blob-store.js +458 -0
  50. package/dist/s3-segment-store.d.ts +95 -0
  51. package/dist/s3-segment-store.js +380 -0
  52. package/dist/schema.d.ts +82 -286
  53. package/dist/schema.js +133 -9
  54. package/dist/scopes.d.ts +48 -0
  55. package/dist/scopes.js +98 -0
  56. package/dist/segment-download.d.ts +14 -0
  57. package/dist/segment-download.js +125 -0
  58. package/dist/segment-store.d.ts +81 -0
  59. package/dist/segment-store.js +63 -0
  60. package/dist/signed-url.d.ts +139 -0
  61. package/dist/signed-url.js +143 -0
  62. package/dist/sigv4.d.ts +97 -0
  63. package/dist/sigv4.js +161 -0
  64. package/dist/sqlite-blob-store.d.ts +22 -0
  65. package/dist/sqlite-blob-store.js +86 -0
  66. package/dist/sqlite-dialect.d.ts +101 -0
  67. package/dist/sqlite-dialect.js +250 -0
  68. package/dist/sqlite-image.d.ts +27 -0
  69. package/dist/sqlite-image.js +83 -0
  70. package/dist/sqlite-lease-store.d.ts +21 -0
  71. package/dist/sqlite-lease-store.js +94 -0
  72. package/dist/sqlite-segment-store.d.ts +25 -0
  73. package/dist/sqlite-segment-store.js +115 -0
  74. package/dist/sqlite-storage.d.ts +47 -0
  75. package/dist/sqlite-storage.js +477 -0
  76. package/dist/storage.d.ts +252 -0
  77. package/dist/storage.js +1 -0
  78. package/dist/validate.d.ts +83 -0
  79. package/dist/validate.js +44 -0
  80. package/package.json +17 -325
  81. package/src/admin.ts +312 -0
  82. package/src/blob-handlers.ts +340 -0
  83. package/src/blob-store.ts +0 -0
  84. package/src/content-encoding.ts +94 -0
  85. package/src/context.ts +176 -0
  86. package/src/crdt-merger.ts +33 -0
  87. package/src/d1-storage.ts +0 -0
  88. package/src/errors.ts +241 -19
  89. package/src/events-ring.ts +116 -0
  90. package/src/events.ts +291 -0
  91. package/src/frame-bytes.ts +94 -0
  92. package/src/handler.ts +678 -0
  93. package/src/index.ts +44 -27
  94. package/src/lease-store.ts +0 -0
  95. package/src/pg-executor-pglite.ts +63 -0
  96. package/src/pg-executor.ts +87 -0
  97. package/src/postgres-fanout.ts +143 -0
  98. package/src/postgres-storage.ts +1182 -0
  99. package/src/prune.ts +63 -289
  100. package/src/pull.ts +505 -1805
  101. package/src/push.ts +623 -1006
  102. package/src/realtime.ts +1082 -0
  103. package/src/relational-rows.ts +633 -0
  104. package/src/s3-blob-store.ts +616 -0
  105. package/src/s3-segment-store.ts +519 -0
  106. package/src/schema.ts +242 -281
  107. package/src/scopes.ts +127 -0
  108. package/src/segment-download.ts +169 -0
  109. package/src/segment-store.ts +154 -0
  110. package/src/signed-url.ts +308 -0
  111. package/src/sigv4.ts +267 -0
  112. package/src/sqlite-blob-store.ts +125 -0
  113. package/src/sqlite-dialect.ts +337 -0
  114. package/src/sqlite-image.ts +124 -0
  115. package/src/sqlite-lease-store.ts +144 -0
  116. package/src/sqlite-segment-store.ts +204 -0
  117. package/src/sqlite-storage.ts +792 -0
  118. package/src/storage.ts +316 -0
  119. package/src/validate.ts +123 -0
  120. package/dist/auth-leases.d.ts +0 -85
  121. package/dist/auth-leases.d.ts.map +0 -1
  122. package/dist/auth-leases.js +0 -368
  123. package/dist/auth-leases.js.map +0 -1
  124. package/dist/better-sqlite3.d.ts +0 -22
  125. package/dist/better-sqlite3.d.ts.map +0 -1
  126. package/dist/better-sqlite3.js +0 -16
  127. package/dist/better-sqlite3.js.map +0 -1
  128. package/dist/blobs/access.d.ts +0 -35
  129. package/dist/blobs/access.d.ts.map +0 -1
  130. package/dist/blobs/access.js +0 -149
  131. package/dist/blobs/access.js.map +0 -1
  132. package/dist/blobs/adapters/database.d.ts +0 -100
  133. package/dist/blobs/adapters/database.d.ts.map +0 -1
  134. package/dist/blobs/adapters/database.js +0 -247
  135. package/dist/blobs/adapters/database.js.map +0 -1
  136. package/dist/blobs/index.d.ts +0 -9
  137. package/dist/blobs/index.d.ts.map +0 -1
  138. package/dist/blobs/index.js +0 -9
  139. package/dist/blobs/index.js.map +0 -1
  140. package/dist/blobs/manager.d.ts +0 -220
  141. package/dist/blobs/manager.d.ts.map +0 -1
  142. package/dist/blobs/manager.js +0 -625
  143. package/dist/blobs/manager.js.map +0 -1
  144. package/dist/blobs/migrate.d.ts +0 -27
  145. package/dist/blobs/migrate.d.ts.map +0 -1
  146. package/dist/blobs/migrate.js +0 -127
  147. package/dist/blobs/migrate.js.map +0 -1
  148. package/dist/blobs/types.d.ts +0 -58
  149. package/dist/blobs/types.d.ts.map +0 -1
  150. package/dist/blobs/types.js +0 -5
  151. package/dist/blobs/types.js.map +0 -1
  152. package/dist/bun-sqlite.d.ts +0 -19
  153. package/dist/bun-sqlite.d.ts.map +0 -1
  154. package/dist/bun-sqlite.js +0 -19
  155. package/dist/bun-sqlite.js.map +0 -1
  156. package/dist/clients.d.ts +0 -15
  157. package/dist/clients.d.ts.map +0 -1
  158. package/dist/clients.js +0 -7
  159. package/dist/clients.js.map +0 -1
  160. package/dist/cloudflare/durable-object.d.ts +0 -93
  161. package/dist/cloudflare/durable-object.d.ts.map +0 -1
  162. package/dist/cloudflare/durable-object.js +0 -210
  163. package/dist/cloudflare/durable-object.js.map +0 -1
  164. package/dist/cloudflare/index.d.ts +0 -22
  165. package/dist/cloudflare/index.d.ts.map +0 -1
  166. package/dist/cloudflare/index.js +0 -22
  167. package/dist/cloudflare/index.js.map +0 -1
  168. package/dist/cloudflare/r2.d.ts +0 -180
  169. package/dist/cloudflare/r2.d.ts.map +0 -1
  170. package/dist/cloudflare/r2.js +0 -258
  171. package/dist/cloudflare/r2.js.map +0 -1
  172. package/dist/cloudflare/scope-cache.d.ts +0 -54
  173. package/dist/cloudflare/scope-cache.d.ts.map +0 -1
  174. package/dist/cloudflare/scope-cache.js +0 -223
  175. package/dist/cloudflare/scope-cache.js.map +0 -1
  176. package/dist/cloudflare/sentry.d.ts +0 -47
  177. package/dist/cloudflare/sentry.d.ts.map +0 -1
  178. package/dist/cloudflare/sentry.js +0 -163
  179. package/dist/cloudflare/sentry.js.map +0 -1
  180. package/dist/cloudflare/worker.d.ts +0 -46
  181. package/dist/cloudflare/worker.d.ts.map +0 -1
  182. package/dist/cloudflare/worker.js +0 -63
  183. package/dist/cloudflare/worker.js.map +0 -1
  184. package/dist/commit-integrity.d.ts +0 -38
  185. package/dist/commit-integrity.d.ts.map +0 -1
  186. package/dist/commit-integrity.js +0 -260
  187. package/dist/commit-integrity.js.map +0 -1
  188. package/dist/compaction.d.ts +0 -27
  189. package/dist/compaction.d.ts.map +0 -1
  190. package/dist/compaction.js +0 -49
  191. package/dist/compaction.js.map +0 -1
  192. package/dist/crdt-yjs/index.d.ts +0 -99
  193. package/dist/crdt-yjs/index.d.ts.map +0 -1
  194. package/dist/crdt-yjs/index.js +0 -629
  195. package/dist/crdt-yjs/index.js.map +0 -1
  196. package/dist/d1.d.ts +0 -13
  197. package/dist/d1.d.ts.map +0 -1
  198. package/dist/d1.js +0 -14
  199. package/dist/d1.js.map +0 -1
  200. package/dist/dialect/base.d.ts +0 -93
  201. package/dist/dialect/base.d.ts.map +0 -1
  202. package/dist/dialect/base.js +0 -181
  203. package/dist/dialect/base.js.map +0 -1
  204. package/dist/dialect/helpers.d.ts +0 -16
  205. package/dist/dialect/helpers.d.ts.map +0 -1
  206. package/dist/dialect/helpers.js +0 -83
  207. package/dist/dialect/helpers.js.map +0 -1
  208. package/dist/dialect/index.d.ts +0 -7
  209. package/dist/dialect/index.d.ts.map +0 -1
  210. package/dist/dialect/index.js +0 -7
  211. package/dist/dialect/index.js.map +0 -1
  212. package/dist/dialect/types.d.ts +0 -187
  213. package/dist/dialect/types.d.ts.map +0 -1
  214. package/dist/dialect/types.js +0 -8
  215. package/dist/dialect/types.js.map +0 -1
  216. package/dist/encrypted-crdt.d.ts +0 -67
  217. package/dist/encrypted-crdt.d.ts.map +0 -1
  218. package/dist/encrypted-crdt.js +0 -426
  219. package/dist/encrypted-crdt.js.map +0 -1
  220. package/dist/errors.d.ts.map +0 -1
  221. package/dist/errors.js.map +0 -1
  222. package/dist/filesystem/index.d.ts +0 -44
  223. package/dist/filesystem/index.d.ts.map +0 -1
  224. package/dist/filesystem/index.js +0 -164
  225. package/dist/filesystem/index.js.map +0 -1
  226. package/dist/handlers/collection.d.ts +0 -20
  227. package/dist/handlers/collection.d.ts.map +0 -1
  228. package/dist/handlers/collection.js +0 -42
  229. package/dist/handlers/collection.js.map +0 -1
  230. package/dist/handlers/create-handler.d.ts +0 -156
  231. package/dist/handlers/create-handler.d.ts.map +0 -1
  232. package/dist/handlers/create-handler.js +0 -626
  233. package/dist/handlers/create-handler.js.map +0 -1
  234. package/dist/handlers/index.d.ts +0 -4
  235. package/dist/handlers/index.d.ts.map +0 -1
  236. package/dist/handlers/index.js +0 -4
  237. package/dist/handlers/index.js.map +0 -1
  238. package/dist/handlers/types.d.ts +0 -295
  239. package/dist/handlers/types.d.ts.map +0 -1
  240. package/dist/handlers/types.js +0 -2
  241. package/dist/handlers/types.js.map +0 -1
  242. package/dist/helpers/conflict.d.ts +0 -52
  243. package/dist/helpers/conflict.d.ts.map +0 -1
  244. package/dist/helpers/conflict.js +0 -49
  245. package/dist/helpers/conflict.js.map +0 -1
  246. package/dist/helpers/emitted-change.d.ts +0 -56
  247. package/dist/helpers/emitted-change.d.ts.map +0 -1
  248. package/dist/helpers/emitted-change.js +0 -46
  249. package/dist/helpers/emitted-change.js.map +0 -1
  250. package/dist/helpers/index.d.ts +0 -12
  251. package/dist/helpers/index.d.ts.map +0 -1
  252. package/dist/helpers/index.js +0 -12
  253. package/dist/helpers/index.js.map +0 -1
  254. package/dist/helpers/paginate.d.ts +0 -49
  255. package/dist/helpers/paginate.d.ts.map +0 -1
  256. package/dist/helpers/paginate.js +0 -54
  257. package/dist/helpers/paginate.js.map +0 -1
  258. package/dist/helpers/scope-authorization.d.ts +0 -7
  259. package/dist/helpers/scope-authorization.d.ts.map +0 -1
  260. package/dist/helpers/scope-authorization.js +0 -19
  261. package/dist/helpers/scope-authorization.js.map +0 -1
  262. package/dist/helpers/scope-commit-index.d.ts +0 -12
  263. package/dist/helpers/scope-commit-index.d.ts.map +0 -1
  264. package/dist/helpers/scope-commit-index.js +0 -35
  265. package/dist/helpers/scope-commit-index.js.map +0 -1
  266. package/dist/helpers/scope-strings.d.ts +0 -74
  267. package/dist/helpers/scope-strings.d.ts.map +0 -1
  268. package/dist/helpers/scope-strings.js +0 -82
  269. package/dist/helpers/scope-strings.js.map +0 -1
  270. package/dist/hono/api-key-auth.d.ts +0 -49
  271. package/dist/hono/api-key-auth.d.ts.map +0 -1
  272. package/dist/hono/api-key-auth.js +0 -108
  273. package/dist/hono/api-key-auth.js.map +0 -1
  274. package/dist/hono/audit-redaction.d.ts +0 -20
  275. package/dist/hono/audit-redaction.d.ts.map +0 -1
  276. package/dist/hono/audit-redaction.js +0 -85
  277. package/dist/hono/audit-redaction.js.map +0 -1
  278. package/dist/hono/blobs.d.ts +0 -75
  279. package/dist/hono/blobs.d.ts.map +0 -1
  280. package/dist/hono/blobs.js +0 -586
  281. package/dist/hono/blobs.js.map +0 -1
  282. package/dist/hono/console/gateway.d.ts +0 -42
  283. package/dist/hono/console/gateway.d.ts.map +0 -1
  284. package/dist/hono/console/gateway.js +0 -2158
  285. package/dist/hono/console/gateway.js.map +0 -1
  286. package/dist/hono/console/live-auth.d.ts +0 -7
  287. package/dist/hono/console/live-auth.d.ts.map +0 -1
  288. package/dist/hono/console/live-auth.js +0 -39
  289. package/dist/hono/console/live-auth.js.map +0 -1
  290. package/dist/hono/console/route-descriptor.d.ts +0 -6
  291. package/dist/hono/console/route-descriptor.d.ts.map +0 -1
  292. package/dist/hono/console/route-descriptor.js +0 -16
  293. package/dist/hono/console/route-descriptor.js.map +0 -1
  294. package/dist/hono/console/routes/api-keys.d.ts +0 -8
  295. package/dist/hono/console/routes/api-keys.d.ts.map +0 -1
  296. package/dist/hono/console/routes/api-keys.js +0 -575
  297. package/dist/hono/console/routes/api-keys.js.map +0 -1
  298. package/dist/hono/console/routes/clients.d.ts +0 -8
  299. package/dist/hono/console/routes/clients.d.ts.map +0 -1
  300. package/dist/hono/console/routes/clients.js +0 -322
  301. package/dist/hono/console/routes/clients.js.map +0 -1
  302. package/dist/hono/console/routes/commits.d.ts +0 -8
  303. package/dist/hono/console/routes/commits.d.ts.map +0 -1
  304. package/dist/hono/console/routes/commits.js +0 -882
  305. package/dist/hono/console/routes/commits.js.map +0 -1
  306. package/dist/hono/console/routes/context.d.ts +0 -188
  307. package/dist/hono/console/routes/context.d.ts.map +0 -1
  308. package/dist/hono/console/routes/context.js +0 -633
  309. package/dist/hono/console/routes/context.js.map +0 -1
  310. package/dist/hono/console/routes/events.d.ts +0 -8
  311. package/dist/hono/console/routes/events.d.ts.map +0 -1
  312. package/dist/hono/console/routes/events.js +0 -501
  313. package/dist/hono/console/routes/events.js.map +0 -1
  314. package/dist/hono/console/routes/maintenance.d.ts +0 -8
  315. package/dist/hono/console/routes/maintenance.d.ts.map +0 -1
  316. package/dist/hono/console/routes/maintenance.js +0 -341
  317. package/dist/hono/console/routes/maintenance.js.map +0 -1
  318. package/dist/hono/console/routes/shared.d.ts +0 -174
  319. package/dist/hono/console/routes/shared.d.ts.map +0 -1
  320. package/dist/hono/console/routes/shared.js +0 -583
  321. package/dist/hono/console/routes/shared.js.map +0 -1
  322. package/dist/hono/console/routes/stats.d.ts +0 -8
  323. package/dist/hono/console/routes/stats.d.ts.map +0 -1
  324. package/dist/hono/console/routes/stats.js +0 -295
  325. package/dist/hono/console/routes/stats.js.map +0 -1
  326. package/dist/hono/console/routes/storage.d.ts +0 -8
  327. package/dist/hono/console/routes/storage.d.ts.map +0 -1
  328. package/dist/hono/console/routes/storage.js +0 -121
  329. package/dist/hono/console/routes/storage.js.map +0 -1
  330. package/dist/hono/console/routes.d.ts +0 -36
  331. package/dist/hono/console/routes.d.ts.map +0 -1
  332. package/dist/hono/console/routes.js +0 -112
  333. package/dist/hono/console/routes.js.map +0 -1
  334. package/dist/hono/console/schema-errors.d.ts +0 -2
  335. package/dist/hono/console/schema-errors.d.ts.map +0 -1
  336. package/dist/hono/console/schema-errors.js +0 -17
  337. package/dist/hono/console/schema-errors.js.map +0 -1
  338. package/dist/hono/console/schemas.d.ts +0 -1515
  339. package/dist/hono/console/schemas.d.ts.map +0 -1
  340. package/dist/hono/console/schemas.js +0 -661
  341. package/dist/hono/console/schemas.js.map +0 -1
  342. package/dist/hono/console/types.d.ts +0 -213
  343. package/dist/hono/console/types.d.ts.map +0 -1
  344. package/dist/hono/console/types.js +0 -2
  345. package/dist/hono/console/types.js.map +0 -1
  346. package/dist/hono/console/ui.d.ts +0 -38
  347. package/dist/hono/console/ui.d.ts.map +0 -1
  348. package/dist/hono/console/ui.js +0 -43
  349. package/dist/hono/console/ui.js.map +0 -1
  350. package/dist/hono/create-server.d.ts +0 -71
  351. package/dist/hono/create-server.d.ts.map +0 -1
  352. package/dist/hono/create-server.js +0 -121
  353. package/dist/hono/create-server.js.map +0 -1
  354. package/dist/hono/errors.d.ts +0 -13
  355. package/dist/hono/errors.d.ts.map +0 -1
  356. package/dist/hono/errors.js +0 -21
  357. package/dist/hono/errors.js.map +0 -1
  358. package/dist/hono/index.d.ts +0 -20
  359. package/dist/hono/index.d.ts.map +0 -1
  360. package/dist/hono/index.js +0 -31
  361. package/dist/hono/index.js.map +0 -1
  362. package/dist/hono/openapi.d.ts +0 -72
  363. package/dist/hono/openapi.d.ts.map +0 -1
  364. package/dist/hono/openapi.js +0 -99
  365. package/dist/hono/openapi.js.map +0 -1
  366. package/dist/hono/proxy/connection-manager.d.ts +0 -78
  367. package/dist/hono/proxy/connection-manager.d.ts.map +0 -1
  368. package/dist/hono/proxy/connection-manager.js +0 -251
  369. package/dist/hono/proxy/connection-manager.js.map +0 -1
  370. package/dist/hono/proxy/index.d.ts +0 -8
  371. package/dist/hono/proxy/index.d.ts.map +0 -1
  372. package/dist/hono/proxy/index.js +0 -8
  373. package/dist/hono/proxy/index.js.map +0 -1
  374. package/dist/hono/proxy/routes.d.ts +0 -86
  375. package/dist/hono/proxy/routes.d.ts.map +0 -1
  376. package/dist/hono/proxy/routes.js +0 -183
  377. package/dist/hono/proxy/routes.js.map +0 -1
  378. package/dist/hono/rate-limit.d.ts +0 -101
  379. package/dist/hono/rate-limit.d.ts.map +0 -1
  380. package/dist/hono/rate-limit.js +0 -184
  381. package/dist/hono/rate-limit.js.map +0 -1
  382. package/dist/hono/realtime-sync-packs.d.ts +0 -43
  383. package/dist/hono/realtime-sync-packs.d.ts.map +0 -1
  384. package/dist/hono/realtime-sync-packs.js +0 -219
  385. package/dist/hono/realtime-sync-packs.js.map +0 -1
  386. package/dist/hono/routes/audit.d.ts +0 -12
  387. package/dist/hono/routes/audit.d.ts.map +0 -1
  388. package/dist/hono/routes/audit.js +0 -385
  389. package/dist/hono/routes/audit.js.map +0 -1
  390. package/dist/hono/routes/auth-leases.d.ts +0 -8
  391. package/dist/hono/routes/auth-leases.d.ts.map +0 -1
  392. package/dist/hono/routes/auth-leases.js +0 -88
  393. package/dist/hono/routes/auth-leases.js.map +0 -1
  394. package/dist/hono/routes/combined.d.ts +0 -8
  395. package/dist/hono/routes/combined.d.ts.map +0 -1
  396. package/dist/hono/routes/combined.js +0 -392
  397. package/dist/hono/routes/combined.js.map +0 -1
  398. package/dist/hono/routes/context.d.ts +0 -314
  399. package/dist/hono/routes/context.d.ts.map +0 -1
  400. package/dist/hono/routes/context.js +0 -1147
  401. package/dist/hono/routes/context.js.map +0 -1
  402. package/dist/hono/routes/health.d.ts +0 -8
  403. package/dist/hono/routes/health.d.ts.map +0 -1
  404. package/dist/hono/routes/health.js +0 -16
  405. package/dist/hono/routes/health.js.map +0 -1
  406. package/dist/hono/routes/realtime.d.ts +0 -9
  407. package/dist/hono/routes/realtime.d.ts.map +0 -1
  408. package/dist/hono/routes/realtime.js +0 -626
  409. package/dist/hono/routes/realtime.js.map +0 -1
  410. package/dist/hono/routes/shared.d.ts +0 -689
  411. package/dist/hono/routes/shared.d.ts.map +0 -1
  412. package/dist/hono/routes/shared.js +0 -972
  413. package/dist/hono/routes/shared.js.map +0 -1
  414. package/dist/hono/routes/snapshots.d.ts +0 -10
  415. package/dist/hono/routes/snapshots.d.ts.map +0 -1
  416. package/dist/hono/routes/snapshots.js +0 -281
  417. package/dist/hono/routes/snapshots.js.map +0 -1
  418. package/dist/hono/routes.d.ts +0 -19
  419. package/dist/hono/routes.d.ts.map +0 -1
  420. package/dist/hono/routes.js +0 -37
  421. package/dist/hono/routes.js.map +0 -1
  422. package/dist/hono/validation.d.ts +0 -5
  423. package/dist/hono/validation.d.ts.map +0 -1
  424. package/dist/hono/validation.js +0 -47
  425. package/dist/hono/validation.js.map +0 -1
  426. package/dist/hono/websocket-origin.d.ts +0 -9
  427. package/dist/hono/websocket-origin.d.ts.map +0 -1
  428. package/dist/hono/websocket-origin.js +0 -96
  429. package/dist/hono/websocket-origin.js.map +0 -1
  430. package/dist/hono/ws.d.ts +0 -341
  431. package/dist/hono/ws.d.ts.map +0 -1
  432. package/dist/hono/ws.js +0 -711
  433. package/dist/hono/ws.js.map +0 -1
  434. package/dist/index.d.ts.map +0 -1
  435. package/dist/index.js.map +0 -1
  436. package/dist/libsql.d.ts +0 -29
  437. package/dist/libsql.d.ts.map +0 -1
  438. package/dist/libsql.js +0 -25
  439. package/dist/libsql.js.map +0 -1
  440. package/dist/migrate.d.ts +0 -14
  441. package/dist/migrate.d.ts.map +0 -1
  442. package/dist/migrate.js +0 -13
  443. package/dist/migrate.js.map +0 -1
  444. package/dist/neon.d.ts +0 -21
  445. package/dist/neon.d.ts.map +0 -1
  446. package/dist/neon.js +0 -22
  447. package/dist/neon.js.map +0 -1
  448. package/dist/notify.d.ts +0 -77
  449. package/dist/notify.d.ts.map +0 -1
  450. package/dist/notify.js +0 -233
  451. package/dist/notify.js.map +0 -1
  452. package/dist/pglite.d.ts +0 -37
  453. package/dist/pglite.d.ts.map +0 -1
  454. package/dist/pglite.js +0 -37
  455. package/dist/pglite.js.map +0 -1
  456. package/dist/plugins/index.d.ts +0 -2
  457. package/dist/plugins/index.d.ts.map +0 -1
  458. package/dist/plugins/index.js +0 -2
  459. package/dist/plugins/index.js.map +0 -1
  460. package/dist/plugins/types.d.ts +0 -73
  461. package/dist/plugins/types.d.ts.map +0 -1
  462. package/dist/plugins/types.js +0 -30
  463. package/dist/plugins/types.js.map +0 -1
  464. package/dist/postgres/index.d.ts +0 -68
  465. package/dist/postgres/index.d.ts.map +0 -1
  466. package/dist/postgres/index.js +0 -924
  467. package/dist/postgres/index.js.map +0 -1
  468. package/dist/proxy/collection.d.ts +0 -7
  469. package/dist/proxy/collection.d.ts.map +0 -1
  470. package/dist/proxy/collection.js +0 -6
  471. package/dist/proxy/collection.js.map +0 -1
  472. package/dist/proxy/handler.d.ts +0 -42
  473. package/dist/proxy/handler.d.ts.map +0 -1
  474. package/dist/proxy/handler.js +0 -102
  475. package/dist/proxy/handler.js.map +0 -1
  476. package/dist/proxy/index.d.ts +0 -9
  477. package/dist/proxy/index.d.ts.map +0 -1
  478. package/dist/proxy/index.js +0 -14
  479. package/dist/proxy/index.js.map +0 -1
  480. package/dist/proxy/mutation-detector.d.ts +0 -35
  481. package/dist/proxy/mutation-detector.d.ts.map +0 -1
  482. package/dist/proxy/mutation-detector.js +0 -246
  483. package/dist/proxy/mutation-detector.js.map +0 -1
  484. package/dist/proxy/oplog.d.ts +0 -30
  485. package/dist/proxy/oplog.d.ts.map +0 -1
  486. package/dist/proxy/oplog.js +0 -137
  487. package/dist/proxy/oplog.js.map +0 -1
  488. package/dist/proxy/types.d.ts +0 -44
  489. package/dist/proxy/types.d.ts.map +0 -1
  490. package/dist/proxy/types.js +0 -7
  491. package/dist/proxy/types.js.map +0 -1
  492. package/dist/prune.d.ts.map +0 -1
  493. package/dist/prune.js.map +0 -1
  494. package/dist/pull.d.ts.map +0 -1
  495. package/dist/pull.js.map +0 -1
  496. package/dist/push.d.ts.map +0 -1
  497. package/dist/push.js.map +0 -1
  498. package/dist/realtime/in-memory.d.ts +0 -13
  499. package/dist/realtime/in-memory.d.ts.map +0 -1
  500. package/dist/realtime/in-memory.js +0 -28
  501. package/dist/realtime/in-memory.js.map +0 -1
  502. package/dist/realtime/index.d.ts +0 -4
  503. package/dist/realtime/index.d.ts.map +0 -1
  504. package/dist/realtime/index.js +0 -3
  505. package/dist/realtime/index.js.map +0 -1
  506. package/dist/realtime/types.d.ts +0 -62
  507. package/dist/realtime/types.d.ts.map +0 -1
  508. package/dist/realtime/types.js +0 -19
  509. package/dist/realtime/types.js.map +0 -1
  510. package/dist/relay/client-role/forward-engine.d.ts +0 -63
  511. package/dist/relay/client-role/forward-engine.d.ts.map +0 -1
  512. package/dist/relay/client-role/forward-engine.js +0 -267
  513. package/dist/relay/client-role/forward-engine.js.map +0 -1
  514. package/dist/relay/client-role/index.d.ts +0 -9
  515. package/dist/relay/client-role/index.d.ts.map +0 -1
  516. package/dist/relay/client-role/index.js +0 -9
  517. package/dist/relay/client-role/index.js.map +0 -1
  518. package/dist/relay/client-role/pull-engine.d.ts +0 -72
  519. package/dist/relay/client-role/pull-engine.d.ts.map +0 -1
  520. package/dist/relay/client-role/pull-engine.js +0 -249
  521. package/dist/relay/client-role/pull-engine.js.map +0 -1
  522. package/dist/relay/client-role/sequence-mapper.d.ts +0 -65
  523. package/dist/relay/client-role/sequence-mapper.d.ts.map +0 -1
  524. package/dist/relay/client-role/sequence-mapper.js +0 -161
  525. package/dist/relay/client-role/sequence-mapper.js.map +0 -1
  526. package/dist/relay/evaluation/relay-paths.d.ts +0 -41
  527. package/dist/relay/evaluation/relay-paths.d.ts.map +0 -1
  528. package/dist/relay/evaluation/relay-paths.js +0 -504
  529. package/dist/relay/evaluation/relay-paths.js.map +0 -1
  530. package/dist/relay/evaluation/rust-boundary.d.ts +0 -47
  531. package/dist/relay/evaluation/rust-boundary.d.ts.map +0 -1
  532. package/dist/relay/evaluation/rust-boundary.js +0 -220
  533. package/dist/relay/evaluation/rust-boundary.js.map +0 -1
  534. package/dist/relay/index.d.ts +0 -37
  535. package/dist/relay/index.d.ts.map +0 -1
  536. package/dist/relay/index.js +0 -44
  537. package/dist/relay/index.js.map +0 -1
  538. package/dist/relay/migrate.d.ts +0 -18
  539. package/dist/relay/migrate.d.ts.map +0 -1
  540. package/dist/relay/migrate.js +0 -99
  541. package/dist/relay/migrate.js.map +0 -1
  542. package/dist/relay/mode-manager.d.ts +0 -60
  543. package/dist/relay/mode-manager.d.ts.map +0 -1
  544. package/dist/relay/mode-manager.js +0 -114
  545. package/dist/relay/mode-manager.js.map +0 -1
  546. package/dist/relay/realtime.d.ts +0 -90
  547. package/dist/relay/realtime.d.ts.map +0 -1
  548. package/dist/relay/realtime.js +0 -147
  549. package/dist/relay/realtime.js.map +0 -1
  550. package/dist/relay/relay.d.ts +0 -190
  551. package/dist/relay/relay.d.ts.map +0 -1
  552. package/dist/relay/relay.js +0 -320
  553. package/dist/relay/relay.js.map +0 -1
  554. package/dist/relay/schema.d.ts +0 -158
  555. package/dist/relay/schema.d.ts.map +0 -1
  556. package/dist/relay/schema.js +0 -7
  557. package/dist/relay/schema.js.map +0 -1
  558. package/dist/relay/server-role/index.d.ts +0 -56
  559. package/dist/relay/server-role/index.d.ts.map +0 -1
  560. package/dist/relay/server-role/index.js +0 -193
  561. package/dist/relay/server-role/index.js.map +0 -1
  562. package/dist/relay/server-role/pull.d.ts +0 -27
  563. package/dist/relay/server-role/pull.d.ts.map +0 -1
  564. package/dist/relay/server-role/pull.js +0 -24
  565. package/dist/relay/server-role/pull.js.map +0 -1
  566. package/dist/relay/server-role/push.d.ts +0 -29
  567. package/dist/relay/server-role/push.d.ts.map +0 -1
  568. package/dist/relay/server-role/push.js +0 -122
  569. package/dist/relay/server-role/push.js.map +0 -1
  570. package/dist/s3/index.d.ts +0 -83
  571. package/dist/s3/index.d.ts.map +0 -1
  572. package/dist/s3/index.js +0 -223
  573. package/dist/s3/index.js.map +0 -1
  574. package/dist/schema.d.ts.map +0 -1
  575. package/dist/schema.js.map +0 -1
  576. package/dist/service-worker/index.d.ts +0 -114
  577. package/dist/service-worker/index.d.ts.map +0 -1
  578. package/dist/service-worker/index.js +0 -487
  579. package/dist/service-worker/index.js.map +0 -1
  580. package/dist/snapshot-artifacts/sqlite-bun.d.ts +0 -15
  581. package/dist/snapshot-artifacts/sqlite-bun.d.ts.map +0 -1
  582. package/dist/snapshot-artifacts/sqlite-bun.js +0 -128
  583. package/dist/snapshot-artifacts/sqlite-bun.js.map +0 -1
  584. package/dist/snapshot-artifacts.d.ts +0 -206
  585. package/dist/snapshot-artifacts.d.ts.map +0 -1
  586. package/dist/snapshot-artifacts.js +0 -545
  587. package/dist/snapshot-artifacts.js.map +0 -1
  588. package/dist/snapshot-chunks/db-metadata.d.ts +0 -56
  589. package/dist/snapshot-chunks/db-metadata.d.ts.map +0 -1
  590. package/dist/snapshot-chunks/db-metadata.js +0 -360
  591. package/dist/snapshot-chunks/db-metadata.js.map +0 -1
  592. package/dist/snapshot-chunks/index.d.ts +0 -8
  593. package/dist/snapshot-chunks/index.d.ts.map +0 -1
  594. package/dist/snapshot-chunks/index.js +0 -8
  595. package/dist/snapshot-chunks/index.js.map +0 -1
  596. package/dist/snapshot-chunks/types.d.ts +0 -80
  597. package/dist/snapshot-chunks/types.d.ts.map +0 -1
  598. package/dist/snapshot-chunks/types.js +0 -8
  599. package/dist/snapshot-chunks/types.js.map +0 -1
  600. package/dist/snapshot-chunks.d.ts +0 -90
  601. package/dist/snapshot-chunks.d.ts.map +0 -1
  602. package/dist/snapshot-chunks.js +0 -304
  603. package/dist/snapshot-chunks.js.map +0 -1
  604. package/dist/sqlite/index.d.ts +0 -53
  605. package/dist/sqlite/index.d.ts.map +0 -1
  606. package/dist/sqlite/index.js +0 -795
  607. package/dist/sqlite/index.js.map +0 -1
  608. package/dist/sqlite3.d.ts +0 -22
  609. package/dist/sqlite3.d.ts.map +0 -1
  610. package/dist/sqlite3.js +0 -99
  611. package/dist/sqlite3.js.map +0 -1
  612. package/dist/stats.d.ts +0 -28
  613. package/dist/stats.d.ts.map +0 -1
  614. package/dist/stats.js +0 -93
  615. package/dist/stats.js.map +0 -1
  616. package/dist/subscriptions/cache.d.ts +0 -58
  617. package/dist/subscriptions/cache.d.ts.map +0 -1
  618. package/dist/subscriptions/cache.js +0 -250
  619. package/dist/subscriptions/cache.js.map +0 -1
  620. package/dist/subscriptions/index.d.ts +0 -3
  621. package/dist/subscriptions/index.d.ts.map +0 -1
  622. package/dist/subscriptions/index.js +0 -3
  623. package/dist/subscriptions/index.js.map +0 -1
  624. package/dist/subscriptions/resolve.d.ts +0 -40
  625. package/dist/subscriptions/resolve.d.ts.map +0 -1
  626. package/dist/subscriptions/resolve.js +0 -275
  627. package/dist/subscriptions/resolve.js.map +0 -1
  628. package/dist/sync.d.ts +0 -24
  629. package/dist/sync.d.ts.map +0 -1
  630. package/dist/sync.js +0 -27
  631. package/dist/sync.js.map +0 -1
  632. package/src/auth-leases.ts +0 -649
  633. package/src/better-sqlite3.ts +0 -35
  634. package/src/blobs/access.ts +0 -244
  635. package/src/blobs/adapters/database.ts +0 -397
  636. package/src/blobs/index.ts +0 -9
  637. package/src/blobs/manager.ts +0 -901
  638. package/src/blobs/migrate.ts +0 -158
  639. package/src/blobs/types.ts +0 -74
  640. package/src/bun-sqlite-ambient.d.ts +0 -19
  641. package/src/bun-sqlite.ts +0 -27
  642. package/src/clients.ts +0 -22
  643. package/src/cloudflare/durable-object.ts +0 -289
  644. package/src/cloudflare/index.ts +0 -22
  645. package/src/cloudflare/r2.ts +0 -526
  646. package/src/cloudflare/scope-cache.ts +0 -341
  647. package/src/cloudflare/sentry.ts +0 -230
  648. package/src/cloudflare/worker.ts +0 -77
  649. package/src/commit-integrity.ts +0 -371
  650. package/src/compaction.ts +0 -77
  651. package/src/crdt-yjs/index.ts +0 -931
  652. package/src/d1.ts +0 -16
  653. package/src/dialect/base.ts +0 -360
  654. package/src/dialect/helpers.ts +0 -92
  655. package/src/dialect/index.ts +0 -7
  656. package/src/dialect/types.ts +0 -247
  657. package/src/encrypted-crdt.ts +0 -786
  658. package/src/filesystem/index.ts +0 -262
  659. package/src/handlers/collection.ts +0 -121
  660. package/src/handlers/create-handler.ts +0 -1134
  661. package/src/handlers/index.ts +0 -3
  662. package/src/handlers/types.ts +0 -403
  663. package/src/helpers/conflict.ts +0 -64
  664. package/src/helpers/emitted-change.ts +0 -69
  665. package/src/helpers/index.ts +0 -12
  666. package/src/helpers/paginate.ts +0 -82
  667. package/src/helpers/scope-authorization.ts +0 -27
  668. package/src/helpers/scope-commit-index.ts +0 -52
  669. package/src/helpers/scope-strings.ts +0 -101
  670. package/src/hono/api-key-auth.ts +0 -177
  671. package/src/hono/audit-redaction.ts +0 -135
  672. package/src/hono/blobs.ts +0 -851
  673. package/src/hono/console/gateway.ts +0 -3046
  674. package/src/hono/console/live-auth.ts +0 -46
  675. package/src/hono/console/route-descriptor.ts +0 -22
  676. package/src/hono/console/routes/api-keys.ts +0 -721
  677. package/src/hono/console/routes/clients.ts +0 -447
  678. package/src/hono/console/routes/commits.ts +0 -1137
  679. package/src/hono/console/routes/context.ts +0 -956
  680. package/src/hono/console/routes/events.ts +0 -669
  681. package/src/hono/console/routes/maintenance.ts +0 -461
  682. package/src/hono/console/routes/shared.ts +0 -816
  683. package/src/hono/console/routes/stats.ts +0 -392
  684. package/src/hono/console/routes/storage.ts +0 -159
  685. package/src/hono/console/routes.ts +0 -146
  686. package/src/hono/console/schema-errors.ts +0 -23
  687. package/src/hono/console/schemas.ts +0 -914
  688. package/src/hono/console/types.ts +0 -223
  689. package/src/hono/console/ui.ts +0 -100
  690. package/src/hono/create-server.ts +0 -230
  691. package/src/hono/errors.ts +0 -50
  692. package/src/hono/index.ts +0 -54
  693. package/src/hono/openapi.ts +0 -139
  694. package/src/hono/proxy/connection-manager.ts +0 -340
  695. package/src/hono/proxy/index.ts +0 -8
  696. package/src/hono/proxy/routes.ts +0 -272
  697. package/src/hono/rate-limit.ts +0 -319
  698. package/src/hono/realtime-sync-packs.ts +0 -354
  699. package/src/hono/routes/audit.ts +0 -499
  700. package/src/hono/routes/auth-leases.ts +0 -119
  701. package/src/hono/routes/combined.ts +0 -583
  702. package/src/hono/routes/context.ts +0 -1629
  703. package/src/hono/routes/health.ts +0 -26
  704. package/src/hono/routes/realtime.ts +0 -808
  705. package/src/hono/routes/shared.ts +0 -1626
  706. package/src/hono/routes/snapshots.ts +0 -345
  707. package/src/hono/routes.ts +0 -68
  708. package/src/hono/validation.ts +0 -81
  709. package/src/hono/websocket-origin.ts +0 -131
  710. package/src/hono/ws.ts +0 -1134
  711. package/src/libsql.ts +0 -51
  712. package/src/migrate.ts +0 -20
  713. package/src/neon.ts +0 -28
  714. package/src/notify.ts +0 -341
  715. package/src/pglite.ts +0 -68
  716. package/src/plugins/index.ts +0 -1
  717. package/src/plugins/types.ts +0 -144
  718. package/src/postgres/index.ts +0 -1291
  719. package/src/proxy/collection.ts +0 -17
  720. package/src/proxy/handler.ts +0 -159
  721. package/src/proxy/index.ts +0 -21
  722. package/src/proxy/mutation-detector.ts +0 -281
  723. package/src/proxy/oplog.ts +0 -181
  724. package/src/proxy/types.ts +0 -46
  725. package/src/realtime/in-memory.ts +0 -33
  726. package/src/realtime/index.ts +0 -7
  727. package/src/realtime/types.ts +0 -90
  728. package/src/relay/bun-types.d.ts +0 -50
  729. package/src/relay/client-role/forward-engine.ts +0 -355
  730. package/src/relay/client-role/index.ts +0 -9
  731. package/src/relay/client-role/pull-engine.ts +0 -329
  732. package/src/relay/client-role/sequence-mapper.ts +0 -201
  733. package/src/relay/evaluation/relay-paths.ts +0 -699
  734. package/src/relay/evaluation/rust-boundary.ts +0 -464
  735. package/src/relay/index.ts +0 -50
  736. package/src/relay/migrate.ts +0 -113
  737. package/src/relay/mode-manager.ts +0 -142
  738. package/src/relay/realtime.ts +0 -207
  739. package/src/relay/relay.ts +0 -431
  740. package/src/relay/schema.ts +0 -171
  741. package/src/relay/server-role/index.ts +0 -338
  742. package/src/relay/server-role/pull.ts +0 -43
  743. package/src/relay/server-role/push.ts +0 -164
  744. package/src/s3/index.ts +0 -346
  745. package/src/service-worker/index.ts +0 -773
  746. package/src/snapshot-artifacts/sqlite-bun.ts +0 -168
  747. package/src/snapshot-artifacts.ts +0 -896
  748. package/src/snapshot-chunks/db-metadata.ts +0 -537
  749. package/src/snapshot-chunks/index.ts +0 -8
  750. package/src/snapshot-chunks/types.ts +0 -105
  751. package/src/snapshot-chunks.ts +0 -453
  752. package/src/sqlite/index.ts +0 -1064
  753. package/src/sqlite3.ts +0 -137
  754. package/src/stats.ts +0 -180
  755. package/src/subscriptions/cache.ts +0 -376
  756. package/src/subscriptions/index.ts +0 -2
  757. package/src/subscriptions/resolve.ts +0 -357
  758. package/src/sync.ts +0 -111
package/src/pull.ts CHANGED
@@ -1,1865 +1,565 @@
1
+ /**
2
+ * Pull: incremental commit delivery, cursors, bootstrap segments
3
+ * (SPEC.md §4, §5).
4
+ */
1
5
  import {
2
- type BinarySnapshotColumn,
3
- type BinarySnapshotColumnType,
4
- type BinarySnapshotRowsEncoder,
5
- bytesToReadableStream,
6
- captureSyncException,
7
- concatByteChunks,
8
- countSyncMetric,
9
- createSnapshotManifest,
10
- distributionSyncMetric,
11
- encodeBinarySnapshotTable,
12
- gzipBytes,
13
- randomId,
14
- type ScopeValues,
15
- SYNC_SCOPED_SNAPSHOT_ARTIFACT_KIND_SQLITE_V1,
16
- SYNC_SNAPSHOT_CHUNK_COMPRESSION,
17
- SYNC_SNAPSHOT_CHUNK_ENCODING,
18
- type SyncBootstrapState,
19
- type SyncChange,
20
- type SyncCommit,
21
- type SyncCrdtStateVectorHint,
22
- type SyncPullRequest,
23
- type SyncPullResponse,
24
- type SyncPullSubscriptionResponse,
25
- type SyncScopedSnapshotArtifactKind,
26
- type SyncSnapshot,
27
- type SyncSnapshotArtifactCompression,
28
- type SyncSnapshotArtifactsRequest,
29
- type SyncSnapshotChunkRef,
30
- sha256Hex,
31
- snapshotScopeDigestFromCacheKey,
32
- startSyncSpan,
6
+ type CommitChange,
7
+ decodeRow,
8
+ encodeRowsSegment,
9
+ type PullHeaderFrame,
10
+ type ResponseFrame,
11
+ type ScopeMap,
12
+ type SegmentRow,
13
+ type SubscriptionFrame,
33
14
  } from '@syncular/core';
34
- import type { Kysely } from 'kysely';
35
- import {
36
- createWireSubscriptionIntegrity,
37
- SYNCULAR_COMMIT_GENESIS_ROOT,
38
- } from './commit-integrity';
39
- import type {
40
- DbExecutor,
41
- IncrementalPullRow,
42
- ServerSyncDialect,
43
- } from './dialect/types';
44
- import {
45
- getServerBootstrapOrderFor,
46
- type ServerHandlerCollection,
47
- } from './handlers/collection';
48
- import type { ServerTableHandler, SyncServerAuth } from './handlers/types';
49
- import { EXTERNAL_CLIENT_ID } from './notify';
50
- import {
51
- type SyncServerPullPlugin,
52
- sortServerPullPlugins,
53
- } from './plugins/types';
54
- import type { SyncCoreDb } from './schema';
55
- import {
56
- createScopedSnapshotArtifactScopeCacheKey,
57
- readBestScopedSnapshotArtifactRefForPageCapacity,
58
- } from './snapshot-artifacts';
59
- import {
60
- createSnapshotChunkScopeCacheKey,
61
- insertSnapshotChunk,
62
- readSnapshotChunkRefByPageKey,
63
- type SnapshotChunkRefWithContinuation,
64
- } from './snapshot-chunks';
65
- import type { SnapshotChunkStorage } from './snapshot-chunks/types';
66
- import {
67
- createMemoryScopeCache,
68
- type ScopeCacheBackend,
69
- } from './subscriptions/cache';
70
- import { resolveEffectiveScopesForSubscriptions } from './subscriptions/resolve';
71
-
72
- const defaultScopeCache = createMemoryScopeCache();
73
- const DEFAULT_MAX_BINARY_SNAPSHOT_BUNDLE_ROWS = 50_000;
74
- const DEFAULT_SNAPSHOT_CHUNK_GZIP_LEVEL = 1;
75
- const MAX_PULL_TRANSACTION_RETRIES = 2;
76
- const PULL_TRANSACTION_RETRY_DELAY_MS = 15;
77
-
78
- interface PullBootstrapTimings {
79
- snapshotQueryMs: number;
80
- binaryEncodeMs: number;
81
- chunkCacheLookupMs: number;
82
- artifactCacheLookupMs: number;
83
- chunkGzipMs: number;
84
- chunkHashMs: number;
85
- chunkPersistMs: number;
86
- }
87
-
88
- interface SnapshotChunkEncodeResult {
89
- body: Uint8Array;
90
- sha256: string;
91
- gzipMs: number;
92
- hashMs: number;
93
- }
94
-
95
- function toResponseChunkRef(ref: SyncSnapshotChunkRef): SyncSnapshotChunkRef {
96
- return {
97
- id: ref.id,
98
- byteLength: ref.byteLength,
99
- sha256: ref.sha256,
100
- encoding: ref.encoding,
101
- compression: ref.compression,
102
- };
103
- }
104
-
105
- async function createChunkedSnapshotManifest(args: {
106
- table: string;
107
- asOfCommitSeq: number;
108
- scopeKey: string;
109
- rowCursor: string | null;
110
- rowLimit: number;
111
- nextRowCursor: string | null;
112
- isFirstPage: boolean;
113
- isLastPage: boolean;
114
- chunks: readonly SyncSnapshotChunkRef[];
115
- }): Promise<SyncSnapshot['manifest']> {
116
- return createSnapshotManifest({
117
- version: 1,
118
- table: args.table,
119
- asOfCommitSeq: args.asOfCommitSeq,
120
- scopeDigest: snapshotScopeDigestFromCacheKey(args.scopeKey),
121
- rowCursor: args.rowCursor,
122
- rowLimit: args.rowLimit,
123
- nextRowCursor: args.nextRowCursor,
124
- isFirstPage: args.isFirstPage,
125
- isLastPage: args.isLastPage,
126
- chunks: args.chunks.map(toResponseChunkRef),
127
- });
128
- }
15
+ import type { SyncRequestContext } from './context';
16
+ import { clockOf, limitsOf } from './context';
17
+ import type { PullSegmentSummary } from './events';
18
+ import type { CompiledSchema, CompiledTable } from './schema';
19
+ import { scopeDigest } from './scopes';
20
+ import type { SegmentRecord } from './segment-store';
21
+ import { issueSegmentUrl } from './signed-url';
22
+ import type { SqliteImageBuilder } from './sqlite-image';
23
+ import type { StoredCommit, StoredRow } from './storage';
129
24
 
130
- interface SnapshotArtifactSelection {
131
- artifactKind: SyncScopedSnapshotArtifactKind;
132
- compression: SyncSnapshotArtifactCompression;
133
- schemaVersion: string;
134
- featureSet: readonly string[];
135
- }
136
-
137
- function normalizeFeatureSet(
138
- features: readonly string[] | undefined
139
- ): string[] {
140
- return Array.from(new Set(features ?? [])).sort();
141
- }
142
-
143
- function resolveSnapshotArtifactSelection(
144
- request: SyncSnapshotArtifactsRequest | undefined,
145
- clientSchemaVersion: number
146
- ): SnapshotArtifactSelection | null {
147
- if (!request) return null;
148
- if (
149
- !request.artifactKinds.includes(
150
- SYNC_SCOPED_SNAPSHOT_ARTIFACT_KIND_SQLITE_V1
151
- )
152
- ) {
153
- return null;
154
- }
155
- if (!request.compressions?.includes(SYNC_SNAPSHOT_CHUNK_COMPRESSION)) {
156
- return null;
25
+ /**
26
+ * Resolve the §5.3 image builder: the host-injected one if present, else the
27
+ * in-tree `buildSqliteImage` on a Bun runtime (dynamic import so `bun:sqlite`
28
+ * is never a static dep of the neutral core), else `undefined` (rows lane).
29
+ * Memoized so the dynamic import happens at most once per process.
30
+ */
31
+ let cachedDefaultBuilder: SqliteImageBuilder | null | undefined;
32
+ async function resolveImageBuilder(
33
+ ctx: SyncRequestContext,
34
+ ): Promise<SqliteImageBuilder | undefined> {
35
+ if (ctx.sqliteImageBuilder !== undefined) return ctx.sqliteImageBuilder;
36
+ if (cachedDefaultBuilder === undefined) {
37
+ const hasBun = (globalThis as { Bun?: unknown }).Bun !== undefined;
38
+ cachedDefaultBuilder = hasBun
39
+ ? (await import('./sqlite-image')).buildSqliteImage
40
+ : null;
157
41
  }
158
- return {
159
- artifactKind: SYNC_SCOPED_SNAPSHOT_ARTIFACT_KIND_SQLITE_V1,
160
- compression: SYNC_SNAPSHOT_CHUNK_COMPRESSION,
161
- schemaVersion: String(clientSchemaVersion),
162
- featureSet: normalizeFeatureSet(request.featureSet),
163
- };
42
+ return cachedDefaultBuilder ?? undefined;
164
43
  }
165
44
 
166
- function resolveSnapshotBinaryColumns<
167
- DB extends SyncCoreDb,
168
- Auth extends SyncServerAuth,
169
- >(
170
- handler: ServerTableHandler<DB, Auth>,
171
- schemaVersion: number
172
- ): readonly BinarySnapshotColumn[] | undefined {
173
- const versioned = handler.snapshotBinaryColumnsForVersion?.(schemaVersion);
174
- return versioned === undefined
175
- ? handler.snapshotBinaryColumns
176
- : (versioned ?? undefined);
45
+ /** §4.2 accept bitmask. */
46
+ export const ACCEPT_INLINE_ROWS = 1 << 0;
47
+ export const ACCEPT_EXTERNAL_ROWS = 1 << 1;
48
+ export const ACCEPT_SQLITE = 1 << 2;
49
+ export const ACCEPT_SIGNED_URLS = 1 << 3;
50
+
51
+ export interface PullLimits {
52
+ readonly limitCommits: number;
53
+ readonly limitSnapshotRows: number;
54
+ readonly maxSnapshotPages: number;
55
+ readonly accept: number;
177
56
  }
178
57
 
179
- function resolveSnapshotBinaryEncoder<
180
- DB extends SyncCoreDb,
181
- Auth extends SyncServerAuth,
182
- >(
183
- handler: ServerTableHandler<DB, Auth>,
184
- schemaVersion: number
185
- ): BinarySnapshotRowsEncoder | undefined {
186
- const versioned = handler.snapshotBinaryEncoderForVersion?.(schemaVersion);
187
- return versioned === undefined
188
- ? handler.snapshotBinaryEncoder
189
- : (versioned ?? undefined);
58
+ function clamp(value: number, min: number, max: number, dflt: number): number {
59
+ if (value === 0) return dflt;
60
+ return Math.min(max, Math.max(min, value));
190
61
  }
191
62
 
192
- function createPullBootstrapTimings(): PullBootstrapTimings {
63
+ /** §4.2 defaults and silent clamps (the v1 values). */
64
+ export function clampPullLimits(header: PullHeaderFrame): PullLimits {
193
65
  return {
194
- snapshotQueryMs: 0,
195
- binaryEncodeMs: 0,
196
- chunkCacheLookupMs: 0,
197
- artifactCacheLookupMs: 0,
198
- chunkGzipMs: 0,
199
- chunkHashMs: 0,
200
- chunkPersistMs: 0,
66
+ limitCommits: clamp(header.limitCommits, 1, 1000, 1000),
67
+ limitSnapshotRows: clamp(header.limitSnapshotRows, 1, 50000, 1000),
68
+ maxSnapshotPages: clamp(header.maxSnapshotPages, 1, 50, 4),
69
+ accept: header.accept,
201
70
  };
202
71
  }
203
72
 
204
- function resolveBinarySnapshotBundleRowLimit(args: {
205
- limitSnapshotRows: number;
206
- pagesRemaining: number;
207
- }): number {
208
- const pageSize = Math.max(1, args.limitSnapshotRows);
209
- const pagesRemaining = Math.max(1, args.pagesRemaining);
210
- const maxBundlePages = Math.max(
211
- 1,
212
- Math.ceil(DEFAULT_MAX_BINARY_SNAPSHOT_BUNDLE_ROWS / pageSize)
213
- );
214
- return pageSize * Math.min(pagesRemaining, maxBundlePages);
215
- }
216
-
217
- async function gzipByteChunks(
218
- chunks: readonly Uint8Array[],
219
- gzipLevel: number
220
- ): Promise<Uint8Array> {
221
- return gzipBytes(concatByteChunks(chunks), {
222
- level: gzipLevel,
223
- });
73
+ export interface SubscriptionPlan {
74
+ readonly frame: SubscriptionFrame;
75
+ readonly table: CompiledTable;
76
+ readonly status: 'active' | 'revoked';
77
+ readonly effective: ScopeMap;
224
78
  }
225
79
 
226
- async function encodeCompressedSnapshotChunk(
227
- chunks: readonly Uint8Array[],
228
- gzipLevel: number
229
- ): Promise<SnapshotChunkEncodeResult> {
230
- const gzipStartedAt = Date.now();
231
- const body = await gzipByteChunks(chunks, gzipLevel);
232
- const gzipMs = Math.max(0, Date.now() - gzipStartedAt);
233
- const hashStartedAt = Date.now();
234
- const sha256 = await sha256Hex(body);
235
- const hashMs = Math.max(0, Date.now() - hashStartedAt);
236
- return { body, sha256, gzipMs, hashMs };
80
+ export interface SubscriptionResult {
81
+ readonly nextCursor: number;
82
+ readonly active: boolean;
237
83
  }
238
84
 
239
- async function encodeCompressedSnapshotChunkToStream(
240
- chunks: readonly Uint8Array[],
241
- gzipLevel: number
242
- ): Promise<{
243
- stream: ReadableStream<Uint8Array>;
244
- byteLength: number;
245
- sha256: string;
246
- gzipMs: number;
247
- hashMs: number;
248
- }> {
249
- const encoded = await encodeCompressedSnapshotChunk(chunks, gzipLevel);
250
- return {
251
- stream: bytesToReadableStream(encoded.body),
252
- byteLength: encoded.body.length,
253
- sha256: encoded.sha256,
254
- gzipMs: encoded.gzipMs,
255
- hashMs: encoded.hashMs,
256
- };
257
- }
258
-
259
- interface SnapshotColumnInference {
260
- name: string;
261
- type: BinarySnapshotColumnType | null;
262
- nullable: boolean;
263
- presentCount: number;
85
+ /**
86
+ * Mutable per-section collector for the `pull.served` event. Only created
87
+ * when an events sink is configured — with `trace` undefined the pull path
88
+ * does zero extra work (`trace?.…` short-circuits argument evaluation).
89
+ */
90
+ export interface PullSectionTrace {
91
+ readonly segments: PullSegmentSummary[];
264
92
  }
265
93
 
266
- function encodeBinarySnapshotRows(
267
- table: string,
268
- rows: readonly unknown[],
269
- columns?: readonly BinarySnapshotColumn[]
270
- ): Uint8Array {
271
- const recordRows = rows.map((row) => toSnapshotRecordRow(table, row));
272
- return encodeBinarySnapshotTable({
273
- table,
274
- columns: columns ?? inferBinarySnapshotColumns(recordRows),
275
- rows: recordRows,
276
- });
94
+ interface BootstrapToken {
95
+ asOfCommitSeq: number;
96
+ tables: string[];
97
+ tableIndex: number;
98
+ rowCursor: string | null;
277
99
  }
278
100
 
279
- function toSnapshotRecordRow(
101
+ function parseBootstrapToken(
102
+ raw: string | undefined,
280
103
  table: string,
281
- row: unknown
282
- ): Record<string, unknown> {
283
- if (
284
- row == null ||
285
- typeof row !== 'object' ||
286
- Array.isArray(row) ||
287
- row instanceof Uint8Array ||
288
- row instanceof ArrayBuffer
289
- ) {
290
- throw new Error(
291
- `Cannot encode binary snapshot for table ${table}: snapshot rows must be objects`
292
- );
293
- }
294
- return row as Record<string, unknown>;
295
- }
296
-
297
- function inferBinarySnapshotColumns(
298
- rows: readonly Record<string, unknown>[]
299
- ): BinarySnapshotColumn[] {
300
- const columns: SnapshotColumnInference[] = [];
301
- const columnsByName = new Map<string, SnapshotColumnInference>();
302
-
303
- for (const row of rows) {
304
- for (const name in row) {
305
- if (!Object.hasOwn(row, name)) continue;
306
- const value = row[name];
307
- let column = columnsByName.get(name);
308
- if (!column) {
309
- column = { name, type: null, nullable: false, presentCount: 0 };
310
- columnsByName.set(name, column);
311
- columns.push(column);
312
- }
313
- column.presentCount += 1;
314
- if (value == null) {
315
- column.nullable = true;
316
- continue;
317
- }
318
- column.type = mergeBinarySnapshotColumnTypes(
319
- column.type,
320
- inferBinarySnapshotColumnType(value)
321
- );
104
+ ): BootstrapToken | undefined {
105
+ if (raw === undefined) return undefined;
106
+ try {
107
+ const parsed = JSON.parse(raw) as Partial<BootstrapToken>;
108
+ if (
109
+ typeof parsed.asOfCommitSeq !== 'number' ||
110
+ !Array.isArray(parsed.tables) ||
111
+ typeof parsed.tableIndex !== 'number' ||
112
+ (parsed.rowCursor !== null && typeof parsed.rowCursor !== 'string') ||
113
+ parsed.tables[0] !== table
114
+ ) {
115
+ return undefined;
322
116
  }
117
+ return {
118
+ asOfCommitSeq: parsed.asOfCommitSeq,
119
+ tables: parsed.tables as string[],
120
+ tableIndex: parsed.tableIndex,
121
+ rowCursor: parsed.rowCursor ?? null,
122
+ };
123
+ } catch {
124
+ return undefined;
323
125
  }
126
+ }
324
127
 
325
- return columns.map((column) => ({
326
- name: column.name,
327
- type: column.type ?? 'json',
328
- ...(column.nullable || column.presentCount < rows.length
329
- ? { nullable: true }
128
+ function commitFrame(table: string, commit: StoredCommit): ResponseFrame {
129
+ const changes: CommitChange[] = commit.changes.map((change) => ({
130
+ tableIndex: 0,
131
+ rowId: change.rowId,
132
+ op: change.op,
133
+ ...(change.rowVersion !== undefined
134
+ ? { rowVersion: change.rowVersion }
330
135
  : {}),
136
+ scopes: change.scopes,
137
+ ...(change.payload !== undefined ? { row: change.payload } : {}),
331
138
  }));
139
+ return {
140
+ type: 'COMMIT',
141
+ commitSeq: commit.commitSeq,
142
+ createdAtMs: commit.createdAtMs,
143
+ actorId: commit.actorId,
144
+ tables: [table],
145
+ changes,
146
+ };
332
147
  }
333
148
 
334
- function inferBinarySnapshotColumnType(
335
- value: unknown
336
- ): BinarySnapshotColumnType {
337
- if (typeof value === 'string') return 'string';
338
- if (typeof value === 'boolean') return 'boolean';
339
- if (typeof value === 'bigint') return 'integer';
340
- if (typeof value === 'number') {
341
- return Number.isSafeInteger(value) ? 'integer' : 'float';
342
- }
343
- if (value instanceof Uint8Array || value instanceof ArrayBuffer) {
344
- return 'bytes';
345
- }
346
- return 'json';
347
- }
348
-
349
- function mergeBinarySnapshotColumnTypes(
350
- current: BinarySnapshotColumnType | null,
351
- next: BinarySnapshotColumnType
352
- ): BinarySnapshotColumnType {
353
- if (!current || current === next) return next;
354
- if (
355
- (current === 'integer' && next === 'float') ||
356
- (current === 'float' && next === 'integer')
357
- ) {
358
- return 'float';
149
+ function chunkRows(rows: SegmentRow[], size: number): SegmentRow[][] {
150
+ if (rows.length === 0) return [];
151
+ const blocks: SegmentRow[][] = [];
152
+ for (let i = 0; i < rows.length; i += size) {
153
+ blocks.push(rows.slice(i, i + size));
359
154
  }
360
- return 'json';
361
- }
362
-
363
- export interface PullResult {
364
- response: SyncPullResponse;
365
- /**
366
- * Effective scopes for all active subscriptions (for cursor tracking).
367
- * Maps subscription ID to effective scopes.
368
- */
369
- effectiveScopes: ScopeValues;
370
- /** Minimum nextCursor across active subscriptions (for pruning cursor tracking). */
371
- clientCursor: number;
372
- /** Internal bootstrap timing breakdown used for benchmark-gated diagnostics. */
373
- bootstrapTimings?: PullBootstrapTimings;
155
+ return blocks;
156
+ }
157
+
158
+ /** §5.4 signed-URL fields for a descriptor, when the client asked (bit 3). */
159
+ async function signedUrlFields(
160
+ ctx: SyncRequestContext,
161
+ limits: PullLimits,
162
+ segmentId: string,
163
+ digest: string,
164
+ now: number,
165
+ ): Promise<{ url?: string; urlExpiresAtMs?: number }> {
166
+ if ((limits.accept & ACCEPT_SIGNED_URLS) === 0 || !ctx.signedUrls) return {};
167
+ return issueSegmentUrl(ctx.signedUrls, {
168
+ segmentId,
169
+ partition: ctx.partition,
170
+ scopeDigest: digest,
171
+ nowMs: now,
172
+ });
374
173
  }
375
174
 
376
- interface PendingExternalChunkWrite {
377
- snapshot: SyncSnapshot;
378
- cacheLookup: {
379
- partitionId: string;
380
- scopeKey: string;
381
- scope: string;
382
- asOfCommitSeq: number;
383
- rowCursor: string | null;
384
- rowLimit: number;
385
- nextRowCursor: string | null;
386
- isLastPage: boolean;
175
+ function segmentRefFrame(
176
+ record: SegmentRecord,
177
+ extra: { url?: string; urlExpiresAtMs?: number },
178
+ ): ResponseFrame {
179
+ return {
180
+ type: 'SEGMENT_REF',
181
+ segmentId: record.segmentId,
182
+ mediaType: record.mediaType,
183
+ table: record.table,
184
+ byteLength: record.byteLength,
185
+ rowCount: record.rowCount,
186
+ asOfCommitSeq: record.asOfCommitSeq,
187
+ scopeDigest: record.scopeDigest,
188
+ ...(record.rowCursor !== null ? { rowCursor: record.rowCursor } : {}),
189
+ ...(record.nextRowCursor !== null
190
+ ? { nextRowCursor: record.nextRowCursor }
191
+ : {}),
192
+ ...(extra.url !== undefined ? { url: extra.url } : {}),
193
+ ...(extra.urlExpiresAtMs !== undefined
194
+ ? { urlExpiresAtMs: extra.urlExpiresAtMs }
195
+ : {}),
387
196
  };
388
- payloadParts: Uint8Array[];
389
- expiresAt: string;
390
- }
391
-
392
- async function runWithConcurrency<T>(
393
- items: readonly T[],
394
- concurrency: number,
395
- worker: (item: T) => Promise<void>
396
- ): Promise<void> {
397
- if (items.length === 0) return;
398
-
399
- const workerCount = Math.max(1, Math.min(concurrency, items.length));
400
- let nextIndex = 0;
401
-
402
- async function runWorker(): Promise<void> {
403
- while (nextIndex < items.length) {
404
- const index = nextIndex;
405
- nextIndex += 1;
406
- const item = items[index];
407
- if (item === undefined) continue;
408
- await worker(item);
409
- }
410
- }
411
-
412
- await Promise.all(Array.from({ length: workerCount }, () => runWorker()));
413
197
  }
414
198
 
415
199
  /**
416
- * Sanitize a numeric limit parameter with bounds checking.
417
- * Handles NaN, negative values, and undefined.
200
+ * §5.3 sqlite-image lane: only at the start of a table, only when the
201
+ * client advertised bit 2, and only when the snapshot exceeds one rows
202
+ * page. Reuses an unexpired stored image for the same (partition, table,
203
+ * schemaVersion, scope digest, pin) instead of rebuilding — the
204
+ * bootstrap-storm rule. Returns false when the table is not eligible
205
+ * (the rows lane takes over).
418
206
  */
419
- function sanitizeLimit(
420
- value: number | undefined,
421
- defaultValue: number,
422
- min: number,
423
- max: number
424
- ): number {
425
- if (value === undefined || value === null) return defaultValue;
426
- if (Number.isNaN(value)) return defaultValue;
427
- return Math.max(min, Math.min(max, value));
428
- }
429
-
430
- function sanitizeGzipLevel(value: number | undefined): number {
431
- if (value === undefined || value === null || !Number.isFinite(value)) {
432
- return DEFAULT_SNAPSHOT_CHUNK_GZIP_LEVEL;
433
- }
434
- return Math.max(0, Math.min(9, Math.trunc(value)));
435
- }
436
-
437
- function isSerializablePullError(error: Error): boolean {
438
- const withCode = error as Error & { code?: string };
439
- return (
440
- withCode.code === '40001' ||
441
- error.message.toLowerCase().includes('could not serialize access')
207
+ async function* sqliteImageSegment(
208
+ ctx: SyncRequestContext,
209
+ schema: CompiledSchema,
210
+ limits: PullLimits,
211
+ plan: SubscriptionPlan,
212
+ asOf: number,
213
+ digest: string,
214
+ trace: PullSectionTrace | undefined,
215
+ ): AsyncGenerator<ResponseFrame, boolean> {
216
+ const { storage, segments, partition } = ctx;
217
+ const now = clockOf(ctx)();
218
+ const existing = await segments.find(
219
+ {
220
+ partition,
221
+ table: plan.table.name,
222
+ schemaVersion: schema.version,
223
+ mediaType: 'sqlite',
224
+ scopeDigest: digest,
225
+ asOfCommitSeq: asOf,
226
+ },
227
+ now,
442
228
  );
443
- }
444
-
445
- async function delay(ms: number): Promise<void> {
446
- await new Promise((resolve) => {
447
- setTimeout(resolve, ms);
448
- });
449
- }
450
-
451
- /**
452
- * Merge all scope values into a flat ScopeValues for cursor tracking.
453
- */
454
- function mergeScopes(subscriptions: { scopes: ScopeValues }[]): ScopeValues {
455
- const result: Record<string, Set<string>> = {};
456
-
457
- for (const sub of subscriptions) {
458
- for (const [key, value] of Object.entries(sub.scopes)) {
459
- if (!result[key]) result[key] = new Set();
460
- const arr = Array.isArray(value) ? value : [value];
461
- for (const v of arr) result[key].add(v);
462
- }
463
- }
464
-
465
- const merged: ScopeValues = {};
466
- for (const [key, set] of Object.entries(result)) {
467
- const arr = Array.from(set);
468
- if (arr.length === 0) continue;
469
- merged[key] = arr.length === 1 ? arr[0]! : arr;
470
- }
471
- return merged;
472
- }
473
-
474
- interface PullResponseStats {
475
- subscriptionCount: number;
476
- activeSubscriptionCount: number;
477
- revokedSubscriptionCount: number;
478
- bootstrapSubscriptionCount: number;
479
- commitCount: number;
480
- changeCount: number;
481
- snapshotPageCount: number;
482
- }
483
-
484
- function assertPullChangeIdentityUnchanged(
485
- source: string,
486
- before: SyncChange,
487
- after: SyncChange
488
- ): void {
489
- if (before.table !== after.table) {
490
- throw new Error(
491
- `${source} cannot change change.table (${before.table} -> ${after.table})`
229
+ if (existing !== undefined) {
230
+ trace?.segments.push({
231
+ mediaType: 'sqlite',
232
+ delivery: 'ref',
233
+ origin: 'reused',
234
+ bytes: existing.byteLength,
235
+ rows: existing.rowCount,
236
+ });
237
+ yield segmentRefFrame(
238
+ existing,
239
+ await signedUrlFields(ctx, limits, existing.segmentId, digest, now),
492
240
  );
241
+ return true;
493
242
  }
494
- if (before.row_id !== after.row_id) {
495
- throw new Error(
496
- `${source} cannot change change.row_id (${before.row_id} -> ${after.row_id})`
497
- );
243
+ // Eligibility probe (§5.3): image only when the snapshot exceeds one
244
+ // rows page — smaller tables stay on the (typically inline) rows lane.
245
+ const probe = await storage.scanRows(partition, {
246
+ table: plan.table.name,
247
+ scopeFilter: plan.effective,
248
+ afterRowId: null,
249
+ limit: limits.limitSnapshotRows + 1,
250
+ });
251
+ if (probe.length <= limits.limitSnapshotRows) return false;
252
+ // §5.3: building an image needs a SQLite engine. The host injects the
253
+ // builder through `sqliteImageBuilder`; when omitted we default to the
254
+ // in-tree `buildSqliteImage` ONLY on a Bun runtime, reached by a *dynamic*
255
+ // import so `bun:sqlite` is never a static dependency of the pull path
256
+ // (TODO §4.2 neutrality — enforced by test/runtime-neutrality.test.ts). On
257
+ // Workers/edge (no `Bun`) this yields the rows lane for a bit-2 client — a
258
+ // support floor, not a fallback (§5.3: sqlite is an *accept*, not a demand).
259
+ const buildImage = await resolveImageBuilder(ctx);
260
+ if (buildImage === undefined) return false;
261
+ // The probe rows are the snapshot's first page — keep them and scan on
262
+ // from the probe's cursor instead of re-reading the whole prefix (the
263
+ // scan is keyset-ordered by rowId, so the concatenation is exactly the
264
+ // rows a single full scan would return).
265
+ const rows: StoredRow[] = [...probe];
266
+ let afterRowId: string | null = probe[probe.length - 1]?.rowId ?? null;
267
+ for (;;) {
268
+ const scanned: StoredRow[] = await storage.scanRows(partition, {
269
+ table: plan.table.name,
270
+ scopeFilter: plan.effective,
271
+ afterRowId,
272
+ limit: 50_000,
273
+ });
274
+ rows.push(...scanned);
275
+ const last = scanned[scanned.length - 1];
276
+ if (scanned.length < 50_000 || last === undefined) break;
277
+ afterRowId = last.rowId;
498
278
  }
499
- if (before.op !== after.op) {
500
- throw new Error(
501
- `${source} cannot change change.op (${before.op} -> ${after.op})`
279
+ const bytes = buildImage({
280
+ table: plan.table,
281
+ schemaVersion: schema.version,
282
+ asOfCommitSeq: asOf,
283
+ scopeDigest: digest,
284
+ rows,
285
+ });
286
+ const record = await segments.put(
287
+ {
288
+ partition,
289
+ table: plan.table.name,
290
+ schemaVersion: schema.version,
291
+ mediaType: 'sqlite',
292
+ scopeDigest: digest,
293
+ asOfCommitSeq: asOf,
294
+ rowCount: rows.length,
295
+ rowCursor: null,
296
+ nextRowCursor: null,
297
+ },
298
+ bytes,
299
+ now,
300
+ );
301
+ trace?.segments.push({
302
+ mediaType: 'sqlite',
303
+ delivery: 'ref',
304
+ origin: 'built',
305
+ bytes: record.byteLength,
306
+ rows: record.rowCount,
307
+ });
308
+ yield segmentRefFrame(
309
+ record,
310
+ await signedUrlFields(ctx, limits, record.segmentId, digest, now),
311
+ );
312
+ return true;
313
+ }
314
+
315
+ async function* bootstrapSegments(
316
+ ctx: SyncRequestContext,
317
+ schema: CompiledSchema,
318
+ limits: PullLimits,
319
+ plan: SubscriptionPlan,
320
+ asOf: number,
321
+ startRowCursor: string | null,
322
+ trace: PullSectionTrace | undefined,
323
+ ): AsyncGenerator<
324
+ ResponseFrame,
325
+ { complete: boolean; rowCursor: string | null }
326
+ > {
327
+ const { storage, segments, partition } = ctx;
328
+ const serverLimits = limitsOf(ctx);
329
+ const digest = await scopeDigest(plan.effective);
330
+ const now = clockOf(ctx)();
331
+ // §5.3: the sqlite-image lane — whole-table, chosen only at the start
332
+ // of a table (a mid-table resume stays on the rows lane, never
333
+ // switching lanes), and only when the client advertised bit 2.
334
+ // §5.11: a table with any encrypted column is image-INELIGIBLE — an image
335
+ // copies ciphertext wholesale with no per-row decrypt pass, so it MUST be
336
+ // served on the rows lane (which decrypts per row on the client).
337
+ if (
338
+ (limits.accept & ACCEPT_SQLITE) !== 0 &&
339
+ startRowCursor === null &&
340
+ plan.table.encryptedColumnIndices.length === 0
341
+ ) {
342
+ const imaged = yield* sqliteImageSegment(
343
+ ctx,
344
+ schema,
345
+ limits,
346
+ plan,
347
+ asOf,
348
+ digest,
349
+ trace,
502
350
  );
351
+ if (imaged) return { complete: true, rowCursor: null };
503
352
  }
504
- }
505
-
506
- function projectPullChangeForClientSchema<
507
- DB extends SyncCoreDb,
508
- Auth extends SyncServerAuth,
509
- >(args: {
510
- handler: ServerTableHandler<DB, Auth>;
511
- change: SyncChange;
512
- schemaVersion: number;
513
- }): SyncChange {
514
- const projected = args.handler.projectChangeForVersion
515
- ? args.handler.projectChangeForVersion(args.change, args.schemaVersion)
516
- : args.change;
517
- assertPullChangeIdentityUnchanged(
518
- 'Server table handler projectChangeForVersion',
519
- args.change,
520
- projected
521
- );
522
- return projected;
523
- }
524
-
525
- async function transformPullChanges<
526
- DB extends SyncCoreDb,
527
- Auth extends SyncServerAuth,
528
- >(args: {
529
- plugins: readonly SyncServerPullPlugin<DB, Auth>[];
530
- ctx: { db: DbExecutor<DB>; actorId: string; auth: Auth };
531
- tableHandler: ServerTableHandler<DB, Auth>;
532
- subscription: {
533
- id: string;
534
- table: string;
535
- scopes: ScopeValues;
536
- params: Record<string, unknown> | undefined;
537
- cursor: number;
538
- crdtStateVectors: readonly SyncCrdtStateVectorHint[];
539
- };
540
- changes: readonly SyncChange[];
541
- }): Promise<SyncChange[]> {
542
- let changes = [...args.changes];
543
- for (const plugin of args.plugins) {
544
- if (!plugin.transformPullChanges) continue;
545
- const nextChanges = await plugin.transformPullChanges({
546
- ctx: args.ctx,
547
- tableHandler: args.tableHandler,
548
- subscription: args.subscription,
549
- changes,
353
+ let rowCursor = startRowCursor;
354
+ for (let page = 0; page < limits.maxSnapshotPages; page++) {
355
+ const scanned = await storage.scanRows(partition, {
356
+ table: plan.table.name,
357
+ scopeFilter: plan.effective,
358
+ afterRowId: rowCursor,
359
+ limit: limits.limitSnapshotRows + 1,
360
+ });
361
+ const pageRows = scanned.slice(0, limits.limitSnapshotRows);
362
+ const hasMore = scanned.length > limits.limitSnapshotRows;
363
+ // §5.2: every row record carries the row's current server_version.
364
+ const decoded = pageRows.map((row: StoredRow) => ({
365
+ serverVersion: row.serverVersion,
366
+ values: decodeRow(plan.table.columns, row.payload),
367
+ }));
368
+ const bytes = encodeRowsSegment({
369
+ table: plan.table.name,
370
+ schemaVersion: schema.version,
371
+ columns: plan.table.columns,
372
+ blocks: chunkRows(decoded, 1000),
550
373
  });
551
- if (nextChanges.length !== changes.length) {
552
- throw new Error(
553
- `Server pull plugin "${plugin.name}" cannot change pull change count (${changes.length} -> ${nextChanges.length})`
374
+ const lastRow = pageRows[pageRows.length - 1];
375
+ const nextRowCursor =
376
+ hasMore && lastRow !== undefined ? lastRow.rowId : null;
377
+
378
+ const canInline = (limits.accept & ACCEPT_INLINE_ROWS) !== 0;
379
+ const canExternal = (limits.accept & ACCEPT_EXTERNAL_ROWS) !== 0;
380
+ const inline =
381
+ canInline &&
382
+ (bytes.length <= serverLimits.inlineSegmentMaxBytes || !canExternal);
383
+ if (inline) {
384
+ trace?.segments.push({
385
+ mediaType: 'rows',
386
+ delivery: 'inline',
387
+ origin: 'built',
388
+ bytes: bytes.length,
389
+ rows: pageRows.length,
390
+ });
391
+ yield { type: 'SEGMENT_INLINE', payload: bytes };
392
+ } else {
393
+ const record = await segments.put(
394
+ {
395
+ partition,
396
+ table: plan.table.name,
397
+ schemaVersion: schema.version,
398
+ mediaType: 'rows',
399
+ scopeDigest: digest,
400
+ asOfCommitSeq: asOf,
401
+ rowCount: pageRows.length,
402
+ rowCursor,
403
+ nextRowCursor,
404
+ },
405
+ bytes,
406
+ now,
554
407
  );
555
- }
556
- for (let i = 0; i < changes.length; i += 1) {
557
- assertPullChangeIdentityUnchanged(
558
- `Server pull plugin "${plugin.name}"`,
559
- changes[i]!,
560
- nextChanges[i]!
408
+ trace?.segments.push({
409
+ mediaType: 'rows',
410
+ delivery: 'ref',
411
+ origin: 'built',
412
+ bytes: record.byteLength,
413
+ rows: record.rowCount,
414
+ });
415
+ yield segmentRefFrame(
416
+ record,
417
+ await signedUrlFields(ctx, limits, record.segmentId, digest, now),
561
418
  );
562
419
  }
563
- changes = [...nextChanges];
420
+ if (!hasMore) return { complete: true, rowCursor: null };
421
+ rowCursor = nextRowCursor;
564
422
  }
565
- return changes;
423
+ return { complete: false, rowCursor };
566
424
  }
567
425
 
568
- function summarizePullResponse(response: SyncPullResponse): PullResponseStats {
569
- const subscriptions = response.subscriptions ?? [];
570
- let activeSubscriptionCount = 0;
571
- let revokedSubscriptionCount = 0;
572
- let bootstrapSubscriptionCount = 0;
573
- let commitCount = 0;
574
- let changeCount = 0;
575
- let snapshotPageCount = 0;
576
-
577
- for (const sub of subscriptions) {
578
- if (sub.status === 'revoked') {
579
- revokedSubscriptionCount += 1;
580
- } else {
581
- activeSubscriptionCount += 1;
582
- }
426
+ /**
427
+ * Produce the `SUB_START … SUB_END` section for one subscription (§1.6),
428
+ * returning the cursor recorded for the retention watermark (§4.5).
429
+ */
430
+ export async function* subscriptionSection(
431
+ ctx: SyncRequestContext,
432
+ schema: CompiledSchema,
433
+ limits: PullLimits,
434
+ plan: SubscriptionPlan,
435
+ maxSeq: number,
436
+ horizonSeq: number,
437
+ trace?: PullSectionTrace,
438
+ ): AsyncGenerator<ResponseFrame, SubscriptionResult> {
439
+ const sub = plan.frame;
440
+
441
+ if (plan.status === 'revoked') {
442
+ yield {
443
+ type: 'SUB_START',
444
+ id: sub.id,
445
+ status: 'revoked',
446
+ reasonCode: 'sync.scope_revoked',
447
+ effectiveScopes: {},
448
+ bootstrap: false,
449
+ };
450
+ yield { type: 'SUB_END', nextCursor: sub.cursor };
451
+ return { nextCursor: sub.cursor, active: false };
452
+ }
583
453
 
584
- if (sub.bootstrap) {
585
- bootstrapSubscriptionCount += 1;
586
- }
454
+ const token = parseBootstrapToken(sub.bootstrapState, sub.table);
455
+
456
+ // §4.6: a cursor behind the horizon (and not resuming a bootstrap)
457
+ // cannot compute deltas — answer `reset` and echo the cursor.
458
+ if (token === undefined && sub.cursor >= 0 && sub.cursor < horizonSeq) {
459
+ yield {
460
+ type: 'SUB_START',
461
+ id: sub.id,
462
+ status: 'reset',
463
+ reasonCode: 'sync.cursor_expired',
464
+ effectiveScopes: {},
465
+ bootstrap: false,
466
+ };
467
+ yield { type: 'SUB_END', nextCursor: sub.cursor };
468
+ return { nextCursor: sub.cursor, active: false };
469
+ }
587
470
 
588
- const commits = sub.commits ?? [];
589
- commitCount += commits.length;
590
- for (const commit of commits) {
591
- changeCount += commit.changes?.length ?? 0;
471
+ const bootstrapping =
472
+ token !== undefined || sub.cursor < 0 || sub.cursor > maxSeq;
473
+
474
+ if (bootstrapping) {
475
+ // §4.7: resume at the pinned point unless the pin fell behind the
476
+ // horizon (or the token is unusable) — then restart from a fresh pin.
477
+ const resume =
478
+ token !== undefined && token.asOfCommitSeq >= horizonSeq
479
+ ? token
480
+ : undefined;
481
+ const asOf = resume?.asOfCommitSeq ?? maxSeq;
482
+ const startCursor = resume?.rowCursor ?? null;
483
+ yield {
484
+ type: 'SUB_START',
485
+ id: sub.id,
486
+ status: 'active',
487
+ reasonCode: '',
488
+ effectiveScopes: plan.effective,
489
+ bootstrap: true,
490
+ };
491
+ const outcome = yield* bootstrapSegments(
492
+ ctx,
493
+ schema,
494
+ limits,
495
+ plan,
496
+ asOf,
497
+ startCursor,
498
+ trace,
499
+ );
500
+ if (outcome.complete) {
501
+ yield { type: 'SUB_END', nextCursor: asOf };
502
+ } else {
503
+ const nextToken: BootstrapToken = {
504
+ asOfCommitSeq: asOf,
505
+ tables: [sub.table],
506
+ tableIndex: 0,
507
+ rowCursor: outcome.rowCursor,
508
+ };
509
+ yield {
510
+ type: 'SUB_END',
511
+ nextCursor: asOf,
512
+ bootstrapState: JSON.stringify(nextToken),
513
+ };
592
514
  }
593
-
594
- snapshotPageCount += sub.snapshots?.length ?? 0;
515
+ return { nextCursor: asOf, active: true };
595
516
  }
596
517
 
597
- return {
598
- subscriptionCount: subscriptions.length,
599
- activeSubscriptionCount,
600
- revokedSubscriptionCount,
601
- bootstrapSubscriptionCount,
602
- commitCount,
603
- changeCount,
604
- snapshotPageCount,
605
- };
606
- }
607
-
608
- function recordPullMetrics(args: {
609
- status: string;
610
- dedupeRows: boolean;
611
- durationMs: number;
612
- stats: PullResponseStats;
613
- }): void {
614
- const { status, dedupeRows, durationMs, stats } = args;
615
- const attributes = {
616
- status,
617
- dedupe_rows: dedupeRows,
518
+ // Incremental (§4.5): window cursor < commitSeq <= maxSeq, oldest first,
519
+ // cut off at limitCommits total changes, never splitting a commit.
520
+ yield {
521
+ type: 'SUB_START',
522
+ id: sub.id,
523
+ status: 'active',
524
+ reasonCode: '',
525
+ effectiveScopes: plan.effective,
526
+ bootstrap: false,
618
527
  };
619
-
620
- countSyncMetric('sync.server.pull.requests', 1, { attributes });
621
- distributionSyncMetric('sync.server.pull.duration_ms', durationMs, {
622
- unit: 'millisecond',
623
- attributes,
624
- });
625
- distributionSyncMetric(
626
- 'sync.server.pull.subscriptions',
627
- stats.subscriptionCount,
628
- { attributes }
629
- );
630
- distributionSyncMetric(
631
- 'sync.server.pull.active_subscriptions',
632
- stats.activeSubscriptionCount,
633
- { attributes }
634
- );
635
- distributionSyncMetric(
636
- 'sync.server.pull.revoked_subscriptions',
637
- stats.revokedSubscriptionCount,
638
- { attributes }
639
- );
640
- distributionSyncMetric(
641
- 'sync.server.pull.bootstrap_subscriptions',
642
- stats.bootstrapSubscriptionCount,
643
- { attributes }
644
- );
645
- distributionSyncMetric('sync.server.pull.commits', stats.commitCount, {
646
- attributes,
528
+ const commits = await ctx.storage.readCommitWindow(ctx.partition, {
529
+ table: sub.table,
530
+ scopeFilter: plan.effective,
531
+ afterSeq: sub.cursor,
532
+ throughSeq: maxSeq,
533
+ limitChanges: limits.limitCommits + 1,
647
534
  });
648
- distributionSyncMetric('sync.server.pull.changes', stats.changeCount, {
649
- attributes,
650
- });
651
- distributionSyncMetric(
652
- 'sync.server.pull.snapshot_pages',
653
- stats.snapshotPageCount,
654
- { attributes }
655
- );
656
- }
657
-
658
- async function readLatestExternalCommitByTable<DB extends SyncCoreDb>(
659
- trx: DbExecutor<DB>,
660
- args: { partitionId: string; afterCursor: number; tables: string[] }
661
- ): Promise<Map<string, number>> {
662
- const tableNames = Array.from(
663
- new Set(args.tables.filter((table) => typeof table === 'string'))
664
- );
665
- const latestByTable = new Map<string, number>();
666
- if (tableNames.length === 0) {
667
- return latestByTable;
668
- }
669
-
670
- type SyncExecutor = Pick<Kysely<SyncCoreDb>, 'selectFrom'>;
671
- const executor = trx as SyncExecutor;
672
- const rows = await executor
673
- .selectFrom('sync_table_commits as tc')
674
- .innerJoin('sync_commits as cm', (join) =>
675
- join
676
- .onRef('cm.commit_seq', '=', 'tc.commit_seq')
677
- .onRef('cm.partition_id', '=', 'tc.partition_id')
678
- )
679
- .select(['tc.table as table'])
680
- .select((eb) => eb.fn.max('tc.commit_seq').as('latest_commit_seq'))
681
- .where('tc.partition_id', '=', args.partitionId)
682
- .where('cm.client_id', '=', EXTERNAL_CLIENT_ID)
683
- .where('cm.change_count', '=', 0)
684
- .where('tc.commit_seq', '>', args.afterCursor)
685
- .where('tc.table', 'in', tableNames)
686
- .groupBy('tc.table')
687
- .execute();
688
-
689
- for (const row of rows) {
690
- const commitSeq = Number(row.latest_commit_seq ?? -1);
691
- if (!Number.isFinite(commitSeq) || commitSeq < 0) continue;
692
- latestByTable.set(row.table, commitSeq);
693
- }
694
-
695
- return latestByTable;
696
- }
697
-
698
- export async function pull<
699
- DB extends SyncCoreDb,
700
- Auth extends SyncServerAuth,
701
- >(args: {
702
- db: Kysely<DB>;
703
- dialect: ServerSyncDialect;
704
- handlers: ServerHandlerCollection<DB, Auth>;
705
- auth: Auth;
706
- request: SyncPullRequest;
707
- /**
708
- * Optional snapshot chunk storage adapter.
709
- * When provided, stores chunk bodies in external storage (S3, etc.)
710
- * instead of inline in the database.
711
- */
712
- chunkStorage?: SnapshotChunkStorage;
713
- /**
714
- * Optional shared scope cache backend.
715
- * Request-local memoization is always applied, even with custom backends.
716
- * Defaults to process-local memory cache.
717
- */
718
- scopeCache?: ScopeCacheBackend;
719
- /**
720
- * Gzip compression level for generated snapshot chunks. The protocol remains
721
- * gzip-only; this tunes CPU/size tradeoffs for deployments that know their
722
- * network constraints.
723
- *
724
- * Default: 1, range: 0-9.
725
- */
726
- snapshotChunkGzipLevel?: number;
727
- /**
728
- * Schema/cache semantic version included in generated snapshot chunk cache
729
- * keys. Changing this value invalidates cached bootstrap chunks without
730
- * requiring table data to change.
731
- */
732
- snapshotChunkCacheSchemaVersion?: number | string | null;
733
- /**
734
- * Optional server plugins for protocol-level pull transforms.
735
- */
736
- plugins?: readonly SyncServerPullPlugin<DB, Auth>[];
737
- }): Promise<PullResult> {
738
- const { request, dialect } = args;
739
- const db = args.db;
740
- const pullPlugins = sortServerPullPlugins(args.plugins);
741
- const partitionId = args.auth.partitionId ?? 'default';
742
- const snapshotChunkGzipLevel = sanitizeGzipLevel(args.snapshotChunkGzipLevel);
743
- const clientSchemaVersion = request.schemaVersion;
744
- if (!Number.isInteger(clientSchemaVersion) || clientSchemaVersion < 1) {
745
- throw new Error('Pull request schemaVersion must be a positive integer');
746
- }
747
- const snapshotChunkCacheSchemaVersion =
748
- args.snapshotChunkCacheSchemaVersion === null ||
749
- args.snapshotChunkCacheSchemaVersion === undefined
750
- ? clientSchemaVersion
751
- : `${clientSchemaVersion}:${args.snapshotChunkCacheSchemaVersion}`;
752
- const requestedSubscriptionCount = Array.isArray(request.subscriptions)
753
- ? request.subscriptions.length
754
- : 0;
755
- const startedAtMs = Date.now();
756
-
757
- return startSyncSpan(
758
- {
759
- name: 'sync.server.pull',
760
- op: 'sync.pull',
761
- attributes: {
762
- requested_subscription_count: requestedSubscriptionCount,
763
- dedupe_rows: request.dedupeRows === true,
764
- },
765
- },
766
- async (span) => {
767
- try {
768
- // Validate and sanitize request limits
769
- const limitCommits = sanitizeLimit(request.limitCommits, 1000, 1, 1000);
770
- const limitSnapshotRows = sanitizeLimit(
771
- request.limitSnapshotRows,
772
- 1000,
773
- 1,
774
- 50000
775
- );
776
- const maxSnapshotPages = sanitizeLimit(
777
- request.maxSnapshotPages,
778
- 4,
779
- 1,
780
- 50
781
- );
782
- const dedupeRows = request.dedupeRows === true;
783
- const snapshotChunkEncoding = SYNC_SNAPSHOT_CHUNK_ENCODING;
784
- const snapshotArtifactSelection = resolveSnapshotArtifactSelection(
785
- request.snapshotArtifacts,
786
- clientSchemaVersion
787
- );
788
- const snapshotArtifactSchemaVersion =
789
- snapshotArtifactSelection?.schemaVersion ?? null;
790
- // Resolve effective scopes for each subscription
791
- const resolved = await resolveEffectiveScopesForSubscriptions({
792
- db,
793
- auth: args.auth,
794
- subscriptions: request.subscriptions ?? [],
795
- handlers: args.handlers,
796
- scopeCache: args.scopeCache ?? defaultScopeCache,
797
- });
798
-
799
- for (
800
- let attemptIndex = 0;
801
- attemptIndex < MAX_PULL_TRANSACTION_RETRIES;
802
- attemptIndex += 1
803
- ) {
804
- const pendingExternalChunkWrites: PendingExternalChunkWrite[] = [];
805
- const bootstrapTimings = createPullBootstrapTimings();
806
-
807
- try {
808
- const result = await dialect.executeInTransaction(
809
- db,
810
- async (trx) => {
811
- await dialect.setRepeatableRead(trx);
812
-
813
- const maxCommitSeq = await dialect.readMaxCommitSeq(trx, {
814
- partitionId,
815
- });
816
- const minCommitSeq = await dialect.readMinCommitSeq(trx, {
817
- partitionId,
818
- });
819
-
820
- const subResponses: SyncPullSubscriptionResponse[] = [];
821
- const activeSubscriptions: { scopes: ScopeValues }[] = [];
822
- const nextCursors: number[] = [];
823
-
824
- // Detect external data changes (synthetic commits from notifyExternalDataChange)
825
- // Compute minimum cursor across all active subscriptions to scope the query.
826
- let minSubCursor = Number.MAX_SAFE_INTEGER;
827
- const activeTables = new Set<string>();
828
- for (const sub of resolved) {
829
- if (
830
- sub.status === 'revoked' ||
831
- Object.keys(sub.scopes).length === 0
832
- )
833
- continue;
834
- activeTables.add(sub.table);
835
- const cursor = Math.max(-1, sub.cursor ?? -1);
836
- if (cursor >= 0 && cursor < minSubCursor) {
837
- minSubCursor = cursor;
838
- }
839
- }
840
-
841
- const maxExternalCommitByTable =
842
- minSubCursor < Number.MAX_SAFE_INTEGER && minSubCursor >= 0
843
- ? await readLatestExternalCommitByTable(trx, {
844
- partitionId,
845
- afterCursor: minSubCursor,
846
- tables: Array.from(activeTables),
847
- })
848
- : new Map<string, number>();
849
-
850
- for (const sub of resolved) {
851
- const cursor = Math.max(-1, sub.cursor ?? -1);
852
- // Validate table handler exists (throws if not registered)
853
- if (!args.handlers.byTable.has(sub.table)) {
854
- throw new Error(`Unknown table: ${sub.table}`);
855
- }
856
-
857
- if (
858
- sub.status === 'revoked' ||
859
- Object.keys(sub.scopes).length === 0
860
- ) {
861
- subResponses.push({
862
- id: sub.id,
863
- status: 'revoked',
864
- scopes: {},
865
- bootstrap: false,
866
- nextCursor: cursor,
867
- commits: [],
868
- });
869
- continue;
870
- }
871
-
872
- const effectiveScopes = sub.scopes;
873
- activeSubscriptions.push({ scopes: effectiveScopes });
874
- const latestExternalCommitForTable =
875
- maxExternalCommitByTable.get(sub.table);
876
-
877
- const needsBootstrap =
878
- sub.bootstrapState != null ||
879
- cursor < 0 ||
880
- cursor > maxCommitSeq ||
881
- (minCommitSeq > 0 && cursor < minCommitSeq - 1) ||
882
- (latestExternalCommitForTable !== undefined &&
883
- latestExternalCommitForTable > cursor);
884
-
885
- if (needsBootstrap) {
886
- const tables = getServerBootstrapOrderFor(
887
- args.handlers,
888
- sub.table
889
- ).map((handler) => handler.table);
890
- const initState: SyncBootstrapState = {
891
- asOfCommitSeq: maxCommitSeq,
892
- tables,
893
- tableIndex: 0,
894
- rowCursor: null,
895
- };
896
-
897
- const requestedState = sub.bootstrapState ?? null;
898
- const state =
899
- requestedState &&
900
- typeof requestedState.asOfCommitSeq === 'number' &&
901
- Array.isArray(requestedState.tables) &&
902
- typeof requestedState.tableIndex === 'number'
903
- ? (requestedState as SyncBootstrapState)
904
- : initState;
905
-
906
- // If the bootstrap state's asOfCommitSeq is no longer catch-up-able, restart bootstrap.
907
- const effectiveState =
908
- state.asOfCommitSeq < minCommitSeq - 1
909
- ? initState
910
- : state;
911
-
912
- const tableName =
913
- effectiveState.tables[effectiveState.tableIndex];
914
-
915
- // No tables (or ran past the end): treat bootstrap as complete.
916
- if (!tableName) {
917
- subResponses.push({
918
- id: sub.id,
919
- status: 'active',
920
- scopes: effectiveScopes,
921
- bootstrap: true,
922
- bootstrapState: null,
923
- nextCursor: effectiveState.asOfCommitSeq,
924
- commits: [],
925
- snapshots: [],
926
- });
927
- nextCursors.push(effectiveState.asOfCommitSeq);
928
- continue;
929
- }
930
-
931
- const snapshots: SyncSnapshot[] = [];
932
- let nextState: SyncBootstrapState | null = effectiveState;
933
- const cacheKey = await createSnapshotChunkScopeCacheKey({
934
- partitionId,
935
- scopes: effectiveScopes,
936
- schemaVersion: snapshotChunkCacheSchemaVersion,
937
- encoding: snapshotChunkEncoding,
938
- compression: SYNC_SNAPSHOT_CHUNK_COMPRESSION,
939
- gzipLevel: snapshotChunkGzipLevel,
940
- });
941
- const artifactScopeKey =
942
- snapshotArtifactSelection && snapshotArtifactSchemaVersion
943
- ? await createScopedSnapshotArtifactScopeCacheKey({
944
- partitionId,
945
- subscriptionId: sub.id,
946
- scopes: effectiveScopes,
947
- schemaVersion: snapshotArtifactSchemaVersion,
948
- artifactKind:
949
- snapshotArtifactSelection.artifactKind,
950
- compression: snapshotArtifactSelection.compression,
951
- features: snapshotArtifactSelection.featureSet,
952
- })
953
- : null;
954
-
955
- interface SnapshotBundle {
956
- table: string;
957
- tableIndex: number;
958
- startCursor: string | null;
959
- nextRowCursor: string | null;
960
- isFirstPage: boolean;
961
- isLastPage: boolean;
962
- pageCount: number;
963
- cacheRowLimit: number | null;
964
- ttlMs: number;
965
- binaryColumns?: readonly BinarySnapshotColumn[];
966
- binaryEncoder?: BinarySnapshotRowsEncoder;
967
- binaryRows: unknown[];
968
- }
969
-
970
- const createSnapshotBundle = (
971
- table: string,
972
- tableIndex: number,
973
- rowCursor: string | null,
974
- ttlMs: number,
975
- binaryColumns?: readonly BinarySnapshotColumn[],
976
- binaryEncoder?: BinarySnapshotRowsEncoder
977
- ): SnapshotBundle => {
978
- return {
979
- table,
980
- tableIndex,
981
- startCursor: rowCursor,
982
- nextRowCursor: null,
983
- isFirstPage: rowCursor == null,
984
- isLastPage: false,
985
- pageCount: 0,
986
- cacheRowLimit: null,
987
- ttlMs,
988
- binaryColumns,
989
- binaryEncoder,
990
- binaryRows: [],
991
- };
992
- };
993
-
994
- const snapshotBootstrapStateAfter = (args: {
995
- tableIndex: number;
996
- nextRowCursor: string | null;
997
- isLastPage: boolean;
998
- }): SyncBootstrapState | null => {
999
- if (!args.isLastPage) {
1000
- return {
1001
- ...effectiveState,
1002
- tableIndex: args.tableIndex,
1003
- rowCursor: args.nextRowCursor,
1004
- };
1005
- }
1006
- if (args.tableIndex + 1 < effectiveState.tables.length) {
1007
- return {
1008
- ...effectiveState,
1009
- tableIndex: args.tableIndex + 1,
1010
- rowCursor: null,
1011
- };
1012
- }
1013
- return null;
1014
- };
1015
-
1016
- const encodeSnapshotBundlePayload = (
1017
- bundle: SnapshotBundle
1018
- ): Uint8Array[] => {
1019
- const encodeStartedAt = Date.now();
1020
- const payload = bundle.binaryEncoder
1021
- ? bundle.binaryEncoder(bundle.binaryRows)
1022
- : encodeBinarySnapshotRows(
1023
- bundle.table,
1024
- bundle.binaryRows,
1025
- bundle.binaryColumns
1026
- );
1027
- bootstrapTimings.binaryEncodeMs += Math.max(
1028
- 0,
1029
- Date.now() - encodeStartedAt
1030
- );
1031
- return [payload];
1032
- };
1033
-
1034
- const flushSnapshotBundle = async (
1035
- bundle: SnapshotBundle
1036
- ): Promise<void> => {
1037
- const nowIso = new Date().toISOString();
1038
- const bundleRowLimit = Math.max(
1039
- 1,
1040
- bundle.cacheRowLimit ??
1041
- limitSnapshotRows * bundle.pageCount
1042
- );
1043
-
1044
- const cacheLookupStartedAt = Date.now();
1045
- const cached = await readSnapshotChunkRefByPageKey(trx, {
1046
- partitionId,
1047
- scopeKey: cacheKey,
1048
- scope: bundle.table,
1049
- asOfCommitSeq: effectiveState.asOfCommitSeq,
1050
- rowCursor: bundle.startCursor,
1051
- rowLimit: bundleRowLimit,
1052
- encoding: snapshotChunkEncoding,
1053
- compression: SYNC_SNAPSHOT_CHUNK_COMPRESSION,
1054
- nowIso,
1055
- });
1056
- bootstrapTimings.chunkCacheLookupMs += Math.max(
1057
- 0,
1058
- Date.now() - cacheLookupStartedAt
1059
- );
1060
-
1061
- let chunkRef: SyncSnapshotChunkRef | null = cached;
1062
- if (!chunkRef) {
1063
- const expiresAt = new Date(
1064
- Date.now() + Math.max(1000, bundle.ttlMs)
1065
- ).toISOString();
1066
-
1067
- if (args.chunkStorage) {
1068
- const snapshot: SyncSnapshot = {
1069
- table: bundle.table,
1070
- rows: [],
1071
- chunks: [],
1072
- isFirstPage: bundle.isFirstPage,
1073
- isLastPage: bundle.isLastPage,
1074
- bootstrapStateAfter: snapshotBootstrapStateAfter({
1075
- tableIndex: bundle.tableIndex,
1076
- nextRowCursor: bundle.nextRowCursor,
1077
- isLastPage: bundle.isLastPage,
1078
- }),
1079
- };
1080
- snapshots.push(snapshot);
1081
- pendingExternalChunkWrites.push({
1082
- snapshot,
1083
- cacheLookup: {
1084
- partitionId,
1085
- scopeKey: cacheKey,
1086
- scope: bundle.table,
1087
- asOfCommitSeq: effectiveState.asOfCommitSeq,
1088
- rowCursor: bundle.startCursor,
1089
- rowLimit: bundleRowLimit,
1090
- nextRowCursor: bundle.nextRowCursor,
1091
- isLastPage: bundle.isLastPage,
1092
- },
1093
- payloadParts: encodeSnapshotBundlePayload(bundle),
1094
- expiresAt,
1095
- });
1096
- return;
1097
- }
1098
- const encodedChunk =
1099
- await encodeCompressedSnapshotChunk(
1100
- encodeSnapshotBundlePayload(bundle),
1101
- snapshotChunkGzipLevel
1102
- );
1103
- bootstrapTimings.chunkGzipMs += encodedChunk.gzipMs;
1104
- bootstrapTimings.chunkHashMs += encodedChunk.hashMs;
1105
- const chunkId = randomId();
1106
- const chunkPersistStartedAt = Date.now();
1107
- chunkRef = await insertSnapshotChunk(trx, {
1108
- chunkId,
1109
- partitionId,
1110
- scopeKey: cacheKey,
1111
- scope: bundle.table,
1112
- asOfCommitSeq: effectiveState.asOfCommitSeq,
1113
- rowCursor: bundle.startCursor,
1114
- rowLimit: bundleRowLimit,
1115
- nextRowCursor: bundle.nextRowCursor,
1116
- isLastPage: bundle.isLastPage,
1117
- encoding: snapshotChunkEncoding,
1118
- compression: SYNC_SNAPSHOT_CHUNK_COMPRESSION,
1119
- sha256: encodedChunk.sha256,
1120
- body: encodedChunk.body,
1121
- expiresAt,
1122
- });
1123
- bootstrapTimings.chunkPersistMs += Math.max(
1124
- 0,
1125
- Date.now() - chunkPersistStartedAt
1126
- );
1127
- }
1128
-
1129
- const chunk = toResponseChunkRef(chunkRef);
1130
- snapshots.push({
1131
- table: bundle.table,
1132
- rows: [],
1133
- chunks: [chunk],
1134
- manifest: await createChunkedSnapshotManifest({
1135
- table: bundle.table,
1136
- asOfCommitSeq: effectiveState.asOfCommitSeq,
1137
- scopeKey: cacheKey,
1138
- rowCursor: bundle.startCursor,
1139
- rowLimit: bundleRowLimit,
1140
- nextRowCursor: bundle.nextRowCursor,
1141
- isFirstPage: bundle.isFirstPage,
1142
- isLastPage: bundle.isLastPage,
1143
- chunks: [chunk],
1144
- }),
1145
- isFirstPage: bundle.isFirstPage,
1146
- isLastPage: bundle.isLastPage,
1147
- bootstrapStateAfter: snapshotBootstrapStateAfter({
1148
- tableIndex: bundle.tableIndex,
1149
- nextRowCursor: bundle.nextRowCursor,
1150
- isLastPage: bundle.isLastPage,
1151
- }),
1152
- });
1153
- };
1154
-
1155
- let activeBundle: SnapshotBundle | null = null;
1156
-
1157
- for (
1158
- let pageIndex = 0;
1159
- pageIndex < maxSnapshotPages;
1160
- pageIndex++
1161
- ) {
1162
- if (!nextState) break;
1163
-
1164
- const nextTableName: string | undefined =
1165
- nextState.tables[nextState.tableIndex];
1166
- if (!nextTableName) {
1167
- if (activeBundle) {
1168
- activeBundle.isLastPage = true;
1169
- await flushSnapshotBundle(activeBundle);
1170
- activeBundle = null;
1171
- }
1172
- nextState = null;
1173
- break;
1174
- }
1175
-
1176
- const tableHandler =
1177
- args.handlers.byTable.get(nextTableName);
1178
- if (!tableHandler) {
1179
- throw new Error(`Unknown table: ${nextTableName}`);
1180
- }
1181
- if (
1182
- !activeBundle ||
1183
- activeBundle.table !== nextTableName
1184
- ) {
1185
- if (activeBundle) {
1186
- await flushSnapshotBundle(activeBundle);
1187
- }
1188
- activeBundle = createSnapshotBundle(
1189
- nextTableName,
1190
- nextState.tableIndex,
1191
- nextState.rowCursor,
1192
- tableHandler.snapshotChunkTtlMs ??
1193
- 24 * 60 * 60 * 1000,
1194
- resolveSnapshotBinaryColumns(
1195
- tableHandler,
1196
- clientSchemaVersion
1197
- ),
1198
- resolveSnapshotBinaryEncoder(
1199
- tableHandler,
1200
- clientSchemaVersion
1201
- )
1202
- );
1203
- }
1204
-
1205
- if (artifactScopeKey && activeBundle.pageCount === 0) {
1206
- const pagesRemaining = Math.max(
1207
- 1,
1208
- maxSnapshotPages - pageIndex
1209
- );
1210
- const artifactRowLimit =
1211
- resolveBinarySnapshotBundleRowLimit({
1212
- limitSnapshotRows,
1213
- pagesRemaining,
1214
- });
1215
- const artifactLookupStartedAt = Date.now();
1216
- const artifact =
1217
- await readBestScopedSnapshotArtifactRefForPageCapacity(
1218
- trx,
1219
- {
1220
- partitionId,
1221
- scopeKey: artifactScopeKey,
1222
- subscriptionId: sub.id,
1223
- table: nextTableName,
1224
- asOfCommitSeq: effectiveState.asOfCommitSeq,
1225
- rowCursor: nextState.rowCursor,
1226
- maxRowLimit: artifactRowLimit,
1227
- artifactKind:
1228
- snapshotArtifactSelection!.artifactKind,
1229
- schemaVersion: snapshotArtifactSchemaVersion!,
1230
- compression:
1231
- snapshotArtifactSelection!.compression,
1232
- }
1233
- );
1234
- bootstrapTimings.artifactCacheLookupMs += Math.max(
1235
- 0,
1236
- Date.now() - artifactLookupStartedAt
1237
- );
1238
-
1239
- if (
1240
- artifact &&
1241
- (artifact.isLastPage ||
1242
- artifact.nextRowCursor !== null)
1243
- ) {
1244
- snapshots.push({
1245
- table: nextTableName,
1246
- rows: [],
1247
- artifacts: [artifact],
1248
- isFirstPage: artifact.isFirstPage,
1249
- isLastPage: artifact.isLastPage,
1250
- bootstrapStateAfter: snapshotBootstrapStateAfter({
1251
- tableIndex: nextState.tableIndex,
1252
- nextRowCursor: artifact.nextRowCursor,
1253
- isLastPage: artifact.isLastPage,
1254
- }),
1255
- });
1256
- activeBundle = null;
1257
- const selectedArtifactRowLimit =
1258
- artifact.manifest.rowLimit;
1259
- pageIndex +=
1260
- Math.max(
1261
- 1,
1262
- Math.ceil(
1263
- selectedArtifactRowLimit / limitSnapshotRows
1264
- )
1265
- ) - 1;
1266
-
1267
- nextState = snapshotBootstrapStateAfter({
1268
- tableIndex: nextState.tableIndex,
1269
- nextRowCursor: artifact.nextRowCursor,
1270
- isLastPage: artifact.isLastPage,
1271
- });
1272
- continue;
1273
- }
1274
- }
1275
-
1276
- if (activeBundle.pageCount === 0) {
1277
- const pagesRemaining = Math.max(
1278
- 1,
1279
- maxSnapshotPages - pageIndex
1280
- );
1281
- const cachedRowLimit =
1282
- resolveBinarySnapshotBundleRowLimit({
1283
- limitSnapshotRows,
1284
- pagesRemaining,
1285
- });
1286
- activeBundle.cacheRowLimit = cachedRowLimit;
1287
- const cacheLookupStartedAt = Date.now();
1288
- const cached: SnapshotChunkRefWithContinuation | null =
1289
- await readSnapshotChunkRefByPageKey(trx, {
1290
- partitionId,
1291
- scopeKey: cacheKey,
1292
- scope: nextTableName,
1293
- asOfCommitSeq: effectiveState.asOfCommitSeq,
1294
- rowCursor: nextState.rowCursor,
1295
- rowLimit: cachedRowLimit,
1296
- encoding: snapshotChunkEncoding,
1297
- compression: SYNC_SNAPSHOT_CHUNK_COMPRESSION,
1298
- });
1299
- bootstrapTimings.chunkCacheLookupMs += Math.max(
1300
- 0,
1301
- Date.now() - cacheLookupStartedAt
1302
- );
1303
-
1304
- if (
1305
- cached &&
1306
- (cached.isLastPage || cached.nextRowCursor !== null)
1307
- ) {
1308
- const chunk = toResponseChunkRef(cached);
1309
- snapshots.push({
1310
- table: nextTableName,
1311
- rows: [],
1312
- chunks: [chunk],
1313
- manifest: await createChunkedSnapshotManifest({
1314
- table: nextTableName,
1315
- asOfCommitSeq: effectiveState.asOfCommitSeq,
1316
- scopeKey: cacheKey,
1317
- rowCursor: nextState.rowCursor,
1318
- rowLimit: cachedRowLimit,
1319
- nextRowCursor: cached.nextRowCursor,
1320
- isFirstPage: nextState.rowCursor == null,
1321
- isLastPage: cached.isLastPage,
1322
- chunks: [chunk],
1323
- }),
1324
- isFirstPage: nextState.rowCursor == null,
1325
- isLastPage: cached.isLastPage,
1326
- bootstrapStateAfter: snapshotBootstrapStateAfter({
1327
- tableIndex: nextState.tableIndex,
1328
- nextRowCursor: cached.nextRowCursor,
1329
- isLastPage: cached.isLastPage,
1330
- }),
1331
- });
1332
- activeBundle = null;
1333
- pageIndex +=
1334
- Math.max(
1335
- 1,
1336
- Math.ceil(cachedRowLimit / limitSnapshotRows)
1337
- ) - 1;
1338
-
1339
- nextState = snapshotBootstrapStateAfter({
1340
- tableIndex: nextState.tableIndex,
1341
- nextRowCursor: cached.nextRowCursor,
1342
- isLastPage: cached.isLastPage,
1343
- });
1344
- continue;
1345
- }
1346
- }
1347
-
1348
- const snapshotQueryStartedAt = Date.now();
1349
- const page: {
1350
- rows: unknown[];
1351
- nextCursor: string | null;
1352
- } = await tableHandler.snapshot(
1353
- {
1354
- db: trx,
1355
- actorId: args.auth.actorId,
1356
- auth: args.auth,
1357
- scopeValues: effectiveScopes,
1358
- cursor: nextState.rowCursor,
1359
- limit: limitSnapshotRows,
1360
- schemaVersion: clientSchemaVersion,
1361
- },
1362
- sub.params
1363
- );
1364
- bootstrapTimings.snapshotQueryMs += Math.max(
1365
- 0,
1366
- Date.now() - snapshotQueryStartedAt
1367
- );
1368
-
1369
- const pageRows = page.rows ?? [];
1370
- activeBundle.nextRowCursor = page.nextCursor;
1371
- activeBundle.binaryRows.push(...pageRows);
1372
- activeBundle.pageCount += 1;
1373
-
1374
- if (page.nextCursor != null) {
1375
- const shouldFlushBinaryBundle =
1376
- activeBundle.binaryRows.length >=
1377
- DEFAULT_MAX_BINARY_SNAPSHOT_BUNDLE_ROWS;
1378
- if (shouldFlushBinaryBundle) {
1379
- await flushSnapshotBundle(activeBundle);
1380
- activeBundle = null;
1381
- }
1382
- nextState = {
1383
- ...nextState,
1384
- rowCursor: page.nextCursor,
1385
- };
1386
- continue;
1387
- }
1388
-
1389
- activeBundle.isLastPage = true;
1390
- await flushSnapshotBundle(activeBundle);
1391
- activeBundle = null;
1392
-
1393
- if (nextState.tableIndex + 1 < nextState.tables.length) {
1394
- nextState = {
1395
- ...nextState,
1396
- tableIndex: nextState.tableIndex + 1,
1397
- rowCursor: null,
1398
- };
1399
- continue;
1400
- }
1401
-
1402
- nextState = null;
1403
- break;
1404
- }
1405
-
1406
- if (activeBundle) {
1407
- await flushSnapshotBundle(activeBundle);
1408
- }
1409
-
1410
- subResponses.push({
1411
- id: sub.id,
1412
- status: 'active',
1413
- scopes: effectiveScopes,
1414
- bootstrap: true,
1415
- bootstrapState: nextState,
1416
- nextCursor: effectiveState.asOfCommitSeq,
1417
- commits: [],
1418
- snapshots,
1419
- });
1420
- nextCursors.push(effectiveState.asOfCommitSeq);
1421
- continue;
1422
- }
1423
-
1424
- // Incremental pull for this subscription. The dialect row query
1425
- // carries the scanned commit-window max when matching rows exist,
1426
- // so we only need a separate commit-window scan when the row query
1427
- // returns no matches at all.
1428
- const incrementalRows: IncrementalPullRow[] = [];
1429
- let maxScannedCommitSeq = cursor;
1430
-
1431
- for await (const row of dialect.iterateIncrementalPullRows(
1432
- trx,
1433
- {
1434
- partitionId,
1435
- table: sub.table,
1436
- scopes: effectiveScopes,
1437
- cursor,
1438
- limitCommits,
1439
- }
1440
- )) {
1441
- incrementalRows.push(row);
1442
- maxScannedCommitSeq = Math.max(
1443
- maxScannedCommitSeq,
1444
- row.scanned_max_commit_seq ?? row.commit_seq
1445
- );
1446
- }
1447
-
1448
- if (incrementalRows.length === 0) {
1449
- const scannedCommitSeqs =
1450
- await dialect.readCommitSeqsForPull(trx, {
1451
- partitionId,
1452
- cursor,
1453
- limitCommits,
1454
- tables: [sub.table],
1455
- });
1456
- maxScannedCommitSeq =
1457
- scannedCommitSeqs.length > 0
1458
- ? scannedCommitSeqs[scannedCommitSeqs.length - 1]!
1459
- : cursor;
1460
-
1461
- if (scannedCommitSeqs.length === 0) {
1462
- subResponses.push({
1463
- id: sub.id,
1464
- status: 'active',
1465
- scopes: effectiveScopes,
1466
- bootstrap: false,
1467
- nextCursor: cursor,
1468
- commits: [],
1469
- });
1470
- nextCursors.push(cursor);
1471
- continue;
1472
- }
1473
- }
1474
-
1475
- const tableHandler = args.handlers.byTable.get(sub.table);
1476
- if (!tableHandler) {
1477
- throw new Error(`Unknown table: ${sub.table}`);
1478
- }
1479
-
1480
- const incrementalItems = incrementalRows.map((r) => ({
1481
- commitSeq: r.commit_seq,
1482
- createdAt: r.created_at,
1483
- actorId: r.actor_id,
1484
- change: projectPullChangeForClientSchema({
1485
- handler: tableHandler,
1486
- schemaVersion: clientSchemaVersion,
1487
- change: {
1488
- table: r.table,
1489
- row_id: r.row_id,
1490
- op: r.op,
1491
- row_json: r.row_json,
1492
- row_version: r.row_version,
1493
- scopes: r.scopes,
1494
- } satisfies SyncChange,
1495
- }),
1496
- }));
1497
-
1498
- if (pullPlugins.length > 0 && incrementalItems.length > 0) {
1499
- const transformedChanges = await transformPullChanges({
1500
- plugins: pullPlugins,
1501
- ctx: {
1502
- db: trx,
1503
- actorId: args.auth.actorId,
1504
- auth: args.auth,
1505
- },
1506
- tableHandler,
1507
- subscription: {
1508
- id: sub.id,
1509
- table: sub.table,
1510
- scopes: effectiveScopes,
1511
- params: sub.params,
1512
- cursor,
1513
- crdtStateVectors: sub.crdtStateVectors,
1514
- },
1515
- changes: incrementalItems.map((item) => item.change),
1516
- });
1517
- for (let i = 0; i < incrementalItems.length; i += 1) {
1518
- incrementalItems[i]!.change = transformedChanges[i]!;
1519
- }
1520
- }
1521
-
1522
- let nextCursor = cursor;
1523
-
1524
- if (dedupeRows) {
1525
- const latestByRowKey = new Map<
1526
- string,
1527
- {
1528
- commitSeq: number;
1529
- createdAt: string;
1530
- actorId: string;
1531
- change: SyncChange;
1532
- }
1533
- >();
1534
-
1535
- for (const item of incrementalItems) {
1536
- nextCursor = Math.max(nextCursor, item.commitSeq);
1537
- const rowKey = `${item.change.table}\u0000${item.change.row_id}`;
1538
-
1539
- // Move row keys to insertion tail so Map iteration yields
1540
- // "latest change wins" order without a full array sort.
1541
- if (latestByRowKey.has(rowKey)) {
1542
- latestByRowKey.delete(rowKey);
1543
- }
1544
- latestByRowKey.set(rowKey, {
1545
- commitSeq: item.commitSeq,
1546
- createdAt: item.createdAt,
1547
- actorId: item.actorId,
1548
- change: item.change,
1549
- });
1550
- }
1551
-
1552
- nextCursor = Math.max(nextCursor, maxScannedCommitSeq);
1553
-
1554
- if (latestByRowKey.size === 0) {
1555
- subResponses.push({
1556
- id: sub.id,
1557
- status: 'active',
1558
- scopes: effectiveScopes,
1559
- bootstrap: false,
1560
- nextCursor,
1561
- commits: [],
1562
- });
1563
- nextCursors.push(nextCursor);
1564
- continue;
1565
- }
1566
-
1567
- const commits: SyncCommit[] = [];
1568
- for (const item of latestByRowKey.values()) {
1569
- const lastCommit = commits[commits.length - 1];
1570
- if (
1571
- !lastCommit ||
1572
- lastCommit.commitSeq !== item.commitSeq
1573
- ) {
1574
- commits.push({
1575
- commitSeq: item.commitSeq,
1576
- createdAt: item.createdAt,
1577
- actorId: item.actorId,
1578
- changes: [item.change],
1579
- });
1580
- continue;
1581
- }
1582
- lastCommit.changes.push(item.change);
1583
- }
1584
- const integrity = await createWireSubscriptionIntegrity({
1585
- partitionId,
1586
- subscriptionId: sub.id,
1587
- previousRoot:
1588
- typeof sub.verifiedRoot === 'string'
1589
- ? sub.verifiedRoot
1590
- : SYNCULAR_COMMIT_GENESIS_ROOT,
1591
- commits,
1592
- });
1593
-
1594
- subResponses.push({
1595
- id: sub.id,
1596
- status: 'active',
1597
- scopes: effectiveScopes,
1598
- bootstrap: false,
1599
- nextCursor,
1600
- ...(integrity ? { integrity } : {}),
1601
- commits,
1602
- });
1603
- nextCursors.push(nextCursor);
1604
- continue;
1605
- }
1606
-
1607
- const commits: SyncCommit[] = [];
1608
-
1609
- for (const item of incrementalItems) {
1610
- nextCursor = Math.max(nextCursor, item.commitSeq);
1611
- const seq = item.commitSeq;
1612
- let commit = commits[commits.length - 1];
1613
- if (!commit || commit.commitSeq !== seq) {
1614
- commit = {
1615
- commitSeq: seq,
1616
- createdAt: item.createdAt,
1617
- actorId: item.actorId,
1618
- changes: [],
1619
- };
1620
- commits.push(commit);
1621
- }
1622
-
1623
- commit.changes.push(item.change);
1624
- }
1625
-
1626
- const integrity = await createWireSubscriptionIntegrity({
1627
- partitionId,
1628
- subscriptionId: sub.id,
1629
- previousRoot:
1630
- typeof sub.verifiedRoot === 'string'
1631
- ? sub.verifiedRoot
1632
- : SYNCULAR_COMMIT_GENESIS_ROOT,
1633
- commits,
1634
- });
1635
-
1636
- nextCursor = Math.max(nextCursor, maxScannedCommitSeq);
1637
-
1638
- if (commits.length === 0) {
1639
- subResponses.push({
1640
- id: sub.id,
1641
- status: 'active',
1642
- scopes: effectiveScopes,
1643
- bootstrap: false,
1644
- nextCursor,
1645
- commits: [],
1646
- });
1647
- nextCursors.push(nextCursor);
1648
- continue;
1649
- }
1650
-
1651
- subResponses.push({
1652
- id: sub.id,
1653
- status: 'active',
1654
- scopes: effectiveScopes,
1655
- bootstrap: false,
1656
- nextCursor,
1657
- ...(integrity ? { integrity } : {}),
1658
- commits,
1659
- });
1660
- nextCursors.push(nextCursor);
1661
- }
1662
-
1663
- const effectiveScopes = mergeScopes(activeSubscriptions);
1664
- const clientCursor =
1665
- nextCursors.length > 0
1666
- ? Math.min(...nextCursors)
1667
- : maxCommitSeq;
1668
-
1669
- return {
1670
- response: {
1671
- ok: true as const,
1672
- subscriptions: subResponses,
1673
- },
1674
- effectiveScopes,
1675
- clientCursor,
1676
- };
1677
- }
1678
- );
1679
-
1680
- const chunkStorage = args.chunkStorage;
1681
- if (chunkStorage && pendingExternalChunkWrites.length > 0) {
1682
- await runWithConcurrency(
1683
- pendingExternalChunkWrites,
1684
- 4,
1685
- async (pending) => {
1686
- const cacheLookupStartedAt = Date.now();
1687
- let chunkRef: SyncSnapshotChunkRef | null =
1688
- await readSnapshotChunkRefByPageKey(db, {
1689
- partitionId: pending.cacheLookup.partitionId,
1690
- scopeKey: pending.cacheLookup.scopeKey,
1691
- scope: pending.cacheLookup.scope,
1692
- asOfCommitSeq: pending.cacheLookup.asOfCommitSeq,
1693
- rowCursor: pending.cacheLookup.rowCursor,
1694
- rowLimit: pending.cacheLookup.rowLimit,
1695
- encoding: snapshotChunkEncoding,
1696
- compression: SYNC_SNAPSHOT_CHUNK_COMPRESSION,
1697
- });
1698
- bootstrapTimings.chunkCacheLookupMs += Math.max(
1699
- 0,
1700
- Date.now() - cacheLookupStartedAt
1701
- );
1702
-
1703
- if (!chunkRef) {
1704
- if (chunkStorage.storeChunkStream) {
1705
- const {
1706
- stream: bodyStream,
1707
- byteLength,
1708
- sha256,
1709
- gzipMs,
1710
- hashMs,
1711
- } = await encodeCompressedSnapshotChunkToStream(
1712
- pending.payloadParts,
1713
- snapshotChunkGzipLevel
1714
- );
1715
- bootstrapTimings.chunkGzipMs += gzipMs;
1716
- bootstrapTimings.chunkHashMs += hashMs;
1717
- const chunkPersistStartedAt = Date.now();
1718
- chunkRef = await chunkStorage.storeChunkStream({
1719
- partitionId: pending.cacheLookup.partitionId,
1720
- scopeKey: pending.cacheLookup.scopeKey,
1721
- scope: pending.cacheLookup.scope,
1722
- asOfCommitSeq: pending.cacheLookup.asOfCommitSeq,
1723
- rowCursor: pending.cacheLookup.rowCursor,
1724
- rowLimit: pending.cacheLookup.rowLimit,
1725
- nextRowCursor: pending.cacheLookup.nextRowCursor,
1726
- isLastPage: pending.cacheLookup.isLastPage,
1727
- encoding: snapshotChunkEncoding,
1728
- compression: SYNC_SNAPSHOT_CHUNK_COMPRESSION,
1729
- sha256,
1730
- byteLength,
1731
- bodyStream,
1732
- expiresAt: pending.expiresAt,
1733
- });
1734
- bootstrapTimings.chunkPersistMs += Math.max(
1735
- 0,
1736
- Date.now() - chunkPersistStartedAt
1737
- );
1738
- } else {
1739
- const encodedChunk = await encodeCompressedSnapshotChunk(
1740
- pending.payloadParts,
1741
- snapshotChunkGzipLevel
1742
- );
1743
- bootstrapTimings.chunkGzipMs += encodedChunk.gzipMs;
1744
- bootstrapTimings.chunkHashMs += encodedChunk.hashMs;
1745
- const chunkPersistStartedAt = Date.now();
1746
- chunkRef = await chunkStorage.storeChunk({
1747
- partitionId: pending.cacheLookup.partitionId,
1748
- scopeKey: pending.cacheLookup.scopeKey,
1749
- scope: pending.cacheLookup.scope,
1750
- asOfCommitSeq: pending.cacheLookup.asOfCommitSeq,
1751
- rowCursor: pending.cacheLookup.rowCursor,
1752
- rowLimit: pending.cacheLookup.rowLimit,
1753
- nextRowCursor: pending.cacheLookup.nextRowCursor,
1754
- isLastPage: pending.cacheLookup.isLastPage,
1755
- encoding: snapshotChunkEncoding,
1756
- compression: SYNC_SNAPSHOT_CHUNK_COMPRESSION,
1757
- sha256: encodedChunk.sha256,
1758
- body: encodedChunk.body,
1759
- expiresAt: pending.expiresAt,
1760
- });
1761
- bootstrapTimings.chunkPersistMs += Math.max(
1762
- 0,
1763
- Date.now() - chunkPersistStartedAt
1764
- );
1765
- }
1766
- }
1767
-
1768
- const chunk = toResponseChunkRef(chunkRef);
1769
- pending.snapshot.chunks = [chunk];
1770
- pending.snapshot.manifest =
1771
- await createChunkedSnapshotManifest({
1772
- table: pending.snapshot.table,
1773
- asOfCommitSeq: pending.cacheLookup.asOfCommitSeq,
1774
- scopeKey: pending.cacheLookup.scopeKey,
1775
- rowCursor: pending.cacheLookup.rowCursor,
1776
- rowLimit: pending.cacheLookup.rowLimit,
1777
- nextRowCursor: pending.cacheLookup.nextRowCursor,
1778
- isFirstPage: pending.snapshot.isFirstPage,
1779
- isLastPage: pending.cacheLookup.isLastPage,
1780
- chunks: [chunk],
1781
- });
1782
- }
1783
- );
1784
- }
1785
-
1786
- const durationMs = Math.max(0, Date.now() - startedAtMs);
1787
- const stats = summarizePullResponse(result.response);
1788
-
1789
- span.setAttribute('status', 'ok');
1790
- span.setAttribute('duration_ms', durationMs);
1791
- span.setAttribute('subscription_count', stats.subscriptionCount);
1792
- span.setAttribute('commit_count', stats.commitCount);
1793
- span.setAttribute('change_count', stats.changeCount);
1794
- span.setAttribute('snapshot_page_count', stats.snapshotPageCount);
1795
- span.setAttributes({
1796
- bootstrap_snapshot_query_ms: bootstrapTimings.snapshotQueryMs,
1797
- bootstrap_snapshot_binary_encode_ms:
1798
- bootstrapTimings.binaryEncodeMs,
1799
- bootstrap_chunk_cache_lookup_ms:
1800
- bootstrapTimings.chunkCacheLookupMs,
1801
- bootstrap_artifact_cache_lookup_ms:
1802
- bootstrapTimings.artifactCacheLookupMs,
1803
- bootstrap_chunk_gzip_ms: bootstrapTimings.chunkGzipMs,
1804
- bootstrap_chunk_hash_ms: bootstrapTimings.chunkHashMs,
1805
- bootstrap_chunk_persist_ms: bootstrapTimings.chunkPersistMs,
1806
- });
1807
- span.setStatus('ok');
1808
-
1809
- recordPullMetrics({
1810
- status: 'ok',
1811
- dedupeRows,
1812
- durationMs,
1813
- stats,
1814
- });
1815
-
1816
- return {
1817
- ...result,
1818
- bootstrapTimings,
1819
- };
1820
- } catch (error) {
1821
- if (
1822
- error instanceof Error &&
1823
- attemptIndex < MAX_PULL_TRANSACTION_RETRIES - 1 &&
1824
- isSerializablePullError(error)
1825
- ) {
1826
- await delay(PULL_TRANSACTION_RETRY_DELAY_MS * (attemptIndex + 1));
1827
- continue;
1828
- }
1829
- throw error;
1830
- }
1831
- }
1832
-
1833
- throw new Error('Pull transaction retry loop exhausted unexpectedly');
1834
- } catch (error) {
1835
- const durationMs = Math.max(0, Date.now() - startedAtMs);
1836
-
1837
- span.setAttribute('status', 'error');
1838
- span.setAttribute('duration_ms', durationMs);
1839
- span.setStatus('error');
1840
-
1841
- recordPullMetrics({
1842
- status: 'error',
1843
- dedupeRows: request.dedupeRows === true,
1844
- durationMs,
1845
- stats: {
1846
- subscriptionCount: 0,
1847
- activeSubscriptionCount: 0,
1848
- revokedSubscriptionCount: 0,
1849
- bootstrapSubscriptionCount: 0,
1850
- commitCount: 0,
1851
- changeCount: 0,
1852
- snapshotPageCount: 0,
1853
- },
1854
- });
1855
-
1856
- captureSyncException(error, {
1857
- event: 'sync.server.pull',
1858
- requestedSubscriptionCount,
1859
- dedupeRows: request.dedupeRows === true,
1860
- });
1861
- throw error;
1862
- }
535
+ let delivered = 0;
536
+ let deliveredCommits = 0;
537
+ let lastDeliveredSeq = sub.cursor;
538
+ for (const commit of commits) {
539
+ if (
540
+ delivered > 0 &&
541
+ delivered + commit.changes.length > limits.limitCommits
542
+ ) {
543
+ break;
1863
544
  }
1864
- );
545
+ yield commitFrame(sub.table, commit);
546
+ delivered += commit.changes.length;
547
+ deliveredCommits += 1;
548
+ lastDeliveredSeq = commit.commitSeq;
549
+ if (delivered >= limits.limitCommits) break;
550
+ }
551
+ const totalReturned = commits.reduce((n, c) => n + c.changes.length, 0);
552
+ // The window is proven exhausted only when the storage lookahead came
553
+ // back under budget (it scanned to `throughSeq`) and every returned
554
+ // commit was delivered.
555
+ const exhausted =
556
+ totalReturned <= limits.limitCommits && deliveredCommits === commits.length;
557
+ // §4.5: the cursor advances even when no matching changes exist; when
558
+ // the change limit truncated the window it stops at the last fully
559
+ // delivered commit.
560
+ const nextCursor = exhausted
561
+ ? Math.max(sub.cursor, maxSeq)
562
+ : Math.max(sub.cursor, lastDeliveredSeq);
563
+ yield { type: 'SUB_END', nextCursor };
564
+ return { nextCursor, active: true };
1865
565
  }