@livestore/common 0.4.0-dev.9 → 0.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (513) hide show
  1. package/dist/.tsbuildinfo +1 -1
  2. package/dist/ClientSessionLeaderThreadProxy.d.ts +27 -12
  3. package/dist/ClientSessionLeaderThreadProxy.d.ts.map +1 -1
  4. package/dist/ClientSessionLeaderThreadProxy.js.map +1 -1
  5. package/dist/WorkerTransportError.d.ts +11 -0
  6. package/dist/WorkerTransportError.d.ts.map +1 -0
  7. package/dist/WorkerTransportError.js +11 -0
  8. package/dist/WorkerTransportError.js.map +1 -0
  9. package/dist/adapter-types.d.ts +37 -7
  10. package/dist/adapter-types.d.ts.map +1 -1
  11. package/dist/adapter-types.js +27 -1
  12. package/dist/adapter-types.js.map +1 -1
  13. package/dist/bounded-collections.d.ts.map +1 -1
  14. package/dist/bounded-collections.js +6 -4
  15. package/dist/bounded-collections.js.map +1 -1
  16. package/dist/debug-info.d.ts.map +1 -1
  17. package/dist/debug-info.js +33 -6
  18. package/dist/debug-info.js.map +1 -1
  19. package/dist/devtools/devtools-compatibility.test.d.ts +2 -0
  20. package/dist/devtools/devtools-compatibility.test.d.ts.map +1 -0
  21. package/dist/devtools/devtools-compatibility.test.js +15 -0
  22. package/dist/devtools/devtools-compatibility.test.js.map +1 -0
  23. package/dist/devtools/devtools-messages-client-session.d.ts +55 -24
  24. package/dist/devtools/devtools-messages-client-session.d.ts.map +1 -1
  25. package/dist/devtools/devtools-messages-client-session.js +22 -5
  26. package/dist/devtools/devtools-messages-client-session.js.map +1 -1
  27. package/dist/devtools/devtools-messages-common.d.ts +11 -14
  28. package/dist/devtools/devtools-messages-common.d.ts.map +1 -1
  29. package/dist/devtools/devtools-messages-common.js +7 -9
  30. package/dist/devtools/devtools-messages-common.js.map +1 -1
  31. package/dist/devtools/devtools-messages-leader.d.ts +65 -30
  32. package/dist/devtools/devtools-messages-leader.d.ts.map +1 -1
  33. package/dist/devtools/devtools-messages-leader.js +29 -11
  34. package/dist/devtools/devtools-messages-leader.js.map +1 -1
  35. package/dist/devtools/devtools-sessioninfo.d.ts +14 -2
  36. package/dist/devtools/devtools-sessioninfo.d.ts.map +1 -1
  37. package/dist/devtools/devtools-sessioninfo.js +7 -4
  38. package/dist/devtools/devtools-sessioninfo.js.map +1 -1
  39. package/dist/devtools/mod.d.ts +13 -2
  40. package/dist/devtools/mod.d.ts.map +1 -1
  41. package/dist/devtools/mod.js +10 -3
  42. package/dist/devtools/mod.js.map +1 -1
  43. package/dist/errors.d.ts +48 -18
  44. package/dist/errors.d.ts.map +1 -1
  45. package/dist/errors.js +20 -12
  46. package/dist/errors.js.map +1 -1
  47. package/dist/index.d.ts +4 -1
  48. package/dist/index.d.ts.map +1 -1
  49. package/dist/index.js +4 -1
  50. package/dist/index.js.map +1 -1
  51. package/dist/leader-thread/LeaderSyncProcessor.d.ts +53 -6
  52. package/dist/leader-thread/LeaderSyncProcessor.d.ts.map +1 -1
  53. package/dist/leader-thread/LeaderSyncProcessor.js +325 -257
  54. package/dist/leader-thread/LeaderSyncProcessor.js.map +1 -1
  55. package/dist/leader-thread/RejectedPushError.d.ts +107 -0
  56. package/dist/leader-thread/RejectedPushError.d.ts.map +1 -0
  57. package/dist/leader-thread/RejectedPushError.js +78 -0
  58. package/dist/leader-thread/RejectedPushError.js.map +1 -0
  59. package/dist/leader-thread/connection.js +1 -1
  60. package/dist/leader-thread/connection.js.map +1 -1
  61. package/dist/leader-thread/eventlog.d.ts +19 -14
  62. package/dist/leader-thread/eventlog.d.ts.map +1 -1
  63. package/dist/leader-thread/eventlog.js +78 -18
  64. package/dist/leader-thread/eventlog.js.map +1 -1
  65. package/dist/leader-thread/leader-worker-devtools.d.ts +1 -2
  66. package/dist/leader-thread/leader-worker-devtools.d.ts.map +1 -1
  67. package/dist/leader-thread/leader-worker-devtools.js +90 -58
  68. package/dist/leader-thread/leader-worker-devtools.js.map +1 -1
  69. package/dist/leader-thread/make-leader-thread-layer.d.ts +15 -7
  70. package/dist/leader-thread/make-leader-thread-layer.d.ts.map +1 -1
  71. package/dist/leader-thread/make-leader-thread-layer.js +49 -17
  72. package/dist/leader-thread/make-leader-thread-layer.js.map +1 -1
  73. package/dist/leader-thread/make-leader-thread-layer.test.d.ts +2 -0
  74. package/dist/leader-thread/make-leader-thread-layer.test.d.ts.map +1 -0
  75. package/dist/leader-thread/make-leader-thread-layer.test.js +32 -0
  76. package/dist/leader-thread/make-leader-thread-layer.test.js.map +1 -0
  77. package/dist/leader-thread/materialize-event.d.ts +1 -1
  78. package/dist/leader-thread/materialize-event.d.ts.map +1 -1
  79. package/dist/leader-thread/materialize-event.js +28 -9
  80. package/dist/leader-thread/materialize-event.js.map +1 -1
  81. package/dist/leader-thread/mod.d.ts +1 -0
  82. package/dist/leader-thread/mod.d.ts.map +1 -1
  83. package/dist/leader-thread/mod.js +1 -0
  84. package/dist/leader-thread/mod.js.map +1 -1
  85. package/dist/leader-thread/recreate-db.d.ts +2 -2
  86. package/dist/leader-thread/recreate-db.d.ts.map +1 -1
  87. package/dist/leader-thread/recreate-db.js +6 -6
  88. package/dist/leader-thread/recreate-db.js.map +1 -1
  89. package/dist/leader-thread/shutdown-channel.d.ts +2 -2
  90. package/dist/leader-thread/shutdown-channel.d.ts.map +1 -1
  91. package/dist/leader-thread/shutdown-channel.js +2 -2
  92. package/dist/leader-thread/shutdown-channel.js.map +1 -1
  93. package/dist/leader-thread/stream-events.d.ts +56 -0
  94. package/dist/leader-thread/stream-events.d.ts.map +1 -0
  95. package/dist/leader-thread/stream-events.js +167 -0
  96. package/dist/leader-thread/stream-events.js.map +1 -0
  97. package/dist/leader-thread/types.d.ts +95 -17
  98. package/dist/leader-thread/types.d.ts.map +1 -1
  99. package/dist/leader-thread/types.js +13 -0
  100. package/dist/leader-thread/types.js.map +1 -1
  101. package/dist/logging.d.ts +40 -0
  102. package/dist/logging.d.ts.map +1 -0
  103. package/dist/logging.js +33 -0
  104. package/dist/logging.js.map +1 -0
  105. package/dist/make-client-session.d.ts +5 -3
  106. package/dist/make-client-session.d.ts.map +1 -1
  107. package/dist/make-client-session.js +7 -4
  108. package/dist/make-client-session.js.map +1 -1
  109. package/dist/materializer-helper.d.ts +6 -6
  110. package/dist/materializer-helper.d.ts.map +1 -1
  111. package/dist/materializer-helper.js +18 -8
  112. package/dist/materializer-helper.js.map +1 -1
  113. package/dist/otel.d.ts +2 -1
  114. package/dist/otel.d.ts.map +1 -1
  115. package/dist/otel.js +7 -2
  116. package/dist/otel.js.map +1 -1
  117. package/dist/rematerialize-from-eventlog.d.ts +3 -3
  118. package/dist/rematerialize-from-eventlog.d.ts.map +1 -1
  119. package/dist/rematerialize-from-eventlog.js +40 -29
  120. package/dist/rematerialize-from-eventlog.js.map +1 -1
  121. package/dist/schema/EventDef/define.d.ts +161 -0
  122. package/dist/schema/EventDef/define.d.ts.map +1 -0
  123. package/dist/schema/EventDef/define.js +140 -0
  124. package/dist/schema/EventDef/define.js.map +1 -0
  125. package/dist/schema/EventDef/deprecated.d.ts +99 -0
  126. package/dist/schema/EventDef/deprecated.d.ts.map +1 -0
  127. package/dist/schema/EventDef/deprecated.js +144 -0
  128. package/dist/schema/EventDef/deprecated.js.map +1 -0
  129. package/dist/schema/EventDef/deprecated.test.d.ts +2 -0
  130. package/dist/schema/EventDef/deprecated.test.d.ts.map +1 -0
  131. package/dist/schema/EventDef/deprecated.test.js +95 -0
  132. package/dist/schema/EventDef/deprecated.test.js.map +1 -0
  133. package/dist/schema/EventDef/event-def.d.ts +110 -0
  134. package/dist/schema/EventDef/event-def.d.ts.map +1 -0
  135. package/dist/schema/EventDef/event-def.js +2 -0
  136. package/dist/schema/EventDef/event-def.js.map +1 -0
  137. package/dist/schema/EventDef/facts.d.ts +118 -0
  138. package/dist/schema/EventDef/facts.d.ts.map +1 -0
  139. package/dist/schema/EventDef/facts.js +53 -0
  140. package/dist/schema/EventDef/facts.js.map +1 -0
  141. package/dist/schema/EventDef/materializer.d.ts +155 -0
  142. package/dist/schema/EventDef/materializer.d.ts.map +1 -0
  143. package/dist/schema/EventDef/materializer.js +83 -0
  144. package/dist/schema/EventDef/materializer.js.map +1 -0
  145. package/dist/schema/EventDef/mod.d.ts +6 -0
  146. package/dist/schema/EventDef/mod.d.ts.map +1 -0
  147. package/dist/schema/EventDef/mod.js +6 -0
  148. package/dist/schema/EventDef/mod.js.map +1 -0
  149. package/dist/schema/EventSequenceNumber/client.d.ts +136 -0
  150. package/dist/schema/EventSequenceNumber/client.d.ts.map +1 -0
  151. package/dist/schema/EventSequenceNumber/client.js +193 -0
  152. package/dist/schema/EventSequenceNumber/client.js.map +1 -0
  153. package/dist/schema/EventSequenceNumber/global.d.ts +15 -0
  154. package/dist/schema/EventSequenceNumber/global.d.ts.map +1 -0
  155. package/dist/schema/EventSequenceNumber/global.js +14 -0
  156. package/dist/schema/EventSequenceNumber/global.js.map +1 -0
  157. package/dist/schema/EventSequenceNumber/mod.d.ts +37 -0
  158. package/dist/schema/EventSequenceNumber/mod.d.ts.map +1 -0
  159. package/dist/schema/EventSequenceNumber/mod.js +37 -0
  160. package/dist/schema/EventSequenceNumber/mod.js.map +1 -0
  161. package/dist/schema/EventSequenceNumber.test.js +44 -44
  162. package/dist/schema/EventSequenceNumber.test.js.map +1 -1
  163. package/dist/schema/{LiveStoreEvent.d.ts → LiveStoreEvent/client.d.ts} +102 -111
  164. package/dist/schema/LiveStoreEvent/client.d.ts.map +1 -0
  165. package/dist/schema/LiveStoreEvent/client.js +176 -0
  166. package/dist/schema/LiveStoreEvent/client.js.map +1 -0
  167. package/dist/schema/LiveStoreEvent/client.test.d.ts +2 -0
  168. package/dist/schema/LiveStoreEvent/client.test.d.ts.map +1 -0
  169. package/dist/schema/LiveStoreEvent/client.test.js +111 -0
  170. package/dist/schema/LiveStoreEvent/client.test.js.map +1 -0
  171. package/dist/schema/LiveStoreEvent/for-event-def.d.ts +52 -0
  172. package/dist/schema/LiveStoreEvent/for-event-def.d.ts.map +1 -0
  173. package/dist/schema/LiveStoreEvent/for-event-def.js +2 -0
  174. package/dist/schema/LiveStoreEvent/for-event-def.js.map +1 -0
  175. package/dist/schema/LiveStoreEvent/global.d.ts +36 -0
  176. package/dist/schema/LiveStoreEvent/global.d.ts.map +1 -0
  177. package/dist/schema/LiveStoreEvent/global.js +31 -0
  178. package/dist/schema/LiveStoreEvent/global.js.map +1 -0
  179. package/dist/schema/LiveStoreEvent/input.d.ts +46 -0
  180. package/dist/schema/LiveStoreEvent/input.d.ts.map +1 -0
  181. package/dist/schema/LiveStoreEvent/input.js +26 -0
  182. package/dist/schema/LiveStoreEvent/input.js.map +1 -0
  183. package/dist/schema/LiveStoreEvent/mod.d.ts +5 -0
  184. package/dist/schema/LiveStoreEvent/mod.d.ts.map +1 -0
  185. package/dist/schema/LiveStoreEvent/mod.js +5 -0
  186. package/dist/schema/LiveStoreEvent/mod.js.map +1 -0
  187. package/dist/schema/events.d.ts +1 -1
  188. package/dist/schema/events.d.ts.map +1 -1
  189. package/dist/schema/events.js +1 -1
  190. package/dist/schema/events.js.map +1 -1
  191. package/dist/schema/mod.d.ts +6 -4
  192. package/dist/schema/mod.d.ts.map +1 -1
  193. package/dist/schema/mod.js +5 -4
  194. package/dist/schema/mod.js.map +1 -1
  195. package/dist/schema/schema.d.ts +16 -1
  196. package/dist/schema/schema.d.ts.map +1 -1
  197. package/dist/schema/schema.js +32 -4
  198. package/dist/schema/schema.js.map +1 -1
  199. package/dist/schema/state/sqlite/client-document-def.d.ts +2 -1
  200. package/dist/schema/state/sqlite/client-document-def.d.ts.map +1 -1
  201. package/dist/schema/state/sqlite/client-document-def.js +36 -15
  202. package/dist/schema/state/sqlite/client-document-def.js.map +1 -1
  203. package/dist/schema/state/sqlite/client-document-def.test.js +121 -2
  204. package/dist/schema/state/sqlite/client-document-def.test.js.map +1 -1
  205. package/dist/schema/state/sqlite/column-annotations.d.ts.map +1 -1
  206. package/dist/schema/state/sqlite/column-annotations.js +1 -1
  207. package/dist/schema/state/sqlite/column-annotations.js.map +1 -1
  208. package/dist/schema/state/sqlite/column-annotations.test.js +2 -2
  209. package/dist/schema/state/sqlite/column-annotations.test.js.map +1 -1
  210. package/dist/schema/state/sqlite/column-def.d.ts.map +1 -1
  211. package/dist/schema/state/sqlite/column-def.js +96 -47
  212. package/dist/schema/state/sqlite/column-def.js.map +1 -1
  213. package/dist/schema/state/sqlite/column-def.test.js +51 -12
  214. package/dist/schema/state/sqlite/column-def.test.js.map +1 -1
  215. package/dist/schema/state/sqlite/column-spec.d.ts.map +1 -1
  216. package/dist/schema/state/sqlite/column-spec.js +30 -12
  217. package/dist/schema/state/sqlite/column-spec.js.map +1 -1
  218. package/dist/schema/state/sqlite/column-spec.test.js +24 -15
  219. package/dist/schema/state/sqlite/column-spec.test.js.map +1 -1
  220. package/dist/schema/state/sqlite/db-schema/ast/sqlite.js +2 -2
  221. package/dist/schema/state/sqlite/db-schema/ast/sqlite.js.map +1 -1
  222. package/dist/schema/state/sqlite/db-schema/dsl/field-defs.d.ts +16 -10
  223. package/dist/schema/state/sqlite/db-schema/dsl/field-defs.d.ts.map +1 -1
  224. package/dist/schema/state/sqlite/db-schema/dsl/field-defs.js +15 -4
  225. package/dist/schema/state/sqlite/db-schema/dsl/field-defs.js.map +1 -1
  226. package/dist/schema/state/sqlite/db-schema/dsl/field-defs.test.js +1 -1
  227. package/dist/schema/state/sqlite/db-schema/dsl/field-defs.test.js.map +1 -1
  228. package/dist/schema/state/sqlite/db-schema/dsl/mod.d.ts +1 -1
  229. package/dist/schema/state/sqlite/db-schema/dsl/mod.d.ts.map +1 -1
  230. package/dist/schema/state/sqlite/db-schema/dsl/mod.js +1 -1
  231. package/dist/schema/state/sqlite/db-schema/dsl/mod.js.map +1 -1
  232. package/dist/schema/state/sqlite/mod.d.ts +2 -2
  233. package/dist/schema/state/sqlite/mod.d.ts.map +1 -1
  234. package/dist/schema/state/sqlite/mod.js +5 -7
  235. package/dist/schema/state/sqlite/mod.js.map +1 -1
  236. package/dist/schema/state/sqlite/query-builder/api.d.ts +51 -22
  237. package/dist/schema/state/sqlite/query-builder/api.d.ts.map +1 -1
  238. package/dist/schema/state/sqlite/query-builder/astToSql.d.ts.map +1 -1
  239. package/dist/schema/state/sqlite/query-builder/astToSql.js +99 -22
  240. package/dist/schema/state/sqlite/query-builder/astToSql.js.map +1 -1
  241. package/dist/schema/state/sqlite/query-builder/impl.d.ts +1 -1
  242. package/dist/schema/state/sqlite/query-builder/impl.d.ts.map +1 -1
  243. package/dist/schema/state/sqlite/query-builder/impl.js +28 -15
  244. package/dist/schema/state/sqlite/query-builder/impl.js.map +1 -1
  245. package/dist/schema/state/sqlite/query-builder/impl.test.js +231 -93
  246. package/dist/schema/state/sqlite/query-builder/impl.test.js.map +1 -1
  247. package/dist/schema/state/sqlite/schema-helpers.d.ts +2 -2
  248. package/dist/schema/state/sqlite/schema-helpers.d.ts.map +1 -1
  249. package/dist/schema/state/sqlite/schema-helpers.js +24 -14
  250. package/dist/schema/state/sqlite/schema-helpers.js.map +1 -1
  251. package/dist/schema/state/sqlite/schema-helpers.test.d.ts +2 -0
  252. package/dist/schema/state/sqlite/schema-helpers.test.d.ts.map +1 -0
  253. package/dist/schema/state/sqlite/schema-helpers.test.js +36 -0
  254. package/dist/schema/state/sqlite/schema-helpers.test.js.map +1 -0
  255. package/dist/schema/state/sqlite/{system-tables.d.ts → system-tables/eventlog-tables.d.ts} +21 -450
  256. package/dist/schema/state/sqlite/system-tables/eventlog-tables.d.ts.map +1 -0
  257. package/dist/schema/state/sqlite/system-tables/eventlog-tables.js +54 -0
  258. package/dist/schema/state/sqlite/system-tables/eventlog-tables.js.map +1 -0
  259. package/dist/schema/state/sqlite/system-tables/mod.d.ts +3 -0
  260. package/dist/schema/state/sqlite/system-tables/mod.d.ts.map +1 -0
  261. package/dist/schema/state/sqlite/system-tables/mod.js +3 -0
  262. package/dist/schema/state/sqlite/system-tables/mod.js.map +1 -0
  263. package/dist/schema/state/sqlite/system-tables/state-tables.d.ts +456 -0
  264. package/dist/schema/state/sqlite/system-tables/state-tables.d.ts.map +1 -0
  265. package/dist/schema/state/sqlite/system-tables/state-tables.js +55 -0
  266. package/dist/schema/state/sqlite/system-tables/state-tables.js.map +1 -0
  267. package/dist/schema/state/sqlite/table-def.d.ts +5 -3
  268. package/dist/schema/state/sqlite/table-def.d.ts.map +1 -1
  269. package/dist/schema/state/sqlite/table-def.js +1 -1
  270. package/dist/schema/state/sqlite/table-def.js.map +1 -1
  271. package/dist/schema/state/sqlite/table-def.test.js +92 -3
  272. package/dist/schema/state/sqlite/table-def.test.js.map +1 -1
  273. package/dist/schema/unknown-events.d.ts +47 -0
  274. package/dist/schema/unknown-events.d.ts.map +1 -0
  275. package/dist/schema/unknown-events.js +69 -0
  276. package/dist/schema/unknown-events.js.map +1 -0
  277. package/dist/schema-management/__tests__/migrations-autoincrement-quoting.test.d.ts +2 -0
  278. package/dist/schema-management/__tests__/migrations-autoincrement-quoting.test.d.ts.map +1 -0
  279. package/dist/schema-management/__tests__/migrations-autoincrement-quoting.test.js +73 -0
  280. package/dist/schema-management/__tests__/migrations-autoincrement-quoting.test.js.map +1 -0
  281. package/dist/schema-management/common.js +2 -2
  282. package/dist/schema-management/common.js.map +1 -1
  283. package/dist/schema-management/migrations.d.ts +32 -2
  284. package/dist/schema-management/migrations.d.ts.map +1 -1
  285. package/dist/schema-management/migrations.js +38 -6
  286. package/dist/schema-management/migrations.js.map +1 -1
  287. package/dist/schema-management/validate-schema.d.ts +3 -3
  288. package/dist/schema-management/validate-schema.d.ts.map +1 -1
  289. package/dist/schema-management/validate-schema.js +2 -2
  290. package/dist/schema-management/validate-schema.js.map +1 -1
  291. package/dist/sql-queries/sql-queries.d.ts.map +1 -1
  292. package/dist/sql-queries/sql-queries.js +18 -6
  293. package/dist/sql-queries/sql-queries.js.map +1 -1
  294. package/dist/sql-queries/sql-query-builder.d.ts.map +1 -1
  295. package/dist/sql-queries/sql-query-builder.js.map +1 -1
  296. package/dist/sqlite-db-helper.js +3 -3
  297. package/dist/sqlite-db-helper.js.map +1 -1
  298. package/dist/sqlite-types.d.ts +5 -5
  299. package/dist/sqlite-types.d.ts.map +1 -1
  300. package/dist/sqlite-types.js.map +1 -1
  301. package/dist/sync/ClientSessionSyncProcessor.d.ts +12 -12
  302. package/dist/sync/ClientSessionSyncProcessor.d.ts.map +1 -1
  303. package/dist/sync/ClientSessionSyncProcessor.js +99 -114
  304. package/dist/sync/ClientSessionSyncProcessor.js.map +1 -1
  305. package/dist/sync/errors.d.ts +0 -33
  306. package/dist/sync/errors.d.ts.map +1 -1
  307. package/dist/sync/errors.js +5 -22
  308. package/dist/sync/errors.js.map +1 -1
  309. package/dist/sync/index.d.ts +2 -0
  310. package/dist/sync/index.d.ts.map +1 -1
  311. package/dist/sync/index.js +2 -0
  312. package/dist/sync/index.js.map +1 -1
  313. package/dist/sync/mock-sync-backend.d.ts +10 -8
  314. package/dist/sync/mock-sync-backend.d.ts.map +1 -1
  315. package/dist/sync/mock-sync-backend.js +71 -69
  316. package/dist/sync/mock-sync-backend.js.map +1 -1
  317. package/dist/sync/next/compact-events.d.ts.map +1 -1
  318. package/dist/sync/next/compact-events.js +11 -12
  319. package/dist/sync/next/compact-events.js.map +1 -1
  320. package/dist/sync/next/facts.d.ts +5 -5
  321. package/dist/sync/next/facts.d.ts.map +1 -1
  322. package/dist/sync/next/facts.js +7 -8
  323. package/dist/sync/next/facts.js.map +1 -1
  324. package/dist/sync/next/history-dag-common.d.ts +54 -15
  325. package/dist/sync/next/history-dag-common.d.ts.map +1 -1
  326. package/dist/sync/next/history-dag-common.js +198 -9
  327. package/dist/sync/next/history-dag-common.js.map +1 -1
  328. package/dist/sync/next/history-dag.d.ts.map +1 -1
  329. package/dist/sync/next/history-dag.js +11 -11
  330. package/dist/sync/next/history-dag.js.map +1 -1
  331. package/dist/sync/next/rebase-events.d.ts +5 -5
  332. package/dist/sync/next/rebase-events.d.ts.map +1 -1
  333. package/dist/sync/next/rebase-events.js +6 -6
  334. package/dist/sync/next/rebase-events.js.map +1 -1
  335. package/dist/sync/next/test/compact-events.calculator.test.js +2 -2
  336. package/dist/sync/next/test/compact-events.calculator.test.js.map +1 -1
  337. package/dist/sync/next/test/compact-events.test.d.ts.map +1 -1
  338. package/dist/sync/next/test/compact-events.test.js +2 -2
  339. package/dist/sync/next/test/compact-events.test.js.map +1 -1
  340. package/dist/sync/next/test/event-fixtures.d.ts +2 -2
  341. package/dist/sync/next/test/event-fixtures.d.ts.map +1 -1
  342. package/dist/sync/next/test/event-fixtures.js +11 -11
  343. package/dist/sync/next/test/event-fixtures.js.map +1 -1
  344. package/dist/sync/sync-backend-kv.d.ts +3 -3
  345. package/dist/sync/sync-backend-kv.d.ts.map +1 -1
  346. package/dist/sync/sync-backend-kv.js +3 -3
  347. package/dist/sync/sync-backend-kv.js.map +1 -1
  348. package/dist/sync/sync-backend.d.ts +33 -13
  349. package/dist/sync/sync-backend.d.ts.map +1 -1
  350. package/dist/sync/sync-backend.js +38 -1
  351. package/dist/sync/sync-backend.js.map +1 -1
  352. package/dist/sync/sync.d.ts +23 -2
  353. package/dist/sync/sync.d.ts.map +1 -1
  354. package/dist/sync/syncstate.d.ts +55 -55
  355. package/dist/sync/syncstate.d.ts.map +1 -1
  356. package/dist/sync/syncstate.js +80 -98
  357. package/dist/sync/syncstate.js.map +1 -1
  358. package/dist/sync/syncstate.test.js +221 -132
  359. package/dist/sync/syncstate.test.js.map +1 -1
  360. package/dist/sync/transport-chunking.d.ts +36 -0
  361. package/dist/sync/transport-chunking.d.ts.map +1 -0
  362. package/dist/sync/transport-chunking.js +56 -0
  363. package/dist/sync/transport-chunking.js.map +1 -0
  364. package/dist/sync/validate-push-payload.d.ts +2 -2
  365. package/dist/sync/validate-push-payload.d.ts.map +1 -1
  366. package/dist/sync/validate-push-payload.js +4 -6
  367. package/dist/sync/validate-push-payload.js.map +1 -1
  368. package/dist/testing/event-factory.d.ts +3 -3
  369. package/dist/testing/event-factory.d.ts.map +1 -1
  370. package/dist/testing/event-factory.js +5 -7
  371. package/dist/testing/event-factory.js.map +1 -1
  372. package/dist/util.js +2 -2
  373. package/dist/util.js.map +1 -1
  374. package/dist/version.d.ts +24 -5
  375. package/dist/version.d.ts.map +1 -1
  376. package/dist/version.js +25 -8
  377. package/dist/version.js.map +1 -1
  378. package/package.json +67 -15
  379. package/src/ClientSessionLeaderThreadProxy.ts +27 -12
  380. package/src/WorkerTransportError.ts +12 -0
  381. package/src/adapter-types.ts +50 -7
  382. package/src/bounded-collections.ts +6 -5
  383. package/src/debug-info.ts +37 -6
  384. package/src/devtools/devtools-compatibility.test.ts +18 -0
  385. package/src/devtools/devtools-messages-client-session.ts +22 -4
  386. package/src/devtools/devtools-messages-common.ts +7 -12
  387. package/src/devtools/devtools-messages-leader.ts +29 -10
  388. package/src/devtools/devtools-sessioninfo.ts +8 -5
  389. package/src/devtools/mod.ts +11 -2
  390. package/src/errors.ts +32 -24
  391. package/src/index.ts +4 -1
  392. package/src/leader-thread/LeaderSyncProcessor.ts +523 -373
  393. package/src/leader-thread/RejectedPushError.ts +106 -0
  394. package/src/leader-thread/connection.ts +1 -1
  395. package/src/leader-thread/eventlog.ts +112 -39
  396. package/src/leader-thread/leader-worker-devtools.ts +201 -120
  397. package/src/leader-thread/make-leader-thread-layer.test.ts +44 -0
  398. package/src/leader-thread/make-leader-thread-layer.ts +125 -40
  399. package/src/leader-thread/materialize-event.ts +40 -10
  400. package/src/leader-thread/mod.ts +1 -0
  401. package/src/leader-thread/recreate-db.ts +7 -7
  402. package/src/leader-thread/shutdown-channel.ts +4 -8
  403. package/src/leader-thread/stream-events.ts +206 -0
  404. package/src/leader-thread/types.ts +68 -18
  405. package/src/logging.ts +62 -0
  406. package/src/make-client-session.ts +11 -5
  407. package/src/materializer-helper.ts +27 -16
  408. package/src/otel.ts +13 -2
  409. package/src/rematerialize-from-eventlog.ts +61 -51
  410. package/src/schema/EventDef/define.ts +217 -0
  411. package/src/schema/EventDef/deprecated.test.ts +129 -0
  412. package/src/schema/EventDef/deprecated.ts +175 -0
  413. package/src/schema/EventDef/event-def.ts +125 -0
  414. package/src/schema/EventDef/facts.ts +135 -0
  415. package/src/schema/EventDef/materializer.ts +172 -0
  416. package/src/schema/EventDef/mod.ts +5 -0
  417. package/src/schema/EventSequenceNumber/client.ts +257 -0
  418. package/src/schema/EventSequenceNumber/global.ts +19 -0
  419. package/src/schema/EventSequenceNumber/mod.ts +37 -0
  420. package/src/schema/EventSequenceNumber.test.ts +72 -53
  421. package/src/schema/LiveStoreEvent/client.test.ts +129 -0
  422. package/src/schema/LiveStoreEvent/client.ts +235 -0
  423. package/src/schema/LiveStoreEvent/for-event-def.ts +60 -0
  424. package/src/schema/LiveStoreEvent/global.ts +45 -0
  425. package/src/schema/LiveStoreEvent/input.ts +63 -0
  426. package/src/schema/LiveStoreEvent/mod.ts +4 -0
  427. package/src/schema/events.ts +1 -1
  428. package/src/schema/mod.ts +6 -4
  429. package/src/schema/schema.ts +46 -5
  430. package/src/schema/state/sqlite/client-document-def.test.ts +144 -5
  431. package/src/schema/state/sqlite/client-document-def.ts +47 -34
  432. package/src/schema/state/sqlite/column-annotations.test.ts +3 -2
  433. package/src/schema/state/sqlite/column-annotations.ts +2 -1
  434. package/src/schema/state/sqlite/column-def.test.ts +66 -12
  435. package/src/schema/state/sqlite/column-def.ts +119 -47
  436. package/src/schema/state/sqlite/column-spec.test.ts +32 -17
  437. package/src/schema/state/sqlite/column-spec.ts +37 -11
  438. package/src/schema/state/sqlite/db-schema/ast/sqlite.ts +2 -2
  439. package/src/schema/state/sqlite/db-schema/dsl/field-defs.test.ts +2 -1
  440. package/src/schema/state/sqlite/db-schema/dsl/field-defs.ts +41 -15
  441. package/src/schema/state/sqlite/db-schema/dsl/mod.ts +13 -19
  442. package/src/schema/state/sqlite/mod.ts +7 -8
  443. package/src/schema/state/sqlite/query-builder/api.ts +55 -17
  444. package/src/schema/state/sqlite/query-builder/astToSql.ts +110 -21
  445. package/src/schema/state/sqlite/query-builder/impl.test.ts +267 -93
  446. package/src/schema/state/sqlite/query-builder/impl.ts +26 -13
  447. package/src/schema/state/sqlite/schema-helpers.test.ts +44 -0
  448. package/src/schema/state/sqlite/schema-helpers.ts +30 -22
  449. package/src/schema/state/sqlite/system-tables/eventlog-tables.ts +64 -0
  450. package/src/schema/state/sqlite/system-tables/mod.ts +2 -0
  451. package/src/schema/state/sqlite/system-tables/state-tables.ts +69 -0
  452. package/src/schema/state/sqlite/table-def.test.ts +114 -3
  453. package/src/schema/state/sqlite/table-def.ts +16 -22
  454. package/src/schema/unknown-events.ts +131 -0
  455. package/src/schema-management/__tests__/migrations-autoincrement-quoting.test.ts +88 -0
  456. package/src/schema-management/common.ts +2 -2
  457. package/src/schema-management/migrations.ts +42 -9
  458. package/src/schema-management/validate-schema.ts +3 -3
  459. package/src/sql-queries/sql-queries.ts +18 -6
  460. package/src/sql-queries/sql-query-builder.ts +1 -0
  461. package/src/sqlite-db-helper.ts +3 -3
  462. package/src/sqlite-types.ts +6 -5
  463. package/src/sync/ClientSessionSyncProcessor.ts +152 -142
  464. package/src/sync/errors.ts +12 -24
  465. package/src/sync/index.ts +2 -0
  466. package/src/sync/mock-sync-backend.ts +146 -104
  467. package/src/sync/next/compact-events.ts +10 -11
  468. package/src/sync/next/facts.ts +13 -14
  469. package/src/sync/next/history-dag-common.ts +280 -26
  470. package/src/sync/next/history-dag.ts +17 -13
  471. package/src/sync/next/rebase-events.ts +12 -12
  472. package/src/sync/next/test/compact-events.calculator.test.ts +3 -2
  473. package/src/sync/next/test/compact-events.test.ts +4 -3
  474. package/src/sync/next/test/event-fixtures.ts +13 -13
  475. package/src/sync/sync-backend-kv.ts +4 -3
  476. package/src/sync/sync-backend.ts +66 -17
  477. package/src/sync/sync.ts +24 -2
  478. package/src/sync/syncstate.test.ts +583 -419
  479. package/src/sync/syncstate.ts +127 -122
  480. package/src/sync/transport-chunking.ts +90 -0
  481. package/src/sync/validate-push-payload.ts +6 -8
  482. package/src/testing/event-factory.ts +10 -12
  483. package/src/util.ts +2 -2
  484. package/src/version.ts +33 -8
  485. package/dist/schema/EventDef.d.ts +0 -126
  486. package/dist/schema/EventDef.d.ts.map +0 -1
  487. package/dist/schema/EventDef.js +0 -46
  488. package/dist/schema/EventDef.js.map +0 -1
  489. package/dist/schema/EventSequenceNumber.d.ts +0 -80
  490. package/dist/schema/EventSequenceNumber.d.ts.map +0 -1
  491. package/dist/schema/EventSequenceNumber.js +0 -139
  492. package/dist/schema/EventSequenceNumber.js.map +0 -1
  493. package/dist/schema/LiveStoreEvent.d.ts.map +0 -1
  494. package/dist/schema/LiveStoreEvent.js +0 -147
  495. package/dist/schema/LiveStoreEvent.js.map +0 -1
  496. package/dist/schema/state/sqlite/system-tables.d.ts.map +0 -1
  497. package/dist/schema/state/sqlite/system-tables.js +0 -81
  498. package/dist/schema/state/sqlite/system-tables.js.map +0 -1
  499. package/dist/sync/next/graphology.d.ts +0 -8
  500. package/dist/sync/next/graphology.d.ts.map +0 -1
  501. package/dist/sync/next/graphology.js +0 -30
  502. package/dist/sync/next/graphology.js.map +0 -1
  503. package/dist/sync/next/graphology_.d.ts +0 -3
  504. package/dist/sync/next/graphology_.d.ts.map +0 -1
  505. package/dist/sync/next/graphology_.js +0 -3
  506. package/dist/sync/next/graphology_.js.map +0 -1
  507. package/src/schema/EventDef.ts +0 -222
  508. package/src/schema/EventSequenceNumber.ts +0 -199
  509. package/src/schema/LiveStoreEvent.ts +0 -286
  510. package/src/schema/state/sqlite/system-tables.ts +0 -106
  511. package/src/sync/next/ambient.d.ts +0 -3
  512. package/src/sync/next/graphology.ts +0 -41
  513. package/src/sync/next/graphology_.ts +0 -2
@@ -1,14 +1,18 @@
1
- import { casesHandled, isNotUndefined, LS_DEV, shouldNeverHappen, TRACE_VERBOSE } from '@livestore/utils';
2
- import { BucketQueue, Cause, Deferred, Duration, Effect, Exit, FiberHandle, Layer, Option, OtelTracer, pipe, Queue, ReadonlyArray, Schedule, Stream, Subscribable, SubscriptionRef, } from '@livestore/utils/effect';
3
- import { UnexpectedError, } from "../adapter-types.js";
1
+ import { casesHandled, isNotUndefined, LS_DEV, TRACE_VERBOSE } from '@livestore/utils';
2
+ import { BucketQueue, Cause, Deferred, Duration, Effect, Exit, FiberHandle, Layer, Option, Queue, ReadonlyArray, Schedule, Schema, Stream, Subscribable, SubscriptionRef, } from '@livestore/utils/effect';
3
+ import { UnknownError } from "../adapter-types.js";
4
+ import { IntentionalShutdownCause } from "../errors.js";
4
5
  import { makeMaterializerHash } from "../materializer-helper.js";
5
- import { EventSequenceNumber, getEventDef, LiveStoreEvent, SystemTables } from "../schema/mod.js";
6
- import { LeaderAheadError, } from "../sync/sync.js";
6
+ import { EventSequenceNumber, LiveStoreEvent, resolveEventDef, SystemTables } from "../schema/mod.js";
7
+ import { EVENTLOG_META_TABLE, SYNC_STATUS_TABLE } from "../schema/state/sqlite/system-tables/eventlog-tables.js";
8
+ import { isRejectedPushError, LeaderAheadError, NonMonotonicBatchError, StaleRebaseGenerationError } from "./RejectedPushError.js";
7
9
  import * as SyncState from "../sync/syncstate.js";
8
10
  import { sql } from "../util.js";
9
11
  import * as Eventlog from "./eventlog.js";
10
12
  import { rollback } from "./materialize-event.js";
11
13
  import { LeaderThreadCtx } from "./types.js";
14
+ /** Serialize value to JSON string for trace attributes */
15
+ const jsonStringify = Schema.encodeSync(Schema.parseJson());
12
16
  /**
13
17
  * The LeaderSyncProcessor manages synchronization of events between
14
18
  * the local state and the sync backend, ensuring efficient and orderly processing.
@@ -27,11 +31,11 @@ import { LeaderThreadCtx } from "./types.js";
27
31
  * - Maintains events in ascending order.
28
32
  * - Uses `Deferred` objects to resolve/reject events based on application success.
29
33
  * - Processes events from the queue, applying events in batches.
30
- * - Controlled by a `Latch` to manage execution flow.
31
- * - The latch closes on pull receipt and re-opens post-pull completion.
34
+ * - Controlled by a mutex (`Semaphore(1)`) to ensure mutual exclusion between local push and backend pull processing.
35
+ * - The backend pull side acquires the mutex before processing and releases it on post-pull completion.
32
36
  * - Processes up to `maxBatchSize` events per cycle.
33
37
  *
34
- * Currently we're advancing the state db and eventlog in lockstep, but we could also decouple this in the future
38
+ * Currently, we're advancing the state db and eventlog in lockstep, but we could also decouple this in the future
35
39
  *
36
40
  * Tricky concurrency scenarios:
37
41
  * - Queued local push batches becoming invalid due to a prior local push item being rejected.
@@ -39,23 +43,20 @@ import { LeaderThreadCtx } from "./types.js";
39
43
  *
40
44
  * See ClientSessionSyncProcessor for how the leader and session sync processors are similar/different.
41
45
  */
42
- export const makeLeaderSyncProcessor = ({ schema, dbState, initialBlockingSyncContext, initialSyncState, onError, livePull, params, testing, }) => Effect.gen(function* () {
46
+ export const makeLeaderSyncProcessor = ({ schema, dbState, initialBlockingSyncContext, initialSyncState, onError, onBackendIdMismatch, livePull, params, testing, }) => Effect.gen(function* () {
43
47
  const syncBackendPushQueue = yield* BucketQueue.make();
44
- const localPushBatchSize = params.localPushBatchSize ?? 1;
45
- const backendPushBatchSize = params.backendPushBatchSize ?? 2;
48
+ const localPushBatchSize = params.localPushBatchSize ?? 10;
49
+ const backendPushBatchSize = params.backendPushBatchSize ?? 50;
46
50
  const syncStateSref = yield* SubscriptionRef.make(undefined);
47
- const isClientEvent = (eventEncoded) => {
48
- const { eventDef } = getEventDef(schema, eventEncoded.name);
49
- return eventDef.options.clientOnly;
50
- };
51
+ const isClientEvent = (eventEncoded) => schema.eventsDefsMap.get(eventEncoded.name)?.options.clientOnly ?? false;
51
52
  const connectedClientSessionPullQueues = yield* makePullQueueSet;
52
53
  // This context depends on data from `boot`, we should find a better implementation to avoid this ref indirection.
53
54
  const ctxRef = {
54
55
  current: undefined,
55
56
  };
56
57
  const localPushesQueue = yield* BucketQueue.make();
57
- const localPushesLatch = yield* Effect.makeLatch(true);
58
- const pullLatch = yield* Effect.makeLatch(true);
58
+ // Ensures mutual exclusion between local push and backend pull processing.
59
+ const localPushBackendPullMutex = yield* Effect.makeSemaphore(1);
59
60
  /**
60
61
  * Additionally to the `syncStateSref` we also need the `pushHeadRef` in order to prevent old/duplicate
61
62
  * events from being pushed in a scenario like this:
@@ -65,9 +66,9 @@ export const makeLeaderSyncProcessor = ({ schema, dbState, initialBlockingSyncCo
65
66
  *
66
67
  * Thus the purpose of the pushHeadRef is the guard the integrity of the local push queue
67
68
  */
68
- const pushHeadRef = { current: EventSequenceNumber.ROOT };
69
+ const pushHeadRef = { current: EventSequenceNumber.Client.ROOT };
69
70
  const advancePushHead = (eventNum) => {
70
- pushHeadRef.current = EventSequenceNumber.max(pushHeadRef.current, eventNum);
71
+ pushHeadRef.current = EventSequenceNumber.Client.max(pushHeadRef.current, eventNum);
71
72
  };
72
73
  // NOTE: New events are only pushed to sync backend after successful local push processing
73
74
  const push = (newEvents, options) => Effect.gen(function* () {
@@ -77,7 +78,7 @@ export const makeLeaderSyncProcessor = ({ schema, dbState, initialBlockingSyncCo
77
78
  yield* validatePushBatch(newEvents, pushHeadRef.current);
78
79
  advancePushHead(newEvents.at(-1).seqNum);
79
80
  const waitForProcessing = options?.waitForProcessing ?? false;
80
- if (waitForProcessing) {
81
+ if (waitForProcessing === true) {
81
82
  const deferreds = yield* Effect.forEach(newEvents, () => Deferred.make());
82
83
  const items = newEvents.map((eventEncoded, i) => [eventEncoded, deferreds[i]]);
83
84
  yield* BucketQueue.offerAll(localPushesQueue, items);
@@ -90,34 +91,51 @@ export const makeLeaderSyncProcessor = ({ schema, dbState, initialBlockingSyncCo
90
91
  }).pipe(Effect.withSpan('@livestore/common:LeaderSyncProcessor:push', {
91
92
  attributes: {
92
93
  batchSize: newEvents.length,
93
- batch: TRACE_VERBOSE ? newEvents : undefined,
94
+ batch: TRACE_VERBOSE === true ? newEvents : undefined,
94
95
  },
95
- links: ctxRef.current?.span ? [{ _tag: 'SpanLink', span: ctxRef.current.span, attributes: {} }] : undefined,
96
+ links: ctxRef.current?.span !== undefined
97
+ ? [{ _tag: 'SpanLink', span: ctxRef.current.span, attributes: {} }]
98
+ : undefined,
96
99
  }));
97
100
  const pushPartial = ({ event: { name, args }, clientId, sessionId }) => Effect.gen(function* () {
98
- const syncState = yield* syncStateSref;
99
- if (syncState === undefined)
100
- return shouldNeverHappen('Not initialized');
101
- const { eventDef } = getEventDef(schema, name);
102
- const eventEncoded = new LiveStoreEvent.EncodedWithMeta({
101
+ const syncState = yield* Effect.fromNullable(yield* syncStateSref).pipe(Effect.orDieDebugger);
102
+ const resolution = yield* resolveEventDef(schema, {
103
+ operation: '@livestore/common:LeaderSyncProcessor:pushPartial',
104
+ event: {
105
+ name,
106
+ args,
107
+ clientId,
108
+ sessionId,
109
+ seqNum: syncState.localHead,
110
+ },
111
+ });
112
+ if (resolution._tag === 'unknown') {
113
+ // Ignore partial pushes for unrecognised events – they are still
114
+ // persisted server-side once a schema update ships.
115
+ return;
116
+ }
117
+ const eventEncoded = new LiveStoreEvent.Client.EncodedWithMeta({
103
118
  name,
104
119
  args,
105
120
  clientId,
106
121
  sessionId,
107
- ...EventSequenceNumber.nextPair({ seqNum: syncState.localHead, isClient: eventDef.options.clientOnly }),
122
+ ...EventSequenceNumber.Client.nextPair({
123
+ seqNum: syncState.localHead,
124
+ isClient: resolution.eventDef.options.clientOnly,
125
+ }),
108
126
  });
109
127
  yield* push([eventEncoded]);
110
- }).pipe(Effect.catchTag('LeaderAheadError', Effect.orDie));
128
+ }).pipe(
129
+ // pushPartial constructs the event sequence number internally, so these errors should never happen.
130
+ Effect.catchIf(isRejectedPushError, Effect.die));
111
131
  // Starts various background loops
112
132
  const boot = Effect.gen(function* () {
113
133
  const span = yield* Effect.currentSpan.pipe(Effect.orDie);
114
- const otelSpan = yield* OtelTracer.currentOtelSpan.pipe(Effect.catchAll(() => Effect.succeed(undefined)));
115
134
  const { devtools, shutdownChannel } = yield* LeaderThreadCtx;
116
135
  const runtime = yield* Effect.runtime();
117
136
  ctxRef.current = {
118
- otelSpan,
119
137
  span,
120
- devtoolsLatch: devtools.enabled ? devtools.syncBackendLatch : undefined,
138
+ devtoolsLatch: devtools.enabled === true ? devtools.syncBackendLatch : undefined,
121
139
  runtime,
122
140
  };
123
141
  /** State transitions need to happen atomically, so we use a Ref to track the state */
@@ -125,35 +143,34 @@ export const makeLeaderSyncProcessor = ({ schema, dbState, initialBlockingSyncCo
125
143
  // Rehydrate sync queue
126
144
  if (initialSyncState.pending.length > 0) {
127
145
  const globalPendingEvents = initialSyncState.pending
128
- // Don't sync clientOnly events
146
+ // Don't sync client-local events
129
147
  .filter((eventEncoded) => {
130
- const { eventDef } = getEventDef(schema, eventEncoded.name);
131
- return eventDef.options.clientOnly === false;
148
+ const eventDef = schema.eventsDefsMap.get(eventEncoded.name);
149
+ return eventDef === undefined ? true : eventDef.options.clientOnly === false;
132
150
  });
133
151
  if (globalPendingEvents.length > 0) {
134
152
  yield* BucketQueue.offerAll(syncBackendPushQueue, globalPendingEvents);
135
153
  }
136
154
  }
155
+ const handleBackendIdMismatchError = (error) => handleBackendIdMismatch({ error, onBackendIdMismatch, shutdownChannel });
137
156
  const maybeShutdownOnError = (cause) => Effect.gen(function* () {
138
157
  if (onError === 'ignore') {
139
- if (LS_DEV) {
158
+ if (LS_DEV === true) {
140
159
  yield* Effect.logDebug(`Ignoring sync error (${cause._tag === 'Fail' ? cause.error._tag : cause._tag})`, Cause.pretty(cause));
141
160
  }
142
161
  return;
143
162
  }
144
- const errorToSend = Cause.isFailType(cause) ? cause.error : UnexpectedError.make({ cause });
163
+ const errorToSend = Cause.isFailType(cause) === true ? cause.error : UnknownError.make({ cause });
145
164
  yield* shutdownChannel.send(errorToSend).pipe(Effect.orDie);
146
- return yield* Effect.die(cause);
165
+ return yield* Effect.failCause(cause).pipe(Effect.orDie);
147
166
  });
148
167
  yield* backgroundApplyLocalPushes({
149
- localPushesLatch,
168
+ localPushBackendPullMutex,
150
169
  localPushesQueue,
151
- pullLatch,
152
170
  syncStateSref,
153
171
  syncBackendPushQueue,
154
172
  schema,
155
173
  isClientEvent,
156
- otelSpan,
157
174
  connectedClientSessionPullQueues,
158
175
  localPushBatchSize,
159
176
  testing: {
@@ -163,10 +180,9 @@ export const makeLeaderSyncProcessor = ({ schema, dbState, initialBlockingSyncCo
163
180
  const backendPushingFiberHandle = yield* FiberHandle.make();
164
181
  const backendPushingEffect = backgroundBackendPushing({
165
182
  syncBackendPushQueue,
166
- otelSpan,
167
183
  devtoolsLatch: ctxRef.current?.devtoolsLatch,
168
184
  backendPushBatchSize,
169
- }).pipe(Effect.catchAllCause(maybeShutdownOnError));
185
+ }).pipe(Effect.catchTag('BackendIdMismatchError', handleBackendIdMismatchError), Effect.catchAllCause(maybeShutdownOnError));
170
186
  yield* FiberHandle.run(backendPushingFiberHandle, backendPushingEffect);
171
187
  yield* backgroundBackendPulling({
172
188
  isClientEvent,
@@ -180,19 +196,19 @@ export const makeLeaderSyncProcessor = ({ schema, dbState, initialBlockingSyncCo
180
196
  yield* FiberHandle.run(backendPushingFiberHandle, backendPushingEffect);
181
197
  }),
182
198
  syncStateSref,
183
- localPushesLatch,
184
- pullLatch,
199
+ localPushBackendPullMutex,
185
200
  livePull,
186
201
  dbState,
187
- otelSpan,
188
202
  initialBlockingSyncContext,
189
203
  devtoolsLatch: ctxRef.current?.devtoolsLatch,
190
204
  connectedClientSessionPullQueues,
191
205
  advancePushHead,
192
206
  }).pipe(Effect.retry({
193
- // We want to retry pulling if we've lost connection to the sync backend
194
- while: (cause) => cause._tag === 'IsOfflineError',
195
- }), Effect.catchAllCause(maybeShutdownOnError),
207
+ // Retry pulling when we've lost connection to the sync backend
208
+ // We're using `until` with a refinement instead of `while` to narrow `IsOfflineError` out of the error type.
209
+ // See https://github.com/Effect-TS/effect/issues/6122
210
+ until: (error) => error._tag !== 'IsOfflineError',
211
+ }), Effect.catchTag('BackendIdMismatchError', handleBackendIdMismatchError), Effect.catchAllCause(maybeShutdownOnError),
196
212
  // Needed to avoid `Fiber terminated with an unhandled error` logs which seem to happen because of the `Effect.retry` above.
197
213
  // This might be a bug in Effect. Only seems to happen in the browser.
198
214
  Effect.provide(Layer.setUnhandledErrorLogLevel(Option.none())), Effect.forkScoped);
@@ -216,17 +232,9 @@ export const makeLeaderSyncProcessor = ({ schema, dbState, initialBlockingSyncCo
216
232
  - full new state db snapshot in the "rebase" case
217
233
  - downside: importing the snapshot is expensive
218
234
  */
219
- const pullQueue = ({ cursor }) => {
220
- const runtime = ctxRef.current?.runtime ?? shouldNeverHappen('Not initialized');
221
- return connectedClientSessionPullQueues.makeQueue(cursor).pipe(Effect.provide(runtime));
222
- };
235
+ const pullQueue = ({ cursor }) => Effect.fromNullable(ctxRef.current?.runtime).pipe(Effect.orDieDebugger, Effect.flatMap((runtime) => connectedClientSessionPullQueues.makeQueue(cursor).pipe(Effect.provide(runtime))));
223
236
  const syncState = Subscribable.make({
224
- get: Effect.gen(function* () {
225
- const syncState = yield* syncStateSref;
226
- if (syncState === undefined)
227
- return shouldNeverHappen('Not initialized');
228
- return syncState;
229
- }),
237
+ get: syncStateSref.pipe(Effect.flatMap(Effect.fromNullable), Effect.orDieDebugger),
230
238
  changes: syncStateSref.changes.pipe(Stream.filter(isNotUndefined)),
231
239
  });
232
240
  return {
@@ -238,100 +246,103 @@ export const makeLeaderSyncProcessor = ({ schema, dbState, initialBlockingSyncCo
238
246
  syncState,
239
247
  };
240
248
  });
241
- const backgroundApplyLocalPushes = ({ localPushesLatch, localPushesQueue, pullLatch, syncStateSref, syncBackendPushQueue, schema, isClientEvent, otelSpan, connectedClientSessionPullQueues, localPushBatchSize, testing, }) => Effect.gen(function* () {
249
+ const backgroundApplyLocalPushes = ({ localPushBackendPullMutex, localPushesQueue, syncStateSref, syncBackendPushQueue, schema, isClientEvent, connectedClientSessionPullQueues, localPushBatchSize, testing, }) => Effect.gen(function* () {
242
250
  while (true) {
243
251
  if (testing.delay !== undefined) {
244
252
  yield* testing.delay.pipe(Effect.withSpan('localPushProcessingDelay'));
245
253
  }
246
254
  const batchItems = yield* BucketQueue.takeBetween(localPushesQueue, 1, localPushBatchSize);
247
- // Wait for the backend pulling to finish
248
- yield* localPushesLatch.await;
249
- // Prevent backend pull processing until this local push is finished
250
- yield* pullLatch.close;
251
- const syncState = yield* syncStateSref;
252
- if (syncState === undefined)
253
- return shouldNeverHappen('Not initialized');
254
- const currentRebaseGeneration = syncState.localHead.rebaseGeneration;
255
- // Since the rebase generation might have changed since enqueuing, we need to filter out items with older generation
256
- // It's important that we filter after we got localPushesLatch, otherwise we might filter with the old generation
257
- const [newEvents, deferreds] = pipe(batchItems, ReadonlyArray.filter(([eventEncoded]) => eventEncoded.seqNum.rebaseGeneration === currentRebaseGeneration), ReadonlyArray.unzip);
258
- if (newEvents.length === 0) {
259
- // console.log('dropping old-gen batch', currentLocalPushGenerationRef.current)
260
- // Allow the backend pulling to start
261
- yield* pullLatch.open;
262
- continue;
263
- }
264
- const mergeResult = SyncState.merge({
265
- syncState,
266
- payload: { _tag: 'local-push', newEvents },
267
- isClientEvent,
268
- isEqualEvent: LiveStoreEvent.isEqualEncoded,
269
- });
270
- switch (mergeResult._tag) {
271
- case 'unexpected-error': {
272
- otelSpan?.addEvent(`push:unexpected-error`, {
273
- batchSize: newEvents.length,
274
- newEvents: TRACE_VERBOSE ? JSON.stringify(newEvents) : undefined,
255
+ // Applies a batch of local pushes, guarded by the localPushBackendPullMutex to ensure mutual exclusion with backend pulling
256
+ yield* Effect.gen(function* () {
257
+ const syncState = yield* Effect.fromNullable(yield* syncStateSref).pipe(Effect.orDieDebugger);
258
+ const currentRebaseGeneration = syncState.localHead.rebaseGeneration;
259
+ // Since the rebase generation might have changed since enqueuing, we need to filter out items with older generation
260
+ // It's important that we filter after acquiring the localPushBackendPullMutex, otherwise we might filter with the old generation
261
+ const [droppedItems, filteredItems] = ReadonlyArray.partition(batchItems, ([eventEncoded]) => eventEncoded.seqNum.rebaseGeneration >= currentRebaseGeneration);
262
+ if (droppedItems.length > 0) {
263
+ yield* Effect.spanEvent(`push:drop-old-generation`, {
264
+ droppedCount: droppedItems.length,
265
+ currentRebaseGeneration,
275
266
  });
276
- return yield* new UnexpectedError({ cause: mergeResult.message });
267
+ /**
268
+ * Dropped pushes may still have a deferred awaiting completion.
269
+ * Fail it so the caller learns the leader advanced and resubmits with the updated generation.
270
+ */
271
+ yield* Effect.forEach(droppedItems.filter((item) => item[1] !== undefined), ([eventEncoded, deferred]) => Deferred.fail(deferred, StaleRebaseGenerationError.make({
272
+ currentRebaseGeneration,
273
+ providedRebaseGeneration: eventEncoded.seqNum.rebaseGeneration,
274
+ sessionId: eventEncoded.sessionId,
275
+ })));
277
276
  }
278
- case 'rebase': {
279
- return shouldNeverHappen('The leader thread should never have to rebase due to a local push');
277
+ if (filteredItems.length === 0) {
278
+ return;
280
279
  }
281
- case 'reject': {
282
- otelSpan?.addEvent(`push:reject`, {
283
- batchSize: newEvents.length,
284
- mergeResult: TRACE_VERBOSE ? JSON.stringify(mergeResult) : undefined,
285
- });
286
- // TODO: how to test this?
287
- const nextRebaseGeneration = currentRebaseGeneration + 1;
288
- const providedNum = newEvents.at(0).seqNum;
289
- // All subsequent pushes with same generation should be rejected as well
290
- // We're also handling the case where the localPushQueue already contains events
291
- // from the next generation which we preserve in the queue
292
- const remainingEventsMatchingGeneration = yield* BucketQueue.takeSplitWhere(localPushesQueue, ([eventEncoded]) => eventEncoded.seqNum.rebaseGeneration >= nextRebaseGeneration);
293
- // TODO we still need to better understand and handle this scenario
294
- if (LS_DEV && (yield* BucketQueue.size(localPushesQueue)) > 0) {
295
- console.log('localPushesQueue is not empty', yield* BucketQueue.size(localPushesQueue));
296
- // biome-ignore lint/suspicious/noDebugger: debugging
297
- debugger;
280
+ const [newEvents, deferreds] = ReadonlyArray.unzip(filteredItems);
281
+ yield* Effect.annotateCurrentSpan({
282
+ 'batchSize': newEvents.length,
283
+ ...(TRACE_VERBOSE === true ? { 'newEvents': jsonStringify(newEvents) } : {}),
284
+ });
285
+ const mergeResult = yield* SyncState.merge({
286
+ syncState,
287
+ payload: { _tag: 'local-push', newEvents },
288
+ isClientEvent,
289
+ isEqualEvent: LiveStoreEvent.Client.isEqualEncoded,
290
+ });
291
+ switch (mergeResult._tag) {
292
+ case 'rebase': {
293
+ return yield* Effect.dieDebugger('The leader thread should never have to rebase due to a local push');
294
+ }
295
+ case 'reject': {
296
+ yield* Effect.spanEvent(`push:reject`, {
297
+ batchSize: newEvents.length,
298
+ ...(TRACE_VERBOSE === true ? { mergeResult: jsonStringify(mergeResult) } : {}),
299
+ });
300
+ // TODO: how to test this?
301
+ const nextRebaseGeneration = currentRebaseGeneration + 1;
302
+ const providedNum = newEvents.at(0).seqNum;
303
+ // All subsequent pushes with same generation should be rejected as well
304
+ // We're also handling the case where the localPushQueue already contains events
305
+ // from the next generation which we preserve in the queue
306
+ const remainingEventsMatchingGeneration = yield* BucketQueue.takeSplitWhere(localPushesQueue, ([eventEncoded]) => eventEncoded.seqNum.rebaseGeneration >= nextRebaseGeneration);
307
+ // TODO we still need to better understand and handle this scenario
308
+ if (LS_DEV === true && (yield* BucketQueue.size(localPushesQueue)) > 0) {
309
+ console.log('localPushesQueue is not empty', yield* BucketQueue.size(localPushesQueue));
310
+ // oxlint-disable-next-line eslint(no-debugger) -- intentional breakpoint for unexpected queue state
311
+ debugger;
312
+ }
313
+ const allDeferredsToReject = [
314
+ ...deferreds,
315
+ ...remainingEventsMatchingGeneration.map(([_, deferred]) => deferred),
316
+ ].filter(isNotUndefined);
317
+ yield* Effect.forEach(allDeferredsToReject, (deferred) => Deferred.fail(deferred, LeaderAheadError.make({ minimumExpectedNum: mergeResult.expectedMinimumId, providedNum, sessionId: newEvents.at(0).sessionId })));
318
+ // In this case we're skipping state update and down/upstream processing
319
+ // We've cleared the local push queue and are now waiting for new local pushes / backend pulls
320
+ return;
321
+ }
322
+ case 'advance': {
323
+ break;
324
+ }
325
+ default: {
326
+ casesHandled(mergeResult);
298
327
  }
299
- const allDeferredsToReject = [
300
- ...deferreds,
301
- ...remainingEventsMatchingGeneration.map(([_, deferred]) => deferred),
302
- ].filter(isNotUndefined);
303
- yield* Effect.forEach(allDeferredsToReject, (deferred) => Deferred.fail(deferred, LeaderAheadError.make({ minimumExpectedNum: mergeResult.expectedMinimumId, providedNum })));
304
- // Allow the backend pulling to start
305
- yield* pullLatch.open;
306
- // In this case we're skipping state update and down/upstream processing
307
- // We've cleared the local push queue and are now waiting for new local pushes / backend pulls
308
- continue;
309
- }
310
- case 'advance': {
311
- break;
312
- }
313
- default: {
314
- casesHandled(mergeResult);
315
328
  }
316
- }
317
- yield* SubscriptionRef.set(syncStateSref, mergeResult.newSyncState);
318
- yield* connectedClientSessionPullQueues.offer({
319
- payload: SyncState.PayloadUpstreamAdvance.make({ newEvents: mergeResult.newEvents }),
320
- leaderHead: mergeResult.newSyncState.localHead,
321
- });
322
- otelSpan?.addEvent(`push:advance`, {
323
- batchSize: newEvents.length,
324
- mergeResult: TRACE_VERBOSE ? JSON.stringify(mergeResult) : undefined,
325
- });
326
- // Don't sync clientOnly events
327
- const filteredBatch = mergeResult.newEvents.filter((eventEncoded) => {
328
- const { eventDef } = getEventDef(schema, eventEncoded.name);
329
- return eventDef.options.clientOnly === false;
330
- });
331
- yield* BucketQueue.offerAll(syncBackendPushQueue, filteredBatch);
332
- yield* materializeEventsBatch({ batchItems: mergeResult.newEvents, deferreds });
333
- // Allow the backend pulling to start
334
- yield* pullLatch.open;
329
+ yield* SubscriptionRef.set(syncStateSref, mergeResult.newSyncState);
330
+ yield* connectedClientSessionPullQueues.offer({
331
+ payload: SyncState.PayloadUpstreamAdvance.make({ newEvents: mergeResult.newEvents }),
332
+ leaderHead: mergeResult.newSyncState.localHead,
333
+ });
334
+ yield* Effect.spanEvent(`push:advance`, {
335
+ batchSize: newEvents.length,
336
+ ...(TRACE_VERBOSE === true ? { mergeResult: jsonStringify(mergeResult) } : {}),
337
+ });
338
+ // Don't sync client-local events
339
+ const filteredBatch = mergeResult.newEvents.filter((eventEncoded) => {
340
+ const eventDef = schema.eventsDefsMap.get(eventEncoded.name);
341
+ return eventDef === undefined ? true : eventDef.options.clientOnly === false;
342
+ });
343
+ yield* BucketQueue.offerAll(syncBackendPushQueue, filteredBatch);
344
+ yield* materializeEventsBatch({ batchItems: mergeResult.newEvents, deferreds });
345
+ }).pipe(localPushBackendPullMutex.withPermits(1));
335
346
  }
336
347
  });
337
348
  // TODO how to handle errors gracefully
@@ -341,7 +352,7 @@ const materializeEventsBatch = ({ batchItems, deferreds }) => Effect.gen(functio
341
352
  db.execute('BEGIN TRANSACTION', undefined); // Start the transaction
342
353
  dbEventlog.execute('BEGIN TRANSACTION', undefined); // Start the transaction
343
354
  yield* Effect.addFinalizer((exit) => Effect.gen(function* () {
344
- if (Exit.isSuccess(exit))
355
+ if (Exit.isSuccess(exit) === true)
345
356
  return;
346
357
  // Rollback in case of an error
347
358
  db.execute('ROLLBACK', undefined);
@@ -360,129 +371,134 @@ const materializeEventsBatch = ({ batchItems, deferreds }) => Effect.gen(functio
360
371
  }).pipe(Effect.uninterruptible, Effect.scoped, Effect.withSpan('@livestore/common:LeaderSyncProcessor:materializeEventItems', {
361
372
  attributes: { batchSize: batchItems.length },
362
373
  }), Effect.tapCauseLogPretty);
363
- const backgroundBackendPulling = ({ isClientEvent, restartBackendPushing, otelSpan, dbState, syncStateSref, localPushesLatch, livePull, pullLatch, devtoolsLatch, initialBlockingSyncContext, connectedClientSessionPullQueues, advancePushHead, }) => Effect.gen(function* () {
374
+ const backgroundBackendPulling = Effect.fn('@livestore/common:LeaderSyncProcessor:backend-pulling')(function* ({ isClientEvent, restartBackendPushing, dbState, syncStateSref, localPushBackendPullMutex, livePull, devtoolsLatch, initialBlockingSyncContext, connectedClientSessionPullQueues, advancePushHead, }) {
364
375
  const { syncBackend, dbState: db, dbEventlog, schema } = yield* LeaderThreadCtx;
365
376
  if (syncBackend === undefined)
366
377
  return;
367
- const onNewPullChunk = (newEvents, pageInfo) => Effect.gen(function* () {
368
- if (newEvents.length === 0)
378
+ let pullMutexHeld = false;
379
+ const releasePullMutexIfHeld = Effect.gen(function* () {
380
+ if (pullMutexHeld === false)
369
381
  return;
382
+ pullMutexHeld = false;
383
+ yield* localPushBackendPullMutex.release(1);
384
+ });
385
+ const isPullPaginationComplete = (pageInfo) => pageInfo._tag === 'NoMore';
386
+ const onNewPullChunk = (newEvents, pageInfo) => Effect.gen(function* () {
370
387
  if (devtoolsLatch !== undefined) {
371
388
  yield* devtoolsLatch.await;
372
389
  }
373
- // Prevent more local pushes from being processed until this pull is finished
374
- yield* localPushesLatch.close;
375
- // Wait for pending local pushes to finish
376
- yield* pullLatch.await;
377
- const syncState = yield* syncStateSref;
378
- if (syncState === undefined)
379
- return shouldNeverHappen('Not initialized');
380
- const mergeResult = SyncState.merge({
381
- syncState,
382
- payload: SyncState.PayloadUpstreamAdvance.make({ newEvents }),
383
- isClientEvent,
384
- isEqualEvent: LiveStoreEvent.isEqualEncoded,
385
- ignoreClientEvents: true,
386
- });
387
- if (mergeResult._tag === 'reject') {
388
- return shouldNeverHappen('The leader thread should never reject upstream advances');
390
+ if (newEvents.length === 0) {
391
+ if (isPullPaginationComplete(pageInfo) === true) {
392
+ yield* releasePullMutexIfHeld;
393
+ }
394
+ return;
389
395
  }
390
- else if (mergeResult._tag === 'unexpected-error') {
391
- otelSpan?.addEvent(`pull:unexpected-error`, {
392
- newEventsCount: newEvents.length,
393
- newEvents: TRACE_VERBOSE ? JSON.stringify(newEvents) : undefined,
394
- });
395
- return yield* new UnexpectedError({ cause: mergeResult.message });
396
+ // Prevent more local pushes from being processed until this pull pagination sequence is finished.
397
+ if (pullMutexHeld === false) {
398
+ yield* localPushBackendPullMutex.take(1);
399
+ pullMutexHeld = true;
396
400
  }
397
- const newBackendHead = newEvents.at(-1).seqNum;
398
- Eventlog.updateBackendHead(dbEventlog, newBackendHead);
399
- if (mergeResult._tag === 'rebase') {
400
- otelSpan?.addEvent(`pull:rebase[${mergeResult.newSyncState.localHead.rebaseGeneration}]`, {
401
- newEventsCount: newEvents.length,
402
- newEvents: TRACE_VERBOSE ? JSON.stringify(newEvents) : undefined,
403
- rollbackCount: mergeResult.rollbackEvents.length,
404
- mergeResult: TRACE_VERBOSE ? JSON.stringify(mergeResult) : undefined,
401
+ const chunkExit = yield* Effect.gen(function* () {
402
+ const syncState = yield* Effect.fromNullable(yield* syncStateSref).pipe(Effect.orDieDebugger);
403
+ yield* Effect.annotateCurrentSpan({
404
+ 'merge.newEventsCount': newEvents.length,
405
+ ...(TRACE_VERBOSE === true ? { 'merge.newEvents': jsonStringify(newEvents) } : {}),
405
406
  });
406
- const globalRebasedPendingEvents = mergeResult.newSyncState.pending.filter((event) => {
407
- const { eventDef } = getEventDef(schema, event.name);
408
- return eventDef.options.clientOnly === false;
407
+ const mergeResult = yield* SyncState.merge({
408
+ syncState,
409
+ payload: SyncState.PayloadUpstreamAdvance.make({ newEvents }),
410
+ isClientEvent,
411
+ isEqualEvent: LiveStoreEvent.Client.isEqualEncoded,
412
+ ignoreClientEvents: true,
409
413
  });
410
- yield* restartBackendPushing(globalRebasedPendingEvents);
411
- if (mergeResult.rollbackEvents.length > 0) {
412
- yield* rollback({
413
- dbState: db,
414
- dbEventlog,
415
- eventNumsToRollback: mergeResult.rollbackEvents.map((_) => _.seqNum),
414
+ if (mergeResult._tag === 'reject') {
415
+ return yield* Effect.dieDebugger('The leader thread should never reject upstream advances');
416
+ }
417
+ const newBackendHead = newEvents.at(-1).seqNum;
418
+ Eventlog.updateBackendHead(dbEventlog, newBackendHead);
419
+ if (mergeResult._tag === 'rebase') {
420
+ yield* Effect.spanEvent(`pull:rebase[${mergeResult.newSyncState.localHead.rebaseGeneration}]`, {
421
+ newEventsCount: newEvents.length,
422
+ ...(TRACE_VERBOSE === true ? { newEvents: jsonStringify(newEvents) } : {}),
423
+ rollbackCount: mergeResult.rollbackEvents.length,
424
+ ...(TRACE_VERBOSE === true ? { mergeResult: jsonStringify(mergeResult) } : {}),
425
+ });
426
+ const globalRebasedPendingEvents = mergeResult.newSyncState.pending.filter((event) => {
427
+ const eventDef = schema.eventsDefsMap.get(event.name);
428
+ return eventDef === undefined ? true : eventDef.options.clientOnly === false;
429
+ });
430
+ yield* restartBackendPushing(globalRebasedPendingEvents);
431
+ if (mergeResult.rollbackEvents.length > 0) {
432
+ yield* rollback({
433
+ dbState: db,
434
+ dbEventlog,
435
+ eventNumsToRollback: mergeResult.rollbackEvents.map((_) => _.seqNum),
436
+ });
437
+ }
438
+ yield* connectedClientSessionPullQueues.offer({
439
+ payload: SyncState.payloadFromMergeResult(mergeResult),
440
+ leaderHead: mergeResult.newSyncState.localHead,
416
441
  });
417
442
  }
418
- yield* connectedClientSessionPullQueues.offer({
419
- payload: SyncState.payloadFromMergeResult(mergeResult),
420
- leaderHead: mergeResult.newSyncState.localHead,
421
- });
422
- }
423
- else {
424
- otelSpan?.addEvent(`pull:advance`, {
425
- newEventsCount: newEvents.length,
426
- mergeResult: TRACE_VERBOSE ? JSON.stringify(mergeResult) : undefined,
427
- });
428
- // Ensure push fiber is active after advance by restarting with current pending (non-client) events
429
- const globalPendingEvents = mergeResult.newSyncState.pending.filter((event) => {
430
- const { eventDef } = getEventDef(schema, event.name);
431
- return eventDef.options.clientOnly === false;
432
- });
433
- yield* restartBackendPushing(globalPendingEvents);
434
- yield* connectedClientSessionPullQueues.offer({
435
- payload: SyncState.payloadFromMergeResult(mergeResult),
436
- leaderHead: mergeResult.newSyncState.localHead,
437
- });
438
- if (mergeResult.confirmedEvents.length > 0) {
439
- // `mergeResult.confirmedEvents` don't contain the correct sync metadata, so we need to use
440
- // `newEvents` instead which we filter via `mergeResult.confirmedEvents`
441
- const confirmedNewEvents = newEvents.filter((event) => mergeResult.confirmedEvents.some((confirmedEvent) => EventSequenceNumber.isEqual(event.seqNum, confirmedEvent.seqNum)));
442
- yield* Eventlog.updateSyncMetadata(confirmedNewEvents).pipe(UnexpectedError.mapToUnexpectedError);
443
+ else {
444
+ yield* Effect.spanEvent(`pull:advance`, {
445
+ newEventsCount: newEvents.length,
446
+ ...(TRACE_VERBOSE === true ? { mergeResult: jsonStringify(mergeResult) } : {}),
447
+ });
448
+ // Ensure push fiber is active after advance by restarting with current pending (non-client) events
449
+ const globalPendingEvents = mergeResult.newSyncState.pending.filter((event) => {
450
+ const eventDef = schema.eventsDefsMap.get(event.name);
451
+ return eventDef === undefined ? true : eventDef.options.clientOnly === false;
452
+ });
453
+ yield* restartBackendPushing(globalPendingEvents);
454
+ yield* connectedClientSessionPullQueues.offer({
455
+ payload: SyncState.payloadFromMergeResult(mergeResult),
456
+ leaderHead: mergeResult.newSyncState.localHead,
457
+ });
458
+ if (mergeResult.confirmedEvents.length > 0) {
459
+ // `mergeResult.confirmedEvents` don't contain the correct sync metadata, so we need to use
460
+ // `newEvents` instead which we filter via `mergeResult.confirmedEvents`
461
+ const confirmedNewEvents = newEvents.filter((event) => mergeResult.confirmedEvents.some((confirmedEvent) => EventSequenceNumber.Client.isEqual(event.seqNum, confirmedEvent.seqNum)));
462
+ yield* Eventlog.updateSyncMetadata(confirmedNewEvents).pipe(Effect.orDieDebugger);
463
+ }
443
464
  }
465
+ // Removes the changeset rows which are no longer needed as we'll never have to rollback beyond this point
466
+ trimChangesetRows(db, newBackendHead);
467
+ advancePushHead(mergeResult.newSyncState.localHead);
468
+ yield* materializeEventsBatch({ batchItems: mergeResult.newEvents, deferreds: undefined });
469
+ yield* SubscriptionRef.set(syncStateSref, mergeResult.newSyncState);
470
+ }).pipe(Effect.exit);
471
+ if (Exit.isFailure(chunkExit) === true) {
472
+ yield* releasePullMutexIfHeld;
473
+ return yield* Effect.failCause(chunkExit.cause);
444
474
  }
445
- // Removes the changeset rows which are no longer needed as we'll never have to rollback beyond this point
446
- trimChangesetRows(db, newBackendHead);
447
- advancePushHead(mergeResult.newSyncState.localHead);
448
- yield* materializeEventsBatch({ batchItems: mergeResult.newEvents, deferreds: undefined });
449
- yield* SubscriptionRef.set(syncStateSref, mergeResult.newSyncState);
450
- // Allow local pushes to be processed again
451
- if (pageInfo._tag === 'NoMore') {
452
- yield* localPushesLatch.open;
475
+ if (isPullPaginationComplete(pageInfo) === true) {
476
+ yield* releasePullMutexIfHeld;
453
477
  }
454
478
  });
455
- const syncState = yield* syncStateSref;
456
- if (syncState === undefined)
457
- return shouldNeverHappen('Not initialized');
479
+ const syncState = yield* Effect.fromNullable(yield* syncStateSref).pipe(Effect.orDieDebugger);
458
480
  const cursorInfo = yield* Eventlog.getSyncBackendCursorInfo({ remoteHead: syncState.upstreamHead.global });
459
481
  const hashMaterializerResult = makeMaterializerHash({ schema, dbState });
460
482
  yield* syncBackend.pull(cursorInfo, { live: livePull }).pipe(
461
483
  // TODO only take from queue while connected
462
484
  Stream.tap(({ batch, pageInfo }) => Effect.gen(function* () {
463
- // yield* Effect.spanEvent('batch', {
464
- // attributes: {
465
- // batchSize: batch.length,
466
- // batch: TRACE_VERBOSE ? batch : undefined,
467
- // },
468
- // })
469
485
  // NOTE we only want to take process events when the sync backend is connected
470
486
  // (e.g. needed for simulating being offline)
471
487
  // TODO remove when there's a better way to handle this in stream above
472
488
  yield* SubscriptionRef.waitUntil(syncBackend.isConnected, (isConnected) => isConnected === true);
473
- yield* onNewPullChunk(batch.map((_) => LiveStoreEvent.EncodedWithMeta.fromGlobal(_.eventEncoded, {
489
+ yield* onNewPullChunk(batch.map((_) => LiveStoreEvent.Client.EncodedWithMeta.fromGlobal(_.eventEncoded, {
474
490
  syncMetadata: _.metadata,
475
491
  // TODO we can't really know the materializer result here yet beyond the first event batch item as we need to materialize it one by one first
476
492
  // This is a bug and needs to be fixed https://github.com/livestorejs/livestore/issues/503#issuecomment-3114533165
477
- materializerHashLeader: hashMaterializerResult(LiveStoreEvent.encodedFromGlobal(_.eventEncoded)),
493
+ materializerHashLeader: hashMaterializerResult(LiveStoreEvent.Global.toClientEncoded(_.eventEncoded)),
478
494
  materializerHashSession: Option.none(),
479
495
  })), pageInfo);
480
496
  yield* initialBlockingSyncContext.update({ processed: batch.length, pageInfo });
481
- })), Stream.runDrain, Effect.interruptible);
497
+ })), Stream.runDrain, Effect.interruptible, Effect.ensuring(releasePullMutexIfHeld));
482
498
  // Should only ever happen when livePull is false
483
499
  yield* Effect.logDebug('backend-pulling finished', { livePull });
484
- }).pipe(Effect.withSpan('@livestore/common:LeaderSyncProcessor:backend-pulling'));
485
- const backgroundBackendPushing = ({ syncBackendPushQueue, otelSpan, devtoolsLatch, backendPushBatchSize, }) => Effect.gen(function* () {
500
+ });
501
+ const backgroundBackendPushing = Effect.fn('@livestore/common:LeaderSyncProcessor:backend-pushing')(function* ({ syncBackendPushQueue, devtoolsLatch, backendPushBatchSize, }) {
486
502
  const { syncBackend } = yield* LeaderThreadCtx;
487
503
  if (syncBackend === undefined)
488
504
  return;
@@ -493,45 +509,47 @@ const backgroundBackendPushing = ({ syncBackendPushQueue, otelSpan, devtoolsLatc
493
509
  if (devtoolsLatch !== undefined) {
494
510
  yield* devtoolsLatch.await;
495
511
  }
496
- otelSpan?.addEvent('backend-push', {
512
+ yield* Effect.spanEvent('backend-push', {
497
513
  batchSize: queueItems.length,
498
- batch: TRACE_VERBOSE ? JSON.stringify(queueItems) : undefined,
514
+ ...(TRACE_VERBOSE === true ? { batch: jsonStringify(queueItems) } : {}),
499
515
  });
500
516
  // Push with declarative retry/backoff using Effect schedules
501
517
  // - Exponential backoff starting at 1s and doubling (1s, 2s, 4s, 8s, 16s, 30s ...)
502
518
  // - Delay clamped at 30s (continues retrying at 30s)
503
519
  // - Resets automatically after successful push
504
520
  // TODO(metrics): expose counters/gauges for retry attempts and queue health via devtools/metrics
505
- // Only retry for transient UnexpectedError cases
506
- const isRetryable = (err) => err._tag === 'InvalidPushError' && err.cause._tag === 'LiveStore.UnexpectedError';
507
- // Input: InvalidPushError | IsOfflineError, Output: Duration
508
- const retrySchedule = Schedule.exponential(Duration.seconds(1)).pipe(Schedule.andThenEither(Schedule.spaced(Duration.seconds(30))), // clamp at 30 second intervals
509
- Schedule.compose(Schedule.elapsed), Schedule.whileInput(isRetryable));
510
521
  yield* Effect.gen(function* () {
511
522
  const iteration = yield* Schedule.CurrentIterationMetadata;
512
523
  const pushResult = yield* syncBackend.push(queueItems.map((_) => _.toGlobal())).pipe(Effect.either);
513
524
  const retries = iteration.recurrence;
514
525
  if (retries > 0 && pushResult._tag === 'Right') {
515
- otelSpan?.addEvent('backend-push-retry-success', { retries, batchSize: queueItems.length });
526
+ yield* Effect.spanEvent('backend-push-retry-success', { retries, batchSize: queueItems.length });
516
527
  }
517
528
  if (pushResult._tag === 'Left') {
518
- otelSpan?.addEvent('backend-push-error', {
529
+ yield* Effect.spanEvent('backend-push-error', {
519
530
  error: pushResult.left.toString(),
520
531
  retries,
521
532
  batchSize: queueItems.length,
522
533
  });
523
534
  const error = pushResult.left;
524
- if (error._tag === 'IsOfflineError' ||
525
- (error._tag === 'InvalidPushError' && error.cause._tag === 'ServerAheadError')) {
535
+ if (error._tag === 'ServerAheadError') {
526
536
  // It's a core part of the sync protocol that the sync backend will emit a new pull chunk alongside the ServerAheadError
527
537
  yield* Effect.logDebug('handled backend-push-error (waiting for interupt caused by pull)', { error });
528
538
  return yield* Effect.never;
529
539
  }
530
540
  return yield* error;
531
541
  }
532
- }).pipe(Effect.retry(retrySchedule));
542
+ }).pipe(
543
+ // Retry transient errors
544
+ Effect.retry({
545
+ schedule: Schedule.exponential(Duration.seconds(1)).pipe(Schedule.modifyDelay((_, delay) => Duration.min(delay, Duration.seconds(30))) // Cap delay at 30s intervals.
546
+ ),
547
+ while: (error) => error._tag === 'IsOfflineError' || error._tag === 'UnknownError',
548
+ }),
549
+ // This is needed to narrow the Error type. Our retry policy runs indefinitely, but Effect.retry does not narrow the Error type.
550
+ Effect.catchIf((error) => error._tag === 'IsOfflineError' || error._tag === 'UnknownError', Effect.die));
533
551
  }
534
- }).pipe(Effect.interruptible, Effect.withSpan('@livestore/common:LeaderSyncProcessor:backend-pushing'));
552
+ }, Effect.interruptible);
535
553
  const trimChangesetRows = (db, newHead) => {
536
554
  // Since we're using the session changeset rows to query for the current head,
537
555
  // we're keeping at least one row for the current head, and thus are using `<` instead of `<=`
@@ -551,15 +569,15 @@ const makePullQueueSet = Effect.gen(function* () {
551
569
  const queue = yield* Queue.unbounded().pipe(Effect.acquireRelease(Queue.shutdown));
552
570
  yield* Effect.addFinalizer(() => Effect.sync(() => set.delete(queue)));
553
571
  const payloadsSinceCursor = Array.from(cachedPayloads.entries())
554
- .flatMap(([seqNumStr, payloads]) => payloads.map((payload) => ({ payload, seqNum: EventSequenceNumber.fromString(seqNumStr) })))
555
- .filter(({ seqNum }) => EventSequenceNumber.isGreaterThan(seqNum, cursor))
556
- .toSorted((a, b) => EventSequenceNumber.compare(a.seqNum, b.seqNum))
572
+ .flatMap(([seqNumStr, payloads]) => payloads.map((payload) => ({ payload, seqNum: EventSequenceNumber.Client.fromString(seqNumStr) })))
573
+ .filter(({ seqNum }) => EventSequenceNumber.Client.isGreaterThan(seqNum, cursor))
574
+ .toSorted((a, b) => EventSequenceNumber.Client.compare(a.seqNum, b.seqNum))
557
575
  .map(({ payload }) => {
558
576
  if (payload._tag === 'upstream-advance') {
559
577
  return {
560
578
  payload: {
561
579
  _tag: 'upstream-advance',
562
- newEvents: ReadonlyArray.dropWhile(payload.newEvents, (eventEncoded) => EventSequenceNumber.isGreaterThanOrEqual(cursor, eventEncoded.seqNum)),
580
+ newEvents: ReadonlyArray.dropWhile(payload.newEvents, (eventEncoded) => EventSequenceNumber.Client.isGreaterThanOrEqual(cursor, eventEncoded.seqNum)),
563
581
  },
564
582
  };
565
583
  }
@@ -599,8 +617,8 @@ const makePullQueueSet = Effect.gen(function* () {
599
617
  return queue;
600
618
  });
601
619
  const offer = (item) => Effect.gen(function* () {
602
- const seqNumStr = EventSequenceNumber.toString(item.leaderHead);
603
- if (cachedPayloads.has(seqNumStr)) {
620
+ const seqNumStr = EventSequenceNumber.Client.toString(item.leaderHead);
621
+ if (cachedPayloads.has(seqNumStr) === true) {
604
622
  cachedPayloads.get(seqNumStr).push(item.payload);
605
623
  }
606
624
  else {
@@ -620,22 +638,72 @@ const makePullQueueSet = Effect.gen(function* () {
620
638
  offer,
621
639
  };
622
640
  });
641
+ /**
642
+ * Validate a client-provided batch before it is admitted to the leader queue.
643
+ * Ensures the numbers form a strictly increasing chain and that the first
644
+ * event sits ahead of the current push head.
645
+ */
623
646
  const validatePushBatch = (batch, pushHead) => Effect.gen(function* () {
624
647
  if (batch.length === 0) {
625
648
  return;
626
649
  }
627
- // Make sure batch is monotonically increasing
650
+ // Defensive check: callers should already provide a strictly increasing sequence
651
+ // of event numbers.
628
652
  for (let i = 1; i < batch.length; i++) {
629
- if (EventSequenceNumber.isGreaterThanOrEqual(batch[i - 1].seqNum, batch[i].seqNum)) {
630
- shouldNeverHappen(`Events must be ordered in monotonically ascending order by eventNum. Received: [${batch.map((e) => EventSequenceNumber.toString(e.seqNum)).join(', ')}]`);
653
+ if (EventSequenceNumber.Client.isGreaterThanOrEqual(batch[i - 1].seqNum, batch[i].seqNum) === true) {
654
+ return yield* NonMonotonicBatchError.make({
655
+ precedingSeqNum: batch[i - 1].seqNum,
656
+ violatingSeqNum: batch[i].seqNum,
657
+ violationIndex: i,
658
+ sessionId: batch[i].sessionId,
659
+ });
631
660
  }
632
661
  }
633
- // Make sure smallest sequence number is > pushHead
634
- if (EventSequenceNumber.isGreaterThanOrEqual(pushHead, batch[0].seqNum)) {
662
+ // Reject stale batches whose first event is at or behind the leader's push head.
663
+ if (EventSequenceNumber.Client.isGreaterThanOrEqual(pushHead, batch[0].seqNum) === true) {
635
664
  return yield* LeaderAheadError.make({
636
665
  minimumExpectedNum: pushHead,
637
666
  providedNum: batch[0].seqNum,
667
+ sessionId: batch[0].sessionId,
638
668
  });
639
669
  }
640
670
  });
671
+ /**
672
+ * Handles a BackendIdMismatchError based on the configured behavior.
673
+ * This occurs when the sync backend has been reset and has a new identity.
674
+ */
675
+ const handleBackendIdMismatch = Effect.fn('@livestore/common:LeaderSyncProcessor:handleBackendIdMismatch')(function* ({ error, onBackendIdMismatch, shutdownChannel, }) {
676
+ const { dbEventlog, dbState } = yield* LeaderThreadCtx;
677
+ if (onBackendIdMismatch === 'reset') {
678
+ yield* Effect.logWarning('Sync backend identity changed (backend was reset). Clearing local storage and shutting down.', error);
679
+ // Clear local databases so the client can start fresh on next boot
680
+ yield* clearLocalDatabases({ dbEventlog, dbState });
681
+ // Send shutdown signal with special reason
682
+ yield* shutdownChannel.send(IntentionalShutdownCause.make({ reason: 'backend-id-mismatch' })).pipe(Effect.orDie);
683
+ return yield* Effect.die(error);
684
+ }
685
+ if (onBackendIdMismatch === 'shutdown') {
686
+ yield* Effect.logWarning('Sync backend identity changed (backend was reset). Shutting down without clearing local storage.', error);
687
+ yield* shutdownChannel.send(error).pipe(Effect.orDie);
688
+ return yield* Effect.die(error);
689
+ }
690
+ // ignore mode
691
+ if (LS_DEV === true) {
692
+ yield* Effect.logDebug('Ignoring BackendIdMismatchError (sync backend was reset but client continues with stale data)', error);
693
+ }
694
+ });
695
+ /**
696
+ * Clears local databases (eventlog and state) so the client can start fresh on next boot.
697
+ * This is used when the sync backend identity has changed (i.e. backend was reset).
698
+ */
699
+ const clearLocalDatabases = ({ dbEventlog, dbState }) => Effect.sync(() => {
700
+ // Clear eventlog tables
701
+ dbEventlog.execute(sql `DELETE FROM ${EVENTLOG_META_TABLE}`);
702
+ dbEventlog.execute(sql `DELETE FROM ${SYNC_STATUS_TABLE}`);
703
+ // Drop all state tables - they'll be recreated on next boot
704
+ const tables = dbState.select(sql `SELECT name FROM sqlite_master WHERE type='table' AND name NOT LIKE 'sqlite_%'`);
705
+ for (const { name } of tables) {
706
+ dbState.execute(`DROP TABLE IF EXISTS "${name}"`);
707
+ }
708
+ });
641
709
  //# sourceMappingURL=LeaderSyncProcessor.js.map