@livestore/common 0.4.0-dev.1 → 0.4.0-dev.11

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (263) hide show
  1. package/dist/.tsbuildinfo +1 -1
  2. package/dist/ClientSessionLeaderThreadProxy.d.ts +8 -3
  3. package/dist/ClientSessionLeaderThreadProxy.d.ts.map +1 -1
  4. package/dist/ClientSessionLeaderThreadProxy.js.map +1 -1
  5. package/dist/adapter-types.d.ts +9 -3
  6. package/dist/adapter-types.d.ts.map +1 -1
  7. package/dist/adapter-types.js.map +1 -1
  8. package/dist/devtools/devtools-messages-client-session.d.ts +21 -21
  9. package/dist/devtools/devtools-messages-common.d.ts +7 -14
  10. package/dist/devtools/devtools-messages-common.d.ts.map +1 -1
  11. package/dist/devtools/devtools-messages-common.js +1 -6
  12. package/dist/devtools/devtools-messages-common.js.map +1 -1
  13. package/dist/devtools/devtools-messages-leader.d.ts +27 -25
  14. package/dist/devtools/devtools-messages-leader.d.ts.map +1 -1
  15. package/dist/errors.d.ts +47 -5
  16. package/dist/errors.d.ts.map +1 -1
  17. package/dist/errors.js +22 -3
  18. package/dist/errors.js.map +1 -1
  19. package/dist/leader-thread/LeaderSyncProcessor.d.ts +7 -3
  20. package/dist/leader-thread/LeaderSyncProcessor.d.ts.map +1 -1
  21. package/dist/leader-thread/LeaderSyncProcessor.js +135 -52
  22. package/dist/leader-thread/LeaderSyncProcessor.js.map +1 -1
  23. package/dist/leader-thread/eventlog.d.ts +4 -10
  24. package/dist/leader-thread/eventlog.d.ts.map +1 -1
  25. package/dist/leader-thread/eventlog.js +4 -6
  26. package/dist/leader-thread/eventlog.js.map +1 -1
  27. package/dist/leader-thread/leader-worker-devtools.d.ts +1 -1
  28. package/dist/leader-thread/leader-worker-devtools.js +6 -2
  29. package/dist/leader-thread/leader-worker-devtools.js.map +1 -1
  30. package/dist/leader-thread/make-leader-thread-layer.d.ts +1 -2
  31. package/dist/leader-thread/make-leader-thread-layer.d.ts.map +1 -1
  32. package/dist/leader-thread/make-leader-thread-layer.js +68 -19
  33. package/dist/leader-thread/make-leader-thread-layer.js.map +1 -1
  34. package/dist/leader-thread/make-leader-thread-layer.test.d.ts +2 -0
  35. package/dist/leader-thread/make-leader-thread-layer.test.d.ts.map +1 -0
  36. package/dist/leader-thread/make-leader-thread-layer.test.js +32 -0
  37. package/dist/leader-thread/make-leader-thread-layer.test.js.map +1 -0
  38. package/dist/leader-thread/materialize-event.d.ts +2 -2
  39. package/dist/leader-thread/materialize-event.d.ts.map +1 -1
  40. package/dist/leader-thread/materialize-event.js +23 -9
  41. package/dist/leader-thread/materialize-event.js.map +1 -1
  42. package/dist/leader-thread/recreate-db.d.ts +2 -3
  43. package/dist/leader-thread/recreate-db.d.ts.map +1 -1
  44. package/dist/leader-thread/recreate-db.js +1 -1
  45. package/dist/leader-thread/recreate-db.js.map +1 -1
  46. package/dist/leader-thread/shutdown-channel.d.ts +2 -2
  47. package/dist/leader-thread/shutdown-channel.d.ts.map +1 -1
  48. package/dist/leader-thread/shutdown-channel.js +2 -2
  49. package/dist/leader-thread/shutdown-channel.js.map +1 -1
  50. package/dist/leader-thread/types.d.ts +7 -5
  51. package/dist/leader-thread/types.d.ts.map +1 -1
  52. package/dist/leader-thread/types.js.map +1 -1
  53. package/dist/materializer-helper.d.ts +1 -1
  54. package/dist/materializer-helper.d.ts.map +1 -1
  55. package/dist/materializer-helper.js +20 -4
  56. package/dist/materializer-helper.js.map +1 -1
  57. package/dist/rematerialize-from-eventlog.d.ts +1 -1
  58. package/dist/rematerialize-from-eventlog.d.ts.map +1 -1
  59. package/dist/rematerialize-from-eventlog.js +25 -16
  60. package/dist/rematerialize-from-eventlog.js.map +1 -1
  61. package/dist/schema/EventDef.d.ts +3 -0
  62. package/dist/schema/EventDef.d.ts.map +1 -1
  63. package/dist/schema/EventDef.js.map +1 -1
  64. package/dist/schema/EventSequenceNumber.d.ts +4 -1
  65. package/dist/schema/EventSequenceNumber.d.ts.map +1 -1
  66. package/dist/schema/EventSequenceNumber.js +4 -1
  67. package/dist/schema/EventSequenceNumber.js.map +1 -1
  68. package/dist/schema/EventSequenceNumber.test.js +2 -2
  69. package/dist/schema/LiveStoreEvent.d.ts +1 -1
  70. package/dist/schema/LiveStoreEvent.d.ts.map +1 -1
  71. package/dist/schema/LiveStoreEvent.js +1 -2
  72. package/dist/schema/LiveStoreEvent.js.map +1 -1
  73. package/dist/schema/mod.d.ts +2 -0
  74. package/dist/schema/mod.d.ts.map +1 -1
  75. package/dist/schema/mod.js +1 -0
  76. package/dist/schema/mod.js.map +1 -1
  77. package/dist/schema/schema.d.ts +15 -0
  78. package/dist/schema/schema.d.ts.map +1 -1
  79. package/dist/schema/schema.js +26 -1
  80. package/dist/schema/schema.js.map +1 -1
  81. package/dist/schema/state/sqlite/client-document-def.d.ts +35 -5
  82. package/dist/schema/state/sqlite/client-document-def.d.ts.map +1 -1
  83. package/dist/schema/state/sqlite/client-document-def.js +95 -4
  84. package/dist/schema/state/sqlite/client-document-def.js.map +1 -1
  85. package/dist/schema/state/sqlite/client-document-def.test.js +16 -0
  86. package/dist/schema/state/sqlite/client-document-def.test.js.map +1 -1
  87. package/dist/schema/state/sqlite/column-annotations.d.ts.map +1 -1
  88. package/dist/schema/state/sqlite/column-annotations.js +14 -6
  89. package/dist/schema/state/sqlite/column-annotations.js.map +1 -1
  90. package/dist/schema/state/sqlite/column-def.d.ts +6 -2
  91. package/dist/schema/state/sqlite/column-def.d.ts.map +1 -1
  92. package/dist/schema/state/sqlite/column-def.js +128 -185
  93. package/dist/schema/state/sqlite/column-def.js.map +1 -1
  94. package/dist/schema/state/sqlite/column-def.test.js +116 -73
  95. package/dist/schema/state/sqlite/column-def.test.js.map +1 -1
  96. package/dist/schema/state/sqlite/db-schema/ast/sqlite.d.ts +2 -1
  97. package/dist/schema/state/sqlite/db-schema/ast/sqlite.d.ts.map +1 -1
  98. package/dist/schema/state/sqlite/db-schema/ast/sqlite.js +23 -6
  99. package/dist/schema/state/sqlite/db-schema/ast/sqlite.js.map +1 -1
  100. package/dist/schema/state/sqlite/db-schema/dsl/mod.d.ts.map +1 -1
  101. package/dist/schema/state/sqlite/db-schema/dsl/mod.js +2 -1
  102. package/dist/schema/state/sqlite/db-schema/dsl/mod.js.map +1 -1
  103. package/dist/schema/state/sqlite/mod.d.ts +1 -1
  104. package/dist/schema/state/sqlite/mod.d.ts.map +1 -1
  105. package/dist/schema/state/sqlite/mod.js +1 -1
  106. package/dist/schema/state/sqlite/mod.js.map +1 -1
  107. package/dist/schema/state/sqlite/query-builder/api.d.ts +5 -2
  108. package/dist/schema/state/sqlite/query-builder/api.d.ts.map +1 -1
  109. package/dist/schema/state/sqlite/query-builder/impl.d.ts.map +1 -1
  110. package/dist/schema/state/sqlite/query-builder/impl.js +6 -2
  111. package/dist/schema/state/sqlite/query-builder/impl.js.map +1 -1
  112. package/dist/schema/state/sqlite/query-builder/impl.test.js +137 -2
  113. package/dist/schema/state/sqlite/query-builder/impl.test.js.map +1 -1
  114. package/dist/schema/state/sqlite/system-tables.d.ts +42 -6
  115. package/dist/schema/state/sqlite/system-tables.d.ts.map +1 -1
  116. package/dist/schema/state/sqlite/system-tables.js +2 -0
  117. package/dist/schema/state/sqlite/system-tables.js.map +1 -1
  118. package/dist/schema/state/sqlite/table-def.d.ts +4 -4
  119. package/dist/schema/state/sqlite/table-def.d.ts.map +1 -1
  120. package/dist/schema/state/sqlite/table-def.js +2 -2
  121. package/dist/schema/state/sqlite/table-def.js.map +1 -1
  122. package/dist/schema/state/sqlite/table-def.test.js +87 -2
  123. package/dist/schema/state/sqlite/table-def.test.js.map +1 -1
  124. package/dist/schema/unknown-events.d.ts +47 -0
  125. package/dist/schema/unknown-events.d.ts.map +1 -0
  126. package/dist/schema/unknown-events.js +69 -0
  127. package/dist/schema/unknown-events.js.map +1 -0
  128. package/dist/sql-queries/sql-query-builder.d.ts.map +1 -1
  129. package/dist/sql-queries/sql-query-builder.js +2 -1
  130. package/dist/sql-queries/sql-query-builder.js.map +1 -1
  131. package/dist/sync/ClientSessionSyncProcessor.d.ts +9 -11
  132. package/dist/sync/ClientSessionSyncProcessor.d.ts.map +1 -1
  133. package/dist/sync/ClientSessionSyncProcessor.js +36 -33
  134. package/dist/sync/ClientSessionSyncProcessor.js.map +1 -1
  135. package/dist/sync/errors.d.ts +61 -0
  136. package/dist/sync/errors.d.ts.map +1 -0
  137. package/dist/sync/errors.js +36 -0
  138. package/dist/sync/errors.js.map +1 -0
  139. package/dist/sync/index.d.ts +3 -0
  140. package/dist/sync/index.d.ts.map +1 -1
  141. package/dist/sync/index.js +3 -0
  142. package/dist/sync/index.js.map +1 -1
  143. package/dist/sync/mock-sync-backend.d.ts +23 -0
  144. package/dist/sync/mock-sync-backend.d.ts.map +1 -0
  145. package/dist/sync/mock-sync-backend.js +114 -0
  146. package/dist/sync/mock-sync-backend.js.map +1 -0
  147. package/dist/sync/next/compact-events.d.ts.map +1 -1
  148. package/dist/sync/next/compact-events.js +4 -5
  149. package/dist/sync/next/compact-events.js.map +1 -1
  150. package/dist/sync/next/facts.d.ts.map +1 -1
  151. package/dist/sync/next/facts.js +1 -2
  152. package/dist/sync/next/facts.js.map +1 -1
  153. package/dist/sync/next/history-dag-common.d.ts +50 -11
  154. package/dist/sync/next/history-dag-common.d.ts.map +1 -1
  155. package/dist/sync/next/history-dag-common.js +193 -4
  156. package/dist/sync/next/history-dag-common.js.map +1 -1
  157. package/dist/sync/next/history-dag.d.ts.map +1 -1
  158. package/dist/sync/next/history-dag.js +3 -1
  159. package/dist/sync/next/history-dag.js.map +1 -1
  160. package/dist/sync/sync-backend-kv.d.ts +7 -0
  161. package/dist/sync/sync-backend-kv.d.ts.map +1 -0
  162. package/dist/sync/sync-backend-kv.js +18 -0
  163. package/dist/sync/sync-backend-kv.js.map +1 -0
  164. package/dist/sync/sync-backend.d.ts +105 -0
  165. package/dist/sync/sync-backend.d.ts.map +1 -0
  166. package/dist/sync/sync-backend.js +61 -0
  167. package/dist/sync/sync-backend.js.map +1 -0
  168. package/dist/sync/sync.d.ts +6 -84
  169. package/dist/sync/sync.d.ts.map +1 -1
  170. package/dist/sync/sync.js +2 -27
  171. package/dist/sync/sync.js.map +1 -1
  172. package/dist/sync/syncstate.test.js +16 -15
  173. package/dist/sync/syncstate.test.js.map +1 -1
  174. package/dist/sync/transport-chunking.d.ts +36 -0
  175. package/dist/sync/transport-chunking.d.ts.map +1 -0
  176. package/dist/sync/transport-chunking.js +56 -0
  177. package/dist/sync/transport-chunking.js.map +1 -0
  178. package/dist/sync/validate-push-payload.d.ts +1 -1
  179. package/dist/sync/validate-push-payload.d.ts.map +1 -1
  180. package/dist/sync/validate-push-payload.js +6 -6
  181. package/dist/sync/validate-push-payload.js.map +1 -1
  182. package/dist/testing/event-factory.d.ts +68 -0
  183. package/dist/testing/event-factory.d.ts.map +1 -0
  184. package/dist/testing/event-factory.js +80 -0
  185. package/dist/testing/event-factory.js.map +1 -0
  186. package/dist/testing/mod.d.ts +2 -0
  187. package/dist/testing/mod.d.ts.map +1 -0
  188. package/dist/testing/mod.js +2 -0
  189. package/dist/testing/mod.js.map +1 -0
  190. package/dist/version.d.ts +2 -2
  191. package/dist/version.d.ts.map +1 -1
  192. package/dist/version.js +2 -2
  193. package/dist/version.js.map +1 -1
  194. package/package.json +7 -8
  195. package/src/ClientSessionLeaderThreadProxy.ts +8 -3
  196. package/src/adapter-types.ts +13 -3
  197. package/src/devtools/devtools-messages-common.ts +1 -8
  198. package/src/errors.ts +33 -4
  199. package/src/leader-thread/LeaderSyncProcessor.ts +204 -63
  200. package/src/leader-thread/eventlog.ts +10 -6
  201. package/src/leader-thread/leader-worker-devtools.ts +6 -2
  202. package/src/leader-thread/make-leader-thread-layer.test.ts +44 -0
  203. package/src/leader-thread/make-leader-thread-layer.ts +137 -26
  204. package/src/leader-thread/materialize-event.ts +34 -9
  205. package/src/leader-thread/recreate-db.ts +11 -3
  206. package/src/leader-thread/shutdown-channel.ts +16 -2
  207. package/src/leader-thread/types.ts +7 -5
  208. package/src/materializer-helper.ts +22 -5
  209. package/src/rematerialize-from-eventlog.ts +33 -23
  210. package/src/schema/EventDef.ts +3 -0
  211. package/src/schema/EventSequenceNumber.test.ts +2 -2
  212. package/src/schema/EventSequenceNumber.ts +5 -2
  213. package/src/schema/LiveStoreEvent.ts +1 -2
  214. package/src/schema/mod.ts +2 -0
  215. package/src/schema/schema.ts +37 -1
  216. package/src/schema/state/sqlite/client-document-def.test.ts +17 -0
  217. package/src/schema/state/sqlite/client-document-def.ts +117 -5
  218. package/src/schema/state/sqlite/column-annotations.ts +16 -6
  219. package/src/schema/state/sqlite/column-def.test.ts +150 -93
  220. package/src/schema/state/sqlite/column-def.ts +136 -203
  221. package/src/schema/state/sqlite/db-schema/ast/sqlite.ts +26 -6
  222. package/src/schema/state/sqlite/db-schema/dsl/mod.ts +2 -1
  223. package/src/schema/state/sqlite/mod.ts +1 -0
  224. package/src/schema/state/sqlite/query-builder/api.ts +7 -2
  225. package/src/schema/state/sqlite/query-builder/impl.test.ts +187 -6
  226. package/src/schema/state/sqlite/query-builder/impl.ts +8 -2
  227. package/src/schema/state/sqlite/system-tables.ts +2 -0
  228. package/src/schema/state/sqlite/table-def.test.ts +112 -2
  229. package/src/schema/state/sqlite/table-def.ts +9 -8
  230. package/src/schema/unknown-events.ts +131 -0
  231. package/src/sql-queries/sql-query-builder.ts +2 -1
  232. package/src/sync/ClientSessionSyncProcessor.ts +56 -49
  233. package/src/sync/errors.ts +38 -0
  234. package/src/sync/index.ts +3 -0
  235. package/src/sync/mock-sync-backend.ts +184 -0
  236. package/src/sync/next/compact-events.ts +4 -5
  237. package/src/sync/next/facts.ts +1 -3
  238. package/src/sync/next/history-dag-common.ts +272 -21
  239. package/src/sync/next/history-dag.ts +3 -1
  240. package/src/sync/sync-backend-kv.ts +22 -0
  241. package/src/sync/sync-backend.ts +185 -0
  242. package/src/sync/sync.ts +6 -89
  243. package/src/sync/syncstate.test.ts +17 -15
  244. package/src/sync/transport-chunking.ts +90 -0
  245. package/src/sync/validate-push-payload.ts +6 -7
  246. package/src/testing/event-factory.ts +133 -0
  247. package/src/testing/mod.ts +1 -0
  248. package/src/version.ts +2 -2
  249. package/dist/schema-management/migrations.test.d.ts +0 -2
  250. package/dist/schema-management/migrations.test.d.ts.map +0 -1
  251. package/dist/schema-management/migrations.test.js +0 -52
  252. package/dist/schema-management/migrations.test.js.map +0 -1
  253. package/dist/sync/next/graphology.d.ts +0 -8
  254. package/dist/sync/next/graphology.d.ts.map +0 -1
  255. package/dist/sync/next/graphology.js +0 -30
  256. package/dist/sync/next/graphology.js.map +0 -1
  257. package/dist/sync/next/graphology_.d.ts +0 -3
  258. package/dist/sync/next/graphology_.d.ts.map +0 -1
  259. package/dist/sync/next/graphology_.js +0 -3
  260. package/dist/sync/next/graphology_.js.map +0 -1
  261. package/src/sync/next/ambient.d.ts +0 -3
  262. package/src/sync/next/graphology.ts +0 -41
  263. package/src/sync/next/graphology_.ts +0 -2
@@ -13,12 +13,13 @@ import {
13
13
  Stream,
14
14
  Subscribable,
15
15
  } from '@livestore/utils/effect'
16
- import * as otel from '@opentelemetry/api'
16
+ import type * as otel from '@opentelemetry/api'
17
17
 
18
- import { type ClientSession, SyncError, type UnexpectedError } from '../adapter-types.ts'
18
+ import { type ClientSession, UnexpectedError } from '../adapter-types.ts'
19
+ import type { MaterializeError } from '../errors.ts'
19
20
  import * as EventSequenceNumber from '../schema/EventSequenceNumber.ts'
20
21
  import * as LiveStoreEvent from '../schema/LiveStoreEvent.ts'
21
- import { getEventDef, type LiveStoreSchema } from '../schema/mod.ts'
22
+ import type { LiveStoreSchema } from '../schema/mod.ts'
22
23
  import * as SyncState from './syncstate.ts'
23
24
 
24
25
  /**
@@ -51,16 +52,19 @@ export const makeClientSessionSyncProcessor = ({
51
52
  clientSession: ClientSession
52
53
  runtime: Runtime.Runtime<Scope.Scope>
53
54
  materializeEvent: (
54
- eventDecoded: LiveStoreEvent.AnyDecoded,
55
- options: { otelContext: otel.Context; withChangeset: boolean; materializerHashLeader: Option.Option<number> },
56
- ) => {
57
- writeTables: Set<string>
58
- sessionChangeset:
59
- | { _tag: 'sessionChangeset'; data: Uint8Array<ArrayBuffer>; debug: any }
60
- | { _tag: 'no-op' }
61
- | { _tag: 'unset' }
62
- materializerHash: Option.Option<number>
63
- }
55
+ eventEncoded: LiveStoreEvent.EncodedWithMeta,
56
+ options: { withChangeset: boolean; materializerHashLeader: Option.Option<number> },
57
+ ) => Effect.Effect<
58
+ {
59
+ writeTables: Set<string>
60
+ sessionChangeset:
61
+ | { _tag: 'sessionChangeset'; data: Uint8Array<ArrayBuffer>; debug: any }
62
+ | { _tag: 'no-op' }
63
+ | { _tag: 'unset' }
64
+ materializerHash: Option.Option<number>
65
+ },
66
+ MaterializeError
67
+ >
64
68
  rollback: (changeset: Uint8Array<ArrayBuffer>) => void
65
69
  refreshTables: (tables: Set<string>) => void
66
70
  span: otel.Span
@@ -91,23 +95,27 @@ export const makeClientSessionSyncProcessor = ({
91
95
  }),
92
96
  }
93
97
 
94
- /** Only used for debugging / observability, it's not relied upon for correctness of the sync processor. */
98
+ /** Only used for debugging / observability / testing, it's not relied upon for correctness of the sync processor. */
95
99
  const syncStateUpdateQueue = Queue.unbounded<SyncState.SyncState>().pipe(Effect.runSync)
96
100
  const isClientEvent = (eventEncoded: LiveStoreEvent.EncodedWithMeta) =>
97
- getEventDef(schema, eventEncoded.name).eventDef.options.clientOnly
101
+ schema.eventsDefsMap.get(eventEncoded.name)?.options.clientOnly ?? false
98
102
 
99
103
  /** We're queuing push requests to reduce the number of messages sent to the leader by batching them */
100
104
  const leaderPushQueue = BucketQueue.make<LiveStoreEvent.EncodedWithMeta>().pipe(Effect.runSync)
101
105
 
102
- const push: ClientSessionSyncProcessor['push'] = (batch, { otelContext }) => {
106
+ const push: ClientSessionSyncProcessor['push'] = Effect.fn('client-session-sync-processor:push')(function* (batch) {
103
107
  // TODO validate batch
104
108
 
105
109
  let baseEventSequenceNumber = syncStateRef.current.localHead
106
110
  const encodedEventDefs = batch.map(({ name, args }) => {
107
- const eventDef = getEventDef(schema, name)
111
+ const eventDef = schema.eventsDefsMap.get(name)
112
+ if (eventDef === undefined) {
113
+ return shouldNeverHappen(`No event definition found for \`${name}\`.`)
114
+ }
108
115
  const nextNumPair = EventSequenceNumber.nextPair({
109
116
  seqNum: baseEventSequenceNumber,
110
- isClient: eventDef.eventDef.options.clientOnly,
117
+ isClient: eventDef.options.clientOnly,
118
+ rebaseGeneration: baseEventSequenceNumber.rebaseGeneration,
111
119
  })
112
120
  baseEventSequenceNumber = nextNumPair.seqNum
113
121
  return new LiveStoreEvent.EncodedWithMeta(
@@ -128,33 +136,35 @@ export const makeClientSessionSyncProcessor = ({
128
136
  isEqualEvent: LiveStoreEvent.isEqualEncoded,
129
137
  })
130
138
 
139
+ yield* Effect.annotateCurrentSpan({
140
+ batchSize: encodedEventDefs.length,
141
+ mergeResultTag: mergeResult._tag,
142
+ eventCounts: encodedEventDefs.reduce<Record<string, number>>((acc, event) => {
143
+ acc[event.name] = (acc[event.name] ?? 0) + 1
144
+ return acc
145
+ }, {}),
146
+ ...(TRACE_VERBOSE && { mergeResult: JSON.stringify(mergeResult) }),
147
+ })
148
+
131
149
  if (mergeResult._tag === 'unexpected-error') {
132
150
  return shouldNeverHappen('Unexpected error in client-session-sync-processor', mergeResult.message)
133
151
  }
134
152
 
135
- span.addEvent('local-push', {
136
- batchSize: encodedEventDefs.length,
137
- mergeResult: TRACE_VERBOSE ? JSON.stringify(mergeResult) : undefined,
138
- })
139
-
140
153
  if (mergeResult._tag !== 'advance') {
141
154
  return shouldNeverHappen(`Expected advance, got ${mergeResult._tag}`)
142
155
  }
143
156
 
144
157
  syncStateRef.current = mergeResult.newSyncState
145
- syncStateUpdateQueue.offer(mergeResult.newSyncState).pipe(Effect.runSync)
158
+ yield* syncStateUpdateQueue.offer(mergeResult.newSyncState)
146
159
 
147
160
  // Materialize events to state
148
161
  const writeTables = new Set<string>()
149
162
  for (const event of mergeResult.newEvents) {
150
- // TODO avoid encoding and decoding here again
151
- const decodedEventDef = Schema.decodeSync(eventSchema)(event)
152
163
  const {
153
164
  writeTables: newWriteTables,
154
165
  sessionChangeset,
155
166
  materializerHash,
156
- } = materializeEvent(decodedEventDef, {
157
- otelContext,
167
+ } = yield* materializeEvent(event, {
158
168
  withChangeset: true,
159
169
  materializerHashLeader: Option.none(),
160
170
  })
@@ -167,10 +177,10 @@ export const makeClientSessionSyncProcessor = ({
167
177
 
168
178
  // Trigger push to leader
169
179
  // console.debug('pushToLeader', encodedEventDefs.length, ...encodedEventDefs.map((_) => _.toJSON()))
170
- BucketQueue.offerAll(leaderPushQueue, encodedEventDefs).pipe(Effect.runSync)
180
+ yield* BucketQueue.offerAll(leaderPushQueue, encodedEventDefs)
171
181
 
172
182
  return { writeTables }
173
- }
183
+ })
174
184
 
175
185
  const debugInfo = {
176
186
  rebaseCount: 0,
@@ -178,8 +188,6 @@ export const makeClientSessionSyncProcessor = ({
178
188
  rejectCount: 0,
179
189
  }
180
190
 
181
- const otelContext = otel.trace.setSpan(otel.context.active(), span)
182
-
183
191
  const boot: ClientSessionSyncProcessor['boot'] = Effect.gen(function* () {
184
192
  if (confirmUnsavedChanges && typeof window !== 'undefined' && typeof window.addEventListener === 'function') {
185
193
  const onBeforeUnload = (event: BeforeUnloadEvent) => {
@@ -229,13 +237,12 @@ export const makeClientSessionSyncProcessor = ({
229
237
  })
230
238
 
231
239
  if (mergeResult._tag === 'unexpected-error') {
232
- return yield* new SyncError({ cause: mergeResult.message })
240
+ return yield* new UnexpectedError({ cause: mergeResult.message })
233
241
  } else if (mergeResult._tag === 'reject') {
234
242
  return shouldNeverHappen('Unexpected reject in client-session-sync-processor', mergeResult)
235
243
  }
236
244
 
237
245
  syncStateRef.current = mergeResult.newSyncState
238
- yield* syncStateUpdateQueue.offer(mergeResult.newSyncState)
239
246
 
240
247
  if (mergeResult._tag === 'rebase') {
241
248
  span.addEvent('merge:pull:rebase', {
@@ -244,7 +251,6 @@ export const makeClientSessionSyncProcessor = ({
244
251
  newEventsCount: mergeResult.newEvents.length,
245
252
  rollbackCount: mergeResult.rollbackEvents.length,
246
253
  res: TRACE_VERBOSE ? JSON.stringify(mergeResult) : undefined,
247
- rebaseGeneration: mergeResult.newSyncState.localHead.rebaseGeneration,
248
254
  })
249
255
 
250
256
  debugInfo.rebaseCount++
@@ -294,18 +300,19 @@ export const makeClientSessionSyncProcessor = ({
294
300
  debugInfo.advanceCount++
295
301
  }
296
302
 
297
- if (mergeResult.newEvents.length === 0) return
303
+ if (mergeResult.newEvents.length === 0) {
304
+ // If there are no new events, we need to update the sync state as well
305
+ yield* syncStateUpdateQueue.offer(mergeResult.newSyncState)
306
+ return
307
+ }
298
308
 
299
309
  const writeTables = new Set<string>()
300
310
  for (const event of mergeResult.newEvents) {
301
- // TODO apply changeset if available (will require tracking of write tables as well)
302
- const decodedEventDef = Schema.decodeSync(eventSchema)(event)
303
311
  const {
304
312
  writeTables: newWriteTables,
305
313
  sessionChangeset,
306
314
  materializerHash,
307
- } = materializeEvent(decodedEventDef, {
308
- otelContext,
315
+ } = yield* materializeEvent(event, {
309
316
  withChangeset: true,
310
317
  materializerHashLeader: event.meta.materializerHashLeader,
311
318
  })
@@ -318,6 +325,9 @@ export const makeClientSessionSyncProcessor = ({
318
325
  }
319
326
 
320
327
  refreshTables(writeTables)
328
+
329
+ // We're only triggering the sync state update after all events have been materialized
330
+ yield* syncStateUpdateQueue.offer(mergeResult.newSyncState)
321
331
  }).pipe(
322
332
  Effect.tapCauseLogPretty,
323
333
  Effect.catchAllCause((cause) => clientSession.shutdown(Exit.failCause(cause))),
@@ -364,10 +374,7 @@ export const makeClientSessionSyncProcessor = ({
364
374
  export interface ClientSessionSyncProcessor {
365
375
  push: (
366
376
  batch: ReadonlyArray<LiveStoreEvent.PartialAnyDecoded>,
367
- options: { otelContext: otel.Context },
368
- ) => {
369
- writeTables: Set<string>
370
- }
377
+ ) => Effect.Effect<{ writeTables: Set<string> }, MaterializeError>
371
378
  boot: Effect.Effect<void, UnexpectedError, Scope.Scope>
372
379
  /**
373
380
  * Only used for debugging / observability.
@@ -388,11 +395,11 @@ const SIMULATION_ENABLED = true
388
395
  // Warning: High values for the simulation params can lead to very long test runs since those get multiplied with the number of events
389
396
  export const ClientSessionSyncProcessorSimulationParams = Schema.Struct({
390
397
  pull: Schema.Struct({
391
- '1_before_leader_push_fiber_interrupt': Schema.Int.pipe(Schema.between(0, 25)),
392
- '2_before_leader_push_queue_clear': Schema.Int.pipe(Schema.between(0, 25)),
393
- '3_before_rebase_rollback': Schema.Int.pipe(Schema.between(0, 25)),
394
- '4_before_leader_push_queue_offer': Schema.Int.pipe(Schema.between(0, 25)),
395
- '5_before_leader_push_fiber_run': Schema.Int.pipe(Schema.between(0, 25)),
398
+ '1_before_leader_push_fiber_interrupt': Schema.Int.pipe(Schema.between(0, 15)),
399
+ '2_before_leader_push_queue_clear': Schema.Int.pipe(Schema.between(0, 15)),
400
+ '3_before_rebase_rollback': Schema.Int.pipe(Schema.between(0, 15)),
401
+ '4_before_leader_push_queue_offer': Schema.Int.pipe(Schema.between(0, 15)),
402
+ '5_before_leader_push_fiber_run': Schema.Int.pipe(Schema.between(0, 15)),
396
403
  }),
397
404
  })
398
405
  type ClientSessionSyncProcessorSimulationParams = typeof ClientSessionSyncProcessorSimulationParams.Type
@@ -0,0 +1,38 @@
1
+ import { Schema } from '@livestore/utils/effect'
2
+ import { UnexpectedError } from '../errors.ts'
3
+ import { EventSequenceNumber } from '../schema/mod.ts'
4
+
5
+ export class IsOfflineError extends Schema.TaggedError<IsOfflineError>()('IsOfflineError', {
6
+ cause: Schema.Defect,
7
+ }) {}
8
+
9
+ /** Unique ID generated by the backend when its created. Used to check whether the backend identity has changed. */
10
+ export const BackendId = Schema.String.annotations({ title: '@livestore/sync-cf:BackendId' })
11
+
12
+ export class BackendIdMismatchError extends Schema.TaggedError<BackendIdMismatchError>()('BackendIdMismatchError', {
13
+ expected: BackendId,
14
+ received: BackendId,
15
+ }) {}
16
+
17
+ export class ServerAheadError extends Schema.TaggedError<ServerAheadError>()('ServerAheadError', {
18
+ minimumExpectedNum: EventSequenceNumber.GlobalEventSequenceNumber,
19
+ providedNum: EventSequenceNumber.GlobalEventSequenceNumber,
20
+ }) {}
21
+
22
+ export class InvalidPushError extends Schema.TaggedError<InvalidPushError>()('InvalidPushError', {
23
+ cause: Schema.Union(UnexpectedError, ServerAheadError, BackendIdMismatchError),
24
+ }) {}
25
+
26
+ export class InvalidPullError extends Schema.TaggedError<InvalidPullError>()('InvalidPullError', {
27
+ cause: Schema.Defect,
28
+ }) {}
29
+
30
+ export class LeaderAheadError extends Schema.TaggedError<LeaderAheadError>()('LeaderAheadError', {
31
+ minimumExpectedNum: EventSequenceNumber.EventSequenceNumber,
32
+ providedNum: EventSequenceNumber.EventSequenceNumber,
33
+ /** Generation number the client session should use for subsequent pushes */
34
+ // nextGeneration: Schema.Number,
35
+ }) {}
36
+
37
+ export const SyncError = Schema.Union(InvalidPushError, InvalidPullError)
38
+ export type SyncError = typeof SyncError.Type
package/src/sync/index.ts CHANGED
@@ -1,3 +1,6 @@
1
1
  export * from './ClientSessionSyncProcessor.ts'
2
+ export * from './mock-sync-backend.ts'
2
3
  export * from './sync.ts'
4
+ export { NetworkStatus } from './sync-backend.ts'
5
+ export * from './transport-chunking.ts'
3
6
  export * from './validate-push-payload.ts'
@@ -0,0 +1,184 @@
1
+ import type { Schema, Scope } from '@livestore/utils/effect'
2
+ import { Effect, Mailbox, Option, Queue, Stream, SubscriptionRef } from '@livestore/utils/effect'
3
+ import { UnexpectedError } from '../errors.ts'
4
+ import { EventSequenceNumber, type LiveStoreEvent } from '../schema/mod.ts'
5
+ import { InvalidPushError } from './errors.ts'
6
+ import * as SyncBackend from './sync-backend.ts'
7
+ import { validatePushPayload } from './validate-push-payload.ts'
8
+
9
+ export interface MockSyncBackend {
10
+ pushedEvents: Stream.Stream<LiveStoreEvent.AnyEncodedGlobal>
11
+ connect: Effect.Effect<void>
12
+ disconnect: Effect.Effect<void>
13
+ makeSyncBackend: Effect.Effect<SyncBackend.SyncBackend, UnexpectedError, Scope.Scope>
14
+ advance: (...batch: LiveStoreEvent.AnyEncodedGlobal[]) => Effect.Effect<void>
15
+ /** Fail the next N push calls with an InvalidPushError (or custom error) */
16
+ failNextPushes: (
17
+ count: number,
18
+ error?: (batch: ReadonlyArray<LiveStoreEvent.AnyEncodedGlobal>) => Effect.Effect<never, InvalidPushError>,
19
+ ) => Effect.Effect<void>
20
+ }
21
+
22
+ export interface MockSyncBackendOptions {
23
+ /** Chunk size for non-live pulls; defaults to 100 */
24
+ nonLiveChunkSize?: number
25
+ /** Initial connected state; defaults to false */
26
+ startConnected?: boolean
27
+ // TODO add a "flaky" mode to simulate transient network / server failures for pull/push
28
+ }
29
+
30
+ export const makeMockSyncBackend = (
31
+ options?: MockSyncBackendOptions,
32
+ ): Effect.Effect<MockSyncBackend, UnexpectedError, Scope.Scope> =>
33
+ Effect.gen(function* () {
34
+ const syncEventSequenceNumberRef = { current: EventSequenceNumber.ROOT.global }
35
+ const syncPullQueue = yield* Queue.unbounded<LiveStoreEvent.AnyEncodedGlobal>()
36
+ const pushedEventsQueue = yield* Mailbox.make<LiveStoreEvent.AnyEncodedGlobal>()
37
+ const syncIsConnectedRef = yield* SubscriptionRef.make(options?.startConnected ?? false)
38
+ const allEventsRef: { current: LiveStoreEvent.AnyEncodedGlobal[] } = { current: [] }
39
+
40
+ const span = yield* Effect.currentSpan.pipe(Effect.orDie)
41
+
42
+ const semaphore = yield* Effect.makeSemaphore(1)
43
+
44
+ // TODO improve the API and implementation of simulating errors
45
+ const failCounterRef = yield* SubscriptionRef.make(0)
46
+ const failEffectRef = yield* SubscriptionRef.make<
47
+ ((batch: ReadonlyArray<LiveStoreEvent.AnyEncodedGlobal>) => Effect.Effect<never, InvalidPushError>) | undefined
48
+ >(undefined)
49
+
50
+ const makeSyncBackend = Effect.gen(function* () {
51
+ const nonLiveChunkSize = Math.max(1, options?.nonLiveChunkSize ?? 100)
52
+
53
+ // TODO consider making offline state actively error pull/push.
54
+ // Currently, offline only reflects in `isConnected`, while operations still succeed,
55
+ // mirroring how some real providers behave during transient disconnects.
56
+ return SyncBackend.of<Schema.JsonValue>({
57
+ isConnected: syncIsConnectedRef,
58
+ connect: SubscriptionRef.set(syncIsConnectedRef, true),
59
+ ping: Effect.void,
60
+ pull: (cursor, options) =>
61
+ (options?.live
62
+ ? Stream.concat(
63
+ Stream.make(SyncBackend.pullResItemEmpty()),
64
+ Stream.fromQueue(syncPullQueue).pipe(
65
+ Stream.chunks,
66
+ Stream.map((chunk) => ({
67
+ batch: [...chunk].map((eventEncoded) => ({ eventEncoded, metadata: Option.none() })),
68
+ pageInfo: SyncBackend.pageInfoNoMore,
69
+ })),
70
+ ),
71
+ )
72
+ : Stream.fromEffect(
73
+ Effect.sync(() => {
74
+ const lastSeen = cursor.pipe(
75
+ Option.match({
76
+ onNone: () => EventSequenceNumber.ROOT.global,
77
+ onSome: (_) => _.eventSequenceNumber,
78
+ }),
79
+ )
80
+ // All events with seqNum greater than lastSeen
81
+ const slice = allEventsRef.current.filter((e) => e.seqNum > lastSeen)
82
+ // Split into configured chunk size
83
+ const chunks: { events: LiveStoreEvent.AnyEncodedGlobal[]; remaining: number }[] = []
84
+ for (let i = 0; i < slice.length; i += nonLiveChunkSize) {
85
+ const end = Math.min(i + nonLiveChunkSize, slice.length)
86
+ const remaining = Math.max(slice.length - end, 0)
87
+ chunks.push({ events: slice.slice(i, end), remaining })
88
+ }
89
+ if (chunks.length === 0) {
90
+ chunks.push({ events: [], remaining: 0 })
91
+ }
92
+ return chunks
93
+ }),
94
+ ).pipe(
95
+ Stream.flatMap((chunks) =>
96
+ Stream.fromIterable(chunks).pipe(
97
+ Stream.map(({ events, remaining }) => ({
98
+ batch: events.map((eventEncoded) => ({ eventEncoded, metadata: Option.none() })),
99
+ pageInfo: remaining > 0 ? SyncBackend.pageInfoMoreKnown(remaining) : SyncBackend.pageInfoNoMore,
100
+ })),
101
+ ),
102
+ ),
103
+ )
104
+ ).pipe(Stream.withSpan('MockSyncBackend:pull', { parent: span })),
105
+ push: (batch) =>
106
+ Effect.gen(function* () {
107
+ yield* validatePushPayload(batch, syncEventSequenceNumberRef.current)
108
+
109
+ const remaining = yield* SubscriptionRef.get(failCounterRef)
110
+ if (remaining > 0) {
111
+ const maybeFail = yield* SubscriptionRef.get(failEffectRef)
112
+ // decrement counter first
113
+ yield* SubscriptionRef.set(failCounterRef, remaining - 1)
114
+ if (maybeFail) {
115
+ return yield* maybeFail(batch)
116
+ }
117
+ return yield* new InvalidPushError({
118
+ cause: new UnexpectedError({ cause: new Error('MockSyncBackend: simulated push failure') }),
119
+ })
120
+ }
121
+
122
+ yield* Effect.sleep(10).pipe(Effect.withSpan('MockSyncBackend:push:sleep')) // Simulate network latency
123
+
124
+ yield* pushedEventsQueue.offerAll(batch)
125
+ yield* syncPullQueue.offerAll(batch)
126
+ allEventsRef.current = allEventsRef.current.concat(batch)
127
+
128
+ syncEventSequenceNumberRef.current = batch.at(-1)!.seqNum
129
+ }).pipe(
130
+ Effect.withSpan('MockSyncBackend:push', {
131
+ parent: span,
132
+ attributes: {
133
+ nums: batch.map((_) => _.seqNum),
134
+ },
135
+ }),
136
+ semaphore.withPermits(1),
137
+ ),
138
+ metadata: {
139
+ name: '@livestore/mock-sync',
140
+ description: 'Just a mock sync backend',
141
+ },
142
+ supports: {
143
+ pullPageInfoKnown: true,
144
+ pullLive: true,
145
+ },
146
+ })
147
+ })
148
+
149
+ const advance = (...batch: LiveStoreEvent.AnyEncodedGlobal[]) =>
150
+ Effect.gen(function* () {
151
+ syncEventSequenceNumberRef.current = batch.at(-1)!.seqNum
152
+ allEventsRef.current = allEventsRef.current.concat(batch)
153
+ yield* syncPullQueue.offerAll(batch)
154
+ }).pipe(
155
+ Effect.withSpan('MockSyncBackend:advance', {
156
+ parent: span,
157
+ attributes: { nums: batch.map((_) => _.seqNum) },
158
+ }),
159
+ semaphore.withPermits(1),
160
+ )
161
+
162
+ const connect = SubscriptionRef.set(syncIsConnectedRef, true)
163
+ const disconnect = SubscriptionRef.set(syncIsConnectedRef, false)
164
+
165
+ const failNextPushes = (
166
+ count: number,
167
+ error?: (batch: ReadonlyArray<LiveStoreEvent.AnyEncodedGlobal>) => Effect.Effect<never, InvalidPushError>,
168
+ ) =>
169
+ Effect.gen(function* () {
170
+ yield* SubscriptionRef.set(failCounterRef, count)
171
+ yield* SubscriptionRef.set(failEffectRef, error)
172
+ })
173
+
174
+ return {
175
+ syncEventSequenceNumberRef,
176
+ syncPullQueue,
177
+ pushedEvents: Mailbox.toStream(pushedEventsQueue),
178
+ connect,
179
+ disconnect,
180
+ makeSyncBackend,
181
+ advance,
182
+ failNextPushes,
183
+ }
184
+ }).pipe(Effect.withSpanScoped('MockSyncBackend'))
@@ -1,6 +1,5 @@
1
1
  import { EventSequenceNumber } from '../../schema/mod.ts'
2
2
  import { replacesFacts } from './facts.ts'
3
- import { graphologyDag } from './graphology_.ts'
4
3
  import type { HistoryDag } from './history-dag-common.ts'
5
4
  import { emptyHistoryDag } from './history-dag-common.ts'
6
5
 
@@ -17,7 +16,7 @@ export const compactEvents = (inputDag: HistoryDag): { dag: HistoryDag; compacte
17
16
  const dag = inputDag.copy()
18
17
  const compactedEventCount = 0
19
18
 
20
- const orderedEventSequenceNumberStrs = graphologyDag.topologicalSort(dag).reverse()
19
+ const orderedEventSequenceNumberStrs = dag.topologicalNodeIds().reverse()
21
20
 
22
21
  // drop root
23
22
  orderedEventSequenceNumberStrs.pop()
@@ -116,7 +115,7 @@ const findSubDagsInHistory = (
116
115
  const subDags: HistoryDag[] = []
117
116
  const allOutsideDependencies: string[][] = []
118
117
 
119
- for (const eventNumStr of graphologyDag.topologicalSort(inputDag)) {
118
+ for (const eventNumStr of inputDag.topologicalNodeIds()) {
120
119
  if (eventNumStr === upToExclEventSequenceNumberStr) {
121
120
  break
122
121
  }
@@ -189,8 +188,8 @@ const dagReplacesDag = (dagA: HistoryDag, dagB: HistoryDag): boolean => {
189
188
  }
190
189
 
191
190
  // TODO write tests that covers deterministic order when DAGs have branches
192
- const nodeEntriesA = graphologyDag.topologicalSort(dagA).map((nodeId) => dagA.getNodeAttributes(nodeId))
193
- const nodeEntriesB = graphologyDag.topologicalSort(dagB).map((nodeId) => dagB.getNodeAttributes(nodeId))
191
+ const nodeEntriesA = dagA.topologicalNodeIds().map((nodeId) => dagA.getNodeAttributes(nodeId))
192
+ const nodeEntriesB = dagB.topologicalNodeIds().map((nodeId) => dagB.getNodeAttributes(nodeId))
194
193
 
195
194
  for (let i = 0; i < nodeEntriesA.length; i++) {
196
195
  const nodeA = nodeEntriesA[i]!
@@ -1,5 +1,4 @@
1
1
  import { notYetImplemented } from '@livestore/utils'
2
-
3
2
  import type {
4
3
  EventDefFactInput,
5
4
  EventDefFacts,
@@ -8,7 +7,6 @@ import type {
8
7
  FactsCallback,
9
8
  } from '../../schema/EventDef.ts'
10
9
  import type * as EventSequenceNumber from '../../schema/EventSequenceNumber.ts'
11
- import { graphologyDag } from './graphology_.ts'
12
10
  import { EMPTY_FACT_VALUE, type HistoryDag, type HistoryDagNode } from './history-dag-common.ts'
13
11
 
14
12
  export const factsSnapshotForEvents = (
@@ -34,7 +32,7 @@ export const factsSnapshotForDag = (
34
32
  ): EventDefFactsSnapshot => {
35
33
  const facts = new Map<string, any>()
36
34
 
37
- const orderedEventSequenceNumberStrs = graphologyDag.topologicalSort(dag)
35
+ const orderedEventSequenceNumberStrs = dag.topologicalNodeIds()
38
36
 
39
37
  for (let i = 0; i < orderedEventSequenceNumberStrs.length; i++) {
40
38
  const event = dag.getNodeAttributes(orderedEventSequenceNumberStrs[i]!)