@livestore/common 0.4.0-dev.1 → 0.4.0-dev.10

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (253) hide show
  1. package/dist/.tsbuildinfo +1 -1
  2. package/dist/ClientSessionLeaderThreadProxy.d.ts +7 -2
  3. package/dist/ClientSessionLeaderThreadProxy.d.ts.map +1 -1
  4. package/dist/ClientSessionLeaderThreadProxy.js.map +1 -1
  5. package/dist/adapter-types.d.ts +9 -3
  6. package/dist/adapter-types.d.ts.map +1 -1
  7. package/dist/adapter-types.js.map +1 -1
  8. package/dist/devtools/devtools-messages-client-session.d.ts +21 -21
  9. package/dist/devtools/devtools-messages-common.d.ts +7 -14
  10. package/dist/devtools/devtools-messages-common.d.ts.map +1 -1
  11. package/dist/devtools/devtools-messages-common.js +1 -6
  12. package/dist/devtools/devtools-messages-common.js.map +1 -1
  13. package/dist/devtools/devtools-messages-leader.d.ts +27 -25
  14. package/dist/devtools/devtools-messages-leader.d.ts.map +1 -1
  15. package/dist/errors.d.ts +47 -5
  16. package/dist/errors.d.ts.map +1 -1
  17. package/dist/errors.js +22 -3
  18. package/dist/errors.js.map +1 -1
  19. package/dist/leader-thread/LeaderSyncProcessor.d.ts +7 -3
  20. package/dist/leader-thread/LeaderSyncProcessor.d.ts.map +1 -1
  21. package/dist/leader-thread/LeaderSyncProcessor.js +122 -49
  22. package/dist/leader-thread/LeaderSyncProcessor.js.map +1 -1
  23. package/dist/leader-thread/eventlog.d.ts +4 -10
  24. package/dist/leader-thread/eventlog.d.ts.map +1 -1
  25. package/dist/leader-thread/eventlog.js +4 -6
  26. package/dist/leader-thread/eventlog.js.map +1 -1
  27. package/dist/leader-thread/leader-worker-devtools.d.ts +1 -1
  28. package/dist/leader-thread/leader-worker-devtools.js +6 -2
  29. package/dist/leader-thread/leader-worker-devtools.js.map +1 -1
  30. package/dist/leader-thread/make-leader-thread-layer.d.ts +1 -2
  31. package/dist/leader-thread/make-leader-thread-layer.d.ts.map +1 -1
  32. package/dist/leader-thread/make-leader-thread-layer.js +68 -19
  33. package/dist/leader-thread/make-leader-thread-layer.js.map +1 -1
  34. package/dist/leader-thread/make-leader-thread-layer.test.d.ts +2 -0
  35. package/dist/leader-thread/make-leader-thread-layer.test.d.ts.map +1 -0
  36. package/dist/leader-thread/make-leader-thread-layer.test.js +32 -0
  37. package/dist/leader-thread/make-leader-thread-layer.test.js.map +1 -0
  38. package/dist/leader-thread/materialize-event.d.ts +2 -2
  39. package/dist/leader-thread/materialize-event.d.ts.map +1 -1
  40. package/dist/leader-thread/materialize-event.js +23 -9
  41. package/dist/leader-thread/materialize-event.js.map +1 -1
  42. package/dist/leader-thread/recreate-db.d.ts +2 -3
  43. package/dist/leader-thread/recreate-db.d.ts.map +1 -1
  44. package/dist/leader-thread/recreate-db.js +1 -1
  45. package/dist/leader-thread/recreate-db.js.map +1 -1
  46. package/dist/leader-thread/shutdown-channel.d.ts +2 -2
  47. package/dist/leader-thread/shutdown-channel.d.ts.map +1 -1
  48. package/dist/leader-thread/shutdown-channel.js +2 -2
  49. package/dist/leader-thread/shutdown-channel.js.map +1 -1
  50. package/dist/leader-thread/types.d.ts +7 -5
  51. package/dist/leader-thread/types.d.ts.map +1 -1
  52. package/dist/leader-thread/types.js.map +1 -1
  53. package/dist/materializer-helper.d.ts +1 -1
  54. package/dist/materializer-helper.d.ts.map +1 -1
  55. package/dist/materializer-helper.js +20 -4
  56. package/dist/materializer-helper.js.map +1 -1
  57. package/dist/rematerialize-from-eventlog.d.ts +1 -1
  58. package/dist/rematerialize-from-eventlog.d.ts.map +1 -1
  59. package/dist/rematerialize-from-eventlog.js +25 -16
  60. package/dist/rematerialize-from-eventlog.js.map +1 -1
  61. package/dist/schema/EventDef.d.ts +3 -0
  62. package/dist/schema/EventDef.d.ts.map +1 -1
  63. package/dist/schema/EventDef.js.map +1 -1
  64. package/dist/schema/LiveStoreEvent.d.ts +1 -1
  65. package/dist/schema/LiveStoreEvent.d.ts.map +1 -1
  66. package/dist/schema/LiveStoreEvent.js +1 -2
  67. package/dist/schema/LiveStoreEvent.js.map +1 -1
  68. package/dist/schema/mod.d.ts +2 -0
  69. package/dist/schema/mod.d.ts.map +1 -1
  70. package/dist/schema/mod.js +1 -0
  71. package/dist/schema/mod.js.map +1 -1
  72. package/dist/schema/schema.d.ts +15 -0
  73. package/dist/schema/schema.d.ts.map +1 -1
  74. package/dist/schema/schema.js +26 -1
  75. package/dist/schema/schema.js.map +1 -1
  76. package/dist/schema/state/sqlite/client-document-def.d.ts +35 -5
  77. package/dist/schema/state/sqlite/client-document-def.d.ts.map +1 -1
  78. package/dist/schema/state/sqlite/client-document-def.js +95 -4
  79. package/dist/schema/state/sqlite/client-document-def.js.map +1 -1
  80. package/dist/schema/state/sqlite/client-document-def.test.js +16 -0
  81. package/dist/schema/state/sqlite/client-document-def.test.js.map +1 -1
  82. package/dist/schema/state/sqlite/column-annotations.d.ts.map +1 -1
  83. package/dist/schema/state/sqlite/column-annotations.js +14 -6
  84. package/dist/schema/state/sqlite/column-annotations.js.map +1 -1
  85. package/dist/schema/state/sqlite/column-def.d.ts +6 -2
  86. package/dist/schema/state/sqlite/column-def.d.ts.map +1 -1
  87. package/dist/schema/state/sqlite/column-def.js +122 -185
  88. package/dist/schema/state/sqlite/column-def.js.map +1 -1
  89. package/dist/schema/state/sqlite/column-def.test.js +116 -73
  90. package/dist/schema/state/sqlite/column-def.test.js.map +1 -1
  91. package/dist/schema/state/sqlite/db-schema/ast/sqlite.d.ts +2 -1
  92. package/dist/schema/state/sqlite/db-schema/ast/sqlite.d.ts.map +1 -1
  93. package/dist/schema/state/sqlite/db-schema/ast/sqlite.js +23 -6
  94. package/dist/schema/state/sqlite/db-schema/ast/sqlite.js.map +1 -1
  95. package/dist/schema/state/sqlite/db-schema/dsl/mod.d.ts.map +1 -1
  96. package/dist/schema/state/sqlite/db-schema/dsl/mod.js +2 -1
  97. package/dist/schema/state/sqlite/db-schema/dsl/mod.js.map +1 -1
  98. package/dist/schema/state/sqlite/mod.d.ts +1 -1
  99. package/dist/schema/state/sqlite/mod.d.ts.map +1 -1
  100. package/dist/schema/state/sqlite/mod.js +1 -1
  101. package/dist/schema/state/sqlite/mod.js.map +1 -1
  102. package/dist/schema/state/sqlite/query-builder/api.d.ts +5 -2
  103. package/dist/schema/state/sqlite/query-builder/api.d.ts.map +1 -1
  104. package/dist/schema/state/sqlite/query-builder/impl.d.ts.map +1 -1
  105. package/dist/schema/state/sqlite/query-builder/impl.js +6 -2
  106. package/dist/schema/state/sqlite/query-builder/impl.js.map +1 -1
  107. package/dist/schema/state/sqlite/query-builder/impl.test.js +137 -2
  108. package/dist/schema/state/sqlite/query-builder/impl.test.js.map +1 -1
  109. package/dist/schema/state/sqlite/system-tables.d.ts +42 -6
  110. package/dist/schema/state/sqlite/system-tables.d.ts.map +1 -1
  111. package/dist/schema/state/sqlite/system-tables.js +2 -0
  112. package/dist/schema/state/sqlite/system-tables.js.map +1 -1
  113. package/dist/schema/state/sqlite/table-def.d.ts +4 -4
  114. package/dist/schema/state/sqlite/table-def.d.ts.map +1 -1
  115. package/dist/schema/state/sqlite/table-def.js +2 -2
  116. package/dist/schema/state/sqlite/table-def.js.map +1 -1
  117. package/dist/schema/state/sqlite/table-def.test.js +51 -2
  118. package/dist/schema/state/sqlite/table-def.test.js.map +1 -1
  119. package/dist/schema/unknown-events.d.ts +47 -0
  120. package/dist/schema/unknown-events.d.ts.map +1 -0
  121. package/dist/schema/unknown-events.js +69 -0
  122. package/dist/schema/unknown-events.js.map +1 -0
  123. package/dist/sql-queries/sql-query-builder.d.ts.map +1 -1
  124. package/dist/sql-queries/sql-query-builder.js +2 -1
  125. package/dist/sql-queries/sql-query-builder.js.map +1 -1
  126. package/dist/sync/ClientSessionSyncProcessor.d.ts +9 -11
  127. package/dist/sync/ClientSessionSyncProcessor.d.ts.map +1 -1
  128. package/dist/sync/ClientSessionSyncProcessor.js +35 -33
  129. package/dist/sync/ClientSessionSyncProcessor.js.map +1 -1
  130. package/dist/sync/errors.d.ts +61 -0
  131. package/dist/sync/errors.d.ts.map +1 -0
  132. package/dist/sync/errors.js +36 -0
  133. package/dist/sync/errors.js.map +1 -0
  134. package/dist/sync/index.d.ts +3 -0
  135. package/dist/sync/index.d.ts.map +1 -1
  136. package/dist/sync/index.js +3 -0
  137. package/dist/sync/index.js.map +1 -1
  138. package/dist/sync/mock-sync-backend.d.ts +23 -0
  139. package/dist/sync/mock-sync-backend.d.ts.map +1 -0
  140. package/dist/sync/mock-sync-backend.js +114 -0
  141. package/dist/sync/mock-sync-backend.js.map +1 -0
  142. package/dist/sync/next/compact-events.d.ts.map +1 -1
  143. package/dist/sync/next/compact-events.js +4 -5
  144. package/dist/sync/next/compact-events.js.map +1 -1
  145. package/dist/sync/next/facts.d.ts.map +1 -1
  146. package/dist/sync/next/facts.js +1 -2
  147. package/dist/sync/next/facts.js.map +1 -1
  148. package/dist/sync/next/history-dag-common.d.ts +50 -11
  149. package/dist/sync/next/history-dag-common.d.ts.map +1 -1
  150. package/dist/sync/next/history-dag-common.js +193 -4
  151. package/dist/sync/next/history-dag-common.js.map +1 -1
  152. package/dist/sync/next/history-dag.d.ts.map +1 -1
  153. package/dist/sync/next/history-dag.js +3 -1
  154. package/dist/sync/next/history-dag.js.map +1 -1
  155. package/dist/sync/sync-backend-kv.d.ts +7 -0
  156. package/dist/sync/sync-backend-kv.d.ts.map +1 -0
  157. package/dist/sync/sync-backend-kv.js +18 -0
  158. package/dist/sync/sync-backend-kv.js.map +1 -0
  159. package/dist/sync/sync-backend.d.ts +105 -0
  160. package/dist/sync/sync-backend.d.ts.map +1 -0
  161. package/dist/sync/sync-backend.js +61 -0
  162. package/dist/sync/sync-backend.js.map +1 -0
  163. package/dist/sync/sync.d.ts +6 -84
  164. package/dist/sync/sync.d.ts.map +1 -1
  165. package/dist/sync/sync.js +2 -27
  166. package/dist/sync/sync.js.map +1 -1
  167. package/dist/sync/transport-chunking.d.ts +36 -0
  168. package/dist/sync/transport-chunking.d.ts.map +1 -0
  169. package/dist/sync/transport-chunking.js +56 -0
  170. package/dist/sync/transport-chunking.js.map +1 -0
  171. package/dist/sync/validate-push-payload.d.ts +1 -1
  172. package/dist/sync/validate-push-payload.d.ts.map +1 -1
  173. package/dist/sync/validate-push-payload.js +6 -6
  174. package/dist/sync/validate-push-payload.js.map +1 -1
  175. package/dist/testing/event-factory.d.ts +68 -0
  176. package/dist/testing/event-factory.d.ts.map +1 -0
  177. package/dist/testing/event-factory.js +80 -0
  178. package/dist/testing/event-factory.js.map +1 -0
  179. package/dist/testing/mod.d.ts +2 -0
  180. package/dist/testing/mod.d.ts.map +1 -0
  181. package/dist/testing/mod.js +2 -0
  182. package/dist/testing/mod.js.map +1 -0
  183. package/dist/version.d.ts +2 -2
  184. package/dist/version.d.ts.map +1 -1
  185. package/dist/version.js +2 -2
  186. package/dist/version.js.map +1 -1
  187. package/package.json +7 -8
  188. package/src/ClientSessionLeaderThreadProxy.ts +7 -2
  189. package/src/adapter-types.ts +13 -3
  190. package/src/devtools/devtools-messages-common.ts +1 -8
  191. package/src/errors.ts +33 -4
  192. package/src/leader-thread/LeaderSyncProcessor.ts +179 -57
  193. package/src/leader-thread/eventlog.ts +10 -6
  194. package/src/leader-thread/leader-worker-devtools.ts +6 -2
  195. package/src/leader-thread/make-leader-thread-layer.test.ts +44 -0
  196. package/src/leader-thread/make-leader-thread-layer.ts +137 -26
  197. package/src/leader-thread/materialize-event.ts +34 -9
  198. package/src/leader-thread/recreate-db.ts +11 -3
  199. package/src/leader-thread/shutdown-channel.ts +16 -2
  200. package/src/leader-thread/types.ts +7 -5
  201. package/src/materializer-helper.ts +22 -5
  202. package/src/rematerialize-from-eventlog.ts +33 -23
  203. package/src/schema/EventDef.ts +3 -0
  204. package/src/schema/LiveStoreEvent.ts +1 -2
  205. package/src/schema/mod.ts +2 -0
  206. package/src/schema/schema.ts +37 -1
  207. package/src/schema/state/sqlite/client-document-def.test.ts +17 -0
  208. package/src/schema/state/sqlite/client-document-def.ts +117 -5
  209. package/src/schema/state/sqlite/column-annotations.ts +16 -6
  210. package/src/schema/state/sqlite/column-def.test.ts +150 -93
  211. package/src/schema/state/sqlite/column-def.ts +128 -203
  212. package/src/schema/state/sqlite/db-schema/ast/sqlite.ts +26 -6
  213. package/src/schema/state/sqlite/db-schema/dsl/mod.ts +2 -1
  214. package/src/schema/state/sqlite/mod.ts +1 -0
  215. package/src/schema/state/sqlite/query-builder/api.ts +7 -2
  216. package/src/schema/state/sqlite/query-builder/impl.test.ts +187 -6
  217. package/src/schema/state/sqlite/query-builder/impl.ts +8 -2
  218. package/src/schema/state/sqlite/system-tables.ts +2 -0
  219. package/src/schema/state/sqlite/table-def.test.ts +64 -2
  220. package/src/schema/state/sqlite/table-def.ts +9 -8
  221. package/src/schema/unknown-events.ts +131 -0
  222. package/src/sql-queries/sql-query-builder.ts +2 -1
  223. package/src/sync/ClientSessionSyncProcessor.ts +55 -49
  224. package/src/sync/errors.ts +38 -0
  225. package/src/sync/index.ts +3 -0
  226. package/src/sync/mock-sync-backend.ts +184 -0
  227. package/src/sync/next/compact-events.ts +4 -5
  228. package/src/sync/next/facts.ts +1 -3
  229. package/src/sync/next/history-dag-common.ts +272 -21
  230. package/src/sync/next/history-dag.ts +3 -1
  231. package/src/sync/sync-backend-kv.ts +22 -0
  232. package/src/sync/sync-backend.ts +185 -0
  233. package/src/sync/sync.ts +6 -89
  234. package/src/sync/transport-chunking.ts +90 -0
  235. package/src/sync/validate-push-payload.ts +6 -7
  236. package/src/testing/event-factory.ts +133 -0
  237. package/src/testing/mod.ts +1 -0
  238. package/src/version.ts +2 -2
  239. package/dist/schema-management/migrations.test.d.ts +0 -2
  240. package/dist/schema-management/migrations.test.d.ts.map +0 -1
  241. package/dist/schema-management/migrations.test.js +0 -52
  242. package/dist/schema-management/migrations.test.js.map +0 -1
  243. package/dist/sync/next/graphology.d.ts +0 -8
  244. package/dist/sync/next/graphology.d.ts.map +0 -1
  245. package/dist/sync/next/graphology.js +0 -30
  246. package/dist/sync/next/graphology.js.map +0 -1
  247. package/dist/sync/next/graphology_.d.ts +0 -3
  248. package/dist/sync/next/graphology_.d.ts.map +0 -1
  249. package/dist/sync/next/graphology_.js +0 -3
  250. package/dist/sync/next/graphology_.js.map +0 -1
  251. package/src/sync/next/ambient.d.ts +0 -3
  252. package/src/sync/next/graphology.ts +0 -41
  253. package/src/sync/next/graphology_.ts +0 -2
@@ -13,12 +13,13 @@ import {
13
13
  Stream,
14
14
  Subscribable,
15
15
  } from '@livestore/utils/effect'
16
- import * as otel from '@opentelemetry/api'
16
+ import type * as otel from '@opentelemetry/api'
17
17
 
18
- import { type ClientSession, SyncError, type UnexpectedError } from '../adapter-types.ts'
18
+ import { type ClientSession, UnexpectedError } from '../adapter-types.ts'
19
+ import type { MaterializeError } from '../errors.ts'
19
20
  import * as EventSequenceNumber from '../schema/EventSequenceNumber.ts'
20
21
  import * as LiveStoreEvent from '../schema/LiveStoreEvent.ts'
21
- import { getEventDef, type LiveStoreSchema } from '../schema/mod.ts'
22
+ import type { LiveStoreSchema } from '../schema/mod.ts'
22
23
  import * as SyncState from './syncstate.ts'
23
24
 
24
25
  /**
@@ -51,16 +52,19 @@ export const makeClientSessionSyncProcessor = ({
51
52
  clientSession: ClientSession
52
53
  runtime: Runtime.Runtime<Scope.Scope>
53
54
  materializeEvent: (
54
- eventDecoded: LiveStoreEvent.AnyDecoded,
55
- options: { otelContext: otel.Context; withChangeset: boolean; materializerHashLeader: Option.Option<number> },
56
- ) => {
57
- writeTables: Set<string>
58
- sessionChangeset:
59
- | { _tag: 'sessionChangeset'; data: Uint8Array<ArrayBuffer>; debug: any }
60
- | { _tag: 'no-op' }
61
- | { _tag: 'unset' }
62
- materializerHash: Option.Option<number>
63
- }
55
+ eventEncoded: LiveStoreEvent.EncodedWithMeta,
56
+ options: { withChangeset: boolean; materializerHashLeader: Option.Option<number> },
57
+ ) => Effect.Effect<
58
+ {
59
+ writeTables: Set<string>
60
+ sessionChangeset:
61
+ | { _tag: 'sessionChangeset'; data: Uint8Array<ArrayBuffer>; debug: any }
62
+ | { _tag: 'no-op' }
63
+ | { _tag: 'unset' }
64
+ materializerHash: Option.Option<number>
65
+ },
66
+ MaterializeError
67
+ >
64
68
  rollback: (changeset: Uint8Array<ArrayBuffer>) => void
65
69
  refreshTables: (tables: Set<string>) => void
66
70
  span: otel.Span
@@ -91,23 +95,26 @@ export const makeClientSessionSyncProcessor = ({
91
95
  }),
92
96
  }
93
97
 
94
- /** Only used for debugging / observability, it's not relied upon for correctness of the sync processor. */
98
+ /** Only used for debugging / observability / testing, it's not relied upon for correctness of the sync processor. */
95
99
  const syncStateUpdateQueue = Queue.unbounded<SyncState.SyncState>().pipe(Effect.runSync)
96
100
  const isClientEvent = (eventEncoded: LiveStoreEvent.EncodedWithMeta) =>
97
- getEventDef(schema, eventEncoded.name).eventDef.options.clientOnly
101
+ schema.eventsDefsMap.get(eventEncoded.name)?.options.clientOnly ?? false
98
102
 
99
103
  /** We're queuing push requests to reduce the number of messages sent to the leader by batching them */
100
104
  const leaderPushQueue = BucketQueue.make<LiveStoreEvent.EncodedWithMeta>().pipe(Effect.runSync)
101
105
 
102
- const push: ClientSessionSyncProcessor['push'] = (batch, { otelContext }) => {
106
+ const push: ClientSessionSyncProcessor['push'] = Effect.fn('client-session-sync-processor:push')(function* (batch) {
103
107
  // TODO validate batch
104
108
 
105
109
  let baseEventSequenceNumber = syncStateRef.current.localHead
106
110
  const encodedEventDefs = batch.map(({ name, args }) => {
107
- const eventDef = getEventDef(schema, name)
111
+ const eventDef = schema.eventsDefsMap.get(name)
112
+ if (eventDef === undefined) {
113
+ return shouldNeverHappen(`No event definition found for \`${name}\`.`)
114
+ }
108
115
  const nextNumPair = EventSequenceNumber.nextPair({
109
116
  seqNum: baseEventSequenceNumber,
110
- isClient: eventDef.eventDef.options.clientOnly,
117
+ isClient: eventDef.options.clientOnly,
111
118
  })
112
119
  baseEventSequenceNumber = nextNumPair.seqNum
113
120
  return new LiveStoreEvent.EncodedWithMeta(
@@ -128,33 +135,35 @@ export const makeClientSessionSyncProcessor = ({
128
135
  isEqualEvent: LiveStoreEvent.isEqualEncoded,
129
136
  })
130
137
 
138
+ yield* Effect.annotateCurrentSpan({
139
+ batchSize: encodedEventDefs.length,
140
+ mergeResultTag: mergeResult._tag,
141
+ eventCounts: encodedEventDefs.reduce<Record<string, number>>((acc, event) => {
142
+ acc[event.name] = (acc[event.name] ?? 0) + 1
143
+ return acc
144
+ }, {}),
145
+ ...(TRACE_VERBOSE && { mergeResult: JSON.stringify(mergeResult) }),
146
+ })
147
+
131
148
  if (mergeResult._tag === 'unexpected-error') {
132
149
  return shouldNeverHappen('Unexpected error in client-session-sync-processor', mergeResult.message)
133
150
  }
134
151
 
135
- span.addEvent('local-push', {
136
- batchSize: encodedEventDefs.length,
137
- mergeResult: TRACE_VERBOSE ? JSON.stringify(mergeResult) : undefined,
138
- })
139
-
140
152
  if (mergeResult._tag !== 'advance') {
141
153
  return shouldNeverHappen(`Expected advance, got ${mergeResult._tag}`)
142
154
  }
143
155
 
144
156
  syncStateRef.current = mergeResult.newSyncState
145
- syncStateUpdateQueue.offer(mergeResult.newSyncState).pipe(Effect.runSync)
157
+ yield* syncStateUpdateQueue.offer(mergeResult.newSyncState)
146
158
 
147
159
  // Materialize events to state
148
160
  const writeTables = new Set<string>()
149
161
  for (const event of mergeResult.newEvents) {
150
- // TODO avoid encoding and decoding here again
151
- const decodedEventDef = Schema.decodeSync(eventSchema)(event)
152
162
  const {
153
163
  writeTables: newWriteTables,
154
164
  sessionChangeset,
155
165
  materializerHash,
156
- } = materializeEvent(decodedEventDef, {
157
- otelContext,
166
+ } = yield* materializeEvent(event, {
158
167
  withChangeset: true,
159
168
  materializerHashLeader: Option.none(),
160
169
  })
@@ -167,10 +176,10 @@ export const makeClientSessionSyncProcessor = ({
167
176
 
168
177
  // Trigger push to leader
169
178
  // console.debug('pushToLeader', encodedEventDefs.length, ...encodedEventDefs.map((_) => _.toJSON()))
170
- BucketQueue.offerAll(leaderPushQueue, encodedEventDefs).pipe(Effect.runSync)
179
+ yield* BucketQueue.offerAll(leaderPushQueue, encodedEventDefs)
171
180
 
172
181
  return { writeTables }
173
- }
182
+ })
174
183
 
175
184
  const debugInfo = {
176
185
  rebaseCount: 0,
@@ -178,8 +187,6 @@ export const makeClientSessionSyncProcessor = ({
178
187
  rejectCount: 0,
179
188
  }
180
189
 
181
- const otelContext = otel.trace.setSpan(otel.context.active(), span)
182
-
183
190
  const boot: ClientSessionSyncProcessor['boot'] = Effect.gen(function* () {
184
191
  if (confirmUnsavedChanges && typeof window !== 'undefined' && typeof window.addEventListener === 'function') {
185
192
  const onBeforeUnload = (event: BeforeUnloadEvent) => {
@@ -229,13 +236,12 @@ export const makeClientSessionSyncProcessor = ({
229
236
  })
230
237
 
231
238
  if (mergeResult._tag === 'unexpected-error') {
232
- return yield* new SyncError({ cause: mergeResult.message })
239
+ return yield* new UnexpectedError({ cause: mergeResult.message })
233
240
  } else if (mergeResult._tag === 'reject') {
234
241
  return shouldNeverHappen('Unexpected reject in client-session-sync-processor', mergeResult)
235
242
  }
236
243
 
237
244
  syncStateRef.current = mergeResult.newSyncState
238
- yield* syncStateUpdateQueue.offer(mergeResult.newSyncState)
239
245
 
240
246
  if (mergeResult._tag === 'rebase') {
241
247
  span.addEvent('merge:pull:rebase', {
@@ -244,7 +250,6 @@ export const makeClientSessionSyncProcessor = ({
244
250
  newEventsCount: mergeResult.newEvents.length,
245
251
  rollbackCount: mergeResult.rollbackEvents.length,
246
252
  res: TRACE_VERBOSE ? JSON.stringify(mergeResult) : undefined,
247
- rebaseGeneration: mergeResult.newSyncState.localHead.rebaseGeneration,
248
253
  })
249
254
 
250
255
  debugInfo.rebaseCount++
@@ -294,18 +299,19 @@ export const makeClientSessionSyncProcessor = ({
294
299
  debugInfo.advanceCount++
295
300
  }
296
301
 
297
- if (mergeResult.newEvents.length === 0) return
302
+ if (mergeResult.newEvents.length === 0) {
303
+ // If there are no new events, we need to update the sync state as well
304
+ yield* syncStateUpdateQueue.offer(mergeResult.newSyncState)
305
+ return
306
+ }
298
307
 
299
308
  const writeTables = new Set<string>()
300
309
  for (const event of mergeResult.newEvents) {
301
- // TODO apply changeset if available (will require tracking of write tables as well)
302
- const decodedEventDef = Schema.decodeSync(eventSchema)(event)
303
310
  const {
304
311
  writeTables: newWriteTables,
305
312
  sessionChangeset,
306
313
  materializerHash,
307
- } = materializeEvent(decodedEventDef, {
308
- otelContext,
314
+ } = yield* materializeEvent(event, {
309
315
  withChangeset: true,
310
316
  materializerHashLeader: event.meta.materializerHashLeader,
311
317
  })
@@ -318,6 +324,9 @@ export const makeClientSessionSyncProcessor = ({
318
324
  }
319
325
 
320
326
  refreshTables(writeTables)
327
+
328
+ // We're only triggering the sync state update after all events have been materialized
329
+ yield* syncStateUpdateQueue.offer(mergeResult.newSyncState)
321
330
  }).pipe(
322
331
  Effect.tapCauseLogPretty,
323
332
  Effect.catchAllCause((cause) => clientSession.shutdown(Exit.failCause(cause))),
@@ -364,10 +373,7 @@ export const makeClientSessionSyncProcessor = ({
364
373
  export interface ClientSessionSyncProcessor {
365
374
  push: (
366
375
  batch: ReadonlyArray<LiveStoreEvent.PartialAnyDecoded>,
367
- options: { otelContext: otel.Context },
368
- ) => {
369
- writeTables: Set<string>
370
- }
376
+ ) => Effect.Effect<{ writeTables: Set<string> }, MaterializeError>
371
377
  boot: Effect.Effect<void, UnexpectedError, Scope.Scope>
372
378
  /**
373
379
  * Only used for debugging / observability.
@@ -388,11 +394,11 @@ const SIMULATION_ENABLED = true
388
394
  // Warning: High values for the simulation params can lead to very long test runs since those get multiplied with the number of events
389
395
  export const ClientSessionSyncProcessorSimulationParams = Schema.Struct({
390
396
  pull: Schema.Struct({
391
- '1_before_leader_push_fiber_interrupt': Schema.Int.pipe(Schema.between(0, 25)),
392
- '2_before_leader_push_queue_clear': Schema.Int.pipe(Schema.between(0, 25)),
393
- '3_before_rebase_rollback': Schema.Int.pipe(Schema.between(0, 25)),
394
- '4_before_leader_push_queue_offer': Schema.Int.pipe(Schema.between(0, 25)),
395
- '5_before_leader_push_fiber_run': Schema.Int.pipe(Schema.between(0, 25)),
397
+ '1_before_leader_push_fiber_interrupt': Schema.Int.pipe(Schema.between(0, 15)),
398
+ '2_before_leader_push_queue_clear': Schema.Int.pipe(Schema.between(0, 15)),
399
+ '3_before_rebase_rollback': Schema.Int.pipe(Schema.between(0, 15)),
400
+ '4_before_leader_push_queue_offer': Schema.Int.pipe(Schema.between(0, 15)),
401
+ '5_before_leader_push_fiber_run': Schema.Int.pipe(Schema.between(0, 15)),
396
402
  }),
397
403
  })
398
404
  type ClientSessionSyncProcessorSimulationParams = typeof ClientSessionSyncProcessorSimulationParams.Type
@@ -0,0 +1,38 @@
1
+ import { Schema } from '@livestore/utils/effect'
2
+ import { UnexpectedError } from '../errors.ts'
3
+ import { EventSequenceNumber } from '../schema/mod.ts'
4
+
5
+ export class IsOfflineError extends Schema.TaggedError<IsOfflineError>()('IsOfflineError', {
6
+ cause: Schema.Defect,
7
+ }) {}
8
+
9
+ /** Unique ID generated by the backend when its created. Used to check whether the backend identity has changed. */
10
+ export const BackendId = Schema.String.annotations({ title: '@livestore/sync-cf:BackendId' })
11
+
12
+ export class BackendIdMismatchError extends Schema.TaggedError<BackendIdMismatchError>()('BackendIdMismatchError', {
13
+ expected: BackendId,
14
+ received: BackendId,
15
+ }) {}
16
+
17
+ export class ServerAheadError extends Schema.TaggedError<ServerAheadError>()('ServerAheadError', {
18
+ minimumExpectedNum: EventSequenceNumber.GlobalEventSequenceNumber,
19
+ providedNum: EventSequenceNumber.GlobalEventSequenceNumber,
20
+ }) {}
21
+
22
+ export class InvalidPushError extends Schema.TaggedError<InvalidPushError>()('InvalidPushError', {
23
+ cause: Schema.Union(UnexpectedError, ServerAheadError, BackendIdMismatchError),
24
+ }) {}
25
+
26
+ export class InvalidPullError extends Schema.TaggedError<InvalidPullError>()('InvalidPullError', {
27
+ cause: Schema.Defect,
28
+ }) {}
29
+
30
+ export class LeaderAheadError extends Schema.TaggedError<LeaderAheadError>()('LeaderAheadError', {
31
+ minimumExpectedNum: EventSequenceNumber.EventSequenceNumber,
32
+ providedNum: EventSequenceNumber.EventSequenceNumber,
33
+ /** Generation number the client session should use for subsequent pushes */
34
+ // nextGeneration: Schema.Number,
35
+ }) {}
36
+
37
+ export const SyncError = Schema.Union(InvalidPushError, InvalidPullError)
38
+ export type SyncError = typeof SyncError.Type
package/src/sync/index.ts CHANGED
@@ -1,3 +1,6 @@
1
1
  export * from './ClientSessionSyncProcessor.ts'
2
+ export * from './mock-sync-backend.ts'
2
3
  export * from './sync.ts'
4
+ export { NetworkStatus } from './sync-backend.ts'
5
+ export * from './transport-chunking.ts'
3
6
  export * from './validate-push-payload.ts'
@@ -0,0 +1,184 @@
1
+ import type { Schema, Scope } from '@livestore/utils/effect'
2
+ import { Effect, Mailbox, Option, Queue, Stream, SubscriptionRef } from '@livestore/utils/effect'
3
+ import { UnexpectedError } from '../errors.ts'
4
+ import { EventSequenceNumber, type LiveStoreEvent } from '../schema/mod.ts'
5
+ import { InvalidPushError } from './errors.ts'
6
+ import * as SyncBackend from './sync-backend.ts'
7
+ import { validatePushPayload } from './validate-push-payload.ts'
8
+
9
+ export interface MockSyncBackend {
10
+ pushedEvents: Stream.Stream<LiveStoreEvent.AnyEncodedGlobal>
11
+ connect: Effect.Effect<void>
12
+ disconnect: Effect.Effect<void>
13
+ makeSyncBackend: Effect.Effect<SyncBackend.SyncBackend, UnexpectedError, Scope.Scope>
14
+ advance: (...batch: LiveStoreEvent.AnyEncodedGlobal[]) => Effect.Effect<void>
15
+ /** Fail the next N push calls with an InvalidPushError (or custom error) */
16
+ failNextPushes: (
17
+ count: number,
18
+ error?: (batch: ReadonlyArray<LiveStoreEvent.AnyEncodedGlobal>) => Effect.Effect<never, InvalidPushError>,
19
+ ) => Effect.Effect<void>
20
+ }
21
+
22
+ export interface MockSyncBackendOptions {
23
+ /** Chunk size for non-live pulls; defaults to 100 */
24
+ nonLiveChunkSize?: number
25
+ /** Initial connected state; defaults to false */
26
+ startConnected?: boolean
27
+ // TODO add a "flaky" mode to simulate transient network / server failures for pull/push
28
+ }
29
+
30
+ export const makeMockSyncBackend = (
31
+ options?: MockSyncBackendOptions,
32
+ ): Effect.Effect<MockSyncBackend, UnexpectedError, Scope.Scope> =>
33
+ Effect.gen(function* () {
34
+ const syncEventSequenceNumberRef = { current: EventSequenceNumber.ROOT.global }
35
+ const syncPullQueue = yield* Queue.unbounded<LiveStoreEvent.AnyEncodedGlobal>()
36
+ const pushedEventsQueue = yield* Mailbox.make<LiveStoreEvent.AnyEncodedGlobal>()
37
+ const syncIsConnectedRef = yield* SubscriptionRef.make(options?.startConnected ?? false)
38
+ const allEventsRef: { current: LiveStoreEvent.AnyEncodedGlobal[] } = { current: [] }
39
+
40
+ const span = yield* Effect.currentSpan.pipe(Effect.orDie)
41
+
42
+ const semaphore = yield* Effect.makeSemaphore(1)
43
+
44
+ // TODO improve the API and implementation of simulating errors
45
+ const failCounterRef = yield* SubscriptionRef.make(0)
46
+ const failEffectRef = yield* SubscriptionRef.make<
47
+ ((batch: ReadonlyArray<LiveStoreEvent.AnyEncodedGlobal>) => Effect.Effect<never, InvalidPushError>) | undefined
48
+ >(undefined)
49
+
50
+ const makeSyncBackend = Effect.gen(function* () {
51
+ const nonLiveChunkSize = Math.max(1, options?.nonLiveChunkSize ?? 100)
52
+
53
+ // TODO consider making offline state actively error pull/push.
54
+ // Currently, offline only reflects in `isConnected`, while operations still succeed,
55
+ // mirroring how some real providers behave during transient disconnects.
56
+ return SyncBackend.of<Schema.JsonValue>({
57
+ isConnected: syncIsConnectedRef,
58
+ connect: SubscriptionRef.set(syncIsConnectedRef, true),
59
+ ping: Effect.void,
60
+ pull: (cursor, options) =>
61
+ (options?.live
62
+ ? Stream.concat(
63
+ Stream.make(SyncBackend.pullResItemEmpty()),
64
+ Stream.fromQueue(syncPullQueue).pipe(
65
+ Stream.chunks,
66
+ Stream.map((chunk) => ({
67
+ batch: [...chunk].map((eventEncoded) => ({ eventEncoded, metadata: Option.none() })),
68
+ pageInfo: SyncBackend.pageInfoNoMore,
69
+ })),
70
+ ),
71
+ )
72
+ : Stream.fromEffect(
73
+ Effect.sync(() => {
74
+ const lastSeen = cursor.pipe(
75
+ Option.match({
76
+ onNone: () => EventSequenceNumber.ROOT.global,
77
+ onSome: (_) => _.eventSequenceNumber,
78
+ }),
79
+ )
80
+ // All events with seqNum greater than lastSeen
81
+ const slice = allEventsRef.current.filter((e) => e.seqNum > lastSeen)
82
+ // Split into configured chunk size
83
+ const chunks: { events: LiveStoreEvent.AnyEncodedGlobal[]; remaining: number }[] = []
84
+ for (let i = 0; i < slice.length; i += nonLiveChunkSize) {
85
+ const end = Math.min(i + nonLiveChunkSize, slice.length)
86
+ const remaining = Math.max(slice.length - end, 0)
87
+ chunks.push({ events: slice.slice(i, end), remaining })
88
+ }
89
+ if (chunks.length === 0) {
90
+ chunks.push({ events: [], remaining: 0 })
91
+ }
92
+ return chunks
93
+ }),
94
+ ).pipe(
95
+ Stream.flatMap((chunks) =>
96
+ Stream.fromIterable(chunks).pipe(
97
+ Stream.map(({ events, remaining }) => ({
98
+ batch: events.map((eventEncoded) => ({ eventEncoded, metadata: Option.none() })),
99
+ pageInfo: remaining > 0 ? SyncBackend.pageInfoMoreKnown(remaining) : SyncBackend.pageInfoNoMore,
100
+ })),
101
+ ),
102
+ ),
103
+ )
104
+ ).pipe(Stream.withSpan('MockSyncBackend:pull', { parent: span })),
105
+ push: (batch) =>
106
+ Effect.gen(function* () {
107
+ yield* validatePushPayload(batch, syncEventSequenceNumberRef.current)
108
+
109
+ const remaining = yield* SubscriptionRef.get(failCounterRef)
110
+ if (remaining > 0) {
111
+ const maybeFail = yield* SubscriptionRef.get(failEffectRef)
112
+ // decrement counter first
113
+ yield* SubscriptionRef.set(failCounterRef, remaining - 1)
114
+ if (maybeFail) {
115
+ return yield* maybeFail(batch)
116
+ }
117
+ return yield* new InvalidPushError({
118
+ cause: new UnexpectedError({ cause: new Error('MockSyncBackend: simulated push failure') }),
119
+ })
120
+ }
121
+
122
+ yield* Effect.sleep(10).pipe(Effect.withSpan('MockSyncBackend:push:sleep')) // Simulate network latency
123
+
124
+ yield* pushedEventsQueue.offerAll(batch)
125
+ yield* syncPullQueue.offerAll(batch)
126
+ allEventsRef.current = allEventsRef.current.concat(batch)
127
+
128
+ syncEventSequenceNumberRef.current = batch.at(-1)!.seqNum
129
+ }).pipe(
130
+ Effect.withSpan('MockSyncBackend:push', {
131
+ parent: span,
132
+ attributes: {
133
+ nums: batch.map((_) => _.seqNum),
134
+ },
135
+ }),
136
+ semaphore.withPermits(1),
137
+ ),
138
+ metadata: {
139
+ name: '@livestore/mock-sync',
140
+ description: 'Just a mock sync backend',
141
+ },
142
+ supports: {
143
+ pullPageInfoKnown: true,
144
+ pullLive: true,
145
+ },
146
+ })
147
+ })
148
+
149
+ const advance = (...batch: LiveStoreEvent.AnyEncodedGlobal[]) =>
150
+ Effect.gen(function* () {
151
+ syncEventSequenceNumberRef.current = batch.at(-1)!.seqNum
152
+ allEventsRef.current = allEventsRef.current.concat(batch)
153
+ yield* syncPullQueue.offerAll(batch)
154
+ }).pipe(
155
+ Effect.withSpan('MockSyncBackend:advance', {
156
+ parent: span,
157
+ attributes: { nums: batch.map((_) => _.seqNum) },
158
+ }),
159
+ semaphore.withPermits(1),
160
+ )
161
+
162
+ const connect = SubscriptionRef.set(syncIsConnectedRef, true)
163
+ const disconnect = SubscriptionRef.set(syncIsConnectedRef, false)
164
+
165
+ const failNextPushes = (
166
+ count: number,
167
+ error?: (batch: ReadonlyArray<LiveStoreEvent.AnyEncodedGlobal>) => Effect.Effect<never, InvalidPushError>,
168
+ ) =>
169
+ Effect.gen(function* () {
170
+ yield* SubscriptionRef.set(failCounterRef, count)
171
+ yield* SubscriptionRef.set(failEffectRef, error)
172
+ })
173
+
174
+ return {
175
+ syncEventSequenceNumberRef,
176
+ syncPullQueue,
177
+ pushedEvents: Mailbox.toStream(pushedEventsQueue),
178
+ connect,
179
+ disconnect,
180
+ makeSyncBackend,
181
+ advance,
182
+ failNextPushes,
183
+ }
184
+ }).pipe(Effect.withSpanScoped('MockSyncBackend'))
@@ -1,6 +1,5 @@
1
1
  import { EventSequenceNumber } from '../../schema/mod.ts'
2
2
  import { replacesFacts } from './facts.ts'
3
- import { graphologyDag } from './graphology_.ts'
4
3
  import type { HistoryDag } from './history-dag-common.ts'
5
4
  import { emptyHistoryDag } from './history-dag-common.ts'
6
5
 
@@ -17,7 +16,7 @@ export const compactEvents = (inputDag: HistoryDag): { dag: HistoryDag; compacte
17
16
  const dag = inputDag.copy()
18
17
  const compactedEventCount = 0
19
18
 
20
- const orderedEventSequenceNumberStrs = graphologyDag.topologicalSort(dag).reverse()
19
+ const orderedEventSequenceNumberStrs = dag.topologicalNodeIds().reverse()
21
20
 
22
21
  // drop root
23
22
  orderedEventSequenceNumberStrs.pop()
@@ -116,7 +115,7 @@ const findSubDagsInHistory = (
116
115
  const subDags: HistoryDag[] = []
117
116
  const allOutsideDependencies: string[][] = []
118
117
 
119
- for (const eventNumStr of graphologyDag.topologicalSort(inputDag)) {
118
+ for (const eventNumStr of inputDag.topologicalNodeIds()) {
120
119
  if (eventNumStr === upToExclEventSequenceNumberStr) {
121
120
  break
122
121
  }
@@ -189,8 +188,8 @@ const dagReplacesDag = (dagA: HistoryDag, dagB: HistoryDag): boolean => {
189
188
  }
190
189
 
191
190
  // TODO write tests that covers deterministic order when DAGs have branches
192
- const nodeEntriesA = graphologyDag.topologicalSort(dagA).map((nodeId) => dagA.getNodeAttributes(nodeId))
193
- const nodeEntriesB = graphologyDag.topologicalSort(dagB).map((nodeId) => dagB.getNodeAttributes(nodeId))
191
+ const nodeEntriesA = dagA.topologicalNodeIds().map((nodeId) => dagA.getNodeAttributes(nodeId))
192
+ const nodeEntriesB = dagB.topologicalNodeIds().map((nodeId) => dagB.getNodeAttributes(nodeId))
194
193
 
195
194
  for (let i = 0; i < nodeEntriesA.length; i++) {
196
195
  const nodeA = nodeEntriesA[i]!
@@ -1,5 +1,4 @@
1
1
  import { notYetImplemented } from '@livestore/utils'
2
-
3
2
  import type {
4
3
  EventDefFactInput,
5
4
  EventDefFacts,
@@ -8,7 +7,6 @@ import type {
8
7
  FactsCallback,
9
8
  } from '../../schema/EventDef.ts'
10
9
  import type * as EventSequenceNumber from '../../schema/EventSequenceNumber.ts'
11
- import { graphologyDag } from './graphology_.ts'
12
10
  import { EMPTY_FACT_VALUE, type HistoryDag, type HistoryDagNode } from './history-dag-common.ts'
13
11
 
14
12
  export const factsSnapshotForEvents = (
@@ -34,7 +32,7 @@ export const factsSnapshotForDag = (
34
32
  ): EventDefFactsSnapshot => {
35
33
  const facts = new Map<string, any>()
36
34
 
37
- const orderedEventSequenceNumberStrs = graphologyDag.topologicalSort(dag)
35
+ const orderedEventSequenceNumberStrs = dag.topologicalNodeIds()
38
36
 
39
37
  for (let i = 0; i < orderedEventSequenceNumberStrs.length; i++) {
40
38
  const event = dag.getNodeAttributes(orderedEventSequenceNumberStrs[i]!)