@livestore/common 0.4.0-dev.20 → 0.4.0-dev.22

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (127) hide show
  1. package/dist/.tsbuildinfo +1 -1
  2. package/dist/ClientSessionLeaderThreadProxy.d.ts +10 -0
  3. package/dist/ClientSessionLeaderThreadProxy.d.ts.map +1 -1
  4. package/dist/ClientSessionLeaderThreadProxy.js.map +1 -1
  5. package/dist/adapter-types.d.ts +23 -0
  6. package/dist/adapter-types.d.ts.map +1 -1
  7. package/dist/adapter-types.js +27 -1
  8. package/dist/adapter-types.js.map +1 -1
  9. package/dist/devtools/devtools-messages-client-session.d.ts +42 -22
  10. package/dist/devtools/devtools-messages-client-session.d.ts.map +1 -1
  11. package/dist/devtools/devtools-messages-client-session.js +12 -1
  12. package/dist/devtools/devtools-messages-client-session.js.map +1 -1
  13. package/dist/devtools/devtools-messages-common.d.ts +12 -6
  14. package/dist/devtools/devtools-messages-common.d.ts.map +1 -1
  15. package/dist/devtools/devtools-messages-common.js +7 -2
  16. package/dist/devtools/devtools-messages-common.js.map +1 -1
  17. package/dist/devtools/devtools-messages-leader.d.ts +47 -25
  18. package/dist/devtools/devtools-messages-leader.d.ts.map +1 -1
  19. package/dist/devtools/devtools-messages-leader.js +13 -1
  20. package/dist/devtools/devtools-messages-leader.js.map +1 -1
  21. package/dist/leader-thread/LeaderSyncProcessor.d.ts +33 -0
  22. package/dist/leader-thread/LeaderSyncProcessor.d.ts.map +1 -1
  23. package/dist/leader-thread/LeaderSyncProcessor.js +12 -12
  24. package/dist/leader-thread/LeaderSyncProcessor.js.map +1 -1
  25. package/dist/leader-thread/eventlog.d.ts +6 -1
  26. package/dist/leader-thread/eventlog.d.ts.map +1 -1
  27. package/dist/leader-thread/eventlog.js +59 -2
  28. package/dist/leader-thread/eventlog.js.map +1 -1
  29. package/dist/leader-thread/leader-worker-devtools.js +38 -6
  30. package/dist/leader-thread/leader-worker-devtools.js.map +1 -1
  31. package/dist/leader-thread/make-leader-thread-layer.d.ts +4 -2
  32. package/dist/leader-thread/make-leader-thread-layer.d.ts.map +1 -1
  33. package/dist/leader-thread/make-leader-thread-layer.js +5 -1
  34. package/dist/leader-thread/make-leader-thread-layer.js.map +1 -1
  35. package/dist/leader-thread/materialize-event.d.ts.map +1 -1
  36. package/dist/leader-thread/materialize-event.js +3 -0
  37. package/dist/leader-thread/materialize-event.js.map +1 -1
  38. package/dist/leader-thread/mod.d.ts +1 -0
  39. package/dist/leader-thread/mod.d.ts.map +1 -1
  40. package/dist/leader-thread/mod.js +1 -0
  41. package/dist/leader-thread/mod.js.map +1 -1
  42. package/dist/leader-thread/stream-events.d.ts +56 -0
  43. package/dist/leader-thread/stream-events.d.ts.map +1 -0
  44. package/dist/leader-thread/stream-events.js +166 -0
  45. package/dist/leader-thread/stream-events.js.map +1 -0
  46. package/dist/leader-thread/types.d.ts +77 -1
  47. package/dist/leader-thread/types.d.ts.map +1 -1
  48. package/dist/leader-thread/types.js +13 -0
  49. package/dist/leader-thread/types.js.map +1 -1
  50. package/dist/otel.d.ts +2 -1
  51. package/dist/otel.d.ts.map +1 -1
  52. package/dist/otel.js +5 -0
  53. package/dist/otel.js.map +1 -1
  54. package/dist/schema/EventDef/define.d.ts +14 -0
  55. package/dist/schema/EventDef/define.d.ts.map +1 -1
  56. package/dist/schema/EventDef/define.js +1 -0
  57. package/dist/schema/EventDef/define.js.map +1 -1
  58. package/dist/schema/EventDef/deprecated.d.ts +99 -0
  59. package/dist/schema/EventDef/deprecated.d.ts.map +1 -0
  60. package/dist/schema/EventDef/deprecated.js +144 -0
  61. package/dist/schema/EventDef/deprecated.js.map +1 -0
  62. package/dist/schema/EventDef/deprecated.test.d.ts +2 -0
  63. package/dist/schema/EventDef/deprecated.test.d.ts.map +1 -0
  64. package/dist/schema/EventDef/deprecated.test.js +95 -0
  65. package/dist/schema/EventDef/deprecated.test.js.map +1 -0
  66. package/dist/schema/EventDef/event-def.d.ts +4 -0
  67. package/dist/schema/EventDef/event-def.d.ts.map +1 -1
  68. package/dist/schema/EventDef/mod.d.ts +1 -0
  69. package/dist/schema/EventDef/mod.d.ts.map +1 -1
  70. package/dist/schema/EventDef/mod.js +1 -0
  71. package/dist/schema/EventDef/mod.js.map +1 -1
  72. package/dist/schema/LiveStoreEvent/client.d.ts +6 -6
  73. package/dist/schema/state/sqlite/client-document-def.d.ts +1 -0
  74. package/dist/schema/state/sqlite/client-document-def.d.ts.map +1 -1
  75. package/dist/schema/state/sqlite/client-document-def.js +17 -8
  76. package/dist/schema/state/sqlite/client-document-def.js.map +1 -1
  77. package/dist/schema/state/sqlite/client-document-def.test.js +120 -1
  78. package/dist/schema/state/sqlite/client-document-def.test.js.map +1 -1
  79. package/dist/schema/state/sqlite/column-def.test.js +2 -3
  80. package/dist/schema/state/sqlite/column-def.test.js.map +1 -1
  81. package/dist/schema/state/sqlite/db-schema/dsl/mod.d.ts.map +1 -1
  82. package/dist/schema/state/sqlite/db-schema/dsl/mod.js.map +1 -1
  83. package/dist/schema/state/sqlite/query-builder/api.d.ts +29 -12
  84. package/dist/schema/state/sqlite/query-builder/api.d.ts.map +1 -1
  85. package/dist/schema/state/sqlite/query-builder/astToSql.d.ts.map +1 -1
  86. package/dist/schema/state/sqlite/query-builder/astToSql.js +71 -1
  87. package/dist/schema/state/sqlite/query-builder/astToSql.js.map +1 -1
  88. package/dist/schema/state/sqlite/query-builder/impl.test.js +109 -1
  89. package/dist/schema/state/sqlite/query-builder/impl.test.js.map +1 -1
  90. package/dist/schema/state/sqlite/table-def.d.ts.map +1 -1
  91. package/dist/sync/ClientSessionSyncProcessor.d.ts.map +1 -1
  92. package/dist/sync/ClientSessionSyncProcessor.js +6 -2
  93. package/dist/sync/ClientSessionSyncProcessor.js.map +1 -1
  94. package/dist/version.d.ts +7 -1
  95. package/dist/version.d.ts.map +1 -1
  96. package/dist/version.js +8 -1
  97. package/dist/version.js.map +1 -1
  98. package/package.json +4 -4
  99. package/src/ClientSessionLeaderThreadProxy.ts +10 -0
  100. package/src/adapter-types.ts +30 -0
  101. package/src/devtools/devtools-messages-client-session.ts +12 -0
  102. package/src/devtools/devtools-messages-common.ts +7 -3
  103. package/src/devtools/devtools-messages-leader.ts +13 -0
  104. package/src/leader-thread/LeaderSyncProcessor.ts +116 -42
  105. package/src/leader-thread/eventlog.ts +80 -4
  106. package/src/leader-thread/leader-worker-devtools.ts +52 -6
  107. package/src/leader-thread/make-leader-thread-layer.ts +8 -0
  108. package/src/leader-thread/materialize-event.ts +4 -0
  109. package/src/leader-thread/mod.ts +1 -0
  110. package/src/leader-thread/stream-events.ts +201 -0
  111. package/src/leader-thread/types.ts +49 -1
  112. package/src/otel.ts +10 -0
  113. package/src/schema/EventDef/define.ts +16 -0
  114. package/src/schema/EventDef/deprecated.test.ts +128 -0
  115. package/src/schema/EventDef/deprecated.ts +175 -0
  116. package/src/schema/EventDef/event-def.ts +5 -0
  117. package/src/schema/EventDef/mod.ts +1 -0
  118. package/src/schema/state/sqlite/client-document-def.test.ts +140 -2
  119. package/src/schema/state/sqlite/client-document-def.ts +25 -26
  120. package/src/schema/state/sqlite/column-def.test.ts +2 -3
  121. package/src/schema/state/sqlite/db-schema/dsl/mod.ts +10 -16
  122. package/src/schema/state/sqlite/query-builder/api.ts +31 -4
  123. package/src/schema/state/sqlite/query-builder/astToSql.ts +81 -1
  124. package/src/schema/state/sqlite/query-builder/impl.test.ts +141 -1
  125. package/src/schema/state/sqlite/table-def.ts +9 -8
  126. package/src/sync/ClientSessionSyncProcessor.ts +26 -13
  127. package/src/version.ts +9 -1
@@ -107,6 +107,17 @@ export class Ping extends LSDClientSessionReqResMessage('LSD.ClientSession.Ping'
107
107
 
108
108
  export class Pong extends LSDClientSessionReqResMessage('LSD.ClientSession.Pong', {}) {}
109
109
 
110
+ /**
111
+ * Sent by the app when DevTools version doesn't match.
112
+ * Contains the app's actual version so DevTools can display a meaningful error.
113
+ */
114
+ export class VersionMismatch extends LSDClientSessionReqResMessage('LSD.ClientSession.VersionMismatch', {
115
+ /** The version running in the app */
116
+ appVersion: Schema.String,
117
+ /** The version that was sent by DevTools (that caused the mismatch) */
118
+ receivedVersion: Schema.String,
119
+ }) {}
120
+
110
121
  export class Disconnect extends LSDClientSessionChannelMessage('LSD.ClientSession.Disconnect', {}) {}
111
122
 
112
123
  export const MessageToApp = Schema.Union(
@@ -136,6 +147,7 @@ export const MessageFromApp = Schema.Union(
136
147
  LiveQueriesRes,
137
148
  Disconnect,
138
149
  Pong,
150
+ VersionMismatch,
139
151
  SyncHeadRes,
140
152
  ).annotations({ identifier: 'LSD.ClientSession.MessageFromApp' })
141
153
 
@@ -1,13 +1,17 @@
1
1
  import { Schema } from '@livestore/utils/effect'
2
2
 
3
- import { liveStoreVersion as pkgVersion } from '../version.ts'
4
-
5
3
  export { NetworkStatus } from '../sync/sync-backend.ts'
6
4
 
7
5
  export const requestId = Schema.String
8
6
  export const clientId = Schema.String
9
7
  export const sessionId = Schema.String
10
- export const liveStoreVersion = Schema.Literal(pkgVersion)
8
+ /**
9
+ * Version field for devtools messages.
10
+ * Uses Schema.String to accept messages from any version (backwards/forwards compatible).
11
+ * Version checking happens at the application layer after message parsing succeeds,
12
+ * allowing DevTools to respond with a proper VersionMismatch error instead of silent rejection.
13
+ */
14
+ export const liveStoreVersion = Schema.String
11
15
 
12
16
  export const LSDMessage = <Tag extends string, Fields extends Schema.Struct.Fields>(tag: Tag, fields: Fields) =>
13
17
  Schema.TaggedStruct(tag, {
@@ -77,6 +77,7 @@ export class SnapshotRes extends LSDReqResMessage('LSD.Leader.SnapshotRes', {
77
77
  export const LoadDatabaseFile = LeaderReqResMessage('LSD.Leader.LoadDatabaseFile', {
78
78
  payload: {
79
79
  data: Transferable.Uint8Array as Schema.Schema<Uint8Array<ArrayBuffer>>,
80
+ batchId: Schema.optional(Schema.String),
80
81
  },
81
82
  success: {},
82
83
  error: {
@@ -110,6 +111,17 @@ export class Ping extends LSDReqResMessage('LSD.Leader.Ping', {}) {}
110
111
 
111
112
  export class Pong extends LSDReqResMessage('LSD.Leader.Pong', {}) {}
112
113
 
114
+ /**
115
+ * Sent by the app when DevTools version doesn't match.
116
+ * Contains the app's actual version so DevTools can display a meaningful error.
117
+ */
118
+ export class VersionMismatch extends LSDReqResMessage('LSD.Leader.VersionMismatch', {
119
+ /** The version running in the app */
120
+ appVersion: Schema.String,
121
+ /** The version that was sent by DevTools (that caused the mismatch) */
122
+ receivedVersion: Schema.String,
123
+ }) {}
124
+
113
125
  export class Disconnect extends LSDReqResMessage('LSD.Leader.Disconnect', {}) {}
114
126
 
115
127
  export const SetSyncLatch = LeaderReqResMessage('LSD.Leader.SetSyncLatch', {
@@ -180,6 +192,7 @@ export const MessageFromApp = Schema.Union(
180
192
  NetworkStatusRes,
181
193
  CommitEventRes,
182
194
  Pong,
195
+ VersionMismatch,
183
196
  DatabaseFileInfoRes,
184
197
  SyncHistoryRes,
185
198
  SyncingInfoRes,
@@ -37,6 +37,11 @@ import { rollback } from './materialize-event.ts'
37
37
  import type { InitialBlockingSyncContext, LeaderSyncProcessor } from './types.ts'
38
38
  import { LeaderThreadCtx } from './types.ts'
39
39
 
40
+ // WORKAROUND: @effect/opentelemetry mis-parses `Span.addEvent(name, attributes)` and treats the attributes object as a
41
+ // time input, causing `TypeError: {} is not iterable` at runtime.
42
+ // Upstream: https://github.com/Effect-TS/effect/pull/5929
43
+ // TODO: simplify back to the 2-arg overload once the upstream fix is released and adopted.
44
+
40
45
  type LocalPushQueueItem = [
41
46
  event: LiveStoreEvent.Client.EncodedWithMeta,
42
47
  deferred: Deferred.Deferred<void, LeaderAheadError> | undefined,
@@ -90,10 +95,43 @@ export const makeLeaderSyncProcessor = ({
90
95
  onError: 'shutdown' | 'ignore'
91
96
  params: {
92
97
  /**
98
+ * Maximum number of local events to process per batch cycle.
99
+ *
100
+ * This controls how many events from client sessions are applied to the local state
101
+ * in a single iteration before yielding to allow potential backend pulls.
102
+ *
103
+ * **Trade-offs:**
104
+ * - **Lower values (1-5):** More responsive to remote updates since pull processing can
105
+ * interleave more frequently. Better for high-conflict scenarios where rebases are common.
106
+ * Slightly higher per-event overhead due to more frequent transaction commits.
107
+ *
108
+ * - **Higher values (10-50+):** Better throughput for bulk local writes as more events are
109
+ * batched into a single transaction. However, may delay remote update processing and
110
+ * increase rebase complexity if many local events queue up during a slow pull.
111
+ *
112
+ * - **Very high values (100+):** Risk of starvation for pull processing if local pushes
113
+ * arrive continuously. May cause larger rollbacks during rebases. Not recommended
114
+ * unless you have a write-heavy workload with minimal remote synchronization.
115
+ *
93
116
  * @default 10
94
117
  */
95
118
  localPushBatchSize?: number
96
119
  /**
120
+ * Maximum number of events to push to the sync backend per batch.
121
+ *
122
+ * This controls how many events are sent in a single push request to the remote server.
123
+ *
124
+ * **Trade-offs:**
125
+ * - **Lower values (1-10):** Lower latency for each push operation. Faster feedback on
126
+ * push success/failure. Slightly higher network overhead due to more requests.
127
+ *
128
+ * - **Higher values (50-100):** Better network efficiency by amortizing request overhead.
129
+ * Preferred for high-throughput scenarios. May increase latency to first confirmation.
130
+ *
131
+ * - **Very high values (200+):** Risk of hitting server request size limits or timeouts.
132
+ * A single failed request loses the entire batch (will be retried). May cause memory
133
+ * pressure if events accumulate faster than they can be pushed.
134
+ *
97
135
  * @default 50
98
136
  */
99
137
  backendPushBatchSize?: number
@@ -111,8 +149,8 @@ export const makeLeaderSyncProcessor = ({
111
149
  }): Effect.Effect<LeaderSyncProcessor, UnknownError, Scope.Scope> =>
112
150
  Effect.gen(function* () {
113
151
  const syncBackendPushQueue = yield* BucketQueue.make<LiveStoreEvent.Client.EncodedWithMeta>()
114
- const localPushBatchSize = params.localPushBatchSize ?? 1
115
- const backendPushBatchSize = params.backendPushBatchSize ?? 2
152
+ const localPushBatchSize = params.localPushBatchSize ?? 10
153
+ const backendPushBatchSize = params.backendPushBatchSize ?? 50
116
154
 
117
155
  const syncStateSref = yield* SubscriptionRef.make<SyncState.SyncState | undefined>(undefined)
118
156
 
@@ -443,10 +481,14 @@ const backgroundApplyLocalPushes = ({
443
481
  )
444
482
 
445
483
  if (droppedItems.length > 0) {
446
- otelSpan?.addEvent(`push:drop-old-generation`, {
447
- droppedCount: droppedItems.length,
448
- currentRebaseGeneration,
449
- })
484
+ otelSpan?.addEvent(
485
+ `push:drop-old-generation`,
486
+ {
487
+ droppedCount: droppedItems.length,
488
+ currentRebaseGeneration,
489
+ },
490
+ undefined,
491
+ )
450
492
 
451
493
  /**
452
494
  * Dropped pushes may still have a deferred awaiting completion.
@@ -484,20 +526,28 @@ const backgroundApplyLocalPushes = ({
484
526
 
485
527
  switch (mergeResult._tag) {
486
528
  case 'unknown-error': {
487
- otelSpan?.addEvent(`push:unknown-error`, {
488
- batchSize: newEvents.length,
489
- newEvents: TRACE_VERBOSE ? JSON.stringify(newEvents) : undefined,
490
- })
529
+ otelSpan?.addEvent(
530
+ `push:unknown-error`,
531
+ {
532
+ batchSize: newEvents.length,
533
+ newEvents: TRACE_VERBOSE ? JSON.stringify(newEvents) : undefined,
534
+ },
535
+ undefined,
536
+ )
491
537
  return yield* new UnknownError({ cause: mergeResult.message })
492
538
  }
493
539
  case 'rebase': {
494
540
  return shouldNeverHappen('The leader thread should never have to rebase due to a local push')
495
541
  }
496
542
  case 'reject': {
497
- otelSpan?.addEvent(`push:reject`, {
498
- batchSize: newEvents.length,
499
- mergeResult: TRACE_VERBOSE ? JSON.stringify(mergeResult) : undefined,
500
- })
543
+ otelSpan?.addEvent(
544
+ `push:reject`,
545
+ {
546
+ batchSize: newEvents.length,
547
+ mergeResult: TRACE_VERBOSE ? JSON.stringify(mergeResult) : undefined,
548
+ },
549
+ undefined,
550
+ )
501
551
 
502
552
  // TODO: how to test this?
503
553
  const nextRebaseGeneration = currentRebaseGeneration + 1
@@ -552,10 +602,14 @@ const backgroundApplyLocalPushes = ({
552
602
  leaderHead: mergeResult.newSyncState.localHead,
553
603
  })
554
604
 
555
- otelSpan?.addEvent(`push:advance`, {
556
- batchSize: newEvents.length,
557
- mergeResult: TRACE_VERBOSE ? JSON.stringify(mergeResult) : undefined,
558
- })
605
+ otelSpan?.addEvent(
606
+ `push:advance`,
607
+ {
608
+ batchSize: newEvents.length,
609
+ mergeResult: TRACE_VERBOSE ? JSON.stringify(mergeResult) : undefined,
610
+ },
611
+ undefined,
612
+ )
559
613
 
560
614
  // Don't sync client-local events
561
615
  const filteredBatch = mergeResult.newEvents.filter((eventEncoded) => {
@@ -686,10 +740,14 @@ const backgroundBackendPulling = ({
686
740
  if (mergeResult._tag === 'reject') {
687
741
  return shouldNeverHappen('The leader thread should never reject upstream advances')
688
742
  } else if (mergeResult._tag === 'unknown-error') {
689
- otelSpan?.addEvent(`pull:unknown-error`, {
690
- newEventsCount: newEvents.length,
691
- newEvents: TRACE_VERBOSE ? JSON.stringify(newEvents) : undefined,
692
- })
743
+ otelSpan?.addEvent(
744
+ `pull:unknown-error`,
745
+ {
746
+ newEventsCount: newEvents.length,
747
+ newEvents: TRACE_VERBOSE ? JSON.stringify(newEvents) : undefined,
748
+ },
749
+ undefined,
750
+ )
693
751
  return yield* new UnknownError({ cause: mergeResult.message })
694
752
  }
695
753
 
@@ -698,12 +756,16 @@ const backgroundBackendPulling = ({
698
756
  Eventlog.updateBackendHead(dbEventlog, newBackendHead)
699
757
 
700
758
  if (mergeResult._tag === 'rebase') {
701
- otelSpan?.addEvent(`pull:rebase[${mergeResult.newSyncState.localHead.rebaseGeneration}]`, {
702
- newEventsCount: newEvents.length,
703
- newEvents: TRACE_VERBOSE ? JSON.stringify(newEvents) : undefined,
704
- rollbackCount: mergeResult.rollbackEvents.length,
705
- mergeResult: TRACE_VERBOSE ? JSON.stringify(mergeResult) : undefined,
706
- })
759
+ otelSpan?.addEvent(
760
+ `pull:rebase[${mergeResult.newSyncState.localHead.rebaseGeneration}]`,
761
+ {
762
+ newEventsCount: newEvents.length,
763
+ newEvents: TRACE_VERBOSE ? JSON.stringify(newEvents) : undefined,
764
+ rollbackCount: mergeResult.rollbackEvents.length,
765
+ mergeResult: TRACE_VERBOSE ? JSON.stringify(mergeResult) : undefined,
766
+ },
767
+ undefined,
768
+ )
707
769
 
708
770
  const globalRebasedPendingEvents = mergeResult.newSyncState.pending.filter((event) => {
709
771
  const eventDef = schema.eventsDefsMap.get(event.name)
@@ -724,10 +786,14 @@ const backgroundBackendPulling = ({
724
786
  leaderHead: mergeResult.newSyncState.localHead,
725
787
  })
726
788
  } else {
727
- otelSpan?.addEvent(`pull:advance`, {
728
- newEventsCount: newEvents.length,
729
- mergeResult: TRACE_VERBOSE ? JSON.stringify(mergeResult) : undefined,
730
- })
789
+ otelSpan?.addEvent(
790
+ `pull:advance`,
791
+ {
792
+ newEventsCount: newEvents.length,
793
+ mergeResult: TRACE_VERBOSE ? JSON.stringify(mergeResult) : undefined,
794
+ },
795
+ undefined,
796
+ )
731
797
 
732
798
  // Ensure push fiber is active after advance by restarting with current pending (non-client) events
733
799
  const globalPendingEvents = mergeResult.newSyncState.pending.filter((event) => {
@@ -837,10 +903,14 @@ const backgroundBackendPushing = ({
837
903
  yield* devtoolsLatch.await
838
904
  }
839
905
 
840
- otelSpan?.addEvent('backend-push', {
841
- batchSize: queueItems.length,
842
- batch: TRACE_VERBOSE ? JSON.stringify(queueItems) : undefined,
843
- })
906
+ otelSpan?.addEvent(
907
+ 'backend-push',
908
+ {
909
+ batchSize: queueItems.length,
910
+ batch: TRACE_VERBOSE ? JSON.stringify(queueItems) : undefined,
911
+ },
912
+ undefined,
913
+ )
844
914
 
845
915
  // Push with declarative retry/backoff using Effect schedules
846
916
  // - Exponential backoff starting at 1s and doubling (1s, 2s, 4s, 8s, 16s, 30s ...)
@@ -867,15 +937,19 @@ const backgroundBackendPushing = ({
867
937
 
868
938
  const retries = iteration.recurrence
869
939
  if (retries > 0 && pushResult._tag === 'Right') {
870
- otelSpan?.addEvent('backend-push-retry-success', { retries, batchSize: queueItems.length })
940
+ otelSpan?.addEvent('backend-push-retry-success', { retries, batchSize: queueItems.length }, undefined)
871
941
  }
872
942
 
873
943
  if (pushResult._tag === 'Left') {
874
- otelSpan?.addEvent('backend-push-error', {
875
- error: pushResult.left.toString(),
876
- retries,
877
- batchSize: queueItems.length,
878
- })
944
+ otelSpan?.addEvent(
945
+ 'backend-push-error',
946
+ {
947
+ error: pushResult.left.toString(),
948
+ retries,
949
+ batchSize: queueItems.length,
950
+ },
951
+ undefined,
952
+ )
879
953
  const error = pushResult.left
880
954
  if (
881
955
  error._tag === 'IsOfflineError' ||
@@ -1,6 +1,5 @@
1
1
  import { LS_DEV, shouldNeverHappen } from '@livestore/utils'
2
- import { Effect, Option, Schema } from '@livestore/utils/effect'
3
-
2
+ import { Chunk, Effect, Option, Schema } from '@livestore/utils/effect'
4
3
  import type { SqliteDb } from '../adapter-types.ts'
5
4
  import * as EventSequenceNumber from '../schema/EventSequenceNumber/mod.ts'
6
5
  import * as LiveStoreEvent from '../schema/LiveStoreEvent/mod.ts'
@@ -16,8 +15,8 @@ import { insertRow, updateRows } from '../sql-queries/sql-queries.ts'
16
15
  import type { PreparedBindValues } from '../util.ts'
17
16
  import { sql } from '../util.ts'
18
17
  import { execSql } from './connection.ts'
19
- import type { InitialSyncInfo } from './types.ts'
20
- import { LeaderThreadCtx } from './types.ts'
18
+ import type { InitialSyncInfo, StreamEventsOptions } from './types.ts'
19
+ import { LeaderThreadCtx, STREAM_EVENTS_BATCH_SIZE_DEFAULT } from './types.ts'
21
20
 
22
21
  export const initEventlogDb = (dbEventlog: SqliteDb) =>
23
22
  Effect.gen(function* () {
@@ -100,6 +99,83 @@ export const getEventsSince = ({
100
99
  .sort((a, b) => EventSequenceNumber.Client.compare(a.seqNum, b.seqNum))
101
100
  }
102
101
 
102
+ export const getEventsFromEventlog = ({
103
+ dbEventlog,
104
+ options,
105
+ }: {
106
+ dbEventlog: SqliteDb
107
+ options: StreamEventsOptions
108
+ }): Effect.Effect<Chunk.Chunk<LiveStoreEvent.Client.Encoded>> =>
109
+ Effect.gen(function* () {
110
+ const since = options.since ?? EventSequenceNumber.Client.ROOT
111
+ const batchSize = options.batchSize ?? STREAM_EVENTS_BATCH_SIZE_DEFAULT
112
+
113
+ const makeQuery = () => {
114
+ let query = eventlogMetaTable.where('seqNumGlobal', '>', since.global)
115
+
116
+ if (options.until) {
117
+ query = query.where('seqNumGlobal', '<=', options.until.global)
118
+ }
119
+
120
+ if (options.filter && options.filter.length > 0) {
121
+ query = query.where({ name: { op: 'IN', value: options.filter } })
122
+ }
123
+
124
+ if (options.clientIds && options.clientIds.length > 0) {
125
+ query = query.where({ clientId: { op: 'IN', value: options.clientIds } })
126
+ }
127
+
128
+ if (options.sessionIds && options.sessionIds.length > 0) {
129
+ query = query.where({ sessionId: { op: 'IN', value: options.sessionIds } })
130
+ }
131
+
132
+ if (options.includeClientOnly !== true) {
133
+ query = query.where('seqNumClient', '<=', EventSequenceNumber.Client.DEFAULT)
134
+ }
135
+
136
+ return query
137
+ .orderBy([
138
+ { col: 'seqNumGlobal', direction: 'asc' },
139
+ { col: 'seqNumClient', direction: 'asc' },
140
+ ])
141
+ .limit(batchSize)
142
+ }
143
+
144
+ const eventlogEvents = yield* Effect.sync(() => dbEventlog.select(makeQuery()))
145
+
146
+ if (eventlogEvents.length === 0) {
147
+ return Chunk.empty<LiveStoreEvent.Client.Encoded>()
148
+ }
149
+
150
+ const spanAttributes = {
151
+ 'livestore.eventLog.since': since.global,
152
+ 'livestore.eventLog.until': options.until?.global,
153
+ }
154
+
155
+ return yield* Effect.sync(() => {
156
+ const encodedEvents = eventlogEvents.map((eventlogEvent) => {
157
+ return LiveStoreEvent.Client.Encoded.make({
158
+ name: eventlogEvent.name,
159
+ args: eventlogEvent.argsJson,
160
+ seqNum: {
161
+ global: eventlogEvent.seqNumGlobal,
162
+ client: eventlogEvent.seqNumClient,
163
+ rebaseGeneration: eventlogEvent.seqNumRebaseGeneration,
164
+ },
165
+ parentSeqNum: {
166
+ global: eventlogEvent.parentSeqNumGlobal,
167
+ client: eventlogEvent.parentSeqNumClient,
168
+ rebaseGeneration: eventlogEvent.parentSeqNumRebaseGeneration,
169
+ },
170
+ clientId: eventlogEvent.clientId,
171
+ sessionId: eventlogEvent.sessionId,
172
+ })
173
+ })
174
+
175
+ return Chunk.fromIterable(encodedEvents)
176
+ }).pipe(Effect.withSpan('@livestore/common:eventlog:getEventsFromEventlog', { attributes: spanAttributes }))
177
+ })
178
+
103
179
  export const getClientHeadFromDb = (dbEventlog: SqliteDb): EventSequenceNumber.Client.Composite => {
104
180
  const res = dbEventlog.select<{
105
181
  seqNumGlobal: EventSequenceNumber.Global.Type
@@ -94,6 +94,22 @@ const listenToDevtools = ({
94
94
  type RequestId = string
95
95
  const handledRequestIds = new Set<RequestId>()
96
96
 
97
+ type LoadDatabaseKind = 'state' | 'eventlog'
98
+ const loadDatabaseBatchTracker = new Map<string, Set<LoadDatabaseKind>>()
99
+
100
+ const registerBatchProgress = (batchId: string, kind: LoadDatabaseKind) => {
101
+ const entry = loadDatabaseBatchTracker.get(batchId) ?? new Set<LoadDatabaseKind>()
102
+ entry.add(kind)
103
+ loadDatabaseBatchTracker.set(batchId, entry)
104
+ const finished = entry.has('state') && entry.has('eventlog')
105
+
106
+ if (finished) {
107
+ loadDatabaseBatchTracker.delete(batchId)
108
+ }
109
+
110
+ return finished
111
+ }
112
+
97
113
  yield* incomingMessages.pipe(
98
114
  Stream.tap((decodedEvent) =>
99
115
  Effect.gen(function* () {
@@ -122,6 +138,17 @@ const listenToDevtools = ({
122
138
 
123
139
  switch (decodedEvent._tag) {
124
140
  case 'LSD.Leader.Ping': {
141
+ // Check version mismatch and respond with VersionMismatch if versions don't match
142
+ if (decodedEvent.liveStoreVersion !== liveStoreVersion) {
143
+ yield* sendMessage(
144
+ Devtools.Leader.VersionMismatch.make({
145
+ ...reqPayload,
146
+ appVersion: liveStoreVersion,
147
+ receivedVersion: decodedEvent.liveStoreVersion,
148
+ }),
149
+ )
150
+ return
151
+ }
125
152
  yield* sendMessage(Devtools.Leader.Pong.make({ ...reqPayload }))
126
153
  return
127
154
  }
@@ -133,7 +160,7 @@ const listenToDevtools = ({
133
160
  return
134
161
  }
135
162
  case 'LSD.Leader.LoadDatabaseFile.Request': {
136
- const { data } = decodedEvent
163
+ const { data, batchId } = decodedEvent
137
164
 
138
165
  const handleLoadDb = Effect.gen(function* () {
139
166
  const tableNames = yield* Effect.acquireRelease(makeSqliteDb({ _tag: 'in-memory' }), (db) =>
@@ -148,25 +175,44 @@ const listenToDevtools = ({
148
175
  ),
149
176
  )
150
177
 
178
+ let databaseKind: LoadDatabaseKind | undefined
179
+
151
180
  if (tableNames.has(SystemTables.EVENTLOG_META_TABLE)) {
152
- // Is eventlog db
181
+ databaseKind = 'eventlog'
153
182
  yield* SubscriptionRef.set(shutdownStateSubRef, 'shutting-down')
154
183
  yield* Effect.try(() => void dbEventlog.import(data))
155
- yield* Effect.try(() => void dbState.destroy())
184
+
185
+ if (batchId === undefined) {
186
+ yield* Effect.try(() => void dbState.destroy())
187
+ }
156
188
  } else if (
157
189
  tableNames.has(SystemTables.SCHEMA_META_TABLE) &&
158
190
  tableNames.has(SystemTables.SCHEMA_EVENT_DEFS_META_TABLE)
159
191
  ) {
160
- // Is state db
192
+ databaseKind = 'state'
161
193
  yield* SubscriptionRef.set(shutdownStateSubRef, 'shutting-down')
162
194
  yield* Effect.try(() => void dbState.import(data))
163
- yield* Effect.try(() => void dbEventlog.destroy())
195
+
196
+ if (batchId === undefined) {
197
+ yield* Effect.try(() => void dbEventlog.destroy())
198
+ }
164
199
  } else {
165
200
  return yield* Effect.fail({ _tag: 'unsupported-database' } as const)
166
201
  }
167
202
 
203
+ const resolvedDatabaseKind = databaseKind
204
+ if (resolvedDatabaseKind === undefined) {
205
+ return yield* Effect.fail({ _tag: 'unsupported-database' } as const)
206
+ }
207
+
208
+ const shouldShutdown =
209
+ batchId === undefined ? true : registerBatchProgress(batchId, resolvedDatabaseKind)
210
+
168
211
  yield* sendMessage(Devtools.Leader.LoadDatabaseFile.Success.make({ ...reqPayload }))
169
- yield* shutdownChannel.send(IntentionalShutdownCause.make({ reason: 'devtools-import' }))
212
+
213
+ if (shouldShutdown) {
214
+ yield* shutdownChannel.send(IntentionalShutdownCause.make({ reason: 'devtools-import' }))
215
+ }
170
216
  })
171
217
 
172
218
  yield* handleLoadDb.pipe(
@@ -55,6 +55,8 @@ export interface MakeLeaderThreadLayerParams {
55
55
  dbEventlog: LeaderSqliteDb
56
56
  devtoolsOptions: DevtoolsOptions
57
57
  shutdownChannel: ShutdownChannel
58
+ /** Boot warning to emit (e.g., OPFS unavailable in private browsing) */
59
+ bootWarning?: BootStatus
58
60
  params?: {
59
61
  localPushBatchSize?: number
60
62
  backendPushBatchSize?: number
@@ -80,6 +82,7 @@ export const makeLeaderThreadLayer = ({
80
82
  dbEventlog,
81
83
  devtoolsOptions,
82
84
  shutdownChannel,
85
+ bootWarning,
83
86
  params,
84
87
  testing,
85
88
  }: MakeLeaderThreadLayerParams): Layer.Layer<LeaderThreadCtx, UnknownError, Scope.Scope | HttpClient.HttpClient> =>
@@ -89,6 +92,11 @@ export const makeLeaderThreadLayer = ({
89
92
 
90
93
  const bootStatusQueue = yield* Queue.unbounded<BootStatus>().pipe(Effect.acquireRelease(Queue.shutdown))
91
94
 
95
+ // Emit boot warning if present (e.g., OPFS unavailable in private browsing)
96
+ if (bootWarning !== undefined) {
97
+ yield* Queue.offer(bootStatusQueue, bootWarning)
98
+ }
99
+
92
100
  const dbEventlogMissing = !hasEventlogTables(dbEventlog)
93
101
 
94
102
  // Either happens on initial boot or if schema changes
@@ -3,6 +3,7 @@ import { Effect, Option, ReadonlyArray, Schema } from '@livestore/utils/effect'
3
3
 
4
4
  import { MaterializeError, MaterializerHashMismatchError, type SqliteDb } from '../adapter-types.ts'
5
5
  import { getExecStatementsFromMaterializer, hashMaterializerResults } from '../materializer-helper.ts'
6
+ import { logDeprecationWarnings } from '../schema/EventDef/deprecated.ts'
6
7
  import type { LiveStoreSchema } from '../schema/mod.ts'
7
8
  import { EventSequenceNumber, resolveEventDef, SystemTables, UNKNOWN_EVENT_SCHEMA_HASH } from '../schema/mod.ts'
8
9
  import { insertRow } from '../sql-queries/index.ts'
@@ -62,6 +63,9 @@ export const makeMaterializeEvent = ({
62
63
 
63
64
  const { eventDef, materializer } = resolution
64
65
 
66
+ // Log deprecation warnings for deprecated events/fields
67
+ yield* logDeprecationWarnings(eventDef, eventEncoded.args as Record<string, unknown>)
68
+
65
69
  const execArgsArr = getExecStatementsFromMaterializer({
66
70
  eventDef,
67
71
  materializer,
@@ -5,4 +5,5 @@ export * from './make-leader-thread-layer.ts'
5
5
  export * from './materialize-event.ts'
6
6
  export * from './recreate-db.ts'
7
7
  export * as ShutdownChannel from './shutdown-channel.ts'
8
+ export * from './stream-events.ts'
8
9
  export * from './types.ts'