@livestore/common 0.4.0-dev.20 → 0.4.0-dev.22
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/.tsbuildinfo +1 -1
- package/dist/ClientSessionLeaderThreadProxy.d.ts +10 -0
- package/dist/ClientSessionLeaderThreadProxy.d.ts.map +1 -1
- package/dist/ClientSessionLeaderThreadProxy.js.map +1 -1
- package/dist/adapter-types.d.ts +23 -0
- package/dist/adapter-types.d.ts.map +1 -1
- package/dist/adapter-types.js +27 -1
- package/dist/adapter-types.js.map +1 -1
- package/dist/devtools/devtools-messages-client-session.d.ts +42 -22
- package/dist/devtools/devtools-messages-client-session.d.ts.map +1 -1
- package/dist/devtools/devtools-messages-client-session.js +12 -1
- package/dist/devtools/devtools-messages-client-session.js.map +1 -1
- package/dist/devtools/devtools-messages-common.d.ts +12 -6
- package/dist/devtools/devtools-messages-common.d.ts.map +1 -1
- package/dist/devtools/devtools-messages-common.js +7 -2
- package/dist/devtools/devtools-messages-common.js.map +1 -1
- package/dist/devtools/devtools-messages-leader.d.ts +47 -25
- package/dist/devtools/devtools-messages-leader.d.ts.map +1 -1
- package/dist/devtools/devtools-messages-leader.js +13 -1
- package/dist/devtools/devtools-messages-leader.js.map +1 -1
- package/dist/leader-thread/LeaderSyncProcessor.d.ts +33 -0
- package/dist/leader-thread/LeaderSyncProcessor.d.ts.map +1 -1
- package/dist/leader-thread/LeaderSyncProcessor.js +12 -12
- package/dist/leader-thread/LeaderSyncProcessor.js.map +1 -1
- package/dist/leader-thread/eventlog.d.ts +6 -1
- package/dist/leader-thread/eventlog.d.ts.map +1 -1
- package/dist/leader-thread/eventlog.js +59 -2
- package/dist/leader-thread/eventlog.js.map +1 -1
- package/dist/leader-thread/leader-worker-devtools.js +38 -6
- package/dist/leader-thread/leader-worker-devtools.js.map +1 -1
- package/dist/leader-thread/make-leader-thread-layer.d.ts +4 -2
- package/dist/leader-thread/make-leader-thread-layer.d.ts.map +1 -1
- package/dist/leader-thread/make-leader-thread-layer.js +5 -1
- package/dist/leader-thread/make-leader-thread-layer.js.map +1 -1
- package/dist/leader-thread/materialize-event.d.ts.map +1 -1
- package/dist/leader-thread/materialize-event.js +3 -0
- package/dist/leader-thread/materialize-event.js.map +1 -1
- package/dist/leader-thread/mod.d.ts +1 -0
- package/dist/leader-thread/mod.d.ts.map +1 -1
- package/dist/leader-thread/mod.js +1 -0
- package/dist/leader-thread/mod.js.map +1 -1
- package/dist/leader-thread/stream-events.d.ts +56 -0
- package/dist/leader-thread/stream-events.d.ts.map +1 -0
- package/dist/leader-thread/stream-events.js +166 -0
- package/dist/leader-thread/stream-events.js.map +1 -0
- package/dist/leader-thread/types.d.ts +77 -1
- package/dist/leader-thread/types.d.ts.map +1 -1
- package/dist/leader-thread/types.js +13 -0
- package/dist/leader-thread/types.js.map +1 -1
- package/dist/otel.d.ts +2 -1
- package/dist/otel.d.ts.map +1 -1
- package/dist/otel.js +5 -0
- package/dist/otel.js.map +1 -1
- package/dist/schema/EventDef/define.d.ts +14 -0
- package/dist/schema/EventDef/define.d.ts.map +1 -1
- package/dist/schema/EventDef/define.js +1 -0
- package/dist/schema/EventDef/define.js.map +1 -1
- package/dist/schema/EventDef/deprecated.d.ts +99 -0
- package/dist/schema/EventDef/deprecated.d.ts.map +1 -0
- package/dist/schema/EventDef/deprecated.js +144 -0
- package/dist/schema/EventDef/deprecated.js.map +1 -0
- package/dist/schema/EventDef/deprecated.test.d.ts +2 -0
- package/dist/schema/EventDef/deprecated.test.d.ts.map +1 -0
- package/dist/schema/EventDef/deprecated.test.js +95 -0
- package/dist/schema/EventDef/deprecated.test.js.map +1 -0
- package/dist/schema/EventDef/event-def.d.ts +4 -0
- package/dist/schema/EventDef/event-def.d.ts.map +1 -1
- package/dist/schema/EventDef/mod.d.ts +1 -0
- package/dist/schema/EventDef/mod.d.ts.map +1 -1
- package/dist/schema/EventDef/mod.js +1 -0
- package/dist/schema/EventDef/mod.js.map +1 -1
- package/dist/schema/LiveStoreEvent/client.d.ts +6 -6
- package/dist/schema/state/sqlite/client-document-def.d.ts +1 -0
- package/dist/schema/state/sqlite/client-document-def.d.ts.map +1 -1
- package/dist/schema/state/sqlite/client-document-def.js +17 -8
- package/dist/schema/state/sqlite/client-document-def.js.map +1 -1
- package/dist/schema/state/sqlite/client-document-def.test.js +120 -1
- package/dist/schema/state/sqlite/client-document-def.test.js.map +1 -1
- package/dist/schema/state/sqlite/column-def.test.js +2 -3
- package/dist/schema/state/sqlite/column-def.test.js.map +1 -1
- package/dist/schema/state/sqlite/db-schema/dsl/mod.d.ts.map +1 -1
- package/dist/schema/state/sqlite/db-schema/dsl/mod.js.map +1 -1
- package/dist/schema/state/sqlite/query-builder/api.d.ts +29 -12
- package/dist/schema/state/sqlite/query-builder/api.d.ts.map +1 -1
- package/dist/schema/state/sqlite/query-builder/astToSql.d.ts.map +1 -1
- package/dist/schema/state/sqlite/query-builder/astToSql.js +71 -1
- package/dist/schema/state/sqlite/query-builder/astToSql.js.map +1 -1
- package/dist/schema/state/sqlite/query-builder/impl.test.js +109 -1
- package/dist/schema/state/sqlite/query-builder/impl.test.js.map +1 -1
- package/dist/schema/state/sqlite/table-def.d.ts.map +1 -1
- package/dist/sync/ClientSessionSyncProcessor.d.ts.map +1 -1
- package/dist/sync/ClientSessionSyncProcessor.js +6 -2
- package/dist/sync/ClientSessionSyncProcessor.js.map +1 -1
- package/dist/version.d.ts +7 -1
- package/dist/version.d.ts.map +1 -1
- package/dist/version.js +8 -1
- package/dist/version.js.map +1 -1
- package/package.json +4 -4
- package/src/ClientSessionLeaderThreadProxy.ts +10 -0
- package/src/adapter-types.ts +30 -0
- package/src/devtools/devtools-messages-client-session.ts +12 -0
- package/src/devtools/devtools-messages-common.ts +7 -3
- package/src/devtools/devtools-messages-leader.ts +13 -0
- package/src/leader-thread/LeaderSyncProcessor.ts +116 -42
- package/src/leader-thread/eventlog.ts +80 -4
- package/src/leader-thread/leader-worker-devtools.ts +52 -6
- package/src/leader-thread/make-leader-thread-layer.ts +8 -0
- package/src/leader-thread/materialize-event.ts +4 -0
- package/src/leader-thread/mod.ts +1 -0
- package/src/leader-thread/stream-events.ts +201 -0
- package/src/leader-thread/types.ts +49 -1
- package/src/otel.ts +10 -0
- package/src/schema/EventDef/define.ts +16 -0
- package/src/schema/EventDef/deprecated.test.ts +128 -0
- package/src/schema/EventDef/deprecated.ts +175 -0
- package/src/schema/EventDef/event-def.ts +5 -0
- package/src/schema/EventDef/mod.ts +1 -0
- package/src/schema/state/sqlite/client-document-def.test.ts +140 -2
- package/src/schema/state/sqlite/client-document-def.ts +25 -26
- package/src/schema/state/sqlite/column-def.test.ts +2 -3
- package/src/schema/state/sqlite/db-schema/dsl/mod.ts +10 -16
- package/src/schema/state/sqlite/query-builder/api.ts +31 -4
- package/src/schema/state/sqlite/query-builder/astToSql.ts +81 -1
- package/src/schema/state/sqlite/query-builder/impl.test.ts +141 -1
- package/src/schema/state/sqlite/table-def.ts +9 -8
- package/src/sync/ClientSessionSyncProcessor.ts +26 -13
- package/src/version.ts +9 -1
|
@@ -107,6 +107,17 @@ export class Ping extends LSDClientSessionReqResMessage('LSD.ClientSession.Ping'
|
|
|
107
107
|
|
|
108
108
|
export class Pong extends LSDClientSessionReqResMessage('LSD.ClientSession.Pong', {}) {}
|
|
109
109
|
|
|
110
|
+
/**
|
|
111
|
+
* Sent by the app when DevTools version doesn't match.
|
|
112
|
+
* Contains the app's actual version so DevTools can display a meaningful error.
|
|
113
|
+
*/
|
|
114
|
+
export class VersionMismatch extends LSDClientSessionReqResMessage('LSD.ClientSession.VersionMismatch', {
|
|
115
|
+
/** The version running in the app */
|
|
116
|
+
appVersion: Schema.String,
|
|
117
|
+
/** The version that was sent by DevTools (that caused the mismatch) */
|
|
118
|
+
receivedVersion: Schema.String,
|
|
119
|
+
}) {}
|
|
120
|
+
|
|
110
121
|
export class Disconnect extends LSDClientSessionChannelMessage('LSD.ClientSession.Disconnect', {}) {}
|
|
111
122
|
|
|
112
123
|
export const MessageToApp = Schema.Union(
|
|
@@ -136,6 +147,7 @@ export const MessageFromApp = Schema.Union(
|
|
|
136
147
|
LiveQueriesRes,
|
|
137
148
|
Disconnect,
|
|
138
149
|
Pong,
|
|
150
|
+
VersionMismatch,
|
|
139
151
|
SyncHeadRes,
|
|
140
152
|
).annotations({ identifier: 'LSD.ClientSession.MessageFromApp' })
|
|
141
153
|
|
|
@@ -1,13 +1,17 @@
|
|
|
1
1
|
import { Schema } from '@livestore/utils/effect'
|
|
2
2
|
|
|
3
|
-
import { liveStoreVersion as pkgVersion } from '../version.ts'
|
|
4
|
-
|
|
5
3
|
export { NetworkStatus } from '../sync/sync-backend.ts'
|
|
6
4
|
|
|
7
5
|
export const requestId = Schema.String
|
|
8
6
|
export const clientId = Schema.String
|
|
9
7
|
export const sessionId = Schema.String
|
|
10
|
-
|
|
8
|
+
/**
|
|
9
|
+
* Version field for devtools messages.
|
|
10
|
+
* Uses Schema.String to accept messages from any version (backwards/forwards compatible).
|
|
11
|
+
* Version checking happens at the application layer after message parsing succeeds,
|
|
12
|
+
* allowing DevTools to respond with a proper VersionMismatch error instead of silent rejection.
|
|
13
|
+
*/
|
|
14
|
+
export const liveStoreVersion = Schema.String
|
|
11
15
|
|
|
12
16
|
export const LSDMessage = <Tag extends string, Fields extends Schema.Struct.Fields>(tag: Tag, fields: Fields) =>
|
|
13
17
|
Schema.TaggedStruct(tag, {
|
|
@@ -77,6 +77,7 @@ export class SnapshotRes extends LSDReqResMessage('LSD.Leader.SnapshotRes', {
|
|
|
77
77
|
export const LoadDatabaseFile = LeaderReqResMessage('LSD.Leader.LoadDatabaseFile', {
|
|
78
78
|
payload: {
|
|
79
79
|
data: Transferable.Uint8Array as Schema.Schema<Uint8Array<ArrayBuffer>>,
|
|
80
|
+
batchId: Schema.optional(Schema.String),
|
|
80
81
|
},
|
|
81
82
|
success: {},
|
|
82
83
|
error: {
|
|
@@ -110,6 +111,17 @@ export class Ping extends LSDReqResMessage('LSD.Leader.Ping', {}) {}
|
|
|
110
111
|
|
|
111
112
|
export class Pong extends LSDReqResMessage('LSD.Leader.Pong', {}) {}
|
|
112
113
|
|
|
114
|
+
/**
|
|
115
|
+
* Sent by the app when DevTools version doesn't match.
|
|
116
|
+
* Contains the app's actual version so DevTools can display a meaningful error.
|
|
117
|
+
*/
|
|
118
|
+
export class VersionMismatch extends LSDReqResMessage('LSD.Leader.VersionMismatch', {
|
|
119
|
+
/** The version running in the app */
|
|
120
|
+
appVersion: Schema.String,
|
|
121
|
+
/** The version that was sent by DevTools (that caused the mismatch) */
|
|
122
|
+
receivedVersion: Schema.String,
|
|
123
|
+
}) {}
|
|
124
|
+
|
|
113
125
|
export class Disconnect extends LSDReqResMessage('LSD.Leader.Disconnect', {}) {}
|
|
114
126
|
|
|
115
127
|
export const SetSyncLatch = LeaderReqResMessage('LSD.Leader.SetSyncLatch', {
|
|
@@ -180,6 +192,7 @@ export const MessageFromApp = Schema.Union(
|
|
|
180
192
|
NetworkStatusRes,
|
|
181
193
|
CommitEventRes,
|
|
182
194
|
Pong,
|
|
195
|
+
VersionMismatch,
|
|
183
196
|
DatabaseFileInfoRes,
|
|
184
197
|
SyncHistoryRes,
|
|
185
198
|
SyncingInfoRes,
|
|
@@ -37,6 +37,11 @@ import { rollback } from './materialize-event.ts'
|
|
|
37
37
|
import type { InitialBlockingSyncContext, LeaderSyncProcessor } from './types.ts'
|
|
38
38
|
import { LeaderThreadCtx } from './types.ts'
|
|
39
39
|
|
|
40
|
+
// WORKAROUND: @effect/opentelemetry mis-parses `Span.addEvent(name, attributes)` and treats the attributes object as a
|
|
41
|
+
// time input, causing `TypeError: {} is not iterable` at runtime.
|
|
42
|
+
// Upstream: https://github.com/Effect-TS/effect/pull/5929
|
|
43
|
+
// TODO: simplify back to the 2-arg overload once the upstream fix is released and adopted.
|
|
44
|
+
|
|
40
45
|
type LocalPushQueueItem = [
|
|
41
46
|
event: LiveStoreEvent.Client.EncodedWithMeta,
|
|
42
47
|
deferred: Deferred.Deferred<void, LeaderAheadError> | undefined,
|
|
@@ -90,10 +95,43 @@ export const makeLeaderSyncProcessor = ({
|
|
|
90
95
|
onError: 'shutdown' | 'ignore'
|
|
91
96
|
params: {
|
|
92
97
|
/**
|
|
98
|
+
* Maximum number of local events to process per batch cycle.
|
|
99
|
+
*
|
|
100
|
+
* This controls how many events from client sessions are applied to the local state
|
|
101
|
+
* in a single iteration before yielding to allow potential backend pulls.
|
|
102
|
+
*
|
|
103
|
+
* **Trade-offs:**
|
|
104
|
+
* - **Lower values (1-5):** More responsive to remote updates since pull processing can
|
|
105
|
+
* interleave more frequently. Better for high-conflict scenarios where rebases are common.
|
|
106
|
+
* Slightly higher per-event overhead due to more frequent transaction commits.
|
|
107
|
+
*
|
|
108
|
+
* - **Higher values (10-50+):** Better throughput for bulk local writes as more events are
|
|
109
|
+
* batched into a single transaction. However, may delay remote update processing and
|
|
110
|
+
* increase rebase complexity if many local events queue up during a slow pull.
|
|
111
|
+
*
|
|
112
|
+
* - **Very high values (100+):** Risk of starvation for pull processing if local pushes
|
|
113
|
+
* arrive continuously. May cause larger rollbacks during rebases. Not recommended
|
|
114
|
+
* unless you have a write-heavy workload with minimal remote synchronization.
|
|
115
|
+
*
|
|
93
116
|
* @default 10
|
|
94
117
|
*/
|
|
95
118
|
localPushBatchSize?: number
|
|
96
119
|
/**
|
|
120
|
+
* Maximum number of events to push to the sync backend per batch.
|
|
121
|
+
*
|
|
122
|
+
* This controls how many events are sent in a single push request to the remote server.
|
|
123
|
+
*
|
|
124
|
+
* **Trade-offs:**
|
|
125
|
+
* - **Lower values (1-10):** Lower latency for each push operation. Faster feedback on
|
|
126
|
+
* push success/failure. Slightly higher network overhead due to more requests.
|
|
127
|
+
*
|
|
128
|
+
* - **Higher values (50-100):** Better network efficiency by amortizing request overhead.
|
|
129
|
+
* Preferred for high-throughput scenarios. May increase latency to first confirmation.
|
|
130
|
+
*
|
|
131
|
+
* - **Very high values (200+):** Risk of hitting server request size limits or timeouts.
|
|
132
|
+
* A single failed request loses the entire batch (will be retried). May cause memory
|
|
133
|
+
* pressure if events accumulate faster than they can be pushed.
|
|
134
|
+
*
|
|
97
135
|
* @default 50
|
|
98
136
|
*/
|
|
99
137
|
backendPushBatchSize?: number
|
|
@@ -111,8 +149,8 @@ export const makeLeaderSyncProcessor = ({
|
|
|
111
149
|
}): Effect.Effect<LeaderSyncProcessor, UnknownError, Scope.Scope> =>
|
|
112
150
|
Effect.gen(function* () {
|
|
113
151
|
const syncBackendPushQueue = yield* BucketQueue.make<LiveStoreEvent.Client.EncodedWithMeta>()
|
|
114
|
-
const localPushBatchSize = params.localPushBatchSize ??
|
|
115
|
-
const backendPushBatchSize = params.backendPushBatchSize ??
|
|
152
|
+
const localPushBatchSize = params.localPushBatchSize ?? 10
|
|
153
|
+
const backendPushBatchSize = params.backendPushBatchSize ?? 50
|
|
116
154
|
|
|
117
155
|
const syncStateSref = yield* SubscriptionRef.make<SyncState.SyncState | undefined>(undefined)
|
|
118
156
|
|
|
@@ -443,10 +481,14 @@ const backgroundApplyLocalPushes = ({
|
|
|
443
481
|
)
|
|
444
482
|
|
|
445
483
|
if (droppedItems.length > 0) {
|
|
446
|
-
otelSpan?.addEvent(
|
|
447
|
-
|
|
448
|
-
|
|
449
|
-
|
|
484
|
+
otelSpan?.addEvent(
|
|
485
|
+
`push:drop-old-generation`,
|
|
486
|
+
{
|
|
487
|
+
droppedCount: droppedItems.length,
|
|
488
|
+
currentRebaseGeneration,
|
|
489
|
+
},
|
|
490
|
+
undefined,
|
|
491
|
+
)
|
|
450
492
|
|
|
451
493
|
/**
|
|
452
494
|
* Dropped pushes may still have a deferred awaiting completion.
|
|
@@ -484,20 +526,28 @@ const backgroundApplyLocalPushes = ({
|
|
|
484
526
|
|
|
485
527
|
switch (mergeResult._tag) {
|
|
486
528
|
case 'unknown-error': {
|
|
487
|
-
otelSpan?.addEvent(
|
|
488
|
-
|
|
489
|
-
|
|
490
|
-
|
|
529
|
+
otelSpan?.addEvent(
|
|
530
|
+
`push:unknown-error`,
|
|
531
|
+
{
|
|
532
|
+
batchSize: newEvents.length,
|
|
533
|
+
newEvents: TRACE_VERBOSE ? JSON.stringify(newEvents) : undefined,
|
|
534
|
+
},
|
|
535
|
+
undefined,
|
|
536
|
+
)
|
|
491
537
|
return yield* new UnknownError({ cause: mergeResult.message })
|
|
492
538
|
}
|
|
493
539
|
case 'rebase': {
|
|
494
540
|
return shouldNeverHappen('The leader thread should never have to rebase due to a local push')
|
|
495
541
|
}
|
|
496
542
|
case 'reject': {
|
|
497
|
-
otelSpan?.addEvent(
|
|
498
|
-
|
|
499
|
-
|
|
500
|
-
|
|
543
|
+
otelSpan?.addEvent(
|
|
544
|
+
`push:reject`,
|
|
545
|
+
{
|
|
546
|
+
batchSize: newEvents.length,
|
|
547
|
+
mergeResult: TRACE_VERBOSE ? JSON.stringify(mergeResult) : undefined,
|
|
548
|
+
},
|
|
549
|
+
undefined,
|
|
550
|
+
)
|
|
501
551
|
|
|
502
552
|
// TODO: how to test this?
|
|
503
553
|
const nextRebaseGeneration = currentRebaseGeneration + 1
|
|
@@ -552,10 +602,14 @@ const backgroundApplyLocalPushes = ({
|
|
|
552
602
|
leaderHead: mergeResult.newSyncState.localHead,
|
|
553
603
|
})
|
|
554
604
|
|
|
555
|
-
otelSpan?.addEvent(
|
|
556
|
-
|
|
557
|
-
|
|
558
|
-
|
|
605
|
+
otelSpan?.addEvent(
|
|
606
|
+
`push:advance`,
|
|
607
|
+
{
|
|
608
|
+
batchSize: newEvents.length,
|
|
609
|
+
mergeResult: TRACE_VERBOSE ? JSON.stringify(mergeResult) : undefined,
|
|
610
|
+
},
|
|
611
|
+
undefined,
|
|
612
|
+
)
|
|
559
613
|
|
|
560
614
|
// Don't sync client-local events
|
|
561
615
|
const filteredBatch = mergeResult.newEvents.filter((eventEncoded) => {
|
|
@@ -686,10 +740,14 @@ const backgroundBackendPulling = ({
|
|
|
686
740
|
if (mergeResult._tag === 'reject') {
|
|
687
741
|
return shouldNeverHappen('The leader thread should never reject upstream advances')
|
|
688
742
|
} else if (mergeResult._tag === 'unknown-error') {
|
|
689
|
-
otelSpan?.addEvent(
|
|
690
|
-
|
|
691
|
-
|
|
692
|
-
|
|
743
|
+
otelSpan?.addEvent(
|
|
744
|
+
`pull:unknown-error`,
|
|
745
|
+
{
|
|
746
|
+
newEventsCount: newEvents.length,
|
|
747
|
+
newEvents: TRACE_VERBOSE ? JSON.stringify(newEvents) : undefined,
|
|
748
|
+
},
|
|
749
|
+
undefined,
|
|
750
|
+
)
|
|
693
751
|
return yield* new UnknownError({ cause: mergeResult.message })
|
|
694
752
|
}
|
|
695
753
|
|
|
@@ -698,12 +756,16 @@ const backgroundBackendPulling = ({
|
|
|
698
756
|
Eventlog.updateBackendHead(dbEventlog, newBackendHead)
|
|
699
757
|
|
|
700
758
|
if (mergeResult._tag === 'rebase') {
|
|
701
|
-
otelSpan?.addEvent(
|
|
702
|
-
|
|
703
|
-
|
|
704
|
-
|
|
705
|
-
|
|
706
|
-
|
|
759
|
+
otelSpan?.addEvent(
|
|
760
|
+
`pull:rebase[${mergeResult.newSyncState.localHead.rebaseGeneration}]`,
|
|
761
|
+
{
|
|
762
|
+
newEventsCount: newEvents.length,
|
|
763
|
+
newEvents: TRACE_VERBOSE ? JSON.stringify(newEvents) : undefined,
|
|
764
|
+
rollbackCount: mergeResult.rollbackEvents.length,
|
|
765
|
+
mergeResult: TRACE_VERBOSE ? JSON.stringify(mergeResult) : undefined,
|
|
766
|
+
},
|
|
767
|
+
undefined,
|
|
768
|
+
)
|
|
707
769
|
|
|
708
770
|
const globalRebasedPendingEvents = mergeResult.newSyncState.pending.filter((event) => {
|
|
709
771
|
const eventDef = schema.eventsDefsMap.get(event.name)
|
|
@@ -724,10 +786,14 @@ const backgroundBackendPulling = ({
|
|
|
724
786
|
leaderHead: mergeResult.newSyncState.localHead,
|
|
725
787
|
})
|
|
726
788
|
} else {
|
|
727
|
-
otelSpan?.addEvent(
|
|
728
|
-
|
|
729
|
-
|
|
730
|
-
|
|
789
|
+
otelSpan?.addEvent(
|
|
790
|
+
`pull:advance`,
|
|
791
|
+
{
|
|
792
|
+
newEventsCount: newEvents.length,
|
|
793
|
+
mergeResult: TRACE_VERBOSE ? JSON.stringify(mergeResult) : undefined,
|
|
794
|
+
},
|
|
795
|
+
undefined,
|
|
796
|
+
)
|
|
731
797
|
|
|
732
798
|
// Ensure push fiber is active after advance by restarting with current pending (non-client) events
|
|
733
799
|
const globalPendingEvents = mergeResult.newSyncState.pending.filter((event) => {
|
|
@@ -837,10 +903,14 @@ const backgroundBackendPushing = ({
|
|
|
837
903
|
yield* devtoolsLatch.await
|
|
838
904
|
}
|
|
839
905
|
|
|
840
|
-
otelSpan?.addEvent(
|
|
841
|
-
|
|
842
|
-
|
|
843
|
-
|
|
906
|
+
otelSpan?.addEvent(
|
|
907
|
+
'backend-push',
|
|
908
|
+
{
|
|
909
|
+
batchSize: queueItems.length,
|
|
910
|
+
batch: TRACE_VERBOSE ? JSON.stringify(queueItems) : undefined,
|
|
911
|
+
},
|
|
912
|
+
undefined,
|
|
913
|
+
)
|
|
844
914
|
|
|
845
915
|
// Push with declarative retry/backoff using Effect schedules
|
|
846
916
|
// - Exponential backoff starting at 1s and doubling (1s, 2s, 4s, 8s, 16s, 30s ...)
|
|
@@ -867,15 +937,19 @@ const backgroundBackendPushing = ({
|
|
|
867
937
|
|
|
868
938
|
const retries = iteration.recurrence
|
|
869
939
|
if (retries > 0 && pushResult._tag === 'Right') {
|
|
870
|
-
otelSpan?.addEvent('backend-push-retry-success', { retries, batchSize: queueItems.length })
|
|
940
|
+
otelSpan?.addEvent('backend-push-retry-success', { retries, batchSize: queueItems.length }, undefined)
|
|
871
941
|
}
|
|
872
942
|
|
|
873
943
|
if (pushResult._tag === 'Left') {
|
|
874
|
-
otelSpan?.addEvent(
|
|
875
|
-
error
|
|
876
|
-
|
|
877
|
-
|
|
878
|
-
|
|
944
|
+
otelSpan?.addEvent(
|
|
945
|
+
'backend-push-error',
|
|
946
|
+
{
|
|
947
|
+
error: pushResult.left.toString(),
|
|
948
|
+
retries,
|
|
949
|
+
batchSize: queueItems.length,
|
|
950
|
+
},
|
|
951
|
+
undefined,
|
|
952
|
+
)
|
|
879
953
|
const error = pushResult.left
|
|
880
954
|
if (
|
|
881
955
|
error._tag === 'IsOfflineError' ||
|
|
@@ -1,6 +1,5 @@
|
|
|
1
1
|
import { LS_DEV, shouldNeverHappen } from '@livestore/utils'
|
|
2
|
-
import { Effect, Option, Schema } from '@livestore/utils/effect'
|
|
3
|
-
|
|
2
|
+
import { Chunk, Effect, Option, Schema } from '@livestore/utils/effect'
|
|
4
3
|
import type { SqliteDb } from '../adapter-types.ts'
|
|
5
4
|
import * as EventSequenceNumber from '../schema/EventSequenceNumber/mod.ts'
|
|
6
5
|
import * as LiveStoreEvent from '../schema/LiveStoreEvent/mod.ts'
|
|
@@ -16,8 +15,8 @@ import { insertRow, updateRows } from '../sql-queries/sql-queries.ts'
|
|
|
16
15
|
import type { PreparedBindValues } from '../util.ts'
|
|
17
16
|
import { sql } from '../util.ts'
|
|
18
17
|
import { execSql } from './connection.ts'
|
|
19
|
-
import type { InitialSyncInfo } from './types.ts'
|
|
20
|
-
import { LeaderThreadCtx } from './types.ts'
|
|
18
|
+
import type { InitialSyncInfo, StreamEventsOptions } from './types.ts'
|
|
19
|
+
import { LeaderThreadCtx, STREAM_EVENTS_BATCH_SIZE_DEFAULT } from './types.ts'
|
|
21
20
|
|
|
22
21
|
export const initEventlogDb = (dbEventlog: SqliteDb) =>
|
|
23
22
|
Effect.gen(function* () {
|
|
@@ -100,6 +99,83 @@ export const getEventsSince = ({
|
|
|
100
99
|
.sort((a, b) => EventSequenceNumber.Client.compare(a.seqNum, b.seqNum))
|
|
101
100
|
}
|
|
102
101
|
|
|
102
|
+
export const getEventsFromEventlog = ({
|
|
103
|
+
dbEventlog,
|
|
104
|
+
options,
|
|
105
|
+
}: {
|
|
106
|
+
dbEventlog: SqliteDb
|
|
107
|
+
options: StreamEventsOptions
|
|
108
|
+
}): Effect.Effect<Chunk.Chunk<LiveStoreEvent.Client.Encoded>> =>
|
|
109
|
+
Effect.gen(function* () {
|
|
110
|
+
const since = options.since ?? EventSequenceNumber.Client.ROOT
|
|
111
|
+
const batchSize = options.batchSize ?? STREAM_EVENTS_BATCH_SIZE_DEFAULT
|
|
112
|
+
|
|
113
|
+
const makeQuery = () => {
|
|
114
|
+
let query = eventlogMetaTable.where('seqNumGlobal', '>', since.global)
|
|
115
|
+
|
|
116
|
+
if (options.until) {
|
|
117
|
+
query = query.where('seqNumGlobal', '<=', options.until.global)
|
|
118
|
+
}
|
|
119
|
+
|
|
120
|
+
if (options.filter && options.filter.length > 0) {
|
|
121
|
+
query = query.where({ name: { op: 'IN', value: options.filter } })
|
|
122
|
+
}
|
|
123
|
+
|
|
124
|
+
if (options.clientIds && options.clientIds.length > 0) {
|
|
125
|
+
query = query.where({ clientId: { op: 'IN', value: options.clientIds } })
|
|
126
|
+
}
|
|
127
|
+
|
|
128
|
+
if (options.sessionIds && options.sessionIds.length > 0) {
|
|
129
|
+
query = query.where({ sessionId: { op: 'IN', value: options.sessionIds } })
|
|
130
|
+
}
|
|
131
|
+
|
|
132
|
+
if (options.includeClientOnly !== true) {
|
|
133
|
+
query = query.where('seqNumClient', '<=', EventSequenceNumber.Client.DEFAULT)
|
|
134
|
+
}
|
|
135
|
+
|
|
136
|
+
return query
|
|
137
|
+
.orderBy([
|
|
138
|
+
{ col: 'seqNumGlobal', direction: 'asc' },
|
|
139
|
+
{ col: 'seqNumClient', direction: 'asc' },
|
|
140
|
+
])
|
|
141
|
+
.limit(batchSize)
|
|
142
|
+
}
|
|
143
|
+
|
|
144
|
+
const eventlogEvents = yield* Effect.sync(() => dbEventlog.select(makeQuery()))
|
|
145
|
+
|
|
146
|
+
if (eventlogEvents.length === 0) {
|
|
147
|
+
return Chunk.empty<LiveStoreEvent.Client.Encoded>()
|
|
148
|
+
}
|
|
149
|
+
|
|
150
|
+
const spanAttributes = {
|
|
151
|
+
'livestore.eventLog.since': since.global,
|
|
152
|
+
'livestore.eventLog.until': options.until?.global,
|
|
153
|
+
}
|
|
154
|
+
|
|
155
|
+
return yield* Effect.sync(() => {
|
|
156
|
+
const encodedEvents = eventlogEvents.map((eventlogEvent) => {
|
|
157
|
+
return LiveStoreEvent.Client.Encoded.make({
|
|
158
|
+
name: eventlogEvent.name,
|
|
159
|
+
args: eventlogEvent.argsJson,
|
|
160
|
+
seqNum: {
|
|
161
|
+
global: eventlogEvent.seqNumGlobal,
|
|
162
|
+
client: eventlogEvent.seqNumClient,
|
|
163
|
+
rebaseGeneration: eventlogEvent.seqNumRebaseGeneration,
|
|
164
|
+
},
|
|
165
|
+
parentSeqNum: {
|
|
166
|
+
global: eventlogEvent.parentSeqNumGlobal,
|
|
167
|
+
client: eventlogEvent.parentSeqNumClient,
|
|
168
|
+
rebaseGeneration: eventlogEvent.parentSeqNumRebaseGeneration,
|
|
169
|
+
},
|
|
170
|
+
clientId: eventlogEvent.clientId,
|
|
171
|
+
sessionId: eventlogEvent.sessionId,
|
|
172
|
+
})
|
|
173
|
+
})
|
|
174
|
+
|
|
175
|
+
return Chunk.fromIterable(encodedEvents)
|
|
176
|
+
}).pipe(Effect.withSpan('@livestore/common:eventlog:getEventsFromEventlog', { attributes: spanAttributes }))
|
|
177
|
+
})
|
|
178
|
+
|
|
103
179
|
export const getClientHeadFromDb = (dbEventlog: SqliteDb): EventSequenceNumber.Client.Composite => {
|
|
104
180
|
const res = dbEventlog.select<{
|
|
105
181
|
seqNumGlobal: EventSequenceNumber.Global.Type
|
|
@@ -94,6 +94,22 @@ const listenToDevtools = ({
|
|
|
94
94
|
type RequestId = string
|
|
95
95
|
const handledRequestIds = new Set<RequestId>()
|
|
96
96
|
|
|
97
|
+
type LoadDatabaseKind = 'state' | 'eventlog'
|
|
98
|
+
const loadDatabaseBatchTracker = new Map<string, Set<LoadDatabaseKind>>()
|
|
99
|
+
|
|
100
|
+
const registerBatchProgress = (batchId: string, kind: LoadDatabaseKind) => {
|
|
101
|
+
const entry = loadDatabaseBatchTracker.get(batchId) ?? new Set<LoadDatabaseKind>()
|
|
102
|
+
entry.add(kind)
|
|
103
|
+
loadDatabaseBatchTracker.set(batchId, entry)
|
|
104
|
+
const finished = entry.has('state') && entry.has('eventlog')
|
|
105
|
+
|
|
106
|
+
if (finished) {
|
|
107
|
+
loadDatabaseBatchTracker.delete(batchId)
|
|
108
|
+
}
|
|
109
|
+
|
|
110
|
+
return finished
|
|
111
|
+
}
|
|
112
|
+
|
|
97
113
|
yield* incomingMessages.pipe(
|
|
98
114
|
Stream.tap((decodedEvent) =>
|
|
99
115
|
Effect.gen(function* () {
|
|
@@ -122,6 +138,17 @@ const listenToDevtools = ({
|
|
|
122
138
|
|
|
123
139
|
switch (decodedEvent._tag) {
|
|
124
140
|
case 'LSD.Leader.Ping': {
|
|
141
|
+
// Check version mismatch and respond with VersionMismatch if versions don't match
|
|
142
|
+
if (decodedEvent.liveStoreVersion !== liveStoreVersion) {
|
|
143
|
+
yield* sendMessage(
|
|
144
|
+
Devtools.Leader.VersionMismatch.make({
|
|
145
|
+
...reqPayload,
|
|
146
|
+
appVersion: liveStoreVersion,
|
|
147
|
+
receivedVersion: decodedEvent.liveStoreVersion,
|
|
148
|
+
}),
|
|
149
|
+
)
|
|
150
|
+
return
|
|
151
|
+
}
|
|
125
152
|
yield* sendMessage(Devtools.Leader.Pong.make({ ...reqPayload }))
|
|
126
153
|
return
|
|
127
154
|
}
|
|
@@ -133,7 +160,7 @@ const listenToDevtools = ({
|
|
|
133
160
|
return
|
|
134
161
|
}
|
|
135
162
|
case 'LSD.Leader.LoadDatabaseFile.Request': {
|
|
136
|
-
const { data } = decodedEvent
|
|
163
|
+
const { data, batchId } = decodedEvent
|
|
137
164
|
|
|
138
165
|
const handleLoadDb = Effect.gen(function* () {
|
|
139
166
|
const tableNames = yield* Effect.acquireRelease(makeSqliteDb({ _tag: 'in-memory' }), (db) =>
|
|
@@ -148,25 +175,44 @@ const listenToDevtools = ({
|
|
|
148
175
|
),
|
|
149
176
|
)
|
|
150
177
|
|
|
178
|
+
let databaseKind: LoadDatabaseKind | undefined
|
|
179
|
+
|
|
151
180
|
if (tableNames.has(SystemTables.EVENTLOG_META_TABLE)) {
|
|
152
|
-
|
|
181
|
+
databaseKind = 'eventlog'
|
|
153
182
|
yield* SubscriptionRef.set(shutdownStateSubRef, 'shutting-down')
|
|
154
183
|
yield* Effect.try(() => void dbEventlog.import(data))
|
|
155
|
-
|
|
184
|
+
|
|
185
|
+
if (batchId === undefined) {
|
|
186
|
+
yield* Effect.try(() => void dbState.destroy())
|
|
187
|
+
}
|
|
156
188
|
} else if (
|
|
157
189
|
tableNames.has(SystemTables.SCHEMA_META_TABLE) &&
|
|
158
190
|
tableNames.has(SystemTables.SCHEMA_EVENT_DEFS_META_TABLE)
|
|
159
191
|
) {
|
|
160
|
-
|
|
192
|
+
databaseKind = 'state'
|
|
161
193
|
yield* SubscriptionRef.set(shutdownStateSubRef, 'shutting-down')
|
|
162
194
|
yield* Effect.try(() => void dbState.import(data))
|
|
163
|
-
|
|
195
|
+
|
|
196
|
+
if (batchId === undefined) {
|
|
197
|
+
yield* Effect.try(() => void dbEventlog.destroy())
|
|
198
|
+
}
|
|
164
199
|
} else {
|
|
165
200
|
return yield* Effect.fail({ _tag: 'unsupported-database' } as const)
|
|
166
201
|
}
|
|
167
202
|
|
|
203
|
+
const resolvedDatabaseKind = databaseKind
|
|
204
|
+
if (resolvedDatabaseKind === undefined) {
|
|
205
|
+
return yield* Effect.fail({ _tag: 'unsupported-database' } as const)
|
|
206
|
+
}
|
|
207
|
+
|
|
208
|
+
const shouldShutdown =
|
|
209
|
+
batchId === undefined ? true : registerBatchProgress(batchId, resolvedDatabaseKind)
|
|
210
|
+
|
|
168
211
|
yield* sendMessage(Devtools.Leader.LoadDatabaseFile.Success.make({ ...reqPayload }))
|
|
169
|
-
|
|
212
|
+
|
|
213
|
+
if (shouldShutdown) {
|
|
214
|
+
yield* shutdownChannel.send(IntentionalShutdownCause.make({ reason: 'devtools-import' }))
|
|
215
|
+
}
|
|
170
216
|
})
|
|
171
217
|
|
|
172
218
|
yield* handleLoadDb.pipe(
|
|
@@ -55,6 +55,8 @@ export interface MakeLeaderThreadLayerParams {
|
|
|
55
55
|
dbEventlog: LeaderSqliteDb
|
|
56
56
|
devtoolsOptions: DevtoolsOptions
|
|
57
57
|
shutdownChannel: ShutdownChannel
|
|
58
|
+
/** Boot warning to emit (e.g., OPFS unavailable in private browsing) */
|
|
59
|
+
bootWarning?: BootStatus
|
|
58
60
|
params?: {
|
|
59
61
|
localPushBatchSize?: number
|
|
60
62
|
backendPushBatchSize?: number
|
|
@@ -80,6 +82,7 @@ export const makeLeaderThreadLayer = ({
|
|
|
80
82
|
dbEventlog,
|
|
81
83
|
devtoolsOptions,
|
|
82
84
|
shutdownChannel,
|
|
85
|
+
bootWarning,
|
|
83
86
|
params,
|
|
84
87
|
testing,
|
|
85
88
|
}: MakeLeaderThreadLayerParams): Layer.Layer<LeaderThreadCtx, UnknownError, Scope.Scope | HttpClient.HttpClient> =>
|
|
@@ -89,6 +92,11 @@ export const makeLeaderThreadLayer = ({
|
|
|
89
92
|
|
|
90
93
|
const bootStatusQueue = yield* Queue.unbounded<BootStatus>().pipe(Effect.acquireRelease(Queue.shutdown))
|
|
91
94
|
|
|
95
|
+
// Emit boot warning if present (e.g., OPFS unavailable in private browsing)
|
|
96
|
+
if (bootWarning !== undefined) {
|
|
97
|
+
yield* Queue.offer(bootStatusQueue, bootWarning)
|
|
98
|
+
}
|
|
99
|
+
|
|
92
100
|
const dbEventlogMissing = !hasEventlogTables(dbEventlog)
|
|
93
101
|
|
|
94
102
|
// Either happens on initial boot or if schema changes
|
|
@@ -3,6 +3,7 @@ import { Effect, Option, ReadonlyArray, Schema } from '@livestore/utils/effect'
|
|
|
3
3
|
|
|
4
4
|
import { MaterializeError, MaterializerHashMismatchError, type SqliteDb } from '../adapter-types.ts'
|
|
5
5
|
import { getExecStatementsFromMaterializer, hashMaterializerResults } from '../materializer-helper.ts'
|
|
6
|
+
import { logDeprecationWarnings } from '../schema/EventDef/deprecated.ts'
|
|
6
7
|
import type { LiveStoreSchema } from '../schema/mod.ts'
|
|
7
8
|
import { EventSequenceNumber, resolveEventDef, SystemTables, UNKNOWN_EVENT_SCHEMA_HASH } from '../schema/mod.ts'
|
|
8
9
|
import { insertRow } from '../sql-queries/index.ts'
|
|
@@ -62,6 +63,9 @@ export const makeMaterializeEvent = ({
|
|
|
62
63
|
|
|
63
64
|
const { eventDef, materializer } = resolution
|
|
64
65
|
|
|
66
|
+
// Log deprecation warnings for deprecated events/fields
|
|
67
|
+
yield* logDeprecationWarnings(eventDef, eventEncoded.args as Record<string, unknown>)
|
|
68
|
+
|
|
65
69
|
const execArgsArr = getExecStatementsFromMaterializer({
|
|
66
70
|
eventDef,
|
|
67
71
|
materializer,
|
package/src/leader-thread/mod.ts
CHANGED