@livestore/common 0.3.0-dev.10 → 0.3.0-dev.12
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/.tsbuildinfo +1 -1
- package/dist/adapter-types.d.ts +62 -30
- package/dist/adapter-types.d.ts.map +1 -1
- package/dist/adapter-types.js +12 -0
- package/dist/adapter-types.js.map +1 -1
- package/dist/devtools/devtool-message-leader.d.ts +2 -0
- package/dist/devtools/devtool-message-leader.d.ts.map +1 -0
- package/dist/devtools/devtool-message-leader.js +2 -0
- package/dist/devtools/devtool-message-leader.js.map +1 -0
- package/dist/devtools/devtools-bridge.d.ts +10 -7
- package/dist/devtools/devtools-bridge.d.ts.map +1 -1
- package/dist/devtools/devtools-messages-client-session.d.ts +370 -0
- package/dist/devtools/devtools-messages-client-session.d.ts.map +1 -0
- package/dist/devtools/devtools-messages-client-session.js +77 -0
- package/dist/devtools/devtools-messages-client-session.js.map +1 -0
- package/dist/devtools/devtools-messages-common.d.ts +57 -0
- package/dist/devtools/devtools-messages-common.d.ts.map +1 -0
- package/dist/devtools/devtools-messages-common.js +44 -0
- package/dist/devtools/devtools-messages-common.js.map +1 -0
- package/dist/devtools/devtools-messages-leader.d.ts +437 -0
- package/dist/devtools/devtools-messages-leader.d.ts.map +1 -0
- package/dist/devtools/devtools-messages-leader.js +132 -0
- package/dist/devtools/devtools-messages-leader.js.map +1 -0
- package/dist/devtools/devtools-messages.d.ts +3 -580
- package/dist/devtools/devtools-messages.d.ts.map +1 -1
- package/dist/devtools/devtools-messages.js +3 -174
- package/dist/devtools/devtools-messages.js.map +1 -1
- package/dist/init-singleton-tables.d.ts +2 -2
- package/dist/init-singleton-tables.d.ts.map +1 -1
- package/dist/init-singleton-tables.js.map +1 -1
- package/dist/leader-thread/LeaderSyncProcessor.d.ts +4 -4
- package/dist/leader-thread/LeaderSyncProcessor.d.ts.map +1 -1
- package/dist/leader-thread/LeaderSyncProcessor.js +64 -36
- package/dist/leader-thread/LeaderSyncProcessor.js.map +1 -1
- package/dist/leader-thread/apply-mutation.d.ts.map +1 -1
- package/dist/leader-thread/apply-mutation.js +4 -4
- package/dist/leader-thread/apply-mutation.js.map +1 -1
- package/dist/leader-thread/connection.d.ts +34 -6
- package/dist/leader-thread/connection.d.ts.map +1 -1
- package/dist/leader-thread/connection.js +22 -7
- package/dist/leader-thread/connection.js.map +1 -1
- package/dist/leader-thread/leader-worker-devtools.js +67 -36
- package/dist/leader-thread/leader-worker-devtools.js.map +1 -1
- package/dist/leader-thread/make-leader-thread-layer.d.ts +6 -6
- package/dist/leader-thread/make-leader-thread-layer.d.ts.map +1 -1
- package/dist/leader-thread/make-leader-thread-layer.js +38 -13
- package/dist/leader-thread/make-leader-thread-layer.js.map +1 -1
- package/dist/leader-thread/mutationlog.d.ts +4 -4
- package/dist/leader-thread/mutationlog.d.ts.map +1 -1
- package/dist/leader-thread/mutationlog.js +6 -6
- package/dist/leader-thread/mutationlog.js.map +1 -1
- package/dist/leader-thread/pull-queue-set.d.ts.map +1 -1
- package/dist/leader-thread/recreate-db.d.ts +4 -2
- package/dist/leader-thread/recreate-db.d.ts.map +1 -1
- package/dist/leader-thread/recreate-db.js +27 -22
- package/dist/leader-thread/recreate-db.js.map +1 -1
- package/dist/leader-thread/types.d.ts +32 -17
- package/dist/leader-thread/types.d.ts.map +1 -1
- package/dist/leader-thread/types.js +0 -2
- package/dist/leader-thread/types.js.map +1 -1
- package/dist/query-builder/api.d.ts +2 -2
- package/dist/query-builder/api.d.ts.map +1 -1
- package/dist/query-builder/impl.js.map +1 -1
- package/dist/query-builder/impl.test.js +16 -1
- package/dist/query-builder/impl.test.js.map +1 -1
- package/dist/query-info.d.ts +3 -3
- package/dist/query-info.d.ts.map +1 -1
- package/dist/rehydrate-from-mutationlog.d.ts +3 -3
- package/dist/rehydrate-from-mutationlog.d.ts.map +1 -1
- package/dist/rehydrate-from-mutationlog.js.map +1 -1
- package/dist/schema/EventId.d.ts +1 -0
- package/dist/schema/EventId.d.ts.map +1 -1
- package/dist/schema/EventId.js +3 -0
- package/dist/schema/EventId.js.map +1 -1
- package/dist/schema/mutations.d.ts +1 -1
- package/dist/schema/system-tables.d.ts +1 -1
- package/dist/schema-management/common.d.ts +3 -3
- package/dist/schema-management/common.d.ts.map +1 -1
- package/dist/schema-management/common.js.map +1 -1
- package/dist/schema-management/migrations.d.ts +5 -5
- package/dist/schema-management/migrations.d.ts.map +1 -1
- package/dist/schema-management/migrations.js +6 -1
- package/dist/schema-management/migrations.js.map +1 -1
- package/dist/sync/ClientSessionSyncProcessor.d.ts +8 -12
- package/dist/sync/ClientSessionSyncProcessor.d.ts.map +1 -1
- package/dist/sync/ClientSessionSyncProcessor.js +31 -13
- package/dist/sync/ClientSessionSyncProcessor.js.map +1 -1
- package/dist/sync/next/test/mutation-fixtures.d.ts +7 -7
- package/dist/version.d.ts +1 -1
- package/dist/version.js +1 -1
- package/package.json +3 -3
- package/src/adapter-types.ts +52 -33
- package/src/devtools/devtools-bridge.ts +10 -7
- package/src/devtools/devtools-messages-client-session.ts +125 -0
- package/src/devtools/devtools-messages-common.ts +81 -0
- package/src/devtools/devtools-messages-leader.ts +176 -0
- package/src/devtools/devtools-messages.ts +3 -246
- package/src/init-singleton-tables.ts +2 -2
- package/src/leader-thread/LeaderSyncProcessor.ts +94 -46
- package/src/leader-thread/apply-mutation.ts +5 -5
- package/src/leader-thread/connection.ts +54 -9
- package/src/leader-thread/leader-worker-devtools.ts +105 -41
- package/src/leader-thread/make-leader-thread-layer.ts +55 -22
- package/src/leader-thread/mutationlog.ts +9 -9
- package/src/leader-thread/recreate-db.ts +33 -24
- package/src/leader-thread/types.ts +38 -21
- package/src/query-builder/api.ts +3 -3
- package/src/query-builder/impl.test.ts +22 -1
- package/src/query-builder/impl.ts +2 -2
- package/src/query-info.ts +3 -3
- package/src/rehydrate-from-mutationlog.ts +3 -3
- package/src/schema/EventId.ts +4 -0
- package/src/schema-management/common.ts +3 -3
- package/src/schema-management/migrations.ts +12 -8
- package/src/sync/ClientSessionSyncProcessor.ts +38 -22
- package/src/version.ts +1 -1
@@ -1,5 +1,5 @@
|
|
1
|
-
import { isNotUndefined, shouldNeverHappen, TRACE_VERBOSE } from '@livestore/utils'
|
2
|
-
import type { HttpClient, Scope } from '@livestore/utils/effect'
|
1
|
+
import { isNotUndefined, LS_DEV, shouldNeverHappen, TRACE_VERBOSE } from '@livestore/utils'
|
2
|
+
import type { HttpClient, Scope, Tracer } from '@livestore/utils/effect'
|
3
3
|
import {
|
4
4
|
BucketQueue,
|
5
5
|
Deferred,
|
@@ -16,7 +16,7 @@ import {
|
|
16
16
|
} from '@livestore/utils/effect'
|
17
17
|
import type * as otel from '@opentelemetry/api'
|
18
18
|
|
19
|
-
import type {
|
19
|
+
import type { SqliteDb } from '../adapter-types.js'
|
20
20
|
import { UnexpectedError } from '../adapter-types.js'
|
21
21
|
import type { LiveStoreSchema, SessionChangesetMetaRow } from '../schema/mod.js'
|
22
22
|
import {
|
@@ -67,13 +67,13 @@ type PushQueueItem = [
|
|
67
67
|
export const makeLeaderSyncProcessor = ({
|
68
68
|
schema,
|
69
69
|
dbMissing,
|
70
|
-
|
70
|
+
dbMutationLog,
|
71
71
|
initialBlockingSyncContext,
|
72
72
|
}: {
|
73
73
|
schema: LiveStoreSchema
|
74
|
-
/** Only used to know whether we can safely query
|
74
|
+
/** Only used to know whether we can safely query dbMutationLog during setup execution */
|
75
75
|
dbMissing: boolean
|
76
|
-
|
76
|
+
dbMutationLog: SqliteDb
|
77
77
|
initialBlockingSyncContext: InitialBlockingSyncContext
|
78
78
|
}): Effect.Effect<LeaderSyncProcessor, UnexpectedError, Scope.Scope> =>
|
79
79
|
Effect.gen(function* () {
|
@@ -86,7 +86,16 @@ export const makeLeaderSyncProcessor = ({
|
|
86
86
|
return mutationDef.options.localOnly
|
87
87
|
}
|
88
88
|
|
89
|
-
|
89
|
+
// This context depends on data from `boot`, we should find a better implementation to avoid this ref indirection.
|
90
|
+
const ctxRef = {
|
91
|
+
current: undefined as
|
92
|
+
| undefined
|
93
|
+
| {
|
94
|
+
otelSpan: otel.Span | undefined
|
95
|
+
span: Tracer.Span
|
96
|
+
devtoolsLatch: Effect.Latch | undefined
|
97
|
+
},
|
98
|
+
}
|
90
99
|
|
91
100
|
const localPushesQueue = yield* BucketQueue.make<PushQueueItem>()
|
92
101
|
const localPushesLatch = yield* Effect.makeLatch(true)
|
@@ -119,9 +128,7 @@ export const makeLeaderSyncProcessor = ({
|
|
119
128
|
batchSize: newEvents.length,
|
120
129
|
batch: TRACE_VERBOSE ? newEvents : undefined,
|
121
130
|
},
|
122
|
-
links:
|
123
|
-
? [{ _tag: 'SpanLink', span: OtelTracer.makeExternalSpan(spanRef.current.spanContext()), attributes: {} }]
|
124
|
-
: undefined,
|
131
|
+
links: ctxRef.current?.span ? [{ _tag: 'SpanLink', span: ctxRef.current.span, attributes: {} }] : undefined,
|
125
132
|
}),
|
126
133
|
)
|
127
134
|
|
@@ -145,11 +152,18 @@ export const makeLeaderSyncProcessor = ({
|
|
145
152
|
// Starts various background loops
|
146
153
|
const boot: LeaderSyncProcessor['boot'] = ({ dbReady }) =>
|
147
154
|
Effect.gen(function* () {
|
148
|
-
const span = yield*
|
149
|
-
|
155
|
+
const span = yield* Effect.currentSpan.pipe(Effect.orDie)
|
156
|
+
const otelSpan = yield* OtelTracer.currentOtelSpan.pipe(Effect.catchAll(() => Effect.succeed(undefined)))
|
157
|
+
const { devtools } = yield* LeaderThreadCtx
|
150
158
|
|
151
|
-
|
152
|
-
|
159
|
+
ctxRef.current = {
|
160
|
+
otelSpan,
|
161
|
+
span,
|
162
|
+
devtoolsLatch: devtools.enabled ? devtools.syncBackendLatch : undefined,
|
163
|
+
}
|
164
|
+
|
165
|
+
const initialBackendHead = dbMissing ? EventId.ROOT.global : getBackendHeadFromDb(dbMutationLog)
|
166
|
+
const initialLocalHead = dbMissing ? EventId.ROOT : getLocalHeadFromDb(dbMutationLog)
|
153
167
|
|
154
168
|
if (initialBackendHead > initialLocalHead.global) {
|
155
169
|
return shouldNeverHappen(
|
@@ -193,14 +207,19 @@ export const makeLeaderSyncProcessor = ({
|
|
193
207
|
syncBackendQueue,
|
194
208
|
schema,
|
195
209
|
isLocalEvent,
|
196
|
-
|
210
|
+
otelSpan,
|
197
211
|
}).pipe(Effect.tapCauseLogPretty, Effect.forkScoped)
|
198
212
|
|
199
213
|
const backendPushingFiberHandle = yield* FiberHandle.make()
|
200
214
|
|
201
215
|
yield* FiberHandle.run(
|
202
216
|
backendPushingFiberHandle,
|
203
|
-
backgroundBackendPushing({
|
217
|
+
backgroundBackendPushing({
|
218
|
+
dbReady,
|
219
|
+
syncBackendQueue,
|
220
|
+
otelSpan,
|
221
|
+
devtoolsLatch: ctxRef.current?.devtoolsLatch,
|
222
|
+
}).pipe(Effect.tapCauseLogPretty),
|
204
223
|
)
|
205
224
|
|
206
225
|
yield* backgroundBackendPulling({
|
@@ -219,15 +238,23 @@ export const makeLeaderSyncProcessor = ({
|
|
219
238
|
// Restart pushing fiber
|
220
239
|
yield* FiberHandle.run(
|
221
240
|
backendPushingFiberHandle,
|
222
|
-
backgroundBackendPushing({
|
241
|
+
backgroundBackendPushing({
|
242
|
+
dbReady,
|
243
|
+
syncBackendQueue,
|
244
|
+
otelSpan,
|
245
|
+
devtoolsLatch: ctxRef.current?.devtoolsLatch,
|
246
|
+
}).pipe(Effect.tapCauseLogPretty),
|
223
247
|
)
|
224
248
|
}),
|
225
249
|
syncStateSref,
|
226
250
|
localPushesLatch,
|
227
251
|
pullLatch,
|
228
|
-
|
252
|
+
otelSpan,
|
229
253
|
initialBlockingSyncContext,
|
254
|
+
devtoolsLatch: ctxRef.current?.devtoolsLatch,
|
230
255
|
}).pipe(Effect.tapCauseLogPretty, Effect.forkScoped)
|
256
|
+
|
257
|
+
return { initialLeaderHead: initialLocalHead }
|
231
258
|
}).pipe(Effect.withSpanScoped('@livestore/common:leader-thread:syncing'))
|
232
259
|
|
233
260
|
return {
|
@@ -253,7 +280,7 @@ const backgroundApplyLocalPushes = ({
|
|
253
280
|
syncBackendQueue,
|
254
281
|
schema,
|
255
282
|
isLocalEvent,
|
256
|
-
|
283
|
+
otelSpan,
|
257
284
|
}: {
|
258
285
|
pullLatch: Effect.Latch
|
259
286
|
localPushesLatch: Effect.Latch
|
@@ -262,7 +289,7 @@ const backgroundApplyLocalPushes = ({
|
|
262
289
|
syncBackendQueue: BucketQueue.BucketQueue<MutationEvent.EncodedWithMeta>
|
263
290
|
schema: LiveStoreSchema
|
264
291
|
isLocalEvent: (mutationEventEncoded: MutationEvent.EncodedWithMeta) => boolean
|
265
|
-
|
292
|
+
otelSpan: otel.Span | undefined
|
266
293
|
}) =>
|
267
294
|
Effect.gen(function* () {
|
268
295
|
const { connectedClientSessionPullQueues } = yield* LeaderThreadCtx
|
@@ -293,7 +320,7 @@ const backgroundApplyLocalPushes = ({
|
|
293
320
|
if (updateResult._tag === 'rebase') {
|
294
321
|
return shouldNeverHappen('The leader thread should never have to rebase due to a local push')
|
295
322
|
} else if (updateResult._tag === 'reject') {
|
296
|
-
|
323
|
+
otelSpan?.addEvent('local-push:reject', {
|
297
324
|
batchSize: newEvents.length,
|
298
325
|
updateResult: TRACE_VERBOSE ? JSON.stringify(updateResult) : undefined,
|
299
326
|
})
|
@@ -331,7 +358,7 @@ const backgroundApplyLocalPushes = ({
|
|
331
358
|
remaining: 0,
|
332
359
|
})
|
333
360
|
|
334
|
-
|
361
|
+
otelSpan?.addEvent('local-push', {
|
335
362
|
batchSize: newEvents.length,
|
336
363
|
updateResult: TRACE_VERBOSE ? JSON.stringify(updateResult) : undefined,
|
337
364
|
})
|
@@ -361,14 +388,14 @@ type ApplyMutationItems = (_: {
|
|
361
388
|
const makeApplyMutationItems: Effect.Effect<ApplyMutationItems, UnexpectedError, LeaderThreadCtx | Scope.Scope> =
|
362
389
|
Effect.gen(function* () {
|
363
390
|
const leaderThreadCtx = yield* LeaderThreadCtx
|
364
|
-
const { db,
|
391
|
+
const { dbReadModel: db, dbMutationLog } = leaderThreadCtx
|
365
392
|
|
366
393
|
const applyMutation = yield* makeApplyMutation
|
367
394
|
|
368
395
|
return ({ batchItems, deferreds }) =>
|
369
396
|
Effect.gen(function* () {
|
370
397
|
db.execute('BEGIN TRANSACTION', undefined) // Start the transaction
|
371
|
-
|
398
|
+
dbMutationLog.execute('BEGIN TRANSACTION', undefined) // Start the transaction
|
372
399
|
|
373
400
|
yield* Effect.addFinalizer((exit) =>
|
374
401
|
Effect.gen(function* () {
|
@@ -376,7 +403,7 @@ const makeApplyMutationItems: Effect.Effect<ApplyMutationItems, UnexpectedError,
|
|
376
403
|
|
377
404
|
// Rollback in case of an error
|
378
405
|
db.execute('ROLLBACK', undefined)
|
379
|
-
|
406
|
+
dbMutationLog.execute('ROLLBACK', undefined)
|
380
407
|
}),
|
381
408
|
)
|
382
409
|
|
@@ -389,7 +416,7 @@ const makeApplyMutationItems: Effect.Effect<ApplyMutationItems, UnexpectedError,
|
|
389
416
|
}
|
390
417
|
|
391
418
|
db.execute('COMMIT', undefined) // Commit the transaction
|
392
|
-
|
419
|
+
dbMutationLog.execute('COMMIT', undefined) // Commit the transaction
|
393
420
|
}).pipe(
|
394
421
|
Effect.uninterruptible,
|
395
422
|
Effect.scoped,
|
@@ -406,10 +433,11 @@ const backgroundBackendPulling = ({
|
|
406
433
|
initialBackendHead,
|
407
434
|
isLocalEvent,
|
408
435
|
restartBackendPushing,
|
409
|
-
|
436
|
+
otelSpan,
|
410
437
|
syncStateSref,
|
411
438
|
localPushesLatch,
|
412
439
|
pullLatch,
|
440
|
+
devtoolsLatch,
|
413
441
|
initialBlockingSyncContext,
|
414
442
|
}: {
|
415
443
|
dbReady: Deferred.Deferred<void>
|
@@ -418,14 +446,21 @@ const backgroundBackendPulling = ({
|
|
418
446
|
restartBackendPushing: (
|
419
447
|
filteredRebasedPending: ReadonlyArray<MutationEvent.EncodedWithMeta>,
|
420
448
|
) => Effect.Effect<void, UnexpectedError, LeaderThreadCtx | HttpClient.HttpClient>
|
421
|
-
|
449
|
+
otelSpan: otel.Span | undefined
|
422
450
|
syncStateSref: SubscriptionRef.SubscriptionRef<SyncState.SyncState | undefined>
|
423
451
|
localPushesLatch: Effect.Latch
|
424
452
|
pullLatch: Effect.Latch
|
453
|
+
devtoolsLatch: Effect.Latch | undefined
|
425
454
|
initialBlockingSyncContext: InitialBlockingSyncContext
|
426
455
|
}) =>
|
427
456
|
Effect.gen(function* () {
|
428
|
-
const {
|
457
|
+
const {
|
458
|
+
syncBackend,
|
459
|
+
dbReadModel: db,
|
460
|
+
dbMutationLog,
|
461
|
+
connectedClientSessionPullQueues,
|
462
|
+
schema,
|
463
|
+
} = yield* LeaderThreadCtx
|
429
464
|
|
430
465
|
if (syncBackend === undefined) return
|
431
466
|
|
@@ -437,6 +472,10 @@ const backgroundBackendPulling = ({
|
|
437
472
|
Effect.gen(function* () {
|
438
473
|
if (newEvents.length === 0) return
|
439
474
|
|
475
|
+
if (devtoolsLatch !== undefined) {
|
476
|
+
yield* devtoolsLatch.await
|
477
|
+
}
|
478
|
+
|
440
479
|
// Prevent more local pushes from being processed until this pull is finished
|
441
480
|
yield* localPushesLatch.close
|
442
481
|
|
@@ -462,10 +501,10 @@ const backgroundBackendPulling = ({
|
|
462
501
|
|
463
502
|
const newBackendHead = newEvents.at(-1)!.id
|
464
503
|
|
465
|
-
updateBackendHead(
|
504
|
+
updateBackendHead(dbMutationLog, newBackendHead)
|
466
505
|
|
467
506
|
if (updateResult._tag === 'rebase') {
|
468
|
-
|
507
|
+
otelSpan?.addEvent('backend-pull:rebase', {
|
469
508
|
newEventsCount: newEvents.length,
|
470
509
|
newEvents: TRACE_VERBOSE ? JSON.stringify(newEvents) : undefined,
|
471
510
|
rollbackCount: updateResult.eventsToRollback.length,
|
@@ -479,7 +518,7 @@ const backgroundBackendPulling = ({
|
|
479
518
|
yield* restartBackendPushing(filteredRebasedPending)
|
480
519
|
|
481
520
|
if (updateResult.eventsToRollback.length > 0) {
|
482
|
-
yield* rollback({ db,
|
521
|
+
yield* rollback({ db, dbMutationLog, eventIdsToRollback: updateResult.eventsToRollback.map((_) => _.id) })
|
483
522
|
}
|
484
523
|
|
485
524
|
yield* connectedClientSessionPullQueues.offer({
|
@@ -492,7 +531,7 @@ const backgroundBackendPulling = ({
|
|
492
531
|
remaining,
|
493
532
|
})
|
494
533
|
} else {
|
495
|
-
|
534
|
+
otelSpan?.addEvent('backend-pull:advance', {
|
496
535
|
newEventsCount: newEvents.length,
|
497
536
|
updateResult: TRACE_VERBOSE ? JSON.stringify(updateResult) : undefined,
|
498
537
|
})
|
@@ -549,11 +588,11 @@ const backgroundBackendPulling = ({
|
|
549
588
|
|
550
589
|
const rollback = ({
|
551
590
|
db,
|
552
|
-
|
591
|
+
dbMutationLog,
|
553
592
|
eventIdsToRollback,
|
554
593
|
}: {
|
555
|
-
db:
|
556
|
-
|
594
|
+
db: SqliteDb
|
595
|
+
dbMutationLog: SqliteDb
|
557
596
|
eventIdsToRollback: EventId.EventId[]
|
558
597
|
}) =>
|
559
598
|
Effect.gen(function* () {
|
@@ -578,7 +617,7 @@ const rollback = ({
|
|
578
617
|
)
|
579
618
|
|
580
619
|
// Delete the mutation log rows
|
581
|
-
|
620
|
+
dbMutationLog.execute(
|
582
621
|
sql`DELETE FROM ${MUTATION_LOG_META_TABLE} WHERE (idGlobal, idLocal) IN (${eventIdsToRollback.map((id) => `(${id.global}, ${id.local})`).join(', ')})`,
|
583
622
|
)
|
584
623
|
}).pipe(
|
@@ -589,7 +628,7 @@ const rollback = ({
|
|
589
628
|
|
590
629
|
const getCursorInfo = (remoteHead: EventId.GlobalEventId) =>
|
591
630
|
Effect.gen(function* () {
|
592
|
-
const {
|
631
|
+
const { dbMutationLog } = yield* LeaderThreadCtx
|
593
632
|
|
594
633
|
if (remoteHead === EventId.ROOT.global) return Option.none()
|
595
634
|
|
@@ -598,7 +637,7 @@ const getCursorInfo = (remoteHead: EventId.GlobalEventId) =>
|
|
598
637
|
}).pipe(Schema.pluck('syncMetadataJson'), Schema.Array, Schema.head)
|
599
638
|
|
600
639
|
const syncMetadataOption = yield* Effect.sync(() =>
|
601
|
-
|
640
|
+
dbMutationLog.select<{ syncMetadataJson: string }>(
|
602
641
|
sql`SELECT syncMetadataJson FROM ${MUTATION_LOG_META_TABLE} WHERE idGlobal = ${remoteHead} ORDER BY idLocal ASC LIMIT 1`,
|
603
642
|
),
|
604
643
|
).pipe(Effect.andThen(Schema.decode(MutationlogQuerySchema)), Effect.map(Option.flatten), Effect.orDie)
|
@@ -612,14 +651,16 @@ const getCursorInfo = (remoteHead: EventId.GlobalEventId) =>
|
|
612
651
|
const backgroundBackendPushing = ({
|
613
652
|
dbReady,
|
614
653
|
syncBackendQueue,
|
615
|
-
|
654
|
+
otelSpan,
|
655
|
+
devtoolsLatch,
|
616
656
|
}: {
|
617
657
|
dbReady: Deferred.Deferred<void>
|
618
658
|
syncBackendQueue: BucketQueue.BucketQueue<MutationEvent.EncodedWithMeta>
|
619
|
-
|
659
|
+
otelSpan: otel.Span | undefined
|
660
|
+
devtoolsLatch: Effect.Latch | undefined
|
620
661
|
}) =>
|
621
662
|
Effect.gen(function* () {
|
622
|
-
const { syncBackend,
|
663
|
+
const { syncBackend, dbMutationLog } = yield* LeaderThreadCtx
|
623
664
|
if (syncBackend === undefined) return
|
624
665
|
|
625
666
|
yield* dbReady
|
@@ -632,7 +673,11 @@ const backgroundBackendPushing = ({
|
|
632
673
|
|
633
674
|
yield* SubscriptionRef.waitUntil(syncBackend.isConnected, (isConnected) => isConnected === true)
|
634
675
|
|
635
|
-
|
676
|
+
if (devtoolsLatch !== undefined) {
|
677
|
+
yield* devtoolsLatch.await
|
678
|
+
}
|
679
|
+
|
680
|
+
otelSpan?.addEvent('backend-push', {
|
636
681
|
batchSize: queueItems.length,
|
637
682
|
batch: TRACE_VERBOSE ? JSON.stringify(queueItems) : undefined,
|
638
683
|
})
|
@@ -641,7 +686,10 @@ const backgroundBackendPushing = ({
|
|
641
686
|
const pushResult = yield* syncBackend.push(queueItems.map((_) => _.toGlobal())).pipe(Effect.either)
|
642
687
|
|
643
688
|
if (pushResult._tag === 'Left') {
|
644
|
-
|
689
|
+
if (LS_DEV) {
|
690
|
+
yield* Effect.logDebug('backend-push-error', { error: pushResult.left.toString() })
|
691
|
+
}
|
692
|
+
otelSpan?.addEvent('backend-push-error', { error: pushResult.left.toString() })
|
645
693
|
// wait for interrupt caused by background pulling which will then restart pushing
|
646
694
|
return yield* Effect.never
|
647
695
|
}
|
@@ -652,7 +700,7 @@ const backgroundBackendPushing = ({
|
|
652
700
|
for (let i = 0; i < queueItems.length; i++) {
|
653
701
|
const mutationEventEncoded = queueItems[i]!
|
654
702
|
yield* execSql(
|
655
|
-
|
703
|
+
dbMutationLog,
|
656
704
|
...updateRows({
|
657
705
|
tableName: MUTATION_LOG_META_TABLE,
|
658
706
|
columns: mutationLogMetaTable.sqliteDef.columns,
|
@@ -664,7 +712,7 @@ const backgroundBackendPushing = ({
|
|
664
712
|
}
|
665
713
|
}).pipe(Effect.interruptible, Effect.withSpan('@livestore/common:leader-thread:syncing:backend-pushing'))
|
666
714
|
|
667
|
-
const trimChangesetRows = (db:
|
715
|
+
const trimChangesetRows = (db: SqliteDb, newHead: EventId.EventId) => {
|
668
716
|
// Since we're using the session changeset rows to query for the current head,
|
669
717
|
// we're keeping at least one row for the current head, and thus are using `<` instead of `<=`
|
670
718
|
db.execute(sql`DELETE FROM ${SESSION_CHANGESET_META_TABLE} WHERE idGlobal < ${newHead.global}`)
|
@@ -2,7 +2,7 @@ import { memoizeByRef, shouldNeverHappen } from '@livestore/utils'
|
|
2
2
|
import type { Scope } from '@livestore/utils/effect'
|
3
3
|
import { Effect, Option, Schema } from '@livestore/utils/effect'
|
4
4
|
|
5
|
-
import type {
|
5
|
+
import type { SqliteDb, SqliteError, UnexpectedError } from '../index.js'
|
6
6
|
import { getExecArgsFromMutation } from '../mutation.js'
|
7
7
|
import {
|
8
8
|
type LiveStoreSchema,
|
@@ -38,7 +38,7 @@ export const makeApplyMutation: Effect.Effect<ApplyMutation, never, Scope.Scope
|
|
38
38
|
|
39
39
|
return (mutationEventEncoded, options) =>
|
40
40
|
Effect.gen(function* () {
|
41
|
-
const { schema, db,
|
41
|
+
const { schema, dbReadModel: db, dbMutationLog } = leaderThreadCtx
|
42
42
|
const skipMutationLog = options?.skipMutationLog ?? false
|
43
43
|
|
44
44
|
const mutationName = mutationEventEncoded.mutation
|
@@ -92,7 +92,7 @@ export const makeApplyMutation: Effect.Effect<ApplyMutation, never, Scope.Scope
|
|
92
92
|
// write to mutation_log
|
93
93
|
const excludeFromMutationLog = shouldExcludeMutationFromLog(mutationName, mutationEventEncoded)
|
94
94
|
if (skipMutationLog === false && excludeFromMutationLog === false) {
|
95
|
-
yield* insertIntoMutationLog(mutationEventEncoded,
|
95
|
+
yield* insertIntoMutationLog(mutationEventEncoded, dbMutationLog, mutationDefSchemaHashMap)
|
96
96
|
} else {
|
97
97
|
// console.debug('[@livestore/common:leader-thread] skipping mutation log write', mutation, statementSql, bindValues)
|
98
98
|
}
|
@@ -111,7 +111,7 @@ export const makeApplyMutation: Effect.Effect<ApplyMutation, never, Scope.Scope
|
|
111
111
|
|
112
112
|
const insertIntoMutationLog = (
|
113
113
|
mutationEventEncoded: MutationEvent.AnyEncoded,
|
114
|
-
|
114
|
+
dbMutationLog: SqliteDb,
|
115
115
|
mutationDefSchemaHashMap: Map<string, number>,
|
116
116
|
) =>
|
117
117
|
Effect.gen(function* () {
|
@@ -121,7 +121,7 @@ const insertIntoMutationLog = (
|
|
121
121
|
|
122
122
|
// TODO use prepared statements
|
123
123
|
yield* execSql(
|
124
|
-
|
124
|
+
dbMutationLog,
|
125
125
|
...insertRow({
|
126
126
|
tableName: MUTATION_LOG_META_TABLE,
|
127
127
|
columns: mutationLogMetaTable.sqliteDef.columns,
|
@@ -1,7 +1,7 @@
|
|
1
1
|
// import type { WaSqlite } from '@livestore/sqlite-wasm'
|
2
2
|
import { Effect } from '@livestore/utils/effect'
|
3
3
|
|
4
|
-
import type {
|
4
|
+
import type { SqliteDb } from '../adapter-types.js'
|
5
5
|
import { SqliteError } from '../adapter-types.js'
|
6
6
|
import type { BindValues } from '../sql-queries/index.js'
|
7
7
|
import type { PreparedBindValues } from '../util.js'
|
@@ -12,21 +12,66 @@ namespace WaSqlite {
|
|
12
12
|
export type SQLiteError = any
|
13
13
|
}
|
14
14
|
|
15
|
-
|
15
|
+
type ConnectionOptions = {
|
16
|
+
/**
|
17
|
+
* The database connection locking mode.
|
18
|
+
*
|
19
|
+
* @remarks
|
20
|
+
*
|
21
|
+
* This **option is ignored** when used on an **in-memory database** as they can only operate in exclusive locking mode.
|
22
|
+
* In-memory databases can’t share state between connections (unless using a
|
23
|
+
* {@link https://www.sqlite.org/sharedcache.html#shared_cache_and_in_memory_databases|shared cache}),
|
24
|
+
* making concurrent access impossible. This is functionally equivalent to exclusive locking.
|
25
|
+
*
|
26
|
+
* @defaultValue
|
27
|
+
* The default is `"NORMAL"` unless it was unless overridden at compile-time using `SQLITE_DEFAULT_LOCKING_MODE`.
|
28
|
+
*
|
29
|
+
* @see {@link https://www.sqlite.org/pragma.html#pragma_locking_mode|`locking_mode` pragma}
|
30
|
+
*/
|
31
|
+
lockingMode?: 'NORMAL' | 'EXCLUSIVE'
|
32
|
+
|
33
|
+
/**
|
34
|
+
* Whether to enforce foreign key constraints.
|
35
|
+
*
|
36
|
+
* @privateRemarks
|
37
|
+
*
|
38
|
+
* We require a value for this option to minimize future problems, as the default value might change in future
|
39
|
+
* versions of SQLite.
|
40
|
+
*
|
41
|
+
* @see {@link https://www.sqlite.org/pragma.html#pragma_foreign_keys|`foreign_keys` pragma}
|
42
|
+
*/
|
43
|
+
foreignKeys: boolean
|
44
|
+
}
|
45
|
+
|
46
|
+
export const configureConnection = (sqliteDb: SqliteDb, { foreignKeys, lockingMode }: ConnectionOptions) =>
|
16
47
|
execSql(
|
17
|
-
|
48
|
+
sqliteDb,
|
49
|
+
// We use the WAL journal mode is significantly faster in most scenarios than the traditional rollback journal mode.
|
50
|
+
// It specifically significantly improves write performance. However, when using the WAL journal mode, transactions
|
51
|
+
// that involve changes against multiple ATTACHed databases are atomic for each database but are not atomic
|
52
|
+
// across all databases as a set. Additionally, it is not possible to change the page size after entering WAL mode,
|
53
|
+
// whether on an empty database or by using VACUUM or the backup API. To change the page size, we must switch to the
|
54
|
+
// rollback journal mode.
|
55
|
+
//
|
56
|
+
// When connected to an in-memory database, the WAL journal mode option is ignored because an in-memory database can
|
57
|
+
// only be in either the MEMORY or OFF options. By default, an in-memory database is in the MEMORY option, which
|
58
|
+
// means that it stores the rollback journal in volatile RAM. This saves disk I/O but at the expense of safety and
|
59
|
+
// integrity. If the thread using SQLite crashes in the middle of a transaction, then the database file will very
|
60
|
+
// likely go corrupt.
|
18
61
|
sql`
|
62
|
+
-- disable WAL until we have it working properly
|
63
|
+
-- PRAGMA journal_mode=WAL;
|
19
64
|
PRAGMA page_size=8192;
|
20
|
-
PRAGMA
|
21
|
-
${
|
65
|
+
PRAGMA foreign_keys=${foreignKeys ? 'ON' : 'OFF'};
|
66
|
+
${lockingMode === undefined ? '' : sql`PRAGMA locking_mode=${lockingMode};`}
|
22
67
|
`,
|
23
68
|
{},
|
24
69
|
)
|
25
70
|
|
26
|
-
export const execSql = (
|
71
|
+
export const execSql = (sqliteDb: SqliteDb, sql: string, bind: BindValues) => {
|
27
72
|
const bindValues = prepareBindValues(bind, sql)
|
28
73
|
return Effect.try({
|
29
|
-
try: () =>
|
74
|
+
try: () => sqliteDb.execute(sql, bindValues),
|
30
75
|
catch: (cause) =>
|
31
76
|
new SqliteError({ cause, query: { bindValues, sql }, code: (cause as WaSqlite.SQLiteError).code }),
|
32
77
|
}).pipe(
|
@@ -48,9 +93,9 @@ export const execSql = (syncDb: SynchronousDatabase, sql: string, bind: BindValu
|
|
48
93
|
// }
|
49
94
|
|
50
95
|
// TODO actually use prepared statements
|
51
|
-
export const execSqlPrepared = (
|
96
|
+
export const execSqlPrepared = (sqliteDb: SqliteDb, sql: string, bindValues: PreparedBindValues) => {
|
52
97
|
return Effect.try({
|
53
|
-
try: () =>
|
98
|
+
try: () => sqliteDb.execute(sql, bindValues),
|
54
99
|
catch: (cause) =>
|
55
100
|
new SqliteError({ cause, query: { bindValues, sql }, code: (cause as WaSqlite.SQLiteError).code }),
|
56
101
|
}).pipe(
|