@livestore/common 0.4.0-dev.1 → 0.4.0-dev.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/.tsbuildinfo +1 -1
- package/dist/ClientSessionLeaderThreadProxy.d.ts +7 -2
- package/dist/ClientSessionLeaderThreadProxy.d.ts.map +1 -1
- package/dist/ClientSessionLeaderThreadProxy.js.map +1 -1
- package/dist/adapter-types.d.ts +9 -3
- package/dist/adapter-types.d.ts.map +1 -1
- package/dist/adapter-types.js.map +1 -1
- package/dist/devtools/devtools-messages-client-session.d.ts +21 -21
- package/dist/devtools/devtools-messages-common.d.ts +7 -14
- package/dist/devtools/devtools-messages-common.d.ts.map +1 -1
- package/dist/devtools/devtools-messages-common.js +1 -6
- package/dist/devtools/devtools-messages-common.js.map +1 -1
- package/dist/devtools/devtools-messages-leader.d.ts +27 -25
- package/dist/devtools/devtools-messages-leader.d.ts.map +1 -1
- package/dist/errors.d.ts +47 -5
- package/dist/errors.d.ts.map +1 -1
- package/dist/errors.js +22 -3
- package/dist/errors.js.map +1 -1
- package/dist/leader-thread/LeaderSyncProcessor.d.ts +7 -3
- package/dist/leader-thread/LeaderSyncProcessor.d.ts.map +1 -1
- package/dist/leader-thread/LeaderSyncProcessor.js +122 -49
- package/dist/leader-thread/LeaderSyncProcessor.js.map +1 -1
- package/dist/leader-thread/eventlog.d.ts +4 -10
- package/dist/leader-thread/eventlog.d.ts.map +1 -1
- package/dist/leader-thread/eventlog.js +4 -6
- package/dist/leader-thread/eventlog.js.map +1 -1
- package/dist/leader-thread/leader-worker-devtools.d.ts +1 -1
- package/dist/leader-thread/leader-worker-devtools.js +6 -2
- package/dist/leader-thread/leader-worker-devtools.js.map +1 -1
- package/dist/leader-thread/make-leader-thread-layer.d.ts +1 -2
- package/dist/leader-thread/make-leader-thread-layer.d.ts.map +1 -1
- package/dist/leader-thread/make-leader-thread-layer.js +68 -19
- package/dist/leader-thread/make-leader-thread-layer.js.map +1 -1
- package/dist/leader-thread/make-leader-thread-layer.test.d.ts +2 -0
- package/dist/leader-thread/make-leader-thread-layer.test.d.ts.map +1 -0
- package/dist/leader-thread/make-leader-thread-layer.test.js +32 -0
- package/dist/leader-thread/make-leader-thread-layer.test.js.map +1 -0
- package/dist/leader-thread/materialize-event.d.ts +2 -2
- package/dist/leader-thread/materialize-event.d.ts.map +1 -1
- package/dist/leader-thread/materialize-event.js +23 -9
- package/dist/leader-thread/materialize-event.js.map +1 -1
- package/dist/leader-thread/recreate-db.d.ts +2 -3
- package/dist/leader-thread/recreate-db.d.ts.map +1 -1
- package/dist/leader-thread/recreate-db.js +1 -1
- package/dist/leader-thread/recreate-db.js.map +1 -1
- package/dist/leader-thread/shutdown-channel.d.ts +2 -2
- package/dist/leader-thread/shutdown-channel.d.ts.map +1 -1
- package/dist/leader-thread/shutdown-channel.js +2 -2
- package/dist/leader-thread/shutdown-channel.js.map +1 -1
- package/dist/leader-thread/types.d.ts +7 -5
- package/dist/leader-thread/types.d.ts.map +1 -1
- package/dist/leader-thread/types.js.map +1 -1
- package/dist/materializer-helper.d.ts +1 -1
- package/dist/materializer-helper.d.ts.map +1 -1
- package/dist/materializer-helper.js +20 -4
- package/dist/materializer-helper.js.map +1 -1
- package/dist/rematerialize-from-eventlog.d.ts +1 -1
- package/dist/rematerialize-from-eventlog.d.ts.map +1 -1
- package/dist/rematerialize-from-eventlog.js +25 -16
- package/dist/rematerialize-from-eventlog.js.map +1 -1
- package/dist/schema/EventDef.d.ts +3 -0
- package/dist/schema/EventDef.d.ts.map +1 -1
- package/dist/schema/EventDef.js.map +1 -1
- package/dist/schema/LiveStoreEvent.d.ts +1 -1
- package/dist/schema/LiveStoreEvent.d.ts.map +1 -1
- package/dist/schema/LiveStoreEvent.js +1 -2
- package/dist/schema/LiveStoreEvent.js.map +1 -1
- package/dist/schema/mod.d.ts +2 -0
- package/dist/schema/mod.d.ts.map +1 -1
- package/dist/schema/mod.js +1 -0
- package/dist/schema/mod.js.map +1 -1
- package/dist/schema/schema.d.ts +15 -0
- package/dist/schema/schema.d.ts.map +1 -1
- package/dist/schema/schema.js +26 -1
- package/dist/schema/schema.js.map +1 -1
- package/dist/schema/state/sqlite/client-document-def.d.ts +35 -5
- package/dist/schema/state/sqlite/client-document-def.d.ts.map +1 -1
- package/dist/schema/state/sqlite/client-document-def.js +95 -4
- package/dist/schema/state/sqlite/client-document-def.js.map +1 -1
- package/dist/schema/state/sqlite/client-document-def.test.js +16 -0
- package/dist/schema/state/sqlite/client-document-def.test.js.map +1 -1
- package/dist/schema/state/sqlite/column-annotations.d.ts.map +1 -1
- package/dist/schema/state/sqlite/column-annotations.js +14 -6
- package/dist/schema/state/sqlite/column-annotations.js.map +1 -1
- package/dist/schema/state/sqlite/column-def.d.ts +6 -2
- package/dist/schema/state/sqlite/column-def.d.ts.map +1 -1
- package/dist/schema/state/sqlite/column-def.js +122 -185
- package/dist/schema/state/sqlite/column-def.js.map +1 -1
- package/dist/schema/state/sqlite/column-def.test.js +116 -73
- package/dist/schema/state/sqlite/column-def.test.js.map +1 -1
- package/dist/schema/state/sqlite/db-schema/ast/sqlite.d.ts +2 -1
- package/dist/schema/state/sqlite/db-schema/ast/sqlite.d.ts.map +1 -1
- package/dist/schema/state/sqlite/db-schema/ast/sqlite.js +23 -6
- package/dist/schema/state/sqlite/db-schema/ast/sqlite.js.map +1 -1
- package/dist/schema/state/sqlite/db-schema/dsl/mod.d.ts.map +1 -1
- package/dist/schema/state/sqlite/db-schema/dsl/mod.js +2 -1
- package/dist/schema/state/sqlite/db-schema/dsl/mod.js.map +1 -1
- package/dist/schema/state/sqlite/mod.d.ts +1 -1
- package/dist/schema/state/sqlite/mod.d.ts.map +1 -1
- package/dist/schema/state/sqlite/mod.js +1 -1
- package/dist/schema/state/sqlite/mod.js.map +1 -1
- package/dist/schema/state/sqlite/query-builder/api.d.ts +5 -2
- package/dist/schema/state/sqlite/query-builder/api.d.ts.map +1 -1
- package/dist/schema/state/sqlite/query-builder/impl.d.ts.map +1 -1
- package/dist/schema/state/sqlite/query-builder/impl.js +6 -2
- package/dist/schema/state/sqlite/query-builder/impl.js.map +1 -1
- package/dist/schema/state/sqlite/query-builder/impl.test.js +137 -2
- package/dist/schema/state/sqlite/query-builder/impl.test.js.map +1 -1
- package/dist/schema/state/sqlite/system-tables.d.ts +42 -6
- package/dist/schema/state/sqlite/system-tables.d.ts.map +1 -1
- package/dist/schema/state/sqlite/system-tables.js +2 -0
- package/dist/schema/state/sqlite/system-tables.js.map +1 -1
- package/dist/schema/state/sqlite/table-def.d.ts +4 -4
- package/dist/schema/state/sqlite/table-def.d.ts.map +1 -1
- package/dist/schema/state/sqlite/table-def.js +2 -2
- package/dist/schema/state/sqlite/table-def.js.map +1 -1
- package/dist/schema/state/sqlite/table-def.test.js +51 -2
- package/dist/schema/state/sqlite/table-def.test.js.map +1 -1
- package/dist/schema/unknown-events.d.ts +47 -0
- package/dist/schema/unknown-events.d.ts.map +1 -0
- package/dist/schema/unknown-events.js +69 -0
- package/dist/schema/unknown-events.js.map +1 -0
- package/dist/sql-queries/sql-query-builder.d.ts.map +1 -1
- package/dist/sql-queries/sql-query-builder.js +2 -1
- package/dist/sql-queries/sql-query-builder.js.map +1 -1
- package/dist/sync/ClientSessionSyncProcessor.d.ts +9 -11
- package/dist/sync/ClientSessionSyncProcessor.d.ts.map +1 -1
- package/dist/sync/ClientSessionSyncProcessor.js +35 -33
- package/dist/sync/ClientSessionSyncProcessor.js.map +1 -1
- package/dist/sync/errors.d.ts +61 -0
- package/dist/sync/errors.d.ts.map +1 -0
- package/dist/sync/errors.js +36 -0
- package/dist/sync/errors.js.map +1 -0
- package/dist/sync/index.d.ts +3 -0
- package/dist/sync/index.d.ts.map +1 -1
- package/dist/sync/index.js +3 -0
- package/dist/sync/index.js.map +1 -1
- package/dist/sync/mock-sync-backend.d.ts +23 -0
- package/dist/sync/mock-sync-backend.d.ts.map +1 -0
- package/dist/sync/mock-sync-backend.js +114 -0
- package/dist/sync/mock-sync-backend.js.map +1 -0
- package/dist/sync/next/compact-events.d.ts.map +1 -1
- package/dist/sync/next/compact-events.js +4 -5
- package/dist/sync/next/compact-events.js.map +1 -1
- package/dist/sync/next/facts.d.ts.map +1 -1
- package/dist/sync/next/facts.js +1 -2
- package/dist/sync/next/facts.js.map +1 -1
- package/dist/sync/next/history-dag-common.d.ts +50 -11
- package/dist/sync/next/history-dag-common.d.ts.map +1 -1
- package/dist/sync/next/history-dag-common.js +193 -4
- package/dist/sync/next/history-dag-common.js.map +1 -1
- package/dist/sync/next/history-dag.d.ts.map +1 -1
- package/dist/sync/next/history-dag.js +3 -1
- package/dist/sync/next/history-dag.js.map +1 -1
- package/dist/sync/sync-backend-kv.d.ts +7 -0
- package/dist/sync/sync-backend-kv.d.ts.map +1 -0
- package/dist/sync/sync-backend-kv.js +18 -0
- package/dist/sync/sync-backend-kv.js.map +1 -0
- package/dist/sync/sync-backend.d.ts +105 -0
- package/dist/sync/sync-backend.d.ts.map +1 -0
- package/dist/sync/sync-backend.js +61 -0
- package/dist/sync/sync-backend.js.map +1 -0
- package/dist/sync/sync.d.ts +6 -84
- package/dist/sync/sync.d.ts.map +1 -1
- package/dist/sync/sync.js +2 -27
- package/dist/sync/sync.js.map +1 -1
- package/dist/sync/transport-chunking.d.ts +36 -0
- package/dist/sync/transport-chunking.d.ts.map +1 -0
- package/dist/sync/transport-chunking.js +56 -0
- package/dist/sync/transport-chunking.js.map +1 -0
- package/dist/sync/validate-push-payload.d.ts +1 -1
- package/dist/sync/validate-push-payload.d.ts.map +1 -1
- package/dist/sync/validate-push-payload.js +6 -6
- package/dist/sync/validate-push-payload.js.map +1 -1
- package/dist/testing/event-factory.d.ts +68 -0
- package/dist/testing/event-factory.d.ts.map +1 -0
- package/dist/testing/event-factory.js +80 -0
- package/dist/testing/event-factory.js.map +1 -0
- package/dist/testing/mod.d.ts +2 -0
- package/dist/testing/mod.d.ts.map +1 -0
- package/dist/testing/mod.js +2 -0
- package/dist/testing/mod.js.map +1 -0
- package/dist/version.d.ts +2 -2
- package/dist/version.d.ts.map +1 -1
- package/dist/version.js +2 -2
- package/dist/version.js.map +1 -1
- package/package.json +7 -8
- package/src/ClientSessionLeaderThreadProxy.ts +7 -2
- package/src/adapter-types.ts +13 -3
- package/src/devtools/devtools-messages-common.ts +1 -8
- package/src/errors.ts +33 -4
- package/src/leader-thread/LeaderSyncProcessor.ts +179 -57
- package/src/leader-thread/eventlog.ts +10 -6
- package/src/leader-thread/leader-worker-devtools.ts +6 -2
- package/src/leader-thread/make-leader-thread-layer.test.ts +44 -0
- package/src/leader-thread/make-leader-thread-layer.ts +137 -26
- package/src/leader-thread/materialize-event.ts +34 -9
- package/src/leader-thread/recreate-db.ts +11 -3
- package/src/leader-thread/shutdown-channel.ts +16 -2
- package/src/leader-thread/types.ts +7 -5
- package/src/materializer-helper.ts +22 -5
- package/src/rematerialize-from-eventlog.ts +33 -23
- package/src/schema/EventDef.ts +3 -0
- package/src/schema/LiveStoreEvent.ts +1 -2
- package/src/schema/mod.ts +2 -0
- package/src/schema/schema.ts +37 -1
- package/src/schema/state/sqlite/client-document-def.test.ts +17 -0
- package/src/schema/state/sqlite/client-document-def.ts +117 -5
- package/src/schema/state/sqlite/column-annotations.ts +16 -6
- package/src/schema/state/sqlite/column-def.test.ts +150 -93
- package/src/schema/state/sqlite/column-def.ts +128 -203
- package/src/schema/state/sqlite/db-schema/ast/sqlite.ts +26 -6
- package/src/schema/state/sqlite/db-schema/dsl/mod.ts +2 -1
- package/src/schema/state/sqlite/mod.ts +1 -0
- package/src/schema/state/sqlite/query-builder/api.ts +7 -2
- package/src/schema/state/sqlite/query-builder/impl.test.ts +187 -6
- package/src/schema/state/sqlite/query-builder/impl.ts +8 -2
- package/src/schema/state/sqlite/system-tables.ts +2 -0
- package/src/schema/state/sqlite/table-def.test.ts +64 -2
- package/src/schema/state/sqlite/table-def.ts +9 -8
- package/src/schema/unknown-events.ts +131 -0
- package/src/sql-queries/sql-query-builder.ts +2 -1
- package/src/sync/ClientSessionSyncProcessor.ts +55 -49
- package/src/sync/errors.ts +38 -0
- package/src/sync/index.ts +3 -0
- package/src/sync/mock-sync-backend.ts +184 -0
- package/src/sync/next/compact-events.ts +4 -5
- package/src/sync/next/facts.ts +1 -3
- package/src/sync/next/history-dag-common.ts +272 -21
- package/src/sync/next/history-dag.ts +3 -1
- package/src/sync/sync-backend-kv.ts +22 -0
- package/src/sync/sync-backend.ts +185 -0
- package/src/sync/sync.ts +6 -89
- package/src/sync/transport-chunking.ts +90 -0
- package/src/sync/validate-push-payload.ts +6 -7
- package/src/testing/event-factory.ts +133 -0
- package/src/testing/mod.ts +1 -0
- package/src/version.ts +2 -2
- package/dist/schema-management/migrations.test.d.ts +0 -2
- package/dist/schema-management/migrations.test.d.ts.map +0 -1
- package/dist/schema-management/migrations.test.js +0 -52
- package/dist/schema-management/migrations.test.js.map +0 -1
- package/dist/sync/next/graphology.d.ts +0 -8
- package/dist/sync/next/graphology.d.ts.map +0 -1
- package/dist/sync/next/graphology.js +0 -30
- package/dist/sync/next/graphology.js.map +0 -1
- package/dist/sync/next/graphology_.d.ts +0 -3
- package/dist/sync/next/graphology_.d.ts.map +0 -1
- package/dist/sync/next/graphology_.js +0 -3
- package/dist/sync/next/graphology_.js.map +0 -1
- package/src/sync/next/ambient.d.ts +0 -3
- package/src/sync/next/graphology.ts +0 -41
- package/src/sync/next/graphology_.ts +0 -2
@@ -13,12 +13,13 @@ import {
|
|
13
13
|
Stream,
|
14
14
|
Subscribable,
|
15
15
|
} from '@livestore/utils/effect'
|
16
|
-
import * as otel from '@opentelemetry/api'
|
16
|
+
import type * as otel from '@opentelemetry/api'
|
17
17
|
|
18
|
-
import { type ClientSession,
|
18
|
+
import { type ClientSession, UnexpectedError } from '../adapter-types.ts'
|
19
|
+
import type { MaterializeError } from '../errors.ts'
|
19
20
|
import * as EventSequenceNumber from '../schema/EventSequenceNumber.ts'
|
20
21
|
import * as LiveStoreEvent from '../schema/LiveStoreEvent.ts'
|
21
|
-
import {
|
22
|
+
import type { LiveStoreSchema } from '../schema/mod.ts'
|
22
23
|
import * as SyncState from './syncstate.ts'
|
23
24
|
|
24
25
|
/**
|
@@ -51,16 +52,19 @@ export const makeClientSessionSyncProcessor = ({
|
|
51
52
|
clientSession: ClientSession
|
52
53
|
runtime: Runtime.Runtime<Scope.Scope>
|
53
54
|
materializeEvent: (
|
54
|
-
|
55
|
-
options: {
|
56
|
-
) =>
|
57
|
-
|
58
|
-
|
59
|
-
|
60
|
-
|
61
|
-
|
62
|
-
|
63
|
-
|
55
|
+
eventEncoded: LiveStoreEvent.EncodedWithMeta,
|
56
|
+
options: { withChangeset: boolean; materializerHashLeader: Option.Option<number> },
|
57
|
+
) => Effect.Effect<
|
58
|
+
{
|
59
|
+
writeTables: Set<string>
|
60
|
+
sessionChangeset:
|
61
|
+
| { _tag: 'sessionChangeset'; data: Uint8Array<ArrayBuffer>; debug: any }
|
62
|
+
| { _tag: 'no-op' }
|
63
|
+
| { _tag: 'unset' }
|
64
|
+
materializerHash: Option.Option<number>
|
65
|
+
},
|
66
|
+
MaterializeError
|
67
|
+
>
|
64
68
|
rollback: (changeset: Uint8Array<ArrayBuffer>) => void
|
65
69
|
refreshTables: (tables: Set<string>) => void
|
66
70
|
span: otel.Span
|
@@ -91,23 +95,26 @@ export const makeClientSessionSyncProcessor = ({
|
|
91
95
|
}),
|
92
96
|
}
|
93
97
|
|
94
|
-
/** Only used for debugging / observability, it's not relied upon for correctness of the sync processor. */
|
98
|
+
/** Only used for debugging / observability / testing, it's not relied upon for correctness of the sync processor. */
|
95
99
|
const syncStateUpdateQueue = Queue.unbounded<SyncState.SyncState>().pipe(Effect.runSync)
|
96
100
|
const isClientEvent = (eventEncoded: LiveStoreEvent.EncodedWithMeta) =>
|
97
|
-
|
101
|
+
schema.eventsDefsMap.get(eventEncoded.name)?.options.clientOnly ?? false
|
98
102
|
|
99
103
|
/** We're queuing push requests to reduce the number of messages sent to the leader by batching them */
|
100
104
|
const leaderPushQueue = BucketQueue.make<LiveStoreEvent.EncodedWithMeta>().pipe(Effect.runSync)
|
101
105
|
|
102
|
-
const push: ClientSessionSyncProcessor['push'] = (batch
|
106
|
+
const push: ClientSessionSyncProcessor['push'] = Effect.fn('client-session-sync-processor:push')(function* (batch) {
|
103
107
|
// TODO validate batch
|
104
108
|
|
105
109
|
let baseEventSequenceNumber = syncStateRef.current.localHead
|
106
110
|
const encodedEventDefs = batch.map(({ name, args }) => {
|
107
|
-
const eventDef =
|
111
|
+
const eventDef = schema.eventsDefsMap.get(name)
|
112
|
+
if (eventDef === undefined) {
|
113
|
+
return shouldNeverHappen(`No event definition found for \`${name}\`.`)
|
114
|
+
}
|
108
115
|
const nextNumPair = EventSequenceNumber.nextPair({
|
109
116
|
seqNum: baseEventSequenceNumber,
|
110
|
-
isClient: eventDef.
|
117
|
+
isClient: eventDef.options.clientOnly,
|
111
118
|
})
|
112
119
|
baseEventSequenceNumber = nextNumPair.seqNum
|
113
120
|
return new LiveStoreEvent.EncodedWithMeta(
|
@@ -128,33 +135,35 @@ export const makeClientSessionSyncProcessor = ({
|
|
128
135
|
isEqualEvent: LiveStoreEvent.isEqualEncoded,
|
129
136
|
})
|
130
137
|
|
138
|
+
yield* Effect.annotateCurrentSpan({
|
139
|
+
batchSize: encodedEventDefs.length,
|
140
|
+
mergeResultTag: mergeResult._tag,
|
141
|
+
eventCounts: encodedEventDefs.reduce<Record<string, number>>((acc, event) => {
|
142
|
+
acc[event.name] = (acc[event.name] ?? 0) + 1
|
143
|
+
return acc
|
144
|
+
}, {}),
|
145
|
+
...(TRACE_VERBOSE && { mergeResult: JSON.stringify(mergeResult) }),
|
146
|
+
})
|
147
|
+
|
131
148
|
if (mergeResult._tag === 'unexpected-error') {
|
132
149
|
return shouldNeverHappen('Unexpected error in client-session-sync-processor', mergeResult.message)
|
133
150
|
}
|
134
151
|
|
135
|
-
span.addEvent('local-push', {
|
136
|
-
batchSize: encodedEventDefs.length,
|
137
|
-
mergeResult: TRACE_VERBOSE ? JSON.stringify(mergeResult) : undefined,
|
138
|
-
})
|
139
|
-
|
140
152
|
if (mergeResult._tag !== 'advance') {
|
141
153
|
return shouldNeverHappen(`Expected advance, got ${mergeResult._tag}`)
|
142
154
|
}
|
143
155
|
|
144
156
|
syncStateRef.current = mergeResult.newSyncState
|
145
|
-
syncStateUpdateQueue.offer(mergeResult.newSyncState)
|
157
|
+
yield* syncStateUpdateQueue.offer(mergeResult.newSyncState)
|
146
158
|
|
147
159
|
// Materialize events to state
|
148
160
|
const writeTables = new Set<string>()
|
149
161
|
for (const event of mergeResult.newEvents) {
|
150
|
-
// TODO avoid encoding and decoding here again
|
151
|
-
const decodedEventDef = Schema.decodeSync(eventSchema)(event)
|
152
162
|
const {
|
153
163
|
writeTables: newWriteTables,
|
154
164
|
sessionChangeset,
|
155
165
|
materializerHash,
|
156
|
-
} = materializeEvent(
|
157
|
-
otelContext,
|
166
|
+
} = yield* materializeEvent(event, {
|
158
167
|
withChangeset: true,
|
159
168
|
materializerHashLeader: Option.none(),
|
160
169
|
})
|
@@ -167,10 +176,10 @@ export const makeClientSessionSyncProcessor = ({
|
|
167
176
|
|
168
177
|
// Trigger push to leader
|
169
178
|
// console.debug('pushToLeader', encodedEventDefs.length, ...encodedEventDefs.map((_) => _.toJSON()))
|
170
|
-
BucketQueue.offerAll(leaderPushQueue, encodedEventDefs)
|
179
|
+
yield* BucketQueue.offerAll(leaderPushQueue, encodedEventDefs)
|
171
180
|
|
172
181
|
return { writeTables }
|
173
|
-
}
|
182
|
+
})
|
174
183
|
|
175
184
|
const debugInfo = {
|
176
185
|
rebaseCount: 0,
|
@@ -178,8 +187,6 @@ export const makeClientSessionSyncProcessor = ({
|
|
178
187
|
rejectCount: 0,
|
179
188
|
}
|
180
189
|
|
181
|
-
const otelContext = otel.trace.setSpan(otel.context.active(), span)
|
182
|
-
|
183
190
|
const boot: ClientSessionSyncProcessor['boot'] = Effect.gen(function* () {
|
184
191
|
if (confirmUnsavedChanges && typeof window !== 'undefined' && typeof window.addEventListener === 'function') {
|
185
192
|
const onBeforeUnload = (event: BeforeUnloadEvent) => {
|
@@ -229,13 +236,12 @@ export const makeClientSessionSyncProcessor = ({
|
|
229
236
|
})
|
230
237
|
|
231
238
|
if (mergeResult._tag === 'unexpected-error') {
|
232
|
-
return yield* new
|
239
|
+
return yield* new UnexpectedError({ cause: mergeResult.message })
|
233
240
|
} else if (mergeResult._tag === 'reject') {
|
234
241
|
return shouldNeverHappen('Unexpected reject in client-session-sync-processor', mergeResult)
|
235
242
|
}
|
236
243
|
|
237
244
|
syncStateRef.current = mergeResult.newSyncState
|
238
|
-
yield* syncStateUpdateQueue.offer(mergeResult.newSyncState)
|
239
245
|
|
240
246
|
if (mergeResult._tag === 'rebase') {
|
241
247
|
span.addEvent('merge:pull:rebase', {
|
@@ -244,7 +250,6 @@ export const makeClientSessionSyncProcessor = ({
|
|
244
250
|
newEventsCount: mergeResult.newEvents.length,
|
245
251
|
rollbackCount: mergeResult.rollbackEvents.length,
|
246
252
|
res: TRACE_VERBOSE ? JSON.stringify(mergeResult) : undefined,
|
247
|
-
rebaseGeneration: mergeResult.newSyncState.localHead.rebaseGeneration,
|
248
253
|
})
|
249
254
|
|
250
255
|
debugInfo.rebaseCount++
|
@@ -294,18 +299,19 @@ export const makeClientSessionSyncProcessor = ({
|
|
294
299
|
debugInfo.advanceCount++
|
295
300
|
}
|
296
301
|
|
297
|
-
if (mergeResult.newEvents.length === 0)
|
302
|
+
if (mergeResult.newEvents.length === 0) {
|
303
|
+
// If there are no new events, we need to update the sync state as well
|
304
|
+
yield* syncStateUpdateQueue.offer(mergeResult.newSyncState)
|
305
|
+
return
|
306
|
+
}
|
298
307
|
|
299
308
|
const writeTables = new Set<string>()
|
300
309
|
for (const event of mergeResult.newEvents) {
|
301
|
-
// TODO apply changeset if available (will require tracking of write tables as well)
|
302
|
-
const decodedEventDef = Schema.decodeSync(eventSchema)(event)
|
303
310
|
const {
|
304
311
|
writeTables: newWriteTables,
|
305
312
|
sessionChangeset,
|
306
313
|
materializerHash,
|
307
|
-
} = materializeEvent(
|
308
|
-
otelContext,
|
314
|
+
} = yield* materializeEvent(event, {
|
309
315
|
withChangeset: true,
|
310
316
|
materializerHashLeader: event.meta.materializerHashLeader,
|
311
317
|
})
|
@@ -318,6 +324,9 @@ export const makeClientSessionSyncProcessor = ({
|
|
318
324
|
}
|
319
325
|
|
320
326
|
refreshTables(writeTables)
|
327
|
+
|
328
|
+
// We're only triggering the sync state update after all events have been materialized
|
329
|
+
yield* syncStateUpdateQueue.offer(mergeResult.newSyncState)
|
321
330
|
}).pipe(
|
322
331
|
Effect.tapCauseLogPretty,
|
323
332
|
Effect.catchAllCause((cause) => clientSession.shutdown(Exit.failCause(cause))),
|
@@ -364,10 +373,7 @@ export const makeClientSessionSyncProcessor = ({
|
|
364
373
|
export interface ClientSessionSyncProcessor {
|
365
374
|
push: (
|
366
375
|
batch: ReadonlyArray<LiveStoreEvent.PartialAnyDecoded>,
|
367
|
-
|
368
|
-
) => {
|
369
|
-
writeTables: Set<string>
|
370
|
-
}
|
376
|
+
) => Effect.Effect<{ writeTables: Set<string> }, MaterializeError>
|
371
377
|
boot: Effect.Effect<void, UnexpectedError, Scope.Scope>
|
372
378
|
/**
|
373
379
|
* Only used for debugging / observability.
|
@@ -388,11 +394,11 @@ const SIMULATION_ENABLED = true
|
|
388
394
|
// Warning: High values for the simulation params can lead to very long test runs since those get multiplied with the number of events
|
389
395
|
export const ClientSessionSyncProcessorSimulationParams = Schema.Struct({
|
390
396
|
pull: Schema.Struct({
|
391
|
-
'1_before_leader_push_fiber_interrupt': Schema.Int.pipe(Schema.between(0,
|
392
|
-
'2_before_leader_push_queue_clear': Schema.Int.pipe(Schema.between(0,
|
393
|
-
'3_before_rebase_rollback': Schema.Int.pipe(Schema.between(0,
|
394
|
-
'4_before_leader_push_queue_offer': Schema.Int.pipe(Schema.between(0,
|
395
|
-
'5_before_leader_push_fiber_run': Schema.Int.pipe(Schema.between(0,
|
397
|
+
'1_before_leader_push_fiber_interrupt': Schema.Int.pipe(Schema.between(0, 15)),
|
398
|
+
'2_before_leader_push_queue_clear': Schema.Int.pipe(Schema.between(0, 15)),
|
399
|
+
'3_before_rebase_rollback': Schema.Int.pipe(Schema.between(0, 15)),
|
400
|
+
'4_before_leader_push_queue_offer': Schema.Int.pipe(Schema.between(0, 15)),
|
401
|
+
'5_before_leader_push_fiber_run': Schema.Int.pipe(Schema.between(0, 15)),
|
396
402
|
}),
|
397
403
|
})
|
398
404
|
type ClientSessionSyncProcessorSimulationParams = typeof ClientSessionSyncProcessorSimulationParams.Type
|
@@ -0,0 +1,38 @@
|
|
1
|
+
import { Schema } from '@livestore/utils/effect'
|
2
|
+
import { UnexpectedError } from '../errors.ts'
|
3
|
+
import { EventSequenceNumber } from '../schema/mod.ts'
|
4
|
+
|
5
|
+
export class IsOfflineError extends Schema.TaggedError<IsOfflineError>()('IsOfflineError', {
|
6
|
+
cause: Schema.Defect,
|
7
|
+
}) {}
|
8
|
+
|
9
|
+
/** Unique ID generated by the backend when its created. Used to check whether the backend identity has changed. */
|
10
|
+
export const BackendId = Schema.String.annotations({ title: '@livestore/sync-cf:BackendId' })
|
11
|
+
|
12
|
+
export class BackendIdMismatchError extends Schema.TaggedError<BackendIdMismatchError>()('BackendIdMismatchError', {
|
13
|
+
expected: BackendId,
|
14
|
+
received: BackendId,
|
15
|
+
}) {}
|
16
|
+
|
17
|
+
export class ServerAheadError extends Schema.TaggedError<ServerAheadError>()('ServerAheadError', {
|
18
|
+
minimumExpectedNum: EventSequenceNumber.GlobalEventSequenceNumber,
|
19
|
+
providedNum: EventSequenceNumber.GlobalEventSequenceNumber,
|
20
|
+
}) {}
|
21
|
+
|
22
|
+
export class InvalidPushError extends Schema.TaggedError<InvalidPushError>()('InvalidPushError', {
|
23
|
+
cause: Schema.Union(UnexpectedError, ServerAheadError, BackendIdMismatchError),
|
24
|
+
}) {}
|
25
|
+
|
26
|
+
export class InvalidPullError extends Schema.TaggedError<InvalidPullError>()('InvalidPullError', {
|
27
|
+
cause: Schema.Defect,
|
28
|
+
}) {}
|
29
|
+
|
30
|
+
export class LeaderAheadError extends Schema.TaggedError<LeaderAheadError>()('LeaderAheadError', {
|
31
|
+
minimumExpectedNum: EventSequenceNumber.EventSequenceNumber,
|
32
|
+
providedNum: EventSequenceNumber.EventSequenceNumber,
|
33
|
+
/** Generation number the client session should use for subsequent pushes */
|
34
|
+
// nextGeneration: Schema.Number,
|
35
|
+
}) {}
|
36
|
+
|
37
|
+
export const SyncError = Schema.Union(InvalidPushError, InvalidPullError)
|
38
|
+
export type SyncError = typeof SyncError.Type
|
package/src/sync/index.ts
CHANGED
@@ -0,0 +1,184 @@
|
|
1
|
+
import type { Schema, Scope } from '@livestore/utils/effect'
|
2
|
+
import { Effect, Mailbox, Option, Queue, Stream, SubscriptionRef } from '@livestore/utils/effect'
|
3
|
+
import { UnexpectedError } from '../errors.ts'
|
4
|
+
import { EventSequenceNumber, type LiveStoreEvent } from '../schema/mod.ts'
|
5
|
+
import { InvalidPushError } from './errors.ts'
|
6
|
+
import * as SyncBackend from './sync-backend.ts'
|
7
|
+
import { validatePushPayload } from './validate-push-payload.ts'
|
8
|
+
|
9
|
+
export interface MockSyncBackend {
|
10
|
+
pushedEvents: Stream.Stream<LiveStoreEvent.AnyEncodedGlobal>
|
11
|
+
connect: Effect.Effect<void>
|
12
|
+
disconnect: Effect.Effect<void>
|
13
|
+
makeSyncBackend: Effect.Effect<SyncBackend.SyncBackend, UnexpectedError, Scope.Scope>
|
14
|
+
advance: (...batch: LiveStoreEvent.AnyEncodedGlobal[]) => Effect.Effect<void>
|
15
|
+
/** Fail the next N push calls with an InvalidPushError (or custom error) */
|
16
|
+
failNextPushes: (
|
17
|
+
count: number,
|
18
|
+
error?: (batch: ReadonlyArray<LiveStoreEvent.AnyEncodedGlobal>) => Effect.Effect<never, InvalidPushError>,
|
19
|
+
) => Effect.Effect<void>
|
20
|
+
}
|
21
|
+
|
22
|
+
export interface MockSyncBackendOptions {
|
23
|
+
/** Chunk size for non-live pulls; defaults to 100 */
|
24
|
+
nonLiveChunkSize?: number
|
25
|
+
/** Initial connected state; defaults to false */
|
26
|
+
startConnected?: boolean
|
27
|
+
// TODO add a "flaky" mode to simulate transient network / server failures for pull/push
|
28
|
+
}
|
29
|
+
|
30
|
+
export const makeMockSyncBackend = (
|
31
|
+
options?: MockSyncBackendOptions,
|
32
|
+
): Effect.Effect<MockSyncBackend, UnexpectedError, Scope.Scope> =>
|
33
|
+
Effect.gen(function* () {
|
34
|
+
const syncEventSequenceNumberRef = { current: EventSequenceNumber.ROOT.global }
|
35
|
+
const syncPullQueue = yield* Queue.unbounded<LiveStoreEvent.AnyEncodedGlobal>()
|
36
|
+
const pushedEventsQueue = yield* Mailbox.make<LiveStoreEvent.AnyEncodedGlobal>()
|
37
|
+
const syncIsConnectedRef = yield* SubscriptionRef.make(options?.startConnected ?? false)
|
38
|
+
const allEventsRef: { current: LiveStoreEvent.AnyEncodedGlobal[] } = { current: [] }
|
39
|
+
|
40
|
+
const span = yield* Effect.currentSpan.pipe(Effect.orDie)
|
41
|
+
|
42
|
+
const semaphore = yield* Effect.makeSemaphore(1)
|
43
|
+
|
44
|
+
// TODO improve the API and implementation of simulating errors
|
45
|
+
const failCounterRef = yield* SubscriptionRef.make(0)
|
46
|
+
const failEffectRef = yield* SubscriptionRef.make<
|
47
|
+
((batch: ReadonlyArray<LiveStoreEvent.AnyEncodedGlobal>) => Effect.Effect<never, InvalidPushError>) | undefined
|
48
|
+
>(undefined)
|
49
|
+
|
50
|
+
const makeSyncBackend = Effect.gen(function* () {
|
51
|
+
const nonLiveChunkSize = Math.max(1, options?.nonLiveChunkSize ?? 100)
|
52
|
+
|
53
|
+
// TODO consider making offline state actively error pull/push.
|
54
|
+
// Currently, offline only reflects in `isConnected`, while operations still succeed,
|
55
|
+
// mirroring how some real providers behave during transient disconnects.
|
56
|
+
return SyncBackend.of<Schema.JsonValue>({
|
57
|
+
isConnected: syncIsConnectedRef,
|
58
|
+
connect: SubscriptionRef.set(syncIsConnectedRef, true),
|
59
|
+
ping: Effect.void,
|
60
|
+
pull: (cursor, options) =>
|
61
|
+
(options?.live
|
62
|
+
? Stream.concat(
|
63
|
+
Stream.make(SyncBackend.pullResItemEmpty()),
|
64
|
+
Stream.fromQueue(syncPullQueue).pipe(
|
65
|
+
Stream.chunks,
|
66
|
+
Stream.map((chunk) => ({
|
67
|
+
batch: [...chunk].map((eventEncoded) => ({ eventEncoded, metadata: Option.none() })),
|
68
|
+
pageInfo: SyncBackend.pageInfoNoMore,
|
69
|
+
})),
|
70
|
+
),
|
71
|
+
)
|
72
|
+
: Stream.fromEffect(
|
73
|
+
Effect.sync(() => {
|
74
|
+
const lastSeen = cursor.pipe(
|
75
|
+
Option.match({
|
76
|
+
onNone: () => EventSequenceNumber.ROOT.global,
|
77
|
+
onSome: (_) => _.eventSequenceNumber,
|
78
|
+
}),
|
79
|
+
)
|
80
|
+
// All events with seqNum greater than lastSeen
|
81
|
+
const slice = allEventsRef.current.filter((e) => e.seqNum > lastSeen)
|
82
|
+
// Split into configured chunk size
|
83
|
+
const chunks: { events: LiveStoreEvent.AnyEncodedGlobal[]; remaining: number }[] = []
|
84
|
+
for (let i = 0; i < slice.length; i += nonLiveChunkSize) {
|
85
|
+
const end = Math.min(i + nonLiveChunkSize, slice.length)
|
86
|
+
const remaining = Math.max(slice.length - end, 0)
|
87
|
+
chunks.push({ events: slice.slice(i, end), remaining })
|
88
|
+
}
|
89
|
+
if (chunks.length === 0) {
|
90
|
+
chunks.push({ events: [], remaining: 0 })
|
91
|
+
}
|
92
|
+
return chunks
|
93
|
+
}),
|
94
|
+
).pipe(
|
95
|
+
Stream.flatMap((chunks) =>
|
96
|
+
Stream.fromIterable(chunks).pipe(
|
97
|
+
Stream.map(({ events, remaining }) => ({
|
98
|
+
batch: events.map((eventEncoded) => ({ eventEncoded, metadata: Option.none() })),
|
99
|
+
pageInfo: remaining > 0 ? SyncBackend.pageInfoMoreKnown(remaining) : SyncBackend.pageInfoNoMore,
|
100
|
+
})),
|
101
|
+
),
|
102
|
+
),
|
103
|
+
)
|
104
|
+
).pipe(Stream.withSpan('MockSyncBackend:pull', { parent: span })),
|
105
|
+
push: (batch) =>
|
106
|
+
Effect.gen(function* () {
|
107
|
+
yield* validatePushPayload(batch, syncEventSequenceNumberRef.current)
|
108
|
+
|
109
|
+
const remaining = yield* SubscriptionRef.get(failCounterRef)
|
110
|
+
if (remaining > 0) {
|
111
|
+
const maybeFail = yield* SubscriptionRef.get(failEffectRef)
|
112
|
+
// decrement counter first
|
113
|
+
yield* SubscriptionRef.set(failCounterRef, remaining - 1)
|
114
|
+
if (maybeFail) {
|
115
|
+
return yield* maybeFail(batch)
|
116
|
+
}
|
117
|
+
return yield* new InvalidPushError({
|
118
|
+
cause: new UnexpectedError({ cause: new Error('MockSyncBackend: simulated push failure') }),
|
119
|
+
})
|
120
|
+
}
|
121
|
+
|
122
|
+
yield* Effect.sleep(10).pipe(Effect.withSpan('MockSyncBackend:push:sleep')) // Simulate network latency
|
123
|
+
|
124
|
+
yield* pushedEventsQueue.offerAll(batch)
|
125
|
+
yield* syncPullQueue.offerAll(batch)
|
126
|
+
allEventsRef.current = allEventsRef.current.concat(batch)
|
127
|
+
|
128
|
+
syncEventSequenceNumberRef.current = batch.at(-1)!.seqNum
|
129
|
+
}).pipe(
|
130
|
+
Effect.withSpan('MockSyncBackend:push', {
|
131
|
+
parent: span,
|
132
|
+
attributes: {
|
133
|
+
nums: batch.map((_) => _.seqNum),
|
134
|
+
},
|
135
|
+
}),
|
136
|
+
semaphore.withPermits(1),
|
137
|
+
),
|
138
|
+
metadata: {
|
139
|
+
name: '@livestore/mock-sync',
|
140
|
+
description: 'Just a mock sync backend',
|
141
|
+
},
|
142
|
+
supports: {
|
143
|
+
pullPageInfoKnown: true,
|
144
|
+
pullLive: true,
|
145
|
+
},
|
146
|
+
})
|
147
|
+
})
|
148
|
+
|
149
|
+
const advance = (...batch: LiveStoreEvent.AnyEncodedGlobal[]) =>
|
150
|
+
Effect.gen(function* () {
|
151
|
+
syncEventSequenceNumberRef.current = batch.at(-1)!.seqNum
|
152
|
+
allEventsRef.current = allEventsRef.current.concat(batch)
|
153
|
+
yield* syncPullQueue.offerAll(batch)
|
154
|
+
}).pipe(
|
155
|
+
Effect.withSpan('MockSyncBackend:advance', {
|
156
|
+
parent: span,
|
157
|
+
attributes: { nums: batch.map((_) => _.seqNum) },
|
158
|
+
}),
|
159
|
+
semaphore.withPermits(1),
|
160
|
+
)
|
161
|
+
|
162
|
+
const connect = SubscriptionRef.set(syncIsConnectedRef, true)
|
163
|
+
const disconnect = SubscriptionRef.set(syncIsConnectedRef, false)
|
164
|
+
|
165
|
+
const failNextPushes = (
|
166
|
+
count: number,
|
167
|
+
error?: (batch: ReadonlyArray<LiveStoreEvent.AnyEncodedGlobal>) => Effect.Effect<never, InvalidPushError>,
|
168
|
+
) =>
|
169
|
+
Effect.gen(function* () {
|
170
|
+
yield* SubscriptionRef.set(failCounterRef, count)
|
171
|
+
yield* SubscriptionRef.set(failEffectRef, error)
|
172
|
+
})
|
173
|
+
|
174
|
+
return {
|
175
|
+
syncEventSequenceNumberRef,
|
176
|
+
syncPullQueue,
|
177
|
+
pushedEvents: Mailbox.toStream(pushedEventsQueue),
|
178
|
+
connect,
|
179
|
+
disconnect,
|
180
|
+
makeSyncBackend,
|
181
|
+
advance,
|
182
|
+
failNextPushes,
|
183
|
+
}
|
184
|
+
}).pipe(Effect.withSpanScoped('MockSyncBackend'))
|
@@ -1,6 +1,5 @@
|
|
1
1
|
import { EventSequenceNumber } from '../../schema/mod.ts'
|
2
2
|
import { replacesFacts } from './facts.ts'
|
3
|
-
import { graphologyDag } from './graphology_.ts'
|
4
3
|
import type { HistoryDag } from './history-dag-common.ts'
|
5
4
|
import { emptyHistoryDag } from './history-dag-common.ts'
|
6
5
|
|
@@ -17,7 +16,7 @@ export const compactEvents = (inputDag: HistoryDag): { dag: HistoryDag; compacte
|
|
17
16
|
const dag = inputDag.copy()
|
18
17
|
const compactedEventCount = 0
|
19
18
|
|
20
|
-
const orderedEventSequenceNumberStrs =
|
19
|
+
const orderedEventSequenceNumberStrs = dag.topologicalNodeIds().reverse()
|
21
20
|
|
22
21
|
// drop root
|
23
22
|
orderedEventSequenceNumberStrs.pop()
|
@@ -116,7 +115,7 @@ const findSubDagsInHistory = (
|
|
116
115
|
const subDags: HistoryDag[] = []
|
117
116
|
const allOutsideDependencies: string[][] = []
|
118
117
|
|
119
|
-
for (const eventNumStr of
|
118
|
+
for (const eventNumStr of inputDag.topologicalNodeIds()) {
|
120
119
|
if (eventNumStr === upToExclEventSequenceNumberStr) {
|
121
120
|
break
|
122
121
|
}
|
@@ -189,8 +188,8 @@ const dagReplacesDag = (dagA: HistoryDag, dagB: HistoryDag): boolean => {
|
|
189
188
|
}
|
190
189
|
|
191
190
|
// TODO write tests that covers deterministic order when DAGs have branches
|
192
|
-
const nodeEntriesA =
|
193
|
-
const nodeEntriesB =
|
191
|
+
const nodeEntriesA = dagA.topologicalNodeIds().map((nodeId) => dagA.getNodeAttributes(nodeId))
|
192
|
+
const nodeEntriesB = dagB.topologicalNodeIds().map((nodeId) => dagB.getNodeAttributes(nodeId))
|
194
193
|
|
195
194
|
for (let i = 0; i < nodeEntriesA.length; i++) {
|
196
195
|
const nodeA = nodeEntriesA[i]!
|
package/src/sync/next/facts.ts
CHANGED
@@ -1,5 +1,4 @@
|
|
1
1
|
import { notYetImplemented } from '@livestore/utils'
|
2
|
-
|
3
2
|
import type {
|
4
3
|
EventDefFactInput,
|
5
4
|
EventDefFacts,
|
@@ -8,7 +7,6 @@ import type {
|
|
8
7
|
FactsCallback,
|
9
8
|
} from '../../schema/EventDef.ts'
|
10
9
|
import type * as EventSequenceNumber from '../../schema/EventSequenceNumber.ts'
|
11
|
-
import { graphologyDag } from './graphology_.ts'
|
12
10
|
import { EMPTY_FACT_VALUE, type HistoryDag, type HistoryDagNode } from './history-dag-common.ts'
|
13
11
|
|
14
12
|
export const factsSnapshotForEvents = (
|
@@ -34,7 +32,7 @@ export const factsSnapshotForDag = (
|
|
34
32
|
): EventDefFactsSnapshot => {
|
35
33
|
const facts = new Map<string, any>()
|
36
34
|
|
37
|
-
const orderedEventSequenceNumberStrs =
|
35
|
+
const orderedEventSequenceNumberStrs = dag.topologicalNodeIds()
|
38
36
|
|
39
37
|
for (let i = 0; i < orderedEventSequenceNumberStrs.length; i++) {
|
40
38
|
const event = dag.getNodeAttributes(orderedEventSequenceNumberStrs[i]!)
|