@livestore/common 0.3.0-dev.16 → 0.3.0-dev.17
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/.tsbuildinfo +1 -1
- package/dist/adapter-types.d.ts +5 -4
- package/dist/adapter-types.d.ts.map +1 -1
- package/dist/adapter-types.js.map +1 -1
- package/dist/bounded-collections.d.ts +1 -1
- package/dist/bounded-collections.d.ts.map +1 -1
- package/dist/debug-info.d.ts.map +1 -1
- package/dist/derived-mutations.d.ts.map +1 -1
- package/dist/devtools/devtools-messages-client-session.d.ts +21 -21
- package/dist/devtools/devtools-messages-common.d.ts +6 -6
- package/dist/devtools/devtools-messages-common.d.ts.map +1 -1
- package/dist/devtools/devtools-messages-leader.d.ts +28 -28
- package/dist/devtools/index.d.ts.map +1 -1
- package/dist/init-singleton-tables.d.ts.map +1 -1
- package/dist/leader-thread/LeaderSyncProcessor.d.ts +3 -1
- package/dist/leader-thread/LeaderSyncProcessor.d.ts.map +1 -1
- package/dist/leader-thread/LeaderSyncProcessor.js +124 -43
- package/dist/leader-thread/LeaderSyncProcessor.js.map +1 -1
- package/dist/leader-thread/apply-mutation.d.ts.map +1 -1
- package/dist/leader-thread/apply-mutation.js +8 -2
- package/dist/leader-thread/apply-mutation.js.map +1 -1
- package/dist/leader-thread/connection.d.ts.map +1 -1
- package/dist/leader-thread/leader-worker-devtools.d.ts.map +1 -1
- package/dist/leader-thread/make-leader-thread-layer.d.ts.map +1 -1
- package/dist/leader-thread/make-leader-thread-layer.js +1 -0
- package/dist/leader-thread/make-leader-thread-layer.js.map +1 -1
- package/dist/leader-thread/mutationlog.d.ts.map +1 -1
- package/dist/leader-thread/pull-queue-set.d.ts +3 -3
- package/dist/leader-thread/pull-queue-set.d.ts.map +1 -1
- package/dist/leader-thread/pull-queue-set.js +9 -0
- package/dist/leader-thread/pull-queue-set.js.map +1 -1
- package/dist/leader-thread/shutdown-channel.d.ts +2 -5
- package/dist/leader-thread/shutdown-channel.d.ts.map +1 -1
- package/dist/leader-thread/shutdown-channel.js +2 -4
- package/dist/leader-thread/shutdown-channel.js.map +1 -1
- package/dist/leader-thread/types.d.ts +7 -2
- package/dist/leader-thread/types.d.ts.map +1 -1
- package/dist/mutation.d.ts.map +1 -1
- package/dist/otel.d.ts.map +1 -1
- package/dist/query-builder/api.d.ts.map +1 -1
- package/dist/query-builder/impl.d.ts.map +1 -1
- package/dist/rehydrate-from-mutationlog.d.ts.map +1 -1
- package/dist/schema/EventId.d.ts +8 -0
- package/dist/schema/EventId.d.ts.map +1 -1
- package/dist/schema/EventId.js +14 -0
- package/dist/schema/EventId.js.map +1 -1
- package/dist/schema/MutationEvent.d.ts.map +1 -1
- package/dist/schema/MutationEvent.js +1 -1
- package/dist/schema/MutationEvent.js.map +1 -1
- package/dist/schema/db-schema/ast/sqlite.d.ts.map +1 -1
- package/dist/schema/db-schema/ast/validate.d.ts.map +1 -1
- package/dist/schema/db-schema/dsl/field-defs.d.ts.map +1 -1
- package/dist/schema/db-schema/dsl/field-defs.js.map +1 -1
- package/dist/schema/db-schema/dsl/mod.d.ts.map +1 -1
- package/dist/schema/db-schema/dsl/mod.js.map +1 -1
- package/dist/schema/db-schema/hash.d.ts.map +1 -1
- package/dist/schema/mutations.d.ts.map +1 -1
- package/dist/schema/schema-helpers.d.ts.map +1 -1
- package/dist/schema/schema.d.ts +3 -1
- package/dist/schema/schema.d.ts.map +1 -1
- package/dist/schema/table-def.d.ts +1 -8
- package/dist/schema/table-def.d.ts.map +1 -1
- package/dist/schema-management/common.d.ts.map +1 -1
- package/dist/schema-management/migrations.d.ts.map +1 -1
- package/dist/schema-management/validate-mutation-defs.d.ts.map +1 -1
- package/dist/sql-queries/misc.d.ts.map +1 -1
- package/dist/sql-queries/sql-queries.d.ts.map +1 -1
- package/dist/sql-queries/sql-query-builder.d.ts.map +1 -1
- package/dist/sql-queries/types.d.ts.map +1 -1
- package/dist/sync/ClientSessionSyncProcessor.d.ts +11 -1
- package/dist/sync/ClientSessionSyncProcessor.d.ts.map +1 -1
- package/dist/sync/ClientSessionSyncProcessor.js +48 -14
- package/dist/sync/ClientSessionSyncProcessor.js.map +1 -1
- package/dist/sync/next/compact-events.d.ts.map +1 -1
- package/dist/sync/next/facts.d.ts.map +1 -1
- package/dist/sync/next/history-dag.d.ts.map +1 -1
- package/dist/sync/next/rebase-events.d.ts.map +1 -1
- package/dist/sync/next/test/mutation-fixtures.d.ts.map +1 -1
- package/dist/sync/sync.d.ts +14 -9
- package/dist/sync/sync.d.ts.map +1 -1
- package/dist/sync/sync.js +7 -3
- package/dist/sync/sync.js.map +1 -1
- package/dist/sync/syncstate.d.ts +132 -21
- package/dist/sync/syncstate.d.ts.map +1 -1
- package/dist/sync/syncstate.js +129 -41
- package/dist/sync/syncstate.js.map +1 -1
- package/dist/sync/syncstate.test.js +19 -7
- package/dist/sync/syncstate.test.js.map +1 -1
- package/dist/sync/validate-push-payload.d.ts.map +1 -1
- package/dist/util.d.ts.map +1 -1
- package/dist/version.d.ts +1 -1
- package/dist/version.js +1 -1
- package/package.json +2 -2
- package/src/adapter-types.ts +5 -4
- package/src/leader-thread/LeaderSyncProcessor.ts +164 -54
- package/src/leader-thread/apply-mutation.ts +17 -2
- package/src/leader-thread/make-leader-thread-layer.ts +1 -0
- package/src/leader-thread/pull-queue-set.ts +10 -1
- package/src/leader-thread/shutdown-channel.ts +2 -4
- package/src/leader-thread/types.ts +8 -2
- package/src/schema/EventId.ts +16 -0
- package/src/schema/MutationEvent.ts +1 -1
- package/src/schema/db-schema/dsl/field-defs.ts +1 -2
- package/src/schema/db-schema/dsl/mod.ts +1 -1
- package/src/sync/ClientSessionSyncProcessor.ts +78 -13
- package/src/sync/sync.ts +7 -4
- package/src/sync/syncstate.test.ts +32 -14
- package/src/sync/syncstate.ts +145 -60
- package/src/version.ts +1 -1
@@ -1,6 +1,6 @@
|
|
1
1
|
import { LS_DEV, shouldNeverHappen, TRACE_VERBOSE } from '@livestore/utils'
|
2
2
|
import type { Runtime, Scope } from '@livestore/utils/effect'
|
3
|
-
import { Effect, Queue, Schema, Stream, Subscribable } from '@livestore/utils/effect'
|
3
|
+
import { BucketQueue, Effect, FiberHandle, Queue, Schema, Stream, Subscribable } from '@livestore/utils/effect'
|
4
4
|
import * as otel from '@opentelemetry/api'
|
5
5
|
|
6
6
|
import type { ClientSession, UnexpectedError } from '../adapter-types.js'
|
@@ -26,6 +26,7 @@ export const makeClientSessionSyncProcessor = ({
|
|
26
26
|
rollback,
|
27
27
|
refreshTables,
|
28
28
|
span,
|
29
|
+
params,
|
29
30
|
}: {
|
30
31
|
schema: LiveStoreSchema
|
31
32
|
clientSession: ClientSession
|
@@ -40,6 +41,9 @@ export const makeClientSessionSyncProcessor = ({
|
|
40
41
|
rollback: (changeset: Uint8Array) => void
|
41
42
|
refreshTables: (tables: Set<string>) => void
|
42
43
|
span: otel.Span
|
44
|
+
params: {
|
45
|
+
leaderPushBatchSize: number
|
46
|
+
}
|
43
47
|
}): ClientSessionSyncProcessor => {
|
44
48
|
const mutationEventSchema = MutationEvent.makeMutationEventSchemaMemo(schema)
|
45
49
|
|
@@ -59,6 +63,9 @@ export const makeClientSessionSyncProcessor = ({
|
|
59
63
|
return mutationDef.options.clientOnly
|
60
64
|
}
|
61
65
|
|
66
|
+
/** We're queuing push requests to reduce the number of messages sent to the leader by batching them */
|
67
|
+
const leaderPushQueue = BucketQueue.make<MutationEvent.EncodedWithMeta>().pipe(Effect.runSync)
|
68
|
+
|
62
69
|
const push: ClientSessionSyncProcessor['push'] = (batch, { otelContext }) => {
|
63
70
|
// TODO validate batch
|
64
71
|
|
@@ -84,6 +91,10 @@ export const makeClientSessionSyncProcessor = ({
|
|
84
91
|
isEqualEvent: MutationEvent.isEqualEncoded,
|
85
92
|
})
|
86
93
|
|
94
|
+
if (updateResult._tag === 'unexpected-error') {
|
95
|
+
return shouldNeverHappen('Unexpected error in client-session-sync-processor', updateResult.cause)
|
96
|
+
}
|
97
|
+
|
87
98
|
span.addEvent('local-push', {
|
88
99
|
batchSize: encodedMutationEvents.length,
|
89
100
|
updateResult: TRACE_VERBOSE ? JSON.stringify(updateResult) : undefined,
|
@@ -108,13 +119,17 @@ export const makeClientSessionSyncProcessor = ({
|
|
108
119
|
}
|
109
120
|
|
110
121
|
// console.debug('pushToLeader', encodedMutationEvents.length, ...encodedMutationEvents.map((_) => _.toJSON()))
|
111
|
-
|
112
|
-
.push(encodedMutationEvents)
|
113
|
-
.pipe(Effect.tapCauseLogPretty, Effect.provide(runtime), Effect.runFork)
|
122
|
+
BucketQueue.offerAll(leaderPushQueue, encodedMutationEvents).pipe(Effect.runSync)
|
114
123
|
|
115
124
|
return { writeTables }
|
116
125
|
}
|
117
126
|
|
127
|
+
const debugInfo = {
|
128
|
+
rebaseCount: 0,
|
129
|
+
advanceCount: 0,
|
130
|
+
rejectCount: 0,
|
131
|
+
}
|
132
|
+
|
118
133
|
const otelContext = otel.trace.setSpan(otel.context.active(), span)
|
119
134
|
|
120
135
|
const boot: ClientSessionSyncProcessor['boot'] = Effect.gen(function* () {
|
@@ -133,6 +148,20 @@ export const makeClientSessionSyncProcessor = ({
|
|
133
148
|
)
|
134
149
|
}
|
135
150
|
|
151
|
+
const leaderPushingFiberHandle = yield* FiberHandle.make()
|
152
|
+
|
153
|
+
const backgroundLeaderPushing = Effect.gen(function* () {
|
154
|
+
const batch = yield* BucketQueue.takeBetween(leaderPushQueue, 1, params.leaderPushBatchSize)
|
155
|
+
yield* clientSession.leaderThread.mutations.push(batch).pipe(
|
156
|
+
Effect.catchTag('LeaderAheadError', () => {
|
157
|
+
debugInfo.rejectCount++
|
158
|
+
return BucketQueue.clear(leaderPushQueue)
|
159
|
+
}),
|
160
|
+
)
|
161
|
+
}).pipe(Effect.forever, Effect.interruptible, Effect.tapCauseLogPretty)
|
162
|
+
|
163
|
+
yield* FiberHandle.run(leaderPushingFiberHandle, backgroundLeaderPushing)
|
164
|
+
|
136
165
|
yield* clientSession.leaderThread.mutations.pull.pipe(
|
137
166
|
Stream.tap(({ payload, remaining }) =>
|
138
167
|
Effect.gen(function* () {
|
@@ -148,9 +177,10 @@ export const makeClientSessionSyncProcessor = ({
|
|
148
177
|
isEqualEvent: MutationEvent.isEqualEncoded,
|
149
178
|
})
|
150
179
|
|
151
|
-
if (updateResult._tag === '
|
152
|
-
|
153
|
-
|
180
|
+
if (updateResult._tag === 'unexpected-error') {
|
181
|
+
return yield* Effect.fail(updateResult.cause)
|
182
|
+
} else if (updateResult._tag === 'reject') {
|
183
|
+
return shouldNeverHappen('Unexpected reject in client-session-sync-processor', updateResult)
|
154
184
|
}
|
155
185
|
|
156
186
|
syncStateRef.current = updateResult.newSyncState
|
@@ -165,11 +195,21 @@ export const makeClientSessionSyncProcessor = ({
|
|
165
195
|
res: TRACE_VERBOSE ? JSON.stringify(updateResult) : undefined,
|
166
196
|
remaining,
|
167
197
|
})
|
198
|
+
|
199
|
+
debugInfo.rebaseCount++
|
200
|
+
|
201
|
+
yield* FiberHandle.clear(leaderPushingFiberHandle)
|
202
|
+
|
203
|
+
// Reset the leader push queue since we're rebasing and will push again
|
204
|
+
yield* BucketQueue.clear(leaderPushQueue)
|
205
|
+
|
206
|
+
yield* FiberHandle.run(leaderPushingFiberHandle, backgroundLeaderPushing)
|
207
|
+
|
168
208
|
if (LS_DEV) {
|
169
209
|
Effect.logDebug(
|
170
210
|
'pull:rebase: rollback',
|
171
211
|
updateResult.eventsToRollback.length,
|
172
|
-
...updateResult.eventsToRollback.map((_) => _.toJSON()),
|
212
|
+
...updateResult.eventsToRollback.slice(0, 10).map((_) => _.toJSON()),
|
173
213
|
).pipe(Effect.provide(runtime), Effect.runSync)
|
174
214
|
}
|
175
215
|
|
@@ -181,9 +221,7 @@ export const makeClientSessionSyncProcessor = ({
|
|
181
221
|
}
|
182
222
|
}
|
183
223
|
|
184
|
-
|
185
|
-
.push(updateResult.newSyncState.pending)
|
186
|
-
.pipe(Effect.tapCauseLogPretty, Effect.provide(runtime), Effect.runFork)
|
224
|
+
yield* BucketQueue.offerAll(leaderPushQueue, updateResult.newSyncState.pending)
|
187
225
|
} else {
|
188
226
|
span.addEvent('pull:advance', {
|
189
227
|
payloadTag: payload._tag,
|
@@ -192,6 +230,8 @@ export const makeClientSessionSyncProcessor = ({
|
|
192
230
|
res: TRACE_VERBOSE ? JSON.stringify(updateResult) : undefined,
|
193
231
|
remaining,
|
194
232
|
})
|
233
|
+
|
234
|
+
debugInfo.advanceCount++
|
195
235
|
}
|
196
236
|
|
197
237
|
if (updateResult.newEvents.length === 0) return
|
@@ -208,10 +248,14 @@ export const makeClientSessionSyncProcessor = ({
|
|
208
248
|
}
|
209
249
|
|
210
250
|
refreshTables(writeTables)
|
211
|
-
})
|
251
|
+
}).pipe(
|
252
|
+
Effect.tapCauseLogPretty,
|
253
|
+
Effect.catchAllCause((cause) => Effect.sync(() => clientSession.shutdown(cause))),
|
254
|
+
),
|
212
255
|
),
|
213
256
|
Stream.runDrain,
|
214
257
|
Effect.forever, // NOTE Whenever the leader changes, we need to re-start the stream
|
258
|
+
Effect.withSpan('client-session-sync-processor:pull'),
|
215
259
|
Effect.tapCauseLogPretty,
|
216
260
|
Effect.forkScoped,
|
217
261
|
)
|
@@ -228,6 +272,21 @@ export const makeClientSessionSyncProcessor = ({
|
|
228
272
|
}),
|
229
273
|
changes: Stream.fromQueue(syncStateUpdateQueue),
|
230
274
|
}),
|
275
|
+
debug: {
|
276
|
+
print: () =>
|
277
|
+
Effect.gen(function* () {
|
278
|
+
console.log('debugInfo', debugInfo)
|
279
|
+
console.log('syncState', syncStateRef.current)
|
280
|
+
const pushQueueSize = yield* BucketQueue.size(leaderPushQueue)
|
281
|
+
console.log('pushQueueSize', pushQueueSize)
|
282
|
+
const pushQueueItems = yield* BucketQueue.peekAll(leaderPushQueue)
|
283
|
+
console.log(
|
284
|
+
'pushQueueItems',
|
285
|
+
pushQueueItems.map((_) => _.toJSON()),
|
286
|
+
)
|
287
|
+
}).pipe(Effect.provide(runtime), Effect.runSync),
|
288
|
+
debugInfo: () => debugInfo,
|
289
|
+
},
|
231
290
|
} satisfies ClientSessionSyncProcessor
|
232
291
|
}
|
233
292
|
|
@@ -239,6 +298,12 @@ export interface ClientSessionSyncProcessor {
|
|
239
298
|
writeTables: Set<string>
|
240
299
|
}
|
241
300
|
boot: Effect.Effect<void, UnexpectedError, Scope.Scope>
|
242
|
-
|
243
301
|
syncState: Subscribable.Subscribable<SyncState.SyncState>
|
302
|
+
debug: {
|
303
|
+
print: () => void
|
304
|
+
debugInfo: () => {
|
305
|
+
rebaseCount: number
|
306
|
+
advanceCount: number
|
307
|
+
}
|
308
|
+
}
|
244
309
|
}
|
package/src/sync/sync.ts
CHANGED
@@ -67,13 +67,16 @@ export class InvalidPushError extends Schema.TaggedError<InvalidPushError>()('In
|
|
67
67
|
minimumExpectedId: Schema.Number,
|
68
68
|
providedId: Schema.Number,
|
69
69
|
}),
|
70
|
-
Schema.TaggedStruct('LeaderAhead', {
|
71
|
-
minimumExpectedId: EventId.EventId,
|
72
|
-
providedId: EventId.EventId,
|
73
|
-
}),
|
74
70
|
),
|
75
71
|
}) {}
|
76
72
|
|
77
73
|
export class InvalidPullError extends Schema.TaggedError<InvalidPullError>()('InvalidPullError', {
|
78
74
|
message: Schema.String,
|
79
75
|
}) {}
|
76
|
+
|
77
|
+
export class LeaderAheadError extends Schema.TaggedError<LeaderAheadError>()('LeaderAheadError', {
|
78
|
+
minimumExpectedId: EventId.EventId,
|
79
|
+
providedId: EventId.EventId,
|
80
|
+
/** Generation number the client session should use for subsequent pushes */
|
81
|
+
// nextGeneration: Schema.Number,
|
82
|
+
}) {}
|
@@ -17,7 +17,7 @@ class TestEvent extends MutationEvent.EncodedWithMeta {
|
|
17
17
|
parentId: EventId.make(parentId),
|
18
18
|
mutation: 'a',
|
19
19
|
args: payload,
|
20
|
-
|
20
|
+
|
21
21
|
clientId: 'static-local-id',
|
22
22
|
sessionId: undefined,
|
23
23
|
})
|
@@ -87,7 +87,7 @@ describe('syncstate', () => {
|
|
87
87
|
}
|
88
88
|
expect(result.newSyncState.upstreamHead).toMatchObject(e_0_1_e_1_1.id)
|
89
89
|
expect(result.newSyncState.localHead).toMatchObject(e_1_0_e_2_0.id)
|
90
|
-
expectEventArraysEqual(result.newEvents, [e_0_0_e_1_0, e_0_1_e_1_1])
|
90
|
+
expectEventArraysEqual(result.newEvents, [e_0_0_e_1_0, e_0_1_e_1_1, e_1_0_e_2_0])
|
91
91
|
expectEventArraysEqual(result.eventsToRollback, [e_0_0, e_0_1, e_1_0])
|
92
92
|
})
|
93
93
|
|
@@ -118,7 +118,7 @@ describe('syncstate', () => {
|
|
118
118
|
}
|
119
119
|
expect(result.newSyncState.upstreamHead).toMatchObject(e_0_1_e_1_0.id)
|
120
120
|
expect(result.newSyncState.localHead).toMatchObject(e_1_0_e_2_0.id)
|
121
|
-
expectEventArraysEqual(result.newEvents, [e_0_1_e_1_0])
|
121
|
+
expectEventArraysEqual(result.newEvents, [e_0_1_e_1_0, e_1_0_e_2_0])
|
122
122
|
expectEventArraysEqual(result.eventsToRollback, [e_0_1, e_1_0])
|
123
123
|
})
|
124
124
|
|
@@ -148,12 +148,11 @@ describe('syncstate', () => {
|
|
148
148
|
upstreamHead: EventId.ROOT,
|
149
149
|
localHead: e_0_0.id,
|
150
150
|
})
|
151
|
-
|
152
|
-
|
153
|
-
|
154
|
-
|
155
|
-
|
156
|
-
).toThrow()
|
151
|
+
const result = run({
|
152
|
+
syncState,
|
153
|
+
payload: { _tag: 'upstream-rebase', rollbackUntil: e_0_0.id, newEvents: [e_1_0] },
|
154
|
+
})
|
155
|
+
expect(result).toMatchObject({ _tag: 'unexpected-error' })
|
157
156
|
})
|
158
157
|
|
159
158
|
it('should work for empty incoming', () => {
|
@@ -185,7 +184,8 @@ describe('syncstate', () => {
|
|
185
184
|
upstreamHead: EventId.ROOT,
|
186
185
|
localHead: e_0_0.id,
|
187
186
|
})
|
188
|
-
|
187
|
+
const result = run({ syncState, payload: { _tag: 'upstream-advance', newEvents: [e_0_1, e_0_0] } })
|
188
|
+
expect(result).toMatchObject({ _tag: 'unexpected-error' })
|
189
189
|
})
|
190
190
|
|
191
191
|
it('should throw error if newEvents are not sorted in ascending order by eventId (global)', () => {
|
@@ -195,7 +195,8 @@ describe('syncstate', () => {
|
|
195
195
|
upstreamHead: EventId.ROOT,
|
196
196
|
localHead: e_0_0.id,
|
197
197
|
})
|
198
|
-
|
198
|
+
const result = run({ syncState, payload: { _tag: 'upstream-advance', newEvents: [e_1_0, e_0_0] } })
|
199
|
+
expect(result).toMatchObject({ _tag: 'unexpected-error' })
|
199
200
|
})
|
200
201
|
|
201
202
|
it('should acknowledge pending event when receiving matching event', () => {
|
@@ -329,6 +330,17 @@ describe('syncstate', () => {
|
|
329
330
|
expect(result.newSyncState.localHead).toMatchObject(e_1_0.id)
|
330
331
|
expect(result.newEvents).toStrictEqual([e_1_0])
|
331
332
|
})
|
333
|
+
|
334
|
+
it('should fail if incoming event is ≤ local head', () => {
|
335
|
+
const syncState = new SyncState.SyncState({
|
336
|
+
pending: [],
|
337
|
+
rollbackTail: [],
|
338
|
+
upstreamHead: e_1_0.id,
|
339
|
+
localHead: e_1_0.id,
|
340
|
+
})
|
341
|
+
const result = run({ syncState, payload: { _tag: 'upstream-advance', newEvents: [e_0_0] } })
|
342
|
+
expect(result).toMatchObject({ _tag: 'unexpected-error' })
|
343
|
+
})
|
332
344
|
})
|
333
345
|
|
334
346
|
describe('upstream-advance: rebase', () => {
|
@@ -513,14 +525,20 @@ const expectEventArraysEqual = (
|
|
513
525
|
})
|
514
526
|
}
|
515
527
|
|
516
|
-
function expectAdvance(
|
528
|
+
function expectAdvance(
|
529
|
+
result: typeof SyncState.UpdateResult.Type,
|
530
|
+
): asserts result is typeof SyncState.UpdateResultAdvance.Type {
|
517
531
|
expect(result._tag).toBe('advance')
|
518
532
|
}
|
519
533
|
|
520
|
-
function expectRebase(
|
534
|
+
function expectRebase(
|
535
|
+
result: typeof SyncState.UpdateResult.Type,
|
536
|
+
): asserts result is typeof SyncState.UpdateResultRebase.Type {
|
521
537
|
expect(result._tag).toBe('rebase')
|
522
538
|
}
|
523
539
|
|
524
|
-
function expectReject(
|
540
|
+
function expectReject(
|
541
|
+
result: typeof SyncState.UpdateResult.Type,
|
542
|
+
): asserts result is typeof SyncState.UpdateResultReject.Type {
|
525
543
|
expect(result._tag).toBe('reject')
|
526
544
|
}
|
package/src/sync/syncstate.ts
CHANGED
@@ -1,6 +1,7 @@
|
|
1
|
-
import {
|
2
|
-
import { ReadonlyArray, Schema } from '@livestore/utils/effect'
|
1
|
+
import { casesHandled } from '@livestore/utils'
|
2
|
+
import { Match, ReadonlyArray, Schema } from '@livestore/utils/effect'
|
3
3
|
|
4
|
+
import { UnexpectedError } from '../adapter-types.js'
|
4
5
|
import * as EventId from '../schema/EventId.js'
|
5
6
|
import * as MutationEvent from '../schema/MutationEvent.js'
|
6
7
|
|
@@ -79,31 +80,104 @@ export const PayloadUpstream = Schema.Union(PayloadUpstreamRebase, PayloadUpstre
|
|
79
80
|
|
80
81
|
export type PayloadUpstream = typeof PayloadUpstream.Type
|
81
82
|
|
82
|
-
|
83
|
-
|
84
|
-
|
85
|
-
|
83
|
+
/** Only used for debugging purposes */
|
84
|
+
export class UpdateContext extends Schema.Class<UpdateContext>('UpdateContext')({
|
85
|
+
payload: Payload,
|
86
|
+
syncState: SyncState,
|
87
|
+
}) {
|
88
|
+
toJSON = (): any => {
|
89
|
+
const payload = Match.value(this.payload).pipe(
|
90
|
+
Match.tag('local-push', () => ({
|
91
|
+
_tag: 'local-push',
|
92
|
+
newEvents: this.payload.newEvents.map((e) => e.toJSON()),
|
93
|
+
})),
|
94
|
+
Match.tag('upstream-advance', () => ({
|
95
|
+
_tag: 'upstream-advance',
|
96
|
+
newEvents: this.payload.newEvents.map((e) => e.toJSON()),
|
97
|
+
})),
|
98
|
+
Match.tag('upstream-rebase', () => ({
|
99
|
+
_tag: 'upstream-rebase',
|
100
|
+
newEvents: this.payload.newEvents.map((e) => e.toJSON()),
|
101
|
+
})),
|
102
|
+
Match.exhaustive,
|
103
|
+
)
|
104
|
+
return {
|
105
|
+
payload,
|
106
|
+
syncState: this.syncState.toJSON(),
|
107
|
+
}
|
108
|
+
}
|
109
|
+
}
|
110
|
+
|
111
|
+
export class UpdateResultAdvance extends Schema.Class<UpdateResultAdvance>('UpdateResultAdvance')({
|
112
|
+
_tag: Schema.Literal('advance'),
|
113
|
+
newSyncState: SyncState,
|
86
114
|
/** Events which weren't pending before the update */
|
87
|
-
newEvents:
|
115
|
+
newEvents: Schema.Array(MutationEvent.EncodedWithMeta),
|
116
|
+
updateContext: UpdateContext,
|
117
|
+
}) {
|
118
|
+
toJSON = (): any => {
|
119
|
+
return {
|
120
|
+
_tag: this._tag,
|
121
|
+
newSyncState: this.newSyncState.toJSON(),
|
122
|
+
newEvents: this.newEvents.map((e) => e.toJSON()),
|
123
|
+
updateContext: this.updateContext.toJSON(),
|
124
|
+
}
|
125
|
+
}
|
88
126
|
}
|
89
127
|
|
90
|
-
export
|
91
|
-
_tag: 'rebase'
|
92
|
-
newSyncState: SyncState
|
93
|
-
previousSyncState: SyncState
|
128
|
+
export class UpdateResultRebase extends Schema.Class<UpdateResultRebase>('UpdateResultRebase')({
|
129
|
+
_tag: Schema.Literal('rebase'),
|
130
|
+
newSyncState: SyncState,
|
94
131
|
/** Events which weren't pending before the update */
|
95
|
-
newEvents:
|
96
|
-
eventsToRollback:
|
132
|
+
newEvents: Schema.Array(MutationEvent.EncodedWithMeta),
|
133
|
+
eventsToRollback: Schema.Array(MutationEvent.EncodedWithMeta),
|
134
|
+
updateContext: UpdateContext,
|
135
|
+
}) {
|
136
|
+
toJSON = (): any => {
|
137
|
+
return {
|
138
|
+
_tag: this._tag,
|
139
|
+
newSyncState: this.newSyncState.toJSON(),
|
140
|
+
newEvents: this.newEvents.map((e) => e.toJSON()),
|
141
|
+
eventsToRollback: this.eventsToRollback.map((e) => e.toJSON()),
|
142
|
+
updateContext: this.updateContext.toJSON(),
|
143
|
+
}
|
144
|
+
}
|
97
145
|
}
|
98
146
|
|
99
|
-
export
|
100
|
-
_tag: 'reject'
|
101
|
-
previousSyncState: SyncState
|
147
|
+
export class UpdateResultReject extends Schema.Class<UpdateResultReject>('UpdateResultReject')({
|
148
|
+
_tag: Schema.Literal('reject'),
|
102
149
|
/** The minimum id that the new events must have */
|
103
|
-
expectedMinimumId: EventId.EventId
|
150
|
+
expectedMinimumId: EventId.EventId,
|
151
|
+
updateContext: UpdateContext,
|
152
|
+
}) {
|
153
|
+
toJSON = (): any => {
|
154
|
+
return {
|
155
|
+
_tag: this._tag,
|
156
|
+
expectedMinimumId: `(${this.expectedMinimumId.global},${this.expectedMinimumId.client})`,
|
157
|
+
updateContext: this.updateContext.toJSON(),
|
158
|
+
}
|
159
|
+
}
|
104
160
|
}
|
105
161
|
|
106
|
-
export
|
162
|
+
export class UpdateResultUnexpectedError extends Schema.Class<UpdateResultUnexpectedError>(
|
163
|
+
'UpdateResultUnexpectedError',
|
164
|
+
)({
|
165
|
+
_tag: Schema.Literal('unexpected-error'),
|
166
|
+
cause: UnexpectedError,
|
167
|
+
}) {}
|
168
|
+
|
169
|
+
export class UpdateResult extends Schema.Union(
|
170
|
+
UpdateResultAdvance,
|
171
|
+
UpdateResultRebase,
|
172
|
+
UpdateResultReject,
|
173
|
+
UpdateResultUnexpectedError,
|
174
|
+
) {}
|
175
|
+
|
176
|
+
const unexpectedError = (cause: unknown): UpdateResultUnexpectedError =>
|
177
|
+
UpdateResultUnexpectedError.make({
|
178
|
+
_tag: 'unexpected-error',
|
179
|
+
cause: new UnexpectedError({ cause }),
|
180
|
+
})
|
107
181
|
|
108
182
|
export const updateSyncState = ({
|
109
183
|
syncState,
|
@@ -118,7 +192,7 @@ export const updateSyncState = ({
|
|
118
192
|
isEqualEvent: (a: MutationEvent.EncodedWithMeta, b: MutationEvent.EncodedWithMeta) => boolean
|
119
193
|
/** This is used in the leader which should ignore local events when receiving an upstream-advance payload */
|
120
194
|
ignoreLocalEvents?: boolean
|
121
|
-
}): UpdateResult => {
|
195
|
+
}): typeof UpdateResult.Type => {
|
122
196
|
const trimRollbackTail = (
|
123
197
|
rollbackTail: ReadonlyArray<MutationEvent.EncodedWithMeta>,
|
124
198
|
): ReadonlyArray<MutationEvent.EncodedWithMeta> => {
|
@@ -129,6 +203,8 @@ export const updateSyncState = ({
|
|
129
203
|
return rollbackTail.slice(index + 1)
|
130
204
|
}
|
131
205
|
|
206
|
+
const updateContext = UpdateContext.make({ payload, syncState })
|
207
|
+
|
132
208
|
switch (payload._tag) {
|
133
209
|
case 'upstream-rebase': {
|
134
210
|
// Find the index of the rollback event in the rollback tail
|
@@ -136,7 +212,7 @@ export const updateSyncState = ({
|
|
136
212
|
EventId.isEqual(event.id, payload.rollbackUntil),
|
137
213
|
)
|
138
214
|
if (rollbackIndex === -1) {
|
139
|
-
return
|
215
|
+
return unexpectedError(
|
140
216
|
`Rollback event not found in rollback tail. Rollback until: [${payload.rollbackUntil.global},${payload.rollbackUntil.client}]. Rollback tail: [${syncState.rollbackTail.map((e) => e.toString()).join(', ')}]`,
|
141
217
|
)
|
142
218
|
}
|
@@ -153,7 +229,7 @@ export const updateSyncState = ({
|
|
153
229
|
isLocalEvent,
|
154
230
|
})
|
155
231
|
|
156
|
-
return {
|
232
|
+
return UpdateResultRebase.make({
|
157
233
|
_tag: 'rebase',
|
158
234
|
newSyncState: new SyncState({
|
159
235
|
pending: rebasedPending,
|
@@ -161,15 +237,15 @@ export const updateSyncState = ({
|
|
161
237
|
upstreamHead: newUpstreamHead,
|
162
238
|
localHead: rebasedPending.at(-1)?.id ?? newUpstreamHead,
|
163
239
|
}),
|
164
|
-
|
165
|
-
newEvents: payload.newEvents,
|
240
|
+
newEvents: [...payload.newEvents, ...rebasedPending],
|
166
241
|
eventsToRollback,
|
167
|
-
|
242
|
+
updateContext,
|
243
|
+
})
|
168
244
|
}
|
169
245
|
|
170
246
|
case 'upstream-advance': {
|
171
247
|
if (payload.newEvents.length === 0) {
|
172
|
-
return {
|
248
|
+
return UpdateResultAdvance.make({
|
173
249
|
_tag: 'advance',
|
174
250
|
newSyncState: new SyncState({
|
175
251
|
pending: syncState.pending,
|
@@ -177,18 +253,27 @@ export const updateSyncState = ({
|
|
177
253
|
upstreamHead: syncState.upstreamHead,
|
178
254
|
localHead: syncState.localHead,
|
179
255
|
}),
|
180
|
-
previousSyncState: syncState,
|
181
256
|
newEvents: [],
|
182
|
-
|
257
|
+
updateContext,
|
258
|
+
})
|
183
259
|
}
|
184
260
|
|
185
261
|
// Validate that newEvents are sorted in ascending order by eventId
|
186
262
|
for (let i = 1; i < payload.newEvents.length; i++) {
|
187
263
|
if (EventId.isGreaterThan(payload.newEvents[i - 1]!.id, payload.newEvents[i]!.id)) {
|
188
|
-
return
|
264
|
+
return unexpectedError(
|
265
|
+
`Events must be sorted in ascending order by eventId. Received: [${payload.newEvents.map((e) => `(${e.id.global},${e.id.client})`).join(', ')}]`,
|
266
|
+
)
|
189
267
|
}
|
190
268
|
}
|
191
269
|
|
270
|
+
// Validate that incoming events are larger than upstream head
|
271
|
+
if (EventId.isGreaterThan(syncState.upstreamHead, payload.newEvents[0]!.id)) {
|
272
|
+
return unexpectedError(
|
273
|
+
`Incoming events must be greater than upstream head. Expected greater than: [${syncState.upstreamHead.global},${syncState.upstreamHead.client}]. Received: [${payload.newEvents.map((e) => `(${e.id.global},${e.id.client})`).join(', ')}]`,
|
274
|
+
)
|
275
|
+
}
|
276
|
+
|
192
277
|
const newUpstreamHead = payload.newEvents.at(-1)!.id
|
193
278
|
|
194
279
|
const divergentPendingIndex = findDivergencePoint({
|
@@ -235,7 +320,7 @@ export const updateSyncState = ({
|
|
235
320
|
return true
|
236
321
|
})
|
237
322
|
|
238
|
-
return {
|
323
|
+
return UpdateResultAdvance.make({
|
239
324
|
_tag: 'advance',
|
240
325
|
newSyncState: new SyncState({
|
241
326
|
pending: pendingRemaining,
|
@@ -243,9 +328,9 @@ export const updateSyncState = ({
|
|
243
328
|
upstreamHead: newUpstreamHead,
|
244
329
|
localHead: pendingRemaining.at(-1)?.id ?? newUpstreamHead,
|
245
330
|
}),
|
246
|
-
previousSyncState: syncState,
|
247
331
|
newEvents,
|
248
|
-
|
332
|
+
updateContext,
|
333
|
+
})
|
249
334
|
} else {
|
250
335
|
const divergentPending = syncState.pending.slice(divergentPendingIndex)
|
251
336
|
const rebasedPending = rebaseEvents({
|
@@ -262,7 +347,7 @@ export const updateSyncState = ({
|
|
262
347
|
ignoreLocalEvents,
|
263
348
|
})
|
264
349
|
|
265
|
-
return {
|
350
|
+
return UpdateResultRebase.make({
|
266
351
|
_tag: 'rebase',
|
267
352
|
newSyncState: new SyncState({
|
268
353
|
pending: rebasedPending,
|
@@ -270,16 +355,21 @@ export const updateSyncState = ({
|
|
270
355
|
upstreamHead: newUpstreamHead,
|
271
356
|
localHead: rebasedPending.at(-1)!.id,
|
272
357
|
}),
|
273
|
-
previousSyncState: syncState,
|
274
358
|
newEvents: [...payload.newEvents.slice(divergentNewEventsIndex), ...rebasedPending],
|
275
359
|
eventsToRollback: [...syncState.rollbackTail, ...divergentPending],
|
276
|
-
|
360
|
+
updateContext,
|
361
|
+
})
|
277
362
|
}
|
278
363
|
}
|
279
364
|
|
280
365
|
case 'local-push': {
|
281
366
|
if (payload.newEvents.length === 0) {
|
282
|
-
return {
|
367
|
+
return UpdateResultAdvance.make({
|
368
|
+
_tag: 'advance',
|
369
|
+
newSyncState: syncState,
|
370
|
+
newEvents: [],
|
371
|
+
updateContext,
|
372
|
+
})
|
283
373
|
}
|
284
374
|
|
285
375
|
const newEventsFirst = payload.newEvents.at(0)!
|
@@ -287,9 +377,13 @@ export const updateSyncState = ({
|
|
287
377
|
|
288
378
|
if (invalidEventId) {
|
289
379
|
const expectedMinimumId = EventId.nextPair(syncState.localHead, true).id
|
290
|
-
return {
|
380
|
+
return UpdateResultReject.make({
|
381
|
+
_tag: 'reject',
|
382
|
+
expectedMinimumId,
|
383
|
+
updateContext,
|
384
|
+
})
|
291
385
|
} else {
|
292
|
-
return {
|
386
|
+
return UpdateResultAdvance.make({
|
293
387
|
_tag: 'advance',
|
294
388
|
newSyncState: new SyncState({
|
295
389
|
pending: [...syncState.pending, ...payload.newEvents],
|
@@ -297,33 +391,15 @@ export const updateSyncState = ({
|
|
297
391
|
upstreamHead: syncState.upstreamHead,
|
298
392
|
localHead: payload.newEvents.at(-1)!.id,
|
299
393
|
}),
|
300
|
-
previousSyncState: syncState,
|
301
394
|
newEvents: payload.newEvents,
|
302
|
-
|
395
|
+
updateContext,
|
396
|
+
})
|
303
397
|
}
|
304
398
|
}
|
305
399
|
|
306
|
-
|
307
|
-
|
308
|
-
|
309
|
-
// if (startIndex === -1) {
|
310
|
-
// return shouldNeverHappen('New rollback start event not found in rollback tail')
|
311
|
-
// }
|
312
|
-
|
313
|
-
// // Keep only the events from the start index onwards
|
314
|
-
// const newRollbackTail = syncState.rollbackTail.slice(startIndex)
|
315
|
-
|
316
|
-
// return {
|
317
|
-
// _tag: 'advance',
|
318
|
-
// syncState: {
|
319
|
-
// pending: syncState.pending,
|
320
|
-
// rollbackTail: newRollbackTail,
|
321
|
-
// upstreamHead: syncState.upstreamHead,
|
322
|
-
// localHead: syncState.localHead,
|
323
|
-
// },
|
324
|
-
// newEvents: [],
|
325
|
-
// }
|
326
|
-
// }
|
400
|
+
default: {
|
401
|
+
casesHandled(payload)
|
402
|
+
}
|
327
403
|
}
|
328
404
|
}
|
329
405
|
|
@@ -385,3 +461,12 @@ const rebaseEvents = ({
|
|
385
461
|
return newEvent
|
386
462
|
})
|
387
463
|
}
|
464
|
+
|
465
|
+
/**
|
466
|
+
* TODO: Implement this
|
467
|
+
*
|
468
|
+
* In certain scenarios e.g. when the client session has a queue of upstream update results,
|
469
|
+
* it could make sense to "flatten" update results into a single update result which the client session
|
470
|
+
* can process more efficiently which avoids push-threshing
|
471
|
+
*/
|
472
|
+
const _flattenUpdateResults = (_updateResults: ReadonlyArray<UpdateResult>) => {}
|
package/src/version.ts
CHANGED
@@ -2,7 +2,7 @@
|
|
2
2
|
// import packageJson from '../package.json' with { type: 'json' }
|
3
3
|
// export const liveStoreVersion = packageJson.version
|
4
4
|
|
5
|
-
export const liveStoreVersion = '0.3.0-dev.
|
5
|
+
export const liveStoreVersion = '0.3.0-dev.17' as const
|
6
6
|
|
7
7
|
/**
|
8
8
|
* This version number is incremented whenever the internal storage format changes in a breaking way.
|