@livestore/sync-cf 0.4.0-dev.9 → 0.5.0-dev.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +7 -8
- package/dist/.tsbuildinfo +1 -1
- package/dist/cf-worker/do/durable-object.d.ts +1 -1
- package/dist/cf-worker/do/durable-object.d.ts.map +1 -1
- package/dist/cf-worker/do/durable-object.js +36 -18
- package/dist/cf-worker/do/durable-object.js.map +1 -1
- package/dist/cf-worker/do/layer.d.ts +24 -20
- package/dist/cf-worker/do/layer.d.ts.map +1 -1
- package/dist/cf-worker/do/layer.js +57 -24
- package/dist/cf-worker/do/layer.js.map +1 -1
- package/dist/cf-worker/do/pull.d.ts +8 -3
- package/dist/cf-worker/do/pull.d.ts.map +1 -1
- package/dist/cf-worker/do/pull.js +26 -21
- package/dist/cf-worker/do/pull.js.map +1 -1
- package/dist/cf-worker/do/push.d.ts +20 -6
- package/dist/cf-worker/do/push.d.ts.map +1 -1
- package/dist/cf-worker/do/push.js +84 -56
- package/dist/cf-worker/do/push.js.map +1 -1
- package/dist/cf-worker/do/push.test.d.ts +2 -0
- package/dist/cf-worker/do/push.test.d.ts.map +1 -0
- package/dist/cf-worker/do/push.test.js +30 -0
- package/dist/cf-worker/do/push.test.js.map +1 -0
- package/dist/cf-worker/do/sqlite.d.ts +49 -52
- package/dist/cf-worker/do/sqlite.d.ts.map +1 -1
- package/dist/cf-worker/do/sqlite.js +14 -5
- package/dist/cf-worker/do/sqlite.js.map +1 -1
- package/dist/cf-worker/do/sync-storage.d.ts +15 -9
- package/dist/cf-worker/do/sync-storage.d.ts.map +1 -1
- package/dist/cf-worker/do/sync-storage.js +111 -25
- package/dist/cf-worker/do/sync-storage.js.map +1 -1
- package/dist/cf-worker/do/transport/do-rpc-server.d.ts +2 -2
- package/dist/cf-worker/do/transport/do-rpc-server.d.ts.map +1 -1
- package/dist/cf-worker/do/transport/do-rpc-server.js +31 -23
- package/dist/cf-worker/do/transport/do-rpc-server.js.map +1 -1
- package/dist/cf-worker/do/transport/http-rpc-server.d.ts +5 -3
- package/dist/cf-worker/do/transport/http-rpc-server.d.ts.map +1 -1
- package/dist/cf-worker/do/transport/http-rpc-server.js +28 -19
- package/dist/cf-worker/do/transport/http-rpc-server.js.map +1 -1
- package/dist/cf-worker/do/transport/ws-rpc-server.d.ts +3 -2
- package/dist/cf-worker/do/transport/ws-rpc-server.d.ts.map +1 -1
- package/dist/cf-worker/do/transport/ws-rpc-server.js +32 -10
- package/dist/cf-worker/do/transport/ws-rpc-server.js.map +1 -1
- package/dist/cf-worker/shared.d.ts +124 -34
- package/dist/cf-worker/shared.d.ts.map +1 -1
- package/dist/cf-worker/shared.js +52 -17
- package/dist/cf-worker/shared.js.map +1 -1
- package/dist/cf-worker/worker.d.ts +47 -39
- package/dist/cf-worker/worker.d.ts.map +1 -1
- package/dist/cf-worker/worker.js +54 -35
- package/dist/cf-worker/worker.js.map +1 -1
- package/dist/client/transport/do-rpc-client.d.ts +13 -4
- package/dist/client/transport/do-rpc-client.d.ts.map +1 -1
- package/dist/client/transport/do-rpc-client.js +74 -28
- package/dist/client/transport/do-rpc-client.js.map +1 -1
- package/dist/client/transport/http-rpc-client.d.ts +3 -3
- package/dist/client/transport/http-rpc-client.d.ts.map +1 -1
- package/dist/client/transport/http-rpc-client.js +51 -22
- package/dist/client/transport/http-rpc-client.js.map +1 -1
- package/dist/client/transport/ws-rpc-client.d.ts +4 -3
- package/dist/client/transport/ws-rpc-client.d.ts.map +1 -1
- package/dist/client/transport/ws-rpc-client.js +40 -22
- package/dist/client/transport/ws-rpc-client.js.map +1 -1
- package/dist/common/constants.d.ts +7 -0
- package/dist/common/constants.d.ts.map +1 -0
- package/dist/common/constants.js +17 -0
- package/dist/common/constants.js.map +1 -0
- package/dist/common/do-rpc-schema.d.ts +42 -43
- package/dist/common/do-rpc-schema.d.ts.map +1 -1
- package/dist/common/do-rpc-schema.js +5 -5
- package/dist/common/do-rpc-schema.js.map +1 -1
- package/dist/common/http-rpc-schema.d.ts +37 -38
- package/dist/common/http-rpc-schema.d.ts.map +1 -1
- package/dist/common/http-rpc-schema.js +7 -7
- package/dist/common/http-rpc-schema.js.map +1 -1
- package/dist/common/mod.d.ts +5 -3
- package/dist/common/mod.d.ts.map +1 -1
- package/dist/common/mod.js +4 -2
- package/dist/common/mod.js.map +1 -1
- package/dist/common/sync-message-types.d.ts +113 -130
- package/dist/common/sync-message-types.d.ts.map +1 -1
- package/dist/common/sync-message-types.js +29 -17
- package/dist/common/sync-message-types.js.map +1 -1
- package/dist/common/ws-rpc-schema.d.ts +34 -35
- package/dist/common/ws-rpc-schema.d.ts.map +1 -1
- package/dist/common/ws-rpc-schema.js +5 -5
- package/dist/common/ws-rpc-schema.js.map +1 -1
- package/package.json +57 -15
- package/src/cf-worker/do/durable-object.ts +59 -41
- package/src/cf-worker/do/layer.ts +153 -88
- package/src/cf-worker/do/pull.ts +48 -31
- package/src/cf-worker/do/push.test.ts +40 -0
- package/src/cf-worker/do/push.ts +163 -125
- package/src/cf-worker/do/sqlite.ts +15 -5
- package/src/cf-worker/do/sync-storage.ts +171 -38
- package/src/cf-worker/do/transport/do-rpc-server.ts +40 -40
- package/src/cf-worker/do/transport/http-rpc-server.ts +37 -20
- package/src/cf-worker/do/transport/ws-rpc-server.ts +44 -16
- package/src/cf-worker/shared.ts +152 -36
- package/src/cf-worker/worker.ts +110 -58
- package/src/client/transport/do-rpc-client.ts +106 -40
- package/src/client/transport/http-rpc-client.ts +74 -38
- package/src/client/transport/ws-rpc-client.ts +55 -29
- package/src/common/constants.ts +18 -0
- package/src/common/do-rpc-schema.ts +6 -5
- package/src/common/http-rpc-schema.ts +8 -7
- package/src/common/mod.ts +4 -3
- package/src/common/sync-message-types.ts +19 -19
- package/src/common/ws-rpc-schema.ts +6 -5
- package/dist/cf-worker/do/ws-chunking.d.ts +0 -22
- package/dist/cf-worker/do/ws-chunking.d.ts.map +0 -1
- package/dist/cf-worker/do/ws-chunking.js +0 -49
- package/dist/cf-worker/do/ws-chunking.js.map +0 -1
- package/src/cf-worker/do/ws-chunking.ts +0 -76
package/src/cf-worker/do/pull.ts
CHANGED
|
@@ -1,9 +1,11 @@
|
|
|
1
|
-
import { BackendIdMismatchError,
|
|
2
|
-
import {
|
|
1
|
+
import { BackendIdMismatchError, SyncBackend, UnknownError } from '@livestore/common'
|
|
2
|
+
import { splitArrayBySize } from '@livestore/common/sync'
|
|
3
|
+
import { Effect, Option, Schema, Stream } from '@livestore/utils/effect'
|
|
4
|
+
|
|
5
|
+
import { MAX_PULL_EVENTS_PER_MESSAGE, MAX_WS_MESSAGE_BYTES } from '../../common/constants.ts'
|
|
3
6
|
import { SyncMessage } from '../../common/mod.ts'
|
|
4
|
-
import {
|
|
5
|
-
import
|
|
6
|
-
import { splitChunkBySize } from './ws-chunking.ts'
|
|
7
|
+
import type { ForwardedHeaders } from '../shared.ts'
|
|
8
|
+
import * as DoCtx from './layer.ts'
|
|
7
9
|
|
|
8
10
|
const encodePullResponse = Schema.encodeSync(SyncMessage.PullResponse)
|
|
9
11
|
|
|
@@ -16,17 +18,26 @@ const encodePullResponse = Schema.encodeSync(SyncMessage.PullResponse)
|
|
|
16
18
|
// DO RPC:
|
|
17
19
|
// - Further chunks will be emitted manually in `push.ts`
|
|
18
20
|
// - If the client sends a `Interrupt` RPC message, TODO
|
|
19
|
-
export const makeEndingPullStream = (
|
|
20
|
-
req
|
|
21
|
-
payload
|
|
22
|
-
|
|
21
|
+
export const makeEndingPullStream = ({
|
|
22
|
+
req,
|
|
23
|
+
payload,
|
|
24
|
+
headers,
|
|
25
|
+
}: {
|
|
26
|
+
req: SyncMessage.PullRequest
|
|
27
|
+
payload: Schema.Json | undefined
|
|
28
|
+
headers: ForwardedHeaders | undefined
|
|
29
|
+
}): Stream.Stream<SyncMessage.PullResponse, UnknownError | BackendIdMismatchError, DoCtx.DoCtx> =>
|
|
23
30
|
Effect.gen(function* () {
|
|
24
|
-
const { doOptions, backendId, storeId, storage } = yield* DoCtx
|
|
31
|
+
const { doOptions, backendId, storeId, storage } = yield* DoCtx.DoCtx
|
|
25
32
|
|
|
26
|
-
if (doOptions?.onPull) {
|
|
27
|
-
yield* Effect.
|
|
28
|
-
|
|
29
|
-
|
|
33
|
+
if (doOptions?.onPull !== undefined) {
|
|
34
|
+
yield* Effect.trySyncOrPromiseOrEffect(() =>
|
|
35
|
+
doOptions.onPull!(req, {
|
|
36
|
+
storeId,
|
|
37
|
+
...(payload !== undefined ? { payload } : {}),
|
|
38
|
+
...(headers !== undefined ? { headers } : {}),
|
|
39
|
+
}),
|
|
40
|
+
).pipe(UnknownError.mapToUnknownError)
|
|
30
41
|
}
|
|
31
42
|
|
|
32
43
|
if (req.cursor._tag === 'Some' && req.cursor.value.backendId !== backendId) {
|
|
@@ -38,8 +49,8 @@ export const makeEndingPullStream = (
|
|
|
38
49
|
)
|
|
39
50
|
|
|
40
51
|
return storedEvents.pipe(
|
|
41
|
-
Stream.
|
|
42
|
-
|
|
52
|
+
Stream.mapArrayEffect(
|
|
53
|
+
splitArrayBySize({
|
|
43
54
|
maxItems: MAX_PULL_EVENTS_PER_MESSAGE,
|
|
44
55
|
maxBytes: MAX_WS_MESSAGE_BYTES,
|
|
45
56
|
encode: (batch) =>
|
|
@@ -48,23 +59,27 @@ export const makeEndingPullStream = (
|
|
|
48
59
|
),
|
|
49
60
|
}),
|
|
50
61
|
),
|
|
51
|
-
Stream.mapAccum(
|
|
52
|
-
|
|
53
|
-
|
|
62
|
+
Stream.mapAccum(
|
|
63
|
+
() => total,
|
|
64
|
+
(remaining, array) => {
|
|
65
|
+
const nextRemaining = Math.max(0, remaining - array.length)
|
|
54
66
|
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
67
|
+
return [
|
|
68
|
+
nextRemaining,
|
|
69
|
+
[
|
|
70
|
+
SyncMessage.PullResponse.make({
|
|
71
|
+
batch: array,
|
|
72
|
+
pageInfo: nextRemaining > 0 ? SyncBackend.pageInfoMoreKnown(nextRemaining) : SyncBackend.pageInfoNoMore,
|
|
73
|
+
backendId,
|
|
74
|
+
}),
|
|
75
|
+
],
|
|
76
|
+
] as const
|
|
77
|
+
},
|
|
78
|
+
),
|
|
64
79
|
Stream.tap(
|
|
65
80
|
Effect.fn(function* (res) {
|
|
66
|
-
if (doOptions?.onPullRes) {
|
|
67
|
-
yield* Effect.
|
|
81
|
+
if (doOptions?.onPullRes !== undefined) {
|
|
82
|
+
yield* Effect.trySyncOrPromiseOrEffect(() => doOptions.onPullRes!(res)).pipe(UnknownError.mapToUnknownError)
|
|
68
83
|
}
|
|
69
84
|
}),
|
|
70
85
|
),
|
|
@@ -72,6 +87,8 @@ export const makeEndingPullStream = (
|
|
|
72
87
|
)
|
|
73
88
|
}).pipe(
|
|
74
89
|
Stream.unwrap,
|
|
75
|
-
Stream.mapError((cause) =>
|
|
90
|
+
Stream.mapError((cause) =>
|
|
91
|
+
cause._tag === 'BackendIdMismatchError' || cause._tag === 'UnknownError' ? cause : new UnknownError({ cause }),
|
|
92
|
+
),
|
|
76
93
|
Stream.withSpan('cloudflare-provider:pull'),
|
|
77
94
|
)
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
import { Vitest } from '@livestore/utils-dev/node-vitest'
|
|
2
|
+
import { Deferred, Duration, Effect, Fiber, Semaphore } from '@livestore/utils/effect'
|
|
3
|
+
|
|
4
|
+
import { runSerializedPushAdmission } from './push.ts'
|
|
5
|
+
|
|
6
|
+
Vitest.describe('sync-cf push admission', () => {
|
|
7
|
+
Vitest.live('publishes an admitted push before honoring interruption', () =>
|
|
8
|
+
Effect.gen(function* () {
|
|
9
|
+
const semaphore = yield* Semaphore.make(1)
|
|
10
|
+
const persisted = yield* Deferred.make<void>()
|
|
11
|
+
const allowBroadcast = yield* Deferred.make<void>()
|
|
12
|
+
const broadcasted = yield* Deferred.make<void>()
|
|
13
|
+
|
|
14
|
+
const admissionFiber = yield* runSerializedPushAdmission(
|
|
15
|
+
semaphore,
|
|
16
|
+
Effect.gen(function* () {
|
|
17
|
+
yield* Deferred.succeed(persisted, undefined)
|
|
18
|
+
yield* Deferred.await(allowBroadcast)
|
|
19
|
+
yield* Deferred.succeed(broadcasted, undefined)
|
|
20
|
+
}),
|
|
21
|
+
).pipe(Effect.forkChild)
|
|
22
|
+
|
|
23
|
+
// Reproduce cancellation after persistence but before the pull broadcast handoff.
|
|
24
|
+
yield* Deferred.await(persisted)
|
|
25
|
+
const interruptStarted = yield* Deferred.make<void>()
|
|
26
|
+
const interruptFiber = yield* Effect.gen(function* () {
|
|
27
|
+
yield* Deferred.succeed(interruptStarted, undefined)
|
|
28
|
+
yield* Fiber.interrupt(admissionFiber)
|
|
29
|
+
}).pipe(Effect.forkChild)
|
|
30
|
+
yield* Deferred.await(interruptStarted)
|
|
31
|
+
yield* Effect.yieldNow
|
|
32
|
+
|
|
33
|
+
yield* Deferred.succeed(allowBroadcast, undefined)
|
|
34
|
+
yield* Deferred.await(broadcasted).pipe(Effect.timeout(Duration.seconds(1)))
|
|
35
|
+
yield* Fiber.join(interruptFiber)
|
|
36
|
+
|
|
37
|
+
Vitest.expect(yield* Deferred.isDone(broadcasted)).toBe(true)
|
|
38
|
+
}),
|
|
39
|
+
)
|
|
40
|
+
})
|
package/src/cf-worker/do/push.ts
CHANGED
|
@@ -1,36 +1,38 @@
|
|
|
1
|
-
import {
|
|
2
|
-
BackendIdMismatchError,
|
|
3
|
-
InvalidPushError,
|
|
4
|
-
ServerAheadError,
|
|
5
|
-
SyncBackend,
|
|
6
|
-
UnexpectedError,
|
|
7
|
-
} from '@livestore/common'
|
|
1
|
+
import { BackendIdMismatchError, ServerAheadError, SyncBackend, UnknownError } from '@livestore/common'
|
|
8
2
|
import { type CfTypes, emitStreamResponse } from '@livestore/common-cf'
|
|
9
|
-
import {
|
|
3
|
+
import { splitArrayBySize } from '@livestore/common/sync'
|
|
4
|
+
import { Effect, Option, ReadonlyArray as EffectArray, type RpcMessage, Schema } from '@livestore/utils/effect'
|
|
5
|
+
|
|
6
|
+
import { MAX_PUSH_EVENTS_PER_REQUEST, MAX_WS_MESSAGE_BYTES } from '../../common/constants.ts'
|
|
10
7
|
import { SyncMessage } from '../../common/mod.ts'
|
|
11
8
|
import {
|
|
12
9
|
type Env,
|
|
13
|
-
|
|
14
|
-
MAX_WS_MESSAGE_BYTES,
|
|
10
|
+
type ForwardedHeaders,
|
|
15
11
|
type MakeDurableObjectClassOptions,
|
|
12
|
+
rpcSubscriptionKeyPrefix,
|
|
13
|
+
type RpcSubscription,
|
|
16
14
|
type StoreId,
|
|
17
15
|
WebSocketAttachmentSchema,
|
|
18
16
|
} from '../shared.ts'
|
|
19
|
-
import
|
|
20
|
-
import { splitChunkBySize } from './ws-chunking.ts'
|
|
17
|
+
import * as DoCtx from './layer.ts'
|
|
21
18
|
|
|
22
|
-
const
|
|
19
|
+
const pullResponseJsonSchema = Schema.toCodecJson(SyncMessage.PullResponse)
|
|
20
|
+
const encodePullResponse = Schema.encodeSync(pullResponseJsonSchema)
|
|
21
|
+
const jsonStringify = Schema.encodeSync(Schema.fromJsonString(Schema.Unknown))
|
|
23
22
|
type PullBatchItem = SyncMessage.PullResponse['batch'][number]
|
|
23
|
+
type PushBatchItem = SyncMessage.PushRequest['batch'][number]
|
|
24
24
|
|
|
25
25
|
export const makePush =
|
|
26
26
|
({
|
|
27
27
|
payload,
|
|
28
|
+
headers,
|
|
28
29
|
options,
|
|
29
30
|
storeId,
|
|
30
31
|
ctx,
|
|
31
32
|
env,
|
|
32
33
|
}: {
|
|
33
|
-
payload: Schema.
|
|
34
|
+
payload: Schema.Json | undefined
|
|
35
|
+
headers: ForwardedHeaders | undefined
|
|
34
36
|
options: MakeDurableObjectClassOptions | undefined
|
|
35
37
|
storeId: StoreId
|
|
36
38
|
ctx: CfTypes.DurableObjectState
|
|
@@ -39,59 +41,64 @@ export const makePush =
|
|
|
39
41
|
(pushRequest: Omit<SyncMessage.PushRequest, '_tag'>) =>
|
|
40
42
|
Effect.gen(function* () {
|
|
41
43
|
// yield* Effect.log(`Pushing ${decodedMessage.batch.length} events`, decodedMessage.batch)
|
|
42
|
-
const { backendId, storage, currentHeadRef, updateCurrentHead,
|
|
44
|
+
const { backendId, storage, currentHeadRef, updateCurrentHead, pushSemaphore } = yield* DoCtx.DoCtx
|
|
43
45
|
|
|
44
46
|
if (pushRequest.batch.length === 0) {
|
|
45
47
|
return SyncMessage.PushAck.make({})
|
|
46
48
|
}
|
|
47
49
|
|
|
48
|
-
if (options?.onPush) {
|
|
49
|
-
yield* Effect.
|
|
50
|
-
|
|
51
|
-
|
|
50
|
+
if (options?.onPush !== undefined) {
|
|
51
|
+
yield* Effect.trySyncOrPromiseOrEffect(() =>
|
|
52
|
+
options.onPush!(pushRequest, {
|
|
53
|
+
storeId,
|
|
54
|
+
...(payload !== undefined ? { payload } : {}),
|
|
55
|
+
...(headers !== undefined ? { headers } : {}),
|
|
56
|
+
}),
|
|
57
|
+
).pipe(UnknownError.mapToUnknownError)
|
|
52
58
|
}
|
|
53
59
|
|
|
54
60
|
if (pushRequest.backendId._tag === 'Some' && pushRequest.backendId.value !== backendId) {
|
|
55
61
|
return yield* new BackendIdMismatchError({ expected: backendId, received: pushRequest.backendId.value })
|
|
56
62
|
}
|
|
57
63
|
|
|
58
|
-
//
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
//
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
64
|
+
// A push is one ordered admit → persist → publish operation. `blockConcurrencyWhile` below protects
|
|
65
|
+
// the Durable Object state transition, while this semaphore keeps later pushes from overtaking this
|
|
66
|
+
// push during broadcasting without holding Cloudflare's concurrency gate across network I/O.
|
|
67
|
+
yield* runSerializedPushAdmission(
|
|
68
|
+
pushSemaphore,
|
|
69
|
+
Effect.gen(function* () {
|
|
70
|
+
// Keep the head check, durable append, and in-memory head update atomic with respect to other
|
|
71
|
+
// Durable Object requests. Broadcasting intentionally happens after this gate is released.
|
|
72
|
+
const { createdAt } = yield* Effect.gen(function* () {
|
|
73
|
+
const currentHead = currentHeadRef.current
|
|
74
|
+
// TODO handle clientId unique conflict
|
|
75
|
+
// Validate the batch
|
|
76
|
+
const firstEventParent = pushRequest.batch[0]!.parentSeqNum
|
|
77
|
+
if (firstEventParent !== currentHead) {
|
|
78
|
+
return yield* new ServerAheadError({ minimumExpectedNum: currentHead, providedNum: firstEventParent })
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
const createdAt = new Date().toISOString()
|
|
82
|
+
|
|
83
|
+
// TODO possibly model this as a queue in order to speed up subsequent pushes
|
|
84
|
+
yield* storage.appendEvents(pushRequest.batch, createdAt)
|
|
85
|
+
|
|
86
|
+
updateCurrentHead(pushRequest.batch.at(-1)!.seqNum)
|
|
87
|
+
|
|
88
|
+
return { createdAt }
|
|
89
|
+
}).pipe(blockConcurrencyWhile(ctx))
|
|
90
|
+
|
|
91
|
+
const connectedClients = ctx.getWebSockets()
|
|
92
|
+
|
|
93
|
+
// Preparing chunks of responses to make sure we don't exceed the WS message size limit.
|
|
94
|
+
if (EffectArray.isReadonlyArrayNonEmpty(pushRequest.batch) === false) {
|
|
95
|
+
return
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
const responses = yield* splitArrayBySize({
|
|
99
|
+
maxItems: MAX_PUSH_EVENTS_PER_REQUEST,
|
|
93
100
|
maxBytes: MAX_WS_MESSAGE_BYTES,
|
|
94
|
-
encode: (items) =>
|
|
101
|
+
encode: (items: ReadonlyArray<PushBatchItem>) =>
|
|
95
102
|
encodePullResponse(
|
|
96
103
|
SyncMessage.PullResponse.make({
|
|
97
104
|
batch: items.map(
|
|
@@ -104,105 +111,136 @@ export const makePush =
|
|
|
104
111
|
backendId,
|
|
105
112
|
}),
|
|
106
113
|
),
|
|
107
|
-
})
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
return {
|
|
121
|
-
response,
|
|
122
|
-
encoded: Schema.encodeSync(SyncMessage.PullResponse)(response),
|
|
123
|
-
}
|
|
124
|
-
}),
|
|
125
|
-
)
|
|
126
|
-
|
|
127
|
-
// Dual broadcasting: WebSocket + RPC clients
|
|
114
|
+
})(pushRequest.batch).pipe(
|
|
115
|
+
Effect.map((eventBatch) =>
|
|
116
|
+
eventBatch.map((events) => {
|
|
117
|
+
const batchWithMetadata = events.map((eventEncoded) => ({
|
|
118
|
+
eventEncoded,
|
|
119
|
+
metadata: Option.some(SyncMessage.SyncMetadata.make({ createdAt })),
|
|
120
|
+
}))
|
|
121
|
+
|
|
122
|
+
const response = SyncMessage.PullResponse.make({
|
|
123
|
+
batch: batchWithMetadata,
|
|
124
|
+
pageInfo: SyncBackend.pageInfoNoMore,
|
|
125
|
+
backendId,
|
|
126
|
+
})
|
|
128
127
|
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
128
|
+
return {
|
|
129
|
+
response,
|
|
130
|
+
encoded: encodePullResponse(response),
|
|
131
|
+
}
|
|
132
|
+
}),
|
|
133
|
+
),
|
|
134
|
+
)
|
|
135
|
+
|
|
136
|
+
// Dual broadcasting: WebSocket + RPC clients
|
|
137
|
+
|
|
138
|
+
// Broadcast to WebSocket clients
|
|
139
|
+
if (connectedClients.length > 0) {
|
|
140
|
+
for (const { response, encoded } of responses) {
|
|
141
|
+
// Only calling once for now.
|
|
142
|
+
if (options?.onPullRes !== undefined) {
|
|
143
|
+
yield* Effect.trySyncOrPromiseOrEffect(() => options.onPullRes!(response)).pipe(
|
|
144
|
+
UnknownError.mapToUnknownError,
|
|
145
|
+
)
|
|
146
|
+
}
|
|
136
147
|
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
148
|
+
// NOTE we're also sending the pullRes chunk to the pushing ws client as confirmation
|
|
149
|
+
for (const conn of connectedClients) {
|
|
150
|
+
const attachment = yield* Schema.decodeEffect(WebSocketAttachmentSchema)(conn.deserializeAttachment())
|
|
151
|
+
|
|
152
|
+
// We're doing something a bit "advanced" here as we're directly emitting Effect RPC-compatible
|
|
153
|
+
// response messsages on the Effect RPC-managed websocket connection to the WS client.
|
|
154
|
+
// For this we need to get the RPC `requestId` from the WebSocket attachment.
|
|
155
|
+
for (const requestId of attachment.pullRequestIds) {
|
|
156
|
+
const res: RpcMessage.ResponseChunkEncoded = {
|
|
157
|
+
_tag: 'Chunk',
|
|
158
|
+
requestId,
|
|
159
|
+
values: [encoded],
|
|
160
|
+
}
|
|
161
|
+
conn.send(jsonStringify(res))
|
|
149
162
|
}
|
|
150
|
-
conn.send(JSON.stringify(res))
|
|
151
163
|
}
|
|
152
164
|
}
|
|
165
|
+
|
|
166
|
+
yield* Effect.logDebug(`Broadcasted to ${connectedClients.length} WebSocket clients`)
|
|
153
167
|
}
|
|
154
168
|
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
for (const
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
169
|
+
// Subscribers live in the DO's KV storage (single source of truth), so fan-out survives reconstruction.
|
|
170
|
+
// emitStreamResponse reconstructs each client stub from its callerContext.
|
|
171
|
+
const rpcSubscriptions = Array.from(
|
|
172
|
+
ctx.storage.kv.list<RpcSubscription>({ prefix: rpcSubscriptionKeyPrefix }),
|
|
173
|
+
)
|
|
174
|
+
if (rpcSubscriptions.length > 0) {
|
|
175
|
+
for (const [, subscription] of rpcSubscriptions) {
|
|
176
|
+
for (const { encoded } of responses) {
|
|
177
|
+
yield* emitStreamResponse({
|
|
178
|
+
callerContext: subscription.callerContext,
|
|
179
|
+
env,
|
|
180
|
+
requestId: subscription.requestId,
|
|
181
|
+
storeId: subscription.storeId,
|
|
182
|
+
values: [encoded],
|
|
183
|
+
}).pipe(Effect.tapCauseLogPretty, Effect.exit)
|
|
184
|
+
}
|
|
168
185
|
}
|
|
169
|
-
}
|
|
170
186
|
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
Effect.tapCauseLogPretty,
|
|
175
|
-
Effect.withSpan('push-rpc-broadcast'),
|
|
176
|
-
Effect.uninterruptible, // We need to make sure Effect RPC doesn't interrupt this fiber
|
|
177
|
-
Effect.fork,
|
|
187
|
+
yield* Effect.logDebug(`Broadcasted to ${rpcSubscriptions.length} RPC clients`)
|
|
188
|
+
}
|
|
189
|
+
}).pipe(Effect.tapCauseLogPretty, Effect.withSpan('push-rpc-broadcast')),
|
|
178
190
|
)
|
|
179
191
|
|
|
180
|
-
//
|
|
181
|
-
|
|
182
|
-
|
|
192
|
+
// Acknowledge only after the committed batch has been published, so a client cannot observe an
|
|
193
|
+
// advanced server head while the corresponding pull response is still waiting in a detached fiber.
|
|
183
194
|
return SyncMessage.PushAck.make({})
|
|
184
195
|
}).pipe(
|
|
185
196
|
Effect.tap(
|
|
186
197
|
Effect.fn(function* (message) {
|
|
187
|
-
if (options?.onPushRes) {
|
|
188
|
-
yield* Effect.
|
|
198
|
+
if (options?.onPushRes !== undefined) {
|
|
199
|
+
yield* Effect.trySyncOrPromiseOrEffect(() => options.onPushRes!(message)).pipe(
|
|
200
|
+
UnknownError.mapToUnknownError,
|
|
201
|
+
)
|
|
189
202
|
}
|
|
190
203
|
}),
|
|
191
204
|
),
|
|
192
|
-
Effect.mapError((cause) =>
|
|
205
|
+
Effect.mapError((cause) =>
|
|
206
|
+
cause._tag === 'BackendIdMismatchError' || cause._tag === 'ServerAheadError' || cause._tag === 'UnknownError'
|
|
207
|
+
? cause
|
|
208
|
+
: new UnknownError({ cause }),
|
|
209
|
+
),
|
|
193
210
|
Effect.withSpan('sync-cf:do:push', { attributes: { storeId, batchSize: pushRequest.batch.length } }),
|
|
194
211
|
)
|
|
195
212
|
|
|
196
213
|
/**
|
|
214
|
+
* Serializes the complete push admission lifecycle, including publication to pull subscribers.
|
|
215
|
+
*
|
|
216
|
+
* This complements `blockConcurrencyWhile`: that Cloudflare primitive makes the durable state transition
|
|
217
|
+
* atomic with respect to other Durable Object requests, whereas this semaphore preserves push order after
|
|
218
|
+
* the storage gate is released and broadcasting begins. Keeping broadcast outside `blockConcurrencyWhile`
|
|
219
|
+
* avoids holding Cloudflare's concurrency gate across network I/O.
|
|
220
|
+
*
|
|
221
|
+
* Once admitted, the effect is uninterruptible so cancellation cannot leave a committed head without its
|
|
222
|
+
* matching pull response. Such a gap would make later pushers receive `ServerAhead` while waiting for a
|
|
223
|
+
* response that was never published.
|
|
224
|
+
*/
|
|
225
|
+
export const runSerializedPushAdmission = <A, E, R>(
|
|
226
|
+
semaphore: DoCtx.Service['pushSemaphore'],
|
|
227
|
+
admission: Effect.Effect<A, E, R>,
|
|
228
|
+
): Effect.Effect<A, E, R> => semaphore.withPermits(1)(admission.pipe(Effect.uninterruptible))
|
|
229
|
+
|
|
230
|
+
/**
|
|
231
|
+
* Runs the storage-backed state transition behind Cloudflare's Durable Object concurrency gate.
|
|
232
|
+
* This scope should remain limited to local persistence and head mutation; ordered publication is handled
|
|
233
|
+
* separately by `runSerializedPushAdmission` so outbound work does not block unrelated Durable Object events.
|
|
234
|
+
*
|
|
197
235
|
* @see https://developers.cloudflare.com/durable-objects/api/state/#blockconcurrencywhile
|
|
198
236
|
*/
|
|
199
237
|
const blockConcurrencyWhile =
|
|
200
238
|
(ctx: CfTypes.DurableObjectState) =>
|
|
201
239
|
<A, E, R>(eff: Effect.Effect<A, E, R>) =>
|
|
202
240
|
Effect.gen(function* () {
|
|
203
|
-
const
|
|
241
|
+
const services = yield* Effect.context<R>()
|
|
204
242
|
const exit = yield* Effect.promise(() =>
|
|
205
|
-
ctx.blockConcurrencyWhile(() => eff.pipe(Effect.
|
|
243
|
+
ctx.blockConcurrencyWhile(() => eff.pipe(Effect.runPromiseExitWith(services))),
|
|
206
244
|
)
|
|
207
245
|
|
|
208
246
|
return yield* exit
|
|
@@ -1,15 +1,21 @@
|
|
|
1
1
|
import { EventSequenceNumber, State } from '@livestore/common/schema'
|
|
2
2
|
import { Schema } from '@livestore/utils/effect'
|
|
3
|
+
|
|
3
4
|
import { PERSISTENCE_FORMAT_VERSION } from '../shared.ts'
|
|
4
5
|
|
|
6
|
+
/**
|
|
7
|
+
* Main event log table storing all LiveStore events.
|
|
8
|
+
*
|
|
9
|
+
* ⚠️ IMPORTANT: Any changes to this schema require bumping PERSISTENCE_FORMAT_VERSION in shared.ts
|
|
10
|
+
*/
|
|
5
11
|
export const eventlogTable = State.SQLite.table({
|
|
6
12
|
// NOTE actual table name is determined at runtime to use proper storeId
|
|
7
13
|
name: `eventlog_${PERSISTENCE_FORMAT_VERSION}_$storeId`,
|
|
8
14
|
columns: {
|
|
9
|
-
seqNum: State.SQLite.integer({ primaryKey: true, schema: EventSequenceNumber.
|
|
10
|
-
parentSeqNum: State.SQLite.integer({ schema: EventSequenceNumber.
|
|
15
|
+
seqNum: State.SQLite.integer({ primaryKey: true, schema: EventSequenceNumber.Global.Schema }),
|
|
16
|
+
parentSeqNum: State.SQLite.integer({ schema: EventSequenceNumber.Global.Schema }),
|
|
11
17
|
name: State.SQLite.text({}),
|
|
12
|
-
args: State.SQLite.text({ schema: Schema.
|
|
18
|
+
args: State.SQLite.text({ schema: Schema.fromJsonString(Schema.Any), nullable: true }),
|
|
13
19
|
/** ISO date format. Currently only used for debugging purposes. */
|
|
14
20
|
createdAt: State.SQLite.text({}),
|
|
15
21
|
clientId: State.SQLite.text({}),
|
|
@@ -17,12 +23,16 @@ export const eventlogTable = State.SQLite.table({
|
|
|
17
23
|
},
|
|
18
24
|
})
|
|
19
25
|
|
|
20
|
-
/**
|
|
26
|
+
/**
|
|
27
|
+
* Context metadata table - one row per Durable Object.
|
|
28
|
+
*
|
|
29
|
+
* ⚠️ IMPORTANT: Any changes to this schema require bumping PERSISTENCE_FORMAT_VERSION in shared.ts
|
|
30
|
+
*/
|
|
21
31
|
export const contextTable = State.SQLite.table({
|
|
22
32
|
name: `context_${PERSISTENCE_FORMAT_VERSION}`,
|
|
23
33
|
columns: {
|
|
24
34
|
storeId: State.SQLite.text({ primaryKey: true }),
|
|
25
|
-
currentHead: State.SQLite.integer({ schema: EventSequenceNumber.
|
|
35
|
+
currentHead: State.SQLite.integer({ schema: EventSequenceNumber.Global.Schema }),
|
|
26
36
|
backendId: State.SQLite.text({}),
|
|
27
37
|
},
|
|
28
38
|
})
|