@livestore/sync-cf 0.4.0-dev.9 → 0.5.0-dev.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (113) hide show
  1. package/README.md +7 -8
  2. package/dist/.tsbuildinfo +1 -1
  3. package/dist/cf-worker/do/durable-object.d.ts +1 -1
  4. package/dist/cf-worker/do/durable-object.d.ts.map +1 -1
  5. package/dist/cf-worker/do/durable-object.js +36 -18
  6. package/dist/cf-worker/do/durable-object.js.map +1 -1
  7. package/dist/cf-worker/do/layer.d.ts +24 -20
  8. package/dist/cf-worker/do/layer.d.ts.map +1 -1
  9. package/dist/cf-worker/do/layer.js +57 -24
  10. package/dist/cf-worker/do/layer.js.map +1 -1
  11. package/dist/cf-worker/do/pull.d.ts +8 -3
  12. package/dist/cf-worker/do/pull.d.ts.map +1 -1
  13. package/dist/cf-worker/do/pull.js +26 -21
  14. package/dist/cf-worker/do/pull.js.map +1 -1
  15. package/dist/cf-worker/do/push.d.ts +20 -6
  16. package/dist/cf-worker/do/push.d.ts.map +1 -1
  17. package/dist/cf-worker/do/push.js +84 -56
  18. package/dist/cf-worker/do/push.js.map +1 -1
  19. package/dist/cf-worker/do/push.test.d.ts +2 -0
  20. package/dist/cf-worker/do/push.test.d.ts.map +1 -0
  21. package/dist/cf-worker/do/push.test.js +30 -0
  22. package/dist/cf-worker/do/push.test.js.map +1 -0
  23. package/dist/cf-worker/do/sqlite.d.ts +49 -52
  24. package/dist/cf-worker/do/sqlite.d.ts.map +1 -1
  25. package/dist/cf-worker/do/sqlite.js +14 -5
  26. package/dist/cf-worker/do/sqlite.js.map +1 -1
  27. package/dist/cf-worker/do/sync-storage.d.ts +15 -9
  28. package/dist/cf-worker/do/sync-storage.d.ts.map +1 -1
  29. package/dist/cf-worker/do/sync-storage.js +111 -25
  30. package/dist/cf-worker/do/sync-storage.js.map +1 -1
  31. package/dist/cf-worker/do/transport/do-rpc-server.d.ts +2 -2
  32. package/dist/cf-worker/do/transport/do-rpc-server.d.ts.map +1 -1
  33. package/dist/cf-worker/do/transport/do-rpc-server.js +31 -23
  34. package/dist/cf-worker/do/transport/do-rpc-server.js.map +1 -1
  35. package/dist/cf-worker/do/transport/http-rpc-server.d.ts +5 -3
  36. package/dist/cf-worker/do/transport/http-rpc-server.d.ts.map +1 -1
  37. package/dist/cf-worker/do/transport/http-rpc-server.js +28 -19
  38. package/dist/cf-worker/do/transport/http-rpc-server.js.map +1 -1
  39. package/dist/cf-worker/do/transport/ws-rpc-server.d.ts +3 -2
  40. package/dist/cf-worker/do/transport/ws-rpc-server.d.ts.map +1 -1
  41. package/dist/cf-worker/do/transport/ws-rpc-server.js +32 -10
  42. package/dist/cf-worker/do/transport/ws-rpc-server.js.map +1 -1
  43. package/dist/cf-worker/shared.d.ts +124 -34
  44. package/dist/cf-worker/shared.d.ts.map +1 -1
  45. package/dist/cf-worker/shared.js +52 -17
  46. package/dist/cf-worker/shared.js.map +1 -1
  47. package/dist/cf-worker/worker.d.ts +47 -39
  48. package/dist/cf-worker/worker.d.ts.map +1 -1
  49. package/dist/cf-worker/worker.js +54 -35
  50. package/dist/cf-worker/worker.js.map +1 -1
  51. package/dist/client/transport/do-rpc-client.d.ts +13 -4
  52. package/dist/client/transport/do-rpc-client.d.ts.map +1 -1
  53. package/dist/client/transport/do-rpc-client.js +74 -28
  54. package/dist/client/transport/do-rpc-client.js.map +1 -1
  55. package/dist/client/transport/http-rpc-client.d.ts +3 -3
  56. package/dist/client/transport/http-rpc-client.d.ts.map +1 -1
  57. package/dist/client/transport/http-rpc-client.js +51 -22
  58. package/dist/client/transport/http-rpc-client.js.map +1 -1
  59. package/dist/client/transport/ws-rpc-client.d.ts +4 -3
  60. package/dist/client/transport/ws-rpc-client.d.ts.map +1 -1
  61. package/dist/client/transport/ws-rpc-client.js +40 -22
  62. package/dist/client/transport/ws-rpc-client.js.map +1 -1
  63. package/dist/common/constants.d.ts +7 -0
  64. package/dist/common/constants.d.ts.map +1 -0
  65. package/dist/common/constants.js +17 -0
  66. package/dist/common/constants.js.map +1 -0
  67. package/dist/common/do-rpc-schema.d.ts +42 -43
  68. package/dist/common/do-rpc-schema.d.ts.map +1 -1
  69. package/dist/common/do-rpc-schema.js +5 -5
  70. package/dist/common/do-rpc-schema.js.map +1 -1
  71. package/dist/common/http-rpc-schema.d.ts +37 -38
  72. package/dist/common/http-rpc-schema.d.ts.map +1 -1
  73. package/dist/common/http-rpc-schema.js +7 -7
  74. package/dist/common/http-rpc-schema.js.map +1 -1
  75. package/dist/common/mod.d.ts +5 -3
  76. package/dist/common/mod.d.ts.map +1 -1
  77. package/dist/common/mod.js +4 -2
  78. package/dist/common/mod.js.map +1 -1
  79. package/dist/common/sync-message-types.d.ts +113 -130
  80. package/dist/common/sync-message-types.d.ts.map +1 -1
  81. package/dist/common/sync-message-types.js +29 -17
  82. package/dist/common/sync-message-types.js.map +1 -1
  83. package/dist/common/ws-rpc-schema.d.ts +34 -35
  84. package/dist/common/ws-rpc-schema.d.ts.map +1 -1
  85. package/dist/common/ws-rpc-schema.js +5 -5
  86. package/dist/common/ws-rpc-schema.js.map +1 -1
  87. package/package.json +57 -15
  88. package/src/cf-worker/do/durable-object.ts +59 -41
  89. package/src/cf-worker/do/layer.ts +153 -88
  90. package/src/cf-worker/do/pull.ts +48 -31
  91. package/src/cf-worker/do/push.test.ts +40 -0
  92. package/src/cf-worker/do/push.ts +163 -125
  93. package/src/cf-worker/do/sqlite.ts +15 -5
  94. package/src/cf-worker/do/sync-storage.ts +171 -38
  95. package/src/cf-worker/do/transport/do-rpc-server.ts +40 -40
  96. package/src/cf-worker/do/transport/http-rpc-server.ts +37 -20
  97. package/src/cf-worker/do/transport/ws-rpc-server.ts +44 -16
  98. package/src/cf-worker/shared.ts +152 -36
  99. package/src/cf-worker/worker.ts +110 -58
  100. package/src/client/transport/do-rpc-client.ts +106 -40
  101. package/src/client/transport/http-rpc-client.ts +74 -38
  102. package/src/client/transport/ws-rpc-client.ts +55 -29
  103. package/src/common/constants.ts +18 -0
  104. package/src/common/do-rpc-schema.ts +6 -5
  105. package/src/common/http-rpc-schema.ts +8 -7
  106. package/src/common/mod.ts +4 -3
  107. package/src/common/sync-message-types.ts +19 -19
  108. package/src/common/ws-rpc-schema.ts +6 -5
  109. package/dist/cf-worker/do/ws-chunking.d.ts +0 -22
  110. package/dist/cf-worker/do/ws-chunking.d.ts.map +0 -1
  111. package/dist/cf-worker/do/ws-chunking.js +0 -49
  112. package/dist/cf-worker/do/ws-chunking.js.map +0 -1
  113. package/src/cf-worker/do/ws-chunking.ts +0 -76
@@ -1,9 +1,11 @@
1
- import { BackendIdMismatchError, InvalidPullError, SyncBackend, UnexpectedError } from '@livestore/common'
2
- import { Chunk, Effect, Option, Schema, Stream } from '@livestore/utils/effect'
1
+ import { BackendIdMismatchError, SyncBackend, UnknownError } from '@livestore/common'
2
+ import { splitArrayBySize } from '@livestore/common/sync'
3
+ import { Effect, Option, Schema, Stream } from '@livestore/utils/effect'
4
+
5
+ import { MAX_PULL_EVENTS_PER_MESSAGE, MAX_WS_MESSAGE_BYTES } from '../../common/constants.ts'
3
6
  import { SyncMessage } from '../../common/mod.ts'
4
- import { MAX_PULL_EVENTS_PER_MESSAGE, MAX_WS_MESSAGE_BYTES } from '../shared.ts'
5
- import { DoCtx } from './layer.ts'
6
- import { splitChunkBySize } from './ws-chunking.ts'
7
+ import type { ForwardedHeaders } from '../shared.ts'
8
+ import * as DoCtx from './layer.ts'
7
9
 
8
10
  const encodePullResponse = Schema.encodeSync(SyncMessage.PullResponse)
9
11
 
@@ -16,17 +18,26 @@ const encodePullResponse = Schema.encodeSync(SyncMessage.PullResponse)
16
18
  // DO RPC:
17
19
  // - Further chunks will be emitted manually in `push.ts`
18
20
  // - If the client sends a `Interrupt` RPC message, TODO
19
- export const makeEndingPullStream = (
20
- req: SyncMessage.PullRequest,
21
- payload: Schema.JsonValue | undefined,
22
- ): Stream.Stream<SyncMessage.PullResponse, InvalidPullError, DoCtx> =>
21
+ export const makeEndingPullStream = ({
22
+ req,
23
+ payload,
24
+ headers,
25
+ }: {
26
+ req: SyncMessage.PullRequest
27
+ payload: Schema.Json | undefined
28
+ headers: ForwardedHeaders | undefined
29
+ }): Stream.Stream<SyncMessage.PullResponse, UnknownError | BackendIdMismatchError, DoCtx.DoCtx> =>
23
30
  Effect.gen(function* () {
24
- const { doOptions, backendId, storeId, storage } = yield* DoCtx
31
+ const { doOptions, backendId, storeId, storage } = yield* DoCtx.DoCtx
25
32
 
26
- if (doOptions?.onPull) {
27
- yield* Effect.tryAll(() => doOptions!.onPull!(req, { storeId, payload })).pipe(
28
- UnexpectedError.mapToUnexpectedError,
29
- )
33
+ if (doOptions?.onPull !== undefined) {
34
+ yield* Effect.trySyncOrPromiseOrEffect(() =>
35
+ doOptions.onPull!(req, {
36
+ storeId,
37
+ ...(payload !== undefined ? { payload } : {}),
38
+ ...(headers !== undefined ? { headers } : {}),
39
+ }),
40
+ ).pipe(UnknownError.mapToUnknownError)
30
41
  }
31
42
 
32
43
  if (req.cursor._tag === 'Some' && req.cursor.value.backendId !== backendId) {
@@ -38,8 +49,8 @@ export const makeEndingPullStream = (
38
49
  )
39
50
 
40
51
  return storedEvents.pipe(
41
- Stream.mapChunks(
42
- splitChunkBySize({
52
+ Stream.mapArrayEffect(
53
+ splitArrayBySize({
43
54
  maxItems: MAX_PULL_EVENTS_PER_MESSAGE,
44
55
  maxBytes: MAX_WS_MESSAGE_BYTES,
45
56
  encode: (batch) =>
@@ -48,23 +59,27 @@ export const makeEndingPullStream = (
48
59
  ),
49
60
  }),
50
61
  ),
51
- Stream.mapAccum(total, (remaining, chunk) => {
52
- const asArray = Chunk.toReadonlyArray(chunk)
53
- const nextRemaining = Math.max(0, remaining - asArray.length)
62
+ Stream.mapAccum(
63
+ () => total,
64
+ (remaining, array) => {
65
+ const nextRemaining = Math.max(0, remaining - array.length)
54
66
 
55
- return [
56
- nextRemaining,
57
- SyncMessage.PullResponse.make({
58
- batch: asArray,
59
- pageInfo: nextRemaining > 0 ? SyncBackend.pageInfoMoreKnown(nextRemaining) : SyncBackend.pageInfoNoMore,
60
- backendId,
61
- }),
62
- ] as const
63
- }),
67
+ return [
68
+ nextRemaining,
69
+ [
70
+ SyncMessage.PullResponse.make({
71
+ batch: array,
72
+ pageInfo: nextRemaining > 0 ? SyncBackend.pageInfoMoreKnown(nextRemaining) : SyncBackend.pageInfoNoMore,
73
+ backendId,
74
+ }),
75
+ ],
76
+ ] as const
77
+ },
78
+ ),
64
79
  Stream.tap(
65
80
  Effect.fn(function* (res) {
66
- if (doOptions?.onPullRes) {
67
- yield* Effect.tryAll(() => doOptions.onPullRes!(res)).pipe(UnexpectedError.mapToUnexpectedError)
81
+ if (doOptions?.onPullRes !== undefined) {
82
+ yield* Effect.trySyncOrPromiseOrEffect(() => doOptions.onPullRes!(res)).pipe(UnknownError.mapToUnknownError)
68
83
  }
69
84
  }),
70
85
  ),
@@ -72,6 +87,8 @@ export const makeEndingPullStream = (
72
87
  )
73
88
  }).pipe(
74
89
  Stream.unwrap,
75
- Stream.mapError((cause) => InvalidPullError.make({ cause })),
90
+ Stream.mapError((cause) =>
91
+ cause._tag === 'BackendIdMismatchError' || cause._tag === 'UnknownError' ? cause : new UnknownError({ cause }),
92
+ ),
76
93
  Stream.withSpan('cloudflare-provider:pull'),
77
94
  )
@@ -0,0 +1,40 @@
1
+ import { Vitest } from '@livestore/utils-dev/node-vitest'
2
+ import { Deferred, Duration, Effect, Fiber, Semaphore } from '@livestore/utils/effect'
3
+
4
+ import { runSerializedPushAdmission } from './push.ts'
5
+
6
+ Vitest.describe('sync-cf push admission', () => {
7
+ Vitest.live('publishes an admitted push before honoring interruption', () =>
8
+ Effect.gen(function* () {
9
+ const semaphore = yield* Semaphore.make(1)
10
+ const persisted = yield* Deferred.make<void>()
11
+ const allowBroadcast = yield* Deferred.make<void>()
12
+ const broadcasted = yield* Deferred.make<void>()
13
+
14
+ const admissionFiber = yield* runSerializedPushAdmission(
15
+ semaphore,
16
+ Effect.gen(function* () {
17
+ yield* Deferred.succeed(persisted, undefined)
18
+ yield* Deferred.await(allowBroadcast)
19
+ yield* Deferred.succeed(broadcasted, undefined)
20
+ }),
21
+ ).pipe(Effect.forkChild)
22
+
23
+ // Reproduce cancellation after persistence but before the pull broadcast handoff.
24
+ yield* Deferred.await(persisted)
25
+ const interruptStarted = yield* Deferred.make<void>()
26
+ const interruptFiber = yield* Effect.gen(function* () {
27
+ yield* Deferred.succeed(interruptStarted, undefined)
28
+ yield* Fiber.interrupt(admissionFiber)
29
+ }).pipe(Effect.forkChild)
30
+ yield* Deferred.await(interruptStarted)
31
+ yield* Effect.yieldNow
32
+
33
+ yield* Deferred.succeed(allowBroadcast, undefined)
34
+ yield* Deferred.await(broadcasted).pipe(Effect.timeout(Duration.seconds(1)))
35
+ yield* Fiber.join(interruptFiber)
36
+
37
+ Vitest.expect(yield* Deferred.isDone(broadcasted)).toBe(true)
38
+ }),
39
+ )
40
+ })
@@ -1,36 +1,38 @@
1
- import {
2
- BackendIdMismatchError,
3
- InvalidPushError,
4
- ServerAheadError,
5
- SyncBackend,
6
- UnexpectedError,
7
- } from '@livestore/common'
1
+ import { BackendIdMismatchError, ServerAheadError, SyncBackend, UnknownError } from '@livestore/common'
8
2
  import { type CfTypes, emitStreamResponse } from '@livestore/common-cf'
9
- import { Chunk, Effect, Option, type RpcMessage, Schema } from '@livestore/utils/effect'
3
+ import { splitArrayBySize } from '@livestore/common/sync'
4
+ import { Effect, Option, ReadonlyArray as EffectArray, type RpcMessage, Schema } from '@livestore/utils/effect'
5
+
6
+ import { MAX_PUSH_EVENTS_PER_REQUEST, MAX_WS_MESSAGE_BYTES } from '../../common/constants.ts'
10
7
  import { SyncMessage } from '../../common/mod.ts'
11
8
  import {
12
9
  type Env,
13
- MAX_PULL_EVENTS_PER_MESSAGE,
14
- MAX_WS_MESSAGE_BYTES,
10
+ type ForwardedHeaders,
15
11
  type MakeDurableObjectClassOptions,
12
+ rpcSubscriptionKeyPrefix,
13
+ type RpcSubscription,
16
14
  type StoreId,
17
15
  WebSocketAttachmentSchema,
18
16
  } from '../shared.ts'
19
- import { DoCtx } from './layer.ts'
20
- import { splitChunkBySize } from './ws-chunking.ts'
17
+ import * as DoCtx from './layer.ts'
21
18
 
22
- const encodePullResponse = Schema.encodeSync(SyncMessage.PullResponse)
19
+ const pullResponseJsonSchema = Schema.toCodecJson(SyncMessage.PullResponse)
20
+ const encodePullResponse = Schema.encodeSync(pullResponseJsonSchema)
21
+ const jsonStringify = Schema.encodeSync(Schema.fromJsonString(Schema.Unknown))
23
22
  type PullBatchItem = SyncMessage.PullResponse['batch'][number]
23
+ type PushBatchItem = SyncMessage.PushRequest['batch'][number]
24
24
 
25
25
  export const makePush =
26
26
  ({
27
27
  payload,
28
+ headers,
28
29
  options,
29
30
  storeId,
30
31
  ctx,
31
32
  env,
32
33
  }: {
33
- payload: Schema.JsonValue | undefined
34
+ payload: Schema.Json | undefined
35
+ headers: ForwardedHeaders | undefined
34
36
  options: MakeDurableObjectClassOptions | undefined
35
37
  storeId: StoreId
36
38
  ctx: CfTypes.DurableObjectState
@@ -39,59 +41,64 @@ export const makePush =
39
41
  (pushRequest: Omit<SyncMessage.PushRequest, '_tag'>) =>
40
42
  Effect.gen(function* () {
41
43
  // yield* Effect.log(`Pushing ${decodedMessage.batch.length} events`, decodedMessage.batch)
42
- const { backendId, storage, currentHeadRef, updateCurrentHead, rpcSubscriptions } = yield* DoCtx
44
+ const { backendId, storage, currentHeadRef, updateCurrentHead, pushSemaphore } = yield* DoCtx.DoCtx
43
45
 
44
46
  if (pushRequest.batch.length === 0) {
45
47
  return SyncMessage.PushAck.make({})
46
48
  }
47
49
 
48
- if (options?.onPush) {
49
- yield* Effect.tryAll(() => options.onPush!(pushRequest, { storeId, payload })).pipe(
50
- UnexpectedError.mapToUnexpectedError,
51
- )
50
+ if (options?.onPush !== undefined) {
51
+ yield* Effect.trySyncOrPromiseOrEffect(() =>
52
+ options.onPush!(pushRequest, {
53
+ storeId,
54
+ ...(payload !== undefined ? { payload } : {}),
55
+ ...(headers !== undefined ? { headers } : {}),
56
+ }),
57
+ ).pipe(UnknownError.mapToUnknownError)
52
58
  }
53
59
 
54
60
  if (pushRequest.backendId._tag === 'Some' && pushRequest.backendId.value !== backendId) {
55
61
  return yield* new BackendIdMismatchError({ expected: backendId, received: pushRequest.backendId.value })
56
62
  }
57
63
 
58
- // This part of the code needs to run sequentially to avoid race conditions
59
- const { createdAt } = yield* Effect.gen(function* () {
60
- const currentHead = currentHeadRef.current
61
- // TODO handle clientId unique conflict
62
- // Validate the batch
63
- const firstEventParent = pushRequest.batch[0]!.parentSeqNum
64
- if (firstEventParent !== currentHead) {
65
- // yield* Effect.logDebug('ServerAheadError: backend head mismatch', {
66
- // expectedHead: currentHead,
67
- // providedHead: firstEventParent,
68
- // batchSize: pushRequest.batch.length,
69
- // backendId,
70
- // })
71
-
72
- return yield* new ServerAheadError({ minimumExpectedNum: currentHead, providedNum: firstEventParent })
73
- }
74
-
75
- const createdAt = new Date().toISOString()
76
-
77
- // TODO possibly model this as a queue in order to speed up subsequent pushes
78
- yield* storage.appendEvents(pushRequest.batch, createdAt)
79
-
80
- updateCurrentHead(pushRequest.batch.at(-1)!.seqNum)
81
-
82
- return { createdAt }
83
- }).pipe(blockConcurrencyWhile(ctx))
84
-
85
- // Run in background but already return the push ack to the client
86
- yield* Effect.gen(function* () {
87
- const connectedClients = ctx.getWebSockets()
88
-
89
- // Preparing chunks of responses to make sure we don't exceed the WS message size limit.
90
- const responses = Chunk.fromIterable(pushRequest.batch).pipe(
91
- splitChunkBySize({
92
- maxItems: MAX_PULL_EVENTS_PER_MESSAGE,
64
+ // A push is one ordered admit persist publish operation. `blockConcurrencyWhile` below protects
65
+ // the Durable Object state transition, while this semaphore keeps later pushes from overtaking this
66
+ // push during broadcasting without holding Cloudflare's concurrency gate across network I/O.
67
+ yield* runSerializedPushAdmission(
68
+ pushSemaphore,
69
+ Effect.gen(function* () {
70
+ // Keep the head check, durable append, and in-memory head update atomic with respect to other
71
+ // Durable Object requests. Broadcasting intentionally happens after this gate is released.
72
+ const { createdAt } = yield* Effect.gen(function* () {
73
+ const currentHead = currentHeadRef.current
74
+ // TODO handle clientId unique conflict
75
+ // Validate the batch
76
+ const firstEventParent = pushRequest.batch[0]!.parentSeqNum
77
+ if (firstEventParent !== currentHead) {
78
+ return yield* new ServerAheadError({ minimumExpectedNum: currentHead, providedNum: firstEventParent })
79
+ }
80
+
81
+ const createdAt = new Date().toISOString()
82
+
83
+ // TODO possibly model this as a queue in order to speed up subsequent pushes
84
+ yield* storage.appendEvents(pushRequest.batch, createdAt)
85
+
86
+ updateCurrentHead(pushRequest.batch.at(-1)!.seqNum)
87
+
88
+ return { createdAt }
89
+ }).pipe(blockConcurrencyWhile(ctx))
90
+
91
+ const connectedClients = ctx.getWebSockets()
92
+
93
+ // Preparing chunks of responses to make sure we don't exceed the WS message size limit.
94
+ if (EffectArray.isReadonlyArrayNonEmpty(pushRequest.batch) === false) {
95
+ return
96
+ }
97
+
98
+ const responses = yield* splitArrayBySize({
99
+ maxItems: MAX_PUSH_EVENTS_PER_REQUEST,
93
100
  maxBytes: MAX_WS_MESSAGE_BYTES,
94
- encode: (items) =>
101
+ encode: (items: ReadonlyArray<PushBatchItem>) =>
95
102
  encodePullResponse(
96
103
  SyncMessage.PullResponse.make({
97
104
  batch: items.map(
@@ -104,105 +111,136 @@ export const makePush =
104
111
  backendId,
105
112
  }),
106
113
  ),
107
- }),
108
- Chunk.map((eventsChunk) => {
109
- const batchWithMetadata = Chunk.toReadonlyArray(eventsChunk).map((eventEncoded) => ({
110
- eventEncoded,
111
- metadata: Option.some(SyncMessage.SyncMetadata.make({ createdAt })),
112
- }))
113
-
114
- const response = SyncMessage.PullResponse.make({
115
- batch: batchWithMetadata,
116
- pageInfo: SyncBackend.pageInfoNoMore,
117
- backendId,
118
- })
119
-
120
- return {
121
- response,
122
- encoded: Schema.encodeSync(SyncMessage.PullResponse)(response),
123
- }
124
- }),
125
- )
126
-
127
- // Dual broadcasting: WebSocket + RPC clients
114
+ })(pushRequest.batch).pipe(
115
+ Effect.map((eventBatch) =>
116
+ eventBatch.map((events) => {
117
+ const batchWithMetadata = events.map((eventEncoded) => ({
118
+ eventEncoded,
119
+ metadata: Option.some(SyncMessage.SyncMetadata.make({ createdAt })),
120
+ }))
121
+
122
+ const response = SyncMessage.PullResponse.make({
123
+ batch: batchWithMetadata,
124
+ pageInfo: SyncBackend.pageInfoNoMore,
125
+ backendId,
126
+ })
128
127
 
129
- // Broadcast to WebSocket clients
130
- if (connectedClients.length > 0) {
131
- for (const { response, encoded } of responses) {
132
- // Only calling once for now.
133
- if (options?.onPullRes) {
134
- yield* Effect.tryAll(() => options.onPullRes!(response)).pipe(UnexpectedError.mapToUnexpectedError)
135
- }
128
+ return {
129
+ response,
130
+ encoded: encodePullResponse(response),
131
+ }
132
+ }),
133
+ ),
134
+ )
135
+
136
+ // Dual broadcasting: WebSocket + RPC clients
137
+
138
+ // Broadcast to WebSocket clients
139
+ if (connectedClients.length > 0) {
140
+ for (const { response, encoded } of responses) {
141
+ // Only calling once for now.
142
+ if (options?.onPullRes !== undefined) {
143
+ yield* Effect.trySyncOrPromiseOrEffect(() => options.onPullRes!(response)).pipe(
144
+ UnknownError.mapToUnknownError,
145
+ )
146
+ }
136
147
 
137
- // NOTE we're also sending the pullRes chunk to the pushing ws client as confirmation
138
- for (const conn of connectedClients) {
139
- const attachment = Schema.decodeSync(WebSocketAttachmentSchema)(conn.deserializeAttachment())
140
-
141
- // We're doing something a bit "advanced" here as we're directly emitting Effect RPC-compatible
142
- // response messsages on the Effect RPC-managed websocket connection to the WS client.
143
- // For this we need to get the RPC `requestId` from the WebSocket attachment.
144
- for (const requestId of attachment.pullRequestIds) {
145
- const res: RpcMessage.ResponseChunkEncoded = {
146
- _tag: 'Chunk',
147
- requestId,
148
- values: [encoded],
148
+ // NOTE we're also sending the pullRes chunk to the pushing ws client as confirmation
149
+ for (const conn of connectedClients) {
150
+ const attachment = yield* Schema.decodeEffect(WebSocketAttachmentSchema)(conn.deserializeAttachment())
151
+
152
+ // We're doing something a bit "advanced" here as we're directly emitting Effect RPC-compatible
153
+ // response messsages on the Effect RPC-managed websocket connection to the WS client.
154
+ // For this we need to get the RPC `requestId` from the WebSocket attachment.
155
+ for (const requestId of attachment.pullRequestIds) {
156
+ const res: RpcMessage.ResponseChunkEncoded = {
157
+ _tag: 'Chunk',
158
+ requestId,
159
+ values: [encoded],
160
+ }
161
+ conn.send(jsonStringify(res))
149
162
  }
150
- conn.send(JSON.stringify(res))
151
163
  }
152
164
  }
165
+
166
+ yield* Effect.logDebug(`Broadcasted to ${connectedClients.length} WebSocket clients`)
153
167
  }
154
168
 
155
- yield* Effect.logDebug(`Broadcasted to ${connectedClients.length} WebSocket clients`)
156
- }
157
-
158
- // RPC broadcasting would require reconstructing client stubs from clientIds
159
- if (rpcSubscriptions.size > 0) {
160
- for (const subscription of rpcSubscriptions.values()) {
161
- for (const { encoded } of responses) {
162
- yield* emitStreamResponse({
163
- callerContext: subscription.callerContext,
164
- env,
165
- requestId: subscription.requestId,
166
- values: [encoded],
167
- }).pipe(Effect.tapCauseLogPretty, Effect.exit)
169
+ // Subscribers live in the DO's KV storage (single source of truth), so fan-out survives reconstruction.
170
+ // emitStreamResponse reconstructs each client stub from its callerContext.
171
+ const rpcSubscriptions = Array.from(
172
+ ctx.storage.kv.list<RpcSubscription>({ prefix: rpcSubscriptionKeyPrefix }),
173
+ )
174
+ if (rpcSubscriptions.length > 0) {
175
+ for (const [, subscription] of rpcSubscriptions) {
176
+ for (const { encoded } of responses) {
177
+ yield* emitStreamResponse({
178
+ callerContext: subscription.callerContext,
179
+ env,
180
+ requestId: subscription.requestId,
181
+ storeId: subscription.storeId,
182
+ values: [encoded],
183
+ }).pipe(Effect.tapCauseLogPretty, Effect.exit)
184
+ }
168
185
  }
169
- }
170
186
 
171
- yield* Effect.logDebug(`Broadcasted to ${rpcSubscriptions.size} RPC clients`)
172
- }
173
- }).pipe(
174
- Effect.tapCauseLogPretty,
175
- Effect.withSpan('push-rpc-broadcast'),
176
- Effect.uninterruptible, // We need to make sure Effect RPC doesn't interrupt this fiber
177
- Effect.fork,
187
+ yield* Effect.logDebug(`Broadcasted to ${rpcSubscriptions.length} RPC clients`)
188
+ }
189
+ }).pipe(Effect.tapCauseLogPretty, Effect.withSpan('push-rpc-broadcast')),
178
190
  )
179
191
 
180
- // We need to yield here to make sure the fork above is kicked off before we let Effect RPC finish the request
181
- yield* Effect.yieldNow()
182
-
192
+ // Acknowledge only after the committed batch has been published, so a client cannot observe an
193
+ // advanced server head while the corresponding pull response is still waiting in a detached fiber.
183
194
  return SyncMessage.PushAck.make({})
184
195
  }).pipe(
185
196
  Effect.tap(
186
197
  Effect.fn(function* (message) {
187
- if (options?.onPushRes) {
188
- yield* Effect.tryAll(() => options.onPushRes!(message)).pipe(UnexpectedError.mapToUnexpectedError)
198
+ if (options?.onPushRes !== undefined) {
199
+ yield* Effect.trySyncOrPromiseOrEffect(() => options.onPushRes!(message)).pipe(
200
+ UnknownError.mapToUnknownError,
201
+ )
189
202
  }
190
203
  }),
191
204
  ),
192
- Effect.mapError((cause) => InvalidPushError.make({ cause })),
205
+ Effect.mapError((cause) =>
206
+ cause._tag === 'BackendIdMismatchError' || cause._tag === 'ServerAheadError' || cause._tag === 'UnknownError'
207
+ ? cause
208
+ : new UnknownError({ cause }),
209
+ ),
193
210
  Effect.withSpan('sync-cf:do:push', { attributes: { storeId, batchSize: pushRequest.batch.length } }),
194
211
  )
195
212
 
196
213
  /**
214
+ * Serializes the complete push admission lifecycle, including publication to pull subscribers.
215
+ *
216
+ * This complements `blockConcurrencyWhile`: that Cloudflare primitive makes the durable state transition
217
+ * atomic with respect to other Durable Object requests, whereas this semaphore preserves push order after
218
+ * the storage gate is released and broadcasting begins. Keeping broadcast outside `blockConcurrencyWhile`
219
+ * avoids holding Cloudflare's concurrency gate across network I/O.
220
+ *
221
+ * Once admitted, the effect is uninterruptible so cancellation cannot leave a committed head without its
222
+ * matching pull response. Such a gap would make later pushers receive `ServerAhead` while waiting for a
223
+ * response that was never published.
224
+ */
225
+ export const runSerializedPushAdmission = <A, E, R>(
226
+ semaphore: DoCtx.Service['pushSemaphore'],
227
+ admission: Effect.Effect<A, E, R>,
228
+ ): Effect.Effect<A, E, R> => semaphore.withPermits(1)(admission.pipe(Effect.uninterruptible))
229
+
230
+ /**
231
+ * Runs the storage-backed state transition behind Cloudflare's Durable Object concurrency gate.
232
+ * This scope should remain limited to local persistence and head mutation; ordered publication is handled
233
+ * separately by `runSerializedPushAdmission` so outbound work does not block unrelated Durable Object events.
234
+ *
197
235
  * @see https://developers.cloudflare.com/durable-objects/api/state/#blockconcurrencywhile
198
236
  */
199
237
  const blockConcurrencyWhile =
200
238
  (ctx: CfTypes.DurableObjectState) =>
201
239
  <A, E, R>(eff: Effect.Effect<A, E, R>) =>
202
240
  Effect.gen(function* () {
203
- const runtime = yield* Effect.runtime<R>()
241
+ const services = yield* Effect.context<R>()
204
242
  const exit = yield* Effect.promise(() =>
205
- ctx.blockConcurrencyWhile(() => eff.pipe(Effect.provide(runtime), Effect.runPromiseExit)),
243
+ ctx.blockConcurrencyWhile(() => eff.pipe(Effect.runPromiseExitWith(services))),
206
244
  )
207
245
 
208
246
  return yield* exit
@@ -1,15 +1,21 @@
1
1
  import { EventSequenceNumber, State } from '@livestore/common/schema'
2
2
  import { Schema } from '@livestore/utils/effect'
3
+
3
4
  import { PERSISTENCE_FORMAT_VERSION } from '../shared.ts'
4
5
 
6
+ /**
7
+ * Main event log table storing all LiveStore events.
8
+ *
9
+ * ⚠️ IMPORTANT: Any changes to this schema require bumping PERSISTENCE_FORMAT_VERSION in shared.ts
10
+ */
5
11
  export const eventlogTable = State.SQLite.table({
6
12
  // NOTE actual table name is determined at runtime to use proper storeId
7
13
  name: `eventlog_${PERSISTENCE_FORMAT_VERSION}_$storeId`,
8
14
  columns: {
9
- seqNum: State.SQLite.integer({ primaryKey: true, schema: EventSequenceNumber.GlobalEventSequenceNumber }),
10
- parentSeqNum: State.SQLite.integer({ schema: EventSequenceNumber.GlobalEventSequenceNumber }),
15
+ seqNum: State.SQLite.integer({ primaryKey: true, schema: EventSequenceNumber.Global.Schema }),
16
+ parentSeqNum: State.SQLite.integer({ schema: EventSequenceNumber.Global.Schema }),
11
17
  name: State.SQLite.text({}),
12
- args: State.SQLite.text({ schema: Schema.parseJson(Schema.Any), nullable: true }),
18
+ args: State.SQLite.text({ schema: Schema.fromJsonString(Schema.Any), nullable: true }),
13
19
  /** ISO date format. Currently only used for debugging purposes. */
14
20
  createdAt: State.SQLite.text({}),
15
21
  clientId: State.SQLite.text({}),
@@ -17,12 +23,16 @@ export const eventlogTable = State.SQLite.table({
17
23
  },
18
24
  })
19
25
 
20
- /** Will only ever have one row per durable object. */
26
+ /**
27
+ * Context metadata table - one row per Durable Object.
28
+ *
29
+ * ⚠️ IMPORTANT: Any changes to this schema require bumping PERSISTENCE_FORMAT_VERSION in shared.ts
30
+ */
21
31
  export const contextTable = State.SQLite.table({
22
32
  name: `context_${PERSISTENCE_FORMAT_VERSION}`,
23
33
  columns: {
24
34
  storeId: State.SQLite.text({ primaryKey: true }),
25
- currentHead: State.SQLite.integer({ schema: EventSequenceNumber.GlobalEventSequenceNumber }),
35
+ currentHead: State.SQLite.integer({ schema: EventSequenceNumber.Global.Schema }),
26
36
  backendId: State.SQLite.text({}),
27
37
  },
28
38
  })