@livestore/sync-cf 0.4.0-dev.1 → 0.4.0-dev.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +60 -0
- package/dist/.tsbuildinfo +1 -1
- package/dist/cf-worker/do/durable-object.d.ts +45 -0
- package/dist/cf-worker/do/durable-object.d.ts.map +1 -0
- package/dist/cf-worker/do/durable-object.js +150 -0
- package/dist/cf-worker/do/durable-object.js.map +1 -0
- package/dist/cf-worker/do/layer.d.ts +34 -0
- package/dist/cf-worker/do/layer.d.ts.map +1 -0
- package/dist/cf-worker/do/layer.js +91 -0
- package/dist/cf-worker/do/layer.js.map +1 -0
- package/dist/cf-worker/do/pull.d.ts +6 -0
- package/dist/cf-worker/do/pull.d.ts.map +1 -0
- package/dist/cf-worker/do/pull.js +47 -0
- package/dist/cf-worker/do/pull.js.map +1 -0
- package/dist/cf-worker/do/push.d.ts +14 -0
- package/dist/cf-worker/do/push.d.ts.map +1 -0
- package/dist/cf-worker/do/push.js +131 -0
- package/dist/cf-worker/do/push.js.map +1 -0
- package/dist/cf-worker/{durable-object.d.ts → do/sqlite.d.ts} +77 -70
- package/dist/cf-worker/do/sqlite.d.ts.map +1 -0
- package/dist/cf-worker/do/sqlite.js +27 -0
- package/dist/cf-worker/do/sqlite.js.map +1 -0
- package/dist/cf-worker/do/sync-storage.d.ts +25 -0
- package/dist/cf-worker/do/sync-storage.d.ts.map +1 -0
- package/dist/cf-worker/do/sync-storage.js +190 -0
- package/dist/cf-worker/do/sync-storage.js.map +1 -0
- package/dist/cf-worker/do/transport/do-rpc-server.d.ts +9 -0
- package/dist/cf-worker/do/transport/do-rpc-server.d.ts.map +1 -0
- package/dist/cf-worker/do/transport/do-rpc-server.js +45 -0
- package/dist/cf-worker/do/transport/do-rpc-server.js.map +1 -0
- package/dist/cf-worker/do/transport/http-rpc-server.d.ts +7 -0
- package/dist/cf-worker/do/transport/http-rpc-server.d.ts.map +1 -0
- package/dist/cf-worker/do/transport/http-rpc-server.js +24 -0
- package/dist/cf-worker/do/transport/http-rpc-server.js.map +1 -0
- package/dist/cf-worker/do/transport/ws-rpc-server.d.ts +4 -0
- package/dist/cf-worker/do/transport/ws-rpc-server.d.ts.map +1 -0
- package/dist/cf-worker/do/transport/ws-rpc-server.js +21 -0
- package/dist/cf-worker/do/transport/ws-rpc-server.js.map +1 -0
- package/dist/cf-worker/mod.d.ts +4 -2
- package/dist/cf-worker/mod.d.ts.map +1 -1
- package/dist/cf-worker/mod.js +3 -2
- package/dist/cf-worker/mod.js.map +1 -1
- package/dist/cf-worker/shared.d.ts +147 -0
- package/dist/cf-worker/shared.d.ts.map +1 -0
- package/dist/cf-worker/shared.js +32 -0
- package/dist/cf-worker/shared.js.map +1 -0
- package/dist/cf-worker/worker.d.ts +45 -45
- package/dist/cf-worker/worker.d.ts.map +1 -1
- package/dist/cf-worker/worker.js +51 -39
- package/dist/cf-worker/worker.js.map +1 -1
- package/dist/client/mod.d.ts +4 -0
- package/dist/client/mod.d.ts.map +1 -0
- package/dist/client/mod.js +4 -0
- package/dist/client/mod.js.map +1 -0
- package/dist/client/transport/do-rpc-client.d.ts +40 -0
- package/dist/client/transport/do-rpc-client.d.ts.map +1 -0
- package/dist/client/transport/do-rpc-client.js +117 -0
- package/dist/client/transport/do-rpc-client.js.map +1 -0
- package/dist/client/transport/http-rpc-client.d.ts +43 -0
- package/dist/client/transport/http-rpc-client.d.ts.map +1 -0
- package/dist/client/transport/http-rpc-client.js +103 -0
- package/dist/client/transport/http-rpc-client.js.map +1 -0
- package/dist/client/transport/ws-rpc-client.d.ts +45 -0
- package/dist/client/transport/ws-rpc-client.d.ts.map +1 -0
- package/dist/client/transport/ws-rpc-client.js +108 -0
- package/dist/client/transport/ws-rpc-client.js.map +1 -0
- package/dist/common/constants.d.ts +7 -0
- package/dist/common/constants.d.ts.map +1 -0
- package/dist/common/constants.js +17 -0
- package/dist/common/constants.js.map +1 -0
- package/dist/common/do-rpc-schema.d.ts +76 -0
- package/dist/common/do-rpc-schema.d.ts.map +1 -0
- package/dist/common/do-rpc-schema.js +48 -0
- package/dist/common/do-rpc-schema.js.map +1 -0
- package/dist/common/http-rpc-schema.d.ts +58 -0
- package/dist/common/http-rpc-schema.d.ts.map +1 -0
- package/dist/common/http-rpc-schema.js +37 -0
- package/dist/common/http-rpc-schema.js.map +1 -0
- package/dist/common/mod.d.ts +8 -1
- package/dist/common/mod.d.ts.map +1 -1
- package/dist/common/mod.js +7 -1
- package/dist/common/mod.js.map +1 -1
- package/dist/common/{ws-message-types.d.ts → sync-message-types.d.ts} +119 -153
- package/dist/common/sync-message-types.d.ts.map +1 -0
- package/dist/common/sync-message-types.js +60 -0
- package/dist/common/sync-message-types.js.map +1 -0
- package/dist/common/ws-rpc-schema.d.ts +55 -0
- package/dist/common/ws-rpc-schema.d.ts.map +1 -0
- package/dist/common/ws-rpc-schema.js +32 -0
- package/dist/common/ws-rpc-schema.js.map +1 -0
- package/package.json +7 -8
- package/src/cf-worker/do/durable-object.ts +237 -0
- package/src/cf-worker/do/layer.ts +128 -0
- package/src/cf-worker/do/pull.ts +77 -0
- package/src/cf-worker/do/push.ts +205 -0
- package/src/cf-worker/do/sqlite.ts +28 -0
- package/src/cf-worker/do/sync-storage.ts +321 -0
- package/src/cf-worker/do/transport/do-rpc-server.ts +84 -0
- package/src/cf-worker/do/transport/http-rpc-server.ts +37 -0
- package/src/cf-worker/do/transport/ws-rpc-server.ts +34 -0
- package/src/cf-worker/mod.ts +4 -2
- package/src/cf-worker/shared.ts +112 -0
- package/src/cf-worker/worker.ts +91 -105
- package/src/client/mod.ts +3 -0
- package/src/client/transport/do-rpc-client.ts +191 -0
- package/src/client/transport/http-rpc-client.ts +225 -0
- package/src/client/transport/ws-rpc-client.ts +202 -0
- package/src/common/constants.ts +18 -0
- package/src/common/do-rpc-schema.ts +54 -0
- package/src/common/http-rpc-schema.ts +40 -0
- package/src/common/mod.ts +10 -1
- package/src/common/sync-message-types.ts +117 -0
- package/src/common/ws-rpc-schema.ts +36 -0
- package/dist/cf-worker/cf-types.d.ts +0 -2
- package/dist/cf-worker/cf-types.d.ts.map +0 -1
- package/dist/cf-worker/cf-types.js +0 -2
- package/dist/cf-worker/cf-types.js.map +0 -1
- package/dist/cf-worker/durable-object.d.ts.map +0 -1
- package/dist/cf-worker/durable-object.js +0 -317
- package/dist/cf-worker/durable-object.js.map +0 -1
- package/dist/common/ws-message-types.d.ts.map +0 -1
- package/dist/common/ws-message-types.js +0 -57
- package/dist/common/ws-message-types.js.map +0 -1
- package/dist/sync-impl/mod.d.ts +0 -2
- package/dist/sync-impl/mod.d.ts.map +0 -1
- package/dist/sync-impl/mod.js +0 -2
- package/dist/sync-impl/mod.js.map +0 -1
- package/dist/sync-impl/ws-impl.d.ts +0 -7
- package/dist/sync-impl/ws-impl.d.ts.map +0 -1
- package/dist/sync-impl/ws-impl.js +0 -175
- package/dist/sync-impl/ws-impl.js.map +0 -1
- package/src/cf-worker/cf-types.ts +0 -12
- package/src/cf-worker/durable-object.ts +0 -478
- package/src/common/ws-message-types.ts +0 -114
- package/src/sync-impl/mod.ts +0 -1
- package/src/sync-impl/ws-impl.ts +0 -274
|
@@ -0,0 +1,225 @@
|
|
|
1
|
+
import { InvalidPullError, InvalidPushError, SyncBackend, UnexpectedError } from '@livestore/common'
|
|
2
|
+
import type { EventSequenceNumber } from '@livestore/common/schema'
|
|
3
|
+
import { splitChunkBySize } from '@livestore/common/sync'
|
|
4
|
+
import { omit } from '@livestore/utils'
|
|
5
|
+
import {
|
|
6
|
+
Chunk,
|
|
7
|
+
type Duration,
|
|
8
|
+
Effect,
|
|
9
|
+
HttpClient,
|
|
10
|
+
HttpClientRequest,
|
|
11
|
+
identity,
|
|
12
|
+
Layer,
|
|
13
|
+
Option,
|
|
14
|
+
RpcClient,
|
|
15
|
+
RpcSerialization,
|
|
16
|
+
Schedule,
|
|
17
|
+
Schema,
|
|
18
|
+
Stream,
|
|
19
|
+
SubscriptionRef,
|
|
20
|
+
UrlParams,
|
|
21
|
+
} from '@livestore/utils/effect'
|
|
22
|
+
import { MAX_HTTP_REQUEST_BYTES, MAX_PUSH_EVENTS_PER_REQUEST } from '../../common/constants.ts'
|
|
23
|
+
import { SyncHttpRpc } from '../../common/http-rpc-schema.ts'
|
|
24
|
+
import { SearchParamsSchema } from '../../common/mod.ts'
|
|
25
|
+
import type { SyncMetadata } from '../../common/sync-message-types.ts'
|
|
26
|
+
|
|
27
|
+
export interface HttpSyncOptions {
|
|
28
|
+
/**
|
|
29
|
+
* URL of the sync backend
|
|
30
|
+
*
|
|
31
|
+
* @example
|
|
32
|
+
* ```ts
|
|
33
|
+
* const syncBackend = makeHttpSync({ url: 'https://sync.example.com' })
|
|
34
|
+
* ```
|
|
35
|
+
*/
|
|
36
|
+
url: string
|
|
37
|
+
headers?: Record<string, string>
|
|
38
|
+
livePull?: {
|
|
39
|
+
/**
|
|
40
|
+
* How often to poll for new events
|
|
41
|
+
* @default 5 seconds
|
|
42
|
+
*/
|
|
43
|
+
pollInterval?: Duration.DurationInput
|
|
44
|
+
}
|
|
45
|
+
ping?: {
|
|
46
|
+
/**
|
|
47
|
+
* @default true
|
|
48
|
+
*/
|
|
49
|
+
enabled?: boolean
|
|
50
|
+
/**
|
|
51
|
+
* How long to wait for a ping response before timing out
|
|
52
|
+
* @default 10 seconds
|
|
53
|
+
*/
|
|
54
|
+
requestTimeout?: Duration.DurationInput
|
|
55
|
+
/**
|
|
56
|
+
* How often to send ping requests
|
|
57
|
+
* @default 10 seconds
|
|
58
|
+
*/
|
|
59
|
+
requestInterval?: Duration.DurationInput
|
|
60
|
+
}
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
/**
|
|
64
|
+
* Note: This implementation requires the `enable_request_signal` compatibility flag to properly support `pull` streaming responses
|
|
65
|
+
*/
|
|
66
|
+
export const makeHttpSync =
|
|
67
|
+
(options: HttpSyncOptions): SyncBackend.SyncBackendConstructor<SyncMetadata> =>
|
|
68
|
+
({ storeId, payload }) =>
|
|
69
|
+
Effect.gen(function* () {
|
|
70
|
+
// Based on ping responses
|
|
71
|
+
const isConnected = yield* SubscriptionRef.make(false)
|
|
72
|
+
|
|
73
|
+
const livePullInterval = options.livePull?.pollInterval ?? 5_000
|
|
74
|
+
|
|
75
|
+
const urlParamsData = yield* Schema.encode(SearchParamsSchema)({
|
|
76
|
+
storeId,
|
|
77
|
+
payload,
|
|
78
|
+
transport: 'http',
|
|
79
|
+
}).pipe(UnexpectedError.mapToUnexpectedError)
|
|
80
|
+
|
|
81
|
+
const urlParams = UrlParams.fromInput(urlParamsData)
|
|
82
|
+
|
|
83
|
+
// Setup HTTP RPC Protocol
|
|
84
|
+
const HttpProtocolLive = RpcClient.layerProtocolHttp({
|
|
85
|
+
url: `${options.url}?${UrlParams.toString(urlParams)}`,
|
|
86
|
+
transformClient: HttpClient.mapRequest((request) =>
|
|
87
|
+
request.pipe(
|
|
88
|
+
HttpClientRequest.setHeaders({
|
|
89
|
+
...options.headers,
|
|
90
|
+
// Used in CF Worker to identify the store (additionally to storeId embedded in the RPC requests)
|
|
91
|
+
'x-livestore-store-id': storeId,
|
|
92
|
+
}),
|
|
93
|
+
),
|
|
94
|
+
),
|
|
95
|
+
}).pipe(Layer.provide(RpcSerialization.layerJson))
|
|
96
|
+
|
|
97
|
+
const rpcClient = yield* RpcClient.make(SyncHttpRpc).pipe(Effect.provide(HttpProtocolLive))
|
|
98
|
+
|
|
99
|
+
const pingTimeout = options.ping?.requestTimeout ?? 10_000
|
|
100
|
+
|
|
101
|
+
const ping: SyncBackend.SyncBackend<SyncMetadata>['ping'] = Effect.gen(function* () {
|
|
102
|
+
yield* rpcClient.SyncHttpRpc.Ping({ storeId, payload })
|
|
103
|
+
|
|
104
|
+
yield* SubscriptionRef.set(isConnected, true)
|
|
105
|
+
}).pipe(
|
|
106
|
+
UnexpectedError.mapToUnexpectedError,
|
|
107
|
+
Effect.timeout(pingTimeout),
|
|
108
|
+
Effect.catchTag('TimeoutException', () => SubscriptionRef.set(isConnected, false)),
|
|
109
|
+
)
|
|
110
|
+
|
|
111
|
+
const pingInterval = options.ping?.requestInterval ?? 10_000
|
|
112
|
+
|
|
113
|
+
if (options.ping?.enabled !== false) {
|
|
114
|
+
// Automatically ping the server to keep the connection alive
|
|
115
|
+
yield* ping.pipe(Effect.repeat(Schedule.spaced(pingInterval)), Effect.tapCauseLogPretty, Effect.forkScoped)
|
|
116
|
+
}
|
|
117
|
+
|
|
118
|
+
// Helps already establish a TCP connection to the server
|
|
119
|
+
const connect = ping.pipe(UnexpectedError.mapToUnexpectedError)
|
|
120
|
+
|
|
121
|
+
const backendIdHelper = yield* SyncBackend.makeBackendIdHelper
|
|
122
|
+
|
|
123
|
+
const mapCursor = (cursor: Option.Option<{ eventSequenceNumber: number }>) =>
|
|
124
|
+
cursor.pipe(
|
|
125
|
+
Option.map((a) => ({
|
|
126
|
+
eventSequenceNumber: a.eventSequenceNumber as EventSequenceNumber.GlobalEventSequenceNumber,
|
|
127
|
+
backendId: backendIdHelper.get().pipe(Option.getOrThrow),
|
|
128
|
+
})),
|
|
129
|
+
)
|
|
130
|
+
|
|
131
|
+
const pull: SyncBackend.SyncBackend<SyncMetadata>['pull'] = (cursor, options) =>
|
|
132
|
+
rpcClient.SyncHttpRpc.Pull({
|
|
133
|
+
storeId,
|
|
134
|
+
payload,
|
|
135
|
+
cursor: mapCursor(cursor),
|
|
136
|
+
}).pipe(
|
|
137
|
+
options?.live
|
|
138
|
+
? // Phase 2: Simulate `live` pull by polling for new events
|
|
139
|
+
Stream.concatWithLastElement((lastElement) => {
|
|
140
|
+
const initialPhase2Cursor = lastElement.pipe(
|
|
141
|
+
Option.flatMap((_) => Option.fromNullable(_.batch.at(-1)?.eventEncoded.seqNum)),
|
|
142
|
+
Option.map((eventSequenceNumber) => ({ eventSequenceNumber })),
|
|
143
|
+
Option.orElse(() => cursor),
|
|
144
|
+
mapCursor,
|
|
145
|
+
)
|
|
146
|
+
|
|
147
|
+
return Stream.unfoldChunkEffect(initialPhase2Cursor, (currentCursor) =>
|
|
148
|
+
Effect.gen(function* () {
|
|
149
|
+
yield* Effect.sleep(livePullInterval)
|
|
150
|
+
|
|
151
|
+
const items = yield* rpcClient.SyncHttpRpc.Pull({ storeId, payload, cursor: currentCursor }).pipe(
|
|
152
|
+
Stream.runCollect,
|
|
153
|
+
)
|
|
154
|
+
|
|
155
|
+
const nextCursor = Chunk.last(items).pipe(
|
|
156
|
+
Option.flatMap((item) => Option.fromNullable(item.batch.at(-1)?.eventEncoded.seqNum)),
|
|
157
|
+
Option.map((eventSequenceNumber) => ({ eventSequenceNumber })),
|
|
158
|
+
Option.orElse(() => currentCursor),
|
|
159
|
+
mapCursor,
|
|
160
|
+
)
|
|
161
|
+
|
|
162
|
+
return Option.some([items, nextCursor])
|
|
163
|
+
}),
|
|
164
|
+
)
|
|
165
|
+
})
|
|
166
|
+
: identity,
|
|
167
|
+
Stream.tap((res) => backendIdHelper.lazySet(res.backendId)),
|
|
168
|
+
Stream.map((res) => omit(res, ['backendId'])),
|
|
169
|
+
Stream.mapError((cause) => (cause._tag === 'InvalidPullError' ? cause : InvalidPullError.make({ cause }))),
|
|
170
|
+
Stream.withSpan('http-sync-client:pull'),
|
|
171
|
+
)
|
|
172
|
+
|
|
173
|
+
const pushSemaphore = yield* Effect.makeSemaphore(1)
|
|
174
|
+
|
|
175
|
+
const push: SyncBackend.SyncBackend<SyncMetadata>['push'] = (batch) =>
|
|
176
|
+
Effect.gen(function* () {
|
|
177
|
+
if (batch.length === 0) {
|
|
178
|
+
return
|
|
179
|
+
}
|
|
180
|
+
|
|
181
|
+
const backendId = backendIdHelper.get()
|
|
182
|
+
const batchChunks = yield* Chunk.fromIterable(batch).pipe(
|
|
183
|
+
splitChunkBySize({
|
|
184
|
+
maxItems: MAX_PUSH_EVENTS_PER_REQUEST,
|
|
185
|
+
maxBytes: MAX_HTTP_REQUEST_BYTES,
|
|
186
|
+
encode: (items) => ({
|
|
187
|
+
batch: items,
|
|
188
|
+
storeId,
|
|
189
|
+
payload,
|
|
190
|
+
backendId,
|
|
191
|
+
}),
|
|
192
|
+
}),
|
|
193
|
+
Effect.mapError((cause) => new InvalidPushError({ cause: new UnexpectedError({ cause }) })),
|
|
194
|
+
)
|
|
195
|
+
|
|
196
|
+
for (const chunk of Chunk.toReadonlyArray(batchChunks)) {
|
|
197
|
+
const chunkArray = Chunk.toReadonlyArray(chunk)
|
|
198
|
+
yield* rpcClient.SyncHttpRpc.Push({ storeId, payload, batch: chunkArray, backendId })
|
|
199
|
+
}
|
|
200
|
+
}).pipe(
|
|
201
|
+
pushSemaphore.withPermits(1),
|
|
202
|
+
Effect.mapError((cause) =>
|
|
203
|
+
cause._tag === 'InvalidPushError' ? cause : new InvalidPushError({ cause: new UnexpectedError({ cause }) }),
|
|
204
|
+
),
|
|
205
|
+
Effect.withSpan('http-sync-client:push'),
|
|
206
|
+
)
|
|
207
|
+
|
|
208
|
+
return SyncBackend.of({
|
|
209
|
+
connect,
|
|
210
|
+
isConnected,
|
|
211
|
+
pull,
|
|
212
|
+
push,
|
|
213
|
+
ping,
|
|
214
|
+
metadata: {
|
|
215
|
+
name: '@livestore/cf-sync-http',
|
|
216
|
+
description: 'LiveStore sync backend implementation using HTTP RPC',
|
|
217
|
+
protocol: 'http',
|
|
218
|
+
url: options.url,
|
|
219
|
+
},
|
|
220
|
+
supports: {
|
|
221
|
+
pullPageInfoKnown: true,
|
|
222
|
+
pullLive: true,
|
|
223
|
+
},
|
|
224
|
+
})
|
|
225
|
+
})
|
|
@@ -0,0 +1,202 @@
|
|
|
1
|
+
import { InvalidPullError, InvalidPushError, IsOfflineError, SyncBackend, UnexpectedError } from '@livestore/common'
|
|
2
|
+
import type { LiveStoreEvent } from '@livestore/common/schema'
|
|
3
|
+
import { splitChunkBySize } from '@livestore/common/sync'
|
|
4
|
+
import { omit } from '@livestore/utils'
|
|
5
|
+
import {
|
|
6
|
+
Chunk,
|
|
7
|
+
type Duration,
|
|
8
|
+
Effect,
|
|
9
|
+
Layer,
|
|
10
|
+
Option,
|
|
11
|
+
RpcClient,
|
|
12
|
+
RpcSerialization,
|
|
13
|
+
Schedule,
|
|
14
|
+
Schema,
|
|
15
|
+
type Scope,
|
|
16
|
+
Socket,
|
|
17
|
+
Stream,
|
|
18
|
+
SubscriptionRef,
|
|
19
|
+
UrlParams,
|
|
20
|
+
type WebSocket,
|
|
21
|
+
} from '@livestore/utils/effect'
|
|
22
|
+
import { MAX_PUSH_EVENTS_PER_REQUEST, MAX_WS_MESSAGE_BYTES } from '../../common/constants.ts'
|
|
23
|
+
import { SearchParamsSchema } from '../../common/mod.ts'
|
|
24
|
+
import type { SyncMetadata } from '../../common/sync-message-types.ts'
|
|
25
|
+
import { SyncWsRpc } from '../../common/ws-rpc-schema.ts'
|
|
26
|
+
|
|
27
|
+
export interface WsSyncOptions {
|
|
28
|
+
/**
|
|
29
|
+
* URL of the sync backend
|
|
30
|
+
*
|
|
31
|
+
* The protocol can either `http`/`https` or `ws`/`wss`
|
|
32
|
+
*
|
|
33
|
+
* @example 'https://sync.example.com'
|
|
34
|
+
*/
|
|
35
|
+
url: string
|
|
36
|
+
/**
|
|
37
|
+
* Optional WebSocket factory for custom WebSocket implementations (e.g., Cloudflare Durable Objects)
|
|
38
|
+
* If not provided, uses standard WebSocket from @livestore/utils/effect
|
|
39
|
+
*/
|
|
40
|
+
webSocketFactory?: (wsUrl: string) => Effect.Effect<globalThis.WebSocket, WebSocket.WebSocketError, Scope.Scope>
|
|
41
|
+
ping?: {
|
|
42
|
+
/**
|
|
43
|
+
* @default true
|
|
44
|
+
*/
|
|
45
|
+
enabled?: boolean
|
|
46
|
+
/**
|
|
47
|
+
* How long to wait for a ping response before timing out
|
|
48
|
+
* @default 10 seconds
|
|
49
|
+
*/
|
|
50
|
+
requestTimeout?: Duration.DurationInput
|
|
51
|
+
/**
|
|
52
|
+
* How often to send ping requests
|
|
53
|
+
* @default 10 seconds
|
|
54
|
+
*/
|
|
55
|
+
requestInterval?: Duration.DurationInput
|
|
56
|
+
}
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
/**
|
|
60
|
+
* Creates a sync backend that uses WebSocket to communicate with the sync backend.
|
|
61
|
+
*
|
|
62
|
+
* @example
|
|
63
|
+
* ```ts
|
|
64
|
+
* import { makeWsSync } from '@livestore/sync-cf/client'
|
|
65
|
+
*
|
|
66
|
+
* const syncBackend = makeWsSync({ url: 'wss://sync.example.com' })
|
|
67
|
+
*/
|
|
68
|
+
export const makeWsSync =
|
|
69
|
+
(options: WsSyncOptions): SyncBackend.SyncBackendConstructor<SyncMetadata> =>
|
|
70
|
+
({ storeId, payload }) =>
|
|
71
|
+
Effect.gen(function* () {
|
|
72
|
+
const urlParamsData = yield* Schema.encode(SearchParamsSchema)({
|
|
73
|
+
storeId,
|
|
74
|
+
payload,
|
|
75
|
+
transport: 'ws',
|
|
76
|
+
}).pipe(UnexpectedError.mapToUnexpectedError)
|
|
77
|
+
|
|
78
|
+
const urlParams = UrlParams.fromInput(urlParamsData)
|
|
79
|
+
const wsUrl = `${options.url}?${UrlParams.toString(urlParams)}`
|
|
80
|
+
|
|
81
|
+
const isConnected = yield* SubscriptionRef.make(false)
|
|
82
|
+
|
|
83
|
+
// TODO bring this back in a cross-platform way
|
|
84
|
+
// If the browser already tells us we're offline, then we'll at least wait until the browser
|
|
85
|
+
// thinks we're online again. (We'll only know for sure once the WS conneciton is established.)
|
|
86
|
+
// while (typeof navigator !== 'undefined' && navigator.onLine === false) {
|
|
87
|
+
// yield* Effect.sleep(1000)
|
|
88
|
+
// }
|
|
89
|
+
// TODO bring this back in a cross-platform way
|
|
90
|
+
// if (navigator.onLine === false) {
|
|
91
|
+
// yield* Effect.async((cb) => self.addEventListener('online', () => cb(Effect.void)))
|
|
92
|
+
// }
|
|
93
|
+
|
|
94
|
+
const pingInterval = options.ping?.requestInterval ?? 10_000
|
|
95
|
+
|
|
96
|
+
const ProtocolLive = RpcClient.layerProtocolSocketWithIsConnected({
|
|
97
|
+
isConnected,
|
|
98
|
+
retryTransientErrors: Schedule.fixed(1000),
|
|
99
|
+
pingSchedule: Schedule.once.pipe(Schedule.andThen(Schedule.fixed(pingInterval))),
|
|
100
|
+
url: wsUrl,
|
|
101
|
+
}).pipe(
|
|
102
|
+
Layer.provide(Socket.layerWebSocket(wsUrl)),
|
|
103
|
+
Layer.provide(Socket.layerWebSocketConstructorGlobal),
|
|
104
|
+
Layer.provide(RpcSerialization.layerJson),
|
|
105
|
+
)
|
|
106
|
+
|
|
107
|
+
// Warning: we need to build the layer here eagerly to tie it to the scope
|
|
108
|
+
// instead of using `Effect.provide(ProtocolLive)` which would close the layer scope too early
|
|
109
|
+
const ctx = yield* Layer.build(ProtocolLive)
|
|
110
|
+
|
|
111
|
+
const rpcClient = yield* RpcClient.make(SyncWsRpc).pipe(Effect.provide(ctx))
|
|
112
|
+
|
|
113
|
+
const pingTimeout = options.ping?.requestTimeout ?? 10_000
|
|
114
|
+
|
|
115
|
+
const ping = Effect.gen(function* () {
|
|
116
|
+
const pinger = yield* RpcClient.SocketPinger.pipe(Effect.provide(ctx))
|
|
117
|
+
yield* pinger.ping
|
|
118
|
+
yield* SubscriptionRef.set(isConnected, true)
|
|
119
|
+
}).pipe(
|
|
120
|
+
Effect.timeout(pingTimeout),
|
|
121
|
+
Effect.catchTag('TimeoutException', () => SubscriptionRef.set(isConnected, false)),
|
|
122
|
+
UnexpectedError.mapToUnexpectedError,
|
|
123
|
+
Effect.withSpan('ping'),
|
|
124
|
+
)
|
|
125
|
+
|
|
126
|
+
const backendIdHelper = yield* SyncBackend.makeBackendIdHelper
|
|
127
|
+
|
|
128
|
+
return SyncBackend.of<SyncMetadata>({
|
|
129
|
+
isConnected,
|
|
130
|
+
connect: ping,
|
|
131
|
+
pull: (cursor, options) =>
|
|
132
|
+
rpcClient.SyncWsRpc.Pull({
|
|
133
|
+
storeId,
|
|
134
|
+
payload,
|
|
135
|
+
cursor: cursor.pipe(
|
|
136
|
+
Option.map((a) => ({
|
|
137
|
+
eventSequenceNumber: a.eventSequenceNumber,
|
|
138
|
+
backendId: backendIdHelper.get().pipe(Option.getOrThrow),
|
|
139
|
+
})),
|
|
140
|
+
),
|
|
141
|
+
live: options?.live ?? false,
|
|
142
|
+
}).pipe(
|
|
143
|
+
Stream.tap((res) => backendIdHelper.lazySet(res.backendId)),
|
|
144
|
+
Stream.map((res) => omit(res, ['backendId'])),
|
|
145
|
+
Stream.mapError((cause) =>
|
|
146
|
+
cause._tag === 'RpcClientError' && Socket.isSocketError(cause.cause)
|
|
147
|
+
? new IsOfflineError({ cause: cause.cause })
|
|
148
|
+
: cause._tag === 'InvalidPullError'
|
|
149
|
+
? cause
|
|
150
|
+
: InvalidPullError.make({ cause }),
|
|
151
|
+
),
|
|
152
|
+
Stream.withSpan('pull'),
|
|
153
|
+
),
|
|
154
|
+
|
|
155
|
+
push: (batch) =>
|
|
156
|
+
Effect.gen(function* () {
|
|
157
|
+
if (batch.length === 0) return
|
|
158
|
+
|
|
159
|
+
const encodePayload = (batch: ReadonlyArray<LiveStoreEvent.AnyEncodedGlobal>) => ({
|
|
160
|
+
storeId,
|
|
161
|
+
payload,
|
|
162
|
+
batch,
|
|
163
|
+
backendId: backendIdHelper.get(),
|
|
164
|
+
})
|
|
165
|
+
|
|
166
|
+
const chunksChunk = yield* Chunk.fromIterable(batch).pipe(
|
|
167
|
+
splitChunkBySize({
|
|
168
|
+
maxItems: MAX_PUSH_EVENTS_PER_REQUEST,
|
|
169
|
+
maxBytes: MAX_WS_MESSAGE_BYTES,
|
|
170
|
+
encode: encodePayload,
|
|
171
|
+
}),
|
|
172
|
+
Effect.mapError((cause) => new InvalidPushError({ cause: new UnexpectedError({ cause }) })),
|
|
173
|
+
)
|
|
174
|
+
|
|
175
|
+
for (const sub of chunksChunk) {
|
|
176
|
+
yield* rpcClient.SyncWsRpc.Push({
|
|
177
|
+
storeId,
|
|
178
|
+
payload,
|
|
179
|
+
batch: Chunk.toReadonlyArray(sub),
|
|
180
|
+
backendId: backendIdHelper.get(),
|
|
181
|
+
}).pipe(
|
|
182
|
+
Effect.mapError((cause) =>
|
|
183
|
+
cause._tag === 'InvalidPushError'
|
|
184
|
+
? cause
|
|
185
|
+
: new InvalidPushError({ cause: new UnexpectedError({ cause }) }),
|
|
186
|
+
),
|
|
187
|
+
)
|
|
188
|
+
}
|
|
189
|
+
}).pipe(Effect.withSpan('push')),
|
|
190
|
+
ping,
|
|
191
|
+
metadata: {
|
|
192
|
+
name: '@livestore/cf-sync',
|
|
193
|
+
description: 'LiveStore sync backend implementation using Cloudflare Workers & Durable Objects',
|
|
194
|
+
protocol: 'ws',
|
|
195
|
+
url: options.url,
|
|
196
|
+
},
|
|
197
|
+
supports: {
|
|
198
|
+
pullPageInfoKnown: true,
|
|
199
|
+
pullLive: true,
|
|
200
|
+
},
|
|
201
|
+
})
|
|
202
|
+
})
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
// Shared transport limits for Cloudflare sync provider
|
|
2
|
+
// Keep payloads comfortably below ~1MB frame caps across Cloudflare transports.
|
|
3
|
+
// References:
|
|
4
|
+
// - Durable Objects WebSockets + hibernation best practices:
|
|
5
|
+
// https://developers.cloudflare.com/durable-objects/best-practices/websockets/
|
|
6
|
+
// - Workers platform limits (general context):
|
|
7
|
+
// https://developers.cloudflare.com/workers/platform/limits/
|
|
8
|
+
// Empirically, frames just below 1MB can fail on hibernated DO WebSockets; we use 900_000 bytes to keep a safety margin.
|
|
9
|
+
export const MAX_TRANSPORT_PAYLOAD_BYTES = 900_000
|
|
10
|
+
|
|
11
|
+
export const MAX_WS_MESSAGE_BYTES = MAX_TRANSPORT_PAYLOAD_BYTES
|
|
12
|
+
export const MAX_DO_RPC_REQUEST_BYTES = MAX_TRANSPORT_PAYLOAD_BYTES
|
|
13
|
+
export const MAX_HTTP_REQUEST_BYTES = MAX_TRANSPORT_PAYLOAD_BYTES
|
|
14
|
+
|
|
15
|
+
// Upper bound for items per message/request. Mirrors server broadcast chunking.
|
|
16
|
+
// Not Cloudflare-enforced; chosen to balance payload size and latency.
|
|
17
|
+
export const MAX_PULL_EVENTS_PER_MESSAGE = 100
|
|
18
|
+
export const MAX_PUSH_EVENTS_PER_REQUEST = 100
|
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
import { InvalidPullError, InvalidPushError } from '@livestore/common'
|
|
2
|
+
import { Rpc, RpcGroup, Schema } from '@livestore/utils/effect'
|
|
3
|
+
import * as SyncMessage from './sync-message-types.ts'
|
|
4
|
+
|
|
5
|
+
const commonPayloadFields = {
|
|
6
|
+
/**
|
|
7
|
+
* While the storeId is already implied by the durable object, we still need the explicit storeId
|
|
8
|
+
* since a DO doesn't know its own id.name value. 🫠
|
|
9
|
+
* https://community.cloudflare.com/t/how-can-i-get-the-name-of-a-durable-object-from-itself/505961/8
|
|
10
|
+
*/
|
|
11
|
+
storeId: Schema.String,
|
|
12
|
+
/** Needed for various reasons (e.g. auth) */
|
|
13
|
+
payload: Schema.optional(Schema.JsonValue),
|
|
14
|
+
}
|
|
15
|
+
|
|
16
|
+
export class SyncDoRpc extends RpcGroup.make(
|
|
17
|
+
Rpc.make('SyncDoRpc.Pull', {
|
|
18
|
+
payload: {
|
|
19
|
+
/** Omitting the cursor will start from the beginning */
|
|
20
|
+
cursor: SyncMessage.PullRequest.fields.cursor,
|
|
21
|
+
// TODO rename
|
|
22
|
+
/** Whether to keep the pull stream alive and wait for more events */
|
|
23
|
+
rpcContext: Schema.optional(
|
|
24
|
+
Schema.Struct({
|
|
25
|
+
callerContext: Schema.Struct({
|
|
26
|
+
bindingName: Schema.String,
|
|
27
|
+
durableObjectId: Schema.String,
|
|
28
|
+
}),
|
|
29
|
+
}),
|
|
30
|
+
),
|
|
31
|
+
...commonPayloadFields,
|
|
32
|
+
},
|
|
33
|
+
success: Schema.Struct({
|
|
34
|
+
rpcRequestId: Schema.String,
|
|
35
|
+
...SyncMessage.PullResponse.fields,
|
|
36
|
+
}),
|
|
37
|
+
error: InvalidPullError,
|
|
38
|
+
stream: true,
|
|
39
|
+
}),
|
|
40
|
+
Rpc.make('SyncDoRpc.Push', {
|
|
41
|
+
payload: {
|
|
42
|
+
...SyncMessage.PushRequest.fields,
|
|
43
|
+
...commonPayloadFields,
|
|
44
|
+
},
|
|
45
|
+
success: SyncMessage.PushAck,
|
|
46
|
+
error: InvalidPushError,
|
|
47
|
+
}),
|
|
48
|
+
Rpc.make('SyncDoRpc.Ping', {
|
|
49
|
+
payload: {
|
|
50
|
+
...commonPayloadFields,
|
|
51
|
+
},
|
|
52
|
+
success: Schema.Void,
|
|
53
|
+
}),
|
|
54
|
+
) {}
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
import { InvalidPullError, InvalidPushError, UnexpectedError } from '@livestore/common'
|
|
2
|
+
import { Rpc, RpcGroup, Schema } from '@livestore/utils/effect'
|
|
3
|
+
import * as SyncMessage from './sync-message-types.ts'
|
|
4
|
+
|
|
5
|
+
/**
|
|
6
|
+
* HTTP RPC Schema for LiveStore CF Sync Provider
|
|
7
|
+
*
|
|
8
|
+
* This defines the RPC endpoints available over HTTP transport.
|
|
9
|
+
* Unlike WebSocket transport which maintains persistent connections,
|
|
10
|
+
* HTTP transport uses request/response patterns for each operation.
|
|
11
|
+
*/
|
|
12
|
+
export class SyncHttpRpc extends RpcGroup.make(
|
|
13
|
+
Rpc.make('SyncHttpRpc.Pull', {
|
|
14
|
+
payload: Schema.Struct({
|
|
15
|
+
storeId: Schema.String,
|
|
16
|
+
payload: Schema.optional(Schema.JsonValue),
|
|
17
|
+
...SyncMessage.PullRequest.fields,
|
|
18
|
+
}),
|
|
19
|
+
success: SyncMessage.PullResponse,
|
|
20
|
+
error: InvalidPullError,
|
|
21
|
+
stream: true,
|
|
22
|
+
}),
|
|
23
|
+
Rpc.make('SyncHttpRpc.Push', {
|
|
24
|
+
payload: Schema.Struct({
|
|
25
|
+
storeId: Schema.String,
|
|
26
|
+
payload: Schema.optional(Schema.JsonValue),
|
|
27
|
+
...SyncMessage.PushRequest.fields,
|
|
28
|
+
}),
|
|
29
|
+
success: SyncMessage.PushAck,
|
|
30
|
+
error: InvalidPushError,
|
|
31
|
+
}),
|
|
32
|
+
Rpc.make('SyncHttpRpc.Ping', {
|
|
33
|
+
payload: Schema.Struct({
|
|
34
|
+
storeId: Schema.String,
|
|
35
|
+
payload: Schema.optional(Schema.JsonValue),
|
|
36
|
+
}),
|
|
37
|
+
success: SyncMessage.Pong,
|
|
38
|
+
error: UnexpectedError,
|
|
39
|
+
}),
|
|
40
|
+
) {}
|
package/src/common/mod.ts
CHANGED
|
@@ -1,8 +1,17 @@
|
|
|
1
|
+
import { OversizeChunkItemError, splitChunkBySize } from '@livestore/common/sync'
|
|
1
2
|
import { Schema } from '@livestore/utils/effect'
|
|
2
3
|
|
|
3
|
-
export
|
|
4
|
+
export type { CfTypes } from '@livestore/common-cf'
|
|
5
|
+
export * from './constants.ts'
|
|
6
|
+
export { SyncHttpRpc } from './http-rpc-schema.ts'
|
|
7
|
+
export * as SyncMessage from './sync-message-types.ts'
|
|
8
|
+
export { OversizeChunkItemError, splitChunkBySize }
|
|
4
9
|
|
|
5
10
|
export const SearchParamsSchema = Schema.Struct({
|
|
6
11
|
storeId: Schema.String,
|
|
7
12
|
payload: Schema.compose(Schema.StringFromUriComponent, Schema.parseJson(Schema.JsonValue)).pipe(Schema.UndefinedOr),
|
|
13
|
+
// NOTE `do-rpc` is handled differently
|
|
14
|
+
transport: Schema.Union(Schema.Literal('http'), Schema.Literal('ws')),
|
|
8
15
|
})
|
|
16
|
+
|
|
17
|
+
export type SearchParams = typeof SearchParamsSchema.Type
|
|
@@ -0,0 +1,117 @@
|
|
|
1
|
+
import { BackendId, SyncBackend } from '@livestore/common'
|
|
2
|
+
import { EventSequenceNumber, LiveStoreEvent } from '@livestore/common/schema'
|
|
3
|
+
import { Schema } from '@livestore/utils/effect'
|
|
4
|
+
|
|
5
|
+
/**
|
|
6
|
+
* Common sync message types shared between different transport modes (WS, HTTP, RPC)
|
|
7
|
+
*
|
|
8
|
+
* These are the canonical message definitions used across all transport implementations.
|
|
9
|
+
*/
|
|
10
|
+
|
|
11
|
+
export const SyncMetadata = Schema.TaggedStruct('SyncMessage.SyncMetadata', {
|
|
12
|
+
/** ISO date format */
|
|
13
|
+
createdAt: Schema.String,
|
|
14
|
+
}).annotations({ title: '@livestore/sync-cf:SyncMetadata' })
|
|
15
|
+
|
|
16
|
+
export type SyncMetadata = typeof SyncMetadata.Type
|
|
17
|
+
|
|
18
|
+
export const PullRequest = Schema.Struct({
|
|
19
|
+
/** Omitting the cursor will start from the beginning */
|
|
20
|
+
cursor: Schema.Option(
|
|
21
|
+
Schema.Struct({
|
|
22
|
+
backendId: BackendId,
|
|
23
|
+
eventSequenceNumber: EventSequenceNumber.GlobalEventSequenceNumber,
|
|
24
|
+
}),
|
|
25
|
+
),
|
|
26
|
+
}).annotations({ title: '@livestore/sync-cf:PullRequest' })
|
|
27
|
+
|
|
28
|
+
export type PullRequest = typeof PullRequest.Type
|
|
29
|
+
|
|
30
|
+
export const PullResponse = Schema.Struct({
|
|
31
|
+
batch: Schema.Array(
|
|
32
|
+
Schema.Struct({
|
|
33
|
+
eventEncoded: LiveStoreEvent.AnyEncodedGlobal,
|
|
34
|
+
metadata: Schema.Option(SyncMetadata),
|
|
35
|
+
}),
|
|
36
|
+
),
|
|
37
|
+
pageInfo: SyncBackend.PullResPageInfo,
|
|
38
|
+
backendId: BackendId,
|
|
39
|
+
}).annotations({ title: '@livestore/sync-cf:PullResponse' })
|
|
40
|
+
|
|
41
|
+
export const emptyPullResponse = (backendId: string) =>
|
|
42
|
+
PullResponse.make({
|
|
43
|
+
batch: [],
|
|
44
|
+
pageInfo: SyncBackend.pageInfoNoMore,
|
|
45
|
+
backendId,
|
|
46
|
+
})
|
|
47
|
+
|
|
48
|
+
export type PullResponse = typeof PullResponse.Type
|
|
49
|
+
|
|
50
|
+
export const PushRequest = Schema.Struct({
|
|
51
|
+
batch: Schema.Array(LiveStoreEvent.AnyEncodedGlobal),
|
|
52
|
+
backendId: Schema.Option(BackendId),
|
|
53
|
+
}).annotations({ title: '@livestore/sync-cf:PushRequest' })
|
|
54
|
+
|
|
55
|
+
export type PushRequest = typeof PushRequest.Type
|
|
56
|
+
|
|
57
|
+
export const PushAck = Schema.Struct({}).annotations({
|
|
58
|
+
title: '@livestore/sync-cf:PushAck',
|
|
59
|
+
})
|
|
60
|
+
|
|
61
|
+
export type PushAck = typeof PushAck.Type
|
|
62
|
+
|
|
63
|
+
export const Ping = Schema.TaggedStruct('SyncMessage.Ping', {}).annotations({ title: '@livestore/sync-cf:Ping' })
|
|
64
|
+
|
|
65
|
+
export type Ping = typeof Ping.Type
|
|
66
|
+
|
|
67
|
+
export const Pong = Schema.TaggedStruct('SyncMessage.Pong', {}).annotations({ title: '@livestore/sync-cf:Pong' })
|
|
68
|
+
|
|
69
|
+
export type Pong = typeof Pong.Type
|
|
70
|
+
|
|
71
|
+
// Admin operations
|
|
72
|
+
export const AdminResetRoomRequest = Schema.TaggedStruct('SyncMessage.AdminResetRoomRequest', {
|
|
73
|
+
adminSecret: Schema.String,
|
|
74
|
+
}).annotations({ title: '@livestore/sync-cf:AdminResetRoomRequest' })
|
|
75
|
+
|
|
76
|
+
export type AdminResetRoomRequest = typeof AdminResetRoomRequest.Type
|
|
77
|
+
|
|
78
|
+
export const AdminResetRoomResponse = Schema.TaggedStruct('SyncMessage.AdminResetRoomResponse', {}).annotations({
|
|
79
|
+
title: '@livestore/sync-cf:AdminResetRoomResponse',
|
|
80
|
+
})
|
|
81
|
+
|
|
82
|
+
export type AdminResetRoomResponse = typeof AdminResetRoomResponse.Type
|
|
83
|
+
|
|
84
|
+
export const AdminInfoRequest = Schema.TaggedStruct('SyncMessage.AdminInfoRequest', {
|
|
85
|
+
adminSecret: Schema.String,
|
|
86
|
+
}).annotations({ title: '@livestore/sync-cf:AdminInfoRequest' })
|
|
87
|
+
|
|
88
|
+
export type AdminInfoRequest = typeof AdminInfoRequest.Type
|
|
89
|
+
|
|
90
|
+
export const AdminInfoResponse = Schema.TaggedStruct('SyncMessage.AdminInfoResponse', {
|
|
91
|
+
info: Schema.Struct({
|
|
92
|
+
durableObjectId: Schema.String,
|
|
93
|
+
}),
|
|
94
|
+
}).annotations({ title: '@livestore/sync-cf:AdminInfoResponse' })
|
|
95
|
+
|
|
96
|
+
export type AdminInfoResponse = typeof AdminInfoResponse.Type
|
|
97
|
+
|
|
98
|
+
export const BackendToClientMessage = Schema.Union(
|
|
99
|
+
PullResponse,
|
|
100
|
+
PushAck,
|
|
101
|
+
Pong,
|
|
102
|
+
AdminResetRoomResponse,
|
|
103
|
+
AdminInfoResponse,
|
|
104
|
+
)
|
|
105
|
+
export type BackendToClientMessage = typeof BackendToClientMessage.Type
|
|
106
|
+
|
|
107
|
+
export const ClientToBackendMessage = Schema.Union(
|
|
108
|
+
PullRequest,
|
|
109
|
+
PushRequest,
|
|
110
|
+
Ping,
|
|
111
|
+
AdminResetRoomRequest,
|
|
112
|
+
AdminInfoRequest,
|
|
113
|
+
)
|
|
114
|
+
export type ClientToBackendMessage = typeof ClientToBackendMessage.Type
|
|
115
|
+
|
|
116
|
+
export const Message = Schema.Union(BackendToClientMessage, ClientToBackendMessage)
|
|
117
|
+
export type Message = typeof Message.Type
|