@livestore/sync-cf 0.4.0-dev.3 → 0.4.0-dev.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +60 -0
- package/dist/.tsbuildinfo +1 -1
- package/dist/cf-worker/do/durable-object.d.ts +45 -0
- package/dist/cf-worker/do/durable-object.d.ts.map +1 -0
- package/dist/cf-worker/do/durable-object.js +154 -0
- package/dist/cf-worker/do/durable-object.js.map +1 -0
- package/dist/cf-worker/do/layer.d.ts +34 -0
- package/dist/cf-worker/do/layer.d.ts.map +1 -0
- package/dist/cf-worker/do/layer.js +68 -0
- package/dist/cf-worker/do/layer.js.map +1 -0
- package/dist/cf-worker/do/pull.d.ts +6 -0
- package/dist/cf-worker/do/pull.d.ts.map +1 -0
- package/dist/cf-worker/do/pull.js +39 -0
- package/dist/cf-worker/do/pull.js.map +1 -0
- package/dist/cf-worker/do/push.d.ts +14 -0
- package/dist/cf-worker/do/push.d.ts.map +1 -0
- package/dist/cf-worker/do/push.js +99 -0
- package/dist/cf-worker/do/push.js.map +1 -0
- package/dist/cf-worker/do/sqlite.d.ts +196 -0
- package/dist/cf-worker/do/sqlite.d.ts.map +1 -0
- package/dist/cf-worker/do/sqlite.js +27 -0
- package/dist/cf-worker/do/sqlite.js.map +1 -0
- package/dist/cf-worker/do/sync-storage.d.ts +17 -0
- package/dist/cf-worker/do/sync-storage.d.ts.map +1 -0
- package/dist/cf-worker/do/sync-storage.js +73 -0
- package/dist/cf-worker/do/sync-storage.js.map +1 -0
- package/dist/cf-worker/do/transport/do-rpc-server.d.ts +8 -0
- package/dist/cf-worker/do/transport/do-rpc-server.d.ts.map +1 -0
- package/dist/cf-worker/do/transport/do-rpc-server.js +45 -0
- package/dist/cf-worker/do/transport/do-rpc-server.js.map +1 -0
- package/dist/cf-worker/do/transport/http-rpc-server.d.ts +7 -0
- package/dist/cf-worker/do/transport/http-rpc-server.d.ts.map +1 -0
- package/dist/cf-worker/do/transport/http-rpc-server.js +24 -0
- package/dist/cf-worker/do/transport/http-rpc-server.js.map +1 -0
- package/dist/cf-worker/do/transport/ws-rpc-server.d.ts +4 -0
- package/dist/cf-worker/do/transport/ws-rpc-server.d.ts.map +1 -0
- package/dist/cf-worker/do/transport/ws-rpc-server.js +21 -0
- package/dist/cf-worker/do/transport/ws-rpc-server.js.map +1 -0
- package/dist/cf-worker/mod.d.ts +4 -2
- package/dist/cf-worker/mod.d.ts.map +1 -1
- package/dist/cf-worker/mod.js +3 -2
- package/dist/cf-worker/mod.js.map +1 -1
- package/dist/cf-worker/shared.d.ts +127 -0
- package/dist/cf-worker/shared.d.ts.map +1 -0
- package/dist/cf-worker/shared.js +26 -0
- package/dist/cf-worker/shared.js.map +1 -0
- package/dist/cf-worker/worker.d.ts +36 -21
- package/dist/cf-worker/worker.d.ts.map +1 -1
- package/dist/cf-worker/worker.js +39 -32
- package/dist/cf-worker/worker.js.map +1 -1
- package/dist/client/mod.d.ts +4 -0
- package/dist/client/mod.d.ts.map +1 -0
- package/dist/client/mod.js +4 -0
- package/dist/client/mod.js.map +1 -0
- package/dist/client/transport/do-rpc-client.d.ts +40 -0
- package/dist/client/transport/do-rpc-client.d.ts.map +1 -0
- package/dist/client/transport/do-rpc-client.js +102 -0
- package/dist/client/transport/do-rpc-client.js.map +1 -0
- package/dist/client/transport/http-rpc-client.d.ts +43 -0
- package/dist/client/transport/http-rpc-client.d.ts.map +1 -0
- package/dist/client/transport/http-rpc-client.js +87 -0
- package/dist/client/transport/http-rpc-client.js.map +1 -0
- package/dist/client/transport/ws-rpc-client.d.ts +45 -0
- package/dist/client/transport/ws-rpc-client.d.ts.map +1 -0
- package/dist/client/transport/ws-rpc-client.js +94 -0
- package/dist/client/transport/ws-rpc-client.js.map +1 -0
- package/dist/common/do-rpc-schema.d.ts +76 -0
- package/dist/common/do-rpc-schema.d.ts.map +1 -0
- package/dist/common/do-rpc-schema.js +48 -0
- package/dist/common/do-rpc-schema.js.map +1 -0
- package/dist/common/http-rpc-schema.d.ts +58 -0
- package/dist/common/http-rpc-schema.d.ts.map +1 -0
- package/dist/common/http-rpc-schema.js +37 -0
- package/dist/common/http-rpc-schema.js.map +1 -0
- package/dist/common/mod.d.ts +5 -1
- package/dist/common/mod.d.ts.map +1 -1
- package/dist/common/mod.js +4 -1
- package/dist/common/mod.js.map +1 -1
- package/dist/common/sync-message-types.d.ts +236 -0
- package/dist/common/sync-message-types.d.ts.map +1 -0
- package/dist/common/sync-message-types.js +60 -0
- package/dist/common/sync-message-types.js.map +1 -0
- package/dist/common/ws-rpc-schema.d.ts +55 -0
- package/dist/common/ws-rpc-schema.d.ts.map +1 -0
- package/dist/common/ws-rpc-schema.js +32 -0
- package/dist/common/ws-rpc-schema.js.map +1 -0
- package/package.json +7 -8
- package/src/cf-worker/do/durable-object.ts +241 -0
- package/src/cf-worker/do/layer.ts +107 -0
- package/src/cf-worker/do/pull.ts +64 -0
- package/src/cf-worker/do/push.ts +162 -0
- package/src/cf-worker/do/sqlite.ts +28 -0
- package/src/cf-worker/do/sync-storage.ts +126 -0
- package/src/cf-worker/do/transport/do-rpc-server.ts +82 -0
- package/src/cf-worker/do/transport/http-rpc-server.ts +37 -0
- package/src/cf-worker/do/transport/ws-rpc-server.ts +34 -0
- package/src/cf-worker/mod.ts +4 -2
- package/src/cf-worker/shared.ts +95 -0
- package/src/cf-worker/worker.ts +72 -63
- package/src/client/mod.ts +3 -0
- package/src/client/transport/do-rpc-client.ts +171 -0
- package/src/client/transport/http-rpc-client.ts +205 -0
- package/src/client/transport/ws-rpc-client.ts +182 -0
- package/src/common/do-rpc-schema.ts +54 -0
- package/src/common/http-rpc-schema.ts +40 -0
- package/src/common/mod.ts +8 -1
- package/src/common/sync-message-types.ts +117 -0
- package/src/common/ws-rpc-schema.ts +36 -0
- package/src/cf-worker/cf-types.ts +0 -12
- package/src/cf-worker/durable-object.ts +0 -478
- package/src/common/ws-message-types.ts +0 -114
- package/src/sync-impl/mod.ts +0 -1
- package/src/sync-impl/ws-impl.ts +0 -274
|
@@ -0,0 +1,171 @@
|
|
|
1
|
+
import { InvalidPullError, InvalidPushError, SyncBackend, UnexpectedError } from '@livestore/common'
|
|
2
|
+
import { type CfTypes, layerProtocolDurableObject } from '@livestore/common-cf'
|
|
3
|
+
import { omit, shouldNeverHappen } from '@livestore/utils'
|
|
4
|
+
import {
|
|
5
|
+
Effect,
|
|
6
|
+
identity,
|
|
7
|
+
Layer,
|
|
8
|
+
Mailbox,
|
|
9
|
+
Option,
|
|
10
|
+
RpcClient,
|
|
11
|
+
RpcSerialization,
|
|
12
|
+
Schema,
|
|
13
|
+
Stream,
|
|
14
|
+
SubscriptionRef,
|
|
15
|
+
} from '@livestore/utils/effect'
|
|
16
|
+
import type { SyncBackendRpcInterface } from '../../cf-worker/shared.ts'
|
|
17
|
+
import { SyncDoRpc } from '../../common/do-rpc-schema.ts'
|
|
18
|
+
import { SyncMessage } from '../../common/mod.ts'
|
|
19
|
+
import type { SyncMetadata } from '../../common/sync-message-types.ts'
|
|
20
|
+
|
|
21
|
+
export interface SyncBackendRpcStub extends CfTypes.DurableObjectStub, SyncBackendRpcInterface {}
|
|
22
|
+
|
|
23
|
+
// TODO we probably need better scoping for the requestIdMailboxMap (i.e. support multiple stores, ...)
|
|
24
|
+
type EffectRpcRequestId = string // 0, 1, 2, ...
|
|
25
|
+
const requestIdMailboxMap = new Map<EffectRpcRequestId, Mailbox.Mailbox<SyncMessage.PullResponse>>()
|
|
26
|
+
|
|
27
|
+
export interface DoRpcSyncOptions {
|
|
28
|
+
/** Durable Object stub that implements the SyncDoRpc interface */
|
|
29
|
+
syncBackendStub: SyncBackendRpcStub
|
|
30
|
+
/** Information about this DurableObject instance so the Sync DO instance can call back to this instance */
|
|
31
|
+
durableObjectContext: {
|
|
32
|
+
/** See `wrangler.toml` for the binding name */
|
|
33
|
+
bindingName: string
|
|
34
|
+
/** `state.id.toString()` in the DO */
|
|
35
|
+
durableObjectId: string
|
|
36
|
+
}
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
/**
|
|
40
|
+
* Creates a sync backend that uses Durable Object RPC to communicate with the sync backend.
|
|
41
|
+
*
|
|
42
|
+
* Used internally by `@livestore/adapter-cf` to connect to the sync backend.
|
|
43
|
+
*/
|
|
44
|
+
export const makeDoRpcSync =
|
|
45
|
+
({ syncBackendStub, durableObjectContext }: DoRpcSyncOptions): SyncBackend.SyncBackendConstructor<SyncMetadata> =>
|
|
46
|
+
({ storeId, payload }) =>
|
|
47
|
+
Effect.gen(function* () {
|
|
48
|
+
const isConnected = yield* SubscriptionRef.make(true)
|
|
49
|
+
|
|
50
|
+
const ProtocolLive = layerProtocolDurableObject({
|
|
51
|
+
callRpc: (payload) => syncBackendStub.rpc(payload),
|
|
52
|
+
callerContext: durableObjectContext,
|
|
53
|
+
}).pipe(Layer.provide(RpcSerialization.layerJson))
|
|
54
|
+
|
|
55
|
+
const context = yield* Layer.build(ProtocolLive)
|
|
56
|
+
|
|
57
|
+
const rpcClient = yield* RpcClient.make(SyncDoRpc).pipe(Effect.provide(context))
|
|
58
|
+
|
|
59
|
+
// Nothing to do here
|
|
60
|
+
const connect = Effect.void
|
|
61
|
+
|
|
62
|
+
const backendIdHelper = yield* SyncBackend.makeBackendIdHelper
|
|
63
|
+
|
|
64
|
+
const pull: SyncBackend.SyncBackend<SyncMetadata>['pull'] = (cursor, options) =>
|
|
65
|
+
rpcClient.SyncDoRpc.Pull({
|
|
66
|
+
cursor: cursor.pipe(
|
|
67
|
+
Option.map((a) => ({
|
|
68
|
+
eventSequenceNumber: a.eventSequenceNumber,
|
|
69
|
+
backendId: backendIdHelper.get().pipe(Option.getOrThrow),
|
|
70
|
+
})),
|
|
71
|
+
),
|
|
72
|
+
storeId,
|
|
73
|
+
rpcContext: options?.live ? { callerContext: durableObjectContext } : undefined,
|
|
74
|
+
}).pipe(
|
|
75
|
+
options?.live
|
|
76
|
+
? Stream.concatWithLastElement((res) =>
|
|
77
|
+
Effect.gen(function* () {
|
|
78
|
+
if (res._tag === 'None')
|
|
79
|
+
return shouldNeverHappen('There should at least be a no-more page info response')
|
|
80
|
+
|
|
81
|
+
const mailbox = yield* Mailbox.make<SyncMessage.PullResponse>().pipe(
|
|
82
|
+
Effect.acquireRelease((mailbox) => mailbox.shutdown),
|
|
83
|
+
)
|
|
84
|
+
|
|
85
|
+
requestIdMailboxMap.set(res.value.rpcRequestId, mailbox)
|
|
86
|
+
|
|
87
|
+
return Mailbox.toStream(mailbox)
|
|
88
|
+
}).pipe(Stream.unwrapScoped),
|
|
89
|
+
)
|
|
90
|
+
: identity,
|
|
91
|
+
Stream.tap((res) => backendIdHelper.lazySet(res.backendId)),
|
|
92
|
+
Stream.map((res) => omit(res, ['backendId'])),
|
|
93
|
+
Stream.mapError((cause) => (cause._tag === 'InvalidPullError' ? cause : InvalidPullError.make({ cause }))),
|
|
94
|
+
Stream.withSpan('rpc-sync-client:pull'),
|
|
95
|
+
)
|
|
96
|
+
|
|
97
|
+
const push: SyncBackend.SyncBackend<{ createdAt: string }>['push'] = (batch) =>
|
|
98
|
+
Effect.gen(function* () {
|
|
99
|
+
if (batch.length === 0) {
|
|
100
|
+
return
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
yield* rpcClient.SyncDoRpc.Push({ batch, storeId, backendId: backendIdHelper.get() })
|
|
104
|
+
}).pipe(
|
|
105
|
+
Effect.mapError((cause) =>
|
|
106
|
+
cause._tag === 'InvalidPushError'
|
|
107
|
+
? cause
|
|
108
|
+
: InvalidPushError.make({ cause: new UnexpectedError({ cause }) }),
|
|
109
|
+
),
|
|
110
|
+
Effect.withSpan('rpc-sync-client:push'),
|
|
111
|
+
)
|
|
112
|
+
|
|
113
|
+
const ping: SyncBackend.SyncBackend<{ createdAt: string }>['ping'] = rpcClient.SyncDoRpc.Ping({
|
|
114
|
+
storeId,
|
|
115
|
+
payload,
|
|
116
|
+
}).pipe(UnexpectedError.mapToUnexpectedError, Effect.withSpan('rpc-sync-client:ping'))
|
|
117
|
+
|
|
118
|
+
return SyncBackend.of({
|
|
119
|
+
connect,
|
|
120
|
+
isConnected,
|
|
121
|
+
pull,
|
|
122
|
+
push,
|
|
123
|
+
ping,
|
|
124
|
+
metadata: {
|
|
125
|
+
name: 'rpc-sync-client',
|
|
126
|
+
description: 'Cloudflare Durable Object RPC Sync Client',
|
|
127
|
+
protocol: 'rpc',
|
|
128
|
+
storeId,
|
|
129
|
+
},
|
|
130
|
+
supports: {
|
|
131
|
+
pullPageInfoKnown: true,
|
|
132
|
+
pullLive: true,
|
|
133
|
+
},
|
|
134
|
+
})
|
|
135
|
+
}).pipe(Effect.withSpan('rpc-sync-client:makeDoRpcSync'))
|
|
136
|
+
|
|
137
|
+
/**
|
|
138
|
+
*
|
|
139
|
+
* ```ts
|
|
140
|
+
* import { DurableObject } from 'cloudflare:workers'
|
|
141
|
+
* import { ClientDoWithRpcCallback } from '@livestore/common-cf'
|
|
142
|
+
*
|
|
143
|
+
* export class MyDurableObject extends DurableObject implements ClientDoWithRpcCallback {
|
|
144
|
+
* // ...
|
|
145
|
+
*
|
|
146
|
+
* async syncUpdateRpc(payload: RpcMessage.ResponseChunkEncoded) {
|
|
147
|
+
* return handleSyncUpdateRpc(payload)
|
|
148
|
+
* }
|
|
149
|
+
* }
|
|
150
|
+
* ```
|
|
151
|
+
*/
|
|
152
|
+
export const handleSyncUpdateRpc = (payload: unknown) =>
|
|
153
|
+
Effect.gen(function* () {
|
|
154
|
+
const decodedPayload = yield* Schema.decodeUnknown(ResponseChunkEncoded)(payload)
|
|
155
|
+
const decoded = yield* Schema.decodeUnknown(SyncMessage.PullResponse)(decodedPayload.values[0]!)
|
|
156
|
+
|
|
157
|
+
const pullStreamMailbox = requestIdMailboxMap.get(decodedPayload.requestId)
|
|
158
|
+
|
|
159
|
+
if (pullStreamMailbox === undefined) {
|
|
160
|
+
// Case: DO was hibernated, so we need to manually update the store
|
|
161
|
+
yield* Effect.log(`No mailbox found for ${decodedPayload.requestId}`)
|
|
162
|
+
} else {
|
|
163
|
+
// Case: DO was still alive, so the existing `pull` will pick up the new events
|
|
164
|
+
yield* pullStreamMailbox.offer(decoded)
|
|
165
|
+
}
|
|
166
|
+
}).pipe(Effect.withSpan('rpc-sync-client:rpcCallback'), Effect.tapCauseLogPretty, Effect.runPromise)
|
|
167
|
+
|
|
168
|
+
const ResponseChunkEncoded = Schema.Struct({
|
|
169
|
+
requestId: Schema.String,
|
|
170
|
+
values: Schema.Array(Schema.Any),
|
|
171
|
+
})
|
|
@@ -0,0 +1,205 @@
|
|
|
1
|
+
import { InvalidPullError, InvalidPushError, SyncBackend, UnexpectedError } from '@livestore/common'
|
|
2
|
+
import type { EventSequenceNumber } from '@livestore/common/schema'
|
|
3
|
+
import { omit } from '@livestore/utils'
|
|
4
|
+
import {
|
|
5
|
+
Chunk,
|
|
6
|
+
type Duration,
|
|
7
|
+
Effect,
|
|
8
|
+
HttpClient,
|
|
9
|
+
HttpClientRequest,
|
|
10
|
+
identity,
|
|
11
|
+
Layer,
|
|
12
|
+
Option,
|
|
13
|
+
RpcClient,
|
|
14
|
+
RpcSerialization,
|
|
15
|
+
Schedule,
|
|
16
|
+
Schema,
|
|
17
|
+
Stream,
|
|
18
|
+
SubscriptionRef,
|
|
19
|
+
UrlParams,
|
|
20
|
+
} from '@livestore/utils/effect'
|
|
21
|
+
import { SyncHttpRpc } from '../../common/http-rpc-schema.ts'
|
|
22
|
+
import { SearchParamsSchema } from '../../common/mod.ts'
|
|
23
|
+
import type { SyncMetadata } from '../../common/sync-message-types.ts'
|
|
24
|
+
|
|
25
|
+
export interface HttpSyncOptions {
|
|
26
|
+
/**
|
|
27
|
+
* URL of the sync backend
|
|
28
|
+
*
|
|
29
|
+
* @example
|
|
30
|
+
* ```ts
|
|
31
|
+
* const syncBackend = makeHttpSync({ url: 'https://sync.example.com' })
|
|
32
|
+
* ```
|
|
33
|
+
*/
|
|
34
|
+
url: string
|
|
35
|
+
headers?: Record<string, string>
|
|
36
|
+
livePull?: {
|
|
37
|
+
/**
|
|
38
|
+
* How often to poll for new events
|
|
39
|
+
* @default 5 seconds
|
|
40
|
+
*/
|
|
41
|
+
pollInterval?: Duration.DurationInput
|
|
42
|
+
}
|
|
43
|
+
ping?: {
|
|
44
|
+
/**
|
|
45
|
+
* @default true
|
|
46
|
+
*/
|
|
47
|
+
enabled?: boolean
|
|
48
|
+
/**
|
|
49
|
+
* How long to wait for a ping response before timing out
|
|
50
|
+
* @default 10 seconds
|
|
51
|
+
*/
|
|
52
|
+
requestTimeout?: Duration.DurationInput
|
|
53
|
+
/**
|
|
54
|
+
* How often to send ping requests
|
|
55
|
+
* @default 10 seconds
|
|
56
|
+
*/
|
|
57
|
+
requestInterval?: Duration.DurationInput
|
|
58
|
+
}
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
/**
|
|
62
|
+
* Note: This implementation requires the `enable_request_signal` compatibility flag to properly support `pull` streaming responses
|
|
63
|
+
*/
|
|
64
|
+
export const makeHttpSync =
|
|
65
|
+
(options: HttpSyncOptions): SyncBackend.SyncBackendConstructor<SyncMetadata> =>
|
|
66
|
+
({ storeId, payload }) =>
|
|
67
|
+
Effect.gen(function* () {
|
|
68
|
+
// Based on ping responses
|
|
69
|
+
const isConnected = yield* SubscriptionRef.make(false)
|
|
70
|
+
|
|
71
|
+
const livePullInterval = options.livePull?.pollInterval ?? 5_000
|
|
72
|
+
|
|
73
|
+
const urlParamsData = yield* Schema.encode(SearchParamsSchema)({
|
|
74
|
+
storeId,
|
|
75
|
+
payload,
|
|
76
|
+
transport: 'http',
|
|
77
|
+
}).pipe(UnexpectedError.mapToUnexpectedError)
|
|
78
|
+
|
|
79
|
+
const urlParams = UrlParams.fromInput(urlParamsData)
|
|
80
|
+
|
|
81
|
+
// Setup HTTP RPC Protocol
|
|
82
|
+
const HttpProtocolLive = RpcClient.layerProtocolHttp({
|
|
83
|
+
url: `${options.url}?${UrlParams.toString(urlParams)}`,
|
|
84
|
+
transformClient: HttpClient.mapRequest((request) =>
|
|
85
|
+
request.pipe(
|
|
86
|
+
HttpClientRequest.setHeaders({
|
|
87
|
+
...options.headers,
|
|
88
|
+
// Used in CF Worker to identify the store (additionally to storeId embedded in the RPC requests)
|
|
89
|
+
'x-livestore-store-id': storeId,
|
|
90
|
+
}),
|
|
91
|
+
),
|
|
92
|
+
),
|
|
93
|
+
}).pipe(Layer.provide(RpcSerialization.layerJson))
|
|
94
|
+
|
|
95
|
+
const rpcClient = yield* RpcClient.make(SyncHttpRpc).pipe(Effect.provide(HttpProtocolLive))
|
|
96
|
+
|
|
97
|
+
const pingTimeout = options.ping?.requestTimeout ?? 10_000
|
|
98
|
+
|
|
99
|
+
const ping: SyncBackend.SyncBackend<SyncMetadata>['ping'] = Effect.gen(function* () {
|
|
100
|
+
yield* rpcClient.SyncHttpRpc.Ping({ storeId, payload })
|
|
101
|
+
|
|
102
|
+
yield* SubscriptionRef.set(isConnected, true)
|
|
103
|
+
}).pipe(
|
|
104
|
+
UnexpectedError.mapToUnexpectedError,
|
|
105
|
+
Effect.timeout(pingTimeout),
|
|
106
|
+
Effect.catchTag('TimeoutException', () => SubscriptionRef.set(isConnected, false)),
|
|
107
|
+
)
|
|
108
|
+
|
|
109
|
+
const pingInterval = options.ping?.requestInterval ?? 10_000
|
|
110
|
+
|
|
111
|
+
if (options.ping?.enabled !== false) {
|
|
112
|
+
// Automatically ping the server to keep the connection alive
|
|
113
|
+
yield* ping.pipe(Effect.repeat(Schedule.spaced(pingInterval)), Effect.tapCauseLogPretty, Effect.forkScoped)
|
|
114
|
+
}
|
|
115
|
+
|
|
116
|
+
// Helps already establish a TCP connection to the server
|
|
117
|
+
const connect = ping.pipe(UnexpectedError.mapToUnexpectedError)
|
|
118
|
+
|
|
119
|
+
const backendIdHelper = yield* SyncBackend.makeBackendIdHelper
|
|
120
|
+
|
|
121
|
+
const mapCursor = (cursor: Option.Option<{ eventSequenceNumber: number }>) =>
|
|
122
|
+
cursor.pipe(
|
|
123
|
+
Option.map((a) => ({
|
|
124
|
+
eventSequenceNumber: a.eventSequenceNumber as EventSequenceNumber.GlobalEventSequenceNumber,
|
|
125
|
+
backendId: backendIdHelper.get().pipe(Option.getOrThrow),
|
|
126
|
+
})),
|
|
127
|
+
)
|
|
128
|
+
|
|
129
|
+
const pull: SyncBackend.SyncBackend<SyncMetadata>['pull'] = (cursor, options) =>
|
|
130
|
+
rpcClient.SyncHttpRpc.Pull({
|
|
131
|
+
storeId,
|
|
132
|
+
payload,
|
|
133
|
+
cursor: mapCursor(cursor),
|
|
134
|
+
}).pipe(
|
|
135
|
+
options?.live
|
|
136
|
+
? // Phase 2: Simulate `live` pull by polling for new events
|
|
137
|
+
Stream.concatWithLastElement((lastElement) => {
|
|
138
|
+
const initialPhase2Cursor = lastElement.pipe(
|
|
139
|
+
Option.flatMap((_) => Option.fromNullable(_.batch.at(-1)?.eventEncoded.seqNum)),
|
|
140
|
+
Option.map((eventSequenceNumber) => ({ eventSequenceNumber })),
|
|
141
|
+
Option.orElse(() => cursor),
|
|
142
|
+
mapCursor,
|
|
143
|
+
)
|
|
144
|
+
|
|
145
|
+
return Stream.unfoldChunkEffect(initialPhase2Cursor, (currentCursor) =>
|
|
146
|
+
Effect.gen(function* () {
|
|
147
|
+
yield* Effect.sleep(livePullInterval)
|
|
148
|
+
|
|
149
|
+
const items = yield* rpcClient.SyncHttpRpc.Pull({ storeId, payload, cursor: currentCursor }).pipe(
|
|
150
|
+
Stream.runCollect,
|
|
151
|
+
)
|
|
152
|
+
|
|
153
|
+
const nextCursor = Chunk.last(items).pipe(
|
|
154
|
+
Option.flatMap((item) => Option.fromNullable(item.batch.at(-1)?.eventEncoded.seqNum)),
|
|
155
|
+
Option.map((eventSequenceNumber) => ({ eventSequenceNumber })),
|
|
156
|
+
Option.orElse(() => currentCursor),
|
|
157
|
+
mapCursor,
|
|
158
|
+
)
|
|
159
|
+
|
|
160
|
+
return Option.some([items, nextCursor])
|
|
161
|
+
}),
|
|
162
|
+
)
|
|
163
|
+
})
|
|
164
|
+
: identity,
|
|
165
|
+
Stream.tap((res) => backendIdHelper.lazySet(res.backendId)),
|
|
166
|
+
Stream.map((res) => omit(res, ['backendId'])),
|
|
167
|
+
Stream.mapError((cause) => (cause._tag === 'InvalidPullError' ? cause : InvalidPullError.make({ cause }))),
|
|
168
|
+
Stream.withSpan('http-sync-client:pull'),
|
|
169
|
+
)
|
|
170
|
+
|
|
171
|
+
const pushSemaphore = yield* Effect.makeSemaphore(1)
|
|
172
|
+
|
|
173
|
+
const push: SyncBackend.SyncBackend<SyncMetadata>['push'] = (batch) =>
|
|
174
|
+
Effect.gen(function* () {
|
|
175
|
+
if (batch.length === 0) {
|
|
176
|
+
return
|
|
177
|
+
}
|
|
178
|
+
|
|
179
|
+
yield* rpcClient.SyncHttpRpc.Push({ storeId, payload, batch, backendId: backendIdHelper.get() })
|
|
180
|
+
}).pipe(
|
|
181
|
+
pushSemaphore.withPermits(1),
|
|
182
|
+
Effect.mapError((cause) =>
|
|
183
|
+
cause._tag === 'InvalidPushError' ? cause : new InvalidPushError({ cause: new UnexpectedError({ cause }) }),
|
|
184
|
+
),
|
|
185
|
+
Effect.withSpan('http-sync-client:push'),
|
|
186
|
+
)
|
|
187
|
+
|
|
188
|
+
return SyncBackend.of({
|
|
189
|
+
connect,
|
|
190
|
+
isConnected,
|
|
191
|
+
pull,
|
|
192
|
+
push,
|
|
193
|
+
ping,
|
|
194
|
+
metadata: {
|
|
195
|
+
name: '@livestore/cf-sync-http',
|
|
196
|
+
description: 'LiveStore sync backend implementation using HTTP RPC',
|
|
197
|
+
protocol: 'http',
|
|
198
|
+
url: options.url,
|
|
199
|
+
},
|
|
200
|
+
supports: {
|
|
201
|
+
pullPageInfoKnown: true,
|
|
202
|
+
pullLive: true,
|
|
203
|
+
},
|
|
204
|
+
})
|
|
205
|
+
})
|
|
@@ -0,0 +1,182 @@
|
|
|
1
|
+
import { InvalidPullError, InvalidPushError, IsOfflineError, SyncBackend, UnexpectedError } from '@livestore/common'
|
|
2
|
+
import { omit } from '@livestore/utils'
|
|
3
|
+
import {
|
|
4
|
+
type Duration,
|
|
5
|
+
Effect,
|
|
6
|
+
Layer,
|
|
7
|
+
Option,
|
|
8
|
+
RpcClient,
|
|
9
|
+
RpcSerialization,
|
|
10
|
+
Schedule,
|
|
11
|
+
Schema,
|
|
12
|
+
type Scope,
|
|
13
|
+
Socket,
|
|
14
|
+
Stream,
|
|
15
|
+
SubscriptionRef,
|
|
16
|
+
UrlParams,
|
|
17
|
+
type WebSocket,
|
|
18
|
+
} from '@livestore/utils/effect'
|
|
19
|
+
import { SearchParamsSchema } from '../../common/mod.ts'
|
|
20
|
+
import type { SyncMetadata } from '../../common/sync-message-types.ts'
|
|
21
|
+
import { SyncWsRpc } from '../../common/ws-rpc-schema.ts'
|
|
22
|
+
|
|
23
|
+
export interface WsSyncOptions {
|
|
24
|
+
/**
|
|
25
|
+
* URL of the sync backend
|
|
26
|
+
*
|
|
27
|
+
* The protocol can either `http`/`https` or `ws`/`wss`
|
|
28
|
+
*
|
|
29
|
+
* @example 'https://sync.example.com'
|
|
30
|
+
*/
|
|
31
|
+
url: string
|
|
32
|
+
/**
|
|
33
|
+
* Optional WebSocket factory for custom WebSocket implementations (e.g., Cloudflare Durable Objects)
|
|
34
|
+
* If not provided, uses standard WebSocket from @livestore/utils/effect
|
|
35
|
+
*/
|
|
36
|
+
webSocketFactory?: (wsUrl: string) => Effect.Effect<globalThis.WebSocket, WebSocket.WebSocketError, Scope.Scope>
|
|
37
|
+
ping?: {
|
|
38
|
+
/**
|
|
39
|
+
* @default true
|
|
40
|
+
*/
|
|
41
|
+
enabled?: boolean
|
|
42
|
+
/**
|
|
43
|
+
* How long to wait for a ping response before timing out
|
|
44
|
+
* @default 10 seconds
|
|
45
|
+
*/
|
|
46
|
+
requestTimeout?: Duration.DurationInput
|
|
47
|
+
/**
|
|
48
|
+
* How often to send ping requests
|
|
49
|
+
* @default 10 seconds
|
|
50
|
+
*/
|
|
51
|
+
requestInterval?: Duration.DurationInput
|
|
52
|
+
}
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
/**
|
|
56
|
+
* Creates a sync backend that uses WebSocket to communicate with the sync backend.
|
|
57
|
+
*
|
|
58
|
+
* @example
|
|
59
|
+
* ```ts
|
|
60
|
+
* import { makeWsSync } from '@livestore/sync-cf/client'
|
|
61
|
+
*
|
|
62
|
+
* const syncBackend = makeWsSync({ url: 'wss://sync.example.com' })
|
|
63
|
+
*/
|
|
64
|
+
export const makeWsSync =
|
|
65
|
+
(options: WsSyncOptions): SyncBackend.SyncBackendConstructor<SyncMetadata> =>
|
|
66
|
+
({ storeId, payload }) =>
|
|
67
|
+
Effect.gen(function* () {
|
|
68
|
+
const urlParamsData = yield* Schema.encode(SearchParamsSchema)({
|
|
69
|
+
storeId,
|
|
70
|
+
payload,
|
|
71
|
+
transport: 'ws',
|
|
72
|
+
}).pipe(UnexpectedError.mapToUnexpectedError)
|
|
73
|
+
|
|
74
|
+
const urlParams = UrlParams.fromInput(urlParamsData)
|
|
75
|
+
const wsUrl = `${options.url}?${UrlParams.toString(urlParams)}`
|
|
76
|
+
|
|
77
|
+
const isConnected = yield* SubscriptionRef.make(false)
|
|
78
|
+
|
|
79
|
+
// TODO bring this back in a cross-platform way
|
|
80
|
+
// If the browser already tells us we're offline, then we'll at least wait until the browser
|
|
81
|
+
// thinks we're online again. (We'll only know for sure once the WS conneciton is established.)
|
|
82
|
+
// while (typeof navigator !== 'undefined' && navigator.onLine === false) {
|
|
83
|
+
// yield* Effect.sleep(1000)
|
|
84
|
+
// }
|
|
85
|
+
// TODO bring this back in a cross-platform way
|
|
86
|
+
// if (navigator.onLine === false) {
|
|
87
|
+
// yield* Effect.async((cb) => self.addEventListener('online', () => cb(Effect.void)))
|
|
88
|
+
// }
|
|
89
|
+
|
|
90
|
+
const pingInterval = options.ping?.requestInterval ?? 10_000
|
|
91
|
+
|
|
92
|
+
const ProtocolLive = RpcClient.layerProtocolSocketWithIsConnected({
|
|
93
|
+
isConnected,
|
|
94
|
+
retryTransientErrors: Schedule.fixed(1000),
|
|
95
|
+
pingSchedule: Schedule.once.pipe(Schedule.andThen(Schedule.fixed(pingInterval))),
|
|
96
|
+
url: wsUrl,
|
|
97
|
+
}).pipe(
|
|
98
|
+
Layer.provide(Socket.layerWebSocket(wsUrl)),
|
|
99
|
+
Layer.provide(Socket.layerWebSocketConstructorGlobal),
|
|
100
|
+
Layer.provide(RpcSerialization.layerJson),
|
|
101
|
+
)
|
|
102
|
+
|
|
103
|
+
// Warning: we need to build the layer here eagerly to tie it to the scope
|
|
104
|
+
// instead of using `Effect.provide(ProtocolLive)` which would close the layer scope too early
|
|
105
|
+
const ctx = yield* Layer.build(ProtocolLive)
|
|
106
|
+
|
|
107
|
+
const rpcClient = yield* RpcClient.make(SyncWsRpc).pipe(Effect.provide(ctx))
|
|
108
|
+
|
|
109
|
+
const pingTimeout = options.ping?.requestTimeout ?? 10_000
|
|
110
|
+
|
|
111
|
+
const ping = Effect.gen(function* () {
|
|
112
|
+
const pinger = yield* RpcClient.SocketPinger.pipe(Effect.provide(ctx))
|
|
113
|
+
yield* pinger.ping
|
|
114
|
+
yield* SubscriptionRef.set(isConnected, true)
|
|
115
|
+
}).pipe(
|
|
116
|
+
Effect.timeout(pingTimeout),
|
|
117
|
+
Effect.catchTag('TimeoutException', () => SubscriptionRef.set(isConnected, false)),
|
|
118
|
+
UnexpectedError.mapToUnexpectedError,
|
|
119
|
+
Effect.withSpan('ping'),
|
|
120
|
+
)
|
|
121
|
+
|
|
122
|
+
const backendIdHelper = yield* SyncBackend.makeBackendIdHelper
|
|
123
|
+
|
|
124
|
+
return SyncBackend.of<SyncMetadata>({
|
|
125
|
+
isConnected,
|
|
126
|
+
connect: ping,
|
|
127
|
+
pull: (cursor, options) =>
|
|
128
|
+
rpcClient.SyncWsRpc.Pull({
|
|
129
|
+
storeId,
|
|
130
|
+
payload,
|
|
131
|
+
cursor: cursor.pipe(
|
|
132
|
+
Option.map((a) => ({
|
|
133
|
+
eventSequenceNumber: a.eventSequenceNumber,
|
|
134
|
+
backendId: backendIdHelper.get().pipe(Option.getOrThrow),
|
|
135
|
+
})),
|
|
136
|
+
),
|
|
137
|
+
live: options?.live ?? false,
|
|
138
|
+
}).pipe(
|
|
139
|
+
Stream.tap((res) => backendIdHelper.lazySet(res.backendId)),
|
|
140
|
+
Stream.map((res) => omit(res, ['backendId'])),
|
|
141
|
+
Stream.mapError((cause) =>
|
|
142
|
+
cause._tag === 'RpcClientError' && Socket.isSocketError(cause.cause)
|
|
143
|
+
? new IsOfflineError({ cause: cause.cause })
|
|
144
|
+
: cause._tag === 'InvalidPullError'
|
|
145
|
+
? cause
|
|
146
|
+
: InvalidPullError.make({ cause }),
|
|
147
|
+
),
|
|
148
|
+
Stream.withSpan('pull'),
|
|
149
|
+
),
|
|
150
|
+
|
|
151
|
+
push: (batch) =>
|
|
152
|
+
Effect.gen(function* () {
|
|
153
|
+
if (batch.length === 0) {
|
|
154
|
+
return
|
|
155
|
+
}
|
|
156
|
+
|
|
157
|
+
return yield* rpcClient.SyncWsRpc.Push({
|
|
158
|
+
storeId,
|
|
159
|
+
payload,
|
|
160
|
+
batch,
|
|
161
|
+
backendId: backendIdHelper.get(),
|
|
162
|
+
}).pipe(
|
|
163
|
+
Effect.mapError((cause) =>
|
|
164
|
+
cause._tag === 'InvalidPushError'
|
|
165
|
+
? cause
|
|
166
|
+
: new InvalidPushError({ cause: new UnexpectedError({ cause }) }),
|
|
167
|
+
),
|
|
168
|
+
)
|
|
169
|
+
}).pipe(Effect.withSpan('push')),
|
|
170
|
+
ping,
|
|
171
|
+
metadata: {
|
|
172
|
+
name: '@livestore/cf-sync',
|
|
173
|
+
description: 'LiveStore sync backend implementation using Cloudflare Workers & Durable Objects',
|
|
174
|
+
protocol: 'ws',
|
|
175
|
+
url: options.url,
|
|
176
|
+
},
|
|
177
|
+
supports: {
|
|
178
|
+
pullPageInfoKnown: true,
|
|
179
|
+
pullLive: true,
|
|
180
|
+
},
|
|
181
|
+
})
|
|
182
|
+
})
|
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
import { InvalidPullError, InvalidPushError } from '@livestore/common'
|
|
2
|
+
import { Rpc, RpcGroup, Schema } from '@livestore/utils/effect'
|
|
3
|
+
import * as SyncMessage from './sync-message-types.ts'
|
|
4
|
+
|
|
5
|
+
const commonPayloadFields = {
|
|
6
|
+
/**
|
|
7
|
+
* While the storeId is already implied by the durable object, we still need the explicit storeId
|
|
8
|
+
* since a DO doesn't know its own id.name value. 🫠
|
|
9
|
+
* https://community.cloudflare.com/t/how-can-i-get-the-name-of-a-durable-object-from-itself/505961/8
|
|
10
|
+
*/
|
|
11
|
+
storeId: Schema.String,
|
|
12
|
+
/** Needed for various reasons (e.g. auth) */
|
|
13
|
+
payload: Schema.optional(Schema.JsonValue),
|
|
14
|
+
}
|
|
15
|
+
|
|
16
|
+
export class SyncDoRpc extends RpcGroup.make(
|
|
17
|
+
Rpc.make('SyncDoRpc.Pull', {
|
|
18
|
+
payload: {
|
|
19
|
+
/** Omitting the cursor will start from the beginning */
|
|
20
|
+
cursor: SyncMessage.PullRequest.fields.cursor,
|
|
21
|
+
// TODO rename
|
|
22
|
+
/** Whether to keep the pull stream alive and wait for more events */
|
|
23
|
+
rpcContext: Schema.optional(
|
|
24
|
+
Schema.Struct({
|
|
25
|
+
callerContext: Schema.Struct({
|
|
26
|
+
bindingName: Schema.String,
|
|
27
|
+
durableObjectId: Schema.String,
|
|
28
|
+
}),
|
|
29
|
+
}),
|
|
30
|
+
),
|
|
31
|
+
...commonPayloadFields,
|
|
32
|
+
},
|
|
33
|
+
success: Schema.Struct({
|
|
34
|
+
rpcRequestId: Schema.String,
|
|
35
|
+
...SyncMessage.PullResponse.fields,
|
|
36
|
+
}),
|
|
37
|
+
error: InvalidPullError,
|
|
38
|
+
stream: true,
|
|
39
|
+
}),
|
|
40
|
+
Rpc.make('SyncDoRpc.Push', {
|
|
41
|
+
payload: {
|
|
42
|
+
...SyncMessage.PushRequest.fields,
|
|
43
|
+
...commonPayloadFields,
|
|
44
|
+
},
|
|
45
|
+
success: SyncMessage.PushAck,
|
|
46
|
+
error: InvalidPushError,
|
|
47
|
+
}),
|
|
48
|
+
Rpc.make('SyncDoRpc.Ping', {
|
|
49
|
+
payload: {
|
|
50
|
+
...commonPayloadFields,
|
|
51
|
+
},
|
|
52
|
+
success: Schema.Void,
|
|
53
|
+
}),
|
|
54
|
+
) {}
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
import { InvalidPullError, InvalidPushError, UnexpectedError } from '@livestore/common'
|
|
2
|
+
import { Rpc, RpcGroup, Schema } from '@livestore/utils/effect'
|
|
3
|
+
import * as SyncMessage from './sync-message-types.ts'
|
|
4
|
+
|
|
5
|
+
/**
|
|
6
|
+
* HTTP RPC Schema for LiveStore CF Sync Provider
|
|
7
|
+
*
|
|
8
|
+
* This defines the RPC endpoints available over HTTP transport.
|
|
9
|
+
* Unlike WebSocket transport which maintains persistent connections,
|
|
10
|
+
* HTTP transport uses request/response patterns for each operation.
|
|
11
|
+
*/
|
|
12
|
+
export class SyncHttpRpc extends RpcGroup.make(
|
|
13
|
+
Rpc.make('SyncHttpRpc.Pull', {
|
|
14
|
+
payload: Schema.Struct({
|
|
15
|
+
storeId: Schema.String,
|
|
16
|
+
payload: Schema.optional(Schema.JsonValue),
|
|
17
|
+
...SyncMessage.PullRequest.fields,
|
|
18
|
+
}),
|
|
19
|
+
success: SyncMessage.PullResponse,
|
|
20
|
+
error: InvalidPullError,
|
|
21
|
+
stream: true,
|
|
22
|
+
}),
|
|
23
|
+
Rpc.make('SyncHttpRpc.Push', {
|
|
24
|
+
payload: Schema.Struct({
|
|
25
|
+
storeId: Schema.String,
|
|
26
|
+
payload: Schema.optional(Schema.JsonValue),
|
|
27
|
+
...SyncMessage.PushRequest.fields,
|
|
28
|
+
}),
|
|
29
|
+
success: SyncMessage.PushAck,
|
|
30
|
+
error: InvalidPushError,
|
|
31
|
+
}),
|
|
32
|
+
Rpc.make('SyncHttpRpc.Ping', {
|
|
33
|
+
payload: Schema.Struct({
|
|
34
|
+
storeId: Schema.String,
|
|
35
|
+
payload: Schema.optional(Schema.JsonValue),
|
|
36
|
+
}),
|
|
37
|
+
success: SyncMessage.Pong,
|
|
38
|
+
error: UnexpectedError,
|
|
39
|
+
}),
|
|
40
|
+
) {}
|
package/src/common/mod.ts
CHANGED
|
@@ -1,8 +1,15 @@
|
|
|
1
1
|
import { Schema } from '@livestore/utils/effect'
|
|
2
2
|
|
|
3
|
-
export
|
|
3
|
+
export type { CfTypes } from '@livestore/common-cf'
|
|
4
|
+
|
|
5
|
+
export { SyncHttpRpc } from './http-rpc-schema.ts'
|
|
6
|
+
export * as SyncMessage from './sync-message-types.ts'
|
|
4
7
|
|
|
5
8
|
export const SearchParamsSchema = Schema.Struct({
|
|
6
9
|
storeId: Schema.String,
|
|
7
10
|
payload: Schema.compose(Schema.StringFromUriComponent, Schema.parseJson(Schema.JsonValue)).pipe(Schema.UndefinedOr),
|
|
11
|
+
// NOTE `do-rpc` is handled differently
|
|
12
|
+
transport: Schema.Union(Schema.Literal('http'), Schema.Literal('ws')),
|
|
8
13
|
})
|
|
14
|
+
|
|
15
|
+
export type SearchParams = typeof SearchParamsSchema.Type
|