@livestore/sync-cf 0.4.0-dev.21 → 0.4.0-dev.23
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +6 -7
- package/dist/.tsbuildinfo +1 -1
- package/dist/cf-worker/do/durable-object.d.ts.map +1 -1
- package/dist/cf-worker/do/durable-object.js +12 -8
- package/dist/cf-worker/do/durable-object.js.map +1 -1
- package/dist/cf-worker/do/layer.d.ts +1 -1
- package/dist/cf-worker/do/layer.d.ts.map +1 -1
- package/dist/cf-worker/do/layer.js +2 -2
- package/dist/cf-worker/do/layer.js.map +1 -1
- package/dist/cf-worker/do/pull.d.ts +7 -2
- package/dist/cf-worker/do/pull.d.ts.map +1 -1
- package/dist/cf-worker/do/pull.js +12 -6
- package/dist/cf-worker/do/pull.js.map +1 -1
- package/dist/cf-worker/do/push.d.ts +5 -4
- package/dist/cf-worker/do/push.d.ts.map +1 -1
- package/dist/cf-worker/do/push.js +18 -11
- package/dist/cf-worker/do/push.js.map +1 -1
- package/dist/cf-worker/do/sqlite.d.ts.map +1 -1
- package/dist/cf-worker/do/sqlite.js.map +1 -1
- package/dist/cf-worker/do/sync-storage.d.ts +1 -1
- package/dist/cf-worker/do/sync-storage.d.ts.map +1 -1
- package/dist/cf-worker/do/sync-storage.js +2 -1
- package/dist/cf-worker/do/sync-storage.js.map +1 -1
- package/dist/cf-worker/do/transport/do-rpc-server.d.ts.map +1 -1
- package/dist/cf-worker/do/transport/do-rpc-server.js +13 -7
- package/dist/cf-worker/do/transport/do-rpc-server.js.map +1 -1
- package/dist/cf-worker/do/transport/http-rpc-server.d.ts +2 -1
- package/dist/cf-worker/do/transport/http-rpc-server.d.ts.map +1 -1
- package/dist/cf-worker/do/transport/http-rpc-server.js +17 -14
- package/dist/cf-worker/do/transport/http-rpc-server.js.map +1 -1
- package/dist/cf-worker/do/transport/ws-rpc-server.d.ts +2 -1
- package/dist/cf-worker/do/transport/ws-rpc-server.d.ts.map +1 -1
- package/dist/cf-worker/do/transport/ws-rpc-server.js +30 -8
- package/dist/cf-worker/do/transport/ws-rpc-server.js.map +1 -1
- package/dist/cf-worker/shared.d.ts +60 -15
- package/dist/cf-worker/shared.d.ts.map +1 -1
- package/dist/cf-worker/shared.js +27 -0
- package/dist/cf-worker/shared.js.map +1 -1
- package/dist/cf-worker/worker.d.ts +31 -31
- package/dist/cf-worker/worker.d.ts.map +1 -1
- package/dist/cf-worker/worker.js +27 -30
- package/dist/cf-worker/worker.js.map +1 -1
- package/dist/client/transport/do-rpc-client.d.ts.map +1 -1
- package/dist/client/transport/do-rpc-client.js +11 -7
- package/dist/client/transport/do-rpc-client.js.map +1 -1
- package/dist/client/transport/http-rpc-client.d.ts.map +1 -1
- package/dist/client/transport/http-rpc-client.js +10 -6
- package/dist/client/transport/http-rpc-client.js.map +1 -1
- package/dist/client/transport/ws-rpc-client.d.ts.map +1 -1
- package/dist/client/transport/ws-rpc-client.js +11 -11
- package/dist/client/transport/ws-rpc-client.js.map +1 -1
- package/dist/common/do-rpc-schema.d.ts +3 -3
- package/dist/common/do-rpc-schema.d.ts.map +1 -1
- package/dist/common/do-rpc-schema.js +3 -3
- package/dist/common/do-rpc-schema.js.map +1 -1
- package/dist/common/http-rpc-schema.d.ts +3 -3
- package/dist/common/http-rpc-schema.d.ts.map +1 -1
- package/dist/common/http-rpc-schema.js +3 -3
- package/dist/common/http-rpc-schema.js.map +1 -1
- package/dist/common/sync-message-types.d.ts +2 -2
- package/dist/common/ws-rpc-schema.d.ts +3 -3
- package/dist/common/ws-rpc-schema.d.ts.map +1 -1
- package/dist/common/ws-rpc-schema.js +3 -3
- package/dist/common/ws-rpc-schema.js.map +1 -1
- package/package.json +71 -13
- package/src/cf-worker/do/durable-object.ts +18 -10
- package/src/cf-worker/do/layer.ts +4 -3
- package/src/cf-worker/do/pull.ts +28 -9
- package/src/cf-worker/do/push.ts +29 -10
- package/src/cf-worker/do/sqlite.ts +1 -0
- package/src/cf-worker/do/sync-storage.ts +4 -2
- package/src/cf-worker/do/transport/do-rpc-server.ts +18 -7
- package/src/cf-worker/do/transport/http-rpc-server.ts +27 -21
- package/src/cf-worker/do/transport/ws-rpc-server.ts +40 -12
- package/src/cf-worker/shared.ts +89 -11
- package/src/cf-worker/worker.ts +64 -47
- package/src/client/transport/do-rpc-client.ts +20 -14
- package/src/client/transport/http-rpc-client.ts +19 -13
- package/src/client/transport/ws-rpc-client.ts +39 -36
- package/src/common/do-rpc-schema.ts +4 -3
- package/src/common/http-rpc-schema.ts +4 -3
- package/src/common/ws-rpc-schema.ts +4 -3
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
/// <reference types="@cloudflare/workers-types" />
|
|
2
2
|
|
|
3
3
|
import { DurableObject } from 'cloudflare:workers'
|
|
4
|
+
|
|
4
5
|
import { type CfTypes, setupDurableObjectWebSocketRpc } from '@livestore/common-cf'
|
|
5
6
|
import { CfDeclare } from '@livestore/common-cf/declare'
|
|
6
7
|
import {
|
|
@@ -14,8 +15,10 @@ import {
|
|
|
14
15
|
Schema,
|
|
15
16
|
type Scope,
|
|
16
17
|
} from '@livestore/utils/effect'
|
|
18
|
+
|
|
17
19
|
import {
|
|
18
20
|
type Env,
|
|
21
|
+
extractForwardedHeaders,
|
|
19
22
|
type MakeDurableObjectClassOptions,
|
|
20
23
|
matchSyncRequest,
|
|
21
24
|
type SyncBackendRpcInterface,
|
|
@@ -33,7 +36,7 @@ declare class Response extends CfDeclare.Response {}
|
|
|
33
36
|
declare class WebSocketPair extends CfDeclare.WebSocketPair {}
|
|
34
37
|
declare class WebSocketRequestResponsePair extends CfDeclare.WebSocketRequestResponsePair {}
|
|
35
38
|
|
|
36
|
-
const DurableObjectBase = DurableObject
|
|
39
|
+
const DurableObjectBase = DurableObject as any as new (
|
|
37
40
|
state: CfTypes.DurableObjectState,
|
|
38
41
|
env: Env,
|
|
39
42
|
) => CfTypes.DurableObject & { ctx: CfTypes.DurableObjectState; env: Env }
|
|
@@ -87,14 +90,14 @@ export const makeDurableObject: MakeDurableObjectClass = (options) => {
|
|
|
87
90
|
|
|
88
91
|
const Logging = Logger.consoleWithThread('SyncDo')
|
|
89
92
|
|
|
90
|
-
const Observability = options?.otel?.baseUrl
|
|
91
|
-
? Otlp.layer({
|
|
93
|
+
const Observability: Layer.Layer<never> = options?.otel?.baseUrl !== undefined
|
|
94
|
+
? (Otlp.layer({
|
|
92
95
|
baseUrl: options.otel.baseUrl,
|
|
93
96
|
tracerExportInterval: 50,
|
|
94
97
|
resource: {
|
|
95
98
|
serviceName: options.otel.serviceName ?? 'sync-cf-do',
|
|
96
99
|
},
|
|
97
|
-
}).pipe(Layer.provide(FetchHttpClient.layer))
|
|
100
|
+
}).pipe(Layer.provide(FetchHttpClient.layer)) as Layer.Layer<never>)
|
|
98
101
|
: Layer.empty
|
|
99
102
|
|
|
100
103
|
return class SyncBackendDOBase extends DurableObjectBase implements SyncBackendRpcInterface {
|
|
@@ -106,7 +109,7 @@ export const makeDurableObject: MakeDurableObjectClass = (options) => {
|
|
|
106
109
|
const WebSocketRpcServerLive = makeRpcServer({ doSelf: this, doOptions: options })
|
|
107
110
|
|
|
108
111
|
// This registers the `webSocketMessage` and `webSocketClose` handlers
|
|
109
|
-
if (enabledTransports.has('ws')) {
|
|
112
|
+
if (enabledTransports.has('ws') === true) {
|
|
110
113
|
setupDurableObjectWebSocketRpc({
|
|
111
114
|
doSelf: this,
|
|
112
115
|
rpcLayer: WebSocketRpcServerLive,
|
|
@@ -142,7 +145,7 @@ export const makeDurableObject: MakeDurableObjectClass = (options) => {
|
|
|
142
145
|
}
|
|
143
146
|
}
|
|
144
147
|
|
|
145
|
-
fetch = async (request: Request): Promise<Response> =>
|
|
148
|
+
override fetch = async (request: Request): Promise<Response> =>
|
|
146
149
|
Effect.gen(this, function* () {
|
|
147
150
|
const searchParams = matchSyncRequest(request)
|
|
148
151
|
if (searchParams === undefined) {
|
|
@@ -155,16 +158,20 @@ export const makeDurableObject: MakeDurableObjectClass = (options) => {
|
|
|
155
158
|
throw new Error(`Transport ${transport} is not enabled (based on \`options.enabledTransports\`)`)
|
|
156
159
|
}
|
|
157
160
|
|
|
161
|
+
// Extract headers to forward based on configuration (available for all transports)
|
|
162
|
+
const headers = extractForwardedHeaders(request, options?.forwardHeaders)
|
|
163
|
+
|
|
158
164
|
if (transport === 'http') {
|
|
159
|
-
return yield* this.handleHttp(request)
|
|
165
|
+
return yield* this.handleHttp(request, headers)
|
|
160
166
|
}
|
|
161
167
|
|
|
162
168
|
if (transport === 'ws') {
|
|
163
169
|
const { 0: client, 1: server } = new WebSocketPair()
|
|
164
170
|
|
|
165
171
|
// Since we're using websocket hibernation, we need to remember the storeId for subsequent `webSocketMessage` calls
|
|
172
|
+
// Also store forwarded headers so they're available after hibernation resume
|
|
166
173
|
server.serializeAttachment(
|
|
167
|
-
Schema.encodeSync(WebSocketAttachmentSchema)({ storeId, payload, pullRequestIds: [] }),
|
|
174
|
+
Schema.encodeSync(WebSocketAttachmentSchema)({ storeId, payload, pullRequestIds: [], headers }),
|
|
168
175
|
)
|
|
169
176
|
|
|
170
177
|
// See https://developers.cloudflare.com/durable-objects/examples/websocket-hibernation-server
|
|
@@ -220,10 +227,11 @@ export const makeDurableObject: MakeDurableObjectClass = (options) => {
|
|
|
220
227
|
*
|
|
221
228
|
* Requires the `enable_request_signal` compatibility flag to properly support `pull` streaming responses
|
|
222
229
|
*/
|
|
223
|
-
private handleHttp = (request: CfTypes.Request) =>
|
|
230
|
+
private handleHttp = (request: CfTypes.Request, forwardedHeaders: Record<string, string> | undefined) =>
|
|
224
231
|
createHttpRpcHandler({
|
|
225
232
|
request,
|
|
226
|
-
responseHeaders: options
|
|
233
|
+
...(options?.http?.responseHeaders !== undefined ? { responseHeaders: options.http.responseHeaders } : {}),
|
|
234
|
+
...(forwardedHeaders !== undefined ? { forwardedHeaders } : {}),
|
|
227
235
|
}).pipe(Effect.withSpan('@livestore/sync-cf:durable-object:handleHttp'))
|
|
228
236
|
|
|
229
237
|
private runEffectAsPromise = <T, E = never>(effect: Effect.Effect<T, E, Scope.Scope>): Promise<T> =>
|
|
@@ -1,9 +1,10 @@
|
|
|
1
1
|
import { UnknownError } from '@livestore/common'
|
|
2
|
-
import { EventSequenceNumber, State } from '@livestore/common/schema'
|
|
3
2
|
import type { CfTypes } from '@livestore/common-cf'
|
|
3
|
+
import { EventSequenceNumber, State } from '@livestore/common/schema'
|
|
4
4
|
import { shouldNeverHappen } from '@livestore/utils'
|
|
5
5
|
import { Effect, Predicate } from '@livestore/utils/effect'
|
|
6
6
|
import { nanoid } from '@livestore/utils/nanoid'
|
|
7
|
+
|
|
7
8
|
import type { Env, MakeDurableObjectClassOptions, RpcSubscription } from '../shared.ts'
|
|
8
9
|
import { contextTable, eventlogTable } from './sqlite.ts'
|
|
9
10
|
import { makeStorage } from './sync-storage.ts'
|
|
@@ -27,7 +28,7 @@ export class DoCtx extends Effect.Service<DoCtx>()('DoCtx', {
|
|
|
27
28
|
}
|
|
28
29
|
|
|
29
30
|
const getStoreId = (from: CfTypes.Request | { storeId: string }) => {
|
|
30
|
-
if (Predicate.hasProperty(from, 'url')) {
|
|
31
|
+
if (Predicate.hasProperty(from, 'url') === true) {
|
|
31
32
|
const url = new URL(from.url)
|
|
32
33
|
return (
|
|
33
34
|
url.searchParams.get('storeId') ?? shouldNeverHappen(`No storeId provided in request URL search params`)
|
|
@@ -42,7 +43,7 @@ export class DoCtx extends Effect.Service<DoCtx>()('DoCtx', {
|
|
|
42
43
|
const opt = doOptions?.storage
|
|
43
44
|
if (opt?._tag === 'd1') {
|
|
44
45
|
const db = (doSelf.env as any)[opt.binding]
|
|
45
|
-
if (
|
|
46
|
+
if (db == null) {
|
|
46
47
|
return yield* UnknownError.make({ cause: new Error(`D1 binding '${opt.binding}' not found on env`) })
|
|
47
48
|
}
|
|
48
49
|
return { _tag: 'd1' as const, db }
|
package/src/cf-worker/do/pull.ts
CHANGED
|
@@ -1,8 +1,10 @@
|
|
|
1
|
-
import { BackendIdMismatchError,
|
|
1
|
+
import { BackendIdMismatchError, SyncBackend, UnknownError } from '@livestore/common'
|
|
2
2
|
import { splitChunkBySize } from '@livestore/common/sync'
|
|
3
3
|
import { Chunk, Effect, Option, Schema, Stream } from '@livestore/utils/effect'
|
|
4
|
+
|
|
4
5
|
import { MAX_PULL_EVENTS_PER_MESSAGE, MAX_WS_MESSAGE_BYTES } from '../../common/constants.ts'
|
|
5
6
|
import { SyncMessage } from '../../common/mod.ts'
|
|
7
|
+
import type { ForwardedHeaders } from '../shared.ts'
|
|
6
8
|
import { DoCtx } from './layer.ts'
|
|
7
9
|
|
|
8
10
|
const encodePullResponse = Schema.encodeSync(SyncMessage.PullResponse)
|
|
@@ -16,15 +18,28 @@ const encodePullResponse = Schema.encodeSync(SyncMessage.PullResponse)
|
|
|
16
18
|
// DO RPC:
|
|
17
19
|
// - Further chunks will be emitted manually in `push.ts`
|
|
18
20
|
// - If the client sends a `Interrupt` RPC message, TODO
|
|
19
|
-
export const makeEndingPullStream = (
|
|
20
|
-
req
|
|
21
|
-
payload
|
|
22
|
-
|
|
21
|
+
export const makeEndingPullStream = ({
|
|
22
|
+
req,
|
|
23
|
+
payload,
|
|
24
|
+
headers,
|
|
25
|
+
}: {
|
|
26
|
+
req: SyncMessage.PullRequest
|
|
27
|
+
payload: Schema.JsonValue | undefined
|
|
28
|
+
headers: ForwardedHeaders | undefined
|
|
29
|
+
}): Stream.Stream<SyncMessage.PullResponse, UnknownError | BackendIdMismatchError, DoCtx> =>
|
|
23
30
|
Effect.gen(function* () {
|
|
24
31
|
const { doOptions, backendId, storeId, storage } = yield* DoCtx
|
|
25
32
|
|
|
26
|
-
if (doOptions?.onPull) {
|
|
27
|
-
yield* Effect.tryAll(() =>
|
|
33
|
+
if (doOptions?.onPull !== undefined) {
|
|
34
|
+
yield* Effect.tryAll(() =>
|
|
35
|
+
doOptions.onPull!(req, {
|
|
36
|
+
storeId,
|
|
37
|
+
...(payload !== undefined ? { payload } : {}),
|
|
38
|
+
...(headers !== undefined ? { headers } : {}),
|
|
39
|
+
}),
|
|
40
|
+
).pipe(
|
|
41
|
+
UnknownError.mapToUnknownError,
|
|
42
|
+
)
|
|
28
43
|
}
|
|
29
44
|
|
|
30
45
|
if (req.cursor._tag === 'Some' && req.cursor.value.backendId !== backendId) {
|
|
@@ -61,7 +76,7 @@ export const makeEndingPullStream = (
|
|
|
61
76
|
}),
|
|
62
77
|
Stream.tap(
|
|
63
78
|
Effect.fn(function* (res) {
|
|
64
|
-
if (doOptions?.onPullRes) {
|
|
79
|
+
if (doOptions?.onPullRes !== undefined) {
|
|
65
80
|
yield* Effect.tryAll(() => doOptions.onPullRes!(res)).pipe(UnknownError.mapToUnknownError)
|
|
66
81
|
}
|
|
67
82
|
}),
|
|
@@ -70,6 +85,10 @@ export const makeEndingPullStream = (
|
|
|
70
85
|
)
|
|
71
86
|
}).pipe(
|
|
72
87
|
Stream.unwrap,
|
|
73
|
-
Stream.mapError((cause) =>
|
|
88
|
+
Stream.mapError((cause) =>
|
|
89
|
+
cause._tag === 'BackendIdMismatchError' || cause._tag === 'UnknownError'
|
|
90
|
+
? cause
|
|
91
|
+
: new UnknownError({ cause }),
|
|
92
|
+
),
|
|
74
93
|
Stream.withSpan('cloudflare-provider:pull'),
|
|
75
94
|
)
|
package/src/cf-worker/do/push.ts
CHANGED
|
@@ -1,30 +1,39 @@
|
|
|
1
1
|
import {
|
|
2
2
|
BackendIdMismatchError,
|
|
3
|
-
InvalidPushError,
|
|
4
3
|
ServerAheadError,
|
|
5
4
|
SyncBackend,
|
|
6
5
|
UnknownError,
|
|
7
6
|
} from '@livestore/common'
|
|
8
|
-
import { splitChunkBySize } from '@livestore/common/sync'
|
|
9
7
|
import { type CfTypes, emitStreamResponse } from '@livestore/common-cf'
|
|
8
|
+
import { splitChunkBySize } from '@livestore/common/sync'
|
|
10
9
|
import { Chunk, Effect, Option, type RpcMessage, Schema } from '@livestore/utils/effect'
|
|
10
|
+
|
|
11
11
|
import { MAX_PUSH_EVENTS_PER_REQUEST, MAX_WS_MESSAGE_BYTES } from '../../common/constants.ts'
|
|
12
12
|
import { SyncMessage } from '../../common/mod.ts'
|
|
13
|
-
import {
|
|
13
|
+
import {
|
|
14
|
+
type Env,
|
|
15
|
+
type ForwardedHeaders,
|
|
16
|
+
type MakeDurableObjectClassOptions,
|
|
17
|
+
type StoreId,
|
|
18
|
+
WebSocketAttachmentSchema,
|
|
19
|
+
} from '../shared.ts'
|
|
14
20
|
import { DoCtx } from './layer.ts'
|
|
15
21
|
|
|
16
22
|
const encodePullResponse = Schema.encodeSync(SyncMessage.PullResponse)
|
|
23
|
+
const jsonStringify = Schema.encodeSync(Schema.parseJson())
|
|
17
24
|
type PullBatchItem = SyncMessage.PullResponse['batch'][number]
|
|
18
25
|
|
|
19
26
|
export const makePush =
|
|
20
27
|
({
|
|
21
28
|
payload,
|
|
29
|
+
headers,
|
|
22
30
|
options,
|
|
23
31
|
storeId,
|
|
24
32
|
ctx,
|
|
25
33
|
env,
|
|
26
34
|
}: {
|
|
27
35
|
payload: Schema.JsonValue | undefined
|
|
36
|
+
headers: ForwardedHeaders | undefined
|
|
28
37
|
options: MakeDurableObjectClassOptions | undefined
|
|
29
38
|
storeId: StoreId
|
|
30
39
|
ctx: CfTypes.DurableObjectState
|
|
@@ -39,8 +48,14 @@ export const makePush =
|
|
|
39
48
|
return SyncMessage.PushAck.make({})
|
|
40
49
|
}
|
|
41
50
|
|
|
42
|
-
if (options?.onPush) {
|
|
43
|
-
yield* Effect.tryAll(() =>
|
|
51
|
+
if (options?.onPush !== undefined) {
|
|
52
|
+
yield* Effect.tryAll(() =>
|
|
53
|
+
options.onPush!(pushRequest, {
|
|
54
|
+
storeId,
|
|
55
|
+
...(payload !== undefined ? { payload } : {}),
|
|
56
|
+
...(headers !== undefined ? { headers } : {}),
|
|
57
|
+
}),
|
|
58
|
+
).pipe(
|
|
44
59
|
UnknownError.mapToUnknownError,
|
|
45
60
|
)
|
|
46
61
|
}
|
|
@@ -126,13 +141,13 @@ export const makePush =
|
|
|
126
141
|
if (connectedClients.length > 0) {
|
|
127
142
|
for (const { response, encoded } of responses) {
|
|
128
143
|
// Only calling once for now.
|
|
129
|
-
if (options?.onPullRes) {
|
|
144
|
+
if (options?.onPullRes !== undefined) {
|
|
130
145
|
yield* Effect.tryAll(() => options.onPullRes!(response)).pipe(UnknownError.mapToUnknownError)
|
|
131
146
|
}
|
|
132
147
|
|
|
133
148
|
// NOTE we're also sending the pullRes chunk to the pushing ws client as confirmation
|
|
134
149
|
for (const conn of connectedClients) {
|
|
135
|
-
const attachment = Schema.
|
|
150
|
+
const attachment = yield* Schema.decode(WebSocketAttachmentSchema)(conn.deserializeAttachment())
|
|
136
151
|
|
|
137
152
|
// We're doing something a bit "advanced" here as we're directly emitting Effect RPC-compatible
|
|
138
153
|
// response messsages on the Effect RPC-managed websocket connection to the WS client.
|
|
@@ -143,7 +158,7 @@ export const makePush =
|
|
|
143
158
|
requestId,
|
|
144
159
|
values: [encoded],
|
|
145
160
|
}
|
|
146
|
-
conn.send(
|
|
161
|
+
conn.send(jsonStringify(res))
|
|
147
162
|
}
|
|
148
163
|
}
|
|
149
164
|
}
|
|
@@ -180,12 +195,16 @@ export const makePush =
|
|
|
180
195
|
}).pipe(
|
|
181
196
|
Effect.tap(
|
|
182
197
|
Effect.fn(function* (message) {
|
|
183
|
-
if (options?.onPushRes) {
|
|
198
|
+
if (options?.onPushRes !== undefined) {
|
|
184
199
|
yield* Effect.tryAll(() => options.onPushRes!(message)).pipe(UnknownError.mapToUnknownError)
|
|
185
200
|
}
|
|
186
201
|
}),
|
|
187
202
|
),
|
|
188
|
-
Effect.mapError((cause) =>
|
|
203
|
+
Effect.mapError((cause) =>
|
|
204
|
+
cause._tag === 'BackendIdMismatchError' || cause._tag === 'ServerAheadError' || cause._tag === 'UnknownError'
|
|
205
|
+
? cause
|
|
206
|
+
: new UnknownError({ cause }),
|
|
207
|
+
),
|
|
189
208
|
Effect.withSpan('sync-cf:do:push', { attributes: { storeId, batchSize: pushRequest.batch.length } }),
|
|
190
209
|
)
|
|
191
210
|
|
|
@@ -1,7 +1,8 @@
|
|
|
1
1
|
import { UnknownError } from '@livestore/common'
|
|
2
|
-
import type { LiveStoreEvent } from '@livestore/common/schema'
|
|
3
2
|
import type { CfTypes } from '@livestore/common-cf'
|
|
3
|
+
import type { LiveStoreEvent } from '@livestore/common/schema'
|
|
4
4
|
import { Chunk, Effect, Option, Schema, Stream } from '@livestore/utils/effect'
|
|
5
|
+
|
|
5
6
|
import { SyncMetadata } from '../../common/sync-message-types.ts'
|
|
6
7
|
import { PERSISTENCE_FORMAT_VERSION, type StoreId } from '../shared.ts'
|
|
7
8
|
import { eventlogTable } from './sqlite.ts'
|
|
@@ -51,6 +52,7 @@ export const makeStorage = (
|
|
|
51
52
|
const D1_MIN_PAGE_SIZE = 1
|
|
52
53
|
|
|
53
54
|
const decodeEventlogRows = Schema.decodeUnknownSync(Schema.Array(eventlogTable.rowSchema))
|
|
55
|
+
const jsonStringify = Schema.encodeSync(Schema.parseJson())
|
|
54
56
|
const textEncoder = new TextEncoder()
|
|
55
57
|
|
|
56
58
|
const decreaseLimit = (limit: number) => Math.max(D1_MIN_PAGE_SIZE, Math.floor(limit / 2))
|
|
@@ -120,7 +122,7 @@ export const makeStorage = (
|
|
|
120
122
|
return Option.none()
|
|
121
123
|
}
|
|
122
124
|
|
|
123
|
-
const encodedSize = textEncoder.encode(
|
|
125
|
+
const encodedSize = textEncoder.encode(jsonStringify(rawEvents)).byteLength
|
|
124
126
|
|
|
125
127
|
if (encodedSize > D1_TARGET_RESPONSE_BYTES && state.limit > D1_MIN_PAGE_SIZE) {
|
|
126
128
|
const nextLimit = decreaseLimit(state.limit)
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { UnknownError } from '@livestore/common'
|
|
2
2
|
import { type CfTypes, toDurableObjectHandler } from '@livestore/common-cf'
|
|
3
3
|
import {
|
|
4
4
|
Effect,
|
|
@@ -11,6 +11,7 @@ import {
|
|
|
11
11
|
RpcSerialization,
|
|
12
12
|
Stream,
|
|
13
13
|
} from '@livestore/utils/effect'
|
|
14
|
+
|
|
14
15
|
import { SyncDoRpc } from '../../../common/do-rpc-schema.ts'
|
|
15
16
|
import { SyncMessage } from '../../../common/mod.ts'
|
|
16
17
|
import { DoCtx, type DoCtxInput } from '../layer.ts'
|
|
@@ -39,17 +40,18 @@ export const createDoRpcHandler = (
|
|
|
39
40
|
const { rpcSubscriptions } = yield* DoCtx
|
|
40
41
|
|
|
41
42
|
// TODO rename `req.rpcContext` to something more appropriate
|
|
42
|
-
if (req.rpcContext) {
|
|
43
|
+
if (req.rpcContext !== undefined) {
|
|
43
44
|
rpcSubscriptions.set(req.storeId, {
|
|
44
45
|
storeId: req.storeId,
|
|
45
|
-
payload: req.payload,
|
|
46
46
|
subscribedAt: Date.now(),
|
|
47
47
|
requestId: Headers.get(headers, 'x-rpc-request-id').pipe(Option.getOrThrow),
|
|
48
48
|
callerContext: req.rpcContext.callerContext,
|
|
49
|
+
...(req.payload !== undefined ? { payload: req.payload } : {}),
|
|
49
50
|
})
|
|
50
51
|
}
|
|
51
52
|
|
|
52
|
-
|
|
53
|
+
// DO-RPC doesn't have HTTP headers context - headers are undefined
|
|
54
|
+
return makeEndingPullStream({ req, payload: req.payload, headers: undefined })
|
|
53
55
|
}).pipe(
|
|
54
56
|
Stream.unwrap,
|
|
55
57
|
Stream.map((res) => ({
|
|
@@ -57,18 +59,27 @@ export const createDoRpcHandler = (
|
|
|
57
59
|
rpcRequestId: Headers.get(headers, 'x-rpc-request-id').pipe(Option.getOrThrow),
|
|
58
60
|
})),
|
|
59
61
|
Stream.provideLayer(DoCtx.Default({ ...input, from: { storeId: req.storeId } })),
|
|
60
|
-
Stream.mapError((cause) =>
|
|
62
|
+
Stream.mapError((cause) =>
|
|
63
|
+
cause._tag === 'UnknownError' || cause._tag === 'BackendIdMismatchError'
|
|
64
|
+
? cause
|
|
65
|
+
: new UnknownError({ cause }),
|
|
66
|
+
),
|
|
61
67
|
Stream.tapErrorCause(Effect.log),
|
|
62
68
|
),
|
|
63
69
|
'SyncDoRpc.Push': (req) =>
|
|
64
70
|
Effect.gen(this, function* () {
|
|
65
71
|
const { doOptions, ctx, env, storeId } = yield* DoCtx
|
|
66
|
-
|
|
72
|
+
// DO-RPC doesn't have HTTP headers context - headers are undefined
|
|
73
|
+
const push = makePush({ storeId, payload: req.payload, headers: undefined, options: doOptions, ctx, env })
|
|
67
74
|
|
|
68
75
|
return yield* push(req)
|
|
69
76
|
}).pipe(
|
|
70
77
|
Effect.provide(DoCtx.Default({ ...input, from: { storeId: req.storeId } })),
|
|
71
|
-
Effect.mapError((cause) =>
|
|
78
|
+
Effect.mapError((cause) =>
|
|
79
|
+
cause._tag === 'UnknownError' || cause._tag === 'ServerAheadError' || cause._tag === 'BackendIdMismatchError'
|
|
80
|
+
? cause
|
|
81
|
+
: new UnknownError({ cause }),
|
|
82
|
+
),
|
|
72
83
|
Effect.tapCauseLogPretty,
|
|
73
84
|
),
|
|
74
85
|
})
|
|
@@ -1,45 +1,50 @@
|
|
|
1
1
|
import type { CfTypes } from '@livestore/common-cf'
|
|
2
2
|
import { Effect, HttpApp, Layer, RpcSerialization, RpcServer } from '@livestore/utils/effect'
|
|
3
|
+
|
|
3
4
|
import { SyncHttpRpc } from '../../../common/http-rpc-schema.ts'
|
|
4
5
|
import * as SyncMessage from '../../../common/sync-message-types.ts'
|
|
6
|
+
import { headersRecordToMap } from '../../shared.ts'
|
|
5
7
|
import { DoCtx } from '../layer.ts'
|
|
6
8
|
import { makeEndingPullStream } from '../pull.ts'
|
|
7
9
|
import { makePush } from '../push.ts'
|
|
8
10
|
|
|
9
|
-
export const createHttpRpcHandler = ({
|
|
11
|
+
export const createHttpRpcHandler = Effect.fn('createHttpRpcHandler')(function* ({
|
|
10
12
|
request,
|
|
11
13
|
responseHeaders,
|
|
14
|
+
forwardedHeaders,
|
|
12
15
|
}: {
|
|
13
16
|
request: CfTypes.Request
|
|
14
17
|
responseHeaders?: Record<string, string>
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
}
|
|
18
|
+
forwardedHeaders?: Record<string, string>
|
|
19
|
+
}) {
|
|
20
|
+
const handlerLayer = createHttpRpcLayer(forwardedHeaders)
|
|
21
|
+
const httpApp = RpcServer.toHttpApp(SyncHttpRpc).pipe(Effect.provide(handlerLayer))
|
|
22
|
+
const webHandler = yield* httpApp.pipe(Effect.map(HttpApp.toWebHandler))
|
|
23
|
+
|
|
24
|
+
const response = yield* Effect.promise(
|
|
25
|
+
() => webHandler(request as TODO as Request) as TODO as Promise<CfTypes.Response>,
|
|
26
|
+
).pipe(Effect.timeout(10000))
|
|
27
|
+
|
|
28
|
+
if (responseHeaders !== undefined) {
|
|
29
|
+
for (const [key, value] of Object.entries(responseHeaders)) {
|
|
30
|
+
response.headers.set(key, value)
|
|
29
31
|
}
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
return response
|
|
35
|
+
})
|
|
30
36
|
|
|
31
|
-
|
|
32
|
-
|
|
37
|
+
const createHttpRpcLayer = (forwardedHeaders: Record<string, string> | undefined) => {
|
|
38
|
+
const headers = headersRecordToMap(forwardedHeaders)
|
|
33
39
|
|
|
34
|
-
const createHttpRpcLayer =
|
|
35
40
|
// TODO implement admin requests
|
|
36
|
-
SyncHttpRpc.toLayer({
|
|
37
|
-
'SyncHttpRpc.Pull': (req) => makeEndingPullStream(req, req.payload),
|
|
41
|
+
return SyncHttpRpc.toLayer({
|
|
42
|
+
'SyncHttpRpc.Pull': (req) => makeEndingPullStream({ req, payload: req.payload, headers }),
|
|
38
43
|
|
|
39
44
|
'SyncHttpRpc.Push': (req) =>
|
|
40
45
|
Effect.gen(function* () {
|
|
41
46
|
const { ctx, env, doOptions, storeId } = yield* DoCtx
|
|
42
|
-
const push = makePush({ payload: undefined, options: doOptions, storeId, ctx, env })
|
|
47
|
+
const push = makePush({ payload: undefined, headers, options: doOptions, storeId, ctx, env })
|
|
43
48
|
|
|
44
49
|
return yield* push(req)
|
|
45
50
|
}),
|
|
@@ -49,3 +54,4 @@ const createHttpRpcLayer =
|
|
|
49
54
|
Layer.provideMerge(RpcServer.layerProtocolHttp({ path: '/http-rpc' })),
|
|
50
55
|
Layer.provideMerge(RpcSerialization.layerJson),
|
|
51
56
|
)
|
|
57
|
+
}
|
|
@@ -1,34 +1,62 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import {
|
|
1
|
+
import { UnknownError } from '@livestore/common'
|
|
2
|
+
import { WsContext } from '@livestore/common-cf'
|
|
3
|
+
import { Effect, identity, Layer, RpcServer, Schema, Stream } from '@livestore/utils/effect'
|
|
4
|
+
|
|
3
5
|
import { SyncWsRpc } from '../../../common/ws-rpc-schema.ts'
|
|
6
|
+
import { headersRecordToMap, WebSocketAttachmentSchema } from '../../shared.ts'
|
|
4
7
|
import { DoCtx, type DoCtxInput } from '../layer.ts'
|
|
5
8
|
import { makeEndingPullStream } from '../pull.ts'
|
|
6
9
|
import { makePush } from '../push.ts'
|
|
7
10
|
|
|
8
11
|
export const makeRpcServer = ({ doSelf, doOptions }: Omit<DoCtxInput, 'from'>) => {
|
|
9
|
-
// TODO implement admin requests
|
|
10
12
|
const handlersLayer = SyncWsRpc.toLayer({
|
|
11
13
|
'SyncWsRpc.Pull': (req) =>
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
req.
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
14
|
+
Effect.gen(function* () {
|
|
15
|
+
const headers = yield* getForwardedHeaders
|
|
16
|
+
return makeEndingPullStream({ req, payload: req.payload, headers }).pipe(
|
|
17
|
+
// Needed to keep the stream alive on the client side for phase 2 (i.e. not send the `Exit` stream RPC message)
|
|
18
|
+
req.live === true ? Stream.concat(Stream.never) : identity,
|
|
19
|
+
Stream.provideLayer(DoCtx.Default({ doSelf, doOptions, from: { storeId: req.storeId } })),
|
|
20
|
+
Stream.mapError((cause) =>
|
|
21
|
+
cause._tag === 'UnknownError' || cause._tag === 'BackendIdMismatchError'
|
|
22
|
+
? cause
|
|
23
|
+
: new UnknownError({ cause }),
|
|
24
|
+
),
|
|
25
|
+
)
|
|
26
|
+
}).pipe(Stream.unwrap),
|
|
19
27
|
'SyncWsRpc.Push': (req) =>
|
|
20
28
|
Effect.gen(function* () {
|
|
21
29
|
const { doOptions, storeId, ctx, env } = yield* DoCtx
|
|
30
|
+
const headers = yield* getForwardedHeaders
|
|
22
31
|
|
|
23
|
-
const push = makePush({ options: doOptions, storeId, payload: req.payload, ctx, env })
|
|
32
|
+
const push = makePush({ options: doOptions, storeId, payload: req.payload, headers, ctx, env })
|
|
24
33
|
|
|
25
34
|
return yield* push(req)
|
|
26
35
|
}).pipe(
|
|
27
36
|
Effect.provide(DoCtx.Default({ doSelf, doOptions, from: { storeId: req.storeId } })),
|
|
28
|
-
Effect.mapError((cause) =>
|
|
37
|
+
Effect.mapError((cause) =>
|
|
38
|
+
cause._tag === 'UnknownError' || cause._tag === 'ServerAheadError' || cause._tag === 'BackendIdMismatchError'
|
|
39
|
+
? cause
|
|
40
|
+
: new UnknownError({ cause }),
|
|
41
|
+
),
|
|
29
42
|
Effect.tapCauseLogPretty,
|
|
30
43
|
),
|
|
31
44
|
})
|
|
32
45
|
|
|
33
46
|
return RpcServer.layer(SyncWsRpc).pipe(Layer.provide(handlersLayer))
|
|
34
47
|
}
|
|
48
|
+
|
|
49
|
+
/** Extracts forwarded headers from the WebSocket attachment */
|
|
50
|
+
const getForwardedHeaders = Effect.gen(function* () {
|
|
51
|
+
const { ws } = yield* WsContext
|
|
52
|
+
const attachment = ws.deserializeAttachment()
|
|
53
|
+
const decoded = Schema.decodeUnknownEither(WebSocketAttachmentSchema)(attachment)
|
|
54
|
+
if (decoded._tag === 'Left') {
|
|
55
|
+
yield* Effect.logError('Failed to decode WebSocket attachment for forwarded headers', { error: decoded.left })
|
|
56
|
+
ws.close(1011, 'invalid-attachment')
|
|
57
|
+
return yield* Effect.die('Invalid WebSocket attachment (headers decode failed)')
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
const headers = headersRecordToMap(decoded.right.headers)
|
|
61
|
+
return headers
|
|
62
|
+
})
|