@livestore/sync-cf 0.4.0-dev.3 → 0.4.0-dev.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +60 -0
- package/dist/.tsbuildinfo +1 -1
- package/dist/cf-worker/do/durable-object.d.ts +45 -0
- package/dist/cf-worker/do/durable-object.d.ts.map +1 -0
- package/dist/cf-worker/do/durable-object.js +154 -0
- package/dist/cf-worker/do/durable-object.js.map +1 -0
- package/dist/cf-worker/do/layer.d.ts +34 -0
- package/dist/cf-worker/do/layer.d.ts.map +1 -0
- package/dist/cf-worker/do/layer.js +68 -0
- package/dist/cf-worker/do/layer.js.map +1 -0
- package/dist/cf-worker/do/pull.d.ts +6 -0
- package/dist/cf-worker/do/pull.d.ts.map +1 -0
- package/dist/cf-worker/do/pull.js +39 -0
- package/dist/cf-worker/do/pull.js.map +1 -0
- package/dist/cf-worker/do/push.d.ts +14 -0
- package/dist/cf-worker/do/push.d.ts.map +1 -0
- package/dist/cf-worker/do/push.js +99 -0
- package/dist/cf-worker/do/push.js.map +1 -0
- package/dist/cf-worker/do/sqlite.d.ts +196 -0
- package/dist/cf-worker/do/sqlite.d.ts.map +1 -0
- package/dist/cf-worker/do/sqlite.js +27 -0
- package/dist/cf-worker/do/sqlite.js.map +1 -0
- package/dist/cf-worker/do/sync-storage.d.ts +17 -0
- package/dist/cf-worker/do/sync-storage.d.ts.map +1 -0
- package/dist/cf-worker/do/sync-storage.js +73 -0
- package/dist/cf-worker/do/sync-storage.js.map +1 -0
- package/dist/cf-worker/do/transport/do-rpc-server.d.ts +8 -0
- package/dist/cf-worker/do/transport/do-rpc-server.d.ts.map +1 -0
- package/dist/cf-worker/do/transport/do-rpc-server.js +45 -0
- package/dist/cf-worker/do/transport/do-rpc-server.js.map +1 -0
- package/dist/cf-worker/do/transport/http-rpc-server.d.ts +7 -0
- package/dist/cf-worker/do/transport/http-rpc-server.d.ts.map +1 -0
- package/dist/cf-worker/do/transport/http-rpc-server.js +24 -0
- package/dist/cf-worker/do/transport/http-rpc-server.js.map +1 -0
- package/dist/cf-worker/do/transport/ws-rpc-server.d.ts +4 -0
- package/dist/cf-worker/do/transport/ws-rpc-server.d.ts.map +1 -0
- package/dist/cf-worker/do/transport/ws-rpc-server.js +21 -0
- package/dist/cf-worker/do/transport/ws-rpc-server.js.map +1 -0
- package/dist/cf-worker/mod.d.ts +4 -2
- package/dist/cf-worker/mod.d.ts.map +1 -1
- package/dist/cf-worker/mod.js +3 -2
- package/dist/cf-worker/mod.js.map +1 -1
- package/dist/cf-worker/shared.d.ts +127 -0
- package/dist/cf-worker/shared.d.ts.map +1 -0
- package/dist/cf-worker/shared.js +26 -0
- package/dist/cf-worker/shared.js.map +1 -0
- package/dist/cf-worker/worker.d.ts +36 -21
- package/dist/cf-worker/worker.d.ts.map +1 -1
- package/dist/cf-worker/worker.js +39 -32
- package/dist/cf-worker/worker.js.map +1 -1
- package/dist/client/mod.d.ts +4 -0
- package/dist/client/mod.d.ts.map +1 -0
- package/dist/client/mod.js +4 -0
- package/dist/client/mod.js.map +1 -0
- package/dist/client/transport/do-rpc-client.d.ts +40 -0
- package/dist/client/transport/do-rpc-client.d.ts.map +1 -0
- package/dist/client/transport/do-rpc-client.js +102 -0
- package/dist/client/transport/do-rpc-client.js.map +1 -0
- package/dist/client/transport/http-rpc-client.d.ts +43 -0
- package/dist/client/transport/http-rpc-client.d.ts.map +1 -0
- package/dist/client/transport/http-rpc-client.js +87 -0
- package/dist/client/transport/http-rpc-client.js.map +1 -0
- package/dist/client/transport/ws-rpc-client.d.ts +45 -0
- package/dist/client/transport/ws-rpc-client.d.ts.map +1 -0
- package/dist/client/transport/ws-rpc-client.js +94 -0
- package/dist/client/transport/ws-rpc-client.js.map +1 -0
- package/dist/common/do-rpc-schema.d.ts +76 -0
- package/dist/common/do-rpc-schema.d.ts.map +1 -0
- package/dist/common/do-rpc-schema.js +48 -0
- package/dist/common/do-rpc-schema.js.map +1 -0
- package/dist/common/http-rpc-schema.d.ts +58 -0
- package/dist/common/http-rpc-schema.d.ts.map +1 -0
- package/dist/common/http-rpc-schema.js +37 -0
- package/dist/common/http-rpc-schema.js.map +1 -0
- package/dist/common/mod.d.ts +5 -1
- package/dist/common/mod.d.ts.map +1 -1
- package/dist/common/mod.js +4 -1
- package/dist/common/mod.js.map +1 -1
- package/dist/common/sync-message-types.d.ts +236 -0
- package/dist/common/sync-message-types.d.ts.map +1 -0
- package/dist/common/sync-message-types.js +60 -0
- package/dist/common/sync-message-types.js.map +1 -0
- package/dist/common/ws-rpc-schema.d.ts +55 -0
- package/dist/common/ws-rpc-schema.d.ts.map +1 -0
- package/dist/common/ws-rpc-schema.js +32 -0
- package/dist/common/ws-rpc-schema.js.map +1 -0
- package/package.json +7 -8
- package/src/cf-worker/do/durable-object.ts +241 -0
- package/src/cf-worker/do/layer.ts +107 -0
- package/src/cf-worker/do/pull.ts +64 -0
- package/src/cf-worker/do/push.ts +162 -0
- package/src/cf-worker/do/sqlite.ts +28 -0
- package/src/cf-worker/do/sync-storage.ts +126 -0
- package/src/cf-worker/do/transport/do-rpc-server.ts +82 -0
- package/src/cf-worker/do/transport/http-rpc-server.ts +37 -0
- package/src/cf-worker/do/transport/ws-rpc-server.ts +34 -0
- package/src/cf-worker/mod.ts +4 -2
- package/src/cf-worker/shared.ts +95 -0
- package/src/cf-worker/worker.ts +72 -63
- package/src/client/mod.ts +3 -0
- package/src/client/transport/do-rpc-client.ts +171 -0
- package/src/client/transport/http-rpc-client.ts +205 -0
- package/src/client/transport/ws-rpc-client.ts +182 -0
- package/src/common/do-rpc-schema.ts +54 -0
- package/src/common/http-rpc-schema.ts +40 -0
- package/src/common/mod.ts +8 -1
- package/src/common/sync-message-types.ts +117 -0
- package/src/common/ws-rpc-schema.ts +36 -0
- package/src/cf-worker/cf-types.ts +0 -12
- package/src/cf-worker/durable-object.ts +0 -478
- package/src/common/ws-message-types.ts +0 -114
- package/src/sync-impl/mod.ts +0 -1
- package/src/sync-impl/ws-impl.ts +0 -274
|
@@ -0,0 +1,126 @@
|
|
|
1
|
+
import { UnexpectedError } from '@livestore/common'
|
|
2
|
+
import type { LiveStoreEvent } from '@livestore/common/schema'
|
|
3
|
+
import type { CfTypes } from '@livestore/common-cf'
|
|
4
|
+
import { Effect, Option, Schema } from '@livestore/utils/effect'
|
|
5
|
+
import { SyncMetadata } from '../../common/sync-message-types.ts'
|
|
6
|
+
import { type Env, PERSISTENCE_FORMAT_VERSION, type StoreId } from '../shared.ts'
|
|
7
|
+
import { eventlogTable } from './sqlite.ts'
|
|
8
|
+
|
|
9
|
+
export type SyncStorage = {
|
|
10
|
+
dbName: string
|
|
11
|
+
// getHead: Effect.Effect<EventSequenceNumber.GlobalEventSequenceNumber, UnexpectedError>
|
|
12
|
+
getEvents: (
|
|
13
|
+
cursor: number | undefined,
|
|
14
|
+
) => Effect.Effect<
|
|
15
|
+
ReadonlyArray<{ eventEncoded: LiveStoreEvent.AnyEncodedGlobal; metadata: Option.Option<SyncMetadata> }>,
|
|
16
|
+
UnexpectedError
|
|
17
|
+
>
|
|
18
|
+
appendEvents: (
|
|
19
|
+
batch: ReadonlyArray<LiveStoreEvent.AnyEncodedGlobal>,
|
|
20
|
+
createdAt: string,
|
|
21
|
+
) => Effect.Effect<void, UnexpectedError>
|
|
22
|
+
resetStore: Effect.Effect<void, UnexpectedError>
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
export const makeStorage = (ctx: CfTypes.DurableObjectState, env: Env, storeId: StoreId): SyncStorage => {
|
|
26
|
+
const dbName = `eventlog_${PERSISTENCE_FORMAT_VERSION}_${toValidTableName(storeId)}`
|
|
27
|
+
|
|
28
|
+
const execDb = <T>(cb: (db: CfTypes.D1Database) => Promise<CfTypes.D1Result<T>>) =>
|
|
29
|
+
Effect.tryPromise({
|
|
30
|
+
try: () => cb(env.DB),
|
|
31
|
+
catch: (error) => new UnexpectedError({ cause: error, payload: { dbName } }),
|
|
32
|
+
}).pipe(
|
|
33
|
+
Effect.map((_) => _.results),
|
|
34
|
+
Effect.withSpan('@livestore/sync-cf:durable-object:execDb'),
|
|
35
|
+
)
|
|
36
|
+
|
|
37
|
+
// const getHead: Effect.Effect<EventSequenceNumber.GlobalEventSequenceNumber, UnexpectedError> = Effect.gen(
|
|
38
|
+
// function* () {
|
|
39
|
+
// const result = yield* execDb<{ seqNum: EventSequenceNumber.GlobalEventSequenceNumber }>((db) =>
|
|
40
|
+
// db.prepare(`SELECT seqNum FROM ${dbName} ORDER BY seqNum DESC LIMIT 1`).all(),
|
|
41
|
+
// )
|
|
42
|
+
|
|
43
|
+
// return result[0]?.seqNum ?? EventSequenceNumber.ROOT.global
|
|
44
|
+
// },
|
|
45
|
+
// ).pipe(UnexpectedError.mapToUnexpectedError)
|
|
46
|
+
|
|
47
|
+
// TODO support streaming
|
|
48
|
+
const getEvents = (
|
|
49
|
+
cursor: number | undefined,
|
|
50
|
+
): Effect.Effect<
|
|
51
|
+
ReadonlyArray<{ eventEncoded: LiveStoreEvent.AnyEncodedGlobal; metadata: Option.Option<SyncMetadata> }>,
|
|
52
|
+
UnexpectedError
|
|
53
|
+
> =>
|
|
54
|
+
Effect.gen(function* () {
|
|
55
|
+
const whereClause = cursor === undefined ? '' : `WHERE seqNum > ${cursor}`
|
|
56
|
+
const sql = `SELECT * FROM ${dbName} ${whereClause} ORDER BY seqNum ASC`
|
|
57
|
+
// TODO handle case where `cursor` was not found
|
|
58
|
+
const rawEvents = yield* execDb((db) => db.prepare(sql).all())
|
|
59
|
+
const events = Schema.decodeUnknownSync(Schema.Array(eventlogTable.rowSchema))(rawEvents).map(
|
|
60
|
+
({ createdAt, ...eventEncoded }) => ({
|
|
61
|
+
eventEncoded,
|
|
62
|
+
metadata: Option.some(SyncMetadata.make({ createdAt })),
|
|
63
|
+
}),
|
|
64
|
+
)
|
|
65
|
+
return events
|
|
66
|
+
}).pipe(
|
|
67
|
+
UnexpectedError.mapToUnexpectedError,
|
|
68
|
+
Effect.withSpan('@livestore/sync-cf:durable-object:getEvents', { attributes: { dbName, cursor } }),
|
|
69
|
+
)
|
|
70
|
+
|
|
71
|
+
const appendEvents: SyncStorage['appendEvents'] = (batch, createdAt) =>
|
|
72
|
+
Effect.gen(function* () {
|
|
73
|
+
// If there are no events, do nothing.
|
|
74
|
+
if (batch.length === 0) return
|
|
75
|
+
|
|
76
|
+
// CF D1 limits:
|
|
77
|
+
// Maximum bound parameters per query 100, Maximum arguments per SQL function 32
|
|
78
|
+
// Thus we need to split the batch into chunks of max (100/7=)14 events each.
|
|
79
|
+
const CHUNK_SIZE = 14
|
|
80
|
+
|
|
81
|
+
for (let i = 0; i < batch.length; i += CHUNK_SIZE) {
|
|
82
|
+
const chunk = batch.slice(i, i + CHUNK_SIZE)
|
|
83
|
+
|
|
84
|
+
// Create a list of placeholders ("(?, ?, ?, ?, ?, ?, ?)"), corresponding to each event.
|
|
85
|
+
const valuesPlaceholders = chunk.map(() => '(?, ?, ?, ?, ?, ?, ?)').join(', ')
|
|
86
|
+
const sql = `INSERT INTO ${dbName} (seqNum, parentSeqNum, args, name, createdAt, clientId, sessionId) VALUES ${valuesPlaceholders}`
|
|
87
|
+
// Flatten the event properties into a parameters array.
|
|
88
|
+
const params = chunk.flatMap((event) => [
|
|
89
|
+
event.seqNum,
|
|
90
|
+
event.parentSeqNum,
|
|
91
|
+
event.args === undefined ? null : JSON.stringify(event.args),
|
|
92
|
+
event.name,
|
|
93
|
+
createdAt,
|
|
94
|
+
event.clientId,
|
|
95
|
+
event.sessionId,
|
|
96
|
+
])
|
|
97
|
+
|
|
98
|
+
yield* execDb((db) =>
|
|
99
|
+
db
|
|
100
|
+
.prepare(sql)
|
|
101
|
+
.bind(...params)
|
|
102
|
+
.run(),
|
|
103
|
+
)
|
|
104
|
+
}
|
|
105
|
+
}).pipe(
|
|
106
|
+
UnexpectedError.mapToUnexpectedError,
|
|
107
|
+
Effect.withSpan('@livestore/sync-cf:durable-object:appendEvents', {
|
|
108
|
+
attributes: { dbName, batchLength: batch.length },
|
|
109
|
+
}),
|
|
110
|
+
)
|
|
111
|
+
|
|
112
|
+
const resetStore = Effect.promise(() => ctx.storage.deleteAll()).pipe(
|
|
113
|
+
UnexpectedError.mapToUnexpectedError,
|
|
114
|
+
Effect.withSpan('@livestore/sync-cf:durable-object:resetStore'),
|
|
115
|
+
)
|
|
116
|
+
|
|
117
|
+
return {
|
|
118
|
+
dbName,
|
|
119
|
+
// getHead,
|
|
120
|
+
getEvents,
|
|
121
|
+
appendEvents,
|
|
122
|
+
resetStore,
|
|
123
|
+
}
|
|
124
|
+
}
|
|
125
|
+
|
|
126
|
+
const toValidTableName = (str: string) => str.replaceAll(/[^a-zA-Z0-9]/g, '_')
|
|
@@ -0,0 +1,82 @@
|
|
|
1
|
+
import { InvalidPullError, InvalidPushError } from '@livestore/common'
|
|
2
|
+
import { toDurableObjectHandler } from '@livestore/common-cf'
|
|
3
|
+
import {
|
|
4
|
+
Effect,
|
|
5
|
+
Headers,
|
|
6
|
+
HttpServer,
|
|
7
|
+
Layer,
|
|
8
|
+
Logger,
|
|
9
|
+
LogLevel,
|
|
10
|
+
Option,
|
|
11
|
+
RpcSerialization,
|
|
12
|
+
Stream,
|
|
13
|
+
} from '@livestore/utils/effect'
|
|
14
|
+
import { SyncDoRpc } from '../../../common/do-rpc-schema.ts'
|
|
15
|
+
import { SyncMessage } from '../../../common/mod.ts'
|
|
16
|
+
import { DoCtx, type DoCtxInput } from '../layer.ts'
|
|
17
|
+
import { makeEndingPullStream } from '../pull.ts'
|
|
18
|
+
import { makePush } from '../push.ts'
|
|
19
|
+
|
|
20
|
+
export interface DoRpcHandlerOptions {
|
|
21
|
+
payload: Uint8Array<ArrayBuffer>
|
|
22
|
+
input: Omit<DoCtxInput, 'from'>
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
export const createDoRpcHandler = (options: DoRpcHandlerOptions) =>
|
|
26
|
+
Effect.gen(this, function* () {
|
|
27
|
+
const { payload, input } = options
|
|
28
|
+
// const { rpcSubscriptions, backendId, doOptions, ctx, env } = yield* DoCtx
|
|
29
|
+
|
|
30
|
+
// TODO add admin RPCs
|
|
31
|
+
const RpcLive = SyncDoRpc.toLayer({
|
|
32
|
+
'SyncDoRpc.Ping': (_req) => {
|
|
33
|
+
return Effect.succeed(SyncMessage.Pong.make({}))
|
|
34
|
+
},
|
|
35
|
+
'SyncDoRpc.Pull': (req, { headers }) =>
|
|
36
|
+
Effect.gen(this, function* () {
|
|
37
|
+
const { rpcSubscriptions } = yield* DoCtx
|
|
38
|
+
|
|
39
|
+
// TODO rename `req.rpcContext` to something more appropriate
|
|
40
|
+
if (req.rpcContext) {
|
|
41
|
+
rpcSubscriptions.set(req.storeId, {
|
|
42
|
+
storeId: req.storeId,
|
|
43
|
+
payload: req.payload,
|
|
44
|
+
subscribedAt: Date.now(),
|
|
45
|
+
requestId: Headers.get(headers, 'x-rpc-request-id').pipe(Option.getOrThrow),
|
|
46
|
+
callerContext: req.rpcContext.callerContext,
|
|
47
|
+
})
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
return makeEndingPullStream(req, req.payload)
|
|
51
|
+
}).pipe(
|
|
52
|
+
Stream.unwrap,
|
|
53
|
+
Stream.map((res) => ({
|
|
54
|
+
...res,
|
|
55
|
+
rpcRequestId: Headers.get(headers, 'x-rpc-request-id').pipe(Option.getOrThrow),
|
|
56
|
+
})),
|
|
57
|
+
Stream.provideLayer(DoCtx.Default({ ...input, from: { storeId: req.storeId } })),
|
|
58
|
+
Stream.mapError((cause) => (cause._tag === 'InvalidPullError' ? cause : InvalidPullError.make({ cause }))),
|
|
59
|
+
Stream.tapErrorCause(Effect.log),
|
|
60
|
+
),
|
|
61
|
+
'SyncDoRpc.Push': (req) =>
|
|
62
|
+
Effect.gen(this, function* () {
|
|
63
|
+
const { doOptions, ctx, env, storeId } = yield* DoCtx
|
|
64
|
+
const push = makePush({ storeId, payload: req.payload, options: doOptions, ctx, env })
|
|
65
|
+
|
|
66
|
+
return yield* push(req)
|
|
67
|
+
}).pipe(
|
|
68
|
+
Effect.provide(DoCtx.Default({ ...input, from: { storeId: req.storeId } })),
|
|
69
|
+
Effect.mapError((cause) => (cause._tag === 'InvalidPushError' ? cause : InvalidPushError.make({ cause }))),
|
|
70
|
+
Effect.tapCauseLogPretty,
|
|
71
|
+
),
|
|
72
|
+
})
|
|
73
|
+
|
|
74
|
+
const handler = toDurableObjectHandler(SyncDoRpc, {
|
|
75
|
+
layer: Layer.mergeAll(RpcLive, RpcSerialization.layerJson, HttpServer.layerContext).pipe(
|
|
76
|
+
Layer.provide(Logger.consoleWithThread('SyncDo')),
|
|
77
|
+
Layer.provide(Logger.minimumLogLevel(LogLevel.Debug)),
|
|
78
|
+
),
|
|
79
|
+
})
|
|
80
|
+
|
|
81
|
+
return yield* handler(payload)
|
|
82
|
+
}).pipe(Effect.withSpan('createDoRpcHandler'))
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
import type { CfTypes } from '@livestore/common-cf'
|
|
2
|
+
import { Effect, HttpApp, Layer, RpcSerialization, RpcServer } from '@livestore/utils/effect'
|
|
3
|
+
import { SyncHttpRpc } from '../../../common/http-rpc-schema.ts'
|
|
4
|
+
import * as SyncMessage from '../../../common/sync-message-types.ts'
|
|
5
|
+
import { DoCtx } from '../layer.ts'
|
|
6
|
+
import { makeEndingPullStream } from '../pull.ts'
|
|
7
|
+
import { makePush } from '../push.ts'
|
|
8
|
+
|
|
9
|
+
export const createHttpRpcHandler = ({ request }: { request: CfTypes.Request }) =>
|
|
10
|
+
Effect.gen(function* () {
|
|
11
|
+
const handlerLayer = createHttpRpcLayer
|
|
12
|
+
const httpApp = RpcServer.toHttpApp(SyncHttpRpc).pipe(Effect.provide(handlerLayer))
|
|
13
|
+
const webHandler = yield* httpApp.pipe(Effect.map(HttpApp.toWebHandler))
|
|
14
|
+
|
|
15
|
+
return yield* Effect.promise(
|
|
16
|
+
() => webHandler(request as TODO as Request) as TODO as Promise<CfTypes.Response>,
|
|
17
|
+
).pipe(Effect.timeout(10000))
|
|
18
|
+
}).pipe(Effect.withSpan('createHttpRpcHandler'))
|
|
19
|
+
|
|
20
|
+
const createHttpRpcLayer =
|
|
21
|
+
// TODO implement admin requests
|
|
22
|
+
SyncHttpRpc.toLayer({
|
|
23
|
+
'SyncHttpRpc.Pull': (req) => makeEndingPullStream(req, req.payload),
|
|
24
|
+
|
|
25
|
+
'SyncHttpRpc.Push': (req) =>
|
|
26
|
+
Effect.gen(function* () {
|
|
27
|
+
const { ctx, env, doOptions, storeId } = yield* DoCtx
|
|
28
|
+
const push = makePush({ payload: undefined, options: doOptions, storeId, ctx, env })
|
|
29
|
+
|
|
30
|
+
return yield* push(req)
|
|
31
|
+
}),
|
|
32
|
+
|
|
33
|
+
'SyncHttpRpc.Ping': () => Effect.succeed(SyncMessage.Pong.make({})),
|
|
34
|
+
}).pipe(
|
|
35
|
+
Layer.provideMerge(RpcServer.layerProtocolHttp({ path: '/http-rpc' })),
|
|
36
|
+
Layer.provideMerge(RpcSerialization.layerJson),
|
|
37
|
+
)
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
import { InvalidPullError, InvalidPushError } from '@livestore/common'
|
|
2
|
+
import { Effect, identity, Layer, RpcServer, Stream } from '@livestore/utils/effect'
|
|
3
|
+
import { SyncWsRpc } from '../../../common/ws-rpc-schema.ts'
|
|
4
|
+
import { DoCtx, type DoCtxInput } from '../layer.ts'
|
|
5
|
+
import { makeEndingPullStream } from '../pull.ts'
|
|
6
|
+
import { makePush } from '../push.ts'
|
|
7
|
+
|
|
8
|
+
export const makeRpcServer = ({ doSelf, doOptions }: Omit<DoCtxInput, 'from'>) => {
|
|
9
|
+
// TODO implement admin requests
|
|
10
|
+
const handlersLayer = SyncWsRpc.toLayer({
|
|
11
|
+
'SyncWsRpc.Pull': (req) =>
|
|
12
|
+
makeEndingPullStream(req, req.payload).pipe(
|
|
13
|
+
// Needed to keep the stream alive on the client side for phase 2 (i.e. not send the `Exit` stream RPC message)
|
|
14
|
+
req.live ? Stream.concat(Stream.never) : identity,
|
|
15
|
+
Stream.provideLayer(DoCtx.Default({ doSelf, doOptions, from: { storeId: req.storeId } })),
|
|
16
|
+
Stream.mapError((cause) => (cause._tag === 'InvalidPullError' ? cause : InvalidPullError.make({ cause }))),
|
|
17
|
+
// Stream.tapErrorCause(Effect.log),
|
|
18
|
+
),
|
|
19
|
+
'SyncWsRpc.Push': (req) =>
|
|
20
|
+
Effect.gen(function* () {
|
|
21
|
+
const { doOptions, storeId, ctx, env } = yield* DoCtx
|
|
22
|
+
|
|
23
|
+
const push = makePush({ options: doOptions, storeId, payload: req.payload, ctx, env })
|
|
24
|
+
|
|
25
|
+
return yield* push(req)
|
|
26
|
+
}).pipe(
|
|
27
|
+
Effect.provide(DoCtx.Default({ doSelf, doOptions, from: { storeId: req.storeId } })),
|
|
28
|
+
Effect.mapError((cause) => (cause._tag === 'InvalidPushError' ? cause : InvalidPushError.make({ cause }))),
|
|
29
|
+
Effect.tapCauseLogPretty,
|
|
30
|
+
),
|
|
31
|
+
})
|
|
32
|
+
|
|
33
|
+
return RpcServer.layer(SyncWsRpc).pipe(Layer.provide(handlersLayer))
|
|
34
|
+
}
|
package/src/cf-worker/mod.ts
CHANGED
|
@@ -1,3 +1,5 @@
|
|
|
1
|
-
export
|
|
2
|
-
export
|
|
1
|
+
export type { CfTypes } from '@livestore/common-cf'
|
|
2
|
+
export { CfDeclare } from '@livestore/common-cf/declare'
|
|
3
|
+
export * from './do/durable-object.ts'
|
|
4
|
+
export * from './shared.ts'
|
|
3
5
|
export * from './worker.ts'
|
|
@@ -0,0 +1,95 @@
|
|
|
1
|
+
import type { InvalidPullError, InvalidPushError } from '@livestore/common'
|
|
2
|
+
import type { CfTypes } from '@livestore/common-cf'
|
|
3
|
+
import { Effect, type Option, Schema, UrlParams } from '@livestore/utils/effect'
|
|
4
|
+
import { SearchParamsSchema, SyncMessage } from '../common/mod.ts'
|
|
5
|
+
|
|
6
|
+
export interface Env {
|
|
7
|
+
/** Eventlog database */
|
|
8
|
+
DB: CfTypes.D1Database
|
|
9
|
+
ADMIN_SECRET: string
|
|
10
|
+
}
|
|
11
|
+
|
|
12
|
+
export type MakeDurableObjectClassOptions = {
|
|
13
|
+
onPush?: (
|
|
14
|
+
message: SyncMessage.PushRequest,
|
|
15
|
+
context: { storeId: StoreId; payload?: Schema.JsonValue },
|
|
16
|
+
) => Effect.SyncOrPromiseOrEffect<void>
|
|
17
|
+
onPushRes?: (message: SyncMessage.PushAck | InvalidPushError) => Effect.SyncOrPromiseOrEffect<void>
|
|
18
|
+
onPull?: (
|
|
19
|
+
message: SyncMessage.PullRequest,
|
|
20
|
+
context: { storeId: StoreId; payload?: Schema.JsonValue },
|
|
21
|
+
) => Effect.SyncOrPromiseOrEffect<void>
|
|
22
|
+
onPullRes?: (message: SyncMessage.PullResponse | InvalidPullError) => Effect.SyncOrPromiseOrEffect<void>
|
|
23
|
+
// TODO make storage configurable: D1, DO SQLite, later: external SQLite
|
|
24
|
+
|
|
25
|
+
/**
|
|
26
|
+
* Enabled transports for sync backend
|
|
27
|
+
* - `http`: HTTP JSON-RPC
|
|
28
|
+
* - `ws`: WebSocket
|
|
29
|
+
* - `do-rpc`: Durable Object RPC calls (only works in combination with `@livestore/adapter-cf`)
|
|
30
|
+
*
|
|
31
|
+
* @default Set(['http', 'ws', 'do-rpc'])
|
|
32
|
+
*/
|
|
33
|
+
enabledTransports?: Set<'http' | 'ws' | 'do-rpc'>
|
|
34
|
+
|
|
35
|
+
otel?: {
|
|
36
|
+
baseUrl?: string
|
|
37
|
+
serviceName?: string
|
|
38
|
+
}
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
export type StoreId = string
|
|
42
|
+
export type DurableObjectId = string
|
|
43
|
+
|
|
44
|
+
/**
|
|
45
|
+
* Needs to be bumped when the storage format changes (e.g. eventlogTable schema changes)
|
|
46
|
+
*
|
|
47
|
+
* Changing this version number will lead to a "soft reset".
|
|
48
|
+
*/
|
|
49
|
+
export const PERSISTENCE_FORMAT_VERSION = 7
|
|
50
|
+
|
|
51
|
+
export const DEFAULT_SYNC_DURABLE_OBJECT_NAME = 'SYNC_BACKEND_DO'
|
|
52
|
+
|
|
53
|
+
export const encodeOutgoingMessage = Schema.encodeSync(Schema.parseJson(SyncMessage.BackendToClientMessage))
|
|
54
|
+
export const encodeIncomingMessage = Schema.encodeSync(Schema.parseJson(SyncMessage.ClientToBackendMessage))
|
|
55
|
+
|
|
56
|
+
export const getSyncRequestSearchParams = (request: CfTypes.Request): Option.Option<typeof SearchParamsSchema.Type> => {
|
|
57
|
+
const url = new URL(request.url)
|
|
58
|
+
const urlParams = UrlParams.fromInput(url.searchParams)
|
|
59
|
+
const paramsResult = UrlParams.schemaStruct(SearchParamsSchema)(urlParams).pipe(Effect.option, Effect.runSync)
|
|
60
|
+
|
|
61
|
+
return paramsResult
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
export const PULL_CHUNK_SIZE = 100
|
|
65
|
+
|
|
66
|
+
// RPC subscription storage (TODO refactor)
|
|
67
|
+
export type RpcSubscription = {
|
|
68
|
+
storeId: StoreId
|
|
69
|
+
payload?: Schema.JsonValue
|
|
70
|
+
subscribedAt: number
|
|
71
|
+
/** Effect RPC request ID */
|
|
72
|
+
requestId: string
|
|
73
|
+
callerContext: {
|
|
74
|
+
bindingName: string
|
|
75
|
+
durableObjectId: string
|
|
76
|
+
}
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
/**
|
|
80
|
+
* Durable Object interface supporting the DO RPC protocol for DO <> DO syncing.
|
|
81
|
+
*/
|
|
82
|
+
export interface SyncBackendRpcInterface {
|
|
83
|
+
__DURABLE_OBJECT_BRAND: never
|
|
84
|
+
rpc(payload: Uint8Array): Promise<Uint8Array | CfTypes.ReadableStream>
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
export const WebSocketAttachmentSchema = Schema.parseJson(
|
|
88
|
+
Schema.Struct({
|
|
89
|
+
// Same across all websocket connections
|
|
90
|
+
storeId: Schema.String,
|
|
91
|
+
// Different for each websocket connection
|
|
92
|
+
payload: Schema.optional(Schema.JsonValue),
|
|
93
|
+
pullRequestIds: Schema.Array(Schema.String),
|
|
94
|
+
}),
|
|
95
|
+
)
|
package/src/cf-worker/worker.ts
CHANGED
|
@@ -1,17 +1,15 @@
|
|
|
1
|
-
import type * as CfWorker from '@cloudflare/workers-types'
|
|
2
1
|
import { UnexpectedError } from '@livestore/common'
|
|
3
2
|
import type { Schema } from '@livestore/utils/effect'
|
|
4
|
-
import { Effect
|
|
5
|
-
|
|
6
|
-
import {
|
|
7
|
-
|
|
8
|
-
import type { Env } from './durable-object.ts'
|
|
3
|
+
import { Effect } from '@livestore/utils/effect'
|
|
4
|
+
import type { CfTypes, SearchParams } from '../common/mod.ts'
|
|
5
|
+
import type { CfDeclare } from './mod.ts'
|
|
6
|
+
import { DEFAULT_SYNC_DURABLE_OBJECT_NAME, type Env, getSyncRequestSearchParams } from './shared.ts'
|
|
9
7
|
|
|
10
8
|
// NOTE We need to redeclare runtime types here to avoid type conflicts with the lib.dom Response type.
|
|
11
|
-
declare class Response extends
|
|
9
|
+
declare class Response extends CfDeclare.Response {}
|
|
12
10
|
|
|
13
11
|
export namespace HelperTypes {
|
|
14
|
-
type AnyDON =
|
|
12
|
+
type AnyDON = CfTypes.DurableObjectNamespace<undefined>
|
|
15
13
|
|
|
16
14
|
type DOKeys<T> = {
|
|
17
15
|
[K in keyof T]-?: T[K] extends AnyDON ? K : never
|
|
@@ -27,11 +25,11 @@ export namespace HelperTypes {
|
|
|
27
25
|
* type PlatformEnv = {
|
|
28
26
|
* DB: D1Database
|
|
29
27
|
* ADMIN_TOKEN: string
|
|
30
|
-
*
|
|
28
|
+
* SYNC_BACKEND_DO: DurableObjectNamespace<SyncBackendDO>
|
|
31
29
|
* }
|
|
32
30
|
* export default makeWorker<PlatformEnv>({
|
|
33
|
-
* durableObject: { name: "
|
|
34
|
-
* // ^ (property) name?: "
|
|
31
|
+
* durableObject: { name: "SYNC_BACKEND_DO" },
|
|
32
|
+
* // ^ (property) name?: "SYNC_BACKEND_DO" | undefined
|
|
35
33
|
* });
|
|
36
34
|
*/
|
|
37
35
|
export type ExtractDurableObjectKeys<TEnv = Env> = DOKeys<NonBuiltins<TEnv>> extends never
|
|
@@ -40,12 +38,12 @@ export namespace HelperTypes {
|
|
|
40
38
|
}
|
|
41
39
|
|
|
42
40
|
// HINT: If we ever extend user's custom worker RPC, type T can help here with expected return type safety. Currently unused.
|
|
43
|
-
export type CFWorker<TEnv extends Env = Env, _T extends
|
|
41
|
+
export type CFWorker<TEnv extends Env = Env, _T extends CfTypes.Rpc.DurableObjectBranded | undefined = undefined> = {
|
|
44
42
|
fetch: <CFHostMetada = unknown>(
|
|
45
|
-
request:
|
|
43
|
+
request: CfTypes.Request<CFHostMetada>,
|
|
46
44
|
env: TEnv,
|
|
47
|
-
ctx:
|
|
48
|
-
) => Promise<
|
|
45
|
+
ctx: CfTypes.ExecutionContext,
|
|
46
|
+
) => Promise<CfTypes.Response>
|
|
49
47
|
}
|
|
50
48
|
|
|
51
49
|
export type MakeWorkerOptions<TEnv extends Env = Env> = {
|
|
@@ -61,7 +59,7 @@ export type MakeWorkerOptions<TEnv extends Env = Env> = {
|
|
|
61
59
|
/**
|
|
62
60
|
* Needs to match the binding name from the wrangler config
|
|
63
61
|
*
|
|
64
|
-
* @default '
|
|
62
|
+
* @default 'SYNC_BACKEND_DO'
|
|
65
63
|
*/
|
|
66
64
|
name?: HelperTypes.ExtractDurableObjectKeys<TEnv>
|
|
67
65
|
}
|
|
@@ -69,7 +67,7 @@ export type MakeWorkerOptions<TEnv extends Env = Env> = {
|
|
|
69
67
|
|
|
70
68
|
export const makeWorker = <
|
|
71
69
|
TEnv extends Env = Env,
|
|
72
|
-
TDurableObjectRpc extends
|
|
70
|
+
TDurableObjectRpc extends CfTypes.Rpc.DurableObjectBranded | undefined = undefined,
|
|
73
71
|
>(
|
|
74
72
|
options: MakeWorkerOptions<TEnv> = {},
|
|
75
73
|
): CFWorker<TEnv, TDurableObjectRpc> => {
|
|
@@ -77,16 +75,7 @@ export const makeWorker = <
|
|
|
77
75
|
fetch: async (request, env, _ctx) => {
|
|
78
76
|
const url = new URL(request.url)
|
|
79
77
|
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
if (request.method === 'GET' && url.pathname === '/') {
|
|
83
|
-
return new Response('Info: WebSocket sync backend endpoint for @livestore/sync-cf.', {
|
|
84
|
-
status: 200,
|
|
85
|
-
headers: { 'Content-Type': 'text/plain' },
|
|
86
|
-
})
|
|
87
|
-
}
|
|
88
|
-
|
|
89
|
-
const corsHeaders: CfWorker.HeadersInit = options.enableCORS
|
|
78
|
+
const corsHeaders: CfTypes.HeadersInit = options.enableCORS
|
|
90
79
|
? {
|
|
91
80
|
'Access-Control-Allow-Origin': '*',
|
|
92
81
|
'Access-Control-Allow-Methods': 'GET, POST, OPTIONS',
|
|
@@ -101,11 +90,28 @@ export const makeWorker = <
|
|
|
101
90
|
})
|
|
102
91
|
}
|
|
103
92
|
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
93
|
+
const requestParamsResult = getSyncRequestSearchParams(request)
|
|
94
|
+
|
|
95
|
+
// Check if this is a sync request first, before showing info message
|
|
96
|
+
if (requestParamsResult._tag === 'Some') {
|
|
97
|
+
return handleSyncRequest<TEnv, TDurableObjectRpc>({
|
|
98
|
+
request,
|
|
99
|
+
searchParams: requestParamsResult.value,
|
|
100
|
+
env,
|
|
101
|
+
ctx: _ctx,
|
|
102
|
+
options: {
|
|
103
|
+
headers: corsHeaders,
|
|
104
|
+
validatePayload: options.validatePayload,
|
|
105
|
+
durableObject: options.durableObject,
|
|
106
|
+
},
|
|
107
|
+
})
|
|
108
|
+
}
|
|
109
|
+
|
|
110
|
+
// Only show info message for GET requests to / without sync parameters
|
|
111
|
+
if (request.method === 'GET' && url.pathname === '/') {
|
|
112
|
+
return new Response('Info: Sync backend endpoint for @livestore/sync-cf.', {
|
|
113
|
+
status: 200,
|
|
114
|
+
headers: { 'Content-Type': 'text/plain' },
|
|
109
115
|
})
|
|
110
116
|
}
|
|
111
117
|
|
|
@@ -124,7 +130,7 @@ export const makeWorker = <
|
|
|
124
130
|
}
|
|
125
131
|
|
|
126
132
|
/**
|
|
127
|
-
* Handles `/
|
|
133
|
+
* Handles `/sync` endpoint.
|
|
128
134
|
*
|
|
129
135
|
* @example
|
|
130
136
|
* ```ts
|
|
@@ -137,46 +143,49 @@ export const makeWorker = <
|
|
|
137
143
|
*
|
|
138
144
|
* export default {
|
|
139
145
|
* fetch: async (request, env, ctx) => {
|
|
140
|
-
*
|
|
141
|
-
*
|
|
146
|
+
* const requestParamsResult = getSyncRequestSearchParams(request)
|
|
147
|
+
*
|
|
148
|
+
* // Is LiveStore sync request
|
|
149
|
+
* if (requestParamsResult._tag === 'Some') {
|
|
150
|
+
* return handleSyncRequest({
|
|
151
|
+
* request,
|
|
152
|
+
* searchParams: requestParamsResult.value,
|
|
153
|
+
* env,
|
|
154
|
+
* ctx,
|
|
155
|
+
* options: { headers: {}, validatePayload }
|
|
156
|
+
* })
|
|
142
157
|
* }
|
|
143
158
|
*
|
|
144
159
|
* return new Response('Invalid path', { status: 400 })
|
|
145
|
-
* return new Response('Invalid path', { status: 400 })
|
|
146
160
|
* }
|
|
147
161
|
* }
|
|
148
162
|
* ```
|
|
149
163
|
*
|
|
150
164
|
* @throws {UnexpectedError} If the payload is invalid
|
|
151
165
|
*/
|
|
152
|
-
export const
|
|
166
|
+
export const handleSyncRequest = <
|
|
153
167
|
TEnv extends Env = Env,
|
|
154
|
-
TDurableObjectRpc extends
|
|
168
|
+
TDurableObjectRpc extends CfTypes.Rpc.DurableObjectBranded | undefined = undefined,
|
|
155
169
|
CFHostMetada = unknown,
|
|
156
|
-
>(
|
|
157
|
-
request
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
options
|
|
161
|
-
|
|
170
|
+
>({
|
|
171
|
+
request,
|
|
172
|
+
searchParams,
|
|
173
|
+
env,
|
|
174
|
+
options = {},
|
|
175
|
+
}: {
|
|
176
|
+
request: CfTypes.Request<CFHostMetada>
|
|
177
|
+
searchParams: SearchParams
|
|
178
|
+
env: TEnv
|
|
179
|
+
/** Only there for type-level reasons */
|
|
180
|
+
ctx: CfTypes.ExecutionContext
|
|
181
|
+
options?: {
|
|
182
|
+
headers?: CfTypes.HeadersInit
|
|
162
183
|
durableObject?: MakeWorkerOptions<TEnv>['durableObject']
|
|
163
184
|
validatePayload?: (payload: Schema.JsonValue | undefined, context: { storeId: string }) => void | Promise<void>
|
|
164
|
-
}
|
|
165
|
-
): Promise<
|
|
185
|
+
}
|
|
186
|
+
}): Promise<CfTypes.Response> =>
|
|
166
187
|
Effect.gen(function* () {
|
|
167
|
-
const
|
|
168
|
-
|
|
169
|
-
const urlParams = UrlParams.fromInput(url.searchParams)
|
|
170
|
-
const paramsResult = yield* UrlParams.schemaStruct(SearchParamsSchema)(urlParams).pipe(Effect.either)
|
|
171
|
-
|
|
172
|
-
if (paramsResult._tag === 'Left') {
|
|
173
|
-
return new Response(`Invalid search params: ${paramsResult.left.toString()}`, {
|
|
174
|
-
status: 500,
|
|
175
|
-
headers: options?.headers,
|
|
176
|
-
})
|
|
177
|
-
}
|
|
178
|
-
|
|
179
|
-
const { storeId, payload } = paramsResult.right
|
|
188
|
+
const { storeId, payload, transport } = searchParams
|
|
180
189
|
|
|
181
190
|
if (options.validatePayload !== undefined) {
|
|
182
191
|
const result = yield* Effect.promise(async () => options.validatePayload!(payload, { storeId })).pipe(
|
|
@@ -190,7 +199,7 @@ export const handleWebSocket = <
|
|
|
190
199
|
}
|
|
191
200
|
}
|
|
192
201
|
|
|
193
|
-
const durableObjectName = options.durableObject?.name ??
|
|
202
|
+
const durableObjectName = options.durableObject?.name ?? DEFAULT_SYNC_DURABLE_OBJECT_NAME
|
|
194
203
|
if (!(durableObjectName in env)) {
|
|
195
204
|
return new Response(
|
|
196
205
|
`Failed dependency: Required Durable Object binding '${durableObjectName as string}' not available`,
|
|
@@ -203,19 +212,19 @@ export const handleWebSocket = <
|
|
|
203
212
|
|
|
204
213
|
const durableObjectNamespace = env[
|
|
205
214
|
durableObjectName as keyof TEnv
|
|
206
|
-
] as
|
|
215
|
+
] as CfTypes.DurableObjectNamespace<TDurableObjectRpc>
|
|
207
216
|
|
|
208
217
|
const id = durableObjectNamespace.idFromName(storeId)
|
|
209
218
|
const durableObject = durableObjectNamespace.get(id)
|
|
210
219
|
|
|
220
|
+
// Handle WebSocket upgrade request
|
|
211
221
|
const upgradeHeader = request.headers.get('Upgrade')
|
|
212
|
-
if (
|
|
222
|
+
if (transport === 'ws' && (upgradeHeader === null || upgradeHeader !== 'websocket')) {
|
|
213
223
|
return new Response('Durable Object expected Upgrade: websocket', {
|
|
214
224
|
status: 426,
|
|
215
225
|
headers: options?.headers,
|
|
216
226
|
})
|
|
217
227
|
}
|
|
218
228
|
|
|
219
|
-
// Cloudflare Durable Object type clashing with lib.dom Response type, which is why we need the casts here.
|
|
220
229
|
return yield* Effect.promise(() => durableObject.fetch(request))
|
|
221
230
|
}).pipe(Effect.tapCauseLogPretty, Effect.runPromise)
|