@livestore/sync-cf 0.4.0-dev.3 → 0.4.0-dev.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +60 -0
- package/dist/.tsbuildinfo +1 -1
- package/dist/cf-worker/do/durable-object.d.ts +45 -0
- package/dist/cf-worker/do/durable-object.d.ts.map +1 -0
- package/dist/cf-worker/do/durable-object.js +154 -0
- package/dist/cf-worker/do/durable-object.js.map +1 -0
- package/dist/cf-worker/do/layer.d.ts +34 -0
- package/dist/cf-worker/do/layer.d.ts.map +1 -0
- package/dist/cf-worker/do/layer.js +68 -0
- package/dist/cf-worker/do/layer.js.map +1 -0
- package/dist/cf-worker/do/pull.d.ts +6 -0
- package/dist/cf-worker/do/pull.d.ts.map +1 -0
- package/dist/cf-worker/do/pull.js +39 -0
- package/dist/cf-worker/do/pull.js.map +1 -0
- package/dist/cf-worker/do/push.d.ts +14 -0
- package/dist/cf-worker/do/push.d.ts.map +1 -0
- package/dist/cf-worker/do/push.js +99 -0
- package/dist/cf-worker/do/push.js.map +1 -0
- package/dist/cf-worker/do/sqlite.d.ts +196 -0
- package/dist/cf-worker/do/sqlite.d.ts.map +1 -0
- package/dist/cf-worker/do/sqlite.js +27 -0
- package/dist/cf-worker/do/sqlite.js.map +1 -0
- package/dist/cf-worker/do/sync-storage.d.ts +17 -0
- package/dist/cf-worker/do/sync-storage.d.ts.map +1 -0
- package/dist/cf-worker/do/sync-storage.js +73 -0
- package/dist/cf-worker/do/sync-storage.js.map +1 -0
- package/dist/cf-worker/do/transport/do-rpc-server.d.ts +8 -0
- package/dist/cf-worker/do/transport/do-rpc-server.d.ts.map +1 -0
- package/dist/cf-worker/do/transport/do-rpc-server.js +45 -0
- package/dist/cf-worker/do/transport/do-rpc-server.js.map +1 -0
- package/dist/cf-worker/do/transport/http-rpc-server.d.ts +7 -0
- package/dist/cf-worker/do/transport/http-rpc-server.d.ts.map +1 -0
- package/dist/cf-worker/do/transport/http-rpc-server.js +24 -0
- package/dist/cf-worker/do/transport/http-rpc-server.js.map +1 -0
- package/dist/cf-worker/do/transport/ws-rpc-server.d.ts +4 -0
- package/dist/cf-worker/do/transport/ws-rpc-server.d.ts.map +1 -0
- package/dist/cf-worker/do/transport/ws-rpc-server.js +21 -0
- package/dist/cf-worker/do/transport/ws-rpc-server.js.map +1 -0
- package/dist/cf-worker/mod.d.ts +4 -2
- package/dist/cf-worker/mod.d.ts.map +1 -1
- package/dist/cf-worker/mod.js +3 -2
- package/dist/cf-worker/mod.js.map +1 -1
- package/dist/cf-worker/shared.d.ts +127 -0
- package/dist/cf-worker/shared.d.ts.map +1 -0
- package/dist/cf-worker/shared.js +26 -0
- package/dist/cf-worker/shared.js.map +1 -0
- package/dist/cf-worker/worker.d.ts +36 -21
- package/dist/cf-worker/worker.d.ts.map +1 -1
- package/dist/cf-worker/worker.js +39 -32
- package/dist/cf-worker/worker.js.map +1 -1
- package/dist/client/mod.d.ts +4 -0
- package/dist/client/mod.d.ts.map +1 -0
- package/dist/client/mod.js +4 -0
- package/dist/client/mod.js.map +1 -0
- package/dist/client/transport/do-rpc-client.d.ts +40 -0
- package/dist/client/transport/do-rpc-client.d.ts.map +1 -0
- package/dist/client/transport/do-rpc-client.js +102 -0
- package/dist/client/transport/do-rpc-client.js.map +1 -0
- package/dist/client/transport/http-rpc-client.d.ts +43 -0
- package/dist/client/transport/http-rpc-client.d.ts.map +1 -0
- package/dist/client/transport/http-rpc-client.js +87 -0
- package/dist/client/transport/http-rpc-client.js.map +1 -0
- package/dist/client/transport/ws-rpc-client.d.ts +45 -0
- package/dist/client/transport/ws-rpc-client.d.ts.map +1 -0
- package/dist/client/transport/ws-rpc-client.js +94 -0
- package/dist/client/transport/ws-rpc-client.js.map +1 -0
- package/dist/common/do-rpc-schema.d.ts +76 -0
- package/dist/common/do-rpc-schema.d.ts.map +1 -0
- package/dist/common/do-rpc-schema.js +48 -0
- package/dist/common/do-rpc-schema.js.map +1 -0
- package/dist/common/http-rpc-schema.d.ts +58 -0
- package/dist/common/http-rpc-schema.d.ts.map +1 -0
- package/dist/common/http-rpc-schema.js +37 -0
- package/dist/common/http-rpc-schema.js.map +1 -0
- package/dist/common/mod.d.ts +5 -1
- package/dist/common/mod.d.ts.map +1 -1
- package/dist/common/mod.js +4 -1
- package/dist/common/mod.js.map +1 -1
- package/dist/common/sync-message-types.d.ts +236 -0
- package/dist/common/sync-message-types.d.ts.map +1 -0
- package/dist/common/sync-message-types.js +60 -0
- package/dist/common/sync-message-types.js.map +1 -0
- package/dist/common/ws-rpc-schema.d.ts +55 -0
- package/dist/common/ws-rpc-schema.d.ts.map +1 -0
- package/dist/common/ws-rpc-schema.js +32 -0
- package/dist/common/ws-rpc-schema.js.map +1 -0
- package/package.json +7 -8
- package/src/cf-worker/do/durable-object.ts +241 -0
- package/src/cf-worker/do/layer.ts +107 -0
- package/src/cf-worker/do/pull.ts +64 -0
- package/src/cf-worker/do/push.ts +162 -0
- package/src/cf-worker/do/sqlite.ts +28 -0
- package/src/cf-worker/do/sync-storage.ts +126 -0
- package/src/cf-worker/do/transport/do-rpc-server.ts +82 -0
- package/src/cf-worker/do/transport/http-rpc-server.ts +37 -0
- package/src/cf-worker/do/transport/ws-rpc-server.ts +34 -0
- package/src/cf-worker/mod.ts +4 -2
- package/src/cf-worker/shared.ts +95 -0
- package/src/cf-worker/worker.ts +72 -63
- package/src/client/mod.ts +3 -0
- package/src/client/transport/do-rpc-client.ts +171 -0
- package/src/client/transport/http-rpc-client.ts +205 -0
- package/src/client/transport/ws-rpc-client.ts +182 -0
- package/src/common/do-rpc-schema.ts +54 -0
- package/src/common/http-rpc-schema.ts +40 -0
- package/src/common/mod.ts +8 -1
- package/src/common/sync-message-types.ts +117 -0
- package/src/common/ws-rpc-schema.ts +36 -0
- package/src/cf-worker/cf-types.ts +0 -12
- package/src/cf-worker/durable-object.ts +0 -478
- package/src/common/ws-message-types.ts +0 -114
- package/src/sync-impl/mod.ts +0 -1
- package/src/sync-impl/ws-impl.ts +0 -274
|
@@ -0,0 +1,117 @@
|
|
|
1
|
+
import { BackendId, SyncBackend } from '@livestore/common'
|
|
2
|
+
import { EventSequenceNumber, LiveStoreEvent } from '@livestore/common/schema'
|
|
3
|
+
import { Schema } from '@livestore/utils/effect'
|
|
4
|
+
|
|
5
|
+
/**
|
|
6
|
+
* Common sync message types shared between different transport modes (WS, HTTP, RPC)
|
|
7
|
+
*
|
|
8
|
+
* These are the canonical message definitions used across all transport implementations.
|
|
9
|
+
*/
|
|
10
|
+
|
|
11
|
+
export const SyncMetadata = Schema.TaggedStruct('SyncMessage.SyncMetadata', {
|
|
12
|
+
/** ISO date format */
|
|
13
|
+
createdAt: Schema.String,
|
|
14
|
+
}).annotations({ title: '@livestore/sync-cf:SyncMetadata' })
|
|
15
|
+
|
|
16
|
+
export type SyncMetadata = typeof SyncMetadata.Type
|
|
17
|
+
|
|
18
|
+
export const PullRequest = Schema.Struct({
|
|
19
|
+
/** Omitting the cursor will start from the beginning */
|
|
20
|
+
cursor: Schema.Option(
|
|
21
|
+
Schema.Struct({
|
|
22
|
+
backendId: BackendId,
|
|
23
|
+
eventSequenceNumber: EventSequenceNumber.GlobalEventSequenceNumber,
|
|
24
|
+
}),
|
|
25
|
+
),
|
|
26
|
+
}).annotations({ title: '@livestore/sync-cf:PullRequest' })
|
|
27
|
+
|
|
28
|
+
export type PullRequest = typeof PullRequest.Type
|
|
29
|
+
|
|
30
|
+
export const PullResponse = Schema.Struct({
|
|
31
|
+
batch: Schema.Array(
|
|
32
|
+
Schema.Struct({
|
|
33
|
+
eventEncoded: LiveStoreEvent.AnyEncodedGlobal,
|
|
34
|
+
metadata: Schema.Option(SyncMetadata),
|
|
35
|
+
}),
|
|
36
|
+
),
|
|
37
|
+
pageInfo: SyncBackend.PullResPageInfo,
|
|
38
|
+
backendId: BackendId,
|
|
39
|
+
}).annotations({ title: '@livestore/sync-cf:PullResponse' })
|
|
40
|
+
|
|
41
|
+
export const emptyPullResponse = (backendId: string) =>
|
|
42
|
+
PullResponse.make({
|
|
43
|
+
batch: [],
|
|
44
|
+
pageInfo: SyncBackend.pageInfoNoMore,
|
|
45
|
+
backendId,
|
|
46
|
+
})
|
|
47
|
+
|
|
48
|
+
export type PullResponse = typeof PullResponse.Type
|
|
49
|
+
|
|
50
|
+
export const PushRequest = Schema.Struct({
|
|
51
|
+
batch: Schema.Array(LiveStoreEvent.AnyEncodedGlobal),
|
|
52
|
+
backendId: Schema.Option(BackendId),
|
|
53
|
+
}).annotations({ title: '@livestore/sync-cf:PushRequest' })
|
|
54
|
+
|
|
55
|
+
export type PushRequest = typeof PushRequest.Type
|
|
56
|
+
|
|
57
|
+
export const PushAck = Schema.Struct({}).annotations({
|
|
58
|
+
title: '@livestore/sync-cf:PushAck',
|
|
59
|
+
})
|
|
60
|
+
|
|
61
|
+
export type PushAck = typeof PushAck.Type
|
|
62
|
+
|
|
63
|
+
export const Ping = Schema.TaggedStruct('SyncMessage.Ping', {}).annotations({ title: '@livestore/sync-cf:Ping' })
|
|
64
|
+
|
|
65
|
+
export type Ping = typeof Ping.Type
|
|
66
|
+
|
|
67
|
+
export const Pong = Schema.TaggedStruct('SyncMessage.Pong', {}).annotations({ title: '@livestore/sync-cf:Pong' })
|
|
68
|
+
|
|
69
|
+
export type Pong = typeof Pong.Type
|
|
70
|
+
|
|
71
|
+
// Admin operations
|
|
72
|
+
export const AdminResetRoomRequest = Schema.TaggedStruct('SyncMessage.AdminResetRoomRequest', {
|
|
73
|
+
adminSecret: Schema.String,
|
|
74
|
+
}).annotations({ title: '@livestore/sync-cf:AdminResetRoomRequest' })
|
|
75
|
+
|
|
76
|
+
export type AdminResetRoomRequest = typeof AdminResetRoomRequest.Type
|
|
77
|
+
|
|
78
|
+
export const AdminResetRoomResponse = Schema.TaggedStruct('SyncMessage.AdminResetRoomResponse', {}).annotations({
|
|
79
|
+
title: '@livestore/sync-cf:AdminResetRoomResponse',
|
|
80
|
+
})
|
|
81
|
+
|
|
82
|
+
export type AdminResetRoomResponse = typeof AdminResetRoomResponse.Type
|
|
83
|
+
|
|
84
|
+
export const AdminInfoRequest = Schema.TaggedStruct('SyncMessage.AdminInfoRequest', {
|
|
85
|
+
adminSecret: Schema.String,
|
|
86
|
+
}).annotations({ title: '@livestore/sync-cf:AdminInfoRequest' })
|
|
87
|
+
|
|
88
|
+
export type AdminInfoRequest = typeof AdminInfoRequest.Type
|
|
89
|
+
|
|
90
|
+
export const AdminInfoResponse = Schema.TaggedStruct('SyncMessage.AdminInfoResponse', {
|
|
91
|
+
info: Schema.Struct({
|
|
92
|
+
durableObjectId: Schema.String,
|
|
93
|
+
}),
|
|
94
|
+
}).annotations({ title: '@livestore/sync-cf:AdminInfoResponse' })
|
|
95
|
+
|
|
96
|
+
export type AdminInfoResponse = typeof AdminInfoResponse.Type
|
|
97
|
+
|
|
98
|
+
export const BackendToClientMessage = Schema.Union(
|
|
99
|
+
PullResponse,
|
|
100
|
+
PushAck,
|
|
101
|
+
Pong,
|
|
102
|
+
AdminResetRoomResponse,
|
|
103
|
+
AdminInfoResponse,
|
|
104
|
+
)
|
|
105
|
+
export type BackendToClientMessage = typeof BackendToClientMessage.Type
|
|
106
|
+
|
|
107
|
+
export const ClientToBackendMessage = Schema.Union(
|
|
108
|
+
PullRequest,
|
|
109
|
+
PushRequest,
|
|
110
|
+
Ping,
|
|
111
|
+
AdminResetRoomRequest,
|
|
112
|
+
AdminInfoRequest,
|
|
113
|
+
)
|
|
114
|
+
export type ClientToBackendMessage = typeof ClientToBackendMessage.Type
|
|
115
|
+
|
|
116
|
+
export const Message = Schema.Union(BackendToClientMessage, ClientToBackendMessage)
|
|
117
|
+
export type Message = typeof Message.Type
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
import { InvalidPullError, InvalidPushError } from '@livestore/common'
|
|
2
|
+
import { Rpc, RpcGroup, Schema } from '@livestore/utils/effect'
|
|
3
|
+
import * as SyncMessage from './sync-message-types.ts'
|
|
4
|
+
|
|
5
|
+
/**
|
|
6
|
+
* WebSocket RPC Schema for LiveStore CF Sync Provider
|
|
7
|
+
*
|
|
8
|
+
* This defines the RPC endpoints available over WebSocket transport.
|
|
9
|
+
* Unlike HTTP transport which uses request/response patterns for each operation,
|
|
10
|
+
* WebSocket transport maintains a persistent connection and uses streaming responses.
|
|
11
|
+
*/
|
|
12
|
+
export class SyncWsRpc extends RpcGroup.make(
|
|
13
|
+
Rpc.make('SyncWsRpc.Pull', {
|
|
14
|
+
payload: Schema.Struct({
|
|
15
|
+
storeId: Schema.String,
|
|
16
|
+
payload: Schema.optional(Schema.JsonValue),
|
|
17
|
+
/** Whether to keep the pull stream alive and wait for more events */
|
|
18
|
+
live: Schema.Boolean,
|
|
19
|
+
...SyncMessage.PullRequest.fields,
|
|
20
|
+
}),
|
|
21
|
+
success: SyncMessage.PullResponse,
|
|
22
|
+
error: InvalidPullError,
|
|
23
|
+
stream: true,
|
|
24
|
+
}),
|
|
25
|
+
Rpc.make('SyncWsRpc.Push', {
|
|
26
|
+
payload: Schema.Struct({
|
|
27
|
+
storeId: Schema.String,
|
|
28
|
+
payload: Schema.optional(Schema.JsonValue),
|
|
29
|
+
...SyncMessage.PushRequest.fields,
|
|
30
|
+
}),
|
|
31
|
+
success: SyncMessage.PushAck,
|
|
32
|
+
error: InvalidPushError,
|
|
33
|
+
}),
|
|
34
|
+
// Ping <> Pong is handled by DO WS auto-response
|
|
35
|
+
// TODO add admin RPCs
|
|
36
|
+
) {}
|
|
@@ -1,478 +0,0 @@
|
|
|
1
|
-
import { UnexpectedError } from '@livestore/common'
|
|
2
|
-
import { EventSequenceNumber, type LiveStoreEvent, State } from '@livestore/common/schema'
|
|
3
|
-
import { shouldNeverHappen } from '@livestore/utils'
|
|
4
|
-
import { Effect, Logger, LogLevel, Option, Schema, UrlParams } from '@livestore/utils/effect'
|
|
5
|
-
import { SearchParamsSchema, WSMessage } from '../common/mod.ts'
|
|
6
|
-
import type { SyncMetadata } from '../common/ws-message-types.ts'
|
|
7
|
-
import type * as CfWorker from './cf-types.ts'
|
|
8
|
-
|
|
9
|
-
// NOTE We need to redeclare runtime types here to avoid type conflicts with the lib.dom Response type.
|
|
10
|
-
declare class Response extends CfWorker.Response {}
|
|
11
|
-
declare class WebSocketPair extends CfWorker.WebSocketPair {}
|
|
12
|
-
declare class WebSocketRequestResponsePair extends CfWorker.WebSocketRequestResponsePair {}
|
|
13
|
-
|
|
14
|
-
export interface Env {
|
|
15
|
-
DB: CfWorker.D1Database
|
|
16
|
-
ADMIN_SECRET: string
|
|
17
|
-
}
|
|
18
|
-
|
|
19
|
-
const encodeOutgoingMessage = Schema.encodeSync(Schema.parseJson(WSMessage.BackendToClientMessage))
|
|
20
|
-
const encodeIncomingMessage = Schema.encodeSync(Schema.parseJson(WSMessage.ClientToBackendMessage))
|
|
21
|
-
|
|
22
|
-
export const eventlogTable = State.SQLite.table({
|
|
23
|
-
// NOTE actual table name is determined at runtime
|
|
24
|
-
name: 'eventlog_$PERSISTENCE_FORMAT_VERSION_$storeId',
|
|
25
|
-
columns: {
|
|
26
|
-
seqNum: State.SQLite.integer({ primaryKey: true, schema: EventSequenceNumber.GlobalEventSequenceNumber }),
|
|
27
|
-
parentSeqNum: State.SQLite.integer({ schema: EventSequenceNumber.GlobalEventSequenceNumber }),
|
|
28
|
-
name: State.SQLite.text({}),
|
|
29
|
-
args: State.SQLite.text({ schema: Schema.parseJson(Schema.Any), nullable: true }),
|
|
30
|
-
/** ISO date format. Currently only used for debugging purposes. */
|
|
31
|
-
createdAt: State.SQLite.text({}),
|
|
32
|
-
clientId: State.SQLite.text({}),
|
|
33
|
-
sessionId: State.SQLite.text({}),
|
|
34
|
-
},
|
|
35
|
-
})
|
|
36
|
-
|
|
37
|
-
const WebSocketAttachmentSchema = Schema.parseJson(
|
|
38
|
-
Schema.Struct({
|
|
39
|
-
storeId: Schema.String,
|
|
40
|
-
payload: Schema.optional(Schema.JsonValue),
|
|
41
|
-
}),
|
|
42
|
-
)
|
|
43
|
-
|
|
44
|
-
export const PULL_CHUNK_SIZE = 100
|
|
45
|
-
|
|
46
|
-
/**
|
|
47
|
-
* Needs to be bumped when the storage format changes (e.g. eventlogTable schema changes)
|
|
48
|
-
*
|
|
49
|
-
* Changing this version number will lead to a "soft reset".
|
|
50
|
-
*/
|
|
51
|
-
export const PERSISTENCE_FORMAT_VERSION = 7
|
|
52
|
-
|
|
53
|
-
export type MakeDurableObjectClassOptions = {
|
|
54
|
-
onPush?: (
|
|
55
|
-
message: WSMessage.PushReq,
|
|
56
|
-
context: { storeId: string; payload?: Schema.JsonValue },
|
|
57
|
-
) => Effect.Effect<void> | Promise<void>
|
|
58
|
-
onPushRes?: (message: WSMessage.PushAck | WSMessage.Error) => Effect.Effect<void> | Promise<void>
|
|
59
|
-
onPull?: (
|
|
60
|
-
message: WSMessage.PullReq,
|
|
61
|
-
context: { storeId: string; payload?: Schema.JsonValue },
|
|
62
|
-
) => Effect.Effect<void> | Promise<void>
|
|
63
|
-
onPullRes?: (message: WSMessage.PullRes | WSMessage.Error) => Effect.Effect<void> | Promise<void>
|
|
64
|
-
}
|
|
65
|
-
|
|
66
|
-
export type MakeDurableObjectClass = (options?: MakeDurableObjectClassOptions) => {
|
|
67
|
-
new (ctx: CfWorker.DurableObjectState, env: Env): CfWorker.DurableObject
|
|
68
|
-
}
|
|
69
|
-
|
|
70
|
-
/**
|
|
71
|
-
* Creates a Durable Object class for handling WebSocket-based sync.
|
|
72
|
-
*
|
|
73
|
-
* Example:
|
|
74
|
-
* ```ts
|
|
75
|
-
* // In your Cloudflare Worker file
|
|
76
|
-
* import { makeDurableObject } from '@livestore/sync-cf/cf-worker'
|
|
77
|
-
*
|
|
78
|
-
* export class WebSocketServer extends makeDurableObject({
|
|
79
|
-
* onPush: async (message) => {
|
|
80
|
-
* console.log('onPush', message.batch)
|
|
81
|
-
* },
|
|
82
|
-
* onPull: async (message) => {
|
|
83
|
-
* console.log('onPull', message)
|
|
84
|
-
* },
|
|
85
|
-
* }) {}
|
|
86
|
-
* ```
|
|
87
|
-
*
|
|
88
|
-
* ```toml
|
|
89
|
-
* # wrangler.toml
|
|
90
|
-
* [new_classes]
|
|
91
|
-
* WebSocketServer = "src/websocket-server.ts"
|
|
92
|
-
* ```
|
|
93
|
-
*/
|
|
94
|
-
export const makeDurableObject: MakeDurableObjectClass = (options) => {
|
|
95
|
-
return class WebSocketServerBase implements CfWorker.DurableObject, CfWorker.Rpc.DurableObjectBranded {
|
|
96
|
-
__DURABLE_OBJECT_BRAND = 'WebSocketServerBase' as never
|
|
97
|
-
ctx: CfWorker.DurableObjectState
|
|
98
|
-
env: Env
|
|
99
|
-
|
|
100
|
-
constructor(ctx: CfWorker.DurableObjectState, env: Env) {
|
|
101
|
-
this.ctx = ctx
|
|
102
|
-
this.env = env
|
|
103
|
-
}
|
|
104
|
-
|
|
105
|
-
/** Needed to prevent concurrent pushes */
|
|
106
|
-
private pushSemaphore = Effect.makeSemaphore(1).pipe(Effect.runSync)
|
|
107
|
-
|
|
108
|
-
private currentHead: EventSequenceNumber.GlobalEventSequenceNumber | 'uninitialized' = 'uninitialized'
|
|
109
|
-
|
|
110
|
-
fetch = async (request: CfWorker.Request): Promise<CfWorker.Response> =>
|
|
111
|
-
Effect.sync(() => {
|
|
112
|
-
const { storeId, payload } = getRequestSearchParams(request)
|
|
113
|
-
const storage = makeStorage(this.ctx, this.env, storeId)
|
|
114
|
-
|
|
115
|
-
const { 0: client, 1: server } = new WebSocketPair()
|
|
116
|
-
|
|
117
|
-
// Since we're using websocket hibernation, we need to remember the storeId for subsequent `webSocketMessage` calls
|
|
118
|
-
server.serializeAttachment(Schema.encodeSync(WebSocketAttachmentSchema)({ storeId, payload }))
|
|
119
|
-
|
|
120
|
-
// See https://developers.cloudflare.com/durable-objects/examples/websocket-hibernation-server
|
|
121
|
-
|
|
122
|
-
this.ctx.acceptWebSocket(server)
|
|
123
|
-
|
|
124
|
-
this.ctx.setWebSocketAutoResponse(
|
|
125
|
-
new WebSocketRequestResponsePair(
|
|
126
|
-
encodeIncomingMessage(WSMessage.Ping.make({ requestId: 'ping' })),
|
|
127
|
-
encodeOutgoingMessage(WSMessage.Pong.make({ requestId: 'ping' })),
|
|
128
|
-
),
|
|
129
|
-
)
|
|
130
|
-
|
|
131
|
-
const colSpec = State.SQLite.makeColumnSpec(eventlogTable.sqliteDef.ast)
|
|
132
|
-
this.env.DB.exec(`CREATE TABLE IF NOT EXISTS ${storage.dbName} (${colSpec}) strict`)
|
|
133
|
-
|
|
134
|
-
return new Response(null, {
|
|
135
|
-
status: 101,
|
|
136
|
-
webSocket: client,
|
|
137
|
-
})
|
|
138
|
-
}).pipe(Effect.tapCauseLogPretty, Effect.runPromise)
|
|
139
|
-
|
|
140
|
-
webSocketMessage = (ws: CfWorker.WebSocket, message: ArrayBuffer | string): Promise<void> | undefined => {
|
|
141
|
-
const decodedMessageRes = Schema.decodeUnknownEither(Schema.parseJson(WSMessage.ClientToBackendMessage))(message)
|
|
142
|
-
|
|
143
|
-
if (decodedMessageRes._tag === 'Left') {
|
|
144
|
-
Effect.logError('Invalid message received', { message }).pipe(
|
|
145
|
-
Effect.provide(Logger.prettyWithThread('durable-object')),
|
|
146
|
-
Effect.runSync,
|
|
147
|
-
)
|
|
148
|
-
return
|
|
149
|
-
}
|
|
150
|
-
|
|
151
|
-
const decodedMessage = decodedMessageRes.right
|
|
152
|
-
|
|
153
|
-
const requestId = decodedMessage.requestId
|
|
154
|
-
|
|
155
|
-
return Effect.gen(this, function* () {
|
|
156
|
-
const { storeId, payload } = yield* Schema.decode(WebSocketAttachmentSchema)(ws.deserializeAttachment())
|
|
157
|
-
const storage = makeStorage(this.ctx, this.env, storeId)
|
|
158
|
-
|
|
159
|
-
switch (decodedMessage._tag) {
|
|
160
|
-
// TODO allow pulling concurrently to not block incoming push requests
|
|
161
|
-
case 'WSMessage.PullReq': {
|
|
162
|
-
if (options?.onPull) {
|
|
163
|
-
yield* Effect.tryAll(() => options.onPull!(decodedMessage, { storeId, payload }))
|
|
164
|
-
}
|
|
165
|
-
|
|
166
|
-
const respond = (message: WSMessage.PullRes) =>
|
|
167
|
-
Effect.gen(function* () {
|
|
168
|
-
if (options?.onPullRes) {
|
|
169
|
-
yield* Effect.tryAll(() => options.onPullRes!(message))
|
|
170
|
-
}
|
|
171
|
-
|
|
172
|
-
if (ws.readyState !== WebSocket.OPEN) {
|
|
173
|
-
yield* Effect.logWarning('WebSocket not open, skipping send', {
|
|
174
|
-
readyState: ws.readyState,
|
|
175
|
-
message,
|
|
176
|
-
})
|
|
177
|
-
return
|
|
178
|
-
}
|
|
179
|
-
|
|
180
|
-
yield* Effect.try({
|
|
181
|
-
try: () => ws.send(encodeOutgoingMessage(message)),
|
|
182
|
-
catch: (cause) =>
|
|
183
|
-
new UnexpectedError({ cause, note: 'Failed to send pull response', payload: { message } }),
|
|
184
|
-
})
|
|
185
|
-
})
|
|
186
|
-
|
|
187
|
-
const cursor = decodedMessage.cursor
|
|
188
|
-
|
|
189
|
-
// TODO use streaming
|
|
190
|
-
const remainingEvents = yield* storage.getEvents(cursor)
|
|
191
|
-
|
|
192
|
-
// Send at least one response, even if there are no events
|
|
193
|
-
const batches =
|
|
194
|
-
remainingEvents.length === 0
|
|
195
|
-
? [[]]
|
|
196
|
-
: Array.from({ length: Math.ceil(remainingEvents.length / PULL_CHUNK_SIZE) }, (_, i) =>
|
|
197
|
-
remainingEvents.slice(i * PULL_CHUNK_SIZE, (i + 1) * PULL_CHUNK_SIZE),
|
|
198
|
-
)
|
|
199
|
-
|
|
200
|
-
for (const [index, batch] of batches.entries()) {
|
|
201
|
-
const remaining = Math.max(0, remainingEvents.length - (index + 1) * PULL_CHUNK_SIZE)
|
|
202
|
-
yield* respond(WSMessage.PullRes.make({ batch, remaining, requestId: { context: 'pull', requestId } }))
|
|
203
|
-
}
|
|
204
|
-
|
|
205
|
-
break
|
|
206
|
-
}
|
|
207
|
-
case 'WSMessage.PushReq': {
|
|
208
|
-
const respond = (message: WSMessage.PushAck | WSMessage.Error) =>
|
|
209
|
-
Effect.gen(function* () {
|
|
210
|
-
if (options?.onPushRes) {
|
|
211
|
-
yield* Effect.tryAll(() => options.onPushRes!(message))
|
|
212
|
-
}
|
|
213
|
-
ws.send(encodeOutgoingMessage(message))
|
|
214
|
-
})
|
|
215
|
-
|
|
216
|
-
if (decodedMessage.batch.length === 0) {
|
|
217
|
-
yield* respond(WSMessage.PushAck.make({ requestId }))
|
|
218
|
-
return
|
|
219
|
-
}
|
|
220
|
-
|
|
221
|
-
yield* this.pushSemaphore.take(1)
|
|
222
|
-
|
|
223
|
-
if (options?.onPush) {
|
|
224
|
-
yield* Effect.tryAll(() => options.onPush!(decodedMessage, { storeId, payload }))
|
|
225
|
-
}
|
|
226
|
-
|
|
227
|
-
// TODO check whether we could use the Durable Object storage for this to speed up the lookup
|
|
228
|
-
// const expectedParentNum = yield* storage.getHead
|
|
229
|
-
let currentHead: EventSequenceNumber.GlobalEventSequenceNumber
|
|
230
|
-
if (this.currentHead === 'uninitialized') {
|
|
231
|
-
const currentHeadFromStorage = yield* Effect.promise(() => this.ctx.storage.get('currentHead'))
|
|
232
|
-
// console.log('currentHeadFromStorage', currentHeadFromStorage)
|
|
233
|
-
if (currentHeadFromStorage === undefined) {
|
|
234
|
-
// console.log('currentHeadFromStorage is null, getting from D1')
|
|
235
|
-
// currentHead = yield* storage.getHead
|
|
236
|
-
// console.log('currentHeadFromStorage is null, using root')
|
|
237
|
-
currentHead = EventSequenceNumber.ROOT.global
|
|
238
|
-
} else {
|
|
239
|
-
currentHead = currentHeadFromStorage as EventSequenceNumber.GlobalEventSequenceNumber
|
|
240
|
-
}
|
|
241
|
-
} else {
|
|
242
|
-
// console.log('currentHead is already initialized', this.currentHead)
|
|
243
|
-
currentHead = this.currentHead
|
|
244
|
-
}
|
|
245
|
-
|
|
246
|
-
// TODO handle clientId unique conflict
|
|
247
|
-
// Validate the batch
|
|
248
|
-
const firstEvent = decodedMessage.batch[0]!
|
|
249
|
-
if (firstEvent.parentSeqNum !== currentHead) {
|
|
250
|
-
const err = WSMessage.Error.make({
|
|
251
|
-
message: `Invalid parent event number. Received e${firstEvent.parentSeqNum} but expected e${currentHead}`,
|
|
252
|
-
requestId,
|
|
253
|
-
})
|
|
254
|
-
|
|
255
|
-
yield* Effect.logError(err)
|
|
256
|
-
|
|
257
|
-
yield* respond(err)
|
|
258
|
-
yield* this.pushSemaphore.release(1)
|
|
259
|
-
return
|
|
260
|
-
}
|
|
261
|
-
|
|
262
|
-
yield* respond(WSMessage.PushAck.make({ requestId }))
|
|
263
|
-
|
|
264
|
-
const createdAt = new Date().toISOString()
|
|
265
|
-
|
|
266
|
-
// NOTE we're not waiting for this to complete yet to allow the broadcast to happen right away
|
|
267
|
-
// while letting the async storage write happen in the background
|
|
268
|
-
const storeFiber = yield* storage.appendEvents(decodedMessage.batch, createdAt).pipe(Effect.fork)
|
|
269
|
-
|
|
270
|
-
this.currentHead = decodedMessage.batch.at(-1)!.seqNum
|
|
271
|
-
yield* Effect.promise(() => this.ctx.storage.put('currentHead', this.currentHead))
|
|
272
|
-
|
|
273
|
-
yield* this.pushSemaphore.release(1)
|
|
274
|
-
|
|
275
|
-
const connectedClients = this.ctx.getWebSockets()
|
|
276
|
-
|
|
277
|
-
// console.debug(`Broadcasting push batch to ${this.subscribedWebSockets.size} clients`)
|
|
278
|
-
if (connectedClients.length > 0) {
|
|
279
|
-
// TODO refactor to batch api
|
|
280
|
-
const pullRes = WSMessage.PullRes.make({
|
|
281
|
-
batch: decodedMessage.batch.map((eventEncoded) => ({
|
|
282
|
-
eventEncoded,
|
|
283
|
-
metadata: Option.some({ createdAt }),
|
|
284
|
-
})),
|
|
285
|
-
remaining: 0,
|
|
286
|
-
requestId: { context: 'push', requestId },
|
|
287
|
-
})
|
|
288
|
-
const pullResEnc = encodeOutgoingMessage(pullRes)
|
|
289
|
-
|
|
290
|
-
// Only calling once for now.
|
|
291
|
-
if (options?.onPullRes) {
|
|
292
|
-
yield* Effect.tryAll(() => options.onPullRes!(pullRes))
|
|
293
|
-
}
|
|
294
|
-
|
|
295
|
-
// NOTE we're also sending the pullRes to the pushing ws client as a confirmation
|
|
296
|
-
for (const conn of connectedClients) {
|
|
297
|
-
conn.send(pullResEnc)
|
|
298
|
-
}
|
|
299
|
-
}
|
|
300
|
-
|
|
301
|
-
// Wait for the storage write to complete before finishing this request
|
|
302
|
-
yield* storeFiber
|
|
303
|
-
|
|
304
|
-
break
|
|
305
|
-
}
|
|
306
|
-
case 'WSMessage.AdminResetRoomReq': {
|
|
307
|
-
if (decodedMessage.adminSecret !== this.env.ADMIN_SECRET) {
|
|
308
|
-
ws.send(encodeOutgoingMessage(WSMessage.Error.make({ message: 'Invalid admin secret', requestId })))
|
|
309
|
-
return
|
|
310
|
-
}
|
|
311
|
-
|
|
312
|
-
yield* storage.resetStore
|
|
313
|
-
ws.send(encodeOutgoingMessage(WSMessage.AdminResetRoomRes.make({ requestId })))
|
|
314
|
-
|
|
315
|
-
break
|
|
316
|
-
}
|
|
317
|
-
case 'WSMessage.AdminInfoReq': {
|
|
318
|
-
if (decodedMessage.adminSecret !== this.env.ADMIN_SECRET) {
|
|
319
|
-
ws.send(encodeOutgoingMessage(WSMessage.Error.make({ message: 'Invalid admin secret', requestId })))
|
|
320
|
-
return
|
|
321
|
-
}
|
|
322
|
-
|
|
323
|
-
ws.send(
|
|
324
|
-
encodeOutgoingMessage(
|
|
325
|
-
WSMessage.AdminInfoRes.make({ requestId, info: { durableObjectId: this.ctx.id.toString() } }),
|
|
326
|
-
),
|
|
327
|
-
)
|
|
328
|
-
|
|
329
|
-
break
|
|
330
|
-
}
|
|
331
|
-
default: {
|
|
332
|
-
console.error('unsupported message', decodedMessage)
|
|
333
|
-
return shouldNeverHappen()
|
|
334
|
-
}
|
|
335
|
-
}
|
|
336
|
-
}).pipe(
|
|
337
|
-
Effect.withSpan(`@livestore/sync-cf:durable-object:webSocketMessage:${decodedMessage._tag}`, {
|
|
338
|
-
attributes: { requestId },
|
|
339
|
-
}),
|
|
340
|
-
Effect.tapCauseLogPretty,
|
|
341
|
-
Effect.tapErrorCause((cause) =>
|
|
342
|
-
Effect.sync(() =>
|
|
343
|
-
ws.send(encodeOutgoingMessage(WSMessage.Error.make({ message: cause.toString(), requestId }))),
|
|
344
|
-
),
|
|
345
|
-
),
|
|
346
|
-
Logger.withMinimumLogLevel(LogLevel.Debug),
|
|
347
|
-
Effect.provide(Logger.prettyWithThread('durable-object')),
|
|
348
|
-
Effect.runPromise,
|
|
349
|
-
)
|
|
350
|
-
}
|
|
351
|
-
|
|
352
|
-
webSocketClose = async (
|
|
353
|
-
ws: CfWorker.WebSocket,
|
|
354
|
-
code: number,
|
|
355
|
-
_reason: string,
|
|
356
|
-
_wasClean: boolean,
|
|
357
|
-
): Promise<void> => {
|
|
358
|
-
// If the client closes the connection, the runtime will invoke the webSocketClose() handler.
|
|
359
|
-
ws.close(code, 'Durable Object is closing WebSocket')
|
|
360
|
-
}
|
|
361
|
-
}
|
|
362
|
-
}
|
|
363
|
-
|
|
364
|
-
type SyncStorage = {
|
|
365
|
-
dbName: string
|
|
366
|
-
// getHead: Effect.Effect<EventSequenceNumber.GlobalEventSequenceNumber, UnexpectedError>
|
|
367
|
-
getEvents: (
|
|
368
|
-
cursor: number | undefined,
|
|
369
|
-
) => Effect.Effect<
|
|
370
|
-
ReadonlyArray<{ eventEncoded: LiveStoreEvent.AnyEncodedGlobal; metadata: Option.Option<SyncMetadata> }>,
|
|
371
|
-
UnexpectedError
|
|
372
|
-
>
|
|
373
|
-
appendEvents: (
|
|
374
|
-
batch: ReadonlyArray<LiveStoreEvent.AnyEncodedGlobal>,
|
|
375
|
-
createdAt: string,
|
|
376
|
-
) => Effect.Effect<void, UnexpectedError>
|
|
377
|
-
resetStore: Effect.Effect<void, UnexpectedError>
|
|
378
|
-
}
|
|
379
|
-
|
|
380
|
-
const makeStorage = (ctx: any, env: Env, storeId: string): SyncStorage => {
|
|
381
|
-
const dbName = `eventlog_${PERSISTENCE_FORMAT_VERSION}_${toValidTableName(storeId)}`
|
|
382
|
-
|
|
383
|
-
const execDb = <T>(cb: (db: CfWorker.D1Database) => Promise<CfWorker.D1Result<T>>) =>
|
|
384
|
-
Effect.tryPromise({
|
|
385
|
-
try: () => cb(env.DB),
|
|
386
|
-
catch: (error) => new UnexpectedError({ cause: error, payload: { dbName } }),
|
|
387
|
-
}).pipe(
|
|
388
|
-
Effect.map((_) => _.results),
|
|
389
|
-
Effect.withSpan('@livestore/sync-cf:durable-object:execDb'),
|
|
390
|
-
)
|
|
391
|
-
|
|
392
|
-
// const getHead: Effect.Effect<EventSequenceNumber.GlobalEventSequenceNumber, UnexpectedError> = Effect.gen(
|
|
393
|
-
// function* () {
|
|
394
|
-
// const result = yield* execDb<{ seqNum: EventSequenceNumber.GlobalEventSequenceNumber }>((db) =>
|
|
395
|
-
// db.prepare(`SELECT seqNum FROM ${dbName} ORDER BY seqNum DESC LIMIT 1`).all(),
|
|
396
|
-
// )
|
|
397
|
-
|
|
398
|
-
// return result[0]?.seqNum ?? EventSequenceNumber.ROOT.global
|
|
399
|
-
// },
|
|
400
|
-
// ).pipe(UnexpectedError.mapToUnexpectedError)
|
|
401
|
-
|
|
402
|
-
const getEvents = (
|
|
403
|
-
cursor: number | undefined,
|
|
404
|
-
): Effect.Effect<
|
|
405
|
-
ReadonlyArray<{ eventEncoded: LiveStoreEvent.AnyEncodedGlobal; metadata: Option.Option<SyncMetadata> }>,
|
|
406
|
-
UnexpectedError
|
|
407
|
-
> =>
|
|
408
|
-
Effect.gen(function* () {
|
|
409
|
-
const whereClause = cursor === undefined ? '' : `WHERE seqNum > ${cursor}`
|
|
410
|
-
const sql = `SELECT * FROM ${dbName} ${whereClause} ORDER BY seqNum ASC`
|
|
411
|
-
// TODO handle case where `cursor` was not found
|
|
412
|
-
const rawEvents = yield* execDb((db) => db.prepare(sql).all())
|
|
413
|
-
const events = Schema.decodeUnknownSync(Schema.Array(eventlogTable.rowSchema))(rawEvents).map(
|
|
414
|
-
({ createdAt, ...eventEncoded }) => ({
|
|
415
|
-
eventEncoded,
|
|
416
|
-
metadata: Option.some({ createdAt }),
|
|
417
|
-
}),
|
|
418
|
-
)
|
|
419
|
-
return events
|
|
420
|
-
}).pipe(UnexpectedError.mapToUnexpectedError)
|
|
421
|
-
|
|
422
|
-
const appendEvents: SyncStorage['appendEvents'] = (batch, createdAt) =>
|
|
423
|
-
Effect.gen(function* () {
|
|
424
|
-
// If there are no events, do nothing.
|
|
425
|
-
if (batch.length === 0) return
|
|
426
|
-
|
|
427
|
-
// CF D1 limits:
|
|
428
|
-
// Maximum bound parameters per query 100, Maximum arguments per SQL function 32
|
|
429
|
-
// Thus we need to split the batch into chunks of max (100/7=)14 events each.
|
|
430
|
-
const CHUNK_SIZE = 14
|
|
431
|
-
|
|
432
|
-
for (let i = 0; i < batch.length; i += CHUNK_SIZE) {
|
|
433
|
-
const chunk = batch.slice(i, i + CHUNK_SIZE)
|
|
434
|
-
|
|
435
|
-
// Create a list of placeholders ("(?, ?, ?, ?, ?, ?, ?)"), corresponding to each event.
|
|
436
|
-
const valuesPlaceholders = chunk.map(() => '(?, ?, ?, ?, ?, ?, ?)').join(', ')
|
|
437
|
-
const sql = `INSERT INTO ${dbName} (seqNum, parentSeqNum, args, name, createdAt, clientId, sessionId) VALUES ${valuesPlaceholders}`
|
|
438
|
-
// Flatten the event properties into a parameters array.
|
|
439
|
-
const params = chunk.flatMap((event) => [
|
|
440
|
-
event.seqNum,
|
|
441
|
-
event.parentSeqNum,
|
|
442
|
-
event.args === undefined ? null : JSON.stringify(event.args),
|
|
443
|
-
event.name,
|
|
444
|
-
createdAt,
|
|
445
|
-
event.clientId,
|
|
446
|
-
event.sessionId,
|
|
447
|
-
])
|
|
448
|
-
|
|
449
|
-
yield* execDb((db) =>
|
|
450
|
-
db
|
|
451
|
-
.prepare(sql)
|
|
452
|
-
.bind(...params)
|
|
453
|
-
.run(),
|
|
454
|
-
)
|
|
455
|
-
}
|
|
456
|
-
}).pipe(UnexpectedError.mapToUnexpectedError)
|
|
457
|
-
|
|
458
|
-
const resetStore = Effect.gen(function* () {
|
|
459
|
-
yield* Effect.promise(() => ctx.storage.deleteAll())
|
|
460
|
-
}).pipe(UnexpectedError.mapToUnexpectedError)
|
|
461
|
-
|
|
462
|
-
return {
|
|
463
|
-
dbName,
|
|
464
|
-
// getHead,
|
|
465
|
-
getEvents,
|
|
466
|
-
appendEvents,
|
|
467
|
-
resetStore,
|
|
468
|
-
}
|
|
469
|
-
}
|
|
470
|
-
|
|
471
|
-
const getRequestSearchParams = (request: CfWorker.Request) => {
|
|
472
|
-
const url = new URL(request.url)
|
|
473
|
-
const urlParams = UrlParams.fromInput(url.searchParams)
|
|
474
|
-
const paramsResult = UrlParams.schemaStruct(SearchParamsSchema)(urlParams).pipe(Effect.runSync)
|
|
475
|
-
return paramsResult
|
|
476
|
-
}
|
|
477
|
-
|
|
478
|
-
const toValidTableName = (str: string) => str.replaceAll(/[^a-zA-Z0-9]/g, '_')
|