@livestore/sync-cf 0.3.0-dev.2 → 0.3.0-dev.21

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (48) hide show
  1. package/dist/.tsbuildinfo +1 -1
  2. package/dist/cf-worker/durable-object.d.ts +42 -37
  3. package/dist/cf-worker/durable-object.d.ts.map +1 -1
  4. package/dist/cf-worker/durable-object.js +224 -152
  5. package/dist/cf-worker/durable-object.js.map +1 -1
  6. package/dist/cf-worker/mod.d.ts +3 -0
  7. package/dist/cf-worker/mod.d.ts.map +1 -0
  8. package/dist/cf-worker/mod.js +3 -0
  9. package/dist/cf-worker/mod.js.map +1 -0
  10. package/dist/cf-worker/worker.d.ts +6 -0
  11. package/dist/cf-worker/worker.d.ts.map +1 -0
  12. package/dist/cf-worker/worker.js +29 -0
  13. package/dist/cf-worker/worker.js.map +1 -0
  14. package/dist/common/{index.js → mod.d.ts} +1 -1
  15. package/dist/common/mod.d.ts.map +1 -0
  16. package/dist/common/{index.d.ts → mod.js} +1 -1
  17. package/dist/common/mod.js.map +1 -0
  18. package/dist/common/ws-message-types.d.ts +58 -233
  19. package/dist/common/ws-message-types.d.ts.map +1 -1
  20. package/dist/common/ws-message-types.js +6 -11
  21. package/dist/common/ws-message-types.js.map +1 -1
  22. package/dist/sync-impl/mod.d.ts +2 -0
  23. package/dist/sync-impl/mod.d.ts.map +1 -0
  24. package/dist/sync-impl/mod.js +2 -0
  25. package/dist/sync-impl/mod.js.map +1 -0
  26. package/dist/sync-impl/ws-impl.d.ts +3 -12
  27. package/dist/sync-impl/ws-impl.d.ts.map +1 -1
  28. package/dist/sync-impl/ws-impl.js +73 -28
  29. package/dist/sync-impl/ws-impl.js.map +1 -1
  30. package/package.json +15 -13
  31. package/src/cf-worker/durable-object.ts +299 -172
  32. package/src/cf-worker/mod.ts +2 -0
  33. package/src/cf-worker/worker.ts +39 -0
  34. package/src/common/ws-message-types.ts +7 -22
  35. package/src/sync-impl/ws-impl.ts +85 -47
  36. package/dist/cf-worker/index.d.ts +0 -8
  37. package/dist/cf-worker/index.d.ts.map +0 -1
  38. package/dist/cf-worker/index.js +0 -67
  39. package/dist/cf-worker/index.js.map +0 -1
  40. package/dist/common/index.d.ts.map +0 -1
  41. package/dist/common/index.js.map +0 -1
  42. package/dist/sync-impl/index.d.ts +0 -2
  43. package/dist/sync-impl/index.d.ts.map +0 -1
  44. package/dist/sync-impl/index.js +0 -2
  45. package/dist/sync-impl/index.js.map +0 -1
  46. package/src/cf-worker/index.ts +0 -84
  47. /package/src/common/{index.ts → mod.ts} +0 -0
  48. /package/src/sync-impl/{index.ts → mod.ts} +0 -0
@@ -1,14 +1,14 @@
1
- import { makeColumnSpec } from '@livestore/common'
1
+ import { makeColumnSpec, UnexpectedError } from '@livestore/common'
2
2
  import { DbSchema, EventId, type MutationEvent } from '@livestore/common/schema'
3
3
  import { shouldNeverHappen } from '@livestore/utils'
4
4
  import { Effect, Logger, LogLevel, Option, Schema } from '@livestore/utils/effect'
5
5
  import { DurableObject } from 'cloudflare:workers'
6
6
 
7
- import { WSMessage } from '../common/index.js'
7
+ import { WSMessage } from '../common/mod.js'
8
8
  import type { SyncMetadata } from '../common/ws-message-types.js'
9
9
 
10
10
  export interface Env {
11
- WEBSOCKET_SERVER: DurableObjectNamespace<WebSocketServer>
11
+ WEBSOCKET_SERVER: DurableObjectNamespace
12
12
  DB: D1Database
13
13
  ADMIN_SECRET: string
14
14
  }
@@ -19,51 +19,83 @@ const encodeOutgoingMessage = Schema.encodeSync(Schema.parseJson(WSMessage.Backe
19
19
  const encodeIncomingMessage = Schema.encodeSync(Schema.parseJson(WSMessage.ClientToBackendMessage))
20
20
  const decodeIncomingMessage = Schema.decodeUnknownEither(Schema.parseJson(WSMessage.ClientToBackendMessage))
21
21
 
22
- // NOTE actual table name is determined at runtime by `WebSocketServer.dbName`
23
- export const mutationLogTable = DbSchema.table('__unused', {
24
- idGlobal: DbSchema.integer({ primaryKey: true }),
25
- parentIdGlobal: DbSchema.integer({}),
22
+ // NOTE actual table name is determined at runtime
23
+ export const mutationLogTable = DbSchema.table('mutation_log_${PERSISTENCE_FORMAT_VERSION}_${storeId}', {
24
+ id: DbSchema.integer({ primaryKey: true, schema: EventId.GlobalEventId }),
25
+ parentId: DbSchema.integer({ schema: EventId.GlobalEventId }),
26
26
  mutation: DbSchema.text({}),
27
27
  args: DbSchema.text({ schema: Schema.parseJson(Schema.Any) }),
28
- /** ISO date format */
28
+ /** ISO date format. Currently only used for debugging purposes. */
29
29
  createdAt: DbSchema.text({}),
30
+ clientId: DbSchema.text({}),
30
31
  })
31
32
 
32
- // Durable Object
33
- export class WebSocketServer extends DurableObject<Env> {
34
- dbName = `mutation_log_${this.ctx.id.toString()}`
35
- storage = makeStorage(this.ctx, this.env, this.dbName)
33
+ const WebSocketAttachmentSchema = Schema.parseJson(
34
+ Schema.Struct({
35
+ storeId: Schema.String,
36
+ }),
37
+ )
38
+
39
+ export const PULL_CHUNK_SIZE = 100
40
+
41
+ /**
42
+ * Needs to be bumped when the storage format changes (e.g. mutationLogTable schema changes)
43
+ *
44
+ * Changing this version number will lead to a "soft reset".
45
+ */
46
+ export const PERSISTENCE_FORMAT_VERSION = 3
47
+
48
+ export type MakeDurableObjectClassOptions = {
49
+ onPush?: (message: WSMessage.PushReq) => Effect.Effect<void> | Promise<void>
50
+ onPushRes?: (message: WSMessage.PushAck | WSMessage.Error) => Effect.Effect<void> | Promise<void>
51
+ onPull?: (message: WSMessage.PullReq) => Effect.Effect<void> | Promise<void>
52
+ onPullRes?: (message: WSMessage.PullRes | WSMessage.Error) => Effect.Effect<void> | Promise<void>
53
+ }
36
54
 
37
- constructor(ctx: DurableObjectState, env: Env) {
38
- super(ctx, env)
39
- }
55
+ export type MakeDurableObjectClass = (options?: MakeDurableObjectClassOptions) => {
56
+ new (ctx: DurableObjectState, env: Env): DurableObject<Env>
57
+ }
40
58
 
41
- fetch = async (_request: Request) =>
42
- Effect.gen(this, function* () {
43
- const { 0: client, 1: server } = new WebSocketPair()
59
+ export const makeDurableObject: MakeDurableObjectClass = (options) => {
60
+ return class WebSocketServerBase extends DurableObject<Env> {
61
+ /** Needed to prevent concurrent pushes */
62
+ private pushSemaphore = Effect.makeSemaphore(1).pipe(Effect.runSync)
44
63
 
45
- // See https://developers.cloudflare.com/durable-objects/examples/websocket-hibernation-server
64
+ constructor(ctx: DurableObjectState, env: Env) {
65
+ super(ctx, env)
66
+ }
46
67
 
47
- this.ctx.acceptWebSocket(server)
68
+ fetch = async (request: Request) =>
69
+ Effect.gen(this, function* () {
70
+ const storeId = getStoreId(request)
71
+ const storage = makeStorage(this.ctx, this.env, storeId)
48
72
 
49
- this.ctx.setWebSocketAutoResponse(
50
- new WebSocketRequestResponsePair(
51
- encodeIncomingMessage(WSMessage.Ping.make({ requestId: 'ping' })),
52
- encodeOutgoingMessage(WSMessage.Pong.make({ requestId: 'ping' })),
53
- ),
54
- )
73
+ const { 0: client, 1: server } = new WebSocketPair()
55
74
 
56
- const colSpec = makeColumnSpec(mutationLogTable.sqliteDef.ast)
57
- this.env.DB.exec(`CREATE TABLE IF NOT EXISTS ${this.dbName} (${colSpec}) strict`)
75
+ // Since we're using websocket hibernation, we need to remember the storeId for subsequent `webSocketMessage` calls
76
+ server.serializeAttachment(Schema.encodeSync(WebSocketAttachmentSchema)({ storeId }))
58
77
 
59
- return new Response(null, {
60
- status: 101,
61
- webSocket: client,
62
- })
63
- }).pipe(Effect.tapCauseLogPretty, Effect.runPromise)
78
+ // See https://developers.cloudflare.com/durable-objects/examples/websocket-hibernation-server
64
79
 
65
- webSocketMessage = (ws: WebSocketClient, message: ArrayBuffer | string) =>
66
- Effect.gen(this, function* () {
80
+ this.ctx.acceptWebSocket(server)
81
+
82
+ this.ctx.setWebSocketAutoResponse(
83
+ new WebSocketRequestResponsePair(
84
+ encodeIncomingMessage(WSMessage.Ping.make({ requestId: 'ping' })),
85
+ encodeOutgoingMessage(WSMessage.Pong.make({ requestId: 'ping' })),
86
+ ),
87
+ )
88
+
89
+ const colSpec = makeColumnSpec(mutationLogTable.sqliteDef.ast)
90
+ this.env.DB.exec(`CREATE TABLE IF NOT EXISTS ${storage.dbName} (${colSpec}) strict`)
91
+
92
+ return new Response(null, {
93
+ status: 101,
94
+ webSocket: client,
95
+ })
96
+ }).pipe(Effect.tapCauseLogPretty, Effect.runPromise)
97
+
98
+ webSocketMessage = (ws: WebSocketClient, message: ArrayBuffer | string) => {
67
99
  const decodedMessageRes = decodeIncomingMessage(message)
68
100
 
69
101
  if (decodedMessageRes._tag === 'Left') {
@@ -74,188 +106,283 @@ export class WebSocketServer extends DurableObject<Env> {
74
106
  const decodedMessage = decodedMessageRes.right
75
107
  const requestId = decodedMessage.requestId
76
108
 
77
- try {
78
- switch (decodedMessage._tag) {
79
- case 'WSMessage.PullReq': {
80
- const cursor = decodedMessage.cursor
81
- const CHUNK_SIZE = 100
109
+ return Effect.gen(this, function* () {
110
+ const { storeId } = yield* Schema.decode(WebSocketAttachmentSchema)(ws.deserializeAttachment())
111
+ const storage = makeStorage(this.ctx, this.env, storeId)
112
+
113
+ try {
114
+ switch (decodedMessage._tag) {
115
+ // TODO allow pulling concurrently to not block incoming push requests
116
+ case 'WSMessage.PullReq': {
117
+ if (options?.onPull) {
118
+ yield* Effect.tryAll(() => options.onPull!(decodedMessage))
119
+ }
82
120
 
83
- // TODO use streaming
84
- const remainingEvents = [...(yield* Effect.promise(() => this.storage.getEvents(cursor)))]
121
+ const respond = (message: WSMessage.PullRes) =>
122
+ Effect.gen(function* () {
123
+ if (options?.onPullRes) {
124
+ yield* Effect.tryAll(() => options.onPullRes!(message))
125
+ }
126
+ ws.send(encodeOutgoingMessage(message))
127
+ })
85
128
 
86
- // NOTE we want to make sure the WS server responds at least once with `InitRes` even if `events` is empty
87
- while (true) {
88
- const events = remainingEvents.splice(0, CHUNK_SIZE)
129
+ const cursor = decodedMessage.cursor
89
130
 
90
- ws.send(
91
- encodeOutgoingMessage(WSMessage.PullRes.make({ events, remaining: remainingEvents.length, requestId })),
92
- )
131
+ // TODO use streaming
132
+ const remainingEvents = yield* storage.getEvents(cursor)
133
+
134
+ // Send at least one response, even if there are no events
135
+ const batches =
136
+ remainingEvents.length === 0
137
+ ? [[]]
138
+ : Array.from({ length: Math.ceil(remainingEvents.length / PULL_CHUNK_SIZE) }, (_, i) =>
139
+ remainingEvents.slice(i * PULL_CHUNK_SIZE, (i + 1) * PULL_CHUNK_SIZE),
140
+ )
93
141
 
94
- if (remainingEvents.length === 0) {
95
- break
142
+ for (const [index, batch] of batches.entries()) {
143
+ const remaining = Math.max(0, remainingEvents.length - (index + 1) * PULL_CHUNK_SIZE)
144
+ yield* respond(WSMessage.PullRes.make({ batch, remaining, requestId: { context: 'pull', requestId } }))
96
145
  }
146
+
147
+ break
97
148
  }
149
+ case 'WSMessage.PushReq': {
150
+ const respond = (message: WSMessage.PushAck | WSMessage.Error) =>
151
+ Effect.gen(function* () {
152
+ if (options?.onPushRes) {
153
+ yield* Effect.tryAll(() => options.onPushRes!(message))
154
+ }
155
+ ws.send(encodeOutgoingMessage(message))
156
+ })
98
157
 
99
- break
100
- }
101
- case 'WSMessage.PushReq': {
102
- // TODO check whether we could use the Durable Object storage for this to speed up the lookup
103
- const latestEvent = yield* Effect.promise(() => this.storage.getLatestEvent())
104
- const expectedParentId = latestEvent?.id ?? EventId.ROOT
105
-
106
- let i = 0
107
- for (const mutationEventEncoded of decodedMessage.batch) {
108
- if (mutationEventEncoded.parentId.global !== expectedParentId.global + i) {
158
+ if (decodedMessage.batch.length === 0) {
159
+ yield* respond(WSMessage.PushAck.make({ requestId }))
160
+ return
161
+ }
162
+
163
+ yield* this.pushSemaphore.take(1)
164
+
165
+ if (options?.onPush) {
166
+ yield* Effect.tryAll(() => options.onPush!(decodedMessage))
167
+ }
168
+
169
+ // TODO check whether we could use the Durable Object storage for this to speed up the lookup
170
+ const expectedParentId = yield* storage.getHead
171
+
172
+ // TODO handle clientId unique conflict
173
+ // Validate the batch
174
+ const firstEvent = decodedMessage.batch[0]!
175
+ if (firstEvent.parentId !== expectedParentId) {
109
176
  const err = WSMessage.Error.make({
110
- message: `Invalid parent id. Received ${mutationEventEncoded.parentId.global} but expected ${expectedParentId.global}`,
177
+ message: `Invalid parent id. Received ${firstEvent.parentId} but expected ${expectedParentId}`,
111
178
  requestId,
112
179
  })
113
180
 
114
- yield* Effect.fail(err).pipe(Effect.ignoreLogged)
181
+ yield* Effect.logError(err)
115
182
 
116
- ws.send(encodeOutgoingMessage(err))
183
+ yield* respond(err)
184
+ yield* this.pushSemaphore.release(1)
117
185
  return
118
186
  }
119
187
 
120
- // TODO handle clientId unique conflict
188
+ yield* respond(WSMessage.PushAck.make({ requestId }))
121
189
 
122
- const createdAt = new Date().toISOString()
190
+ yield* this.pushSemaphore.release(1)
123
191
 
124
- // NOTE we're currently not blocking on this to allow broadcasting right away
125
- const storePromise = this.storage.appendEvent(mutationEventEncoded, createdAt)
126
-
127
- ws.send(
128
- encodeOutgoingMessage(
129
- WSMessage.PushAck.make({ mutationId: mutationEventEncoded.id.global, requestId }),
130
- ),
131
- )
192
+ const createdAt = new Date().toISOString()
132
193
 
133
- // console.debug(`Broadcasting mutation event to ${this.subscribedWebSockets.size} clients`)
194
+ // NOTE we're not waiting for this to complete yet to allow the broadcast to happen right away
195
+ // while letting the async storage write happen in the background
196
+ const storeFiber = yield* storage.appendEvents(decodedMessage.batch, createdAt).pipe(Effect.fork)
134
197
 
135
198
  const connectedClients = this.ctx.getWebSockets()
136
199
 
200
+ // console.debug(`Broadcasting push batch to ${this.subscribedWebSockets.size} clients`)
137
201
  if (connectedClients.length > 0) {
138
- const broadcastMessage = encodeOutgoingMessage(
139
- // TODO refactor to batch api
140
- WSMessage.PushBroadcast.make({
202
+ // TODO refactor to batch api
203
+ const pullRes = WSMessage.PullRes.make({
204
+ batch: decodedMessage.batch.map((mutationEventEncoded) => ({
141
205
  mutationEventEncoded,
142
206
  metadata: Option.some({ createdAt }),
143
- }),
144
- )
207
+ })),
208
+ remaining: 0,
209
+ requestId: { context: 'push', requestId },
210
+ })
211
+ const pullResEnc = encodeOutgoingMessage(pullRes)
145
212
 
213
+ // Only calling once for now.
214
+ if (options?.onPullRes) {
215
+ yield* Effect.tryAll(() => options.onPullRes!(pullRes))
216
+ }
217
+
218
+ // NOTE we're also sending the pullRes to the pushing ws client as a confirmation
146
219
  for (const conn of connectedClients) {
147
- console.log('Broadcasting to client', conn === ws ? 'self' : 'other')
148
- // if (conn !== ws) {
149
- conn.send(broadcastMessage)
150
- // }
220
+ conn.send(pullResEnc)
151
221
  }
152
222
  }
153
223
 
154
- yield* Effect.promise(() => storePromise)
224
+ // Wait for the storage write to complete before finishing this request
225
+ yield* storeFiber
155
226
 
156
- i++
157
- }
158
-
159
- break
160
- }
161
- case 'WSMessage.AdminResetRoomReq': {
162
- if (decodedMessage.adminSecret !== this.env.ADMIN_SECRET) {
163
- ws.send(encodeOutgoingMessage(WSMessage.Error.make({ message: 'Invalid admin secret', requestId })))
164
- return
227
+ break
165
228
  }
229
+ case 'WSMessage.AdminResetRoomReq': {
230
+ if (decodedMessage.adminSecret !== this.env.ADMIN_SECRET) {
231
+ ws.send(encodeOutgoingMessage(WSMessage.Error.make({ message: 'Invalid admin secret', requestId })))
232
+ return
233
+ }
166
234
 
167
- yield* Effect.promise(() => this.storage.resetRoom())
168
- ws.send(encodeOutgoingMessage(WSMessage.AdminResetRoomRes.make({ requestId })))
235
+ yield* storage.resetStore
236
+ ws.send(encodeOutgoingMessage(WSMessage.AdminResetRoomRes.make({ requestId })))
169
237
 
170
- break
171
- }
172
- case 'WSMessage.AdminInfoReq': {
173
- if (decodedMessage.adminSecret !== this.env.ADMIN_SECRET) {
174
- ws.send(encodeOutgoingMessage(WSMessage.Error.make({ message: 'Invalid admin secret', requestId })))
175
- return
238
+ break
176
239
  }
240
+ case 'WSMessage.AdminInfoReq': {
241
+ if (decodedMessage.adminSecret !== this.env.ADMIN_SECRET) {
242
+ ws.send(encodeOutgoingMessage(WSMessage.Error.make({ message: 'Invalid admin secret', requestId })))
243
+ return
244
+ }
177
245
 
178
- ws.send(
179
- encodeOutgoingMessage(
180
- WSMessage.AdminInfoRes.make({ requestId, info: { durableObjectId: this.ctx.id.toString() } }),
181
- ),
182
- )
246
+ ws.send(
247
+ encodeOutgoingMessage(
248
+ WSMessage.AdminInfoRes.make({ requestId, info: { durableObjectId: this.ctx.id.toString() } }),
249
+ ),
250
+ )
183
251
 
184
- break
185
- }
186
- default: {
187
- console.error('unsupported message', decodedMessage)
188
- return shouldNeverHappen()
252
+ break
253
+ }
254
+ default: {
255
+ console.error('unsupported message', decodedMessage)
256
+ return shouldNeverHappen()
257
+ }
189
258
  }
259
+ } catch (error: any) {
260
+ ws.send(encodeOutgoingMessage(WSMessage.Error.make({ message: error.message, requestId })))
190
261
  }
191
- } catch (error: any) {
192
- ws.send(encodeOutgoingMessage(WSMessage.Error.make({ message: error.message, requestId })))
193
- }
194
- }).pipe(
195
- Effect.withSpan('@livestore/sync-cf:durable-object:webSocketMessage'),
196
- Effect.tapCauseLogPretty,
197
- Logger.withMinimumLogLevel(LogLevel.Debug),
198
- Effect.provide(Logger.pretty),
199
- Effect.runPromise,
200
- )
201
-
202
- webSocketClose = async (ws: WebSocketClient, code: number, _reason: string, _wasClean: boolean) => {
203
- // If the client closes the connection, the runtime will invoke the webSocketClose() handler.
204
- ws.close(code, 'Durable Object is closing WebSocket')
205
- }
206
- }
262
+ }).pipe(
263
+ Effect.withSpan(`@livestore/sync-cf:durable-object:webSocketMessage:${decodedMessage._tag}`, {
264
+ attributes: { requestId },
265
+ }),
266
+ Effect.tapCauseLogPretty,
267
+ Logger.withMinimumLogLevel(LogLevel.Debug),
268
+ Effect.provide(Logger.prettyWithThread('durable-object')),
269
+ Effect.runPromise,
270
+ )
271
+ }
207
272
 
208
- const makeStorage = (ctx: DurableObjectState, env: Env, dbName: string) => {
209
- const getLatestEvent = async (): Promise<MutationEvent.Any | undefined> => {
210
- const rawEvents = await env.DB.prepare(`SELECT * FROM ${dbName} ORDER BY idGlobal DESC LIMIT 1`).all()
211
- if (rawEvents.error) {
212
- throw new Error(rawEvents.error)
273
+ webSocketClose = async (ws: WebSocketClient, code: number, _reason: string, _wasClean: boolean) => {
274
+ // If the client closes the connection, the runtime will invoke the webSocketClose() handler.
275
+ ws.close(code, 'Durable Object is closing WebSocket')
213
276
  }
214
- const events = Schema.decodeUnknownSync(Schema.Array(mutationLogTable.schema))(rawEvents.results).map((e) => ({
215
- ...e,
216
- // TODO remove local ids
217
- id: { global: e.idGlobal, local: 0 },
218
- parentId: { global: e.parentIdGlobal, local: 0 },
219
- }))
220
- return events[0]
221
277
  }
278
+ }
222
279
 
223
- const getEvents = async (
280
+ type SyncStorage = {
281
+ dbName: string
282
+ getHead: Effect.Effect<EventId.GlobalEventId, UnexpectedError>
283
+ getEvents: (
224
284
  cursor: number | undefined,
225
- ): Promise<
226
- ReadonlyArray<{ mutationEventEncoded: MutationEvent.AnyEncoded; metadata: Option.Option<SyncMetadata> }>
227
- > => {
228
- const whereClause = cursor === undefined ? '' : `WHERE idGlobal > ${cursor}`
229
- const sql = `SELECT * FROM ${dbName} ${whereClause} ORDER BY idGlobal ASC`
230
- // TODO handle case where `cursor` was not found
231
- const rawEvents = await env.DB.prepare(sql).all()
232
- if (rawEvents.error) {
233
- throw new Error(rawEvents.error)
234
- }
235
- const events = Schema.decodeUnknownSync(Schema.Array(mutationLogTable.schema))(rawEvents.results).map(
236
- ({ createdAt, ...e }) => ({
237
- mutationEventEncoded: {
238
- ...e,
239
- // TODO remove local ids
240
- id: { global: e.idGlobal, local: 0 },
241
- parentId: { global: e.parentIdGlobal, local: 0 },
242
- },
243
- metadata: Option.some({ createdAt }),
244
- }),
285
+ ) => Effect.Effect<
286
+ ReadonlyArray<{ mutationEventEncoded: MutationEvent.AnyEncodedGlobal; metadata: Option.Option<SyncMetadata> }>,
287
+ UnexpectedError
288
+ >
289
+ appendEvents: (
290
+ batch: ReadonlyArray<MutationEvent.AnyEncodedGlobal>,
291
+ createdAt: string,
292
+ ) => Effect.Effect<void, UnexpectedError>
293
+ resetStore: Effect.Effect<void, UnexpectedError>
294
+ }
295
+
296
+ const makeStorage = (ctx: DurableObjectState, env: Env, storeId: string): SyncStorage => {
297
+ const dbName = `mutation_log_${PERSISTENCE_FORMAT_VERSION}_${toValidTableName(storeId)}`
298
+
299
+ const execDb = <T>(cb: (db: D1Database) => Promise<D1Result<T>>) =>
300
+ Effect.tryPromise({
301
+ try: () => cb(env.DB),
302
+ catch: (error) => new UnexpectedError({ cause: error, payload: { dbName } }),
303
+ }).pipe(
304
+ Effect.map((_) => _.results),
305
+ Effect.withSpan('@livestore/sync-cf:durable-object:execDb'),
245
306
  )
246
- return events
247
- }
248
307
 
249
- const appendEvent = async (event: MutationEvent.Any, createdAt: string) => {
250
- const sql = `INSERT INTO ${dbName} (idGlobal, parentIdGlobal, args, mutation, createdAt) VALUES (?, ?, ?, ?, ?)`
251
- await env.DB.prepare(sql)
252
- .bind(event.id.global, event.parentId.global, JSON.stringify(event.args), event.mutation, createdAt)
253
- .run()
254
- }
308
+ const getHead: Effect.Effect<EventId.GlobalEventId, UnexpectedError> = Effect.gen(function* () {
309
+ const result = yield* execDb<{ id: EventId.GlobalEventId }>((db) =>
310
+ db.prepare(`SELECT id FROM ${dbName} ORDER BY id DESC LIMIT 1`).all(),
311
+ )
255
312
 
256
- const resetRoom = async () => {
257
- await ctx.storage.deleteAll()
258
- }
313
+ return result[0]?.id ?? EventId.ROOT.global
314
+ }).pipe(UnexpectedError.mapToUnexpectedError)
259
315
 
260
- return { getLatestEvent, getEvents, appendEvent, resetRoom }
316
+ const getEvents = (
317
+ cursor: number | undefined,
318
+ ): Effect.Effect<
319
+ ReadonlyArray<{ mutationEventEncoded: MutationEvent.AnyEncodedGlobal; metadata: Option.Option<SyncMetadata> }>,
320
+ UnexpectedError
321
+ > =>
322
+ Effect.gen(function* () {
323
+ const whereClause = cursor === undefined ? '' : `WHERE id > ${cursor}`
324
+ const sql = `SELECT * FROM ${dbName} ${whereClause} ORDER BY id ASC`
325
+ // TODO handle case where `cursor` was not found
326
+ const rawEvents = yield* execDb((db) => db.prepare(sql).all())
327
+ const events = Schema.decodeUnknownSync(Schema.Array(mutationLogTable.schema))(rawEvents).map(
328
+ ({ createdAt, ...mutationEventEncoded }) => ({
329
+ mutationEventEncoded,
330
+ metadata: Option.some({ createdAt }),
331
+ }),
332
+ )
333
+ return events
334
+ }).pipe(UnexpectedError.mapToUnexpectedError)
335
+
336
+ const appendEvents: SyncStorage['appendEvents'] = (batch, createdAt) =>
337
+ Effect.gen(function* () {
338
+ // If there are no events, do nothing.
339
+ if (batch.length === 0) return
340
+
341
+ // CF D1 limits:
342
+ // Maximum bound parameters per query 100, Maximum arguments per SQL function 32
343
+ // Thus we need to split the batch into chunks of max (100/6=)16 events each.
344
+ const CHUNK_SIZE = 16
345
+
346
+ for (let i = 0; i < batch.length; i += CHUNK_SIZE) {
347
+ const chunk = batch.slice(i, i + CHUNK_SIZE)
348
+
349
+ // Create a list of placeholders ("(?, ?, ?, ?, ?), …") corresponding to each event.
350
+ const valuesPlaceholders = chunk.map(() => '(?, ?, ?, ?, ?, ?)').join(', ')
351
+ const sql = `INSERT INTO ${dbName} (id, parentId, args, mutation, createdAt, clientId) VALUES ${valuesPlaceholders}`
352
+ // Flatten the event properties into a parameters array.
353
+ const params = chunk.flatMap((event) => [
354
+ event.id,
355
+ event.parentId,
356
+ JSON.stringify(event.args),
357
+ event.mutation,
358
+ createdAt,
359
+ event.clientId,
360
+ ])
361
+
362
+ yield* execDb((db) =>
363
+ db
364
+ .prepare(sql)
365
+ .bind(...params)
366
+ .run(),
367
+ )
368
+ }
369
+ }).pipe(UnexpectedError.mapToUnexpectedError)
370
+
371
+ const resetStore = Effect.gen(function* () {
372
+ yield* Effect.promise(() => ctx.storage.deleteAll())
373
+ }).pipe(UnexpectedError.mapToUnexpectedError)
374
+
375
+ return { dbName, getHead, getEvents, appendEvents, resetStore }
261
376
  }
377
+
378
+ const getStoreId = (request: Request) => {
379
+ const url = new URL(request.url)
380
+ const searchParams = url.searchParams
381
+ const storeId = searchParams.get('storeId')
382
+ if (storeId === null) {
383
+ throw new Error('storeId search param is required')
384
+ }
385
+ return storeId
386
+ }
387
+
388
+ const toValidTableName = (str: string) => str.replaceAll(/[^a-zA-Z0-9]/g, '_')
@@ -0,0 +1,2 @@
1
+ export * from './durable-object.js'
2
+ export * from './worker.js'
@@ -0,0 +1,39 @@
1
+ import type { Env } from './durable-object.js'
2
+
3
+ export type CFWorker = {
4
+ fetch: (request: Request, env: Env, ctx: ExecutionContext) => Promise<Response>
5
+ }
6
+
7
+ export const makeWorker = (): CFWorker => {
8
+ return {
9
+ fetch: async (request, env, _ctx) => {
10
+ const url = new URL(request.url)
11
+ const searchParams = url.searchParams
12
+ const storeId = searchParams.get('storeId')
13
+
14
+ if (storeId === null) {
15
+ return new Response('storeId search param is required', { status: 400 })
16
+ }
17
+
18
+ const id = env.WEBSOCKET_SERVER.idFromName(storeId)
19
+ const durableObject = env.WEBSOCKET_SERVER.get(id)
20
+
21
+ if (url.pathname.endsWith('/websocket')) {
22
+ const upgradeHeader = request.headers.get('Upgrade')
23
+ if (!upgradeHeader || upgradeHeader !== 'websocket') {
24
+ return new Response('Durable Object expected Upgrade: websocket', { status: 426 })
25
+ }
26
+
27
+ return durableObject.fetch(request)
28
+ }
29
+
30
+ return new Response(null, {
31
+ status: 400,
32
+ statusText: 'Bad Request',
33
+ headers: {
34
+ 'Content-Type': 'text/plain',
35
+ },
36
+ })
37
+ },
38
+ }
39
+ }