@livestore/sync-cf 0.2.0 → 0.3.0-dev.1

package/src/cf-worker/durable-object.ts

@@ -1,10 +1,11 @@
-import { makeColumnSpec, ROOT_ID } from '@livestore/common'
-import { DbSchema, type MutationEvent, mutationEventSchemaAny } from '@livestore/common/schema'
+import { makeColumnSpec } from '@livestore/common'
+import { DbSchema, EventId, type MutationEvent } from '@livestore/common/schema'
 import { shouldNeverHappen } from '@livestore/utils'
-import { Effect, Schema } from '@livestore/utils/effect'
+import { Effect, Logger, LogLevel, Option, Schema } from '@livestore/utils/effect'
 import { DurableObject } from 'cloudflare:workers'
 
 import { WSMessage } from '../common/index.js'
+import type { SyncMetadata } from '../common/ws-message-types.js'
 
 export interface Env {
   WEBSOCKET_SERVER: DurableObjectNamespace<WebSocketServer>
@@ -18,11 +19,14 @@ const encodeOutgoingMessage = Schema.encodeSync(Schema.parseJson(WSMessage.Backe
 const encodeIncomingMessage = Schema.encodeSync(Schema.parseJson(WSMessage.ClientToBackendMessage))
 const decodeIncomingMessage = Schema.decodeUnknownEither(Schema.parseJson(WSMessage.ClientToBackendMessage))
 
+// NOTE actual table name is determined at runtime by `WebSocketServer.dbName`
 export const mutationLogTable = DbSchema.table('__unused', {
   idGlobal: DbSchema.integer({ primaryKey: true }),
   parentIdGlobal: DbSchema.integer({}),
   mutation: DbSchema.text({}),
   args: DbSchema.text({ schema: Schema.parseJson(Schema.Any) }),
+  /** ISO date format */
+  createdAt: DbSchema.text({}),
 })
 
 // Durable Object
@@ -58,129 +62,142 @@ export class WebSocketServer extends DurableObject<Env> {
       })
     }).pipe(Effect.tapCauseLogPretty, Effect.runPromise)
 
-  webSocketMessage = async (ws: WebSocketClient, message: ArrayBuffer | string) => {
-    const decodedMessageRes = decodeIncomingMessage(message)
+  webSocketMessage = (ws: WebSocketClient, message: ArrayBuffer | string) =>
+    Effect.gen(this, function* () {
+      const decodedMessageRes = decodeIncomingMessage(message)
 
-    if (decodedMessageRes._tag === 'Left') {
-      console.error('Invalid message received', decodedMessageRes.left)
-      return
-    }
+      if (decodedMessageRes._tag === 'Left') {
+        console.error('Invalid message received', decodedMessageRes.left)
+        return
+      }
 
-    const decodedMessage = decodedMessageRes.right
-    const requestId = decodedMessage.requestId
+      const decodedMessage = decodedMessageRes.right
+      const requestId = decodedMessage.requestId
 
-    try {
-      switch (decodedMessage._tag) {
-        case 'WSMessage.PullReq': {
-          const cursor = decodedMessage.cursor
-          const CHUNK_SIZE = 100
+      try {
+        switch (decodedMessage._tag) {
+          case 'WSMessage.PullReq': {
+            const cursor = decodedMessage.cursor
+            const CHUNK_SIZE = 100
 
-          // TODO use streaming
-          const remainingEvents = [...(await this.storage.getEvents(cursor))]
+            // TODO use streaming
+            const remainingEvents = [...(yield* Effect.promise(() => this.storage.getEvents(cursor)))]
 
-          // NOTE we want to make sure the WS server responds at least once with `InitRes` even if `events` is empty
-          while (true) {
-            const events = remainingEvents.splice(0, CHUNK_SIZE)
-            const encodedEvents = Schema.encodeSync(Schema.Array(mutationEventSchemaAny))(events)
-            const hasMore = remainingEvents.length > 0
+            // NOTE we want to make sure the WS server responds at least once with `InitRes` even if `events` is empty
+            while (true) {
+              const events = remainingEvents.splice(0, CHUNK_SIZE)
 
-            ws.send(encodeOutgoingMessage(WSMessage.PullRes.make({ events: encodedEvents, hasMore, requestId })))
+              ws.send(
+                encodeOutgoingMessage(WSMessage.PullRes.make({ events, remaining: remainingEvents.length, requestId })),
+              )
 
-            if (hasMore === false) {
-              break
+              if (remainingEvents.length === 0) {
+                break
+              }
             }
+
+            break
           }
+          case 'WSMessage.PushReq': {
+            // TODO check whether we could use the Durable Object storage for this to speed up the lookup
+            const latestEvent = yield* Effect.promise(() => this.storage.getLatestEvent())
+            const expectedParentId = latestEvent?.id ?? EventId.ROOT
+
+            let i = 0
+            for (const mutationEventEncoded of decodedMessage.batch) {
+              if (mutationEventEncoded.parentId.global !== expectedParentId.global + i) {
+                const err = WSMessage.Error.make({
+                  message: `Invalid parent id. Received ${mutationEventEncoded.parentId.global} but expected ${expectedParentId.global}`,
+                  requestId,
+                })
 
-          break
-        }
-        case 'WSMessage.PushReq': {
-          // TODO check whether we could use the Durable Object storage for this to speed up the lookup
-          const latestEvent = await this.storage.getLatestEvent()
-          const expectedParentId = latestEvent?.id ?? ROOT_ID
+                yield* Effect.fail(err).pipe(Effect.ignoreLogged)
 
-          if (decodedMessage.mutationEventEncoded.parentId.global !== expectedParentId.global) {
-            ws.send(
-              encodeOutgoingMessage(
-                WSMessage.Error.make({
-                  message: `Invalid parent id. Received ${decodedMessage.mutationEventEncoded.parentId.global} but expected ${expectedParentId.global}`,
-                  requestId,
-                }),
-              ),
-            )
-            return
-          }
+                ws.send(encodeOutgoingMessage(err))
+                return
+              }
 
-          // TODO handle clientId unique conflict
+              // TODO handle clientId unique conflict
 
-          // NOTE we're currently not blocking on this to allow broadcasting right away
-          const storePromise = decodedMessage.persisted
-            ? this.storage.appendEvent(decodedMessage.mutationEventEncoded)
-            : Promise.resolve()
+              const createdAt = new Date().toISOString()
 
-          ws.send(
-            encodeOutgoingMessage(
-              WSMessage.PushAck.make({ mutationId: decodedMessage.mutationEventEncoded.id.global, requestId }),
-            ),
-          )
+              // NOTE we're currently not blocking on this to allow broadcasting right away
+              const storePromise = this.storage.appendEvent(mutationEventEncoded, createdAt)
 
-          // console.debug(`Broadcasting mutation event to ${this.subscribedWebSockets.size} clients`)
+              ws.send(
+                encodeOutgoingMessage(
+                  WSMessage.PushAck.make({ mutationId: mutationEventEncoded.id.global, requestId }),
+                ),
+              )
 
-          const connectedClients = this.ctx.getWebSockets()
+              // console.debug(`Broadcasting mutation event to ${this.subscribedWebSockets.size} clients`)
 
-          if (connectedClients.length > 0) {
-            const broadcastMessage = encodeOutgoingMessage(
-              WSMessage.PushBroadcast.make({
-                mutationEventEncoded: decodedMessage.mutationEventEncoded,
-                persisted: decodedMessage.persisted,
-              }),
-            )
+              const connectedClients = this.ctx.getWebSockets()
 
-            for (const conn of connectedClients) {
-              console.log('Broadcasting to client', conn === ws ? 'self' : 'other')
-              // if (conn !== ws) {
-              conn.send(broadcastMessage)
-              // }
-            }
-          }
+              if (connectedClients.length > 0) {
+                const broadcastMessage = encodeOutgoingMessage(
+                  // TODO refactor to batch api
+                  WSMessage.PushBroadcast.make({
+                    mutationEventEncoded,
+                    metadata: Option.some({ createdAt }),
+                  }),
+                )
 
-          await storePromise
+                for (const conn of connectedClients) {
+                  console.log('Broadcasting to client', conn === ws ? 'self' : 'other')
+                  // if (conn !== ws) {
+                  conn.send(broadcastMessage)
+                  // }
+                }
+              }
 
-          break
-        }
-        case 'WSMessage.AdminResetRoomReq': {
-          if (decodedMessage.adminSecret !== this.env.ADMIN_SECRET) {
-            ws.send(encodeOutgoingMessage(WSMessage.Error.make({ message: 'Invalid admin secret', requestId })))
-            return
+              yield* Effect.promise(() => storePromise)
+
+              i++
+            }
+
+            break
           }
+          case 'WSMessage.AdminResetRoomReq': {
+            if (decodedMessage.adminSecret !== this.env.ADMIN_SECRET) {
+              ws.send(encodeOutgoingMessage(WSMessage.Error.make({ message: 'Invalid admin secret', requestId })))
+              return
+            }
 
-          await this.storage.resetRoom()
-          ws.send(encodeOutgoingMessage(WSMessage.AdminResetRoomRes.make({ requestId })))
+            yield* Effect.promise(() => this.storage.resetRoom())
+            ws.send(encodeOutgoingMessage(WSMessage.AdminResetRoomRes.make({ requestId })))
 
-          break
-        }
-        case 'WSMessage.AdminInfoReq': {
-          if (decodedMessage.adminSecret !== this.env.ADMIN_SECRET) {
-            ws.send(encodeOutgoingMessage(WSMessage.Error.make({ message: 'Invalid admin secret', requestId })))
-            return
+            break
           }
+          case 'WSMessage.AdminInfoReq': {
+            if (decodedMessage.adminSecret !== this.env.ADMIN_SECRET) {
+              ws.send(encodeOutgoingMessage(WSMessage.Error.make({ message: 'Invalid admin secret', requestId })))
+              return
+            }
 
-          ws.send(
-            encodeOutgoingMessage(
-              WSMessage.AdminInfoRes.make({ requestId, info: { durableObjectId: this.ctx.id.toString() } }),
-            ),
-          )
+            ws.send(
+              encodeOutgoingMessage(
+                WSMessage.AdminInfoRes.make({ requestId, info: { durableObjectId: this.ctx.id.toString() } }),
+              ),
+            )
 
-          break
-        }
-        default: {
-          console.error('unsupported message', decodedMessage)
-          return shouldNeverHappen()
+            break
+          }
+          default: {
+            console.error('unsupported message', decodedMessage)
+            return shouldNeverHappen()
+          }
         }
+      } catch (error: any) {
+        ws.send(encodeOutgoingMessage(WSMessage.Error.make({ message: error.message, requestId })))
       }
-    } catch (error: any) {
-      ws.send(encodeOutgoingMessage(WSMessage.Error.make({ message: error.message, requestId })))
-    }
-  }
+    }).pipe(
+      Effect.withSpan('@livestore/sync-cf:durable-object:webSocketMessage'),
+      Effect.tapCauseLogPretty,
+      Logger.withMinimumLogLevel(LogLevel.Debug),
+      Effect.provide(Logger.pretty),
+      Effect.runPromise,
+    )
 
   webSocketClose = async (ws: WebSocketClient, code: number, _reason: string, _wasClean: boolean) => {
     // If the client closes the connection, the runtime will invoke the webSocketClose() handler.
@@ -196,31 +213,43 @@ const makeStorage = (ctx: DurableObjectState, env: Env, dbName: string) => {
     }
     const events = Schema.decodeUnknownSync(Schema.Array(mutationLogTable.schema))(rawEvents.results).map((e) => ({
       ...e,
+      // TODO remove local ids
       id: { global: e.idGlobal, local: 0 },
       parentId: { global: e.parentIdGlobal, local: 0 },
     }))
     return events[0]
   }
 
-  const getEvents = async (cursor: number | undefined): Promise<ReadonlyArray<MutationEvent.Any>> => {
-    const whereClause = cursor ? `WHERE idGlobal > ${cursor}` : ''
+  const getEvents = async (
+    cursor: number | undefined,
+  ): Promise<
+    ReadonlyArray<{ mutationEventEncoded: MutationEvent.AnyEncoded; metadata: Option.Option<SyncMetadata> }>
+  > => {
+    const whereClause = cursor === undefined ? '' : `WHERE idGlobal > ${cursor}`
+    const sql = `SELECT * FROM ${dbName} ${whereClause} ORDER BY idGlobal ASC`
     // TODO handle case where `cursor` was not found
-    const rawEvents = await env.DB.prepare(`SELECT * FROM ${dbName} ${whereClause} ORDER BY idGlobal ASC`).all()
+    const rawEvents = await env.DB.prepare(sql).all()
     if (rawEvents.error) {
       throw new Error(rawEvents.error)
     }
-    const events = Schema.decodeUnknownSync(Schema.Array(mutationLogTable.schema))(rawEvents.results).map((e) => ({
-      ...e,
-      id: { global: e.idGlobal, local: 0 },
-      parentId: { global: e.parentIdGlobal, local: 0 },
-    }))
+    const events = Schema.decodeUnknownSync(Schema.Array(mutationLogTable.schema))(rawEvents.results).map(
+      ({ createdAt, ...e }) => ({
+        mutationEventEncoded: {
+          ...e,
+          // TODO remove local ids
+          id: { global: e.idGlobal, local: 0 },
+          parentId: { global: e.parentIdGlobal, local: 0 },
+        },
+        metadata: Option.some({ createdAt }),
+      }),
+    )
     return events
   }
 
-  const appendEvent = async (event: MutationEvent.Any) => {
-    const sql = `INSERT INTO ${dbName} (idGlobal, parentIdGlobal, args, mutation) VALUES (?, ?, ?, ?)`
+  const appendEvent = async (event: MutationEvent.Any, createdAt: string) => {
+    const sql = `INSERT INTO ${dbName} (idGlobal, parentIdGlobal, args, mutation, createdAt) VALUES (?, ?, ?, ?, ?)`
     await env.DB.prepare(sql)
-      .bind(event.id.global, event.parentId.global, JSON.stringify(event.args), event.mutation)
+      .bind(event.id.global, event.parentId.global, JSON.stringify(event.args), event.mutation, createdAt)
       .run()
   }
 

package/src/cf-worker/index.ts

@@ -1,7 +1,7 @@
 /// <reference no-default-lib="true"/>
 /// <reference lib="esnext" />
 
-// import { mutationEventSchemaEncodedAny } from '@livestore/common/schema'
+// import { EncodedAny } from '@livestore/common/schema'
 // import { Effect, HttpServer, Schema } from '@livestore/utils/effect'
 
 import type { Env } from './durable-object.js'

package/src/common/ws-message-types.ts

@@ -1,4 +1,4 @@
-import { mutationEventSchemaEncodedAny } from '@livestore/common/schema'
+import { MutationEvent } from '@livestore/common/schema'
 import { Schema } from '@livestore/utils/effect'
 
 export const PullReq = Schema.TaggedStruct('WSMessage.PullReq', {
@@ -9,27 +9,36 @@ export const PullReq = Schema.TaggedStruct('WSMessage.PullReq', {
 
 export type PullReq = typeof PullReq.Type
 
+export const SyncMetadata = Schema.Struct({
+  /** ISO date format */
+  createdAt: Schema.String,
+})
+
+export type SyncMetadata = typeof SyncMetadata.Type
+
 export const PullRes = Schema.TaggedStruct('WSMessage.PullRes', {
   requestId: Schema.String,
-  // /** The  */
-  // cursor: Schema.String,
-  events: Schema.Array(mutationEventSchemaEncodedAny),
-  hasMore: Schema.Boolean,
+  events: Schema.Array(
+    Schema.Struct({
+      mutationEventEncoded: MutationEvent.EncodedAny,
+      metadata: Schema.Option(SyncMetadata),
+    }),
+  ),
+  remaining: Schema.Number,
 })
 
 export type PullRes = typeof PullRes.Type
 
 export const PushBroadcast = Schema.TaggedStruct('WSMessage.PushBroadcast', {
-  mutationEventEncoded: mutationEventSchemaEncodedAny,
-  persisted: Schema.Boolean,
+  mutationEventEncoded: MutationEvent.EncodedAny,
+  metadata: Schema.Option(SyncMetadata),
 })
 
 export type PushBroadcast = typeof PushBroadcast.Type
 
 export const PushReq = Schema.TaggedStruct('WSMessage.PushReq', {
   requestId: Schema.String,
-  mutationEventEncoded: mutationEventSchemaEncodedAny,
-  persisted: Schema.Boolean,
+  batch: Schema.Array(MutationEvent.EncodedAny),
 })
 
 export type PushReq = typeof PushReq.Type