@livestore/sync-cf 0.0.58-dev.6 → 0.0.58-dev.7

package/src/cf-worker/durable-object.ts

@@ -1,5 +1,5 @@
 import { makeColumnSpec } from '@livestore/common'
-import { DbSchema, type MutationEvent } from '@livestore/common/schema'
+import { DbSchema, type MutationEvent, mutationEventSchemaAny } from '@livestore/common/schema'
 import { shouldNeverHappen } from '@livestore/utils'
 import { Effect, Schema } from '@livestore/utils/effect'
 import { DurableObject } from 'cloudflare:workers'
@@ -14,14 +14,15 @@ export interface Env {
 
 type WebSocketClient = WebSocket
 
-const encodeMessage = Schema.encodeSync(Schema.parseJson(WSMessage.Message))
-const decodeMessage = Schema.decodeUnknownEither(Schema.parseJson(WSMessage.Message))
+const encodeOutgoingMessage = Schema.encodeSync(Schema.parseJson(WSMessage.BackendToClientMessage))
+const encodeIncomingMessage = Schema.encodeSync(Schema.parseJson(WSMessage.ClientToBackendMessage))
+const decodeIncomingMessage = Schema.decodeUnknownEither(Schema.parseJson(WSMessage.ClientToBackendMessage))
 
 export const mutationLogTable = DbSchema.table('__unused', {
-  // TODO add parent ids (see https://vlcn.io/blog/crdt-substrate)
-  id: DbSchema.text({ primaryKey: true }),
-  mutation: DbSchema.text({ nullable: false }),
-  args: DbSchema.text({ nullable: false, schema: Schema.parseJson(Schema.Any) }),
+  id: DbSchema.integer({ primaryKey: true }),
+  parentId: DbSchema.integer({}),
+  mutation: DbSchema.text({}),
+  args: DbSchema.text({ schema: Schema.parseJson(Schema.Any) }),
 })
 
 // Durable Object
@@ -43,8 +44,8 @@ export class WebSocketServer extends DurableObject<Env> {
 
       this.ctx.setWebSocketAutoResponse(
         new WebSocketRequestResponsePair(
-          encodeMessage(WSMessage.Ping.make({ requestId: 'ping' })),
-          encodeMessage(WSMessage.Pong.make({ requestId: 'ping' })),
+          encodeIncomingMessage(WSMessage.Ping.make({ requestId: 'ping' })),
+          encodeOutgoingMessage(WSMessage.Pong.make({ requestId: 'ping' })),
         ),
       )
 
@@ -58,7 +59,7 @@ export class WebSocketServer extends DurableObject<Env> {
     }).pipe(Effect.tapCauseLogPretty, Effect.runPromise)
 
   webSocketMessage = async (ws: WebSocketClient, message: ArrayBuffer | string) => {
-    const decodedMessageRes = decodeMessage(message)
+    const decodedMessageRes = decodeIncomingMessage(message)
 
     if (decodedMessageRes._tag === 'Left') {
       console.error('Invalid message received', decodedMessageRes.left)
@@ -68,91 +69,116 @@ export class WebSocketServer extends DurableObject<Env> {
     const decodedMessage = decodedMessageRes.right
     const requestId = decodedMessage.requestId
 
-    switch (decodedMessage._tag) {
-      case 'WSMessage.PullReq': {
-        const cursor = decodedMessage.cursor
-        const CHUNK_SIZE = 100
+    try {
+      switch (decodedMessage._tag) {
+        case 'WSMessage.PullReq': {
+          const cursor = decodedMessage.cursor
+          const CHUNK_SIZE = 100
 
-        // TODO use streaming
-        const remainingEvents = [...(await this.storage.getEvents(cursor))]
+          // TODO use streaming
+          const remainingEvents = [...(await this.storage.getEvents(cursor))]
 
-        // NOTE we want to make sure the WS server responds at least once with `InitRes` even if `events` is empty
-        while (true) {
-          const events = remainingEvents.splice(0, CHUNK_SIZE)
-          const hasMore = remainingEvents.length > 0
+          // NOTE we want to make sure the WS server responds at least once with `InitRes` even if `events` is empty
+          while (true) {
+            const events = remainingEvents.splice(0, CHUNK_SIZE)
+            const encodedEvents = Schema.encodeSync(Schema.Array(mutationEventSchemaAny))(events)
+            const hasMore = remainingEvents.length > 0
 
-          ws.send(encodeMessage(WSMessage.PullRes.make({ events, hasMore, requestId })))
+            ws.send(encodeOutgoingMessage(WSMessage.PullRes.make({ events: encodedEvents, hasMore, requestId })))
 
-          if (hasMore === false) {
-            break
+            if (hasMore === false) {
+              break
+            }
           }
+
+          break
         }
+        case 'WSMessage.PushReq': {
+          // TODO check whether we could use the Durable Object storage for this to speed up the lookup
+          const latestEvent = await this.storage.getLatestEvent()
+          const expectedParentId = latestEvent?.id ?? { global: 0, local: 0 }
+
+          if (decodedMessage.mutationEventEncoded.parentId !== expectedParentId) {
+            ws.send(
+              encodeOutgoingMessage(
+                WSMessage.Error.make({
+                  message: `Invalid parent id. Received ${decodedMessage.mutationEventEncoded.parentId} but expected ${expectedParentId}`,
+                  requestId,
+                }),
+              ),
+            )
+            return
+          }
 
-        break
-      }
-      case 'WSMessage.PushReq': {
-        // NOTE we're currently not blocking on this to allow broadcasting right away
-        // however we should do some mutation validation first (e.g. checking parent event id)
-        const storePromise = decodedMessage.persisted
-          ? this.storage.appendEvent(decodedMessage.mutationEventEncoded)
-          : Promise.resolve()
-
-        ws.send(
-          encodeMessage(WSMessage.PushAck.make({ mutationId: decodedMessage.mutationEventEncoded.id, requestId })),
-        )
-
-        // console.debug(`Broadcasting mutation event to ${this.subscribedWebSockets.size} clients`)
-
-        const connectedClients = this.ctx.getWebSockets()
-
-        if (connectedClients.length > 0) {
-          const broadcastMessage = encodeMessage(
-            WSMessage.PushBroadcast.make({
-              mutationEventEncoded: decodedMessage.mutationEventEncoded,
-              requestId,
-              persisted: decodedMessage.persisted,
-            }),
+          // TODO handle clientId unique conflict
+
+          // NOTE we're currently not blocking on this to allow broadcasting right away
+          const storePromise = decodedMessage.persisted
+            ? this.storage.appendEvent(decodedMessage.mutationEventEncoded)
+            : Promise.resolve()
+
+          ws.send(
+            encodeOutgoingMessage(
+              WSMessage.PushAck.make({ mutationId: decodedMessage.mutationEventEncoded.id.global, requestId }),
+            ),
           )
 
-          for (const conn of connectedClients) {
-            console.log('Broadcasting to client', conn === ws ? 'self' : 'other')
-            if (conn !== ws) {
+          // console.debug(`Broadcasting mutation event to ${this.subscribedWebSockets.size} clients`)
+
+          const connectedClients = this.ctx.getWebSockets()
+
+          if (connectedClients.length > 0) {
+            const broadcastMessage = encodeOutgoingMessage(
+              WSMessage.PushBroadcast.make({
+                mutationEventEncoded: decodedMessage.mutationEventEncoded,
+                persisted: decodedMessage.persisted,
+              }),
+            )
+
+            for (const conn of connectedClients) {
+              console.log('Broadcasting to client', conn === ws ? 'self' : 'other')
+              // if (conn !== ws) {
               conn.send(broadcastMessage)
+              // }
             }
           }
-        }
 
-        await storePromise
+          await storePromise
 
-        break
-      }
-      case 'WSMessage.AdminResetRoomReq': {
-        if (decodedMessage.adminSecret !== this.env.ADMIN_SECRET) {
-          ws.send(encodeMessage(WSMessage.Error.make({ message: 'Invalid admin secret', requestId })))
-          return
+          break
         }
+        case 'WSMessage.AdminResetRoomReq': {
+          if (decodedMessage.adminSecret !== this.env.ADMIN_SECRET) {
+            ws.send(encodeOutgoingMessage(WSMessage.Error.make({ message: 'Invalid admin secret', requestId })))
+            return
+          }
 
-        await this.storage.resetRoom()
-        ws.send(encodeMessage(WSMessage.AdminResetRoomRes.make({ requestId })))
+          await this.storage.resetRoom()
+          ws.send(encodeOutgoingMessage(WSMessage.AdminResetRoomRes.make({ requestId })))
 
-        break
-      }
-      case 'WSMessage.AdminInfoReq': {
-        if (decodedMessage.adminSecret !== this.env.ADMIN_SECRET) {
-          ws.send(encodeMessage(WSMessage.Error.make({ message: 'Invalid admin secret', requestId })))
-          return
+          break
         }
+        case 'WSMessage.AdminInfoReq': {
+          if (decodedMessage.adminSecret !== this.env.ADMIN_SECRET) {
+            ws.send(encodeOutgoingMessage(WSMessage.Error.make({ message: 'Invalid admin secret', requestId })))
+            return
+          }
 
-        ws.send(
-          encodeMessage(WSMessage.AdminInfoRes.make({ requestId, info: { durableObjectId: this.ctx.id.toString() } })),
-        )
+          ws.send(
+            encodeOutgoingMessage(
+              WSMessage.AdminInfoRes.make({ requestId, info: { durableObjectId: this.ctx.id.toString() } }),
+            ),
+          )
 
-        break
-      }
-      default: {
-        console.error('unsupported message', decodedMessage)
-        return shouldNeverHappen()
+          break
+        }
+        default: {
+          console.error('unsupported message', decodedMessage)
+          return shouldNeverHappen()
+        }
       }
+    } catch (error: any) {
+      ws.send(encodeOutgoingMessage(WSMessage.Error.make({ message: error.message, requestId })))
     }
   }
 
@@ -163,25 +189,42 @@ export class WebSocketServer extends DurableObject<Env> {
 }
 
 const makeStorage = (ctx: DurableObjectState, env: Env, dbName: string) => {
-  const getEvents = async (cursor: string | undefined): Promise<ReadonlyArray<MutationEvent.Any>> => {
-    const whereClause = cursor ? `WHERE id > '${cursor}'` : ''
+  const getLatestEvent = async (): Promise<MutationEvent.Any | undefined> => {
+    const rawEvents = await env.DB.prepare(`SELECT * FROM ${dbName} ORDER BY id DESC LIMIT 1`).all()
+    if (rawEvents.error) {
+      throw new Error(rawEvents.error)
+    }
+    const events = Schema.decodeUnknownSync(Schema.Array(mutationLogTable.schema))(rawEvents.results).map((e) => ({
+      ...e,
+      id: { global: e.id, local: 0 },
+      parentId: { global: e.parentId, local: 0 },
+    }))
+    return events[0]
+  }
+
+  const getEvents = async (cursor: number | undefined): Promise<ReadonlyArray<MutationEvent.Any>> => {
+    const whereClause = cursor ? `WHERE id > ${cursor}` : ''
     // TODO handle case where `cursor` was not found
     const rawEvents = await env.DB.prepare(`SELECT * FROM ${dbName} ${whereClause} ORDER BY id ASC`).all()
     if (rawEvents.error) {
       throw new Error(rawEvents.error)
     }
-    const events = Schema.decodeUnknownSync(Schema.Array(mutationLogTable.schema))(rawEvents.results)
+    const events = Schema.decodeUnknownSync(Schema.Array(mutationLogTable.schema))(rawEvents.results).map((e) => ({
+      ...e,
+      id: { global: e.id, local: 0 },
+      parentId: { global: e.parentId, local: 0 },
+    }))
     return events
   }
 
   const appendEvent = async (event: MutationEvent.Any) => {
-    const sql = `INSERT INTO ${dbName} (id, args, mutation) VALUES (?, ?, ?)`
-    await env.DB.prepare(sql).bind(event.id, JSON.stringify(event.args), event.mutation).run()
+    const sql = `INSERT INTO ${dbName} (id, parentId, args, mutation) VALUES (?, ?, ?, ?)`
+    await env.DB.prepare(sql).bind(event.id, event.parentId, JSON.stringify(event.args), event.mutation).run()
   }
 
   const resetRoom = async () => {
     await ctx.storage.deleteAll()
   }
 
-  return { getEvents, appendEvent, resetRoom }
+  return { getLatestEvent, getEvents, appendEvent, resetRoom }
 }

package/src/common/ws-message-types.ts

@@ -4,7 +4,7 @@ import { Schema } from '@livestore/utils/effect'
 export const PullReq = Schema.TaggedStruct('WSMessage.PullReq', {
   requestId: Schema.String,
   /** Omitting the cursor will start from the beginning */
-  cursor: Schema.optional(Schema.String),
+  cursor: Schema.optional(Schema.Number),
 })
 
 export type PullReq = typeof PullReq.Type
@@ -20,7 +20,6 @@ export const PullRes = Schema.TaggedStruct('WSMessage.PullRes', {
 export type PullRes = typeof PullRes.Type
 
 export const PushBroadcast = Schema.TaggedStruct('WSMessage.PushBroadcast', {
-  requestId: Schema.String,
   mutationEventEncoded: mutationEventSchemaEncodedAny,
   persisted: Schema.Boolean,
 })
@@ -37,7 +36,7 @@ export type PushReq = typeof PushReq.Type
 
 export const PushAck = Schema.TaggedStruct('WSMessage.PushAck', {
   requestId: Schema.String,
-  mutationId: Schema.String,
+  mutationId: Schema.Number,
 })
 
 export type PushAck = typeof PushAck.Type
@@ -105,5 +104,16 @@ export const Message = Schema.Union(
 export type Message = typeof Message.Type
 export type MessageEncoded = typeof Message.Encoded
 
-export const IncomingMessage = Schema.Union(PullRes, PushBroadcast, PushAck, Error, Pong)
-export type IncomingMessage = typeof IncomingMessage.Type
+export const BackendToClientMessage = Schema.Union(
+  PullRes,
+  PushBroadcast,
+  PushAck,
+  AdminResetRoomRes,
+  AdminInfoRes,
+  Error,
+  Pong,
+)
+export type BackendToClientMessage = typeof BackendToClientMessage.Type
+
+export const ClientToBackendMessage = Schema.Union(PullReq, PushReq, AdminResetRoomReq, AdminInfoReq, Ping)
+export type ClientToBackendMessage = typeof ClientToBackendMessage.Type

package/src/sync-impl/ws-impl.ts

@@ -2,9 +2,9 @@
 
 import type { SyncBackend, SyncBackendOptionsBase } from '@livestore/common'
 import { InvalidPullError, InvalidPushError } from '@livestore/common'
-import { cuid } from '@livestore/utils/cuid'
 import type { Scope } from '@livestore/utils/effect'
 import { Deferred, Effect, Option, PubSub, Queue, Schema, Stream, SubscriptionRef } from '@livestore/utils/effect'
+import { nanoid } from '@livestore/utils/nanoid'
 
 import { WSMessage } from '../common/index.js'
 
@@ -34,25 +34,34 @@ export const makeWsSync = (options: WsSyncOptions): Effect.Effect<SyncBackend<nu
       isConnected,
       pull: (args, { listenForNew }) =>
         listenForNew
-          ? Stream.fromPubSub(incomingMessages).pipe(
-              Stream.tap((_) =>
-                _._tag === 'WSMessage.Error' ? new InvalidPullError({ message: _.message }) : Effect.void,
-              ),
-              Stream.filter(Schema.is(WSMessage.PushBroadcast)),
-              Stream.map((_) => ({
-                mutationEventEncoded: _.mutationEventEncoded,
-                persisted: _.persisted,
-                metadata,
-              })),
-            )
+          ? Effect.gen(function* () {
+              const requestId = nanoid()
+              const cursor = Option.getOrUndefined(args)?.cursor.global
+
+              yield* send(WSMessage.PullReq.make({ cursor, requestId }))
+
+              return Stream.fromPubSub(incomingMessages).pipe(
+                Stream.filter((_) => (_._tag === 'WSMessage.PullRes' ? _.requestId === requestId : true)),
+                Stream.tap((_) =>
+                  _._tag === 'WSMessage.Error' ? new InvalidPullError({ message: _.message }) : Effect.void,
+                ),
+                Stream.filter(Schema.is(Schema.Union(WSMessage.PushBroadcast, WSMessage.PullRes))),
+                Stream.map((msg) =>
+                  msg._tag === 'WSMessage.PushBroadcast'
+                    ? [{ mutationEventEncoded: msg.mutationEventEncoded, persisted: msg.persisted, metadata }]
+                    : msg.events.map((_) => ({ mutationEventEncoded: _, metadata, persisted: true })),
+                ),
+                Stream.flattenIterables,
+              )
+            }).pipe(Stream.unwrap)
           : Effect.gen(function* () {
-              const requestId = cuid()
-              const cursor = Option.getOrUndefined(args)?.cursor
+              const requestId = nanoid()
+              const cursor = Option.getOrUndefined(args)?.cursor.global
 
               yield* send(WSMessage.PullReq.make({ cursor, requestId }))
 
               return Stream.fromPubSub(incomingMessages).pipe(
-                Stream.filter((_) => _.requestId === requestId),
+                Stream.filter((_) => _._tag !== 'WSMessage.PushBroadcast' && _.requestId === requestId),
                 Stream.tap((_) =>
                   _._tag === 'WSMessage.Error' ? new InvalidPullError({ message: _.message }) : Effect.void,
                 ),
@@ -70,17 +79,17 @@ export const makeWsSync = (options: WsSyncOptions): Effect.Effect<SyncBackend<nu
       push: (mutationEventEncoded, persisted) =>
         Effect.gen(function* () {
           const ready = yield* Deferred.make<void, InvalidPushError>()
-          const requestId = cuid()
+          const requestId = nanoid()
 
           yield* Stream.fromPubSub(incomingMessages).pipe(
-            Stream.filter((_) => _.requestId === requestId),
+            Stream.filter((_) => _._tag !== 'WSMessage.PushBroadcast' && _.requestId === requestId),
             Stream.tap((_) =>
               _._tag === 'WSMessage.Error'
                 ? Deferred.fail(ready, new InvalidPushError({ message: _.message }))
                 : Effect.void,
             ),
             Stream.filter(Schema.is(WSMessage.PushAck)),
-            Stream.filter((_) => _.mutationId === mutationEventEncoded.id),
+            Stream.filter((_) => _.mutationId === mutationEventEncoded.id.global),
             Stream.take(1),
             Stream.tap(() => Deferred.succeed(ready, void 0)),
             Stream.runDrain,
@@ -104,7 +113,7 @@ const connect = (wsUrl: string) =>
     const isConnected = yield* SubscriptionRef.make(false)
     const wsRef: { current: WebSocket | undefined } = { current: undefined }
 
-    const incomingMessages = yield* PubSub.unbounded<Exclude<WSMessage.IncomingMessage, WSMessage.Pong>>()
+    const incomingMessages = yield* PubSub.unbounded<Exclude<WSMessage.BackendToClientMessage, WSMessage.Pong>>()
 
     const waitUntilOnline = isConnected.changes.pipe(Stream.filter(Boolean), Stream.take(1), Stream.runDrain)
 
@@ -132,7 +141,9 @@ const connect = (wsUrl: string) =>
       const pongMessages = yield* Queue.unbounded<WSMessage.Pong>()
 
       const messageHandler = (event: MessageEvent<any>): void => {
-        const decodedEventRes = Schema.decodeUnknownEither(Schema.parseJson(WSMessage.IncomingMessage))(event.data)
+        const decodedEventRes = Schema.decodeUnknownEither(Schema.parseJson(WSMessage.BackendToClientMessage))(
+          event.data,
+        )
 
         if (decodedEventRes._tag === 'Left') {
           console.error('Sync: Invalid message received', decodedEventRes.left)

package/tsconfig.json

@@ -3,6 +3,7 @@
   "compilerOptions": {
     "outDir": "./dist",
     "rootDir": "./src",
+    "target": "es2022",
     "tsBuildInfoFile": "./dist/.tsbuildinfo",
     "types": ["@cloudflare/workers-types"]
   },