@livestore/sync-cf 0.4.0-dev.0 → 0.4.0-dev.10

package/src/cf-worker/do/push.ts

@@ -0,0 +1,205 @@
+import {
+  BackendIdMismatchError,
+  InvalidPushError,
+  ServerAheadError,
+  SyncBackend,
+  UnexpectedError,
+} from '@livestore/common'
+import { splitChunkBySize } from '@livestore/common/sync'
+import { type CfTypes, emitStreamResponse } from '@livestore/common-cf'
+import { Chunk, Effect, Option, type RpcMessage, Schema } from '@livestore/utils/effect'
+import { MAX_PUSH_EVENTS_PER_REQUEST, MAX_WS_MESSAGE_BYTES } from '../../common/constants.ts'
+import { SyncMessage } from '../../common/mod.ts'
+import { type Env, type MakeDurableObjectClassOptions, type StoreId, WebSocketAttachmentSchema } from '../shared.ts'
+import { DoCtx } from './layer.ts'
+
+const encodePullResponse = Schema.encodeSync(SyncMessage.PullResponse)
+type PullBatchItem = SyncMessage.PullResponse['batch'][number]
+
+export const makePush =
+  ({
+    payload,
+    options,
+    storeId,
+    ctx,
+    env,
+  }: {
+    payload: Schema.JsonValue | undefined
+    options: MakeDurableObjectClassOptions | undefined
+    storeId: StoreId
+    ctx: CfTypes.DurableObjectState
+    env: Env
+  }) =>
+  (pushRequest: Omit<SyncMessage.PushRequest, '_tag'>) =>
+    Effect.gen(function* () {
+      // yield* Effect.log(`Pushing ${decodedMessage.batch.length} events`, decodedMessage.batch)
+      const { backendId, storage, currentHeadRef, updateCurrentHead, rpcSubscriptions } = yield* DoCtx
+
+      if (pushRequest.batch.length === 0) {
+        return SyncMessage.PushAck.make({})
+      }
+
+      if (options?.onPush) {
+        yield* Effect.tryAll(() => options.onPush!(pushRequest, { storeId, payload })).pipe(
+          UnexpectedError.mapToUnexpectedError,
+        )
+      }
+
+      if (pushRequest.backendId._tag === 'Some' && pushRequest.backendId.value !== backendId) {
+        return yield* new BackendIdMismatchError({ expected: backendId, received: pushRequest.backendId.value })
+      }
+
+      // This part of the code needs to run sequentially to avoid race conditions
+      const { createdAt } = yield* Effect.gen(function* () {
+        const currentHead = currentHeadRef.current
+        // TODO handle clientId unique conflict
+        // Validate the batch
+        const firstEventParent = pushRequest.batch[0]!.parentSeqNum
+        if (firstEventParent !== currentHead) {
+          // yield* Effect.logDebug('ServerAheadError: backend head mismatch', {
+          //   expectedHead: currentHead,
+          //   providedHead: firstEventParent,
+          //   batchSize: pushRequest.batch.length,
+          //   backendId,
+          // })
+
+          return yield* new ServerAheadError({ minimumExpectedNum: currentHead, providedNum: firstEventParent })
+        }
+
+        const createdAt = new Date().toISOString()
+
+        // TODO possibly model this as a queue in order to speed up subsequent pushes
+        yield* storage.appendEvents(pushRequest.batch, createdAt)
+
+        updateCurrentHead(pushRequest.batch.at(-1)!.seqNum)
+
+        return { createdAt }
+      }).pipe(blockConcurrencyWhile(ctx))
+
+      // Run in background but already return the push ack to the client
+      yield* Effect.gen(function* () {
+        const connectedClients = ctx.getWebSockets()
+
+        // Preparing chunks of responses to make sure we don't exceed the WS message size limit.
+        const responses = yield* Chunk.fromIterable(pushRequest.batch).pipe(
+          splitChunkBySize({
+            maxItems: MAX_PUSH_EVENTS_PER_REQUEST,
+            maxBytes: MAX_WS_MESSAGE_BYTES,
+            encode: (items) =>
+              encodePullResponse(
+                SyncMessage.PullResponse.make({
+                  batch: items.map(
+                    (eventEncoded): PullBatchItem => ({
+                      eventEncoded,
+                      metadata: Option.some(SyncMessage.SyncMetadata.make({ createdAt })),
+                    }),
+                  ),
+                  pageInfo: SyncBackend.pageInfoNoMore,
+                  backendId,
+                }),
+              ),
+          }),
+          Effect.map(
+            Chunk.map((eventsChunk) => {
+              const batchWithMetadata = Chunk.toReadonlyArray(eventsChunk).map((eventEncoded) => ({
+                eventEncoded,
+                metadata: Option.some(SyncMessage.SyncMetadata.make({ createdAt })),
+              }))
+
+              const response = SyncMessage.PullResponse.make({
+                batch: batchWithMetadata,
+                pageInfo: SyncBackend.pageInfoNoMore,
+                backendId,
+              })
+
+              return {
+                response,
+                encoded: Schema.encodeSync(SyncMessage.PullResponse)(response),
+              }
+            }),
+          ),
+        )
+
+        // Dual broadcasting: WebSocket + RPC clients
+
+        // Broadcast to WebSocket clients
+        if (connectedClients.length > 0) {
+          for (const { response, encoded } of responses) {
+            // Only calling once for now.
+            if (options?.onPullRes) {
+              yield* Effect.tryAll(() => options.onPullRes!(response)).pipe(UnexpectedError.mapToUnexpectedError)
+            }
+
+            // NOTE we're also sending the pullRes chunk to the pushing ws client as confirmation
+            for (const conn of connectedClients) {
+              const attachment = Schema.decodeSync(WebSocketAttachmentSchema)(conn.deserializeAttachment())
+
+              // We're doing something a bit "advanced" here as we're directly emitting Effect RPC-compatible
+              // response messsages on the Effect RPC-managed websocket connection to the WS client.
+              // For this we need to get the RPC `requestId` from the WebSocket attachment.
+              for (const requestId of attachment.pullRequestIds) {
+                const res: RpcMessage.ResponseChunkEncoded = {
+                  _tag: 'Chunk',
+                  requestId,
+                  values: [encoded],
+                }
+                conn.send(JSON.stringify(res))
+              }
+            }
+          }
+
+          yield* Effect.logDebug(`Broadcasted to ${connectedClients.length} WebSocket clients`)
+        }
+
+        // RPC broadcasting would require reconstructing client stubs from clientIds
+        if (rpcSubscriptions.size > 0) {
+          for (const subscription of rpcSubscriptions.values()) {
+            for (const { encoded } of responses) {
+              yield* emitStreamResponse({
+                callerContext: subscription.callerContext,
+                env,
+                requestId: subscription.requestId,
+                values: [encoded],
+              }).pipe(Effect.tapCauseLogPretty, Effect.exit)
+            }
+          }
+
+          yield* Effect.logDebug(`Broadcasted to ${rpcSubscriptions.size} RPC clients`)
+        }
+      }).pipe(
+        Effect.tapCauseLogPretty,
+        Effect.withSpan('push-rpc-broadcast'),
+        Effect.uninterruptible, // We need to make sure Effect RPC doesn't interrupt this fiber
+        Effect.fork,
+      )
+
+      // We need to yield here to make sure the fork above is kicked off before we let Effect RPC finish the request
+      yield* Effect.yieldNow()
+
+      return SyncMessage.PushAck.make({})
+    }).pipe(
+      Effect.tap(
+        Effect.fn(function* (message) {
+          if (options?.onPushRes) {
+            yield* Effect.tryAll(() => options.onPushRes!(message)).pipe(UnexpectedError.mapToUnexpectedError)
+          }
+        }),
+      ),
+      Effect.mapError((cause) => InvalidPushError.make({ cause })),
+      Effect.withSpan('sync-cf:do:push', { attributes: { storeId, batchSize: pushRequest.batch.length } }),
+    )
+
+/**
+ * @see https://developers.cloudflare.com/durable-objects/api/state/#blockconcurrencywhile
+ */
+const blockConcurrencyWhile =
+  (ctx: CfTypes.DurableObjectState) =>
+  <A, E, R>(eff: Effect.Effect<A, E, R>) =>
+    Effect.gen(function* () {
+      const runtime = yield* Effect.runtime<R>()
+      const exit = yield* Effect.promise(() =>
+        ctx.blockConcurrencyWhile(() => eff.pipe(Effect.provide(runtime), Effect.runPromiseExit)),
+      )
+
+      return yield* exit
+    })

package/src/cf-worker/do/sqlite.ts

@@ -0,0 +1,28 @@
+import { EventSequenceNumber, State } from '@livestore/common/schema'
+import { Schema } from '@livestore/utils/effect'
+import { PERSISTENCE_FORMAT_VERSION } from '../shared.ts'
+
+export const eventlogTable = State.SQLite.table({
+  // NOTE actual table name is determined at runtime to use proper storeId
+  name: `eventlog_${PERSISTENCE_FORMAT_VERSION}_$storeId`,
+  columns: {
+    seqNum: State.SQLite.integer({ primaryKey: true, schema: EventSequenceNumber.GlobalEventSequenceNumber }),
+    parentSeqNum: State.SQLite.integer({ schema: EventSequenceNumber.GlobalEventSequenceNumber }),
+    name: State.SQLite.text({}),
+    args: State.SQLite.text({ schema: Schema.parseJson(Schema.Any), nullable: true }),
+    /** ISO date format. Currently only used for debugging purposes. */
+    createdAt: State.SQLite.text({}),
+    clientId: State.SQLite.text({}),
+    sessionId: State.SQLite.text({}),
+  },
+})
+
+/** Will only ever have one row per durable object. */
+export const contextTable = State.SQLite.table({
+  name: `context_${PERSISTENCE_FORMAT_VERSION}`,
+  columns: {
+    storeId: State.SQLite.text({ primaryKey: true }),
+    currentHead: State.SQLite.integer({ schema: EventSequenceNumber.GlobalEventSequenceNumber }),
+    backendId: State.SQLite.text({}),
+  },
+})

package/src/cf-worker/do/sync-storage.ts

@@ -0,0 +1,321 @@
+import { UnexpectedError } from '@livestore/common'
+import type { LiveStoreEvent } from '@livestore/common/schema'
+import type { CfTypes } from '@livestore/common-cf'
+import { Chunk, Effect, Option, Schema, Stream } from '@livestore/utils/effect'
+import { SyncMetadata } from '../../common/sync-message-types.ts'
+import { PERSISTENCE_FORMAT_VERSION, type StoreId } from '../shared.ts'
+import { eventlogTable } from './sqlite.ts'
+
+export type SyncStorage = {
+  dbName: string
+  getEvents: (cursor: number | undefined) => Effect.Effect<
+    {
+      total: number
+      stream: Stream.Stream<
+        { eventEncoded: LiveStoreEvent.AnyEncodedGlobal; metadata: Option.Option<SyncMetadata> },
+        UnexpectedError
+      >
+    },
+    UnexpectedError
+  >
+  appendEvents: (
+    batch: ReadonlyArray<LiveStoreEvent.AnyEncodedGlobal>,
+    createdAt: string,
+  ) => Effect.Effect<void, UnexpectedError>
+  resetStore: Effect.Effect<void, UnexpectedError>
+}
+
+export const makeStorage = (
+  ctx: CfTypes.DurableObjectState,
+  storeId: StoreId,
+  engine: { _tag: 'd1'; db: CfTypes.D1Database } | { _tag: 'do-sqlite' },
+): SyncStorage => {
+  const dbName = `eventlog_${PERSISTENCE_FORMAT_VERSION}_${toValidTableName(storeId)}`
+
+  const execDb = <T>(cb: (db: CfTypes.D1Database) => Promise<CfTypes.D1Result<T>>) =>
+    Effect.tryPromise({
+      try: () => cb(engine._tag === 'd1' ? engine.db : (undefined as never)),
+      catch: (error) => new UnexpectedError({ cause: error, payload: { dbName } }),
+    }).pipe(
+      Effect.map((_) => _.results),
+      Effect.withSpan('@livestore/sync-cf:durable-object:execDb'),
+    )
+
+  // Cloudflare's D1 HTTP endpoint rejects JSON responses once they exceed ~1MB.
+  // Keep individual SELECT batches comfortably below that threshold so we can
+  // serve large histories without tripping the limit.
+  const D1_MAX_JSON_RESPONSE_BYTES = 1_000_000
+  const D1_RESPONSE_SAFETY_MARGIN_BYTES = 64 * 1024
+  const D1_TARGET_RESPONSE_BYTES = D1_MAX_JSON_RESPONSE_BYTES - D1_RESPONSE_SAFETY_MARGIN_BYTES
+  const D1_INITIAL_PAGE_SIZE = 256
+  const D1_MIN_PAGE_SIZE = 1
+
+  const decodeEventlogRows = Schema.decodeUnknownSync(Schema.Array(eventlogTable.rowSchema))
+  const textEncoder = new TextEncoder()
+
+  const decreaseLimit = (limit: number) => Math.max(D1_MIN_PAGE_SIZE, Math.floor(limit / 2))
+  const increaseLimit = (limit: number) => Math.min(D1_INITIAL_PAGE_SIZE, limit * 2)
+
+  const computeNextLimit = (limit: number, encodedSize: number) => {
+    if (encodedSize > D1_TARGET_RESPONSE_BYTES && limit > D1_MIN_PAGE_SIZE) {
+      const next = decreaseLimit(limit)
+      return next === limit ? limit : next
+    }
+
+    if (encodedSize < D1_TARGET_RESPONSE_BYTES / 2 && limit < D1_INITIAL_PAGE_SIZE) {
+      const next = increaseLimit(limit)
+      return next === limit ? limit : next
+    }
+
+    return limit
+  }
+
+  const getEventsD1 = (
+    cursor: number | undefined,
+  ): Effect.Effect<
+    {
+      total: number
+      stream: Stream.Stream<
+        { eventEncoded: LiveStoreEvent.AnyEncodedGlobal; metadata: Option.Option<SyncMetadata> },
+        UnexpectedError
+      >
+    },
+    UnexpectedError
+  > =>
+    Effect.gen(function* () {
+      const countStatement =
+        cursor === undefined
+          ? `SELECT COUNT(*) as total FROM ${dbName}`
+          : `SELECT COUNT(*) as total FROM ${dbName} WHERE seqNum > ?`
+
+      const countRows = yield* execDb<{ total: number }>((db) => {
+        const prepared = db.prepare(countStatement)
+        return cursor === undefined ? prepared.all() : prepared.bind(cursor).all()
+      })
+
+      const total = Number(countRows[0]?.total ?? 0)
+
+      type State = { cursor: number | undefined; limit: number }
+      type EmittedEvent = { eventEncoded: LiveStoreEvent.AnyEncodedGlobal; metadata: Option.Option<SyncMetadata> }
+
+      const initialState: State = { cursor, limit: D1_INITIAL_PAGE_SIZE }
+
+      const fetchPage = (
+        state: State,
+      ): Effect.Effect<Option.Option<readonly [Chunk.Chunk<EmittedEvent>, State]>, UnexpectedError> =>
+        Effect.gen(function* () {
+          const statement =
+            state.cursor === undefined
+              ? `SELECT * FROM ${dbName} ORDER BY seqNum ASC LIMIT ?`
+              : `SELECT * FROM ${dbName} WHERE seqNum > ? ORDER BY seqNum ASC LIMIT ?`
+
+          const rawEvents = yield* execDb((db) => {
+            const prepared = db.prepare(statement)
+            return state.cursor === undefined
+              ? prepared.bind(state.limit).all()
+              : prepared.bind(state.cursor, state.limit).all()
+          })
+
+          if (rawEvents.length === 0) {
+            return Option.none()
+          }
+
+          const encodedSize = textEncoder.encode(JSON.stringify(rawEvents)).byteLength
+
+          if (encodedSize > D1_TARGET_RESPONSE_BYTES && state.limit > D1_MIN_PAGE_SIZE) {
+            const nextLimit = decreaseLimit(state.limit)
+
+            if (nextLimit !== state.limit) {
+              return yield* fetchPage({ cursor: state.cursor, limit: nextLimit })
+            }
+          }
+
+          const decodedRows = Chunk.fromIterable(decodeEventlogRows(rawEvents))
+
+          const eventsChunk = Chunk.map(decodedRows, ({ createdAt, ...eventEncoded }) => ({
+            eventEncoded,
+            metadata: Option.some(SyncMetadata.make({ createdAt })),
+          }))
+
+          const lastSeqNum = Chunk.unsafeLast(decodedRows).seqNum
+          const nextState: State = { cursor: lastSeqNum, limit: computeNextLimit(state.limit, encodedSize) }
+
+          return Option.some([eventsChunk, nextState] as const)
+        })
+
+      const stream = Stream.unfoldChunkEffect(initialState, fetchPage)
+
+      return { total, stream }
+    }).pipe(
+      UnexpectedError.mapToUnexpectedError,
+      Effect.withSpan('@livestore/sync-cf:durable-object:getEvents', {
+        attributes: { dbName, cursor, engine: engine._tag },
+      }),
+    )
+
+  const appendEventsD1: SyncStorage['appendEvents'] = (batch, createdAt) =>
+    Effect.gen(function* () {
+      // If there are no events, do nothing.
+      if (batch.length === 0) return
+
+      // CF D1 limits:
+      // Maximum bound parameters per query	100, Maximum arguments per SQL function	32
+      // Thus we need to split the batch into chunks of max (100/7=)14 events each.
+      const CHUNK_SIZE = 14
+
+      for (let i = 0; i < batch.length; i += CHUNK_SIZE) {
+        const chunk = batch.slice(i, i + CHUNK_SIZE)
+
+        // Create a list of placeholders ("(?, ?, ?, ?, ?, ?, ?)"), corresponding to each event.
+        const valuesPlaceholders = chunk.map(() => '(?, ?, ?, ?, ?, ?, ?)').join(', ')
+        const sql = `INSERT INTO ${dbName} (seqNum, parentSeqNum, args, name, createdAt, clientId, sessionId) VALUES ${valuesPlaceholders}`
+        // Flatten the event properties into a parameters array.
+        const params = chunk.flatMap((event) => [
+          event.seqNum,
+          event.parentSeqNum,
+          event.args === undefined ? null : JSON.stringify(event.args),
+          event.name,
+          createdAt,
+          event.clientId,
+          event.sessionId,
+        ])
+
+        yield* execDb((db) =>
+          db
+            .prepare(sql)
+            .bind(...params)
+            .run(),
+        )
+      }
+    }).pipe(
+      UnexpectedError.mapToUnexpectedError,
+      Effect.withSpan('@livestore/sync-cf:durable-object:appendEvents', {
+        attributes: { dbName, batchLength: batch.length, engine: engine._tag },
+      }),
+    )
+
+  const resetStore = Effect.promise(() => ctx.storage.deleteAll()).pipe(
+    UnexpectedError.mapToUnexpectedError,
+    Effect.withSpan('@livestore/sync-cf:durable-object:resetStore'),
+  )
+
+  // DO SQLite engine implementation
+  const getEventsDoSqlite = (
+    cursor: number | undefined,
+  ): Effect.Effect<
+    {
+      total: number
+      stream: Stream.Stream<
+        { eventEncoded: LiveStoreEvent.AnyEncodedGlobal; metadata: Option.Option<SyncMetadata> },
+        UnexpectedError
+      >
+    },
+    UnexpectedError
+  > =>
+    Effect.gen(function* () {
+      const selectCountSql =
+        cursor === undefined
+          ? `SELECT COUNT(*) as total FROM "${dbName}"`
+          : `SELECT COUNT(*) as total FROM "${dbName}" WHERE seqNum > ?`
+
+      let total = 0
+      try {
+        const cursorIter =
+          cursor === undefined ? ctx.storage.sql.exec(selectCountSql) : ctx.storage.sql.exec(selectCountSql, cursor)
+        for (const row of cursorIter) {
+          total = Number((row as any).total ?? 0)
+        }
+      } catch (error) {
+        return yield* Effect.fail(new UnexpectedError({ cause: error, payload: { dbName, stage: 'count' } }))
+      }
+
+      type State = { cursor: number | undefined }
+      type EmittedEvent = { eventEncoded: LiveStoreEvent.AnyEncodedGlobal; metadata: Option.Option<SyncMetadata> }
+
+      const DO_PAGE_SIZE = 256
+      const initialState: State = { cursor }
+
+      const fetchPage = (
+        state: State,
+      ): Effect.Effect<Option.Option<readonly [Chunk.Chunk<EmittedEvent>, State]>, UnexpectedError> =>
+        Effect.try({
+          try: () => {
+            const sql =
+              state.cursor === undefined
+                ? `SELECT * FROM "${dbName}" ORDER BY seqNum ASC LIMIT ?`
+                : `SELECT * FROM "${dbName}" WHERE seqNum > ? ORDER BY seqNum ASC LIMIT ?`
+
+            const iter =
+              state.cursor === undefined
+                ? ctx.storage.sql.exec(sql, DO_PAGE_SIZE)
+                : ctx.storage.sql.exec(sql, state.cursor, DO_PAGE_SIZE)
+
+            const rows: any[] = []
+            for (const row of iter) rows.push(row)
+
+            if (rows.length === 0) {
+              return Option.none()
+            }
+
+            const decodedRows = Chunk.fromIterable(decodeEventlogRows(rows))
+            const eventsChunk = Chunk.map(decodedRows, ({ createdAt, ...eventEncoded }) => ({
+              eventEncoded,
+              metadata: Option.some(SyncMetadata.make({ createdAt })),
+            }))
+
+            const lastSeqNum = Chunk.unsafeLast(decodedRows).seqNum
+            const nextState: State = { cursor: lastSeqNum }
+
+            return Option.some([eventsChunk, nextState] as const)
+          },
+          catch: (error) => new UnexpectedError({ cause: error, payload: { dbName, stage: 'select' } }),
+        })
+
+      const stream = Stream.unfoldChunkEffect(initialState, fetchPage)
+
+      return { total, stream }
+    }).pipe(
+      UnexpectedError.mapToUnexpectedError,
+      Effect.withSpan('@livestore/sync-cf:durable-object:getEvents', {
+        attributes: { dbName, cursor, engine: engine._tag },
+      }),
+    )
+
+  const appendEventsDoSqlite: SyncStorage['appendEvents'] = (batch, createdAt) =>
+    Effect.try({
+      try: () => {
+        if (batch.length === 0) return
+        // Keep params per statement within conservative limits (align with D1 bound params ~100)
+        const CHUNK_SIZE = 14
+        for (let i = 0; i < batch.length; i += CHUNK_SIZE) {
+          const chunk = batch.slice(i, i + CHUNK_SIZE)
+          const placeholders = chunk.map(() => '(?, ?, ?, ?, ?, ?, ?)').join(', ')
+          const sql = `INSERT INTO "${dbName}" (seqNum, parentSeqNum, args, name, createdAt, clientId, sessionId) VALUES ${placeholders}`
+          const params = chunk.flatMap((event) => [
+            event.seqNum,
+            event.parentSeqNum,
+            event.args === undefined ? null : JSON.stringify(event.args),
+            event.name,
+            createdAt,
+            event.clientId,
+            event.sessionId,
+          ])
+          ctx.storage.sql.exec(sql, ...params)
+        }
+      },
+      catch: (error) => new UnexpectedError({ cause: error, payload: { dbName, stage: 'insert' } }),
+    }).pipe(
+      Effect.withSpan('@livestore/sync-cf:durable-object:appendEvents', {
+        attributes: { dbName, batchLength: batch.length, engine: engine._tag },
+      }),
+      UnexpectedError.mapToUnexpectedError,
+    )
+
+  if (engine._tag === 'd1') {
+    return { dbName, getEvents: getEventsD1, appendEvents: appendEventsD1, resetStore }
+  }
+
+  return { dbName, getEvents: getEventsDoSqlite, appendEvents: appendEventsDoSqlite, resetStore }
+}
+
+const toValidTableName = (str: string) => str.replaceAll(/[^a-zA-Z0-9]/g, '_')

package/src/cf-worker/do/transport/do-rpc-server.ts

@@ -0,0 +1,84 @@
+import { InvalidPullError, InvalidPushError } from '@livestore/common'
+import { type CfTypes, toDurableObjectHandler } from '@livestore/common-cf'
+import {
+  Effect,
+  Headers,
+  HttpServer,
+  Layer,
+  Logger,
+  LogLevel,
+  Option,
+  RpcSerialization,
+  Stream,
+} from '@livestore/utils/effect'
+import { SyncDoRpc } from '../../../common/do-rpc-schema.ts'
+import { SyncMessage } from '../../../common/mod.ts'
+import { DoCtx, type DoCtxInput } from '../layer.ts'
+import { makeEndingPullStream } from '../pull.ts'
+import { makePush } from '../push.ts'
+
+export interface DoRpcHandlerOptions {
+  payload: Uint8Array<ArrayBuffer>
+  input: Omit<DoCtxInput, 'from'>
+}
+
+export const createDoRpcHandler = (
+  options: DoRpcHandlerOptions,
+): Effect.Effect<Uint8Array<ArrayBuffer> | CfTypes.ReadableStream> =>
+  Effect.gen(this, function* () {
+    const { payload, input } = options
+    // const { rpcSubscriptions, backendId, doOptions, ctx, env } = yield* DoCtx
+
+    // TODO add admin RPCs
+    const RpcLive = SyncDoRpc.toLayer({
+      'SyncDoRpc.Ping': (_req) => {
+        return Effect.succeed(SyncMessage.Pong.make({}))
+      },
+      'SyncDoRpc.Pull': (req, { headers }) =>
+        Effect.gen(this, function* () {
+          const { rpcSubscriptions } = yield* DoCtx
+
+          // TODO rename `req.rpcContext` to something more appropriate
+          if (req.rpcContext) {
+            rpcSubscriptions.set(req.storeId, {
+              storeId: req.storeId,
+              payload: req.payload,
+              subscribedAt: Date.now(),
+              requestId: Headers.get(headers, 'x-rpc-request-id').pipe(Option.getOrThrow),
+              callerContext: req.rpcContext.callerContext,
+            })
+          }
+
+          return makeEndingPullStream(req, req.payload)
+        }).pipe(
+          Stream.unwrap,
+          Stream.map((res) => ({
+            ...res,
+            rpcRequestId: Headers.get(headers, 'x-rpc-request-id').pipe(Option.getOrThrow),
+          })),
+          Stream.provideLayer(DoCtx.Default({ ...input, from: { storeId: req.storeId } })),
+          Stream.mapError((cause) => (cause._tag === 'InvalidPullError' ? cause : InvalidPullError.make({ cause }))),
+          Stream.tapErrorCause(Effect.log),
+        ),
+      'SyncDoRpc.Push': (req) =>
+        Effect.gen(this, function* () {
+          const { doOptions, ctx, env, storeId } = yield* DoCtx
+          const push = makePush({ storeId, payload: req.payload, options: doOptions, ctx, env })
+
+          return yield* push(req)
+        }).pipe(
+          Effect.provide(DoCtx.Default({ ...input, from: { storeId: req.storeId } })),
+          Effect.mapError((cause) => (cause._tag === 'InvalidPushError' ? cause : InvalidPushError.make({ cause }))),
+          Effect.tapCauseLogPretty,
+        ),
+    })
+
+    const handler = toDurableObjectHandler(SyncDoRpc, {
+      layer: Layer.mergeAll(RpcLive, RpcSerialization.layerJson, HttpServer.layerContext).pipe(
+        Layer.provide(Logger.consoleWithThread('SyncDo')),
+        Layer.provide(Logger.minimumLogLevel(LogLevel.Debug)),
+      ),
+    })
+
+    return yield* handler(payload)
+  }).pipe(Effect.withSpan('createDoRpcHandler'))

package/src/cf-worker/do/transport/http-rpc-server.ts

@@ -0,0 +1,37 @@
+import type { CfTypes } from '@livestore/common-cf'
+import { Effect, HttpApp, Layer, RpcSerialization, RpcServer } from '@livestore/utils/effect'
+import { SyncHttpRpc } from '../../../common/http-rpc-schema.ts'
+import * as SyncMessage from '../../../common/sync-message-types.ts'
+import { DoCtx } from '../layer.ts'
+import { makeEndingPullStream } from '../pull.ts'
+import { makePush } from '../push.ts'
+
+export const createHttpRpcHandler = ({ request }: { request: CfTypes.Request }) =>
+  Effect.gen(function* () {
+    const handlerLayer = createHttpRpcLayer
+    const httpApp = RpcServer.toHttpApp(SyncHttpRpc).pipe(Effect.provide(handlerLayer))
+    const webHandler = yield* httpApp.pipe(Effect.map(HttpApp.toWebHandler))
+
+    return yield* Effect.promise(
+      () => webHandler(request as TODO as Request) as TODO as Promise<CfTypes.Response>,
+    ).pipe(Effect.timeout(10000))
+  }).pipe(Effect.withSpan('createHttpRpcHandler'))
+
+const createHttpRpcLayer =
+  // TODO implement admin requests
+  SyncHttpRpc.toLayer({
+    'SyncHttpRpc.Pull': (req) => makeEndingPullStream(req, req.payload),
+
+    'SyncHttpRpc.Push': (req) =>
+      Effect.gen(function* () {
+        const { ctx, env, doOptions, storeId } = yield* DoCtx
+        const push = makePush({ payload: undefined, options: doOptions, storeId, ctx, env })
+
+        return yield* push(req)
+      }),
+
+    'SyncHttpRpc.Ping': () => Effect.succeed(SyncMessage.Pong.make({})),
+  }).pipe(
+    Layer.provideMerge(RpcServer.layerProtocolHttp({ path: '/http-rpc' })),
+    Layer.provideMerge(RpcSerialization.layerJson),
+  )