@livestore/sync-cf 0.4.0-dev.9 → 0.5.0-dev.0

package/src/cf-worker/shared.ts

@@ -1,27 +1,81 @@
-import type { InvalidPullError, InvalidPushError } from '@livestore/common'
+import type { UnknownError } from '@livestore/common'
 import type { CfTypes } from '@livestore/common-cf'
-import { Effect, Schema, UrlParams } from '@livestore/utils/effect'
+import type { Effect } from '@livestore/utils/effect'
+import { Result, Schema } from '@livestore/utils/effect'
+
 import type { SearchParams } from '../common/mod.ts'
 import { SearchParamsSchema, SyncMessage } from '../common/mod.ts'
 
-export interface Env {
-  /** Eventlog database */
-  DB: CfTypes.D1Database
-  ADMIN_SECRET: string
+export type Env = {}
+
+/** Headers forwarded from the request to callbacks */
+export type ForwardedHeaders = ReadonlyMap<string, string>
+
+/**
+ * Configuration for forwarding request headers to DO callbacks.
+ * - `string[]`: List of header names to forward (case-insensitive)
+ * - `(request) => Record<string, string>`: Custom extraction function (sync)
+ */
+export type ForwardHeadersOption = readonly string[] | ((request: CfTypes.Request) => Record<string, string>)
+
+/** Context passed to onPush/onPull callbacks */
+export type CallbackContext = {
+  storeId: StoreId
+  payload?: Schema.Json
+  /** Headers forwarded from the request (only present if `forwardHeaders` is configured) */
+  headers?: ForwardedHeaders
 }
 
 export type MakeDurableObjectClassOptions = {
-  onPush?: (
-    message: SyncMessage.PushRequest,
-    context: { storeId: StoreId; payload?: Schema.JsonValue },
-  ) => Effect.SyncOrPromiseOrEffect<void>
-  onPushRes?: (message: SyncMessage.PushAck | InvalidPushError) => Effect.SyncOrPromiseOrEffect<void>
-  onPull?: (
-    message: SyncMessage.PullRequest,
-    context: { storeId: StoreId; payload?: Schema.JsonValue },
-  ) => Effect.SyncOrPromiseOrEffect<void>
-  onPullRes?: (message: SyncMessage.PullResponse | InvalidPullError) => Effect.SyncOrPromiseOrEffect<void>
-  // TODO make storage configurable: D1, DO SQLite, later: external SQLite
+  onPush?: (message: SyncMessage.PushRequest, context: CallbackContext) => Effect.SyncOrPromiseOrEffect<void>
+  onPushRes?: (message: SyncMessage.PushAck | UnknownError) => Effect.SyncOrPromiseOrEffect<void>
+  onPull?: (message: SyncMessage.PullRequest, context: CallbackContext) => Effect.SyncOrPromiseOrEffect<void>
+  onPullRes?: (message: SyncMessage.PullResponse | UnknownError) => Effect.SyncOrPromiseOrEffect<void>
+
+  /**
+   * Forward request headers to `onPush`/`onPull` callbacks for authentication.
+   *
+   * This enables cookie-based or header-based authentication patterns where
+   * you need access to request headers inside the Durable Object.
+   *
+   * @example Forward specific headers by name (case-insensitive)
+   * ```ts
+   * makeDurableObject({
+   *   forwardHeaders: ['cookie', 'authorization'],
+   *   onPush: async (message, { headers }) => {
+   *     const cookie = headers?.get('cookie')
+   *     const session = await validateSession(cookie)
+   *   },
+   * })
+   * ```
+   *
+   * @example Custom extraction function for derived values
+   * ```ts
+   * makeDurableObject({
+   *   forwardHeaders: (request) => ({
+   *     'x-user-id': request.headers.get('x-user-id') ?? '',
+   *     'x-session': request.headers.get('cookie')?.split('session=')[1]?.split(';')[0] ?? '',
+   *   }),
+   *   onPush: async (message, { headers }) => {
+   *     const userId = headers?.get('x-user-id')
+   *   },
+   * })
+   * ```
+   */
+  forwardHeaders?: ForwardHeadersOption
+  /**
+   * Storage engine for event persistence.
+   * - Default: `{ _tag: 'do-sqlite' }` (Durable Object SQLite)
+   * - D1: `{ _tag: 'd1', binding: string }` where `binding` is the D1 binding name in wrangler.toml.
+   *
+   * If omitted, the runtime defaults to DO SQLite. For backwards-compatibility, if an env binding named
+   * `DB` exists and looks like a D1Database, D1 will be used.
+   *
+   * Trade-offs:
+   * - DO SQLite: simpler deploy, data co-located with DO, not externally queryable
+   * - D1: centralized DB, inspectable with DB tools, extra network hop and JSON size limits
+   */
+  storage?: { _tag: 'do-sqlite' } | { _tag: 'd1'; binding: string }
 
   /**
    * Enabled transports for sync backend
@@ -33,6 +87,26 @@ export type MakeDurableObjectClassOptions = {
    */
   enabledTransports?: Set<'http' | 'ws' | 'do-rpc'>
 
+  /**
+   * Custom HTTP response headers for HTTP transport
+   * These headers will be added to all HTTP RPC responses (Pull, Push, Ping)
+   *
+   * @example
+   * ```ts
+   * {
+   *   http: {
+   *     responseHeaders: {
+   *       'Access-Control-Allow-Origin': '*',
+   *       'Cache-Control': 'no-cache'
+   *     }
+   *   }
+   * }
+   * ```
+   */
+  http?: {
+    responseHeaders?: Record<string, string>
+  }
+
   otel?: {
     baseUrl?: string
     serviceName?: string
@@ -43,14 +117,27 @@ export type StoreId = string
 export type DurableObjectId = string
 
 /**
- * Needs to be bumped when the storage format changes (e.g. eventlogTable schema changes)
+ * CRITICAL: Increment this version whenever you modify the database schema structure.
+ *
+ * Bump required when:
+ * - Adding/removing/renaming columns in eventlogTable or contextTable (see sqlite.ts)
+ * - Changing column types or constraints
+ * - Modifying primary keys or indexes
  *
- * Changing this version number will lead to a "soft reset".
+ * Bump NOT required when:
+ * - Changing query patterns, pagination logic, or streaming behavior
+ * - Adding new tables (as long as existing table schemas remain unchanged)
+ * - Updating implementation details in sync-storage.ts
+ *
+ * Impact: Changing this version triggers a "soft reset" - new table names are created
+ * and old data becomes inaccessible (but remains in storage).
  */
 export const PERSISTENCE_FORMAT_VERSION = 7
 
-export const encodeOutgoingMessage = Schema.encodeSync(Schema.parseJson(SyncMessage.BackendToClientMessage))
-export const encodeIncomingMessage = Schema.encodeSync(Schema.parseJson(SyncMessage.ClientToBackendMessage))
+export const encodeOutgoingMessage = Schema.encodeSync(Schema.fromJsonString(SyncMessage.BackendToClientMessage))
+export const encodeIncomingMessage = Schema.encodeSync(Schema.fromJsonString(SyncMessage.ClientToBackendMessage))
+
+const SearchParamsFromUrlSearchParams = Schema.fromURLSearchParams(SearchParamsSchema)
 
 /**
  * Extracts the LiveStore sync search parameters from a request. Returns
@@ -59,27 +146,19 @@ export const encodeIncomingMessage = Schema.encodeSync(Schema.parseJson(SyncMess
  */
 export const matchSyncRequest = (request: CfTypes.Request): SearchParams | undefined => {
   const url = new URL(request.url)
-  const urlParams = UrlParams.fromInput(url.searchParams)
-  const paramsResult = UrlParams.schemaStruct(SearchParamsSchema)(urlParams).pipe(Effect.option, Effect.runSync)
+  const paramsResult = Schema.decodeUnknownResult(SearchParamsFromUrlSearchParams)(url.searchParams)
 
-  if (paramsResult._tag === 'None') {
+  if (Result.isFailure(paramsResult) === true) {
     return undefined
   }
 
-  return paramsResult.value
+  return paramsResult.success
 }
 
-export const MAX_PULL_EVENTS_PER_MESSAGE = 100
-
-// Cloudflare hibernated WebSocket frames begin failing just below 1MB. Keep our
-// payloads comfortably beneath that ceiling so we don't rely on implementation
-// quirks of local dev servers.
-export const MAX_WS_MESSAGE_BYTES = 900_000
-
 // RPC subscription storage (TODO refactor)
 export type RpcSubscription = {
   storeId: StoreId
-  payload?: Schema.JsonValue
+  payload?: Schema.Json
   subscribedAt: number
   /** Effect RPC request ID */
   requestId: string
@@ -89,6 +168,9 @@ export type RpcSubscription = {
   }
 }
 
+/** Key prefix for DO-RPC subscriptions persisted in the DO's synchronous KV storage. */
+export const rpcSubscriptionKeyPrefix = 'rpc-sub:'
+
 /**
  * Durable Object interface supporting the DO RPC protocol for DO <> DO syncing.
  */
@@ -97,12 +179,46 @@ export interface SyncBackendRpcInterface {
   rpc(payload: Uint8Array): Promise<Uint8Array | CfTypes.ReadableStream>
 }
 
-export const WebSocketAttachmentSchema = Schema.parseJson(
+export const WebSocketAttachmentSchema = Schema.fromJsonString(
   Schema.Struct({
     // Same across all websocket connections
     storeId: Schema.String,
     // Different for each websocket connection
-    payload: Schema.optional(Schema.JsonValue),
-    pullRequestIds: Schema.Array(Schema.String),
+    payload: Schema.optional(Schema.Json),
+    pullRequestIds: Schema.Array(Schema.Union([Schema.String, Schema.Finite])),
+    // Headers forwarded from the initial request (via forwardHeaders option)
+    headers: Schema.optional(Schema.Record(Schema.String, Schema.String)),
   }),
 )
+
+/** Helper to extract headers from a request based on the forwardHeaders option */
+export const extractForwardedHeaders = (
+  request: CfTypes.Request,
+  forwardHeaders: ForwardHeadersOption | undefined,
+): Record<string, string> | undefined => {
+  if (forwardHeaders === undefined) {
+    return undefined
+  }
+
+  if (typeof forwardHeaders === 'function') {
+    return forwardHeaders(request)
+  }
+
+  // Array of header names - extract them case-insensitively
+  const result: Record<string, string> = {}
+  for (const name of forwardHeaders) {
+    const value = request.headers.get(name)
+    if (value !== null) {
+      result[name.toLowerCase()] = value
+    }
+  }
+  return Object.keys(result).length > 0 ? result : undefined
+}
+
+/** Convert a headers record to a ReadonlyMap */
+export const headersRecordToMap = (headers: Record<string, string> | undefined): ForwardedHeaders | undefined => {
+  if (headers === undefined) {
+    return undefined
+  }
+  return new Map(Object.entries(headers).map(([key, value]) => [key.toLowerCase(), value]))
+}

package/src/cf-worker/worker.ts

@@ -1,13 +1,14 @@
-import { UnexpectedError } from '@livestore/common'
+import { env as importedEnv } from 'cloudflare:workers'
+
+import { UnknownError } from '@livestore/common'
 import type { HelperTypes } from '@livestore/common-cf'
-import type { Schema } from '@livestore/utils/effect'
-import { Effect } from '@livestore/utils/effect'
+import { CfDeclare } from '@livestore/common-cf/declare'
+import { Effect, Result, Schema } from '@livestore/utils/effect'
+
 import type { CfTypes, SearchParams } from '../common/mod.ts'
-import type { CfDeclare } from './mod.ts'
-import { type Env, matchSyncRequest } from './shared.ts'
+import { type Env, type ForwardedHeaders, matchSyncRequest } from './shared.ts'
 
-// NOTE We need to redeclare runtime types here to avoid type conflicts with the lib.dom Response type.
-declare class Response extends CfDeclare.Response {}
+const Response = CfDeclare.Response
 
 // HINT: If we ever extend user's custom worker RPC, type T can help here with expected return type safety. Currently unused.
 export type CFWorker<TEnv extends Env = Env, _T extends CfTypes.Rpc.DurableObjectBranded | undefined = undefined> = {
@@ -18,21 +19,46 @@ export type CFWorker<TEnv extends Env = Env, _T extends CfTypes.Rpc.DurableObjec
   ) => Promise<CfTypes.Response>
 }
 
+/** Context passed to validatePayload callback */
+export type ValidatePayloadContext = {
+  storeId: string
+  /** Request headers (raw, not filtered by forwardHeaders) */
+  headers: ForwardedHeaders
+}
+
 /**
  * Options accepted by {@link makeWorker}. The Durable Object binding has to be
  * supplied explicitly so we never fall back to deprecated defaults when Cloudflare config changes.
  */
-export type MakeWorkerOptions<TEnv extends Env = Env> = {
+export type MakeWorkerOptions<TEnv extends Env = Env, TSyncPayload = Schema.Json> = {
   /**
    * Binding name of the sync Durable Object declared in wrangler config.
    */
   syncBackendBinding: HelperTypes.ExtractDurableObjectKeys<TEnv>
   /**
-   * Validates the payload during WebSocket connection establishment.
+   * Optionally pass a schema to decode the client-provided payload into a typed object
+   * before calling {@link validatePayload}. If omitted, the raw JSON value is forwarded.
+   */
+  syncPayloadSchema?: Schema.Decoder<TSyncPayload>
+  /**
+   * Validates the (optionally decoded) payload during WebSocket connection establishment.
+   * If {@link syncPayloadSchema} is provided, `payload` will be of the schema's inferred type.
+   *
+   * The context includes request headers for cookie-based or header-based authentication.
+   *
+   * @example Cookie-based authentication
+   * ```ts
+   * validatePayload: async (payload, { storeId, headers }) => {
+   *   const cookie = headers.get('cookie')
+   *   const session = await validateSessionFromCookie(cookie)
+   *   if (!session) throw new Error('Unauthorized')
+   * }
+   * ```
+   *
    * Note: This runs only at connection time, not for individual push events.
-   * For push event validation, use the `onPush` callback in the durable object.
+   * For push event validation, use the `onPush` callback in the Durable Object.
    */
-  validatePayload?: (payload: Schema.JsonValue | undefined, context: { storeId: string }) => void | Promise<void>
+  validatePayload?: (payload: TSyncPayload, context: ValidatePayloadContext) => void | Promise<void>
   /** @default false */
   enableCORS?: boolean
 }
@@ -47,22 +73,24 @@ export type MakeWorkerOptions<TEnv extends Env = Env> = {
 export const makeWorker = <
   TEnv extends Env = Env,
   TDurableObjectRpc extends CfTypes.Rpc.DurableObjectBranded | undefined = undefined,
+  TSyncPayload = Schema.Json,
 >(
-  options: MakeWorkerOptions<TEnv>,
+  options: MakeWorkerOptions<TEnv, TSyncPayload>,
 ): CFWorker<TEnv, TDurableObjectRpc> => {
   return {
     fetch: async (request, env, _ctx) => {
       const url = new URL(request.url)
 
-      const corsHeaders: CfTypes.HeadersInit = options.enableCORS
-        ? {
-            'Access-Control-Allow-Origin': '*',
-            'Access-Control-Allow-Methods': 'GET, POST, OPTIONS',
-            'Access-Control-Allow-Headers': request.headers.get('Access-Control-Request-Headers') ?? '*',
-          }
-        : {}
+      const corsHeaders: CfTypes.HeadersInit =
+        options.enableCORS === true
+          ? {
+              'Access-Control-Allow-Origin': '*',
+              'Access-Control-Allow-Methods': 'GET, POST, OPTIONS',
+              'Access-Control-Allow-Headers': request.headers.get('Access-Control-Request-Headers') ?? '*',
+            }
+          : {}
 
-      if (request.method === 'OPTIONS' && options.enableCORS) {
+      if (request.method === 'OPTIONS' && options.enableCORS === true) {
         return new Response(null, {
           status: 204,
           headers: corsHeaders,
@@ -73,7 +101,7 @@ export const makeWorker = <
 
       // Check if this is a sync request first, before showing info message
       if (searchParams !== undefined) {
-        return handleSyncRequest<TEnv, TDurableObjectRpc>({
+        return handleSyncRequest<TEnv, TDurableObjectRpc, unknown, TSyncPayload>({
           request,
           searchParams,
           env,
@@ -81,6 +109,7 @@ export const makeWorker = <
           syncBackendBinding: options.syncBackendBinding,
           headers: corsHeaders,
           validatePayload: options.validatePayload,
+          syncPayloadSchema: options.syncPayloadSchema,
         })
       }
 
@@ -106,83 +135,106 @@ export const makeWorker = <
   }
 }
 
+/** Convert CF Request headers to a ForwardedHeaders map */
+const requestHeadersToMap = (request: CfTypes.Request): ForwardedHeaders => {
+  const result = new Map<string, string>()
+  request.headers.forEach((value, key) => {
+    result.set(key.toLowerCase(), value)
+  })
+  return result
+}
+
 /**
- * Handles `/sync` endpoint.
+ * Handles LiveStore sync requests (e.g. with search params `?storeId=...&transport=...`).
  *
- * @example
+ * @example Token-based authentication
  * ```ts
- * const validatePayload = (payload: Schema.JsonValue | undefined, context: { storeId: string }) => {
- *   console.log(`Validating connection for store: ${context.storeId}`)
+ * const validatePayload = (payload: Schema.Json | undefined, context: { storeId: string }) => {
  *   if (payload?.authToken !== 'insecure-token-change-me') {
  *     throw new Error('Invalid auth token')
  *   }
  * }
+ * ```
  *
- * export default {
- *   fetch: async (request, env, ctx) => {
- *     const searchParams = matchSyncRequest(request)
- *
- *     // Is LiveStore sync request
- *     if (searchParams !== undefined) {
- *       return handleSyncRequest({
- *         request,
- *         searchParams,
- *         env,
- *         ctx,
- *         syncBackendBinding: 'SYNC_BACKEND_DO',
- *         headers: {},
- *         validatePayload,
- *       })
- *     }
- *
- *     return new Response('Invalid path', { status: 400 })
- *   }
+ * @example Cookie-based authentication
+ * ```ts
+ * const validatePayload = async (payload: Schema.Json | undefined, { storeId, headers }) => {
+ *   const cookie = headers.get('cookie')
+ *   const session = await validateSessionFromCookie(cookie)
+ *   if (!session) throw new Error('Unauthorized')
  * }
  * ```
  *
- * @throws {UnexpectedError} If the payload is invalid
+ * @throws {UnknownError} If the payload is invalid
  */
 export const handleSyncRequest = <
   TEnv extends Env = Env,
   TDurableObjectRpc extends CfTypes.Rpc.DurableObjectBranded | undefined = undefined,
   CFHostMetada = unknown,
+  TSyncPayload = Schema.Json,
 >({
   request,
   searchParams: { storeId, payload, transport },
-  env,
+  env: explicitlyProvidedEnv,
   syncBackendBinding,
   headers,
   validatePayload,
+  syncPayloadSchema,
 }: {
   request: CfTypes.Request<CFHostMetada>
   searchParams: SearchParams
-  env: TEnv
+  env?: TEnv | undefined
   /** Only there for type-level reasons */
   ctx: CfTypes.ExecutionContext
   /** Binding name of the sync backend Durable Object */
-  syncBackendBinding: MakeWorkerOptions<TEnv>['syncBackendBinding']
+  syncBackendBinding: MakeWorkerOptions<TEnv, TSyncPayload>['syncBackendBinding']
   headers?: CfTypes.HeadersInit | undefined
-  validatePayload?: (payload: Schema.JsonValue | undefined, context: { storeId: string }) => void | Promise<void>
+  validatePayload?: MakeWorkerOptions<TEnv, TSyncPayload>['validatePayload']
+  syncPayloadSchema?: MakeWorkerOptions<TEnv, TSyncPayload>['syncPayloadSchema']
 }): Promise<CfTypes.Response> =>
   Effect.gen(function* () {
     if (validatePayload !== undefined) {
-      const result = yield* Effect.promise(async () => validatePayload!(payload, { storeId })).pipe(
-        UnexpectedError.mapToUnexpectedError,
-        Effect.either,
-      )
+      // Convert request headers to a Map for the validation context
+      const requestHeaders = requestHeadersToMap(request)
 
-      if (result._tag === 'Left') {
-        console.error('Invalid payload', result.left)
-        return new Response(result.left.toString(), { status: 400, headers })
+      // Always decode with the supplied schema when present, even if payload is undefined.
+      // This ensures required payloads are enforced by the schema.
+      if (syncPayloadSchema !== undefined) {
+        const decodedResult = Schema.decodeUnknownResult(syncPayloadSchema)(payload)
+        if (Result.isFailure(decodedResult) === true) {
+          const message = decodedResult.failure.toString()
+          console.error('Invalid payload (decode failed)', message)
+          return new Response(message, { status: 400, ...(headers !== undefined ? { headers } : {}) })
+        }
+
+        const result = yield* Effect.promise(async () =>
+          validatePayload(decodedResult.success, { storeId, headers: requestHeaders }),
+        ).pipe(UnknownError.mapToUnknownError, Effect.result)
+
+        if (Result.isFailure(result) === true) {
+          console.error('Invalid payload (validation failed)', result.failure)
+          return new Response(result.failure.toString(), { status: 400, ...(headers !== undefined ? { headers } : {}) })
+        }
+      } else {
+        const result = yield* Effect.promise(async () =>
+          validatePayload(payload as TSyncPayload, { storeId, headers: requestHeaders }),
+        ).pipe(UnknownError.mapToUnknownError, Effect.result)
+
+        if (Result.isFailure(result) === true) {
+          console.error('Invalid payload (validation failed)', result.failure)
+          return new Response(result.failure.toString(), { status: 400, ...(headers !== undefined ? { headers } : {}) })
+        }
       }
     }
 
+    const env = explicitlyProvidedEnv ?? (importedEnv as TEnv)
+
     if (!(syncBackendBinding in env)) {
       return new Response(
         `Failed dependency: Required Durable Object binding '${syncBackendBinding as string}' not available`,
         {
           status: 424,
-          headers,
+          ...(headers !== undefined ? { headers } : {}),
         },
       )
     }
@@ -199,7 +251,7 @@ export const handleSyncRequest = <
     if (transport === 'ws' && (upgradeHeader === null || upgradeHeader !== 'websocket')) {
       return new Response('Durable Object expected Upgrade: websocket', {
         status: 426,
-        headers,
+        ...(headers !== undefined ? { headers } : {}),
       })
     }