@livestore/sync-s2 0.0.0-snapshot-446f5de211c4578498f20693bd2998869be3e796

package/src/make-s2-url.ts

@@ -0,0 +1,23 @@
+import { shouldNeverHappen } from '@livestore/utils'
+import { Schema } from '@livestore/utils/effect'
+import * as ApiSchema from './api-schema.ts'
+
+export const makeS2StreamName = (storeId: string) => storeId.replace(/[^a-zA-Z0-9_-]/g, '-').slice(0, 100)
+
+/**
+ * Decode `args` from URLSearchParams using Effect Schema, mirroring Electric's approach.
+ */
+export const decodePullArgsFromSearchParams = (searchParams: URLSearchParams): typeof ApiSchema.PullArgs.Type => {
+  const UrlParamsSchema = Schema.Struct({ args: ApiSchema.ArgsSchema })
+  const argsResult = Schema.decodeUnknownEither(UrlParamsSchema)(Object.fromEntries(searchParams.entries()))
+
+  if (argsResult._tag === 'Left') {
+    return shouldNeverHappen(
+      'Invalid search params provided to decodePullArgsFromSearchParams',
+      searchParams,
+      Object.fromEntries(searchParams.entries()),
+    )
+  }
+
+  return argsResult.right.args
+}

package/src/mod.ts

@@ -0,0 +1,7 @@
+export * as ApiSchema from './api-schema.ts'
+export * as HttpClientGenerated from './http-client-generated.ts'
+export * from './make-s2-url.ts'
+export * from './s2-proxy-helpers.ts'
+export { makeSyncBackend, type SyncS2Options } from './sync-provider.ts'
+export type { S2SeqNum as S2SeqNumType, SyncMetadata as SyncMetadataType } from './types.ts'
+export { S2SeqNum, SyncMetadata, s2SeqNum } from './types.ts'

package/src/s2-proxy-helpers.ts

@@ -0,0 +1,196 @@
+/**
+ * Helper functions for implementing S2 API proxies.
+ * These utilities reduce duplication when building HTTP endpoints that bridge to S2.
+ */
+
+import type { LiveStoreEvent } from '@livestore/livestore'
+import type { PullArgs } from './api-schema.ts'
+import { makeS2StreamName } from './make-s2-url.ts'
+
+/** Configuration for S2 connections */
+export interface S2Config {
+  basin: string
+  token: string
+  /** @default 'https://aws.s2.dev/v1' */
+  accountBase?: string
+  /** @default 'https://{basin}.b.aws.s2.dev/v1' */
+  basinBase?: string
+}
+
+// URL construction helpers
+export const getBasinUrl = (config: S2Config, path: string): string => {
+  const base = config.basinBase ?? `https://${config.basin}.b.aws.s2.dev/v1`
+  return `${base}${path}`
+}
+
+export const getAccountUrl = (config: S2Config, path: string): string => {
+  const base = config.accountBase ?? 'https://aws.s2.dev/v1'
+  return `${base}${path}`
+}
+
+export const getStreamRecordsUrl = (
+  config: S2Config,
+  stream: string,
+  params?: { seq_num?: number; count?: number; clamp?: boolean },
+): string => {
+  const base = getBasinUrl(config, `/streams/${encodeURIComponent(stream)}/records`)
+  if (!params) return base
+
+  const searchParams = new URLSearchParams()
+  /** seq_num - The sequence number to start from. See: https://docs.s2.dev/api#seq_num */
+  if (params.seq_num !== undefined) searchParams.append('seq_num', params.seq_num.toString())
+  /** count - Maximum number of changes to return. See: https://docs.s2.dev/api#count */
+  if (params.count !== undefined) searchParams.append('count', params.count.toString())
+  /** clamp - Whether to clamp the response to the requested count. See: https://docs.s2.dev/api#clamp */
+  if (params.clamp !== undefined) searchParams.append('clamp', params.clamp.toString())
+
+  return searchParams.toString() ? `${base}?${searchParams}` : base
+}
+
+// Header helpers
+export const getAuthHeaders = (token: string): Record<string, string> => ({
+  Authorization: `Bearer ${token}`,
+})
+
+export const getSSEHeaders = (token: string): Record<string, string> => ({
+  ...getAuthHeaders(token),
+  accept: 'text/event-stream',
+  's2-format': 'raw',
+})
+
+export const getPushHeaders = (token: string): Record<string, string> => ({
+  ...getAuthHeaders(token),
+  'content-type': 'application/json',
+  's2-format': 'raw',
+})
+
+// S2 operation helpers
+export const ensureBasin = async (config: S2Config): Promise<void> => {
+  try {
+    await fetch(getAccountUrl(config, '/basins'), {
+      method: 'POST',
+      headers: {
+        ...getAuthHeaders(config.token),
+        'content-type': 'application/json',
+      },
+      body: JSON.stringify({ basin: config.basin }),
+    })
+  } catch {
+    // Ignore errors - basin might already exist
+  }
+}
+
+export const ensureStream = async (config: S2Config, stream: string): Promise<void> => {
+  try {
+    await fetch(getBasinUrl(config, '/streams'), {
+      method: 'POST',
+      headers: {
+        ...getAuthHeaders(config.token),
+        'content-type': 'application/json',
+      },
+      body: JSON.stringify({ stream }),
+    })
+  } catch {
+    // Ignore errors - stream might already exist
+  }
+}
+
+// Request construction helpers
+export const buildPullRequest = ({
+  config,
+  args,
+}: {
+  config: S2Config
+  args: PullArgs
+}): {
+  url: string
+  headers: Record<string, string>
+} => {
+  const streamName = makeS2StreamName(args.storeId)
+  // Convert cursor (last seen record) to seq_num (where to start reading)
+  // cursor points to last processed record, seq_num needs to be the next record
+  const seq_num = args.s2SeqNum === 'from-start' ? 0 : args.s2SeqNum + 1
+
+  if (args.live) {
+    const url = getStreamRecordsUrl(config, streamName, { seq_num, clamp: true })
+    return { url, headers: getSSEHeaders(config.token) }
+  } else {
+    // Non-live pulls also stream over SSE: we request a very large `count` and
+    // rely on S2 closing the stream once the page tail is reached. This keeps
+    // both live and paged reads on the same transport while still giving us a
+    // natural end-of-stream signal.
+    const url = getStreamRecordsUrl(config, streamName, { seq_num, count: Number.MAX_SAFE_INTEGER, clamp: true })
+    return { url, headers: getSSEHeaders(config.token) }
+  }
+}
+
+export const buildPushRequest = ({
+  config,
+  storeId,
+  batch,
+}: {
+  config: S2Config
+  storeId: string
+  batch: readonly LiveStoreEvent.AnyEncodedGlobal[]
+}): {
+  url: string
+  method: 'POST'
+  headers: Record<string, string>
+  /** JSON-encoded batch */
+  body: string
+} => {
+  const streamName = makeS2StreamName(storeId)
+  const url = getBasinUrl(config, `/streams/${encodeURIComponent(streamName)}/records`)
+  return {
+    url,
+    method: 'POST',
+    headers: getPushHeaders(config.token),
+    body: JSON.stringify(formatBatchForS2(batch)),
+  }
+}
+
+// Response helpers
+export const emptyBatchResponse = (): Response => {
+  return new Response(JSON.stringify({ records: [] }), {
+    headers: { 'content-type': 'application/json' },
+  })
+}
+
+export const sseKeepAliveResponse = (): Response => {
+  return new Response('event: ping\ndata: {}\n\n', {
+    status: 200,
+    headers: { 'content-type': 'text/event-stream', 'cache-control': 'no-cache' },
+  })
+}
+
+export const successResponse = (): Response => {
+  const body = { success: true }
+  return new Response(JSON.stringify(body), {
+    headers: { 'content-type': 'application/json' },
+  })
+}
+
+export const errorResponse = (message: string, status = 500): Response => {
+  return new Response(JSON.stringify({ error: message }), {
+    status,
+    headers: { 'content-type': 'application/json' },
+  })
+}
+
+// Batch formatting helper
+export const formatBatchForS2 = (
+  batch: readonly LiveStoreEvent.AnyEncodedGlobal[],
+): { records: { body: string }[] } => {
+  return {
+    records: batch.map((ev) => ({ body: JSON.stringify(ev) })),
+  }
+}
+
+export const asCurl = (request: { url: string; method: string; headers: Record<string, string>; body?: string }) => {
+  const url = request.url
+  const method = request.method
+  const headers = Object.entries(request.headers).map(([key, value]) => `-H "${key}: ${value}"`)
+  const body = request.body
+  const headersStr = headers.join(' ')
+  return `curl -X ${method} ${url} ${headersStr} ${body ? `-d '${body}'` : ''}`
+}

package/src/sync-provider.ts

@@ -0,0 +1,267 @@
+/**
+ * S2 Sync Provider — Client Overview
+ *
+ * Architecture
+ * - This package implements LiveStore's SyncBackend over a simple HTTP API that we call the "API proxy".
+ *   The proxy exposes three verbs compatible with LiveStore's sync contract:
+ *     - GET `/?args=...`   → pull pages of events
+ *     - POST `/`           → push a batch of events
+ *     - HEAD `/`           → ping (reachability)
+ * - In tests, the API proxy bridges to the hosted S2 service. The proxy remains focused on app logic; all
+ *   reusable logic (schemas, helpers, client behavior) is factored into this package.
+ *
+ * LiveStore → S2 mapping
+ * - storeId → S2 stream name (sanitized). Each store maps to one S2 stream.
+ * - LiveStore event (AnyEncodedGlobal) → S2 record body (string). We JSON-encode the event and set it as the
+ *   S2 record body. We currently do not rely on S2 record headers.
+ * - Sequence numbers → INDEPENDENT systems. LiveStore's seqNum (in event payload) tracks logical event ordering;
+ *   S2's seq_num tracks physical stream position. Both start at 0 but are decoupled by design to support future
+ *   optimizations like compaction. SyncMetadata tracks S2's position separately for cursor management.
+ * - Pull (cursor) → S2 `seq_num`. The cursor uses SyncMetadata.s2SeqNum for stream positioning, not LiveStore's seqNum.
+ * - Live pulling → S2 SSE tail. SSE streaming provides real-time event delivery without polling.
+ *
+ * S2 constraints & considerations
+ * - Append limits: respect S2 batch count/byte limits; add retries/backoff for 4xx/5xx.
+ * - Provisioning: basin / stream lifecycle is kept outside of the client (API proxy / service concern).
+ * - Formatting: when speaking to S2 directly, prefer `s2-format: raw` for JSON record bodies and ensure UTF‑8.
+ * - Sequence numbers: S2 assigns `seq_num`; LiveStore's event sequence is preserved inside the event payload.
+ *   DO NOT couple these systems together or assume 1:1 correspondence.
+ *
+ * Errors
+ * - push → InvalidPushError on non‑2xx; pull → InvalidPullError on non‑2xx; ping/connect map timeouts to offline.
+ * - The proxy should surface helpful status codes and error bodies.
+ */
+import { InvalidPullError, InvalidPushError, SyncBackend, UnexpectedError } from '@livestore/common'
+import type { EventSequenceNumber } from '@livestore/common/schema'
+import { shouldNeverHappen } from '@livestore/utils'
+import {
+  type Duration,
+  Effect,
+  HttpClient,
+  HttpClientRequest,
+  HttpClientResponse,
+  Option,
+  Schedule,
+  Schema,
+  Sse,
+  Stream,
+  SubscriptionRef,
+} from '@livestore/utils/effect'
+import * as ApiSchema from './api-schema.ts'
+import { decodeReadBatch } from './decode.ts'
+import * as HttpClientGenerated from './http-client-generated.ts'
+import type { SyncMetadata } from './types.ts'
+
+export interface SyncS2Options {
+  endpoint:
+    | string
+    | {
+        push: string
+        pull: string
+        ping: string
+      }
+  ping?: {
+    /** Enable periodic ping; default true */
+    enabled?: boolean
+    /** Timeout for individual ping request; default 10s */
+    requestTimeout?: Duration.DurationInput
+    /** Interval between ping requests; default 10s */
+    requestInterval?: Duration.DurationInput
+  }
+  retry?: {
+    /** Custom retry schedule for non-live pulls (default: 2 recurs, 100ms spaced) */
+    pull?: Schedule.Schedule<number, InvalidPullError>
+    /** Custom retry schedule for pushes (default: 2 recurs, 100ms spaced) */
+    push?: Schedule.Schedule<number, InvalidPushError>
+  }
+}
+
+export const defaultRetry = Schedule.compose(Schedule.recurs(2), Schedule.spaced(100))
+
+export const makeSyncBackend =
+  ({ endpoint, ping: pingOptions, retry }: SyncS2Options): SyncBackend.SyncBackendConstructor<SyncMetadata> =>
+  ({ storeId, payload }) =>
+    Effect.gen(function* () {
+      const isConnected = yield* SubscriptionRef.make(false)
+      const pullEndpoint = typeof endpoint === 'string' ? endpoint : endpoint.pull
+      const pushEndpoint = typeof endpoint === 'string' ? endpoint : endpoint.push
+      const pingEndpoint = typeof endpoint === 'string' ? endpoint : endpoint.ping
+
+      const httpClient = yield* HttpClient.HttpClient
+
+      const pullEndpointHasSameOrigin =
+        pullEndpoint.startsWith('/') ||
+        (globalThis.location !== undefined && globalThis.location.origin === new URL(pullEndpoint).origin)
+
+      const pingTimeout = pingOptions?.requestTimeout ?? 10_000
+
+      const ping: SyncBackend.SyncBackend<SyncMetadata>['ping'] = Effect.gen(function* () {
+        yield* httpClient.pipe(HttpClient.filterStatusOk).head(pingEndpoint)
+        yield* SubscriptionRef.set(isConnected, true)
+      }).pipe(
+        UnexpectedError.mapToUnexpectedError,
+        Effect.timeout(pingTimeout),
+        Effect.catchTag('TimeoutException', () => SubscriptionRef.set(isConnected, false)),
+      )
+
+      const pingInterval = pingOptions?.requestInterval ?? 10_000
+      if (pingOptions?.enabled !== false) {
+        yield* ping.pipe(Effect.repeat(Schedule.spaced(pingInterval)), Effect.tapCauseLogPretty, Effect.forkScoped)
+      }
+
+      // No need to connect if the pull endpoint has the same origin as the current page
+      const connect: SyncBackend.SyncBackend<SyncMetadata>['connect'] = pullEndpointHasSameOrigin
+        ? Effect.void
+        : ping.pipe(UnexpectedError.mapToUnexpectedError)
+
+      const runPullSse = (
+        cursor: Option.Option<{
+          eventSequenceNumber: EventSequenceNumber.GlobalEventSequenceNumber
+          metadata: Option.Option<SyncMetadata>
+        }>,
+        live: boolean,
+      ): Stream.Stream<SyncBackend.PullResItem<SyncMetadata>, InvalidPullError> => {
+        // Extract S2 seqNum from metadata for SSE cursor
+        const s2SeqNum = cursor.pipe(
+          Option.flatMap((_) => _.metadata),
+          Option.map((_) => _.s2SeqNum),
+          Option.getOrElse(() => 'from-start' as const),
+        )
+
+        const argsJson = Schema.encodeSync(ApiSchema.ArgsSchema)({ storeId, payload, s2SeqNum, live })
+        const url = `${pullEndpoint}?args=${argsJson}`
+
+        return httpClient
+          .execute(HttpClientRequest.get(url).pipe(HttpClientRequest.setHeaders({ accept: 'text/event-stream' })))
+          .pipe(
+            HttpClientResponse.stream,
+            // decode text and split into lines
+            Stream.decodeText('utf8'),
+            Stream.pipeThroughChannel(Sse.makeChannel()),
+            // Filter out pings, map errors to stream failures
+            Stream.mapEffect(
+              Effect.fnUntraced(function* (msg) {
+                const evt = msg.event.toLowerCase()
+                if (evt === 'ping') return Option.none()
+                if (evt === 'error') {
+                  return yield* new InvalidPullError({ cause: new Error(`SSE error: ${msg.data}`) })
+                }
+                if (evt === 'batch') {
+                  const readBatch = yield* Schema.decode(Schema.parseJson(HttpClientGenerated.ReadBatch))(msg.data)
+                  const batch = decodeReadBatch(readBatch)
+
+                  const lastSeqNum = batch.at(-1)?.eventEncoded.seqNum
+                  const tailSeqNum = readBatch.tail?.seq_num
+                  const remaining =
+                    lastSeqNum !== undefined && tailSeqNum !== undefined ? tailSeqNum - lastSeqNum : undefined
+
+                  return Option.some({
+                    batch,
+                    pageInfo:
+                      remaining !== undefined && remaining > 0
+                        ? SyncBackend.pageInfoMoreKnown(remaining)
+                        : SyncBackend.pageInfoNoMore,
+                  })
+                }
+                // emitted when reached the end of the stream
+                if (evt === 'message' && msg.data === '[DONE]') {
+                  return Option.none()
+                }
+                return shouldNeverHappen(`Unexpected SSE event: ${evt}`, msg)
+              }),
+            ),
+            Stream.filterMap((_) => _), // filter out Option.none()
+            Stream.mapError((cause) => (cause._tag === 'InvalidPullError' ? cause : new InvalidPullError({ cause }))),
+            Stream.retry(retry?.pull ?? defaultRetry),
+          )
+      }
+
+      const ssePull = (
+        startCursor: Option.Option<{
+          eventSequenceNumber: EventSequenceNumber.GlobalEventSequenceNumber
+          metadata: Option.Option<SyncMetadata>
+        }>,
+      ): Stream.Stream<SyncBackend.PullResItem<SyncMetadata>, InvalidPullError> => {
+        const computeNextCursor = (
+          lastItem: Option.Option<SyncBackend.PullResItem<SyncMetadata>>,
+          current: Option.Option<{
+            eventSequenceNumber: EventSequenceNumber.GlobalEventSequenceNumber
+            metadata: Option.Option<SyncMetadata>
+          }>,
+        ) =>
+          lastItem.pipe(
+            Option.flatMap((item) => {
+              const lastBatchItem = item.batch.at(-1)
+              if (!lastBatchItem) return Option.none()
+              return Option.some({
+                eventSequenceNumber: lastBatchItem.eventEncoded.seqNum,
+                metadata: lastBatchItem.metadata,
+              })
+            }),
+            Option.orElse(() => current),
+          )
+
+        const loop = (
+          cursor: Option.Option<{
+            eventSequenceNumber: EventSequenceNumber.GlobalEventSequenceNumber
+            metadata: Option.Option<SyncMetadata>
+          }>,
+          isFirst: boolean,
+        ): Stream.Stream<SyncBackend.PullResItem<SyncMetadata>, InvalidPullError> => {
+          const sseStream = (live: boolean) =>
+            runPullSse(cursor, live).pipe(
+              Stream.emitIfEmpty({
+                batch: [],
+                pageInfo: SyncBackend.pageInfoNoMore,
+              } as SyncBackend.PullResItem<SyncMetadata>),
+            )
+
+          const stream = isFirst ? sseStream(false) : sseStream(true)
+
+          return stream.pipe(
+            // Reconnect from last item if stream
+            Stream.concatWithLastElement((lastItem) => loop(computeNextCursor(lastItem, cursor), false)),
+          )
+        }
+
+        return loop(startCursor, true)
+      }
+
+      return SyncBackend.of({
+        connect,
+        pull: (cursor, options) => {
+          if (options?.live) {
+            return ssePull(cursor)
+          } else {
+            return runPullSse(cursor, false).pipe(
+              Stream.emitIfEmpty({
+                batch: [],
+                pageInfo: SyncBackend.pageInfoNoMore,
+              } as SyncBackend.PullResItem<SyncMetadata>),
+            )
+          }
+        },
+        push: (batch) =>
+          HttpClientRequest.schemaBodyJson(ApiSchema.PushPayload)(HttpClientRequest.post(pushEndpoint), {
+            storeId,
+            batch,
+          }).pipe(
+            Effect.andThen(httpClient.pipe(HttpClient.filterStatusOk).execute),
+            Effect.andThen(HttpClientResponse.schemaBodyJson(ApiSchema.PushResponse)),
+            Effect.mapError((cause) => InvalidPushError.make({ cause: UnexpectedError.make({ cause }) })),
+            Effect.retry(retry?.push ?? defaultRetry),
+          ),
+        ping,
+        isConnected,
+        metadata: {
+          name: '@livestore/sync-s2',
+          description: 'LiveStore sync backend implementation for S2',
+          protocol: 'http',
+          endpoint,
+        },
+        supports: {
+          pullPageInfoKnown: false,
+          pullLive: true,
+        },
+      })
+    })

package/src/types.ts

@@ -0,0 +1,26 @@
+import { Brand, Schema } from '@livestore/utils/effect'
+
+/**
+ * Branded type for S2's sequence numbers to ensure type safety
+ * and prevent accidental mixing with LiveStore's sequence numbers.
+ *
+ * S2 sequence numbers:
+ * - Start at 0 for the first record in a stream
+ * - Are assigned sequentially by S2 for each appended record
+ * - Are used for physical stream positioning (reading from a specific point)
+ * - Are completely independent from LiveStore's logical event sequence numbers
+ */
+export type S2SeqNum = Brand.Branded<number, 'S2SeqNum'>
+export const s2SeqNum = Brand.nominal<S2SeqNum>()
+export const S2SeqNum = Schema.fromBrand(s2SeqNum)(Schema.Int.pipe(Schema.greaterThanOrEqualTo(0)))
+
+/**
+ * Metadata for tracking S2-specific cursor information.
+ * This is separate from LiveStore's event sequence numbers to maintain
+ * proper abstraction boundaries between storage and application logic.
+ */
+export const SyncMetadata = Schema.Struct({
+  /** S2's seq_num for stream positioning */
+  s2SeqNum: S2SeqNum,
+})
+export type SyncMetadata = typeof SyncMetadata.Type