@livestore/sync-s2 0.4.0-dev.8 → 0.4.0

package/src/s2-proxy-helpers.ts

@@ -4,7 +4,9 @@
  */
 
 import type { LiveStoreEvent } from '@livestore/livestore'
+
 import type { PullArgs } from './api-schema.ts'
+import { chunkEventsForS2 } from './limits.ts'
 import { makeS2StreamName } from './make-s2-url.ts'
 
 /** Configuration for S2 connections */
@@ -15,8 +17,20 @@ export interface S2Config {
   accountBase?: string
   /** @default 'https://{basin}.b.aws.s2.dev/v1' */
   basinBase?: string
+  /**
+   * When true, adds `S2-Basin` header to requests. This is required for s2-lite
+   * (the open-source self-hosted S2) which uses header-based basin routing instead
+   * of subdomain-based routing used by hosted S2.
+   * @see https://github.com/s2-streamstore/s2-lite
+   */
+  lite?: boolean
 }
 
+export const isLiteMode = (config: S2Config): boolean => config.lite === true
+
+const getBasinHeader = (config: S2Config): Record<string, string> =>
+  isLiteMode(config) === true ? { 's2-basin': config.basin } : {}
+
 // URL construction helpers
 export const getBasinUrl = (config: S2Config, path: string): string => {
   const base = config.basinBase ?? `https://${config.basin}.b.aws.s2.dev/v1`
@@ -31,10 +45,10 @@ export const getAccountUrl = (config: S2Config, path: string): string => {
 export const getStreamRecordsUrl = (
   config: S2Config,
   stream: string,
-  params?: { seq_num?: number; count?: number; clamp?: boolean },
+  params?: { seq_num?: number; count?: number; clamp?: boolean; wait?: number },
 ): string => {
   const base = getBasinUrl(config, `/streams/${encodeURIComponent(stream)}/records`)
-  if (!params) return base
+  if (params == null) return base
 
   const searchParams = new URLSearchParams()
   /** seq_num - The sequence number to start from. See: https://docs.s2.dev/api#seq_num */
@@ -43,8 +57,11 @@ export const getStreamRecordsUrl = (
   if (params.count !== undefined) searchParams.append('count', params.count.toString())
   /** clamp - Whether to clamp the response to the requested count. See: https://docs.s2.dev/api#clamp */
   if (params.clamp !== undefined) searchParams.append('clamp', params.clamp.toString())
+  /** wait - How long to wait for new records before returning. See: https://docs.s2.dev/api#wait */
+  if (params.wait !== undefined) searchParams.append('wait', params.wait.toString())
 
-  return searchParams.toString() ? `${base}?${searchParams}` : base
+  const searchParamsString = searchParams.toString()
+  return searchParamsString.length > 0 ? `${base}?${searchParams}` : base
 }
 
 // Header helpers
@@ -52,14 +69,16 @@ export const getAuthHeaders = (token: string): Record<string, string> => ({
   Authorization: `Bearer ${token}`,
 })
 
-export const getSSEHeaders = (token: string): Record<string, string> => ({
-  ...getAuthHeaders(token),
+export const getSSEHeaders = (config: S2Config): Record<string, string> => ({
+  ...getAuthHeaders(config.token),
+  ...getBasinHeader(config),
   accept: 'text/event-stream',
   's2-format': 'raw',
 })
 
-export const getPushHeaders = (token: string): Record<string, string> => ({
-  ...getAuthHeaders(token),
+export const getPushHeaders = (config: S2Config): Record<string, string> => ({
+  ...getAuthHeaders(config.token),
+  ...getBasinHeader(config),
   'content-type': 'application/json',
   's2-format': 'raw',
 })
@@ -86,6 +105,7 @@ export const ensureStream = async (config: S2Config, stream: string): Promise<vo
       method: 'POST',
       headers: {
         ...getAuthHeaders(config.token),
+        ...getBasinHeader(config),
         'content-type': 'application/json',
       },
       body: JSON.stringify({ stream }),
@@ -111,42 +131,50 @@ export const buildPullRequest = ({
   // cursor points to last processed record, seq_num needs to be the next record
   const seq_num = args.s2SeqNum === 'from-start' ? 0 : args.s2SeqNum + 1
 
-  if (args.live) {
+  if (args.live === true) {
     const url = getStreamRecordsUrl(config, streamName, { seq_num, clamp: true })
-    return { url, headers: getSSEHeaders(config.token) }
+    return { url, headers: getSSEHeaders(config) }
   } else {
-    // Non-live pulls also stream over SSE: we request a very large `count` and
-    // rely on S2 closing the stream once the page tail is reached. This keeps
-    // both live and paged reads on the same transport while still giving us a
-    // natural end-of-stream signal.
-    const url = getStreamRecordsUrl(config, streamName, { seq_num, count: Number.MAX_SAFE_INTEGER, clamp: true })
-    return { url, headers: getSSEHeaders(config.token) }
+    // Non-live pulls also stream over SSE. We ask S2 to return immediately when
+    // the tail is reached by setting wait=0 which gives us an explicit
+    // end-of-stream without requesting an arbitrarily large page size.
+    const url = getStreamRecordsUrl(config, streamName, { seq_num, wait: 0, clamp: true })
+    return { url, headers: getSSEHeaders(config) }
   }
 }
 
-export const buildPushRequest = ({
+export interface S2PushRequest {
+  readonly url: string
+  readonly method: 'POST'
+  readonly headers: Record<string, string>
+  readonly body: string
+}
+
+/**
+ * Builds one or more append requests against S2. The helper applies the
+ * documented 1 MiB / 1000-record limits via `chunkEventsForS2`, so callers
+ * receive a request per compliant chunk instead of hitting 413 responses at
+ * runtime.
+ */
+export const buildPushRequests = ({
   config,
   storeId,
   batch,
 }: {
   config: S2Config
   storeId: string
-  batch: readonly LiveStoreEvent.AnyEncodedGlobal[]
-}): {
-  url: string
-  method: 'POST'
-  headers: Record<string, string>
-  /** JSON-encoded batch */
-  body: string
-} => {
+  batch: readonly LiveStoreEvent.Global.Encoded[]
+}): ReadonlyArray<S2PushRequest> => {
   const streamName = makeS2StreamName(storeId)
   const url = getBasinUrl(config, `/streams/${encodeURIComponent(streamName)}/records`)
-  return {
+  const chunks = chunkEventsForS2(batch)
+
+  return chunks.map((chunk) => ({
     url,
-    method: 'POST',
-    headers: getPushHeaders(config.token),
-    body: JSON.stringify(formatBatchForS2(batch)),
-  }
+    method: 'POST' as const,
+    headers: getPushHeaders(config),
+    body: JSON.stringify({ records: chunk.records }),
+  }))
 }
 
 // Response helpers
@@ -176,21 +204,3 @@ export const errorResponse = (message: string, status = 500): Response => {
     headers: { 'content-type': 'application/json' },
   })
 }
-
-// Batch formatting helper
-export const formatBatchForS2 = (
-  batch: readonly LiveStoreEvent.AnyEncodedGlobal[],
-): { records: { body: string }[] } => {
-  return {
-    records: batch.map((ev) => ({ body: JSON.stringify(ev) })),
-  }
-}
-
-export const asCurl = (request: { url: string; method: string; headers: Record<string, string>; body?: string }) => {
-  const url = request.url
-  const method = request.method
-  const headers = Object.entries(request.headers).map(([key, value]) => `-H "${key}: ${value}"`)
-  const body = request.body
-  const headersStr = headers.join(' ')
-  return `curl -X ${method} ${url} ${headersStr} ${body ? `-d '${body}'` : ''}`
-}

package/src/sync-provider.ts

@@ -28,10 +28,10 @@
  *   DO NOT couple these systems together or assume 1:1 correspondence.
  *
  * Errors
- * - push → InvalidPushError on non‑2xx; pull → InvalidPullError on non‑2xx; ping/connect map timeouts to offline.
+ * - push → UnknownError on non‑2xx; pull → UnknownError on non‑2xx; ping/connect map timeouts to offline.
  * - The proxy should surface helpful status codes and error bodies.
  */
-import { InvalidPullError, InvalidPushError, SyncBackend, UnexpectedError } from '@livestore/common'
+import { SyncBackend, UnknownError } from '@livestore/common'
 import type { EventSequenceNumber } from '@livestore/common/schema'
 import { shouldNeverHappen } from '@livestore/utils'
 import {
@@ -47,9 +47,11 @@ import {
   Stream,
   SubscriptionRef,
 } from '@livestore/utils/effect'
+
 import * as ApiSchema from './api-schema.ts'
 import { decodeReadBatch } from './decode.ts'
 import * as HttpClientGenerated from './http-client-generated.ts'
+import { chunkEventsForS2, S2LimitExceededError } from './limits.ts'
 import type { SyncMetadata } from './types.ts'
 
 export interface SyncS2Options {
@@ -70,14 +72,26 @@ export interface SyncS2Options {
   }
   retry?: {
     /** Custom retry schedule for non-live pulls (default: 2 recurs, 100ms spaced) */
-    pull?: Schedule.Schedule<number, InvalidPullError>
+    pull?: Schedule.Schedule<number, UnknownError>
     /** Custom retry schedule for pushes (default: 2 recurs, 100ms spaced) */
-    push?: Schedule.Schedule<number, InvalidPushError>
+    push?: Schedule.Schedule<number, UnknownError>
   }
 }
 
 export const defaultRetry = Schedule.compose(Schedule.recurs(2), Schedule.spaced(100))
 
+const getBrowserOrigin = () => {
+  if (typeof globalThis !== 'object' || globalThis === null || !('location' in globalThis)) {
+    return undefined
+  }
+
+  const { location } = globalThis as typeof globalThis & {
+    location?: { origin?: unknown } | undefined
+  }
+
+  return typeof location?.origin === 'string' ? location.origin : undefined
+}
+
 export const makeSyncBackend =
   ({ endpoint, ping: pingOptions, retry }: SyncS2Options): SyncBackend.SyncBackendConstructor<SyncMetadata> =>
   ({ storeId, payload }) =>
@@ -89,9 +103,9 @@ export const makeSyncBackend =
 
       const httpClient = yield* HttpClient.HttpClient
 
+      const browserOrigin = getBrowserOrigin()
       const pullEndpointHasSameOrigin =
-        pullEndpoint.startsWith('/') ||
-        (globalThis.location !== undefined && globalThis.location.origin === new URL(pullEndpoint).origin)
+        pullEndpoint.startsWith('/') || (browserOrigin !== undefined && browserOrigin === new URL(pullEndpoint).origin)
 
       const pingTimeout = pingOptions?.requestTimeout ?? 10_000
 
@@ -99,7 +113,7 @@ export const makeSyncBackend =
         yield* httpClient.pipe(HttpClient.filterStatusOk).head(pingEndpoint)
         yield* SubscriptionRef.set(isConnected, true)
       }).pipe(
-        UnexpectedError.mapToUnexpectedError,
+        UnknownError.mapToUnknownError,
         Effect.timeout(pingTimeout),
         Effect.catchTag('TimeoutException', () => SubscriptionRef.set(isConnected, false)),
       )
@@ -110,17 +124,16 @@ export const makeSyncBackend =
       }
 
       // No need to connect if the pull endpoint has the same origin as the current page
-      const connect: SyncBackend.SyncBackend<SyncMetadata>['connect'] = pullEndpointHasSameOrigin
-        ? Effect.void
-        : ping.pipe(UnexpectedError.mapToUnexpectedError)
+      const connect: SyncBackend.SyncBackend<SyncMetadata>['connect'] =
+        pullEndpointHasSameOrigin === true ? Effect.void : ping.pipe(UnknownError.mapToUnknownError)
 
       const runPullSse = (
         cursor: Option.Option<{
-          eventSequenceNumber: EventSequenceNumber.GlobalEventSequenceNumber
+          eventSequenceNumber: EventSequenceNumber.Global.Type
           metadata: Option.Option<SyncMetadata>
         }>,
         live: boolean,
-      ): Stream.Stream<SyncBackend.PullResItem<SyncMetadata>, InvalidPullError> => {
+      ): Stream.Stream<SyncBackend.PullResItem<SyncMetadata>, UnknownError> => {
         // Extract S2 seqNum from metadata for SSE cursor
         const s2SeqNum = cursor.pipe(
           Option.flatMap((_) => _.metadata),
@@ -144,7 +157,7 @@ export const makeSyncBackend =
                 const evt = msg.event.toLowerCase()
                 if (evt === 'ping') return Option.none()
                 if (evt === 'error') {
-                  return yield* new InvalidPullError({ cause: new Error(`SSE error: ${msg.data}`) })
+                  return yield* new UnknownError({ cause: new Error(`SSE error: ${msg.data}`) })
                 }
                 if (evt === 'batch') {
                   const readBatch = yield* Schema.decode(Schema.parseJson(HttpClientGenerated.ReadBatch))(msg.data)
@@ -176,28 +189,30 @@ export const makeSyncBackend =
               }),
             ),
             Stream.filterMap((_) => _), // filter out Option.none()
-            Stream.mapError((cause) => (cause._tag === 'InvalidPullError' ? cause : new InvalidPullError({ cause }))),
+            Stream.mapError((cause) =>
+              cause._tag === 'UnknownError' ? cause : new UnknownError({ cause }),
+            ),
             Stream.retry(retry?.pull ?? defaultRetry),
           )
       }
 
       const ssePull = (
         startCursor: Option.Option<{
-          eventSequenceNumber: EventSequenceNumber.GlobalEventSequenceNumber
+          eventSequenceNumber: EventSequenceNumber.Global.Type
           metadata: Option.Option<SyncMetadata>
         }>,
-      ): Stream.Stream<SyncBackend.PullResItem<SyncMetadata>, InvalidPullError> => {
+      ): Stream.Stream<SyncBackend.PullResItem<SyncMetadata>, UnknownError> => {
         const computeNextCursor = (
           lastItem: Option.Option<SyncBackend.PullResItem<SyncMetadata>>,
           current: Option.Option<{
-            eventSequenceNumber: EventSequenceNumber.GlobalEventSequenceNumber
+            eventSequenceNumber: EventSequenceNumber.Global.Type
             metadata: Option.Option<SyncMetadata>
           }>,
         ) =>
           lastItem.pipe(
             Option.flatMap((item) => {
               const lastBatchItem = item.batch.at(-1)
-              if (!lastBatchItem) return Option.none()
+              if (lastBatchItem == null) return Option.none()
               return Option.some({
                 eventSequenceNumber: lastBatchItem.eventEncoded.seqNum,
                 metadata: lastBatchItem.metadata,
@@ -208,11 +223,11 @@ export const makeSyncBackend =
 
         const loop = (
           cursor: Option.Option<{
-            eventSequenceNumber: EventSequenceNumber.GlobalEventSequenceNumber
+            eventSequenceNumber: EventSequenceNumber.Global.Type
             metadata: Option.Option<SyncMetadata>
           }>,
           isFirst: boolean,
-        ): Stream.Stream<SyncBackend.PullResItem<SyncMetadata>, InvalidPullError> => {
+        ): Stream.Stream<SyncBackend.PullResItem<SyncMetadata>, UnknownError> => {
           const sseStream = (live: boolean) =>
             runPullSse(cursor, live).pipe(
               Stream.emitIfEmpty({
@@ -221,7 +236,7 @@ export const makeSyncBackend =
               } as SyncBackend.PullResItem<SyncMetadata>),
             )
 
-          const stream = isFirst ? sseStream(false) : sseStream(true)
+          const stream = isFirst === true ? sseStream(false) : sseStream(true)
 
           return stream.pipe(
             // Reconnect from last item if stream
@@ -235,7 +250,7 @@ export const makeSyncBackend =
       return SyncBackend.of({
         connect,
         pull: (cursor, options) => {
-          if (options?.live) {
+          if (options?.live === true) {
             return ssePull(cursor)
           } else {
             return runPullSse(cursor, false).pipe(
@@ -247,15 +262,47 @@ export const makeSyncBackend =
           }
         },
         push: (batch) =>
-          HttpClientRequest.schemaBodyJson(ApiSchema.PushPayload)(HttpClientRequest.post(pushEndpoint), {
-            storeId,
-            batch,
-          }).pipe(
-            Effect.andThen(httpClient.pipe(HttpClient.filterStatusOk).execute),
-            Effect.andThen(HttpClientResponse.schemaBodyJson(ApiSchema.PushResponse)),
-            Effect.mapError((cause) => InvalidPushError.make({ cause: UnexpectedError.make({ cause }) })),
-            Effect.retry(retry?.push ?? defaultRetry),
-          ),
+          Effect.gen(function* () {
+            const toUnknownError = (cause: unknown): UnknownError => {
+              if (cause instanceof UnknownError) {
+                return cause
+              }
+
+              if (cause instanceof S2LimitExceededError) {
+                const note =
+                  cause.limitType === 'record-metered-bytes'
+                    ? `S2 record exceeded ${cause.max} metered bytes (actual: ${cause.actual})`
+                    : `S2 batch exceeded ${cause.max} (type: ${cause.limitType}, actual: ${cause.actual})`
+
+                return new UnknownError({
+                  cause,
+                  note,
+                  payload: {
+                    limitType: cause.limitType,
+                    max: cause.max,
+                    actual: cause.actual,
+                    recordIndex: cause.recordIndex,
+                  },
+                })
+              }
+
+              return new UnknownError({ cause })
+            }
+
+            const chunks = yield* Effect.sync(() => chunkEventsForS2(batch)).pipe(Effect.mapError(toUnknownError))
+
+            for (const chunk of chunks) {
+              yield* HttpClientRequest.schemaBodyJson(ApiSchema.PushPayload)(HttpClientRequest.post(pushEndpoint), {
+                storeId,
+                batch: chunk.events,
+              }).pipe(
+                Effect.andThen(httpClient.pipe(HttpClient.filterStatusOk).execute),
+                Effect.andThen(HttpClientResponse.schemaBodyJson(ApiSchema.PushResponse)),
+                Effect.mapError(toUnknownError),
+                Effect.retry(retry?.push ?? defaultRetry),
+              )
+            }
+          }),
         ping,
         isConnected,
         metadata: {