@livestore/sync-s2 0.0.0-snapshot-39b490a4d9054515b0012244297c45505059cf72 → 0.0.0-snapshot-4b42ef6ef0c52dda5b8633a10addf45134e4cafe

package/src/limits.ts

@@ -0,0 +1,135 @@
+import type { LiveStoreEvent } from '@livestore/common/schema'
+import { splitChunkBySize } from '@livestore/common/sync'
+import { Chunk, Effect, Schema } from '@livestore/utils/effect'
+
+const textEncoder = new TextEncoder()
+
+/**
+ * Maximum metered size of a single record (docs: https://s2.dev/docs/limits#records).
+ */
+export const MAX_RECORD_METERED_BYTES = 1_048_576 // 1 MiB
+
+/**
+ * Maximum combined metered size of a batch append (docs: https://s2.dev/docs/limits#records).
+ */
+export const MAX_BATCH_METERED_BYTES = 1_048_576 // 1 MiB
+
+/**
+ * Maximum number of records per append (docs: https://s2.dev/docs/limits#records).
+ */
+export const MAX_RECORDS_PER_BATCH = 1_000
+
+const LimitType = Schema.Literal('record-metered-bytes', 'batch-metered-bytes', 'batch-count')
+
+export class S2LimitExceededError extends Schema.TaggedError<S2LimitExceededError>()('S2LimitExceededError', {
+  limitType: LimitType,
+  max: Schema.Number,
+  actual: Schema.Number,
+  recordIndex: Schema.optional(Schema.Number),
+}) {}
+
+export interface AppendRecordBody {
+  readonly body?: string
+  readonly headers?: ReadonlyArray<{ readonly name: string; readonly value: string }>
+}
+
+// S2 measures bodies/headers in UTF‑8 bytes; centralising this helper keeps the
+// formula readable and consistent with the docs.
+const utf8ByteLength = (value: string): number => textEncoder.encode(value).byteLength
+
+export const computeRecordMeteredBytes = (record: AppendRecordBody): number => {
+  const headers = record.headers ?? []
+  const headerCount = headers.length
+  const headerBytes = headers.reduce(
+    (acc, header) => acc + utf8ByteLength(header.name) + utf8ByteLength(header.value),
+    0,
+  )
+  const bodyBytes = record.body === undefined ? 0 : utf8ByteLength(record.body)
+  return 8 + 2 * headerCount + headerBytes + bodyBytes
+}
+
+export const computeBatchMeteredBytes = (records: ReadonlyArray<AppendRecordBody>): number =>
+  records.reduce((acc, record) => acc + computeRecordMeteredBytes(record), 0)
+
+interface PreparedEvent {
+  readonly event: LiveStoreEvent.AnyEncodedGlobal
+  readonly record: AppendRecordBody
+  readonly meteredBytes: number
+  readonly index: number
+}
+
+export interface S2Chunk {
+  readonly events: ReadonlyArray<LiveStoreEvent.AnyEncodedGlobal>
+  readonly records: ReadonlyArray<AppendRecordBody>
+  readonly meteredBytes: number
+}
+
+// Pre-stringify events and pre-compute per-record metered bytes so we only pay
+// the JSON cost once when chunking large batches.
+const convertEventsToPrepared = (events: ReadonlyArray<LiveStoreEvent.AnyEncodedGlobal>): PreparedEvent[] =>
+  events.map((event, index) => {
+    const body = JSON.stringify(event)
+    const record: AppendRecordBody = { body }
+    const meteredBytes = computeRecordMeteredBytes(record)
+
+    if (meteredBytes > MAX_RECORD_METERED_BYTES) {
+      throw new S2LimitExceededError({
+        limitType: 'record-metered-bytes',
+        max: MAX_RECORD_METERED_BYTES,
+        actual: meteredBytes,
+        recordIndex: index,
+      })
+    }
+
+    return { event, record, meteredBytes, index }
+  })
+
+// Summarises a chunk’s metered bytes. Passed to splitChunkBySize so we enforce
+// S2 limits directly instead of relying on JSON size heuristics.
+const makeChunkMeasure = (items: ReadonlyArray<PreparedEvent>): number =>
+  items.reduce((acc, item) => acc + item.meteredBytes, 0)
+
+const mapPreparedChunks = (chunks: Chunk.Chunk<Chunk.Chunk<PreparedEvent>>): ReadonlyArray<S2Chunk> =>
+  Chunk.toReadonlyArray(chunks).map((chunk) => {
+    const chunkItems = Chunk.toReadonlyArray(chunk)
+    const events = chunkItems.map((item) => item.event)
+    const records = chunkItems.map((item) => item.record)
+    return {
+      events,
+      records,
+      meteredBytes: makeChunkMeasure(chunkItems),
+    }
+  })
+
+export const chunkEventsForS2 = (events: ReadonlyArray<LiveStoreEvent.AnyEncodedGlobal>): ReadonlyArray<S2Chunk> => {
+  if (events.length === 0) {
+    return []
+  }
+
+  const prepared = convertEventsToPrepared(events)
+
+  try {
+    const chunks = Chunk.fromIterable(prepared).pipe(
+      splitChunkBySize({
+        maxItems: MAX_RECORDS_PER_BATCH,
+        maxBytes: MAX_BATCH_METERED_BYTES,
+        encode: (items) => ({ records: items.map((item) => item.record) }),
+        measure: makeChunkMeasure,
+      }),
+      Effect.runSync,
+    )
+
+    return mapPreparedChunks(chunks)
+  } catch (error) {
+    if (error && typeof error === 'object' && (error as any)._tag === 'OversizeChunkItemError') {
+      const oversize = error as { size: number; maxBytes: number; _tag: string }
+      throw new S2LimitExceededError({
+        limitType: 'record-metered-bytes',
+        max: oversize.maxBytes,
+        actual: oversize.size,
+      })
+    }
+
+    throw error
+  }
+}

package/src/mod.ts

@@ -1,5 +1,6 @@
 export * as ApiSchema from './api-schema.ts'
 export * as HttpClientGenerated from './http-client-generated.ts'
+export * from './limits.ts'
 export * from './make-s2-url.ts'
 export * from './s2-proxy-helpers.ts'
 export { makeSyncBackend, type SyncS2Options } from './sync-provider.ts'

package/src/s2-proxy-helpers.ts

@@ -5,6 +5,7 @@
 
 import type { LiveStoreEvent } from '@livestore/livestore'
 import type { PullArgs } from './api-schema.ts'
+import { chunkEventsForS2 } from './limits.ts'
 import { makeS2StreamName } from './make-s2-url.ts'
 
 /** Configuration for S2 connections */
@@ -125,7 +126,20 @@ export const buildPullRequest = ({
   }
 }
 
-export const buildPushRequest = ({
+export interface S2PushRequest {
+  readonly url: string
+  readonly method: 'POST'
+  readonly headers: Record<string, string>
+  readonly body: string
+}
+
+/**
+ * Builds one or more append requests against S2. The helper applies the
+ * documented 1 MiB / 1000-record limits via `chunkEventsForS2`, so callers
+ * receive a request per compliant chunk instead of hitting 413 responses at
+ * runtime.
+ */
+export const buildPushRequests = ({
   config,
   storeId,
   batch,
@@ -133,21 +147,17 @@ export const buildPushRequest = ({
   config: S2Config
   storeId: string
   batch: readonly LiveStoreEvent.AnyEncodedGlobal[]
-}): {
-  url: string
-  method: 'POST'
-  headers: Record<string, string>
-  /** JSON-encoded batch */
-  body: string
-} => {
+}): ReadonlyArray<S2PushRequest> => {
   const streamName = makeS2StreamName(storeId)
   const url = getBasinUrl(config, `/streams/${encodeURIComponent(streamName)}/records`)
-  return {
+  const chunks = chunkEventsForS2(batch)
+
+  return chunks.map((chunk) => ({
     url,
-    method: 'POST',
+    method: 'POST' as const,
     headers: getPushHeaders(config.token),
-    body: JSON.stringify(formatBatchForS2(batch)),
-  }
+    body: JSON.stringify({ records: chunk.records }),
+  }))
 }
 
 // Response helpers
@@ -177,21 +187,3 @@ export const errorResponse = (message: string, status = 500): Response => {
     headers: { 'content-type': 'application/json' },
   })
 }
-
-// Batch formatting helper
-export const formatBatchForS2 = (
-  batch: readonly LiveStoreEvent.AnyEncodedGlobal[],
-): { records: { body: string }[] } => {
-  return {
-    records: batch.map((ev) => ({ body: JSON.stringify(ev) })),
-  }
-}
-
-export const asCurl = (request: { url: string; method: string; headers: Record<string, string>; body?: string }) => {
-  const url = request.url
-  const method = request.method
-  const headers = Object.entries(request.headers).map(([key, value]) => `-H "${key}: ${value}"`)
-  const body = request.body
-  const headersStr = headers.join(' ')
-  return `curl -X ${method} ${url} ${headersStr} ${body ? `-d '${body}'` : ''}`
-}

package/src/sync-provider.ts

@@ -50,6 +50,7 @@ import {
 import * as ApiSchema from './api-schema.ts'
 import { decodeReadBatch } from './decode.ts'
 import * as HttpClientGenerated from './http-client-generated.ts'
+import { chunkEventsForS2, S2LimitExceededError } from './limits.ts'
 import type { SyncMetadata } from './types.ts'
 
 export interface SyncS2Options {
@@ -247,15 +248,53 @@ export const makeSyncBackend =
           }
         },
         push: (batch) =>
-          HttpClientRequest.schemaBodyJson(ApiSchema.PushPayload)(HttpClientRequest.post(pushEndpoint), {
-            storeId,
-            batch,
-          }).pipe(
-            Effect.andThen(httpClient.pipe(HttpClient.filterStatusOk).execute),
-            Effect.andThen(HttpClientResponse.schemaBodyJson(ApiSchema.PushResponse)),
-            Effect.mapError((cause) => InvalidPushError.make({ cause: UnexpectedError.make({ cause }) })),
-            Effect.retry(retry?.push ?? defaultRetry),
-          ),
+          Effect.gen(function* () {
+            const makeInvalidPushError = (cause: unknown): InvalidPushError => {
+              if (cause instanceof InvalidPushError) {
+                return cause
+              }
+
+              if (cause instanceof UnexpectedError) {
+                return new InvalidPushError({ cause })
+              }
+
+              if (cause instanceof S2LimitExceededError) {
+                const note =
+                  cause.limitType === 'record-metered-bytes'
+                    ? `S2 record exceeded ${cause.max} metered bytes (actual: ${cause.actual})`
+                    : `S2 batch exceeded ${cause.max} (type: ${cause.limitType}, actual: ${cause.actual})`
+
+                return new InvalidPushError({
+                  cause: new UnexpectedError({
+                    cause,
+                    note,
+                    payload: {
+                      limitType: cause.limitType,
+                      max: cause.max,
+                      actual: cause.actual,
+                      recordIndex: cause.recordIndex,
+                    },
+                  }),
+                })
+              }
+
+              return new InvalidPushError({ cause: new UnexpectedError({ cause }) })
+            }
+
+            const chunks = yield* Effect.sync(() => chunkEventsForS2(batch)).pipe(Effect.mapError(makeInvalidPushError))
+
+            for (const chunk of chunks) {
+              yield* HttpClientRequest.schemaBodyJson(ApiSchema.PushPayload)(HttpClientRequest.post(pushEndpoint), {
+                storeId,
+                batch: chunk.events,
+              }).pipe(
+                Effect.andThen(httpClient.pipe(HttpClient.filterStatusOk).execute),
+                Effect.andThen(HttpClientResponse.schemaBodyJson(ApiSchema.PushResponse)),
+                Effect.mapError(makeInvalidPushError),
+                Effect.retry(retry?.push ?? defaultRetry),
+              )
+            }
+          }),
         ping,
         isConnected,
         metadata: {