@electric-sql/client 1.5.6 → 1.5.8

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@electric-sql/client",
   "description": "Postgres everywhere - your data, in sync, wherever you need it.",
-  "version": "1.5.6",
+  "version": "1.5.8",
   "author": "ElectricSQL team and contributors.",
   "bugs": {
     "url": "https://github.com/electric-sql/electric/issues"

package/src/client.ts

@@ -15,7 +15,12 @@ import {
   encodeWhereClause,
   quoteIdentifier,
 } from './column-mapper'
-import { getOffset, isUpToDateMessage, isChangeMessage } from './helpers'
+import {
+  getOffset,
+  isUpToDateMessage,
+  isChangeMessage,
+  bigintSafeStringify,
+} from './helpers'
 import {
   FetchError,
   FetchBackoffAbortError,
@@ -89,6 +94,8 @@ const RESERVED_PARAMS: Set<ReservedParamKeys> = new Set([
   CACHE_BUSTER_QUERY_PARAM,
 ])
 
+const TROUBLESHOOTING_URL = `https://electric-sql.com/docs/guides/troubleshooting`
+
 type Replica = `full` | `default`
 export type LogMode = `changes_only` | `full`
 
@@ -601,6 +608,15 @@ export class ShapeStream<T extends Row<unknown> = Row>
   #unsubscribeFromVisibilityChanges?: () => void
   #unsubscribeFromWakeDetection?: () => void
   #maxStaleCacheRetries = 3
+  // Fast-loop detection: track recent non-live requests to detect tight retry
+  // loops caused by proxy/CDN misconfiguration or stale client-side caches
+  #recentRequestEntries: Array<{ timestamp: number; offset: string }> = []
+  #fastLoopWindowMs = 500
+  #fastLoopThreshold = 5
+  #fastLoopBackoffBaseMs = 100
+  #fastLoopBackoffMaxMs = 5_000
+  #fastLoopConsecutiveCount = 0
+  #fastLoopMaxCount = 5
 
   constructor(options: ShapeStreamOptions<GetExtensions<T>>) {
     this.options = { subscribe: true, ...options }
@@ -745,6 +761,8 @@ export class ShapeStream<T extends Row<unknown> = Row>
           if (this.#syncState instanceof ErrorState) {
             this.#syncState = this.#syncState.retry()
           }
+          this.#fastLoopConsecutiveCount = 0
+          this.#recentRequestEntries = []
 
           // Restart from current offset
           this.#started = false
@@ -788,6 +806,14 @@ export class ShapeStream<T extends Row<unknown> = Row>
       return
     }
 
+    // Only check for fast loops on non-live requests; live polling is expected to be rapid
+    if (!this.#syncState.isUpToDate) {
+      await this.#checkFastLoop()
+    } else {
+      this.#fastLoopConsecutiveCount = 0
+      this.#recentRequestEntries = []
+    }
+
     let resumingFromPause = false
     if (this.#syncState instanceof PausedState) {
       resumingFromPause = true
@@ -893,6 +919,88 @@ export class ShapeStream<T extends Row<unknown> = Row>
     return this.#requestShape()
   }
 
+  /**
+   * Detects tight retry loops (e.g., from stale client-side caches or
+   * proxy/CDN misconfiguration) and attempts recovery. On first detection,
+   * clears client-side caches (in-memory and localStorage) and resets the
+   * stream to fetch from scratch.
+   * If the loop persists, applies exponential backoff and eventually throws.
+   */
+  async #checkFastLoop(): Promise<void> {
+    const now = Date.now()
+    const currentOffset = this.#syncState.offset
+
+    this.#recentRequestEntries = this.#recentRequestEntries.filter(
+      (e) => now - e.timestamp < this.#fastLoopWindowMs
+    )
+    this.#recentRequestEntries.push({ timestamp: now, offset: currentOffset })
+
+    // Only flag as a fast loop if requests are stuck at the same offset.
+    // Normal rapid syncing advances the offset with each response.
+    const sameOffsetCount = this.#recentRequestEntries.filter(
+      (e) => e.offset === currentOffset
+    ).length
+
+    if (sameOffsetCount < this.#fastLoopThreshold) return
+
+    this.#fastLoopConsecutiveCount++
+
+    if (this.#fastLoopConsecutiveCount >= this.#fastLoopMaxCount) {
+      throw new FetchError(
+        502,
+        undefined,
+        undefined,
+        {},
+        this.options.url,
+        `Client is stuck in a fast retry loop ` +
+          `(${this.#fastLoopThreshold} requests in ${this.#fastLoopWindowMs}ms at the same offset, ` +
+          `repeated ${this.#fastLoopMaxCount} times). ` +
+          `Client-side caches were cleared automatically on first detection, but the loop persists. ` +
+          `This usually indicates a proxy or CDN misconfiguration. ` +
+          `Common causes:\n` +
+          `  - Proxy is not including query parameters (handle, offset) in its cache key\n` +
+          `  - CDN is serving stale 409 responses\n` +
+          `  - Proxy is stripping required Electric headers from responses\n` +
+          `For more information visit the troubleshooting guide: ${TROUBLESHOOTING_URL}`
+      )
+    }
+
+    if (this.#fastLoopConsecutiveCount === 1) {
+      console.warn(
+        `[Electric] Detected fast retry loop ` +
+          `(${this.#fastLoopThreshold} requests in ${this.#fastLoopWindowMs}ms at the same offset). ` +
+          `Clearing client-side caches and resetting stream to recover. ` +
+          `If this persists, check that your proxy includes all query parameters ` +
+          `(especially 'handle' and 'offset') in its cache key, ` +
+          `and that required Electric headers are forwarded to the client. ` +
+          `For more information visit the troubleshooting guide: ${TROUBLESHOOTING_URL}`
+      )
+
+      if (this.#currentFetchUrl) {
+        const shapeKey = canonicalShapeKey(this.#currentFetchUrl)
+        expiredShapesCache.delete(shapeKey)
+        upToDateTracker.delete(shapeKey)
+      } else {
+        expiredShapesCache.clear()
+        upToDateTracker.clear()
+      }
+      this.#reset()
+      this.#recentRequestEntries = []
+      return
+    }
+
+    // Exponential backoff with full jitter
+    const maxDelay = Math.min(
+      this.#fastLoopBackoffMaxMs,
+      this.#fastLoopBackoffBaseMs * Math.pow(2, this.#fastLoopConsecutiveCount)
+    )
+    const delayMs = Math.floor(Math.random() * maxDelay)
+
+    await new Promise((resolve) => setTimeout(resolve, delayMs))
+
+    this.#recentRequestEntries = []
+  }
+
   async #constructUrl(
     url: string,
     resumingFromPause: boolean,
@@ -987,7 +1095,7 @@ export class ShapeStream<T extends Row<unknown> = Row>
         // Serialize params as JSON to keep the parameter name constant for proxy configs
         fetchUrl.searchParams.set(
           SUBSET_PARAM_WHERE_PARAMS,
-          JSON.stringify(subsetParams.params)
+          bigintSafeStringify(subsetParams.params)
         )
       if (subsetParams.limit)
         setQueryParam(fetchUrl, SUBSET_PARAM_LIMIT, subsetParams.limit)
@@ -1110,7 +1218,7 @@ export class ShapeStream<T extends Row<unknown> = Row>
           `CDN continues serving stale cached responses after ${this.#maxStaleCacheRetries} retry attempts. ` +
             `This indicates a severe proxy/CDN misconfiguration. ` +
             `Check that your proxy includes all query parameters (especially 'handle' and 'offset') in its cache key. ` +
-            `For more information visit the troubleshooting guide: https://electric-sql.com/docs/guides/troubleshooting`
+            `For more information visit the troubleshooting guide: ${TROUBLESHOOTING_URL}`
         )
       }
       console.warn(
@@ -1118,7 +1226,7 @@ export class ShapeStream<T extends Row<unknown> = Row>
           `This should not happen and indicates a proxy/CDN caching misconfiguration. ` +
           `The response contained handle "${shapeHandle}" which was previously marked as expired. ` +
           `Check that your proxy includes all query parameters (especially 'handle' and 'offset') in its cache key. ` +
-          `For more information visit the troubleshooting guide: https://electric-sql.com/docs/guides/troubleshooting ` +
+          `For more information visit the troubleshooting guide: ${TROUBLESHOOTING_URL} ` +
           `Retrying with a random cache buster to bypass the stale cache (attempt ${this.#syncState.staleCacheRetryCount}/${this.#maxStaleCacheRetries}).`
       )
       throw new StaleCacheError(
@@ -1261,7 +1369,7 @@ export class ShapeStream<T extends Row<unknown> = Row>
     const batch = this.#messageParser.parse<Array<Message<T>>>(messages, schema)
 
     if (!Array.isArray(batch)) {
-      const preview = JSON.stringify(batch)?.slice(0, 200)
+      const preview = bigintSafeStringify(batch)?.slice(0, 200)
       throw new FetchError(
         response.status,
         `Received non-array response body from shape endpoint. ` +
@@ -1650,7 +1758,8 @@ export class ShapeStream<T extends Row<unknown> = Row>
     }, 30_000)
 
     try {
-      const { metadata, data } = await this.fetchSnapshot(opts)
+      const { metadata, data, responseOffset, responseHandle } =
+        await this.fetchSnapshot(opts)
 
       const dataWithEndBoundary = (data as Array<Message<T>>).concat([
         { headers: { control: `snapshot-end`, ...metadata } },
@@ -1663,6 +1772,31 @@ export class ShapeStream<T extends Row<unknown> = Row>
       )
       this.#onMessages(dataWithEndBoundary, false)
 
+      // On cold start the stream's offset is still at "now". Advance it
+      // to the snapshot's position so no updates are missed in between.
+      if (responseOffset !== null || responseHandle !== null) {
+        const transition = this.#syncState.handleResponseMetadata({
+          status: 200,
+          responseHandle,
+          responseOffset,
+          responseCursor: null,
+          expiredHandle: null,
+          now: Date.now(),
+          maxStaleCacheRetries: this.#maxStaleCacheRetries,
+          createCacheBuster: () =>
+            `${Date.now()}-${Math.random().toString(36).substring(2, 9)}`,
+        })
+        if (transition.action === `accepted`) {
+          this.#syncState = transition.state
+        } else {
+          console.warn(
+            `[Electric] Snapshot response metadata was not accepted ` +
+              `by state "${this.#syncState.kind}" (action: ${transition.action}). ` +
+              `Stream offset was not advanced from snapshot.`
+          )
+        }
+      }
+
       return {
         metadata,
         data,
@@ -1682,11 +1816,13 @@ export class ShapeStream<T extends Row<unknown> = Row>
    * `subsetMethod: 'POST'` on the stream to send parameters in the request body instead.
    *
    * @param opts - The options for the snapshot request.
-   * @returns The metadata and the data for the snapshot.
+   * @returns The metadata, data, and the response's offset/handle for state advancement.
    */
   async fetchSnapshot(opts: SubsetParams): Promise<{
     metadata: SnapshotMetadata
     data: Array<ChangeMessage<T>>
+    responseOffset: Offset | null
+    responseHandle: string | null
   }> {
     const method = opts.method ?? this.options.subsetMethod ?? `GET`
     const usePost = method === `POST`
@@ -1703,7 +1839,7 @@ export class ShapeStream<T extends Row<unknown> = Row>
           ...result.requestHeaders,
           'Content-Type': `application/json`,
         },
-        body: JSON.stringify(this.#buildSubsetBody(opts)),
+        body: bigintSafeStringify(this.#buildSubsetBody(opts)),
       }
     } else {
       const result = await this.#constructUrl(this.options.url, true, opts)
@@ -1757,7 +1893,11 @@ export class ShapeStream<T extends Row<unknown> = Row>
       schema
     )
 
-    return { metadata, data }
+    const responseOffset =
+      (response.headers.get(CHUNK_LAST_OFFSET_HEADER) as Offset) || null
+    const responseHandle = response.headers.get(SHAPE_HANDLE_HEADER)
+
+    return { metadata, data, responseOffset, responseHandle }
   }
 
   #buildSubsetBody(opts: SubsetParams): Record<string, unknown> {

package/src/expired-shapes-cache.ts

@@ -66,6 +66,11 @@ export class ExpiredShapesCache {
     this.data = {}
     this.save()
   }
+
+  delete(shapeUrl: string): void {
+    delete this.data[shapeUrl]
+    this.save()
+  }
 }
 
 // Module-level singleton instance

package/src/fetch.ts

@@ -44,9 +44,9 @@ export interface BackoffOptions {
 }
 
 export const BackoffDefaults = {
-  initialDelay: 100,
-  maxDelay: 60_000, // Cap at 60s - reasonable for long-lived connections
-  multiplier: 1.3,
+  initialDelay: 1_000,
+  maxDelay: 32_000,
+  multiplier: 2,
   maxRetries: Infinity, // Retry forever - clients may go offline and come back
 }
 

package/src/helpers.ts

@@ -51,7 +51,7 @@ export function isChangeMessage<T extends Row<unknown> = Row>(
 export function isControlMessage<T extends Row<unknown> = Row>(
   message: Message<T>
 ): message is ControlMessage {
-  return message != null && !isChangeMessage(message)
+  return message != null && `headers` in message && `control` in message.headers
 }
 
 export function isUpToDateMessage<T extends Row<unknown> = Row>(
@@ -71,6 +71,21 @@ export function getOffset(message: ControlMessage): Offset | undefined {
   return lsn ? (`${lsn}_0` as Offset) : undefined
 }
 
+function bigintReplacer(_key: string, value: unknown): unknown {
+  return typeof value === `bigint` ? value.toString() : value
+}
+
+/**
+ * BigInt-safe version of JSON.stringify.
+ * Converts BigInt values to their string representation (as JSON strings,
+ * e.g. `{ id: 42n }` becomes `{"id":"42"}`) instead of throwing.
+ * Assumes input is a JSON-serializable value — passing `undefined` at the
+ * top level will return `undefined` (matching `JSON.stringify` behavior).
+ */
+export function bigintSafeStringify(value: unknown): string {
+  return JSON.stringify(value, bigintReplacer)
+}
+
 /**
  * Checks if a transaction is visible in a snapshot.
  *

package/src/shape-stream-state.ts

@@ -704,6 +704,16 @@ export class PausedState extends ShapeStreamState {
     return this.previousState.replayCursor
   }
 
+  handleResponseMetadata(
+    input: ResponseMetadataInput
+  ): ResponseMetadataTransition {
+    const transition = this.previousState.handleResponseMetadata(input)
+    if (transition.action === `accepted`) {
+      return { action: `accepted`, state: new PausedState(transition.state) }
+    }
+    return transition
+  }
+
   withHandle(handle: string): PausedState {
     return new PausedState(this.previousState.withHandle(handle))
   }

package/src/shape.ts

@@ -1,5 +1,9 @@
 import { Message, Offset, Row } from './types'
-import { isChangeMessage, isControlMessage } from './helpers'
+import {
+  isChangeMessage,
+  isControlMessage,
+  bigintSafeStringify,
+} from './helpers'
 import { FetchError } from './error'
 import { LogMode, ShapeStreamInterface } from './client'
 
@@ -141,7 +145,7 @@ export class Shape<T extends Row<unknown> = Row> {
     params: Parameters<ShapeStreamInterface<T>[`requestSnapshot`]>[0]
   ): Promise<void> {
     // Track this snapshot request for future re-execution on shape rotation
-    const key = JSON.stringify(params)
+    const key = bigintSafeStringify(params)
     this.#requestedSubSnapshots.add(key)
     // Ensure the stream is up-to-date so schema is available for parsing
     await this.#awaitUpToDate()

package/src/types.ts

@@ -91,7 +91,7 @@ export type SubsetParams = {
   /** Legacy string format WHERE clause */
   where?: string
   /** Positional parameter values for WHERE clause */
-  params?: Record<string, string>
+  params?: Record<string, string | bigint | number>
   /** Maximum number of rows to return */
   limit?: number
   /** Number of rows to skip */

package/src/up-to-date-tracker.ts

@@ -151,6 +151,11 @@ export class UpToDateTracker {
     }
     this.save()
   }
+
+  delete(shapeKey: string): void {
+    delete this.data[shapeKey]
+    this.save()
+  }
 }
 
 // Module-level singleton instance