@electric-sql/client 1.1.1 → 1.1.3

package/src/client.ts

@@ -296,10 +296,51 @@ export interface ShapeStreamOptions<T = never> {
 
   /**
    * A function for handling shapestream errors.
-   * This is optional, when it is not provided any shapestream errors will be thrown.
-   * If the function returns an object containing parameters and/or headers
-   * the shapestream will apply those changes and try syncing again.
-   * If the function returns void the shapestream is stopped.
+   *
+   * **Automatic retries**: The client automatically retries 5xx server errors, network
+   * errors, and 429 rate limits with exponential backoff. The `onError` callback is
+   * only invoked after these automatic retries are exhausted, or for non-retryable
+   * errors like 4xx client errors.
+   *
+   * When not provided, non-retryable errors will be thrown and syncing will stop.
+   *
+   * **Return value behavior**:
+   * - Return an **object** (RetryOpts or empty `{}`) to retry syncing:
+   *   - `{}` - Retry with the same params and headers
+   *   - `{ params }` - Retry with modified params
+   *   - `{ headers }` - Retry with modified headers (e.g., refreshed auth token)
+   *   - `{ params, headers }` - Retry with both modified
+   * - Return **void** or **undefined** to stop the stream permanently
+   *
+   * **Important**: If you want syncing to continue after an error (e.g., to retry
+   * on network failures), you MUST return at least an empty object `{}`. Simply
+   * logging the error and returning nothing will stop syncing.
+   *
+   * Supports async functions that return `Promise<void | RetryOpts>`.
+   *
+   * @example
+   * ```typescript
+   * // Retry on network errors, stop on others
+   * onError: (error) => {
+   *   console.error('Stream error:', error)
+   *   if (error instanceof FetchError && error.status >= 500) {
+   *     return {} // Retry with same params
+   *   }
+   *   // Return void to stop on other errors
+   * }
+   * ```
+   *
+   * @example
+   * ```typescript
+   * // Refresh auth token on 401
+   * onError: async (error) => {
+   *   if (error instanceof FetchError && error.status === 401) {
+   *     const newToken = await refreshAuthToken()
+   *     return { headers: { Authorization: `Bearer ${newToken}` } }
+   *   }
+   *   return {} // Retry other errors
+   * }
+   * ```
    */
   onError?: ShapeStreamErrorHandler
 }
@@ -441,6 +482,13 @@ export class ShapeStream<T extends Row<unknown> = Row>
   #activeSnapshotRequests = 0 // counter for concurrent snapshot requests
   #midStreamPromise?: Promise<void>
   #midStreamPromiseResolver?: () => void
+  #lastSseConnectionStartTime?: number
+  #minSseConnectionDuration = 1000 // Minimum expected SSE connection duration (1 second)
+  #consecutiveShortSseConnections = 0
+  #maxShortSseConnections = 3 // Fall back to long polling after this many short connections
+  #sseFallbackToLongPolling = false
+  #sseBackoffBaseDelay = 100 // Base delay for exponential backoff (ms)
+  #sseBackoffMaxDelay = 5000 // Maximum delay cap (ms)
 
   constructor(options: ShapeStreamOptions<GetExtensions<T>>) {
     this.options = { subscribe: true, ...options }
@@ -507,32 +555,61 @@ export class ShapeStream<T extends Row<unknown> = Row>
       await this.#requestShape()
     } catch (err) {
       this.#error = err
+
+      // Check if onError handler wants to retry
       if (this.#onError) {
         const retryOpts = await this.#onError(err as Error)
-        if (typeof retryOpts === `object`) {
-          this.#reset()
-
-          if (`params` in retryOpts) {
-            this.options.params = retryOpts.params
+        // Guard against null (typeof null === "object" in JavaScript)
+        if (retryOpts && typeof retryOpts === `object`) {
+          // Update params/headers but don't reset offset
+          // We want to continue from where we left off, not refetch everything
+          if (retryOpts.params) {
+            // Merge new params with existing params to preserve other parameters
+            this.options.params = {
+              ...(this.options.params ?? {}),
+              ...retryOpts.params,
+            }
           }
 
-          if (`headers` in retryOpts) {
-            this.options.headers = retryOpts.headers
+          if (retryOpts.headers) {
+            // Merge new headers with existing headers to preserve other headers
+            this.options.headers = {
+              ...(this.options.headers ?? {}),
+              ...retryOpts.headers,
+            }
           }
 
-          // Restart
+          // Clear the error since we're retrying
+          this.#error = null
+
+          // Restart from current offset
           this.#started = false
-          this.#start()
+          await this.#start()
+          return
         }
+        // onError returned void, meaning it doesn't want to retry
+        // This is an unrecoverable error, notify subscribers
+        if (err instanceof Error) {
+          this.#sendErrorToSubscribers(err)
+        }
+        this.#connected = false
+        this.#tickPromiseRejecter?.()
         return
       }
 
-      // If no handler is provided for errors just throw so the error still bubbles up.
-      throw err
-    } finally {
+      // No onError handler provided, this is an unrecoverable error
+      // Notify subscribers and throw
+      if (err instanceof Error) {
+        this.#sendErrorToSubscribers(err)
+      }
       this.#connected = false
       this.#tickPromiseRejecter?.()
+      throw err
     }
+
+    // Normal completion, clean up
+    this.#connected = false
+    this.#tickPromiseRejecter?.()
   }
 
   async #requestShape(): Promise<void> {
@@ -613,12 +690,10 @@ export class ShapeStream<T extends Row<unknown> = Row>
         )
         return this.#requestShape()
       } else {
-        // Notify subscribers
-        this.#sendErrorToSubscribers(e)
-
         // errors that have reached this point are not actionable without
         // additional user input, such as 400s or failures to read the
-        // body of a response, so we exit the loop
+        // body of a response, so we exit the loop and let #start handle it
+        // Note: We don't notify subscribers here because onError might recover
         throw e
       }
     } finally {
@@ -832,7 +907,8 @@ export class ShapeStream<T extends Row<unknown> = Row>
       this.#isUpToDate &&
       useSse &&
       !this.#isRefreshing &&
-      !opts.resumingFromPause
+      !opts.resumingFromPause &&
+      !this.#sseFallbackToLongPolling
     ) {
       opts.fetchUrl.searchParams.set(EXPERIMENTAL_LIVE_SSE_QUERY_PARAM, `true`)
       opts.fetchUrl.searchParams.set(LIVE_SSE_QUERY_PARAM, `true`)
@@ -871,6 +947,10 @@ export class ShapeStream<T extends Row<unknown> = Row>
   }): Promise<void> {
     const { fetchUrl, requestAbortController, headers } = opts
     const fetch = this.#sseFetchClient
+
+    // Track when the SSE connection starts
+    this.#lastSseConnectionStartTime = Date.now()
+
     try {
       let buffer: Array<Message<T>> = []
       await fetchEventSource(fetchUrl.toString(), {
@@ -916,6 +996,44 @@ export class ShapeStream<T extends Row<unknown> = Row>
         throw new FetchBackoffAbortError()
       }
       throw error
+    } finally {
+      // Check if the SSE connection closed too quickly
+      // This can happen when responses are cached or when the proxy/server
+      // is misconfigured for SSE and closes the connection immediately
+      const connectionDuration = Date.now() - this.#lastSseConnectionStartTime!
+      const wasAborted = requestAbortController.signal.aborted
+
+      if (connectionDuration < this.#minSseConnectionDuration && !wasAborted) {
+        // Connection was too short - likely a cached response or misconfiguration
+        this.#consecutiveShortSseConnections++
+
+        if (
+          this.#consecutiveShortSseConnections >= this.#maxShortSseConnections
+        ) {
+          // Too many short connections - fall back to long polling
+          this.#sseFallbackToLongPolling = true
+          console.warn(
+            `[Electric] SSE connections are closing immediately (possibly due to proxy buffering or misconfiguration). ` +
+              `Falling back to long polling. ` +
+              `Your proxy must support streaming SSE responses (not buffer the complete response). ` +
+              `Configuration: Nginx add 'X-Accel-Buffering: no', Caddy add 'flush_interval -1' to reverse_proxy. ` +
+              `Note: Do NOT disable caching entirely - Electric uses cache headers to enable request collapsing for efficiency.`
+          )
+        } else {
+          // Add exponential backoff with full jitter to prevent tight infinite loop
+          // Formula: random(0, min(cap, base * 2^attempt))
+          const maxDelay = Math.min(
+            this.#sseBackoffMaxDelay,
+            this.#sseBackoffBaseDelay *
+              Math.pow(2, this.#consecutiveShortSseConnections)
+          )
+          const delayMs = Math.floor(Math.random() * maxDelay)
+          await new Promise((resolve) => setTimeout(resolve, delayMs))
+        }
+      } else if (connectionDuration >= this.#minSseConnectionDuration) {
+        // Connection was healthy - reset counter
+        this.#consecutiveShortSseConnections = 0
+      }
     }
   }
 
@@ -1090,6 +1208,9 @@ export class ShapeStream<T extends Row<unknown> = Row>
     this.#connected = false
     this.#schema = undefined
     this.#activeSnapshotRequests = 0
+    // Reset SSE fallback state to try SSE again after reset
+    this.#consecutiveShortSseConnections = 0
+    this.#sseFallbackToLongPolling = false
   }
 
   /**

package/src/constants.ts

@@ -30,6 +30,7 @@ export const SUBSET_PARAM_WHERE_PARAMS = `subset__params`
 // Query parameters that should be passed through when proxying Electric requests
 export const ELECTRIC_PROTOCOL_QUERY_PARAMS: Array<string> = [
   LIVE_QUERY_PARAM,
+  LIVE_SSE_QUERY_PARAM,
   SHAPE_HANDLE_QUERY_PARAM,
   OFFSET_QUERY_PARAM,
   LIVE_CACHE_BUSTER_QUERY_PARAM,

package/src/fetch.ts

@@ -38,21 +38,8 @@ export interface BackoffOptions {
    * Maximum number of retry attempts before giving up.
    * Set to Infinity (default) for indefinite retries - needed for offline scenarios
    * where clients may go offline and come back later.
-   *
-   * The retry budget provides protection against retry storms even with infinite retries.
    */
   maxRetries?: number
-  /**
-   * Percentage of requests that can be retries (0.1 = 10%)
-   *
-   * This is the primary load shedding mechanism. It limits the *rate* of retries,
-   * not the total count. Even with infinite retries, at most 10% of your traffic
-   * will be retries, preventing retry storms from amplifying server load.
-   *
-   * The budget resets every 60 seconds, so a temporary spike of errors won't
-   * permanently exhaust the budget.
-   */
-  retryBudgetPercent?: number
 }
 
 export const BackoffDefaults = {
@@ -60,7 +47,31 @@ export const BackoffDefaults = {
   maxDelay: 60_000, // Cap at 60s - reasonable for long-lived connections
   multiplier: 1.3,
   maxRetries: Infinity, // Retry forever - clients may go offline and come back
-  retryBudgetPercent: 0.1, // 10% retry budget prevents amplification
+}
+
+/**
+ * Parse Retry-After header value and return delay in milliseconds
+ * Supports both delta-seconds format and HTTP-date format
+ * Returns 0 if header is not present or invalid
+ */
+export function parseRetryAfterHeader(retryAfter: string | undefined): number {
+  if (!retryAfter) return 0
+
+  // Try parsing as seconds (delta-seconds format)
+  const retryAfterSec = Number(retryAfter)
+  if (Number.isFinite(retryAfterSec) && retryAfterSec > 0) {
+    return retryAfterSec * 1000
+  }
+
+  // Try parsing as HTTP-date
+  const retryDate = Date.parse(retryAfter)
+  if (!isNaN(retryDate)) {
+    // Handle clock skew: clamp to non-negative, cap at reasonable max
+    const deltaMs = retryDate - Date.now()
+    return Math.max(0, Math.min(deltaMs, 3600_000)) // Cap at 1 hour
+  }
+
+  return 0
 }
 
 export function createFetchWithBackoff(
@@ -74,37 +85,7 @@ export function createFetchWithBackoff(
     debug = false,
     onFailedAttempt,
     maxRetries = Infinity,
-    retryBudgetPercent = 0.1,
   } = backoffOptions
-
-  // Retry budget tracking (closure-scoped)
-  // Resets every minute to prevent retry storms
-  let totalRequests = 0
-  let totalRetries = 0
-  let budgetResetTime = Date.now() + 60_000
-
-  function checkRetryBudget(percent: number): boolean {
-    const now = Date.now()
-    if (now > budgetResetTime) {
-      totalRequests = 0
-      totalRetries = 0
-      budgetResetTime = now + 60_000
-    }
-
-    totalRequests++
-
-    // Allow retries for first 10 requests to avoid cold start issues
-    if (totalRequests < 10) return true
-
-    const currentRetryRate = totalRetries / totalRequests
-    const hasCapacity = currentRetryRate < percent
-
-    if (hasCapacity) {
-      totalRetries++
-    }
-
-    return hasCapacity
-  }
   return async (...args: Parameters<typeof fetch>): Promise<Response> => {
     const url = args[0]
     const options = args[1]
@@ -116,8 +97,6 @@ export function createFetchWithBackoff(
       try {
         const result = await fetchClient(...args)
         if (result.ok) {
-          // Reset backoff on successful request
-          delay = initialDelay
           return result
         }
 
@@ -137,9 +116,9 @@ export function createFetchWithBackoff(
           // Any client errors cannot be backed off on, leave it to the caller to handle.
           throw e
         } else {
-          // Check retry budget and max retries
+          // Check max retries
           attempt++
-          if (attempt >= maxRetries) {
+          if (attempt > maxRetries) {
             if (debug) {
               console.log(
                 `Max retries reached (${attempt}/${maxRetries}), giving up`
@@ -148,50 +127,20 @@ export function createFetchWithBackoff(
             throw e
           }
 
-          // Check retry budget - this is our primary load shedding mechanism
-          // It limits the *rate* of retries (10% of traffic) not the count
-          // This prevents retry storms even with infinite retries
-          if (!checkRetryBudget(retryBudgetPercent)) {
-            if (debug) {
-              console.log(
-                `Retry budget exhausted (attempt ${attempt}), backing off`
-              )
-            }
-            // Wait for maxDelay before checking budget again
-            // This prevents tight retry loops when budget is exhausted
-            await new Promise((resolve) => setTimeout(resolve, maxDelay))
-            // Don't throw - continue retrying after the wait
-            // This allows offline clients to eventually reconnect
-            continue
-          }
-
           // Calculate wait time honoring server-driven backoff as a floor
           // Precedence: max(serverMinimum, min(clientMaxDelay, backoffWithJitter))
 
           // 1. Parse server-provided Retry-After (if present)
-          let serverMinimumMs = 0
-          if (e instanceof FetchError && e.headers) {
-            const retryAfter = e.headers[`retry-after`]
-            if (retryAfter) {
-              const retryAfterSec = Number(retryAfter)
-              if (Number.isFinite(retryAfterSec) && retryAfterSec > 0) {
-                // Retry-After in seconds
-                serverMinimumMs = retryAfterSec * 1000
-              } else {
-                // Retry-After as HTTP date
-                const retryDate = Date.parse(retryAfter)
-                if (!isNaN(retryDate)) {
-                  // Handle clock skew: clamp to non-negative, cap at reasonable max
-                  const deltaMs = retryDate - Date.now()
-                  serverMinimumMs = Math.max(0, Math.min(deltaMs, 3600_000)) // Cap at 1 hour
-                }
-              }
-            }
-          }
-
-          // 2. Calculate client backoff with full jitter
-          const jitter = Math.random() * delay
-          const clientBackoffMs = Math.min(jitter, maxDelay)
+          const serverMinimumMs =
+            e instanceof FetchError && e.headers
+              ? parseRetryAfterHeader(e.headers[`retry-after`])
+              : 0
+
+          // 2. Calculate client backoff with full jitter strategy
+          // Full jitter: random_between(0, min(cap, exponential_backoff))
+          // See: https://aws.amazon.com/blogs/architecture/exponential-backoff-and-jitter/
+          const jitter = Math.random() * delay // random value between 0 and current delay
+          const clientBackoffMs = Math.min(jitter, maxDelay) // cap at maxDelay
 
           // 3. Server minimum is the floor, client cap is the ceiling
           const waitMs = Math.max(serverMinimumMs, clientBackoffMs)