@electric-sql/client 1.3.1 → 1.4.1

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@electric-sql/client",
   "description": "Postgres everywhere - your data, in sync, wherever you need it.",
-  "version": "1.3.1",
+  "version": "1.4.1",
   "author": "ElectricSQL team and contributors.",
   "bugs": {
     "url": "https://github.com/electric-sql/electric/issues"

package/src/client.ts

@@ -24,6 +24,7 @@ import {
   MissingShapeHandleError,
   ReservedParamError,
   MissingHeadersError,
+  StaleCacheError,
 } from './error'
 import {
   BackoffDefaults,
@@ -59,7 +60,11 @@ import {
   SUBSET_PARAM_LIMIT,
   SUBSET_PARAM_OFFSET,
   SUBSET_PARAM_ORDER_BY,
+  SUBSET_PARAM_WHERE_EXPR,
+  SUBSET_PARAM_ORDER_BY_EXPR,
+  CACHE_BUSTER_QUERY_PARAM,
 } from './constants'
+import { compileExpression, compileOrderBy } from './expression-compiler'
 import {
   EventSourceMessage,
   fetchEventSource,
@@ -73,6 +78,7 @@ const RESERVED_PARAMS: Set<ReservedParamKeys> = new Set([
   SHAPE_HANDLE_QUERY_PARAM,
   LIVE_QUERY_PARAM,
   OFFSET_QUERY_PARAM,
+  CACHE_BUSTER_QUERY_PARAM,
 ])
 
 type Replica = `full` | `default`
@@ -138,6 +144,7 @@ type ReservedParamKeys =
   | typeof SHAPE_HANDLE_QUERY_PARAM
   | typeof LIVE_QUERY_PARAM
   | typeof OFFSET_QUERY_PARAM
+  | typeof CACHE_BUSTER_QUERY_PARAM
   | `subset__${string}`
 
 /**
@@ -570,6 +577,9 @@ export class ShapeStream<T extends Row<unknown> = Row>
   #sseBackoffBaseDelay = 100 // Base delay for exponential backoff (ms)
   #sseBackoffMaxDelay = 5000 // Maximum delay cap (ms)
   #unsubscribeFromVisibilityChanges?: () => void
+  #staleCacheBuster?: string // Cache buster set when stale CDN response detected, used on retry requests to bypass cache
+  #staleCacheRetryCount = 0
+  #maxStaleCacheRetries = 3
 
   // Derived state: we're in replay mode if we have a last seen cursor
   get #replayMode(): boolean {
@@ -669,7 +679,8 @@ export class ShapeStream<T extends Row<unknown> = Row>
       if (this.#onError) {
         const retryOpts = await this.#onError(err as Error)
         // Guard against null (typeof null === "object" in JavaScript)
-        if (retryOpts && typeof retryOpts === `object`) {
+        const isRetryable = !(err instanceof MissingHeadersError)
+        if (retryOpts && typeof retryOpts === `object` && isRetryable) {
           // Update params/headers but don't reset offset
           // We want to continue from where we left off, not refetch everything
           if (retryOpts.params) {
@@ -780,6 +791,15 @@ export class ShapeStream<T extends Row<unknown> = Row>
         }
         return // interrupted
       }
+
+      if (e instanceof StaleCacheError) {
+        // Received a stale cached response from CDN with an expired handle.
+        // The #staleCacheBuster has been set in #onInitialResponse, so retry
+        // the request which will include a random cache buster to bypass the
+        // misconfigured CDN cache.
+        return this.#requestShape()
+      }
+
       if (!(e instanceof FetchError)) throw e // should never happen
 
       if (e.status == 409) {
@@ -890,13 +910,29 @@ export class ShapeStream<T extends Row<unknown> = Row>
     }
 
     if (subsetParams) {
-      if (subsetParams.where && typeof subsetParams.where === `string`) {
+      // Prefer structured expressions when available (allows proper columnMapper application)
+      // Fall back to legacy string format for backwards compatibility
+      if (subsetParams.whereExpr) {
+        // Compile structured expression with columnMapper applied
+        const compiledWhere = compileExpression(
+          subsetParams.whereExpr,
+          this.options.columnMapper?.encode
+        )
+        setQueryParam(fetchUrl, SUBSET_PARAM_WHERE, compiledWhere)
+        // Also send the structured expression for servers that support it
+        fetchUrl.searchParams.set(
+          SUBSET_PARAM_WHERE_EXPR,
+          JSON.stringify(subsetParams.whereExpr)
+        )
+      } else if (subsetParams.where && typeof subsetParams.where === `string`) {
+        // Legacy string format (no columnMapper applied to already-compiled SQL)
         const encodedWhere = encodeWhereClause(
           subsetParams.where,
           this.options.columnMapper?.encode
         )
         setQueryParam(fetchUrl, SUBSET_PARAM_WHERE, encodedWhere)
       }
+
       if (subsetParams.params)
         // Serialize params as JSON to keep the parameter name constant for proxy configs
         fetchUrl.searchParams.set(
@@ -907,7 +943,25 @@ export class ShapeStream<T extends Row<unknown> = Row>
         setQueryParam(fetchUrl, SUBSET_PARAM_LIMIT, subsetParams.limit)
       if (subsetParams.offset)
         setQueryParam(fetchUrl, SUBSET_PARAM_OFFSET, subsetParams.offset)
-      if (subsetParams.orderBy && typeof subsetParams.orderBy === `string`) {
+
+      // Prefer structured ORDER BY expressions when available
+      if (subsetParams.orderByExpr) {
+        // Compile structured ORDER BY with columnMapper applied
+        const compiledOrderBy = compileOrderBy(
+          subsetParams.orderByExpr,
+          this.options.columnMapper?.encode
+        )
+        setQueryParam(fetchUrl, SUBSET_PARAM_ORDER_BY, compiledOrderBy)
+        // Also send the structured expression for servers that support it
+        fetchUrl.searchParams.set(
+          SUBSET_PARAM_ORDER_BY_EXPR,
+          JSON.stringify(subsetParams.orderByExpr)
+        )
+      } else if (
+        subsetParams.orderBy &&
+        typeof subsetParams.orderBy === `string`
+      ) {
+        // Legacy string format
         const encodedOrderBy = encodeWhereClause(
           subsetParams.orderBy,
           this.options.columnMapper?.encode
@@ -948,6 +1002,15 @@ export class ShapeStream<T extends Row<unknown> = Row>
       fetchUrl.searchParams.set(EXPIRED_HANDLE_QUERY_PARAM, expiredHandle)
     }
 
+    // Add random cache buster if we received a stale response from CDN
+    // This forces a fresh request bypassing the misconfigured CDN cache
+    if (this.#staleCacheBuster) {
+      fetchUrl.searchParams.set(
+        CACHE_BUSTER_QUERY_PARAM,
+        this.#staleCacheBuster
+      )
+    }
+
     // sort query params in-place for stable URLs and improved cache hits
     fetchUrl.searchParams.sort()
 
@@ -993,6 +1056,46 @@ export class ShapeStream<T extends Row<unknown> = Row>
         : null
       if (shapeHandle !== expiredHandle) {
         this.#shapeHandle = shapeHandle
+        // Clear cache buster after successful response with valid handle
+        if (this.#staleCacheBuster) {
+          this.#staleCacheBuster = undefined
+          this.#staleCacheRetryCount = 0
+        }
+      } else if (this.#shapeHandle === undefined) {
+        // We received a stale response from cache and don't have a handle yet.
+        // Instead of accepting the stale handle, throw an error to trigger a retry
+        // with a random cache buster to bypass the CDN cache.
+        this.#staleCacheRetryCount++
+        // Cancel the response body to release the connection before retrying
+        await response.body?.cancel()
+        if (this.#staleCacheRetryCount > this.#maxStaleCacheRetries) {
+          throw new FetchError(
+            502,
+            undefined,
+            undefined,
+            {},
+            this.#currentFetchUrl?.toString() ?? ``,
+            `CDN continues serving stale cached responses after ${this.#maxStaleCacheRetries} retry attempts. ` +
+              `This indicates a severe proxy/CDN misconfiguration. ` +
+              `Check that your proxy includes all query parameters (especially 'handle' and 'offset') in its cache key. ` +
+              `For more information visit the troubleshooting guide: https://electric-sql.com/docs/guides/troubleshooting`
+          )
+        }
+        console.warn(
+          `[Electric] Received stale cached response with expired shape handle. ` +
+            `This should not happen and indicates a proxy/CDN caching misconfiguration. ` +
+            `The response contained handle "${shapeHandle}" which was previously marked as expired. ` +
+            `Check that your proxy includes all query parameters (especially 'handle' and 'offset') in its cache key. ` +
+            `For more information visit the troubleshooting guide: https://electric-sql.com/docs/guides/troubleshooting ` +
+            `Retrying with a random cache buster to bypass the stale cache (attempt ${this.#staleCacheRetryCount}/${this.#maxStaleCacheRetries}).`
+        )
+        // Generate a random cache buster for the retry
+        this.#staleCacheBuster = `${Date.now()}-${Math.random().toString(36).substring(2, 9)}`
+        throw new StaleCacheError(
+          `Received stale cached response with expired handle "${shapeHandle}". ` +
+            `This indicates a proxy/CDN caching misconfiguration. ` +
+            `Check that your proxy includes all query parameters (especially 'handle' and 'offset') in its cache key.`
+        )
       } else {
         console.warn(
           `[Electric] Received stale cached response with expired shape handle. ` +
@@ -1453,6 +1556,9 @@ export class ShapeStream<T extends Row<unknown> = Row>
     // Reset SSE fallback state to try SSE again after reset
     this.#consecutiveShortSseConnections = 0
     this.#sseFallbackToLongPolling = false
+    // Reset stale cache retry state
+    this.#staleCacheBuster = undefined
+    this.#staleCacheRetryCount = 0
   }
 
   /**

package/src/constants.ts

@@ -26,6 +26,9 @@ export const SUBSET_PARAM_LIMIT = `subset__limit`
 export const SUBSET_PARAM_OFFSET = `subset__offset`
 export const SUBSET_PARAM_ORDER_BY = `subset__order_by`
 export const SUBSET_PARAM_WHERE_PARAMS = `subset__params`
+export const SUBSET_PARAM_WHERE_EXPR = `subset__where_expr`
+export const SUBSET_PARAM_ORDER_BY_EXPR = `subset__order_by_expr`
+export const CACHE_BUSTER_QUERY_PARAM = `cache-buster` // Random cache buster to bypass stale CDN responses
 
 // Query parameters that should be passed through when proxying Electric requests
 export const ELECTRIC_PROTOCOL_QUERY_PARAMS: Array<string> = [
@@ -41,4 +44,7 @@ export const ELECTRIC_PROTOCOL_QUERY_PARAMS: Array<string> = [
   SUBSET_PARAM_OFFSET,
   SUBSET_PARAM_ORDER_BY,
   SUBSET_PARAM_WHERE_PARAMS,
+  SUBSET_PARAM_WHERE_EXPR,
+  SUBSET_PARAM_ORDER_BY_EXPR,
+  CACHE_BUSTER_QUERY_PARAM,
 ]

package/src/error.ts

@@ -116,3 +116,10 @@ export class MissingHeadersError extends Error {
     super(msg)
   }
 }
+
+export class StaleCacheError extends Error {
+  constructor(message: string) {
+    super(message)
+    this.name = `StaleCacheError`
+  }
+}

package/src/expression-compiler.ts

@@ -0,0 +1,132 @@
+import { SerializedExpression, SerializedOrderByClause } from './types'
+import { quoteIdentifier } from './column-mapper'
+
+/**
+ * Compiles a serialized expression into a SQL string.
+ * Applies columnMapper transformations to column references.
+ *
+ * @param expr - The serialized expression to compile
+ * @param columnMapper - Optional function to transform column names (e.g., camelCase to snake_case)
+ * @returns The compiled SQL string
+ *
+ * @example
+ * ```typescript
+ * const expr = { type: 'ref', column: 'userId' }
+ * compileExpression(expr, camelToSnake) // '"user_id"'
+ * ```
+ */
+export function compileExpression(
+  expr: SerializedExpression,
+  columnMapper?: (col: string) => string
+): string {
+  switch (expr.type) {
+    case `ref`: {
+      // Apply columnMapper, then quote
+      const mappedColumn = columnMapper
+        ? columnMapper(expr.column)
+        : expr.column
+      return quoteIdentifier(mappedColumn)
+    }
+    case `val`:
+      return `$${expr.paramIndex}`
+    case `func`:
+      return compileFunction(expr, columnMapper)
+    default: {
+      // TypeScript exhaustiveness check
+      const _exhaustive: never = expr
+      throw new Error(`Unknown expression type: ${JSON.stringify(_exhaustive)}`)
+    }
+  }
+}
+
+/**
+ * Compiles a function expression into SQL.
+ */
+function compileFunction(
+  expr: { type: `func`; name: string; args: SerializedExpression[] },
+  columnMapper?: (col: string) => string
+): string {
+  const args = expr.args.map((arg) => compileExpression(arg, columnMapper))
+
+  switch (expr.name) {
+    // Binary comparison operators
+    case `eq`:
+      return `${args[0]} = ${args[1]}`
+    case `gt`:
+      return `${args[0]} > ${args[1]}`
+    case `gte`:
+      return `${args[0]} >= ${args[1]}`
+    case `lt`:
+      return `${args[0]} < ${args[1]}`
+    case `lte`:
+      return `${args[0]} <= ${args[1]}`
+
+    // Logical operators
+    case `and`:
+      return args.map((a) => `(${a})`).join(` AND `)
+    case `or`:
+      return args.map((a) => `(${a})`).join(` OR `)
+    case `not`:
+      return `NOT (${args[0]})`
+
+    // Special operators
+    case `in`:
+      return `${args[0]} = ANY(${args[1]})`
+    case `like`:
+      return `${args[0]} LIKE ${args[1]}`
+    case `ilike`:
+      return `${args[0]} ILIKE ${args[1]}`
+    case `isNull`:
+    case `isUndefined`:
+      return `${args[0]} IS NULL`
+
+    // String functions
+    case `upper`:
+      return `UPPER(${args[0]})`
+    case `lower`:
+      return `LOWER(${args[0]})`
+    case `length`:
+      return `LENGTH(${args[0]})`
+    case `concat`:
+      return `CONCAT(${args.join(`, `)})`
+
+    // Other functions
+    case `coalesce`:
+      return `COALESCE(${args.join(`, `)})`
+
+    default:
+      throw new Error(`Unknown function: ${expr.name}`)
+  }
+}
+
+/**
+ * Compiles serialized ORDER BY clauses into a SQL string.
+ * Applies columnMapper transformations to column references.
+ *
+ * @param clauses - The serialized ORDER BY clauses to compile
+ * @param columnMapper - Optional function to transform column names
+ * @returns The compiled SQL ORDER BY string
+ *
+ * @example
+ * ```typescript
+ * const clauses = [{ column: 'createdAt', direction: 'desc', nulls: 'first' }]
+ * compileOrderBy(clauses, camelToSnake) // '"created_at" DESC NULLS FIRST'
+ * ```
+ */
+export function compileOrderBy(
+  clauses: SerializedOrderByClause[],
+  columnMapper?: (col: string) => string
+): string {
+  return clauses
+    .map((clause) => {
+      const mappedColumn = columnMapper
+        ? columnMapper(clause.column)
+        : clause.column
+      let sql = quoteIdentifier(mappedColumn)
+      if (clause.direction === `desc`) sql += ` DESC`
+      if (clause.nulls === `first`) sql += ` NULLS FIRST`
+      if (clause.nulls === `last`) sql += ` NULLS LAST`
+      return sql
+    })
+    .join(`, `)
+}

package/src/index.ts

@@ -16,3 +16,4 @@ export {
   snakeToCamel,
   camelToSnake,
 } from './column-mapper'
+export { compileExpression, compileOrderBy } from './expression-compiler'

package/src/types.ts

@@ -68,12 +68,40 @@ export type MoveTag = string
  */
 export type MoveOutPattern = { pos: number; value: string }
 
+/**
+ * Serialized expression types for structured subset queries.
+ * These allow Electric to properly apply columnMapper transformations
+ * before generating the final SQL.
+ */
+export type SerializedExpression =
+  | { type: `ref`; column: string } // Column reference
+  | { type: `val`; paramIndex: number } // Parameter placeholder ($1, $2, etc.)
+  | { type: `func`; name: string; args: SerializedExpression[] } // Operator/function
+
+/**
+ * Serialized ORDER BY clause for structured subset queries.
+ */
+export type SerializedOrderByClause = {
+  column: string
+  direction?: `asc` | `desc` // omitted means 'asc'
+  nulls?: `first` | `last`
+}
+
 export type SubsetParams = {
+  /** Legacy string format WHERE clause */
   where?: string
+  /** Positional parameter values for WHERE clause */
   params?: Record<string, string>
+  /** Maximum number of rows to return */
   limit?: number
+  /** Number of rows to skip */
   offset?: number
+  /** Legacy string format ORDER BY clause */
   orderBy?: string
+  /** Structured WHERE expression (preferred when available) */
+  whereExpr?: SerializedExpression
+  /** Structured ORDER BY clauses (preferred when available) */
+  orderByExpr?: SerializedOrderByClause[]
 }
 
 export type ControlMessage = {