@electric-sql/client 1.5.10 → 1.5.12

package/skills/electric-shapes/SKILL.md

@@ -0,0 +1,339 @@
+---
+name: electric-shapes
+description: >
+  Configure ShapeStream and Shape to sync a Postgres table to the client.
+  Covers ShapeStreamOptions (url, table, where, columns, replica, offset,
+  handle), custom type parsers (timestamptz, jsonb, int8), column mappers
+  (snakeCamelMapper, createColumnMapper), onError retry semantics, backoff
+  options, log modes (full, changes_only), requestSnapshot, fetchSnapshot,
+  subscribe/unsubscribe, and Shape materialized view. Load when setting up
+  sync, configuring shapes, parsing types, or handling sync errors.
+type: core
+library: electric
+library_version: '1.5.10'
+sources:
+  - 'electric-sql/electric:packages/typescript-client/src/client.ts'
+  - 'electric-sql/electric:packages/typescript-client/src/shape.ts'
+  - 'electric-sql/electric:packages/typescript-client/src/types.ts'
+  - 'electric-sql/electric:packages/typescript-client/src/parser.ts'
+  - 'electric-sql/electric:packages/typescript-client/src/column-mapper.ts'
+  - 'electric-sql/electric:website/docs/guides/shapes.md'
+---
+
+# Electric — Shape Streaming
+
+## Setup
+
+```ts
+import { ShapeStream, Shape } from '@electric-sql/client'
+
+const stream = new ShapeStream({
+  url: '/api/todos', // Your proxy route, NOT direct Electric URL
+  // Built-in parsers auto-handle: bool, int2, int4, float4, float8, json, jsonb
+  // Add custom parsers for other types (see references/type-parsers.md)
+  parser: {
+    timestamptz: (date: string) => new Date(date),
+  },
+})
+
+const shape = new Shape(stream)
+
+shape.subscribe(({ rows }) => {
+  console.log('synced rows:', rows)
+})
+
+// Wait for initial sync
+const rows = await shape.rows
+```
+
+## Core Patterns
+
+### Filter rows with WHERE clause and positional params
+
+```ts
+const stream = new ShapeStream({
+  url: '/api/todos',
+  params: {
+    table: 'todos',
+    where: 'user_id = $1 AND status = $2',
+    params: { '1': userId, '2': 'active' },
+  },
+})
+```
+
+### Select specific columns (must include primary key)
+
+```ts
+const stream = new ShapeStream({
+  url: '/api/todos',
+  params: {
+    table: 'todos',
+    columns: ['id', 'title', 'status'], // PK required
+  },
+})
+```
+
+### Map column names between snake_case and camelCase
+
+```ts
+import { ShapeStream, snakeCamelMapper } from '@electric-sql/client'
+
+const stream = new ShapeStream({
+  url: '/api/todos',
+  columnMapper: snakeCamelMapper(),
+})
+// DB column "created_at" arrives as "createdAt" in client
+// WHERE clauses auto-translate: "createdAt" → "created_at"
+```
+
+### Handle errors with retry
+
+```ts
+const stream = new ShapeStream({
+  url: '/api/todos',
+  onError: (error) => {
+    console.error('sync error', error)
+    return {} // Return {} to retry; returning void stops the stream
+  },
+})
+```
+
+For auth token refresh on 401 errors, see electric-proxy-auth/SKILL.md.
+
+### Resume from stored offset
+
+```ts
+const stream = new ShapeStream({
+  url: '/api/todos',
+  offset: storedOffset, // Both offset AND handle required
+  handle: storedHandle,
+})
+```
+
+### Get replica with old values on update
+
+```ts
+const stream = new ShapeStream({
+  url: '/api/todos',
+  params: {
+    table: 'todos',
+    replica: 'full', // Sends unchanged columns + old_value on updates
+  },
+})
+```
+
+## Common Mistakes
+
+### CRITICAL Returning void from onError stops sync permanently
+
+Wrong:
+
+```ts
+const stream = new ShapeStream({
+  url: '/api/todos',
+  onError: (error) => {
+    console.error('sync error', error)
+    // Returning nothing = stream stops forever
+  },
+})
+```
+
+Correct:
+
+```ts
+const stream = new ShapeStream({
+  url: '/api/todos',
+  onError: (error) => {
+    console.error('sync error', error)
+    return {} // Return {} to retry
+  },
+})
+```
+
+`onError` returning `undefined` signals the stream to permanently stop. Return at least `{}` to retry, or return `{ headers, params }` to retry with updated values.
+
+Source: `packages/typescript-client/src/client.ts:409-418`
+
+### HIGH Using columns without including primary key
+
+Wrong:
+
+```ts
+const stream = new ShapeStream({
+  url: '/api/todos',
+  params: {
+    table: 'todos',
+    columns: ['title', 'status'],
+  },
+})
+```
+
+Correct:
+
+```ts
+const stream = new ShapeStream({
+  url: '/api/todos',
+  params: {
+    table: 'todos',
+    columns: ['id', 'title', 'status'],
+  },
+})
+```
+
+Server returns 400 error. The `columns` list must always include the primary key column(s).
+
+Source: `website/docs/guides/shapes.md`
+
+### HIGH Setting offset without handle for resumption
+
+Wrong:
+
+```ts
+new ShapeStream({
+  url: '/api/todos',
+  offset: storedOffset,
+})
+```
+
+Correct:
+
+```ts
+new ShapeStream({
+  url: '/api/todos',
+  offset: storedOffset,
+  handle: storedHandle,
+})
+```
+
+Throws `MissingShapeHandleError`. Both `offset` AND `handle` are required to resume a stream from a stored position.
+
+Source: `packages/typescript-client/src/client.ts:1997-2003`
+
+### HIGH Using non-deterministic functions in WHERE clause
+
+Wrong:
+
+```ts
+const stream = new ShapeStream({
+  url: '/api/events',
+  params: {
+    table: 'events',
+    where: 'start_time > now()',
+  },
+})
+```
+
+Correct:
+
+```ts
+const stream = new ShapeStream({
+  url: '/api/events',
+  params: {
+    table: 'events',
+    where: 'start_time > $1',
+    params: { '1': new Date().toISOString() },
+  },
+})
+```
+
+Server rejects WHERE clauses with non-deterministic functions like `now()`, `random()`, `count()`. Use static values or positional params.
+
+Source: `packages/sync-service/lib/electric/replication/eval/env/known_functions.ex`
+
+### HIGH Not parsing custom Postgres types
+
+Wrong:
+
+```ts
+const stream = new ShapeStream({
+  url: '/api/events',
+})
+// createdAt will be string "2024-01-15T10:30:00.000Z", not a Date
+```
+
+Correct:
+
+```ts
+const stream = new ShapeStream({
+  url: '/api/events',
+  parser: {
+    timestamptz: (date: string) => new Date(date),
+    timestamp: (date: string) => new Date(date),
+  },
+})
+```
+
+Electric auto-parses `bool`, `int2`, `int4`, `float4`, `float8`, `json`, `jsonb`, and `int8` (→ BigInt). All other types arrive as strings — add custom parsers for `timestamptz`, `date`, `numeric`, etc. See [references/type-parsers.md](references/type-parsers.md) for the full list.
+
+Source: `AGENTS.md:300-308`
+
+### MEDIUM Using reserved parameter names in params
+
+Wrong:
+
+```ts
+const stream = new ShapeStream({
+  url: '/api/todos',
+  params: {
+    table: 'todos',
+    cursor: 'abc', // Reserved!
+    offset: '0', // Reserved!
+  },
+})
+```
+
+Correct:
+
+```ts
+const stream = new ShapeStream({
+  url: '/api/todos',
+  params: {
+    table: 'todos',
+    page_cursor: 'abc',
+    page_offset: '0',
+  },
+})
+```
+
+Throws `ReservedParamError`. Names `cursor`, `handle`, `live`, `offset`, `cache-buster`, and all `subset__*` prefixed params are reserved by the Electric protocol.
+
+Source: `packages/typescript-client/src/client.ts:1984-1985`
+
+### MEDIUM Mutating shape options on a running stream
+
+Wrong:
+
+```ts
+const stream = new ShapeStream({
+  url: '/api/todos',
+  params: { table: 'todos', where: "status = 'active'" },
+})
+// Later...
+stream.options.params.where = "status = 'done'" // No effect!
+```
+
+Correct:
+
+```ts
+// Create a new stream with different params
+const newStream = new ShapeStream({
+  url: '/api/todos',
+  params: { table: 'todos', where: "status = 'done'" },
+})
+```
+
+Shapes are immutable per subscription. Changing params on a running stream has no effect. Create a new `ShapeStream` instance for different filters.
+
+Source: `AGENTS.md:106`
+
+## References
+
+- [WHERE clause supported types and functions](references/where-clause.md)
+- [Built-in type parsers](references/type-parsers.md)
+
+See also: electric-proxy-auth/SKILL.md — Shape URLs must point to proxy routes, not directly to Electric.
+See also: electric-debugging/SKILL.md — onError semantics and backoff are essential for diagnosing sync problems.
+
+## Version
+
+Targets @electric-sql/client v1.5.10.

package/skills/electric-shapes/references/type-parsers.md

@@ -0,0 +1,64 @@
+# Electric Shapes — Type Parser Reference
+
+## Built-in Parsers
+
+These parsers are applied automatically. All other types arrive as strings.
+
+| Postgres Type | Parser        | Output Type | Notes                      |
+| ------------- | ------------- | ----------- | -------------------------- |
+| `int2`        | `parseNumber` | `number`    |                            |
+| `int4`        | `parseNumber` | `number`    |                            |
+| `int8`        | `parseBigInt` | `BigInt`    | Returns BigInt, not number |
+| `float4`      | `parseNumber` | `number`    |                            |
+| `float8`      | `parseNumber` | `number`    |                            |
+| `bool`        | `parseBool`   | `boolean`   |                            |
+| `json`        | `parseJson`   | `object`    |                            |
+| `jsonb`       | `parseJson`   | `object`    |                            |
+
+## Common Custom Parsers
+
+```ts
+const stream = new ShapeStream({
+  url: '/api/items',
+  parser: {
+    timestamptz: (date: string) => new Date(date),
+    timestamp: (date: string) => new Date(date),
+    date: (date: string) => new Date(date),
+    numeric: (n: string) => parseFloat(n),
+    interval: (i: string) => i, // Keep as string or use a library
+  },
+})
+```
+
+## Parser Signature
+
+```ts
+type ParseFunction<Extensions = never> = (
+  value: string,
+  additionalInfo?: Omit<ColumnInfo, 'type' | 'dims'>
+) => Value<Extensions>
+```
+
+The `additionalInfo` parameter provides column metadata like `precision`, `scale`, `max_length`, `not_null`.
+
+## NULL Handling
+
+If a column has `not_null: true` in the schema and a `NULL` value is received, the parser throws `ParserNullValueError`. This indicates a schema mismatch.
+
+## Transformer vs Parser
+
+- **Parser**: converts individual column values by Postgres type name
+- **Transformer**: transforms the entire row object after parsing
+
+```ts
+const stream = new ShapeStream({
+  url: '/api/items',
+  parser: {
+    timestamptz: (date: string) => new Date(date),
+  },
+  transformer: (row) => ({
+    ...row,
+    fullName: `${row.firstName} ${row.lastName}`,
+  }),
+})
+```

package/skills/electric-shapes/references/where-clause.md

@@ -0,0 +1,64 @@
+# Electric Shapes — WHERE Clause Reference
+
+## Supported Column Types
+
+| Type                       | Example                               | Notes                 |
+| -------------------------- | ------------------------------------- | --------------------- |
+| `text`, `varchar`, `char`  | `name = 'Alice'`                      | String comparison     |
+| `int2`, `int4`, `int8`     | `age > 21`                            | Numeric comparison    |
+| `float4`, `float8`         | `price < 9.99`                        | Float comparison      |
+| `bool`                     | `active = true`                       | Boolean               |
+| `uuid`                     | `id = '550e8400-...'`                 | UUID comparison       |
+| `date`                     | `created > '2024-01-01'`              | Date comparison       |
+| `timestamp`, `timestamptz` | `updated_at > '2024-01-01T00:00:00Z'` | Timestamp comparison  |
+| `interval`                 | `duration > '1 hour'`                 | Interval comparison   |
+| `numeric`                  | `amount >= 100.50`                    | Arbitrary precision   |
+| `arrays`                   | `tags && ARRAY['urgent']`             | Array operations      |
+| `enum`                     | `status::text IN ('active', 'done')`  | **Must cast to text** |
+
+## Unsupported
+
+- `timetz` — not supported in WHERE
+- Non-deterministic functions: `now()`, `random()`, `count()`, `current_timestamp`
+- Aggregate functions
+- Subqueries (experimental, requires `ELECTRIC_FEATURE_FLAGS=allow_subqueries`)
+
+## Positional Parameters
+
+```ts
+// Array format
+params: { where: 'org_id = $1 AND role = $2', params: ['org-123', 'admin'] }
+
+// Object format
+params: { where: 'org_id = $1 AND role = $2', params: { '1': 'org-123', '2': 'admin' } }
+```
+
+## Operators
+
+| Operator                 | Example                           |
+| ------------------------ | --------------------------------- |
+| `=`, `!=`, `<>`          | `status = 'active'`               |
+| `<`, `>`, `<=`, `>=`     | `age >= 18`                       |
+| `IN`                     | `status IN ('active', 'pending')` |
+| `NOT IN`                 | `status NOT IN ('deleted')`       |
+| `LIKE`, `ILIKE`          | `name ILIKE '%john%'`             |
+| `IS NULL`, `IS NOT NULL` | `deleted_at IS NULL`              |
+| `AND`, `OR`, `NOT`       | `active = true AND age > 18`      |
+| `BETWEEN`                | `age BETWEEN 18 AND 65`           |
+| `ANY`, `ALL`             | Array comparisons                 |
+
+## Enum Gotcha
+
+Enum columns require explicit `::text` cast:
+
+```ts
+// Wrong — fails silently or errors
+params: {
+  where: "status IN ('active', 'done')"
+}
+
+// Correct
+params: {
+  where: "status::text IN ('active', 'done')"
+}
+```

package/src/client.ts

@@ -96,6 +96,10 @@ const RESERVED_PARAMS: Set<ReservedParamKeys> = new Set([
 
 const TROUBLESHOOTING_URL = `https://electric-sql.com/docs/guides/troubleshooting`
 
+function createCacheBuster(): string {
+  return `${Date.now()}-${Math.random().toString(36).substring(2, 9)}`
+}
+
 type Replica = `full` | `default`
 export type LogMode = `changes_only` | `full`
 
@@ -617,6 +621,7 @@ export class ShapeStream<T extends Row<unknown> = Row>
   #fastLoopBackoffMaxMs = 5_000
   #fastLoopConsecutiveCount = 0
   #fastLoopMaxCount = 5
+  #refetchCacheBuster?: string
 
   constructor(options: ShapeStreamOptions<GetExtensions<T>>) {
     this.options = { subscribe: true, ...options }
@@ -875,10 +880,10 @@ export class ShapeStream<T extends Row<unknown> = Row>
       if (!(e instanceof FetchError)) throw e // should never happen
 
       if (e.status == 409) {
-        // Upon receiving a 409, we should start from scratch
-        // with the newly provided shape handle, or a fallback
-        // pseudo-handle based on the current one to act as a
-        // consistent cache buster
+        // Upon receiving a 409, start from scratch with the newly
+        // provided shape handle. If the header is missing (e.g. proxy
+        // stripped it), reset without a handle and use a random
+        // cache-buster query param to ensure the retry URL is unique.
 
         // Store the current shape URL as expired to avoid future 409s
         if (this.#syncState.handle) {
@@ -886,8 +891,14 @@ export class ShapeStream<T extends Row<unknown> = Row>
           expiredShapesCache.markExpired(shapeKey, this.#syncState.handle)
         }
 
-        const newShapeHandle =
-          e.headers[SHAPE_HANDLE_HEADER] || `${this.#syncState.handle!}-next`
+        const newShapeHandle = e.headers[SHAPE_HANDLE_HEADER]
+        if (!newShapeHandle) {
+          console.warn(
+            `[Electric] Received 409 response without a shape handle header. ` +
+              `This likely indicates a proxy or CDN stripping required headers.`
+          )
+          this.#refetchCacheBuster = createCacheBuster()
+        }
         this.#reset(newShapeHandle)
 
         // must refetch control message might be in a list or not depending
@@ -1144,6 +1155,16 @@ export class ShapeStream<T extends Row<unknown> = Row>
       fetchUrl.searchParams.set(EXPIRED_HANDLE_QUERY_PARAM, expiredHandle)
     }
 
+    // Add one-shot cache buster when a 409 response lacked a handle header
+    // (e.g. proxy stripped it). Ensures each retry has a unique URL.
+    if (this.#refetchCacheBuster) {
+      fetchUrl.searchParams.set(
+        CACHE_BUSTER_QUERY_PARAM,
+        this.#refetchCacheBuster
+      )
+      this.#refetchCacheBuster = undefined
+    }
+
     // sort query params in-place for stable URLs and improved cache hits
     fetchUrl.searchParams.sort()
 
@@ -1199,8 +1220,7 @@ export class ShapeStream<T extends Row<unknown> = Row>
       expiredHandle,
       now: Date.now(),
       maxStaleCacheRetries: this.#maxStaleCacheRetries,
-      createCacheBuster: () =>
-        `${Date.now()}-${Math.random().toString(36).substring(2, 9)}`,
+      createCacheBuster,
     })
 
     this.#syncState = transition.state
@@ -1786,8 +1806,7 @@ export class ShapeStream<T extends Row<unknown> = Row>
           expiredHandle: null,
           now: Date.now(),
           maxStaleCacheRetries: this.#maxStaleCacheRetries,
-          createCacheBuster: () =>
-            `${Date.now()}-${Math.random().toString(36).substring(2, 9)}`,
+          createCacheBuster,
         })
         if (transition.action === `accepted`) {
           this.#syncState = transition.state
@@ -1869,9 +1888,16 @@ export class ShapeStream<T extends Row<unknown> = Row>
 
         // For snapshot 409s, only update the handle — don't reset offset/schema/etc.
         // The main stream is paused and should not be disturbed.
-        const nextHandle =
-          e.headers[SHAPE_HANDLE_HEADER] || `${usedHandle ?? `handle`}-next`
-        this.#syncState = this.#syncState.withHandle(nextHandle)
+        const nextHandle = e.headers[SHAPE_HANDLE_HEADER]
+        if (nextHandle) {
+          this.#syncState = this.#syncState.withHandle(nextHandle)
+        } else {
+          console.warn(
+            `[Electric] Received 409 response without a shape handle header. ` +
+              `This likely indicates a proxy or CDN stripping required headers.`
+          )
+          this.#refetchCacheBuster = createCacheBuster()
+        }
 
         return this.fetchSnapshot(opts)
       }