@electric-sql/client 0.7.3 → 0.9.0

package/src/client.ts

@@ -11,7 +11,10 @@ import { isUpToDateMessage } from './helpers'
 import {
   FetchError,
   FetchBackoffAbortError,
-  MissingHeadersError,
+  MissingShapeUrlError,
+  InvalidSignalError,
+  MissingShapeHandleError,
+  ReservedParamError,
 } from './error'
 import {
   BackoffDefaults,
@@ -31,27 +34,49 @@ import {
   SHAPE_HANDLE_QUERY_PARAM,
   SHAPE_SCHEMA_HEADER,
   WHERE_QUERY_PARAM,
-  DATABASE_ID_QUERY_PARAM,
   TABLE_QUERY_PARAM,
   REPLICA_PARAM,
 } from './constants'
 
 const RESERVED_PARAMS = new Set([
-  DATABASE_ID_QUERY_PARAM,
-  COLUMNS_QUERY_PARAM,
   LIVE_CACHE_BUSTER_QUERY_PARAM,
   SHAPE_HANDLE_QUERY_PARAM,
   LIVE_QUERY_PARAM,
   OFFSET_QUERY_PARAM,
-  TABLE_QUERY_PARAM,
-  WHERE_QUERY_PARAM,
-  REPLICA_PARAM,
 ])
 
 type Replica = `full` | `default`
 
+/**
+ * PostgreSQL-specific shape parameters that can be provided externally
+ */
+type PostgresParams = {
+  /** The root table for the shape. Not required if you set the table in your proxy. */
+  table?: string
+
+  /**
+   * The columns to include in the shape.
+   * Must include primary keys, and can only include valid columns.
+   */
+  columns?: string[]
+
+  /** The where clauses for the shape */
+  where?: string
+
+  /**
+   * If `replica` is `default` (the default) then Electric will only send the
+   * changed columns in an update.
+   *
+   * If it's `full` Electric will send the entire row with both changed and
+   * unchanged values.
+   *
+   * Setting `replica` to `full` will result in higher bandwidth
+   * usage and so is not generally recommended.
+   */
+  replica?: Replica
+}
+
 type ReservedParamKeys =
-  | typeof DATABASE_ID_QUERY_PARAM
   | typeof COLUMNS_QUERY_PARAM
   | typeof LIVE_CACHE_BUSTER_QUERY_PARAM
   | typeof SHAPE_HANDLE_QUERY_PARAM
@@ -61,7 +86,41 @@ type ReservedParamKeys =
   | typeof WHERE_QUERY_PARAM
   | typeof REPLICA_PARAM
 
-type ParamsRecord = Omit<Record<string, string>, ReservedParamKeys>
+/**
+ * External params type - what users provide.
+ * Includes documented PostgreSQL params and allows string or string[] values for any additional params.
+ */
+type ExternalParamsRecord = Partial<PostgresParams> & {
+  [K in string as K extends ReservedParamKeys ? never : K]: string | string[]
+}
+
+/**
+ * Internal params type - used within the library.
+ * All values are converted to strings.
+ */
+type InternalParamsRecord = {
+  [K in string as K extends ReservedParamKeys ? never : K]: string
+}
+
+/**
+ * Helper function to convert external params to internal format
+ */
+function toInternalParams(params: ExternalParamsRecord): InternalParamsRecord {
+  const result: InternalParamsRecord = {}
+  for (const [key, value] of Object.entries(params)) {
+    result[key] = Array.isArray(value) ? value.join(`,`) : value
+  }
+  return result
+}
+
+type RetryOpts = {
+  params?: ExternalParamsRecord
+  headers?: Record<string, string>
+}
+
+type ShapeStreamErrorHandler = (
+  error: Error
+) => void | RetryOpts | Promise<void | RetryOpts>
 
 /**
  * Options for constructing a ShapeStream.
@@ -73,39 +132,6 @@ export interface ShapeStreamOptions<T = never> {
    */
   url: string
 
-  /**
-   * Which database to use.
-   * This is optional unless Electric is used with multiple databases.
-   */
-  databaseId?: string
-
-  /**
-   * The root table for the shape. Passed as a query parameter. Not required if you set the table in your proxy.
-   */
-  table?: string
-
-  /**
-   * The where clauses for the shape.
-   */
-  where?: string
-
-  /**
-   * The columns to include in the shape.
-   * Must include primary keys, and can only inlude valid columns.
-   */
-  columns?: string[]
-
-  /**
-   * If `replica` is `default` (the default) then Electric will only send the
-   * changed columns in an update.
-   *
-   * If it's `full` Electric will send the entire row with both changed and
-   * unchanged values.
-   *
-   * Setting `replica` to `full` will obviously result in higher bandwidth
-   * usage and so is not recommended.
-   */
-  replica?: Replica
   /**
    * The "offset" on the shape log. This is typically not set as the ShapeStream
    * will handle this automatically. A common scenario where you might pass an offset
@@ -115,12 +141,12 @@ export interface ShapeStreamOptions<T = never> {
    * so it knows at what point in the shape to catch you up from.
    */
   offset?: Offset
+
   /**
    * Similar to `offset`, this isn't typically used unless you're maintaining
    * a cache of the shape log.
    */
-  shapeHandle?: string
-  backoffOptions?: BackoffOptions
+  handle?: string
 
   /**
    * HTTP headers to attach to requests made by the client.
@@ -132,18 +158,32 @@ export interface ShapeStreamOptions<T = never> {
    * Additional request parameters to attach to the URL.
    * These will be merged with Electric's standard parameters.
    * Note: You cannot use Electric's reserved parameter names
-   * (table, where, columns, offset, handle, live, cursor, database_id, replica).
+   * (offset, handle, live, cursor).
+   *
+   * PostgreSQL-specific options like table, where, columns, and replica
+   * should be specified here.
    */
-  params?: ParamsRecord
+  params?: ExternalParamsRecord
 
   /**
    * Automatically fetch updates to the Shape. If you just want to sync the current
    * shape and stop, pass false.
    */
   subscribe?: boolean
+
   signal?: AbortSignal
   fetchClient?: typeof fetch
+  backoffOptions?: BackoffOptions
   parser?: Parser<T>
+
+  /**
+   * A function for handling shapestream errors.
+   * This is optional, when it is not provided any shapestream errors will be thrown.
+   * If the function returns an object containing parameters and/or headers
+   * the shapestream will apply those changes and try syncing again.
+   * If the function returns void the shapestream is stopped.
+   */
+  onError?: ShapeStreamErrorHandler
 }
 
 export interface ShapeStreamInterface<T extends Row<unknown> = Row> {
@@ -151,12 +191,7 @@ export interface ShapeStreamInterface<T extends Row<unknown> = Row> {
     callback: (messages: Message<T>[]) => MaybePromise<void>,
     onError?: (error: FetchError | Error) => void
   ): void
-  unsubscribeAllUpToDateSubscribers(): void
   unsubscribeAll(): void
-  subscribeOnceToUpToDate(
-    callback: () => MaybePromise<void>,
-    error: (err: FetchError | Error) => void
-  ): () => void
 
   isLoading(): boolean
   lastSyncedAt(): number | undefined
@@ -166,6 +201,7 @@ export interface ShapeStreamInterface<T extends Row<unknown> = Row> {
   isUpToDate: boolean
   lastOffset: Offset
   shapeHandle?: string
+  error?: unknown
 }
 
 /**
@@ -208,6 +244,7 @@ export class ShapeStream<T extends Row<unknown> = Row>
   }
 
   readonly options: ShapeStreamOptions<GetExtensions<T>>
+  #error: unknown = null
 
   readonly #fetchClient: typeof fetch
   readonly #messageParser: MessageParser<T>
@@ -219,10 +256,6 @@ export class ShapeStream<T extends Row<unknown> = Row>
       ((error: Error) => void) | undefined,
     ]
   >()
-  readonly #upToDateSubscribers = new Map<
-    number,
-    [() => void, (error: FetchError | Error) => void]
-  >()
 
   #lastOffset: Offset
   #liveCacheBuster: string // Seconds since our Electric Epoch 😎
@@ -230,20 +263,17 @@ export class ShapeStream<T extends Row<unknown> = Row>
   #isUpToDate: boolean = false
   #connected: boolean = false
   #shapeHandle?: string
-  #databaseId?: string
   #schema?: Schema
-  #error?: unknown
-  #replica?: Replica
+  #onError?: ShapeStreamErrorHandler
 
   constructor(options: ShapeStreamOptions<GetExtensions<T>>) {
-    validateOptions(options)
     this.options = { subscribe: true, ...options }
+    validateOptions(this.options)
     this.#lastOffset = this.options.offset ?? `-1`
     this.#liveCacheBuster = ``
-    this.#shapeHandle = this.options.shapeHandle
-    this.#databaseId = this.options.databaseId
+    this.#shapeHandle = this.options.handle
     this.#messageParser = new MessageParser<T>(options.parser)
-    this.#replica = this.options.replica
+    this.#onError = this.options.onError
 
     const baseFetchClient =
       options.fetchClient ??
@@ -261,13 +291,17 @@ export class ShapeStream<T extends Row<unknown> = Row>
       createFetchWithChunkBuffer(fetchWithBackoffClient)
     )
 
-    this.start()
+    this.#start()
   }
 
   get shapeHandle() {
     return this.#shapeHandle
   }
 
+  get error() {
+    return this.#error
+  }
+
   get isUpToDate() {
     return this.#isUpToDate
   }
@@ -276,20 +310,14 @@ export class ShapeStream<T extends Row<unknown> = Row>
     return this.#lastOffset
   }
 
-  get error() {
-    return this.#error
-  }
-
-  async start() {
-    this.#isUpToDate = false
-
-    const { url, table, where, columns, signal } = this.options
-
+  async #start() {
     try {
       while (
-        (!signal?.aborted && !this.#isUpToDate) ||
+        (!this.options.signal?.aborted && !this.#isUpToDate) ||
         this.options.subscribe
       ) {
+        const { url, signal } = this.options
+
         const fetchUrl = new URL(url)
 
         // Add any custom parameters first
@@ -304,16 +332,30 @@ export class ShapeStream<T extends Row<unknown> = Row>
             )
           }
 
-          for (const [key, value] of Object.entries(this.options.params)) {
-            fetchUrl.searchParams.set(key, value)
+          // Add PostgreSQL-specific parameters from params
+          const params = toInternalParams(this.options.params)
+          if (params.table)
+            fetchUrl.searchParams.set(TABLE_QUERY_PARAM, params.table)
+          if (params.where)
+            fetchUrl.searchParams.set(WHERE_QUERY_PARAM, params.where)
+          if (params.columns)
+            fetchUrl.searchParams.set(COLUMNS_QUERY_PARAM, params.columns)
+          if (params.replica)
+            fetchUrl.searchParams.set(REPLICA_PARAM, params.replica)
+
+          // Add any remaining custom parameters
+          const customParams = { ...params }
+          delete customParams.table
+          delete customParams.where
+          delete customParams.columns
+          delete customParams.replica
+
+          for (const [key, value] of Object.entries(customParams)) {
+            fetchUrl.searchParams.set(key, value as string)
           }
         }
 
         // Add Electric's internal parameters
-        if (table) fetchUrl.searchParams.set(TABLE_QUERY_PARAM, table)
-        if (where) fetchUrl.searchParams.set(WHERE_QUERY_PARAM, where)
-        if (columns && columns.length > 0)
-          fetchUrl.searchParams.set(COLUMNS_QUERY_PARAM, columns.join(`,`))
         fetchUrl.searchParams.set(OFFSET_QUERY_PARAM, this.#lastOffset)
 
         if (this.#isUpToDate) {
@@ -332,16 +374,8 @@ export class ShapeStream<T extends Row<unknown> = Row>
           )
         }
 
-        if (this.#databaseId) {
-          fetchUrl.searchParams.set(DATABASE_ID_QUERY_PARAM, this.#databaseId!)
-        }
-
-        if (
-          (this.#replica ?? ShapeStream.Replica.DEFAULT) !=
-          ShapeStream.Replica.DEFAULT
-        ) {
-          fetchUrl.searchParams.set(REPLICA_PARAM, this.#replica as string)
-        }
+        // sort query params in-place for stable URLs and improved cache hits
+        fetchUrl.searchParams.sort()
 
         let response!: Response
         try {
@@ -352,7 +386,6 @@ export class ShapeStream<T extends Row<unknown> = Row>
           this.#connected = true
         } catch (e) {
           if (e instanceof FetchBackoffAbortError) break // interrupted
-          if (e instanceof MissingHeadersError) throw e
           if (!(e instanceof FetchError)) throw e // should never happen
           if (e.status == 409) {
             // Upon receiving a 409, we should start from scratch
@@ -363,7 +396,6 @@ export class ShapeStream<T extends Row<unknown> = Row>
             continue
           } else if (e.status >= 400 && e.status < 500) {
             // Notify subscribers
-            this.#sendErrorToUpToDateSubscribers(e)
             this.#sendErrorToSubscribers(e)
 
             // 400 errors are not actionable without additional user input,
@@ -405,7 +437,6 @@ export class ShapeStream<T extends Row<unknown> = Row>
 
         // Update isUpToDate
         if (batch.length > 0) {
-          const prevUpToDate = this.#isUpToDate
           const lastMessage = batch[batch.length - 1]
           if (isUpToDateMessage(lastMessage)) {
             this.#lastSyncedAt = Date.now()
@@ -413,13 +444,31 @@ export class ShapeStream<T extends Row<unknown> = Row>
           }
 
           await this.#publish(batch)
-          if (!prevUpToDate && this.#isUpToDate) {
-            this.#notifyUpToDateSubscribers()
-          }
         }
       }
     } catch (err) {
       this.#error = err
+      if (this.#onError) {
+        const retryOpts = await this.#onError(err as Error)
+        if (typeof retryOpts === `object`) {
+          this.#reset()
+
+          if (`params` in retryOpts) {
+            this.options.params = retryOpts.params
+          }
+
+          if (`headers` in retryOpts) {
+            this.options.headers = retryOpts.headers
+          }
+
+          // Restart
+          this.#start()
+        }
+        return
+      }
+
+      // If no handler is provided for errors just throw so the error still bubbles up.
+      throw err
     } finally {
       this.#connected = false
     }
@@ -427,7 +476,7 @@ export class ShapeStream<T extends Row<unknown> = Row>
 
   subscribe(
     callback: (messages: Message<T>[]) => MaybePromise<void>,
-    onError?: (error: FetchError | Error) => void
+    onError: (error: Error) => void = () => {}
   ) {
     const subscriptionId = Math.random()
 
@@ -442,23 +491,6 @@ export class ShapeStream<T extends Row<unknown> = Row>
     this.#subscribers.clear()
   }
 
-  subscribeOnceToUpToDate(
-    callback: () => MaybePromise<void>,
-    error: (err: FetchError | Error) => void
-  ) {
-    const subscriptionId = Math.random()
-
-    this.#upToDateSubscribers.set(subscriptionId, [callback, error])
-
-    return () => {
-      this.#upToDateSubscribers.delete(subscriptionId)
-    }
-  }
-
-  unsubscribeAllUpToDateSubscribers(): void {
-    this.#upToDateSubscribers.clear()
-  }
-
   /** Unix time at which we last synced. Undefined when `isLoading` is true. */
   lastSyncedAt(): number | undefined {
     return this.#lastSyncedAt
@@ -477,7 +509,7 @@ export class ShapeStream<T extends Row<unknown> = Row>
 
   /** True during initial fetch. False afterwise.  */
   isLoading(): boolean {
-    return !this.isUpToDate
+    return !this.#isUpToDate
   }
 
   async #publish(messages: Message<T>[]): Promise<void> {
@@ -500,26 +532,14 @@ export class ShapeStream<T extends Row<unknown> = Row>
     })
   }
 
-  #notifyUpToDateSubscribers() {
-    this.#upToDateSubscribers.forEach(([callback]) => {
-      callback()
-    })
-  }
-
-  #sendErrorToUpToDateSubscribers(error: FetchError | Error) {
-    this.#upToDateSubscribers.forEach(([_, errorCallback]) =>
-      errorCallback(error)
-    )
-  }
-
   /**
    * Resets the state of the stream, optionally with a provided
    * shape handle
    */
-  #reset(shapeHandle?: string) {
+  #reset(handle?: string) {
     this.#lastOffset = `-1`
     this.#liveCacheBuster = ``
-    this.#shapeHandle = shapeHandle
+    this.#shapeHandle = handle
     this.#isUpToDate = false
     this.#connected = false
     this.#schema = undefined
@@ -528,22 +548,18 @@ export class ShapeStream<T extends Row<unknown> = Row>
 
 function validateOptions<T>(options: Partial<ShapeStreamOptions<T>>): void {
   if (!options.url) {
-    throw new Error(`Invalid shape options. It must provide the url`)
+    throw new MissingShapeUrlError()
   }
   if (options.signal && !(options.signal instanceof AbortSignal)) {
-    throw new Error(
-      `Invalid signal option. It must be an instance of AbortSignal.`
-    )
+    throw new InvalidSignalError()
   }
 
   if (
     options.offset !== undefined &&
     options.offset !== `-1` &&
-    !options.shapeHandle
+    !options.handle
   ) {
-    throw new Error(
-      `shapeHandle is required if this isn't an initial fetch (i.e. offset > -1)`
-    )
+    throw new MissingShapeHandleError()
   }
 
   // Check for reserved parameter names
@@ -552,9 +568,7 @@ function validateOptions<T>(options: Partial<ShapeStreamOptions<T>>): void {
       RESERVED_PARAMS.has(key)
     )
     if (reservedParams.length > 0) {
-      throw new Error(
-        `Cannot use reserved Electric parameter names in custom params: ${reservedParams.join(`, `)}`
-      )
+      throw new ReservedParamError(reservedParams)
     }
   }
   return

package/src/constants.ts

@@ -3,7 +3,6 @@ export const SHAPE_HANDLE_HEADER = `electric-handle`
 export const CHUNK_LAST_OFFSET_HEADER = `electric-offset`
 export const SHAPE_SCHEMA_HEADER = `electric-schema`
 export const CHUNK_UP_TO_DATE_HEADER = `electric-up-to-date`
-export const DATABASE_ID_QUERY_PARAM = `database_id`
 export const COLUMNS_QUERY_PARAM = `columns`
 export const LIVE_CACHE_BUSTER_QUERY_PARAM = `cursor`
 export const SHAPE_HANDLE_QUERY_PARAM = `handle`

package/src/error.ts

@@ -46,6 +46,60 @@ export class FetchError extends Error {
 export class FetchBackoffAbortError extends Error {
   constructor() {
     super(`Fetch with backoff aborted`)
+    this.name = `FetchBackoffAbortError`
+  }
+}
+
+export class InvalidShapeOptionsError extends Error {
+  constructor(message: string) {
+    super(message)
+    this.name = `InvalidShapeOptionsError`
+  }
+}
+
+export class MissingShapeUrlError extends Error {
+  constructor() {
+    super(`Invalid shape options: missing required url parameter`)
+    this.name = `MissingShapeUrlError`
+  }
+}
+
+export class InvalidSignalError extends Error {
+  constructor() {
+    super(`Invalid signal option. It must be an instance of AbortSignal.`)
+    this.name = `InvalidSignalError`
+  }
+}
+
+export class MissingShapeHandleError extends Error {
+  constructor() {
+    super(
+      `shapeHandle is required if this isn't an initial fetch (i.e. offset > -1)`
+    )
+    this.name = `MissingShapeHandleError`
+  }
+}
+
+export class ReservedParamError extends Error {
+  constructor(reservedParams: string[]) {
+    super(
+      `Cannot use reserved Electric parameter names in custom params: ${reservedParams.join(`, `)}`
+    )
+    this.name = `ReservedParamError`
+  }
+}
+
+export class ParserNullValueError extends Error {
+  constructor(columnName: string) {
+    super(`Column "${columnName ?? `unknown`}" does not allow NULL values`)
+    this.name = `ParserNullValueError`
+  }
+}
+
+export class ShapeStreamAlreadyRunningError extends Error {
+  constructor() {
+    super(`ShapeStream is already running`)
+    this.name = `ShapeStreamAlreadyRunningError`
   }
 }
 
@@ -56,6 +110,7 @@ export class MissingHeadersError extends Error {
       msg += `- ${h}\n`
     })
     msg += `\nThis is often due to a proxy not setting CORS correctly so that all Electric headers can be read by the client.`
+    msg += `\nFor more information visit the troubleshooting guide: /docs/guides/troubleshooting/missing-headers`
     super(msg)
   }
 }

package/src/fetch.ts

@@ -171,7 +171,7 @@ export function createFetchWithResponseHeadersCheck(
       const missingHeaders: Array<string> = []
 
       const addMissingHeaders = (requiredHeaders: Array<string>) =>
-        requiredHeaders.filter((h) => !headers.has(h))
+        missingHeaders.push(...requiredHeaders.filter((h) => !headers.has(h)))
       addMissingHeaders(requiredElectricResponseHeaders)
 
       const input = args[0]
@@ -312,6 +312,7 @@ function getNextChunkUrl(url: string, res: Response): string | void {
 
   nextUrl.searchParams.set(SHAPE_HANDLE_QUERY_PARAM, shapeHandle)
   nextUrl.searchParams.set(OFFSET_QUERY_PARAM, lastOffset)
+  nextUrl.searchParams.sort()
   return nextUrl.toString()
 }
 

package/src/parser.ts

@@ -1,4 +1,5 @@
 import { ColumnInfo, GetExtensions, Message, Row, Schema, Value } from './types'
+import { ParserNullValueError } from './error'
 
 type NullToken = null | `NULL`
 type Token = Exclude<string, NullToken>
@@ -162,7 +163,7 @@ function makeNullableParser<Extensions>(
   return (value: NullableToken) => {
     if (isPgNull(value)) {
       if (!isNullable) {
-        throw new Error(`Column ${columnName ?? `unknown`} is not nullable`)
+        throw new ParserNullValueError(columnName ?? `unknown`)
       }
       return null
     }

package/src/shape.ts

@@ -21,7 +21,12 @@ export type ShapeChangedCallback<T extends Row<unknown> = Row> = (data: {
  * @param {ShapeStream<T extends Row>} - the underlying shape stream
  * @example
  * ```
- * const shapeStream = new ShapeStream<{ foo: number }>(url: `http://localhost:3000/v1/shape`, table: `foo`})
+ * const shapeStream = new ShapeStream<{ foo: number }>({
+ *   url: `http://localhost:3000/v1/shape`,
+ *   params: {
+ *     table: `foo`
+ *   }
+ * })
  * const shape = new Shape(shapeStream)
  * ```
  *
@@ -55,15 +60,6 @@ export class Shape<T extends Row<unknown> = Row> {
       this.#process.bind(this),
       this.#handleError.bind(this)
     )
-    const unsubscribe = this.#stream.subscribeOnceToUpToDate(
-      () => {
-        unsubscribe()
-      },
-      (e) => {
-        this.#handleError(e)
-        throw e
-      }
-    )
   }
 
   get isUpToDate(): boolean {
@@ -74,6 +70,10 @@ export class Shape<T extends Row<unknown> = Row> {
     return this.#stream.lastOffset
   }
 
+  get handle(): string | undefined {
+    return this.#stream.shapeHandle
+  }
+
   get rows(): Promise<T[]> {
     return this.value.then((v) => Array.from(v.values()))
   }