@electric-sql/client 0.4.1 → 0.5.1

package/src/constants.ts

@@ -0,0 +1,7 @@
+export const SHAPE_ID_HEADER = `x-electric-shape-id`
+export const CHUNK_LAST_OFFSET_HEADER = `x-electric-chunk-last-offset`
+export const SHAPE_SCHEMA_HEADER = `x-electric-schema`
+export const SHAPE_ID_QUERY_PARAM = `shape_id`
+export const OFFSET_QUERY_PARAM = `offset`
+export const WHERE_QUERY_PARAM = `where`
+export const LIVE_QUERY_PARAM = `live`

package/src/error.ts

@@ -0,0 +1,50 @@
+export class FetchError extends Error {
+  status: number
+  text?: string
+  json?: object
+  headers: Record<string, string>
+
+  constructor(
+    status: number,
+    text: string | undefined,
+    json: object | undefined,
+    headers: Record<string, string>,
+    public url: string,
+    message?: string
+  ) {
+    super(
+      message ||
+        `HTTP Error ${status} at ${url}: ${text ?? JSON.stringify(json)}`
+    )
+    this.name = `FetchError`
+    this.status = status
+    this.text = text
+    this.json = json
+    this.headers = headers
+  }
+
+  static async fromResponse(
+    response: Response,
+    url: string
+  ): Promise<FetchError> {
+    const status = response.status
+    const headers = Object.fromEntries([...response.headers.entries()])
+    let text: string | undefined = undefined
+    let json: object | undefined = undefined
+
+    const contentType = response.headers.get(`content-type`)
+    if (contentType && contentType.includes(`application/json`)) {
+      json = (await response.json()) as object
+    } else {
+      text = await response.text()
+    }
+
+    return new FetchError(status, text, json, headers, url)
+  }
+}
+
+export class FetchBackoffAbortError extends Error {
+  constructor() {
+    super(`Fetch with backoff aborted`)
+  }
+}

package/src/fetch.ts

@@ -0,0 +1,79 @@
+import { FetchError, FetchBackoffAbortError } from './error'
+
+export interface BackoffOptions {
+  /**
+   * Initial delay before retrying in milliseconds
+   */
+  initialDelay: number
+  /**
+   * Maximum retry delay in milliseconds
+   */
+  maxDelay: number
+  multiplier: number
+  onFailedAttempt?: () => void
+  debug?: boolean
+}
+
+export const BackoffDefaults = {
+  initialDelay: 100,
+  maxDelay: 10_000,
+  multiplier: 1.3,
+}
+
+export function createFetchWithBackoff(
+  fetchClient: typeof fetch,
+  backoffOptions: BackoffOptions = BackoffDefaults
+): typeof fetch {
+  const {
+    initialDelay,
+    maxDelay,
+    multiplier,
+    debug = false,
+    onFailedAttempt,
+  } = backoffOptions
+  return async (...args: Parameters<typeof fetch>): Promise<Response> => {
+    const url = args[0]
+    const options = args[1]
+
+    let delay = initialDelay
+    let attempt = 0
+
+    /* eslint-disable no-constant-condition -- we re-fetch the shape log
+     * continuously until we get a non-ok response. For recoverable errors,
+     * we retry the fetch with exponential backoff. Users can pass in an
+     * AbortController to abort the fetching an any point.
+     * */
+    while (true) {
+      /* eslint-enable no-constant-condition */
+      try {
+        const result = await fetchClient(...args)
+        if (result.ok) return result
+        else throw await FetchError.fromResponse(result, url.toString())
+      } catch (e) {
+        onFailedAttempt?.()
+        if (options?.signal?.aborted) {
+          throw new FetchBackoffAbortError()
+        } else if (
+          e instanceof FetchError &&
+          e.status >= 400 &&
+          e.status < 500
+        ) {
+          // Any client errors cannot be backed off on, leave it to the caller to handle.
+          throw e
+        } else {
+          // Exponentially backoff on errors.
+          // Wait for the current delay duration
+          await new Promise((resolve) => setTimeout(resolve, delay))
+
+          // Increase the delay for the next attempt
+          delay = Math.min(delay * multiplier, maxDelay)
+
+          if (debug) {
+            attempt++
+            console.log(`Retry attempt #${attempt} after ${delay}ms`)
+          }
+        }
+      }
+    }
+  }
+}

package/src/helpers.ts

@@ -45,3 +45,9 @@ export function isControlMessage<T extends Row = Row>(
 ): message is ControlMessage {
   return !isChangeMessage(message)
 }
+
+export function isUpToDateMessage<T extends Row = Row>(
+  message: Message<T>
+): message is ControlMessage & { up_to_date: true } {
+  return isControlMessage(message) && message.headers.control === `up-to-date`
+}

package/src/index.ts

@@ -1,3 +1,6 @@
 export * from './client'
+export * from './shape'
 export * from './types'
-export * from './helpers'
+export { isChangeMessage, isControlMessage } from './helpers'
+export { FetchError } from './error'
+export { type BackoffOptions, BackoffDefaults } from './fetch'

package/src/queue.ts

@@ -0,0 +1,62 @@
+import { MaybePromise } from './types'
+
+function isThenable(value: MaybePromise<void>): value is Promise<void> {
+  return (
+    !!value &&
+    typeof value === `object` &&
+    `then` in value &&
+    typeof value.then === `function`
+  )
+}
+
+/**
+ * Processes messages asynchronously in order.
+ */
+export class AsyncProcessingQueue {
+  #processingChain: MaybePromise<void> = undefined
+
+  public process(callback: () => MaybePromise<void>): MaybePromise<void> {
+    this.#processingChain = isThenable(this.#processingChain)
+      ? this.#processingChain.then(callback)
+      : callback()
+    return this.#processingChain
+  }
+
+  public async waitForProcessing(): Promise<void> {
+    let currentChain: MaybePromise<void>
+    do {
+      currentChain = this.#processingChain
+      await currentChain
+    } while (this.#processingChain !== currentChain)
+  }
+}
+
+export interface MessageProcessorInterface<T> {
+  process(messages: T): MaybePromise<void>
+  waitForProcessing(): Promise<void>
+}
+
+/**
+ * Receives messages, puts them on a queue and processes
+ * them synchronously or asynchronously by passing to a
+ * registered callback function.
+ *
+ * @constructor
+ * @param {(message: T) => MaybePromise<void>} callback function
+ */
+export class MessageProcessor<T> implements MessageProcessorInterface<T> {
+  readonly #queue = new AsyncProcessingQueue()
+  readonly #callback: (messages: T) => MaybePromise<void>
+
+  constructor(callback: (messages: T) => MaybePromise<void>) {
+    this.#callback = callback
+  }
+
+  public process(messages: T): void {
+    this.#queue.process(() => this.#callback(messages))
+  }
+
+  public async waitForProcessing(): Promise<void> {
+    await this.#queue.waitForProcessing()
+  }
+}

package/src/shape.ts

@@ -0,0 +1,197 @@
+import { Message, Row } from './types'
+import { isChangeMessage, isControlMessage } from './helpers'
+import { FetchError } from './error'
+import { ShapeStreamInterface } from './client'
+
+export type ShapeData<T extends Row = Row> = Map<string, T>
+export type ShapeChangedCallback<T extends Row = Row> = (
+  value: ShapeData<T>
+) => void
+
+/**
+ * A Shape is an object that subscribes to a shape log,
+ * keeps a materialised shape `.value` in memory and
+ * notifies subscribers when the value has changed.
+ *
+ * It can be used without a framework and as a primitive
+ * to simplify developing framework hooks.
+ *
+ * @constructor
+ * @param {ShapeStream<T extends Row>} - the underlying shape stream
+ * @example
+ * ```
+ * const shapeStream = new ShapeStream<{ foo: number }>(url: 'http://localhost:3000/v1/shape/foo'})
+ * const shape = new Shape(shapeStream)
+ * ```
+ *
+ * `value` returns a promise that resolves the Shape data once the Shape has been
+ * fully loaded (and when resuming from being offline):
+ *
+ *     const value = await shape.value
+ *
+ * `valueSync` returns the current data synchronously:
+ *
+ *     const value = shape.valueSync
+ *
+ *  Subscribe to updates. Called whenever the shape updates in Postgres.
+ *
+ *     shape.subscribe(shapeData => {
+ *       console.log(shapeData)
+ *     })
+ */
+export class Shape<T extends Row = Row> {
+  readonly #stream: ShapeStreamInterface<T>
+
+  readonly #data: ShapeData<T> = new Map()
+  readonly #subscribers = new Map<number, ShapeChangedCallback<T>>()
+
+  #hasNotifiedSubscribersUpToDate: boolean = false
+  #error: FetchError | false = false
+
+  constructor(stream: ShapeStreamInterface<T>) {
+    this.#stream = stream
+    this.#stream.subscribe(
+      this.#process.bind(this),
+      this.#handleError.bind(this)
+    )
+    const unsubscribe = this.#stream.subscribeOnceToUpToDate(
+      () => {
+        unsubscribe()
+      },
+      (e) => {
+        this.#handleError(e)
+        throw e
+      }
+    )
+  }
+
+  get isUpToDate(): boolean {
+    return this.#stream.isUpToDate
+  }
+
+  get value(): Promise<ShapeData<T>> {
+    return new Promise((resolve, reject) => {
+      if (this.#stream.isUpToDate) {
+        resolve(this.valueSync)
+      } else {
+        const unsubscribe = this.subscribe((shapeData) => {
+          unsubscribe()
+          if (this.#error) reject(this.#error)
+          resolve(shapeData)
+        })
+      }
+    })
+  }
+
+  get valueSync() {
+    return this.#data
+  }
+
+  get error() {
+    return this.#error
+  }
+
+  /** Unix time at which we last synced. Undefined when `isLoading` is true. */
+  lastSyncedAt(): number | undefined {
+    return this.#stream.lastSyncedAt()
+  }
+
+  /** Time elapsed since last sync (in ms). Infinity if we did not yet sync. */
+  lastSynced() {
+    return this.#stream.lastSynced()
+  }
+
+  /** True during initial fetch. False afterwise.  */
+  isLoading() {
+    return this.#stream.isLoading()
+  }
+
+  /** Indicates if we are connected to the Electric sync service. */
+  isConnected(): boolean {
+    return this.#stream.isConnected()
+  }
+
+  subscribe(callback: ShapeChangedCallback<T>): () => void {
+    const subscriptionId = Math.random()
+
+    this.#subscribers.set(subscriptionId, callback)
+
+    return () => {
+      this.#subscribers.delete(subscriptionId)
+    }
+  }
+
+  unsubscribeAll(): void {
+    this.#subscribers.clear()
+  }
+
+  get numSubscribers() {
+    return this.#subscribers.size
+  }
+
+  #process(messages: Message<T>[]): void {
+    let dataMayHaveChanged = false
+    let isUpToDate = false
+    let newlyUpToDate = false
+
+    messages.forEach((message) => {
+      if (isChangeMessage(message)) {
+        dataMayHaveChanged = [`insert`, `update`, `delete`].includes(
+          message.headers.operation
+        )
+
+        switch (message.headers.operation) {
+          case `insert`:
+            this.#data.set(message.key, message.value)
+            break
+          case `update`:
+            this.#data.set(message.key, {
+              ...this.#data.get(message.key)!,
+              ...message.value,
+            })
+            break
+          case `delete`:
+            this.#data.delete(message.key)
+            break
+        }
+      }
+
+      if (isControlMessage(message)) {
+        switch (message.headers.control) {
+          case `up-to-date`:
+            isUpToDate = true
+            if (!this.#hasNotifiedSubscribersUpToDate) {
+              newlyUpToDate = true
+            }
+            break
+          case `must-refetch`:
+            this.#data.clear()
+            this.#error = false
+            isUpToDate = false
+            newlyUpToDate = false
+            break
+        }
+      }
+    })
+
+    // Always notify subscribers when the Shape first is up to date.
+    // FIXME this would be cleaner with a simple state machine.
+    if (newlyUpToDate || (isUpToDate && dataMayHaveChanged)) {
+      this.#hasNotifiedSubscribersUpToDate = true
+      this.#notify()
+    }
+  }
+
+  #handleError(e: Error): void {
+    if (e instanceof FetchError) {
+      this.#error = e
+      this.#notify()
+    }
+  }
+
+  #notify(): void {
+    this.#subscribers.forEach((callback) => {
+      callback(this.valueSync)
+    })
+  }
+}

package/src/types.ts

@@ -108,3 +108,5 @@ export type TypedMessages<T extends Row = Row> = {
   messages: Array<Message<T>>
   schema: ColumnInfo
 }
+
+export type MaybePromise<T> = T | Promise<T>