@electric-sql/client 1.0.0-beta.3 → 1.0.0-beta.5

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@electric-sql/client",
   "description": "Postgres everywhere - your data, in sync, wherever you need it.",
-  "version": "1.0.0-beta.3",
+  "version": "1.0.0-beta.5",
   "author": "ElectricSQL team and contributors.",
   "bugs": {
     "url": "https://github.com/electric-sql/electric/issues"

package/src/client.ts

@@ -36,6 +36,7 @@ import {
   WHERE_QUERY_PARAM,
   TABLE_QUERY_PARAM,
   REPLICA_PARAM,
+  FORCE_DISCONNECT_AND_REFRESH,
 } from './constants'
 
 const RESERVED_PARAMS: Set<ReservedParamKeys> = new Set([
@@ -68,7 +69,8 @@ export interface PostgresParams {
    * changed columns in an update.
    *
    * If it's `full` Electric will send the entire row with both changed and
-   * unchanged values.
+   * unchanged values. `old_value` will also be present on update messages,
+   * containing the previous value for changed columns.
    *
    * Setting `replica` to `full` will result in higher bandwidth
    * usage and so is not generally recommended.
@@ -254,11 +256,14 @@ export interface ShapeStreamInterface<T extends Row<unknown> = Row> {
   lastSyncedAt(): number | undefined
   lastSynced(): number
   isConnected(): boolean
+  hasStarted(): boolean
 
   isUpToDate: boolean
   lastOffset: Offset
   shapeHandle?: string
   error?: unknown
+
+  forceDisconnectAndRefresh(): Promise<void>
 }
 
 /**
@@ -323,6 +328,11 @@ export class ShapeStream<T extends Row<unknown> = Row>
   #shapeHandle?: string
   #schema?: Schema
   #onError?: ShapeStreamErrorHandler
+  #requestAbortController?: AbortController
+  #isRefreshing = false
+  #tickPromise?: Promise<void>
+  #tickPromiseResolver?: () => void
+  #tickPromiseRejecter?: (reason?: unknown) => void
 
   constructor(options: ShapeStreamOptions<GetExtensions<T>>) {
     this.options = { subscribe: true, ...options }
@@ -419,7 +429,9 @@ export class ShapeStream<T extends Row<unknown> = Row>
         fetchUrl.searchParams.set(OFFSET_QUERY_PARAM, this.#lastOffset)
 
         if (this.#isUpToDate) {
-          fetchUrl.searchParams.set(LIVE_QUERY_PARAM, `true`)
+          if (!this.#isRefreshing) {
+            fetchUrl.searchParams.set(LIVE_QUERY_PARAM, `true`)
+          }
           fetchUrl.searchParams.set(
             LIVE_CACHE_BUSTER_QUERY_PARAM,
             this.#liveCacheBuster
@@ -437,16 +449,44 @@ export class ShapeStream<T extends Row<unknown> = Row>
         // sort query params in-place for stable URLs and improved cache hits
         fetchUrl.searchParams.sort()
 
+        // Create a new AbortController for this request
+        this.#requestAbortController = new AbortController()
+
+        // If user provided a signal, listen to it and pass on the reason for the abort
+        let abortListener: (() => void) | undefined
+        if (signal) {
+          abortListener = () => {
+            this.#requestAbortController?.abort(signal.reason)
+          }
+          signal.addEventListener(`abort`, abortListener, { once: true })
+          if (signal.aborted) {
+            // If the signal is already aborted, abort the request immediately
+            this.#requestAbortController?.abort(signal.reason)
+          }
+        }
+
         let response!: Response
         try {
           response = await this.#fetchClient(fetchUrl.toString(), {
-            signal,
+            signal: this.#requestAbortController.signal,
             headers: requestHeaders,
           })
           this.#connected = true
         } catch (e) {
+          // Handle abort error triggered by refresh
+          if (
+            (e instanceof FetchError || e instanceof FetchBackoffAbortError) &&
+            this.#requestAbortController.signal.aborted &&
+            this.#requestAbortController.signal.reason ===
+              FORCE_DISCONNECT_AND_REFRESH
+          ) {
+            // Loop back to the top of the while loop to start a new request
+            continue
+          }
+
           if (e instanceof FetchBackoffAbortError) break // interrupted
           if (!(e instanceof FetchError)) throw e // should never happen
+
           if (e.status == 409) {
             // Upon receiving a 409, we should start from scratch
             // with the newly provided shape handle
@@ -462,6 +502,11 @@ export class ShapeStream<T extends Row<unknown> = Row>
             // so we exit the loop
             throw e
           }
+        } finally {
+          if (abortListener && signal) {
+            signal.removeEventListener(`abort`, abortListener)
+          }
+          this.#requestAbortController = undefined
         }
 
         const { headers, status } = response
@@ -505,6 +550,8 @@ export class ShapeStream<T extends Row<unknown> = Row>
 
           await this.#publish(batch)
         }
+
+        this.#tickPromiseResolver?.()
       }
     } catch (err) {
       this.#error = err
@@ -532,6 +579,7 @@ export class ShapeStream<T extends Row<unknown> = Row>
       throw err
     } finally {
       this.#connected = false
+      this.#tickPromiseRejecter?.()
     }
   }
 
@@ -574,6 +622,44 @@ export class ShapeStream<T extends Row<unknown> = Row>
     return !this.#isUpToDate
   }
 
+  hasStarted(): boolean {
+    return this.#started
+  }
+
+  /** Await the next tick of the request loop */
+  async #nextTick() {
+    if (this.#tickPromise) {
+      return this.#tickPromise
+    }
+    this.#tickPromise = new Promise((resolve, reject) => {
+      this.#tickPromiseResolver = resolve
+      this.#tickPromiseRejecter = reject
+    })
+    this.#tickPromise.finally(() => {
+      this.#tickPromise = undefined
+      this.#tickPromiseResolver = undefined
+      this.#tickPromiseRejecter = undefined
+    })
+    return this.#tickPromise
+  }
+
+  /**
+   * Refreshes the shape stream.
+   * This preemptively aborts any ongoing long poll and reconnects without
+   * long polling, ensuring that the stream receives an up to date message with the
+   * latest LSN from Postgres at that point in time.
+   */
+  async forceDisconnectAndRefresh(): Promise<void> {
+    this.#isRefreshing = true
+    if (this.#isUpToDate && !this.#requestAbortController?.signal.aborted) {
+      // If we are "up to date", any current request will be a "live" request
+      // and needs to be aborted
+      this.#requestAbortController?.abort(FORCE_DISCONNECT_AND_REFRESH)
+    }
+    await this.#nextTick()
+    this.#isRefreshing = false
+  }
+
   async #publish(messages: Message<T>[]): Promise<void> {
     await Promise.all(
       Array.from(this.#subscribers.values()).map(async ([callback, __]) => {

package/src/constants.ts

@@ -11,3 +11,4 @@ export const OFFSET_QUERY_PARAM = `offset`
 export const TABLE_QUERY_PARAM = `table`
 export const WHERE_QUERY_PARAM = `where`
 export const REPLICA_PARAM = `replica`
+export const FORCE_DISCONNECT_AND_REFRESH = `force-disconnect-and-refresh`

package/src/fetch.ts

@@ -264,10 +264,8 @@ class PrefetchQueue {
     const aborter = new AbortController()
 
     try {
-      const request = this.#fetchClient(url, {
-        ...(args[1] ?? {}),
-        signal: chainAborter(aborter, args[1]?.signal),
-      })
+      const { signal, cleanup } = chainAborter(aborter, args[1]?.signal)
+      const request = this.#fetchClient(url, { ...(args[1] ?? {}), signal })
       this.#prefetchQueue.set(url, [request, aborter])
       request
         .then((response) => {
@@ -286,6 +284,7 @@ class PrefetchQueue {
           return this.#prefetch(nextUrl, args[1])
         })
         .catch(() => {})
+        .finally(cleanup)
     } catch (_) {
       // ignore prefetch errors
     }
@@ -324,12 +323,31 @@ function getNextChunkUrl(url: string, res: Response): string | void {
 function chainAborter(
   aborter: AbortController,
   sourceSignal?: AbortSignal | null
-): AbortSignal {
-  if (!sourceSignal) return aborter.signal
-  if (sourceSignal.aborted) aborter.abort()
-  else
-    sourceSignal.addEventListener(`abort`, () => aborter.abort(), {
+): {
+  signal: AbortSignal
+  cleanup: () => void
+} {
+  let cleanup = noop
+  if (!sourceSignal) {
+    // no-op, nothing to chain to
+  } else if (sourceSignal.aborted) {
+    // source signal is already aborted, abort immediately
+    aborter.abort()
+  } else {
+    // chain to source signal abort event, and add callback to unlink
+    // the aborter to avoid memory leaks
+    const abortParent = () => aborter.abort()
+    sourceSignal.addEventListener(`abort`, abortParent, {
       once: true,
+      signal: aborter.signal,
     })
-  return aborter.signal
+    cleanup = () => sourceSignal.removeEventListener(`abort`, abortParent)
+  }
+
+  return {
+    signal: aborter.signal,
+    cleanup,
+  }
 }
+
+function noop() {}

package/src/parser.ts

@@ -104,7 +104,12 @@ export class MessageParser<T extends Row<unknown>> {
       // is needed because there could be a column named `value`
       // and the value associated to that column will be a string or null.
       // But `typeof null === 'object'` so we need to make an explicit check.
-      if (key === `value` && typeof value === `object` && value !== null) {
+      // We also parse the `old_value`, which appears on updates when `replica=full`.
+      if (
+        (key === `value` || key === `old_value`) &&
+        typeof value === `object` &&
+        value !== null
+      ) {
         // Parse the row values
         const row = value as Record<string, Value<GetExtensions<T>>>
         Object.keys(row).forEach((key) => {

package/src/shape.ts

@@ -9,6 +9,8 @@ export type ShapeChangedCallback<T extends Row<unknown> = Row> = (data: {
   rows: T[]
 }) => void
 
+type ShapeStatus = `syncing` | `up-to-date`
+
 /**
  * A Shape is an object that subscribes to a shape log,
  * keeps a materialised shape `.rows` in memory and
@@ -50,8 +52,7 @@ export class Shape<T extends Row<unknown> = Row> {
 
   readonly #data: ShapeData<T> = new Map()
   readonly #subscribers = new Map<number, ShapeChangedCallback<T>>()
-
-  #hasNotifiedSubscribersUpToDate: boolean = false
+  #status: ShapeStatus = `syncing`
   #error: FetchError | false = false
 
   constructor(stream: ShapeStreamInterface<T>) {
@@ -63,7 +64,7 @@ export class Shape<T extends Row<unknown> = Row> {
   }
 
   get isUpToDate(): boolean {
-    return this.stream.isUpToDate
+    return this.#status === `up-to-date`
   }
 
   get lastOffset(): Offset {
@@ -143,16 +144,11 @@ export class Shape<T extends Row<unknown> = Row> {
   }
 
   #process(messages: Message<T>[]): void {
-    let dataMayHaveChanged = false
-    let isUpToDate = false
-    let newlyUpToDate = false
+    let shouldNotify = false
 
     messages.forEach((message) => {
       if (isChangeMessage(message)) {
-        dataMayHaveChanged = [`insert`, `update`, `delete`].includes(
-          message.headers.operation
-        )
-
+        shouldNotify = this.#updateShapeStatus(`syncing`)
         switch (message.headers.operation) {
           case `insert`:
             this.#data.set(message.key, message.value)
@@ -172,28 +168,24 @@ export class Shape<T extends Row<unknown> = Row> {
       if (isControlMessage(message)) {
         switch (message.headers.control) {
           case `up-to-date`:
-            isUpToDate = true
-            if (!this.#hasNotifiedSubscribersUpToDate) {
-              newlyUpToDate = true
-            }
+            shouldNotify = this.#updateShapeStatus(`up-to-date`)
             break
           case `must-refetch`:
             this.#data.clear()
             this.#error = false
-            this.#hasNotifiedSubscribersUpToDate = false
-            isUpToDate = false
-            newlyUpToDate = false
+            shouldNotify = this.#updateShapeStatus(`syncing`)
             break
         }
       }
     })
 
-    // Always notify subscribers when the Shape first is up to date.
-    // FIXME this would be cleaner with a simple state machine.
-    if (newlyUpToDate || (isUpToDate && dataMayHaveChanged)) {
-      this.#hasNotifiedSubscribersUpToDate = true
-      this.#notify()
-    }
+    if (shouldNotify) this.#notify()
+  }
+
+  #updateShapeStatus(status: ShapeStatus): boolean {
+    const stateChanged = this.#status !== status
+    this.#status = status
+    return stateChanged && status === `up-to-date`
   }
 
   #handleError(e: Error): void {

package/src/types.ts

@@ -32,6 +32,7 @@ export type ControlMessage = {
 export type ChangeMessage<T extends Row<unknown> = Row> = {
   key: string
   value: T
+  old_value?: Partial<T> // Only provided for updates if `replica` is `full`
   headers: Header & { operation: Operation }
 }