@electric-sql/client 1.0.9 → 1.0.11

package/src/client.ts

@@ -5,9 +5,11 @@ import {
   Row,
   MaybePromise,
   GetExtensions,
+  ChangeMessage,
+  SnapshotMetadata,
 } from './types'
-import { MessageParser, Parser } from './parser'
-import { getOffset, isUpToDateMessage } from './helpers'
+import { MessageParser, Parser, TransformFunction } from './parser'
+import { getOffset, isUpToDateMessage, isChangeMessage } from './helpers'
 import {
   FetchError,
   FetchBackoffAbortError,
@@ -28,6 +30,7 @@ import {
   CHUNK_LAST_OFFSET_HEADER,
   LIVE_CACHE_BUSTER_HEADER,
   LIVE_CACHE_BUSTER_QUERY_PARAM,
+  EXPIRED_HANDLE_QUERY_PARAM,
   COLUMNS_QUERY_PARAM,
   LIVE_QUERY_PARAM,
   OFFSET_QUERY_PARAM,
@@ -41,11 +44,20 @@ import {
   FORCE_DISCONNECT_AND_REFRESH,
   PAUSE_STREAM,
   EXPERIMENTAL_LIVE_SSE_QUERY_PARAM,
+  ELECTRIC_PROTOCOL_QUERY_PARAMS,
+  LOG_MODE_QUERY_PARAM,
+  SUBSET_PARAM_WHERE,
+  SUBSET_PARAM_WHERE_PARAMS,
+  SUBSET_PARAM_LIMIT,
+  SUBSET_PARAM_OFFSET,
+  SUBSET_PARAM_ORDER_BY,
 } from './constants'
 import {
   EventSourceMessage,
   fetchEventSource,
 } from '@microsoft/fetch-event-source'
+import { expiredShapesCache } from './expired-shapes-cache'
+import { SnapshotTracker } from './snapshot-tracker'
 
 const RESERVED_PARAMS: Set<ReservedParamKeys> = new Set([
   LIVE_CACHE_BUSTER_QUERY_PARAM,
@@ -55,6 +67,7 @@ const RESERVED_PARAMS: Set<ReservedParamKeys> = new Set([
 ])
 
 type Replica = `full` | `default`
+export type LogMode = `changes_only` | `full`
 
 /**
  * PostgreSQL-specific shape parameters that can be provided externally
@@ -111,11 +124,20 @@ export type ExternalParamsRecord<T extends Row<unknown> = Row> = {
   [K in string]: ParamValue | undefined
 } & Partial<PostgresParams<T>> & { [K in ReservedParamKeys]?: never }
 
+export type SubsetParams = {
+  where?: string
+  params?: Record<string, string>
+  limit?: number
+  offset?: number
+  orderBy?: string
+}
+
 type ReservedParamKeys =
   | typeof LIVE_CACHE_BUSTER_QUERY_PARAM
   | typeof SHAPE_HANDLE_QUERY_PARAM
   | typeof LIVE_QUERY_PARAM
   | typeof OFFSET_QUERY_PARAM
+  | `subset__${string}`
 
 /**
  * External headers type - what users provide.
@@ -255,10 +277,16 @@ export interface ShapeStreamOptions<T = never> {
    */
   experimentalLiveSse?: boolean
 
+  /**
+   * Initial data loading mode
+   */
+  mode?: LogMode
+
   signal?: AbortSignal
   fetchClient?: typeof fetch
   backoffOptions?: BackoffOptions
   parser?: Parser<T>
+  transformer?: TransformFunction<T>
 
   /**
    * A function for handling shapestream errors.
@@ -289,8 +317,37 @@ export interface ShapeStreamInterface<T extends Row<unknown> = Row> {
   lastOffset: Offset
   shapeHandle?: string
   error?: unknown
+  mode: LogMode
 
   forceDisconnectAndRefresh(): Promise<void>
+
+  requestSnapshot(params: {
+    where?: string
+    params?: Record<string, string>
+    limit: number
+    offset?: number
+    orderBy: string
+  }): Promise<{
+    metadata: SnapshotMetadata
+    data: Array<Message<T>>
+  }>
+}
+
+/**
+ * Creates a canonical shape key from a URL excluding only Electric protocol parameters
+ */
+function canonicalShapeKey(url: URL): string {
+  const cleanUrl = new URL(url.origin + url.pathname)
+
+  // Copy all params except Electric protocol ones that vary between requests
+  for (const [key, value] of url.searchParams) {
+    if (!ELECTRIC_PROTOCOL_QUERY_PARAMS.includes(key)) {
+      cleanUrl.searchParams.set(key, value)
+    }
+  }
+
+  cleanUrl.searchParams.sort()
+  return cleanUrl.toString()
 }
 
 /**
@@ -362,8 +419,10 @@ export class ShapeStream<T extends Row<unknown> = Row>
   #liveCacheBuster: string // Seconds since our Electric Epoch 😎
   #lastSyncedAt?: number // unix time
   #isUpToDate: boolean = false
+  #isMidStream: boolean = true
   #connected: boolean = false
   #shapeHandle?: string
+  #mode: LogMode
   #schema?: Schema
   #onError?: ShapeStreamErrorHandler
   #requestAbortController?: AbortController
@@ -372,6 +431,10 @@ export class ShapeStream<T extends Row<unknown> = Row>
   #tickPromiseResolver?: () => void
   #tickPromiseRejecter?: (reason?: unknown) => void
   #messageChain = Promise.resolve<void[]>([]) // promise chain for incoming messages
+  #snapshotTracker = new SnapshotTracker()
+  #activeSnapshotRequests = 0 // counter for concurrent snapshot requests
+  #midStreamPromise?: Promise<void>
+  #midStreamPromiseResolver?: () => void
 
   constructor(options: ShapeStreamOptions<GetExtensions<T>>) {
     this.options = { subscribe: true, ...options }
@@ -379,8 +442,12 @@ export class ShapeStream<T extends Row<unknown> = Row>
     this.#lastOffset = this.options.offset ?? `-1`
     this.#liveCacheBuster = ``
     this.#shapeHandle = this.options.handle
-    this.#messageParser = new MessageParser<T>(options.parser)
+    this.#messageParser = new MessageParser<T>(
+      options.parser,
+      options.transformer
+    )
     this.#onError = this.options.onError
+    this.#mode = this.options.mode ?? `full`
 
     const baseFetchClient =
       options.fetchClient ??
@@ -423,6 +490,10 @@ export class ShapeStream<T extends Row<unknown> = Row>
     return this.#lastOffset
   }
 
+  get mode() {
+    return this.#mode
+  }
+
   async #start(): Promise<void> {
     this.#started = true
 
@@ -517,6 +588,13 @@ export class ShapeStream<T extends Row<unknown> = Row>
         // with the newly provided shape handle, or a fallback
         // pseudo-handle based on the current one to act as a
         // consistent cache buster
+
+        // Store the current shape URL as expired to avoid future 409s
+        if (this.#shapeHandle) {
+          const shapeKey = canonicalShapeKey(fetchUrl)
+          expiredShapesCache.markExpired(shapeKey, this.#shapeHandle)
+        }
+
         const newShapeHandle =
           e.headers[SHAPE_HANDLE_HEADER] || `${this.#shapeHandle!}-next`
         this.#reset(newShapeHandle)
@@ -542,7 +620,11 @@ export class ShapeStream<T extends Row<unknown> = Row>
     return this.#requestShape()
   }
 
-  async #constructUrl(url: string, resumingFromPause: boolean) {
+  async #constructUrl(
+    url: string,
+    resumingFromPause: boolean,
+    subsetParams?: SubsetParams
+  ) {
     // Resolve headers and params in parallel
     const [requestHeaders, params] = await Promise.all([
       resolveHeaders(this.options.headers),
@@ -552,9 +634,7 @@ export class ShapeStream<T extends Row<unknown> = Row>
     ])
 
     // Validate params after resolution
-    if (params) {
-      validateParams(params)
-    }
+    if (params) validateParams(params)
 
     const fetchUrl = new URL(url)
 
@@ -581,8 +661,22 @@ export class ShapeStream<T extends Row<unknown> = Row>
       }
     }
 
+    if (subsetParams) {
+      if (subsetParams.where)
+        setQueryParam(fetchUrl, SUBSET_PARAM_WHERE, subsetParams.where)
+      if (subsetParams.params)
+        setQueryParam(fetchUrl, SUBSET_PARAM_WHERE_PARAMS, subsetParams.params)
+      if (subsetParams.limit)
+        setQueryParam(fetchUrl, SUBSET_PARAM_LIMIT, subsetParams.limit)
+      if (subsetParams.offset)
+        setQueryParam(fetchUrl, SUBSET_PARAM_OFFSET, subsetParams.offset)
+      if (subsetParams.orderBy)
+        setQueryParam(fetchUrl, SUBSET_PARAM_ORDER_BY, subsetParams.orderBy)
+    }
+
     // Add Electric's internal parameters
     fetchUrl.searchParams.set(OFFSET_QUERY_PARAM, this.#lastOffset)
+    fetchUrl.searchParams.set(LOG_MODE_QUERY_PARAM, this.#mode)
 
     if (this.#isUpToDate) {
       // If we are resuming from a paused state, we don't want to perform a live request
@@ -602,6 +696,13 @@ export class ShapeStream<T extends Row<unknown> = Row>
       fetchUrl.searchParams.set(SHAPE_HANDLE_QUERY_PARAM, this.#shapeHandle!)
     }
 
+    // Add cache buster for shapes known to be expired to prevent 409s
+    const shapeKey = canonicalShapeKey(fetchUrl)
+    const expiredHandle = expiredShapesCache.getExpiredHandle(shapeKey)
+    if (expiredHandle) {
+      fetchUrl.searchParams.set(EXPIRED_HANDLE_QUERY_PARAM, expiredHandle)
+    }
+
     // sort query params in-place for stable URLs and improved cache hits
     fetchUrl.searchParams.sort()
 
@@ -667,6 +768,9 @@ export class ShapeStream<T extends Row<unknown> = Row>
   async #onMessages(batch: Array<Message<T>>, isSseMessage = false) {
     // Update isUpToDate
     if (batch.length > 0) {
+      // Set isMidStream to true when we receive any data
+      this.#isMidStream = true
+
       const lastMessage = batch[batch.length - 1]
       if (isUpToDateMessage(lastMessage)) {
         if (isSseMessage) {
@@ -680,9 +784,21 @@ export class ShapeStream<T extends Row<unknown> = Row>
         }
         this.#lastSyncedAt = Date.now()
         this.#isUpToDate = true
+        // Set isMidStream to false when we see an up-to-date message
+        this.#isMidStream = false
+        // Resolve the promise waiting for mid-stream to end
+        this.#midStreamPromiseResolver?.()
       }
 
-      await this.#publish(batch)
+      // Filter messages using snapshot tracker
+      const messagesToProcess = batch.filter((message) => {
+        if (isChangeMessage(message)) {
+          return !this.#snapshotTracker.shouldRejectMessage(message)
+        }
+        return true // Always process control messages
+      })
+
+      await this.#publish(messagesToProcess)
     }
   }
 
@@ -866,6 +982,24 @@ export class ShapeStream<T extends Row<unknown> = Row>
     return this.#tickPromise
   }
 
+  /** Await until we're not in the middle of a stream (i.e., until we see an up-to-date message) */
+  async #waitForStreamEnd() {
+    if (!this.#isMidStream) {
+      return
+    }
+    if (this.#midStreamPromise) {
+      return this.#midStreamPromise
+    }
+    this.#midStreamPromise = new Promise((resolve) => {
+      this.#midStreamPromiseResolver = resolve
+    })
+    this.#midStreamPromise.finally(() => {
+      this.#midStreamPromise = undefined
+      this.#midStreamPromiseResolver = undefined
+    })
+    return this.#midStreamPromise
+  }
+
   /**
    * Refreshes the shape stream.
    * This preemptively aborts any ongoing long poll and reconnects without
@@ -938,8 +1072,108 @@ export class ShapeStream<T extends Row<unknown> = Row>
     this.#liveCacheBuster = ``
     this.#shapeHandle = handle
     this.#isUpToDate = false
+    this.#isMidStream = true
     this.#connected = false
     this.#schema = undefined
+    this.#activeSnapshotRequests = 0
+  }
+
+  /**
+   * Request a snapshot for subset of data.
+   *
+   * Only available when mode is `changes_only`.
+   * Returns the insertion point & the data, but more importantly injects the data
+   * into the subscribed data stream. Returned value is unlikely to be useful for the caller,
+   * unless the caller has complicated additional logic.
+   *
+   * Data will be injected in a way that's also tracking further incoming changes, and it'll
+   * skip the ones that are already in the snapshot.
+   *
+   * @param opts - The options for the snapshot request.
+   * @returns The metadata and the data for the snapshot.
+   */
+  async requestSnapshot(opts: SubsetParams): Promise<{
+    metadata: SnapshotMetadata
+    data: Array<ChangeMessage<T>>
+  }> {
+    if (this.#mode === `full`) {
+      throw new Error(
+        `Snapshot requests are not supported in ${this.#mode} mode, as the consumer is guaranteed to observe all data`
+      )
+    }
+    // We shouldn't be getting a snapshot on a shape that's not started
+    if (!this.#started) await this.#start()
+
+    // Wait until we're not mid-stream before pausing
+    // This ensures we don't pause in the middle of a transaction
+    await this.#waitForStreamEnd()
+
+    // Pause the stream if this is the first snapshot request
+    this.#activeSnapshotRequests++
+
+    try {
+      if (this.#activeSnapshotRequests === 1) {
+        // Currently this cannot throw, but in case it can later it's in this try block to not have a stuck counter
+        this.#pause()
+      }
+
+      const { fetchUrl, requestHeaders } = await this.#constructUrl(
+        this.options.url,
+        true,
+        opts
+      )
+
+      const { metadata, data } = await this.#fetchSnapshot(
+        fetchUrl,
+        requestHeaders
+      )
+
+      const dataWithEndBoundary = (data as Array<Message<T>>).concat([
+        { headers: { control: `snapshot-end`, ...metadata } },
+      ])
+
+      this.#snapshotTracker.addSnapshot(
+        metadata,
+        new Set(data.map((message) => message.key))
+      )
+      this.#onMessages(dataWithEndBoundary, false)
+
+      return {
+        metadata,
+        data,
+      }
+    } finally {
+      // Resume the stream if this was the last snapshot request
+      this.#activeSnapshotRequests--
+      if (this.#activeSnapshotRequests === 0) {
+        this.#resume()
+      }
+    }
+  }
+
+  async #fetchSnapshot(url: URL, headers: Record<string, string>) {
+    const response = await this.#fetchClient(url.toString(), { headers })
+
+    if (!response.ok) {
+      throw new FetchError(
+        response.status,
+        undefined,
+        undefined,
+        Object.fromEntries([...response.headers.entries()]),
+        url.toString()
+      )
+    }
+
+    const { metadata, data } = await response.json()
+    const batch = this.#messageParser.parse<Array<ChangeMessage<T>>>(
+      JSON.stringify(data),
+      this.#schema!
+    )
+
+    return {
+      metadata,
+      data: batch,
+    }
   }
 }
 
@@ -969,6 +1203,7 @@ function validateOptions<T>(options: Partial<ShapeStreamOptions<T>>): void {
   if (
     options.offset !== undefined &&
     options.offset !== `-1` &&
+    options.offset !== `now` &&
     !options.handle
   ) {
     throw new MissingShapeHandleError()

package/src/constants.ts

@@ -5,6 +5,7 @@ export const SHAPE_SCHEMA_HEADER = `electric-schema`
 export const CHUNK_UP_TO_DATE_HEADER = `electric-up-to-date`
 export const COLUMNS_QUERY_PARAM = `columns`
 export const LIVE_CACHE_BUSTER_QUERY_PARAM = `cursor`
+export const EXPIRED_HANDLE_QUERY_PARAM = `expired_handle`
 export const SHAPE_HANDLE_QUERY_PARAM = `handle`
 export const LIVE_QUERY_PARAM = `live`
 export const OFFSET_QUERY_PARAM = `offset`
@@ -15,6 +16,12 @@ export const WHERE_PARAMS_PARAM = `params`
 export const EXPERIMENTAL_LIVE_SSE_QUERY_PARAM = `experimental_live_sse`
 export const FORCE_DISCONNECT_AND_REFRESH = `force-disconnect-and-refresh`
 export const PAUSE_STREAM = `pause-stream`
+export const LOG_MODE_QUERY_PARAM = `log`
+export const SUBSET_PARAM_WHERE = `subset__where`
+export const SUBSET_PARAM_LIMIT = `subset__limit`
+export const SUBSET_PARAM_OFFSET = `subset__offset`
+export const SUBSET_PARAM_ORDER_BY = `subset__order_by`
+export const SUBSET_PARAM_WHERE_PARAMS = `subset__params`
 
 // Query parameters that should be passed through when proxying Electric requests
 export const ELECTRIC_PROTOCOL_QUERY_PARAMS: Array<string> = [
@@ -22,4 +29,11 @@ export const ELECTRIC_PROTOCOL_QUERY_PARAMS: Array<string> = [
   SHAPE_HANDLE_QUERY_PARAM,
   OFFSET_QUERY_PARAM,
   LIVE_CACHE_BUSTER_QUERY_PARAM,
+  EXPIRED_HANDLE_QUERY_PARAM,
+  LOG_MODE_QUERY_PARAM,
+  SUBSET_PARAM_WHERE,
+  SUBSET_PARAM_LIMIT,
+  SUBSET_PARAM_OFFSET,
+  SUBSET_PARAM_ORDER_BY,
+  SUBSET_PARAM_WHERE_PARAMS,
 ]

package/src/expired-shapes-cache.ts

@@ -0,0 +1,72 @@
+interface ExpiredShapeCacheEntry {
+  expiredHandle: string
+  lastUsed: number
+}
+
+/**
+ * LRU cache for tracking expired shapes with automatic cleanup
+ */
+export class ExpiredShapesCache {
+  private data: Record<string, ExpiredShapeCacheEntry> = {}
+  private max: number = 250
+  private readonly storageKey = `electric_expired_shapes`
+
+  getExpiredHandle(shapeUrl: string): string | null {
+    const entry = this.data[shapeUrl]
+    if (entry) {
+      // Update last used time when accessed
+      entry.lastUsed = Date.now()
+      this.save()
+      return entry.expiredHandle
+    }
+    return null
+  }
+
+  markExpired(shapeUrl: string, handle: string): void {
+    this.data[shapeUrl] = { expiredHandle: handle, lastUsed: Date.now() }
+
+    const keys = Object.keys(this.data)
+    if (keys.length > this.max) {
+      const oldest = keys.reduce((min, k) =>
+        this.data[k].lastUsed < this.data[min].lastUsed ? k : min
+      )
+      delete this.data[oldest]
+    }
+
+    this.save()
+  }
+
+  private save(): void {
+    if (typeof localStorage === `undefined`) return
+    try {
+      localStorage.setItem(this.storageKey, JSON.stringify(this.data))
+    } catch {
+      // Ignore localStorage errors
+    }
+  }
+
+  private load(): void {
+    if (typeof localStorage === `undefined`) return
+    try {
+      const stored = localStorage.getItem(this.storageKey)
+      if (stored) {
+        this.data = JSON.parse(stored)
+      }
+    } catch {
+      // Ignore localStorage errors, start fresh
+      this.data = {}
+    }
+  }
+
+  constructor() {
+    this.load()
+  }
+
+  clear(): void {
+    this.data = {}
+    this.save()
+  }
+}
+
+// Module-level singleton instance
+export const expiredShapesCache = new ExpiredShapesCache()

package/src/fetch.ts

@@ -5,6 +5,11 @@ import {
   OFFSET_QUERY_PARAM,
   SHAPE_HANDLE_HEADER,
   SHAPE_HANDLE_QUERY_PARAM,
+  SUBSET_PARAM_LIMIT,
+  SUBSET_PARAM_OFFSET,
+  SUBSET_PARAM_ORDER_BY,
+  SUBSET_PARAM_WHERE,
+  SUBSET_PARAM_WHERE_PARAMS,
 } from './constants'
 import {
   FetchError,
@@ -206,11 +211,24 @@ export function createFetchWithResponseHeadersCheck(
 
       const addMissingHeaders = (requiredHeaders: Array<string>) =>
         missingHeaders.push(...requiredHeaders.filter((h) => !headers.has(h)))
-      addMissingHeaders(requiredElectricResponseHeaders)
 
       const input = args[0]
       const urlString = input.toString()
       const url = new URL(urlString)
+
+      // Snapshot responses (subset params) return a JSON object and do not include Electric chunk headers
+      const isSnapshotRequest = [
+        SUBSET_PARAM_WHERE,
+        SUBSET_PARAM_WHERE_PARAMS,
+        SUBSET_PARAM_LIMIT,
+        SUBSET_PARAM_OFFSET,
+        SUBSET_PARAM_ORDER_BY,
+      ].some((p) => url.searchParams.has(p))
+      if (isSnapshotRequest) {
+        return response
+      }
+
+      addMissingHeaders(requiredElectricResponseHeaders)
       if (url.searchParams.get(LIVE_QUERY_PARAM) === `true`) {
         addMissingHeaders(requiredLiveResponseHeaders)
       }

package/src/helpers.ts

@@ -1,4 +1,12 @@
-import { ChangeMessage, ControlMessage, Message, Offset, Row } from './types'
+import {
+  ChangeMessage,
+  ControlMessage,
+  Message,
+  NormalizedPgSnapshot,
+  Offset,
+  PostgresSnapshot,
+  Row,
+} from './types'
 
 /**
  * Type guard for checking {@link Message} is {@link ChangeMessage}.
@@ -64,3 +72,28 @@ export function getOffset(message: ControlMessage): Offset | undefined {
   }
   return `${lsn}_0` as Offset
 }
+
+/**
+ * Checks if a transaction is visible in a snapshot.
+ *
+ * @param txid - the transaction id to check
+ * @param snapshot - the information about the snapshot
+ * @returns true if the transaction is visible in the snapshot
+ */
+export function isVisibleInSnapshot(
+  txid: number | bigint | `${bigint}`,
+  snapshot: PostgresSnapshot | NormalizedPgSnapshot
+): boolean {
+  const xid = BigInt(txid)
+  const xmin = BigInt(snapshot.xmin)
+  const xmax = BigInt(snapshot.xmax)
+  const xip = snapshot.xip_list.map(BigInt)
+
+  // If the transaction id is less than the minimum transaction id, it is visible in the snapshot.
+  // If the transaction id is less than the maximum transaction id and not in the list of active
+  //   transactions at the time of the snapshot, it has been committed before the snapshot was taken
+  //   and is therefore visible in the snapshot.
+  // Otherwise, it is not visible in the snapshot.
+
+  return xid < xmin || (xid < xmax && !xip.includes(xid))
+}

package/src/index.ts

@@ -1,7 +1,11 @@
 export * from './client'
 export * from './shape'
 export * from './types'
-export { isChangeMessage, isControlMessage } from './helpers'
+export {
+  isChangeMessage,
+  isControlMessage,
+  isVisibleInSnapshot,
+} from './helpers'
 export { FetchError } from './error'
 export { type BackoffOptions, BackoffDefaults } from './fetch'
 export { ELECTRIC_PROTOCOL_QUERY_PARAMS } from './constants'

package/src/parser.ts

@@ -19,6 +19,10 @@ export type Parser<Extensions = never> = {
   [key: string]: ParseFunction<Extensions>
 }
 
+export type TransformFunction<Extensions = never> = (
+  message: Row<Extensions>
+) => Row<Extensions>
+
 const parseNumber = (value: string) => Number(value)
 const parseBool = (value: string) => value === `true` || value === `t`
 const parseBigInt = (value: string) => BigInt(value)
@@ -94,11 +98,16 @@ export function pgArrayParser<Extensions>(
 
 export class MessageParser<T extends Row<unknown>> {
   private parser: Parser<GetExtensions<T>>
-  constructor(parser?: Parser<GetExtensions<T>>) {
+  private transformer?: TransformFunction<GetExtensions<T>>
+  constructor(
+    parser?: Parser<GetExtensions<T>>,
+    transformer?: TransformFunction<GetExtensions<T>>
+  ) {
     // Merge the provided parser with the default parser
     // to use the provided parser whenever defined
     // and otherwise fall back to the default parser
     this.parser = { ...defaultParser, ...parser }
+    this.transformer = transformer
   }
 
   parse<Result>(messages: string, schema: Schema): Result {
@@ -118,6 +127,8 @@ export class MessageParser<T extends Row<unknown>> {
         Object.keys(row).forEach((key) => {
           row[key] = this.parseRow(key, row[key] as NullableToken, schema)
         })
+
+        if (this.transformer) value = this.transformer(value)
       }
       return value
     }) as Result