@electric-sql/client 1.0.9 → 1.0.11

package/src/shape.ts

@@ -1,7 +1,7 @@
 import { Message, Offset, Row } from './types'
 import { isChangeMessage, isControlMessage } from './helpers'
 import { FetchError } from './error'
-import { ShapeStreamInterface } from './client'
+import { LogMode, ShapeStreamInterface } from './client'
 
 export type ShapeData<T extends Row<unknown> = Row> = Map<string, T>
 export type ShapeChangedCallback<T extends Row<unknown> = Row> = (data: {
@@ -52,6 +52,9 @@ export class Shape<T extends Row<unknown> = Row> {
 
   readonly #data: ShapeData<T> = new Map()
   readonly #subscribers = new Map<number, ShapeChangedCallback<T>>()
+  readonly #insertedKeys = new Set<string>()
+  readonly #requestedSubSnapshots = new Set<string>()
+  #reexecuteSnapshotsPending = false
   #status: ShapeStatus = `syncing`
   #error: FetchError | false = false
 
@@ -125,6 +128,26 @@ export class Shape<T extends Row<unknown> = Row> {
     return this.stream.isConnected()
   }
 
+  /** Current log mode of the underlying stream */
+  get mode(): LogMode {
+    return this.stream.mode
+  }
+
+  /**
+   * Request a snapshot for subset of data. Only available when mode is changes_only.
+   * Returns void; data will be emitted via the stream and processed by this Shape.
+   */
+  async requestSnapshot(
+    params: Parameters<ShapeStreamInterface<T>[`requestSnapshot`]>[0]
+  ): Promise<void> {
+    // Track this snapshot request for future re-execution on shape rotation
+    const key = JSON.stringify(params)
+    this.#requestedSubSnapshots.add(key)
+    // Ensure the stream is up-to-date so schema is available for parsing
+    await this.#awaitUpToDate()
+    await this.stream.requestSnapshot(params)
+  }
+
   subscribe(callback: ShapeChangedCallback<T>): () => void {
     const subscriptionId = Math.random()
 
@@ -149,19 +172,43 @@ export class Shape<T extends Row<unknown> = Row> {
     messages.forEach((message) => {
       if (isChangeMessage(message)) {
         shouldNotify = this.#updateShapeStatus(`syncing`)
-        switch (message.headers.operation) {
-          case `insert`:
-            this.#data.set(message.key, message.value)
-            break
-          case `update`:
-            this.#data.set(message.key, {
-              ...this.#data.get(message.key)!,
-              ...message.value,
-            })
-            break
-          case `delete`:
-            this.#data.delete(message.key)
-            break
+        if (this.mode === `full`) {
+          switch (message.headers.operation) {
+            case `insert`:
+              this.#data.set(message.key, message.value)
+              break
+            case `update`:
+              this.#data.set(message.key, {
+                ...this.#data.get(message.key)!,
+                ...message.value,
+              })
+              break
+            case `delete`:
+              this.#data.delete(message.key)
+              break
+          }
+        } else {
+          // changes_only: only apply updates/deletes for keys for which we observed an insert
+          switch (message.headers.operation) {
+            case `insert`:
+              this.#insertedKeys.add(message.key)
+              this.#data.set(message.key, message.value)
+              break
+            case `update`:
+              if (this.#insertedKeys.has(message.key)) {
+                this.#data.set(message.key, {
+                  ...this.#data.get(message.key)!,
+                  ...message.value,
+                })
+              }
+              break
+            case `delete`:
+              if (this.#insertedKeys.has(message.key)) {
+                this.#data.delete(message.key)
+                this.#insertedKeys.delete(message.key)
+              }
+              break
+          }
         }
       }
 
@@ -169,11 +216,18 @@ export class Shape<T extends Row<unknown> = Row> {
         switch (message.headers.control) {
           case `up-to-date`:
             shouldNotify = this.#updateShapeStatus(`up-to-date`)
+            if (this.#reexecuteSnapshotsPending) {
+              this.#reexecuteSnapshotsPending = false
+              void this.#reexecuteSnapshots()
+            }
             break
           case `must-refetch`:
             this.#data.clear()
+            this.#insertedKeys.clear()
             this.#error = false
             shouldNotify = this.#updateShapeStatus(`syncing`)
+            // Flag to re-execute sub-snapshots once the new shape is up-to-date
+            this.#reexecuteSnapshotsPending = true
             break
         }
       }
@@ -182,6 +236,42 @@ export class Shape<T extends Row<unknown> = Row> {
     if (shouldNotify) this.#notify()
   }
 
+  async #reexecuteSnapshots(): Promise<void> {
+    // Wait until stream is up-to-date again (ensures schema is available)
+    await this.#awaitUpToDate()
+
+    // Re-execute all snapshots concurrently
+    await Promise.all(
+      Array.from(this.#requestedSubSnapshots).map(async (jsonParams) => {
+        try {
+          const snapshot = JSON.parse(jsonParams)
+          await this.stream.requestSnapshot(snapshot)
+        } catch (_) {
+          // Ignore and continue; errors will be surfaced via stream onError
+        }
+      })
+    )
+  }
+
+  async #awaitUpToDate(): Promise<void> {
+    if (this.stream.isUpToDate) return
+    await new Promise<void>((resolve) => {
+      const check = () => {
+        if (this.stream.isUpToDate) {
+          clearInterval(interval)
+          unsub()
+          resolve()
+        }
+      }
+      const interval = setInterval(check, 10)
+      const unsub = this.stream.subscribe(
+        () => check(),
+        () => check()
+      )
+      check()
+    })
+  }
+
   #updateShapeStatus(status: ShapeStatus): boolean {
     const stateChanged = this.#status !== status
     this.#status = status

package/src/snapshot-tracker.ts

@@ -0,0 +1,88 @@
+import { isVisibleInSnapshot } from './helpers'
+import { Row, SnapshotMetadata } from './types'
+import { ChangeMessage } from './types'
+
+/**
+ * Tracks active snapshots and filters out duplicate change messages that are already included in snapshots.
+ *
+ * When requesting a snapshot in changes_only mode, we need to track which transactions were included in the
+ * snapshot to avoid processing duplicate changes that arrive via the live stream. This class maintains that
+ * tracking state and provides methods to:
+ *
+ * - Add new snapshots for tracking via addSnapshot()
+ * - Remove completed snapshots via removeSnapshot()
+ * - Check if incoming changes should be filtered via shouldRejectMessage()
+ */
+export class SnapshotTracker {
+  private activeSnapshots: Map<
+    number,
+    { xmin: bigint; xmax: bigint; xip_list: bigint[]; keys: Set<string> }
+  > = new Map()
+  private xmaxSnapshots: Map<bigint, Set<number>> = new Map()
+  private snapshotsByDatabaseLsn: Map<bigint, Set<number>> = new Map()
+
+  /**
+   * Add a new snapshot for tracking
+   */
+  addSnapshot(metadata: SnapshotMetadata, keys: Set<string>): void {
+    this.activeSnapshots.set(metadata.snapshot_mark, {
+      xmin: BigInt(metadata.xmin),
+      xmax: BigInt(metadata.xmax),
+      xip_list: metadata.xip_list.map(BigInt),
+      keys,
+    })
+    const xmaxSet =
+      this.xmaxSnapshots
+        .get(BigInt(metadata.xmax))
+        ?.add(metadata.snapshot_mark) ?? new Set([metadata.snapshot_mark])
+    this.xmaxSnapshots.set(BigInt(metadata.xmax), xmaxSet)
+    const databaseLsnSet =
+      this.snapshotsByDatabaseLsn
+        .get(BigInt(metadata.database_lsn))
+        ?.add(metadata.snapshot_mark) ?? new Set([metadata.snapshot_mark])
+    this.snapshotsByDatabaseLsn.set(
+      BigInt(metadata.database_lsn),
+      databaseLsnSet
+    )
+  }
+
+  /**
+   * Remove a snapshot from tracking
+   */
+  removeSnapshot(snapshotMark: number): void {
+    this.activeSnapshots.delete(snapshotMark)
+  }
+
+  /**
+   * Check if a change message should be filtered because its already in an active snapshot
+   * Returns true if the message should be filtered out (not processed)
+   */
+  shouldRejectMessage(message: ChangeMessage<Row<unknown>>): boolean {
+    const txids = message.headers.txids || []
+    if (txids.length === 0) return false
+
+    const xid = Math.max(...txids) // Use the maximum transaction ID
+
+    for (const [xmax, snapshots] of this.xmaxSnapshots.entries()) {
+      if (xid >= xmax) {
+        for (const snapshot of snapshots) {
+          this.removeSnapshot(snapshot)
+        }
+      }
+    }
+
+    return [...this.activeSnapshots.values()].some(
+      (x) => x.keys.has(message.key) && isVisibleInSnapshot(xid, x)
+    )
+  }
+
+  lastSeenUpdate(newDatabaseLsn: bigint): void {
+    for (const [dbLsn, snapshots] of this.snapshotsByDatabaseLsn.entries()) {
+      if (dbLsn <= newDatabaseLsn) {
+        for (const snapshot of snapshots) {
+          this.removeSnapshot(snapshot)
+        }
+      }
+    }
+  }
+}

package/src/types.ts

@@ -14,10 +14,33 @@ export type Value<Extensions = never> =
 
 export type Row<Extensions = never> = Record<string, Value<Extensions>>
 
-export type GetExtensions<T extends Row<unknown>> =
-  T extends Row<infer Extensions> ? Extensions : never
+// Check if `T` extends the base Row type without extensions
+// if yes, it has no extensions so we return `never`
+// otherwise, we infer the extensions from the Row type
+export type GetExtensions<T> = [T] extends [Row<never>]
+  ? never
+  : [T] extends [Row<infer E>]
+    ? E
+    : never
+
+export type Offset =
+  | `-1`
+  | `now`
+  | `${number}_${number}`
+  | `${bigint}_${number}`
+
+/** Information about transaction visibility for a snapshot. All fields are encoded as strings, but should be treated as uint64. */
+export type PostgresSnapshot = {
+  xmin: `${bigint}`
+  xmax: `${bigint}`
+  xip_list: `${bigint}`[]
+}
 
-export type Offset = `-1` | `${number}_${number}` | `${bigint}_${number}`
+export type NormalizedPgSnapshot = {
+  xmin: bigint
+  xmax: bigint
+  xip_list: bigint[]
+}
 
 interface Header {
   [key: Exclude<string, `operation` | `control`>]: Value
@@ -26,17 +49,19 @@ interface Header {
 export type Operation = `insert` | `update` | `delete`
 
 export type ControlMessage = {
-  headers: Header & {
-    control: `up-to-date` | `must-refetch`
-    global_last_seen_lsn?: string
-  }
+  headers:
+    | (Header & {
+        control: `up-to-date` | `must-refetch`
+        global_last_seen_lsn?: string
+      })
+    | (Header & { control: `snapshot-end` } & PostgresSnapshot)
 }
 
 export type ChangeMessage<T extends Row<unknown> = Row> = {
   key: string
   value: T
   old_value?: Partial<T> // Only provided for updates if `replica` is `full`
-  headers: Header & { operation: Operation }
+  headers: Header & { operation: Operation; txids?: number[] }
 }
 
 // Define the type for a record
@@ -125,3 +150,18 @@ export type TypedMessages<T extends Row<unknown> = Row> = {
 }
 
 export type MaybePromise<T> = T | Promise<T>
+
+/**
+ * Metadata that allows the consumer to know which changes have been incorporated into this snapshot.
+ *
+ * For any data that has a known transaction ID `xid` (and e.g. a key that's part of the snapshot):
+ * - if `xid` < `xmin` - included, change can be skipped
+ * - if `xid` < `xmax` AND `xid` not in `xip` - included, change can be skipped
+ * - if `xid` < `xmax` AND `xid` in `xip` - parallel, not included, change must be processed
+ * - if `xid` >= `xmax` - not included, change must be processed, and we can stop filtering after we see this
+ */
+export type SnapshotMetadata = {
+  /** Random number that's reflected in the `snapshot_mark` header on the snapshot items. */
+  snapshot_mark: number
+  database_lsn: string
+} & PostgresSnapshot