@electric-sql/client 1.0.10 → 1.0.11

package/src/snapshot-tracker.ts

@@ -0,0 +1,88 @@
+import { isVisibleInSnapshot } from './helpers'
+import { Row, SnapshotMetadata } from './types'
+import { ChangeMessage } from './types'
+
+/**
+ * Tracks active snapshots and filters out duplicate change messages that are already included in snapshots.
+ *
+ * When requesting a snapshot in changes_only mode, we need to track which transactions were included in the
+ * snapshot to avoid processing duplicate changes that arrive via the live stream. This class maintains that
+ * tracking state and provides methods to:
+ *
+ * - Add new snapshots for tracking via addSnapshot()
+ * - Remove completed snapshots via removeSnapshot()
+ * - Check if incoming changes should be filtered via shouldRejectMessage()
+ */
+export class SnapshotTracker {
+  private activeSnapshots: Map<
+    number,
+    { xmin: bigint; xmax: bigint; xip_list: bigint[]; keys: Set<string> }
+  > = new Map()
+  private xmaxSnapshots: Map<bigint, Set<number>> = new Map()
+  private snapshotsByDatabaseLsn: Map<bigint, Set<number>> = new Map()
+
+  /**
+   * Add a new snapshot for tracking
+   */
+  addSnapshot(metadata: SnapshotMetadata, keys: Set<string>): void {
+    this.activeSnapshots.set(metadata.snapshot_mark, {
+      xmin: BigInt(metadata.xmin),
+      xmax: BigInt(metadata.xmax),
+      xip_list: metadata.xip_list.map(BigInt),
+      keys,
+    })
+    const xmaxSet =
+      this.xmaxSnapshots
+        .get(BigInt(metadata.xmax))
+        ?.add(metadata.snapshot_mark) ?? new Set([metadata.snapshot_mark])
+    this.xmaxSnapshots.set(BigInt(metadata.xmax), xmaxSet)
+    const databaseLsnSet =
+      this.snapshotsByDatabaseLsn
+        .get(BigInt(metadata.database_lsn))
+        ?.add(metadata.snapshot_mark) ?? new Set([metadata.snapshot_mark])
+    this.snapshotsByDatabaseLsn.set(
+      BigInt(metadata.database_lsn),
+      databaseLsnSet
+    )
+  }
+
+  /**
+   * Remove a snapshot from tracking
+   */
+  removeSnapshot(snapshotMark: number): void {
+    this.activeSnapshots.delete(snapshotMark)
+  }
+
+  /**
+   * Check if a change message should be filtered because its already in an active snapshot
+   * Returns true if the message should be filtered out (not processed)
+   */
+  shouldRejectMessage(message: ChangeMessage<Row<unknown>>): boolean {
+    const txids = message.headers.txids || []
+    if (txids.length === 0) return false
+
+    const xid = Math.max(...txids) // Use the maximum transaction ID
+
+    for (const [xmax, snapshots] of this.xmaxSnapshots.entries()) {
+      if (xid >= xmax) {
+        for (const snapshot of snapshots) {
+          this.removeSnapshot(snapshot)
+        }
+      }
+    }
+
+    return [...this.activeSnapshots.values()].some(
+      (x) => x.keys.has(message.key) && isVisibleInSnapshot(xid, x)
+    )
+  }
+
+  lastSeenUpdate(newDatabaseLsn: bigint): void {
+    for (const [dbLsn, snapshots] of this.snapshotsByDatabaseLsn.entries()) {
+      if (dbLsn <= newDatabaseLsn) {
+        for (const snapshot of snapshots) {
+          this.removeSnapshot(snapshot)
+        }
+      }
+    }
+  }
+}

package/src/types.ts

@@ -23,7 +23,24 @@ export type GetExtensions<T> = [T] extends [Row<never>]
     ? E
     : never
 
-export type Offset = `-1` | `${number}_${number}` | `${bigint}_${number}`
+export type Offset =
+  | `-1`
+  | `now`
+  | `${number}_${number}`
+  | `${bigint}_${number}`
+
+/** Information about transaction visibility for a snapshot. All fields are encoded as strings, but should be treated as uint64. */
+export type PostgresSnapshot = {
+  xmin: `${bigint}`
+  xmax: `${bigint}`
+  xip_list: `${bigint}`[]
+}
+
+export type NormalizedPgSnapshot = {
+  xmin: bigint
+  xmax: bigint
+  xip_list: bigint[]
+}
 
 interface Header {
   [key: Exclude<string, `operation` | `control`>]: Value
@@ -32,17 +49,19 @@ interface Header {
 export type Operation = `insert` | `update` | `delete`
 
 export type ControlMessage = {
-  headers: Header & {
-    control: `up-to-date` | `must-refetch`
-    global_last_seen_lsn?: string
-  }
+  headers:
+    | (Header & {
+        control: `up-to-date` | `must-refetch`
+        global_last_seen_lsn?: string
+      })
+    | (Header & { control: `snapshot-end` } & PostgresSnapshot)
 }
 
 export type ChangeMessage<T extends Row<unknown> = Row> = {
   key: string
   value: T
   old_value?: Partial<T> // Only provided for updates if `replica` is `full`
-  headers: Header & { operation: Operation }
+  headers: Header & { operation: Operation; txids?: number[] }
 }
 
 // Define the type for a record
@@ -131,3 +150,18 @@ export type TypedMessages<T extends Row<unknown> = Row> = {
 }
 
 export type MaybePromise<T> = T | Promise<T>
+
+/**
+ * Metadata that allows the consumer to know which changes have been incorporated into this snapshot.
+ *
+ * For any data that has a known transaction ID `xid` (and e.g. a key that's part of the snapshot):
+ * - if `xid` < `xmin` - included, change can be skipped
+ * - if `xid` < `xmax` AND `xid` not in `xip` - included, change can be skipped
+ * - if `xid` < `xmax` AND `xid` in `xip` - parallel, not included, change must be processed
+ * - if `xid` >= `xmax` - not included, change must be processed, and we can stop filtering after we see this
+ */
+export type SnapshotMetadata = {
+  /** Random number that's reflected in the `snapshot_mark` header on the snapshot items. */
+  snapshot_mark: number
+  database_lsn: string
+} & PostgresSnapshot