@tanstack/electric-db-collection 0.2.15 → 0.2.16

package/src/electric.ts

@@ -14,8 +14,24 @@ import {
   TimeoutWaitingForTxIdError,
 } from './errors'
 import { compileSQL } from './sql-compiler'
+import {
+  addTagToIndex,
+  findRowsMatchingPattern,
+  getTagLength,
+  isMoveOutMessage,
+  removeTagFromIndex,
+  tagMatchesPattern,
+} from './tag-index'
+import type {
+  MoveOutPattern,
+  MoveTag,
+  ParsedMoveTag,
+  RowId,
+  TagIndex,
+} from './tag-index'
 import type {
   BaseCollectionConfig,
+  ChangeMessageOrDeleteKeyMessage,
   CollectionConfig,
   DeleteMutationFnParams,
   InsertMutationFnParams,
@@ -288,6 +304,15 @@ function isSnapshotEndMessage<T extends Row<unknown>>(
   return isControlMessage(message) && message.headers.control === `snapshot-end`
 }
 
+function isSubsetEndMessage<T extends Row<unknown>>(
+  message: Message<T>,
+): message is ControlMessage & { headers: { control: `subset-end` } } {
+  return (
+    isControlMessage(message) &&
+    (message.headers.control as string) === `subset-end`
+  )
+}
+
 function parseSnapshotMessage(message: SnapshotEndMessage): PostgresSnapshot {
   return {
     xmin: message.headers.xmin,
@@ -871,6 +896,231 @@ function createElectricSync<T extends Row<unknown>>(
   // Store for the relation schema information
   const relationSchema = new Store<string | undefined>(undefined)
 
+  const tagCache = new Map<MoveTag, ParsedMoveTag>()
+
+  // Parses a tag string into a MoveTag.
+  // It memoizes the result parsed tag such that future calls
+  // for the same tag string return the same MoveTag array.
+  const parseTag = (tag: MoveTag): ParsedMoveTag => {
+    const cachedTag = tagCache.get(tag)
+    if (cachedTag) {
+      return cachedTag
+    }
+
+    const parsedTag = tag.split(`|`)
+    tagCache.set(tag, parsedTag)
+    return parsedTag
+  }
+
+  // Tag tracking state
+  const rowTagSets = new Map<RowId, Set<MoveTag>>()
+  const tagIndex: TagIndex = []
+  let tagLength: number | undefined = undefined
+
+  /**
+   * Initialize the tag index with the correct length
+   */
+  const initializeTagIndex = (length: number): void => {
+    if (tagIndex.length < length) {
+      // Extend the index array to the required length
+      for (let i = tagIndex.length; i < length; i++) {
+        tagIndex[i] = new Map()
+      }
+    }
+  }
+
+  /**
+   * Add tags to a row and update the tag index
+   */
+  const addTagsToRow = (
+    tags: Array<MoveTag>,
+    rowId: RowId,
+    rowTagSet: Set<MoveTag>,
+  ): void => {
+    for (const tag of tags) {
+      const parsedTag = parseTag(tag)
+
+      // Infer tag length from first tag
+      if (tagLength === undefined) {
+        tagLength = getTagLength(parsedTag)
+        initializeTagIndex(tagLength)
+      }
+
+      // Validate tag length matches
+      const currentTagLength = getTagLength(parsedTag)
+      if (currentTagLength !== tagLength) {
+        debug(
+          `${collectionId ? `[${collectionId}] ` : ``}Tag length mismatch: expected ${tagLength}, got ${currentTagLength}`,
+        )
+        continue
+      }
+
+      rowTagSet.add(tag)
+      addTagToIndex(parsedTag, rowId, tagIndex, tagLength)
+    }
+  }
+
+  /**
+   * Remove tags from a row and update the tag index
+   */
+  const removeTagsFromRow = (
+    removedTags: Array<MoveTag>,
+    rowId: RowId,
+    rowTagSet: Set<MoveTag>,
+  ): void => {
+    if (tagLength === undefined) {
+      return
+    }
+
+    for (const tag of removedTags) {
+      const parsedTag = parseTag(tag)
+      rowTagSet.delete(tag)
+      removeTagFromIndex(parsedTag, rowId, tagIndex, tagLength)
+      // We aggresively evict the tag from the cache
+      // if this tag is shared with another row
+      // and is not removed from that other row
+      // then next time we encounter the tag it will be parsed again
+      tagCache.delete(tag)
+    }
+  }
+
+  /**
+   * Process tags for a change message (add and remove tags)
+   */
+  const processTagsForChangeMessage = (
+    tags: Array<MoveTag> | undefined,
+    removedTags: Array<MoveTag> | undefined,
+    rowId: RowId,
+  ): Set<MoveTag> => {
+    // Initialize tag set for this row if it doesn't exist (needed for checking deletion)
+    if (!rowTagSets.has(rowId)) {
+      rowTagSets.set(rowId, new Set())
+    }
+    const rowTagSet = rowTagSets.get(rowId)!
+
+    // Add new tags
+    if (tags) {
+      addTagsToRow(tags, rowId, rowTagSet)
+    }
+
+    // Remove tags
+    if (removedTags) {
+      removeTagsFromRow(removedTags, rowId, rowTagSet)
+    }
+
+    return rowTagSet
+  }
+
+  /**
+   * Clear all tag tracking state (used when truncating)
+   */
+  const clearTagTrackingState = (): void => {
+    rowTagSets.clear()
+    tagIndex.length = 0
+    tagLength = undefined
+  }
+
+  /**
+   * Remove all tags for a row from both the tag set and the index
+   * Used when a row is deleted
+   */
+  const clearTagsForRow = (rowId: RowId): void => {
+    if (tagLength === undefined) {
+      return
+    }
+
+    const rowTagSet = rowTagSets.get(rowId)
+    if (!rowTagSet) {
+      return
+    }
+
+    // Remove each tag from the index
+    for (const tag of rowTagSet) {
+      const parsedTag = parseTag(tag)
+      const currentTagLength = getTagLength(parsedTag)
+      if (currentTagLength === tagLength) {
+        removeTagFromIndex(parsedTag, rowId, tagIndex, tagLength)
+      }
+      tagCache.delete(tag)
+    }
+
+    // Remove the row from the tag sets map
+    rowTagSets.delete(rowId)
+  }
+
+  /**
+   * Remove matching tags from a row based on a pattern
+   * Returns true if the row's tag set is now empty
+   */
+  const removeMatchingTagsFromRow = (
+    rowId: RowId,
+    pattern: MoveOutPattern,
+  ): boolean => {
+    const rowTagSet = rowTagSets.get(rowId)
+    if (!rowTagSet) {
+      return false
+    }
+
+    // Find tags that match this pattern and remove them
+    for (const tag of rowTagSet) {
+      const parsedTag = parseTag(tag)
+      if (tagMatchesPattern(parsedTag, pattern)) {
+        rowTagSet.delete(tag)
+        removeTagFromIndex(parsedTag, rowId, tagIndex, tagLength!)
+      }
+    }
+
+    // Check if row's tag set is now empty
+    if (rowTagSet.size === 0) {
+      rowTagSets.delete(rowId)
+      return true
+    }
+
+    return false
+  }
+
+  /**
+   * Process move-out event: remove matching tags from rows and delete rows with empty tag sets
+   */
+  const processMoveOutEvent = (
+    patterns: Array<MoveOutPattern>,
+    begin: () => void,
+    write: (message: ChangeMessageOrDeleteKeyMessage<T>) => void,
+    transactionStarted: boolean,
+  ): boolean => {
+    if (tagLength === undefined) {
+      debug(
+        `${collectionId ? `[${collectionId}] ` : ``}Received move-out message but no tag length set yet, ignoring`,
+      )
+      return transactionStarted
+    }
+
+    let txStarted = transactionStarted
+
+    // Process all patterns and collect rows to delete
+    for (const pattern of patterns) {
+      // Find all rows that match this pattern
+      const affectedRowIds = findRowsMatchingPattern(pattern, tagIndex)
+
+      for (const rowId of affectedRowIds) {
+        if (removeMatchingTagsFromRow(rowId, pattern)) {
+          // Delete rows with empty tag sets
+          if (!txStarted) {
+            begin()
+            txStarted = true
+          }
+
+          write({
+            type: `delete`,
+            key: rowId,
+          })
+        }
+      }
+    }
+
+    return txStarted
+  }
+
   /**
    * Get the sync metadata for insert operations
    * @returns Record containing relation information
@@ -983,6 +1233,38 @@ function createElectricSync<T extends Row<unknown>>(
         syncMode === `progressive` && !hasReceivedUpToDate
       const bufferedMessages: Array<Message<T>> = [] // Buffer change messages during initial sync
 
+      /**
+       * Process a change message: handle tags and write the mutation
+       */
+      const processChangeMessage = (changeMessage: Message<T>) => {
+        if (!isChangeMessage(changeMessage)) {
+          return
+        }
+
+        // Process tags if present
+        const tags = changeMessage.headers.tags
+        const removedTags = changeMessage.headers.removed_tags
+        const hasTags = tags || removedTags
+
+        const rowId = collection.getKeyFromItem(changeMessage.value)
+        const operation = changeMessage.headers.operation
+
+        if (operation === `delete`) {
+          clearTagsForRow(rowId)
+        } else if (hasTags) {
+          processTagsForChangeMessage(tags, removedTags, rowId)
+        }
+
+        write({
+          type: changeMessage.headers.operation,
+          value: changeMessage.value,
+          // Include the primary key and relation info in the metadata
+          metadata: {
+            ...changeMessage.headers,
+          },
+        })
+      }
+
       // Create deduplicated loadSubset wrapper for non-eager modes
       // This prevents redundant snapshot requests when multiple concurrent
       // live queries request overlapping or subset predicates
@@ -997,8 +1279,8 @@ function createElectricSync<T extends Row<unknown>>(
       })
 
       unsubscribeStream = stream.subscribe((messages: Array<Message<T>>) => {
-        let hasUpToDate = false
-        let hasSnapshotEnd = false
+        // Track commit point type - up-to-date takes precedence as it also triggers progressive mode atomic swap
+        let commitPoint: `up-to-date` | `subset-end` | null = null
 
         // Clear the current batch buffer at the START of processing a new batch
         // This preserves messages from the previous batch until new ones arrive,
@@ -1008,7 +1290,7 @@ function createElectricSync<T extends Row<unknown>>(
 
         for (const message of messages) {
           // Add message to current batch buffer (for race condition handling)
-          if (isChangeMessage(message)) {
+          if (isChangeMessage(message) || isMoveOutMessage(message)) {
             currentBatchMessages.setState((currentBuffer) => {
               const newBuffer = [...currentBuffer, message]
               // Limit buffer size for safety
@@ -1065,23 +1347,35 @@ function createElectricSync<T extends Row<unknown>>(
                 transactionStarted = true
               }
 
-              write({
-                type: message.headers.operation,
-                value: message.value,
-                // Include the primary key and relation info in the metadata
-                metadata: {
-                  ...message.headers,
-                },
-              })
+              processChangeMessage(message)
             }
           } else if (isSnapshotEndMessage(message)) {
-            // Skip snapshot-end tracking during buffered initial sync (will be extracted during atomic swap)
+            // Track postgres snapshot metadata for resolving awaiting mutations
+            // Skip during buffered initial sync (will be extracted during atomic swap)
             if (!isBufferingInitialSync()) {
               newSnapshots.push(parseSnapshotMessage(message))
             }
-            hasSnapshotEnd = true
           } else if (isUpToDateMessage(message)) {
-            hasUpToDate = true
+            // up-to-date takes precedence - also triggers progressive mode atomic swap
+            commitPoint = `up-to-date`
+          } else if (isSubsetEndMessage(message)) {
+            // subset-end triggers commit but not progressive mode atomic swap
+            if (commitPoint !== `up-to-date`) {
+              commitPoint = `subset-end`
+            }
+          } else if (isMoveOutMessage(message)) {
+            // Handle move-out event: buffer if buffering, otherwise process immediately
+            if (isBufferingInitialSync()) {
+              bufferedMessages.push(message)
+            } else {
+              // Normal processing: process move-out immediately
+              transactionStarted = processMoveOutEvent(
+                message.headers.patterns,
+                begin,
+                write,
+                transactionStarted,
+              )
+            }
           } else if (isMustRefetchMessage(message)) {
             debug(
               `${collectionId ? `[${collectionId}] ` : ``}Received must-refetch message, starting transaction with truncate`,
@@ -1095,21 +1389,23 @@ function createElectricSync<T extends Row<unknown>>(
 
             truncate()
 
+            // Clear tag tracking state
+            clearTagTrackingState()
+
             // Reset the loadSubset deduplication state since we're starting fresh
             // This ensures that previously loaded predicates don't prevent refetching after truncate
             loadSubsetDedupe?.reset()
 
             // Reset flags so we continue accumulating changes until next up-to-date
-            hasUpToDate = false
-            hasSnapshotEnd = false
+            commitPoint = null
             hasReceivedUpToDate = false // Reset for progressive mode (isBufferingInitialSync will reflect this)
             bufferedMessages.length = 0 // Clear buffered messages
           }
         }
 
-        if (hasUpToDate || hasSnapshotEnd) {
-          // PROGRESSIVE MODE: Atomic swap on first up-to-date
-          if (isBufferingInitialSync() && hasUpToDate) {
+        if (commitPoint !== null) {
+          // PROGRESSIVE MODE: Atomic swap on first up-to-date (not subset-end)
+          if (isBufferingInitialSync() && commitPoint === `up-to-date`) {
             debug(
               `${collectionId ? `[${collectionId}] ` : ``}Progressive mode: Performing atomic swap with ${bufferedMessages.length} buffered messages`,
             )
@@ -1120,16 +1416,13 @@ function createElectricSync<T extends Row<unknown>>(
             // Truncate to clear all snapshot data
             truncate()
 
+            // Clear tag tracking state for atomic swap
+            clearTagTrackingState()
+
             // Apply all buffered change messages and extract txids/snapshots
             for (const bufferedMsg of bufferedMessages) {
               if (isChangeMessage(bufferedMsg)) {
-                write({
-                  type: bufferedMsg.headers.operation,
-                  value: bufferedMsg.value,
-                  metadata: {
-                    ...bufferedMsg.headers,
-                  },
-                })
+                processChangeMessage(bufferedMsg)
 
                 // Extract txids from buffered messages (will be committed to store after transaction)
                 if (hasTxids(bufferedMsg)) {
@@ -1140,6 +1433,14 @@ function createElectricSync<T extends Row<unknown>>(
               } else if (isSnapshotEndMessage(bufferedMsg)) {
                 // Extract snapshots from buffered messages (will be committed to store after transaction)
                 newSnapshots.push(parseSnapshotMessage(bufferedMsg))
+              } else if (isMoveOutMessage(bufferedMsg)) {
+                // Process buffered move-out messages during atomic swap
+                processMoveOutEvent(
+                  bufferedMsg.headers.patterns,
+                  begin,
+                  write,
+                  transactionStarted,
+                )
               }
             }
 
@@ -1155,25 +1456,16 @@ function createElectricSync<T extends Row<unknown>>(
             )
           } else {
             // Normal mode or on-demand: commit transaction if one was started
-            // In eager mode, only commit on snapshot-end if we've already received
-            // the first up-to-date, because the snapshot-end in the log could be from
-            // a significant period before the stream is actually up to date
-            const shouldCommit =
-              hasUpToDate || syncMode === `on-demand` || hasReceivedUpToDate
-
-            if (transactionStarted && shouldCommit) {
+            // Both up-to-date and subset-end trigger a commit
+            if (transactionStarted) {
               commit()
               transactionStarted = false
             }
           }
-
-          if (hasUpToDate || (hasSnapshotEnd && syncMode === `on-demand`)) {
-            // Mark the collection as ready now that sync is up to date
-            wrappedMarkReady(isBufferingInitialSync())
-          }
+          wrappedMarkReady(isBufferingInitialSync())
 
           // Track that we've received the first up-to-date for progressive mode
-          if (hasUpToDate) {
+          if (commitPoint === `up-to-date`) {
             hasReceivedUpToDate = true
           }
 
@@ -1204,12 +1496,11 @@ function createElectricSync<T extends Row<unknown>>(
             return seen
           })
 
-          // Resolve all matched pending matches on up-to-date or snapshot-end in on-demand mode
+          // Resolve all matched pending matches on up-to-date or subset-end
           // Set batchCommitted BEFORE resolving to avoid timing window where late awaitMatch
           // calls could register as "matched" after resolver pass already ran
-          if (hasUpToDate || (hasSnapshotEnd && syncMode === `on-demand`)) {
-            batchCommitted.setState(() => true)
-          }
+          batchCommitted.setState(() => true)
+
           resolveMatchedPendingMatches()
         }
       })

package/src/tag-index.ts

@@ -0,0 +1,160 @@
+// Import Row and Message types for the isEventMessage function
+import type { Message, Row } from '@electric-sql/client'
+
+export type RowId = string | number
+export type MoveTag = string
+export type ParsedMoveTag = Array<string>
+export type Position = number
+export type Value = string
+export type MoveOutPattern = {
+  pos: Position
+  value: Value
+}
+
+const TAG_WILDCARD = `_`
+
+/**
+ * Event message type for move-out events
+ */
+export interface EventMessage {
+  headers: {
+    event: `move-out`
+    patterns: Array<MoveOutPattern>
+  }
+}
+
+/**
+ * Tag index structure: array indexed by position, maps value to set of row IDs.
+ * For example:
+ * ```example
+ * const tag1 = [a, b, c]
+ * const tag2 = [a, b, d]
+ * const tag3 = [a, d, e]
+ *
+ * // Index is:
+ * [
+ *   new Map([a -> <rows with a on index 0>])
+ *   new Map([b -> <rows with b on index 1>, d -> <rows with d on index 1>])
+ *   new Map([c -> <rows with c on index 2>, d -> <rows with d on index 2>, e -> <rows with e on index 2>])
+ * ]
+ * ```
+ */
+export type TagIndex = Array<Map<Value, Set<RowId>>>
+
+/**
+ * Abstraction to get the value at a specific position in a tag
+ */
+export function getValue(tag: ParsedMoveTag, position: Position): Value {
+  if (position >= tag.length) {
+    throw new Error(`Position out of bounds`)
+  }
+  return tag[position]!
+}
+
+/**
+ * Abstraction to extract position and value from a pattern.
+ */
+function getPositionalValue(pattern: MoveOutPattern): {
+  pos: number
+  value: string
+} {
+  return pattern
+}
+
+/**
+ * Abstraction to get the length of a tag
+ */
+export function getTagLength(tag: ParsedMoveTag): number {
+  return tag.length
+}
+
+/**
+ * Check if a tag matches a pattern.
+ * A tag matches if the value at the pattern's position equals the pattern's value,
+ * or if the value at that position is "_" (wildcard).
+ */
+export function tagMatchesPattern(
+  tag: ParsedMoveTag,
+  pattern: MoveOutPattern,
+): boolean {
+  const { pos, value } = getPositionalValue(pattern)
+  const tagValue = getValue(tag, pos)
+  return tagValue === value || tagValue === TAG_WILDCARD
+}
+
+/**
+ * Add a tag to the index for efficient pattern matching
+ */
+export function addTagToIndex(
+  tag: ParsedMoveTag,
+  rowId: RowId,
+  index: TagIndex,
+  tagLength: number,
+): void {
+  for (let i = 0; i < tagLength; i++) {
+    const value = getValue(tag, i)
+
+    // Only index non-wildcard values
+    if (value !== TAG_WILDCARD) {
+      const positionIndex = index[i]!
+      if (!positionIndex.has(value)) {
+        positionIndex.set(value, new Set())
+      }
+
+      const tags = positionIndex.get(value)!
+      tags.add(rowId)
+    }
+  }
+}
+
+/**
+ * Remove a tag from the index
+ */
+export function removeTagFromIndex(
+  tag: ParsedMoveTag,
+  rowId: RowId,
+  index: TagIndex,
+  tagLength: number,
+): void {
+  for (let i = 0; i < tagLength; i++) {
+    const value = getValue(tag, i)
+
+    // Only remove non-wildcard values
+    if (value !== TAG_WILDCARD) {
+      const positionIndex = index[i]
+      if (positionIndex) {
+        const rowSet = positionIndex.get(value)
+        if (rowSet) {
+          rowSet.delete(rowId)
+
+          // Clean up empty sets
+          if (rowSet.size === 0) {
+            positionIndex.delete(value)
+          }
+        }
+      }
+    }
+  }
+}
+
+/**
+ * Find all rows that match a given pattern
+ */
+export function findRowsMatchingPattern(
+  pattern: MoveOutPattern,
+  index: TagIndex,
+): Set<RowId> {
+  const { pos, value } = getPositionalValue(pattern)
+  const positionIndex = index[pos]
+  const rowSet = positionIndex?.get(value)
+  return rowSet ?? new Set()
+}
+
+/**
+ * Check if a message is an event message with move-out event
+ */
+export function isMoveOutMessage<T extends Row<unknown>>(
+  message: Message<T>,
+): message is Message<T> & EventMessage {
+  return message.headers.event === `move-out`
+}