@tanstack/electric-db-collection 0.2.14 → 0.2.16

package/dist/cjs/tag-index.cjs

@@ -0,0 +1,67 @@
+"use strict";
+Object.defineProperty(exports, Symbol.toStringTag, { value: "Module" });
+const TAG_WILDCARD = `_`;
+function getValue(tag, position) {
+  if (position >= tag.length) {
+    throw new Error(`Position out of bounds`);
+  }
+  return tag[position];
+}
+function getPositionalValue(pattern) {
+  return pattern;
+}
+function getTagLength(tag) {
+  return tag.length;
+}
+function tagMatchesPattern(tag, pattern) {
+  const { pos, value } = getPositionalValue(pattern);
+  const tagValue = getValue(tag, pos);
+  return tagValue === value || tagValue === TAG_WILDCARD;
+}
+function addTagToIndex(tag, rowId, index, tagLength) {
+  for (let i = 0; i < tagLength; i++) {
+    const value = getValue(tag, i);
+    if (value !== TAG_WILDCARD) {
+      const positionIndex = index[i];
+      if (!positionIndex.has(value)) {
+        positionIndex.set(value, /* @__PURE__ */ new Set());
+      }
+      const tags = positionIndex.get(value);
+      tags.add(rowId);
+    }
+  }
+}
+function removeTagFromIndex(tag, rowId, index, tagLength) {
+  for (let i = 0; i < tagLength; i++) {
+    const value = getValue(tag, i);
+    if (value !== TAG_WILDCARD) {
+      const positionIndex = index[i];
+      if (positionIndex) {
+        const rowSet = positionIndex.get(value);
+        if (rowSet) {
+          rowSet.delete(rowId);
+          if (rowSet.size === 0) {
+            positionIndex.delete(value);
+          }
+        }
+      }
+    }
+  }
+}
+function findRowsMatchingPattern(pattern, index) {
+  const { pos, value } = getPositionalValue(pattern);
+  const positionIndex = index[pos];
+  const rowSet = positionIndex?.get(value);
+  return rowSet ?? /* @__PURE__ */ new Set();
+}
+function isMoveOutMessage(message) {
+  return message.headers.event === `move-out`;
+}
+exports.addTagToIndex = addTagToIndex;
+exports.findRowsMatchingPattern = findRowsMatchingPattern;
+exports.getTagLength = getTagLength;
+exports.getValue = getValue;
+exports.isMoveOutMessage = isMoveOutMessage;
+exports.removeTagFromIndex = removeTagFromIndex;
+exports.tagMatchesPattern = tagMatchesPattern;
+//# sourceMappingURL=tag-index.cjs.map

package/dist/cjs/tag-index.cjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"tag-index.cjs","sources":["../../src/tag-index.ts"],"sourcesContent":["// Import Row and Message types for the isEventMessage function\nimport type { Message, Row } from '@electric-sql/client'\n\nexport type RowId = string | number\nexport type MoveTag = string\nexport type ParsedMoveTag = Array<string>\nexport type Position = number\nexport type Value = string\nexport type MoveOutPattern = {\n  pos: Position\n  value: Value\n}\n\nconst TAG_WILDCARD = `_`\n\n/**\n * Event message type for move-out events\n */\nexport interface EventMessage {\n  headers: {\n    event: `move-out`\n    patterns: Array<MoveOutPattern>\n  }\n}\n\n/**\n * Tag index structure: array indexed by position, maps value to set of row IDs.\n * For example:\n * ```example\n * const tag1 = [a, b, c]\n * const tag2 = [a, b, d]\n * const tag3 = [a, d, e]\n *\n * // Index is:\n * [\n *   new Map([a -> <rows with a on index 0>])\n *   new Map([b -> <rows with b on index 1>, d -> <rows with d on index 1>])\n *   new Map([c -> <rows with c on index 2>, d -> <rows with d on index 2>, e -> <rows with e on index 2>])\n * ]\n * ```\n */\nexport type TagIndex = Array<Map<Value, Set<RowId>>>\n\n/**\n * Abstraction to get the value at a specific position in a tag\n */\nexport function getValue(tag: ParsedMoveTag, position: Position): Value {\n  if (position >= tag.length) {\n    throw new Error(`Position out of bounds`)\n  }\n  return tag[position]!\n}\n\n/**\n * Abstraction to extract position and value from a pattern.\n */\nfunction getPositionalValue(pattern: MoveOutPattern): {\n  pos: number\n  value: string\n} {\n  return pattern\n}\n\n/**\n * Abstraction to get the length of a tag\n */\nexport function getTagLength(tag: ParsedMoveTag): number {\n  return tag.length\n}\n\n/**\n * Check if a tag matches a pattern.\n * A tag matches if the value at the pattern's position equals the pattern's value,\n * or if the value at that position is \"_\" (wildcard).\n */\nexport function tagMatchesPattern(\n  tag: ParsedMoveTag,\n  pattern: MoveOutPattern,\n): boolean {\n  const { pos, value } = getPositionalValue(pattern)\n  const tagValue = getValue(tag, pos)\n  return tagValue === value || tagValue === TAG_WILDCARD\n}\n\n/**\n * Add a tag to the index for efficient pattern matching\n */\nexport function addTagToIndex(\n  tag: ParsedMoveTag,\n  rowId: RowId,\n  index: TagIndex,\n  tagLength: number,\n): void {\n  for (let i = 0; i < tagLength; i++) {\n    const value = getValue(tag, i)\n\n    // Only index non-wildcard values\n    if (value !== TAG_WILDCARD) {\n      const positionIndex = index[i]!\n      if (!positionIndex.has(value)) {\n        positionIndex.set(value, new Set())\n      }\n\n      const tags = positionIndex.get(value)!\n      tags.add(rowId)\n    }\n  }\n}\n\n/**\n * Remove a tag from the index\n */\nexport function removeTagFromIndex(\n  tag: ParsedMoveTag,\n  rowId: RowId,\n  index: TagIndex,\n  tagLength: number,\n): void {\n  for (let i = 0; i < tagLength; i++) {\n    const value = getValue(tag, i)\n\n    // Only remove non-wildcard values\n    if (value !== TAG_WILDCARD) {\n      const positionIndex = index[i]\n      if (positionIndex) {\n        const rowSet = positionIndex.get(value)\n        if (rowSet) {\n          rowSet.delete(rowId)\n\n          // Clean up empty sets\n          if (rowSet.size === 0) {\n            positionIndex.delete(value)\n          }\n        }\n      }\n    }\n  }\n}\n\n/**\n * Find all rows that match a given pattern\n */\nexport function findRowsMatchingPattern(\n  pattern: MoveOutPattern,\n  index: TagIndex,\n): Set<RowId> {\n  const { pos, value } = getPositionalValue(pattern)\n  const positionIndex = index[pos]\n  const rowSet = positionIndex?.get(value)\n  return rowSet ?? new Set()\n}\n\n/**\n * Check if a message is an event message with move-out event\n */\nexport function isMoveOutMessage<T extends Row<unknown>>(\n  message: Message<T>,\n): message is Message<T> & EventMessage {\n  return message.headers.event === `move-out`\n}\n"],"names":[],"mappings":";;AAaA,MAAM,eAAe;AAiCd,SAAS,SAAS,KAAoB,UAA2B;AACtE,MAAI,YAAY,IAAI,QAAQ;AAC1B,UAAM,IAAI,MAAM,wBAAwB;AAAA,EAC1C;AACA,SAAO,IAAI,QAAQ;AACrB;AAKA,SAAS,mBAAmB,SAG1B;AACA,SAAO;AACT;AAKO,SAAS,aAAa,KAA4B;AACvD,SAAO,IAAI;AACb;AAOO,SAAS,kBACd,KACA,SACS;AACT,QAAM,EAAE,KAAK,UAAU,mBAAmB,OAAO;AACjD,QAAM,WAAW,SAAS,KAAK,GAAG;AAClC,SAAO,aAAa,SAAS,aAAa;AAC5C;AAKO,SAAS,cACd,KACA,OACA,OACA,WACM;AACN,WAAS,IAAI,GAAG,IAAI,WAAW,KAAK;AAClC,UAAM,QAAQ,SAAS,KAAK,CAAC;AAG7B,QAAI,UAAU,cAAc;AAC1B,YAAM,gBAAgB,MAAM,CAAC;AAC7B,UAAI,CAAC,cAAc,IAAI,KAAK,GAAG;AAC7B,sBAAc,IAAI,OAAO,oBAAI,IAAA,CAAK;AAAA,MACpC;AAEA,YAAM,OAAO,cAAc,IAAI,KAAK;AACpC,WAAK,IAAI,KAAK;AAAA,IAChB;AAAA,EACF;AACF;AAKO,SAAS,mBACd,KACA,OACA,OACA,WACM;AACN,WAAS,IAAI,GAAG,IAAI,WAAW,KAAK;AAClC,UAAM,QAAQ,SAAS,KAAK,CAAC;AAG7B,QAAI,UAAU,cAAc;AAC1B,YAAM,gBAAgB,MAAM,CAAC;AAC7B,UAAI,eAAe;AACjB,cAAM,SAAS,cAAc,IAAI,KAAK;AACtC,YAAI,QAAQ;AACV,iBAAO,OAAO,KAAK;AAGnB,cAAI,OAAO,SAAS,GAAG;AACrB,0BAAc,OAAO,KAAK;AAAA,UAC5B;AAAA,QACF;AAAA,MACF;AAAA,IACF;AAAA,EACF;AACF;AAKO,SAAS,wBACd,SACA,OACY;AACZ,QAAM,EAAE,KAAK,UAAU,mBAAmB,OAAO;AACjD,QAAM,gBAAgB,MAAM,GAAG;AAC/B,QAAM,SAAS,eAAe,IAAI,KAAK;AACvC,SAAO,8BAAc,IAAA;AACvB;AAKO,SAAS,iBACd,SACsC;AACtC,SAAO,QAAQ,QAAQ,UAAU;AACnC;;;;;;;;"}

package/dist/cjs/tag-index.d.cts

@@ -0,0 +1,66 @@
+import { Message, Row } from '@electric-sql/client';
+export type RowId = string | number;
+export type MoveTag = string;
+export type ParsedMoveTag = Array<string>;
+export type Position = number;
+export type Value = string;
+export type MoveOutPattern = {
+    pos: Position;
+    value: Value;
+};
+/**
+ * Event message type for move-out events
+ */
+export interface EventMessage {
+    headers: {
+        event: `move-out`;
+        patterns: Array<MoveOutPattern>;
+    };
+}
+/**
+ * Tag index structure: array indexed by position, maps value to set of row IDs.
+ * For example:
+ * ```example
+ * const tag1 = [a, b, c]
+ * const tag2 = [a, b, d]
+ * const tag3 = [a, d, e]
+ *
+ * // Index is:
+ * [
+ *   new Map([a -> <rows with a on index 0>])
+ *   new Map([b -> <rows with b on index 1>, d -> <rows with d on index 1>])
+ *   new Map([c -> <rows with c on index 2>, d -> <rows with d on index 2>, e -> <rows with e on index 2>])
+ * ]
+ * ```
+ */
+export type TagIndex = Array<Map<Value, Set<RowId>>>;
+/**
+ * Abstraction to get the value at a specific position in a tag
+ */
+export declare function getValue(tag: ParsedMoveTag, position: Position): Value;
+/**
+ * Abstraction to get the length of a tag
+ */
+export declare function getTagLength(tag: ParsedMoveTag): number;
+/**
+ * Check if a tag matches a pattern.
+ * A tag matches if the value at the pattern's position equals the pattern's value,
+ * or if the value at that position is "_" (wildcard).
+ */
+export declare function tagMatchesPattern(tag: ParsedMoveTag, pattern: MoveOutPattern): boolean;
+/**
+ * Add a tag to the index for efficient pattern matching
+ */
+export declare function addTagToIndex(tag: ParsedMoveTag, rowId: RowId, index: TagIndex, tagLength: number): void;
+/**
+ * Remove a tag from the index
+ */
+export declare function removeTagFromIndex(tag: ParsedMoveTag, rowId: RowId, index: TagIndex, tagLength: number): void;
+/**
+ * Find all rows that match a given pattern
+ */
+export declare function findRowsMatchingPattern(pattern: MoveOutPattern, index: TagIndex): Set<RowId>;
+/**
+ * Check if a message is an event message with move-out event
+ */
+export declare function isMoveOutMessage<T extends Row<unknown>>(message: Message<T>): message is Message<T> & EventMessage;

package/dist/esm/electric.js

@@ -5,6 +5,7 @@ import DebugModule from "debug";
 import { DeduplicatedLoadSubset, and } from "@tanstack/db";
 import { StreamAbortedError, ExpectedNumberInAwaitTxIdError, TimeoutWaitingForTxIdError, TimeoutWaitingForMatchError } from "./errors.js";
 import { compileSQL } from "./sql-compiler.js";
+import { isMoveOutMessage, getTagLength, addTagToIndex, removeTagFromIndex, findRowsMatchingPattern, tagMatchesPattern } from "./tag-index.js";
 const debug = DebugModule.debug(`ts/db:electric`);
 const ELECTRIC_TEST_HOOKS = Symbol(`electricTestHooks`);
 function isUpToDateMessage(message) {
@@ -16,6 +17,9 @@ function isMustRefetchMessage(message) {
 function isSnapshotEndMessage(message) {
   return isControlMessage(message) && message.headers.control === `snapshot-end`;
 }
+function isSubsetEndMessage(message) {
+  return isControlMessage(message) && message.headers.control === `subset-end`;
+}
 function parseSnapshotMessage(message) {
   return {
     xmin: message.headers.xmin,
@@ -337,6 +341,134 @@ function createElectricSync(shapeOptions, options) {
   } = options;
   const MAX_BATCH_MESSAGES = 1e3;
   const relationSchema = new Store(void 0);
+  const tagCache = /* @__PURE__ */ new Map();
+  const parseTag = (tag) => {
+    const cachedTag = tagCache.get(tag);
+    if (cachedTag) {
+      return cachedTag;
+    }
+    const parsedTag = tag.split(`|`);
+    tagCache.set(tag, parsedTag);
+    return parsedTag;
+  };
+  const rowTagSets = /* @__PURE__ */ new Map();
+  const tagIndex = [];
+  let tagLength = void 0;
+  const initializeTagIndex = (length) => {
+    if (tagIndex.length < length) {
+      for (let i = tagIndex.length; i < length; i++) {
+        tagIndex[i] = /* @__PURE__ */ new Map();
+      }
+    }
+  };
+  const addTagsToRow = (tags, rowId, rowTagSet) => {
+    for (const tag of tags) {
+      const parsedTag = parseTag(tag);
+      if (tagLength === void 0) {
+        tagLength = getTagLength(parsedTag);
+        initializeTagIndex(tagLength);
+      }
+      const currentTagLength = getTagLength(parsedTag);
+      if (currentTagLength !== tagLength) {
+        debug(
+          `${collectionId ? `[${collectionId}] ` : ``}Tag length mismatch: expected ${tagLength}, got ${currentTagLength}`
+        );
+        continue;
+      }
+      rowTagSet.add(tag);
+      addTagToIndex(parsedTag, rowId, tagIndex, tagLength);
+    }
+  };
+  const removeTagsFromRow = (removedTags, rowId, rowTagSet) => {
+    if (tagLength === void 0) {
+      return;
+    }
+    for (const tag of removedTags) {
+      const parsedTag = parseTag(tag);
+      rowTagSet.delete(tag);
+      removeTagFromIndex(parsedTag, rowId, tagIndex, tagLength);
+      tagCache.delete(tag);
+    }
+  };
+  const processTagsForChangeMessage = (tags, removedTags, rowId) => {
+    if (!rowTagSets.has(rowId)) {
+      rowTagSets.set(rowId, /* @__PURE__ */ new Set());
+    }
+    const rowTagSet = rowTagSets.get(rowId);
+    if (tags) {
+      addTagsToRow(tags, rowId, rowTagSet);
+    }
+    if (removedTags) {
+      removeTagsFromRow(removedTags, rowId, rowTagSet);
+    }
+    return rowTagSet;
+  };
+  const clearTagTrackingState = () => {
+    rowTagSets.clear();
+    tagIndex.length = 0;
+    tagLength = void 0;
+  };
+  const clearTagsForRow = (rowId) => {
+    if (tagLength === void 0) {
+      return;
+    }
+    const rowTagSet = rowTagSets.get(rowId);
+    if (!rowTagSet) {
+      return;
+    }
+    for (const tag of rowTagSet) {
+      const parsedTag = parseTag(tag);
+      const currentTagLength = getTagLength(parsedTag);
+      if (currentTagLength === tagLength) {
+        removeTagFromIndex(parsedTag, rowId, tagIndex, tagLength);
+      }
+      tagCache.delete(tag);
+    }
+    rowTagSets.delete(rowId);
+  };
+  const removeMatchingTagsFromRow = (rowId, pattern) => {
+    const rowTagSet = rowTagSets.get(rowId);
+    if (!rowTagSet) {
+      return false;
+    }
+    for (const tag of rowTagSet) {
+      const parsedTag = parseTag(tag);
+      if (tagMatchesPattern(parsedTag, pattern)) {
+        rowTagSet.delete(tag);
+        removeTagFromIndex(parsedTag, rowId, tagIndex, tagLength);
+      }
+    }
+    if (rowTagSet.size === 0) {
+      rowTagSets.delete(rowId);
+      return true;
+    }
+    return false;
+  };
+  const processMoveOutEvent = (patterns, begin, write, transactionStarted) => {
+    if (tagLength === void 0) {
+      debug(
+        `${collectionId ? `[${collectionId}] ` : ``}Received move-out message but no tag length set yet, ignoring`
+      );
+      return transactionStarted;
+    }
+    let txStarted = transactionStarted;
+    for (const pattern of patterns) {
+      const affectedRowIds = findRowsMatchingPattern(pattern, tagIndex);
+      for (const rowId of affectedRowIds) {
+        if (removeMatchingTagsFromRow(rowId, pattern)) {
+          if (!txStarted) {
+            begin();
+            txStarted = true;
+          }
+          write({
+            type: `delete`,
+            key: rowId
+          });
+        }
+      }
+    }
+    return txStarted;
+  };
   const getSyncMetadata = () => {
     const schema = relationSchema.state || `public`;
     return {
@@ -411,6 +543,29 @@ You can provide an 'onError' handler on the shapeOptions to handle this error, a
       let hasReceivedUpToDate = false;
       const isBufferingInitialSync = () => syncMode === `progressive` && !hasReceivedUpToDate;
       const bufferedMessages = [];
+      const processChangeMessage = (changeMessage) => {
+        if (!isChangeMessage(changeMessage)) {
+          return;
+        }
+        const tags = changeMessage.headers.tags;
+        const removedTags = changeMessage.headers.removed_tags;
+        const hasTags = tags || removedTags;
+        const rowId = collection.getKeyFromItem(changeMessage.value);
+        const operation = changeMessage.headers.operation;
+        if (operation === `delete`) {
+          clearTagsForRow(rowId);
+        } else if (hasTags) {
+          processTagsForChangeMessage(tags, removedTags, rowId);
+        }
+        write({
+          type: changeMessage.headers.operation,
+          value: changeMessage.value,
+          // Include the primary key and relation info in the metadata
+          metadata: {
+            ...changeMessage.headers
+          }
+        });
+      };
       const loadSubsetDedupe = createLoadSubsetDedupe({
         stream,
         syncMode,
@@ -421,12 +576,11 @@ You can provide an 'onError' handler on the shapeOptions to handle this error, a
         collectionId
       });
       unsubscribeStream = stream.subscribe((messages) => {
-        let hasUpToDate = false;
-        let hasSnapshotEnd = false;
+        let commitPoint = null;
         currentBatchMessages.setState(() => []);
         batchCommitted.setState(() => false);
         for (const message of messages) {
-          if (isChangeMessage(message)) {
+          if (isChangeMessage(message) || isMoveOutMessage(message)) {
             currentBatchMessages.setState((currentBuffer) => {
               const newBuffer = [...currentBuffer, message];
               if (newBuffer.length > MAX_BATCH_MESSAGES) {
@@ -466,22 +620,29 @@ You can provide an 'onError' handler on the shapeOptions to handle this error, a
                 begin();
                 transactionStarted = true;
               }
-              write({
-                type: message.headers.operation,
-                value: message.value,
-                // Include the primary key and relation info in the metadata
-                metadata: {
-                  ...message.headers
-                }
-              });
+              processChangeMessage(message);
             }
           } else if (isSnapshotEndMessage(message)) {
             if (!isBufferingInitialSync()) {
               newSnapshots.push(parseSnapshotMessage(message));
             }
-            hasSnapshotEnd = true;
           } else if (isUpToDateMessage(message)) {
-            hasUpToDate = true;
+            commitPoint = `up-to-date`;
+          } else if (isSubsetEndMessage(message)) {
+            if (commitPoint !== `up-to-date`) {
+              commitPoint = `subset-end`;
+            }
+          } else if (isMoveOutMessage(message)) {
+            if (isBufferingInitialSync()) {
+              bufferedMessages.push(message);
+            } else {
+              transactionStarted = processMoveOutEvent(
+                message.headers.patterns,
+                begin,
+                write,
+                transactionStarted
+              );
+            }
           } else if (isMustRefetchMessage(message)) {
             debug(
               `${collectionId ? `[${collectionId}] ` : ``}Received must-refetch message, starting transaction with truncate`
@@ -491,29 +652,24 @@ You can provide an 'onError' handler on the shapeOptions to handle this error, a
               transactionStarted = true;
             }
             truncate();
+            clearTagTrackingState();
             loadSubsetDedupe?.reset();
-            hasUpToDate = false;
-            hasSnapshotEnd = false;
+            commitPoint = null;
             hasReceivedUpToDate = false;
             bufferedMessages.length = 0;
           }
         }
-        if (hasUpToDate || hasSnapshotEnd) {
-          if (isBufferingInitialSync() && hasUpToDate) {
+        if (commitPoint !== null) {
+          if (isBufferingInitialSync() && commitPoint === `up-to-date`) {
             debug(
               `${collectionId ? `[${collectionId}] ` : ``}Progressive mode: Performing atomic swap with ${bufferedMessages.length} buffered messages`
             );
             begin();
             truncate();
+            clearTagTrackingState();
             for (const bufferedMsg of bufferedMessages) {
               if (isChangeMessage(bufferedMsg)) {
-                write({
-                  type: bufferedMsg.headers.operation,
-                  value: bufferedMsg.value,
-                  metadata: {
-                    ...bufferedMsg.headers
-                  }
-                });
+                processChangeMessage(bufferedMsg);
                 if (hasTxids(bufferedMsg)) {
                   bufferedMsg.headers.txids?.forEach(
                     (txid) => newTxids.add(txid)
@@ -521,6 +677,13 @@ You can provide an 'onError' handler on the shapeOptions to handle this error, a
                 }
               } else if (isSnapshotEndMessage(bufferedMsg)) {
                 newSnapshots.push(parseSnapshotMessage(bufferedMsg));
+              } else if (isMoveOutMessage(bufferedMsg)) {
+                processMoveOutEvent(
+                  bufferedMsg.headers.patterns,
+                  begin,
+                  write,
+                  transactionStarted
+                );
               }
             }
             commit();
@@ -529,16 +692,13 @@ You can provide an 'onError' handler on the shapeOptions to handle this error, a
               `${collectionId ? `[${collectionId}] ` : ``}Progressive mode: Atomic swap complete, now in normal sync mode`
             );
           } else {
-            const shouldCommit = hasUpToDate || syncMode === `on-demand` || hasReceivedUpToDate;
-            if (transactionStarted && shouldCommit) {
+            if (transactionStarted) {
               commit();
               transactionStarted = false;
             }
           }
-          if (hasUpToDate || hasSnapshotEnd && syncMode === `on-demand`) {
-            wrappedMarkReady(isBufferingInitialSync());
-          }
-          if (hasUpToDate) {
+          wrappedMarkReady(isBufferingInitialSync());
+          if (commitPoint === `up-to-date`) {
             hasReceivedUpToDate = true;
           }
           seenTxids.setState((currentTxids) => {
@@ -564,9 +724,7 @@ You can provide an 'onError' handler on the shapeOptions to handle this error, a
             newSnapshots.length = 0;
             return seen;
           });
-          if (hasUpToDate || hasSnapshotEnd && syncMode === `on-demand`) {
-            batchCommitted.setState(() => true);
-          }
+          batchCommitted.setState(() => true);
           resolveMatchedPendingMatches();
         }
       });