@tanstack/electric-db-collection 0.4.10 → 0.5.1

package/dist/esm/electric.js

@@ -1,12 +1,11 @@
 import { isVisibleInSnapshot, isChangeMessage, ShapeStream, isControlMessage } from "@electric-sql/client";
 import { isChangeMessage as isChangeMessage2, isControlMessage as isControlMessage2 } from "@electric-sql/client";
 import DebugModule from "debug";
-import { withCollectionSyncConfigFactory, withCollectionConfigFactory, withCollectionSyncConfigCleanup, DeduplicatedLoadSubset, and, LoadSubsetOperationAbortedError } from "@tanstack/db";
+import { withCollectionSyncConfigFactory, withCollectionConfigFactory, validateSyncPersistenceCapability, warnOnce, withCollectionSyncConfigCleanup, DeduplicatedLoadSubset, and, LoadSubsetOperationAbortedError } from "@tanstack/db";
 import { StreamAbortedError, ExpectedNumberInAwaitTxIdError, TimeoutWaitingForTxIdError, TimeoutWaitingForMatchError } from "./errors.js";
 import { compileSQL } from "./sql-compiler.js";
 import { isMoveOutMessage, isMoveInMessage, deriveDisjunctPositions, getTagLength, addTagToIndex, parseTag, removeTagFromIndex, findRowsMatchingPattern, rowVisible, tagMatchesPattern } from "./tag-index.js";
 const debug = DebugModule.debug(`ts/db:electric`);
-const FORCE_DISCONNECT_AND_REFRESH_TIMEOUT_MS = 250;
 const ELECTRIC_TEST_HOOKS = /* @__PURE__ */ Symbol(`electricTestHooks`);
 function exportElectricSyncMeta(evidence) {
   const resume = evidence.hydratedResumeState;
@@ -158,6 +157,7 @@ function createLoadSubsetDedupe({
   commit,
   getCommitCursor,
   waitForCommitsAfter,
+  recordSnapshotRow,
   collectionId,
   encodeColumnName,
   signal
@@ -200,7 +200,9 @@ function createLoadSubsetDedupe({
               metadata: { ...row.headers }
             });
           }
-          await commit(opts.signal);
+          const applied = commit(opts.signal);
+          for (const row of rows) recordSnapshotRow?.(row.value, row);
+          if (applied !== true) await applied;
           debug(`${logPrefix}Applied snapshot with ${rows.length} rows`);
         }
       } catch (error) {
@@ -216,47 +218,6 @@ function createLoadSubsetDedupe({
       return;
     }
     const { cursor, where, orderBy, limit } = opts;
-    if (stream.isUpToDate) {
-      let timeoutId;
-      const abortSignals = [signal, opts.signal].filter(
-        (candidate) => candidate !== void 0
-      );
-      let rejectAbort = () => {
-      };
-      const aborted = new Promise((_resolve, reject) => {
-        rejectAbort = reject;
-      });
-      const abort = (event) => rejectAbort(abortReason(event.currentTarget));
-      for (const abortSignal of abortSignals) {
-        abortSignal.addEventListener(`abort`, abort, { once: true });
-      }
-      try {
-        await Promise.race([
-          stream.forceDisconnectAndRefresh(),
-          new Promise((resolve) => {
-            timeoutId = setTimeout(
-              resolve,
-              FORCE_DISCONNECT_AND_REFRESH_TIMEOUT_MS
-            );
-          }),
-          aborted
-        ]);
-      } catch (error) {
-        if (signal.aborted || opts.signal?.aborted) throw error;
-        if (handleSnapshotError(error, `forceDisconnectAndRefresh`)) {
-          return;
-        }
-        debug(
-          `${logPrefix}forceDisconnectAndRefresh failed, proceeding to requestSnapshot: %o`,
-          error
-        );
-      } finally {
-        clearTimeout(timeoutId);
-        for (const abortSignal of abortSignals) {
-          abortSignal.removeEventListener(`abort`, abort);
-        }
-      }
-    }
     throwIfAborted();
     try {
       if (cursor) {
@@ -376,7 +337,7 @@ class ElectricLifecycle {
       waiter.resolve(true);
     }
   }
-  async awaitTxId(txId, timeout = 5e3) {
+  async awaitTxId(txId, timeout = 15e3) {
     debug(
       `${this.collectionId ? `[${this.collectionId}] ` : ``}awaitTxId called with txid %d`,
       txId
@@ -399,7 +360,7 @@ class ElectricLifecycle {
       });
     });
   }
-  async awaitMatch(matchFn, timeout = 3e3) {
+  async awaitMatch(matchFn, timeout = 15e3) {
     debug(
       `${this.collectionId ? `[${this.collectionId}] ` : ``}awaitMatch called with custom function`
     );
@@ -495,12 +456,20 @@ function electricCollectionOptions(config) {
   });
   const sync = createSync();
   const processMatchingStrategy = async (result, waitForTxId) => {
-    if (result && `txid` in result) {
-      const timeout = result.timeout;
-      if (Array.isArray(result.txid)) {
-        await Promise.all(result.txid.map((txid) => waitForTxId(txid, timeout)));
+    const canHaveProperties = typeof result === `object` && result !== null || typeof result === `function`;
+    if (canHaveProperties && `txid` in result) {
+      const strategy = result;
+      warnOnce(
+        "electric-collection-txid-return",
+        "[TanStack DB] DEPRECATED: Returning { txid } from mutation handlers is deprecated and will be removed in v1.0. Use `await collection.utils.awaitTxId(txid)` instead of returning { txid }. See migration guide: https://tanstack.com/db/latest/docs/collections/electric-collection#persistence-handlers--synchronization"
+      );
+      const timeout = strategy.timeout;
+      if (Array.isArray(strategy.txid)) {
+        await Promise.all(
+          strategy.txid.map((txid) => waitForTxId(txid, timeout))
+        );
       } else {
-        await waitForTxId(result.txid, timeout);
+        await waitForTxId(strategy.txid, timeout);
       }
     }
   };
@@ -514,7 +483,6 @@ function electricCollectionOptions(config) {
       handlerResult,
       getMutationAwaitTxId(params)
     );
-    return handlerResult;
   } : void 0;
   const wrappedOnUpdate = config.onUpdate ? async (params) => {
     const handlerResult = await config.onUpdate(params);
@@ -522,7 +490,6 @@ function electricCollectionOptions(config) {
       handlerResult,
       getMutationAwaitTxId(params)
     );
-    return handlerResult;
   } : void 0;
   const wrappedOnDelete = config.onDelete ? async (params) => {
     const handlerResult = await config.onDelete(params);
@@ -530,7 +497,6 @@ function electricCollectionOptions(config) {
       handlerResult,
       getMutationAwaitTxId(params)
     );
-    return handlerResult;
   } : void 0;
   const {
     shapeOptions: _shapeOptions,
@@ -601,7 +567,6 @@ function electricCollectionOptions(config) {
 function createElectricSync(shapeOptions, options) {
   const { getLifecycle, syncMode, collectionId, testHooks } = options;
   let relationSchema;
-  let warnedUnverifiableResume = false;
   const createTagState = () => {
     const tagCache = /* @__PURE__ */ new Map();
     const parseTag$1 = (tag) => {
@@ -843,9 +808,13 @@ function createElectricSync(shapeOptions, options) {
         const persistedResumeState2 = metadata?.collection.get(`electric:resume`);
         return parseElectricResumeState(persistedResumeState2);
       };
-      const persistedMetadata = metadata;
-      const scanPersisted = persistedMetadata?.row.scanPersisted;
-      const whenHydrated = persistedMetadata?.row.whenHydrated;
+      const persistence = metadata === void 0 ? null : validateSyncPersistenceCapability(metadata.persistence);
+      const hydrateBaseline = persistence?.hydrateBaseline;
+      const resumeSnapshot = persistence?.resumeSnapshot;
+      const certifyResumeSnapshot = resumeSnapshot?.certify;
+      const getKeySetEvidence = resumeSnapshot?.getKeySetEvidence;
+      const expectCurrentCommit = resumeSnapshot?.expectCurrentCommit;
+      const persistedKeySetEvidence = getKeySetEvidence?.();
       const persistedResumeState = getNewestElectricResumeState(
         readPersistedResumeState(),
         lifecycle.resumeState
@@ -855,24 +824,19 @@ function createElectricSync(shapeOptions, options) {
         params: shapeOptions.params
       });
       const hasIncompatiblePersistedResume = persistedResumeState?.kind === `resume` && persistedResumeState.shapeId !== shapeIdentity;
-      const hasUnverifiablePersistedResume = shapeOptions.offset === void 0 && shapeOptions.handle === void 0 && persistedResumeState?.kind === `resume` && scanPersisted !== void 0 && whenHydrated === void 0;
-      if (hasUnverifiablePersistedResume && !warnedUnverifiableResume) {
-        warnedUnverifiableResume = true;
-        console.warn(
-          `Electric persistence cannot verify hydration for saved resume state. Update the persistence adapter alongside Electric to enable safe resume.`
-        );
-      }
-      const needsFullSnapshot = shapeOptions.offset === void 0 && shapeOptions.handle === void 0 && persistedResumeState !== void 0 && (persistedResumeState.kind === `reset` || !retainsTagState && persistedResumeState.requiresTagState !== false);
-      const canUsePersistedResume = shapeOptions.offset === void 0 && shapeOptions.handle === void 0 && persistedResumeState?.kind === `resume` && !hasIncompatiblePersistedResume && !hasUnverifiablePersistedResume && // Cached rows do not contain authoritative tag/active-condition state.
-      // Unknown (older) metadata is conservative; untagged shapes still resume.
+      const lacksCompletePersistedKeySet = persistedResumeState?.kind === `resume` && getKeySetEvidence !== void 0 && persistedKeySetEvidence?.status !== `consistent`;
+      const needsFullSnapshot = shapeOptions.offset === void 0 && shapeOptions.handle === void 0 && persistedResumeState !== void 0 && (persistedResumeState.kind === `reset` || lacksCompletePersistedKeySet || !retainsTagState && persistedResumeState.requiresTagState !== false);
+      const canUsePersistedResume = shapeOptions.offset === void 0 && shapeOptions.handle === void 0 && persistedResumeState?.kind === `resume` && !hasIncompatiblePersistedResume && // Cached rows do not contain authoritative tag/active-condition state.
+      // Only a complete adapter ledger can justify a persisted cursor.
       !needsFullSnapshot;
       const hasExplicitResumeOffset = shapeOptions.offset !== void 0 && shapeOptions.offset !== `-1`;
       if (!canUsePersistedResume && !hasExplicitResumeOffset) {
         clearTagTrackingState();
       }
       const receivesCompleteRows = shapeOptions.params?.replica === `full`;
+      const requiresKeySetCertification = canUsePersistedResume && certifyResumeSnapshot !== void 0;
       const requiresCompleteResume = syncMode !== `on-demand` && (canUsePersistedResume || hasExplicitResumeOffset && !receivesCompleteRows);
-      let freshSnapshotPending = (syncMode === `eager` || needsFullSnapshot) && !canUsePersistedResume && !hasExplicitResumeOffset && whenHydrated !== void 0;
+      let freshSnapshotPending = (syncMode === `eager` || needsFullSnapshot) && !canUsePersistedResume && !hasExplicitResumeOffset && hydrateBaseline !== void 0;
       let progressiveReadyGate = null;
       let streamErrorVersion = 0;
       const wrappedMarkReady = (isBuffering, expectedErrorVersion = streamErrorVersion) => {
@@ -939,6 +903,39 @@ function createElectricSync(shapeOptions, options) {
       let resetGeneration = 0;
       const isBufferingInitialSync = () => syncMode === `progressive` && !hasReceivedUpToDate && !isResettingSnapshot;
       const bufferedMessages = [];
+      const progressiveSnapshotRows = /* @__PURE__ */ new Map();
+      const pendingPresence = /* @__PURE__ */ new Map();
+      let usesBaseline = true;
+      let pendingPresenceRevision = 0;
+      const recordPendingPresence = (rowId, present) => {
+        pendingPresenceRevision++;
+        pendingPresence.set(rowId, present);
+      };
+      const recordPendingTruncate = () => {
+        pendingPresenceRevision++;
+        pendingPresence.clear();
+        progressiveSnapshotRows.clear();
+        usesBaseline = false;
+      };
+      const commitSourceTransaction = () => {
+        const committedPresenceRevision = pendingPresenceRevision;
+        const applied = commit();
+        if (metadata?.persistence) {
+          const retireAppliedPresence = () => {
+            if (pendingPresenceRevision === committedPresenceRevision) {
+              pendingPresence.clear();
+              usesBaseline = true;
+            }
+          };
+          if (applied === true) retireAppliedPresence();
+          else void applied.then(retireAppliedPresence, () => void 0);
+        }
+        return applied;
+      };
+      const beginSourceTransaction = () => {
+        if (metadata?.persistence) begin({ immediate: true });
+        else begin();
+      };
       const syncedKeys = /* @__PURE__ */ new Set();
       let resumeInvalid = false;
       const stageResumeMetadata = () => {
@@ -961,7 +958,7 @@ function createElectricSync(shapeOptions, options) {
         lifecycle.resumeState = resumeState;
         metadata?.collection.set(`electric:resume`, resumeState);
       };
-      const commitResetResumeMetadataImmediately = () => {
+      const commitResetResumeMetadataImmediately = (expectInResumeSnapshot = false) => {
         const resetState = {
           kind: `reset`,
           updatedAt: Date.now()
@@ -970,11 +967,14 @@ function createElectricSync(shapeOptions, options) {
         if (metadata) {
           begin({ immediate: true });
           metadata.collection.set(`electric:resume`, resetState);
+          if (expectInResumeSnapshot) {
+            expectCurrentCommit?.();
+          }
           commit();
         }
       };
-      if (hasIncompatiblePersistedResume || hasUnverifiablePersistedResume || needsFullSnapshot && persistedResumeState.kind === `resume`) {
-        commitResetResumeMetadataImmediately();
+      if (hasIncompatiblePersistedResume || needsFullSnapshot && persistedResumeState.kind === `resume`) {
+        commitResetResumeMetadataImmediately(true);
       }
       const processChangeMessage = (changeMessage) => {
         if (!isChangeMessage(changeMessage)) {
@@ -1021,6 +1021,10 @@ function createElectricSync(shapeOptions, options) {
         commit,
         getCommitCursor: () => commitSequence,
         waitForCommitsAfter,
+        recordSnapshotRow: (row, message) => {
+          recordPendingPresence(collection.getKeyFromItem(row), true);
+          progressiveSnapshotRows.set(collection.getKeyFromItem(row), message);
+        },
         collectionId,
         // Pass the columnMapper's encode function to transform column names
         // (e.g., camelCase to snake_case) when compiling SQL for subset queries
@@ -1028,7 +1032,23 @@ function createElectricSync(shapeOptions, options) {
         // Pass abort signal so requestSnapshot errors can be ignored during cleanup
         signal: abortController.signal
       });
-      const resumeKeysPromise = requiresCompleteResume || freshSnapshotPending ? whenHydrated?.() : void 0;
+      const resumeKeysPromise = requiresCompleteResume || freshSnapshotPending ? hydrateBaseline ? (async () => {
+        await hydrateBaseline();
+        const currentKeySetEvidence = getKeySetEvidence?.();
+        if (canUsePersistedResume && currentKeySetEvidence?.status !== `consistent`) {
+          throw new Error(
+            `Electric persisted resume baseline could not be certified during hydration`
+          );
+        }
+      })() : void 0 : requiresKeySetCertification ? (async () => {
+        await certifyResumeSnapshot();
+        const currentKeySetEvidence = getKeySetEvidence?.();
+        if (currentKeySetEvidence?.status !== `consistent`) {
+          throw new Error(
+            `Electric persisted resume baseline could not be certified`
+          );
+        }
+      })() : void 0;
       let areResumeKeysReady = !resumeKeysPromise;
       const pendingResumeBatches = [];
       let unsubscribeStream = () => {
@@ -1060,31 +1080,34 @@ function createElectricSync(shapeOptions, options) {
         }
         if (freshSnapshotPending) {
           freshSnapshotPending = false;
-          begin();
+          beginSourceTransaction();
           transactionStarted = true;
           truncate();
+          recordPendingTruncate();
           syncedKeys.clear();
           clearTagTrackingState();
           isResettingSnapshot = true;
           resetGeneration++;
         }
-        const pendingPresence = /* @__PURE__ */ new Map();
-        let usesBaseline = true;
-        for (const pending of collection._state.pendingSyncedTransactions) {
-          if (pending.truncate) {
-            pendingPresence.clear();
-            usesBaseline = false;
-          }
-          for (const operation of pending.operations) {
-            pendingPresence.set(operation.key, operation.type !== `delete`);
+        if (!metadata?.persistence && !transactionStarted) {
+          pendingPresence.clear();
+          usesBaseline = true;
+          for (const pending of collection._state.pendingSyncedTransactions) {
+            if (pending.truncate) {
+              pendingPresence.clear();
+              usesBaseline = false;
+            }
+            for (const operation of pending.operations) {
+              pendingPresence.set(operation.key, operation.type !== `delete`);
+            }
           }
-        }
-        for (const message of bufferedMessages) {
-          if (isChangeMessage(message)) {
-            pendingPresence.set(
-              collection.getKeyFromItem(message.value),
-              message.headers.operation !== `delete`
-            );
+          for (const message of bufferedMessages) {
+            if (isChangeMessage(message)) {
+              pendingPresence.set(
+                collection.getKeyFromItem(message.value),
+                message.headers.operation !== `delete`
+              );
+            }
           }
         }
         let commitPoint = null;
@@ -1106,7 +1129,10 @@ function createElectricSync(shapeOptions, options) {
               }
               if (!receivesCompleteRows) continue;
             }
-            pendingPresence.set(rowId, operation !== `delete`);
+            recordPendingPresence(rowId, operation !== `delete`);
+            if (operation !== `update`) {
+              progressiveSnapshotRows.delete(rowId);
+            }
           }
           if (isChangeMessage(message)) {
             const schema = message.headers.schema;
@@ -1114,10 +1140,18 @@ function createElectricSync(shapeOptions, options) {
               relationSchema = schema;
             }
             if (isBufferingInitialSync() && !transactionStarted) {
+              if (message.headers.operation === `update`) {
+                const rowId = collection.getKeyFromItem(message.value);
+                const snapshotRow = progressiveSnapshotRows.get(rowId);
+                if (snapshotRow) {
+                  bufferedMessages.push(snapshotRow);
+                  progressiveSnapshotRows.delete(rowId);
+                }
+              }
               bufferedMessages.push(message);
             } else {
               if (!transactionStarted) {
-                begin();
+                beginSourceTransaction();
                 transactionStarted = true;
               }
               processChangeMessage(message);
@@ -1138,11 +1172,11 @@ function createElectricSync(shapeOptions, options) {
             } else {
               transactionStarted = processMoveOutEvent(
                 message.headers.patterns,
-                begin,
+                beginSourceTransaction,
                 write,
                 transactionStarted,
                 (rowId) => {
-                  pendingPresence.set(rowId, false);
+                  recordPendingPresence(rowId, false);
                   syncedKeys.delete(rowId);
                 }
               );
@@ -1159,20 +1193,20 @@ function createElectricSync(shapeOptions, options) {
             );
             commitResetResumeMetadataImmediately();
             if (!transactionStarted) {
-              begin();
+              beginSourceTransaction();
               transactionStarted = true;
             }
             truncate();
             clearTagTrackingState();
             syncedKeys.clear();
-            pendingPresence.clear();
-            usesBaseline = false;
+            recordPendingTruncate();
             isResettingSnapshot = true;
             resetGeneration++;
             loadSubsetDedupe?.reset();
             commitPoint = null;
             hasReceivedUpToDate = false;
             bufferedMessages.length = 0;
+            progressiveSnapshotRows.clear();
           }
         }
         if (needsFullSnapshot && isResettingSnapshot && commitPoint === `subset-end`)
@@ -1186,7 +1220,7 @@ function createElectricSync(shapeOptions, options) {
             debug(
               `${collectionId ? `[${collectionId}] ` : ``}Progressive mode: Performing atomic swap with ${bufferedMessages.length} buffered messages`
             );
-            begin();
+            beginSourceTransaction();
             truncate();
             clearTagTrackingState();
             syncedKeys.clear();
@@ -1209,7 +1243,7 @@ function createElectricSync(shapeOptions, options) {
                   // normal-stream transactionStarted flag is still false.
                   true,
                   (rowId) => {
-                    pendingPresence.set(rowId, false);
+                    recordPendingPresence(rowId, false);
                     syncedKeys.delete(rowId);
                   }
                 );
@@ -1218,8 +1252,9 @@ function createElectricSync(shapeOptions, options) {
               }
             }
             stageResumeMetadata();
-            applied = commit();
+            applied = commitSourceTransaction();
             bufferedMessages.length = 0;
+            progressiveSnapshotRows.clear();
             debug(
               `${collectionId ? `[${collectionId}] ` : ``}Progressive mode: Atomic swap complete, now in normal sync mode`
             );
@@ -1228,12 +1263,12 @@ function createElectricSync(shapeOptions, options) {
               if (!isResettingSnapshot || finishesReset) {
                 stageResumeMetadata();
               }
-              applied = commit();
+              applied = commitSourceTransaction();
               transactionStarted = false;
             } else if (commitPoint === `up-to-date` && metadata) {
-              begin();
+              beginSourceTransaction();
               stageResumeMetadata();
-              applied = commit();
+              applied = commitSourceTransaction();
             }
           }
           const readyErrorVersion = streamErrorVersion;
@@ -1242,7 +1277,15 @@ function createElectricSync(shapeOptions, options) {
           } else {
             void applied.then(
               () => wrappedMarkReady(wasBufferingInitialSync, readyErrorVersion),
-              () => void 0
+              (error) => {
+                if (!isActiveLifecycle() || abortController.signal.aborted) {
+                  return;
+                }
+                streamErrorVersion++;
+                unsubscribeStream();
+                abortController.abort();
+                if (collection.status !== `error`) markError(error);
+              }
             );
           }
           if (finishesReset) {