@tanstack/db-sqlite-persistence-core 0.4.5 → 0.4.8

package/dist/esm/persisted.d.ts

@@ -49,6 +49,7 @@ export type TxCommitted = {
     latestRowVersion: number;
 } & ({
     requiresFullReload: true;
+    reconciliationReload?: true;
 } | {
     requiresFullReload: false;
     changedRows: Array<{
@@ -149,6 +150,41 @@ export type ApplyCommittedTxResponse = {
     sourceCode?: string | number;
     path?: string | ReadonlyArray<string | number>;
 };
+/** The durable version read before a source transaction enters coordination. */
+export type CommittedTxAnchor = {
+    latestRowVersion: number;
+    resetEpoch: number;
+};
+export type ReconcileCommittedTxResponse = {
+    type: `rpc:reconcileCommittedTx:res`;
+    rpcId: string;
+    ok: true;
+    alreadyApplied: boolean;
+    committed: {
+        term: number;
+        seq: number;
+        rowVersion: number;
+    };
+} | {
+    type: `rpc:reconcileCommittedTx:res`;
+    rpcId: string;
+    ok: false;
+    code: `NOT_LEADER` | `UNDETERMINED` | `PERSISTENCE_ERROR`;
+    error: string;
+    sourceCode?: string | number;
+    path?: string | ReadonlyArray<string | number>;
+};
+export type ReconciledCommittedTx = {
+    kind: `already-applied` | `applied-now`;
+    committed: {
+        term: number;
+        seq: number;
+        rowVersion: number;
+    };
+    latestRowVersion: number;
+} | {
+    kind: `unknown`;
+};
 export type PullSinceRequest = {
     type: `rpc:pullSince:req`;
     rpcId: string;
@@ -299,6 +335,8 @@ export interface PersistenceAdapter {
         resetEpoch: number;
     }>;
     applyCommittedTx: (collectionId: string, tx: PersistedTx) => Promise<void>;
+    /** Atomically certify or apply one source transaction after a lost response. */
+    reconcileCommittedTx?: (collectionId: string, tx: PersistedTx, anchor: CommittedTxAnchor) => Promise<ReconciledCommittedTx>;
     loadCollectionMetadata?: (collectionId: string) => Promise<Array<{
         key: string;
         value: unknown;
@@ -311,6 +349,12 @@ export interface PersistenceAdapter {
         latestSeq: number;
         latestRowVersion: number;
     }>;
+    /** Reserve a new durable election term before announcing Browser leadership. */
+    reserveLeadershipTerm?: (collectionId: string, observedTerm: number) => Promise<{
+        latestTerm: number;
+        latestSeq: number;
+        latestRowVersion: number;
+    }>;
     /**
      * Runs one complete logical hydrate as a non-preemptible scheduler unit.
      * The callback must use the supplied unscheduled adapter for all nested
@@ -365,6 +409,8 @@ export interface PersistedCollectionCoordinator {
      * loan after the call; ordinary work uses the registered collection adapter.
      */
     requestApplyCommittedTx: (collectionId: string, tx: PersistedTx, scopedAdapter?: HydrationPersistenceAdapter) => Promise<ApplyCommittedTxResponse>;
+    /** Available only when the elected adapter can certify the durable outcome. */
+    reconcileCommittedTx?: (collectionId: string, tx: PersistedTx, anchor: CommittedTxAnchor, scopedAdapter?: HydrationPersistenceAdapter) => Promise<ReconcileCommittedTxResponse>;
     /** The scoped adapter is leader-local and is never serialized to a follower. */
     pullSince?: (collectionId: string, fromRowVersion: number, scopedAdapter?: HydrationPersistenceAdapter) => Promise<PullSinceResponse>;
 }

package/dist/esm/persisted.js

@@ -1,5 +1,5 @@
-import { safeRandomUUID, withCollectionConfigFactory, whenSyncAccepted, SyncTransactionAbortedError, getLoadSubsetDemandKey, compileSingleRowExpression, toBooleanPredicate, SYNC_PERSISTENCE_VERSION, SYNC_PERSISTENCE_PROTOCOL, PERSISTED_READINESS, NoPendingSyncTransactionCommitError, NoPendingSyncTransactionWriteError, withAcceptedReceipt } from "@tanstack/db";
-import { InvalidPersistedCollectionConfigError, DuplicateRemoteSubsetOwnerError, toPersistedCollectionDurabilityError, InvalidPersistedStorageKeyEncodingError, InvalidPersistedStorageKeyError, InvalidSyncConfigError, InvalidPersistedCollectionCoordinatorError, PersistedCollectionDurabilityError, InvalidPersistenceAdapterError } from "./errors.js";
+import { safeRandomUUID, withCollectionConfigFactory, whenSyncAccepted, SyncTransactionAbortedError, getLoadSubsetDemandKey, equalPersistedSnapshotValues, compileSingleRowExpression, toBooleanPredicate, SYNC_PERSISTENCE_VERSION, SYNC_PERSISTENCE_PROTOCOL, PERSISTED_READINESS, NoPendingSyncTransactionCommitError, NoPendingSyncTransactionWriteError, withAcceptedReceipt } from "@tanstack/db";
+import { InvalidPersistedCollectionConfigError, DuplicateRemoteSubsetOwnerError, toPersistedCollectionDurabilityError, InvalidPersistedStorageKeyEncodingError, InvalidPersistedStorageKeyError, InvalidSyncConfigError, InvalidPersistedCollectionCoordinatorError, PersistedCollectionDurabilityError, IndeterminateCommitError, InvalidPersistenceAdapterError } from "./errors.js";
 import { serializeSQLiteBigInt, serializeSQLiteTemporal } from "./sqlite-value.js";
 import { toTransportedLoadSubsetOptions, toProcessLocalLoadSubsetOptions, validateRemoteSubsetOptions } from "./remote-subset-wire.js";
 import { reportRemoteSubsetOwnerError, unloadRemoteSubsetOwner } from "./remote-subset-owner.js";
@@ -391,7 +391,7 @@ function isTxCommittedPayload(payload) {
     return false;
   }
   if (payload.requiresFullReload) {
-    return true;
+    return payload.reconciliationReload === void 0 || payload.reconciliationReload === true;
   }
   return Array.isArray(payload.changedRows) && Array.isArray(payload.deletedKeys) && (payload.rowMetadataMutations === void 0 || Array.isArray(payload.rowMetadataMutations)) && (payload.collectionMetadataMutations === void 0 || Array.isArray(payload.collectionMetadataMutations));
 }
@@ -457,6 +457,7 @@ class PersistedCollectionRuntime {
     this.latestTerm = 0;
     this.latestSeq = 0;
     this.latestRowVersion = 0;
+    this.publicRowVersion = 0;
     this.localTerm = 1;
     this.localSeq = 0;
     this.localRowVersion = 0;
@@ -1134,6 +1135,8 @@ class PersistedCollectionRuntime {
     this.lifecycleGeneration++;
     this.persistedReadiness?.set({ status: `loading` });
     this.startupSettled = false;
+    this.publicRowVersion = 0;
+    this.resetTermFence = void 0;
     this.hydratedDemands.clear();
     this.startupMetadataPromise = null;
     this.startPromise = null;
@@ -1178,6 +1181,7 @@ class PersistedCollectionRuntime {
   }
   async hydrateSubsetUnsafe(options, config, adapter) {
     let rowsLoaded = false;
+    let boundRowVersion;
     let replayFailure;
     const resetSequence = this.resetSequence;
     this.hydrationSequence++;
@@ -1197,6 +1201,7 @@ class PersistedCollectionRuntime {
           if (config.lifecycleGeneration !== this.lifecycleGeneration) return;
           if (resetSequence !== this.resetSequence) return;
           this.bindResumeSnapshotEvidence(snapshot);
+          boundRowVersion = snapshot.latestRowVersion;
         } else {
           rows = await this.loadSubsetRowsUnsafe(options, adapter);
         }
@@ -1209,6 +1214,12 @@ class PersistedCollectionRuntime {
           }
           const applied = this.applyRowsToCollection(rows);
           await whenSyncAccepted(applied);
+          if (boundRowVersion !== void 0) {
+            this.publicRowVersion = Math.max(
+              this.publicRowVersion,
+              boundRowVersion
+            );
+          }
         }
       } finally {
         if (this.activeHydrationContext === hydrationContext && this.hydratingGeneration === config.lifecycleGeneration) {
@@ -1372,7 +1383,9 @@ class PersistedCollectionRuntime {
   }
   isExpectedResumeGeneration(generation) {
     const expected = this.persistedResumeGeneration;
-    return expected !== void 0 && expected.latestTerm === generation.latestTerm && expected.latestSeq === generation.latestSeq && expected.latestRowVersion === generation.latestRowVersion && expected.resetEpoch === generation.resetEpoch;
+    return expected !== void 0 && expected.latestRowVersion === generation.latestRowVersion && expected.resetEpoch === generation.resetEpoch && (expected.latestTerm === generation.latestTerm && expected.latestSeq === generation.latestSeq || // A no-write election reserves a new term at sequence zero. It does
+    // not change the persisted rows or the source resume cursor.
+    generation.latestTerm > expected.latestTerm && generation.latestSeq === 0);
   }
   bindResumeSnapshotEvidence(snapshot) {
     const generation = this.getResumeSnapshotGeneration(snapshot);
@@ -1414,6 +1427,27 @@ class PersistedCollectionRuntime {
       return this.syncControls.commit?.() ?? true;
     });
   }
+  matchesCollectionSnapshot(rows, collectionMetadata) {
+    if (!this.collection || !this.syncControls.metadata) return false;
+    const base = this.collection.base;
+    if (base.size !== rows.size) return false;
+    for (const [key, row] of rows) {
+      if (!base.has(key) || !equalPersistedSnapshotValues(base.get(key), row.value) || !equalPersistedSnapshotValues(
+        this.syncControls.metadata.row.get(key),
+        row.metadata
+      )) {
+        return false;
+      }
+    }
+    const currentMetadata = this.syncControls.metadata.collection.list();
+    if (currentMetadata.length !== collectionMetadata.length) return false;
+    const expectedMetadata = new Map(
+      collectionMetadata.map(({ key, value }) => [key, value])
+    );
+    return currentMetadata.every(
+      ({ key, value }) => expectedMetadata.has(key) && equalPersistedSnapshotValues(expectedMetadata.get(key), value)
+    );
+  }
   async flushQueuedHydrationTransactionsUnsafe(adapter) {
     let operationFailure;
     while (this.queuedHydrationTransactions.length > 0) {
@@ -1498,16 +1532,55 @@ class PersistedCollectionRuntime {
     if (!transaction.truncate && transaction.operations.length === 0 && transaction.rowMetadataWrites.size === 0 && transaction.collectionMetadataWrites.size === 0) {
       return;
     }
+    const anchor = this.persistedResumeGeneration && {
+      latestRowVersion: this.latestRowVersion,
+      resetEpoch: this.persistedResumeGeneration.resetEpoch
+    };
     const streamPosition = this.nextLocalStreamPosition();
     const tx = this.createPersistedTxFromOperations(transaction, streamPosition);
-    const response = await this.persistence.coordinator.requestApplyCommittedTx(
-      this.collectionId,
-      tx,
-      // A hydration-scoped adapter is already inside the shared scheduler.
-      // Ordinary source commits pass no scoped adapter so the coordinator
-      // enters that scheduler before taking the database-wide writer lock.
-      scopedAdapter
-    );
+    let response;
+    try {
+      response = await this.persistence.coordinator.requestApplyCommittedTx(
+        this.collectionId,
+        tx,
+        // A hydration-scoped adapter is already inside the shared scheduler.
+        // Ordinary source commits enter it before taking the writer lock.
+        scopedAdapter
+      );
+    } catch (error) {
+      if (!(error instanceof IndeterminateCommitError) || !anchor || !this.persistence.coordinator.reconcileCommittedTx) {
+        throw error;
+      }
+      let reconciled;
+      try {
+        reconciled = await this.persistence.coordinator.reconcileCommittedTx(
+          this.collectionId,
+          tx,
+          anchor,
+          scopedAdapter
+        );
+      } catch (reconcileError) {
+        if (reconcileError instanceof PersistedCollectionDurabilityError) {
+          throw reconcileError;
+        }
+        throw error;
+      }
+      if (!reconciled.ok) {
+        if (reconciled.code === `PERSISTENCE_ERROR`) {
+          throw new PersistedCollectionDurabilityError(
+            `Failed to reconcile collection "${this.collectionId}": ${reconciled.error}`,
+            {
+              cause: reconciled,
+              code: reconciled.sourceCode ?? reconciled.code,
+              path: reconciled.path
+            }
+          );
+        }
+        throw error;
+      }
+      this.recordDurableSourceCommit(transaction, reconciled.committed);
+      return;
+    }
     if (!response.ok) {
       if (response.code === `PERSISTENCE_ERROR`) {
         const error = new PersistedCollectionDurabilityError(
@@ -1524,18 +1597,24 @@ class PersistedCollectionRuntime {
         `failed to apply external sync transaction through coordinator: ${response.error}`
       );
     }
+    this.recordDurableSourceCommit(transaction, {
+      term: response.term,
+      seq: response.seq,
+      rowVersion: response.latestRowVersion
+    });
+  }
+  recordDurableSourceCommit(transaction, position) {
     if (transaction.lifecycleGeneration !== this.lifecycleGeneration) return;
-    this.observeStreamPosition(
-      response.term,
-      response.seq,
-      response.latestRowVersion
-    );
+    this.observeStreamPosition(position.term, position.seq, position.rowVersion);
+    if (position.rowVersion === this.publicRowVersion + 1) {
+      this.publicRowVersion = position.rowVersion;
+    }
     if (transaction.expectedResumeGenerationOwner === this.resumeGenerationOwner && this.persistedResumeGeneration !== void 0) {
       this.persistedResumeGeneration = {
         ...this.persistedResumeGeneration,
-        latestTerm: response.term,
-        latestSeq: response.seq,
-        latestRowVersion: response.latestRowVersion
+        latestTerm: position.term,
+        latestSeq: position.seq,
+        latestRowVersion: position.rowVersion
       };
     }
   }
@@ -1898,9 +1977,17 @@ class PersistedCollectionRuntime {
     }
     if (isCollectionResetPayload(payload)) {
       void this.applyMutex.run(async () => {
-        await this.runInHydrationScope(
-          (adapter) => this.truncateAndReloadUnsafe(adapter, lifecycleGeneration)
-        );
+        await this.runInHydrationScope(async (adapter) => {
+          await this.truncateAndReloadUnsafe(adapter, lifecycleGeneration);
+          await this.rebaseDurableStreamFromSnapshotUnsafe(
+            adapter,
+            lifecycleGeneration
+          );
+          if (lifecycleGeneration === this.lifecycleGeneration) {
+            this.publicRowVersion = 0;
+            this.resetTermFence = this.latestTerm;
+          }
+        });
         if (lifecycleGeneration === this.lifecycleGeneration) {
           await this.flushQueuedTxCommittedUnsafe();
         }
@@ -1923,21 +2010,42 @@ class PersistedCollectionRuntime {
   }
   async processCommittedTxUnsafe(txCommitted, lifecycleGeneration = this.lifecycleGeneration) {
     if (lifecycleGeneration !== this.lifecycleGeneration) return;
-    if (txCommitted.term < this.latestTerm) {
-      return;
+    if (this.resetTermFence !== void 0) {
+      if (txCommitted.term <= this.resetTermFence) {
+        await this.runInHydrationScope(
+          (adapter) => this.reloadActiveSubsetsUnsafe(adapter)
+        );
+        return;
+      }
     }
-    if (txCommitted.term === this.latestTerm && txCommitted.seq <= this.latestSeq) {
+    if (txCommitted.requiresFullReload && txCommitted.reconciliationReload) {
+      await this.invalidateFromCommittedTxUnsafe(
+        txCommitted,
+        this.persistence.adapter
+      );
+      if (lifecycleGeneration === this.lifecycleGeneration) {
+        this.observeStreamPosition(
+          txCommitted.term,
+          txCommitted.seq,
+          txCommitted.latestRowVersion
+        );
+        this.publicRowVersion = Math.max(
+          this.publicRowVersion,
+          txCommitted.latestRowVersion
+        );
+        await this.flushQueuedTxCommittedUnsafe();
+      }
       return;
     }
+    if (txCommitted.latestRowVersion <= this.publicRowVersion) return;
+    const observedButUnpublished = txCommitted.term < this.latestTerm || txCommitted.term === this.latestTerm && txCommitted.seq <= this.latestSeq;
     const hasGapInCurrentTerm = txCommitted.term === this.latestTerm && txCommitted.seq > this.latestSeq + 1;
     const hasGapAcrossTerms = txCommitted.term > this.latestTerm && txCommitted.seq > 1;
-    const hasGap = hasGapInCurrentTerm || hasGapAcrossTerms;
+    const hasGap = observedButUnpublished || txCommitted.latestRowVersion > this.publicRowVersion + 1 || hasGapInCurrentTerm || hasGapAcrossTerms;
     if (hasGap) {
       await this.recoverFromSeqGapUnsafe(lifecycleGeneration);
       if (lifecycleGeneration !== this.lifecycleGeneration) return;
-      if (txCommitted.term < this.latestTerm || txCommitted.term === this.latestTerm && txCommitted.seq <= this.latestSeq) {
-        return;
-      }
+      if (txCommitted.latestRowVersion <= this.publicRowVersion) return;
     }
     this.observeStreamPosition(
       txCommitted.term,
@@ -1949,9 +2057,27 @@ class PersistedCollectionRuntime {
       this.persistence.adapter
     );
     if (lifecycleGeneration === this.lifecycleGeneration) {
+      this.publicRowVersion = Math.max(
+        this.publicRowVersion,
+        txCommitted.latestRowVersion
+      );
       await this.flushQueuedTxCommittedUnsafe();
     }
   }
+  async rebaseDurableStreamFromSnapshotUnsafe(adapter, lifecycleGeneration) {
+    if (lifecycleGeneration !== this.lifecycleGeneration) return;
+    const snapshot = await adapter.loadResumeSnapshot(this.collectionId, {
+      includeRows: false
+    });
+    if (lifecycleGeneration !== this.lifecycleGeneration) return;
+    this.latestRowVersion = snapshot.latestRowVersion;
+    this.localRowVersion = snapshot.latestRowVersion;
+    this.observeStreamPosition(
+      snapshot.latestTerm,
+      snapshot.latestSeq,
+      snapshot.latestRowVersion
+    );
+  }
   async recoverFromSeqGapUnsafe(lifecycleGeneration) {
     if (lifecycleGeneration !== this.lifecycleGeneration) return;
     if (this.persistence.coordinator.pullSince && this.latestRowVersion >= 0) {
@@ -1959,7 +2085,7 @@ class PersistedCollectionRuntime {
       try {
         pullResponse = await this.persistence.coordinator.pullSince(
           this.collectionId,
-          this.latestRowVersion
+          this.publicRowVersion
         );
       } catch (error) {
         console.warn(`Failed pullSince recovery attempt:`, error);
@@ -1976,6 +2102,10 @@ class PersistedCollectionRuntime {
             await this.runInHydrationScope(
               (adapter) => this.reloadActiveSubsetsUnsafe(adapter)
             );
+            this.publicRowVersion = Math.max(
+              this.publicRowVersion,
+              pullResponse.latestRowVersion
+            );
             return;
           }
           const deltas = pullResponse.deltas;
@@ -1983,6 +2113,10 @@ class PersistedCollectionRuntime {
             await this.runInHydrationScope(
               (adapter) => this.reloadActiveSubsetsUnsafe(adapter)
             );
+            this.publicRowVersion = Math.max(
+              this.publicRowVersion,
+              pullResponse.latestRowVersion
+            );
             return;
           }
           await this.runInHydrationScope(async (adapter) => {
@@ -2006,13 +2140,21 @@ class PersistedCollectionRuntime {
               if (lifecycleGeneration !== this.lifecycleGeneration) return;
             }
           });
+          this.publicRowVersion = Math.max(
+            this.publicRowVersion,
+            pullResponse.latestRowVersion
+          );
           return;
         }
       }
     }
     if (lifecycleGeneration !== this.lifecycleGeneration) return;
     await this.runInHydrationScope(
-      (adapter) => this.truncateAndReloadUnsafe(adapter, lifecycleGeneration)
+      (adapter) => this.reloadActiveSubsetsUnsafe(adapter)
+    );
+    this.publicRowVersion = Math.max(
+      this.publicRowVersion,
+      this.latestRowVersion
     );
     if (this.mode === `sync-present`) {
       for (const options of this.activeSubsets.values()) {
@@ -2036,12 +2178,12 @@ class PersistedCollectionRuntime {
     await this.reloadActiveSubsetsUnsafe(adapter);
   }
   async invalidateFromCommittedTxUnsafe(txCommitted, adapter) {
-    const reloadActiveSubsets = () => this.runInHydrationScope(
-      (scopedAdapter) => this.reloadActiveSubsetsUnsafe(scopedAdapter),
+    const reloadActiveSubsets = (skipUnchangedReplace = false) => this.runInHydrationScope(
+      (scopedAdapter) => this.reloadActiveSubsetsUnsafe(scopedAdapter, skipUnchangedReplace),
       adapter
     );
     if (txCommitted.requiresFullReload) {
-      await reloadActiveSubsets();
+      await reloadActiveSubsets(txCommitted.reconciliationReload === true);
       return;
     }
     const changedKeyCount = txCommitted.changedRows.length + txCommitted.deletedKeys.length;
@@ -2105,7 +2247,7 @@ class PersistedCollectionRuntime {
     });
     await whenSyncAccepted(applied);
   }
-  async reloadActiveSubsetsUnsafe(adapter) {
+  async reloadActiveSubsetsUnsafe(adapter, skipUnchangedReplace = false) {
     const lifecycleGeneration = this.lifecycleGeneration;
     const truncateGeneration = this.sourceTruncateGeneration;
     const activeSubsetOptions = this.activeSubsets.size > 0 ? Array.from(this.activeSubsets.values()) : [{}];
@@ -2131,15 +2273,29 @@ class PersistedCollectionRuntime {
       for (const key of mergedRows.keys()) {
         hydrationContext.suppliedRowKeys.add(key);
       }
-      const applied = this.replaceCollectionSnapshot(
-        Array.from(mergedRows.entries()).map(([key, row]) => ({
-          key,
-          value: row.value,
-          metadata: row.metadata
-        })),
-        collectionMetadata
-      );
-      await whenSyncAccepted(applied);
+      if (skipUnchangedReplace && this.matchesCollectionSnapshot(mergedRows, collectionMetadata)) {
+        const orphanKeys = this.collection?._syncedRowMetadataKeysOutsideBase() ?? [];
+        if (orphanKeys.length > 0) {
+          const applied = this.withInternalApply(() => {
+            this.syncControls.begin?.();
+            for (const key of orphanKeys) {
+              this.syncControls.metadata?.row.delete(key);
+            }
+            return this.syncControls.commit?.() ?? true;
+          });
+          await whenSyncAccepted(applied);
+        }
+      } else {
+        const applied = this.replaceCollectionSnapshot(
+          Array.from(mergedRows.entries()).map(([key, row]) => ({
+            key,
+            value: row.value,
+            metadata: row.metadata
+          })),
+          collectionMetadata
+        );
+        await whenSyncAccepted(applied);
+      }
     } finally {
       if (this.activeHydrationContext === hydrationContext && this.hydratingGeneration === lifecycleGeneration) {
         this.hydratingGeneration = null;