@tanstack/db-sqlite-persistence-core 0.4.7 → 0.4.9

package/dist/esm/broadcast-coordinator.js

@@ -1,7 +1,9 @@
 import { safeRandomUUID } from "@tanstack/db";
-import { DuplicateRemoteSubsetOwnerError, RetryableRemoteSubsetAcquisitionError, IndeterminateCommitError, toPersistedCollectionDurabilityError, PersistedCollectionDurabilityError } from "./errors.js";
+import { isValidCommittedTxAnchor } from "./committed-tx-anchor.js";
+import { DuplicateRemoteSubsetOwnerError, InvalidPersistenceAdapterError, RetryableRemoteSubsetAcquisitionError, IndeterminateCommitError, InvalidPersistedCollectionConfigError, toPersistedCollectionDurabilityError, PersistedCollectionDurabilityError } from "./errors.js";
 import { toTransportedLoadSubsetOptions, toProcessLocalLoadSubsetOptions } from "./remote-subset-wire.js";
 import { unloadRemoteSubsetOwner } from "./remote-subset-owner.js";
+import { managedStorageTarget, eagerStorageTarget } from "./persisted.js";
 const HEARTBEAT_INTERVAL_MS = 3e3;
 const LEADERSHIP_RETRY_DELAY_MS = 200;
 const RPC_TIMEOUT_MS = 1e4;
@@ -11,10 +13,16 @@ const REMOTE_SUBSET_REPLAY_RETRY_ATTEMPTS = 2;
 const RPC_DEDUPE_RETENTION_MS = 6e4;
 const WRITER_LOCK_BUSY_RETRY_MS = 50;
 const WRITER_LOCK_MAX_RETRIES = 20;
+function requireDurableElection(adapter) {
+  if (typeof adapter.reserveLeadershipTerm !== `function`) {
+    throw new InvalidPersistenceAdapterError(`reserveLeadershipTerm`);
+  }
+}
 class BroadcastCollectionCoordinator {
   constructor(options) {
     this.nodeId = safeRandomUUID();
     this.collectionAdapters = /* @__PURE__ */ new Map();
+    this.collectionClaimIds = /* @__PURE__ */ new Map();
     this.remoteSubsetOwners = /* @__PURE__ */ new Map();
     this.remoteSubsetIds = /* @__PURE__ */ new Map();
     this.outboundRemoteSubsetAcquisitions = /* @__PURE__ */ new Map();
@@ -27,6 +35,7 @@ class BroadcastCollectionCoordinator {
     this.appliedEnvelopePruneTimer = null;
     this.rejectDisposed = null;
     this.disposed = false;
+    if (options.adapter) requireDurableElection(options.adapter);
     this.dbName = options.dbName;
     this.coordinatorName = options.coordinatorName;
     this.defaultAdapter = options.adapter ?? null;
@@ -65,11 +74,18 @@ class BroadcastCollectionCoordinator {
    * Called by a host persistence factory to wire the internally-created adapter.
    */
   setAdapter(adapter) {
+    requireDurableElection(adapter);
     this.defaultAdapter = adapter;
   }
   /** Register the persistence adapter that owns one collection. */
-  setAdapterForCollection(collectionId, adapter) {
+  setAdapterForCollection(collectionId, adapter, cacheGenerationClaimId) {
+    requireDurableElection(adapter);
     this.collectionAdapters.set(collectionId, adapter);
+    if (cacheGenerationClaimId) {
+      this.collectionClaimIds.set(collectionId, cacheGenerationClaimId);
+    } else {
+      this.collectionClaimIds.delete(collectionId);
+    }
   }
   registerRemoteSubsetOwner(collectionId, owner) {
     if (this.remoteSubsetOwners.has(collectionId)) {
@@ -213,10 +229,29 @@ class BroadcastCollectionCoordinator {
     }
     try {
       await acquisition.release;
+    } catch (error) {
+      this.scheduleRemoteSubsetReleaseRetry(acquisition, options);
+      throw error;
     } finally {
       acquisition.release = null;
     }
   }
+  scheduleRemoteSubsetReleaseRetry(acquisition, options) {
+    const key = remoteSubsetAcquisitionKey(
+      acquisition.collectionId,
+      acquisition.acquisitionId
+    );
+    if (this.isDisposed() || this.outboundRemoteSubsetAcquisitions.get(key) !== acquisition || acquisition.retryTimer !== null) {
+      return;
+    }
+    acquisition.retryTimer = setTimeout(() => {
+      acquisition.retryTimer = null;
+      void this.requestReleaseRemoteSubset(
+        acquisition.collectionId,
+        options
+      ).catch(() => void 0);
+    }, RPC_RETRY_DELAY_MS);
+  }
   async acquireRemoteSubset(acquisition) {
     if (acquisition.inFlight) return acquisition.inFlight;
     const route = { localOwner: false };
@@ -276,7 +311,9 @@ class BroadcastCollectionCoordinator {
     try {
       await work;
       acquired = true;
-      this.cancelRemoteSubsetReplayRetry(acquisition);
+      if (!acquisition.releaseRequested) {
+        this.cancelRemoteSubsetReplayRetry(acquisition);
+      }
     } finally {
       if (acquisition.inFlight === work) acquisition.inFlight = null;
       const current = this.collections.get(acquisition.collectionId);
@@ -293,21 +330,34 @@ class BroadcastCollectionCoordinator {
       }
     }
   }
-  async requestEnsurePersistedIndex(collectionId, signature, spec, scopedAdapter, localEnsureCompleted = false) {
+  async requestEnsurePersistedIndex(collectionId, signature, spec, scopedAdapter, localEnsureCompleted = false, cacheGenerationClaimId) {
     if (this.isLeader(collectionId)) {
       if (localEnsureCompleted) return;
-      await (scopedAdapter ?? this.requireAdapter(collectionId)).ensureIndex(
-        collectionId,
-        signature,
-        spec
-      );
+      const adapter = scopedAdapter ?? this.requireAdapter(collectionId);
+      if (cacheGenerationClaimId) {
+        await adapter.ensureIndex(
+          managedStorageTarget(collectionId, cacheGenerationClaimId),
+          signature,
+          spec,
+          {
+            cacheGenerationClaimId
+          }
+        );
+      } else {
+        await adapter.ensureIndex(
+          eagerStorageTarget(collectionId),
+          signature,
+          spec
+        );
+      }
       return;
     }
     const response = await this.sendRPC(collectionId, {
       type: `rpc:ensurePersistedIndex:req`,
       rpcId: safeRandomUUID(),
       signature,
-      spec
+      spec,
+      cacheGenerationClaimId
     });
     if (!response.ok) {
       throw new Error(
@@ -315,7 +365,7 @@ class BroadcastCollectionCoordinator {
       );
     }
   }
-  async requestApplyLocalMutations(collectionId, mutations) {
+  async requestApplyLocalMutations(collectionId, mutations, cacheGenerationClaimId) {
     const initialRoute = this.waitForInitialRoute(collectionId);
     if (initialRoute) await initialRoute;
     if (this.isLeader(collectionId)) {
@@ -323,14 +373,16 @@ class BroadcastCollectionCoordinator {
         type: `rpc:applyLocalMutations:req`,
         rpcId: safeRandomUUID(),
         envelopeId: safeRandomUUID(),
-        mutations
+        mutations,
+        cacheGenerationClaimId
       });
     }
     return this.sendRPC(collectionId, {
       type: `rpc:applyLocalMutations:req`,
       rpcId: safeRandomUUID(),
       envelopeId: safeRandomUUID(),
-      mutations
+      mutations,
+      cacheGenerationClaimId
     });
   }
   async requestApplyCommittedTx(collectionId, tx, scopedAdapter) {
@@ -347,14 +399,39 @@ class BroadcastCollectionCoordinator {
     }
     return this.sendRPC(collectionId, request);
   }
-  async pullSince(collectionId, fromRowVersion, scopedAdapter) {
+  async reconcileCommittedTx(collectionId, tx, anchor, scopedAdapter) {
+    const initialRoute = this.waitForInitialRoute(collectionId);
+    if (initialRoute) await initialRoute;
+    let response;
+    for (let attempt = 0; attempt <= RPC_RETRY_ATTEMPTS; attempt++) {
+      if (attempt > 0) await sleep(RPC_RETRY_DELAY_MS * attempt);
+      const request = {
+        type: `rpc:reconcileCommittedTx:req`,
+        rpcId: safeRandomUUID(),
+        tx,
+        anchor
+      };
+      response = this.isLeader(collectionId) ? await this.handleReconcileCommittedTx(
+        collectionId,
+        request,
+        scopedAdapter
+      ) : await this.sendRPC(
+        collectionId,
+        request
+      );
+      if (response.ok || response.code !== `NOT_LEADER`) return response;
+    }
+    return response;
+  }
+  async pullSince(collectionId, fromRowVersion, scopedAdapter, cacheGenerationClaimId) {
     if (this.isLeader(collectionId)) {
       return this.handlePullSince(
         collectionId,
         {
           type: `rpc:pullSince:req`,
           rpcId: safeRandomUUID(),
-          fromRowVersion
+          fromRowVersion,
+          cacheGenerationClaimId
         },
         scopedAdapter
       );
@@ -362,7 +439,8 @@ class BroadcastCollectionCoordinator {
     return this.sendRPC(collectionId, {
       type: `rpc:pullSince:req`,
       rpcId: safeRandomUUID(),
-      fromRowVersion
+      fromRowVersion,
+      cacheGenerationClaimId
     });
   }
   // -----------------------------------------------------------------------
@@ -395,6 +473,7 @@ class BroadcastCollectionCoordinator {
     this.channel.close();
     this.collections.clear();
     this.collectionAdapters.clear();
+    this.collectionClaimIds.clear();
     for (const collectionId of this.remoteSubsetOwners.keys()) {
       this.releaseInboundRemoteSubsetAcquisitions(collectionId);
     }
@@ -452,16 +531,24 @@ class BroadcastCollectionCoordinator {
           }
           try {
             const adapter = this.requireAdapter(collectionId);
-            if (adapter.getStreamPosition) {
-              const pos = await adapter.getStreamPosition(collectionId);
-              state.latestTerm = pos.latestTerm;
-              state.latestSeq = pos.latestSeq;
-              state.latestRowVersion = pos.latestRowVersion;
-            }
+            const reserveTerm = adapter.reserveLeadershipTerm;
+            if (!reserveTerm)
+              throw new InvalidPersistenceAdapterError(`reserveLeadershipTerm`);
+            const pos = await this.withWriterLock(() => {
+              const claimId = this.collectionClaimIds.get(collectionId);
+              return reserveTerm.call(
+                adapter,
+                claimId ? managedStorageTarget(collectionId, claimId) : eagerStorageTarget(collectionId),
+                state.latestTerm,
+                { cacheGenerationClaimId: claimId }
+              );
+            });
+            state.latestTerm = pos.latestTerm;
+            state.latestSeq = pos.latestSeq;
+            state.latestRowVersion = pos.latestRowVersion;
             if (this.isDisposed() || !this.participatesInLeadership(state) || this.collections.get(collectionId) !== state || abortController.signal.aborted || state.lockAbortController !== abortController) {
               return;
             }
-            state.latestTerm++;
             state.isLeader = true;
             state.leaderId = this.nodeId;
             this.notifyRouteWaiters(state);
@@ -602,6 +689,7 @@ class BroadcastCollectionCoordinator {
     this.releaseLeadership(collectionId, state);
     this.collections.delete(collectionId);
     this.collectionAdapters.delete(collectionId);
+    this.collectionClaimIds.delete(collectionId);
     const prefix = `${JSON.stringify([collectionId]).slice(0, -1)},`;
     for (const key of this.appliedEnvelopes.keys()) {
       if (key.startsWith(prefix)) this.appliedEnvelopes.delete(key);
@@ -718,6 +806,7 @@ class BroadcastCollectionCoordinator {
       const heartbeat = payload;
       if (typeof heartbeat.leaderId === `string` && typeof heartbeat.term === `number` && typeof heartbeat.latestSeq === `number` && typeof heartbeat.latestRowVersion === `number`) {
         const state2 = this.getOrCreateCollectionState(envelope.collectionId);
+        if (state2.isLeader) return;
         if (heartbeat.term < state2.latestTerm) return;
         const changedLeader = state2.leaderId !== heartbeat.leaderId;
         state2.leaderId = heartbeat.leaderId;
@@ -898,6 +987,8 @@ class BroadcastCollectionCoordinator {
         return this.handleApplyLocalMutations(collectionId, request);
       case `rpc:applyCommittedTx:req`:
         return this.handleApplyCommittedTx(collectionId, request);
+      case `rpc:reconcileCommittedTx:req`:
+        return this.handleReconcileCommittedTx(collectionId, request);
       case `rpc:pullSince:req`:
         return this.handlePullSince(collectionId, request);
     }
@@ -1012,10 +1103,7 @@ class BroadcastCollectionCoordinator {
     acquisition.released = true;
     acquisition.release = (async () => {
       try {
-        await acquisition.load;
-      } catch {
-      }
-      try {
+        await acquisition.admitted;
         if (acquisition.transferred) {
           await unloadRemoteSubsetOwner(acquisition.owner, acquisition.options);
         }
@@ -1131,6 +1219,7 @@ class BroadcastCollectionCoordinator {
       owner: options.owner,
       options: options.options,
       load: Promise.resolve(),
+      admitted: Promise.resolve(),
       transferred: false,
       released: false,
       terminalRelease: false,
@@ -1138,6 +1227,10 @@ class BroadcastCollectionCoordinator {
     };
     this.releasedRemoteSubsetAcquisitionTimes.delete(key);
     this.inboundRemoteSubsetAcquisitions.set(key, acquisition);
+    let resolveAdmission;
+    acquisition.admitted = new Promise((resolve) => {
+      resolveAdmission = resolve;
+    });
     let resolveLoad;
     let rejectLoad;
     acquisition.load = new Promise((resolve, reject) => {
@@ -1150,6 +1243,8 @@ class BroadcastCollectionCoordinator {
       void Promise.resolve(load).then(resolveLoad, rejectLoad);
     } catch (error) {
       rejectLoad(error);
+    } finally {
+      resolveAdmission();
     }
     try {
       await acquisition.load;
@@ -1174,10 +1269,23 @@ class BroadcastCollectionCoordinator {
     }
   }
   async handleEnsurePersistedIndex(collectionId, request) {
-    await this.withScheduledWriterLock(
-      collectionId,
-      (adapter) => adapter.ensureIndex(collectionId, request.signature, request.spec)
-    );
+    await this.withScheduledWriterLock(collectionId, (adapter) => {
+      if (request.cacheGenerationClaimId) {
+        return adapter.ensureIndex(
+          managedStorageTarget(collectionId, request.cacheGenerationClaimId),
+          request.signature,
+          request.spec,
+          {
+            cacheGenerationClaimId: request.cacheGenerationClaimId
+          }
+        );
+      }
+      return adapter.ensureIndex(
+        eagerStorageTarget(collectionId),
+        request.signature,
+        request.spec
+      );
+    });
     return {
       type: `rpc:ensurePersistedIndex:res`,
       rpcId: request.rpcId,
@@ -1206,6 +1314,11 @@ class BroadcastCollectionCoordinator {
     });
   }
   async applyLocalMutationsOnce(collectionId, request) {
+    if (this.collectionClaimIds.has(collectionId) && !request.cacheGenerationClaimId) {
+      throw new InvalidPersistedCollectionConfigError(
+        `A managed local mutation requires a persisted cache claim`
+      );
+    }
     const state = this.collections.get(collectionId);
     if (!state || !state.isLeader) {
       return {
@@ -1231,6 +1344,7 @@ class BroadcastCollectionCoordinator {
     }
     const pendingTx = {
       txId: safeRandomUUID(),
+      cacheGenerationClaimId: request.cacheGenerationClaimId,
       mutations: request.mutations.map((m) => ({
         type: m.type,
         key: m.key,
@@ -1391,6 +1505,10 @@ class BroadcastCollectionCoordinator {
       }
     );
     this.pruneAppliedEnvelopes();
+    this.publishCommittedTx(collectionId, state, tx);
+    return response;
+  }
+  publishCommittedTx(collectionId, state, tx, forceReload = false) {
     const committedBase = {
       type: `tx:committed`,
       term: tx.term,
@@ -1398,9 +1516,10 @@ class BroadcastCollectionCoordinator {
       txId: tx.txId,
       latestRowVersion: tx.rowVersion
     };
-    const committedPayload = tx.truncate ? {
+    const committedPayload = tx.truncate || forceReload ? {
       ...committedBase,
-      requiresFullReload: true
+      requiresFullReload: true,
+      ...forceReload ? { reconciliationReload: true } : {}
     } : {
       ...committedBase,
       requiresFullReload: false,
@@ -1424,24 +1543,123 @@ class BroadcastCollectionCoordinator {
     for (const subscriber of state.subscribers) {
       subscriber(committed);
     }
+  }
+  async handleReconcileCommittedTx(collectionId, request, scopedAdapter) {
+    const state = this.collections.get(collectionId);
+    const notLeader = () => ({
+      type: `rpc:reconcileCommittedTx:res`,
+      rpcId: request.rpcId,
+      ok: false,
+      code: `NOT_LEADER`,
+      error: `not the leader for ${collectionId}`
+    });
+    if (!state?.isLeader) return notLeader();
+    if (!isValidCommittedTxAnchor(request.anchor) || !hasReconciliationTxPayload(request.tx)) {
+      return {
+        type: `rpc:reconcileCommittedTx:res`,
+        rpcId: request.rpcId,
+        ok: false,
+        code: `UNDETERMINED`,
+        error: `the source transaction has no valid durable proof`
+      };
+    }
+    const expectedTerm = state.latestTerm;
+    const result = await this.withScheduledWriterLock(
+      collectionId,
+      async (adapter) => {
+        const candidate = this.nextPersistedTxIfLeader(
+          collectionId,
+          state,
+          expectedTerm,
+          request.tx
+        );
+        if (candidate === null) return null;
+        if (!adapter.reconcileCommittedTx) return { kind: `unknown` };
+        try {
+          const reconciled = await adapter.reconcileCommittedTx(
+            candidate.cacheGenerationClaimId ? managedStorageTarget(
+              collectionId,
+              candidate.cacheGenerationClaimId
+            ) : eagerStorageTarget(collectionId),
+            candidate,
+            request.anchor
+          );
+          if (reconciled.kind !== `unknown`) {
+            state.latestRowVersion = Math.max(
+              state.latestRowVersion,
+              reconciled.latestRowVersion
+            );
+            if (reconciled.kind === `applied-now`) {
+              state.latestSeq = reconciled.committed.seq;
+            }
+          }
+          return reconciled;
+        } catch (error) {
+          throw toPersistedCollectionDurabilityError(collectionId, error);
+        }
+      },
+      scopedAdapter
+    );
+    if (result === null) return notLeader();
+    if (result.kind === `unknown`) {
+      return {
+        type: `rpc:reconcileCommittedTx:res`,
+        rpcId: request.rpcId,
+        ok: false,
+        code: `UNDETERMINED`,
+        error: `the durable outcome cannot be certified`
+      };
+    }
+    const current = {
+      term: state.latestTerm,
+      seq: state.latestSeq,
+      rowVersion: state.latestRowVersion
+    };
+    const response = {
+      type: `rpc:reconcileCommittedTx:res`,
+      rpcId: request.rpcId,
+      ok: true,
+      alreadyApplied: result.kind === `already-applied`,
+      committed: result.committed
+    };
+    if (!this.isDisposed() && this.collections.get(collectionId) === state) {
+      this.publishCommittedTx(
+        collectionId,
+        state,
+        result.kind === `already-applied` ? { ...request.tx, ...current } : { ...request.tx, ...result.committed },
+        result.kind === `already-applied`
+      );
+    }
     return response;
   }
+  nextPersistedTxIfLeader(collectionId, state, expectedTerm, pendingTx) {
+    if (this.isDisposed() || this.collections.get(collectionId) !== state || !state.isLeader || state.leaderId !== this.nodeId || state.latestTerm !== expectedTerm) {
+      return null;
+    }
+    return {
+      ...pendingTx,
+      term: state.latestTerm,
+      seq: state.latestSeq + 1,
+      rowVersion: state.latestRowVersion + 1
+    };
+  }
   async applyDurablyAtNextStreamPosition(collectionId, state, pendingTx, scopedAdapter) {
     const expectedTerm = state.latestTerm;
     return this.withScheduledWriterLock(
       collectionId,
       async (adapter) => {
-        if (this.isDisposed() || this.collections.get(collectionId) !== state || !state.isLeader || state.leaderId !== this.nodeId || state.latestTerm !== expectedTerm) {
-          return null;
-        }
-        const tx = {
-          ...pendingTx,
-          term: state.latestTerm,
-          seq: state.latestSeq + 1,
-          rowVersion: state.latestRowVersion + 1
-        };
+        const tx = this.nextPersistedTxIfLeader(
+          collectionId,
+          state,
+          expectedTerm,
+          pendingTx
+        );
+        if (tx === null) return null;
         try {
-          await adapter.applyCommittedTx(collectionId, tx);
+          await adapter.applyCommittedTx(
+            tx.cacheGenerationClaimId ? managedStorageTarget(collectionId, tx.cacheGenerationClaimId) : eagerStorageTarget(collectionId),
+            tx
+          );
         } catch (error) {
           throw toPersistedCollectionDurabilityError(collectionId, error);
         }
@@ -1466,7 +1684,13 @@ class BroadcastCollectionCoordinator {
         requiresFullReload: true
       };
     }
-    const result = await adapter.pullSince(collectionId, request.fromRowVersion);
+    const result = await adapter.pullSince(
+      request.cacheGenerationClaimId ? managedStorageTarget(collectionId, request.cacheGenerationClaimId) : eagerStorageTarget(collectionId),
+      request.fromRowVersion,
+      {
+        cacheGenerationClaimId: request.cacheGenerationClaimId
+      }
+    );
     if (result.requiresFullReload) {
       return {
         type: `rpc:pullSince:res`,
@@ -1582,12 +1806,18 @@ function isRPCRequest(payload) {
     case `rpc:ensurePersistedIndex:req`:
     case `rpc:applyLocalMutations:req`:
     case `rpc:applyCommittedTx:req`:
+    case `rpc:reconcileCommittedTx:req`:
     case `rpc:pullSince:req`:
       return true;
     default:
       return false;
   }
 }
+function hasReconciliationTxPayload(value) {
+  if (!value || typeof value !== `object`) return false;
+  const tx = value;
+  return typeof tx.txId === `string` && Array.isArray(tx.mutations);
+}
 function sleep(ms) {
   return new Promise((resolve) => setTimeout(resolve, ms));
 }
@@ -1661,6 +1891,15 @@ function createRPCErrorResponse(request, cause) {
         code: `CONFLICT`,
         error
       };
+    case `rpc:reconcileCommittedTx:req`:
+      return {
+        type: `rpc:reconcileCommittedTx:res`,
+        rpcId: request.rpcId,
+        ok: false,
+        code: cause instanceof PersistedCollectionDurabilityError ? `PERSISTENCE_ERROR` : `UNDETERMINED`,
+        error,
+        ...cause instanceof PersistedCollectionDurabilityError ? toSafeDurabilityDetails(cause) : {}
+      };
     case `rpc:pullSince:req`:
       return {
         type: `rpc:pullSince:res`,