@tanstack/db-sqlite-persistence-core 0.4.5 → 0.4.8

package/dist/cjs/sqlite-core-adapter.d.cts

@@ -1,5 +1,5 @@
 import { LoadSubsetOptions } from '@tanstack/db';
-import { HydrationPersistenceAdapter, PersistedIndexSpec, PersistedKeySetEvidence, PersistedRowScanOptions, PersistedScannedRow, PersistedTx, PersistenceAdapter, ReplayableTxDelta, SQLiteDriver } from './persisted.cjs';
+import { CommittedTxAnchor, HydrationPersistenceAdapter, PersistedIndexSpec, PersistedKeySetEvidence, PersistedRowScanOptions, PersistedScannedRow, PersistedTx, PersistenceAdapter, ReconciledCommittedTx, ReplayableTxDelta, SQLiteDriver } from './persisted.cjs';
 type SQLiteCoreAdapterSchemaMismatchPolicy = `sync-present-reset` | `sync-absent-error` | `reset`;
 export type SQLiteCoreAdapterOptions = {
     driver: SQLiteDriver;
@@ -83,7 +83,10 @@ export declare class SQLiteCorePersistenceAdapter implements PersistenceAdapter
     }>;
     private loadResumeSnapshotUnscheduled;
     applyCommittedTx(collectionId: string, tx: PersistedTx): Promise<void>;
+    reconcileCommittedTx(collectionId: string, tx: PersistedTx, anchor: CommittedTxAnchor): Promise<ReconciledCommittedTx>;
     private applyCommittedTxUnscheduled;
+    private reconcileCommittedTxUnscheduled;
+    private applyCommittedTxInternal;
     loadCollectionMetadata(collectionId: string): Promise<Array<{
         key: string;
         value: unknown;
@@ -95,6 +98,12 @@ export declare class SQLiteCorePersistenceAdapter implements PersistenceAdapter
     private ensureIndexUnscheduled;
     markIndexRemoved(collectionId: string, signature: string): Promise<void>;
     private markIndexRemovedUnscheduled;
+    reserveLeadershipTerm(collectionId: string, observedTerm: number): Promise<{
+        latestTerm: number;
+        latestSeq: number;
+        latestRowVersion: number;
+    }>;
+    private reserveLeadershipTermUnscheduled;
     getStreamPosition(collectionId: string): Promise<{
         latestTerm: number;
         latestSeq: number;

package/dist/esm/broadcast-coordinator.d.ts

@@ -1,7 +1,7 @@
 import { LoadSubsetOptions } from '@tanstack/db';
 import { RemoteSubsetOwner } from './remote-subset-owner.js';
-import { ApplyCommittedTxResponse, ApplyLocalMutationsResponse, HydrationPersistenceAdapter, PersistedCollectionCoordinator, PersistedIndexSpec, PersistedMutationEnvelope, PersistedTx, PersistenceAdapter, ProtocolEnvelope, PullSinceResponse } from './persisted.js';
-type CoordinatorAdapter = Pick<PersistenceAdapter, `loadSubset` | `applyCommittedTx` | `ensureIndex` | `runInRegularScope`> & {
+import { ApplyCommittedTxResponse, ApplyLocalMutationsResponse, CommittedTxAnchor, HydrationPersistenceAdapter, PersistedCollectionCoordinator, PersistedIndexSpec, PersistedMutationEnvelope, PersistedTx, PersistenceAdapter, ProtocolEnvelope, PullSinceResponse, ReconcileCommittedTxResponse } from './persisted.js';
+type CoordinatorAdapter = Pick<PersistenceAdapter, `loadSubset` | `applyCommittedTx` | `reconcileCommittedTx` | `ensureIndex` | `runInRegularScope`> & {
     pullSince?: (collectionId: string, fromRowVersion: number) => Promise<{
         latestRowVersion: number;
         requiresFullReload: true;
@@ -16,6 +16,11 @@ type CoordinatorAdapter = Pick<PersistenceAdapter, `loadSubset` | `applyCommitte
         latestSeq: number;
         latestRowVersion: number;
     }>;
+    reserveLeadershipTerm?: (collectionId: string, observedTerm: number) => Promise<{
+        latestTerm: number;
+        latestSeq: number;
+        latestRowVersion: number;
+    }>;
 };
 /** @internal Shared by the Browser and Electron persistence hosts. */
 export type BroadcastCollectionCoordinatorOptions = {
@@ -69,6 +74,7 @@ export declare class BroadcastCollectionCoordinator implements PersistedCollecti
     requestEnsurePersistedIndex(collectionId: string, signature: string, spec: PersistedIndexSpec, scopedAdapter?: HydrationPersistenceAdapter, localEnsureCompleted?: boolean): Promise<void>;
     requestApplyLocalMutations(collectionId: string, mutations: Array<PersistedMutationEnvelope>): Promise<ApplyLocalMutationsResponse>;
     requestApplyCommittedTx(collectionId: string, tx: PersistedTx, scopedAdapter?: HydrationPersistenceAdapter): Promise<ApplyCommittedTxResponse>;
+    reconcileCommittedTx(collectionId: string, tx: PersistedTx, anchor: CommittedTxAnchor, scopedAdapter?: HydrationPersistenceAdapter): Promise<ReconcileCommittedTxResponse>;
     pullSince(collectionId: string, fromRowVersion: number, scopedAdapter?: HydrationPersistenceAdapter): Promise<PullSinceResponse>;
     dispose(): void;
     private getOrCreateCollectionState;
@@ -105,6 +111,9 @@ export declare class BroadcastCollectionCoordinator implements PersistedCollecti
     private handleApplyCommittedTx;
     private runDeduplicatedEnvelope;
     private applyCommittedTxOnce;
+    private publishCommittedTx;
+    private handleReconcileCommittedTx;
+    private nextPersistedTxIfLeader;
     private applyDurablyAtNextStreamPosition;
     private handlePullSince;
     private withScheduledWriterLock;

package/dist/esm/broadcast-coordinator.js

@@ -1,5 +1,6 @@
 import { safeRandomUUID } from "@tanstack/db";
-import { DuplicateRemoteSubsetOwnerError, RetryableRemoteSubsetAcquisitionError, IndeterminateCommitError, toPersistedCollectionDurabilityError, PersistedCollectionDurabilityError } from "./errors.js";
+import { isValidCommittedTxAnchor } from "./committed-tx-anchor.js";
+import { DuplicateRemoteSubsetOwnerError, InvalidPersistenceAdapterError, RetryableRemoteSubsetAcquisitionError, IndeterminateCommitError, toPersistedCollectionDurabilityError, PersistedCollectionDurabilityError } from "./errors.js";
 import { toTransportedLoadSubsetOptions, toProcessLocalLoadSubsetOptions } from "./remote-subset-wire.js";
 import { unloadRemoteSubsetOwner } from "./remote-subset-owner.js";
 const HEARTBEAT_INTERVAL_MS = 3e3;
@@ -11,6 +12,11 @@ const REMOTE_SUBSET_REPLAY_RETRY_ATTEMPTS = 2;
 const RPC_DEDUPE_RETENTION_MS = 6e4;
 const WRITER_LOCK_BUSY_RETRY_MS = 50;
 const WRITER_LOCK_MAX_RETRIES = 20;
+function requireDurableElection(adapter) {
+  if (typeof adapter.reserveLeadershipTerm !== `function`) {
+    throw new InvalidPersistenceAdapterError(`reserveLeadershipTerm`);
+  }
+}
 class BroadcastCollectionCoordinator {
   constructor(options) {
     this.nodeId = safeRandomUUID();
@@ -27,6 +33,7 @@ class BroadcastCollectionCoordinator {
     this.appliedEnvelopePruneTimer = null;
     this.rejectDisposed = null;
     this.disposed = false;
+    if (options.adapter) requireDurableElection(options.adapter);
     this.dbName = options.dbName;
     this.coordinatorName = options.coordinatorName;
     this.defaultAdapter = options.adapter ?? null;
@@ -65,10 +72,12 @@ class BroadcastCollectionCoordinator {
    * Called by a host persistence factory to wire the internally-created adapter.
    */
   setAdapter(adapter) {
+    requireDurableElection(adapter);
     this.defaultAdapter = adapter;
   }
   /** Register the persistence adapter that owns one collection. */
   setAdapterForCollection(collectionId, adapter) {
+    requireDurableElection(adapter);
     this.collectionAdapters.set(collectionId, adapter);
   }
   registerRemoteSubsetOwner(collectionId, owner) {
@@ -347,6 +356,30 @@ class BroadcastCollectionCoordinator {
     }
     return this.sendRPC(collectionId, request);
   }
+  async reconcileCommittedTx(collectionId, tx, anchor, scopedAdapter) {
+    const initialRoute = this.waitForInitialRoute(collectionId);
+    if (initialRoute) await initialRoute;
+    let response;
+    for (let attempt = 0; attempt <= RPC_RETRY_ATTEMPTS; attempt++) {
+      if (attempt > 0) await sleep(RPC_RETRY_DELAY_MS * attempt);
+      const request = {
+        type: `rpc:reconcileCommittedTx:req`,
+        rpcId: safeRandomUUID(),
+        tx,
+        anchor
+      };
+      response = this.isLeader(collectionId) ? await this.handleReconcileCommittedTx(
+        collectionId,
+        request,
+        scopedAdapter
+      ) : await this.sendRPC(
+        collectionId,
+        request
+      );
+      if (response.ok || response.code !== `NOT_LEADER`) return response;
+    }
+    return response;
+  }
   async pullSince(collectionId, fromRowVersion, scopedAdapter) {
     if (this.isLeader(collectionId)) {
       return this.handlePullSince(
@@ -452,16 +485,18 @@ class BroadcastCollectionCoordinator {
           }
           try {
             const adapter = this.requireAdapter(collectionId);
-            if (adapter.getStreamPosition) {
-              const pos = await adapter.getStreamPosition(collectionId);
-              state.latestTerm = pos.latestTerm;
-              state.latestSeq = pos.latestSeq;
-              state.latestRowVersion = pos.latestRowVersion;
-            }
+            const reserveTerm = adapter.reserveLeadershipTerm;
+            if (!reserveTerm)
+              throw new InvalidPersistenceAdapterError(`reserveLeadershipTerm`);
+            const pos = await this.withWriterLock(
+              () => reserveTerm.call(adapter, collectionId, state.latestTerm)
+            );
+            state.latestTerm = pos.latestTerm;
+            state.latestSeq = pos.latestSeq;
+            state.latestRowVersion = pos.latestRowVersion;
             if (this.isDisposed() || !this.participatesInLeadership(state) || this.collections.get(collectionId) !== state || abortController.signal.aborted || state.lockAbortController !== abortController) {
               return;
             }
-            state.latestTerm++;
             state.isLeader = true;
             state.leaderId = this.nodeId;
             this.notifyRouteWaiters(state);
@@ -718,6 +753,7 @@ class BroadcastCollectionCoordinator {
       const heartbeat = payload;
       if (typeof heartbeat.leaderId === `string` && typeof heartbeat.term === `number` && typeof heartbeat.latestSeq === `number` && typeof heartbeat.latestRowVersion === `number`) {
         const state2 = this.getOrCreateCollectionState(envelope.collectionId);
+        if (state2.isLeader) return;
         if (heartbeat.term < state2.latestTerm) return;
         const changedLeader = state2.leaderId !== heartbeat.leaderId;
         state2.leaderId = heartbeat.leaderId;
@@ -898,6 +934,8 @@ class BroadcastCollectionCoordinator {
         return this.handleApplyLocalMutations(collectionId, request);
       case `rpc:applyCommittedTx:req`:
         return this.handleApplyCommittedTx(collectionId, request);
+      case `rpc:reconcileCommittedTx:req`:
+        return this.handleReconcileCommittedTx(collectionId, request);
       case `rpc:pullSince:req`:
         return this.handlePullSince(collectionId, request);
     }
@@ -1391,6 +1429,10 @@ class BroadcastCollectionCoordinator {
       }
     );
     this.pruneAppliedEnvelopes();
+    this.publishCommittedTx(collectionId, state, tx);
+    return response;
+  }
+  publishCommittedTx(collectionId, state, tx, forceReload = false) {
     const committedBase = {
       type: `tx:committed`,
       term: tx.term,
@@ -1398,9 +1440,10 @@ class BroadcastCollectionCoordinator {
       txId: tx.txId,
       latestRowVersion: tx.rowVersion
     };
-    const committedPayload = tx.truncate ? {
+    const committedPayload = tx.truncate || forceReload ? {
       ...committedBase,
-      requiresFullReload: true
+      requiresFullReload: true,
+      ...forceReload ? { reconciliationReload: true } : {}
     } : {
       ...committedBase,
       requiresFullReload: false,
@@ -1424,22 +1467,115 @@ class BroadcastCollectionCoordinator {
     for (const subscriber of state.subscribers) {
       subscriber(committed);
     }
+  }
+  async handleReconcileCommittedTx(collectionId, request, scopedAdapter) {
+    const state = this.collections.get(collectionId);
+    const notLeader = () => ({
+      type: `rpc:reconcileCommittedTx:res`,
+      rpcId: request.rpcId,
+      ok: false,
+      code: `NOT_LEADER`,
+      error: `not the leader for ${collectionId}`
+    });
+    if (!state?.isLeader) return notLeader();
+    if (!isValidCommittedTxAnchor(request.anchor) || !hasReconciliationTxPayload(request.tx)) {
+      return {
+        type: `rpc:reconcileCommittedTx:res`,
+        rpcId: request.rpcId,
+        ok: false,
+        code: `UNDETERMINED`,
+        error: `the source transaction has no valid durable proof`
+      };
+    }
+    const expectedTerm = state.latestTerm;
+    const result = await this.withScheduledWriterLock(
+      collectionId,
+      async (adapter) => {
+        const candidate = this.nextPersistedTxIfLeader(
+          collectionId,
+          state,
+          expectedTerm,
+          request.tx
+        );
+        if (candidate === null) return null;
+        if (!adapter.reconcileCommittedTx) return { kind: `unknown` };
+        try {
+          const reconciled = await adapter.reconcileCommittedTx(
+            collectionId,
+            candidate,
+            request.anchor
+          );
+          if (reconciled.kind !== `unknown`) {
+            state.latestRowVersion = Math.max(
+              state.latestRowVersion,
+              reconciled.latestRowVersion
+            );
+            if (reconciled.kind === `applied-now`) {
+              state.latestSeq = reconciled.committed.seq;
+            }
+          }
+          return reconciled;
+        } catch (error) {
+          throw toPersistedCollectionDurabilityError(collectionId, error);
+        }
+      },
+      scopedAdapter
+    );
+    if (result === null) return notLeader();
+    if (result.kind === `unknown`) {
+      return {
+        type: `rpc:reconcileCommittedTx:res`,
+        rpcId: request.rpcId,
+        ok: false,
+        code: `UNDETERMINED`,
+        error: `the durable outcome cannot be certified`
+      };
+    }
+    const current = {
+      term: state.latestTerm,
+      seq: state.latestSeq,
+      rowVersion: state.latestRowVersion
+    };
+    const response = {
+      type: `rpc:reconcileCommittedTx:res`,
+      rpcId: request.rpcId,
+      ok: true,
+      alreadyApplied: result.kind === `already-applied`,
+      committed: result.committed
+    };
+    if (!this.isDisposed() && this.collections.get(collectionId) === state) {
+      this.publishCommittedTx(
+        collectionId,
+        state,
+        result.kind === `already-applied` ? { ...request.tx, ...current } : { ...request.tx, ...result.committed },
+        result.kind === `already-applied`
+      );
+    }
     return response;
   }
+  nextPersistedTxIfLeader(collectionId, state, expectedTerm, pendingTx) {
+    if (this.isDisposed() || this.collections.get(collectionId) !== state || !state.isLeader || state.leaderId !== this.nodeId || state.latestTerm !== expectedTerm) {
+      return null;
+    }
+    return {
+      ...pendingTx,
+      term: state.latestTerm,
+      seq: state.latestSeq + 1,
+      rowVersion: state.latestRowVersion + 1
+    };
+  }
   async applyDurablyAtNextStreamPosition(collectionId, state, pendingTx, scopedAdapter) {
     const expectedTerm = state.latestTerm;
     return this.withScheduledWriterLock(
       collectionId,
       async (adapter) => {
-        if (this.isDisposed() || this.collections.get(collectionId) !== state || !state.isLeader || state.leaderId !== this.nodeId || state.latestTerm !== expectedTerm) {
-          return null;
-        }
-        const tx = {
-          ...pendingTx,
-          term: state.latestTerm,
-          seq: state.latestSeq + 1,
-          rowVersion: state.latestRowVersion + 1
-        };
+        const tx = this.nextPersistedTxIfLeader(
+          collectionId,
+          state,
+          expectedTerm,
+          pendingTx
+        );
+        if (tx === null) return null;
         try {
           await adapter.applyCommittedTx(collectionId, tx);
         } catch (error) {
@@ -1582,12 +1718,18 @@ function isRPCRequest(payload) {
     case `rpc:ensurePersistedIndex:req`:
     case `rpc:applyLocalMutations:req`:
     case `rpc:applyCommittedTx:req`:
+    case `rpc:reconcileCommittedTx:req`:
     case `rpc:pullSince:req`:
       return true;
     default:
       return false;
   }
 }
+function hasReconciliationTxPayload(value) {
+  if (!value || typeof value !== `object`) return false;
+  const tx = value;
+  return typeof tx.txId === `string` && Array.isArray(tx.mutations);
+}
 function sleep(ms) {
   return new Promise((resolve) => setTimeout(resolve, ms));
 }
@@ -1661,6 +1803,15 @@ function createRPCErrorResponse(request, cause) {
         code: `CONFLICT`,
         error
       };
+    case `rpc:reconcileCommittedTx:req`:
+      return {
+        type: `rpc:reconcileCommittedTx:res`,
+        rpcId: request.rpcId,
+        ok: false,
+        code: cause instanceof PersistedCollectionDurabilityError ? `PERSISTENCE_ERROR` : `UNDETERMINED`,
+        error,
+        ...cause instanceof PersistedCollectionDurabilityError ? toSafeDurabilityDetails(cause) : {}
+      };
     case `rpc:pullSince:req`:
       return {
         type: `rpc:pullSince:res`,