@tanstack/db-sqlite-persistence-core 0.4.7 → 0.4.9

package/src/broadcast-coordinator.ts

@@ -1,7 +1,10 @@
 import { safeRandomUUID } from '@tanstack/db'
+import { isValidCommittedTxAnchor } from './committed-tx-anchor'
 import {
   DuplicateRemoteSubsetOwnerError,
   IndeterminateCommitError,
+  InvalidPersistedCollectionConfigError,
+  InvalidPersistenceAdapterError,
   PersistedCollectionDurabilityError,
   RetryableRemoteSubsetAcquisitionError,
   toPersistedCollectionDurabilityError,
@@ -11,6 +14,7 @@ import {
   toTransportedLoadSubsetOptions,
 } from './remote-subset-wire'
 import { unloadRemoteSubsetOwner } from './remote-subset-owner'
+import { eagerStorageTarget, managedStorageTarget } from './persisted'
 import type { LoadSubsetOptions } from '@tanstack/db'
 import type { IndeterminateCommitRequestType } from './errors'
 import type { TransportedLoadSubsetOptions } from './remote-subset-wire'
@@ -18,6 +22,7 @@ import type { RemoteSubsetOwner } from './remote-subset-owner'
 import type {
   ApplyCommittedTxResponse,
   ApplyLocalMutationsResponse,
+  CommittedTxAnchor,
   EnsureRemoteSubsetRequest,
   EnsureRemoteSubsetResponse,
   HydrationPersistenceAdapter,
@@ -25,10 +30,12 @@ import type {
   PersistedIndexSpec,
   PersistedMutationEnvelope,
   PersistedRowMetadataMutation,
+  PersistedStorageTarget,
   PersistedTx,
   PersistenceAdapter,
   ProtocolEnvelope,
   PullSinceResponse,
+  ReconcileCommittedTxResponse,
   ReleaseRemoteSubsetRequest,
   ReleaseRemoteSubsetResponse,
   TxCommitted,
@@ -60,12 +67,14 @@ type RPCRequest =
       rpcId: string
       signature: string
       spec: PersistedIndexSpec
+      cacheGenerationClaimId?: string
     }
   | {
       type: `rpc:applyLocalMutations:req`
       rpcId: string
       envelopeId: string
       mutations: Array<PersistedMutationEnvelope>
+      cacheGenerationClaimId?: string
     }
   | {
       type: `rpc:applyCommittedTx:req`
@@ -73,10 +82,17 @@ type RPCRequest =
       envelopeId: string
       tx: PersistedTx
     }
+  | {
+      type: `rpc:reconcileCommittedTx:req`
+      rpcId: string
+      tx: PersistedTx
+      anchor: CommittedTxAnchor
+    }
   | {
       type: `rpc:pullSince:req`
       rpcId: string
       fromRowVersion: number
+      cacheGenerationClaimId?: string
     }
 
 type RPCResponse =
@@ -90,6 +106,7 @@ type RPCResponse =
     }
   | ApplyLocalMutationsResponse
   | ApplyCommittedTxResponse
+  | ReconcileCommittedTxResponse
   | PullSinceResponse
 
 type PendingRPC = {
@@ -115,11 +132,16 @@ type CollectionState = {
 // belong to the persisted sync wrapper, not the elected writer transport.
 type CoordinatorAdapter = Pick<
   PersistenceAdapter,
-  `loadSubset` | `applyCommittedTx` | `ensureIndex` | `runInRegularScope`
+  | `loadSubset`
+  | `applyCommittedTx`
+  | `reconcileCommittedTx`
+  | `ensureIndex`
+  | `runInRegularScope`
 > & {
   pullSince?: (
-    collectionId: string,
+    target: PersistedStorageTarget,
     fromRowVersion: number,
+    ctx?: { cacheGenerationClaimId?: string },
   ) => Promise<
     | {
         latestRowVersion: number
@@ -132,13 +154,31 @@ type CoordinatorAdapter = Pick<
         deletedKeys: Array<string | number>
       }
   >
-  getStreamPosition?: (collectionId: string) => Promise<{
+  getStreamPosition?: (
+    target: PersistedStorageTarget,
+    ctx?: { cacheGenerationClaimId?: string },
+  ) => Promise<{
+    latestTerm: number
+    latestSeq: number
+    latestRowVersion: number
+  }>
+  reserveLeadershipTerm?: (
+    target: PersistedStorageTarget,
+    observedTerm: number,
+    ctx?: { cacheGenerationClaimId?: string },
+  ) => Promise<{
     latestTerm: number
     latestSeq: number
     latestRowVersion: number
   }>
 }
 
+function requireDurableElection(adapter: CoordinatorAdapter): void {
+  if (typeof adapter.reserveLeadershipTerm !== `function`) {
+    throw new InvalidPersistenceAdapterError(`reserveLeadershipTerm`)
+  }
+}
+
 type ActiveRemoteSubsetAcquisition = {
   collectionId: string
   requesterId: string
@@ -146,6 +186,7 @@ type ActiveRemoteSubsetAcquisition = {
   owner: RemoteSubsetOwner
   options: TransportedLoadSubsetOptions
   load: Promise<void>
+  admitted: Promise<void>
   transferred: boolean
   released: boolean
   terminalRelease: boolean
@@ -230,6 +271,7 @@ export class BroadcastCollectionCoordinator implements PersistedCollectionCoordi
   private readonly coordinatorName: string
   private defaultAdapter: CoordinatorAdapter | null
   private readonly collectionAdapters = new Map<string, CoordinatorAdapter>()
+  private readonly collectionClaimIds = new Map<string, string>()
   private readonly remoteSubsetOwners = new Map<string, RemoteSubsetOwner>()
   private readonly remoteSubsetIds = new Map<
     string,
@@ -282,6 +324,7 @@ export class BroadcastCollectionCoordinator implements PersistedCollectionCoordi
   }
 
   constructor(options: BroadcastCollectionCoordinatorOptions) {
+    if (options.adapter) requireDurableElection(options.adapter)
     this.dbName = options.dbName
     this.coordinatorName = options.coordinatorName
     this.defaultAdapter = options.adapter ?? null
@@ -300,6 +343,7 @@ export class BroadcastCollectionCoordinator implements PersistedCollectionCoordi
    * Called by a host persistence factory to wire the internally-created adapter.
    */
   setAdapter(adapter: CoordinatorAdapter): void {
+    requireDurableElection(adapter)
     this.defaultAdapter = adapter
   }
 
@@ -307,8 +351,15 @@ export class BroadcastCollectionCoordinator implements PersistedCollectionCoordi
   setAdapterForCollection(
     collectionId: string,
     adapter: CoordinatorAdapter,
+    cacheGenerationClaimId?: string,
   ): void {
+    requireDurableElection(adapter)
     this.collectionAdapters.set(collectionId, adapter)
+    if (cacheGenerationClaimId) {
+      this.collectionClaimIds.set(collectionId, cacheGenerationClaimId)
+    } else {
+      this.collectionClaimIds.delete(collectionId)
+    }
   }
 
   registerRemoteSubsetOwner(
@@ -491,11 +542,38 @@ export class BroadcastCollectionCoordinator implements PersistedCollectionCoordi
     }
     try {
       await acquisition.release
+    } catch (error) {
+      this.scheduleRemoteSubsetReleaseRetry(acquisition, options)
+      throw error
     } finally {
       acquisition.release = null
     }
   }
 
+  private scheduleRemoteSubsetReleaseRetry(
+    acquisition: OutboundRemoteSubsetAcquisition,
+    options: LoadSubsetOptions,
+  ): void {
+    const key = remoteSubsetAcquisitionKey(
+      acquisition.collectionId,
+      acquisition.acquisitionId,
+    )
+    if (
+      this.isDisposed() ||
+      this.outboundRemoteSubsetAcquisitions.get(key) !== acquisition ||
+      acquisition.retryTimer !== null
+    ) {
+      return
+    }
+    acquisition.retryTimer = setTimeout(() => {
+      acquisition.retryTimer = null
+      void this.requestReleaseRemoteSubset(
+        acquisition.collectionId,
+        options,
+      ).catch(() => undefined)
+    }, RPC_RETRY_DELAY_MS)
+  }
+
   private async acquireRemoteSubset(
     acquisition: OutboundRemoteSubsetAcquisition,
   ): Promise<void> {
@@ -571,7 +649,11 @@ export class BroadcastCollectionCoordinator implements PersistedCollectionCoordi
     try {
       await work
       acquired = true
-      this.cancelRemoteSubsetReplayRetry(acquisition)
+      // Release may have failed while this acquisition was in flight. In that
+      // case the shared timer carries release retry debt, not replay work.
+      if (!acquisition.releaseRequested) {
+        this.cancelRemoteSubsetReplayRetry(acquisition)
+      }
     } finally {
       if (acquisition.inFlight === work) acquisition.inFlight = null
       const current = this.collections.get(acquisition.collectionId)
@@ -604,14 +686,27 @@ export class BroadcastCollectionCoordinator implements PersistedCollectionCoordi
     spec: PersistedIndexSpec,
     scopedAdapter?: HydrationPersistenceAdapter,
     localEnsureCompleted = false,
+    cacheGenerationClaimId?: string,
   ): Promise<void> {
     if (this.isLeader(collectionId)) {
       if (localEnsureCompleted) return
-      await (scopedAdapter ?? this.requireAdapter(collectionId)).ensureIndex(
-        collectionId,
-        signature,
-        spec,
-      )
+      const adapter = scopedAdapter ?? this.requireAdapter(collectionId)
+      if (cacheGenerationClaimId) {
+        await adapter.ensureIndex(
+          managedStorageTarget(collectionId, cacheGenerationClaimId),
+          signature,
+          spec,
+          {
+            cacheGenerationClaimId,
+          },
+        )
+      } else {
+        await adapter.ensureIndex(
+          eagerStorageTarget(collectionId),
+          signature,
+          spec,
+        )
+      }
       return
     }
 
@@ -625,6 +720,7 @@ export class BroadcastCollectionCoordinator implements PersistedCollectionCoordi
       rpcId: safeRandomUUID(),
       signature,
       spec,
+      cacheGenerationClaimId,
     })
 
     if (!response.ok) {
@@ -637,6 +733,7 @@ export class BroadcastCollectionCoordinator implements PersistedCollectionCoordi
   async requestApplyLocalMutations(
     collectionId: string,
     mutations: Array<PersistedMutationEnvelope>,
+    cacheGenerationClaimId?: string,
   ): Promise<ApplyLocalMutationsResponse> {
     const initialRoute = this.waitForInitialRoute(collectionId)
     if (initialRoute) await initialRoute
@@ -646,6 +743,7 @@ export class BroadcastCollectionCoordinator implements PersistedCollectionCoordi
         rpcId: safeRandomUUID(),
         envelopeId: safeRandomUUID(),
         mutations,
+        cacheGenerationClaimId,
       })
     }
 
@@ -654,6 +752,7 @@ export class BroadcastCollectionCoordinator implements PersistedCollectionCoordi
       rpcId: safeRandomUUID(),
       envelopeId: safeRandomUUID(),
       mutations,
+      cacheGenerationClaimId,
     })
   }
 
@@ -677,10 +776,48 @@ export class BroadcastCollectionCoordinator implements PersistedCollectionCoordi
     return this.sendRPC<ApplyCommittedTxResponse>(collectionId, request)
   }
 
+  async reconcileCommittedTx(
+    collectionId: string,
+    tx: PersistedTx,
+    anchor: CommittedTxAnchor,
+    scopedAdapter?: HydrationPersistenceAdapter,
+  ): Promise<ReconcileCommittedTxResponse> {
+    const initialRoute = this.waitForInitialRoute(collectionId)
+    if (initialRoute) await initialRoute
+    // Each attempt checks the exact txId under the elected writer lock, so a
+    // stale NOT_LEADER reply can follow the replacement route safely.
+    let response: ReconcileCommittedTxResponse | undefined
+    for (let attempt = 0; attempt <= RPC_RETRY_ATTEMPTS; attempt++) {
+      if (attempt > 0) await sleep(RPC_RETRY_DELAY_MS * attempt)
+      const request: Extract<
+        RPCRequest,
+        { type: `rpc:reconcileCommittedTx:req` }
+      > = {
+        type: `rpc:reconcileCommittedTx:req`,
+        rpcId: safeRandomUUID(),
+        tx,
+        anchor,
+      }
+      response = this.isLeader(collectionId)
+        ? await this.handleReconcileCommittedTx(
+            collectionId,
+            request,
+            scopedAdapter,
+          )
+        : await this.sendRPC<ReconcileCommittedTxResponse>(
+            collectionId,
+            request,
+          )
+      if (response.ok || response.code !== `NOT_LEADER`) return response
+    }
+    return response!
+  }
+
   async pullSince(
     collectionId: string,
     fromRowVersion: number,
     scopedAdapter?: HydrationPersistenceAdapter,
+    cacheGenerationClaimId?: string,
   ): Promise<PullSinceResponse> {
     if (this.isLeader(collectionId)) {
       return this.handlePullSince(
@@ -689,6 +826,7 @@ export class BroadcastCollectionCoordinator implements PersistedCollectionCoordi
           type: `rpc:pullSince:req`,
           rpcId: safeRandomUUID(),
           fromRowVersion,
+          cacheGenerationClaimId,
         },
         scopedAdapter,
       )
@@ -698,6 +836,7 @@ export class BroadcastCollectionCoordinator implements PersistedCollectionCoordi
       type: `rpc:pullSince:req`,
       rpcId: safeRandomUUID(),
       fromRowVersion,
+      cacheGenerationClaimId,
     })
   }
 
@@ -736,6 +875,7 @@ export class BroadcastCollectionCoordinator implements PersistedCollectionCoordi
     this.channel.close()
     this.collections.clear()
     this.collectionAdapters.clear()
+    this.collectionClaimIds.clear()
     for (const collectionId of this.remoteSubsetOwners.keys()) {
       this.releaseInboundRemoteSubsetAcquisitions(collectionId)
     }
@@ -806,14 +946,28 @@ export class BroadcastCollectionCoordinator implements PersistedCollectionCoordi
           }
 
           try {
-            // Restore stream position from DB before claiming leadership
+            // Reserve an election term durably before advertising this route.
+            // Election bypasses the shared scheduler because a hydrate may be
+            // waiting for its first writer route. The writer Web Lock orders
+            // reservation after any old owner's in-flight durable write.
             const adapter = this.requireAdapter(collectionId)
-            if (adapter.getStreamPosition) {
-              const pos = await adapter.getStreamPosition(collectionId)
-              state.latestTerm = pos.latestTerm
-              state.latestSeq = pos.latestSeq
-              state.latestRowVersion = pos.latestRowVersion
-            }
+            const reserveTerm = adapter.reserveLeadershipTerm
+            if (!reserveTerm)
+              throw new InvalidPersistenceAdapterError(`reserveLeadershipTerm`)
+            const pos = await this.withWriterLock(() => {
+              const claimId = this.collectionClaimIds.get(collectionId)
+              return reserveTerm.call(
+                adapter,
+                claimId
+                  ? managedStorageTarget(collectionId, claimId)
+                  : eagerStorageTarget(collectionId),
+                state.latestTerm,
+                { cacheGenerationClaimId: claimId },
+              )
+            })
+            state.latestTerm = pos.latestTerm
+            state.latestSeq = pos.latestSeq
+            state.latestRowVersion = pos.latestRowVersion
 
             if (
               this.isDisposed() ||
@@ -825,7 +979,6 @@ export class BroadcastCollectionCoordinator implements PersistedCollectionCoordi
               return
             }
 
-            state.latestTerm++
             state.isLeader = true
             state.leaderId = this.nodeId
             this.notifyRouteWaiters(state)
@@ -1005,6 +1158,7 @@ export class BroadcastCollectionCoordinator implements PersistedCollectionCoordi
     this.releaseLeadership(collectionId, state)
     this.collections.delete(collectionId)
     this.collectionAdapters.delete(collectionId)
+    this.collectionClaimIds.delete(collectionId)
     const prefix = `${JSON.stringify([collectionId]).slice(0, -1)},`
     for (const key of this.appliedEnvelopes.keys()) {
       if (key.startsWith(prefix)) this.appliedEnvelopes.delete(key)
@@ -1175,6 +1329,9 @@ export class BroadcastCollectionCoordinator implements PersistedCollectionCoordi
         typeof heartbeat.latestRowVersion === `number`
       ) {
         const state = this.getOrCreateCollectionState(envelope.collectionId)
+        // The local Web Lock owns this route even if an older heartbeat arrives
+        // while leadership changes.
+        if (state.isLeader) return
         if (heartbeat.term < state.latestTerm) return
         const changedLeader = state.leaderId !== heartbeat.leaderId
         state.leaderId = heartbeat.leaderId
@@ -1415,6 +1572,8 @@ export class BroadcastCollectionCoordinator implements PersistedCollectionCoordi
         return this.handleApplyLocalMutations(collectionId, request)
       case `rpc:applyCommittedTx:req`:
         return this.handleApplyCommittedTx(collectionId, request)
+      case `rpc:reconcileCommittedTx:req`:
+        return this.handleReconcileCommittedTx(collectionId, request)
       case `rpc:pullSince:req`:
         return this.handlePullSince(collectionId, request)
     }
@@ -1546,12 +1705,11 @@ export class BroadcastCollectionCoordinator implements PersistedCollectionCoordi
     acquisition.released = true
     acquisition.release = (async () => {
       try {
-        await acquisition.load
-      } catch {
-        // A returned promise transfers the lease even when initial loading fails.
-      }
-      try {
+        await acquisition.admitted
         if (acquisition.transferred) {
+          // The owner may require unload to settle its load. Invocation of the
+          // owner already transferred this lease; source settlement is not a
+          // prerequisite for giving it the cancellation opportunity.
           await unloadRemoteSubsetOwner(acquisition.owner, acquisition.options)
         }
       } finally {
@@ -1711,6 +1869,7 @@ export class BroadcastCollectionCoordinator implements PersistedCollectionCoordi
       owner: options.owner,
       options: options.options,
       load: Promise.resolve(),
+      admitted: Promise.resolve(),
       transferred: false,
       released: false,
       terminalRelease: false,
@@ -1718,6 +1877,10 @@ export class BroadcastCollectionCoordinator implements PersistedCollectionCoordi
     }
     this.releasedRemoteSubsetAcquisitionTimes.delete(key)
     this.inboundRemoteSubsetAcquisitions.set(key, acquisition)
+    let resolveAdmission!: () => void
+    acquisition.admitted = new Promise<void>((resolve) => {
+      resolveAdmission = resolve
+    })
     let resolveLoad!: () => void
     let rejectLoad!: (error: unknown) => void
     acquisition.load = new Promise<void>((resolve, reject) => {
@@ -1730,6 +1893,8 @@ export class BroadcastCollectionCoordinator implements PersistedCollectionCoordi
       void Promise.resolve(load).then(resolveLoad, rejectLoad)
     } catch (error) {
       rejectLoad(error)
+    } finally {
+      resolveAdmission()
     }
 
     try {
@@ -1765,11 +1930,26 @@ export class BroadcastCollectionCoordinator implements PersistedCollectionCoordi
       rpcId: string
       signature: string
       spec: PersistedIndexSpec
+      cacheGenerationClaimId?: string
     },
   ): Promise<RPCResponse> {
-    await this.withScheduledWriterLock(collectionId, (adapter) =>
-      adapter.ensureIndex(collectionId, request.signature, request.spec),
-    )
+    await this.withScheduledWriterLock(collectionId, (adapter) => {
+      if (request.cacheGenerationClaimId) {
+        return adapter.ensureIndex(
+          managedStorageTarget(collectionId, request.cacheGenerationClaimId),
+          request.signature,
+          request.spec,
+          {
+            cacheGenerationClaimId: request.cacheGenerationClaimId,
+          },
+        )
+      }
+      return adapter.ensureIndex(
+        eagerStorageTarget(collectionId),
+        request.signature,
+        request.spec,
+      )
+    })
     return {
       type: `rpc:ensurePersistedIndex:res`,
       rpcId: request.rpcId,
@@ -1784,6 +1964,7 @@ export class BroadcastCollectionCoordinator implements PersistedCollectionCoordi
       rpcId: string
       envelopeId: string
       mutations: Array<PersistedMutationEnvelope>
+      cacheGenerationClaimId?: string
     },
   ): Promise<ApplyLocalMutationsResponse> {
     const envelopeKey = appliedEnvelopeKey(collectionId, request.envelopeId)
@@ -1817,6 +1998,14 @@ export class BroadcastCollectionCoordinator implements PersistedCollectionCoordi
     collectionId: string,
     request: Extract<RPCRequest, { type: `rpc:applyLocalMutations:req` }>,
   ): Promise<ApplyLocalMutationsResponse> {
+    if (
+      this.collectionClaimIds.has(collectionId) &&
+      !request.cacheGenerationClaimId
+    ) {
+      throw new InvalidPersistedCollectionConfigError(
+        `A managed local mutation requires a persisted cache claim`,
+      )
+    }
     const state = this.collections.get(collectionId)
     if (!state || !state.isLeader) {
       return {
@@ -1846,6 +2035,7 @@ export class BroadcastCollectionCoordinator implements PersistedCollectionCoordi
     }
     const pendingTx = {
       txId: safeRandomUUID(),
+      cacheGenerationClaimId: request.cacheGenerationClaimId,
       mutations: request.mutations.map((m) => ({
         type: m.type,
         key: m.key,
@@ -2056,6 +2246,17 @@ export class BroadcastCollectionCoordinator implements PersistedCollectionCoordi
     )
     this.pruneAppliedEnvelopes()
 
+    this.publishCommittedTx(collectionId, state, tx)
+
+    return response
+  }
+
+  private publishCommittedTx(
+    collectionId: string,
+    state: CollectionState,
+    tx: PersistedTx,
+    forceReload = false,
+  ): void {
     const committedBase = {
       type: `tx:committed` as const,
       term: tx.term,
@@ -2063,26 +2264,28 @@ export class BroadcastCollectionCoordinator implements PersistedCollectionCoordi
       txId: tx.txId,
       latestRowVersion: tx.rowVersion,
     }
-    const committedPayload: TxCommitted = tx.truncate
-      ? {
-          ...committedBase,
-          requiresFullReload: true,
-        }
-      : {
-          ...committedBase,
-          requiresFullReload: false,
-          changedRows: tx.mutations
-            .filter((mutation) => mutation.type !== `delete`)
-            .map((mutation) => ({
-              key: mutation.key,
-              value: mutation.value,
-            })),
-          deletedKeys: tx.mutations
-            .filter((mutation) => mutation.type === `delete`)
-            .map((mutation) => mutation.key),
-          rowMetadataMutations: tx.rowMetadataMutations,
-          collectionMetadataMutations: tx.collectionMetadataMutations,
-        }
+    const committedPayload: TxCommitted =
+      tx.truncate || forceReload
+        ? {
+            ...committedBase,
+            requiresFullReload: true,
+            ...(forceReload ? { reconciliationReload: true as const } : {}),
+          }
+        : {
+            ...committedBase,
+            requiresFullReload: false,
+            changedRows: tx.mutations
+              .filter((mutation) => mutation.type !== `delete`)
+              .map((mutation) => ({
+                key: mutation.key,
+                value: mutation.value,
+              })),
+            deletedKeys: tx.mutations
+              .filter((mutation) => mutation.type === `delete`)
+              .map((mutation) => mutation.key),
+            rowMetadataMutations: tx.rowMetadataMutations,
+            collectionMetadataMutations: tx.collectionMetadataMutations,
+          }
     const committed: ProtocolEnvelope<TxCommitted> = {
       v: 1,
       dbName: this.dbName,
@@ -2095,10 +2298,133 @@ export class BroadcastCollectionCoordinator implements PersistedCollectionCoordi
     for (const subscriber of state.subscribers) {
       subscriber(committed)
     }
+  }
+
+  private async handleReconcileCommittedTx(
+    collectionId: string,
+    request: Extract<RPCRequest, { type: `rpc:reconcileCommittedTx:req` }>,
+    scopedAdapter?: HydrationPersistenceAdapter,
+  ): Promise<ReconcileCommittedTxResponse> {
+    const state = this.collections.get(collectionId)
+    const notLeader = (): ReconcileCommittedTxResponse => ({
+      type: `rpc:reconcileCommittedTx:res`,
+      rpcId: request.rpcId,
+      ok: false,
+      code: `NOT_LEADER`,
+      error: `not the leader for ${collectionId}`,
+    })
+    if (!state?.isLeader) return notLeader()
+    if (
+      !isValidCommittedTxAnchor(request.anchor) ||
+      !hasReconciliationTxPayload(request.tx)
+    ) {
+      return {
+        type: `rpc:reconcileCommittedTx:res`,
+        rpcId: request.rpcId,
+        ok: false,
+        code: `UNDETERMINED`,
+        error: `the source transaction has no valid durable proof`,
+      }
+    }
+
+    const expectedTerm = state.latestTerm
+    const result = await this.withScheduledWriterLock(
+      collectionId,
+      async (adapter) => {
+        const candidate = this.nextPersistedTxIfLeader(
+          collectionId,
+          state,
+          expectedTerm,
+          request.tx,
+        )
+        if (candidate === null) return null
+        if (!adapter.reconcileCommittedTx) return { kind: `unknown` as const }
+        try {
+          const reconciled = await adapter.reconcileCommittedTx(
+            candidate.cacheGenerationClaimId
+              ? managedStorageTarget(
+                  collectionId,
+                  candidate.cacheGenerationClaimId,
+                )
+              : eagerStorageTarget(collectionId),
+            candidate,
+            request.anchor,
+          )
+          if (reconciled.kind !== `unknown`) {
+            state.latestRowVersion = Math.max(
+              state.latestRowVersion,
+              reconciled.latestRowVersion,
+            )
+            if (reconciled.kind === `applied-now`) {
+              state.latestSeq = reconciled.committed.seq
+            }
+          }
+          return reconciled
+        } catch (error) {
+          throw toPersistedCollectionDurabilityError(collectionId, error)
+        }
+      },
+      scopedAdapter,
+    )
+    if (result === null) return notLeader()
+    if (result.kind === `unknown`) {
+      return {
+        type: `rpc:reconcileCommittedTx:res`,
+        rpcId: request.rpcId,
+        ok: false,
+        code: `UNDETERMINED`,
+        error: `the durable outcome cannot be certified`,
+      }
+    }
 
+    const current = {
+      term: state.latestTerm,
+      seq: state.latestSeq,
+      rowVersion: state.latestRowVersion,
+    }
+    const response: ReconcileCommittedTxResponse = {
+      type: `rpc:reconcileCommittedTx:res`,
+      rpcId: request.rpcId,
+      ok: true,
+      alreadyApplied: result.kind === `already-applied`,
+      committed: result.committed,
+    }
+    if (!this.isDisposed() && this.collections.get(collectionId) === state) {
+      this.publishCommittedTx(
+        collectionId,
+        state,
+        result.kind === `already-applied`
+          ? { ...request.tx, ...current }
+          : { ...request.tx, ...result.committed },
+        result.kind === `already-applied`,
+      )
+    }
     return response
   }
 
+  private nextPersistedTxIfLeader(
+    collectionId: string,
+    state: CollectionState,
+    expectedTerm: number,
+    pendingTx: Omit<PersistedTx, `term` | `seq` | `rowVersion`>,
+  ): PersistedTx | null {
+    if (
+      this.isDisposed() ||
+      this.collections.get(collectionId) !== state ||
+      !state.isLeader ||
+      state.leaderId !== this.nodeId ||
+      state.latestTerm !== expectedTerm
+    ) {
+      return null
+    }
+    return {
+      ...pendingTx,
+      term: state.latestTerm,
+      seq: state.latestSeq + 1,
+      rowVersion: state.latestRowVersion + 1,
+    }
+  }
+
   private async applyDurablyAtNextStreamPosition(
     collectionId: string,
     state: CollectionState,
@@ -2109,23 +2435,20 @@ export class BroadcastCollectionCoordinator implements PersistedCollectionCoordi
     return this.withScheduledWriterLock(
       collectionId,
       async (adapter) => {
-        if (
-          this.isDisposed() ||
-          this.collections.get(collectionId) !== state ||
-          !state.isLeader ||
-          state.leaderId !== this.nodeId ||
-          state.latestTerm !== expectedTerm
-        ) {
-          return null
-        }
-        const tx: PersistedTx = {
-          ...pendingTx,
-          term: state.latestTerm,
-          seq: state.latestSeq + 1,
-          rowVersion: state.latestRowVersion + 1,
-        }
+        const tx = this.nextPersistedTxIfLeader(
+          collectionId,
+          state,
+          expectedTerm,
+          pendingTx,
+        )
+        if (tx === null) return null
         try {
-          await adapter.applyCommittedTx(collectionId, tx)
+          await adapter.applyCommittedTx(
+            tx.cacheGenerationClaimId
+              ? managedStorageTarget(collectionId, tx.cacheGenerationClaimId)
+              : eagerStorageTarget(collectionId),
+            tx,
+          )
         } catch (error) {
           throw toPersistedCollectionDurabilityError(collectionId, error)
         }
@@ -2143,6 +2466,7 @@ export class BroadcastCollectionCoordinator implements PersistedCollectionCoordi
       type: `rpc:pullSince:req`
       rpcId: string
       fromRowVersion: number
+      cacheGenerationClaimId?: string
     },
     scopedAdapter?: HydrationPersistenceAdapter,
   ): Promise<PullSinceResponse> {
@@ -2161,7 +2485,15 @@ export class BroadcastCollectionCoordinator implements PersistedCollectionCoordi
       }
     }
 
-    const result = await adapter.pullSince(collectionId, request.fromRowVersion)
+    const result = await adapter.pullSince(
+      request.cacheGenerationClaimId
+        ? managedStorageTarget(collectionId, request.cacheGenerationClaimId)
+        : eagerStorageTarget(collectionId),
+      request.fromRowVersion,
+      {
+        cacheGenerationClaimId: request.cacheGenerationClaimId,
+      },
+    )
 
     if (result.requiresFullReload) {
       return {
@@ -2315,6 +2647,7 @@ function isRPCRequest(payload: unknown): payload is RPCRequest {
     case `rpc:ensurePersistedIndex:req`:
     case `rpc:applyLocalMutations:req`:
     case `rpc:applyCommittedTx:req`:
+    case `rpc:reconcileCommittedTx:req`:
     case `rpc:pullSince:req`:
       return true
     default:
@@ -2322,6 +2655,12 @@ function isRPCRequest(payload: unknown): payload is RPCRequest {
   }
 }
 
+function hasReconciliationTxPayload(value: unknown): boolean {
+  if (!value || typeof value !== `object`) return false
+  const tx = value as Record<string, unknown>
+  return typeof tx.txId === `string` && Array.isArray(tx.mutations)
+}
+
 function sleep(ms: number): Promise<void> {
   return new Promise((resolve) => setTimeout(resolve, ms))
 }
@@ -2411,6 +2750,20 @@ function createRPCErrorResponse(
         code: `CONFLICT`,
         error,
       }
+    case `rpc:reconcileCommittedTx:req`:
+      return {
+        type: `rpc:reconcileCommittedTx:res`,
+        rpcId: request.rpcId,
+        ok: false,
+        code:
+          cause instanceof PersistedCollectionDurabilityError
+            ? `PERSISTENCE_ERROR`
+            : `UNDETERMINED`,
+        error,
+        ...(cause instanceof PersistedCollectionDurabilityError
+          ? toSafeDurabilityDetails(cause)
+          : {}),
+      }
     case `rpc:pullSince:req`:
       return {
         type: `rpc:pullSince:res`,