@tanstack/db-sqlite-persistence-core 0.4.1 → 0.4.4

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@tanstack/db-sqlite-persistence-core",
-  "version": "0.4.1",
+  "version": "0.4.4",
   "description": "SQLite persisted collection core for TanStack DB",
   "author": "TanStack Team",
   "license": "MIT",
@@ -50,7 +50,7 @@
   ],
   "dependencies": {
     "@standard-schema/spec": "^1.1.0",
-    "@tanstack/db": "0.11.1"
+    "@tanstack/db": "0.12.0"
   },
   "peerDependencies": {
     "typescript": ">=4.7"

package/src/errors.ts

@@ -90,8 +90,7 @@ export function toPersistedCollectionDurabilityError(
 }
 
 export type IndeterminateCommitRequestType =
-  | `rpc:applyLocalMutations:req`
-  | `rpc:applyCommittedTx:req`
+  `rpc:applyLocalMutations:req` | `rpc:applyCommittedTx:req`
 
 export type IndeterminateCommitErrorOptions = {
   collectionId: string

package/src/persisted.ts

@@ -9,6 +9,8 @@ import {
   getLoadSubsetDemandKey,
   safeRandomUUID,
   toBooleanPredicate,
+  whenSyncAccepted,
+  withAcceptedReceipt,
   withCollectionConfigFactory,
 } from '@tanstack/db'
 import {
@@ -289,8 +291,7 @@ export type PersistedRowMetadataMutation<
 > = { type: `set`; key: TKey; value: unknown } | { type: `delete`; key: TKey }
 
 export type PersistedCollectionMetadataMutation =
-  | { type: `set`; key: string; value: unknown }
-  | { type: `delete`; key: string }
+  { type: `set`; key: string; value: unknown } | { type: `delete`; key: string }
 
 export type ReplayableTxDelta<
   T extends Record<string, unknown> = Record<string, unknown>,
@@ -738,7 +739,7 @@ const DEFAULT_DB_NAME = `tanstack-db`
 const REMOTE_ENSURE_RETRY_DELAY_MS = 50
 
 type SyncControlFns<T extends object, TKey extends string | number> = {
-  begin: ((options?: { immediate?: boolean }) => void) | null
+  begin: (() => void) | null
   write:
     | ((
         message:
@@ -1046,7 +1047,6 @@ type BufferedSyncTransaction<T extends object, TKey extends string | number> = {
   truncate: boolean
   internal: boolean
   lifecycleGeneration: number
-  beginOptions?: { immediate?: boolean }
   expectedResumeGenerationOwner?: symbol
   signal?: AbortSignal
   prependHydrationRows: (
@@ -1066,6 +1066,8 @@ type OpenSyncTransaction<T extends object, TKey extends string | number> = Omit<
   applicationReceipt?: SyncAppliedReceipt
   operationKeys: Set<TKey>
   queuedBecauseHydrating: boolean
+  // Set by `reserveCommitTurn`: subset hydration waits for this commit.
+  reservesCommitTurn?: boolean
   hasDependentSuccessor: boolean
   publicationAdmissionWaiters?: Set<{
     resolve: () => void
@@ -1326,15 +1328,13 @@ class PersistedCollectionRuntime<
   private resumeGenerationOwner = Symbol(`persisted resume generation owner`)
   private lifecycleGeneration = 0
   private internalApplyDepth = 0
-  private sourcePublicationWaitDepth = 0
   private appliedReceiptSequence = 0
   private syncErrorReported = false
   private reportedSyncError: unknown
   private readonly pendingAppliedReceipts = new Map<number, Promise<void>>()
   private hydratingGeneration: number | null = null
   private terminalFailure:
-    | { lifecycleGeneration: number; error: unknown }
-    | undefined
+    { lifecycleGeneration: number; error: unknown } | undefined
   private coordinatorUnsubscribe: (() => void) | null = null
   private remoteSubsetOwnerUnsubscribe: (() => void) | null = null
   private indexAddedUnsubscribe: (() => void) | null = null
@@ -1409,10 +1409,20 @@ class PersistedCollectionRuntime<
     return receipt
   }
 
-  private async waitForAppliedReceiptsAfter(cursor: number): Promise<void> {
+  // Startup waits for acceptance: a local mutation's handler can wait for
+  // startup, and the rows publish when that mutation settles. Readiness waits
+  // for them in core. A subset load waits for visibility.
+  private async waitForAppliedReceiptsAfter(
+    cursor: number,
+    until: `accepted` | `visible` = `visible`,
+  ): Promise<void> {
     await Promise.all(
       Array.from(this.pendingAppliedReceipts, ([sequence, receipt]) =>
-        sequence > cursor ? receipt : undefined,
+        sequence <= cursor
+          ? undefined
+          : until === `accepted`
+            ? whenSyncAccepted(receipt)
+            : receipt,
       ),
     )
   }
@@ -1623,7 +1633,10 @@ class PersistedCollectionRuntime<
           startup.appliedCursor !== undefined &&
           lifecycleGeneration === this.lifecycleGeneration
         ) {
-          await this.waitForAppliedReceiptsAfter(startup.appliedCursor)
+          await this.waitForAppliedReceiptsAfter(
+            startup.appliedCursor,
+            `accepted`,
+          )
         }
       }
       if (lifecycleGeneration === this.lifecycleGeneration) {
@@ -1664,7 +1677,7 @@ class PersistedCollectionRuntime<
         appliedCursor !== undefined &&
         lifecycleGeneration === this.lifecycleGeneration
       ) {
-        await this.waitForAppliedReceiptsAfter(appliedCursor)
+        await this.waitForAppliedReceiptsAfter(appliedCursor, `accepted`)
       }
     })().catch((error) => {
       throw this.markTerminalFailure(error, lifecycleGeneration)
@@ -1795,7 +1808,7 @@ class PersistedCollectionRuntime<
     const applied = this.replaceCollectionMetadataSnapshot(
       snapshot.collectionMetadata,
     )
-    if (applied !== true) await applied
+    await whenSyncAccepted(applied)
   }
 
   private async loadCollectionMetadataSnapshot(
@@ -1827,7 +1840,7 @@ class PersistedCollectionRuntime<
       .map(({ key }) => key)
 
     return this.withInternalApply(() => {
-      this.syncControls.begin?.({ immediate: true })
+      this.syncControls.begin?.()
 
       currentKeys.forEach((key) => {
         if (!nextMetadata.has(key)) {
@@ -2072,16 +2085,6 @@ class PersistedCollectionRuntime<
   ): Promise<void> {
     const failure = this.getCurrentTerminalFailure()
     if (failure) return Promise.reject(failure.error)
-    if (
-      transaction.beginOptions?.immediate &&
-      this.sourcePublicationWaitDepth > 0
-    ) {
-      return Promise.reject(
-        new InvalidPersistedCollectionConfigError(
-          `immediate persisted source replay cannot enter while an earlier source publication is waiting`,
-        ),
-      )
-    }
     return this.applyMutex.run(async () => {
       await this.applyBufferedSyncTransactionUnsafe(transaction)
     })
@@ -2373,7 +2376,7 @@ class PersistedCollectionRuntime<
             hydrationContext.suppliedRowKeys.add(row.key)
           }
           const applied = this.applyRowsToCollection(rows)
-          if (applied !== true) await applied
+          await whenSyncAccepted(applied)
         }
       } finally {
         if (
@@ -2433,8 +2436,7 @@ class PersistedCollectionRuntime<
     adapter: HydrationPersistenceAdapter,
   ): Promise<void> {
     let snapshotRows:
-      | Map<TKey, { key: TKey; value: T; metadata?: unknown }>
-      | undefined
+      Map<TKey, { key: TKey; value: T; metadata?: unknown }> | undefined
     type RecoveryPresence = `present` | `absent` | `unknown`
     const recoveredPresence = new Map<TKey, RecoveryPresence>()
     let snapshotInvalidatedByTruncate = false
@@ -2570,7 +2572,7 @@ class PersistedCollectionRuntime<
     }
 
     return this.withInternalApply(() => {
-      this.syncControls.begin?.({ immediate: true })
+      this.syncControls.begin?.()
 
       for (const row of rows) {
         if (this.collection?._hasHydratedKey(row.key)) {
@@ -2668,7 +2670,7 @@ class PersistedCollectionRuntime<
       .map(({ key }) => key)
 
     return this.withInternalApply(() => {
-      this.syncControls.begin?.({ immediate: true })
+      this.syncControls.begin?.()
       this.syncControls.truncate?.()
 
       for (const row of rows) {
@@ -2739,14 +2741,17 @@ class PersistedCollectionRuntime<
         : transaction.applyToCollection()
       applicationReturned = true
       abortedDuringApplication = transaction.signal?.aborted === true
-      if (applied !== true) {
-        this.sourcePublicationWaitDepth++
-        try {
-          await applied
-        } finally {
-          this.sourcePublicationWaitDepth--
-        }
-      }
+      // Inside the apply mutex, internal work waits only for acceptance: a
+      // local mutation's handler may need this mutex, and the rows publish
+      // when that mutation settles. A source transaction is stored once it is
+      // visible, except one held by a persisting optimistic transaction,
+      // which is stored now so a handler can await its acceptance.
+      if (transaction.internal) await whenSyncAccepted(applied)
+      else if (
+        applied !== true &&
+        !this.collection?._state.hasPersistingTransaction()
+      )
+        await applied
       this.throwIfLifecycleReplaced(transaction.lifecycleGeneration)
 
       if (!transaction.internal) {
@@ -2972,7 +2977,7 @@ class PersistedCollectionRuntime<
     }
 
     const applied = this.withInternalApply(() => {
-      this.syncControls.begin?.({ immediate: true })
+      this.syncControls.begin?.()
 
       for (const mutation of mutations) {
         if (mutation.type === `delete`) {
@@ -2991,9 +2996,10 @@ class PersistedCollectionRuntime<
       return this.syncControls.commit?.() ?? true
     })
 
-    if (applied !== true) {
-      await applied
-    }
+    // The mutation handler awaits this confirmation, so wait only for
+    // acceptance: the rows become visible when the mutation settles.
+    const accepted = whenSyncAccepted(applied)
+    if (accepted !== true) await accepted
   }
 
   private filterMutationsForCollection(
@@ -3545,11 +3551,11 @@ class PersistedCollectionRuntime<
     this.resetSequence++
     if (this.syncControls.begin && this.syncControls.commit) {
       const applied = this.withInternalApply(() => {
-        this.syncControls.begin?.({ immediate: true })
+        this.syncControls.begin?.()
         this.syncControls.truncate?.()
         return this.syncControls.commit?.() ?? true
       })
-      if (applied !== true) await applied
+      await whenSyncAccepted(applied)
     }
 
     if (lifecycleGeneration !== this.lifecycleGeneration) return
@@ -3600,7 +3606,7 @@ class PersistedCollectionRuntime<
     )
 
     const applied = this.withInternalApply(() => {
-      this.syncControls.begin?.({ immediate: true })
+      this.syncControls.begin?.()
 
       for (const {
         key: changedKey,
@@ -3646,7 +3652,7 @@ class PersistedCollectionRuntime<
 
       return this.syncControls.commit?.() ?? true
     })
-    if (applied !== true) await applied
+    await whenSyncAccepted(applied)
   }
 
   private async reloadActiveSubsetsUnsafe(
@@ -3692,7 +3698,7 @@ class PersistedCollectionRuntime<
         })),
         collectionMetadata,
       )
-      if (applied !== true) await applied
+      await whenSyncAccepted(applied)
     } finally {
       if (
         this.activeHydrationContext === hydrationContext &&
@@ -3934,6 +3940,19 @@ function createWrappedSyncConfig<
       ) => {
         removePendingPublicationTransaction(transaction)
       }
+      // A source receipt is accepted once its rows are durable, and resolves
+      // when core makes them visible.
+      const visibleAfterDurable = (
+        transaction: OpenSyncTransaction<T, TKey>,
+        durable: Promise<void>,
+      ): Promise<void> => {
+        const visible = durable.then(async () => {
+          const receipt = transaction.applicationReceipt
+          if (receipt !== undefined && receipt !== true) await receipt
+        })
+        void visible.catch(() => undefined)
+        return withAcceptedReceipt(visible, durable)
+      }
       const settleRuntimeTransaction = (
         transaction: OpenSyncTransaction<T, TKey>,
         applied: Promise<void>,
@@ -4072,14 +4091,7 @@ function createWrappedSyncConfig<
           }
           transaction.deferredHydrationMetadataDeleteKeys.clear()
 
-          // A buffered source replay is part of the hydrate that owns it.
-          // It must not wait for a mutation whose persistence is queued
-          // behind that hydrate's apply mutex.
-          params.begin(
-            transaction.queuedBecauseHydrating
-              ? { immediate: true }
-              : transaction.beginOptions,
-          )
+          params.begin()
           if (transaction.truncate) params.truncate()
           for (const operation of transaction.operations) {
             if (operation.type === `delete`) {
@@ -4106,14 +4118,10 @@ function createWrappedSyncConfig<
 
           const applied = params.commit(signal)
           transaction.applicationReceipt = applied
-          if (applied === true) {
-            removePendingPublicationTransaction(transaction)
-          } else {
-            void applied.then(
-              () => removePendingPublicationTransaction(transaction),
-              () => removePendingPublicationTransaction(transaction),
-            )
-          }
+          // Core accepted the transaction, so it is no longer staged here;
+          // the receipt resolves when it is visible.
+          removePendingPublicationTransaction(transaction)
+          if (applied !== true) void applied.catch(() => undefined)
           return applied
         } catch (error) {
           removePendingPublicationTransaction(transaction)
@@ -4135,6 +4143,11 @@ function createWrappedSyncConfig<
       const persistenceCapability: SyncPersistenceCapabilityV1<TKey> = {
         protocol: SYNC_PERSISTENCE_PROTOCOL,
         version: SYNC_PERSISTENCE_VERSION,
+        reserveCommitTurn: () => {
+          const openTransaction = getOpenTransaction()
+          if (openTransaction && !openTransaction.internal)
+            openTransaction.reservesCommitTurn = true
+        },
         hydrateBaseline: async () => {
           if (startupState.cleanedUp) return
           try {
@@ -4192,7 +4205,7 @@ function createWrappedSyncConfig<
             })
             .catch(() => undefined)
         },
-        begin: (options?: { immediate?: boolean }) => {
+        begin: () => {
           if (startupState.cleanedUp) return undefined
           const terminalFailure = getTerminalFailure()
           const internal = runtime.isApplyingInternally()
@@ -4205,7 +4218,6 @@ function createWrappedSyncConfig<
             truncate: false,
             internal,
             lifecycleGeneration: runtime.getLifecycleGeneration(),
-            beginOptions: options,
             operationKeys: new Set(),
             hasDependentSuccessor: false,
             queuedBecauseHydrating:
@@ -4522,7 +4534,6 @@ function createWrappedSyncConfig<
             truncate: openTransaction.truncate,
             internal: false,
             lifecycleGeneration: openTransaction.lifecycleGeneration,
-            beginOptions: openTransaction.beginOptions,
             expectedResumeGenerationOwner:
               openTransaction.expectedResumeGenerationOwner,
             signal,
@@ -4576,7 +4587,7 @@ function createWrappedSyncConfig<
             })
             settlePublicationAdmissionWaiters(openTransaction)
             settleRuntimeTransaction(openTransaction, applied)
-            return applied
+            return visibleAfterDurable(openTransaction, applied)
           }
 
           let applied: Promise<void>
@@ -4588,7 +4599,7 @@ function createWrappedSyncConfig<
           }
           settlePublicationAdmissionWaiters(openTransaction)
           settleRuntimeTransaction(openTransaction, applied)
-          return applied
+          return visibleAfterDurable(openTransaction, applied)
         },
       }
 
@@ -4728,9 +4739,7 @@ function createWrappedSyncConfig<
 
           const openTransaction = getOpenTransaction()
           const needsPublicationAdmission =
-            openTransaction &&
-            !openTransaction.internal &&
-            openTransaction.beginOptions?.immediate
+            openTransaction?.reservesCommitTurn === true
           const hydrated =
             sourceResultSettled && !needsPublicationAdmission
               ? runtime.loadHydratedSubset(options, (loadOptions) =>
@@ -4751,13 +4760,9 @@ function createWrappedSyncConfig<
               return
             }
             const currentTransaction = getOpenTransaction()
-            // An immediate source transaction reserves its FIFO turn before
+            // A source transaction that reserved its FIFO turn commits before
             // subset hydration advances the generation it was built against.
-            if (
-              currentTransaction &&
-              !currentTransaction.internal &&
-              currentTransaction.beginOptions?.immediate
-            ) {
+            if (currentTransaction?.reservesCommitTurn) {
               await waitForPublicationAdmission(
                 currentTransaction,
                 options,

package/src/remote-subset-wire.ts

@@ -2,12 +2,7 @@ import { IR } from '@tanstack/db'
 import type { LoadSubsetOptions } from '@tanstack/db'
 
 export type RemoteSubsetWirePrimitive =
-  | undefined
-  | null
-  | boolean
-  | string
-  | number
-  | bigint
+  undefined | null | boolean | string | number | bigint
 
 export type RemoteSubsetWireTypedArray =
   | Int8Array