@tanstack/powersync-db-collection 0.1.67 → 0.2.1

package/src/powersync.ts

@@ -1,4 +1,4 @@
-import { DiffTriggerOperation, sanitizeSQL } from '@powersync/common'
+import { DiffTriggerOperation, LogLevels, sanitizeSQL } from '@powersync/common'
 import { or, withCollectionConfigFactory } from '@tanstack/db'
 import { compileSQLite } from './sqlite-compiler'
 import { PendingOperationStore } from './PendingOperationStore'
@@ -17,8 +17,6 @@ import type {
 import type {
   AnyTableColumnType,
   ExtractedTable,
-  ExtractedTableColumns,
-  MapBaseColumnType,
   OptionalExtractedTable,
 } from './helpers'
 import type {
@@ -26,7 +24,6 @@ import type {
   ConfigWithArbitraryCollectionTypes,
   ConfigWithSQLiteInputType,
   ConfigWithSQLiteTypes,
-  CustomSQLiteSerializer,
   EnhancedPowerSyncCollectionConfig,
   InferPowerSyncOutputType,
   PowerSyncCollectionConfig,
@@ -36,6 +33,14 @@ import type { PendingOperation } from './PendingOperationStore'
 import type { StandardSchemaV1 } from '@standard-schema/spec'
 import type { LockContext, Table, TriggerDiffRecord } from '@powersync/common'
 
+function isPromiseLike(value: unknown): value is PromiseLike<unknown> {
+  return (
+    !!value &&
+    (typeof value === `object` || typeof value === `function`) &&
+    typeof (value as { then?: unknown }).then === `function`
+  )
+}
+
 /**
  * Creates PowerSync collection options for use with a standard Collection.
  *
@@ -275,12 +280,15 @@ function createPowerSyncCollectionConfig<
       return validation.value
     } else if (`issues` in validation) {
       const issueMessage = `Failed to validate incoming data for ${viewName}. Issues: ${validation.issues.map((issue) => `${issue.path} - ${issue.message}`)}`
-      database.logger.error(issueMessage)
+      database.logger.log({ level: LogLevels.error, message: issueMessage })
       onDeserializationError!(validation)
       throw new Error(issueMessage)
     } else {
       const unknownErrorMessage = `Unknown deserialization error for ${viewName}`
-      database.logger.error(unknownErrorMessage)
+      database.logger.log({
+        level: LogLevels.error,
+        message: unknownErrorMessage,
+      })
       onDeserializationError!({ issues: [{ message: unknownErrorMessage }] })
       throw new Error(unknownErrorMessage)
     }
@@ -324,6 +332,8 @@ function createPowerSyncCollectionConfig<
         | ((options?: { context?: LockContext }) => Promise<void>)
         | null = null
       let trackingSetup: Promise<void> | null = null
+      let activeTrackingDisposal: Promise<void> | null = null
+      let abortTrackingDisposal: Promise<void> | null = null
 
       if (syncMode === `eager`) {
         return runEagerSync()
@@ -347,11 +357,35 @@ function createPowerSyncCollectionConfig<
 
         const dispose = disposeTracking
         if (!dispose) {
+          // Reconciliation may have claimed the disposer after this caller
+          // observed setup. Join that invocation instead of settling early.
+          await activeTrackingDisposal
           return
         }
 
         disposeTracking = null
-        await dispose(context ? { context } : undefined)
+        let disposal: Promise<void>
+        try {
+          disposal = Promise.resolve(dispose(context ? { context } : undefined))
+        } catch (error) {
+          disposal = Promise.reject(error)
+        }
+        activeTrackingDisposal = disposal
+        try {
+          await disposal
+        } finally {
+          if (activeTrackingDisposal === disposal) {
+            activeTrackingDisposal = null
+          }
+        }
+      }
+
+      function disposeTrackingAfterAbort(): Promise<void> {
+        const disposal = (abortTrackingDisposal ??= safelyDisposeTracking())
+        // Abort event dispatch does not observe a listener's returned promise.
+        // The eager cleanup callback awaits this same promise below.
+        void disposal.catch(() => undefined)
+        return disposal
       }
 
       async function establishTracking(
@@ -392,6 +426,7 @@ function createPowerSyncCollectionConfig<
         return await database.triggers.createDiffTrigger({
           source: viewName,
           destination: trackedTableName,
+          columns: table.columns.map((column) => column.name),
           setupContext,
           when,
           hooks: {
@@ -416,9 +451,10 @@ function createPowerSyncCollectionConfig<
                 }
                 appliedReceipts.push(commit())
               }
-              database.logger.info(
-                `Sync is ready for ${viewName} into ${trackedTableName}`,
-              )
+              database.logger.log({
+                level: LogLevels.info,
+                message: `Sync is ready for ${viewName} into ${trackedTableName}`,
+              })
             },
           },
         })
@@ -442,10 +478,11 @@ function createPowerSyncCollectionConfig<
             await flushDiffRecordsWithContext(context, ignoredReceipts)
           })
           .catch((error) => {
-            database.logger.error(
-              `An error has been detected in the sync handler`,
+            database.logger.log({
+              level: LogLevels.error,
+              message: `An error has been detected in the sync handler`,
               error,
-            )
+            })
           })
       }
 
@@ -503,18 +540,20 @@ function createPowerSyncCollectionConfig<
           // transaction that currently parks it.
           pendingOperationStore.resolvePendingFor(pendingOperations)
         } catch (error) {
-          database.logger.error(
-            `An error has been detected in the sync handler`,
+          database.logger.log({
+            level: LogLevels.error,
+            message: `An error has been detected in the sync handler`,
             error,
-          )
+          })
         }
       }
 
       // The sync function needs to be synchronous.
       async function start(afterOnChangeRegistered?: () => Promise<void>) {
-        database.logger.info(
-          `Sync is starting for ${viewName} into ${trackedTableName}`,
-        )
+        database.logger.log({
+          level: LogLevels.info,
+          message: `Sync is starting for ${viewName} into ${trackedTableName}`,
+        })
         database.onChangeWithCallback(
           {
             onChange: async () => {
@@ -532,12 +571,12 @@ function createPowerSyncCollectionConfig<
 
         // If the abort controller was aborted while processing the request above
         if (abortController.signal.aborted) {
-          await safelyDisposeTracking()
+          await disposeTrackingAfterAbort()
         } else {
           abortController.signal.addEventListener(
             `abort`,
-            async () => {
-              await safelyDisposeTracking()
+            () => {
+              void disposeTrackingAfterAbort()
             },
             { once: true },
           )
@@ -548,14 +587,14 @@ function createPowerSyncCollectionConfig<
       // Registers a diff trigger for the entire table.
       function runEagerSync() {
         let onUnload: CleanupFn | void | null = null
+        let onUnloadStarted = false
 
-        start(async () => {
+        const startup = start(async () => {
           const cleanup = await restConfig.onLoad?.()
+          onUnload = cleanup
           if (abortController.signal.aborted) {
-            cleanup?.()
             return
           }
-          onUnload = cleanup
 
           const appliedReceipts: Array<SyncAppliedReceipt> = []
           await establishTracking(
@@ -584,21 +623,61 @@ function createPowerSyncCollectionConfig<
           await Promise.all(appliedReceipts)
           markReady()
         }).catch((error) => {
-          database.logger.error(
-            `Could not start syncing process for ${viewName} into ${trackedTableName}`,
+          database.logger.log({
+            level: LogLevels.error,
+            message: `Could not start syncing process for ${viewName} into ${trackedTableName}`,
             error,
-          )
+          })
           if (collection.status === `loading`) {
             markError(error)
           }
         })
 
-        return () => {
-          database.logger.info(
-            `Sync has been stopped for ${viewName} into ${trackedTableName}`,
-          )
+        const invokeOnUnload = (): void | Promise<void> => {
+          if (onUnloadStarted || !onUnload) return
+          onUnloadStarted = true
+          const cleanup = onUnload
+          onUnload = null
+          return cleanup()
+        }
+
+        return async () => {
+          database.logger.log({
+            level: LogLevels.info,
+            message: `Sync has been stopped for ${viewName} into ${trackedTableName}`,
+          })
           abortController.abort()
-          onUnload?.()
+          const trackingDisposal = disposeTrackingAfterAbort()
+
+          let onUnloadFailure: { error: unknown } | undefined
+          const settleOnUnload = async () => {
+            try {
+              await invokeOnUnload()
+            } catch (error) {
+              onUnloadFailure ??= { error }
+            }
+          }
+
+          // Invoke an already-installed callback in this stack, but also wait
+          // for startup to publish a callback acquired concurrently with abort.
+          const settlements = await Promise.allSettled([
+            startup,
+            trackingDisposal,
+            settleOnUnload(),
+          ])
+          const disposalSettlement = settlements[1]
+          await settleOnUnload()
+          if (disposalSettlement.status === `rejected` && onUnloadFailure) {
+            throw new AggregateError(
+              [disposalSettlement.reason, onUnloadFailure.error],
+              `PowerSync tracking disposal and load-hook cleanup both failed`,
+              { cause: disposalSettlement.reason },
+            )
+          }
+          if (disposalSettlement.status === `rejected`) {
+            throw disposalSettlement.reason
+          }
+          if (onUnloadFailure) throw onUnloadFailure.error
         }
       }
 
@@ -614,22 +693,40 @@ function createPowerSyncCollectionConfig<
           options: LoadSubsetOptions
           failures: number
         }
+        type DemandCleanupTask = {
+          order: number
+          promise: Promise<unknown>
+        }
+        type DemandCleanupFailure = {
+          order: number
+          error: unknown
+        }
+        type DemandCleanupCollector = {
+          tasks: Array<DemandCleanupTask>
+          ownedTasks: Set<Promise<unknown>>
+          failures: Array<DemandCleanupFailure>
+          nextOrder: number
+        }
 
         const demands = new Map<LoadSubsetOptions, DemandRecord>()
         const releasedSubsets = new WeakSet<LoadSubsetOptions>()
         const pendingReleases: Array<PendingRelease> = []
+        const pendingDemandLoads = new Set<Promise<void>>()
+        const pendingDemandCleanups = new Set<Promise<unknown>>()
         let stopped = false
         let trackingRevision = 0
         let reconciledTrackingRevision = 0
         let rebuildPromise: Promise<void> | null = null
         let drainingReleases = false
         let releaseRetryTimer: ReturnType<typeof setTimeout> | undefined
+        let demandCleanupCollector: DemandCleanupCollector | null = null
         const startup = start()
         void startup.catch((error) =>
-          database.logger.error(
-            `Could not start syncing process for ${viewName} into ${trackedTableName}`,
+          database.logger.log({
+            level: LogLevels.error,
+            message: `Could not start syncing process for ${viewName} into ${trackedTableName}`,
             error,
-          ),
+          }),
         )
 
         const activeWhereExpressions = () =>
@@ -653,51 +750,70 @@ function createPowerSyncCollectionConfig<
               if (!isCurrent()) return
 
               const active = activeWhereExpressions()
-              if (active.length === 0) return
-              const combinedWhere =
-                active.length === 1
-                  ? active[0]
-                  : or(active[0], active[1], ...active.slice(2))
-              const compiledNewData = compileSQLite(
-                { where: combinedWhere },
-                { jsonColumn: 'NEW.data' },
-              )
-              const compiledOldData = compileSQLite(
-                { where: combinedWhere },
-                { jsonColumn: 'OLD.data' },
-              )
-              const compiledView = compileSQLite({ where: combinedWhere })
-              const newDataWhenClause = toInlinedWhereClause(compiledNewData)
-              const oldDataWhenClause = toInlinedWhereClause(compiledOldData)
-              const viewWhereClause = toInlinedWhereClause(compiledView)
-
-              await establishTracking(
-                {
-                  setupContext: ctx,
-                  when: {
-                    [DiffTriggerOperation.INSERT]: newDataWhenClause,
-                    [DiffTriggerOperation.UPDATE]: `(${newDataWhenClause}) OR (${oldDataWhenClause})`,
-                    [DiffTriggerOperation.DELETE]: oldDataWhenClause,
+              // Tracking was absent during an error. Positive baseline rows
+              // alone cannot reveal deletes or predicate exits from that gap.
+              const missing =
+                collection.status === `error`
+                  ? new Set(collection.keys())
+                  : undefined
+              if (active.length > 0) {
+                const combinedWhere =
+                  active.length === 1
+                    ? active[0]
+                    : or(active[0], active[1], ...active.slice(2))
+                const compiledNewData = compileSQLite(
+                  { where: combinedWhere },
+                  { jsonColumn: 'NEW.data' },
+                )
+                const compiledOldData = compileSQLite(
+                  { where: combinedWhere },
+                  { jsonColumn: 'OLD.data' },
+                )
+                const compiledView = compileSQLite({ where: combinedWhere })
+                const newDataWhenClause = toInlinedWhereClause(compiledNewData)
+                const oldDataWhenClause = toInlinedWhereClause(compiledOldData)
+                const viewWhereClause = toInlinedWhereClause(compiledView)
+                await establishTracking(
+                  {
+                    setupContext: ctx,
+                    when: {
+                      [DiffTriggerOperation.INSERT]: newDataWhenClause,
+                      [DiffTriggerOperation.UPDATE]: `(${newDataWhenClause}) OR (${oldDataWhenClause})`,
+                      [DiffTriggerOperation.DELETE]: oldDataWhenClause,
+                    },
+                    writeType: (rowId: string) =>
+                      collection.has(rowId) ? `update` : `insert`,
+                    batchQuery: (
+                      lockContext: LockContext,
+                      batchSize: number,
+                      cursor: number,
+                    ) =>
+                      lockContext
+                        .getAll<TableType>(
+                          `SELECT * FROM ${viewName} WHERE ${viewWhereClause} LIMIT ? OFFSET ?`,
+                          [batchSize, cursor],
+                        )
+                        .then((rows) => {
+                          for (const row of rows) missing?.delete(row.id)
+                          return rows
+                        }),
                   },
-                  writeType: (rowId: string) =>
-                    collection.has(rowId) ? `update` : `insert`,
-                  batchQuery: (
-                    lockContext: LockContext,
-                    batchSize: number,
-                    cursor: number,
-                  ) =>
-                    lockContext.getAll<TableType>(
-                      `SELECT * FROM ${viewName} WHERE ${viewWhereClause} LIMIT ? OFFSET ?`,
-                      [batchSize, cursor],
-                    ),
-                },
-                appliedReceipts,
-              )
+                  appliedReceipts,
+                )
+              }
               if (!isCurrent()) await safelyDisposeTracking(ctx)
+              else if (missing?.size) {
+                begin()
+                for (const key of missing) write({ type: `delete`, key })
+                appliedReceipts.push(commit())
+              }
             })
             await Promise.all(appliedReceipts)
             if (isCurrent()) {
               reconciledTrackingRevision = revision
+              // Replacing the trigger alone is not recovery: its baseline
+              // writes must also be applied before the source is ready again.
+              if (collection.status === `error`) markReady()
             }
           }
         }
@@ -742,6 +858,13 @@ function createPowerSyncCollectionConfig<
             if (cleanup) demand.cleanup = cleanup
           } catch (error) {
             demands.delete(options)
+            const collector = demandCleanupCollector
+            if (collector) {
+              collector.failures.push({
+                order: collector.nextOrder++,
+                error,
+              })
+            }
             throw error
           }
 
@@ -753,8 +876,7 @@ function createPowerSyncCollectionConfig<
             options.signal?.aborted ||
             demands.get(options) !== demand
           ) {
-            demands.delete(options)
-            demand.cleanup?.()
+            cleanupDemand(demand)
             return
           }
 
@@ -763,6 +885,14 @@ function createPowerSyncCollectionConfig<
           await rebuildTracking()
         }
 
+        const trackLoadSubset = (options: LoadSubsetOptions): Promise<void> => {
+          const task = loadSubset(options)
+          pendingDemandLoads.add(task)
+          const finish = () => pendingDemandLoads.delete(task)
+          void task.then(finish, finish)
+          return task
+        }
+
         const toInlinedWhereClause = (compiled: {
           where?: string
           params: Array<unknown>
@@ -775,15 +905,64 @@ function createPowerSyncCollectionConfig<
           )
         }
 
+        const reportDemandCleanupFailure = (error: unknown): void => {
+          database.logger.log({
+            level: LogLevels.error,
+            message: `Could not clean up subset hook for ${viewName}`,
+            error,
+          })
+        }
+
         const cleanupDemand = (demand: DemandRecord): void => {
           demands.delete(demand.options)
+          const collector = demandCleanupCollector
+          const order = collector ? collector.nextOrder++ : undefined
           try {
-            demand.cleanup?.()
+            const cleanup = demand.cleanup
+            demand.cleanup = undefined
+            const result: unknown = cleanup?.()
+            if (!isPromiseLike(result)) return
+
+            const task = Promise.resolve(result)
+            pendingDemandCleanups.add(task)
+            const finish = () => pendingDemandCleanups.delete(task)
+            void task.then(finish, finish)
+            if (collector) {
+              collector.tasks.push({ order: order!, promise: task })
+              collector.ownedTasks.add(task)
+              void task.catch(() => undefined)
+            } else {
+              void task.catch((error) => {
+                if (!demandCleanupCollector?.ownedTasks.has(task)) {
+                  reportDemandCleanupFailure(error)
+                }
+              })
+            }
           } catch (error) {
-            database.logger.error(
-              `Could not clean up subset hook for ${viewName}`,
-              error,
-            )
+            if (collector) {
+              collector.failures.push({ order: order!, error })
+            } else {
+              reportDemandCleanupFailure(error)
+            }
+          }
+        }
+
+        const throwDemandCleanupFailures = (
+          collector: DemandCleanupCollector,
+          primaryFailures: ReadonlyArray<unknown> = [],
+        ): void => {
+          const errors = [
+            ...primaryFailures,
+            ...collector.failures
+              .slice()
+              .sort((left, right) => left.order - right.order)
+              .map(({ error }) => error),
+          ]
+          if (errors.length === 1) throw errors[0]
+          if (errors.length > 1) {
+            throw new AggregateError(errors, `PowerSync cleanup failed`, {
+              cause: errors[0],
+            })
           }
         }
 
@@ -859,10 +1038,11 @@ function createPowerSyncCollectionConfig<
                 )
                 retryDelay =
                   retryDelay === 0 ? delay : Math.min(retryDelay, delay)
-                database.logger.error(
-                  `Could not release subset tracking for ${viewName}; retrying`,
+                database.logger.log({
+                  level: LogLevels.error,
+                  message: `Could not release subset tracking for ${viewName}; retrying`,
                   error,
-                )
+                })
               }
             }
           } finally {
@@ -896,16 +1076,62 @@ function createPowerSyncCollectionConfig<
             stopped = true
             clearTimeout(releaseRetryTimer)
             releaseRetryTimer = undefined
-            database.logger.info(
-              `Sync has been stopped for ${viewName} into ${trackedTableName}`,
-            )
+            database.logger.log({
+              level: LogLevels.info,
+              message: `Sync has been stopped for ${viewName} into ${trackedTableName}`,
+            })
             abortController.abort()
+            const trackingDisposal = disposeTrackingAfterAbort()
+            const pendingLoads = [...pendingDemandLoads]
+            const pendingTasks = [...pendingDemandCleanups]
+            const collector: DemandCleanupCollector = {
+              tasks: pendingTasks.map((promise, order) => ({ order, promise })),
+              ownedTasks: new Set(pendingTasks),
+              failures: [],
+              nextOrder: pendingTasks.length,
+            }
+            demandCleanupCollector = collector
             for (const demand of demands.values()) {
               cleanupDemand(demand)
             }
             pendingReleases.length = 0
+
+            return (async () => {
+              const ownershipSettlements = await Promise.allSettled([
+                trackingDisposal,
+                ...pendingLoads,
+              ])
+              const disposalSettlement = ownershipSettlements[0]
+
+              let settledTasks = 0
+              while (settledTasks < collector.tasks.length) {
+                const tasks = collector.tasks.slice(settledTasks)
+                settledTasks = collector.tasks.length
+                const outcomes = await Promise.allSettled(
+                  tasks.map(({ promise }) => promise),
+                )
+                for (const [index, outcome] of outcomes.entries()) {
+                  if (outcome.status === `rejected`) {
+                    collector.failures.push({
+                      order: tasks[index]!.order,
+                      error: outcome.reason,
+                    })
+                  }
+                }
+              }
+              throwDemandCleanupFailures(
+                collector,
+                disposalSettlement.status === `rejected`
+                  ? [disposalSettlement.reason]
+                  : [],
+              )
+            })().finally(() => {
+              if (demandCleanupCollector === collector) {
+                demandCleanupCollector = null
+              }
+            })
           },
-          loadSubset: (options: LoadSubsetOptions) => loadSubset(options),
+          loadSubset: trackLoadSubset,
           unloadSubset,
         }
       }
@@ -946,18 +1172,7 @@ function createPowerSyncCollectionConfig<
         trackedTableName,
         metadataIsTracked,
         serializeValue: (value) =>
-          serializeForSQLite(
-            value,
-            // This is required by the input generic
-            table as Table<
-              MapBaseColumnType<InferPowerSyncOutputType<TTable, TSchema>>
-            >,
-            // Coerce serializer to the shape that corresponds to the Table constructed from OutputType
-            serializer as CustomSQLiteSerializer<
-              OutputType,
-              ExtractedTableColumns<Table<MapBaseColumnType<OutputType>>>
-            >,
-          ),
+          serializeForSQLite<TTable>(value, table, serializer),
       }),
     },
   }

package/src/serialization.ts

@@ -2,9 +2,9 @@ import { ColumnType } from '@powersync/common'
 import type { Table } from '@powersync/common'
 import type { CustomSQLiteSerializer } from './definitions'
 import type {
+  AnyTableColumnType,
   ExtractedTable,
   ExtractedTableColumns,
-  MapBaseColumnType,
 } from './helpers'
 
 /**
@@ -13,10 +13,6 @@ import type {
  * This function takes an object representing a row, a table schema, and an optional custom serializer map.
  * It returns a new object with values transformed to be compatible with SQLite column types.
  *
- * ## Generics
- * - `TOutput`: The shape of the input object, typically matching the row data.
- * - `TTable`: The table schema, which must match the keys of `TOutput`.
- *
  * ## Parameters
  * - `value`: The object to serialize (row data).
  * - `tableSchema`: The schema describing the SQLite table columns and types.
@@ -37,17 +33,12 @@ import type {
  * - Throws if a key in `value` does not exist in the schema.
  * - Throws if a value cannot be converted to the required SQLite type.
  */
-export function serializeForSQLite<
-  TOutput extends Record<string, unknown>,
-  // The keys should match
-  TTable extends Table<MapBaseColumnType<TOutput>> = Table<
-    MapBaseColumnType<TOutput>
-  >,
->(
-  value: TOutput,
+export function serializeForSQLite<TTable extends Table>(
+  value: AnyTableColumnType<TTable>,
   tableSchema: TTable,
-  customSerializer: Partial<
-    CustomSQLiteSerializer<TOutput, ExtractedTableColumns<TTable>>
+  customSerializer: CustomSQLiteSerializer<
+    AnyTableColumnType<TTable>,
+    ExtractedTableColumns<TTable>
   > = {},
 ): ExtractedTable<TTable> {
   return Object.fromEntries(
@@ -68,7 +59,7 @@ export function serializeForSQLite<
 
       const customTransform = customSerializer[key]
       if (customTransform) {
-        return [key, customTransform(value as TOutput[string])]
+        return [key, customTransform(value)]
       }
 
       // Map to the output
@@ -98,5 +89,5 @@ export function serializeForSQLite<
           }
       }
     }),
-  )
+  ) as ExtractedTable<TTable>
 }