@tanstack/powersync-db-collection 0.0.0

package/package.json

@@ -0,0 +1,71 @@
+{
+  "name": "@tanstack/powersync-db-collection",
+  "description": "PowerSync collection for TanStack DB",
+  "version": "0.0.0",
+  "dependencies": {
+    "@standard-schema/spec": "^1.0.0",
+    "@tanstack/db": "workspace:*",
+    "@tanstack/store": "^0.8.0",
+    "debug": "^4.4.3",
+    "p-defer": "^4.0.1"
+  },
+  "peerDependencies": {
+    "@powersync/common": "^1.41.0"
+  },
+  "devDependencies": {
+    "@powersync/common": "^1.41.0",
+    "@powersync/node": "^0.13.0",
+    "@types/debug": "^4.1.12",
+    "@vitest/coverage-istanbul": "^3.2.4"
+  },
+  "exports": {
+    ".": {
+      "import": {
+        "types": "./dist/esm/index.d.ts",
+        "default": "./dist/esm/index.js"
+      },
+      "require": {
+        "types": "./dist/cjs/index.d.cts",
+        "default": "./dist/cjs/index.cjs"
+      }
+    },
+    "./package.json": "./package.json"
+  },
+  "files": [
+    "dist",
+    "src"
+  ],
+  "main": "dist/cjs/index.cjs",
+  "module": "dist/esm/index.js",
+  "packageManager": "pnpm@10.17.0",
+  "author": "POWERSYNC",
+  "license": "MIT",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/TanStack/db.git",
+    "directory": "packages/powersync-db-collection"
+  },
+  "homepage": "https://tanstack.com/db",
+  "keywords": [
+    "powersync",
+    "realtime",
+    "local-first",
+    "sync-engine",
+    "sync",
+    "replication",
+    "opfs",
+    "indexeddb",
+    "localstorage",
+    "optimistic",
+    "typescript"
+  ],
+  "scripts": {
+    "build": "vite build",
+    "dev": "vite build --watch",
+    "lint": "eslint . --fix",
+    "test": "npx vitest --run"
+  },
+  "sideEffects": false,
+  "type": "module",
+  "types": "dist/esm/index.d.ts"
+}

package/src/PendingOperationStore.ts

@@ -0,0 +1,54 @@
+import pDefer from "p-defer"
+import type { DiffTriggerOperation } from "@powersync/common"
+import type { DeferredPromise } from "p-defer"
+
+export type PendingOperation = {
+  tableName: string
+  operation: DiffTriggerOperation
+  id: string
+  timestamp: string
+}
+
+/**
+ * Optimistic mutations have their optimistic state discarded once transactions have
+ * been applied.
+ * We need to ensure that an applied transaction has been observed by the sync diff trigger
+ * before resolving the transaction application call.
+ * This store allows registering a wait for a pending operation to have been observed.
+ */
+export class PendingOperationStore {
+  private pendingOperations = new Map<PendingOperation, DeferredPromise<void>>()
+
+  /**
+   * Globally accessible PendingOperationStore
+   */
+  static GLOBAL = new PendingOperationStore()
+
+  /**
+   * @returns A promise which will resolve once the specified operation has been seen.
+   */
+  waitFor(operation: PendingOperation): Promise<void> {
+    const managedPromise = pDefer<void>()
+    this.pendingOperations.set(operation, managedPromise)
+    return managedPromise.promise
+  }
+
+  /**
+   * Marks a set of operations as seen. This will resolve any pending promises.
+   */
+  resolvePendingFor(operations: Array<PendingOperation>) {
+    for (const operation of operations) {
+      for (const [pendingOp, deferred] of this.pendingOperations.entries()) {
+        if (
+          pendingOp.tableName == operation.tableName &&
+          pendingOp.operation == operation.operation &&
+          pendingOp.id == operation.id &&
+          pendingOp.timestamp == operation.timestamp
+        ) {
+          deferred.resolve()
+          this.pendingOperations.delete(pendingOp)
+        }
+      }
+    }
+  }
+}

package/src/PowerSyncTransactor.ts

@@ -0,0 +1,271 @@
+import { sanitizeSQL } from "@powersync/common"
+import DebugModule from "debug"
+import { asPowerSyncRecord, mapOperationToPowerSync } from "./helpers"
+import { PendingOperationStore } from "./PendingOperationStore"
+import type { AbstractPowerSyncDatabase, LockContext } from "@powersync/common"
+import type { PendingMutation, Transaction } from "@tanstack/db"
+import type { EnhancedPowerSyncCollectionConfig } from "./definitions"
+import type { PendingOperation } from "./PendingOperationStore"
+
+const debug = DebugModule.debug(`ts/db:powersync`)
+
+export type TransactorOptions = {
+  database: AbstractPowerSyncDatabase
+}
+
+/**
+ * Applies mutations to the PowerSync database. This method is called automatically by the collection's
+ * insert, update, and delete operations. You typically don't need to call this directly unless you
+ * have special transaction requirements.
+ *
+ * @example
+ * ```typescript
+ * // Create a collection
+ * const collection = createCollection(
+ *   powerSyncCollectionOptions<Document>({
+ *     database: db,
+ *     table: APP_SCHEMA.props.documents,
+ *   })
+ * )
+ *
+ * const addTx = createTransaction({
+ *   autoCommit: false,
+ *   mutationFn: async ({ transaction }) => {
+ *     await new PowerSyncTransactor({ database: db }).applyTransaction(transaction)
+ *   },
+ * })
+ *
+ * addTx.mutate(() => {
+ *   for (let i = 0; i < 5; i++) {
+ *     collection.insert({ id: randomUUID(), name: `tx-${i}` })
+ *   }
+ * })
+ *
+ * await addTx.commit()
+ * await addTx.isPersisted.promise
+ * ```
+ *
+ * @param transaction - The transaction containing mutations to apply
+ * @returns A promise that resolves when the mutations have been persisted to PowerSync
+ */
+export class PowerSyncTransactor {
+  database: AbstractPowerSyncDatabase
+  pendingOperationStore: PendingOperationStore
+
+  constructor(options: TransactorOptions) {
+    this.database = options.database
+    this.pendingOperationStore = PendingOperationStore.GLOBAL
+  }
+
+  /**
+   * Persists a {@link Transaction} to the PowerSync SQLite database.
+   */
+  async applyTransaction(transaction: Transaction<any>) {
+    const { mutations } = transaction
+
+    if (mutations.length == 0) {
+      return
+    }
+    /**
+     * The transaction might contain operations for different collections.
+     * We can do some optimizations for single-collection transactions.
+     */
+    const mutationsCollectionIds = mutations.map(
+      (mutation) => mutation.collection.id
+    )
+    const collectionIds = Array.from(new Set(mutationsCollectionIds))
+    const lastCollectionMutationIndexes = new Map<string, number>()
+    const allCollections = collectionIds
+      .map((id) => mutations.find((mutation) => mutation.collection.id == id)!)
+      .map((mutation) => mutation.collection)
+    for (const collectionId of collectionIds) {
+      lastCollectionMutationIndexes.set(
+        collectionId,
+        mutationsCollectionIds.lastIndexOf(collectionId)
+      )
+    }
+
+    // Check all the observers are ready before taking a lock
+    await Promise.all(
+      allCollections.map(async (collection) => {
+        if (collection.isReady()) {
+          return
+        }
+        await new Promise<void>((resolve) => collection.onFirstReady(resolve))
+      })
+    )
+
+    // Persist to PowerSync
+    const { whenComplete } = await this.database.writeTransaction(
+      async (tx) => {
+        const pendingOperations: Array<PendingOperation | null> = []
+
+        for (const [index, mutation] of mutations.entries()) {
+          /**
+           * Each collection processes events independently. We need to make sure the
+           * last operation for each collection has been observed.
+           */
+          const shouldWait =
+            index == lastCollectionMutationIndexes.get(mutation.collection.id)
+          switch (mutation.type) {
+            case `insert`:
+              pendingOperations.push(
+                await this.handleInsert(mutation, tx, shouldWait)
+              )
+              break
+            case `update`:
+              pendingOperations.push(
+                await this.handleUpdate(mutation, tx, shouldWait)
+              )
+              break
+            case `delete`:
+              pendingOperations.push(
+                await this.handleDelete(mutation, tx, shouldWait)
+              )
+              break
+          }
+        }
+
+        /**
+         * Return a promise from the writeTransaction, without awaiting it.
+         * This promise will resolve once the entire transaction has been
+         * observed via the diff triggers.
+         * We return without awaiting in order to free the write lock.
+         */
+        return {
+          whenComplete: Promise.all(
+            pendingOperations
+              .filter((op) => !!op)
+              .map((op) => this.pendingOperationStore.waitFor(op))
+          ),
+        }
+      }
+    )
+
+    // Wait for the change to be observed via the diff trigger
+    await whenComplete
+  }
+
+  protected async handleInsert(
+    mutation: PendingMutation<any>,
+    context: LockContext,
+    waitForCompletion: boolean = false
+  ): Promise<PendingOperation | null> {
+    debug(`insert`, mutation)
+
+    return this.handleOperationWithCompletion(
+      mutation,
+      context,
+      waitForCompletion,
+      async (tableName, mutation, serializeValue) => {
+        const values = serializeValue(mutation.modified)
+        const keys = Object.keys(values).map((key) => sanitizeSQL`${key}`)
+
+        await context.execute(
+          `
+        INSERT into ${tableName} 
+            (${keys.join(`, `)}) 
+        VALUES 
+            (${keys.map((_) => `?`).join(`, `)})
+        `,
+          Object.values(values)
+        )
+      }
+    )
+  }
+
+  protected async handleUpdate(
+    mutation: PendingMutation<any>,
+    context: LockContext,
+    waitForCompletion: boolean = false
+  ): Promise<PendingOperation | null> {
+    debug(`update`, mutation)
+
+    return this.handleOperationWithCompletion(
+      mutation,
+      context,
+      waitForCompletion,
+      async (tableName, mutation, serializeValue) => {
+        const values = serializeValue(mutation.modified)
+        const keys = Object.keys(values).map((key) => sanitizeSQL`${key}`)
+
+        await context.execute(
+          `
+        UPDATE ${tableName} 
+        SET ${keys.map((key) => `${key} = ?`).join(`, `)}
+        WHERE id = ?
+        `,
+          [...Object.values(values), asPowerSyncRecord(mutation.modified).id]
+        )
+      }
+    )
+  }
+
+  protected async handleDelete(
+    mutation: PendingMutation<any>,
+    context: LockContext,
+    waitForCompletion: boolean = false
+  ): Promise<PendingOperation | null> {
+    debug(`update`, mutation)
+
+    return this.handleOperationWithCompletion(
+      mutation,
+      context,
+      waitForCompletion,
+      async (tableName, mutation) => {
+        await context.execute(
+          `
+        DELETE FROM ${tableName} WHERE id = ?
+        `,
+          [asPowerSyncRecord(mutation.original).id]
+        )
+      }
+    )
+  }
+
+  /**
+   * Helper function which wraps a persistence operation by:
+   * - Fetching the mutation's collection's SQLite table details
+   * - Executing the mutation
+   * - Returning the last pending diff operation if required
+   */
+  protected async handleOperationWithCompletion(
+    mutation: PendingMutation<any>,
+    context: LockContext,
+    waitForCompletion: boolean,
+    handler: (
+      tableName: string,
+      mutation: PendingMutation<any>,
+      serializeValue: (value: any) => Record<string, unknown>
+    ) => Promise<void>
+  ): Promise<PendingOperation | null> {
+    if (
+      typeof (mutation.collection.config as any).utils?.getMeta != `function`
+    ) {
+      throw new Error(`Could not get tableName from mutation's collection config.
+        The provided mutation might not have originated from PowerSync.`)
+    }
+
+    const { tableName, trackedTableName, serializeValue } = (
+      mutation.collection
+        .config as unknown as EnhancedPowerSyncCollectionConfig<any>
+    ).utils.getMeta()
+
+    await handler(sanitizeSQL`${tableName}`, mutation, serializeValue)
+
+    if (!waitForCompletion) {
+      return null
+    }
+
+    // Need to get the operation in order to wait for it
+    const diffOperation = await context.get<{ id: string; timestamp: string }>(
+      sanitizeSQL`SELECT id, timestamp FROM ${trackedTableName} ORDER BY timestamp DESC LIMIT 1`
+    )
+    return {
+      tableName,
+      id: diffOperation.id,
+      operation: mapOperationToPowerSync(mutation.type),
+      timestamp: diffOperation.timestamp,
+    }
+  }
+}

package/src/definitions.ts

@@ -0,0 +1,274 @@
+import type { AbstractPowerSyncDatabase, Table } from "@powersync/common"
+import type { StandardSchemaV1 } from "@standard-schema/spec"
+import type {
+  BaseCollectionConfig,
+  CollectionConfig,
+  InferSchemaOutput,
+} from "@tanstack/db"
+import type {
+  AnyTableColumnType,
+  ExtractedTable,
+  OptionalExtractedTable,
+  PowerSyncRecord,
+} from "./helpers"
+
+/**
+ * Small helper which determines the output type if:
+ * - Standard SQLite types are to be used OR
+ * - If the provided schema should be used.
+ */
+export type InferPowerSyncOutputType<
+  TTable extends Table = Table,
+  TSchema extends StandardSchemaV1<PowerSyncRecord> = never,
+> = TSchema extends never ? ExtractedTable<TTable> : InferSchemaOutput<TSchema>
+
+/**
+ * A mapping type for custom serialization of object properties to SQLite-compatible values.
+ *
+ * This type allows you to override, for keys in the input object (`TOutput`), a function that transforms
+ * the value to the corresponding SQLite type (`TSQLite`). Keys not specified will use the default SQLite serialization.
+ *
+ * ## Generics
+ * - `TOutput`: The input object type, representing the row data to be serialized.
+ * - `TSQLite`: The target SQLite-compatible type for each property, typically inferred from the table schema.
+ *
+ * ## Usage
+ * Use this type to define a map of serialization functions for specific keys when you need custom handling
+ * (e.g., converting complex objects, formatting dates, or handling enums).
+ *
+ * Example:
+ * ```ts
+ * const serializer: CustomSQLiteSerializer<MyRowType, MySQLiteType> = {
+ *   createdAt: (date) => date.toISOString(),
+ *   status: (status) => status ? 1 : 0,
+ *   meta: (meta) => JSON.stringify(meta),
+ * };
+ * ```
+ *
+ * ## Behavior
+ * - Each key maps to a function that receives the value and returns the SQLite-compatible value.
+ * - Used by `serializeForSQLite` to override default serialization for specific columns.
+ */
+export type CustomSQLiteSerializer<
+  TOutput extends Record<string, unknown>,
+  TSQLite extends Record<string, unknown>,
+> = Partial<{
+  [Key in keyof TOutput]: (
+    value: TOutput[Key]
+  ) => Key extends keyof TSQLite ? TSQLite[Key] : never
+}>
+
+export type SerializerConfig<
+  TOutput extends Record<string, unknown>,
+  TSQLite extends Record<string, unknown>,
+> = {
+  /**
+   * Optional partial serializer object for customizing how individual columns are serialized for SQLite.
+   *
+   * This should be a partial map of column keys to serialization functions, following the
+   * {@link CustomSQLiteSerializer} type. Each function receives the column value and returns a value
+   * compatible with SQLite storage.
+   *
+   * If not provided for a column, the default behavior is used:
+   *   - `TEXT`: Strings are stored as-is; Dates are converted to ISO strings; other types are JSON-stringified.
+   *   - `INTEGER`/`REAL`: Numbers are stored as-is; booleans are mapped to 1/0.
+   *
+   * Use this option to override serialization for specific columns, such as formatting dates, handling enums,
+   * or serializing complex objects.
+   *
+   * Example:
+   * ```typescript
+   * serializer: {
+   *   createdAt: (date) => date.getTime(), // Store as timestamp
+   *   meta: (meta) => JSON.stringify(meta), // Custom object serialization
+   * }
+   * ```
+   */
+  serializer?: CustomSQLiteSerializer<TOutput, TSQLite>
+
+  /**
+   * Application logic should ensure that incoming synced data is always valid.
+   * Failing to deserialize and apply incoming changes results in data inconsistency - which is a fatal error.
+   * Use this callback to react to deserialization errors.
+   */
+  onDeserializationError: (error: StandardSchemaV1.FailureResult) => void
+}
+
+/**
+ * Config for when TInput and TOutput are both the SQLite types.
+ */
+export type ConfigWithSQLiteTypes = {}
+
+/**
+ * Config where TInput is the SQLite types while TOutput can be defined by TSchema.
+ * We can use the same schema to validate TInput and incoming SQLite changes.
+ */
+export type ConfigWithSQLiteInputType<
+  TTable extends Table,
+  TSchema extends StandardSchemaV1<
+    // TInput is the SQLite types.
+    OptionalExtractedTable<TTable>,
+    AnyTableColumnType<TTable>
+  >,
+> = SerializerConfig<
+  StandardSchemaV1.InferOutput<TSchema>,
+  ExtractedTable<TTable>
+> & {
+  schema: TSchema
+}
+
+/**
+ * Config where TInput and TOutput have arbitrarily typed values.
+ * The keys of the types need to equal the SQLite types.
+ * Since TInput is not the SQLite types, we require a schema in order to deserialize incoming SQLite updates. The schema should validate from SQLite to TOutput.
+ */
+export type ConfigWithArbitraryCollectionTypes<
+  TTable extends Table,
+  TSchema extends StandardSchemaV1<
+    // The input and output must have the same keys, the value types can be arbitrary
+    AnyTableColumnType<TTable>,
+    AnyTableColumnType<TTable>
+  >,
+> = SerializerConfig<
+  StandardSchemaV1.InferOutput<TSchema>,
+  ExtractedTable<TTable>
+> & {
+  schema: TSchema
+  /**
+   * Schema for deserializing and validating input data from the sync stream.
+   *
+   * This schema defines how to transform and validate data coming from SQLite types (as stored in the database)
+   * into the desired output types (`TOutput`) expected by your application or validation logic.
+   *
+   * The generic parameters allow for arbitrary input and output types, so you can specify custom conversion rules
+   * for each column. This is especially useful when your application expects richer types (e.g., Date, enums, objects)
+   * than what SQLite natively supports.
+   *
+   * Use this to ensure that incoming data from the sync stream is properly converted and validated before use.
+   *
+   * Example:
+   * ```typescript
+   * deserializationSchema: z.object({
+   *   createdAt: z.preprocess((val) => new Date(val as string), z.date()),
+   *   meta: z.preprocess((val) => JSON.parse(val as string), z.object({ ... })),
+   * })
+   * ```
+   *
+   * This enables robust type safety and validation for incoming data, bridging the gap between SQLite storage
+   * and your application's expected types.
+   */
+  deserializationSchema: StandardSchemaV1<
+    ExtractedTable<TTable>,
+    StandardSchemaV1.InferOutput<TSchema>
+  >
+}
+export type BasePowerSyncCollectionConfig<
+  TTable extends Table = Table,
+  TSchema extends StandardSchemaV1 = never,
+> = Omit<
+  BaseCollectionConfig<ExtractedTable<TTable>, string, TSchema>,
+  `onInsert` | `onUpdate` | `onDelete` | `getKey`
+> & {
+  /** The PowerSync schema Table definition */
+  table: TTable
+  /** The PowerSync database instance */
+  database: AbstractPowerSyncDatabase
+  /**
+   * The maximum number of documents to read from the SQLite table
+   * in a single batch during the initial sync between PowerSync and the
+   * in-memory TanStack DB collection.
+   *
+   * @remarks
+   * - Defaults to {@link DEFAULT_BATCH_SIZE} if not specified.
+   * - Larger values reduce the number of round trips to the storage
+   *   engine but increase memory usage per batch.
+   * - Smaller values may lower memory usage and allow earlier
+   *   streaming of initial results, at the cost of more query calls.
+   */
+  syncBatchSize?: number
+}
+
+/**
+ * Configuration interface for PowerSync collection options.
+ * @template TTable - The PowerSync table schema definition
+ * @template TSchema - The validation schema type
+ */
+/**
+ * Configuration options for creating a PowerSync collection.
+ *
+ * @example
+ * ```typescript
+ * const APP_SCHEMA = new Schema({
+ *   documents: new Table({
+ *     name: column.text,
+ *   }),
+ * })
+ *
+ * const db = new PowerSyncDatabase({
+ *   database: {
+ *     dbFilename: "test.sqlite",
+ *   },
+ *   schema: APP_SCHEMA,
+ * })
+ *
+ * const collection = createCollection(
+ *   powerSyncCollectionOptions({
+ *     database: db,
+ *     table: APP_SCHEMA.props.documents
+ *   })
+ * )
+ * ```
+ */
+export type PowerSyncCollectionConfig<
+  TTable extends Table = Table,
+  TSchema extends StandardSchemaV1<any> = never,
+> = BasePowerSyncCollectionConfig<TTable, TSchema> &
+  (
+    | ConfigWithSQLiteTypes
+    | ConfigWithSQLiteInputType<TTable, TSchema>
+    | ConfigWithArbitraryCollectionTypes<TTable, TSchema>
+  )
+
+/**
+ * Metadata for the PowerSync Collection.
+ */
+export type PowerSyncCollectionMeta<TTable extends Table = Table> = {
+  /**
+   * The SQLite table representing the collection.
+   */
+  tableName: string
+  /**
+   * The internal table used to track diffs for the collection.
+   */
+  trackedTableName: string
+
+  /**
+   * Serializes a collection value to the SQLite type
+   */
+  serializeValue: (value: any) => ExtractedTable<TTable>
+}
+
+/**
+ * A CollectionConfig which includes utilities for PowerSync.
+ */
+export type EnhancedPowerSyncCollectionConfig<
+  TTable extends Table,
+  OutputType extends Record<string, unknown> = Record<string, unknown>,
+  TSchema extends StandardSchemaV1 = never,
+> = CollectionConfig<OutputType, string, TSchema> & {
+  id?: string
+  utils: PowerSyncCollectionUtils<TTable>
+  schema?: TSchema
+}
+
+/**
+ * Collection-level utilities for PowerSync.
+ */
+export type PowerSyncCollectionUtils<TTable extends Table = Table> = {
+  getMeta: () => PowerSyncCollectionMeta<TTable>
+}
+
+/**
+ * Default value for {@link PowerSyncCollectionConfig#syncBatchSize}.
+ */
+export const DEFAULT_BATCH_SIZE = 1000