@tanstack/db 0.5.33 → 0.6.1

package/skills/db-core/live-queries/SKILL.md

@@ -7,10 +7,14 @@ description: >
   isUndefined, and, or, not. Aggregates: count, sum, avg, min, max. String
   functions: upper, lower, length, concat, coalesce. Math: add. $selected
   namespace. createLiveQueryCollection. Derived collections. Predicate push-down.
-  Incremental view maintenance via differential dataflow (d2ts).
+  Incremental view maintenance via differential dataflow (d2ts). Virtual
+  properties ($synced, $origin, $key, $collectionId). Includes subqueries
+  for hierarchical data. toArray and concat(toArray(...)) scalar includes.
+  queryOnce for one-shot queries. createEffect for reactive side effects
+  (onEnter, onUpdate, onExit, onBatch).
 type: sub-skill
 library: db
-library_version: '0.5.30'
+library_version: '0.6.0'
 sources:
   - 'TanStack/db:docs/guides/live-queries.md'
   - 'TanStack/db:packages/db/src/query/builder/index.ts'
@@ -191,6 +195,162 @@ const activeUserPosts = createLiveQueryCollection((q) =>
 
 Create derived collections once at module scope and reuse them. Do not recreate on every render or navigation.
 
+## Virtual Properties
+
+Live query results include computed, read-only virtual properties on every row:
+
+- `$synced`: `true` when the row is confirmed by sync; `false` when it is still optimistic.
+- `$origin`: `"local"` if the last confirmed change came from this client, otherwise `"remote"`.
+- `$key`: the row key for the result.
+- `$collectionId`: the source collection ID.
+
+These props are added automatically and can be used in `where`, `select`, and `orderBy` clauses. Do not persist them back to storage.
+
+## Includes (Subqueries in Select)
+
+Embed a correlated subquery inside `select()` to produce hierarchical (nested) data. The subquery must contain a `where` with an `eq()` that correlates a parent field with a child field. Three materialization modes are available.
+
+### Collection includes (default)
+
+Return a child `Collection` on each parent row:
+
+```ts
+import { eq, createLiveQueryCollection } from '@tanstack/db'
+
+const projectsWithIssues = createLiveQueryCollection((q) =>
+  q.from({ p: projectsCollection }).select(({ p }) => ({
+    id: p.id,
+    name: p.name,
+    issues: q
+      .from({ i: issuesCollection })
+      .where(({ i }) => eq(i.projectId, p.id))
+      .select(({ i }) => ({
+        id: i.id,
+        title: i.title,
+      })),
+  })),
+)
+
+// Each row's `issues` is a live Collection
+for (const project of projectsWithIssues) {
+  console.log(project.name, project.issues.toArray)
+}
+```
+
+### Array includes with toArray()
+
+Wrap the subquery in `toArray()` to get a plain array of scalar values instead of a Collection:
+
+```ts
+import { eq, toArray, createLiveQueryCollection } from '@tanstack/db'
+
+const messagesWithParts = createLiveQueryCollection((q) =>
+  q.from({ m: messagesCollection }).select(({ m }) => ({
+    id: m.id,
+    contentParts: toArray(
+      q
+        .from({ c: chunksCollection })
+        .where(({ c }) => eq(c.messageId, m.id))
+        .orderBy(({ c }) => c.timestamp)
+        .select(({ c }) => c.text),
+    ),
+  })),
+)
+// row.contentParts is string[]
+```
+
+### Concatenated scalar with concat(toArray())
+
+Wrap `toArray()` in `concat()` to join the scalar results into a single string:
+
+```ts
+import { eq, toArray, concat, createLiveQueryCollection } from '@tanstack/db'
+
+const messagesWithContent = createLiveQueryCollection((q) =>
+  q.from({ m: messagesCollection }).select(({ m }) => ({
+    id: m.id,
+    content: concat(
+      toArray(
+        q
+          .from({ c: chunksCollection })
+          .where(({ c }) => eq(c.messageId, m.id))
+          .orderBy(({ c }) => c.timestamp)
+          .select(({ c }) => c.text),
+      ),
+    ),
+  })),
+)
+// row.content is a single concatenated string
+```
+
+### Includes rules
+
+- The subquery **must** have a `where` clause with an `eq()` correlating a parent alias with a child alias. The library extracts this automatically as the join condition.
+- `toArray()` and `concat(toArray())` require the subquery to use a **scalar** `select` (e.g., `select(({ c }) => c.text)`), not an object select.
+- Collection includes (bare subquery) require an **object** `select`.
+- Includes subqueries are compiled into the same incremental pipeline as the parent query -- they are not separate live queries.
+
+## One-Shot Queries with queryOnce
+
+For non-reactive, one-time snapshots use `queryOnce`. It creates a live query collection, preloads it, extracts the results, and cleans up automatically.
+
+```ts
+import { eq, queryOnce } from '@tanstack/db'
+
+const activeUsers = await queryOnce((q) =>
+  q
+    .from({ user: usersCollection })
+    .where(({ user }) => eq(user.active, true))
+    .select(({ user }) => ({ id: user.id, name: user.name })),
+)
+
+// With findOne — resolves to T | undefined
+const user = await queryOnce((q) =>
+  q
+    .from({ user: usersCollection })
+    .where(({ user }) => eq(user.id, userId))
+    .findOne(),
+)
+```
+
+Use `queryOnce` for scripts, loaders, data export, tests, or AI/LLM context building. For UI bindings and reactive updates, use live queries instead.
+
+## Reactive Effects (createEffect)
+
+Reactive effects respond to query result _changes_ without materializing the full result set. Effects fire callbacks when rows enter, exit, or update within a query result — like a database trigger on an arbitrary live query.
+
+```ts
+import { createEffect, eq } from '@tanstack/db'
+
+const effect = createEffect({
+  query: (q) =>
+    q
+      .from({ msg: messagesCollection })
+      .where(({ msg }) => eq(msg.role, 'user')),
+  skipInitial: true,
+  onEnter: async (event, ctx) => {
+    await processNewMessage(event.value, { signal: ctx.signal })
+  },
+  onExit: (event) => {
+    console.log('Message left result set:', event.key)
+  },
+  onError: (error, event) => {
+    console.error(`Failed to process ${event.key}:`, error)
+  },
+})
+
+// Dispose when no longer needed
+await effect.dispose()
+```
+
+| Use case                        | Approach                                              |
+| ------------------------------- | ----------------------------------------------------- |
+| Display query results in UI     | Live query collection + `useLiveQuery`                |
+| React to changes (side effects) | `createEffect` with `onEnter` / `onUpdate` / `onExit` |
+| Inspect full batch of changes   | `createEffect` with `onBatch`                         |
+
+Key options: `id` (optional), `query`, `skipInitial` (skip existing rows on init), `onEnter`, `onUpdate`, `onExit`, `onBatch`, `onError`, `onSourceError`. The `ctx.signal` aborts when the effect is disposed.
+
 ## Common Mistakes
 
 ### CRITICAL: Using === instead of eq()

package/skills/db-core/mutations-optimistic/SKILL.md

@@ -9,7 +9,7 @@ description: >
   onInsert/onUpdate/onDelete handlers. PendingMutation type. Transaction.isPersisted.
 type: sub-skill
 library: db
-library_version: '0.5.30'
+library_version: '0.6.0'
 sources:
   - 'TanStack/db:docs/guides/mutations.md'
   - 'TanStack/db:packages/db/src/transactions.ts'

package/skills/db-core/persistence/SKILL.md

@@ -0,0 +1,241 @@
+---
+name: db-core/persistence
+description: >
+  SQLite-backed persistence for TanStack DB collections. persistedCollectionOptions
+  wraps any adapter (Electric, Query, PowerSync, or local-only) with durable local
+  storage. Platform adapters: browser (WA-SQLite OPFS), React Native (op-sqlite),
+  Expo (expo-sqlite), Electron (IPC), Node (better-sqlite3), Capacitor, Tauri,
+  Cloudflare Durable Objects. Multi-tab/multi-process coordination via
+  BrowserCollectionCoordinator / ElectronCollectionCoordinator /
+  SingleProcessCoordinator. schemaVersion for migration resets. Local-only mode
+  for offline-first without a server.
+type: sub-skill
+library: db
+library_version: '0.6.0'
+sources:
+  - 'TanStack/db:packages/db-sqlite-persistence-core/src/persisted.ts'
+  - 'TanStack/db:packages/browser-db-sqlite-persistence/src/index.ts'
+  - 'TanStack/db:packages/react-native-db-sqlite-persistence/src/index.ts'
+  - 'TanStack/db:packages/expo-db-sqlite-persistence/src/index.ts'
+  - 'TanStack/db:packages/electron-db-sqlite-persistence/src/index.ts'
+  - 'TanStack/db:packages/node-db-sqlite-persistence/src/index.ts'
+  - 'TanStack/db:examples/react/offline-transactions/src/db/persisted-todos.ts'
+  - 'TanStack/db:examples/react-native/shopping-list/src/db/collections.ts'
+---
+
+This skill builds on db-core and db-core/collection-setup. Read those first.
+
+# SQLite Persistence
+
+TanStack DB persistence adds a durable SQLite-backed layer to any collection. Data survives page reloads, app restarts, and offline periods. The server remains authoritative for synced collections -- persistence provides a local cache that hydrates instantly.
+
+## Choosing a Platform Package
+
+| Platform       | Package                                                      | Create function                              |
+| -------------- | ------------------------------------------------------------ | -------------------------------------------- |
+| Browser (OPFS) | `@tanstack/browser-db-sqlite-persistence`                    | `createBrowserWASQLitePersistence`           |
+| React Native   | `@tanstack/react-native-db-sqlite-persistence`               | `createReactNativeSQLitePersistence`         |
+| Expo           | `@tanstack/expo-db-sqlite-persistence`                       | `createExpoSQLitePersistence`                |
+| Electron       | `@tanstack/electron-db-sqlite-persistence`                   | `createElectronSQLitePersistence` (renderer) |
+| Node.js        | `@tanstack/node-db-sqlite-persistence`                       | `createNodeSQLitePersistence`                |
+| Capacitor      | `@tanstack/capacitor-db-sqlite-persistence`                  | `createCapacitorSQLitePersistence`           |
+| Tauri          | `@tanstack/tauri-db-sqlite-persistence`                      | `createTauriSQLitePersistence`               |
+| Cloudflare DO  | `@tanstack/cloudflare-durable-objects-db-sqlite-persistence` | `createCloudflareDOSQLitePersistence`        |
+
+All platform packages re-export `persistedCollectionOptions` from the core.
+
+## Local-Only Persistence (No Server)
+
+For purely local data with no sync backend:
+
+```ts
+import { createCollection } from '@tanstack/react-db'
+import {
+  BrowserCollectionCoordinator,
+  createBrowserWASQLitePersistence,
+  openBrowserWASQLiteOPFSDatabase,
+  persistedCollectionOptions,
+} from '@tanstack/browser-db-sqlite-persistence'
+
+const database = await openBrowserWASQLiteOPFSDatabase({
+  databaseName: 'my-app.sqlite',
+})
+
+const coordinator = new BrowserCollectionCoordinator({
+  dbName: 'my-app',
+})
+
+const persistence = createBrowserWASQLitePersistence({
+  database,
+  coordinator,
+})
+
+const draftsCollection = createCollection(
+  persistedCollectionOptions<Draft, string>({
+    id: 'drafts',
+    getKey: (d) => d.id,
+    persistence,
+    schemaVersion: 1,
+  }),
+)
+```
+
+Local-only collections provide `collection.utils.acceptMutations()` for applying mutations directly.
+
+## Synced Persistence (Wrapping an Adapter)
+
+Spread an existing adapter's options into `persistedCollectionOptions` to add persistence on top of sync:
+
+```ts
+import { createCollection } from '@tanstack/react-db'
+import { electricCollectionOptions } from '@tanstack/electric-db-collection'
+import {
+  createReactNativeSQLitePersistence,
+  persistedCollectionOptions,
+} from '@tanstack/react-native-db-sqlite-persistence'
+
+const persistence = createReactNativeSQLitePersistence({ database })
+
+const todosCollection = createCollection(
+  persistedCollectionOptions({
+    ...electricCollectionOptions({
+      id: 'todos',
+      shapeOptions: { url: '/api/electric/todos' },
+      getKey: (item) => item.id,
+    }),
+    persistence,
+    schemaVersion: 1,
+  }),
+)
+```
+
+This works with any adapter: `electricCollectionOptions`, `queryCollectionOptions`, `powerSyncCollectionOptions`, etc. The `persistedCollectionOptions` wrapper intercepts the sync layer to persist data as it flows through.
+
+## Multi-Tab / Multi-Process Coordination
+
+Coordinators handle leader election and cross-instance communication so only one tab/process owns the database writer.
+
+| Platform                              | Coordinator                     | Mechanism                                      |
+| ------------------------------------- | ------------------------------- | ---------------------------------------------- |
+| Browser                               | `BrowserCollectionCoordinator`  | BroadcastChannel + Web Locks                   |
+| Electron                              | `ElectronCollectionCoordinator` | IPC (main holds DB, renderer accesses via RPC) |
+| Single-process (RN, Expo, Node, etc.) | `SingleProcessCoordinator`      | No-op (always leader)                          |
+
+Browser example:
+
+```ts
+import { BrowserCollectionCoordinator } from '@tanstack/browser-db-sqlite-persistence'
+
+const coordinator = new BrowserCollectionCoordinator({
+  dbName: 'my-app',
+})
+
+// Pass to persistence
+const persistence = createBrowserWASQLitePersistence({ database, coordinator })
+
+// Cleanup on shutdown
+coordinator.dispose()
+```
+
+Electron requires setup in both processes:
+
+```ts
+// Main process
+import { exposeElectronSQLitePersistence } from '@tanstack/electron-db-sqlite-persistence'
+exposeElectronSQLitePersistence({ persistence, ipcMain })
+
+// Renderer process
+import {
+  createElectronSQLitePersistence,
+  ElectronCollectionCoordinator,
+} from '@tanstack/electron-db-sqlite-persistence'
+
+const coordinator = new ElectronCollectionCoordinator({ dbName: 'my-app' })
+const persistence = createElectronSQLitePersistence({
+  ipcRenderer: window.electron.ipcRenderer,
+  coordinator,
+})
+```
+
+## Schema Versioning
+
+`schemaVersion` tracks the shape of persisted data. When the stored version doesn't match the code, the collection resets (drops and reloads from server for synced collections, or throws for local-only).
+
+```ts
+persistedCollectionOptions({
+  // ...
+  schemaVersion: 2, // bump when you change the data shape
+})
+```
+
+There is no custom migration function -- a version mismatch triggers a full reset. For synced collections this is safe because the server re-supplies the data.
+
+## Key Options
+
+| Option          | Type                             | Description                                              |
+| --------------- | -------------------------------- | -------------------------------------------------------- |
+| `persistence`   | `PersistedCollectionPersistence` | Platform adapter + coordinator                           |
+| `schemaVersion` | `number`                         | Data version (default 1). Bump on schema changes         |
+| `id`            | `string`                         | Required for local-only. Collection identifier in SQLite |
+
+## Common Mistakes
+
+### CRITICAL Using local-only persistence without an `id`
+
+Wrong:
+
+```ts
+persistedCollectionOptions({
+  getKey: (d) => d.id,
+  persistence,
+  // missing id — generates random UUID each session, data won't persist across reloads
+})
+```
+
+Correct:
+
+```ts
+persistedCollectionOptions({
+  id: 'drafts',
+  getKey: (d) => d.id,
+  persistence,
+})
+```
+
+Without an explicit `id`, the code generates a random UUID each session, so persisted data is silently abandoned on every reload. Local-only persisted collections must always provide an `id`. Synced collections derive it from the adapter config.
+
+### HIGH Forgetting the coordinator in multi-tab apps
+
+Wrong:
+
+```ts
+const persistence = createBrowserWASQLitePersistence({ database })
+// No coordinator — concurrent tabs corrupt the database
+```
+
+Correct:
+
+```ts
+const coordinator = new BrowserCollectionCoordinator({ dbName: 'my-app' })
+const persistence = createBrowserWASQLitePersistence({ database, coordinator })
+```
+
+Without a coordinator, multiple browser tabs write to SQLite concurrently, causing data corruption. Always use `BrowserCollectionCoordinator` in browser environments.
+
+### HIGH Not bumping schemaVersion after changing data shape
+
+If you add, remove, or rename fields in your collection type but keep the same `schemaVersion`, the persisted SQLite data will have the old shape. For synced collections, bump the version to trigger a reset and re-sync.
+
+### MEDIUM Not disposing the coordinator on cleanup
+
+```ts
+// On app shutdown or hot module reload
+coordinator.dispose()
+await database.close?.()
+```
+
+Failing to dispose leaks BroadcastChannel subscriptions and Web Lock handles.
+
+See also: db-core/collection-setup/SKILL.md — for adapter selection and collection configuration.
+
+See also: offline/SKILL.md — for offline transaction queueing (complements persistence).

package/skills/meta-framework/SKILL.md

@@ -9,7 +9,7 @@ description: >
   loader APIs.
 type: composition
 library: db
-library_version: '0.5.30'
+library_version: '0.6.0'
 requires:
   - db-core
   - db-core/collection-setup

package/src/collection/change-events.ts

@@ -22,6 +22,7 @@ import type {
 import type { CollectionImpl } from './index.js'
 import type { SingleRowRefProxy } from '../query/builder/ref-proxy'
 import type { BasicExpression, OrderBy } from '../query/ir.js'
+import type { WithVirtualProps } from '../virtual-props.js'
 
 /**
  * Returns the current state of the collection as an array of changes
@@ -58,14 +59,14 @@ export function currentStateAsChanges<
   T extends object,
   TKey extends string | number,
 >(
-  collection: CollectionLike<T, TKey>,
+  collection: CollectionLike<WithVirtualProps<T, TKey>, TKey>,
   options: CurrentStateAsChangesOptions = {},
-): Array<ChangeMessage<T>> | void {
+): Array<ChangeMessage<WithVirtualProps<T, TKey>, TKey>> | void {
   // Helper function to collect filtered results
   const collectFilteredResults = (
-    filterFn?: (value: T) => boolean,
-  ): Array<ChangeMessage<T>> => {
-    const result: Array<ChangeMessage<T>> = []
+    filterFn?: (value: WithVirtualProps<T, TKey>) => boolean,
+  ): Array<ChangeMessage<WithVirtualProps<T, TKey>, TKey>> => {
+    const result: Array<ChangeMessage<WithVirtualProps<T, TKey>, TKey>> = []
     for (const [key, value] of collection.entries()) {
       // If no filter function is provided, include all items
       if (filterFn?.(value) ?? true) {
@@ -106,7 +107,7 @@ export function currentStateAsChanges<
     }
 
     // Convert keys to change messages
-    const result: Array<ChangeMessage<T>> = []
+    const result: Array<ChangeMessage<WithVirtualProps<T, TKey>, TKey>> = []
     for (const key of orderedKeys) {
       const value = collection.get(key)
       if (value !== undefined) {
@@ -138,7 +139,7 @@ export function currentStateAsChanges<
 
     if (optimizationResult.canOptimize) {
       // Use index optimization
-      const result: Array<ChangeMessage<T>> = []
+      const result: Array<ChangeMessage<WithVirtualProps<T, TKey>, TKey>> = []
       for (const key of optimizationResult.matchingKeys) {
         const value = collection.get(key)
         if (value !== undefined) {
@@ -241,9 +242,12 @@ export function createFilterFunctionFromExpression<T extends object>(
  * @param options - The subscription options containing the where clause
  * @returns A filtered callback function
  */
-export function createFilteredCallback<T extends object>(
+export function createFilteredCallback<
+  T extends object,
+  TKey extends string | number = string | number,
+>(
   originalCallback: (changes: Array<ChangeMessage<T>>) => void,
-  options: SubscribeChangesOptions,
+  options: SubscribeChangesOptions<T, TKey>,
 ): (changes: Array<ChangeMessage<T>>) => void {
   const filterFn = createFilterFunctionFromExpression(options.whereExpression!)
 

package/src/collection/changes.ts

@@ -10,6 +10,8 @@ import type { CollectionLifecycleManager } from './lifecycle.js'
 import type { CollectionSyncManager } from './sync.js'
 import type { CollectionEventsManager } from './events.js'
 import type { CollectionImpl } from './index.js'
+import type { CollectionStateManager } from './state.js'
+import type { WithVirtualProps } from '../virtual-props.js'
 
 export class CollectionChangesManager<
   TOutput extends object = Record<string, unknown>,
@@ -21,6 +23,7 @@ export class CollectionChangesManager<
   private sync!: CollectionSyncManager<TOutput, TKey, TSchema, TInput>
   private events!: CollectionEventsManager
   private collection!: CollectionImpl<TOutput, TKey, any, TSchema, TInput>
+  private state!: CollectionStateManager<TOutput, TKey, TSchema, TInput>
 
   public activeSubscribersCount = 0
   public changeSubscriptions = new Set<CollectionSubscription>()
@@ -37,11 +40,13 @@ export class CollectionChangesManager<
     sync: CollectionSyncManager<TOutput, TKey, TSchema, TInput>
     events: CollectionEventsManager
     collection: CollectionImpl<TOutput, TKey, any, TSchema, TInput>
+    state: CollectionStateManager<TOutput, TKey, TSchema, TInput>
   }) {
     this.lifecycle = deps.lifecycle
     this.sync = deps.sync
     this.events = deps.events
     this.collection = deps.collection
+    this.state = deps.state
   }
 
   /**
@@ -55,6 +60,16 @@ export class CollectionChangesManager<
     }
   }
 
+  /**
+   * Enriches a change message with virtual properties ($synced, $origin, $key, $collectionId).
+   * Uses the "add-if-missing" pattern to preserve virtual properties from upstream collections.
+   */
+  private enrichChangeWithVirtualProps(
+    change: ChangeMessage<TOutput, TKey>,
+  ): ChangeMessage<WithVirtualProps<TOutput, TKey>, TKey> {
+    return this.state.enrichChangeMessage(change)
+  }
+
   /**
    * Emit events either immediately or batch them for later emission
    */
@@ -70,26 +85,32 @@ export class CollectionChangesManager<
     }
 
     // Either we're not batching, or we're forcing emission (user action or ending batch cycle)
-    let eventsToEmit = changes
+    let rawEvents = changes
 
     if (forceEmit) {
       // Force emit is used to end a batch (e.g. after a sync commit). Combine any
       // buffered optimistic events with the final changes so subscribers see the
       // whole picture, even if the sync diff is empty.
       if (this.batchedEvents.length > 0) {
-        eventsToEmit = [...this.batchedEvents, ...changes]
+        rawEvents = [...this.batchedEvents, ...changes]
       }
       this.batchedEvents = []
       this.shouldBatchEvents = false
     }
 
-    if (eventsToEmit.length === 0) {
+    if (rawEvents.length === 0) {
       return
     }
 
+    // Enrich all change messages with virtual properties
+    // This uses the "add-if-missing" pattern to preserve pass-through semantics
+    const enrichedEvents: Array<
+      ChangeMessage<WithVirtualProps<TOutput, TKey>, TKey>
+    > = rawEvents.map((change) => this.enrichChangeWithVirtualProps(change))
+
     // Emit to all listeners
     for (const subscription of this.changeSubscriptions) {
-      subscription.emitEvents(eventsToEmit)
+      subscription.emitEvents(enrichedEvents)
     }
   }
 
@@ -97,8 +118,10 @@ export class CollectionChangesManager<
    * Subscribe to changes in the collection
    */
   public subscribeChanges(
-    callback: (changes: Array<ChangeMessage<TOutput>>) => void,
-    options: SubscribeChangesOptions<TOutput> = {},
+    callback: (
+      changes: Array<ChangeMessage<WithVirtualProps<TOutput, TKey>>>,
+    ) => void,
+    options: SubscribeChangesOptions<TOutput, TKey> = {},
   ): CollectionSubscription {
     // Start sync and track subscriber
     this.addSubscriber()
@@ -113,7 +136,7 @@ export class CollectionChangesManager<
     const { where, ...opts } = options
     let whereExpression = opts.whereExpression
     if (where) {
-      const proxy = createSingleRowRefProxy<TOutput>()
+      const proxy = createSingleRowRefProxy<WithVirtualProps<TOutput, TKey>>()
       const result = where(proxy)
       whereExpression = toExpression(result)
     }