@tanstack/db 0.0.6 → 0.0.8

package/src/index.ts

@@ -9,4 +9,4 @@ export * from "./proxy"
 export * from "./query/index.js"
 
 // Re-export some stuff explicitly to ensure the type & value is exported
-export { Collection } from "./collection"
+export type { Collection } from "./collection"

package/src/proxy.ts

@@ -642,7 +642,7 @@ export function createChangeProxy<
         return value
       },
 
-      set(sobj, prop, value) {
+      set(_sobj, prop, value) {
         const currentValue = changeTracker.copy_[prop as keyof T]
         debugLog(
           `set called for property ${String(prop)}, current:`,
@@ -716,7 +716,7 @@ export function createChangeProxy<
         return true
       },
 
-      defineProperty(ptarget, prop, descriptor) {
+      defineProperty(_ptarget, prop, descriptor) {
         // const result = Reflect.defineProperty(
         //   changeTracker.copy_,
         //   prop,

package/src/query/compiled-query.ts

@@ -1,7 +1,7 @@
 import { D2, MessageType, MultiSet, output } from "@electric-sql/d2ts"
-import { Effect, batch } from "@tanstack/store"
-import { Collection } from "../collection.js"
+import { createCollection } from "../collection.js"
 import { compileQueryPipeline } from "./pipeline-compiler.js"
+import type { Collection } from "../collection.js"
 import type { ChangeMessage, SyncConfig } from "../types.js"
 import type {
   IStreamBuilder,
@@ -26,7 +26,7 @@ export class CompiledQuery<TResults extends object = Record<string, unknown>> {
   private resultCollection: Collection<TResults>
   public state: `compiled` | `running` | `stopped` = `compiled`
   private version = 0
-  private unsubscribeEffect?: () => void
+  private unsubscribeCallbacks: Array<() => void> = []
 
   constructor(queryBuilder: QueryBuilder<Context<Schema>>) {
     const query = queryBuilder._query
@@ -97,9 +97,9 @@ export class CompiledQuery<TResults extends object = Record<string, unknown>> {
 
     this.graph = graph
     this.inputs = inputs
-    this.resultCollection = new Collection<TResults>({
+    this.resultCollection = createCollection<TResults>({
       id: crypto.randomUUID(), // TODO: remove when we don't require any more
-      getId: (val) => {
+      getKey: (val: unknown) => {
         return (val as any)._key
       },
       sync: {
@@ -115,12 +115,12 @@ export class CompiledQuery<TResults extends object = Record<string, unknown>> {
   private sendChangesToInput(
     inputKey: string,
     changes: Array<ChangeMessage>,
-    getId: (item: ChangeMessage[`value`]) => any
+    getKey: (item: ChangeMessage[`value`]) => any
   ) {
     const input = this.inputs[inputKey]!
     const multiSetArray: MultiSetArray<unknown> = []
     for (const change of changes) {
-      const key = getId(change.value)
+      const key = getKey(change.value)
       if (change.type === `insert`) {
         multiSetArray.push([[key, change.value], 1])
       } else if (change.type === `update`) {
@@ -160,39 +160,29 @@ export class CompiledQuery<TResults extends object = Record<string, unknown>> {
       throw new Error(`Query is stopped`)
     }
 
-    batch(() => {
-      Object.entries(this.inputCollections).forEach(([key, collection]) => {
-        this.sendChangesToInput(
-          key,
-          collection.currentStateAsChanges(),
-          collection.config.getId
-        )
-      })
-      this.incrementVersion()
-      this.sendFrontierToAllInputs()
-      this.runGraph()
+    // Send initial state
+    Object.entries(this.inputCollections).forEach(([key, collection]) => {
+      this.sendChangesToInput(
+        key,
+        collection.currentStateAsChanges(),
+        collection.config.getKey
+      )
     })
+    this.incrementVersion()
+    this.sendFrontierToAllInputs()
+    this.runGraph()
+
+    // Subscribe to changes
+    Object.entries(this.inputCollections).forEach(([key, collection]) => {
+      const unsubscribe = collection.subscribeChanges((changes) => {
+        this.sendChangesToInput(key, changes, collection.config.getKey)
+        this.incrementVersion()
+        this.sendFrontierToAllInputs()
+        this.runGraph()
+      })
 
-    const changeEffect = new Effect({
-      fn: () => {
-        batch(() => {
-          Object.entries(this.inputCollections).forEach(([key, collection]) => {
-            this.sendChangesToInput(
-              key,
-              collection.derivedChanges.state,
-              collection.config.getId
-            )
-          })
-          this.incrementVersion()
-          this.sendFrontierToAllInputs()
-          this.runGraph()
-        })
-      },
-      deps: Object.values(this.inputCollections).map(
-        (collection) => collection.derivedChanges
-      ),
+      this.unsubscribeCallbacks.push(unsubscribe)
     })
-    this.unsubscribeEffect = changeEffect.mount()
 
     this.state = `running`
     return () => {
@@ -201,8 +191,8 @@ export class CompiledQuery<TResults extends object = Record<string, unknown>> {
   }
 
   stop() {
-    this.unsubscribeEffect?.()
-    this.unsubscribeEffect = undefined
+    this.unsubscribeCallbacks.forEach((unsubscribe) => unsubscribe())
+    this.unsubscribeCallbacks = []
     this.state = `stopped`
   }
 }

package/src/query/order-by.ts

@@ -13,6 +13,13 @@ import type {
   NamespacedRow,
 } from "../types"
 
+type OrderByItem = {
+  operand: ConditionOperand
+  direction: `asc` | `desc`
+}
+
+type OrderByItems = Array<OrderByItem>
+
 export function processOrderBy(
   resultPipeline: NamespacedAndKeyedStream,
   query: Query,
@@ -41,10 +48,7 @@ export function processOrderBy(
   }
 
   // Normalize orderBy to an array of objects
-  const orderByItems: Array<{
-    operand: ConditionOperand
-    direction: `asc` | `desc`
-  }> = []
+  const orderByItems: OrderByItems = []
 
   if (typeof query.orderBy === `string`) {
     // Handle string format: '@column'
@@ -84,22 +88,13 @@ export function processOrderBy(
   const valueExtractor = (namespacedRow: NamespacedRow) => {
     // For multiple orderBy columns, create a composite key
     if (orderByItems.length > 1) {
-      return orderByItems.map((item) => {
-        const val = evaluateOperandOnNamespacedRow(
+      return orderByItems.map((item) =>
+        evaluateOperandOnNamespacedRow(
           namespacedRow,
           item.operand,
           mainTableAlias
         )
-
-        // Reverse the value for 'desc' ordering
-        return item.direction === `desc` && typeof val === `number`
-          ? -val
-          : item.direction === `desc` && typeof val === `string`
-            ? String.fromCharCode(
-                ...[...val].map((c) => 0xffff - c.charCodeAt(0))
-              )
-            : val
-      })
+      )
     } else if (orderByItems.length === 1) {
       // For a single orderBy column, use the value directly
       const item = orderByItems[0]
@@ -108,66 +103,24 @@ export function processOrderBy(
         item!.operand,
         mainTableAlias
       )
-
-      // Reverse the value for 'desc' ordering
-      return item!.direction === `desc` && typeof val === `number`
-        ? -val
-        : item!.direction === `desc` && typeof val === `string`
-          ? String.fromCharCode(
-              ...[...val].map((c) => 0xffff - c.charCodeAt(0))
-            )
-          : val
+      return val
     }
 
     // Default case - no ordering
     return null
   }
 
-  const comparator = (a: unknown, b: unknown): number => {
-    // if a and b are both numbers compare them directly
-    if (typeof a === `number` && typeof b === `number`) {
-      return a - b
-    }
-    // if a and b are both strings, compare them lexicographically
+  const ascComparator = (a: any, b: any): number => {
+    // if a and b are both strings, compare them based on locale
     if (typeof a === `string` && typeof b === `string`) {
       return a.localeCompare(b)
     }
-    // if a and b are both booleans, compare them
-    if (typeof a === `boolean` && typeof b === `boolean`) {
-      return a === b ? 0 : a ? 1 : -1
-    }
-    // if a and b are both dates, compare them
-    if (a instanceof Date && b instanceof Date) {
-      return a.getTime() - b.getTime()
-    }
-    // if a and b are both null, return 0
-    if (a === null || b === null) {
-      return 0
-    }
 
     // if a and b are both arrays, compare them element by element
     if (Array.isArray(a) && Array.isArray(b)) {
       for (let i = 0; i < Math.min(a.length, b.length); i++) {
-        // Get the values from the array
-        const aVal = a[i]
-        const bVal = b[i]
-
         // Compare the values
-        let result: number
-
-        if (typeof aVal === `boolean` && typeof bVal === `boolean`) {
-          // Special handling for booleans - false comes before true
-          result = aVal === bVal ? 0 : aVal ? 1 : -1
-        } else if (typeof aVal === `number` && typeof bVal === `number`) {
-          // Numeric comparison
-          result = aVal - bVal
-        } else if (typeof aVal === `string` && typeof bVal === `string`) {
-          // String comparison
-          result = aVal.localeCompare(bVal)
-        } else {
-          // Default comparison using the general comparator
-          result = comparator(aVal, bVal)
-        }
+        const result = ascComparator(a[i], b[i])
 
         if (result !== 0) {
           return result
@@ -176,13 +129,62 @@ export function processOrderBy(
       // All elements are equal up to the minimum length
       return a.length - b.length
     }
-    // if a and b are both null/undefined, return 0
-    if (a == null && b == null) {
-      return 0
+
+    // If at least one of the values is an object then we don't really know how to meaningfully compare them
+    // therefore we turn them into strings and compare those
+    // There are 2 exceptions:
+    // 1) if both objects are dates then we can compare them
+    // 2) if either object is nullish then we can't call toString on it
+    const bothObjects = typeof a === `object` && typeof b === `object`
+    const bothDates = a instanceof Date && b instanceof Date
+    const notNull = a !== null && b !== null
+    if (bothObjects && !bothDates && notNull) {
+      // Every object should support `toString`
+      return a.toString().localeCompare(b.toString())
+    }
+
+    if (a < b) return -1
+    if (a > b) return 1
+    return 0
+  }
+
+  const descComparator = (a: unknown, b: unknown): number => {
+    return ascComparator(b, a)
+  }
+
+  // Create a multi-property comparator that respects the order and direction of each property
+  const makeComparator = (orderByProps: OrderByItems) => {
+    return (a: unknown, b: unknown) => {
+      // If we're comparing arrays (multiple properties), compare each property in order
+      if (orderByProps.length > 1) {
+        // `a` and `b` must be arrays since `orderByItems.length > 1`
+        // hence the extracted values must be arrays
+        const arrayA = a as Array<unknown>
+        const arrayB = b as Array<unknown>
+        for (let i = 0; i < orderByProps.length; i++) {
+          const direction = orderByProps[i]!.direction
+          const compareFn =
+            direction === `desc` ? descComparator : ascComparator
+          const result = compareFn(arrayA[i], arrayB[i])
+          if (result !== 0) {
+            return result
+          }
+        }
+        // should normally always be 0 because
+        // both values are extracted based on orderByItems
+        return arrayA.length - arrayB.length
+      }
+
+      // Single property comparison
+      if (orderByProps.length === 1) {
+        const direction = orderByProps[0]!.direction
+        return direction === `desc` ? descComparator(a, b) : ascComparator(a, b)
+      }
+
+      return ascComparator(a, b)
     }
-    // Fallback to string comparison for all other cases
-    return (a as any).toString().localeCompare((b as any).toString())
   }
+  const comparator = makeComparator(orderByItems)
 
   // Apply the appropriate orderBy operator based on whether an ORDER_INDEX column is requested
   if (hasOrderIndexColumn) {

package/src/query/schema.ts

@@ -197,7 +197,7 @@ export type SelectCallback<TContext extends Context = Context> = (
   context: TContext extends { schema: infer S } ? S : any
 ) => any
 
-export type As<TContext extends Context = Context> = string
+export type As<_TContext extends Context = Context> = string
 
 export type From<TContext extends Context = Context> = InputReference<{
   baseSchema: TContext[`baseSchema`]
@@ -219,9 +219,9 @@ export type GroupBy<TContext extends Context = Context> =
   | PropertyReference<TContext>
   | Array<PropertyReference<TContext>>
 
-export type Limit<TContext extends Context = Context> = number
+export type Limit<_TContext extends Context = Context> = number
 
-export type Offset<TContext extends Context = Context> = number
+export type Offset<_TContext extends Context = Context> = number
 
 export interface BaseQuery<TContext extends Context = Context> {
   // The select clause is an array of either plain strings or objects mapping alias names

package/src/transactions.ts

@@ -1,6 +1,7 @@
 import { createDeferred } from "./deferred"
 import type { Deferred } from "./deferred"
 import type {
+  MutationFn,
   PendingMutation,
   TransactionConfig,
   TransactionState,
@@ -24,8 +25,8 @@ function generateUUID() {
   })
 }
 
-const transactions: Array<Transaction> = []
-let transactionStack: Array<Transaction> = []
+const transactions: Array<Transaction<any>> = []
+let transactionStack: Array<Transaction<any>> = []
 
 export function createTransaction(config: TransactionConfig): Transaction {
   if (typeof config.mutationFn === `undefined`) {
@@ -51,27 +52,27 @@ export function getActiveTransaction(): Transaction | undefined {
   }
 }
 
-function registerTransaction(tx: Transaction) {
+function registerTransaction(tx: Transaction<any>) {
   transactionStack.push(tx)
 }
 
-function unregisterTransaction(tx: Transaction) {
+function unregisterTransaction(tx: Transaction<any>) {
   transactionStack = transactionStack.filter((t) => t.id !== tx.id)
 }
 
-function removeFromPendingList(tx: Transaction) {
+function removeFromPendingList(tx: Transaction<any>) {
   const index = transactions.findIndex((t) => t.id === tx.id)
   if (index !== -1) {
     transactions.splice(index, 1)
   }
 }
 
-export class Transaction {
+export class Transaction<T extends object = Record<string, unknown>> {
   public id: string
   public state: TransactionState
-  public mutationFn
-  public mutations: Array<PendingMutation<any>>
-  public isPersisted: Deferred<Transaction>
+  public mutationFn: MutationFn<T>
+  public mutations: Array<PendingMutation<T>>
+  public isPersisted: Deferred<Transaction<T>>
   public autoCommit: boolean
   public createdAt: Date
   public metadata: Record<string, unknown>
@@ -80,12 +81,12 @@ export class Transaction {
     error: Error
   }
 
-  constructor(config: TransactionConfig) {
+  constructor(config: TransactionConfig<T>) {
     this.id = config.id!
     this.mutationFn = config.mutationFn
     this.state = `pending`
     this.mutations = []
-    this.isPersisted = createDeferred()
+    this.isPersisted = createDeferred<Transaction<T>>()
     this.autoCommit = config.autoCommit ?? true
     this.createdAt = new Date()
     this.metadata = config.metadata ?? {}
@@ -99,7 +100,7 @@ export class Transaction {
     }
   }
 
-  mutate(callback: () => void): Transaction {
+  mutate(callback: () => void): Transaction<T> {
     if (this.state !== `pending`) {
       throw `You can no longer call .mutate() as the transaction is no longer pending`
     }
@@ -121,7 +122,7 @@ export class Transaction {
   applyMutations(mutations: Array<PendingMutation<any>>): void {
     for (const newMutation of mutations) {
       const existingIndex = this.mutations.findIndex(
-        (m) => m.key === newMutation.key
+        (m) => m.globalKey === newMutation.globalKey
       )
 
       if (existingIndex >= 0) {
@@ -134,7 +135,7 @@ export class Transaction {
     }
   }
 
-  rollback(config?: { isSecondaryRollback?: boolean }): Transaction {
+  rollback(config?: { isSecondaryRollback?: boolean }): Transaction<T> {
     const isSecondaryRollback = config?.isSecondaryRollback ?? false
     if (this.state === `completed`) {
       throw `You can no longer call .rollback() as the transaction is already completed`
@@ -146,10 +147,10 @@ export class Transaction {
     // and roll them back as well.
     if (!isSecondaryRollback) {
       const mutationIds = new Set()
-      this.mutations.forEach((m) => mutationIds.add(m.key))
+      this.mutations.forEach((m) => mutationIds.add(m.globalKey))
       for (const t of transactions) {
         t.state === `pending` &&
-          t.mutations.some((m) => mutationIds.has(m.key)) &&
+          t.mutations.some((m) => mutationIds.has(m.globalKey)) &&
           t.rollback({ isSecondaryRollback: true })
       }
     }
@@ -166,14 +167,14 @@ export class Transaction {
     const hasCalled = new Set()
     for (const mutation of this.mutations) {
       if (!hasCalled.has(mutation.collection.id)) {
-        mutation.collection.transactions.setState((state) => state)
+        mutation.collection.onTransactionStateChange()
         mutation.collection.commitPendingTransactions()
         hasCalled.add(mutation.collection.id)
       }
     }
   }
 
-  async commit(): Promise<Transaction> {
+  async commit(): Promise<Transaction<T>> {
     if (this.state !== `pending`) {
       throw `You can no longer call .commit() as the transaction is no longer pending`
     }
@@ -189,10 +190,11 @@ export class Transaction {
     // Run mutationFn
     try {
       // At this point we know there's at least one mutation
-      // Use type assertion to tell TypeScript about this guarantee
-      const transactionWithMutations =
-        this as unknown as TransactionWithMutations
-      await this.mutationFn({ transaction: transactionWithMutations })
+      // We've already verified mutations is non-empty, so this cast is safe
+      // Use a direct type assertion instead of object spreading to preserve the original type
+      await this.mutationFn({
+        transaction: this as unknown as TransactionWithMutations<T>,
+      })
 
       this.setState(`completed`)
       this.touchCollection()

package/src/types.ts

@@ -5,32 +5,50 @@ import type { Transaction } from "./transactions"
 
 export type TransactionState = `pending` | `persisting` | `completed` | `failed`
 
+/**
+ * Represents a utility function that can be attached to a collection
+ */
+export type Fn = (...args: Array<any>) => any
+
+/**
+ * A record of utility functions that can be attached to a collection
+ */
+export type UtilsRecord = Record<string, Fn>
+
 /**
  * Represents a pending mutation within a transaction
  * Contains information about the original and modified data, as well as metadata
  */
 export interface PendingMutation<T extends object = Record<string, unknown>> {
   mutationId: string
-  original: Record<string, unknown>
-  modified: Record<string, unknown>
-  changes: Record<string, unknown>
+  original: Partial<T>
+  modified: T
+  changes: Partial<T>
+  globalKey: string
   key: any
   type: OperationType
   metadata: unknown
   syncMetadata: Record<string, unknown>
   createdAt: Date
   updatedAt: Date
-  collection: Collection<T>
+  collection: Collection<T, any>
 }
 
 /**
  * Configuration options for creating a new transaction
  */
-export type MutationFnParams = {
-  transaction: Transaction
+export type MutationFnParams<T extends object = Record<string, unknown>> = {
+  transaction: TransactionWithMutations<T>
 }
 
-export type MutationFn = (params: MutationFnParams) => Promise<any>
+export type MutationFn<T extends object = Record<string, unknown>> = (
+  params: MutationFnParams<T>
+) => Promise<any>
+
+/**
+ * Represents a non-empty array (at least one element)
+ */
+export type NonEmptyArray<T> = [T, ...Array<T>]
 
 /**
  * Utility type for a Transaction with at least one mutation
@@ -38,16 +56,16 @@ export type MutationFn = (params: MutationFnParams) => Promise<any>
  */
 export type TransactionWithMutations<
   T extends object = Record<string, unknown>,
-> = Transaction & {
-  mutations: [PendingMutation<T>, ...Array<PendingMutation<T>>]
+> = Transaction<T> & {
+  mutations: NonEmptyArray<PendingMutation<T>>
 }
 
-export interface TransactionConfig {
+export interface TransactionConfig<T extends object = Record<string, unknown>> {
   /** Unique identifier for the transaction */
   id?: string
   /* If the transaction should autocommit after a mutate call or should commit be called explicitly */
   autoCommit?: boolean
-  mutationFn: MutationFn
+  mutationFn: MutationFn<T>
   /** Custom metadata to associate with the transaction */
   metadata?: Record<string, unknown>
 }
@@ -68,9 +86,12 @@ export type Row<TExtensions = never> = Record<string, Value<TExtensions>>
 
 export type OperationType = `insert` | `update` | `delete`
 
-export interface SyncConfig<T extends object = Record<string, unknown>> {
+export interface SyncConfig<
+  T extends object = Record<string, unknown>,
+  TKey extends string | number = string | number,
+> {
   sync: (params: {
-    collection: Collection<T>
+    collection: Collection<T, TKey>
     begin: () => void
     write: (message: Omit<ChangeMessage<T>, `key`>) => void
     commit: () => void
@@ -83,8 +104,11 @@ export interface SyncConfig<T extends object = Record<string, unknown>> {
   getSyncMetadata?: () => Record<string, unknown>
 }
 
-export interface ChangeMessage<T extends object = Record<string, unknown>> {
-  key: any
+export interface ChangeMessage<
+  T extends object = Record<string, unknown>,
+  TKey extends string | number = string | number,
+> {
+  key: TKey
   value: T
   previousValue?: T
   type: OperationType
@@ -124,11 +148,14 @@ export interface InsertConfig {
   metadata?: Record<string, unknown>
 }
 
-export interface CollectionConfig<T extends object = Record<string, unknown>> {
+export interface CollectionConfig<
+  T extends object = Record<string, unknown>,
+  TKey extends string | number = string | number,
+> {
   // If an id isn't passed in, a UUID will be
   // generated for it.
   id?: string
-  sync: SyncConfig<T>
+  sync: SyncConfig<T, TKey>
   schema?: StandardSchema<T>
   /**
    * Function to extract the ID from an object
@@ -137,27 +164,27 @@ export interface CollectionConfig<T extends object = Record<string, unknown>> {
    * @returns The ID string for the item
    * @example
    * // For a collection with a 'uuid' field as the primary key
-   * getId: (item) => item.uuid
+   * getKey: (item) => item.uuid
    */
-  getId: (item: T) => any
+  getKey: (item: T) => TKey
   /**
    * Optional asynchronous handler function called before an insert operation
    * @param params Object containing transaction and mutation information
    * @returns Promise resolving to any value
    */
-  onInsert?: MutationFn
+  onInsert?: MutationFn<T>
   /**
    * Optional asynchronous handler function called before an update operation
    * @param params Object containing transaction and mutation information
    * @returns Promise resolving to any value
    */
-  onUpdate?: MutationFn
+  onUpdate?: MutationFn<T>
   /**
    * Optional asynchronous handler function called before a delete operation
    * @param params Object containing transaction and mutation information
    * @returns Promise resolving to any value
    */
-  onDelete?: MutationFn
+  onDelete?: MutationFn<T>
 }
 
 export type ChangesPayload<T extends object = Record<string, unknown>> = Array<
@@ -192,3 +219,8 @@ export type KeyedNamespacedRow = [unknown, NamespacedRow]
  * a `select` clause.
  */
 export type NamespacedAndKeyedStream = IStreamBuilder<KeyedNamespacedRow>
+
+export type ChangeListener<
+  T extends object = Record<string, unknown>,
+  TKey extends string | number = string | number,
+> = (changes: Array<ChangeMessage<T, TKey>>) => void