@tanstack/db 0.0.22 → 0.0.24

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@tanstack/db",
   "description": "A reactive client store for building super fast apps on sync",
-  "version": "0.0.22",
+  "version": "0.0.24",
   "dependencies": {
     "@electric-sql/d2mini": "^0.1.7",
     "@standard-schema/spec": "^1.0.0"

package/src/collection.ts

@@ -12,15 +12,17 @@ import type {
   OperationConfig,
   OptimisticChangeMessage,
   PendingMutation,
+  ResolveInsertInput,
   ResolveType,
   StandardSchema,
   Transaction as TransactionType,
+  TransactionWithMutations,
   UtilsRecord,
 } from "./types"
 import type { StandardSchemaV1 } from "@standard-schema/spec"
 
 // Store collections in memory
-export const collectionsStore = new Map<string, CollectionImpl<any, any>>()
+export const collectionsStore = new Map<string, CollectionImpl<any, any, any>>()
 
 interface PendingSyncedTransaction<T extends object = Record<string, unknown>> {
   committed: boolean
@@ -32,12 +34,15 @@ interface PendingSyncedTransaction<T extends object = Record<string, unknown>> {
  * @template T - The type of items in the collection
  * @template TKey - The type of the key for the collection
  * @template TUtils - The utilities record type
+ * @template TInsertInput - The type for insert operations (can be different from T for schemas with defaults)
  */
 export interface Collection<
   T extends object = Record<string, unknown>,
   TKey extends string | number = string | number,
   TUtils extends UtilsRecord = {},
-> extends CollectionImpl<T, TKey> {
+  TSchema extends StandardSchemaV1 = StandardSchemaV1,
+  TInsertInput extends object = T,
+> extends CollectionImpl<T, TKey, TUtils, TSchema, TInsertInput> {
   readonly utils: TUtils
 }
 
@@ -124,12 +129,22 @@ export function createCollection<
   options: CollectionConfig<
     ResolveType<TExplicit, TSchema, TFallback>,
     TKey,
-    TSchema
+    TSchema,
+    ResolveInsertInput<TExplicit, TSchema, TFallback>
   > & { utils?: TUtils }
-): Collection<ResolveType<TExplicit, TSchema, TFallback>, TKey, TUtils> {
+): Collection<
+  ResolveType<TExplicit, TSchema, TFallback>,
+  TKey,
+  TUtils,
+  TSchema,
+  ResolveInsertInput<TExplicit, TSchema, TFallback>
+> {
   const collection = new CollectionImpl<
     ResolveType<TExplicit, TSchema, TFallback>,
-    TKey
+    TKey,
+    TUtils,
+    TSchema,
+    ResolveInsertInput<TExplicit, TSchema, TFallback>
   >(options)
 
   // Copy utils to both top level and .utils namespace
@@ -142,7 +157,9 @@ export function createCollection<
   return collection as Collection<
     ResolveType<TExplicit, TSchema, TFallback>,
     TKey,
-    TUtils
+    TUtils,
+    TSchema,
+    ResolveInsertInput<TExplicit, TSchema, TFallback>
   >
 }
 
@@ -179,8 +196,10 @@ export class CollectionImpl<
   T extends object = Record<string, unknown>,
   TKey extends string | number = string | number,
   TUtils extends UtilsRecord = {},
+  TSchema extends StandardSchemaV1 = StandardSchemaV1,
+  TInsertInput extends object = T,
 > {
-  public config: CollectionConfig<T, TKey, any>
+  public config: CollectionConfig<T, TKey, TSchema, TInsertInput>
 
   // Core state - make public for testing
   public transactions: SortedMap<string, Transaction<any>>
@@ -312,7 +331,7 @@ export class CollectionImpl<
    * @param config - Configuration object for the collection
    * @throws Error if sync config is missing
    */
-  constructor(config: CollectionConfig<T, TKey, any>) {
+  constructor(config: CollectionConfig<T, TKey, TSchema, TInsertInput>) {
     // eslint-disable-next-line
     if (!config) {
       throw new Error(`Collection requires a config`)
@@ -1322,9 +1341,11 @@ export class CollectionImpl<
    *   console.log('Insert failed:', error)
    * }
    */
-  insert = (data: T | Array<T>, config?: InsertConfig) => {
+  insert = (
+    data: TInsertInput | Array<TInsertInput>,
+    config?: InsertConfig
+  ) => {
     this.validateCollectionUsable(`insert`)
-
     const ambientTransaction = getActiveTransaction()
 
     // If no ambient transaction exists, check for an onInsert handler early
@@ -1335,7 +1356,7 @@ export class CollectionImpl<
     }
 
     const items = Array.isArray(data) ? data : [data]
-    const mutations: Array<PendingMutation<T, `insert`>> = []
+    const mutations: Array<PendingMutation<T>> = []
 
     // Create mutations for each item
     items.forEach((item) => {
@@ -1343,7 +1364,7 @@ export class CollectionImpl<
       const validatedData = this.validateData(item, `insert`)
 
       // Check if an item with this ID already exists in the collection
-      const key = this.getKeyFromItem(item)
+      const key = this.getKeyFromItem(validatedData)
       if (this.has(key)) {
         throw `Cannot insert document with ID "${key}" because it already exists in the collection`
       }
@@ -1353,7 +1374,15 @@ export class CollectionImpl<
         mutationId: crypto.randomUUID(),
         original: {},
         modified: validatedData,
-        changes: validatedData,
+        // Pick the values from validatedData based on what's passed in - this is for cases
+        // where a schema has default values. The validated data has the extra default
+        // values but for changes, we just want to show the data that was actually passed in.
+        changes: Object.fromEntries(
+          Object.keys(item).map((k) => [
+            k,
+            validatedData[k as keyof typeof validatedData],
+          ])
+        ) as TInsertInput,
         globalKey,
         key,
         metadata: config?.metadata as unknown,
@@ -1381,8 +1410,12 @@ export class CollectionImpl<
       const directOpTransaction = createTransaction<T>({
         mutationFn: async (params) => {
           // Call the onInsert handler with the transaction and collection
-          return this.config.onInsert!({
-            ...params,
+          return await this.config.onInsert!({
+            transaction:
+              params.transaction as unknown as TransactionWithMutations<
+                TInsertInput,
+                `insert`
+              >,
             collection: this as unknown as Collection<T, TKey, TUtils>,
           })
         },
@@ -1526,7 +1559,7 @@ export class CollectionImpl<
     }
 
     // Create mutations for each object that has changes
-    const mutations: Array<PendingMutation<T, `update`>> = keysArray
+    const mutations: Array<PendingMutation<T, `update`, this>> = keysArray
       .map((key, index) => {
         const itemChanges = changesArray[index] // User-provided changes for this specific item
 
@@ -1581,7 +1614,7 @@ export class CollectionImpl<
           collection: this,
         }
       })
-      .filter(Boolean) as Array<PendingMutation<T, `update`>>
+      .filter(Boolean) as Array<PendingMutation<T, `update`, this>>
 
     // If no changes were made, return an empty transaction early
     if (mutations.length === 0) {
@@ -1609,7 +1642,11 @@ export class CollectionImpl<
       mutationFn: async (params) => {
         // Call the onUpdate handler with the transaction and collection
         return this.config.onUpdate!({
-          ...params,
+          transaction:
+            params.transaction as unknown as TransactionWithMutations<
+              T,
+              `update`
+            >,
           collection: this as unknown as Collection<T, TKey, TUtils>,
         })
       },
@@ -1677,7 +1714,7 @@ export class CollectionImpl<
     }
 
     const keysArray = Array.isArray(keys) ? keys : [keys]
-    const mutations: Array<PendingMutation<T, `delete`>> = []
+    const mutations: Array<PendingMutation<T, `delete`, this>> = []
 
     for (const key of keysArray) {
       if (!this.has(key)) {
@@ -1686,7 +1723,7 @@ export class CollectionImpl<
         )
       }
       const globalKey = this.generateGlobalKey(key, this.get(key)!)
-      const mutation: PendingMutation<T, `delete`> = {
+      const mutation: PendingMutation<T, `delete`, this> = {
         mutationId: crypto.randomUUID(),
         original: this.get(key)!,
         modified: this.get(key)!,
@@ -1724,7 +1761,11 @@ export class CollectionImpl<
       mutationFn: async (params) => {
         // Call the onDelete handler with the transaction and collection
         return this.config.onDelete!({
-          ...params,
+          transaction:
+            params.transaction as unknown as TransactionWithMutations<
+              T,
+              `delete`
+            >,
           collection: this as unknown as Collection<T, TKey, TUtils>,
         })
       },

package/src/local-storage.ts

@@ -393,7 +393,7 @@ export function localStorageCollectionOptions<
     // Remove items
     params.transaction.mutations.forEach((mutation) => {
       // For delete operations, mutation.original contains the full object
-      const key = config.getKey(mutation.original)
+      const key = config.getKey(mutation.original as ResolvedType)
       currentData.delete(key)
     })
 
@@ -506,7 +506,7 @@ function createLocalStorageSync<T extends object>(
   storageKey: string,
   storage: StorageApi,
   storageEventApi: StorageEventApi,
-  getKey: (item: T) => string | number,
+  _getKey: (item: T) => string | number,
   lastKnownData: Map<string | number, StoredItem<T>>
 ): SyncConfig<T> & { manualTrigger?: () => void } {
   let syncParams: Parameters<SyncConfig<T>[`sync`]>[0] | null = null

package/src/proxy.ts

@@ -461,6 +461,30 @@ export function createChangeProxy<
 
         // If the value is a function, bind it to the ptarget
         if (typeof value === `function`) {
+          // For Array methods that modify the array
+          if (Array.isArray(ptarget)) {
+            const methodName = prop.toString()
+            const modifyingMethods = new Set([
+              `pop`,
+              `push`,
+              `shift`,
+              `unshift`,
+              `splice`,
+              `sort`,
+              `reverse`,
+              `fill`,
+              `copyWithin`,
+            ])
+
+            if (modifyingMethods.has(methodName)) {
+              return function (...args: Array<unknown>) {
+                const result = value.apply(changeTracker.copy_, args)
+                markChanged(changeTracker)
+                return result
+              }
+            }
+          }
+
           // For Map and Set methods that modify the collection
           if (ptarget instanceof Map || ptarget instanceof Set) {
             const methodName = prop.toString()

package/src/query/builder/index.ts

@@ -184,6 +184,110 @@ export class BaseQueryBuilder<TContext extends Context = Context> {
     }) as any
   }
 
+  /**
+   * Perform a LEFT JOIN with another table or subquery
+   *
+   * @param source - An object with a single key-value pair where the key is the table alias and the value is a Collection or subquery
+   * @param onCallback - A function that receives table references and returns the join condition
+   * @returns A QueryBuilder with the left joined table available
+   *
+   * @example
+   * ```ts
+   * // Left join users with posts
+   * query
+   *   .from({ users: usersCollection })
+   *   .leftJoin({ posts: postsCollection }, ({users, posts}) => eq(users.id, posts.userId))
+   * ```
+   */
+  leftJoin<TSource extends Source>(
+    source: TSource,
+    onCallback: JoinOnCallback<
+      MergeContext<TContext, SchemaFromSource<TSource>>
+    >
+  ): QueryBuilder<
+    MergeContextWithJoinType<TContext, SchemaFromSource<TSource>, `left`>
+  > {
+    return this.join(source, onCallback, `left`)
+  }
+
+  /**
+   * Perform a RIGHT JOIN with another table or subquery
+   *
+   * @param source - An object with a single key-value pair where the key is the table alias and the value is a Collection or subquery
+   * @param onCallback - A function that receives table references and returns the join condition
+   * @returns A QueryBuilder with the right joined table available
+   *
+   * @example
+   * ```ts
+   * // Right join users with posts
+   * query
+   *   .from({ users: usersCollection })
+   *   .rightJoin({ posts: postsCollection }, ({users, posts}) => eq(users.id, posts.userId))
+   * ```
+   */
+  rightJoin<TSource extends Source>(
+    source: TSource,
+    onCallback: JoinOnCallback<
+      MergeContext<TContext, SchemaFromSource<TSource>>
+    >
+  ): QueryBuilder<
+    MergeContextWithJoinType<TContext, SchemaFromSource<TSource>, `right`>
+  > {
+    return this.join(source, onCallback, `right`)
+  }
+
+  /**
+   * Perform an INNER JOIN with another table or subquery
+   *
+   * @param source - An object with a single key-value pair where the key is the table alias and the value is a Collection or subquery
+   * @param onCallback - A function that receives table references and returns the join condition
+   * @returns A QueryBuilder with the inner joined table available
+   *
+   * @example
+   * ```ts
+   * // Inner join users with posts
+   * query
+   *   .from({ users: usersCollection })
+   *   .innerJoin({ posts: postsCollection }, ({users, posts}) => eq(users.id, posts.userId))
+   * ```
+   */
+  innerJoin<TSource extends Source>(
+    source: TSource,
+    onCallback: JoinOnCallback<
+      MergeContext<TContext, SchemaFromSource<TSource>>
+    >
+  ): QueryBuilder<
+    MergeContextWithJoinType<TContext, SchemaFromSource<TSource>, `inner`>
+  > {
+    return this.join(source, onCallback, `inner`)
+  }
+
+  /**
+   * Perform a FULL JOIN with another table or subquery
+   *
+   * @param source - An object with a single key-value pair where the key is the table alias and the value is a Collection or subquery
+   * @param onCallback - A function that receives table references and returns the join condition
+   * @returns A QueryBuilder with the full joined table available
+   *
+   * @example
+   * ```ts
+   * // Full join users with posts
+   * query
+   *   .from({ users: usersCollection })
+   *   .fullJoin({ posts: postsCollection }, ({users, posts}) => eq(users.id, posts.userId))
+   * ```
+   */
+  fullJoin<TSource extends Source>(
+    source: TSource,
+    onCallback: JoinOnCallback<
+      MergeContext<TContext, SchemaFromSource<TSource>>
+    >
+  ): QueryBuilder<
+    MergeContextWithJoinType<TContext, SchemaFromSource<TSource>, `full`>
+  > {
+    return this.join(source, onCallback, `full`)
+  }
+
   /**
    * Filter rows based on a condition
    *

package/src/query/compiler/index.ts

@@ -1,4 +1,5 @@
 import { distinct, filter, map } from "@electric-sql/d2mini"
+import { optimizeQuery } from "../optimizer.js"
 import { compileExpression } from "./evaluators.js"
 import { processJoins } from "./joins.js"
 import { processGroupBy } from "./group-by.js"
@@ -10,30 +11,35 @@ import type {
   NamespacedAndKeyedStream,
   ResultStream,
 } from "../../types.js"
-
-/**
- * Cache for compiled subqueries to avoid duplicate compilation
- */
-type QueryCache = WeakMap<QueryIR, ResultStream>
+import type { QueryCache, QueryMapping } from "./types.js"
 
 /**
  * Compiles a query2 IR into a D2 pipeline
- * @param query The query IR to compile
+ * @param rawQuery The query IR to compile
  * @param inputs Mapping of collection names to input streams
  * @param cache Optional cache for compiled subqueries (used internally for recursion)
+ * @param queryMapping Optional mapping from optimized queries to original queries
  * @returns A stream builder representing the compiled query
  */
 export function compileQuery(
-  query: QueryIR,
+  rawQuery: QueryIR,
   inputs: Record<string, KeyedStream>,
-  cache: QueryCache = new WeakMap()
+  cache: QueryCache = new WeakMap(),
+  queryMapping: QueryMapping = new WeakMap()
 ): ResultStream {
-  // Check if this query has already been compiled
-  const cachedResult = cache.get(query)
+  // Check if the original raw query has already been compiled
+  const cachedResult = cache.get(rawQuery)
   if (cachedResult) {
     return cachedResult
   }
 
+  // Optimize the query before compilation
+  const query = optimizeQuery(rawQuery)
+
+  // Create mapping from optimized query to original for caching
+  queryMapping.set(query, rawQuery)
+  mapNestedQueries(query, rawQuery, queryMapping)
+
   // Create a copy of the inputs map to avoid modifying the original
   const allInputs = { ...inputs }
 
@@ -44,7 +50,8 @@ export function compileQuery(
   const { alias: mainTableAlias, input: mainInput } = processFrom(
     query.from,
     allInputs,
-    cache
+    cache,
+    queryMapping
   )
   tables[mainTableAlias] = mainInput
 
@@ -68,7 +75,8 @@ export function compileQuery(
       tables,
       mainTableAlias,
       allInputs,
-      cache
+      cache,
+      queryMapping
     )
   }
 
@@ -218,8 +226,8 @@ export function compileQuery(
     )
 
     const result = resultPipeline
-    // Cache the result before returning
-    cache.set(query, result)
+    // Cache the result before returning (use original query as key)
+    cache.set(rawQuery, result)
     return result
   } else if (query.limit !== undefined || query.offset !== undefined) {
     // If there's a limit or offset without orderBy, throw an error
@@ -241,8 +249,8 @@ export function compileQuery(
   )
 
   const result = resultPipeline
-  // Cache the result before returning
-  cache.set(query, result)
+  // Cache the result before returning (use original query as key)
+  cache.set(rawQuery, result)
   return result
 }
 
@@ -252,7 +260,8 @@ export function compileQuery(
 function processFrom(
   from: CollectionRef | QueryRef,
   allInputs: Record<string, KeyedStream>,
-  cache: QueryCache
+  cache: QueryCache,
+  queryMapping: QueryMapping
 ): { alias: string; input: KeyedStream } {
   switch (from.type) {
     case `collectionRef`: {
@@ -265,8 +274,16 @@ function processFrom(
       return { alias: from.alias, input }
     }
     case `queryRef`: {
+      // Find the original query for caching purposes
+      const originalQuery = queryMapping.get(from.query) || from.query
+
       // Recursively compile the sub-query with cache
-      const subQueryInput = compileQuery(from.query, allInputs, cache)
+      const subQueryInput = compileQuery(
+        originalQuery,
+        allInputs,
+        cache,
+        queryMapping
+      )
 
       // Subqueries may return [key, [value, orderByIndex]] (with ORDER BY) or [key, value] (without ORDER BY)
       // We need to extract just the value for use in parent queries
@@ -283,3 +300,53 @@ function processFrom(
       throw new Error(`Unsupported FROM type: ${(from as any).type}`)
   }
 }
+
+/**
+ * Recursively maps optimized subqueries to their original queries for proper caching.
+ * This ensures that when we encounter the same QueryRef object in different contexts,
+ * we can find the original query to check the cache.
+ */
+function mapNestedQueries(
+  optimizedQuery: QueryIR,
+  originalQuery: QueryIR,
+  queryMapping: QueryMapping
+): void {
+  // Map the FROM clause if it's a QueryRef
+  if (
+    optimizedQuery.from.type === `queryRef` &&
+    originalQuery.from.type === `queryRef`
+  ) {
+    queryMapping.set(optimizedQuery.from.query, originalQuery.from.query)
+    // Recursively map nested queries
+    mapNestedQueries(
+      optimizedQuery.from.query,
+      originalQuery.from.query,
+      queryMapping
+    )
+  }
+
+  // Map JOIN clauses if they exist
+  if (optimizedQuery.join && originalQuery.join) {
+    for (
+      let i = 0;
+      i < optimizedQuery.join.length && i < originalQuery.join.length;
+      i++
+    ) {
+      const optimizedJoin = optimizedQuery.join[i]!
+      const originalJoin = originalQuery.join[i]!
+
+      if (
+        optimizedJoin.from.type === `queryRef` &&
+        originalJoin.from.type === `queryRef`
+      ) {
+        queryMapping.set(optimizedJoin.from.query, originalJoin.from.query)
+        // Recursively map nested queries in joins
+        mapNestedQueries(
+          optimizedJoin.from.query,
+          originalJoin.from.query,
+          queryMapping
+        )
+      }
+    }
+  }
+}

package/src/query/compiler/joins.ts

@@ -7,18 +7,13 @@ import {
 import { compileExpression } from "./evaluators.js"
 import { compileQuery } from "./index.js"
 import type { IStreamBuilder, JoinType } from "@electric-sql/d2mini"
-import type { CollectionRef, JoinClause, QueryIR, QueryRef } from "../ir.js"
+import type { CollectionRef, JoinClause, QueryRef } from "../ir.js"
 import type {
   KeyedStream,
   NamespacedAndKeyedStream,
   NamespacedRow,
-  ResultStream,
 } from "../../types.js"
-
-/**
- * Cache for compiled subqueries to avoid duplicate compilation
- */
-type QueryCache = WeakMap<QueryIR, ResultStream>
+import type { QueryCache, QueryMapping } from "./types.js"
 
 /**
  * Processes all join clauses in a query
@@ -29,7 +24,8 @@ export function processJoins(
   tables: Record<string, KeyedStream>,
   mainTableAlias: string,
   allInputs: Record<string, KeyedStream>,
-  cache: QueryCache
+  cache: QueryCache,
+  queryMapping: QueryMapping
 ): NamespacedAndKeyedStream {
   let resultPipeline = pipeline
 
@@ -40,7 +36,8 @@ export function processJoins(
       tables,
       mainTableAlias,
       allInputs,
-      cache
+      cache,
+      queryMapping
     )
   }
 
@@ -56,13 +53,15 @@ function processJoin(
   tables: Record<string, KeyedStream>,
   mainTableAlias: string,
   allInputs: Record<string, KeyedStream>,
-  cache: QueryCache
+  cache: QueryCache,
+  queryMapping: QueryMapping
 ): NamespacedAndKeyedStream {
   // Get the joined table alias and input stream
   const { alias: joinedTableAlias, input: joinedInput } = processJoinSource(
     joinClause.from,
     allInputs,
-    cache
+    cache,
+    queryMapping
   )
 
   // Add the joined table to the tables map
@@ -128,7 +127,8 @@ function processJoin(
 function processJoinSource(
   from: CollectionRef | QueryRef,
   allInputs: Record<string, KeyedStream>,
-  cache: QueryCache
+  cache: QueryCache,
+  queryMapping: QueryMapping
 ): { alias: string; input: KeyedStream } {
   switch (from.type) {
     case `collectionRef`: {
@@ -141,8 +141,16 @@ function processJoinSource(
       return { alias: from.alias, input }
     }
     case `queryRef`: {
+      // Find the original query for caching purposes
+      const originalQuery = queryMapping.get(from.query) || from.query
+
       // Recursively compile the sub-query with cache
-      const subQueryInput = compileQuery(from.query, allInputs, cache)
+      const subQueryInput = compileQuery(
+        originalQuery,
+        allInputs,
+        cache,
+        queryMapping
+      )
 
       // Subqueries may return [key, [value, orderByIndex]] (with ORDER BY) or [key, value] (without ORDER BY)
       // We need to extract just the value for use in parent queries

package/src/query/compiler/types.ts

@@ -0,0 +1,12 @@
+import type { QueryIR } from "../ir.js"
+import type { ResultStream } from "../../types.js"
+
+/**
+ * Cache for compiled subqueries to avoid duplicate compilation
+ */
+export type QueryCache = WeakMap<QueryIR, ResultStream>
+
+/**
+ * Mapping from optimized queries back to their original queries for caching
+ */
+export type QueryMapping = WeakMap<QueryIR, QueryIR>