@tanstack/db 0.0.10 → 0.0.12

package/src/collection.ts

@@ -11,23 +11,16 @@ import type {
   OperationConfig,
   OptimisticChangeMessage,
   PendingMutation,
+  ResolveType,
   StandardSchema,
   Transaction as TransactionType,
   UtilsRecord,
 } from "./types"
+import type { StandardSchemaV1 } from "@standard-schema/spec"
 
 // Store collections in memory
 export const collectionsStore = new Map<string, CollectionImpl<any, any>>()
 
-// Map to track loading collections
-const loadingCollectionResolvers = new Map<
-  string,
-  {
-    promise: Promise<CollectionImpl<any, any>>
-    resolve: (value: CollectionImpl<any, any>) => void
-  }
->()
-
 interface PendingSyncedTransaction<T extends object = Record<string, unknown>> {
   committed: boolean
   operations: Array<OptimisticChangeMessage<T>>
@@ -36,6 +29,7 @@ interface PendingSyncedTransaction<T extends object = Record<string, unknown>> {
 /**
  * Enhanced Collection interface that includes both data type T and utilities TUtils
  * @template T - The type of items in the collection
+ * @template TKey - The type of the key for the collection
  * @template TUtils - The utilities record type
  */
 export interface Collection<
@@ -49,20 +43,53 @@ export interface Collection<
 /**
  * Creates a new Collection instance with the given configuration
  *
- * @template T - The type of items in the collection
+ * @template TExplicit - The explicit type of items in the collection (highest priority)
  * @template TKey - The type of the key for the collection
  * @template TUtils - The utilities record type
+ * @template TSchema - The schema type for validation and type inference (second priority)
+ * @template TFallback - The fallback type if no explicit or schema type is provided
  * @param options - Collection options with optional utilities
  * @returns A new Collection with utilities exposed both at top level and under .utils
+ *
+ * @example
+ * // Using explicit type
+ * const todos = createCollection<Todo>({
+ *   getKey: (todo) => todo.id,
+ *   sync: { sync: () => {} }
+ * })
+ *
+ * // Using schema for type inference (preferred as it also gives you client side validation)
+ * const todoSchema = z.object({
+ *   id: z.string(),
+ *   title: z.string(),
+ *   completed: z.boolean()
+ * })
+ *
+ * const todos = createCollection({
+ *   schema: todoSchema,
+ *   getKey: (todo) => todo.id,
+ *   sync: { sync: () => {} }
+ * })
+ *
+ * // Note: You must provide either an explicit type or a schema, but not both
  */
 export function createCollection<
-  T extends object = Record<string, unknown>,
+  TExplicit = unknown,
   TKey extends string | number = string | number,
   TUtils extends UtilsRecord = {},
+  TSchema extends StandardSchemaV1 = StandardSchemaV1,
+  TFallback extends object = Record<string, unknown>,
 >(
-  options: CollectionConfig<T, TKey> & { utils?: TUtils }
-): Collection<T, TKey, TUtils> {
-  const collection = new CollectionImpl<T, TKey>(options)
+  options: CollectionConfig<
+    ResolveType<TExplicit, TSchema, TFallback>,
+    TKey,
+    TSchema
+  > & { utils?: TUtils }
+): Collection<ResolveType<TExplicit, TSchema, TFallback>, TKey, TUtils> {
+  const collection = new CollectionImpl<
+    ResolveType<TExplicit, TSchema, TFallback>,
+    TKey
+  >(options)
 
   // Copy utils to both top level and .utils namespace
   if (options.utils) {
@@ -71,98 +98,11 @@ export function createCollection<
     collection.utils = {} as TUtils
   }
 
-  return collection as Collection<T, TKey, TUtils>
-}
-
-/**
- * Preloads a collection with the given configuration
- * Returns a promise that resolves once the sync tool has done its first commit (initial sync is finished)
- * If the collection has already loaded, it resolves immediately
- *
- * This function is useful in route loaders or similar pre-rendering scenarios where you want
- * to ensure data is available before a route transition completes. It uses the same shared collection
- * instance that will be used by useCollection, ensuring data consistency.
- *
- * @example
- * ```typescript
- * // In a route loader
- * async function loader({ params }) {
- *   await preloadCollection({
- *     id: `users-${params.userId}`,
- *     sync: { ... },
- *   });
- *
- *   return null;
- * }
- * ```
- *
- * @template T - The type of items in the collection
- * @param config - Configuration for the collection, including id and sync
- * @returns Promise that resolves when the initial sync is finished
- */
-export function preloadCollection<
-  T extends object = Record<string, unknown>,
-  TKey extends string | number = string | number,
->(config: CollectionConfig<T, TKey>): Promise<CollectionImpl<T, TKey>> {
-  if (!config.id) {
-    throw new Error(`The id property is required for preloadCollection`)
-  }
-
-  // If the collection is already fully loaded, return a resolved promise
-  if (
-    collectionsStore.has(config.id) &&
-    !loadingCollectionResolvers.has(config.id)
-  ) {
-    return Promise.resolve(
-      collectionsStore.get(config.id)! as CollectionImpl<T, TKey>
-    )
-  }
-
-  // If the collection is in the process of loading, return its promise
-  if (loadingCollectionResolvers.has(config.id)) {
-    return loadingCollectionResolvers.get(config.id)!.promise
-  }
-
-  // Create a new collection instance if it doesn't exist
-  if (!collectionsStore.has(config.id)) {
-    collectionsStore.set(
-      config.id,
-      createCollection<T, TKey>({
-        id: config.id,
-        getKey: config.getKey,
-        sync: config.sync,
-        schema: config.schema,
-      })
-    )
-  }
-
-  const collection = collectionsStore.get(config.id)! as CollectionImpl<T, TKey>
-
-  // Create a promise that will resolve after the first commit
-  let resolveFirstCommit: (value: CollectionImpl<T, TKey>) => void
-  const firstCommitPromise = new Promise<CollectionImpl<T, TKey>>((resolve) => {
-    resolveFirstCommit = resolve
-  })
-
-  // Store the loading promise first
-  loadingCollectionResolvers.set(config.id, {
-    promise: firstCommitPromise,
-    resolve: resolveFirstCommit!,
-  })
-
-  // Register a one-time listener for the first commit
-  collection.onFirstCommit(() => {
-    if (!config.id) {
-      throw new Error(`The id property is required for preloadCollection`)
-    }
-    if (loadingCollectionResolvers.has(config.id)) {
-      const resolver = loadingCollectionResolvers.get(config.id)!
-      loadingCollectionResolvers.delete(config.id)
-      resolver.resolve(collection)
-    }
-  })
-
-  return firstCommitPromise
+  return collection as Collection<
+    ResolveType<TExplicit, TSchema, TFallback>,
+    TKey,
+    TUtils
+  >
 }
 
 /**
@@ -184,8 +124,8 @@ export class SchemaValidationError extends Error {
     message?: string
   ) {
     const defaultMessage = `${type === `insert` ? `Insert` : `Update`} validation failed: ${issues
-      .map((issue) => issue.message)
-      .join(`, `)}`
+      .map((issue) => `\n- ${issue.message} - path: ${issue.path}`)
+      .join(``)}`
 
     super(message || defaultMessage)
     this.name = `SchemaValidationError`
@@ -221,7 +161,7 @@ export class CollectionImpl<
 
   private pendingSyncedTransactions: Array<PendingSyncedTransaction<T>> = []
   private syncedKeys = new Set<TKey>()
-  public config: CollectionConfig<T, TKey>
+  public config: CollectionConfig<T, TKey, any>
   private hasReceivedFirstCommit = false
 
   // Array to store one-time commit listeners
@@ -244,7 +184,7 @@ export class CollectionImpl<
    * @param config - Configuration object for the collection
    * @throws Error if sync config is missing
    */
-  constructor(config: CollectionConfig<T, TKey>) {
+  constructor(config: CollectionConfig<T, TKey, any>) {
     // eslint-disable-next-line
     if (!config) {
       throw new Error(`Collection requires a config`)
@@ -803,7 +743,7 @@ export class CollectionImpl<
     }
 
     const items = Array.isArray(data) ? data : [data]
-    const mutations: Array<PendingMutation<T>> = []
+    const mutations: Array<PendingMutation<T, `insert`>> = []
 
     // Create mutations for each item
     items.forEach((item) => {
@@ -817,7 +757,7 @@ export class CollectionImpl<
       }
       const globalKey = this.generateGlobalKey(key, item)
 
-      const mutation: PendingMutation<T> = {
+      const mutation: PendingMutation<T, `insert`> = {
         mutationId: crypto.randomUUID(),
         original: {},
         modified: validatedData,
@@ -987,7 +927,7 @@ export class CollectionImpl<
     }
 
     // Create mutations for each object that has changes
-    const mutations: Array<PendingMutation<T>> = keysArray
+    const mutations: Array<PendingMutation<T, `update`>> = keysArray
       .map((key, index) => {
         const itemChanges = changesArray[index] // User-provided changes for this specific item
 
@@ -1025,9 +965,9 @@ export class CollectionImpl<
 
         return {
           mutationId: crypto.randomUUID(),
-          original: originalItem as Record<string, unknown>,
-          modified: modifiedItem as Record<string, unknown>,
-          changes: validatedUpdatePayload as Record<string, unknown>,
+          original: originalItem,
+          modified: modifiedItem,
+          changes: validatedUpdatePayload as Partial<T>,
           globalKey,
           key,
           metadata: config.metadata as unknown,
@@ -1041,7 +981,7 @@ export class CollectionImpl<
           collection: this,
         }
       })
-      .filter(Boolean) as Array<PendingMutation<T>>
+      .filter(Boolean) as Array<PendingMutation<T, `update`>>
 
     // If no changes were made, return an empty transaction early
     if (mutations.length === 0) {
@@ -1117,15 +1057,20 @@ export class CollectionImpl<
     }
 
     const keysArray = Array.isArray(keys) ? keys : [keys]
-    const mutations: Array<PendingMutation<T>> = []
+    const mutations: Array<PendingMutation<T, `delete`>> = []
 
     for (const key of keysArray) {
+      if (!this.has(key)) {
+        throw new Error(
+          `Collection.delete was called with key '${key}' but there is no item in the collection with this key`
+        )
+      }
       const globalKey = this.generateGlobalKey(key, this.get(key)!)
-      const mutation: PendingMutation<T> = {
+      const mutation: PendingMutation<T, `delete`> = {
         mutationId: crypto.randomUUID(),
-        original: this.get(key) || {},
+        original: this.get(key)!,
         modified: this.get(key)!,
-        changes: this.get(key) || {},
+        changes: this.get(key)!,
         globalKey,
         key,
         metadata: config?.metadata as unknown,

package/src/query/compiled-query.ts

@@ -1,13 +1,13 @@
-import { D2, MessageType, MultiSet, output } from "@electric-sql/d2ts"
+import { D2, MultiSet, output } from "@electric-sql/d2mini"
 import { createCollection } from "../collection.js"
 import { compileQueryPipeline } from "./pipeline-compiler.js"
 import type { Collection } from "../collection.js"
-import type { ChangeMessage, SyncConfig } from "../types.js"
+import type { ChangeMessage, ResolveType, SyncConfig } from "../types.js"
 import type {
   IStreamBuilder,
   MultiSetArray,
   RootStreamBuilder,
-} from "@electric-sql/d2ts"
+} from "@electric-sql/d2mini"
 import type { QueryBuilder, ResultsFromContext } from "./query-builder.js"
 import type { Context, Schema } from "./types.js"
 
@@ -25,7 +25,6 @@ export class CompiledQuery<TResults extends object = Record<string, unknown>> {
   private inputCollections: Record<string, Collection<any>>
   private resultCollection: Collection<TResults>
   public state: `compiled` | `running` | `stopped` = `compiled`
-  private version = 0
   private unsubscribeCallbacks: Array<() => void> = []
 
   constructor(queryBuilder: QueryBuilder<Context<Schema>>) {
@@ -38,58 +37,76 @@ export class CompiledQuery<TResults extends object = Record<string, unknown>> {
 
     this.inputCollections = collections
 
-    const graph = new D2({ initialFrontier: this.version })
+    const graph = new D2()
     const inputs = Object.fromEntries(
       Object.entries(collections).map(([key]) => [key, graph.newInput<any>()])
     )
 
-    const sync: SyncConfig<TResults>[`sync`] = ({ begin, write, commit }) => {
+    // Use TResults directly to ensure type compatibility
+    const sync: SyncConfig<TResults>[`sync`] = ({
+      begin,
+      write,
+      commit,
+      collection,
+    }) => {
       compileQueryPipeline<IStreamBuilder<[unknown, TResults]>>(
         query,
         inputs
       ).pipe(
-        output(({ type, data }) => {
-          if (type === MessageType.DATA) {
-            begin()
-            data.collection
-              .getInner()
-              .reduce((acc, [[key, value], multiplicity]) => {
-                const changes = acc.get(key) || {
-                  deletes: 0,
-                  inserts: 0,
-                  value,
-                }
-                if (multiplicity < 0) {
-                  changes.deletes += Math.abs(multiplicity)
-                } else if (multiplicity > 0) {
-                  changes.inserts += multiplicity
-                  changes.value = value
-                }
-                acc.set(key, changes)
-                return acc
-              }, new Map<unknown, { deletes: number; inserts: number; value: TResults }>())
-              .forEach((changes, rawKey) => {
-                const { deletes, inserts, value } = changes
-                const valueWithKey = { ...value, _key: rawKey }
-                if (inserts && !deletes) {
-                  write({
-                    value: valueWithKey,
-                    type: `insert`,
-                  })
-                } else if (inserts >= deletes) {
-                  write({
-                    value: valueWithKey,
-                    type: `update`,
-                  })
-                } else if (deletes > 0) {
-                  write({
-                    value: valueWithKey,
-                    type: `delete`,
-                  })
-                }
-              })
-            commit()
-          }
+        output((data) => {
+          begin()
+          data
+            .getInner()
+            .reduce((acc, [[key, value], multiplicity]) => {
+              const changes = acc.get(key) || {
+                deletes: 0,
+                inserts: 0,
+                value,
+              }
+              if (multiplicity < 0) {
+                changes.deletes += Math.abs(multiplicity)
+              } else if (multiplicity > 0) {
+                changes.inserts += multiplicity
+                changes.value = value
+              }
+              acc.set(key, changes)
+              return acc
+            }, new Map<unknown, { deletes: number; inserts: number; value: TResults }>())
+            .forEach((changes, rawKey) => {
+              const { deletes, inserts, value } = changes
+              const valueWithKey = { ...value, _key: rawKey }
+
+              // Simple singular insert.
+              if (inserts && deletes === 0) {
+                write({
+                  value: valueWithKey,
+                  type: `insert`,
+                })
+              } else if (
+                // Insert & update(s) (updates are a delete & insert)
+                inserts > deletes ||
+                // Just update(s) but the item is already in the collection (so
+                // was inserted previously).
+                (inserts === deletes &&
+                  collection.has(valueWithKey._key as string | number))
+              ) {
+                write({
+                  value: valueWithKey,
+                  type: `update`,
+                })
+                // Only delete is left as an option
+              } else if (deletes > 0) {
+                write({
+                  value: valueWithKey,
+                  type: `delete`,
+                })
+              } else {
+                throw new Error(
+                  `This should never happen ${JSON.stringify(changes)}`
+                )
+              }
+            })
+          commit()
         })
       )
       graph.finalize()
@@ -98,14 +115,30 @@ export class CompiledQuery<TResults extends object = Record<string, unknown>> {
     this.graph = graph
     this.inputs = inputs
     this.resultCollection = createCollection<TResults>({
-      id: crypto.randomUUID(), // TODO: remove when we don't require any more
       getKey: (val: unknown) => {
         return (val as any)._key
       },
       sync: {
-        sync,
+        sync: sync as unknown as (params: {
+          collection: Collection<
+            ResolveType<TResults, never, Record<string, unknown>>,
+            string | number,
+            {}
+          >
+          begin: () => void
+          write: (
+            message: Omit<
+              ChangeMessage<
+                ResolveType<TResults, never, Record<string, unknown>>,
+                string | number
+              >,
+              `key`
+            >
+          ) => void
+          commit: () => void
+        }) => void,
       },
-    })
+    }) as unknown as Collection<TResults, string | number, {}>
   }
 
   get results() {
@@ -131,22 +164,7 @@ export class CompiledQuery<TResults extends object = Record<string, unknown>> {
         multiSetArray.push([[key, change.value], -1])
       }
     }
-    input.sendData(this.version, new MultiSet(multiSetArray))
-  }
-
-  private sendFrontierToInput(inputKey: string) {
-    const input = this.inputs[inputKey]!
-    input.sendFrontier(this.version)
-  }
-
-  private sendFrontierToAllInputs() {
-    Object.entries(this.inputs).forEach(([key]) => {
-      this.sendFrontierToInput(key)
-    })
-  }
-
-  private incrementVersion() {
-    this.version++
+    input.sendData(new MultiSet(multiSetArray))
   }
 
   private runGraph() {
@@ -168,16 +186,12 @@ export class CompiledQuery<TResults extends object = Record<string, unknown>> {
         collection.config.getKey
       )
     })
-    this.incrementVersion()
-    this.sendFrontierToAllInputs()
     this.runGraph()
 
     // Subscribe to changes
     Object.entries(this.inputCollections).forEach(([key, collection]) => {
       const unsubscribe = collection.subscribeChanges((changes) => {
         this.sendChangesToInput(key, changes, collection.config.getKey)
-        this.incrementVersion()
-        this.sendFrontierToAllInputs()
         this.runGraph()
       })
 

package/src/query/group-by.ts

@@ -1,4 +1,4 @@
-import { groupBy, groupByOperators } from "@electric-sql/d2ts"
+import { groupBy, groupByOperators } from "@electric-sql/d2mini"
 import {
   evaluateOperandOnNamespacedRow,
   extractValueFromNamespacedRow,

package/src/query/joins.ts

@@ -3,11 +3,11 @@ import {
   filter,
   join as joinOperator,
   map,
-} from "@electric-sql/d2ts"
+} from "@electric-sql/d2mini"
 import { evaluateConditionOnNamespacedRow } from "./evaluators.js"
 import { extractJoinKey } from "./extractors.js"
 import type { Query } from "./index.js"
-import type { IStreamBuilder, JoinType } from "@electric-sql/d2ts"
+import type { IStreamBuilder, JoinType } from "@electric-sql/d2mini"
 import type {
   KeyedStream,
   NamespacedAndKeyedStream,

package/src/query/order-by.ts

@@ -3,7 +3,7 @@ import {
   orderBy,
   orderByWithFractionalIndex,
   orderByWithIndex,
-} from "@electric-sql/d2ts"
+} from "@electric-sql/d2mini"
 import { evaluateOperandOnNamespacedRow } from "./extractors"
 import { isOrderIndexFunctionCall } from "./utils"
 import type { ConditionOperand, Query } from "./schema"

package/src/query/pipeline-compiler.ts

@@ -1,11 +1,11 @@
-import { filter, map } from "@electric-sql/d2ts"
+import { filter, map } from "@electric-sql/d2mini"
 import { evaluateWhereOnNamespacedRow } from "./evaluators.js"
 import { processJoinClause } from "./joins.js"
 import { processGroupBy } from "./group-by.js"
 import { processOrderBy } from "./order-by.js"
 import { processSelect } from "./select.js"
 import type { Query } from "./schema.js"
-import type { IStreamBuilder } from "@electric-sql/d2ts"
+import type { IStreamBuilder } from "@electric-sql/d2mini"
 import type {
   InputRow,
   KeyedStream,

package/src/query/select.ts

@@ -1,4 +1,4 @@
-import { map } from "@electric-sql/d2ts"
+import { map } from "@electric-sql/d2mini"
 import {
   evaluateOperandOnNamespacedRow,
   extractValueFromNamespacedRow,

package/src/transactions.ts

@@ -8,36 +8,24 @@ import type {
   TransactionWithMutations,
 } from "./types"
 
-function generateUUID() {
-  // Check if crypto.randomUUID is available (modern browsers and Node.js 15+)
-  if (
-    typeof crypto !== `undefined` &&
-    typeof crypto.randomUUID === `function`
-  ) {
-    return crypto.randomUUID()
-  }
-
-  // Fallback implementation for older environments
-  return `xxxxxxxx-xxxx-4xxx-yxxx-xxxxxxxxxxxx`.replace(/[xy]/g, function (c) {
-    const r = (Math.random() * 16) | 0
-    const v = c === `x` ? r : (r & 0x3) | 0x8
-    return v.toString(16)
-  })
-}
-
 const transactions: Array<Transaction<any>> = []
 let transactionStack: Array<Transaction<any>> = []
 
-export function createTransaction(config: TransactionConfig): Transaction {
+export function createTransaction<
+  TData extends object = Record<string, unknown>,
+>(config: TransactionConfig<TData>): Transaction<TData> {
   if (typeof config.mutationFn === `undefined`) {
     throw `mutationFn is required when creating a transaction`
   }
 
   let transactionId = config.id
   if (!transactionId) {
-    transactionId = generateUUID()
+    transactionId = crypto.randomUUID()
   }
-  const newTransaction = new Transaction({ ...config, id: transactionId })
+  const newTransaction = new Transaction<TData>({
+    ...config,
+    id: transactionId,
+  })
 
   transactions.push(newTransaction)