@tanstack/db 0.1.6 → 0.1.8

package/dist/esm/query/live/collection-config-builder.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"collection-config-builder.js","sources":["../../../../src/query/live/collection-config-builder.ts"],"sourcesContent":["import { D2, output } from \"@tanstack/db-ivm\"\nimport { compileQuery } from \"../compiler/index.js\"\nimport { buildQuery, getQueryIR } from \"../builder/index.js\"\nimport { CollectionSubscriber } from \"./collection-subscriber.js\"\nimport type { RootStreamBuilder } from \"@tanstack/db-ivm\"\nimport type { OrderByOptimizationInfo } from \"../compiler/order-by.js\"\nimport type { Collection } from \"../../collection.js\"\nimport type {\n  CollectionConfig,\n  KeyedStream,\n  ResultStream,\n  SyncConfig,\n} from \"../../types.js\"\nimport type { Context, GetResult } from \"../builder/types.js\"\nimport type { BasicExpression, QueryIR } from \"../ir.js\"\nimport type { LazyCollectionCallbacks } from \"../compiler/joins.js\"\nimport type {\n  Changes,\n  FullSyncState,\n  LiveQueryCollectionConfig,\n  SyncState,\n} from \"./types.js\"\n\n// Global counter for auto-generated collection IDs\nlet liveQueryCollectionCounter = 0\n\nexport class CollectionConfigBuilder<\n  TContext extends Context,\n  TResult extends object = GetResult<TContext>,\n> {\n  private readonly id: string\n  readonly query: QueryIR\n  private readonly collections: Record<string, Collection<any, any, any>>\n\n  // WeakMap to store the keys of the results\n  // so that we can retrieve them in the getKey function\n  private readonly resultKeys = new WeakMap<object, unknown>()\n\n  // WeakMap to store the orderBy index for each result\n  private readonly orderByIndices = new WeakMap<object, string>()\n\n  private readonly compare?: (val1: TResult, val2: TResult) => number\n\n  private graphCache: D2 | undefined\n  private inputsCache: Record<string, RootStreamBuilder<unknown>> | undefined\n  private pipelineCache: ResultStream | undefined\n  public collectionWhereClausesCache:\n    | Map<string, BasicExpression<boolean>>\n    | undefined\n\n  // Map of collection IDs to functions that load keys for that lazy collection\n  readonly lazyCollectionsCallbacks: Record<string, LazyCollectionCallbacks> =\n    {}\n  // Set of collection IDs that are lazy collections\n  readonly lazyCollections = new Set<string>()\n  // Set of collection IDs that include an optimizable ORDER BY clause\n  readonly optimizableOrderByCollections: Record<\n    string,\n    OrderByOptimizationInfo\n  > = {}\n\n  constructor(\n    private readonly config: LiveQueryCollectionConfig<TContext, TResult>\n  ) {\n    // Generate a unique ID if not provided\n    this.id = config.id || `live-query-${++liveQueryCollectionCounter}`\n\n    this.query = buildQueryFromConfig(config)\n    this.collections = extractCollectionsFromQuery(this.query)\n\n    // Create compare function for ordering if the query has orderBy\n    if (this.query.orderBy && this.query.orderBy.length > 0) {\n      this.compare = createOrderByComparator<TResult>(this.orderByIndices)\n    }\n\n    // Compile the base pipeline once initially\n    // This is done to ensure that any errors are thrown immediately and synchronously\n    this.compileBasePipeline()\n  }\n\n  getConfig(): CollectionConfig<TResult> {\n    return {\n      id: this.id,\n      getKey:\n        this.config.getKey ||\n        ((item) => this.resultKeys.get(item) as string | number),\n      sync: this.getSyncConfig(),\n      compare: this.compare,\n      gcTime: this.config.gcTime || 5000, // 5 seconds by default for live queries\n      schema: this.config.schema,\n      onInsert: this.config.onInsert,\n      onUpdate: this.config.onUpdate,\n      onDelete: this.config.onDelete,\n      startSync: this.config.startSync,\n    }\n  }\n\n  // The callback function is called after the graph has run.\n  // This gives the callback a chance to load more data if needed,\n  // that's used to optimize orderBy operators that set a limit,\n  // in order to load some more data if we still don't have enough rows after the pipeline has run.\n  // That can happend because even though we load N rows, the pipeline might filter some of these rows out\n  // causing the orderBy operator to receive less than N rows or even no rows at all.\n  // So this callback would notice that it doesn't have enough rows and load some more.\n  // The callback returns a boolean, when it's true it's done loading data and we can mark the collection as ready.\n  maybeRunGraph(\n    config: Parameters<SyncConfig<TResult>[`sync`]>[0],\n    syncState: FullSyncState,\n    callback?: () => boolean\n  ) {\n    const { begin, commit, markReady } = config\n\n    // We only run the graph if all the collections are ready\n    if (\n      this.allCollectionsReadyOrInitialCommit() &&\n      syncState.subscribedToAllCollections\n    ) {\n      syncState.graph.run()\n      const ready = callback?.() ?? true\n      // On the initial run, we may need to do an empty commit to ensure that\n      // the collection is initialized\n      if (syncState.messagesCount === 0) {\n        begin()\n        commit()\n      }\n      // Mark the collection as ready after the first successful run\n      if (ready && this.allCollectionsReady()) {\n        markReady()\n      }\n    }\n  }\n\n  private getSyncConfig(): SyncConfig<TResult> {\n    return {\n      rowUpdateMode: `full`,\n      sync: this.syncFn.bind(this),\n    }\n  }\n\n  private syncFn(config: Parameters<SyncConfig<TResult>[`sync`]>[0]) {\n    const syncState: SyncState = {\n      messagesCount: 0,\n      subscribedToAllCollections: false,\n      unsubscribeCallbacks: new Set<() => void>(),\n    }\n\n    // Extend the pipeline such that it applies the incoming changes to the collection\n    const fullSyncState = this.extendPipelineWithChangeProcessing(\n      config,\n      syncState\n    )\n\n    const loadMoreDataCallbacks = this.subscribeToAllCollections(\n      config,\n      fullSyncState\n    )\n\n    // Initial run with callback to load more data if needed\n    this.maybeRunGraph(config, fullSyncState, loadMoreDataCallbacks)\n\n    // Return the unsubscribe function\n    return () => {\n      syncState.unsubscribeCallbacks.forEach((unsubscribe) => unsubscribe())\n\n      // Reset caches so a fresh graph/pipeline is compiled on next start\n      // This avoids reusing a finalized D2 graph across GC restarts\n      this.graphCache = undefined\n      this.inputsCache = undefined\n      this.pipelineCache = undefined\n      this.collectionWhereClausesCache = undefined\n    }\n  }\n\n  private compileBasePipeline() {\n    this.graphCache = new D2()\n    this.inputsCache = Object.fromEntries(\n      Object.entries(this.collections).map(([key]) => [\n        key,\n        this.graphCache!.newInput<any>(),\n      ])\n    )\n\n    // Compile the query and get both pipeline and collection WHERE clauses\n    const {\n      pipeline: pipelineCache,\n      collectionWhereClauses: collectionWhereClausesCache,\n    } = compileQuery(\n      this.query,\n      this.inputsCache as Record<string, KeyedStream>,\n      this.collections,\n      this.lazyCollectionsCallbacks,\n      this.lazyCollections,\n      this.optimizableOrderByCollections\n    )\n\n    this.pipelineCache = pipelineCache\n    this.collectionWhereClausesCache = collectionWhereClausesCache\n  }\n\n  private maybeCompileBasePipeline() {\n    if (!this.graphCache || !this.inputsCache || !this.pipelineCache) {\n      this.compileBasePipeline()\n    }\n    return {\n      graph: this.graphCache!,\n      inputs: this.inputsCache!,\n      pipeline: this.pipelineCache!,\n    }\n  }\n\n  private extendPipelineWithChangeProcessing(\n    config: Parameters<SyncConfig<TResult>[`sync`]>[0],\n    syncState: SyncState\n  ): FullSyncState {\n    const { begin, commit } = config\n    const { graph, inputs, pipeline } = this.maybeCompileBasePipeline()\n\n    pipeline.pipe(\n      output((data) => {\n        const messages = data.getInner()\n        syncState.messagesCount += messages.length\n\n        begin()\n        messages\n          .reduce(\n            accumulateChanges<TResult>,\n            new Map<unknown, Changes<TResult>>()\n          )\n          .forEach(this.applyChanges.bind(this, config))\n        commit()\n      })\n    )\n\n    graph.finalize()\n\n    // Extend the sync state with the graph, inputs, and pipeline\n    syncState.graph = graph\n    syncState.inputs = inputs\n    syncState.pipeline = pipeline\n\n    return syncState as FullSyncState\n  }\n\n  private applyChanges(\n    config: Parameters<SyncConfig<TResult>[`sync`]>[0],\n    changes: {\n      deletes: number\n      inserts: number\n      value: TResult\n      orderByIndex: string | undefined\n    },\n    key: unknown\n  ) {\n    const { write, collection } = config\n    const { deletes, inserts, value, orderByIndex } = changes\n\n    // Store the key of the result so that we can retrieve it in the\n    // getKey function\n    this.resultKeys.set(value, key)\n\n    // Store the orderBy index if it exists\n    if (orderByIndex !== undefined) {\n      this.orderByIndices.set(value, orderByIndex)\n    }\n\n    // Simple singular insert.\n    if (inserts && deletes === 0) {\n      write({\n        value,\n        type: `insert`,\n      })\n    } else if (\n      // Insert & update(s) (updates are a delete & insert)\n      inserts > deletes ||\n      // Just update(s) but the item is already in the collection (so\n      // was inserted previously).\n      (inserts === deletes && collection.has(key as string | number))\n    ) {\n      write({\n        value,\n        type: `update`,\n      })\n      // Only delete is left as an option\n    } else if (deletes > 0) {\n      write({\n        value,\n        type: `delete`,\n      })\n    } else {\n      throw new Error(\n        `Could not apply changes: ${JSON.stringify(changes)}. This should never happen.`\n      )\n    }\n  }\n\n  private allCollectionsReady() {\n    return Object.values(this.collections).every((collection) =>\n      collection.isReady()\n    )\n  }\n\n  private allCollectionsReadyOrInitialCommit() {\n    return Object.values(this.collections).every(\n      (collection) =>\n        collection.status === `ready` || collection.status === `initialCommit`\n    )\n  }\n\n  private subscribeToAllCollections(\n    config: Parameters<SyncConfig<TResult>[`sync`]>[0],\n    syncState: FullSyncState\n  ) {\n    const loaders = Object.entries(this.collections).map(\n      ([collectionId, collection]) => {\n        const collectionSubscriber = new CollectionSubscriber(\n          collectionId,\n          collection,\n          config,\n          syncState,\n          this\n        )\n        collectionSubscriber.subscribe()\n\n        const loadMore =\n          collectionSubscriber.loadMoreIfNeeded.bind(collectionSubscriber)\n\n        return loadMore\n      }\n    )\n\n    const loadMoreDataCallback = () => {\n      loaders.map((loader) => loader()) // .every((doneLoading) => doneLoading)\n      return true\n    }\n\n    // Mark the collections as subscribed in the sync state\n    syncState.subscribedToAllCollections = true\n\n    return loadMoreDataCallback\n  }\n}\n\nfunction buildQueryFromConfig<TContext extends Context>(\n  config: LiveQueryCollectionConfig<any, any>\n) {\n  // Build the query using the provided query builder function or instance\n  if (typeof config.query === `function`) {\n    return buildQuery<TContext>(config.query)\n  }\n  return getQueryIR(config.query)\n}\n\nfunction createOrderByComparator<T extends object>(\n  orderByIndices: WeakMap<object, string>\n) {\n  return (val1: T, val2: T): number => {\n    // Use the orderBy index stored in the WeakMap\n    const index1 = orderByIndices.get(val1)\n    const index2 = orderByIndices.get(val2)\n\n    // Compare fractional indices lexicographically\n    if (index1 && index2) {\n      if (index1 < index2) {\n        return -1\n      } else if (index1 > index2) {\n        return 1\n      } else {\n        return 0\n      }\n    }\n\n    // Fallback to no ordering if indices are missing\n    return 0\n  }\n}\n\n/**\n * Helper function to extract collections from a compiled query\n * Traverses the query IR to find all collection references\n * Maps collections by their ID (not alias) as expected by the compiler\n */\nfunction extractCollectionsFromQuery(\n  query: any\n): Record<string, Collection<any, any, any>> {\n  const collections: Record<string, any> = {}\n\n  // Helper function to recursively extract collections from a query or source\n  function extractFromSource(source: any) {\n    if (source.type === `collectionRef`) {\n      collections[source.collection.id] = source.collection\n    } else if (source.type === `queryRef`) {\n      // Recursively extract from subquery\n      extractFromQuery(source.query)\n    }\n  }\n\n  // Helper function to recursively extract collections from a query\n  function extractFromQuery(q: any) {\n    // Extract from FROM clause\n    if (q.from) {\n      extractFromSource(q.from)\n    }\n\n    // Extract from JOIN clauses\n    if (q.join && Array.isArray(q.join)) {\n      for (const joinClause of q.join) {\n        if (joinClause.from) {\n          extractFromSource(joinClause.from)\n        }\n      }\n    }\n  }\n\n  // Start extraction from the root query\n  extractFromQuery(query)\n\n  return collections\n}\n\nfunction accumulateChanges<T>(\n  acc: Map<unknown, Changes<T>>,\n  [[key, tupleData], multiplicity]: [\n    [unknown, [any, string | undefined]],\n    number,\n  ]\n) {\n  // All queries now consistently return [value, orderByIndex] format\n  // where orderByIndex is undefined for queries without ORDER BY\n  const [value, orderByIndex] = tupleData as [T, string | undefined]\n\n  const changes = acc.get(key) || {\n    deletes: 0,\n    inserts: 0,\n    value,\n    orderByIndex,\n  }\n  if (multiplicity < 0) {\n    changes.deletes += Math.abs(multiplicity)\n  } else if (multiplicity > 0) {\n    changes.inserts += multiplicity\n    changes.value = value\n    changes.orderByIndex = orderByIndex\n  }\n  acc.set(key, changes)\n  return acc\n}\n"],"names":[],"mappings":";;;;AAwBA,IAAI,6BAA6B;AAE1B,MAAM,wBAGX;AAAA,EAgCA,YACmB,QACjB;AADiB,SAAA,SAAA;AA1BnB,SAAiB,iCAAiB,QAAA;AAGlC,SAAiB,qCAAqB,QAAA;AAYtC,SAAS,2BACP,CAAA;AAEF,SAAS,sCAAsB,IAAA;AAE/B,SAAS,gCAGL,CAAA;AAMF,SAAK,KAAK,OAAO,MAAM,cAAc,EAAE,0BAA0B;AAEjE,SAAK,QAAQ,qBAAqB,MAAM;AACxC,SAAK,cAAc,4BAA4B,KAAK,KAAK;AAGzD,QAAI,KAAK,MAAM,WAAW,KAAK,MAAM,QAAQ,SAAS,GAAG;AACvD,WAAK,UAAU,wBAAiC,KAAK,cAAc;AAAA,IACrE;AAIA,SAAK,oBAAA;AAAA,EACP;AAAA,EAEA,YAAuC;AACrC,WAAO;AAAA,MACL,IAAI,KAAK;AAAA,MACT,QACE,KAAK,OAAO,WACX,CAAC,SAAS,KAAK,WAAW,IAAI,IAAI;AAAA,MACrC,MAAM,KAAK,cAAA;AAAA,MACX,SAAS,KAAK;AAAA,MACd,QAAQ,KAAK,OAAO,UAAU;AAAA;AAAA,MAC9B,QAAQ,KAAK,OAAO;AAAA,MACpB,UAAU,KAAK,OAAO;AAAA,MACtB,UAAU,KAAK,OAAO;AAAA,MACtB,UAAU,KAAK,OAAO;AAAA,MACtB,WAAW,KAAK,OAAO;AAAA,IAAA;AAAA,EAE3B;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAUA,cACE,QACA,WACA,UACA;AACA,UAAM,EAAE,OAAO,QAAQ,UAAA,IAAc;AAGrC,QACE,KAAK,wCACL,UAAU,4BACV;AACA,gBAAU,MAAM,IAAA;AAChB,YAAM,SAAQ,2CAAgB;AAG9B,UAAI,UAAU,kBAAkB,GAAG;AACjC,cAAA;AACA,eAAA;AAAA,MACF;AAEA,UAAI,SAAS,KAAK,uBAAuB;AACvC,kBAAA;AAAA,MACF;AAAA,IACF;AAAA,EACF;AAAA,EAEQ,gBAAqC;AAC3C,WAAO;AAAA,MACL,eAAe;AAAA,MACf,MAAM,KAAK,OAAO,KAAK,IAAI;AAAA,IAAA;AAAA,EAE/B;AAAA,EAEQ,OAAO,QAAoD;AACjE,UAAM,YAAuB;AAAA,MAC3B,eAAe;AAAA,MACf,4BAA4B;AAAA,MAC5B,0CAA0B,IAAA;AAAA,IAAgB;AAI5C,UAAM,gBAAgB,KAAK;AAAA,MACzB;AAAA,MACA;AAAA,IAAA;AAGF,UAAM,wBAAwB,KAAK;AAAA,MACjC;AAAA,MACA;AAAA,IAAA;AAIF,SAAK,cAAc,QAAQ,eAAe,qBAAqB;AAG/D,WAAO,MAAM;AACX,gBAAU,qBAAqB,QAAQ,CAAC,gBAAgB,aAAa;AAIrE,WAAK,aAAa;AAClB,WAAK,cAAc;AACnB,WAAK,gBAAgB;AACrB,WAAK,8BAA8B;AAAA,IACrC;AAAA,EACF;AAAA,EAEQ,sBAAsB;AAC5B,SAAK,aAAa,IAAI,GAAA;AACtB,SAAK,cAAc,OAAO;AAAA,MACxB,OAAO,QAAQ,KAAK,WAAW,EAAE,IAAI,CAAC,CAAC,GAAG,MAAM;AAAA,QAC9C;AAAA,QACA,KAAK,WAAY,SAAA;AAAA,MAAc,CAChC;AAAA,IAAA;AAIH,UAAM;AAAA,MACJ,UAAU;AAAA,MACV,wBAAwB;AAAA,IAAA,IACtB;AAAA,MACF,KAAK;AAAA,MACL,KAAK;AAAA,MACL,KAAK;AAAA,MACL,KAAK;AAAA,MACL,KAAK;AAAA,MACL,KAAK;AAAA,IAAA;AAGP,SAAK,gBAAgB;AACrB,SAAK,8BAA8B;AAAA,EACrC;AAAA,EAEQ,2BAA2B;AACjC,QAAI,CAAC,KAAK,cAAc,CAAC,KAAK,eAAe,CAAC,KAAK,eAAe;AAChE,WAAK,oBAAA;AAAA,IACP;AACA,WAAO;AAAA,MACL,OAAO,KAAK;AAAA,MACZ,QAAQ,KAAK;AAAA,MACb,UAAU,KAAK;AAAA,IAAA;AAAA,EAEnB;AAAA,EAEQ,mCACN,QACA,WACe;AACf,UAAM,EAAE,OAAO,OAAA,IAAW;AAC1B,UAAM,EAAE,OAAO,QAAQ,SAAA,IAAa,KAAK,yBAAA;AAEzC,aAAS;AAAA,MACP,OAAO,CAAC,SAAS;AACf,cAAM,WAAW,KAAK,SAAA;AACtB,kBAAU,iBAAiB,SAAS;AAEpC,cAAA;AACA,iBACG;AAAA,UACC;AAAA,8BACI,IAAA;AAAA,QAA+B,EAEpC,QAAQ,KAAK,aAAa,KAAK,MAAM,MAAM,CAAC;AAC/C,eAAA;AAAA,MACF,CAAC;AAAA,IAAA;AAGH,UAAM,SAAA;AAGN,cAAU,QAAQ;AAClB,cAAU,SAAS;AACnB,cAAU,WAAW;AAErB,WAAO;AAAA,EACT;AAAA,EAEQ,aACN,QACA,SAMA,KACA;AACA,UAAM,EAAE,OAAO,WAAA,IAAe;AAC9B,UAAM,EAAE,SAAS,SAAS,OAAO,iBAAiB;AAIlD,SAAK,WAAW,IAAI,OAAO,GAAG;AAG9B,QAAI,iBAAiB,QAAW;AAC9B,WAAK,eAAe,IAAI,OAAO,YAAY;AAAA,IAC7C;AAGA,QAAI,WAAW,YAAY,GAAG;AAC5B,YAAM;AAAA,QACJ;AAAA,QACA,MAAM;AAAA,MAAA,CACP;AAAA,IACH;AAAA;AAAA,MAEE,UAAU;AAAA;AAAA,MAGT,YAAY,WAAW,WAAW,IAAI,GAAsB;AAAA,MAC7D;AACA,YAAM;AAAA,QACJ;AAAA,QACA,MAAM;AAAA,MAAA,CACP;AAAA,IAEH,WAAW,UAAU,GAAG;AACtB,YAAM;AAAA,QACJ;AAAA,QACA,MAAM;AAAA,MAAA,CACP;AAAA,IACH,OAAO;AACL,YAAM,IAAI;AAAA,QACR,4BAA4B,KAAK,UAAU,OAAO,CAAC;AAAA,MAAA;AAAA,IAEvD;AAAA,EACF;AAAA,EAEQ,sBAAsB;AAC5B,WAAO,OAAO,OAAO,KAAK,WAAW,EAAE;AAAA,MAAM,CAAC,eAC5C,WAAW,QAAA;AAAA,IAAQ;AAAA,EAEvB;AAAA,EAEQ,qCAAqC;AAC3C,WAAO,OAAO,OAAO,KAAK,WAAW,EAAE;AAAA,MACrC,CAAC,eACC,WAAW,WAAW,WAAW,WAAW,WAAW;AAAA,IAAA;AAAA,EAE7D;AAAA,EAEQ,0BACN,QACA,WACA;AACA,UAAM,UAAU,OAAO,QAAQ,KAAK,WAAW,EAAE;AAAA,MAC/C,CAAC,CAAC,cAAc,UAAU,MAAM;AAC9B,cAAM,uBAAuB,IAAI;AAAA,UAC/B;AAAA,UACA;AAAA,UACA;AAAA,UACA;AAAA,UACA;AAAA,QAAA;AAEF,6BAAqB,UAAA;AAErB,cAAM,WACJ,qBAAqB,iBAAiB,KAAK,oBAAoB;AAEjE,eAAO;AAAA,MACT;AAAA,IAAA;AAGF,UAAM,uBAAuB,MAAM;AACjC,cAAQ,IAAI,CAAC,WAAW,OAAA,CAAQ;AAChC,aAAO;AAAA,IACT;AAGA,cAAU,6BAA6B;AAEvC,WAAO;AAAA,EACT;AACF;AAEA,SAAS,qBACP,QACA;AAEA,MAAI,OAAO,OAAO,UAAU,YAAY;AACtC,WAAO,WAAqB,OAAO,KAAK;AAAA,EAC1C;AACA,SAAO,WAAW,OAAO,KAAK;AAChC;AAEA,SAAS,wBACP,gBACA;AACA,SAAO,CAAC,MAAS,SAAoB;AAEnC,UAAM,SAAS,eAAe,IAAI,IAAI;AACtC,UAAM,SAAS,eAAe,IAAI,IAAI;AAGtC,QAAI,UAAU,QAAQ;AACpB,UAAI,SAAS,QAAQ;AACnB,eAAO;AAAA,MACT,WAAW,SAAS,QAAQ;AAC1B,eAAO;AAAA,MACT,OAAO;AACL,eAAO;AAAA,MACT;AAAA,IACF;AAGA,WAAO;AAAA,EACT;AACF;AAOA,SAAS,4BACP,OAC2C;AAC3C,QAAM,cAAmC,CAAA;AAGzC,WAAS,kBAAkB,QAAa;AACtC,QAAI,OAAO,SAAS,iBAAiB;AACnC,kBAAY,OAAO,WAAW,EAAE,IAAI,OAAO;AAAA,IAC7C,WAAW,OAAO,SAAS,YAAY;AAErC,uBAAiB,OAAO,KAAK;AAAA,IAC/B;AAAA,EACF;AAGA,WAAS,iBAAiB,GAAQ;AAEhC,QAAI,EAAE,MAAM;AACV,wBAAkB,EAAE,IAAI;AAAA,IAC1B;AAGA,QAAI,EAAE,QAAQ,MAAM,QAAQ,EAAE,IAAI,GAAG;AACnC,iBAAW,cAAc,EAAE,MAAM;AAC/B,YAAI,WAAW,MAAM;AACnB,4BAAkB,WAAW,IAAI;AAAA,QACnC;AAAA,MACF;AAAA,IACF;AAAA,EACF;AAGA,mBAAiB,KAAK;AAEtB,SAAO;AACT;AAEA,SAAS,kBACP,KACA,CAAC,CAAC,KAAK,SAAS,GAAG,YAAY,GAI/B;AAGA,QAAM,CAAC,OAAO,YAAY,IAAI;AAE9B,QAAM,UAAU,IAAI,IAAI,GAAG,KAAK;AAAA,IAC9B,SAAS;AAAA,IACT,SAAS;AAAA,IACT;AAAA,IACA;AAAA,EAAA;AAEF,MAAI,eAAe,GAAG;AACpB,YAAQ,WAAW,KAAK,IAAI,YAAY;AAAA,EAC1C,WAAW,eAAe,GAAG;AAC3B,YAAQ,WAAW;AACnB,YAAQ,QAAQ;AAChB,YAAQ,eAAe;AAAA,EACzB;AACA,MAAI,IAAI,KAAK,OAAO;AACpB,SAAO;AACT;"}

package/dist/esm/query/live/collection-subscriber.d.ts

@@ -0,0 +1,28 @@
+import { FullSyncState } from './types.js';
+import { Collection } from '../../collection.js';
+import { SyncConfig } from '../../types.js';
+import { Context, GetResult } from '../builder/types.js';
+import { CollectionConfigBuilder } from './collection-config-builder.js';
+export declare class CollectionSubscriber<TContext extends Context, TResult extends object = GetResult<TContext>> {
+    private collectionId;
+    private collection;
+    private config;
+    private syncState;
+    private collectionConfigBuilder;
+    private sentKeys;
+    private biggest;
+    constructor(collectionId: string, collection: Collection, config: Parameters<SyncConfig<TResult>[`sync`]>[0], syncState: FullSyncState, collectionConfigBuilder: CollectionConfigBuilder<TContext, TResult>);
+    subscribe(): void;
+    private subscribeToChanges;
+    private sendChangesToPipeline;
+    private sendVisibleChangesToPipeline;
+    private loadKeys;
+    private subscribeToAllChanges;
+    private subscribeToMatchingChanges;
+    private subscribeToOrderedChanges;
+    loadMoreIfNeeded(): boolean;
+    private sendChangesToPipelineWithTracking;
+    private loadNextItems;
+    private getWhereClauseFromAlias;
+    private trackSentValues;
+}

package/dist/esm/query/live/collection-subscriber.js

@@ -0,0 +1,272 @@
+import { MultiSet } from "@tanstack/db-ivm";
+import { createFilterFunctionFromExpression } from "../../change-events.js";
+import { convertToBasicExpression } from "../compiler/expressions.js";
+class CollectionSubscriber {
+  constructor(collectionId, collection, config, syncState, collectionConfigBuilder) {
+    this.collectionId = collectionId;
+    this.collection = collection;
+    this.config = config;
+    this.syncState = syncState;
+    this.collectionConfigBuilder = collectionConfigBuilder;
+    this.sentKeys = /* @__PURE__ */ new Set();
+    this.biggest = void 0;
+    this.sendVisibleChangesToPipeline = (changes, loadedInitialState) => {
+      if (loadedInitialState) {
+        return this.sendChangesToPipeline(changes);
+      }
+      const newChanges = [];
+      for (const change of changes) {
+        let newChange = change;
+        if (!this.sentKeys.has(change.key)) {
+          if (change.type === `update`) {
+            newChange = { ...change, type: `insert` };
+          } else if (change.type === `delete`) {
+            continue;
+          }
+          this.sentKeys.add(change.key);
+        }
+        newChanges.push(newChange);
+      }
+      return this.sendChangesToPipeline(newChanges);
+    };
+  }
+  subscribe() {
+    const collectionAlias = findCollectionAlias(
+      this.collectionId,
+      this.collectionConfigBuilder.query
+    );
+    const whereClause = this.getWhereClauseFromAlias(collectionAlias);
+    if (whereClause) {
+      const whereExpression = convertToBasicExpression(
+        whereClause,
+        collectionAlias
+      );
+      if (whereExpression) {
+        this.subscribeToChanges(whereExpression);
+      } else {
+        throw new Error(
+          `Failed to convert WHERE clause to collection filter for collection '${this.collectionId}'. This indicates a bug in the query optimization logic.`
+        );
+      }
+    } else {
+      this.subscribeToChanges();
+    }
+  }
+  subscribeToChanges(whereExpression) {
+    let unsubscribe;
+    if (this.collectionConfigBuilder.lazyCollections.has(this.collectionId)) {
+      unsubscribe = this.subscribeToMatchingChanges(whereExpression);
+    } else if (Object.hasOwn(
+      this.collectionConfigBuilder.optimizableOrderByCollections,
+      this.collectionId
+    )) {
+      unsubscribe = this.subscribeToOrderedChanges(whereExpression);
+    } else {
+      unsubscribe = this.subscribeToAllChanges(whereExpression);
+    }
+    this.syncState.unsubscribeCallbacks.add(unsubscribe);
+  }
+  sendChangesToPipeline(changes, callback) {
+    const input = this.syncState.inputs[this.collectionId];
+    const sentChanges = sendChangesToInput(
+      input,
+      changes,
+      this.collection.config.getKey
+    );
+    const dataLoader = sentChanges > 0 ? callback : void 0;
+    this.collectionConfigBuilder.maybeRunGraph(
+      this.config,
+      this.syncState,
+      dataLoader
+    );
+  }
+  loadKeys(keys, filterFn) {
+    for (const key of keys) {
+      if (this.sentKeys.has(key)) continue;
+      const value = this.collection.get(key);
+      if (value !== void 0 && filterFn(value)) {
+        this.sentKeys.add(key);
+        this.sendChangesToPipeline([{ type: `insert`, key, value }]);
+      }
+    }
+  }
+  subscribeToAllChanges(whereExpression) {
+    const sendChangesToPipeline = this.sendChangesToPipeline.bind(this);
+    const unsubscribe = this.collection.subscribeChanges(
+      sendChangesToPipeline,
+      {
+        includeInitialState: true,
+        ...whereExpression ? { whereExpression } : void 0
+      }
+    );
+    return unsubscribe;
+  }
+  subscribeToMatchingChanges(whereExpression) {
+    let loadedInitialState = false;
+    let sendChanges = false;
+    const sendVisibleChanges = (changes) => {
+      this.sendVisibleChangesToPipeline(
+        sendChanges ? changes : [],
+        loadedInitialState
+      );
+    };
+    const unsubscribe = this.collection.subscribeChanges(sendVisibleChanges, {
+      whereExpression
+    });
+    const filterFn = whereExpression ? createFilterFunctionFromExpression(whereExpression) : () => true;
+    const loadKs = (keys) => {
+      sendChanges = true;
+      return this.loadKeys(keys, filterFn);
+    };
+    this.collectionConfigBuilder.lazyCollectionsCallbacks[this.collectionId] = {
+      loadKeys: loadKs,
+      loadInitialState: () => {
+        if (loadedInitialState) return;
+        loadedInitialState = true;
+        sendChanges = true;
+        const changes = this.collection.currentStateAsChanges({
+          whereExpression
+        });
+        this.sendChangesToPipeline(changes);
+      }
+    };
+    return unsubscribe;
+  }
+  subscribeToOrderedChanges(whereExpression) {
+    const { offset, limit, comparator, dataNeeded } = this.collectionConfigBuilder.optimizableOrderByCollections[this.collectionId];
+    this.loadNextItems(offset + limit);
+    const sendChangesInRange = (changes) => {
+      const splittedChanges = splitUpdates(changes);
+      let filteredChanges = splittedChanges;
+      if (dataNeeded() === 0) {
+        filteredChanges = filterChangesSmallerOrEqualToMax(
+          splittedChanges,
+          comparator,
+          this.biggest
+        );
+      }
+      this.sendChangesToPipeline(
+        filteredChanges,
+        this.loadMoreIfNeeded.bind(this)
+      );
+    };
+    const unsubscribe = this.collection.subscribeChanges(sendChangesInRange, {
+      whereExpression
+    });
+    return unsubscribe;
+  }
+  // This function is called by maybeRunGraph
+  // after each iteration of the query pipeline
+  // to ensure that the orderBy operator has enough data to work with
+  loadMoreIfNeeded() {
+    const orderByInfo = this.collectionConfigBuilder.optimizableOrderByCollections[this.collectionId];
+    if (!orderByInfo) {
+      return true;
+    }
+    const { dataNeeded } = orderByInfo;
+    if (!dataNeeded) {
+      throw new Error(
+        `Missing dataNeeded callback for collection ${this.collectionId}`
+      );
+    }
+    const n = dataNeeded();
+    let noMoreNextItems = false;
+    if (n > 0) {
+      const loadedItems = this.loadNextItems(n);
+      noMoreNextItems = loadedItems === 0;
+    }
+    return n === 0 || noMoreNextItems;
+  }
+  sendChangesToPipelineWithTracking(changes) {
+    const { comparator } = this.collectionConfigBuilder.optimizableOrderByCollections[this.collectionId];
+    const trackedChanges = this.trackSentValues(changes, comparator);
+    this.sendChangesToPipeline(trackedChanges, this.loadMoreIfNeeded.bind(this));
+  }
+  // Loads the next `n` items from the collection
+  // starting from the biggest item it has sent
+  loadNextItems(n) {
+    const { valueExtractorForRawRow, index } = this.collectionConfigBuilder.optimizableOrderByCollections[this.collectionId];
+    const biggestSentRow = this.biggest;
+    const biggestSentValue = biggestSentRow ? valueExtractorForRawRow(biggestSentRow) : biggestSentRow;
+    const nextOrderedKeys = index.take(n, biggestSentValue);
+    const nextInserts = nextOrderedKeys.map((key) => {
+      return { type: `insert`, key, value: this.collection.get(key) };
+    });
+    this.sendChangesToPipelineWithTracking(nextInserts);
+    return nextInserts.length;
+  }
+  getWhereClauseFromAlias(collectionAlias) {
+    const collectionWhereClausesCache = this.collectionConfigBuilder.collectionWhereClausesCache;
+    if (collectionAlias && collectionWhereClausesCache) {
+      return collectionWhereClausesCache.get(collectionAlias);
+    }
+    return void 0;
+  }
+  *trackSentValues(changes, comparator) {
+    for (const change of changes) {
+      this.sentKeys.add(change.key);
+      if (!this.biggest) {
+        this.biggest = change.value;
+      } else if (comparator(this.biggest, change.value) < 0) {
+        this.biggest = change.value;
+      }
+      yield change;
+    }
+  }
+}
+function findCollectionAlias(collectionId, query) {
+  var _a, _b, _c, _d;
+  if (((_a = query.from) == null ? void 0 : _a.type) === `collectionRef` && ((_b = query.from.collection) == null ? void 0 : _b.id) === collectionId) {
+    return query.from.alias;
+  }
+  if (query.join) {
+    for (const joinClause of query.join) {
+      if (((_c = joinClause.from) == null ? void 0 : _c.type) === `collectionRef` && ((_d = joinClause.from.collection) == null ? void 0 : _d.id) === collectionId) {
+        return joinClause.from.alias;
+      }
+    }
+  }
+  return void 0;
+}
+function sendChangesToInput(input, changes, getKey) {
+  const multiSetArray = [];
+  for (const change of changes) {
+    const key = getKey(change.value);
+    if (change.type === `insert`) {
+      multiSetArray.push([[key, change.value], 1]);
+    } else if (change.type === `update`) {
+      multiSetArray.push([[key, change.previousValue], -1]);
+      multiSetArray.push([[key, change.value], 1]);
+    } else {
+      multiSetArray.push([[key, change.value], -1]);
+    }
+  }
+  input.sendData(new MultiSet(multiSetArray));
+  return multiSetArray.length;
+}
+function* splitUpdates(changes) {
+  for (const change of changes) {
+    if (change.type === `update`) {
+      yield { type: `delete`, key: change.key, value: change.previousValue };
+      yield { type: `insert`, key: change.key, value: change.value };
+    } else {
+      yield change;
+    }
+  }
+}
+function* filterChanges(changes, f) {
+  for (const change of changes) {
+    if (f(change)) {
+      yield change;
+    }
+  }
+}
+function* filterChangesSmallerOrEqualToMax(changes, comparator, maxValue) {
+  yield* filterChanges(changes, (change) => {
+    return !maxValue || comparator(change.value, maxValue) <= 0;
+  });
+}
+export {
+  CollectionSubscriber
+};
+//# sourceMappingURL=collection-subscriber.js.map

package/dist/esm/query/live/collection-subscriber.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"collection-subscriber.js","sources":["../../../../src/query/live/collection-subscriber.ts"],"sourcesContent":["import { MultiSet } from \"@tanstack/db-ivm\"\nimport { createFilterFunctionFromExpression } from \"../../change-events.js\"\nimport { convertToBasicExpression } from \"../compiler/expressions.js\"\nimport type { FullSyncState } from \"./types.js\"\nimport type { MultiSetArray, RootStreamBuilder } from \"@tanstack/db-ivm\"\nimport type { Collection } from \"../../collection.js\"\nimport type { ChangeMessage, SyncConfig } from \"../../types.js\"\nimport type { Context, GetResult } from \"../builder/types.js\"\nimport type { BasicExpression } from \"../ir.js\"\nimport type { CollectionConfigBuilder } from \"./collection-config-builder.js\"\n\nexport class CollectionSubscriber<\n  TContext extends Context,\n  TResult extends object = GetResult<TContext>,\n> {\n  // Keep track of the keys we've sent (needed for join and orderBy optimizations)\n  private sentKeys = new Set<string | number>()\n\n  // Keep track of the biggest value we've sent so far (needed for orderBy optimization)\n  private biggest: any = undefined\n\n  constructor(\n    private collectionId: string,\n    private collection: Collection,\n    private config: Parameters<SyncConfig<TResult>[`sync`]>[0],\n    private syncState: FullSyncState,\n    private collectionConfigBuilder: CollectionConfigBuilder<TContext, TResult>\n  ) {}\n\n  subscribe() {\n    const collectionAlias = findCollectionAlias(\n      this.collectionId,\n      this.collectionConfigBuilder.query\n    )\n    const whereClause = this.getWhereClauseFromAlias(collectionAlias)\n\n    if (whereClause) {\n      // Convert WHERE clause to BasicExpression format for collection subscription\n      const whereExpression = convertToBasicExpression(\n        whereClause,\n        collectionAlias!\n      )\n\n      if (whereExpression) {\n        // Use index optimization for this collection\n        this.subscribeToChanges(whereExpression)\n      } else {\n        // This should not happen - if we have a whereClause but can't create whereExpression,\n        // it indicates a bug in our optimization logic\n        throw new Error(\n          `Failed to convert WHERE clause to collection filter for collection '${this.collectionId}'. ` +\n            `This indicates a bug in the query optimization logic.`\n        )\n      }\n    } else {\n      // No WHERE clause for this collection, use regular subscription\n      this.subscribeToChanges()\n    }\n  }\n\n  private subscribeToChanges(whereExpression?: BasicExpression<boolean>) {\n    let unsubscribe: () => void\n    if (this.collectionConfigBuilder.lazyCollections.has(this.collectionId)) {\n      unsubscribe = this.subscribeToMatchingChanges(whereExpression)\n    } else if (\n      Object.hasOwn(\n        this.collectionConfigBuilder.optimizableOrderByCollections,\n        this.collectionId\n      )\n    ) {\n      unsubscribe = this.subscribeToOrderedChanges(whereExpression)\n    } else {\n      unsubscribe = this.subscribeToAllChanges(whereExpression)\n    }\n    this.syncState.unsubscribeCallbacks.add(unsubscribe)\n  }\n\n  private sendChangesToPipeline(\n    changes: Iterable<ChangeMessage<any, string | number>>,\n    callback?: () => boolean\n  ) {\n    const input = this.syncState.inputs[this.collectionId]!\n    const sentChanges = sendChangesToInput(\n      input,\n      changes,\n      this.collection.config.getKey\n    )\n\n    // Do not provide the callback that loads more data\n    // if there's no more data to load\n    // otherwise we end up in an infinite loop trying to load more data\n    const dataLoader = sentChanges > 0 ? callback : undefined\n\n    // We need to call `maybeRunGraph` even if there's no data to load\n    // because we need to mark the collection as ready if it's not already\n    // and that's only done in `maybeRunGraph`\n    this.collectionConfigBuilder.maybeRunGraph(\n      this.config,\n      this.syncState,\n      dataLoader\n    )\n  }\n\n  // Wraps the sendChangesToPipeline function\n  // in order to turn `update`s into `insert`s\n  // for keys that have not been sent to the pipeline yet\n  // and filter out deletes for keys that have not been sent\n  private sendVisibleChangesToPipeline = (\n    changes: Array<ChangeMessage<any, string | number>>,\n    loadedInitialState: boolean\n  ) => {\n    if (loadedInitialState) {\n      // There was no index for the join key\n      // so we loaded the initial state\n      // so we can safely assume that the pipeline has seen all keys\n      return this.sendChangesToPipeline(changes)\n    }\n\n    const newChanges = []\n    for (const change of changes) {\n      let newChange = change\n      if (!this.sentKeys.has(change.key)) {\n        if (change.type === `update`) {\n          newChange = { ...change, type: `insert` }\n        } else if (change.type === `delete`) {\n          // filter out deletes for keys that have not been sent\n          continue\n        }\n        this.sentKeys.add(change.key)\n      }\n      newChanges.push(newChange)\n    }\n\n    return this.sendChangesToPipeline(newChanges)\n  }\n\n  private loadKeys(\n    keys: Iterable<string | number>,\n    filterFn: (item: object) => boolean\n  ) {\n    for (const key of keys) {\n      // Only load the key once\n      if (this.sentKeys.has(key)) continue\n\n      const value = this.collection.get(key)\n      if (value !== undefined && filterFn(value)) {\n        this.sentKeys.add(key)\n        this.sendChangesToPipeline([{ type: `insert`, key, value }])\n      }\n    }\n  }\n\n  private subscribeToAllChanges(\n    whereExpression: BasicExpression<boolean> | undefined\n  ) {\n    const sendChangesToPipeline = this.sendChangesToPipeline.bind(this)\n    const unsubscribe = this.collection.subscribeChanges(\n      sendChangesToPipeline,\n      {\n        includeInitialState: true,\n        ...(whereExpression ? { whereExpression } : undefined),\n      }\n    )\n    return unsubscribe\n  }\n\n  private subscribeToMatchingChanges(\n    whereExpression: BasicExpression<boolean> | undefined\n  ) {\n    // Flag to indicate we have send to whole initial state of the collection\n    // to the pipeline, this is set when there are no indexes that can be used\n    // to filter the changes and so the whole state was requested from the collection\n    let loadedInitialState = false\n\n    // Flag to indicate that we have started sending changes to the pipeline.\n    // This is set to true by either the first call to `loadKeys` or when the\n    // query requests the whole initial state in `loadInitialState`.\n    // Until that point we filter out all changes from subscription to the collection.\n    let sendChanges = false\n\n    const sendVisibleChanges = (\n      changes: Array<ChangeMessage<any, string | number>>\n    ) => {\n      // We filter out changes when sendChanges is false to ensure that we don't send\n      // any changes from the live subscription until the join operator requests either\n      // the initial state or its first key. This is needed otherwise it could receive\n      // changes which are then later subsumed by the initial state (and that would\n      // lead to weird bugs due to the data being received twice).\n      this.sendVisibleChangesToPipeline(\n        sendChanges ? changes : [],\n        loadedInitialState\n      )\n    }\n\n    const unsubscribe = this.collection.subscribeChanges(sendVisibleChanges, {\n      whereExpression,\n    })\n\n    // Create a function that loads keys from the collection\n    // into the query pipeline on demand\n    const filterFn = whereExpression\n      ? createFilterFunctionFromExpression(whereExpression)\n      : () => true\n    const loadKs = (keys: Set<string | number>) => {\n      sendChanges = true\n      return this.loadKeys(keys, filterFn)\n    }\n\n    // Store the functions to load keys and load initial state in the `lazyCollectionsCallbacks` map\n    // This is used by the join operator to dynamically load matching keys from the lazy collection\n    // or to get the full initial state of the collection if there's no index for the join key\n    this.collectionConfigBuilder.lazyCollectionsCallbacks[this.collectionId] = {\n      loadKeys: loadKs,\n      loadInitialState: () => {\n        // Make sure we only load the initial state once\n        if (loadedInitialState) return\n        loadedInitialState = true\n        sendChanges = true\n\n        const changes = this.collection.currentStateAsChanges({\n          whereExpression,\n        })\n        this.sendChangesToPipeline(changes)\n      },\n    }\n    return unsubscribe\n  }\n\n  private subscribeToOrderedChanges(\n    whereExpression: BasicExpression<boolean> | undefined\n  ) {\n    const { offset, limit, comparator, dataNeeded } =\n      this.collectionConfigBuilder.optimizableOrderByCollections[\n        this.collectionId\n      ]!\n\n    // Load the first `offset + limit` values from the index\n    // i.e. the K items from the collection that fall into the requested range: [offset, offset + limit[\n    this.loadNextItems(offset + limit)\n\n    const sendChangesInRange = (\n      changes: Iterable<ChangeMessage<any, string | number>>\n    ) => {\n      // Split live updates into a delete of the old value and an insert of the new value\n      // and filter out changes that are bigger than the biggest value we've sent so far\n      // because they can't affect the topK\n      const splittedChanges = splitUpdates(changes)\n      let filteredChanges = splittedChanges\n      if (dataNeeded!() === 0) {\n        // If the topK is full [..., maxSentValue] then we do not need to send changes > maxSentValue\n        // because they can never make it into the topK.\n        // However, if the topK isn't full yet, we need to also send changes > maxSentValue\n        // because they will make it into the topK\n        filteredChanges = filterChangesSmallerOrEqualToMax(\n          splittedChanges,\n          comparator,\n          this.biggest\n        )\n      }\n      this.sendChangesToPipeline(\n        filteredChanges,\n        this.loadMoreIfNeeded.bind(this)\n      )\n    }\n\n    // Subscribe to changes and only send changes that are smaller than the biggest value we've sent so far\n    // values that are bigger don't need to be sent because they can't affect the topK\n    const unsubscribe = this.collection.subscribeChanges(sendChangesInRange, {\n      whereExpression,\n    })\n\n    return unsubscribe\n  }\n\n  // This function is called by maybeRunGraph\n  // after each iteration of the query pipeline\n  // to ensure that the orderBy operator has enough data to work with\n  loadMoreIfNeeded() {\n    const orderByInfo =\n      this.collectionConfigBuilder.optimizableOrderByCollections[\n        this.collectionId\n      ]\n\n    if (!orderByInfo) {\n      // This query has no orderBy operator\n      // so there's no data to load, just return true\n      return true\n    }\n\n    const { dataNeeded } = orderByInfo\n\n    if (!dataNeeded) {\n      // This should never happen because the topK operator should always set the size callback\n      // which in turn should lead to the orderBy operator setting the dataNeeded callback\n      throw new Error(\n        `Missing dataNeeded callback for collection ${this.collectionId}`\n      )\n    }\n\n    // `dataNeeded` probes the orderBy operator to see if it needs more data\n    // if it needs more data, it returns the number of items it needs\n    const n = dataNeeded()\n    let noMoreNextItems = false\n    if (n > 0) {\n      const loadedItems = this.loadNextItems(n)\n      noMoreNextItems = loadedItems === 0\n    }\n\n    // Indicate that we're done loading data if we didn't need to load more data\n    // or there's no more data to load\n    return n === 0 || noMoreNextItems\n  }\n\n  private sendChangesToPipelineWithTracking(\n    changes: Iterable<ChangeMessage<any, string | number>>\n  ) {\n    const { comparator } =\n      this.collectionConfigBuilder.optimizableOrderByCollections[\n        this.collectionId\n      ]!\n    const trackedChanges = this.trackSentValues(changes, comparator)\n    this.sendChangesToPipeline(trackedChanges, this.loadMoreIfNeeded.bind(this))\n  }\n\n  // Loads the next `n` items from the collection\n  // starting from the biggest item it has sent\n  private loadNextItems(n: number) {\n    const { valueExtractorForRawRow, index } =\n      this.collectionConfigBuilder.optimizableOrderByCollections[\n        this.collectionId\n      ]!\n    const biggestSentRow = this.biggest\n    const biggestSentValue = biggestSentRow\n      ? valueExtractorForRawRow(biggestSentRow)\n      : biggestSentRow\n    // Take the `n` items after the biggest sent value\n    const nextOrderedKeys = index.take(n, biggestSentValue)\n    const nextInserts: Array<ChangeMessage<any, string | number>> =\n      nextOrderedKeys.map((key) => {\n        return { type: `insert`, key, value: this.collection.get(key) }\n      })\n    this.sendChangesToPipelineWithTracking(nextInserts)\n    return nextInserts.length\n  }\n\n  private getWhereClauseFromAlias(\n    collectionAlias: string | undefined\n  ): BasicExpression<boolean> | undefined {\n    const collectionWhereClausesCache =\n      this.collectionConfigBuilder.collectionWhereClausesCache\n    if (collectionAlias && collectionWhereClausesCache) {\n      return collectionWhereClausesCache.get(collectionAlias)\n    }\n    return undefined\n  }\n\n  private *trackSentValues(\n    changes: Iterable<ChangeMessage<any, string | number>>,\n    comparator: (a: any, b: any) => number\n  ) {\n    for (const change of changes) {\n      this.sentKeys.add(change.key)\n\n      if (!this.biggest) {\n        this.biggest = change.value\n      } else if (comparator(this.biggest, change.value) < 0) {\n        this.biggest = change.value\n      }\n\n      yield change\n    }\n  }\n}\n\n/**\n * Finds the alias for a collection ID in the query\n */\nfunction findCollectionAlias(\n  collectionId: string,\n  query: any\n): string | undefined {\n  // Check FROM clause\n  if (\n    query.from?.type === `collectionRef` &&\n    query.from.collection?.id === collectionId\n  ) {\n    return query.from.alias\n  }\n\n  // Check JOIN clauses\n  if (query.join) {\n    for (const joinClause of query.join) {\n      if (\n        joinClause.from?.type === `collectionRef` &&\n        joinClause.from.collection?.id === collectionId\n      ) {\n        return joinClause.from.alias\n      }\n    }\n  }\n\n  return undefined\n}\n\n/**\n * Helper function to send changes to a D2 input stream\n */\nfunction sendChangesToInput(\n  input: RootStreamBuilder<unknown>,\n  changes: Iterable<ChangeMessage>,\n  getKey: (item: ChangeMessage[`value`]) => any\n): number {\n  const multiSetArray: MultiSetArray<unknown> = []\n  for (const change of changes) {\n    const key = getKey(change.value)\n    if (change.type === `insert`) {\n      multiSetArray.push([[key, change.value], 1])\n    } else if (change.type === `update`) {\n      multiSetArray.push([[key, change.previousValue], -1])\n      multiSetArray.push([[key, change.value], 1])\n    } else {\n      // change.type === `delete`\n      multiSetArray.push([[key, change.value], -1])\n    }\n  }\n  input.sendData(new MultiSet(multiSetArray))\n  return multiSetArray.length\n}\n\n/** Splits updates into a delete of the old value and an insert of the new value */\nfunction* splitUpdates<\n  T extends object = Record<string, unknown>,\n  TKey extends string | number = string | number,\n>(\n  changes: Iterable<ChangeMessage<T, TKey>>\n): Generator<ChangeMessage<T, TKey>> {\n  for (const change of changes) {\n    if (change.type === `update`) {\n      yield { type: `delete`, key: change.key, value: change.previousValue! }\n      yield { type: `insert`, key: change.key, value: change.value }\n    } else {\n      yield change\n    }\n  }\n}\n\nfunction* filterChanges<\n  T extends object = Record<string, unknown>,\n  TKey extends string | number = string | number,\n>(\n  changes: Iterable<ChangeMessage<T, TKey>>,\n  f: (change: ChangeMessage<T, TKey>) => boolean\n): Generator<ChangeMessage<T, TKey>> {\n  for (const change of changes) {\n    if (f(change)) {\n      yield change\n    }\n  }\n}\n\n/**\n * Filters changes to only include those that are smaller than the provided max value\n * @param changes - Iterable of changes to filter\n * @param comparator - Comparator function to use for filtering\n * @param maxValue - Range to filter changes within (range boundaries are exclusive)\n * @returns Iterable of changes that fall within the range\n */\nfunction* filterChangesSmallerOrEqualToMax<\n  T extends object = Record<string, unknown>,\n  TKey extends string | number = string | number,\n>(\n  changes: Iterable<ChangeMessage<T, TKey>>,\n  comparator: (a: any, b: any) => number,\n  maxValue: any\n): Generator<ChangeMessage<T, TKey>> {\n  yield* filterChanges(changes, (change) => {\n    return !maxValue || comparator(change.value, maxValue) <= 0\n  })\n}\n"],"names":[],"mappings":";;;AAWO,MAAM,qBAGX;AAAA,EAOA,YACU,cACA,YACA,QACA,WACA,yBACR;AALQ,SAAA,eAAA;AACA,SAAA,aAAA;AACA,SAAA,SAAA;AACA,SAAA,YAAA;AACA,SAAA,0BAAA;AAVV,SAAQ,+BAAe,IAAA;AAGvB,SAAQ,UAAe;AAwFvB,SAAQ,+BAA+B,CACrC,SACA,uBACG;AACH,UAAI,oBAAoB;AAItB,eAAO,KAAK,sBAAsB,OAAO;AAAA,MAC3C;AAEA,YAAM,aAAa,CAAA;AACnB,iBAAW,UAAU,SAAS;AAC5B,YAAI,YAAY;AAChB,YAAI,CAAC,KAAK,SAAS,IAAI,OAAO,GAAG,GAAG;AAClC,cAAI,OAAO,SAAS,UAAU;AAC5B,wBAAY,EAAE,GAAG,QAAQ,MAAM,SAAA;AAAA,UACjC,WAAW,OAAO,SAAS,UAAU;AAEnC;AAAA,UACF;AACA,eAAK,SAAS,IAAI,OAAO,GAAG;AAAA,QAC9B;AACA,mBAAW,KAAK,SAAS;AAAA,MAC3B;AAEA,aAAO,KAAK,sBAAsB,UAAU;AAAA,IAC9C;AAAA,EA3GG;AAAA,EAEH,YAAY;AACV,UAAM,kBAAkB;AAAA,MACtB,KAAK;AAAA,MACL,KAAK,wBAAwB;AAAA,IAAA;AAE/B,UAAM,cAAc,KAAK,wBAAwB,eAAe;AAEhE,QAAI,aAAa;AAEf,YAAM,kBAAkB;AAAA,QACtB;AAAA,QACA;AAAA,MAAA;AAGF,UAAI,iBAAiB;AAEnB,aAAK,mBAAmB,eAAe;AAAA,MACzC,OAAO;AAGL,cAAM,IAAI;AAAA,UACR,uEAAuE,KAAK,YAAY;AAAA,QAAA;AAAA,MAG5F;AAAA,IACF,OAAO;AAEL,WAAK,mBAAA;AAAA,IACP;AAAA,EACF;AAAA,EAEQ,mBAAmB,iBAA4C;AACrE,QAAI;AACJ,QAAI,KAAK,wBAAwB,gBAAgB,IAAI,KAAK,YAAY,GAAG;AACvE,oBAAc,KAAK,2BAA2B,eAAe;AAAA,IAC/D,WACE,OAAO;AAAA,MACL,KAAK,wBAAwB;AAAA,MAC7B,KAAK;AAAA,IAAA,GAEP;AACA,oBAAc,KAAK,0BAA0B,eAAe;AAAA,IAC9D,OAAO;AACL,oBAAc,KAAK,sBAAsB,eAAe;AAAA,IAC1D;AACA,SAAK,UAAU,qBAAqB,IAAI,WAAW;AAAA,EACrD;AAAA,EAEQ,sBACN,SACA,UACA;AACA,UAAM,QAAQ,KAAK,UAAU,OAAO,KAAK,YAAY;AACrD,UAAM,cAAc;AAAA,MAClB;AAAA,MACA;AAAA,MACA,KAAK,WAAW,OAAO;AAAA,IAAA;AAMzB,UAAM,aAAa,cAAc,IAAI,WAAW;AAKhD,SAAK,wBAAwB;AAAA,MAC3B,KAAK;AAAA,MACL,KAAK;AAAA,MACL;AAAA,IAAA;AAAA,EAEJ;AAAA,EAmCQ,SACN,MACA,UACA;AACA,eAAW,OAAO,MAAM;AAEtB,UAAI,KAAK,SAAS,IAAI,GAAG,EAAG;AAE5B,YAAM,QAAQ,KAAK,WAAW,IAAI,GAAG;AACrC,UAAI,UAAU,UAAa,SAAS,KAAK,GAAG;AAC1C,aAAK,SAAS,IAAI,GAAG;AACrB,aAAK,sBAAsB,CAAC,EAAE,MAAM,UAAU,KAAK,MAAA,CAAO,CAAC;AAAA,MAC7D;AAAA,IACF;AAAA,EACF;AAAA,EAEQ,sBACN,iBACA;AACA,UAAM,wBAAwB,KAAK,sBAAsB,KAAK,IAAI;AAClE,UAAM,cAAc,KAAK,WAAW;AAAA,MAClC;AAAA,MACA;AAAA,QACE,qBAAqB;AAAA,QACrB,GAAI,kBAAkB,EAAE,oBAAoB;AAAA,MAAA;AAAA,IAC9C;AAEF,WAAO;AAAA,EACT;AAAA,EAEQ,2BACN,iBACA;AAIA,QAAI,qBAAqB;AAMzB,QAAI,cAAc;AAElB,UAAM,qBAAqB,CACzB,YACG;AAMH,WAAK;AAAA,QACH,cAAc,UAAU,CAAA;AAAA,QACxB;AAAA,MAAA;AAAA,IAEJ;AAEA,UAAM,cAAc,KAAK,WAAW,iBAAiB,oBAAoB;AAAA,MACvE;AAAA,IAAA,CACD;AAID,UAAM,WAAW,kBACb,mCAAmC,eAAe,IAClD,MAAM;AACV,UAAM,SAAS,CAAC,SAA+B;AAC7C,oBAAc;AACd,aAAO,KAAK,SAAS,MAAM,QAAQ;AAAA,IACrC;AAKA,SAAK,wBAAwB,yBAAyB,KAAK,YAAY,IAAI;AAAA,MACzE,UAAU;AAAA,MACV,kBAAkB,MAAM;AAEtB,YAAI,mBAAoB;AACxB,6BAAqB;AACrB,sBAAc;AAEd,cAAM,UAAU,KAAK,WAAW,sBAAsB;AAAA,UACpD;AAAA,QAAA,CACD;AACD,aAAK,sBAAsB,OAAO;AAAA,MACpC;AAAA,IAAA;AAEF,WAAO;AAAA,EACT;AAAA,EAEQ,0BACN,iBACA;AACA,UAAM,EAAE,QAAQ,OAAO,YAAY,WAAA,IACjC,KAAK,wBAAwB,8BAC3B,KAAK,YACP;AAIF,SAAK,cAAc,SAAS,KAAK;AAEjC,UAAM,qBAAqB,CACzB,YACG;AAIH,YAAM,kBAAkB,aAAa,OAAO;AAC5C,UAAI,kBAAkB;AACtB,UAAI,WAAA,MAAkB,GAAG;AAKvB,0BAAkB;AAAA,UAChB;AAAA,UACA;AAAA,UACA,KAAK;AAAA,QAAA;AAAA,MAET;AACA,WAAK;AAAA,QACH;AAAA,QACA,KAAK,iBAAiB,KAAK,IAAI;AAAA,MAAA;AAAA,IAEnC;AAIA,UAAM,cAAc,KAAK,WAAW,iBAAiB,oBAAoB;AAAA,MACvE;AAAA,IAAA,CACD;AAED,WAAO;AAAA,EACT;AAAA;AAAA;AAAA;AAAA,EAKA,mBAAmB;AACjB,UAAM,cACJ,KAAK,wBAAwB,8BAC3B,KAAK,YACP;AAEF,QAAI,CAAC,aAAa;AAGhB,aAAO;AAAA,IACT;AAEA,UAAM,EAAE,eAAe;AAEvB,QAAI,CAAC,YAAY;AAGf,YAAM,IAAI;AAAA,QACR,8CAA8C,KAAK,YAAY;AAAA,MAAA;AAAA,IAEnE;AAIA,UAAM,IAAI,WAAA;AACV,QAAI,kBAAkB;AACtB,QAAI,IAAI,GAAG;AACT,YAAM,cAAc,KAAK,cAAc,CAAC;AACxC,wBAAkB,gBAAgB;AAAA,IACpC;AAIA,WAAO,MAAM,KAAK;AAAA,EACpB;AAAA,EAEQ,kCACN,SACA;AACA,UAAM,EAAE,WAAA,IACN,KAAK,wBAAwB,8BAC3B,KAAK,YACP;AACF,UAAM,iBAAiB,KAAK,gBAAgB,SAAS,UAAU;AAC/D,SAAK,sBAAsB,gBAAgB,KAAK,iBAAiB,KAAK,IAAI,CAAC;AAAA,EAC7E;AAAA;AAAA;AAAA,EAIQ,cAAc,GAAW;AAC/B,UAAM,EAAE,yBAAyB,UAC/B,KAAK,wBAAwB,8BAC3B,KAAK,YACP;AACF,UAAM,iBAAiB,KAAK;AAC5B,UAAM,mBAAmB,iBACrB,wBAAwB,cAAc,IACtC;AAEJ,UAAM,kBAAkB,MAAM,KAAK,GAAG,gBAAgB;AACtD,UAAM,cACJ,gBAAgB,IAAI,CAAC,QAAQ;AAC3B,aAAO,EAAE,MAAM,UAAU,KAAK,OAAO,KAAK,WAAW,IAAI,GAAG,EAAA;AAAA,IAC9D,CAAC;AACH,SAAK,kCAAkC,WAAW;AAClD,WAAO,YAAY;AAAA,EACrB;AAAA,EAEQ,wBACN,iBACsC;AACtC,UAAM,8BACJ,KAAK,wBAAwB;AAC/B,QAAI,mBAAmB,6BAA6B;AAClD,aAAO,4BAA4B,IAAI,eAAe;AAAA,IACxD;AACA,WAAO;AAAA,EACT;AAAA,EAEA,CAAS,gBACP,SACA,YACA;AACA,eAAW,UAAU,SAAS;AAC5B,WAAK,SAAS,IAAI,OAAO,GAAG;AAE5B,UAAI,CAAC,KAAK,SAAS;AACjB,aAAK,UAAU,OAAO;AAAA,MACxB,WAAW,WAAW,KAAK,SAAS,OAAO,KAAK,IAAI,GAAG;AACrD,aAAK,UAAU,OAAO;AAAA,MACxB;AAEA,YAAM;AAAA,IACR;AAAA,EACF;AACF;AAKA,SAAS,oBACP,cACA,OACoB;;AAEpB,QACE,WAAM,SAAN,mBAAY,UAAS,qBACrB,WAAM,KAAK,eAAX,mBAAuB,QAAO,cAC9B;AACA,WAAO,MAAM,KAAK;AAAA,EACpB;AAGA,MAAI,MAAM,MAAM;AACd,eAAW,cAAc,MAAM,MAAM;AACnC,YACE,gBAAW,SAAX,mBAAiB,UAAS,qBAC1B,gBAAW,KAAK,eAAhB,mBAA4B,QAAO,cACnC;AACA,eAAO,WAAW,KAAK;AAAA,MACzB;AAAA,IACF;AAAA,EACF;AAEA,SAAO;AACT;AAKA,SAAS,mBACP,OACA,SACA,QACQ;AACR,QAAM,gBAAwC,CAAA;AAC9C,aAAW,UAAU,SAAS;AAC5B,UAAM,MAAM,OAAO,OAAO,KAAK;AAC/B,QAAI,OAAO,SAAS,UAAU;AAC5B,oBAAc,KAAK,CAAC,CAAC,KAAK,OAAO,KAAK,GAAG,CAAC,CAAC;AAAA,IAC7C,WAAW,OAAO,SAAS,UAAU;AACnC,oBAAc,KAAK,CAAC,CAAC,KAAK,OAAO,aAAa,GAAG,EAAE,CAAC;AACpD,oBAAc,KAAK,CAAC,CAAC,KAAK,OAAO,KAAK,GAAG,CAAC,CAAC;AAAA,IAC7C,OAAO;AAEL,oBAAc,KAAK,CAAC,CAAC,KAAK,OAAO,KAAK,GAAG,EAAE,CAAC;AAAA,IAC9C;AAAA,EACF;AACA,QAAM,SAAS,IAAI,SAAS,aAAa,CAAC;AAC1C,SAAO,cAAc;AACvB;AAGA,UAAU,aAIR,SACmC;AACnC,aAAW,UAAU,SAAS;AAC5B,QAAI,OAAO,SAAS,UAAU;AAC5B,YAAM,EAAE,MAAM,UAAU,KAAK,OAAO,KAAK,OAAO,OAAO,cAAA;AACvD,YAAM,EAAE,MAAM,UAAU,KAAK,OAAO,KAAK,OAAO,OAAO,MAAA;AAAA,IACzD,OAAO;AACL,YAAM;AAAA,IACR;AAAA,EACF;AACF;AAEA,UAAU,cAIR,SACA,GACmC;AACnC,aAAW,UAAU,SAAS;AAC5B,QAAI,EAAE,MAAM,GAAG;AACb,YAAM;AAAA,IACR;AAAA,EACF;AACF;AASA,UAAU,iCAIR,SACA,YACA,UACmC;AACnC,SAAO,cAAc,SAAS,CAAC,WAAW;AACxC,WAAO,CAAC,YAAY,WAAW,OAAO,OAAO,QAAQ,KAAK;AAAA,EAC5D,CAAC;AACH;"}

package/dist/esm/query/live/types.d.ts

@@ -0,0 +1,77 @@
+import { D2, RootStreamBuilder } from '@tanstack/db-ivm';
+import { CollectionConfig, ResultStream } from '../../types.js';
+import { InitialQueryBuilder, QueryBuilder } from '../builder/index.js';
+import { Context, GetResult } from '../builder/types.js';
+export type Changes<T> = {
+    deletes: number;
+    inserts: number;
+    value: T;
+    orderByIndex: string | undefined;
+};
+export type SyncState = {
+    messagesCount: number;
+    subscribedToAllCollections: boolean;
+    unsubscribeCallbacks: Set<() => void>;
+    graph?: D2;
+    inputs?: Record<string, RootStreamBuilder<unknown>>;
+    pipeline?: ResultStream;
+};
+export type FullSyncState = Required<SyncState>;
+/**
+ * Configuration interface for live query collection options
+ *
+ * @example
+ * ```typescript
+ * const config: LiveQueryCollectionConfig<any, any> = {
+ *   // id is optional - will auto-generate "live-query-1", "live-query-2", etc.
+ *   query: (q) => q
+ *     .from({ comment: commentsCollection })
+ *     .join(
+ *       { user: usersCollection },
+ *       ({ comment, user }) => eq(comment.user_id, user.id)
+ *     )
+ *     .where(({ comment }) => eq(comment.active, true))
+ *     .select(({ comment, user }) => ({
+ *       id: comment.id,
+ *       content: comment.content,
+ *       authorName: user.name,
+ *     })),
+ *   // getKey is optional - defaults to using stream key
+ *   getKey: (item) => item.id,
+ * }
+ * ```
+ */
+export interface LiveQueryCollectionConfig<TContext extends Context, TResult extends object = GetResult<TContext> & object> {
+    /**
+     * Unique identifier for the collection
+     * If not provided, defaults to `live-query-${number}` with auto-incrementing number
+     */
+    id?: string;
+    /**
+     * Query builder function that defines the live query
+     */
+    query: ((q: InitialQueryBuilder) => QueryBuilder<TContext>) | QueryBuilder<TContext>;
+    /**
+     * Function to extract the key from result items
+     * If not provided, defaults to using the key from the D2 stream
+     */
+    getKey?: (item: TResult) => string | number;
+    /**
+     * Optional schema for validation
+     */
+    schema?: CollectionConfig<TResult>[`schema`];
+    /**
+     * Optional mutation handlers
+     */
+    onInsert?: CollectionConfig<TResult>[`onInsert`];
+    onUpdate?: CollectionConfig<TResult>[`onUpdate`];
+    onDelete?: CollectionConfig<TResult>[`onDelete`];
+    /**
+     * Start sync / the query immediately
+     */
+    startSync?: boolean;
+    /**
+     * GC time for the collection
+     */
+    gcTime?: number;
+}

package/dist/esm/query/live-query-collection.d.ts

@@ -1,65 +1,8 @@
+import { LiveQueryCollectionConfig } from './live/types.js';
 import { InitialQueryBuilder, QueryBuilder } from './builder/index.js';
 import { Collection } from '../collection.js';
 import { CollectionConfig, UtilsRecord } from '../types.js';
 import { Context, GetResult } from './builder/types.js';
-/**
- * Configuration interface for live query collection options
- *
- * @example
- * ```typescript
- * const config: LiveQueryCollectionConfig<any, any> = {
- *   // id is optional - will auto-generate "live-query-1", "live-query-2", etc.
- *   query: (q) => q
- *     .from({ comment: commentsCollection })
- *     .join(
- *       { user: usersCollection },
- *       ({ comment, user }) => eq(comment.user_id, user.id)
- *     )
- *     .where(({ comment }) => eq(comment.active, true))
- *     .select(({ comment, user }) => ({
- *       id: comment.id,
- *       content: comment.content,
- *       authorName: user.name,
- *     })),
- *   // getKey is optional - defaults to using stream key
- *   getKey: (item) => item.id,
- * }
- * ```
- */
-export interface LiveQueryCollectionConfig<TContext extends Context, TResult extends object = GetResult<TContext> & object> {
-    /**
-     * Unique identifier for the collection
-     * If not provided, defaults to `live-query-${number}` with auto-incrementing number
-     */
-    id?: string;
-    /**
-     * Query builder function that defines the live query
-     */
-    query: ((q: InitialQueryBuilder) => QueryBuilder<TContext>) | QueryBuilder<TContext>;
-    /**
-     * Function to extract the key from result items
-     * If not provided, defaults to using the key from the D2 stream
-     */
-    getKey?: (item: TResult) => string | number;
-    /**
-     * Optional schema for validation
-     */
-    schema?: CollectionConfig<TResult>[`schema`];
-    /**
-     * Optional mutation handlers
-     */
-    onInsert?: CollectionConfig<TResult>[`onInsert`];
-    onUpdate?: CollectionConfig<TResult>[`onUpdate`];
-    onDelete?: CollectionConfig<TResult>[`onDelete`];
-    /**
-     * Start sync / the query immediately
-     */
-    startSync?: boolean;
-    /**
-     * GC time for the collection
-     */
-    gcTime?: number;
-}
 /**
  * Creates live query collection options for use with createCollection
  *