@tanstack/db 0.0.26 → 0.0.29

package/dist/esm/utils/index-optimization.js

@@ -0,0 +1,270 @@
+function findIndexForField(indexes, fieldPath) {
+  for (const index of indexes.values()) {
+    if (index.matchesField(fieldPath)) {
+      return index;
+    }
+  }
+  return void 0;
+}
+function intersectSets(sets) {
+  if (sets.length === 0) return /* @__PURE__ */ new Set();
+  if (sets.length === 1) return new Set(sets[0]);
+  let result = new Set(sets[0]);
+  for (let i = 1; i < sets.length; i++) {
+    const newResult = /* @__PURE__ */ new Set();
+    for (const item of result) {
+      if (sets[i].has(item)) {
+        newResult.add(item);
+      }
+    }
+    result = newResult;
+  }
+  return result;
+}
+function unionSets(sets) {
+  const result = /* @__PURE__ */ new Set();
+  for (const set of sets) {
+    for (const item of set) {
+      result.add(item);
+    }
+  }
+  return result;
+}
+function optimizeExpressionWithIndexes(expression, indexes) {
+  return optimizeQueryRecursive(expression, indexes);
+}
+function optimizeQueryRecursive(expression, indexes) {
+  if (expression.type === `func`) {
+    switch (expression.name) {
+      case `eq`:
+      case `gt`:
+      case `gte`:
+      case `lt`:
+      case `lte`:
+        return optimizeSimpleComparison(expression, indexes);
+      case `and`:
+        return optimizeAndExpression(expression, indexes);
+      case `or`:
+        return optimizeOrExpression(expression, indexes);
+      case `in`:
+        return optimizeInArrayExpression(expression, indexes);
+    }
+  }
+  return { canOptimize: false, matchingKeys: /* @__PURE__ */ new Set() };
+}
+function optimizeCompoundRangeQuery(expression, indexes) {
+  if (expression.type !== `func` || expression.args.length < 2) {
+    return { canOptimize: false, matchingKeys: /* @__PURE__ */ new Set() };
+  }
+  const fieldOperations = /* @__PURE__ */ new Map();
+  for (const arg of expression.args) {
+    if (arg.type === `func` && [`gt`, `gte`, `lt`, `lte`].includes(arg.name)) {
+      const rangeOp = arg;
+      if (rangeOp.args.length === 2) {
+        const leftArg = rangeOp.args[0];
+        const rightArg = rangeOp.args[1];
+        let fieldArg = null;
+        let valueArg = null;
+        let operation = rangeOp.name;
+        if (leftArg.type === `ref` && rightArg.type === `val`) {
+          fieldArg = leftArg;
+          valueArg = rightArg;
+        } else if (leftArg.type === `val` && rightArg.type === `ref`) {
+          fieldArg = rightArg;
+          valueArg = leftArg;
+          switch (operation) {
+            case `gt`:
+              operation = `lt`;
+              break;
+            case `gte`:
+              operation = `lte`;
+              break;
+            case `lt`:
+              operation = `gt`;
+              break;
+            case `lte`:
+              operation = `gte`;
+              break;
+          }
+        }
+        if (fieldArg && valueArg) {
+          const fieldPath = fieldArg.path;
+          const fieldKey = fieldPath.join(`.`);
+          const value = valueArg.value;
+          if (!fieldOperations.has(fieldKey)) {
+            fieldOperations.set(fieldKey, []);
+          }
+          fieldOperations.get(fieldKey).push({ operation, value });
+        }
+      }
+    }
+  }
+  for (const [fieldKey, operations] of fieldOperations) {
+    if (operations.length >= 2) {
+      const fieldPath = fieldKey.split(`.`);
+      const index = findIndexForField(indexes, fieldPath);
+      if (index && index.supports(`gt`) && index.supports(`lt`)) {
+        let from = void 0;
+        let to = void 0;
+        let fromInclusive = true;
+        let toInclusive = true;
+        for (const { operation, value } of operations) {
+          switch (operation) {
+            case `gt`:
+              if (from === void 0 || value > from) {
+                from = value;
+                fromInclusive = false;
+              }
+              break;
+            case `gte`:
+              if (from === void 0 || value > from) {
+                from = value;
+                fromInclusive = true;
+              }
+              break;
+            case `lt`:
+              if (to === void 0 || value < to) {
+                to = value;
+                toInclusive = false;
+              }
+              break;
+            case `lte`:
+              if (to === void 0 || value < to) {
+                to = value;
+                toInclusive = true;
+              }
+              break;
+          }
+        }
+        const matchingKeys = index.rangeQuery({
+          from,
+          to,
+          fromInclusive,
+          toInclusive
+        });
+        return { canOptimize: true, matchingKeys };
+      }
+    }
+  }
+  return { canOptimize: false, matchingKeys: /* @__PURE__ */ new Set() };
+}
+function optimizeSimpleComparison(expression, indexes) {
+  if (expression.type !== `func` || expression.args.length !== 2) {
+    return { canOptimize: false, matchingKeys: /* @__PURE__ */ new Set() };
+  }
+  const leftArg = expression.args[0];
+  const rightArg = expression.args[1];
+  let fieldArg = null;
+  let valueArg = null;
+  let operation = expression.name;
+  if (leftArg.type === `ref` && rightArg.type === `val`) {
+    fieldArg = leftArg;
+    valueArg = rightArg;
+  } else if (leftArg.type === `val` && rightArg.type === `ref`) {
+    fieldArg = rightArg;
+    valueArg = leftArg;
+    switch (operation) {
+      case `gt`:
+        operation = `lt`;
+        break;
+      case `gte`:
+        operation = `lte`;
+        break;
+      case `lt`:
+        operation = `gt`;
+        break;
+      case `lte`:
+        operation = `gte`;
+        break;
+    }
+  }
+  if (fieldArg && valueArg) {
+    const fieldPath = fieldArg.path;
+    const index = findIndexForField(indexes, fieldPath);
+    if (index) {
+      const queryValue = valueArg.value;
+      const indexOperation = operation;
+      if (!index.supports(indexOperation)) {
+        return { canOptimize: false, matchingKeys: /* @__PURE__ */ new Set() };
+      }
+      const matchingKeys = index.lookup(indexOperation, queryValue);
+      return { canOptimize: true, matchingKeys };
+    }
+  }
+  return { canOptimize: false, matchingKeys: /* @__PURE__ */ new Set() };
+}
+function optimizeAndExpression(expression, indexes) {
+  if (expression.type !== `func` || expression.args.length < 2) {
+    return { canOptimize: false, matchingKeys: /* @__PURE__ */ new Set() };
+  }
+  const compoundRangeResult = optimizeCompoundRangeQuery(expression, indexes);
+  if (compoundRangeResult.canOptimize) {
+    return compoundRangeResult;
+  }
+  const results = [];
+  for (const arg of expression.args) {
+    const result = optimizeQueryRecursive(arg, indexes);
+    if (result.canOptimize) {
+      results.push(result);
+    }
+  }
+  if (results.length > 0) {
+    const allMatchingSets = results.map((r) => r.matchingKeys);
+    const intersectedKeys = intersectSets(allMatchingSets);
+    return { canOptimize: true, matchingKeys: intersectedKeys };
+  }
+  return { canOptimize: false, matchingKeys: /* @__PURE__ */ new Set() };
+}
+function optimizeOrExpression(expression, indexes) {
+  if (expression.type !== `func` || expression.args.length < 2) {
+    return { canOptimize: false, matchingKeys: /* @__PURE__ */ new Set() };
+  }
+  const results = [];
+  for (const arg of expression.args) {
+    const result = optimizeQueryRecursive(arg, indexes);
+    if (result.canOptimize) {
+      results.push(result);
+    }
+  }
+  if (results.length > 0) {
+    const allMatchingSets = results.map((r) => r.matchingKeys);
+    const unionedKeys = unionSets(allMatchingSets);
+    return { canOptimize: true, matchingKeys: unionedKeys };
+  }
+  return { canOptimize: false, matchingKeys: /* @__PURE__ */ new Set() };
+}
+function optimizeInArrayExpression(expression, indexes) {
+  if (expression.type !== `func` || expression.args.length !== 2) {
+    return { canOptimize: false, matchingKeys: /* @__PURE__ */ new Set() };
+  }
+  const fieldArg = expression.args[0];
+  const arrayArg = expression.args[1];
+  if (fieldArg.type === `ref` && arrayArg.type === `val` && Array.isArray(arrayArg.value)) {
+    const fieldPath = fieldArg.path;
+    const values = arrayArg.value;
+    const index = findIndexForField(indexes, fieldPath);
+    if (index) {
+      if (index.supports(`in`)) {
+        const matchingKeys = index.lookup(`in`, values);
+        return { canOptimize: true, matchingKeys };
+      } else if (index.supports(`eq`)) {
+        const matchingKeys = /* @__PURE__ */ new Set();
+        for (const value of values) {
+          const keysForValue = index.lookup(`eq`, value);
+          for (const key of keysForValue) {
+            matchingKeys.add(key);
+          }
+        }
+        return { canOptimize: true, matchingKeys };
+      }
+    }
+  }
+  return { canOptimize: false, matchingKeys: /* @__PURE__ */ new Set() };
+}
+export {
+  findIndexForField,
+  intersectSets,
+  optimizeExpressionWithIndexes,
+  unionSets
+};
+//# sourceMappingURL=index-optimization.js.map

package/dist/esm/utils/index-optimization.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index-optimization.js","sources":["../../../src/utils/index-optimization.ts"],"sourcesContent":["/**\n * # Index-Based Query Optimization\n *\n * This module provides utilities for optimizing query expressions by leveraging\n * available indexes to quickly find matching keys instead of scanning all data.\n *\n * This is different from the query structure optimizer in `query/optimizer.ts`\n * which rewrites query IR structure. This module focuses on using indexes during\n * query execution to speed up data filtering.\n *\n * ## Key Features:\n * - Uses indexes to find matching keys for WHERE conditions\n * - Supports AND/OR logic with set operations\n * - Handles range queries (eq, gt, gte, lt, lte)\n * - Optimizes IN array expressions\n */\n\nimport type { BaseIndex, IndexOperation } from \"../indexes/base-index.js\"\nimport type { BasicExpression } from \"../query/ir.js\"\n\n/**\n * Result of index-based query optimization\n */\nexport interface OptimizationResult<TKey> {\n  canOptimize: boolean\n  matchingKeys: Set<TKey>\n}\n\n/**\n * Finds an index that matches a given field path\n */\nexport function findIndexForField<TKey extends string | number>(\n  indexes: Map<number, BaseIndex<TKey>>,\n  fieldPath: Array<string>\n): BaseIndex<TKey> | undefined {\n  for (const index of indexes.values()) {\n    if (index.matchesField(fieldPath)) {\n      return index\n    }\n  }\n  return undefined\n}\n\n/**\n * Intersects multiple sets (AND logic)\n */\nexport function intersectSets<T>(sets: Array<Set<T>>): Set<T> {\n  if (sets.length === 0) return new Set()\n  if (sets.length === 1) return new Set(sets[0])\n\n  let result = new Set(sets[0])\n  for (let i = 1; i < sets.length; i++) {\n    const newResult = new Set<T>()\n    for (const item of result) {\n      if (sets[i]!.has(item)) {\n        newResult.add(item)\n      }\n    }\n    result = newResult\n  }\n  return result\n}\n\n/**\n * Unions multiple sets (OR logic)\n */\nexport function unionSets<T>(sets: Array<Set<T>>): Set<T> {\n  const result = new Set<T>()\n  for (const set of sets) {\n    for (const item of set) {\n      result.add(item)\n    }\n  }\n  return result\n}\n\n/**\n * Optimizes a query expression using available indexes to find matching keys\n */\nexport function optimizeExpressionWithIndexes<TKey extends string | number>(\n  expression: BasicExpression,\n  indexes: Map<number, BaseIndex<TKey>>\n): OptimizationResult<TKey> {\n  return optimizeQueryRecursive(expression, indexes)\n}\n\n/**\n * Recursively optimizes query expressions\n */\nfunction optimizeQueryRecursive<TKey extends string | number>(\n  expression: BasicExpression,\n  indexes: Map<number, BaseIndex<TKey>>\n): OptimizationResult<TKey> {\n  if (expression.type === `func`) {\n    switch (expression.name) {\n      case `eq`:\n      case `gt`:\n      case `gte`:\n      case `lt`:\n      case `lte`:\n        return optimizeSimpleComparison(expression, indexes)\n\n      case `and`:\n        return optimizeAndExpression(expression, indexes)\n\n      case `or`:\n        return optimizeOrExpression(expression, indexes)\n\n      case `in`:\n        return optimizeInArrayExpression(expression, indexes)\n    }\n  }\n\n  return { canOptimize: false, matchingKeys: new Set() }\n}\n\n/**\n * Checks if an expression can be optimized\n */\nexport function canOptimizeExpression<TKey extends string | number>(\n  expression: BasicExpression,\n  indexes: Map<number, BaseIndex<TKey>>\n): boolean {\n  if (expression.type === `func`) {\n    switch (expression.name) {\n      case `eq`:\n      case `gt`:\n      case `gte`:\n      case `lt`:\n      case `lte`:\n        return canOptimizeSimpleComparison(expression, indexes)\n\n      case `and`:\n        return canOptimizeAndExpression(expression, indexes)\n\n      case `or`:\n        return canOptimizeOrExpression(expression, indexes)\n\n      case `in`:\n        return canOptimizeInArrayExpression(expression, indexes)\n    }\n  }\n\n  return false\n}\n\n/**\n * Optimizes compound range queries on the same field\n * Example: WHERE age > 5 AND age < 10\n */\nfunction optimizeCompoundRangeQuery<TKey extends string | number>(\n  expression: BasicExpression,\n  indexes: Map<number, BaseIndex<TKey>>\n): OptimizationResult<TKey> {\n  if (expression.type !== `func` || expression.args.length < 2) {\n    return { canOptimize: false, matchingKeys: new Set() }\n  }\n\n  // Group range operations by field\n  const fieldOperations = new Map<\n    string,\n    Array<{\n      operation: `gt` | `gte` | `lt` | `lte`\n      value: any\n    }>\n  >()\n\n  // Collect all range operations from AND arguments\n  for (const arg of expression.args) {\n    if (arg.type === `func` && [`gt`, `gte`, `lt`, `lte`].includes(arg.name)) {\n      const rangeOp = arg as any\n      if (rangeOp.args.length === 2) {\n        const leftArg = rangeOp.args[0]!\n        const rightArg = rangeOp.args[1]!\n\n        // Check both directions: field op value AND value op field\n        let fieldArg: BasicExpression | null = null\n        let valueArg: BasicExpression | null = null\n        let operation = rangeOp.name as `gt` | `gte` | `lt` | `lte`\n\n        if (leftArg.type === `ref` && rightArg.type === `val`) {\n          // field op value\n          fieldArg = leftArg\n          valueArg = rightArg\n        } else if (leftArg.type === `val` && rightArg.type === `ref`) {\n          // value op field - need to flip the operation\n          fieldArg = rightArg\n          valueArg = leftArg\n\n          // Flip the operation for reverse comparison\n          switch (operation) {\n            case `gt`:\n              operation = `lt`\n              break\n            case `gte`:\n              operation = `lte`\n              break\n            case `lt`:\n              operation = `gt`\n              break\n            case `lte`:\n              operation = `gte`\n              break\n          }\n        }\n\n        if (fieldArg && valueArg) {\n          const fieldPath = (fieldArg as any).path\n          const fieldKey = fieldPath.join(`.`)\n          const value = (valueArg as any).value\n\n          if (!fieldOperations.has(fieldKey)) {\n            fieldOperations.set(fieldKey, [])\n          }\n          fieldOperations.get(fieldKey)!.push({ operation, value })\n        }\n      }\n    }\n  }\n\n  // Check if we have multiple operations on the same field\n  for (const [fieldKey, operations] of fieldOperations) {\n    if (operations.length >= 2) {\n      const fieldPath = fieldKey.split(`.`)\n      const index = findIndexForField(indexes, fieldPath)\n\n      if (index && index.supports(`gt`) && index.supports(`lt`)) {\n        // Build range query options\n        let from: any = undefined\n        let to: any = undefined\n        let fromInclusive = true\n        let toInclusive = true\n\n        for (const { operation, value } of operations) {\n          switch (operation) {\n            case `gt`:\n              if (from === undefined || value > from) {\n                from = value\n                fromInclusive = false\n              }\n              break\n            case `gte`:\n              if (from === undefined || value > from) {\n                from = value\n                fromInclusive = true\n              }\n              break\n            case `lt`:\n              if (to === undefined || value < to) {\n                to = value\n                toInclusive = false\n              }\n              break\n            case `lte`:\n              if (to === undefined || value < to) {\n                to = value\n                toInclusive = true\n              }\n              break\n          }\n        }\n\n        const matchingKeys = (index as any).rangeQuery({\n          from,\n          to,\n          fromInclusive,\n          toInclusive,\n        })\n\n        return { canOptimize: true, matchingKeys }\n      }\n    }\n  }\n\n  return { canOptimize: false, matchingKeys: new Set() }\n}\n\n/**\n * Optimizes simple comparison expressions (eq, gt, gte, lt, lte)\n */\nfunction optimizeSimpleComparison<TKey extends string | number>(\n  expression: BasicExpression,\n  indexes: Map<number, BaseIndex<TKey>>\n): OptimizationResult<TKey> {\n  if (expression.type !== `func` || expression.args.length !== 2) {\n    return { canOptimize: false, matchingKeys: new Set() }\n  }\n\n  const leftArg = expression.args[0]!\n  const rightArg = expression.args[1]!\n\n  // Check both directions: field op value AND value op field\n  let fieldArg: BasicExpression | null = null\n  let valueArg: BasicExpression | null = null\n  let operation = expression.name as `eq` | `gt` | `gte` | `lt` | `lte`\n\n  if (leftArg.type === `ref` && rightArg.type === `val`) {\n    // field op value\n    fieldArg = leftArg\n    valueArg = rightArg\n  } else if (leftArg.type === `val` && rightArg.type === `ref`) {\n    // value op field - need to flip the operation\n    fieldArg = rightArg\n    valueArg = leftArg\n\n    // Flip the operation for reverse comparison\n    switch (operation) {\n      case `gt`:\n        operation = `lt`\n        break\n      case `gte`:\n        operation = `lte`\n        break\n      case `lt`:\n        operation = `gt`\n        break\n      case `lte`:\n        operation = `gte`\n        break\n      // eq stays the same\n    }\n  }\n\n  if (fieldArg && valueArg) {\n    const fieldPath = (fieldArg as any).path\n    const index = findIndexForField(indexes, fieldPath)\n\n    if (index) {\n      const queryValue = (valueArg as any).value\n\n      // Map operation to IndexOperation enum\n      const indexOperation = operation as IndexOperation\n\n      // Check if the index supports this operation\n      if (!index.supports(indexOperation)) {\n        return { canOptimize: false, matchingKeys: new Set() }\n      }\n\n      const matchingKeys = index.lookup(indexOperation, queryValue)\n      return { canOptimize: true, matchingKeys }\n    }\n  }\n\n  return { canOptimize: false, matchingKeys: new Set() }\n}\n\n/**\n * Checks if a simple comparison can be optimized\n */\nfunction canOptimizeSimpleComparison<TKey extends string | number>(\n  expression: BasicExpression,\n  indexes: Map<number, BaseIndex<TKey>>\n): boolean {\n  if (expression.type !== `func` || expression.args.length !== 2) {\n    return false\n  }\n\n  const leftArg = expression.args[0]!\n  const rightArg = expression.args[1]!\n\n  // Check both directions: field op value AND value op field\n  let fieldPath: Array<string> | null = null\n\n  if (leftArg.type === `ref` && rightArg.type === `val`) {\n    fieldPath = (leftArg as any).path\n  } else if (leftArg.type === `val` && rightArg.type === `ref`) {\n    fieldPath = (rightArg as any).path\n  }\n\n  if (fieldPath) {\n    const index = findIndexForField(indexes, fieldPath)\n    return index !== undefined\n  }\n\n  return false\n}\n\n/**\n * Optimizes AND expressions\n */\nfunction optimizeAndExpression<TKey extends string | number>(\n  expression: BasicExpression,\n  indexes: Map<number, BaseIndex<TKey>>\n): OptimizationResult<TKey> {\n  if (expression.type !== `func` || expression.args.length < 2) {\n    return { canOptimize: false, matchingKeys: new Set() }\n  }\n\n  // First, try to optimize compound range queries on the same field\n  const compoundRangeResult = optimizeCompoundRangeQuery(expression, indexes)\n  if (compoundRangeResult.canOptimize) {\n    return compoundRangeResult\n  }\n\n  const results: Array<OptimizationResult<TKey>> = []\n\n  // Try to optimize each part, keep the optimizable ones\n  for (const arg of expression.args) {\n    const result = optimizeQueryRecursive(arg, indexes)\n    if (result.canOptimize) {\n      results.push(result)\n    }\n  }\n\n  if (results.length > 0) {\n    // Use intersectSets utility for AND logic\n    const allMatchingSets = results.map((r) => r.matchingKeys)\n    const intersectedKeys = intersectSets(allMatchingSets)\n    return { canOptimize: true, matchingKeys: intersectedKeys }\n  }\n\n  return { canOptimize: false, matchingKeys: new Set() }\n}\n\n/**\n * Checks if an AND expression can be optimized\n */\nfunction canOptimizeAndExpression<TKey extends string | number>(\n  expression: BasicExpression,\n  indexes: Map<number, BaseIndex<TKey>>\n): boolean {\n  if (expression.type !== `func` || expression.args.length < 2) {\n    return false\n  }\n\n  // If any argument can be optimized, we can gain some speedup\n  return expression.args.some((arg) => canOptimizeExpression(arg, indexes))\n}\n\n/**\n * Optimizes OR expressions\n */\nfunction optimizeOrExpression<TKey extends string | number>(\n  expression: BasicExpression,\n  indexes: Map<number, BaseIndex<TKey>>\n): OptimizationResult<TKey> {\n  if (expression.type !== `func` || expression.args.length < 2) {\n    return { canOptimize: false, matchingKeys: new Set() }\n  }\n\n  const results: Array<OptimizationResult<TKey>> = []\n\n  // Try to optimize each part, keep the optimizable ones\n  for (const arg of expression.args) {\n    const result = optimizeQueryRecursive(arg, indexes)\n    if (result.canOptimize) {\n      results.push(result)\n    }\n  }\n\n  if (results.length > 0) {\n    // Use unionSets utility for OR logic\n    const allMatchingSets = results.map((r) => r.matchingKeys)\n    const unionedKeys = unionSets(allMatchingSets)\n    return { canOptimize: true, matchingKeys: unionedKeys }\n  }\n\n  return { canOptimize: false, matchingKeys: new Set() }\n}\n\n/**\n * Checks if an OR expression can be optimized\n */\nfunction canOptimizeOrExpression<TKey extends string | number>(\n  expression: BasicExpression,\n  indexes: Map<number, BaseIndex<TKey>>\n): boolean {\n  if (expression.type !== `func` || expression.args.length < 2) {\n    return false\n  }\n\n  // If any argument can be optimized, we can gain some speedup\n  return expression.args.some((arg) => canOptimizeExpression(arg, indexes))\n}\n\n/**\n * Optimizes IN array expressions\n */\nfunction optimizeInArrayExpression<TKey extends string | number>(\n  expression: BasicExpression,\n  indexes: Map<number, BaseIndex<TKey>>\n): OptimizationResult<TKey> {\n  if (expression.type !== `func` || expression.args.length !== 2) {\n    return { canOptimize: false, matchingKeys: new Set() }\n  }\n\n  const fieldArg = expression.args[0]!\n  const arrayArg = expression.args[1]!\n\n  if (\n    fieldArg.type === `ref` &&\n    arrayArg.type === `val` &&\n    Array.isArray((arrayArg as any).value)\n  ) {\n    const fieldPath = (fieldArg as any).path\n    const values = (arrayArg as any).value\n    const index = findIndexForField(indexes, fieldPath)\n\n    if (index) {\n      // Check if the index supports IN operation\n      if (index.supports(`in`)) {\n        const matchingKeys = index.lookup(`in`, values)\n        return { canOptimize: true, matchingKeys }\n      } else if (index.supports(`eq`)) {\n        // Fallback to multiple equality lookups\n        const matchingKeys = new Set<TKey>()\n        for (const value of values) {\n          const keysForValue = index.lookup(`eq`, value)\n          for (const key of keysForValue) {\n            matchingKeys.add(key)\n          }\n        }\n        return { canOptimize: true, matchingKeys }\n      }\n    }\n  }\n\n  return { canOptimize: false, matchingKeys: new Set() }\n}\n\n/**\n * Checks if an IN array expression can be optimized\n */\nfunction canOptimizeInArrayExpression<TKey extends string | number>(\n  expression: BasicExpression,\n  indexes: Map<number, BaseIndex<TKey>>\n): boolean {\n  if (expression.type !== `func` || expression.args.length !== 2) {\n    return false\n  }\n\n  const fieldArg = expression.args[0]!\n  const arrayArg = expression.args[1]!\n\n  if (\n    fieldArg.type === `ref` &&\n    arrayArg.type === `val` &&\n    Array.isArray((arrayArg as any).value)\n  ) {\n    const fieldPath = (fieldArg as any).path\n    const index = findIndexForField(indexes, fieldPath)\n    return index !== undefined\n  }\n\n  return false\n}\n"],"names":[],"mappings":"AA+BO,SAAS,kBACd,SACA,WAC6B;AAC7B,aAAW,SAAS,QAAQ,UAAU;AACpC,QAAI,MAAM,aAAa,SAAS,GAAG;AACjC,aAAO;AAAA,IACT;AAAA,EACF;AACA,SAAO;AACT;AAKO,SAAS,cAAiB,MAA6B;AAC5D,MAAI,KAAK,WAAW,EAAG,4BAAW,IAAA;AAClC,MAAI,KAAK,WAAW,EAAG,QAAO,IAAI,IAAI,KAAK,CAAC,CAAC;AAE7C,MAAI,SAAS,IAAI,IAAI,KAAK,CAAC,CAAC;AAC5B,WAAS,IAAI,GAAG,IAAI,KAAK,QAAQ,KAAK;AACpC,UAAM,gCAAgB,IAAA;AACtB,eAAW,QAAQ,QAAQ;AACzB,UAAI,KAAK,CAAC,EAAG,IAAI,IAAI,GAAG;AACtB,kBAAU,IAAI,IAAI;AAAA,MACpB;AAAA,IACF;AACA,aAAS;AAAA,EACX;AACA,SAAO;AACT;AAKO,SAAS,UAAa,MAA6B;AACxD,QAAM,6BAAa,IAAA;AACnB,aAAW,OAAO,MAAM;AACtB,eAAW,QAAQ,KAAK;AACtB,aAAO,IAAI,IAAI;AAAA,IACjB;AAAA,EACF;AACA,SAAO;AACT;AAKO,SAAS,8BACd,YACA,SAC0B;AAC1B,SAAO,uBAAuB,YAAY,OAAO;AACnD;AAKA,SAAS,uBACP,YACA,SAC0B;AAC1B,MAAI,WAAW,SAAS,QAAQ;AAC9B,YAAQ,WAAW,MAAA;AAAA,MACjB,KAAK;AAAA,MACL,KAAK;AAAA,MACL,KAAK;AAAA,MACL,KAAK;AAAA,MACL,KAAK;AACH,eAAO,yBAAyB,YAAY,OAAO;AAAA,MAErD,KAAK;AACH,eAAO,sBAAsB,YAAY,OAAO;AAAA,MAElD,KAAK;AACH,eAAO,qBAAqB,YAAY,OAAO;AAAA,MAEjD,KAAK;AACH,eAAO,0BAA0B,YAAY,OAAO;AAAA,IAAA;AAAA,EAE1D;AAEA,SAAO,EAAE,aAAa,OAAO,cAAc,oBAAI,MAAI;AACrD;AAoCA,SAAS,2BACP,YACA,SAC0B;AAC1B,MAAI,WAAW,SAAS,UAAU,WAAW,KAAK,SAAS,GAAG;AAC5D,WAAO,EAAE,aAAa,OAAO,cAAc,oBAAI,MAAI;AAAA,EACrD;AAGA,QAAM,sCAAsB,IAAA;AAS5B,aAAW,OAAO,WAAW,MAAM;AACjC,QAAI,IAAI,SAAS,UAAU,CAAC,MAAM,OAAO,MAAM,KAAK,EAAE,SAAS,IAAI,IAAI,GAAG;AACxE,YAAM,UAAU;AAChB,UAAI,QAAQ,KAAK,WAAW,GAAG;AAC7B,cAAM,UAAU,QAAQ,KAAK,CAAC;AAC9B,cAAM,WAAW,QAAQ,KAAK,CAAC;AAG/B,YAAI,WAAmC;AACvC,YAAI,WAAmC;AACvC,YAAI,YAAY,QAAQ;AAExB,YAAI,QAAQ,SAAS,SAAS,SAAS,SAAS,OAAO;AAErD,qBAAW;AACX,qBAAW;AAAA,QACb,WAAW,QAAQ,SAAS,SAAS,SAAS,SAAS,OAAO;AAE5D,qBAAW;AACX,qBAAW;AAGX,kBAAQ,WAAA;AAAA,YACN,KAAK;AACH,0BAAY;AACZ;AAAA,YACF,KAAK;AACH,0BAAY;AACZ;AAAA,YACF,KAAK;AACH,0BAAY;AACZ;AAAA,YACF,KAAK;AACH,0BAAY;AACZ;AAAA,UAAA;AAAA,QAEN;AAEA,YAAI,YAAY,UAAU;AACxB,gBAAM,YAAa,SAAiB;AACpC,gBAAM,WAAW,UAAU,KAAK,GAAG;AACnC,gBAAM,QAAS,SAAiB;AAEhC,cAAI,CAAC,gBAAgB,IAAI,QAAQ,GAAG;AAClC,4BAAgB,IAAI,UAAU,EAAE;AAAA,UAClC;AACA,0BAAgB,IAAI,QAAQ,EAAG,KAAK,EAAE,WAAW,OAAO;AAAA,QAC1D;AAAA,MACF;AAAA,IACF;AAAA,EACF;AAGA,aAAW,CAAC,UAAU,UAAU,KAAK,iBAAiB;AACpD,QAAI,WAAW,UAAU,GAAG;AAC1B,YAAM,YAAY,SAAS,MAAM,GAAG;AACpC,YAAM,QAAQ,kBAAkB,SAAS,SAAS;AAElD,UAAI,SAAS,MAAM,SAAS,IAAI,KAAK,MAAM,SAAS,IAAI,GAAG;AAEzD,YAAI,OAAY;AAChB,YAAI,KAAU;AACd,YAAI,gBAAgB;AACpB,YAAI,cAAc;AAElB,mBAAW,EAAE,WAAW,MAAA,KAAW,YAAY;AAC7C,kBAAQ,WAAA;AAAA,YACN,KAAK;AACH,kBAAI,SAAS,UAAa,QAAQ,MAAM;AACtC,uBAAO;AACP,gCAAgB;AAAA,cAClB;AACA;AAAA,YACF,KAAK;AACH,kBAAI,SAAS,UAAa,QAAQ,MAAM;AACtC,uBAAO;AACP,gCAAgB;AAAA,cAClB;AACA;AAAA,YACF,KAAK;AACH,kBAAI,OAAO,UAAa,QAAQ,IAAI;AAClC,qBAAK;AACL,8BAAc;AAAA,cAChB;AACA;AAAA,YACF,KAAK;AACH,kBAAI,OAAO,UAAa,QAAQ,IAAI;AAClC,qBAAK;AACL,8BAAc;AAAA,cAChB;AACA;AAAA,UAAA;AAAA,QAEN;AAEA,cAAM,eAAgB,MAAc,WAAW;AAAA,UAC7C;AAAA,UACA;AAAA,UACA;AAAA,UACA;AAAA,QAAA,CACD;AAED,eAAO,EAAE,aAAa,MAAM,aAAA;AAAA,MAC9B;AAAA,IACF;AAAA,EACF;AAEA,SAAO,EAAE,aAAa,OAAO,cAAc,oBAAI,MAAI;AACrD;AAKA,SAAS,yBACP,YACA,SAC0B;AAC1B,MAAI,WAAW,SAAS,UAAU,WAAW,KAAK,WAAW,GAAG;AAC9D,WAAO,EAAE,aAAa,OAAO,cAAc,oBAAI,MAAI;AAAA,EACrD;AAEA,QAAM,UAAU,WAAW,KAAK,CAAC;AACjC,QAAM,WAAW,WAAW,KAAK,CAAC;AAGlC,MAAI,WAAmC;AACvC,MAAI,WAAmC;AACvC,MAAI,YAAY,WAAW;AAE3B,MAAI,QAAQ,SAAS,SAAS,SAAS,SAAS,OAAO;AAErD,eAAW;AACX,eAAW;AAAA,EACb,WAAW,QAAQ,SAAS,SAAS,SAAS,SAAS,OAAO;AAE5D,eAAW;AACX,eAAW;AAGX,YAAQ,WAAA;AAAA,MACN,KAAK;AACH,oBAAY;AACZ;AAAA,MACF,KAAK;AACH,oBAAY;AACZ;AAAA,MACF,KAAK;AACH,oBAAY;AACZ;AAAA,MACF,KAAK;AACH,oBAAY;AACZ;AAAA,IAAA;AAAA,EAGN;AAEA,MAAI,YAAY,UAAU;AACxB,UAAM,YAAa,SAAiB;AACpC,UAAM,QAAQ,kBAAkB,SAAS,SAAS;AAElD,QAAI,OAAO;AACT,YAAM,aAAc,SAAiB;AAGrC,YAAM,iBAAiB;AAGvB,UAAI,CAAC,MAAM,SAAS,cAAc,GAAG;AACnC,eAAO,EAAE,aAAa,OAAO,cAAc,oBAAI,MAAI;AAAA,MACrD;AAEA,YAAM,eAAe,MAAM,OAAO,gBAAgB,UAAU;AAC5D,aAAO,EAAE,aAAa,MAAM,aAAA;AAAA,IAC9B;AAAA,EACF;AAEA,SAAO,EAAE,aAAa,OAAO,cAAc,oBAAI,MAAI;AACrD;AAoCA,SAAS,sBACP,YACA,SAC0B;AAC1B,MAAI,WAAW,SAAS,UAAU,WAAW,KAAK,SAAS,GAAG;AAC5D,WAAO,EAAE,aAAa,OAAO,cAAc,oBAAI,MAAI;AAAA,EACrD;AAGA,QAAM,sBAAsB,2BAA2B,YAAY,OAAO;AAC1E,MAAI,oBAAoB,aAAa;AACnC,WAAO;AAAA,EACT;AAEA,QAAM,UAA2C,CAAA;AAGjD,aAAW,OAAO,WAAW,MAAM;AACjC,UAAM,SAAS,uBAAuB,KAAK,OAAO;AAClD,QAAI,OAAO,aAAa;AACtB,cAAQ,KAAK,MAAM;AAAA,IACrB;AAAA,EACF;AAEA,MAAI,QAAQ,SAAS,GAAG;AAEtB,UAAM,kBAAkB,QAAQ,IAAI,CAAC,MAAM,EAAE,YAAY;AACzD,UAAM,kBAAkB,cAAc,eAAe;AACrD,WAAO,EAAE,aAAa,MAAM,cAAc,gBAAA;AAAA,EAC5C;AAEA,SAAO,EAAE,aAAa,OAAO,cAAc,oBAAI,MAAI;AACrD;AAoBA,SAAS,qBACP,YACA,SAC0B;AAC1B,MAAI,WAAW,SAAS,UAAU,WAAW,KAAK,SAAS,GAAG;AAC5D,WAAO,EAAE,aAAa,OAAO,cAAc,oBAAI,MAAI;AAAA,EACrD;AAEA,QAAM,UAA2C,CAAA;AAGjD,aAAW,OAAO,WAAW,MAAM;AACjC,UAAM,SAAS,uBAAuB,KAAK,OAAO;AAClD,QAAI,OAAO,aAAa;AACtB,cAAQ,KAAK,MAAM;AAAA,IACrB;AAAA,EACF;AAEA,MAAI,QAAQ,SAAS,GAAG;AAEtB,UAAM,kBAAkB,QAAQ,IAAI,CAAC,MAAM,EAAE,YAAY;AACzD,UAAM,cAAc,UAAU,eAAe;AAC7C,WAAO,EAAE,aAAa,MAAM,cAAc,YAAA;AAAA,EAC5C;AAEA,SAAO,EAAE,aAAa,OAAO,cAAc,oBAAI,MAAI;AACrD;AAoBA,SAAS,0BACP,YACA,SAC0B;AAC1B,MAAI,WAAW,SAAS,UAAU,WAAW,KAAK,WAAW,GAAG;AAC9D,WAAO,EAAE,aAAa,OAAO,cAAc,oBAAI,MAAI;AAAA,EACrD;AAEA,QAAM,WAAW,WAAW,KAAK,CAAC;AAClC,QAAM,WAAW,WAAW,KAAK,CAAC;AAElC,MACE,SAAS,SAAS,SAClB,SAAS,SAAS,SAClB,MAAM,QAAS,SAAiB,KAAK,GACrC;AACA,UAAM,YAAa,SAAiB;AACpC,UAAM,SAAU,SAAiB;AACjC,UAAM,QAAQ,kBAAkB,SAAS,SAAS;AAElD,QAAI,OAAO;AAET,UAAI,MAAM,SAAS,IAAI,GAAG;AACxB,cAAM,eAAe,MAAM,OAAO,MAAM,MAAM;AAC9C,eAAO,EAAE,aAAa,MAAM,aAAA;AAAA,MAC9B,WAAW,MAAM,SAAS,IAAI,GAAG;AAE/B,cAAM,mCAAmB,IAAA;AACzB,mBAAW,SAAS,QAAQ;AAC1B,gBAAM,eAAe,MAAM,OAAO,MAAM,KAAK;AAC7C,qBAAW,OAAO,cAAc;AAC9B,yBAAa,IAAI,GAAG;AAAA,UACtB;AAAA,QACF;AACA,eAAO,EAAE,aAAa,MAAM,aAAA;AAAA,MAC9B;AAAA,IACF;AAAA,EACF;AAEA,SAAO,EAAE,aAAa,OAAO,cAAc,oBAAI,MAAI;AACrD;"}

package/package.json

@@ -1,13 +1,14 @@
 {
   "name": "@tanstack/db",
   "description": "A reactive client store for building super fast apps on sync",
-  "version": "0.0.26",
+  "version": "0.0.29",
   "dependencies": {
     "@electric-sql/d2mini": "^0.1.7",
     "@standard-schema/spec": "^1.0.0"
   },
   "devDependencies": {
-    "@vitest/coverage-istanbul": "^3.0.9"
+    "@vitest/coverage-istanbul": "^3.0.9",
+    "arktype": "^2.1.20"
   },
   "exports": {
     ".": {

package/src/change-events.ts

@@ -0,0 +1,257 @@
+import {
+  createSingleRowRefProxy,
+  toExpression,
+} from "./query/builder/ref-proxy"
+import { compileSingleRowExpression } from "./query/compiler/evaluators.js"
+import { optimizeExpressionWithIndexes } from "./utils/index-optimization.js"
+import type {
+  ChangeMessage,
+  CurrentStateAsChangesOptions,
+  SubscribeChangesOptions,
+} from "./types"
+import type { Collection } from "./collection"
+import type { SingleRowRefProxy } from "./query/builder/ref-proxy"
+import type { BasicExpression } from "./query/ir.js"
+
+/**
+ * Interface for a collection-like object that provides the necessary methods
+ * for the change events system to work
+ */
+export interface CollectionLike<
+  T extends object = Record<string, unknown>,
+  TKey extends string | number = string | number,
+> extends Pick<Collection<T, TKey>, `get` | `has` | `entries` | `indexes`> {}
+
+/**
+ * Returns the current state of the collection as an array of changes
+ * @param collection - The collection to get changes from
+ * @param options - Options including optional where filter
+ * @returns An array of changes
+ * @example
+ * // Get all items as changes
+ * const allChanges = currentStateAsChanges(collection)
+ *
+ * // Get only items matching a condition
+ * const activeChanges = currentStateAsChanges(collection, {
+ *   where: (row) => row.status === 'active'
+ * })
+ *
+ * // Get only items using a pre-compiled expression
+ * const activeChanges = currentStateAsChanges(collection, {
+ *   whereExpression: eq(row.status, 'active')
+ * })
+ */
+export function currentStateAsChanges<
+  T extends object,
+  TKey extends string | number,
+>(
+  collection: CollectionLike<T, TKey>,
+  options: CurrentStateAsChangesOptions<T> = {}
+): Array<ChangeMessage<T>> {
+  // Helper function to collect filtered results
+  const collectFilteredResults = (
+    filterFn?: (value: T) => boolean
+  ): Array<ChangeMessage<T>> => {
+    const result: Array<ChangeMessage<T>> = []
+    for (const [key, value] of collection.entries()) {
+      // If no filter function is provided, include all items
+      if (filterFn?.(value) ?? true) {
+        result.push({
+          type: `insert`,
+          key,
+          value,
+        })
+      }
+    }
+    return result
+  }
+
+  if (!options.where && !options.whereExpression) {
+    // No filtering, return all items
+    return collectFilteredResults()
+  }
+
+  // There's a where clause, let's see if we can use an index
+  try {
+    let expression: BasicExpression<boolean>
+
+    if (options.whereExpression) {
+      // Use the pre-compiled expression directly
+      expression = options.whereExpression
+    } else if (options.where) {
+      // Create the single-row refProxy for the callback
+      const singleRowRefProxy = createSingleRowRefProxy<T>()
+
+      // Execute the callback to get the expression
+      const whereExpression = options.where(singleRowRefProxy)
+
+      // Convert the result to a BasicExpression
+      expression = toExpression(whereExpression)
+    } else {
+      // This should never happen due to the check above, but TypeScript needs it
+      return []
+    }
+
+    // Try to optimize the query using indexes
+    const optimizationResult = optimizeExpressionWithIndexes(
+      expression,
+      collection.indexes
+    )
+
+    if (optimizationResult.canOptimize) {
+      // Use index optimization
+      const result: Array<ChangeMessage<T>> = []
+      for (const key of optimizationResult.matchingKeys) {
+        const value = collection.get(key)
+        if (value !== undefined) {
+          result.push({
+            type: `insert`,
+            key,
+            value,
+          })
+        }
+      }
+      return result
+    } else {
+      // No index found or complex expression, fall back to full scan with filter
+      const filterFn = options.where
+        ? createFilterFunction(options.where)
+        : createFilterFunctionFromExpression(expression)
+
+      return collectFilteredResults(filterFn)
+    }
+  } catch (error) {
+    // If anything goes wrong with the where clause, fall back to full scan
+    console.warn(
+      `Error processing where clause, falling back to full scan:`,
+      error
+    )
+
+    const filterFn = options.where
+      ? createFilterFunction(options.where)
+      : createFilterFunctionFromExpression(options.whereExpression!)
+
+    return collectFilteredResults(filterFn)
+  }
+}
+
+/**
+ * Creates a filter function from a where callback
+ * @param whereCallback - The callback function that defines the filter condition
+ * @returns A function that takes an item and returns true if it matches the filter
+ */
+export function createFilterFunction<T extends object>(
+  whereCallback: (row: SingleRowRefProxy<T>) => any
+): (item: T) => boolean {
+  return (item: T): boolean => {
+    try {
+      // First try the RefProxy approach for query builder functions
+      const singleRowRefProxy = createSingleRowRefProxy<T>()
+      const whereExpression = whereCallback(singleRowRefProxy)
+      const expression = toExpression(whereExpression)
+      const evaluator = compileSingleRowExpression(expression)
+      const result = evaluator(item as Record<string, unknown>)
+      // WHERE clauses should always evaluate to boolean predicates (Kevin's feedback)
+      return result
+    } catch {
+      // If RefProxy approach fails (e.g., arithmetic operations), fall back to direct evaluation
+      try {
+        // Create a simple proxy that returns actual values for arithmetic operations
+        const simpleProxy = new Proxy(item as any, {
+          get(target, prop) {
+            return target[prop]
+          },
+        }) as SingleRowRefProxy<T>
+
+        const result = whereCallback(simpleProxy)
+        return result
+      } catch {
+        // If both approaches fail, exclude the item
+        return false
+      }
+    }
+  }
+}
+
+/**
+ * Creates a filter function from a pre-compiled expression
+ * @param expression - The pre-compiled expression to evaluate
+ * @returns A function that takes an item and returns true if it matches the filter
+ */
+export function createFilterFunctionFromExpression<T extends object>(
+  expression: BasicExpression<boolean>
+): (item: T) => boolean {
+  return (item: T): boolean => {
+    try {
+      const evaluator = compileSingleRowExpression(expression)
+      const result = evaluator(item as Record<string, unknown>)
+      return Boolean(result)
+    } catch {
+      // If evaluation fails, exclude the item
+      return false
+    }
+  }
+}
+
+/**
+ * Creates a filtered callback that only calls the original callback with changes that match the where clause
+ * @param originalCallback - The original callback to filter
+ * @param options - The subscription options containing the where clause
+ * @returns A filtered callback function
+ */
+export function createFilteredCallback<T extends object>(
+  originalCallback: (changes: Array<ChangeMessage<T>>) => void,
+  options: SubscribeChangesOptions<T>
+): (changes: Array<ChangeMessage<T>>) => void {
+  const filterFn = options.whereExpression
+    ? createFilterFunctionFromExpression(options.whereExpression)
+    : createFilterFunction(options.where!)
+
+  return (changes: Array<ChangeMessage<T>>) => {
+    const filteredChanges: Array<ChangeMessage<T>> = []
+
+    for (const change of changes) {
+      if (change.type === `insert`) {
+        // For inserts, check if the new value matches the filter
+        if (filterFn(change.value)) {
+          filteredChanges.push(change)
+        }
+      } else if (change.type === `update`) {
+        // For updates, we need to check both old and new values
+        const newValueMatches = filterFn(change.value)
+        const oldValueMatches = change.previousValue
+          ? filterFn(change.previousValue)
+          : false
+
+        if (newValueMatches && oldValueMatches) {
+          // Both old and new match: emit update
+          filteredChanges.push(change)
+        } else if (newValueMatches && !oldValueMatches) {
+          // New matches but old didn't: emit insert
+          filteredChanges.push({
+            ...change,
+            type: `insert`,
+          })
+        } else if (!newValueMatches && oldValueMatches) {
+          // Old matched but new doesn't: emit delete
+          filteredChanges.push({
+            ...change,
+            type: `delete`,
+            value: change.previousValue!, // Use the previous value for the delete
+          })
+        }
+        // If neither matches, don't emit anything
+      } else {
+        // For deletes, include if the previous value would have matched
+        // (so subscribers know something they were tracking was deleted)
+        if (filterFn(change.value)) {
+          filteredChanges.push(change)
+        }
+      }
+    }
+
+    if (filteredChanges.length > 0) {
+      originalCallback(filteredChanges)
+    }
+  }
+}