@rocicorp/zero 0.25.0-canary.1 → 0.25.0-canary.11

package/out/zqlite/src/sqlite-stat-fanout.js

@@ -0,0 +1,301 @@
+class SQLiteStatFanout {
+  #db;
+  #defaultFanout;
+  /**
+   * Cache of fanout results by table and columns.
+   * Key format: "tableName:col1,col2,col3" (sorted alphabetically)
+   */
+  #cache = /* @__PURE__ */ new Map();
+  /**
+   * Prepared statements for querying SQLite statistics tables.
+   * Prepared once in constructor for performance.
+   */
+  #stat4Stmt;
+  #stat1Stmt;
+  #indexStmt;
+  /**
+   * Creates a new fanout calculator.
+   *
+   * @param db Database instance
+   * @param defaultFanout Default fanout when statistics unavailable (default: 3)
+   *                      - 1: Conservative (assumes FK relationships)
+   *                      - 3: Moderate (recommended, safe middle ground)
+   *                      - 10: SQLite's default (optimistic)
+   */
+  constructor(db, defaultFanout = 3) {
+    this.#db = db;
+    this.#defaultFanout = defaultFanout;
+    this.#stat4Stmt = this.#db.prepare(`
+      SELECT neq, nlt, ndlt, sample
+      FROM sqlite_stat4
+      WHERE tbl = ? AND idx = ?
+      ORDER BY nlt
+    `);
+    this.#stat1Stmt = this.#db.prepare(`
+      SELECT stat
+      FROM sqlite_stat1
+      WHERE tbl = ? AND idx = ?
+    `);
+    this.#indexStmt = this.#db.prepare(`
+      SELECT il.name as index_name, ii.seqno, ii.name as column_name
+      FROM pragma_index_list(?) il
+      JOIN pragma_index_info(il.name) ii
+      ORDER BY il.seq, ii.seqno
+    `);
+  }
+  /**
+   * Gets the fanout factor for join column(s).
+   *
+   * Fanout = average number of child rows per distinct parent key value(s).
+   *
+   * ## Strategy
+   *
+   * 1. Try sqlite_stat4 (best): Histogram with separate NULL/non-NULL samples
+   * 2. Fallback to sqlite_stat1: Average across all rows (includes NULLs)
+   * 3. Fallback to default: When no statistics available
+   *
+   * ## Compound Indexes
+   *
+   * For multi-column joins, finds indexes where ALL columns appear as an
+   * exact prefix. Uses the appropriate depth in stat1/stat4.
+   *
+   * Example:
+   * - Columns: `['customerId', 'storeId']`
+   * - Matches index: `(customerId, storeId, date)` at depth 2
+   * - Uses stat1 parts[2] or stat4 neq[1] for accurate fanout
+   *
+   * ## Caching
+   *
+   * Results are cached per (table, columns) combination. Clear the cache if
+   * you run ANALYZE to update statistics.
+   *
+   * @param tableName Table containing the join column(s)
+   * @param columns Array of column names (one or more columns)
+   * @returns Fanout result with value and source
+   */
+  getFanout(tableName, columns) {
+    const cacheKey = `${tableName}:${[...columns].sort().join(",")}`;
+    const cached = this.#cache.get(cacheKey);
+    if (cached) {
+      return cached;
+    }
+    const stat4Result = this.#getFanoutFromStat4(tableName, columns);
+    if (stat4Result) {
+      this.#cache.set(cacheKey, stat4Result);
+      return stat4Result;
+    }
+    const stat1Result = this.#getFanoutFromStat1(tableName, columns);
+    if (stat1Result) {
+      this.#cache.set(cacheKey, stat1Result);
+      return stat1Result;
+    }
+    const defaultResult = {
+      fanout: this.#defaultFanout,
+      confidence: "none",
+      source: "default"
+    };
+    this.#cache.set(cacheKey, defaultResult);
+    return defaultResult;
+  }
+  /**
+   * Clears the fanout cache.
+   * Call this after running ANALYZE to pick up updated statistics.
+   */
+  clearCache() {
+    this.#cache.clear();
+  }
+  /**
+   * Gets fanout from sqlite_stat4 histogram.
+   *
+   * Queries stat4 samples, decodes to identify NULLs, and returns
+   * the median fanout of non-NULL samples.
+   *
+   * For compound indexes, uses the neq value at the appropriate depth.
+   *
+   * @param columns Array of column names to get fanout for
+   * @returns Fanout result or undefined if stat4 unavailable
+   */
+  #getFanoutFromStat4(tableName, columns) {
+    try {
+      const indexInfo = this.#findIndexForColumns(tableName, columns);
+      if (!indexInfo) {
+        return void 0;
+      }
+      const samples = this.#stat4Stmt.all(
+        tableName,
+        indexInfo.indexName
+      );
+      if (samples.length === 0) {
+        return void 0;
+      }
+      const neqIndex = indexInfo.depth - 1;
+      const decodedSamples = samples.map((s) => {
+        const neqParts = s.neq.split(" ");
+        return {
+          fanout: parseInt(neqParts[neqIndex] ?? neqParts[0], 10),
+          isNull: this.#decodeSampleIsNull(s.sample)
+        };
+      });
+      const nonNullSamples = decodedSamples.filter((s) => !s.isNull);
+      if (nonNullSamples.length === 0) {
+        return {
+          fanout: 0,
+          source: "stat4",
+          confidence: "high"
+        };
+      }
+      const fanouts = nonNullSamples.map((s) => s.fanout).sort((a, b) => a - b);
+      const medianFanout = fanouts.length % 2 === 0 ? Math.floor(
+        (fanouts[fanouts.length / 2 - 1] + fanouts[fanouts.length / 2]) / 2
+      ) : fanouts[Math.floor(fanouts.length / 2)];
+      return {
+        fanout: medianFanout,
+        source: "stat4",
+        confidence: "high"
+      };
+    } catch {
+      return void 0;
+    }
+  }
+  /**
+   * Gets fanout from sqlite_stat1 average.
+   *
+   * Note: This includes NULL rows in the calculation and may overestimate
+   * fanout for sparse foreign keys.
+   *
+   * For compound indexes, uses the stat value at the appropriate depth.
+   *
+   * @param columns Array of column names to get fanout for
+   * @returns Fanout result or undefined if stat1 unavailable
+   */
+  #getFanoutFromStat1(tableName, columns) {
+    try {
+      const indexInfo = this.#findIndexForColumns(tableName, columns);
+      if (!indexInfo) {
+        return void 0;
+      }
+      const result = this.#stat1Stmt.get(tableName, indexInfo.indexName);
+      if (!result) {
+        return void 0;
+      }
+      const parts = result.stat.split(" ");
+      if (parts.length < indexInfo.depth + 1) {
+        return void 0;
+      }
+      const fanout = parseInt(parts[indexInfo.depth], 10);
+      if (isNaN(fanout)) {
+        return void 0;
+      }
+      return {
+        fanout,
+        source: "stat1",
+        confidence: "med"
+      };
+    } catch {
+      return void 0;
+    }
+  }
+  /**
+   * Finds an index that can be used to get statistics for column(s).
+   *
+   * Uses pragma_index_list and pragma_index_info to reliably get index
+   * column names, avoiding brittle SQL parsing. Includes all indices:
+   * user-created (CREATE INDEX), PRIMARY KEY, and UNIQUE constraints.
+   *
+   * Uses flexible matching: Finds indexes where ALL columns appear in the
+   * first N positions, regardless of order. This works because SQLite statistics
+   * at depth N represent the fanout for the combination of the first N columns,
+   * and combinations are order-independent.
+   *
+   * Example:
+   * - columns: ['customerId', 'storeId']
+   * - Matches: (customerId, storeId, date) at depth 2 ✅
+   * - Matches: (storeId, customerId, date) at depth 2 ✅ (flexible order)
+   * - Does NOT match: (date, customerId, storeId) ❌ (columns not in first 2 positions)
+   * - Does NOT match: (customerId, date, storeId) ❌ (storeId not in first 2 positions)
+   *
+   * @param columns Array of column names (order-independent for matching)
+   * @returns Index info with name and depth, or undefined if no match
+   */
+  #findIndexForColumns(tableName, columns) {
+    try {
+      const rows = this.#indexStmt.all(tableName);
+      const indexMap = /* @__PURE__ */ new Map();
+      for (const row of rows) {
+        const cols = indexMap.get(row.index_name) ?? [];
+        cols.push(row.column_name);
+        indexMap.set(row.index_name, cols);
+      }
+      for (const [indexName, indexColumns] of indexMap) {
+        if (this.#isPrefixMatch(columns, indexColumns)) {
+          return {
+            indexName,
+            depth: columns.length
+          };
+        }
+      }
+      return void 0;
+    } catch {
+      return void 0;
+    }
+  }
+  /**
+   * Checks if all queryColumns exist in the first N positions of indexColumns,
+   * regardless of order.
+   *
+   * This allows flexible matching: constraint {a, b} matches both index (a, b, c)
+   * and index (b, a, c) at depth 2, since both represent the fanout for the
+   * combination of columns a and b.
+   *
+   * Gaps are NOT allowed: constraint {a, c} does NOT match index (a, b, c)
+   * because no depth represents just (a, c) without b. Statistics are cumulative
+   * from position 0.
+   *
+   * @param queryColumns Columns we're looking for (from constraint)
+   * @param indexColumns Columns in the index (in order)
+   * @returns true if all queryColumns exist in indexColumns[0...queryColumns.length-1]
+   */
+  #isPrefixMatch(queryColumns, indexColumns) {
+    if (queryColumns.length > indexColumns.length) {
+      return false;
+    }
+    const indexPrefix = indexColumns.slice(0, queryColumns.length);
+    const indexPrefixLower = new Set(indexPrefix.map((col) => col.toLowerCase()));
+    const queryColumnsLower = queryColumns.map((col) => col.toLowerCase());
+    return queryColumnsLower.every((queryCol) => indexPrefixLower.has(queryCol));
+  }
+  /**
+   * Decodes a sqlite_stat4 sample value to check if it's NULL.
+   *
+   * SQLite record format (simplified):
+   * - Varint: header size
+   * - Serial types for each column (one byte each typically)
+   * - Actual data
+   *
+   * Serial type 0 = NULL
+   * Serial type 1 = 8-bit int
+   * Serial type 2 = 16-bit int
+   * Serial type 3 = 24-bit int
+   * etc.
+   *
+   * We only need to check the first column's serial type.
+   *
+   * @param sample Binary-encoded sample from stat4
+   * @returns true if the sample value is NULL
+   */
+  #decodeSampleIsNull(sample) {
+    if (sample.length === 0) {
+      return true;
+    }
+    const headerSize = sample[0];
+    if (headerSize === 0 || headerSize >= sample.length) {
+      return true;
+    }
+    const serialType = sample[1];
+    return serialType === 0;
+  }
+}
+export {
+  SQLiteStatFanout
+};
+//# sourceMappingURL=sqlite-stat-fanout.js.map

package/out/zqlite/src/sqlite-stat-fanout.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"sqlite-stat-fanout.js","sources":["../../../../zqlite/src/sqlite-stat-fanout.ts"],"sourcesContent":["import type {Database} from './db.ts';\n\n/**\n * Result of fanout calculation from SQLite statistics.\n */\nexport interface FanoutResult {\n  /**\n   * The fanout value (average rows per distinct value of the join column).\n   * For non-NULL joins, this represents how many child rows exist per parent key.\n   */\n  fanout: number;\n  confidence: 'high' | 'med' | 'none';\n\n  /**\n   * Source of the fanout calculation.\n   * - 'stat4': From sqlite_stat4 histogram (most accurate, excludes NULLs)\n   * - 'stat1': From sqlite_stat1 average (includes NULLs, may overestimate)\n   * - 'default': Fallback constant when statistics unavailable\n   */\n  source: 'stat4' | 'stat1' | 'default';\n}\n\n/**\n * Sample from sqlite_stat4 histogram.\n */\ninterface Stat4Sample {\n  /** \"N1 N2\" = rows equal to sample (N1=first col, N2=if composite) */\n  neq: string;\n  /** \"N1 N2\" = rows less than sample */\n  nlt: string;\n  /** \"N1 N2\" = distinct values less than sample */\n  ndlt: string;\n  /** The actual sample value (binary encoded) */\n  sample: Buffer;\n}\n\n/**\n * Computes join fanout factors from SQLite statistics tables.\n *\n * Fanout is the average number of child rows per distinct parent key value,\n * used to estimate join cardinality in query planning.\n *\n * ## Problem\n *\n * sqlite_stat1 includes NULL rows in its calculation, which can significantly\n * overestimate fanout for sparse foreign keys:\n *\n * ```\n * Example: 100 tasks, 20 with project_id, 80 with NULL\n * - stat1 reports: \"100 17\" → fanout = 17 (WRONG - includes NULLs)\n * - stat4 shows: NULL samples with fanout=80, non-NULL samples with fanout=4\n * - True fanout: 4 (CORRECT)\n * ```\n *\n * ## Solution\n *\n * This class uses sqlite_stat4 histogram to separate NULL and non-NULL samples,\n * providing accurate fanout for non-NULL joins.\n *\n * ## Usage\n *\n * ```typescript\n * const calculator = new SQLiteStatFanout(db);\n *\n * // Get fanout for posts.userId → users.id join\n * const result = calculator.getFanout('posts', 'userId');\n *\n * if (result.source === 'stat4') {\n *   // Accurate: excludes NULLs, samples actual distribution\n *   console.log(`Fanout: ${result.fanout} (from stat4)`);\n * } else if (result.source === 'stat1') {\n *   // Conservative: includes NULLs, may overestimate\n *   console.log(`Fanout: ${result.fanout} (from stat1, includes NULLs)`);\n * } else {\n *   // Fallback: no statistics available\n *   console.log(`Fanout: ${result.fanout} (default estimate)`);\n * }\n * ```\n *\n * ## Requirements\n *\n * - SQLite compiled with ENABLE_STAT4 (most builds include this)\n * - `ANALYZE` command run on the database\n * - Index exists on the join column\n *\n * @see https://sqlite.org/fileformat2.html#stat4tab\n * @see packages/zql/src/planner/SELECTIVITY_PLAN.md\n */\nexport class SQLiteStatFanout {\n  readonly #db: Database;\n  readonly #defaultFanout: number;\n\n  /**\n   * Cache of fanout results by table and columns.\n   * Key format: \"tableName:col1,col2,col3\" (sorted alphabetically)\n   */\n  readonly #cache = new Map<string, FanoutResult>();\n\n  /**\n   * Prepared statements for querying SQLite statistics tables.\n   * Prepared once in constructor for performance.\n   */\n  readonly #stat4Stmt: ReturnType<Database['prepare']>;\n  readonly #stat1Stmt: ReturnType<Database['prepare']>;\n  readonly #indexStmt: ReturnType<Database['prepare']>;\n\n  /**\n   * Creates a new fanout calculator.\n   *\n   * @param db Database instance\n   * @param defaultFanout Default fanout when statistics unavailable (default: 3)\n   *                      - 1: Conservative (assumes FK relationships)\n   *                      - 3: Moderate (recommended, safe middle ground)\n   *                      - 10: SQLite's default (optimistic)\n   */\n  constructor(db: Database, defaultFanout = 3) {\n    this.#db = db;\n    this.#defaultFanout = defaultFanout;\n\n    // Prepare SQL statements once for reuse across multiple getFanout() calls\n    this.#stat4Stmt = this.#db.prepare(`\n      SELECT neq, nlt, ndlt, sample\n      FROM sqlite_stat4\n      WHERE tbl = ? AND idx = ?\n      ORDER BY nlt\n    `);\n\n    this.#stat1Stmt = this.#db.prepare(`\n      SELECT stat\n      FROM sqlite_stat1\n      WHERE tbl = ? AND idx = ?\n    `);\n\n    this.#indexStmt = this.#db.prepare(`\n      SELECT il.name as index_name, ii.seqno, ii.name as column_name\n      FROM pragma_index_list(?) il\n      JOIN pragma_index_info(il.name) ii\n      ORDER BY il.seq, ii.seqno\n    `);\n  }\n\n  /**\n   * Gets the fanout factor for join column(s).\n   *\n   * Fanout = average number of child rows per distinct parent key value(s).\n   *\n   * ## Strategy\n   *\n   * 1. Try sqlite_stat4 (best): Histogram with separate NULL/non-NULL samples\n   * 2. Fallback to sqlite_stat1: Average across all rows (includes NULLs)\n   * 3. Fallback to default: When no statistics available\n   *\n   * ## Compound Indexes\n   *\n   * For multi-column joins, finds indexes where ALL columns appear as an\n   * exact prefix. Uses the appropriate depth in stat1/stat4.\n   *\n   * Example:\n   * - Columns: `['customerId', 'storeId']`\n   * - Matches index: `(customerId, storeId, date)` at depth 2\n   * - Uses stat1 parts[2] or stat4 neq[1] for accurate fanout\n   *\n   * ## Caching\n   *\n   * Results are cached per (table, columns) combination. Clear the cache if\n   * you run ANALYZE to update statistics.\n   *\n   * @param tableName Table containing the join column(s)\n   * @param columns Array of column names (one or more columns)\n   * @returns Fanout result with value and source\n   */\n  getFanout(tableName: string, columns: string[]): FanoutResult {\n    // Cache key uses sorted columns for consistency\n    const cacheKey = `${tableName}:${[...columns].sort().join(',')}`;\n    const cached = this.#cache.get(cacheKey);\n    if (cached) {\n      return cached;\n    }\n\n    // Strategy 1: Try stat4 first (most accurate)\n    // NOTE: columns are NOT sorted - preserves Object.keys() order from constraint\n    // Matching is order-independent (flexible), but we keep original order for consistency\n    const stat4Result = this.#getFanoutFromStat4(tableName, columns);\n    if (stat4Result) {\n      this.#cache.set(cacheKey, stat4Result);\n      return stat4Result;\n    }\n\n    // Strategy 2: Fallback to stat1 (includes NULLs)\n    const stat1Result = this.#getFanoutFromStat1(tableName, columns);\n    if (stat1Result) {\n      this.#cache.set(cacheKey, stat1Result);\n      return stat1Result;\n    }\n\n    // Strategy 3: Use default\n    const defaultResult: FanoutResult = {\n      fanout: this.#defaultFanout,\n      confidence: 'none',\n      source: 'default',\n    };\n    this.#cache.set(cacheKey, defaultResult);\n    return defaultResult;\n  }\n\n  /**\n   * Clears the fanout cache.\n   * Call this after running ANALYZE to pick up updated statistics.\n   */\n  clearCache(): void {\n    this.#cache.clear();\n  }\n\n  /**\n   * Gets fanout from sqlite_stat4 histogram.\n   *\n   * Queries stat4 samples, decodes to identify NULLs, and returns\n   * the median fanout of non-NULL samples.\n   *\n   * For compound indexes, uses the neq value at the appropriate depth.\n   *\n   * @param columns Array of column names to get fanout for\n   * @returns Fanout result or undefined if stat4 unavailable\n   */\n  #getFanoutFromStat4(\n    tableName: string,\n    columns: string[],\n  ): FanoutResult | undefined {\n    try {\n      // Find index containing the columns as a prefix\n      const indexInfo = this.#findIndexForColumns(tableName, columns);\n      if (!indexInfo) {\n        return undefined;\n      }\n\n      // Query stat4 samples for this index (using prepared statement)\n      const samples = this.#stat4Stmt.all(\n        tableName,\n        indexInfo.indexName,\n      ) as Stat4Sample[];\n\n      if (samples.length === 0) {\n        return undefined;\n      }\n\n      // Decode samples and separate NULL from non-NULL\n      // Use depth-1 for neq array index (depth is 1-based, array is 0-based)\n      const neqIndex = indexInfo.depth - 1;\n      const decodedSamples = samples.map(s => {\n        const neqParts = s.neq.split(' ');\n        return {\n          fanout: parseInt(neqParts[neqIndex] ?? neqParts[0], 10),\n          isNull: this.#decodeSampleIsNull(s.sample),\n        };\n      });\n\n      const nonNullSamples = decodedSamples.filter(s => !s.isNull);\n\n      if (nonNullSamples.length === 0) {\n        // All samples are NULL - return fanout of 0 since NULLs don't match in joins\n        return {\n          fanout: 0,\n          source: 'stat4',\n          confidence: 'high',\n        };\n      }\n\n      // Use median of non-NULL fanouts (more robust than average)\n      const fanouts = nonNullSamples.map(s => s.fanout).sort((a, b) => a - b);\n      const medianFanout =\n        fanouts.length % 2 === 0\n          ? Math.floor(\n              (fanouts[fanouts.length / 2 - 1] + fanouts[fanouts.length / 2]) /\n                2,\n            )\n          : fanouts[Math.floor(fanouts.length / 2)];\n\n      return {\n        fanout: medianFanout,\n        source: 'stat4',\n        confidence: 'high',\n      };\n    } catch {\n      // stat4 table may not exist or query may fail\n      return undefined;\n    }\n  }\n\n  /**\n   * Gets fanout from sqlite_stat1 average.\n   *\n   * Note: This includes NULL rows in the calculation and may overestimate\n   * fanout for sparse foreign keys.\n   *\n   * For compound indexes, uses the stat value at the appropriate depth.\n   *\n   * @param columns Array of column names to get fanout for\n   * @returns Fanout result or undefined if stat1 unavailable\n   */\n  #getFanoutFromStat1(\n    tableName: string,\n    columns: string[],\n  ): FanoutResult | undefined {\n    try {\n      // Find index containing the columns as a prefix\n      const indexInfo = this.#findIndexForColumns(tableName, columns);\n      if (!indexInfo) {\n        return undefined;\n      }\n\n      // Query stat1 for this index (using prepared statement)\n      const result = this.#stat1Stmt.get(tableName, indexInfo.indexName) as\n        | {stat: string}\n        | undefined;\n\n      if (!result) {\n        return undefined;\n      }\n\n      const parts = result.stat.split(' ');\n      // Check if we have enough parts for the requested depth\n      if (parts.length < indexInfo.depth + 1) {\n        return undefined;\n      }\n\n      const fanout = parseInt(parts[indexInfo.depth], 10);\n      if (isNaN(fanout)) {\n        return undefined;\n      }\n\n      return {\n        fanout,\n        source: 'stat1',\n        confidence: 'med',\n      };\n    } catch {\n      return undefined;\n    }\n  }\n\n  /**\n   * Finds an index that can be used to get statistics for column(s).\n   *\n   * Uses pragma_index_list and pragma_index_info to reliably get index\n   * column names, avoiding brittle SQL parsing. Includes all indices:\n   * user-created (CREATE INDEX), PRIMARY KEY, and UNIQUE constraints.\n   *\n   * Uses flexible matching: Finds indexes where ALL columns appear in the\n   * first N positions, regardless of order. This works because SQLite statistics\n   * at depth N represent the fanout for the combination of the first N columns,\n   * and combinations are order-independent.\n   *\n   * Example:\n   * - columns: ['customerId', 'storeId']\n   * - Matches: (customerId, storeId, date) at depth 2 ✅\n   * - Matches: (storeId, customerId, date) at depth 2 ✅ (flexible order)\n   * - Does NOT match: (date, customerId, storeId) ❌ (columns not in first 2 positions)\n   * - Does NOT match: (customerId, date, storeId) ❌ (storeId not in first 2 positions)\n   *\n   * @param columns Array of column names (order-independent for matching)\n   * @returns Index info with name and depth, or undefined if no match\n   */\n  #findIndexForColumns(\n    tableName: string,\n    columns: string[],\n  ): {indexName: string; depth: number} | undefined {\n    try {\n      // Query returns all columns for all indexes (including PK/UNIQUE) in order\n      const rows = this.#indexStmt.all(tableName) as {\n        index_name: string;\n        seqno: number;\n        column_name: string;\n      }[];\n\n      // Group by index name\n      const indexMap = new Map<string, string[]>();\n      for (const row of rows) {\n        const cols = indexMap.get(row.index_name) ?? [];\n        cols.push(row.column_name);\n        indexMap.set(row.index_name, cols);\n      }\n\n      // Check each index for prefix match\n      for (const [indexName, indexColumns] of indexMap) {\n        if (this.#isPrefixMatch(columns, indexColumns)) {\n          return {\n            indexName,\n            depth: columns.length,\n          };\n        }\n      }\n\n      return undefined;\n    } catch {\n      return undefined;\n    }\n  }\n\n  /**\n   * Checks if all queryColumns exist in the first N positions of indexColumns,\n   * regardless of order.\n   *\n   * This allows flexible matching: constraint {a, b} matches both index (a, b, c)\n   * and index (b, a, c) at depth 2, since both represent the fanout for the\n   * combination of columns a and b.\n   *\n   * Gaps are NOT allowed: constraint {a, c} does NOT match index (a, b, c)\n   * because no depth represents just (a, c) without b. Statistics are cumulative\n   * from position 0.\n   *\n   * @param queryColumns Columns we're looking for (from constraint)\n   * @param indexColumns Columns in the index (in order)\n   * @returns true if all queryColumns exist in indexColumns[0...queryColumns.length-1]\n   */\n  #isPrefixMatch(queryColumns: string[], indexColumns: string[]): boolean {\n    if (queryColumns.length > indexColumns.length) {\n      return false;\n    }\n\n    // Get the prefix of the index that we're checking against\n    const indexPrefix = indexColumns.slice(0, queryColumns.length);\n\n    // Normalize to lowercase for case-insensitive comparison\n    const indexPrefixLower = new Set(indexPrefix.map(col => col.toLowerCase()));\n    const queryColumnsLower = queryColumns.map(col => col.toLowerCase());\n\n    // Check if ALL query columns exist in the index prefix\n    return queryColumnsLower.every(queryCol => indexPrefixLower.has(queryCol));\n  }\n\n  /**\n   * Decodes a sqlite_stat4 sample value to check if it's NULL.\n   *\n   * SQLite record format (simplified):\n   * - Varint: header size\n   * - Serial types for each column (one byte each typically)\n   * - Actual data\n   *\n   * Serial type 0 = NULL\n   * Serial type 1 = 8-bit int\n   * Serial type 2 = 16-bit int\n   * Serial type 3 = 24-bit int\n   * etc.\n   *\n   * We only need to check the first column's serial type.\n   *\n   * @param sample Binary-encoded sample from stat4\n   * @returns true if the sample value is NULL\n   */\n  #decodeSampleIsNull(sample: Buffer): boolean {\n    if (sample.length === 0) {\n      return true;\n    }\n\n    // Read header size (varint - simplified: assume single byte)\n    const headerSize = sample[0];\n\n    if (headerSize === 0 || headerSize >= sample.length) {\n      return true;\n    }\n\n    // Read first serial type (at position 1)\n    const serialType = sample[1];\n\n    // Serial type 0 = NULL\n    return serialType === 0;\n  }\n}\n"],"names":[],"mappings":"AAwFO,MAAM,iBAAiB;AAAA,EACnB;AAAA,EACA;AAAA;AAAA;AAAA;AAAA;AAAA,EAMA,6BAAa,IAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAMb;AAAA,EACA;AAAA,EACA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAWT,YAAY,IAAc,gBAAgB,GAAG;AAC3C,SAAK,MAAM;AACX,SAAK,iBAAiB;AAGtB,SAAK,aAAa,KAAK,IAAI,QAAQ;AAAA;AAAA;AAAA;AAAA;AAAA,KAKlC;AAED,SAAK,aAAa,KAAK,IAAI,QAAQ;AAAA;AAAA;AAAA;AAAA,KAIlC;AAED,SAAK,aAAa,KAAK,IAAI,QAAQ;AAAA;AAAA;AAAA;AAAA;AAAA,KAKlC;AAAA,EACH;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAgCA,UAAU,WAAmB,SAAiC;AAE5D,UAAM,WAAW,GAAG,SAAS,IAAI,CAAC,GAAG,OAAO,EAAE,KAAA,EAAO,KAAK,GAAG,CAAC;AAC9D,UAAM,SAAS,KAAK,OAAO,IAAI,QAAQ;AACvC,QAAI,QAAQ;AACV,aAAO;AAAA,IACT;AAKA,UAAM,cAAc,KAAK,oBAAoB,WAAW,OAAO;AAC/D,QAAI,aAAa;AACf,WAAK,OAAO,IAAI,UAAU,WAAW;AACrC,aAAO;AAAA,IACT;AAGA,UAAM,cAAc,KAAK,oBAAoB,WAAW,OAAO;AAC/D,QAAI,aAAa;AACf,WAAK,OAAO,IAAI,UAAU,WAAW;AACrC,aAAO;AAAA,IACT;AAGA,UAAM,gBAA8B;AAAA,MAClC,QAAQ,KAAK;AAAA,MACb,YAAY;AAAA,MACZ,QAAQ;AAAA,IAAA;AAEV,SAAK,OAAO,IAAI,UAAU,aAAa;AACvC,WAAO;AAAA,EACT;AAAA;AAAA;AAAA;AAAA;AAAA,EAMA,aAAmB;AACjB,SAAK,OAAO,MAAA;AAAA,EACd;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAaA,oBACE,WACA,SAC0B;AAC1B,QAAI;AAEF,YAAM,YAAY,KAAK,qBAAqB,WAAW,OAAO;AAC9D,UAAI,CAAC,WAAW;AACd,eAAO;AAAA,MACT;AAGA,YAAM,UAAU,KAAK,WAAW;AAAA,QAC9B;AAAA,QACA,UAAU;AAAA,MAAA;AAGZ,UAAI,QAAQ,WAAW,GAAG;AACxB,eAAO;AAAA,MACT;AAIA,YAAM,WAAW,UAAU,QAAQ;AACnC,YAAM,iBAAiB,QAAQ,IAAI,CAAA,MAAK;AACtC,cAAM,WAAW,EAAE,IAAI,MAAM,GAAG;AAChC,eAAO;AAAA,UACL,QAAQ,SAAS,SAAS,QAAQ,KAAK,SAAS,CAAC,GAAG,EAAE;AAAA,UACtD,QAAQ,KAAK,oBAAoB,EAAE,MAAM;AAAA,QAAA;AAAA,MAE7C,CAAC;AAED,YAAM,iBAAiB,eAAe,OAAO,CAAA,MAAK,CAAC,EAAE,MAAM;AAE3D,UAAI,eAAe,WAAW,GAAG;AAE/B,eAAO;AAAA,UACL,QAAQ;AAAA,UACR,QAAQ;AAAA,UACR,YAAY;AAAA,QAAA;AAAA,MAEhB;AAGA,YAAM,UAAU,eAAe,IAAI,CAAA,MAAK,EAAE,MAAM,EAAE,KAAK,CAAC,GAAG,MAAM,IAAI,CAAC;AACtE,YAAM,eACJ,QAAQ,SAAS,MAAM,IACnB,KAAK;AAAA,SACF,QAAQ,QAAQ,SAAS,IAAI,CAAC,IAAI,QAAQ,QAAQ,SAAS,CAAC,KAC3D;AAAA,MAAA,IAEJ,QAAQ,KAAK,MAAM,QAAQ,SAAS,CAAC,CAAC;AAE5C,aAAO;AAAA,QACL,QAAQ;AAAA,QACR,QAAQ;AAAA,QACR,YAAY;AAAA,MAAA;AAAA,IAEhB,QAAQ;AAEN,aAAO;AAAA,IACT;AAAA,EACF;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAaA,oBACE,WACA,SAC0B;AAC1B,QAAI;AAEF,YAAM,YAAY,KAAK,qBAAqB,WAAW,OAAO;AAC9D,UAAI,CAAC,WAAW;AACd,eAAO;AAAA,MACT;AAGA,YAAM,SAAS,KAAK,WAAW,IAAI,WAAW,UAAU,SAAS;AAIjE,UAAI,CAAC,QAAQ;AACX,eAAO;AAAA,MACT;AAEA,YAAM,QAAQ,OAAO,KAAK,MAAM,GAAG;AAEnC,UAAI,MAAM,SAAS,UAAU,QAAQ,GAAG;AACtC,eAAO;AAAA,MACT;AAEA,YAAM,SAAS,SAAS,MAAM,UAAU,KAAK,GAAG,EAAE;AAClD,UAAI,MAAM,MAAM,GAAG;AACjB,eAAO;AAAA,MACT;AAEA,aAAO;AAAA,QACL;AAAA,QACA,QAAQ;AAAA,QACR,YAAY;AAAA,MAAA;AAAA,IAEhB,QAAQ;AACN,aAAO;AAAA,IACT;AAAA,EACF;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAwBA,qBACE,WACA,SACgD;AAChD,QAAI;AAEF,YAAM,OAAO,KAAK,WAAW,IAAI,SAAS;AAO1C,YAAM,+BAAe,IAAA;AACrB,iBAAW,OAAO,MAAM;AACtB,cAAM,OAAO,SAAS,IAAI,IAAI,UAAU,KAAK,CAAA;AAC7C,aAAK,KAAK,IAAI,WAAW;AACzB,iBAAS,IAAI,IAAI,YAAY,IAAI;AAAA,MACnC;AAGA,iBAAW,CAAC,WAAW,YAAY,KAAK,UAAU;AAChD,YAAI,KAAK,eAAe,SAAS,YAAY,GAAG;AAC9C,iBAAO;AAAA,YACL;AAAA,YACA,OAAO,QAAQ;AAAA,UAAA;AAAA,QAEnB;AAAA,MACF;AAEA,aAAO;AAAA,IACT,QAAQ;AACN,aAAO;AAAA,IACT;AAAA,EACF;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAkBA,eAAe,cAAwB,cAAiC;AACtE,QAAI,aAAa,SAAS,aAAa,QAAQ;AAC7C,aAAO;AAAA,IACT;AAGA,UAAM,cAAc,aAAa,MAAM,GAAG,aAAa,MAAM;AAG7D,UAAM,mBAAmB,IAAI,IAAI,YAAY,IAAI,CAAA,QAAO,IAAI,YAAA,CAAa,CAAC;AAC1E,UAAM,oBAAoB,aAAa,IAAI,CAAA,QAAO,IAAI,aAAa;AAGnE,WAAO,kBAAkB,MAAM,CAAA,aAAY,iBAAiB,IAAI,QAAQ,CAAC;AAAA,EAC3E;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAqBA,oBAAoB,QAAyB;AAC3C,QAAI,OAAO,WAAW,GAAG;AACvB,aAAO;AAAA,IACT;AAGA,UAAM,aAAa,OAAO,CAAC;AAE3B,QAAI,eAAe,KAAK,cAAc,OAAO,QAAQ;AACnD,aAAO;AAAA,IACT;AAGA,UAAM,aAAa,OAAO,CAAC;AAG3B,WAAO,eAAe;AAAA,EACxB;AACF;"}

package/out/zqlite/src/table-source.d.ts

@@ -2,10 +2,11 @@ import type { LogContext } from '@rocicorp/logger';
 import type { LogConfig } from '../../otel/src/log-options.ts';
 import type { Condition, Ordering } from '../../zero-protocol/src/ast.ts';
 import type { Row } from '../../zero-protocol/src/data.ts';
+import type { PrimaryKey } from '../../zero-protocol/src/primary-key.ts';
 import type { SchemaValue, ValueType } from '../../zero-schema/src/table-schema.ts';
 import type { DebugDelegate } from '../../zql/src/builder/debug-delegate.ts';
-import type { Source, SourceChange, SourceInput } from '../../zql/src/ivm/source.ts';
-import { Database } from './db.ts';
+import { type Source, type SourceChange, type SourceInput } from '../../zql/src/ivm/source.ts';
+import type { Database } from './db.ts';
 /**
  * A source that is backed by a SQLite table.
  *
@@ -22,8 +23,18 @@ import { Database } from './db.ts';
  */
 export declare class TableSource implements Source {
     #private;
-    constructor(logContext: LogContext, logConfig: LogConfig, db: Database, tableName: string, columns: Record<string, SchemaValue>, primaryKey: readonly [string, ...string[]]);
-    get table(): string;
+    /**
+     * @param shouldYield a function called after each row is read from the database,
+     * which should return true if the source should yield the special 'yield' value
+     * to yield control back to the caller at the end of the pipeline.  Can
+     * also throw an error to abort the pipeline processing.
+     */
+    constructor(logContext: LogContext, logConfig: LogConfig, db: Database, tableName: string, columns: Record<string, SchemaValue>, primaryKey: PrimaryKey, shouldYield?: () => boolean);
+    get tableSchema(): {
+        name: string;
+        columns: Record<string, SchemaValue>;
+        primaryKey: readonly [string, ...string[]];
+    };
     /**
      * Sets the db (snapshot) to use, to facilitate the Snapshotter leapfrog
      * algorithm for concurrent traversal of historic timelines.

package/out/zqlite/src/table-source.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"table-source.d.ts","sourceRoot":"","sources":["../../../../zqlite/src/table-source.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AACjD,OAAO,KAAK,EAAC,SAAS,EAAC,MAAM,+BAA+B,CAAC;AAM7D,OAAO,KAAK,EAAC,SAAS,EAAE,QAAQ,EAAC,MAAM,gCAAgC,CAAC;AACxE,OAAO,KAAK,EAAC,GAAG,EAAQ,MAAM,iCAAiC,CAAC;AAEhE,OAAO,KAAK,EACV,WAAW,EACX,SAAS,EACV,MAAM,uCAAuC,CAAC;AAC/C,OAAO,KAAK,EAAC,aAAa,EAAC,MAAM,yCAAyC,CAAC;AAoB3E,OAAO,KAAK,EACV,MAAM,EACN,YAAY,EACZ,WAAW,EACZ,MAAM,6BAA6B,CAAC;AAErC,OAAO,EAAC,QAAQ,EAAY,MAAM,SAAS,CAAC;AAe5C;;;;;;;;;;;;;GAaG;AACH,qBAAa,WAAY,YAAW,MAAM;;gBActC,UAAU,EAAE,UAAU,EACtB,SAAS,EAAE,SAAS,EACpB,EAAE,EAAE,QAAQ,EACZ,SAAS,EAAE,MAAM,EACjB,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,WAAW,CAAC,EACpC,UAAU,EAAE,SAAS,CAAC,MAAM,EAAE,GAAG,MAAM,EAAE,CAAC;IAgB5C,IAAI,KAAK,WAER;IAED;;;OAGG;IACH,KAAK,CAAC,EAAE,EAAE,QAAQ;IA4FlB,OAAO,CACL,IAAI,EAAE,QAAQ,EACd,OAAO,CAAC,EAAE,SAAS,EACnB,aAAa,CAAC,EAAE,GAAG,CAAC,MAAM,CAAC,EAC3B,KAAK,CAAC,EAAE,aAAa;IAsCvB,WAAW,CAAC,GAAG,EAAE,GAAG,GAAG,GAAG;IAsF1B,IAAI,CAAC,MAAM,EAAE,YAAY,GAAG,IAAI;IAM/B,OAAO,CAAC,MAAM,EAAE,YAAY;IA2F7B;;;;;;OAMG;IACH,MAAM,CAAC,MAAM,EAAE,GAAG,GAAG,GAAG,GAAG,SAAS;CA+CrC;AA6BD,wBAAgB,aAAa,CAC3B,OAAO,EAAE,SAAS,MAAM,EAAE,EAC1B,GAAG,EAAE,GAAG,EACR,WAAW,EAAE,MAAM,CAAC,MAAM,EAAE,WAAW,CAAC,GACvC,SAAS,OAAO,EAAE,CAEpB;AAED,wBAAgB,gBAAgB,CAAC,IAAI,EAAE,SAAS,wCAa/C;AAED,wBAAgB,eAAe,CAC7B,UAAU,EAAE,MAAM,CAAC,MAAM,EAAE,WAAW,CAAC,EACvC,GAAG,EAAE,GAAG,EACR,SAAS,EAAE,MAAM,GAChB,GAAG,CAaL;AAwCD,qBAAa,qBAAsB,SAAQ,KAAK;CAAG"}
+{"version":3,"file":"table-source.d.ts","sourceRoot":"","sources":["../../../../zqlite/src/table-source.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAEjD,OAAO,KAAK,EAAC,SAAS,EAAC,MAAM,+BAA+B,CAAC;AAK7D,OAAO,KAAK,EAAC,SAAS,EAAE,QAAQ,EAAC,MAAM,gCAAgC,CAAC;AACxE,OAAO,KAAK,EAAC,GAAG,EAAQ,MAAM,iCAAiC,CAAC;AAChE,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,wCAAwC,CAAC;AACvE,OAAO,KAAK,EACV,WAAW,EACX,SAAS,EACV,MAAM,uCAAuC,CAAC;AAC/C,OAAO,KAAK,EAAC,aAAa,EAAC,MAAM,yCAAyC,CAAC;AAe3E,OAAO,EACL,KAAK,MAAM,EACX,KAAK,YAAY,EACjB,KAAK,WAAW,EACjB,MAAM,6BAA6B,CAAC;AAErC,OAAO,KAAK,EAAC,QAAQ,EAAY,MAAM,SAAS,CAAC;AAqBjD;;;;;;;;;;;;;GAaG;AACH,qBAAa,WAAY,YAAW,MAAM;;IAcxC;;;;;OAKG;gBAED,UAAU,EAAE,UAAU,EACtB,SAAS,EAAE,SAAS,EACpB,EAAE,EAAE,QAAQ,EACZ,SAAS,EAAE,MAAM,EACjB,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,WAAW,CAAC,EACpC,UAAU,EAAE,UAAU,EACtB,WAAW,gBAAc;IAiB3B,IAAI,WAAW;;;;MAMd;IAED;;;OAGG;IACH,KAAK,CAAC,EAAE,EAAE,QAAQ;IA4FlB,OAAO,CACL,IAAI,EAAE,QAAQ,EACd,OAAO,CAAC,EAAE,SAAS,EACnB,aAAa,CAAC,EAAE,GAAG,CAAC,MAAM,CAAC,EAC3B,KAAK,CAAC,EAAE,aAAa;IAuCvB,WAAW,CAAC,GAAG,EAAE,GAAG,GAAG,GAAG;IA4G1B,IAAI,CAAC,MAAM,EAAE,YAAY,GAAG,IAAI;IAM/B,OAAO,CAAC,MAAM,EAAE,YAAY;IA2F7B;;;;;;OAMG;IACH,MAAM,CAAC,MAAM,EAAE,GAAG,GAAG,GAAG,GAAG,SAAS;CA6BrC;AA6BD,wBAAgB,aAAa,CAC3B,OAAO,EAAE,SAAS,MAAM,EAAE,EAC1B,GAAG,EAAE,GAAG,EACR,WAAW,EAAE,MAAM,CAAC,MAAM,EAAE,WAAW,CAAC,GACvC,SAAS,OAAO,EAAE,CAEpB;AAED,wBAAgB,gBAAgB,CAAC,IAAI,EAAE,SAAS,wCAa/C;AAED,wBAAgB,eAAe,CAC7B,UAAU,EAAE,MAAM,CAAC,MAAM,EAAE,WAAW,CAAC,EACvC,GAAG,EAAE,GAAG,EACR,SAAS,EAAE,MAAM,GAChB,GAAG,CAaL;AAwCD,qBAAa,qBAAsB,SAAQ,KAAK;CAAG"}