@rocicorp/zero 1.11.0-canary.2 → 1.11.0-canary.21

package/out/zqlite/src/sqlite-stat-fanout.js

@@ -1,3 +1,6 @@
+import { getOrInsertComputed } from "../../shared/src/map.js";
+import { newArray } from "../../shared/src/arrays.js";
+import { isSampleNull } from "./sqlite-stat4-sample.js";
 //#region ../zqlite/src/sqlite-stat-fanout.ts
 /**
 * Computes join fanout factors from SQLite statistics tables.
@@ -178,7 +181,7 @@ var SQLiteStatFanout = class {
 				const neqParts = s.neq.split(" ");
 				return {
 					fanout: parseInt(neqParts[neqIndex] ?? neqParts[0], 10),
-					isNull: this.#decodeSampleIsNull(s.sample)
+					isNull: isSampleNull(s.sample)
 				};
 			}).filter((s) => !s.isNull);
 			if (nonNullSamples.length === 0) return {
@@ -252,11 +255,7 @@ var SQLiteStatFanout = class {
 		try {
 			const rows = this.#indexStmt.all(tableName);
 			const indexMap = /* @__PURE__ */ new Map();
-			for (const row of rows) {
-				const cols = indexMap.get(row.index_name) ?? [];
-				cols.push(row.column_name);
-				indexMap.set(row.index_name, cols);
-			}
+			for (const row of rows) getOrInsertComputed(indexMap, row.index_name, newArray).push(row.column_name);
 			for (const [indexName, indexColumns] of indexMap) if (this.#isPrefixMatch(columns, indexColumns)) return {
 				indexName,
 				depth: columns.length
@@ -288,31 +287,6 @@ var SQLiteStatFanout = class {
 		const indexPrefixLower = new Set(indexPrefix.map((col) => col.toLowerCase()));
 		return queryColumns.map((col) => col.toLowerCase()).every((queryCol) => indexPrefixLower.has(queryCol));
 	}
-	/**
-	* Decodes a sqlite_stat4 sample value to check if it's NULL.
-	*
-	* SQLite record format (simplified):
-	* - Varint: header size
-	* - Serial types for each column (one byte each typically)
-	* - Actual data
-	*
-	* Serial type 0 = NULL
-	* Serial type 1 = 8-bit int
-	* Serial type 2 = 16-bit int
-	* Serial type 3 = 24-bit int
-	* etc.
-	*
-	* We only need to check the first column's serial type.
-	*
-	* @param sample Binary-encoded sample from stat4
-	* @returns true if the sample value is NULL
-	*/
-	#decodeSampleIsNull(sample) {
-		if (sample.length === 0) return true;
-		const headerSize = sample[0];
-		if (headerSize === 0 || headerSize >= sample.length) return true;
-		return sample[1] === 0;
-	}
 };
 //#endregion
 export { SQLiteStatFanout };

package/out/zqlite/src/sqlite-stat-fanout.js.map

@@ -1 +1 @@
-{"version":3,"file":"sqlite-stat-fanout.js","names":["#db","#defaultFanout","#cache","#stat4Stmt","#stat1Stmt","#indexStmt","#getFanoutFromStat4","#getFanoutFromStat1","#findIndexForColumns","#decodeSampleIsNull","#isPrefixMatch"],"sources":["../../../../zqlite/src/sqlite-stat-fanout.ts"],"sourcesContent":["import type {Database} from './db.ts';\n\n/**\n * Result of fanout calculation from SQLite statistics.\n */\nexport interface FanoutResult {\n  /**\n   * The fanout value (average rows per distinct value of the join column).\n   * For non-NULL joins, this represents how many child rows exist per parent key.\n   */\n  fanout: number;\n  confidence: 'high' | 'med' | 'none';\n\n  /**\n   * Source of the fanout calculation.\n   * - 'stat4': From sqlite_stat4 histogram (most accurate, excludes NULLs)\n   * - 'stat1': From sqlite_stat1 average (includes NULLs, may overestimate)\n   * - 'default': Fallback constant when statistics unavailable\n   */\n  source: 'stat4' | 'stat1' | 'default';\n}\n\n/**\n * Sample from sqlite_stat4 histogram.\n */\ninterface Stat4Sample {\n  /** \"N1 N2\" = rows equal to sample (N1=first col, N2=if composite) */\n  neq: string;\n  /** \"N1 N2\" = rows less than sample */\n  nlt: string;\n  /** \"N1 N2\" = distinct values less than sample */\n  ndlt: string;\n  /** The actual sample value (binary encoded) */\n  sample: Buffer;\n}\n\n/**\n * Computes join fanout factors from SQLite statistics tables.\n *\n * Fanout is the average number of child rows per distinct parent key value,\n * used to estimate join cardinality in query planning.\n *\n * ## Problem\n *\n * sqlite_stat1 includes NULL rows in its calculation, which can significantly\n * overestimate fanout for sparse foreign keys:\n *\n * ```\n * Example: 100 tasks, 20 with project_id, 80 with NULL\n * - stat1 reports: \"100 17\" → fanout = 17 (WRONG - includes NULLs)\n * - stat4 shows: NULL samples with fanout=80, non-NULL samples with fanout=4\n * - True fanout: 4 (CORRECT)\n * ```\n *\n * ## Solution\n *\n * This class uses sqlite_stat4 histogram to separate NULL and non-NULL samples,\n * providing accurate fanout for non-NULL joins.\n *\n * ## Usage\n *\n * ```typescript\n * const calculator = new SQLiteStatFanout(db);\n *\n * // Get fanout for posts.userId → users.id join\n * const result = calculator.getFanout('posts', 'userId');\n *\n * if (result.source === 'stat4') {\n *   // Accurate: excludes NULLs, samples actual distribution\n *   console.log(`Fanout: ${result.fanout} (from stat4)`);\n * } else if (result.source === 'stat1') {\n *   // Conservative: includes NULLs, may overestimate\n *   console.log(`Fanout: ${result.fanout} (from stat1, includes NULLs)`);\n * } else {\n *   // Fallback: no statistics available\n *   console.log(`Fanout: ${result.fanout} (default estimate)`);\n * }\n * ```\n *\n * ## Requirements\n *\n * - SQLite compiled with ENABLE_STAT4 (most builds include this)\n * - `ANALYZE` command run on the database\n * - Index exists on the join column\n *\n * @see https://sqlite.org/fileformat2.html#stat4tab\n * @see packages/zql/src/planner/SELECTIVITY_PLAN.md\n */\nexport class SQLiteStatFanout {\n  readonly #db: Database;\n  readonly #defaultFanout: number;\n\n  /**\n   * Cache of fanout results by table and columns.\n   * Key format: \"tableName:col1,col2,col3\" (sorted alphabetically)\n   */\n  readonly #cache = new Map<string, FanoutResult>();\n\n  /**\n   * Prepared statements for querying SQLite statistics tables.\n   * Prepared once in constructor for performance.\n   */\n  readonly #stat4Stmt: ReturnType<Database['prepare']>;\n  readonly #stat1Stmt: ReturnType<Database['prepare']>;\n  readonly #indexStmt: ReturnType<Database['prepare']>;\n\n  /**\n   * Creates a new fanout calculator.\n   *\n   * @param db Database instance\n   * @param defaultFanout Default fanout when statistics unavailable (default: 3)\n   *                      - 1: Conservative (assumes FK relationships)\n   *                      - 3: Moderate (recommended, safe middle ground)\n   *                      - 10: SQLite's default (optimistic)\n   */\n  constructor(db: Database, defaultFanout = 3) {\n    this.#db = db;\n    this.#defaultFanout = defaultFanout;\n\n    // Prepare SQL statements once for reuse across multiple getFanout() calls\n    this.#stat4Stmt = this.#db.prepare(`\n      SELECT neq, nlt, ndlt, sample\n      FROM sqlite_stat4\n      WHERE tbl = ? AND idx = ?\n      ORDER BY nlt\n    `);\n\n    this.#stat1Stmt = this.#db.prepare(`\n      SELECT stat\n      FROM sqlite_stat1\n      WHERE tbl = ? AND idx = ?\n    `);\n\n    this.#indexStmt = this.#db.prepare(`\n      SELECT il.name as index_name, ii.seqno, ii.name as column_name\n      FROM pragma_index_list(?) il\n      JOIN pragma_index_info(il.name) ii\n      WHERE il.partial = 0\n      ORDER BY il.seq, ii.seqno\n    `);\n  }\n\n  /**\n   * Gets the fanout factor for join column(s).\n   *\n   * Fanout = average number of child rows per distinct parent key value(s).\n   *\n   * ## Strategy\n   *\n   * 1. Try sqlite_stat4 (best): Histogram with separate NULL/non-NULL samples\n   * 2. Fallback to sqlite_stat1: Average across all rows (includes NULLs)\n   * 3. Fallback to default: When no statistics available\n   *\n   * ## Compound Indexes\n   *\n   * For multi-column joins, finds indexes where ALL columns appear as an\n   * exact prefix. Uses the appropriate depth in stat1/stat4.\n   *\n   * Example:\n   * - Columns: `['customerId', 'storeId']`\n   * - Matches index: `(customerId, storeId, date)` at depth 2\n   * - Uses stat1 parts[2] or stat4 neq[1] for accurate fanout\n   *\n   * ## Caching\n   *\n   * Results are cached per (table, columns) combination. Clear the cache if\n   * you run ANALYZE to update statistics.\n   *\n   * @param tableName Table containing the join column(s)\n   * @param columns Array of column names (one or more columns)\n   * @returns Fanout result with value and source\n   */\n  getFanout(tableName: string, columns: string[]): FanoutResult {\n    // Cache key uses sorted columns for consistency\n    const cacheKey = `${tableName}:${columns.toSorted().join(',')}`;\n    const cached = this.#cache.get(cacheKey);\n    if (cached) {\n      return cached;\n    }\n\n    // Strategy 1: Try stat4 first (most accurate)\n    // NOTE: columns are NOT sorted - preserves Object.keys() order from constraint\n    // Matching is order-independent (flexible), but we keep original order for consistency\n    const stat4Result = this.#getFanoutFromStat4(tableName, columns);\n    if (stat4Result) {\n      this.#cache.set(cacheKey, stat4Result);\n      return stat4Result;\n    }\n\n    // Strategy 2: Fallback to stat1 (includes NULLs)\n    const stat1Result = this.#getFanoutFromStat1(tableName, columns);\n    if (stat1Result) {\n      this.#cache.set(cacheKey, stat1Result);\n      return stat1Result;\n    }\n\n    // Strategy 3: Use default\n    const defaultResult: FanoutResult = {\n      fanout: this.#defaultFanout,\n      confidence: 'none',\n      source: 'default',\n    };\n    this.#cache.set(cacheKey, defaultResult);\n    return defaultResult;\n  }\n\n  /**\n   * Clears the fanout cache.\n   * Call this after running ANALYZE to pick up updated statistics.\n   */\n  clearCache(): void {\n    this.#cache.clear();\n  }\n\n  /**\n   * Gets fanout from sqlite_stat4 histogram.\n   *\n   * Queries stat4 samples, decodes to identify NULLs, and returns\n   * the median fanout of non-NULL samples.\n   *\n   * For compound indexes, uses the neq value at the appropriate depth.\n   *\n   * @param columns Array of column names to get fanout for\n   * @returns Fanout result or undefined if stat4 unavailable\n   */\n  #getFanoutFromStat4(\n    tableName: string,\n    columns: string[],\n  ): FanoutResult | undefined {\n    try {\n      // Find index containing the columns as a prefix\n      const indexInfo = this.#findIndexForColumns(tableName, columns);\n      if (!indexInfo) {\n        return undefined;\n      }\n\n      // Query stat4 samples for this index (using prepared statement)\n      const samples = this.#stat4Stmt.all(\n        tableName,\n        indexInfo.indexName,\n      ) as Stat4Sample[];\n\n      if (samples.length === 0) {\n        return undefined;\n      }\n\n      // Decode samples and separate NULL from non-NULL\n      // Use depth-1 for neq array index (depth is 1-based, array is 0-based)\n      const neqIndex = indexInfo.depth - 1;\n      const decodedSamples = samples.map(s => {\n        const neqParts = s.neq.split(' ');\n        return {\n          fanout: parseInt(neqParts[neqIndex] ?? neqParts[0], 10),\n          isNull: this.#decodeSampleIsNull(s.sample),\n        };\n      });\n\n      const nonNullSamples = decodedSamples.filter(s => !s.isNull);\n\n      if (nonNullSamples.length === 0) {\n        // All samples are NULL - return fanout of 0 since NULLs don't match in joins\n        return {\n          fanout: 0,\n          source: 'stat4',\n          confidence: 'high',\n        };\n      }\n\n      // Use median of non-NULL fanouts (more robust than average)\n      const fanouts = nonNullSamples.map(s => s.fanout).sort((a, b) => a - b);\n      const medianFanout =\n        fanouts.length % 2 === 0\n          ? Math.floor(\n              (fanouts[fanouts.length / 2 - 1] + fanouts[fanouts.length / 2]) /\n                2,\n            )\n          : fanouts[Math.floor(fanouts.length / 2)];\n\n      return {\n        fanout: medianFanout,\n        source: 'stat4',\n        confidence: 'high',\n      };\n    } catch {\n      // stat4 table may not exist or query may fail\n      return undefined;\n    }\n  }\n\n  /**\n   * Gets fanout from sqlite_stat1 average.\n   *\n   * Note: This includes NULL rows in the calculation and may overestimate\n   * fanout for sparse foreign keys.\n   *\n   * For compound indexes, uses the stat value at the appropriate depth.\n   *\n   * @param columns Array of column names to get fanout for\n   * @returns Fanout result or undefined if stat1 unavailable\n   */\n  #getFanoutFromStat1(\n    tableName: string,\n    columns: string[],\n  ): FanoutResult | undefined {\n    try {\n      // Find index containing the columns as a prefix\n      const indexInfo = this.#findIndexForColumns(tableName, columns);\n      if (!indexInfo) {\n        return undefined;\n      }\n\n      // Query stat1 for this index (using prepared statement)\n      const result = this.#stat1Stmt.get(tableName, indexInfo.indexName) as\n        | {stat: string}\n        | undefined;\n\n      if (!result) {\n        return undefined;\n      }\n\n      const parts = result.stat.split(' ');\n      // Check if we have enough parts for the requested depth\n      if (parts.length < indexInfo.depth + 1) {\n        return undefined;\n      }\n\n      const fanout = parseInt(parts[indexInfo.depth], 10);\n      if (isNaN(fanout)) {\n        return undefined;\n      }\n\n      return {\n        fanout,\n        source: 'stat1',\n        confidence: 'med',\n      };\n    } catch {\n      return undefined;\n    }\n  }\n\n  /**\n   * Finds an index that can be used to get statistics for column(s).\n   *\n   * Uses pragma_index_list and pragma_index_info to reliably get index\n   * column names, avoiding brittle SQL parsing. Includes all indices:\n   * user-created (CREATE INDEX), PRIMARY KEY, and UNIQUE constraints.\n   *\n   * Uses flexible matching: Finds indexes where ALL columns appear in the\n   * first N positions, regardless of order. This works because SQLite statistics\n   * at depth N represent the fanout for the combination of the first N columns,\n   * and combinations are order-independent.\n   *\n   * Example:\n   * - columns: ['customerId', 'storeId']\n   * - Matches: (customerId, storeId, date) at depth 2 ✅\n   * - Matches: (storeId, customerId, date) at depth 2 ✅ (flexible order)\n   * - Does NOT match: (date, customerId, storeId) ❌ (columns not in first 2 positions)\n   * - Does NOT match: (customerId, date, storeId) ❌ (storeId not in first 2 positions)\n   *\n   * @param columns Array of column names (order-independent for matching)\n   * @returns Index info with name and depth, or undefined if no match\n   */\n  #findIndexForColumns(\n    tableName: string,\n    columns: string[],\n  ): {indexName: string; depth: number} | undefined {\n    try {\n      // Query returns all columns for all indexes (including PK/UNIQUE) in order\n      const rows = this.#indexStmt.all(tableName) as {\n        index_name: string;\n        seqno: number;\n        column_name: string;\n      }[];\n\n      // Group by index name\n      const indexMap = new Map<string, string[]>();\n      for (const row of rows) {\n        const cols = indexMap.get(row.index_name) ?? [];\n        cols.push(row.column_name);\n        indexMap.set(row.index_name, cols);\n      }\n\n      // Check each index for prefix match\n      for (const [indexName, indexColumns] of indexMap) {\n        if (this.#isPrefixMatch(columns, indexColumns)) {\n          return {\n            indexName,\n            depth: columns.length,\n          };\n        }\n      }\n\n      return undefined;\n    } catch {\n      return undefined;\n    }\n  }\n\n  /**\n   * Checks if all queryColumns exist in the first N positions of indexColumns,\n   * regardless of order.\n   *\n   * This allows flexible matching: constraint {a, b} matches both index (a, b, c)\n   * and index (b, a, c) at depth 2, since both represent the fanout for the\n   * combination of columns a and b.\n   *\n   * Gaps are NOT allowed: constraint {a, c} does NOT match index (a, b, c)\n   * because no depth represents just (a, c) without b. Statistics are cumulative\n   * from position 0.\n   *\n   * @param queryColumns Columns we're looking for (from constraint)\n   * @param indexColumns Columns in the index (in order)\n   * @returns true if all queryColumns exist in indexColumns[0...queryColumns.length-1]\n   */\n  #isPrefixMatch(queryColumns: string[], indexColumns: string[]): boolean {\n    if (queryColumns.length > indexColumns.length) {\n      return false;\n    }\n\n    // Get the prefix of the index that we're checking against\n    const indexPrefix = indexColumns.slice(0, queryColumns.length);\n\n    // Normalize to lowercase for case-insensitive comparison\n    const indexPrefixLower = new Set(indexPrefix.map(col => col.toLowerCase()));\n    const queryColumnsLower = queryColumns.map(col => col.toLowerCase());\n\n    // Check if ALL query columns exist in the index prefix\n    return queryColumnsLower.every(queryCol => indexPrefixLower.has(queryCol));\n  }\n\n  /**\n   * Decodes a sqlite_stat4 sample value to check if it's NULL.\n   *\n   * SQLite record format (simplified):\n   * - Varint: header size\n   * - Serial types for each column (one byte each typically)\n   * - Actual data\n   *\n   * Serial type 0 = NULL\n   * Serial type 1 = 8-bit int\n   * Serial type 2 = 16-bit int\n   * Serial type 3 = 24-bit int\n   * etc.\n   *\n   * We only need to check the first column's serial type.\n   *\n   * @param sample Binary-encoded sample from stat4\n   * @returns true if the sample value is NULL\n   */\n  #decodeSampleIsNull(sample: Buffer): boolean {\n    if (sample.length === 0) {\n      return true;\n    }\n\n    // Read header size (varint - simplified: assume single byte)\n    const headerSize = sample[0];\n\n    if (headerSize === 0 || headerSize >= sample.length) {\n      return true;\n    }\n\n    // Read first serial type (at position 1)\n    const serialType = sample[1];\n\n    // Serial type 0 = NULL\n    return serialType === 0;\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAwFA,IAAa,mBAAb,MAA8B;CAC5B;CACA;;;;;CAMA,yBAAkB,IAAI,IAA0B;;;;;CAMhD;CACA;CACA;;;;;;;;;;CAWA,YAAY,IAAc,gBAAgB,GAAG;EAC3C,KAAKA,MAAM;EACX,KAAKC,iBAAiB;EAGtB,KAAKE,aAAa,KAAKH,IAAI,QAAQ;;;;;KAKlC;EAED,KAAKI,aAAa,KAAKJ,IAAI,QAAQ;;;;KAIlC;EAED,KAAKK,aAAa,KAAKL,IAAI,QAAQ;;;;;;KAMlC;CACH;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAgCA,UAAU,WAAmB,SAAiC;EAE5D,MAAM,WAAW,GAAG,UAAU,GAAG,QAAQ,SAAS,EAAE,KAAK,GAAG;EAC5D,MAAM,SAAS,KAAKE,OAAO,IAAI,QAAQ;EACvC,IAAI,QACF,OAAO;EAMT,MAAM,cAAc,KAAKI,oBAAoB,WAAW,OAAO;EAC/D,IAAI,aAAa;GACf,KAAKJ,OAAO,IAAI,UAAU,WAAW;GACrC,OAAO;EACT;EAGA,MAAM,cAAc,KAAKK,oBAAoB,WAAW,OAAO;EAC/D,IAAI,aAAa;GACf,KAAKL,OAAO,IAAI,UAAU,WAAW;GACrC,OAAO;EACT;EAGA,MAAM,gBAA8B;GAClC,QAAQ,KAAKD;GACb,YAAY;GACZ,QAAQ;EACV;EACA,KAAKC,OAAO,IAAI,UAAU,aAAa;EACvC,OAAO;CACT;;;;;CAMA,aAAmB;EACjB,KAAKA,OAAO,MAAM;CACpB;;;;;;;;;;;;CAaA,oBACE,WACA,SAC0B;EAC1B,IAAI;GAEF,MAAM,YAAY,KAAKM,qBAAqB,WAAW,OAAO;GAC9D,IAAI,CAAC,WACH;GAIF,MAAM,UAAU,KAAKL,WAAW,IAC9B,WACA,UAAU,SACZ;GAEA,IAAI,QAAQ,WAAW,GACrB;GAKF,MAAM,WAAW,UAAU,QAAQ;GASnC,MAAM,iBARiB,QAAQ,KAAI,MAAK;IACtC,MAAM,WAAW,EAAE,IAAI,MAAM,GAAG;IAChC,OAAO;KACL,QAAQ,SAAS,SAAS,aAAa,SAAS,IAAI,EAAE;KACtD,QAAQ,KAAKM,oBAAoB,EAAE,MAAM;IAC3C;GACF,CAEuB,EAAe,QAAO,MAAK,CAAC,EAAE,MAAM;GAE3D,IAAI,eAAe,WAAW,GAE5B,OAAO;IACL,QAAQ;IACR,QAAQ;IACR,YAAY;GACd;GAIF,MAAM,UAAU,eAAe,KAAI,MAAK,EAAE,MAAM,EAAE,MAAM,GAAG,MAAM,IAAI,CAAC;GAStE,OAAO;IACL,QARA,QAAQ,SAAS,MAAM,IACnB,KAAK,OACF,QAAQ,QAAQ,SAAS,IAAI,KAAK,QAAQ,QAAQ,SAAS,MAC1D,CACJ,IACA,QAAQ,KAAK,MAAM,QAAQ,SAAS,CAAC;IAIzC,QAAQ;IACR,YAAY;GACd;EACF,QAAQ;GAEN;EACF;CACF;;;;;;;;;;;;CAaA,oBACE,WACA,SAC0B;EAC1B,IAAI;GAEF,MAAM,YAAY,KAAKD,qBAAqB,WAAW,OAAO;GAC9D,IAAI,CAAC,WACH;GAIF,MAAM,SAAS,KAAKJ,WAAW,IAAI,WAAW,UAAU,SAAS;GAIjE,IAAI,CAAC,QACH;GAGF,MAAM,QAAQ,OAAO,KAAK,MAAM,GAAG;GAEnC,IAAI,MAAM,SAAS,UAAU,QAAQ,GACnC;GAGF,MAAM,SAAS,SAAS,MAAM,UAAU,QAAQ,EAAE;GAClD,IAAI,MAAM,MAAM,GACd;GAGF,OAAO;IACL;IACA,QAAQ;IACR,YAAY;GACd;EACF,QAAQ;GACN;EACF;CACF;;;;;;;;;;;;;;;;;;;;;;;CAwBA,qBACE,WACA,SACgD;EAChD,IAAI;GAEF,MAAM,OAAO,KAAKC,WAAW,IAAI,SAAS;GAO1C,MAAM,2BAAW,IAAI,IAAsB;GAC3C,KAAK,MAAM,OAAO,MAAM;IACtB,MAAM,OAAO,SAAS,IAAI,IAAI,UAAU,KAAK,CAAC;IAC9C,KAAK,KAAK,IAAI,WAAW;IACzB,SAAS,IAAI,IAAI,YAAY,IAAI;GACnC;GAGA,KAAK,MAAM,CAAC,WAAW,iBAAiB,UACtC,IAAI,KAAKK,eAAe,SAAS,YAAY,GAC3C,OAAO;IACL;IACA,OAAO,QAAQ;GACjB;GAIJ;EACF,QAAQ;GACN;EACF;CACF;;;;;;;;;;;;;;;;;CAkBA,eAAe,cAAwB,cAAiC;EACtE,IAAI,aAAa,SAAS,aAAa,QACrC,OAAO;EAIT,MAAM,cAAc,aAAa,MAAM,GAAG,aAAa,MAAM;EAG7D,MAAM,mBAAmB,IAAI,IAAI,YAAY,KAAI,QAAO,IAAI,YAAY,CAAC,CAAC;EAI1E,OAH0B,aAAa,KAAI,QAAO,IAAI,YAAY,CAG3D,EAAkB,OAAM,aAAY,iBAAiB,IAAI,QAAQ,CAAC;CAC3E;;;;;;;;;;;;;;;;;;;;CAqBA,oBAAoB,QAAyB;EAC3C,IAAI,OAAO,WAAW,GACpB,OAAO;EAIT,MAAM,aAAa,OAAO;EAE1B,IAAI,eAAe,KAAK,cAAc,OAAO,QAC3C,OAAO;EAOT,OAHmB,OAAO,OAGJ;CACxB;AACF"}
+{"version":3,"file":"sqlite-stat-fanout.js","names":["#db","#defaultFanout","#cache","#stat4Stmt","#stat1Stmt","#indexStmt","#getFanoutFromStat4","#getFanoutFromStat1","#findIndexForColumns","#isPrefixMatch"],"sources":["../../../../zqlite/src/sqlite-stat-fanout.ts"],"sourcesContent":["import {newArray} from '../../shared/src/arrays.ts';\nimport {getOrInsertComputed} from '../../shared/src/map.ts';\nimport type {Database} from './db.ts';\nimport {isSampleNull} from './sqlite-stat4-sample.ts';\n\n/**\n * Result of fanout calculation from SQLite statistics.\n */\nexport interface FanoutResult {\n  /**\n   * The fanout value (average rows per distinct value of the join column).\n   * For non-NULL joins, this represents how many child rows exist per parent key.\n   */\n  fanout: number;\n  confidence: 'high' | 'med' | 'none';\n\n  /**\n   * Source of the fanout calculation.\n   * - 'stat4': From sqlite_stat4 histogram (most accurate, excludes NULLs)\n   * - 'stat1': From sqlite_stat1 average (includes NULLs, may overestimate)\n   * - 'default': Fallback constant when statistics unavailable\n   */\n  source: 'stat4' | 'stat1' | 'default';\n}\n\n/**\n * Sample from sqlite_stat4 histogram.\n */\ninterface Stat4Sample {\n  /** \"N1 N2\" = rows equal to sample (N1=first col, N2=if composite) */\n  neq: string;\n  /** \"N1 N2\" = rows less than sample */\n  nlt: string;\n  /** \"N1 N2\" = distinct values less than sample */\n  ndlt: string;\n  /** The actual sample value (binary encoded) */\n  sample: Buffer;\n}\n\n/**\n * Computes join fanout factors from SQLite statistics tables.\n *\n * Fanout is the average number of child rows per distinct parent key value,\n * used to estimate join cardinality in query planning.\n *\n * ## Problem\n *\n * sqlite_stat1 includes NULL rows in its calculation, which can significantly\n * overestimate fanout for sparse foreign keys:\n *\n * ```\n * Example: 100 tasks, 20 with project_id, 80 with NULL\n * - stat1 reports: \"100 17\" → fanout = 17 (WRONG - includes NULLs)\n * - stat4 shows: NULL samples with fanout=80, non-NULL samples with fanout=4\n * - True fanout: 4 (CORRECT)\n * ```\n *\n * ## Solution\n *\n * This class uses sqlite_stat4 histogram to separate NULL and non-NULL samples,\n * providing accurate fanout for non-NULL joins.\n *\n * ## Usage\n *\n * ```typescript\n * const calculator = new SQLiteStatFanout(db);\n *\n * // Get fanout for posts.userId → users.id join\n * const result = calculator.getFanout('posts', 'userId');\n *\n * if (result.source === 'stat4') {\n *   // Accurate: excludes NULLs, samples actual distribution\n *   console.log(`Fanout: ${result.fanout} (from stat4)`);\n * } else if (result.source === 'stat1') {\n *   // Conservative: includes NULLs, may overestimate\n *   console.log(`Fanout: ${result.fanout} (from stat1, includes NULLs)`);\n * } else {\n *   // Fallback: no statistics available\n *   console.log(`Fanout: ${result.fanout} (default estimate)`);\n * }\n * ```\n *\n * ## Requirements\n *\n * - SQLite compiled with ENABLE_STAT4 (most builds include this)\n * - `ANALYZE` command run on the database\n * - Index exists on the join column\n *\n * @see https://sqlite.org/fileformat2.html#stat4tab\n * @see packages/zql/src/planner/SELECTIVITY_PLAN.md\n */\nexport class SQLiteStatFanout {\n  readonly #db: Database;\n  readonly #defaultFanout: number;\n\n  /**\n   * Cache of fanout results by table and columns.\n   * Key format: \"tableName:col1,col2,col3\" (sorted alphabetically)\n   */\n  readonly #cache = new Map<string, FanoutResult>();\n\n  /**\n   * Prepared statements for querying SQLite statistics tables.\n   * Prepared once in constructor for performance.\n   */\n  readonly #stat4Stmt: ReturnType<Database['prepare']>;\n  readonly #stat1Stmt: ReturnType<Database['prepare']>;\n  readonly #indexStmt: ReturnType<Database['prepare']>;\n\n  /**\n   * Creates a new fanout calculator.\n   *\n   * @param db Database instance\n   * @param defaultFanout Default fanout when statistics unavailable (default: 3)\n   *                      - 1: Conservative (assumes FK relationships)\n   *                      - 3: Moderate (recommended, safe middle ground)\n   *                      - 10: SQLite's default (optimistic)\n   */\n  constructor(db: Database, defaultFanout = 3) {\n    this.#db = db;\n    this.#defaultFanout = defaultFanout;\n\n    // Prepare SQL statements once for reuse across multiple getFanout() calls\n    this.#stat4Stmt = this.#db.prepare(`\n      SELECT neq, nlt, ndlt, sample\n      FROM sqlite_stat4\n      WHERE tbl = ? AND idx = ?\n      ORDER BY nlt\n    `);\n\n    this.#stat1Stmt = this.#db.prepare(`\n      SELECT stat\n      FROM sqlite_stat1\n      WHERE tbl = ? AND idx = ?\n    `);\n\n    this.#indexStmt = this.#db.prepare(`\n      SELECT il.name as index_name, ii.seqno, ii.name as column_name\n      FROM pragma_index_list(?) il\n      JOIN pragma_index_info(il.name) ii\n      WHERE il.partial = 0\n      ORDER BY il.seq, ii.seqno\n    `);\n  }\n\n  /**\n   * Gets the fanout factor for join column(s).\n   *\n   * Fanout = average number of child rows per distinct parent key value(s).\n   *\n   * ## Strategy\n   *\n   * 1. Try sqlite_stat4 (best): Histogram with separate NULL/non-NULL samples\n   * 2. Fallback to sqlite_stat1: Average across all rows (includes NULLs)\n   * 3. Fallback to default: When no statistics available\n   *\n   * ## Compound Indexes\n   *\n   * For multi-column joins, finds indexes where ALL columns appear as an\n   * exact prefix. Uses the appropriate depth in stat1/stat4.\n   *\n   * Example:\n   * - Columns: `['customerId', 'storeId']`\n   * - Matches index: `(customerId, storeId, date)` at depth 2\n   * - Uses stat1 parts[2] or stat4 neq[1] for accurate fanout\n   *\n   * ## Caching\n   *\n   * Results are cached per (table, columns) combination. Clear the cache if\n   * you run ANALYZE to update statistics.\n   *\n   * @param tableName Table containing the join column(s)\n   * @param columns Array of column names (one or more columns)\n   * @returns Fanout result with value and source\n   */\n  getFanout(tableName: string, columns: string[]): FanoutResult {\n    // Cache key uses sorted columns for consistency\n    const cacheKey = `${tableName}:${columns.toSorted().join(',')}`;\n    const cached = this.#cache.get(cacheKey);\n    if (cached) {\n      return cached;\n    }\n\n    // Strategy 1: Try stat4 first (most accurate)\n    // NOTE: columns are NOT sorted - preserves Object.keys() order from constraint\n    // Matching is order-independent (flexible), but we keep original order for consistency\n    const stat4Result = this.#getFanoutFromStat4(tableName, columns);\n    if (stat4Result) {\n      this.#cache.set(cacheKey, stat4Result);\n      return stat4Result;\n    }\n\n    // Strategy 2: Fallback to stat1 (includes NULLs)\n    const stat1Result = this.#getFanoutFromStat1(tableName, columns);\n    if (stat1Result) {\n      this.#cache.set(cacheKey, stat1Result);\n      return stat1Result;\n    }\n\n    // Strategy 3: Use default\n    const defaultResult: FanoutResult = {\n      fanout: this.#defaultFanout,\n      confidence: 'none',\n      source: 'default',\n    };\n    this.#cache.set(cacheKey, defaultResult);\n    return defaultResult;\n  }\n\n  /**\n   * Clears the fanout cache.\n   * Call this after running ANALYZE to pick up updated statistics.\n   */\n  clearCache(): void {\n    this.#cache.clear();\n  }\n\n  /**\n   * Gets fanout from sqlite_stat4 histogram.\n   *\n   * Queries stat4 samples, decodes to identify NULLs, and returns\n   * the median fanout of non-NULL samples.\n   *\n   * For compound indexes, uses the neq value at the appropriate depth.\n   *\n   * @param columns Array of column names to get fanout for\n   * @returns Fanout result or undefined if stat4 unavailable\n   */\n  #getFanoutFromStat4(\n    tableName: string,\n    columns: string[],\n  ): FanoutResult | undefined {\n    try {\n      // Find index containing the columns as a prefix\n      const indexInfo = this.#findIndexForColumns(tableName, columns);\n      if (!indexInfo) {\n        return undefined;\n      }\n\n      // Query stat4 samples for this index (using prepared statement)\n      const samples = this.#stat4Stmt.all(\n        tableName,\n        indexInfo.indexName,\n      ) as Stat4Sample[];\n\n      if (samples.length === 0) {\n        return undefined;\n      }\n\n      // Decode samples and separate NULL from non-NULL\n      // Use depth-1 for neq array index (depth is 1-based, array is 0-based)\n      const neqIndex = indexInfo.depth - 1;\n      const decodedSamples = samples.map(s => {\n        const neqParts = s.neq.split(' ');\n        return {\n          fanout: parseInt(neqParts[neqIndex] ?? neqParts[0], 10),\n          isNull: isSampleNull(s.sample),\n        };\n      });\n\n      const nonNullSamples = decodedSamples.filter(s => !s.isNull);\n\n      if (nonNullSamples.length === 0) {\n        // All samples are NULL - return fanout of 0 since NULLs don't match in joins\n        return {\n          fanout: 0,\n          source: 'stat4',\n          confidence: 'high',\n        };\n      }\n\n      // Use median of non-NULL fanouts (more robust than average)\n      const fanouts = nonNullSamples.map(s => s.fanout).sort((a, b) => a - b);\n      const medianFanout =\n        fanouts.length % 2 === 0\n          ? Math.floor(\n              (fanouts[fanouts.length / 2 - 1] + fanouts[fanouts.length / 2]) /\n                2,\n            )\n          : fanouts[Math.floor(fanouts.length / 2)];\n\n      return {\n        fanout: medianFanout,\n        source: 'stat4',\n        confidence: 'high',\n      };\n    } catch {\n      // stat4 table may not exist or query may fail\n      return undefined;\n    }\n  }\n\n  /**\n   * Gets fanout from sqlite_stat1 average.\n   *\n   * Note: This includes NULL rows in the calculation and may overestimate\n   * fanout for sparse foreign keys.\n   *\n   * For compound indexes, uses the stat value at the appropriate depth.\n   *\n   * @param columns Array of column names to get fanout for\n   * @returns Fanout result or undefined if stat1 unavailable\n   */\n  #getFanoutFromStat1(\n    tableName: string,\n    columns: string[],\n  ): FanoutResult | undefined {\n    try {\n      // Find index containing the columns as a prefix\n      const indexInfo = this.#findIndexForColumns(tableName, columns);\n      if (!indexInfo) {\n        return undefined;\n      }\n\n      // Query stat1 for this index (using prepared statement)\n      const result = this.#stat1Stmt.get(tableName, indexInfo.indexName) as\n        | {stat: string}\n        | undefined;\n\n      if (!result) {\n        return undefined;\n      }\n\n      const parts = result.stat.split(' ');\n      // Check if we have enough parts for the requested depth\n      if (parts.length < indexInfo.depth + 1) {\n        return undefined;\n      }\n\n      const fanout = parseInt(parts[indexInfo.depth], 10);\n      if (isNaN(fanout)) {\n        return undefined;\n      }\n\n      return {\n        fanout,\n        source: 'stat1',\n        confidence: 'med',\n      };\n    } catch {\n      return undefined;\n    }\n  }\n\n  /**\n   * Finds an index that can be used to get statistics for column(s).\n   *\n   * Uses pragma_index_list and pragma_index_info to reliably get index\n   * column names, avoiding brittle SQL parsing. Includes all indices:\n   * user-created (CREATE INDEX), PRIMARY KEY, and UNIQUE constraints.\n   *\n   * Uses flexible matching: Finds indexes where ALL columns appear in the\n   * first N positions, regardless of order. This works because SQLite statistics\n   * at depth N represent the fanout for the combination of the first N columns,\n   * and combinations are order-independent.\n   *\n   * Example:\n   * - columns: ['customerId', 'storeId']\n   * - Matches: (customerId, storeId, date) at depth 2 ✅\n   * - Matches: (storeId, customerId, date) at depth 2 ✅ (flexible order)\n   * - Does NOT match: (date, customerId, storeId) ❌ (columns not in first 2 positions)\n   * - Does NOT match: (customerId, date, storeId) ❌ (storeId not in first 2 positions)\n   *\n   * @param columns Array of column names (order-independent for matching)\n   * @returns Index info with name and depth, or undefined if no match\n   */\n  #findIndexForColumns(\n    tableName: string,\n    columns: string[],\n  ): {indexName: string; depth: number} | undefined {\n    try {\n      // Query returns all columns for all indexes (including PK/UNIQUE) in order\n      const rows = this.#indexStmt.all(tableName) as {\n        index_name: string;\n        seqno: number;\n        column_name: string;\n      }[];\n\n      // Group by index name\n      const indexMap = new Map<string, string[]>();\n      for (const row of rows) {\n        getOrInsertComputed(indexMap, row.index_name, newArray).push(\n          row.column_name,\n        );\n      }\n\n      // Check each index for prefix match\n      for (const [indexName, indexColumns] of indexMap) {\n        if (this.#isPrefixMatch(columns, indexColumns)) {\n          return {\n            indexName,\n            depth: columns.length,\n          };\n        }\n      }\n\n      return undefined;\n    } catch {\n      return undefined;\n    }\n  }\n\n  /**\n   * Checks if all queryColumns exist in the first N positions of indexColumns,\n   * regardless of order.\n   *\n   * This allows flexible matching: constraint {a, b} matches both index (a, b, c)\n   * and index (b, a, c) at depth 2, since both represent the fanout for the\n   * combination of columns a and b.\n   *\n   * Gaps are NOT allowed: constraint {a, c} does NOT match index (a, b, c)\n   * because no depth represents just (a, c) without b. Statistics are cumulative\n   * from position 0.\n   *\n   * @param queryColumns Columns we're looking for (from constraint)\n   * @param indexColumns Columns in the index (in order)\n   * @returns true if all queryColumns exist in indexColumns[0...queryColumns.length-1]\n   */\n  #isPrefixMatch(queryColumns: string[], indexColumns: string[]): boolean {\n    if (queryColumns.length > indexColumns.length) {\n      return false;\n    }\n\n    // Get the prefix of the index that we're checking against\n    const indexPrefix = indexColumns.slice(0, queryColumns.length);\n\n    // Normalize to lowercase for case-insensitive comparison\n    const indexPrefixLower = new Set(indexPrefix.map(col => col.toLowerCase()));\n    const queryColumnsLower = queryColumns.map(col => col.toLowerCase());\n\n    // Check if ALL query columns exist in the index prefix\n    return queryColumnsLower.every(queryCol => indexPrefixLower.has(queryCol));\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AA2FA,IAAa,mBAAb,MAA8B;CAC5B;CACA;;;;;CAMA,yBAAkB,IAAI,IAA0B;;;;;CAMhD;CACA;CACA;;;;;;;;;;CAWA,YAAY,IAAc,gBAAgB,GAAG;EAC3C,KAAKA,MAAM;EACX,KAAKC,iBAAiB;EAGtB,KAAKE,aAAa,KAAKH,IAAI,QAAQ;;;;;KAKlC;EAED,KAAKI,aAAa,KAAKJ,IAAI,QAAQ;;;;KAIlC;EAED,KAAKK,aAAa,KAAKL,IAAI,QAAQ;;;;;;KAMlC;CACH;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAgCA,UAAU,WAAmB,SAAiC;EAE5D,MAAM,WAAW,GAAG,UAAU,GAAG,QAAQ,SAAS,EAAE,KAAK,GAAG;EAC5D,MAAM,SAAS,KAAKE,OAAO,IAAI,QAAQ;EACvC,IAAI,QACF,OAAO;EAMT,MAAM,cAAc,KAAKI,oBAAoB,WAAW,OAAO;EAC/D,IAAI,aAAa;GACf,KAAKJ,OAAO,IAAI,UAAU,WAAW;GACrC,OAAO;EACT;EAGA,MAAM,cAAc,KAAKK,oBAAoB,WAAW,OAAO;EAC/D,IAAI,aAAa;GACf,KAAKL,OAAO,IAAI,UAAU,WAAW;GACrC,OAAO;EACT;EAGA,MAAM,gBAA8B;GAClC,QAAQ,KAAKD;GACb,YAAY;GACZ,QAAQ;EACV;EACA,KAAKC,OAAO,IAAI,UAAU,aAAa;EACvC,OAAO;CACT;;;;;CAMA,aAAmB;EACjB,KAAKA,OAAO,MAAM;CACpB;;;;;;;;;;;;CAaA,oBACE,WACA,SAC0B;EAC1B,IAAI;GAEF,MAAM,YAAY,KAAKM,qBAAqB,WAAW,OAAO;GAC9D,IAAI,CAAC,WACH;GAIF,MAAM,UAAU,KAAKL,WAAW,IAC9B,WACA,UAAU,SACZ;GAEA,IAAI,QAAQ,WAAW,GACrB;GAKF,MAAM,WAAW,UAAU,QAAQ;GASnC,MAAM,iBARiB,QAAQ,KAAI,MAAK;IACtC,MAAM,WAAW,EAAE,IAAI,MAAM,GAAG;IAChC,OAAO;KACL,QAAQ,SAAS,SAAS,aAAa,SAAS,IAAI,EAAE;KACtD,QAAQ,aAAa,EAAE,MAAM;IAC/B;GACF,CAEuB,EAAe,QAAO,MAAK,CAAC,EAAE,MAAM;GAE3D,IAAI,eAAe,WAAW,GAE5B,OAAO;IACL,QAAQ;IACR,QAAQ;IACR,YAAY;GACd;GAIF,MAAM,UAAU,eAAe,KAAI,MAAK,EAAE,MAAM,EAAE,MAAM,GAAG,MAAM,IAAI,CAAC;GAStE,OAAO;IACL,QARA,QAAQ,SAAS,MAAM,IACnB,KAAK,OACF,QAAQ,QAAQ,SAAS,IAAI,KAAK,QAAQ,QAAQ,SAAS,MAC1D,CACJ,IACA,QAAQ,KAAK,MAAM,QAAQ,SAAS,CAAC;IAIzC,QAAQ;IACR,YAAY;GACd;EACF,QAAQ;GAEN;EACF;CACF;;;;;;;;;;;;CAaA,oBACE,WACA,SAC0B;EAC1B,IAAI;GAEF,MAAM,YAAY,KAAKK,qBAAqB,WAAW,OAAO;GAC9D,IAAI,CAAC,WACH;GAIF,MAAM,SAAS,KAAKJ,WAAW,IAAI,WAAW,UAAU,SAAS;GAIjE,IAAI,CAAC,QACH;GAGF,MAAM,QAAQ,OAAO,KAAK,MAAM,GAAG;GAEnC,IAAI,MAAM,SAAS,UAAU,QAAQ,GACnC;GAGF,MAAM,SAAS,SAAS,MAAM,UAAU,QAAQ,EAAE;GAClD,IAAI,MAAM,MAAM,GACd;GAGF,OAAO;IACL;IACA,QAAQ;IACR,YAAY;GACd;EACF,QAAQ;GACN;EACF;CACF;;;;;;;;;;;;;;;;;;;;;;;CAwBA,qBACE,WACA,SACgD;EAChD,IAAI;GAEF,MAAM,OAAO,KAAKC,WAAW,IAAI,SAAS;GAO1C,MAAM,2BAAW,IAAI,IAAsB;GAC3C,KAAK,MAAM,OAAO,MAChB,oBAAoB,UAAU,IAAI,YAAY,QAAQ,EAAE,KACtD,IAAI,WACN;GAIF,KAAK,MAAM,CAAC,WAAW,iBAAiB,UACtC,IAAI,KAAKI,eAAe,SAAS,YAAY,GAC3C,OAAO;IACL;IACA,OAAO,QAAQ;GACjB;GAIJ;EACF,QAAQ;GACN;EACF;CACF;;;;;;;;;;;;;;;;;CAkBA,eAAe,cAAwB,cAAiC;EACtE,IAAI,aAAa,SAAS,aAAa,QACrC,OAAO;EAIT,MAAM,cAAc,aAAa,MAAM,GAAG,aAAa,MAAM;EAG7D,MAAM,mBAAmB,IAAI,IAAI,YAAY,KAAI,QAAO,IAAI,YAAY,CAAC,CAAC;EAI1E,OAH0B,aAAa,KAAI,QAAO,IAAI,YAAY,CAG3D,EAAkB,OAAM,aAAY,iBAAiB,IAAI,QAAQ,CAAC;CAC3E;AACF"}

package/out/zqlite/src/sqlite-stat4-sample.d.ts

@@ -0,0 +1,37 @@
+/**
+ * Decoding of `sqlite_stat4.sample` values.
+ *
+ * A sample is a SQLite record: a header holding one serial type per column,
+ * followed by the column data. We only ever look at the serial types, which
+ * say what *kind* of value each column holds without revealing the value
+ * itself. Two callers need this:
+ *
+ * - `SQLiteStatFanout`, which separates NULL from non-NULL samples so that
+ *   join fanout is not skewed by a sparse foreign key.
+ * - the `/plannerz` admin endpoint, which reports the shape of the histogram
+ *   without returning row data.
+ *
+ * @see https://sqlite.org/fileformat2.html#record_format
+ * @see https://sqlite.org/fileformat2.html#stat4tab
+ */
+/**
+ * The kind of value a stat4 sample column holds, derived from its serial type.
+ * `unknown` covers the internal serial types 10 and 11, which do not appear in
+ * a well-formed record.
+ */
+export type SampleValueKind = 'null' | 'integer' | 'real' | 'text' | 'blob' | 'unknown';
+/**
+ * Decodes the serial types in a stat4 sample's header.
+ *
+ * The sample's trailing rowid is included as a final column, matching the
+ * record SQLite stores.
+ *
+ * @returns one kind per column, or an empty array if `sample` is empty or its
+ *          header is malformed.
+ */
+export declare function decodeSampleKinds(sample: Buffer): SampleValueKind[];
+/**
+ * Whether the first column of a sample is NULL.
+ */
+export declare function isSampleNull(sample: Buffer): boolean;
+//# sourceMappingURL=sqlite-stat4-sample.d.ts.map

package/out/zqlite/src/sqlite-stat4-sample.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"sqlite-stat4-sample.d.ts","sourceRoot":"","sources":["../../../../zqlite/src/sqlite-stat4-sample.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;GAeG;AAEH;;;;GAIG;AACH,MAAM,MAAM,eAAe,GACvB,MAAM,GACN,SAAS,GACT,MAAM,GACN,MAAM,GACN,MAAM,GACN,SAAS,CAAC;AAEd;;;;;;;;GAQG;AACH,wBAAgB,iBAAiB,CAAC,MAAM,EAAE,MAAM,GAAG,eAAe,EAAE,CAuBnE;AAED;;GAEG;AACH,wBAAgB,YAAY,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAGpD"}

package/out/zqlite/src/sqlite-stat4-sample.js

@@ -0,0 +1,61 @@
+//#region ../zqlite/src/sqlite-stat4-sample.ts
+/**
+* Decodes the serial types in a stat4 sample's header.
+*
+* The sample's trailing rowid is included as a final column, matching the
+* record SQLite stores.
+*
+* @returns one kind per column, or an empty array if `sample` is empty or its
+*          header is malformed.
+*/
+function decodeSampleKinds(sample) {
+	const header = readVarint(sample, 0);
+	if (header === void 0) return [];
+	const [headerSize, serialTypesStart] = header;
+	if (headerSize < serialTypesStart || headerSize > sample.length) return [];
+	const kinds = [];
+	for (let pos = serialTypesStart; pos < headerSize;) {
+		const next = readVarint(sample, pos);
+		if (next === void 0) return [];
+		const [serialType, end] = next;
+		kinds.push(kindOfSerialType(serialType));
+		pos = end;
+	}
+	return kinds;
+}
+/**
+* Whether the first column of a sample is NULL.
+*/
+function isSampleNull(sample) {
+	const kinds = decodeSampleKinds(sample);
+	return kinds.length === 0 || kinds[0] === "null";
+}
+function kindOfSerialType(serialType) {
+	if (serialType === 0) return "null";
+	if (serialType <= 6 || serialType === 8 || serialType === 9) return "integer";
+	if (serialType === 7) return "real";
+	if (serialType <= 11) return "unknown";
+	return serialType % 2 === 0 ? "blob" : "text";
+}
+/**
+* Reads the varint at `pos`.
+*
+* @returns the value and the offset just past it, or `undefined` if the varint
+*          runs off the end of the buffer. Values are read as `number`; a
+*          varint large enough to lose precision is not something a well-formed
+*          header or serial type contains.
+*/
+function readVarint(buf, pos) {
+	let value = 0;
+	for (let i = 0; i < 9; i++) {
+		if (pos + i >= buf.length) return;
+		const byte = buf[pos + i];
+		if (i === 8) return [value * 256 + byte, pos + 9];
+		value = value * 128 + (byte & 127);
+		if ((byte & 128) === 0) return [value, pos + i + 1];
+	}
+}
+//#endregion
+export { decodeSampleKinds, isSampleNull };
+
+//# sourceMappingURL=sqlite-stat4-sample.js.map

package/out/zqlite/src/sqlite-stat4-sample.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"sqlite-stat4-sample.js","names":[],"sources":["../../../../zqlite/src/sqlite-stat4-sample.ts"],"sourcesContent":["/**\n * Decoding of `sqlite_stat4.sample` values.\n *\n * A sample is a SQLite record: a header holding one serial type per column,\n * followed by the column data. We only ever look at the serial types, which\n * say what *kind* of value each column holds without revealing the value\n * itself. Two callers need this:\n *\n * - `SQLiteStatFanout`, which separates NULL from non-NULL samples so that\n *   join fanout is not skewed by a sparse foreign key.\n * - the `/plannerz` admin endpoint, which reports the shape of the histogram\n *   without returning row data.\n *\n * @see https://sqlite.org/fileformat2.html#record_format\n * @see https://sqlite.org/fileformat2.html#stat4tab\n */\n\n/**\n * The kind of value a stat4 sample column holds, derived from its serial type.\n * `unknown` covers the internal serial types 10 and 11, which do not appear in\n * a well-formed record.\n */\nexport type SampleValueKind =\n  | 'null'\n  | 'integer'\n  | 'real'\n  | 'text'\n  | 'blob'\n  | 'unknown';\n\n/**\n * Decodes the serial types in a stat4 sample's header.\n *\n * The sample's trailing rowid is included as a final column, matching the\n * record SQLite stores.\n *\n * @returns one kind per column, or an empty array if `sample` is empty or its\n *          header is malformed.\n */\nexport function decodeSampleKinds(sample: Buffer): SampleValueKind[] {\n  const header = readVarint(sample, 0);\n  if (header === undefined) {\n    return [];\n  }\n  // `headerSize` counts the size varint itself, so the serial types run from\n  // the end of that varint to `headerSize`, and cannot exceed the record.\n  const [headerSize, serialTypesStart] = header;\n  if (headerSize < serialTypesStart || headerSize > sample.length) {\n    return [];\n  }\n\n  const kinds: SampleValueKind[] = [];\n  for (let pos = serialTypesStart; pos < headerSize;) {\n    const next = readVarint(sample, pos);\n    if (next === undefined) {\n      return [];\n    }\n    const [serialType, end] = next;\n    kinds.push(kindOfSerialType(serialType));\n    pos = end;\n  }\n  return kinds;\n}\n\n/**\n * Whether the first column of a sample is NULL.\n */\nexport function isSampleNull(sample: Buffer): boolean {\n  const kinds = decodeSampleKinds(sample);\n  return kinds.length === 0 || kinds[0] === 'null';\n}\n\nfunction kindOfSerialType(serialType: number): SampleValueKind {\n  if (serialType === 0) {\n    return 'null';\n  }\n  if (serialType <= 6 || serialType === 8 || serialType === 9) {\n    return 'integer';\n  }\n  if (serialType === 7) {\n    return 'real';\n  }\n  if (serialType <= 11) {\n    return 'unknown'; // 10 and 11 are for internal use.\n  }\n  return serialType % 2 === 0 ? 'blob' : 'text';\n}\n\n/**\n * Reads the varint at `pos`.\n *\n * @returns the value and the offset just past it, or `undefined` if the varint\n *          runs off the end of the buffer. Values are read as `number`; a\n *          varint large enough to lose precision is not something a well-formed\n *          header or serial type contains.\n */\nfunction readVarint(\n  buf: Buffer,\n  pos: number,\n): [value: number, end: number] | undefined {\n  let value = 0;\n  // A varint is at most 9 bytes, the last of which contributes all 8 bits.\n  for (let i = 0; i < 9; i++) {\n    if (pos + i >= buf.length) {\n      return undefined;\n    }\n    const byte = buf[pos + i];\n    if (i === 8) {\n      return [value * 256 + byte, pos + 9];\n    }\n    value = value * 128 + (byte & 0x7f);\n    if ((byte & 0x80) === 0) {\n      return [value, pos + i + 1];\n    }\n  }\n  return undefined;\n}\n"],"mappings":";;;;;;;;;;AAuCA,SAAgB,kBAAkB,QAAmC;CACnE,MAAM,SAAS,WAAW,QAAQ,CAAC;CACnC,IAAI,WAAW,KAAA,GACb,OAAO,CAAC;CAIV,MAAM,CAAC,YAAY,oBAAoB;CACvC,IAAI,aAAa,oBAAoB,aAAa,OAAO,QACvD,OAAO,CAAC;CAGV,MAAM,QAA2B,CAAC;CAClC,KAAK,IAAI,MAAM,kBAAkB,MAAM,aAAa;EAClD,MAAM,OAAO,WAAW,QAAQ,GAAG;EACnC,IAAI,SAAS,KAAA,GACX,OAAO,CAAC;EAEV,MAAM,CAAC,YAAY,OAAO;EAC1B,MAAM,KAAK,iBAAiB,UAAU,CAAC;EACvC,MAAM;CACR;CACA,OAAO;AACT;;;;AAKA,SAAgB,aAAa,QAAyB;CACpD,MAAM,QAAQ,kBAAkB,MAAM;CACtC,OAAO,MAAM,WAAW,KAAK,MAAM,OAAO;AAC5C;AAEA,SAAS,iBAAiB,YAAqC;CAC7D,IAAI,eAAe,GACjB,OAAO;CAET,IAAI,cAAc,KAAK,eAAe,KAAK,eAAe,GACxD,OAAO;CAET,IAAI,eAAe,GACjB,OAAO;CAET,IAAI,cAAc,IAChB,OAAO;CAET,OAAO,aAAa,MAAM,IAAI,SAAS;AACzC;;;;;;;;;AAUA,SAAS,WACP,KACA,KAC0C;CAC1C,IAAI,QAAQ;CAEZ,KAAK,IAAI,IAAI,GAAG,IAAI,GAAG,KAAK;EAC1B,IAAI,MAAM,KAAK,IAAI,QACjB;EAEF,MAAM,OAAO,IAAI,MAAM;EACvB,IAAI,MAAM,GACR,OAAO,CAAC,QAAQ,MAAM,MAAM,MAAM,CAAC;EAErC,QAAQ,QAAQ,OAAO,OAAO;EAC9B,KAAK,OAAO,SAAU,GACpB,OAAO,CAAC,OAAO,MAAM,IAAI,CAAC;CAE9B;AAEF"}

package/out/zqlite/src/table-source.d.ts

@@ -8,6 +8,25 @@ import type { DebugDelegate } from '../../zql/src/builder/debug-delegate.ts';
 import { type Source, type SourceChange, type SourceInput } from '../../zql/src/ivm/source.ts';
 import type { Stream } from '../../zql/src/ivm/stream.ts';
 import type { Database } from './db.ts';
+export type TableSourceOptions = {
+    /**
+     * When set, a pushed change that the filters of every connection reject
+     * is neither pushed nor applied to the backing table, on the grounds that
+     * no connection can observe it. This departs from the `Source.push`
+     * contract (a push commits the change to the source), so it is only
+     * appropriate when the caller does not read the table through any other
+     * path and replaces the table's contents afterwards, as the view-syncer's
+     * pipeline driver does when it advances to the next snapshot.
+     */
+    skipUnobservableChanges?: boolean | undefined;
+    /**
+     * When set, pushed changes are held in an in-memory {@link PendingDelta}
+     * merged into every read instead of being written to the backing table.
+     * The delta is dropped when the source moves to the next snapshot. This is
+     * the initial mode; see {@link TableSource.setDeferWrites}.
+     */
+    deferWrites?: boolean | undefined;
+};
 /**
  * A source that is backed by a SQLite table.
  *
@@ -30,7 +49,12 @@ export declare class TableSource implements Source {
      * to yield control back to the caller at the end of the pipeline.  Can
      * also throw an error to abort the pipeline processing.
      */
-    constructor(logContext: LogContext, logConfig: LogConfig, db: Database, tableName: string, columns: Record<string, SchemaValue>, primaryKey: PrimaryKey, shouldYield?: () => boolean);
+    constructor(logContext: LogContext, logConfig: LogConfig, db: Database, tableName: string, columns: Record<string, SchemaValue>, primaryKey: PrimaryKey, shouldYield?: () => boolean, options?: TableSourceOptions);
+    /**
+     * The number of rows read from SQLite by fetches, over the lifetime of this
+     * source. Rows that SQLite scans and filters out itself are not counted.
+     */
+    get rowsRead(): number;
     get tableSchema(): {
         name: string;
         columns: Record<string, SchemaValue>;
@@ -41,10 +65,66 @@ export declare class TableSource implements Source {
      * algorithm for concurrent traversal of historic timelines.
      */
     setDB(db: Database): void;
+    /**
+     * Sets whether the changes pushed from now on are held in memory (see
+     * {@link TableSourceOptions.deferWrites}) or written to the backing
+     * snapshot. The two must not be mixed within one snapshot, so this may only
+     * be called before the first push after construction or {@link setDB}. To
+     * switch to writing through partway, see {@link writePendingChanges}.
+     */
+    setDeferWrites(deferWrites: boolean): void;
+    /**
+     * Drops the changes held in memory (see {@link setDeferWrites}), for an
+     * advancement that has been abandoned. The source then no longer reflects
+     * the changes pushed to it, so it must not be read again until it has moved
+     * to another snapshot ({@link setDB}), if it is used again at all.
+     */
+    discardPendingChanges(): void;
+    /**
+     * Writes the changes held in memory (see {@link setDeferWrites}) to the
+     * backing snapshot, which is left as if they had been written through (in
+     * the source's columns), and writes the changes pushed from then on through
+     * to it. Yields when the
+     * source's `shouldYield` says to, and must not be interleaved with a push.
+     */
+    writePendingChanges(): Stream<'yield'>;
     connect(sort: Ordering | undefined, filters?: Condition, splitEditKeys?: Set<string>, debug?: DebugDelegate): SourceInput;
     toSQLiteRow(row: Row): Row;
     push(change: SourceChange): Stream<'yield'>;
-    genPush(change: SourceChange): Generator<"yield" | undefined, void, unknown>;
+    /**
+     * The number of rows held in memory for changes that have been pushed but
+     * not written, i.e. with `deferWrites`, since the last {@link setDB}.
+     */
+    get pendingRows(): number;
+    /** Estimated bytes retained by deferred changes and their indexes. */
+    get pendingBytes(): number;
+    genPush(change: SourceChange): Stream<'yield' | undefined>;
+    /**
+     * Reconciles a set of rows that the caller found in the *backing snapshot*
+     * with the changes this source has applied but not written.
+     *
+     * A view-syncer computes the rows a replicated change collides with -- by
+     * change-log key for a delete, by every unique key for a set -- against the
+     * snapshot it is reading. When derivation writes through, that snapshot has
+     * already absorbed the earlier changes of the same advancement, so the
+     * probe sees them. When derivation is deferred they live here instead, and
+     * the probe has to be brought up to date or the source is pushed a change
+     * that contradicts its own state (a delete of a row the batch already
+     * removed, an edit whose old row is stale).
+     *
+     * Two ways the batch can disagree with the probe, both handled here: a
+     * probed row may have been removed, or may have been edited (so the
+     * caller's copy is stale, or it no longer collides at all).
+     *
+     * The batch cannot have edited a row the probe missed *into* collision. The
+     * change log keeps one entry per row key, so every row the batch holds is
+     * that row's value at the advancement's target version, as is `next`, and
+     * two distinct rows of one version cannot share a non-null unique key.
+     *
+     * Returns `base` untouched when derivation is not deferred, which is what
+     * makes this a no-op for the write-through path.
+     */
+    reconcilePendingConflicts(base: readonly Row[], next: Row | null, rowKey: Row, uniqueKeys: readonly (readonly string[])[]): readonly Row[];
     /**
      * Retrieves a row from the backing DB by a unique key, or `undefined` if such a
      * row does not exist. This is not used in the IVM pipeline but is useful

package/out/zqlite/src/table-source.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"table-source.d.ts","sourceRoot":"","sources":["../../../../zqlite/src/table-source.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAEjD,OAAO,KAAK,EAAC,SAAS,EAAC,MAAM,+BAA+B,CAAC;AAK7D,OAAO,KAAK,EAAC,SAAS,EAAE,QAAQ,EAAC,MAAM,gCAAgC,CAAC;AACxE,OAAO,KAAK,EAAC,GAAG,EAAQ,MAAM,iCAAiC,CAAC;AAChE,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,wCAAwC,CAAC;AACvE,OAAO,KAAK,EACV,WAAW,EACX,SAAS,EACV,MAAM,uCAAuC,CAAC;AAC/C,OAAO,KAAK,EAAC,aAAa,EAAC,MAAM,yCAAyC,CAAC;AAmB3E,OAAO,EACL,KAAK,MAAM,EACX,KAAK,YAAY,EACjB,KAAK,WAAW,EACjB,MAAM,6BAA6B,CAAC;AACrC,OAAO,KAAK,EAAC,MAAM,EAAC,MAAM,6BAA6B,CAAC;AAExD,OAAO,KAAK,EAAC,QAAQ,EAAY,MAAM,SAAS,CAAC;AAoBjD;;;;;;;;;;;;;GAaG;AACH,qBAAa,WAAY,YAAW,MAAM;;IAexC;;;;;OAKG;IACH,YACE,UAAU,EAAE,UAAU,EACtB,SAAS,EAAE,SAAS,EACpB,EAAE,EAAE,QAAQ,EACZ,SAAS,EAAE,MAAM,EACjB,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,WAAW,CAAC,EACpC,UAAU,EAAE,UAAU,EACtB,WAAW,gBAAc,EAe1B;IAED,IAAI,WAAW;QAEX,IAAI;QACJ,OAAO;QACP,UAAU;MAEb;IAED;;;OAGG;IACH,KAAK,CAAC,EAAE,EAAE,QAAQ,QAEjB;IA0FD,OAAO,CACL,IAAI,EAAE,QAAQ,GAAG,SAAS,EAC1B,OAAO,CAAC,EAAE,SAAS,EACnB,aAAa,CAAC,EAAE,GAAG,CAAC,MAAM,CAAC,EAC3B,KAAK,CAAC,EAAE,aAAa,eA4CtB;IAED,WAAW,CAAC,GAAG,EAAE,GAAG,GAAG,GAAG,CAOzB;IAkIA,IAAI,CAAC,MAAM,EAAE,YAAY,GAAG,MAAM,CAAC,OAAO,CAAC,CAM3C;IAEA,OAAO,CAAC,MAAM,EAAE,YAAY,iDAgB5B;IAqFD;;;;;;OAMG;IACH,MAAM,CAAC,MAAM,EAAE,GAAG,GAAG,GAAG,GAAG,SAAS,CAanC;CAmBF;AA4CD,wBAAgB,aAAa,CAC3B,OAAO,EAAE,SAAS,MAAM,EAAE,EAC1B,GAAG,EAAE,GAAG,EACR,WAAW,EAAE,MAAM,CAAC,MAAM,EAAE,WAAW,CAAC,GACvC,SAAS,OAAO,EAAE,CAEpB;AAED,wBAAgB,gBAAgB,CAAC,IAAI,EAAE,SAAS,wCAa/C;AAED,wBAAgB,eAAe,CAC7B,UAAU,EAAE,MAAM,CAAC,MAAM,EAAE,WAAW,CAAC,EACvC,GAAG,EAAE,GAAG,EACR,SAAS,EAAE,MAAM,GAChB,GAAG,CAaL;AAwCD,qBAAa,qBAAsB,SAAQ,KAAK;CAAG"}
+{"version":3,"file":"table-source.d.ts","sourceRoot":"","sources":["../../../../zqlite/src/table-source.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAEjD,OAAO,KAAK,EAAC,SAAS,EAAC,MAAM,+BAA+B,CAAC;AAM7D,OAAO,KAAK,EAAC,SAAS,EAAE,QAAQ,EAAC,MAAM,gCAAgC,CAAC;AACxE,OAAO,KAAK,EAAC,GAAG,EAAQ,MAAM,iCAAiC,CAAC;AAChE,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,wCAAwC,CAAC;AACvE,OAAO,KAAK,EACV,WAAW,EACX,SAAS,EACV,MAAM,uCAAuC,CAAC;AAC/C,OAAO,KAAK,EAAC,aAAa,EAAC,MAAM,yCAAyC,CAAC;AAwB3E,OAAO,EACL,KAAK,MAAM,EACX,KAAK,YAAY,EACjB,KAAK,WAAW,EACjB,MAAM,6BAA6B,CAAC;AACrC,OAAO,KAAK,EAAC,MAAM,EAAC,MAAM,6BAA6B,CAAC;AAExD,OAAO,KAAK,EAAC,QAAQ,EAAY,MAAM,SAAS,CAAC;AA0BjD,MAAM,MAAM,kBAAkB,GAAG;IAC/B;;;;;;;;OAQG;IACH,uBAAuB,CAAC,EAAE,OAAO,GAAG,SAAS,CAAC;IAE9C;;;;;OAKG;IACH,WAAW,CAAC,EAAE,OAAO,GAAG,SAAS,CAAC;CACnC,CAAC;AAEF;;;;;;;;;;;;;GAaG;AACH,qBAAa,WAAY,YAAW,MAAM;;IA2BxC;;;;;OAKG;IACH,YACE,UAAU,EAAE,UAAU,EACtB,SAAS,EAAE,SAAS,EACpB,EAAE,EAAE,QAAQ,EACZ,SAAS,EAAE,MAAM,EACjB,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,WAAW,CAAC,EACpC,UAAU,EAAE,UAAU,EACtB,WAAW,gBAAc,EACzB,OAAO,GAAE,kBAAuB,EAqBjC;IAED;;;OAGG;IACH,IAAI,QAAQ,IAAI,MAAM,CAErB;IAED,IAAI,WAAW;QAEX,IAAI;QACJ,OAAO;QACP,UAAU;MAEb;IAED;;;OAGG;IACH,KAAK,CAAC,EAAE,EAAE,QAAQ,QAKjB;IAED;;;;;;OAMG;IACH,cAAc,CAAC,WAAW,EAAE,OAAO,QAUlC;IAED;;;;;OAKG;IACH,qBAAqB,IAAI,IAAI,CAE5B;IAED;;;;;;OAMG;IACF,mBAAmB,IAAI,MAAM,CAAC,OAAO,CAAC,CAwBtC;IA0FD,OAAO,CACL,IAAI,EAAE,QAAQ,GAAG,SAAS,EAC1B,OAAO,CAAC,EAAE,SAAS,EACnB,aAAa,CAAC,EAAE,GAAG,CAAC,MAAM,CAAC,EAC3B,KAAK,CAAC,EAAE,aAAa,eA8CtB;IAED,WAAW,CAAC,GAAG,EAAE,GAAG,GAAG,GAAG,CAOzB;IAkMA,IAAI,CAAC,MAAM,EAAE,YAAY,GAAG,MAAM,CAAC,OAAO,CAAC,CAM3C;IAED;;;OAGG;IACH,IAAI,WAAW,IAAI,MAAM,CAExB;IAED,sEAAsE;IACtE,IAAI,YAAY,IAAI,MAAM,CAEzB;IAEA,OAAO,CAAC,MAAM,EAAE,YAAY,GAAG,MAAM,CAAC,OAAO,GAAG,SAAS,CAAC,CAsC1D;IAuED;;;;;;;;;;;;;;;;;;;;;;;;OAwBG;IACH,yBAAyB,CACvB,IAAI,EAAE,SAAS,GAAG,EAAE,EACpB,IAAI,EAAE,GAAG,GAAG,IAAI,EAChB,MAAM,EAAE,GAAG,EACX,UAAU,EAAE,SAAS,CAAC,SAAS,MAAM,EAAE,CAAC,EAAE,GACzC,SAAS,GAAG,EAAE,CAoChB;IA6DD;;;;;;OAMG;IACH,MAAM,CAAC,MAAM,EAAE,GAAG,GAAG,GAAG,GAAG,SAAS,CAgCnC;CAmBF;AAoFD,wBAAgB,aAAa,CAC3B,OAAO,EAAE,SAAS,MAAM,EAAE,EAC1B,GAAG,EAAE,GAAG,EACR,WAAW,EAAE,MAAM,CAAC,MAAM,EAAE,WAAW,CAAC,GACvC,SAAS,OAAO,EAAE,CAEpB;AAED,wBAAgB,gBAAgB,CAAC,IAAI,EAAE,SAAS,wCAa/C;AAED,wBAAgB,eAAe,CAC7B,UAAU,EAAE,MAAM,CAAC,MAAM,EAAE,WAAW,CAAC,EACvC,GAAG,EAAE,GAAG,EACR,SAAS,EAAE,MAAM,GAChB,GAAG,CAaL;AAyCD,qBAAa,qBAAsB,SAAQ,KAAK;CAAG"}