@rocicorp/zero 1.12.0-canary.1 → 1.12.0-canary.11

package/out/zqlite/src/query-delegate.js.map

@@ -1 +1 @@
-{"version":3,"file":"query-delegate.js","names":["#lc","#db","#schema","#sources","#logConfig","#commitObservers"],"sources":["../../../../zqlite/src/query-delegate.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport type {LogConfig} from '../../otel/src/log-options.ts';\nimport {getOrInsertComputed} from '../../shared/src/map.ts';\nimport type {Schema} from '../../zero-types/src/schema.ts';\nimport type {Source} from '../../zql/src/ivm/source.ts';\nimport {QueryDelegateBase} from '../../zql/src/query/query-delegate-base.ts';\nimport type {CommitListener} from '../../zql/src/query/query-delegate.ts';\nimport type {Database} from './db.ts';\nimport {TableSource} from './table-source.ts';\n\nexport class QueryDelegateImpl extends QueryDelegateBase {\n  readonly #lc: LogContext;\n  readonly #db: Database;\n  readonly #schema: Schema;\n  readonly #sources: Map<string, Source> = new Map();\n  readonly #logConfig: LogConfig;\n  readonly defaultQueryComplete = true;\n  readonly #commitObservers = new Set<() => void>();\n\n  constructor(\n    lc: LogContext,\n    db: Database,\n    schema: Schema,\n    logConfig?: LogConfig,\n  ) {\n    super();\n    this.#lc = lc.withContext('class', 'QueryDelegateImpl');\n    this.#db = db;\n    this.#schema = schema;\n    this.#logConfig = logConfig ?? {\n      format: 'text',\n      ivmSampling: 0,\n      level: 'info',\n      slowHydrateThreshold: 0,\n      slowAdvanceThreshold: 0,\n      queryStatsIntervalSeconds: 0,\n      slowRowThreshold: 0,\n      planWarningRowThreshold: 0,\n      planWarningCostThreshold: 0,\n    };\n  }\n\n  getSource(tableName: string): Source {\n    return getOrInsertComputed(this.#sources, tableName, tableName => {\n      const tableSchema = this.#schema.tables[tableName];\n      return new TableSource(\n        this.#lc,\n        this.#logConfig,\n        this.#db,\n        tableName,\n        tableSchema.columns,\n        tableSchema.primaryKey,\n      );\n    });\n  }\n\n  onTransactionCommit(cb: CommitListener) {\n    this.#commitObservers.add(cb);\n    return () => {\n      this.#commitObservers.delete(cb);\n    };\n  }\n  override batchViewUpdates<T>(applyViewUpdates: () => T): T {\n    const ret = applyViewUpdates();\n    for (const observer of this.#commitObservers) {\n      observer();\n    }\n    return ret;\n  }\n}\n"],"mappings":";;;;AAUA,IAAa,oBAAb,cAAuC,kBAAkB;CACvD;CACA;CACA;CACA,2BAAyC,IAAI,IAAI;CACjD;CACA,uBAAgC;CAChC,mCAA4B,IAAI,IAAgB;CAEhD,YACE,IACA,IACA,QACA,WACA;EACA,MAAM;EACN,KAAKA,MAAM,GAAG,YAAY,SAAS,mBAAmB;EACtD,KAAKC,MAAM;EACX,KAAKC,UAAU;EACf,KAAKE,aAAa,aAAa;GAC7B,QAAQ;GACR,aAAa;GACb,OAAO;GACP,sBAAsB;GACtB,sBAAsB;GACtB,2BAA2B;GAC3B,kBAAkB;GAClB,yBAAyB;GACzB,0BAA0B;EAC5B;CACF;CAEA,UAAU,WAA2B;EACnC,OAAO,oBAAoB,KAAKD,UAAU,YAAW,cAAa;GAChE,MAAM,cAAc,KAAKD,QAAQ,OAAO;GACxC,OAAO,IAAI,YACT,KAAKF,KACL,KAAKI,YACL,KAAKH,KACL,WACA,YAAY,SACZ,YAAY,UACd;EACF,CAAC;CACH;CAEA,oBAAoB,IAAoB;EACtC,KAAKI,iBAAiB,IAAI,EAAE;EAC5B,aAAa;GACX,KAAKA,iBAAiB,OAAO,EAAE;EACjC;CACF;CACA,iBAA6B,kBAA8B;EACzD,MAAM,MAAM,iBAAiB;EAC7B,KAAK,MAAM,YAAY,KAAKA,kBAC1B,SAAS;EAEX,OAAO;CACT;AACF"}
+{"version":3,"file":"query-delegate.js","names":[],"sources":["../../../../zqlite/src/query-delegate.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport type {LogConfig} from '../../otel/src/log-options.ts';\nimport {getOrInsertComputed} from '../../shared/src/map.ts';\nimport type {Schema} from '../../zero-types/src/schema.ts';\nimport type {Source} from '../../zql/src/ivm/source.ts';\nimport {QueryDelegateBase} from '../../zql/src/query/query-delegate-base.ts';\nimport type {CommitListener} from '../../zql/src/query/query-delegate.ts';\nimport type {Database} from './db.ts';\nimport {TableSource} from './table-source.ts';\n\nexport class QueryDelegateImpl extends QueryDelegateBase {\n  readonly #lc: LogContext;\n  readonly #db: Database;\n  readonly #schema: Schema;\n  readonly #sources: Map<string, Source> = new Map();\n  readonly #logConfig: LogConfig;\n  readonly defaultQueryComplete = true;\n  readonly #commitObservers = new Set<() => void>();\n\n  constructor(\n    lc: LogContext,\n    db: Database,\n    schema: Schema,\n    logConfig?: LogConfig,\n  ) {\n    super();\n    this.#lc = lc.withContext('class', 'QueryDelegateImpl');\n    this.#db = db;\n    this.#schema = schema;\n    this.#logConfig = logConfig ?? {\n      format: 'text',\n      ivmSampling: 0,\n      level: 'info',\n      slowHydrateThreshold: 0,\n      slowAdvanceThreshold: 0,\n      queryStatsIntervalSeconds: 0,\n      slowRowThreshold: 0,\n      planWarningRowThreshold: 0,\n      planWarningCostThreshold: 0,\n    };\n  }\n\n  getSource(tableName: string): Source {\n    return getOrInsertComputed(this.#sources, tableName, tableName => {\n      const tableSchema = this.#schema.tables[tableName];\n      return new TableSource(\n        this.#lc,\n        this.#logConfig,\n        this.#db,\n        tableName,\n        tableSchema.columns,\n        tableSchema.primaryKey,\n      );\n    });\n  }\n\n  onTransactionCommit(cb: CommitListener) {\n    this.#commitObservers.add(cb);\n    return () => {\n      this.#commitObservers.delete(cb);\n    };\n  }\n  override batchViewUpdates<T>(applyViewUpdates: () => T): T {\n    const ret = applyViewUpdates();\n    for (const observer of this.#commitObservers) {\n      observer();\n    }\n    return ret;\n  }\n}\n"],"mappings":";;;;AAUA,IAAa,oBAAb,cAAuC,kBAAkB;CACvD;CACA;CACA;CACA,2BAAyC,IAAI,IAAI;CACjD;CACA,uBAAgC;CAChC,mCAA4B,IAAI,IAAgB;CAEhD,YACE,IACA,IACA,QACA,WACA;EACA,MAAM;EACN,KAAK,MAAM,GAAG,YAAY,SAAS,mBAAmB;EACtD,KAAK,MAAM;EACX,KAAK,UAAU;EACf,KAAK,aAAa,aAAa;GAC7B,QAAQ;GACR,aAAa;GACb,OAAO;GACP,sBAAsB;GACtB,sBAAsB;GACtB,2BAA2B;GAC3B,kBAAkB;GAClB,yBAAyB;GACzB,0BAA0B;EAC5B;CACF;CAEA,UAAU,WAA2B;EACnC,OAAO,oBAAoB,KAAK,UAAU,YAAW,cAAa;GAChE,MAAM,cAAc,KAAK,QAAQ,OAAO;GACxC,OAAO,IAAI,YACT,KAAK,KACL,KAAK,YACL,KAAK,KACL,WACA,YAAY,SACZ,YAAY,UACd;EACF,CAAC;CACH;CAEA,oBAAoB,IAAoB;EACtC,KAAK,iBAAiB,IAAI,EAAE;EAC5B,aAAa;GACX,KAAK,iBAAiB,OAAO,EAAE;EACjC;CACF;CACA,iBAA6B,kBAA8B;EACzD,MAAM,MAAM,iBAAiB;EAC7B,KAAK,MAAM,YAAY,KAAK,kBAC1B,SAAS;EAEX,OAAO;CACT;AACF"}

package/out/zqlite/src/resolve-scalar-subqueries.js

@@ -157,13 +157,10 @@ function collectConstraints(condition, constraints) {
 				if (condition.left.type === "column" && condition.right.type === "literal") constraints.set(condition.left.name, condition.right.value);
 			}
 			break;
-		case "and":
-			for (const c of condition.conditions) collectConstraints(c, constraints);
-			break;
-		default: break;
+		case "and": for (const c of condition.conditions) collectConstraints(c, constraints);
 	}
 }
 //#endregion
-export { resolveSimpleScalarSubqueries };
+export { extractLiteralEqualityConstraints, isSimpleSubquery, resolveSimpleScalarSubqueries };
 
 //# sourceMappingURL=resolve-scalar-subqueries.js.map

package/out/zqlite/src/resolve-scalar-subqueries.js.map

@@ -1 +1 @@
-{"version":3,"file":"resolve-scalar-subqueries.js","names":[],"sources":["../../../../zqlite/src/resolve-scalar-subqueries.ts"],"sourcesContent":["import type {\n  AST,\n  Condition,\n  CorrelatedSubqueryCondition,\n  LiteralValue,\n  SimpleCondition,\n  System,\n} from '../../zero-protocol/src/ast.ts';\nimport type {PrimaryKey} from '../../zero-protocol/src/primary-key.ts';\n\ntype TableSpecWithUniqueKeys = {\n  tableSpec: {\n    uniqueKeys: PrimaryKey[];\n  };\n};\n\nexport type CompanionSubquery = {\n  /** The original scalar subquery AST (the subquery table query). */\n  ast: AST;\n  /** The field in the subquery row whose value was resolved. */\n  childField: string;\n  /** The system that added the gate. */\n  system: System | undefined;\n  /** The resolved value, `null` if a row matched but the field was `NULL`,\n   * or `undefined` if no row matched. */\n  resolvedValue: LiteralValue | null | undefined;\n};\n\n/**\n * A `scalar` hint that could not be honored: the subquery was not provably\n * limited to one row, so the gate was left as a plain EXISTS.\n */\nexport type IgnoredScalarHint = {\n  /** The subquery's table. */\n  table: string;\n  /** The unique keys that were available to pin it. */\n  uniqueKeys: readonly PrimaryKey[];\n};\n\nexport type ResolveResult = {\n  ast: AST;\n  companions: CompanionSubquery[];\n  /**\n   * Hints the caller may want to surface: each one is a `{scalar: true}` the\n   * author asked for and did not get. See {@link IgnoredScalarHint}.\n   */\n  ignoredScalarHints: IgnoredScalarHint[];\n};\n\n/** Accumulators threaded through the recursion. */\ntype Out = {\n  companions: CompanionSubquery[];\n  ignoredScalarHints: IgnoredScalarHint[];\n};\n\n/**\n * Callback that executes a scalar subquery and returns the value of\n * `childField` from the (at most one) matching row, or `undefined`\n * if no rows match.\n */\nexport type ScalarExecutor = (\n  subqueryAST: AST,\n  childField: string,\n) => LiteralValue | null | undefined;\n\n/**\n * Resolves \"simple\" scalar subqueries by calling the provided executor\n * and replacing them with literal conditions. A scalar subquery is simple\n * when all columns of at least one unique index on the subquery table are\n * equality-constrained by literal values in the subquery's WHERE clause\n * (using only AND conjunctions).\n *\n * Non-simple scalar subqueries are left untouched for the existing\n * EXISTS rewrite in buildPipelineInternal.\n *\n * Returns the resolved AST and a list of companion subquery ASTs whose\n * rows need to be synced to the client for the EXISTS rewrite to work.\n */\nexport function resolveSimpleScalarSubqueries(\n  ast: AST,\n  tableSpecs: Map<string, TableSpecWithUniqueKeys>,\n  execute: ScalarExecutor,\n): ResolveResult {\n  const out: Out = {companions: [], ignoredScalarHints: []};\n  const resolved = resolveASTRecursive(ast, tableSpecs, execute, out);\n  return {ast: resolved, ...out};\n}\n\nfunction resolveASTRecursive(\n  ast: AST,\n  tableSpecs: Map<string, TableSpecWithUniqueKeys>,\n  execute: ScalarExecutor,\n  out: Out,\n): AST {\n  const where = ast.where\n    ? resolveCondition(ast.where, tableSpecs, execute, out)\n    : undefined;\n\n  const related = ast.related?.map(r => ({\n    ...r,\n    subquery: resolveASTRecursive(r.subquery, tableSpecs, execute, out),\n  }));\n\n  if (where !== ast.where || related !== ast.related) {\n    return {...ast, where, related};\n  }\n  return ast;\n}\n\nfunction resolveCondition(\n  condition: Condition,\n  tableSpecs: Map<string, TableSpecWithUniqueKeys>,\n  execute: ScalarExecutor,\n  out: Out,\n): Condition {\n  switch (condition.type) {\n    case 'correlatedSubquery':\n      if (condition.scalar) {\n        return resolveScalarSubquery(condition, tableSpecs, execute, out);\n      }\n      // Non-scalar correlated subquery: recurse into its subquery\n      {\n        const resolvedSubquery = resolveASTRecursive(\n          condition.related.subquery,\n          tableSpecs,\n          execute,\n          out,\n        );\n        if (resolvedSubquery !== condition.related.subquery) {\n          return {\n            ...condition,\n            related: {...condition.related, subquery: resolvedSubquery},\n          };\n        }\n        return condition;\n      }\n    case 'and':\n    case 'or': {\n      const resolved = condition.conditions.map(c =>\n        resolveCondition(c, tableSpecs, execute, out),\n      );\n      if (resolved.every((c, i) => c === condition.conditions[i])) {\n        return condition;\n      }\n      return {type: condition.type, conditions: resolved};\n    }\n    default:\n      return condition;\n  }\n}\n\nfunction resolveScalarSubquery(\n  condition: CorrelatedSubqueryCondition,\n  tableSpecs: Map<string, TableSpecWithUniqueKeys>,\n  execute: ScalarExecutor,\n  out: Out,\n): Condition {\n  const parentField = condition.related.correlation.parentField[0];\n  const childField = condition.related.correlation.childField[0];\n\n  // Recursively resolve any scalar subqueries nested in the\n  // subquery's own WHERE (and related) before evaluating this one.\n  const subquery = resolveASTRecursive(\n    condition.related.subquery,\n    tableSpecs,\n    execute,\n    out,\n  );\n\n  if (!isSimpleSubquery(subquery, tableSpecs)) {\n    // The author asked for a scalar rewrite and is not getting one. Record it\n    // so the caller can say so — silently falling back is the whole reason the\n    // hint is easy to get wrong.\n    out.ignoredScalarHints.push({\n      table: subquery.table,\n      uniqueKeys: tableSpecs.get(subquery.table)?.tableSpec.uniqueKeys ?? [],\n    });\n    // Return with the (possibly partially-resolved) subquery.\n    if (subquery !== condition.related.subquery) {\n      return {\n        ...condition,\n        related: {...condition.related, subquery},\n      };\n    }\n    return condition;\n  }\n\n  const value = execute(subquery, childField);\n\n  // Record the companion subquery AST so its rows are synced to the client.\n  // The client rewrites scalar subqueries to EXISTS and needs those rows.\n  out.companions.push({\n    ast: subquery,\n    childField,\n    system: condition.related.system,\n    resolvedValue: value,\n  });\n\n  if (value === undefined || value === null) {\n    // No row matched, or the matched row's `childField` is NULL (which can\n    // never satisfy the correlation, since `parentField = NULL` is never\n    // true). Either way the correlated EXISTS is false for *every* parent\n    // row, so the gate collapses to a constant: false for EXISTS, and its\n    // negation — true — for NOT EXISTS.\n    return condition.op === 'EXISTS' ? ALWAYS_FALSE : ALWAYS_TRUE;\n  }\n\n  const op = condition.op === 'EXISTS' ? '=' : 'IS NOT';\n  return {\n    type: 'simple',\n    op,\n    left: {type: 'column', name: parentField},\n    right: {type: 'literal', value},\n  } satisfies SimpleCondition;\n}\n\nconst ALWAYS_FALSE: SimpleCondition = {\n  type: 'simple',\n  op: '=',\n  left: {type: 'literal', value: 1},\n  right: {type: 'literal', value: 0},\n};\n\nconst ALWAYS_TRUE: SimpleCondition = {\n  type: 'simple',\n  op: '=',\n  left: {type: 'literal', value: 1},\n  right: {type: 'literal', value: 1},\n};\n\n/**\n * Checks if the subquery is guaranteed to return at most one deterministic row.\n *\n * This is true when all columns of at least one unique index on the subquery\n * table are equality-constrained by literal values in the WHERE clause\n * (using only AND conjunctions).\n */\nexport function isSimpleSubquery(\n  subquery: AST,\n  tableSpecs: Map<string, TableSpecWithUniqueKeys>,\n): boolean {\n  const spec = tableSpecs.get(subquery.table);\n  if (!spec) {\n    return false;\n  }\n\n  if (!subquery.where) {\n    return false;\n  }\n\n  const constraints = extractLiteralEqualityConstraints(subquery.where);\n  if (constraints.size === 0) {\n    return false;\n  }\n\n  const {uniqueKeys} = spec.tableSpec;\n  return uniqueKeys.some(key => key.every(col => constraints.has(col)));\n}\n\n/**\n * Extracts column=literal equality constraints from a condition tree,\n * only following AND conjunctions (not OR).\n */\nexport function extractLiteralEqualityConstraints(\n  condition: Condition,\n): Map<string, LiteralValue> {\n  const constraints = new Map<string, LiteralValue>();\n  collectConstraints(condition, constraints);\n  return constraints;\n}\n\nfunction collectConstraints(\n  condition: Condition,\n  constraints: Map<string, LiteralValue>,\n): void {\n  switch (condition.type) {\n    case 'simple':\n      if (condition.op === '=') {\n        if (\n          condition.left.type === 'column' &&\n          condition.right.type === 'literal'\n        ) {\n          constraints.set(condition.left.name, condition.right.value);\n        }\n      }\n      break;\n    case 'and':\n      for (const c of condition.conditions) {\n        collectConstraints(c, constraints);\n      }\n      break;\n    // OR, correlatedSubquery (non-scalar) — don't contribute constraints\n    default:\n      break;\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;AA8EA,SAAgB,8BACd,KACA,YACA,SACe;CACf,MAAM,MAAW;EAAC,YAAY,CAAC;EAAG,oBAAoB,CAAC;CAAC;CAExD,OAAO;EAAC,KADS,oBAAoB,KAAK,YAAY,SAAS,GAClD;EAAU,GAAG;CAAG;AAC/B;AAEA,SAAS,oBACP,KACA,YACA,SACA,KACK;CACL,MAAM,QAAQ,IAAI,QACd,iBAAiB,IAAI,OAAO,YAAY,SAAS,GAAG,IACpD,KAAA;CAEJ,MAAM,UAAU,IAAI,SAAS,KAAI,OAAM;EACrC,GAAG;EACH,UAAU,oBAAoB,EAAE,UAAU,YAAY,SAAS,GAAG;CACpE,EAAE;CAEF,IAAI,UAAU,IAAI,SAAS,YAAY,IAAI,SACzC,OAAO;EAAC,GAAG;EAAK;EAAO;CAAO;CAEhC,OAAO;AACT;AAEA,SAAS,iBACP,WACA,YACA,SACA,KACW;CACX,QAAQ,UAAU,MAAlB;EACE,KAAK;GACH,IAAI,UAAU,QACZ,OAAO,sBAAsB,WAAW,YAAY,SAAS,GAAG;GAGlE;IACE,MAAM,mBAAmB,oBACvB,UAAU,QAAQ,UAClB,YACA,SACA,GACF;IACA,IAAI,qBAAqB,UAAU,QAAQ,UACzC,OAAO;KACL,GAAG;KACH,SAAS;MAAC,GAAG,UAAU;MAAS,UAAU;KAAgB;IAC5D;IAEF,OAAO;GACT;EACF,KAAK;EACL,KAAK,MAAM;GACT,MAAM,WAAW,UAAU,WAAW,KAAI,MACxC,iBAAiB,GAAG,YAAY,SAAS,GAAG,CAC9C;GACA,IAAI,SAAS,OAAO,GAAG,MAAM,MAAM,UAAU,WAAW,EAAE,GACxD,OAAO;GAET,OAAO;IAAC,MAAM,UAAU;IAAM,YAAY;GAAQ;EACpD;EACA,SACE,OAAO;CACX;AACF;AAEA,SAAS,sBACP,WACA,YACA,SACA,KACW;CACX,MAAM,cAAc,UAAU,QAAQ,YAAY,YAAY;CAC9D,MAAM,aAAa,UAAU,QAAQ,YAAY,WAAW;CAI5D,MAAM,WAAW,oBACf,UAAU,QAAQ,UAClB,YACA,SACA,GACF;CAEA,IAAI,CAAC,iBAAiB,UAAU,UAAU,GAAG;EAI3C,IAAI,mBAAmB,KAAK;GAC1B,OAAO,SAAS;GAChB,YAAY,WAAW,IAAI,SAAS,KAAK,GAAG,UAAU,cAAc,CAAC;EACvE,CAAC;EAED,IAAI,aAAa,UAAU,QAAQ,UACjC,OAAO;GACL,GAAG;GACH,SAAS;IAAC,GAAG,UAAU;IAAS;GAAQ;EAC1C;EAEF,OAAO;CACT;CAEA,MAAM,QAAQ,QAAQ,UAAU,UAAU;CAI1C,IAAI,WAAW,KAAK;EAClB,KAAK;EACL;EACA,QAAQ,UAAU,QAAQ;EAC1B,eAAe;CACjB,CAAC;CAED,IAAI,UAAU,KAAA,KAAa,UAAU,MAMnC,OAAO,UAAU,OAAO,WAAW,eAAe;CAIpD,OAAO;EACL,MAAM;EACN,IAHS,UAAU,OAAO,WAAW,MAAM;EAI3C,MAAM;GAAC,MAAM;GAAU,MAAM;EAAW;EACxC,OAAO;GAAC,MAAM;GAAW;EAAK;CAChC;AACF;AAEA,IAAM,eAAgC;CACpC,MAAM;CACN,IAAI;CACJ,MAAM;EAAC,MAAM;EAAW,OAAO;CAAC;CAChC,OAAO;EAAC,MAAM;EAAW,OAAO;CAAC;AACnC;AAEA,IAAM,cAA+B;CACnC,MAAM;CACN,IAAI;CACJ,MAAM;EAAC,MAAM;EAAW,OAAO;CAAC;CAChC,OAAO;EAAC,MAAM;EAAW,OAAO;CAAC;AACnC;;;;;;;;AASA,SAAgB,iBACd,UACA,YACS;CACT,MAAM,OAAO,WAAW,IAAI,SAAS,KAAK;CAC1C,IAAI,CAAC,MACH,OAAO;CAGT,IAAI,CAAC,SAAS,OACZ,OAAO;CAGT,MAAM,cAAc,kCAAkC,SAAS,KAAK;CACpE,IAAI,YAAY,SAAS,GACvB,OAAO;CAGT,MAAM,EAAC,eAAc,KAAK;CAC1B,OAAO,WAAW,MAAK,QAAO,IAAI,OAAM,QAAO,YAAY,IAAI,GAAG,CAAC,CAAC;AACtE;;;;;AAMA,SAAgB,kCACd,WAC2B;CAC3B,MAAM,8BAAc,IAAI,IAA0B;CAClD,mBAAmB,WAAW,WAAW;CACzC,OAAO;AACT;AAEA,SAAS,mBACP,WACA,aACM;CACN,QAAQ,UAAU,MAAlB;EACE,KAAK;GACH,IAAI,UAAU,OAAO;QAEjB,UAAU,KAAK,SAAS,YACxB,UAAU,MAAM,SAAS,WAEzB,YAAY,IAAI,UAAU,KAAK,MAAM,UAAU,MAAM,KAAK;GAAA;GAG9D;EACF,KAAK;GACH,KAAK,MAAM,KAAK,UAAU,YACxB,mBAAmB,GAAG,WAAW;GAEnC;EAEF,SACE;CACJ;AACF"}
+{"version":3,"file":"resolve-scalar-subqueries.js","names":[],"sources":["../../../../zqlite/src/resolve-scalar-subqueries.ts"],"sourcesContent":["import type {\n  AST,\n  Condition,\n  CorrelatedSubqueryCondition,\n  LiteralValue,\n  SimpleCondition,\n  System,\n} from '../../zero-protocol/src/ast.ts';\nimport type {PrimaryKey} from '../../zero-protocol/src/primary-key.ts';\n\ntype TableSpecWithUniqueKeys = {\n  tableSpec: {\n    uniqueKeys: PrimaryKey[];\n  };\n};\n\nexport type CompanionSubquery = {\n  /** The original scalar subquery AST (the subquery table query). */\n  ast: AST;\n  /** The field in the subquery row whose value was resolved. */\n  childField: string;\n  /** The system that added the gate. */\n  system: System | undefined;\n  /** The resolved value, `null` if a row matched but the field was `NULL`,\n   * or `undefined` if no row matched. */\n  resolvedValue: LiteralValue | null | undefined;\n};\n\n/**\n * A `scalar` hint that could not be honored: the subquery was not provably\n * limited to one row, so the gate was left as a plain EXISTS.\n */\nexport type IgnoredScalarHint = {\n  /** The subquery's table. */\n  table: string;\n  /** The unique keys that were available to pin it. */\n  uniqueKeys: readonly PrimaryKey[];\n};\n\nexport type ResolveResult = {\n  ast: AST;\n  companions: CompanionSubquery[];\n  /**\n   * Hints the caller may want to surface: each one is a `{scalar: true}` the\n   * author asked for and did not get. See {@link IgnoredScalarHint}.\n   */\n  ignoredScalarHints: IgnoredScalarHint[];\n};\n\n/** Accumulators threaded through the recursion. */\ntype Out = {\n  companions: CompanionSubquery[];\n  ignoredScalarHints: IgnoredScalarHint[];\n};\n\n/**\n * Callback that executes a scalar subquery and returns the value of\n * `childField` from the (at most one) matching row, or `undefined`\n * if no rows match.\n */\nexport type ScalarExecutor = (\n  subqueryAST: AST,\n  childField: string,\n) => LiteralValue | null | undefined;\n\n/**\n * Resolves \"simple\" scalar subqueries by calling the provided executor\n * and replacing them with literal conditions. A scalar subquery is simple\n * when all columns of at least one unique index on the subquery table are\n * equality-constrained by literal values in the subquery's WHERE clause\n * (using only AND conjunctions).\n *\n * Non-simple scalar subqueries are left untouched for the existing\n * EXISTS rewrite in buildPipelineInternal.\n *\n * Returns the resolved AST and a list of companion subquery ASTs whose\n * rows need to be synced to the client for the EXISTS rewrite to work.\n */\nexport function resolveSimpleScalarSubqueries(\n  ast: AST,\n  tableSpecs: Map<string, TableSpecWithUniqueKeys>,\n  execute: ScalarExecutor,\n): ResolveResult {\n  const out: Out = {companions: [], ignoredScalarHints: []};\n  const resolved = resolveASTRecursive(ast, tableSpecs, execute, out);\n  return {ast: resolved, ...out};\n}\n\nfunction resolveASTRecursive(\n  ast: AST,\n  tableSpecs: Map<string, TableSpecWithUniqueKeys>,\n  execute: ScalarExecutor,\n  out: Out,\n): AST {\n  const where = ast.where\n    ? resolveCondition(ast.where, tableSpecs, execute, out)\n    : undefined;\n\n  const related = ast.related?.map(r => ({\n    ...r,\n    subquery: resolveASTRecursive(r.subquery, tableSpecs, execute, out),\n  }));\n\n  if (where !== ast.where || related !== ast.related) {\n    return {...ast, where, related};\n  }\n  return ast;\n}\n\nfunction resolveCondition(\n  condition: Condition,\n  tableSpecs: Map<string, TableSpecWithUniqueKeys>,\n  execute: ScalarExecutor,\n  out: Out,\n): Condition {\n  switch (condition.type) {\n    case 'correlatedSubquery':\n      if (condition.scalar) {\n        return resolveScalarSubquery(condition, tableSpecs, execute, out);\n      }\n      // Non-scalar correlated subquery: recurse into its subquery\n      {\n        const resolvedSubquery = resolveASTRecursive(\n          condition.related.subquery,\n          tableSpecs,\n          execute,\n          out,\n        );\n        if (resolvedSubquery !== condition.related.subquery) {\n          return {\n            ...condition,\n            related: {...condition.related, subquery: resolvedSubquery},\n          };\n        }\n        return condition;\n      }\n    case 'and':\n    case 'or': {\n      const resolved = condition.conditions.map(c =>\n        resolveCondition(c, tableSpecs, execute, out),\n      );\n      if (resolved.every((c, i) => c === condition.conditions[i])) {\n        return condition;\n      }\n      return {type: condition.type, conditions: resolved};\n    }\n    default:\n      return condition;\n  }\n}\n\nfunction resolveScalarSubquery(\n  condition: CorrelatedSubqueryCondition,\n  tableSpecs: Map<string, TableSpecWithUniqueKeys>,\n  execute: ScalarExecutor,\n  out: Out,\n): Condition {\n  const parentField = condition.related.correlation.parentField[0];\n  const childField = condition.related.correlation.childField[0];\n\n  // Recursively resolve any scalar subqueries nested in the\n  // subquery's own WHERE (and related) before evaluating this one.\n  const subquery = resolveASTRecursive(\n    condition.related.subquery,\n    tableSpecs,\n    execute,\n    out,\n  );\n\n  if (!isSimpleSubquery(subquery, tableSpecs)) {\n    // The author asked for a scalar rewrite and is not getting one. Record it\n    // so the caller can say so — silently falling back is the whole reason the\n    // hint is easy to get wrong.\n    out.ignoredScalarHints.push({\n      table: subquery.table,\n      uniqueKeys: tableSpecs.get(subquery.table)?.tableSpec.uniqueKeys ?? [],\n    });\n    // Return with the (possibly partially-resolved) subquery.\n    if (subquery !== condition.related.subquery) {\n      return {\n        ...condition,\n        related: {...condition.related, subquery},\n      };\n    }\n    return condition;\n  }\n\n  const value = execute(subquery, childField);\n\n  // Record the companion subquery AST so its rows are synced to the client.\n  // The client rewrites scalar subqueries to EXISTS and needs those rows.\n  out.companions.push({\n    ast: subquery,\n    childField,\n    system: condition.related.system,\n    resolvedValue: value,\n  });\n\n  if (value === undefined || value === null) {\n    // No row matched, or the matched row's `childField` is NULL (which can\n    // never satisfy the correlation, since `parentField = NULL` is never\n    // true). Either way the correlated EXISTS is false for *every* parent\n    // row, so the gate collapses to a constant: false for EXISTS, and its\n    // negation — true — for NOT EXISTS.\n    return condition.op === 'EXISTS' ? ALWAYS_FALSE : ALWAYS_TRUE;\n  }\n\n  const op = condition.op === 'EXISTS' ? '=' : 'IS NOT';\n  return {\n    type: 'simple',\n    op,\n    left: {type: 'column', name: parentField},\n    right: {type: 'literal', value},\n  } satisfies SimpleCondition;\n}\n\nconst ALWAYS_FALSE: SimpleCondition = {\n  type: 'simple',\n  op: '=',\n  left: {type: 'literal', value: 1},\n  right: {type: 'literal', value: 0},\n};\n\nconst ALWAYS_TRUE: SimpleCondition = {\n  type: 'simple',\n  op: '=',\n  left: {type: 'literal', value: 1},\n  right: {type: 'literal', value: 1},\n};\n\n/**\n * Checks if the subquery is guaranteed to return at most one deterministic row.\n *\n * This is true when all columns of at least one unique index on the subquery\n * table are equality-constrained by literal values in the WHERE clause\n * (using only AND conjunctions).\n */\nexport function isSimpleSubquery(\n  subquery: AST,\n  tableSpecs: Map<string, TableSpecWithUniqueKeys>,\n): boolean {\n  const spec = tableSpecs.get(subquery.table);\n  if (!spec) {\n    return false;\n  }\n\n  if (!subquery.where) {\n    return false;\n  }\n\n  const constraints = extractLiteralEqualityConstraints(subquery.where);\n  if (constraints.size === 0) {\n    return false;\n  }\n\n  const {uniqueKeys} = spec.tableSpec;\n  return uniqueKeys.some(key => key.every(col => constraints.has(col)));\n}\n\n/**\n * Extracts column=literal equality constraints from a condition tree,\n * only following AND conjunctions (not OR).\n */\nexport function extractLiteralEqualityConstraints(\n  condition: Condition,\n): Map<string, LiteralValue> {\n  const constraints = new Map<string, LiteralValue>();\n  collectConstraints(condition, constraints);\n  return constraints;\n}\n\nfunction collectConstraints(\n  condition: Condition,\n  constraints: Map<string, LiteralValue>,\n): void {\n  switch (condition.type) {\n    case 'simple':\n      if (condition.op === '=') {\n        if (\n          condition.left.type === 'column' &&\n          condition.right.type === 'literal'\n        ) {\n          constraints.set(condition.left.name, condition.right.value);\n        }\n      }\n      break;\n    case 'and':\n      for (const c of condition.conditions) {\n        collectConstraints(c, constraints);\n      }\n      break;\n    // OR, correlatedSubquery (non-scalar) — don't contribute constraints\n    default:\n      break;\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;AA8EA,SAAgB,8BACd,KACA,YACA,SACe;CACf,MAAM,MAAW;EAAC,YAAY,CAAC;EAAG,oBAAoB,CAAC;CAAC;CAExD,OAAO;EAAC,KADS,oBAAoB,KAAK,YAAY,SAAS,GAClD;EAAU,GAAG;CAAG;AAC/B;AAEA,SAAS,oBACP,KACA,YACA,SACA,KACK;CACL,MAAM,QAAQ,IAAI,QACd,iBAAiB,IAAI,OAAO,YAAY,SAAS,GAAG,IACpD,KAAA;CAEJ,MAAM,UAAU,IAAI,SAAS,KAAI,OAAM;EACrC,GAAG;EACH,UAAU,oBAAoB,EAAE,UAAU,YAAY,SAAS,GAAG;CACpE,EAAE;CAEF,IAAI,UAAU,IAAI,SAAS,YAAY,IAAI,SACzC,OAAO;EAAC,GAAG;EAAK;EAAO;CAAO;CAEhC,OAAO;AACT;AAEA,SAAS,iBACP,WACA,YACA,SACA,KACW;CACX,QAAQ,UAAU,MAAlB;EACE,KAAK;GACH,IAAI,UAAU,QACZ,OAAO,sBAAsB,WAAW,YAAY,SAAS,GAAG;GAGlE;IACE,MAAM,mBAAmB,oBACvB,UAAU,QAAQ,UAClB,YACA,SACA,GACF;IACA,IAAI,qBAAqB,UAAU,QAAQ,UACzC,OAAO;KACL,GAAG;KACH,SAAS;MAAC,GAAG,UAAU;MAAS,UAAU;KAAgB;IAC5D;IAEF,OAAO;GACT;EACF,KAAK;EACL,KAAK,MAAM;GACT,MAAM,WAAW,UAAU,WAAW,KAAI,MACxC,iBAAiB,GAAG,YAAY,SAAS,GAAG,CAC9C;GACA,IAAI,SAAS,OAAO,GAAG,MAAM,MAAM,UAAU,WAAW,EAAE,GACxD,OAAO;GAET,OAAO;IAAC,MAAM,UAAU;IAAM,YAAY;GAAQ;EACpD;EACA,SACE,OAAO;CACX;AACF;AAEA,SAAS,sBACP,WACA,YACA,SACA,KACW;CACX,MAAM,cAAc,UAAU,QAAQ,YAAY,YAAY;CAC9D,MAAM,aAAa,UAAU,QAAQ,YAAY,WAAW;CAI5D,MAAM,WAAW,oBACf,UAAU,QAAQ,UAClB,YACA,SACA,GACF;CAEA,IAAI,CAAC,iBAAiB,UAAU,UAAU,GAAG;EAI3C,IAAI,mBAAmB,KAAK;GAC1B,OAAO,SAAS;GAChB,YAAY,WAAW,IAAI,SAAS,KAAK,CAAC,EAAE,UAAU,cAAc,CAAC;EACvE,CAAC;EAED,IAAI,aAAa,UAAU,QAAQ,UACjC,OAAO;GACL,GAAG;GACH,SAAS;IAAC,GAAG,UAAU;IAAS;GAAQ;EAC1C;EAEF,OAAO;CACT;CAEA,MAAM,QAAQ,QAAQ,UAAU,UAAU;CAI1C,IAAI,WAAW,KAAK;EAClB,KAAK;EACL;EACA,QAAQ,UAAU,QAAQ;EAC1B,eAAe;CACjB,CAAC;CAED,IAAI,UAAU,KAAA,KAAa,UAAU,MAMnC,OAAO,UAAU,OAAO,WAAW,eAAe;CAIpD,OAAO;EACL,MAAM;EACN,IAHS,UAAU,OAAO,WAAW,MAAM;EAI3C,MAAM;GAAC,MAAM;GAAU,MAAM;EAAW;EACxC,OAAO;GAAC,MAAM;GAAW;EAAK;CAChC;AACF;AAEA,IAAM,eAAgC;CACpC,MAAM;CACN,IAAI;CACJ,MAAM;EAAC,MAAM;EAAW,OAAO;CAAC;CAChC,OAAO;EAAC,MAAM;EAAW,OAAO;CAAC;AACnC;AAEA,IAAM,cAA+B;CACnC,MAAM;CACN,IAAI;CACJ,MAAM;EAAC,MAAM;EAAW,OAAO;CAAC;CAChC,OAAO;EAAC,MAAM;EAAW,OAAO;CAAC;AACnC;;;;;;;;AASA,SAAgB,iBACd,UACA,YACS;CACT,MAAM,OAAO,WAAW,IAAI,SAAS,KAAK;CAC1C,IAAI,CAAC,MACH,OAAO;CAGT,IAAI,CAAC,SAAS,OACZ,OAAO;CAGT,MAAM,cAAc,kCAAkC,SAAS,KAAK;CACpE,IAAI,YAAY,SAAS,GACvB,OAAO;CAGT,MAAM,EAAC,eAAc,KAAK;CAC1B,OAAO,WAAW,MAAK,QAAO,IAAI,OAAM,QAAO,YAAY,IAAI,GAAG,CAAC,CAAC;AACtE;;;;;AAMA,SAAgB,kCACd,WAC2B;CAC3B,MAAM,8BAAc,IAAI,IAA0B;CAClD,mBAAmB,WAAW,WAAW;CACzC,OAAO;AACT;AAEA,SAAS,mBACP,WACA,aACM;CACN,QAAQ,UAAU,MAAlB;EACE,KAAK;GACH,IAAI,UAAU,OAAO,KAEjB;QAAA,UAAU,KAAK,SAAS,YACxB,UAAU,MAAM,SAAS,WAEzB,YAAY,IAAI,UAAU,KAAK,MAAM,UAAU,MAAM,KAAK;GAAA;GAG9D;EACF,KAAK,OACH,KAAK,MAAM,KAAK,UAAU,YACxB,mBAAmB,GAAG,WAAW;CAMvC;AACF"}

package/out/zqlite/src/sqlite-cost-model.js

@@ -21,7 +21,8 @@ function createSQLiteCostModel(db, tableSpecs) {
 	return (tableName, sort, filters, constraint) => {
 		const noSubqueryFilters = transformFilters(filters).filters;
 		const { zqlSpec } = must(tableSpecs.get(tableName));
-		const sql = compileInline(buildSelectQuery(tableName, zqlSpec, constraint, noSubqueryFilters, sort, void 0, void 0));
+		const query = buildSelectQuery(tableName, zqlSpec, constraint, noSubqueryFilters, sort, void 0, void 0);
+		const sql = compileInline(query);
 		const loops = getScanstatusLoops(db.prepare(sql));
 		assert(loops.length > 0, `Expected scanstatus to return at least one loop for query: ${sql}`);
 		return {
@@ -124,6 +125,6 @@ function btreeCost(rows) {
 	return rows * Math.log2(rows) / 10;
 }
 //#endregion
-export { createSQLiteCostModel };
+export { btreeCost, createSQLiteCostModel };
 
 //# sourceMappingURL=sqlite-cost-model.js.map

package/out/zqlite/src/sqlite-cost-model.js.map

@@ -1 +1 @@
-{"version":3,"file":"sqlite-cost-model.js","names":[],"sources":["../../../../zqlite/src/sqlite-cost-model.ts"],"sourcesContent":["import SQLite3Database from '@rocicorp/zero-sqlite3';\nimport {assert} from '../../shared/src/asserts.ts';\nimport {must} from '../../shared/src/must.ts';\nimport type {Condition, Ordering} from '../../zero-protocol/src/ast.ts';\nimport type {SchemaValue} from '../../zero-types/src/schema-value.ts';\nimport {transformFilters} from '../../zql/src/builder/filter.ts';\nimport type {\n  AccessPlan,\n  ConnectionCostModel,\n  CostModelCost,\n} from '../../zql/src/planner/planner-connection.ts';\nimport type {PlannerConstraint} from '../../zql/src/planner/planner-constraint.ts';\nimport type {Database, Statement} from './db.ts';\nimport {compileInline} from './internal/sql-inline.ts';\nimport {buildSelectQuery} from './query-builder.ts';\nimport {SQLiteStatFanout} from './sqlite-stat-fanout.ts';\n\n/**\n * Loop information returned by SQLite's scanstatus API.\n */\ninterface ScanstatusLoop {\n  /** Unique identifier for this loop */\n  selectId: number;\n  /** Parent loop ID, or 0 for root loops */\n  parentId: number;\n  /** Estimated rows emitted per turn of parent loop */\n  est: number;\n  /** EXPLAIN text for this loop to determine: b-tree vs list subquery */\n  explain: string;\n}\n\n/**\n * Creates a SQLite-based cost model for query planning.\n * Uses SQLite's scanstatus API to estimate query costs based on the actual\n * SQLite query planner's analysis.\n *\n * @param db Database instance for preparing statements\n * @param tableSpecs Map of table names to their table specs with ZQL schemas\n * @returns ConnectionCostModel function for use with the planner\n */\nexport function createSQLiteCostModel(\n  db: Database,\n  tableSpecs: Map<string, {zqlSpec: Record<string, SchemaValue>}>,\n): ConnectionCostModel {\n  const fanoutEstimator = new SQLiteStatFanout(db);\n  const tableRows = tableRowsReader(db);\n  return (\n    tableName: string,\n    sort: Ordering,\n    filters: Condition | undefined,\n    constraint: PlannerConstraint | undefined,\n  ): CostModelCost => {\n    // Transform filters to remove correlated subqueries.\n    //\n    // We use `transformFilters` (the same function the runtime uses to\n    // decide what to push to the source at connection time). It returns:\n    // - For AND: the non-CSQ conjuncts (a *superset* of the original).\n    // - For OR with any CSQ branch: `undefined` (the OR-with-CSQ is not\n    //   pushable, and matches a strict superset of any single branch).\n    //\n    // This matches what the runtime actually applies at the source. A\n    // previous implementation here had wrong OR semantics — it dropped the\n    // CSQ branch and returned the simple branch alone, which is a strict\n    // *subset* of the original OR's match set, causing the cost model to\n    // under-estimate scan rows for OR-with-CSQ shapes and biasing the\n    // planner toward semi-join plans. See the comment in\n    // `packages/zql/src/builder/filter.ts:transformFilters`.\n    const noSubqueryFilters = transformFilters(filters).filters;\n\n    // Build the SQL query using the same logic as actual queries\n    const {zqlSpec} = must(tableSpecs.get(tableName));\n\n    const query = buildSelectQuery(\n      tableName,\n      zqlSpec,\n      constraint,\n      noSubqueryFilters,\n      sort,\n      undefined, // reverse is undefined here\n      undefined, // start is undefined here\n    );\n\n    // Use compileInline to inline actual values into the SQL for cost estimation.\n    // This allows SQLite's query planner to see real values and make better decisions\n    // about index usage and query plans. This is safe here because it's only used for\n    // cost estimation, not for executing user-facing queries (which use parameterized\n    // queries via the standard compile() function).\n    const sql = compileInline(query);\n\n    // Prepare statement to get scanstatus information\n    const stmt = db.prepare(sql);\n\n    // Get scanstatus loops from the prepared statement\n    const loops = getScanstatusLoops(stmt);\n\n    // Scanstatus should always be available - if we get no loops, something is wrong\n    assert(\n      loops.length > 0,\n      `Expected scanstatus to return at least one loop for query: ${sql}`,\n    );\n\n    const ret = estimateCost(loops, (columns: string[]) =>\n      fanoutEstimator.getFanout(tableName, columns),\n    );\n\n    return {...ret, plan: accessPlan(loops, tableRows(tableName))};\n  };\n}\n\n/**\n * Returns a function that reads the number of rows in a table from\n * `sqlite_stat1`, or `undefined` if the table has not been analyzed.\n *\n * The counts are cached: they only feed plan warnings, for which the count\n * as of the first plan is close enough.\n */\nfunction tableRowsReader(db: Database): (table: string) => number | undefined {\n  let stmt: Statement | undefined;\n  const read = (table: string): number | undefined => {\n    try {\n      stmt ??= db.prepare(\n        'SELECT stat FROM sqlite_stat1 WHERE tbl = ? LIMIT 1',\n      );\n      // Every row for a table starts with the number of rows in the table.\n      const row = stmt.get<{stat: string} | undefined>(table);\n      const rows = row ? parseInt(row.stat, 10) : NaN;\n      return Number.isNaN(rows) ? undefined : rows;\n    } catch {\n      // sqlite_stat1 does not exist until the database is analyzed.\n      return undefined;\n    }\n  };\n  // Tables without statistics are cached too, as `undefined`.\n  const cache = new Map<string, number | undefined>();\n  return table => {\n    if (!cache.has(table)) {\n      cache.set(table, read(table));\n    }\n    return cache.get(table);\n  };\n}\n\n/**\n * Describes how SQLite reads the rows, from the EXPLAIN text of the\n * statement's top-level loops, e.g.:\n *\n * - `SEARCH issue USING INDEX issue_project (projectID=?)`\n * - `SCAN comment USING INDEX sqlite_autoindex_comment_1` (every row, in the\n *   order of the index)\n * - `SCAN issue` (every row, in table order)\n * - `USE TEMP B-TREE FOR ORDER BY` (a full sort)\n * - `USE TEMP B-TREE FOR LAST TERM OF ORDER BY` or `... RIGHT PART OF ORDER\n *   BY` (sorting runs of rows that tie on the index)\n */\nfunction accessPlan(\n  loops: ScanstatusLoop[],\n  tableRows: number | undefined,\n): AccessPlan {\n  const [first, ...rest] = loops.filter(loop => loop.parentId === 0);\n  let sort: AccessPlan['sort'] = 'none';\n  for (const {explain} of rest) {\n    if (\n      explain.startsWith('USE TEMP B-TREE FOR') &&\n      explain.includes('ORDER BY')\n    ) {\n      sort = explain === 'USE TEMP B-TREE FOR ORDER BY' ? 'full' : 'partial';\n    }\n  }\n  return {\n    access: first?.explain.startsWith('SCAN ') ? 'scan' : 'search',\n    sort,\n    tableRows,\n  };\n}\n\n/**\n * Gets scanstatus loop information from a prepared statement.\n * Iterates through all query elements and extracts loop statistics.\n *\n * Uses SQLITE_SCANSTAT_COMPLEX flag (1) to get all loops including sorting operations.\n *\n * @param stmt Prepared statement to get scanstatus from\n * @returns Array of loop information, or empty array if scanstatus unavailable\n */\nfunction getScanstatusLoops(stmt: Statement): ScanstatusLoop[] {\n  const loops: ScanstatusLoop[] = [];\n\n  // Iterate through query elements by incrementing idx until we get undefined\n  // which indicates we've reached the end\n  for (let idx = 0; ; idx++) {\n    const selectId = stmt.scanStatus(\n      idx,\n      SQLite3Database.SQLITE_SCANSTAT_SELECTID,\n      1,\n    );\n\n    if (selectId === undefined) {\n      break;\n    }\n\n    // A MULTI-INDEX OR nested in a branch of another one gets a scanstatus\n    // entry without an OP_Explain: SQLite reports its SELECTID and PARENTID\n    // as -1 and its EXPLAIN as NULL. The nested OR's own `MULTI-INDEX OR` and\n    // `INDEX n` loops are reported separately, and the entry is never a\n    // top-level loop, so nothing the estimate reads is lost by skipping it.\n    if (selectId === -1) {\n      continue;\n    }\n\n    loops.push({\n      selectId: must(selectId),\n      parentId: must(\n        stmt.scanStatus(idx, SQLite3Database.SQLITE_SCANSTAT_PARENTID, 1),\n      ),\n      explain: must(\n        stmt.scanStatus(idx, SQLite3Database.SQLITE_SCANSTAT_EXPLAIN, 1),\n      ),\n      est: must(stmt.scanStatus(idx, SQLite3Database.SQLITE_SCANSTAT_EST, 1)),\n    });\n  }\n\n  return loops.sort((a, b) => a.selectId - b.selectId);\n}\n\n/**\n * Estimates the cost of a query based on scanstats from sqlite3_stmt_scanstatus_v2\n */\nfunction estimateCost(\n  scanstats: ScanstatusLoop[],\n  fanout: CostModelCost['fanout'],\n): CostModelCost {\n  // Sort by selectId to process in execution order\n  const sorted = scanstats.toSorted((a, b) => a.selectId - b.selectId);\n\n  let totalRows = 0;\n  let totalCost = 0;\n\n  // Identify if there are multiple top-level (parentId=0) operations\n  // If so, the first is typically the scan, and subsequent ones are sorts\n  const topLevelOps = sorted.filter(s => s.parentId === 0);\n\n  // We only consider top level ops since ZQL queries are single-table when hitting SQLite.\n  // We do have a nested op in the case of `WHERE x IN (:arg)` but it is negligible\n  // assuming :arg is small.\n  let firstLoop = true;\n  for (const op of topLevelOps) {\n    if (firstLoop) {\n      // First top-level op is the main scan\n      // and determines the total number of rows output.\n      totalRows = op.est;\n      firstLoop = false;\n    } else {\n      if (op.explain.includes('ORDER BY')) {\n        totalCost += btreeCost(totalRows);\n      }\n    }\n  }\n\n  return {\n    rows: totalRows,\n    startupCost: totalCost,\n    fanout,\n  };\n}\n\nexport function btreeCost(rows: number): number {\n  // B-Tree construction is ~O(n log n) so we estimate the cost as such.\n  // We divide the cost by 10 because sorting in SQLite is ~10x faster\n  // than bringing the data into JS and sorting there.\n  return (rows * Math.log2(rows)) / 10;\n}\n"],"mappings":";;;;;;;;;;;;;;;;;AAwCA,SAAgB,sBACd,IACA,YACqB;CACrB,MAAM,kBAAkB,IAAI,iBAAiB,EAAE;CAC/C,MAAM,YAAY,gBAAgB,EAAE;CACpC,QACE,WACA,MACA,SACA,eACkB;EAgBlB,MAAM,oBAAoB,iBAAiB,OAAO,EAAE;EAGpD,MAAM,EAAC,YAAW,KAAK,WAAW,IAAI,SAAS,CAAC;EAiBhD,MAAM,MAAM,cAfE,iBACZ,WACA,SACA,YACA,mBACA,MACA,KAAA,GACA,KAAA,CAQwB,CAAK;EAM/B,MAAM,QAAQ,mBAHD,GAAG,QAAQ,GAGS,CAAI;EAGrC,OACE,MAAM,SAAS,GACf,8DAA8D,KAChE;EAMA,OAAO;GAAC,GAJI,aAAa,QAAQ,YAC/B,gBAAgB,UAAU,WAAW,OAAO,CAGnC;GAAK,MAAM,WAAW,OAAO,UAAU,SAAS,CAAC;EAAC;CAC/D;AACF;;;;;;;;AASA,SAAS,gBAAgB,IAAqD;CAC5E,IAAI;CACJ,MAAM,QAAQ,UAAsC;EAClD,IAAI;GACF,SAAS,GAAG,QACV,qDACF;GAEA,MAAM,MAAM,KAAK,IAAgC,KAAK;GACtD,MAAM,OAAO,MAAM,SAAS,IAAI,MAAM,EAAE,IAAI;GAC5C,OAAO,OAAO,MAAM,IAAI,IAAI,KAAA,IAAY;EAC1C,QAAQ;GAEN;EACF;CACF;CAEA,MAAM,wBAAQ,IAAI,IAAgC;CAClD,QAAO,UAAS;EACd,IAAI,CAAC,MAAM,IAAI,KAAK,GAClB,MAAM,IAAI,OAAO,KAAK,KAAK,CAAC;EAE9B,OAAO,MAAM,IAAI,KAAK;CACxB;AACF;;;;;;;;;;;;;AAcA,SAAS,WACP,OACA,WACY;CACZ,MAAM,CAAC,OAAO,GAAG,QAAQ,MAAM,QAAO,SAAQ,KAAK,aAAa,CAAC;CACjE,IAAI,OAA2B;CAC/B,KAAK,MAAM,EAAC,aAAY,MACtB,IACE,QAAQ,WAAW,qBAAqB,KACxC,QAAQ,SAAS,UAAU,GAE3B,OAAO,YAAY,iCAAiC,SAAS;CAGjE,OAAO;EACL,QAAQ,OAAO,QAAQ,WAAW,OAAO,IAAI,SAAS;EACtD;EACA;CACF;AACF;;;;;;;;;;AAWA,SAAS,mBAAmB,MAAmC;CAC7D,MAAM,QAA0B,CAAC;CAIjC,KAAK,IAAI,MAAM,IAAK,OAAO;EACzB,MAAM,WAAW,KAAK,WACpB,KACA,gBAAgB,0BAChB,CACF;EAEA,IAAI,aAAa,KAAA,GACf;EAQF,IAAI,aAAa,IACf;EAGF,MAAM,KAAK;GACT,UAAU,KAAK,QAAQ;GACvB,UAAU,KACR,KAAK,WAAW,KAAK,gBAAgB,0BAA0B,CAAC,CAClE;GACA,SAAS,KACP,KAAK,WAAW,KAAK,gBAAgB,yBAAyB,CAAC,CACjE;GACA,KAAK,KAAK,KAAK,WAAW,KAAK,gBAAgB,qBAAqB,CAAC,CAAC;EACxE,CAAC;CACH;CAEA,OAAO,MAAM,MAAM,GAAG,MAAM,EAAE,WAAW,EAAE,QAAQ;AACrD;;;;AAKA,SAAS,aACP,WACA,QACe;CAEf,MAAM,SAAS,UAAU,UAAU,GAAG,MAAM,EAAE,WAAW,EAAE,QAAQ;CAEnE,IAAI,YAAY;CAChB,IAAI,YAAY;CAIhB,MAAM,cAAc,OAAO,QAAO,MAAK,EAAE,aAAa,CAAC;CAKvD,IAAI,YAAY;CAChB,KAAK,MAAM,MAAM,aACf,IAAI,WAAW;EAGb,YAAY,GAAG;EACf,YAAY;CACd,OACE,IAAI,GAAG,QAAQ,SAAS,UAAU,GAChC,aAAa,UAAU,SAAS;CAKtC,OAAO;EACL,MAAM;EACN,aAAa;EACb;CACF;AACF;AAEA,SAAgB,UAAU,MAAsB;CAI9C,OAAQ,OAAO,KAAK,KAAK,IAAI,IAAK;AACpC"}
+{"version":3,"file":"sqlite-cost-model.js","names":[],"sources":["../../../../zqlite/src/sqlite-cost-model.ts"],"sourcesContent":["import SQLite3Database from '@rocicorp/zero-sqlite3';\nimport {assert} from '../../shared/src/asserts.ts';\nimport {must} from '../../shared/src/must.ts';\nimport type {Condition, Ordering} from '../../zero-protocol/src/ast.ts';\nimport type {SchemaValue} from '../../zero-types/src/schema-value.ts';\nimport {transformFilters} from '../../zql/src/builder/filter.ts';\nimport type {\n  AccessPlan,\n  ConnectionCostModel,\n  CostModelCost,\n} from '../../zql/src/planner/planner-connection.ts';\nimport type {PlannerConstraint} from '../../zql/src/planner/planner-constraint.ts';\nimport type {Database, Statement} from './db.ts';\nimport {compileInline} from './internal/sql-inline.ts';\nimport {buildSelectQuery} from './query-builder.ts';\nimport {SQLiteStatFanout} from './sqlite-stat-fanout.ts';\n\n/**\n * Loop information returned by SQLite's scanstatus API.\n */\ninterface ScanstatusLoop {\n  /** Unique identifier for this loop */\n  selectId: number;\n  /** Parent loop ID, or 0 for root loops */\n  parentId: number;\n  /** Estimated rows emitted per turn of parent loop */\n  est: number;\n  /** EXPLAIN text for this loop to determine: b-tree vs list subquery */\n  explain: string;\n}\n\n/**\n * Creates a SQLite-based cost model for query planning.\n * Uses SQLite's scanstatus API to estimate query costs based on the actual\n * SQLite query planner's analysis.\n *\n * @param db Database instance for preparing statements\n * @param tableSpecs Map of table names to their table specs with ZQL schemas\n * @returns ConnectionCostModel function for use with the planner\n */\nexport function createSQLiteCostModel(\n  db: Database,\n  tableSpecs: Map<string, {zqlSpec: Record<string, SchemaValue>}>,\n): ConnectionCostModel {\n  const fanoutEstimator = new SQLiteStatFanout(db);\n  const tableRows = tableRowsReader(db);\n  return (\n    tableName: string,\n    sort: Ordering,\n    filters: Condition | undefined,\n    constraint: PlannerConstraint | undefined,\n  ): CostModelCost => {\n    // Transform filters to remove correlated subqueries.\n    //\n    // We use `transformFilters` (the same function the runtime uses to\n    // decide what to push to the source at connection time). It returns:\n    // - For AND: the non-CSQ conjuncts (a *superset* of the original).\n    // - For OR with any CSQ branch: `undefined` (the OR-with-CSQ is not\n    //   pushable, and matches a strict superset of any single branch).\n    //\n    // This matches what the runtime actually applies at the source. A\n    // previous implementation here had wrong OR semantics — it dropped the\n    // CSQ branch and returned the simple branch alone, which is a strict\n    // *subset* of the original OR's match set, causing the cost model to\n    // under-estimate scan rows for OR-with-CSQ shapes and biasing the\n    // planner toward semi-join plans. See the comment in\n    // `packages/zql/src/builder/filter.ts:transformFilters`.\n    const noSubqueryFilters = transformFilters(filters).filters;\n\n    // Build the SQL query using the same logic as actual queries\n    const {zqlSpec} = must(tableSpecs.get(tableName));\n\n    const query = buildSelectQuery(\n      tableName,\n      zqlSpec,\n      constraint,\n      noSubqueryFilters,\n      sort,\n      undefined, // reverse is undefined here\n      undefined, // start is undefined here\n    );\n\n    // Use compileInline to inline actual values into the SQL for cost estimation.\n    // This allows SQLite's query planner to see real values and make better decisions\n    // about index usage and query plans. This is safe here because it's only used for\n    // cost estimation, not for executing user-facing queries (which use parameterized\n    // queries via the standard compile() function).\n    const sql = compileInline(query);\n\n    // Prepare statement to get scanstatus information\n    const stmt = db.prepare(sql);\n\n    // Get scanstatus loops from the prepared statement\n    const loops = getScanstatusLoops(stmt);\n\n    // Scanstatus should always be available - if we get no loops, something is wrong\n    assert(\n      loops.length > 0,\n      `Expected scanstatus to return at least one loop for query: ${sql}`,\n    );\n\n    const ret = estimateCost(loops, (columns: string[]) =>\n      fanoutEstimator.getFanout(tableName, columns),\n    );\n\n    return {...ret, plan: accessPlan(loops, tableRows(tableName))};\n  };\n}\n\n/**\n * Returns a function that reads the number of rows in a table from\n * `sqlite_stat1`, or `undefined` if the table has not been analyzed.\n *\n * The counts are cached: they only feed plan warnings, for which the count\n * as of the first plan is close enough.\n */\nfunction tableRowsReader(db: Database): (table: string) => number | undefined {\n  let stmt: Statement | undefined;\n  const read = (table: string): number | undefined => {\n    try {\n      stmt ??= db.prepare(\n        'SELECT stat FROM sqlite_stat1 WHERE tbl = ? LIMIT 1',\n      );\n      // Every row for a table starts with the number of rows in the table.\n      const row = stmt.get<{stat: string} | undefined>(table);\n      const rows = row ? parseInt(row.stat, 10) : NaN;\n      return Number.isNaN(rows) ? undefined : rows;\n    } catch {\n      // sqlite_stat1 does not exist until the database is analyzed.\n      return undefined;\n    }\n  };\n  // Tables without statistics are cached too, as `undefined`.\n  const cache = new Map<string, number | undefined>();\n  return table => {\n    if (!cache.has(table)) {\n      cache.set(table, read(table));\n    }\n    return cache.get(table);\n  };\n}\n\n/**\n * Describes how SQLite reads the rows, from the EXPLAIN text of the\n * statement's top-level loops, e.g.:\n *\n * - `SEARCH issue USING INDEX issue_project (projectID=?)`\n * - `SCAN comment USING INDEX sqlite_autoindex_comment_1` (every row, in the\n *   order of the index)\n * - `SCAN issue` (every row, in table order)\n * - `USE TEMP B-TREE FOR ORDER BY` (a full sort)\n * - `USE TEMP B-TREE FOR LAST TERM OF ORDER BY` or `... RIGHT PART OF ORDER\n *   BY` (sorting runs of rows that tie on the index)\n */\nfunction accessPlan(\n  loops: ScanstatusLoop[],\n  tableRows: number | undefined,\n): AccessPlan {\n  const [first, ...rest] = loops.filter(loop => loop.parentId === 0);\n  let sort: AccessPlan['sort'] = 'none';\n  for (const {explain} of rest) {\n    if (\n      explain.startsWith('USE TEMP B-TREE FOR') &&\n      explain.includes('ORDER BY')\n    ) {\n      sort = explain === 'USE TEMP B-TREE FOR ORDER BY' ? 'full' : 'partial';\n    }\n  }\n  return {\n    access: first?.explain.startsWith('SCAN ') ? 'scan' : 'search',\n    sort,\n    tableRows,\n  };\n}\n\n/**\n * Gets scanstatus loop information from a prepared statement.\n * Iterates through all query elements and extracts loop statistics.\n *\n * Uses SQLITE_SCANSTAT_COMPLEX flag (1) to get all loops including sorting operations.\n *\n * @param stmt Prepared statement to get scanstatus from\n * @returns Array of loop information, or empty array if scanstatus unavailable\n */\nfunction getScanstatusLoops(stmt: Statement): ScanstatusLoop[] {\n  const loops: ScanstatusLoop[] = [];\n\n  // Iterate through query elements by incrementing idx until we get undefined\n  // which indicates we've reached the end\n  for (let idx = 0; ; idx++) {\n    const selectId = stmt.scanStatus(\n      idx,\n      SQLite3Database.SQLITE_SCANSTAT_SELECTID,\n      1,\n    );\n\n    if (selectId === undefined) {\n      break;\n    }\n\n    // A MULTI-INDEX OR nested in a branch of another one gets a scanstatus\n    // entry without an OP_Explain: SQLite reports its SELECTID and PARENTID\n    // as -1 and its EXPLAIN as NULL. The nested OR's own `MULTI-INDEX OR` and\n    // `INDEX n` loops are reported separately, and the entry is never a\n    // top-level loop, so nothing the estimate reads is lost by skipping it.\n    if (selectId === -1) {\n      continue;\n    }\n\n    loops.push({\n      selectId: must(selectId),\n      parentId: must(\n        stmt.scanStatus(idx, SQLite3Database.SQLITE_SCANSTAT_PARENTID, 1),\n      ),\n      explain: must(\n        stmt.scanStatus(idx, SQLite3Database.SQLITE_SCANSTAT_EXPLAIN, 1),\n      ),\n      est: must(stmt.scanStatus(idx, SQLite3Database.SQLITE_SCANSTAT_EST, 1)),\n    });\n  }\n\n  return loops.sort((a, b) => a.selectId - b.selectId);\n}\n\n/**\n * Estimates the cost of a query based on scanstats from sqlite3_stmt_scanstatus_v2\n */\nfunction estimateCost(\n  scanstats: ScanstatusLoop[],\n  fanout: CostModelCost['fanout'],\n): CostModelCost {\n  // Sort by selectId to process in execution order\n  const sorted = scanstats.toSorted((a, b) => a.selectId - b.selectId);\n\n  let totalRows = 0;\n  let totalCost = 0;\n\n  // Identify if there are multiple top-level (parentId=0) operations\n  // If so, the first is typically the scan, and subsequent ones are sorts\n  const topLevelOps = sorted.filter(s => s.parentId === 0);\n\n  // We only consider top level ops since ZQL queries are single-table when hitting SQLite.\n  // We do have a nested op in the case of `WHERE x IN (:arg)` but it is negligible\n  // assuming :arg is small.\n  let firstLoop = true;\n  for (const op of topLevelOps) {\n    if (firstLoop) {\n      // First top-level op is the main scan\n      // and determines the total number of rows output.\n      totalRows = op.est;\n      firstLoop = false;\n    } else {\n      if (op.explain.includes('ORDER BY')) {\n        totalCost += btreeCost(totalRows);\n      }\n    }\n  }\n\n  return {\n    rows: totalRows,\n    startupCost: totalCost,\n    fanout,\n  };\n}\n\nexport function btreeCost(rows: number): number {\n  // B-Tree construction is ~O(n log n) so we estimate the cost as such.\n  // We divide the cost by 10 because sorting in SQLite is ~10x faster\n  // than bringing the data into JS and sorting there.\n  return (rows * Math.log2(rows)) / 10;\n}\n"],"mappings":";;;;;;;;;;;;;;;;;AAwCA,SAAgB,sBACd,IACA,YACqB;CACrB,MAAM,kBAAkB,IAAI,iBAAiB,EAAE;CAC/C,MAAM,YAAY,gBAAgB,EAAE;CACpC,QACE,WACA,MACA,SACA,eACkB;EAgBlB,MAAM,oBAAoB,iBAAiB,OAAO,CAAC,CAAC;EAGpD,MAAM,EAAC,YAAW,KAAK,WAAW,IAAI,SAAS,CAAC;EAEhD,MAAM,QAAQ,iBACZ,WACA,SACA,YACA,mBACA,MACA,KAAA,GACA,KAAA,CACF;EAOA,MAAM,MAAM,cAAc,KAAK;EAM/B,MAAM,QAAQ,mBAHD,GAAG,QAAQ,GAGS,CAAI;EAGrC,OACE,MAAM,SAAS,GACf,8DAA8D,KAChE;EAMA,OAAO;GAAC,GAJI,aAAa,QAAQ,YAC/B,gBAAgB,UAAU,WAAW,OAAO,CAGnC;GAAK,MAAM,WAAW,OAAO,UAAU,SAAS,CAAC;EAAC;CAC/D;AACF;;;;;;;;AASA,SAAS,gBAAgB,IAAqD;CAC5E,IAAI;CACJ,MAAM,QAAQ,UAAsC;EAClD,IAAI;GACF,SAAS,GAAG,QACV,qDACF;GAEA,MAAM,MAAM,KAAK,IAAgC,KAAK;GACtD,MAAM,OAAO,MAAM,SAAS,IAAI,MAAM,EAAE,IAAI;GAC5C,OAAO,OAAO,MAAM,IAAI,IAAI,KAAA,IAAY;EAC1C,QAAQ;GAEN;EACF;CACF;CAEA,MAAM,wBAAQ,IAAI,IAAgC;CAClD,QAAO,UAAS;EACd,IAAI,CAAC,MAAM,IAAI,KAAK,GAClB,MAAM,IAAI,OAAO,KAAK,KAAK,CAAC;EAE9B,OAAO,MAAM,IAAI,KAAK;CACxB;AACF;;;;;;;;;;;;;AAcA,SAAS,WACP,OACA,WACY;CACZ,MAAM,CAAC,OAAO,GAAG,QAAQ,MAAM,QAAO,SAAQ,KAAK,aAAa,CAAC;CACjE,IAAI,OAA2B;CAC/B,KAAK,MAAM,EAAC,aAAY,MACtB,IACE,QAAQ,WAAW,qBAAqB,KACxC,QAAQ,SAAS,UAAU,GAE3B,OAAO,YAAY,iCAAiC,SAAS;CAGjE,OAAO;EACL,QAAQ,OAAO,QAAQ,WAAW,OAAO,IAAI,SAAS;EACtD;EACA;CACF;AACF;;;;;;;;;;AAWA,SAAS,mBAAmB,MAAmC;CAC7D,MAAM,QAA0B,CAAC;CAIjC,KAAK,IAAI,MAAM,IAAK,OAAO;EACzB,MAAM,WAAW,KAAK,WACpB,KACA,gBAAgB,0BAChB,CACF;EAEA,IAAI,aAAa,KAAA,GACf;EAQF,IAAI,aAAa,IACf;EAGF,MAAM,KAAK;GACT,UAAU,KAAK,QAAQ;GACvB,UAAU,KACR,KAAK,WAAW,KAAK,gBAAgB,0BAA0B,CAAC,CAClE;GACA,SAAS,KACP,KAAK,WAAW,KAAK,gBAAgB,yBAAyB,CAAC,CACjE;GACA,KAAK,KAAK,KAAK,WAAW,KAAK,gBAAgB,qBAAqB,CAAC,CAAC;EACxE,CAAC;CACH;CAEA,OAAO,MAAM,MAAM,GAAG,MAAM,EAAE,WAAW,EAAE,QAAQ;AACrD;;;;AAKA,SAAS,aACP,WACA,QACe;CAEf,MAAM,SAAS,UAAU,UAAU,GAAG,MAAM,EAAE,WAAW,EAAE,QAAQ;CAEnE,IAAI,YAAY;CAChB,IAAI,YAAY;CAIhB,MAAM,cAAc,OAAO,QAAO,MAAK,EAAE,aAAa,CAAC;CAKvD,IAAI,YAAY;CAChB,KAAK,MAAM,MAAM,aACf,IAAI,WAAW;EAGb,YAAY,GAAG;EACf,YAAY;CACd,OACE,IAAI,GAAG,QAAQ,SAAS,UAAU,GAChC,aAAa,UAAU,SAAS;CAKtC,OAAO;EACL,MAAM;EACN,aAAa;EACb;CACF;AACF;AAEA,SAAgB,UAAU,MAAsB;CAI9C,OAAQ,OAAO,KAAK,KAAK,IAAI,IAAK;AACpC"}

package/out/zqlite/src/sqlite-stat-fanout.js.map

@@ -1 +1 @@
-{"version":3,"file":"sqlite-stat-fanout.js","names":["#db","#defaultFanout","#cache","#stat4Stmt","#stat1Stmt","#indexStmt","#getFanoutFromStat4","#getFanoutFromStat1","#findIndexForColumns","#isPrefixMatch"],"sources":["../../../../zqlite/src/sqlite-stat-fanout.ts"],"sourcesContent":["import {newArray} from '../../shared/src/arrays.ts';\nimport {getOrInsertComputed} from '../../shared/src/map.ts';\nimport type {Database} from './db.ts';\nimport {isSampleNull} from './sqlite-stat4-sample.ts';\n\n/**\n * Result of fanout calculation from SQLite statistics.\n */\nexport interface FanoutResult {\n  /**\n   * The fanout value (average rows per distinct value of the join column).\n   * For non-NULL joins, this represents how many child rows exist per parent key.\n   */\n  fanout: number;\n  confidence: 'high' | 'med' | 'none';\n\n  /**\n   * Source of the fanout calculation.\n   * - 'stat4': From sqlite_stat4 histogram (most accurate, excludes NULLs)\n   * - 'stat1': From sqlite_stat1 average (includes NULLs, may overestimate)\n   * - 'default': Fallback constant when statistics unavailable\n   */\n  source: 'stat4' | 'stat1' | 'default';\n}\n\n/**\n * Sample from sqlite_stat4 histogram.\n */\ninterface Stat4Sample {\n  /** \"N1 N2\" = rows equal to sample (N1=first col, N2=if composite) */\n  neq: string;\n  /** \"N1 N2\" = rows less than sample */\n  nlt: string;\n  /** \"N1 N2\" = distinct values less than sample */\n  ndlt: string;\n  /** The actual sample value (binary encoded) */\n  sample: Buffer;\n}\n\n/**\n * Computes join fanout factors from SQLite statistics tables.\n *\n * Fanout is the average number of child rows per distinct parent key value,\n * used to estimate join cardinality in query planning.\n *\n * ## Problem\n *\n * sqlite_stat1 includes NULL rows in its calculation, which can significantly\n * overestimate fanout for sparse foreign keys:\n *\n * ```\n * Example: 100 tasks, 20 with project_id, 80 with NULL\n * - stat1 reports: \"100 17\" → fanout = 17 (WRONG - includes NULLs)\n * - stat4 shows: NULL samples with fanout=80, non-NULL samples with fanout=4\n * - True fanout: 4 (CORRECT)\n * ```\n *\n * ## Solution\n *\n * This class uses sqlite_stat4 histogram to separate NULL and non-NULL samples,\n * providing accurate fanout for non-NULL joins.\n *\n * ## Usage\n *\n * ```typescript\n * const calculator = new SQLiteStatFanout(db);\n *\n * // Get fanout for posts.userId → users.id join\n * const result = calculator.getFanout('posts', 'userId');\n *\n * if (result.source === 'stat4') {\n *   // Accurate: excludes NULLs, samples actual distribution\n *   console.log(`Fanout: ${result.fanout} (from stat4)`);\n * } else if (result.source === 'stat1') {\n *   // Conservative: includes NULLs, may overestimate\n *   console.log(`Fanout: ${result.fanout} (from stat1, includes NULLs)`);\n * } else {\n *   // Fallback: no statistics available\n *   console.log(`Fanout: ${result.fanout} (default estimate)`);\n * }\n * ```\n *\n * ## Requirements\n *\n * - SQLite compiled with ENABLE_STAT4 (most builds include this)\n * - `ANALYZE` command run on the database\n * - Index exists on the join column\n *\n * @see https://sqlite.org/fileformat2.html#stat4tab\n * @see packages/zql/src/planner/SELECTIVITY_PLAN.md\n */\nexport class SQLiteStatFanout {\n  readonly #db: Database;\n  readonly #defaultFanout: number;\n\n  /**\n   * Cache of fanout results by table and columns.\n   * Key format: \"tableName:col1,col2,col3\" (sorted alphabetically)\n   */\n  readonly #cache = new Map<string, FanoutResult>();\n\n  /**\n   * Prepared statements for querying SQLite statistics tables.\n   * Prepared once in constructor for performance.\n   */\n  readonly #stat4Stmt: ReturnType<Database['prepare']>;\n  readonly #stat1Stmt: ReturnType<Database['prepare']>;\n  readonly #indexStmt: ReturnType<Database['prepare']>;\n\n  /**\n   * Creates a new fanout calculator.\n   *\n   * @param db Database instance\n   * @param defaultFanout Default fanout when statistics unavailable (default: 3)\n   *                      - 1: Conservative (assumes FK relationships)\n   *                      - 3: Moderate (recommended, safe middle ground)\n   *                      - 10: SQLite's default (optimistic)\n   */\n  constructor(db: Database, defaultFanout = 3) {\n    this.#db = db;\n    this.#defaultFanout = defaultFanout;\n\n    // Prepare SQL statements once for reuse across multiple getFanout() calls\n    this.#stat4Stmt = this.#db.prepare(`\n      SELECT neq, nlt, ndlt, sample\n      FROM sqlite_stat4\n      WHERE tbl = ? AND idx = ?\n      ORDER BY nlt\n    `);\n\n    this.#stat1Stmt = this.#db.prepare(`\n      SELECT stat\n      FROM sqlite_stat1\n      WHERE tbl = ? AND idx = ?\n    `);\n\n    this.#indexStmt = this.#db.prepare(`\n      SELECT il.name as index_name, ii.seqno, ii.name as column_name\n      FROM pragma_index_list(?) il\n      JOIN pragma_index_info(il.name) ii\n      WHERE il.partial = 0\n      ORDER BY il.seq, ii.seqno\n    `);\n  }\n\n  /**\n   * Gets the fanout factor for join column(s).\n   *\n   * Fanout = average number of child rows per distinct parent key value(s).\n   *\n   * ## Strategy\n   *\n   * 1. Try sqlite_stat4 (best): Histogram with separate NULL/non-NULL samples\n   * 2. Fallback to sqlite_stat1: Average across all rows (includes NULLs)\n   * 3. Fallback to default: When no statistics available\n   *\n   * ## Compound Indexes\n   *\n   * For multi-column joins, finds indexes where ALL columns appear as an\n   * exact prefix. Uses the appropriate depth in stat1/stat4.\n   *\n   * Example:\n   * - Columns: `['customerId', 'storeId']`\n   * - Matches index: `(customerId, storeId, date)` at depth 2\n   * - Uses stat1 parts[2] or stat4 neq[1] for accurate fanout\n   *\n   * ## Caching\n   *\n   * Results are cached per (table, columns) combination. Clear the cache if\n   * you run ANALYZE to update statistics.\n   *\n   * @param tableName Table containing the join column(s)\n   * @param columns Array of column names (one or more columns)\n   * @returns Fanout result with value and source\n   */\n  getFanout(tableName: string, columns: string[]): FanoutResult {\n    // Cache key uses sorted columns for consistency\n    const cacheKey = `${tableName}:${columns.toSorted().join(',')}`;\n    const cached = this.#cache.get(cacheKey);\n    if (cached) {\n      return cached;\n    }\n\n    // Strategy 1: Try stat4 first (most accurate)\n    // NOTE: columns are NOT sorted - preserves Object.keys() order from constraint\n    // Matching is order-independent (flexible), but we keep original order for consistency\n    const stat4Result = this.#getFanoutFromStat4(tableName, columns);\n    if (stat4Result) {\n      this.#cache.set(cacheKey, stat4Result);\n      return stat4Result;\n    }\n\n    // Strategy 2: Fallback to stat1 (includes NULLs)\n    const stat1Result = this.#getFanoutFromStat1(tableName, columns);\n    if (stat1Result) {\n      this.#cache.set(cacheKey, stat1Result);\n      return stat1Result;\n    }\n\n    // Strategy 3: Use default\n    const defaultResult: FanoutResult = {\n      fanout: this.#defaultFanout,\n      confidence: 'none',\n      source: 'default',\n    };\n    this.#cache.set(cacheKey, defaultResult);\n    return defaultResult;\n  }\n\n  /**\n   * Clears the fanout cache.\n   * Call this after running ANALYZE to pick up updated statistics.\n   */\n  clearCache(): void {\n    this.#cache.clear();\n  }\n\n  /**\n   * Gets fanout from sqlite_stat4 histogram.\n   *\n   * Queries stat4 samples, decodes to identify NULLs, and returns\n   * the median fanout of non-NULL samples.\n   *\n   * For compound indexes, uses the neq value at the appropriate depth.\n   *\n   * @param columns Array of column names to get fanout for\n   * @returns Fanout result or undefined if stat4 unavailable\n   */\n  #getFanoutFromStat4(\n    tableName: string,\n    columns: string[],\n  ): FanoutResult | undefined {\n    try {\n      // Find index containing the columns as a prefix\n      const indexInfo = this.#findIndexForColumns(tableName, columns);\n      if (!indexInfo) {\n        return undefined;\n      }\n\n      // Query stat4 samples for this index (using prepared statement)\n      const samples = this.#stat4Stmt.all(\n        tableName,\n        indexInfo.indexName,\n      ) as Stat4Sample[];\n\n      if (samples.length === 0) {\n        return undefined;\n      }\n\n      // Decode samples and separate NULL from non-NULL\n      // Use depth-1 for neq array index (depth is 1-based, array is 0-based)\n      const neqIndex = indexInfo.depth - 1;\n      const decodedSamples = samples.map(s => {\n        const neqParts = s.neq.split(' ');\n        return {\n          fanout: parseInt(neqParts[neqIndex] ?? neqParts[0], 10),\n          isNull: isSampleNull(s.sample),\n        };\n      });\n\n      const nonNullSamples = decodedSamples.filter(s => !s.isNull);\n\n      if (nonNullSamples.length === 0) {\n        // All samples are NULL - return fanout of 0 since NULLs don't match in joins\n        return {\n          fanout: 0,\n          source: 'stat4',\n          confidence: 'high',\n        };\n      }\n\n      // Use median of non-NULL fanouts (more robust than average)\n      const fanouts = nonNullSamples.map(s => s.fanout).sort((a, b) => a - b);\n      const medianFanout =\n        fanouts.length % 2 === 0\n          ? Math.floor(\n              (fanouts[fanouts.length / 2 - 1] + fanouts[fanouts.length / 2]) /\n                2,\n            )\n          : fanouts[Math.floor(fanouts.length / 2)];\n\n      return {\n        fanout: medianFanout,\n        source: 'stat4',\n        confidence: 'high',\n      };\n    } catch {\n      // stat4 table may not exist or query may fail\n      return undefined;\n    }\n  }\n\n  /**\n   * Gets fanout from sqlite_stat1 average.\n   *\n   * Note: This includes NULL rows in the calculation and may overestimate\n   * fanout for sparse foreign keys.\n   *\n   * For compound indexes, uses the stat value at the appropriate depth.\n   *\n   * @param columns Array of column names to get fanout for\n   * @returns Fanout result or undefined if stat1 unavailable\n   */\n  #getFanoutFromStat1(\n    tableName: string,\n    columns: string[],\n  ): FanoutResult | undefined {\n    try {\n      // Find index containing the columns as a prefix\n      const indexInfo = this.#findIndexForColumns(tableName, columns);\n      if (!indexInfo) {\n        return undefined;\n      }\n\n      // Query stat1 for this index (using prepared statement)\n      const result = this.#stat1Stmt.get(tableName, indexInfo.indexName) as\n        | {stat: string}\n        | undefined;\n\n      if (!result) {\n        return undefined;\n      }\n\n      const parts = result.stat.split(' ');\n      // Check if we have enough parts for the requested depth\n      if (parts.length < indexInfo.depth + 1) {\n        return undefined;\n      }\n\n      const fanout = parseInt(parts[indexInfo.depth], 10);\n      if (isNaN(fanout)) {\n        return undefined;\n      }\n\n      return {\n        fanout,\n        source: 'stat1',\n        confidence: 'med',\n      };\n    } catch {\n      return undefined;\n    }\n  }\n\n  /**\n   * Finds an index that can be used to get statistics for column(s).\n   *\n   * Uses pragma_index_list and pragma_index_info to reliably get index\n   * column names, avoiding brittle SQL parsing. Includes all indices:\n   * user-created (CREATE INDEX), PRIMARY KEY, and UNIQUE constraints.\n   *\n   * Uses flexible matching: Finds indexes where ALL columns appear in the\n   * first N positions, regardless of order. This works because SQLite statistics\n   * at depth N represent the fanout for the combination of the first N columns,\n   * and combinations are order-independent.\n   *\n   * Example:\n   * - columns: ['customerId', 'storeId']\n   * - Matches: (customerId, storeId, date) at depth 2 ✅\n   * - Matches: (storeId, customerId, date) at depth 2 ✅ (flexible order)\n   * - Does NOT match: (date, customerId, storeId) ❌ (columns not in first 2 positions)\n   * - Does NOT match: (customerId, date, storeId) ❌ (storeId not in first 2 positions)\n   *\n   * @param columns Array of column names (order-independent for matching)\n   * @returns Index info with name and depth, or undefined if no match\n   */\n  #findIndexForColumns(\n    tableName: string,\n    columns: string[],\n  ): {indexName: string; depth: number} | undefined {\n    try {\n      // Query returns all columns for all indexes (including PK/UNIQUE) in order\n      const rows = this.#indexStmt.all(tableName) as {\n        index_name: string;\n        seqno: number;\n        column_name: string;\n      }[];\n\n      // Group by index name\n      const indexMap = new Map<string, string[]>();\n      for (const row of rows) {\n        getOrInsertComputed(indexMap, row.index_name, newArray).push(\n          row.column_name,\n        );\n      }\n\n      // Check each index for prefix match\n      for (const [indexName, indexColumns] of indexMap) {\n        if (this.#isPrefixMatch(columns, indexColumns)) {\n          return {\n            indexName,\n            depth: columns.length,\n          };\n        }\n      }\n\n      return undefined;\n    } catch {\n      return undefined;\n    }\n  }\n\n  /**\n   * Checks if all queryColumns exist in the first N positions of indexColumns,\n   * regardless of order.\n   *\n   * This allows flexible matching: constraint {a, b} matches both index (a, b, c)\n   * and index (b, a, c) at depth 2, since both represent the fanout for the\n   * combination of columns a and b.\n   *\n   * Gaps are NOT allowed: constraint {a, c} does NOT match index (a, b, c)\n   * because no depth represents just (a, c) without b. Statistics are cumulative\n   * from position 0.\n   *\n   * @param queryColumns Columns we're looking for (from constraint)\n   * @param indexColumns Columns in the index (in order)\n   * @returns true if all queryColumns exist in indexColumns[0...queryColumns.length-1]\n   */\n  #isPrefixMatch(queryColumns: string[], indexColumns: string[]): boolean {\n    if (queryColumns.length > indexColumns.length) {\n      return false;\n    }\n\n    // Get the prefix of the index that we're checking against\n    const indexPrefix = indexColumns.slice(0, queryColumns.length);\n\n    // Normalize to lowercase for case-insensitive comparison\n    const indexPrefixLower = new Set(indexPrefix.map(col => col.toLowerCase()));\n    const queryColumnsLower = queryColumns.map(col => col.toLowerCase());\n\n    // Check if ALL query columns exist in the index prefix\n    return queryColumnsLower.every(queryCol => indexPrefixLower.has(queryCol));\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AA2FA,IAAa,mBAAb,MAA8B;CAC5B;CACA;;;;;CAMA,yBAAkB,IAAI,IAA0B;;;;;CAMhD;CACA;CACA;;;;;;;;;;CAWA,YAAY,IAAc,gBAAgB,GAAG;EAC3C,KAAKA,MAAM;EACX,KAAKC,iBAAiB;EAGtB,KAAKE,aAAa,KAAKH,IAAI,QAAQ;;;;;KAKlC;EAED,KAAKI,aAAa,KAAKJ,IAAI,QAAQ;;;;KAIlC;EAED,KAAKK,aAAa,KAAKL,IAAI,QAAQ;;;;;;KAMlC;CACH;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAgCA,UAAU,WAAmB,SAAiC;EAE5D,MAAM,WAAW,GAAG,UAAU,GAAG,QAAQ,SAAS,EAAE,KAAK,GAAG;EAC5D,MAAM,SAAS,KAAKE,OAAO,IAAI,QAAQ;EACvC,IAAI,QACF,OAAO;EAMT,MAAM,cAAc,KAAKI,oBAAoB,WAAW,OAAO;EAC/D,IAAI,aAAa;GACf,KAAKJ,OAAO,IAAI,UAAU,WAAW;GACrC,OAAO;EACT;EAGA,MAAM,cAAc,KAAKK,oBAAoB,WAAW,OAAO;EAC/D,IAAI,aAAa;GACf,KAAKL,OAAO,IAAI,UAAU,WAAW;GACrC,OAAO;EACT;EAGA,MAAM,gBAA8B;GAClC,QAAQ,KAAKD;GACb,YAAY;GACZ,QAAQ;EACV;EACA,KAAKC,OAAO,IAAI,UAAU,aAAa;EACvC,OAAO;CACT;;;;;CAMA,aAAmB;EACjB,KAAKA,OAAO,MAAM;CACpB;;;;;;;;;;;;CAaA,oBACE,WACA,SAC0B;EAC1B,IAAI;GAEF,MAAM,YAAY,KAAKM,qBAAqB,WAAW,OAAO;GAC9D,IAAI,CAAC,WACH;GAIF,MAAM,UAAU,KAAKL,WAAW,IAC9B,WACA,UAAU,SACZ;GAEA,IAAI,QAAQ,WAAW,GACrB;GAKF,MAAM,WAAW,UAAU,QAAQ;GASnC,MAAM,iBARiB,QAAQ,KAAI,MAAK;IACtC,MAAM,WAAW,EAAE,IAAI,MAAM,GAAG;IAChC,OAAO;KACL,QAAQ,SAAS,SAAS,aAAa,SAAS,IAAI,EAAE;KACtD,QAAQ,aAAa,EAAE,MAAM;IAC/B;GACF,CAEuB,EAAe,QAAO,MAAK,CAAC,EAAE,MAAM;GAE3D,IAAI,eAAe,WAAW,GAE5B,OAAO;IACL,QAAQ;IACR,QAAQ;IACR,YAAY;GACd;GAIF,MAAM,UAAU,eAAe,KAAI,MAAK,EAAE,MAAM,EAAE,MAAM,GAAG,MAAM,IAAI,CAAC;GAStE,OAAO;IACL,QARA,QAAQ,SAAS,MAAM,IACnB,KAAK,OACF,QAAQ,QAAQ,SAAS,IAAI,KAAK,QAAQ,QAAQ,SAAS,MAC1D,CACJ,IACA,QAAQ,KAAK,MAAM,QAAQ,SAAS,CAAC;IAIzC,QAAQ;IACR,YAAY;GACd;EACF,QAAQ;GAEN;EACF;CACF;;;;;;;;;;;;CAaA,oBACE,WACA,SAC0B;EAC1B,IAAI;GAEF,MAAM,YAAY,KAAKK,qBAAqB,WAAW,OAAO;GAC9D,IAAI,CAAC,WACH;GAIF,MAAM,SAAS,KAAKJ,WAAW,IAAI,WAAW,UAAU,SAAS;GAIjE,IAAI,CAAC,QACH;GAGF,MAAM,QAAQ,OAAO,KAAK,MAAM,GAAG;GAEnC,IAAI,MAAM,SAAS,UAAU,QAAQ,GACnC;GAGF,MAAM,SAAS,SAAS,MAAM,UAAU,QAAQ,EAAE;GAClD,IAAI,MAAM,MAAM,GACd;GAGF,OAAO;IACL;IACA,QAAQ;IACR,YAAY;GACd;EACF,QAAQ;GACN;EACF;CACF;;;;;;;;;;;;;;;;;;;;;;;CAwBA,qBACE,WACA,SACgD;EAChD,IAAI;GAEF,MAAM,OAAO,KAAKC,WAAW,IAAI,SAAS;GAO1C,MAAM,2BAAW,IAAI,IAAsB;GAC3C,KAAK,MAAM,OAAO,MAChB,oBAAoB,UAAU,IAAI,YAAY,QAAQ,EAAE,KACtD,IAAI,WACN;GAIF,KAAK,MAAM,CAAC,WAAW,iBAAiB,UACtC,IAAI,KAAKI,eAAe,SAAS,YAAY,GAC3C,OAAO;IACL;IACA,OAAO,QAAQ;GACjB;GAIJ;EACF,QAAQ;GACN;EACF;CACF;;;;;;;;;;;;;;;;;CAkBA,eAAe,cAAwB,cAAiC;EACtE,IAAI,aAAa,SAAS,aAAa,QACrC,OAAO;EAIT,MAAM,cAAc,aAAa,MAAM,GAAG,aAAa,MAAM;EAG7D,MAAM,mBAAmB,IAAI,IAAI,YAAY,KAAI,QAAO,IAAI,YAAY,CAAC,CAAC;EAI1E,OAH0B,aAAa,KAAI,QAAO,IAAI,YAAY,CAG3D,EAAkB,OAAM,aAAY,iBAAiB,IAAI,QAAQ,CAAC;CAC3E;AACF"}
+{"version":3,"file":"sqlite-stat-fanout.js","names":[],"sources":["../../../../zqlite/src/sqlite-stat-fanout.ts"],"sourcesContent":["import {newArray} from '../../shared/src/arrays.ts';\nimport {getOrInsertComputed} from '../../shared/src/map.ts';\nimport type {Database} from './db.ts';\nimport {isSampleNull} from './sqlite-stat4-sample.ts';\n\n/**\n * Result of fanout calculation from SQLite statistics.\n */\nexport interface FanoutResult {\n  /**\n   * The fanout value (average rows per distinct value of the join column).\n   * For non-NULL joins, this represents how many child rows exist per parent key.\n   */\n  fanout: number;\n  confidence: 'high' | 'med' | 'none';\n\n  /**\n   * Source of the fanout calculation.\n   * - 'stat4': From sqlite_stat4 histogram (most accurate, excludes NULLs)\n   * - 'stat1': From sqlite_stat1 average (includes NULLs, may overestimate)\n   * - 'default': Fallback constant when statistics unavailable\n   */\n  source: 'stat4' | 'stat1' | 'default';\n}\n\n/**\n * Sample from sqlite_stat4 histogram.\n */\ninterface Stat4Sample {\n  /** \"N1 N2\" = rows equal to sample (N1=first col, N2=if composite) */\n  neq: string;\n  /** \"N1 N2\" = rows less than sample */\n  nlt: string;\n  /** \"N1 N2\" = distinct values less than sample */\n  ndlt: string;\n  /** The actual sample value (binary encoded) */\n  sample: Buffer;\n}\n\n/**\n * Computes join fanout factors from SQLite statistics tables.\n *\n * Fanout is the average number of child rows per distinct parent key value,\n * used to estimate join cardinality in query planning.\n *\n * ## Problem\n *\n * sqlite_stat1 includes NULL rows in its calculation, which can significantly\n * overestimate fanout for sparse foreign keys:\n *\n * ```\n * Example: 100 tasks, 20 with project_id, 80 with NULL\n * - stat1 reports: \"100 17\" → fanout = 17 (WRONG - includes NULLs)\n * - stat4 shows: NULL samples with fanout=80, non-NULL samples with fanout=4\n * - True fanout: 4 (CORRECT)\n * ```\n *\n * ## Solution\n *\n * This class uses sqlite_stat4 histogram to separate NULL and non-NULL samples,\n * providing accurate fanout for non-NULL joins.\n *\n * ## Usage\n *\n * ```typescript\n * const calculator = new SQLiteStatFanout(db);\n *\n * // Get fanout for posts.userId → users.id join\n * const result = calculator.getFanout('posts', 'userId');\n *\n * if (result.source === 'stat4') {\n *   // Accurate: excludes NULLs, samples actual distribution\n *   console.log(`Fanout: ${result.fanout} (from stat4)`);\n * } else if (result.source === 'stat1') {\n *   // Conservative: includes NULLs, may overestimate\n *   console.log(`Fanout: ${result.fanout} (from stat1, includes NULLs)`);\n * } else {\n *   // Fallback: no statistics available\n *   console.log(`Fanout: ${result.fanout} (default estimate)`);\n * }\n * ```\n *\n * ## Requirements\n *\n * - SQLite compiled with ENABLE_STAT4 (most builds include this)\n * - `ANALYZE` command run on the database\n * - Index exists on the join column\n *\n * @see https://sqlite.org/fileformat2.html#stat4tab\n * @see packages/zql/src/planner/SELECTIVITY_PLAN.md\n */\nexport class SQLiteStatFanout {\n  readonly #db: Database;\n  readonly #defaultFanout: number;\n\n  /**\n   * Cache of fanout results by table and columns.\n   * Key format: \"tableName:col1,col2,col3\" (sorted alphabetically)\n   */\n  readonly #cache = new Map<string, FanoutResult>();\n\n  /**\n   * Prepared statements for querying SQLite statistics tables.\n   * Prepared once in constructor for performance.\n   */\n  readonly #stat4Stmt: ReturnType<Database['prepare']>;\n  readonly #stat1Stmt: ReturnType<Database['prepare']>;\n  readonly #indexStmt: ReturnType<Database['prepare']>;\n\n  /**\n   * Creates a new fanout calculator.\n   *\n   * @param db Database instance\n   * @param defaultFanout Default fanout when statistics unavailable (default: 3)\n   *                      - 1: Conservative (assumes FK relationships)\n   *                      - 3: Moderate (recommended, safe middle ground)\n   *                      - 10: SQLite's default (optimistic)\n   */\n  constructor(db: Database, defaultFanout = 3) {\n    this.#db = db;\n    this.#defaultFanout = defaultFanout;\n\n    // Prepare SQL statements once for reuse across multiple getFanout() calls\n    this.#stat4Stmt = this.#db.prepare(`\n      SELECT neq, nlt, ndlt, sample\n      FROM sqlite_stat4\n      WHERE tbl = ? AND idx = ?\n      ORDER BY nlt\n    `);\n\n    this.#stat1Stmt = this.#db.prepare(`\n      SELECT stat\n      FROM sqlite_stat1\n      WHERE tbl = ? AND idx = ?\n    `);\n\n    this.#indexStmt = this.#db.prepare(`\n      SELECT il.name as index_name, ii.seqno, ii.name as column_name\n      FROM pragma_index_list(?) il\n      JOIN pragma_index_info(il.name) ii\n      WHERE il.partial = 0\n      ORDER BY il.seq, ii.seqno\n    `);\n  }\n\n  /**\n   * Gets the fanout factor for join column(s).\n   *\n   * Fanout = average number of child rows per distinct parent key value(s).\n   *\n   * ## Strategy\n   *\n   * 1. Try sqlite_stat4 (best): Histogram with separate NULL/non-NULL samples\n   * 2. Fallback to sqlite_stat1: Average across all rows (includes NULLs)\n   * 3. Fallback to default: When no statistics available\n   *\n   * ## Compound Indexes\n   *\n   * For multi-column joins, finds indexes where ALL columns appear as an\n   * exact prefix. Uses the appropriate depth in stat1/stat4.\n   *\n   * Example:\n   * - Columns: `['customerId', 'storeId']`\n   * - Matches index: `(customerId, storeId, date)` at depth 2\n   * - Uses stat1 parts[2] or stat4 neq[1] for accurate fanout\n   *\n   * ## Caching\n   *\n   * Results are cached per (table, columns) combination. Clear the cache if\n   * you run ANALYZE to update statistics.\n   *\n   * @param tableName Table containing the join column(s)\n   * @param columns Array of column names (one or more columns)\n   * @returns Fanout result with value and source\n   */\n  getFanout(tableName: string, columns: string[]): FanoutResult {\n    // Cache key uses sorted columns for consistency\n    const cacheKey = `${tableName}:${columns.toSorted().join(',')}`;\n    const cached = this.#cache.get(cacheKey);\n    if (cached) {\n      return cached;\n    }\n\n    // Strategy 1: Try stat4 first (most accurate)\n    // NOTE: columns are NOT sorted - preserves Object.keys() order from constraint\n    // Matching is order-independent (flexible), but we keep original order for consistency\n    const stat4Result = this.#getFanoutFromStat4(tableName, columns);\n    if (stat4Result) {\n      this.#cache.set(cacheKey, stat4Result);\n      return stat4Result;\n    }\n\n    // Strategy 2: Fallback to stat1 (includes NULLs)\n    const stat1Result = this.#getFanoutFromStat1(tableName, columns);\n    if (stat1Result) {\n      this.#cache.set(cacheKey, stat1Result);\n      return stat1Result;\n    }\n\n    // Strategy 3: Use default\n    const defaultResult: FanoutResult = {\n      fanout: this.#defaultFanout,\n      confidence: 'none',\n      source: 'default',\n    };\n    this.#cache.set(cacheKey, defaultResult);\n    return defaultResult;\n  }\n\n  /**\n   * Clears the fanout cache.\n   * Call this after running ANALYZE to pick up updated statistics.\n   */\n  clearCache(): void {\n    this.#cache.clear();\n  }\n\n  /**\n   * Gets fanout from sqlite_stat4 histogram.\n   *\n   * Queries stat4 samples, decodes to identify NULLs, and returns\n   * the median fanout of non-NULL samples.\n   *\n   * For compound indexes, uses the neq value at the appropriate depth.\n   *\n   * @param columns Array of column names to get fanout for\n   * @returns Fanout result or undefined if stat4 unavailable\n   */\n  #getFanoutFromStat4(\n    tableName: string,\n    columns: string[],\n  ): FanoutResult | undefined {\n    try {\n      // Find index containing the columns as a prefix\n      const indexInfo = this.#findIndexForColumns(tableName, columns);\n      if (!indexInfo) {\n        return undefined;\n      }\n\n      // Query stat4 samples for this index (using prepared statement)\n      const samples = this.#stat4Stmt.all(\n        tableName,\n        indexInfo.indexName,\n      ) as Stat4Sample[];\n\n      if (samples.length === 0) {\n        return undefined;\n      }\n\n      // Decode samples and separate NULL from non-NULL\n      // Use depth-1 for neq array index (depth is 1-based, array is 0-based)\n      const neqIndex = indexInfo.depth - 1;\n      const decodedSamples = samples.map(s => {\n        const neqParts = s.neq.split(' ');\n        return {\n          fanout: parseInt(neqParts[neqIndex] ?? neqParts[0], 10),\n          isNull: isSampleNull(s.sample),\n        };\n      });\n\n      const nonNullSamples = decodedSamples.filter(s => !s.isNull);\n\n      if (nonNullSamples.length === 0) {\n        // All samples are NULL - return fanout of 0 since NULLs don't match in joins\n        return {\n          fanout: 0,\n          source: 'stat4',\n          confidence: 'high',\n        };\n      }\n\n      // Use median of non-NULL fanouts (more robust than average)\n      const fanouts = nonNullSamples.map(s => s.fanout).sort((a, b) => a - b);\n      const medianFanout =\n        fanouts.length % 2 === 0\n          ? Math.floor(\n              (fanouts[fanouts.length / 2 - 1] + fanouts[fanouts.length / 2]) /\n                2,\n            )\n          : fanouts[Math.floor(fanouts.length / 2)];\n\n      return {\n        fanout: medianFanout,\n        source: 'stat4',\n        confidence: 'high',\n      };\n    } catch {\n      // stat4 table may not exist or query may fail\n      return undefined;\n    }\n  }\n\n  /**\n   * Gets fanout from sqlite_stat1 average.\n   *\n   * Note: This includes NULL rows in the calculation and may overestimate\n   * fanout for sparse foreign keys.\n   *\n   * For compound indexes, uses the stat value at the appropriate depth.\n   *\n   * @param columns Array of column names to get fanout for\n   * @returns Fanout result or undefined if stat1 unavailable\n   */\n  #getFanoutFromStat1(\n    tableName: string,\n    columns: string[],\n  ): FanoutResult | undefined {\n    try {\n      // Find index containing the columns as a prefix\n      const indexInfo = this.#findIndexForColumns(tableName, columns);\n      if (!indexInfo) {\n        return undefined;\n      }\n\n      // Query stat1 for this index (using prepared statement)\n      const result = this.#stat1Stmt.get(tableName, indexInfo.indexName) as\n        | {stat: string}\n        | undefined;\n\n      if (!result) {\n        return undefined;\n      }\n\n      const parts = result.stat.split(' ');\n      // Check if we have enough parts for the requested depth\n      if (parts.length < indexInfo.depth + 1) {\n        return undefined;\n      }\n\n      const fanout = parseInt(parts[indexInfo.depth], 10);\n      if (isNaN(fanout)) {\n        return undefined;\n      }\n\n      return {\n        fanout,\n        source: 'stat1',\n        confidence: 'med',\n      };\n    } catch {\n      return undefined;\n    }\n  }\n\n  /**\n   * Finds an index that can be used to get statistics for column(s).\n   *\n   * Uses pragma_index_list and pragma_index_info to reliably get index\n   * column names, avoiding brittle SQL parsing. Includes all indices:\n   * user-created (CREATE INDEX), PRIMARY KEY, and UNIQUE constraints.\n   *\n   * Uses flexible matching: Finds indexes where ALL columns appear in the\n   * first N positions, regardless of order. This works because SQLite statistics\n   * at depth N represent the fanout for the combination of the first N columns,\n   * and combinations are order-independent.\n   *\n   * Example:\n   * - columns: ['customerId', 'storeId']\n   * - Matches: (customerId, storeId, date) at depth 2 ✅\n   * - Matches: (storeId, customerId, date) at depth 2 ✅ (flexible order)\n   * - Does NOT match: (date, customerId, storeId) ❌ (columns not in first 2 positions)\n   * - Does NOT match: (customerId, date, storeId) ❌ (storeId not in first 2 positions)\n   *\n   * @param columns Array of column names (order-independent for matching)\n   * @returns Index info with name and depth, or undefined if no match\n   */\n  #findIndexForColumns(\n    tableName: string,\n    columns: string[],\n  ): {indexName: string; depth: number} | undefined {\n    try {\n      // Query returns all columns for all indexes (including PK/UNIQUE) in order\n      const rows = this.#indexStmt.all(tableName) as {\n        index_name: string;\n        seqno: number;\n        column_name: string;\n      }[];\n\n      // Group by index name\n      const indexMap = new Map<string, string[]>();\n      for (const row of rows) {\n        getOrInsertComputed(indexMap, row.index_name, newArray).push(\n          row.column_name,\n        );\n      }\n\n      // Check each index for prefix match\n      for (const [indexName, indexColumns] of indexMap) {\n        if (this.#isPrefixMatch(columns, indexColumns)) {\n          return {\n            indexName,\n            depth: columns.length,\n          };\n        }\n      }\n\n      return undefined;\n    } catch {\n      return undefined;\n    }\n  }\n\n  /**\n   * Checks if all queryColumns exist in the first N positions of indexColumns,\n   * regardless of order.\n   *\n   * This allows flexible matching: constraint {a, b} matches both index (a, b, c)\n   * and index (b, a, c) at depth 2, since both represent the fanout for the\n   * combination of columns a and b.\n   *\n   * Gaps are NOT allowed: constraint {a, c} does NOT match index (a, b, c)\n   * because no depth represents just (a, c) without b. Statistics are cumulative\n   * from position 0.\n   *\n   * @param queryColumns Columns we're looking for (from constraint)\n   * @param indexColumns Columns in the index (in order)\n   * @returns true if all queryColumns exist in indexColumns[0...queryColumns.length-1]\n   */\n  #isPrefixMatch(queryColumns: string[], indexColumns: string[]): boolean {\n    if (queryColumns.length > indexColumns.length) {\n      return false;\n    }\n\n    // Get the prefix of the index that we're checking against\n    const indexPrefix = indexColumns.slice(0, queryColumns.length);\n\n    // Normalize to lowercase for case-insensitive comparison\n    const indexPrefixLower = new Set(indexPrefix.map(col => col.toLowerCase()));\n    const queryColumnsLower = queryColumns.map(col => col.toLowerCase());\n\n    // Check if ALL query columns exist in the index prefix\n    return queryColumnsLower.every(queryCol => indexPrefixLower.has(queryCol));\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AA2FA,IAAa,mBAAb,MAA8B;CAC5B;CACA;;;;;CAMA,yBAAkB,IAAI,IAA0B;;;;;CAMhD;CACA;CACA;;;;;;;;;;CAWA,YAAY,IAAc,gBAAgB,GAAG;EAC3C,KAAK,MAAM;EACX,KAAK,iBAAiB;EAGtB,KAAK,aAAa,KAAK,IAAI,QAAQ;;;;;KAKlC;EAED,KAAK,aAAa,KAAK,IAAI,QAAQ;;;;KAIlC;EAED,KAAK,aAAa,KAAK,IAAI,QAAQ;;;;;;KAMlC;CACH;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAgCA,UAAU,WAAmB,SAAiC;EAE5D,MAAM,WAAW,GAAG,UAAU,GAAG,QAAQ,SAAS,CAAC,CAAC,KAAK,GAAG;EAC5D,MAAM,SAAS,KAAK,OAAO,IAAI,QAAQ;EACvC,IAAI,QACF,OAAO;EAMT,MAAM,cAAc,KAAK,oBAAoB,WAAW,OAAO;EAC/D,IAAI,aAAa;GACf,KAAK,OAAO,IAAI,UAAU,WAAW;GACrC,OAAO;EACT;EAGA,MAAM,cAAc,KAAK,oBAAoB,WAAW,OAAO;EAC/D,IAAI,aAAa;GACf,KAAK,OAAO,IAAI,UAAU,WAAW;GACrC,OAAO;EACT;EAGA,MAAM,gBAA8B;GAClC,QAAQ,KAAK;GACb,YAAY;GACZ,QAAQ;EACV;EACA,KAAK,OAAO,IAAI,UAAU,aAAa;EACvC,OAAO;CACT;;;;;CAMA,aAAmB;EACjB,KAAK,OAAO,MAAM;CACpB;;;;;;;;;;;;CAaA,oBACE,WACA,SAC0B;EAC1B,IAAI;GAEF,MAAM,YAAY,KAAK,qBAAqB,WAAW,OAAO;GAC9D,IAAI,CAAC,WACH;GAIF,MAAM,UAAU,KAAK,WAAW,IAC9B,WACA,UAAU,SACZ;GAEA,IAAI,QAAQ,WAAW,GACrB;GAKF,MAAM,WAAW,UAAU,QAAQ;GASnC,MAAM,iBARiB,QAAQ,KAAI,MAAK;IACtC,MAAM,WAAW,EAAE,IAAI,MAAM,GAAG;IAChC,OAAO;KACL,QAAQ,SAAS,SAAS,aAAa,SAAS,IAAI,EAAE;KACtD,QAAQ,aAAa,EAAE,MAAM;IAC/B;GACF,CAEuB,CAAA,CAAe,QAAO,MAAK,CAAC,EAAE,MAAM;GAE3D,IAAI,eAAe,WAAW,GAE5B,OAAO;IACL,QAAQ;IACR,QAAQ;IACR,YAAY;GACd;GAIF,MAAM,UAAU,eAAe,KAAI,MAAK,EAAE,MAAM,CAAC,CAAC,MAAM,GAAG,MAAM,IAAI,CAAC;GAStE,OAAO;IACL,QARA,QAAQ,SAAS,MAAM,IACnB,KAAK,OACF,QAAQ,QAAQ,SAAS,IAAI,KAAK,QAAQ,QAAQ,SAAS,MAC1D,CACJ,IACA,QAAQ,KAAK,MAAM,QAAQ,SAAS,CAAC;IAIzC,QAAQ;IACR,YAAY;GACd;EACF,QAAQ;GAEN;EACF;CACF;;;;;;;;;;;;CAaA,oBACE,WACA,SAC0B;EAC1B,IAAI;GAEF,MAAM,YAAY,KAAK,qBAAqB,WAAW,OAAO;GAC9D,IAAI,CAAC,WACH;GAIF,MAAM,SAAS,KAAK,WAAW,IAAI,WAAW,UAAU,SAAS;GAIjE,IAAI,CAAC,QACH;GAGF,MAAM,QAAQ,OAAO,KAAK,MAAM,GAAG;GAEnC,IAAI,MAAM,SAAS,UAAU,QAAQ,GACnC;GAGF,MAAM,SAAS,SAAS,MAAM,UAAU,QAAQ,EAAE;GAClD,IAAI,MAAM,MAAM,GACd;GAGF,OAAO;IACL;IACA,QAAQ;IACR,YAAY;GACd;EACF,QAAQ;GACN;EACF;CACF;;;;;;;;;;;;;;;;;;;;;;;CAwBA,qBACE,WACA,SACgD;EAChD,IAAI;GAEF,MAAM,OAAO,KAAK,WAAW,IAAI,SAAS;GAO1C,MAAM,2BAAW,IAAI,IAAsB;GAC3C,KAAK,MAAM,OAAO,MAChB,oBAAoB,UAAU,IAAI,YAAY,QAAQ,CAAC,CAAC,KACtD,IAAI,WACN;GAIF,KAAK,MAAM,CAAC,WAAW,iBAAiB,UACtC,IAAI,KAAK,eAAe,SAAS,YAAY,GAC3C,OAAO;IACL;IACA,OAAO,QAAQ;GACjB;GAIJ;EACF,QAAQ;GACN;EACF;CACF;;;;;;;;;;;;;;;;;CAkBA,eAAe,cAAwB,cAAiC;EACtE,IAAI,aAAa,SAAS,aAAa,QACrC,OAAO;EAIT,MAAM,cAAc,aAAa,MAAM,GAAG,aAAa,MAAM;EAG7D,MAAM,mBAAmB,IAAI,IAAI,YAAY,KAAI,QAAO,IAAI,YAAY,CAAC,CAAC;EAI1E,OAH0B,aAAa,KAAI,QAAO,IAAI,YAAY,CAG3D,CAAA,CAAkB,OAAM,aAAY,iBAAiB,IAAI,QAAQ,CAAC;CAC3E;AACF"}

package/out/zqlite/src/table-source.d.ts

@@ -129,7 +129,7 @@ export declare class TableSource implements Source {
      * Returns `base` untouched when derivation is not deferred, which is what
      * makes this a no-op for the write-through path.
      */
-    reconcilePendingConflicts(base: readonly Row[], next: Row | null, rowKey: Row, uniqueKeys: readonly (readonly string[])[]): readonly Row[];
+    reconcilePendingConflicts(base: readonly Row[], next: Row | null, rowKey: SQLiteRow, uniqueKeys: readonly (readonly string[])[]): readonly Row[];
     /**
      * Retrieves a row from the backing DB by a unique key, or `undefined` if such a
      * row does not exist. This is not used in the IVM pipeline but is useful
@@ -139,9 +139,26 @@ export declare class TableSource implements Source {
      */
     getRow(rowKey: Row): Row | undefined;
 }
-export declare function toSQLiteTypes(columns: readonly string[], row: Row, columnTypes: Record<string, SchemaValue>): readonly unknown[];
 export declare function toSQLiteTypeName(type: ValueType): "INTEGER" | "NULL" | "REAL" | "TEXT";
-export declare function fromSQLiteTypes(valueTypes: Record<string, SchemaValue>, row: Row, tableName: string): Row;
+/**
+ * A column value as read from SQLite with `safeIntegers(true)`, i.e. before
+ * {@link fromSQLiteTypes} converts it to a {@link Value}.
+ */
+export type SQLiteValue = bigint | number | string | null;
+/** A row as read from SQLite, before {@link fromSQLiteTypes}. */
+export type SQLiteRow = Readonly<Record<string, SQLiteValue>>;
+export declare function fromSQLiteTypes(valueTypes: Record<string, SchemaValue>, row: SQLiteRow, tableName: string): Row;
+/** A result column of a raw row and the type it converts to. */
+export type RawColumn = {
+    readonly name: string;
+    readonly type: ValueType;
+};
+export declare function rawColumn(valueTypes: Record<string, SchemaValue>, name: string, tableName: string): RawColumn;
+/**
+ * Like {@link fromSQLiteTypes}, for a row read with `raw(true)`: the row's
+ * values are `values[offset]` up to `values[offset + columns.length - 1]`.
+ */
+export declare function fromSQLiteRawRow(columns: readonly RawColumn[], values: readonly SQLiteValue[], tableName: string, offset?: number): Row;
 export declare class UnsupportedValueError extends Error {
 }
 //# sourceMappingURL=table-source.d.ts.map

package/out/zqlite/src/table-source.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"table-source.d.ts","sourceRoot":"","sources":["../../../../zqlite/src/table-source.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAEjD,OAAO,KAAK,EAAC,SAAS,EAAC,MAAM,+BAA+B,CAAC;AAM7D,OAAO,KAAK,EAAC,SAAS,EAAE,QAAQ,EAAC,MAAM,gCAAgC,CAAC;AACxE,OAAO,KAAK,EAAC,GAAG,EAAQ,MAAM,iCAAiC,CAAC;AAChE,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,wCAAwC,CAAC;AACvE,OAAO,KAAK,EACV,WAAW,EACX,SAAS,EACV,MAAM,uCAAuC,CAAC;AAC/C,OAAO,KAAK,EAAC,aAAa,EAAC,MAAM,yCAAyC,CAAC;AAuB3E,OAAO,EACL,KAAK,MAAM,EACX,KAAK,YAAY,EACjB,KAAK,WAAW,EACjB,MAAM,6BAA6B,CAAC;AACrC,OAAO,KAAK,EAAC,MAAM,EAAC,MAAM,6BAA6B,CAAC;AAExD,OAAO,KAAK,EAAC,QAAQ,EAAY,MAAM,SAAS,CAAC;AAyBjD,MAAM,MAAM,kBAAkB,GAAG;IAC/B;;;;;;;;OAQG;IACH,uBAAuB,CAAC,EAAE,OAAO,GAAG,SAAS,CAAC;IAE9C;;;;;OAKG;IACH,WAAW,CAAC,EAAE,OAAO,GAAG,SAAS,CAAC;CACnC,CAAC;AAEF;;;;;;;;;;;;;GAaG;AACH,qBAAa,WAAY,YAAW,MAAM;;IAwBxC;;;;;OAKG;IACH,YACE,UAAU,EAAE,UAAU,EACtB,SAAS,EAAE,SAAS,EACpB,EAAE,EAAE,QAAQ,EACZ,SAAS,EAAE,MAAM,EACjB,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,WAAW,CAAC,EACpC,UAAU,EAAE,UAAU,EACtB,WAAW,gBAAc,EACzB,OAAO,GAAE,kBAAuB,EAqBjC;IAED;;;OAGG;IACH,IAAI,QAAQ,IAAI,MAAM,CAErB;IAED,IAAI,WAAW;QAEX,IAAI;QACJ,OAAO;QACP,UAAU;MAEb;IAED;;;OAGG;IACH,KAAK,CAAC,EAAE,EAAE,QAAQ,QAKjB;IAED;;;;;;OAMG;IACH,cAAc,CAAC,WAAW,EAAE,OAAO,QAUlC;IAED;;;;;OAKG;IACH,qBAAqB,IAAI,IAAI,CAE5B;IAED;;;;;;OAMG;IACF,mBAAmB,IAAI,MAAM,CAAC,OAAO,CAAC,CAwBtC;IAwED,OAAO,CACL,IAAI,EAAE,QAAQ,GAAG,SAAS,EAC1B,OAAO,CAAC,EAAE,SAAS,EACnB,aAAa,CAAC,EAAE,GAAG,CAAC,MAAM,CAAC,EAC3B,KAAK,CAAC,EAAE,aAAa,eA4CtB;IAED,WAAW,CAAC,GAAG,EAAE,GAAG,GAAG,GAAG,CAOzB;IAmMA,IAAI,CAAC,MAAM,EAAE,YAAY,GAAG,MAAM,CAAC,OAAO,CAAC,CAM3C;IAED;;;OAGG;IACH,IAAI,WAAW,IAAI,MAAM,CAExB;IAED,sEAAsE;IACtE,IAAI,YAAY,IAAI,MAAM,CAEzB;IAED;;;OAGG;IACH,YAAY,CAAC,GAAG,EAAE,GAAG,GAAG,OAAO,CAU9B;IAmBA,OAAO,CAAC,MAAM,EAAE,YAAY,GAAG,MAAM,CAAC,OAAO,GAAG,SAAS,CAAC,CA6C1D;IAuED;;;;;;;;;;;;;;;;;;;;;;;;OAwBG;IACH,yBAAyB,CACvB,IAAI,EAAE,SAAS,GAAG,EAAE,EACpB,IAAI,EAAE,GAAG,GAAG,IAAI,EAChB,MAAM,EAAE,GAAG,EACX,UAAU,EAAE,SAAS,CAAC,SAAS,MAAM,EAAE,CAAC,EAAE,GACzC,SAAS,GAAG,EAAE,CAoChB;IA6DD;;;;;;OAMG;IACH,MAAM,CAAC,MAAM,EAAE,GAAG,GAAG,GAAG,GAAG,SAAS,CAkCnC;CAmBF;AAoFD,wBAAgB,aAAa,CAC3B,OAAO,EAAE,SAAS,MAAM,EAAE,EAC1B,GAAG,EAAE,GAAG,EACR,WAAW,EAAE,MAAM,CAAC,MAAM,EAAE,WAAW,CAAC,GACvC,SAAS,OAAO,EAAE,CAEpB;AAED,wBAAgB,gBAAgB,CAAC,IAAI,EAAE,SAAS,wCAa/C;AAED,wBAAgB,eAAe,CAC7B,UAAU,EAAE,MAAM,CAAC,MAAM,EAAE,WAAW,CAAC,EACvC,GAAG,EAAE,GAAG,EACR,SAAS,EAAE,MAAM,GAChB,GAAG,CAaL;AAyCD,qBAAa,qBAAsB,SAAQ,KAAK;CAAG"}
+{"version":3,"file":"table-source.d.ts","sourceRoot":"","sources":["../../../../zqlite/src/table-source.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAEjD,OAAO,KAAK,EAAC,SAAS,EAAC,MAAM,+BAA+B,CAAC;AAM7D,OAAO,KAAK,EAAC,SAAS,EAAE,QAAQ,EAAC,MAAM,gCAAgC,CAAC;AACxE,OAAO,KAAK,EAAC,GAAG,EAAQ,MAAM,iCAAiC,CAAC;AAChE,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,wCAAwC,CAAC;AACvE,OAAO,KAAK,EACV,WAAW,EACX,SAAS,EACV,MAAM,uCAAuC,CAAC;AAC/C,OAAO,KAAK,EAAC,aAAa,EAAC,MAAM,yCAAyC,CAAC;AAwB3E,OAAO,EACL,KAAK,MAAM,EACX,KAAK,YAAY,EAEjB,KAAK,WAAW,EACjB,MAAM,6BAA6B,CAAC;AACrC,OAAO,KAAK,EAAC,MAAM,EAAC,MAAM,6BAA6B,CAAC;AAExD,OAAO,KAAK,EAAc,QAAQ,EAAY,MAAM,SAAS,CAAC;AAiC9D,MAAM,MAAM,kBAAkB,GAAG;IAC/B;;;;;;;;OAQG;IACH,uBAAuB,CAAC,EAAE,OAAO,GAAG,SAAS,CAAC;IAE9C;;;;;OAKG;IACH,WAAW,CAAC,EAAE,OAAO,GAAG,SAAS,CAAC;CACnC,CAAC;AAEF;;;;;;;;;;;;;GAaG;AACH,qBAAa,WAAY,YAAW,MAAM;;IAyBxC;;;;;OAKG;IACH,YACE,UAAU,EAAE,UAAU,EACtB,SAAS,EAAE,SAAS,EACpB,EAAE,EAAE,QAAQ,EACZ,SAAS,EAAE,MAAM,EACjB,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,WAAW,CAAC,EACpC,UAAU,EAAE,UAAU,EACtB,WAAW,gBAAc,EACzB,OAAO,GAAE,kBAAuB,EAqBjC;IAED;;;OAGG;IACH,IAAI,QAAQ,IAAI,MAAM,CAErB;IAED,IAAI,WAAW;QAEX,IAAI;QACJ,OAAO;QACP,UAAU;MAEb;IAED;;;OAGG;IACH,KAAK,CAAC,EAAE,EAAE,QAAQ,QAKjB;IAED;;;;;;OAMG;IACH,cAAc,CAAC,WAAW,EAAE,OAAO,QAUlC;IAED;;;;;OAKG;IACH,qBAAqB,IAAI,IAAI,CAE5B;IAED;;;;;;OAMG;IACF,mBAAmB,IAAI,MAAM,CAAC,OAAO,CAAC,CAwBtC;IAwED,OAAO,CACL,IAAI,EAAE,QAAQ,GAAG,SAAS,EAC1B,OAAO,CAAC,EAAE,SAAS,EACnB,aAAa,CAAC,EAAE,GAAG,CAAC,MAAM,CAAC,EAC3B,KAAK,CAAC,EAAE,aAAa,eAqDtB;IAED,WAAW,CAAC,GAAG,EAAE,GAAG,GAAG,GAAG,CAOzB;IAsRA,IAAI,CAAC,MAAM,EAAE,YAAY,GAAG,MAAM,CAAC,OAAO,CAAC,CAM3C;IAED;;;OAGG;IACH,IAAI,WAAW,IAAI,MAAM,CAExB;IAED,sEAAsE;IACtE,IAAI,YAAY,IAAI,MAAM,CAEzB;IAED;;;OAGG;IACH,YAAY,CAAC,GAAG,EAAE,GAAG,GAAG,OAAO,CAU9B;IAmBA,OAAO,CAAC,MAAM,EAAE,YAAY,GAAG,MAAM,CAAC,OAAO,GAAG,SAAS,CAAC,CA6C1D;IAuED;;;;;;;;;;;;;;;;;;;;;;;;OAwBG;IACH,yBAAyB,CACvB,IAAI,EAAE,SAAS,GAAG,EAAE,EACpB,IAAI,EAAE,GAAG,GAAG,IAAI,EAChB,MAAM,EAAE,SAAS,EACjB,UAAU,EAAE,SAAS,CAAC,SAAS,MAAM,EAAE,CAAC,EAAE,GACzC,SAAS,GAAG,EAAE,CAoChB;IA6DD;;;;;;OAMG;IACH,MAAM,CAAC,MAAM,EAAE,GAAG,GAAG,GAAG,GAAG,SAAS,CAsCnC;CACF;AA4FD,wBAAgB,gBAAgB,CAAC,IAAI,EAAE,SAAS,wCAa/C;AAED;;;GAGG;AACH,MAAM,MAAM,WAAW,GAAG,MAAM,GAAG,MAAM,GAAG,MAAM,GAAG,IAAI,CAAC;AAE1D,iEAAiE;AACjE,MAAM,MAAM,SAAS,GAAG,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,WAAW,CAAC,CAAC,CAAC;AAE9D,wBAAgB,eAAe,CAC7B,UAAU,EAAE,MAAM,CAAC,MAAM,EAAE,WAAW,CAAC,EACvC,GAAG,EAAE,SAAS,EACd,SAAS,EAAE,MAAM,GAChB,GAAG,CAcL;AAED,gEAAgE;AAChE,MAAM,MAAM,SAAS,GAAG;IACtB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,IAAI,EAAE,SAAS,CAAC;CAC1B,CAAC;AAEF,wBAAgB,SAAS,CACvB,UAAU,EAAE,MAAM,CAAC,MAAM,EAAE,WAAW,CAAC,EACvC,IAAI,EAAE,MAAM,EACZ,SAAS,EAAE,MAAM,GAChB,SAAS,CASX;AAaD;;;GAGG;AACH,wBAAgB,gBAAgB,CAC9B,OAAO,EAAE,SAAS,SAAS,EAAE,EAC7B,MAAM,EAAE,SAAS,WAAW,EAAE,EAC9B,SAAS,EAAE,MAAM,EACjB,MAAM,SAAI,GACT,GAAG,CAaL;AAwCD,qBAAa,qBAAsB,SAAQ,KAAK;CAAG"}

package/out/zqlite/src/table-source.js

@@ -1,19 +1,22 @@
 import { assert, unreachable } from "../../shared/src/asserts.js";
 import { getOrInsertComputed } from "../../shared/src/map.js";
 import { must } from "../../shared/src/must.js";
-import { assertOrderingIncludesPK } from "../../zql/src/query/complete-ordering.js";
+import { assignProperty } from "../../shared/src/objects.js";
 import { makeComparator } from "../../zql/src/ivm/data.js";
+import { assertOrderingIncludesPK } from "../../zql/src/query/complete-ordering.js";
 import { createPredicate, transformFilters } from "../../zql/src/builder/filter.js";
 import { genPushAndWriteWithSplitEdit, generateWithOverlay, generateWithOverlayUnordered, generateWithStart } from "../../zql/src/ivm/memory-source.js";
-import { timeSampled } from "../../otel/src/maybe-time.js";
-import { recordIndexUsage } from "./index-usage.js";
 import { compile, format, sql } from "./internal/sql.js";
+import { buildCountQuery, buildGroupCountsQuery, toSQLiteType } from "./query-builder.js";
+import { FetchSQLCache } from "./fetch-sql.js";
+import { recordIndexUsage } from "./index-usage.js";
 import { StatementCache } from "./internal/statement-cache.js";
 import { NOT_OVERRIDDEN, PendingDelta, generateWithPendingDelta, generateWithPendingDeltaUnordered } from "./pending-delta.js";
-import { buildSelectQueries, toSQLiteType } from "./query-builder.js";
 import SQLite3Database from "@rocicorp/zero-sqlite3";
 //#region ../zqlite/src/table-source.ts
 var eventCount = 0;
+var FIRST_BATCH_ROWS = 1;
+var MAX_BATCH_ROWS = 128;
 /**
 * A source that is backed by a SQLite table.
 *
@@ -30,6 +33,7 @@ var eventCount = 0;
 */
 var TableSource = class {
 	#dbCache = /* @__PURE__ */ new WeakMap();
+	#rawColumns = /* @__PURE__ */ new WeakMap();
 	#connections = [];
 	#table;
 	#columns;
@@ -166,9 +170,12 @@ var TableSource = class {
 		const transformedFilters = transformFilters(filters);
 		const unordered = sort === void 0;
 		const primaryKeySort = this.#primaryKeySort;
+		const fetchSQL = new FetchSQLCache(this.#table, this.#columns, transformedFilters.filters, sort);
 		const input = {
 			getSchema: () => schema,
-			fetch: (req) => this.#fetch(req, connection),
+			fetch: (req) => this.#fetch(req, connection, fetchSQL),
+			count: (constraint) => this.#count(constraint, transformedFilters.filters),
+			groupCounts: (columns) => this.#groupCounts(columns, transformedFilters.filters),
 			setOutput: (output) => {
 				connection.output = output;
 			},
@@ -200,9 +207,41 @@ var TableSource = class {
 	toSQLiteRow(row) {
 		return Object.fromEntries(Object.entries(row).map(([key, value]) => [key, toSQLiteType(value, this.#columns[key].type)]));
 	}
-	*#fetch(req, connection) {
+	/**
+	* The number of rows a fetch with `constraint` would return, counted by
+	* SQLite. `undefined` while a push is in flight: the overlay and pending
+	* writes are applied row by row, so the caller fetches instead.
+	*/
+	#count(constraint, filters) {
+		if (this.#overlay !== void 0 || this.#delta !== void 0 && !this.#delta.isEmpty) return;
+		const { text, values } = format(buildCountQuery(this.#table, this.#columns, constraint, filters));
+		return this.#stmts.cache.use(text, ({ statement }) => {
+			const count = statement.safeIntegers(false).pluck(true).get(...values);
+			recordIndexUsage(statement, this.#table);
+			return count;
+		});
+	}
+	/**
+	* The distinct values of `columns` among the rows that pass `filters`, each
+	* with its number of rows, grouped by SQLite. `undefined` while a push is in
+	* flight, as for {@link #count}.
+	*/
+	#groupCounts(columns, filters) {
+		if (this.#overlay !== void 0 || this.#delta !== void 0 && !this.#delta.isEmpty) return;
+		const rawColumns = columns.map((c) => rawColumn(this.#columns, c, this.#table));
+		const { text, values } = format(buildGroupCountsQuery(this.#table, columns, filters));
+		return this.#stmts.cache.use(text, ({ statement }) => {
+			const rows = statement.safeIntegers(true).raw(true).all(...values);
+			recordIndexUsage(statement, this.#table);
+			return rows.map((values) => ({
+				row: fromSQLiteRawRow(rawColumns, values, this.#table),
+				count: Number(values[columns.length])
+			}));
+		});
+	}
+	*#fetch(req, connection, fetchSQL) {
 		const { sort, debug } = connection;
-		const rows = this.#queryRows(this.#requestToSQL(req, connection.filters?.condition, sort), debug);
+		const rows = this.#queryRows(fetchSQL.get(req), debug);
 		const overlayPredicate = mergeOverlayPredicate(connection.filters?.predicate, req.filter);
 		try {
 			if (sort) {
@@ -214,22 +253,20 @@ var TableSource = class {
 		}
 	}
 	/**
-	* The rows of `queries`, run one after another as {@link buildSelectQueries}
+	* The rows of `queries`, run one after another as `buildSelectQueries`
 	* requires. Each statement is run only once the previous one is exhausted,
 	* so a fetch that stops early never runs the rest.
 	*/
 	*#queryRows(queries, debug) {
 		const { cache } = this.#stmts;
-		for (const query of queries) {
-			const sqlAndBindings = format(query);
-			const cachedStatement = cache.get(sqlAndBindings.text);
-			cachedStatement.statement.safeIntegers(true);
-			const rowIterator = cachedStatement.statement.iterate(...sqlAndBindings.values);
+		for (const sqlAndBindings of queries) {
+			const cachedStatement = cache.getExact(sqlAndBindings.text);
+			const cursor = cachedStatement.statement.batches(sqlAndBindings.values);
 			try {
 				debug?.initQuery(this.#table, sqlAndBindings.text);
-				yield* this.#mapFromSQLiteTypes(this.#columns, rowIterator, sqlAndBindings.text, debug);
+				yield* this.#readRows(cursor, this.#rawColumnsFor(cachedStatement.statement), sqlAndBindings.text, debug);
 			} finally {
-				rowIterator.return?.();
+				cursor.close();
 				recordIndexUsage(cachedStatement.statement, this.#table);
 				if (debug) {
 					let totalNvisit = 0;
@@ -265,16 +302,29 @@ var TableSource = class {
 		const deltaRows = delta.rowsFor(sort ?? this.#primaryKeySort, req.constraint, sort ? req.reverse : false, predicate, req.multiConstraints, sort ? req.start?.row : void 0);
 		return compare ? generateWithPendingDelta(baseRows, deltaRows, delta, compare) : generateWithPendingDeltaUnordered(baseRows, deltaRows, delta);
 	}
-	*#mapFromSQLiteTypes(valueTypes, rowIterator, query, debug) {
-		let result;
-		do {
-			result = timeSampled(this.#lc, ++eventCount, this.#logConfig.ivmSampling, () => rowIterator.next(), this.#logConfig.slowRowThreshold, () => `table-source.next took too long for ${query}. Are you missing an index?`);
-			if (result.done) break;
-			this.#rowsRead++;
-			const row = fromSQLiteTypes(valueTypes, result.value, this.#table);
-			debug?.rowVended(this.#table, query, row);
-			yield row;
-		} while (!result.done);
+	#rawColumnsFor(statement) {
+		return getOrInsertComputed(this.#rawColumns, statement, () => statement.columns().map(({ name }) => rawColumn(this.#columns, name, this.#table)));
+	}
+	*#readRows(cursor, columns, query, debug) {
+		for (let capacity = FIRST_BATCH_ROWS;;) {
+			const { ivmSampling, slowRowThreshold } = this.#logConfig;
+			const sampled = ivmSampling > 0 && ++eventCount % ivmSampling === 0;
+			const start = sampled ? performance.now() : 0;
+			const values = cursor.next(capacity);
+			const n = values.length / columns.length;
+			if (sampled) {
+				const duration = performance.now() - start;
+				if (duration > slowRowThreshold * Math.max(n, 1)) this.#lc.warn?.(`table-source.next took too long for ${query}. Are you missing an index? duration: ${duration}ms`);
+			}
+			if (n === 0) return;
+			for (let r = 0; r < n; r++) {
+				const row = fromSQLiteRawRow(columns, values, this.#table, r * columns.length);
+				this.#rowsRead++;
+				debug?.rowVended(this.#table, query, row);
+				yield row;
+			}
+			capacity = Math.min(capacity * 2, MAX_BATCH_ROWS);
+		}
 	}
 	*push(change) {
 		for (const result of this.genPush(change)) if (result === "yield") yield result;
@@ -388,7 +438,7 @@ var TableSource = class {
 	}
 	#syncedDeleteKey(rowKey) {
 		const synced = {};
-		for (const [col, value] of Object.entries(rowKey)) if (Object.hasOwn(this.#columns, col)) synced[col] = value;
+		for (const [col, value] of Object.entries(rowKey)) if (Object.hasOwn(this.#columns, col)) assignProperty(synced, col, value);
 		return Object.entries(fromSQLiteTypes(this.#columns, synced, this.#table));
 	}
 	/**
@@ -437,12 +487,11 @@ var TableSource = class {
 	getRow(rowKey) {
 		const keyCols = Object.keys(rowKey);
 		const stmt = this.#getRowStmt(keyCols);
-		const raw = this.#stmts.cache.use(stmt, ({ statement }) => {
-			const row = statement.safeIntegers(true).get(...toSQLiteTypes(keyCols, rowKey, this.#columns));
+		const row = this.#stmts.cache.use(stmt, ({ statement }) => {
+			const values = statement.safeIntegers(true).raw(true).get(...toSQLiteTypes(keyCols, rowKey, this.#columns));
 			recordIndexUsage(statement, this.#table);
-			return row;
+			return values === void 0 ? void 0 : fromSQLiteRawRow(this.#rawColumnsFor(statement), values, this.#table);
 		});
-		const row = raw ? fromSQLiteTypes(this.#columns, raw, this.#table) : raw;
 		const delta = this.#delta;
 		if (delta === void 0 || delta.isEmpty) return row;
 		if (row !== void 0) {
@@ -452,9 +501,6 @@ var TableSource = class {
 		}
 		return delta.getByColumns(keyCols, rowKey);
 	}
-	#requestToSQL(request, filters, order) {
-		return buildSelectQueries(this.#table, this.#columns, request.constraint, filters, order, request.reverse, request.start, request.multiConstraints, request.filter);
-	}
 };
 function mergeOverlayPredicate(connPredicate, reqFilter) {
 	if (!reqFilter) return connPredicate;
@@ -504,11 +550,34 @@ function fromSQLiteTypes(valueTypes, row, tableName) {
 	const newRow = {};
 	for (const key of Object.keys(row)) {
 		const valueType = valueTypes[key];
-		if (valueType === void 0) {
-			const columnList = Object.keys(valueTypes).sort().join(", ");
-			throw new Error(`Invalid column "${key}" for table "${tableName}". Synced columns include ${columnList}`);
-		}
-		newRow[key] = fromSQLiteType(valueType.type, row[key], key, tableName);
+		if (valueType === void 0) throw invalidColumnError(valueTypes, key, tableName);
+		assignProperty(newRow, key, fromSQLiteType(valueType.type, row[key], key, tableName));
+	}
+	return newRow;
+}
+function rawColumn(valueTypes, name, tableName) {
+	const valueType = Object.hasOwn(valueTypes, name) ? valueTypes[name] : void 0;
+	if (valueType === void 0) throw invalidColumnError(valueTypes, name, tableName);
+	return {
+		name,
+		type: valueType.type
+	};
+}
+function invalidColumnError(valueTypes, name, tableName) {
+	const columnList = Object.keys(valueTypes).sort().join(", ");
+	return /* @__PURE__ */ new Error(`Invalid column "${name}" for table "${tableName}". Synced columns include ${columnList}`);
+}
+/**
+* Like {@link fromSQLiteTypes}, for a row read with `raw(true)`: the row's
+* values are `values[offset]` up to `values[offset + columns.length - 1]`.
+*/
+function fromSQLiteRawRow(columns, values, tableName, offset = 0) {
+	const newRow = {};
+	for (let i = 0; i < columns.length; i++) {
+		const { name, type } = columns[i];
+		const value = fromSQLiteType(type, values[offset + i], name, tableName);
+		if (name === "__proto__") assignProperty(newRow, name, value);
+		else newRow[name] = value;
 	}
 	return newRow;
 }
@@ -520,9 +589,8 @@ function fromSQLiteType(valueType, v, column, tableName) {
 		case "string":
 		case "null":
 			if (typeof v === "bigint") {
-				const bi = v;
-				if (bi > Number.MAX_SAFE_INTEGER || bi < Number.MIN_SAFE_INTEGER) throw new UnsupportedValueError(`value ${bi} (in ${tableName}.${column}) is outside of supported bounds`);
-				return Number(bi);
+				if (v > Number.MAX_SAFE_INTEGER || v < Number.MIN_SAFE_INTEGER) throw new UnsupportedValueError(`value ${v} (in ${tableName}.${column}) is outside of supported bounds`);
+				return Number(v);
 			}
 			return v;
 		case "json": try {
@@ -550,6 +618,6 @@ function* generateWithYields(stream, shouldYield) {
 	}
 }
 //#endregion
-export { TableSource, fromSQLiteTypes };
+export { TableSource, UnsupportedValueError, fromSQLiteRawRow, fromSQLiteTypes, rawColumn };
 
 //# sourceMappingURL=table-source.js.map