npm - @rocicorp/zero - Versions diffs - 1.11.0-canary.12 → 1.11.0-canary.14 - Mend

@rocicorp/zero 1.11.0-canary.12 → 1.11.0-canary.14

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (260) hide show

package/out/zqlite/src/pending-delta.js.map ADDED Viewed

@@ -0,0 +1 @@

+ {"version":3,"file":"pending-delta.js","names":["#primaryKey","#singleColumnKey","#firstKeyValues","#byKey","#indexes","#rowBytes","#indexBytes","#liveCount","#key","#track","#removeFromIndexes","#addToIndexes","#getOrCreateIndex"],"sources":["../../../../zqlite/src/pending-delta.ts"],"sourcesContent":["import {BTreeSet} from '../../shared/src/btree-set.ts';\nimport {hasOwn} from '../../shared/src/has-own.ts';\nimport type {Ordering, OrderPart} from '../../zero-protocol/src/ast.ts';\nimport type {Row, Value} from '../../zero-protocol/src/data.ts';\nimport type {PrimaryKey} from '../../zero-protocol/src/primary-key.ts';\nimport {\n constraintMatchesPrimaryKey,\n constraintMatchesRow,\n type Constraint,\n} from '../../zql/src/ivm/constraint.ts';\nimport type {Comparator} from '../../zql/src/ivm/data.ts';\nimport {\n generateRows,\n makeBoundComparator,\n maxValue,\n minValue,\n type RowBound,\n} from '../../zql/src/ivm/memory-source.ts';\nimport type {MultiConstraint} from '../../zql/src/ivm/operator.ts';\n\n/**\n * The batch overlay that lets a `TableSource` derive a transaction without\n * writing to the database it is reading from.\n *\n * A `TableSource` normally makes each change durable in its backing SQLite\n * connection as soon as it has been vended to every output, because change\n * *k+1* has to read a base that already includes changes *1..k* -- IVM's\n * joins fetch from the source mid-push. On the server that backing connection\n * is a `BEGIN CONCURRENT` snapshot that is never committed and always rolled\n * back, so those writes exist only to serve reads for the rest of the pass.\n *\n * A `PendingDelta` holds them in memory instead. The base stays pinned at the\n * state it had when the pass began and this object supplies *1..k*, merged\n * into every leaf scan. The durable copy comes from the replicator's own\n * commit, which the source picks up when it leapfrogs to the next snapshot.\n *\n * State is coalesced by primary key: whatever the batch did to a row, all that\n * matters downstream is the row's final value, or that it is gone. A key that\n * appears here overrides the base unconditionally -- an added row has no base\n * to override, an edited row's base copy is suppressed in favor of the new\n * value, and a removed row's base copy is suppressed with nothing to put in\n * its place.\n *\n * Live rows are additionally kept in lazily built sorted indexes, one per\n * `[...constraintKeys, ...sort]` shape a fetch has asked for, so that merging\n * the delta into a constrained scan costs `O(log D + matches)` rather than a\n * walk of the whole delta. This mirrors `MemorySource`'s index handling; see\n * its `#fetch` for the bound construction the scans here reproduce.\n */\nexport class PendingDelta {\n readonly #primaryKey: PrimaryKey;\n readonly #singleColumnKey: string | undefined;\n\n /**\n * For a composite primary key, the first key column's value of every key in\n * `#byKey`. {@link overrides} runs for every base row of a merged fetch, and\n * building a composite key allocates, so this rules most rows out first.\n */\n readonly #firstKeyValues = new Set<Value>();\n\n /**\n * The coalesced net state, keyed by primary key. A `undefined` value is a\n * tombstone: the batch removed the row, so the base copy is suppressed and\n * nothing replaces it. Membership alone is what suppresses a base row, so\n * this map is also the \"is this key overridden\" test.\n */\n readonly #byKey = new Map<unknown, Row | undefined>();\n\n /** Sorted views over the live (non-tombstone) rows, keyed by `JSON(sort)`. */\n readonly #indexes = new Map<string, BTreeSet<Row>>();\n\n #liveCount = 0;\n #rowBytes = 0;\n #indexBytes = 0;\n\n constructor(primaryKey: PrimaryKey) {\n this.#primaryKey = primaryKey;\n this.#singleColumnKey = primaryKey.length === 1 ? primaryKey[0] : undefined;\n }\n\n /**\n * True when the source can take its untouched fast path: no suppression\n * test per base row, no merge, no index maintenance.\n */\n get isEmpty(): boolean {\n return this.#byKey.size === 0;\n }\n\n /** The number of rows the batch has touched, tombstones included. */\n get size(): number {\n return this.#byKey.size;\n }\n\n /**\n * Estimated retained bytes, including keys, tombstones, and sorted indexes.\n * This is a budgeting heuristic, not a measurement of the JavaScript heap.\n */\n get estimatedBytes(): number {\n // Budget 32 bytes per live row per index for references and B-tree nodes.\n return (\n this.#rowBytes +\n this.#indexBytes +\n this.#liveCount * this.#indexes.size * 32\n );\n }\n\n #key(row: Row): unknown {\n const single = this.#singleColumnKey;\n if (single !== undefined) {\n return row[single];\n }\n const key = this.#primaryKey.map(k => row[k]);\n return JSON.stringify(key);\n }\n\n #track(row: Row, key: unknown): void {\n if (this.#byKey.has(key)) {\n return;\n }\n this.#rowBytes += 64 + estimateBytes(key as Value);\n if (this.#singleColumnKey === undefined) {\n this.#firstKeyValues.add(row[this.#primaryKey[0]]);\n }\n }\n\n /** Records that `row` is now present with this value. */\n set(row: Row): void {\n const key = this.#key(row);\n const existing = this.#byKey.get(key);\n this.#track(row, key);\n this.#rowBytes +=\n estimateBytes(row) -\n (existing === undefined ? 0 : estimateBytes(existing));\n if (existing !== undefined) {\n // A live row for this key is already indexed under its old value, which\n // may sort differently. Retract it before inserting the new one.\n this.#removeFromIndexes(existing);\n this.#liveCount--;\n }\n this.#byKey.set(key, row);\n this.#addToIndexes(row);\n this.#liveCount++;\n }\n\n /** Records that `row` is now absent. */\n delete(row: Row): void {\n const key = this.#key(row);\n const existing = this.#byKey.get(key);\n this.#track(row, key);\n if (existing !== undefined) {\n this.#rowBytes -= estimateBytes(existing);\n this.#removeFromIndexes(existing);\n this.#liveCount--;\n }\n this.#byKey.set(key, undefined);\n }\n\n /**\n * Whether the batch overrides this base row. Callers use it to suppress base\n * rows during a merge, so it must be answered by primary key -- the value\n * carried by the base row is exactly what the delta is replacing.\n */\n overrides(row: Row): boolean {\n if (\n this.#singleColumnKey === undefined &&\n !this.#firstKeyValues.has(row[this.#primaryKey[0]])\n ) {\n return false;\n }\n return this.#byKey.has(this.#key(row));\n }\n\n /**\n * The batch's value for a row, by primary key. Returns `NOT_OVERRIDDEN` when\n * the batch has not touched the key at all (so the caller should consult the\n * base) and `undefined` when the batch removed it.\n */\n get(row: Row): Row | undefined | typeof NOT_OVERRIDDEN {\n const key = this.#key(row);\n if (!this.#byKey.has(key)) {\n return NOT_OVERRIDDEN;\n }\n return this.#byKey.get(key);\n }\n\n /**\n * Looks a row up by an arbitrary (not necessarily primary) unique key. Only\n * reachable when the delta is non-empty, which is never the case for the one\n * caller that uses a non-primary key, so a walk is fine here.\n */\n getByColumns(keyCols: readonly string[], keyRow: Row): Row | undefined {\n for (const row of this.#byKey.values()) {\n if (row !== undefined && keyCols.every(c => row[c] === keyRow[c])) {\n return row;\n }\n }\n return undefined;\n }\n\n #addToIndexes(row: Row): void {\n for (const data of this.#indexes.values()) {\n data.add(row);\n }\n }\n\n #removeFromIndexes(row: Row): void {\n for (const data of this.#indexes.values()) {\n data.delete(row);\n }\n }\n\n #getOrCreateIndex(sort: Ordering): BTreeSet<Row> {\n const indexKey = JSON.stringify(sort);\n let data = this.#indexes.get(indexKey);\n if (data === undefined) {\n data = new BTreeSet<Row>(makeBoundComparator(sort) as Comparator);\n for (const row of this.#byKey.values()) {\n if (row !== undefined) {\n data.add(row);\n }\n }\n this.#indexes.set(indexKey, data);\n this.#indexBytes += 128 + estimateBytes(indexKey) + estimateBytes(sort);\n }\n return data;\n }\n\n /**\n * The live rows this batch contributes to a fetch, in the same order the\n * backing SQL query returns its rows, so the two streams can be merged.\n *\n * `constraint`, `multiConstraints` and `filterPredicate` are the ones SQL\n * already applied to the base rows; they are applied here so the delta rows\n * are subject to the same query. `start` is the fetch's start row: the scan\n * begins there rather than at the start of the constraint span. It is\n * inclusive, so the caller still applies an `after` basis.\n */\n *rowsFor(\n sort: Ordering,\n constraint: Constraint | undefined,\n reverse: boolean | undefined,\n filterPredicate: ((row: Row) => boolean | undefined) | undefined,\n multiConstraints: readonly MultiConstraint[] | undefined,\n start?: Row | undefined,\n ): Iterable<Row> {\n if (this.#liveCount === 0) {\n return;\n }\n\n // Lead the index with the constraint keys so a constrained fetch scans\n // only the matching span; within that span rows come out in `sort` order,\n // which is the order SQL returned the base rows in.\n const indexSort: OrderPart[] = [];\n if (constraint) {\n for (const key of Object.keys(constraint)) {\n indexSort.push([key, 'asc']);\n }\n }\n // Constraining by the whole primary key admits at most one row, so the\n // requested sort adds nothing.\n const sorted =\n !constraint || !constraintMatchesPrimaryKey(constraint, this.#primaryKey);\n if (sorted) {\n indexSort.push(...sort);\n }\n\n const data = this.#getOrCreateIndex(indexSort);\n\n let scanStart: RowBound | undefined;\n if (\n start !== undefined &&\n sorted &&\n (!constraint || constraintMatchesRow(constraint, start))\n ) {\n // `start` carries every index column, and within the constraint span it\n // agrees with the constraint, so it is the scan's bound as it stands.\n // A `start` outside the span bounds nothing the scan can use.\n scanStart = start;\n } else if (constraint) {\n // The first row matching the constraint is not simply the constraint\n // values with everything else absent: a `desc` part puts absent values\n // last. The min/max sentinels say \"the first row with these constraint\n // values\" regardless of direction. Same construction as `MemorySource`.\n scanStart = {};\n for (const [key, dir] of indexSort) {\n if (hasOwn(constraint, key)) {\n scanStart[key] = constraint[key] as Value;\n } else if (reverse) {\n scanStart[key] = dir === 'asc' ? maxValue : minValue;\n } else {\n scanStart[key] = dir === 'asc' ? minValue : maxValue;\n }\n }\n }\n\n for (const row of generateRows(data, scanStart, reverse)) {\n if (constraint && !constraintMatchesRow(constraint, row)) {\n // Rows are sorted by the constraint keys first, so matches are\n // contiguous and the first miss ends the span.\n break;\n }\n if (multiConstraints && !matchesMultiConstraints(row, multiConstraints)) {\n continue;\n }\n if (filterPredicate && !filterPredicate(row)) {\n continue;\n }\n yield row;\n }\n }\n\n /**\n * A row with the primary key of each row the batch has touched, removed\n * rows included. A live row is returned as is.\n */\n *touchedKeys(): Iterable<Row> {\n const single = this.#singleColumnKey;\n for (const [key, row] of this.#byKey) {\n if (row !== undefined) {\n yield row;\n } else if (single !== undefined) {\n yield {[single]: key as Value};\n } else {\n const values = JSON.parse(key as string) as Value[];\n yield Object.fromEntries(\n this.#primaryKey.map((k, i) => [k, values[i]]),\n );\n }\n }\n }\n\n /** The rows the batch has added or edited, and not since removed. */\n *liveRows(): Iterable<Row> {\n for (const row of this.#byKey.values()) {\n if (row !== undefined) {\n yield row;\n }\n }\n }\n\n /** Drops the batch. Called when the source moves to a snapshot that has it. */\n clear(): void {\n this.#byKey.clear();\n this.#firstKeyValues.clear();\n this.#indexes.clear();\n this.#liveCount = 0;\n this.#rowBytes = 0;\n this.#indexBytes = 0;\n }\n}\n\n// Allow for object/property storage and UTF-16 strings without allocating a\n// serialized copy. Shared values may be counted more than once intentionally.\nfunction estimateBytes(value: Value): number {\n if (typeof value === 'string') {\n return 24 + value.length * 2;\n }\n if (value === null || typeof value !== 'object') {\n return 8;\n }\n let bytes = 32;\n if (Array.isArray(value)) {\n for (const item of value) {\n bytes += 8 + estimateBytes(item);\n }\n } else {\n for (const key in value) {\n bytes += 16 + estimateBytes(key) + estimateBytes((value as Row)[key]);\n }\n }\n return bytes;\n}\n\nexport const NOT_OVERRIDDEN = Symbol('not-overridden');\n\nfunction matchesMultiConstraints(\n row: Row,\n multiConstraints: readonly MultiConstraint[],\n): boolean {\n for (const mc of multiConstraints) {\n if (mc.length === 0) {\n continue;\n }\n let any = false;\n for (const c of mc) {\n if (constraintMatchesRow(c, row)) {\n any = true;\n break;\n }\n }\n if (!any) {\n return false;\n }\n }\n return true;\n}\n\n/**\n * Merges a batch's rows into an ordered base stream and drops the base rows\n * the batch overrides.\n *\n * Both streams must already be in `compare` order. A delta row that ties with\n * a base row is the *same* row by primary key -- every fetch ordering includes\n * the primary key -- so the base copy is suppressed and the delta's value\n * takes its place at that position.\n */\nexport function* generateWithPendingDelta(\n baseRows: Iterable<Row>,\n deltaRows: Iterable<Row>,\n delta: PendingDelta,\n compare: Comparator,\n): IterableIterator<Row> {\n const deltaIter = deltaRows[Symbol.iterator]();\n let pending = deltaIter.next();\n try {\n for (const row of baseRows) {\n while (!pending.done && compare(pending.value, row) < 0) {\n yield pending.value;\n pending = deltaIter.next();\n }\n if (!delta.overrides(row)) {\n yield row;\n }\n }\n while (!pending.done) {\n yield pending.value;\n pending = deltaIter.next();\n }\n } finally {\n // Propagate early termination so the delta iterator can release the BTree\n // cursor it is walking, the same contract `mergeSortedStreams` documents.\n if (!pending.done) {\n deltaIter.return?.(undefined);\n }\n }\n}\n\n/**\n * The unordered counterpart. Nothing downstream depends on position, so the\n * surviving base rows are emitted first and the batch's rows follow.\n */\nexport function* generateWithPendingDeltaUnordered(\n baseRows: Iterable<Row>,\n deltaRows: Iterable<Row>,\n delta: PendingDelta,\n): IterableIterator<Row> {\n for (const row of baseRows) {\n if (!delta.overrides(row)) {\n yield row;\n }\n }\n yield* deltaRows;\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAiDA,IAAa,eAAb,MAA0B;CACxB;CACA;;;;;;CAOA,kCAA2B,IAAI,IAAW;;;;;;;CAQ1C,yBAAkB,IAAI,IAA8B;;CAGpD,2BAAoB,IAAI,IAA2B;CAEnD,aAAa;CACb,YAAY;CACZ,cAAc;CAEd,YAAY,YAAwB;EAClC,KAAKA,cAAc;EACnB,KAAKC,mBAAmB,WAAW,WAAW,IAAI,WAAW,KAAK,KAAA;CACpE;;;;;CAMA,IAAI,UAAmB;EACrB,OAAO,KAAKE,OAAO,SAAS;CAC9B;;CAGA,IAAI,OAAe;EACjB,OAAO,KAAKA,OAAO;CACrB;;;;;CAMA,IAAI,iBAAyB;EAE3B,OACE,KAAKE,YACL,KAAKC,cACL,KAAKC,aAAa,KAAKH,SAAS,OAAO;CAE3C;CAEA,KAAK,KAAmB;EACtB,MAAM,SAAS,KAAKH;EACpB,IAAI,WAAW,KAAA,GACb,OAAO,IAAI;EAEb,MAAM,MAAM,KAAKD,YAAY,KAAI,MAAK,IAAI,EAAE;EAC5C,OAAO,KAAK,UAAU,GAAG;CAC3B;CAEA,OAAO,KAAU,KAAoB;EACnC,IAAI,KAAKG,OAAO,IAAI,GAAG,GACrB;EAEF,KAAKE,aAAa,KAAK,cAAc,GAAY;EACjD,IAAI,KAAKJ,qBAAqB,KAAA,GAC5B,KAAKC,gBAAgB,IAAI,IAAI,KAAKF,YAAY,GAAG;CAErD;;CAGA,IAAI,KAAgB;EAClB,MAAM,MAAM,KAAKQ,KAAK,GAAG;EACzB,MAAM,WAAW,KAAKL,OAAO,IAAI,GAAG;EACpC,KAAKM,OAAO,KAAK,GAAG;EACpB,KAAKJ,aACH,cAAc,GAAG,KAChB,aAAa,KAAA,IAAY,IAAI,cAAc,QAAQ;EACtD,IAAI,aAAa,KAAA,GAAW;GAG1B,KAAKK,mBAAmB,QAAQ;GAChC,KAAKH;EACP;EACA,KAAKJ,OAAO,IAAI,KAAK,GAAG;EACxB,KAAKQ,cAAc,GAAG;EACtB,KAAKJ;CACP;;CAGA,OAAO,KAAgB;EACrB,MAAM,MAAM,KAAKC,KAAK,GAAG;EACzB,MAAM,WAAW,KAAKL,OAAO,IAAI,GAAG;EACpC,KAAKM,OAAO,KAAK,GAAG;EACpB,IAAI,aAAa,KAAA,GAAW;GAC1B,KAAKJ,aAAa,cAAc,QAAQ;GACxC,KAAKK,mBAAmB,QAAQ;GAChC,KAAKH;EACP;EACA,KAAKJ,OAAO,IAAI,KAAK,KAAA,CAAS;CAChC;;;;;;CAOA,UAAU,KAAmB;EAC3B,IACE,KAAKF,qBAAqB,KAAA,KAC1B,CAAC,KAAKC,gBAAgB,IAAI,IAAI,KAAKF,YAAY,GAAG,GAElD,OAAO;EAET,OAAO,KAAKG,OAAO,IAAI,KAAKK,KAAK,GAAG,CAAC;CACvC;;;;;;CAOA,IAAI,KAAmD;EACrD,MAAM,MAAM,KAAKA,KAAK,GAAG;EACzB,IAAI,CAAC,KAAKL,OAAO,IAAI,GAAG,GACtB,OAAO;EAET,OAAO,KAAKA,OAAO,IAAI,GAAG;CAC5B;;;;;;CAOA,aAAa,SAA4B,QAA8B;EACrE,KAAK,MAAM,OAAO,KAAKA,OAAO,OAAO,GACnC,IAAI,QAAQ,KAAA,KAAa,QAAQ,OAAM,MAAK,IAAI,OAAO,OAAO,EAAE,GAC9D,OAAO;CAIb;CAEA,cAAc,KAAgB;EAC5B,KAAK,MAAM,QAAQ,KAAKC,SAAS,OAAO,GACtC,KAAK,IAAI,GAAG;CAEhB;CAEA,mBAAmB,KAAgB;EACjC,KAAK,MAAM,QAAQ,KAAKA,SAAS,OAAO,GACtC,KAAK,OAAO,GAAG;CAEnB;CAEA,kBAAkB,MAA+B;EAC/C,MAAM,WAAW,KAAK,UAAU,IAAI;EACpC,IAAI,OAAO,KAAKA,SAAS,IAAI,QAAQ;EACrC,IAAI,SAAS,KAAA,GAAW;GACtB,OAAO,IAAI,SAAc,oBAAoB,IAAI,CAAe;GAChE,KAAK,MAAM,OAAO,KAAKD,OAAO,OAAO,GACnC,IAAI,QAAQ,KAAA,GACV,KAAK,IAAI,GAAG;GAGhB,KAAKC,SAAS,IAAI,UAAU,IAAI;GAChC,KAAKE,eAAe,MAAM,cAAc,QAAQ,IAAI,cAAc,IAAI;EACxE;EACA,OAAO;CACT;;;;;;;;;;;CAYA,CAAC,QACC,MACA,YACA,SACA,iBACA,kBACA,OACe;EACf,IAAI,KAAKC,eAAe,GACtB;EAMF,MAAM,YAAyB,CAAC;EAChC,IAAI,YACF,KAAK,MAAM,OAAO,OAAO,KAAK,UAAU,GACtC,UAAU,KAAK,CAAC,KAAK,KAAK,CAAC;EAK/B,MAAM,SACJ,CAAC,cAAc,CAAC,4BAA4B,YAAY,KAAKP,WAAW;EAC1E,IAAI,QACF,UAAU,KAAK,GAAG,IAAI;EAGxB,MAAM,OAAO,KAAKY,kBAAkB,SAAS;EAE7C,IAAI;EACJ,IACE,UAAU,KAAA,KACV,WACC,CAAC,cAAc,qBAAqB,YAAY,KAAK,IAKtD,YAAY;OACP,IAAI,YAAY;GAKrB,YAAY,CAAC;GACb,KAAK,MAAM,CAAC,KAAK,QAAQ,WACvB,IAAI,OAAO,YAAY,GAAG,GACxB,UAAU,OAAO,WAAW;QACvB,IAAI,SACT,UAAU,OAAO,QAAQ,QAAQ,WAAW;QAE5C,UAAU,OAAO,QAAQ,QAAQ,WAAW;EAGlD;EAEA,KAAK,MAAM,OAAO,aAAa,MAAM,WAAW,OAAO,GAAG;GACxD,IAAI,cAAc,CAAC,qBAAqB,YAAY,GAAG,GAGrD;GAEF,IAAI,oBAAoB,CAAC,wBAAwB,KAAK,gBAAgB,GACpE;GAEF,IAAI,mBAAmB,CAAC,gBAAgB,GAAG,GACzC;GAEF,MAAM;EACR;CACF;;;;;CAMA,CAAC,cAA6B;EAC5B,MAAM,SAAS,KAAKX;EACpB,KAAK,MAAM,CAAC,KAAK,QAAQ,KAAKE,QAC5B,IAAI,QAAQ,KAAA,GACV,MAAM;OACD,IAAI,WAAW,KAAA,GACpB,MAAM,GAAE,SAAS,IAAY;OACxB;GACL,MAAM,SAAS,KAAK,MAAM,GAAa;GACvC,MAAM,OAAO,YACX,KAAKH,YAAY,KAAK,GAAG,MAAM,CAAC,GAAG,OAAO,EAAE,CAAC,CAC/C;EACF;CAEJ;;CAGA,CAAC,WAA0B;EACzB,KAAK,MAAM,OAAO,KAAKG,OAAO,OAAO,GACnC,IAAI,QAAQ,KAAA,GACV,MAAM;CAGZ;;CAGA,QAAc;EACZ,KAAKA,OAAO,MAAM;EAClB,KAAKD,gBAAgB,MAAM;EAC3B,KAAKE,SAAS,MAAM;EACpB,KAAKG,aAAa;EAClB,KAAKF,YAAY;EACjB,KAAKC,cAAc;CACrB;AACF;AAIA,SAAS,cAAc,OAAsB;CAC3C,IAAI,OAAO,UAAU,UACnB,OAAO,KAAK,MAAM,SAAS;CAE7B,IAAI,UAAU,QAAQ,OAAO,UAAU,UACrC,OAAO;CAET,IAAI,QAAQ;CACZ,IAAI,MAAM,QAAQ,KAAK,GACrB,KAAK,MAAM,QAAQ,OACjB,SAAS,IAAI,cAAc,IAAI;MAGjC,KAAK,MAAM,OAAO,OAChB,SAAS,KAAK,cAAc,GAAG,IAAI,cAAe,MAAc,IAAI;CAGxE,OAAO;AACT;AAEA,IAAa,iBAAiB,OAAO,gBAAgB;AAErD,SAAS,wBACP,KACA,kBACS;CACT,KAAK,MAAM,MAAM,kBAAkB;EACjC,IAAI,GAAG,WAAW,GAChB;EAEF,IAAI,MAAM;EACV,KAAK,MAAM,KAAK,IACd,IAAI,qBAAqB,GAAG,GAAG,GAAG;GAChC,MAAM;GACN;EACF;EAEF,IAAI,CAAC,KACH,OAAO;CAEX;CACA,OAAO;AACT;;;;;;;;;;AAWA,UAAiB,yBACf,UACA,WACA,OACA,SACuB;CACvB,MAAM,YAAY,UAAU,OAAO,UAAU;CAC7C,IAAI,UAAU,UAAU,KAAK;CAC7B,IAAI;EACF,KAAK,MAAM,OAAO,UAAU;GAC1B,OAAO,CAAC,QAAQ,QAAQ,QAAQ,QAAQ,OAAO,GAAG,IAAI,GAAG;IACvD,MAAM,QAAQ;IACd,UAAU,UAAU,KAAK;GAC3B;GACA,IAAI,CAAC,MAAM,UAAU,GAAG,GACtB,MAAM;EAEV;EACA,OAAO,CAAC,QAAQ,MAAM;GACpB,MAAM,QAAQ;GACd,UAAU,UAAU,KAAK;EAC3B;CACF,UAAU;EAGR,IAAI,CAAC,QAAQ,MACX,UAAU,SAAS,KAAA,CAAS;CAEhC;AACF;;;;;AAMA,UAAiB,kCACf,UACA,WACA,OACuB;CACvB,KAAK,MAAM,OAAO,UAChB,IAAI,CAAC,MAAM,UAAU,GAAG,GACtB,MAAM;CAGV,OAAO;AACT"}

package/out/zqlite/src/query-delegate.d.ts.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"query-delegate.d.ts","sourceRoot":"","sources":["../../../../zqlite/src/query-delegate.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AACjD,OAAO,KAAK,EAAC,SAAS,EAAC,MAAM,+BAA+B,CAAC;AAE7D,OAAO,KAAK,EAAC,MAAM,EAAC,MAAM,gCAAgC,CAAC;AAC3D,OAAO,KAAK,EAAC,MAAM,EAAC,MAAM,6BAA6B,CAAC;AACxD,OAAO,EAAC,iBAAiB,EAAC,MAAM,4CAA4C,CAAC;AAC7E,OAAO,KAAK,EAAC,cAAc,EAAC,MAAM,uCAAuC,CAAC;AAC1E,OAAO,KAAK,EAAC,QAAQ,EAAC,MAAM,SAAS,CAAC;AAGtC,qBAAa,iBAAkB,SAAQ,iBAAiB;;IAMtD,QAAQ,CAAC,oBAAoB,QAAQ;IAGrC,YACE,EAAE,EAAE,UAAU,EACd,EAAE,EAAE,QAAQ,EACZ,MAAM,EAAE,MAAM,EACd,SAAS,CAAC,EAAE,SAAS,~~EAatB~~;IAED,SAAS,CAAC,SAAS,EAAE,MAAM,GAAG,MAAM,CAYnC;IAED,mBAAmB,CAAC,EAAE,EAAE,cAAc,cAKrC;IACQ,gBAAgB,CAAC,CAAC,EAAE,gBAAgB,EAAE,MAAM,CAAC,GAAG,CAAC,CAMzD;CACF"}
1	+ {"version":3,"file":"query-delegate.d.ts","sourceRoot":"","sources":["../../../../zqlite/src/query-delegate.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AACjD,OAAO,KAAK,EAAC,SAAS,EAAC,MAAM,+BAA+B,CAAC;AAE7D,OAAO,KAAK,EAAC,MAAM,EAAC,MAAM,gCAAgC,CAAC;AAC3D,OAAO,KAAK,EAAC,MAAM,EAAC,MAAM,6BAA6B,CAAC;AACxD,OAAO,EAAC,iBAAiB,EAAC,MAAM,4CAA4C,CAAC;AAC7E,OAAO,KAAK,EAAC,cAAc,EAAC,MAAM,uCAAuC,CAAC;AAC1E,OAAO,KAAK,EAAC,QAAQ,EAAC,MAAM,SAAS,CAAC;AAGtC,qBAAa,iBAAkB,SAAQ,iBAAiB;;IAMtD,QAAQ,CAAC,oBAAoB,QAAQ;IAGrC,YACE,EAAE,EAAE,UAAU,EACd,EAAE,EAAE,QAAQ,EACZ,MAAM,EAAE,MAAM,EACd,SAAS,CAAC,EAAE,SAAS,EAiBtB;IAED,SAAS,CAAC,SAAS,EAAE,MAAM,GAAG,MAAM,CAYnC;IAED,mBAAmB,CAAC,EAAE,EAAE,cAAc,cAKrC;IACQ,gBAAgB,CAAC,CAAC,EAAE,gBAAgB,EAAE,MAAM,CAAC,GAAG,CAAC,CAMzD;CACF"}

package/out/zqlite/src/query-delegate.js CHANGED Viewed

@@ -20,7 +20,11 @@ var QueryDelegateImpl = class extends QueryDelegateBase {
 			ivmSampling: 0,
 			level: "info",
 			slowHydrateThreshold: 0,
-			slowRowThreshold: 0
+			slowAdvanceThreshold: 0,
+			queryStatsIntervalSeconds: 0,
+			slowRowThreshold: 0,
+			planWarningRowThreshold: 0,
+			planWarningCostThreshold: 0
 		};
 	}
 	getSource(tableName) {

package/out/zqlite/src/query-delegate.js.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"query-delegate.js","names":["#lc","#db","#schema","#sources","#logConfig","#commitObservers"],"sources":["../../../../zqlite/src/query-delegate.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport type {LogConfig} from '../../otel/src/log-options.ts';\nimport {getOrInsertComputed} from '../../shared/src/map.ts';\nimport type {Schema} from '../../zero-types/src/schema.ts';\nimport type {Source} from '../../zql/src/ivm/source.ts';\nimport {QueryDelegateBase} from '../../zql/src/query/query-delegate-base.ts';\nimport type {CommitListener} from '../../zql/src/query/query-delegate.ts';\nimport type {Database} from './db.ts';\nimport {TableSource} from './table-source.ts';\n\nexport class QueryDelegateImpl extends QueryDelegateBase {\n readonly #lc: LogContext;\n readonly #db: Database;\n readonly #schema: Schema;\n readonly #sources: Map<string, Source> = new Map();\n readonly #logConfig: LogConfig;\n readonly defaultQueryComplete = true;\n readonly #commitObservers = new Set<() => void>();\n\n constructor(\n lc: LogContext,\n db: Database,\n schema: Schema,\n logConfig?: LogConfig,\n ) {\n super();\n this.#lc = lc.withContext('class', 'QueryDelegateImpl');\n this.#db = db;\n this.#schema = schema;\n this.#logConfig = logConfig ?? {\n format: 'text',\n ivmSampling: 0,\n level: 'info',\n slowHydrateThreshold: 0,\n slowRowThreshold: 0,\n };\n }\n\n getSource(tableName: string): Source {\n return getOrInsertComputed(this.#sources, tableName, tableName => {\n const tableSchema = this.#schema.tables[tableName];\n return new TableSource(\n this.#lc,\n this.#logConfig,\n this.#db,\n tableName,\n tableSchema.columns,\n tableSchema.primaryKey,\n );\n });\n }\n\n onTransactionCommit(cb: CommitListener) {\n this.#commitObservers.add(cb);\n return () => {\n this.#commitObservers.delete(cb);\n };\n }\n override batchViewUpdates<T>(applyViewUpdates: () => T): T {\n const ret = applyViewUpdates();\n for (const observer of this.#commitObservers) {\n observer();\n }\n return ret;\n }\n}\n"],"mappings":";;;;AAUA,IAAa,oBAAb,cAAuC,kBAAkB;CACvD;CACA;CACA;CACA,2BAAyC,IAAI,IAAI;CACjD;CACA,uBAAgC;CAChC,mCAA4B,IAAI,IAAgB;CAEhD,YACE,IACA,IACA,QACA,WACA;EACA,MAAM;EACN,KAAKA,MAAM,GAAG,YAAY,SAAS,mBAAmB;EACtD,KAAKC,MAAM;EACX,KAAKC,UAAU;EACf,KAAKE,aAAa,aAAa;GAC7B,QAAQ;GACR,aAAa;GACb,OAAO;GACP,sBAAsB;GACtB,kBAAkB;~~EACpB~~;CACF;CAEA,UAAU,WAA2B;EACnC,OAAO,oBAAoB,KAAKD,UAAU,YAAW,cAAa;GAChE,MAAM,cAAc,KAAKD,QAAQ,OAAO;GACxC,OAAO,IAAI,YACT,KAAKF,KACL,KAAKI,YACL,KAAKH,KACL,WACA,YAAY,SACZ,YAAY,UACd;EACF,CAAC;CACH;CAEA,oBAAoB,IAAoB;EACtC,KAAKI,iBAAiB,IAAI,EAAE;EAC5B,aAAa;GACX,KAAKA,iBAAiB,OAAO,EAAE;EACjC;CACF;CACA,iBAA6B,kBAA8B;EACzD,MAAM,MAAM,iBAAiB;EAC7B,KAAK,MAAM,YAAY,KAAKA,kBAC1B,SAAS;EAEX,OAAO;CACT;AACF"}
1	+ {"version":3,"file":"query-delegate.js","names":["#lc","#db","#schema","#sources","#logConfig","#commitObservers"],"sources":["../../../../zqlite/src/query-delegate.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport type {LogConfig} from '../../otel/src/log-options.ts';\nimport {getOrInsertComputed} from '../../shared/src/map.ts';\nimport type {Schema} from '../../zero-types/src/schema.ts';\nimport type {Source} from '../../zql/src/ivm/source.ts';\nimport {QueryDelegateBase} from '../../zql/src/query/query-delegate-base.ts';\nimport type {CommitListener} from '../../zql/src/query/query-delegate.ts';\nimport type {Database} from './db.ts';\nimport {TableSource} from './table-source.ts';\n\nexport class QueryDelegateImpl extends QueryDelegateBase {\n readonly #lc: LogContext;\n readonly #db: Database;\n readonly #schema: Schema;\n readonly #sources: Map<string, Source> = new Map();\n readonly #logConfig: LogConfig;\n readonly defaultQueryComplete = true;\n readonly #commitObservers = new Set<() => void>();\n\n constructor(\n lc: LogContext,\n db: Database,\n schema: Schema,\n logConfig?: LogConfig,\n ) {\n super();\n this.#lc = lc.withContext('class', 'QueryDelegateImpl');\n this.#db = db;\n this.#schema = schema;\n this.#logConfig = logConfig ?? {\n format: 'text',\n ivmSampling: 0,\n level: 'info',\n slowHydrateThreshold: 0,\n slowAdvanceThreshold: 0,\n queryStatsIntervalSeconds: 0,\n slowRowThreshold: 0,\n planWarningRowThreshold: 0,\n planWarningCostThreshold: 0,\n };\n }\n\n getSource(tableName: string): Source {\n return getOrInsertComputed(this.#sources, tableName, tableName => {\n const tableSchema = this.#schema.tables[tableName];\n return new TableSource(\n this.#lc,\n this.#logConfig,\n this.#db,\n tableName,\n tableSchema.columns,\n tableSchema.primaryKey,\n );\n });\n }\n\n onTransactionCommit(cb: CommitListener) {\n this.#commitObservers.add(cb);\n return () => {\n this.#commitObservers.delete(cb);\n };\n }\n override batchViewUpdates<T>(applyViewUpdates: () => T): T {\n const ret = applyViewUpdates();\n for (const observer of this.#commitObservers) {\n observer();\n }\n return ret;\n }\n}\n"],"mappings":";;;;AAUA,IAAa,oBAAb,cAAuC,kBAAkB;CACvD;CACA;CACA;CACA,2BAAyC,IAAI,IAAI;CACjD;CACA,uBAAgC;CAChC,mCAA4B,IAAI,IAAgB;CAEhD,YACE,IACA,IACA,QACA,WACA;EACA,MAAM;EACN,KAAKA,MAAM,GAAG,YAAY,SAAS,mBAAmB;EACtD,KAAKC,MAAM;EACX,KAAKC,UAAU;EACf,KAAKE,aAAa,aAAa;GAC7B,QAAQ;GACR,aAAa;GACb,OAAO;GACP,sBAAsB;GACtB,sBAAsB;GACtB,2BAA2B;GAC3B,kBAAkB;GAClB,yBAAyB;GACzB,0BAA0B;EAC5B;CACF;CAEA,UAAU,WAA2B;EACnC,OAAO,oBAAoB,KAAKD,UAAU,YAAW,cAAa;GAChE,MAAM,cAAc,KAAKD,QAAQ,OAAO;GACxC,OAAO,IAAI,YACT,KAAKF,KACL,KAAKI,YACL,KAAKH,KACL,WACA,YAAY,SACZ,YAAY,UACd;EACF,CAAC;CACH;CAEA,oBAAoB,IAAoB;EACtC,KAAKI,iBAAiB,IAAI,EAAE;EAC5B,aAAa;GACX,KAAKA,iBAAiB,OAAO,EAAE;EACjC;CACF;CACA,iBAA6B,kBAA8B;EACzD,MAAM,MAAM,iBAAiB;EAC7B,KAAK,MAAM,YAAY,KAAKA,kBAC1B,SAAS;EAEX,OAAO;CACT;AACF"}

package/out/zqlite/src/sqlite-cost-model.d.ts.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"sqlite-cost-model.d.ts","sourceRoot":"","sources":["../../../../zqlite/src/sqlite-cost-model.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAC,WAAW,EAAC,MAAM,sCAAsC,CAAC;AAEtE,OAAO,KAAK,~~EACV~~,mBAAmB,EAEpB,MAAM,6CAA6C,CAAC;AAErD,OAAO,KAAK,EAAC,QAAQ,EAAY,MAAM,SAAS,CAAC;AAmBjD;;;;;;;;GAQG;AACH,wBAAgB,qBAAqB,CACnC,EAAE,EAAE,QAAQ,EACZ,UAAU,EAAE,GAAG,CAAC,MAAM,EAAE;IAAC,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,WAAW,CAAC,CAAA;CAAC,CAAC,GAC9D,mBAAmB,~~CA+DrB~~;~~AAmFD~~,wBAAgB,SAAS,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CAK9C"}
1	+ {"version":3,"file":"sqlite-cost-model.d.ts","sourceRoot":"","sources":["../../../../zqlite/src/sqlite-cost-model.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAC,WAAW,EAAC,MAAM,sCAAsC,CAAC;AAEtE,OAAO,KAAK,EAEV,mBAAmB,EAEpB,MAAM,6CAA6C,CAAC;AAErD,OAAO,KAAK,EAAC,QAAQ,EAAY,MAAM,SAAS,CAAC;AAmBjD;;;;;;;;GAQG;AACH,wBAAgB,qBAAqB,CACnC,EAAE,EAAE,QAAQ,EACZ,UAAU,EAAE,GAAG,CAAC,MAAM,EAAE;IAAC,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,WAAW,CAAC,CAAA;CAAC,CAAC,GAC9D,mBAAmB,CAgErB;AAqJD,wBAAgB,SAAS,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CAK9C"}

package/out/zqlite/src/sqlite-cost-model.js CHANGED Viewed

@@ -17,13 +17,64 @@ import SQLite3Database from "@rocicorp/zero-sqlite3";
 */
 function createSQLiteCostModel(db, tableSpecs) {
 	const fanoutEstimator = new SQLiteStatFanout(db);
+	const tableRows = tableRowsReader(db);
 	return (tableName, sort, filters, constraint) => {
 		const noSubqueryFilters = transformFilters(filters).filters;
 		const { zqlSpec } = must(tableSpecs.get(tableName));
 		const sql = compileInline(buildSelectQuery(tableName, zqlSpec, constraint, noSubqueryFilters, sort, void 0, void 0));
 		const loops = getScanstatusLoops(db.prepare(sql));
 		assert(loops.length > 0, `Expected scanstatus to return at least one loop for query: ${sql}`);
-		return estimateCost(loops, (columns) => fanoutEstimator.getFanout(tableName, columns));
+		return {
+			...estimateCost(loops, (columns) => fanoutEstimator.getFanout(tableName, columns)),
+			plan: accessPlan(loops, tableRows(tableName))
+		};
+	};
+}
+/**
+* Returns a function that reads the number of rows in a table from
+* `sqlite_stat1`, or `undefined` if the table has not been analyzed.
+*
+* The counts are cached: they only feed plan warnings, for which the count
+* as of the first plan is close enough.
+*/
+function tableRowsReader(db) {
+	let stmt;
+	const read = (table) => {
+		try {
+			stmt ??= db.prepare("SELECT stat FROM sqlite_stat1 WHERE tbl = ? LIMIT 1");
+			const row = stmt.get(table);
+			const rows = row ? parseInt(row.stat, 10) : NaN;
+			return Number.isNaN(rows) ? void 0 : rows;
+		} catch {
+			return;
+		}
+	};
+	const cache = /* @__PURE__ */ new Map();
+	return (table) => {
+		if (!cache.has(table)) cache.set(table, read(table));
+		return cache.get(table);
+	};
+}
+/**
+* Describes how SQLite reads the rows, from the EXPLAIN text of the
+* statement's top-level loops, e.g.:
+*
+* - `SEARCH issue USING INDEX issue_project (projectID=?)`
+* - `SCAN comment USING INDEX sqlite_autoindex_comment_1` (every row, in the
+*   order of the index)
+* - `SCAN issue` (every row, in table order)
+* - `USE TEMP B-TREE FOR ORDER BY` (a full sort)
+* - `USE TEMP B-TREE FOR LAST TERM OF ORDER BY` or `... RIGHT PART OF ORDER
+*   BY` (sorting runs of rows that tie on the index)
+*/
+function accessPlan(loops, tableRows) {
+	const [first, ...rest] = loops.filter((loop) => loop.parentId === 0);
+	let sort = "none";
+	for (const { explain } of rest) if (explain.startsWith("USE TEMP B-TREE FOR") && explain.includes("ORDER BY")) sort = explain === "USE TEMP B-TREE FOR ORDER BY" ? "full" : "partial";
+	return {
+		access: first?.explain.startsWith("SCAN ") ? "scan" : "search",
+		sort,
+		tableRows
 	};
 }
 /**

package/out/zqlite/src/sqlite-cost-model.js.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"sqlite-cost-model.js","names":[],"sources":["../../../../zqlite/src/sqlite-cost-model.ts"],"sourcesContent":["import SQLite3Database from '@rocicorp/zero-sqlite3';\nimport {assert} from '../../shared/src/asserts.ts';\nimport {must} from '../../shared/src/must.ts';\nimport type {Condition, Ordering} from '../../zero-protocol/src/ast.ts';\nimport type {SchemaValue} from '../../zero-types/src/schema-value.ts';\nimport {transformFilters} from '../../zql/src/builder/filter.ts';\nimport type {\n ConnectionCostModel,\n CostModelCost,\n} from '../../zql/src/planner/planner-connection.ts';\nimport type {PlannerConstraint} from '../../zql/src/planner/planner-constraint.ts';\nimport type {Database, Statement} from './db.ts';\nimport {compileInline} from './internal/sql-inline.ts';\nimport {buildSelectQuery} from './query-builder.ts';\nimport {SQLiteStatFanout} from './sqlite-stat-fanout.ts';\n\n/*\n Loop information returned by SQLite's scanstatus API.\n /\ninterface ScanstatusLoop {\n /* Unique identifier for this loop /\n selectId: number;\n /* Parent loop ID, or 0 for root loops /\n parentId: number;\n /* Estimated rows emitted per turn of parent loop /\n est: number;\n /* EXPLAIN text for this loop to determine: b-tree vs list subquery /\n explain: string;\n}\n\n/\n Creates a SQLite-based cost model for query planning.\n * Uses SQLite's scanstatus API to estimate query costs based on the actual\n * SQLite query planner's analysis.\n \n @param db Database instance for preparing statements\n * @param tableSpecs Map of table names to their table specs with ZQL schemas\n * @returns ConnectionCostModel function for use with the planner\n /\nexport function createSQLiteCostModel(\n db: Database,\n tableSpecs: Map<string, {zqlSpec: Record<string, SchemaValue>}>,\n): ConnectionCostModel {\n const fanoutEstimator = new SQLiteStatFanout(db);\n return (\n tableName: string,\n sort: Ordering,\n filters: Condition \| undefined,\n constraint: PlannerConstraint \| undefined,\n ): CostModelCost => {\n // Transform filters to remove correlated subqueries.\n //\n // We use `transformFilters` (the same function the runtime uses to\n // decide what to push to the source at connection time). It returns:\n // - For AND: the non-CSQ conjuncts (a superset* of the original).\n // - For OR with any CSQ branch: `undefined` (the OR-with-CSQ is not\n // pushable, and matches a strict superset of any single branch).\n //\n // This matches what the runtime actually applies at the source. A\n // previous implementation here had wrong OR semantics — it dropped the\n // CSQ branch and returned the simple branch alone, which is a strict\n // subset of the original OR's match set, causing the cost model to\n // under-estimate scan rows for OR-with-CSQ shapes and biasing the\n // planner toward semi-join plans. See the comment in\n // `packages/zql/src/builder/filter.ts:transformFilters`.\n const noSubqueryFilters = transformFilters(filters).filters;\n\n // Build the SQL query using the same logic as actual queries\n const {zqlSpec} = must(tableSpecs.get(tableName));\n\n const query = buildSelectQuery(\n tableName,\n zqlSpec,\n constraint,\n noSubqueryFilters,\n sort,\n undefined, // reverse is undefined here\n undefined, // start is undefined here\n );\n\n // Use compileInline to inline actual values into the SQL for cost estimation.\n // This allows SQLite's query planner to see real values and make better decisions\n // about index usage and query plans. This is safe here because it's only used for\n // cost estimation, not for executing user-facing queries (which use parameterized\n // queries via the standard compile() function).\n const sql = compileInline(query);\n\n // Prepare statement to get scanstatus information\n const stmt = db.prepare(sql);\n\n // Get scanstatus loops from the prepared statement\n const loops = getScanstatusLoops(stmt);\n\n // Scanstatus should always be available - if we get no loops, something is wrong\n assert(\n loops.length > 0,\n `Expected scanstatus to return at least one loop for query: ${sql}`,\n );\n\n const ret = estimateCost(loops, (columns: string[]) =>\n fanoutEstimator.getFanout(tableName, columns),\n );\n\n return ret;\n };\n}\n\n/*\n Gets scanstatus loop information from a prepared statement.\n * Iterates through all query elements and extracts loop statistics.\n \n Uses SQLITE_SCANSTAT_COMPLEX flag (1) to get all loops including sorting operations.\n \n @param stmt Prepared statement to get scanstatus from\n * @returns Array of loop information, or empty array if scanstatus unavailable\n /\nfunction getScanstatusLoops(stmt: Statement): ScanstatusLoop[] {\n const loops: ScanstatusLoop[] = [];\n\n // Iterate through query elements by incrementing idx until we get undefined\n // which indicates we've reached the end\n for (let idx = 0; ; idx++) {\n const selectId = stmt.scanStatus(\n idx,\n SQLite3Database.SQLITE_SCANSTAT_SELECTID,\n 1,\n );\n\n if (selectId === undefined) {\n break;\n }\n\n loops.push({\n selectId: must(selectId),\n parentId: must(\n stmt.scanStatus(idx, SQLite3Database.SQLITE_SCANSTAT_PARENTID, 1),\n ),\n explain: must(\n stmt.scanStatus(idx, SQLite3Database.SQLITE_SCANSTAT_EXPLAIN, 1),\n ),\n est: must(stmt.scanStatus(idx, SQLite3Database.SQLITE_SCANSTAT_EST, 1)),\n });\n }\n\n return loops.sort((a, b) => a.selectId - b.selectId);\n}\n\n/\n Estimates the cost of a query based on scanstats from sqlite3_stmt_scanstatus_v2\n /\nfunction estimateCost(\n scanstats: ScanstatusLoop[],\n fanout: CostModelCost['fanout'],\n): CostModelCost {\n // Sort by selectId to process in execution order\n const sorted = scanstats.toSorted((a, b) => a.selectId - b.selectId);\n\n let totalRows = 0;\n let totalCost = 0;\n\n // Identify if there are multiple top-level (parentId=0) operations\n // If so, the first is typically the scan, and subsequent ones are sorts\n const topLevelOps = sorted.filter(s => s.parentId === 0);\n\n // We only consider top level ops since ZQL queries are single-table when hitting SQLite.\n // We do have a nested op in the case of `WHERE x IN (:arg)` but it is negligible\n // assuming :arg is small.\n let firstLoop = true;\n for (const op of topLevelOps) {\n if (firstLoop) {\n // First top-level op is the main scan\n // and determines the total number of rows output.\n totalRows = op.est;\n firstLoop = false;\n } else {\n if (op.explain.includes('ORDER BY')) {\n totalCost += btreeCost(totalRows);\n }\n }\n }\n\n return {\n rows: totalRows,\n startupCost: totalCost,\n fanout,\n };\n}\n\nexport function btreeCost(rows: number): number {\n // B-Tree construction is ~O(n log n) so we estimate the cost as such.\n // We divide the cost by 10 because sorting in SQLite is ~10x faster\n // than bringing the data into JS and sorting there.\n return (rows Math.log2(rows)) / 10;\n}\n"],"mappings":";;;;;;;;;;;;;;;;;AAuCA,SAAgB,sBACd,IACA,YACqB;CACrB,MAAM,kBAAkB,IAAI,iBAAiB,EAAE;CAC/C,QACE,WACA,MACA,SACA,eACkB;EAgBlB,MAAM,oBAAoB,iBAAiB,OAAO,EAAE;EAGpD,MAAM,EAAC,YAAW,KAAK,WAAW,IAAI,SAAS,CAAC;EAiBhD,MAAM,MAAM,cAfE,iBACZ,WACA,SACA,YACA,mBACA,MACA,KAAA,GACA,KAAA,CAQwB,CAAK;EAM/B,MAAM,QAAQ,mBAHD,GAAG,QAAQ,GAGS,CAAI;EAGrC,OACE,MAAM,SAAS,GACf,8DAA8D,KAChE;EAMA,OAJY,aAAa,QAAQ,YAC/B,gBAAgB,UAAU,WAAW,OAAO,CAGvC;CACT;AACF;;;;;;;;;;AAWA,SAAS,mBAAmB,MAAmC;CAC7D,MAAM,QAA0B,CAAC;CAIjC,KAAK,IAAI,MAAM,IAAK,OAAO;EACzB,MAAM,WAAW,KAAK,WACpB,KACA,gBAAgB,0BAChB,CACF;EAEA,IAAI,aAAa,KAAA,GACf;EAGF,MAAM,KAAK;GACT,UAAU,KAAK,QAAQ;GACvB,UAAU,KACR,KAAK,WAAW,KAAK,gBAAgB,0BAA0B,CAAC,CAClE;GACA,SAAS,KACP,KAAK,WAAW,KAAK,gBAAgB,yBAAyB,CAAC,CACjE;GACA,KAAK,KAAK,KAAK,WAAW,KAAK,gBAAgB,qBAAqB,CAAC,CAAC;EACxE,CAAC;CACH;CAEA,OAAO,MAAM,MAAM,GAAG,MAAM,EAAE,WAAW,EAAE,QAAQ;AACrD;;;;AAKA,SAAS,aACP,WACA,QACe;CAEf,MAAM,SAAS,UAAU,UAAU,GAAG,MAAM,EAAE,WAAW,EAAE,QAAQ;CAEnE,IAAI,YAAY;CAChB,IAAI,YAAY;CAIhB,MAAM,cAAc,OAAO,QAAO,MAAK,EAAE,aAAa,CAAC;CAKvD,IAAI,YAAY;CAChB,KAAK,MAAM,MAAM,aACf,IAAI,WAAW;EAGb,YAAY,GAAG;EACf,YAAY;CACd,OACE,IAAI,GAAG,QAAQ,SAAS,UAAU,GAChC,aAAa,UAAU,SAAS;CAKtC,OAAO;EACL,MAAM;EACN,aAAa;EACb;CACF;AACF;AAEA,SAAgB,UAAU,MAAsB;CAI9C,OAAQ,OAAO,KAAK,KAAK,IAAI,IAAK;AACpC"}
1	+ {"version":3,"file":"sqlite-cost-model.js","names":[],"sources":["../../../../zqlite/src/sqlite-cost-model.ts"],"sourcesContent":["import SQLite3Database from '@rocicorp/zero-sqlite3';\nimport {assert} from '../../shared/src/asserts.ts';\nimport {must} from '../../shared/src/must.ts';\nimport type {Condition, Ordering} from '../../zero-protocol/src/ast.ts';\nimport type {SchemaValue} from '../../zero-types/src/schema-value.ts';\nimport {transformFilters} from '../../zql/src/builder/filter.ts';\nimport type {\n AccessPlan,\n ConnectionCostModel,\n CostModelCost,\n} from '../../zql/src/planner/planner-connection.ts';\nimport type {PlannerConstraint} from '../../zql/src/planner/planner-constraint.ts';\nimport type {Database, Statement} from './db.ts';\nimport {compileInline} from './internal/sql-inline.ts';\nimport {buildSelectQuery} from './query-builder.ts';\nimport {SQLiteStatFanout} from './sqlite-stat-fanout.ts';\n\n/*\n Loop information returned by SQLite's scanstatus API.\n /\ninterface ScanstatusLoop {\n /* Unique identifier for this loop /\n selectId: number;\n /* Parent loop ID, or 0 for root loops /\n parentId: number;\n /* Estimated rows emitted per turn of parent loop /\n est: number;\n /* EXPLAIN text for this loop to determine: b-tree vs list subquery /\n explain: string;\n}\n\n/\n Creates a SQLite-based cost model for query planning.\n * Uses SQLite's scanstatus API to estimate query costs based on the actual\n * SQLite query planner's analysis.\n \n @param db Database instance for preparing statements\n * @param tableSpecs Map of table names to their table specs with ZQL schemas\n * @returns ConnectionCostModel function for use with the planner\n /\nexport function createSQLiteCostModel(\n db: Database,\n tableSpecs: Map<string, {zqlSpec: Record<string, SchemaValue>}>,\n): ConnectionCostModel {\n const fanoutEstimator = new SQLiteStatFanout(db);\n const tableRows = tableRowsReader(db);\n return (\n tableName: string,\n sort: Ordering,\n filters: Condition \| undefined,\n constraint: PlannerConstraint \| undefined,\n ): CostModelCost => {\n // Transform filters to remove correlated subqueries.\n //\n // We use `transformFilters` (the same function the runtime uses to\n // decide what to push to the source at connection time). It returns:\n // - For AND: the non-CSQ conjuncts (a superset* of the original).\n // - For OR with any CSQ branch: `undefined` (the OR-with-CSQ is not\n // pushable, and matches a strict superset of any single branch).\n //\n // This matches what the runtime actually applies at the source. A\n // previous implementation here had wrong OR semantics — it dropped the\n // CSQ branch and returned the simple branch alone, which is a strict\n // subset of the original OR's match set, causing the cost model to\n // under-estimate scan rows for OR-with-CSQ shapes and biasing the\n // planner toward semi-join plans. See the comment in\n // `packages/zql/src/builder/filter.ts:transformFilters`.\n const noSubqueryFilters = transformFilters(filters).filters;\n\n // Build the SQL query using the same logic as actual queries\n const {zqlSpec} = must(tableSpecs.get(tableName));\n\n const query = buildSelectQuery(\n tableName,\n zqlSpec,\n constraint,\n noSubqueryFilters,\n sort,\n undefined, // reverse is undefined here\n undefined, // start is undefined here\n );\n\n // Use compileInline to inline actual values into the SQL for cost estimation.\n // This allows SQLite's query planner to see real values and make better decisions\n // about index usage and query plans. This is safe here because it's only used for\n // cost estimation, not for executing user-facing queries (which use parameterized\n // queries via the standard compile() function).\n const sql = compileInline(query);\n\n // Prepare statement to get scanstatus information\n const stmt = db.prepare(sql);\n\n // Get scanstatus loops from the prepared statement\n const loops = getScanstatusLoops(stmt);\n\n // Scanstatus should always be available - if we get no loops, something is wrong\n assert(\n loops.length > 0,\n `Expected scanstatus to return at least one loop for query: ${sql}`,\n );\n\n const ret = estimateCost(loops, (columns: string[]) =>\n fanoutEstimator.getFanout(tableName, columns),\n );\n\n return {...ret, plan: accessPlan(loops, tableRows(tableName))};\n };\n}\n\n/*\n Returns a function that reads the number of rows in a table from\n * `sqlite_stat1`, or `undefined` if the table has not been analyzed.\n \n The counts are cached: they only feed plan warnings, for which the count\n * as of the first plan is close enough.\n /\nfunction tableRowsReader(db: Database): (table: string) => number \| undefined {\n let stmt: Statement \| undefined;\n const read = (table: string): number \| undefined => {\n try {\n stmt ??= db.prepare(\n 'SELECT stat FROM sqlite_stat1 WHERE tbl = ? LIMIT 1',\n );\n // Every row for a table starts with the number of rows in the table.\n const row = stmt.get<{stat: string} \| undefined>(table);\n const rows = row ? parseInt(row.stat, 10) : NaN;\n return Number.isNaN(rows) ? undefined : rows;\n } catch {\n // sqlite_stat1 does not exist until the database is analyzed.\n return undefined;\n }\n };\n // Tables without statistics are cached too, as `undefined`.\n const cache = new Map<string, number \| undefined>();\n return table => {\n if (!cache.has(table)) {\n cache.set(table, read(table));\n }\n return cache.get(table);\n };\n}\n\n/\n Describes how SQLite reads the rows, from the EXPLAIN text of the\n * statement's top-level loops, e.g.:\n \n - `SEARCH issue USING INDEX issue_project (projectID=?)`\n * - `SCAN comment USING INDEX sqlite_autoindex_comment_1` (every row, in the\n * order of the index)\n * - `SCAN issue` (every row, in table order)\n * - `USE TEMP B-TREE FOR ORDER BY` (a full sort)\n * - `USE TEMP B-TREE FOR LAST TERM OF ORDER BY` or `... RIGHT PART OF ORDER\n * BY` (sorting runs of rows that tie on the index)\n /\nfunction accessPlan(\n loops: ScanstatusLoop[],\n tableRows: number \| undefined,\n): AccessPlan {\n const [first, ...rest] = loops.filter(loop => loop.parentId === 0);\n let sort: AccessPlan['sort'] = 'none';\n for (const {explain} of rest) {\n if (\n explain.startsWith('USE TEMP B-TREE FOR') &&\n explain.includes('ORDER BY')\n ) {\n sort = explain === 'USE TEMP B-TREE FOR ORDER BY' ? 'full' : 'partial';\n }\n }\n return {\n access: first?.explain.startsWith('SCAN ') ? 'scan' : 'search',\n sort,\n tableRows,\n };\n}\n\n/\n Gets scanstatus loop information from a prepared statement.\n * Iterates through all query elements and extracts loop statistics.\n \n Uses SQLITE_SCANSTAT_COMPLEX flag (1) to get all loops including sorting operations.\n \n @param stmt Prepared statement to get scanstatus from\n * @returns Array of loop information, or empty array if scanstatus unavailable\n /\nfunction getScanstatusLoops(stmt: Statement): ScanstatusLoop[] {\n const loops: ScanstatusLoop[] = [];\n\n // Iterate through query elements by incrementing idx until we get undefined\n // which indicates we've reached the end\n for (let idx = 0; ; idx++) {\n const selectId = stmt.scanStatus(\n idx,\n SQLite3Database.SQLITE_SCANSTAT_SELECTID,\n 1,\n );\n\n if (selectId === undefined) {\n break;\n }\n\n loops.push({\n selectId: must(selectId),\n parentId: must(\n stmt.scanStatus(idx, SQLite3Database.SQLITE_SCANSTAT_PARENTID, 1),\n ),\n explain: must(\n stmt.scanStatus(idx, SQLite3Database.SQLITE_SCANSTAT_EXPLAIN, 1),\n ),\n est: must(stmt.scanStatus(idx, SQLite3Database.SQLITE_SCANSTAT_EST, 1)),\n });\n }\n\n return loops.sort((a, b) => a.selectId - b.selectId);\n}\n\n/\n Estimates the cost of a query based on scanstats from sqlite3_stmt_scanstatus_v2\n /\nfunction estimateCost(\n scanstats: ScanstatusLoop[],\n fanout: CostModelCost['fanout'],\n): CostModelCost {\n // Sort by selectId to process in execution order\n const sorted = scanstats.toSorted((a, b) => a.selectId - b.selectId);\n\n let totalRows = 0;\n let totalCost = 0;\n\n // Identify if there are multiple top-level (parentId=0) operations\n // If so, the first is typically the scan, and subsequent ones are sorts\n const topLevelOps = sorted.filter(s => s.parentId === 0);\n\n // We only consider top level ops since ZQL queries are single-table when hitting SQLite.\n // We do have a nested op in the case of `WHERE x IN (:arg)` but it is negligible\n // assuming :arg is small.\n let firstLoop = true;\n for (const op of topLevelOps) {\n if (firstLoop) {\n // First top-level op is the main scan\n // and determines the total number of rows output.\n totalRows = op.est;\n firstLoop = false;\n } else {\n if (op.explain.includes('ORDER BY')) {\n totalCost += btreeCost(totalRows);\n }\n }\n }\n\n return {\n rows: totalRows,\n startupCost: totalCost,\n fanout,\n };\n}\n\nexport function btreeCost(rows: number): number {\n // B-Tree construction is ~O(n log n) so we estimate the cost as such.\n // We divide the cost by 10 because sorting in SQLite is ~10x faster\n // than bringing the data into JS and sorting there.\n return (rows Math.log2(rows)) / 10;\n}\n"],"mappings":";;;;;;;;;;;;;;;;;AAwCA,SAAgB,sBACd,IACA,YACqB;CACrB,MAAM,kBAAkB,IAAI,iBAAiB,EAAE;CAC/C,MAAM,YAAY,gBAAgB,EAAE;CACpC,QACE,WACA,MACA,SACA,eACkB;EAgBlB,MAAM,oBAAoB,iBAAiB,OAAO,EAAE;EAGpD,MAAM,EAAC,YAAW,KAAK,WAAW,IAAI,SAAS,CAAC;EAiBhD,MAAM,MAAM,cAfE,iBACZ,WACA,SACA,YACA,mBACA,MACA,KAAA,GACA,KAAA,CAQwB,CAAK;EAM/B,MAAM,QAAQ,mBAHD,GAAG,QAAQ,GAGS,CAAI;EAGrC,OACE,MAAM,SAAS,GACf,8DAA8D,KAChE;EAMA,OAAO;GAAC,GAJI,aAAa,QAAQ,YAC/B,gBAAgB,UAAU,WAAW,OAAO,CAGnC;GAAK,MAAM,WAAW,OAAO,UAAU,SAAS,CAAC;EAAC;CAC/D;AACF;;;;;;;;AASA,SAAS,gBAAgB,IAAqD;CAC5E,IAAI;CACJ,MAAM,QAAQ,UAAsC;EAClD,IAAI;GACF,SAAS,GAAG,QACV,qDACF;GAEA,MAAM,MAAM,KAAK,IAAgC,KAAK;GACtD,MAAM,OAAO,MAAM,SAAS,IAAI,MAAM,EAAE,IAAI;GAC5C,OAAO,OAAO,MAAM,IAAI,IAAI,KAAA,IAAY;EAC1C,QAAQ;GAEN;EACF;CACF;CAEA,MAAM,wBAAQ,IAAI,IAAgC;CAClD,QAAO,UAAS;EACd,IAAI,CAAC,MAAM,IAAI,KAAK,GAClB,MAAM,IAAI,OAAO,KAAK,KAAK,CAAC;EAE9B,OAAO,MAAM,IAAI,KAAK;CACxB;AACF;;;;;;;;;;;;;AAcA,SAAS,WACP,OACA,WACY;CACZ,MAAM,CAAC,OAAO,GAAG,QAAQ,MAAM,QAAO,SAAQ,KAAK,aAAa,CAAC;CACjE,IAAI,OAA2B;CAC/B,KAAK,MAAM,EAAC,aAAY,MACtB,IACE,QAAQ,WAAW,qBAAqB,KACxC,QAAQ,SAAS,UAAU,GAE3B,OAAO,YAAY,iCAAiC,SAAS;CAGjE,OAAO;EACL,QAAQ,OAAO,QAAQ,WAAW,OAAO,IAAI,SAAS;EACtD;EACA;CACF;AACF;;;;;;;;;;AAWA,SAAS,mBAAmB,MAAmC;CAC7D,MAAM,QAA0B,CAAC;CAIjC,KAAK,IAAI,MAAM,IAAK,OAAO;EACzB,MAAM,WAAW,KAAK,WACpB,KACA,gBAAgB,0BAChB,CACF;EAEA,IAAI,aAAa,KAAA,GACf;EAGF,MAAM,KAAK;GACT,UAAU,KAAK,QAAQ;GACvB,UAAU,KACR,KAAK,WAAW,KAAK,gBAAgB,0BAA0B,CAAC,CAClE;GACA,SAAS,KACP,KAAK,WAAW,KAAK,gBAAgB,yBAAyB,CAAC,CACjE;GACA,KAAK,KAAK,KAAK,WAAW,KAAK,gBAAgB,qBAAqB,CAAC,CAAC;EACxE,CAAC;CACH;CAEA,OAAO,MAAM,MAAM,GAAG,MAAM,EAAE,WAAW,EAAE,QAAQ;AACrD;;;;AAKA,SAAS,aACP,WACA,QACe;CAEf,MAAM,SAAS,UAAU,UAAU,GAAG,MAAM,EAAE,WAAW,EAAE,QAAQ;CAEnE,IAAI,YAAY;CAChB,IAAI,YAAY;CAIhB,MAAM,cAAc,OAAO,QAAO,MAAK,EAAE,aAAa,CAAC;CAKvD,IAAI,YAAY;CAChB,KAAK,MAAM,MAAM,aACf,IAAI,WAAW;EAGb,YAAY,GAAG;EACf,YAAY;CACd,OACE,IAAI,GAAG,QAAQ,SAAS,UAAU,GAChC,aAAa,UAAU,SAAS;CAKtC,OAAO;EACL,MAAM;EACN,aAAa;EACb;CACF;AACF;AAEA,SAAgB,UAAU,MAAsB;CAI9C,OAAQ,OAAO,KAAK,KAAK,IAAI,IAAK;AACpC"}

package/out/zqlite/src/sqlite-stat-fanout.d.ts.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"sqlite-stat-fanout.d.ts","sourceRoot":"","sources":["../../../../zqlite/src/sqlite-stat-fanout.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAC,QAAQ,EAAC,MAAM,SAAS,CAAC;~~AAEtC~~;;GAEG;AACH,MAAM,WAAW,YAAY;IAC3B;;;OAGG;IACH,MAAM,EAAE,MAAM,CAAC;IACf,UAAU,EAAE,MAAM,GAAG,KAAK,GAAG,MAAM,CAAC;IAEpC;;;;;OAKG;IACH,MAAM,EAAE,OAAO,GAAG,OAAO,GAAG,SAAS,CAAC;CACvC;AAgBD;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAmDG;AACH,qBAAa,gBAAgB;;IAkB3B;;;;;;;;OAQG;IACH,YAAY,EAAE,EAAE,QAAQ,EAAE,aAAa,SAAI,EAyB1C;IAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OA6BG;IACH,SAAS,CAAC,SAAS,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,GAAG,YAAY,CAgC5D;IAED;;;OAGG;IACH,UAAU,IAAI,IAAI,CAEjB;~~CAgQF~~"}
1	+ {"version":3,"file":"sqlite-stat-fanout.d.ts","sourceRoot":"","sources":["../../../../zqlite/src/sqlite-stat-fanout.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAC,QAAQ,EAAC,MAAM,SAAS,CAAC;AAGtC;;GAEG;AACH,MAAM,WAAW,YAAY;IAC3B;;;OAGG;IACH,MAAM,EAAE,MAAM,CAAC;IACf,UAAU,EAAE,MAAM,GAAG,KAAK,GAAG,MAAM,CAAC;IAEpC;;;;;OAKG;IACH,MAAM,EAAE,OAAO,GAAG,OAAO,GAAG,SAAS,CAAC;CACvC;AAgBD;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAmDG;AACH,qBAAa,gBAAgB;;IAkB3B;;;;;;;;OAQG;IACH,YAAY,EAAE,EAAE,QAAQ,EAAE,aAAa,SAAI,EAyB1C;IAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OA6BG;IACH,SAAS,CAAC,SAAS,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,GAAG,YAAY,CAgC5D;IAED;;;OAGG;IACH,UAAU,IAAI,IAAI,CAEjB;CA0NF"}

package/out/zqlite/src/sqlite-stat-fanout.js CHANGED Viewed

@@ -1,5 +1,6 @@
 import { getOrInsertComputed } from "../../shared/src/map.js";
 import { newArray } from "../../shared/src/arrays.js";
+import { isSampleNull } from "./sqlite-stat4-sample.js";
 //#region ../zqlite/src/sqlite-stat-fanout.ts
 /**
 * Computes join fanout factors from SQLite statistics tables.
@@ -180,7 +181,7 @@ var SQLiteStatFanout = class {
 				const neqParts = s.neq.split(" ");
 				return {
 					fanout: parseInt(neqParts[neqIndex] ?? neqParts[0], 10),
-					isNull: this.#decodeSampleIsNull(s.sample)
+					isNull: isSampleNull(s.sample)
 				};
 			}).filter((s) => !s.isNull);
 			if (nonNullSamples.length === 0) return {
@@ -286,31 +287,6 @@ var SQLiteStatFanout = class {
 		const indexPrefixLower = new Set(indexPrefix.map((col) => col.toLowerCase()));
 		return queryColumns.map((col) => col.toLowerCase()).every((queryCol) => indexPrefixLower.has(queryCol));
 	}
-	/**
-	* Decodes a sqlite_stat4 sample value to check if it's NULL.
-	*
-	* SQLite record format (simplified):
-	* - Varint: header size
-	* - Serial types for each column (one byte each typically)
-	* - Actual data
-	*
-	* Serial type 0 = NULL
-	* Serial type 1 = 8-bit int
-	* Serial type 2 = 16-bit int
-	* Serial type 3 = 24-bit int
-	* etc.
-	*
-	* We only need to check the first column's serial type.
-	*
-	* @param sample Binary-encoded sample from stat4
-	* @returns true if the sample value is NULL
-	*/
-	#decodeSampleIsNull(sample) {
-		if (sample.length === 0) return true;
-		const headerSize = sample[0];
-		if (headerSize === 0 || headerSize >= sample.length) return true;
-		return sample[1] === 0;
-	}
 };
 //#endregion
 export { SQLiteStatFanout };

package/out/zqlite/src/sqlite-stat-fanout.js.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"sqlite-stat-fanout.js","names":["#db","#defaultFanout","#cache","#stat4Stmt","#stat1Stmt","#indexStmt","#getFanoutFromStat4","#getFanoutFromStat1","#findIndexForColumns","#decodeSampleIsNull","#isPrefixMatch"],"sources":["../../../../zqlite/src/sqlite-stat-fanout.ts"],"sourcesContent":["import {newArray} from '../../shared/src/arrays.ts';\nimport {getOrInsertComputed} from '../../shared/src/map.ts';\nimport type {Database} from './db.ts';\n\n/*\n Result of fanout calculation from SQLite statistics.\n /\nexport interface FanoutResult {\n /\n The fanout value (average rows per distinct value of the join column).\n * For non-NULL joins, this represents how many child rows exist per parent key.\n /\n fanout: number;\n confidence: 'high' \| 'med' \| 'none';\n\n /\n Source of the fanout calculation.\n * - 'stat4': From sqlite_stat4 histogram (most accurate, excludes NULLs)\n * - 'stat1': From sqlite_stat1 average (includes NULLs, may overestimate)\n * - 'default': Fallback constant when statistics unavailable\n /\n source: 'stat4' \| 'stat1' \| 'default';\n}\n\n/\n Sample from sqlite_stat4 histogram.\n /\ninterface Stat4Sample {\n /* \"N1 N2\" = rows equal to sample (N1=first col, N2=if composite) /\n neq: string;\n /* \"N1 N2\" = rows less than sample /\n nlt: string;\n /* \"N1 N2\" = distinct values less than sample /\n ndlt: string;\n /* The actual sample value (binary encoded) /\n sample: Buffer;\n}\n\n/\n Computes join fanout factors from SQLite statistics tables.\n \n Fanout is the average number of child rows per distinct parent key value,\n * used to estimate join cardinality in query planning.\n \n ## Problem\n \n sqlite_stat1 includes NULL rows in its calculation, which can significantly\n * overestimate fanout for sparse foreign keys:\n \n ```\n * Example: 100 tasks, 20 with project_id, 80 with NULL\n * - stat1 reports: \"100 17\" → fanout = 17 (WRONG - includes NULLs)\n * - stat4 shows: NULL samples with fanout=80, non-NULL samples with fanout=4\n * - True fanout: 4 (CORRECT)\n * ```\n \n ## Solution\n \n This class uses sqlite_stat4 histogram to separate NULL and non-NULL samples,\n * providing accurate fanout for non-NULL joins.\n \n ## Usage\n \n ```typescript\n * const calculator = new SQLiteStatFanout(db);\n \n // Get fanout for posts.userId → users.id join\n * const result = calculator.getFanout('posts', 'userId');\n \n if (result.source === 'stat4') {\n * // Accurate: excludes NULLs, samples actual distribution\n * console.log(`Fanout: ${result.fanout} (from stat4)`);\n * } else if (result.source === 'stat1') {\n * // Conservative: includes NULLs, may overestimate\n * console.log(`Fanout: ${result.fanout} (from stat1, includes NULLs)`);\n * } else {\n * // Fallback: no statistics available\n * console.log(`Fanout: ${result.fanout} (default estimate)`);\n * }\n * ```\n \n ## Requirements\n \n - SQLite compiled with ENABLE_STAT4 (most builds include this)\n * - `ANALYZE` command run on the database\n * - Index exists on the join column\n \n @see https://sqlite.org/fileformat2.html#stat4tab\n * @see packages/zql/src/planner/SELECTIVITY_PLAN.md\n /\nexport class SQLiteStatFanout {\n readonly #db: Database;\n readonly #defaultFanout: number;\n\n /\n Cache of fanout results by table and columns.\n * Key format: \"tableName:col1,col2,col3\" (sorted alphabetically)\n /\n readonly #cache = new Map<string, FanoutResult>();\n\n /\n Prepared statements for querying SQLite statistics tables.\n * Prepared once in constructor for performance.\n /\n readonly #stat4Stmt: ReturnType<Database['prepare']>;\n readonly #stat1Stmt: ReturnType<Database['prepare']>;\n readonly #indexStmt: ReturnType<Database['prepare']>;\n\n /\n Creates a new fanout calculator.\n \n @param db Database instance\n * @param defaultFanout Default fanout when statistics unavailable (default: 3)\n * - 1: Conservative (assumes FK relationships)\n * - 3: Moderate (recommended, safe middle ground)\n * - 10: SQLite's default (optimistic)\n /\n constructor(db: Database, defaultFanout = 3) {\n this.#db = db;\n this.#defaultFanout = defaultFanout;\n\n // Prepare SQL statements once for reuse across multiple getFanout() calls\n this.#stat4Stmt = this.#db.prepare(`\n SELECT neq, nlt, ndlt, sample\n FROM sqlite_stat4\n WHERE tbl = ? AND idx = ?\n ORDER BY nlt\n `);\n\n this.#stat1Stmt = this.#db.prepare(`\n SELECT stat\n FROM sqlite_stat1\n WHERE tbl = ? AND idx = ?\n `);\n\n this.#indexStmt = this.#db.prepare(`\n SELECT il.name as index_name, ii.seqno, ii.name as column_name\n FROM pragma_index_list(?) il\n JOIN pragma_index_info(il.name) ii\n WHERE il.partial = 0\n ORDER BY il.seq, ii.seqno\n `);\n }\n\n /\n Gets the fanout factor for join column(s).\n \n Fanout = average number of child rows per distinct parent key value(s).\n \n ## Strategy\n \n 1. Try sqlite_stat4 (best): Histogram with separate NULL/non-NULL samples\n * 2. Fallback to sqlite_stat1: Average across all rows (includes NULLs)\n * 3. Fallback to default: When no statistics available\n \n ## Compound Indexes\n \n For multi-column joins, finds indexes where ALL columns appear as an\n * exact prefix. Uses the appropriate depth in stat1/stat4.\n \n Example:\n * - Columns: `['customerId', 'storeId']`\n * - Matches index: `(customerId, storeId, date)` at depth 2\n * - Uses stat1 parts[2] or stat4 neq[1] for accurate fanout\n \n ## Caching\n \n Results are cached per (table, columns) combination. Clear the cache if\n * you run ANALYZE to update statistics.\n \n @param tableName Table containing the join column(s)\n * @param columns Array of column names (one or more columns)\n * @returns Fanout result with value and source\n /\n getFanout(tableName: string, columns: string[]): FanoutResult {\n // Cache key uses sorted columns for consistency\n const cacheKey = `${tableName}:${columns.toSorted().join(',')}`;\n const cached = this.#cache.get(cacheKey);\n if (cached) {\n return cached;\n }\n\n // Strategy 1: Try stat4 first (most accurate)\n // NOTE: columns are NOT sorted - preserves Object.keys() order from constraint\n // Matching is order-independent (flexible), but we keep original order for consistency\n const stat4Result = this.#getFanoutFromStat4(tableName, columns);\n if (stat4Result) {\n this.#cache.set(cacheKey, stat4Result);\n return stat4Result;\n }\n\n // Strategy 2: Fallback to stat1 (includes NULLs)\n const stat1Result = this.#getFanoutFromStat1(tableName, columns);\n if (stat1Result) {\n this.#cache.set(cacheKey, stat1Result);\n return stat1Result;\n }\n\n // Strategy 3: Use default\n const defaultResult: FanoutResult = {\n fanout: this.#defaultFanout,\n confidence: 'none',\n source: 'default',\n };\n this.#cache.set(cacheKey, defaultResult);\n return defaultResult;\n }\n\n /\n Clears the fanout cache.\n * Call this after running ANALYZE to pick up updated statistics.\n /\n clearCache(): void {\n this.#cache.clear();\n }\n\n /\n Gets fanout from sqlite_stat4 histogram.\n \n Queries stat4 samples, decodes to identify NULLs, and returns\n * the median fanout of non-NULL samples.\n \n For compound indexes, uses the neq value at the appropriate depth.\n \n @param columns Array of column names to get fanout for\n * @returns Fanout result or undefined if stat4 unavailable\n /\n #getFanoutFromStat4(\n tableName: string,\n columns: string[],\n ): FanoutResult \| undefined {\n try {\n // Find index containing the columns as a prefix\n const indexInfo = this.#findIndexForColumns(tableName, columns);\n if (!indexInfo) {\n return undefined;\n }\n\n // Query stat4 samples for this index (using prepared statement)\n const samples = this.#stat4Stmt.all(\n tableName,\n indexInfo.indexName,\n ) as Stat4Sample[];\n\n if (samples.length === 0) {\n return undefined;\n }\n\n // Decode samples and separate NULL from non-NULL\n // Use depth-1 for neq array index (depth is 1-based, array is 0-based)\n const neqIndex = indexInfo.depth - 1;\n const decodedSamples = samples.map(s => {\n const neqParts = s.neq.split(' ');\n return {\n fanout: parseInt(neqParts[neqIndex] ?? neqParts[0], 10),\n isNull: this.#decodeSampleIsNull(s.sample),\n };\n });\n\n const nonNullSamples = decodedSamples.filter(s => !s.isNull);\n\n if (nonNullSamples.length === 0) {\n // All samples are NULL - return fanout of 0 since NULLs don't match in joins\n return {\n fanout: 0,\n source: 'stat4',\n confidence: 'high',\n };\n }\n\n // Use median of non-NULL fanouts (more robust than average)\n const fanouts = nonNullSamples.map(s => s.fanout).sort((a, b) => a - b);\n const medianFanout =\n fanouts.length % 2 === 0\n ? Math.floor(\n (fanouts[fanouts.length / 2 - 1] + fanouts[fanouts.length / 2]) /\n 2,\n )\n : fanouts[Math.floor(fanouts.length / 2)];\n\n return {\n fanout: medianFanout,\n source: 'stat4',\n confidence: 'high',\n };\n } catch {\n // stat4 table may not exist or query may fail\n return undefined;\n }\n }\n\n /\n Gets fanout from sqlite_stat1 average.\n \n Note: This includes NULL rows in the calculation and may overestimate\n * fanout for sparse foreign keys.\n \n For compound indexes, uses the stat value at the appropriate depth.\n \n @param columns Array of column names to get fanout for\n * @returns Fanout result or undefined if stat1 unavailable\n /\n #getFanoutFromStat1(\n tableName: string,\n columns: string[],\n ): FanoutResult \| undefined {\n try {\n // Find index containing the columns as a prefix\n const indexInfo = this.#findIndexForColumns(tableName, columns);\n if (!indexInfo) {\n return undefined;\n }\n\n // Query stat1 for this index (using prepared statement)\n const result = this.#stat1Stmt.get(tableName, indexInfo.indexName) as\n \| {stat: string}\n \| undefined;\n\n if (!result) {\n return undefined;\n }\n\n const parts = result.stat.split(' ');\n // Check if we have enough parts for the requested depth\n if (parts.length < indexInfo.depth + 1) {\n return undefined;\n }\n\n const fanout = parseInt(parts[indexInfo.depth], 10);\n if (isNaN(fanout)) {\n return undefined;\n }\n\n return {\n fanout,\n source: 'stat1',\n confidence: 'med',\n };\n } catch {\n return undefined;\n }\n }\n\n /\n Finds an index that can be used to get statistics for column(s).\n \n Uses pragma_index_list and pragma_index_info to reliably get index\n * column names, avoiding brittle SQL parsing. Includes all indices:\n * user-created (CREATE INDEX), PRIMARY KEY, and UNIQUE constraints.\n \n Uses flexible matching: Finds indexes where ALL columns appear in the\n * first N positions, regardless of order. This works because SQLite statistics\n * at depth N represent the fanout for the combination of the first N columns,\n * and combinations are order-independent.\n \n Example:\n * - columns: ['customerId', 'storeId']\n * - Matches: (customerId, storeId, date) at depth 2 ✅\n * - Matches: (storeId, customerId, date) at depth 2 ✅ (flexible order)\n * - Does NOT match: (date, customerId, storeId) ❌ (columns not in first 2 positions)\n * - Does NOT match: (customerId, date, storeId) ❌ (storeId not in first 2 positions)\n \n @param columns Array of column names (order-independent for matching)\n * @returns Index info with name and depth, or undefined if no match\n /\n #findIndexForColumns(\n tableName: string,\n columns: string[],\n ): {indexName: string; depth: number} \| undefined {\n try {\n // Query returns all columns for all indexes (including PK/UNIQUE) in order\n const rows = this.#indexStmt.all(tableName) as {\n index_name: string;\n seqno: number;\n column_name: string;\n }[];\n\n // Group by index name\n const indexMap = new Map<string, string[]>();\n for (const row of rows) {\n getOrInsertComputed(indexMap, row.index_name, newArray).push(\n row.column_name,\n );\n }\n\n // Check each index for prefix match\n for (const [indexName, indexColumns] of indexMap) {\n if (this.#isPrefixMatch(columns, indexColumns)) {\n return {\n indexName,\n depth: columns.length,\n };\n }\n }\n\n return undefined;\n } catch {\n return undefined;\n }\n }\n\n /\n Checks if all queryColumns exist in the first N positions of indexColumns,\n * regardless of order.\n \n This allows flexible matching: constraint {a, b} matches both index (a, b, c)\n * and index (b, a, c) at depth 2, since both represent the fanout for the\n * combination of columns a and b.\n \n Gaps are NOT allowed: constraint {a, c} does NOT match index (a, b, c)\n * because no depth represents just (a, c) without b. Statistics are cumulative\n * from position 0.\n \n @param queryColumns Columns we're looking for (from constraint)\n * @param indexColumns Columns in the index (in order)\n * @returns true if all queryColumns exist in indexColumns[0...queryColumns.length-1]\n /\n #isPrefixMatch(queryColumns: string[], indexColumns: string[]): boolean {\n if (queryColumns.length > indexColumns.length) {\n return false;\n }\n\n // Get the prefix of the index that we're checking against\n const indexPrefix = indexColumns.slice(0, queryColumns.length);\n\n // Normalize to lowercase for case-insensitive comparison\n const indexPrefixLower = new Set(indexPrefix.map(col => col.toLowerCase()));\n const queryColumnsLower = queryColumns.map(col => col.toLowerCase());\n\n // Check if ALL query columns exist in the index prefix\n return queryColumnsLower.every(queryCol => indexPrefixLower.has(queryCol));\n }\n\n /\n Decodes a sqlite_stat4 sample value to check if it's NULL.\n \n SQLite record format (simplified):\n * - Varint: header size\n * - Serial types for each column (one byte each typically)\n * - Actual data\n \n Serial type 0 = NULL\n * Serial type 1 = 8-bit int\n * Serial type 2 = 16-bit int\n * Serial type 3 = 24-bit int\n * etc.\n \n We only need to check the first column's serial type.\n \n @param sample Binary-encoded sample from stat4\n * @returns true if the sample value is NULL\n */\n #decodeSampleIsNull(sample: Buffer): boolean {\n if (sample.length === 0) {\n return true;\n }\n\n // Read header size (varint - simplified: assume single byte)\n const headerSize = sample[0];\n\n if (headerSize === 0 \|\| headerSize >= sample.length) {\n return true;\n }\n\n // Read first serial type (at position 1)\n const serialType = sample[1];\n\n // Serial type 0 = NULL\n return serialType === 0;\n }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AA0FA,IAAa,mBAAb,MAA8B;CAC5B;CACA;;;;;CAMA,yBAAkB,IAAI,IAA0B;;;;;CAMhD;CACA;CACA;;;;;;;;;;CAWA,YAAY,IAAc,gBAAgB,GAAG;EAC3C,KAAKA,MAAM;EACX,KAAKC,iBAAiB;EAGtB,KAAKE,aAAa,KAAKH,IAAI,QAAQ;;;;;KAKlC;EAED,KAAKI,aAAa,KAAKJ,IAAI,QAAQ;;;;KAIlC;EAED,KAAKK,aAAa,KAAKL,IAAI,QAAQ;;;;;;KAMlC;CACH;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAgCA,UAAU,WAAmB,SAAiC;EAE5D,MAAM,WAAW,GAAG,UAAU,GAAG,QAAQ,SAAS,EAAE,KAAK,GAAG;EAC5D,MAAM,SAAS,KAAKE,OAAO,IAAI,QAAQ;EACvC,IAAI,QACF,OAAO;EAMT,MAAM,cAAc,KAAKI,oBAAoB,WAAW,OAAO;EAC/D,IAAI,aAAa;GACf,KAAKJ,OAAO,IAAI,UAAU,WAAW;GACrC,OAAO;EACT;EAGA,MAAM,cAAc,KAAKK,oBAAoB,WAAW,OAAO;EAC/D,IAAI,aAAa;GACf,KAAKL,OAAO,IAAI,UAAU,WAAW;GACrC,OAAO;EACT;EAGA,MAAM,gBAA8B;GAClC,QAAQ,KAAKD;GACb,YAAY;GACZ,QAAQ;EACV;EACA,KAAKC,OAAO,IAAI,UAAU,aAAa;EACvC,OAAO;CACT;;;;;CAMA,aAAmB;EACjB,KAAKA,OAAO,MAAM;CACpB;;;;;;;;;;;;CAaA,oBACE,WACA,SAC0B;EAC1B,IAAI;GAEF,MAAM,YAAY,KAAKM,qBAAqB,WAAW,OAAO;GAC9D,IAAI,CAAC,WACH;GAIF,MAAM,UAAU,KAAKL,WAAW,IAC9B,WACA,UAAU,SACZ;GAEA,IAAI,QAAQ,WAAW,GACrB;GAKF,MAAM,WAAW,UAAU,QAAQ;GASnC,MAAM,iBARiB,QAAQ,KAAI,MAAK;IACtC,MAAM,WAAW,EAAE,IAAI,MAAM,GAAG;IAChC,OAAO;KACL,QAAQ,SAAS,SAAS,aAAa,SAAS,IAAI,EAAE;KACtD,QAAQ,KAAKM,oBAAoB,EAAE,MAAM;IAC3C;GACF,CAEuB,EAAe,QAAO,MAAK,CAAC,EAAE,MAAM;GAE3D,IAAI,eAAe,WAAW,GAE5B,OAAO;IACL,QAAQ;IACR,QAAQ;IACR,YAAY;GACd;GAIF,MAAM,UAAU,eAAe,KAAI,MAAK,EAAE,MAAM,EAAE,MAAM,GAAG,MAAM,IAAI,CAAC;GAStE,OAAO;IACL,QARA,QAAQ,SAAS,MAAM,IACnB,KAAK,OACF,QAAQ,QAAQ,SAAS,IAAI,KAAK,QAAQ,QAAQ,SAAS,MAC1D,CACJ,IACA,QAAQ,KAAK,MAAM,QAAQ,SAAS,CAAC;IAIzC,QAAQ;IACR,YAAY;GACd;EACF,QAAQ;GAEN;EACF;CACF;;;;;;;;;;;;CAaA,oBACE,WACA,SAC0B;EAC1B,IAAI;GAEF,MAAM,YAAY,KAAKD,qBAAqB,WAAW,OAAO;GAC9D,IAAI,CAAC,WACH;GAIF,MAAM,SAAS,KAAKJ,WAAW,IAAI,WAAW,UAAU,SAAS;GAIjE,IAAI,CAAC,QACH;GAGF,MAAM,QAAQ,OAAO,KAAK,MAAM,GAAG;GAEnC,IAAI,MAAM,SAAS,UAAU,QAAQ,GACnC;GAGF,MAAM,SAAS,SAAS,MAAM,UAAU,QAAQ,EAAE;GAClD,IAAI,MAAM,MAAM,GACd;GAGF,OAAO;IACL;IACA,QAAQ;IACR,YAAY;GACd;EACF,QAAQ;GACN;EACF;CACF;;;;;;;;;;;;;;;;;;;;;;;CAwBA,qBACE,WACA,SACgD;EAChD,IAAI;GAEF,MAAM,OAAO,KAAKC,WAAW,IAAI,SAAS;GAO1C,MAAM,2BAAW,IAAI,IAAsB;GAC3C,KAAK,MAAM,OAAO,MAChB,oBAAoB,UAAU,IAAI,YAAY,QAAQ,EAAE,KACtD,IAAI,WACN;GAIF,KAAK,MAAM,CAAC,WAAW,iBAAiB,UACtC,IAAI,KAAKK,eAAe,SAAS,YAAY,GAC3C,OAAO;IACL;IACA,OAAO,QAAQ;GACjB;GAIJ;EACF,QAAQ;GACN;EACF;CACF;;;;;;;;;;;;;;;;;CAkBA,eAAe,cAAwB,cAAiC;EACtE,IAAI,aAAa,SAAS,aAAa,QACrC,OAAO;EAIT,MAAM,cAAc,aAAa,MAAM,GAAG,aAAa,MAAM;EAG7D,MAAM,mBAAmB,IAAI,IAAI,YAAY,KAAI,QAAO,IAAI,YAAY,CAAC,CAAC;EAI1E,OAH0B,aAAa,KAAI,QAAO,IAAI,YAAY,CAG3D,EAAkB,OAAM,aAAY,iBAAiB,IAAI,QAAQ,CAAC;CAC3E;;;;;;;;;;;;;;;;;;;;CAqBA,oBAAoB,QAAyB;EAC3C,IAAI,OAAO,WAAW,GACpB,OAAO;EAIT,MAAM,aAAa,OAAO;EAE1B,IAAI,eAAe,KAAK,cAAc,OAAO,QAC3C,OAAO;EAOT,OAHmB,OAAO,OAGJ;CACxB;AACF"}
1	+ {"version":3,"file":"sqlite-stat-fanout.js","names":["#db","#defaultFanout","#cache","#stat4Stmt","#stat1Stmt","#indexStmt","#getFanoutFromStat4","#getFanoutFromStat1","#findIndexForColumns","#isPrefixMatch"],"sources":["../../../../zqlite/src/sqlite-stat-fanout.ts"],"sourcesContent":["import {newArray} from '../../shared/src/arrays.ts';\nimport {getOrInsertComputed} from '../../shared/src/map.ts';\nimport type {Database} from './db.ts';\nimport {isSampleNull} from './sqlite-stat4-sample.ts';\n\n/*\n Result of fanout calculation from SQLite statistics.\n /\nexport interface FanoutResult {\n /\n The fanout value (average rows per distinct value of the join column).\n * For non-NULL joins, this represents how many child rows exist per parent key.\n /\n fanout: number;\n confidence: 'high' \| 'med' \| 'none';\n\n /\n Source of the fanout calculation.\n * - 'stat4': From sqlite_stat4 histogram (most accurate, excludes NULLs)\n * - 'stat1': From sqlite_stat1 average (includes NULLs, may overestimate)\n * - 'default': Fallback constant when statistics unavailable\n /\n source: 'stat4' \| 'stat1' \| 'default';\n}\n\n/\n Sample from sqlite_stat4 histogram.\n /\ninterface Stat4Sample {\n /* \"N1 N2\" = rows equal to sample (N1=first col, N2=if composite) /\n neq: string;\n /* \"N1 N2\" = rows less than sample /\n nlt: string;\n /* \"N1 N2\" = distinct values less than sample /\n ndlt: string;\n /* The actual sample value (binary encoded) /\n sample: Buffer;\n}\n\n/\n Computes join fanout factors from SQLite statistics tables.\n \n Fanout is the average number of child rows per distinct parent key value,\n * used to estimate join cardinality in query planning.\n \n ## Problem\n \n sqlite_stat1 includes NULL rows in its calculation, which can significantly\n * overestimate fanout for sparse foreign keys:\n \n ```\n * Example: 100 tasks, 20 with project_id, 80 with NULL\n * - stat1 reports: \"100 17\" → fanout = 17 (WRONG - includes NULLs)\n * - stat4 shows: NULL samples with fanout=80, non-NULL samples with fanout=4\n * - True fanout: 4 (CORRECT)\n * ```\n \n ## Solution\n \n This class uses sqlite_stat4 histogram to separate NULL and non-NULL samples,\n * providing accurate fanout for non-NULL joins.\n \n ## Usage\n \n ```typescript\n * const calculator = new SQLiteStatFanout(db);\n \n // Get fanout for posts.userId → users.id join\n * const result = calculator.getFanout('posts', 'userId');\n \n if (result.source === 'stat4') {\n * // Accurate: excludes NULLs, samples actual distribution\n * console.log(`Fanout: ${result.fanout} (from stat4)`);\n * } else if (result.source === 'stat1') {\n * // Conservative: includes NULLs, may overestimate\n * console.log(`Fanout: ${result.fanout} (from stat1, includes NULLs)`);\n * } else {\n * // Fallback: no statistics available\n * console.log(`Fanout: ${result.fanout} (default estimate)`);\n * }\n * ```\n \n ## Requirements\n \n - SQLite compiled with ENABLE_STAT4 (most builds include this)\n * - `ANALYZE` command run on the database\n * - Index exists on the join column\n \n @see https://sqlite.org/fileformat2.html#stat4tab\n * @see packages/zql/src/planner/SELECTIVITY_PLAN.md\n /\nexport class SQLiteStatFanout {\n readonly #db: Database;\n readonly #defaultFanout: number;\n\n /\n Cache of fanout results by table and columns.\n * Key format: \"tableName:col1,col2,col3\" (sorted alphabetically)\n /\n readonly #cache = new Map<string, FanoutResult>();\n\n /\n Prepared statements for querying SQLite statistics tables.\n * Prepared once in constructor for performance.\n /\n readonly #stat4Stmt: ReturnType<Database['prepare']>;\n readonly #stat1Stmt: ReturnType<Database['prepare']>;\n readonly #indexStmt: ReturnType<Database['prepare']>;\n\n /\n Creates a new fanout calculator.\n \n @param db Database instance\n * @param defaultFanout Default fanout when statistics unavailable (default: 3)\n * - 1: Conservative (assumes FK relationships)\n * - 3: Moderate (recommended, safe middle ground)\n * - 10: SQLite's default (optimistic)\n /\n constructor(db: Database, defaultFanout = 3) {\n this.#db = db;\n this.#defaultFanout = defaultFanout;\n\n // Prepare SQL statements once for reuse across multiple getFanout() calls\n this.#stat4Stmt = this.#db.prepare(`\n SELECT neq, nlt, ndlt, sample\n FROM sqlite_stat4\n WHERE tbl = ? AND idx = ?\n ORDER BY nlt\n `);\n\n this.#stat1Stmt = this.#db.prepare(`\n SELECT stat\n FROM sqlite_stat1\n WHERE tbl = ? AND idx = ?\n `);\n\n this.#indexStmt = this.#db.prepare(`\n SELECT il.name as index_name, ii.seqno, ii.name as column_name\n FROM pragma_index_list(?) il\n JOIN pragma_index_info(il.name) ii\n WHERE il.partial = 0\n ORDER BY il.seq, ii.seqno\n `);\n }\n\n /\n Gets the fanout factor for join column(s).\n \n Fanout = average number of child rows per distinct parent key value(s).\n \n ## Strategy\n \n 1. Try sqlite_stat4 (best): Histogram with separate NULL/non-NULL samples\n * 2. Fallback to sqlite_stat1: Average across all rows (includes NULLs)\n * 3. Fallback to default: When no statistics available\n \n ## Compound Indexes\n \n For multi-column joins, finds indexes where ALL columns appear as an\n * exact prefix. Uses the appropriate depth in stat1/stat4.\n \n Example:\n * - Columns: `['customerId', 'storeId']`\n * - Matches index: `(customerId, storeId, date)` at depth 2\n * - Uses stat1 parts[2] or stat4 neq[1] for accurate fanout\n \n ## Caching\n \n Results are cached per (table, columns) combination. Clear the cache if\n * you run ANALYZE to update statistics.\n \n @param tableName Table containing the join column(s)\n * @param columns Array of column names (one or more columns)\n * @returns Fanout result with value and source\n /\n getFanout(tableName: string, columns: string[]): FanoutResult {\n // Cache key uses sorted columns for consistency\n const cacheKey = `${tableName}:${columns.toSorted().join(',')}`;\n const cached = this.#cache.get(cacheKey);\n if (cached) {\n return cached;\n }\n\n // Strategy 1: Try stat4 first (most accurate)\n // NOTE: columns are NOT sorted - preserves Object.keys() order from constraint\n // Matching is order-independent (flexible), but we keep original order for consistency\n const stat4Result = this.#getFanoutFromStat4(tableName, columns);\n if (stat4Result) {\n this.#cache.set(cacheKey, stat4Result);\n return stat4Result;\n }\n\n // Strategy 2: Fallback to stat1 (includes NULLs)\n const stat1Result = this.#getFanoutFromStat1(tableName, columns);\n if (stat1Result) {\n this.#cache.set(cacheKey, stat1Result);\n return stat1Result;\n }\n\n // Strategy 3: Use default\n const defaultResult: FanoutResult = {\n fanout: this.#defaultFanout,\n confidence: 'none',\n source: 'default',\n };\n this.#cache.set(cacheKey, defaultResult);\n return defaultResult;\n }\n\n /\n Clears the fanout cache.\n * Call this after running ANALYZE to pick up updated statistics.\n /\n clearCache(): void {\n this.#cache.clear();\n }\n\n /\n Gets fanout from sqlite_stat4 histogram.\n \n Queries stat4 samples, decodes to identify NULLs, and returns\n * the median fanout of non-NULL samples.\n \n For compound indexes, uses the neq value at the appropriate depth.\n \n @param columns Array of column names to get fanout for\n * @returns Fanout result or undefined if stat4 unavailable\n /\n #getFanoutFromStat4(\n tableName: string,\n columns: string[],\n ): FanoutResult \| undefined {\n try {\n // Find index containing the columns as a prefix\n const indexInfo = this.#findIndexForColumns(tableName, columns);\n if (!indexInfo) {\n return undefined;\n }\n\n // Query stat4 samples for this index (using prepared statement)\n const samples = this.#stat4Stmt.all(\n tableName,\n indexInfo.indexName,\n ) as Stat4Sample[];\n\n if (samples.length === 0) {\n return undefined;\n }\n\n // Decode samples and separate NULL from non-NULL\n // Use depth-1 for neq array index (depth is 1-based, array is 0-based)\n const neqIndex = indexInfo.depth - 1;\n const decodedSamples = samples.map(s => {\n const neqParts = s.neq.split(' ');\n return {\n fanout: parseInt(neqParts[neqIndex] ?? neqParts[0], 10),\n isNull: isSampleNull(s.sample),\n };\n });\n\n const nonNullSamples = decodedSamples.filter(s => !s.isNull);\n\n if (nonNullSamples.length === 0) {\n // All samples are NULL - return fanout of 0 since NULLs don't match in joins\n return {\n fanout: 0,\n source: 'stat4',\n confidence: 'high',\n };\n }\n\n // Use median of non-NULL fanouts (more robust than average)\n const fanouts = nonNullSamples.map(s => s.fanout).sort((a, b) => a - b);\n const medianFanout =\n fanouts.length % 2 === 0\n ? Math.floor(\n (fanouts[fanouts.length / 2 - 1] + fanouts[fanouts.length / 2]) /\n 2,\n )\n : fanouts[Math.floor(fanouts.length / 2)];\n\n return {\n fanout: medianFanout,\n source: 'stat4',\n confidence: 'high',\n };\n } catch {\n // stat4 table may not exist or query may fail\n return undefined;\n }\n }\n\n /\n Gets fanout from sqlite_stat1 average.\n \n Note: This includes NULL rows in the calculation and may overestimate\n * fanout for sparse foreign keys.\n \n For compound indexes, uses the stat value at the appropriate depth.\n \n @param columns Array of column names to get fanout for\n * @returns Fanout result or undefined if stat1 unavailable\n /\n #getFanoutFromStat1(\n tableName: string,\n columns: string[],\n ): FanoutResult \| undefined {\n try {\n // Find index containing the columns as a prefix\n const indexInfo = this.#findIndexForColumns(tableName, columns);\n if (!indexInfo) {\n return undefined;\n }\n\n // Query stat1 for this index (using prepared statement)\n const result = this.#stat1Stmt.get(tableName, indexInfo.indexName) as\n \| {stat: string}\n \| undefined;\n\n if (!result) {\n return undefined;\n }\n\n const parts = result.stat.split(' ');\n // Check if we have enough parts for the requested depth\n if (parts.length < indexInfo.depth + 1) {\n return undefined;\n }\n\n const fanout = parseInt(parts[indexInfo.depth], 10);\n if (isNaN(fanout)) {\n return undefined;\n }\n\n return {\n fanout,\n source: 'stat1',\n confidence: 'med',\n };\n } catch {\n return undefined;\n }\n }\n\n /\n Finds an index that can be used to get statistics for column(s).\n \n Uses pragma_index_list and pragma_index_info to reliably get index\n * column names, avoiding brittle SQL parsing. Includes all indices:\n * user-created (CREATE INDEX), PRIMARY KEY, and UNIQUE constraints.\n \n Uses flexible matching: Finds indexes where ALL columns appear in the\n * first N positions, regardless of order. This works because SQLite statistics\n * at depth N represent the fanout for the combination of the first N columns,\n * and combinations are order-independent.\n \n Example:\n * - columns: ['customerId', 'storeId']\n * - Matches: (customerId, storeId, date) at depth 2 ✅\n * - Matches: (storeId, customerId, date) at depth 2 ✅ (flexible order)\n * - Does NOT match: (date, customerId, storeId) ❌ (columns not in first 2 positions)\n * - Does NOT match: (customerId, date, storeId) ❌ (storeId not in first 2 positions)\n \n @param columns Array of column names (order-independent for matching)\n * @returns Index info with name and depth, or undefined if no match\n /\n #findIndexForColumns(\n tableName: string,\n columns: string[],\n ): {indexName: string; depth: number} \| undefined {\n try {\n // Query returns all columns for all indexes (including PK/UNIQUE) in order\n const rows = this.#indexStmt.all(tableName) as {\n index_name: string;\n seqno: number;\n column_name: string;\n }[];\n\n // Group by index name\n const indexMap = new Map<string, string[]>();\n for (const row of rows) {\n getOrInsertComputed(indexMap, row.index_name, newArray).push(\n row.column_name,\n );\n }\n\n // Check each index for prefix match\n for (const [indexName, indexColumns] of indexMap) {\n if (this.#isPrefixMatch(columns, indexColumns)) {\n return {\n indexName,\n depth: columns.length,\n };\n }\n }\n\n return undefined;\n } catch {\n return undefined;\n }\n }\n\n /\n Checks if all queryColumns exist in the first N positions of indexColumns,\n * regardless of order.\n \n This allows flexible matching: constraint {a, b} matches both index (a, b, c)\n * and index (b, a, c) at depth 2, since both represent the fanout for the\n * combination of columns a and b.\n \n Gaps are NOT allowed: constraint {a, c} does NOT match index (a, b, c)\n * because no depth represents just (a, c) without b. Statistics are cumulative\n * from position 0.\n \n @param queryColumns Columns we're looking for (from constraint)\n * @param indexColumns Columns in the index (in order)\n * @returns true if all queryColumns exist in indexColumns[0...queryColumns.length-1]\n */\n #isPrefixMatch(queryColumns: string[], indexColumns: string[]): boolean {\n if (queryColumns.length > indexColumns.length) {\n return false;\n }\n\n // Get the prefix of the index that we're checking against\n const indexPrefix = indexColumns.slice(0, queryColumns.length);\n\n // Normalize to lowercase for case-insensitive comparison\n const indexPrefixLower = new Set(indexPrefix.map(col => col.toLowerCase()));\n const queryColumnsLower = queryColumns.map(col => col.toLowerCase());\n\n // Check if ALL query columns exist in the index prefix\n return queryColumnsLower.every(queryCol => indexPrefixLower.has(queryCol));\n }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AA2FA,IAAa,mBAAb,MAA8B;CAC5B;CACA;;;;;CAMA,yBAAkB,IAAI,IAA0B;;;;;CAMhD;CACA;CACA;;;;;;;;;;CAWA,YAAY,IAAc,gBAAgB,GAAG;EAC3C,KAAKA,MAAM;EACX,KAAKC,iBAAiB;EAGtB,KAAKE,aAAa,KAAKH,IAAI,QAAQ;;;;;KAKlC;EAED,KAAKI,aAAa,KAAKJ,IAAI,QAAQ;;;;KAIlC;EAED,KAAKK,aAAa,KAAKL,IAAI,QAAQ;;;;;;KAMlC;CACH;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAgCA,UAAU,WAAmB,SAAiC;EAE5D,MAAM,WAAW,GAAG,UAAU,GAAG,QAAQ,SAAS,EAAE,KAAK,GAAG;EAC5D,MAAM,SAAS,KAAKE,OAAO,IAAI,QAAQ;EACvC,IAAI,QACF,OAAO;EAMT,MAAM,cAAc,KAAKI,oBAAoB,WAAW,OAAO;EAC/D,IAAI,aAAa;GACf,KAAKJ,OAAO,IAAI,UAAU,WAAW;GACrC,OAAO;EACT;EAGA,MAAM,cAAc,KAAKK,oBAAoB,WAAW,OAAO;EAC/D,IAAI,aAAa;GACf,KAAKL,OAAO,IAAI,UAAU,WAAW;GACrC,OAAO;EACT;EAGA,MAAM,gBAA8B;GAClC,QAAQ,KAAKD;GACb,YAAY;GACZ,QAAQ;EACV;EACA,KAAKC,OAAO,IAAI,UAAU,aAAa;EACvC,OAAO;CACT;;;;;CAMA,aAAmB;EACjB,KAAKA,OAAO,MAAM;CACpB;;;;;;;;;;;;CAaA,oBACE,WACA,SAC0B;EAC1B,IAAI;GAEF,MAAM,YAAY,KAAKM,qBAAqB,WAAW,OAAO;GAC9D,IAAI,CAAC,WACH;GAIF,MAAM,UAAU,KAAKL,WAAW,IAC9B,WACA,UAAU,SACZ;GAEA,IAAI,QAAQ,WAAW,GACrB;GAKF,MAAM,WAAW,UAAU,QAAQ;GASnC,MAAM,iBARiB,QAAQ,KAAI,MAAK;IACtC,MAAM,WAAW,EAAE,IAAI,MAAM,GAAG;IAChC,OAAO;KACL,QAAQ,SAAS,SAAS,aAAa,SAAS,IAAI,EAAE;KACtD,QAAQ,aAAa,EAAE,MAAM;IAC/B;GACF,CAEuB,EAAe,QAAO,MAAK,CAAC,EAAE,MAAM;GAE3D,IAAI,eAAe,WAAW,GAE5B,OAAO;IACL,QAAQ;IACR,QAAQ;IACR,YAAY;GACd;GAIF,MAAM,UAAU,eAAe,KAAI,MAAK,EAAE,MAAM,EAAE,MAAM,GAAG,MAAM,IAAI,CAAC;GAStE,OAAO;IACL,QARA,QAAQ,SAAS,MAAM,IACnB,KAAK,OACF,QAAQ,QAAQ,SAAS,IAAI,KAAK,QAAQ,QAAQ,SAAS,MAC1D,CACJ,IACA,QAAQ,KAAK,MAAM,QAAQ,SAAS,CAAC;IAIzC,QAAQ;IACR,YAAY;GACd;EACF,QAAQ;GAEN;EACF;CACF;;;;;;;;;;;;CAaA,oBACE,WACA,SAC0B;EAC1B,IAAI;GAEF,MAAM,YAAY,KAAKK,qBAAqB,WAAW,OAAO;GAC9D,IAAI,CAAC,WACH;GAIF,MAAM,SAAS,KAAKJ,WAAW,IAAI,WAAW,UAAU,SAAS;GAIjE,IAAI,CAAC,QACH;GAGF,MAAM,QAAQ,OAAO,KAAK,MAAM,GAAG;GAEnC,IAAI,MAAM,SAAS,UAAU,QAAQ,GACnC;GAGF,MAAM,SAAS,SAAS,MAAM,UAAU,QAAQ,EAAE;GAClD,IAAI,MAAM,MAAM,GACd;GAGF,OAAO;IACL;IACA,QAAQ;IACR,YAAY;GACd;EACF,QAAQ;GACN;EACF;CACF;;;;;;;;;;;;;;;;;;;;;;;CAwBA,qBACE,WACA,SACgD;EAChD,IAAI;GAEF,MAAM,OAAO,KAAKC,WAAW,IAAI,SAAS;GAO1C,MAAM,2BAAW,IAAI,IAAsB;GAC3C,KAAK,MAAM,OAAO,MAChB,oBAAoB,UAAU,IAAI,YAAY,QAAQ,EAAE,KACtD,IAAI,WACN;GAIF,KAAK,MAAM,CAAC,WAAW,iBAAiB,UACtC,IAAI,KAAKI,eAAe,SAAS,YAAY,GAC3C,OAAO;IACL;IACA,OAAO,QAAQ;GACjB;GAIJ;EACF,QAAQ;GACN;EACF;CACF;;;;;;;;;;;;;;;;;CAkBA,eAAe,cAAwB,cAAiC;EACtE,IAAI,aAAa,SAAS,aAAa,QACrC,OAAO;EAIT,MAAM,cAAc,aAAa,MAAM,GAAG,aAAa,MAAM;EAG7D,MAAM,mBAAmB,IAAI,IAAI,YAAY,KAAI,QAAO,IAAI,YAAY,CAAC,CAAC;EAI1E,OAH0B,aAAa,KAAI,QAAO,IAAI,YAAY,CAG3D,EAAkB,OAAM,aAAY,iBAAiB,IAAI,QAAQ,CAAC;CAC3E;AACF"}

package/out/zqlite/src/sqlite-stat4-sample.d.ts ADDED Viewed

@@ -0,0 +1,37 @@
+/**
+ * Decoding of `sqlite_stat4.sample` values.
+ *
+ * A sample is a SQLite record: a header holding one serial type per column,
+ * followed by the column data. We only ever look at the serial types, which
+ * say what *kind* of value each column holds without revealing the value
+ * itself. Two callers need this:
+ *
+ * - `SQLiteStatFanout`, which separates NULL from non-NULL samples so that
+ *   join fanout is not skewed by a sparse foreign key.
+ * - the `/plannerz` admin endpoint, which reports the shape of the histogram
+ *   without returning row data.
+ *
+ * @see https://sqlite.org/fileformat2.html#record_format
+ * @see https://sqlite.org/fileformat2.html#stat4tab
+ */
+/**
+ * The kind of value a stat4 sample column holds, derived from its serial type.
+ * `unknown` covers the internal serial types 10 and 11, which do not appear in
+ * a well-formed record.
+ */
+export type SampleValueKind = 'null' | 'integer' | 'real' | 'text' | 'blob' | 'unknown';
+/**
+ * Decodes the serial types in a stat4 sample's header.
+ *
+ * The sample's trailing rowid is included as a final column, matching the
+ * record SQLite stores.
+ *
+ * @returns one kind per column, or an empty array if `sample` is empty or its
+ *          header is malformed.
+ */
+export declare function decodeSampleKinds(sample: Buffer): SampleValueKind[];
+/**
+ * Whether the first column of a sample is NULL.
+ */
+export declare function isSampleNull(sample: Buffer): boolean;
+//# sourceMappingURL=sqlite-stat4-sample.d.ts.map

package/out/zqlite/src/sqlite-stat4-sample.d.ts.map ADDED Viewed

@@ -0,0 +1 @@

+ {"version":3,"file":"sqlite-stat4-sample.d.ts","sourceRoot":"","sources":["../../../../zqlite/src/sqlite-stat4-sample.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;GAeG;AAEH;;;;GAIG;AACH,MAAM,MAAM,eAAe,GACvB,MAAM,GACN,SAAS,GACT,MAAM,GACN,MAAM,GACN,MAAM,GACN,SAAS,CAAC;AAEd;;;;;;;;GAQG;AACH,wBAAgB,iBAAiB,CAAC,MAAM,EAAE,MAAM,GAAG,eAAe,EAAE,CAuBnE;AAED;;GAEG;AACH,wBAAgB,YAAY,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAGpD"}

package/out/zqlite/src/sqlite-stat4-sample.js ADDED Viewed

@@ -0,0 +1,61 @@
+//#region ../zqlite/src/sqlite-stat4-sample.ts
+/**
+* Decodes the serial types in a stat4 sample's header.
+*
+* The sample's trailing rowid is included as a final column, matching the
+* record SQLite stores.
+*
+* @returns one kind per column, or an empty array if `sample` is empty or its
+*          header is malformed.
+*/
+function decodeSampleKinds(sample) {
+	const header = readVarint(sample, 0);
+	if (header === void 0) return [];
+	const [headerSize, serialTypesStart] = header;
+	if (headerSize < serialTypesStart || headerSize > sample.length) return [];
+	const kinds = [];
+	for (let pos = serialTypesStart; pos < headerSize;) {
+		const next = readVarint(sample, pos);
+		if (next === void 0) return [];
+		const [serialType, end] = next;
+		kinds.push(kindOfSerialType(serialType));
+		pos = end;
+	}
+	return kinds;
+}
+/**
+* Whether the first column of a sample is NULL.
+*/
+function isSampleNull(sample) {
+	const kinds = decodeSampleKinds(sample);
+	return kinds.length === 0 || kinds[0] === "null";
+}
+function kindOfSerialType(serialType) {
+	if (serialType === 0) return "null";
+	if (serialType <= 6 || serialType === 8 || serialType === 9) return "integer";
+	if (serialType === 7) return "real";
+	if (serialType <= 11) return "unknown";
+	return serialType % 2 === 0 ? "blob" : "text";
+}
+/**
+* Reads the varint at `pos`.
+*
+* @returns the value and the offset just past it, or `undefined` if the varint
+*          runs off the end of the buffer. Values are read as `number`; a
+*          varint large enough to lose precision is not something a well-formed
+*          header or serial type contains.
+*/
+function readVarint(buf, pos) {
+	let value = 0;
+	for (let i = 0; i < 9; i++) {
+		if (pos + i >= buf.length) return;
+		const byte = buf[pos + i];
+		if (i === 8) return [value * 256 + byte, pos + 9];
+		value = value * 128 + (byte & 127);
+		if ((byte & 128) === 0) return [value, pos + i + 1];
+	}
+}
+//#endregion
+export { decodeSampleKinds, isSampleNull };
+//# sourceMappingURL=sqlite-stat4-sample.js.map

package/out/zqlite/src/sqlite-stat4-sample.js.map ADDED Viewed

@@ -0,0 +1 @@

+ {"version":3,"file":"sqlite-stat4-sample.js","names":[],"sources":["../../../../zqlite/src/sqlite-stat4-sample.ts"],"sourcesContent":["/**\n * Decoding of `sqlite_stat4.sample` values.\n *\n * A sample is a SQLite record: a header holding one serial type per column,\n * followed by the column data. We only ever look at the serial types, which\n * say what *kind* of value each column holds without revealing the value\n * itself. Two callers need this:\n *\n * - `SQLiteStatFanout`, which separates NULL from non-NULL samples so that\n * join fanout is not skewed by a sparse foreign key.\n * - the `/plannerz` admin endpoint, which reports the shape of the histogram\n * without returning row data.\n *\n * @see https://sqlite.org/fileformat2.html#record_format\n * @see https://sqlite.org/fileformat2.html#stat4tab\n */\n\n/**\n * The kind of value a stat4 sample column holds, derived from its serial type.\n * `unknown` covers the internal serial types 10 and 11, which do not appear in\n * a well-formed record.\n */\nexport type SampleValueKind =\n | 'null'\n | 'integer'\n | 'real'\n | 'text'\n | 'blob'\n | 'unknown';\n\n/**\n * Decodes the serial types in a stat4 sample's header.\n *\n * The sample's trailing rowid is included as a final column, matching the\n * record SQLite stores.\n *\n * @returns one kind per column, or an empty array if `sample` is empty or its\n * header is malformed.\n */\nexport function decodeSampleKinds(sample: Buffer): SampleValueKind[] {\n const header = readVarint(sample, 0);\n if (header === undefined) {\n return [];\n }\n // `headerSize` counts the size varint itself, so the serial types run from\n // the end of that varint to `headerSize`, and cannot exceed the record.\n const [headerSize, serialTypesStart] = header;\n if (headerSize < serialTypesStart || headerSize > sample.length) {\n return [];\n }\n\n const kinds: SampleValueKind[] = [];\n for (let pos = serialTypesStart; pos < headerSize;) {\n const next = readVarint(sample, pos);\n if (next === undefined) {\n return [];\n }\n const [serialType, end] = next;\n kinds.push(kindOfSerialType(serialType));\n pos = end;\n }\n return kinds;\n}\n\n/**\n * Whether the first column of a sample is NULL.\n */\nexport function isSampleNull(sample: Buffer): boolean {\n const kinds = decodeSampleKinds(sample);\n return kinds.length === 0 || kinds[0] === 'null';\n}\n\nfunction kindOfSerialType(serialType: number): SampleValueKind {\n if (serialType === 0) {\n return 'null';\n }\n if (serialType <= 6 || serialType === 8 || serialType === 9) {\n return 'integer';\n }\n if (serialType === 7) {\n return 'real';\n }\n if (serialType <= 11) {\n return 'unknown'; // 10 and 11 are for internal use.\n }\n return serialType % 2 === 0 ? 'blob' : 'text';\n}\n\n/**\n * Reads the varint at `pos`.\n *\n * @returns the value and the offset just past it, or `undefined` if the varint\n * runs off the end of the buffer. Values are read as `number`; a\n * varint large enough to lose precision is not something a well-formed\n * header or serial type contains.\n */\nfunction readVarint(\n buf: Buffer,\n pos: number,\n): [value: number, end: number] | undefined {\n let value = 0;\n // A varint is at most 9 bytes, the last of which contributes all 8 bits.\n for (let i = 0; i < 9; i++) {\n if (pos + i >= buf.length) {\n return undefined;\n }\n const byte = buf[pos + i];\n if (i === 8) {\n return [value * 256 + byte, pos + 9];\n }\n value = value * 128 + (byte & 0x7f);\n if ((byte & 0x80) === 0) {\n return [value, pos + i + 1];\n }\n }\n return undefined;\n}\n"],"mappings":";;;;;;;;;;AAuCA,SAAgB,kBAAkB,QAAmC;CACnE,MAAM,SAAS,WAAW,QAAQ,CAAC;CACnC,IAAI,WAAW,KAAA,GACb,OAAO,CAAC;CAIV,MAAM,CAAC,YAAY,oBAAoB;CACvC,IAAI,aAAa,oBAAoB,aAAa,OAAO,QACvD,OAAO,CAAC;CAGV,MAAM,QAA2B,CAAC;CAClC,KAAK,IAAI,MAAM,kBAAkB,MAAM,aAAa;EAClD,MAAM,OAAO,WAAW,QAAQ,GAAG;EACnC,IAAI,SAAS,KAAA,GACX,OAAO,CAAC;EAEV,MAAM,CAAC,YAAY,OAAO;EAC1B,MAAM,KAAK,iBAAiB,UAAU,CAAC;EACvC,MAAM;CACR;CACA,OAAO;AACT;;;;AAKA,SAAgB,aAAa,QAAyB;CACpD,MAAM,QAAQ,kBAAkB,MAAM;CACtC,OAAO,MAAM,WAAW,KAAK,MAAM,OAAO;AAC5C;AAEA,SAAS,iBAAiB,YAAqC;CAC7D,IAAI,eAAe,GACjB,OAAO;CAET,IAAI,cAAc,KAAK,eAAe,KAAK,eAAe,GACxD,OAAO;CAET,IAAI,eAAe,GACjB,OAAO;CAET,IAAI,cAAc,IAChB,OAAO;CAET,OAAO,aAAa,MAAM,IAAI,SAAS;AACzC;;;;;;;;;AAUA,SAAS,WACP,KACA,KAC0C;CAC1C,IAAI,QAAQ;CAEZ,KAAK,IAAI,IAAI,GAAG,IAAI,GAAG,KAAK;EAC1B,IAAI,MAAM,KAAK,IAAI,QACjB;EAEF,MAAM,OAAO,IAAI,MAAM;EACvB,IAAI,MAAM,GACR,OAAO,CAAC,QAAQ,MAAM,MAAM,MAAM,CAAC;EAErC,QAAQ,QAAQ,OAAO,OAAO;EAC9B,KAAK,OAAO,SAAU,GACpB,OAAO,CAAC,OAAO,MAAM,IAAI,CAAC;CAE9B;AAEF"}

package/out/zqlite/src/table-source.d.ts CHANGED Viewed

@@ -19,6 +19,13 @@ export type TableSourceOptions = {
      * pipeline driver does when it advances to the next snapshot.
      */
     skipUnobservableChanges?: boolean | undefined;
+    /**
+     * When set, pushed changes are held in an in-memory {@link PendingDelta}
+     * merged into every read instead of being written to the backing table.
+     * The delta is dropped when the source moves to the next snapshot. This is
+     * the initial mode; see {@link TableSource.setDeferWrites}.
+     */
+    deferWrites?: boolean | undefined;
 };
 /**
  * A source that is backed by a SQLite table.
@@ -43,6 +50,11 @@ export declare class TableSource implements Source {
      * also throw an error to abort the pipeline processing.
      */
     constructor(logContext: LogContext, logConfig: LogConfig, db: Database, tableName: string, columns: Record<string, SchemaValue>, primaryKey: PrimaryKey, shouldYield?: () => boolean, options?: TableSourceOptions);
+    /**
+     * The number of rows read from SQLite by fetches, over the lifetime of this
+     * source. Rows that SQLite scans and filters out itself are not counted.
+     */
+    get rowsRead(): number;
     get tableSchema(): {
         name: string;
         columns: Record<string, SchemaValue>;
@@ -53,11 +65,67 @@ export declare class TableSource implements Source {
      * algorithm for concurrent traversal of historic timelines.
      */
     setDB(db: Database): void;
+    /**
+     * Sets whether the changes pushed from now on are held in memory (see
+     * {@link TableSourceOptions.deferWrites}) or written to the backing
+     * snapshot. The two must not be mixed within one snapshot, so this may only
+     * be called before the first push after construction or {@link setDB}. To
+     * switch to writing through partway, see {@link writePendingChanges}.
+     */
+    setDeferWrites(deferWrites: boolean): void;
+    /**
+     * Drops the changes held in memory (see {@link setDeferWrites}), for an
+     * advancement that has been abandoned. The source then no longer reflects
+     * the changes pushed to it, so it must not be read again until it has moved
+     * to another snapshot ({@link setDB}), if it is used again at all.
+     */
+    discardPendingChanges(): void;
+    /**
+     * Writes the changes held in memory (see {@link setDeferWrites}) to the
+     * backing snapshot, which is left as if they had been written through (in
+     * the source's columns), and writes the changes pushed from then on through
+     * to it. Yields when the
+     * source's `shouldYield` says to, and must not be interleaved with a push.
+     */
+    writePendingChanges(): Stream<'yield'>;
     hasConnections(): boolean;
     connect(sort: Ordering | undefined, filters?: Condition, splitEditKeys?: Set<string>, debug?: DebugDelegate): SourceInput;
     toSQLiteRow(row: Row): Row;
     push(change: SourceChange): Stream<'yield'>;
+    /**
+     * The number of rows held in memory for changes that have been pushed but
+     * not written, i.e. with `deferWrites`, since the last {@link setDB}.
+     */
+    get pendingRows(): number;
+    /** Estimated bytes retained by deferred changes and their indexes. */
+    get pendingBytes(): number;
     genPush(change: SourceChange): Stream<'yield' | undefined>;
+    /**
+     * Reconciles a set of rows that the caller found in the *backing snapshot*
+     * with the changes this source has applied but not written.
+     *
+     * A view-syncer computes the rows a replicated change collides with -- by
+     * change-log key for a delete, by every unique key for a set -- against the
+     * snapshot it is reading. When derivation writes through, that snapshot has
+     * already absorbed the earlier changes of the same advancement, so the
+     * probe sees them. When derivation is deferred they live here instead, and
+     * the probe has to be brought up to date or the source is pushed a change
+     * that contradicts its own state (a delete of a row the batch already
+     * removed, an edit whose old row is stale).
+     *
+     * Two ways the batch can disagree with the probe, both handled here: a
+     * probed row may have been removed, or may have been edited (so the
+     * caller's copy is stale, or it no longer collides at all).
+     *
+     * The batch cannot have edited a row the probe missed *into* collision. The
+     * change log keeps one entry per row key, so every row the batch holds is
+     * that row's value at the advancement's target version, as is `next`, and
+     * two distinct rows of one version cannot share a non-null unique key.
+     *
+     * Returns `base` untouched when derivation is not deferred, which is what
+     * makes this a no-op for the write-through path.
+     */
+    reconcilePendingConflicts(base: readonly Row[], next: Row | null, rowKey: Row, uniqueKeys: readonly (readonly string[])[]): readonly Row[];
     /**
      * Retrieves a row from the backing DB by a unique key, or `undefined` if such a
      * row does not exist. This is not used in the IVM pipeline but is useful

package/out/zqlite/src/table-source.d.ts.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"table-source.d.ts","sourceRoot":"","sources":["../../../../zqlite/src/table-source.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAEjD,OAAO,KAAK,EAAC,SAAS,EAAC,MAAM,+BAA+B,CAAC;AAM7D,OAAO,KAAK,EAAC,SAAS,EAAE,QAAQ,EAAC,MAAM,gCAAgC,CAAC;AACxE,OAAO,KAAK,EAAC,GAAG,EAAQ,MAAM,iCAAiC,CAAC;AAChE,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,wCAAwC,CAAC;AACvE,OAAO,KAAK,EACV,WAAW,EACX,SAAS,EACV,MAAM,uCAAuC,CAAC;AAC/C,OAAO,KAAK,EAAC,aAAa,EAAC,MAAM,yCAAyC,CAAC;~~AAoB3E~~,OAAO,EACL,KAAK,MAAM,EACX,KAAK,YAAY,EACjB,KAAK,WAAW,EACjB,MAAM,6BAA6B,CAAC;AACrC,OAAO,KAAK,EAAC,MAAM,EAAC,MAAM,6BAA6B,CAAC;AAExD,OAAO,KAAK,EAAC,QAAQ,EAAY,MAAM,SAAS,CAAC;~~AAoBjD~~,MAAM,MAAM,kBAAkB,GAAG;IAC/B;;;;;;;;OAQG;IACH,uBAAuB,CAAC,EAAE,OAAO,GAAG,SAAS,CAAC;~~CAC/C~~,CAAC;AAEF;;;;;;;;;;;;;GAaG;AACH,qBAAa,WAAY,YAAW,MAAM;;~~IAmBxC~~;;;;;OAKG;IACH,YACE,UAAU,EAAE,UAAU,EACtB,SAAS,EAAE,SAAS,EACpB,EAAE,EAAE,QAAQ,EACZ,SAAS,EAAE,MAAM,EACjB,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,WAAW,CAAC,EACpC,UAAU,EAAE,UAAU,EACtB,WAAW,gBAAc,EACzB,OAAO,GAAE,kBAAuB,~~EAiBjC~~;IAED,IAAI,WAAW;QAEX,IAAI;QACJ,OAAO;QACP,UAAU;MAEb;IAED;;;OAGG;IACH,KAAK,CAAC,EAAE,EAAE,QAAQ,~~QAEjB~~;IA0FD,cAAc,IAAI,OAAO,CAExB;IAED,OAAO,CACL,IAAI,EAAE,QAAQ,GAAG,SAAS,EAC1B,OAAO,CAAC,EAAE,SAAS,EACnB,aAAa,CAAC,EAAE,GAAG,CAAC,MAAM,CAAC,EAC3B,KAAK,CAAC,EAAE,aAAa,eA8CtB;IAED,WAAW,CAAC,GAAG,EAAE,GAAG,GAAG,GAAG,CAOzB;~~IAkIA~~,IAAI,CAAC,MAAM,EAAE,YAAY,GAAG,MAAM,CAAC,OAAO,CAAC,CAM3C;IAEA,OAAO,CAAC,MAAM,EAAE,YAAY,GAAG,MAAM,CAAC,OAAO,GAAG,SAAS,CAAC,~~CA4B1D~~;~~IAkFD~~;;;;;;OAMG;IACH,MAAM,CAAC,MAAM,EAAE,GAAG,GAAG,GAAG,GAAG,SAAS,~~CAanC~~;CAmBF;~~AA4CD~~,wBAAgB,aAAa,CAC3B,OAAO,EAAE,SAAS,MAAM,EAAE,EAC1B,GAAG,EAAE,GAAG,EACR,WAAW,EAAE,MAAM,CAAC,MAAM,EAAE,WAAW,CAAC,GACvC,SAAS,OAAO,EAAE,CAEpB;AAED,wBAAgB,gBAAgB,CAAC,IAAI,EAAE,SAAS,wCAa/C;AAED,wBAAgB,eAAe,CAC7B,UAAU,EAAE,MAAM,CAAC,MAAM,EAAE,WAAW,CAAC,EACvC,GAAG,EAAE,GAAG,EACR,SAAS,EAAE,MAAM,GAChB,GAAG,CAaL;AAyCD,qBAAa,qBAAsB,SAAQ,KAAK;CAAG"}
1	+ {"version":3,"file":"table-source.d.ts","sourceRoot":"","sources":["../../../../zqlite/src/table-source.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAEjD,OAAO,KAAK,EAAC,SAAS,EAAC,MAAM,+BAA+B,CAAC;AAM7D,OAAO,KAAK,EAAC,SAAS,EAAE,QAAQ,EAAC,MAAM,gCAAgC,CAAC;AACxE,OAAO,KAAK,EAAC,GAAG,EAAQ,MAAM,iCAAiC,CAAC;AAChE,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,wCAAwC,CAAC;AACvE,OAAO,KAAK,EACV,WAAW,EACX,SAAS,EACV,MAAM,uCAAuC,CAAC;AAC/C,OAAO,KAAK,EAAC,aAAa,EAAC,MAAM,yCAAyC,CAAC;AAwB3E,OAAO,EACL,KAAK,MAAM,EACX,KAAK,YAAY,EACjB,KAAK,WAAW,EACjB,MAAM,6BAA6B,CAAC;AACrC,OAAO,KAAK,EAAC,MAAM,EAAC,MAAM,6BAA6B,CAAC;AAExD,OAAO,KAAK,EAAC,QAAQ,EAAY,MAAM,SAAS,CAAC;AA0BjD,MAAM,MAAM,kBAAkB,GAAG;IAC/B;;;;;;;;OAQG;IACH,uBAAuB,CAAC,EAAE,OAAO,GAAG,SAAS,CAAC;IAE9C;;;;;OAKG;IACH,WAAW,CAAC,EAAE,OAAO,GAAG,SAAS,CAAC;CACnC,CAAC;AAEF;;;;;;;;;;;;;GAaG;AACH,qBAAa,WAAY,YAAW,MAAM;;IA2BxC;;;;;OAKG;IACH,YACE,UAAU,EAAE,UAAU,EACtB,SAAS,EAAE,SAAS,EACpB,EAAE,EAAE,QAAQ,EACZ,SAAS,EAAE,MAAM,EACjB,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,WAAW,CAAC,EACpC,UAAU,EAAE,UAAU,EACtB,WAAW,gBAAc,EACzB,OAAO,GAAE,kBAAuB,EAqBjC;IAED;;;OAGG;IACH,IAAI,QAAQ,IAAI,MAAM,CAErB;IAED,IAAI,WAAW;QAEX,IAAI;QACJ,OAAO;QACP,UAAU;MAEb;IAED;;;OAGG;IACH,KAAK,CAAC,EAAE,EAAE,QAAQ,QAKjB;IAED;;;;;;OAMG;IACH,cAAc,CAAC,WAAW,EAAE,OAAO,QAUlC;IAED;;;;;OAKG;IACH,qBAAqB,IAAI,IAAI,CAE5B;IAED;;;;;;OAMG;IACF,mBAAmB,IAAI,MAAM,CAAC,OAAO,CAAC,CAwBtC;IA0FD,cAAc,IAAI,OAAO,CAExB;IAED,OAAO,CACL,IAAI,EAAE,QAAQ,GAAG,SAAS,EAC1B,OAAO,CAAC,EAAE,SAAS,EACnB,aAAa,CAAC,EAAE,GAAG,CAAC,MAAM,CAAC,EAC3B,KAAK,CAAC,EAAE,aAAa,eA8CtB;IAED,WAAW,CAAC,GAAG,EAAE,GAAG,GAAG,GAAG,CAOzB;IAiLA,IAAI,CAAC,MAAM,EAAE,YAAY,GAAG,MAAM,CAAC,OAAO,CAAC,CAM3C;IAED;;;OAGG;IACH,IAAI,WAAW,IAAI,MAAM,CAExB;IAED,sEAAsE;IACtE,IAAI,YAAY,IAAI,MAAM,CAEzB;IAEA,OAAO,CAAC,MAAM,EAAE,YAAY,GAAG,MAAM,CAAC,OAAO,GAAG,SAAS,CAAC,CAsC1D;IAuED;;;;;;;;;;;;;;;;;;;;;;;;OAwBG;IACH,yBAAyB,CACvB,IAAI,EAAE,SAAS,GAAG,EAAE,EACpB,IAAI,EAAE,GAAG,GAAG,IAAI,EAChB,MAAM,EAAE,GAAG,EACX,UAAU,EAAE,SAAS,CAAC,SAAS,MAAM,EAAE,CAAC,EAAE,GACzC,SAAS,GAAG,EAAE,CAoChB;IA6DD;;;;;;OAMG;IACH,MAAM,CAAC,MAAM,EAAE,GAAG,GAAG,GAAG,GAAG,SAAS,CAgCnC;CAmBF;AAoFD,wBAAgB,aAAa,CAC3B,OAAO,EAAE,SAAS,MAAM,EAAE,EAC1B,GAAG,EAAE,GAAG,EACR,WAAW,EAAE,MAAM,CAAC,MAAM,EAAE,WAAW,CAAC,GACvC,SAAS,OAAO,EAAE,CAEpB;AAED,wBAAgB,gBAAgB,CAAC,IAAI,EAAE,SAAS,wCAa/C;AAED,wBAAgB,eAAe,CAC7B,UAAU,EAAE,MAAM,CAAC,MAAM,EAAE,WAAW,CAAC,EACvC,GAAG,EAAE,GAAG,EACR,SAAS,EAAE,MAAM,GAChB,GAAG,CAaL;AAyCD,qBAAa,qBAAsB,SAAQ,KAAK;CAAG"}