@rocicorp/zero 1.11.0-canary.3 → 1.11.0-canary.30

package/out/zqlite/src/sqlite-stat4-sample.js

@@ -0,0 +1,61 @@
+//#region ../zqlite/src/sqlite-stat4-sample.ts
+/**
+* Decodes the serial types in a stat4 sample's header.
+*
+* The sample's trailing rowid is included as a final column, matching the
+* record SQLite stores.
+*
+* @returns one kind per column, or an empty array if `sample` is empty or its
+*          header is malformed.
+*/
+function decodeSampleKinds(sample) {
+	const header = readVarint(sample, 0);
+	if (header === void 0) return [];
+	const [headerSize, serialTypesStart] = header;
+	if (headerSize < serialTypesStart || headerSize > sample.length) return [];
+	const kinds = [];
+	for (let pos = serialTypesStart; pos < headerSize;) {
+		const next = readVarint(sample, pos);
+		if (next === void 0) return [];
+		const [serialType, end] = next;
+		kinds.push(kindOfSerialType(serialType));
+		pos = end;
+	}
+	return kinds;
+}
+/**
+* Whether the first column of a sample is NULL.
+*/
+function isSampleNull(sample) {
+	const kinds = decodeSampleKinds(sample);
+	return kinds.length === 0 || kinds[0] === "null";
+}
+function kindOfSerialType(serialType) {
+	if (serialType === 0) return "null";
+	if (serialType <= 6 || serialType === 8 || serialType === 9) return "integer";
+	if (serialType === 7) return "real";
+	if (serialType <= 11) return "unknown";
+	return serialType % 2 === 0 ? "blob" : "text";
+}
+/**
+* Reads the varint at `pos`.
+*
+* @returns the value and the offset just past it, or `undefined` if the varint
+*          runs off the end of the buffer. Values are read as `number`; a
+*          varint large enough to lose precision is not something a well-formed
+*          header or serial type contains.
+*/
+function readVarint(buf, pos) {
+	let value = 0;
+	for (let i = 0; i < 9; i++) {
+		if (pos + i >= buf.length) return;
+		const byte = buf[pos + i];
+		if (i === 8) return [value * 256 + byte, pos + 9];
+		value = value * 128 + (byte & 127);
+		if ((byte & 128) === 0) return [value, pos + i + 1];
+	}
+}
+//#endregion
+export { decodeSampleKinds, isSampleNull };
+
+//# sourceMappingURL=sqlite-stat4-sample.js.map

package/out/zqlite/src/sqlite-stat4-sample.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"sqlite-stat4-sample.js","names":[],"sources":["../../../../zqlite/src/sqlite-stat4-sample.ts"],"sourcesContent":["/**\n * Decoding of `sqlite_stat4.sample` values.\n *\n * A sample is a SQLite record: a header holding one serial type per column,\n * followed by the column data. We only ever look at the serial types, which\n * say what *kind* of value each column holds without revealing the value\n * itself. Two callers need this:\n *\n * - `SQLiteStatFanout`, which separates NULL from non-NULL samples so that\n *   join fanout is not skewed by a sparse foreign key.\n * - the `/plannerz` admin endpoint, which reports the shape of the histogram\n *   without returning row data.\n *\n * @see https://sqlite.org/fileformat2.html#record_format\n * @see https://sqlite.org/fileformat2.html#stat4tab\n */\n\n/**\n * The kind of value a stat4 sample column holds, derived from its serial type.\n * `unknown` covers the internal serial types 10 and 11, which do not appear in\n * a well-formed record.\n */\nexport type SampleValueKind =\n  | 'null'\n  | 'integer'\n  | 'real'\n  | 'text'\n  | 'blob'\n  | 'unknown';\n\n/**\n * Decodes the serial types in a stat4 sample's header.\n *\n * The sample's trailing rowid is included as a final column, matching the\n * record SQLite stores.\n *\n * @returns one kind per column, or an empty array if `sample` is empty or its\n *          header is malformed.\n */\nexport function decodeSampleKinds(sample: Buffer): SampleValueKind[] {\n  const header = readVarint(sample, 0);\n  if (header === undefined) {\n    return [];\n  }\n  // `headerSize` counts the size varint itself, so the serial types run from\n  // the end of that varint to `headerSize`, and cannot exceed the record.\n  const [headerSize, serialTypesStart] = header;\n  if (headerSize < serialTypesStart || headerSize > sample.length) {\n    return [];\n  }\n\n  const kinds: SampleValueKind[] = [];\n  for (let pos = serialTypesStart; pos < headerSize;) {\n    const next = readVarint(sample, pos);\n    if (next === undefined) {\n      return [];\n    }\n    const [serialType, end] = next;\n    kinds.push(kindOfSerialType(serialType));\n    pos = end;\n  }\n  return kinds;\n}\n\n/**\n * Whether the first column of a sample is NULL.\n */\nexport function isSampleNull(sample: Buffer): boolean {\n  const kinds = decodeSampleKinds(sample);\n  return kinds.length === 0 || kinds[0] === 'null';\n}\n\nfunction kindOfSerialType(serialType: number): SampleValueKind {\n  if (serialType === 0) {\n    return 'null';\n  }\n  if (serialType <= 6 || serialType === 8 || serialType === 9) {\n    return 'integer';\n  }\n  if (serialType === 7) {\n    return 'real';\n  }\n  if (serialType <= 11) {\n    return 'unknown'; // 10 and 11 are for internal use.\n  }\n  return serialType % 2 === 0 ? 'blob' : 'text';\n}\n\n/**\n * Reads the varint at `pos`.\n *\n * @returns the value and the offset just past it, or `undefined` if the varint\n *          runs off the end of the buffer. Values are read as `number`; a\n *          varint large enough to lose precision is not something a well-formed\n *          header or serial type contains.\n */\nfunction readVarint(\n  buf: Buffer,\n  pos: number,\n): [value: number, end: number] | undefined {\n  let value = 0;\n  // A varint is at most 9 bytes, the last of which contributes all 8 bits.\n  for (let i = 0; i < 9; i++) {\n    if (pos + i >= buf.length) {\n      return undefined;\n    }\n    const byte = buf[pos + i];\n    if (i === 8) {\n      return [value * 256 + byte, pos + 9];\n    }\n    value = value * 128 + (byte & 0x7f);\n    if ((byte & 0x80) === 0) {\n      return [value, pos + i + 1];\n    }\n  }\n  return undefined;\n}\n"],"mappings":";;;;;;;;;;AAuCA,SAAgB,kBAAkB,QAAmC;CACnE,MAAM,SAAS,WAAW,QAAQ,CAAC;CACnC,IAAI,WAAW,KAAA,GACb,OAAO,CAAC;CAIV,MAAM,CAAC,YAAY,oBAAoB;CACvC,IAAI,aAAa,oBAAoB,aAAa,OAAO,QACvD,OAAO,CAAC;CAGV,MAAM,QAA2B,CAAC;CAClC,KAAK,IAAI,MAAM,kBAAkB,MAAM,aAAa;EAClD,MAAM,OAAO,WAAW,QAAQ,GAAG;EACnC,IAAI,SAAS,KAAA,GACX,OAAO,CAAC;EAEV,MAAM,CAAC,YAAY,OAAO;EAC1B,MAAM,KAAK,iBAAiB,UAAU,CAAC;EACvC,MAAM;CACR;CACA,OAAO;AACT;;;;AAKA,SAAgB,aAAa,QAAyB;CACpD,MAAM,QAAQ,kBAAkB,MAAM;CACtC,OAAO,MAAM,WAAW,KAAK,MAAM,OAAO;AAC5C;AAEA,SAAS,iBAAiB,YAAqC;CAC7D,IAAI,eAAe,GACjB,OAAO;CAET,IAAI,cAAc,KAAK,eAAe,KAAK,eAAe,GACxD,OAAO;CAET,IAAI,eAAe,GACjB,OAAO;CAET,IAAI,cAAc,IAChB,OAAO;CAET,OAAO,aAAa,MAAM,IAAI,SAAS;AACzC;;;;;;;;;AAUA,SAAS,WACP,KACA,KAC0C;CAC1C,IAAI,QAAQ;CAEZ,KAAK,IAAI,IAAI,GAAG,IAAI,GAAG,KAAK;EAC1B,IAAI,MAAM,KAAK,IAAI,QACjB;EAEF,MAAM,OAAO,IAAI,MAAM;EACvB,IAAI,MAAM,GACR,OAAO,CAAC,QAAQ,MAAM,MAAM,MAAM,CAAC;EAErC,QAAQ,QAAQ,OAAO,OAAO;EAC9B,KAAK,OAAO,SAAU,GACpB,OAAO,CAAC,OAAO,MAAM,IAAI,CAAC;CAE9B;AAEF"}

package/out/zqlite/src/table-source.d.ts

@@ -8,6 +8,25 @@ import type { DebugDelegate } from '../../zql/src/builder/debug-delegate.ts';
 import { type Source, type SourceChange, type SourceInput } from '../../zql/src/ivm/source.ts';
 import type { Stream } from '../../zql/src/ivm/stream.ts';
 import type { Database } from './db.ts';
+export type TableSourceOptions = {
+    /**
+     * When set, a pushed change that the filters of every connection reject
+     * is neither pushed nor applied to the backing table, on the grounds that
+     * no connection can observe it. This departs from the `Source.push`
+     * contract (a push commits the change to the source), so it is only
+     * appropriate when the caller does not read the table through any other
+     * path and replaces the table's contents afterwards, as the view-syncer's
+     * pipeline driver does when it advances to the next snapshot.
+     */
+    skipUnobservableChanges?: boolean | undefined;
+    /**
+     * When set, pushed changes are held in an in-memory {@link PendingDelta}
+     * merged into every read instead of being written to the backing table.
+     * The delta is dropped when the source moves to the next snapshot. This is
+     * the initial mode; see {@link TableSource.setDeferWrites}.
+     */
+    deferWrites?: boolean | undefined;
+};
 /**
  * A source that is backed by a SQLite table.
  *
@@ -30,7 +49,12 @@ export declare class TableSource implements Source {
      * to yield control back to the caller at the end of the pipeline.  Can
      * also throw an error to abort the pipeline processing.
      */
-    constructor(logContext: LogContext, logConfig: LogConfig, db: Database, tableName: string, columns: Record<string, SchemaValue>, primaryKey: PrimaryKey, shouldYield?: () => boolean);
+    constructor(logContext: LogContext, logConfig: LogConfig, db: Database, tableName: string, columns: Record<string, SchemaValue>, primaryKey: PrimaryKey, shouldYield?: () => boolean, options?: TableSourceOptions);
+    /**
+     * The number of rows read from SQLite by fetches, over the lifetime of this
+     * source. Rows that SQLite scans and filters out itself are not counted.
+     */
+    get rowsRead(): number;
     get tableSchema(): {
         name: string;
         columns: Record<string, SchemaValue>;
@@ -41,10 +65,71 @@ export declare class TableSource implements Source {
      * algorithm for concurrent traversal of historic timelines.
      */
     setDB(db: Database): void;
+    /**
+     * Sets whether the changes pushed from now on are held in memory (see
+     * {@link TableSourceOptions.deferWrites}) or written to the backing
+     * snapshot. The two must not be mixed within one snapshot, so this may only
+     * be called before the first push after construction or {@link setDB}. To
+     * switch to writing through partway, see {@link writePendingChanges}.
+     */
+    setDeferWrites(deferWrites: boolean): void;
+    /**
+     * Drops the changes held in memory (see {@link setDeferWrites}), for an
+     * advancement that has been abandoned. The source then no longer reflects
+     * the changes pushed to it, so it must not be read again until it has moved
+     * to another snapshot ({@link setDB}), if it is used again at all.
+     */
+    discardPendingChanges(): void;
+    /**
+     * Writes the changes held in memory (see {@link setDeferWrites}) to the
+     * backing snapshot, which is left as if they had been written through (in
+     * the source's columns), and writes the changes pushed from then on through
+     * to it. Yields when the
+     * source's `shouldYield` says to, and must not be interleaved with a push.
+     */
+    writePendingChanges(): Stream<'yield'>;
     connect(sort: Ordering | undefined, filters?: Condition, splitEditKeys?: Set<string>, debug?: DebugDelegate): SourceInput;
     toSQLiteRow(row: Row): Row;
     push(change: SourceChange): Stream<'yield'>;
-    genPush(change: SourceChange): Generator<"yield" | undefined, void, unknown>;
+    /**
+     * The number of rows held in memory for changes that have been pushed but
+     * not written, i.e. with `deferWrites`, since the last {@link setDB}.
+     */
+    get pendingRows(): number;
+    /** Estimated bytes retained by deferred changes and their indexes. */
+    get pendingBytes(): number;
+    /**
+     * Returns whether any connected pipeline could observe the given row.
+     * If false, this row is completely invisible to this TableSource's connections.
+     */
+    mayAcceptRow(row: Row): boolean;
+    genPush(change: SourceChange): Stream<'yield' | undefined>;
+    /**
+     * Reconciles a set of rows that the caller found in the *backing snapshot*
+     * with the changes this source has applied but not written.
+     *
+     * A view-syncer computes the rows a replicated change collides with -- by
+     * change-log key for a delete, by every unique key for a set -- against the
+     * snapshot it is reading. When derivation writes through, that snapshot has
+     * already absorbed the earlier changes of the same advancement, so the
+     * probe sees them. When derivation is deferred they live here instead, and
+     * the probe has to be brought up to date or the source is pushed a change
+     * that contradicts its own state (a delete of a row the batch already
+     * removed, an edit whose old row is stale).
+     *
+     * Two ways the batch can disagree with the probe, both handled here: a
+     * probed row may have been removed, or may have been edited (so the
+     * caller's copy is stale, or it no longer collides at all).
+     *
+     * The batch cannot have edited a row the probe missed *into* collision. The
+     * change log keeps one entry per row key, so every row the batch holds is
+     * that row's value at the advancement's target version, as is `next`, and
+     * two distinct rows of one version cannot share a non-null unique key.
+     *
+     * Returns `base` untouched when derivation is not deferred, which is what
+     * makes this a no-op for the write-through path.
+     */
+    reconcilePendingConflicts(base: readonly Row[], next: Row | null, rowKey: Row, uniqueKeys: readonly (readonly string[])[]): readonly Row[];
     /**
      * Retrieves a row from the backing DB by a unique key, or `undefined` if such a
      * row does not exist. This is not used in the IVM pipeline but is useful

package/out/zqlite/src/table-source.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"table-source.d.ts","sourceRoot":"","sources":["../../../../zqlite/src/table-source.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAEjD,OAAO,KAAK,EAAC,SAAS,EAAC,MAAM,+BAA+B,CAAC;AAK7D,OAAO,KAAK,EAAC,SAAS,EAAE,QAAQ,EAAC,MAAM,gCAAgC,CAAC;AACxE,OAAO,KAAK,EAAC,GAAG,EAAQ,MAAM,iCAAiC,CAAC;AAChE,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,wCAAwC,CAAC;AACvE,OAAO,KAAK,EACV,WAAW,EACX,SAAS,EACV,MAAM,uCAAuC,CAAC;AAC/C,OAAO,KAAK,EAAC,aAAa,EAAC,MAAM,yCAAyC,CAAC;AAmB3E,OAAO,EACL,KAAK,MAAM,EACX,KAAK,YAAY,EACjB,KAAK,WAAW,EACjB,MAAM,6BAA6B,CAAC;AACrC,OAAO,KAAK,EAAC,MAAM,EAAC,MAAM,6BAA6B,CAAC;AAExD,OAAO,KAAK,EAAC,QAAQ,EAAY,MAAM,SAAS,CAAC;AAoBjD;;;;;;;;;;;;;GAaG;AACH,qBAAa,WAAY,YAAW,MAAM;;IAexC;;;;;OAKG;IACH,YACE,UAAU,EAAE,UAAU,EACtB,SAAS,EAAE,SAAS,EACpB,EAAE,EAAE,QAAQ,EACZ,SAAS,EAAE,MAAM,EACjB,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,WAAW,CAAC,EACpC,UAAU,EAAE,UAAU,EACtB,WAAW,gBAAc,EAe1B;IAED,IAAI,WAAW;QAEX,IAAI;QACJ,OAAO;QACP,UAAU;MAEb;IAED;;;OAGG;IACH,KAAK,CAAC,EAAE,EAAE,QAAQ,QAEjB;IA0FD,OAAO,CACL,IAAI,EAAE,QAAQ,GAAG,SAAS,EAC1B,OAAO,CAAC,EAAE,SAAS,EACnB,aAAa,CAAC,EAAE,GAAG,CAAC,MAAM,CAAC,EAC3B,KAAK,CAAC,EAAE,aAAa,eA4CtB;IAED,WAAW,CAAC,GAAG,EAAE,GAAG,GAAG,GAAG,CAOzB;IAkIA,IAAI,CAAC,MAAM,EAAE,YAAY,GAAG,MAAM,CAAC,OAAO,CAAC,CAM3C;IAEA,OAAO,CAAC,MAAM,EAAE,YAAY,iDAgB5B;IAqFD;;;;;;OAMG;IACH,MAAM,CAAC,MAAM,EAAE,GAAG,GAAG,GAAG,GAAG,SAAS,CAanC;CAmBF;AA4CD,wBAAgB,aAAa,CAC3B,OAAO,EAAE,SAAS,MAAM,EAAE,EAC1B,GAAG,EAAE,GAAG,EACR,WAAW,EAAE,MAAM,CAAC,MAAM,EAAE,WAAW,CAAC,GACvC,SAAS,OAAO,EAAE,CAEpB;AAED,wBAAgB,gBAAgB,CAAC,IAAI,EAAE,SAAS,wCAa/C;AAED,wBAAgB,eAAe,CAC7B,UAAU,EAAE,MAAM,CAAC,MAAM,EAAE,WAAW,CAAC,EACvC,GAAG,EAAE,GAAG,EACR,SAAS,EAAE,MAAM,GAChB,GAAG,CAaL;AAwCD,qBAAa,qBAAsB,SAAQ,KAAK;CAAG"}
+{"version":3,"file":"table-source.d.ts","sourceRoot":"","sources":["../../../../zqlite/src/table-source.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAEjD,OAAO,KAAK,EAAC,SAAS,EAAC,MAAM,+BAA+B,CAAC;AAM7D,OAAO,KAAK,EAAC,SAAS,EAAE,QAAQ,EAAC,MAAM,gCAAgC,CAAC;AACxE,OAAO,KAAK,EAAC,GAAG,EAAQ,MAAM,iCAAiC,CAAC;AAChE,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,wCAAwC,CAAC;AACvE,OAAO,KAAK,EACV,WAAW,EACX,SAAS,EACV,MAAM,uCAAuC,CAAC;AAC/C,OAAO,KAAK,EAAC,aAAa,EAAC,MAAM,yCAAyC,CAAC;AAuB3E,OAAO,EACL,KAAK,MAAM,EACX,KAAK,YAAY,EACjB,KAAK,WAAW,EACjB,MAAM,6BAA6B,CAAC;AACrC,OAAO,KAAK,EAAC,MAAM,EAAC,MAAM,6BAA6B,CAAC;AAExD,OAAO,KAAK,EAAC,QAAQ,EAAY,MAAM,SAAS,CAAC;AAyBjD,MAAM,MAAM,kBAAkB,GAAG;IAC/B;;;;;;;;OAQG;IACH,uBAAuB,CAAC,EAAE,OAAO,GAAG,SAAS,CAAC;IAE9C;;;;;OAKG;IACH,WAAW,CAAC,EAAE,OAAO,GAAG,SAAS,CAAC;CACnC,CAAC;AAEF;;;;;;;;;;;;;GAaG;AACH,qBAAa,WAAY,YAAW,MAAM;;IAwBxC;;;;;OAKG;IACH,YACE,UAAU,EAAE,UAAU,EACtB,SAAS,EAAE,SAAS,EACpB,EAAE,EAAE,QAAQ,EACZ,SAAS,EAAE,MAAM,EACjB,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,WAAW,CAAC,EACpC,UAAU,EAAE,UAAU,EACtB,WAAW,gBAAc,EACzB,OAAO,GAAE,kBAAuB,EAqBjC;IAED;;;OAGG;IACH,IAAI,QAAQ,IAAI,MAAM,CAErB;IAED,IAAI,WAAW;QAEX,IAAI;QACJ,OAAO;QACP,UAAU;MAEb;IAED;;;OAGG;IACH,KAAK,CAAC,EAAE,EAAE,QAAQ,QAKjB;IAED;;;;;;OAMG;IACH,cAAc,CAAC,WAAW,EAAE,OAAO,QAUlC;IAED;;;;;OAKG;IACH,qBAAqB,IAAI,IAAI,CAE5B;IAED;;;;;;OAMG;IACF,mBAAmB,IAAI,MAAM,CAAC,OAAO,CAAC,CAwBtC;IAwED,OAAO,CACL,IAAI,EAAE,QAAQ,GAAG,SAAS,EAC1B,OAAO,CAAC,EAAE,SAAS,EACnB,aAAa,CAAC,EAAE,GAAG,CAAC,MAAM,CAAC,EAC3B,KAAK,CAAC,EAAE,aAAa,eA4CtB;IAED,WAAW,CAAC,GAAG,EAAE,GAAG,GAAG,GAAG,CAOzB;IAmMA,IAAI,CAAC,MAAM,EAAE,YAAY,GAAG,MAAM,CAAC,OAAO,CAAC,CAM3C;IAED;;;OAGG;IACH,IAAI,WAAW,IAAI,MAAM,CAExB;IAED,sEAAsE;IACtE,IAAI,YAAY,IAAI,MAAM,CAEzB;IAED;;;OAGG;IACH,YAAY,CAAC,GAAG,EAAE,GAAG,GAAG,OAAO,CAU9B;IAmBA,OAAO,CAAC,MAAM,EAAE,YAAY,GAAG,MAAM,CAAC,OAAO,GAAG,SAAS,CAAC,CA6C1D;IAuED;;;;;;;;;;;;;;;;;;;;;;;;OAwBG;IACH,yBAAyB,CACvB,IAAI,EAAE,SAAS,GAAG,EAAE,EACpB,IAAI,EAAE,GAAG,GAAG,IAAI,EAChB,MAAM,EAAE,GAAG,EACX,UAAU,EAAE,SAAS,CAAC,SAAS,MAAM,EAAE,CAAC,EAAE,GACzC,SAAS,GAAG,EAAE,CAoChB;IA6DD;;;;;;OAMG;IACH,MAAM,CAAC,MAAM,EAAE,GAAG,GAAG,GAAG,GAAG,SAAS,CAkCnC;CAmBF;AAoFD,wBAAgB,aAAa,CAC3B,OAAO,EAAE,SAAS,MAAM,EAAE,EAC1B,GAAG,EAAE,GAAG,EACR,WAAW,EAAE,MAAM,CAAC,MAAM,EAAE,WAAW,CAAC,GACvC,SAAS,OAAO,EAAE,CAEpB;AAED,wBAAgB,gBAAgB,CAAC,IAAI,EAAE,SAAS,wCAa/C;AAED,wBAAgB,eAAe,CAC7B,UAAU,EAAE,MAAM,CAAC,MAAM,EAAE,WAAW,CAAC,EACvC,GAAG,EAAE,GAAG,EACR,SAAS,EAAE,MAAM,GAChB,GAAG,CAaL;AAyCD,qBAAa,qBAAsB,SAAQ,KAAK;CAAG"}

package/out/zqlite/src/table-source.js

@@ -1,13 +1,16 @@
 import { assert, unreachable } from "../../shared/src/asserts.js";
+import { getOrInsertComputed } from "../../shared/src/map.js";
 import { must } from "../../shared/src/must.js";
 import { assertOrderingIncludesPK } from "../../zql/src/query/complete-ordering.js";
 import { makeComparator } from "../../zql/src/ivm/data.js";
 import { createPredicate, transformFilters } from "../../zql/src/builder/filter.js";
 import { genPushAndWriteWithSplitEdit, generateWithOverlay, generateWithOverlayUnordered, generateWithStart } from "../../zql/src/ivm/memory-source.js";
 import { timeSampled } from "../../otel/src/maybe-time.js";
+import { recordIndexUsage } from "./index-usage.js";
 import { compile, format, sql } from "./internal/sql.js";
 import { StatementCache } from "./internal/statement-cache.js";
-import { buildSelectQuery, toSQLiteType } from "./query-builder.js";
+import { NOT_OVERRIDDEN, PendingDelta, generateWithPendingDelta, generateWithPendingDeltaUnordered } from "./pending-delta.js";
+import { buildSelectQueries, toSQLiteType } from "./query-builder.js";
 import SQLite3Database from "@rocicorp/zero-sqlite3";
 //#region ../zqlite/src/table-source.ts
 var eventCount = 0;
@@ -35,25 +38,45 @@ var TableSource = class {
 	#logConfig;
 	#lc;
 	#shouldYield;
+	#skipUnobservableChanges;
+	#primaryKeySort;
 	#stmts;
 	#overlay;
 	#pushEpoch = 0;
 	/**
+	* When present, derivation is read-only: changes accumulate here instead of
+	* being written to (and later rolled back out of) the backing snapshot. See
+	* {@link PendingDelta}.
+	*/
+	#delta;
+	#rowsRead = 0;
+	/**
 	* @param shouldYield a function called after each row is read from the database,
 	* which should return true if the source should yield the special 'yield' value
 	* to yield control back to the caller at the end of the pipeline.  Can
 	* also throw an error to abort the pipeline processing.
 	*/
-	constructor(logContext, logConfig, db, tableName, columns, primaryKey, shouldYield = () => false) {
+	constructor(logContext, logConfig, db, tableName, columns, primaryKey, shouldYield = () => false, options = {}) {
 		this.#lc = logContext;
 		this.#logConfig = logConfig;
 		this.#table = tableName;
 		this.#columns = columns;
 		this.#uniqueIndexes = getUniqueIndexes(db, tableName);
 		this.#primaryKey = primaryKey;
+		this.#primaryKeySort = primaryKey.map((k) => [k, "asc"]);
 		this.#stmts = this.#getStatementsFor(db);
 		this.#shouldYield = shouldYield;
-		assert(this.#uniqueIndexes.has(JSON.stringify(primaryKey.toSorted())), `primary key ${primaryKey} does not have a UNIQUE index`);
+		this.#skipUnobservableChanges = options.skipUnobservableChanges ?? false;
+		this.#delta = options.deferWrites ? new PendingDelta(primaryKey) : void 0;
+		const primaryKeyStr = JSON.stringify(primaryKey.toSorted());
+		assert(this.#uniqueIndexes.has(primaryKeyStr), `primary key ${primaryKeyStr} does not have a UNIQUE index`);
+	}
+	/**
+	* The number of rows read from SQLite by fetches, over the lifetime of this
+	* source. Rows that SQLite scans and filters out itself are not counted.
+	*/
+	get rowsRead() {
+		return this.#rowsRead;
 	}
 	get tableSchema() {
 		return {
@@ -68,6 +91,49 @@ var TableSource = class {
 	*/
 	setDB(db) {
 		this.#stmts = this.#getStatementsFor(db);
+		this.#delta?.clear();
+	}
+	/**
+	* Sets whether the changes pushed from now on are held in memory (see
+	* {@link TableSourceOptions.deferWrites}) or written to the backing
+	* snapshot. The two must not be mixed within one snapshot, so this may only
+	* be called before the first push after construction or {@link setDB}. To
+	* switch to writing through partway, see {@link writePendingChanges}.
+	*/
+	setDeferWrites(deferWrites) {
+		assert(this.#delta === void 0 || this.#delta.isEmpty, "Cannot change how writes are applied while changes are pending");
+		if (!deferWrites) this.#delta = void 0;
+		else this.#delta ??= new PendingDelta(this.#primaryKey);
+	}
+	/**
+	* Drops the changes held in memory (see {@link setDeferWrites}), for an
+	* advancement that has been abandoned. The source then no longer reflects
+	* the changes pushed to it, so it must not be read again until it has moved
+	* to another snapshot ({@link setDB}), if it is used again at all.
+	*/
+	discardPendingChanges() {
+		this.#delta?.clear();
+	}
+	/**
+	* Writes the changes held in memory (see {@link setDeferWrites}) to the
+	* backing snapshot, which is left as if they had been written through (in
+	* the source's columns), and writes the changes pushed from then on through
+	* to it. Yields when the
+	* source's `shouldYield` says to, and must not be interleaved with a push.
+	*/
+	*writePendingChanges() {
+		const delta = this.#delta;
+		if (delta === void 0) return;
+		for (const key of delta.touchedKeys()) {
+			this.#stmts.delete.run(...toSQLiteTypes(this.#primaryKey, key, this.#columns));
+			if (this.#shouldYield()) yield "yield";
+		}
+		const columns = Object.keys(this.#columns);
+		for (const row of delta.liveRows()) {
+			this.#stmts.insert.run(...toSQLiteTypes(columns, row, this.#columns));
+			if (this.#shouldYield()) yield "yield";
+		}
+		this.#delta = void 0;
 	}
 	#getStatementsFor(db) {
 		const cached = this.#dbCache.get(db);
@@ -76,9 +142,7 @@ var TableSource = class {
 			cache: new StatementCache(db),
 			insert: db.prepare(compile(sql`INSERT INTO ${sql.ident(this.#table)} (${sql.join(Object.keys(this.#columns).map((c) => sql.ident(c)), ", ")}) VALUES (${sql.__dangerous__rawValue(Array.from({ length: Object.keys(this.#columns).length }).fill("?").join(","))})`)),
 			delete: db.prepare(compile(sql`DELETE FROM ${sql.ident(this.#table)} WHERE ${sql.join(this.#primaryKey.map((k) => sql`${sql.ident(k)}=?`), " AND ")}`)),
-			update: Object.keys(this.#columns).length > this.#primaryKey.length ? db.prepare(compile(sql`UPDATE ${sql.ident(this.#table)} SET ${sql.join(nonPrimaryKeys(this.#columns, this.#primaryKey).map((c) => sql`${sql.ident(c)}=?`), ",")} WHERE ${sql.join(this.#primaryKey.map((k) => sql`${sql.ident(k)}=?`), " AND ")}`)) : void 0,
-			checkExists: db.prepare(compile(sql`SELECT 1 AS "exists" FROM ${sql.ident(this.#table)} WHERE ${sql.join(this.#primaryKey.map((k) => sql`${sql.ident(k)}=?`), " AND ")} LIMIT 1`)),
-			getExisting: db.prepare(compile(sql`SELECT * FROM ${sql.ident(this.#table)} WHERE ${sql.join(this.#primaryKey.map((k) => sql`${sql.ident(k)}=?`), " AND ")}`))
+			update: Object.keys(this.#columns).length > this.#primaryKey.length ? db.prepare(compile(sql`UPDATE ${sql.ident(this.#table)} SET ${sql.join(nonPrimaryKeys(this.#columns, this.#primaryKey).map((c) => sql`${sql.ident(c)}=?`), ",")} WHERE ${sql.join(this.#primaryKey.map((k) => sql`${sql.ident(k)}=?`), " AND ")}`)) : void 0
 		};
 		this.#dbCache.set(db, stmts);
 		return stmts;
@@ -101,7 +165,7 @@ var TableSource = class {
 	connect(sort, filters, splitEditKeys, debug) {
 		const transformedFilters = transformFilters(filters);
 		const unordered = sort === void 0;
-		const primaryKeySort = this.#primaryKey.map((k) => [k, "asc"]);
+		const primaryKeySort = this.#primaryKeySort;
 		const input = {
 			getSchema: () => schema,
 			fetch: (req) => this.#fetch(req, connection),
@@ -138,41 +202,75 @@ var TableSource = class {
 	}
 	*#fetch(req, connection) {
 		const { sort, debug } = connection;
-		const sqlAndBindings = format(this.#requestToSQL(req, connection.filters?.condition, sort));
-		const cachedStatement = this.#stmts.cache.get(sqlAndBindings.text);
-		cachedStatement.statement.safeIntegers(true);
-		const rowIterator = cachedStatement.statement.iterate(...sqlAndBindings.values);
+		const rows = this.#queryRows(this.#requestToSQL(req, connection.filters?.condition, sort), debug);
 		const overlayPredicate = mergeOverlayPredicate(connection.filters?.predicate, req.filter);
 		try {
-			debug?.initQuery(this.#table, sqlAndBindings.text);
 			if (sort) {
 				const comparator = makeComparator(sort, req.reverse);
-				yield* generateWithStart(generateWithYields(generateWithOverlay(req.start?.row, this.#mapFromSQLiteTypes(this.#columns, rowIterator, sqlAndBindings.text, debug), req.constraint, this.#overlay, connection.lastPushedEpoch, comparator, comparator, overlayPredicate, req.multiConstraints), this.#shouldYield), req.start, comparator);
-			} else yield* generateWithYields(generateWithOverlayUnordered(this.#mapFromSQLiteTypes(this.#columns, rowIterator, sqlAndBindings.text, debug), req.constraint, this.#overlay, connection.lastPushedEpoch, this.#primaryKey, overlayPredicate, req.multiConstraints), this.#shouldYield);
+				yield* generateWithStart(generateWithYields(generateWithOverlay(req.start?.row, this.#withPendingDelta(rows, req, overlayPredicate, sort, comparator), req.constraint, this.#overlay, connection.lastPushedEpoch, comparator, comparator, overlayPredicate, req.multiConstraints), this.#shouldYield), req.start, comparator);
+			} else yield* generateWithYields(generateWithOverlayUnordered(this.#withPendingDelta(rows, req, overlayPredicate, void 0, void 0), req.constraint, this.#overlay, connection.lastPushedEpoch, this.#primaryKey, overlayPredicate, req.multiConstraints), this.#shouldYield);
 		} finally {
-			rowIterator.return?.();
-			if (debug) {
-				let totalNvisit = 0;
-				const planLines = [];
-				for (let i = 0;; i++) {
-					const nvisit = cachedStatement.statement.scanStatus(i, SQLite3Database.SQLITE_SCANSTAT_NVISIT, 1);
-					if (nvisit === void 0) break;
-					totalNvisit += Number(nvisit);
-					const explain = cachedStatement.statement.scanStatus(i, SQLite3Database.SQLITE_SCANSTAT_EXPLAIN, 1);
-					if (typeof explain === "string" && explain.length > 0) planLines.push(explain);
+			rows.return();
+		}
+	}
+	/**
+	* The rows of `queries`, run one after another as {@link buildSelectQueries}
+	* requires. Each statement is run only once the previous one is exhausted,
+	* so a fetch that stops early never runs the rest.
+	*/
+	*#queryRows(queries, debug) {
+		const { cache } = this.#stmts;
+		for (const query of queries) {
+			const sqlAndBindings = format(query);
+			const cachedStatement = cache.get(sqlAndBindings.text);
+			cachedStatement.statement.safeIntegers(true);
+			const rowIterator = cachedStatement.statement.iterate(...sqlAndBindings.values);
+			try {
+				debug?.initQuery(this.#table, sqlAndBindings.text);
+				yield* this.#mapFromSQLiteTypes(this.#columns, rowIterator, sqlAndBindings.text, debug);
+			} finally {
+				rowIterator.return?.();
+				recordIndexUsage(cachedStatement.statement, this.#table);
+				if (debug) {
+					let totalNvisit = 0;
+					const planLines = [];
+					for (let i = 0;; i++) {
+						const nvisit = cachedStatement.statement.scanStatus(i, SQLite3Database.SQLITE_SCANSTAT_NVISIT, 1);
+						if (nvisit === void 0) break;
+						totalNvisit += Number(nvisit);
+						const explain = cachedStatement.statement.scanStatus(i, SQLite3Database.SQLITE_SCANSTAT_EXPLAIN, 1);
+						if (typeof explain === "string" && explain.length > 0) planLines.push(explain);
+					}
+					if (totalNvisit !== 0) debug.recordNVisit(this.#table, sqlAndBindings.text, totalNvisit);
+					if (planLines.length > 0) debug.recordExplain(this.#table, sqlAndBindings.text, planLines);
+					cachedStatement.statement.scanStatusReset();
 				}
-				if (totalNvisit !== 0) debug.recordNVisit(this.#table, sqlAndBindings.text, totalNvisit);
-				if (planLines.length > 0) debug.recordExplain(this.#table, sqlAndBindings.text, planLines);
-				cachedStatement.statement.scanStatusReset();
+				cache.return(cachedStatement);
 			}
-			this.#stmts.cache.return(cachedStatement);
 		}
 	}
+	/**
+	* Splices the batch overlay into a base row stream, if there is one. When
+	* derivation is not deferred, or the batch is empty -- every fetch during
+	* hydration, and every fetch before the first change of a pass -- this is
+	* the identity and the source behaves exactly as it did before.
+	*
+	* `predicate` is the connection's filters ANDed with `req.filter`, which
+	* SQL applies to the base rows. `sort` and `compare` are absent for
+	* unordered fetches, where nothing downstream depends on position.
+	*/
+	#withPendingDelta(baseRows, req, predicate, sort, compare) {
+		const delta = this.#delta;
+		if (delta === void 0 || delta.isEmpty) return baseRows;
+		const deltaRows = delta.rowsFor(sort ?? this.#primaryKeySort, req.constraint, sort ? req.reverse : false, predicate, req.multiConstraints, sort ? req.start?.row : void 0);
+		return compare ? generateWithPendingDelta(baseRows, deltaRows, delta, compare) : generateWithPendingDeltaUnordered(baseRows, deltaRows, delta);
+	}
 	*#mapFromSQLiteTypes(valueTypes, rowIterator, query, debug) {
 		let result;
 		do {
 			result = timeSampled(this.#lc, ++eventCount, this.#logConfig.ivmSampling, () => rowIterator.next(), this.#logConfig.slowRowThreshold, () => `table-source.next took too long for ${query}. Are you missing an index?`);
 			if (result.done) break;
+			this.#rowsRead++;
 			const row = fromSQLiteTypes(valueTypes, result.value, this.#table);
 			debug?.rowVended(this.#table, query, row);
 			yield row;
@@ -181,13 +279,51 @@ var TableSource = class {
 	*push(change) {
 		for (const result of this.genPush(change)) if (result === "yield") yield result;
 	}
+	/**
+	* The number of rows held in memory for changes that have been pushed but
+	* not written, i.e. with `deferWrites`, since the last {@link setDB}.
+	*/
+	get pendingRows() {
+		return this.#delta?.size ?? 0;
+	}
+	/** Estimated bytes retained by deferred changes and their indexes. */
+	get pendingBytes() {
+		return this.#delta?.estimatedBytes ?? 0;
+	}
+	/**
+	* Returns whether any connected pipeline could observe the given row.
+	* If false, this row is completely invisible to this TableSource's connections.
+	*/
+	mayAcceptRow(row) {
+		if (!this.#skipUnobservableChanges || this.#connections.length === 0) return true;
+		for (const conn of this.#connections) if (!conn.filters || conn.filters.predicate(row)) return true;
+		return false;
+	}
+	#mayAcceptChange(change) {
+		if (!this.#skipUnobservableChanges || this.#connections.length === 0) return true;
+		const row = change[1];
+		if (this.mayAcceptRow(row)) return true;
+		if (change[0] === 2) {
+			const oldRow = change[2];
+			if (oldRow && this.mayAcceptRow(oldRow)) return true;
+		}
+		return false;
+	}
 	*genPush(change) {
-		const exists = (row) => this.#stmts.checkExists.get(...toSQLiteTypes(this.#primaryKey, row, this.#columns))?.exists === 1;
+		if (this.#skipUnobservableChanges && !this.#mayAcceptChange(change)) {
+			if (change[0] === 1) this.#writeChange(change);
+			return;
+		}
 		const setOverlay = (o) => this.#overlay = o;
 		const writeChange = (c) => this.#writeChange(c);
-		yield* genPushAndWriteWithSplitEdit(this.#connections, change, exists, setOverlay, writeChange, () => ++this.#pushEpoch);
+		yield* genPushAndWriteWithSplitEdit(this.#connections, change, void 0, setOverlay, writeChange, () => ++this.#pushEpoch);
 	}
 	#writeChange(change) {
+		const delta = this.#delta;
+		if (delta !== void 0) {
+			this.#writeChangeToDelta(delta, change);
+			return;
+		}
 		switch (change[0]) {
 			case 0:
 				this.#stmts.insert.run(...toSQLiteTypes(Object.keys(this.#columns), change[1], this.#columns));
@@ -211,15 +347,85 @@ var TableSource = class {
 			default: unreachable(change);
 		}
 	}
+	/**
+	* Reconciles a set of rows that the caller found in the *backing snapshot*
+	* with the changes this source has applied but not written.
+	*
+	* A view-syncer computes the rows a replicated change collides with -- by
+	* change-log key for a delete, by every unique key for a set -- against the
+	* snapshot it is reading. When derivation writes through, that snapshot has
+	* already absorbed the earlier changes of the same advancement, so the
+	* probe sees them. When derivation is deferred they live here instead, and
+	* the probe has to be brought up to date or the source is pushed a change
+	* that contradicts its own state (a delete of a row the batch already
+	* removed, an edit whose old row is stale).
+	*
+	* Two ways the batch can disagree with the probe, both handled here: a
+	* probed row may have been removed, or may have been edited (so the
+	* caller's copy is stale, or it no longer collides at all).
+	*
+	* The batch cannot have edited a row the probe missed *into* collision. The
+	* change log keeps one entry per row key, so every row the batch holds is
+	* that row's value at the advancement's target version, as is `next`, and
+	* two distinct rows of one version cannot share a non-null unique key.
+	*
+	* Returns `base` untouched when derivation is not deferred, which is what
+	* makes this a no-op for the write-through path.
+	*/
+	reconcilePendingConflicts(base, next, rowKey, uniqueKeys) {
+		const delta = this.#delta;
+		if (delta === void 0 || delta.isEmpty) return base;
+		const deleteKey = next === null ? this.#syncedDeleteKey(rowKey) : void 0;
+		const out = [];
+		for (const row of base) {
+			const pending = delta.get(row);
+			const current = pending === NOT_OVERRIDDEN ? row : pending;
+			if (current === void 0) continue;
+			if (deleteKey !== void 0 ? !deleteKey.every(([col, value]) => sqliteValuesEqual(current[col], value, this.#columns[col].type)) : !collides(current, must(next), uniqueKeys, this.#primaryKey, this.#columns)) continue;
+			out.push(current);
+		}
+		return out;
+	}
+	#syncedDeleteKey(rowKey) {
+		const synced = {};
+		for (const [col, value] of Object.entries(rowKey)) if (Object.hasOwn(this.#columns, col)) synced[col] = value;
+		return Object.entries(fromSQLiteTypes(this.#columns, synced, this.#table));
+	}
+	/**
+	* The batch-overlay counterpart of {@link #writeChange}. It must reproduce
+	* that method's semantics exactly, including its treatment of an edit that
+	* cannot be expressed as an `UPDATE` -- a changed primary key, or a table
+	* whose columns are all part of the key -- as a delete followed by an
+	* insert.
+	*/
+	#writeChangeToDelta(delta, change) {
+		switch (change[0]) {
+			case 0:
+				delta.set(change[1]);
+				break;
+			case 1:
+				delta.delete(change[1]);
+				break;
+			case 2: {
+				const oldRow = change[2];
+				const row = change[1];
+				if (canUseUpdate(oldRow, row, this.#columns, this.#primaryKey)) delta.set({
+					...oldRow,
+					...row
+				});
+				else {
+					delta.delete(oldRow);
+					delta.set(row);
+				}
+				break;
+			}
+			default: unreachable(change);
+		}
+	}
 	#getRowStmtCache = /* @__PURE__ */ new Map();
 	#getRowStmt(keyCols) {
 		const keyString = JSON.stringify(keyCols);
-		let stmt = this.#getRowStmtCache.get(keyString);
-		if (!stmt) {
-			stmt = compile(sql`SELECT ${this.#allColumns} FROM ${sql.ident(this.#table)} WHERE ${sql.join(keyCols.map((k) => sql`${sql.ident(k)}=?`), sql` AND`)}`);
-			this.#getRowStmtCache.set(keyString, stmt);
-		}
-		return stmt;
+		return getOrInsertComputed(this.#getRowStmtCache, keyString, () => compile(sql`SELECT ${this.#allColumns} FROM ${sql.ident(this.#table)} WHERE ${sql.join(keyCols.map((k) => sql`${sql.ident(k)}=?`), sql` AND`)}`));
 	}
 	/**
 	* Retrieves a row from the backing DB by a unique key, or `undefined` if such a
@@ -231,12 +437,23 @@ var TableSource = class {
 	getRow(rowKey) {
 		const keyCols = Object.keys(rowKey);
 		const stmt = this.#getRowStmt(keyCols);
-		const row = this.#stmts.cache.use(stmt, (cached) => cached.statement.safeIntegers(true).get(...toSQLiteTypes(keyCols, rowKey, this.#columns)));
-		if (row) return fromSQLiteTypes(this.#columns, row, this.#table);
-		return row;
+		const raw = this.#stmts.cache.use(stmt, ({ statement }) => {
+			const row = statement.safeIntegers(true).get(...toSQLiteTypes(keyCols, rowKey, this.#columns));
+			recordIndexUsage(statement, this.#table);
+			return row;
+		});
+		const row = raw ? fromSQLiteTypes(this.#columns, raw, this.#table) : raw;
+		const delta = this.#delta;
+		if (delta === void 0 || delta.isEmpty) return row;
+		if (row !== void 0) {
+			const overridden = delta.get(row);
+			if (overridden === NOT_OVERRIDDEN) return row;
+			if (overridden !== void 0 && keyCols.every((c) => overridden[c] === rowKey[c])) return overridden;
+		}
+		return delta.getByColumns(keyCols, rowKey);
 	}
 	#requestToSQL(request, filters, order) {
-		return buildSelectQuery(this.#table, this.#columns, request.constraint, filters, order, request.reverse, request.start, request.multiConstraints, request.filter);
+		return buildSelectQueries(this.#table, this.#columns, request.constraint, filters, order, request.reverse, request.start, request.multiConstraints, request.filter);
 	}
 };
 function mergeOverlayPredicate(connPredicate, reqFilter) {
@@ -245,6 +462,22 @@ function mergeOverlayPredicate(connPredicate, reqFilter) {
 	if (!connPredicate) return reqPredicate;
 	return (row) => connPredicate(row) && reqPredicate(row);
 }
+/**
+* Whether `a` occupies the same row identity as `b` -- the same primary key,
+* or the same value on any unique key, which is what a replicated write
+* conflicts on.
+*/
+function collides(a, b, uniqueKeys, primaryKey, columns) {
+	if (primaryKey.every((col) => sqliteValuesEqual(a[col], b[col], columns[col].type))) return true;
+	for (const key of uniqueKeys) {
+		if (key.some((col) => b[col] === null || b[col] === void 0)) continue;
+		if (key.every((col) => sqliteValuesEqual(a[col], b[col], columns[col].type))) return true;
+	}
+	return false;
+}
+function sqliteValuesEqual(a, b, type) {
+	return toSQLiteType(a, type) === toSQLiteType(b, type);
+}
 function getUniqueIndexes(db, tableName) {
 	const sqlAndBindings = format(sql`
     SELECT idx.name, json_group_array(col.name) as columnsJSON
@@ -287,8 +520,9 @@ function fromSQLiteType(valueType, v, column, tableName) {
 		case "string":
 		case "null":
 			if (typeof v === "bigint") {
-				if (v > Number.MAX_SAFE_INTEGER || v < Number.MIN_SAFE_INTEGER) throw new UnsupportedValueError(`value ${v} (in ${tableName}.${column}) is outside of supported bounds`);
-				return Number(v);
+				const bi = v;
+				if (bi > Number.MAX_SAFE_INTEGER || bi < Number.MIN_SAFE_INTEGER) throw new UnsupportedValueError(`value ${bi} (in ${tableName}.${column}) is outside of supported bounds`);
+				return Number(bi);
 			}
 			return v;
 		case "json": try {