@rocicorp/zero 1.12.0-canary.0 → 1.12.0-canary.10

package/out/zql/src/query/validate-input.js.map

@@ -1 +1 @@
-{"version":3,"file":"validate-input.js","names":[],"sources":["../../../../../zql/src/query/validate-input.ts"],"sourcesContent":["import type {StandardSchemaV1} from '@standard-schema/spec';\n\nexport class InputValidationError extends Error {\n  readonly result: StandardSchemaV1.FailureResult;\n\n  // will get picked up by `getErrorDetails` when mapping to an `ApplicationError`\n  readonly details: {\n    type: 'InputValidationError';\n    result: StandardSchemaV1.FailureResult;\n  };\n\n  constructor(message: string, result: StandardSchemaV1.FailureResult) {\n    super(message);\n    this.name = 'InputValidationError';\n    this.result = result;\n    this.details = {type: 'InputValidationError', result};\n  }\n}\n\n/**\n * Validates input using a StandardSchema validator if provided.\n * This is shared validation logic used by both defineQuery and defineMutator.\n *\n * @param name - The name of the query or mutator (for error messages)\n * @param input - The input value to validate\n * @param validator - Optional StandardSchema validator\n * @param kind - Type of definition ('query' or 'mutator') for error messages\n * @returns The validated output (either transformed by validator or input as-is)\n * @throws Error if validation fails or if an async validator is used\n * @internal\n */\nexport function validateInput<TInput, TOutput>(\n  name: string,\n  input: TInput,\n  validator: StandardSchemaV1<TInput, TOutput> | undefined,\n  kind: 'query' | 'mutator',\n): TOutput {\n  if (!validator) {\n    // No validator, so input and output are the same\n    return input as unknown as TOutput;\n  }\n\n  const result = validator['~standard'].validate(input);\n  if (result instanceof Promise) {\n    throw new Error(\n      `Async validators are not supported. ${titleCase(kind)} name ${name}`,\n    );\n  }\n  if (result.issues) {\n    throw new InputValidationError(\n      `Validation failed for ${kind} ${name}: ${result.issues\n        .map(issue => issue.message)\n        .join(', ')}`,\n      result,\n    );\n  }\n  return result.value;\n}\n\nfunction titleCase(kind: string): string {\n  return kind.charAt(0).toUpperCase() + kind.slice(1);\n}\n"],"mappings":";AAEA,IAAa,uBAAb,cAA0C,MAAM;CAC9C;CAGA;CAKA,YAAY,SAAiB,QAAwC;EACnE,MAAM,OAAO;EACb,KAAK,OAAO;EACZ,KAAK,SAAS;EACd,KAAK,UAAU;GAAC,MAAM;GAAwB;EAAM;CACtD;AACF;;;;;;;;;;;;;AAcA,SAAgB,cACd,MACA,OACA,WACA,MACS;CACT,IAAI,CAAC,WAEH,OAAO;CAGT,MAAM,SAAS,UAAU,aAAa,SAAS,KAAK;CACpD,IAAI,kBAAkB,SACpB,MAAM,IAAI,MACR,uCAAuC,UAAU,IAAI,EAAE,QAAQ,MACjE;CAEF,IAAI,OAAO,QACT,MAAM,IAAI,qBACR,yBAAyB,KAAK,GAAG,KAAK,IAAI,OAAO,OAC9C,KAAI,UAAS,MAAM,OAAO,EAC1B,KAAK,IAAI,KACZ,MACF;CAEF,OAAO,OAAO;AAChB;AAEA,SAAS,UAAU,MAAsB;CACvC,OAAO,KAAK,OAAO,CAAC,EAAE,YAAY,IAAI,KAAK,MAAM,CAAC;AACpD"}
+{"version":3,"file":"validate-input.js","names":[],"sources":["../../../../../zql/src/query/validate-input.ts"],"sourcesContent":["import type {StandardSchemaV1} from '@standard-schema/spec';\n\nexport class InputValidationError extends Error {\n  readonly result: StandardSchemaV1.FailureResult;\n\n  // will get picked up by `getErrorDetails` when mapping to an `ApplicationError`\n  readonly details: {\n    type: 'InputValidationError';\n    result: StandardSchemaV1.FailureResult;\n  };\n\n  constructor(message: string, result: StandardSchemaV1.FailureResult) {\n    super(message);\n    this.name = 'InputValidationError';\n    this.result = result;\n    this.details = {type: 'InputValidationError', result};\n  }\n}\n\n/**\n * Validates input using a StandardSchema validator if provided.\n * This is shared validation logic used by both defineQuery and defineMutator.\n *\n * @param name - The name of the query or mutator (for error messages)\n * @param input - The input value to validate\n * @param validator - Optional StandardSchema validator\n * @param kind - Type of definition ('query' or 'mutator') for error messages\n * @returns The validated output (either transformed by validator or input as-is)\n * @throws Error if validation fails or if an async validator is used\n * @internal\n */\nexport function validateInput<TInput, TOutput>(\n  name: string,\n  input: TInput,\n  validator: StandardSchemaV1<TInput, TOutput> | undefined,\n  kind: 'query' | 'mutator',\n): TOutput {\n  if (!validator) {\n    // No validator, so input and output are the same\n    return input as unknown as TOutput;\n  }\n\n  const result = validator['~standard'].validate(input);\n  if (result instanceof Promise) {\n    throw new Error(\n      `Async validators are not supported. ${titleCase(kind)} name ${name}`,\n    );\n  }\n  if (result.issues) {\n    throw new InputValidationError(\n      `Validation failed for ${kind} ${name}: ${result.issues\n        .map(issue => issue.message)\n        .join(', ')}`,\n      result,\n    );\n  }\n  return result.value;\n}\n\nfunction titleCase(kind: string): string {\n  return kind.charAt(0).toUpperCase() + kind.slice(1);\n}\n"],"mappings":";AAEA,IAAa,uBAAb,cAA0C,MAAM;CAC9C;CAGA;CAKA,YAAY,SAAiB,QAAwC;EACnE,MAAM,OAAO;EACb,KAAK,OAAO;EACZ,KAAK,SAAS;EACd,KAAK,UAAU;GAAC,MAAM;GAAwB;EAAM;CACtD;AACF;;;;;;;;;;;;;AAcA,SAAgB,cACd,MACA,OACA,WACA,MACS;CACT,IAAI,CAAC,WAEH,OAAO;CAGT,MAAM,SAAS,UAAU,YAAY,CAAC,SAAS,KAAK;CACpD,IAAI,kBAAkB,SACpB,MAAM,IAAI,MACR,uCAAuC,UAAU,IAAI,EAAE,QAAQ,MACjE;CAEF,IAAI,OAAO,QACT,MAAM,IAAI,qBACR,yBAAyB,KAAK,GAAG,KAAK,IAAI,OAAO,OAC9C,KAAI,UAAS,MAAM,OAAO,CAAC,CAC3B,KAAK,IAAI,KACZ,MACF;CAEF,OAAO,OAAO;AAChB;AAEA,SAAS,UAAU,MAAsB;CACvC,OAAO,KAAK,OAAO,CAAC,CAAC,CAAC,YAAY,IAAI,KAAK,MAAM,CAAC;AACpD"}

package/out/zqlite/src/database-storage.js

@@ -1,3 +1,5 @@
+import { getSizeOfEntry } from "../../shared/src/size-of-value.js";
+import { compareValues } from "../../zql/src/ivm/data.js";
 import { Database } from "./db.js";
 //#region ../zqlite/src/database-storage.ts
 var CREATE_STORAGE_TABLE = `
@@ -9,12 +11,73 @@ var CREATE_STORAGE_TABLE = `
     PRIMARY KEY(clientGroupID, op, key)
   )
   `;
+/**
+* The most values the storage of a spilled operator keeps in memory. Profiles
+* of the view-syncer gain as much from 64 as from 1,024.
+*/
+var MAX_CACHED_ENTRIES_PER_OP = 64;
+/**
+* When the operators' values go over `maxMemoryBytes`, operators are spilled
+* until they are back under this proportion of it, so that spilling, which
+* sorts the operators by last use, happens in batches.
+*/
+var SPILL_TARGET_PROPORTION = .9;
+/**
+* The heap bytes that operator state takes per byte of {@link getSizeOfEntry}.
+* Measured with `heapUsed` on view-syncer operator state, mostly Cap's
+* `{size, pks}` and Take's `{size, bound}`.
+*/
+var HEAP_BYTES_PER_SIZE = 2.6;
 var defaultOptions = {
 	commitInterval: 5e3,
-	compactionThresholdBytes: 50 * 1024 * 1024
+	compactionThresholdBytes: 52428800,
+	maxMemoryBytes: 0
 };
+/**
+* The storage of one operator. Its values live in `values` until the
+* operator is spilled, and in SQLite afterwards, with the most recently used
+* of them in `cache`.
+*/
+var OpStorage = class {
+	cgID;
+	opID;
+	/** `undefined` once spilled. */
+	values;
+	/** The {@link getSizeOfEntry} of `values`. */
+	size = 0;
+	lastUsed = 0;
+	/**
+	* Once spilled, the operator's most recently read and written values,
+	* `undefined` for a key known to be absent. Operators read the same key
+	* again and again (Cap and Take read their state on every fetch and push),
+	* so this saves most of the SQLite reads.
+	*/
+	cache = /* @__PURE__ */ new Map();
+	constructor(cgID, opID, inMemory) {
+		this.cgID = cgID;
+		this.opID = opID;
+		this.values = inMemory ? /* @__PURE__ */ new Map() : void 0;
+	}
+	cacheSet(key, val) {
+		const { cache } = this;
+		if (cache.size >= MAX_CACHED_ENTRIES_PER_OP && !cache.has(key)) cache.clear();
+		cache.set(key, val);
+	}
+};
+/**
+* Operator storage for the client groups of a sync worker.
+*
+* Each operator keeps its values in a `Map`, so reading a key that was never
+* written does not ask SQLite. When the values of all of the operators go
+* over `maxMemoryBytes`, the least recently used operators spill: their
+* values are written to SQLite, where they stay until the operator is
+* destroyed.
+*
+* Like `MemoryStorage`, `get` returns the stored value itself, which
+* operators do not mutate.
+*/
 var DatabaseStorage = class DatabaseStorage {
-	static create(lc, path, options = defaultOptions) {
+	static create(lc, path, options = {}) {
 		const db = new Database(lc, path);
 		db.unsafeMode(true);
 		db.pragma("locking_mode = EXCLUSIVE");
@@ -30,12 +93,20 @@ var DatabaseStorage = class DatabaseStorage {
 	#options;
 	#db;
 	#numWrites = 0;
-	constructor(db, options = defaultOptions) {
+	/** The operators whose values are in memory. */
+	#inMemory = /* @__PURE__ */ new Set();
+	/** The {@link getSizeOfEntry} of the values of the operators in `#inMemory`. */
+	#size = 0;
+	/** `maxMemoryBytes` in units of {@link getSizeOfEntry}. */
+	#maxSize;
+	/** Orders the operators by last use. */
+	#clock = 0;
+	constructor(db, options = {}) {
 		this.#stmts = {
 			get: db.prepare(`
         SELECT val FROM storage WHERE
           clientGroupID = ? AND op = ? AND key = ?
-      `),
+      `).pluck(true),
 			set: db.prepare(`
         INSERT INTO storage (clientGroupID, op, key, val)
           VALUES(?, ?, ?, ?)
@@ -50,7 +121,7 @@ var DatabaseStorage = class DatabaseStorage {
 			scan: db.prepare(`
         SELECT key, val FROM storage WHERE
           clientGroupID = ? AND op = ? AND key >= ?
-      `),
+      `).raw(true),
 			clear: db.prepare(`
         DELETE FROM storage WHERE clientGroupID = ?
       `),
@@ -61,29 +132,110 @@ var DatabaseStorage = class DatabaseStorage {
 			begin: db.prepare("BEGIN")
 		};
 		this.#stmts.begin.run();
-		this.#options = options;
+		this.#options = {
+			commitInterval: options.commitInterval ?? defaultOptions.commitInterval,
+			compactionThresholdBytes: options.compactionThresholdBytes ?? defaultOptions.compactionThresholdBytes,
+			maxMemoryBytes: options.maxMemoryBytes ?? defaultOptions.maxMemoryBytes
+		};
+		this.#maxSize = this.#options.maxMemoryBytes / HEAP_BYTES_PER_SIZE;
 		this.#db = db;
 	}
 	close() {
 		this.#checkpoint();
 		this.#db.close();
 	}
-	#get(cgID, opID, key, def) {
-		this.#maybeCheckpoint();
-		const row = this.#stmts.get.get(cgID, opID, key);
-		return row ? JSON.parse(row.val) : def;
+	/** The estimated size of the values that operators keep in memory. */
+	get memoryBytes() {
+		return Math.round(this.#size * HEAP_BYTES_PER_SIZE);
 	}
-	#set(cgID, opID, key, val) {
-		this.#maybeCheckpoint();
-		this.#stmts.set.run(cgID, opID, key, JSON.stringify(val));
+	#get(op, key, def) {
+		op.lastUsed = ++this.#clock;
+		const { values } = op;
+		if (values) {
+			const val = values.get(key);
+			return val === void 0 ? def : val;
+		}
+		let val = op.cache.get(key);
+		if (val === void 0 && !op.cache.has(key)) {
+			this.#maybeCheckpoint();
+			const json = this.#stmts.get.get(op.cgID, op.opID, key);
+			val = json === void 0 ? void 0 : JSON.parse(json);
+			op.cacheSet(key, val);
+		}
+		return val === void 0 ? def : val;
 	}
-	#del(cgID, opID, key) {
+	#set(op, key, val) {
+		op.lastUsed = ++this.#clock;
+		const { values } = op;
+		if (values) {
+			if (values.size === 0) this.#inMemory.add(op);
+			const old = values.get(key);
+			const delta = getSizeOfEntry(key, val) - (old === void 0 ? 0 : getSizeOfEntry(key, old));
+			values.set(key, val);
+			op.size += delta;
+			this.#size += delta;
+			if (this.#size > this.#maxSize) this.#spillLeastRecentlyUsed();
+			return;
+		}
+		this.#write(op, key, val);
+		op.cacheSet(key, val);
+	}
+	#del(op, key) {
+		op.lastUsed = ++this.#clock;
+		const { values } = op;
+		if (values) {
+			const old = values.get(key);
+			if (old !== void 0) {
+				values.delete(key);
+				const size = getSizeOfEntry(key, old);
+				op.size -= size;
+				this.#size -= size;
+			}
+			return;
+		}
 		this.#maybeCheckpoint();
-		this.#stmts.del.run(cgID, opID, key);
+		this.#stmts.del.run(op.cgID, op.opID, key);
+		op.cacheSet(key, void 0);
 	}
-	#destroy(cgID, opID) {
+	#write(op, key, val) {
 		this.#maybeCheckpoint();
-		this.#stmts.clearOp.run(cgID, opID);
+		this.#stmts.set.run(op.cgID, op.opID, key, JSON.stringify(val));
+	}
+	/**
+	* Spills the least recently used operators until the values in memory are
+	* under {@link SPILL_TARGET_PROPORTION} of the budget.
+	*/
+	#spillLeastRecentlyUsed() {
+		const target = this.#maxSize * SPILL_TARGET_PROPORTION;
+		const ops = [];
+		for (const op of this.#inMemory) ops.push(op);
+		ops.sort((a, b) => a.lastUsed - b.lastUsed);
+		for (const op of ops) {
+			if (this.#size <= target) break;
+			this.#spill(op);
+		}
+	}
+	#spill(op) {
+		const { values } = op;
+		if (!values) return;
+		for (const [key, val] of values) this.#write(op, key, val);
+		this.#release(op);
+		op.values = void 0;
+	}
+	/** Drops the values that `op` keeps in memory. */
+	#release(op) {
+		this.#size -= op.size;
+		op.size = 0;
+		op.values?.clear();
+		op.cache.clear();
+		this.#inMemory.delete(op);
+	}
+	#destroy(op) {
+		if (!op.values) {
+			this.#maybeCheckpoint();
+			this.#stmts.clearOp.run(op.cgID, op.opID);
+		}
+		this.#release(op);
 	}
 	/**
 	* We don't need to commit every single write to the DB
@@ -99,16 +251,29 @@ var DatabaseStorage = class DatabaseStorage {
 		this.#stmts.begin.run();
 		this.#numWrites = 0;
 	}
-	*#scan(cgID, opID, opts = { prefix: "" }) {
+	*#scan(op, opts = { prefix: "" }) {
 		const { prefix } = opts;
-		for (const { key, val } of this.#stmts.scan.iterate(cgID, opID, prefix)) {
+		const { values } = op;
+		if (values) {
+			const keys = [];
+			for (const key of values.keys()) if (key.startsWith(prefix)) keys.push(key);
+			keys.sort(compareValues);
+			for (const key of keys) {
+				const val = this.#get(op, key);
+				if (val !== void 0) yield [key, val];
+			}
+			return;
+		}
+		for (const [key, val] of this.#stmts.scan.iterate(op.cgID, op.opID, prefix)) {
 			if (!key.startsWith(prefix)) return;
 			yield [key, JSON.parse(val)];
 		}
 	}
 	createClientGroupStorage(cgID) {
+		const ops = /* @__PURE__ */ new Set();
 		const destroy = () => {
 			this.#stmts.clear.run(cgID);
+			for (const op of ops) this.#release(op);
 			this.#checkpoint();
 			this.#db.compact(this.#options.compactionThresholdBytes);
 		};
@@ -116,13 +281,17 @@ var DatabaseStorage = class DatabaseStorage {
 		let nextOpID = 1;
 		return {
 			createStorage: () => {
-				const opID = nextOpID++;
+				const op = new OpStorage(cgID, nextOpID++, this.#options.maxMemoryBytes > 0);
+				ops.add(op);
 				return {
-					get: (key, def) => this.#get(cgID, opID, key, def),
-					set: (key, val) => this.#set(cgID, opID, key, val),
-					del: (key) => this.#del(cgID, opID, key),
-					scan: (opts) => this.#scan(cgID, opID, opts),
-					destroy: () => this.#destroy(cgID, opID)
+					get: (key, def) => this.#get(op, key, def),
+					set: (key, val) => this.#set(op, key, val),
+					del: (key) => this.#del(op, key),
+					scan: (opts) => this.#scan(op, opts),
+					destroy: () => {
+						this.#destroy(op);
+						ops.delete(op);
+					}
 				};
 			},
 			destroy
@@ -130,6 +299,6 @@ var DatabaseStorage = class DatabaseStorage {
 	}
 };
 //#endregion
-export { DatabaseStorage };
+export { CREATE_STORAGE_TABLE, DatabaseStorage };
 
 //# sourceMappingURL=database-storage.js.map

package/out/zqlite/src/database-storage.js.map

@@ -1 +1 @@
-{"version":3,"file":"database-storage.js","names":["#stmts","#options","#db","#checkpoint","#maybeCheckpoint","#numWrites","#scan","#get","#set","#del","#destroy"],"sources":["../../../../zqlite/src/database-storage.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport type {JSONValue} from '../../shared/src/json.ts';\nimport type {Storage} from '../../zql/src/ivm/operator.ts';\nimport type {Stream} from '../../zql/src/ivm/stream.ts';\nimport type {Statement} from './db.ts';\nimport {Database} from './db.ts';\n\nexport interface ClientGroupStorage {\n  /** Creates a {@link Storage} instance for a single operator. */\n  createStorage(): Storage;\n\n  /** Deletes all storage for the client group. */\n  destroy(): void;\n}\n\ntype Statements = {\n  get: Statement;\n  set: Statement;\n  del: Statement;\n  scan: Statement;\n  clear: Statement;\n  clearOp: Statement;\n  commit: Statement;\n  begin: Statement;\n};\n\n// Exported for testing.\nexport const CREATE_STORAGE_TABLE = `\n  CREATE TABLE storage (\n    clientGroupID TEXT,\n    op NUMBER,\n    key TEXT,\n    val TEXT,\n    PRIMARY KEY(clientGroupID, op, key)\n  )\n  `;\n\nconst defaultOptions = {\n  commitInterval: 5_000,\n  compactionThresholdBytes: 50 * 1024 * 1024,\n};\n\nexport class DatabaseStorage {\n  static create(\n    lc: LogContext,\n    path: string,\n    options = defaultOptions,\n  ): DatabaseStorage {\n    // SQLite is used for ephemeral storage (i.e. similar to RAM) that can spill to\n    // disk to avoid consuming too much memory. Each worker thread gets its own\n    // database (file) and acts as the single reader/writer of the DB, so\n    // `locking_mode` is set to `EXCLUSIVE` for performance. Similarly, since\n    // durability is not important, `synchronous` is set to `OFF` for performance.\n    const db = new Database(lc, path);\n    db.unsafeMode(true); // Allows journal_mode = OFF\n    db.pragma('locking_mode = EXCLUSIVE');\n    db.pragma('foreign_keys = OFF');\n    db.pragma('journal_mode = OFF');\n    db.pragma('synchronous = OFF');\n    db.pragma('auto_vacuum = INCREMENTAL');\n\n    db.prepare(CREATE_STORAGE_TABLE).run();\n    lc.debug?.(`Created DatabaseStorage backed by ${path}`);\n    return new DatabaseStorage(db, options);\n  }\n\n  readonly #stmts: Statements;\n  readonly #options: typeof defaultOptions;\n  readonly #db: Database;\n  #numWrites = 0;\n\n  constructor(db: Database, options = defaultOptions) {\n    this.#stmts = {\n      get: db.prepare(`\n        SELECT val FROM storage WHERE\n          clientGroupID = ? AND op = ? AND key = ?\n      `),\n      set: db.prepare(`\n        INSERT INTO storage (clientGroupID, op, key, val)\n          VALUES(?, ?, ?, ?)\n        ON CONFLICT(clientGroupID, op, key) \n        DO \n          UPDATE SET val = excluded.val\n      `),\n      del: db.prepare(`\n        DELETE FROM storage WHERE\n          clientGroupID = ? AND op = ? AND key = ?\n      `),\n      scan: db.prepare(`\n        SELECT key, val FROM storage WHERE\n          clientGroupID = ? AND op = ? AND key >= ?\n      `),\n      clear: db.prepare(`\n        DELETE FROM storage WHERE clientGroupID = ?\n      `),\n      clearOp: db.prepare(`\n        DELETE FROM storage WHERE clientGroupID = ? AND op = ?\n      `),\n      commit: db.prepare('COMMIT'),\n      begin: db.prepare('BEGIN'),\n    };\n    this.#stmts.begin.run();\n    this.#options = options;\n    this.#db = db;\n  }\n\n  close() {\n    this.#checkpoint();\n    this.#db.close();\n  }\n\n  #get(\n    cgID: string,\n    opID: number,\n    key: string,\n    def?: JSONValue,\n  ): JSONValue | undefined {\n    this.#maybeCheckpoint();\n    const row = this.#stmts.get.get<{val: string}>(cgID, opID, key);\n    return row ? JSON.parse(row.val) : def;\n  }\n\n  #set(cgID: string, opID: number, key: string, val: JSONValue) {\n    this.#maybeCheckpoint();\n    this.#stmts.set.run(cgID, opID, key, JSON.stringify(val));\n  }\n\n  #del(cgID: string, opID: number, key: string) {\n    this.#maybeCheckpoint();\n    this.#stmts.del.run(cgID, opID, key);\n  }\n\n  #destroy(cgID: string, opID: number) {\n    this.#maybeCheckpoint();\n    this.#stmts.clearOp.run(cgID, opID);\n  }\n\n  /**\n   * We don't need to commit every single write to the DB\n   * since we're not concerned with durability.\n   * Waiting on commits can be expensive, so we commit\n   * every `COMMIT_INTERVAL` writes.\n   */\n  #maybeCheckpoint() {\n    if (++this.#numWrites >= this.#options.commitInterval) {\n      this.#checkpoint();\n    }\n  }\n\n  #checkpoint() {\n    this.#stmts.commit.run();\n    this.#stmts.begin.run();\n    this.#numWrites = 0;\n  }\n\n  *#scan(\n    cgID: string,\n    opID: number,\n    opts: {prefix: string} = {prefix: ''},\n  ): Stream<[string, JSONValue]> {\n    const {prefix} = opts;\n    for (const {key, val} of this.#stmts.scan.iterate<{\n      key: string;\n      val: string;\n    }>(cgID, opID, prefix)) {\n      if (!key.startsWith(prefix)) {\n        return;\n      }\n      yield [key, JSON.parse(val)];\n    }\n  }\n\n  createClientGroupStorage(cgID: string): ClientGroupStorage {\n    const destroy = () => {\n      this.#stmts.clear.run(cgID);\n      this.#checkpoint();\n      this.#db.compact(this.#options.compactionThresholdBytes);\n    };\n    this.#stmts.clear.run(cgID);\n\n    let nextOpID = 1;\n    return {\n      createStorage: () => {\n        const opID = nextOpID++;\n        return {\n          get: (key, def?) => this.#get(cgID, opID, key, def),\n          set: (key, val) => this.#set(cgID, opID, key, val),\n          del: key => this.#del(cgID, opID, key),\n          scan: opts => this.#scan(cgID, opID, opts),\n          destroy: () => this.#destroy(cgID, opID),\n        };\n      },\n\n      destroy,\n    };\n  }\n}\n"],"mappings":";;AA2BA,IAAa,uBAAuB;;;;;;;;;AAUpC,IAAM,iBAAiB;CACrB,gBAAgB;CAChB,0BAA0B,KAAK,OAAO;AACxC;AAEA,IAAa,kBAAb,MAAa,gBAAgB;CAC3B,OAAO,OACL,IACA,MACA,UAAU,gBACO;EAMjB,MAAM,KAAK,IAAI,SAAS,IAAI,IAAI;EAChC,GAAG,WAAW,IAAI;EAClB,GAAG,OAAO,0BAA0B;EACpC,GAAG,OAAO,oBAAoB;EAC9B,GAAG,OAAO,oBAAoB;EAC9B,GAAG,OAAO,mBAAmB;EAC7B,GAAG,OAAO,2BAA2B;EAErC,GAAG,QAAQ,oBAAoB,EAAE,IAAI;EACrC,GAAG,QAAQ,qCAAqC,MAAM;EACtD,OAAO,IAAI,gBAAgB,IAAI,OAAO;CACxC;CAEA;CACA;CACA;CACA,aAAa;CAEb,YAAY,IAAc,UAAU,gBAAgB;EAClD,KAAKA,SAAS;GACZ,KAAK,GAAG,QAAQ;;;OAGf;GACD,KAAK,GAAG,QAAQ;;;;;;OAMf;GACD,KAAK,GAAG,QAAQ;;;OAGf;GACD,MAAM,GAAG,QAAQ;;;OAGhB;GACD,OAAO,GAAG,QAAQ;;OAEjB;GACD,SAAS,GAAG,QAAQ;;OAEnB;GACD,QAAQ,GAAG,QAAQ,QAAQ;GAC3B,OAAO,GAAG,QAAQ,OAAO;EAC3B;EACA,KAAKA,OAAO,MAAM,IAAI;EACtB,KAAKC,WAAW;EAChB,KAAKC,MAAM;CACb;CAEA,QAAQ;EACN,KAAKC,YAAY;EACjB,KAAKD,IAAI,MAAM;CACjB;CAEA,KACE,MACA,MACA,KACA,KACuB;EACvB,KAAKE,iBAAiB;EACtB,MAAM,MAAM,KAAKJ,OAAO,IAAI,IAAmB,MAAM,MAAM,GAAG;EAC9D,OAAO,MAAM,KAAK,MAAM,IAAI,GAAG,IAAI;CACrC;CAEA,KAAK,MAAc,MAAc,KAAa,KAAgB;EAC5D,KAAKI,iBAAiB;EACtB,KAAKJ,OAAO,IAAI,IAAI,MAAM,MAAM,KAAK,KAAK,UAAU,GAAG,CAAC;CAC1D;CAEA,KAAK,MAAc,MAAc,KAAa;EAC5C,KAAKI,iBAAiB;EACtB,KAAKJ,OAAO,IAAI,IAAI,MAAM,MAAM,GAAG;CACrC;CAEA,SAAS,MAAc,MAAc;EACnC,KAAKI,iBAAiB;EACtB,KAAKJ,OAAO,QAAQ,IAAI,MAAM,IAAI;CACpC;;;;;;;CAQA,mBAAmB;EACjB,IAAI,EAAE,KAAKK,cAAc,KAAKJ,SAAS,gBACrC,KAAKE,YAAY;CAErB;CAEA,cAAc;EACZ,KAAKH,OAAO,OAAO,IAAI;EACvB,KAAKA,OAAO,MAAM,IAAI;EACtB,KAAKK,aAAa;CACpB;CAEA,CAACC,MACC,MACA,MACA,OAAyB,EAAC,QAAQ,GAAE,GACP;EAC7B,MAAM,EAAC,WAAU;EACjB,KAAK,MAAM,EAAC,KAAK,SAAQ,KAAKN,OAAO,KAAK,QAGvC,MAAM,MAAM,MAAM,GAAG;GACtB,IAAI,CAAC,IAAI,WAAW,MAAM,GACxB;GAEF,MAAM,CAAC,KAAK,KAAK,MAAM,GAAG,CAAC;EAC7B;CACF;CAEA,yBAAyB,MAAkC;EACzD,MAAM,gBAAgB;GACpB,KAAKA,OAAO,MAAM,IAAI,IAAI;GAC1B,KAAKG,YAAY;GACjB,KAAKD,IAAI,QAAQ,KAAKD,SAAS,wBAAwB;EACzD;EACA,KAAKD,OAAO,MAAM,IAAI,IAAI;EAE1B,IAAI,WAAW;EACf,OAAO;GACL,qBAAqB;IACnB,MAAM,OAAO;IACb,OAAO;KACL,MAAM,KAAK,QAAS,KAAKO,KAAK,MAAM,MAAM,KAAK,GAAG;KAClD,MAAM,KAAK,QAAQ,KAAKC,KAAK,MAAM,MAAM,KAAK,GAAG;KACjD,MAAK,QAAO,KAAKC,KAAK,MAAM,MAAM,GAAG;KACrC,OAAM,SAAQ,KAAKH,MAAM,MAAM,MAAM,IAAI;KACzC,eAAe,KAAKI,SAAS,MAAM,IAAI;IACzC;GACF;GAEA;EACF;CACF;AACF"}
+{"version":3,"file":"database-storage.js","names":[],"sources":["../../../../zqlite/src/database-storage.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport type {JSONValue} from '../../shared/src/json.ts';\nimport {getSizeOfEntry} from '../../shared/src/size-of-value.ts';\nimport {compareValues} from '../../zql/src/ivm/data.ts';\nimport type {Storage} from '../../zql/src/ivm/operator.ts';\nimport type {Stream} from '../../zql/src/ivm/stream.ts';\nimport type {Statement} from './db.ts';\nimport {Database} from './db.ts';\n\nexport interface ClientGroupStorage {\n  /** Creates a {@link Storage} instance for a single operator. */\n  createStorage(): Storage;\n\n  /** Deletes all storage for the client group. */\n  destroy(): void;\n}\n\ntype Statements = {\n  get: Statement;\n  set: Statement;\n  del: Statement;\n  scan: Statement;\n  clear: Statement;\n  clearOp: Statement;\n  commit: Statement;\n  begin: Statement;\n};\n\n// Exported for testing.\nexport const CREATE_STORAGE_TABLE = `\n  CREATE TABLE storage (\n    clientGroupID TEXT,\n    op NUMBER,\n    key TEXT,\n    val TEXT,\n    PRIMARY KEY(clientGroupID, op, key)\n  )\n  `;\n\n/**\n * The most values the storage of a spilled operator keeps in memory. Profiles\n * of the view-syncer gain as much from 64 as from 1,024.\n */\nconst MAX_CACHED_ENTRIES_PER_OP = 64;\n\n/**\n * When the operators' values go over `maxMemoryBytes`, operators are spilled\n * until they are back under this proportion of it, so that spilling, which\n * sorts the operators by last use, happens in batches.\n */\nconst SPILL_TARGET_PROPORTION = 0.9;\n\n/**\n * The heap bytes that operator state takes per byte of {@link getSizeOfEntry}.\n * Measured with `heapUsed` on view-syncer operator state, mostly Cap's\n * `{size, pks}` and Take's `{size, bound}`.\n */\nconst HEAP_BYTES_PER_SIZE = 2.6;\n\nexport type DatabaseStorageOptions = {\n  commitInterval?: number | undefined;\n  compactionThresholdBytes?: number | undefined;\n  /**\n   * The most heap bytes, as estimated from {@link getSizeOfEntry}, that the\n   * operators of all client groups keep in memory before the least recently\n   * used ones spill to SQLite. 0, the default, keeps every operator's values\n   * in SQLite.\n   */\n  maxMemoryBytes?: number | undefined;\n};\n\nconst defaultOptions = {\n  commitInterval: 5_000,\n  compactionThresholdBytes: 50 * 1024 * 1024,\n  maxMemoryBytes: 0,\n};\n\n/**\n * The storage of one operator. Its values live in `values` until the\n * operator is spilled, and in SQLite afterwards, with the most recently used\n * of them in `cache`.\n */\nclass OpStorage {\n  readonly cgID: string;\n  readonly opID: number;\n  /** `undefined` once spilled. */\n  values: Map<string, JSONValue> | undefined;\n  /** The {@link getSizeOfEntry} of `values`. */\n  size = 0;\n  lastUsed = 0;\n  /**\n   * Once spilled, the operator's most recently read and written values,\n   * `undefined` for a key known to be absent. Operators read the same key\n   * again and again (Cap and Take read their state on every fetch and push),\n   * so this saves most of the SQLite reads.\n   */\n  readonly cache = new Map<string, JSONValue | undefined>();\n\n  constructor(cgID: string, opID: number, inMemory: boolean) {\n    this.cgID = cgID;\n    this.opID = opID;\n    this.values = inMemory ? new Map() : undefined;\n  }\n\n  cacheSet(key: string, val: JSONValue | undefined) {\n    const {cache} = this;\n    if (cache.size >= MAX_CACHED_ENTRIES_PER_OP && !cache.has(key)) {\n      cache.clear();\n    }\n    cache.set(key, val);\n  }\n}\n\n/**\n * Operator storage for the client groups of a sync worker.\n *\n * Each operator keeps its values in a `Map`, so reading a key that was never\n * written does not ask SQLite. When the values of all of the operators go\n * over `maxMemoryBytes`, the least recently used operators spill: their\n * values are written to SQLite, where they stay until the operator is\n * destroyed.\n *\n * Like `MemoryStorage`, `get` returns the stored value itself, which\n * operators do not mutate.\n */\nexport class DatabaseStorage {\n  static create(\n    lc: LogContext,\n    path: string,\n    options: DatabaseStorageOptions = {},\n  ): DatabaseStorage {\n    // SQLite is used for ephemeral storage (i.e. similar to RAM) that can spill to\n    // disk to avoid consuming too much memory. Each worker thread gets its own\n    // database (file) and acts as the single reader/writer of the DB, so\n    // `locking_mode` is set to `EXCLUSIVE` for performance. Similarly, since\n    // durability is not important, `synchronous` is set to `OFF` for performance.\n    const db = new Database(lc, path);\n    db.unsafeMode(true); // Allows journal_mode = OFF\n    db.pragma('locking_mode = EXCLUSIVE');\n    db.pragma('foreign_keys = OFF');\n    db.pragma('journal_mode = OFF');\n    db.pragma('synchronous = OFF');\n    db.pragma('auto_vacuum = INCREMENTAL');\n\n    db.prepare(CREATE_STORAGE_TABLE).run();\n    lc.debug?.(`Created DatabaseStorage backed by ${path}`);\n    return new DatabaseStorage(db, options);\n  }\n\n  readonly #stmts: Statements;\n  readonly #options: typeof defaultOptions;\n  readonly #db: Database;\n  #numWrites = 0;\n\n  /** The operators whose values are in memory. */\n  readonly #inMemory = new Set<OpStorage>();\n  /** The {@link getSizeOfEntry} of the values of the operators in `#inMemory`. */\n  #size = 0;\n  /** `maxMemoryBytes` in units of {@link getSizeOfEntry}. */\n  readonly #maxSize: number;\n  /** Orders the operators by last use. */\n  #clock = 0;\n\n  constructor(db: Database, options: DatabaseStorageOptions = {}) {\n    this.#stmts = {\n      get: db\n        .prepare(\n          `\n        SELECT val FROM storage WHERE\n          clientGroupID = ? AND op = ? AND key = ?\n      `,\n        )\n        .pluck(true),\n      set: db.prepare(`\n        INSERT INTO storage (clientGroupID, op, key, val)\n          VALUES(?, ?, ?, ?)\n        ON CONFLICT(clientGroupID, op, key) \n        DO \n          UPDATE SET val = excluded.val\n      `),\n      del: db.prepare(`\n        DELETE FROM storage WHERE\n          clientGroupID = ? AND op = ? AND key = ?\n      `),\n      scan: db\n        .prepare(\n          `\n        SELECT key, val FROM storage WHERE\n          clientGroupID = ? AND op = ? AND key >= ?\n      `,\n        )\n        .raw(true),\n      clear: db.prepare(`\n        DELETE FROM storage WHERE clientGroupID = ?\n      `),\n      clearOp: db.prepare(`\n        DELETE FROM storage WHERE clientGroupID = ? AND op = ?\n      `),\n      commit: db.prepare('COMMIT'),\n      begin: db.prepare('BEGIN'),\n    };\n    this.#stmts.begin.run();\n    this.#options = {\n      commitInterval: options.commitInterval ?? defaultOptions.commitInterval,\n      compactionThresholdBytes:\n        options.compactionThresholdBytes ??\n        defaultOptions.compactionThresholdBytes,\n      maxMemoryBytes: options.maxMemoryBytes ?? defaultOptions.maxMemoryBytes,\n    };\n    this.#maxSize = this.#options.maxMemoryBytes / HEAP_BYTES_PER_SIZE;\n    this.#db = db;\n  }\n\n  close() {\n    this.#checkpoint();\n    this.#db.close();\n  }\n\n  /** The estimated size of the values that operators keep in memory. */\n  get memoryBytes(): number {\n    return Math.round(this.#size * HEAP_BYTES_PER_SIZE);\n  }\n\n  #get(op: OpStorage, key: string, def?: JSONValue): JSONValue | undefined {\n    op.lastUsed = ++this.#clock;\n    const {values} = op;\n    if (values) {\n      const val = values.get(key);\n      return val === undefined ? def : val;\n    }\n    let val = op.cache.get(key);\n    if (val === undefined && !op.cache.has(key)) {\n      this.#maybeCheckpoint();\n      const json = this.#stmts.get.get<string | undefined>(\n        op.cgID,\n        op.opID,\n        key,\n      );\n      val = json === undefined ? undefined : JSON.parse(json);\n      op.cacheSet(key, val);\n    }\n    return val === undefined ? def : val;\n  }\n\n  #set(op: OpStorage, key: string, val: JSONValue) {\n    op.lastUsed = ++this.#clock;\n    const {values} = op;\n    if (values) {\n      if (values.size === 0) {\n        // The operator may have been released by destroy().\n        this.#inMemory.add(op);\n      }\n      const old = values.get(key);\n      const delta =\n        getSizeOfEntry(key, val) -\n        (old === undefined ? 0 : getSizeOfEntry(key, old));\n      values.set(key, val);\n      op.size += delta;\n      this.#size += delta;\n      if (this.#size > this.#maxSize) {\n        this.#spillLeastRecentlyUsed();\n      }\n      return;\n    }\n    this.#write(op, key, val);\n    // The cache is updated only once the database is, so a failed write\n    // leaves it consistent with the database.\n    op.cacheSet(key, val);\n  }\n\n  #del(op: OpStorage, key: string) {\n    op.lastUsed = ++this.#clock;\n    const {values} = op;\n    if (values) {\n      const old = values.get(key);\n      if (old !== undefined) {\n        values.delete(key);\n        const size = getSizeOfEntry(key, old);\n        op.size -= size;\n        this.#size -= size;\n      }\n      return;\n    }\n    this.#maybeCheckpoint();\n    this.#stmts.del.run(op.cgID, op.opID, key);\n    op.cacheSet(key, undefined);\n  }\n\n  #write(op: OpStorage, key: string, val: JSONValue) {\n    this.#maybeCheckpoint();\n    this.#stmts.set.run(op.cgID, op.opID, key, JSON.stringify(val));\n  }\n\n  /**\n   * Spills the least recently used operators until the values in memory are\n   * under {@link SPILL_TARGET_PROPORTION} of the budget.\n   */\n  #spillLeastRecentlyUsed() {\n    const target = this.#maxSize * SPILL_TARGET_PROPORTION;\n    const ops: OpStorage[] = [];\n    for (const op of this.#inMemory) {\n      ops.push(op);\n    }\n    ops.sort((a, b) => a.lastUsed - b.lastUsed);\n    for (const op of ops) {\n      if (this.#size <= target) {\n        break;\n      }\n      this.#spill(op);\n    }\n  }\n\n  #spill(op: OpStorage) {\n    const {values} = op;\n    if (!values) {\n      return;\n    }\n    for (const [key, val] of values) {\n      this.#write(op, key, val);\n    }\n    this.#release(op);\n    op.values = undefined;\n  }\n\n  /** Drops the values that `op` keeps in memory. */\n  #release(op: OpStorage) {\n    this.#size -= op.size;\n    op.size = 0;\n    op.values?.clear();\n    op.cache.clear();\n    this.#inMemory.delete(op);\n  }\n\n  #destroy(op: OpStorage) {\n    if (!op.values) {\n      this.#maybeCheckpoint();\n      this.#stmts.clearOp.run(op.cgID, op.opID);\n    }\n    this.#release(op);\n  }\n\n  /**\n   * We don't need to commit every single write to the DB\n   * since we're not concerned with durability.\n   * Waiting on commits can be expensive, so we commit\n   * every `COMMIT_INTERVAL` writes.\n   */\n  #maybeCheckpoint() {\n    if (++this.#numWrites >= this.#options.commitInterval) {\n      this.#checkpoint();\n    }\n  }\n\n  #checkpoint() {\n    this.#stmts.commit.run();\n    this.#stmts.begin.run();\n    this.#numWrites = 0;\n  }\n\n  *#scan(\n    op: OpStorage,\n    opts: {prefix: string} = {prefix: ''},\n  ): Stream<[string, JSONValue]> {\n    const {prefix} = opts;\n    const {values} = op;\n    if (values) {\n      const keys: string[] = [];\n      for (const key of values.keys()) {\n        if (key.startsWith(prefix)) {\n          keys.push(key);\n        }\n      }\n      // Sorted like SQLite's BINARY collation, which compares UTF-8 bytes.\n      keys.sort(compareValues);\n      for (const key of keys) {\n        // Read through #get, which skips keys deleted since and still works\n        // if the operator spills while the scan is suspended.\n        const val = this.#get(op, key);\n        if (val !== undefined) {\n          yield [key, val];\n        }\n      }\n      return;\n    }\n    for (const [key, val] of this.#stmts.scan.iterate<[string, string]>(\n      op.cgID,\n      op.opID,\n      prefix,\n    )) {\n      if (!key.startsWith(prefix)) {\n        return;\n      }\n      yield [key, JSON.parse(val)];\n    }\n  }\n\n  createClientGroupStorage(cgID: string): ClientGroupStorage {\n    // The group's live operator storages.\n    const ops = new Set<OpStorage>();\n    const destroy = () => {\n      this.#stmts.clear.run(cgID);\n      for (const op of ops) {\n        this.#release(op);\n      }\n      this.#checkpoint();\n      this.#db.compact(this.#options.compactionThresholdBytes);\n    };\n    this.#stmts.clear.run(cgID);\n\n    let nextOpID = 1;\n    return {\n      createStorage: () => {\n        const op = new OpStorage(\n          cgID,\n          nextOpID++,\n          this.#options.maxMemoryBytes > 0,\n        );\n        ops.add(op);\n        return {\n          get: (key, def?) => this.#get(op, key, def),\n          set: (key, val) => this.#set(op, key, val),\n          del: key => this.#del(op, key),\n          scan: opts => this.#scan(op, opts),\n          destroy: () => {\n            this.#destroy(op);\n            ops.delete(op);\n          },\n        };\n      },\n\n      destroy,\n    };\n  }\n}\n"],"mappings":";;;;AA6BA,IAAa,uBAAuB;;;;;;;;;;;;;AAcpC,IAAM,4BAA4B;;;;;;AAOlC,IAAM,0BAA0B;;;;;;AAOhC,IAAM,sBAAsB;AAc5B,IAAM,iBAAiB;CACrB,gBAAgB;CAChB,0BAA0B;CAC1B,gBAAgB;AAClB;;;;;;AAOA,IAAM,YAAN,MAAgB;CACd;CACA;;CAEA;;CAEA,OAAO;CACP,WAAW;;;;;;;CAOX,wBAAiB,IAAI,IAAmC;CAExD,YAAY,MAAc,MAAc,UAAmB;EACzD,KAAK,OAAO;EACZ,KAAK,OAAO;EACZ,KAAK,SAAS,2BAAW,IAAI,IAAI,IAAI,KAAA;CACvC;CAEA,SAAS,KAAa,KAA4B;EAChD,MAAM,EAAC,UAAS;EAChB,IAAI,MAAM,QAAQ,6BAA6B,CAAC,MAAM,IAAI,GAAG,GAC3D,MAAM,MAAM;EAEd,MAAM,IAAI,KAAK,GAAG;CACpB;AACF;;;;;;;;;;;;;AAcA,IAAa,kBAAb,MAAa,gBAAgB;CAC3B,OAAO,OACL,IACA,MACA,UAAkC,CAAC,GAClB;EAMjB,MAAM,KAAK,IAAI,SAAS,IAAI,IAAI;EAChC,GAAG,WAAW,IAAI;EAClB,GAAG,OAAO,0BAA0B;EACpC,GAAG,OAAO,oBAAoB;EAC9B,GAAG,OAAO,oBAAoB;EAC9B,GAAG,OAAO,mBAAmB;EAC7B,GAAG,OAAO,2BAA2B;EAErC,GAAG,QAAQ,oBAAoB,CAAC,CAAC,IAAI;EACrC,GAAG,QAAQ,qCAAqC,MAAM;EACtD,OAAO,IAAI,gBAAgB,IAAI,OAAO;CACxC;CAEA;CACA;CACA;CACA,aAAa;;CAGb,4BAAqB,IAAI,IAAe;;CAExC,QAAQ;;CAER;;CAEA,SAAS;CAET,YAAY,IAAc,UAAkC,CAAC,GAAG;EAC9D,KAAK,SAAS;GACZ,KAAK,GACF,QACC;;;OAIF,CAAC,CACA,MAAM,IAAI;GACb,KAAK,GAAG,QAAQ;;;;;;OAMf;GACD,KAAK,GAAG,QAAQ;;;OAGf;GACD,MAAM,GACH,QACC;;;OAIF,CAAC,CACA,IAAI,IAAI;GACX,OAAO,GAAG,QAAQ;;OAEjB;GACD,SAAS,GAAG,QAAQ;;OAEnB;GACD,QAAQ,GAAG,QAAQ,QAAQ;GAC3B,OAAO,GAAG,QAAQ,OAAO;EAC3B;EACA,KAAK,OAAO,MAAM,IAAI;EACtB,KAAK,WAAW;GACd,gBAAgB,QAAQ,kBAAkB,eAAe;GACzD,0BACE,QAAQ,4BACR,eAAe;GACjB,gBAAgB,QAAQ,kBAAkB,eAAe;EAC3D;EACA,KAAK,WAAW,KAAK,SAAS,iBAAiB;EAC/C,KAAK,MAAM;CACb;CAEA,QAAQ;EACN,KAAK,YAAY;EACjB,KAAK,IAAI,MAAM;CACjB;;CAGA,IAAI,cAAsB;EACxB,OAAO,KAAK,MAAM,KAAK,QAAQ,mBAAmB;CACpD;CAEA,KAAK,IAAe,KAAa,KAAwC;EACvE,GAAG,WAAW,EAAE,KAAK;EACrB,MAAM,EAAC,WAAU;EACjB,IAAI,QAAQ;GACV,MAAM,MAAM,OAAO,IAAI,GAAG;GAC1B,OAAO,QAAQ,KAAA,IAAY,MAAM;EACnC;EACA,IAAI,MAAM,GAAG,MAAM,IAAI,GAAG;EAC1B,IAAI,QAAQ,KAAA,KAAa,CAAC,GAAG,MAAM,IAAI,GAAG,GAAG;GAC3C,KAAK,iBAAiB;GACtB,MAAM,OAAO,KAAK,OAAO,IAAI,IAC3B,GAAG,MACH,GAAG,MACH,GACF;GACA,MAAM,SAAS,KAAA,IAAY,KAAA,IAAY,KAAK,MAAM,IAAI;GACtD,GAAG,SAAS,KAAK,GAAG;EACtB;EACA,OAAO,QAAQ,KAAA,IAAY,MAAM;CACnC;CAEA,KAAK,IAAe,KAAa,KAAgB;EAC/C,GAAG,WAAW,EAAE,KAAK;EACrB,MAAM,EAAC,WAAU;EACjB,IAAI,QAAQ;GACV,IAAI,OAAO,SAAS,GAElB,KAAK,UAAU,IAAI,EAAE;GAEvB,MAAM,MAAM,OAAO,IAAI,GAAG;GAC1B,MAAM,QACJ,eAAe,KAAK,GAAG,KACtB,QAAQ,KAAA,IAAY,IAAI,eAAe,KAAK,GAAG;GAClD,OAAO,IAAI,KAAK,GAAG;GACnB,GAAG,QAAQ;GACX,KAAK,SAAS;GACd,IAAI,KAAK,QAAQ,KAAK,UACpB,KAAK,wBAAwB;GAE/B;EACF;EACA,KAAK,OAAO,IAAI,KAAK,GAAG;EAGxB,GAAG,SAAS,KAAK,GAAG;CACtB;CAEA,KAAK,IAAe,KAAa;EAC/B,GAAG,WAAW,EAAE,KAAK;EACrB,MAAM,EAAC,WAAU;EACjB,IAAI,QAAQ;GACV,MAAM,MAAM,OAAO,IAAI,GAAG;GAC1B,IAAI,QAAQ,KAAA,GAAW;IACrB,OAAO,OAAO,GAAG;IACjB,MAAM,OAAO,eAAe,KAAK,GAAG;IACpC,GAAG,QAAQ;IACX,KAAK,SAAS;GAChB;GACA;EACF;EACA,KAAK,iBAAiB;EACtB,KAAK,OAAO,IAAI,IAAI,GAAG,MAAM,GAAG,MAAM,GAAG;EACzC,GAAG,SAAS,KAAK,KAAA,CAAS;CAC5B;CAEA,OAAO,IAAe,KAAa,KAAgB;EACjD,KAAK,iBAAiB;EACtB,KAAK,OAAO,IAAI,IAAI,GAAG,MAAM,GAAG,MAAM,KAAK,KAAK,UAAU,GAAG,CAAC;CAChE;;;;;CAMA,0BAA0B;EACxB,MAAM,SAAS,KAAK,WAAW;EAC/B,MAAM,MAAmB,CAAC;EAC1B,KAAK,MAAM,MAAM,KAAK,WACpB,IAAI,KAAK,EAAE;EAEb,IAAI,MAAM,GAAG,MAAM,EAAE,WAAW,EAAE,QAAQ;EAC1C,KAAK,MAAM,MAAM,KAAK;GACpB,IAAI,KAAK,SAAS,QAChB;GAEF,KAAK,OAAO,EAAE;EAChB;CACF;CAEA,OAAO,IAAe;EACpB,MAAM,EAAC,WAAU;EACjB,IAAI,CAAC,QACH;EAEF,KAAK,MAAM,CAAC,KAAK,QAAQ,QACvB,KAAK,OAAO,IAAI,KAAK,GAAG;EAE1B,KAAK,SAAS,EAAE;EAChB,GAAG,SAAS,KAAA;CACd;;CAGA,SAAS,IAAe;EACtB,KAAK,SAAS,GAAG;EACjB,GAAG,OAAO;EACV,GAAG,QAAQ,MAAM;EACjB,GAAG,MAAM,MAAM;EACf,KAAK,UAAU,OAAO,EAAE;CAC1B;CAEA,SAAS,IAAe;EACtB,IAAI,CAAC,GAAG,QAAQ;GACd,KAAK,iBAAiB;GACtB,KAAK,OAAO,QAAQ,IAAI,GAAG,MAAM,GAAG,IAAI;EAC1C;EACA,KAAK,SAAS,EAAE;CAClB;;;;;;;CAQA,mBAAmB;EACjB,IAAI,EAAE,KAAK,cAAc,KAAK,SAAS,gBACrC,KAAK,YAAY;CAErB;CAEA,cAAc;EACZ,KAAK,OAAO,OAAO,IAAI;EACvB,KAAK,OAAO,MAAM,IAAI;EACtB,KAAK,aAAa;CACpB;CAEA,CAAC,MACC,IACA,OAAyB,EAAC,QAAQ,GAAE,GACP;EAC7B,MAAM,EAAC,WAAU;EACjB,MAAM,EAAC,WAAU;EACjB,IAAI,QAAQ;GACV,MAAM,OAAiB,CAAC;GACxB,KAAK,MAAM,OAAO,OAAO,KAAK,GAC5B,IAAI,IAAI,WAAW,MAAM,GACvB,KAAK,KAAK,GAAG;GAIjB,KAAK,KAAK,aAAa;GACvB,KAAK,MAAM,OAAO,MAAM;IAGtB,MAAM,MAAM,KAAK,KAAK,IAAI,GAAG;IAC7B,IAAI,QAAQ,KAAA,GACV,MAAM,CAAC,KAAK,GAAG;GAEnB;GACA;EACF;EACA,KAAK,MAAM,CAAC,KAAK,QAAQ,KAAK,OAAO,KAAK,QACxC,GAAG,MACH,GAAG,MACH,MACF,GAAG;GACD,IAAI,CAAC,IAAI,WAAW,MAAM,GACxB;GAEF,MAAM,CAAC,KAAK,KAAK,MAAM,GAAG,CAAC;EAC7B;CACF;CAEA,yBAAyB,MAAkC;EAEzD,MAAM,sBAAM,IAAI,IAAe;EAC/B,MAAM,gBAAgB;GACpB,KAAK,OAAO,MAAM,IAAI,IAAI;GAC1B,KAAK,MAAM,MAAM,KACf,KAAK,SAAS,EAAE;GAElB,KAAK,YAAY;GACjB,KAAK,IAAI,QAAQ,KAAK,SAAS,wBAAwB;EACzD;EACA,KAAK,OAAO,MAAM,IAAI,IAAI;EAE1B,IAAI,WAAW;EACf,OAAO;GACL,qBAAqB;IACnB,MAAM,KAAK,IAAI,UACb,MACA,YACA,KAAK,SAAS,iBAAiB,CACjC;IACA,IAAI,IAAI,EAAE;IACV,OAAO;KACL,MAAM,KAAK,QAAS,KAAK,KAAK,IAAI,KAAK,GAAG;KAC1C,MAAM,KAAK,QAAQ,KAAK,KAAK,IAAI,KAAK,GAAG;KACzC,MAAK,QAAO,KAAK,KAAK,IAAI,GAAG;KAC7B,OAAM,SAAQ,KAAK,MAAM,IAAI,IAAI;KACjC,eAAe;MACb,KAAK,SAAS,EAAE;MAChB,IAAI,OAAO,EAAE;KACf;IACF;GACF;GAEA;EACF;CACF;AACF"}

package/out/zqlite/src/db.d.ts

@@ -1,6 +1,6 @@
 import { type Attributes } from '@opentelemetry/api';
 import type { LogContext } from '@rocicorp/logger';
-import SQLite3Database, { type RunResult, type Statement as SQLite3Statement } from '@rocicorp/zero-sqlite3';
+import SQLite3Database, { type ColumnDefinition, type RunResult, type Statement as SQLite3Statement, type StatementIterator as SQLite3StatementIterator } from '@rocicorp/zero-sqlite3';
 export declare class Database implements Disposable {
     #private;
     constructor(lc: LogContext, path: string, options?: SQLite3Database.Options, slowQueryThreshold?: number);
@@ -37,10 +37,55 @@ export declare class Statement {
     readonly scanStatusReset: SQLite3Statement['scanStatusReset'];
     constructor(lc: LogContext, attrs: Attributes, stmt: SQLite3Statement, threshold: number);
     safeIntegers(useBigInt: boolean): this;
+    /**
+     * Return rows as arrays of column values instead of objects. Turning it
+     * off has no effect in pluck mode, as in better-sqlite3.
+     */
+    raw(enabled: boolean): this;
+    /**
+     * Return the first column of each row instead of the row. Turning it off
+     * has no effect in raw mode, as in better-sqlite3.
+     */
+    pluck(enabled: boolean): this;
+    /** The result columns, in the order of the values in a raw row. */
+    columns(): ColumnDefinition[];
     run(...params: unknown[]): RunResult;
     get<T>(...params: unknown[]): T;
     all<T>(...params: unknown[]): T[];
     iterate<T>(...params: unknown[]): IterableIterator<T>;
+    /**
+     * Reads the statement's rows in batches. Integers are read as with
+     * `safeIntegers(true)`. The cursor must be {@link BatchCursor.close closed}.
+     */
+    batches(params: readonly unknown[]): BatchCursor;
+}
+/**
+ * Reads a statement's rows a batch at a time with one native call per batch
+ * (`nextBatch`), instead of one per row as `next()` does.
+ */
+export declare class BatchCursor {
+    #private;
+    constructor(log: IterationLog, it: SQLite3StatementIterator, columns: number);
+    /**
+     * Reads up to `capacity` rows and returns their values in one row-major
+     * array, empty once the rows are exhausted.
+     */
+    next(capacity: number): (bigint | number | string | null)[];
+    /**
+     * Ends the cursor, abandoning any rows that were not read, and logs if the
+     * iteration was slow. Must be called once the rows are consumed.
+     */
+    close(): void;
+}
+/**
+ * Logs slow iterations: once the time spent in SQLite, or the total time
+ * including the consumer's work between rows, reaches the threshold.
+ */
+declare class IterationLog {
+    #private;
+    constructor(lc: LogContext, attrs: Attributes, slowQueryThreshold: number);
+    addSQLiteTime(elapsed: number): void;
+    log(): void;
 }
 /**
  * An error indicating that the Database failed to open. This essentially
@@ -49,4 +94,5 @@ export declare class Statement {
  */
 export declare class DatabaseInitError extends Error {
 }
+export {};
 //# sourceMappingURL=db.d.ts.map

package/out/zqlite/src/db.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"db.d.ts","sourceRoot":"","sources":["../../../../zqlite/src/db.ts"],"names":[],"mappings":"AAAA,OAAO,EAAQ,KAAK,UAAU,EAAC,MAAM,oBAAoB,CAAC;AAC1D,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AACjD,OAAO,eAAe,EAAE,EAEtB,KAAK,SAAS,EACd,KAAK,SAAS,IAAI,gBAAgB,EACnC,MAAM,wBAAwB,CAAC;AAehC,qBAAa,QAAS,YAAW,UAAU;;IAMzC,YACE,EAAE,EAAE,UAAU,EACd,IAAI,EAAE,MAAM,EACZ,OAAO,CAAC,EAAE,eAAe,CAAC,OAAO,EACjC,kBAAkB,SAAM,EA4BzB;IAED,OAAO,CAAC,GAAG,EAAE,MAAM,GAAG,SAAS,CAY9B;IAED,IAAI,CAAC,GAAG,EAAE,MAAM,GAAG,IAAI,CAEtB;IAED,MAAM,CAAC,CAAC,GAAG,OAAO,EAAE,GAAG,EAAE,MAAM,GAAG,CAAC,EAAE,CAEpC;IAMD,OAAO,CAAC,sBAAsB,EAAE,MAAM,QAuCrC;IAED,UAAU,CAAC,MAAM,EAAE,OAAO,QAEzB;IAED;;;;;;OAMG;IACH,UAAU,CAAC,OAAO,EAAE,OAAO,QAE1B;IAmBD;;;;;OAKG;IACH,KAAK,CAAC,EAAC,QAAe,EAAC,GAAE;QAAC,QAAQ,CAAC,EAAE,OAAO,GAAG,SAAS,CAAA;KAAM,GAAG,IAAI,CAcpE;IAED,WAAW,CAAC,CAAC,EAAE,EAAE,EAAE,MAAM,CAAC,GAAG,CAAC,CAE7B;IAED,IAAI,IAAI,WAEP;IAED,IAAI,aAAa,YAEhB;IAED,CAAC,MAAM,CAAC,OAAO,CAAC,IAAI,IAAI,CAEvB;CACF;AAED,qBAAa,SAAS;;IAKpB,QAAQ,CAAC,UAAU,EAAE,gBAAgB,CAAC,cAAc,CAAC,CAAC;IACtD,QAAQ,CAAC,eAAe,EAAE,gBAAgB,CAAC,iBAAiB,CAAC,CAAC;IAE9D,YACE,EAAE,EAAE,UAAU,EACd,KAAK,EAAE,UAAU,EACjB,IAAI,EAAE,gBAAgB,EACtB,SAAS,EAAE,MAAM,EAQlB;IAED,YAAY,CAAC,SAAS,EAAE,OAAO,GAAG,IAAI,CAGrC;IAED,GAAG,CAAC,GAAG,MAAM,EAAE,OAAO,EAAE,GAAG,SAAS,CAKnC;IAED,GAAG,CAAC,CAAC,EAAE,GAAG,MAAM,EAAE,OAAO,EAAE,GAAG,CAAC,CAK9B;IAED,GAAG,CAAC,CAAC,EAAE,GAAG,MAAM,EAAE,OAAO,EAAE,GAAG,CAAC,EAAE,CAKhC;IAWD,OAAO,CAAC,CAAC,EAAE,GAAG,MAAM,EAAE,OAAO,EAAE,GAAG,gBAAgB,CAAC,CAAC,CAAC,CAOpD;CACF;AAmID;;;;GAIG;AACH,qBAAa,iBAAkB,SAAQ,KAAK;CAAG"}
+{"version":3,"file":"db.d.ts","sourceRoot":"","sources":["../../../../zqlite/src/db.ts"],"names":[],"mappings":"AAAA,OAAO,EAAQ,KAAK,UAAU,EAAC,MAAM,oBAAoB,CAAC;AAC1D,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AACjD,OAAO,eAAe,EAAE,EAEtB,KAAK,gBAAgB,EACrB,KAAK,SAAS,EACd,KAAK,SAAS,IAAI,gBAAgB,EAClC,KAAK,iBAAiB,IAAI,wBAAwB,EACnD,MAAM,wBAAwB,CAAC;AAehC,qBAAa,QAAS,YAAW,UAAU;;IAMzC,YACE,EAAE,EAAE,UAAU,EACd,IAAI,EAAE,MAAM,EACZ,OAAO,CAAC,EAAE,eAAe,CAAC,OAAO,EACjC,kBAAkB,SAAM,EA4BzB;IAED,OAAO,CAAC,GAAG,EAAE,MAAM,GAAG,SAAS,CAY9B;IAED,IAAI,CAAC,GAAG,EAAE,MAAM,GAAG,IAAI,CAEtB;IAED,MAAM,CAAC,CAAC,GAAG,OAAO,EAAE,GAAG,EAAE,MAAM,GAAG,CAAC,EAAE,CAEpC;IAMD,OAAO,CAAC,sBAAsB,EAAE,MAAM,QAuCrC;IAED,UAAU,CAAC,MAAM,EAAE,OAAO,QAEzB;IAED;;;;;;OAMG;IACH,UAAU,CAAC,OAAO,EAAE,OAAO,QAE1B;IAmBD;;;;;OAKG;IACH,KAAK,CAAC,EAAC,QAAe,EAAC,GAAE;QAAC,QAAQ,CAAC,EAAE,OAAO,GAAG,SAAS,CAAA;KAAM,GAAG,IAAI,CAcpE;IAED,WAAW,CAAC,CAAC,EAAE,EAAE,EAAE,MAAM,CAAC,GAAG,CAAC,CAE7B;IAED,IAAI,IAAI,WAEP;IAED,IAAI,aAAa,YAEhB;IAED,CAAC,MAAM,CAAC,OAAO,CAAC,IAAI,IAAI,CAEvB;CACF;AAED,qBAAa,SAAS;;IAQpB,QAAQ,CAAC,UAAU,EAAE,gBAAgB,CAAC,cAAc,CAAC,CAAC;IACtD,QAAQ,CAAC,eAAe,EAAE,gBAAgB,CAAC,iBAAiB,CAAC,CAAC;IAK9D,YACE,EAAE,EAAE,UAAU,EACd,KAAK,EAAE,UAAU,EACjB,IAAI,EAAE,gBAAgB,EACtB,SAAS,EAAE,MAAM,EAQlB;IAED,YAAY,CAAC,SAAS,EAAE,OAAO,GAAG,IAAI,CAOrC;IAED;;;OAGG;IACH,GAAG,CAAC,OAAO,EAAE,OAAO,GAAG,IAAI,CAM1B;IAED;;;OAGG;IACH,KAAK,CAAC,OAAO,EAAE,OAAO,GAAG,IAAI,CAM5B;IAED,mEAAmE;IACnE,OAAO,IAAI,gBAAgB,EAAE,CAE5B;IAED,GAAG,CAAC,GAAG,MAAM,EAAE,OAAO,EAAE,GAAG,SAAS,CAKnC;IAED,GAAG,CAAC,CAAC,EAAE,GAAG,MAAM,EAAE,OAAO,EAAE,GAAG,CAAC,CAK9B;IAED,GAAG,CAAC,CAAC,EAAE,GAAG,MAAM,EAAE,OAAO,EAAE,GAAG,CAAC,EAAE,CAKhC;IAWD,OAAO,CAAC,CAAC,EAAE,GAAG,MAAM,EAAE,OAAO,EAAE,GAAG,gBAAgB,CAAC,CAAC,CAAC,CAOpD;IAED;;;OAGG;IACH,OAAO,CAAC,MAAM,EAAE,SAAS,OAAO,EAAE,GAAG,WAAW,CAW/C;CACF;AAED;;;GAGG;AACH,qBAAa,WAAW;;IAMtB,YACE,GAAG,EAAE,YAAY,EACjB,EAAE,EAAE,wBAAwB,EAC5B,OAAO,EAAE,MAAM,EAKhB;IAED;;;OAGG;IACH,IAAI,CAAC,QAAQ,EAAE,MAAM,GAAG,CAAC,MAAM,GAAG,MAAM,GAAG,MAAM,GAAG,IAAI,CAAC,EAAE,CA0B1D;IAED;;;OAGG;IACH,KAAK,IAAI,IAAI,CAMZ;CACF;AAyBD;;;GAGG;AACH,cAAM,YAAY;;IAQhB,YAAY,EAAE,EAAE,UAAU,EAAE,KAAK,EAAE,UAAU,EAAE,kBAAkB,EAAE,MAAM,EAKxE;IAED,aAAa,CAAC,OAAO,EAAE,MAAM,GAAG,IAAI,CAEnC;IAED,GAAG,SAwCF;CACF;AAwDD;;;;GAIG;AACH,qBAAa,iBAAkB,SAAQ,KAAK;CAAG"}

package/out/zqlite/src/db.js

@@ -5,7 +5,7 @@ import SQLite3Database, { SqliteError } from "@rocicorp/zero-sqlite3";
 //#region ../zqlite/src/db.ts
 var tracer = trace.getTracer("view-syncer", version);
 var AUTO_VACUUM_INCREMENTAL = 2;
-var MB = 1024 * 1024;
+var MB = 1048576;
 function mb(bytes) {
 	return (bytes / MB).toFixed(2);
 }
@@ -120,8 +120,12 @@ var Statement = class {
 	#lc;
 	#threshold;
 	#attrs;
+	#mode = "object";
 	scanStatus;
 	scanStatusReset;
+	#columnCount;
+	#safeIntegers;
+	#iterator;
 	constructor(lc, attrs, stmt, threshold) {
 		this.#lc = lc.withContext("class", "Statement");
 		this.#attrs = attrs;
@@ -131,9 +135,38 @@ var Statement = class {
 		this.scanStatusReset = this.#stmt.scanStatusReset.bind(this.#stmt);
 	}
 	safeIntegers(useBigInt) {
-		this.#stmt.safeIntegers(useBigInt);
+		if (this.#safeIntegers !== useBigInt) {
+			this.#stmt.safeIntegers(useBigInt);
+			this.#safeIntegers = useBigInt;
+		}
+		return this;
+	}
+	/**
+	* Return rows as arrays of column values instead of objects. Turning it
+	* off has no effect in pluck mode, as in better-sqlite3.
+	*/
+	raw(enabled) {
+		if (enabled ? this.#mode !== "raw" : this.#mode === "raw") {
+			this.#stmt.raw(enabled);
+			this.#mode = enabled ? "raw" : "object";
+		}
+		return this;
+	}
+	/**
+	* Return the first column of each row instead of the row. Turning it off
+	* has no effect in raw mode, as in better-sqlite3.
+	*/
+	pluck(enabled) {
+		if (enabled ? this.#mode !== "pluck" : this.#mode === "pluck") {
+			this.#stmt.pluck(enabled);
+			this.#mode = enabled ? "pluck" : "object";
+		}
 		return this;
 	}
+	/** The result columns, in the order of the values in a raw row. */
+	columns() {
+		return this.#stmt.columns();
+	}
 	run(...params) {
 		const start = performance.now();
 		const ret = this.#stmt.run(...params);
@@ -161,6 +194,62 @@ var Statement = class {
 	iterate(...params) {
 		return new LoggingIterableIterator(this.#lc, this.#attrs, this.#stmt.iterate(...params), this.#threshold);
 	}
+	/**
+	* Reads the statement's rows in batches. Integers are read as with
+	* `safeIntegers(true)`. The cursor must be {@link BatchCursor.close closed}.
+	*/
+	batches(params) {
+		this.#columnCount ??= this.#stmt.columns().length;
+		const log = new IterationLog(this.#lc, this.#attrs, this.#threshold);
+		this.safeIntegers(true).raw(true);
+		return new BatchCursor(log, this.#iterator === void 0 ? this.#iterator = this.#stmt.iterate(...params) : this.#iterator.restart(...params), this.#columnCount);
+	}
+};
+/**
+* Reads a statement's rows a batch at a time with one native call per batch
+* (`nextBatch`), instead of one per row as `next()` does.
+*/
+var BatchCursor = class {
+	#log;
+	#it;
+	#columns;
+	#done = false;
+	constructor(log, it, columns) {
+		this.#log = log;
+		this.#it = it;
+		this.#columns = columns;
+	}
+	/**
+	* Reads up to `capacity` rows and returns their values in one row-major
+	* array, empty once the rows are exhausted.
+	*/
+	next(capacity) {
+		if (this.#done) return [];
+		const start = performance.now();
+		let values;
+		try {
+			values = this.#it.nextBatch(capacity);
+		} catch (e) {
+			this.#done = true;
+			this.#log.log();
+			throw e;
+		} finally {
+			this.#log.addSQLiteTime(performance.now() - start);
+		}
+		if (values.length < capacity * this.#columns) this.#done = true;
+		return values;
+	}
+	/**
+	* Ends the cursor, abandoning any rows that were not read, and logs if the
+	* iteration was slow. Must be called once the rows are consumed.
+	*/
+	close() {
+		if (!this.#done) {
+			this.#done = true;
+			this.#it.return?.();
+		}
+		this.#log.log();
+	}
 };
 var ITERATOR_TOTAL_LOG_WINDOW_MS = 6e4;
 var ITERATOR_TOTAL_LOG_LIMIT = 10;
@@ -181,31 +270,27 @@ function admitIteratorTotalLog(now) {
 	suppressedIteratorTotalLogs = 0;
 	return suppressedCount;
 }
-var LoggingIterableIterator = class {
+/**
+* Logs slow iterations: once the time spent in SQLite, or the total time
+* including the consumer's work between rows, reaches the threshold.
+*/
+var IterationLog = class {
 	#lc;
-	#it;
 	#threshold;
 	#attrs;
 	#start;
-	#sqliteRowTimeSum;
+	#sqliteTimeSum = 0;
 	#logged = false;
-	constructor(lc, attrs, it, slowQueryThreshold) {
+	constructor(lc, attrs, slowQueryThreshold) {
 		this.#lc = lc;
 		this.#attrs = attrs;
-		this.#it = it;
 		this.#start = performance.now();
 		this.#threshold = slowQueryThreshold;
-		this.#sqliteRowTimeSum = 0;
 	}
-	next() {
-		const start = performance.now();
-		const ret = this.#it.next();
-		const elapsed = performance.now() - start;
-		this.#sqliteRowTimeSum += elapsed;
-		if (ret.done) this.#log();
-		return ret;
+	addSQLiteTime(elapsed) {
+		this.#sqliteTimeSum += elapsed;
 	}
-	#log() {
+	log() {
 		if (this.#logged) return;
 		this.#logged = true;
 		const now = performance.now();
@@ -216,26 +301,41 @@ var LoggingIterableIterator = class {
 				...this.#attrs,
 				type: "total",
 				method: "iterate",
-				sqliteMs: this.#sqliteRowTimeSum,
+				sqliteMs: this.#sqliteTimeSum,
 				...suppressedCount > 0 ? { suppressedCount } : void 0
 			}, "Slow SQLite iterator total time (including consumer work and waits between rows)");
 		}
-		if (this.#sqliteRowTimeSum >= this.#threshold) logSlow(this.#lc.withContext("method", "iterate").withContext("type", "sqlite"), this.#sqliteRowTimeSum, {
+		if (this.#sqliteTimeSum >= this.#threshold) logSlow(this.#lc.withContext("method", "iterate").withContext("type", "sqlite"), this.#sqliteTimeSum, {
 			...this.#attrs,
 			type: "sqlite",
 			method: "iterate",
 			totalMs
 		});
 	}
+};
+var LoggingIterableIterator = class {
+	#it;
+	#log;
+	constructor(lc, attrs, it, slowQueryThreshold) {
+		this.#it = it;
+		this.#log = new IterationLog(lc, attrs, slowQueryThreshold);
+	}
+	next() {
+		const start = performance.now();
+		const ret = this.#it.next();
+		this.#log.addSQLiteTime(performance.now() - start);
+		if (ret.done) this.#log.log();
+		return ret;
+	}
 	[Symbol.iterator]() {
 		return this;
 	}
 	return() {
-		this.#log();
+		this.#log.log();
 		return this.#it.return?.();
 	}
 	throw(e) {
-		this.#log();
+		this.#log.log();
 		return this.#it.throw?.(e);
 	}
 };
@@ -251,6 +351,6 @@ function logSlow(lc, elapsed, attrs, message = "Slow SQLite query") {
 */
 var DatabaseInitError = class extends Error {};
 //#endregion
-export { Database, DatabaseInitError };
+export { BatchCursor, Database, DatabaseInitError, Statement };
 
 //# sourceMappingURL=db.js.map