@rocicorp/zero 1.10.0-canary.2 → 1.10.0-canary.3

package/out/zero-cache/src/services/change-streamer/sqlite-change-log-read-router.js

@@ -0,0 +1,157 @@
+import { Database } from "../../../../zqlite/src/db.js";
+import { readChangeLogBounds, readChangeLogMeta } from "../replicator/change-log-db.js";
+import { isSampledForShard } from "./shard-sampling.js";
+//#region ../zero-cache/src/services/change-streamer/sqlite-change-log-read-router.ts
+var DEFAULT_FAILURE_COOLDOWN_MS = 3e4;
+/**
+* Selects and pins the catchup source shared by `/snapshot` and `/changes`.
+*
+* A post-registration SQLite failure opens a process-local breaker. Requests
+* use PG during the cooldown; the first request after it expires is the
+* bounded probe. A successful readiness inspection closes the breaker, while
+* another unavailable result rearms it. A writer failure opens it permanently
+* because the writer remains disabled and its file stays absent for the life
+* of the process.
+*/
+var SQLiteChangeLogReadRouter = class {
+	#shard;
+	#readPercent;
+	#coldReadPercent;
+	#retentionMs;
+	#inspect;
+	#failureCooldownMs;
+	#now;
+	#pins = /* @__PURE__ */ new Map();
+	#breakerUntilMs;
+	#permanentlyDisabled = false;
+	constructor(opts) {
+		this.#shard = opts.shard;
+		this.#readPercent = opts.readPercent;
+		this.#coldReadPercent = opts.coldReadPercent ?? 0;
+		this.#retentionMs = opts.retentionMs;
+		this.#inspect = opts.inspect;
+		this.#failureCooldownMs = opts.failureCooldownMs ?? DEFAULT_FAILURE_COOLDOWN_MS;
+		this.#now = opts.now ?? Date.now;
+	}
+	/** Chooses and pins a source for a snapshot reservation. */
+	pin(taskID) {
+		const route = this.#choose(taskID);
+		this.#pins.set(taskID, route);
+		return route;
+	}
+	/** Returns the source pinned by an active snapshot reservation. */
+	peek(taskID) {
+		const route = this.#pins.get(taskID);
+		return route && this.#asPinned(route);
+	}
+	/** Consumes a snapshot pin, or makes the stable choice for a direct retry. */
+	consume(taskID) {
+		const route = this.#pins.get(taskID);
+		if (route === void 0) return this.#choose(taskID);
+		this.#pins.delete(taskID);
+		return this.#asPinned(route);
+	}
+	/** Releases a pin whose snapshot reservation ended before subscribing. */
+	release(taskID) {
+		this.#pins.delete(taskID);
+	}
+	/**
+	* Replaces a pinned SQLite route with PG, for a reservation whose backup
+	* the log cannot cover -- most often a log seeded after that backup.
+	*
+	* The pin moves, not just the reservation's advertised bounds. Confirming
+	* with PG bounds while the task's `/changes` request still resolves to
+	* SQLite is exactly the mismatch the reservation path exists to prevent,
+	* so the two have to change together.
+	*
+	* A task with no pin is left alone: `consume` makes it a fresh choice.
+	*/
+	demote(taskID) {
+		const route = {
+			source: "pg",
+			reason: "backup-uncovered"
+		};
+		if (this.#pins.has(taskID)) this.#pins.set(taskID, route);
+		return route;
+	}
+	/** Opens the post-registration failure breaker. */
+	trip(permanent = false) {
+		if (permanent) {
+			this.#permanentlyDisabled = true;
+			this.#breakerUntilMs = void 0;
+			return;
+		}
+		if (!this.#permanentlyDisabled) this.#breakerUntilMs = this.#now() + this.#failureCooldownMs;
+	}
+	#choose(taskID) {
+		const now = this.#now();
+		if (this.#permanentlyDisabled || this.#breakerUntilMs !== void 0 && now < this.#breakerUntilMs) return {
+			source: "pg",
+			reason: "breaker-open"
+		};
+		let coverage;
+		try {
+			coverage = this.#inspect();
+		} catch {
+			coverage = void 0;
+		}
+		if (coverage === void 0) {
+			if (this.#breakerUntilMs !== void 0) this.#breakerUntilMs = now + this.#failureCooldownMs;
+			return {
+				source: "pg",
+				reason: "log-unavailable"
+			};
+		}
+		this.#breakerUntilMs = void 0;
+		const cold = now - coverage.seededAtMs < this.#retentionMs;
+		if (cold && !isSampledForShard(this.#shard, taskID, this.#coldReadPercent)) return {
+			source: "pg",
+			reason: "cold-log",
+			coverage
+		};
+		if (!isSampledForShard(this.#shard, taskID, this.#readPercent)) return {
+			source: "pg",
+			reason: "percentage",
+			coverage
+		};
+		return {
+			source: "sqlite",
+			reason: cold ? "selected-cold" : "selected",
+			coverage
+		};
+	}
+	#asPinned(route) {
+		return {
+			...route,
+			pinned: true
+		};
+	}
+};
+/**
+* Reads the coverage and validates the v2 schema and RMv2 identity in one
+* short-lived readonly connection. It never creates the log.
+*/
+function inspectSQLiteChangeLog(lc, changeLogFile, identity) {
+	let db;
+	try {
+		db = new Database(lc.withContext("component", "sqlite-change-log-read-router"), changeLogFile, { readonly: true });
+		const meta = readChangeLogMeta(db);
+		if (meta.schemaVersion !== 4 || meta.epoch !== identity.epoch || meta.generation !== identity.generation || meta.replicaID !== identity.replicaID) return;
+		const bounds = readChangeLogBounds(db);
+		if (bounds.minWatermark === null || bounds.headWatermark === null) return;
+		return {
+			seededAtMs: meta.seededAtMs,
+			seedWatermark: meta.seedWatermark,
+			minWatermark: bounds.minWatermark,
+			headWatermark: bounds.headWatermark
+		};
+	} catch {
+		return;
+	} finally {
+		db?.close();
+	}
+}
+//#endregion
+export { SQLiteChangeLogReadRouter, inspectSQLiteChangeLog };
+
+//# sourceMappingURL=sqlite-change-log-read-router.js.map

package/out/zero-cache/src/services/change-streamer/sqlite-change-log-read-router.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"sqlite-change-log-read-router.js","names":["#shard","#readPercent","#coldReadPercent","#retentionMs","#inspect","#failureCooldownMs","#now","#pins","#choose","#asPinned","#permanentlyDisabled","#breakerUntilMs"],"sources":["../../../../../../zero-cache/src/services/change-streamer/sqlite-change-log-read-router.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {Database} from '../../../../zqlite/src/db.ts';\nimport type {ShardID} from '../../types/shards.ts';\nimport {\n  CHANGE_LOG_DB_SCHEMA_VERSION,\n  readChangeLogBounds,\n  readChangeLogMeta,\n  type ChangeLogIdentity,\n} from '../replicator/change-log-db.ts';\nimport {isSampledForShard} from './shard-sampling.ts';\n\nexport type ChangeLogReadSource = 'pg' | 'sqlite';\n\nexport type SQLiteChangeLogCoverage = {\n  readonly seededAtMs: number;\n  readonly seedWatermark: string;\n  readonly minWatermark: string;\n  readonly headWatermark: string;\n};\n\nexport type ChangeLogReadRouteReason =\n  | 'selected'\n  // Selected while the log was still inside its warm-up window. Kept\n  // distinct from 'selected' so bootstrap serving is separable in metrics\n  // from steady-state serving.\n  | 'selected-cold'\n  | 'percentage'\n  | 'cold-log'\n  | 'log-unavailable'\n  | 'breaker-open'\n  // Pinned to SQLite, then demoted because the log could not cover the\n  // backup the reservation is restoring from. See {@link\n  // SQLiteChangeLogReadRouter.demote}.\n  | 'backup-uncovered';\n\nexport type ChangeLogReadRoute = {\n  readonly source: ChangeLogReadSource;\n  readonly reason: ChangeLogReadRouteReason;\n  readonly pinned?: boolean | undefined;\n  readonly coverage?: SQLiteChangeLogCoverage | undefined;\n};\n\nexport type SQLiteChangeLogReadRouterOptions = {\n  readonly shard: ShardID;\n  readonly readPercent: number;\n  /**\n   * Percentage of eligible tasks served from a log that has not yet aged\n   * through {@link retentionMs}. Defaults to zero, which keeps every such\n   * task on PG.\n   */\n  readonly coldReadPercent?: number | undefined;\n  readonly retentionMs: number;\n  readonly inspect: () => SQLiteChangeLogCoverage | undefined;\n  readonly failureCooldownMs?: number | undefined;\n  readonly now?: (() => number) | undefined;\n};\n\nconst DEFAULT_FAILURE_COOLDOWN_MS = 30_000;\n\n/**\n * Selects and pins the catchup source shared by `/snapshot` and `/changes`.\n *\n * A post-registration SQLite failure opens a process-local breaker. Requests\n * use PG during the cooldown; the first request after it expires is the\n * bounded probe. A successful readiness inspection closes the breaker, while\n * another unavailable result rearms it. A writer failure opens it permanently\n * because the writer remains disabled and its file stays absent for the life\n * of the process.\n */\nexport class SQLiteChangeLogReadRouter {\n  readonly #shard: ShardID;\n  readonly #readPercent: number;\n  readonly #coldReadPercent: number;\n  readonly #retentionMs: number;\n  readonly #inspect: () => SQLiteChangeLogCoverage | undefined;\n  readonly #failureCooldownMs: number;\n  readonly #now: () => number;\n  readonly #pins = new Map<string, ChangeLogReadRoute>();\n\n  #breakerUntilMs: number | undefined;\n  #permanentlyDisabled = false;\n\n  constructor(opts: SQLiteChangeLogReadRouterOptions) {\n    this.#shard = opts.shard;\n    this.#readPercent = opts.readPercent;\n    this.#coldReadPercent = opts.coldReadPercent ?? 0;\n    this.#retentionMs = opts.retentionMs;\n    this.#inspect = opts.inspect;\n    this.#failureCooldownMs =\n      opts.failureCooldownMs ?? DEFAULT_FAILURE_COOLDOWN_MS;\n    this.#now = opts.now ?? Date.now;\n  }\n\n  /** Chooses and pins a source for a snapshot reservation. */\n  pin(taskID: string): ChangeLogReadRoute {\n    const route = this.#choose(taskID);\n    this.#pins.set(taskID, route);\n    return route;\n  }\n\n  /** Returns the source pinned by an active snapshot reservation. */\n  peek(taskID: string): ChangeLogReadRoute | undefined {\n    const route = this.#pins.get(taskID);\n    return route && this.#asPinned(route);\n  }\n\n  /** Consumes a snapshot pin, or makes the stable choice for a direct retry. */\n  consume(taskID: string): ChangeLogReadRoute {\n    const route = this.#pins.get(taskID);\n    if (route === undefined) {\n      return this.#choose(taskID);\n    }\n    this.#pins.delete(taskID);\n    return this.#asPinned(route);\n  }\n\n  /** Releases a pin whose snapshot reservation ended before subscribing. */\n  release(taskID: string): void {\n    this.#pins.delete(taskID);\n  }\n\n  /**\n   * Replaces a pinned SQLite route with PG, for a reservation whose backup\n   * the log cannot cover -- most often a log seeded after that backup.\n   *\n   * The pin moves, not just the reservation's advertised bounds. Confirming\n   * with PG bounds while the task's `/changes` request still resolves to\n   * SQLite is exactly the mismatch the reservation path exists to prevent,\n   * so the two have to change together.\n   *\n   * A task with no pin is left alone: `consume` makes it a fresh choice.\n   */\n  demote(taskID: string): ChangeLogReadRoute {\n    const route: ChangeLogReadRoute = {\n      source: 'pg',\n      reason: 'backup-uncovered',\n    };\n    if (this.#pins.has(taskID)) {\n      this.#pins.set(taskID, route);\n    }\n    return route;\n  }\n\n  /** Opens the post-registration failure breaker. */\n  trip(permanent = false): void {\n    if (permanent) {\n      this.#permanentlyDisabled = true;\n      this.#breakerUntilMs = undefined;\n      return;\n    }\n    if (!this.#permanentlyDisabled) {\n      this.#breakerUntilMs = this.#now() + this.#failureCooldownMs;\n    }\n  }\n\n  #choose(taskID: string): ChangeLogReadRoute {\n    const now = this.#now();\n    if (\n      this.#permanentlyDisabled ||\n      (this.#breakerUntilMs !== undefined && now < this.#breakerUntilMs)\n    ) {\n      return {source: 'pg', reason: 'breaker-open'};\n    }\n\n    // Inspect eligibility even at readPercent=0. Slice 11 starts its rollout\n    // at zero specifically so warm/cold/unavailable rates are visible before\n    // any subscriber is routed to SQLite.\n    let coverage: SQLiteChangeLogCoverage | undefined;\n    try {\n      coverage = this.#inspect();\n    } catch {\n      coverage = undefined;\n    }\n    if (coverage === undefined) {\n      // This was a breaker probe rather than ordinary startup unavailability.\n      // Rearm it so a busy retry loop does not turn into an open/read loop.\n      if (this.#breakerUntilMs !== undefined) {\n        this.#breakerUntilMs = now + this.#failureCooldownMs;\n      }\n      return {source: 'pg', reason: 'log-unavailable'};\n    }\n    // A successful probe restores eligibility. This happens before the gates\n    // below so that declining a task -- for a cold log or for the percentage\n    // -- does not leave the cooldown armed against the next ordinary\n    // unavailability.\n    this.#breakerUntilMs = undefined;\n    // A log seeded less than a retention window ago holds less history than\n    // the purger promises to keep, so it is the case that most often cannot\n    // cover a follower. It is also the only path that has no PG to fall back\n    // to once PG is retired, which is the argument for exercising it now\n    // rather than meeting it at the cutover: `coldReadPercent` is that dial.\n    const cold = now - coverage.seededAtMs < this.#retentionMs;\n    if (\n      cold &&\n      !isSampledForShard(this.#shard, taskID, this.#coldReadPercent)\n    ) {\n      return {source: 'pg', reason: 'cold-log', coverage};\n    }\n\n    // Keeping the percentage decision after readiness makes zero-percent\n    // rollouts emit eligibility metrics without serving reads.\n    if (!isSampledForShard(this.#shard, taskID, this.#readPercent)) {\n      return {source: 'pg', reason: 'percentage', coverage};\n    }\n    return {\n      source: 'sqlite',\n      reason: cold ? 'selected-cold' : 'selected',\n      coverage,\n    };\n  }\n\n  #asPinned(route: ChangeLogReadRoute): ChangeLogReadRoute {\n    return {...route, pinned: true};\n  }\n}\n\n/**\n * Reads the coverage and validates the v2 schema and RMv2 identity in one\n * short-lived readonly connection. It never creates the log.\n */\nexport function inspectSQLiteChangeLog(\n  lc: LogContext,\n  changeLogFile: string,\n  identity: ChangeLogIdentity,\n): SQLiteChangeLogCoverage | undefined {\n  let db: Database | undefined;\n  try {\n    db = new Database(\n      lc.withContext('component', 'sqlite-change-log-read-router'),\n      changeLogFile,\n      {readonly: true},\n    );\n    const meta = readChangeLogMeta(db);\n    if (\n      meta.schemaVersion !== CHANGE_LOG_DB_SCHEMA_VERSION ||\n      meta.epoch !== identity.epoch ||\n      meta.generation !== identity.generation ||\n      meta.replicaID !== identity.replicaID\n    ) {\n      return undefined;\n    }\n    const bounds = readChangeLogBounds(db);\n    if (bounds.minWatermark === null || bounds.headWatermark === null) {\n      return undefined;\n    }\n    return {\n      seededAtMs: meta.seededAtMs,\n      seedWatermark: meta.seedWatermark,\n      minWatermark: bounds.minWatermark,\n      headWatermark: bounds.headWatermark,\n    };\n  } catch {\n    return undefined;\n  } finally {\n    db?.close();\n  }\n}\n"],"mappings":";;;;AAyDA,IAAM,8BAA8B;;;;;;;;;;;AAYpC,IAAa,4BAAb,MAAuC;CACrC;CACA;CACA;CACA;CACA;CACA;CACA;CACA,wBAAiB,IAAI,IAAgC;CAErD;CACA,uBAAuB;CAEvB,YAAY,MAAwC;EAClD,KAAKA,SAAS,KAAK;EACnB,KAAKC,eAAe,KAAK;EACzB,KAAKC,mBAAmB,KAAK,mBAAmB;EAChD,KAAKC,eAAe,KAAK;EACzB,KAAKC,WAAW,KAAK;EACrB,KAAKC,qBACH,KAAK,qBAAqB;EAC5B,KAAKC,OAAO,KAAK,OAAO,KAAK;CAC/B;;CAGA,IAAI,QAAoC;EACtC,MAAM,QAAQ,KAAKE,QAAQ,MAAM;EACjC,KAAKD,MAAM,IAAI,QAAQ,KAAK;EAC5B,OAAO;CACT;;CAGA,KAAK,QAAgD;EACnD,MAAM,QAAQ,KAAKA,MAAM,IAAI,MAAM;EACnC,OAAO,SAAS,KAAKE,UAAU,KAAK;CACtC;;CAGA,QAAQ,QAAoC;EAC1C,MAAM,QAAQ,KAAKF,MAAM,IAAI,MAAM;EACnC,IAAI,UAAU,KAAA,GACZ,OAAO,KAAKC,QAAQ,MAAM;EAE5B,KAAKD,MAAM,OAAO,MAAM;EACxB,OAAO,KAAKE,UAAU,KAAK;CAC7B;;CAGA,QAAQ,QAAsB;EAC5B,KAAKF,MAAM,OAAO,MAAM;CAC1B;;;;;;;;;;;;CAaA,OAAO,QAAoC;EACzC,MAAM,QAA4B;GAChC,QAAQ;GACR,QAAQ;EACV;EACA,IAAI,KAAKA,MAAM,IAAI,MAAM,GACvB,KAAKA,MAAM,IAAI,QAAQ,KAAK;EAE9B,OAAO;CACT;;CAGA,KAAK,YAAY,OAAa;EAC5B,IAAI,WAAW;GACb,KAAKG,uBAAuB;GAC5B,KAAKC,kBAAkB,KAAA;GACvB;EACF;EACA,IAAI,CAAC,KAAKD,sBACR,KAAKC,kBAAkB,KAAKL,KAAK,IAAI,KAAKD;CAE9C;CAEA,QAAQ,QAAoC;EAC1C,MAAM,MAAM,KAAKC,KAAK;EACtB,IACE,KAAKI,wBACJ,KAAKC,oBAAoB,KAAA,KAAa,MAAM,KAAKA,iBAElD,OAAO;GAAC,QAAQ;GAAM,QAAQ;EAAc;EAM9C,IAAI;EACJ,IAAI;GACF,WAAW,KAAKP,SAAS;EAC3B,QAAQ;GACN,WAAW,KAAA;EACb;EACA,IAAI,aAAa,KAAA,GAAW;GAG1B,IAAI,KAAKO,oBAAoB,KAAA,GAC3B,KAAKA,kBAAkB,MAAM,KAAKN;GAEpC,OAAO;IAAC,QAAQ;IAAM,QAAQ;GAAiB;EACjD;EAKA,KAAKM,kBAAkB,KAAA;EAMvB,MAAM,OAAO,MAAM,SAAS,aAAa,KAAKR;EAC9C,IACE,QACA,CAAC,kBAAkB,KAAKH,QAAQ,QAAQ,KAAKE,gBAAgB,GAE7D,OAAO;GAAC,QAAQ;GAAM,QAAQ;GAAY;EAAQ;EAKpD,IAAI,CAAC,kBAAkB,KAAKF,QAAQ,QAAQ,KAAKC,YAAY,GAC3D,OAAO;GAAC,QAAQ;GAAM,QAAQ;GAAc;EAAQ;EAEtD,OAAO;GACL,QAAQ;GACR,QAAQ,OAAO,kBAAkB;GACjC;EACF;CACF;CAEA,UAAU,OAA+C;EACvD,OAAO;GAAC,GAAG;GAAO,QAAQ;EAAI;CAChC;AACF;;;;;AAMA,SAAgB,uBACd,IACA,eACA,UACqC;CACrC,IAAI;CACJ,IAAI;EACF,KAAK,IAAI,SACP,GAAG,YAAY,aAAa,+BAA+B,GAC3D,eACA,EAAC,UAAU,KAAI,CACjB;EACA,MAAM,OAAO,kBAAkB,EAAE;EACjC,IACE,KAAK,kBAAA,KACL,KAAK,UAAU,SAAS,SACxB,KAAK,eAAe,SAAS,cAC7B,KAAK,cAAc,SAAS,WAE5B;EAEF,MAAM,SAAS,oBAAoB,EAAE;EACrC,IAAI,OAAO,iBAAiB,QAAQ,OAAO,kBAAkB,MAC3D;EAEF,OAAO;GACL,YAAY,KAAK;GACjB,eAAe,KAAK;GACpB,cAAc,OAAO;GACrB,eAAe,OAAO;EACxB;CACF,QAAQ;EACN;CACF,UAAU;EACR,IAAI,MAAM;CACZ;AACF"}

package/out/zero-cache/src/services/change-streamer/sqlite-change-log-reader.d.ts

@@ -55,7 +55,7 @@ export declare class SQLiteChangeLogReader implements Disposable {
     #private;
     constructor(lc: LogContext, changeLogFile: string);
     plan(fromWatermark: string): CatchupPlan;
-    read(fromWatermark: string, throughWatermark: string, batchSize: number, signal?: AbortSignal | undefined): AsyncIterable<readonly WatermarkedChange[]>;
+    read(fromWatermark: string, throughWatermark: string, batchSize: number, signal?: AbortSignal | undefined, maxRows?: number | undefined): AsyncIterable<readonly WatermarkedChange[]>;
     close(): void;
     [Symbol.dispose](): void;
 }

package/out/zero-cache/src/services/change-streamer/sqlite-change-log-reader.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"sqlite-change-log-reader.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/change-streamer/sqlite-change-log-reader.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AASjD,OAAO,KAAK,EAAC,iBAAiB,EAAC,MAAM,sBAAsB,CAAC;AAE5D,MAAM,MAAM,WAAW,GACnB;IACE,QAAQ,CAAC,IAAI,EAAE,OAAO,CAAC;IACvB,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC;IAC9B,QAAQ,CAAC,aAAa,EAAE,MAAM,CAAC;CAChC,GACD;IAAC,QAAQ,CAAC,IAAI,EAAE,OAAO,CAAC;IAAC,QAAQ,CAAC,aAAa,EAAE,MAAM,CAAA;CAAC,GACxD;IACE,QAAQ,CAAC,IAAI,EAAE,SAAS,CAAC;IACzB,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC;IAC9B,QAAQ,CAAC,aAAa,EAAE,MAAM,CAAC;CAChC,GAKD;IAAC,QAAQ,CAAC,IAAI,EAAE,WAAW,CAAA;CAAC,CAAC;AAiBjC;;;;GAIG;AACH,eAAO,MAAM,8BAA8B,uNAQ1C,CAAC;AAEF;;;;;;;;;;;;GAYG;AACH,eAAO,MAAM,0BAA0B,ucAOtC,CAAC;AAEF,2EAA2E;AAC3E,eAAO,MAAM,gCAAgC,wOAW5C,CAAC;AAMF;;;;;;;;;;;;;;GAcG;AACH,qBAAa,qBAAsB,YAAW,UAAU;;gBAK1C,EAAE,EAAE,UAAU,EAAE,aAAa,EAAE,MAAM;IAQjD,IAAI,CAAC,aAAa,EAAE,MAAM,GAAG,WAAW;IAqBjC,IAAI,CACT,aAAa,EAAE,MAAM,EACrB,gBAAgB,EAAE,MAAM,EACxB,SAAS,EAAE,MAAM,EACjB,MAAM,CAAC,EAAE,WAAW,GAAG,SAAS,GAC/B,aAAa,CAAC,SAAS,iBAAiB,EAAE,CAAC;IA2D9C,KAAK,IAAI,IAAI;IAQb,CAAC,MAAM,CAAC,OAAO,CAAC,IAAI,IAAI;CAuCzB"}
+{"version":3,"file":"sqlite-change-log-reader.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/change-streamer/sqlite-change-log-reader.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AASjD,OAAO,KAAK,EAAC,iBAAiB,EAAC,MAAM,sBAAsB,CAAC;AAE5D,MAAM,MAAM,WAAW,GACnB;IACE,QAAQ,CAAC,IAAI,EAAE,OAAO,CAAC;IACvB,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC;IAC9B,QAAQ,CAAC,aAAa,EAAE,MAAM,CAAC;CAChC,GACD;IAAC,QAAQ,CAAC,IAAI,EAAE,OAAO,CAAC;IAAC,QAAQ,CAAC,aAAa,EAAE,MAAM,CAAA;CAAC,GACxD;IACE,QAAQ,CAAC,IAAI,EAAE,SAAS,CAAC;IACzB,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC;IAC9B,QAAQ,CAAC,aAAa,EAAE,MAAM,CAAC;CAChC,GAKD;IAAC,QAAQ,CAAC,IAAI,EAAE,WAAW,CAAA;CAAC,CAAC;AAiBjC;;;;GAIG;AACH,eAAO,MAAM,8BAA8B,uNAQ1C,CAAC;AAEF;;;;;;;;;;;;GAYG;AACH,eAAO,MAAM,0BAA0B,ucAOtC,CAAC;AAEF,2EAA2E;AAC3E,eAAO,MAAM,gCAAgC,wOAW5C,CAAC;AAMF;;;;;;;;;;;;;;GAcG;AACH,qBAAa,qBAAsB,YAAW,UAAU;;gBAK1C,EAAE,EAAE,UAAU,EAAE,aAAa,EAAE,MAAM;IAQjD,IAAI,CAAC,aAAa,EAAE,MAAM,GAAG,WAAW;IAqBjC,IAAI,CACT,aAAa,EAAE,MAAM,EACrB,gBAAgB,EAAE,MAAM,EACxB,SAAS,EAAE,MAAM,EACjB,MAAM,CAAC,EAAE,WAAW,GAAG,SAAS,EAChC,OAAO,CAAC,EAAE,MAAM,GAAG,SAAS,GAC3B,aAAa,CAAC,SAAS,iBAAiB,EAAE,CAAC;IA6E9C,KAAK,IAAI,IAAI;IAQb,CAAC,MAAM,CAAC,OAAO,CAAC,IAAI,IAAI;CAuCzB"}

package/out/zero-cache/src/services/change-streamer/sqlite-change-log-reader.js

@@ -99,17 +99,21 @@ var SQLiteChangeLogReader = class {
 			headWatermark
 		};
 	}
-	async *read(fromWatermark, throughWatermark, batchSize, signal) {
+	async *read(fromWatermark, throughWatermark, batchSize, signal, maxRows) {
 		assert(Number.isSafeInteger(batchSize) && batchSize > 0, "SQLite change log batch size must be a positive safe integer");
+		assert(maxRows === void 0 || Number.isSafeInteger(maxRows) && maxRows > 0, "SQLite change log row limit must be a positive safe integer");
 		this.#assertOpen();
 		const statements = this.#prepare();
 		assert(statements !== void 0, "SQLite change log must be ready before it is read");
 		let lastWatermark = fromWatermark;
 		let lastPos = Number.MAX_SAFE_INTEGER;
 		let lastTag;
+		let rowsRead = 0;
 		while (true) {
 			this.#throwIfAborted(signal);
-			const rows = statements.readBatch.all(lastWatermark, lastPos, throughWatermark, batchSize);
+			const remainingRows = maxRows === void 0 ? batchSize : maxRows - rowsRead;
+			if (remainingRows === 0) break;
+			const rows = statements.readBatch.all(lastWatermark, lastPos, throughWatermark, Math.min(batchSize, remainingRows));
 			this.#throwIfAborted(signal);
 			if (rows.length === 0) break;
 			const changes = rows.map(reconstructWatermarkedChange);
@@ -118,9 +122,10 @@ var SQLiteChangeLogReader = class {
 			lastWatermark = last.watermark;
 			lastPos = last.pos;
 			lastTag = last.tag;
+			rowsRead += rows.length;
 			yield changes;
 		}
-		if (fromWatermark !== throughWatermark) assert(lastWatermark === throughWatermark && lastTag === "commit", () => `SQLite change log did not end at pinned head ${throughWatermark}`);
+		if (fromWatermark !== throughWatermark && (maxRows === void 0 || rowsRead < maxRows)) assert(lastWatermark === throughWatermark && lastTag === "commit", () => `SQLite change log did not end at pinned head ${throughWatermark}`);
 	}
 	close() {
 		if (this.#closed) return;

package/out/zero-cache/src/services/change-streamer/sqlite-change-log-reader.js.map

@@ -1 +1 @@
-{"version":3,"file":"sqlite-change-log-reader.js","names":["#db","#assertOpen","#prepare","#throwIfAborted","#closed","#statements","#streamTableExists"],"sources":["../../../../../../zero-cache/src/services/change-streamer/sqlite-change-log-reader.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {AbortError} from '../../../../shared/src/abort-error.ts';\nimport {assert} from '../../../../shared/src/asserts.ts';\nimport {Database, type Statement} from '../../../../zqlite/src/db.ts';\nimport {CHANGE_LOG_STREAM_TABLE} from '../replicator/change-log-db.ts';\nimport {\n  reconstructWatermarkedChange,\n  type ChangeLogEntry,\n} from './change-log-codec.ts';\nimport type {WatermarkedChange} from './change-streamer.ts';\n\nexport type CatchupPlan =\n  | {\n      readonly kind: 'range';\n      readonly minWatermark: string;\n      readonly headWatermark: string;\n    }\n  | {readonly kind: 'ahead'; readonly headWatermark: string}\n  | {\n      readonly kind: 'too-old';\n      readonly minWatermark: string;\n      readonly headWatermark: string;\n    }\n  // The log exists as a file but has no content to serve: the replicator has\n  // not created or reconciled it yet. Subscriber selection treats this as\n  // \"decline and fall back to PG catchup\", which is why it is a plan rather\n  // than an error.\n  | {readonly kind: 'not-ready'};\n\ntype PlanRow = {\n  readonly headWatermark: string | null;\n  readonly minWatermark: string | null;\n  readonly boundaryExists: number;\n};\n\ntype ChangeLogRow = ChangeLogEntry & {\n  readonly pos: number;\n};\n\ntype PreparedStatements = {\n  readonly plan: Statement;\n  readonly readBatch: Statement;\n};\n\n/**\n * Seeks to the final position so validating a boundary does not scan every row\n * of a large transaction. Exported so the focused query-plan test covers the\n * production subquery.\n */\nexport const SQLITE_CHANGE_LOG_BOUNDARY_SQL = /*sql*/ `\n  SELECT\n    \"precommit\" = @fromWatermark AND\n      \"tag\" = 'commit' AS \"boundaryExists\"\n  FROM \"${CHANGE_LOG_STREAM_TABLE}\"\n  WHERE \"watermark\" = @fromWatermark\n  ORDER BY \"pos\" DESC\n  LIMIT 1\n`;\n\n/**\n * The log describes its own bounds; nothing here reads the replica.\n * `max(\"watermark\")` is the head because a transaction's rows commit atomically,\n * so the log never ends mid-transaction.\n *\n * `min` and `max` are deliberately kept as *separate* scalar subqueries: SQLite\n * applies its single-aggregate index optimization only when the aggregate is\n * alone in its query, so `SELECT min(x), max(x) FROM t` degrades to a full scan\n * while two scalar subqueries are two index seeks. The query-plan test pins\n * this.\n *\n * Exported so that test can cover the production query.\n */\nexport const SQLITE_CHANGE_LOG_PLAN_SQL = /*sql*/ `\n  SELECT\n    (SELECT max(\"watermark\") FROM \"${CHANGE_LOG_STREAM_TABLE}\")\n      AS \"headWatermark\",\n    (SELECT min(\"watermark\") FROM \"${CHANGE_LOG_STREAM_TABLE}\")\n      AS \"minWatermark\",\n    coalesce((${SQLITE_CHANGE_LOG_BOUNDARY_SQL}), 0) AS \"boundaryExists\"\n`;\n\n/** Exported so the focused query-plan test covers the production query. */\nexport const SQLITE_CHANGE_LOG_READ_BATCH_SQL = /*sql*/ `\n  SELECT\n    \"watermark\",\n    \"pos\",\n    \"tag\",\n    \"change\"\n  FROM \"${CHANGE_LOG_STREAM_TABLE}\"\n  WHERE (\"watermark\", \"pos\") > (?, ?)\n    AND \"watermark\" <= ?\n  ORDER BY \"watermark\", \"pos\"\n  LIMIT ?\n`;\n\nconst TABLE_EXISTS_SQL = /*sql*/ `\n  SELECT 1 FROM \"sqlite_master\" WHERE \"type\" = 'table' AND \"name\" = ?\n`;\n\n/**\n * Reads the change log without retaining a SQLite snapshot while a consumer\n * processes a batch.\n *\n * The log lives in its own database beside the replica\n * (`changeLogFileName(replicaFile)`), which this reader opens read-only; it\n * never opens the replica. The connection therefore keeps the journal mode\n * configured by the change-log writer. Callers pin a head with {@link plan}\n * before passing it to {@link read}.\n *\n * A change-streamer can start before the replicator has created the log, so the\n * stream table may not exist yet. Statements are prepared on first use rather\n * than in the constructor, and {@link plan} reports `not-ready` until they can\n * be; a reader constructed too early becomes usable once the writer catches up.\n */\nexport class SQLiteChangeLogReader implements Disposable {\n  readonly #db: Database;\n  #statements: PreparedStatements | undefined;\n  #closed = false;\n\n  constructor(lc: LogContext, changeLogFile: string) {\n    this.#db = new Database(\n      lc.withContext('component', 'sqlite-change-log-reader'),\n      changeLogFile,\n      {readonly: true},\n    );\n  }\n\n  plan(fromWatermark: string): CatchupPlan {\n    this.#assertOpen();\n    const statements = this.#prepare();\n    if (statements === undefined) {\n      return {kind: 'not-ready'};\n    }\n    const row = statements.plan.get<PlanRow>({fromWatermark});\n    if (row.headWatermark === null || row.minWatermark === null) {\n      return {kind: 'not-ready'};\n    }\n    const {headWatermark, minWatermark} = row;\n\n    if (fromWatermark > headWatermark) {\n      return {kind: 'ahead', headWatermark};\n    }\n    if (fromWatermark < minWatermark || row.boundaryExists === 0) {\n      return {kind: 'too-old', minWatermark, headWatermark};\n    }\n    return {kind: 'range', minWatermark, headWatermark};\n  }\n\n  async *read(\n    fromWatermark: string,\n    throughWatermark: string,\n    batchSize: number,\n    signal?: AbortSignal | undefined,\n  ): AsyncIterable<readonly WatermarkedChange[]> {\n    assert(\n      Number.isSafeInteger(batchSize) && batchSize > 0,\n      'SQLite change log batch size must be a positive safe integer',\n    );\n    this.#assertOpen();\n    const statements = this.#prepare();\n    assert(\n      statements !== undefined,\n      'SQLite change log must be ready before it is read',\n    );\n\n    let lastWatermark = fromWatermark;\n    // The first query must be strictly after the requested transaction. Every\n    // real stream position is far below this sentinel.\n    let lastPos = Number.MAX_SAFE_INTEGER;\n    let lastTag: string | undefined;\n\n    while (true) {\n      this.#throwIfAborted(signal);\n      const rows = statements.readBatch.all<ChangeLogRow>(\n        lastWatermark,\n        lastPos,\n        throughWatermark,\n        batchSize,\n      );\n      this.#throwIfAborted(signal);\n\n      if (rows.length === 0) {\n        break;\n      }\n\n      const changes = rows.map(reconstructWatermarkedChange);\n      const last = rows.at(-1);\n      assert(last !== undefined, 'non-empty batch must have a final row');\n      lastWatermark = last.watermark;\n      lastPos = last.pos;\n      lastTag = last.tag;\n\n      // all() has exhausted the SELECT and closed its implicit read\n      // transaction. This yield therefore cannot pin the WAL while subscriber\n      // flow control is pending.\n      yield changes;\n    }\n\n    // `throughWatermark` was pinned from `max(\"watermark\")` of this same table,\n    // so reaching it no longer depends on the log agreeing with the replica's\n    // `_zero.replicationState`, which is what this assertion used to guard.\n    // What remains is a concurrent purge of the head transaction, which the\n    // purge policy (never past the latest durable transaction) does not do.\n    if (fromWatermark !== throughWatermark) {\n      assert(\n        lastWatermark === throughWatermark && lastTag === 'commit',\n        () =>\n          `SQLite change log did not end at pinned head ${throughWatermark}`,\n      );\n    }\n  }\n\n  close(): void {\n    if (this.#closed) {\n      return;\n    }\n    this.#closed = true;\n    this.#db.close();\n  }\n\n  [Symbol.dispose](): void {\n    this.close();\n  }\n\n  /**\n   * Prepares the statements once the stream table exists, memoizing them.\n   * Returns `undefined` while the log has not been created, which is a normal\n   * startup state rather than an error.\n   */\n  #prepare(): PreparedStatements | undefined {\n    if (this.#statements === undefined && this.#streamTableExists()) {\n      this.#statements = {\n        plan: this.#db.prepare(SQLITE_CHANGE_LOG_PLAN_SQL),\n        readBatch: this.#db.prepare(SQLITE_CHANGE_LOG_READ_BATCH_SQL),\n      };\n    }\n    return this.#statements;\n  }\n\n  #streamTableExists(): boolean {\n    return (\n      this.#db\n        .prepare(TABLE_EXISTS_SQL)\n        .get<{1: number} | undefined>(CHANGE_LOG_STREAM_TABLE) !== undefined\n    );\n  }\n\n  #assertOpen(): void {\n    if (this.#closed) {\n      throw new AbortError('SQLite change log reader is closed');\n    }\n  }\n\n  #throwIfAborted(signal: AbortSignal | undefined): void {\n    this.#assertOpen();\n    if (signal?.aborted) {\n      throw new AbortError('SQLite change log read aborted');\n    }\n  }\n}\n"],"mappings":";;;;;;;;;;;AAiDA,IAAa,iCAAyC;;;;UAI5C,wBAAwB;;;;;;;;;;;;;;;;;;AAmBlC,IAAa,6BAAqC;;qCAEb,wBAAwB;;qCAExB,wBAAwB;;gBAE7C,+BAA+B;;;AAI/C,IAAa,mCAA2C;;;;;;UAM9C,wBAAwB;;;;;;AAOlC,IAAM,mBAA2B;;;;;;;;;;;;;;;;;;AAmBjC,IAAa,wBAAb,MAAyD;CACvD;CACA;CACA,UAAU;CAEV,YAAY,IAAgB,eAAuB;EACjD,KAAKA,MAAM,IAAI,SACb,GAAG,YAAY,aAAa,0BAA0B,GACtD,eACA,EAAC,UAAU,KAAI,CACjB;CACF;CAEA,KAAK,eAAoC;EACvC,KAAKC,YAAY;EACjB,MAAM,aAAa,KAAKC,SAAS;EACjC,IAAI,eAAe,KAAA,GACjB,OAAO,EAAC,MAAM,YAAW;EAE3B,MAAM,MAAM,WAAW,KAAK,IAAa,EAAC,cAAa,CAAC;EACxD,IAAI,IAAI,kBAAkB,QAAQ,IAAI,iBAAiB,MACrD,OAAO,EAAC,MAAM,YAAW;EAE3B,MAAM,EAAC,eAAe,iBAAgB;EAEtC,IAAI,gBAAgB,eAClB,OAAO;GAAC,MAAM;GAAS;EAAa;EAEtC,IAAI,gBAAgB,gBAAgB,IAAI,mBAAmB,GACzD,OAAO;GAAC,MAAM;GAAW;GAAc;EAAa;EAEtD,OAAO;GAAC,MAAM;GAAS;GAAc;EAAa;CACpD;CAEA,OAAO,KACL,eACA,kBACA,WACA,QAC6C;EAC7C,OACE,OAAO,cAAc,SAAS,KAAK,YAAY,GAC/C,8DACF;EACA,KAAKD,YAAY;EACjB,MAAM,aAAa,KAAKC,SAAS;EACjC,OACE,eAAe,KAAA,GACf,mDACF;EAEA,IAAI,gBAAgB;EAGpB,IAAI,UAAU,OAAO;EACrB,IAAI;EAEJ,OAAO,MAAM;GACX,KAAKC,gBAAgB,MAAM;GAC3B,MAAM,OAAO,WAAW,UAAU,IAChC,eACA,SACA,kBACA,SACF;GACA,KAAKA,gBAAgB,MAAM;GAE3B,IAAI,KAAK,WAAW,GAClB;GAGF,MAAM,UAAU,KAAK,IAAI,4BAA4B;GACrD,MAAM,OAAO,KAAK,GAAG,EAAE;GACvB,OAAO,SAAS,KAAA,GAAW,uCAAuC;GAClE,gBAAgB,KAAK;GACrB,UAAU,KAAK;GACf,UAAU,KAAK;GAKf,MAAM;EACR;EAOA,IAAI,kBAAkB,kBACpB,OACE,kBAAkB,oBAAoB,YAAY,gBAEhD,gDAAgD,kBACpD;CAEJ;CAEA,QAAc;EACZ,IAAI,KAAKC,SACP;EAEF,KAAKA,UAAU;EACf,KAAKJ,IAAI,MAAM;CACjB;CAEA,CAAC,OAAO,WAAiB;EACvB,KAAK,MAAM;CACb;;;;;;CAOA,WAA2C;EACzC,IAAI,KAAKK,gBAAgB,KAAA,KAAa,KAAKC,mBAAmB,GAC5D,KAAKD,cAAc;GACjB,MAAM,KAAKL,IAAI,QAAQ,0BAA0B;GACjD,WAAW,KAAKA,IAAI,QAAQ,gCAAgC;EAC9D;EAEF,OAAO,KAAKK;CACd;CAEA,qBAA8B;EAC5B,OACE,KAAKL,IACF,QAAQ,gBAAgB,EACxB,IAA6B,uBAAuB,MAAM,KAAA;CAEjE;CAEA,cAAoB;EAClB,IAAI,KAAKI,SACP,MAAM,IAAI,WAAW,oCAAoC;CAE7D;CAEA,gBAAgB,QAAuC;EACrD,KAAKH,YAAY;EACjB,IAAI,QAAQ,SACV,MAAM,IAAI,WAAW,gCAAgC;CAEzD;AACF"}
+{"version":3,"file":"sqlite-change-log-reader.js","names":["#db","#assertOpen","#prepare","#throwIfAborted","#closed","#statements","#streamTableExists"],"sources":["../../../../../../zero-cache/src/services/change-streamer/sqlite-change-log-reader.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {AbortError} from '../../../../shared/src/abort-error.ts';\nimport {assert} from '../../../../shared/src/asserts.ts';\nimport {Database, type Statement} from '../../../../zqlite/src/db.ts';\nimport {CHANGE_LOG_STREAM_TABLE} from '../replicator/change-log-db.ts';\nimport {\n  reconstructWatermarkedChange,\n  type ChangeLogEntry,\n} from './change-log-codec.ts';\nimport type {WatermarkedChange} from './change-streamer.ts';\n\nexport type CatchupPlan =\n  | {\n      readonly kind: 'range';\n      readonly minWatermark: string;\n      readonly headWatermark: string;\n    }\n  | {readonly kind: 'ahead'; readonly headWatermark: string}\n  | {\n      readonly kind: 'too-old';\n      readonly minWatermark: string;\n      readonly headWatermark: string;\n    }\n  // The log exists as a file but has no content to serve: the replicator has\n  // not created or reconciled it yet. Subscriber selection treats this as\n  // \"decline and fall back to PG catchup\", which is why it is a plan rather\n  // than an error.\n  | {readonly kind: 'not-ready'};\n\ntype PlanRow = {\n  readonly headWatermark: string | null;\n  readonly minWatermark: string | null;\n  readonly boundaryExists: number;\n};\n\ntype ChangeLogRow = ChangeLogEntry & {\n  readonly pos: number;\n};\n\ntype PreparedStatements = {\n  readonly plan: Statement;\n  readonly readBatch: Statement;\n};\n\n/**\n * Seeks to the final position so validating a boundary does not scan every row\n * of a large transaction. Exported so the focused query-plan test covers the\n * production subquery.\n */\nexport const SQLITE_CHANGE_LOG_BOUNDARY_SQL = /*sql*/ `\n  SELECT\n    \"precommit\" = @fromWatermark AND\n      \"tag\" = 'commit' AS \"boundaryExists\"\n  FROM \"${CHANGE_LOG_STREAM_TABLE}\"\n  WHERE \"watermark\" = @fromWatermark\n  ORDER BY \"pos\" DESC\n  LIMIT 1\n`;\n\n/**\n * The log describes its own bounds; nothing here reads the replica.\n * `max(\"watermark\")` is the head because a transaction's rows commit atomically,\n * so the log never ends mid-transaction.\n *\n * `min` and `max` are deliberately kept as *separate* scalar subqueries: SQLite\n * applies its single-aggregate index optimization only when the aggregate is\n * alone in its query, so `SELECT min(x), max(x) FROM t` degrades to a full scan\n * while two scalar subqueries are two index seeks. The query-plan test pins\n * this.\n *\n * Exported so that test can cover the production query.\n */\nexport const SQLITE_CHANGE_LOG_PLAN_SQL = /*sql*/ `\n  SELECT\n    (SELECT max(\"watermark\") FROM \"${CHANGE_LOG_STREAM_TABLE}\")\n      AS \"headWatermark\",\n    (SELECT min(\"watermark\") FROM \"${CHANGE_LOG_STREAM_TABLE}\")\n      AS \"minWatermark\",\n    coalesce((${SQLITE_CHANGE_LOG_BOUNDARY_SQL}), 0) AS \"boundaryExists\"\n`;\n\n/** Exported so the focused query-plan test covers the production query. */\nexport const SQLITE_CHANGE_LOG_READ_BATCH_SQL = /*sql*/ `\n  SELECT\n    \"watermark\",\n    \"pos\",\n    \"tag\",\n    \"change\"\n  FROM \"${CHANGE_LOG_STREAM_TABLE}\"\n  WHERE (\"watermark\", \"pos\") > (?, ?)\n    AND \"watermark\" <= ?\n  ORDER BY \"watermark\", \"pos\"\n  LIMIT ?\n`;\n\nconst TABLE_EXISTS_SQL = /*sql*/ `\n  SELECT 1 FROM \"sqlite_master\" WHERE \"type\" = 'table' AND \"name\" = ?\n`;\n\n/**\n * Reads the change log without retaining a SQLite snapshot while a consumer\n * processes a batch.\n *\n * The log lives in its own database beside the replica\n * (`changeLogFileName(replicaFile)`), which this reader opens read-only; it\n * never opens the replica. The connection therefore keeps the journal mode\n * configured by the change-log writer. Callers pin a head with {@link plan}\n * before passing it to {@link read}.\n *\n * A change-streamer can start before the replicator has created the log, so the\n * stream table may not exist yet. Statements are prepared on first use rather\n * than in the constructor, and {@link plan} reports `not-ready` until they can\n * be; a reader constructed too early becomes usable once the writer catches up.\n */\nexport class SQLiteChangeLogReader implements Disposable {\n  readonly #db: Database;\n  #statements: PreparedStatements | undefined;\n  #closed = false;\n\n  constructor(lc: LogContext, changeLogFile: string) {\n    this.#db = new Database(\n      lc.withContext('component', 'sqlite-change-log-reader'),\n      changeLogFile,\n      {readonly: true},\n    );\n  }\n\n  plan(fromWatermark: string): CatchupPlan {\n    this.#assertOpen();\n    const statements = this.#prepare();\n    if (statements === undefined) {\n      return {kind: 'not-ready'};\n    }\n    const row = statements.plan.get<PlanRow>({fromWatermark});\n    if (row.headWatermark === null || row.minWatermark === null) {\n      return {kind: 'not-ready'};\n    }\n    const {headWatermark, minWatermark} = row;\n\n    if (fromWatermark > headWatermark) {\n      return {kind: 'ahead', headWatermark};\n    }\n    if (fromWatermark < minWatermark || row.boundaryExists === 0) {\n      return {kind: 'too-old', minWatermark, headWatermark};\n    }\n    return {kind: 'range', minWatermark, headWatermark};\n  }\n\n  async *read(\n    fromWatermark: string,\n    throughWatermark: string,\n    batchSize: number,\n    signal?: AbortSignal | undefined,\n    maxRows?: number | undefined,\n  ): AsyncIterable<readonly WatermarkedChange[]> {\n    assert(\n      Number.isSafeInteger(batchSize) && batchSize > 0,\n      'SQLite change log batch size must be a positive safe integer',\n    );\n    assert(\n      maxRows === undefined || (Number.isSafeInteger(maxRows) && maxRows > 0),\n      'SQLite change log row limit must be a positive safe integer',\n    );\n    this.#assertOpen();\n    const statements = this.#prepare();\n    assert(\n      statements !== undefined,\n      'SQLite change log must be ready before it is read',\n    );\n\n    let lastWatermark = fromWatermark;\n    // The first query must be strictly after the requested transaction. Every\n    // real stream position is far below this sentinel.\n    let lastPos = Number.MAX_SAFE_INTEGER;\n    let lastTag: string | undefined;\n    let rowsRead = 0;\n\n    while (true) {\n      this.#throwIfAborted(signal);\n      const remainingRows =\n        maxRows === undefined ? batchSize : maxRows - rowsRead;\n      if (remainingRows === 0) {\n        break;\n      }\n      const rows = statements.readBatch.all<ChangeLogRow>(\n        lastWatermark,\n        lastPos,\n        throughWatermark,\n        Math.min(batchSize, remainingRows),\n      );\n      this.#throwIfAborted(signal);\n\n      if (rows.length === 0) {\n        break;\n      }\n\n      const changes = rows.map(reconstructWatermarkedChange);\n      const last = rows.at(-1);\n      assert(last !== undefined, 'non-empty batch must have a final row');\n      lastWatermark = last.watermark;\n      lastPos = last.pos;\n      lastTag = last.tag;\n      rowsRead += rows.length;\n\n      // all() has exhausted the SELECT and closed its implicit read\n      // transaction. This yield therefore cannot pin the WAL while subscriber\n      // flow control is pending.\n      yield changes;\n    }\n\n    // `throughWatermark` was pinned from `max(\"watermark\")` of this same table,\n    // so reaching it no longer depends on the log agreeing with the replica's\n    // `_zero.replicationState`, which is what this assertion used to guard.\n    // What remains is a concurrent purge of the head transaction, which the\n    // purge policy (never past the latest durable transaction) does not do.\n    // A read that stopped at `maxRows` cannot know whether more rows follow,\n    // so completeness is only asserted for a read that ran out on its own.\n    // Detecting a short bounded read is left to the caller, which set the\n    // limit and is the only side that knows what a full range should hold.\n    if (\n      fromWatermark !== throughWatermark &&\n      (maxRows === undefined || rowsRead < maxRows)\n    ) {\n      assert(\n        lastWatermark === throughWatermark && lastTag === 'commit',\n        () =>\n          `SQLite change log did not end at pinned head ${throughWatermark}`,\n      );\n    }\n  }\n\n  close(): void {\n    if (this.#closed) {\n      return;\n    }\n    this.#closed = true;\n    this.#db.close();\n  }\n\n  [Symbol.dispose](): void {\n    this.close();\n  }\n\n  /**\n   * Prepares the statements once the stream table exists, memoizing them.\n   * Returns `undefined` while the log has not been created, which is a normal\n   * startup state rather than an error.\n   */\n  #prepare(): PreparedStatements | undefined {\n    if (this.#statements === undefined && this.#streamTableExists()) {\n      this.#statements = {\n        plan: this.#db.prepare(SQLITE_CHANGE_LOG_PLAN_SQL),\n        readBatch: this.#db.prepare(SQLITE_CHANGE_LOG_READ_BATCH_SQL),\n      };\n    }\n    return this.#statements;\n  }\n\n  #streamTableExists(): boolean {\n    return (\n      this.#db\n        .prepare(TABLE_EXISTS_SQL)\n        .get<{1: number} | undefined>(CHANGE_LOG_STREAM_TABLE) !== undefined\n    );\n  }\n\n  #assertOpen(): void {\n    if (this.#closed) {\n      throw new AbortError('SQLite change log reader is closed');\n    }\n  }\n\n  #throwIfAborted(signal: AbortSignal | undefined): void {\n    this.#assertOpen();\n    if (signal?.aborted) {\n      throw new AbortError('SQLite change log read aborted');\n    }\n  }\n}\n"],"mappings":";;;;;;;;;;;AAiDA,IAAa,iCAAyC;;;;UAI5C,wBAAwB;;;;;;;;;;;;;;;;;;AAmBlC,IAAa,6BAAqC;;qCAEb,wBAAwB;;qCAExB,wBAAwB;;gBAE7C,+BAA+B;;;AAI/C,IAAa,mCAA2C;;;;;;UAM9C,wBAAwB;;;;;;AAOlC,IAAM,mBAA2B;;;;;;;;;;;;;;;;;;AAmBjC,IAAa,wBAAb,MAAyD;CACvD;CACA;CACA,UAAU;CAEV,YAAY,IAAgB,eAAuB;EACjD,KAAKA,MAAM,IAAI,SACb,GAAG,YAAY,aAAa,0BAA0B,GACtD,eACA,EAAC,UAAU,KAAI,CACjB;CACF;CAEA,KAAK,eAAoC;EACvC,KAAKC,YAAY;EACjB,MAAM,aAAa,KAAKC,SAAS;EACjC,IAAI,eAAe,KAAA,GACjB,OAAO,EAAC,MAAM,YAAW;EAE3B,MAAM,MAAM,WAAW,KAAK,IAAa,EAAC,cAAa,CAAC;EACxD,IAAI,IAAI,kBAAkB,QAAQ,IAAI,iBAAiB,MACrD,OAAO,EAAC,MAAM,YAAW;EAE3B,MAAM,EAAC,eAAe,iBAAgB;EAEtC,IAAI,gBAAgB,eAClB,OAAO;GAAC,MAAM;GAAS;EAAa;EAEtC,IAAI,gBAAgB,gBAAgB,IAAI,mBAAmB,GACzD,OAAO;GAAC,MAAM;GAAW;GAAc;EAAa;EAEtD,OAAO;GAAC,MAAM;GAAS;GAAc;EAAa;CACpD;CAEA,OAAO,KACL,eACA,kBACA,WACA,QACA,SAC6C;EAC7C,OACE,OAAO,cAAc,SAAS,KAAK,YAAY,GAC/C,8DACF;EACA,OACE,YAAY,KAAA,KAAc,OAAO,cAAc,OAAO,KAAK,UAAU,GACrE,6DACF;EACA,KAAKD,YAAY;EACjB,MAAM,aAAa,KAAKC,SAAS;EACjC,OACE,eAAe,KAAA,GACf,mDACF;EAEA,IAAI,gBAAgB;EAGpB,IAAI,UAAU,OAAO;EACrB,IAAI;EACJ,IAAI,WAAW;EAEf,OAAO,MAAM;GACX,KAAKC,gBAAgB,MAAM;GAC3B,MAAM,gBACJ,YAAY,KAAA,IAAY,YAAY,UAAU;GAChD,IAAI,kBAAkB,GACpB;GAEF,MAAM,OAAO,WAAW,UAAU,IAChC,eACA,SACA,kBACA,KAAK,IAAI,WAAW,aAAa,CACnC;GACA,KAAKA,gBAAgB,MAAM;GAE3B,IAAI,KAAK,WAAW,GAClB;GAGF,MAAM,UAAU,KAAK,IAAI,4BAA4B;GACrD,MAAM,OAAO,KAAK,GAAG,EAAE;GACvB,OAAO,SAAS,KAAA,GAAW,uCAAuC;GAClE,gBAAgB,KAAK;GACrB,UAAU,KAAK;GACf,UAAU,KAAK;GACf,YAAY,KAAK;GAKjB,MAAM;EACR;EAWA,IACE,kBAAkB,qBACjB,YAAY,KAAA,KAAa,WAAW,UAErC,OACE,kBAAkB,oBAAoB,YAAY,gBAEhD,gDAAgD,kBACpD;CAEJ;CAEA,QAAc;EACZ,IAAI,KAAKC,SACP;EAEF,KAAKA,UAAU;EACf,KAAKJ,IAAI,MAAM;CACjB;CAEA,CAAC,OAAO,WAAiB;EACvB,KAAK,MAAM;CACb;;;;;;CAOA,WAA2C;EACzC,IAAI,KAAKK,gBAAgB,KAAA,KAAa,KAAKC,mBAAmB,GAC5D,KAAKD,cAAc;GACjB,MAAM,KAAKL,IAAI,QAAQ,0BAA0B;GACjD,WAAW,KAAKA,IAAI,QAAQ,gCAAgC;EAC9D;EAEF,OAAO,KAAKK;CACd;CAEA,qBAA8B;EAC5B,OACE,KAAKL,IACF,QAAQ,gBAAgB,EACxB,IAA6B,uBAAuB,MAAM,KAAA;CAEjE;CAEA,cAAoB;EAClB,IAAI,KAAKI,SACP,MAAM,IAAI,WAAW,oCAAoC;CAE7D;CAEA,gBAAgB,QAAuC;EACrD,KAAKH,YAAY;EACjB,IAAI,QAAQ,SACV,MAAM,IAAI,WAAW,gCAAgC;CAEzD;AACF"}

package/out/zero-cache/src/services/change-streamer/storer.d.ts

@@ -8,6 +8,7 @@ import type { UpstreamStatusMessage } from '../change-source/protocol/current/st
 import { type CookieSet } from '../replicator/change-log-cookies.ts';
 import type { ReplicatorMode } from '../replicator/replicator.ts';
 import type { Service } from '../service.ts';
+import { type ChangeLogEntry } from './change-log-codec.ts';
 import type { Subscriber } from './subscriber.ts';
 export type TuningOptions = {
     backPressureLimitHeapProportion: number;
@@ -73,6 +74,28 @@ export declare class Storer implements Service {
         cookies: CookieSet;
     }>;
     getMinWatermarkForCatchup(): Promise<string | null>;
+    /**
+     * Returns the retained Postgres catchup bounds.
+     * `minWatermark` is the oldest transaction, or `null` for a new replica.
+     * `lastWatermark` is the durable head.
+     */
+    getCatchupBounds(): Promise<{
+        minWatermark: string | null;
+        lastWatermark: string;
+    }>;
+    /**
+     * Lists at most `limit` committed transaction watermarks in
+     * `(afterWatermark, throughWatermark]`, in ascending order.
+     */
+    listCommitWatermarks(afterWatermark: string, throughWatermark: string, limit: number): Promise<string[]>;
+    /**
+     * Reads `(afterWatermark, throughWatermark]` in stream order and bounded batches.
+     * This query matches the subscriber catchup query.
+     *
+     * Both queries fail when an escaped NUL reaches `change->'tag'`.
+     * Postgres cannot convert that value to text.
+     */
+    readCatchupRange(afterWatermark: string, throughWatermark: string, batchRows?: number): AsyncIterable<ChangeLogEntry[]>;
     purgeRecordsBefore(watermark: string): Promise<number>;
     /**
      * @returns The JSON stringified stream message to be sent downstream.

package/out/zero-cache/src/services/change-streamer/storer.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"storer.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/change-streamer/storer.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAUjD,OAAO,EAAC,eAAe,EAAC,MAAM,8BAA8B,CAAC;AAC7D,OAAO,EAAC,KAAK,UAAU,EAA2B,MAAM,mBAAmB,CAAC;AAC5E,OAAO,EAAY,KAAK,OAAO,EAAC,MAAM,uBAAuB,CAAC;AAE9D,OAAO,EAKL,KAAK,eAAe,EAMrB,MAAM,sCAAsC,CAAC;AAC9C,OAAO,EACL,KAAK,gBAAgB,EACrB,KAAK,MAAM,EACZ,MAAM,iDAAiD,CAAC;AACzD,OAAO,KAAK,EAAC,qBAAqB,EAAC,MAAM,6CAA6C,CAAC;AACvF,OAAO,EAIL,KAAK,SAAS,EAEf,MAAM,qCAAqC,CAAC;AAC7C,OAAO,KAAK,EAAC,cAAc,EAAC,MAAM,6BAA6B,CAAC;AAChE,OAAO,KAAK,EAAC,OAAO,EAAC,MAAM,eAAe,CAAC;AAc3C,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,iBAAiB,CAAC;AAsChD,MAAM,MAAM,aAAa,GAAG;IAC1B,+BAA+B,EAAE,MAAM,CAAC;IACxC,kBAAkB,EAAE,MAAM,CAAC;IAC3B,kBAAkB,EAAE,MAAM,CAAC;IAC3B,cAAc,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;CACrC,CAAC;AAaF;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA6BG;AACH,qBAAa,MAAO,YAAW,OAAO;;IACpC,QAAQ,CAAC,EAAE,YAAY;gBAoBrB,EAAE,EAAE,UAAU,EACd,KAAK,EAAE,OAAO,EACd,MAAM,EAAE,MAAM,EACd,gBAAgB,EAAE,MAAM,EACxB,iBAAiB,EAAE,MAAM,EACzB,EAAE,EAAE,UAAU,EACd,cAAc,EAAE,MAAM,EACtB,WAAW,EAAE,CAAC,CAAC,EAAE,MAAM,GAAG,qBAAqB,KAAK,IAAI,EACxD,OAAO,EAAE,CAAC,GAAG,EAAE,KAAK,KAAK,IAAI,EAC7B,EACE,+BAA+B,EAC/B,kBAAkB,EAClB,kBAAkB,EAClB,cAAuB,GACxB,EAAE,aAAa;IAiCZ,eAAe,CAAC,SAAS,CAAC,EAAE,SAAS,GAAG,IAAI;IA2BlD;;;;;;;;;;;;;;;;OAgBG;IACG,sCAAsC,IAAI,OAAO,CAAC;QACtD,aAAa,EAAE,MAAM,CAAC;QACtB,gBAAgB,EAAE,eAAe,EAAE,CAAC;QACpC,OAAO,EAAE,SAAS,CAAC;KACpB,CAAC;IAoDI,yBAAyB,IAAI,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC;IAOzD,kBAAkB,CAAC,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;IAyCtD;;OAEG;IACH,KAAK,CAAC,SAAS,EAAE,MAAM,EAAE,IAAI,EAAE,gBAAgB;IAkB/C,KAAK;IAIL,OAAO,CAAC,UAAU,EAAE,UAAU,EAAE,IAAI,EAAE,cAAc;IAOpD,YAAY,IAAI,OAAO,CAAC,IAAI,CAAC,GAAG,SAAS;IA6FzC;;;OAGG;IACG,GAAG;IAmdT;;;OAGG;IACG,YAAY;IAQlB;;;;OAIG;IACG,IAAI;CAcX;AAED,qBAAa,SAAS;;IAGpB,QAAQ,CAAC,cAAc,EAAE,MAAM,CAAC;IAChC,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC;gBAG5B,EAAE,EAAE,UAAU,EACd,EAAE,EAAE,eAAe,EACnB,cAAc,EAAE,MAAM,EACtB,SAAS,EAAE,MAAM;IAUb,OAAO;CAWd;AAED,qBAAa,WAAW;;gBAKV,EAAE,EAAE,UAAU,EAAE,KAAK,EAAE,OAAO,EAAE,EAAE,EAAE,UAAU;IAWpD,OAAO;CA4Bd"}
+{"version":3,"file":"storer.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/change-streamer/storer.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAUjD,OAAO,EAAC,eAAe,EAAC,MAAM,8BAA8B,CAAC;AAC7D,OAAO,EAAC,KAAK,UAAU,EAA2B,MAAM,mBAAmB,CAAC;AAC5E,OAAO,EAAY,KAAK,OAAO,EAAC,MAAM,uBAAuB,CAAC;AAE9D,OAAO,EAKL,KAAK,eAAe,EAMrB,MAAM,sCAAsC,CAAC;AAC9C,OAAO,EACL,KAAK,gBAAgB,EACrB,KAAK,MAAM,EACZ,MAAM,iDAAiD,CAAC;AACzD,OAAO,KAAK,EAAC,qBAAqB,EAAC,MAAM,6CAA6C,CAAC;AACvF,OAAO,EAIL,KAAK,SAAS,EAEf,MAAM,qCAAqC,CAAC;AAC7C,OAAO,KAAK,EAAC,cAAc,EAAC,MAAM,6BAA6B,CAAC;AAChE,OAAO,KAAK,EAAC,OAAO,EAAC,MAAM,eAAe,CAAC;AAC3C,OAAO,EAIL,KAAK,cAAc,EACpB,MAAM,uBAAuB,CAAC;AAQ/B,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,iBAAiB,CAAC;AAsChD,MAAM,MAAM,aAAa,GAAG;IAC1B,+BAA+B,EAAE,MAAM,CAAC;IACxC,kBAAkB,EAAE,MAAM,CAAC;IAC3B,kBAAkB,EAAE,MAAM,CAAC;IAC3B,cAAc,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;CACrC,CAAC;AAaF;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA6BG;AACH,qBAAa,MAAO,YAAW,OAAO;;IACpC,QAAQ,CAAC,EAAE,YAAY;gBAoBrB,EAAE,EAAE,UAAU,EACd,KAAK,EAAE,OAAO,EACd,MAAM,EAAE,MAAM,EACd,gBAAgB,EAAE,MAAM,EACxB,iBAAiB,EAAE,MAAM,EACzB,EAAE,EAAE,UAAU,EACd,cAAc,EAAE,MAAM,EACtB,WAAW,EAAE,CAAC,CAAC,EAAE,MAAM,GAAG,qBAAqB,KAAK,IAAI,EACxD,OAAO,EAAE,CAAC,GAAG,EAAE,KAAK,KAAK,IAAI,EAC7B,EACE,+BAA+B,EAC/B,kBAAkB,EAClB,kBAAkB,EAClB,cAAuB,GACxB,EAAE,aAAa;IAiCZ,eAAe,CAAC,SAAS,CAAC,EAAE,SAAS,GAAG,IAAI;IA2BlD;;;;;;;;;;;;;;;;OAgBG;IACG,sCAAsC,IAAI,OAAO,CAAC;QACtD,aAAa,EAAE,MAAM,CAAC;QACtB,gBAAgB,EAAE,eAAe,EAAE,CAAC;QACpC,OAAO,EAAE,SAAS,CAAC;KACpB,CAAC;IAoDI,yBAAyB,IAAI,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC;IAOzD;;;;OAIG;IACG,gBAAgB,IAAI,OAAO,CAAC;QAChC,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;QAC5B,aAAa,EAAE,MAAM,CAAC;KACvB,CAAC;IAUF;;;OAGG;IACG,oBAAoB,CACxB,cAAc,EAAE,MAAM,EACtB,gBAAgB,EAAE,MAAM,EACxB,KAAK,EAAE,MAAM,GACZ,OAAO,CAAC,MAAM,EAAE,CAAC;IAWpB;;;;;;OAMG;IACH,gBAAgB,CACd,cAAc,EAAE,MAAM,EACtB,gBAAgB,EAAE,MAAM,EACxB,SAAS,SAAO,GACf,aAAa,CAAC,cAAc,EAAE,CAAC;IAclC,kBAAkB,CAAC,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;IAyCtD;;OAEG;IACH,KAAK,CAAC,SAAS,EAAE,MAAM,EAAE,IAAI,EAAE,gBAAgB;IAkB/C,KAAK;IAIL,OAAO,CAAC,UAAU,EAAE,UAAU,EAAE,IAAI,EAAE,cAAc;IAOpD,YAAY,IAAI,OAAO,CAAC,IAAI,CAAC,GAAG,SAAS;IA6FzC;;;OAGG;IACG,GAAG;IAmdT;;;OAGG;IACG,YAAY;IAQlB;;;;OAIG;IACG,IAAI;CAcX;AAED,qBAAa,SAAS;;IAGpB,QAAQ,CAAC,cAAc,EAAE,MAAM,CAAC;IAChC,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC;gBAG5B,EAAE,EAAE,UAAU,EACd,EAAE,EAAE,eAAe,EACnB,cAAc,EAAE,MAAM,EACtB,SAAS,EAAE,MAAM;IAUb,OAAO;CAWd;AAED,qBAAa,WAAW;;gBAKV,EAAE,EAAE,UAAU,EAAE,KAAK,EAAE,OAAO,EAAE,EAAE,EAAE,UAAU;IAWpD,OAAO;CA4Bd"}

package/out/zero-cache/src/services/change-streamer/storer.js

@@ -161,6 +161,46 @@ var Storer = class {
       SELECT min(watermark) as "minWatermark" FROM ${this.#cdc("changeLog")}`;
 		return minWatermark;
 	}
+	/**
+	* Returns the retained Postgres catchup bounds.
+	* `minWatermark` is the oldest transaction, or `null` for a new replica.
+	* `lastWatermark` is the durable head.
+	*/
+	async getCatchupBounds() {
+		const [bounds] = await this.#db`
+      SELECT
+        (SELECT min(watermark) FROM ${this.#cdc("changeLog")}) as "minWatermark",
+        (SELECT "lastWatermark" FROM ${this.#cdc("replicationState")}) as "lastWatermark"`;
+		return bounds;
+	}
+	/**
+	* Lists at most `limit` committed transaction watermarks in
+	* `(afterWatermark, throughWatermark]`, in ascending order.
+	*/
+	async listCommitWatermarks(afterWatermark, throughWatermark, limit) {
+		return (await this.#db`
+      SELECT watermark FROM ${this.#cdc("changeLog")}
+       WHERE precommit IS NOT NULL
+         AND watermark > ${afterWatermark}
+         AND watermark <= ${throughWatermark}
+       ORDER BY watermark
+       LIMIT ${limit}`).map(({ watermark }) => watermark);
+	}
+	/**
+	* Reads `(afterWatermark, throughWatermark]` in stream order and bounded batches.
+	* This query matches the subscriber catchup query.
+	*
+	* Both queries fail when an escaped NUL reaches `change->'tag'`.
+	* Postgres cannot convert that value to text.
+	*/
+	readCatchupRange(afterWatermark, throughWatermark, batchRows = 2e3) {
+		assert(Number.isSafeInteger(batchRows) && batchRows > 0, "Postgres change log batch size must be a positive safe integer");
+		return this.#db`
+      SELECT watermark, change->'tag' as tag, change::text FROM ${this.#cdc("changeLog")}
+       WHERE watermark > ${afterWatermark}
+         AND watermark <= ${throughWatermark}
+       ORDER BY watermark, pos`.cursor(batchRows);
+	}
 	purgeRecordsBefore(watermark) {
 		return runTx(this.#db, async (sql) => {
 			await sql`