@rocicorp/zero 1.11.0-canary.3 → 1.11.0-canary.30

package/out/zero-cache/src/services/litestream/config-v5.yml

@@ -1,9 +1,11 @@
 snapshot:
   interval: ${ZERO_LITESTREAM_SNAPSHOT_BACKUP_INTERVAL_HOURS}h
   retention: ${ZERO_LITESTREAM_SNAPSHOT_RETENTION_INTERVAL_HOURS}h
+  mode: compaction
 
-# Retain raw L0 files well past worst-case restore duration so continuous
-# compaction can't delete a file an in-progress restore still needs.
+# Retain raw L0 files well past the worst-case restore or snapshot-compaction
+# duration, so continuous compaction can't delete a file that an in-progress
+# restore or compaction snapshot still needs.
 l0-retention: 1h
 l0-retention-check-interval: 5m
 
@@ -25,11 +27,7 @@ validation:
 
 dbs:
   - path: ${ZERO_REPLICA_FILE}
-    # The monitor-interval is the steady-state interval at which the db / wal
-    # is checked to create new l0 files and/or checkpoint the wal. The
-    # application will also explicitly trigger a "/sync" if the wal reaches
-    # ZERO_LITESTREAM_FORCE_CHECKPOINT_THRESHOLD_MB.
-    monitor-interval: ${ZERO_LITESTREAM_INCREMENTAL_BACKUP_INTERVAL_SECONDS}s
+    monitor-interval: ${ZERO_LITESTREAM_MONITOR_INTERVAL_SECONDS}s
     min-checkpoint-page-count: ${ZERO_LITESTREAM_MIN_CHECKPOINT_PAGE_COUNT}
     truncate-page-n: ${ZERO_LITESTREAM_TRUNCATE_PAGE_N}
     replica:
@@ -45,6 +43,7 @@ dbs:
 logging:
   level: ${ZERO_LITESTREAM_LOG_LEVEL}
   type: ${ZERO_LOG_FORMAT}
+  stderr: true # output logs to stderr so that command output can be parsed from stdout
 
 addr: localhost:${LITESTREAM_PORT}
 

package/out/zero-cache/src/services/litestream/litestream-checkpointer.js

@@ -1,7 +1,8 @@
 import { AbortError } from "../../../../shared/src/abort-error.js";
 import { RunningState } from "../running-state.js";
 //#region ../zero-cache/src/services/litestream/litestream-checkpointer.ts
-var LITESTREAM_SYNC_TIMEOUT_MS = 1e4;
+var SOFT_CHECKPOINT_WAIT_MS = 1e4;
+var LITESTREAM_SYNC_TIMEOUT_MS = 5 * 6e4;
 var PAUSE_POLL_INTERVAL_MS = 1e3;
 var PAUSE_LOG_INTERVAL_POLLS = 30;
 /**
@@ -13,11 +14,17 @@ var PAUSE_LOG_INTERVAL_POLLS = 30;
 *  - Soft (`checkpointThresholdPages`): after each committed transaction,
 *    once the un-checkpointed WAL backlog reaches the threshold, force an
 *    immediate litestream sync (which seals the WAL to LTX and attempts
-*    litestream's own PASSIVE checkpoint). Non-blocking beyond the sync call
-*    itself. Backs off by a full threshold's worth of WAL growth whenever an
-*    attempt fails to drain the WAL (e.g. litestream is mid-snapshot and its
-*    checkpoints are being skipped), so repeated failures don't reattempt on
-*    every commit.
+*    litestream's own PASSIVE checkpoint) and block the write path until it
+*    completes, but only up to {@link SOFT_CHECKPOINT_WAIT_MS}. Waiting is the
+*    backpressure: it throttles upstream reads to litestream's WAL-seal rate
+*    and, in the common case, gives litestream a window to checkpoint without
+*    writer interference. A single sync is kept in flight at a time
+*    (`#pendingSync`): if one is still running when the budget elapses, the
+*    writer proceeds and the next commit waits for that sync rather than
+*    issuing (and orphaning) another. Backs off by a full threshold's worth of
+*    WAL growth whenever an attempt fails to drain the WAL (e.g. litestream
+*    is mid-snapshot and its checkpoints are being skipped), so repeated
+*    failures don't reattempt on every commit.
 *  - Hard (`walMaxPages`, optional): if the WAL still exceeds this cap after
 *    a soft attempt, pause (block the caller) polling litestream until the
 *    WAL drains below it. This is the fail-safe against unbounded WAL growth
@@ -33,8 +40,9 @@ var LitestreamCheckpointer = class {
 	#maxWalPages;
 	#state = new RunningState("litestream-checkpointer");
 	#nextWalThreshold;
+	#pendingSync;
 	constructor(lc, db, litestream, config) {
-		this.#lc = lc.withContext("litestream-checkpointer");
+		this.#lc = lc.withContext("component", "litestream-checkpointer");
 		this.#db = db;
 		this.#litestream = litestream;
 		this.#attemptChunk = config.checkpointThresholdPages;
@@ -55,26 +63,47 @@ var LitestreamCheckpointer = class {
 	* that litestream's (disabled) truncate-page-n would have served.
 	*/
 	async maybeCheckpoint() {
-		const walPages = this.#getNumWalPages();
+		let walPages = this.#getNumWalPages();
 		if (walPages < this.#nextWalThreshold) return;
+		this.#pendingSync ??= this.#sync(walPages);
+		await this.#waitFor(this.#pendingSync, SOFT_CHECKPOINT_WAIT_MS);
+		walPages = this.#getNumWalPages();
+		if (this.#maxWalPages !== void 0 && walPages > this.#maxWalPages) walPages = await this.#pauseUntilWalDrained(this.#maxWalPages);
+		this.#nextWalThreshold = walPages + this.#attemptChunk;
+	}
+	async #waitFor(p, timeoutMs) {
+		if (await Promise.race([p, this.#state.sleep(timeoutMs).then(() => "timed-out")]) === "timed-out") this.#lc.warn?.(`timed out waiting for /sync result (${timeoutMs}ms)`);
+	}
+	/**
+	* Runs a single litestream sync, logs its outcome, and clears
+	* {@link #pendingSync} when it settles. Never rejects: sync failures and
+	* post-sync bookkeeping errors (e.g. the db being closed during shutdown)
+	* are caught and logged, so the shared promise can be awaited from multiple
+	* places without risking an unhandled rejection.
+	*/
+	async #sync(walPagesBefore) {
 		const start = performance.now();
 		let result;
 		let err;
 		try {
-			result = await this.#litestream.sync({
-				wait: false,
-				timeoutMs: LITESTREAM_SYNC_TIMEOUT_MS
-			}, this.#state.signal);
+			try {
+				result = await this.#litestream.sync({
+					wait: false,
+					timeoutMs: LITESTREAM_SYNC_TIMEOUT_MS
+				}, this.#state.signal);
+			} catch (e) {
+				err = e;
+			}
+			const elapsed = performance.now() - start;
+			const newWalPages = this.#getNumWalPages();
+			if (newWalPages < this.#nextWalThreshold) this.#lc.info?.(`checkpointed ${walPagesBefore} -> ${newWalPages} wal pages (${elapsed.toFixed(2)} ms)`, result);
+			else if (result) this.#lc.warn?.(`checkpoint skipped ${walPagesBefore} -> ${newWalPages}. a snapshot may be in progress (${elapsed.toFixed(2)} ms)`, result);
+			else this.#lc.warn?.(`checkpoint failed ${walPagesBefore} -> ${newWalPages} (${elapsed.toFixed(2)} ms)`, err);
 		} catch (e) {
-			err = e;
+			this.#lc.warn?.("litestream-checkpointer sync post-processing failed", e);
+		} finally {
+			this.#pendingSync = void 0;
 		}
-		const elapsed = performance.now() - start;
-		let newWalPages = this.#getNumWalPages();
-		if (newWalPages < this.#nextWalThreshold) this.#lc.info?.(`checkpointed ${walPages} -> ${newWalPages} wal pages (${elapsed.toFixed(2)} ms)`, result);
-		else if (result) this.#lc.warn?.(`checkpoint skipped ${walPages} -> ${newWalPages}. a snapshot may be in progress (${elapsed.toFixed(2)} ms)`, result);
-		else this.#lc.warn?.(`checkpoint failed ${walPages} -> ${newWalPages} (${elapsed.toFixed(2)} ms)`, err);
-		if (this.#maxWalPages !== void 0 && newWalPages > this.#maxWalPages) newWalPages = await this.#pauseUntilWalDrained(this.#maxWalPages);
-		this.#nextWalThreshold = newWalPages + this.#attemptChunk;
 	}
 	async #pauseUntilWalDrained(maxWalPages) {
 		for (let i = 0; this.#state.shouldRun(); i++) {
@@ -85,10 +114,8 @@ var LitestreamCheckpointer = class {
 			}
 			if (i % PAUSE_LOG_INTERVAL_POLLS === 0) this.#lc.warn?.(`waiting for litestream to successfully checkpoint ${walPages} wal pages`);
 			await this.#state.sleep(PAUSE_POLL_INTERVAL_MS);
-			await this.#litestream.sync({
-				wait: false,
-				timeoutMs: LITESTREAM_SYNC_TIMEOUT_MS
-			}, this.#state.signal).catch((e) => this.#lc.warn?.(`litestream /sync failed`, e));
+			this.#pendingSync ??= this.#sync(this.#getNumWalPages());
+			await this.#pendingSync;
 		}
 		throw new AbortError("litestream-controller stopped");
 	}

package/out/zero-cache/src/services/litestream/litestream-checkpointer.js.map

@@ -1 +1 @@
-{"version":3,"file":"litestream-checkpointer.js","names":["#lc","#db","#litestream","#attemptChunk","#maxWalPages","#state","#nextWalThreshold","#getNumWalPages","#pauseUntilWalDrained"],"sources":["../../../../../../zero-cache/src/services/litestream/litestream-checkpointer.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {AbortError} from '../../../../shared/src/abort-error.ts';\nimport type {Database} from '../../../../zqlite/src/db.ts';\nimport type {ForceCheckpointConfig} from '../replicator/write-worker-client.ts';\nimport {RunningState} from '../running-state.ts';\nimport type {\n  LitestreamController,\n  SyncResponse,\n} from './litestream-controller.ts';\n\nconst LITESTREAM_SYNC_TIMEOUT_MS = 10_000;\nconst PAUSE_POLL_INTERVAL_MS = 1_000;\nconst PAUSE_LOG_INTERVAL_POLLS = 30;\n\n/**\n * Forces litestream to checkpoint the WAL from the write path, so a sustained\n * high write rate can't grow the WAL until litestream's own blocking TRUNCATE\n * checkpoint locks the db (see ReplicationPauseCheckpointConfig).\n *\n * Two tiers:\n *  - Soft (`checkpointThresholdPages`): after each committed transaction,\n *    once the un-checkpointed WAL backlog reaches the threshold, force an\n *    immediate litestream sync (which seals the WAL to LTX and attempts\n *    litestream's own PASSIVE checkpoint). Non-blocking beyond the sync call\n *    itself. Backs off by a full threshold's worth of WAL growth whenever an\n *    attempt fails to drain the WAL (e.g. litestream is mid-snapshot and its\n *    checkpoints are being skipped), so repeated failures don't reattempt on\n *    every commit.\n *  - Hard (`walMaxPages`, optional): if the WAL still exceeds this cap after\n *    a soft attempt, pause (block the caller) polling litestream until the\n *    WAL drains below it. This is the fail-safe against unbounded WAL growth\n *    when litestream structurally cannot checkpoint — e.g. an in-progress\n *    snapshot holds litestream's checkpoint lock for its full duration, so\n *    neither PASSIVE nor TRUNCATE checkpoints can run until it completes.\n */\nexport class LitestreamCheckpointer {\n  readonly #lc: LogContext;\n  readonly #db: Database;\n  readonly #litestream: LitestreamController;\n  readonly #attemptChunk: number;\n  readonly #maxWalPages: number | undefined;\n  readonly #state = new RunningState('litestream-checkpointer');\n  #nextWalThreshold: number;\n\n  constructor(\n    lc: LogContext,\n    db: Database,\n    litestream: LitestreamController,\n    config: ForceCheckpointConfig,\n  ) {\n    this.#lc = lc.withContext('litestream-checkpointer');\n    this.#db = db;\n    this.#litestream = litestream;\n    this.#attemptChunk = config.checkpointThresholdPages;\n    this.#nextWalThreshold = config.checkpointThresholdPages;\n    this.#maxWalPages = config.maxWalPages;\n  }\n\n  #getNumWalPages(): number {\n    // wal_checkpoint(NOOP) reads WAL status without doing any work: `log` is\n    // the current WAL frame count and `checkpointed` is how many of those\n    // frames are already applied to the main db, so `log - checkpointed` is\n    // the un-checkpointed backlog that a forced sync would actually drain.\n    // (Both reset once litestream restarts the WAL.)\n    const [{log, checkpointed}] = this.#db.pragma<{\n      log: number;\n      checkpointed: number;\n    }>('wal_checkpoint(NOOP)');\n    return log - checkpointed;\n  }\n\n  /**\n   * Call after a committed transaction. No-ops unless the WAL has grown to\n   * at least the current threshold, in which case it triggers a litestream\n   * sync to perform a checkpoint. Assuming that snapshots are disabled, the\n   * checkpoint always succeeds. In the case where the server is configured to\n   * perform snapshots and checkpoints fail, the \"emergency break\"\n   * {@link #pauseUntilWalDrained()} option provides the disk use protection\n   * that litestream's (disabled) truncate-page-n would have served.\n   */\n  async maybeCheckpoint(): Promise<void> {\n    const walPages = this.#getNumWalPages();\n    if (walPages < this.#nextWalThreshold) {\n      return;\n    }\n\n    const start = performance.now();\n    let result: SyncResponse | undefined;\n    let err: unknown;\n    try {\n      result = await this.#litestream.sync(\n        {\n          wait: false,\n          timeoutMs: LITESTREAM_SYNC_TIMEOUT_MS,\n        },\n        this.#state.signal,\n      );\n    } catch (e) {\n      err = e;\n    }\n\n    const elapsed = performance.now() - start;\n    let newWalPages = this.#getNumWalPages();\n    if (newWalPages < this.#nextWalThreshold) {\n      this.#lc.info?.(\n        `checkpointed ${walPages} -> ${newWalPages} wal pages (${elapsed.toFixed(2)} ms)`,\n        result,\n      );\n    } else if (result) {\n      this.#lc.warn?.(\n        `checkpoint skipped ${walPages} -> ${newWalPages}. a snapshot may be in progress (${elapsed.toFixed(2)} ms)`,\n        result,\n      );\n    } else {\n      this.#lc.warn?.(\n        `checkpoint failed ${walPages} -> ${newWalPages} (${elapsed.toFixed(2)} ms)`,\n        err,\n      );\n    }\n\n    if (this.#maxWalPages !== undefined && newWalPages > this.#maxWalPages) {\n      // Emergency brake when best-effort checkpoints are being skipped\n      // (i.e. during a snapshot) and the WAL is approaching disk capacity.\n      // Pausing replication is favorable to litestream's truncate-page-n\n      // because the latter involves re-encoding the database under the lock\n      // (https://github.com/benbjohnson/litestream/issues/1332), which would\n      // incur the full latency of an additional snapshot.\n      newWalPages = await this.#pauseUntilWalDrained(this.#maxWalPages);\n    }\n\n    // In the common case where checkpoints are successful, this is always\n    // just `attemptChunk`. When a checkpoint doesn't drain the WAL (e.g.\n    // litestream is performing a snapshot), wait for another chunk of growth\n    // before the next attempt.\n    this.#nextWalThreshold = newWalPages + this.#attemptChunk;\n  }\n\n  async #pauseUntilWalDrained(maxWalPages: number): Promise<number> {\n    for (let i = 0; this.#state.shouldRun(); i++) {\n      const walPages = this.#getNumWalPages();\n      if (walPages < maxWalPages) {\n        this.#lc.info?.(\n          `wal size (${walPages} pages) is under the max limit of ${maxWalPages}`,\n        );\n        return walPages;\n      }\n\n      // Log every 30 seconds.\n      if (i % PAUSE_LOG_INTERVAL_POLLS === 0) {\n        this.#lc.warn?.(\n          `waiting for litestream to successfully checkpoint ${walPages} wal pages`,\n        );\n      }\n      await this.#state.sleep(PAUSE_POLL_INTERVAL_MS);\n      await this.#litestream\n        .sync(\n          {wait: false, timeoutMs: LITESTREAM_SYNC_TIMEOUT_MS},\n          this.#state.signal,\n        )\n        .catch(e => this.#lc.warn?.(`litestream /sync failed`, e));\n    }\n    throw new AbortError('litestream-controller stopped');\n  }\n\n  /** Releases the underlying litestream controller's connection. */\n  close(): void {\n    this.#litestream.close();\n    this.#state.stop(this.#lc);\n  }\n}\n"],"mappings":";;;AAUA,IAAM,6BAA6B;AACnC,IAAM,yBAAyB;AAC/B,IAAM,2BAA2B;;;;;;;;;;;;;;;;;;;;;;AAuBjC,IAAa,yBAAb,MAAoC;CAClC;CACA;CACA;CACA;CACA;CACA,SAAkB,IAAI,aAAa,yBAAyB;CAC5D;CAEA,YACE,IACA,IACA,YACA,QACA;EACA,KAAKA,MAAM,GAAG,YAAY,yBAAyB;EACnD,KAAKC,MAAM;EACX,KAAKC,cAAc;EACnB,KAAKC,gBAAgB,OAAO;EAC5B,KAAKG,oBAAoB,OAAO;EAChC,KAAKF,eAAe,OAAO;CAC7B;CAEA,kBAA0B;EAMxB,MAAM,CAAC,EAAC,KAAK,kBAAiB,KAAKH,IAAI,OAGpC,sBAAsB;EACzB,OAAO,MAAM;CACf;;;;;;;;;;CAWA,MAAM,kBAAiC;EACrC,MAAM,WAAW,KAAKM,gBAAgB;EACtC,IAAI,WAAW,KAAKD,mBAClB;EAGF,MAAM,QAAQ,YAAY,IAAI;EAC9B,IAAI;EACJ,IAAI;EACJ,IAAI;GACF,SAAS,MAAM,KAAKJ,YAAY,KAC9B;IACE,MAAM;IACN,WAAW;GACb,GACA,KAAKG,OAAO,MACd;EACF,SAAS,GAAG;GACV,MAAM;EACR;EAEA,MAAM,UAAU,YAAY,IAAI,IAAI;EACpC,IAAI,cAAc,KAAKE,gBAAgB;EACvC,IAAI,cAAc,KAAKD,mBACrB,KAAKN,IAAI,OACP,gBAAgB,SAAS,MAAM,YAAY,cAAc,QAAQ,QAAQ,CAAC,EAAE,OAC5E,MACF;OACK,IAAI,QACT,KAAKA,IAAI,OACP,sBAAsB,SAAS,MAAM,YAAY,mCAAmC,QAAQ,QAAQ,CAAC,EAAE,OACvG,MACF;OAEA,KAAKA,IAAI,OACP,qBAAqB,SAAS,MAAM,YAAY,IAAI,QAAQ,QAAQ,CAAC,EAAE,OACvE,GACF;EAGF,IAAI,KAAKI,iBAAiB,KAAA,KAAa,cAAc,KAAKA,cAOxD,cAAc,MAAM,KAAKI,sBAAsB,KAAKJ,YAAY;EAOlE,KAAKE,oBAAoB,cAAc,KAAKH;CAC9C;CAEA,MAAMK,sBAAsB,aAAsC;EAChE,KAAK,IAAI,IAAI,GAAG,KAAKH,OAAO,UAAU,GAAG,KAAK;GAC5C,MAAM,WAAW,KAAKE,gBAAgB;GACtC,IAAI,WAAW,aAAa;IAC1B,KAAKP,IAAI,OACP,aAAa,SAAS,oCAAoC,aAC5D;IACA,OAAO;GACT;GAGA,IAAI,IAAI,6BAA6B,GACnC,KAAKA,IAAI,OACP,qDAAqD,SAAS,WAChE;GAEF,MAAM,KAAKK,OAAO,MAAM,sBAAsB;GAC9C,MAAM,KAAKH,YACR,KACC;IAAC,MAAM;IAAO,WAAW;GAA0B,GACnD,KAAKG,OAAO,MACd,EACC,OAAM,MAAK,KAAKL,IAAI,OAAO,2BAA2B,CAAC,CAAC;EAC7D;EACA,MAAM,IAAI,WAAW,+BAA+B;CACtD;;CAGA,QAAc;EACZ,KAAKE,YAAY,MAAM;EACvB,KAAKG,OAAO,KAAK,KAAKL,GAAG;CAC3B;AACF"}
+{"version":3,"file":"litestream-checkpointer.js","names":["#lc","#db","#litestream","#attemptChunk","#maxWalPages","#state","#nextWalThreshold","#getNumWalPages","#pendingSync","#sync","#waitFor","#pauseUntilWalDrained"],"sources":["../../../../../../zero-cache/src/services/litestream/litestream-checkpointer.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {AbortError} from '../../../../shared/src/abort-error.ts';\nimport type {Database} from '../../../../zqlite/src/db.ts';\nimport type {ForceCheckpointConfig} from '../replicator/write-worker-client.ts';\nimport {RunningState} from '../running-state.ts';\nimport type {\n  LitestreamController,\n  SyncResponse,\n} from './litestream-controller.ts';\n\n// How long the soft path blocks the write path waiting for an in-flight sync\n// before letting the writer proceed. This is a backpressure knob, *not* a\n// request timeout: letting it elapse leaves the sync running (see #pendingSync)\n// rather than abandoning it. In normal operation litestream seals + runs its\n// PASSIVE checkpoint well within this budget; a sync that regularly exceeds it\n// indicates a litestream-side problem to fix (e.g. unintended snapshot), not a\n// value to raise.\nexport const SOFT_CHECKPOINT_WAIT_MS = 10_000;\n// Liveness bound on the /sync request itself, well above the soft wait budget.\n// It exists only to recover from a genuinely wedged litestream (so #pendingSync\n// can't hang forever).\nconst LITESTREAM_SYNC_TIMEOUT_MS = 5 * 60_000;\nconst PAUSE_POLL_INTERVAL_MS = 1_000;\nconst PAUSE_LOG_INTERVAL_POLLS = 30;\n\n/**\n * Forces litestream to checkpoint the WAL from the write path, so a sustained\n * high write rate can't grow the WAL until litestream's own blocking TRUNCATE\n * checkpoint locks the db (see ReplicationPauseCheckpointConfig).\n *\n * Two tiers:\n *  - Soft (`checkpointThresholdPages`): after each committed transaction,\n *    once the un-checkpointed WAL backlog reaches the threshold, force an\n *    immediate litestream sync (which seals the WAL to LTX and attempts\n *    litestream's own PASSIVE checkpoint) and block the write path until it\n *    completes, but only up to {@link SOFT_CHECKPOINT_WAIT_MS}. Waiting is the\n *    backpressure: it throttles upstream reads to litestream's WAL-seal rate\n *    and, in the common case, gives litestream a window to checkpoint without\n *    writer interference. A single sync is kept in flight at a time\n *    (`#pendingSync`): if one is still running when the budget elapses, the\n *    writer proceeds and the next commit waits for that sync rather than\n *    issuing (and orphaning) another. Backs off by a full threshold's worth of\n *    WAL growth whenever an attempt fails to drain the WAL (e.g. litestream\n *    is mid-snapshot and its checkpoints are being skipped), so repeated\n *    failures don't reattempt on every commit.\n *  - Hard (`walMaxPages`, optional): if the WAL still exceeds this cap after\n *    a soft attempt, pause (block the caller) polling litestream until the\n *    WAL drains below it. This is the fail-safe against unbounded WAL growth\n *    when litestream structurally cannot checkpoint — e.g. an in-progress\n *    snapshot holds litestream's checkpoint lock for its full duration, so\n *    neither PASSIVE nor TRUNCATE checkpoints can run until it completes.\n */\nexport class LitestreamCheckpointer {\n  readonly #lc: LogContext;\n  readonly #db: Database;\n  readonly #litestream: LitestreamController;\n  readonly #attemptChunk: number;\n  readonly #maxWalPages: number | undefined;\n  readonly #state = new RunningState('litestream-checkpointer');\n  #nextWalThreshold: number;\n  #pendingSync: Promise<void> | undefined;\n\n  constructor(\n    lc: LogContext,\n    db: Database,\n    litestream: LitestreamController,\n    config: ForceCheckpointConfig,\n  ) {\n    this.#lc = lc.withContext('component', 'litestream-checkpointer');\n    this.#db = db;\n    this.#litestream = litestream;\n    this.#attemptChunk = config.checkpointThresholdPages;\n    this.#nextWalThreshold = config.checkpointThresholdPages;\n    this.#maxWalPages = config.maxWalPages;\n  }\n\n  #getNumWalPages(): number {\n    // wal_checkpoint(NOOP) reads WAL status without doing any work: `log` is\n    // the current WAL frame count and `checkpointed` is how many of those\n    // frames are already applied to the main db, so `log - checkpointed` is\n    // the un-checkpointed backlog that a forced sync would actually drain.\n    // (Both reset once litestream restarts the WAL.)\n    const [{log, checkpointed}] = this.#db.pragma<{\n      log: number;\n      checkpointed: number;\n    }>('wal_checkpoint(NOOP)');\n    return log - checkpointed;\n  }\n\n  /**\n   * Call after a committed transaction. No-ops unless the WAL has grown to\n   * at least the current threshold, in which case it triggers a litestream\n   * sync to perform a checkpoint. Assuming that snapshots are disabled, the\n   * checkpoint always succeeds. In the case where the server is configured to\n   * perform snapshots and checkpoints fail, the \"emergency break\"\n   * {@link #pauseUntilWalDrained()} option provides the disk use protection\n   * that litestream's (disabled) truncate-page-n would have served.\n   */\n  async maybeCheckpoint(): Promise<void> {\n    let walPages = this.#getNumWalPages();\n    if (walPages < this.#nextWalThreshold) {\n      return;\n    }\n\n    // Single-flight: reuse the outstanding sync if one is already running.\n    this.#pendingSync ??= this.#sync(walPages);\n\n    // Soft backpressure: block the write path until the sync completes, but no\n    // longer than the wait budget. Letting the budget elapse lets the writer\n    // proceed *without* cancelling the sync (it keeps running, tracked by\n    // #pendingSync); unbounded WAL growth is caught by the hard cap below.\n    await this.#waitFor(this.#pendingSync, SOFT_CHECKPOINT_WAIT_MS);\n\n    walPages = this.#getNumWalPages();\n    if (this.#maxWalPages !== undefined && walPages > this.#maxWalPages) {\n      // Emergency brake when best-effort checkpoints are being skipped\n      // (i.e. during a snapshot) and the WAL is approaching disk capacity.\n      // Pausing replication is favorable to litestream's truncate-page-n\n      // because the latter involves re-encoding the database under the lock\n      // (https://github.com/benbjohnson/litestream/issues/1332), which would\n      // incur the full latency of an additional snapshot.\n      walPages = await this.#pauseUntilWalDrained(this.#maxWalPages);\n    }\n\n    // In the common case where checkpoints are successful, this is always\n    // just `attemptChunk`. When a checkpoint doesn't drain the WAL (e.g.\n    // litestream is performing a snapshot), wait for another chunk of growth\n    // before the next attempt.\n    this.#nextWalThreshold = walPages + this.#attemptChunk;\n  }\n\n  async #waitFor(p: Promise<void>, timeoutMs: number): Promise<void> {\n    const result = await Promise.race([\n      p,\n      this.#state.sleep(timeoutMs).then(() => 'timed-out' as const),\n    ]);\n    if (result === 'timed-out') {\n      this.#lc.warn?.(`timed out waiting for /sync result (${timeoutMs}ms)`);\n    }\n  }\n\n  /**\n   * Runs a single litestream sync, logs its outcome, and clears\n   * {@link #pendingSync} when it settles. Never rejects: sync failures and\n   * post-sync bookkeeping errors (e.g. the db being closed during shutdown)\n   * are caught and logged, so the shared promise can be awaited from multiple\n   * places without risking an unhandled rejection.\n   */\n  async #sync(walPagesBefore: number): Promise<void> {\n    const start = performance.now();\n    let result: SyncResponse | undefined;\n    let err: unknown;\n    try {\n      try {\n        result = await this.#litestream.sync(\n          {wait: false, timeoutMs: LITESTREAM_SYNC_TIMEOUT_MS},\n          this.#state.signal,\n        );\n      } catch (e) {\n        err = e;\n      }\n\n      const elapsed = performance.now() - start;\n      const newWalPages = this.#getNumWalPages();\n      if (newWalPages < this.#nextWalThreshold) {\n        this.#lc.info?.(\n          `checkpointed ${walPagesBefore} -> ${newWalPages} wal pages (${elapsed.toFixed(2)} ms)`,\n          result,\n        );\n      } else if (result) {\n        this.#lc.warn?.(\n          `checkpoint skipped ${walPagesBefore} -> ${newWalPages}. a snapshot may be in progress (${elapsed.toFixed(2)} ms)`,\n          result,\n        );\n      } else {\n        this.#lc.warn?.(\n          `checkpoint failed ${walPagesBefore} -> ${newWalPages} (${elapsed.toFixed(2)} ms)`,\n          err,\n        );\n      }\n    } catch (e) {\n      this.#lc.warn?.('litestream-checkpointer sync post-processing failed', e);\n    } finally {\n      this.#pendingSync = undefined;\n    }\n  }\n\n  async #pauseUntilWalDrained(maxWalPages: number): Promise<number> {\n    for (let i = 0; this.#state.shouldRun(); i++) {\n      const walPages = this.#getNumWalPages();\n      if (walPages < maxWalPages) {\n        this.#lc.info?.(\n          `wal size (${walPages} pages) is under the max limit of ${maxWalPages}`,\n        );\n        return walPages;\n      }\n\n      // Log every 30 seconds.\n      if (i % PAUSE_LOG_INTERVAL_POLLS === 0) {\n        this.#lc.warn?.(\n          `waiting for litestream to successfully checkpoint ${walPages} wal pages`,\n        );\n      }\n      await this.#state.sleep(PAUSE_POLL_INTERVAL_MS);\n      // Single-flight: reuse the outstanding sync if one is already running.\n      this.#pendingSync ??= this.#sync(this.#getNumWalPages());\n      // await the full liveness timeout configured in #sync()\n      await this.#pendingSync;\n    }\n    throw new AbortError('litestream-controller stopped');\n  }\n\n  /** Releases the underlying litestream controller's connection. */\n  close(): void {\n    this.#litestream.close();\n    this.#state.stop(this.#lc);\n  }\n}\n"],"mappings":";;;AAiBA,IAAa,0BAA0B;AAIvC,IAAM,6BAA6B,IAAI;AACvC,IAAM,yBAAyB;AAC/B,IAAM,2BAA2B;;;;;;;;;;;;;;;;;;;;;;;;;;;;AA6BjC,IAAa,yBAAb,MAAoC;CAClC;CACA;CACA;CACA;CACA;CACA,SAAkB,IAAI,aAAa,yBAAyB;CAC5D;CACA;CAEA,YACE,IACA,IACA,YACA,QACA;EACA,KAAKA,MAAM,GAAG,YAAY,aAAa,yBAAyB;EAChE,KAAKC,MAAM;EACX,KAAKC,cAAc;EACnB,KAAKC,gBAAgB,OAAO;EAC5B,KAAKG,oBAAoB,OAAO;EAChC,KAAKF,eAAe,OAAO;CAC7B;CAEA,kBAA0B;EAMxB,MAAM,CAAC,EAAC,KAAK,kBAAiB,KAAKH,IAAI,OAGpC,sBAAsB;EACzB,OAAO,MAAM;CACf;;;;;;;;;;CAWA,MAAM,kBAAiC;EACrC,IAAI,WAAW,KAAKM,gBAAgB;EACpC,IAAI,WAAW,KAAKD,mBAClB;EAIF,KAAKE,iBAAiB,KAAKC,MAAM,QAAQ;EAMzC,MAAM,KAAKC,SAAS,KAAKF,cAAc,uBAAuB;EAE9D,WAAW,KAAKD,gBAAgB;EAChC,IAAI,KAAKH,iBAAiB,KAAA,KAAa,WAAW,KAAKA,cAOrD,WAAW,MAAM,KAAKO,sBAAsB,KAAKP,YAAY;EAO/D,KAAKE,oBAAoB,WAAW,KAAKH;CAC3C;CAEA,MAAMO,SAAS,GAAkB,WAAkC;EAKjE,IAAI,MAJiB,QAAQ,KAAK,CAChC,GACA,KAAKL,OAAO,MAAM,SAAS,EAAE,WAAW,WAAoB,CAC9D,CAAC,MACc,aACb,KAAKL,IAAI,OAAO,uCAAuC,UAAU,IAAI;CAEzE;;;;;;;;CASA,MAAMS,MAAM,gBAAuC;EACjD,MAAM,QAAQ,YAAY,IAAI;EAC9B,IAAI;EACJ,IAAI;EACJ,IAAI;GACF,IAAI;IACF,SAAS,MAAM,KAAKP,YAAY,KAC9B;KAAC,MAAM;KAAO,WAAW;IAA0B,GACnD,KAAKG,OAAO,MACd;GACF,SAAS,GAAG;IACV,MAAM;GACR;GAEA,MAAM,UAAU,YAAY,IAAI,IAAI;GACpC,MAAM,cAAc,KAAKE,gBAAgB;GACzC,IAAI,cAAc,KAAKD,mBACrB,KAAKN,IAAI,OACP,gBAAgB,eAAe,MAAM,YAAY,cAAc,QAAQ,QAAQ,CAAC,EAAE,OAClF,MACF;QACK,IAAI,QACT,KAAKA,IAAI,OACP,sBAAsB,eAAe,MAAM,YAAY,mCAAmC,QAAQ,QAAQ,CAAC,EAAE,OAC7G,MACF;QAEA,KAAKA,IAAI,OACP,qBAAqB,eAAe,MAAM,YAAY,IAAI,QAAQ,QAAQ,CAAC,EAAE,OAC7E,GACF;EAEJ,SAAS,GAAG;GACV,KAAKA,IAAI,OAAO,uDAAuD,CAAC;EAC1E,UAAU;GACR,KAAKQ,eAAe,KAAA;EACtB;CACF;CAEA,MAAMG,sBAAsB,aAAsC;EAChE,KAAK,IAAI,IAAI,GAAG,KAAKN,OAAO,UAAU,GAAG,KAAK;GAC5C,MAAM,WAAW,KAAKE,gBAAgB;GACtC,IAAI,WAAW,aAAa;IAC1B,KAAKP,IAAI,OACP,aAAa,SAAS,oCAAoC,aAC5D;IACA,OAAO;GACT;GAGA,IAAI,IAAI,6BAA6B,GACnC,KAAKA,IAAI,OACP,qDAAqD,SAAS,WAChE;GAEF,MAAM,KAAKK,OAAO,MAAM,sBAAsB;GAE9C,KAAKG,iBAAiB,KAAKC,MAAM,KAAKF,gBAAgB,CAAC;GAEvD,MAAM,KAAKC;EACb;EACA,MAAM,IAAI,WAAW,+BAA+B;CACtD;;CAGA,QAAc;EACZ,KAAKN,YAAY,MAAM;EACvB,KAAKG,OAAO,KAAK,KAAKL,GAAG;CAC3B;AACF"}

package/out/zero-cache/src/services/litestream/litestream-controller.d.ts

@@ -28,9 +28,17 @@ export type SyncOptions = {
      * returning without waiting on the backup store. Use false on the write path
      * where the goal is to keep the WAL small, not to durably back it up.
      */
-    wait?: boolean;
+    wait?: boolean | undefined;
     /** Overall request timeout in milliseconds. Defaults to 30s. */
-    timeoutMs?: number;
+    timeoutMs?: number | undefined;
+    /**
+     * Server-side deadline in seconds, sent as litestream's `timeout`. When set,
+     * litestream ends the request itself (with an error response) once it
+     * elapses. Set it below {@link timeoutMs} so that litestream, rather than
+     * the client, ends a slow request. (litestream defaults to 30s when `wait`
+     * is set, and to no deadline otherwise.)
+     */
+    serverTimeoutSeconds?: number | undefined;
 };
 /**
  * IPC client for the litestream control server. litestream serves an HTTP API

package/out/zero-cache/src/services/litestream/litestream-controller.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"litestream-controller.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/litestream/litestream-controller.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAEjD;;;;;;;GAOG;AACH,wBAAgB,oBAAoB,CAAC,WAAW,EAAE,MAAM,GAAG,MAAM,CAEhE;AAID,qEAAqE;AACrE,MAAM,MAAM,YAAY,GAAG;IACzB;;;;OAIG;IACH,MAAM,EAAE,QAAQ,GAAG,cAAc,GAAG,WAAW,CAAC;IAChD,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,CAAC;IACb,eAAe,EAAE,MAAM,CAAC;CACzB,CAAC;AAEF,MAAM,MAAM,WAAW,GAAG;IACxB;;;;;;OAMG;IACH,IAAI,CAAC,EAAE,OAAO,CAAC;IAEf,gEAAgE;IAChE,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB,CAAC;AAEF;;;;;;;;;GASG;AACH,qBAAa,oBAAoB;;IAM/B;;;;;;OAMG;IACH,YACE,EAAE,EAAE,UAAU,EACd,WAAW,EAAE,MAAM,EACnB,UAAU,SAAoC,EAQ/C;IAED;;;;;;;;;;OAUG;IACG,IAAI,CACR,IAAI,GAAE,WAAgB,EACtB,MAAM,CAAC,EAAE,WAAW,GACnB,OAAO,CAAC,YAAY,CAAC,CAUvB;IAED,iDAAiD;IACjD,KAAK,IAAI,IAAI,CAEZ;CA4DF"}
+{"version":3,"file":"litestream-controller.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/litestream/litestream-controller.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAEjD;;;;;;;GAOG;AACH,wBAAgB,oBAAoB,CAAC,WAAW,EAAE,MAAM,GAAG,MAAM,CAEhE;AAID,qEAAqE;AACrE,MAAM,MAAM,YAAY,GAAG;IACzB;;;;OAIG;IACH,MAAM,EAAE,QAAQ,GAAG,cAAc,GAAG,WAAW,CAAC;IAChD,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,CAAC;IACb,eAAe,EAAE,MAAM,CAAC;CACzB,CAAC;AAEF,MAAM,MAAM,WAAW,GAAG;IACxB;;;;;;OAMG;IACH,IAAI,CAAC,EAAE,OAAO,GAAG,SAAS,CAAC;IAE3B,gEAAgE;IAChE,SAAS,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAE/B;;;;;;OAMG;IACH,oBAAoB,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;CAC3C,CAAC;AAEF;;;;;;;;;GASG;AACH,qBAAa,oBAAoB;;IAM/B;;;;;;OAMG;IACH,YACE,EAAE,EAAE,UAAU,EACd,WAAW,EAAE,MAAM,EACnB,UAAU,SAAoC,EAQ/C;IAED;;;;;;;;;;OAUG;IACG,IAAI,CACR,IAAI,GAAE,WAAgB,EACtB,MAAM,CAAC,EAAE,WAAW,GACnB,OAAO,CAAC,YAAY,CAAC,CAkBvB;IAED,iDAAiD;IACjD,KAAK,IAAI,IAAI,CAEZ;CA4DF"}

package/out/zero-cache/src/services/litestream/litestream-controller.js

@@ -56,10 +56,11 @@ var LitestreamController = class {
 	* than blocking application progress on it.
 	*/
 	async sync(opts = {}, signal) {
-		const { wait = false, timeoutMs = DEFAULT_REQUEST_TIMEOUT_MS } = opts;
+		const { wait = false, timeoutMs = DEFAULT_REQUEST_TIMEOUT_MS, serverTimeoutSeconds } = opts;
 		const result = await this.#post("/sync", {
 			path: this.#replicaFile,
-			wait
+			wait,
+			...serverTimeoutSeconds ? { timeout: serverTimeoutSeconds } : {}
 		}, timeoutMs, signal);
 		this.#lc.debug?.("litestream sync", result);
 		return result;

package/out/zero-cache/src/services/litestream/litestream-controller.js.map

@@ -1 +1 @@
-{"version":3,"file":"litestream-controller.js","names":["#lc","#replicaFile","#socketPath","#agent","#post"],"sources":["../../../../../../zero-cache/src/services/litestream/litestream-controller.ts"],"sourcesContent":["import type {IncomingMessage} from 'node:http';\nimport {Agent, request} from 'node:http';\nimport {json as readJSON} from 'node:stream/consumers';\nimport type {LogContext} from '@rocicorp/logger';\n\n/**\n * Path of the litestream control socket, derived from the replica file's\n * directory. The socket must live on a filesystem the (non-root) runtime user\n * can write — the replica directory is guaranteed writable since litestream\n * writes the replica there — so it must NOT default to a root-owned location\n * like /var/run. Kept in sync with config-v5.yml's `socket.path`, which is set\n * from `ZERO_LITESTREAM_SOCKET_PATH` (see getLitestream() in commands.ts).\n */\nexport function litestreamSocketPath(replicaFile: string): string {\n  return `${replicaFile}.sock`;\n}\n\nconst DEFAULT_REQUEST_TIMEOUT_MS = 30_000;\n\n/** Response body from litestream's `POST /sync` control endpoint. */\nexport type SyncResponse = {\n  /**\n   * `synced` when `wait` was true and the upload completed, `synced_local`\n   * when the WAL was sealed locally without waiting on the remote replica, or\n   * `no_change` when there was nothing to sync.\n   */\n  status: 'synced' | 'synced_local' | 'no_change';\n  path: string;\n  txid: number;\n  replicated_txid: number;\n};\n\nexport type SyncOptions = {\n  /**\n   * When true, block until the sealed WAL has been uploaded to the remote\n   * replica (litestream's SyncAndWait). When false (the default), litestream\n   * only seals the WAL into local LTX files and runs its checkpoint thresholds,\n   * returning without waiting on the backup store. Use false on the write path\n   * where the goal is to keep the WAL small, not to durably back it up.\n   */\n  wait?: boolean;\n\n  /** Overall request timeout in milliseconds. Defaults to 30s. */\n  timeoutMs?: number;\n};\n\n/**\n * IPC client for the litestream control server. litestream serves an HTTP API\n * over a Unix domain socket (enabled via the `socket:` block in config-v5.yml);\n * because zero-cache runs litestream as a subprocess in the same container, the\n * socket is reachable directly on the local filesystem.\n *\n * A single keep-alive connection is held open and reused across calls, so a\n * high-frequency caller (e.g. write-path backpressure) does not pay a\n * connect/handshake per request.\n */\nexport class LitestreamController {\n  readonly #lc: LogContext;\n  readonly #replicaFile: string;\n  readonly #socketPath: string;\n  readonly #agent: Agent;\n\n  /**\n   * @param replicaFile Path of the replica database litestream is managing —\n   *   the same value passed to litestream as `ZERO_REPLICA_FILE`. This is the\n   *   `path` litestream matches control requests against.\n   * @param socketPath Path of the control socket; defaults to the path derived\n   *   from `replicaFile`, matching config-v5.yml's `socket.path`.\n   */\n  constructor(\n    lc: LogContext,\n    replicaFile: string,\n    socketPath = litestreamSocketPath(replicaFile),\n  ) {\n    this.#lc = lc.withContext('component', 'litestream-controller');\n    this.#replicaFile = replicaFile;\n    this.#socketPath = socketPath;\n    // keepAlive with a single socket keeps one warm connection to the control\n    // socket that is reused across calls.\n    this.#agent = new Agent({keepAlive: true, maxSockets: 1});\n  }\n\n  /**\n   * Forces litestream to sync the WAL into LTX files immediately and run its\n   * checkpoint thresholds, instead of waiting for the next monitor-interval\n   * tick. The checkpoint itself is still litestream's usual (non-blocking)\n   * PASSIVE attempt, so it is skipped if the database is busy — call this once\n   * writes are quiesced.\n   *\n   * Best-effort: callers should tolerate rejection (e.g. ECONNREFUSED/ENOENT\n   * while litestream is restarting or before it has created the socket) rather\n   * than blocking application progress on it.\n   */\n  async sync(\n    opts: SyncOptions = {},\n    signal?: AbortSignal,\n  ): Promise<SyncResponse> {\n    const {wait = false, timeoutMs = DEFAULT_REQUEST_TIMEOUT_MS} = opts;\n    const result = await this.#post<SyncResponse>(\n      '/sync',\n      {path: this.#replicaFile, wait},\n      timeoutMs,\n      signal,\n    );\n    this.#lc.debug?.('litestream sync', result);\n    return result;\n  }\n\n  /** Releases the pooled keep-alive connection. */\n  close(): void {\n    this.#agent.destroy();\n  }\n\n  // Note: Uses ye'old node:http for its unix socketPath support.\n  //   fetch() would have less boiler-plate, but it would require importing\n  //   the `unidici` package to provide a fetch Agent that supports sockets.\n  #post<T>(\n    path: string,\n    body: unknown,\n    timeoutMs: number,\n    signal?: AbortSignal,\n  ): Promise<T> {\n    const payload = JSON.stringify(body);\n    return new Promise<T>((resolve, reject) => {\n      const req = request(\n        {\n          socketPath: this.#socketPath,\n          method: 'POST',\n          path,\n          agent: this.#agent,\n          timeout: timeoutMs,\n          headers: {\n            'content-type': 'application/json',\n            'content-length': Buffer.byteLength(payload),\n          },\n          signal,\n        },\n        (res: IncomingMessage) => {\n          const status = res.statusCode ?? 0;\n          // litestream's success and error responses are both always JSON\n          // (writeJSON / writeJSONError in server.go).\n          readJSON(res).then(\n            body => {\n              if (status < 200 || status >= 300) {\n                reject(\n                  new Error(\n                    `litestream ${path} failed: ${status} ${JSON.stringify(body)}`,\n                  ),\n                );\n              } else {\n                resolve(body as T);\n              }\n            },\n            (e: unknown) =>\n              reject(\n                new Error(`litestream ${path} returned invalid JSON`, {\n                  cause: e,\n                }),\n              ),\n          );\n        },\n      );\n      req.on('timeout', () =>\n        req.destroy(\n          new Error(`litestream ${path} timed out after ${timeoutMs}ms`),\n        ),\n      );\n      req.on('error', reject);\n      req.end(payload);\n    });\n  }\n}\n"],"mappings":";;;;;;;;;;;AAaA,SAAgB,qBAAqB,aAA6B;CAChE,OAAO,GAAG,YAAY;AACxB;AAEA,IAAM,6BAA6B;;;;;;;;;;;AAuCnC,IAAa,uBAAb,MAAkC;CAChC;CACA;CACA;CACA;;;;;;;;CASA,YACE,IACA,aACA,aAAa,qBAAqB,WAAW,GAC7C;EACA,KAAKA,MAAM,GAAG,YAAY,aAAa,uBAAuB;EAC9D,KAAKC,eAAe;EACpB,KAAKC,cAAc;EAGnB,KAAKC,SAAS,IAAI,MAAM;GAAC,WAAW;GAAM,YAAY;EAAC,CAAC;CAC1D;;;;;;;;;;;;CAaA,MAAM,KACJ,OAAoB,CAAC,GACrB,QACuB;EACvB,MAAM,EAAC,OAAO,OAAO,YAAY,+BAA8B;EAC/D,MAAM,SAAS,MAAM,KAAKC,MACxB,SACA;GAAC,MAAM,KAAKH;GAAc;EAAI,GAC9B,WACA,MACF;EACA,KAAKD,IAAI,QAAQ,mBAAmB,MAAM;EAC1C,OAAO;CACT;;CAGA,QAAc;EACZ,KAAKG,OAAO,QAAQ;CACtB;CAKA,MACE,MACA,MACA,WACA,QACY;EACZ,MAAM,UAAU,KAAK,UAAU,IAAI;EACnC,OAAO,IAAI,SAAY,SAAS,WAAW;GACzC,MAAM,MAAM,QACV;IACE,YAAY,KAAKD;IACjB,QAAQ;IACR;IACA,OAAO,KAAKC;IACZ,SAAS;IACT,SAAS;KACP,gBAAgB;KAChB,kBAAkB,OAAO,WAAW,OAAO;IAC7C;IACA;GACF,IACC,QAAyB;IACxB,MAAM,SAAS,IAAI,cAAc;IAGjC,KAAS,GAAG,EAAE,MACZ,SAAQ;KACN,IAAI,SAAS,OAAO,UAAU,KAC5B,uBACE,IAAI,MACF,cAAc,KAAK,WAAW,OAAO,GAAG,KAAK,UAAU,IAAI,GAC7D,CACF;UAEA,QAAQ,IAAS;IAErB,IACC,MACC,OACE,IAAI,MAAM,cAAc,KAAK,yBAAyB,EACpD,OAAO,EACT,CAAC,CACH,CACJ;GACF,CACF;GACA,IAAI,GAAG,iBACL,IAAI,wBACF,IAAI,MAAM,cAAc,KAAK,mBAAmB,UAAU,GAAG,CAC/D,CACF;GACA,IAAI,GAAG,SAAS,MAAM;GACtB,IAAI,IAAI,OAAO;EACjB,CAAC;CACH;AACF"}
+{"version":3,"file":"litestream-controller.js","names":["#lc","#replicaFile","#socketPath","#agent","#post"],"sources":["../../../../../../zero-cache/src/services/litestream/litestream-controller.ts"],"sourcesContent":["import type {IncomingMessage} from 'node:http';\nimport {Agent, request} from 'node:http';\nimport {json as readJSON} from 'node:stream/consumers';\nimport type {LogContext} from '@rocicorp/logger';\n\n/**\n * Path of the litestream control socket, derived from the replica file's\n * directory. The socket must live on a filesystem the (non-root) runtime user\n * can write — the replica directory is guaranteed writable since litestream\n * writes the replica there — so it must NOT default to a root-owned location\n * like /var/run. Kept in sync with config-v5.yml's `socket.path`, which is set\n * from `ZERO_LITESTREAM_SOCKET_PATH` (see getLitestream() in commands.ts).\n */\nexport function litestreamSocketPath(replicaFile: string): string {\n  return `${replicaFile}.sock`;\n}\n\nconst DEFAULT_REQUEST_TIMEOUT_MS = 30_000;\n\n/** Response body from litestream's `POST /sync` control endpoint. */\nexport type SyncResponse = {\n  /**\n   * `synced` when `wait` was true and the upload completed, `synced_local`\n   * when the WAL was sealed locally without waiting on the remote replica, or\n   * `no_change` when there was nothing to sync.\n   */\n  status: 'synced' | 'synced_local' | 'no_change';\n  path: string;\n  txid: number;\n  replicated_txid: number;\n};\n\nexport type SyncOptions = {\n  /**\n   * When true, block until the sealed WAL has been uploaded to the remote\n   * replica (litestream's SyncAndWait). When false (the default), litestream\n   * only seals the WAL into local LTX files and runs its checkpoint thresholds,\n   * returning without waiting on the backup store. Use false on the write path\n   * where the goal is to keep the WAL small, not to durably back it up.\n   */\n  wait?: boolean | undefined;\n\n  /** Overall request timeout in milliseconds. Defaults to 30s. */\n  timeoutMs?: number | undefined;\n\n  /**\n   * Server-side deadline in seconds, sent as litestream's `timeout`. When set,\n   * litestream ends the request itself (with an error response) once it\n   * elapses. Set it below {@link timeoutMs} so that litestream, rather than\n   * the client, ends a slow request. (litestream defaults to 30s when `wait`\n   * is set, and to no deadline otherwise.)\n   */\n  serverTimeoutSeconds?: number | undefined;\n};\n\n/**\n * IPC client for the litestream control server. litestream serves an HTTP API\n * over a Unix domain socket (enabled via the `socket:` block in config-v5.yml);\n * because zero-cache runs litestream as a subprocess in the same container, the\n * socket is reachable directly on the local filesystem.\n *\n * A single keep-alive connection is held open and reused across calls, so a\n * high-frequency caller (e.g. write-path backpressure) does not pay a\n * connect/handshake per request.\n */\nexport class LitestreamController {\n  readonly #lc: LogContext;\n  readonly #replicaFile: string;\n  readonly #socketPath: string;\n  readonly #agent: Agent;\n\n  /**\n   * @param replicaFile Path of the replica database litestream is managing —\n   *   the same value passed to litestream as `ZERO_REPLICA_FILE`. This is the\n   *   `path` litestream matches control requests against.\n   * @param socketPath Path of the control socket; defaults to the path derived\n   *   from `replicaFile`, matching config-v5.yml's `socket.path`.\n   */\n  constructor(\n    lc: LogContext,\n    replicaFile: string,\n    socketPath = litestreamSocketPath(replicaFile),\n  ) {\n    this.#lc = lc.withContext('component', 'litestream-controller');\n    this.#replicaFile = replicaFile;\n    this.#socketPath = socketPath;\n    // keepAlive with a single socket keeps one warm connection to the control\n    // socket that is reused across calls.\n    this.#agent = new Agent({keepAlive: true, maxSockets: 1});\n  }\n\n  /**\n   * Forces litestream to sync the WAL into LTX files immediately and run its\n   * checkpoint thresholds, instead of waiting for the next monitor-interval\n   * tick. The checkpoint itself is still litestream's usual (non-blocking)\n   * PASSIVE attempt, so it is skipped if the database is busy — call this once\n   * writes are quiesced.\n   *\n   * Best-effort: callers should tolerate rejection (e.g. ECONNREFUSED/ENOENT\n   * while litestream is restarting or before it has created the socket) rather\n   * than blocking application progress on it.\n   */\n  async sync(\n    opts: SyncOptions = {},\n    signal?: AbortSignal,\n  ): Promise<SyncResponse> {\n    const {\n      wait = false,\n      timeoutMs = DEFAULT_REQUEST_TIMEOUT_MS,\n      serverTimeoutSeconds,\n    } = opts;\n    const result = await this.#post<SyncResponse>(\n      '/sync',\n      {\n        path: this.#replicaFile,\n        wait,\n        ...(serverTimeoutSeconds ? {timeout: serverTimeoutSeconds} : {}),\n      },\n      timeoutMs,\n      signal,\n    );\n    this.#lc.debug?.('litestream sync', result);\n    return result;\n  }\n\n  /** Releases the pooled keep-alive connection. */\n  close(): void {\n    this.#agent.destroy();\n  }\n\n  // Note: Uses ye'old node:http for its unix socketPath support.\n  //   fetch() would have less boiler-plate, but it would require importing\n  //   the `unidici` package to provide a fetch Agent that supports sockets.\n  #post<T>(\n    path: string,\n    body: unknown,\n    timeoutMs: number,\n    signal?: AbortSignal,\n  ): Promise<T> {\n    const payload = JSON.stringify(body);\n    return new Promise<T>((resolve, reject) => {\n      const req = request(\n        {\n          socketPath: this.#socketPath,\n          method: 'POST',\n          path,\n          agent: this.#agent,\n          timeout: timeoutMs,\n          headers: {\n            'content-type': 'application/json',\n            'content-length': Buffer.byteLength(payload),\n          },\n          signal,\n        },\n        (res: IncomingMessage) => {\n          const status = res.statusCode ?? 0;\n          // litestream's success and error responses are both always JSON\n          // (writeJSON / writeJSONError in server.go).\n          readJSON(res).then(\n            body => {\n              if (status < 200 || status >= 300) {\n                reject(\n                  new Error(\n                    `litestream ${path} failed: ${status} ${JSON.stringify(body)}`,\n                  ),\n                );\n              } else {\n                resolve(body as T);\n              }\n            },\n            (e: unknown) =>\n              reject(\n                new Error(`litestream ${path} returned invalid JSON`, {\n                  cause: e,\n                }),\n              ),\n          );\n        },\n      );\n      req.on('timeout', () =>\n        req.destroy(\n          new Error(`litestream ${path} timed out after ${timeoutMs}ms`),\n        ),\n      );\n      req.on('error', reject);\n      req.end(payload);\n    });\n  }\n}\n"],"mappings":";;;;;;;;;;;AAaA,SAAgB,qBAAqB,aAA6B;CAChE,OAAO,GAAG,YAAY;AACxB;AAEA,IAAM,6BAA6B;;;;;;;;;;;AAgDnC,IAAa,uBAAb,MAAkC;CAChC;CACA;CACA;CACA;;;;;;;;CASA,YACE,IACA,aACA,aAAa,qBAAqB,WAAW,GAC7C;EACA,KAAKA,MAAM,GAAG,YAAY,aAAa,uBAAuB;EAC9D,KAAKC,eAAe;EACpB,KAAKC,cAAc;EAGnB,KAAKC,SAAS,IAAI,MAAM;GAAC,WAAW;GAAM,YAAY;EAAC,CAAC;CAC1D;;;;;;;;;;;;CAaA,MAAM,KACJ,OAAoB,CAAC,GACrB,QACuB;EACvB,MAAM,EACJ,OAAO,OACP,YAAY,4BACZ,yBACE;EACJ,MAAM,SAAS,MAAM,KAAKC,MACxB,SACA;GACE,MAAM,KAAKH;GACX;GACA,GAAI,uBAAuB,EAAC,SAAS,qBAAoB,IAAI,CAAC;EAChE,GACA,WACA,MACF;EACA,KAAKD,IAAI,QAAQ,mBAAmB,MAAM;EAC1C,OAAO;CACT;;CAGA,QAAc;EACZ,KAAKG,OAAO,QAAQ;CACtB;CAKA,MACE,MACA,MACA,WACA,QACY;EACZ,MAAM,UAAU,KAAK,UAAU,IAAI;EACnC,OAAO,IAAI,SAAY,SAAS,WAAW;GACzC,MAAM,MAAM,QACV;IACE,YAAY,KAAKD;IACjB,QAAQ;IACR;IACA,OAAO,KAAKC;IACZ,SAAS;IACT,SAAS;KACP,gBAAgB;KAChB,kBAAkB,OAAO,WAAW,OAAO;IAC7C;IACA;GACF,IACC,QAAyB;IACxB,MAAM,SAAS,IAAI,cAAc;IAGjC,KAAS,GAAG,EAAE,MACZ,SAAQ;KACN,IAAI,SAAS,OAAO,UAAU,KAC5B,uBACE,IAAI,MACF,cAAc,KAAK,WAAW,OAAO,GAAG,KAAK,UAAU,IAAI,GAC7D,CACF;UAEA,QAAQ,IAAS;IAErB,IACC,MACC,OACE,IAAI,MAAM,cAAc,KAAK,yBAAyB,EACpD,OAAO,EACT,CAAC,CACH,CACJ;GACF,CACF;GACA,IAAI,GAAG,iBACL,IAAI,wBACF,IAAI,MAAM,cAAc,KAAK,mBAAmB,UAAU,GAAG,CAC/D,CACF;GACA,IAAI,GAAG,SAAS,MAAM;GACtB,IAAI,IAAI,OAAO;EACjB,CAAC;CACH;AACF"}

package/out/zero-cache/src/services/litestream/restore-progress.d.ts

@@ -0,0 +1,24 @@
+import type { LogContext } from '@rocicorp/logger';
+import type { ViewSyncerStatusEvent } from '../../../../zero-events/src/status.ts';
+type PublishFn = (lc: LogContext, event: ViewSyncerStatusEvent) => void;
+/**
+ * Publishes the progress of a view-syncer's litestream restore.
+ *
+ * `litestream restore` writes the database page by page into
+ * `<replica>.tmp` and renames it to the replica when it is done, so the
+ * size of the temporary file is the number of bytes restored so far.
+ */
+export declare class RestoreProgressReporter {
+    #private;
+    constructor(lc: LogContext, replicaFile: string, publishFn?: PublishFn);
+    /**
+     * Publishes a `Restoring` event now and then every `intervalMs` while the
+     * number of restored bytes changes, until {@link stop} or {@link done}.
+     */
+    start(totalBytes: number | undefined, intervalMs?: number): void;
+    stop(): void;
+    /** Publishes a `Restored` event with the size of the restored replica. */
+    done(): void;
+}
+export {};
+//# sourceMappingURL=restore-progress.d.ts.map

package/out/zero-cache/src/services/litestream/restore-progress.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"restore-progress.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/litestream/restore-progress.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AACjD,OAAO,KAAK,EAGV,qBAAqB,EACtB,MAAM,uCAAuC,CAAC;AAK/C,KAAK,SAAS,GAAG,CAAC,EAAE,EAAE,UAAU,EAAE,KAAK,EAAE,qBAAqB,KAAK,IAAI,CAAC;AAExE;;;;;;GAMG;AACH,qBAAa,uBAAuB;;IAUlC,YACE,EAAE,EAAE,UAAU,EACd,WAAW,EAAE,MAAM,EACnB,SAAS,GAAE,SAAwB,EAKpC;IAED;;;OAGG;IACH,KAAK,CAAC,UAAU,EAAE,MAAM,GAAG,SAAS,EAAE,UAAU,SAAsB,QAcrE;IAED,IAAI,SAGH;IAED,0EAA0E;IAC1E,IAAI,SAGH;CA2CF"}

package/out/zero-cache/src/services/litestream/restore-progress.js

@@ -0,0 +1,91 @@
+import { publishEvent } from "../../observability/events.js";
+import { statSync } from "node:fs";
+//#region ../zero-cache/src/services/litestream/restore-progress.ts
+var PUBLISH_INTERVAL_MS = 5e3;
+/**
+* Publishes the progress of a view-syncer's litestream restore.
+*
+* `litestream restore` writes the database page by page into
+* `<replica>.tmp` and renames it to the replica when it is done, so the
+* size of the temporary file is the number of bytes restored so far.
+*/
+var RestoreProgressReporter = class {
+	#lc;
+	#replicaFile;
+	#publishFn;
+	#start = performance.now();
+	#totalBytes;
+	#lastBytes;
+	#sawTempFile = false;
+	#timer;
+	constructor(lc, replicaFile, publishFn = publishEvent) {
+		this.#lc = lc;
+		this.#replicaFile = replicaFile;
+		this.#publishFn = publishFn;
+	}
+	/**
+	* Publishes a `Restoring` event now and then every `intervalMs` while the
+	* number of restored bytes changes, until {@link stop} or {@link done}.
+	*/
+	start(totalBytes, intervalMs = PUBLISH_INTERVAL_MS) {
+		this.stop();
+		this.#start = performance.now();
+		if (totalBytes !== this.#totalBytes) {
+			this.#lastBytes = void 0;
+			this.#totalBytes = totalBytes;
+		}
+		this.#sawTempFile = false;
+		this.#update(0);
+		this.#timer = setInterval(() => this.#check(), intervalMs);
+	}
+	stop() {
+		clearInterval(this.#timer);
+		this.#timer = void 0;
+	}
+	/** Publishes a `Restored` event with the size of the restored replica. */
+	done() {
+		this.stop();
+		this.#publish("Restored", fileSize(this.#replicaFile) ?? 0);
+	}
+	#check() {
+		const tempBytes = fileSize(`${this.#replicaFile}.tmp`);
+		if (tempBytes !== void 0) {
+			this.#sawTempFile = true;
+			this.#update(tempBytes);
+		} else if (this.#sawTempFile) this.#update(fileSize(this.#replicaFile) ?? 0);
+	}
+	#update(bytes) {
+		if (bytes !== this.#lastBytes) {
+			this.#lastBytes = bytes;
+			this.#publish("Restoring", bytes);
+		}
+	}
+	#publish(stage, bytes) {
+		const restoreStatus = {
+			bytes,
+			totalBytes: this.#totalBytes,
+			elapsedMs: Math.round(performance.now() - this.#start)
+		};
+		const event = {
+			type: "zero/events/status/view-syncer/v1",
+			component: "view-syncer",
+			status: "OK",
+			stage,
+			description: stage === "Restoring" ? "Restoring replica from backup" : "Restored replica from backup",
+			time: (/* @__PURE__ */ new Date()).toISOString(),
+			state: { restoreStatus }
+		};
+		this.#publishFn(this.#lc, event);
+	}
+};
+function fileSize(file) {
+	try {
+		return statSync(file).size;
+	} catch {
+		return;
+	}
+}
+//#endregion
+export { RestoreProgressReporter };
+
+//# sourceMappingURL=restore-progress.js.map

package/out/zero-cache/src/services/litestream/restore-progress.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"restore-progress.js","names":["#lc","#replicaFile","#publishFn","#start","#totalBytes","#lastBytes","#sawTempFile","#update","#timer","#check","#publish"],"sources":["../../../../../../zero-cache/src/services/litestream/restore-progress.ts"],"sourcesContent":["import {statSync} from 'node:fs';\nimport type {LogContext} from '@rocicorp/logger';\nimport type {\n  RestoreStatus,\n  ViewSyncerStage,\n  ViewSyncerStatusEvent,\n} from '../../../../zero-events/src/status.ts';\nimport {publishEvent} from '../../observability/events.ts';\n\nconst PUBLISH_INTERVAL_MS = 5000;\n\ntype PublishFn = (lc: LogContext, event: ViewSyncerStatusEvent) => void;\n\n/**\n * Publishes the progress of a view-syncer's litestream restore.\n *\n * `litestream restore` writes the database page by page into\n * `<replica>.tmp` and renames it to the replica when it is done, so the\n * size of the temporary file is the number of bytes restored so far.\n */\nexport class RestoreProgressReporter {\n  readonly #lc: LogContext;\n  readonly #replicaFile: string;\n  readonly #publishFn: PublishFn;\n  #start = performance.now();\n  #totalBytes: number | undefined;\n  #lastBytes: number | undefined;\n  #sawTempFile = false;\n  #timer: NodeJS.Timeout | undefined;\n\n  constructor(\n    lc: LogContext,\n    replicaFile: string,\n    publishFn: PublishFn = publishEvent,\n  ) {\n    this.#lc = lc;\n    this.#replicaFile = replicaFile;\n    this.#publishFn = publishFn;\n  }\n\n  /**\n   * Publishes a `Restoring` event now and then every `intervalMs` while the\n   * number of restored bytes changes, until {@link stop} or {@link done}.\n   */\n  start(totalBytes: number | undefined, intervalMs = PUBLISH_INTERVAL_MS) {\n    this.stop();\n    // Measure the restore itself, not the wait for a restorable backup.\n    this.#start = performance.now();\n    if (totalBytes !== this.#totalBytes) {\n      // Publish the new total even if the restored bytes are unchanged.\n      this.#lastBytes = undefined;\n      this.#totalBytes = totalBytes;\n    }\n    // Start from 0 rather than checking now: a `.tmp` left behind by an\n    // interrupted restore is only deleted when the restore starts.\n    this.#sawTempFile = false;\n    this.#update(0);\n    this.#timer = setInterval(() => this.#check(), intervalMs);\n  }\n\n  stop() {\n    clearInterval(this.#timer);\n    this.#timer = undefined;\n  }\n\n  /** Publishes a `Restored` event with the size of the restored replica. */\n  done() {\n    this.stop();\n    this.#publish('Restored', fileSize(this.#replicaFile) ?? 0);\n  }\n\n  #check() {\n    const tempBytes = fileSize(`${this.#replicaFile}.tmp`);\n    if (tempBytes !== undefined) {\n      this.#sawTempFile = true;\n      this.#update(tempBytes);\n    } else if (this.#sawTempFile) {\n      // litestream renames `.tmp` to the replica before its post-restore\n      // integrity check, so once `.tmp` is gone the replica holds the bytes.\n      // Before `.tmp` appears, an existing replica is not restore progress\n      // (it may be invalid and about to be deleted).\n      this.#update(fileSize(this.#replicaFile) ?? 0);\n    }\n  }\n\n  #update(bytes: number) {\n    if (bytes !== this.#lastBytes) {\n      this.#lastBytes = bytes;\n      this.#publish('Restoring', bytes);\n    }\n  }\n\n  #publish(stage: ViewSyncerStage, bytes: number) {\n    const restoreStatus: RestoreStatus = {\n      bytes,\n      totalBytes: this.#totalBytes,\n      elapsedMs: Math.round(performance.now() - this.#start),\n    };\n    const event: ViewSyncerStatusEvent = {\n      type: 'zero/events/status/view-syncer/v1',\n      component: 'view-syncer',\n      status: 'OK',\n      stage,\n      description:\n        stage === 'Restoring'\n          ? 'Restoring replica from backup'\n          : 'Restored replica from backup',\n      time: new Date().toISOString(),\n      state: {restoreStatus},\n    };\n    this.#publishFn(this.#lc, event);\n  }\n}\n\nfunction fileSize(file: string): number | undefined {\n  try {\n    return statSync(file).size;\n  } catch {\n    return undefined;\n  }\n}\n"],"mappings":";;;AASA,IAAM,sBAAsB;;;;;;;;AAW5B,IAAa,0BAAb,MAAqC;CACnC;CACA;CACA;CACA,SAAS,YAAY,IAAI;CACzB;CACA;CACA,eAAe;CACf;CAEA,YACE,IACA,aACA,YAAuB,cACvB;EACA,KAAKA,MAAM;EACX,KAAKC,eAAe;EACpB,KAAKC,aAAa;CACpB;;;;;CAMA,MAAM,YAAgC,aAAa,qBAAqB;EACtE,KAAK,KAAK;EAEV,KAAKC,SAAS,YAAY,IAAI;EAC9B,IAAI,eAAe,KAAKC,aAAa;GAEnC,KAAKC,aAAa,KAAA;GAClB,KAAKD,cAAc;EACrB;EAGA,KAAKE,eAAe;EACpB,KAAKC,QAAQ,CAAC;EACd,KAAKC,SAAS,kBAAkB,KAAKC,OAAO,GAAG,UAAU;CAC3D;CAEA,OAAO;EACL,cAAc,KAAKD,MAAM;EACzB,KAAKA,SAAS,KAAA;CAChB;;CAGA,OAAO;EACL,KAAK,KAAK;EACV,KAAKE,SAAS,YAAY,SAAS,KAAKT,YAAY,KAAK,CAAC;CAC5D;CAEA,SAAS;EACP,MAAM,YAAY,SAAS,GAAG,KAAKA,aAAa,KAAK;EACrD,IAAI,cAAc,KAAA,GAAW;GAC3B,KAAKK,eAAe;GACpB,KAAKC,QAAQ,SAAS;EACxB,OAAO,IAAI,KAAKD,cAKd,KAAKC,QAAQ,SAAS,KAAKN,YAAY,KAAK,CAAC;CAEjD;CAEA,QAAQ,OAAe;EACrB,IAAI,UAAU,KAAKI,YAAY;GAC7B,KAAKA,aAAa;GAClB,KAAKK,SAAS,aAAa,KAAK;EAClC;CACF;CAEA,SAAS,OAAwB,OAAe;EAC9C,MAAM,gBAA+B;GACnC;GACA,YAAY,KAAKN;GACjB,WAAW,KAAK,MAAM,YAAY,IAAI,IAAI,KAAKD,MAAM;EACvD;EACA,MAAM,QAA+B;GACnC,MAAM;GACN,WAAW;GACX,QAAQ;GACR;GACA,aACE,UAAU,cACN,kCACA;GACN,uBAAM,IAAI,KAAK,GAAE,YAAY;GAC7B,OAAO,EAAC,cAAa;EACvB;EACA,KAAKD,WAAW,KAAKF,KAAK,KAAK;CACjC;AACF;AAEA,SAAS,SAAS,MAAkC;CAClD,IAAI;EACF,OAAO,SAAS,IAAI,EAAE;CACxB,QAAQ;EACN;CACF;AACF"}

package/out/zero-cache/src/services/litestream/wal-keeper.d.ts

@@ -0,0 +1,33 @@
+import type { LogContext } from '@rocicorp/logger';
+import type { Database } from '../../../../zqlite/src/db.ts';
+/**
+ * Keeps the WAL of a replica intact until `litestream replicate` has attached
+ * to it, when the backup continues from the local litestream state (see
+ * `ResumableBackup`), e.g. after a forking `litestream restore`.
+ *
+ * Litestream needs its latest local L0 file to correspond to the state of
+ * the WAL to continue replication from it rather than creating a new snapshot.
+ * Normally, the last connection to close a database checkpoints and deletes
+ * its WAL, which loses the salt and makes litestream fall back to the
+ * snapshot. The keeper prevents that because every connection holds a SHARED
+ * lock on the database file while it is open in WAL mode, so while the keeper
+ * is open, no other connection's close can be the last one.
+ *
+ * While the keeper is held, nothing can change the journal mode or take an
+ * exclusive lock on the replica, as those fail with SQLITE_BUSY. The backup
+ * replicator keeps a replica that is already in 'wal' mode as is, and skips
+ * a due VACUUM (see `setupReplica`). Incremental schema migrations run in WAL
+ * mode (see `runSchemaMigrations`).
+ */
+export declare class WalKeeper {
+    #private;
+    /**
+     * Takes ownership of `db`, an open connection to the replica, which it
+     * closes when released.
+     */
+    constructor(lc: LogContext, db: Database);
+    get held(): boolean;
+    /** Closes the keeper connection. Idempotent. */
+    release(reason: string): void;
+}
+//# sourceMappingURL=wal-keeper.d.ts.map

package/out/zero-cache/src/services/litestream/wal-keeper.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"wal-keeper.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/litestream/wal-keeper.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AACjD,OAAO,KAAK,EAAC,QAAQ,EAAC,MAAM,8BAA8B,CAAC;AAE3D;;;;;;;;;;;;;;;;;;GAkBG;AACH,qBAAa,SAAS;;IAIpB;;;OAGG;IACH,YAAY,EAAE,EAAE,UAAU,EAAE,EAAE,EAAE,QAAQ,EAMvC;IAED,IAAI,IAAI,IAAI,OAAO,CAElB;IAED,gDAAgD;IAChD,OAAO,CAAC,MAAM,EAAE,MAAM,QAMrB;CACF"}

package/out/zero-cache/src/services/litestream/wal-keeper.js

@@ -0,0 +1,49 @@
+//#region ../zero-cache/src/services/litestream/wal-keeper.ts
+/**
+* Keeps the WAL of a replica intact until `litestream replicate` has attached
+* to it, when the backup continues from the local litestream state (see
+* `ResumableBackup`), e.g. after a forking `litestream restore`.
+*
+* Litestream needs its latest local L0 file to correspond to the state of
+* the WAL to continue replication from it rather than creating a new snapshot.
+* Normally, the last connection to close a database checkpoints and deletes
+* its WAL, which loses the salt and makes litestream fall back to the
+* snapshot. The keeper prevents that because every connection holds a SHARED
+* lock on the database file while it is open in WAL mode, so while the keeper
+* is open, no other connection's close can be the last one.
+*
+* While the keeper is held, nothing can change the journal mode or take an
+* exclusive lock on the replica, as those fail with SQLITE_BUSY. The backup
+* replicator keeps a replica that is already in 'wal' mode as is, and skips
+* a due VACUUM (see `setupReplica`). Incremental schema migrations run in WAL
+* mode (see `runSchemaMigrations`).
+*/
+var WalKeeper = class {
+	#lc;
+	#db;
+	/**
+	* Takes ownership of `db`, an open connection to the replica, which it
+	* closes when released.
+	*/
+	constructor(lc, db) {
+		this.#lc = lc.withContext("component", "wal-keeper");
+		db.prepare("SELECT count(*) FROM sqlite_master").get();
+		this.#db = db;
+		this.#lc.info?.(`holding the restored WAL of ${db.name}`);
+	}
+	get held() {
+		return this.#db !== null;
+	}
+	/** Closes the keeper connection. Idempotent. */
+	release(reason) {
+		if (this.#db) {
+			this.#lc.info?.(`releasing the restored WAL: ${reason}`);
+			this.#db.close();
+			this.#db = null;
+		}
+	}
+};
+//#endregion
+export { WalKeeper };
+
+//# sourceMappingURL=wal-keeper.js.map

package/out/zero-cache/src/services/litestream/wal-keeper.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"wal-keeper.js","names":["#lc","#db"],"sources":["../../../../../../zero-cache/src/services/litestream/wal-keeper.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport type {Database} from '../../../../zqlite/src/db.ts';\n\n/**\n * Keeps the WAL of a replica intact until `litestream replicate` has attached\n * to it, when the backup continues from the local litestream state (see\n * `ResumableBackup`), e.g. after a forking `litestream restore`.\n *\n * Litestream needs its latest local L0 file to correspond to the state of\n * the WAL to continue replication from it rather than creating a new snapshot.\n * Normally, the last connection to close a database checkpoints and deletes\n * its WAL, which loses the salt and makes litestream fall back to the\n * snapshot. The keeper prevents that because every connection holds a SHARED\n * lock on the database file while it is open in WAL mode, so while the keeper\n * is open, no other connection's close can be the last one.\n *\n * While the keeper is held, nothing can change the journal mode or take an\n * exclusive lock on the replica, as those fail with SQLITE_BUSY. The backup\n * replicator keeps a replica that is already in 'wal' mode as is, and skips\n * a due VACUUM (see `setupReplica`). Incremental schema migrations run in WAL\n * mode (see `runSchemaMigrations`).\n */\nexport class WalKeeper {\n  readonly #lc: LogContext;\n  #db: Database | null;\n\n  /**\n   * Takes ownership of `db`, an open connection to the replica, which it\n   * closes when released.\n   */\n  constructor(lc: LogContext, db: Database) {\n    this.#lc = lc.withContext('component', 'wal-keeper');\n    // The SHARED lock is taken by the first read (if no read has taken it).\n    db.prepare('SELECT count(*) FROM sqlite_master').get();\n    this.#db = db;\n    this.#lc.info?.(`holding the restored WAL of ${db.name}`);\n  }\n\n  get held(): boolean {\n    return this.#db !== null;\n  }\n\n  /** Closes the keeper connection. Idempotent. */\n  release(reason: string) {\n    if (this.#db) {\n      this.#lc.info?.(`releasing the restored WAL: ${reason}`);\n      this.#db.close();\n      this.#db = null;\n    }\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;AAsBA,IAAa,YAAb,MAAuB;CACrB;CACA;;;;;CAMA,YAAY,IAAgB,IAAc;EACxC,KAAKA,MAAM,GAAG,YAAY,aAAa,YAAY;EAEnD,GAAG,QAAQ,oCAAoC,EAAE,IAAI;EACrD,KAAKC,MAAM;EACX,KAAKD,IAAI,OAAO,+BAA+B,GAAG,MAAM;CAC1D;CAEA,IAAI,OAAgB;EAClB,OAAO,KAAKC,QAAQ;CACtB;;CAGA,QAAQ,QAAgB;EACtB,IAAI,KAAKA,KAAK;GACZ,KAAKD,IAAI,OAAO,+BAA+B,QAAQ;GACvD,KAAKC,IAAI,MAAM;GACf,KAAKA,MAAM;EACb;CACF;AACF"}

package/out/zero-cache/src/services/mutagen/mutagen.d.ts

@@ -14,6 +14,11 @@ export type MutationError = [
     desc: string
 ];
 export interface Mutagen extends RefCountedService {
+    /**
+     * Callers must hold a {@link ref} for the duration of a push (i.e. across
+     * all of its mutations). The service stops, closing its replica handle,
+     * when its ref count drops to zero.
+     */
     processMutation(mutation: Mutation, authData: JWTPayload | undefined, customMutatorsEnabled: boolean): Promise<MutationError | undefined>;
 }
 export declare class MutagenService implements Mutagen, Service {

package/out/zero-cache/src/services/mutagen/mutagen.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"mutagen.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/mutagen/mutagen.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAEjD,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,MAAM,CAAC;AACrC,OAAO,QAAQ,MAAM,UAAU,CAAC;AAGhC,OAAO,EAAC,SAAS,EAAC,MAAM,6CAA6C,CAAC;AAOtE,OAAO,EACL,KAAK,YAAY,EAEjB,KAAK,QAAQ,EACb,KAAK,QAAQ,EAEb,KAAK,QAAQ,EACd,MAAM,2CAA2C,CAAC;AAKnD,OAAO,KAAK,EAAC,eAAe,EAAC,MAAM,4CAA4C,CAAC;AAEhF,OAAO,EAEL,KAAK,eAAe,EACrB,MAAM,gCAAgC,CAAC;AACxC,OAAO,EAAC,KAAK,UAAU,EAAC,MAAM,6BAA6B,CAAC;AAK5D,OAAO,KAAK,EAAC,UAAU,EAAE,mBAAmB,EAAC,MAAM,mBAAmB,CAAC;AACvE,OAAO,EAAiB,KAAK,OAAO,EAAC,MAAM,uBAAuB,CAAC;AAEnE,OAAO,KAAK,EAAC,iBAAiB,EAAE,OAAO,EAAC,MAAM,eAAe,CAAC;AAK9D,MAAM,MAAM,aAAa,GAAG;IAC1B,IAAI,EAAE,SAAS,CAAC,cAAc,GAAG,SAAS,CAAC,mBAAmB;IAC9D,IAAI,EAAE,MAAM;CACb,CAAC;AAEF,MAAM,WAAW,OAAQ,SAAQ,iBAAiB;IAChD,eAAe,CACb,QAAQ,EAAE,QAAQ,EAClB,QAAQ,EAAE,UAAU,GAAG,SAAS,EAChC,qBAAqB,EAAE,OAAO,GAC7B,OAAO,CAAC,aAAa,GAAG,SAAS,CAAC,CAAC;CACvC;AAED,qBAAa,cAAe,YAAW,OAAO,EAAE,OAAO;;IACrD,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;IAiBpB,YACE,EAAE,EAAE,UAAU,EACd,KAAK,EAAE,OAAO,EACd,aAAa,EAAE,MAAM,EACrB,QAAQ,EAAE,UAAU,EACpB,MAAM,EAAE,UAAU,EAClB,iBAAiB,EAAE,eAAe,EAwBnC;IAED,GAAG,SAGF;IAED,KAAK,SAMJ;IAED,OAAO,IAAI,OAAO,CAEjB;IAED,eAAe,CACb,QAAQ,EAAE,QAAQ,EAClB,QAAQ,EAAE,UAAU,GAAG,SAAS,EAChC,qBAAqB,UAAQ,GAC5B,OAAO,CAAC,aAAa,GAAG,SAAS,CAAC,CAqBpC;IAED,GAAG,IAAI,OAAO,CAAC,IAAI,CAAC,CAEnB;IAED,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC,CAQpB;CACF;AAID,wBAAsB,eAAe,CACnC,EAAE,EAAE,UAAU,EACd,QAAQ,EAAE,UAAU,GAAG,SAAS,EAChC,EAAE,EAAE,UAAU,EACd,KAAK,EAAE,OAAO,EACd,aAAa,EAAE,MAAM,EACrB,QAAQ,EAAE,QAAQ,EAClB,eAAe,EAAE,eAAe,EAChC,SAAS,CAAC,EAAE,MAAM,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,EAAE,cAAc;AACtD,qBAAqB,UAAQ,GAC5B,OAAO,CAAC,aAAa,GAAG,SAAS,CAAC,CAwIpC;AAED,wBAAsB,qBAAqB,CACzC,EAAE,EAAE,UAAU,EACd,EAAE,EAAE,mBAAmB,EACvB,QAAQ,EAAE,UAAU,GAAG,SAAS,EAChC,KAAK,EAAE,OAAO,EACd,aAAa,EAAE,MAAM,EACrB,QAAQ,EAAE,YAAY,EACtB,SAAS,EAAE,OAAO,EAClB,UAAU,EAAE,eAAe,iBAgE5B;AAED,wBAAgB,YAAY,CAC1B,EAAE,EAAE,QAAQ,CAAC,cAAc,EAC3B,MAAM,EAAE,QAAQ,GACf,QAAQ,CAAC,YAAY,CAAC,QAAQ,CAAC,GAAG,EAAE,CAAC,CAKvC;AAED,wBAAgB,YAAY,CAC1B,EAAE,EAAE,QAAQ,CAAC,cAAc,EAC3B,GAAG,EAAE,QAAQ,GACZ,QAAQ,CAAC,YAAY,CAAC,QAAQ,CAAC,GAAG,EAAE,CAAC,CAOvC"}
+{"version":3,"file":"mutagen.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/mutagen/mutagen.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAEjD,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,MAAM,CAAC;AACrC,OAAO,QAAQ,MAAM,UAAU,CAAC;AAGhC,OAAO,EAAC,SAAS,EAAC,MAAM,6CAA6C,CAAC;AAOtE,OAAO,EACL,KAAK,YAAY,EAEjB,KAAK,QAAQ,EACb,KAAK,QAAQ,EAEb,KAAK,QAAQ,EACd,MAAM,2CAA2C,CAAC;AAKnD,OAAO,KAAK,EAAC,eAAe,EAAC,MAAM,4CAA4C,CAAC;AAEhF,OAAO,EAEL,KAAK,eAAe,EACrB,MAAM,gCAAgC,CAAC;AACxC,OAAO,EAAC,KAAK,UAAU,EAAC,MAAM,6BAA6B,CAAC;AAK5D,OAAO,KAAK,EAAC,UAAU,EAAE,mBAAmB,EAAC,MAAM,mBAAmB,CAAC;AACvE,OAAO,EAAiB,KAAK,OAAO,EAAC,MAAM,uBAAuB,CAAC;AAEnE,OAAO,KAAK,EAAC,iBAAiB,EAAE,OAAO,EAAC,MAAM,eAAe,CAAC;AAK9D,MAAM,MAAM,aAAa,GAAG;IAC1B,IAAI,EAAE,SAAS,CAAC,cAAc,GAAG,SAAS,CAAC,mBAAmB;IAC9D,IAAI,EAAE,MAAM;CACb,CAAC;AAEF,MAAM,WAAW,OAAQ,SAAQ,iBAAiB;IAChD;;;;OAIG;IACH,eAAe,CACb,QAAQ,EAAE,QAAQ,EAClB,QAAQ,EAAE,UAAU,GAAG,SAAS,EAChC,qBAAqB,EAAE,OAAO,GAC7B,OAAO,CAAC,aAAa,GAAG,SAAS,CAAC,CAAC;CACvC;AAED,qBAAa,cAAe,YAAW,OAAO,EAAE,OAAO;;IACrD,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;IAiBpB,YACE,EAAE,EAAE,UAAU,EACd,KAAK,EAAE,OAAO,EACd,aAAa,EAAE,MAAM,EACrB,QAAQ,EAAE,UAAU,EACpB,MAAM,EAAE,UAAU,EAClB,iBAAiB,EAAE,eAAe,EAwBnC;IAED,GAAG,SAGF;IAED,KAAK,SAMJ;IAED,OAAO,IAAI,OAAO,CAEjB;IAED,eAAe,CACb,QAAQ,EAAE,QAAQ,EAClB,QAAQ,EAAE,UAAU,GAAG,SAAS,EAChC,qBAAqB,UAAQ,GAC5B,OAAO,CAAC,aAAa,GAAG,SAAS,CAAC,CAqBpC;IAED,GAAG,IAAI,OAAO,CAAC,IAAI,CAAC,CAEnB;IAED,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC,CAiBpB;CACF;AAID,wBAAsB,eAAe,CACnC,EAAE,EAAE,UAAU,EACd,QAAQ,EAAE,UAAU,GAAG,SAAS,EAChC,EAAE,EAAE,UAAU,EACd,KAAK,EAAE,OAAO,EACd,aAAa,EAAE,MAAM,EACrB,QAAQ,EAAE,QAAQ,EAClB,eAAe,EAAE,eAAe,EAChC,SAAS,CAAC,EAAE,MAAM,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,EAAE,cAAc;AACtD,qBAAqB,UAAQ,GAC5B,OAAO,CAAC,aAAa,GAAG,SAAS,CAAC,CAwIpC;AAED,wBAAsB,qBAAqB,CACzC,EAAE,EAAE,UAAU,EACd,EAAE,EAAE,mBAAmB,EACvB,QAAQ,EAAE,UAAU,GAAG,SAAS,EAChC,KAAK,EAAE,OAAO,EACd,aAAa,EAAE,MAAM,EACrB,QAAQ,EAAE,YAAY,EACtB,SAAS,EAAE,OAAO,EAClB,UAAU,EAAE,eAAe,iBAgE5B;AAED,wBAAgB,YAAY,CAC1B,EAAE,EAAE,QAAQ,CAAC,cAAc,EAC3B,MAAM,EAAE,QAAQ,GACf,QAAQ,CAAC,YAAY,CAAC,QAAQ,CAAC,GAAG,EAAE,CAAC,CAKvC;AAED,wBAAgB,YAAY,CAC1B,EAAE,EAAE,QAAQ,CAAC,cAAc,EAC3B,GAAG,EAAE,QAAQ,GACZ,QAAQ,CAAC,YAAY,CAAC,QAAQ,CAAC,GAAG,EAAE,CAAC,CAOvC"}

package/out/zero-cache/src/services/mutagen/mutagen.js

@@ -63,8 +63,14 @@ var MutagenService = class {
 	}
 	stop() {
 		if (this.#isStopped) return this.#stopped.promise;
-		this.#writeAuthorizer.destroy();
 		this.#isStopped = true;
+		try {
+			this.#writeAuthorizer.destroy();
+		} catch (e) {
+			this.#lc.error?.("error destroying write authorizer storage", e);
+		} finally {
+			this.#replica.close({ optimize: false });
+		}
 		this.#stopped.resolve();
 		return this.#stopped.promise;
 	}