@rocicorp/zero 0.25.0-canary.7 → 0.25.0-canary.9

package/out/zero-cache/src/services/change-source/protocol/current.js

@@ -1,7 +1,35 @@
-export * from "./current/control.js";
-export * from "./current/data.js";
-export * from "./current/downstream.js";
-export * from "./current/path.js";
-export * from "./current/status.js";
-export * from "./current/upstream.js";
-//# sourceMappingURL=current.js.map
+import { resetRequiredSchema } from "./current/control.js";
+import { addColumnSchema, beginSchema, commitSchema, createIndexSchema, createTableSchema, dataChangeSchema, deleteSchema, dropColumnSchema, dropIndexSchema, dropTableSchema, insertSchema, relationSchema, renameTableSchema, rollbackSchema, rowSchema, truncateSchema, updateColumnSchema, updateSchema } from "./current/data.js";
+import { changeStreamControlSchema, changeStreamDataSchema, changeStreamMessageSchema } from "./current/downstream.js";
+import { CHANGE_SOURCE_PATH } from "./current/path.js";
+import { statusMessageSchema, statusSchema } from "./current/status.js";
+import { changeSourceUpstreamSchema } from "./current/upstream.js";
+export {
+  CHANGE_SOURCE_PATH,
+  addColumnSchema,
+  beginSchema,
+  changeSourceUpstreamSchema,
+  changeStreamControlSchema,
+  changeStreamDataSchema,
+  changeStreamMessageSchema,
+  commitSchema,
+  createIndexSchema,
+  createTableSchema,
+  dataChangeSchema,
+  deleteSchema,
+  dropColumnSchema,
+  dropIndexSchema,
+  dropTableSchema,
+  insertSchema,
+  relationSchema,
+  renameTableSchema,
+  resetRequiredSchema,
+  rollbackSchema,
+  rowSchema,
+  statusMessageSchema,
+  statusSchema,
+  truncateSchema,
+  updateColumnSchema,
+  updateSchema
+};
+//# sourceMappingURL=current.js.map

package/out/zero-cache/src/services/change-source/protocol/current.js.map

@@ -1 +1 @@
-{"version":3,"file":"current.js","sourceRoot":"","sources":["../../../../../../../zero-cache/src/services/change-source/protocol/current.ts"],"names":[],"mappings":"AAAA,cAAc,sBAAsB,CAAC;AACrC,cAAc,mBAAmB,CAAC;AAClC,cAAc,yBAAyB,CAAC;AACxC,cAAc,mBAAmB,CAAC;AAClC,cAAc,qBAAqB,CAAC;AACpC,cAAc,uBAAuB,CAAC"}
+{"version":3,"file":"current.js","sources":[],"sourcesContent":[],"names":[],"mappings":";;;;;;"}

package/out/zero-cache/src/services/change-source/replica-schema.js

@@ -1,59 +1,76 @@
-import { SqliteError } from '@rocicorp/zero-sqlite3';
+import { SqliteError } from "@rocicorp/zero-sqlite3";
 import { must } from "../../../../shared/src/must.js";
-import { runSchemaMigrations, } from "../../db/migration-lite.js";
+import { runSchemaMigrations } from "../../db/migration-lite.js";
 import { listTables } from "../../db/lite-tables.js";
 import { AutoResetSignal } from "../change-streamer/schema/tables.js";
-import { CREATE_RUNTIME_EVENTS_TABLE, recordEvent, } from "../replicator/schema/replication-state.js";
-import { ColumnMetadataStore, CREATE_COLUMN_METADATA_TABLE, } from "./column-metadata.js";
-export async function initReplica(log, debugName, dbPath, initialSync) {
-    const setupMigration = {
-        migrateSchema: (log, tx) => initialSync(log, tx),
-        minSafeVersion: 1,
-    };
-    try {
-        await runSchemaMigrations(log, debugName, dbPath, setupMigration, schemaVersionMigrationMap);
-    }
-    catch (e) {
-        if (e instanceof SqliteError && e.code === 'SQLITE_CORRUPT') {
-            throw new AutoResetSignal(e.message);
-        }
-        throw e;
+import { recordEvent, CREATE_RUNTIME_EVENTS_TABLE } from "../replicator/schema/replication-state.js";
+import { ColumnMetadataStore, CREATE_COLUMN_METADATA_TABLE } from "./column-metadata.js";
+async function initReplica(log, debugName, dbPath, initialSync) {
+  const setupMigration = {
+    migrateSchema: (log2, tx) => initialSync(log2, tx),
+    minSafeVersion: 1
+  };
+  try {
+    await runSchemaMigrations(
+      log,
+      debugName,
+      dbPath,
+      setupMigration,
+      schemaVersionMigrationMap
+    );
+  } catch (e) {
+    if (e instanceof SqliteError && e.code === "SQLITE_CORRUPT") {
+      throw new AutoResetSignal(e.message);
     }
+    throw e;
+  }
 }
-export async function upgradeReplica(log, debugName, dbPath) {
-    await runSchemaMigrations(log, debugName, dbPath, 
+async function upgradeReplica(log, debugName, dbPath) {
+  await runSchemaMigrations(
+    log,
+    debugName,
+    dbPath,
     // setupMigration should never be invoked
     {
-        migrateSchema: () => {
-            throw new Error('This should only be called for already synced replicas');
-        },
-    }, schemaVersionMigrationMap);
+      migrateSchema: () => {
+        throw new Error(
+          "This should only be called for already synced replicas"
+        );
+      }
+    },
+    schemaVersionMigrationMap
+  );
 }
-export const schemaVersionMigrationMap = {
-    // There's no incremental migration from v1. Just reset the replica.
-    4: {
-        migrateSchema: () => {
-            throw new AutoResetSignal('upgrading replica to new schema');
-        },
-        minSafeVersion: 3,
+const schemaVersionMigrationMap = {
+  // There's no incremental migration from v1. Just reset the replica.
+  4: {
+    migrateSchema: () => {
+      throw new AutoResetSignal("upgrading replica to new schema");
     },
-    5: {
-        migrateSchema: (_, db) => {
-            db.exec(CREATE_RUNTIME_EVENTS_TABLE);
-        },
-        migrateData: (_, db) => {
-            recordEvent(db, 'upgrade');
-        },
+    minSafeVersion: 3
+  },
+  5: {
+    migrateSchema: (_, db) => {
+      db.exec(CREATE_RUNTIME_EVENTS_TABLE);
     },
-    6: {
-        migrateSchema: (_, db) => {
-            db.exec(CREATE_COLUMN_METADATA_TABLE);
-        },
-        migrateData: (_, db) => {
-            const store = ColumnMetadataStore.getInstance(db);
-            const tables = listTables(db);
-            must(store).populateFromExistingTables(tables);
-        },
+    migrateData: (_, db) => {
+      recordEvent(db, "upgrade");
+    }
+  },
+  6: {
+    migrateSchema: (_, db) => {
+      db.exec(CREATE_COLUMN_METADATA_TABLE);
     },
+    migrateData: (_, db) => {
+      const store = ColumnMetadataStore.getInstance(db);
+      const tables = listTables(db);
+      must(store).populateFromExistingTables(tables);
+    }
+  }
+};
+export {
+  initReplica,
+  schemaVersionMigrationMap,
+  upgradeReplica
 };
-//# sourceMappingURL=replica-schema.js.map
+//# sourceMappingURL=replica-schema.js.map

package/out/zero-cache/src/services/change-source/replica-schema.js.map

@@ -1 +1 @@
-{"version":3,"file":"replica-schema.js","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/change-source/replica-schema.ts"],"names":[],"mappings":"AACA,OAAO,EAAC,WAAW,EAAC,MAAM,wBAAwB,CAAC;AACnD,OAAO,EAAC,IAAI,EAAC,MAAM,gCAAgC,CAAC;AAEpD,OAAO,EACL,mBAAmB,GAGpB,MAAM,4BAA4B,CAAC;AACpC,OAAO,EAAC,UAAU,EAAC,MAAM,yBAAyB,CAAC;AACnD,OAAO,EAAC,eAAe,EAAC,MAAM,qCAAqC,CAAC;AACpE,OAAO,EACL,2BAA2B,EAC3B,WAAW,GACZ,MAAM,2CAA2C,CAAC;AACnD,OAAO,EACL,mBAAmB,EACnB,4BAA4B,GAC7B,MAAM,sBAAsB,CAAC;AAE9B,MAAM,CAAC,KAAK,UAAU,WAAW,CAC/B,GAAe,EACf,SAAiB,EACjB,MAAc,EACd,WAA4D;IAE5D,MAAM,cAAc,GAAc;QAChC,aAAa,EAAE,CAAC,GAAG,EAAE,EAAE,EAAE,EAAE,CAAC,WAAW,CAAC,GAAG,EAAE,EAAE,CAAC;QAChD,cAAc,EAAE,CAAC;KAClB,CAAC;IAEF,IAAI,CAAC;QACH,MAAM,mBAAmB,CACvB,GAAG,EACH,SAAS,EACT,MAAM,EACN,cAAc,EACd,yBAAyB,CAC1B,CAAC;IACJ,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACX,IAAI,CAAC,YAAY,WAAW,IAAI,CAAC,CAAC,IAAI,KAAK,gBAAgB,EAAE,CAAC;YAC5D,MAAM,IAAI,eAAe,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC;QACvC,CAAC;QACD,MAAM,CAAC,CAAC;IACV,CAAC;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,cAAc,CAClC,GAAe,EACf,SAAiB,EACjB,MAAc;IAEd,MAAM,mBAAmB,CACvB,GAAG,EACH,SAAS,EACT,MAAM;IACN,yCAAyC;IACzC;QACE,aAAa,EAAE,GAAG,EAAE;YAClB,MAAM,IAAI,KAAK,CACb,wDAAwD,CACzD,CAAC;QACJ,CAAC;KACF,EACD,yBAAyB,CAC1B,CAAC;AACJ,CAAC;AAED,MAAM,CAAC,MAAM,yBAAyB,GAA4B;IAChE,oEAAoE;IACpE,CAAC,EAAE;QACD,aAAa,EAAE,GAAG,EAAE;YAClB,MAAM,IAAI,eAAe,CAAC,iCAAiC,CAAC,CAAC;QAC/D,CAAC;QACD,cAAc,EAAE,CAAC;KAClB;IAED,CAAC,EAAE;QACD,aAAa,EAAE,CAAC,CAAC,EAAE,EAAE,EAAE,EAAE;YACvB,EAAE,CAAC,IAAI,CAAC,2BAA2B,CAAC,CAAC;QACvC,CAAC;QACD,WAAW,EAAE,CAAC,CAAC,EAAE,EAAE,EAAE,EAAE;YACrB,WAAW,CAAC,EAAE,EAAE,SAAS,CAAC,CAAC;QAC7B,CAAC;KACF;IAED,CAAC,EAAE;QACD,aAAa,EAAE,CAAC,CAAC,EAAE,EAAE,EAAE,EAAE;YACvB,EAAE,CAAC,IAAI,CAAC,4BAA4B,CAAC,CAAC;QACxC,CAAC;QACD,WAAW,EAAE,CAAC,CAAC,EAAE,EAAE,EAAE,EAAE;YACrB,MAAM,KAAK,GAAG,mBAAmB,CAAC,WAAW,CAAC,EAAE,CAAC,CAAC;YAClD,MAAM,MAAM,GAAG,UAAU,CAAC,EAAE,CAAC,CAAC;YAC9B,IAAI,CAAC,KAAK,CAAC,CAAC,0BAA0B,CAAC,MAAM,CAAC,CAAC;QACjD,CAAC;KACF;CACF,CAAC"}
+{"version":3,"file":"replica-schema.js","sources":["../../../../../../zero-cache/src/services/change-source/replica-schema.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {SqliteError} from '@rocicorp/zero-sqlite3';\nimport {must} from '../../../../shared/src/must.ts';\nimport type {Database} from '../../../../zqlite/src/db.ts';\nimport {\n  runSchemaMigrations,\n  type IncrementalMigrationMap,\n  type Migration,\n} from '../../db/migration-lite.ts';\nimport {listTables} from '../../db/lite-tables.ts';\nimport {AutoResetSignal} from '../change-streamer/schema/tables.ts';\nimport {\n  CREATE_RUNTIME_EVENTS_TABLE,\n  recordEvent,\n} from '../replicator/schema/replication-state.ts';\nimport {\n  ColumnMetadataStore,\n  CREATE_COLUMN_METADATA_TABLE,\n} from './column-metadata.ts';\n\nexport async function initReplica(\n  log: LogContext,\n  debugName: string,\n  dbPath: string,\n  initialSync: (lc: LogContext, tx: Database) => Promise<void>,\n): Promise<void> {\n  const setupMigration: Migration = {\n    migrateSchema: (log, tx) => initialSync(log, tx),\n    minSafeVersion: 1,\n  };\n\n  try {\n    await runSchemaMigrations(\n      log,\n      debugName,\n      dbPath,\n      setupMigration,\n      schemaVersionMigrationMap,\n    );\n  } catch (e) {\n    if (e instanceof SqliteError && e.code === 'SQLITE_CORRUPT') {\n      throw new AutoResetSignal(e.message);\n    }\n    throw e;\n  }\n}\n\nexport async function upgradeReplica(\n  log: LogContext,\n  debugName: string,\n  dbPath: string,\n) {\n  await runSchemaMigrations(\n    log,\n    debugName,\n    dbPath,\n    // setupMigration should never be invoked\n    {\n      migrateSchema: () => {\n        throw new Error(\n          'This should only be called for already synced replicas',\n        );\n      },\n    },\n    schemaVersionMigrationMap,\n  );\n}\n\nexport const schemaVersionMigrationMap: IncrementalMigrationMap = {\n  // There's no incremental migration from v1. Just reset the replica.\n  4: {\n    migrateSchema: () => {\n      throw new AutoResetSignal('upgrading replica to new schema');\n    },\n    minSafeVersion: 3,\n  },\n\n  5: {\n    migrateSchema: (_, db) => {\n      db.exec(CREATE_RUNTIME_EVENTS_TABLE);\n    },\n    migrateData: (_, db) => {\n      recordEvent(db, 'upgrade');\n    },\n  },\n\n  6: {\n    migrateSchema: (_, db) => {\n      db.exec(CREATE_COLUMN_METADATA_TABLE);\n    },\n    migrateData: (_, db) => {\n      const store = ColumnMetadataStore.getInstance(db);\n      const tables = listTables(db);\n      must(store).populateFromExistingTables(tables);\n    },\n  },\n};\n"],"names":["log"],"mappings":";;;;;;;AAoBA,eAAsB,YACpB,KACA,WACA,QACA,aACe;AACf,QAAM,iBAA4B;AAAA,IAChC,eAAe,CAACA,MAAK,OAAO,YAAYA,MAAK,EAAE;AAAA,IAC/C,gBAAgB;AAAA,EAAA;AAGlB,MAAI;AACF,UAAM;AAAA,MACJ;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,IAAA;AAAA,EAEJ,SAAS,GAAG;AACV,QAAI,aAAa,eAAe,EAAE,SAAS,kBAAkB;AAC3D,YAAM,IAAI,gBAAgB,EAAE,OAAO;AAAA,IACrC;AACA,UAAM;AAAA,EACR;AACF;AAEA,eAAsB,eACpB,KACA,WACA,QACA;AACA,QAAM;AAAA,IACJ;AAAA,IACA;AAAA,IACA;AAAA;AAAA,IAEA;AAAA,MACE,eAAe,MAAM;AACnB,cAAM,IAAI;AAAA,UACR;AAAA,QAAA;AAAA,MAEJ;AAAA,IAAA;AAAA,IAEF;AAAA,EAAA;AAEJ;AAEO,MAAM,4BAAqD;AAAA;AAAA,EAEhE,GAAG;AAAA,IACD,eAAe,MAAM;AACnB,YAAM,IAAI,gBAAgB,iCAAiC;AAAA,IAC7D;AAAA,IACA,gBAAgB;AAAA,EAAA;AAAA,EAGlB,GAAG;AAAA,IACD,eAAe,CAAC,GAAG,OAAO;AACxB,SAAG,KAAK,2BAA2B;AAAA,IACrC;AAAA,IACA,aAAa,CAAC,GAAG,OAAO;AACtB,kBAAY,IAAI,SAAS;AAAA,IAC3B;AAAA,EAAA;AAAA,EAGF,GAAG;AAAA,IACD,eAAe,CAAC,GAAG,OAAO;AACxB,SAAG,KAAK,4BAA4B;AAAA,IACtC;AAAA,IACA,aAAa,CAAC,GAAG,OAAO;AACtB,YAAM,QAAQ,oBAAoB,YAAY,EAAE;AAChD,YAAM,SAAS,WAAW,EAAE;AAC5B,WAAK,KAAK,EAAE,2BAA2B,MAAM;AAAA,IAC/C;AAAA,EAAA;AAEJ;"}

package/out/zero-cache/src/services/change-streamer/backup-monitor.js

@@ -1,174 +1,152 @@
-import parsePrometheusTextFormat from 'parse-prometheus-text-format';
+import parsePrometheusTextFormat from "parse-prometheus-text-format";
 import { promiseVoid } from "../../../../shared/src/resolved-promises.js";
 import { Subscription } from "../../types/subscription.js";
 import { RunningState } from "../running-state.js";
-export const CHECK_INTERVAL_MS = 60 * 1000;
-const MIN_CLEANUP_DELAY_MS = 30 * 1000;
-/**
- * The BackupMonitor polls the litestream "/metrics" endpoint to track the
- * watermark (label) value of the `litestream_replica_progress` gauge and
- * schedules cleanup of change log entries that can be purged as a result.
- *
- * See: https: *github.com/rocicorp/litestream/pull/3
- *
- * Note that change log entries cannot simply be purged as soon as they
- * have been applied and backed up by litestream. Consider the case in which
- * litestream backs up new wal segments every minute, but it takes 5 minutes
- * to restore a replica: if a zero-cache starts restoring a replica at
- * minute 0, and new watermarks are replicated at minutes 1, 2, 3, 4, and 5,
- * purging changelog records as soon as those watermarks are replicated would
- * result in the zero-cache not being able to catch up from minute 0 once it
- * has finished restoring the replica.
- *
- * The `/snapshot` reservation protocol is used to prevent premature change
- * log cleanup:
- * - Clients restoring a snapshot initiate a `/snapshot` request and hold that
- *   request open while it restores its snapshot, prepares it, and
- *   starts its subscription to the change stream. During this time, no
- *   cleanups are scheduled.
- * - When the subscription is started, the interval since the beginning of
- *   of the reservation is tracked to increase the background cleanup delay
- *   interval if needed. The reservation is ended (and request closed), and
- *   cleanup scheduling is resumed with the current delay interval.
- *
- * Note that the reservation request is the primary mechanism by which
- * premature change log cleanup is prevented. The cleanup delay interval is
- * a secondary safeguard.
- */
-export class BackupMonitor {
-    id = 'backup-monitor';
-    #lc;
-    #backupURL;
-    #metricsEndpoint;
-    #changeStreamer;
-    #state = new RunningState(this.id);
-    #reservations = new Map();
-    #watermarks = new Map();
-    #lastWatermark = '';
-    #cleanupDelayMs;
-    #checkMetricsTimer;
-    constructor(lc, backupURL, metricsEndpoint, changeStreamer, initialCleanupDelayMs) {
-        this.#lc = lc.withContext('component', this.id);
-        this.#backupURL = backupURL;
-        this.#metricsEndpoint = metricsEndpoint;
-        this.#changeStreamer = changeStreamer;
-        this.#cleanupDelayMs = Math.max(initialCleanupDelayMs, MIN_CLEANUP_DELAY_MS);
-        this.#lc.info?.(`backup monitor started ${initialCleanupDelayMs} ms after snapshot restore`);
+const CHECK_INTERVAL_MS = 60 * 1e3;
+const MIN_CLEANUP_DELAY_MS = 30 * 1e3;
+class BackupMonitor {
+  id = "backup-monitor";
+  #lc;
+  #backupURL;
+  #metricsEndpoint;
+  #changeStreamer;
+  #state = new RunningState(this.id);
+  #reservations = /* @__PURE__ */ new Map();
+  #watermarks = /* @__PURE__ */ new Map();
+  #lastWatermark = "";
+  #cleanupDelayMs;
+  #checkMetricsTimer;
+  constructor(lc, backupURL, metricsEndpoint, changeStreamer, initialCleanupDelayMs) {
+    this.#lc = lc.withContext("component", this.id);
+    this.#backupURL = backupURL;
+    this.#metricsEndpoint = metricsEndpoint;
+    this.#changeStreamer = changeStreamer;
+    this.#cleanupDelayMs = Math.max(
+      initialCleanupDelayMs,
+      MIN_CLEANUP_DELAY_MS
+      // purely for peace of mind
+    );
+    this.#lc.info?.(
+      `backup monitor started ${initialCleanupDelayMs} ms after snapshot restore`
+    );
+  }
+  run() {
+    this.#lc.info?.(
+      `monitoring backups at ${this.#metricsEndpoint} with ${this.#cleanupDelayMs} ms cleanup delay`
+    );
+    this.#checkMetricsTimer = setInterval(
+      this.checkWatermarksAndScheduleCleanup,
+      CHECK_INTERVAL_MS
+    );
+    return this.#state.stopped();
+  }
+  async startSnapshotReservation(taskID) {
+    this.#lc.info?.(`pausing change-log cleanup while ${taskID} snapshots`);
+    this.#reservations.get(taskID)?.sub.cancel();
+    const sub = Subscription.create({
+      // If the reservation still exists when the connection closes
+      // (e.g. subscriber crashed), clean it up without updating the
+      // cleanup delay.
+      cleanup: () => this.endReservation(taskID, false)
+    });
+    this.#reservations.set(taskID, { start: /* @__PURE__ */ new Date(), sub });
+    const changeLogState = await this.#changeStreamer.getChangeLogState();
+    sub.push([
+      "status",
+      { tag: "status", backupURL: this.#backupURL, ...changeLogState }
+    ]);
+    return sub;
+  }
+  endReservation(taskID, updateCleanupDelay = true) {
+    const res = this.#reservations.get(taskID);
+    if (res === void 0) {
+      return;
     }
-    run() {
-        this.#lc.info?.(`monitoring backups at ${this.#metricsEndpoint} with ` +
-            `${this.#cleanupDelayMs} ms cleanup delay`);
-        this.#checkMetricsTimer = setInterval(this.checkWatermarksAndScheduleCleanup, CHECK_INTERVAL_MS);
-        return this.#state.stopped();
+    this.#reservations.delete(taskID);
+    const { start, sub } = res;
+    sub.cancel();
+    if (updateCleanupDelay) {
+      const duration = Date.now() - start.getTime();
+      this.#lc.info?.(`snapshot initialized by ${taskID} in ${duration} ms`);
+      if (duration > this.#cleanupDelayMs) {
+        this.#cleanupDelayMs = duration;
+        this.#lc.info?.(`increased cleanup delay to ${duration} ms`);
+      }
     }
-    async startSnapshotReservation(taskID) {
-        this.#lc.info?.(`pausing change-log cleanup while ${taskID} snapshots`);
-        // In the case of retries, only track the last reservation.
-        this.#reservations.get(taskID)?.sub.cancel();
-        const sub = Subscription.create({
-            // If the reservation still exists when the connection closes
-            // (e.g. subscriber crashed), clean it up without updating the
-            // cleanup delay.
-            cleanup: () => this.endReservation(taskID, false),
-        });
-        this.#reservations.set(taskID, { start: new Date(), sub });
-        const changeLogState = await this.#changeStreamer.getChangeLogState();
-        sub.push([
-            'status',
-            { tag: 'status', backupURL: this.#backupURL, ...changeLogState },
-        ]);
-        return sub;
+  }
+  // Exported for testing
+  checkWatermarksAndScheduleCleanup = async () => {
+    try {
+      await this.#checkWatermarks();
+    } catch (e) {
+      this.#lc.warn?.(`unable to fetch metrics at ${this.#metricsEndpoint}`, e);
     }
-    endReservation(taskID, updateCleanupDelay = true) {
-        const res = this.#reservations.get(taskID);
-        if (res === undefined) {
-            return;
-        }
-        this.#reservations.delete(taskID);
-        const { start, sub } = res;
-        sub.cancel(); // closes the connection if still open
-        if (updateCleanupDelay) {
-            const duration = Date.now() - start.getTime();
-            this.#lc.info?.(`snapshot initialized by ${taskID} in ${duration} ms`);
-            if (duration > this.#cleanupDelayMs) {
-                this.#cleanupDelayMs = duration;
-                this.#lc.info?.(`increased cleanup delay to ${duration} ms`);
-            }
-        }
+    try {
+      this.#scheduleCleanup();
+    } catch (e) {
+      this.#lc.warn?.(`error scheduling cleanup`, e);
     }
-    // Exported for testing
-    checkWatermarksAndScheduleCleanup = async () => {
-        try {
-            await this.#checkWatermarks();
-        }
-        catch (e) {
-            this.#lc.warn?.(`unable to fetch metrics at ${this.#metricsEndpoint}`, e);
-        }
-        try {
-            this.#scheduleCleanup();
-        }
-        catch (e) {
-            this.#lc.warn?.(`error scheduling cleanup`, e);
-        }
-    };
-    async #checkWatermarks() {
-        const resp = await fetch(this.#metricsEndpoint);
-        if (!resp.ok) {
-            this.#lc.warn?.(`unable to fetch metrics at ${this.#metricsEndpoint}`, await resp.text());
-            return;
-        }
-        const families = parsePrometheusTextFormat(await resp.text());
-        for (const family of families) {
-            if (family.type === 'GAUGE' &&
-                family.name === 'litestream_replica_progress') {
-                for (const metric of family.metrics) {
-                    const watermark = metric.labels?.watermark;
-                    if (watermark &&
-                        watermark > this.#lastWatermark &&
-                        !this.#watermarks.has(watermark)) {
-                        const time = new Date(parseFloat(metric.value) * 1000);
-                        this.#lc.info?.(`replicated watermark=${watermark} to ${metric.labels?.name}` +
-                            ` at ${time.toISOString()}.`);
-                        this.#watermarks.set(watermark, time);
-                    }
-                }
-            }
-        }
+  };
+  async #checkWatermarks() {
+    const resp = await fetch(this.#metricsEndpoint);
+    if (!resp.ok) {
+      this.#lc.warn?.(
+        `unable to fetch metrics at ${this.#metricsEndpoint}`,
+        await resp.text()
+      );
+      return;
     }
-    #scheduleCleanup() {
-        if (this.#reservations.size > 0) {
-            this.#lc.info?.(`watermark cleanup paused for snapshot(s): ${[...this.#reservations.keys()]}`);
-            return;
-        }
-        const latestCleanupTime = Date.now() - this.#cleanupDelayMs;
-        let maxWatermark = '';
-        for (const [watermark, backupTime] of this.#watermarks.entries()) {
-            if (backupTime.getTime() <= latestCleanupTime &&
-                watermark > maxWatermark) {
-                maxWatermark = watermark;
-            }
-        }
-        if (maxWatermark.length) {
-            this.#changeStreamer.scheduleCleanup(maxWatermark);
-            for (const watermark of this.#watermarks.keys()) {
-                if (watermark <= maxWatermark) {
-                    this.#watermarks.delete(watermark);
-                }
-            }
-            this.#lastWatermark = maxWatermark;
+    const families = parsePrometheusTextFormat(await resp.text());
+    for (const family of families) {
+      if (family.type === "GAUGE" && family.name === "litestream_replica_progress") {
+        for (const metric of family.metrics) {
+          const watermark = metric.labels?.watermark;
+          if (watermark && watermark > this.#lastWatermark && !this.#watermarks.has(watermark)) {
+            const time = new Date(parseFloat(metric.value) * 1e3);
+            this.#lc.info?.(
+              `replicated watermark=${watermark} to ${metric.labels?.name} at ${time.toISOString()}.`
+            );
+            this.#watermarks.set(watermark, time);
+          }
         }
+      }
     }
-    stop() {
-        clearInterval(this.#checkMetricsTimer);
-        for (const { sub } of this.#reservations.values()) {
-            // Close any pending reservations. This commonly happens when a new
-            // replication-manager makes a `/snapshot` reservation on the existing
-            // replication-manager, and then shuts it down when it takes over the
-            // replication slot.
-            sub.cancel();
+  }
+  #scheduleCleanup() {
+    if (this.#reservations.size > 0) {
+      this.#lc.info?.(
+        `watermark cleanup paused for snapshot(s): ${[...this.#reservations.keys()]}`
+      );
+      return;
+    }
+    const latestCleanupTime = Date.now() - this.#cleanupDelayMs;
+    let maxWatermark = "";
+    for (const [watermark, backupTime] of this.#watermarks.entries()) {
+      if (backupTime.getTime() <= latestCleanupTime && watermark > maxWatermark) {
+        maxWatermark = watermark;
+      }
+    }
+    if (maxWatermark.length) {
+      this.#changeStreamer.scheduleCleanup(maxWatermark);
+      for (const watermark of this.#watermarks.keys()) {
+        if (watermark <= maxWatermark) {
+          this.#watermarks.delete(watermark);
         }
-        this.#state.stop(this.#lc);
-        return promiseVoid;
+      }
+      this.#lastWatermark = maxWatermark;
+    }
+  }
+  stop() {
+    clearInterval(this.#checkMetricsTimer);
+    for (const { sub } of this.#reservations.values()) {
+      sub.cancel();
     }
+    this.#state.stop(this.#lc);
+    return promiseVoid;
+  }
 }
-//# sourceMappingURL=backup-monitor.js.map
+export {
+  BackupMonitor,
+  CHECK_INTERVAL_MS
+};
+//# sourceMappingURL=backup-monitor.js.map

package/out/zero-cache/src/services/change-streamer/backup-monitor.js.map

@@ -1 +1 @@
-{"version":3,"file":"backup-monitor.js","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/change-streamer/backup-monitor.ts"],"names":[],"mappings":"AACA,OAAO,yBAAyB,MAAM,8BAA8B,CAAC;AACrE,OAAO,EAAC,WAAW,EAAC,MAAM,6CAA6C,CAAC;AACxE,OAAO,EAAC,YAAY,EAAC,MAAM,6BAA6B,CAAC;AACzD,OAAO,EAAC,YAAY,EAAC,MAAM,qBAAqB,CAAC;AAKjD,MAAM,CAAC,MAAM,iBAAiB,GAAG,EAAE,GAAG,IAAI,CAAC;AAC3C,MAAM,oBAAoB,GAAG,EAAE,GAAG,IAAI,CAAC;AAOvC;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA8BG;AACH,MAAM,OAAO,aAAa;IACf,EAAE,GAAG,gBAAgB,CAAC;IACtB,GAAG,CAAa;IAChB,UAAU,CAAS;IACnB,gBAAgB,CAAS;IACzB,eAAe,CAAwB;IACvC,MAAM,GAAG,IAAI,YAAY,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IAEnC,aAAa,GAAG,IAAI,GAAG,EAAuB,CAAC;IAC/C,WAAW,GAAG,IAAI,GAAG,EAAgB,CAAC;IAE/C,cAAc,GAAW,EAAE,CAAC;IAC5B,eAAe,CAAS;IACxB,kBAAkB,CAA6B;IAE/C,YACE,EAAc,EACd,SAAiB,EACjB,eAAuB,EACvB,cAAqC,EACrC,qBAA6B;QAE7B,IAAI,CAAC,GAAG,GAAG,EAAE,CAAC,WAAW,CAAC,WAAW,EAAE,IAAI,CAAC,EAAE,CAAC,CAAC;QAChD,IAAI,CAAC,UAAU,GAAG,SAAS,CAAC;QAC5B,IAAI,CAAC,gBAAgB,GAAG,eAAe,CAAC;QACxC,IAAI,CAAC,eAAe,GAAG,cAAc,CAAC;QACtC,IAAI,CAAC,eAAe,GAAG,IAAI,CAAC,GAAG,CAC7B,qBAAqB,EACrB,oBAAoB,CACrB,CAAC;QAEF,IAAI,CAAC,GAAG,CAAC,IAAI,EAAE,CACb,0BAA0B,qBAAqB,4BAA4B,CAC5E,CAAC;IACJ,CAAC;IAED,GAAG;QACD,IAAI,CAAC,GAAG,CAAC,IAAI,EAAE,CACb,yBAAyB,IAAI,CAAC,gBAAgB,QAAQ;YACpD,GAAG,IAAI,CAAC,eAAe,mBAAmB,CAC7C,CAAC;QACF,IAAI,CAAC,kBAAkB,GAAG,WAAW,CACnC,IAAI,CAAC,iCAAiC,EACtC,iBAAiB,CAClB,CAAC;QACF,OAAO,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;IAC/B,CAAC;IAED,KAAK,CAAC,wBAAwB,CAC5B,MAAc;QAEd,IAAI,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC,oCAAoC,MAAM,YAAY,CAAC,CAAC;QACxE,2DAA2D;QAC3D,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,GAAG,CAAC,MAAM,EAAE,CAAC;QAE7C,MAAM,GAAG,GAAG,YAAY,CAAC,MAAM,CAAkB;YAC/C,6DAA6D;YAC7D,8DAA8D;YAC9D,iBAAiB;YACjB,OAAO,EAAE,GAAG,EAAE,CAAC,IAAI,CAAC,cAAc,CAAC,MAAM,EAAE,KAAK,CAAC;SAClD,CAAC,CAAC;QACH,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,MAAM,EAAE,EAAC,KAAK,EAAE,IAAI,IAAI,EAAE,EAAE,GAAG,EAAC,CAAC,CAAC;QACzD,MAAM,cAAc,GAAG,MAAM,IAAI,CAAC,eAAe,CAAC,iBAAiB,EAAE,CAAC;QACtE,GAAG,CAAC,IAAI,CAAC;YACP,QAAQ;YACR,EAAC,GAAG,EAAE,QAAQ,EAAE,SAAS,EAAE,IAAI,CAAC,UAAU,EAAE,GAAG,cAAc,EAAC;SAC/D,CAAC,CAAC;QACH,OAAO,GAAG,CAAC;IACb,CAAC;IAED,cAAc,CAAC,MAAc,EAAE,kBAAkB,GAAG,IAAI;QACtD,MAAM,GAAG,GAAG,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;QAC3C,IAAI,GAAG,KAAK,SAAS,EAAE,CAAC;YACtB,OAAO;QACT,CAAC;QACD,IAAI,CAAC,aAAa,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;QAClC,MAAM,EAAC,KAAK,EAAE,GAAG,EAAC,GAAG,GAAG,CAAC;QACzB,GAAG,CAAC,MAAM,EAAE,CAAC,CAAC,sCAAsC;QAEpD,IAAI,kBAAkB,EAAE,CAAC;YACvB,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK,CAAC,OAAO,EAAE,CAAC;YAC9C,IAAI,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC,2BAA2B,MAAM,OAAO,QAAQ,KAAK,CAAC,CAAC;YACvE,IAAI,QAAQ,GAAG,IAAI,CAAC,eAAe,EAAE,CAAC;gBACpC,IAAI,CAAC,eAAe,GAAG,QAAQ,CAAC;gBAChC,IAAI,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC,8BAA8B,QAAQ,KAAK,CAAC,CAAC;YAC/D,CAAC;QACH,CAAC;IACH,CAAC;IAED,uBAAuB;IACd,iCAAiC,GAAG,KAAK,IAAI,EAAE;QACtD,IAAI,CAAC;YACH,MAAM,IAAI,CAAC,gBAAgB,EAAE,CAAC;QAChC,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,IAAI,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC,8BAA8B,IAAI,CAAC,gBAAgB,EAAE,EAAE,CAAC,CAAC,CAAC;QAC5E,CAAC;QACD,IAAI,CAAC;YACH,IAAI,CAAC,gBAAgB,EAAE,CAAC;QAC1B,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,IAAI,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC,0BAA0B,EAAE,CAAC,CAAC,CAAC;QACjD,CAAC;IACH,CAAC,CAAC;IAEF,KAAK,CAAC,gBAAgB;QACpB,MAAM,IAAI,GAAG,MAAM,KAAK,CAAC,IAAI,CAAC,gBAAgB,CAAC,CAAC;QAChD,IAAI,CAAC,IAAI,CAAC,EAAE,EAAE,CAAC;YACb,IAAI,CAAC,GAAG,CAAC,IAAI,EAAE,CACb,8BAA8B,IAAI,CAAC,gBAAgB,EAAE,EACrD,MAAM,IAAI,CAAC,IAAI,EAAE,CAClB,CAAC;YACF,OAAO;QACT,CAAC;QACD,MAAM,QAAQ,GAAG,yBAAyB,CAAC,MAAM,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC;QAC9D,KAAK,MAAM,MAAM,IAAI,QAAQ,EAAE,CAAC;YAC9B,IACE,MAAM,CAAC,IAAI,KAAK,OAAO;gBACvB,MAAM,CAAC,IAAI,KAAK,6BAA6B,EAC7C,CAAC;gBACD,KAAK,MAAM,MAAM,IAAI,MAAM,CAAC,OAAO,EAAE,CAAC;oBACpC,MAAM,SAAS,GAAG,MAAM,CAAC,MAAM,EAAE,SAAS,CAAC;oBAC3C,IACE,SAAS;wBACT,SAAS,GAAG,IAAI,CAAC,cAAc;wBAC/B,CAAC,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,SAAS,CAAC,EAChC,CAAC;wBACD,MAAM,IAAI,GAAG,IAAI,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,IAAI,CAAC,CAAC;wBACvD,IAAI,CAAC,GAAG,CAAC,IAAI,EAAE,CACb,wBAAwB,SAAS,OAAO,MAAM,CAAC,MAAM,EAAE,IAAI,EAAE;4BAC3D,OAAO,IAAI,CAAC,WAAW,EAAE,GAAG,CAC/B,CAAC;wBACF,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,SAAS,EAAE,IAAI,CAAC,CAAC;oBACxC,CAAC;gBACH,CAAC;YACH,CAAC;QACH,CAAC;IACH,CAAC;IAED,gBAAgB;QACd,IAAI,IAAI,CAAC,aAAa,CAAC,IAAI,GAAG,CAAC,EAAE,CAAC;YAChC,IAAI,CAAC,GAAG,CAAC,IAAI,EAAE,CACb,6CAA6C,CAAC,GAAG,IAAI,CAAC,aAAa,CAAC,IAAI,EAAE,CAAC,EAAE,CAC9E,CAAC;YACF,OAAO;QACT,CAAC;QACD,MAAM,iBAAiB,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC,eAAe,CAAC;QAC5D,IAAI,YAAY,GAAG,EAAE,CAAC;QACtB,KAAK,MAAM,CAAC,SAAS,EAAE,UAAU,CAAC,IAAI,IAAI,CAAC,WAAW,CAAC,OAAO,EAAE,EAAE,CAAC;YACjE,IACE,UAAU,CAAC,OAAO,EAAE,IAAI,iBAAiB;gBACzC,SAAS,GAAG,YAAY,EACxB,CAAC;gBACD,YAAY,GAAG,SAAS,CAAC;YAC3B,CAAC;QACH,CAAC;QACD,IAAI,YAAY,CAAC,MAAM,EAAE,CAAC;YACxB,IAAI,CAAC,eAAe,CAAC,eAAe,CAAC,YAAY,CAAC,CAAC;YACnD,KAAK,MAAM,SAAS,IAAI,IAAI,CAAC,WAAW,CAAC,IAAI,EAAE,EAAE,CAAC;gBAChD,IAAI,SAAS,IAAI,YAAY,EAAE,CAAC;oBAC9B,IAAI,CAAC,WAAW,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;gBACrC,CAAC;YACH,CAAC;YACD,IAAI,CAAC,cAAc,GAAG,YAAY,CAAC;QACrC,CAAC;IACH,CAAC;IAED,IAAI;QACF,aAAa,CAAC,IAAI,CAAC,kBAAkB,CAAC,CAAC;QACvC,KAAK,MAAM,EAAC,GAAG,EAAC,IAAI,IAAI,CAAC,aAAa,CAAC,MAAM,EAAE,EAAE,CAAC;YAChD,mEAAmE;YACnE,sEAAsE;YACtE,qEAAqE;YACrE,oBAAoB;YACpB,GAAG,CAAC,MAAM,EAAE,CAAC;QACf,CAAC;QACD,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QAC3B,OAAO,WAAW,CAAC;IACrB,CAAC;CACF"}
+{"version":3,"file":"backup-monitor.js","sources":["../../../../../../zero-cache/src/services/change-streamer/backup-monitor.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport parsePrometheusTextFormat from 'parse-prometheus-text-format';\nimport {promiseVoid} from '../../../../shared/src/resolved-promises.ts';\nimport {Subscription} from '../../types/subscription.ts';\nimport {RunningState} from '../running-state.ts';\nimport type {Service} from '../service.ts';\nimport type {ChangeStreamerService} from './change-streamer.ts';\nimport type {SnapshotMessage} from './snapshot.ts';\n\nexport const CHECK_INTERVAL_MS = 60 * 1000;\nconst MIN_CLEANUP_DELAY_MS = 30 * 1000;\n\ntype Reservation = {\n  start: Date;\n  sub: Subscription<SnapshotMessage>;\n};\n\n/**\n * The BackupMonitor polls the litestream \"/metrics\" endpoint to track the\n * watermark (label) value of the `litestream_replica_progress` gauge and\n * schedules cleanup of change log entries that can be purged as a result.\n *\n * See: https: *github.com/rocicorp/litestream/pull/3\n *\n * Note that change log entries cannot simply be purged as soon as they\n * have been applied and backed up by litestream. Consider the case in which\n * litestream backs up new wal segments every minute, but it takes 5 minutes\n * to restore a replica: if a zero-cache starts restoring a replica at\n * minute 0, and new watermarks are replicated at minutes 1, 2, 3, 4, and 5,\n * purging changelog records as soon as those watermarks are replicated would\n * result in the zero-cache not being able to catch up from minute 0 once it\n * has finished restoring the replica.\n *\n * The `/snapshot` reservation protocol is used to prevent premature change\n * log cleanup:\n * - Clients restoring a snapshot initiate a `/snapshot` request and hold that\n *   request open while it restores its snapshot, prepares it, and\n *   starts its subscription to the change stream. During this time, no\n *   cleanups are scheduled.\n * - When the subscription is started, the interval since the beginning of\n *   of the reservation is tracked to increase the background cleanup delay\n *   interval if needed. The reservation is ended (and request closed), and\n *   cleanup scheduling is resumed with the current delay interval.\n *\n * Note that the reservation request is the primary mechanism by which\n * premature change log cleanup is prevented. The cleanup delay interval is\n * a secondary safeguard.\n */\nexport class BackupMonitor implements Service {\n  readonly id = 'backup-monitor';\n  readonly #lc: LogContext;\n  readonly #backupURL: string;\n  readonly #metricsEndpoint: string;\n  readonly #changeStreamer: ChangeStreamerService;\n  readonly #state = new RunningState(this.id);\n\n  readonly #reservations = new Map<string, Reservation>();\n  readonly #watermarks = new Map<string, Date>();\n\n  #lastWatermark: string = '';\n  #cleanupDelayMs: number;\n  #checkMetricsTimer: NodeJS.Timeout | undefined;\n\n  constructor(\n    lc: LogContext,\n    backupURL: string,\n    metricsEndpoint: string,\n    changeStreamer: ChangeStreamerService,\n    initialCleanupDelayMs: number,\n  ) {\n    this.#lc = lc.withContext('component', this.id);\n    this.#backupURL = backupURL;\n    this.#metricsEndpoint = metricsEndpoint;\n    this.#changeStreamer = changeStreamer;\n    this.#cleanupDelayMs = Math.max(\n      initialCleanupDelayMs,\n      MIN_CLEANUP_DELAY_MS, // purely for peace of mind\n    );\n\n    this.#lc.info?.(\n      `backup monitor started ${initialCleanupDelayMs} ms after snapshot restore`,\n    );\n  }\n\n  run(): Promise<void> {\n    this.#lc.info?.(\n      `monitoring backups at ${this.#metricsEndpoint} with ` +\n        `${this.#cleanupDelayMs} ms cleanup delay`,\n    );\n    this.#checkMetricsTimer = setInterval(\n      this.checkWatermarksAndScheduleCleanup,\n      CHECK_INTERVAL_MS,\n    );\n    return this.#state.stopped();\n  }\n\n  async startSnapshotReservation(\n    taskID: string,\n  ): Promise<Subscription<SnapshotMessage>> {\n    this.#lc.info?.(`pausing change-log cleanup while ${taskID} snapshots`);\n    // In the case of retries, only track the last reservation.\n    this.#reservations.get(taskID)?.sub.cancel();\n\n    const sub = Subscription.create<SnapshotMessage>({\n      // If the reservation still exists when the connection closes\n      // (e.g. subscriber crashed), clean it up without updating the\n      // cleanup delay.\n      cleanup: () => this.endReservation(taskID, false),\n    });\n    this.#reservations.set(taskID, {start: new Date(), sub});\n    const changeLogState = await this.#changeStreamer.getChangeLogState();\n    sub.push([\n      'status',\n      {tag: 'status', backupURL: this.#backupURL, ...changeLogState},\n    ]);\n    return sub;\n  }\n\n  endReservation(taskID: string, updateCleanupDelay = true) {\n    const res = this.#reservations.get(taskID);\n    if (res === undefined) {\n      return;\n    }\n    this.#reservations.delete(taskID);\n    const {start, sub} = res;\n    sub.cancel(); // closes the connection if still open\n\n    if (updateCleanupDelay) {\n      const duration = Date.now() - start.getTime();\n      this.#lc.info?.(`snapshot initialized by ${taskID} in ${duration} ms`);\n      if (duration > this.#cleanupDelayMs) {\n        this.#cleanupDelayMs = duration;\n        this.#lc.info?.(`increased cleanup delay to ${duration} ms`);\n      }\n    }\n  }\n\n  // Exported for testing\n  readonly checkWatermarksAndScheduleCleanup = async () => {\n    try {\n      await this.#checkWatermarks();\n    } catch (e) {\n      this.#lc.warn?.(`unable to fetch metrics at ${this.#metricsEndpoint}`, e);\n    }\n    try {\n      this.#scheduleCleanup();\n    } catch (e) {\n      this.#lc.warn?.(`error scheduling cleanup`, e);\n    }\n  };\n\n  async #checkWatermarks() {\n    const resp = await fetch(this.#metricsEndpoint);\n    if (!resp.ok) {\n      this.#lc.warn?.(\n        `unable to fetch metrics at ${this.#metricsEndpoint}`,\n        await resp.text(),\n      );\n      return;\n    }\n    const families = parsePrometheusTextFormat(await resp.text());\n    for (const family of families) {\n      if (\n        family.type === 'GAUGE' &&\n        family.name === 'litestream_replica_progress'\n      ) {\n        for (const metric of family.metrics) {\n          const watermark = metric.labels?.watermark;\n          if (\n            watermark &&\n            watermark > this.#lastWatermark &&\n            !this.#watermarks.has(watermark)\n          ) {\n            const time = new Date(parseFloat(metric.value) * 1000);\n            this.#lc.info?.(\n              `replicated watermark=${watermark} to ${metric.labels?.name}` +\n                ` at ${time.toISOString()}.`,\n            );\n            this.#watermarks.set(watermark, time);\n          }\n        }\n      }\n    }\n  }\n\n  #scheduleCleanup() {\n    if (this.#reservations.size > 0) {\n      this.#lc.info?.(\n        `watermark cleanup paused for snapshot(s): ${[...this.#reservations.keys()]}`,\n      );\n      return;\n    }\n    const latestCleanupTime = Date.now() - this.#cleanupDelayMs;\n    let maxWatermark = '';\n    for (const [watermark, backupTime] of this.#watermarks.entries()) {\n      if (\n        backupTime.getTime() <= latestCleanupTime &&\n        watermark > maxWatermark\n      ) {\n        maxWatermark = watermark;\n      }\n    }\n    if (maxWatermark.length) {\n      this.#changeStreamer.scheduleCleanup(maxWatermark);\n      for (const watermark of this.#watermarks.keys()) {\n        if (watermark <= maxWatermark) {\n          this.#watermarks.delete(watermark);\n        }\n      }\n      this.#lastWatermark = maxWatermark;\n    }\n  }\n\n  stop(): Promise<void> {\n    clearInterval(this.#checkMetricsTimer);\n    for (const {sub} of this.#reservations.values()) {\n      // Close any pending reservations. This commonly happens when a new\n      // replication-manager makes a `/snapshot` reservation on the existing\n      // replication-manager, and then shuts it down when it takes over the\n      // replication slot.\n      sub.cancel();\n    }\n    this.#state.stop(this.#lc);\n    return promiseVoid;\n  }\n}\n"],"names":[],"mappings":";;;;AASO,MAAM,oBAAoB,KAAK;AACtC,MAAM,uBAAuB,KAAK;AAsC3B,MAAM,cAAiC;AAAA,EACnC,KAAK;AAAA,EACL;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA,SAAS,IAAI,aAAa,KAAK,EAAE;AAAA,EAEjC,oCAAoB,IAAA;AAAA,EACpB,kCAAkB,IAAA;AAAA,EAE3B,iBAAyB;AAAA,EACzB;AAAA,EACA;AAAA,EAEA,YACE,IACA,WACA,iBACA,gBACA,uBACA;AACA,SAAK,MAAM,GAAG,YAAY,aAAa,KAAK,EAAE;AAC9C,SAAK,aAAa;AAClB,SAAK,mBAAmB;AACxB,SAAK,kBAAkB;AACvB,SAAK,kBAAkB,KAAK;AAAA,MAC1B;AAAA,MACA;AAAA;AAAA,IAAA;AAGF,SAAK,IAAI;AAAA,MACP,0BAA0B,qBAAqB;AAAA,IAAA;AAAA,EAEnD;AAAA,EAEA,MAAqB;AACnB,SAAK,IAAI;AAAA,MACP,yBAAyB,KAAK,gBAAgB,SACzC,KAAK,eAAe;AAAA,IAAA;AAE3B,SAAK,qBAAqB;AAAA,MACxB,KAAK;AAAA,MACL;AAAA,IAAA;AAEF,WAAO,KAAK,OAAO,QAAA;AAAA,EACrB;AAAA,EAEA,MAAM,yBACJ,QACwC;AACxC,SAAK,IAAI,OAAO,oCAAoC,MAAM,YAAY;AAEtE,SAAK,cAAc,IAAI,MAAM,GAAG,IAAI,OAAA;AAEpC,UAAM,MAAM,aAAa,OAAwB;AAAA;AAAA;AAAA;AAAA,MAI/C,SAAS,MAAM,KAAK,eAAe,QAAQ,KAAK;AAAA,IAAA,CACjD;AACD,SAAK,cAAc,IAAI,QAAQ,EAAC,OAAO,oBAAI,QAAQ,KAAI;AACvD,UAAM,iBAAiB,MAAM,KAAK,gBAAgB,kBAAA;AAClD,QAAI,KAAK;AAAA,MACP;AAAA,MACA,EAAC,KAAK,UAAU,WAAW,KAAK,YAAY,GAAG,eAAA;AAAA,IAAc,CAC9D;AACD,WAAO;AAAA,EACT;AAAA,EAEA,eAAe,QAAgB,qBAAqB,MAAM;AACxD,UAAM,MAAM,KAAK,cAAc,IAAI,MAAM;AACzC,QAAI,QAAQ,QAAW;AACrB;AAAA,IACF;AACA,SAAK,cAAc,OAAO,MAAM;AAChC,UAAM,EAAC,OAAO,IAAA,IAAO;AACrB,QAAI,OAAA;AAEJ,QAAI,oBAAoB;AACtB,YAAM,WAAW,KAAK,IAAA,IAAQ,MAAM,QAAA;AACpC,WAAK,IAAI,OAAO,2BAA2B,MAAM,OAAO,QAAQ,KAAK;AACrE,UAAI,WAAW,KAAK,iBAAiB;AACnC,aAAK,kBAAkB;AACvB,aAAK,IAAI,OAAO,8BAA8B,QAAQ,KAAK;AAAA,MAC7D;AAAA,IACF;AAAA,EACF;AAAA;AAAA,EAGS,oCAAoC,YAAY;AACvD,QAAI;AACF,YAAM,KAAK,iBAAA;AAAA,IACb,SAAS,GAAG;AACV,WAAK,IAAI,OAAO,8BAA8B,KAAK,gBAAgB,IAAI,CAAC;AAAA,IAC1E;AACA,QAAI;AACF,WAAK,iBAAA;AAAA,IACP,SAAS,GAAG;AACV,WAAK,IAAI,OAAO,4BAA4B,CAAC;AAAA,IAC/C;AAAA,EACF;AAAA,EAEA,MAAM,mBAAmB;AACvB,UAAM,OAAO,MAAM,MAAM,KAAK,gBAAgB;AAC9C,QAAI,CAAC,KAAK,IAAI;AACZ,WAAK,IAAI;AAAA,QACP,8BAA8B,KAAK,gBAAgB;AAAA,QACnD,MAAM,KAAK,KAAA;AAAA,MAAK;AAElB;AAAA,IACF;AACA,UAAM,WAAW,0BAA0B,MAAM,KAAK,MAAM;AAC5D,eAAW,UAAU,UAAU;AAC7B,UACE,OAAO,SAAS,WAChB,OAAO,SAAS,+BAChB;AACA,mBAAW,UAAU,OAAO,SAAS;AACnC,gBAAM,YAAY,OAAO,QAAQ;AACjC,cACE,aACA,YAAY,KAAK,kBACjB,CAAC,KAAK,YAAY,IAAI,SAAS,GAC/B;AACA,kBAAM,OAAO,IAAI,KAAK,WAAW,OAAO,KAAK,IAAI,GAAI;AACrD,iBAAK,IAAI;AAAA,cACP,wBAAwB,SAAS,OAAO,OAAO,QAAQ,IAAI,OAClD,KAAK,YAAA,CAAa;AAAA,YAAA;AAE7B,iBAAK,YAAY,IAAI,WAAW,IAAI;AAAA,UACtC;AAAA,QACF;AAAA,MACF;AAAA,IACF;AAAA,EACF;AAAA,EAEA,mBAAmB;AACjB,QAAI,KAAK,cAAc,OAAO,GAAG;AAC/B,WAAK,IAAI;AAAA,QACP,6CAA6C,CAAC,GAAG,KAAK,cAAc,KAAA,CAAM,CAAC;AAAA,MAAA;AAE7E;AAAA,IACF;AACA,UAAM,oBAAoB,KAAK,IAAA,IAAQ,KAAK;AAC5C,QAAI,eAAe;AACnB,eAAW,CAAC,WAAW,UAAU,KAAK,KAAK,YAAY,WAAW;AAChE,UACE,WAAW,QAAA,KAAa,qBACxB,YAAY,cACZ;AACA,uBAAe;AAAA,MACjB;AAAA,IACF;AACA,QAAI,aAAa,QAAQ;AACvB,WAAK,gBAAgB,gBAAgB,YAAY;AACjD,iBAAW,aAAa,KAAK,YAAY,KAAA,GAAQ;AAC/C,YAAI,aAAa,cAAc;AAC7B,eAAK,YAAY,OAAO,SAAS;AAAA,QACnC;AAAA,MACF;AACA,WAAK,iBAAiB;AAAA,IACxB;AAAA,EACF;AAAA,EAEA,OAAsB;AACpB,kBAAc,KAAK,kBAAkB;AACrC,eAAW,EAAC,IAAA,KAAQ,KAAK,cAAc,UAAU;AAK/C,UAAI,OAAA;AAAA,IACN;AACA,SAAK,OAAO,KAAK,KAAK,GAAG;AACzB,WAAO;AAAA,EACT;AACF;"}

package/out/zero-cache/src/services/change-streamer/change-streamer-http.d.ts

@@ -1,17 +1,26 @@
-import { LogContext } from '@rocicorp/logger';
-import { IncomingMessage } from 'node:http';
+import type { LogContext } from '@rocicorp/logger';
+import type { IncomingMessage } from 'node:http';
 import type { ZeroConfig } from '../../config/zero-config.ts';
 import { type Worker } from '../../types/processes.ts';
 import { type ShardID } from '../../types/shards.ts';
 import { type Source } from '../../types/streams.ts';
-import { HttpService, type Options } from '../http-service.ts';
+import { HttpService } from '../http-service.ts';
+import type { Service } from '../service.ts';
 import type { BackupMonitor } from './backup-monitor.ts';
 import { type ChangeStreamer, type Downstream, type SubscriberContext } from './change-streamer.ts';
 import { type SnapshotMessage } from './snapshot.ts';
+type Options = {
+    port: number;
+    startupDelayMs: number;
+    startupDelayKeepalives: number;
+};
 export declare class ChangeStreamerHttpServer extends HttpService {
     #private;
     readonly id = "change-streamer-http-server";
-    constructor(lc: LogContext, config: ZeroConfig, opts: Options, parent: Worker, changeStreamer: ChangeStreamer, backupMonitor: BackupMonitor | null);
+    constructor(lc: LogContext, config: ZeroConfig, opts: Options, parent: Worker, changeStreamer: ChangeStreamer & Service, backupMonitor: BackupMonitor | null);
+    protected _onStart(): void;
+    protected _onHeartbeat(count: number): void;
+    protected _onStop(): Promise<void>;
 }
 export declare class ChangeStreamerHttpClient implements ChangeStreamer {
     #private;

package/out/zero-cache/src/services/change-streamer/change-streamer-http.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"change-streamer-http.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/change-streamer/change-streamer-http.ts"],"names":[],"mappings":"AACA,OAAO,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAC5C,OAAO,EAAC,eAAe,EAAC,MAAM,WAAW,CAAC;AAI1C,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,6BAA6B,CAAC;AAG5D,OAAO,EAAC,KAAK,MAAM,EAAC,MAAM,0BAA0B,CAAC;AACrD,OAAO,EAAC,KAAK,OAAO,EAAC,MAAM,uBAAuB,CAAC;AACnD,OAAO,EAAsB,KAAK,MAAM,EAAC,MAAM,wBAAwB,CAAC;AAIxE,OAAO,EAAC,WAAW,EAAE,KAAK,OAAO,EAAC,MAAM,oBAAoB,CAAC;AAC7D,OAAO,KAAK,EAAC,aAAa,EAAC,MAAM,qBAAqB,CAAC;AACvD,OAAO,EAGL,KAAK,cAAc,EACnB,KAAK,UAAU,EACf,KAAK,iBAAiB,EACvB,MAAM,sBAAsB,CAAC;AAE9B,OAAO,EAAwB,KAAK,eAAe,EAAC,MAAM,eAAe,CAAC;AAW1E,qBAAa,wBAAyB,SAAQ,WAAW;;IACvD,QAAQ,CAAC,EAAE,iCAAiC;gBAK1C,EAAE,EAAE,UAAU,EACd,MAAM,EAAE,UAAU,EAClB,IAAI,EAAE,OAAO,EACb,MAAM,EAAE,MAAM,EACd,cAAc,EAAE,cAAc,EAC9B,aAAa,EAAE,aAAa,GAAG,IAAI;CAyGtC;AAED,qBAAa,wBAAyB,YAAW,cAAc;;gBAO3D,EAAE,EAAE,UAAU,EACd,OAAO,EAAE,OAAO,EAChB,QAAQ,EAAE,MAAM,EAChB,iBAAiB,EAAE,MAAM,GAAG,SAAS;IA+BjC,eAAe,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,eAAe,CAAC,CAAC;IASjE,SAAS,CAAC,GAAG,EAAE,iBAAiB,GAAG,OAAO,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC;CAQrE;AAED,KAAK,cAAc,GAAG,IAAI,CAAC,eAAe,EAAE,KAAK,GAAG,SAAS,CAAC,CAAC;AAE/D,wBAAgB,oBAAoB,CAAC,GAAG,EAAE,cAAc,GAAG,iBAAiB,CAc3E"}
+{"version":3,"file":"change-streamer-http.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/change-streamer/change-streamer-http.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AACjD,OAAO,KAAK,EAAC,eAAe,EAAC,MAAM,WAAW,CAAC;AAI/C,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,6BAA6B,CAAC;AAG5D,OAAO,EAAC,KAAK,MAAM,EAAC,MAAM,0BAA0B,CAAC;AACrD,OAAO,EAAC,KAAK,OAAO,EAAC,MAAM,uBAAuB,CAAC;AACnD,OAAO,EAAsB,KAAK,MAAM,EAAC,MAAM,wBAAwB,CAAC;AAIxE,OAAO,EAAC,WAAW,EAAC,MAAM,oBAAoB,CAAC;AAC/C,OAAO,KAAK,EAAC,OAAO,EAAC,MAAM,eAAe,CAAC;AAC3C,OAAO,KAAK,EAAC,aAAa,EAAC,MAAM,qBAAqB,CAAC;AACvD,OAAO,EAGL,KAAK,cAAc,EACnB,KAAK,UAAU,EACf,KAAK,iBAAiB,EACvB,MAAM,sBAAsB,CAAC;AAE9B,OAAO,EAAwB,KAAK,eAAe,EAAC,MAAM,eAAe,CAAC;AAW1E,KAAK,OAAO,GAAG;IACb,IAAI,EAAE,MAAM,CAAC;IACb,cAAc,EAAE,MAAM,CAAC;IACvB,sBAAsB,EAAE,MAAM,CAAC;CAChC,CAAC;AAEF,qBAAa,wBAAyB,SAAQ,WAAW;;IACvD,QAAQ,CAAC,EAAE,iCAAiC;gBAO1C,EAAE,EAAE,UAAU,EACd,MAAM,EAAE,UAAU,EAClB,IAAI,EAAE,OAAO,EACb,MAAM,EAAE,MAAM,EACd,cAAc,EAAE,cAAc,GAAG,OAAO,EACxC,aAAa,EAAE,aAAa,GAAG,IAAI;cA+HlB,QAAQ,IAAI,IAAI;cAWhB,YAAY,CAAC,KAAK,EAAE,MAAM,GAAG,IAAI;cAS3B,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC;CAKlD;AAED,qBAAa,wBAAyB,YAAW,cAAc;;gBAO3D,EAAE,EAAE,UAAU,EACd,OAAO,EAAE,OAAO,EAChB,QAAQ,EAAE,MAAM,EAChB,iBAAiB,EAAE,MAAM,GAAG,SAAS;IA+BjC,eAAe,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,eAAe,CAAC,CAAC;IASjE,SAAS,CAAC,GAAG,EAAE,iBAAiB,GAAG,OAAO,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC;CAQrE;AAED,KAAK,cAAc,GAAG,IAAI,CAAC,eAAe,EAAE,KAAK,GAAG,SAAS,CAAC,CAAC;AAE/D,wBAAgB,oBAAoB,CAAC,GAAG,EAAE,cAAc,GAAG,iBAAiB,CAc3E"}