@rocicorp/zero 1.6.0-canary.11 → 1.6.0-canary.13

package/out/zero-cache/src/services/change-streamer/backup-monitor.js.map

@@ -1 +1 @@
-{"version":3,"file":"backup-monitor.js","names":["#lc","#replicaFile","#backupURL","#metricsEndpoint","#changeStreamer","#state","#reservations","#watermarks","#cleanupDelayMs","#checkMetricsTimer","#initBackupLagMetric","#checkWatermarks","#scheduleCleanup","#fetchWatermarks","#lastWatermark","#latestBackupTime"],"sources":["../../../../../../zero-cache/src/services/change-streamer/backup-monitor.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport parsePrometheusTextFormat from 'parse-prometheus-text-format';\nimport {promiseVoid} from '../../../../shared/src/resolved-promises.ts';\nimport {Database} from '../../../../zqlite/src/db.ts';\nimport {getOrCreateGauge} from '../../observability/metrics.ts';\nimport {Subscription} from '../../types/subscription.ts';\nimport {RunningState} from '../running-state.ts';\nimport type {Service} from '../service.ts';\nimport type {ChangeStreamerService} from './change-streamer.ts';\nimport type {SnapshotMessage} from './snapshot.ts';\n\nexport const CHECK_INTERVAL_MS = 60_000;\nconst MIN_CLEANUP_DELAY_MS = 30_000;\n\ntype Reservation = {\n  start: Date;\n  sub: Subscription<SnapshotMessage>;\n};\n\n/**\n * The BackupMonitor polls the litestream \"/metrics\" endpoint to track the\n * watermark (label) value of the `litestream_replica_progress` gauge and\n * schedules cleanup of change log entries that can be purged as a result.\n *\n * See: https://github.com/rocicorp/litestream/pull/3\n *\n * Note that change log entries cannot simply be purged as soon as they\n * have been applied and backed up by litestream. Consider the case in which\n * litestream backs up new wal segments every minute, but it takes 5 minutes\n * to restore a replica: if a zero-cache starts restoring a replica at\n * minute 0, and new watermarks are replicated at minutes 1, 2, 3, 4, and 5,\n * purging changelog records as soon as those watermarks are replicated would\n * result in the zero-cache not being able to catch up from minute 0 once it\n * has finished restoring the replica.\n *\n * The `/snapshot` reservation protocol is used to prevent premature change\n * log cleanup:\n * - Clients restoring a snapshot initiate a `/snapshot` request and hold that\n *   request open while it restores its snapshot, prepares it, and\n *   starts its subscription to the change stream. During this time, no\n *   cleanups are scheduled.\n * - When the subscription is started, the interval since the beginning of\n *   of the reservation is tracked to increase the background cleanup delay\n *   interval if needed. The reservation is ended (and request closed), and\n *   cleanup scheduling is resumed with the current delay interval.\n *\n * Note that the reservation request is the primary mechanism by which\n * premature change log cleanup is prevented. The cleanup delay interval is\n * a secondary safeguard.\n */\nexport class BackupMonitor implements Service {\n  readonly id = 'backup-monitor';\n  readonly #lc: LogContext;\n  readonly #replicaFile: string;\n  readonly #backupURL: string;\n  readonly #metricsEndpoint: string;\n  readonly #changeStreamer: ChangeStreamerService;\n  readonly #state = new RunningState(this.id);\n\n  readonly #reservations = new Map<string, Reservation>();\n  readonly #watermarks = new Map<string, Date>();\n\n  #lastWatermark: string = '';\n  #latestBackupTime: Date | null = null;\n  #cleanupDelayMs: number;\n  #checkMetricsTimer: NodeJS.Timeout | undefined;\n\n  constructor(\n    lc: LogContext,\n    replicaFile: string,\n    backupURL: string,\n    metricsEndpoint: string,\n    changeStreamer: ChangeStreamerService,\n    initialCleanupDelayMs: number,\n  ) {\n    this.#lc = lc.withContext('component', this.id);\n    this.#replicaFile = replicaFile;\n    this.#backupURL = backupURL;\n    this.#metricsEndpoint = metricsEndpoint;\n    this.#changeStreamer = changeStreamer;\n    this.#cleanupDelayMs = Math.max(\n      initialCleanupDelayMs,\n      MIN_CLEANUP_DELAY_MS, // purely for peace of mind\n    );\n\n    this.#lc.info?.(\n      `backup monitor started ${initialCleanupDelayMs} ms after snapshot restore`,\n    );\n  }\n\n  run(): Promise<void> {\n    this.#lc.info?.(\n      `monitoring backups at ${this.#metricsEndpoint} with ` +\n        `${this.#cleanupDelayMs} ms cleanup delay`,\n    );\n    this.#checkMetricsTimer = setInterval(\n      this.checkWatermarksAndScheduleCleanup,\n      CHECK_INTERVAL_MS,\n    );\n    this.#initBackupLagMetric();\n    return this.#state.stopped();\n  }\n\n  startSnapshotReservation(taskID: string): Subscription<SnapshotMessage> {\n    this.#lc.info?.(`pausing change-log cleanup while ${taskID} snapshots`);\n    // In the case of retries, only track the last reservation.\n    this.#reservations.get(taskID)?.sub.cancel();\n\n    const sub = Subscription.create<SnapshotMessage>({\n      // If the reservation still exists when the connection closes\n      // (e.g. subscriber crashed), clean it up without updating the\n      // cleanup delay.\n      cleanup: () => this.endReservation(taskID, false),\n    });\n    this.#reservations.set(taskID, {start: new Date(), sub});\n    // Note: the Subscription must be returned immediately so that the\n    //       websocket can begin sending liveness pings.\n    void this.#changeStreamer\n      .getChangeLogState()\n      .then(changeLogState => {\n        sub.push([\n          'status',\n          {tag: 'status', backupURL: this.#backupURL, ...changeLogState},\n        ]);\n      })\n      .catch(e => {\n        this.#lc.warn?.(`failing snapshot reservation`, e);\n        sub.fail(e);\n      });\n    return sub;\n  }\n\n  endReservation(taskID: string, updateCleanupDelay = true) {\n    const res = this.#reservations.get(taskID);\n    if (res === undefined) {\n      return;\n    }\n    this.#reservations.delete(taskID);\n    const {start, sub} = res;\n    sub.cancel(); // closes the connection if still open\n\n    if (updateCleanupDelay) {\n      const duration = Date.now() - start.getTime();\n      this.#lc.info?.(`snapshot initialized by ${taskID} in ${duration} ms`);\n      if (duration > this.#cleanupDelayMs) {\n        this.#cleanupDelayMs = duration;\n        this.#lc.info?.(`increased cleanup delay to ${duration} ms`);\n      }\n    }\n  }\n\n  // Exported for testing\n  readonly checkWatermarksAndScheduleCleanup = async () => {\n    try {\n      await this.#checkWatermarks();\n    } catch (e) {\n      this.#lc.warn?.(`unable to fetch metrics at ${this.#metricsEndpoint}`, e);\n    }\n    try {\n      this.#scheduleCleanup();\n    } catch (e) {\n      this.#lc.warn?.(`error scheduling cleanup`, e);\n    }\n  };\n\n  async *#fetchWatermarks(): AsyncGenerator<{\n    watermark: string;\n    time: Date;\n    name?: string | undefined;\n  }> {\n    const metricsEndpoint = this.#metricsEndpoint;\n    const signal = this.#state.signal;\n    let resp;\n    try {\n      resp = await fetch(metricsEndpoint, {signal});\n    } catch (e) {\n      if (signal.aborted) {\n        // not an error.\n        return;\n      }\n      // Treat exceptions from fetch (e.g. network errors) as non-fatal, and simply\n      // log them and skip the watermark check until the next interval.\n      this.#lc.warn?.(`unable to fetch metrics at ${this.#metricsEndpoint}`, e);\n      return;\n    }\n    if (!resp.ok) {\n      this.#lc.warn?.(\n        `unable to fetch metrics at ${this.#metricsEndpoint}: ${await resp.text()}`,\n      );\n      return;\n    }\n\n    const families = parsePrometheusTextFormat(await resp.text());\n    for (const family of families) {\n      if (\n        family.type === 'GAUGE' &&\n        family.name === 'litestream_replica_progress'\n      ) {\n        for (const metric of family.metrics) {\n          const watermark = metric.labels?.watermark;\n          const name = metric.labels?.name;\n          const time = new Date(parseFloat(metric.value) * 1000);\n\n          if (watermark) {\n            yield {watermark, time, name};\n          }\n        }\n      }\n    }\n  }\n\n  async #checkWatermarks() {\n    for await (const {watermark, name, time} of this.#fetchWatermarks()) {\n      if (watermark > this.#lastWatermark && !this.#watermarks.has(watermark)) {\n        this.#lc.info?.(\n          `replicated watermark=${watermark} to ${name}` +\n            ` at ${time.toISOString()}.`,\n        );\n        this.#watermarks.set(watermark, time);\n        this.#latestBackupTime = time;\n      }\n    }\n    return this.#latestBackupTime;\n  }\n\n  #scheduleCleanup() {\n    if (this.#reservations.size > 0) {\n      this.#lc.info?.(\n        `watermark cleanup paused for snapshot(s): ${[...this.#reservations.keys()]}`,\n      );\n      return;\n    }\n    const latestCleanupTime = Date.now() - this.#cleanupDelayMs;\n    let maxWatermark = '';\n    for (const [watermark, backupTime] of this.#watermarks.entries()) {\n      if (\n        backupTime.getTime() <= latestCleanupTime &&\n        watermark > maxWatermark\n      ) {\n        maxWatermark = watermark;\n      }\n    }\n    if (maxWatermark.length) {\n      this.#changeStreamer.scheduleCleanup(maxWatermark);\n      for (const watermark of this.#watermarks.keys()) {\n        if (watermark <= maxWatermark) {\n          this.#watermarks.delete(watermark);\n        }\n      }\n      this.#lastWatermark = maxWatermark;\n    }\n  }\n\n  stop(): Promise<void> {\n    clearInterval(this.#checkMetricsTimer);\n    for (const {sub} of this.#reservations.values()) {\n      // Close any pending reservations. This commonly happens when a new\n      // replication-manager makes a `/snapshot` reservation on the existing\n      // replication-manager, and then shuts it down when it takes over the\n      // replication slot.\n      sub.cancel();\n    }\n    this.#state.stop(this.#lc);\n    return promiseVoid;\n  }\n\n  #initBackupLagMetric() {\n    getOrCreateGauge('replica', 'backup_lag', {\n      description:\n        'Latency from when a change is written to the replica ' +\n        'to when it is backed up to litestream. It is expected to create a saw ' +\n        'pattern from 0 to the configured ZERO_LITESTREAM_INCREMENTAL_BACKUP_INTERVAL_MINUTES.',\n      unit: 'millisecond',\n    }).addCallback(async o => {\n      // For legacy litestream, we use the watermark metric (and its associated\n      // backup time) exported by litestream metrics to determine the time of\n      // of the backed up watermark. This is technically imprecise--it would be\n      // more correct to use the committed writeTimeMs--but it is good enough\n      // in that it serves the purpose of detecting a non-functioning backup.\n      // With litestream v5, this can be made more precise by querying the\n      // _zero.replicationState row from the backup directly using an LTX-based\n      // database reader.\n      const latestBackup = await this.#checkWatermarks();\n      if (!latestBackup) {\n        this.#lc.warn?.(\n          `no backed up watermarks. unable to report replica.backup_lag`,\n        );\n        return;\n      }\n      const db = new Database(this.#lc, this.#replicaFile, {readonly: true});\n      try {\n        const {writeTimeMs} = db\n          .prepare(/*sql*/ `SELECT writeTimeMs FROM \"_zero.replicationState\"`)\n          .get<{writeTimeMs: number}>();\n        const backupLag = Math.max(0, writeTimeMs - latestBackup.getTime());\n        o.observe(backupLag);\n      } catch (e) {\n        this.#lc.warn?.(`error measuring replica.backup_lag metric`, e);\n      } finally {\n        db.close();\n      }\n    });\n  }\n}\n"],"mappings":";;;;;;;AAWA,IAAa,oBAAoB;AACjC,IAAM,uBAAuB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAsC7B,IAAa,gBAAb,MAA8C;CAC5C,KAAc;CACd;CACA;CACA;CACA;CACA;CACA,SAAkB,IAAI,aAAa,KAAK,EAAE;CAE1C,gCAAyB,IAAI,IAAyB;CACtD,8BAAuB,IAAI,IAAkB;CAE7C,iBAAyB;CACzB,oBAAiC;CACjC;CACA;CAEA,YACE,IACA,aACA,WACA,iBACA,gBACA,uBACA;EACA,KAAKA,MAAM,GAAG,YAAY,aAAa,KAAK,EAAE;EAC9C,KAAKC,eAAe;EACpB,KAAKC,aAAa;EAClB,KAAKC,mBAAmB;EACxB,KAAKC,kBAAkB;EACvB,KAAKI,kBAAkB,KAAK,IAC1B,uBACA,oBACF;EAEA,KAAKR,IAAI,OACP,0BAA0B,sBAAsB,2BAClD;CACF;CAEA,MAAqB;EACnB,KAAKA,IAAI,OACP,yBAAyB,KAAKG,iBAAiB,QAC1C,KAAKK,gBAAgB,kBAC5B;EACA,KAAKC,qBAAqB,YACxB,KAAK,mCACL,iBACF;EACA,KAAKC,qBAAqB;EAC1B,OAAO,KAAKL,OAAO,QAAQ;CAC7B;CAEA,yBAAyB,QAA+C;EACtE,KAAKL,IAAI,OAAO,oCAAoC,OAAO,WAAW;EAEtE,KAAKM,cAAc,IAAI,MAAM,GAAG,IAAI,OAAO;EAE3C,MAAM,MAAM,aAAa,OAAwB,EAI/C,eAAe,KAAK,eAAe,QAAQ,KAAK,EAClD,CAAC;EACD,KAAKA,cAAc,IAAI,QAAQ;GAAC,uBAAO,IAAI,KAAK;GAAG;EAAG,CAAC;EAGvD,KAAUF,gBACP,kBAAkB,EAClB,MAAK,mBAAkB;GACtB,IAAI,KAAK,CACP,UACA;IAAC,KAAK;IAAU,WAAW,KAAKF;IAAY,GAAG;GAAc,CAC/D,CAAC;EACH,CAAC,EACA,OAAM,MAAK;GACV,KAAKF,IAAI,OAAO,gCAAgC,CAAC;GACjD,IAAI,KAAK,CAAC;EACZ,CAAC;EACH,OAAO;CACT;CAEA,eAAe,QAAgB,qBAAqB,MAAM;EACxD,MAAM,MAAM,KAAKM,cAAc,IAAI,MAAM;EACzC,IAAI,QAAQ,KAAA,GACV;EAEF,KAAKA,cAAc,OAAO,MAAM;EAChC,MAAM,EAAC,OAAO,QAAO;EACrB,IAAI,OAAO;EAEX,IAAI,oBAAoB;GACtB,MAAM,WAAW,KAAK,IAAI,IAAI,MAAM,QAAQ;GAC5C,KAAKN,IAAI,OAAO,2BAA2B,OAAO,MAAM,SAAS,IAAI;GACrE,IAAI,WAAW,KAAKQ,iBAAiB;IACnC,KAAKA,kBAAkB;IACvB,KAAKR,IAAI,OAAO,8BAA8B,SAAS,IAAI;GAC7D;EACF;CACF;CAGA,oCAA6C,YAAY;EACvD,IAAI;GACF,MAAM,KAAKW,iBAAiB;EAC9B,SAAS,GAAG;GACV,KAAKX,IAAI,OAAO,8BAA8B,KAAKG,oBAAoB,CAAC;EAC1E;EACA,IAAI;GACF,KAAKS,iBAAiB;EACxB,SAAS,GAAG;GACV,KAAKZ,IAAI,OAAO,4BAA4B,CAAC;EAC/C;CACF;CAEA,OAAOa,mBAIJ;EACD,MAAM,kBAAkB,KAAKV;EAC7B,MAAM,SAAS,KAAKE,OAAO;EAC3B,IAAI;EACJ,IAAI;GACF,OAAO,MAAM,MAAM,iBAAiB,EAAC,OAAM,CAAC;EAC9C,SAAS,GAAG;GACV,IAAI,OAAO,SAET;GAIF,KAAKL,IAAI,OAAO,8BAA8B,KAAKG,oBAAoB,CAAC;GACxE;EACF;EACA,IAAI,CAAC,KAAK,IAAI;GACZ,KAAKH,IAAI,OACP,8BAA8B,KAAKG,iBAAiB,IAAI,MAAM,KAAK,KAAK,GAC1E;GACA;EACF;EAEA,MAAM,WAAW,0BAA0B,MAAM,KAAK,KAAK,CAAC;EAC5D,KAAK,MAAM,UAAU,UACnB,IACE,OAAO,SAAS,WAChB,OAAO,SAAS,+BAEhB,KAAK,MAAM,UAAU,OAAO,SAAS;GACnC,MAAM,YAAY,OAAO,QAAQ;GACjC,MAAM,OAAO,OAAO,QAAQ;GAC5B,MAAM,uBAAO,IAAI,KAAK,WAAW,OAAO,KAAK,IAAI,GAAI;GAErD,IAAI,WACF,MAAM;IAAC;IAAW;IAAM;GAAI;EAEhC;CAGN;CAEA,MAAMQ,mBAAmB;EACvB,WAAW,MAAM,EAAC,WAAW,MAAM,UAAS,KAAKE,iBAAiB,GAChE,IAAI,YAAY,KAAKC,kBAAkB,CAAC,KAAKP,YAAY,IAAI,SAAS,GAAG;GACvE,KAAKP,IAAI,OACP,wBAAwB,UAAU,MAAM,KAAA,MAC/B,KAAK,YAAY,EAAE,EAC9B;GACA,KAAKO,YAAY,IAAI,WAAW,IAAI;GACpC,KAAKQ,oBAAoB;EAC3B;EAEF,OAAO,KAAKA;CACd;CAEA,mBAAmB;EACjB,IAAI,KAAKT,cAAc,OAAO,GAAG;GAC/B,KAAKN,IAAI,OACP,6CAA6C,CAAC,GAAG,KAAKM,cAAc,KAAK,CAAC,GAC5E;GACA;EACF;EACA,MAAM,oBAAoB,KAAK,IAAI,IAAI,KAAKE;EAC5C,IAAI,eAAe;EACnB,KAAK,MAAM,CAAC,WAAW,eAAe,KAAKD,YAAY,QAAQ,GAC7D,IACE,WAAW,QAAQ,KAAK,qBACxB,YAAY,cAEZ,eAAe;EAGnB,IAAI,aAAa,QAAQ;GACvB,KAAKH,gBAAgB,gBAAgB,YAAY;GACjD,KAAK,MAAM,aAAa,KAAKG,YAAY,KAAK,GAC5C,IAAI,aAAa,cACf,KAAKA,YAAY,OAAO,SAAS;GAGrC,KAAKO,iBAAiB;EACxB;CACF;CAEA,OAAsB;EACpB,cAAc,KAAKL,kBAAkB;EACrC,KAAK,MAAM,EAAC,SAAQ,KAAKH,cAAc,OAAO,GAK5C,IAAI,OAAO;EAEb,KAAKD,OAAO,KAAK,KAAKL,GAAG;EACzB,OAAO;CACT;CAEA,uBAAuB;EACrB,iBAAiB,WAAW,cAAc;GACxC,aACE;GAGF,MAAM;EACR,CAAC,EAAE,YAAY,OAAM,MAAK;GASxB,MAAM,eAAe,MAAM,KAAKW,iBAAiB;GACjD,IAAI,CAAC,cAAc;IACjB,KAAKX,IAAI,OACP,8DACF;IACA;GACF;GACA,MAAM,KAAK,IAAI,SAAS,KAAKA,KAAK,KAAKC,cAAc,EAAC,UAAU,KAAI,CAAC;GACrE,IAAI;IACF,MAAM,EAAC,gBAAe,GACnB,QAAgB,kDAAkD,EAClE,IAA2B;IAC9B,MAAM,YAAY,KAAK,IAAI,GAAG,cAAc,aAAa,QAAQ,CAAC;IAClE,EAAE,QAAQ,SAAS;GACrB,SAAS,GAAG;IACV,KAAKD,IAAI,OAAO,6CAA6C,CAAC;GAChE,UAAU;IACR,GAAG,MAAM;GACX;EACF,CAAC;CACH;AACF"}
+{"version":3,"file":"backup-monitor.js","names":["#lc","#replicaFile","#backupURL","#metricsEndpoint","#changeStreamer","#state","#reservations","#watermarks","#cleanupDelayMs","#checkMetricsTimer","#initBackupLagMetric","#checkWatermarks","#scheduleCleanup","#fetchWatermarks","#lastWatermark","#latestBackupTime"],"sources":["../../../../../../zero-cache/src/services/change-streamer/backup-monitor.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport parsePrometheusTextFormat from 'parse-prometheus-text-format';\nimport {promiseVoid} from '../../../../shared/src/resolved-promises.ts';\nimport {Database} from '../../../../zqlite/src/db.ts';\nimport {getOrCreateGauge} from '../../observability/metrics.ts';\nimport {Subscription} from '../../types/subscription.ts';\nimport {RunningState} from '../running-state.ts';\nimport type {Service} from '../service.ts';\nimport type {ChangeStreamerService} from './change-streamer.ts';\nimport type {SnapshotMessage} from './snapshot.ts';\n\nexport const CHECK_INTERVAL_MS = 60_000;\nconst MIN_CLEANUP_DELAY_MS = 30_000;\n\ntype Reservation = {\n  start: Date;\n  sub: Subscription<SnapshotMessage>;\n};\n\n/**\n * The BackupMonitor polls the litestream \"/metrics\" endpoint to track the\n * watermark (label) value of the `litestream_replica_progress` gauge and\n * schedules cleanup of change log entries that can be purged as a result.\n *\n * See: https://github.com/rocicorp/litestream/pull/3\n *\n * Note that change log entries cannot simply be purged as soon as they\n * have been applied and backed up by litestream. Consider the case in which\n * litestream backs up new wal segments every minute, but it takes 5 minutes\n * to restore a replica: if a zero-cache starts restoring a replica at\n * minute 0, and new watermarks are replicated at minutes 1, 2, 3, 4, and 5,\n * purging changelog records as soon as those watermarks are replicated would\n * result in the zero-cache not being able to catch up from minute 0 once it\n * has finished restoring the replica.\n *\n * The `/snapshot` reservation protocol is used to prevent premature change\n * log cleanup:\n * - Clients restoring a snapshot initiate a `/snapshot` request and hold that\n *   request open while it restores its snapshot, prepares it, and\n *   starts its subscription to the change stream. During this time, no\n *   cleanups are scheduled.\n * - When the subscription is started, the interval since the beginning of\n *   of the reservation is tracked to increase the background cleanup delay\n *   interval if needed. The reservation is ended (and request closed), and\n *   cleanup scheduling is resumed with the current delay interval.\n *\n * Note that the reservation request is the primary mechanism by which\n * premature change log cleanup is prevented. The cleanup delay interval is\n * a secondary safeguard.\n */\nexport class BackupMonitor implements Service {\n  readonly id = 'backup-monitor';\n  readonly #lc: LogContext;\n  readonly #replicaFile: string;\n  readonly #backupURL: string;\n  readonly #metricsEndpoint: string;\n  readonly #changeStreamer: ChangeStreamerService;\n  readonly #state = new RunningState(this.id);\n\n  readonly #reservations = new Map<string, Reservation>();\n  readonly #watermarks = new Map<string, Date>();\n\n  #lastWatermark: string = '';\n  #latestBackupTime: Date | null = null;\n  #cleanupDelayMs: number;\n  #checkMetricsTimer: NodeJS.Timeout | undefined;\n\n  constructor(\n    lc: LogContext,\n    replicaFile: string,\n    backupURL: string,\n    metricsEndpoint: string,\n    changeStreamer: ChangeStreamerService,\n    initialCleanupDelayMs: number,\n  ) {\n    this.#lc = lc.withContext('component', this.id);\n    this.#replicaFile = replicaFile;\n    this.#backupURL = backupURL;\n    this.#metricsEndpoint = metricsEndpoint;\n    this.#changeStreamer = changeStreamer;\n    this.#cleanupDelayMs = Math.max(\n      initialCleanupDelayMs,\n      MIN_CLEANUP_DELAY_MS, // purely for peace of mind\n    );\n\n    this.#lc.info?.(\n      `backup monitor started ${initialCleanupDelayMs} ms after snapshot restore`,\n    );\n  }\n\n  run(): Promise<void> {\n    this.#lc.info?.(\n      `monitoring backups at ${this.#metricsEndpoint} with ` +\n        `${this.#cleanupDelayMs} ms cleanup delay`,\n    );\n    this.#checkMetricsTimer = setInterval(\n      this.checkWatermarksAndScheduleCleanup,\n      CHECK_INTERVAL_MS,\n    );\n    this.#initBackupLagMetric();\n    return this.#state.stopped();\n  }\n\n  startSnapshotReservation(taskID: string): Subscription<SnapshotMessage> {\n    this.#lc.info?.(`pausing change-log cleanup while ${taskID} snapshots`);\n    // In the case of retries, only track the last reservation.\n    this.#reservations.get(taskID)?.sub.cancel();\n\n    const sub = Subscription.create<SnapshotMessage>({\n      // If the reservation still exists when the connection closes\n      // (e.g. subscriber crashed), clean it up without updating the\n      // cleanup delay.\n      cleanup: () => this.endReservation(taskID, false),\n    });\n    this.#reservations.set(taskID, {start: new Date(), sub});\n    // Note: the Subscription must be returned immediately so that the\n    //       websocket can begin sending liveness pings.\n    void this.#changeStreamer\n      .getChangeLogState()\n      .then(changeLogState => {\n        sub.push([\n          'status',\n          {tag: 'status', backupURL: this.#backupURL, ...changeLogState},\n        ]);\n      })\n      .catch(e => {\n        this.#lc.warn?.(`failing snapshot reservation`, e);\n        sub.fail(e);\n      });\n    return sub;\n  }\n\n  endReservation(taskID: string, updateCleanupDelay = true) {\n    const res = this.#reservations.get(taskID);\n    if (res === undefined) {\n      return;\n    }\n    this.#reservations.delete(taskID);\n    const {start, sub} = res;\n    sub.cancel(); // closes the connection if still open\n\n    if (updateCleanupDelay) {\n      const duration = Date.now() - start.getTime();\n      this.#lc.info?.(`snapshot initialized by ${taskID} in ${duration} ms`);\n      if (duration > this.#cleanupDelayMs) {\n        this.#cleanupDelayMs = duration;\n        this.#lc.info?.(`increased cleanup delay to ${duration} ms`);\n      }\n    }\n  }\n\n  // Exported for testing\n  readonly checkWatermarksAndScheduleCleanup = async () => {\n    try {\n      await this.#checkWatermarks();\n    } catch (e) {\n      this.#lc.warn?.(`unable to fetch metrics at ${this.#metricsEndpoint}`, e);\n    }\n    try {\n      this.#scheduleCleanup();\n    } catch (e) {\n      this.#lc.warn?.(`error scheduling cleanup`, e);\n    }\n  };\n\n  async *#fetchWatermarks(): AsyncGenerator<{\n    watermark: string;\n    time: Date;\n    name?: string | undefined;\n  }> {\n    const metricsEndpoint = this.#metricsEndpoint;\n    const signal = this.#state.signal;\n    let resp;\n    try {\n      resp = await fetch(metricsEndpoint, {signal});\n    } catch (e) {\n      if (signal.aborted) {\n        // not an error.\n        return;\n      }\n      // Treat exceptions from fetch (e.g. network errors) as non-fatal, and simply\n      // log them and skip the watermark check until the next interval.\n      this.#lc.warn?.(`unable to fetch metrics at ${this.#metricsEndpoint}`, e);\n      return;\n    }\n    if (!resp.ok) {\n      this.#lc.warn?.(\n        `unable to fetch metrics at ${this.#metricsEndpoint}: ${await resp.text()}`,\n      );\n      return;\n    }\n\n    const families = parsePrometheusTextFormat(await resp.text());\n    for (const family of families) {\n      if (\n        family.type === 'GAUGE' &&\n        family.name === 'litestream_replica_progress'\n      ) {\n        for (const metric of family.metrics) {\n          const watermark = metric.labels?.watermark;\n          const name = metric.labels?.name;\n          const time = new Date(parseFloat(metric.value) * 1000);\n\n          if (watermark) {\n            yield {watermark, time, name};\n          }\n        }\n      }\n    }\n  }\n\n  async #checkWatermarks() {\n    for await (const {watermark, name, time} of this.#fetchWatermarks()) {\n      if (watermark > this.#lastWatermark && !this.#watermarks.has(watermark)) {\n        this.#lc.info?.(\n          `replicated watermark=${watermark} to ${name}` +\n            ` at ${time.toISOString()}.`,\n        );\n        this.#watermarks.set(watermark, time);\n        this.#latestBackupTime = time;\n      }\n    }\n    return this.#latestBackupTime;\n  }\n\n  #scheduleCleanup() {\n    if (this.#reservations.size > 0) {\n      this.#lc.info?.(\n        `watermark cleanup paused for snapshot(s): ${[...this.#reservations.keys()]}`,\n      );\n      return;\n    }\n    const latestCleanupTime = Date.now() - this.#cleanupDelayMs;\n    let maxWatermark = '';\n    for (const [watermark, backupTime] of this.#watermarks.entries()) {\n      if (\n        backupTime.getTime() <= latestCleanupTime &&\n        watermark > maxWatermark\n      ) {\n        maxWatermark = watermark;\n      }\n    }\n    if (maxWatermark.length) {\n      this.#changeStreamer.scheduleCleanup(maxWatermark);\n      for (const watermark of this.#watermarks.keys()) {\n        if (watermark <= maxWatermark) {\n          this.#watermarks.delete(watermark);\n        }\n      }\n      this.#lastWatermark = maxWatermark;\n    }\n  }\n\n  stop(): Promise<void> {\n    clearInterval(this.#checkMetricsTimer);\n    for (const {sub} of this.#reservations.values()) {\n      // Close any pending reservations. This commonly happens when a new\n      // replication-manager makes a `/snapshot` reservation on the existing\n      // replication-manager, and then shuts it down when it takes over the\n      // replication slot.\n      sub.cancel();\n    }\n    this.#state.stop(this.#lc);\n    return promiseVoid;\n  }\n\n  #initBackupLagMetric() {\n    getOrCreateGauge('replica', 'backup_lag', {\n      description:\n        'Latency from when a change is written to the replica ' +\n        'to when it is backed up to litestream. It is expected to create a saw ' +\n        'pattern from 0 to the configured ZERO_LITESTREAM_INCREMENTAL_BACKUP_INTERVAL_MINUTES.',\n      unit: 'millisecond',\n    }).addCallback(async o => {\n      // For legacy litestream, we use the watermark metric (and its associated\n      // backup time) exported by litestream metrics to determine the time of\n      // of the backed up watermark. This is technically imprecise--it would be\n      // more correct to use the committed writeTimeMs--but it is good enough\n      // in that it serves the purpose of detecting a non-functioning backup.\n      // With litestream v5, this can be made more precise by querying the\n      // _zero.replicationState row from the backup directly using an LTX-based\n      // database reader.\n      const latestBackup = await this.#checkWatermarks();\n      if (!latestBackup) {\n        this.#lc.warn?.(\n          `no backed up watermarks. unable to report replica.backup_lag`,\n        );\n        return;\n      }\n      const db = new Database(this.#lc, this.#replicaFile, {readonly: true});\n      try {\n        const {writeTimeMs} = db\n          .prepare(/*sql*/ `SELECT writeTimeMs FROM \"_zero.replicationState\"`)\n          .get<{writeTimeMs: number}>();\n        const backupLag = Math.max(0, writeTimeMs - latestBackup.getTime());\n        o.observe(backupLag);\n      } catch (e) {\n        this.#lc.warn?.(`error measuring replica.backup_lag metric`, e);\n      } finally {\n        db.close();\n      }\n    });\n  }\n}\n"],"mappings":";;;;;;;AAWA,IAAa,oBAAoB;AACjC,IAAM,uBAAuB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAsC7B,IAAa,gBAAb,MAA8C;CAC5C,KAAc;CACd;CACA;CACA;CACA;CACA;CACA,SAAkB,IAAI,aAAa,KAAK,GAAG;CAE3C,gCAAyB,IAAI,KAA0B;CACvD,8BAAuB,IAAI,KAAmB;CAE9C,iBAAyB;CACzB,oBAAiC;CACjC;CACA;CAEA,YACE,IACA,aACA,WACA,iBACA,gBACA,uBACA;AACA,QAAA,KAAW,GAAG,YAAY,aAAa,KAAK,GAAG;AAC/C,QAAA,cAAoB;AACpB,QAAA,YAAkB;AAClB,QAAA,kBAAwB;AACxB,QAAA,iBAAuB;AACvB,QAAA,iBAAuB,KAAK,IAC1B,uBACA,qBACD;AAED,QAAA,GAAS,OACP,0BAA0B,sBAAsB,4BACjD;;CAGH,MAAqB;AACnB,QAAA,GAAS,OACP,yBAAyB,MAAA,gBAAsB,QAC1C,MAAA,eAAqB,mBAC3B;AACD,QAAA,oBAA0B,YACxB,KAAK,mCACL,kBACD;AACD,QAAA,qBAA2B;AAC3B,SAAO,MAAA,MAAY,SAAS;;CAG9B,yBAAyB,QAA+C;AACtE,QAAA,GAAS,OAAO,oCAAoC,OAAO,YAAY;AAEvE,QAAA,aAAmB,IAAI,OAAO,EAAE,IAAI,QAAQ;EAE5C,MAAM,MAAM,aAAa,OAAwB,EAI/C,eAAe,KAAK,eAAe,QAAQ,MAAM,EAClD,CAAC;AACF,QAAA,aAAmB,IAAI,QAAQ;GAAC,uBAAO,IAAI,MAAM;GAAE;GAAI,CAAC;AAGnD,QAAA,eACF,mBAAmB,CACnB,MAAK,mBAAkB;AACtB,OAAI,KAAK,CACP,UACA;IAAC,KAAK;IAAU,WAAW,MAAA;IAAiB,GAAG;IAAe,CAC/D,CAAC;IACF,CACD,OAAM,MAAK;AACV,SAAA,GAAS,OAAO,gCAAgC,EAAE;AAClD,OAAI,KAAK,EAAE;IACX;AACJ,SAAO;;CAGT,eAAe,QAAgB,qBAAqB,MAAM;EACxD,MAAM,MAAM,MAAA,aAAmB,IAAI,OAAO;AAC1C,MAAI,QAAQ,KAAA,EACV;AAEF,QAAA,aAAmB,OAAO,OAAO;EACjC,MAAM,EAAC,OAAO,QAAO;AACrB,MAAI,QAAQ;AAEZ,MAAI,oBAAoB;GACtB,MAAM,WAAW,KAAK,KAAK,GAAG,MAAM,SAAS;AAC7C,SAAA,GAAS,OAAO,2BAA2B,OAAO,MAAM,SAAS,KAAK;AACtE,OAAI,WAAW,MAAA,gBAAsB;AACnC,UAAA,iBAAuB;AACvB,UAAA,GAAS,OAAO,8BAA8B,SAAS,KAAK;;;;CAMlE,oCAA6C,YAAY;AACvD,MAAI;AACF,SAAM,MAAA,iBAAuB;WACtB,GAAG;AACV,SAAA,GAAS,OAAO,8BAA8B,MAAA,mBAAyB,EAAE;;AAE3E,MAAI;AACF,SAAA,iBAAuB;WAChB,GAAG;AACV,SAAA,GAAS,OAAO,4BAA4B,EAAE;;;CAIlD,QAAA,kBAIG;EACD,MAAM,kBAAkB,MAAA;EACxB,MAAM,SAAS,MAAA,MAAY;EAC3B,IAAI;AACJ,MAAI;AACF,UAAO,MAAM,MAAM,iBAAiB,EAAC,QAAO,CAAC;WACtC,GAAG;AACV,OAAI,OAAO,QAET;AAIF,SAAA,GAAS,OAAO,8BAA8B,MAAA,mBAAyB,EAAE;AACzE;;AAEF,MAAI,CAAC,KAAK,IAAI;AACZ,SAAA,GAAS,OACP,8BAA8B,MAAA,gBAAsB,IAAI,MAAM,KAAK,MAAM,GAC1E;AACD;;EAGF,MAAM,WAAW,0BAA0B,MAAM,KAAK,MAAM,CAAC;AAC7D,OAAK,MAAM,UAAU,SACnB,KACE,OAAO,SAAS,WAChB,OAAO,SAAS,8BAEhB,MAAK,MAAM,UAAU,OAAO,SAAS;GACnC,MAAM,YAAY,OAAO,QAAQ;GACjC,MAAM,OAAO,OAAO,QAAQ;GAC5B,MAAM,uBAAO,IAAI,KAAK,WAAW,OAAO,MAAM,GAAG,IAAK;AAEtD,OAAI,UACF,OAAM;IAAC;IAAW;IAAM;IAAK;;;CAOvC,OAAA,kBAAyB;AACvB,aAAW,MAAM,EAAC,WAAW,MAAM,UAAS,MAAA,iBAAuB,CACjE,KAAI,YAAY,MAAA,iBAAuB,CAAC,MAAA,WAAiB,IAAI,UAAU,EAAE;AACvE,SAAA,GAAS,OACP,wBAAwB,UAAU,MAAM,KAAA,MAC/B,KAAK,aAAa,CAAC,GAC7B;AACD,SAAA,WAAiB,IAAI,WAAW,KAAK;AACrC,SAAA,mBAAyB;;AAG7B,SAAO,MAAA;;CAGT,mBAAmB;AACjB,MAAI,MAAA,aAAmB,OAAO,GAAG;AAC/B,SAAA,GAAS,OACP,6CAA6C,CAAC,GAAG,MAAA,aAAmB,MAAM,CAAC,GAC5E;AACD;;EAEF,MAAM,oBAAoB,KAAK,KAAK,GAAG,MAAA;EACvC,IAAI,eAAe;AACnB,OAAK,MAAM,CAAC,WAAW,eAAe,MAAA,WAAiB,SAAS,CAC9D,KACE,WAAW,SAAS,IAAI,qBACxB,YAAY,aAEZ,gBAAe;AAGnB,MAAI,aAAa,QAAQ;AACvB,SAAA,eAAqB,gBAAgB,aAAa;AAClD,QAAK,MAAM,aAAa,MAAA,WAAiB,MAAM,CAC7C,KAAI,aAAa,aACf,OAAA,WAAiB,OAAO,UAAU;AAGtC,SAAA,gBAAsB;;;CAI1B,OAAsB;AACpB,gBAAc,MAAA,kBAAwB;AACtC,OAAK,MAAM,EAAC,SAAQ,MAAA,aAAmB,QAAQ,CAK7C,KAAI,QAAQ;AAEd,QAAA,MAAY,KAAK,MAAA,GAAS;AAC1B,SAAO;;CAGT,uBAAuB;AACrB,mBAAiB,WAAW,cAAc;GACxC,aACE;GAGF,MAAM;GACP,CAAC,CAAC,YAAY,OAAM,MAAK;GASxB,MAAM,eAAe,MAAM,MAAA,iBAAuB;AAClD,OAAI,CAAC,cAAc;AACjB,UAAA,GAAS,OACP,+DACD;AACD;;GAEF,MAAM,KAAK,IAAI,SAAS,MAAA,IAAU,MAAA,aAAmB,EAAC,UAAU,MAAK,CAAC;AACtE,OAAI;IACF,MAAM,EAAC,gBAAe,GACnB,QAAgB,mDAAmD,CACnE,KAA4B;IAC/B,MAAM,YAAY,KAAK,IAAI,GAAG,cAAc,aAAa,SAAS,CAAC;AACnE,MAAE,QAAQ,UAAU;YACb,GAAG;AACV,UAAA,GAAS,OAAO,6CAA6C,EAAE;aACvD;AACR,OAAG,OAAO;;IAEZ"}

package/out/zero-cache/src/services/change-streamer/broadcast.js.map

@@ -1 +1 @@
-{"version":3,"file":"broadcast.js","names":["#pending","#completed","#done","#watermark","#majority","#start","#markCompleted","#setDone","#latestCompleted","#isDone","#logWithState"],"sources":["../../../../../../zero-cache/src/services/change-streamer/broadcast.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {resolver} from '@rocicorp/resolver';\nimport type {WatermarkedChange} from './change-streamer-service.ts';\nimport type {Subscriber} from './subscriber.ts';\n\n/**\n * Initiates and tracks the progress of a change broadcasted to\n * a set of subscribers.\n *\n * Creating a `Broadcast` automatically initiates the send.\n *\n * By default, {@link Broadcast.done} resolves when all subscribers\n * have acked the change. However, {@link Broadcast.checkProgress()}\n * can be called to resolve the broadcast earlier based on the flow\n * control policy.\n */\nexport class Broadcast {\n  /**\n   * Sends the change to the subscribers without the tracking machinery.\n   * This is suitable for fire-and-forget (i.e. pipelined) sends.\n   */\n  static withoutTracking(\n    subscribers: Iterable<Subscriber>,\n    change: WatermarkedChange,\n  ) {\n    for (const sub of subscribers) {\n      void sub.send(change);\n    }\n  }\n\n  readonly #pending: Set<Subscriber>;\n  readonly #completed: Completed[];\n  readonly #done = resolver();\n  #isDone = false;\n\n  readonly #watermark: string;\n  readonly #majority: number;\n\n  readonly #start = performance.now();\n  #latestCompleted = Number.MAX_VALUE;\n\n  /**\n   * Broadcasts the `change` to the `subscribers` and tracks their\n   * completion.\n   */\n  constructor(subscribers: Iterable<Subscriber>, change: WatermarkedChange) {\n    this.#pending = new Set(subscribers);\n    this.#completed = [];\n    this.#watermark = change[0];\n    this.#majority = Math.floor(this.#pending.size / 2) + 1;\n\n    for (const sub of this.#pending) {\n      const changes = sub.numPending + 1; // add one for this `change`\n      void sub\n        .send(change)\n        .catch(() => {})\n        .finally(() => this.#markCompleted(sub, changes));\n    }\n\n    // set done if there are no subscribers (mainly for tests)\n    if (this.#pending.size === 0) {\n      this.#setDone();\n    }\n  }\n\n  #markCompleted(sub: Subscriber, changes: number) {\n    const elapsed = (this.#latestCompleted = performance.now()) - this.#start;\n    this.#completed.push({sub, changes, elapsed});\n    this.#pending.delete(sub);\n    if (this.#pending.size === 0) {\n      this.#setDone();\n    }\n  }\n\n  #setDone() {\n    this.#isDone = true;\n    this.#done.resolve();\n  }\n\n  get isDone(): boolean {\n    return this.#isDone;\n  }\n\n  get done(): Promise<void> {\n    return this.#done.promise;\n  }\n\n  /**\n   * Checks for pathological situations in which flow should be reenabled\n   * before all subscribers have acked.\n   *\n   * ### Background\n   *\n   * The purpose of flow control is to pull upstream replication changes\n   * no faster than the rate as they are processed by downstream subscribers\n   * in the steady state. In the change-streamer, this is done by occasionally\n   * waiting for ACKs from subscribers before continuing; without doing so,\n   * I/O buffers fill up and cause the system to spend most of its time in GC.\n   *\n   * However, the naive algorithm of always waiting for all subscribers (e.g.\n   * `Promise.all()`) can behave poorly in scenarios where subscribers\n   * are imbalanced:\n   * * New subscribers may have a backlog of changes to catch up with.\n   *   Having all subscribers wait for the new subscriber to catch up results\n   *   in delaying the entire application.\n   * * Broken TCP connections similarly require all subscribers to wait until\n   *   connection liveness checks kick in and disconnect the subscriber.\n   *\n   * A simplistic approach is to add a limit to the amount of time waiting for\n   * subscribers, i.e. an ack timeout. However, deciding what this timeout\n   * should be is non-trivial because of the heterogeneous nature of changes;\n   * while most changes operate on single rows and are relatively predictable\n   * in terms of running time, some changes are table-wide operations and can\n   * legitimately take an arbitrary amount of time. In such scenarios, a\n   * timeout that is too short can stop progress on replication altogether.\n   *\n   * ### Consensus-based Timeout Algorithm\n   *\n   * To address these shortcomings, a \"consensus-based timeout\" algorithm is\n   * used:\n   * * Wait for more than half of the subscribers to finish. (In\n   *   case of a single node, or the case of one replication-manager\n   *   and one view-syncer, this reduces to waiting for all subscribers.)\n   * * Once more than half of the subscribers have finished, proceed after\n   *   a fixed timeout elapses (e.g. 1 second), even if not all subscribers\n   *   have finished.\n   *\n   * In other words, the subscribers themselves are used to determine the\n   * timeout of each batch of changes; the majority determines this when\n   * they complete, upon which a timeout is logically started.\n   *\n   * In the common case, the remaining subscribers finish soon afterward and\n   * the timeout never elapses. However, in pathological cases where a minority\n   * of subscribers have a disproportionate amount of load, some will still\n   * be processing (or otherwise unresponsive). These subscribers are given\n   * a bounded amount of time to catch up at each flushed batch, up to the\n   * timeout interval. This guarantees eventual catchup because the\n   * subscribers with a backlog of changes necessarily have a higher\n   * processing rate than the subscribers that finished (and are made to wait).\n   *\n   * ### Not implemented: Broken connection detection\n   *\n   * If a subscriber has not made progress for a certain interval, the\n   * algorithm could theoretically drop it preemptively, supplementing the\n   * existing websocket-level liveness checks.\n   *\n   * However, a more reliable approach would be to change the replicator\n   * to use non-blocking writes, and subsequently increase the frequency of\n   * connection-level liveness checks. The current synchronous replica writes\n   * can delay both ping responsiveness and change progress arbitrarily (e.g.\n   * a large index creation); an independently liveness check that is not\n   * delayed by synchronous writes on the subscriber would be a more failsafe\n   * solution.\n   *\n   * @returns `true` if the broadcast was already done or was marked done.\n   */\n  checkProgress(\n    lc: LogContext,\n    flowControlConsensusPaddingMs: number,\n    now: number,\n  ) {\n    if (this.#pending.size === 0) {\n      return true;\n    }\n    const elapsed = now - this.#start;\n    if (this.#completed.length < this.#majority) {\n      if (elapsed >= 1000) {\n        this.#logWithState(\n          lc,\n          `waiting for at least ${this.#majority} subscribers to finish`,\n          elapsed,\n        );\n      }\n      return false;\n    }\n    // Note: In the implementation, #latestCompleted is always updated,\n    // even after the majority is reached. This is fine and does not affect\n    // the important properties of the algorithm.\n    if (now - this.#latestCompleted >= flowControlConsensusPaddingMs) {\n      this.#logWithState(\n        lc,\n        `continuing with ${this.#pending.size} subscriber(s) still pending`,\n        elapsed,\n      );\n      this.#setDone();\n      return true;\n    }\n    return false;\n  }\n\n  #logWithState(lc: LogContext, msg: string, elapsed: number) {\n    lc.withContext('watermark', this.#watermark).info?.(\n      `${msg} (${elapsed.toFixed(3)} ms)`,\n      {\n        completed: this.#completed.map(d => ({\n          id: d.sub.id,\n          processed: d.changes,\n          elapsed: d.elapsed,\n        })),\n        pending: Array.from(this.#pending, sub => ({\n          id: sub.id,\n          ...sub.getStats(),\n        })),\n      },\n    );\n  }\n}\n\n/** Tracks the completed result of a single subscriber. */\ntype Completed = {\n  sub: Subscriber;\n  /** The number of changes processed. */\n  changes: number;\n  /** The elapsed milliseconds. */\n  elapsed: number;\n};\n"],"mappings":";;;;;;;;;;;;;AAgBA,IAAa,YAAb,MAAuB;;;;;CAKrB,OAAO,gBACL,aACA,QACA;EACA,KAAK,MAAM,OAAO,aAChB,IAAS,KAAK,MAAM;CAExB;CAEA;CACA;CACA,QAAiB,SAAS;CAC1B,UAAU;CAEV;CACA;CAEA,SAAkB,YAAY,IAAI;CAClC,mBAAmB,OAAO;;;;;CAM1B,YAAY,aAAmC,QAA2B;EACxE,KAAKA,WAAW,IAAI,IAAI,WAAW;EACnC,KAAKC,aAAa,CAAC;EACnB,KAAKE,aAAa,OAAO;EACzB,KAAKC,YAAY,KAAK,MAAM,KAAKJ,SAAS,OAAO,CAAC,IAAI;EAEtD,KAAK,MAAM,OAAO,KAAKA,UAAU;GAC/B,MAAM,UAAU,IAAI,aAAa;GACjC,IACG,KAAK,MAAM,EACX,YAAY,CAAC,CAAC,EACd,cAAc,KAAKM,eAAe,KAAK,OAAO,CAAC;EACpD;EAGA,IAAI,KAAKN,SAAS,SAAS,GACzB,KAAKO,SAAS;CAElB;CAEA,eAAe,KAAiB,SAAiB;EAC/C,MAAM,WAAW,KAAKC,mBAAmB,YAAY,IAAI,KAAK,KAAKH;EACnE,KAAKJ,WAAW,KAAK;GAAC;GAAK;GAAS;EAAO,CAAC;EAC5C,KAAKD,SAAS,OAAO,GAAG;EACxB,IAAI,KAAKA,SAAS,SAAS,GACzB,KAAKO,SAAS;CAElB;CAEA,WAAW;EACT,KAAKE,UAAU;EACf,KAAKP,MAAM,QAAQ;CACrB;CAEA,IAAI,SAAkB;EACpB,OAAO,KAAKO;CACd;CAEA,IAAI,OAAsB;EACxB,OAAO,KAAKP,MAAM;CACpB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAuEA,cACE,IACA,+BACA,KACA;EACA,IAAI,KAAKF,SAAS,SAAS,GACzB,OAAO;EAET,MAAM,UAAU,MAAM,KAAKK;EAC3B,IAAI,KAAKJ,WAAW,SAAS,KAAKG,WAAW;GAC3C,IAAI,WAAW,KACb,KAAKM,cACH,IACA,wBAAwB,KAAKN,UAAU,yBACvC,OACF;GAEF,OAAO;EACT;EAIA,IAAI,MAAM,KAAKI,oBAAoB,+BAA+B;GAChE,KAAKE,cACH,IACA,mBAAmB,KAAKV,SAAS,KAAK,+BACtC,OACF;GACA,KAAKO,SAAS;GACd,OAAO;EACT;EACA,OAAO;CACT;CAEA,cAAc,IAAgB,KAAa,SAAiB;EAC1D,GAAG,YAAY,aAAa,KAAKJ,UAAU,EAAE,OAC3C,GAAG,IAAI,IAAI,QAAQ,QAAQ,CAAC,EAAE,OAC9B;GACE,WAAW,KAAKF,WAAW,KAAI,OAAM;IACnC,IAAI,EAAE,IAAI;IACV,WAAW,EAAE;IACb,SAAS,EAAE;GACb,EAAE;GACF,SAAS,MAAM,KAAK,KAAKD,WAAU,SAAQ;IACzC,IAAI,IAAI;IACR,GAAG,IAAI,SAAS;GAClB,EAAE;EACJ,CACF;CACF;AACF"}
+{"version":3,"file":"broadcast.js","names":["#pending","#completed","#done","#watermark","#majority","#start","#markCompleted","#setDone","#latestCompleted","#isDone","#logWithState"],"sources":["../../../../../../zero-cache/src/services/change-streamer/broadcast.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {resolver} from '@rocicorp/resolver';\nimport type {WatermarkedChange} from './change-streamer-service.ts';\nimport type {Subscriber} from './subscriber.ts';\n\n/**\n * Initiates and tracks the progress of a change broadcasted to\n * a set of subscribers.\n *\n * Creating a `Broadcast` automatically initiates the send.\n *\n * By default, {@link Broadcast.done} resolves when all subscribers\n * have acked the change. However, {@link Broadcast.checkProgress()}\n * can be called to resolve the broadcast earlier based on the flow\n * control policy.\n */\nexport class Broadcast {\n  /**\n   * Sends the change to the subscribers without the tracking machinery.\n   * This is suitable for fire-and-forget (i.e. pipelined) sends.\n   */\n  static withoutTracking(\n    subscribers: Iterable<Subscriber>,\n    change: WatermarkedChange,\n  ) {\n    for (const sub of subscribers) {\n      void sub.send(change);\n    }\n  }\n\n  readonly #pending: Set<Subscriber>;\n  readonly #completed: Completed[];\n  readonly #done = resolver();\n  #isDone = false;\n\n  readonly #watermark: string;\n  readonly #majority: number;\n\n  readonly #start = performance.now();\n  #latestCompleted = Number.MAX_VALUE;\n\n  /**\n   * Broadcasts the `change` to the `subscribers` and tracks their\n   * completion.\n   */\n  constructor(subscribers: Iterable<Subscriber>, change: WatermarkedChange) {\n    this.#pending = new Set(subscribers);\n    this.#completed = [];\n    this.#watermark = change[0];\n    this.#majority = Math.floor(this.#pending.size / 2) + 1;\n\n    for (const sub of this.#pending) {\n      const changes = sub.numPending + 1; // add one for this `change`\n      void sub\n        .send(change)\n        .catch(() => {})\n        .finally(() => this.#markCompleted(sub, changes));\n    }\n\n    // set done if there are no subscribers (mainly for tests)\n    if (this.#pending.size === 0) {\n      this.#setDone();\n    }\n  }\n\n  #markCompleted(sub: Subscriber, changes: number) {\n    const elapsed = (this.#latestCompleted = performance.now()) - this.#start;\n    this.#completed.push({sub, changes, elapsed});\n    this.#pending.delete(sub);\n    if (this.#pending.size === 0) {\n      this.#setDone();\n    }\n  }\n\n  #setDone() {\n    this.#isDone = true;\n    this.#done.resolve();\n  }\n\n  get isDone(): boolean {\n    return this.#isDone;\n  }\n\n  get done(): Promise<void> {\n    return this.#done.promise;\n  }\n\n  /**\n   * Checks for pathological situations in which flow should be reenabled\n   * before all subscribers have acked.\n   *\n   * ### Background\n   *\n   * The purpose of flow control is to pull upstream replication changes\n   * no faster than the rate as they are processed by downstream subscribers\n   * in the steady state. In the change-streamer, this is done by occasionally\n   * waiting for ACKs from subscribers before continuing; without doing so,\n   * I/O buffers fill up and cause the system to spend most of its time in GC.\n   *\n   * However, the naive algorithm of always waiting for all subscribers (e.g.\n   * `Promise.all()`) can behave poorly in scenarios where subscribers\n   * are imbalanced:\n   * * New subscribers may have a backlog of changes to catch up with.\n   *   Having all subscribers wait for the new subscriber to catch up results\n   *   in delaying the entire application.\n   * * Broken TCP connections similarly require all subscribers to wait until\n   *   connection liveness checks kick in and disconnect the subscriber.\n   *\n   * A simplistic approach is to add a limit to the amount of time waiting for\n   * subscribers, i.e. an ack timeout. However, deciding what this timeout\n   * should be is non-trivial because of the heterogeneous nature of changes;\n   * while most changes operate on single rows and are relatively predictable\n   * in terms of running time, some changes are table-wide operations and can\n   * legitimately take an arbitrary amount of time. In such scenarios, a\n   * timeout that is too short can stop progress on replication altogether.\n   *\n   * ### Consensus-based Timeout Algorithm\n   *\n   * To address these shortcomings, a \"consensus-based timeout\" algorithm is\n   * used:\n   * * Wait for more than half of the subscribers to finish. (In\n   *   case of a single node, or the case of one replication-manager\n   *   and one view-syncer, this reduces to waiting for all subscribers.)\n   * * Once more than half of the subscribers have finished, proceed after\n   *   a fixed timeout elapses (e.g. 1 second), even if not all subscribers\n   *   have finished.\n   *\n   * In other words, the subscribers themselves are used to determine the\n   * timeout of each batch of changes; the majority determines this when\n   * they complete, upon which a timeout is logically started.\n   *\n   * In the common case, the remaining subscribers finish soon afterward and\n   * the timeout never elapses. However, in pathological cases where a minority\n   * of subscribers have a disproportionate amount of load, some will still\n   * be processing (or otherwise unresponsive). These subscribers are given\n   * a bounded amount of time to catch up at each flushed batch, up to the\n   * timeout interval. This guarantees eventual catchup because the\n   * subscribers with a backlog of changes necessarily have a higher\n   * processing rate than the subscribers that finished (and are made to wait).\n   *\n   * ### Not implemented: Broken connection detection\n   *\n   * If a subscriber has not made progress for a certain interval, the\n   * algorithm could theoretically drop it preemptively, supplementing the\n   * existing websocket-level liveness checks.\n   *\n   * However, a more reliable approach would be to change the replicator\n   * to use non-blocking writes, and subsequently increase the frequency of\n   * connection-level liveness checks. The current synchronous replica writes\n   * can delay both ping responsiveness and change progress arbitrarily (e.g.\n   * a large index creation); an independently liveness check that is not\n   * delayed by synchronous writes on the subscriber would be a more failsafe\n   * solution.\n   *\n   * @returns `true` if the broadcast was already done or was marked done.\n   */\n  checkProgress(\n    lc: LogContext,\n    flowControlConsensusPaddingMs: number,\n    now: number,\n  ) {\n    if (this.#pending.size === 0) {\n      return true;\n    }\n    const elapsed = now - this.#start;\n    if (this.#completed.length < this.#majority) {\n      if (elapsed >= 1000) {\n        this.#logWithState(\n          lc,\n          `waiting for at least ${this.#majority} subscribers to finish`,\n          elapsed,\n        );\n      }\n      return false;\n    }\n    // Note: In the implementation, #latestCompleted is always updated,\n    // even after the majority is reached. This is fine and does not affect\n    // the important properties of the algorithm.\n    if (now - this.#latestCompleted >= flowControlConsensusPaddingMs) {\n      this.#logWithState(\n        lc,\n        `continuing with ${this.#pending.size} subscriber(s) still pending`,\n        elapsed,\n      );\n      this.#setDone();\n      return true;\n    }\n    return false;\n  }\n\n  #logWithState(lc: LogContext, msg: string, elapsed: number) {\n    lc.withContext('watermark', this.#watermark).info?.(\n      `${msg} (${elapsed.toFixed(3)} ms)`,\n      {\n        completed: this.#completed.map(d => ({\n          id: d.sub.id,\n          processed: d.changes,\n          elapsed: d.elapsed,\n        })),\n        pending: Array.from(this.#pending, sub => ({\n          id: sub.id,\n          ...sub.getStats(),\n        })),\n      },\n    );\n  }\n}\n\n/** Tracks the completed result of a single subscriber. */\ntype Completed = {\n  sub: Subscriber;\n  /** The number of changes processed. */\n  changes: number;\n  /** The elapsed milliseconds. */\n  elapsed: number;\n};\n"],"mappings":";;;;;;;;;;;;;AAgBA,IAAa,YAAb,MAAuB;;;;;CAKrB,OAAO,gBACL,aACA,QACA;AACA,OAAK,MAAM,OAAO,YACX,KAAI,KAAK,OAAO;;CAIzB;CACA;CACA,QAAiB,UAAU;CAC3B,UAAU;CAEV;CACA;CAEA,SAAkB,YAAY,KAAK;CACnC,mBAAmB,OAAO;;;;;CAM1B,YAAY,aAAmC,QAA2B;AACxE,QAAA,UAAgB,IAAI,IAAI,YAAY;AACpC,QAAA,YAAkB,EAAE;AACpB,QAAA,YAAkB,OAAO;AACzB,QAAA,WAAiB,KAAK,MAAM,MAAA,QAAc,OAAO,EAAE,GAAG;AAEtD,OAAK,MAAM,OAAO,MAAA,SAAe;GAC/B,MAAM,UAAU,IAAI,aAAa;AAC5B,OACF,KAAK,OAAO,CACZ,YAAY,GAAG,CACf,cAAc,MAAA,cAAoB,KAAK,QAAQ,CAAC;;AAIrD,MAAI,MAAA,QAAc,SAAS,EACzB,OAAA,SAAe;;CAInB,eAAe,KAAiB,SAAiB;EAC/C,MAAM,WAAW,MAAA,kBAAwB,YAAY,KAAK,IAAI,MAAA;AAC9D,QAAA,UAAgB,KAAK;GAAC;GAAK;GAAS;GAAQ,CAAC;AAC7C,QAAA,QAAc,OAAO,IAAI;AACzB,MAAI,MAAA,QAAc,SAAS,EACzB,OAAA,SAAe;;CAInB,WAAW;AACT,QAAA,SAAe;AACf,QAAA,KAAW,SAAS;;CAGtB,IAAI,SAAkB;AACpB,SAAO,MAAA;;CAGT,IAAI,OAAsB;AACxB,SAAO,MAAA,KAAW;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAwEpB,cACE,IACA,+BACA,KACA;AACA,MAAI,MAAA,QAAc,SAAS,EACzB,QAAO;EAET,MAAM,UAAU,MAAM,MAAA;AACtB,MAAI,MAAA,UAAgB,SAAS,MAAA,UAAgB;AAC3C,OAAI,WAAW,IACb,OAAA,aACE,IACA,wBAAwB,MAAA,SAAe,yBACvC,QACD;AAEH,UAAO;;AAKT,MAAI,MAAM,MAAA,mBAAyB,+BAA+B;AAChE,SAAA,aACE,IACA,mBAAmB,MAAA,QAAc,KAAK,+BACtC,QACD;AACD,SAAA,SAAe;AACf,UAAO;;AAET,SAAO;;CAGT,cAAc,IAAgB,KAAa,SAAiB;AAC1D,KAAG,YAAY,aAAa,MAAA,UAAgB,CAAC,OAC3C,GAAG,IAAI,IAAI,QAAQ,QAAQ,EAAE,CAAC,OAC9B;GACE,WAAW,MAAA,UAAgB,KAAI,OAAM;IACnC,IAAI,EAAE,IAAI;IACV,WAAW,EAAE;IACb,SAAS,EAAE;IACZ,EAAE;GACH,SAAS,MAAM,KAAK,MAAA,UAAe,SAAQ;IACzC,IAAI,IAAI;IACR,GAAG,IAAI,UAAU;IAClB,EAAE;GACJ,CACF"}

package/out/zero-cache/src/services/change-streamer/change-streamer-http.js.map

@@ -1 +1 @@
-{"version":3,"file":"change-streamer-http.js","names":["#lc","#opts","#changeStreamer","#backupMonitor","#subscribe","#reserveSnapshot","#receiveWebsocket","#getBackupMonitor","#ensureChangeStreamerStarted","#changeStreamerStarted","#shardID","#changeDB","#changeStreamerURI","#resolveChangeStreamer"],"sources":["../../../../../../zero-cache/src/services/change-streamer/change-streamer-http.ts"],"sourcesContent":["import type {IncomingMessage} from 'node:http';\nimport websocket from '@fastify/websocket';\nimport type {LogContext} from '@rocicorp/logger';\nimport WebSocket from 'ws';\nimport {assert} from '../../../../shared/src/asserts.ts';\nimport {must} from '../../../../shared/src/must.ts';\nimport type {IncomingMessageSubset} from '../../types/http.ts';\nimport {pgClient, type PostgresDB} from '../../types/pg.ts';\nimport {type Worker} from '../../types/processes.ts';\nimport {type ShardID} from '../../types/shards.ts';\nimport {\n  streamIn,\n  streamOut,\n  streamOutStringified,\n  type Source,\n} from '../../types/streams.ts';\nimport {URLParams} from '../../types/url-params.ts';\nimport {installWebSocketReceiver} from '../../types/websocket-handoff.ts';\nimport {closeWithError, PROTOCOL_ERROR} from '../../types/ws.ts';\nimport {HttpService} from '../http-service.ts';\nimport type {BackupMonitor} from './backup-monitor.ts';\nimport {\n  downstreamSchema,\n  PROTOCOL_VERSION,\n  type ChangeStreamer,\n  type ChangeStreamerService,\n  type Downstream,\n  type SubscriberContext,\n} from './change-streamer.ts';\nimport {discoverChangeStreamerAddress} from './schema/tables.ts';\nimport {snapshotMessageSchema, type SnapshotMessage} from './snapshot.ts';\n\nconst MIN_SUPPORTED_PROTOCOL_VERSION = 1;\n\nconst SNAPSHOT_PATH_PATTERN = '/replication/:version/snapshot';\nconst CHANGES_PATH_PATTERN = '/replication/:version/changes';\nconst PATH_REGEX = /\\/replication\\/v(?<version>\\d+)\\/(changes|snapshot)$/;\n\nconst SNAPSHOT_PATH = `/replication/v${PROTOCOL_VERSION}/snapshot`;\nconst CHANGES_PATH = `/replication/v${PROTOCOL_VERSION}/changes`;\n\ntype Options = {\n  port: number;\n  keepaliveTimeoutMs: number | undefined;\n  startupDelayMs: number;\n};\n\nexport class ChangeStreamerHttpServer extends HttpService {\n  readonly id = 'change-streamer-http-server';\n  readonly #lc: LogContext;\n  readonly #opts: Options;\n  readonly #changeStreamer: ChangeStreamerService;\n  readonly #backupMonitor: BackupMonitor | null;\n\n  constructor(\n    lc: LogContext,\n    opts: Options,\n    parent: Worker,\n    changeStreamer: ChangeStreamerService,\n    backupMonitor: BackupMonitor | null,\n  ) {\n    super('change-streamer-http-server', lc, opts, async fastify => {\n      await fastify.register(websocket);\n\n      fastify.get(CHANGES_PATH_PATTERN, {websocket: true}, this.#subscribe);\n      fastify.get(\n        SNAPSHOT_PATH_PATTERN,\n        {websocket: true},\n        this.#reserveSnapshot,\n      );\n\n      installWebSocketReceiver<'snapshot' | 'changes'>(\n        lc,\n        fastify.websocketServer,\n        this.#receiveWebsocket,\n        parent,\n      );\n    });\n\n    this.#lc = lc;\n    this.#opts = opts;\n    this.#changeStreamer = changeStreamer;\n    this.#backupMonitor = backupMonitor;\n  }\n\n  #getBackupMonitor() {\n    return must(\n      this.#backupMonitor,\n      'replication-manager is not configured with a ZERO_LITESTREAM_BACKUP_URL',\n    );\n  }\n\n  // Called when receiving a web socket via the main dispatcher handoff.\n  readonly #receiveWebsocket = (\n    ws: WebSocket,\n    action: 'changes' | 'snapshot',\n    msg: IncomingMessageSubset,\n  ) => {\n    switch (action) {\n      case 'snapshot':\n        return this.#reserveSnapshot(ws, msg);\n      case 'changes':\n        return this.#subscribe(ws, msg);\n      default:\n        closeWithError(\n          this._lc,\n          ws,\n          `invalid action \"${action}\" received in handoff`,\n        );\n        return;\n    }\n  };\n\n  readonly #reserveSnapshot = (ws: WebSocket, req: RequestHeaders) => {\n    try {\n      const url = new URL(\n        req.url ?? '',\n        req.headers.origin ?? 'http://localhost',\n      );\n      checkProtocolVersion(url.pathname);\n      const taskID = url.searchParams.get('taskID');\n      if (!taskID) {\n        throw new Error('Missing taskID in snapshot request');\n      }\n      const downstream =\n        this.#getBackupMonitor().startSnapshotReservation(taskID);\n      void streamOut(this._lc, downstream, ws);\n    } catch (err) {\n      closeWithError(this._lc, ws, err, PROTOCOL_ERROR);\n    }\n  };\n\n  readonly #subscribe = async (ws: WebSocket, req: RequestHeaders) => {\n    try {\n      const ctx = getSubscriberContext(req);\n      if (ctx.mode === 'serving') {\n        this.#ensureChangeStreamerStarted('incoming subscription');\n      }\n\n      const downstream = await this.#changeStreamer.subscribe(ctx);\n      if (ctx.initial && ctx.taskID && this.#backupMonitor) {\n        // Now that the change-streamer knows about the subscriber and watermark,\n        // end the reservation to safely resume scheduling cleanup.\n        this.#backupMonitor.endReservation(ctx.taskID);\n      }\n      void streamOutStringified(this._lc, downstream, ws);\n    } catch (err) {\n      closeWithError(this._lc, ws, err, PROTOCOL_ERROR);\n    }\n  };\n\n  #changeStreamerStarted = false;\n\n  #ensureChangeStreamerStarted(reason: string) {\n    if (!this.#changeStreamerStarted && this._state.shouldRun()) {\n      this.#lc.info?.(`starting ChangeStreamerService: ${reason}`);\n      void this.#changeStreamer\n        .run()\n        .catch(e =>\n          this.#lc.warn?.(`ChangeStreamerService ended with error`, e),\n        )\n        .finally(() => this.stop());\n\n      this.#changeStreamerStarted = true;\n    }\n  }\n\n  protected override _onStart(): void {\n    const {startupDelayMs} = this.#opts;\n    this._state.setTimeout(\n      () =>\n        this.#ensureChangeStreamerStarted(\n          `startup delay elapsed (${startupDelayMs} ms)`,\n        ),\n      startupDelayMs,\n    );\n  }\n\n  protected override async _onStop(): Promise<void> {\n    if (this.#changeStreamerStarted) {\n      await this.#changeStreamer.stop();\n    }\n  }\n}\n\nexport class ChangeStreamerHttpClient implements ChangeStreamer {\n  readonly #lc: LogContext;\n  readonly #shardID: ShardID;\n  readonly #changeDB: PostgresDB;\n  readonly #changeStreamerURI: string | undefined;\n\n  constructor(\n    lc: LogContext,\n    shardID: ShardID,\n    changeDB: string,\n    changeStreamerURI: string | undefined,\n  ) {\n    this.#lc = lc;\n    this.#shardID = shardID;\n    // Create a pg client with a single short-lived connection for the purpose\n    // of change-streamer discovery (i.e. ChangeDB as DNS).\n    this.#changeDB = pgClient(lc, changeDB, 'change-streamer-discovery', {\n      max: 1,\n      ['idle_timeout']: 15,\n    });\n    this.#changeStreamerURI = changeStreamerURI;\n  }\n\n  async #resolveChangeStreamer(path: string) {\n    let baseURL = this.#changeStreamerURI;\n    if (!baseURL) {\n      const address = await discoverChangeStreamerAddress(\n        this.#shardID,\n        this.#changeDB,\n      );\n      if (!address) {\n        throw new Error(`no change-streamer is running`);\n      }\n      baseURL = address.includes('://') ? `${address}/` : `ws://${address}/`;\n    }\n    const uri = new URL(path, baseURL);\n    this.#lc.info?.(`connecting to change-streamer@${uri}`);\n    return uri;\n  }\n\n  async reserveSnapshot(taskID: string): Promise<Source<SnapshotMessage>> {\n    const uri = await this.#resolveChangeStreamer(SNAPSHOT_PATH);\n\n    const params = new URLSearchParams({taskID});\n    const ws = new WebSocket(uri + `?${params.toString()}`);\n\n    return streamIn(this.#lc, ws, snapshotMessageSchema);\n  }\n\n  async subscribe(ctx: SubscriberContext): Promise<Source<Downstream>> {\n    const uri = await this.#resolveChangeStreamer(CHANGES_PATH);\n\n    const params = getParams(ctx);\n    const ws = new WebSocket(uri + `?${params.toString()}`);\n\n    return streamIn(this.#lc, ws, downstreamSchema);\n  }\n}\n\ntype RequestHeaders = Pick<IncomingMessage, 'url' | 'headers'>;\n\nexport function getSubscriberContext(req: RequestHeaders): SubscriberContext {\n  const url = new URL(req.url ?? '', req.headers.origin ?? 'http://localhost');\n  const protocolVersion = checkProtocolVersion(url.pathname);\n  const params = new URLParams(url);\n\n  return {\n    protocolVersion,\n    id: params.get('id', true),\n    taskID: params.get('taskID', false),\n    mode: params.get('mode', false) === 'backup' ? 'backup' : 'serving',\n    replicaVersion: params.get('replicaVersion', true),\n    watermark: params.get('watermark', true),\n    initial: params.getBoolean('initial'),\n  };\n}\n\nfunction checkProtocolVersion(pathname: string): number {\n  const match = PATH_REGEX.exec(pathname);\n  if (!match) {\n    throw new Error(`invalid path: ${pathname}`);\n  }\n  const v = Number(match.groups?.version);\n  if (\n    Number.isNaN(v) ||\n    v > PROTOCOL_VERSION ||\n    v < MIN_SUPPORTED_PROTOCOL_VERSION\n  ) {\n    throw new Error(\n      `Cannot service client at protocol v${v}. ` +\n        `Supported protocols: [v${MIN_SUPPORTED_PROTOCOL_VERSION} ... v${PROTOCOL_VERSION}]`,\n    );\n  }\n  return v;\n}\n\n// This is called from the client-side (i.e. the replicator).\nfunction getParams(ctx: SubscriberContext): URLSearchParams {\n  // The protocolVersion is hard-coded into the CHANGES_PATH.\n  const {protocolVersion, ...stringParams} = ctx;\n  assert(\n    protocolVersion === PROTOCOL_VERSION,\n    `replicator should be setting protocolVersion to ${PROTOCOL_VERSION}`,\n  );\n  return new URLSearchParams({\n    ...stringParams,\n    taskID: ctx.taskID ? ctx.taskID : '',\n    initial: ctx.initial ? 'true' : 'false',\n  });\n}\n"],"mappings":";;;;;;;;;;;;;;;AAgCA,IAAM,iCAAiC;AAEvC,IAAM,wBAAwB;AAC9B,IAAM,uBAAuB;AAC7B,IAAM,aAAa;AAEnB,IAAM,gBAAgB;AACtB,IAAM,eAAe;AAQrB,IAAa,2BAAb,cAA8C,YAAY;CACxD,KAAc;CACd;CACA;CACA;CACA;CAEA,YACE,IACA,MACA,QACA,gBACA,eACA;EACA,MAAM,+BAA+B,IAAI,MAAM,OAAM,YAAW;GAC9D,MAAM,QAAQ,SAAS,SAAS;GAEhC,QAAQ,IAAI,sBAAsB,EAAC,WAAW,KAAI,GAAG,KAAKI,UAAU;GACpE,QAAQ,IACN,uBACA,EAAC,WAAW,KAAI,GAChB,KAAKC,gBACP;GAEA,yBACE,IACA,QAAQ,iBACR,KAAKC,mBACL,MACF;EACF,CAAC;EAED,KAAKN,MAAM;EACX,KAAKC,QAAQ;EACb,KAAKC,kBAAkB;EACvB,KAAKC,iBAAiB;CACxB;CAEA,oBAAoB;EAClB,OAAO,KACL,KAAKA,gBACL,yEACF;CACF;CAGA,qBACE,IACA,QACA,QACG;EACH,QAAQ,QAAR;GACE,KAAK,YACH,OAAO,KAAKE,iBAAiB,IAAI,GAAG;GACtC,KAAK,WACH,OAAO,KAAKD,WAAW,IAAI,GAAG;GAChC;IACE,eACE,KAAK,KACL,IACA,mBAAmB,OAAO,sBAC5B;IACA;EACJ;CACF;CAEA,oBAA6B,IAAe,QAAwB;EAClE,IAAI;GACF,MAAM,MAAM,IAAI,IACd,IAAI,OAAO,IACX,IAAI,QAAQ,UAAU,kBACxB;GACA,qBAAqB,IAAI,QAAQ;GACjC,MAAM,SAAS,IAAI,aAAa,IAAI,QAAQ;GAC5C,IAAI,CAAC,QACH,MAAM,IAAI,MAAM,oCAAoC;GAEtD,MAAM,aACJ,KAAKG,kBAAkB,EAAE,yBAAyB,MAAM;GAC1D,UAAe,KAAK,KAAK,YAAY,EAAE;EACzC,SAAS,KAAK;GACZ,eAAe,KAAK,KAAK,IAAI,KAAK,cAAc;EAClD;CACF;CAEA,aAAsB,OAAO,IAAe,QAAwB;EAClE,IAAI;GACF,MAAM,MAAM,qBAAqB,GAAG;GACpC,IAAI,IAAI,SAAS,WACf,KAAKC,6BAA6B,uBAAuB;GAG3D,MAAM,aAAa,MAAM,KAAKN,gBAAgB,UAAU,GAAG;GAC3D,IAAI,IAAI,WAAW,IAAI,UAAU,KAAKC,gBAGpC,KAAKA,eAAe,eAAe,IAAI,MAAM;GAE/C,qBAA0B,KAAK,KAAK,YAAY,EAAE;EACpD,SAAS,KAAK;GACZ,eAAe,KAAK,KAAK,IAAI,KAAK,cAAc;EAClD;CACF;CAEA,yBAAyB;CAEzB,6BAA6B,QAAgB;EAC3C,IAAI,CAAC,KAAKM,0BAA0B,KAAK,OAAO,UAAU,GAAG;GAC3D,KAAKT,IAAI,OAAO,mCAAmC,QAAQ;GAC3D,KAAUE,gBACP,IAAI,EACJ,OAAM,MACL,KAAKF,IAAI,OAAO,0CAA0C,CAAC,CAC7D,EACC,cAAc,KAAK,KAAK,CAAC;GAE5B,KAAKS,yBAAyB;EAChC;CACF;CAEA,WAAoC;EAClC,MAAM,EAAC,mBAAkB,KAAKR;EAC9B,KAAK,OAAO,iBAER,KAAKO,6BACH,0BAA0B,eAAe,KAC3C,GACF,cACF;CACF;CAEA,MAAyB,UAAyB;EAChD,IAAI,KAAKC,wBACP,MAAM,KAAKP,gBAAgB,KAAK;CAEpC;AACF;AAEA,IAAa,2BAAb,MAAgE;CAC9D;CACA;CACA;CACA;CAEA,YACE,IACA,SACA,UACA,mBACA;EACA,KAAKF,MAAM;EACX,KAAKU,WAAW;EAGhB,KAAKC,YAAY,SAAS,IAAI,UAAU,6BAA6B;GACnE,KAAK;IACJ,iBAAiB;EACpB,CAAC;EACD,KAAKC,qBAAqB;CAC5B;CAEA,MAAMC,uBAAuB,MAAc;EACzC,IAAI,UAAU,KAAKD;EACnB,IAAI,CAAC,SAAS;GACZ,MAAM,UAAU,MAAM,8BACpB,KAAKF,UACL,KAAKC,SACP;GACA,IAAI,CAAC,SACH,MAAM,IAAI,MAAM,+BAA+B;GAEjD,UAAU,QAAQ,SAAS,KAAK,IAAI,GAAG,QAAQ,KAAK,QAAQ,QAAQ;EACtE;EACA,MAAM,MAAM,IAAI,IAAI,MAAM,OAAO;EACjC,KAAKX,IAAI,OAAO,iCAAiC,KAAK;EACtD,OAAO;CACT;CAEA,MAAM,gBAAgB,QAAkD;EAItE,MAAM,KAAK,IAAI,YAAU,MAHP,KAAKa,uBAAuB,aAAa,IAG5B,IAAI,IADhB,gBAAgB,EAAC,OAAM,CACP,EAAO,SAAS,GAAG;EAEtD,OAAO,SAAS,KAAKb,KAAK,IAAI,qBAAqB;CACrD;CAEA,MAAM,UAAU,KAAqD;EAInE,MAAM,KAAK,IAAI,YAAU,MAHP,KAAKa,uBAAuB,YAAY,IAG3B,IADhB,UAAU,GACU,EAAO,SAAS,GAAG;EAEtD,OAAO,SAAS,KAAKb,KAAK,IAAI,gBAAgB;CAChD;AACF;AAIA,SAAgB,qBAAqB,KAAwC;CAC3E,MAAM,MAAM,IAAI,IAAI,IAAI,OAAO,IAAI,IAAI,QAAQ,UAAU,kBAAkB;CAC3E,MAAM,kBAAkB,qBAAqB,IAAI,QAAQ;CACzD,MAAM,SAAS,IAAI,UAAU,GAAG;CAEhC,OAAO;EACL;EACA,IAAI,OAAO,IAAI,MAAM,IAAI;EACzB,QAAQ,OAAO,IAAI,UAAU,KAAK;EAClC,MAAM,OAAO,IAAI,QAAQ,KAAK,MAAM,WAAW,WAAW;EAC1D,gBAAgB,OAAO,IAAI,kBAAkB,IAAI;EACjD,WAAW,OAAO,IAAI,aAAa,IAAI;EACvC,SAAS,OAAO,WAAW,SAAS;CACtC;AACF;AAEA,SAAS,qBAAqB,UAA0B;CACtD,MAAM,QAAQ,WAAW,KAAK,QAAQ;CACtC,IAAI,CAAC,OACH,MAAM,IAAI,MAAM,iBAAiB,UAAU;CAE7C,MAAM,IAAI,OAAO,MAAM,QAAQ,OAAO;CACtC,IACE,OAAO,MAAM,CAAC,KACd,IAAA,KACA,IAAI,gCAEJ,MAAM,IAAI,MACR,sCAAsC,EAAE,2BACZ,+BAA+B,SAC7D;CAEF,OAAO;AACT;AAGA,SAAS,UAAU,KAAyC;CAE1D,MAAM,EAAC,iBAAiB,GAAG,iBAAgB;CAC3C,OACE,oBAAA,GACA,mDACF;CACA,OAAO,IAAI,gBAAgB;EACzB,GAAG;EACH,QAAQ,IAAI,SAAS,IAAI,SAAS;EAClC,SAAS,IAAI,UAAU,SAAS;CAClC,CAAC;AACH"}
+{"version":3,"file":"change-streamer-http.js","names":["#lc","#opts","#changeStreamer","#backupMonitor","#subscribe","#reserveSnapshot","#receiveWebsocket","#getBackupMonitor","#ensureChangeStreamerStarted","#changeStreamerStarted","#shardID","#changeDB","#changeStreamerURI","#resolveChangeStreamer"],"sources":["../../../../../../zero-cache/src/services/change-streamer/change-streamer-http.ts"],"sourcesContent":["import type {IncomingMessage} from 'node:http';\nimport websocket from '@fastify/websocket';\nimport type {LogContext} from '@rocicorp/logger';\nimport WebSocket from 'ws';\nimport {assert} from '../../../../shared/src/asserts.ts';\nimport {must} from '../../../../shared/src/must.ts';\nimport type {IncomingMessageSubset} from '../../types/http.ts';\nimport {pgClient, type PostgresDB} from '../../types/pg.ts';\nimport {type Worker} from '../../types/processes.ts';\nimport {type ShardID} from '../../types/shards.ts';\nimport {\n  streamIn,\n  streamOut,\n  streamOutStringified,\n  type Source,\n} from '../../types/streams.ts';\nimport {URLParams} from '../../types/url-params.ts';\nimport {installWebSocketReceiver} from '../../types/websocket-handoff.ts';\nimport {closeWithError, PROTOCOL_ERROR} from '../../types/ws.ts';\nimport {HttpService} from '../http-service.ts';\nimport type {BackupMonitor} from './backup-monitor.ts';\nimport {\n  downstreamSchema,\n  PROTOCOL_VERSION,\n  type ChangeStreamer,\n  type ChangeStreamerService,\n  type Downstream,\n  type SubscriberContext,\n} from './change-streamer.ts';\nimport {discoverChangeStreamerAddress} from './schema/tables.ts';\nimport {snapshotMessageSchema, type SnapshotMessage} from './snapshot.ts';\n\nconst MIN_SUPPORTED_PROTOCOL_VERSION = 1;\n\nconst SNAPSHOT_PATH_PATTERN = '/replication/:version/snapshot';\nconst CHANGES_PATH_PATTERN = '/replication/:version/changes';\nconst PATH_REGEX = /\\/replication\\/v(?<version>\\d+)\\/(changes|snapshot)$/;\n\nconst SNAPSHOT_PATH = `/replication/v${PROTOCOL_VERSION}/snapshot`;\nconst CHANGES_PATH = `/replication/v${PROTOCOL_VERSION}/changes`;\n\ntype Options = {\n  port: number;\n  keepaliveTimeoutMs: number | undefined;\n  startupDelayMs: number;\n};\n\nexport class ChangeStreamerHttpServer extends HttpService {\n  readonly id = 'change-streamer-http-server';\n  readonly #lc: LogContext;\n  readonly #opts: Options;\n  readonly #changeStreamer: ChangeStreamerService;\n  readonly #backupMonitor: BackupMonitor | null;\n\n  constructor(\n    lc: LogContext,\n    opts: Options,\n    parent: Worker,\n    changeStreamer: ChangeStreamerService,\n    backupMonitor: BackupMonitor | null,\n  ) {\n    super('change-streamer-http-server', lc, opts, async fastify => {\n      await fastify.register(websocket);\n\n      fastify.get(CHANGES_PATH_PATTERN, {websocket: true}, this.#subscribe);\n      fastify.get(\n        SNAPSHOT_PATH_PATTERN,\n        {websocket: true},\n        this.#reserveSnapshot,\n      );\n\n      installWebSocketReceiver<'snapshot' | 'changes'>(\n        lc,\n        fastify.websocketServer,\n        this.#receiveWebsocket,\n        parent,\n      );\n    });\n\n    this.#lc = lc;\n    this.#opts = opts;\n    this.#changeStreamer = changeStreamer;\n    this.#backupMonitor = backupMonitor;\n  }\n\n  #getBackupMonitor() {\n    return must(\n      this.#backupMonitor,\n      'replication-manager is not configured with a ZERO_LITESTREAM_BACKUP_URL',\n    );\n  }\n\n  // Called when receiving a web socket via the main dispatcher handoff.\n  readonly #receiveWebsocket = (\n    ws: WebSocket,\n    action: 'changes' | 'snapshot',\n    msg: IncomingMessageSubset,\n  ) => {\n    switch (action) {\n      case 'snapshot':\n        return this.#reserveSnapshot(ws, msg);\n      case 'changes':\n        return this.#subscribe(ws, msg);\n      default:\n        closeWithError(\n          this._lc,\n          ws,\n          `invalid action \"${action}\" received in handoff`,\n        );\n        return;\n    }\n  };\n\n  readonly #reserveSnapshot = (ws: WebSocket, req: RequestHeaders) => {\n    try {\n      const url = new URL(\n        req.url ?? '',\n        req.headers.origin ?? 'http://localhost',\n      );\n      checkProtocolVersion(url.pathname);\n      const taskID = url.searchParams.get('taskID');\n      if (!taskID) {\n        throw new Error('Missing taskID in snapshot request');\n      }\n      const downstream =\n        this.#getBackupMonitor().startSnapshotReservation(taskID);\n      void streamOut(this._lc, downstream, ws);\n    } catch (err) {\n      closeWithError(this._lc, ws, err, PROTOCOL_ERROR);\n    }\n  };\n\n  readonly #subscribe = async (ws: WebSocket, req: RequestHeaders) => {\n    try {\n      const ctx = getSubscriberContext(req);\n      if (ctx.mode === 'serving') {\n        this.#ensureChangeStreamerStarted('incoming subscription');\n      }\n\n      const downstream = await this.#changeStreamer.subscribe(ctx);\n      if (ctx.initial && ctx.taskID && this.#backupMonitor) {\n        // Now that the change-streamer knows about the subscriber and watermark,\n        // end the reservation to safely resume scheduling cleanup.\n        this.#backupMonitor.endReservation(ctx.taskID);\n      }\n      void streamOutStringified(this._lc, downstream, ws);\n    } catch (err) {\n      closeWithError(this._lc, ws, err, PROTOCOL_ERROR);\n    }\n  };\n\n  #changeStreamerStarted = false;\n\n  #ensureChangeStreamerStarted(reason: string) {\n    if (!this.#changeStreamerStarted && this._state.shouldRun()) {\n      this.#lc.info?.(`starting ChangeStreamerService: ${reason}`);\n      void this.#changeStreamer\n        .run()\n        .catch(e =>\n          this.#lc.warn?.(`ChangeStreamerService ended with error`, e),\n        )\n        .finally(() => this.stop());\n\n      this.#changeStreamerStarted = true;\n    }\n  }\n\n  protected override _onStart(): void {\n    const {startupDelayMs} = this.#opts;\n    this._state.setTimeout(\n      () =>\n        this.#ensureChangeStreamerStarted(\n          `startup delay elapsed (${startupDelayMs} ms)`,\n        ),\n      startupDelayMs,\n    );\n  }\n\n  protected override async _onStop(): Promise<void> {\n    if (this.#changeStreamerStarted) {\n      await this.#changeStreamer.stop();\n    }\n  }\n}\n\nexport class ChangeStreamerHttpClient implements ChangeStreamer {\n  readonly #lc: LogContext;\n  readonly #shardID: ShardID;\n  readonly #changeDB: PostgresDB;\n  readonly #changeStreamerURI: string | undefined;\n\n  constructor(\n    lc: LogContext,\n    shardID: ShardID,\n    changeDB: string,\n    changeStreamerURI: string | undefined,\n  ) {\n    this.#lc = lc;\n    this.#shardID = shardID;\n    // Create a pg client with a single short-lived connection for the purpose\n    // of change-streamer discovery (i.e. ChangeDB as DNS).\n    this.#changeDB = pgClient(lc, changeDB, 'change-streamer-discovery', {\n      max: 1,\n      ['idle_timeout']: 15,\n    });\n    this.#changeStreamerURI = changeStreamerURI;\n  }\n\n  async #resolveChangeStreamer(path: string) {\n    let baseURL = this.#changeStreamerURI;\n    if (!baseURL) {\n      const address = await discoverChangeStreamerAddress(\n        this.#shardID,\n        this.#changeDB,\n      );\n      if (!address) {\n        throw new Error(`no change-streamer is running`);\n      }\n      baseURL = address.includes('://') ? `${address}/` : `ws://${address}/`;\n    }\n    const uri = new URL(path, baseURL);\n    this.#lc.info?.(`connecting to change-streamer@${uri}`);\n    return uri;\n  }\n\n  async reserveSnapshot(taskID: string): Promise<Source<SnapshotMessage>> {\n    const uri = await this.#resolveChangeStreamer(SNAPSHOT_PATH);\n\n    const params = new URLSearchParams({taskID});\n    const ws = new WebSocket(uri + `?${params.toString()}`);\n\n    return streamIn(this.#lc, ws, snapshotMessageSchema);\n  }\n\n  async subscribe(ctx: SubscriberContext): Promise<Source<Downstream>> {\n    const uri = await this.#resolveChangeStreamer(CHANGES_PATH);\n\n    const params = getParams(ctx);\n    const ws = new WebSocket(uri + `?${params.toString()}`);\n\n    return streamIn(this.#lc, ws, downstreamSchema);\n  }\n}\n\ntype RequestHeaders = Pick<IncomingMessage, 'url' | 'headers'>;\n\nexport function getSubscriberContext(req: RequestHeaders): SubscriberContext {\n  const url = new URL(req.url ?? '', req.headers.origin ?? 'http://localhost');\n  const protocolVersion = checkProtocolVersion(url.pathname);\n  const params = new URLParams(url);\n\n  return {\n    protocolVersion,\n    id: params.get('id', true),\n    taskID: params.get('taskID', false),\n    mode: params.get('mode', false) === 'backup' ? 'backup' : 'serving',\n    replicaVersion: params.get('replicaVersion', true),\n    watermark: params.get('watermark', true),\n    initial: params.getBoolean('initial'),\n  };\n}\n\nfunction checkProtocolVersion(pathname: string): number {\n  const match = PATH_REGEX.exec(pathname);\n  if (!match) {\n    throw new Error(`invalid path: ${pathname}`);\n  }\n  const v = Number(match.groups?.version);\n  if (\n    Number.isNaN(v) ||\n    v > PROTOCOL_VERSION ||\n    v < MIN_SUPPORTED_PROTOCOL_VERSION\n  ) {\n    throw new Error(\n      `Cannot service client at protocol v${v}. ` +\n        `Supported protocols: [v${MIN_SUPPORTED_PROTOCOL_VERSION} ... v${PROTOCOL_VERSION}]`,\n    );\n  }\n  return v;\n}\n\n// This is called from the client-side (i.e. the replicator).\nfunction getParams(ctx: SubscriberContext): URLSearchParams {\n  // The protocolVersion is hard-coded into the CHANGES_PATH.\n  const {protocolVersion, ...stringParams} = ctx;\n  assert(\n    protocolVersion === PROTOCOL_VERSION,\n    `replicator should be setting protocolVersion to ${PROTOCOL_VERSION}`,\n  );\n  return new URLSearchParams({\n    ...stringParams,\n    taskID: ctx.taskID ? ctx.taskID : '',\n    initial: ctx.initial ? 'true' : 'false',\n  });\n}\n"],"mappings":";;;;;;;;;;;;;;;AAgCA,IAAM,iCAAiC;AAEvC,IAAM,wBAAwB;AAC9B,IAAM,uBAAuB;AAC7B,IAAM,aAAa;AAEnB,IAAM,gBAAgB;AACtB,IAAM,eAAe;AAQrB,IAAa,2BAAb,cAA8C,YAAY;CACxD,KAAc;CACd;CACA;CACA;CACA;CAEA,YACE,IACA,MACA,QACA,gBACA,eACA;AACA,QAAM,+BAA+B,IAAI,MAAM,OAAM,YAAW;AAC9D,SAAM,QAAQ,SAAS,UAAU;AAEjC,WAAQ,IAAI,sBAAsB,EAAC,WAAW,MAAK,EAAE,MAAA,UAAgB;AACrE,WAAQ,IACN,uBACA,EAAC,WAAW,MAAK,EACjB,MAAA,gBACD;AAED,4BACE,IACA,QAAQ,iBACR,MAAA,kBACA,OACD;IACD;AAEF,QAAA,KAAW;AACX,QAAA,OAAa;AACb,QAAA,iBAAuB;AACvB,QAAA,gBAAsB;;CAGxB,oBAAoB;AAClB,SAAO,KACL,MAAA,eACA,0EACD;;CAIH,qBACE,IACA,QACA,QACG;AACH,UAAQ,QAAR;GACE,KAAK,WACH,QAAO,MAAA,gBAAsB,IAAI,IAAI;GACvC,KAAK,UACH,QAAO,MAAA,UAAgB,IAAI,IAAI;GACjC;AACE,mBACE,KAAK,KACL,IACA,mBAAmB,OAAO,uBAC3B;AACD;;;CAIN,oBAA6B,IAAe,QAAwB;AAClE,MAAI;GACF,MAAM,MAAM,IAAI,IACd,IAAI,OAAO,IACX,IAAI,QAAQ,UAAU,mBACvB;AACD,wBAAqB,IAAI,SAAS;GAClC,MAAM,SAAS,IAAI,aAAa,IAAI,SAAS;AAC7C,OAAI,CAAC,OACH,OAAM,IAAI,MAAM,qCAAqC;GAEvD,MAAM,aACJ,MAAA,kBAAwB,CAAC,yBAAyB,OAAO;AACtD,aAAU,KAAK,KAAK,YAAY,GAAG;WACjC,KAAK;AACZ,kBAAe,KAAK,KAAK,IAAI,KAAK,eAAe;;;CAIrD,aAAsB,OAAO,IAAe,QAAwB;AAClE,MAAI;GACF,MAAM,MAAM,qBAAqB,IAAI;AACrC,OAAI,IAAI,SAAS,UACf,OAAA,4BAAkC,wBAAwB;GAG5D,MAAM,aAAa,MAAM,MAAA,eAAqB,UAAU,IAAI;AAC5D,OAAI,IAAI,WAAW,IAAI,UAAU,MAAA,cAG/B,OAAA,cAAoB,eAAe,IAAI,OAAO;AAE3C,wBAAqB,KAAK,KAAK,YAAY,GAAG;WAC5C,KAAK;AACZ,kBAAe,KAAK,KAAK,IAAI,KAAK,eAAe;;;CAIrD,yBAAyB;CAEzB,6BAA6B,QAAgB;AAC3C,MAAI,CAAC,MAAA,yBAA+B,KAAK,OAAO,WAAW,EAAE;AAC3D,SAAA,GAAS,OAAO,mCAAmC,SAAS;AACvD,SAAA,eACF,KAAK,CACL,OAAM,MACL,MAAA,GAAS,OAAO,0CAA0C,EAAE,CAC7D,CACA,cAAc,KAAK,MAAM,CAAC;AAE7B,SAAA,wBAA8B;;;CAIlC,WAAoC;EAClC,MAAM,EAAC,mBAAkB,MAAA;AACzB,OAAK,OAAO,iBAER,MAAA,4BACE,0BAA0B,eAAe,MAC1C,EACH,eACD;;CAGH,MAAyB,UAAyB;AAChD,MAAI,MAAA,sBACF,OAAM,MAAA,eAAqB,MAAM;;;AAKvC,IAAa,2BAAb,MAAgE;CAC9D;CACA;CACA;CACA;CAEA,YACE,IACA,SACA,UACA,mBACA;AACA,QAAA,KAAW;AACX,QAAA,UAAgB;AAGhB,QAAA,WAAiB,SAAS,IAAI,UAAU,6BAA6B;GACnE,KAAK;IACJ,iBAAiB;GACnB,CAAC;AACF,QAAA,oBAA0B;;CAG5B,OAAA,sBAA6B,MAAc;EACzC,IAAI,UAAU,MAAA;AACd,MAAI,CAAC,SAAS;GACZ,MAAM,UAAU,MAAM,8BACpB,MAAA,SACA,MAAA,SACD;AACD,OAAI,CAAC,QACH,OAAM,IAAI,MAAM,gCAAgC;AAElD,aAAU,QAAQ,SAAS,MAAM,GAAG,GAAG,QAAQ,KAAK,QAAQ,QAAQ;;EAEtE,MAAM,MAAM,IAAI,IAAI,MAAM,QAAQ;AAClC,QAAA,GAAS,OAAO,iCAAiC,MAAM;AACvD,SAAO;;CAGT,MAAM,gBAAgB,QAAkD;EAItE,MAAM,KAAK,IAAI,YAHH,MAAM,MAAA,sBAA4B,cAAc,GAG7B,IADhB,IAAI,gBAAgB,EAAC,QAAO,CAAC,CACF,UAAU,GAAG;AAEvD,SAAO,SAAS,MAAA,IAAU,IAAI,sBAAsB;;CAGtD,MAAM,UAAU,KAAqD;EAInE,MAAM,KAAK,IAAI,YAHH,MAAM,MAAA,sBAA4B,aAAa,GAG5B,IADhB,UAAU,IAAI,CACa,UAAU,GAAG;AAEvD,SAAO,SAAS,MAAA,IAAU,IAAI,iBAAiB;;;AAMnD,SAAgB,qBAAqB,KAAwC;CAC3E,MAAM,MAAM,IAAI,IAAI,IAAI,OAAO,IAAI,IAAI,QAAQ,UAAU,mBAAmB;CAC5E,MAAM,kBAAkB,qBAAqB,IAAI,SAAS;CAC1D,MAAM,SAAS,IAAI,UAAU,IAAI;AAEjC,QAAO;EACL;EACA,IAAI,OAAO,IAAI,MAAM,KAAK;EAC1B,QAAQ,OAAO,IAAI,UAAU,MAAM;EACnC,MAAM,OAAO,IAAI,QAAQ,MAAM,KAAK,WAAW,WAAW;EAC1D,gBAAgB,OAAO,IAAI,kBAAkB,KAAK;EAClD,WAAW,OAAO,IAAI,aAAa,KAAK;EACxC,SAAS,OAAO,WAAW,UAAU;EACtC;;AAGH,SAAS,qBAAqB,UAA0B;CACtD,MAAM,QAAQ,WAAW,KAAK,SAAS;AACvC,KAAI,CAAC,MACH,OAAM,IAAI,MAAM,iBAAiB,WAAW;CAE9C,MAAM,IAAI,OAAO,MAAM,QAAQ,QAAQ;AACvC,KACE,OAAO,MAAM,EAAE,IACf,IAAA,KACA,IAAI,+BAEJ,OAAM,IAAI,MACR,sCAAsC,EAAE,2BACZ,+BAA+B,UAC5D;AAEH,QAAO;;AAIT,SAAS,UAAU,KAAyC;CAE1D,MAAM,EAAC,iBAAiB,GAAG,iBAAgB;AAC3C,QACE,oBAAA,GACA,oDACD;AACD,QAAO,IAAI,gBAAgB;EACzB,GAAG;EACH,QAAQ,IAAI,SAAS,IAAI,SAAS;EAClC,SAAS,IAAI,UAAU,SAAS;EACjC,CAAC"}

package/out/zero-cache/src/services/change-streamer/change-streamer-service.js.map

@@ -1 +1 @@
-{"version":3,"file":"change-streamer-service.js","names":["#lc","#shard","#changeDB","#replicaVersion","#source","#storer","#forwarder","#replicationStatusPublisher","#autoReset","#state","#initialWatermarks","#serving","#txCounter","#changeCounter","#stream","#purgeLock","#latestStatus","#handleControlMessage","#purgeOldChanges"],"sources":["../../../../../../zero-cache/src/services/change-streamer/change-streamer-service.ts"],"sourcesContent":["import {getDefaultHighWaterMark} from 'node:stream';\nimport type {LogContext} from '@rocicorp/logger';\nimport {resolver} from '@rocicorp/resolver';\nimport {unreachable} from '../../../../shared/src/asserts.ts';\nimport {promiseVoid} from '../../../../shared/src/resolved-promises.ts';\nimport {publishCriticalEvent} from '../../observability/events.ts';\nimport {getOrCreateCounter} from '../../observability/metrics.ts';\nimport {\n  min,\n  type AtLeastOne,\n  type LexiVersion,\n} from '../../types/lexi-version.ts';\nimport type {PostgresDB} from '../../types/pg.ts';\nimport type {ShardID} from '../../types/shards.ts';\nimport type {Source} from '../../types/streams.ts';\nimport {Subscription} from '../../types/subscription.ts';\nimport type {\n  ChangeSource,\n  ChangeStream,\n} from '../change-source/change-source.ts';\nimport {\n  type ChangeStreamControl,\n  type ChangeStreamData,\n  type Rollback,\n} from '../change-source/protocol/current/downstream.ts';\nimport {\n  publishReplicationError,\n  replicationStatusError,\n  type ReplicationStatusPublisher,\n} from '../replicator/replication-status.ts';\nimport type {SubscriptionState} from '../replicator/schema/replication-state.ts';\nimport {\n  DEFAULT_MAX_RETRY_DELAY_MS,\n  RunningState,\n  UnrecoverableError,\n} from '../running-state.ts';\nimport {\n  type ChangeStreamerService,\n  type Status,\n  type SubscriberContext,\n} from './change-streamer.ts';\nimport * as ErrorType from './error-type-enum.ts';\nimport {Forwarder} from './forwarder.ts';\nimport {initChangeStreamerSchema} from './schema/init.ts';\nimport {\n  AutoResetSignal,\n  ensureReplicationConfig,\n  markResetRequired,\n} from './schema/tables.ts';\nimport {\n  Storer,\n  type PurgeLock,\n  type TuningOptions as StorerOptions,\n} from './storer.ts';\nimport {Subscriber} from './subscriber.ts';\n\nexport type TuningOptions = StorerOptions & {\n  flowControlConsensusPaddingSeconds: number;\n};\n\n/**\n * Performs initialization and schema migrations to initialize a ChangeStreamerImpl.\n */\nexport async function initializeStreamer(\n  lc: LogContext,\n  shard: ShardID,\n  taskID: string,\n  discoveryAddress: string,\n  discoveryProtocol: string,\n  changeDB: PostgresDB,\n  changeSource: ChangeSource,\n  replicationStatusPublisher: ReplicationStatusPublisher,\n  subscriptionState: SubscriptionState,\n  purgeLock: PurgeLock | null,\n  autoReset: boolean,\n  opts: TuningOptions,\n  setTimeoutFn = setTimeout,\n): Promise<ChangeStreamerService> {\n  // Make sure the ChangeLog DB is set up.\n  await initChangeStreamerSchema(lc, changeDB, shard);\n  await ensureReplicationConfig(\n    lc,\n    changeDB,\n    subscriptionState,\n    shard,\n    autoReset,\n    setTimeoutFn,\n  );\n\n  const {replicaVersion} = subscriptionState;\n  return new ChangeStreamerImpl(\n    lc,\n    shard,\n    taskID,\n    discoveryAddress,\n    discoveryProtocol,\n    changeDB,\n    replicaVersion,\n    changeSource,\n    replicationStatusPublisher,\n    purgeLock,\n    autoReset,\n    opts,\n    setTimeoutFn,\n  );\n}\n\nconst REPLICATION_STATUS_ERROR_DELAY_THRESHOLD_MS = 5000;\n\nexport type ChangeTag = ChangeStreamData[1]['tag'];\n\n/**\n * Internally all Downstream messages (not just commits) are given a watermark.\n * These are used for internal ordering for:\n * 1. Replaying new changes in the Storer\n * 2. Filtering old changes in the Subscriber\n *\n * However, only the watermark for `Commit` messages are exposed to\n * subscribers, as that is the only semantically correct watermark to\n * use for tracking a position in a replication stream.\n *\n * Additionally, the ChangeStreamData is eagerly stringified once, after which\n * the string is passed to the changeLog and all subscribers, eliminating\n * redundant stringification and reducing GC churn.\n */\nexport type WatermarkedChange = [\n  watermark: string,\n  tag: ChangeTag,\n  json: string,\n];\n\n/**\n * Upstream-agnostic dispatch of messages in a {@link ChangeStreamMessage} to a\n * {@link Forwarder} and {@link Storer} to execute the forward-store-ack\n * procedure described in {@link ChangeStreamer}.\n *\n * ### Subscriber Catchup\n *\n * Connecting clients first need to be \"caught up\" to the current watermark\n * (from stored change log entries) before new entries are forwarded to\n * them. This is non-trivial because the replication stream may be in the\n * middle of a pending streamed Transaction for which some entries have\n * already been forwarded but are not yet committed to the store.\n *\n *\n * ```\n * ------------------------------- - - - - - - - - - - - - - - - - - - -\n * | Historic changes in storage |  Pending (streamed) tx  |   Next tx\n * ------------------------------- - - - - - - - - - - - - - - - - - - -\n *                                           Replication stream\n *                                           >  >  >  >  >  >  >  >  >\n *           ^  ---> required catchup --->   ^\n * Subscriber watermark               Subscription begins\n * ```\n *\n * Preemptively buffering the changes of every pending transaction\n * would be wasteful and consume too much memory for large transactions.\n *\n * Instead, the streamer synchronously dispatches changes and subscriptions\n * to the {@link Forwarder} and the {@link Storer} such that the two\n * components are aligned as to where in the stream the subscription started.\n * The two components then coordinate catchup and handoff via the\n * {@link Subscriber} object with the following algorithm:\n *\n * * If the streamer is in the middle of a pending Transaction, the\n *   Subscriber is \"queued\" on both the Forwarder and the Storer. In this\n *   state, new changes are *not* forwarded to the Subscriber, and catchup\n *   is not yet executed.\n * * Once the commit message for the pending Transaction is processed\n *   by the Storer, it begins catchup on the Subscriber (with a READONLY\n *   snapshot so that it does not block subsequent storage operations).\n *   This catchup is thus guaranteed to load the change log entries of\n *   that last Transaction.\n * * When the Forwarder processes that same commit message, it moves the\n *   Subscriber from the \"queued\" to the \"active\" set of clients such that\n *   the Subscriber begins receiving new changes, starting from the next\n *   Transaction.\n * * The Subscriber does not forward those changes, however, if its catchup\n *   is not complete. Until then, it buffers the changes in memory.\n * * Once catchup is complete, the buffered changes are immediately sent\n *   and the Subscriber henceforth forwards changes as they are received.\n *\n * In the (common) case where the streamer is not in the middle of a pending\n * transaction when a subscription begins, the Storer begins catchup\n * immediately and the Forwarder directly adds the Subscriber to its active\n * set. However, the Subscriber still buffers any forwarded messages until\n * its catchup is complete.\n *\n * ### Watermarks and ordering\n *\n * The ChangeStreamerService depends on its {@link ChangeSource} to send\n * changes in contiguous [`begin`, `data` ..., `data`, `commit`] sequences\n * in commit order. This follows Postgres's Logical Replication Protocol\n * Message Flow:\n *\n * https://www.postgresql.org/docs/16/protocol-logical-replication.html#PROTOCOL-LOGICAL-MESSAGES-FLOW\n *\n * > The logical replication protocol sends individual transactions one by one.\n * > This means that all messages between a pair of Begin and Commit messages belong to the same transaction.\n *\n * In order to correctly replay (new) and filter (old) messages to subscribers\n * at different points in the replication stream, these changes must be assigned\n * watermarks such that they preserve the order in which they were received\n * from the ChangeSource.\n *\n * A previous implementation incorrectly derived these watermarks from the Postgres\n * Log Sequence Numbers (LSN) of each message. However, LSNs from concurrent,\n * non-conflicting transactions can overlap, which can result in a `begin` message\n * with an earlier LSN arriving after a `commit` message. For example, the\n * changes for these transactions:\n *\n * ```\n * LSN:   1     2     3  4    5   6   7     8   9      10\n * tx1: begin  data     data     data     commit\n * tx2:             begin    data    data      data  commit\n * ```\n *\n * will arrive as:\n *\n * ```\n * begin1, data2, data4, data6, commit8, begin3, data5, data7, data9, commit10\n * ```\n *\n * Thus, LSN of non-commit messages are not suitable for tracking the sorting\n * order of the replication stream.\n *\n * Instead, the ChangeStreamer uses the following algorithm for deterministic\n * catchup and filtering of changes:\n *\n * * A `commit` message is assigned to a watermark corresponding to its LSN.\n *   These are guaranteed to be in commit order by definition.\n *\n * * `begin` and `data` messages are assigned to the watermark of the\n *   preceding `commit` (the previous transaction, or the replication\n *   slot's starting LSN) plus 1. This guarantees that they will be sorted\n *   after the previously commit transaction even if their LSNs came before it.\n *   This is referred to as the `preCommitWatermark`.\n *\n * * In the ChangeLog DB, messages have a secondary sort column `pos`, which is\n *   the position of the message within its transaction, with the `begin` message\n *   starting at `0`. This guarantees that `begin` and `data` messages will be\n *   fetched in the original ChangeSource order during catchup.\n *\n * `begin` and `data` messages share the same watermark, but this is sufficient for\n * Subscriber filtering because subscribers only know about the `commit` watermarks\n * exposed in the `Downstream` `Commit` message. The Subscriber object thus compares\n * the internal watermarks of the incoming messages against the commit watermark of\n * the caller, updating the watermark at every `Commit` message that is forwarded.\n *\n * ### Cleanup\n *\n * As mentioned in the {@link ChangeStreamer} documentation: \"the ChangeStreamer\n * uses a combination of [the \"initial\", i.e. backup-derived watermark and] ACK\n * responses from connected subscribers to determine the watermark up\n * to which it is safe to purge old change log entries.\"\n *\n * More concretely:\n *\n * * The `initial`, backup-derived watermark is the earliest to which cleanup\n *   should ever happen.\n *\n * * However, it is possible for the replica backup to be *ahead* of a connected\n *   subscriber; and if a network error causes that subscriber to retry from its\n *   last watermark, the change streamer must support it.\n *\n * Thus, before cleaning up to an `initial` backup-derived watermark, the change\n * streamer first confirms that all connected subscribers have also passed\n * that watermark.\n */\nclass ChangeStreamerImpl implements ChangeStreamerService {\n  readonly id: string;\n  readonly #lc: LogContext;\n  readonly #shard: ShardID;\n  readonly #changeDB: PostgresDB;\n  readonly #replicaVersion: string;\n  readonly #source: ChangeSource;\n  readonly #storer: Storer;\n  readonly #forwarder: Forwarder;\n  readonly #replicationStatusPublisher: ReplicationStatusPublisher;\n\n  readonly #autoReset: boolean;\n  readonly #state: RunningState;\n  readonly #initialWatermarks = new Set<string>();\n\n  // Starting the (Postgres) ChangeStream results in killing the previous\n  // Postgres subscriber, potentially creating a gap in which the old\n  // change-streamer has shut down and the new change-streamer has not yet\n  // been recognized as \"healthy\" (and thus does not get any requests).\n  //\n  // To minimize this gap, delay starting the ChangeStream until the first\n  // request from a `serving` replicator, indicating that higher level\n  // load-balancing / routing logic has begun routing requests to this task.\n  readonly #serving = resolver();\n\n  readonly #txCounter = getOrCreateCounter(\n    'replication',\n    'transactions',\n    'Count of replicated transactions',\n  );\n  readonly #changeCounter = getOrCreateCounter(\n    'replication',\n    'changes',\n    'Count of replicated changes (DML or DDL statements)',\n  );\n\n  #latestStatus: Status;\n  #purgeLock: PurgeLock | null;\n  #stream: ChangeStream | undefined;\n\n  constructor(\n    lc: LogContext,\n    shard: ShardID,\n    taskID: string,\n    discoveryAddress: string,\n    discoveryProtocol: string,\n    changeDB: PostgresDB,\n    replicaVersion: string,\n    source: ChangeSource,\n    replicationStatusPublisher: ReplicationStatusPublisher,\n    initialPurgeLock: PurgeLock | null,\n    autoReset: boolean,\n    opts: TuningOptions,\n    setTimeoutFn = setTimeout,\n  ) {\n    this.id = `change-streamer`;\n    this.#lc = lc.withContext('component', 'change-streamer');\n    this.#shard = shard;\n    this.#changeDB = changeDB;\n    this.#replicaVersion = replicaVersion;\n    this.#source = source;\n    this.#storer = new Storer(\n      lc,\n      shard,\n      taskID,\n      discoveryAddress,\n      discoveryProtocol,\n      changeDB,\n      replicaVersion,\n      consumed => this.#stream?.acks.push(['status', consumed[1], consumed[2]]),\n      err => this.stop(err),\n      opts,\n    );\n    this.#forwarder = new Forwarder(lc, {\n      flowControlConsensusPaddingSeconds:\n        opts.flowControlConsensusPaddingSeconds,\n    });\n    this.#replicationStatusPublisher = replicationStatusPublisher;\n    this.#purgeLock = initialPurgeLock;\n    this.#autoReset = autoReset;\n    this.#state = new RunningState(this.id, undefined, setTimeoutFn);\n    this.#latestStatus = {tag: 'status'};\n  }\n\n  async run() {\n    this.#lc.info?.('starting change stream');\n\n    this.#forwarder.startProgressMonitor();\n\n    const lagReport = await this.#source.startLagReporter();\n    if (lagReport) {\n      this.#latestStatus.lagReport = lagReport;\n    }\n\n    // Once this change-streamer acquires \"ownership\" of the change DB,\n    // it is safe to start the storer.\n    await this.#storer.assumeOwnership(this.#purgeLock);\n    this.#purgeLock = null;\n\n    // The threshold in (estimated number of) bytes to send() on subscriber\n    // websockets before `await`-ing the I/O buffers to be ready for more.\n    const flushBytesThreshold = getDefaultHighWaterMark(false);\n\n    while (this.#state.shouldRun()) {\n      let err: unknown;\n      let watermark: string | null = null;\n      let unflushedBytes = 0;\n      try {\n        const {lastWatermark, backfillRequests} =\n          await this.#storer.getStartStreamInitializationParameters();\n        const stream = await this.#source.startStream(\n          lastWatermark,\n          backfillRequests,\n        );\n        this.#storer.run().catch(e => stream.changes.cancel(e));\n\n        this.#stream = stream;\n        if (\n          this.#state.resetBackoff() >\n          REPLICATION_STATUS_ERROR_DELAY_THRESHOLD_MS\n        ) {\n          // After recovering from a backoff for which a replication status\n          // error was published, publish an OK status\n          this.#replicationStatusPublisher.publish(\n            this.#lc,\n            'Replicating',\n            `Replicating from ${lastWatermark}`,\n          );\n        }\n        watermark = null;\n\n        for await (const change of stream.changes) {\n          const [type, msg] = change;\n          switch (type) {\n            case 'status':\n              if (msg.ack) {\n                this.#storer.status(change); // storer acks once it gets through its queue\n              }\n              if (msg.lagReport) {\n                // Lag reports are not stored in the cdc change log, but rather\n                // only forwarded on \"live\" connections. When a new subscriber\n                // is catching up, it is initialized with the #latestStatus\n                // from which it can measure lag while catching up.\n                this.#latestStatus.lagReport = msg.lagReport;\n                this.#forwarder.sendStatus(this.#latestStatus);\n              }\n              continue;\n            case 'control':\n              await this.#handleControlMessage(msg);\n              continue; // control messages are not stored/forwarded\n            case 'begin':\n              watermark = change[2].commitWatermark;\n              break;\n            case 'commit':\n              if (watermark !== change[2].watermark) {\n                throw new UnrecoverableError(\n                  `commit watermark ${change[2].watermark} does not match 'begin' watermark ${watermark}`,\n                );\n              }\n              this.#txCounter.add(1);\n              break;\n            default:\n              if (type === 'data') {\n                this.#changeCounter.add(1);\n              }\n              if (watermark === null) {\n                throw new UnrecoverableError(\n                  `${type} change (${msg.tag}) received before 'begin' message`,\n                );\n              }\n              break;\n          }\n\n          const json = this.#storer.store(watermark, change);\n          const entry: WatermarkedChange = [watermark, change[1].tag, json];\n          unflushedBytes += json.length;\n          if (unflushedBytes < flushBytesThreshold) {\n            // pipeline changes until flushBytesThreshold\n            this.#forwarder.forward(entry);\n          } else {\n            // Wait for messages to clear socket buffers to ensure that they\n            // make their way to subscribers. Without this `await`, the\n            // messages end up being buffered in this process, which:\n            // (1) results in memory pressure and increased GC activity\n            // (2) prevents subscribers from processing the messages as they\n            //     arrive, instead getting them in a large batch after being\n            //     idle while they were queued (causing further delays).\n            await this.#forwarder.forwardWithFlowControl(entry);\n            unflushedBytes = 0;\n          }\n\n          if (type === 'commit' || type === 'rollback') {\n            watermark = null;\n          }\n\n          // Allow the storer to exert back pressure.\n          const readyForMore = this.#storer.readyForMore();\n          if (readyForMore) {\n            await readyForMore;\n          }\n        }\n      } catch (e) {\n        err = e;\n      } finally {\n        this.#stream?.changes.cancel();\n        this.#stream = undefined;\n      }\n\n      // When the change stream is interrupted, abort any pending transaction.\n      if (watermark) {\n        this.#lc.warn?.(`aborting interrupted transaction ${watermark}`);\n        this.#storer.abort();\n        this.#forwarder.forward([watermark, 'rollback', ROLLBACK_JSON]);\n      }\n\n      // Backoff and drain any pending entries in the storer before reconnecting.\n      await Promise.all([\n        this.#storer.stop(),\n        this.#state.backoff(this.#lc, err),\n        this.#state.retryDelay > REPLICATION_STATUS_ERROR_DELAY_THRESHOLD_MS\n          ? publishCriticalEvent(\n              this.#lc,\n              replicationStatusError(this.#lc, 'Replicating', err),\n            )\n          : promiseVoid,\n      ]);\n    }\n\n    this.#forwarder.stopProgressMonitor();\n    this.#lc.info?.('ChangeStreamer stopped');\n  }\n\n  async #handleControlMessage(msg: ChangeStreamControl[1]) {\n    this.#lc.info?.('received control message', msg);\n    const {tag} = msg;\n\n    switch (tag) {\n      case 'reset-required':\n        await markResetRequired(this.#changeDB, this.#shard);\n        await publishReplicationError(\n          this.#lc,\n          'Replicating',\n          msg.message ?? 'Resync required',\n          msg.errorDetails,\n        );\n        if (this.#autoReset) {\n          this.#lc.warn?.('shutting down for auto-reset');\n          await this.stop(new AutoResetSignal());\n        }\n        break;\n      default:\n        unreachable(tag);\n    }\n  }\n\n  subscribe(ctx: SubscriberContext): Promise<Source<string>> {\n    const {protocolVersion, id, mode, replicaVersion, watermark} = ctx;\n    if (mode === 'serving') {\n      this.#serving.resolve();\n    }\n    const downstream = Subscription.create<string>({\n      cleanup: () => this.#forwarder.remove(subscriber),\n    });\n    const subscriber = new Subscriber(\n      protocolVersion,\n      id,\n      watermark,\n      downstream,\n      () => this.#latestStatus,\n    );\n    if (replicaVersion !== this.#replicaVersion) {\n      this.#lc.warn?.(\n        `rejecting subscriber at replica version ${replicaVersion}`,\n      );\n      subscriber.close(\n        ErrorType.WrongReplicaVersion,\n        `current replica version is ${\n          this.#replicaVersion\n        } (requested ${replicaVersion})`,\n      );\n    } else {\n      this.#lc.debug?.(`adding subscriber ${subscriber.id}`);\n\n      this.#forwarder.add(subscriber);\n      this.#storer.catchup(subscriber, mode);\n    }\n    return Promise.resolve(downstream);\n  }\n\n  scheduleCleanup(watermark: string) {\n    const origSize = this.#initialWatermarks.size;\n    this.#initialWatermarks.add(watermark);\n\n    if (origSize === 0) {\n      this.#state.setTimeout(() => this.#purgeOldChanges(), CLEANUP_DELAY_MS);\n    }\n  }\n\n  async getChangeLogState(): Promise<{\n    replicaVersion: string;\n    minWatermark: string;\n  }> {\n    const minWatermark = await this.#storer.getMinWatermarkForCatchup();\n    if (!minWatermark) {\n      this.#lc.warn?.(\n        `Unexpected empty changeLog. Resync if \"Local replica watermark\" errors arise`,\n      );\n    }\n    return {\n      replicaVersion: this.#replicaVersion,\n      minWatermark: minWatermark ?? this.#replicaVersion,\n    };\n  }\n\n  /**\n   * Makes a best effort to purge the change log. In the event of a database\n   * error, exceptions will be logged and swallowed, so this method is safe\n   * to run in a timeout.\n   */\n  async #purgeOldChanges(): Promise<void> {\n    const initial = [...this.#initialWatermarks];\n    if (initial.length === 0) {\n      this.#lc.warn?.('No initial watermarks to check for cleanup'); // Not expected.\n      return;\n    }\n    const current = [...this.#forwarder.getAcks()];\n    if (current.length === 0) {\n      // Also not expected, but possible (e.g. subscriber connects, then disconnects).\n      // Bail to be safe.\n      this.#lc.warn?.('No subscribers to confirm cleanup');\n      return;\n    }\n    try {\n      const earliestInitial = min(...(initial as AtLeastOne<LexiVersion>));\n      const earliestCurrent = min(...(current as AtLeastOne<LexiVersion>));\n      if (earliestCurrent < earliestInitial) {\n        this.#lc.info?.(\n          `At least one client is behind backup (${earliestCurrent} < ${earliestInitial})`,\n        );\n      } else {\n        this.#lc.info?.(`Purging changes before ${earliestInitial} ...`);\n        const start = performance.now();\n        const deleted = await this.#storer.purgeRecordsBefore(earliestInitial);\n        const elapsed = (performance.now() - start).toFixed(2);\n        this.#lc.info?.(\n          `Purged ${deleted} changes before ${earliestInitial} (${elapsed} ms)`,\n        );\n        this.#initialWatermarks.delete(earliestInitial);\n      }\n    } catch (e) {\n      this.#lc.warn?.(`error purging change log`, e);\n    } finally {\n      if (this.#initialWatermarks.size) {\n        // If there are unpurged watermarks to check, schedule the next purge.\n        this.#state.setTimeout(() => this.#purgeOldChanges(), CLEANUP_DELAY_MS);\n      }\n    }\n  }\n\n  async stop(err?: unknown) {\n    this.#state.stop(this.#lc, err);\n    this.#stream?.changes.cancel();\n    await this.#storer.stop();\n    await this.#source.stop();\n  }\n}\n\n// The delay between receiving an initial, backup-based watermark\n// and performing a check of whether to purge records before it.\n// This delay should be long enough to handle situations like the following:\n//\n// 1. `litestream restore` downloads a backup for the `replication-manager`\n// 2. `replication-manager` starts up and runs this `change-streamer`\n// 3. `zero-cache`s that are running on a different replica connect to this\n//    `change-streamer` after exponential backoff retries.\n//\n// It is possible for a `zero-cache`[3] to be behind the backup restored [1].\n// This cleanup delay (30 seconds) is thus set to be a value comfortably\n// longer than the max delay for exponential backoff (10 seconds) in\n// `services/running-state.ts`. This allows the `zero-cache` [3] to reconnect\n// so that the `change-streamer` can track its progress and know when it has\n// surpassed the initial watermark of the backup [1].\nconst CLEANUP_DELAY_MS = DEFAULT_MAX_RETRY_DELAY_MS * 3;\n\nconst ROLLBACK_JSON = JSON.stringify([\n  'rollback',\n  {tag: 'rollback'},\n] satisfies Rollback);\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;AA+DA,eAAsB,mBACpB,IACA,OACA,QACA,kBACA,mBACA,UACA,cACA,4BACA,mBACA,WACA,WACA,MACA,eAAe,YACiB;CAEhC,MAAM,yBAAyB,IAAI,UAAU,KAAK;CAClD,MAAM,wBACJ,IACA,UACA,mBACA,OACA,WACA,YACF;CAEA,MAAM,EAAC,mBAAkB;CACzB,OAAO,IAAI,mBACT,IACA,OACA,QACA,kBACA,mBACA,UACA,gBACA,cACA,4BACA,WACA,WACA,MACA,YACF;AACF;AAEA,IAAM,8CAA8C;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAkKpD,IAAM,qBAAN,MAA0D;CACxD;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CAEA;CACA;CACA,qCAA8B,IAAI,IAAY;CAU9C,WAAoB,SAAS;CAE7B,aAAsB,mBACpB,eACA,gBACA,kCACF;CACA,iBAA0B,mBACxB,eACA,WACA,qDACF;CAEA;CACA;CACA;CAEA,YACE,IACA,OACA,QACA,kBACA,mBACA,UACA,gBACA,QACA,4BACA,kBACA,WACA,MACA,eAAe,YACf;EACA,KAAK,KAAK;EACV,KAAKA,MAAM,GAAG,YAAY,aAAa,iBAAiB;EACxD,KAAKC,SAAS;EACd,KAAKC,YAAY;EACjB,KAAKC,kBAAkB;EACvB,KAAKC,UAAU;EACf,KAAKC,UAAU,IAAI,OACjB,IACA,OACA,QACA,kBACA,mBACA,UACA,iBACA,aAAY,KAAKS,SAAS,KAAK,KAAK;GAAC;GAAU,SAAS;GAAI,SAAS;EAAE,CAAC,IACxE,QAAO,KAAK,KAAK,GAAG,GACpB,IACF;EACA,KAAKR,aAAa,IAAI,UAAU,IAAI,EAClC,oCACE,KAAK,mCACT,CAAC;EACD,KAAKC,8BAA8B;EACnC,KAAKQ,aAAa;EAClB,KAAKP,aAAa;EAClB,KAAKC,SAAS,IAAI,aAAa,KAAK,IAAI,KAAA,GAAW,YAAY;EAC/D,KAAKO,gBAAgB,EAAC,KAAK,SAAQ;CACrC;CAEA,MAAM,MAAM;EACV,KAAKhB,IAAI,OAAO,wBAAwB;EAExC,KAAKM,WAAW,qBAAqB;EAErC,MAAM,YAAY,MAAM,KAAKF,QAAQ,iBAAiB;EACtD,IAAI,WACF,KAAKY,cAAc,YAAY;EAKjC,MAAM,KAAKX,QAAQ,gBAAgB,KAAKU,UAAU;EAClD,KAAKA,aAAa;EAIlB,MAAM,sBAAsB,wBAAwB,KAAK;EAEzD,OAAO,KAAKN,OAAO,UAAU,GAAG;GAC9B,IAAI;GACJ,IAAI,YAA2B;GAC/B,IAAI,iBAAiB;GACrB,IAAI;IACF,MAAM,EAAC,eAAe,qBACpB,MAAM,KAAKJ,QAAQ,uCAAuC;IAC5D,MAAM,SAAS,MAAM,KAAKD,QAAQ,YAChC,eACA,gBACF;IACA,KAAKC,QAAQ,IAAI,EAAE,OAAM,MAAK,OAAO,QAAQ,OAAO,CAAC,CAAC;IAEtD,KAAKS,UAAU;IACf,IACE,KAAKL,OAAO,aAAa,IACzB,6CAIA,KAAKF,4BAA4B,QAC/B,KAAKP,KACL,eACA,oBAAoB,eACtB;IAEF,YAAY;IAEZ,WAAW,MAAM,UAAU,OAAO,SAAS;KACzC,MAAM,CAAC,MAAM,OAAO;KACpB,QAAQ,MAAR;MACE,KAAK;OACH,IAAI,IAAI,KACN,KAAKK,QAAQ,OAAO,MAAM;OAE5B,IAAI,IAAI,WAAW;QAKjB,KAAKW,cAAc,YAAY,IAAI;QACnC,KAAKV,WAAW,WAAW,KAAKU,aAAa;OAC/C;OACA;MACF,KAAK;OACH,MAAM,KAAKC,sBAAsB,GAAG;OACpC;MACF,KAAK;OACH,YAAY,OAAO,GAAG;OACtB;MACF,KAAK;OACH,IAAI,cAAc,OAAO,GAAG,WAC1B,MAAM,IAAI,mBACR,oBAAoB,OAAO,GAAG,UAAU,oCAAoC,WAC9E;OAEF,KAAKL,WAAW,IAAI,CAAC;OACrB;MACF;OACE,IAAI,SAAS,QACX,KAAKC,eAAe,IAAI,CAAC;OAE3B,IAAI,cAAc,MAChB,MAAM,IAAI,mBACR,GAAG,KAAK,WAAW,IAAI,IAAI,kCAC7B;OAEF;KACJ;KAEA,MAAM,OAAO,KAAKR,QAAQ,MAAM,WAAW,MAAM;KACjD,MAAM,QAA2B;MAAC;MAAW,OAAO,GAAG;MAAK;KAAI;KAChE,kBAAkB,KAAK;KACvB,IAAI,iBAAiB,qBAEnB,KAAKC,WAAW,QAAQ,KAAK;UACxB;MAQL,MAAM,KAAKA,WAAW,uBAAuB,KAAK;MAClD,iBAAiB;KACnB;KAEA,IAAI,SAAS,YAAY,SAAS,YAChC,YAAY;KAId,MAAM,eAAe,KAAKD,QAAQ,aAAa;KAC/C,IAAI,cACF,MAAM;IAEV;GACF,SAAS,GAAG;IACV,MAAM;GACR,UAAU;IACR,KAAKS,SAAS,QAAQ,OAAO;IAC7B,KAAKA,UAAU,KAAA;GACjB;GAGA,IAAI,WAAW;IACb,KAAKd,IAAI,OAAO,oCAAoC,WAAW;IAC/D,KAAKK,QAAQ,MAAM;IACnB,KAAKC,WAAW,QAAQ;KAAC;KAAW;KAAY;IAAa,CAAC;GAChE;GAGA,MAAM,QAAQ,IAAI;IAChB,KAAKD,QAAQ,KAAK;IAClB,KAAKI,OAAO,QAAQ,KAAKT,KAAK,GAAG;IACjC,KAAKS,OAAO,aAAa,8CACrB,qBACE,KAAKT,KACL,uBAAuB,KAAKA,KAAK,eAAe,GAAG,CACrD,IACA;GACN,CAAC;EACH;EAEA,KAAKM,WAAW,oBAAoB;EACpC,KAAKN,IAAI,OAAO,wBAAwB;CAC1C;CAEA,MAAMiB,sBAAsB,KAA6B;EACvD,KAAKjB,IAAI,OAAO,4BAA4B,GAAG;EAC/C,MAAM,EAAC,QAAO;EAEd,QAAQ,KAAR;GACE,KAAK;IACH,MAAM,kBAAkB,KAAKE,WAAW,KAAKD,MAAM;IACnD,MAAM,wBACJ,KAAKD,KACL,eACA,IAAI,WAAW,mBACf,IAAI,YACN;IACA,IAAI,KAAKQ,YAAY;KACnB,KAAKR,IAAI,OAAO,8BAA8B;KAC9C,MAAM,KAAK,KAAK,IAAI,gBAAgB,CAAC;IACvC;IACA;GACF,SACE,YAAY,GAAG;EACnB;CACF;CAEA,UAAU,KAAiD;EACzD,MAAM,EAAC,iBAAiB,IAAI,MAAM,gBAAgB,cAAa;EAC/D,IAAI,SAAS,WACX,KAAKW,SAAS,QAAQ;EAExB,MAAM,aAAa,aAAa,OAAe,EAC7C,eAAe,KAAKL,WAAW,OAAO,UAAU,EAClD,CAAC;EACD,MAAM,aAAa,IAAI,WACrB,iBACA,IACA,WACA,kBACM,KAAKU,aACb;EACA,IAAI,mBAAmB,KAAKb,iBAAiB;GAC3C,KAAKH,IAAI,OACP,2CAA2C,gBAC7C;GACA,WAAW,MACT,GACA,8BACE,KAAKG,gBACN,cAAc,eAAe,EAChC;EACF,OAAO;GACL,KAAKH,IAAI,QAAQ,qBAAqB,WAAW,IAAI;GAErD,KAAKM,WAAW,IAAI,UAAU;GAC9B,KAAKD,QAAQ,QAAQ,YAAY,IAAI;EACvC;EACA,OAAO,QAAQ,QAAQ,UAAU;CACnC;CAEA,gBAAgB,WAAmB;EACjC,MAAM,WAAW,KAAKK,mBAAmB;EACzC,KAAKA,mBAAmB,IAAI,SAAS;EAErC,IAAI,aAAa,GACf,KAAKD,OAAO,iBAAiB,KAAKS,iBAAiB,GAAG,gBAAgB;CAE1E;CAEA,MAAM,oBAGH;EACD,MAAM,eAAe,MAAM,KAAKb,QAAQ,0BAA0B;EAClE,IAAI,CAAC,cACH,KAAKL,IAAI,OACP,8EACF;EAEF,OAAO;GACL,gBAAgB,KAAKG;GACrB,cAAc,gBAAgB,KAAKA;EACrC;CACF;;;;;;CAOA,MAAMe,mBAAkC;EACtC,MAAM,UAAU,CAAC,GAAG,KAAKR,kBAAkB;EAC3C,IAAI,QAAQ,WAAW,GAAG;GACxB,KAAKV,IAAI,OAAO,4CAA4C;GAC5D;EACF;EACA,MAAM,UAAU,CAAC,GAAG,KAAKM,WAAW,QAAQ,CAAC;EAC7C,IAAI,QAAQ,WAAW,GAAG;GAGxB,KAAKN,IAAI,OAAO,mCAAmC;GACnD;EACF;EACA,IAAI;GACF,MAAM,kBAAkB,IAAI,GAAI,OAAmC;GACnE,MAAM,kBAAkB,IAAI,GAAI,OAAmC;GACnE,IAAI,kBAAkB,iBACpB,KAAKA,IAAI,OACP,yCAAyC,gBAAgB,KAAK,gBAAgB,EAChF;QACK;IACL,KAAKA,IAAI,OAAO,0BAA0B,gBAAgB,KAAK;IAC/D,MAAM,QAAQ,YAAY,IAAI;IAC9B,MAAM,UAAU,MAAM,KAAKK,QAAQ,mBAAmB,eAAe;IACrE,MAAM,WAAW,YAAY,IAAI,IAAI,OAAO,QAAQ,CAAC;IACrD,KAAKL,IAAI,OACP,UAAU,QAAQ,kBAAkB,gBAAgB,IAAI,QAAQ,KAClE;IACA,KAAKU,mBAAmB,OAAO,eAAe;GAChD;EACF,SAAS,GAAG;GACV,KAAKV,IAAI,OAAO,4BAA4B,CAAC;EAC/C,UAAU;GACR,IAAI,KAAKU,mBAAmB,MAE1B,KAAKD,OAAO,iBAAiB,KAAKS,iBAAiB,GAAG,gBAAgB;EAE1E;CACF;CAEA,MAAM,KAAK,KAAe;EACxB,KAAKT,OAAO,KAAK,KAAKT,KAAK,GAAG;EAC9B,KAAKc,SAAS,QAAQ,OAAO;EAC7B,MAAM,KAAKT,QAAQ,KAAK;EACxB,MAAM,KAAKD,QAAQ,KAAK;CAC1B;AACF;AAiBA,IAAM,mBAAmB,6BAA6B;AAEtD,IAAM,gBAAgB,KAAK,UAAU,CACnC,YACA,EAAC,KAAK,WAAU,CAClB,CAAoB"}
+{"version":3,"file":"change-streamer-service.js","names":["#lc","#shard","#changeDB","#replicaVersion","#source","#storer","#forwarder","#replicationStatusPublisher","#autoReset","#state","#initialWatermarks","#serving","#txCounter","#changeCounter","#stream","#purgeLock","#latestStatus","#handleControlMessage","#purgeOldChanges"],"sources":["../../../../../../zero-cache/src/services/change-streamer/change-streamer-service.ts"],"sourcesContent":["import {getDefaultHighWaterMark} from 'node:stream';\nimport type {LogContext} from '@rocicorp/logger';\nimport {resolver} from '@rocicorp/resolver';\nimport {unreachable} from '../../../../shared/src/asserts.ts';\nimport {promiseVoid} from '../../../../shared/src/resolved-promises.ts';\nimport {publishCriticalEvent} from '../../observability/events.ts';\nimport {getOrCreateCounter} from '../../observability/metrics.ts';\nimport {\n  min,\n  type AtLeastOne,\n  type LexiVersion,\n} from '../../types/lexi-version.ts';\nimport type {PostgresDB} from '../../types/pg.ts';\nimport type {ShardID} from '../../types/shards.ts';\nimport type {Source} from '../../types/streams.ts';\nimport {Subscription} from '../../types/subscription.ts';\nimport type {\n  ChangeSource,\n  ChangeStream,\n} from '../change-source/change-source.ts';\nimport {\n  type ChangeStreamControl,\n  type ChangeStreamData,\n  type Rollback,\n} from '../change-source/protocol/current/downstream.ts';\nimport {\n  publishReplicationError,\n  replicationStatusError,\n  type ReplicationStatusPublisher,\n} from '../replicator/replication-status.ts';\nimport type {SubscriptionState} from '../replicator/schema/replication-state.ts';\nimport {\n  DEFAULT_MAX_RETRY_DELAY_MS,\n  RunningState,\n  UnrecoverableError,\n} from '../running-state.ts';\nimport {\n  type ChangeStreamerService,\n  type Status,\n  type SubscriberContext,\n} from './change-streamer.ts';\nimport * as ErrorType from './error-type-enum.ts';\nimport {Forwarder} from './forwarder.ts';\nimport {initChangeStreamerSchema} from './schema/init.ts';\nimport {\n  AutoResetSignal,\n  ensureReplicationConfig,\n  markResetRequired,\n} from './schema/tables.ts';\nimport {\n  Storer,\n  type PurgeLock,\n  type TuningOptions as StorerOptions,\n} from './storer.ts';\nimport {Subscriber} from './subscriber.ts';\n\nexport type TuningOptions = StorerOptions & {\n  flowControlConsensusPaddingSeconds: number;\n};\n\n/**\n * Performs initialization and schema migrations to initialize a ChangeStreamerImpl.\n */\nexport async function initializeStreamer(\n  lc: LogContext,\n  shard: ShardID,\n  taskID: string,\n  discoveryAddress: string,\n  discoveryProtocol: string,\n  changeDB: PostgresDB,\n  changeSource: ChangeSource,\n  replicationStatusPublisher: ReplicationStatusPublisher,\n  subscriptionState: SubscriptionState,\n  purgeLock: PurgeLock | null,\n  autoReset: boolean,\n  opts: TuningOptions,\n  setTimeoutFn = setTimeout,\n): Promise<ChangeStreamerService> {\n  // Make sure the ChangeLog DB is set up.\n  await initChangeStreamerSchema(lc, changeDB, shard);\n  await ensureReplicationConfig(\n    lc,\n    changeDB,\n    subscriptionState,\n    shard,\n    autoReset,\n    setTimeoutFn,\n  );\n\n  const {replicaVersion} = subscriptionState;\n  return new ChangeStreamerImpl(\n    lc,\n    shard,\n    taskID,\n    discoveryAddress,\n    discoveryProtocol,\n    changeDB,\n    replicaVersion,\n    changeSource,\n    replicationStatusPublisher,\n    purgeLock,\n    autoReset,\n    opts,\n    setTimeoutFn,\n  );\n}\n\nconst REPLICATION_STATUS_ERROR_DELAY_THRESHOLD_MS = 5000;\n\nexport type ChangeTag = ChangeStreamData[1]['tag'];\n\n/**\n * Internally all Downstream messages (not just commits) are given a watermark.\n * These are used for internal ordering for:\n * 1. Replaying new changes in the Storer\n * 2. Filtering old changes in the Subscriber\n *\n * However, only the watermark for `Commit` messages are exposed to\n * subscribers, as that is the only semantically correct watermark to\n * use for tracking a position in a replication stream.\n *\n * Additionally, the ChangeStreamData is eagerly stringified once, after which\n * the string is passed to the changeLog and all subscribers, eliminating\n * redundant stringification and reducing GC churn.\n */\nexport type WatermarkedChange = [\n  watermark: string,\n  tag: ChangeTag,\n  json: string,\n];\n\n/**\n * Upstream-agnostic dispatch of messages in a {@link ChangeStreamMessage} to a\n * {@link Forwarder} and {@link Storer} to execute the forward-store-ack\n * procedure described in {@link ChangeStreamer}.\n *\n * ### Subscriber Catchup\n *\n * Connecting clients first need to be \"caught up\" to the current watermark\n * (from stored change log entries) before new entries are forwarded to\n * them. This is non-trivial because the replication stream may be in the\n * middle of a pending streamed Transaction for which some entries have\n * already been forwarded but are not yet committed to the store.\n *\n *\n * ```\n * ------------------------------- - - - - - - - - - - - - - - - - - - -\n * | Historic changes in storage |  Pending (streamed) tx  |   Next tx\n * ------------------------------- - - - - - - - - - - - - - - - - - - -\n *                                           Replication stream\n *                                           >  >  >  >  >  >  >  >  >\n *           ^  ---> required catchup --->   ^\n * Subscriber watermark               Subscription begins\n * ```\n *\n * Preemptively buffering the changes of every pending transaction\n * would be wasteful and consume too much memory for large transactions.\n *\n * Instead, the streamer synchronously dispatches changes and subscriptions\n * to the {@link Forwarder} and the {@link Storer} such that the two\n * components are aligned as to where in the stream the subscription started.\n * The two components then coordinate catchup and handoff via the\n * {@link Subscriber} object with the following algorithm:\n *\n * * If the streamer is in the middle of a pending Transaction, the\n *   Subscriber is \"queued\" on both the Forwarder and the Storer. In this\n *   state, new changes are *not* forwarded to the Subscriber, and catchup\n *   is not yet executed.\n * * Once the commit message for the pending Transaction is processed\n *   by the Storer, it begins catchup on the Subscriber (with a READONLY\n *   snapshot so that it does not block subsequent storage operations).\n *   This catchup is thus guaranteed to load the change log entries of\n *   that last Transaction.\n * * When the Forwarder processes that same commit message, it moves the\n *   Subscriber from the \"queued\" to the \"active\" set of clients such that\n *   the Subscriber begins receiving new changes, starting from the next\n *   Transaction.\n * * The Subscriber does not forward those changes, however, if its catchup\n *   is not complete. Until then, it buffers the changes in memory.\n * * Once catchup is complete, the buffered changes are immediately sent\n *   and the Subscriber henceforth forwards changes as they are received.\n *\n * In the (common) case where the streamer is not in the middle of a pending\n * transaction when a subscription begins, the Storer begins catchup\n * immediately and the Forwarder directly adds the Subscriber to its active\n * set. However, the Subscriber still buffers any forwarded messages until\n * its catchup is complete.\n *\n * ### Watermarks and ordering\n *\n * The ChangeStreamerService depends on its {@link ChangeSource} to send\n * changes in contiguous [`begin`, `data` ..., `data`, `commit`] sequences\n * in commit order. This follows Postgres's Logical Replication Protocol\n * Message Flow:\n *\n * https://www.postgresql.org/docs/16/protocol-logical-replication.html#PROTOCOL-LOGICAL-MESSAGES-FLOW\n *\n * > The logical replication protocol sends individual transactions one by one.\n * > This means that all messages between a pair of Begin and Commit messages belong to the same transaction.\n *\n * In order to correctly replay (new) and filter (old) messages to subscribers\n * at different points in the replication stream, these changes must be assigned\n * watermarks such that they preserve the order in which they were received\n * from the ChangeSource.\n *\n * A previous implementation incorrectly derived these watermarks from the Postgres\n * Log Sequence Numbers (LSN) of each message. However, LSNs from concurrent,\n * non-conflicting transactions can overlap, which can result in a `begin` message\n * with an earlier LSN arriving after a `commit` message. For example, the\n * changes for these transactions:\n *\n * ```\n * LSN:   1     2     3  4    5   6   7     8   9      10\n * tx1: begin  data     data     data     commit\n * tx2:             begin    data    data      data  commit\n * ```\n *\n * will arrive as:\n *\n * ```\n * begin1, data2, data4, data6, commit8, begin3, data5, data7, data9, commit10\n * ```\n *\n * Thus, LSN of non-commit messages are not suitable for tracking the sorting\n * order of the replication stream.\n *\n * Instead, the ChangeStreamer uses the following algorithm for deterministic\n * catchup and filtering of changes:\n *\n * * A `commit` message is assigned to a watermark corresponding to its LSN.\n *   These are guaranteed to be in commit order by definition.\n *\n * * `begin` and `data` messages are assigned to the watermark of the\n *   preceding `commit` (the previous transaction, or the replication\n *   slot's starting LSN) plus 1. This guarantees that they will be sorted\n *   after the previously commit transaction even if their LSNs came before it.\n *   This is referred to as the `preCommitWatermark`.\n *\n * * In the ChangeLog DB, messages have a secondary sort column `pos`, which is\n *   the position of the message within its transaction, with the `begin` message\n *   starting at `0`. This guarantees that `begin` and `data` messages will be\n *   fetched in the original ChangeSource order during catchup.\n *\n * `begin` and `data` messages share the same watermark, but this is sufficient for\n * Subscriber filtering because subscribers only know about the `commit` watermarks\n * exposed in the `Downstream` `Commit` message. The Subscriber object thus compares\n * the internal watermarks of the incoming messages against the commit watermark of\n * the caller, updating the watermark at every `Commit` message that is forwarded.\n *\n * ### Cleanup\n *\n * As mentioned in the {@link ChangeStreamer} documentation: \"the ChangeStreamer\n * uses a combination of [the \"initial\", i.e. backup-derived watermark and] ACK\n * responses from connected subscribers to determine the watermark up\n * to which it is safe to purge old change log entries.\"\n *\n * More concretely:\n *\n * * The `initial`, backup-derived watermark is the earliest to which cleanup\n *   should ever happen.\n *\n * * However, it is possible for the replica backup to be *ahead* of a connected\n *   subscriber; and if a network error causes that subscriber to retry from its\n *   last watermark, the change streamer must support it.\n *\n * Thus, before cleaning up to an `initial` backup-derived watermark, the change\n * streamer first confirms that all connected subscribers have also passed\n * that watermark.\n */\nclass ChangeStreamerImpl implements ChangeStreamerService {\n  readonly id: string;\n  readonly #lc: LogContext;\n  readonly #shard: ShardID;\n  readonly #changeDB: PostgresDB;\n  readonly #replicaVersion: string;\n  readonly #source: ChangeSource;\n  readonly #storer: Storer;\n  readonly #forwarder: Forwarder;\n  readonly #replicationStatusPublisher: ReplicationStatusPublisher;\n\n  readonly #autoReset: boolean;\n  readonly #state: RunningState;\n  readonly #initialWatermarks = new Set<string>();\n\n  // Starting the (Postgres) ChangeStream results in killing the previous\n  // Postgres subscriber, potentially creating a gap in which the old\n  // change-streamer has shut down and the new change-streamer has not yet\n  // been recognized as \"healthy\" (and thus does not get any requests).\n  //\n  // To minimize this gap, delay starting the ChangeStream until the first\n  // request from a `serving` replicator, indicating that higher level\n  // load-balancing / routing logic has begun routing requests to this task.\n  readonly #serving = resolver();\n\n  readonly #txCounter = getOrCreateCounter(\n    'replication',\n    'transactions',\n    'Count of replicated transactions',\n  );\n  readonly #changeCounter = getOrCreateCounter(\n    'replication',\n    'changes',\n    'Count of replicated changes (DML or DDL statements)',\n  );\n\n  #latestStatus: Status;\n  #purgeLock: PurgeLock | null;\n  #stream: ChangeStream | undefined;\n\n  constructor(\n    lc: LogContext,\n    shard: ShardID,\n    taskID: string,\n    discoveryAddress: string,\n    discoveryProtocol: string,\n    changeDB: PostgresDB,\n    replicaVersion: string,\n    source: ChangeSource,\n    replicationStatusPublisher: ReplicationStatusPublisher,\n    initialPurgeLock: PurgeLock | null,\n    autoReset: boolean,\n    opts: TuningOptions,\n    setTimeoutFn = setTimeout,\n  ) {\n    this.id = `change-streamer`;\n    this.#lc = lc.withContext('component', 'change-streamer');\n    this.#shard = shard;\n    this.#changeDB = changeDB;\n    this.#replicaVersion = replicaVersion;\n    this.#source = source;\n    this.#storer = new Storer(\n      lc,\n      shard,\n      taskID,\n      discoveryAddress,\n      discoveryProtocol,\n      changeDB,\n      replicaVersion,\n      consumed => this.#stream?.acks.push(['status', consumed[1], consumed[2]]),\n      err => this.stop(err),\n      opts,\n    );\n    this.#forwarder = new Forwarder(lc, {\n      flowControlConsensusPaddingSeconds:\n        opts.flowControlConsensusPaddingSeconds,\n    });\n    this.#replicationStatusPublisher = replicationStatusPublisher;\n    this.#purgeLock = initialPurgeLock;\n    this.#autoReset = autoReset;\n    this.#state = new RunningState(this.id, undefined, setTimeoutFn);\n    this.#latestStatus = {tag: 'status'};\n  }\n\n  async run() {\n    this.#lc.info?.('starting change stream');\n\n    this.#forwarder.startProgressMonitor();\n\n    const lagReport = await this.#source.startLagReporter();\n    if (lagReport) {\n      this.#latestStatus.lagReport = lagReport;\n    }\n\n    // Once this change-streamer acquires \"ownership\" of the change DB,\n    // it is safe to start the storer.\n    await this.#storer.assumeOwnership(this.#purgeLock);\n    this.#purgeLock = null;\n\n    // The threshold in (estimated number of) bytes to send() on subscriber\n    // websockets before `await`-ing the I/O buffers to be ready for more.\n    const flushBytesThreshold = getDefaultHighWaterMark(false);\n\n    while (this.#state.shouldRun()) {\n      let err: unknown;\n      let watermark: string | null = null;\n      let unflushedBytes = 0;\n      try {\n        const {lastWatermark, backfillRequests} =\n          await this.#storer.getStartStreamInitializationParameters();\n        const stream = await this.#source.startStream(\n          lastWatermark,\n          backfillRequests,\n        );\n        this.#storer.run().catch(e => stream.changes.cancel(e));\n\n        this.#stream = stream;\n        if (\n          this.#state.resetBackoff() >\n          REPLICATION_STATUS_ERROR_DELAY_THRESHOLD_MS\n        ) {\n          // After recovering from a backoff for which a replication status\n          // error was published, publish an OK status\n          this.#replicationStatusPublisher.publish(\n            this.#lc,\n            'Replicating',\n            `Replicating from ${lastWatermark}`,\n          );\n        }\n        watermark = null;\n\n        for await (const change of stream.changes) {\n          const [type, msg] = change;\n          switch (type) {\n            case 'status':\n              if (msg.ack) {\n                this.#storer.status(change); // storer acks once it gets through its queue\n              }\n              if (msg.lagReport) {\n                // Lag reports are not stored in the cdc change log, but rather\n                // only forwarded on \"live\" connections. When a new subscriber\n                // is catching up, it is initialized with the #latestStatus\n                // from which it can measure lag while catching up.\n                this.#latestStatus.lagReport = msg.lagReport;\n                this.#forwarder.sendStatus(this.#latestStatus);\n              }\n              continue;\n            case 'control':\n              await this.#handleControlMessage(msg);\n              continue; // control messages are not stored/forwarded\n            case 'begin':\n              watermark = change[2].commitWatermark;\n              break;\n            case 'commit':\n              if (watermark !== change[2].watermark) {\n                throw new UnrecoverableError(\n                  `commit watermark ${change[2].watermark} does not match 'begin' watermark ${watermark}`,\n                );\n              }\n              this.#txCounter.add(1);\n              break;\n            default:\n              if (type === 'data') {\n                this.#changeCounter.add(1);\n              }\n              if (watermark === null) {\n                throw new UnrecoverableError(\n                  `${type} change (${msg.tag}) received before 'begin' message`,\n                );\n              }\n              break;\n          }\n\n          const json = this.#storer.store(watermark, change);\n          const entry: WatermarkedChange = [watermark, change[1].tag, json];\n          unflushedBytes += json.length;\n          if (unflushedBytes < flushBytesThreshold) {\n            // pipeline changes until flushBytesThreshold\n            this.#forwarder.forward(entry);\n          } else {\n            // Wait for messages to clear socket buffers to ensure that they\n            // make their way to subscribers. Without this `await`, the\n            // messages end up being buffered in this process, which:\n            // (1) results in memory pressure and increased GC activity\n            // (2) prevents subscribers from processing the messages as they\n            //     arrive, instead getting them in a large batch after being\n            //     idle while they were queued (causing further delays).\n            await this.#forwarder.forwardWithFlowControl(entry);\n            unflushedBytes = 0;\n          }\n\n          if (type === 'commit' || type === 'rollback') {\n            watermark = null;\n          }\n\n          // Allow the storer to exert back pressure.\n          const readyForMore = this.#storer.readyForMore();\n          if (readyForMore) {\n            await readyForMore;\n          }\n        }\n      } catch (e) {\n        err = e;\n      } finally {\n        this.#stream?.changes.cancel();\n        this.#stream = undefined;\n      }\n\n      // When the change stream is interrupted, abort any pending transaction.\n      if (watermark) {\n        this.#lc.warn?.(`aborting interrupted transaction ${watermark}`);\n        this.#storer.abort();\n        this.#forwarder.forward([watermark, 'rollback', ROLLBACK_JSON]);\n      }\n\n      // Backoff and drain any pending entries in the storer before reconnecting.\n      await Promise.all([\n        this.#storer.stop(),\n        this.#state.backoff(this.#lc, err),\n        this.#state.retryDelay > REPLICATION_STATUS_ERROR_DELAY_THRESHOLD_MS\n          ? publishCriticalEvent(\n              this.#lc,\n              replicationStatusError(this.#lc, 'Replicating', err),\n            )\n          : promiseVoid,\n      ]);\n    }\n\n    this.#forwarder.stopProgressMonitor();\n    this.#lc.info?.('ChangeStreamer stopped');\n  }\n\n  async #handleControlMessage(msg: ChangeStreamControl[1]) {\n    this.#lc.info?.('received control message', msg);\n    const {tag} = msg;\n\n    switch (tag) {\n      case 'reset-required':\n        await markResetRequired(this.#changeDB, this.#shard);\n        await publishReplicationError(\n          this.#lc,\n          'Replicating',\n          msg.message ?? 'Resync required',\n          msg.errorDetails,\n        );\n        if (this.#autoReset) {\n          this.#lc.warn?.('shutting down for auto-reset');\n          await this.stop(new AutoResetSignal());\n        }\n        break;\n      default:\n        unreachable(tag);\n    }\n  }\n\n  subscribe(ctx: SubscriberContext): Promise<Source<string>> {\n    const {protocolVersion, id, mode, replicaVersion, watermark} = ctx;\n    if (mode === 'serving') {\n      this.#serving.resolve();\n    }\n    const downstream = Subscription.create<string>({\n      cleanup: () => this.#forwarder.remove(subscriber),\n    });\n    const subscriber = new Subscriber(\n      protocolVersion,\n      id,\n      watermark,\n      downstream,\n      () => this.#latestStatus,\n    );\n    if (replicaVersion !== this.#replicaVersion) {\n      this.#lc.warn?.(\n        `rejecting subscriber at replica version ${replicaVersion}`,\n      );\n      subscriber.close(\n        ErrorType.WrongReplicaVersion,\n        `current replica version is ${\n          this.#replicaVersion\n        } (requested ${replicaVersion})`,\n      );\n    } else {\n      this.#lc.debug?.(`adding subscriber ${subscriber.id}`);\n\n      this.#forwarder.add(subscriber);\n      this.#storer.catchup(subscriber, mode);\n    }\n    return Promise.resolve(downstream);\n  }\n\n  scheduleCleanup(watermark: string) {\n    const origSize = this.#initialWatermarks.size;\n    this.#initialWatermarks.add(watermark);\n\n    if (origSize === 0) {\n      this.#state.setTimeout(() => this.#purgeOldChanges(), CLEANUP_DELAY_MS);\n    }\n  }\n\n  async getChangeLogState(): Promise<{\n    replicaVersion: string;\n    minWatermark: string;\n  }> {\n    const minWatermark = await this.#storer.getMinWatermarkForCatchup();\n    if (!minWatermark) {\n      this.#lc.warn?.(\n        `Unexpected empty changeLog. Resync if \"Local replica watermark\" errors arise`,\n      );\n    }\n    return {\n      replicaVersion: this.#replicaVersion,\n      minWatermark: minWatermark ?? this.#replicaVersion,\n    };\n  }\n\n  /**\n   * Makes a best effort to purge the change log. In the event of a database\n   * error, exceptions will be logged and swallowed, so this method is safe\n   * to run in a timeout.\n   */\n  async #purgeOldChanges(): Promise<void> {\n    const initial = [...this.#initialWatermarks];\n    if (initial.length === 0) {\n      this.#lc.warn?.('No initial watermarks to check for cleanup'); // Not expected.\n      return;\n    }\n    const current = [...this.#forwarder.getAcks()];\n    if (current.length === 0) {\n      // Also not expected, but possible (e.g. subscriber connects, then disconnects).\n      // Bail to be safe.\n      this.#lc.warn?.('No subscribers to confirm cleanup');\n      return;\n    }\n    try {\n      const earliestInitial = min(...(initial as AtLeastOne<LexiVersion>));\n      const earliestCurrent = min(...(current as AtLeastOne<LexiVersion>));\n      if (earliestCurrent < earliestInitial) {\n        this.#lc.info?.(\n          `At least one client is behind backup (${earliestCurrent} < ${earliestInitial})`,\n        );\n      } else {\n        this.#lc.info?.(`Purging changes before ${earliestInitial} ...`);\n        const start = performance.now();\n        const deleted = await this.#storer.purgeRecordsBefore(earliestInitial);\n        const elapsed = (performance.now() - start).toFixed(2);\n        this.#lc.info?.(\n          `Purged ${deleted} changes before ${earliestInitial} (${elapsed} ms)`,\n        );\n        this.#initialWatermarks.delete(earliestInitial);\n      }\n    } catch (e) {\n      this.#lc.warn?.(`error purging change log`, e);\n    } finally {\n      if (this.#initialWatermarks.size) {\n        // If there are unpurged watermarks to check, schedule the next purge.\n        this.#state.setTimeout(() => this.#purgeOldChanges(), CLEANUP_DELAY_MS);\n      }\n    }\n  }\n\n  async stop(err?: unknown) {\n    this.#state.stop(this.#lc, err);\n    this.#stream?.changes.cancel();\n    await this.#storer.stop();\n    await this.#source.stop();\n  }\n}\n\n// The delay between receiving an initial, backup-based watermark\n// and performing a check of whether to purge records before it.\n// This delay should be long enough to handle situations like the following:\n//\n// 1. `litestream restore` downloads a backup for the `replication-manager`\n// 2. `replication-manager` starts up and runs this `change-streamer`\n// 3. `zero-cache`s that are running on a different replica connect to this\n//    `change-streamer` after exponential backoff retries.\n//\n// It is possible for a `zero-cache`[3] to be behind the backup restored [1].\n// This cleanup delay (30 seconds) is thus set to be a value comfortably\n// longer than the max delay for exponential backoff (10 seconds) in\n// `services/running-state.ts`. This allows the `zero-cache` [3] to reconnect\n// so that the `change-streamer` can track its progress and know when it has\n// surpassed the initial watermark of the backup [1].\nconst CLEANUP_DELAY_MS = DEFAULT_MAX_RETRY_DELAY_MS * 3;\n\nconst ROLLBACK_JSON = JSON.stringify([\n  'rollback',\n  {tag: 'rollback'},\n] satisfies Rollback);\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;AA+DA,eAAsB,mBACpB,IACA,OACA,QACA,kBACA,mBACA,UACA,cACA,4BACA,mBACA,WACA,WACA,MACA,eAAe,YACiB;AAEhC,OAAM,yBAAyB,IAAI,UAAU,MAAM;AACnD,OAAM,wBACJ,IACA,UACA,mBACA,OACA,WACA,aACD;CAED,MAAM,EAAC,mBAAkB;AACzB,QAAO,IAAI,mBACT,IACA,OACA,QACA,kBACA,mBACA,UACA,gBACA,cACA,4BACA,WACA,WACA,MACA,aACD;;AAGH,IAAM,8CAA8C;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAkKpD,IAAM,qBAAN,MAA0D;CACxD;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CAEA;CACA;CACA,qCAA8B,IAAI,KAAa;CAU/C,WAAoB,UAAU;CAE9B,aAAsB,mBACpB,eACA,gBACA,mCACD;CACD,iBAA0B,mBACxB,eACA,WACA,sDACD;CAED;CACA;CACA;CAEA,YACE,IACA,OACA,QACA,kBACA,mBACA,UACA,gBACA,QACA,4BACA,kBACA,WACA,MACA,eAAe,YACf;AACA,OAAK,KAAK;AACV,QAAA,KAAW,GAAG,YAAY,aAAa,kBAAkB;AACzD,QAAA,QAAc;AACd,QAAA,WAAiB;AACjB,QAAA,iBAAuB;AACvB,QAAA,SAAe;AACf,QAAA,SAAe,IAAI,OACjB,IACA,OACA,QACA,kBACA,mBACA,UACA,iBACA,aAAY,MAAA,QAAc,KAAK,KAAK;GAAC;GAAU,SAAS;GAAI,SAAS;GAAG,CAAC,GACzE,QAAO,KAAK,KAAK,IAAI,EACrB,KACD;AACD,QAAA,YAAkB,IAAI,UAAU,IAAI,EAClC,oCACE,KAAK,oCACR,CAAC;AACF,QAAA,6BAAmC;AACnC,QAAA,YAAkB;AAClB,QAAA,YAAkB;AAClB,QAAA,QAAc,IAAI,aAAa,KAAK,IAAI,KAAA,GAAW,aAAa;AAChE,QAAA,eAAqB,EAAC,KAAK,UAAS;;CAGtC,MAAM,MAAM;AACV,QAAA,GAAS,OAAO,yBAAyB;AAEzC,QAAA,UAAgB,sBAAsB;EAEtC,MAAM,YAAY,MAAM,MAAA,OAAa,kBAAkB;AACvD,MAAI,UACF,OAAA,aAAmB,YAAY;AAKjC,QAAM,MAAA,OAAa,gBAAgB,MAAA,UAAgB;AACnD,QAAA,YAAkB;EAIlB,MAAM,sBAAsB,wBAAwB,MAAM;AAE1D,SAAO,MAAA,MAAY,WAAW,EAAE;GAC9B,IAAI;GACJ,IAAI,YAA2B;GAC/B,IAAI,iBAAiB;AACrB,OAAI;IACF,MAAM,EAAC,eAAe,qBACpB,MAAM,MAAA,OAAa,wCAAwC;IAC7D,MAAM,SAAS,MAAM,MAAA,OAAa,YAChC,eACA,iBACD;AACD,UAAA,OAAa,KAAK,CAAC,OAAM,MAAK,OAAO,QAAQ,OAAO,EAAE,CAAC;AAEvD,UAAA,SAAe;AACf,QACE,MAAA,MAAY,cAAc,GAC1B,4CAIA,OAAA,2BAAiC,QAC/B,MAAA,IACA,eACA,oBAAoB,gBACrB;AAEH,gBAAY;AAEZ,eAAW,MAAM,UAAU,OAAO,SAAS;KACzC,MAAM,CAAC,MAAM,OAAO;AACpB,aAAQ,MAAR;MACE,KAAK;AACH,WAAI,IAAI,IACN,OAAA,OAAa,OAAO,OAAO;AAE7B,WAAI,IAAI,WAAW;AAKjB,cAAA,aAAmB,YAAY,IAAI;AACnC,cAAA,UAAgB,WAAW,MAAA,aAAmB;;AAEhD;MACF,KAAK;AACH,aAAM,MAAA,qBAA2B,IAAI;AACrC;MACF,KAAK;AACH,mBAAY,OAAO,GAAG;AACtB;MACF,KAAK;AACH,WAAI,cAAc,OAAO,GAAG,UAC1B,OAAM,IAAI,mBACR,oBAAoB,OAAO,GAAG,UAAU,oCAAoC,YAC7E;AAEH,aAAA,UAAgB,IAAI,EAAE;AACtB;MACF;AACE,WAAI,SAAS,OACX,OAAA,cAAoB,IAAI,EAAE;AAE5B,WAAI,cAAc,KAChB,OAAM,IAAI,mBACR,GAAG,KAAK,WAAW,IAAI,IAAI,mCAC5B;AAEH;;KAGJ,MAAM,OAAO,MAAA,OAAa,MAAM,WAAW,OAAO;KAClD,MAAM,QAA2B;MAAC;MAAW,OAAO,GAAG;MAAK;MAAK;AACjE,uBAAkB,KAAK;AACvB,SAAI,iBAAiB,oBAEnB,OAAA,UAAgB,QAAQ,MAAM;UACzB;AAQL,YAAM,MAAA,UAAgB,uBAAuB,MAAM;AACnD,uBAAiB;;AAGnB,SAAI,SAAS,YAAY,SAAS,WAChC,aAAY;KAId,MAAM,eAAe,MAAA,OAAa,cAAc;AAChD,SAAI,aACF,OAAM;;YAGH,GAAG;AACV,UAAM;aACE;AACR,UAAA,QAAc,QAAQ,QAAQ;AAC9B,UAAA,SAAe,KAAA;;AAIjB,OAAI,WAAW;AACb,UAAA,GAAS,OAAO,oCAAoC,YAAY;AAChE,UAAA,OAAa,OAAO;AACpB,UAAA,UAAgB,QAAQ;KAAC;KAAW;KAAY;KAAc,CAAC;;AAIjE,SAAM,QAAQ,IAAI;IAChB,MAAA,OAAa,MAAM;IACnB,MAAA,MAAY,QAAQ,MAAA,IAAU,IAAI;IAClC,MAAA,MAAY,aAAa,8CACrB,qBACE,MAAA,IACA,uBAAuB,MAAA,IAAU,eAAe,IAAI,CACrD,GACD;IACL,CAAC;;AAGJ,QAAA,UAAgB,qBAAqB;AACrC,QAAA,GAAS,OAAO,yBAAyB;;CAG3C,OAAA,qBAA4B,KAA6B;AACvD,QAAA,GAAS,OAAO,4BAA4B,IAAI;EAChD,MAAM,EAAC,QAAO;AAEd,UAAQ,KAAR;GACE,KAAK;AACH,UAAM,kBAAkB,MAAA,UAAgB,MAAA,MAAY;AACpD,UAAM,wBACJ,MAAA,IACA,eACA,IAAI,WAAW,mBACf,IAAI,aACL;AACD,QAAI,MAAA,WAAiB;AACnB,WAAA,GAAS,OAAO,+BAA+B;AAC/C,WAAM,KAAK,KAAK,IAAI,iBAAiB,CAAC;;AAExC;GACF,QACE,aAAY,IAAI;;;CAItB,UAAU,KAAiD;EACzD,MAAM,EAAC,iBAAiB,IAAI,MAAM,gBAAgB,cAAa;AAC/D,MAAI,SAAS,UACX,OAAA,QAAc,SAAS;EAEzB,MAAM,aAAa,aAAa,OAAe,EAC7C,eAAe,MAAA,UAAgB,OAAO,WAAW,EAClD,CAAC;EACF,MAAM,aAAa,IAAI,WACrB,iBACA,IACA,WACA,kBACM,MAAA,aACP;AACD,MAAI,mBAAmB,MAAA,gBAAsB;AAC3C,SAAA,GAAS,OACP,2CAA2C,iBAC5C;AACD,cAAW,MACT,GACA,8BACE,MAAA,eACD,cAAc,eAAe,GAC/B;SACI;AACL,SAAA,GAAS,QAAQ,qBAAqB,WAAW,KAAK;AAEtD,SAAA,UAAgB,IAAI,WAAW;AAC/B,SAAA,OAAa,QAAQ,YAAY,KAAK;;AAExC,SAAO,QAAQ,QAAQ,WAAW;;CAGpC,gBAAgB,WAAmB;EACjC,MAAM,WAAW,MAAA,kBAAwB;AACzC,QAAA,kBAAwB,IAAI,UAAU;AAEtC,MAAI,aAAa,EACf,OAAA,MAAY,iBAAiB,MAAA,iBAAuB,EAAE,iBAAiB;;CAI3E,MAAM,oBAGH;EACD,MAAM,eAAe,MAAM,MAAA,OAAa,2BAA2B;AACnE,MAAI,CAAC,aACH,OAAA,GAAS,OACP,+EACD;AAEH,SAAO;GACL,gBAAgB,MAAA;GAChB,cAAc,gBAAgB,MAAA;GAC/B;;;;;;;CAQH,OAAA,kBAAwC;EACtC,MAAM,UAAU,CAAC,GAAG,MAAA,kBAAwB;AAC5C,MAAI,QAAQ,WAAW,GAAG;AACxB,SAAA,GAAS,OAAO,6CAA6C;AAC7D;;EAEF,MAAM,UAAU,CAAC,GAAG,MAAA,UAAgB,SAAS,CAAC;AAC9C,MAAI,QAAQ,WAAW,GAAG;AAGxB,SAAA,GAAS,OAAO,oCAAoC;AACpD;;AAEF,MAAI;GACF,MAAM,kBAAkB,IAAI,GAAI,QAAoC;GACpE,MAAM,kBAAkB,IAAI,GAAI,QAAoC;AACpE,OAAI,kBAAkB,gBACpB,OAAA,GAAS,OACP,yCAAyC,gBAAgB,KAAK,gBAAgB,GAC/E;QACI;AACL,UAAA,GAAS,OAAO,0BAA0B,gBAAgB,MAAM;IAChE,MAAM,QAAQ,YAAY,KAAK;IAC/B,MAAM,UAAU,MAAM,MAAA,OAAa,mBAAmB,gBAAgB;IACtE,MAAM,WAAW,YAAY,KAAK,GAAG,OAAO,QAAQ,EAAE;AACtD,UAAA,GAAS,OACP,UAAU,QAAQ,kBAAkB,gBAAgB,IAAI,QAAQ,MACjE;AACD,UAAA,kBAAwB,OAAO,gBAAgB;;WAE1C,GAAG;AACV,SAAA,GAAS,OAAO,4BAA4B,EAAE;YACtC;AACR,OAAI,MAAA,kBAAwB,KAE1B,OAAA,MAAY,iBAAiB,MAAA,iBAAuB,EAAE,iBAAiB;;;CAK7E,MAAM,KAAK,KAAe;AACxB,QAAA,MAAY,KAAK,MAAA,IAAU,IAAI;AAC/B,QAAA,QAAc,QAAQ,QAAQ;AAC9B,QAAM,MAAA,OAAa,MAAM;AACzB,QAAM,MAAA,OAAa,MAAM;;;AAmB7B,IAAM,mBAAmB,6BAA6B;AAEtD,IAAM,gBAAgB,KAAK,UAAU,CACnC,YACA,EAAC,KAAK,YAAW,CAClB,CAAoB"}

package/out/zero-cache/src/services/change-streamer/change-streamer.js.map

@@ -1 +1 @@
-{"version":3,"file":"change-streamer.js","names":[],"sources":["../../../../../../zero-cache/src/services/change-streamer/change-streamer.ts"],"sourcesContent":["import type {Enum} from '../../../../shared/src/enum.ts';\nimport * as v from '../../../../shared/src/valita.ts';\nimport type {Source} from '../../types/streams.ts';\nimport {changeStreamDataSchema} from '../change-source/protocol/current/downstream.ts';\nimport type {ReplicatorMode} from '../replicator/replicator.ts';\nimport {changeSourceTimingsSchema} from '../replicator/reporter/report-schema.ts';\nimport type {Service} from '../service.ts';\nimport * as ErrorType from './error-type-enum.ts';\n\ntype ErrorType = Enum<typeof ErrorType>;\n\n/**\n * The ChangeStreamer is the component between replicators (\"subscribers\")\n * and a canonical upstream source of changes (e.g. a Postgres logical\n * replication slot). It facilitates multiple subscribers without incurring\n * the associated upstream expense (e.g. PG replication slots are resource\n * intensive) with a \"forward-store-ack\" procedure.\n *\n * * Changes from the upstream source are immediately **forwarded** to\n *   connected subscribers to minimize latency.\n *\n * * They are then **stored** in a separate DB to facilitate catchup\n *   of connecting subscribers that are behind.\n *\n * * **Acknowledgements** are sent upstream after they are successfully\n *   stored.\n *\n * Unlike Postgres replication slots, in which the progress of a static\n * subscriber is tracked in the replication slot, the ChangeStreamer\n * supports a dynamic set of subscribers (i.e.. zero-caches) that can\n * can continually change.\n *\n * However, it is not the case that the ChangeStreamer needs to support\n * arbitrarily old subscribers. Because the replica is continually\n * backed up to a global location and used to initialize new subscriber\n * tasks, an initial subscription request from a subscriber constitutes\n * a signal for how \"behind\" a new subscriber task can be. This is\n * reflected in the {@link SubscriberContext}, which indicates whether\n * the watermark corresponds to an \"initial\" watermark derived from the\n * replica at task startup.\n *\n * The ChangeStreamer uses a combination of this signal with ACK\n * responses from connected subscribers to determine the watermark up\n * to which it is safe to purge old change log entries.\n */\nexport interface ChangeStreamer {\n  /**\n   * Subscribes to changes based on the supplied subscriber `ctx`,\n   * which indicates the watermark at which the subscriber is up to\n   * date.\n   */\n  subscribe(ctx: SubscriberContext): Promise<Source<Downstream>>;\n}\n\n// v1: v0.18\n//   - Client-side support for JSON_FORMAT. Introduced in 0.18.\n// v2: v0.19\n//   - Adds the \"status\" message which is initially used to signal that the\n//     subscription is valid (i.e. starting at the requested watermark).\n// v3: v0.20\n//   - Adds the \"taskID\" to the subscription context, and support for\n//     the BackupMonitor-mediated \"/snapshot\" request.\n// v4: v0.25\n//   - Adds the \"replicaVersion\" and \"minWatermark\" fields to the \"/snapshot\"\n//     status request so that a subscriber can verify whether its replica,\n//     whether it be restored or existing in a permanent volume, is compatible\n//     with the change-streamer.\n// v5: v0.26\n//   - Moves relation.keyColumns and relation.replicaIdentity to\n//     relation.rowKey: { columns, type }.\n//   - Adds `metadata` to `create-table` message\n//   - Adds `tableMetadata` to `add-column` message\n//   - Adds `table-update-metadata` message\n// v6: v0.26\n//   - Adds support for `backfill` messages\n// v6: v1.0.1  (backwards compatible, no version change)\n//   - Adds lag reporting to status messages\n\nexport const PROTOCOL_VERSION = 6;\n\nexport type SubscriberContext = {\n  /**\n   * The supported change-streamer protocol version.\n   */\n  protocolVersion: number;\n\n  /**\n   * Task ID. This is used to link the request with a preceding snapshot\n   * reservation.\n   */\n  taskID: string | null; // TODO: Make required when v3 is min.\n\n  /**\n   * Subscriber id. This is only used for debugging.\n   */\n  id: string;\n\n  /**\n   * The ReplicatorMode of the subscriber. 'backup' indicates that the\n   * subscriber is local to the `change-streamer` in the `replication-manager`,\n   * while 'serving' indicates that user-facing requests depend on the subscriber.\n   */\n  mode: ReplicatorMode;\n\n  /**\n   * The ChangeStreamer will return an Error if the subscriber is\n   * on a different replica version (i.e. the initial snapshot associated\n   * with the replication slot).\n   */\n  replicaVersion: string;\n\n  /**\n   * The watermark up to which the subscriber is up to date.\n   * Only changes after the watermark will be streamed.\n   */\n  watermark: string;\n\n  /**\n   * Whether this is the first subscription request made by the task,\n   * i.e. indicating that the watermark comes from a restored replica\n   * backup. The ChangeStreamer uses this to determine which changes\n   * are safe to purge from the Storer.\n   */\n  initial: boolean;\n};\n\n/**\n * The StatusMessage payload for now is empty, but can be extended to\n * include meta-level information in the future.\n */\nexport const statusSchema = v.object({\n  tag: v.literal('status'),\n\n  lagReport: v\n    .object({\n      lastTimings: changeSourceTimingsSchema.optional(),\n      nextSendTimeMs: v.number(),\n    })\n    .optional(),\n});\n\nexport type Status = v.Infer<typeof statusSchema>;\n\nexport const statusMessageSchema = v.tuple([v.literal('status'), statusSchema]);\n\n/**\n * A StatusMessage will be immediately sent on a (v2+) subscription to\n * indicate that the subscription is valid (i.e. starting at the requested\n * watermark). Invalid subscriptions will instead result in a\n * SubscriptionError as the first message.\n */\nexport type StatusMessage = v.Infer<typeof statusMessageSchema>;\n\nconst subscriptionErrorSchema = v.object({\n  type: v.number(), // ErrorType\n  message: v.string().optional(),\n});\n\nexport type SubscriptionError = v.Infer<typeof subscriptionErrorSchema>;\n\nconst errorSchema = v.tuple([v.literal('error'), subscriptionErrorSchema]);\n\nexport const downstreamSchema = v.union(\n  statusMessageSchema,\n  changeStreamDataSchema,\n  errorSchema,\n);\n\nexport type Error = v.Infer<typeof errorSchema>;\n\nexport function errorTypeToReadableName(val: ErrorType) {\n  switch (val) {\n    case ErrorType.WrongReplicaVersion:\n      return 'WrongReplicaVersion';\n    case ErrorType.WatermarkTooOld:\n      return 'WatermarkTooOld';\n    case ErrorType.Unknown:\n      return 'Unknown';\n    default:\n      return 'Unknown';\n  }\n}\n\n/**\n * A stream of transactions, each starting with a {@link Begin} message,\n * containing one or more {@link Data} messages, and ending with a\n * {@link Commit} or {@link Rollback} message. The 'commit' tuple\n * includes a `watermark` that should be stored with the committed\n * data and used for resuming a subscription (e.g. in the\n * {@link SubscriberContext}).\n *\n * A {@link SubscriptionError} indicates an unrecoverable error that requires\n * manual intervention (e.g. configuration / operational error).\n */\nexport type Downstream = v.Infer<typeof downstreamSchema>;\n\nexport interface ChangeStreamerService\n  extends Omit<ChangeStreamer, 'subscribe'>, Service {\n  /**\n   * The server-side interface overrides `subscribe()` to return a stream\n   * of already-stringified {@link Downstream} payloads.\n   */\n  subscribe(ctx: SubscriberContext): Promise<Source<string>>;\n\n  /**\n   * Notifies the change streamer of a watermark that has been backed up,\n   * indicating that changes before the watermark can be purged if active\n   * subscribers have progressed beyond the watermark.\n   */\n  scheduleCleanup(watermark: string): void;\n\n  getChangeLogState(): Promise<{\n    replicaVersion: string;\n    minWatermark: string;\n  }>;\n}\n"],"mappings":";;;;;;;;AAkIA,IAAa,eAAe,eAAE,OAAO;CACnC,KAAK,eAAE,QAAQ,QAAQ;CAEvB,WAAW,eACR,OAAO;EACN,aAAa,0BAA0B,SAAS;EAChD,gBAAgB,eAAE,OAAO;CAC3B,CAAC,EACA,SAAS;AACd,CAAC;AAID,IAAa,sBAAsB,eAAE,MAAM,CAAC,eAAE,QAAQ,QAAQ,GAAG,YAAY,CAAC;AAU9E,IAAM,0BAA0B,eAAE,OAAO;CACvC,MAAM,eAAE,OAAO;CACf,SAAS,eAAE,OAAO,EAAE,SAAS;AAC/B,CAAC;AAID,IAAM,cAAc,eAAE,MAAM,CAAC,eAAE,QAAQ,OAAO,GAAG,uBAAuB,CAAC;AAEzE,IAAa,mBAAmB,eAAE,MAChC,qBACA,wBACA,WACF;AAIA,SAAgB,wBAAwB,KAAgB;CACtD,QAAQ,KAAR;EACE,KAAK,GACH,OAAO;EACT,KAAK,GACH,OAAO;EACT,KAAK,GACH,OAAO;EACT,SACE,OAAO;CACX;AACF"}
+{"version":3,"file":"change-streamer.js","names":[],"sources":["../../../../../../zero-cache/src/services/change-streamer/change-streamer.ts"],"sourcesContent":["import type {Enum} from '../../../../shared/src/enum.ts';\nimport * as v from '../../../../shared/src/valita.ts';\nimport type {Source} from '../../types/streams.ts';\nimport {changeStreamDataSchema} from '../change-source/protocol/current/downstream.ts';\nimport type {ReplicatorMode} from '../replicator/replicator.ts';\nimport {changeSourceTimingsSchema} from '../replicator/reporter/report-schema.ts';\nimport type {Service} from '../service.ts';\nimport * as ErrorType from './error-type-enum.ts';\n\ntype ErrorType = Enum<typeof ErrorType>;\n\n/**\n * The ChangeStreamer is the component between replicators (\"subscribers\")\n * and a canonical upstream source of changes (e.g. a Postgres logical\n * replication slot). It facilitates multiple subscribers without incurring\n * the associated upstream expense (e.g. PG replication slots are resource\n * intensive) with a \"forward-store-ack\" procedure.\n *\n * * Changes from the upstream source are immediately **forwarded** to\n *   connected subscribers to minimize latency.\n *\n * * They are then **stored** in a separate DB to facilitate catchup\n *   of connecting subscribers that are behind.\n *\n * * **Acknowledgements** are sent upstream after they are successfully\n *   stored.\n *\n * Unlike Postgres replication slots, in which the progress of a static\n * subscriber is tracked in the replication slot, the ChangeStreamer\n * supports a dynamic set of subscribers (i.e.. zero-caches) that can\n * can continually change.\n *\n * However, it is not the case that the ChangeStreamer needs to support\n * arbitrarily old subscribers. Because the replica is continually\n * backed up to a global location and used to initialize new subscriber\n * tasks, an initial subscription request from a subscriber constitutes\n * a signal for how \"behind\" a new subscriber task can be. This is\n * reflected in the {@link SubscriberContext}, which indicates whether\n * the watermark corresponds to an \"initial\" watermark derived from the\n * replica at task startup.\n *\n * The ChangeStreamer uses a combination of this signal with ACK\n * responses from connected subscribers to determine the watermark up\n * to which it is safe to purge old change log entries.\n */\nexport interface ChangeStreamer {\n  /**\n   * Subscribes to changes based on the supplied subscriber `ctx`,\n   * which indicates the watermark at which the subscriber is up to\n   * date.\n   */\n  subscribe(ctx: SubscriberContext): Promise<Source<Downstream>>;\n}\n\n// v1: v0.18\n//   - Client-side support for JSON_FORMAT. Introduced in 0.18.\n// v2: v0.19\n//   - Adds the \"status\" message which is initially used to signal that the\n//     subscription is valid (i.e. starting at the requested watermark).\n// v3: v0.20\n//   - Adds the \"taskID\" to the subscription context, and support for\n//     the BackupMonitor-mediated \"/snapshot\" request.\n// v4: v0.25\n//   - Adds the \"replicaVersion\" and \"minWatermark\" fields to the \"/snapshot\"\n//     status request so that a subscriber can verify whether its replica,\n//     whether it be restored or existing in a permanent volume, is compatible\n//     with the change-streamer.\n// v5: v0.26\n//   - Moves relation.keyColumns and relation.replicaIdentity to\n//     relation.rowKey: { columns, type }.\n//   - Adds `metadata` to `create-table` message\n//   - Adds `tableMetadata` to `add-column` message\n//   - Adds `table-update-metadata` message\n// v6: v0.26\n//   - Adds support for `backfill` messages\n// v6: v1.0.1  (backwards compatible, no version change)\n//   - Adds lag reporting to status messages\n\nexport const PROTOCOL_VERSION = 6;\n\nexport type SubscriberContext = {\n  /**\n   * The supported change-streamer protocol version.\n   */\n  protocolVersion: number;\n\n  /**\n   * Task ID. This is used to link the request with a preceding snapshot\n   * reservation.\n   */\n  taskID: string | null; // TODO: Make required when v3 is min.\n\n  /**\n   * Subscriber id. This is only used for debugging.\n   */\n  id: string;\n\n  /**\n   * The ReplicatorMode of the subscriber. 'backup' indicates that the\n   * subscriber is local to the `change-streamer` in the `replication-manager`,\n   * while 'serving' indicates that user-facing requests depend on the subscriber.\n   */\n  mode: ReplicatorMode;\n\n  /**\n   * The ChangeStreamer will return an Error if the subscriber is\n   * on a different replica version (i.e. the initial snapshot associated\n   * with the replication slot).\n   */\n  replicaVersion: string;\n\n  /**\n   * The watermark up to which the subscriber is up to date.\n   * Only changes after the watermark will be streamed.\n   */\n  watermark: string;\n\n  /**\n   * Whether this is the first subscription request made by the task,\n   * i.e. indicating that the watermark comes from a restored replica\n   * backup. The ChangeStreamer uses this to determine which changes\n   * are safe to purge from the Storer.\n   */\n  initial: boolean;\n};\n\n/**\n * The StatusMessage payload for now is empty, but can be extended to\n * include meta-level information in the future.\n */\nexport const statusSchema = v.object({\n  tag: v.literal('status'),\n\n  lagReport: v\n    .object({\n      lastTimings: changeSourceTimingsSchema.optional(),\n      nextSendTimeMs: v.number(),\n    })\n    .optional(),\n});\n\nexport type Status = v.Infer<typeof statusSchema>;\n\nexport const statusMessageSchema = v.tuple([v.literal('status'), statusSchema]);\n\n/**\n * A StatusMessage will be immediately sent on a (v2+) subscription to\n * indicate that the subscription is valid (i.e. starting at the requested\n * watermark). Invalid subscriptions will instead result in a\n * SubscriptionError as the first message.\n */\nexport type StatusMessage = v.Infer<typeof statusMessageSchema>;\n\nconst subscriptionErrorSchema = v.object({\n  type: v.number(), // ErrorType\n  message: v.string().optional(),\n});\n\nexport type SubscriptionError = v.Infer<typeof subscriptionErrorSchema>;\n\nconst errorSchema = v.tuple([v.literal('error'), subscriptionErrorSchema]);\n\nexport const downstreamSchema = v.union(\n  statusMessageSchema,\n  changeStreamDataSchema,\n  errorSchema,\n);\n\nexport type Error = v.Infer<typeof errorSchema>;\n\nexport function errorTypeToReadableName(val: ErrorType) {\n  switch (val) {\n    case ErrorType.WrongReplicaVersion:\n      return 'WrongReplicaVersion';\n    case ErrorType.WatermarkTooOld:\n      return 'WatermarkTooOld';\n    case ErrorType.Unknown:\n      return 'Unknown';\n    default:\n      return 'Unknown';\n  }\n}\n\n/**\n * A stream of transactions, each starting with a {@link Begin} message,\n * containing one or more {@link Data} messages, and ending with a\n * {@link Commit} or {@link Rollback} message. The 'commit' tuple\n * includes a `watermark` that should be stored with the committed\n * data and used for resuming a subscription (e.g. in the\n * {@link SubscriberContext}).\n *\n * A {@link SubscriptionError} indicates an unrecoverable error that requires\n * manual intervention (e.g. configuration / operational error).\n */\nexport type Downstream = v.Infer<typeof downstreamSchema>;\n\nexport interface ChangeStreamerService\n  extends Omit<ChangeStreamer, 'subscribe'>, Service {\n  /**\n   * The server-side interface overrides `subscribe()` to return a stream\n   * of already-stringified {@link Downstream} payloads.\n   */\n  subscribe(ctx: SubscriberContext): Promise<Source<string>>;\n\n  /**\n   * Notifies the change streamer of a watermark that has been backed up,\n   * indicating that changes before the watermark can be purged if active\n   * subscribers have progressed beyond the watermark.\n   */\n  scheduleCleanup(watermark: string): void;\n\n  getChangeLogState(): Promise<{\n    replicaVersion: string;\n    minWatermark: string;\n  }>;\n}\n"],"mappings":";;;;;;;;AAkIA,IAAa,eAAe,eAAE,OAAO;CACnC,KAAK,eAAE,QAAQ,SAAS;CAExB,WAAW,eACR,OAAO;EACN,aAAa,0BAA0B,UAAU;EACjD,gBAAgB,eAAE,QAAQ;EAC3B,CAAC,CACD,UAAU;CACd,CAAC;AAIF,IAAa,sBAAsB,eAAE,MAAM,CAAC,eAAE,QAAQ,SAAS,EAAE,aAAa,CAAC;AAU/E,IAAM,0BAA0B,eAAE,OAAO;CACvC,MAAM,eAAE,QAAQ;CAChB,SAAS,eAAE,QAAQ,CAAC,UAAU;CAC/B,CAAC;AAIF,IAAM,cAAc,eAAE,MAAM,CAAC,eAAE,QAAQ,QAAQ,EAAE,wBAAwB,CAAC;AAE1E,IAAa,mBAAmB,eAAE,MAChC,qBACA,wBACA,YACD;AAID,SAAgB,wBAAwB,KAAgB;AACtD,SAAQ,KAAR;EACE,KAAK,EACH,QAAO;EACT,KAAK,EACH,QAAO;EACT,KAAK,EACH,QAAO;EACT,QACE,QAAO"}

package/out/zero-cache/src/services/change-streamer/forwarder.js.map

@@ -1 +1 @@
-{"version":3,"file":"forwarder.js","names":["#lc","#progressMonitorOptions","#active","#queued","#progressMonitor","#trackProgress","#currentBroadcast","#inTransaction","#updateActiveSubscribers"],"sources":["../../../../../../zero-cache/src/services/change-streamer/forwarder.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {joinIterables, wrapIterable} from '../../../../shared/src/iterables.ts';\nimport {Broadcast} from './broadcast.ts';\nimport type {ChangeTag, WatermarkedChange} from './change-streamer-service.ts';\nimport type {Status} from './change-streamer.ts';\nimport type {Subscriber} from './subscriber.ts';\n\nexport type ProgressMonitorOptions = {\n  flowControlConsensusPaddingSeconds: number;\n};\n\nexport class Forwarder {\n  readonly #lc: LogContext;\n  readonly #progressMonitorOptions: ProgressMonitorOptions;\n  readonly #active = new Set<Subscriber>();\n  readonly #queued = new Set<Subscriber>();\n  #inTransaction = false;\n\n  #currentBroadcast: Broadcast | undefined;\n  #progressMonitor: NodeJS.Timeout | undefined;\n\n  constructor(\n    lc: LogContext,\n    opts: ProgressMonitorOptions = {flowControlConsensusPaddingSeconds: 1},\n  ) {\n    this.#lc = lc.withContext('component', 'progress-monitor');\n    this.#progressMonitorOptions = opts;\n  }\n\n  startProgressMonitor() {\n    clearInterval(this.#progressMonitor);\n    this.#progressMonitor = setInterval(this.#trackProgress, 1000);\n  }\n\n  readonly #trackProgress = () => {\n    const now = performance.now();\n    for (const sub of this.#active) {\n      sub.sampleProcessRate(now);\n    }\n\n    const {flowControlConsensusPaddingSeconds} = this.#progressMonitorOptions;\n    // A negative number disables early flow control release.\n    if (flowControlConsensusPaddingSeconds >= 0) {\n      this.#currentBroadcast?.checkProgress(\n        this.#lc,\n        flowControlConsensusPaddingSeconds * 1000,\n        now,\n      );\n    }\n  };\n\n  stopProgressMonitor() {\n    clearInterval(this.#progressMonitor);\n  }\n\n  /**\n   * `add()` is called in lock step with `Storer.catchup()` so that the\n   * two components have an equivalent interpretation of whether a Transaction is\n   * currently being streamed.\n   */\n  add(sub: Subscriber) {\n    if (this.#inTransaction) {\n      this.#queued.add(sub);\n    } else {\n      this.#active.add(sub);\n    }\n  }\n\n  remove(sub: Subscriber) {\n    this.#active.delete(sub);\n    this.#queued.delete(sub);\n    sub.close();\n  }\n\n  /**\n   * `forward()` is called in lockstep with `Storer.store()` so that the\n   * two components have an equivalent interpretation of whether a Transaction is\n   * currently being streamed.\n   *\n   * This version of forward is fire-and-forget, with no flow control. The\n   * change-streamer should call and await {@link forwardWithFlowControl()}\n   * occasionally to avoid memory blowup.\n   */\n  forward(entry: WatermarkedChange) {\n    Broadcast.withoutTracking(this.#active.values(), entry);\n    this.#updateActiveSubscribers(entry[1]);\n  }\n\n  sendStatus(status: Status) {\n    for (const sub of this.#active.values()) {\n      sub.sendStatus(status);\n    }\n  }\n\n  /**\n   * The flow-control-aware equivalent of {@link forward()}, returning a\n   * Promise that resolves when replication should continue.\n   */\n  async forwardWithFlowControl(entry: WatermarkedChange) {\n    const broadcast = new Broadcast(this.#active.values(), entry);\n    this.#updateActiveSubscribers(entry[1]);\n\n    // set for progress tracking\n    this.#currentBroadcast = broadcast;\n\n    await broadcast.done;\n\n    // Technically #currentBroadcast may have changed, so only\n    // unset if it if is still the same.\n    if (this.#currentBroadcast === broadcast) {\n      this.#currentBroadcast = undefined;\n    }\n  }\n\n  #updateActiveSubscribers(tag: ChangeTag) {\n    switch (tag) {\n      case 'begin':\n        // While in a Transaction, all added subscribers are \"queued\" so that no\n        // messages are forwarded to them. This state corresponds to being queued\n        // for catchup in the Storer, which will retrieve historic changes\n        // and call catchup() once the current transaction is committed.\n        this.#inTransaction = true;\n        break;\n      case 'commit':\n      case 'rollback':\n        // Upon commit or rollback, all queued subscribers are transferred to\n        // the active set. This means that they can receive messages starting\n        // from the next transaction.\n        //\n        // Note that if catchup is still in progress (in the Storer), these messages\n        // will be buffered in the backlog until catchup completes.\n        this.#inTransaction = false;\n        for (const sub of this.#queued.values()) {\n          this.#active.add(sub);\n        }\n        this.#queued.clear();\n        break;\n    }\n  }\n\n  getAcks(): Set<string> {\n    return new Set(\n      joinIterables(\n        wrapIterable(this.#active).map(s => s.acked),\n        wrapIterable(this.#queued).map(s => s.acked),\n      ),\n    );\n  }\n}\n"],"mappings":";;;AAWA,IAAa,YAAb,MAAuB;CACrB;CACA;CACA,0BAAmB,IAAI,IAAgB;CACvC,0BAAmB,IAAI,IAAgB;CACvC,iBAAiB;CAEjB;CACA;CAEA,YACE,IACA,OAA+B,EAAC,oCAAoC,EAAC,GACrE;EACA,KAAKA,MAAM,GAAG,YAAY,aAAa,kBAAkB;EACzD,KAAKC,0BAA0B;CACjC;CAEA,uBAAuB;EACrB,cAAc,KAAKG,gBAAgB;EACnC,KAAKA,mBAAmB,YAAY,KAAKC,gBAAgB,GAAI;CAC/D;CAEA,uBAAgC;EAC9B,MAAM,MAAM,YAAY,IAAI;EAC5B,KAAK,MAAM,OAAO,KAAKH,SACrB,IAAI,kBAAkB,GAAG;EAG3B,MAAM,EAAC,uCAAsC,KAAKD;EAElD,IAAI,sCAAsC,GACxC,KAAKK,mBAAmB,cACtB,KAAKN,KACL,qCAAqC,KACrC,GACF;CAEJ;CAEA,sBAAsB;EACpB,cAAc,KAAKI,gBAAgB;CACrC;;;;;;CAOA,IAAI,KAAiB;EACnB,IAAI,KAAKG,gBACP,KAAKJ,QAAQ,IAAI,GAAG;OAEpB,KAAKD,QAAQ,IAAI,GAAG;CAExB;CAEA,OAAO,KAAiB;EACtB,KAAKA,QAAQ,OAAO,GAAG;EACvB,KAAKC,QAAQ,OAAO,GAAG;EACvB,IAAI,MAAM;CACZ;;;;;;;;;;CAWA,QAAQ,OAA0B;EAChC,UAAU,gBAAgB,KAAKD,QAAQ,OAAO,GAAG,KAAK;EACtD,KAAKM,yBAAyB,MAAM,EAAE;CACxC;CAEA,WAAW,QAAgB;EACzB,KAAK,MAAM,OAAO,KAAKN,QAAQ,OAAO,GACpC,IAAI,WAAW,MAAM;CAEzB;;;;;CAMA,MAAM,uBAAuB,OAA0B;EACrD,MAAM,YAAY,IAAI,UAAU,KAAKA,QAAQ,OAAO,GAAG,KAAK;EAC5D,KAAKM,yBAAyB,MAAM,EAAE;EAGtC,KAAKF,oBAAoB;EAEzB,MAAM,UAAU;EAIhB,IAAI,KAAKA,sBAAsB,WAC7B,KAAKA,oBAAoB,KAAA;CAE7B;CAEA,yBAAyB,KAAgB;EACvC,QAAQ,KAAR;GACE,KAAK;IAKH,KAAKC,iBAAiB;IACtB;GACF,KAAK;GACL,KAAK;IAOH,KAAKA,iBAAiB;IACtB,KAAK,MAAM,OAAO,KAAKJ,QAAQ,OAAO,GACpC,KAAKD,QAAQ,IAAI,GAAG;IAEtB,KAAKC,QAAQ,MAAM;IACnB;EACJ;CACF;CAEA,UAAuB;EACrB,OAAO,IAAI,IACT,cACE,aAAa,KAAKD,OAAO,EAAE,KAAI,MAAK,EAAE,KAAK,GAC3C,aAAa,KAAKC,OAAO,EAAE,KAAI,MAAK,EAAE,KAAK,CAC7C,CACF;CACF;AACF"}
+{"version":3,"file":"forwarder.js","names":["#lc","#progressMonitorOptions","#active","#queued","#progressMonitor","#trackProgress","#currentBroadcast","#inTransaction","#updateActiveSubscribers"],"sources":["../../../../../../zero-cache/src/services/change-streamer/forwarder.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {joinIterables, wrapIterable} from '../../../../shared/src/iterables.ts';\nimport {Broadcast} from './broadcast.ts';\nimport type {ChangeTag, WatermarkedChange} from './change-streamer-service.ts';\nimport type {Status} from './change-streamer.ts';\nimport type {Subscriber} from './subscriber.ts';\n\nexport type ProgressMonitorOptions = {\n  flowControlConsensusPaddingSeconds: number;\n};\n\nexport class Forwarder {\n  readonly #lc: LogContext;\n  readonly #progressMonitorOptions: ProgressMonitorOptions;\n  readonly #active = new Set<Subscriber>();\n  readonly #queued = new Set<Subscriber>();\n  #inTransaction = false;\n\n  #currentBroadcast: Broadcast | undefined;\n  #progressMonitor: NodeJS.Timeout | undefined;\n\n  constructor(\n    lc: LogContext,\n    opts: ProgressMonitorOptions = {flowControlConsensusPaddingSeconds: 1},\n  ) {\n    this.#lc = lc.withContext('component', 'progress-monitor');\n    this.#progressMonitorOptions = opts;\n  }\n\n  startProgressMonitor() {\n    clearInterval(this.#progressMonitor);\n    this.#progressMonitor = setInterval(this.#trackProgress, 1000);\n  }\n\n  readonly #trackProgress = () => {\n    const now = performance.now();\n    for (const sub of this.#active) {\n      sub.sampleProcessRate(now);\n    }\n\n    const {flowControlConsensusPaddingSeconds} = this.#progressMonitorOptions;\n    // A negative number disables early flow control release.\n    if (flowControlConsensusPaddingSeconds >= 0) {\n      this.#currentBroadcast?.checkProgress(\n        this.#lc,\n        flowControlConsensusPaddingSeconds * 1000,\n        now,\n      );\n    }\n  };\n\n  stopProgressMonitor() {\n    clearInterval(this.#progressMonitor);\n  }\n\n  /**\n   * `add()` is called in lock step with `Storer.catchup()` so that the\n   * two components have an equivalent interpretation of whether a Transaction is\n   * currently being streamed.\n   */\n  add(sub: Subscriber) {\n    if (this.#inTransaction) {\n      this.#queued.add(sub);\n    } else {\n      this.#active.add(sub);\n    }\n  }\n\n  remove(sub: Subscriber) {\n    this.#active.delete(sub);\n    this.#queued.delete(sub);\n    sub.close();\n  }\n\n  /**\n   * `forward()` is called in lockstep with `Storer.store()` so that the\n   * two components have an equivalent interpretation of whether a Transaction is\n   * currently being streamed.\n   *\n   * This version of forward is fire-and-forget, with no flow control. The\n   * change-streamer should call and await {@link forwardWithFlowControl()}\n   * occasionally to avoid memory blowup.\n   */\n  forward(entry: WatermarkedChange) {\n    Broadcast.withoutTracking(this.#active.values(), entry);\n    this.#updateActiveSubscribers(entry[1]);\n  }\n\n  sendStatus(status: Status) {\n    for (const sub of this.#active.values()) {\n      sub.sendStatus(status);\n    }\n  }\n\n  /**\n   * The flow-control-aware equivalent of {@link forward()}, returning a\n   * Promise that resolves when replication should continue.\n   */\n  async forwardWithFlowControl(entry: WatermarkedChange) {\n    const broadcast = new Broadcast(this.#active.values(), entry);\n    this.#updateActiveSubscribers(entry[1]);\n\n    // set for progress tracking\n    this.#currentBroadcast = broadcast;\n\n    await broadcast.done;\n\n    // Technically #currentBroadcast may have changed, so only\n    // unset if it if is still the same.\n    if (this.#currentBroadcast === broadcast) {\n      this.#currentBroadcast = undefined;\n    }\n  }\n\n  #updateActiveSubscribers(tag: ChangeTag) {\n    switch (tag) {\n      case 'begin':\n        // While in a Transaction, all added subscribers are \"queued\" so that no\n        // messages are forwarded to them. This state corresponds to being queued\n        // for catchup in the Storer, which will retrieve historic changes\n        // and call catchup() once the current transaction is committed.\n        this.#inTransaction = true;\n        break;\n      case 'commit':\n      case 'rollback':\n        // Upon commit or rollback, all queued subscribers are transferred to\n        // the active set. This means that they can receive messages starting\n        // from the next transaction.\n        //\n        // Note that if catchup is still in progress (in the Storer), these messages\n        // will be buffered in the backlog until catchup completes.\n        this.#inTransaction = false;\n        for (const sub of this.#queued.values()) {\n          this.#active.add(sub);\n        }\n        this.#queued.clear();\n        break;\n    }\n  }\n\n  getAcks(): Set<string> {\n    return new Set(\n      joinIterables(\n        wrapIterable(this.#active).map(s => s.acked),\n        wrapIterable(this.#queued).map(s => s.acked),\n      ),\n    );\n  }\n}\n"],"mappings":";;;AAWA,IAAa,YAAb,MAAuB;CACrB;CACA;CACA,0BAAmB,IAAI,KAAiB;CACxC,0BAAmB,IAAI,KAAiB;CACxC,iBAAiB;CAEjB;CACA;CAEA,YACE,IACA,OAA+B,EAAC,oCAAoC,GAAE,EACtE;AACA,QAAA,KAAW,GAAG,YAAY,aAAa,mBAAmB;AAC1D,QAAA,yBAA+B;;CAGjC,uBAAuB;AACrB,gBAAc,MAAA,gBAAsB;AACpC,QAAA,kBAAwB,YAAY,MAAA,eAAqB,IAAK;;CAGhE,uBAAgC;EAC9B,MAAM,MAAM,YAAY,KAAK;AAC7B,OAAK,MAAM,OAAO,MAAA,OAChB,KAAI,kBAAkB,IAAI;EAG5B,MAAM,EAAC,uCAAsC,MAAA;AAE7C,MAAI,sCAAsC,EACxC,OAAA,kBAAwB,cACtB,MAAA,IACA,qCAAqC,KACrC,IACD;;CAIL,sBAAsB;AACpB,gBAAc,MAAA,gBAAsB;;;;;;;CAQtC,IAAI,KAAiB;AACnB,MAAI,MAAA,cACF,OAAA,OAAa,IAAI,IAAI;MAErB,OAAA,OAAa,IAAI,IAAI;;CAIzB,OAAO,KAAiB;AACtB,QAAA,OAAa,OAAO,IAAI;AACxB,QAAA,OAAa,OAAO,IAAI;AACxB,MAAI,OAAO;;;;;;;;;;;CAYb,QAAQ,OAA0B;AAChC,YAAU,gBAAgB,MAAA,OAAa,QAAQ,EAAE,MAAM;AACvD,QAAA,wBAA8B,MAAM,GAAG;;CAGzC,WAAW,QAAgB;AACzB,OAAK,MAAM,OAAO,MAAA,OAAa,QAAQ,CACrC,KAAI,WAAW,OAAO;;;;;;CAQ1B,MAAM,uBAAuB,OAA0B;EACrD,MAAM,YAAY,IAAI,UAAU,MAAA,OAAa,QAAQ,EAAE,MAAM;AAC7D,QAAA,wBAA8B,MAAM,GAAG;AAGvC,QAAA,mBAAyB;AAEzB,QAAM,UAAU;AAIhB,MAAI,MAAA,qBAA2B,UAC7B,OAAA,mBAAyB,KAAA;;CAI7B,yBAAyB,KAAgB;AACvC,UAAQ,KAAR;GACE,KAAK;AAKH,UAAA,gBAAsB;AACtB;GACF,KAAK;GACL,KAAK;AAOH,UAAA,gBAAsB;AACtB,SAAK,MAAM,OAAO,MAAA,OAAa,QAAQ,CACrC,OAAA,OAAa,IAAI,IAAI;AAEvB,UAAA,OAAa,OAAO;AACpB;;;CAIN,UAAuB;AACrB,SAAO,IAAI,IACT,cACE,aAAa,MAAA,OAAa,CAAC,KAAI,MAAK,EAAE,MAAM,EAC5C,aAAa,MAAA,OAAa,CAAC,KAAI,MAAK,EAAE,MAAM,CAC7C,CACF"}

package/out/zero-cache/src/services/change-streamer/replica-monitor.js.map

@@ -1 +1 @@
-{"version":3,"file":"replica-monitor.js","names":["#lc","#replicaFile","#changeStreamer","#state","#lastWatermark"],"sources":["../../../../../../zero-cache/src/services/change-streamer/replica-monitor.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {promiseVoid} from '../../../../shared/src/resolved-promises.ts';\nimport {Database} from '../../../../zqlite/src/db.ts';\nimport {StatementRunner} from '../../db/statements.ts';\nimport {getReplicationState} from '../replicator/schema/replication-state.ts';\nimport {RunningState} from '../running-state.ts';\nimport type {Service} from '../service.ts';\nimport type {ChangeStreamerService} from './change-streamer.ts';\n\nconst CHECK_INTERVAL_MS = 30 * 1000;\n\n/**\n * The single-node equivalent of the {@link BackupMonitor} polls the replica\n * file every 30 seconds and schedules cleanup when the watermark\n * (i.e. stateVersion) moves forward.\n */\nexport class ReplicaMonitor implements Service {\n  readonly id = 'replica-monitor';\n  readonly #lc: LogContext;\n  readonly #replicaFile: string;\n  readonly #changeStreamer: ChangeStreamerService;\n  readonly #state = new RunningState(this.id);\n\n  #lastWatermark: string = '';\n\n  constructor(\n    lc: LogContext,\n    replicaFile: string,\n    changeStreamer: ChangeStreamerService,\n  ) {\n    this.#lc = lc.withContext('component', this.id);\n    this.#replicaFile = replicaFile;\n    this.#changeStreamer = changeStreamer;\n  }\n\n  async run() {\n    this.#lc.info?.(`starting replica monitor`);\n    await this.#state.sleep(CHECK_INTERVAL_MS);\n\n    while (this.#state.shouldRun()) {\n      const db = new Database(this.#lc, this.#replicaFile);\n      try {\n        const {stateVersion} = getReplicationState(new StatementRunner(db));\n        if (stateVersion !== this.#lastWatermark) {\n          this.#lastWatermark = stateVersion;\n          this.#lc.debug?.(`replicated up to watermark ${stateVersion}`);\n          this.#changeStreamer.scheduleCleanup(stateVersion);\n        }\n      } catch (e) {\n        this.#lc.error?.(`Unable to read watermark from replica`, e);\n      } finally {\n        db.close();\n      }\n\n      await this.#state.sleep(CHECK_INTERVAL_MS);\n    }\n    this.#lc.info?.(`replica monitor stopped`);\n  }\n\n  stop() {\n    this.#state.stop(this.#lc);\n    return promiseVoid;\n  }\n}\n"],"mappings":";;;;;;AASA,IAAM,oBAAoB,KAAK;;;;;;AAO/B,IAAa,iBAAb,MAA+C;CAC7C,KAAc;CACd;CACA;CACA;CACA,SAAkB,IAAI,aAAa,KAAK,EAAE;CAE1C,iBAAyB;CAEzB,YACE,IACA,aACA,gBACA;EACA,KAAKA,MAAM,GAAG,YAAY,aAAa,KAAK,EAAE;EAC9C,KAAKC,eAAe;EACpB,KAAKC,kBAAkB;CACzB;CAEA,MAAM,MAAM;EACV,KAAKF,IAAI,OAAO,0BAA0B;EAC1C,MAAM,KAAKG,OAAO,MAAM,iBAAiB;EAEzC,OAAO,KAAKA,OAAO,UAAU,GAAG;GAC9B,MAAM,KAAK,IAAI,SAAS,KAAKH,KAAK,KAAKC,YAAY;GACnD,IAAI;IACF,MAAM,EAAC,iBAAgB,oBAAoB,IAAI,gBAAgB,EAAE,CAAC;IAClE,IAAI,iBAAiB,KAAKG,gBAAgB;KACxC,KAAKA,iBAAiB;KACtB,KAAKJ,IAAI,QAAQ,8BAA8B,cAAc;KAC7D,KAAKE,gBAAgB,gBAAgB,YAAY;IACnD;GACF,SAAS,GAAG;IACV,KAAKF,IAAI,QAAQ,yCAAyC,CAAC;GAC7D,UAAU;IACR,GAAG,MAAM;GACX;GAEA,MAAM,KAAKG,OAAO,MAAM,iBAAiB;EAC3C;EACA,KAAKH,IAAI,OAAO,yBAAyB;CAC3C;CAEA,OAAO;EACL,KAAKG,OAAO,KAAK,KAAKH,GAAG;EACzB,OAAO;CACT;AACF"}
+{"version":3,"file":"replica-monitor.js","names":["#lc","#replicaFile","#changeStreamer","#state","#lastWatermark"],"sources":["../../../../../../zero-cache/src/services/change-streamer/replica-monitor.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {promiseVoid} from '../../../../shared/src/resolved-promises.ts';\nimport {Database} from '../../../../zqlite/src/db.ts';\nimport {StatementRunner} from '../../db/statements.ts';\nimport {getReplicationState} from '../replicator/schema/replication-state.ts';\nimport {RunningState} from '../running-state.ts';\nimport type {Service} from '../service.ts';\nimport type {ChangeStreamerService} from './change-streamer.ts';\n\nconst CHECK_INTERVAL_MS = 30 * 1000;\n\n/**\n * The single-node equivalent of the {@link BackupMonitor} polls the replica\n * file every 30 seconds and schedules cleanup when the watermark\n * (i.e. stateVersion) moves forward.\n */\nexport class ReplicaMonitor implements Service {\n  readonly id = 'replica-monitor';\n  readonly #lc: LogContext;\n  readonly #replicaFile: string;\n  readonly #changeStreamer: ChangeStreamerService;\n  readonly #state = new RunningState(this.id);\n\n  #lastWatermark: string = '';\n\n  constructor(\n    lc: LogContext,\n    replicaFile: string,\n    changeStreamer: ChangeStreamerService,\n  ) {\n    this.#lc = lc.withContext('component', this.id);\n    this.#replicaFile = replicaFile;\n    this.#changeStreamer = changeStreamer;\n  }\n\n  async run() {\n    this.#lc.info?.(`starting replica monitor`);\n    await this.#state.sleep(CHECK_INTERVAL_MS);\n\n    while (this.#state.shouldRun()) {\n      const db = new Database(this.#lc, this.#replicaFile);\n      try {\n        const {stateVersion} = getReplicationState(new StatementRunner(db));\n        if (stateVersion !== this.#lastWatermark) {\n          this.#lastWatermark = stateVersion;\n          this.#lc.debug?.(`replicated up to watermark ${stateVersion}`);\n          this.#changeStreamer.scheduleCleanup(stateVersion);\n        }\n      } catch (e) {\n        this.#lc.error?.(`Unable to read watermark from replica`, e);\n      } finally {\n        db.close();\n      }\n\n      await this.#state.sleep(CHECK_INTERVAL_MS);\n    }\n    this.#lc.info?.(`replica monitor stopped`);\n  }\n\n  stop() {\n    this.#state.stop(this.#lc);\n    return promiseVoid;\n  }\n}\n"],"mappings":";;;;;;AASA,IAAM,oBAAoB,KAAK;;;;;;AAO/B,IAAa,iBAAb,MAA+C;CAC7C,KAAc;CACd;CACA;CACA;CACA,SAAkB,IAAI,aAAa,KAAK,GAAG;CAE3C,iBAAyB;CAEzB,YACE,IACA,aACA,gBACA;AACA,QAAA,KAAW,GAAG,YAAY,aAAa,KAAK,GAAG;AAC/C,QAAA,cAAoB;AACpB,QAAA,iBAAuB;;CAGzB,MAAM,MAAM;AACV,QAAA,GAAS,OAAO,2BAA2B;AAC3C,QAAM,MAAA,MAAY,MAAM,kBAAkB;AAE1C,SAAO,MAAA,MAAY,WAAW,EAAE;GAC9B,MAAM,KAAK,IAAI,SAAS,MAAA,IAAU,MAAA,YAAkB;AACpD,OAAI;IACF,MAAM,EAAC,iBAAgB,oBAAoB,IAAI,gBAAgB,GAAG,CAAC;AACnE,QAAI,iBAAiB,MAAA,eAAqB;AACxC,WAAA,gBAAsB;AACtB,WAAA,GAAS,QAAQ,8BAA8B,eAAe;AAC9D,WAAA,eAAqB,gBAAgB,aAAa;;YAE7C,GAAG;AACV,UAAA,GAAS,QAAQ,yCAAyC,EAAE;aACpD;AACR,OAAG,OAAO;;AAGZ,SAAM,MAAA,MAAY,MAAM,kBAAkB;;AAE5C,QAAA,GAAS,OAAO,0BAA0B;;CAG5C,OAAO;AACL,QAAA,MAAY,KAAK,MAAA,GAAS;AAC1B,SAAO"}

package/out/zero-cache/src/services/change-streamer/schema/init.js

@@ -25,33 +25,37 @@ async function initChangeStreamerSchema(log, db, shard) {
 		await db`
     ALTER TABLE ${db(schema)}."replicationConfig" ADD "resetRequired" BOOL`;
 	}
-	await runSchemaMigrations(log, "change-streamer", schema, db, setupMigration, {
-		2: { migrateSchema: migrateV1toV2 },
-		3: {
-			migrateSchema: async (_, db) => {
-				await db.unsafe(createReplicationStateTable(shard));
-			},
-			migrateData: async (_, db) => {
-				const replicationState = { lastWatermark: await getLastWatermarkV2(db, shard) };
-				await db`
-      TRUNCATE TABLE ${db(schema)}."replicationState"`;
-				await db`
-      INSERT INTO ${db(schema)}."replicationState" ${db(replicationState)}`;
-			}
+	const migrateV2ToV3 = {
+		migrateSchema: async (_, db) => {
+			await db.unsafe(createReplicationStateTable(shard));
 		},
-		4: { migrateSchema: async (_, db) => {
+		migrateData: async (_, db) => {
+			const replicationState = { lastWatermark: await getLastWatermarkV2(db, shard) };
 			await db`
+      TRUNCATE TABLE ${db(schema)}."replicationState"`;
+			await db`
+      INSERT INTO ${db(schema)}."replicationState" ${db(replicationState)}`;
+		}
+	};
+	const migrateV3ToV4 = { migrateSchema: async (_, db) => {
+		await db`
       ALTER TABLE ${db(schema)}."changeLog" ALTER "change" TYPE JSON;
       `;
-		} },
-		5: { migrateSchema: async (_, db) => {
-			await db`
+	} };
+	const migrateV4ToV5 = { migrateSchema: async (_, db) => {
+		await db`
       ALTER TABLE ${db(schema)}."replicationState" ADD "ownerAddress" TEXT;
       `;
-		} },
-		6: { migrateSchema: async (_, db) => {
-			await db.unsafe(createBackfillTables(shard));
-		} }
+	} };
+	const migrateV5ToV6 = { migrateSchema: async (_, db) => {
+		await db.unsafe(createBackfillTables(shard));
+	} };
+	await runSchemaMigrations(log, "change-streamer", schema, db, setupMigration, {
+		2: { migrateSchema: migrateV1toV2 },
+		3: migrateV2ToV3,
+		4: migrateV3ToV4,
+		5: migrateV4ToV5,
+		6: migrateV5ToV6
 	});
 }
 async function getLastWatermarkV2(db, shard) {

package/out/zero-cache/src/services/change-streamer/schema/init.js.map

@@ -1 +1 @@
-{"version":3,"file":"init.js","names":[],"sources":["../../../../../../../zero-cache/src/services/change-streamer/schema/init.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {\n  runSchemaMigrations,\n  type IncrementalMigrationMap,\n  type Migration,\n} from '../../../db/migration.ts';\nimport type {PostgresDB, PostgresTransaction} from '../../../types/pg.ts';\nimport {cdcSchema, type ShardID} from '../../../types/shards.ts';\nimport {\n  createBackfillTables,\n  createReplicationStateTable,\n  setupCDCTables,\n  type ReplicationState,\n} from './tables.ts';\n\nasync function migrateFromLegacySchemas(\n  lc: LogContext,\n  db: PostgresDB,\n  newSchema: string,\n  ...legacy: string[]\n) {\n  const rows = await db<{nspname: string}[]>`\n    SELECT nspname FROM pg_namespace \n      WHERE nspname IN ${db([newSchema, ...legacy])}`.values();\n  // oxlint-disable-next-line unicorn/prefer-set-has -- Small collection, array is more appropriate\n  const names = rows.flat();\n  if (names.includes(newSchema)) {\n    return; // already migrated\n  }\n  for (const schema of legacy) {\n    if (names.includes(schema)) {\n      lc.info?.(`Migrating ${schema} to ${newSchema}`);\n      await db`ALTER SCHEMA ${db(schema)} RENAME TO ${db(newSchema)}`;\n      break;\n    }\n  }\n}\n\nexport async function initChangeStreamerSchema(\n  log: LogContext,\n  db: PostgresDB,\n  shard: ShardID,\n): Promise<void> {\n  const schema = cdcSchema(shard);\n  const {appID} = shard;\n  await migrateFromLegacySchemas(\n    log,\n    db,\n    schema,\n    appID === 'zero' ? `cdc_0` : `cdc_${appID}`,\n    'cdc',\n  );\n\n  const setupMigration: Migration = {\n    migrateSchema: (lc, tx) => setupCDCTables(lc, tx, shard),\n    minSafeVersion: 1,\n  };\n\n  async function migrateV1toV2(_: LogContext, db: PostgresTransaction) {\n    await db`\n    ALTER TABLE ${db(schema)}.\"replicationConfig\" ADD \"resetRequired\" BOOL`;\n  }\n\n  const migrateV2ToV3 = {\n    migrateSchema: async (_: LogContext, db: PostgresTransaction) => {\n      await db.unsafe(createReplicationStateTable(shard));\n    },\n\n    migrateData: async (_: LogContext, db: PostgresTransaction) => {\n      const lastWatermark = await getLastWatermarkV2(db, shard);\n      const replicationState: Partial<ReplicationState> = {lastWatermark};\n      await db`\n      TRUNCATE TABLE ${db(schema)}.\"replicationState\"`;\n      await db`\n      INSERT INTO ${db(schema)}.\"replicationState\" ${db(replicationState)}`;\n    },\n  };\n\n  const migrateV3ToV4 = {\n    migrateSchema: async (_: LogContext, db: PostgresTransaction) => {\n      await db`\n      ALTER TABLE ${db(schema)}.\"changeLog\" ALTER \"change\" TYPE JSON;\n      `;\n    },\n  };\n\n  const migrateV4ToV5 = {\n    migrateSchema: async (_: LogContext, db: PostgresTransaction) => {\n      await db`\n      ALTER TABLE ${db(schema)}.\"replicationState\" ADD \"ownerAddress\" TEXT;\n      `;\n    },\n  };\n\n  const migrateV5ToV6 = {\n    migrateSchema: async (_: LogContext, db: PostgresTransaction) => {\n      await db.unsafe(createBackfillTables(shard));\n    },\n  };\n\n  const schemaVersionMigrationMap: IncrementalMigrationMap = {\n    2: {migrateSchema: migrateV1toV2},\n    3: migrateV2ToV3,\n    4: migrateV3ToV4,\n    5: migrateV4ToV5,\n    6: migrateV5ToV6,\n  };\n\n  await runSchemaMigrations(\n    log,\n    'change-streamer',\n    schema,\n    db,\n    setupMigration,\n    schemaVersionMigrationMap,\n  );\n}\n\nexport async function getLastWatermarkV2(\n  db: PostgresDB,\n  shard: ShardID,\n): Promise<string> {\n  const schema = cdcSchema(shard);\n  const [{max}] = await db<{max: string | null}[]>`\n    SELECT MAX(watermark) as max FROM ${db(schema)}.\"changeLog\"`;\n  if (max !== null) {\n    return max;\n  }\n  // The changeLog is only empty if nothing has been synced since initial-sync.\n  // In this case, the last watermark is the replicaVersion.\n  const [{replicaVersion}] = await db<{replicaVersion: string}[]>`\n    SELECT \"replicaVersion\" FROM ${db(schema)}.\"replicationConfig\"\n    `;\n  return replicaVersion;\n}\n"],"mappings":";;;;AAeA,eAAe,yBACb,IACA,IACA,WACA,GAAG,QACH;CAKA,MAAM,SAAQ,MAJK,EAAuB;;yBAEnB,GAAG,CAAC,WAAW,GAAG,MAAM,CAAC,IAAI,OAAO,GAExC,KAAK;CACxB,IAAI,MAAM,SAAS,SAAS,GAC1B;CAEF,KAAK,MAAM,UAAU,QACnB,IAAI,MAAM,SAAS,MAAM,GAAG;EAC1B,GAAG,OAAO,aAAa,OAAO,MAAM,WAAW;EAC/C,MAAM,EAAE,gBAAgB,GAAG,MAAM,EAAE,aAAa,GAAG,SAAS;EAC5D;CACF;AAEJ;AAEA,eAAsB,yBACpB,KACA,IACA,OACe;CACf,MAAM,SAAS,UAAU,KAAK;CAC9B,MAAM,EAAC,UAAS;CAChB,MAAM,yBACJ,KACA,IACA,QACA,UAAU,SAAS,UAAU,OAAO,SACpC,KACF;CAEA,MAAM,iBAA4B;EAChC,gBAAgB,IAAI,OAAO,eAAe,IAAI,IAAI,KAAK;EACvD,gBAAgB;CAClB;CAEA,eAAe,cAAc,GAAe,IAAyB;EACnE,MAAM,EAAE;kBACM,GAAG,MAAM,EAAE;CAC3B;CA+CA,MAAM,oBACJ,KACA,mBACA,QACA,IACA,gBACA;EAbA,GAAG,EAAC,eAAe,cAAa;EAChC,GAAG;GAtCH,eAAe,OAAO,GAAe,OAA4B;IAC/D,MAAM,GAAG,OAAO,4BAA4B,KAAK,CAAC;GACpD;GAEA,aAAa,OAAO,GAAe,OAA4B;IAE7D,MAAM,mBAA8C,EAAC,eAAA,MADzB,mBAAmB,IAAI,KAAK,EACU;IAClE,MAAM,EAAE;uBACS,GAAG,MAAM,EAAE;IAC5B,MAAM,EAAE;oBACM,GAAG,MAAM,EAAE,sBAAsB,GAAG,gBAAgB;GACpE;EA2BG;EACH,GAAG,EAxBH,eAAe,OAAO,GAAe,OAA4B;GAC/D,MAAM,EAAE;oBACM,GAAG,MAAM,EAAE;;EAE3B,EAoBG;EACH,GAAG,EAjBH,eAAe,OAAO,GAAe,OAA4B;GAC/D,MAAM,EAAE;oBACM,GAAG,MAAM,EAAE;;EAE3B,EAaG;EACH,GAAG,EAVH,eAAe,OAAO,GAAe,OAA4B;GAC/D,MAAM,GAAG,OAAO,qBAAqB,KAAK,CAAC;EAC7C,EAQG;CASH,CACF;AACF;AAEA,eAAsB,mBACpB,IACA,OACiB;CACjB,MAAM,SAAS,UAAU,KAAK;CAC9B,MAAM,CAAC,EAAC,SAAQ,MAAM,EAA0B;wCACV,GAAG,MAAM,EAAE;CACjD,IAAI,QAAQ,MACV,OAAO;CAIT,MAAM,CAAC,EAAC,oBAAmB,MAAM,EAA8B;mCAC9B,GAAG,MAAM,EAAE;;CAE5C,OAAO;AACT"}
+{"version":3,"file":"init.js","names":[],"sources":["../../../../../../../zero-cache/src/services/change-streamer/schema/init.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {\n  runSchemaMigrations,\n  type IncrementalMigrationMap,\n  type Migration,\n} from '../../../db/migration.ts';\nimport type {PostgresDB, PostgresTransaction} from '../../../types/pg.ts';\nimport {cdcSchema, type ShardID} from '../../../types/shards.ts';\nimport {\n  createBackfillTables,\n  createReplicationStateTable,\n  setupCDCTables,\n  type ReplicationState,\n} from './tables.ts';\n\nasync function migrateFromLegacySchemas(\n  lc: LogContext,\n  db: PostgresDB,\n  newSchema: string,\n  ...legacy: string[]\n) {\n  const rows = await db<{nspname: string}[]>`\n    SELECT nspname FROM pg_namespace \n      WHERE nspname IN ${db([newSchema, ...legacy])}`.values();\n  // oxlint-disable-next-line unicorn/prefer-set-has -- Small collection, array is more appropriate\n  const names = rows.flat();\n  if (names.includes(newSchema)) {\n    return; // already migrated\n  }\n  for (const schema of legacy) {\n    if (names.includes(schema)) {\n      lc.info?.(`Migrating ${schema} to ${newSchema}`);\n      await db`ALTER SCHEMA ${db(schema)} RENAME TO ${db(newSchema)}`;\n      break;\n    }\n  }\n}\n\nexport async function initChangeStreamerSchema(\n  log: LogContext,\n  db: PostgresDB,\n  shard: ShardID,\n): Promise<void> {\n  const schema = cdcSchema(shard);\n  const {appID} = shard;\n  await migrateFromLegacySchemas(\n    log,\n    db,\n    schema,\n    appID === 'zero' ? `cdc_0` : `cdc_${appID}`,\n    'cdc',\n  );\n\n  const setupMigration: Migration = {\n    migrateSchema: (lc, tx) => setupCDCTables(lc, tx, shard),\n    minSafeVersion: 1,\n  };\n\n  async function migrateV1toV2(_: LogContext, db: PostgresTransaction) {\n    await db`\n    ALTER TABLE ${db(schema)}.\"replicationConfig\" ADD \"resetRequired\" BOOL`;\n  }\n\n  const migrateV2ToV3 = {\n    migrateSchema: async (_: LogContext, db: PostgresTransaction) => {\n      await db.unsafe(createReplicationStateTable(shard));\n    },\n\n    migrateData: async (_: LogContext, db: PostgresTransaction) => {\n      const lastWatermark = await getLastWatermarkV2(db, shard);\n      const replicationState: Partial<ReplicationState> = {lastWatermark};\n      await db`\n      TRUNCATE TABLE ${db(schema)}.\"replicationState\"`;\n      await db`\n      INSERT INTO ${db(schema)}.\"replicationState\" ${db(replicationState)}`;\n    },\n  };\n\n  const migrateV3ToV4 = {\n    migrateSchema: async (_: LogContext, db: PostgresTransaction) => {\n      await db`\n      ALTER TABLE ${db(schema)}.\"changeLog\" ALTER \"change\" TYPE JSON;\n      `;\n    },\n  };\n\n  const migrateV4ToV5 = {\n    migrateSchema: async (_: LogContext, db: PostgresTransaction) => {\n      await db`\n      ALTER TABLE ${db(schema)}.\"replicationState\" ADD \"ownerAddress\" TEXT;\n      `;\n    },\n  };\n\n  const migrateV5ToV6 = {\n    migrateSchema: async (_: LogContext, db: PostgresTransaction) => {\n      await db.unsafe(createBackfillTables(shard));\n    },\n  };\n\n  const schemaVersionMigrationMap: IncrementalMigrationMap = {\n    2: {migrateSchema: migrateV1toV2},\n    3: migrateV2ToV3,\n    4: migrateV3ToV4,\n    5: migrateV4ToV5,\n    6: migrateV5ToV6,\n  };\n\n  await runSchemaMigrations(\n    log,\n    'change-streamer',\n    schema,\n    db,\n    setupMigration,\n    schemaVersionMigrationMap,\n  );\n}\n\nexport async function getLastWatermarkV2(\n  db: PostgresDB,\n  shard: ShardID,\n): Promise<string> {\n  const schema = cdcSchema(shard);\n  const [{max}] = await db<{max: string | null}[]>`\n    SELECT MAX(watermark) as max FROM ${db(schema)}.\"changeLog\"`;\n  if (max !== null) {\n    return max;\n  }\n  // The changeLog is only empty if nothing has been synced since initial-sync.\n  // In this case, the last watermark is the replicaVersion.\n  const [{replicaVersion}] = await db<{replicaVersion: string}[]>`\n    SELECT \"replicaVersion\" FROM ${db(schema)}.\"replicationConfig\"\n    `;\n  return replicaVersion;\n}\n"],"mappings":";;;;AAeA,eAAe,yBACb,IACA,IACA,WACA,GAAG,QACH;CAKA,MAAM,SAJO,MAAM,EAAuB;;yBAEnB,GAAG,CAAC,WAAW,GAAG,OAAO,CAAC,GAAG,QAAQ,EAEzC,MAAM;AACzB,KAAI,MAAM,SAAS,UAAU,CAC3B;AAEF,MAAK,MAAM,UAAU,OACnB,KAAI,MAAM,SAAS,OAAO,EAAE;AAC1B,KAAG,OAAO,aAAa,OAAO,MAAM,YAAY;AAChD,QAAM,EAAE,gBAAgB,GAAG,OAAO,CAAC,aAAa,GAAG,UAAU;AAC7D;;;AAKN,eAAsB,yBACpB,KACA,IACA,OACe;CACf,MAAM,SAAS,UAAU,MAAM;CAC/B,MAAM,EAAC,UAAS;AAChB,OAAM,yBACJ,KACA,IACA,QACA,UAAU,SAAS,UAAU,OAAO,SACpC,MACD;CAED,MAAM,iBAA4B;EAChC,gBAAgB,IAAI,OAAO,eAAe,IAAI,IAAI,MAAM;EACxD,gBAAgB;EACjB;CAED,eAAe,cAAc,GAAe,IAAyB;AACnE,QAAM,EAAE;kBACM,GAAG,OAAO,CAAC;;CAG3B,MAAM,gBAAgB;EACpB,eAAe,OAAO,GAAe,OAA4B;AAC/D,SAAM,GAAG,OAAO,4BAA4B,MAAM,CAAC;;EAGrD,aAAa,OAAO,GAAe,OAA4B;GAE7D,MAAM,mBAA8C,EAAC,eAD/B,MAAM,mBAAmB,IAAI,MAAM,EACU;AACnE,SAAM,EAAE;uBACS,GAAG,OAAO,CAAC;AAC5B,SAAM,EAAE;oBACM,GAAG,OAAO,CAAC,sBAAsB,GAAG,iBAAiB;;EAEtE;CAED,MAAM,gBAAgB,EACpB,eAAe,OAAO,GAAe,OAA4B;AAC/D,QAAM,EAAE;oBACM,GAAG,OAAO,CAAC;;IAG5B;CAED,MAAM,gBAAgB,EACpB,eAAe,OAAO,GAAe,OAA4B;AAC/D,QAAM,EAAE;oBACM,GAAG,OAAO,CAAC;;IAG5B;CAED,MAAM,gBAAgB,EACpB,eAAe,OAAO,GAAe,OAA4B;AAC/D,QAAM,GAAG,OAAO,qBAAqB,MAAM,CAAC;IAE/C;AAUD,OAAM,oBACJ,KACA,mBACA,QACA,IACA,gBAbyD;EACzD,GAAG,EAAC,eAAe,eAAc;EACjC,GAAG;EACH,GAAG;EACH,GAAG;EACH,GAAG;EACJ,CASA;;AAGH,eAAsB,mBACpB,IACA,OACiB;CACjB,MAAM,SAAS,UAAU,MAAM;CAC/B,MAAM,CAAC,EAAC,SAAQ,MAAM,EAA0B;wCACV,GAAG,OAAO,CAAC;AACjD,KAAI,QAAQ,KACV,QAAO;CAIT,MAAM,CAAC,EAAC,oBAAmB,MAAM,EAA8B;mCAC9B,GAAG,OAAO,CAAC;;AAE5C,QAAO"}