@rocicorp/zero 1.12.0-canary.0 → 1.12.0-canary.10

package/out/zero-cache/src/services/change-streamer/restore-admission.js.map

@@ -1 +1 @@
-{"version":3,"file":"restore-admission.js","names":["#lc","#maxConcurrentRestores","#pendingGrantTTLMs","#readinessGracePeriodMs","#setTimeout","#clearTimeout","#replicaVersion","#replacing","#catchingUp","#unclassified","#grants","#queue","#inFlight","#replacementOpened","#startGrantTTL","#processQueue","#deleteGrant","#release"],"sources":["../../../../../../zero-cache/src/services/change-streamer/restore-admission.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {getOrCreateGauge} from '../../observability/metrics.ts';\nimport type {LocalReplica} from './subscribe.ts';\n\n/**\n * The default time to wait for a subscriber that was granted permission to\n * restore to reconnect (i.e. to open a replacement reservation) after its\n * held connection closes, or after its replacement reservation closes\n * without subscribing (e.g. to retry a failed restore). This covers the\n * subscriber's process exit and restart (e.g. a container restart, the\n * backoff of which starts at 10 seconds in Kubernetes), but not the restore\n * itself, which is tracked by the liveness of its snapshot reservation.\n */\nexport const DEFAULT_PENDING_GRANT_TTL_MS = 90_000;\n\n/**\n * The default time for which a replacement continues to be counted after its\n * subscriber has caught up, which covers the time for the subscriber to start\n * serving (i.e. become ready) after catching up. See\n * {@link RestoreAdmissionOptions.readinessGracePeriodMs}.\n */\nexport const DEFAULT_READINESS_GRACE_PERIOD_MS = 5_000;\n\nexport type RestoreAdmissionOptions = {\n  /** The replica version of the replication-manager. */\n  replicaVersion: string;\n  /**\n   * The maximum number of subscribers that may be restoring (or granted\n   * permission to restore) at the same time.\n   */\n  maxConcurrentRestores: number;\n  pendingGrantTTLMs?: number | undefined;\n  /**\n   * The time for which a replacement continues to be counted after its\n   * subscriber has caught up, so that the next subscriber is not granted\n   * permission to restore (and stop serving) before the restored one has\n   * started serving. Catching up is followed by the subscriber's readiness\n   * checks (e.g. waiting for the next replication lag report), and by the\n   * container runner's detection of its readiness.\n   */\n  readinessGracePeriodMs?: number | undefined;\n  // Injectable timers, for deterministic testing.\n  setTimeoutFn?: typeof setTimeout | undefined;\n  clearTimeoutFn?: typeof clearTimeout | undefined;\n};\n\nexport type AdmissionRequest = {\n  /**\n   * Called when the request is queued, and again whenever its (1-based)\n   * position in the queue changes.\n   */\n  onQueued: (position: number) => void;\n\n  /** Called (once) when the request is granted. */\n  onGranted: () => void;\n};\n\n/** A subscribed replacement, returned by {@link RestoreAdmission.reservationSubscribed}. */\nexport type Replacement = {\n  /**\n   * Called when the subscriber has caught up. The replacement is released\n   * after the readiness grace period.\n   */\n  caughtUp: () => void;\n\n  /** Called when the subscription closes. The replacement is released. */\n  closed: () => void;\n};\n\nexport type AdmissionTicket = {\n  /**\n   * Must be called when the requesting connection closes. A queued request\n   * is removed from the queue. For a granted request, this starts the\n   * timer within which the subscriber is expected to reconnect and open a\n   * replacement reservation.\n   */\n  release(): void;\n};\n\ntype Entry = {\n  readonly request: AdmissionRequest;\n  state: 'queued' | 'granted' | 'released';\n  position: number;\n  grant?: Grant | undefined;\n};\n\ntype Grant = {\n  /** Whether the granted subscriber's connection has closed. */\n  released: boolean;\n  timer: ReturnType<typeof setTimeout> | undefined;\n};\n\n/**\n * Limits the number of view-syncers that replace their replica at the same\n * time, so that view-syncers switching over from an incompatible\n * replication-manager (e.g. after a resync) do not all stop serving at once.\n *\n * Capacity is consumed by:\n * * Replacement restores: snapshot reservations of subscribers that are\n *   replacing an (incompatible or outdated) local replica, i.e. subscribers\n *   that were serving. Restores onto an empty disk (e.g. a new pod) do not\n *   take a serving subscriber out of service, and are not counted. The\n *   reservation's connection is held open for the duration of the restore,\n *   and the subscription that follows it is held open until the restored\n *   subscriber has caught up, so their liveness tracks that of the\n *   replacement without a timeout.\n * * Outstanding grants: permission to restore that has not yet been\n *   followed by a replacement reservation (e.g. while the subscriber drains\n *   and restarts). Grants are not matched to reservations by identity:\n *   any replacement reservation consumes an outstanding grant, and a grant\n *   that is not consumed expires a short time after the granted connection\n *   closes (e.g. if the subscriber reconnected to a different\n *   replication-manager or was deleted).\n *\n * ## Life cycle of a switchover restore\n *\n * The following traces a view-syncer (VS) with a replica at version `R1`,\n * served by an old replication-manager (RM) at `R1`, while a new RM (e.g.\n * after a resync) is at `R2 > R1`. The RestoreAdmission is that of the new\n * RM. Terms in `code` are those used in the code.\n *\n * 1. **Drain.** The old RM receives a SIGTERM, and its `DrainTracker` sends\n *    `status {draining: true}` to its `drainable` (i.e. serving) subscribers.\n *    It continues to stream changes to them, and exits once they have all\n *    disconnected.\n *\n * 2. **Switchover.** The VS's `DrainHandler` opens a second `/subscribe`\n *    with `switchover: true` at the watermark of its last committed\n *    transaction, which is routed to some RM. The VS continues to consume\n *    the old RM's stream. If the RM is compatible (i.e. another old RM),\n *    the subscription is accepted and the VS simply moves to it. If the RM\n *    is the new one, the subscriber's replica version is older, and the\n *    subscription proceeds to step 3.\n *\n * 3. **Queued.** `ChangeStreamerImpl.subscribe()` holds the subscription\n *    (it is not added to the forwarder) and calls {@link request}. The\n *    request is `queued` and `onQueued(position)` sends\n *    `status {restore: {state: 'queued', position}}`, again whenever the\n *    position changes. If the held connection closes while queued (e.g.\n *    the VS's old stream ended and it gave up on the switchover), the\n *    ticket is `release()`d and the request leaves the queue.\n *\n * 4. **Granted.** When the number in flight (`replacing` + `granted`) is\n *    below `maxConcurrentRestores`, the head of the queue is `granted`: a\n *    {@link Grant} is added (counting toward capacity), and `onGranted()`\n *    sends `status {restore: {state: 'granted'}}`. The VS keeps the granted\n *    connection open, and initiates a self-drain (`drainRequested`): it\n *    fails its health checks, drains its clients, and keeps replicating\n *    from the old RM in the meantime.\n *\n * 5. **Restart.** The VS process exits, closing the granted connection.\n *    The ticket is `release()`d, marking the grant as `released` and\n *    starting the pending grant TTL ({@link DEFAULT_PENDING_GRANT_TTL_MS}),\n *    within which a replacement reservation is expected. The drain itself\n *    does not count against the TTL.\n *\n * 6. **Reservation opened.** The restarted VS sends\n *    `reserve-snapshot {taskID, localReplica}`, and\n *    `ChangeStreamerImpl.startSnapshotReservation()` calls\n *    {@link reservationOpened}:\n *    * A `localReplica` of a different replica version (e.g. `R1`) is\n *      `replacing`: the reservation is counted, and consumes an outstanding\n *      grant (preferring a `released` one), so the number in flight is\n *      unchanged.\n *    * A `localReplica` of the same replica version is `unclassified` until\n *      the reservation is confirmed ({@link reservationConfirmed}) with the\n *      backup's minimum watermark. It is `replacing` if its watermark is\n *      older (i.e. it cannot catch up from the change log).\n *    * No `localReplica` (an empty disk) is not counted.\n *\n *    If the VS reached a different RM instead, the grant here expires after\n *    the TTL and frees its capacity, while the replacement is counted (as\n *    an ungranted one) by the RM that it reached.\n *\n * 7. **Restoring.** The VS restores the backup while holding the\n *    reservation's connection open. A retried reservation from the same\n *    task that arrives while the previous one is still open replaces it\n *    (`SnapshotReservations` `onClose`) but does not end it (`onEnd`), so it\n *    remains `replacing`. If the reservation instead closes without\n *    subscribing (e.g. a failed or disconnected restore, after which the VS\n *    retries), {@link reservationClosed} converts its capacity into a\n *    `released` grant, which the retry's replacement reservation consumes\n *    (step 6), or which expires after the pending grant TTL (e.g. if the VS\n *    was deleted).\n *\n * 8. **Subscribed.** Once restored, the VS subscribes on the same\n *    connection at `R2`, which is accepted. `ChangeStreamerImpl.subscribe()`\n *    calls {@link reservationSubscribed}, which moves the replacement from\n *    its reservation (the key leaves `replacing`) to the subscription\n *    (`catchingUp`), and then closes the task's reservation. Its `onEnd`\n *    calls {@link reservationClosed}, which is a no-op for the moved key.\n *\n * 9. **Caught up.** The VS catches up on the change stream from the\n *    backup's watermark. Once it has acked all changes up to the head of\n *    the stream (`Subscriber.caughtUp`), the replacement is released after\n *    the readiness grace period, which covers the time for the VS to start\n *    serving, and the next queued request is granted. If the VS disconnects\n *    first, the replacement is released immediately.\n *\n * A VS that restarts outside of a switchover with an incompatible replica\n * (e.g. after a crash) enters at step 6 without a grant, and its restore\n * consumes capacity all the same, delaying queued requests.\n *\n * ## States\n *\n * | Term          | Where                    | Meaning                          |\n * | ------------- | ------------------------ | -------------------------------- |\n * | queued        | `Entry.state`            | Held, waiting for capacity       |\n * | granted       | `Entry.state`, `#grants` | Permitted to restore, pending a  |\n * |               |                          | replacement reservation          |\n * | released      | `Entry.state`            | The held connection closed       |\n * | released      | `Grant.released`         | The granted connection, or an    |\n * |               |                          | unsubscribed replacement         |\n * |               |                          | reservation, closed; the TTL is  |\n * |               |                          | running                          |\n * | unclassified  | `#unclassified`          | Same-version reservation, not    |\n * |               |                          | yet confirmed                    |\n * | replacing     | `#replacing`             | Open replacement reservation     |\n * | catching up   | `#catchingUp`            | Subscribed replacement, not yet  |\n * |               |                          | caught up (or within the         |\n * |               |                          | readiness grace period)          |\n * | in flight     | `#inFlight`              | `replacing` + `catchingUp` +     |\n * |               |                          | `granted`, bounded by            |\n * |               |                          | `maxConcurrentRestores`          |\n */\nexport class RestoreAdmission {\n  readonly #lc: LogContext;\n  readonly #maxConcurrentRestores: number;\n  readonly #pendingGrantTTLMs: number;\n  readonly #readinessGracePeriodMs: number;\n  readonly #setTimeout: typeof setTimeout;\n  readonly #clearTimeout: typeof clearTimeout;\n\n  readonly #replicaVersion: string;\n  /** Keys (task IDs) of open replacement reservations. */\n  readonly #replacing = new Set<string>();\n  /**\n   * Replacements whose subscribers have subscribed, but have not caught up\n   * (or are within the readiness grace period).\n   */\n  readonly #catchingUp = new Set<object>();\n  /**\n   * The local replicas of open reservations with this replica version, which\n   * are classified upon confirmation of the reservation's minimum watermark.\n   */\n  readonly #unclassified = new Map<string, LocalReplica>();\n  readonly #grants = new Set<Grant>();\n  readonly #queue: Entry[] = [];\n\n  constructor(lc: LogContext, opts: RestoreAdmissionOptions) {\n    this.#lc = lc.withContext('component', 'restore-admission');\n    this.#replicaVersion = opts.replicaVersion;\n    this.#maxConcurrentRestores = Math.max(1, opts.maxConcurrentRestores);\n    this.#pendingGrantTTLMs =\n      opts.pendingGrantTTLMs ?? DEFAULT_PENDING_GRANT_TTL_MS;\n    this.#readinessGracePeriodMs =\n      opts.readinessGracePeriodMs ?? DEFAULT_READINESS_GRACE_PERIOD_MS;\n    this.#setTimeout = opts.setTimeoutFn ?? setTimeout;\n    this.#clearTimeout = opts.clearTimeoutFn ?? clearTimeout;\n\n    getOrCreateGauge(\n      'replication',\n      'restore_admission.queued',\n      'Switchover subscribers waiting for permission to restore a new replica.',\n    ).addCallback(result => result.observe(this.#queue.length));\n    getOrCreateGauge(\n      'replication',\n      'restore_admission.in_flight',\n      'Subscribers replacing, or granted permission to replace, their replica.',\n    ).addCallback(result => result.observe(this.#inFlight));\n  }\n\n  get #inFlight() {\n    return this.#replacing.size + this.#catchingUp.size + this.#grants.size;\n  }\n\n  /** For testing and logging. */\n  get stats() {\n    return {\n      replacing: this.#replacing.size + this.#catchingUp.size,\n      granted: this.#grants.size,\n      queued: this.#queue.length,\n    };\n  }\n\n  /**\n   * Called when a snapshot reservation is opened by a subscriber with the\n   * given `localReplica` (if any). A subscriber with a local replica of a\n   * different replica version is replacing it. One with the same replica\n   * version is replacing it if it is outdated, which is determined by\n   * {@link reservationConfirmed}. Idempotent for the same `key` (e.g. a\n   * retried reservation).\n   */\n  reservationOpened(key: string, localReplica: LocalReplica | undefined) {\n    this.#unclassified.delete(key);\n    if (!localReplica) {\n      return; // Restoring onto an empty disk.\n    }\n    if (localReplica.replicaVersion !== this.#replicaVersion) {\n      this.#replacementOpened(key);\n    } else {\n      this.#unclassified.set(key, localReplica);\n    }\n  }\n\n  /**\n   * Called when a reservation is confirmed with the minimum watermark of the\n   * replica to restore.\n   */\n  reservationConfirmed(key: string, minWatermark: string) {\n    const localReplica = this.#unclassified.get(key);\n    this.#unclassified.delete(key);\n    if (localReplica && localReplica.watermark < minWatermark) {\n      this.#replacementOpened(key); // The local replica is outdated.\n    }\n  }\n\n  /**\n   * Called when a reservation ends. A replacement reservation normally ends\n   * after its task subscribes ({@link reservationSubscribed}), in which case\n   * this is a no-op. A replacement reservation that ends without subscribing\n   * (e.g. a failed or disconnected restore) is most likely followed by a\n   * retry, so its capacity is retained as a released grant (which the retry\n   * consumes) rather than released immediately.\n   */\n  reservationClosed(key: string) {\n    this.#unclassified.delete(key);\n    if (this.#replacing.delete(key)) {\n      const grant: Grant = {released: true, timer: undefined};\n      this.#grants.add(grant);\n      this.#startGrantTTL(grant);\n    }\n  }\n\n  /**\n   * Called when the task of a reservation subscribes (before the reservation\n   * is closed). If the reservation is a replacement, it continues to be\n   * counted until the readiness grace period after the subscriber has caught\n   * up, or until the subscription closes.\n   */\n  reservationSubscribed(key: string): Replacement | undefined {\n    this.#unclassified.delete(key);\n    if (!this.#replacing.delete(key)) {\n      return undefined;\n    }\n    const replacement = {};\n    this.#catchingUp.add(replacement);\n    let timer: ReturnType<typeof setTimeout> | undefined;\n    const release = () => {\n      if (timer) {\n        this.#clearTimeout(timer);\n      }\n      if (this.#catchingUp.delete(replacement)) {\n        this.#processQueue();\n      }\n    };\n    return {\n      caughtUp: () => {\n        if (this.#catchingUp.has(replacement) && !timer) {\n          timer = this.#setTimeout(release, this.#readinessGracePeriodMs);\n        }\n      },\n      closed: release,\n    };\n  }\n\n  #replacementOpened(key: string) {\n    if (this.#replacing.has(key)) {\n      return;\n    }\n    this.#replacing.add(key);\n    // The restore is now tracked by its reservation. Prefer consuming a grant\n    // whose subscriber has already disconnected, as it is the most likely\n    // origin of the reservation.\n    const grants = [...this.#grants];\n    const grant = grants.find(g => g.released) ?? grants[0];\n    if (grant) {\n      this.#deleteGrant(grant);\n    }\n  }\n\n  request(request: AdmissionRequest): AdmissionTicket {\n    const entry: Entry = {request, state: 'queued', position: 0};\n    this.#queue.push(entry);\n    this.#processQueue();\n    return {release: () => this.#release(entry)};\n  }\n\n  #release(entry: Entry) {\n    const {state, grant} = entry;\n    entry.state = 'released';\n    if (state === 'queued') {\n      const i = this.#queue.indexOf(entry);\n      if (i >= 0) {\n        this.#queue.splice(i, 1);\n      }\n      this.#processQueue();\n    } else if (state === 'granted' && grant && this.#grants.has(grant)) {\n      // Wait for the subscriber to reconnect with a replacement reservation.\n      grant.released = true;\n      this.#startGrantTTL(grant);\n    }\n  }\n\n  #startGrantTTL(grant: Grant) {\n    grant.timer = this.#setTimeout(() => {\n      if (this.#grants.has(grant)) {\n        this.#lc.info?.(`grant expired without a replacement reservation`);\n        this.#deleteGrant(grant);\n        this.#processQueue();\n      }\n    }, this.#pendingGrantTTLMs);\n  }\n\n  #deleteGrant(grant: Grant) {\n    if (grant.timer) {\n      this.#clearTimeout(grant.timer);\n    }\n    this.#grants.delete(grant);\n  }\n\n  #processQueue() {\n    while (this.#inFlight < this.#maxConcurrentRestores && this.#queue.length) {\n      const entry = this.#queue.shift()!;\n      entry.state = 'granted';\n      entry.grant = {released: false, timer: undefined};\n      this.#grants.add(entry.grant);\n      this.#lc.info?.(`granting permission to restore`, {...this.stats});\n      entry.request.onGranted();\n    }\n    this.#queue.forEach((entry, i) => {\n      const position = i + 1;\n      if (entry.position !== position) {\n        entry.position = position;\n        entry.request.onQueued(position);\n      }\n    });\n  }\n}\n"],"mappings":";;;;;;;AAqBA,IAAa,oCAAoC;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AA4MjD,IAAa,mBAAb,MAA8B;CAC5B;CACA;CACA;CACA;CACA;CACA;CAEA;;CAEA,6BAAsB,IAAI,IAAY;;;;;CAKtC,8BAAuB,IAAI,IAAY;;;;;CAKvC,gCAAyB,IAAI,IAA0B;CACvD,0BAAmB,IAAI,IAAW;CAClC,SAA2B,CAAC;CAE5B,YAAY,IAAgB,MAA+B;EACzD,KAAKA,MAAM,GAAG,YAAY,aAAa,mBAAmB;EAC1D,KAAKM,kBAAkB,KAAK;EAC5B,KAAKL,yBAAyB,KAAK,IAAI,GAAG,KAAK,qBAAqB;EACpE,KAAKC,qBACH,KAAK,qBAAA;EACP,KAAKC,0BACH,KAAK,0BAAA;EACP,KAAKC,cAAc,KAAK,gBAAgB;EACxC,KAAKC,gBAAgB,KAAK,kBAAkB;EAE5C,iBACE,eACA,4BACA,yEACF,EAAE,aAAY,WAAU,OAAO,QAAQ,KAAKM,OAAO,MAAM,CAAC;EAC1D,iBACE,eACA,+BACA,yEACF,EAAE,aAAY,WAAU,OAAO,QAAQ,KAAKC,SAAS,CAAC;CACxD;CAEA,IAAIA,YAAY;EACd,OAAO,KAAKL,WAAW,OAAO,KAAKC,YAAY,OAAO,KAAKE,QAAQ;CACrE;;CAGA,IAAI,QAAQ;EACV,OAAO;GACL,WAAW,KAAKH,WAAW,OAAO,KAAKC,YAAY;GACnD,SAAS,KAAKE,QAAQ;GACtB,QAAQ,KAAKC,OAAO;EACtB;CACF;;;;;;;;;CAUA,kBAAkB,KAAa,cAAwC;EACrE,KAAKF,cAAc,OAAO,GAAG;EAC7B,IAAI,CAAC,cACH;EAEF,IAAI,aAAa,mBAAmB,KAAKH,iBACvC,KAAKO,mBAAmB,GAAG;OAE3B,KAAKJ,cAAc,IAAI,KAAK,YAAY;CAE5C;;;;;CAMA,qBAAqB,KAAa,cAAsB;EACtD,MAAM,eAAe,KAAKA,cAAc,IAAI,GAAG;EAC/C,KAAKA,cAAc,OAAO,GAAG;EAC7B,IAAI,gBAAgB,aAAa,YAAY,cAC3C,KAAKI,mBAAmB,GAAG;CAE/B;;;;;;;;;CAUA,kBAAkB,KAAa;EAC7B,KAAKJ,cAAc,OAAO,GAAG;EAC7B,IAAI,KAAKF,WAAW,OAAO,GAAG,GAAG;GAC/B,MAAM,QAAe;IAAC,UAAU;IAAM,OAAO,KAAA;GAAS;GACtD,KAAKG,QAAQ,IAAI,KAAK;GACtB,KAAKI,eAAe,KAAK;EAC3B;CACF;;;;;;;CAQA,sBAAsB,KAAsC;EAC1D,KAAKL,cAAc,OAAO,GAAG;EAC7B,IAAI,CAAC,KAAKF,WAAW,OAAO,GAAG,GAC7B;EAEF,MAAM,cAAc,CAAC;EACrB,KAAKC,YAAY,IAAI,WAAW;EAChC,IAAI;EACJ,MAAM,gBAAgB;GACpB,IAAI,OACF,KAAKH,cAAc,KAAK;GAE1B,IAAI,KAAKG,YAAY,OAAO,WAAW,GACrC,KAAKO,cAAc;EAEvB;EACA,OAAO;GACL,gBAAgB;IACd,IAAI,KAAKP,YAAY,IAAI,WAAW,KAAK,CAAC,OACxC,QAAQ,KAAKJ,YAAY,SAAS,KAAKD,uBAAuB;GAElE;GACA,QAAQ;EACV;CACF;CAEA,mBAAmB,KAAa;EAC9B,IAAI,KAAKI,WAAW,IAAI,GAAG,GACzB;EAEF,KAAKA,WAAW,IAAI,GAAG;EAIvB,MAAM,SAAS,CAAC,GAAG,KAAKG,OAAO;EAC/B,MAAM,QAAQ,OAAO,MAAK,MAAK,EAAE,QAAQ,KAAK,OAAO;EACrD,IAAI,OACF,KAAKM,aAAa,KAAK;CAE3B;CAEA,QAAQ,SAA4C;EAClD,MAAM,QAAe;GAAC;GAAS,OAAO;GAAU,UAAU;EAAC;EAC3D,KAAKL,OAAO,KAAK,KAAK;EACtB,KAAKI,cAAc;EACnB,OAAO,EAAC,eAAe,KAAKE,SAAS,KAAK,EAAC;CAC7C;CAEA,SAAS,OAAc;EACrB,MAAM,EAAC,OAAO,UAAS;EACvB,MAAM,QAAQ;EACd,IAAI,UAAU,UAAU;GACtB,MAAM,IAAI,KAAKN,OAAO,QAAQ,KAAK;GACnC,IAAI,KAAK,GACP,KAAKA,OAAO,OAAO,GAAG,CAAC;GAEzB,KAAKI,cAAc;EACrB,OAAO,IAAI,UAAU,aAAa,SAAS,KAAKL,QAAQ,IAAI,KAAK,GAAG;GAElE,MAAM,WAAW;GACjB,KAAKI,eAAe,KAAK;EAC3B;CACF;CAEA,eAAe,OAAc;EAC3B,MAAM,QAAQ,KAAKV,kBAAkB;GACnC,IAAI,KAAKM,QAAQ,IAAI,KAAK,GAAG;IAC3B,KAAKV,IAAI,OAAO,iDAAiD;IACjE,KAAKgB,aAAa,KAAK;IACvB,KAAKD,cAAc;GACrB;EACF,GAAG,KAAKb,kBAAkB;CAC5B;CAEA,aAAa,OAAc;EACzB,IAAI,MAAM,OACR,KAAKG,cAAc,MAAM,KAAK;EAEhC,KAAKK,QAAQ,OAAO,KAAK;CAC3B;CAEA,gBAAgB;EACd,OAAO,KAAKE,YAAY,KAAKX,0BAA0B,KAAKU,OAAO,QAAQ;GACzE,MAAM,QAAQ,KAAKA,OAAO,MAAM;GAChC,MAAM,QAAQ;GACd,MAAM,QAAQ;IAAC,UAAU;IAAO,OAAO,KAAA;GAAS;GAChD,KAAKD,QAAQ,IAAI,MAAM,KAAK;GAC5B,KAAKV,IAAI,OAAO,kCAAkC,EAAC,GAAG,KAAK,MAAK,CAAC;GACjE,MAAM,QAAQ,UAAU;EAC1B;EACA,KAAKW,OAAO,SAAS,OAAO,MAAM;GAChC,MAAM,WAAW,IAAI;GACrB,IAAI,MAAM,aAAa,UAAU;IAC/B,MAAM,WAAW;IACjB,MAAM,QAAQ,SAAS,QAAQ;GACjC;EACF,CAAC;CACH;AACF"}
+{"version":3,"file":"restore-admission.js","names":[],"sources":["../../../../../../zero-cache/src/services/change-streamer/restore-admission.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {getOrCreateGauge} from '../../observability/metrics.ts';\nimport type {LocalReplica} from './subscribe.ts';\n\n/**\n * The default time to wait for a subscriber that was granted permission to\n * restore to reconnect (i.e. to open a replacement reservation) after its\n * held connection closes, or after its replacement reservation closes\n * without subscribing (e.g. to retry a failed restore). This covers the\n * subscriber's process exit and restart (e.g. a container restart, the\n * backoff of which starts at 10 seconds in Kubernetes), but not the restore\n * itself, which is tracked by the liveness of its snapshot reservation.\n */\nconst DEFAULT_PENDING_GRANT_TTL_MS = 90_000;\n\n/**\n * The default time for which a replacement continues to be counted after its\n * subscriber has caught up, which covers the time for the subscriber to start\n * serving (i.e. become ready) after catching up. See\n * {@link RestoreAdmissionOptions.readinessGracePeriodMs}.\n */\nexport const DEFAULT_READINESS_GRACE_PERIOD_MS = 5_000;\n\nexport type RestoreAdmissionOptions = {\n  /** The replica version of the replication-manager. */\n  replicaVersion: string;\n  /**\n   * The maximum number of subscribers that may be restoring (or granted\n   * permission to restore) at the same time.\n   */\n  maxConcurrentRestores: number;\n  pendingGrantTTLMs?: number | undefined;\n  /**\n   * The time for which a replacement continues to be counted after its\n   * subscriber has caught up, so that the next subscriber is not granted\n   * permission to restore (and stop serving) before the restored one has\n   * started serving. Catching up is followed by the subscriber's readiness\n   * checks (e.g. waiting for the next replication lag report), and by the\n   * container runner's detection of its readiness.\n   */\n  readinessGracePeriodMs?: number | undefined;\n  // Injectable timers, for deterministic testing.\n  setTimeoutFn?: typeof setTimeout | undefined;\n  clearTimeoutFn?: typeof clearTimeout | undefined;\n};\n\nexport type AdmissionRequest = {\n  /**\n   * Called when the request is queued, and again whenever its (1-based)\n   * position in the queue changes.\n   */\n  onQueued: (position: number) => void;\n\n  /** Called (once) when the request is granted. */\n  onGranted: () => void;\n};\n\n/** A subscribed replacement, returned by {@link RestoreAdmission.reservationSubscribed}. */\nexport type Replacement = {\n  /**\n   * Called when the subscriber has caught up. The replacement is released\n   * after the readiness grace period.\n   */\n  caughtUp: () => void;\n\n  /** Called when the subscription closes. The replacement is released. */\n  closed: () => void;\n};\n\nexport type AdmissionTicket = {\n  /**\n   * Must be called when the requesting connection closes. A queued request\n   * is removed from the queue. For a granted request, this starts the\n   * timer within which the subscriber is expected to reconnect and open a\n   * replacement reservation.\n   */\n  release(): void;\n};\n\ntype Entry = {\n  readonly request: AdmissionRequest;\n  state: 'queued' | 'granted' | 'released';\n  position: number;\n  grant?: Grant | undefined;\n};\n\ntype Grant = {\n  /** Whether the granted subscriber's connection has closed. */\n  released: boolean;\n  timer: ReturnType<typeof setTimeout> | undefined;\n};\n\n/**\n * Limits the number of view-syncers that replace their replica at the same\n * time, so that view-syncers switching over from an incompatible\n * replication-manager (e.g. after a resync) do not all stop serving at once.\n *\n * Capacity is consumed by:\n * * Replacement restores: snapshot reservations of subscribers that are\n *   replacing an (incompatible or outdated) local replica, i.e. subscribers\n *   that were serving. Restores onto an empty disk (e.g. a new pod) do not\n *   take a serving subscriber out of service, and are not counted. The\n *   reservation's connection is held open for the duration of the restore,\n *   and the subscription that follows it is held open until the restored\n *   subscriber has caught up, so their liveness tracks that of the\n *   replacement without a timeout.\n * * Outstanding grants: permission to restore that has not yet been\n *   followed by a replacement reservation (e.g. while the subscriber drains\n *   and restarts). Grants are not matched to reservations by identity:\n *   any replacement reservation consumes an outstanding grant, and a grant\n *   that is not consumed expires a short time after the granted connection\n *   closes (e.g. if the subscriber reconnected to a different\n *   replication-manager or was deleted).\n *\n * ## Life cycle of a switchover restore\n *\n * The following traces a view-syncer (VS) with a replica at version `R1`,\n * served by an old replication-manager (RM) at `R1`, while a new RM (e.g.\n * after a resync) is at `R2 > R1`. The RestoreAdmission is that of the new\n * RM. Terms in `code` are those used in the code.\n *\n * 1. **Drain.** The old RM receives a SIGTERM, and its `DrainTracker` sends\n *    `status {draining: true}` to its `drainable` (i.e. serving) subscribers.\n *    It continues to stream changes to them, and exits once they have all\n *    disconnected.\n *\n * 2. **Switchover.** The VS's `DrainHandler` opens a second `/subscribe`\n *    with `switchover: true` at the watermark of its last committed\n *    transaction, which is routed to some RM. The VS continues to consume\n *    the old RM's stream. If the RM is compatible (i.e. another old RM),\n *    the subscription is accepted and the VS simply moves to it. If the RM\n *    is the new one, the subscriber's replica version is older, and the\n *    subscription proceeds to step 3.\n *\n * 3. **Queued.** `ChangeStreamerImpl.subscribe()` holds the subscription\n *    (it is not added to the forwarder) and calls {@link request}. The\n *    request is `queued` and `onQueued(position)` sends\n *    `status {restore: {state: 'queued', position}}`, again whenever the\n *    position changes. If the held connection closes while queued (e.g.\n *    the VS's old stream ended and it gave up on the switchover), the\n *    ticket is `release()`d and the request leaves the queue.\n *\n * 4. **Granted.** When the number in flight (`replacing` + `granted`) is\n *    below `maxConcurrentRestores`, the head of the queue is `granted`: a\n *    {@link Grant} is added (counting toward capacity), and `onGranted()`\n *    sends `status {restore: {state: 'granted'}}`. The VS keeps the granted\n *    connection open, and initiates a self-drain (`drainRequested`): it\n *    fails its health checks, drains its clients, and keeps replicating\n *    from the old RM in the meantime.\n *\n * 5. **Restart.** The VS process exits, closing the granted connection.\n *    The ticket is `release()`d, marking the grant as `released` and\n *    starting the pending grant TTL ({@link DEFAULT_PENDING_GRANT_TTL_MS}),\n *    within which a replacement reservation is expected. The drain itself\n *    does not count against the TTL.\n *\n * 6. **Reservation opened.** The restarted VS sends\n *    `reserve-snapshot {taskID, localReplica}`, and\n *    `ChangeStreamerImpl.startSnapshotReservation()` calls\n *    {@link reservationOpened}:\n *    * A `localReplica` of a different replica version (e.g. `R1`) is\n *      `replacing`: the reservation is counted, and consumes an outstanding\n *      grant (preferring a `released` one), so the number in flight is\n *      unchanged.\n *    * A `localReplica` of the same replica version is `unclassified` until\n *      the reservation is confirmed ({@link reservationConfirmed}) with the\n *      backup's minimum watermark. It is `replacing` if its watermark is\n *      older (i.e. it cannot catch up from the change log).\n *    * No `localReplica` (an empty disk) is not counted.\n *\n *    If the VS reached a different RM instead, the grant here expires after\n *    the TTL and frees its capacity, while the replacement is counted (as\n *    an ungranted one) by the RM that it reached.\n *\n * 7. **Restoring.** The VS restores the backup while holding the\n *    reservation's connection open. A retried reservation from the same\n *    task that arrives while the previous one is still open replaces it\n *    (`SnapshotReservations` `onClose`) but does not end it (`onEnd`), so it\n *    remains `replacing`. If the reservation instead closes without\n *    subscribing (e.g. a failed or disconnected restore, after which the VS\n *    retries), {@link reservationClosed} converts its capacity into a\n *    `released` grant, which the retry's replacement reservation consumes\n *    (step 6), or which expires after the pending grant TTL (e.g. if the VS\n *    was deleted).\n *\n * 8. **Subscribed.** Once restored, the VS subscribes on the same\n *    connection at `R2`, which is accepted. `ChangeStreamerImpl.subscribe()`\n *    calls {@link reservationSubscribed}, which moves the replacement from\n *    its reservation (the key leaves `replacing`) to the subscription\n *    (`catchingUp`), and then closes the task's reservation. Its `onEnd`\n *    calls {@link reservationClosed}, which is a no-op for the moved key.\n *\n * 9. **Caught up.** The VS catches up on the change stream from the\n *    backup's watermark. Once it has acked all changes up to the head of\n *    the stream (`Subscriber.caughtUp`), the replacement is released after\n *    the readiness grace period, which covers the time for the VS to start\n *    serving, and the next queued request is granted. If the VS disconnects\n *    first, the replacement is released immediately.\n *\n * A VS that restarts outside of a switchover with an incompatible replica\n * (e.g. after a crash) enters at step 6 without a grant, and its restore\n * consumes capacity all the same, delaying queued requests.\n *\n * ## States\n *\n * | Term          | Where                    | Meaning                          |\n * | ------------- | ------------------------ | -------------------------------- |\n * | queued        | `Entry.state`            | Held, waiting for capacity       |\n * | granted       | `Entry.state`, `#grants` | Permitted to restore, pending a  |\n * |               |                          | replacement reservation          |\n * | released      | `Entry.state`            | The held connection closed       |\n * | released      | `Grant.released`         | The granted connection, or an    |\n * |               |                          | unsubscribed replacement         |\n * |               |                          | reservation, closed; the TTL is  |\n * |               |                          | running                          |\n * | unclassified  | `#unclassified`          | Same-version reservation, not    |\n * |               |                          | yet confirmed                    |\n * | replacing     | `#replacing`             | Open replacement reservation     |\n * | catching up   | `#catchingUp`            | Subscribed replacement, not yet  |\n * |               |                          | caught up (or within the         |\n * |               |                          | readiness grace period)          |\n * | in flight     | `#inFlight`              | `replacing` + `catchingUp` +     |\n * |               |                          | `granted`, bounded by            |\n * |               |                          | `maxConcurrentRestores`          |\n */\nexport class RestoreAdmission {\n  readonly #lc: LogContext;\n  readonly #maxConcurrentRestores: number;\n  readonly #pendingGrantTTLMs: number;\n  readonly #readinessGracePeriodMs: number;\n  readonly #setTimeout: typeof setTimeout;\n  readonly #clearTimeout: typeof clearTimeout;\n\n  readonly #replicaVersion: string;\n  /** Keys (task IDs) of open replacement reservations. */\n  readonly #replacing = new Set<string>();\n  /**\n   * Replacements whose subscribers have subscribed, but have not caught up\n   * (or are within the readiness grace period).\n   */\n  readonly #catchingUp = new Set<object>();\n  /**\n   * The local replicas of open reservations with this replica version, which\n   * are classified upon confirmation of the reservation's minimum watermark.\n   */\n  readonly #unclassified = new Map<string, LocalReplica>();\n  readonly #grants = new Set<Grant>();\n  readonly #queue: Entry[] = [];\n\n  constructor(lc: LogContext, opts: RestoreAdmissionOptions) {\n    this.#lc = lc.withContext('component', 'restore-admission');\n    this.#replicaVersion = opts.replicaVersion;\n    this.#maxConcurrentRestores = Math.max(1, opts.maxConcurrentRestores);\n    this.#pendingGrantTTLMs =\n      opts.pendingGrantTTLMs ?? DEFAULT_PENDING_GRANT_TTL_MS;\n    this.#readinessGracePeriodMs =\n      opts.readinessGracePeriodMs ?? DEFAULT_READINESS_GRACE_PERIOD_MS;\n    this.#setTimeout = opts.setTimeoutFn ?? setTimeout;\n    this.#clearTimeout = opts.clearTimeoutFn ?? clearTimeout;\n\n    getOrCreateGauge(\n      'replication',\n      'restore_admission.queued',\n      'Switchover subscribers waiting for permission to restore a new replica.',\n    ).addCallback(result => result.observe(this.#queue.length));\n    getOrCreateGauge(\n      'replication',\n      'restore_admission.in_flight',\n      'Subscribers replacing, or granted permission to replace, their replica.',\n    ).addCallback(result => result.observe(this.#inFlight));\n  }\n\n  get #inFlight() {\n    return this.#replacing.size + this.#catchingUp.size + this.#grants.size;\n  }\n\n  /** For testing and logging. */\n  get stats() {\n    return {\n      replacing: this.#replacing.size + this.#catchingUp.size,\n      granted: this.#grants.size,\n      queued: this.#queue.length,\n    };\n  }\n\n  /**\n   * Called when a snapshot reservation is opened by a subscriber with the\n   * given `localReplica` (if any). A subscriber with a local replica of a\n   * different replica version is replacing it. One with the same replica\n   * version is replacing it if it is outdated, which is determined by\n   * {@link reservationConfirmed}. Idempotent for the same `key` (e.g. a\n   * retried reservation).\n   */\n  reservationOpened(key: string, localReplica: LocalReplica | undefined) {\n    this.#unclassified.delete(key);\n    if (!localReplica) {\n      return; // Restoring onto an empty disk.\n    }\n    if (localReplica.replicaVersion !== this.#replicaVersion) {\n      this.#replacementOpened(key);\n    } else {\n      this.#unclassified.set(key, localReplica);\n    }\n  }\n\n  /**\n   * Called when a reservation is confirmed with the minimum watermark of the\n   * replica to restore.\n   */\n  reservationConfirmed(key: string, minWatermark: string) {\n    const localReplica = this.#unclassified.get(key);\n    this.#unclassified.delete(key);\n    if (localReplica && localReplica.watermark < minWatermark) {\n      this.#replacementOpened(key); // The local replica is outdated.\n    }\n  }\n\n  /**\n   * Called when a reservation ends. A replacement reservation normally ends\n   * after its task subscribes ({@link reservationSubscribed}), in which case\n   * this is a no-op. A replacement reservation that ends without subscribing\n   * (e.g. a failed or disconnected restore) is most likely followed by a\n   * retry, so its capacity is retained as a released grant (which the retry\n   * consumes) rather than released immediately.\n   */\n  reservationClosed(key: string) {\n    this.#unclassified.delete(key);\n    if (this.#replacing.delete(key)) {\n      const grant: Grant = {released: true, timer: undefined};\n      this.#grants.add(grant);\n      this.#startGrantTTL(grant);\n    }\n  }\n\n  /**\n   * Called when the task of a reservation subscribes (before the reservation\n   * is closed). If the reservation is a replacement, it continues to be\n   * counted until the readiness grace period after the subscriber has caught\n   * up, or until the subscription closes.\n   */\n  reservationSubscribed(key: string): Replacement | undefined {\n    this.#unclassified.delete(key);\n    if (!this.#replacing.delete(key)) {\n      return undefined;\n    }\n    const replacement = {};\n    this.#catchingUp.add(replacement);\n    let timer: ReturnType<typeof setTimeout> | undefined;\n    const release = () => {\n      if (timer) {\n        this.#clearTimeout(timer);\n      }\n      if (this.#catchingUp.delete(replacement)) {\n        this.#processQueue();\n      }\n    };\n    return {\n      caughtUp: () => {\n        if (this.#catchingUp.has(replacement) && !timer) {\n          timer = this.#setTimeout(release, this.#readinessGracePeriodMs);\n        }\n      },\n      closed: release,\n    };\n  }\n\n  #replacementOpened(key: string) {\n    if (this.#replacing.has(key)) {\n      return;\n    }\n    this.#replacing.add(key);\n    // The restore is now tracked by its reservation. Prefer consuming a grant\n    // whose subscriber has already disconnected, as it is the most likely\n    // origin of the reservation.\n    const grants = [...this.#grants];\n    const grant = grants.find(g => g.released) ?? grants[0];\n    if (grant) {\n      this.#deleteGrant(grant);\n    }\n  }\n\n  request(request: AdmissionRequest): AdmissionTicket {\n    const entry: Entry = {request, state: 'queued', position: 0};\n    this.#queue.push(entry);\n    this.#processQueue();\n    return {release: () => this.#release(entry)};\n  }\n\n  #release(entry: Entry) {\n    const {state, grant} = entry;\n    entry.state = 'released';\n    if (state === 'queued') {\n      const i = this.#queue.indexOf(entry);\n      if (i >= 0) {\n        this.#queue.splice(i, 1);\n      }\n      this.#processQueue();\n    } else if (state === 'granted' && grant && this.#grants.has(grant)) {\n      // Wait for the subscriber to reconnect with a replacement reservation.\n      grant.released = true;\n      this.#startGrantTTL(grant);\n    }\n  }\n\n  #startGrantTTL(grant: Grant) {\n    grant.timer = this.#setTimeout(() => {\n      if (this.#grants.has(grant)) {\n        this.#lc.info?.(`grant expired without a replacement reservation`);\n        this.#deleteGrant(grant);\n        this.#processQueue();\n      }\n    }, this.#pendingGrantTTLMs);\n  }\n\n  #deleteGrant(grant: Grant) {\n    if (grant.timer) {\n      this.#clearTimeout(grant.timer);\n    }\n    this.#grants.delete(grant);\n  }\n\n  #processQueue() {\n    while (this.#inFlight < this.#maxConcurrentRestores && this.#queue.length) {\n      const entry = this.#queue.shift()!;\n      entry.state = 'granted';\n      entry.grant = {released: false, timer: undefined};\n      this.#grants.add(entry.grant);\n      this.#lc.info?.(`granting permission to restore`, {...this.stats});\n      entry.request.onGranted();\n    }\n    this.#queue.forEach((entry, i) => {\n      const position = i + 1;\n      if (entry.position !== position) {\n        entry.position = position;\n        entry.request.onQueued(position);\n      }\n    });\n  }\n}\n"],"mappings":";;;;;;;;;;;AAaA,IAAM,+BAA+B;;;;;;;AAQrC,IAAa,oCAAoC;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AA4MjD,IAAa,mBAAb,MAA8B;CAC5B;CACA;CACA;CACA;CACA;CACA;CAEA;;CAEA,6BAAsB,IAAI,IAAY;;;;;CAKtC,8BAAuB,IAAI,IAAY;;;;;CAKvC,gCAAyB,IAAI,IAA0B;CACvD,0BAAmB,IAAI,IAAW;CAClC,SAA2B,CAAC;CAE5B,YAAY,IAAgB,MAA+B;EACzD,KAAK,MAAM,GAAG,YAAY,aAAa,mBAAmB;EAC1D,KAAK,kBAAkB,KAAK;EAC5B,KAAK,yBAAyB,KAAK,IAAI,GAAG,KAAK,qBAAqB;EACpE,KAAK,qBACH,KAAK,qBAAqB;EAC5B,KAAK,0BACH,KAAK,0BAAA;EACP,KAAK,cAAc,KAAK,gBAAgB;EACxC,KAAK,gBAAgB,KAAK,kBAAkB;EAE5C,iBACE,eACA,4BACA,yEACF,CAAC,CAAC,aAAY,WAAU,OAAO,QAAQ,KAAK,OAAO,MAAM,CAAC;EAC1D,iBACE,eACA,+BACA,yEACF,CAAC,CAAC,aAAY,WAAU,OAAO,QAAQ,KAAK,SAAS,CAAC;CACxD;CAEA,IAAI,YAAY;EACd,OAAO,KAAK,WAAW,OAAO,KAAK,YAAY,OAAO,KAAK,QAAQ;CACrE;;CAGA,IAAI,QAAQ;EACV,OAAO;GACL,WAAW,KAAK,WAAW,OAAO,KAAK,YAAY;GACnD,SAAS,KAAK,QAAQ;GACtB,QAAQ,KAAK,OAAO;EACtB;CACF;;;;;;;;;CAUA,kBAAkB,KAAa,cAAwC;EACrE,KAAK,cAAc,OAAO,GAAG;EAC7B,IAAI,CAAC,cACH;EAEF,IAAI,aAAa,mBAAmB,KAAK,iBACvC,KAAK,mBAAmB,GAAG;OAE3B,KAAK,cAAc,IAAI,KAAK,YAAY;CAE5C;;;;;CAMA,qBAAqB,KAAa,cAAsB;EACtD,MAAM,eAAe,KAAK,cAAc,IAAI,GAAG;EAC/C,KAAK,cAAc,OAAO,GAAG;EAC7B,IAAI,gBAAgB,aAAa,YAAY,cAC3C,KAAK,mBAAmB,GAAG;CAE/B;;;;;;;;;CAUA,kBAAkB,KAAa;EAC7B,KAAK,cAAc,OAAO,GAAG;EAC7B,IAAI,KAAK,WAAW,OAAO,GAAG,GAAG;GAC/B,MAAM,QAAe;IAAC,UAAU;IAAM,OAAO,KAAA;GAAS;GACtD,KAAK,QAAQ,IAAI,KAAK;GACtB,KAAK,eAAe,KAAK;EAC3B;CACF;;;;;;;CAQA,sBAAsB,KAAsC;EAC1D,KAAK,cAAc,OAAO,GAAG;EAC7B,IAAI,CAAC,KAAK,WAAW,OAAO,GAAG,GAC7B;EAEF,MAAM,cAAc,CAAC;EACrB,KAAK,YAAY,IAAI,WAAW;EAChC,IAAI;EACJ,MAAM,gBAAgB;GACpB,IAAI,OACF,KAAK,cAAc,KAAK;GAE1B,IAAI,KAAK,YAAY,OAAO,WAAW,GACrC,KAAK,cAAc;EAEvB;EACA,OAAO;GACL,gBAAgB;IACd,IAAI,KAAK,YAAY,IAAI,WAAW,KAAK,CAAC,OACxC,QAAQ,KAAK,YAAY,SAAS,KAAK,uBAAuB;GAElE;GACA,QAAQ;EACV;CACF;CAEA,mBAAmB,KAAa;EAC9B,IAAI,KAAK,WAAW,IAAI,GAAG,GACzB;EAEF,KAAK,WAAW,IAAI,GAAG;EAIvB,MAAM,SAAS,CAAC,GAAG,KAAK,OAAO;EAC/B,MAAM,QAAQ,OAAO,MAAK,MAAK,EAAE,QAAQ,KAAK,OAAO;EACrD,IAAI,OACF,KAAK,aAAa,KAAK;CAE3B;CAEA,QAAQ,SAA4C;EAClD,MAAM,QAAe;GAAC;GAAS,OAAO;GAAU,UAAU;EAAC;EAC3D,KAAK,OAAO,KAAK,KAAK;EACtB,KAAK,cAAc;EACnB,OAAO,EAAC,eAAe,KAAK,SAAS,KAAK,EAAC;CAC7C;CAEA,SAAS,OAAc;EACrB,MAAM,EAAC,OAAO,UAAS;EACvB,MAAM,QAAQ;EACd,IAAI,UAAU,UAAU;GACtB,MAAM,IAAI,KAAK,OAAO,QAAQ,KAAK;GACnC,IAAI,KAAK,GACP,KAAK,OAAO,OAAO,GAAG,CAAC;GAEzB,KAAK,cAAc;EACrB,OAAO,IAAI,UAAU,aAAa,SAAS,KAAK,QAAQ,IAAI,KAAK,GAAG;GAElE,MAAM,WAAW;GACjB,KAAK,eAAe,KAAK;EAC3B;CACF;CAEA,eAAe,OAAc;EAC3B,MAAM,QAAQ,KAAK,kBAAkB;GACnC,IAAI,KAAK,QAAQ,IAAI,KAAK,GAAG;IAC3B,KAAK,IAAI,OAAO,iDAAiD;IACjE,KAAK,aAAa,KAAK;IACvB,KAAK,cAAc;GACrB;EACF,GAAG,KAAK,kBAAkB;CAC5B;CAEA,aAAa,OAAc;EACzB,IAAI,MAAM,OACR,KAAK,cAAc,MAAM,KAAK;EAEhC,KAAK,QAAQ,OAAO,KAAK;CAC3B;CAEA,gBAAgB;EACd,OAAO,KAAK,YAAY,KAAK,0BAA0B,KAAK,OAAO,QAAQ;GACzE,MAAM,QAAQ,KAAK,OAAO,MAAM;GAChC,MAAM,QAAQ;GACd,MAAM,QAAQ;IAAC,UAAU;IAAO,OAAO,KAAA;GAAS;GAChD,KAAK,QAAQ,IAAI,MAAM,KAAK;GAC5B,KAAK,IAAI,OAAO,kCAAkC,EAAC,GAAG,KAAK,MAAK,CAAC;GACjE,MAAM,QAAQ,UAAU;EAC1B;EACA,KAAK,OAAO,SAAS,OAAO,MAAM;GAChC,MAAM,WAAW,IAAI;GACrB,IAAI,MAAM,aAAa,UAAU;IAC/B,MAAM,WAAW;IACjB,MAAM,QAAQ,SAAS,QAAQ;GACjC;EACF,CAAC;CACH;AACF"}

package/out/zero-cache/src/services/change-streamer/schema/init.js

@@ -65,6 +65,6 @@ async function getLastWatermarkV2(db, shard) {
 	return replicaVersion;
 }
 //#endregion
-export { initChangeStreamerSchema };
+export { getLastWatermarkV2, initChangeStreamerSchema };
 
 //# sourceMappingURL=init.js.map

package/out/zero-cache/src/services/change-streamer/schema/init.js.map

@@ -1 +1 @@
-{"version":3,"file":"init.js","names":[],"sources":["../../../../../../../zero-cache/src/services/change-streamer/schema/init.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {\n  runSchemaMigrations,\n  type IncrementalMigrationMap,\n  type Migration,\n} from '../../../db/migration.ts';\nimport type {PostgresDB, PostgresTransaction} from '../../../types/pg.ts';\nimport {cdcSchema, type ShardID} from '../../../types/shards.ts';\nimport {\n  createBackfillTables,\n  createReplicationStateTable,\n  setupCDCTables,\n  type ReplicationState,\n} from './tables.ts';\n\nasync function migrateFromLegacySchemas(\n  lc: LogContext,\n  db: PostgresDB,\n  newSchema: string,\n  ...legacy: string[]\n) {\n  const rows = await db<{nspname: string}[]>`\n    SELECT nspname FROM pg_namespace \n      WHERE nspname IN ${db([newSchema, ...legacy])}`.values();\n  // oxlint-disable-next-line unicorn/prefer-set-has -- Small collection, array is more appropriate\n  const names = rows.flat();\n  if (names.includes(newSchema)) {\n    return; // already migrated\n  }\n  for (const schema of legacy) {\n    if (names.includes(schema)) {\n      lc.info?.(`Migrating ${schema} to ${newSchema}`);\n      await db`ALTER SCHEMA ${db(schema)} RENAME TO ${db(newSchema)}`;\n      break;\n    }\n  }\n}\n\nexport async function initChangeStreamerSchema(\n  log: LogContext,\n  db: PostgresDB,\n  shard: ShardID,\n): Promise<void> {\n  const schema = cdcSchema(shard);\n  const {appID} = shard;\n  await migrateFromLegacySchemas(\n    log,\n    db,\n    schema,\n    appID === 'zero' ? `cdc_0` : `cdc_${appID}`,\n    'cdc',\n  );\n\n  const setupMigration: Migration = {\n    migrateSchema: (lc, tx) => setupCDCTables(lc, tx, shard),\n    minSafeVersion: 1,\n  };\n\n  async function migrateV1toV2(_: LogContext, db: PostgresTransaction) {\n    await db`\n    ALTER TABLE ${db(schema)}.\"replicationConfig\" ADD \"resetRequired\" BOOL`;\n  }\n\n  const migrateV2ToV3 = {\n    migrateSchema: async (_: LogContext, db: PostgresTransaction) => {\n      await db.unsafe(createReplicationStateTable(shard));\n    },\n\n    migrateData: async (_: LogContext, db: PostgresTransaction) => {\n      const lastWatermark = await getLastWatermarkV2(db, shard);\n      const replicationState: Partial<ReplicationState> = {lastWatermark};\n      await db`\n      TRUNCATE TABLE ${db(schema)}.\"replicationState\"`;\n      await db`\n      INSERT INTO ${db(schema)}.\"replicationState\" ${db(replicationState)}`;\n    },\n  };\n\n  const migrateV3ToV4 = {\n    migrateSchema: async (_: LogContext, db: PostgresTransaction) => {\n      await db`\n      ALTER TABLE ${db(schema)}.\"changeLog\" ALTER \"change\" TYPE JSON;\n      `;\n    },\n  };\n\n  const migrateV4ToV5 = {\n    migrateSchema: async (_: LogContext, db: PostgresTransaction) => {\n      await db`\n      ALTER TABLE ${db(schema)}.\"replicationState\" ADD \"ownerAddress\" TEXT;\n      `;\n    },\n  };\n\n  const migrateV5ToV6 = {\n    migrateSchema: async (_: LogContext, db: PostgresTransaction) => {\n      await db.unsafe(createBackfillTables(shard));\n    },\n  };\n\n  const schemaVersionMigrationMap: IncrementalMigrationMap = {\n    2: {migrateSchema: migrateV1toV2},\n    3: migrateV2ToV3,\n    4: migrateV3ToV4,\n    5: migrateV4ToV5,\n    6: migrateV5ToV6,\n  };\n\n  await runSchemaMigrations(\n    log,\n    'change-streamer',\n    schema,\n    db,\n    setupMigration,\n    schemaVersionMigrationMap,\n  );\n}\n\nexport async function getLastWatermarkV2(\n  db: PostgresDB,\n  shard: ShardID,\n): Promise<string> {\n  const schema = cdcSchema(shard);\n  const [{max}] = await db<{max: string | null}[]>`\n    SELECT MAX(watermark) as max FROM ${db(schema)}.\"changeLog\"`;\n  if (max !== null) {\n    return max;\n  }\n  // The changeLog is only empty if nothing has been synced since initial-sync.\n  // In this case, the last watermark is the replicaVersion.\n  const [{replicaVersion}] = await db<{replicaVersion: string}[]>`\n    SELECT \"replicaVersion\" FROM ${db(schema)}.\"replicationConfig\"\n    `;\n  return replicaVersion;\n}\n"],"mappings":";;;;AAeA,eAAe,yBACb,IACA,IACA,WACA,GAAG,QACH;CAKA,MAAM,SAAQ,MAJK,EAAuB;;yBAEnB,GAAG,CAAC,WAAW,GAAG,MAAM,CAAC,IAAI,OAAO,GAExC,KAAK;CACxB,IAAI,MAAM,SAAS,SAAS,GAC1B;CAEF,KAAK,MAAM,UAAU,QACnB,IAAI,MAAM,SAAS,MAAM,GAAG;EAC1B,GAAG,OAAO,aAAa,OAAO,MAAM,WAAW;EAC/C,MAAM,EAAE,gBAAgB,GAAG,MAAM,EAAE,aAAa,GAAG,SAAS;EAC5D;CACF;AAEJ;AAEA,eAAsB,yBACpB,KACA,IACA,OACe;CACf,MAAM,SAAS,UAAU,KAAK;CAC9B,MAAM,EAAC,UAAS;CAChB,MAAM,yBACJ,KACA,IACA,QACA,UAAU,SAAS,UAAU,OAAO,SACpC,KACF;CAEA,MAAM,iBAA4B;EAChC,gBAAgB,IAAI,OAAO,eAAe,IAAI,IAAI,KAAK;EACvD,gBAAgB;CAClB;CAEA,eAAe,cAAc,GAAe,IAAyB;EACnE,MAAM,EAAE;kBACM,GAAG,MAAM,EAAE;CAC3B;CA+CA,MAAM,oBACJ,KACA,mBACA,QACA,IACA,gBACA;EAbA,GAAG,EAAC,eAAe,cAAa;EAChC,GAAG;GAtCH,eAAe,OAAO,GAAe,OAA4B;IAC/D,MAAM,GAAG,OAAO,4BAA4B,KAAK,CAAC;GACpD;GAEA,aAAa,OAAO,GAAe,OAA4B;IAE7D,MAAM,mBAA8C,EAAC,eAAA,MADzB,mBAAmB,IAAI,KAAK,EACU;IAClE,MAAM,EAAE;uBACS,GAAG,MAAM,EAAE;IAC5B,MAAM,EAAE;oBACM,GAAG,MAAM,EAAE,sBAAsB,GAAG,gBAAgB;GACpE;EA2BG;EACH,GAAG,EAxBH,eAAe,OAAO,GAAe,OAA4B;GAC/D,MAAM,EAAE;oBACM,GAAG,MAAM,EAAE;;EAE3B,EAoBG;EACH,GAAG,EAjBH,eAAe,OAAO,GAAe,OAA4B;GAC/D,MAAM,EAAE;oBACM,GAAG,MAAM,EAAE;;EAE3B,EAaG;EACH,GAAG,EAVH,eAAe,OAAO,GAAe,OAA4B;GAC/D,MAAM,GAAG,OAAO,qBAAqB,KAAK,CAAC;EAC7C,EAQG;CASH,CACF;AACF;AAEA,eAAsB,mBACpB,IACA,OACiB;CACjB,MAAM,SAAS,UAAU,KAAK;CAC9B,MAAM,CAAC,EAAC,SAAQ,MAAM,EAA0B;wCACV,GAAG,MAAM,EAAE;CACjD,IAAI,QAAQ,MACV,OAAO;CAIT,MAAM,CAAC,EAAC,oBAAmB,MAAM,EAA8B;mCAC9B,GAAG,MAAM,EAAE;;CAE5C,OAAO;AACT"}
+{"version":3,"file":"init.js","names":[],"sources":["../../../../../../../zero-cache/src/services/change-streamer/schema/init.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {\n  runSchemaMigrations,\n  type IncrementalMigrationMap,\n  type Migration,\n} from '../../../db/migration.ts';\nimport type {PostgresDB, PostgresTransaction} from '../../../types/pg.ts';\nimport {cdcSchema, type ShardID} from '../../../types/shards.ts';\nimport {\n  createBackfillTables,\n  createReplicationStateTable,\n  setupCDCTables,\n  type ReplicationState,\n} from './tables.ts';\n\nasync function migrateFromLegacySchemas(\n  lc: LogContext,\n  db: PostgresDB,\n  newSchema: string,\n  ...legacy: string[]\n) {\n  const rows = await db<{nspname: string}[]>`\n    SELECT nspname FROM pg_namespace \n      WHERE nspname IN ${db([newSchema, ...legacy])}`.values();\n  // oxlint-disable-next-line unicorn/prefer-set-has -- Small collection, array is more appropriate\n  const names = rows.flat();\n  if (names.includes(newSchema)) {\n    return; // already migrated\n  }\n  for (const schema of legacy) {\n    if (names.includes(schema)) {\n      lc.info?.(`Migrating ${schema} to ${newSchema}`);\n      await db`ALTER SCHEMA ${db(schema)} RENAME TO ${db(newSchema)}`;\n      break;\n    }\n  }\n}\n\nexport async function initChangeStreamerSchema(\n  log: LogContext,\n  db: PostgresDB,\n  shard: ShardID,\n): Promise<void> {\n  const schema = cdcSchema(shard);\n  const {appID} = shard;\n  await migrateFromLegacySchemas(\n    log,\n    db,\n    schema,\n    appID === 'zero' ? `cdc_0` : `cdc_${appID}`,\n    'cdc',\n  );\n\n  const setupMigration: Migration = {\n    migrateSchema: (lc, tx) => setupCDCTables(lc, tx, shard),\n    minSafeVersion: 1,\n  };\n\n  async function migrateV1toV2(_: LogContext, db: PostgresTransaction) {\n    await db`\n    ALTER TABLE ${db(schema)}.\"replicationConfig\" ADD \"resetRequired\" BOOL`;\n  }\n\n  const migrateV2ToV3 = {\n    migrateSchema: async (_: LogContext, db: PostgresTransaction) => {\n      await db.unsafe(createReplicationStateTable(shard));\n    },\n\n    migrateData: async (_: LogContext, db: PostgresTransaction) => {\n      const lastWatermark = await getLastWatermarkV2(db, shard);\n      const replicationState: Partial<ReplicationState> = {lastWatermark};\n      await db`\n      TRUNCATE TABLE ${db(schema)}.\"replicationState\"`;\n      await db`\n      INSERT INTO ${db(schema)}.\"replicationState\" ${db(replicationState)}`;\n    },\n  };\n\n  const migrateV3ToV4 = {\n    migrateSchema: async (_: LogContext, db: PostgresTransaction) => {\n      await db`\n      ALTER TABLE ${db(schema)}.\"changeLog\" ALTER \"change\" TYPE JSON;\n      `;\n    },\n  };\n\n  const migrateV4ToV5 = {\n    migrateSchema: async (_: LogContext, db: PostgresTransaction) => {\n      await db`\n      ALTER TABLE ${db(schema)}.\"replicationState\" ADD \"ownerAddress\" TEXT;\n      `;\n    },\n  };\n\n  const migrateV5ToV6 = {\n    migrateSchema: async (_: LogContext, db: PostgresTransaction) => {\n      await db.unsafe(createBackfillTables(shard));\n    },\n  };\n\n  const schemaVersionMigrationMap: IncrementalMigrationMap = {\n    2: {migrateSchema: migrateV1toV2},\n    3: migrateV2ToV3,\n    4: migrateV3ToV4,\n    5: migrateV4ToV5,\n    6: migrateV5ToV6,\n  };\n\n  await runSchemaMigrations(\n    log,\n    'change-streamer',\n    schema,\n    db,\n    setupMigration,\n    schemaVersionMigrationMap,\n  );\n}\n\nexport async function getLastWatermarkV2(\n  db: PostgresDB,\n  shard: ShardID,\n): Promise<string> {\n  const schema = cdcSchema(shard);\n  const [{max}] = await db<{max: string | null}[]>`\n    SELECT MAX(watermark) as max FROM ${db(schema)}.\"changeLog\"`;\n  if (max !== null) {\n    return max;\n  }\n  // The changeLog is only empty if nothing has been synced since initial-sync.\n  // In this case, the last watermark is the replicaVersion.\n  const [{replicaVersion}] = await db<{replicaVersion: string}[]>`\n    SELECT \"replicaVersion\" FROM ${db(schema)}.\"replicationConfig\"\n    `;\n  return replicaVersion;\n}\n"],"mappings":";;;;AAeA,eAAe,yBACb,IACA,IACA,WACA,GAAG,QACH;CAKA,MAAM,SAAQ,MAJK,EAAuB;;yBAEnB,GAAG,CAAC,WAAW,GAAG,MAAM,CAAC,IAAI,OAAO,EAAA,CAExC,KAAK;CACxB,IAAI,MAAM,SAAS,SAAS,GAC1B;CAEF,KAAK,MAAM,UAAU,QACnB,IAAI,MAAM,SAAS,MAAM,GAAG;EAC1B,GAAG,OAAO,aAAa,OAAO,MAAM,WAAW;EAC/C,MAAM,EAAE,gBAAgB,GAAG,MAAM,EAAE,aAAa,GAAG,SAAS;EAC5D;CACF;AAEJ;AAEA,eAAsB,yBACpB,KACA,IACA,OACe;CACf,MAAM,SAAS,UAAU,KAAK;CAC9B,MAAM,EAAC,UAAS;CAChB,MAAM,yBACJ,KACA,IACA,QACA,UAAU,SAAS,UAAU,OAAO,SACpC,KACF;CAEA,MAAM,iBAA4B;EAChC,gBAAgB,IAAI,OAAO,eAAe,IAAI,IAAI,KAAK;EACvD,gBAAgB;CAClB;CAEA,eAAe,cAAc,GAAe,IAAyB;EACnE,MAAM,EAAE;kBACM,GAAG,MAAM,EAAE;CAC3B;CA+CA,MAAM,oBACJ,KACA,mBACA,QACA,IACA,gBACA;EAbA,GAAG,EAAC,eAAe,cAAa;EAChC,GAAG;GAtCH,eAAe,OAAO,GAAe,OAA4B;IAC/D,MAAM,GAAG,OAAO,4BAA4B,KAAK,CAAC;GACpD;GAEA,aAAa,OAAO,GAAe,OAA4B;IAE7D,MAAM,mBAA8C,EAAC,eAAA,MADzB,mBAAmB,IAAI,KAAK,EACU;IAClE,MAAM,EAAE;uBACS,GAAG,MAAM,EAAE;IAC5B,MAAM,EAAE;oBACM,GAAG,MAAM,EAAE,sBAAsB,GAAG,gBAAgB;GACpE;EA2BG;EACH,GAAG,EAxBH,eAAe,OAAO,GAAe,OAA4B;GAC/D,MAAM,EAAE;oBACM,GAAG,MAAM,EAAE;;EAE3B,EAoBG;EACH,GAAG,EAjBH,eAAe,OAAO,GAAe,OAA4B;GAC/D,MAAM,EAAE;oBACM,GAAG,MAAM,EAAE;;EAE3B,EAaG;EACH,GAAG,EAVH,eAAe,OAAO,GAAe,OAA4B;GAC/D,MAAM,GAAG,OAAO,qBAAqB,KAAK,CAAC;EAC7C,EAQG;CASH,CACF;AACF;AAEA,eAAsB,mBACpB,IACA,OACiB;CACjB,MAAM,SAAS,UAAU,KAAK;CAC9B,MAAM,CAAC,EAAC,SAAQ,MAAM,EAA0B;wCACV,GAAG,MAAM,EAAE;CACjD,IAAI,QAAQ,MACV,OAAO;CAIT,MAAM,CAAC,EAAC,oBAAmB,MAAM,EAA8B;mCAC9B,GAAG,MAAM,EAAE;;CAE5C,OAAO;AACT"}

package/out/zero-cache/src/services/change-streamer/schema/tables.js

@@ -313,6 +313,6 @@ async function terminateChangeDBLockHolders(lc, db, shard) {
 	else for (const { pid, applicationName, query, terminated: ok } of terminated) lc.info?.(`terminated blocking backend pid=${pid} app=${applicationName} ok=${ok} query=${query.slice(0, 200)}`);
 }
 //#endregion
-export { AutoResetSignal, checkReplicationConfig, createBackfillTables, createReplicationStateTable, discoverChangeStreamerAddress, ensureReplicationConfig, markResetRequired, reinitializeChangeLog, setupCDCTables };
+export { AutoResetSignal, CHANGE_STREAMER_APP_NAME, checkReplicationConfig, createBackfillTables, createReplicationStateTable, discoverChangeStreamerAddress, ensureReplicationConfig, markResetRequired, reinitializeChangeLog, setupCDCTables };
 
 //# sourceMappingURL=tables.js.map

package/out/zero-cache/src/services/change-streamer/schema/tables.js.map

@@ -1 +1 @@
-{"version":3,"file":"tables.js","names":[],"sources":["../../../../../../../zero-cache/src/services/change-streamer/schema/tables.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {ident} from 'pg-format';\nimport type postgres from 'postgres';\nimport {type PendingQuery, type Row} from 'postgres';\nimport {AbortError} from '../../../../../shared/src/abort-error.ts';\nimport {equals} from '../../../../../shared/src/set-utils.ts';\nimport {runTx} from '../../../db/run-transaction.ts';\nimport {type PostgresDB, type PostgresTransaction} from '../../../types/pg.ts';\nimport {cdcSchema, type ShardID} from '../../../types/shards.ts';\nimport type {\n  BackfillID,\n  Change,\n  TableMetadata,\n} from '../../change-source/protocol/current/data.ts';\nimport type {SubscriptionState} from '../../replicator/schema/replication-state.ts';\n\n// For readability in the sql statements.\nfunction schema(shard: ShardID) {\n  return ident(cdcSchema(shard));\n}\n\nexport const PG_SCHEMA = 'cdc';\n\nfunction createSchema(shard: ShardID) {\n  return /*sql*/ `CREATE SCHEMA IF NOT EXISTS ${schema(shard)};`;\n}\n\nexport type ChangeLogEntry = {\n  // A strictly monotonically increasing, lexicographically sortable\n  // value that uniquely identifies a position in the change stream.\n  watermark: string;\n  change: Change;\n};\n\ntype FullChangeLogEntry = ChangeLogEntry & {pos: number};\n\nfunction createChangeLogTable(shard: ShardID) {\n  // Note: The \"change\" column used to be JSONB, but that was problematic in that\n  // it does not handle the NULL unicode character.\n  // https://vladimir.varank.in/notes/2021/01/you-dont-insert-unicode-null-character-as-postgres-jsonb/\n  return /*sql*/ `\n  CREATE TABLE ${schema(shard)}.\"changeLog\" (\n    watermark  TEXT,\n    pos        INT8,\n    change     JSON NOT NULL,\n    precommit  TEXT,  -- Only exists on commit entries. Purely for debugging.\n    PRIMARY KEY (watermark, pos)\n  );\n`;\n}\n\n/**\n * Tracks the watermark from which to resume the change stream and the\n * current owner (task ID) acting as the single writer to the changeLog.\n */\nexport type ReplicationState = {\n  lastWatermark: string;\n  owner: string | null;\n  ownerAddress: string | null;\n};\n\nexport function createReplicationStateTable(shard: ShardID) {\n  return /*sql*/ `\n  CREATE TABLE ${schema(shard)}.\"replicationState\" (\n    \"lastWatermark\" TEXT NOT NULL,\n    \"owner\" TEXT,\n    \"ownerAddress\" TEXT,\n    \"lock\" INTEGER PRIMARY KEY DEFAULT 1 CHECK (lock=1)\n  );\n`;\n}\n\nexport async function discoverChangeStreamerAddress(\n  shard: ShardID,\n  sql: PostgresDB,\n): Promise<string | null> {\n  const result = await sql<{ownerAddress: string | null}[]> /*sql*/ `\n    SELECT \"ownerAddress\" FROM ${sql(cdcSchema(shard))}.\"replicationState\"`;\n  return result[0].ownerAddress;\n}\n\n/**\n * This mirrors the analogously named table in the SQLite replica\n * (`services/replicator/schema/replication-state.ts`), and is used\n * to detect when the replica has been reset and is no longer compatible\n * with the current ChangeLog.\n */\nexport type ReplicationConfig = {\n  replicaVersion: string;\n  publications: readonly string[];\n};\n\nfunction createReplicationConfigTable(shard: ShardID) {\n  return /*sql*/ `\n  CREATE TABLE ${schema(shard)}.\"replicationConfig\" (\n    \"replicaVersion\" TEXT NOT NULL,\n    \"publications\" TEXT[] NOT NULL,\n    \"resetRequired\" BOOL,\n    \"lock\" INTEGER PRIMARY KEY DEFAULT 1 CHECK (lock=1)\n  );\n`;\n}\n\nexport function createBackfillTables(shard: ShardID) {\n  return /*sql*/ `\n  CREATE TABLE ${schema(shard)}.\"tableMetadata\" (\n    \"schema\" TEXT NOT NULL,\n    \"table\" TEXT NOT NULL,\n    \"metadata\" JSONB NOT NULL,\n    PRIMARY KEY(\"schema\", \"table\")\n  );\n\n  CREATE TABLE ${schema(shard)}.\"backfilling\" (\n    \"schema\" TEXT NOT NULL,\n    \"table\" TEXT NOT NULL,\n    \"column\" TEXT NOT NULL,\n    \"backfill\" JSONB NOT NULL,\n    PRIMARY KEY(\"schema\", \"table\", \"column\")\n  );\n  `;\n}\n\nexport type TableMetadataRow = {\n  schema: string;\n  table: string;\n  metadata: TableMetadata;\n};\n\nexport type BackfillingColumn = {\n  schema: string;\n  table: string;\n  column: string;\n  backfill: BackfillID;\n};\n\nfunction createTables(shard: ShardID) {\n  return (\n    createSchema(shard) +\n    createChangeLogTable(shard) +\n    createReplicationStateTable(shard) +\n    createReplicationConfigTable(shard) +\n    createBackfillTables(shard)\n  );\n}\n\ninterface PurgeLock {\n  release(): Promise<void>;\n}\n\nexport async function setupCDCTables(\n  lc: LogContext,\n  db: postgres.TransactionSql,\n  shard: ShardID,\n) {\n  lc.info?.(`Setting up CDC tables`);\n  await db.unsafe(createTables(shard));\n}\n\nexport async function markResetRequired(sql: PostgresDB, shard: ShardID) {\n  const schema = cdcSchema(shard);\n  await sql`\n  UPDATE ${sql(schema)}.\"replicationConfig\"\n    SET \"resetRequired\" = true`;\n}\n\ntype ReplicationConfigRow = ReplicationConfig & {\n  resetRequired: boolean | null;\n};\n\nexport type ReplicaSubscriptionState = Pick<\n  SubscriptionState,\n  'publications' | 'replicaVersion' | 'watermark'\n>;\n\n/**\n * Validates the `replicationConfig` of the change db against the replica,\n * throwing an {@link AutoResetSignal} if the replica cannot be served from it.\n *\n * @param behindSlot Whether the change log is behind the replication slot,\n *     in which case it is re-initialized from the replica (see\n *     {@link reinitializeChangeLog}) and can thus be reset for a replica\n *     at any watermark.\n * @returns `'init'` if the tables are uninitialized, `'reset'` if they are\n *     for a different replica, or `'compatible'`.\n */\nfunction validateReplicationConfig(\n  lc: LogContext,\n  current: ReplicationConfigRow | undefined,\n  {publications, replicaVersion, watermark}: ReplicaSubscriptionState,\n  autoReset: boolean,\n  behindSlot: boolean,\n): 'init' | 'reset' | 'compatible' {\n  if (!current) {\n    return 'init';\n  }\n  if (\n    current.replicaVersion !== replicaVersion ||\n    !equals(new Set(current.publications), new Set(publications))\n  ) {\n    if (replicaVersion !== watermark && !behindSlot) {\n      throw new AutoResetSignal(\n        `Cannot reset change db@${current.replicaVersion} to ` +\n          `service replica@${replicaVersion} from watermark ${watermark}`,\n      );\n    }\n    return 'reset';\n  }\n  if (current.resetRequired) {\n    if (autoReset) {\n      throw new AutoResetSignal('reset required by replication stream');\n    }\n    lc.error?.(\n      '\\n\\n\\n' +\n        'Reset required but --auto-reset is not enabled.\\n' +\n        'This can happen for upstream databases that do not support event triggers.\\n' +\n        'To correct this, see https://zero.rocicorp.dev/docs/connecting-to-postgres#schema-changes' +\n        '\\n\\n\\n',\n    );\n  }\n  return 'compatible';\n}\n\nfunction selectReplicationConfig(sql: PostgresTransaction, shard: ShardID) {\n  return sql<ReplicationConfigRow[]> /*sql*/ `\n    SELECT \"replicaVersion\", \"publications\", \"resetRequired\" \n      FROM ${sql(cdcSchema(shard))}.\"replicationConfig\"`;\n}\n\n/**\n * Checks, without modifying the change db, that the replica can be served\n * by the change log once {@link ensureReplicationConfig} is called upon\n * taking over the change log. Throws an {@link AutoResetSignal} otherwise.\n *\n * This allows a replication-manager to detect an incompatible replica at\n * startup while leaving the change log to its current owner until then.\n */\nexport async function checkReplicationConfig(\n  lc: LogContext,\n  db: PostgresDB,\n  subscriptionState: ReplicaSubscriptionState,\n  shard: ShardID,\n  autoReset: boolean,\n  behindSlot = false,\n): Promise<void> {\n  const [current] = await runTx(db, sql => selectReplicationConfig(sql, shard));\n  const status = validateReplicationConfig(\n    lc,\n    current,\n    subscriptionState,\n    autoReset,\n    behindSlot,\n  );\n  if (status === 'reset') {\n    lc.info?.(\n      `Data in cdc tables @${current?.replicaVersion} is incompatible ` +\n        `with replica @${subscriptionState.replicaVersion}. ` +\n        `Tables will be reset when taking over the change log.`,\n    );\n  }\n}\n\n/**\n * Initializes the cdc tables for the replica, resetting them if they were\n * written for a different replica (version or publications). This disrupts\n * a replication-manager that is serving the change log, and is thus only\n * done when taking over the change log (see `Storer.assumeOwnershipAtHead`).\n *\n * @param purgeLock A purge lock held by the caller, if any, which is released\n *     before the tables are reset.\n * @param behindSlot See {@link checkReplicationConfig}.\n */\nexport async function ensureReplicationConfig(\n  lc: LogContext,\n  db: PostgresDB,\n  subscriptionState: ReplicaSubscriptionState,\n  shard: ShardID,\n  autoReset: boolean,\n  purgeLock?: PurgeLock,\n  setTimeoutFn: typeof setTimeout = setTimeout,\n  behindSlot = false,\n) {\n  const {publications, replicaVersion} = subscriptionState;\n  const replicaConfig = {publications, replicaVersion};\n  const replicationState: ReplicationState = {\n    lastWatermark: replicaVersion,\n    owner: null,\n    ownerAddress: null,\n  };\n  const schema = cdcSchema(shard);\n\n  await runTx(db, async sql => {\n    const [current] = await selectReplicationConfig(sql, shard);\n    const status = validateReplicationConfig(\n      lc,\n      current,\n      subscriptionState,\n      autoReset,\n      behindSlot,\n    );\n    if (status === 'compatible') {\n      return [];\n    }\n    const stmts: PendingQuery<Row[]>[] = [];\n    if (status === 'reset') {\n      lc.info?.(\n        `Data in cdc tables @${current?.replicaVersion} is incompatible ` +\n          `with replica @${replicaVersion}. Clearing tables.`,\n      );\n      // Release any purge lock held by the caller; the changeLog is\n      // incompatible with the replica and needs to be truncated.\n      void purgeLock?.release();\n\n      // Note: The order of TRUNCATE matters. The replicationState table is\n      //       truncated before the changeLog table, in order to acquire\n      //       (exclusive) locks on the tables in the same order that the\n      //       storer.ts acquires locks when writing the changeLog, namely:\n      //\n      // 1. SELECT ... FROM replicationState FOR UPDATE;\n      // 2. INSERT INTO changeLog ...;\n      //    ...\n      // n. UPDATE replicationState ...;\n      stmts.push(\n        sql`TRUNCATE TABLE ${sql(schema)}.\"replicationState\"`,\n        sql`TRUNCATE TABLE ${sql(schema)}.\"changeLog\"`,\n        sql`TRUNCATE TABLE ${sql(schema)}.\"replicationConfig\"`,\n        sql`TRUNCATE TABLE ${sql(schema)}.\"tableMetadata\"`,\n        sql`TRUNCATE TABLE ${sql(schema)}.\"backfilling\"`,\n      );\n    }\n    // Initialize (or re-initialize TRUNCATED) tables.\n    //\n    // The PG storer uses the earliest changeLog entry as the safe watermark\n    // from which subscribers can be resumed. These initial entries ensure\n    // that subscribers can start from a freshly synced replica, even if new\n    // changes have been replicated and not purged.\n    //\n    // TODO: Replace this with an explicit `firstWatermark` column in the\n    //       change db.\n    const initialTx: FullChangeLogEntry[] = [\n      {watermark: replicaVersion, pos: 0, change: {tag: 'begin'}},\n      {watermark: replicaVersion, pos: 1, change: {tag: 'commit'}},\n    ];\n    stmts.push(\n      sql`INSERT INTO ${sql(schema)}.\"replicationConfig\" ${sql(replicaConfig)}`,\n      sql`INSERT INTO ${sql(schema)}.\"replicationState\"  ${sql(replicationState)} \n            ON CONFLICT (lock) DO UPDATE SET ${sql(replicationState)}`,\n      ...initialTx.map(\n        change => sql`INSERT INTO ${sql(schema)}.\"changeLog\" ${sql(change)}`,\n      ),\n    );\n\n    if (status === 'reset') {\n      return runStatementsAndTerminateLockHolders(\n        lc,\n        db,\n        shard,\n        'ensureReplicationConfig',\n        stmts,\n        setTimeoutFn,\n      );\n    }\n    return Promise.all(stmts);\n  });\n}\n\n// The time to wait for a TRUNCATE in ensureReplicationConfig before\n// terminating blocking backends via terminateChangeDBLockHolders.\nconst LOCK_HOLDER_TERMINATE_TIMEOUT_MS = 5_000;\n\n/**\n * Executes `stmts` that require ACCESS EXCLUSIVE locks (e.g. `TRUNCATE`),\n * which may be blocked by old storer catchup reads. Races against a\n * timeout that terminates the blocking backends if the `stmts` take too\n * long. The check is repeated until the statements complete, since the\n * `stmts` may not yet be waiting on a lock when the timer fires.\n */\nasync function runStatementsAndTerminateLockHolders(\n  lc: LogContext,\n  db: PostgresDB,\n  shard: ShardID,\n  name: string,\n  stmts: PendingQuery<Row[]>[],\n  setTimeoutFn: typeof setTimeout,\n) {\n  let done = false;\n  let timer: ReturnType<typeof setTimeout> | undefined;\n  const scheduleTerminate = () => {\n    timer = setTimeoutFn(async () => {\n      lc.info?.(`${name} blocked, terminating lock holders`);\n      try {\n        await terminateChangeDBLockHolders(lc, db, shard);\n      } catch (e) {\n        lc.warn?.('error terminating lock holders', e);\n      }\n      if (!done) {\n        scheduleTerminate();\n      }\n    }, LOCK_HOLDER_TERMINATE_TIMEOUT_MS);\n  };\n  scheduleTerminate();\n  try {\n    return await Promise.all(stmts);\n  } finally {\n    done = true;\n    clearTimeout(timer);\n  }\n}\n\nexport type ChangeLogReinitialization = {\n  /**\n   * The watermark of the replica from which the change log is re-initialized,\n   * i.e. the watermark from which the change stream will resume.\n   */\n  watermark: string;\n  cookies: {\n    tableMetadata: readonly {\n      schema: string;\n      table: string;\n      metadata: TableMetadata;\n    }[];\n    backfilling: readonly {\n      schema: string;\n      table: string;\n      column: string;\n      backfill: BackfillID;\n    }[];\n  };\n  owner: string;\n  ownerAddress: string;\n};\n\n/**\n * Re-initializes the change log from the state of the replica if it does not\n * contain all of the replica's changes, i.e. if its `lastWatermark` is before\n * the replica's watermark. This is the case if the change log is no longer\n * maintained (e.g. after running in high-availability mode), or if its last\n * owner stopped before persisting changes that it had already forwarded\n * (and backed up). Ownership of the change log is taken in the same\n * transaction, in which:\n *\n * * The `replicationState`, `changeLog`, `tableMetadata`, and `backfilling`\n *   tables are truncated. (The `replicationConfig` is kept;\n *   {@link ensureReplicationConfig} handles changes of the replica version\n *   or publications.)\n * * The change log is seeded with an empty transaction at the replica's\n *   watermark, so that subscribers at that watermark can be caught up, while\n *   subscribers at earlier watermarks are rejected as being too old.\n * * The `tableMetadata` and `backfilling` cookies are set from the replica.\n *\n * Returns `'maintained'` without modifying anything if the change log's\n * `lastWatermark` is at or past the replica's watermark, i.e. the change log\n * contains all of the replica's changes. Because a replication slot is only\n * acked past a transaction once it has been backed up, this means that the\n * change log contains every transaction up to the position of the replica's\n * replication slot, and it can be taken over (and resumed from its head).\n *\n * This is called when the change log's head is behind the position of the\n * replication slot (see `PgChangeLogPurgeLocker`). That position cannot be\n * compared with the watermarks of transactions to determine whether the\n * change log is maintained, as it is also advanced past positions of changes\n * outside of the publication.\n *\n * @param purgeLock A purge lock held by the caller, if any, which is released\n *     before the tables are truncated.\n */\nexport function reinitializeChangeLog(\n  lc: LogContext,\n  db: PostgresDB,\n  shard: ShardID,\n  {watermark, cookies, owner, ownerAddress}: ChangeLogReinitialization,\n  purgeLock?: PurgeLock,\n  setTimeoutFn: typeof setTimeout = setTimeout,\n): Promise<'reinitialized' | 'maintained'> {\n  const schema = cdcSchema(shard);\n  return runTx(db, async sql => {\n    // Lock the replicationState first, which is the order in which the\n    // storer acquires locks when writing the changeLog.\n    const state = await sql<{lastWatermark: string}[]>`\n      SELECT \"lastWatermark\" FROM ${sql(schema)}.\"replicationState\" FOR UPDATE`;\n    const lastWatermark = state[0]?.lastWatermark;\n    if (lastWatermark !== undefined && lastWatermark >= watermark) {\n      lc.info?.(\n        `changeLog@${lastWatermark} contains the replica's changes (@${watermark})`,\n      );\n      return 'maintained';\n    }\n    lc.info?.(\n      `re-initializing changeLog@${lastWatermark} from the replica @${watermark}`,\n    );\n    const replicationState: ReplicationState = {\n      lastWatermark: watermark,\n      owner,\n      ownerAddress,\n    };\n    const initialTx: FullChangeLogEntry[] = [\n      {watermark, pos: 0, change: {tag: 'begin'}},\n      {watermark, pos: 1, change: {tag: 'commit'}},\n    ];\n    // The caller's purge lock would block the TRUNCATE. Releasing it is safe:\n    // a concurrent purge checks the ownership after deleting, which blocks\n    // on the replicationState lock held by this transaction.\n    await purgeLock?.release();\n    await runStatementsAndTerminateLockHolders(\n      lc,\n      db,\n      shard,\n      'reinitializeChangeLog',\n      [\n        sql`TRUNCATE TABLE ${sql(schema)}.\"replicationState\"`,\n        sql`TRUNCATE TABLE ${sql(schema)}.\"changeLog\"`,\n        sql`TRUNCATE TABLE ${sql(schema)}.\"tableMetadata\"`,\n        sql`TRUNCATE TABLE ${sql(schema)}.\"backfilling\"`,\n      ],\n      setTimeoutFn,\n    );\n    await sql`INSERT INTO ${sql(schema)}.\"replicationState\" ${sql(replicationState)}`;\n    for (const entry of initialTx) {\n      await sql`INSERT INTO ${sql(schema)}.\"changeLog\" ${sql(entry)}`;\n    }\n    for (const row of cookies.tableMetadata) {\n      await sql`INSERT INTO ${sql(schema)}.\"tableMetadata\" ${sql({...row})}`;\n    }\n    for (const row of cookies.backfilling) {\n      await sql`INSERT INTO ${sql(schema)}.\"backfilling\" ${sql({...row})}`;\n    }\n    return 'reinitialized';\n  });\n}\n\nexport const CHANGE_STREAMER_APP_NAME = 'zero-change-streamer';\n\nexport class AutoResetSignal extends AbortError {\n  readonly name = 'AutoResetSignal';\n}\n\n/**\n * Terminates zero-cache backends that are blocking the current backend\n * from acquiring locks on CDC tables (e.g., during TRUNCATE).\n *\n * This is used during change-DB takeover when the new replication-manager's\n * `ensureReplicationConfig` needs to TRUNCATE tables, but the old\n * replication-manager's storer is still reading from them (e.g., large\n * catchup cursors).\n *\n * The function:\n * 1. Finds backends waiting for a lock on a TRUNCATE in {schema}\n * 2. Uses `pg_blocking_pids()` to identify which backends are blocking them\n * 3. Terminates blocking backends that have `application_name = 'zero-change-streamer'`\n *\n * Must be called on a **separate connection** from the one that is blocked,\n * since the blocked connection is inside a pending transaction.\n */\nexport async function terminateChangeDBLockHolders(\n  lc: LogContext,\n  db: PostgresDB,\n  shard: ShardID,\n) {\n  const schema = cdcSchema(shard);\n\n  // Step 1: Find backends that are blocked waiting for a lock,\n  // whose query involves a TRUNCATE on this shard's CDC schema.\n  const blocked = await db<{pid: number}[]>`\n    SELECT pid FROM pg_stat_activity\n      WHERE wait_event_type = 'Lock'\n        AND application_name = ${CHANGE_STREAMER_APP_NAME}\n        AND query LIKE ${'%TRUNCATE%' + schema + '%'}`;\n\n  if (blocked.length === 0) {\n    lc.info?.('no blocked TRUNCATE backends found');\n    return;\n  }\n\n  const blockedPids = blocked.map(r => r.pid);\n  lc.info?.(`found blocked TRUNCATE backends: ${JSON.stringify(blockedPids)}`);\n\n  // Step 2: For each blocked backend, find and terminate its blockers\n  // that are zero-change-streamer connections.\n  const terminated = await db<\n    {pid: number; applicationName: string; query: string; terminated: boolean}[]\n  >`\n    SELECT pid, application_name as \"applicationName\", query,\n           pg_terminate_backend(pid) as terminated\n      FROM pg_stat_activity\n      WHERE pid = ANY(\n        SELECT unnest(pg_blocking_pids(blocked.pid))\n          FROM unnest(${blockedPids}::int[]) AS blocked(pid)\n      )\n      AND application_name = ${CHANGE_STREAMER_APP_NAME}\n      AND pid != ALL(${blockedPids}::int[])`;\n\n  if (terminated.length === 0) {\n    lc.info?.(`no ${CHANGE_STREAMER_APP_NAME} blockers found to terminate`);\n  } else {\n    for (const {pid, applicationName, query, terminated: ok} of terminated) {\n      lc.info?.(\n        `terminated blocking backend pid=${pid} app=${applicationName} ok=${ok} query=${query.slice(0, 200)}`,\n      );\n    }\n  }\n}\n"],"mappings":";;;;;;;;AAiBA,SAAS,OAAO,OAAgB;CAC9B,OAAO,MAAM,UAAU,KAAK,CAAC;AAC/B;AAIA,SAAS,aAAa,OAAgB;CACpC,OAAe,+BAA+B,OAAO,KAAK,EAAE;AAC9D;AAWA,SAAS,qBAAqB,OAAgB;CAI5C,OAAe;iBACA,OAAO,KAAK,EAAE;;;;;;;;AAQ/B;AAYA,SAAgB,4BAA4B,OAAgB;CAC1D,OAAe;iBACA,OAAO,KAAK,EAAE;;;;;;;AAO/B;AAEA,eAAsB,8BACpB,OACA,KACwB;CAGxB,QAAO,MAFc,GAA6C;iCACnC,IAAI,UAAU,KAAK,CAAC,EAAE,sBACvC,GAAG;AACnB;AAaA,SAAS,6BAA6B,OAAgB;CACpD,OAAe;iBACA,OAAO,KAAK,EAAE;;;;;;;AAO/B;AAEA,SAAgB,qBAAqB,OAAgB;CACnD,OAAe;iBACA,OAAO,KAAK,EAAE;;;;;;;iBAOd,OAAO,KAAK,EAAE;;;;;;;;AAQ/B;AAeA,SAAS,aAAa,OAAgB;CACpC,OACE,aAAa,KAAK,IAClB,qBAAqB,KAAK,IAC1B,4BAA4B,KAAK,IACjC,6BAA6B,KAAK,IAClC,qBAAqB,KAAK;AAE9B;AAMA,eAAsB,eACpB,IACA,IACA,OACA;CACA,GAAG,OAAO,uBAAuB;CACjC,MAAM,GAAG,OAAO,aAAa,KAAK,CAAC;AACrC;AAEA,eAAsB,kBAAkB,KAAiB,OAAgB;CAEvE,MAAM,GAAG;WACA,IAFM,UAAU,KAEZ,CAAM,EAAE;;AAEvB;;;;;;;;;;;;AAsBA,SAAS,0BACP,IACA,SACA,EAAC,cAAc,gBAAgB,aAC/B,WACA,YACiC;CACjC,IAAI,CAAC,SACH,OAAO;CAET,IACE,QAAQ,mBAAmB,kBAC3B,CAAC,OAAO,IAAI,IAAI,QAAQ,YAAY,GAAG,IAAI,IAAI,YAAY,CAAC,GAC5D;EACA,IAAI,mBAAmB,aAAa,CAAC,YACnC,MAAM,IAAI,gBACR,0BAA0B,QAAQ,eAAe,sBAC5B,eAAe,kBAAkB,WACxD;EAEF,OAAO;CACT;CACA,IAAI,QAAQ,eAAe;EACzB,IAAI,WACF,MAAM,IAAI,gBAAgB,sCAAsC;EAElE,GAAG,QACD,oOAKF;CACF;CACA,OAAO;AACT;AAEA,SAAS,wBAAwB,KAA0B,OAAgB;CACzE,OAAO,GAAoC;;aAEhC,IAAI,UAAU,KAAK,CAAC,EAAE;AACnC;;;;;;;;;AAUA,eAAsB,uBACpB,IACA,IACA,mBACA,OACA,WACA,aAAa,OACE;CACf,MAAM,CAAC,WAAW,MAAM,MAAM,KAAI,QAAO,wBAAwB,KAAK,KAAK,CAAC;CAQ5E,IAPe,0BACb,IACA,SACA,mBACA,WACA,UAEE,MAAW,SACb,GAAG,OACD,uBAAuB,SAAS,eAAe,iCAC5B,kBAAkB,eAAe,wDAEtD;AAEJ;;;;;;;;;;;AAYA,eAAsB,wBACpB,IACA,IACA,mBACA,OACA,WACA,WACA,eAAkC,YAClC,aAAa,OACb;CACA,MAAM,EAAC,cAAc,mBAAkB;CACvC,MAAM,gBAAgB;EAAC;EAAc;CAAc;CACnD,MAAM,mBAAqC;EACzC,eAAe;EACf,OAAO;EACP,cAAc;CAChB;CACA,MAAM,SAAS,UAAU,KAAK;CAE9B,MAAM,MAAM,IAAI,OAAM,QAAO;EAC3B,MAAM,CAAC,WAAW,MAAM,wBAAwB,KAAK,KAAK;EAC1D,MAAM,SAAS,0BACb,IACA,SACA,mBACA,WACA,UACF;EACA,IAAI,WAAW,cACb,OAAO,CAAC;EAEV,MAAM,QAA+B,CAAC;EACtC,IAAI,WAAW,SAAS;GACtB,GAAG,OACD,uBAAuB,SAAS,eAAe,iCAC5B,eAAe,mBACpC;GAGA,WAAgB,QAAQ;GAWxB,MAAM,KACJ,GAAG,kBAAkB,IAAI,MAAM,EAAE,sBACjC,GAAG,kBAAkB,IAAI,MAAM,EAAE,eACjC,GAAG,kBAAkB,IAAI,MAAM,EAAE,uBACjC,GAAG,kBAAkB,IAAI,MAAM,EAAE,mBACjC,GAAG,kBAAkB,IAAI,MAAM,EAAE,eACnC;EACF;EAUA,MAAM,YAAkC,CACtC;GAAC,WAAW;GAAgB,KAAK;GAAG,QAAQ,EAAC,KAAK,QAAO;EAAC,GAC1D;GAAC,WAAW;GAAgB,KAAK;GAAG,QAAQ,EAAC,KAAK,SAAQ;EAAC,CAC7D;EACA,MAAM,KACJ,GAAG,eAAe,IAAI,MAAM,EAAE,uBAAuB,IAAI,aAAa,KACtE,GAAG,eAAe,IAAI,MAAM,EAAE,uBAAuB,IAAI,gBAAgB,EAAE;+CAClC,IAAI,gBAAgB,KAC7D,GAAG,UAAU,KACX,WAAU,GAAG,eAAe,IAAI,MAAM,EAAE,eAAe,IAAI,MAAM,GACnE,CACF;EAEA,IAAI,WAAW,SACb,OAAO,qCACL,IACA,IACA,OACA,2BACA,OACA,YACF;EAEF,OAAO,QAAQ,IAAI,KAAK;CAC1B,CAAC;AACH;AAIA,IAAM,mCAAmC;;;;;;;;AASzC,eAAe,qCACb,IACA,IACA,OACA,MACA,OACA,cACA;CACA,IAAI,OAAO;CACX,IAAI;CACJ,MAAM,0BAA0B;EAC9B,QAAQ,aAAa,YAAY;GAC/B,GAAG,OAAO,GAAG,KAAK,mCAAmC;GACrD,IAAI;IACF,MAAM,6BAA6B,IAAI,IAAI,KAAK;GAClD,SAAS,GAAG;IACV,GAAG,OAAO,kCAAkC,CAAC;GAC/C;GACA,IAAI,CAAC,MACH,kBAAkB;EAEtB,GAAG,gCAAgC;CACrC;CACA,kBAAkB;CAClB,IAAI;EACF,OAAO,MAAM,QAAQ,IAAI,KAAK;CAChC,UAAU;EACR,OAAO;EACP,aAAa,KAAK;CACpB;AACF;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AA2DA,SAAgB,sBACd,IACA,IACA,OACA,EAAC,WAAW,SAAS,OAAO,gBAC5B,WACA,eAAkC,YACO;CACzC,MAAM,SAAS,UAAU,KAAK;CAC9B,OAAO,MAAM,IAAI,OAAM,QAAO;EAK5B,MAAM,iBAAgB,MAFF,GAA8B;oCAClB,IAAI,MAAM,EAAE,iCAChB,IAAI;EAChC,IAAI,kBAAkB,KAAA,KAAa,iBAAiB,WAAW;GAC7D,GAAG,OACD,aAAa,cAAc,oCAAoC,UAAU,EAC3E;GACA,OAAO;EACT;EACA,GAAG,OACD,6BAA6B,cAAc,qBAAqB,WAClE;EACA,MAAM,mBAAqC;GACzC,eAAe;GACf;GACA;EACF;EACA,MAAM,YAAkC,CACtC;GAAC;GAAW,KAAK;GAAG,QAAQ,EAAC,KAAK,QAAO;EAAC,GAC1C;GAAC;GAAW,KAAK;GAAG,QAAQ,EAAC,KAAK,SAAQ;EAAC,CAC7C;EAIA,MAAM,WAAW,QAAQ;EACzB,MAAM,qCACJ,IACA,IACA,OACA,yBACA;GACE,GAAG,kBAAkB,IAAI,MAAM,EAAE;GACjC,GAAG,kBAAkB,IAAI,MAAM,EAAE;GACjC,GAAG,kBAAkB,IAAI,MAAM,EAAE;GACjC,GAAG,kBAAkB,IAAI,MAAM,EAAE;EACnC,GACA,YACF;EACA,MAAM,GAAG,eAAe,IAAI,MAAM,EAAE,sBAAsB,IAAI,gBAAgB;EAC9E,KAAK,MAAM,SAAS,WAClB,MAAM,GAAG,eAAe,IAAI,MAAM,EAAE,eAAe,IAAI,KAAK;EAE9D,KAAK,MAAM,OAAO,QAAQ,eACxB,MAAM,GAAG,eAAe,IAAI,MAAM,EAAE,mBAAmB,IAAI,EAAC,GAAG,IAAG,CAAC;EAErE,KAAK,MAAM,OAAO,QAAQ,aACxB,MAAM,GAAG,eAAe,IAAI,MAAM,EAAE,iBAAiB,IAAI,EAAC,GAAG,IAAG,CAAC;EAEnE,OAAO;CACT,CAAC;AACH;AAEA,IAAa,2BAA2B;AAExC,IAAa,kBAAb,cAAqC,WAAW;CAC9C,OAAgB;AAClB;;;;;;;;;;;;;;;;;;AAmBA,eAAsB,6BACpB,IACA,IACA,OACA;CAKA,MAAM,UAAU,MAAM,EAAmB;;;iCAGV,yBAAyB;yBACjC,eARR,UAAU,KAQa,IAAS;CAE/C,IAAI,QAAQ,WAAW,GAAG;EACxB,GAAG,OAAO,oCAAoC;EAC9C;CACF;CAEA,MAAM,cAAc,QAAQ,KAAI,MAAK,EAAE,GAAG;CAC1C,GAAG,OAAO,oCAAoC,KAAK,UAAU,WAAW,GAAG;CAI3E,MAAM,aAAa,MAAM,EAExB;;;;;;wBAMqB,YAAY;;+BAEL,yBAAyB;uBACjC,YAAY;CAEjC,IAAI,WAAW,WAAW,GACxB,GAAG,OAAO,MAAM,yBAAyB,6BAA6B;MAEtE,KAAK,MAAM,EAAC,KAAK,iBAAiB,OAAO,YAAY,QAAO,YAC1D,GAAG,OACD,mCAAmC,IAAI,OAAO,gBAAgB,MAAM,GAAG,SAAS,MAAM,MAAM,GAAG,GAAG,GACpG;AAGN"}
+{"version":3,"file":"tables.js","names":[],"sources":["../../../../../../../zero-cache/src/services/change-streamer/schema/tables.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {ident} from 'pg-format';\nimport type postgres from 'postgres';\nimport {type PendingQuery, type Row} from 'postgres';\nimport {AbortError} from '../../../../../shared/src/abort-error.ts';\nimport {equals} from '../../../../../shared/src/set-utils.ts';\nimport {runTx} from '../../../db/run-transaction.ts';\nimport {type PostgresDB, type PostgresTransaction} from '../../../types/pg.ts';\nimport {cdcSchema, type ShardID} from '../../../types/shards.ts';\nimport type {\n  BackfillID,\n  Change,\n  TableMetadata,\n} from '../../change-source/protocol/current/data.ts';\nimport type {SubscriptionState} from '../../replicator/schema/replication-state.ts';\n\n// For readability in the sql statements.\nfunction schema(shard: ShardID) {\n  return ident(cdcSchema(shard));\n}\n\nfunction createSchema(shard: ShardID) {\n  return /*sql*/ `CREATE SCHEMA IF NOT EXISTS ${schema(shard)};`;\n}\n\ntype ChangeLogEntry = {\n  // A strictly monotonically increasing, lexicographically sortable\n  // value that uniquely identifies a position in the change stream.\n  watermark: string;\n  change: Change;\n};\n\ntype FullChangeLogEntry = ChangeLogEntry & {pos: number};\n\nfunction createChangeLogTable(shard: ShardID) {\n  // Note: The \"change\" column used to be JSONB, but that was problematic in that\n  // it does not handle the NULL unicode character.\n  // https://vladimir.varank.in/notes/2021/01/you-dont-insert-unicode-null-character-as-postgres-jsonb/\n  return /*sql*/ `\n  CREATE TABLE ${schema(shard)}.\"changeLog\" (\n    watermark  TEXT,\n    pos        INT8,\n    change     JSON NOT NULL,\n    precommit  TEXT,  -- Only exists on commit entries. Purely for debugging.\n    PRIMARY KEY (watermark, pos)\n  );\n`;\n}\n\n/**\n * Tracks the watermark from which to resume the change stream and the\n * current owner (task ID) acting as the single writer to the changeLog.\n */\nexport type ReplicationState = {\n  lastWatermark: string;\n  owner: string | null;\n  ownerAddress: string | null;\n};\n\nexport function createReplicationStateTable(shard: ShardID) {\n  return /*sql*/ `\n  CREATE TABLE ${schema(shard)}.\"replicationState\" (\n    \"lastWatermark\" TEXT NOT NULL,\n    \"owner\" TEXT,\n    \"ownerAddress\" TEXT,\n    \"lock\" INTEGER PRIMARY KEY DEFAULT 1 CHECK (lock=1)\n  );\n`;\n}\n\nexport async function discoverChangeStreamerAddress(\n  shard: ShardID,\n  sql: PostgresDB,\n): Promise<string | null> {\n  const result = await sql<{ownerAddress: string | null}[]> /*sql*/ `\n    SELECT \"ownerAddress\" FROM ${sql(cdcSchema(shard))}.\"replicationState\"`;\n  return result[0].ownerAddress;\n}\n\n/**\n * This mirrors the analogously named table in the SQLite replica\n * (`services/replicator/schema/replication-state.ts`), and is used\n * to detect when the replica has been reset and is no longer compatible\n * with the current ChangeLog.\n */\nexport type ReplicationConfig = {\n  replicaVersion: string;\n  publications: readonly string[];\n};\n\nfunction createReplicationConfigTable(shard: ShardID) {\n  return /*sql*/ `\n  CREATE TABLE ${schema(shard)}.\"replicationConfig\" (\n    \"replicaVersion\" TEXT NOT NULL,\n    \"publications\" TEXT[] NOT NULL,\n    \"resetRequired\" BOOL,\n    \"lock\" INTEGER PRIMARY KEY DEFAULT 1 CHECK (lock=1)\n  );\n`;\n}\n\nexport function createBackfillTables(shard: ShardID) {\n  return /*sql*/ `\n  CREATE TABLE ${schema(shard)}.\"tableMetadata\" (\n    \"schema\" TEXT NOT NULL,\n    \"table\" TEXT NOT NULL,\n    \"metadata\" JSONB NOT NULL,\n    PRIMARY KEY(\"schema\", \"table\")\n  );\n\n  CREATE TABLE ${schema(shard)}.\"backfilling\" (\n    \"schema\" TEXT NOT NULL,\n    \"table\" TEXT NOT NULL,\n    \"column\" TEXT NOT NULL,\n    \"backfill\" JSONB NOT NULL,\n    PRIMARY KEY(\"schema\", \"table\", \"column\")\n  );\n  `;\n}\n\nexport type TableMetadataRow = {\n  schema: string;\n  table: string;\n  metadata: TableMetadata;\n};\n\nexport type BackfillingColumn = {\n  schema: string;\n  table: string;\n  column: string;\n  backfill: BackfillID;\n};\n\nfunction createTables(shard: ShardID) {\n  return (\n    createSchema(shard) +\n    createChangeLogTable(shard) +\n    createReplicationStateTable(shard) +\n    createReplicationConfigTable(shard) +\n    createBackfillTables(shard)\n  );\n}\n\ninterface PurgeLock {\n  release(): Promise<void>;\n}\n\nexport async function setupCDCTables(\n  lc: LogContext,\n  db: postgres.TransactionSql,\n  shard: ShardID,\n) {\n  lc.info?.(`Setting up CDC tables`);\n  await db.unsafe(createTables(shard));\n}\n\nexport async function markResetRequired(sql: PostgresDB, shard: ShardID) {\n  const schema = cdcSchema(shard);\n  await sql`\n  UPDATE ${sql(schema)}.\"replicationConfig\"\n    SET \"resetRequired\" = true`;\n}\n\ntype ReplicationConfigRow = ReplicationConfig & {\n  resetRequired: boolean | null;\n};\n\nexport type ReplicaSubscriptionState = Pick<\n  SubscriptionState,\n  'publications' | 'replicaVersion' | 'watermark'\n>;\n\n/**\n * Validates the `replicationConfig` of the change db against the replica,\n * throwing an {@link AutoResetSignal} if the replica cannot be served from it.\n *\n * @param behindSlot Whether the change log is behind the replication slot,\n *     in which case it is re-initialized from the replica (see\n *     {@link reinitializeChangeLog}) and can thus be reset for a replica\n *     at any watermark.\n * @returns `'init'` if the tables are uninitialized, `'reset'` if they are\n *     for a different replica, or `'compatible'`.\n */\nfunction validateReplicationConfig(\n  lc: LogContext,\n  current: ReplicationConfigRow | undefined,\n  {publications, replicaVersion, watermark}: ReplicaSubscriptionState,\n  autoReset: boolean,\n  behindSlot: boolean,\n): 'init' | 'reset' | 'compatible' {\n  if (!current) {\n    return 'init';\n  }\n  if (\n    current.replicaVersion !== replicaVersion ||\n    !equals(new Set(current.publications), new Set(publications))\n  ) {\n    if (replicaVersion !== watermark && !behindSlot) {\n      throw new AutoResetSignal(\n        `Cannot reset change db@${current.replicaVersion} to ` +\n          `service replica@${replicaVersion} from watermark ${watermark}`,\n      );\n    }\n    return 'reset';\n  }\n  if (current.resetRequired) {\n    if (autoReset) {\n      throw new AutoResetSignal('reset required by replication stream');\n    }\n    lc.error?.(\n      '\\n\\n\\n' +\n        'Reset required but --auto-reset is not enabled.\\n' +\n        'This can happen for upstream databases that do not support event triggers.\\n' +\n        'To correct this, see https://zero.rocicorp.dev/docs/connecting-to-postgres#schema-changes' +\n        '\\n\\n\\n',\n    );\n  }\n  return 'compatible';\n}\n\nfunction selectReplicationConfig(sql: PostgresTransaction, shard: ShardID) {\n  return sql<ReplicationConfigRow[]> /*sql*/ `\n    SELECT \"replicaVersion\", \"publications\", \"resetRequired\" \n      FROM ${sql(cdcSchema(shard))}.\"replicationConfig\"`;\n}\n\n/**\n * Checks, without modifying the change db, that the replica can be served\n * by the change log once {@link ensureReplicationConfig} is called upon\n * taking over the change log. Throws an {@link AutoResetSignal} otherwise.\n *\n * This allows a replication-manager to detect an incompatible replica at\n * startup while leaving the change log to its current owner until then.\n */\nexport async function checkReplicationConfig(\n  lc: LogContext,\n  db: PostgresDB,\n  subscriptionState: ReplicaSubscriptionState,\n  shard: ShardID,\n  autoReset: boolean,\n  behindSlot = false,\n): Promise<void> {\n  const [current] = await runTx(db, sql => selectReplicationConfig(sql, shard));\n  const status = validateReplicationConfig(\n    lc,\n    current,\n    subscriptionState,\n    autoReset,\n    behindSlot,\n  );\n  if (status === 'reset') {\n    lc.info?.(\n      `Data in cdc tables @${current?.replicaVersion} is incompatible ` +\n        `with replica @${subscriptionState.replicaVersion}. ` +\n        `Tables will be reset when taking over the change log.`,\n    );\n  }\n}\n\n/**\n * Initializes the cdc tables for the replica, resetting them if they were\n * written for a different replica (version or publications). This disrupts\n * a replication-manager that is serving the change log, and is thus only\n * done when taking over the change log (see `Storer.assumeOwnershipAtHead`).\n *\n * @param purgeLock A purge lock held by the caller, if any, which is released\n *     before the tables are reset.\n * @param behindSlot See {@link checkReplicationConfig}.\n */\nexport async function ensureReplicationConfig(\n  lc: LogContext,\n  db: PostgresDB,\n  subscriptionState: ReplicaSubscriptionState,\n  shard: ShardID,\n  autoReset: boolean,\n  purgeLock?: PurgeLock,\n  setTimeoutFn: typeof setTimeout = setTimeout,\n  behindSlot = false,\n) {\n  const {publications, replicaVersion} = subscriptionState;\n  const replicaConfig = {publications, replicaVersion};\n  const replicationState: ReplicationState = {\n    lastWatermark: replicaVersion,\n    owner: null,\n    ownerAddress: null,\n  };\n  const schema = cdcSchema(shard);\n\n  await runTx(db, async sql => {\n    const [current] = await selectReplicationConfig(sql, shard);\n    const status = validateReplicationConfig(\n      lc,\n      current,\n      subscriptionState,\n      autoReset,\n      behindSlot,\n    );\n    if (status === 'compatible') {\n      return [];\n    }\n    const stmts: PendingQuery<Row[]>[] = [];\n    if (status === 'reset') {\n      lc.info?.(\n        `Data in cdc tables @${current?.replicaVersion} is incompatible ` +\n          `with replica @${replicaVersion}. Clearing tables.`,\n      );\n      // Release any purge lock held by the caller; the changeLog is\n      // incompatible with the replica and needs to be truncated.\n      void purgeLock?.release();\n\n      // Note: The order of TRUNCATE matters. The replicationState table is\n      //       truncated before the changeLog table, in order to acquire\n      //       (exclusive) locks on the tables in the same order that the\n      //       storer.ts acquires locks when writing the changeLog, namely:\n      //\n      // 1. SELECT ... FROM replicationState FOR UPDATE;\n      // 2. INSERT INTO changeLog ...;\n      //    ...\n      // n. UPDATE replicationState ...;\n      stmts.push(\n        sql`TRUNCATE TABLE ${sql(schema)}.\"replicationState\"`,\n        sql`TRUNCATE TABLE ${sql(schema)}.\"changeLog\"`,\n        sql`TRUNCATE TABLE ${sql(schema)}.\"replicationConfig\"`,\n        sql`TRUNCATE TABLE ${sql(schema)}.\"tableMetadata\"`,\n        sql`TRUNCATE TABLE ${sql(schema)}.\"backfilling\"`,\n      );\n    }\n    // Initialize (or re-initialize TRUNCATED) tables.\n    //\n    // The PG storer uses the earliest changeLog entry as the safe watermark\n    // from which subscribers can be resumed. These initial entries ensure\n    // that subscribers can start from a freshly synced replica, even if new\n    // changes have been replicated and not purged.\n    //\n    // TODO: Replace this with an explicit `firstWatermark` column in the\n    //       change db.\n    const initialTx: FullChangeLogEntry[] = [\n      {watermark: replicaVersion, pos: 0, change: {tag: 'begin'}},\n      {watermark: replicaVersion, pos: 1, change: {tag: 'commit'}},\n    ];\n    stmts.push(\n      sql`INSERT INTO ${sql(schema)}.\"replicationConfig\" ${sql(replicaConfig)}`,\n      sql`INSERT INTO ${sql(schema)}.\"replicationState\"  ${sql(replicationState)} \n            ON CONFLICT (lock) DO UPDATE SET ${sql(replicationState)}`,\n      ...initialTx.map(\n        change => sql`INSERT INTO ${sql(schema)}.\"changeLog\" ${sql(change)}`,\n      ),\n    );\n\n    if (status === 'reset') {\n      return runStatementsAndTerminateLockHolders(\n        lc,\n        db,\n        shard,\n        'ensureReplicationConfig',\n        stmts,\n        setTimeoutFn,\n      );\n    }\n    return Promise.all(stmts);\n  });\n}\n\n// The time to wait for a TRUNCATE in ensureReplicationConfig before\n// terminating blocking backends via terminateChangeDBLockHolders.\nconst LOCK_HOLDER_TERMINATE_TIMEOUT_MS = 5_000;\n\n/**\n * Executes `stmts` that require ACCESS EXCLUSIVE locks (e.g. `TRUNCATE`),\n * which may be blocked by old storer catchup reads. Races against a\n * timeout that terminates the blocking backends if the `stmts` take too\n * long. The check is repeated until the statements complete, since the\n * `stmts` may not yet be waiting on a lock when the timer fires.\n */\nasync function runStatementsAndTerminateLockHolders(\n  lc: LogContext,\n  db: PostgresDB,\n  shard: ShardID,\n  name: string,\n  stmts: PendingQuery<Row[]>[],\n  setTimeoutFn: typeof setTimeout,\n) {\n  let done = false;\n  let timer: ReturnType<typeof setTimeout> | undefined;\n  const scheduleTerminate = () => {\n    timer = setTimeoutFn(async () => {\n      lc.info?.(`${name} blocked, terminating lock holders`);\n      try {\n        await terminateChangeDBLockHolders(lc, db, shard);\n      } catch (e) {\n        lc.warn?.('error terminating lock holders', e);\n      }\n      if (!done) {\n        scheduleTerminate();\n      }\n    }, LOCK_HOLDER_TERMINATE_TIMEOUT_MS);\n  };\n  scheduleTerminate();\n  try {\n    return await Promise.all(stmts);\n  } finally {\n    done = true;\n    clearTimeout(timer);\n  }\n}\n\nexport type ChangeLogReinitialization = {\n  /**\n   * The watermark of the replica from which the change log is re-initialized,\n   * i.e. the watermark from which the change stream will resume.\n   */\n  watermark: string;\n  cookies: {\n    tableMetadata: readonly {\n      schema: string;\n      table: string;\n      metadata: TableMetadata;\n    }[];\n    backfilling: readonly {\n      schema: string;\n      table: string;\n      column: string;\n      backfill: BackfillID;\n    }[];\n  };\n  owner: string;\n  ownerAddress: string;\n};\n\n/**\n * Re-initializes the change log from the state of the replica if it does not\n * contain all of the replica's changes, i.e. if its `lastWatermark` is before\n * the replica's watermark. This is the case if the change log is no longer\n * maintained (e.g. after running in high-availability mode), or if its last\n * owner stopped before persisting changes that it had already forwarded\n * (and backed up). Ownership of the change log is taken in the same\n * transaction, in which:\n *\n * * The `replicationState`, `changeLog`, `tableMetadata`, and `backfilling`\n *   tables are truncated. (The `replicationConfig` is kept;\n *   {@link ensureReplicationConfig} handles changes of the replica version\n *   or publications.)\n * * The change log is seeded with an empty transaction at the replica's\n *   watermark, so that subscribers at that watermark can be caught up, while\n *   subscribers at earlier watermarks are rejected as being too old.\n * * The `tableMetadata` and `backfilling` cookies are set from the replica.\n *\n * Returns `'maintained'` without modifying anything if the change log's\n * `lastWatermark` is at or past the replica's watermark, i.e. the change log\n * contains all of the replica's changes. Because a replication slot is only\n * acked past a transaction once it has been backed up, this means that the\n * change log contains every transaction up to the position of the replica's\n * replication slot, and it can be taken over (and resumed from its head).\n *\n * This is called when the change log's head is behind the position of the\n * replication slot (see `PgChangeLogPurgeLocker`). That position cannot be\n * compared with the watermarks of transactions to determine whether the\n * change log is maintained, as it is also advanced past positions of changes\n * outside of the publication.\n *\n * @param purgeLock A purge lock held by the caller, if any, which is released\n *     before the tables are truncated.\n */\nexport function reinitializeChangeLog(\n  lc: LogContext,\n  db: PostgresDB,\n  shard: ShardID,\n  {watermark, cookies, owner, ownerAddress}: ChangeLogReinitialization,\n  purgeLock?: PurgeLock,\n  setTimeoutFn: typeof setTimeout = setTimeout,\n): Promise<'reinitialized' | 'maintained'> {\n  const schema = cdcSchema(shard);\n  return runTx(db, async sql => {\n    // Lock the replicationState first, which is the order in which the\n    // storer acquires locks when writing the changeLog.\n    const state = await sql<{lastWatermark: string}[]>`\n      SELECT \"lastWatermark\" FROM ${sql(schema)}.\"replicationState\" FOR UPDATE`;\n    const lastWatermark = state[0]?.lastWatermark;\n    if (lastWatermark !== undefined && lastWatermark >= watermark) {\n      lc.info?.(\n        `changeLog@${lastWatermark} contains the replica's changes (@${watermark})`,\n      );\n      return 'maintained';\n    }\n    lc.info?.(\n      `re-initializing changeLog@${lastWatermark} from the replica @${watermark}`,\n    );\n    const replicationState: ReplicationState = {\n      lastWatermark: watermark,\n      owner,\n      ownerAddress,\n    };\n    const initialTx: FullChangeLogEntry[] = [\n      {watermark, pos: 0, change: {tag: 'begin'}},\n      {watermark, pos: 1, change: {tag: 'commit'}},\n    ];\n    // The caller's purge lock would block the TRUNCATE. Releasing it is safe:\n    // a concurrent purge checks the ownership after deleting, which blocks\n    // on the replicationState lock held by this transaction.\n    await purgeLock?.release();\n    await runStatementsAndTerminateLockHolders(\n      lc,\n      db,\n      shard,\n      'reinitializeChangeLog',\n      [\n        sql`TRUNCATE TABLE ${sql(schema)}.\"replicationState\"`,\n        sql`TRUNCATE TABLE ${sql(schema)}.\"changeLog\"`,\n        sql`TRUNCATE TABLE ${sql(schema)}.\"tableMetadata\"`,\n        sql`TRUNCATE TABLE ${sql(schema)}.\"backfilling\"`,\n      ],\n      setTimeoutFn,\n    );\n    await sql`INSERT INTO ${sql(schema)}.\"replicationState\" ${sql(replicationState)}`;\n    for (const entry of initialTx) {\n      await sql`INSERT INTO ${sql(schema)}.\"changeLog\" ${sql(entry)}`;\n    }\n    for (const row of cookies.tableMetadata) {\n      await sql`INSERT INTO ${sql(schema)}.\"tableMetadata\" ${sql({...row})}`;\n    }\n    for (const row of cookies.backfilling) {\n      await sql`INSERT INTO ${sql(schema)}.\"backfilling\" ${sql({...row})}`;\n    }\n    return 'reinitialized';\n  });\n}\n\nexport const CHANGE_STREAMER_APP_NAME = 'zero-change-streamer';\n\nexport class AutoResetSignal extends AbortError {\n  readonly name = 'AutoResetSignal';\n}\n\n/**\n * Terminates zero-cache backends that are blocking the current backend\n * from acquiring locks on CDC tables (e.g., during TRUNCATE).\n *\n * This is used during change-DB takeover when the new replication-manager's\n * `ensureReplicationConfig` needs to TRUNCATE tables, but the old\n * replication-manager's storer is still reading from them (e.g., large\n * catchup cursors).\n *\n * The function:\n * 1. Finds backends waiting for a lock on a TRUNCATE in {schema}\n * 2. Uses `pg_blocking_pids()` to identify which backends are blocking them\n * 3. Terminates blocking backends that have `application_name = 'zero-change-streamer'`\n *\n * Must be called on a **separate connection** from the one that is blocked,\n * since the blocked connection is inside a pending transaction.\n */\nasync function terminateChangeDBLockHolders(\n  lc: LogContext,\n  db: PostgresDB,\n  shard: ShardID,\n) {\n  const schema = cdcSchema(shard);\n\n  // Step 1: Find backends that are blocked waiting for a lock,\n  // whose query involves a TRUNCATE on this shard's CDC schema.\n  const blocked = await db<{pid: number}[]>`\n    SELECT pid FROM pg_stat_activity\n      WHERE wait_event_type = 'Lock'\n        AND application_name = ${CHANGE_STREAMER_APP_NAME}\n        AND query LIKE ${'%TRUNCATE%' + schema + '%'}`;\n\n  if (blocked.length === 0) {\n    lc.info?.('no blocked TRUNCATE backends found');\n    return;\n  }\n\n  const blockedPids = blocked.map(r => r.pid);\n  lc.info?.(`found blocked TRUNCATE backends: ${JSON.stringify(blockedPids)}`);\n\n  // Step 2: For each blocked backend, find and terminate its blockers\n  // that are zero-change-streamer connections.\n  const terminated = await db<\n    {pid: number; applicationName: string; query: string; terminated: boolean}[]\n  >`\n    SELECT pid, application_name as \"applicationName\", query,\n           pg_terminate_backend(pid) as terminated\n      FROM pg_stat_activity\n      WHERE pid = ANY(\n        SELECT unnest(pg_blocking_pids(blocked.pid))\n          FROM unnest(${blockedPids}::int[]) AS blocked(pid)\n      )\n      AND application_name = ${CHANGE_STREAMER_APP_NAME}\n      AND pid != ALL(${blockedPids}::int[])`;\n\n  if (terminated.length === 0) {\n    lc.info?.(`no ${CHANGE_STREAMER_APP_NAME} blockers found to terminate`);\n  } else {\n    for (const {pid, applicationName, query, terminated: ok} of terminated) {\n      lc.info?.(\n        `terminated blocking backend pid=${pid} app=${applicationName} ok=${ok} query=${query.slice(0, 200)}`,\n      );\n    }\n  }\n}\n"],"mappings":";;;;;;;;AAiBA,SAAS,OAAO,OAAgB;CAC9B,OAAO,MAAM,UAAU,KAAK,CAAC;AAC/B;AAEA,SAAS,aAAa,OAAgB;CACpC,OAAe,+BAA+B,OAAO,KAAK,EAAE;AAC9D;AAWA,SAAS,qBAAqB,OAAgB;CAI5C,OAAe;iBACA,OAAO,KAAK,EAAE;;;;;;;;AAQ/B;AAYA,SAAgB,4BAA4B,OAAgB;CAC1D,OAAe;iBACA,OAAO,KAAK,EAAE;;;;;;;AAO/B;AAEA,eAAsB,8BACpB,OACA,KACwB;CAGxB,QAAO,MAFc,GAA6C;iCACnC,IAAI,UAAU,KAAK,CAAC,EAAE,qBAAA,CACvC,EAAE,CAAC;AACnB;AAaA,SAAS,6BAA6B,OAAgB;CACpD,OAAe;iBACA,OAAO,KAAK,EAAE;;;;;;;AAO/B;AAEA,SAAgB,qBAAqB,OAAgB;CACnD,OAAe;iBACA,OAAO,KAAK,EAAE;;;;;;;iBAOd,OAAO,KAAK,EAAE;;;;;;;;AAQ/B;AAeA,SAAS,aAAa,OAAgB;CACpC,OACE,aAAa,KAAK,IAClB,qBAAqB,KAAK,IAC1B,4BAA4B,KAAK,IACjC,6BAA6B,KAAK,IAClC,qBAAqB,KAAK;AAE9B;AAMA,eAAsB,eACpB,IACA,IACA,OACA;CACA,GAAG,OAAO,uBAAuB;CACjC,MAAM,GAAG,OAAO,aAAa,KAAK,CAAC;AACrC;AAEA,eAAsB,kBAAkB,KAAiB,OAAgB;CAEvE,MAAM,GAAG;WACA,IAFM,UAAU,KAEZ,CAAM,EAAE;;AAEvB;;;;;;;;;;;;AAsBA,SAAS,0BACP,IACA,SACA,EAAC,cAAc,gBAAgB,aAC/B,WACA,YACiC;CACjC,IAAI,CAAC,SACH,OAAO;CAET,IACE,QAAQ,mBAAmB,kBAC3B,CAAC,OAAO,IAAI,IAAI,QAAQ,YAAY,GAAG,IAAI,IAAI,YAAY,CAAC,GAC5D;EACA,IAAI,mBAAmB,aAAa,CAAC,YACnC,MAAM,IAAI,gBACR,0BAA0B,QAAQ,eAAe,sBAC5B,eAAe,kBAAkB,WACxD;EAEF,OAAO;CACT;CACA,IAAI,QAAQ,eAAe;EACzB,IAAI,WACF,MAAM,IAAI,gBAAgB,sCAAsC;EAElE,GAAG,QACD,oOAKF;CACF;CACA,OAAO;AACT;AAEA,SAAS,wBAAwB,KAA0B,OAAgB;CACzE,OAAO,GAAoC;;aAEhC,IAAI,UAAU,KAAK,CAAC,EAAE;AACnC;;;;;;;;;AAUA,eAAsB,uBACpB,IACA,IACA,mBACA,OACA,WACA,aAAa,OACE;CACf,MAAM,CAAC,WAAW,MAAM,MAAM,KAAI,QAAO,wBAAwB,KAAK,KAAK,CAAC;CAQ5E,IAPe,0BACb,IACA,SACA,mBACA,WACA,UAEE,MAAW,SACb,GAAG,OACD,uBAAuB,SAAS,eAAe,iCAC5B,kBAAkB,eAAe,wDAEtD;AAEJ;;;;;;;;;;;AAYA,eAAsB,wBACpB,IACA,IACA,mBACA,OACA,WACA,WACA,eAAkC,YAClC,aAAa,OACb;CACA,MAAM,EAAC,cAAc,mBAAkB;CACvC,MAAM,gBAAgB;EAAC;EAAc;CAAc;CACnD,MAAM,mBAAqC;EACzC,eAAe;EACf,OAAO;EACP,cAAc;CAChB;CACA,MAAM,SAAS,UAAU,KAAK;CAE9B,MAAM,MAAM,IAAI,OAAM,QAAO;EAC3B,MAAM,CAAC,WAAW,MAAM,wBAAwB,KAAK,KAAK;EAC1D,MAAM,SAAS,0BACb,IACA,SACA,mBACA,WACA,UACF;EACA,IAAI,WAAW,cACb,OAAO,CAAC;EAEV,MAAM,QAA+B,CAAC;EACtC,IAAI,WAAW,SAAS;GACtB,GAAG,OACD,uBAAuB,SAAS,eAAe,iCAC5B,eAAe,mBACpC;GAGA,WAAgB,QAAQ;GAWxB,MAAM,KACJ,GAAG,kBAAkB,IAAI,MAAM,EAAE,sBACjC,GAAG,kBAAkB,IAAI,MAAM,EAAE,eACjC,GAAG,kBAAkB,IAAI,MAAM,EAAE,uBACjC,GAAG,kBAAkB,IAAI,MAAM,EAAE,mBACjC,GAAG,kBAAkB,IAAI,MAAM,EAAE,eACnC;EACF;EAUA,MAAM,YAAkC,CACtC;GAAC,WAAW;GAAgB,KAAK;GAAG,QAAQ,EAAC,KAAK,QAAO;EAAC,GAC1D;GAAC,WAAW;GAAgB,KAAK;GAAG,QAAQ,EAAC,KAAK,SAAQ;EAAC,CAC7D;EACA,MAAM,KACJ,GAAG,eAAe,IAAI,MAAM,EAAE,uBAAuB,IAAI,aAAa,KACtE,GAAG,eAAe,IAAI,MAAM,EAAE,uBAAuB,IAAI,gBAAgB,EAAE;+CAClC,IAAI,gBAAgB,KAC7D,GAAG,UAAU,KACX,WAAU,GAAG,eAAe,IAAI,MAAM,EAAE,eAAe,IAAI,MAAM,GACnE,CACF;EAEA,IAAI,WAAW,SACb,OAAO,qCACL,IACA,IACA,OACA,2BACA,OACA,YACF;EAEF,OAAO,QAAQ,IAAI,KAAK;CAC1B,CAAC;AACH;AAIA,IAAM,mCAAmC;;;;;;;;AASzC,eAAe,qCACb,IACA,IACA,OACA,MACA,OACA,cACA;CACA,IAAI,OAAO;CACX,IAAI;CACJ,MAAM,0BAA0B;EAC9B,QAAQ,aAAa,YAAY;GAC/B,GAAG,OAAO,GAAG,KAAK,mCAAmC;GACrD,IAAI;IACF,MAAM,6BAA6B,IAAI,IAAI,KAAK;GAClD,SAAS,GAAG;IACV,GAAG,OAAO,kCAAkC,CAAC;GAC/C;GACA,IAAI,CAAC,MACH,kBAAkB;EAEtB,GAAG,gCAAgC;CACrC;CACA,kBAAkB;CAClB,IAAI;EACF,OAAO,MAAM,QAAQ,IAAI,KAAK;CAChC,UAAU;EACR,OAAO;EACP,aAAa,KAAK;CACpB;AACF;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AA2DA,SAAgB,sBACd,IACA,IACA,OACA,EAAC,WAAW,SAAS,OAAO,gBAC5B,WACA,eAAkC,YACO;CACzC,MAAM,SAAS,UAAU,KAAK;CAC9B,OAAO,MAAM,IAAI,OAAM,QAAO;EAK5B,MAAM,iBAAgB,MAFF,GAA8B;oCAClB,IAAI,MAAM,EAAE,gCAAA,CAChB,EAAE,EAAE;EAChC,IAAI,kBAAkB,KAAA,KAAa,iBAAiB,WAAW;GAC7D,GAAG,OACD,aAAa,cAAc,oCAAoC,UAAU,EAC3E;GACA,OAAO;EACT;EACA,GAAG,OACD,6BAA6B,cAAc,qBAAqB,WAClE;EACA,MAAM,mBAAqC;GACzC,eAAe;GACf;GACA;EACF;EACA,MAAM,YAAkC,CACtC;GAAC;GAAW,KAAK;GAAG,QAAQ,EAAC,KAAK,QAAO;EAAC,GAC1C;GAAC;GAAW,KAAK;GAAG,QAAQ,EAAC,KAAK,SAAQ;EAAC,CAC7C;EAIA,MAAM,WAAW,QAAQ;EACzB,MAAM,qCACJ,IACA,IACA,OACA,yBACA;GACE,GAAG,kBAAkB,IAAI,MAAM,EAAE;GACjC,GAAG,kBAAkB,IAAI,MAAM,EAAE;GACjC,GAAG,kBAAkB,IAAI,MAAM,EAAE;GACjC,GAAG,kBAAkB,IAAI,MAAM,EAAE;EACnC,GACA,YACF;EACA,MAAM,GAAG,eAAe,IAAI,MAAM,EAAE,sBAAsB,IAAI,gBAAgB;EAC9E,KAAK,MAAM,SAAS,WAClB,MAAM,GAAG,eAAe,IAAI,MAAM,EAAE,eAAe,IAAI,KAAK;EAE9D,KAAK,MAAM,OAAO,QAAQ,eACxB,MAAM,GAAG,eAAe,IAAI,MAAM,EAAE,mBAAmB,IAAI,EAAC,GAAG,IAAG,CAAC;EAErE,KAAK,MAAM,OAAO,QAAQ,aACxB,MAAM,GAAG,eAAe,IAAI,MAAM,EAAE,iBAAiB,IAAI,EAAC,GAAG,IAAG,CAAC;EAEnE,OAAO;CACT,CAAC;AACH;AAEA,IAAa,2BAA2B;AAExC,IAAa,kBAAb,cAAqC,WAAW;CAC9C,OAAgB;AAClB;;;;;;;;;;;;;;;;;;AAmBA,eAAe,6BACb,IACA,IACA,OACA;CAKA,MAAM,UAAU,MAAM,EAAmB;;;iCAGV,yBAAyB;yBACjC,eARR,UAAU,KAQa,IAAS;CAE/C,IAAI,QAAQ,WAAW,GAAG;EACxB,GAAG,OAAO,oCAAoC;EAC9C;CACF;CAEA,MAAM,cAAc,QAAQ,KAAI,MAAK,EAAE,GAAG;CAC1C,GAAG,OAAO,oCAAoC,KAAK,UAAU,WAAW,GAAG;CAI3E,MAAM,aAAa,MAAM,EAExB;;;;;;wBAMqB,YAAY;;+BAEL,yBAAyB;uBACjC,YAAY;CAEjC,IAAI,WAAW,WAAW,GACxB,GAAG,OAAO,MAAM,yBAAyB,6BAA6B;MAEtE,KAAK,MAAM,EAAC,KAAK,iBAAiB,OAAO,YAAY,QAAO,YAC1D,GAAG,OACD,mCAAmC,IAAI,OAAO,gBAAgB,MAAM,GAAG,SAAS,MAAM,MAAM,GAAG,GAAG,GACpG;AAGN"}

package/out/zero-cache/src/services/change-streamer/server-start-metrics.js

@@ -0,0 +1,42 @@
+import { getOrCreateGauge } from "../../observability/metrics.js";
+//#region ../zero-cache/src/services/change-streamer/server-start-metrics.ts
+/**
+* Records where a replication-manager's start goes, reported once it is
+* ready by the `zero.replication.server_start` gauge, in seconds, labeled
+* by `phase` and `kind` (see {@link InitializeKind}). This is for diagnosis,
+* e.g. of why a start is slow, as the process only starts once.
+*/
+var ServerStartMetrics = class {
+	#phasesMs = /* @__PURE__ */ new Map();
+	#kind;
+	#ready = false;
+	setKind(kind) {
+		this.#kind = kind;
+	}
+	record(phase, ms) {
+		if (ms !== void 0 && !this.#ready) this.#phasesMs.set(phase, ms);
+	}
+	/**
+	* Records the `total` (by default, the time since the process started) and
+	* starts reporting the gauge. Phases recorded afterwards are ignored.
+	*/
+	ready(totalMs = performance.now()) {
+		if (this.#ready) return;
+		this.record("total", totalMs);
+		this.#ready = true;
+		const kind = this.#kind ?? "unknown";
+		getOrCreateGauge("replication", "server_start", {
+			description: "The duration of each phase of the replication-manager start (slot_create, fork_wait, restore, catch_up, backup_confirm, and the total from process start to readiness), by the kind of initialization (fork, resume, rmv1, initial_sync).",
+			unit: "s"
+		}).addCallback((o) => {
+			for (const [phase, ms] of this.#phasesMs) o.observe(ms / 1e3, {
+				phase,
+				kind
+			});
+		});
+	}
+};
+//#endregion
+export { ServerStartMetrics };
+
+//# sourceMappingURL=server-start-metrics.js.map

package/out/zero-cache/src/services/change-streamer/server-start-metrics.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"server-start-metrics.js","names":[],"sources":["../../../../../../zero-cache/src/services/change-streamer/server-start-metrics.ts"],"sourcesContent":["import {getOrCreateGauge} from '../../observability/metrics.ts';\nimport type {InitializeKind} from '../change-source/common/replica-restore.ts';\n\n/**\n * The phases of a replication-manager's start:\n * - `slot_create`: creating the new replica's slot, including retries while\n *   it is blocked by upstream transactions. This is separate from `fork_wait`\n *   because the upstream (e.g. a long-running transaction), not zero-cache or\n *   its capacity, determines how long it takes.\n * - `fork_wait`: selecting the replica to restore from (and claiming its\n *   slot), including the wait for a forked replica to pass the new slot's\n *   position, excluding `slot_create`\n * - `restore`: restoring the replica from its backup\n * - `catch_up`: from initialization to the backup-replicator being caught up\n *   (i.e. its replication lag readiness)\n * - `backup_confirm`: waiting for the first backup to cover the replica\n * - `total`: from the start of the (change-streamer) process to readiness\n *\n * The phases are not exhaustive (e.g. connecting to and preparing the\n * upstream), and `catch_up` and `backup_confirm` overlap.\n */\nexport type ServerStartPhase =\n  | 'slot_create'\n  | 'fork_wait'\n  | 'restore'\n  | 'catch_up'\n  | 'backup_confirm'\n  | 'total';\n\n/**\n * Records where a replication-manager's start goes, reported once it is\n * ready by the `zero.replication.server_start` gauge, in seconds, labeled\n * by `phase` and `kind` (see {@link InitializeKind}). This is for diagnosis,\n * e.g. of why a start is slow, as the process only starts once.\n */\nexport class ServerStartMetrics {\n  readonly #phasesMs = new Map<ServerStartPhase, number>();\n  #kind: InitializeKind | undefined;\n  #ready = false;\n\n  setKind(kind: InitializeKind | undefined) {\n    this.#kind = kind;\n  }\n\n  record(phase: ServerStartPhase, ms: number | undefined) {\n    if (ms !== undefined && !this.#ready) {\n      this.#phasesMs.set(phase, ms);\n    }\n  }\n\n  /**\n   * Records the `total` (by default, the time since the process started) and\n   * starts reporting the gauge. Phases recorded afterwards are ignored.\n   */\n  ready(totalMs = performance.now()) {\n    if (this.#ready) {\n      return;\n    }\n    this.record('total', totalMs);\n    this.#ready = true;\n    const kind = this.#kind ?? 'unknown';\n    getOrCreateGauge('replication', 'server_start', {\n      description:\n        'The duration of each phase of the replication-manager start ' +\n        '(slot_create, fork_wait, restore, catch_up, backup_confirm, and the total ' +\n        'from process start to readiness), by the kind of initialization ' +\n        '(fork, resume, rmv1, initial_sync).',\n      unit: 's',\n    }).addCallback(o => {\n      for (const [phase, ms] of this.#phasesMs) {\n        o.observe(ms / 1000, {phase, kind});\n      }\n    });\n  }\n}\n"],"mappings":";;;;;;;;AAmCA,IAAa,qBAAb,MAAgC;CAC9B,4BAAqB,IAAI,IAA8B;CACvD;CACA,SAAS;CAET,QAAQ,MAAkC;EACxC,KAAK,QAAQ;CACf;CAEA,OAAO,OAAyB,IAAwB;EACtD,IAAI,OAAO,KAAA,KAAa,CAAC,KAAK,QAC5B,KAAK,UAAU,IAAI,OAAO,EAAE;CAEhC;;;;;CAMA,MAAM,UAAU,YAAY,IAAI,GAAG;EACjC,IAAI,KAAK,QACP;EAEF,KAAK,OAAO,SAAS,OAAO;EAC5B,KAAK,SAAS;EACd,MAAM,OAAO,KAAK,SAAS;EAC3B,iBAAiB,eAAe,gBAAgB;GAC9C,aACE;GAIF,MAAM;EACR,CAAC,CAAC,CAAC,aAAY,MAAK;GAClB,KAAK,MAAM,CAAC,OAAO,OAAO,KAAK,WAC7B,EAAE,QAAQ,KAAK,KAAM;IAAC;IAAO;GAAI,CAAC;EAEtC,CAAC;CACH;AACF"}

package/out/zero-cache/src/services/change-streamer/snapshot-message.js

@@ -34,8 +34,7 @@ var statusSchema = valita_exports.object({
 	*/
 	replicaSize: valita_exports.number().optional()
 });
-var statusMessageSchema = valita_exports.tuple([valita_exports.literal("status"), statusSchema]);
-valita_exports.union(statusMessageSchema);
+valita_exports.tuple([valita_exports.literal("status"), statusSchema]);
 //#endregion
 export { statusSchema };
 

package/out/zero-cache/src/services/change-streamer/snapshot-message.js.map

@@ -1 +1 @@
-{"version":3,"file":"snapshot-message.js","names":[],"sources":["../../../../../../zero-cache/src/services/change-streamer/snapshot-message.ts"],"sourcesContent":["/**\n * Pure schema definitions for the `/snapshot` reservation status message.\n *\n * These live in their own module (depending only on valita) so that both\n * {@link snapshot.ts} (which also pulls in the change-streamer HTTP client) and\n * {@link subscribe.ts} can import them without forming an import cycle.\n */\n\nimport * as v from '../../../../shared/src/valita.ts';\n\nexport const statusSchema = v.object({\n  tag: v.literal('status'),\n\n  /**\n   * The location from which litestream should perform the restore.\n   */\n  backupURL: v.string(),\n\n  /**\n   * The `replicaVersion` of the backup. If a subscriber's restored or\n   * existing replica is of a different version, it should delete it and\n   * retry the restore from litestream (i.e. equivalent to a\n   * `WrongReplicaVersion` response from a `/changes` subscription).\n   */\n  replicaVersion: v.string(),\n\n  /**\n   * The earliest watermark from which catchup is possible. If the\n   * subscriber's replica is older that this watermark, it should delete it\n   * and (retry the) restore from litestream (i.e. equivalent to a\n   * `WatermarkTooOld` response from a `/changes` subscription).\n   */\n  minWatermark: v.string(),\n\n  /**\n   * The size in bytes of the replication-manager's replica when the snapshot\n   * was reserved, used as an estimate of the size of the restored replica\n   * when reporting restore progress. Absent from older replication-managers.\n   */\n  replicaSize: v.number().optional(),\n});\n\nexport type SnapshotStatus = v.Infer<typeof statusSchema>;\n\nexport const statusMessageSchema = v.tuple([v.literal('status'), statusSchema]);\n\nexport const snapshotMessageSchema = v.union(statusMessageSchema);\n\nexport type SnapshotMessage = v.Infer<typeof statusMessageSchema>;\n"],"mappings":";;;;;;;;;AAUA,IAAa,eAAe,eAAE,OAAO;CACnC,KAAK,eAAE,QAAQ,QAAQ;;;;CAKvB,WAAW,eAAE,OAAO;;;;;;;CAQpB,gBAAgB,eAAE,OAAO;;;;;;;CAQzB,cAAc,eAAE,OAAO;;;;;;CAOvB,aAAa,eAAE,OAAO,EAAE,SAAS;AACnC,CAAC;AAID,IAAa,sBAAsB,eAAE,MAAM,CAAC,eAAE,QAAQ,QAAQ,GAAG,YAAY,CAAC;AAEzC,eAAE,MAAM,mBAAmB"}
+{"version":3,"file":"snapshot-message.js","names":["v"],"sources":["../../../../../../zero-cache/src/services/change-streamer/snapshot-message.ts"],"sourcesContent":["/**\n * Pure schema definitions for the `/snapshot` reservation status message.\n *\n * These live in their own module (depending only on valita) so that both\n * {@link snapshot.ts} (which also pulls in the change-streamer HTTP client) and\n * {@link subscribe.ts} can import them without forming an import cycle.\n */\n\nimport * as v from '../../../../shared/src/valita.ts';\n\nexport const statusSchema = v.object({\n  tag: v.literal('status'),\n\n  /**\n   * The location from which litestream should perform the restore.\n   */\n  backupURL: v.string(),\n\n  /**\n   * The `replicaVersion` of the backup. If a subscriber's restored or\n   * existing replica is of a different version, it should delete it and\n   * retry the restore from litestream (i.e. equivalent to a\n   * `WrongReplicaVersion` response from a `/changes` subscription).\n   */\n  replicaVersion: v.string(),\n\n  /**\n   * The earliest watermark from which catchup is possible. If the\n   * subscriber's replica is older that this watermark, it should delete it\n   * and (retry the) restore from litestream (i.e. equivalent to a\n   * `WatermarkTooOld` response from a `/changes` subscription).\n   */\n  minWatermark: v.string(),\n\n  /**\n   * The size in bytes of the replication-manager's replica when the snapshot\n   * was reserved, used as an estimate of the size of the restored replica\n   * when reporting restore progress. Absent from older replication-managers.\n   */\n  replicaSize: v.number().optional(),\n});\n\nconst statusMessageSchema = v.tuple([v.literal('status'), statusSchema]);\n\nexport type SnapshotMessage = v.Infer<typeof statusMessageSchema>;\n"],"mappings":";;;;;;;;;AAUA,IAAa,eAAeA,eAAE,OAAO;CACnC,KAAKA,eAAE,QAAQ,QAAQ;;;;CAKvB,WAAWA,eAAE,OAAO;;;;;;;CAQpB,gBAAgBA,eAAE,OAAO;;;;;;;CAQzB,cAAcA,eAAE,OAAO;;;;;;CAOvB,aAAaA,eAAE,OAAO,CAAC,CAAC,SAAS;AACnC,CAAC;AAE2BA,eAAE,MAAM,CAACA,eAAE,QAAQ,QAAQ,GAAG,YAAY,CAAC"}

package/out/zero-cache/src/services/change-streamer/snapshot-reservations.js.map

@@ -1 +1 @@
-{"version":3,"file":"snapshot-reservations.js","names":["#lc","#backupConfig","#callbacks","#reservations","#close","#metricAttrs","#replicaSize","#downstream","#watermark","#delayNoted"],"sources":["../../../../../../zero-cache/src/services/change-streamer/snapshot-reservations.ts"],"sourcesContent":["import {statSync} from 'node:fs';\nimport type {LogContext} from '@rocicorp/logger';\nimport type {Source} from '../../types/streams.ts';\nimport {Subscription} from '../../types/subscription.ts';\nimport {\n  litestreamMonitorMetricAttrs,\n  litestreamSnapshotReservationConfirmDuration,\n  litestreamSnapshotReservationDuration,\n} from '../litestream/metrics.ts';\nimport type {BackupConfig} from './change-streamer-service.ts';\nimport type {SnapshotMessage} from './snapshot-message.ts';\nimport type {ChangeLogReadSource} from './sqlite-change-log-read-router.ts';\n\nexport type ReservationCallbacks = {\n  /**\n   * Called whenever a reservation is closed, including when it is replaced\n   * by a new reservation (e.g. a retry) for the same task.\n   */\n  onClose?: ((taskID: string) => void) | undefined;\n\n  /**\n   * Called when a task's reservation is closed without being replaced,\n   * i.e. when the task no longer holds a reservation.\n   */\n  onEnd?: ((taskID: string) => void) | undefined;\n};\n\nexport class SnapshotReservations {\n  readonly #lc: LogContext;\n  readonly #backupConfig: BackupConfig;\n  readonly #callbacks: ReservationCallbacks;\n  readonly #reservations = new Map<string, Reservation>();\n\n  constructor(\n    lc: LogContext,\n    backupConfig: BackupConfig,\n    callbacks: ReservationCallbacks = {},\n  ) {\n    this.#lc = lc.withContext('component', 'snapshot-reserver');\n    this.#backupConfig = backupConfig;\n    this.#callbacks = callbacks;\n  }\n\n  open(taskID: string): Source<SnapshotMessage> {\n    this.#close(taskID, undefined, true);\n\n    const instanceID = {};\n    const downstream = Subscription.create<SnapshotMessage>({\n      cleanup: () => this.#close(taskID, instanceID),\n    });\n    this.#reservations.set(taskID, new Reservation(instanceID, downstream));\n    this.#lc.info?.(`created snapshot reservation for ${taskID}`);\n    return downstream;\n  }\n\n  close(taskID: string) {\n    this.#close(taskID, undefined);\n  }\n\n  isCurrent(taskID: string, source: Source<SnapshotMessage>): boolean {\n    return this.#reservations.get(taskID)?.owns(source) ?? false;\n  }\n\n  #metricAttrs() {\n    return litestreamMonitorMetricAttrs(\n      this.#backupConfig.backupURL,\n      this.#backupConfig.litestreamVersion,\n      'view_syncer',\n    );\n  }\n\n  #close(\n    taskID: string,\n    cancelledInstanceID: InstanceID | undefined,\n    replaced = false,\n  ) {\n    const res = this.#reservations.get(taskID);\n    if (\n      res &&\n      (!cancelledInstanceID || res.instanceID === cancelledInstanceID)\n    ) {\n      // Note: delete first, so that the reservation is gone when close() is called.\n      this.#reservations.delete(taskID);\n      this.#callbacks.onClose?.(taskID);\n      if (!replaced) {\n        this.#callbacks.onEnd?.(taskID);\n      }\n      res.close();\n\n      const duration = Date.now() - res.startTime.getTime();\n      this.#lc.info?.(\n        `ended snapshot reservation for ${taskID} (${duration} ms)`,\n      );\n      // `result` reports how the reservation ended and `confirmed` whether it\n      // ever received its bounds. They are separate dimensions: a follower\n      // that gives up while waiting for a confirmation is\n      // `result=cancelled, confirmed=false`, which a single collapsed\n      // attribute cannot distinguish from a client that simply went away.\n      litestreamSnapshotReservationDuration().recordMs(duration, {\n        ...this.#metricAttrs(),\n        result: cancelledInstanceID ? 'cancelled' : 'closed',\n        confirmed: res.confirmed(),\n      });\n    }\n  }\n\n  confirmationsRequired() {\n    for (const res of this.#reservations.values()) {\n      if (!res.confirmed()) {\n        return true;\n      }\n    }\n    return false;\n  }\n\n  unconfirmedTaskIDs(): string[] {\n    return [...this.#reservations.entries()]\n      .filter(([, reservation]) => !reservation.confirmed())\n      .map(([taskID]) => taskID);\n  }\n\n  /** Confirms one reservation with the bounds of its pinned read source. */\n  confirmFor(\n    taskID: string,\n    replicaVersion: string,\n    minWatermark: string,\n    source: ChangeLogReadSource,\n  ): void {\n    const res = this.#reservations.get(taskID);\n    if (res && !res.confirmed()) {\n      this.#lc.info?.(\n        `reserving change-log entries since ${minWatermark} for ${taskID}`,\n      );\n      res.confirm(\n        this.#backupConfig.backupURL,\n        replicaVersion,\n        minWatermark,\n        this.#replicaSize(),\n      );\n      // Measured from `open()`, not from the first confirmation attempt: what\n      // matters is how long the follower waited before it could restore.\n      litestreamSnapshotReservationConfirmDuration().recordMs(\n        Date.now() - res.startTime.getTime(),\n        {...this.#metricAttrs(), source},\n      );\n    }\n  }\n\n  #replicaSize(): number | undefined {\n    const {replicaFile} = this.#backupConfig;\n    if (!replicaFile) {\n      return undefined;\n    }\n    try {\n      return statSync(replicaFile).size;\n    } catch (e) {\n      this.#lc.warn?.(`Unable to get the size of ${replicaFile}`, e);\n      return undefined;\n    }\n  }\n\n  /**\n   * Notes that a confirmation was deferred, returning true only the first time\n   * for this reservation. Confirmation is retried on every backup, so an\n   * undeduplicated count would measure backups rather than delayed followers.\n   */\n  noteConfirmationDelayed(taskID: string): boolean {\n    return this.#reservations.get(taskID)?.noteDelayed() ?? false;\n  }\n\n  getReservedWatermarks() {\n    return Array.from(\n      this.#reservations.values(),\n      ({reservedWatermark}) => reservedWatermark,\n    ).filter(watermark => watermark !== null);\n  }\n}\n\ntype InstanceID = {};\n\nclass Reservation {\n  readonly instanceID: InstanceID;\n  readonly startTime: Date = new Date();\n  readonly #downstream: Subscription<SnapshotMessage>;\n  #watermark: string | null = null;\n  #delayNoted = false;\n\n  constructor(\n    instanceID: InstanceID,\n    downstream: Subscription<SnapshotMessage>,\n  ) {\n    this.instanceID = instanceID;\n    this.#downstream = downstream;\n  }\n\n  get reservedWatermark() {\n    return this.#watermark;\n  }\n\n  confirmed() {\n    return this.#watermark !== null;\n  }\n\n  owns(source: Source<SnapshotMessage>): boolean {\n    return this.#downstream === source;\n  }\n\n  noteDelayed(): boolean {\n    if (this.#delayNoted) {\n      return false;\n    }\n    this.#delayNoted = true;\n    return true;\n  }\n\n  confirm(\n    backupURL: string,\n    replicaVersion: string,\n    minWatermark: string,\n    replicaSize: number | undefined,\n  ) {\n    if (this.#watermark === null) {\n      if (this.#downstream.active) {\n        this.#downstream.push([\n          'status',\n          {tag: 'status', backupURL, replicaVersion, minWatermark, replicaSize},\n        ]);\n      }\n      this.#watermark = minWatermark;\n    }\n  }\n\n  close() {\n    this.#downstream.cancel();\n  }\n}\n"],"mappings":";;;;AA2BA,IAAa,uBAAb,MAAkC;CAChC;CACA;CACA;CACA,gCAAyB,IAAI,IAAyB;CAEtD,YACE,IACA,cACA,YAAkC,CAAC,GACnC;EACA,KAAKA,MAAM,GAAG,YAAY,aAAa,mBAAmB;EAC1D,KAAKC,gBAAgB;EACrB,KAAKC,aAAa;CACpB;CAEA,KAAK,QAAyC;EAC5C,KAAKE,OAAO,QAAQ,KAAA,GAAW,IAAI;EAEnC,MAAM,aAAa,CAAC;EACpB,MAAM,aAAa,aAAa,OAAwB,EACtD,eAAe,KAAKA,OAAO,QAAQ,UAAU,EAC/C,CAAC;EACD,KAAKD,cAAc,IAAI,QAAQ,IAAI,YAAY,YAAY,UAAU,CAAC;EACtE,KAAKH,IAAI,OAAO,oCAAoC,QAAQ;EAC5D,OAAO;CACT;CAEA,MAAM,QAAgB;EACpB,KAAKI,OAAO,QAAQ,KAAA,CAAS;CAC/B;CAEA,UAAU,QAAgB,QAA0C;EAClE,OAAO,KAAKD,cAAc,IAAI,MAAM,GAAG,KAAK,MAAM,KAAK;CACzD;CAEA,eAAe;EACb,OAAO,6BACL,KAAKF,cAAc,WACnB,KAAKA,cAAc,mBACnB,aACF;CACF;CAEA,OACE,QACA,qBACA,WAAW,OACX;EACA,MAAM,MAAM,KAAKE,cAAc,IAAI,MAAM;EACzC,IACE,QACC,CAAC,uBAAuB,IAAI,eAAe,sBAC5C;GAEA,KAAKA,cAAc,OAAO,MAAM;GAChC,KAAKD,WAAW,UAAU,MAAM;GAChC,IAAI,CAAC,UACH,KAAKA,WAAW,QAAQ,MAAM;GAEhC,IAAI,MAAM;GAEV,MAAM,WAAW,KAAK,IAAI,IAAI,IAAI,UAAU,QAAQ;GACpD,KAAKF,IAAI,OACP,kCAAkC,OAAO,IAAI,SAAS,KACxD;GAMA,sCAAsC,EAAE,SAAS,UAAU;IACzD,GAAG,KAAKK,aAAa;IACrB,QAAQ,sBAAsB,cAAc;IAC5C,WAAW,IAAI,UAAU;GAC3B,CAAC;EACH;CACF;CAEA,wBAAwB;EACtB,KAAK,MAAM,OAAO,KAAKF,cAAc,OAAO,GAC1C,IAAI,CAAC,IAAI,UAAU,GACjB,OAAO;EAGX,OAAO;CACT;CAEA,qBAA+B;EAC7B,OAAO,CAAC,GAAG,KAAKA,cAAc,QAAQ,CAAC,EACpC,QAAQ,GAAG,iBAAiB,CAAC,YAAY,UAAU,CAAC,EACpD,KAAK,CAAC,YAAY,MAAM;CAC7B;;CAGA,WACE,QACA,gBACA,cACA,QACM;EACN,MAAM,MAAM,KAAKA,cAAc,IAAI,MAAM;EACzC,IAAI,OAAO,CAAC,IAAI,UAAU,GAAG;GAC3B,KAAKH,IAAI,OACP,sCAAsC,aAAa,OAAO,QAC5D;GACA,IAAI,QACF,KAAKC,cAAc,WACnB,gBACA,cACA,KAAKK,aAAa,CACpB;GAGA,6CAA6C,EAAE,SAC7C,KAAK,IAAI,IAAI,IAAI,UAAU,QAAQ,GACnC;IAAC,GAAG,KAAKD,aAAa;IAAG;GAAM,CACjC;EACF;CACF;CAEA,eAAmC;EACjC,MAAM,EAAC,gBAAe,KAAKJ;EAC3B,IAAI,CAAC,aACH;EAEF,IAAI;GACF,OAAO,SAAS,WAAW,EAAE;EAC/B,SAAS,GAAG;GACV,KAAKD,IAAI,OAAO,6BAA6B,eAAe,CAAC;GAC7D;EACF;CACF;;;;;;CAOA,wBAAwB,QAAyB;EAC/C,OAAO,KAAKG,cAAc,IAAI,MAAM,GAAG,YAAY,KAAK;CAC1D;CAEA,wBAAwB;EACtB,OAAO,MAAM,KACX,KAAKA,cAAc,OAAO,IACzB,EAAC,wBAAuB,iBAC3B,EAAE,QAAO,cAAa,cAAc,IAAI;CAC1C;AACF;AAIA,IAAM,cAAN,MAAkB;CAChB;CACA,4BAA2B,IAAI,KAAK;CACpC;CACA,aAA4B;CAC5B,cAAc;CAEd,YACE,YACA,YACA;EACA,KAAK,aAAa;EAClB,KAAKI,cAAc;CACrB;CAEA,IAAI,oBAAoB;EACtB,OAAO,KAAKC;CACd;CAEA,YAAY;EACV,OAAO,KAAKA,eAAe;CAC7B;CAEA,KAAK,QAA0C;EAC7C,OAAO,KAAKD,gBAAgB;CAC9B;CAEA,cAAuB;EACrB,IAAI,KAAKE,aACP,OAAO;EAET,KAAKA,cAAc;EACnB,OAAO;CACT;CAEA,QACE,WACA,gBACA,cACA,aACA;EACA,IAAI,KAAKD,eAAe,MAAM;GAC5B,IAAI,KAAKD,YAAY,QACnB,KAAKA,YAAY,KAAK,CACpB,UACA;IAAC,KAAK;IAAU;IAAW;IAAgB;IAAc;GAAW,CACtE,CAAC;GAEH,KAAKC,aAAa;EACpB;CACF;CAEA,QAAQ;EACN,KAAKD,YAAY,OAAO;CAC1B;AACF"}
+{"version":3,"file":"snapshot-reservations.js","names":[],"sources":["../../../../../../zero-cache/src/services/change-streamer/snapshot-reservations.ts"],"sourcesContent":["import {statSync} from 'node:fs';\nimport type {LogContext} from '@rocicorp/logger';\nimport type {Source} from '../../types/streams.ts';\nimport {Subscription} from '../../types/subscription.ts';\nimport {\n  litestreamMonitorMetricAttrs,\n  litestreamSnapshotReservationConfirmDuration,\n  litestreamSnapshotReservationDuration,\n} from '../litestream/metrics.ts';\nimport type {BackupConfig} from './change-streamer-service.ts';\nimport type {SnapshotMessage} from './snapshot-message.ts';\nimport type {ChangeLogReadSource} from './sqlite-change-log-read-router.ts';\n\nexport type ReservationCallbacks = {\n  /**\n   * Called whenever a reservation is closed, including when it is replaced\n   * by a new reservation (e.g. a retry) for the same task.\n   */\n  onClose?: ((taskID: string) => void) | undefined;\n\n  /**\n   * Called when a task's reservation is closed without being replaced,\n   * i.e. when the task no longer holds a reservation.\n   */\n  onEnd?: ((taskID: string) => void) | undefined;\n};\n\nexport class SnapshotReservations {\n  readonly #lc: LogContext;\n  readonly #backupConfig: BackupConfig;\n  readonly #callbacks: ReservationCallbacks;\n  readonly #reservations = new Map<string, Reservation>();\n\n  constructor(\n    lc: LogContext,\n    backupConfig: BackupConfig,\n    callbacks: ReservationCallbacks = {},\n  ) {\n    this.#lc = lc.withContext('component', 'snapshot-reserver');\n    this.#backupConfig = backupConfig;\n    this.#callbacks = callbacks;\n  }\n\n  open(taskID: string): Source<SnapshotMessage> {\n    this.#close(taskID, undefined, true);\n\n    const instanceID = {};\n    const downstream = Subscription.create<SnapshotMessage>({\n      cleanup: () => this.#close(taskID, instanceID),\n    });\n    this.#reservations.set(taskID, new Reservation(instanceID, downstream));\n    this.#lc.info?.(`created snapshot reservation for ${taskID}`);\n    return downstream;\n  }\n\n  close(taskID: string) {\n    this.#close(taskID, undefined);\n  }\n\n  isCurrent(taskID: string, source: Source<SnapshotMessage>): boolean {\n    return this.#reservations.get(taskID)?.owns(source) ?? false;\n  }\n\n  #metricAttrs() {\n    return litestreamMonitorMetricAttrs(\n      this.#backupConfig.backupURL,\n      this.#backupConfig.litestreamVersion,\n      'view_syncer',\n    );\n  }\n\n  #close(\n    taskID: string,\n    cancelledInstanceID: InstanceID | undefined,\n    replaced = false,\n  ) {\n    const res = this.#reservations.get(taskID);\n    if (\n      res &&\n      (!cancelledInstanceID || res.instanceID === cancelledInstanceID)\n    ) {\n      // Note: delete first, so that the reservation is gone when close() is called.\n      this.#reservations.delete(taskID);\n      this.#callbacks.onClose?.(taskID);\n      if (!replaced) {\n        this.#callbacks.onEnd?.(taskID);\n      }\n      res.close();\n\n      const duration = Date.now() - res.startTime.getTime();\n      this.#lc.info?.(\n        `ended snapshot reservation for ${taskID} (${duration} ms)`,\n      );\n      // `result` reports how the reservation ended and `confirmed` whether it\n      // ever received its bounds. They are separate dimensions: a follower\n      // that gives up while waiting for a confirmation is\n      // `result=cancelled, confirmed=false`, which a single collapsed\n      // attribute cannot distinguish from a client that simply went away.\n      litestreamSnapshotReservationDuration().recordMs(duration, {\n        ...this.#metricAttrs(),\n        result: cancelledInstanceID ? 'cancelled' : 'closed',\n        confirmed: res.confirmed(),\n      });\n    }\n  }\n\n  confirmationsRequired() {\n    for (const res of this.#reservations.values()) {\n      if (!res.confirmed()) {\n        return true;\n      }\n    }\n    return false;\n  }\n\n  unconfirmedTaskIDs(): string[] {\n    return [...this.#reservations.entries()]\n      .filter(([, reservation]) => !reservation.confirmed())\n      .map(([taskID]) => taskID);\n  }\n\n  /** Confirms one reservation with the bounds of its pinned read source. */\n  confirmFor(\n    taskID: string,\n    replicaVersion: string,\n    minWatermark: string,\n    source: ChangeLogReadSource,\n  ): void {\n    const res = this.#reservations.get(taskID);\n    if (res && !res.confirmed()) {\n      this.#lc.info?.(\n        `reserving change-log entries since ${minWatermark} for ${taskID}`,\n      );\n      res.confirm(\n        this.#backupConfig.backupURL,\n        replicaVersion,\n        minWatermark,\n        this.#replicaSize(),\n      );\n      // Measured from `open()`, not from the first confirmation attempt: what\n      // matters is how long the follower waited before it could restore.\n      litestreamSnapshotReservationConfirmDuration().recordMs(\n        Date.now() - res.startTime.getTime(),\n        {...this.#metricAttrs(), source},\n      );\n    }\n  }\n\n  #replicaSize(): number | undefined {\n    const {replicaFile} = this.#backupConfig;\n    if (!replicaFile) {\n      return undefined;\n    }\n    try {\n      return statSync(replicaFile).size;\n    } catch (e) {\n      this.#lc.warn?.(`Unable to get the size of ${replicaFile}`, e);\n      return undefined;\n    }\n  }\n\n  /**\n   * Notes that a confirmation was deferred, returning true only the first time\n   * for this reservation. Confirmation is retried on every backup, so an\n   * undeduplicated count would measure backups rather than delayed followers.\n   */\n  noteConfirmationDelayed(taskID: string): boolean {\n    return this.#reservations.get(taskID)?.noteDelayed() ?? false;\n  }\n\n  getReservedWatermarks() {\n    return Array.from(\n      this.#reservations.values(),\n      ({reservedWatermark}) => reservedWatermark,\n    ).filter(watermark => watermark !== null);\n  }\n}\n\ntype InstanceID = {};\n\nclass Reservation {\n  readonly instanceID: InstanceID;\n  readonly startTime: Date = new Date();\n  readonly #downstream: Subscription<SnapshotMessage>;\n  #watermark: string | null = null;\n  #delayNoted = false;\n\n  constructor(\n    instanceID: InstanceID,\n    downstream: Subscription<SnapshotMessage>,\n  ) {\n    this.instanceID = instanceID;\n    this.#downstream = downstream;\n  }\n\n  get reservedWatermark() {\n    return this.#watermark;\n  }\n\n  confirmed() {\n    return this.#watermark !== null;\n  }\n\n  owns(source: Source<SnapshotMessage>): boolean {\n    return this.#downstream === source;\n  }\n\n  noteDelayed(): boolean {\n    if (this.#delayNoted) {\n      return false;\n    }\n    this.#delayNoted = true;\n    return true;\n  }\n\n  confirm(\n    backupURL: string,\n    replicaVersion: string,\n    minWatermark: string,\n    replicaSize: number | undefined,\n  ) {\n    if (this.#watermark === null) {\n      if (this.#downstream.active) {\n        this.#downstream.push([\n          'status',\n          {tag: 'status', backupURL, replicaVersion, minWatermark, replicaSize},\n        ]);\n      }\n      this.#watermark = minWatermark;\n    }\n  }\n\n  close() {\n    this.#downstream.cancel();\n  }\n}\n"],"mappings":";;;;AA2BA,IAAa,uBAAb,MAAkC;CAChC;CACA;CACA;CACA,gCAAyB,IAAI,IAAyB;CAEtD,YACE,IACA,cACA,YAAkC,CAAC,GACnC;EACA,KAAK,MAAM,GAAG,YAAY,aAAa,mBAAmB;EAC1D,KAAK,gBAAgB;EACrB,KAAK,aAAa;CACpB;CAEA,KAAK,QAAyC;EAC5C,KAAK,OAAO,QAAQ,KAAA,GAAW,IAAI;EAEnC,MAAM,aAAa,CAAC;EACpB,MAAM,aAAa,aAAa,OAAwB,EACtD,eAAe,KAAK,OAAO,QAAQ,UAAU,EAC/C,CAAC;EACD,KAAK,cAAc,IAAI,QAAQ,IAAI,YAAY,YAAY,UAAU,CAAC;EACtE,KAAK,IAAI,OAAO,oCAAoC,QAAQ;EAC5D,OAAO;CACT;CAEA,MAAM,QAAgB;EACpB,KAAK,OAAO,QAAQ,KAAA,CAAS;CAC/B;CAEA,UAAU,QAAgB,QAA0C;EAClE,OAAO,KAAK,cAAc,IAAI,MAAM,CAAC,EAAE,KAAK,MAAM,KAAK;CACzD;CAEA,eAAe;EACb,OAAO,6BACL,KAAK,cAAc,WACnB,KAAK,cAAc,mBACnB,aACF;CACF;CAEA,OACE,QACA,qBACA,WAAW,OACX;EACA,MAAM,MAAM,KAAK,cAAc,IAAI,MAAM;EACzC,IACE,QACC,CAAC,uBAAuB,IAAI,eAAe,sBAC5C;GAEA,KAAK,cAAc,OAAO,MAAM;GAChC,KAAK,WAAW,UAAU,MAAM;GAChC,IAAI,CAAC,UACH,KAAK,WAAW,QAAQ,MAAM;GAEhC,IAAI,MAAM;GAEV,MAAM,WAAW,KAAK,IAAI,IAAI,IAAI,UAAU,QAAQ;GACpD,KAAK,IAAI,OACP,kCAAkC,OAAO,IAAI,SAAS,KACxD;GAMA,sCAAsC,CAAC,CAAC,SAAS,UAAU;IACzD,GAAG,KAAK,aAAa;IACrB,QAAQ,sBAAsB,cAAc;IAC5C,WAAW,IAAI,UAAU;GAC3B,CAAC;EACH;CACF;CAEA,wBAAwB;EACtB,KAAK,MAAM,OAAO,KAAK,cAAc,OAAO,GAC1C,IAAI,CAAC,IAAI,UAAU,GACjB,OAAO;EAGX,OAAO;CACT;CAEA,qBAA+B;EAC7B,OAAO,CAAC,GAAG,KAAK,cAAc,QAAQ,CAAC,CAAC,CACrC,QAAQ,GAAG,iBAAiB,CAAC,YAAY,UAAU,CAAC,CAAC,CACrD,KAAK,CAAC,YAAY,MAAM;CAC7B;;CAGA,WACE,QACA,gBACA,cACA,QACM;EACN,MAAM,MAAM,KAAK,cAAc,IAAI,MAAM;EACzC,IAAI,OAAO,CAAC,IAAI,UAAU,GAAG;GAC3B,KAAK,IAAI,OACP,sCAAsC,aAAa,OAAO,QAC5D;GACA,IAAI,QACF,KAAK,cAAc,WACnB,gBACA,cACA,KAAK,aAAa,CACpB;GAGA,6CAA6C,CAAC,CAAC,SAC7C,KAAK,IAAI,IAAI,IAAI,UAAU,QAAQ,GACnC;IAAC,GAAG,KAAK,aAAa;IAAG;GAAM,CACjC;EACF;CACF;CAEA,eAAmC;EACjC,MAAM,EAAC,gBAAe,KAAK;EAC3B,IAAI,CAAC,aACH;EAEF,IAAI;GACF,OAAO,SAAS,WAAW,CAAC,CAAC;EAC/B,SAAS,GAAG;GACV,KAAK,IAAI,OAAO,6BAA6B,eAAe,CAAC;GAC7D;EACF;CACF;;;;;;CAOA,wBAAwB,QAAyB;EAC/C,OAAO,KAAK,cAAc,IAAI,MAAM,CAAC,EAAE,YAAY,KAAK;CAC1D;CAEA,wBAAwB;EACtB,OAAO,MAAM,KACX,KAAK,cAAc,OAAO,IACzB,EAAC,wBAAuB,iBAC3B,CAAC,CAAC,QAAO,cAAa,cAAc,IAAI;CAC1C;AACF;AAIA,IAAM,cAAN,MAAkB;CAChB;CACA,4BAA2B,IAAI,KAAK;CACpC;CACA,aAA4B;CAC5B,cAAc;CAEd,YACE,YACA,YACA;EACA,KAAK,aAAa;EAClB,KAAK,cAAc;CACrB;CAEA,IAAI,oBAAoB;EACtB,OAAO,KAAK;CACd;CAEA,YAAY;EACV,OAAO,KAAK,eAAe;CAC7B;CAEA,KAAK,QAA0C;EAC7C,OAAO,KAAK,gBAAgB;CAC9B;CAEA,cAAuB;EACrB,IAAI,KAAK,aACP,OAAO;EAET,KAAK,cAAc;EACnB,OAAO;CACT;CAEA,QACE,WACA,gBACA,cACA,aACA;EACA,IAAI,KAAK,eAAe,MAAM;GAC5B,IAAI,KAAK,YAAY,QACnB,KAAK,YAAY,KAAK,CACpB,UACA;IAAC,KAAK;IAAU;IAAW;IAAgB;IAAc;GAAW,CACtE,CAAC;GAEH,KAAK,aAAa;EACpB;CACF;CAEA,QAAQ;EACN,KAAK,YAAY,OAAO;CAC1B;AACF"}

package/out/zero-cache/src/services/change-streamer/sqlite-change-log-catchup.js

@@ -1,7 +1,7 @@
+import { resolver } from "../../../../shared/src/resolver.js";
 import { AbortError } from "../../../../shared/src/abort-error.js";
 import { sleep } from "../../../../shared/src/sleep.js";
 import { getOrCreateCounter, getOrCreateLatencyHistogram, getOrCreateValueHistogram } from "../../observability/metrics.js";
-import { resolver } from "@rocicorp/resolver";
 //#region ../zero-cache/src/services/change-streamer/sqlite-change-log-catchup.ts
 var DEFAULT_BARRIER_POLL_INTERVAL_MS = 1e3;
 var NOOP_CLEANUP_GUARD = { runWhilePurgeBlocked: (register) => Promise.resolve(register()) };
@@ -339,6 +339,6 @@ var SQLiteChangeLogBarrierBacklogError = class extends SQLiteChangeLogBarrierErr
 	name = "SQLiteChangeLogBarrierBacklogError";
 };
 //#endregion
-export { SQLiteChangeLogCatchup };
+export { SQLiteChangeLogBarrierBacklogError, SQLiteChangeLogBarrierTimeoutError, SQLiteChangeLogCatchup };
 
 //# sourceMappingURL=sqlite-change-log-catchup.js.map