@rocicorp/zero 1.11.0 → 1.11.1

package/out/zero-cache/src/services/replicator/drain-handler.js.map

@@ -1 +1 @@
-{"version":3,"file":"drain-handler.js","names":["#lc","#changeStreamer","#state","#options","#committedContext","#restoreGranted","#draining","#isActive","#switchover","#next","#attempt","#holdForRestore"],"sources":["../../../../../../zero-cache/src/services/replicator/drain-handler.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {resolver} from '@rocicorp/resolver';\nimport {readHead, type Sized, type Source} from '../../types/streams.ts';\nimport type {\n  ChangeStreamer,\n  Status,\n  SubscriberContext,\n} from '../change-streamer/change-streamer.ts';\nimport type {SubscribeDownstream} from '../change-streamer/subscribe.ts';\nimport type {RunningState} from '../running-state.ts';\n\n// Bounds the time to wait for a switchover subscription to be established\n// and to receive its first message.\nconst DEFAULT_SWITCHOVER_TIMEOUT_MS = 10_000;\nconst MIN_SWITCHOVER_RETRY_DELAY_MS = 1_000;\nconst MAX_SWITCHOVER_RETRY_DELAY_MS = 30_000;\n\n/**\n * Bounds the time to search for another replication-manager to switch over\n * to, after which the current subscription is closed so that the draining\n * replication-manager can shut down (e.g. when it is the only one). The time\n * during which a subscription is held by another replication-manager (i.e.\n * waiting for permission to restore) does not count towards it.\n *\n * This is well below the default termination grace period of container\n * runners (30 seconds in Kubernetes and ECS), so that the draining\n * replication-manager can shut down cleanly when there is no other.\n */\nconst DEFAULT_SWITCHOVER_SEARCH_TIMEOUT_MS = 20_000;\n\nexport type SwitchoverOptions = {\n  /**\n   * Called when a replication-manager with a newer replica version grants\n   * permission to restore its replica. This should initiate a graceful\n   * restart of the server (during which the current subscription continues\n   * to be served).\n   */\n  onRestoreGranted: () => void;\n\n  /**\n   * The replica file, from which the state of the last committed\n   * transaction is read for switchover subscriptions.\n   */\n  replicaFile: string;\n\n  /** For testing. */\n  timeoutMs?: number | undefined;\n\n  /** For testing. */\n  searchTimeoutMs?: number | undefined;\n};\n\nexport type Downstream = Source<Sized<SubscribeDownstream>>;\n\ntype AttemptOutcome =\n  | 'switched'\n  | 'restoring'\n  // The subscription was held for restore, but closed before a grant.\n  | 'released'\n  | 'retry';\n\n/**\n * Handles the switchover of a serving replica's subscription from a draining\n * replication-manager to another one, without interrupting the current\n * subscription.\n *\n * Upon {@link onChangeStreamerDrain}, the handler repeatedly (with backoff)\n * opens a new `switchover` subscription, at the watermark of the replica's\n * last committed transaction, to whichever replication-manager the\n * connection is routed to. The first message on the new subscription\n * determines the outcome:\n * * `status`: The subscription was accepted (i.e. the replica is compatible).\n *   The current subscription is cancelled (aborting any transaction in\n *   progress), and the new one is made available via\n *   {@link getNextConnection}. Note that the new subscription may start with\n *   transactions that were committed from the current subscription in the\n *   meantime, which the consumer must skip.\n * * `status` with `restore: queued`: The replication-manager has a newer\n *   replica version, and is holding the subscription until it grants\n *   permission to restore. The held subscription is monitored for the grant.\n * * `status` with `restore: granted`: Permission to restore is granted, and\n *   {@link SwitchoverOptions.onRestoreGranted} is called. The granted\n *   subscription is kept open until the process exits, so that the\n *   replication-manager can track the grant until the restore begins.\n * * Anything else (e.g. an error, or a `draining` replication-manager):\n *   The switchover is retried later.\n *\n * If no other replication-manager accepts or holds a switchover subscription\n * within {@link DEFAULT_SWITCHOVER_SEARCH_TIMEOUT_MS} (e.g. because the\n * draining one is the only one), the current subscription is closed so that\n * the draining replication-manager can shut down, and the consumer reconnects\n * on its own.\n *\n * The current subscription continues to be consumed throughout. A\n * switchover is scoped to the subscription from which it switches: If that\n * subscription ends first (e.g. the replication-manager shuts down), the\n * switchover is abandoned, including any held (but not granted)\n * subscription, as the consumer reconnects on its own.\n */\nexport class DrainHandler {\n  readonly #lc: LogContext;\n  readonly #changeStreamer: ChangeStreamer;\n  readonly #state: RunningState;\n  readonly #options: SwitchoverOptions;\n  readonly #committedContext: () => SubscriberContext;\n\n  // The subscription currently being switched from.\n  #draining: Downstream | undefined;\n  #restoreGranted = false;\n  #next: Downstream | undefined;\n\n  /**\n   * @param committedContext Returns the subscriber context as of the\n   *        replica's last committed transaction.\n   */\n  constructor(\n    lc: LogContext,\n    changeStreamer: ChangeStreamer,\n    state: RunningState,\n    options: SwitchoverOptions,\n    committedContext: () => SubscriberContext,\n  ) {\n    this.#lc = lc.withContext('component', 'drain-handler');\n    this.#changeStreamer = changeStreamer;\n    this.#state = state;\n    this.#options = options;\n    this.#committedContext = committedContext;\n  }\n\n  /**\n   * Called when the change-streamer of the `current` subscription is\n   * draining.\n   */\n  onChangeStreamerDrain(current: Downstream) {\n    if (\n      this.#restoreGranted ||\n      this.#draining === current ||\n      !this.#isActive(current)\n    ) {\n      return;\n    }\n    this.#draining = current;\n    this.#lc.info?.(\n      `replication-manager is draining. switching over to another`,\n    );\n    void this.#switchover(current);\n  }\n\n  get hasNextConnection(): boolean {\n    return this.#next !== undefined;\n  }\n\n  /**\n   * Returns (and clears) the subscription that replaces the (now cancelled)\n   * current one, if a switchover succeeded.\n   */\n  getNextConnection(): Downstream | undefined {\n    const next = this.#next;\n    this.#next = undefined;\n    return next;\n  }\n\n  #isActive(current: Downstream) {\n    return !current.signal.aborted && this.#state.shouldRun();\n  }\n\n  async #switchover(current: Downstream) {\n    const searchTimeoutMs =\n      this.#options.searchTimeoutMs ?? DEFAULT_SWITCHOVER_SEARCH_TIMEOUT_MS;\n    let failures = 0;\n    let searchStart = Date.now();\n    while (this.#isActive(current)) {\n      const outcome = await this.#attempt(current);\n      if (outcome === 'switched' || outcome === 'restoring') {\n        return;\n      }\n      if (outcome === 'released') {\n        // Time spent held by another replication-manager does not count\n        // towards the search.\n        failures = 0;\n        searchStart = Date.now();\n      }\n      const remaining = searchStart + searchTimeoutMs - Date.now();\n      if (remaining > 0) {\n        const delay = Math.min(\n          MIN_SWITCHOVER_RETRY_DELAY_MS * 2 ** failures++,\n          MAX_SWITCHOVER_RETRY_DELAY_MS,\n        );\n        await this.#state.sleep(Math.min(delay, remaining));\n        continue;\n      }\n      // The search has timed out.\n      if (this.#isActive(current)) {\n        this.#lc.warn?.(\n          `no other replication-manager found within ${searchTimeoutMs} ms. ` +\n            `disconnecting from the draining replication-manager`,\n        );\n        current.cancel();\n      }\n      return;\n    }\n  }\n\n  async #attempt(current: Downstream): Promise<AttemptOutcome> {\n    const lc = this.#lc;\n    const timeoutMs = this.#options.timeoutMs ?? DEFAULT_SWITCHOVER_TIMEOUT_MS;\n    let candidate: Downstream | undefined;\n    let timer: ReturnType<typeof setTimeout> | undefined;\n    let timedOut = false;\n    try {\n      const ctx = this.#committedContext();\n      lc.info?.(`attempting switchover at ${ctx.watermark}`);\n      const timeout = new Promise<never>((_, reject) => {\n        timer = setTimeout(() => {\n          timedOut = true;\n          reject(new Error(`timed out after ${timeoutMs} ms`));\n        }, timeoutMs);\n      });\n      const subscribing = this.#changeStreamer.subscribe({\n        ...ctx,\n        switchover: true,\n      });\n      // Clean up a connection that is established after a timeout.\n      void subscribing.then(\n        sub => (timedOut ? sub.cancel() : undefined),\n        () => {},\n      );\n      candidate = await Promise.race([subscribing, timeout]);\n      const {head, rest} = readHead(candidate);\n      const first = await Promise.race([head.next(), timeout]);\n      const msg = first.done ? undefined : first.value.data;\n\n      if (!this.#isActive(current)) {\n        // e.g. The current subscription failed in the meantime.\n      } else if (msg?.[0] === 'status') {\n        const {restore, draining} = msg[1];\n        if (restore) {\n          const granted = await this.#holdForRestore(current, rest, restore);\n          if (granted) {\n            return 'restoring';\n          }\n          lc.info?.(`held switchover subscription closed. will retry`);\n          return 'released';\n        }\n        if (!draining) {\n          lc.info?.(`switched over to a new subscription at ${ctx.watermark}`);\n          this.#next = rest;\n          current.cancel();\n          return 'switched';\n        }\n      }\n      lc.warn?.(`switchover rejected. continuing current subscription`, msg);\n    } catch (e) {\n      lc.warn?.(`switchover failed. continuing current subscription`, e);\n    } finally {\n      clearTimeout(timer);\n    }\n    candidate?.cancel();\n    return 'retry';\n  }\n\n  /**\n   * Keeps the held (or granted) subscription open, and initiates a graceful\n   * restart when permission to restore is granted. A held subscription that\n   * has yet to be granted is cancelled if the `current` subscription ends.\n   *\n   * @returns `true` once permission is granted, or `false` if the held\n   *          subscription closes before then.\n   */\n  #holdForRestore(\n    current: Downstream,\n    held: Downstream,\n    restore: NonNullable<Status['restore']>,\n  ): Promise<boolean> {\n    const lc = this.#lc;\n    const result = resolver<boolean>();\n    const unregister = this.#state.cancelOnStop(held);\n    const onCurrentEnded = () => {\n      if (!this.#restoreGranted) {\n        lc.info?.(`abandoning held switchover subscription`);\n        held.cancel();\n      }\n    };\n    current.signal.addEventListener('abort', onCurrentEnded, {once: true});\n\n    const onRestore = (r: NonNullable<Status['restore']>) => {\n      if (r.state === 'queued') {\n        lc.info?.(\n          `waiting for permission to restore a new replica (position ${r.position})`,\n        );\n      } else if (!this.#restoreGranted) {\n        lc.info?.(\n          `granted permission to restore a new replica. restarting gracefully`,\n        );\n        this.#restoreGranted = true;\n        this.#options.onRestoreGranted();\n        result.resolve(true);\n      }\n    };\n    onRestore(restore);\n    void (async () => {\n      try {\n        // Note: The loop must not be exited early, as that would close the\n        // connection.\n        for await (const {data: msg} of held) {\n          if (msg[0] === 'status' && msg[1].restore) {\n            onRestore(msg[1].restore);\n          }\n        }\n      } catch (e) {\n        lc.warn?.(`held switchover subscription failed`, e);\n      } finally {\n        unregister();\n        current.signal.removeEventListener('abort', onCurrentEnded);\n        result.resolve(false);\n      }\n    })();\n    return result.promise;\n  }\n}\n"],"mappings":";;;AAaA,IAAM,gCAAgC;AACtC,IAAM,gCAAgC;AACtC,IAAM,gCAAgC;;;;;;;;;;;;AAatC,IAAM,uCAAuC;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAuE7C,IAAa,eAAb,MAA0B;CACxB;CACA;CACA;CACA;CACA;CAGA;CACA,kBAAkB;CAClB;;;;;CAMA,YACE,IACA,gBACA,OACA,SACA,kBACA;EACA,KAAKA,MAAM,GAAG,YAAY,aAAa,eAAe;EACtD,KAAKC,kBAAkB;EACvB,KAAKC,SAAS;EACd,KAAKC,WAAW;EAChB,KAAKC,oBAAoB;CAC3B;;;;;CAMA,sBAAsB,SAAqB;EACzC,IACE,KAAKC,mBACL,KAAKC,cAAc,WACnB,CAAC,KAAKC,UAAU,OAAO,GAEvB;EAEF,KAAKD,YAAY;EACjB,KAAKN,IAAI,OACP,4DACF;EACA,KAAUQ,YAAY,OAAO;CAC/B;CAEA,IAAI,oBAA6B;EAC/B,OAAO,KAAKC,UAAU,KAAA;CACxB;;;;;CAMA,oBAA4C;EAC1C,MAAM,OAAO,KAAKA;EAClB,KAAKA,QAAQ,KAAA;EACb,OAAO;CACT;CAEA,UAAU,SAAqB;EAC7B,OAAO,CAAC,QAAQ,OAAO,WAAW,KAAKP,OAAO,UAAU;CAC1D;CAEA,MAAMM,YAAY,SAAqB;EACrC,MAAM,kBACJ,KAAKL,SAAS,mBAAmB;EACnC,IAAI,WAAW;EACf,IAAI,cAAc,KAAK,IAAI;EAC3B,OAAO,KAAKI,UAAU,OAAO,GAAG;GAC9B,MAAM,UAAU,MAAM,KAAKG,SAAS,OAAO;GAC3C,IAAI,YAAY,cAAc,YAAY,aACxC;GAEF,IAAI,YAAY,YAAY;IAG1B,WAAW;IACX,cAAc,KAAK,IAAI;GACzB;GACA,MAAM,YAAY,cAAc,kBAAkB,KAAK,IAAI;GAC3D,IAAI,YAAY,GAAG;IACjB,MAAM,QAAQ,KAAK,IACjB,gCAAgC,KAAK,YACrC,6BACF;IACA,MAAM,KAAKR,OAAO,MAAM,KAAK,IAAI,OAAO,SAAS,CAAC;IAClD;GACF;GAEA,IAAI,KAAKK,UAAU,OAAO,GAAG;IAC3B,KAAKP,IAAI,OACP,6CAA6C,gBAAgB,yDAE/D;IACA,QAAQ,OAAO;GACjB;GACA;EACF;CACF;CAEA,MAAMU,SAAS,SAA8C;EAC3D,MAAM,KAAK,KAAKV;EAChB,MAAM,YAAY,KAAKG,SAAS,aAAa;EAC7C,IAAI;EACJ,IAAI;EACJ,IAAI,WAAW;EACf,IAAI;GACF,MAAM,MAAM,KAAKC,kBAAkB;GACnC,GAAG,OAAO,4BAA4B,IAAI,WAAW;GACrD,MAAM,UAAU,IAAI,SAAgB,GAAG,WAAW;IAChD,QAAQ,iBAAiB;KACvB,WAAW;KACX,uBAAO,IAAI,MAAM,mBAAmB,UAAU,IAAI,CAAC;IACrD,GAAG,SAAS;GACd,CAAC;GACD,MAAM,cAAc,KAAKH,gBAAgB,UAAU;IACjD,GAAG;IACH,YAAY;GACd,CAAC;GAED,YAAiB,MACf,QAAQ,WAAW,IAAI,OAAO,IAAI,KAAA,SAC5B,CAAC,CACT;GACA,YAAY,MAAM,QAAQ,KAAK,CAAC,aAAa,OAAO,CAAC;GACrD,MAAM,EAAC,MAAM,SAAQ,SAAS,SAAS;GACvC,MAAM,QAAQ,MAAM,QAAQ,KAAK,CAAC,KAAK,KAAK,GAAG,OAAO,CAAC;GACvD,MAAM,MAAM,MAAM,OAAO,KAAA,IAAY,MAAM,MAAM;GAEjD,IAAI,CAAC,KAAKM,UAAU,OAAO,GAAG,CAE9B,OAAO,IAAI,MAAM,OAAO,UAAU;IAChC,MAAM,EAAC,SAAS,aAAY,IAAI;IAChC,IAAI,SAAS;KAEX,IAAI,MADkB,KAAKI,gBAAgB,SAAS,MAAM,OAAO,GAE/D,OAAO;KAET,GAAG,OAAO,iDAAiD;KAC3D,OAAO;IACT;IACA,IAAI,CAAC,UAAU;KACb,GAAG,OAAO,0CAA0C,IAAI,WAAW;KACnE,KAAKF,QAAQ;KACb,QAAQ,OAAO;KACf,OAAO;IACT;GACF;GACA,GAAG,OAAO,wDAAwD,GAAG;EACvE,SAAS,GAAG;GACV,GAAG,OAAO,sDAAsD,CAAC;EACnE,UAAU;GACR,aAAa,KAAK;EACpB;EACA,WAAW,OAAO;EAClB,OAAO;CACT;;;;;;;;;CAUA,gBACE,SACA,MACA,SACkB;EAClB,MAAM,KAAK,KAAKT;EAChB,MAAM,SAAS,SAAkB;EACjC,MAAM,aAAa,KAAKE,OAAO,aAAa,IAAI;EAChD,MAAM,uBAAuB;GAC3B,IAAI,CAAC,KAAKG,iBAAiB;IACzB,GAAG,OAAO,yCAAyC;IACnD,KAAK,OAAO;GACd;EACF;EACA,QAAQ,OAAO,iBAAiB,SAAS,gBAAgB,EAAC,MAAM,KAAI,CAAC;EAErE,MAAM,aAAa,MAAsC;GACvD,IAAI,EAAE,UAAU,UACd,GAAG,OACD,6DAA6D,EAAE,SAAS,EAC1E;QACK,IAAI,CAAC,KAAKA,iBAAiB;IAChC,GAAG,OACD,oEACF;IACA,KAAKA,kBAAkB;IACvB,KAAKF,SAAS,iBAAiB;IAC/B,OAAO,QAAQ,IAAI;GACrB;EACF;EACA,UAAU,OAAO;EACjB,CAAM,YAAY;GAChB,IAAI;IAGF,WAAW,MAAM,EAAC,MAAM,SAAQ,MAC9B,IAAI,IAAI,OAAO,YAAY,IAAI,GAAG,SAChC,UAAU,IAAI,GAAG,OAAO;GAG9B,SAAS,GAAG;IACV,GAAG,OAAO,uCAAuC,CAAC;GACpD,UAAU;IACR,WAAW;IACX,QAAQ,OAAO,oBAAoB,SAAS,cAAc;IAC1D,OAAO,QAAQ,KAAK;GACtB;EACF,GAAG;EACH,OAAO,OAAO;CAChB;AACF"}
+{"version":3,"file":"drain-handler.js","names":[],"sources":["../../../../../../zero-cache/src/services/replicator/drain-handler.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {resolver} from '@rocicorp/resolver';\nimport {readHead, type Sized, type Source} from '../../types/streams.ts';\nimport type {\n  ChangeStreamer,\n  Status,\n  SubscriberContext,\n} from '../change-streamer/change-streamer.ts';\nimport type {SubscribeDownstream} from '../change-streamer/subscribe.ts';\nimport type {RunningState} from '../running-state.ts';\n\n// Bounds the time to wait for a switchover subscription to be established\n// and to receive its first message.\nconst DEFAULT_SWITCHOVER_TIMEOUT_MS = 10_000;\nconst MIN_SWITCHOVER_RETRY_DELAY_MS = 1_000;\nconst MAX_SWITCHOVER_RETRY_DELAY_MS = 30_000;\n\n/**\n * Bounds the time to search for another replication-manager to switch over\n * to, after which the current subscription is closed so that the draining\n * replication-manager can shut down (e.g. when it is the only one). The time\n * during which a subscription is held by another replication-manager (i.e.\n * waiting for permission to restore) does not count towards it.\n *\n * This is well below the default termination grace period of container\n * runners (30 seconds in Kubernetes and ECS), so that the draining\n * replication-manager can shut down cleanly when there is no other.\n */\nconst DEFAULT_SWITCHOVER_SEARCH_TIMEOUT_MS = 20_000;\n\nexport type SwitchoverOptions = {\n  /**\n   * Called when a replication-manager with a newer replica version grants\n   * permission to restore its replica. This should initiate a graceful\n   * restart of the server (during which the current subscription continues\n   * to be served).\n   */\n  onRestoreGranted: () => void;\n\n  /**\n   * The replica file, from which the state of the last committed\n   * transaction is read for switchover subscriptions.\n   */\n  replicaFile: string;\n\n  /** For testing. */\n  timeoutMs?: number | undefined;\n\n  /** For testing. */\n  searchTimeoutMs?: number | undefined;\n};\n\nexport type Downstream = Source<Sized<SubscribeDownstream>>;\n\ntype AttemptOutcome =\n  | 'switched'\n  | 'restoring'\n  // The subscription was held for restore, but closed before a grant.\n  | 'released'\n  | 'retry';\n\n/**\n * Handles the switchover of a serving replica's subscription from a draining\n * replication-manager to another one, without interrupting the current\n * subscription.\n *\n * Upon {@link onChangeStreamerDrain}, the handler repeatedly (with backoff)\n * opens a new `switchover` subscription, at the watermark of the replica's\n * last committed transaction, to whichever replication-manager the\n * connection is routed to. The first message on the new subscription\n * determines the outcome:\n * * `status`: The subscription was accepted (i.e. the replica is compatible).\n *   The current subscription is cancelled (aborting any transaction in\n *   progress), and the new one is made available via\n *   {@link getNextConnection}. Note that the new subscription may start with\n *   transactions that were committed from the current subscription in the\n *   meantime, which the consumer must skip.\n * * `status` with `restore: queued`: The replication-manager has a newer\n *   replica version, and is holding the subscription until it grants\n *   permission to restore. The held subscription is monitored for the grant.\n * * `status` with `restore: granted`: Permission to restore is granted, and\n *   {@link SwitchoverOptions.onRestoreGranted} is called. The granted\n *   subscription is kept open until the process exits, so that the\n *   replication-manager can track the grant until the restore begins.\n * * Anything else (e.g. an error, or a `draining` replication-manager):\n *   The switchover is retried later.\n *\n * If no other replication-manager accepts or holds a switchover subscription\n * within {@link DEFAULT_SWITCHOVER_SEARCH_TIMEOUT_MS} (e.g. because the\n * draining one is the only one), the current subscription is closed so that\n * the draining replication-manager can shut down, and the consumer reconnects\n * on its own.\n *\n * The current subscription continues to be consumed throughout. A\n * switchover is scoped to the subscription from which it switches: If that\n * subscription ends first (e.g. the replication-manager shuts down), the\n * switchover is abandoned, including any held (but not granted)\n * subscription, as the consumer reconnects on its own.\n */\nexport class DrainHandler {\n  readonly #lc: LogContext;\n  readonly #changeStreamer: ChangeStreamer;\n  readonly #state: RunningState;\n  readonly #options: SwitchoverOptions;\n  readonly #committedContext: () => SubscriberContext;\n\n  // The subscription currently being switched from.\n  #draining: Downstream | undefined;\n  #restoreGranted = false;\n  #next: Downstream | undefined;\n\n  /**\n   * @param committedContext Returns the subscriber context as of the\n   *        replica's last committed transaction.\n   */\n  constructor(\n    lc: LogContext,\n    changeStreamer: ChangeStreamer,\n    state: RunningState,\n    options: SwitchoverOptions,\n    committedContext: () => SubscriberContext,\n  ) {\n    this.#lc = lc.withContext('component', 'drain-handler');\n    this.#changeStreamer = changeStreamer;\n    this.#state = state;\n    this.#options = options;\n    this.#committedContext = committedContext;\n  }\n\n  /**\n   * Called when the change-streamer of the `current` subscription is\n   * draining.\n   */\n  onChangeStreamerDrain(current: Downstream) {\n    if (\n      this.#restoreGranted ||\n      this.#draining === current ||\n      !this.#isActive(current)\n    ) {\n      return;\n    }\n    this.#draining = current;\n    this.#lc.info?.(\n      `replication-manager is draining. switching over to another`,\n    );\n    void this.#switchover(current);\n  }\n\n  get hasNextConnection(): boolean {\n    return this.#next !== undefined;\n  }\n\n  /**\n   * Returns (and clears) the subscription that replaces the (now cancelled)\n   * current one, if a switchover succeeded.\n   */\n  getNextConnection(): Downstream | undefined {\n    const next = this.#next;\n    this.#next = undefined;\n    return next;\n  }\n\n  #isActive(current: Downstream) {\n    return !current.signal.aborted && this.#state.shouldRun();\n  }\n\n  async #switchover(current: Downstream) {\n    const searchTimeoutMs =\n      this.#options.searchTimeoutMs ?? DEFAULT_SWITCHOVER_SEARCH_TIMEOUT_MS;\n    let failures = 0;\n    let searchStart = Date.now();\n    while (this.#isActive(current)) {\n      const outcome = await this.#attempt(current);\n      if (outcome === 'switched' || outcome === 'restoring') {\n        return;\n      }\n      if (outcome === 'released') {\n        // Time spent held by another replication-manager does not count\n        // towards the search.\n        failures = 0;\n        searchStart = Date.now();\n      }\n      const remaining = searchStart + searchTimeoutMs - Date.now();\n      if (remaining > 0) {\n        const delay = Math.min(\n          MIN_SWITCHOVER_RETRY_DELAY_MS * 2 ** failures++,\n          MAX_SWITCHOVER_RETRY_DELAY_MS,\n        );\n        await this.#state.sleep(Math.min(delay, remaining));\n        continue;\n      }\n      // The search has timed out.\n      if (this.#isActive(current)) {\n        this.#lc.warn?.(\n          `no other replication-manager found within ${searchTimeoutMs} ms. ` +\n            `disconnecting from the draining replication-manager`,\n        );\n        current.cancel();\n      }\n      return;\n    }\n  }\n\n  async #attempt(current: Downstream): Promise<AttemptOutcome> {\n    const lc = this.#lc;\n    const timeoutMs = this.#options.timeoutMs ?? DEFAULT_SWITCHOVER_TIMEOUT_MS;\n    let candidate: Downstream | undefined;\n    let timer: ReturnType<typeof setTimeout> | undefined;\n    let timedOut = false;\n    try {\n      const ctx = this.#committedContext();\n      lc.info?.(`attempting switchover at ${ctx.watermark}`);\n      const timeout = new Promise<never>((_, reject) => {\n        timer = setTimeout(() => {\n          timedOut = true;\n          reject(new Error(`timed out after ${timeoutMs} ms`));\n        }, timeoutMs);\n      });\n      const subscribing = this.#changeStreamer.subscribe({\n        ...ctx,\n        switchover: true,\n      });\n      // Clean up a connection that is established after a timeout.\n      void subscribing.then(\n        sub => (timedOut ? sub.cancel() : undefined),\n        () => {},\n      );\n      candidate = await Promise.race([subscribing, timeout]);\n      const {head, rest} = readHead(candidate);\n      const first = await Promise.race([head.next(), timeout]);\n      const msg = first.done ? undefined : first.value.data;\n\n      if (!this.#isActive(current)) {\n        // e.g. The current subscription failed in the meantime.\n      } else if (msg?.[0] === 'status') {\n        const {restore, draining} = msg[1];\n        if (restore) {\n          const granted = await this.#holdForRestore(current, rest, restore);\n          if (granted) {\n            return 'restoring';\n          }\n          lc.info?.(`held switchover subscription closed. will retry`);\n          return 'released';\n        }\n        if (!draining) {\n          lc.info?.(`switched over to a new subscription at ${ctx.watermark}`);\n          this.#next = rest;\n          current.cancel();\n          return 'switched';\n        }\n      }\n      lc.warn?.(`switchover rejected. continuing current subscription`, msg);\n    } catch (e) {\n      lc.warn?.(`switchover failed. continuing current subscription`, e);\n    } finally {\n      clearTimeout(timer);\n    }\n    candidate?.cancel();\n    return 'retry';\n  }\n\n  /**\n   * Keeps the held (or granted) subscription open, and initiates a graceful\n   * restart when permission to restore is granted. A held subscription that\n   * has yet to be granted is cancelled if the `current` subscription ends.\n   *\n   * @returns `true` once permission is granted, or `false` if the held\n   *          subscription closes before then.\n   */\n  #holdForRestore(\n    current: Downstream,\n    held: Downstream,\n    restore: NonNullable<Status['restore']>,\n  ): Promise<boolean> {\n    const lc = this.#lc;\n    const result = resolver<boolean>();\n    const unregister = this.#state.cancelOnStop(held);\n    const onCurrentEnded = () => {\n      if (!this.#restoreGranted) {\n        lc.info?.(`abandoning held switchover subscription`);\n        held.cancel();\n      }\n    };\n    current.signal.addEventListener('abort', onCurrentEnded, {once: true});\n\n    const onRestore = (r: NonNullable<Status['restore']>) => {\n      if (r.state === 'queued') {\n        lc.info?.(\n          `waiting for permission to restore a new replica (position ${r.position})`,\n        );\n      } else if (!this.#restoreGranted) {\n        lc.info?.(\n          `granted permission to restore a new replica. restarting gracefully`,\n        );\n        this.#restoreGranted = true;\n        this.#options.onRestoreGranted();\n        result.resolve(true);\n      }\n    };\n    onRestore(restore);\n    void (async () => {\n      try {\n        // Note: The loop must not be exited early, as that would close the\n        // connection.\n        for await (const {data: msg} of held) {\n          if (msg[0] === 'status' && msg[1].restore) {\n            onRestore(msg[1].restore);\n          }\n        }\n      } catch (e) {\n        lc.warn?.(`held switchover subscription failed`, e);\n      } finally {\n        unregister();\n        current.signal.removeEventListener('abort', onCurrentEnded);\n        result.resolve(false);\n      }\n    })();\n    return result.promise;\n  }\n}\n"],"mappings":";;;AAaA,IAAM,gCAAgC;AACtC,IAAM,gCAAgC;AACtC,IAAM,gCAAgC;;;;;;;;;;;;AAatC,IAAM,uCAAuC;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAuE7C,IAAa,eAAb,MAA0B;CACxB;CACA;CACA;CACA;CACA;CAGA;CACA,kBAAkB;CAClB;;;;;CAMA,YACE,IACA,gBACA,OACA,SACA,kBACA;EACA,KAAK,MAAM,GAAG,YAAY,aAAa,eAAe;EACtD,KAAK,kBAAkB;EACvB,KAAK,SAAS;EACd,KAAK,WAAW;EAChB,KAAK,oBAAoB;CAC3B;;;;;CAMA,sBAAsB,SAAqB;EACzC,IACE,KAAK,mBACL,KAAK,cAAc,WACnB,CAAC,KAAK,UAAU,OAAO,GAEvB;EAEF,KAAK,YAAY;EACjB,KAAK,IAAI,OACP,4DACF;EACA,KAAU,YAAY,OAAO;CAC/B;CAEA,IAAI,oBAA6B;EAC/B,OAAO,KAAK,UAAU,KAAA;CACxB;;;;;CAMA,oBAA4C;EAC1C,MAAM,OAAO,KAAK;EAClB,KAAK,QAAQ,KAAA;EACb,OAAO;CACT;CAEA,UAAU,SAAqB;EAC7B,OAAO,CAAC,QAAQ,OAAO,WAAW,KAAK,OAAO,UAAU;CAC1D;CAEA,MAAM,YAAY,SAAqB;EACrC,MAAM,kBACJ,KAAK,SAAS,mBAAmB;EACnC,IAAI,WAAW;EACf,IAAI,cAAc,KAAK,IAAI;EAC3B,OAAO,KAAK,UAAU,OAAO,GAAG;GAC9B,MAAM,UAAU,MAAM,KAAK,SAAS,OAAO;GAC3C,IAAI,YAAY,cAAc,YAAY,aACxC;GAEF,IAAI,YAAY,YAAY;IAG1B,WAAW;IACX,cAAc,KAAK,IAAI;GACzB;GACA,MAAM,YAAY,cAAc,kBAAkB,KAAK,IAAI;GAC3D,IAAI,YAAY,GAAG;IACjB,MAAM,QAAQ,KAAK,IACjB,gCAAgC,KAAK,YACrC,6BACF;IACA,MAAM,KAAK,OAAO,MAAM,KAAK,IAAI,OAAO,SAAS,CAAC;IAClD;GACF;GAEA,IAAI,KAAK,UAAU,OAAO,GAAG;IAC3B,KAAK,IAAI,OACP,6CAA6C,gBAAgB,yDAE/D;IACA,QAAQ,OAAO;GACjB;GACA;EACF;CACF;CAEA,MAAM,SAAS,SAA8C;EAC3D,MAAM,KAAK,KAAK;EAChB,MAAM,YAAY,KAAK,SAAS,aAAa;EAC7C,IAAI;EACJ,IAAI;EACJ,IAAI,WAAW;EACf,IAAI;GACF,MAAM,MAAM,KAAK,kBAAkB;GACnC,GAAG,OAAO,4BAA4B,IAAI,WAAW;GACrD,MAAM,UAAU,IAAI,SAAgB,GAAG,WAAW;IAChD,QAAQ,iBAAiB;KACvB,WAAW;KACX,uBAAO,IAAI,MAAM,mBAAmB,UAAU,IAAI,CAAC;IACrD,GAAG,SAAS;GACd,CAAC;GACD,MAAM,cAAc,KAAK,gBAAgB,UAAU;IACjD,GAAG;IACH,YAAY;GACd,CAAC;GAED,YAAiB,MACf,QAAQ,WAAW,IAAI,OAAO,IAAI,KAAA,SAC5B,CAAC,CACT;GACA,YAAY,MAAM,QAAQ,KAAK,CAAC,aAAa,OAAO,CAAC;GACrD,MAAM,EAAC,MAAM,SAAQ,SAAS,SAAS;GACvC,MAAM,QAAQ,MAAM,QAAQ,KAAK,CAAC,KAAK,KAAK,GAAG,OAAO,CAAC;GACvD,MAAM,MAAM,MAAM,OAAO,KAAA,IAAY,MAAM,MAAM;GAEjD,IAAI,CAAC,KAAK,UAAU,OAAO,GAAG,CAE9B,OAAO,IAAI,MAAM,OAAO,UAAU;IAChC,MAAM,EAAC,SAAS,aAAY,IAAI;IAChC,IAAI,SAAS;KAEX,IAAI,MADkB,KAAK,gBAAgB,SAAS,MAAM,OAAO,GAE/D,OAAO;KAET,GAAG,OAAO,iDAAiD;KAC3D,OAAO;IACT;IACA,IAAI,CAAC,UAAU;KACb,GAAG,OAAO,0CAA0C,IAAI,WAAW;KACnE,KAAK,QAAQ;KACb,QAAQ,OAAO;KACf,OAAO;IACT;GACF;GACA,GAAG,OAAO,wDAAwD,GAAG;EACvE,SAAS,GAAG;GACV,GAAG,OAAO,sDAAsD,CAAC;EACnE,UAAU;GACR,aAAa,KAAK;EACpB;EACA,WAAW,OAAO;EAClB,OAAO;CACT;;;;;;;;;CAUA,gBACE,SACA,MACA,SACkB;EAClB,MAAM,KAAK,KAAK;EAChB,MAAM,SAAS,SAAkB;EACjC,MAAM,aAAa,KAAK,OAAO,aAAa,IAAI;EAChD,MAAM,uBAAuB;GAC3B,IAAI,CAAC,KAAK,iBAAiB;IACzB,GAAG,OAAO,yCAAyC;IACnD,KAAK,OAAO;GACd;EACF;EACA,QAAQ,OAAO,iBAAiB,SAAS,gBAAgB,EAAC,MAAM,KAAI,CAAC;EAErE,MAAM,aAAa,MAAsC;GACvD,IAAI,EAAE,UAAU,UACd,GAAG,OACD,6DAA6D,EAAE,SAAS,EAC1E;QACK,IAAI,CAAC,KAAK,iBAAiB;IAChC,GAAG,OACD,oEACF;IACA,KAAK,kBAAkB;IACvB,KAAK,SAAS,iBAAiB;IAC/B,OAAO,QAAQ,IAAI;GACrB;EACF;EACA,UAAU,OAAO;EACjB,CAAM,YAAY;GAChB,IAAI;IAGF,WAAW,MAAM,EAAC,MAAM,SAAQ,MAC9B,IAAI,IAAI,OAAO,YAAY,IAAI,EAAE,CAAC,SAChC,UAAU,IAAI,EAAE,CAAC,OAAO;GAG9B,SAAS,GAAG;IACV,GAAG,OAAO,uCAAuC,CAAC;GACpD,UAAU;IACR,WAAW;IACX,QAAQ,OAAO,oBAAoB,SAAS,cAAc;IAC1D,OAAO,QAAQ,KAAK;GACtB;EACF,EAAA,CAAG;EACH,OAAO,OAAO;CAChB;AACF"}

package/out/zero-cache/src/services/replicator/incremental-sync.js

@@ -13,7 +13,7 @@ import { DrainHandler } from "./drain-handler.js";
 import { ReplicationReportRecorder } from "./reporter/recorder.js";
 //#region ../zero-cache/src/services/replicator/incremental-sync.ts
 var MAX_WORKER_BATCH_MESSAGES = 256;
-var MAX_WORKER_BATCH_SIZE = 1024 * 1024;
+var MAX_WORKER_BATCH_SIZE = 1048576;
 var MAX_OLDER_REPLICA_VERSION_REJECTIONS = 20;
 /**
 * The {@link IncrementalSyncer} manages a logical replication stream from upstream,

package/out/zero-cache/src/services/replicator/incremental-sync.js.map

@@ -1 +1 @@
-{"version":3,"file":"incremental-sync.js","names":["#lc","#taskID","#id","#changeStreamer","#worker","#mode","#statusPublisher","#notifier","#reporter","#drainHandler","#state","#replicationEvents","#initialConnection","#committedContext","#subscriberContext","#handleResult","#olderReplicaVersionRejections"],"sources":["../../../../../../zero-cache/src/services/replicator/incremental-sync.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {AbortError} from '../../../../shared/src/abort-error.ts';\nimport type {Enum} from '../../../../shared/src/enum.ts';\nimport {Database} from '../../../../zqlite/src/db.ts';\nimport {mapPostgresToLiteIndex} from '../../db/pg-to-lite.ts';\nimport {StatementRunner} from '../../db/statements.ts';\nimport {getOrCreateCounter} from '../../observability/metrics.ts';\nimport type {Source} from '../../types/streams.ts';\nimport type {DownloadStatus} from '../change-source/protocol/current.ts';\nimport type {ChangeStreamData} from '../change-source/protocol/current/downstream.ts';\nimport type {ReservationFollowup} from '../change-streamer/change-streamer-http.ts';\nimport {\n  errorTypeToReadableName,\n  PROTOCOL_VERSION,\n  type ChangeStreamer,\n  type SubscriberContext,\n} from '../change-streamer/change-streamer.ts';\nimport type * as ErrorType from '../change-streamer/error-type-enum.ts';\nimport {WrongReplicaVersion} from '../change-streamer/error-type-enum.ts';\nimport type {SubscribeDownstream} from '../change-streamer/subscribe.ts';\nimport {RunningState} from '../running-state.ts';\nimport type {CommitResult} from './change-processor.ts';\nimport {\n  DrainHandler,\n  type Downstream,\n  type SwitchoverOptions,\n} from './drain-handler.ts';\nimport {Notifier} from './notifier.ts';\nimport {\n  IndexingProgress,\n  type ReplicationStatusPublisher,\n} from './replication-status.ts';\nimport type {ReplicaState, ReplicatorMode} from './replicator.ts';\nimport {\n  ReplicationReportRecorder,\n  type ReplicationLagReadinessOptions,\n} from './reporter/recorder.ts';\nimport type {ReplicationReport} from './reporter/report-schema.ts';\nimport {readReplicaBackfills} from './schema/backfilling.ts';\nimport {getSubscriptionState} from './schema/replication-state.ts';\nimport type {WriteWorkerClient} from './write-worker-client.ts';\n\ntype ErrorType = Enum<typeof ErrorType>;\n\nconst MAX_WORKER_BATCH_MESSAGES = 256;\nconst MAX_WORKER_BATCH_SIZE = 1024 * 1024;\n\n// The number of times to reconnect after being rejected by a\n// replication-manager with an older replica version (e.g. one that has yet\n// to be replaced by a resync), before accepting the rejection and restarting\n// (e.g. if the upstream was legitimately replaced by one with a lower replica\n// version).\nconst MAX_OLDER_REPLICA_VERSION_REJECTIONS = 20;\n\n/**\n * The {@link IncrementalSyncer} manages a logical replication stream from upstream,\n * handling application lifecycle events (start, stop) and retrying the\n * connection with exponential backoff. The actual handling of the logical\n * replication messages is done by the {@link ChangeProcessor}, which runs\n * in a worker thread via the {@link WriteWorkerClient}.\n */\nexport class IncrementalSyncer {\n  readonly #lc: LogContext;\n  readonly #taskID: string;\n  readonly #id: string;\n  readonly #changeStreamer: ChangeStreamer;\n  readonly #worker: WriteWorkerClient;\n  readonly #mode: ReplicatorMode;\n  readonly #statusPublisher: ReplicationStatusPublisher | null;\n  readonly #notifier: Notifier;\n  readonly #reporter: ReplicationReportRecorder;\n  // Consumed (and cleared) by the first subscribe() in run()'s loop; every\n  // subsequent (re)connection goes through #changeStreamer.subscribe().\n  #initialConnection: ReservationFollowup | undefined;\n\n  // Only set for serving replicas that switch over between\n  // replication-managers.\n  readonly #drainHandler: DrainHandler | undefined;\n  #olderReplicaVersionRejections = 0;\n\n  readonly #state = new RunningState('IncrementalSyncer');\n\n  readonly #replicationEvents = getOrCreateCounter(\n    'replication',\n    'events',\n    'Number of replication events processed',\n  );\n\n  constructor(\n    lc: LogContext,\n    taskID: string,\n    id: string,\n    changeStreamer: ChangeStreamer,\n    worker: WriteWorkerClient,\n    mode: ReplicatorMode,\n    statusPublisher: ReplicationStatusPublisher | null,\n    replicationLagReadinessOptions: ReplicationLagReadinessOptions = {},\n    initialConnection?: ReservationFollowup,\n    switchover?: SwitchoverOptions,\n  ) {\n    this.#lc = lc;\n    this.#taskID = taskID;\n    this.#id = id;\n    this.#changeStreamer = changeStreamer;\n    this.#worker = worker;\n    this.#mode = mode;\n    this.#statusPublisher = statusPublisher;\n    this.#notifier = new Notifier();\n    this.#reporter = new ReplicationReportRecorder(\n      lc,\n      replicationLagReadinessOptions,\n    );\n    this.#initialConnection = initialConnection;\n    // Only serving replicas switch over between replication-managers. In\n    // particular, the in-RM backup-replicator never does.\n    this.#drainHandler =\n      mode === 'serving' && switchover\n        ? new DrainHandler(lc, changeStreamer, this.#state, switchover, () =>\n            this.#committedContext(switchover.replicaFile),\n          )\n        : undefined;\n  }\n\n  ready(): Promise<void> {\n    return this.#reporter.ready();\n  }\n\n  async run() {\n    const lc = this.#lc;\n    let workerError: Error | undefined;\n    this.#worker.onError(err => {\n      workerError ??= err;\n      this.#state.stop(lc, err);\n    });\n    lc.info?.(`Starting IncrementalSyncer`);\n    const {watermark: initialWatermark} =\n      await this.#worker.getSubscriptionState();\n\n    // Notify any waiting subscribers that the replica is ready to be read.\n    // This initial notification intentionally omits replicaReadyTimeMs because\n    // it represents already-current state, not newly-unserved work.\n    void this.#notifier.notifySubscribers({\n      state: 'version-ready',\n      watermark: initialWatermark,\n    });\n\n    while (this.#state.shouldRun()) {\n      const {replicaVersion, watermark, backfills} =\n        await this.#worker.getSubscriptionState();\n\n      let downstream: Downstream | undefined;\n      let unregister = () => {};\n      let err: unknown | undefined;\n\n      try {\n        const ctx = this.#subscriberContext(\n          replicaVersion,\n          watermark,\n          backfills,\n        );\n        // A successful switchover has already established the subscription\n        // on a new connection.\n        const switched = this.#drainHandler?.getNextConnection();\n        // The first connection reuses the still-open reservation connection\n        // (if any) from the backup restore, so the replication-manager that\n        // reserved the change log is the one that serves the subscription.\n        // Every reconnect after that opens a fresh connection.\n        const initialConnection = this.#initialConnection;\n        this.#initialConnection = undefined;\n        downstream =\n          switched ??\n          (initialConnection\n            ? await initialConnection.subscribe(ctx)\n            : await this.#changeStreamer.subscribe(ctx));\n        unregister = this.#state.cancelOnStop(downstream);\n        this.#statusPublisher?.publish(\n          lc,\n          'Replicating',\n          `Replicating from ${watermark}`,\n        );\n\n        let backfill:\n          | {status: DownloadStatus; table: string; columns: string[]}\n          | undefined;\n        let writeBatch: ChangeStreamData[] = [];\n        let writeBatchSize = 0;\n        let inTransaction = false;\n        // Indexes created in the current run of consecutive index creations.\n        let indexing: IndexingProgress | undefined;\n\n        const publishBackfillStatus = (table: string) =>\n          this.#statusPublisher?.publish(\n            lc,\n            'Replicating',\n            `Backfilling ${table} table`,\n            3000,\n            () =>\n              backfill\n                ? {\n                    downloadStatus: [\n                      {\n                        ...backfill.status,\n                        table: backfill.table,\n                        columns: backfill.columns,\n                      },\n                    ],\n                  }\n                : {},\n          );\n\n        const flushWrites = async () => {\n          if (writeBatch.length === 0) {\n            return;\n          }\n          const batch = writeBatch;\n          writeBatch = [];\n          writeBatchSize = 0;\n\n          const result = await this.#worker.processMessages(batch);\n          this.#handleResult(lc, result);\n          if (result?.completedBackfill) {\n            backfill = undefined;\n          }\n        };\n\n        // Whether the current transaction is being skipped.\n        let skipping = false;\n\n        // A subscription established by a switchover starts at the\n        // watermark at which it was requested, and may thus include\n        // transactions that were since committed from the previous\n        // subscription.\n        const shouldSkip = (message: SubscribeDownstream) => {\n          const type = message[0];\n          if (type === 'begin' && message[2].commitWatermark <= watermark) {\n            lc.debug?.(\n              `skipping committed transaction ${message[2].commitWatermark}`,\n            );\n            skipping = true;\n          }\n          if (\n            skipping &&\n            (type === 'begin' ||\n              type === 'data' ||\n              type === 'commit' ||\n              type === 'rollback')\n          ) {\n            skipping = type !== 'commit' && type !== 'rollback';\n            return true;\n          }\n          return false;\n        };\n\n        for await (const {data: message, size} of downstream) {\n          if (shouldSkip(message)) {\n            continue;\n          }\n          // Only reset backoff after receiving an actual message.\n          this.#state.resetBackoff();\n\n          this.#replicationEvents.add(1);\n          switch (message[0]) {\n            case 'reserved':\n              lc.warn?.(`unexpected snapshot reservation message received`, {\n                message: message[0],\n              });\n              break;\n            case 'status': {\n              const {lagReport, draining} = message[1];\n              this.#olderReplicaVersionRejections = 0;\n              if (draining) {\n                this.#drainHandler?.onChangeStreamerDrain(downstream);\n              }\n              if (lagReport) {\n                const report: ReplicationReport = {\n                  nextSendTimeMs: lagReport.nextSendTimeMs,\n                };\n                if (lagReport.lastTimings) {\n                  report.lastTimings = {\n                    ...lagReport.lastTimings,\n                    replicateTimeMs: Date.now(),\n                  };\n                }\n                this.#reporter.record(report);\n              }\n              break;\n            }\n            case 'error': {\n              const {\n                type,\n                message: msg,\n                replicaVersion: rmVersion,\n              } = message[1];\n              if (\n                type === WrongReplicaVersion &&\n                rmVersion !== undefined &&\n                rmVersion < replicaVersion &&\n                this.#olderReplicaVersionRejections++ <\n                  MAX_OLDER_REPLICA_VERSION_REJECTIONS\n              ) {\n                // The replication-manager is behind this replica (e.g. it\n                // has yet to be replaced by a resync). Rather than restoring\n                // its older replica, reconnect (with backoff), which may\n                // reach a different replication-manager.\n                lc.warn?.(\n                  `replication-manager is at an older replica version ` +\n                    `(${rmVersion}) than ${replicaVersion}. reconnecting`,\n                );\n                break;\n              }\n              // Signal from the replication-manager that the view-syncer must\n              // shut down and restore a new backup from litestream.\n              this.stop(\n                lc,\n                // Note: The AbortError indicates a clean / intentional shutdown.\n                new AbortError(\n                  `${errorTypeToReadableName(type as ErrorType)}: ${msg}`,\n                ),\n              );\n              break;\n            }\n            default: {\n              const msg = message[1];\n              if (msg.tag === 'backfill' && msg.status) {\n                const {status, relation} = msg;\n                const first = !backfill;\n                backfill = {\n                  status, // Update the current status\n                  table: relation.name,\n                  columns: [...relation.rowKey.columns, ...msg.columns],\n                };\n                if (first) {\n                  // Start publishing the status every 3 seconds.\n                  publishBackfillStatus(relation.name);\n                }\n              }\n\n              if (msg.tag === 'create-index' && this.#statusPublisher) {\n                // Creating an index on an existing table can take a long\n                // time. Flush preceding changes so that the index creation\n                // is processed (and timed) on its own.\n                await flushWrites();\n                const index = mapPostgresToLiteIndex(msg.spec);\n                indexing ??= new IndexingProgress();\n                indexing.start(index);\n                this.#statusPublisher.publish(\n                  lc,\n                  'Replicating',\n                  `Creating index ${index.name} on ${index.tableName}`,\n                  3000,\n                  indexing.state,\n                );\n                // Exclude the time spent publishing the status.\n                indexing.restartTimer();\n                writeBatch.push(message as ChangeStreamData);\n                writeBatchSize += size;\n                await flushWrites();\n                const elapsed = indexing.finish();\n                lc.info?.(\n                  `Created index ${index.name} (${elapsed.toFixed(3)} ms)`,\n                );\n                this.#statusPublisher.publish(\n                  lc,\n                  'Replicating',\n                  `Created index ${index.name} on ${index.tableName}`,\n                  0,\n                  indexing.state,\n                );\n                if (backfill) {\n                  // Resume reporting the progress of the ongoing backfill.\n                  publishBackfillStatus(backfill.table);\n                }\n                break;\n              }\n              indexing = undefined;\n\n              const type = message[0];\n              const invalidTransactionSequence =\n                type === 'begin' ? inTransaction : !inTransaction;\n              if (type === 'begin') {\n                inTransaction = true;\n              } else if (type === 'commit' || type === 'rollback') {\n                inTransaction = false;\n              }\n\n              writeBatch.push(message as ChangeStreamData);\n              writeBatchSize += size;\n              if (\n                // Waiting here ensures that consuming the commit, which ACKs\n                // it upstream, only happens after the SQLite commit finishes.\n                type === 'commit' ||\n                type === 'rollback' ||\n                // Promptly surface malformed transaction sequences instead\n                // of leaving a partial batch waiting for another message.\n                invalidTransactionSequence ||\n                writeBatch.length >= MAX_WORKER_BATCH_MESSAGES ||\n                writeBatchSize >= MAX_WORKER_BATCH_SIZE\n              ) {\n                await flushWrites();\n              }\n              break;\n            }\n          }\n        }\n        this.#worker.abort();\n      } catch (e) {\n        err = e;\n        this.#worker.abort();\n      } finally {\n        downstream?.cancel();\n        unregister();\n        this.#statusPublisher?.stop();\n      }\n      if (err === undefined && this.#drainHandler?.hasNextConnection) {\n        continue; // Switched over to a new subscription; no backoff.\n      }\n      this.#drainHandler?.getNextConnection()?.cancel();\n      await this.#state.backoff(lc, err);\n    }\n    lc.info?.('IncrementalSyncer stopped');\n    if (workerError) {\n      throw workerError;\n    }\n  }\n\n  #handleResult(lc: LogContext, result: CommitResult | null) {\n    if (!result) {\n      return;\n    }\n    if (result.completedBackfill) {\n      // Publish the final status\n      const status = result.completedBackfill;\n      this.#statusPublisher?.publish(\n        lc,\n        'Replicating',\n        `Backfilled ${status.table} table`,\n        0,\n        () => ({downloadStatus: [status]}),\n      );\n    } else if (result.schemaUpdated) {\n      this.#statusPublisher?.publish(lc, 'Replicating', 'Schema updated');\n    }\n    if (result.watermark && result.changeLogUpdated) {\n      void this.#notifier.notifySubscribers({\n        state: 'version-ready',\n        watermark: result.watermark,\n        replicaReadyTimeMs: Date.now(),\n        upstreamCommitTimeMs: result.upstreamCommitTimeMs,\n      });\n    }\n  }\n\n  subscribe(): Source<ReplicaState> {\n    return this.#notifier.subscribe();\n  }\n\n  #subscriberContext(\n    replicaVersion: string,\n    watermark: string,\n    backfills: SubscriberContext['backfills'],\n  ): SubscriberContext {\n    return {\n      protocolVersion: PROTOCOL_VERSION,\n      taskID: this.#taskID,\n      id: this.#id,\n      mode: this.#mode,\n      watermark,\n      replicaVersion,\n      backfills,\n      ...(this.#drainHandler ? {drainable: true} : {}),\n    };\n  }\n\n  /**\n   * Returns the subscriber context as of the last committed transaction,\n   * read from the replica file (i.e. not from the write worker, whose\n   * connection may be in the middle of a transaction).\n   */\n  #committedContext(replicaFile: string): SubscriberContext {\n    const db = new Database(this.#lc, replicaFile, {readonly: true});\n    try {\n      const {replicaVersion, watermark, backfills} = db.transaction(() => ({\n        ...getSubscriptionState(new StatementRunner(db)),\n        backfills: readReplicaBackfills(db),\n      }));\n      return this.#subscriberContext(replicaVersion, watermark, backfills);\n    } finally {\n      db.close();\n    }\n  }\n\n  stop(lc: LogContext, err?: unknown) {\n    this.#state.stop(lc, err);\n    // Abort any polling loop that the write worker may be in.\n    this.#worker.abort();\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;AA4CA,IAAM,4BAA4B;AAClC,IAAM,wBAAwB,OAAO;AAOrC,IAAM,uCAAuC;;;;;;;;AAS7C,IAAa,oBAAb,MAA+B;CAC7B;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CAGA;CAIA;CACA,iCAAiC;CAEjC,SAAkB,IAAI,aAAa,mBAAmB;CAEtD,qBAA8B,mBAC5B,eACA,UACA,wCACF;CAEA,YACE,IACA,QACA,IACA,gBACA,QACA,MACA,iBACA,iCAAiE,CAAC,GAClE,mBACA,YACA;EACA,KAAKA,MAAM;EACX,KAAKC,UAAU;EACf,KAAKC,MAAM;EACX,KAAKC,kBAAkB;EACvB,KAAKC,UAAU;EACf,KAAKC,QAAQ;EACb,KAAKC,mBAAmB;EACxB,KAAKC,YAAY,IAAI,SAAS;EAC9B,KAAKC,YAAY,IAAI,0BACnB,IACA,8BACF;EACA,KAAKI,qBAAqB;EAG1B,KAAKH,gBACH,SAAS,aAAa,aAClB,IAAI,aAAa,IAAI,gBAAgB,KAAKC,QAAQ,kBAChD,KAAKG,kBAAkB,WAAW,WAAW,CAC/C,IACA,KAAA;CACR;CAEA,QAAuB;EACrB,OAAO,KAAKL,UAAU,MAAM;CAC9B;CAEA,MAAM,MAAM;EACV,MAAM,KAAK,KAAKR;EAChB,IAAI;EACJ,KAAKI,QAAQ,SAAQ,QAAO;GAC1B,gBAAgB;GAChB,KAAKM,OAAO,KAAK,IAAI,GAAG;EAC1B,CAAC;EACD,GAAG,OAAO,4BAA4B;EACtC,MAAM,EAAC,WAAW,qBAChB,MAAM,KAAKN,QAAQ,qBAAqB;EAK1C,KAAUG,UAAU,kBAAkB;GACpC,OAAO;GACP,WAAW;EACb,CAAC;EAED,OAAO,KAAKG,OAAO,UAAU,GAAG;GAC9B,MAAM,EAAC,gBAAgB,WAAW,cAChC,MAAM,KAAKN,QAAQ,qBAAqB;GAE1C,IAAI;GACJ,IAAI,mBAAmB,CAAC;GACxB,IAAI;GAEJ,IAAI;IACF,MAAM,MAAM,KAAKU,mBACf,gBACA,WACA,SACF;IAGA,MAAM,WAAW,KAAKL,eAAe,kBAAkB;IAKvD,MAAM,oBAAoB,KAAKG;IAC/B,KAAKA,qBAAqB,KAAA;IAC1B,aACE,aACC,oBACG,MAAM,kBAAkB,UAAU,GAAG,IACrC,MAAM,KAAKT,gBAAgB,UAAU,GAAG;IAC9C,aAAa,KAAKO,OAAO,aAAa,UAAU;IAChD,KAAKJ,kBAAkB,QACrB,IACA,eACA,oBAAoB,WACtB;IAEA,IAAI;IAGJ,IAAI,aAAiC,CAAC;IACtC,IAAI,iBAAiB;IACrB,IAAI,gBAAgB;IAEpB,IAAI;IAEJ,MAAM,yBAAyB,UAC7B,KAAKA,kBAAkB,QACrB,IACA,eACA,eAAe,MAAM,SACrB,WAEE,WACI,EACE,gBAAgB,CACd;KACE,GAAG,SAAS;KACZ,OAAO,SAAS;KAChB,SAAS,SAAS;IACpB,CACF,EACF,IACA,CAAC,CACT;IAEF,MAAM,cAAc,YAAY;KAC9B,IAAI,WAAW,WAAW,GACxB;KAEF,MAAM,QAAQ;KACd,aAAa,CAAC;KACd,iBAAiB;KAEjB,MAAM,SAAS,MAAM,KAAKF,QAAQ,gBAAgB,KAAK;KACvD,KAAKW,cAAc,IAAI,MAAM;KAC7B,IAAI,QAAQ,mBACV,WAAW,KAAA;IAEf;IAGA,IAAI,WAAW;IAMf,MAAM,cAAc,YAAiC;KACnD,MAAM,OAAO,QAAQ;KACrB,IAAI,SAAS,WAAW,QAAQ,GAAG,mBAAmB,WAAW;MAC/D,GAAG,QACD,kCAAkC,QAAQ,GAAG,iBAC/C;MACA,WAAW;KACb;KACA,IACE,aACC,SAAS,WACR,SAAS,UACT,SAAS,YACT,SAAS,aACX;MACA,WAAW,SAAS,YAAY,SAAS;MACzC,OAAO;KACT;KACA,OAAO;IACT;IAEA,WAAW,MAAM,EAAC,MAAM,SAAS,UAAS,YAAY;KACpD,IAAI,WAAW,OAAO,GACpB;KAGF,KAAKL,OAAO,aAAa;KAEzB,KAAKC,mBAAmB,IAAI,CAAC;KAC7B,QAAQ,QAAQ,IAAhB;MACE,KAAK;OACH,GAAG,OAAO,oDAAoD,EAC5D,SAAS,QAAQ,GACnB,CAAC;OACD;MACF,KAAK,UAAU;OACb,MAAM,EAAC,WAAW,aAAY,QAAQ;OACtC,KAAKK,iCAAiC;OACtC,IAAI,UACF,KAAKP,eAAe,sBAAsB,UAAU;OAEtD,IAAI,WAAW;QACb,MAAM,SAA4B,EAChC,gBAAgB,UAAU,eAC5B;QACA,IAAI,UAAU,aACZ,OAAO,cAAc;SACnB,GAAG,UAAU;SACb,iBAAiB,KAAK,IAAI;QAC5B;QAEF,KAAKD,UAAU,OAAO,MAAM;OAC9B;OACA;MACF;MACA,KAAK,SAAS;OACZ,MAAM,EACJ,MACA,SAAS,KACT,gBAAgB,cACd,QAAQ;OACZ,IACE,SAAA,KACA,cAAc,KAAA,KACd,YAAY,kBACZ,KAAKQ,mCACH,sCACF;QAKA,GAAG,OACD,uDACM,UAAU,SAAS,eAAe,eAC1C;QACA;OACF;OAGA,KAAK,KACH,IAEA,IAAI,WACF,GAAG,wBAAwB,IAAiB,EAAE,IAAI,KACpD,CACF;OACA;MACF;MACA,SAAS;OACP,MAAM,MAAM,QAAQ;OACpB,IAAI,IAAI,QAAQ,cAAc,IAAI,QAAQ;QACxC,MAAM,EAAC,QAAQ,aAAY;QAC3B,MAAM,QAAQ,CAAC;QACf,WAAW;SACT;SACA,OAAO,SAAS;SAChB,SAAS,CAAC,GAAG,SAAS,OAAO,SAAS,GAAG,IAAI,OAAO;QACtD;QACA,IAAI,OAEF,sBAAsB,SAAS,IAAI;OAEvC;OAEA,IAAI,IAAI,QAAQ,kBAAkB,KAAKV,kBAAkB;QAIvD,MAAM,YAAY;QAClB,MAAM,QAAQ,uBAAuB,IAAI,IAAI;QAC7C,aAAa,IAAI,iBAAiB;QAClC,SAAS,MAAM,KAAK;QACpB,KAAKA,iBAAiB,QACpB,IACA,eACA,kBAAkB,MAAM,KAAK,MAAM,MAAM,aACzC,KACA,SAAS,KACX;QAEA,SAAS,aAAa;QACtB,WAAW,KAAK,OAA2B;QAC3C,kBAAkB;QAClB,MAAM,YAAY;QAClB,MAAM,UAAU,SAAS,OAAO;QAChC,GAAG,OACD,iBAAiB,MAAM,KAAK,IAAI,QAAQ,QAAQ,CAAC,EAAE,KACrD;QACA,KAAKA,iBAAiB,QACpB,IACA,eACA,iBAAiB,MAAM,KAAK,MAAM,MAAM,aACxC,GACA,SAAS,KACX;QACA,IAAI,UAEF,sBAAsB,SAAS,KAAK;QAEtC;OACF;OACA,WAAW,KAAA;OAEX,MAAM,OAAO,QAAQ;OACrB,MAAM,6BACJ,SAAS,UAAU,gBAAgB,CAAC;OACtC,IAAI,SAAS,SACX,gBAAgB;YACX,IAAI,SAAS,YAAY,SAAS,YACvC,gBAAgB;OAGlB,WAAW,KAAK,OAA2B;OAC3C,kBAAkB;OAClB,IAGE,SAAS,YACT,SAAS,cAGT,8BACA,WAAW,UAAU,6BACrB,kBAAkB,uBAElB,MAAM,YAAY;OAEpB;MACF;KACF;IACF;IACA,KAAKF,QAAQ,MAAM;GACrB,SAAS,GAAG;IACV,MAAM;IACN,KAAKA,QAAQ,MAAM;GACrB,UAAU;IACR,YAAY,OAAO;IACnB,WAAW;IACX,KAAKE,kBAAkB,KAAK;GAC9B;GACA,IAAI,QAAQ,KAAA,KAAa,KAAKG,eAAe,mBAC3C;GAEF,KAAKA,eAAe,kBAAkB,GAAG,OAAO;GAChD,MAAM,KAAKC,OAAO,QAAQ,IAAI,GAAG;EACnC;EACA,GAAG,OAAO,2BAA2B;EACrC,IAAI,aACF,MAAM;CAEV;CAEA,cAAc,IAAgB,QAA6B;EACzD,IAAI,CAAC,QACH;EAEF,IAAI,OAAO,mBAAmB;GAE5B,MAAM,SAAS,OAAO;GACtB,KAAKJ,kBAAkB,QACrB,IACA,eACA,cAAc,OAAO,MAAM,SAC3B,UACO,EAAC,gBAAgB,CAAC,MAAM,EAAC,EAClC;EACF,OAAO,IAAI,OAAO,eAChB,KAAKA,kBAAkB,QAAQ,IAAI,eAAe,gBAAgB;EAEpE,IAAI,OAAO,aAAa,OAAO,kBAC7B,KAAUC,UAAU,kBAAkB;GACpC,OAAO;GACP,WAAW,OAAO;GAClB,oBAAoB,KAAK,IAAI;GAC7B,sBAAsB,OAAO;EAC/B,CAAC;CAEL;CAEA,YAAkC;EAChC,OAAO,KAAKA,UAAU,UAAU;CAClC;CAEA,mBACE,gBACA,WACA,WACmB;EACnB,OAAO;GACL,iBAAA;GACA,QAAQ,KAAKN;GACb,IAAI,KAAKC;GACT,MAAM,KAAKG;GACX;GACA;GACA;GACA,GAAI,KAAKI,gBAAgB,EAAC,WAAW,KAAI,IAAI,CAAC;EAChD;CACF;;;;;;CAOA,kBAAkB,aAAwC;EACxD,MAAM,KAAK,IAAI,SAAS,KAAKT,KAAK,aAAa,EAAC,UAAU,KAAI,CAAC;EAC/D,IAAI;GACF,MAAM,EAAC,gBAAgB,WAAW,cAAa,GAAG,mBAAmB;IACnE,GAAG,qBAAqB,IAAI,gBAAgB,EAAE,CAAC;IAC/C,WAAW,qBAAqB,EAAE;GACpC,EAAE;GACF,OAAO,KAAKc,mBAAmB,gBAAgB,WAAW,SAAS;EACrE,UAAU;GACR,GAAG,MAAM;EACX;CACF;CAEA,KAAK,IAAgB,KAAe;EAClC,KAAKJ,OAAO,KAAK,IAAI,GAAG;EAExB,KAAKN,QAAQ,MAAM;CACrB;AACF"}
+{"version":3,"file":"incremental-sync.js","names":[],"sources":["../../../../../../zero-cache/src/services/replicator/incremental-sync.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {AbortError} from '../../../../shared/src/abort-error.ts';\nimport type {Enum} from '../../../../shared/src/enum.ts';\nimport {Database} from '../../../../zqlite/src/db.ts';\nimport {mapPostgresToLiteIndex} from '../../db/pg-to-lite.ts';\nimport {StatementRunner} from '../../db/statements.ts';\nimport {getOrCreateCounter} from '../../observability/metrics.ts';\nimport type {Source} from '../../types/streams.ts';\nimport type {DownloadStatus} from '../change-source/protocol/current.ts';\nimport type {ChangeStreamData} from '../change-source/protocol/current/downstream.ts';\nimport type {ReservationFollowup} from '../change-streamer/change-streamer-http.ts';\nimport {\n  errorTypeToReadableName,\n  PROTOCOL_VERSION,\n  type ChangeStreamer,\n  type SubscriberContext,\n} from '../change-streamer/change-streamer.ts';\nimport type * as ErrorType from '../change-streamer/error-type-enum.ts';\nimport {WrongReplicaVersion} from '../change-streamer/error-type-enum.ts';\nimport type {SubscribeDownstream} from '../change-streamer/subscribe.ts';\nimport {RunningState} from '../running-state.ts';\nimport type {CommitResult} from './change-processor.ts';\nimport {\n  DrainHandler,\n  type Downstream,\n  type SwitchoverOptions,\n} from './drain-handler.ts';\nimport {Notifier} from './notifier.ts';\nimport {\n  IndexingProgress,\n  type ReplicationStatusPublisher,\n} from './replication-status.ts';\nimport type {ReplicaState, ReplicatorMode} from './replicator.ts';\nimport {\n  ReplicationReportRecorder,\n  type ReplicationLagReadinessOptions,\n} from './reporter/recorder.ts';\nimport type {ReplicationReport} from './reporter/report-schema.ts';\nimport {readReplicaBackfills} from './schema/backfilling.ts';\nimport {getSubscriptionState} from './schema/replication-state.ts';\nimport type {WriteWorkerClient} from './write-worker-client.ts';\n\ntype ErrorType = Enum<typeof ErrorType>;\n\nconst MAX_WORKER_BATCH_MESSAGES = 256;\nconst MAX_WORKER_BATCH_SIZE = 1024 * 1024;\n\n// The number of times to reconnect after being rejected by a\n// replication-manager with an older replica version (e.g. one that has yet\n// to be replaced by a resync), before accepting the rejection and restarting\n// (e.g. if the upstream was legitimately replaced by one with a lower replica\n// version).\nconst MAX_OLDER_REPLICA_VERSION_REJECTIONS = 20;\n\n/**\n * The {@link IncrementalSyncer} manages a logical replication stream from upstream,\n * handling application lifecycle events (start, stop) and retrying the\n * connection with exponential backoff. The actual handling of the logical\n * replication messages is done by the {@link ChangeProcessor}, which runs\n * in a worker thread via the {@link WriteWorkerClient}.\n */\nexport class IncrementalSyncer {\n  readonly #lc: LogContext;\n  readonly #taskID: string;\n  readonly #id: string;\n  readonly #changeStreamer: ChangeStreamer;\n  readonly #worker: WriteWorkerClient;\n  readonly #mode: ReplicatorMode;\n  readonly #statusPublisher: ReplicationStatusPublisher | null;\n  readonly #notifier: Notifier;\n  readonly #reporter: ReplicationReportRecorder;\n  // Consumed (and cleared) by the first subscribe() in run()'s loop; every\n  // subsequent (re)connection goes through #changeStreamer.subscribe().\n  #initialConnection: ReservationFollowup | undefined;\n\n  // Only set for serving replicas that switch over between\n  // replication-managers.\n  readonly #drainHandler: DrainHandler | undefined;\n  #olderReplicaVersionRejections = 0;\n\n  readonly #state = new RunningState('IncrementalSyncer');\n\n  readonly #replicationEvents = getOrCreateCounter(\n    'replication',\n    'events',\n    'Number of replication events processed',\n  );\n\n  constructor(\n    lc: LogContext,\n    taskID: string,\n    id: string,\n    changeStreamer: ChangeStreamer,\n    worker: WriteWorkerClient,\n    mode: ReplicatorMode,\n    statusPublisher: ReplicationStatusPublisher | null,\n    replicationLagReadinessOptions: ReplicationLagReadinessOptions = {},\n    initialConnection?: ReservationFollowup,\n    switchover?: SwitchoverOptions,\n  ) {\n    this.#lc = lc;\n    this.#taskID = taskID;\n    this.#id = id;\n    this.#changeStreamer = changeStreamer;\n    this.#worker = worker;\n    this.#mode = mode;\n    this.#statusPublisher = statusPublisher;\n    this.#notifier = new Notifier();\n    this.#reporter = new ReplicationReportRecorder(\n      lc,\n      replicationLagReadinessOptions,\n    );\n    this.#initialConnection = initialConnection;\n    // Only serving replicas switch over between replication-managers. In\n    // particular, the in-RM backup-replicator never does.\n    this.#drainHandler =\n      mode === 'serving' && switchover\n        ? new DrainHandler(lc, changeStreamer, this.#state, switchover, () =>\n            this.#committedContext(switchover.replicaFile),\n          )\n        : undefined;\n  }\n\n  ready(): Promise<void> {\n    return this.#reporter.ready();\n  }\n\n  async run() {\n    const lc = this.#lc;\n    let workerError: Error | undefined;\n    this.#worker.onError(err => {\n      workerError ??= err;\n      this.#state.stop(lc, err);\n    });\n    lc.info?.(`Starting IncrementalSyncer`);\n    const {watermark: initialWatermark} =\n      await this.#worker.getSubscriptionState();\n\n    // Notify any waiting subscribers that the replica is ready to be read.\n    // This initial notification intentionally omits replicaReadyTimeMs because\n    // it represents already-current state, not newly-unserved work.\n    void this.#notifier.notifySubscribers({\n      state: 'version-ready',\n      watermark: initialWatermark,\n    });\n\n    while (this.#state.shouldRun()) {\n      const {replicaVersion, watermark, backfills} =\n        await this.#worker.getSubscriptionState();\n\n      let downstream: Downstream | undefined;\n      let unregister = () => {};\n      let err: unknown | undefined;\n\n      try {\n        const ctx = this.#subscriberContext(\n          replicaVersion,\n          watermark,\n          backfills,\n        );\n        // A successful switchover has already established the subscription\n        // on a new connection.\n        const switched = this.#drainHandler?.getNextConnection();\n        // The first connection reuses the still-open reservation connection\n        // (if any) from the backup restore, so the replication-manager that\n        // reserved the change log is the one that serves the subscription.\n        // Every reconnect after that opens a fresh connection.\n        const initialConnection = this.#initialConnection;\n        this.#initialConnection = undefined;\n        downstream =\n          switched ??\n          (initialConnection\n            ? await initialConnection.subscribe(ctx)\n            : await this.#changeStreamer.subscribe(ctx));\n        unregister = this.#state.cancelOnStop(downstream);\n        this.#statusPublisher?.publish(\n          lc,\n          'Replicating',\n          `Replicating from ${watermark}`,\n        );\n\n        let backfill:\n          | {status: DownloadStatus; table: string; columns: string[]}\n          | undefined;\n        let writeBatch: ChangeStreamData[] = [];\n        let writeBatchSize = 0;\n        let inTransaction = false;\n        // Indexes created in the current run of consecutive index creations.\n        let indexing: IndexingProgress | undefined;\n\n        const publishBackfillStatus = (table: string) =>\n          this.#statusPublisher?.publish(\n            lc,\n            'Replicating',\n            `Backfilling ${table} table`,\n            3000,\n            () =>\n              backfill\n                ? {\n                    downloadStatus: [\n                      {\n                        ...backfill.status,\n                        table: backfill.table,\n                        columns: backfill.columns,\n                      },\n                    ],\n                  }\n                : {},\n          );\n\n        const flushWrites = async () => {\n          if (writeBatch.length === 0) {\n            return;\n          }\n          const batch = writeBatch;\n          writeBatch = [];\n          writeBatchSize = 0;\n\n          const result = await this.#worker.processMessages(batch);\n          this.#handleResult(lc, result);\n          if (result?.completedBackfill) {\n            backfill = undefined;\n          }\n        };\n\n        // Whether the current transaction is being skipped.\n        let skipping = false;\n\n        // A subscription established by a switchover starts at the\n        // watermark at which it was requested, and may thus include\n        // transactions that were since committed from the previous\n        // subscription.\n        const shouldSkip = (message: SubscribeDownstream) => {\n          const type = message[0];\n          if (type === 'begin' && message[2].commitWatermark <= watermark) {\n            lc.debug?.(\n              `skipping committed transaction ${message[2].commitWatermark}`,\n            );\n            skipping = true;\n          }\n          if (\n            skipping &&\n            (type === 'begin' ||\n              type === 'data' ||\n              type === 'commit' ||\n              type === 'rollback')\n          ) {\n            skipping = type !== 'commit' && type !== 'rollback';\n            return true;\n          }\n          return false;\n        };\n\n        for await (const {data: message, size} of downstream) {\n          if (shouldSkip(message)) {\n            continue;\n          }\n          // Only reset backoff after receiving an actual message.\n          this.#state.resetBackoff();\n\n          this.#replicationEvents.add(1);\n          switch (message[0]) {\n            case 'reserved':\n              lc.warn?.(`unexpected snapshot reservation message received`, {\n                message: message[0],\n              });\n              break;\n            case 'status': {\n              const {lagReport, draining} = message[1];\n              this.#olderReplicaVersionRejections = 0;\n              if (draining) {\n                this.#drainHandler?.onChangeStreamerDrain(downstream);\n              }\n              if (lagReport) {\n                const report: ReplicationReport = {\n                  nextSendTimeMs: lagReport.nextSendTimeMs,\n                };\n                if (lagReport.lastTimings) {\n                  report.lastTimings = {\n                    ...lagReport.lastTimings,\n                    replicateTimeMs: Date.now(),\n                  };\n                }\n                this.#reporter.record(report);\n              }\n              break;\n            }\n            case 'error': {\n              const {\n                type,\n                message: msg,\n                replicaVersion: rmVersion,\n              } = message[1];\n              if (\n                type === WrongReplicaVersion &&\n                rmVersion !== undefined &&\n                rmVersion < replicaVersion &&\n                this.#olderReplicaVersionRejections++ <\n                  MAX_OLDER_REPLICA_VERSION_REJECTIONS\n              ) {\n                // The replication-manager is behind this replica (e.g. it\n                // has yet to be replaced by a resync). Rather than restoring\n                // its older replica, reconnect (with backoff), which may\n                // reach a different replication-manager.\n                lc.warn?.(\n                  `replication-manager is at an older replica version ` +\n                    `(${rmVersion}) than ${replicaVersion}. reconnecting`,\n                );\n                break;\n              }\n              // Signal from the replication-manager that the view-syncer must\n              // shut down and restore a new backup from litestream.\n              this.stop(\n                lc,\n                // Note: The AbortError indicates a clean / intentional shutdown.\n                new AbortError(\n                  `${errorTypeToReadableName(type as ErrorType)}: ${msg}`,\n                ),\n              );\n              break;\n            }\n            default: {\n              const msg = message[1];\n              if (msg.tag === 'backfill' && msg.status) {\n                const {status, relation} = msg;\n                const first = !backfill;\n                backfill = {\n                  status, // Update the current status\n                  table: relation.name,\n                  columns: [...relation.rowKey.columns, ...msg.columns],\n                };\n                if (first) {\n                  // Start publishing the status every 3 seconds.\n                  publishBackfillStatus(relation.name);\n                }\n              }\n\n              if (msg.tag === 'create-index' && this.#statusPublisher) {\n                // Creating an index on an existing table can take a long\n                // time. Flush preceding changes so that the index creation\n                // is processed (and timed) on its own.\n                await flushWrites();\n                const index = mapPostgresToLiteIndex(msg.spec);\n                indexing ??= new IndexingProgress();\n                indexing.start(index);\n                this.#statusPublisher.publish(\n                  lc,\n                  'Replicating',\n                  `Creating index ${index.name} on ${index.tableName}`,\n                  3000,\n                  indexing.state,\n                );\n                // Exclude the time spent publishing the status.\n                indexing.restartTimer();\n                writeBatch.push(message as ChangeStreamData);\n                writeBatchSize += size;\n                await flushWrites();\n                const elapsed = indexing.finish();\n                lc.info?.(\n                  `Created index ${index.name} (${elapsed.toFixed(3)} ms)`,\n                );\n                this.#statusPublisher.publish(\n                  lc,\n                  'Replicating',\n                  `Created index ${index.name} on ${index.tableName}`,\n                  0,\n                  indexing.state,\n                );\n                if (backfill) {\n                  // Resume reporting the progress of the ongoing backfill.\n                  publishBackfillStatus(backfill.table);\n                }\n                break;\n              }\n              indexing = undefined;\n\n              const type = message[0];\n              const invalidTransactionSequence =\n                type === 'begin' ? inTransaction : !inTransaction;\n              if (type === 'begin') {\n                inTransaction = true;\n              } else if (type === 'commit' || type === 'rollback') {\n                inTransaction = false;\n              }\n\n              writeBatch.push(message as ChangeStreamData);\n              writeBatchSize += size;\n              if (\n                // Waiting here ensures that consuming the commit, which ACKs\n                // it upstream, only happens after the SQLite commit finishes.\n                type === 'commit' ||\n                type === 'rollback' ||\n                // Promptly surface malformed transaction sequences instead\n                // of leaving a partial batch waiting for another message.\n                invalidTransactionSequence ||\n                writeBatch.length >= MAX_WORKER_BATCH_MESSAGES ||\n                writeBatchSize >= MAX_WORKER_BATCH_SIZE\n              ) {\n                await flushWrites();\n              }\n              break;\n            }\n          }\n        }\n        this.#worker.abort();\n      } catch (e) {\n        err = e;\n        this.#worker.abort();\n      } finally {\n        downstream?.cancel();\n        unregister();\n        this.#statusPublisher?.stop();\n      }\n      if (err === undefined && this.#drainHandler?.hasNextConnection) {\n        continue; // Switched over to a new subscription; no backoff.\n      }\n      this.#drainHandler?.getNextConnection()?.cancel();\n      await this.#state.backoff(lc, err);\n    }\n    lc.info?.('IncrementalSyncer stopped');\n    if (workerError) {\n      throw workerError;\n    }\n  }\n\n  #handleResult(lc: LogContext, result: CommitResult | null) {\n    if (!result) {\n      return;\n    }\n    if (result.completedBackfill) {\n      // Publish the final status\n      const status = result.completedBackfill;\n      this.#statusPublisher?.publish(\n        lc,\n        'Replicating',\n        `Backfilled ${status.table} table`,\n        0,\n        () => ({downloadStatus: [status]}),\n      );\n    } else if (result.schemaUpdated) {\n      this.#statusPublisher?.publish(lc, 'Replicating', 'Schema updated');\n    }\n    if (result.watermark && result.changeLogUpdated) {\n      void this.#notifier.notifySubscribers({\n        state: 'version-ready',\n        watermark: result.watermark,\n        replicaReadyTimeMs: Date.now(),\n        upstreamCommitTimeMs: result.upstreamCommitTimeMs,\n      });\n    }\n  }\n\n  subscribe(): Source<ReplicaState> {\n    return this.#notifier.subscribe();\n  }\n\n  #subscriberContext(\n    replicaVersion: string,\n    watermark: string,\n    backfills: SubscriberContext['backfills'],\n  ): SubscriberContext {\n    return {\n      protocolVersion: PROTOCOL_VERSION,\n      taskID: this.#taskID,\n      id: this.#id,\n      mode: this.#mode,\n      watermark,\n      replicaVersion,\n      backfills,\n      ...(this.#drainHandler ? {drainable: true} : {}),\n    };\n  }\n\n  /**\n   * Returns the subscriber context as of the last committed transaction,\n   * read from the replica file (i.e. not from the write worker, whose\n   * connection may be in the middle of a transaction).\n   */\n  #committedContext(replicaFile: string): SubscriberContext {\n    const db = new Database(this.#lc, replicaFile, {readonly: true});\n    try {\n      const {replicaVersion, watermark, backfills} = db.transaction(() => ({\n        ...getSubscriptionState(new StatementRunner(db)),\n        backfills: readReplicaBackfills(db),\n      }));\n      return this.#subscriberContext(replicaVersion, watermark, backfills);\n    } finally {\n      db.close();\n    }\n  }\n\n  stop(lc: LogContext, err?: unknown) {\n    this.#state.stop(lc, err);\n    // Abort any polling loop that the write worker may be in.\n    this.#worker.abort();\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;AA4CA,IAAM,4BAA4B;AAClC,IAAM,wBAAwB;AAO9B,IAAM,uCAAuC;;;;;;;;AAS7C,IAAa,oBAAb,MAA+B;CAC7B;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CAGA;CAIA;CACA,iCAAiC;CAEjC,SAAkB,IAAI,aAAa,mBAAmB;CAEtD,qBAA8B,mBAC5B,eACA,UACA,wCACF;CAEA,YACE,IACA,QACA,IACA,gBACA,QACA,MACA,iBACA,iCAAiE,CAAC,GAClE,mBACA,YACA;EACA,KAAK,MAAM;EACX,KAAK,UAAU;EACf,KAAK,MAAM;EACX,KAAK,kBAAkB;EACvB,KAAK,UAAU;EACf,KAAK,QAAQ;EACb,KAAK,mBAAmB;EACxB,KAAK,YAAY,IAAI,SAAS;EAC9B,KAAK,YAAY,IAAI,0BACnB,IACA,8BACF;EACA,KAAK,qBAAqB;EAG1B,KAAK,gBACH,SAAS,aAAa,aAClB,IAAI,aAAa,IAAI,gBAAgB,KAAK,QAAQ,kBAChD,KAAK,kBAAkB,WAAW,WAAW,CAC/C,IACA,KAAA;CACR;CAEA,QAAuB;EACrB,OAAO,KAAK,UAAU,MAAM;CAC9B;CAEA,MAAM,MAAM;EACV,MAAM,KAAK,KAAK;EAChB,IAAI;EACJ,KAAK,QAAQ,SAAQ,QAAO;GAC1B,gBAAgB;GAChB,KAAK,OAAO,KAAK,IAAI,GAAG;EAC1B,CAAC;EACD,GAAG,OAAO,4BAA4B;EACtC,MAAM,EAAC,WAAW,qBAChB,MAAM,KAAK,QAAQ,qBAAqB;EAK1C,KAAU,UAAU,kBAAkB;GACpC,OAAO;GACP,WAAW;EACb,CAAC;EAED,OAAO,KAAK,OAAO,UAAU,GAAG;GAC9B,MAAM,EAAC,gBAAgB,WAAW,cAChC,MAAM,KAAK,QAAQ,qBAAqB;GAE1C,IAAI;GACJ,IAAI,mBAAmB,CAAC;GACxB,IAAI;GAEJ,IAAI;IACF,MAAM,MAAM,KAAK,mBACf,gBACA,WACA,SACF;IAGA,MAAM,WAAW,KAAK,eAAe,kBAAkB;IAKvD,MAAM,oBAAoB,KAAK;IAC/B,KAAK,qBAAqB,KAAA;IAC1B,aACE,aACC,oBACG,MAAM,kBAAkB,UAAU,GAAG,IACrC,MAAM,KAAK,gBAAgB,UAAU,GAAG;IAC9C,aAAa,KAAK,OAAO,aAAa,UAAU;IAChD,KAAK,kBAAkB,QACrB,IACA,eACA,oBAAoB,WACtB;IAEA,IAAI;IAGJ,IAAI,aAAiC,CAAC;IACtC,IAAI,iBAAiB;IACrB,IAAI,gBAAgB;IAEpB,IAAI;IAEJ,MAAM,yBAAyB,UAC7B,KAAK,kBAAkB,QACrB,IACA,eACA,eAAe,MAAM,SACrB,WAEE,WACI,EACE,gBAAgB,CACd;KACE,GAAG,SAAS;KACZ,OAAO,SAAS;KAChB,SAAS,SAAS;IACpB,CACF,EACF,IACA,CAAC,CACT;IAEF,MAAM,cAAc,YAAY;KAC9B,IAAI,WAAW,WAAW,GACxB;KAEF,MAAM,QAAQ;KACd,aAAa,CAAC;KACd,iBAAiB;KAEjB,MAAM,SAAS,MAAM,KAAK,QAAQ,gBAAgB,KAAK;KACvD,KAAK,cAAc,IAAI,MAAM;KAC7B,IAAI,QAAQ,mBACV,WAAW,KAAA;IAEf;IAGA,IAAI,WAAW;IAMf,MAAM,cAAc,YAAiC;KACnD,MAAM,OAAO,QAAQ;KACrB,IAAI,SAAS,WAAW,QAAQ,EAAE,CAAC,mBAAmB,WAAW;MAC/D,GAAG,QACD,kCAAkC,QAAQ,EAAE,CAAC,iBAC/C;MACA,WAAW;KACb;KACA,IACE,aACC,SAAS,WACR,SAAS,UACT,SAAS,YACT,SAAS,aACX;MACA,WAAW,SAAS,YAAY,SAAS;MACzC,OAAO;KACT;KACA,OAAO;IACT;IAEA,WAAW,MAAM,EAAC,MAAM,SAAS,UAAS,YAAY;KACpD,IAAI,WAAW,OAAO,GACpB;KAGF,KAAK,OAAO,aAAa;KAEzB,KAAK,mBAAmB,IAAI,CAAC;KAC7B,QAAQ,QAAQ,IAAhB;MACE,KAAK;OACH,GAAG,OAAO,oDAAoD,EAC5D,SAAS,QAAQ,GACnB,CAAC;OACD;MACF,KAAK,UAAU;OACb,MAAM,EAAC,WAAW,aAAY,QAAQ;OACtC,KAAK,iCAAiC;OACtC,IAAI,UACF,KAAK,eAAe,sBAAsB,UAAU;OAEtD,IAAI,WAAW;QACb,MAAM,SAA4B,EAChC,gBAAgB,UAAU,eAC5B;QACA,IAAI,UAAU,aACZ,OAAO,cAAc;SACnB,GAAG,UAAU;SACb,iBAAiB,KAAK,IAAI;QAC5B;QAEF,KAAK,UAAU,OAAO,MAAM;OAC9B;OACA;MACF;MACA,KAAK,SAAS;OACZ,MAAM,EACJ,MACA,SAAS,KACT,gBAAgB,cACd,QAAQ;OACZ,IACE,SAAA,KACA,cAAc,KAAA,KACd,YAAY,kBACZ,KAAK,mCACH,sCACF;QAKA,GAAG,OACD,uDACM,UAAU,SAAS,eAAe,eAC1C;QACA;OACF;OAGA,KAAK,KACH,IAEA,IAAI,WACF,GAAG,wBAAwB,IAAiB,EAAE,IAAI,KACpD,CACF;OACA;MACF;MACA,SAAS;OACP,MAAM,MAAM,QAAQ;OACpB,IAAI,IAAI,QAAQ,cAAc,IAAI,QAAQ;QACxC,MAAM,EAAC,QAAQ,aAAY;QAC3B,MAAM,QAAQ,CAAC;QACf,WAAW;SACT;SACA,OAAO,SAAS;SAChB,SAAS,CAAC,GAAG,SAAS,OAAO,SAAS,GAAG,IAAI,OAAO;QACtD;QACA,IAAI,OAEF,sBAAsB,SAAS,IAAI;OAEvC;OAEA,IAAI,IAAI,QAAQ,kBAAkB,KAAK,kBAAkB;QAIvD,MAAM,YAAY;QAClB,MAAM,QAAQ,uBAAuB,IAAI,IAAI;QAC7C,aAAa,IAAI,iBAAiB;QAClC,SAAS,MAAM,KAAK;QACpB,KAAK,iBAAiB,QACpB,IACA,eACA,kBAAkB,MAAM,KAAK,MAAM,MAAM,aACzC,KACA,SAAS,KACX;QAEA,SAAS,aAAa;QACtB,WAAW,KAAK,OAA2B;QAC3C,kBAAkB;QAClB,MAAM,YAAY;QAClB,MAAM,UAAU,SAAS,OAAO;QAChC,GAAG,OACD,iBAAiB,MAAM,KAAK,IAAI,QAAQ,QAAQ,CAAC,EAAE,KACrD;QACA,KAAK,iBAAiB,QACpB,IACA,eACA,iBAAiB,MAAM,KAAK,MAAM,MAAM,aACxC,GACA,SAAS,KACX;QACA,IAAI,UAEF,sBAAsB,SAAS,KAAK;QAEtC;OACF;OACA,WAAW,KAAA;OAEX,MAAM,OAAO,QAAQ;OACrB,MAAM,6BACJ,SAAS,UAAU,gBAAgB,CAAC;OACtC,IAAI,SAAS,SACX,gBAAgB;YACX,IAAI,SAAS,YAAY,SAAS,YACvC,gBAAgB;OAGlB,WAAW,KAAK,OAA2B;OAC3C,kBAAkB;OAClB,IAGE,SAAS,YACT,SAAS,cAGT,8BACA,WAAW,UAAU,6BACrB,kBAAkB,uBAElB,MAAM,YAAY;OAEpB;MACF;KACF;IACF;IACA,KAAK,QAAQ,MAAM;GACrB,SAAS,GAAG;IACV,MAAM;IACN,KAAK,QAAQ,MAAM;GACrB,UAAU;IACR,YAAY,OAAO;IACnB,WAAW;IACX,KAAK,kBAAkB,KAAK;GAC9B;GACA,IAAI,QAAQ,KAAA,KAAa,KAAK,eAAe,mBAC3C;GAEF,KAAK,eAAe,kBAAkB,CAAC,EAAE,OAAO;GAChD,MAAM,KAAK,OAAO,QAAQ,IAAI,GAAG;EACnC;EACA,GAAG,OAAO,2BAA2B;EACrC,IAAI,aACF,MAAM;CAEV;CAEA,cAAc,IAAgB,QAA6B;EACzD,IAAI,CAAC,QACH;EAEF,IAAI,OAAO,mBAAmB;GAE5B,MAAM,SAAS,OAAO;GACtB,KAAK,kBAAkB,QACrB,IACA,eACA,cAAc,OAAO,MAAM,SAC3B,UACO,EAAC,gBAAgB,CAAC,MAAM,EAAC,EAClC;EACF,OAAO,IAAI,OAAO,eAChB,KAAK,kBAAkB,QAAQ,IAAI,eAAe,gBAAgB;EAEpE,IAAI,OAAO,aAAa,OAAO,kBAC7B,KAAU,UAAU,kBAAkB;GACpC,OAAO;GACP,WAAW,OAAO;GAClB,oBAAoB,KAAK,IAAI;GAC7B,sBAAsB,OAAO;EAC/B,CAAC;CAEL;CAEA,YAAkC;EAChC,OAAO,KAAK,UAAU,UAAU;CAClC;CAEA,mBACE,gBACA,WACA,WACmB;EACnB,OAAO;GACL,iBAAA;GACA,QAAQ,KAAK;GACb,IAAI,KAAK;GACT,MAAM,KAAK;GACX;GACA;GACA;GACA,GAAI,KAAK,gBAAgB,EAAC,WAAW,KAAI,IAAI,CAAC;EAChD;CACF;;;;;;CAOA,kBAAkB,aAAwC;EACxD,MAAM,KAAK,IAAI,SAAS,KAAK,KAAK,aAAa,EAAC,UAAU,KAAI,CAAC;EAC/D,IAAI;GACF,MAAM,EAAC,gBAAgB,WAAW,cAAa,GAAG,mBAAmB;IACnE,GAAG,qBAAqB,IAAI,gBAAgB,EAAE,CAAC;IAC/C,WAAW,qBAAqB,EAAE;GACpC,EAAE;GACF,OAAO,KAAK,mBAAmB,gBAAgB,WAAW,SAAS;EACrE,UAAU;GACR,GAAG,MAAM;EACX;CACF;CAEA,KAAK,IAAgB,KAAe;EAClC,KAAK,OAAO,KAAK,IAAI,GAAG;EAExB,KAAK,QAAQ,MAAM;CACrB;AACF"}

package/out/zero-cache/src/services/replicator/notifier.js.map

@@ -1 +1 @@
-{"version":3,"file":"notifier.js","names":["#eventEmitter","#lastStateReceived","#newSubscription"],"sources":["../../../../../../zero-cache/src/services/replicator/notifier.ts"],"sourcesContent":["import {EventEmitter} from 'eventemitter3';\nimport {\n  type PendingResult,\n  type Result,\n  Subscription,\n} from '../../types/subscription.ts';\nimport type {ReplicaState, ReplicaStateNotifier} from './replicator.ts';\n\n/**\n * Handles the semantics of {@link ReplicatorVersionNotifier.subscribe()}\n * notifications, namely:\n *\n * * New subscribers are notified immediately with the latest received\n *   ReplicaState.\n *\n * * Non-latest notifications are discarded if the subscriber is too\n *   busy to consume them.\n *\n * By encapsulating the state for the first behavior (essentially, whether\n * the first notification has been sent by the Replicator), Notifier objects\n * can be chained to simplify fanout from Replicator to View Syncers.\n *\n * In particular, each Syncer Thread can manage a single Subscription to\n * the Replicator across a MessageChannel, which it uses for its own Notifier\n * instance to manage subscriptions from View Syncers within its thread. This\n * way a Replicator only deals with sending notifications to a bounded set\n * of MessageChannel-based subscribers (Syncer Threads), while the dynamic\n * subscribe and unsubscribe traffic from View Syncers remains within each\n * Syncer Thread.\n */\nfunction oldest(\n  curr: number | undefined,\n  prev: number | undefined,\n): number | undefined {\n  if (curr === undefined) {\n    return prev;\n  }\n  if (prev === undefined) {\n    return curr;\n  }\n  return Math.min(curr, prev);\n}\n\nexport class Notifier implements ReplicaStateNotifier {\n  readonly #eventEmitter = new EventEmitter();\n  #lastStateReceived: ReplicaState | undefined;\n\n  get latestState(): ReplicaState | undefined {\n    return this.#lastStateReceived;\n  }\n\n  #newSubscription() {\n    const notify = (state: ReplicaState) => subscription.push(state);\n    const subscription = Subscription.create<ReplicaState>({\n      // A coalesced notification stands in for every notification it\n      // subsumed, so both timestamps keep the *oldest* value: they measure how\n      // long work has been outstanding, and the subsumed watermarks are still\n      // unserved.\n      coalesce: (curr, prev) => ({\n        ...curr,\n        replicaReadyTimeMs: oldest(\n          curr.replicaReadyTimeMs,\n          prev.replicaReadyTimeMs,\n        ),\n        upstreamCommitTimeMs: oldest(\n          curr.upstreamCommitTimeMs,\n          prev.upstreamCommitTimeMs,\n        ),\n      }),\n      cleanup: () => this.#eventEmitter.off('version', notify),\n    });\n    return {notify, subscription};\n  }\n\n  subscribe(): Subscription<ReplicaState> {\n    const {notify, subscription} = this.#newSubscription();\n    this.#eventEmitter.on('version', notify);\n    if (this.#lastStateReceived) {\n      // Per Replicator.subscribe() semantics, the current state of the\n      // replica, if known, is immediately sent on subscribe.\n      notify(this.#lastStateReceived);\n    }\n    return subscription;\n  }\n\n  notifySubscribers(\n    state: ReplicaState = {state: 'version-ready'},\n  ): Promise<Result>[] {\n    this.#lastStateReceived = state;\n    return this.#eventEmitter\n      .listeners('version')\n      .map(notify => notify(state) as unknown as PendingResult)\n      .map(pending => pending.result);\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;AA8BA,SAAS,OACP,MACA,MACoB;CACpB,IAAI,SAAS,KAAA,GACX,OAAO;CAET,IAAI,SAAS,KAAA,GACX,OAAO;CAET,OAAO,KAAK,IAAI,MAAM,IAAI;AAC5B;AAEA,IAAa,WAAb,MAAsD;CACpD,gBAAyB,IAAI,aAAa;CAC1C;CAEA,IAAI,cAAwC;EAC1C,OAAO,KAAKC;CACd;CAEA,mBAAmB;EACjB,MAAM,UAAU,UAAwB,aAAa,KAAK,KAAK;EAC/D,MAAM,eAAe,aAAa,OAAqB;GAKrD,WAAW,MAAM,UAAU;IACzB,GAAG;IACH,oBAAoB,OAClB,KAAK,oBACL,KAAK,kBACP;IACA,sBAAsB,OACpB,KAAK,sBACL,KAAK,oBACP;GACF;GACA,eAAe,KAAKD,cAAc,IAAI,WAAW,MAAM;EACzD,CAAC;EACD,OAAO;GAAC;GAAQ;EAAY;CAC9B;CAEA,YAAwC;EACtC,MAAM,EAAC,QAAQ,iBAAgB,KAAKE,iBAAiB;EACrD,KAAKF,cAAc,GAAG,WAAW,MAAM;EACvC,IAAI,KAAKC,oBAGP,OAAO,KAAKA,kBAAkB;EAEhC,OAAO;CACT;CAEA,kBACE,QAAsB,EAAC,OAAO,gBAAe,GAC1B;EACnB,KAAKA,qBAAqB;EAC1B,OAAO,KAAKD,cACT,UAAU,SAAS,EACnB,KAAI,WAAU,OAAO,KAAK,CAA6B,EACvD,KAAI,YAAW,QAAQ,MAAM;CAClC;AACF"}
+{"version":3,"file":"notifier.js","names":[],"sources":["../../../../../../zero-cache/src/services/replicator/notifier.ts"],"sourcesContent":["import {EventEmitter} from 'eventemitter3';\nimport {\n  type PendingResult,\n  type Result,\n  Subscription,\n} from '../../types/subscription.ts';\nimport type {ReplicaState, ReplicaStateNotifier} from './replicator.ts';\n\n/**\n * Handles the semantics of {@link ReplicatorVersionNotifier.subscribe()}\n * notifications, namely:\n *\n * * New subscribers are notified immediately with the latest received\n *   ReplicaState.\n *\n * * Non-latest notifications are discarded if the subscriber is too\n *   busy to consume them.\n *\n * By encapsulating the state for the first behavior (essentially, whether\n * the first notification has been sent by the Replicator), Notifier objects\n * can be chained to simplify fanout from Replicator to View Syncers.\n *\n * In particular, each Syncer Thread can manage a single Subscription to\n * the Replicator across a MessageChannel, which it uses for its own Notifier\n * instance to manage subscriptions from View Syncers within its thread. This\n * way a Replicator only deals with sending notifications to a bounded set\n * of MessageChannel-based subscribers (Syncer Threads), while the dynamic\n * subscribe and unsubscribe traffic from View Syncers remains within each\n * Syncer Thread.\n */\nfunction oldest(\n  curr: number | undefined,\n  prev: number | undefined,\n): number | undefined {\n  if (curr === undefined) {\n    return prev;\n  }\n  if (prev === undefined) {\n    return curr;\n  }\n  return Math.min(curr, prev);\n}\n\nexport class Notifier implements ReplicaStateNotifier {\n  readonly #eventEmitter = new EventEmitter();\n  #lastStateReceived: ReplicaState | undefined;\n\n  get latestState(): ReplicaState | undefined {\n    return this.#lastStateReceived;\n  }\n\n  #newSubscription() {\n    const notify = (state: ReplicaState) => subscription.push(state);\n    const subscription = Subscription.create<ReplicaState>({\n      // A coalesced notification stands in for every notification it\n      // subsumed, so both timestamps keep the *oldest* value: they measure how\n      // long work has been outstanding, and the subsumed watermarks are still\n      // unserved.\n      coalesce: (curr, prev) => ({\n        ...curr,\n        replicaReadyTimeMs: oldest(\n          curr.replicaReadyTimeMs,\n          prev.replicaReadyTimeMs,\n        ),\n        upstreamCommitTimeMs: oldest(\n          curr.upstreamCommitTimeMs,\n          prev.upstreamCommitTimeMs,\n        ),\n      }),\n      cleanup: () => this.#eventEmitter.off('version', notify),\n    });\n    return {notify, subscription};\n  }\n\n  subscribe(): Subscription<ReplicaState> {\n    const {notify, subscription} = this.#newSubscription();\n    this.#eventEmitter.on('version', notify);\n    if (this.#lastStateReceived) {\n      // Per Replicator.subscribe() semantics, the current state of the\n      // replica, if known, is immediately sent on subscribe.\n      notify(this.#lastStateReceived);\n    }\n    return subscription;\n  }\n\n  notifySubscribers(\n    state: ReplicaState = {state: 'version-ready'},\n  ): Promise<Result>[] {\n    this.#lastStateReceived = state;\n    return this.#eventEmitter\n      .listeners('version')\n      .map(notify => notify(state) as unknown as PendingResult)\n      .map(pending => pending.result);\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;AA8BA,SAAS,OACP,MACA,MACoB;CACpB,IAAI,SAAS,KAAA,GACX,OAAO;CAET,IAAI,SAAS,KAAA,GACX,OAAO;CAET,OAAO,KAAK,IAAI,MAAM,IAAI;AAC5B;AAEA,IAAa,WAAb,MAAsD;CACpD,gBAAyB,IAAI,aAAa;CAC1C;CAEA,IAAI,cAAwC;EAC1C,OAAO,KAAK;CACd;CAEA,mBAAmB;EACjB,MAAM,UAAU,UAAwB,aAAa,KAAK,KAAK;EAC/D,MAAM,eAAe,aAAa,OAAqB;GAKrD,WAAW,MAAM,UAAU;IACzB,GAAG;IACH,oBAAoB,OAClB,KAAK,oBACL,KAAK,kBACP;IACA,sBAAsB,OACpB,KAAK,sBACL,KAAK,oBACP;GACF;GACA,eAAe,KAAK,cAAc,IAAI,WAAW,MAAM;EACzD,CAAC;EACD,OAAO;GAAC;GAAQ;EAAY;CAC9B;CAEA,YAAwC;EACtC,MAAM,EAAC,QAAQ,iBAAgB,KAAK,iBAAiB;EACrD,KAAK,cAAc,GAAG,WAAW,MAAM;EACvC,IAAI,KAAK,oBAGP,OAAO,KAAK,kBAAkB;EAEhC,OAAO;CACT;CAEA,kBACE,QAAsB,EAAC,OAAO,gBAAe,GAC1B;EACnB,KAAK,qBAAqB;EAC1B,OAAO,KAAK,cACT,UAAU,SAAS,CAAC,CACpB,KAAI,WAAU,OAAO,KAAK,CAA6B,CAAC,CACxD,KAAI,YAAW,QAAQ,MAAM;CAClC;AACF"}

package/out/zero-cache/src/services/replicator/replication-status.js

@@ -141,7 +141,7 @@ var IndexingProgress = class {
 	}
 };
 async function publishReplicationError(lc, stage, description, errorDetails, now = /* @__PURE__ */ new Date()) {
-	await publishCriticalEvent(lc, {
+	const event = {
 		type: "zero/events/status/replication/v1",
 		component: "replication",
 		status: "ERROR",
@@ -149,7 +149,8 @@ async function publishReplicationError(lc, stage, description, errorDetails, now
 		description,
 		errorDetails,
 		time: now.toISOString()
-	});
+	};
+	await publishCriticalEvent(lc, event);
 }
 function replicationStatusError(lc, stage, e, db, now = /* @__PURE__ */ new Date()) {
 	const event = replicationStatusEvent(lc, db, stage, "ERROR", String(e), now);
@@ -221,6 +222,6 @@ function getReplicaSize(db) {
 	return pageCount * pageSize;
 }
 //#endregion
-export { IndexingProgress, ReplicationStatusPublisher, publishReplicationError, replicationStatusError };
+export { IndexingProgress, ReplicationStatusPublisher, publishReplicationError, replicationStatusError, replicationStatusEvent };
 
 //# sourceMappingURL=replication-status.js.map

package/out/zero-cache/src/services/replicator/replication-status.js.map

@@ -1 +1 @@
-{"version":3,"file":"replication-status.js","names":["#dbRunner","#publishFn","#publish","#timer","#totalIndexes","#now","#count","#current","#completedMs"],"sources":["../../../../../../zero-cache/src/services/replicator/replication-status.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {createSilentLogContext} from '../../../../shared/src/logging-test-utils.ts';\nimport type {JSONObject} from '../../../../zero-events/src/json.ts';\nimport type {\n  IndexingStatus,\n  ReplicatedIndex,\n  ReplicatedTable,\n  ReplicationStage,\n  ReplicationState,\n  ReplicationStatusEvent,\n  Status,\n} from '../../../../zero-events/src/status.ts';\nimport {Database} from '../../../../zqlite/src/db.ts';\nimport {computeZqlSpecs, listIndexes} from '../../db/lite-tables.ts';\nimport type {LiteIndexSpec, LiteTableSpec} from '../../db/specs.ts';\nimport {\n  makeErrorDetails,\n  publishCriticalEvent,\n} from '../../observability/events.ts';\n\nconst byKeys = (a: [string, unknown], b: [string, unknown]) =>\n  a[0] < b[0] ? -1 : a[0] > b[0] ? 1 : 0;\n\nexport class ReplicationStatusPublisher {\n  readonly #dbRunner: <T>(lc: LogContext, fn: (db: Database) => T) => T;\n  readonly #publishFn: typeof publishCriticalEvent;\n  #timer: NodeJS.Timeout | undefined;\n\n  static forTesting() {\n    return ReplicationStatusPublisher.forReplicaFile(':memory:');\n  }\n\n  static forRunningTransaction(tx: Database, publishFn = publishCriticalEvent) {\n    return new ReplicationStatusPublisher((_lc, fn) => fn(tx), publishFn);\n  }\n\n  static forReplicaFile(file: string, publishFn = publishCriticalEvent) {\n    return new ReplicationStatusPublisher((lc, fn) => {\n      const db = new Database(lc, file, {readonly: true});\n      try {\n        return fn(db);\n      } finally {\n        db.close();\n      }\n    }, publishFn);\n  }\n\n  constructor(\n    dbRunner: <T>(lc: LogContext, fn: (db: Database) => T) => T,\n    publishFn = publishCriticalEvent,\n  ) {\n    this.#dbRunner = dbRunner;\n    this.#publishFn = publishFn;\n  }\n\n  publish(\n    lc: LogContext,\n    stage: ReplicationStage,\n    description?: string,\n    interval = 0,\n    extraState?: () => Partial<ReplicationState>,\n    now = new Date(),\n  ): this {\n    void this.#publish(lc, stage, description, interval, extraState, now);\n    return this;\n  }\n\n  /**\n   * Like {@link publish}, but waits (for at most `maxWaitMs`) for the event\n   * to be sent. Use this before a long synchronous operation (e.g. creating\n   * an index) that blocks the event loop and would otherwise prevent the\n   * event from being sent until the operation completes.\n   */\n  async publishAndFlush(\n    lc: LogContext,\n    stage: ReplicationStage,\n    description?: string,\n    interval = 0,\n    extraState?: () => Partial<ReplicationState>,\n    maxWaitMs = 1000,\n  ): Promise<void> {\n    const published = this.#publish(\n      lc,\n      stage,\n      description,\n      interval,\n      extraState,\n    );\n    let timer: NodeJS.Timeout | undefined;\n    try {\n      await Promise.race([\n        published,\n        new Promise<void>(resolve => {\n          timer = setTimeout(resolve, maxWaitMs);\n        }),\n      ]);\n    } finally {\n      clearTimeout(timer);\n    }\n  }\n\n  #publish(\n    lc: LogContext,\n    stage: ReplicationStage,\n    description: string | undefined,\n    interval: number,\n    extraState: (() => Partial<ReplicationState>) | undefined,\n    now = new Date(),\n  ): Promise<void> {\n    this.stop();\n\n    const event = this.#dbRunner(lc, db =>\n      replicationStatusEvent(lc, db, stage, 'OK', description, now),\n    );\n    if (event.state) {\n      event.state = {\n        ...event.state,\n        ...extraState?.(),\n      };\n    }\n    const published = this.#publishFn(lc, event);\n\n    if (interval) {\n      this.#timer = setInterval(\n        () => this.publish(lc, stage, description, interval, extraState),\n        interval,\n      );\n    }\n    return published;\n  }\n\n  async publishAndThrowError(\n    lc: LogContext,\n    stage: ReplicationStage,\n    e: unknown,\n  ): Promise<never> {\n    this.stop();\n    const event = this.#dbRunner(lc, db =>\n      replicationStatusError(lc, stage, e, db),\n    );\n    await this.#publishFn(lc, event);\n    throw e;\n  }\n\n  stop(): this {\n    clearInterval(this.#timer);\n    return this;\n  }\n}\n\n/**\n * Tracks the progress of a sequence of index creations for reporting in\n * the `indexingStatus` field of {@link ReplicationState}.\n */\nexport class IndexingProgress {\n  readonly #totalIndexes: number | undefined;\n  readonly #now: () => number;\n  #count = 0;\n  #completedMs = 0;\n  #current:\n    | {index: LiteIndexSpec; start: number; elapsedMs: number | undefined}\n    | undefined;\n\n  /**\n   * @param totalIndexes The total number of indexes to be created, if known.\n   */\n  constructor(totalIndexes?: number, now = () => performance.now()) {\n    this.#totalIndexes = totalIndexes;\n    this.#now = now;\n  }\n\n  /** Marks the start of creating the `index`. */\n  start(index: LiteIndexSpec) {\n    this.#count++;\n    this.#current = {index, start: this.#now(), elapsedMs: undefined};\n  }\n\n  /**\n   * Restarts the timer of the current index, e.g. to exclude time spent\n   * reporting its start from its creation time.\n   */\n  restartTimer() {\n    if (this.#current) {\n      this.#current.start = this.#now();\n    }\n  }\n\n  /**\n   * Marks the completion of the index passed to the last call to\n   * {@link start}, returning the milliseconds it took to create.\n   */\n  finish(): number {\n    const current = this.#current;\n    if (!current) {\n      return 0;\n    }\n    if (current.elapsedMs === undefined) {\n      current.elapsedMs = this.#now() - current.start;\n      this.#completedMs += current.elapsedMs;\n    }\n    return current.elapsedMs;\n  }\n\n  /** The `extraState` for {@link ReplicationStatusPublisher.publish}. */\n  readonly state = (): Partial<ReplicationState> => {\n    const status = this.status();\n    return status ? {indexingStatus: status} : {};\n  };\n\n  status(): IndexingStatus | undefined {\n    const current = this.#current;\n    if (!current) {\n      return undefined;\n    }\n    const {index, start, elapsedMs} = current;\n    const done = elapsedMs !== undefined;\n    return {\n      name: index.name,\n      table: index.tableName,\n      columns: Object.keys(index.columns),\n      unique: index.unique,\n      index: this.#count,\n      totalIndexes: this.#totalIndexes,\n      elapsedMs: Math.round(done ? elapsedMs : this.#now() - start),\n      completedMs: Math.round(\n        done ? this.#completedMs - elapsedMs : this.#completedMs,\n      ),\n      done,\n    };\n  }\n}\n\nexport async function publishReplicationError(\n  lc: LogContext,\n  stage: ReplicationStage,\n  description: string,\n  errorDetails?: JSONObject,\n  now = new Date(),\n) {\n  const event: ReplicationStatusEvent = {\n    type: 'zero/events/status/replication/v1',\n    component: 'replication',\n    status: 'ERROR',\n    stage,\n    description,\n    errorDetails,\n    time: now.toISOString(),\n  };\n  await publishCriticalEvent(lc, event);\n}\n\nexport function replicationStatusError(\n  lc: LogContext,\n  stage: ReplicationStage,\n  e: unknown,\n  db?: Database,\n  now = new Date(),\n) {\n  const event = replicationStatusEvent(lc, db, stage, 'ERROR', String(e), now);\n  event.errorDetails = makeErrorDetails(e);\n  return event;\n}\n\n// Exported for testing.\nexport function replicationStatusEvent(\n  lc: LogContext,\n  db: Database | undefined,\n  stage: ReplicationStage,\n  status: Status,\n  description?: string,\n  now = new Date(),\n): ReplicationStatusEvent {\n  const start = performance.now();\n  try {\n    return {\n      type: 'zero/events/status/replication/v1',\n      component: 'replication',\n      status,\n      stage,\n      description,\n      time: now.toISOString(),\n      state: {\n        tables: db ? getReplicatedTables(db) : [],\n        indexes: db ? getReplicatedIndexes(db) : [],\n        replicaSize: db ? getReplicaSize(db) : undefined,\n      },\n    };\n  } catch (e) {\n    lc.warn?.(`Unable to create full ReplicationStatusEvent`, e);\n    return {\n      type: 'zero/events/status/replication/v1',\n      component: 'replication',\n      status,\n      stage,\n      description,\n      time: now.toISOString(),\n      state: {\n        tables: [],\n        indexes: [],\n        replicaSize: 0,\n      },\n    };\n  } finally {\n    const elapsed = (performance.now() - start).toFixed(3);\n    lc.debug?.(`computed schema for replication event (${elapsed} ms)`);\n  }\n}\n\nfunction getReplicatedTables(db: Database): ReplicatedTable[] {\n  const fullTables = new Map<string, LiteTableSpec>();\n  const clientSchema = computeZqlSpecs(\n    createSilentLogContext(), // avoid logging warnings about indexes\n    db,\n    {includeBackfillingColumns: false},\n    new Map(),\n    fullTables,\n  );\n\n  // oxlint-disable-next-line e18e/prefer-array-to-sorted\n  return [...fullTables.entries()].sort(byKeys).map(([table, spec]) => ({\n    table,\n    columns: Object.entries(spec.columns)\n      .sort(byKeys)\n      .map(([column, spec]) => ({\n        column,\n        upstreamType: spec.dataType.split('|')[0],\n        clientType: clientSchema.get(table)?.zqlSpec[column]?.type ?? null,\n      })),\n  }));\n}\n\nfunction getReplicatedIndexes(db: Database): ReplicatedIndex[] {\n  return listIndexes(db).map(({name, tableName: table, columns, unique}) => ({\n    name,\n    table,\n    unique,\n    columns: Object.entries(columns).map(([column, dir]) => ({column, dir})),\n  }));\n}\n\nfunction getReplicaSize(db: Database) {\n  const [{page_count: pageCount}] = db.pragma<{page_count: number}>(\n    'page_count',\n  );\n  const [{page_size: pageSize}] = db.pragma<{page_size: number}>('page_size');\n  return pageCount * pageSize;\n}\n"],"mappings":";;;;;AAoBA,IAAM,UAAU,GAAsB,MACpC,EAAE,KAAK,EAAE,KAAK,KAAK,EAAE,KAAK,EAAE,KAAK,IAAI;AAEvC,IAAa,6BAAb,MAAa,2BAA2B;CACtC;CACA;CACA;CAEA,OAAO,aAAa;EAClB,OAAO,2BAA2B,eAAe,UAAU;CAC7D;CAEA,OAAO,sBAAsB,IAAc,YAAY,sBAAsB;EAC3E,OAAO,IAAI,4BAA4B,KAAK,OAAO,GAAG,EAAE,GAAG,SAAS;CACtE;CAEA,OAAO,eAAe,MAAc,YAAY,sBAAsB;EACpE,OAAO,IAAI,4BAA4B,IAAI,OAAO;GAChD,MAAM,KAAK,IAAI,SAAS,IAAI,MAAM,EAAC,UAAU,KAAI,CAAC;GAClD,IAAI;IACF,OAAO,GAAG,EAAE;GACd,UAAU;IACR,GAAG,MAAM;GACX;EACF,GAAG,SAAS;CACd;CAEA,YACE,UACA,YAAY,sBACZ;EACA,KAAKA,YAAY;EACjB,KAAKC,aAAa;CACpB;CAEA,QACE,IACA,OACA,aACA,WAAW,GACX,YACA,sBAAM,IAAI,KAAK,GACT;EACN,KAAUC,SAAS,IAAI,OAAO,aAAa,UAAU,YAAY,GAAG;EACpE,OAAO;CACT;;;;;;;CAQA,MAAM,gBACJ,IACA,OACA,aACA,WAAW,GACX,YACA,YAAY,KACG;EACf,MAAM,YAAY,KAAKA,SACrB,IACA,OACA,aACA,UACA,UACF;EACA,IAAI;EACJ,IAAI;GACF,MAAM,QAAQ,KAAK,CACjB,WACA,IAAI,SAAc,YAAW;IAC3B,QAAQ,WAAW,SAAS,SAAS;GACvC,CAAC,CACH,CAAC;EACH,UAAU;GACR,aAAa,KAAK;EACpB;CACF;CAEA,SACE,IACA,OACA,aACA,UACA,YACA,sBAAM,IAAI,KAAK,GACA;EACf,KAAK,KAAK;EAEV,MAAM,QAAQ,KAAKF,UAAU,KAAI,OAC/B,uBAAuB,IAAI,IAAI,OAAO,MAAM,aAAa,GAAG,CAC9D;EACA,IAAI,MAAM,OACR,MAAM,QAAQ;GACZ,GAAG,MAAM;GACT,GAAG,aAAa;EAClB;EAEF,MAAM,YAAY,KAAKC,WAAW,IAAI,KAAK;EAE3C,IAAI,UACF,KAAKE,SAAS,kBACN,KAAK,QAAQ,IAAI,OAAO,aAAa,UAAU,UAAU,GAC/D,QACF;EAEF,OAAO;CACT;CAEA,MAAM,qBACJ,IACA,OACA,GACgB;EAChB,KAAK,KAAK;EACV,MAAM,QAAQ,KAAKH,UAAU,KAAI,OAC/B,uBAAuB,IAAI,OAAO,GAAG,EAAE,CACzC;EACA,MAAM,KAAKC,WAAW,IAAI,KAAK;EAC/B,MAAM;CACR;CAEA,OAAa;EACX,cAAc,KAAKE,MAAM;EACzB,OAAO;CACT;AACF;;;;;AAMA,IAAa,mBAAb,MAA8B;CAC5B;CACA;CACA,SAAS;CACT,eAAe;CACf;;;;CAOA,YAAY,cAAuB,YAAY,YAAY,IAAI,GAAG;EAChE,KAAKC,gBAAgB;EACrB,KAAKC,OAAO;CACd;;CAGA,MAAM,OAAsB;EAC1B,KAAKC;EACL,KAAKC,WAAW;GAAC;GAAO,OAAO,KAAKF,KAAK;GAAG,WAAW,KAAA;EAAS;CAClE;;;;;CAMA,eAAe;EACb,IAAI,KAAKE,UACP,KAAKA,SAAS,QAAQ,KAAKF,KAAK;CAEpC;;;;;CAMA,SAAiB;EACf,MAAM,UAAU,KAAKE;EACrB,IAAI,CAAC,SACH,OAAO;EAET,IAAI,QAAQ,cAAc,KAAA,GAAW;GACnC,QAAQ,YAAY,KAAKF,KAAK,IAAI,QAAQ;GAC1C,KAAKG,gBAAgB,QAAQ;EAC/B;EACA,OAAO,QAAQ;CACjB;;CAGA,cAAkD;EAChD,MAAM,SAAS,KAAK,OAAO;EAC3B,OAAO,SAAS,EAAC,gBAAgB,OAAM,IAAI,CAAC;CAC9C;CAEA,SAAqC;EACnC,MAAM,UAAU,KAAKD;EACrB,IAAI,CAAC,SACH;EAEF,MAAM,EAAC,OAAO,OAAO,cAAa;EAClC,MAAM,OAAO,cAAc,KAAA;EAC3B,OAAO;GACL,MAAM,MAAM;GACZ,OAAO,MAAM;GACb,SAAS,OAAO,KAAK,MAAM,OAAO;GAClC,QAAQ,MAAM;GACd,OAAO,KAAKD;GACZ,cAAc,KAAKF;GACnB,WAAW,KAAK,MAAM,OAAO,YAAY,KAAKC,KAAK,IAAI,KAAK;GAC5D,aAAa,KAAK,MAChB,OAAO,KAAKG,eAAe,YAAY,KAAKA,YAC9C;GACA;EACF;CACF;AACF;AAEA,eAAsB,wBACpB,IACA,OACA,aACA,cACA,sBAAM,IAAI,KAAK,GACf;CAUA,MAAM,qBAAqB,IAAI;EAR7B,MAAM;EACN,WAAW;EACX,QAAQ;EACR;EACA;EACA;EACA,MAAM,IAAI,YAAY;CAEO,CAAK;AACtC;AAEA,SAAgB,uBACd,IACA,OACA,GACA,IACA,sBAAM,IAAI,KAAK,GACf;CACA,MAAM,QAAQ,uBAAuB,IAAI,IAAI,OAAO,SAAS,OAAO,CAAC,GAAG,GAAG;CAC3E,MAAM,eAAe,iBAAiB,CAAC;CACvC,OAAO;AACT;AAGA,SAAgB,uBACd,IACA,IACA,OACA,QACA,aACA,sBAAM,IAAI,KAAK,GACS;CACxB,MAAM,QAAQ,YAAY,IAAI;CAC9B,IAAI;EACF,OAAO;GACL,MAAM;GACN,WAAW;GACX;GACA;GACA;GACA,MAAM,IAAI,YAAY;GACtB,OAAO;IACL,QAAQ,KAAK,oBAAoB,EAAE,IAAI,CAAC;IACxC,SAAS,KAAK,qBAAqB,EAAE,IAAI,CAAC;IAC1C,aAAa,KAAK,eAAe,EAAE,IAAI,KAAA;GACzC;EACF;CACF,SAAS,GAAG;EACV,GAAG,OAAO,gDAAgD,CAAC;EAC3D,OAAO;GACL,MAAM;GACN,WAAW;GACX;GACA;GACA;GACA,MAAM,IAAI,YAAY;GACtB,OAAO;IACL,QAAQ,CAAC;IACT,SAAS,CAAC;IACV,aAAa;GACf;EACF;CACF,UAAU;EACR,MAAM,WAAW,YAAY,IAAI,IAAI,OAAO,QAAQ,CAAC;EACrD,GAAG,QAAQ,0CAA0C,QAAQ,KAAK;CACpE;AACF;AAEA,SAAS,oBAAoB,IAAiC;CAC5D,MAAM,6BAAa,IAAI,IAA2B;CAClD,MAAM,eAAe,gBACnB,uBAAuB,GACvB,IACA,EAAC,2BAA2B,MAAK,mBACjC,IAAI,IAAI,GACR,UACF;CAGA,OAAO,CAAC,GAAG,WAAW,QAAQ,CAAC,EAAE,KAAK,MAAM,EAAE,KAAK,CAAC,OAAO,WAAW;EACpE;EACA,SAAS,OAAO,QAAQ,KAAK,OAAO,EACjC,KAAK,MAAM,EACX,KAAK,CAAC,QAAQ,WAAW;GACxB;GACA,cAAc,KAAK,SAAS,MAAM,GAAG,EAAE;GACvC,YAAY,aAAa,IAAI,KAAK,GAAG,QAAQ,SAAS,QAAQ;EAChE,EAAE;CACN,EAAE;AACJ;AAEA,SAAS,qBAAqB,IAAiC;CAC7D,OAAO,YAAY,EAAE,EAAE,KAAK,EAAC,MAAM,WAAW,OAAO,SAAS,cAAa;EACzE;EACA;EACA;EACA,SAAS,OAAO,QAAQ,OAAO,EAAE,KAAK,CAAC,QAAQ,UAAU;GAAC;GAAQ;EAAG,EAAE;CACzE,EAAE;AACJ;AAEA,SAAS,eAAe,IAAc;CACpC,MAAM,CAAC,EAAC,YAAY,eAAc,GAAG,OACnC,YACF;CACA,MAAM,CAAC,EAAC,WAAW,cAAa,GAAG,OAA4B,WAAW;CAC1E,OAAO,YAAY;AACrB"}
+{"version":3,"file":"replication-status.js","names":[],"sources":["../../../../../../zero-cache/src/services/replicator/replication-status.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {createSilentLogContext} from '../../../../shared/src/logging-test-utils.ts';\nimport type {JSONObject} from '../../../../zero-events/src/json.ts';\nimport type {\n  IndexingStatus,\n  ReplicatedIndex,\n  ReplicatedTable,\n  ReplicationStage,\n  ReplicationState,\n  ReplicationStatusEvent,\n  Status,\n} from '../../../../zero-events/src/status.ts';\nimport {Database} from '../../../../zqlite/src/db.ts';\nimport {computeZqlSpecs, listIndexes} from '../../db/lite-tables.ts';\nimport type {LiteIndexSpec, LiteTableSpec} from '../../db/specs.ts';\nimport {\n  makeErrorDetails,\n  publishCriticalEvent,\n} from '../../observability/events.ts';\n\nconst byKeys = (a: [string, unknown], b: [string, unknown]) =>\n  a[0] < b[0] ? -1 : a[0] > b[0] ? 1 : 0;\n\nexport class ReplicationStatusPublisher {\n  readonly #dbRunner: <T>(lc: LogContext, fn: (db: Database) => T) => T;\n  readonly #publishFn: typeof publishCriticalEvent;\n  #timer: NodeJS.Timeout | undefined;\n\n  static forTesting() {\n    return ReplicationStatusPublisher.forReplicaFile(':memory:');\n  }\n\n  static forRunningTransaction(tx: Database, publishFn = publishCriticalEvent) {\n    return new ReplicationStatusPublisher((_lc, fn) => fn(tx), publishFn);\n  }\n\n  static forReplicaFile(file: string, publishFn = publishCriticalEvent) {\n    return new ReplicationStatusPublisher((lc, fn) => {\n      const db = new Database(lc, file, {readonly: true});\n      try {\n        return fn(db);\n      } finally {\n        db.close();\n      }\n    }, publishFn);\n  }\n\n  constructor(\n    dbRunner: <T>(lc: LogContext, fn: (db: Database) => T) => T,\n    publishFn = publishCriticalEvent,\n  ) {\n    this.#dbRunner = dbRunner;\n    this.#publishFn = publishFn;\n  }\n\n  publish(\n    lc: LogContext,\n    stage: ReplicationStage,\n    description?: string,\n    interval = 0,\n    extraState?: () => Partial<ReplicationState>,\n    now = new Date(),\n  ): this {\n    void this.#publish(lc, stage, description, interval, extraState, now);\n    return this;\n  }\n\n  /**\n   * Like {@link publish}, but waits (for at most `maxWaitMs`) for the event\n   * to be sent. Use this before a long synchronous operation (e.g. creating\n   * an index) that blocks the event loop and would otherwise prevent the\n   * event from being sent until the operation completes.\n   */\n  async publishAndFlush(\n    lc: LogContext,\n    stage: ReplicationStage,\n    description?: string,\n    interval = 0,\n    extraState?: () => Partial<ReplicationState>,\n    maxWaitMs = 1000,\n  ): Promise<void> {\n    const published = this.#publish(\n      lc,\n      stage,\n      description,\n      interval,\n      extraState,\n    );\n    let timer: NodeJS.Timeout | undefined;\n    try {\n      await Promise.race([\n        published,\n        new Promise<void>(resolve => {\n          timer = setTimeout(resolve, maxWaitMs);\n        }),\n      ]);\n    } finally {\n      clearTimeout(timer);\n    }\n  }\n\n  #publish(\n    lc: LogContext,\n    stage: ReplicationStage,\n    description: string | undefined,\n    interval: number,\n    extraState: (() => Partial<ReplicationState>) | undefined,\n    now = new Date(),\n  ): Promise<void> {\n    this.stop();\n\n    const event = this.#dbRunner(lc, db =>\n      replicationStatusEvent(lc, db, stage, 'OK', description, now),\n    );\n    if (event.state) {\n      event.state = {\n        ...event.state,\n        ...extraState?.(),\n      };\n    }\n    const published = this.#publishFn(lc, event);\n\n    if (interval) {\n      this.#timer = setInterval(\n        () => this.publish(lc, stage, description, interval, extraState),\n        interval,\n      );\n    }\n    return published;\n  }\n\n  async publishAndThrowError(\n    lc: LogContext,\n    stage: ReplicationStage,\n    e: unknown,\n  ): Promise<never> {\n    this.stop();\n    const event = this.#dbRunner(lc, db =>\n      replicationStatusError(lc, stage, e, db),\n    );\n    await this.#publishFn(lc, event);\n    throw e;\n  }\n\n  stop(): this {\n    clearInterval(this.#timer);\n    return this;\n  }\n}\n\n/**\n * Tracks the progress of a sequence of index creations for reporting in\n * the `indexingStatus` field of {@link ReplicationState}.\n */\nexport class IndexingProgress {\n  readonly #totalIndexes: number | undefined;\n  readonly #now: () => number;\n  #count = 0;\n  #completedMs = 0;\n  #current:\n    | {index: LiteIndexSpec; start: number; elapsedMs: number | undefined}\n    | undefined;\n\n  /**\n   * @param totalIndexes The total number of indexes to be created, if known.\n   */\n  constructor(totalIndexes?: number, now = () => performance.now()) {\n    this.#totalIndexes = totalIndexes;\n    this.#now = now;\n  }\n\n  /** Marks the start of creating the `index`. */\n  start(index: LiteIndexSpec) {\n    this.#count++;\n    this.#current = {index, start: this.#now(), elapsedMs: undefined};\n  }\n\n  /**\n   * Restarts the timer of the current index, e.g. to exclude time spent\n   * reporting its start from its creation time.\n   */\n  restartTimer() {\n    if (this.#current) {\n      this.#current.start = this.#now();\n    }\n  }\n\n  /**\n   * Marks the completion of the index passed to the last call to\n   * {@link start}, returning the milliseconds it took to create.\n   */\n  finish(): number {\n    const current = this.#current;\n    if (!current) {\n      return 0;\n    }\n    if (current.elapsedMs === undefined) {\n      current.elapsedMs = this.#now() - current.start;\n      this.#completedMs += current.elapsedMs;\n    }\n    return current.elapsedMs;\n  }\n\n  /** The `extraState` for {@link ReplicationStatusPublisher.publish}. */\n  readonly state = (): Partial<ReplicationState> => {\n    const status = this.status();\n    return status ? {indexingStatus: status} : {};\n  };\n\n  status(): IndexingStatus | undefined {\n    const current = this.#current;\n    if (!current) {\n      return undefined;\n    }\n    const {index, start, elapsedMs} = current;\n    const done = elapsedMs !== undefined;\n    return {\n      name: index.name,\n      table: index.tableName,\n      columns: Object.keys(index.columns),\n      unique: index.unique,\n      index: this.#count,\n      totalIndexes: this.#totalIndexes,\n      elapsedMs: Math.round(done ? elapsedMs : this.#now() - start),\n      completedMs: Math.round(\n        done ? this.#completedMs - elapsedMs : this.#completedMs,\n      ),\n      done,\n    };\n  }\n}\n\nexport async function publishReplicationError(\n  lc: LogContext,\n  stage: ReplicationStage,\n  description: string,\n  errorDetails?: JSONObject,\n  now = new Date(),\n) {\n  const event: ReplicationStatusEvent = {\n    type: 'zero/events/status/replication/v1',\n    component: 'replication',\n    status: 'ERROR',\n    stage,\n    description,\n    errorDetails,\n    time: now.toISOString(),\n  };\n  await publishCriticalEvent(lc, event);\n}\n\nexport function replicationStatusError(\n  lc: LogContext,\n  stage: ReplicationStage,\n  e: unknown,\n  db?: Database,\n  now = new Date(),\n) {\n  const event = replicationStatusEvent(lc, db, stage, 'ERROR', String(e), now);\n  event.errorDetails = makeErrorDetails(e);\n  return event;\n}\n\n// Exported for testing.\nexport function replicationStatusEvent(\n  lc: LogContext,\n  db: Database | undefined,\n  stage: ReplicationStage,\n  status: Status,\n  description?: string,\n  now = new Date(),\n): ReplicationStatusEvent {\n  const start = performance.now();\n  try {\n    return {\n      type: 'zero/events/status/replication/v1',\n      component: 'replication',\n      status,\n      stage,\n      description,\n      time: now.toISOString(),\n      state: {\n        tables: db ? getReplicatedTables(db) : [],\n        indexes: db ? getReplicatedIndexes(db) : [],\n        replicaSize: db ? getReplicaSize(db) : undefined,\n      },\n    };\n  } catch (e) {\n    lc.warn?.(`Unable to create full ReplicationStatusEvent`, e);\n    return {\n      type: 'zero/events/status/replication/v1',\n      component: 'replication',\n      status,\n      stage,\n      description,\n      time: now.toISOString(),\n      state: {\n        tables: [],\n        indexes: [],\n        replicaSize: 0,\n      },\n    };\n  } finally {\n    const elapsed = (performance.now() - start).toFixed(3);\n    lc.debug?.(`computed schema for replication event (${elapsed} ms)`);\n  }\n}\n\nfunction getReplicatedTables(db: Database): ReplicatedTable[] {\n  const fullTables = new Map<string, LiteTableSpec>();\n  const clientSchema = computeZqlSpecs(\n    createSilentLogContext(), // avoid logging warnings about indexes\n    db,\n    {includeBackfillingColumns: false},\n    new Map(),\n    fullTables,\n  );\n\n  // oxlint-disable-next-line e18e/prefer-array-to-sorted\n  return [...fullTables.entries()].sort(byKeys).map(([table, spec]) => ({\n    table,\n    columns: Object.entries(spec.columns)\n      .sort(byKeys)\n      .map(([column, spec]) => ({\n        column,\n        upstreamType: spec.dataType.split('|')[0],\n        clientType: clientSchema.get(table)?.zqlSpec[column]?.type ?? null,\n      })),\n  }));\n}\n\nfunction getReplicatedIndexes(db: Database): ReplicatedIndex[] {\n  return listIndexes(db).map(({name, tableName: table, columns, unique}) => ({\n    name,\n    table,\n    unique,\n    columns: Object.entries(columns).map(([column, dir]) => ({column, dir})),\n  }));\n}\n\nfunction getReplicaSize(db: Database) {\n  const [{page_count: pageCount}] = db.pragma<{page_count: number}>(\n    'page_count',\n  );\n  const [{page_size: pageSize}] = db.pragma<{page_size: number}>('page_size');\n  return pageCount * pageSize;\n}\n"],"mappings":";;;;;AAoBA,IAAM,UAAU,GAAsB,MACpC,EAAE,KAAK,EAAE,KAAK,KAAK,EAAE,KAAK,EAAE,KAAK,IAAI;AAEvC,IAAa,6BAAb,MAAa,2BAA2B;CACtC;CACA;CACA;CAEA,OAAO,aAAa;EAClB,OAAO,2BAA2B,eAAe,UAAU;CAC7D;CAEA,OAAO,sBAAsB,IAAc,YAAY,sBAAsB;EAC3E,OAAO,IAAI,4BAA4B,KAAK,OAAO,GAAG,EAAE,GAAG,SAAS;CACtE;CAEA,OAAO,eAAe,MAAc,YAAY,sBAAsB;EACpE,OAAO,IAAI,4BAA4B,IAAI,OAAO;GAChD,MAAM,KAAK,IAAI,SAAS,IAAI,MAAM,EAAC,UAAU,KAAI,CAAC;GAClD,IAAI;IACF,OAAO,GAAG,EAAE;GACd,UAAU;IACR,GAAG,MAAM;GACX;EACF,GAAG,SAAS;CACd;CAEA,YACE,UACA,YAAY,sBACZ;EACA,KAAK,YAAY;EACjB,KAAK,aAAa;CACpB;CAEA,QACE,IACA,OACA,aACA,WAAW,GACX,YACA,sBAAM,IAAI,KAAK,GACT;EACN,KAAU,SAAS,IAAI,OAAO,aAAa,UAAU,YAAY,GAAG;EACpE,OAAO;CACT;;;;;;;CAQA,MAAM,gBACJ,IACA,OACA,aACA,WAAW,GACX,YACA,YAAY,KACG;EACf,MAAM,YAAY,KAAK,SACrB,IACA,OACA,aACA,UACA,UACF;EACA,IAAI;EACJ,IAAI;GACF,MAAM,QAAQ,KAAK,CACjB,WACA,IAAI,SAAc,YAAW;IAC3B,QAAQ,WAAW,SAAS,SAAS;GACvC,CAAC,CACH,CAAC;EACH,UAAU;GACR,aAAa,KAAK;EACpB;CACF;CAEA,SACE,IACA,OACA,aACA,UACA,YACA,sBAAM,IAAI,KAAK,GACA;EACf,KAAK,KAAK;EAEV,MAAM,QAAQ,KAAK,UAAU,KAAI,OAC/B,uBAAuB,IAAI,IAAI,OAAO,MAAM,aAAa,GAAG,CAC9D;EACA,IAAI,MAAM,OACR,MAAM,QAAQ;GACZ,GAAG,MAAM;GACT,GAAG,aAAa;EAClB;EAEF,MAAM,YAAY,KAAK,WAAW,IAAI,KAAK;EAE3C,IAAI,UACF,KAAK,SAAS,kBACN,KAAK,QAAQ,IAAI,OAAO,aAAa,UAAU,UAAU,GAC/D,QACF;EAEF,OAAO;CACT;CAEA,MAAM,qBACJ,IACA,OACA,GACgB;EAChB,KAAK,KAAK;EACV,MAAM,QAAQ,KAAK,UAAU,KAAI,OAC/B,uBAAuB,IAAI,OAAO,GAAG,EAAE,CACzC;EACA,MAAM,KAAK,WAAW,IAAI,KAAK;EAC/B,MAAM;CACR;CAEA,OAAa;EACX,cAAc,KAAK,MAAM;EACzB,OAAO;CACT;AACF;;;;;AAMA,IAAa,mBAAb,MAA8B;CAC5B;CACA;CACA,SAAS;CACT,eAAe;CACf;;;;CAOA,YAAY,cAAuB,YAAY,YAAY,IAAI,GAAG;EAChE,KAAK,gBAAgB;EACrB,KAAK,OAAO;CACd;;CAGA,MAAM,OAAsB;EAC1B,KAAK;EACL,KAAK,WAAW;GAAC;GAAO,OAAO,KAAK,KAAK;GAAG,WAAW,KAAA;EAAS;CAClE;;;;;CAMA,eAAe;EACb,IAAI,KAAK,UACP,KAAK,SAAS,QAAQ,KAAK,KAAK;CAEpC;;;;;CAMA,SAAiB;EACf,MAAM,UAAU,KAAK;EACrB,IAAI,CAAC,SACH,OAAO;EAET,IAAI,QAAQ,cAAc,KAAA,GAAW;GACnC,QAAQ,YAAY,KAAK,KAAK,IAAI,QAAQ;GAC1C,KAAK,gBAAgB,QAAQ;EAC/B;EACA,OAAO,QAAQ;CACjB;;CAGA,cAAkD;EAChD,MAAM,SAAS,KAAK,OAAO;EAC3B,OAAO,SAAS,EAAC,gBAAgB,OAAM,IAAI,CAAC;CAC9C;CAEA,SAAqC;EACnC,MAAM,UAAU,KAAK;EACrB,IAAI,CAAC,SACH;EAEF,MAAM,EAAC,OAAO,OAAO,cAAa;EAClC,MAAM,OAAO,cAAc,KAAA;EAC3B,OAAO;GACL,MAAM,MAAM;GACZ,OAAO,MAAM;GACb,SAAS,OAAO,KAAK,MAAM,OAAO;GAClC,QAAQ,MAAM;GACd,OAAO,KAAK;GACZ,cAAc,KAAK;GACnB,WAAW,KAAK,MAAM,OAAO,YAAY,KAAK,KAAK,IAAI,KAAK;GAC5D,aAAa,KAAK,MAChB,OAAO,KAAK,eAAe,YAAY,KAAK,YAC9C;GACA;EACF;CACF;AACF;AAEA,eAAsB,wBACpB,IACA,OACA,aACA,cACA,sBAAM,IAAI,KAAK,GACf;CACA,MAAM,QAAgC;EACpC,MAAM;EACN,WAAW;EACX,QAAQ;EACR;EACA;EACA;EACA,MAAM,IAAI,YAAY;CACxB;CACA,MAAM,qBAAqB,IAAI,KAAK;AACtC;AAEA,SAAgB,uBACd,IACA,OACA,GACA,IACA,sBAAM,IAAI,KAAK,GACf;CACA,MAAM,QAAQ,uBAAuB,IAAI,IAAI,OAAO,SAAS,OAAO,CAAC,GAAG,GAAG;CAC3E,MAAM,eAAe,iBAAiB,CAAC;CACvC,OAAO;AACT;AAGA,SAAgB,uBACd,IACA,IACA,OACA,QACA,aACA,sBAAM,IAAI,KAAK,GACS;CACxB,MAAM,QAAQ,YAAY,IAAI;CAC9B,IAAI;EACF,OAAO;GACL,MAAM;GACN,WAAW;GACX;GACA;GACA;GACA,MAAM,IAAI,YAAY;GACtB,OAAO;IACL,QAAQ,KAAK,oBAAoB,EAAE,IAAI,CAAC;IACxC,SAAS,KAAK,qBAAqB,EAAE,IAAI,CAAC;IAC1C,aAAa,KAAK,eAAe,EAAE,IAAI,KAAA;GACzC;EACF;CACF,SAAS,GAAG;EACV,GAAG,OAAO,gDAAgD,CAAC;EAC3D,OAAO;GACL,MAAM;GACN,WAAW;GACX;GACA;GACA;GACA,MAAM,IAAI,YAAY;GACtB,OAAO;IACL,QAAQ,CAAC;IACT,SAAS,CAAC;IACV,aAAa;GACf;EACF;CACF,UAAU;EACR,MAAM,WAAW,YAAY,IAAI,IAAI,MAAA,CAAO,QAAQ,CAAC;EACrD,GAAG,QAAQ,0CAA0C,QAAQ,KAAK;CACpE;AACF;AAEA,SAAS,oBAAoB,IAAiC;CAC5D,MAAM,6BAAa,IAAI,IAA2B;CAClD,MAAM,eAAe,gBACnB,uBAAuB,GACvB,IACA,EAAC,2BAA2B,MAAK,mBACjC,IAAI,IAAI,GACR,UACF;CAGA,OAAO,CAAC,GAAG,WAAW,QAAQ,CAAC,CAAC,CAAC,KAAK,MAAM,CAAC,CAAC,KAAK,CAAC,OAAO,WAAW;EACpE;EACA,SAAS,OAAO,QAAQ,KAAK,OAAO,CAAC,CAClC,KAAK,MAAM,CAAC,CACZ,KAAK,CAAC,QAAQ,WAAW;GACxB;GACA,cAAc,KAAK,SAAS,MAAM,GAAG,CAAC,CAAC;GACvC,YAAY,aAAa,IAAI,KAAK,CAAC,EAAE,QAAQ,OAAO,EAAE,QAAQ;EAChE,EAAE;CACN,EAAE;AACJ;AAEA,SAAS,qBAAqB,IAAiC;CAC7D,OAAO,YAAY,EAAE,CAAC,CAAC,KAAK,EAAC,MAAM,WAAW,OAAO,SAAS,cAAa;EACzE;EACA;EACA;EACA,SAAS,OAAO,QAAQ,OAAO,CAAC,CAAC,KAAK,CAAC,QAAQ,UAAU;GAAC;GAAQ;EAAG,EAAE;CACzE,EAAE;AACJ;AAEA,SAAS,eAAe,IAAc;CACpC,MAAM,CAAC,EAAC,YAAY,eAAc,GAAG,OACnC,YACF;CACA,MAAM,CAAC,EAAC,WAAW,cAAa,GAAG,OAA4B,WAAW;CAC1E,OAAO,YAAY;AACrB"}

package/out/zero-cache/src/services/replicator/replicator.js.map

@@ -1 +1 @@
-{"version":3,"file":"replicator.js","names":["#lc","#incrementalSyncer","#shouldPublishErrors","#worker","#runPromise"],"sources":["../../../../../../zero-cache/src/services/replicator/replicator.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport type {ReadonlyJSONObject} from '../../../../shared/src/json.ts';\nimport type {Source} from '../../types/streams.ts';\nimport type {ReservationFollowup} from '../change-streamer/change-streamer-http.ts';\nimport type {ChangeStreamer} from '../change-streamer/change-streamer.ts';\nimport type {Service} from '../service.ts';\nimport type {SwitchoverOptions} from './drain-handler.ts';\nimport {IncrementalSyncer} from './incremental-sync.ts';\nimport {\n  publishReplicationError,\n  type ReplicationStatusPublisher,\n} from './replication-status.ts';\nimport type {ReplicationLagReadinessOptions} from './reporter/recorder.ts';\nimport type {WriteWorkerClient} from './write-worker-client.ts';\n\n/** See {@link ReplicaStateNotifier.subscribe()}. */\nexport type ReplicaState = {\n  readonly state: 'version-ready';\n  /**\n   * The replica watermark that became ready. Omitted for the initial\n   * \"current state is ready\" notification, which should not count as newly\n   * unserved work for a freshly-started ViewSyncer.\n   */\n  readonly watermark?: string | undefined;\n  /**\n   * Millisecond epoch when `watermark` became ready to read from the replica.\n   */\n  readonly replicaReadyTimeMs?: number | undefined;\n\n  /**\n   * Millisecond epoch at which the transaction behind `watermark` committed\n   * *upstream*, as reported by the ChangeSource. Unlike `replicaReadyTimeMs`,\n   * which is stamped locally when the change lands in the replica, this is the\n   * origin of the whole pipeline, so `now - upstreamCommitTimeMs` at the point\n   * the change is poked to clients is the end-to-end serving lag.\n   *\n   * Note that this crosses a clock domain: it comes from the upstream\n   * database, whereas it is subtracted from the local clock in the ViewSyncer.\n   * The existing `zero.replication.total_lag` metric spans the same boundary.\n   *\n   * Undefined when the ChangeSource does not report a commit time, and for the\n   * initial \"current state is ready\" notification.\n   */\n  readonly upstreamCommitTimeMs?: number | undefined;\n\n  // Used in tests to verify behavior when additional information\n  // is ferried in the future. Not set in production.\n  readonly testSeqNum?: number | undefined;\n};\n\nexport interface ReplicaStateNotifier {\n  /**\n   * Creates a cancelable subscription of changes in the replica state.\n   *\n   * A `version-ready` message indicates that the replica is ready to be\n   * read, and henceforth that a _new_ version is ready, i.e. whenever a\n   * change is committed to the replica. The `version-ready` message itself\n   * otherwise contains no other information; the subscriber queries the\n   * replica for the current data.\n   *\n   * A `maintenance` state indicates that the replica should not be read from.\n   * If a subscriber is holding any transaction locks, it should release them\n   * until the next `version-ready` signal.\n   *\n   * Upon subscription, the current state of the replica is sent immediately\n   * if known. If multiple notifications occur before the subscriber\n   * can consume them, all but the last notification are discarded by the\n   * Subscription object (i.e. not buffered). Thus, a subscriber only\n   * ever consumes the current (i.e. known) state of the replica. This avoids\n   * a buildup of \"work\" if a subscriber is too busy to consume all\n   * notifications.\n   */\n  subscribe(): Source<ReplicaState>;\n}\n\nexport interface Replicator extends ReplicaStateNotifier {\n  /**\n   * Returns an opaque message for human-readable consumption. This is\n   * purely for ensuring that the Replicator has started at least once to\n   * bootstrap a new replica.\n   */\n  status(): Promise<ReadonlyJSONObject>;\n}\n\nexport type ReplicatorMode = 'backup' | 'serving';\n\nexport class ReplicatorService implements Replicator, Service {\n  readonly id: string;\n  readonly #lc: LogContext;\n  readonly #incrementalSyncer: IncrementalSyncer;\n  readonly #shouldPublishErrors: boolean;\n  readonly #worker: WriteWorkerClient;\n  #runPromise: Promise<void> | undefined;\n\n  constructor(\n    lc: LogContext,\n    taskID: string,\n    id: string,\n    mode: ReplicatorMode,\n    changeStreamer: ChangeStreamer,\n    worker: WriteWorkerClient,\n    statusPublisher: ReplicationStatusPublisher | null,\n    replicationLagReadinessOptions: ReplicationLagReadinessOptions = {},\n    // The still-open connection from the snapshot reservation that produced\n    // the replica this process just restored (serving mode only). Used for\n    // IncrementalSyncer's first subscribe() so the replication-manager that\n    // reserved the change log is the one that serves the subscription.\n    initialConnection?: ReservationFollowup,\n    switchover?: SwitchoverOptions,\n  ) {\n    this.id = id;\n    this.#lc = lc\n      .withContext('component', 'replicator')\n      .withContext('serviceID', this.id);\n    this.#worker = worker;\n    this.#shouldPublishErrors = statusPublisher !== null;\n\n    this.#incrementalSyncer = new IncrementalSyncer(\n      this.#lc,\n      taskID,\n      `${taskID}/${id}`,\n      changeStreamer,\n      worker,\n      mode,\n      statusPublisher,\n      replicationLagReadinessOptions,\n      initialConnection,\n      switchover,\n    );\n  }\n\n  status() {\n    return Promise.resolve({status: 'ok'});\n  }\n\n  ready(): Promise<void> {\n    return this.#incrementalSyncer.ready();\n  }\n\n  run() {\n    this.#runPromise = this.#incrementalSyncer.run().catch(async error => {\n      if (this.#shouldPublishErrors) {\n        await publishReplicationError(\n          this.#lc,\n          'Replicating',\n          'Replication stopped because the replica writer failed',\n        );\n      }\n      throw error;\n    });\n    return this.#runPromise;\n  }\n\n  subscribe(): Source<ReplicaState> {\n    return this.#incrementalSyncer.subscribe();\n  }\n\n  async stop() {\n    this.#incrementalSyncer.stop(this.#lc);\n    // Wait for the syncer's run loop to finish so that any in-flight\n    // worker.processMessages() call completes and clears #pending\n    // before we send the 'stop' message to the worker.\n    await this.#runPromise;\n    await this.#worker.stop();\n  }\n}\n"],"mappings":";;;AAsFA,IAAa,oBAAb,MAA8D;CAC5D;CACA;CACA;CACA;CACA;CACA;CAEA,YACE,IACA,QACA,IACA,MACA,gBACA,QACA,iBACA,iCAAiE,CAAC,GAKlE,mBACA,YACA;EACA,KAAK,KAAK;EACV,KAAKA,MAAM,GACR,YAAY,aAAa,YAAY,EACrC,YAAY,aAAa,KAAK,EAAE;EACnC,KAAKG,UAAU;EACf,KAAKD,uBAAuB,oBAAoB;EAEhD,KAAKD,qBAAqB,IAAI,kBAC5B,KAAKD,KACL,QACA,GAAG,OAAO,GAAG,MACb,gBACA,QACA,MACA,iBACA,gCACA,mBACA,UACF;CACF;CAEA,SAAS;EACP,OAAO,QAAQ,QAAQ,EAAC,QAAQ,KAAI,CAAC;CACvC;CAEA,QAAuB;EACrB,OAAO,KAAKC,mBAAmB,MAAM;CACvC;CAEA,MAAM;EACJ,KAAKG,cAAc,KAAKH,mBAAmB,IAAI,EAAE,MAAM,OAAM,UAAS;GACpE,IAAI,KAAKC,sBACP,MAAM,wBACJ,KAAKF,KACL,eACA,uDACF;GAEF,MAAM;EACR,CAAC;EACD,OAAO,KAAKI;CACd;CAEA,YAAkC;EAChC,OAAO,KAAKH,mBAAmB,UAAU;CAC3C;CAEA,MAAM,OAAO;EACX,KAAKA,mBAAmB,KAAK,KAAKD,GAAG;EAIrC,MAAM,KAAKI;EACX,MAAM,KAAKD,QAAQ,KAAK;CAC1B;AACF"}
+{"version":3,"file":"replicator.js","names":[],"sources":["../../../../../../zero-cache/src/services/replicator/replicator.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport type {ReadonlyJSONObject} from '../../../../shared/src/json.ts';\nimport type {Source} from '../../types/streams.ts';\nimport type {ReservationFollowup} from '../change-streamer/change-streamer-http.ts';\nimport type {ChangeStreamer} from '../change-streamer/change-streamer.ts';\nimport type {Service} from '../service.ts';\nimport type {SwitchoverOptions} from './drain-handler.ts';\nimport {IncrementalSyncer} from './incremental-sync.ts';\nimport {\n  publishReplicationError,\n  type ReplicationStatusPublisher,\n} from './replication-status.ts';\nimport type {ReplicationLagReadinessOptions} from './reporter/recorder.ts';\nimport type {WriteWorkerClient} from './write-worker-client.ts';\n\n/** See {@link ReplicaStateNotifier.subscribe()}. */\nexport type ReplicaState = {\n  readonly state: 'version-ready';\n  /**\n   * The replica watermark that became ready. Omitted for the initial\n   * \"current state is ready\" notification, which should not count as newly\n   * unserved work for a freshly-started ViewSyncer.\n   */\n  readonly watermark?: string | undefined;\n  /**\n   * Millisecond epoch when `watermark` became ready to read from the replica.\n   */\n  readonly replicaReadyTimeMs?: number | undefined;\n\n  /**\n   * Millisecond epoch at which the transaction behind `watermark` committed\n   * *upstream*, as reported by the ChangeSource. Unlike `replicaReadyTimeMs`,\n   * which is stamped locally when the change lands in the replica, this is the\n   * origin of the whole pipeline, so `now - upstreamCommitTimeMs` at the point\n   * the change is poked to clients is the end-to-end serving lag.\n   *\n   * Note that this crosses a clock domain: it comes from the upstream\n   * database, whereas it is subtracted from the local clock in the ViewSyncer.\n   * The existing `zero.replication.total_lag` metric spans the same boundary.\n   *\n   * Undefined when the ChangeSource does not report a commit time, and for the\n   * initial \"current state is ready\" notification.\n   */\n  readonly upstreamCommitTimeMs?: number | undefined;\n\n  // Used in tests to verify behavior when additional information\n  // is ferried in the future. Not set in production.\n  readonly testSeqNum?: number | undefined;\n};\n\nexport interface ReplicaStateNotifier {\n  /**\n   * Creates a cancelable subscription of changes in the replica state.\n   *\n   * A `version-ready` message indicates that the replica is ready to be\n   * read, and henceforth that a _new_ version is ready, i.e. whenever a\n   * change is committed to the replica. The `version-ready` message itself\n   * otherwise contains no other information; the subscriber queries the\n   * replica for the current data.\n   *\n   * A `maintenance` state indicates that the replica should not be read from.\n   * If a subscriber is holding any transaction locks, it should release them\n   * until the next `version-ready` signal.\n   *\n   * Upon subscription, the current state of the replica is sent immediately\n   * if known. If multiple notifications occur before the subscriber\n   * can consume them, all but the last notification are discarded by the\n   * Subscription object (i.e. not buffered). Thus, a subscriber only\n   * ever consumes the current (i.e. known) state of the replica. This avoids\n   * a buildup of \"work\" if a subscriber is too busy to consume all\n   * notifications.\n   */\n  subscribe(): Source<ReplicaState>;\n}\n\nexport interface Replicator extends ReplicaStateNotifier {\n  /**\n   * Returns an opaque message for human-readable consumption. This is\n   * purely for ensuring that the Replicator has started at least once to\n   * bootstrap a new replica.\n   */\n  status(): Promise<ReadonlyJSONObject>;\n}\n\nexport type ReplicatorMode = 'backup' | 'serving';\n\nexport class ReplicatorService implements Replicator, Service {\n  readonly id: string;\n  readonly #lc: LogContext;\n  readonly #incrementalSyncer: IncrementalSyncer;\n  readonly #shouldPublishErrors: boolean;\n  readonly #worker: WriteWorkerClient;\n  #runPromise: Promise<void> | undefined;\n\n  constructor(\n    lc: LogContext,\n    taskID: string,\n    id: string,\n    mode: ReplicatorMode,\n    changeStreamer: ChangeStreamer,\n    worker: WriteWorkerClient,\n    statusPublisher: ReplicationStatusPublisher | null,\n    replicationLagReadinessOptions: ReplicationLagReadinessOptions = {},\n    // The still-open connection from the snapshot reservation that produced\n    // the replica this process just restored (serving mode only). Used for\n    // IncrementalSyncer's first subscribe() so the replication-manager that\n    // reserved the change log is the one that serves the subscription.\n    initialConnection?: ReservationFollowup,\n    switchover?: SwitchoverOptions,\n  ) {\n    this.id = id;\n    this.#lc = lc\n      .withContext('component', 'replicator')\n      .withContext('serviceID', this.id);\n    this.#worker = worker;\n    this.#shouldPublishErrors = statusPublisher !== null;\n\n    this.#incrementalSyncer = new IncrementalSyncer(\n      this.#lc,\n      taskID,\n      `${taskID}/${id}`,\n      changeStreamer,\n      worker,\n      mode,\n      statusPublisher,\n      replicationLagReadinessOptions,\n      initialConnection,\n      switchover,\n    );\n  }\n\n  status() {\n    return Promise.resolve({status: 'ok'});\n  }\n\n  ready(): Promise<void> {\n    return this.#incrementalSyncer.ready();\n  }\n\n  run() {\n    this.#runPromise = this.#incrementalSyncer.run().catch(async error => {\n      if (this.#shouldPublishErrors) {\n        await publishReplicationError(\n          this.#lc,\n          'Replicating',\n          'Replication stopped because the replica writer failed',\n        );\n      }\n      throw error;\n    });\n    return this.#runPromise;\n  }\n\n  subscribe(): Source<ReplicaState> {\n    return this.#incrementalSyncer.subscribe();\n  }\n\n  async stop() {\n    this.#incrementalSyncer.stop(this.#lc);\n    // Wait for the syncer's run loop to finish so that any in-flight\n    // worker.processMessages() call completes and clears #pending\n    // before we send the 'stop' message to the worker.\n    await this.#runPromise;\n    await this.#worker.stop();\n  }\n}\n"],"mappings":";;;AAsFA,IAAa,oBAAb,MAA8D;CAC5D;CACA;CACA;CACA;CACA;CACA;CAEA,YACE,IACA,QACA,IACA,MACA,gBACA,QACA,iBACA,iCAAiE,CAAC,GAKlE,mBACA,YACA;EACA,KAAK,KAAK;EACV,KAAK,MAAM,GACR,YAAY,aAAa,YAAY,CAAC,CACtC,YAAY,aAAa,KAAK,EAAE;EACnC,KAAK,UAAU;EACf,KAAK,uBAAuB,oBAAoB;EAEhD,KAAK,qBAAqB,IAAI,kBAC5B,KAAK,KACL,QACA,GAAG,OAAO,GAAG,MACb,gBACA,QACA,MACA,iBACA,gCACA,mBACA,UACF;CACF;CAEA,SAAS;EACP,OAAO,QAAQ,QAAQ,EAAC,QAAQ,KAAI,CAAC;CACvC;CAEA,QAAuB;EACrB,OAAO,KAAK,mBAAmB,MAAM;CACvC;CAEA,MAAM;EACJ,KAAK,cAAc,KAAK,mBAAmB,IAAI,CAAC,CAAC,MAAM,OAAM,UAAS;GACpE,IAAI,KAAK,sBACP,MAAM,wBACJ,KAAK,KACL,eACA,uDACF;GAEF,MAAM;EACR,CAAC;EACD,OAAO,KAAK;CACd;CAEA,YAAkC;EAChC,OAAO,KAAK,mBAAmB,UAAU;CAC3C;CAEA,MAAM,OAAO;EACX,KAAK,mBAAmB,KAAK,KAAK,GAAG;EAIrC,MAAM,KAAK;EACX,MAAM,KAAK,QAAQ,KAAK;CAC1B;AACF"}

package/out/zero-cache/src/services/replicator/reporter/recorder.js

@@ -129,6 +129,6 @@ function isLagReportingDisabledSignal(report) {
 	return report.nextSendTimeMs === 0 && report.lastTimings === void 0;
 }
 //#endregion
-export { ReplicationReportRecorder };
+export { ReplicationReportRecorder, estimateUpstreamClockSkewMs, isLagReportingDisabledSignal };
 
 //# sourceMappingURL=recorder.js.map

package/out/zero-cache/src/services/replicator/reporter/recorder.js.map

@@ -1 +1 @@
-{"version":3,"file":"recorder.js","names":["#lc","#readinessThresholdMs","#readyGate","#clearTimeout","#readinessTimer","#setReady","#last","#ready"],"sources":["../../../../../../../zero-cache/src/services/replicator/reporter/recorder.ts"],"sourcesContent":["import type {ObservableResult} from '@opentelemetry/api';\nimport type {LogContext, LogLevel} from '@rocicorp/logger';\nimport {resolver} from '@rocicorp/resolver';\nimport {getOrCreateGauge} from '../../../observability/metrics.ts';\nimport type {ReplicationReport} from './report-schema.ts';\n\n// Hook for sanity checking lag reports in development.\nconst LOG_ALL_REPLICATION_REPORTS_AT_DEBUG =\n  process.env.ZERO_LOG_ALL_REPLICATION_REPORTS_AT_DEBUG === '1';\n\nexport type ReplicationLagReadinessOptions = {\n  readinessThresholdMs?: number | undefined;\n  maxReadinessDelayMs?: number | undefined;\n  setTimeoutFn?: typeof setTimeout | undefined;\n  clearTimeoutFn?: typeof clearTimeout | undefined;\n};\n\nexport class ReplicationReportRecorder {\n  readonly #lc: LogContext;\n  readonly #readinessThresholdMs: number;\n  readonly #readyGate = resolver();\n  readonly #clearTimeout: typeof clearTimeout;\n  readonly #readinessTimer: ReturnType<typeof setTimeout>;\n  #ready = false;\n  #last: ReplicationReport | null = null;\n\n  constructor(\n    lc: LogContext,\n    {\n      readinessThresholdMs = 1000,\n      maxReadinessDelayMs = 600_000,\n      setTimeoutFn = setTimeout,\n      clearTimeoutFn = clearTimeout,\n    }: ReplicationLagReadinessOptions = {},\n  ) {\n    this.#lc = lc;\n    this.#readinessThresholdMs = readinessThresholdMs;\n    this.#clearTimeout = clearTimeoutFn;\n\n    this.#readinessTimer = setTimeoutFn(\n      () =>\n        this.#setReady(\n          'warn',\n          `timed out waiting for replication lag readiness threshold of ${this.#readinessThresholdMs} ms`,\n        ),\n      maxReadinessDelayMs,\n    );\n    // The timer should not keep the process alive.\n    this.#readinessTimer?.unref?.();\n  }\n\n  ready(): Promise<void> {\n    return this.#readyGate.promise;\n  }\n\n  record(report: ReplicationReport) {\n    if (isLagReportingDisabledSignal(report)) {\n      this.#setReady(\n        'info',\n        `replica is caught up (replication lag reporting is disabled upstream)`,\n      );\n      return;\n    }\n    this.#last = report;\n\n    const {lastTimings} = report;\n    if (lastTimings) {\n      const total = lastTimings.replicateTimeMs - lastTimings.sendTimeMs;\n      if (total > 10_000) {\n        const log = this.#ready ? 'warn' : 'info';\n        this.#lc[log]?.(`high replication lag: ${total} ms`, {\n          ready: this.#ready,\n          report,\n        });\n      } else if (total > 1_000) {\n        this.#lc.info?.(`replication lag: ${total} ms`, {\n          ready: this.#ready,\n          report,\n        });\n      }\n      if (LOG_ALL_REPLICATION_REPORTS_AT_DEBUG) {\n        this.#lc.debug?.(`replication lag ${total} ms`, {\n          ready: this.#ready,\n          report,\n        });\n      }\n      if (!this.#ready && total <= this.#readinessThresholdMs) {\n        this.#setReady(\n          'info',\n          `replicator is now ready with replication lag (${total} ms) less than ${this.#readinessThresholdMs} ms readiness threshold`,\n        );\n      }\n    }\n  }\n\n  #setReady(log: LogLevel, message: string) {\n    if (this.#ready) {\n      return;\n    }\n    this.#lc[log]?.(message, {lastReport: this.#last});\n    this.#ready = true;\n    this.#clearTimeout(this.#readinessTimer);\n    this.#readyGate.resolve();\n\n    getOrCreateGauge('replication', 'upstream_lag', {\n      description:\n        'Latency from sending an upstream replication report ' +\n        'to receiving it in the replication stream',\n      unit: 'millisecond',\n    }).addCallback(this.reportUpstreamLag);\n\n    getOrCreateGauge('replication', 'replica_lag', {\n      description:\n        'Latency from receiving an upstream replication report ' +\n        'to its reaching the replica',\n      unit: 'millisecond',\n    }).addCallback(this.reportReplicaLag);\n\n    getOrCreateGauge('replication', 'total_lag', {\n      description:\n        'Latency from sending an upstream replication report to its ' +\n        'reaching the replica. This reflects the actual measured ' +\n        'round-trip of the most recently received report and does not ' +\n        'grow when reports stop arriving.',\n      unit: 'millisecond',\n    }).addCallback(this.reportTotalLag);\n\n    getOrCreateGauge('replication', 'last_total_lag', {\n      description:\n        'Latency from sending the most recently received upstream ' +\n        'replication report to its reaching the replica. This is an alias ' +\n        'of replication.total_lag retained for dashboards that explicitly ' +\n        'want the non-extrapolated value.',\n      unit: 'millisecond',\n    }).addCallback(this.reportLastTotalLag);\n\n    getOrCreateGauge('replication', 'upstream_clock_skew', {\n      description:\n        'Estimated offset of the upstream database clock relative to ' +\n        \"zero-cache's. Positive means upstream is running ahead. \" +\n        'Derived from the lag report round-trip, whose send and receive ' +\n        'times are stamped on our clock either side of the upstream ' +\n        'commit timestamp. This matters because ' +\n        'sync.e2e_serving_lag subtracts an upstream commit timestamp from ' +\n        'a local clock, so a skewed upstream clock biases it directly -- ' +\n        'and an upstream clock running ahead biases it *low*, which looks ' +\n        'like healthy serving rather than a broken measurement.',\n      unit: 'millisecond',\n    }).addCallback(this.reportUpstreamClockSkew);\n  }\n\n  readonly reportUpstreamLag = (o: ObservableResult) => {\n    const last = this.#last?.lastTimings;\n    if (last) {\n      o.observe(last.receiveTimeMs - last.sendTimeMs);\n    }\n  };\n\n  readonly reportReplicaLag = (o: ObservableResult) => {\n    const last = this.#last?.lastTimings;\n    if (last) {\n      o.observe(last.replicateTimeMs - last.receiveTimeMs);\n    }\n  };\n\n  readonly reportTotalLag = (o: ObservableResult) => {\n    const last = this.#last?.lastTimings;\n    if (last) {\n      o.observe(last.replicateTimeMs - last.sendTimeMs);\n    }\n  };\n\n  readonly reportLastTotalLag = (o: ObservableResult) => {\n    const last = this.#last?.lastTimings;\n    if (last) {\n      o.observe(last.replicateTimeMs - last.sendTimeMs);\n    }\n  };\n\n  readonly reportUpstreamClockSkew = (o: ObservableResult) => {\n    const last = this.#last?.lastTimings;\n    if (last) {\n      o.observe(estimateUpstreamClockSkewMs(last));\n    }\n  };\n}\n\n/**\n * Estimates how far the upstream database's clock is from ours, in\n * milliseconds, positive meaning upstream is ahead.\n *\n * `sendTimeMs` and `receiveTimeMs` are both stamped in the\n * replication-manager, either side of `commitTimeMs`, which upstream stamps.\n * Assuming the upstream commit fell midway through that round trip, the\n * difference between where it actually landed and that midpoint is the clock\n * offset -- the same round-trip estimate NTP uses.\n *\n * The estimate is biased by any asymmetry between the two legs: the outbound\n * leg is a SQL round trip, while the return leg additionally waits on a WAL\n * flush and the walsender (which the pre-PG-17 path notes can add 50~100ms on\n * an idle database). So treat this as a detector of gross skew -- the kind\n * that invalidates sync.e2e_serving_lag -- and not as a sub-100ms correction.\n *\n * The rigorous bound, if a tighter reading is ever needed: the commit\n * genuinely happened between send and receive, so a `commitTimeMs` outside\n * that window proves skew of at least the amount by which it falls outside,\n * independent of any latency assumption.\n */\nexport function estimateUpstreamClockSkewMs({\n  sendTimeMs,\n  commitTimeMs,\n  receiveTimeMs,\n}: {\n  sendTimeMs: number;\n  commitTimeMs: number;\n  receiveTimeMs: number;\n}): number {\n  return commitTimeMs - (sendTimeMs + receiveTimeMs) / 2;\n}\n\n/**\n * Returns whether the `report` is the signal sent by the change-streamer,\n * once the subscriber is caught up, to indicate that lag reporting is\n * disabled upstream (and thus readiness cannot be gated on lag reports).\n *\n * This is a lag report with a `nextSendTimeMs` of 0 and no `lastTimings`.\n */\nexport function isLagReportingDisabledSignal(\n  report: ReplicationReport,\n): boolean {\n  return report.nextSendTimeMs === 0 && report.lastTimings === undefined;\n}\n"],"mappings":";;;AAOA,IAAM,uCACJ,QAAQ,IAAI,8CAA8C;AAS5D,IAAa,4BAAb,MAAuC;CACrC;CACA;CACA,aAAsB,SAAS;CAC/B;CACA;CACA,SAAS;CACT,QAAkC;CAElC,YACE,IACA,EACE,uBAAuB,KACvB,sBAAsB,KACtB,eAAe,YACf,iBAAiB,iBACiB,CAAC,GACrC;EACA,KAAKA,MAAM;EACX,KAAKC,wBAAwB;EAC7B,KAAKE,gBAAgB;EAErB,KAAKC,kBAAkB,mBAEnB,KAAKC,UACH,QACA,gEAAgE,KAAKJ,sBAAsB,IAC7F,GACF,mBACF;EAEA,KAAKG,iBAAiB,QAAQ;CAChC;CAEA,QAAuB;EACrB,OAAO,KAAKF,WAAW;CACzB;CAEA,OAAO,QAA2B;EAChC,IAAI,6BAA6B,MAAM,GAAG;GACxC,KAAKG,UACH,QACA,uEACF;GACA;EACF;EACA,KAAKC,QAAQ;EAEb,MAAM,EAAC,gBAAe;EACtB,IAAI,aAAa;GACf,MAAM,QAAQ,YAAY,kBAAkB,YAAY;GACxD,IAAI,QAAQ,KAAQ;IAClB,MAAM,MAAM,KAAKC,SAAS,SAAS;IACnC,KAAKP,IAAI,OAAO,yBAAyB,MAAM,MAAM;KACnD,OAAO,KAAKO;KACZ;IACF,CAAC;GACH,OAAO,IAAI,QAAQ,KACjB,KAAKP,IAAI,OAAO,oBAAoB,MAAM,MAAM;IAC9C,OAAO,KAAKO;IACZ;GACF,CAAC;GAEH,IAAI,sCACF,KAAKP,IAAI,QAAQ,mBAAmB,MAAM,MAAM;IAC9C,OAAO,KAAKO;IACZ;GACF,CAAC;GAEH,IAAI,CAAC,KAAKA,UAAU,SAAS,KAAKN,uBAChC,KAAKI,UACH,QACA,iDAAiD,MAAM,iBAAiB,KAAKJ,sBAAsB,wBACrG;EAEJ;CACF;CAEA,UAAU,KAAe,SAAiB;EACxC,IAAI,KAAKM,QACP;EAEF,KAAKP,IAAI,OAAO,SAAS,EAAC,YAAY,KAAKM,MAAK,CAAC;EACjD,KAAKC,SAAS;EACd,KAAKJ,cAAc,KAAKC,eAAe;EACvC,KAAKF,WAAW,QAAQ;EAExB,iBAAiB,eAAe,gBAAgB;GAC9C,aACE;GAEF,MAAM;EACR,CAAC,EAAE,YAAY,KAAK,iBAAiB;EAErC,iBAAiB,eAAe,eAAe;GAC7C,aACE;GAEF,MAAM;EACR,CAAC,EAAE,YAAY,KAAK,gBAAgB;EAEpC,iBAAiB,eAAe,aAAa;GAC3C,aACE;GAIF,MAAM;EACR,CAAC,EAAE,YAAY,KAAK,cAAc;EAElC,iBAAiB,eAAe,kBAAkB;GAChD,aACE;GAIF,MAAM;EACR,CAAC,EAAE,YAAY,KAAK,kBAAkB;EAEtC,iBAAiB,eAAe,uBAAuB;GACrD,aACE;GASF,MAAM;EACR,CAAC,EAAE,YAAY,KAAK,uBAAuB;CAC7C;CAEA,qBAA8B,MAAwB;EACpD,MAAM,OAAO,KAAKI,OAAO;EACzB,IAAI,MACF,EAAE,QAAQ,KAAK,gBAAgB,KAAK,UAAU;CAElD;CAEA,oBAA6B,MAAwB;EACnD,MAAM,OAAO,KAAKA,OAAO;EACzB,IAAI,MACF,EAAE,QAAQ,KAAK,kBAAkB,KAAK,aAAa;CAEvD;CAEA,kBAA2B,MAAwB;EACjD,MAAM,OAAO,KAAKA,OAAO;EACzB,IAAI,MACF,EAAE,QAAQ,KAAK,kBAAkB,KAAK,UAAU;CAEpD;CAEA,sBAA+B,MAAwB;EACrD,MAAM,OAAO,KAAKA,OAAO;EACzB,IAAI,MACF,EAAE,QAAQ,KAAK,kBAAkB,KAAK,UAAU;CAEpD;CAEA,2BAAoC,MAAwB;EAC1D,MAAM,OAAO,KAAKA,OAAO;EACzB,IAAI,MACF,EAAE,QAAQ,4BAA4B,IAAI,CAAC;CAE/C;AACF;;;;;;;;;;;;;;;;;;;;;;AAuBA,SAAgB,4BAA4B,EAC1C,YACA,cACA,iBAKS;CACT,OAAO,gBAAgB,aAAa,iBAAiB;AACvD;;;;;;;;AASA,SAAgB,6BACd,QACS;CACT,OAAO,OAAO,mBAAmB,KAAK,OAAO,gBAAgB,KAAA;AAC/D"}
+{"version":3,"file":"recorder.js","names":[],"sources":["../../../../../../../zero-cache/src/services/replicator/reporter/recorder.ts"],"sourcesContent":["import type {ObservableResult} from '@opentelemetry/api';\nimport type {LogContext, LogLevel} from '@rocicorp/logger';\nimport {resolver} from '@rocicorp/resolver';\nimport {getOrCreateGauge} from '../../../observability/metrics.ts';\nimport type {ReplicationReport} from './report-schema.ts';\n\n// Hook for sanity checking lag reports in development.\nconst LOG_ALL_REPLICATION_REPORTS_AT_DEBUG =\n  process.env.ZERO_LOG_ALL_REPLICATION_REPORTS_AT_DEBUG === '1';\n\nexport type ReplicationLagReadinessOptions = {\n  readinessThresholdMs?: number | undefined;\n  maxReadinessDelayMs?: number | undefined;\n  setTimeoutFn?: typeof setTimeout | undefined;\n  clearTimeoutFn?: typeof clearTimeout | undefined;\n};\n\nexport class ReplicationReportRecorder {\n  readonly #lc: LogContext;\n  readonly #readinessThresholdMs: number;\n  readonly #readyGate = resolver();\n  readonly #clearTimeout: typeof clearTimeout;\n  readonly #readinessTimer: ReturnType<typeof setTimeout>;\n  #ready = false;\n  #last: ReplicationReport | null = null;\n\n  constructor(\n    lc: LogContext,\n    {\n      readinessThresholdMs = 1000,\n      maxReadinessDelayMs = 600_000,\n      setTimeoutFn = setTimeout,\n      clearTimeoutFn = clearTimeout,\n    }: ReplicationLagReadinessOptions = {},\n  ) {\n    this.#lc = lc;\n    this.#readinessThresholdMs = readinessThresholdMs;\n    this.#clearTimeout = clearTimeoutFn;\n\n    this.#readinessTimer = setTimeoutFn(\n      () =>\n        this.#setReady(\n          'warn',\n          `timed out waiting for replication lag readiness threshold of ${this.#readinessThresholdMs} ms`,\n        ),\n      maxReadinessDelayMs,\n    );\n    // The timer should not keep the process alive.\n    this.#readinessTimer?.unref?.();\n  }\n\n  ready(): Promise<void> {\n    return this.#readyGate.promise;\n  }\n\n  record(report: ReplicationReport) {\n    if (isLagReportingDisabledSignal(report)) {\n      this.#setReady(\n        'info',\n        `replica is caught up (replication lag reporting is disabled upstream)`,\n      );\n      return;\n    }\n    this.#last = report;\n\n    const {lastTimings} = report;\n    if (lastTimings) {\n      const total = lastTimings.replicateTimeMs - lastTimings.sendTimeMs;\n      if (total > 10_000) {\n        const log = this.#ready ? 'warn' : 'info';\n        this.#lc[log]?.(`high replication lag: ${total} ms`, {\n          ready: this.#ready,\n          report,\n        });\n      } else if (total > 1_000) {\n        this.#lc.info?.(`replication lag: ${total} ms`, {\n          ready: this.#ready,\n          report,\n        });\n      }\n      if (LOG_ALL_REPLICATION_REPORTS_AT_DEBUG) {\n        this.#lc.debug?.(`replication lag ${total} ms`, {\n          ready: this.#ready,\n          report,\n        });\n      }\n      if (!this.#ready && total <= this.#readinessThresholdMs) {\n        this.#setReady(\n          'info',\n          `replicator is now ready with replication lag (${total} ms) less than ${this.#readinessThresholdMs} ms readiness threshold`,\n        );\n      }\n    }\n  }\n\n  #setReady(log: LogLevel, message: string) {\n    if (this.#ready) {\n      return;\n    }\n    this.#lc[log]?.(message, {lastReport: this.#last});\n    this.#ready = true;\n    this.#clearTimeout(this.#readinessTimer);\n    this.#readyGate.resolve();\n\n    getOrCreateGauge('replication', 'upstream_lag', {\n      description:\n        'Latency from sending an upstream replication report ' +\n        'to receiving it in the replication stream',\n      unit: 'millisecond',\n    }).addCallback(this.reportUpstreamLag);\n\n    getOrCreateGauge('replication', 'replica_lag', {\n      description:\n        'Latency from receiving an upstream replication report ' +\n        'to its reaching the replica',\n      unit: 'millisecond',\n    }).addCallback(this.reportReplicaLag);\n\n    getOrCreateGauge('replication', 'total_lag', {\n      description:\n        'Latency from sending an upstream replication report to its ' +\n        'reaching the replica. This reflects the actual measured ' +\n        'round-trip of the most recently received report and does not ' +\n        'grow when reports stop arriving.',\n      unit: 'millisecond',\n    }).addCallback(this.reportTotalLag);\n\n    getOrCreateGauge('replication', 'last_total_lag', {\n      description:\n        'Latency from sending the most recently received upstream ' +\n        'replication report to its reaching the replica. This is an alias ' +\n        'of replication.total_lag retained for dashboards that explicitly ' +\n        'want the non-extrapolated value.',\n      unit: 'millisecond',\n    }).addCallback(this.reportLastTotalLag);\n\n    getOrCreateGauge('replication', 'upstream_clock_skew', {\n      description:\n        'Estimated offset of the upstream database clock relative to ' +\n        \"zero-cache's. Positive means upstream is running ahead. \" +\n        'Derived from the lag report round-trip, whose send and receive ' +\n        'times are stamped on our clock either side of the upstream ' +\n        'commit timestamp. This matters because ' +\n        'sync.e2e_serving_lag subtracts an upstream commit timestamp from ' +\n        'a local clock, so a skewed upstream clock biases it directly -- ' +\n        'and an upstream clock running ahead biases it *low*, which looks ' +\n        'like healthy serving rather than a broken measurement.',\n      unit: 'millisecond',\n    }).addCallback(this.reportUpstreamClockSkew);\n  }\n\n  readonly reportUpstreamLag = (o: ObservableResult) => {\n    const last = this.#last?.lastTimings;\n    if (last) {\n      o.observe(last.receiveTimeMs - last.sendTimeMs);\n    }\n  };\n\n  readonly reportReplicaLag = (o: ObservableResult) => {\n    const last = this.#last?.lastTimings;\n    if (last) {\n      o.observe(last.replicateTimeMs - last.receiveTimeMs);\n    }\n  };\n\n  readonly reportTotalLag = (o: ObservableResult) => {\n    const last = this.#last?.lastTimings;\n    if (last) {\n      o.observe(last.replicateTimeMs - last.sendTimeMs);\n    }\n  };\n\n  readonly reportLastTotalLag = (o: ObservableResult) => {\n    const last = this.#last?.lastTimings;\n    if (last) {\n      o.observe(last.replicateTimeMs - last.sendTimeMs);\n    }\n  };\n\n  readonly reportUpstreamClockSkew = (o: ObservableResult) => {\n    const last = this.#last?.lastTimings;\n    if (last) {\n      o.observe(estimateUpstreamClockSkewMs(last));\n    }\n  };\n}\n\n/**\n * Estimates how far the upstream database's clock is from ours, in\n * milliseconds, positive meaning upstream is ahead.\n *\n * `sendTimeMs` and `receiveTimeMs` are both stamped in the\n * replication-manager, either side of `commitTimeMs`, which upstream stamps.\n * Assuming the upstream commit fell midway through that round trip, the\n * difference between where it actually landed and that midpoint is the clock\n * offset -- the same round-trip estimate NTP uses.\n *\n * The estimate is biased by any asymmetry between the two legs: the outbound\n * leg is a SQL round trip, while the return leg additionally waits on a WAL\n * flush and the walsender (which the pre-PG-17 path notes can add 50~100ms on\n * an idle database). So treat this as a detector of gross skew -- the kind\n * that invalidates sync.e2e_serving_lag -- and not as a sub-100ms correction.\n *\n * The rigorous bound, if a tighter reading is ever needed: the commit\n * genuinely happened between send and receive, so a `commitTimeMs` outside\n * that window proves skew of at least the amount by which it falls outside,\n * independent of any latency assumption.\n */\nexport function estimateUpstreamClockSkewMs({\n  sendTimeMs,\n  commitTimeMs,\n  receiveTimeMs,\n}: {\n  sendTimeMs: number;\n  commitTimeMs: number;\n  receiveTimeMs: number;\n}): number {\n  return commitTimeMs - (sendTimeMs + receiveTimeMs) / 2;\n}\n\n/**\n * Returns whether the `report` is the signal sent by the change-streamer,\n * once the subscriber is caught up, to indicate that lag reporting is\n * disabled upstream (and thus readiness cannot be gated on lag reports).\n *\n * This is a lag report with a `nextSendTimeMs` of 0 and no `lastTimings`.\n */\nexport function isLagReportingDisabledSignal(\n  report: ReplicationReport,\n): boolean {\n  return report.nextSendTimeMs === 0 && report.lastTimings === undefined;\n}\n"],"mappings":";;;AAOA,IAAM,uCACJ,QAAQ,IAAI,8CAA8C;AAS5D,IAAa,4BAAb,MAAuC;CACrC;CACA;CACA,aAAsB,SAAS;CAC/B;CACA;CACA,SAAS;CACT,QAAkC;CAElC,YACE,IACA,EACE,uBAAuB,KACvB,sBAAsB,KACtB,eAAe,YACf,iBAAiB,iBACiB,CAAC,GACrC;EACA,KAAK,MAAM;EACX,KAAK,wBAAwB;EAC7B,KAAK,gBAAgB;EAErB,KAAK,kBAAkB,mBAEnB,KAAK,UACH,QACA,gEAAgE,KAAK,sBAAsB,IAC7F,GACF,mBACF;EAEA,KAAK,iBAAiB,QAAQ;CAChC;CAEA,QAAuB;EACrB,OAAO,KAAK,WAAW;CACzB;CAEA,OAAO,QAA2B;EAChC,IAAI,6BAA6B,MAAM,GAAG;GACxC,KAAK,UACH,QACA,uEACF;GACA;EACF;EACA,KAAK,QAAQ;EAEb,MAAM,EAAC,gBAAe;EACtB,IAAI,aAAa;GACf,MAAM,QAAQ,YAAY,kBAAkB,YAAY;GACxD,IAAI,QAAQ,KAAQ;IAClB,MAAM,MAAM,KAAK,SAAS,SAAS;IACnC,KAAK,IAAI,IAAI,GAAG,yBAAyB,MAAM,MAAM;KACnD,OAAO,KAAK;KACZ;IACF,CAAC;GACH,OAAO,IAAI,QAAQ,KACjB,KAAK,IAAI,OAAO,oBAAoB,MAAM,MAAM;IAC9C,OAAO,KAAK;IACZ;GACF,CAAC;GAEH,IAAI,sCACF,KAAK,IAAI,QAAQ,mBAAmB,MAAM,MAAM;IAC9C,OAAO,KAAK;IACZ;GACF,CAAC;GAEH,IAAI,CAAC,KAAK,UAAU,SAAS,KAAK,uBAChC,KAAK,UACH,QACA,iDAAiD,MAAM,iBAAiB,KAAK,sBAAsB,wBACrG;EAEJ;CACF;CAEA,UAAU,KAAe,SAAiB;EACxC,IAAI,KAAK,QACP;EAEF,KAAK,IAAI,IAAI,GAAG,SAAS,EAAC,YAAY,KAAK,MAAK,CAAC;EACjD,KAAK,SAAS;EACd,KAAK,cAAc,KAAK,eAAe;EACvC,KAAK,WAAW,QAAQ;EAExB,iBAAiB,eAAe,gBAAgB;GAC9C,aACE;GAEF,MAAM;EACR,CAAC,CAAC,CAAC,YAAY,KAAK,iBAAiB;EAErC,iBAAiB,eAAe,eAAe;GAC7C,aACE;GAEF,MAAM;EACR,CAAC,CAAC,CAAC,YAAY,KAAK,gBAAgB;EAEpC,iBAAiB,eAAe,aAAa;GAC3C,aACE;GAIF,MAAM;EACR,CAAC,CAAC,CAAC,YAAY,KAAK,cAAc;EAElC,iBAAiB,eAAe,kBAAkB;GAChD,aACE;GAIF,MAAM;EACR,CAAC,CAAC,CAAC,YAAY,KAAK,kBAAkB;EAEtC,iBAAiB,eAAe,uBAAuB;GACrD,aACE;GASF,MAAM;EACR,CAAC,CAAC,CAAC,YAAY,KAAK,uBAAuB;CAC7C;CAEA,qBAA8B,MAAwB;EACpD,MAAM,OAAO,KAAK,OAAO;EACzB,IAAI,MACF,EAAE,QAAQ,KAAK,gBAAgB,KAAK,UAAU;CAElD;CAEA,oBAA6B,MAAwB;EACnD,MAAM,OAAO,KAAK,OAAO;EACzB,IAAI,MACF,EAAE,QAAQ,KAAK,kBAAkB,KAAK,aAAa;CAEvD;CAEA,kBAA2B,MAAwB;EACjD,MAAM,OAAO,KAAK,OAAO;EACzB,IAAI,MACF,EAAE,QAAQ,KAAK,kBAAkB,KAAK,UAAU;CAEpD;CAEA,sBAA+B,MAAwB;EACrD,MAAM,OAAO,KAAK,OAAO;EACzB,IAAI,MACF,EAAE,QAAQ,KAAK,kBAAkB,KAAK,UAAU;CAEpD;CAEA,2BAAoC,MAAwB;EAC1D,MAAM,OAAO,KAAK,OAAO;EACzB,IAAI,MACF,EAAE,QAAQ,4BAA4B,IAAI,CAAC;CAE/C;AACF;;;;;;;;;;;;;;;;;;;;;;AAuBA,SAAgB,4BAA4B,EAC1C,YACA,cACA,iBAKS;CACT,OAAO,gBAAgB,aAAa,iBAAiB;AACvD;;;;;;;;AASA,SAAgB,6BACd,QACS;CACT,OAAO,OAAO,mBAAmB,KAAK,OAAO,gBAAgB,KAAA;AAC/D"}

package/out/zero-cache/src/services/replicator/reporter/report-schema.js

@@ -15,6 +15,6 @@ valita_exports.object({
 	nextSendTimeMs: valita_exports.number()
 });
 //#endregion
-export { changeSourceReportSchema, changeSourceTimingsSchema };
+export { changeSourceReportSchema, changeSourceTimingsSchema, replicationTimingsSchema };
 
 //# sourceMappingURL=report-schema.js.map

package/out/zero-cache/src/services/replicator/reporter/report-schema.js.map

@@ -1 +1 @@
-{"version":3,"file":"report-schema.js","names":[],"sources":["../../../../../../../zero-cache/src/services/replicator/reporter/report-schema.ts"],"sourcesContent":["import * as v from '../../../../../shared/src/valita.ts';\n\nexport const changeSourceTimingsSchema = v.object({\n  sendTimeMs: v.number(),\n  commitTimeMs: v.number(),\n  receiveTimeMs: v.number(),\n});\n\nexport const changeSourceReportSchema = v.object({\n  lastTimings: changeSourceTimingsSchema,\n  nextSendTimeMs: v.number(),\n});\n\nexport const replicationTimingsSchema = changeSourceTimingsSchema.extend({\n  replicateTimeMs: v.number(),\n});\n\nexport const replicationReportSchema = v.object({\n  lastTimings: replicationTimingsSchema.optional(),\n  nextSendTimeMs: v.number(),\n});\n\nexport type ChangeSourceTimings = v.Infer<typeof changeSourceTimingsSchema>;\nexport type ChangeSourceReport = v.Infer<typeof changeSourceReportSchema>;\n\nexport type ReplicationTimings = v.Infer<typeof replicationTimingsSchema>;\nexport type ReplicationReport = v.Infer<typeof replicationReportSchema>;\n"],"mappings":";;AAEA,IAAa,4BAA4B,eAAE,OAAO;CAChD,YAAY,eAAE,OAAO;CACrB,cAAc,eAAE,OAAO;CACvB,eAAe,eAAE,OAAO;AAC1B,CAAC;AAED,IAAa,2BAA2B,eAAE,OAAO;CAC/C,aAAa;CACb,gBAAgB,eAAE,OAAO;AAC3B,CAAC;AAED,IAAa,2BAA2B,0BAA0B,OAAO,EACvE,iBAAiB,eAAE,OAAO,EAC5B,CAAC;AAEsC,eAAE,OAAO;CAC9C,aAAa,yBAAyB,SAAS;CAC/C,gBAAgB,eAAE,OAAO;AAC3B,CAAC"}
+{"version":3,"file":"report-schema.js","names":["v"],"sources":["../../../../../../../zero-cache/src/services/replicator/reporter/report-schema.ts"],"sourcesContent":["import * as v from '../../../../../shared/src/valita.ts';\n\nexport const changeSourceTimingsSchema = v.object({\n  sendTimeMs: v.number(),\n  commitTimeMs: v.number(),\n  receiveTimeMs: v.number(),\n});\n\nexport const changeSourceReportSchema = v.object({\n  lastTimings: changeSourceTimingsSchema,\n  nextSendTimeMs: v.number(),\n});\n\nexport const replicationTimingsSchema = changeSourceTimingsSchema.extend({\n  replicateTimeMs: v.number(),\n});\n\nexport const replicationReportSchema = v.object({\n  lastTimings: replicationTimingsSchema.optional(),\n  nextSendTimeMs: v.number(),\n});\n\nexport type ChangeSourceTimings = v.Infer<typeof changeSourceTimingsSchema>;\nexport type ChangeSourceReport = v.Infer<typeof changeSourceReportSchema>;\n\nexport type ReplicationTimings = v.Infer<typeof replicationTimingsSchema>;\nexport type ReplicationReport = v.Infer<typeof replicationReportSchema>;\n"],"mappings":";;AAEA,IAAa,4BAA4BA,eAAE,OAAO;CAChD,YAAYA,eAAE,OAAO;CACrB,cAAcA,eAAE,OAAO;CACvB,eAAeA,eAAE,OAAO;AAC1B,CAAC;AAED,IAAa,2BAA2BA,eAAE,OAAO;CAC/C,aAAa;CACb,gBAAgBA,eAAE,OAAO;AAC3B,CAAC;AAED,IAAa,2BAA2B,0BAA0B,OAAO,EACvE,iBAAiBA,eAAE,OAAO,EAC5B,CAAC;AAEsCA,eAAE,OAAO;CAC9C,aAAa,yBAAyB,SAAS;CAC/C,gBAAgBA,eAAE,OAAO;AAC3B,CAAC"}

package/out/zero-cache/src/services/replicator/schema/backfilling.js

@@ -227,7 +227,7 @@ function populateBackfillingFromColumnMetadata(lc, db) {
         WHERE backfill IS NOT NULL
         ORDER BY table_name, column_name
     `).all();
-	const unresolvable = getOrCreateCounter("replica", "backfilling_unresolvable_rows", "In-flight backfills whose upstream table identity could not be recovered from the replica when \"" + BACKFILLING_TABLE + "\" was created. A best-effort identity is stored; the expected count is 0.");
+	const unresolvable = getOrCreateCounter("replica", "backfilling_unresolvable_rows", "In-flight backfills whose upstream table identity could not be recovered from the replica when \"_zero.backfilling\" was created. A best-effort identity is stored; the expected count is 0.");
 	if (backfilling.length === 0) {
 		unresolvable.add(0);
 		return;

package/out/zero-cache/src/services/replicator/schema/backfilling.js.map

@@ -1 +1 @@
-{"version":3,"file":"backfilling.js","names":["#db","#run","#upsert","#renameTable","#dropTable","#renameColumn","#deleteColumn","#getProgress","#setProgress","#dropColumn"],"sources":["../../../../../../../zero-cache/src/services/replicator/schema/backfilling.ts"],"sourcesContent":["/**\n * The replica's backfill cookie, keyed by upstream identity.\n *\n * `_zero.column_metadata.backfill` already records which columns are being\n * backfilled, and it stays authoritative for `isBackfilling`, which is what the\n * replica's own read paths use. What it cannot do is name the table an upstream\n * would recognize: it is keyed by *lite* table name, and `liteTableName()`\n * collapses the `public` schema and has no inverse anywhere in the tree. A lite\n * name containing a dot is ambiguous — `{public, \"a.b\"}` and `{a, b}` produce\n * the same string — and a table with backfilling columns and no\n * `_zero.tableMetadata` row (legal, since its metadata is optional) cannot be\n * resolved at all.\n *\n * That is the whole of what blocks a change log from being initialized out of a\n * restored replica, so this table carries the identity the reconstruction needs:\n * a straight row copy into the change log's own cookie tables, with nothing to\n * reconstruct and nothing to guess. The replica's copy is the *seed* source; the\n * change log's copy (`replicator/change-log-cookies.ts`) is the *working* set,\n * folded forward from the seed as the log's head advances.\n *\n * The cost is a second copy of `backfill` inside the replica, taken\n * deliberately — though not to save writes: `_zero.column_metadata` is written\n * only by initial sync, the DDL handlers, and backfill completion, so two more\n * columns on it would be free. It is a matter of shape. That table is keyed by\n * lite table name, so `schema`/`table` would be non-key duplicates of its own\n * key, to be held in sync with it across every rename with no answer to which\n * encoding is authoritative. This one is keyed exactly as `cdc.backfilling` and\n * `_zero.changeLogBackfilling` are, which is what makes the change log's\n * initialization comparison a row-set diff rather than a translation, and what\n * lets all three stores interpret the same fold. It also holds in-flight backfills and nothing else,\n * where the same query against `column_metadata` is an unindexed scan over a row\n * per column of every table.\n *\n * This table is write-only from the replica's perspective. Nothing in the\n * replicator reads it; it exists to be read by the change-streamer.\n */\n\nimport type {LogContext} from '@rocicorp/logger';\nimport {unreachable} from '../../../../../shared/src/asserts.ts';\nimport {BigIntJSON} from '../../../../../shared/src/bigint-json.ts';\nimport type {Database, Statement} from '../../../../../zqlite/src/db.ts';\nimport {getOrCreateCounter} from '../../../observability/metrics.ts';\nimport {liteTableName} from '../../../types/names.ts';\nimport type {\n  BackfillID,\n  BackfillProgressMark,\n  Identifier,\n  SchemaChange,\n  TableMetadata,\n} from '../../change-source/protocol/current/data.ts';\nimport type {BackfillRequest} from '../../change-source/protocol/current/upstream.ts';\nimport {\n  backfillRequestsFrom,\n  cookieOps,\n  type CookieOp,\n  type CookieSet,\n} from '../change-log-cookies.ts';\n\nexport const BACKFILLING_TABLE = '_zero.backfilling';\n\n// `backfill` holds the JSON that `cdc.backfilling` holds as JSONB. SQLite has\n// no JSONB, and nothing here queries into the document: it is stored to be\n// handed back to the change source verbatim.\n//\n// `progress` holds the JSON of the BackfillProgressMark up to which the\n// column's backfill data has been received contiguously (see\n// `acceptBackfill()`), or NULL if no data has been received (i.e. the backfill\n// must start from scratch). It is deliberately a separate column from\n// `backfill` so that the cookie set (and its comparison with the change log's\n// cookie jars) is unaffected by it.\nexport const CREATE_BACKFILLING_TABLE = /*sql*/ `\n  CREATE TABLE \"${BACKFILLING_TABLE}\" (\n    \"schema\"   TEXT NOT NULL,\n    \"table\"    TEXT NOT NULL,\n    \"column\"   TEXT NOT NULL,\n    \"backfill\" TEXT NOT NULL,\n    \"progress\" TEXT,\n    PRIMARY KEY (\"schema\", \"table\", \"column\")\n  );\n`;\n\n/**\n * The replica's interpreter of the cookie fold in\n * `replicator/change-log-cookies.ts`.\n *\n * The fold is shared rather than re-implemented for the same reason the\n * Postgres and SQLite change logs share it: the three stores must agree on every\n * transition forever, and the failure mode when they drift is a backfill that is\n * silently never re-requested, on a column that is then silently never\n * populated. Sharing it makes the replica-versus-Postgres comparison a\n * regression test on the interpreters rather than on three implementations of\n * the same switch.\n *\n * Only the backfill half of the fold is interpreted here. The replica's metadata\n * cookie is `_zero.tableMetadata.upstreamMetadata`, which `TableMetadataTracker`\n * already maintains at the same sites, so an `upsert-metadata` op is inert.\n *\n * Statements are lazily prepared, matching `TableMetadataTracker`: a replica\n * that never sees a schema change never prepares any of them.\n */\nexport class BackfillingTracker {\n  readonly #db: Database;\n\n  #upsert: Statement | undefined;\n  #renameTable: Statement | undefined;\n  #dropTable: Statement | undefined;\n  #renameColumn: Statement | undefined;\n  #dropColumn: Statement | undefined;\n  #getProgress: Statement | undefined;\n  #setProgress: Statement | undefined;\n\n  constructor(db: Database) {\n    this.#db = db;\n  }\n\n  /**\n   * Applies the change's backfill-cookie ops, returning the ops that were\n   * applied. Changes that carry no backfill state — index changes, metadata-only\n   * updates, and the `create-table` / `add-column` variants from a change source\n   * that does not support backfill — apply nothing.\n   */\n  apply(change: SchemaChange): CookieOp[] {\n    const ops = cookieOps(change);\n    for (const op of ops) {\n      this.#run(op);\n    }\n    return ops;\n  }\n\n  #run(op: CookieOp): void {\n    switch (op.op) {\n      case 'upsert-metadata':\n        // Maintained by TableMetadataTracker in \"_zero.tableMetadata\".\n        break;\n\n      case 'upsert-backfill':\n        (this.#upsert ??= this.#db.prepare(/*sql*/ `\n          INSERT INTO \"${BACKFILLING_TABLE}\"\n            (\"schema\", \"table\", \"column\", \"backfill\") VALUES (?, ?, ?, ?)\n            ON CONFLICT (\"schema\", \"table\", \"column\")\n            DO UPDATE SET \"backfill\" = excluded.\"backfill\", \"progress\" = NULL\n        `)).run(\n          op.table.schema,\n          op.table.name,\n          op.column,\n          BigIntJSON.stringify(op.backfill),\n        );\n        break;\n\n      case 'rename-table':\n        (this.#renameTable ??= this.#db.prepare(/*sql*/ `\n          UPDATE \"${BACKFILLING_TABLE}\"\n            SET \"schema\" = ?, \"table\" = ? WHERE \"schema\" = ? AND \"table\" = ?\n        `)).run(op.new.schema, op.new.name, op.old.schema, op.old.name);\n        break;\n\n      case 'drop-table':\n        (this.#dropTable ??= this.#db.prepare(/*sql*/ `\n          DELETE FROM \"${BACKFILLING_TABLE}\"\n            WHERE \"schema\" = ? AND \"table\" = ?\n        `)).run(op.table.schema, op.table.name);\n        break;\n\n      case 'rename-column':\n        (this.#renameColumn ??= this.#db.prepare(/*sql*/ `\n          UPDATE \"${BACKFILLING_TABLE}\" SET \"column\" = ?\n            WHERE \"schema\" = ? AND \"table\" = ? AND \"column\" = ?\n        `)).run(op.new, op.table.schema, op.table.name, op.old);\n        break;\n\n      case 'drop-column':\n        this.#deleteColumn(op.table, op.column);\n        break;\n\n      case 'complete-backfill':\n        // A per-column delete rather than an `IN` list, which cannot be a\n        // single prepared statement across arities. The columns of one\n        // completed backfill are few, and this runs only on schema changes.\n        for (const column of op.columns) {\n          this.#deleteColumn(op.table, column);\n        }\n        break;\n\n      default:\n        unreachable(op);\n    }\n  }\n\n  /**\n   * Returns the tracked progress of the backfilling columns of the `table`,\n   * keyed by column name. Columns that are not backfilling are absent.\n   */\n  getProgress(\n    table: Identifier,\n  ): Map<string, BackfillProgressMark | undefined> {\n    const rows = (this.#getProgress ??= this.#db.prepare(/*sql*/ `\n      SELECT \"column\", \"progress\" FROM \"${BACKFILLING_TABLE}\"\n        WHERE \"schema\" = ? AND \"table\" = ?\n    `)).all<{column: string; progress: string | null}>(\n      table.schema,\n      table.name,\n    );\n    return new Map(\n      rows.map(({column, progress}) => [\n        column,\n        progress === null\n          ? undefined\n          : (BigIntJSON.parse(progress) as BackfillProgressMark),\n      ]),\n    );\n  }\n\n  /** Sets (or clears) the tracked progress of a backfilling column. */\n  setProgress(\n    table: Identifier,\n    column: string,\n    progress: BackfillProgressMark | undefined,\n  ): void {\n    (this.#setProgress ??= this.#db.prepare(/*sql*/ `\n      UPDATE \"${BACKFILLING_TABLE}\" SET \"progress\" = ?\n        WHERE \"schema\" = ? AND \"table\" = ? AND \"column\" = ?\n    `)).run(\n      progress === undefined ? null : BigIntJSON.stringify(progress),\n      table.schema,\n      table.name,\n      column,\n    );\n  }\n\n  /**\n   * Marks the backfill of the `columns` as complete. This is the subset of\n   * a `backfill-completed` message's columns (including row key columns)\n   * whose progress is contiguous with the message.\n   */\n  complete(table: Identifier, columns: readonly string[]): void {\n    this.#run({op: 'complete-backfill', table, columns});\n  }\n\n  #deleteColumn(table: Identifier, column: string): void {\n    (this.#dropColumn ??= this.#db.prepare(/*sql*/ `\n      DELETE FROM \"${BACKFILLING_TABLE}\"\n        WHERE \"schema\" = ? AND \"table\" = ? AND \"column\" = ?\n    `)).run(table.schema, table.name, column);\n  }\n}\n\n/**\n * Returns the {@link BackfillRequest}s from the current replica cookies. The\n * result has the same shape as the Postgres initialization result.\n *\n * The caller must pair this snapshot with the replica state version. When the\n * replica trails the change log, the comparison applies the missing schema\n * changes before it compares the results.\n */\nexport function readBackfillRequests(db: Database): BackfillRequest[] {\n  return backfillRequestsFrom(readReplicaCookies(db));\n}\n\n/**\n * Returns the {@link BackfillRequest}s of the replica's pending backfills,\n * including the tracked progress of each column. These are the backfills that\n * a subscriber reports to the change-streamer when starting a subscription.\n *\n * The caller must pair this snapshot with the replica state version.\n */\nexport function readReplicaBackfills(db: Database): BackfillRequest[] {\n  const requests = readBackfillRequests(db);\n  const progress = db\n    .prepare(/*sql*/ `\n      SELECT \"schema\", \"table\", \"column\", \"progress\"\n        FROM \"${BACKFILLING_TABLE}\" WHERE \"progress\" IS NOT NULL\n    `)\n    .all<{schema: string; table: string; column: string; progress: string}>();\n  for (const {schema, table, column, progress: mark} of progress) {\n    const req = requests.find(\n      r => r.table.schema === schema && r.table.name === table,\n    );\n    const col = req?.columns[column];\n    if (col) {\n      col.progress = BigIntJSON.parse(mark) as BackfillProgressMark;\n    }\n  }\n  return requests;\n}\n\n/**\n * The replica's whole cookie set, in the shape the change log holds it — the\n * seed a change log would be initialized from, and the replica-derived half of\n * the initialization comparison.\n *\n * The metadata half is filtered on `upstreamMetadata IS NOT NULL` because\n * `_zero.tableMetadata` carries `minRowVersion` as well, so a row exists for\n * every table whose rows were ever force-re-downloaded, whether or not upstream\n * ever sent metadata for it. Neither cookie store can represent that row:\n * `cdc.tableMetadata.\"metadata\"` and `_zero.changeLogTableMetadata.\"metadata\"`\n * are both `NOT NULL`.\n *\n * Only meaningful paired with `_zero.replicationState.stateVersion`, read in\n * the same snapshot (invariant 15).\n */\nexport function readReplicaCookies(db: Database): CookieSet {\n  const tableMetadata = db\n    .prepare(/*sql*/ `\n      SELECT \"schema\", \"table\", \"upstreamMetadata\" AS \"metadata\"\n        FROM \"_zero.tableMetadata\"\n        WHERE \"upstreamMetadata\" IS NOT NULL\n        ORDER BY \"schema\", \"table\"\n    `)\n    .all<{schema: string; table: string; metadata: string}>()\n    .map(({schema, table, metadata}) => ({\n      schema,\n      table,\n      metadata: BigIntJSON.parse(metadata) as TableMetadata,\n    }));\n\n  const backfilling = db\n    .prepare(/*sql*/ `\n      SELECT \"schema\", \"table\", \"column\", \"backfill\"\n        FROM \"${BACKFILLING_TABLE}\"\n        ORDER BY \"schema\", \"table\", \"column\"\n    `)\n    .all<{schema: string; table: string; column: string; backfill: string}>()\n    .map(({schema, table, column, backfill}) => ({\n      schema,\n      table,\n      column,\n      backfill: BigIntJSON.parse(backfill) as BackfillID,\n    }));\n\n  return {tableMetadata, backfilling};\n}\n\n/**\n * Seeds {@link BACKFILLING_TABLE} from `_zero.column_metadata`, resolving each\n * lite table name back to its upstream identity. Used by the v17 migration.\n *\n * The resolution is exact for a table that has a `_zero.tableMetadata` row, and\n * exact for a lite name with no dot in it — `liteTableName()` only omits the\n * schema for `public`. What is left over is an in-flight backfill, on a\n * non-`public` schema, on a table with no metadata row, at the instant of\n * upgrade. Those get the best-effort first-dot split and are counted, because a\n * wrong identity produces a `BackfillRequest` the change source will reject\n * rather than a silently wrong one, and because the initialization comparison\n * surfaces it before anything depends on it. The expected count is zero.\n *\n * A full replace rather than an insert-if-absent: `migrateData` re-runs after a\n * rollback and roll-forward, during which a v16 zero-cache advanced the replica\n * without knowing this table existed. Rebuilding it from `column_metadata` —\n * which that zero-cache *did* maintain — is what makes the re-run both\n * idempotent and correct.\n */\nexport function populateBackfillingFromColumnMetadata(\n  lc: LogContext,\n  db: Database,\n): void {\n  db.prepare(/*sql*/ `DELETE FROM \"${BACKFILLING_TABLE}\"`).run();\n\n  const backfilling = db\n    .prepare(/*sql*/ `\n      SELECT table_name, column_name, backfill FROM \"_zero.column_metadata\"\n        WHERE backfill IS NOT NULL\n        ORDER BY table_name, column_name\n    `)\n    .all<{table_name: string; column_name: string; backfill: string}>();\n\n  const unresolvable = getOrCreateCounter(\n    'replica',\n    'backfilling_unresolvable_rows',\n    'In-flight backfills whose upstream table identity could not be recovered ' +\n      'from the replica when \"' +\n      BACKFILLING_TABLE +\n      '\" was created. A best-effort identity is stored; the expected count is 0.',\n  );\n  if (backfilling.length === 0) {\n    unresolvable.add(0); // the common case: publish the series anyway\n    return;\n  }\n\n  const byLiteName = new Map<string, Identifier>(\n    db\n      .prepare(/*sql*/ `SELECT \"schema\", \"table\" FROM \"_zero.tableMetadata\"`)\n      .all<{schema: string; table: string}>()\n      .map(({schema, table}) => [\n        liteTableName({schema, name: table}),\n        {schema, name: table},\n      ]),\n  );\n\n  const insert = db.prepare(/*sql*/ `\n    INSERT INTO \"${BACKFILLING_TABLE}\"\n      (\"schema\", \"table\", \"column\", \"backfill\") VALUES (?, ?, ?, ?)\n  `);\n  let unresolved = 0;\n  for (const {table_name: liteName, column_name, backfill} of backfilling) {\n    const known = byLiteName.get(liteName);\n    const dot = known ? -1 : liteName.indexOf('.');\n    const table = known ?? {\n      schema: dot < 0 ? 'public' : liteName.slice(0, dot),\n      name: dot < 0 ? liteName : liteName.slice(dot + 1),\n    };\n    if (!known && dot >= 0) {\n      unresolved++;\n      lc.warn?.(\n        `no metadata for backfilling table \"${liteName}\"; ` +\n          `assuming {schema: \"${table.schema}\", name: \"${table.name}\"}`,\n      );\n    }\n    insert.run(table.schema, table.name, column_name, backfill);\n  }\n  unresolvable.add(unresolved);\n  lc.info?.(\n    `seeded \"${BACKFILLING_TABLE}\" with ${backfilling.length} in-flight ` +\n      `backfill(s), ${unresolved} of which could not be resolved`,\n  );\n}\n"],"mappings":";;;;;;AA0DA,IAAa,oBAAoB;AAYjC,IAAa,2BAAmC;kBAC9B,kBAAkB;;;;;;;;;;;;;;;;;;;;;;;;;;;;AA6BpC,IAAa,qBAAb,MAAgC;CAC9B;CAEA;CACA;CACA;CACA;CACA;CACA;CACA;CAEA,YAAY,IAAc;EACxB,KAAKA,MAAM;CACb;;;;;;;CAQA,MAAM,QAAkC;EACtC,MAAM,MAAM,UAAU,MAAM;EAC5B,KAAK,MAAM,MAAM,KACf,KAAKC,KAAK,EAAE;EAEd,OAAO;CACT;CAEA,KAAK,IAAoB;EACvB,QAAQ,GAAG,IAAX;GACE,KAAK,mBAEH;GAEF,KAAK;IACH,CAAC,KAAKC,YAAY,KAAKF,IAAI,QAAgB;yBAC1B,kBAAkB;;;;SAIlC,GAAG,IACF,GAAG,MAAM,QACT,GAAG,MAAM,MACT,GAAG,QACH,WAAW,UAAU,GAAG,QAAQ,CAClC;IACA;GAEF,KAAK;IACH,CAAC,KAAKG,iBAAiB,KAAKH,IAAI,QAAgB;oBACpC,kBAAkB;;SAE7B,GAAG,IAAI,GAAG,IAAI,QAAQ,GAAG,IAAI,MAAM,GAAG,IAAI,QAAQ,GAAG,IAAI,IAAI;IAC9D;GAEF,KAAK;IACH,CAAC,KAAKI,eAAe,KAAKJ,IAAI,QAAgB;yBAC7B,kBAAkB;;SAElC,GAAG,IAAI,GAAG,MAAM,QAAQ,GAAG,MAAM,IAAI;IACtC;GAEF,KAAK;IACH,CAAC,KAAKK,kBAAkB,KAAKL,IAAI,QAAgB;oBACrC,kBAAkB;;SAE7B,GAAG,IAAI,GAAG,KAAK,GAAG,MAAM,QAAQ,GAAG,MAAM,MAAM,GAAG,GAAG;IACtD;GAEF,KAAK;IACH,KAAKM,cAAc,GAAG,OAAO,GAAG,MAAM;IACtC;GAEF,KAAK;IAIH,KAAK,MAAM,UAAU,GAAG,SACtB,KAAKA,cAAc,GAAG,OAAO,MAAM;IAErC;GAEF,SACE,YAAY,EAAE;EAClB;CACF;;;;;CAMA,YACE,OAC+C;EAC/C,MAAM,QAAQ,KAAKC,iBAAiB,KAAKP,IAAI,QAAgB;0CACvB,kBAAkB;;KAEvD,GAAG,IACF,MAAM,QACN,MAAM,IACR;EACA,OAAO,IAAI,IACT,KAAK,KAAK,EAAC,QAAQ,eAAc,CAC/B,QACA,aAAa,OACT,KAAA,IACC,WAAW,MAAM,QAAQ,CAChC,CAAC,CACH;CACF;;CAGA,YACE,OACA,QACA,UACM;EACN,CAAC,KAAKQ,iBAAiB,KAAKR,IAAI,QAAgB;gBACpC,kBAAkB;;KAE7B,GAAG,IACF,aAAa,KAAA,IAAY,OAAO,WAAW,UAAU,QAAQ,GAC7D,MAAM,QACN,MAAM,MACN,MACF;CACF;;;;;;CAOA,SAAS,OAAmB,SAAkC;EAC5D,KAAKC,KAAK;GAAC,IAAI;GAAqB;GAAO;EAAO,CAAC;CACrD;CAEA,cAAc,OAAmB,QAAsB;EACrD,CAAC,KAAKQ,gBAAgB,KAAKT,IAAI,QAAgB;qBAC9B,kBAAkB;;KAElC,GAAG,IAAI,MAAM,QAAQ,MAAM,MAAM,MAAM;CAC1C;AACF;;;;;;;;;AAUA,SAAgB,qBAAqB,IAAiC;CACpE,OAAO,qBAAqB,mBAAmB,EAAE,CAAC;AACpD;;;;;;;;AASA,SAAgB,qBAAqB,IAAiC;CACpE,MAAM,WAAW,qBAAqB,EAAE;CACxC,MAAM,WAAW,GACd,QAAgB;;gBAEL,kBAAkB;KAC7B,EACA,IAAuE;CAC1E,KAAK,MAAM,EAAC,QAAQ,OAAO,QAAQ,UAAU,UAAS,UAAU;EAI9D,MAAM,MAHM,SAAS,MACnB,MAAK,EAAE,MAAM,WAAW,UAAU,EAAE,MAAM,SAAS,KAEzC,GAAK,QAAQ;EACzB,IAAI,KACF,IAAI,WAAW,WAAW,MAAM,IAAI;CAExC;CACA,OAAO;AACT;;;;;;;;;;;;;;;;AAiBA,SAAgB,mBAAmB,IAAyB;CA6B1D,OAAO;EAAC,eA5Bc,GACnB,QAAgB;;;;;KAKhB,EACA,IAAuD,EACvD,KAAK,EAAC,QAAQ,OAAO,gBAAe;GACnC;GACA;GACA,UAAU,WAAW,MAAM,QAAQ;EACrC,EAgBM;EAAe,aAdH,GACjB,QAAgB;;gBAEL,kBAAkB;;KAE7B,EACA,IAAuE,EACvE,KAAK,EAAC,QAAQ,OAAO,QAAQ,gBAAe;GAC3C;GACA;GACA;GACA,UAAU,WAAW,MAAM,QAAQ;EACrC,EAEqB;CAAW;AACpC;;;;;;;;;;;;;;;;;;;;AAqBA,SAAgB,sCACd,IACA,IACM;CACN,GAAG,QAAgB,gBAAgB,kBAAkB,EAAE,EAAE,IAAI;CAE7D,MAAM,cAAc,GACjB,QAAgB;;;;KAIhB,EACA,IAAiE;CAEpE,MAAM,eAAe,mBACnB,WACA,iCACA,sGAEE,oBACA,4EACJ;CACA,IAAI,YAAY,WAAW,GAAG;EAC5B,aAAa,IAAI,CAAC;EAClB;CACF;CAEA,MAAM,aAAa,IAAI,IACrB,GACG,QAAgB,qDAAqD,EACrE,IAAqC,EACrC,KAAK,EAAC,QAAQ,YAAW,CACxB,cAAc;EAAC;EAAQ,MAAM;CAAK,CAAC,GACnC;EAAC;EAAQ,MAAM;CAAK,CACtB,CAAC,CACL;CAEA,MAAM,SAAS,GAAG,QAAgB;mBACjB,kBAAkB;;GAElC;CACD,IAAI,aAAa;CACjB,KAAK,MAAM,EAAC,YAAY,UAAU,aAAa,cAAa,aAAa;EACvE,MAAM,QAAQ,WAAW,IAAI,QAAQ;EACrC,MAAM,MAAM,QAAQ,KAAK,SAAS,QAAQ,GAAG;EAC7C,MAAM,QAAQ,SAAS;GACrB,QAAQ,MAAM,IAAI,WAAW,SAAS,MAAM,GAAG,GAAG;GAClD,MAAM,MAAM,IAAI,WAAW,SAAS,MAAM,MAAM,CAAC;EACnD;EACA,IAAI,CAAC,SAAS,OAAO,GAAG;GACtB;GACA,GAAG,OACD,sCAAsC,SAAS,wBACvB,MAAM,OAAO,YAAY,MAAM,KAAK,GAC9D;EACF;EACA,OAAO,IAAI,MAAM,QAAQ,MAAM,MAAM,aAAa,QAAQ;CAC5D;CACA,aAAa,IAAI,UAAU;CAC3B,GAAG,OACD,WAAW,kBAAkB,SAAS,YAAY,OAAO,0BACvC,WAAW,gCAC/B;AACF"}
+{"version":3,"file":"backfilling.js","names":[],"sources":["../../../../../../../zero-cache/src/services/replicator/schema/backfilling.ts"],"sourcesContent":["/**\n * The replica's backfill cookie, keyed by upstream identity.\n *\n * `_zero.column_metadata.backfill` already records which columns are being\n * backfilled, and it stays authoritative for `isBackfilling`, which is what the\n * replica's own read paths use. What it cannot do is name the table an upstream\n * would recognize: it is keyed by *lite* table name, and `liteTableName()`\n * collapses the `public` schema and has no inverse anywhere in the tree. A lite\n * name containing a dot is ambiguous — `{public, \"a.b\"}` and `{a, b}` produce\n * the same string — and a table with backfilling columns and no\n * `_zero.tableMetadata` row (legal, since its metadata is optional) cannot be\n * resolved at all.\n *\n * That is the whole of what blocks a change log from being initialized out of a\n * restored replica, so this table carries the identity the reconstruction needs:\n * a straight row copy into the change log's own cookie tables, with nothing to\n * reconstruct and nothing to guess. The replica's copy is the *seed* source; the\n * change log's copy (`replicator/change-log-cookies.ts`) is the *working* set,\n * folded forward from the seed as the log's head advances.\n *\n * The cost is a second copy of `backfill` inside the replica, taken\n * deliberately — though not to save writes: `_zero.column_metadata` is written\n * only by initial sync, the DDL handlers, and backfill completion, so two more\n * columns on it would be free. It is a matter of shape. That table is keyed by\n * lite table name, so `schema`/`table` would be non-key duplicates of its own\n * key, to be held in sync with it across every rename with no answer to which\n * encoding is authoritative. This one is keyed exactly as `cdc.backfilling` and\n * `_zero.changeLogBackfilling` are, which is what makes the change log's\n * initialization comparison a row-set diff rather than a translation, and what\n * lets all three stores interpret the same fold. It also holds in-flight backfills and nothing else,\n * where the same query against `column_metadata` is an unindexed scan over a row\n * per column of every table.\n *\n * This table is write-only from the replica's perspective. Nothing in the\n * replicator reads it; it exists to be read by the change-streamer.\n */\n\nimport type {LogContext} from '@rocicorp/logger';\nimport {unreachable} from '../../../../../shared/src/asserts.ts';\nimport {BigIntJSON} from '../../../../../shared/src/bigint-json.ts';\nimport type {Database, Statement} from '../../../../../zqlite/src/db.ts';\nimport {getOrCreateCounter} from '../../../observability/metrics.ts';\nimport {liteTableName} from '../../../types/names.ts';\nimport type {\n  BackfillID,\n  BackfillProgressMark,\n  Identifier,\n  SchemaChange,\n  TableMetadata,\n} from '../../change-source/protocol/current/data.ts';\nimport type {BackfillRequest} from '../../change-source/protocol/current/upstream.ts';\nimport {\n  backfillRequestsFrom,\n  cookieOps,\n  type CookieOp,\n  type CookieSet,\n} from '../change-log-cookies.ts';\n\nexport const BACKFILLING_TABLE = '_zero.backfilling';\n\n// `backfill` holds the JSON that `cdc.backfilling` holds as JSONB. SQLite has\n// no JSONB, and nothing here queries into the document: it is stored to be\n// handed back to the change source verbatim.\n//\n// `progress` holds the JSON of the BackfillProgressMark up to which the\n// column's backfill data has been received contiguously (see\n// `acceptBackfill()`), or NULL if no data has been received (i.e. the backfill\n// must start from scratch). It is deliberately a separate column from\n// `backfill` so that the cookie set (and its comparison with the change log's\n// cookie jars) is unaffected by it.\nexport const CREATE_BACKFILLING_TABLE = /*sql*/ `\n  CREATE TABLE \"${BACKFILLING_TABLE}\" (\n    \"schema\"   TEXT NOT NULL,\n    \"table\"    TEXT NOT NULL,\n    \"column\"   TEXT NOT NULL,\n    \"backfill\" TEXT NOT NULL,\n    \"progress\" TEXT,\n    PRIMARY KEY (\"schema\", \"table\", \"column\")\n  );\n`;\n\n/**\n * The replica's interpreter of the cookie fold in\n * `replicator/change-log-cookies.ts`.\n *\n * The fold is shared rather than re-implemented for the same reason the\n * Postgres and SQLite change logs share it: the three stores must agree on every\n * transition forever, and the failure mode when they drift is a backfill that is\n * silently never re-requested, on a column that is then silently never\n * populated. Sharing it makes the replica-versus-Postgres comparison a\n * regression test on the interpreters rather than on three implementations of\n * the same switch.\n *\n * Only the backfill half of the fold is interpreted here. The replica's metadata\n * cookie is `_zero.tableMetadata.upstreamMetadata`, which `TableMetadataTracker`\n * already maintains at the same sites, so an `upsert-metadata` op is inert.\n *\n * Statements are lazily prepared, matching `TableMetadataTracker`: a replica\n * that never sees a schema change never prepares any of them.\n */\nexport class BackfillingTracker {\n  readonly #db: Database;\n\n  #upsert: Statement | undefined;\n  #renameTable: Statement | undefined;\n  #dropTable: Statement | undefined;\n  #renameColumn: Statement | undefined;\n  #dropColumn: Statement | undefined;\n  #getProgress: Statement | undefined;\n  #setProgress: Statement | undefined;\n\n  constructor(db: Database) {\n    this.#db = db;\n  }\n\n  /**\n   * Applies the change's backfill-cookie ops, returning the ops that were\n   * applied. Changes that carry no backfill state — index changes, metadata-only\n   * updates, and the `create-table` / `add-column` variants from a change source\n   * that does not support backfill — apply nothing.\n   */\n  apply(change: SchemaChange): CookieOp[] {\n    const ops = cookieOps(change);\n    for (const op of ops) {\n      this.#run(op);\n    }\n    return ops;\n  }\n\n  #run(op: CookieOp): void {\n    switch (op.op) {\n      case 'upsert-metadata':\n        // Maintained by TableMetadataTracker in \"_zero.tableMetadata\".\n        break;\n\n      case 'upsert-backfill':\n        (this.#upsert ??= this.#db.prepare(/*sql*/ `\n          INSERT INTO \"${BACKFILLING_TABLE}\"\n            (\"schema\", \"table\", \"column\", \"backfill\") VALUES (?, ?, ?, ?)\n            ON CONFLICT (\"schema\", \"table\", \"column\")\n            DO UPDATE SET \"backfill\" = excluded.\"backfill\", \"progress\" = NULL\n        `)).run(\n          op.table.schema,\n          op.table.name,\n          op.column,\n          BigIntJSON.stringify(op.backfill),\n        );\n        break;\n\n      case 'rename-table':\n        (this.#renameTable ??= this.#db.prepare(/*sql*/ `\n          UPDATE \"${BACKFILLING_TABLE}\"\n            SET \"schema\" = ?, \"table\" = ? WHERE \"schema\" = ? AND \"table\" = ?\n        `)).run(op.new.schema, op.new.name, op.old.schema, op.old.name);\n        break;\n\n      case 'drop-table':\n        (this.#dropTable ??= this.#db.prepare(/*sql*/ `\n          DELETE FROM \"${BACKFILLING_TABLE}\"\n            WHERE \"schema\" = ? AND \"table\" = ?\n        `)).run(op.table.schema, op.table.name);\n        break;\n\n      case 'rename-column':\n        (this.#renameColumn ??= this.#db.prepare(/*sql*/ `\n          UPDATE \"${BACKFILLING_TABLE}\" SET \"column\" = ?\n            WHERE \"schema\" = ? AND \"table\" = ? AND \"column\" = ?\n        `)).run(op.new, op.table.schema, op.table.name, op.old);\n        break;\n\n      case 'drop-column':\n        this.#deleteColumn(op.table, op.column);\n        break;\n\n      case 'complete-backfill':\n        // A per-column delete rather than an `IN` list, which cannot be a\n        // single prepared statement across arities. The columns of one\n        // completed backfill are few, and this runs only on schema changes.\n        for (const column of op.columns) {\n          this.#deleteColumn(op.table, column);\n        }\n        break;\n\n      default:\n        unreachable(op);\n    }\n  }\n\n  /**\n   * Returns the tracked progress of the backfilling columns of the `table`,\n   * keyed by column name. Columns that are not backfilling are absent.\n   */\n  getProgress(\n    table: Identifier,\n  ): Map<string, BackfillProgressMark | undefined> {\n    const rows = (this.#getProgress ??= this.#db.prepare(/*sql*/ `\n      SELECT \"column\", \"progress\" FROM \"${BACKFILLING_TABLE}\"\n        WHERE \"schema\" = ? AND \"table\" = ?\n    `)).all<{column: string; progress: string | null}>(\n      table.schema,\n      table.name,\n    );\n    return new Map(\n      rows.map(({column, progress}) => [\n        column,\n        progress === null\n          ? undefined\n          : (BigIntJSON.parse(progress) as BackfillProgressMark),\n      ]),\n    );\n  }\n\n  /** Sets (or clears) the tracked progress of a backfilling column. */\n  setProgress(\n    table: Identifier,\n    column: string,\n    progress: BackfillProgressMark | undefined,\n  ): void {\n    (this.#setProgress ??= this.#db.prepare(/*sql*/ `\n      UPDATE \"${BACKFILLING_TABLE}\" SET \"progress\" = ?\n        WHERE \"schema\" = ? AND \"table\" = ? AND \"column\" = ?\n    `)).run(\n      progress === undefined ? null : BigIntJSON.stringify(progress),\n      table.schema,\n      table.name,\n      column,\n    );\n  }\n\n  /**\n   * Marks the backfill of the `columns` as complete. This is the subset of\n   * a `backfill-completed` message's columns (including row key columns)\n   * whose progress is contiguous with the message.\n   */\n  complete(table: Identifier, columns: readonly string[]): void {\n    this.#run({op: 'complete-backfill', table, columns});\n  }\n\n  #deleteColumn(table: Identifier, column: string): void {\n    (this.#dropColumn ??= this.#db.prepare(/*sql*/ `\n      DELETE FROM \"${BACKFILLING_TABLE}\"\n        WHERE \"schema\" = ? AND \"table\" = ? AND \"column\" = ?\n    `)).run(table.schema, table.name, column);\n  }\n}\n\n/**\n * Returns the {@link BackfillRequest}s from the current replica cookies. The\n * result has the same shape as the Postgres initialization result.\n *\n * The caller must pair this snapshot with the replica state version. When the\n * replica trails the change log, the comparison applies the missing schema\n * changes before it compares the results.\n */\nexport function readBackfillRequests(db: Database): BackfillRequest[] {\n  return backfillRequestsFrom(readReplicaCookies(db));\n}\n\n/**\n * Returns the {@link BackfillRequest}s of the replica's pending backfills,\n * including the tracked progress of each column. These are the backfills that\n * a subscriber reports to the change-streamer when starting a subscription.\n *\n * The caller must pair this snapshot with the replica state version.\n */\nexport function readReplicaBackfills(db: Database): BackfillRequest[] {\n  const requests = readBackfillRequests(db);\n  const progress = db\n    .prepare(/*sql*/ `\n      SELECT \"schema\", \"table\", \"column\", \"progress\"\n        FROM \"${BACKFILLING_TABLE}\" WHERE \"progress\" IS NOT NULL\n    `)\n    .all<{schema: string; table: string; column: string; progress: string}>();\n  for (const {schema, table, column, progress: mark} of progress) {\n    const req = requests.find(\n      r => r.table.schema === schema && r.table.name === table,\n    );\n    const col = req?.columns[column];\n    if (col) {\n      col.progress = BigIntJSON.parse(mark) as BackfillProgressMark;\n    }\n  }\n  return requests;\n}\n\n/**\n * The replica's whole cookie set, in the shape the change log holds it — the\n * seed a change log would be initialized from, and the replica-derived half of\n * the initialization comparison.\n *\n * The metadata half is filtered on `upstreamMetadata IS NOT NULL` because\n * `_zero.tableMetadata` carries `minRowVersion` as well, so a row exists for\n * every table whose rows were ever force-re-downloaded, whether or not upstream\n * ever sent metadata for it. Neither cookie store can represent that row:\n * `cdc.tableMetadata.\"metadata\"` and `_zero.changeLogTableMetadata.\"metadata\"`\n * are both `NOT NULL`.\n *\n * Only meaningful paired with `_zero.replicationState.stateVersion`, read in\n * the same snapshot (invariant 15).\n */\nexport function readReplicaCookies(db: Database): CookieSet {\n  const tableMetadata = db\n    .prepare(/*sql*/ `\n      SELECT \"schema\", \"table\", \"upstreamMetadata\" AS \"metadata\"\n        FROM \"_zero.tableMetadata\"\n        WHERE \"upstreamMetadata\" IS NOT NULL\n        ORDER BY \"schema\", \"table\"\n    `)\n    .all<{schema: string; table: string; metadata: string}>()\n    .map(({schema, table, metadata}) => ({\n      schema,\n      table,\n      metadata: BigIntJSON.parse(metadata) as TableMetadata,\n    }));\n\n  const backfilling = db\n    .prepare(/*sql*/ `\n      SELECT \"schema\", \"table\", \"column\", \"backfill\"\n        FROM \"${BACKFILLING_TABLE}\"\n        ORDER BY \"schema\", \"table\", \"column\"\n    `)\n    .all<{schema: string; table: string; column: string; backfill: string}>()\n    .map(({schema, table, column, backfill}) => ({\n      schema,\n      table,\n      column,\n      backfill: BigIntJSON.parse(backfill) as BackfillID,\n    }));\n\n  return {tableMetadata, backfilling};\n}\n\n/**\n * Seeds {@link BACKFILLING_TABLE} from `_zero.column_metadata`, resolving each\n * lite table name back to its upstream identity. Used by the v17 migration.\n *\n * The resolution is exact for a table that has a `_zero.tableMetadata` row, and\n * exact for a lite name with no dot in it — `liteTableName()` only omits the\n * schema for `public`. What is left over is an in-flight backfill, on a\n * non-`public` schema, on a table with no metadata row, at the instant of\n * upgrade. Those get the best-effort first-dot split and are counted, because a\n * wrong identity produces a `BackfillRequest` the change source will reject\n * rather than a silently wrong one, and because the initialization comparison\n * surfaces it before anything depends on it. The expected count is zero.\n *\n * A full replace rather than an insert-if-absent: `migrateData` re-runs after a\n * rollback and roll-forward, during which a v16 zero-cache advanced the replica\n * without knowing this table existed. Rebuilding it from `column_metadata` —\n * which that zero-cache *did* maintain — is what makes the re-run both\n * idempotent and correct.\n */\nexport function populateBackfillingFromColumnMetadata(\n  lc: LogContext,\n  db: Database,\n): void {\n  db.prepare(/*sql*/ `DELETE FROM \"${BACKFILLING_TABLE}\"`).run();\n\n  const backfilling = db\n    .prepare(/*sql*/ `\n      SELECT table_name, column_name, backfill FROM \"_zero.column_metadata\"\n        WHERE backfill IS NOT NULL\n        ORDER BY table_name, column_name\n    `)\n    .all<{table_name: string; column_name: string; backfill: string}>();\n\n  const unresolvable = getOrCreateCounter(\n    'replica',\n    'backfilling_unresolvable_rows',\n    'In-flight backfills whose upstream table identity could not be recovered ' +\n      'from the replica when \"' +\n      BACKFILLING_TABLE +\n      '\" was created. A best-effort identity is stored; the expected count is 0.',\n  );\n  if (backfilling.length === 0) {\n    unresolvable.add(0); // the common case: publish the series anyway\n    return;\n  }\n\n  const byLiteName = new Map<string, Identifier>(\n    db\n      .prepare(/*sql*/ `SELECT \"schema\", \"table\" FROM \"_zero.tableMetadata\"`)\n      .all<{schema: string; table: string}>()\n      .map(({schema, table}) => [\n        liteTableName({schema, name: table}),\n        {schema, name: table},\n      ]),\n  );\n\n  const insert = db.prepare(/*sql*/ `\n    INSERT INTO \"${BACKFILLING_TABLE}\"\n      (\"schema\", \"table\", \"column\", \"backfill\") VALUES (?, ?, ?, ?)\n  `);\n  let unresolved = 0;\n  for (const {table_name: liteName, column_name, backfill} of backfilling) {\n    const known = byLiteName.get(liteName);\n    const dot = known ? -1 : liteName.indexOf('.');\n    const table = known ?? {\n      schema: dot < 0 ? 'public' : liteName.slice(0, dot),\n      name: dot < 0 ? liteName : liteName.slice(dot + 1),\n    };\n    if (!known && dot >= 0) {\n      unresolved++;\n      lc.warn?.(\n        `no metadata for backfilling table \"${liteName}\"; ` +\n          `assuming {schema: \"${table.schema}\", name: \"${table.name}\"}`,\n      );\n    }\n    insert.run(table.schema, table.name, column_name, backfill);\n  }\n  unresolvable.add(unresolved);\n  lc.info?.(\n    `seeded \"${BACKFILLING_TABLE}\" with ${backfilling.length} in-flight ` +\n      `backfill(s), ${unresolved} of which could not be resolved`,\n  );\n}\n"],"mappings":";;;;;;AA0DA,IAAa,oBAAoB;AAYjC,IAAa,2BAAmC;kBAC9B,kBAAkB;;;;;;;;;;;;;;;;;;;;;;;;;;;;AA6BpC,IAAa,qBAAb,MAAgC;CAC9B;CAEA;CACA;CACA;CACA;CACA;CACA;CACA;CAEA,YAAY,IAAc;EACxB,KAAK,MAAM;CACb;;;;;;;CAQA,MAAM,QAAkC;EACtC,MAAM,MAAM,UAAU,MAAM;EAC5B,KAAK,MAAM,MAAM,KACf,KAAK,KAAK,EAAE;EAEd,OAAO;CACT;CAEA,KAAK,IAAoB;EACvB,QAAQ,GAAG,IAAX;GACE,KAAK,mBAEH;GAEF,KAAK;IACH,CAAC,KAAK,YAAY,KAAK,IAAI,QAAgB;yBAC1B,kBAAkB;;;;SAIlC,EAAA,CAAG,IACF,GAAG,MAAM,QACT,GAAG,MAAM,MACT,GAAG,QACH,WAAW,UAAU,GAAG,QAAQ,CAClC;IACA;GAEF,KAAK;IACH,CAAC,KAAK,iBAAiB,KAAK,IAAI,QAAgB;oBACpC,kBAAkB;;SAE7B,EAAA,CAAG,IAAI,GAAG,IAAI,QAAQ,GAAG,IAAI,MAAM,GAAG,IAAI,QAAQ,GAAG,IAAI,IAAI;IAC9D;GAEF,KAAK;IACH,CAAC,KAAK,eAAe,KAAK,IAAI,QAAgB;yBAC7B,kBAAkB;;SAElC,EAAA,CAAG,IAAI,GAAG,MAAM,QAAQ,GAAG,MAAM,IAAI;IACtC;GAEF,KAAK;IACH,CAAC,KAAK,kBAAkB,KAAK,IAAI,QAAgB;oBACrC,kBAAkB;;SAE7B,EAAA,CAAG,IAAI,GAAG,KAAK,GAAG,MAAM,QAAQ,GAAG,MAAM,MAAM,GAAG,GAAG;IACtD;GAEF,KAAK;IACH,KAAK,cAAc,GAAG,OAAO,GAAG,MAAM;IACtC;GAEF,KAAK;IAIH,KAAK,MAAM,UAAU,GAAG,SACtB,KAAK,cAAc,GAAG,OAAO,MAAM;IAErC;GAEF,SACE,YAAY,EAAE;EAClB;CACF;;;;;CAMA,YACE,OAC+C;EAC/C,MAAM,QAAQ,KAAK,iBAAiB,KAAK,IAAI,QAAgB;0CACvB,kBAAkB;;KAEvD,EAAA,CAAG,IACF,MAAM,QACN,MAAM,IACR;EACA,OAAO,IAAI,IACT,KAAK,KAAK,EAAC,QAAQ,eAAc,CAC/B,QACA,aAAa,OACT,KAAA,IACC,WAAW,MAAM,QAAQ,CAChC,CAAC,CACH;CACF;;CAGA,YACE,OACA,QACA,UACM;EACN,CAAC,KAAK,iBAAiB,KAAK,IAAI,QAAgB;gBACpC,kBAAkB;;KAE7B,EAAA,CAAG,IACF,aAAa,KAAA,IAAY,OAAO,WAAW,UAAU,QAAQ,GAC7D,MAAM,QACN,MAAM,MACN,MACF;CACF;;;;;;CAOA,SAAS,OAAmB,SAAkC;EAC5D,KAAK,KAAK;GAAC,IAAI;GAAqB;GAAO;EAAO,CAAC;CACrD;CAEA,cAAc,OAAmB,QAAsB;EACrD,CAAC,KAAK,gBAAgB,KAAK,IAAI,QAAgB;qBAC9B,kBAAkB;;KAElC,EAAA,CAAG,IAAI,MAAM,QAAQ,MAAM,MAAM,MAAM;CAC1C;AACF;;;;;;;;;AAUA,SAAgB,qBAAqB,IAAiC;CACpE,OAAO,qBAAqB,mBAAmB,EAAE,CAAC;AACpD;;;;;;;;AASA,SAAgB,qBAAqB,IAAiC;CACpE,MAAM,WAAW,qBAAqB,EAAE;CACxC,MAAM,WAAW,GACd,QAAgB;;gBAEL,kBAAkB;KAC7B,CAAC,CACD,IAAuE;CAC1E,KAAK,MAAM,EAAC,QAAQ,OAAO,QAAQ,UAAU,UAAS,UAAU;EAI9D,MAAM,MAHM,SAAS,MACnB,MAAK,EAAE,MAAM,WAAW,UAAU,EAAE,MAAM,SAAS,KAEzC,CAAA,EAAK,QAAQ;EACzB,IAAI,KACF,IAAI,WAAW,WAAW,MAAM,IAAI;CAExC;CACA,OAAO;AACT;;;;;;;;;;;;;;;;AAiBA,SAAgB,mBAAmB,IAAyB;CA6B1D,OAAO;EAAC,eA5Bc,GACnB,QAAgB;;;;;KAKhB,CAAC,CACD,IAAuD,CAAC,CACxD,KAAK,EAAC,QAAQ,OAAO,gBAAe;GACnC;GACA;GACA,UAAU,WAAW,MAAM,QAAQ;EACrC,EAgBM;EAAe,aAdH,GACjB,QAAgB;;gBAEL,kBAAkB;;KAE7B,CAAC,CACD,IAAuE,CAAC,CACxE,KAAK,EAAC,QAAQ,OAAO,QAAQ,gBAAe;GAC3C;GACA;GACA;GACA,UAAU,WAAW,MAAM,QAAQ;EACrC,EAEqB;CAAW;AACpC;;;;;;;;;;;;;;;;;;;;AAqBA,SAAgB,sCACd,IACA,IACM;CACN,GAAG,QAAgB,gBAAgB,kBAAkB,EAAE,CAAC,CAAC,IAAI;CAE7D,MAAM,cAAc,GACjB,QAAgB;;;;KAIhB,CAAC,CACD,IAAiE;CAEpE,MAAM,eAAe,mBACnB,WACA,iCACA,8LAIF;CACA,IAAI,YAAY,WAAW,GAAG;EAC5B,aAAa,IAAI,CAAC;EAClB;CACF;CAEA,MAAM,aAAa,IAAI,IACrB,GACG,QAAgB,qDAAqD,CAAC,CACtE,IAAqC,CAAC,CACtC,KAAK,EAAC,QAAQ,YAAW,CACxB,cAAc;EAAC;EAAQ,MAAM;CAAK,CAAC,GACnC;EAAC;EAAQ,MAAM;CAAK,CACtB,CAAC,CACL;CAEA,MAAM,SAAS,GAAG,QAAgB;mBACjB,kBAAkB;;GAElC;CACD,IAAI,aAAa;CACjB,KAAK,MAAM,EAAC,YAAY,UAAU,aAAa,cAAa,aAAa;EACvE,MAAM,QAAQ,WAAW,IAAI,QAAQ;EACrC,MAAM,MAAM,QAAQ,KAAK,SAAS,QAAQ,GAAG;EAC7C,MAAM,QAAQ,SAAS;GACrB,QAAQ,MAAM,IAAI,WAAW,SAAS,MAAM,GAAG,GAAG;GAClD,MAAM,MAAM,IAAI,WAAW,SAAS,MAAM,MAAM,CAAC;EACnD;EACA,IAAI,CAAC,SAAS,OAAO,GAAG;GACtB;GACA,GAAG,OACD,sCAAsC,SAAS,wBACvB,MAAM,OAAO,YAAY,MAAM,KAAK,GAC9D;EACF;EACA,OAAO,IAAI,MAAM,QAAQ,MAAM,MAAM,aAAa,QAAQ;CAC5D;CACA,aAAa,IAAI,UAAU;CAC3B,GAAG,OACD,WAAW,kBAAkB,SAAS,YAAY,OAAO,0BACvC,WAAW,gCAC/B;AACF"}