@rocicorp/zero 1.10.0-canary.0 → 1.10.0-canary.2

package/out/zero-cache/src/services/change-streamer/broadcast.js.map

@@ -1 +1 @@
-{"version":3,"file":"broadcast.js","names":["#pending","#completed","#done","#watermark","#majority","#start","#markCompleted","#setDone","#latestCompleted","#isDone","#releaseMode","#logWithState"],"sources":["../../../../../../zero-cache/src/services/change-streamer/broadcast.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {resolver} from '@rocicorp/resolver';\nimport type {WatermarkedChange} from './change-streamer-service.ts';\nimport type {Subscriber} from './subscriber.ts';\n\nexport type BroadcastReleaseMode = 'all-subscribers' | 'consensus-timeout';\n\n/**\n * Initiates and tracks the progress of a change broadcasted to\n * a set of subscribers.\n *\n * Creating a `Broadcast` automatically initiates the send.\n *\n * By default, {@link Broadcast.done} resolves when all subscribers\n * have acked the change. However, {@link Broadcast.checkProgress()}\n * can be called to resolve the broadcast earlier based on the flow\n * control policy.\n */\nexport class Broadcast {\n  /**\n   * Sends the change to the subscribers without the tracking machinery.\n   * This is suitable for fire-and-forget (i.e. pipelined) sends.\n   */\n  static withoutTracking(\n    subscribers: Iterable<Subscriber>,\n    change: WatermarkedChange,\n  ) {\n    for (const sub of subscribers) {\n      void sub.send(change);\n    }\n  }\n\n  readonly #pending: Set<Subscriber>;\n  readonly #completed: Completed[];\n  readonly #done = resolver();\n  #isDone = false;\n  #releaseMode: BroadcastReleaseMode | undefined;\n\n  readonly #watermark: string;\n  readonly #majority: number;\n\n  readonly #start = performance.now();\n  #latestCompleted = Number.MAX_VALUE;\n\n  /**\n   * Broadcasts the `change` to the `subscribers` and tracks their\n   * completion.\n   */\n  constructor(subscribers: Iterable<Subscriber>, change: WatermarkedChange) {\n    this.#pending = new Set(subscribers);\n    this.#completed = [];\n    this.#watermark = change[0];\n    this.#majority = Math.floor(this.#pending.size / 2) + 1;\n\n    for (const sub of this.#pending) {\n      const changes = sub.numPending + 1; // add one for this `change`\n      void sub\n        .send(change)\n        .catch(() => {})\n        .finally(() => this.#markCompleted(sub, changes));\n    }\n\n    // set done if there are no subscribers (mainly for tests)\n    if (this.#pending.size === 0) {\n      this.#setDone('all-subscribers');\n    }\n  }\n\n  #markCompleted(sub: Subscriber, changes: number) {\n    const elapsed = (this.#latestCompleted = performance.now()) - this.#start;\n    this.#completed.push({sub, changes, elapsed});\n    this.#pending.delete(sub);\n    if (this.#pending.size === 0) {\n      this.#setDone('all-subscribers');\n    }\n  }\n\n  #setDone(releaseMode: BroadcastReleaseMode) {\n    if (this.#isDone) {\n      return;\n    }\n    this.#isDone = true;\n    this.#releaseMode = releaseMode;\n    this.#done.resolve();\n  }\n\n  get isDone(): boolean {\n    return this.#isDone;\n  }\n\n  get done(): Promise<void> {\n    return this.#done.promise;\n  }\n\n  get releaseMode(): BroadcastReleaseMode {\n    return this.#releaseMode ?? 'all-subscribers';\n  }\n\n  /**\n   * Checks for pathological situations in which flow should be reenabled\n   * before all subscribers have acked.\n   *\n   * ### Background\n   *\n   * The purpose of flow control is to pull upstream replication changes\n   * no faster than the rate as they are processed by downstream subscribers\n   * in the steady state. In the change-streamer, this is done by occasionally\n   * waiting for ACKs from subscribers before continuing; without doing so,\n   * I/O buffers fill up and cause the system to spend most of its time in GC.\n   *\n   * However, the naive algorithm of always waiting for all subscribers (e.g.\n   * `Promise.all()`) can behave poorly in scenarios where subscribers\n   * are imbalanced:\n   * * New subscribers may have a backlog of changes to catch up with.\n   *   Having all subscribers wait for the new subscriber to catch up results\n   *   in delaying the entire application.\n   * * Broken TCP connections similarly require all subscribers to wait until\n   *   connection liveness checks kick in and disconnect the subscriber.\n   *\n   * A simplistic approach is to add a limit to the amount of time waiting for\n   * subscribers, i.e. an ack timeout. However, deciding what this timeout\n   * should be is non-trivial because of the heterogeneous nature of changes;\n   * while most changes operate on single rows and are relatively predictable\n   * in terms of running time, some changes are table-wide operations and can\n   * legitimately take an arbitrary amount of time. In such scenarios, a\n   * timeout that is too short can stop progress on replication altogether.\n   *\n   * ### Consensus-based Timeout Algorithm\n   *\n   * To address these shortcomings, a \"consensus-based timeout\" algorithm is\n   * used:\n   * * Wait for more than half of the subscribers to finish. (In\n   *   case of a single node, or the case of one replication-manager\n   *   and one view-syncer, this reduces to waiting for all subscribers.)\n   * * Once more than half of the subscribers have finished, proceed after\n   *   a fixed timeout elapses (e.g. 1 second), even if not all subscribers\n   *   have finished.\n   *\n   * In other words, the subscribers themselves are used to determine the\n   * timeout of each batch of changes; the majority determines this when\n   * they complete, upon which a timeout is logically started.\n   *\n   * In the common case, the remaining subscribers finish soon afterward and\n   * the timeout never elapses. However, in pathological cases where a minority\n   * of subscribers have a disproportionate amount of load, some will still\n   * be processing (or otherwise unresponsive). These subscribers are given\n   * a bounded amount of time to catch up at each flushed batch, up to the\n   * timeout interval. This guarantees eventual catchup because the\n   * subscribers with a backlog of changes necessarily have a higher\n   * processing rate than the subscribers that finished (and are made to wait).\n   *\n   * ### Not implemented: Broken connection detection\n   *\n   * If a subscriber has not made progress for a certain interval, the\n   * algorithm could theoretically drop it preemptively, supplementing the\n   * existing websocket-level liveness checks.\n   *\n   * However, a more reliable approach would be to change the replicator\n   * to use non-blocking writes, and subsequently increase the frequency of\n   * connection-level liveness checks. The current synchronous replica writes\n   * can delay both ping responsiveness and change progress arbitrarily (e.g.\n   * a large index creation); an independently liveness check that is not\n   * delayed by synchronous writes on the subscriber would be a more failsafe\n   * solution.\n   *\n   * @returns `true` if the broadcast was already done or was marked done.\n   */\n  checkProgress(\n    lc: LogContext,\n    flowControlConsensusPaddingMs: number,\n    now: number,\n  ) {\n    if (this.#isDone) {\n      return true;\n    }\n    if (this.#pending.size === 0) {\n      return true;\n    }\n    const elapsed = now - this.#start;\n    if (this.#completed.length < this.#majority) {\n      if (elapsed >= 1000) {\n        this.#logWithState(\n          lc,\n          `waiting for at least ${this.#majority} subscribers to finish`,\n          elapsed,\n        );\n      }\n      return false;\n    }\n    // Note: In the implementation, #latestCompleted is always updated,\n    // even after the majority is reached. This is fine and does not affect\n    // the important properties of the algorithm.\n    if (now - this.#latestCompleted >= flowControlConsensusPaddingMs) {\n      this.#logWithState(\n        lc,\n        `continuing with ${this.#pending.size} subscriber(s) still pending`,\n        elapsed,\n      );\n      this.#setDone('consensus-timeout');\n      return true;\n    }\n    return false;\n  }\n\n  #logWithState(lc: LogContext, msg: string, elapsed: number) {\n    lc.withContext('watermark', this.#watermark).info?.(\n      `${msg} (${elapsed.toFixed(3)} ms)`,\n      {\n        completed: this.#completed.map(d => ({\n          id: d.sub.id,\n          processed: d.changes,\n          elapsed: d.elapsed,\n        })),\n        pending: Array.from(this.#pending, sub => ({\n          id: sub.id,\n          ...sub.getStats(),\n        })),\n      },\n    );\n  }\n}\n\n/** Tracks the completed result of a single subscriber. */\ntype Completed = {\n  sub: Subscriber;\n  /** The number of changes processed. */\n  changes: number;\n  /** The elapsed milliseconds. */\n  elapsed: number;\n};\n"],"mappings":";;;;;;;;;;;;;AAkBA,IAAa,YAAb,MAAuB;;;;;CAKrB,OAAO,gBACL,aACA,QACA;EACA,KAAK,MAAM,OAAO,aAChB,IAAS,KAAK,MAAM;CAExB;CAEA;CACA;CACA,QAAiB,SAAS;CAC1B,UAAU;CACV;CAEA;CACA;CAEA,SAAkB,YAAY,IAAI;CAClC,mBAAmB,OAAO;;;;;CAM1B,YAAY,aAAmC,QAA2B;EACxE,KAAKA,WAAW,IAAI,IAAI,WAAW;EACnC,KAAKC,aAAa,CAAC;EACnB,KAAKE,aAAa,OAAO;EACzB,KAAKC,YAAY,KAAK,MAAM,KAAKJ,SAAS,OAAO,CAAC,IAAI;EAEtD,KAAK,MAAM,OAAO,KAAKA,UAAU;GAC/B,MAAM,UAAU,IAAI,aAAa;GACjC,IACG,KAAK,MAAM,EACX,YAAY,CAAC,CAAC,EACd,cAAc,KAAKM,eAAe,KAAK,OAAO,CAAC;EACpD;EAGA,IAAI,KAAKN,SAAS,SAAS,GACzB,KAAKO,SAAS,iBAAiB;CAEnC;CAEA,eAAe,KAAiB,SAAiB;EAC/C,MAAM,WAAW,KAAKC,mBAAmB,YAAY,IAAI,KAAK,KAAKH;EACnE,KAAKJ,WAAW,KAAK;GAAC;GAAK;GAAS;EAAO,CAAC;EAC5C,KAAKD,SAAS,OAAO,GAAG;EACxB,IAAI,KAAKA,SAAS,SAAS,GACzB,KAAKO,SAAS,iBAAiB;CAEnC;CAEA,SAAS,aAAmC;EAC1C,IAAI,KAAKE,SACP;EAEF,KAAKA,UAAU;EACf,KAAKC,eAAe;EACpB,KAAKR,MAAM,QAAQ;CACrB;CAEA,IAAI,SAAkB;EACpB,OAAO,KAAKO;CACd;CAEA,IAAI,OAAsB;EACxB,OAAO,KAAKP,MAAM;CACpB;CAEA,IAAI,cAAoC;EACtC,OAAO,KAAKQ,gBAAgB;CAC9B;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAuEA,cACE,IACA,+BACA,KACA;EACA,IAAI,KAAKD,SACP,OAAO;EAET,IAAI,KAAKT,SAAS,SAAS,GACzB,OAAO;EAET,MAAM,UAAU,MAAM,KAAKK;EAC3B,IAAI,KAAKJ,WAAW,SAAS,KAAKG,WAAW;GAC3C,IAAI,WAAW,KACb,KAAKO,cACH,IACA,wBAAwB,KAAKP,UAAU,yBACvC,OACF;GAEF,OAAO;EACT;EAIA,IAAI,MAAM,KAAKI,oBAAoB,+BAA+B;GAChE,KAAKG,cACH,IACA,mBAAmB,KAAKX,SAAS,KAAK,+BACtC,OACF;GACA,KAAKO,SAAS,mBAAmB;GACjC,OAAO;EACT;EACA,OAAO;CACT;CAEA,cAAc,IAAgB,KAAa,SAAiB;EAC1D,GAAG,YAAY,aAAa,KAAKJ,UAAU,EAAE,OAC3C,GAAG,IAAI,IAAI,QAAQ,QAAQ,CAAC,EAAE,OAC9B;GACE,WAAW,KAAKF,WAAW,KAAI,OAAM;IACnC,IAAI,EAAE,IAAI;IACV,WAAW,EAAE;IACb,SAAS,EAAE;GACb,EAAE;GACF,SAAS,MAAM,KAAK,KAAKD,WAAU,SAAQ;IACzC,IAAI,IAAI;IACR,GAAG,IAAI,SAAS;GAClB,EAAE;EACJ,CACF;CACF;AACF"}
+{"version":3,"file":"broadcast.js","names":["#lc","#pending","#completed","#done","#watermark","#majority","#start","#earlyReleaseTimeoutProportion","#setTimeout","#clearTimeout","#markCompleted","#setDone","#isDone","#earlyReleaseTimer","#logWithState","#releaseMode"],"sources":["../../../../../../zero-cache/src/services/change-streamer/broadcast.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {resolver} from '@rocicorp/resolver';\nimport type {WatermarkedChange} from './change-streamer.ts';\nimport type {Subscriber} from './subscriber.ts';\n\nexport type BroadcastReleaseMode = 'all-subscribers' | 'consensus-timeout';\n\n/**\n * Enables event-driven early release of a {@link Broadcast}. When provided and a\n * majority has acked, the broadcast schedules its own consensus-timeout release\n * proportional to how long the majority took to ack, instead of relying on the\n * Forwarder's periodic progress tick.\n */\nexport type EarlyReleaseOptions = {\n  /** Time to wait, proportional to majority completion, before releasing. */\n  consensusTimeoutProportion: number;\n  /** Injectable timer, for deterministic testing. */\n  setTimeoutFn?: typeof setTimeout;\n  /** Injectable timer cancellation, for deterministic testing. */\n  clearTimeoutFn?: typeof clearTimeout;\n};\n\n/**\n * Initiates and tracks the progress of a change broadcasted to\n * a set of subscribers, implementing **consensus-based timeouts**\n * to effect short-term graceful degradation when a minority of\n * subscribers are behind.\n *\n * ### Background\n *\n * The purpose of flow control is to pull upstream replication changes\n * no faster than the rate they are processed by downstream subscribers\n * in the steady state. In the change-streamer, this is done by occasionally\n * waiting for ACKs from subscribers before continuing; without doing so,\n * I/O buffers fill up and cause the system to spend most of its time in GC.\n *\n * However, the naive algorithm of always waiting for all subscribers (e.g.\n * `Promise.all()`) can behave poorly in scenarios where subscribers\n * are imbalanced:\n * * New subscribers may have a backlog of changes to catch up with.\n *   Having all subscribers wait for the new subscriber to catch up results\n *   in delaying the entire application.\n * * Broken TCP connections similarly require all subscribers to wait until\n *   connection liveness checks kick in and disconnect the subscriber.\n *\n * A simplistic approach is to add a limit to the amount of time waiting for\n * subscribers, i.e. an ack timeout. However, deciding what this timeout\n * should be is non-trivial because of the heterogeneous nature of changes;\n * while most changes operate on single rows and are relatively predictable\n * in terms of running time, some changes are table-wide operations and can\n * legitimately take an arbitrary amount of time. In such scenarios, a\n * timeout that is too short can stop progress on replication altogether.\n *\n * ### Consensus-based Timeout Algorithm\n *\n * To address these shortcomings, a \"consensus-based timeout\" algorithm is\n * used:\n * * Wait for more than half of the subscribers to finish. (In\n *   case of a single node, or the case of one replication-manager\n *   and one view-syncer, this reduces to waiting for all subscribers.)\n * * Once more than half of the subscribers have finished, proceed after\n *   the configured timeout, proportional to the time it took for the\n *   majority of subscribers to ack, even if not all subscribers have\n *   finished.\n *\n * In other words, the subscribers themselves are used to determine the\n * timeout of each batch of changes; the majority determines this when\n * they complete, upon which a timeout is logically started.\n *\n * In the common case, the remaining subscribers finish soon afterward and\n * the timeout never elapses. However, in pathological cases where a minority\n * of subscribers have a disproportionate amount of load, some will still\n * be processing (or otherwise unresponsive). These subscribers are given\n * a bounded amount of time to catch up at each flushed batch, up to the\n * timeout interval. This guarantees eventual catchup because the\n * subscribers with a backlog of changes necessarily have a higher\n * processing rate than the subscribers that finished (and are made to wait).\n */\nexport class Broadcast {\n  /**\n   * Sends the change to the subscribers without the tracking machinery.\n   * This is suitable for fire-and-forget (i.e. pipelined) sends.\n   */\n  static withoutTracking(\n    subscribers: Iterable<Subscriber>,\n    change: WatermarkedChange,\n  ) {\n    for (const sub of subscribers) {\n      void sub.send(change);\n    }\n  }\n\n  readonly #lc: LogContext;\n  readonly #pending: Set<Subscriber>;\n  readonly #completed: Completed[];\n  readonly #done = resolver();\n  #isDone = false;\n  #releaseMode: BroadcastReleaseMode | undefined;\n\n  readonly #watermark: string;\n  readonly #majority: number;\n\n  readonly #start = performance.now();\n\n  // When set, the broadcast schedules its own consensus-timeout release instead\n  // of waiting for the Forwarder's periodic progress tick.\n  readonly #earlyReleaseTimeoutProportion: number | undefined;\n  readonly #setTimeout: typeof setTimeout;\n  readonly #clearTimeout: typeof clearTimeout;\n  #earlyReleaseTimer: ReturnType<typeof setTimeout> | undefined;\n\n  /**\n   * Broadcasts the `change` to the `subscribers` and tracks their\n   * completion.\n   */\n  constructor(\n    lc: LogContext,\n    subscribers: Iterable<Subscriber>,\n    change: WatermarkedChange,\n    earlyRelease?: EarlyReleaseOptions,\n  ) {\n    this.#lc = lc;\n    this.#pending = new Set(subscribers);\n    this.#completed = [];\n    this.#watermark = change[0];\n    this.#majority = Math.floor(this.#pending.size / 2) + 1;\n    this.#earlyReleaseTimeoutProportion =\n      earlyRelease?.consensusTimeoutProportion;\n    this.#setTimeout = earlyRelease?.setTimeoutFn ?? setTimeout;\n    this.#clearTimeout = earlyRelease?.clearTimeoutFn ?? clearTimeout;\n\n    for (const sub of this.#pending) {\n      const changes = sub.numPending + 1; // add one for this `change`\n      void sub\n        .send(change)\n        .catch(() => {})\n        .finally(() => this.#markCompleted(sub, changes));\n    }\n\n    // set done if there are no subscribers (mainly for tests)\n    if (this.#pending.size === 0) {\n      this.#setDone('all-subscribers');\n    }\n  }\n\n  #markCompleted(sub: Subscriber, changes: number) {\n    if (this.#isDone) {\n      return;\n    }\n    const elapsed = performance.now() - this.#start;\n    sub.trackResponseResult('on-time');\n\n    this.#completed.push({sub, changes, elapsed});\n    this.#pending.delete(sub);\n    if (this.#pending.size === 0) {\n      this.#setDone('all-subscribers');\n      return;\n    }\n    // Event-driven consensus-timeout: the moment a majority acks, arm a single\n    // release timer proportional to how long the majority took to ack. The\n    // broadcast resolves when it fires, unless all subscribers ack first.\n    if (\n      this.#earlyReleaseTimeoutProportion !== undefined &&\n      this.#earlyReleaseTimer === undefined &&\n      this.#completed.length === this.#majority\n    ) {\n      const timeoutMs = Math.ceil(\n        elapsed * this.#earlyReleaseTimeoutProportion,\n      );\n      this.#earlyReleaseTimer = this.#setTimeout(\n        () => this.#setDone('consensus-timeout'),\n        timeoutMs,\n      );\n    }\n  }\n\n  #setDone(releaseMode: BroadcastReleaseMode) {\n    if (this.#isDone) {\n      return;\n    }\n    if (releaseMode === 'consensus-timeout') {\n      const elapsed = performance.now() - this.#start;\n      this.#logWithState(\n        `continuing with ${this.#pending.size} subscriber(s) still pending`,\n        elapsed,\n      );\n    }\n    this.#pending.forEach(sub => sub.trackResponseResult('timed-out'));\n    this.#isDone = true;\n    this.#releaseMode = releaseMode;\n    this.#clearTimeout(this.#earlyReleaseTimer);\n    this.#earlyReleaseTimer = undefined;\n    this.#done.resolve();\n  }\n\n  get isDone(): boolean {\n    return this.#isDone;\n  }\n\n  get done(): Promise<void> {\n    return this.#done.promise;\n  }\n\n  get releaseMode(): BroadcastReleaseMode {\n    return this.#releaseMode ?? 'all-subscribers';\n  }\n\n  logSlowRequests(now: number) {\n    if (this.#isDone || this.#pending.size === 0) {\n      return;\n    }\n    const elapsed = now - this.#start;\n    if (this.#completed.length < this.#majority) {\n      if (elapsed >= 1000) {\n        this.#logWithState(\n          `waiting for at least ${this.#majority} subscribers to finish`,\n          elapsed,\n        );\n      }\n    }\n  }\n\n  #logWithState(msg: string, elapsed: number) {\n    this.#lc\n      .withContext('watermark', this.#watermark)\n      .info?.(`${msg} (${elapsed.toFixed(3)} ms)`, {\n        completed: this.#completed.map(d => ({\n          id: d.sub.id,\n          processed: d.changes,\n          elapsed: d.elapsed,\n        })),\n        pending: Array.from(this.#pending, sub => ({\n          id: sub.id,\n          ...sub.getStats(),\n        })),\n      });\n  }\n}\n\n/** Tracks the completed result of a single subscriber. */\ntype Completed = {\n  sub: Subscriber;\n  /** The number of changes processed. */\n  changes: number;\n  /** The elapsed milliseconds. */\n  elapsed: number;\n};\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AA8EA,IAAa,YAAb,MAAuB;;;;;CAKrB,OAAO,gBACL,aACA,QACA;EACA,KAAK,MAAM,OAAO,aAChB,IAAS,KAAK,MAAM;CAExB;CAEA;CACA;CACA;CACA,QAAiB,SAAS;CAC1B,UAAU;CACV;CAEA;CACA;CAEA,SAAkB,YAAY,IAAI;CAIlC;CACA;CACA;CACA;;;;;CAMA,YACE,IACA,aACA,QACA,cACA;EACA,KAAKA,MAAM;EACX,KAAKC,WAAW,IAAI,IAAI,WAAW;EACnC,KAAKC,aAAa,CAAC;EACnB,KAAKE,aAAa,OAAO;EACzB,KAAKC,YAAY,KAAK,MAAM,KAAKJ,SAAS,OAAO,CAAC,IAAI;EACtD,KAAKM,iCACH,cAAc;EAChB,KAAKC,cAAc,cAAc,gBAAgB;EACjD,KAAKC,gBAAgB,cAAc,kBAAkB;EAErD,KAAK,MAAM,OAAO,KAAKR,UAAU;GAC/B,MAAM,UAAU,IAAI,aAAa;GACjC,IACG,KAAK,MAAM,EACX,YAAY,CAAC,CAAC,EACd,cAAc,KAAKS,eAAe,KAAK,OAAO,CAAC;EACpD;EAGA,IAAI,KAAKT,SAAS,SAAS,GACzB,KAAKU,SAAS,iBAAiB;CAEnC;CAEA,eAAe,KAAiB,SAAiB;EAC/C,IAAI,KAAKC,SACP;EAEF,MAAM,UAAU,YAAY,IAAI,IAAI,KAAKN;EACzC,IAAI,oBAAoB,SAAS;EAEjC,KAAKJ,WAAW,KAAK;GAAC;GAAK;GAAS;EAAO,CAAC;EAC5C,KAAKD,SAAS,OAAO,GAAG;EACxB,IAAI,KAAKA,SAAS,SAAS,GAAG;GAC5B,KAAKU,SAAS,iBAAiB;GAC/B;EACF;EAIA,IACE,KAAKJ,mCAAmC,KAAA,KACxC,KAAKM,uBAAuB,KAAA,KAC5B,KAAKX,WAAW,WAAW,KAAKG,WAChC;GACA,MAAM,YAAY,KAAK,KACrB,UAAU,KAAKE,8BACjB;GACA,KAAKM,qBAAqB,KAAKL,kBACvB,KAAKG,SAAS,mBAAmB,GACvC,SACF;EACF;CACF;CAEA,SAAS,aAAmC;EAC1C,IAAI,KAAKC,SACP;EAEF,IAAI,gBAAgB,qBAAqB;GACvC,MAAM,UAAU,YAAY,IAAI,IAAI,KAAKN;GACzC,KAAKQ,cACH,mBAAmB,KAAKb,SAAS,KAAK,+BACtC,OACF;EACF;EACA,KAAKA,SAAS,SAAQ,QAAO,IAAI,oBAAoB,WAAW,CAAC;EACjE,KAAKW,UAAU;EACf,KAAKG,eAAe;EACpB,KAAKN,cAAc,KAAKI,kBAAkB;EAC1C,KAAKA,qBAAqB,KAAA;EAC1B,KAAKV,MAAM,QAAQ;CACrB;CAEA,IAAI,SAAkB;EACpB,OAAO,KAAKS;CACd;CAEA,IAAI,OAAsB;EACxB,OAAO,KAAKT,MAAM;CACpB;CAEA,IAAI,cAAoC;EACtC,OAAO,KAAKY,gBAAgB;CAC9B;CAEA,gBAAgB,KAAa;EAC3B,IAAI,KAAKH,WAAW,KAAKX,SAAS,SAAS,GACzC;EAEF,MAAM,UAAU,MAAM,KAAKK;EAC3B,IAAI,KAAKJ,WAAW,SAAS,KAAKG;OAC5B,WAAW,KACb,KAAKS,cACH,wBAAwB,KAAKT,UAAU,yBACvC,OACF;EAAA;CAGN;CAEA,cAAc,KAAa,SAAiB;EAC1C,KAAKL,IACF,YAAY,aAAa,KAAKI,UAAU,EACxC,OAAO,GAAG,IAAI,IAAI,QAAQ,QAAQ,CAAC,EAAE,OAAO;GAC3C,WAAW,KAAKF,WAAW,KAAI,OAAM;IACnC,IAAI,EAAE,IAAI;IACV,WAAW,EAAE;IACb,SAAS,EAAE;GACb,EAAE;GACF,SAAS,MAAM,KAAK,KAAKD,WAAU,SAAQ;IACzC,IAAI,IAAI;IACR,GAAG,IAAI,SAAS;GAClB,EAAE;EACJ,CAAC;CACL;AACF"}

package/out/zero-cache/src/services/change-streamer/change-log-codec.d.ts

@@ -0,0 +1,19 @@
+import type { ChangeStreamData } from '../change-source/protocol/current/downstream.ts';
+import type { ChangeTag, WatermarkedChange } from './change-streamer.ts';
+export type ChangeLogEntry = {
+    watermark: string;
+    tag: string;
+    change: string;
+};
+/** Serializes a data-plane message to the canonical downstream JSON form. */
+export declare function serializeChangeStreamData(data: ChangeStreamData): string;
+/**
+ * Extracts the stringified change message from the stringified stream message
+ * (the second tuple element). This allows the stream message to be stringified
+ * once while storing only the change message in the change log for backwards
+ * compatibility.
+ */
+export declare function extractChangeSubstring(streamMessageJSON: string, tag: ChangeTag | undefined): string;
+/** Reconstructs the canonical downstream JSON around a stored change. */
+export declare function reconstructWatermarkedChange(entry: ChangeLogEntry): WatermarkedChange;
+//# sourceMappingURL=change-log-codec.d.ts.map

package/out/zero-cache/src/services/change-streamer/change-log-codec.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"change-log-codec.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/change-streamer/change-log-codec.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAC,gBAAgB,EAAC,MAAM,iDAAiD,CAAC;AACtF,OAAO,KAAK,EAAC,SAAS,EAAE,iBAAiB,EAAC,MAAM,sBAAsB,CAAC;AAEvE,MAAM,MAAM,cAAc,GAAG;IAC3B,SAAS,EAAE,MAAM,CAAC;IAClB,GAAG,EAAE,MAAM,CAAC;IACZ,MAAM,EAAE,MAAM,CAAC;CAChB,CAAC;AAEF,6EAA6E;AAC7E,wBAAgB,yBAAyB,CAAC,IAAI,EAAE,gBAAgB,GAAG,MAAM,CAExE;AAED;;;;;GAKG;AACH,wBAAgB,sBAAsB,CACpC,iBAAiB,EAAE,MAAM,EACzB,GAAG,EAAE,SAAS,GAAG,SAAS,GACzB,MAAM,CAkBR;AAED,yEAAyE;AACzE,wBAAgB,4BAA4B,CAC1C,KAAK,EAAE,cAAc,GACpB,iBAAiB,CAqBnB"}

package/out/zero-cache/src/services/change-streamer/change-log-codec.js

@@ -0,0 +1,50 @@
+import { BigIntJSON } from "../../../../shared/src/bigint-json.js";
+//#region ../zero-cache/src/services/change-streamer/change-log-codec.ts
+/** Serializes a data-plane message to the canonical downstream JSON form. */
+function serializeChangeStreamData(data) {
+	return BigIntJSON.stringify(data);
+}
+/**
+* Extracts the stringified change message from the stringified stream message
+* (the second tuple element). This allows the stream message to be stringified
+* once while storing only the change message in the change log for backwards
+* compatibility.
+*/
+function extractChangeSubstring(streamMessageJSON, tag) {
+	switch (tag) {
+		case "begin":
+		case "commit": return streamMessageJSON.substring(streamMessageJSON.indexOf(",") + 1, streamMessageJSON.lastIndexOf(","));
+		default: return streamMessageJSON.substring(streamMessageJSON.indexOf(",") + 1, streamMessageJSON.lastIndexOf("]"));
+	}
+}
+/** Reconstructs the canonical downstream JSON around a stored change. */
+function reconstructWatermarkedChange(entry) {
+	const { watermark, change } = entry;
+	const tag = entry.tag;
+	switch (tag) {
+		case "begin": return [
+			watermark,
+			tag,
+			`["begin",${change},{"commitWatermark":"${watermark}"}]`
+		];
+		case "commit": return [
+			watermark,
+			tag,
+			`["commit",${change},{"watermark":"${watermark}"}]`
+		];
+		case "rollback": return [
+			watermark,
+			tag,
+			`["rollback",${change}]`
+		];
+		default: return [
+			watermark,
+			tag,
+			`["data",${change}]`
+		];
+	}
+}
+//#endregion
+export { extractChangeSubstring, reconstructWatermarkedChange, serializeChangeStreamData };
+
+//# sourceMappingURL=change-log-codec.js.map

package/out/zero-cache/src/services/change-streamer/change-log-codec.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"change-log-codec.js","names":[],"sources":["../../../../../../zero-cache/src/services/change-streamer/change-log-codec.ts"],"sourcesContent":["import {BigIntJSON} from '../../../../shared/src/bigint-json.ts';\nimport type {ChangeStreamData} from '../change-source/protocol/current/downstream.ts';\nimport type {ChangeTag, WatermarkedChange} from './change-streamer.ts';\n\nexport type ChangeLogEntry = {\n  watermark: string;\n  tag: string;\n  change: string;\n};\n\n/** Serializes a data-plane message to the canonical downstream JSON form. */\nexport function serializeChangeStreamData(data: ChangeStreamData): string {\n  return BigIntJSON.stringify(data);\n}\n\n/**\n * Extracts the stringified change message from the stringified stream message\n * (the second tuple element). This allows the stream message to be stringified\n * once while storing only the change message in the change log for backwards\n * compatibility.\n */\nexport function extractChangeSubstring(\n  streamMessageJSON: string,\n  tag: ChangeTag | undefined,\n): string {\n  switch (tag) {\n    case 'begin':\n    case 'commit':\n      // e.g.\n      // [\"begin\",<message-json>,{\"commitWatermark\":\"92fj2d0s\"}]\n      // [\"commit\",<message-json>,{\"watermark\":\"92fj2d0s\"}]\n      return streamMessageJSON.substring(\n        streamMessageJSON.indexOf(',') + 1,\n        streamMessageJSON.lastIndexOf(','),\n      );\n    default:\n      // [\"data\",<message-json>]\n      return streamMessageJSON.substring(\n        streamMessageJSON.indexOf(',') + 1,\n        streamMessageJSON.lastIndexOf(']'),\n      );\n  }\n}\n\n/** Reconstructs the canonical downstream JSON around a stored change. */\nexport function reconstructWatermarkedChange(\n  entry: ChangeLogEntry,\n): WatermarkedChange {\n  const {watermark, change} = entry;\n  const tag = entry.tag as ChangeTag;\n  switch (tag) {\n    case 'begin':\n      return [\n        watermark,\n        tag,\n        `[\"begin\",${change},{\"commitWatermark\":\"${watermark}\"}]`,\n      ];\n    case 'commit':\n      return [\n        watermark,\n        tag,\n        `[\"commit\",${change},{\"watermark\":\"${watermark}\"}]`,\n      ];\n    case 'rollback':\n      return [watermark, tag, `[\"rollback\",${change}]`];\n    default:\n      return [watermark, tag, `[\"data\",${change}]`];\n  }\n}\n"],"mappings":";;;AAWA,SAAgB,0BAA0B,MAAgC;CACxE,OAAO,WAAW,UAAU,IAAI;AAClC;;;;;;;AAQA,SAAgB,uBACd,mBACA,KACQ;CACR,QAAQ,KAAR;EACE,KAAK;EACL,KAAK,UAIH,OAAO,kBAAkB,UACvB,kBAAkB,QAAQ,GAAG,IAAI,GACjC,kBAAkB,YAAY,GAAG,CACnC;EACF,SAEE,OAAO,kBAAkB,UACvB,kBAAkB,QAAQ,GAAG,IAAI,GACjC,kBAAkB,YAAY,GAAG,CACnC;CACJ;AACF;;AAGA,SAAgB,6BACd,OACmB;CACnB,MAAM,EAAC,WAAW,WAAU;CAC5B,MAAM,MAAM,MAAM;CAClB,QAAQ,KAAR;EACE,KAAK,SACH,OAAO;GACL;GACA;GACA,YAAY,OAAO,uBAAuB,UAAU;EACtD;EACF,KAAK,UACH,OAAO;GACL;GACA;GACA,aAAa,OAAO,iBAAiB,UAAU;EACjD;EACF,KAAK,YACH,OAAO;GAAC;GAAW;GAAK,eAAe,OAAO;EAAE;EAClD,SACE,OAAO;GAAC;GAAW;GAAK,WAAW,OAAO;EAAE;CAChD;AACF"}

package/out/zero-cache/src/services/change-streamer/change-log-initializer.d.ts

@@ -0,0 +1,141 @@
+/**
+ * Where a stream connection's initialization parameters come from.
+ *
+ * A change-streamer starts a stream from three things read at one position: the
+ * watermark to resume at, the {@link BackfillRequest}s to hand the change
+ * source, and the cookie set those requests were derived from. Postgres has
+ * always supplied all three (`Storer.getStartStreamInitializationParameters()`).
+ * The point of this workstream is that a restored replica can supply them
+ * instead, which is what lets the `cdc` schema be decommissioned.
+ *
+ * This is the two-source stage. Both are computed on every stream connection,
+ * compared, and Postgres's is used — so the replica-derived path runs in
+ * production, at full rate, before anything depends on it. The flip is
+ * `initFromPgChangeLog: false`, not a rewrite.
+ *
+ * The shape is {@link UpstreamAcker}'s, deliberately: two stores, one output, a
+ * flag per store, and the Postgres flag carrying the marker that retires it.
+ * That class solved the identical problem for upstream ACKs earlier in this
+ * workstream.
+ *
+ * ### Why the comparison is a fold and not an equality
+ *
+ * The replica is a subscriber downstream of the change-streamer's write loop,
+ * so its state version trails the log's head by however long the replicator is
+ * behind. A raw difference between the two cookie sets is therefore the normal
+ * case and says nothing. What is checked instead is that the replica's set,
+ * carried forward over exactly the schema changes the log holds in the interval,
+ * equals Postgres's set at Postgres's watermark. The log holds precisely those
+ * transactions, schema changes are rare, and the check is exact rather than
+ * approximate — so it also happens to be the strongest available test of the
+ * premise that three interpreters of one fold agree.
+ */
+import type { LogContext } from '@rocicorp/logger';
+import { Database } from '../../../../zqlite/src/db.ts';
+import type { BackfillRequest } from '../change-source/protocol/current/upstream.ts';
+import { type CookieSet } from '../replicator/change-log-cookies.ts';
+import { type ChangeLogResumePoint } from '../replicator/change-log-db.ts';
+/** Everything one stream connection starts from, read at one position. */
+export type InitializationParameters = {
+    readonly lastWatermark: string;
+    readonly backfillRequests: BackfillRequest[];
+    readonly cookies: CookieSet;
+};
+/**
+ * - `equal` — the two stores were at the same watermark and held the same
+ *   cookies. Nothing had to be folded, which is the strongest of the four.
+ * - `equal-after-fold` — the replica trailed, and carrying its set forward over
+ *   the log's schema changes in the interval reproduced Postgres's exactly.
+ *   The expected steady-state result.
+ * - `divergent` — the two disagree at the same position. Alert; investigate
+ *   individually.
+ * - `inconclusive` — the log does not span the interval, so the fold cannot be
+ *   performed. Routine on a cold log, and not a finding.
+ * - `error` — the comparison itself failed: the replica could not be read, or
+ *   the fold threw. Not one of the four classifications, because it is a
+ *   statement about this process rather than about the two stores. Kept
+ *   separate rather than folded into `inconclusive` so that a change-streamer
+ *   that has silently stopped being able to read its replica is visible before
+ *   that read becomes the only source.
+ */
+export type InitComparisonResult = 'equal' | 'equal-after-fold' | 'divergent' | 'inconclusive' | 'error';
+export type ChangeLogInitializerSources = {
+    /** `Storer.getStartStreamInitializationParameters()`. */
+    readonly pgChangeLog: () => Promise<InitializationParameters>;
+    /** Typically {@link replicaInitializationSource}. */
+    readonly replica: () => InitializationParameters;
+    /**
+     * Reconciles the SQLite change log and returns its current resume point.
+     *
+     * When `resumeFrom` is present, it supplies the resume point. Otherwise, a
+     * valid log uses its own head. A new or invalid log uses `seed`.
+     *
+     * Returns `undefined` when the writer is disabled or unavailable.
+     */
+    readonly reconcileChangeLog: (resumeFrom: ChangeLogResumePoint | undefined, seed: () => ChangeLogResumePoint) => ChangeLogResumePoint | undefined;
+    /**
+     * The change log, for the fold. `undefined` before the stream loop's first
+     * reconcile has created it, and again once the writer has failed soft and
+     * deleted it — both of which make the comparison `inconclusive` rather than
+     * an error, since neither says anything about the two stores.
+     */
+    readonly changeLog: () => Database | undefined;
+};
+type Opts = {
+    /**
+     * When true, Postgres supplies the resume point and the cookies for a new
+     * change log.
+     *
+     * When false, a valid log resumes from its own head. The replica supplies the
+     * resume point for a new or invalid log.
+     */
+    readonly initFromPgChangeLog: boolean;
+    /**
+     * Derive the parameters from the replica. When Postgres is enabled, these
+     * parameters are used only for comparison.
+     *
+     * When Postgres is disabled, the replica supplies the seed and the fallback
+     * resume point.
+     */
+    readonly initFromReplica: boolean;
+};
+export declare class ChangeLogInitializer {
+    #private;
+    constructor(lc: LogContext, { initFromPgChangeLog, initFromReplica }: Opts, sources: ChangeLogInitializerSources);
+    /**
+     * Returns the comparison result from the most recent {@link initialize}
+     * call. Returns `undefined` when that call used only one source.
+     *
+     * Tests and metrics use this result. It does not affect initialization.
+     */
+    lastComparison(): InitComparisonResult | undefined;
+    /**
+     * Returns the initialization parameters for one stream connection after it
+     * reconciles the change log.
+     *
+     * The selected source determines how reconciliation works. Keeping both
+     * operations here hides that choice from the caller.
+     */
+    initialize(): Promise<InitializationParameters>;
+}
+/**
+ * Reads the replica's initialization parameters in one snapshot, which is
+ * invariant 15 on the replica side: the state version and the cookies are only
+ * meaningful together.
+ */
+export declare function readReplicaInitializationParameters(db: Database): InitializationParameters;
+/**
+ * The replica source for {@link ChangeLogInitializer}, opening and closing the
+ * file per stream connection rather than holding a handle on a database another
+ * process owns.
+ *
+ * The handle is read-only, which is invariant 19 stated in the API: the
+ * replica's cookie tables are written by the change-processor and by nothing
+ * else. Note that a read-only SQLite connection still needs to attach the
+ * WAL's `-shm`, so this can fail if it opens at a moment when no writer holds
+ * the replica; that surfaces as the `error` classification, which is the point
+ * of running the path under observation before anything depends on it.
+ */
+export declare function replicaInitializationSource(lc: LogContext, replicaFile: string): () => InitializationParameters;
+export {};
+//# sourceMappingURL=change-log-initializer.d.ts.map

package/out/zero-cache/src/services/change-streamer/change-log-initializer.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"change-log-initializer.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/change-streamer/change-log-initializer.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA+BG;AAEH,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAIjD,OAAO,EAAC,QAAQ,EAAC,MAAM,8BAA8B,CAAC;AAKtD,OAAO,KAAK,EAAC,eAAe,EAAC,MAAM,+CAA+C,CAAC;AACnF,OAAO,EAGL,KAAK,SAAS,EACf,MAAM,qCAAqC,CAAC;AAC7C,OAAO,EAEL,KAAK,oBAAoB,EAC1B,MAAM,gCAAgC,CAAC;AAQxC,0EAA0E;AAC1E,MAAM,MAAM,wBAAwB,GAAG;IACrC,QAAQ,CAAC,aAAa,EAAE,MAAM,CAAC;IAC/B,QAAQ,CAAC,gBAAgB,EAAE,eAAe,EAAE,CAAC;IAC7C,QAAQ,CAAC,OAAO,EAAE,SAAS,CAAC;CAC7B,CAAC;AAEF;;;;;;;;;;;;;;;;GAgBG;AACH,MAAM,MAAM,oBAAoB,GAC5B,OAAO,GACP,kBAAkB,GAClB,WAAW,GACX,cAAc,GACd,OAAO,CAAC;AAEZ,MAAM,MAAM,2BAA2B,GAAG;IACxC,yDAAyD;IACzD,QAAQ,CAAC,WAAW,EAAE,MAAM,OAAO,CAAC,wBAAwB,CAAC,CAAC;IAC9D,qDAAqD;IACrD,QAAQ,CAAC,OAAO,EAAE,MAAM,wBAAwB,CAAC;IACjD;;;;;;;OAOG;IACH,QAAQ,CAAC,kBAAkB,EAAE,CAC3B,UAAU,EAAE,oBAAoB,GAAG,SAAS,EAC5C,IAAI,EAAE,MAAM,oBAAoB,KAC7B,oBAAoB,GAAG,SAAS,CAAC;IACtC;;;;;OAKG;IACH,QAAQ,CAAC,SAAS,EAAE,MAAM,QAAQ,GAAG,SAAS,CAAC;CAChD,CAAC;AAEF,KAAK,IAAI,GAAG;IACV;;;;;;OAMG;IAEH,QAAQ,CAAC,mBAAmB,EAAE,OAAO,CAAC;IACtC;;;;;;OAMG;IACH,QAAQ,CAAC,eAAe,EAAE,OAAO,CAAC;CACnC,CAAC;AAcF,qBAAa,oBAAoB;;gBAgB7B,EAAE,EAAE,UAAU,EACd,EAAC,mBAAmB,EAAE,eAAe,EAAC,EAAE,IAAI,EAC5C,OAAO,EAAE,2BAA2B;IAYtC;;;;;OAKG;IACH,cAAc,IAAI,oBAAoB,GAAG,SAAS;IAIlD;;;;;;OAMG;IACG,UAAU,IAAI,OAAO,CAAC,wBAAwB,CAAC;CAoJtD;AAUD;;;;GAIG;AACH,wBAAgB,mCAAmC,CACjD,EAAE,EAAE,QAAQ,GACX,wBAAwB,CAc1B;AAED;;;;;;;;;;;GAWG;AACH,wBAAgB,2BAA2B,CACzC,EAAE,EAAE,UAAU,EACd,WAAW,EAAE,MAAM,GAClB,MAAM,wBAAwB,CAahC"}

package/out/zero-cache/src/services/change-streamer/change-log-initializer.js

@@ -0,0 +1,270 @@
+import { assert } from "../../../../shared/src/asserts.js";
+import { must } from "../../../../shared/src/must.js";
+import { BigIntJSON } from "../../../../shared/src/bigint-json.js";
+import { schemaChangeTags } from "../change-source/protocol/current/schema-change-tags.js";
+import { Database } from "../../../../zqlite/src/db.js";
+import { getOrCreateCounter } from "../../observability/metrics.js";
+import { backfillRequestsFrom, foldCookies } from "../replicator/change-log-cookies.js";
+import { readBackfillRequests, readReplicaCookies } from "../replicator/schema/backfilling.js";
+import { getReplicationState } from "../replicator/schema/replication-state.js";
+import { StatementRunner } from "../../db/statements.js";
+import { CHANGE_LOG_STREAM_TABLE } from "../replicator/change-log-db.js";
+import { SQLITE_CHANGE_LOG_BOUNDARY_SQL } from "./sqlite-change-log-reader.js";
+//#region ../zero-cache/src/services/change-streamer/change-log-initializer.ts
+/**
+* The most change-log rows one comparison will scan.
+*
+* The interval is `(replicaWatermark, pgWatermark]`, so it is normally a
+* handful of transactions — but a replicator that has been stalled can leave it
+* arbitrarily large, and this runs on the stream loop between reconciliation and
+* `startStream`, where latency delays the resumption of replication. Over the
+* cap the comparison declines rather than pays: a chronically lagging replicator
+* shows up as `inconclusive`, which is charted, and not as a stall.
+*/
+var MAX_FOLD_SCAN_ROWS = 1e4;
+var ChangeLogInitializer = class {
+	#lc;
+	#initFromPgChangeLog;
+	#initFromReplica;
+	#sources;
+	#resultCounter = getOrCreateCounter("replica", "sqlite_change_log.init_compare_result", "Comparisons of the replica-derived stream initialization parameters against the Postgres-derived ones, by classification.");
+	#lastComparison;
+	constructor(lc, { initFromPgChangeLog, initFromReplica }, sources) {
+		assert(initFromPgChangeLog || initFromReplica, `At least one of initFromPgChangeLog or initFromReplica must be true`);
+		this.#lc = lc.withContext("component", "change-log-initializer");
+		this.#initFromPgChangeLog = initFromPgChangeLog;
+		this.#initFromReplica = initFromReplica;
+		this.#sources = sources;
+	}
+	/**
+	* Returns the comparison result from the most recent {@link initialize}
+	* call. Returns `undefined` when that call used only one source.
+	*
+	* Tests and metrics use this result. It does not affect initialization.
+	*/
+	lastComparison() {
+		return this.#lastComparison;
+	}
+	/**
+	* Returns the initialization parameters for one stream connection after it
+	* reconciles the change log.
+	*
+	* The selected source determines how reconciliation works. Keeping both
+	* operations here hides that choice from the caller.
+	*/
+	async initialize() {
+		this.#lastComparison = void 0;
+		const pg = this.#initFromPgChangeLog ? await this.#sources.pgChangeLog() : void 0;
+		const replica = this.#initFromReplica ? this.#readReplica(pg === void 0) : void 0;
+		const reconciled = this.#sources.reconcileChangeLog(pg && {
+			resumeWatermark: pg.lastWatermark,
+			cookies: pg.cookies
+		}, () => resumePointOf(must(replica, "no seed for the SQLite change log")));
+		if (pg) {
+			if (replica) this.#lastComparison = this.#compare(pg, replica);
+			return pg;
+		}
+		const resumePoint = reconciled ?? resumePointOf(must(replica, "no initialization source"));
+		return {
+			lastWatermark: resumePoint.resumeWatermark,
+			cookies: resumePoint.cookies,
+			backfillRequests: backfillRequestsFrom(resumePoint.cookies)
+		};
+	}
+	/**
+	* Compares both sources and records the result. Postgres remains
+	* authoritative when both sources are enabled.
+	*/
+	#compare(pg, replica) {
+		let result;
+		try {
+			result = this.#classify(pg, replica);
+		} catch (e) {
+			this.#lc.warn?.("failed to compare change-log initialization sources", e);
+			result = "error";
+		}
+		this.#resultCounter.add(1, { result });
+		this.#lc.debug?.("compared change-log initialization sources", { changeLogInitCompare: {
+			result,
+			pgWatermark: pg.lastWatermark,
+			replicaWatermark: replica.lastWatermark
+		} });
+		return result;
+	}
+	#readReplica(required) {
+		try {
+			return this.#sources.replica();
+		} catch (e) {
+			if (required) throw e;
+			this.#lc.warn?.("failed to derive change-log initialization from the replica", e);
+			this.#resultCounter.add(1, { result: "error" });
+			return;
+		}
+	}
+	#classify(pg, replica) {
+		if (replica.lastWatermark > pg.lastWatermark) return "inconclusive";
+		if (replica.lastWatermark === pg.lastWatermark) return this.#report(pg, replica, replica.cookies, "equal");
+		const db = this.#sources.changeLog();
+		if (db === void 0 || !spansInterval(db, replica.lastWatermark)) return "inconclusive";
+		const changes = readSchemaChanges(db, replica.lastWatermark, pg.lastWatermark);
+		if (changes === void 0) return "inconclusive";
+		return this.#report(pg, replica, foldCookies(replica.cookies, changes), "equal-after-fold", changes.length);
+	}
+	#report(pg, replica, derived, whenEqual, foldedChanges = 0) {
+		const expected = canonicalizeCookies(pg.cookies);
+		const actual = canonicalizeCookies(derived);
+		if (expected === actual) return whenEqual;
+		this.#lc.error?.("the replica-derived change-log cookies do not match Postgres", { changeLogInitCompare: {
+			pgWatermark: pg.lastWatermark,
+			replicaWatermark: replica.lastWatermark,
+			foldedChanges,
+			expected,
+			actual
+		} });
+		return "divergent";
+	}
+};
+var resumePointOf = ({ lastWatermark, cookies }) => ({
+	resumeWatermark: lastWatermark,
+	cookies
+});
+/**
+* Reads the replica's initialization parameters in one snapshot, which is
+* invariant 15 on the replica side: the state version and the cookies are only
+* meaningful together.
+*/
+function readReplicaInitializationParameters(db) {
+	const runner = new StatementRunner(db);
+	runner.begin();
+	try {
+		return {
+			lastWatermark: getReplicationState(runner).stateVersion,
+			backfillRequests: readBackfillRequests(db),
+			cookies: readReplicaCookies(db)
+		};
+	} finally {
+		if (db.inTransaction) runner.rollback();
+	}
+}
+/**
+* The replica source for {@link ChangeLogInitializer}, opening and closing the
+* file per stream connection rather than holding a handle on a database another
+* process owns.
+*
+* The handle is read-only, which is invariant 19 stated in the API: the
+* replica's cookie tables are written by the change-processor and by nothing
+* else. Note that a read-only SQLite connection still needs to attach the
+* WAL's `-shm`, so this can fail if it opens at a moment when no writer holds
+* the replica; that surfaces as the `error` classification, which is the point
+* of running the path under observation before anything depends on it.
+*/
+function replicaInitializationSource(lc, replicaFile) {
+	return () => {
+		const db = new Database(lc.withContext("component", "change-log-initializer"), replicaFile, { readonly: true });
+		try {
+			return readReplicaInitializationParameters(db);
+		} finally {
+			db.close();
+		}
+	};
+}
+/**
+* Whether the log holds the complete interval above `fromWatermark`, using the
+* same predicate catchup uses to decide `too-old`: the log both reaches back
+* that far and contains that transaction's `commit` row, so the next row after
+* it is the first change of the interval rather than the middle of a
+* transaction the log only partly holds.
+*/
+function spansInterval(db, fromWatermark) {
+	if (!streamTableExists(db)) return false;
+	const row = db.prepare(`
+      SELECT
+        (SELECT min("watermark") FROM "${CHANGE_LOG_STREAM_TABLE}")
+          AS "minWatermark",
+        coalesce((${SQLITE_CHANGE_LOG_BOUNDARY_SQL}), 0) AS "boundaryExists"
+    `).get({ fromWatermark });
+	return row.minWatermark !== null && fromWatermark >= row.minWatermark && row.boundaryExists === 1;
+}
+var SCHEMA_CHANGE_TAG_LIST = schemaChangeTags.map((tag) => `'${tag}'`).join(", ");
+/**
+* The log's schema changes over `(after, through]`, in stream order, or
+* `undefined` if the interval is too large to scan (see
+* {@link MAX_FOLD_SCAN_ROWS}).
+*
+* The tag is stored beside the verbatim change so this scan never parses the
+* payloads of ordinary data changes. The row count is still checked first so a
+* stalled replicator cannot make the range scan unbounded.
+*
+* The count is itself capped, at one row past the cap it is deciding. A bare
+* `count(*)` over the interval is `O(interval)`, so the unbounded interval this
+* exists to decline would still be walked in full before being declined —
+* which is the latency the cap is here to avoid, not a cheaper form of it. The
+* `LIMIT` makes the decision `O(MAX_FOLD_SCAN_ROWS)`, and the comparison is
+* unaffected: the subquery returns the true count whenever it is within the
+* cap, and `cap + 1` whenever it is not.
+*/
+function readSchemaChanges(db, after, through) {
+	const { rows } = db.prepare(`
+      SELECT count(*) AS "rows" FROM (
+        SELECT 1 FROM "${CHANGE_LOG_STREAM_TABLE}"
+          WHERE "watermark" > ? AND "watermark" <= ?
+          LIMIT ${MAX_FOLD_SCAN_ROWS + 1}
+      )
+    `).get(after, through);
+	if (rows > MAX_FOLD_SCAN_ROWS) return;
+	return db.prepare(`
+      SELECT "change" FROM "${CHANGE_LOG_STREAM_TABLE}"
+        WHERE "watermark" > ? AND "watermark" <= ?
+          AND "tag" IN (${SCHEMA_CHANGE_TAG_LIST})
+        ORDER BY "watermark", "pos"
+    `).all(after, through).map(({ change }) => BigIntJSON.parse(change));
+}
+function streamTableExists(db) {
+	return db.prepare(`
+        SELECT 1 FROM "sqlite_master" WHERE "type" = 'table' AND "name" = ?
+      `).get(CHANGE_LOG_STREAM_TABLE) !== void 0;
+}
+/**
+* A canonical rendering of a whole cookie set: rows sorted by primary key,
+* object keys sorted at every depth.
+*
+* Both the equality check and the `divergent` log line fall out of this one
+* value, and neither can be defeated by a difference that is not a difference.
+* Postgres stores the cookie documents as `jsonb`, which does not preserve key
+* order, and the two readers order their rows in SQL in two engines under two
+* collations -- `storer.ts` pins `COLLATE "C"` for exactly that reason. Sorting
+* again here means an ordering or key-order regression can never be reported as
+* a divergence, which is a class of false alert that would cost more than it
+* caught.
+*/
+function canonicalizeCookies(cookies) {
+	return BigIntJSON.stringify({
+		tableMetadata: byKey(cookies.tableMetadata, (c) => [c.schema, c.table]).map((c) => [
+			c.schema,
+			c.table,
+			sortKeys(c.metadata)
+		]),
+		backfilling: byKey(cookies.backfilling, (c) => [
+			c.schema,
+			c.table,
+			c.column
+		]).map((c) => [
+			c.schema,
+			c.table,
+			c.column,
+			sortKeys(c.backfill)
+		])
+	});
+}
+function byKey(rows, key) {
+	return rows.map((row) => [JSON.stringify(key(row)), row]).sort(([a], [b]) => a < b ? -1 : a > b ? 1 : 0).map(([, row]) => row);
+}
+function sortKeys(value) {
+	if (Array.isArray(value)) return value.map(sortKeys);
+	if (value === null || typeof value !== "object") return value;
+	return Object.fromEntries(Object.entries(value).sort(([a], [b]) => a < b ? -1 : a > b ? 1 : 0).map(([k, v]) => [k, sortKeys(v)]));
+}
+//#endregion
+export { ChangeLogInitializer, replicaInitializationSource };
+
+//# sourceMappingURL=change-log-initializer.js.map

package/out/zero-cache/src/services/change-streamer/change-log-initializer.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"change-log-initializer.js","names":["#lc","#initFromPgChangeLog","#initFromReplica","#sources","#resultCounter","#lastComparison","#readReplica","#compare","#classify","#report"],"sources":["../../../../../../zero-cache/src/services/change-streamer/change-log-initializer.ts"],"sourcesContent":["/**\n * Where a stream connection's initialization parameters come from.\n *\n * A change-streamer starts a stream from three things read at one position: the\n * watermark to resume at, the {@link BackfillRequest}s to hand the change\n * source, and the cookie set those requests were derived from. Postgres has\n * always supplied all three (`Storer.getStartStreamInitializationParameters()`).\n * The point of this workstream is that a restored replica can supply them\n * instead, which is what lets the `cdc` schema be decommissioned.\n *\n * This is the two-source stage. Both are computed on every stream connection,\n * compared, and Postgres's is used — so the replica-derived path runs in\n * production, at full rate, before anything depends on it. The flip is\n * `initFromPgChangeLog: false`, not a rewrite.\n *\n * The shape is {@link UpstreamAcker}'s, deliberately: two stores, one output, a\n * flag per store, and the Postgres flag carrying the marker that retires it.\n * That class solved the identical problem for upstream ACKs earlier in this\n * workstream.\n *\n * ### Why the comparison is a fold and not an equality\n *\n * The replica is a subscriber downstream of the change-streamer's write loop,\n * so its state version trails the log's head by however long the replicator is\n * behind. A raw difference between the two cookie sets is therefore the normal\n * case and says nothing. What is checked instead is that the replica's set,\n * carried forward over exactly the schema changes the log holds in the interval,\n * equals Postgres's set at Postgres's watermark. The log holds precisely those\n * transactions, schema changes are rare, and the check is exact rather than\n * approximate — so it also happens to be the strongest available test of the\n * premise that three interpreters of one fold agree.\n */\n\nimport type {LogContext} from '@rocicorp/logger';\nimport {assert} from '../../../../shared/src/asserts.ts';\nimport {BigIntJSON} from '../../../../shared/src/bigint-json.ts';\nimport {must} from '../../../../shared/src/must.ts';\nimport {Database} from '../../../../zqlite/src/db.ts';\nimport {StatementRunner} from '../../db/statements.ts';\nimport {getOrCreateCounter} from '../../observability/metrics.ts';\nimport type {SchemaChange} from '../change-source/protocol/current/data.ts';\nimport {schemaChangeTags} from '../change-source/protocol/current/schema-change-tags.ts';\nimport type {BackfillRequest} from '../change-source/protocol/current/upstream.ts';\nimport {\n  backfillRequestsFrom,\n  foldCookies,\n  type CookieSet,\n} from '../replicator/change-log-cookies.ts';\nimport {\n  CHANGE_LOG_STREAM_TABLE,\n  type ChangeLogResumePoint,\n} from '../replicator/change-log-db.ts';\nimport {\n  readBackfillRequests,\n  readReplicaCookies,\n} from '../replicator/schema/backfilling.ts';\nimport {getReplicationState} from '../replicator/schema/replication-state.ts';\nimport {SQLITE_CHANGE_LOG_BOUNDARY_SQL} from './sqlite-change-log-reader.ts';\n\n/** Everything one stream connection starts from, read at one position. */\nexport type InitializationParameters = {\n  readonly lastWatermark: string;\n  readonly backfillRequests: BackfillRequest[];\n  readonly cookies: CookieSet;\n};\n\n/**\n * - `equal` — the two stores were at the same watermark and held the same\n *   cookies. Nothing had to be folded, which is the strongest of the four.\n * - `equal-after-fold` — the replica trailed, and carrying its set forward over\n *   the log's schema changes in the interval reproduced Postgres's exactly.\n *   The expected steady-state result.\n * - `divergent` — the two disagree at the same position. Alert; investigate\n *   individually.\n * - `inconclusive` — the log does not span the interval, so the fold cannot be\n *   performed. Routine on a cold log, and not a finding.\n * - `error` — the comparison itself failed: the replica could not be read, or\n *   the fold threw. Not one of the four classifications, because it is a\n *   statement about this process rather than about the two stores. Kept\n *   separate rather than folded into `inconclusive` so that a change-streamer\n *   that has silently stopped being able to read its replica is visible before\n *   that read becomes the only source.\n */\nexport type InitComparisonResult =\n  | 'equal'\n  | 'equal-after-fold'\n  | 'divergent'\n  | 'inconclusive'\n  | 'error';\n\nexport type ChangeLogInitializerSources = {\n  /** `Storer.getStartStreamInitializationParameters()`. */\n  readonly pgChangeLog: () => Promise<InitializationParameters>;\n  /** Typically {@link replicaInitializationSource}. */\n  readonly replica: () => InitializationParameters;\n  /**\n   * Reconciles the SQLite change log and returns its current resume point.\n   *\n   * When `resumeFrom` is present, it supplies the resume point. Otherwise, a\n   * valid log uses its own head. A new or invalid log uses `seed`.\n   *\n   * Returns `undefined` when the writer is disabled or unavailable.\n   */\n  readonly reconcileChangeLog: (\n    resumeFrom: ChangeLogResumePoint | undefined,\n    seed: () => ChangeLogResumePoint,\n  ) => ChangeLogResumePoint | undefined;\n  /**\n   * The change log, for the fold. `undefined` before the stream loop's first\n   * reconcile has created it, and again once the writer has failed soft and\n   * deleted it — both of which make the comparison `inconclusive` rather than\n   * an error, since neither says anything about the two stores.\n   */\n  readonly changeLog: () => Database | undefined;\n};\n\ntype Opts = {\n  /**\n   * When true, Postgres supplies the resume point and the cookies for a new\n   * change log.\n   *\n   * When false, a valid log resumes from its own head. The replica supplies the\n   * resume point for a new or invalid log.\n   */\n  // TODO: set false when retiring PG\n  readonly initFromPgChangeLog: boolean;\n  /**\n   * Derive the parameters from the replica. When Postgres is enabled, these\n   * parameters are used only for comparison.\n   *\n   * When Postgres is disabled, the replica supplies the seed and the fallback\n   * resume point.\n   */\n  readonly initFromReplica: boolean;\n};\n\n/**\n * The most change-log rows one comparison will scan.\n *\n * The interval is `(replicaWatermark, pgWatermark]`, so it is normally a\n * handful of transactions — but a replicator that has been stalled can leave it\n * arbitrarily large, and this runs on the stream loop between reconciliation and\n * `startStream`, where latency delays the resumption of replication. Over the\n * cap the comparison declines rather than pays: a chronically lagging replicator\n * shows up as `inconclusive`, which is charted, and not as a stall.\n */\nconst MAX_FOLD_SCAN_ROWS = 10_000;\n\nexport class ChangeLogInitializer {\n  readonly #lc: LogContext;\n  readonly #initFromPgChangeLog: boolean;\n  readonly #initFromReplica: boolean;\n  readonly #sources: ChangeLogInitializerSources;\n\n  readonly #resultCounter = getOrCreateCounter(\n    'replica',\n    'sqlite_change_log.init_compare_result',\n    'Comparisons of the replica-derived stream initialization parameters ' +\n      'against the Postgres-derived ones, by classification.',\n  );\n\n  #lastComparison: InitComparisonResult | undefined;\n\n  constructor(\n    lc: LogContext,\n    {initFromPgChangeLog, initFromReplica}: Opts,\n    sources: ChangeLogInitializerSources,\n  ) {\n    assert(\n      initFromPgChangeLog || initFromReplica,\n      `At least one of initFromPgChangeLog or initFromReplica must be true`,\n    );\n    this.#lc = lc.withContext('component', 'change-log-initializer');\n    this.#initFromPgChangeLog = initFromPgChangeLog;\n    this.#initFromReplica = initFromReplica;\n    this.#sources = sources;\n  }\n\n  /**\n   * Returns the comparison result from the most recent {@link initialize}\n   * call. Returns `undefined` when that call used only one source.\n   *\n   * Tests and metrics use this result. It does not affect initialization.\n   */\n  lastComparison(): InitComparisonResult | undefined {\n    return this.#lastComparison;\n  }\n\n  /**\n   * Returns the initialization parameters for one stream connection after it\n   * reconciles the change log.\n   *\n   * The selected source determines how reconciliation works. Keeping both\n   * operations here hides that choice from the caller.\n   */\n  async initialize(): Promise<InitializationParameters> {\n    this.#lastComparison = undefined;\n    const pg = this.#initFromPgChangeLog\n      ? await this.#sources.pgChangeLog()\n      : undefined;\n    // The replica is required when Postgres is disabled. It seeds a new log and\n    // supplies a resume point when the writer is unavailable.\n    const replica = this.#initFromReplica\n      ? this.#readReplica(pg === undefined)\n      : undefined;\n\n    const reconciled = this.#sources.reconcileChangeLog(\n      pg && {resumeWatermark: pg.lastWatermark, cookies: pg.cookies},\n      () => resumePointOf(must(replica, 'no seed for the SQLite change log')),\n    );\n\n    if (pg) {\n      if (replica) {\n        // Compare only after reconciliation. The fold must read the schema\n        // changes from the replica watermark through the Postgres watermark.\n        // Reconciliation makes sure that the log contains that interval.\n        this.#lastComparison = this.#compare(pg, replica);\n      }\n      return pg;\n    }\n\n    // Use the reconciled log position. If the writer is unavailable, use the\n    // replica position.\n    const resumePoint =\n      reconciled ?? resumePointOf(must(replica, 'no initialization source'));\n    return {\n      lastWatermark: resumePoint.resumeWatermark,\n      cookies: resumePoint.cookies,\n      // Derive the requests from these cookies so that the requests and the\n      // watermark describe the same position.\n      backfillRequests: backfillRequestsFrom(resumePoint.cookies),\n    };\n  }\n\n  /**\n   * Compares both sources and records the result. Postgres remains\n   * authoritative when both sources are enabled.\n   */\n  #compare(\n    pg: InitializationParameters,\n    replica: InitializationParameters,\n  ): InitComparisonResult {\n    let result: InitComparisonResult;\n    try {\n      result = this.#classify(pg, replica);\n    } catch (e) {\n      this.#lc.warn?.('failed to compare change-log initialization sources', e);\n      result = 'error';\n    }\n    this.#resultCounter.add(1, {result});\n    this.#lc.debug?.('compared change-log initialization sources', {\n      changeLogInitCompare: {\n        result,\n        pgWatermark: pg.lastWatermark,\n        replicaWatermark: replica.lastWatermark,\n      },\n    });\n    return result;\n  }\n\n  #readReplica(required: boolean): InitializationParameters | undefined {\n    try {\n      return this.#sources.replica();\n    } catch (e) {\n      if (required) {\n        throw e;\n      }\n      // Observational until the flip, so a replica that cannot be read must not\n      // stop replication -- but it must not be quiet either, since the flip\n      // makes this read the only source.\n      this.#lc.warn?.(\n        'failed to derive change-log initialization from the replica',\n        e,\n      );\n      this.#resultCounter.add(1, {result: 'error'});\n      return undefined;\n    }\n  }\n\n  #classify(\n    pg: InitializationParameters,\n    replica: InitializationParameters,\n  ): InitComparisonResult {\n    if (replica.lastWatermark > pg.lastWatermark) {\n      // Transactions reach subscribers before the storer's commit, so a replica\n      // read taken while one is in flight can be a transaction ahead. The fold\n      // only runs forward, so there is nothing to be said either way.\n      return 'inconclusive';\n    }\n    if (replica.lastWatermark === pg.lastWatermark) {\n      return this.#report(pg, replica, replica.cookies, 'equal');\n    }\n\n    const db = this.#sources.changeLog();\n    if (db === undefined || !spansInterval(db, replica.lastWatermark)) {\n      return 'inconclusive';\n    }\n    const changes = readSchemaChanges(\n      db,\n      replica.lastWatermark,\n      pg.lastWatermark,\n    );\n    if (changes === undefined) {\n      return 'inconclusive'; // over MAX_FOLD_SCAN_ROWS\n    }\n    return this.#report(\n      pg,\n      replica,\n      foldCookies(replica.cookies, changes),\n      'equal-after-fold',\n      changes.length,\n    );\n  }\n\n  #report(\n    pg: InitializationParameters,\n    replica: InitializationParameters,\n    derived: CookieSet,\n    whenEqual: InitComparisonResult,\n    foldedChanges = 0,\n  ): InitComparisonResult {\n    const expected = canonicalizeCookies(pg.cookies);\n    const actual = canonicalizeCookies(derived);\n    if (expected === actual) {\n      return whenEqual;\n    }\n    // Both sets are normally empty and are at most a handful of rows, so they\n    // are logged whole: a divergence is investigated individually (`P§:10`)\n    // and the two renderings are the whole of what there is to look at.\n    this.#lc.error?.(\n      'the replica-derived change-log cookies do not match Postgres',\n      {\n        changeLogInitCompare: {\n          pgWatermark: pg.lastWatermark,\n          replicaWatermark: replica.lastWatermark,\n          foldedChanges,\n          expected,\n          actual,\n        },\n      },\n    );\n    return 'divergent';\n  }\n}\n\nconst resumePointOf = ({\n  lastWatermark,\n  cookies,\n}: InitializationParameters): ChangeLogResumePoint => ({\n  resumeWatermark: lastWatermark,\n  cookies,\n});\n\n/**\n * Reads the replica's initialization parameters in one snapshot, which is\n * invariant 15 on the replica side: the state version and the cookies are only\n * meaningful together.\n */\nexport function readReplicaInitializationParameters(\n  db: Database,\n): InitializationParameters {\n  const runner = new StatementRunner(db);\n  runner.begin(); // deferred, i.e. one read snapshot for the three statements\n  try {\n    return {\n      lastWatermark: getReplicationState(runner).stateVersion,\n      backfillRequests: readBackfillRequests(db),\n      cookies: readReplicaCookies(db),\n    };\n  } finally {\n    if (db.inTransaction) {\n      runner.rollback();\n    }\n  }\n}\n\n/**\n * The replica source for {@link ChangeLogInitializer}, opening and closing the\n * file per stream connection rather than holding a handle on a database another\n * process owns.\n *\n * The handle is read-only, which is invariant 19 stated in the API: the\n * replica's cookie tables are written by the change-processor and by nothing\n * else. Note that a read-only SQLite connection still needs to attach the\n * WAL's `-shm`, so this can fail if it opens at a moment when no writer holds\n * the replica; that surfaces as the `error` classification, which is the point\n * of running the path under observation before anything depends on it.\n */\nexport function replicaInitializationSource(\n  lc: LogContext,\n  replicaFile: string,\n): () => InitializationParameters {\n  return () => {\n    const db = new Database(\n      lc.withContext('component', 'change-log-initializer'),\n      replicaFile,\n      {readonly: true},\n    );\n    try {\n      return readReplicaInitializationParameters(db);\n    } finally {\n      db.close();\n    }\n  };\n}\n\n/**\n * Whether the log holds the complete interval above `fromWatermark`, using the\n * same predicate catchup uses to decide `too-old`: the log both reaches back\n * that far and contains that transaction's `commit` row, so the next row after\n * it is the first change of the interval rather than the middle of a\n * transaction the log only partly holds.\n */\nfunction spansInterval(db: Database, fromWatermark: string): boolean {\n  if (!streamTableExists(db)) {\n    return false;\n  }\n  const row = db\n    .prepare(/*sql*/ `\n      SELECT\n        (SELECT min(\"watermark\") FROM \"${CHANGE_LOG_STREAM_TABLE}\")\n          AS \"minWatermark\",\n        coalesce((${SQLITE_CHANGE_LOG_BOUNDARY_SQL}), 0) AS \"boundaryExists\"\n    `)\n    .get<{minWatermark: string | null; boundaryExists: number}>({\n      fromWatermark,\n    });\n  return (\n    row.minWatermark !== null &&\n    fromWatermark >= row.minWatermark &&\n    row.boundaryExists === 1\n  );\n}\n\nconst SCHEMA_CHANGE_TAG_LIST = schemaChangeTags\n  .map(tag => `'${tag}'`)\n  .join(', ');\n\n/**\n * The log's schema changes over `(after, through]`, in stream order, or\n * `undefined` if the interval is too large to scan (see\n * {@link MAX_FOLD_SCAN_ROWS}).\n *\n * The tag is stored beside the verbatim change so this scan never parses the\n * payloads of ordinary data changes. The row count is still checked first so a\n * stalled replicator cannot make the range scan unbounded.\n *\n * The count is itself capped, at one row past the cap it is deciding. A bare\n * `count(*)` over the interval is `O(interval)`, so the unbounded interval this\n * exists to decline would still be walked in full before being declined —\n * which is the latency the cap is here to avoid, not a cheaper form of it. The\n * `LIMIT` makes the decision `O(MAX_FOLD_SCAN_ROWS)`, and the comparison is\n * unaffected: the subquery returns the true count whenever it is within the\n * cap, and `cap + 1` whenever it is not.\n */\nfunction readSchemaChanges(\n  db: Database,\n  after: string,\n  through: string,\n): SchemaChange[] | undefined {\n  const {rows} = db\n    .prepare(/*sql*/ `\n      SELECT count(*) AS \"rows\" FROM (\n        SELECT 1 FROM \"${CHANGE_LOG_STREAM_TABLE}\"\n          WHERE \"watermark\" > ? AND \"watermark\" <= ?\n          LIMIT ${MAX_FOLD_SCAN_ROWS + 1}\n      )\n    `)\n    .get<{rows: number}>(after, through);\n  if (rows > MAX_FOLD_SCAN_ROWS) {\n    return undefined;\n  }\n  return db\n    .prepare(/*sql*/ `\n      SELECT \"change\" FROM \"${CHANGE_LOG_STREAM_TABLE}\"\n        WHERE \"watermark\" > ? AND \"watermark\" <= ?\n          AND \"tag\" IN (${SCHEMA_CHANGE_TAG_LIST})\n        ORDER BY \"watermark\", \"pos\"\n    `)\n    .all<{change: string}>(after, through)\n    .map(({change}) => BigIntJSON.parse(change) as SchemaChange);\n}\n\nfunction streamTableExists(db: Database): boolean {\n  return (\n    db\n      .prepare(/*sql*/ `\n        SELECT 1 FROM \"sqlite_master\" WHERE \"type\" = 'table' AND \"name\" = ?\n      `)\n      .get<{1: number} | undefined>(CHANGE_LOG_STREAM_TABLE) !== undefined\n  );\n}\n\n/**\n * A canonical rendering of a whole cookie set: rows sorted by primary key,\n * object keys sorted at every depth.\n *\n * Both the equality check and the `divergent` log line fall out of this one\n * value, and neither can be defeated by a difference that is not a difference.\n * Postgres stores the cookie documents as `jsonb`, which does not preserve key\n * order, and the two readers order their rows in SQL in two engines under two\n * collations -- `storer.ts` pins `COLLATE \"C\"` for exactly that reason. Sorting\n * again here means an ordering or key-order regression can never be reported as\n * a divergence, which is a class of false alert that would cost more than it\n * caught.\n */\nfunction canonicalizeCookies(cookies: CookieSet): string {\n  return BigIntJSON.stringify({\n    tableMetadata: byKey(cookies.tableMetadata, c => [c.schema, c.table]).map(\n      c => [c.schema, c.table, sortKeys(c.metadata)],\n    ),\n    backfilling: byKey(cookies.backfilling, c => [\n      c.schema,\n      c.table,\n      c.column,\n    ]).map(c => [c.schema, c.table, c.column, sortKeys(c.backfill)]),\n  });\n}\n\nfunction byKey<T>(rows: readonly T[], key: (row: T) => string[]): T[] {\n  return rows\n    .map(row => [JSON.stringify(key(row)), row] as const)\n    .sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0))\n    .map(([, row]) => row);\n}\n\nfunction sortKeys(value: unknown): unknown {\n  if (Array.isArray(value)) {\n    return value.map(sortKeys);\n  }\n  if (value === null || typeof value !== 'object') {\n    return value;\n  }\n  return Object.fromEntries(\n    Object.entries(value)\n      .sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0))\n      .map(([k, v]) => [k, sortKeys(v)]),\n  );\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;AAkJA,IAAM,qBAAqB;AAE3B,IAAa,uBAAb,MAAkC;CAChC;CACA;CACA;CACA;CAEA,iBAA0B,mBACxB,WACA,yCACA,2HAEF;CAEA;CAEA,YACE,IACA,EAAC,qBAAqB,mBACtB,SACA;EACA,OACE,uBAAuB,iBACvB,qEACF;EACA,KAAKA,MAAM,GAAG,YAAY,aAAa,wBAAwB;EAC/D,KAAKC,uBAAuB;EAC5B,KAAKC,mBAAmB;EACxB,KAAKC,WAAW;CAClB;;;;;;;CAQA,iBAAmD;EACjD,OAAO,KAAKE;CACd;;;;;;;;CASA,MAAM,aAAgD;EACpD,KAAKA,kBAAkB,KAAA;EACvB,MAAM,KAAK,KAAKJ,uBACZ,MAAM,KAAKE,SAAS,YAAY,IAChC,KAAA;EAGJ,MAAM,UAAU,KAAKD,mBACjB,KAAKI,aAAa,OAAO,KAAA,CAAS,IAClC,KAAA;EAEJ,MAAM,aAAa,KAAKH,SAAS,mBAC/B,MAAM;GAAC,iBAAiB,GAAG;GAAe,SAAS,GAAG;EAAO,SACvD,cAAc,KAAK,SAAS,mCAAmC,CAAC,CACxE;EAEA,IAAI,IAAI;GACN,IAAI,SAIF,KAAKE,kBAAkB,KAAKE,SAAS,IAAI,OAAO;GAElD,OAAO;EACT;EAIA,MAAM,cACJ,cAAc,cAAc,KAAK,SAAS,0BAA0B,CAAC;EACvE,OAAO;GACL,eAAe,YAAY;GAC3B,SAAS,YAAY;GAGrB,kBAAkB,qBAAqB,YAAY,OAAO;EAC5D;CACF;;;;;CAMA,SACE,IACA,SACsB;EACtB,IAAI;EACJ,IAAI;GACF,SAAS,KAAKC,UAAU,IAAI,OAAO;EACrC,SAAS,GAAG;GACV,KAAKR,IAAI,OAAO,uDAAuD,CAAC;GACxE,SAAS;EACX;EACA,KAAKI,eAAe,IAAI,GAAG,EAAC,OAAM,CAAC;EACnC,KAAKJ,IAAI,QAAQ,8CAA8C,EAC7D,sBAAsB;GACpB;GACA,aAAa,GAAG;GAChB,kBAAkB,QAAQ;EAC5B,EACF,CAAC;EACD,OAAO;CACT;CAEA,aAAa,UAAyD;EACpE,IAAI;GACF,OAAO,KAAKG,SAAS,QAAQ;EAC/B,SAAS,GAAG;GACV,IAAI,UACF,MAAM;GAKR,KAAKH,IAAI,OACP,+DACA,CACF;GACA,KAAKI,eAAe,IAAI,GAAG,EAAC,QAAQ,QAAO,CAAC;GAC5C;EACF;CACF;CAEA,UACE,IACA,SACsB;EACtB,IAAI,QAAQ,gBAAgB,GAAG,eAI7B,OAAO;EAET,IAAI,QAAQ,kBAAkB,GAAG,eAC/B,OAAO,KAAKK,QAAQ,IAAI,SAAS,QAAQ,SAAS,OAAO;EAG3D,MAAM,KAAK,KAAKN,SAAS,UAAU;EACnC,IAAI,OAAO,KAAA,KAAa,CAAC,cAAc,IAAI,QAAQ,aAAa,GAC9D,OAAO;EAET,MAAM,UAAU,kBACd,IACA,QAAQ,eACR,GAAG,aACL;EACA,IAAI,YAAY,KAAA,GACd,OAAO;EAET,OAAO,KAAKM,QACV,IACA,SACA,YAAY,QAAQ,SAAS,OAAO,GACpC,oBACA,QAAQ,MACV;CACF;CAEA,QACE,IACA,SACA,SACA,WACA,gBAAgB,GACM;EACtB,MAAM,WAAW,oBAAoB,GAAG,OAAO;EAC/C,MAAM,SAAS,oBAAoB,OAAO;EAC1C,IAAI,aAAa,QACf,OAAO;EAKT,KAAKT,IAAI,QACP,gEACA,EACE,sBAAsB;GACpB,aAAa,GAAG;GAChB,kBAAkB,QAAQ;GAC1B;GACA;GACA;EACF,EACF,CACF;EACA,OAAO;CACT;AACF;AAEA,IAAM,iBAAiB,EACrB,eACA,eACqD;CACrD,iBAAiB;CACjB;AACF;;;;;;AAOA,SAAgB,oCACd,IAC0B;CAC1B,MAAM,SAAS,IAAI,gBAAgB,EAAE;CACrC,OAAO,MAAM;CACb,IAAI;EACF,OAAO;GACL,eAAe,oBAAoB,MAAM,EAAE;GAC3C,kBAAkB,qBAAqB,EAAE;GACzC,SAAS,mBAAmB,EAAE;EAChC;CACF,UAAU;EACR,IAAI,GAAG,eACL,OAAO,SAAS;CAEpB;AACF;;;;;;;;;;;;;AAcA,SAAgB,4BACd,IACA,aACgC;CAChC,aAAa;EACX,MAAM,KAAK,IAAI,SACb,GAAG,YAAY,aAAa,wBAAwB,GACpD,aACA,EAAC,UAAU,KAAI,CACjB;EACA,IAAI;GACF,OAAO,oCAAoC,EAAE;EAC/C,UAAU;GACR,GAAG,MAAM;EACX;CACF;AACF;;;;;;;;AASA,SAAS,cAAc,IAAc,eAAgC;CACnE,IAAI,CAAC,kBAAkB,EAAE,GACvB,OAAO;CAET,MAAM,MAAM,GACT,QAAgB;;yCAEoB,wBAAwB;;oBAE7C,+BAA+B;KAC9C,EACA,IAA2D,EAC1D,cACF,CAAC;CACH,OACE,IAAI,iBAAiB,QACrB,iBAAiB,IAAI,gBACrB,IAAI,mBAAmB;AAE3B;AAEA,IAAM,yBAAyB,iBAC5B,KAAI,QAAO,IAAI,IAAI,EAAE,EACrB,KAAK,IAAI;;;;;;;;;;;;;;;;;;AAmBZ,SAAS,kBACP,IACA,OACA,SAC4B;CAC5B,MAAM,EAAC,SAAQ,GACZ,QAAgB;;yBAEI,wBAAwB;;kBAE/B,qBAAqB,EAAE;;KAEpC,EACA,IAAoB,OAAO,OAAO;CACrC,IAAI,OAAO,oBACT;CAEF,OAAO,GACJ,QAAgB;8BACS,wBAAwB;;0BAE5B,uBAAuB;;KAE5C,EACA,IAAsB,OAAO,OAAO,EACpC,KAAK,EAAC,aAAY,WAAW,MAAM,MAAM,CAAiB;AAC/D;AAEA,SAAS,kBAAkB,IAAuB;CAChD,OACE,GACG,QAAgB;;OAEhB,EACA,IAA6B,uBAAuB,MAAM,KAAA;AAEjE;;;;;;;;;;;;;;AAeA,SAAS,oBAAoB,SAA4B;CACvD,OAAO,WAAW,UAAU;EAC1B,eAAe,MAAM,QAAQ,gBAAe,MAAK,CAAC,EAAE,QAAQ,EAAE,KAAK,CAAC,EAAE,KACpE,MAAK;GAAC,EAAE;GAAQ,EAAE;GAAO,SAAS,EAAE,QAAQ;EAAC,CAC/C;EACA,aAAa,MAAM,QAAQ,cAAa,MAAK;GAC3C,EAAE;GACF,EAAE;GACF,EAAE;EACJ,CAAC,EAAE,KAAI,MAAK;GAAC,EAAE;GAAQ,EAAE;GAAO,EAAE;GAAQ,SAAS,EAAE,QAAQ;EAAC,CAAC;CACjE,CAAC;AACH;AAEA,SAAS,MAAS,MAAoB,KAAgC;CACpE,OAAO,KACJ,KAAI,QAAO,CAAC,KAAK,UAAU,IAAI,GAAG,CAAC,GAAG,GAAG,CAAU,EACnD,MAAM,CAAC,IAAI,CAAC,OAAQ,IAAI,IAAI,KAAK,IAAI,IAAI,IAAI,CAAE,EAC/C,KAAK,GAAG,SAAS,GAAG;AACzB;AAEA,SAAS,SAAS,OAAyB;CACzC,IAAI,MAAM,QAAQ,KAAK,GACrB,OAAO,MAAM,IAAI,QAAQ;CAE3B,IAAI,UAAU,QAAQ,OAAO,UAAU,UACrC,OAAO;CAET,OAAO,OAAO,YACZ,OAAO,QAAQ,KAAK,EACjB,MAAM,CAAC,IAAI,CAAC,OAAQ,IAAI,IAAI,KAAK,IAAI,IAAI,IAAI,CAAE,EAC/C,KAAK,CAAC,GAAG,OAAO,CAAC,GAAG,SAAS,CAAC,CAAC,CAAC,CACrC;AACF"}

package/out/zero-cache/src/services/change-streamer/change-log-transaction-hash.d.ts

@@ -0,0 +1,19 @@
+export type ChangeLogHashRow = {
+    readonly watermark: string;
+    readonly pos: number;
+    readonly tag: string;
+    readonly change: string;
+    readonly precommit: string | null;
+};
+/**
+ * Computes a stable digest over the fields shared by the Postgres and SQLite
+ * change logs. Values are length-prefixed so different field boundaries cannot
+ * produce the same byte stream.
+ */
+export declare class ChangeLogTransactionHasher {
+    #private;
+    constructor();
+    add(row: ChangeLogHashRow): void;
+    digest(): string;
+}
+//# sourceMappingURL=change-log-transaction-hash.d.ts.map

package/out/zero-cache/src/services/change-streamer/change-log-transaction-hash.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"change-log-transaction-hash.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/change-streamer/change-log-transaction-hash.ts"],"names":[],"mappings":"AAEA,MAAM,MAAM,gBAAgB,GAAG;IAC7B,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAC;IACrB,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAC;IACrB,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;CACnC,CAAC;AAEF;;;;GAIG;AACH,qBAAa,0BAA0B;;;IAQrC,GAAG,CAAC,GAAG,EAAE,gBAAgB,GAAG,IAAI;IAahC,MAAM,IAAI,MAAM;CAGjB"}