@rocicorp/zero 1.11.0 → 1.11.1

package/out/zero-cache/src/services/change-streamer/backfill-state.js.map

@@ -1 +1 @@
-{"version":3,"file":"backfill-state.js","names":["#state","#committed","#beforeTx","#begin","#commit","#rollback","#applyBackfill","#applySchemaChange"],"sources":["../../../../../../zero-cache/src/services/change-streamer/backfill-state.ts"],"sourcesContent":["import {unreachable} from '../../../../shared/src/asserts.ts';\nimport {BigIntJSON} from '../../../../shared/src/bigint-json.ts';\nimport {deepEqual} from '../../../../shared/src/json.ts';\nimport {\n  acceptBackfill,\n  covers,\n} from '../change-source/protocol/backfill-progress.ts';\nimport type {\n  BackfillRequest,\n  ChangeStreamData,\n  Identifier,\n  SchemaChange,\n  TableMetadata,\n} from '../change-source/protocol/current.ts';\nimport {schemaChangeTags} from '../change-source/protocol/current/schema-change-tags.ts';\nimport {cookieOps} from '../replicator/change-log-cookies.ts';\nimport type {ChangeTag, WatermarkedChange} from './change-streamer.ts';\n\nfunction tableKey({schema, name}: Identifier) {\n  return JSON.stringify([schema, name]);\n}\n\ntype Column = BackfillRequest['columns'][string];\n\n/** A mutable BackfillRequest. */\ntype Backfill = {\n  table: {schema: string; name: string; metadata: TableMetadata | null};\n  columns: Record<string, Column>;\n};\n\ntype State = Map<string, Backfill>;\n\nfunction clone(state: State): State {\n  return new Map(\n    Array.from(state, ([key, {table, columns}]) => [\n      key,\n      {table: {...table}, columns: {...columns}},\n    ]),\n  );\n}\n\n/**\n * Tags of messages that affect the backfill state. Other messages need not be\n * parsed.\n */\nconst RELEVANT_TAGS = new Set<ChangeTag>([\n  'backfill',\n  ...schemaChangeTags.filter(t => t !== 'create-index' && t !== 'drop-index'),\n]);\n\n// Serialized changes are typically forwarded (as the same string) to every\n// subscriber, so a single entry cache avoids re-parsing the same message for\n// each subscriber.\nlet lastParsed: {json: string; change: ChangeStreamData} | undefined;\n\nfunction parse(json: string): ChangeStreamData {\n  if (lastParsed?.json !== json) {\n    lastParsed = {json, change: BigIntJSON.parse(json) as ChangeStreamData};\n  }\n  return lastParsed.change;\n}\n\n/**\n * Tracks the state of the pending backfills (and their progress) of a party\n * in the replication stream (e.g. a subscriber, or the change-log itself), by\n * applying the same policies that a subscriber applies to its replica:\n * schema changes are folded with the {@link cookieOps} shared by the\n * cookie jars, and `backfill` / `backfill-completed` messages are accepted\n * or ignored with the continuity policy of {@link acceptBackfill}.\n *\n * Changes are applied transactionally: the changes of a transaction are\n * discarded if it is rolled back, and {@link requests()} only reflects\n * committed transactions.\n */\nexport class BackfillState {\n  #state: State;\n  #beforeTx: State | null = null;\n\n  constructor(requests: readonly BackfillRequest[] = []) {\n    this.#state = new Map(\n      requests\n        .filter(req => Object.keys(req.columns).length > 0)\n        .map(({table, columns}) => [\n          tableKey(table),\n          {table: {...table}, columns: {...columns}},\n        ]),\n    );\n  }\n\n  get #committed(): State {\n    return this.#beforeTx ?? this.#state;\n  }\n\n  /** The (committed) pending backfills. */\n  requests(): BackfillRequest[] {\n    return [...clone(this.#committed).values()];\n  }\n\n  /**\n   * Returns the `requests` with the (committed) progress of this state\n   * applied to the columns that are pending in both, i.e. same table, column,\n   * and backfill ID. Other columns have no progress (i.e. from scratch).\n   */\n  withProgress(requests: readonly BackfillRequest[]): BackfillRequest[] {\n    const state = this.#committed;\n    return requests.map(({table, columns}) => {\n      const tracked = state.get(tableKey(table))?.columns;\n      return {\n        table,\n        columns: Object.fromEntries(\n          Object.entries(columns).map(([name, {id}]) => {\n            const col = tracked?.[name];\n            return [\n              name,\n              col?.progress && deepEqual(col.id, id)\n                ? {id, progress: col.progress}\n                : {id},\n            ];\n          }),\n        ),\n      };\n    });\n  }\n\n  /** Whether there are no (committed) pending backfills. */\n  get empty(): boolean {\n    return this.#committed.size === 0;\n  }\n\n  /**\n   * Returns the columns (as `schema.table.column` strings) with (committed)\n   * pending backfills that are not covered by the (committed) progress of the\n   * `stream`, i.e. columns for which the `stream` will not deliver all of the\n   * necessary data.\n   */\n  uncovered(stream: BackfillState): string[] {\n    const uncovered: string[] = [];\n    const streamState = stream.#committed;\n    for (const [key, {table, columns}] of this.#committed) {\n      const streamColumns = streamState.get(key)?.columns;\n      for (const [name, {id, progress}] of Object.entries(columns)) {\n        const s = streamColumns?.[name];\n        if (!s || !deepEqual(s.id, id) || !covers(s.progress, progress)) {\n          uncovered.push(`${table.schema}.${table.name}.${name}`);\n        }\n      }\n    }\n    return uncovered;\n  }\n\n  /**\n   * Applies a serialized change, parsing it only if it is relevant.\n   *\n   * @returns The columns (as `schema.table.column` strings) of a backfill\n   *          message that were ignored due to the continuity policy.\n   */\n  applySerialized([, tag, json]: WatermarkedChange): string[] {\n    switch (tag) {\n      case 'begin':\n        this.#begin();\n        return [];\n      case 'commit':\n        this.#commit();\n        return [];\n      case 'rollback':\n        this.#rollback();\n        return [];\n    }\n    if (!RELEVANT_TAGS.has(tag)) {\n      return [];\n    }\n    return this.apply(parse(json));\n  }\n\n  /**\n   * Applies a change.\n   *\n   * @returns The columns (as `schema.table.column` strings) of a backfill\n   *          message that were ignored due to the continuity policy.\n   */\n  apply(change: ChangeStreamData): string[] {\n    switch (change[0]) {\n      case 'begin':\n        this.#begin();\n        return [];\n      case 'commit':\n        this.#commit();\n        return [];\n      case 'rollback':\n        this.#rollback();\n        return [];\n    }\n    const msg = change[1];\n    switch (msg.tag) {\n      case 'backfill': {\n        const {relation, columns, progressMarks} = msg;\n        return this.#applyBackfill(relation, columns, (progress, col) => {\n          const result = acceptBackfill(progress, progressMarks);\n          if (result.accept) {\n            col.progress = result.progress;\n            if (col.progress === undefined) {\n              delete col.progress;\n            }\n          }\n          return result.accept;\n        });\n      }\n      case 'backfill-completed': {\n        const {relation, columns, progressMarks} = msg;\n        const backfill = this.#state.get(tableKey(relation));\n        const ignored = this.#applyBackfill(\n          relation,\n          columns,\n          (progress, _, name) => {\n            const {accept} = acceptBackfill(\n              progress,\n              progressMarks && {previous: progressMarks.previous},\n            );\n            if (accept && backfill) {\n              delete backfill.columns[name];\n            }\n            return accept;\n          },\n        );\n        if (backfill && Object.keys(backfill.columns).length === 0) {\n          this.#state.delete(tableKey(relation));\n        }\n        return ignored;\n      }\n      case 'create-table':\n      case 'rename-table':\n      case 'update-table-metadata':\n      case 'add-column':\n      case 'update-column':\n      case 'drop-column':\n      case 'drop-table':\n        this.#applySchemaChange(msg);\n        return [];\n    }\n    return [];\n  }\n\n  #applyBackfill(\n    relation: {schema: string; name: string; rowKey: {columns: string[]}},\n    columns: readonly string[],\n    apply: (progress: Column['progress'], col: Column, name: string) => boolean,\n  ): string[] {\n    const backfill = this.#state.get(tableKey(relation));\n    if (!backfill) {\n      return [];\n    }\n    const ignored: string[] = [];\n    // rowKey columns are excluded from `columns` but are backfilled with them\n    // (e.g. for a new table).\n    for (const name of [...relation.rowKey.columns, ...columns]) {\n      const col = backfill.columns[name];\n      if (!col) {\n        continue;\n      }\n      // Copy-on-write, since the column object may be shared with the\n      // pre-transaction state.\n      const updated = {...col};\n      if (apply(col.progress, updated, name)) {\n        if (name in backfill.columns) {\n          backfill.columns[name] = updated;\n        }\n      } else {\n        ignored.push(`${relation.schema}.${relation.name}.${name}`);\n      }\n    }\n    return ignored;\n  }\n\n  #applySchemaChange(change: SchemaChange) {\n    // Table metadata specified in the same change (e.g. a `create-table` or\n    // `add-column`), for backfills of tables that are not yet tracked.\n    const metadata = new Map<string, TableMetadata>();\n\n    for (const op of cookieOps(change)) {\n      switch (op.op) {\n        case 'upsert-metadata': {\n          const key = tableKey(op.table);\n          metadata.set(key, op.metadata);\n          const backfill = this.#state.get(key);\n          if (backfill) {\n            backfill.table.metadata = op.metadata;\n          }\n          break;\n        }\n        case 'upsert-backfill': {\n          const key = tableKey(op.table);\n          let backfill = this.#state.get(key);\n          if (!backfill) {\n            backfill = {\n              table: {\n                schema: op.table.schema,\n                name: op.table.name,\n                metadata: metadata.get(key) ?? null,\n              },\n              columns: {},\n            };\n            this.#state.set(key, backfill);\n          }\n          // A new backfill always starts from scratch.\n          backfill.columns[op.column] = {id: op.backfill};\n          break;\n        }\n        case 'rename-table': {\n          const backfill = this.#state.get(tableKey(op.old));\n          if (backfill) {\n            this.#state.delete(tableKey(op.old));\n            backfill.table.schema = op.new.schema;\n            backfill.table.name = op.new.name;\n            this.#state.set(tableKey(op.new), backfill);\n          }\n          break;\n        }\n        case 'drop-table':\n          this.#state.delete(tableKey(op.table));\n          break;\n        case 'rename-column': {\n          const backfill = this.#state.get(tableKey(op.table));\n          const col = backfill?.columns[op.old];\n          if (backfill && col) {\n            delete backfill.columns[op.old];\n            backfill.columns[op.new] = col;\n          }\n          break;\n        }\n        case 'drop-column': {\n          const key = tableKey(op.table);\n          const backfill = this.#state.get(key);\n          if (backfill) {\n            delete backfill.columns[op.column];\n            if (Object.keys(backfill.columns).length === 0) {\n              this.#state.delete(key);\n            }\n          }\n          break;\n        }\n        case 'complete-backfill':\n          // Only produced for `backfill-completed`, which is handled\n          // separately (with the continuity policy).\n          throw new Error(`unexpected ${op.op} op`);\n        default:\n          unreachable(op);\n      }\n    }\n  }\n\n  #begin() {\n    // Tolerate a missing commit/rollback, e.g. from an interrupted stream.\n    this.#beforeTx ??= clone(this.#state);\n  }\n\n  #commit() {\n    this.#beforeTx = null;\n  }\n\n  #rollback() {\n    if (this.#beforeTx) {\n      this.#state = this.#beforeTx;\n      this.#beforeTx = null;\n    }\n  }\n}\n"],"mappings":";;;;;;;AAkBA,SAAS,SAAS,EAAC,QAAQ,QAAmB;CAC5C,OAAO,KAAK,UAAU,CAAC,QAAQ,IAAI,CAAC;AACtC;AAYA,SAAS,MAAM,OAAqB;CAClC,OAAO,IAAI,IACT,MAAM,KAAK,QAAQ,CAAC,KAAK,EAAC,OAAO,eAAc,CAC7C,KACA;EAAC,OAAO,EAAC,GAAG,MAAK;EAAG,SAAS,EAAC,GAAG,QAAO;CAAC,CAC3C,CAAC,CACH;AACF;;;;;AAMA,IAAM,gBAAgB,IAAI,IAAe,CACvC,YACA,GAAG,iBAAiB,QAAO,MAAK,MAAM,kBAAkB,MAAM,YAAY,CAC5E,CAAC;AAKD,IAAI;AAEJ,SAAS,MAAM,MAAgC;CAC7C,IAAI,YAAY,SAAS,MACvB,aAAa;EAAC;EAAM,QAAQ,WAAW,MAAM,IAAI;CAAqB;CAExE,OAAO,WAAW;AACpB;;;;;;;;;;;;;AAcA,IAAa,gBAAb,MAA2B;CACzB;CACA,YAA0B;CAE1B,YAAY,WAAuC,CAAC,GAAG;EACrD,KAAKA,SAAS,IAAI,IAChB,SACG,QAAO,QAAO,OAAO,KAAK,IAAI,OAAO,EAAE,SAAS,CAAC,EACjD,KAAK,EAAC,OAAO,cAAa,CACzB,SAAS,KAAK,GACd;GAAC,OAAO,EAAC,GAAG,MAAK;GAAG,SAAS,EAAC,GAAG,QAAO;EAAC,CAC3C,CAAC,CACL;CACF;CAEA,IAAIC,aAAoB;EACtB,OAAO,KAAKC,aAAa,KAAKF;CAChC;;CAGA,WAA8B;EAC5B,OAAO,CAAC,GAAG,MAAM,KAAKC,UAAU,EAAE,OAAO,CAAC;CAC5C;;;;;;CAOA,aAAa,UAAyD;EACpE,MAAM,QAAQ,KAAKA;EACnB,OAAO,SAAS,KAAK,EAAC,OAAO,cAAa;GACxC,MAAM,UAAU,MAAM,IAAI,SAAS,KAAK,CAAC,GAAG;GAC5C,OAAO;IACL;IACA,SAAS,OAAO,YACd,OAAO,QAAQ,OAAO,EAAE,KAAK,CAAC,MAAM,EAAC,UAAS;KAC5C,MAAM,MAAM,UAAU;KACtB,OAAO,CACL,MACA,KAAK,YAAY,UAAU,IAAI,IAAI,EAAE,IACjC;MAAC;MAAI,UAAU,IAAI;KAAQ,IAC3B,EAAC,GAAE,CACT;IACF,CAAC,CACH;GACF;EACF,CAAC;CACH;;CAGA,IAAI,QAAiB;EACnB,OAAO,KAAKA,WAAW,SAAS;CAClC;;;;;;;CAQA,UAAU,QAAiC;EACzC,MAAM,YAAsB,CAAC;EAC7B,MAAM,cAAc,OAAOA;EAC3B,KAAK,MAAM,CAAC,KAAK,EAAC,OAAO,cAAa,KAAKA,YAAY;GACrD,MAAM,gBAAgB,YAAY,IAAI,GAAG,GAAG;GAC5C,KAAK,MAAM,CAAC,MAAM,EAAC,IAAI,eAAc,OAAO,QAAQ,OAAO,GAAG;IAC5D,MAAM,IAAI,gBAAgB;IAC1B,IAAI,CAAC,KAAK,CAAC,UAAU,EAAE,IAAI,EAAE,KAAK,CAAC,OAAO,EAAE,UAAU,QAAQ,GAC5D,UAAU,KAAK,GAAG,MAAM,OAAO,GAAG,MAAM,KAAK,GAAG,MAAM;GAE1D;EACF;EACA,OAAO;CACT;;;;;;;CAQA,gBAAgB,GAAG,KAAK,OAAoC;EAC1D,QAAQ,KAAR;GACE,KAAK;IACH,KAAKE,OAAO;IACZ,OAAO,CAAC;GACV,KAAK;IACH,KAAKC,QAAQ;IACb,OAAO,CAAC;GACV,KAAK;IACH,KAAKC,UAAU;IACf,OAAO,CAAC;EACZ;EACA,IAAI,CAAC,cAAc,IAAI,GAAG,GACxB,OAAO,CAAC;EAEV,OAAO,KAAK,MAAM,MAAM,IAAI,CAAC;CAC/B;;;;;;;CAQA,MAAM,QAAoC;EACxC,QAAQ,OAAO,IAAf;GACE,KAAK;IACH,KAAKF,OAAO;IACZ,OAAO,CAAC;GACV,KAAK;IACH,KAAKC,QAAQ;IACb,OAAO,CAAC;GACV,KAAK;IACH,KAAKC,UAAU;IACf,OAAO,CAAC;EACZ;EACA,MAAM,MAAM,OAAO;EACnB,QAAQ,IAAI,KAAZ;GACE,KAAK,YAAY;IACf,MAAM,EAAC,UAAU,SAAS,kBAAiB;IAC3C,OAAO,KAAKC,eAAe,UAAU,UAAU,UAAU,QAAQ;KAC/D,MAAM,SAAS,eAAe,UAAU,aAAa;KACrD,IAAI,OAAO,QAAQ;MACjB,IAAI,WAAW,OAAO;MACtB,IAAI,IAAI,aAAa,KAAA,GACnB,OAAO,IAAI;KAEf;KACA,OAAO,OAAO;IAChB,CAAC;GACH;GACA,KAAK,sBAAsB;IACzB,MAAM,EAAC,UAAU,SAAS,kBAAiB;IAC3C,MAAM,WAAW,KAAKN,OAAO,IAAI,SAAS,QAAQ,CAAC;IACnD,MAAM,UAAU,KAAKM,eACnB,UACA,UACC,UAAU,GAAG,SAAS;KACrB,MAAM,EAAC,WAAU,eACf,UACA,iBAAiB,EAAC,UAAU,cAAc,SAAQ,CACpD;KACA,IAAI,UAAU,UACZ,OAAO,SAAS,QAAQ;KAE1B,OAAO;IACT,CACF;IACA,IAAI,YAAY,OAAO,KAAK,SAAS,OAAO,EAAE,WAAW,GACvD,KAAKN,OAAO,OAAO,SAAS,QAAQ,CAAC;IAEvC,OAAO;GACT;GACA,KAAK;GACL,KAAK;GACL,KAAK;GACL,KAAK;GACL,KAAK;GACL,KAAK;GACL,KAAK;IACH,KAAKO,mBAAmB,GAAG;IAC3B,OAAO,CAAC;EACZ;EACA,OAAO,CAAC;CACV;CAEA,eACE,UACA,SACA,OACU;EACV,MAAM,WAAW,KAAKP,OAAO,IAAI,SAAS,QAAQ,CAAC;EACnD,IAAI,CAAC,UACH,OAAO,CAAC;EAEV,MAAM,UAAoB,CAAC;EAG3B,KAAK,MAAM,QAAQ,CAAC,GAAG,SAAS,OAAO,SAAS,GAAG,OAAO,GAAG;GAC3D,MAAM,MAAM,SAAS,QAAQ;GAC7B,IAAI,CAAC,KACH;GAIF,MAAM,UAAU,EAAC,GAAG,IAAG;GACvB,IAAI,MAAM,IAAI,UAAU,SAAS,IAAI;QAC/B,QAAQ,SAAS,SACnB,SAAS,QAAQ,QAAQ;GAAA,OAG3B,QAAQ,KAAK,GAAG,SAAS,OAAO,GAAG,SAAS,KAAK,GAAG,MAAM;EAE9D;EACA,OAAO;CACT;CAEA,mBAAmB,QAAsB;EAGvC,MAAM,2BAAW,IAAI,IAA2B;EAEhD,KAAK,MAAM,MAAM,UAAU,MAAM,GAC/B,QAAQ,GAAG,IAAX;GACE,KAAK,mBAAmB;IACtB,MAAM,MAAM,SAAS,GAAG,KAAK;IAC7B,SAAS,IAAI,KAAK,GAAG,QAAQ;IAC7B,MAAM,WAAW,KAAKA,OAAO,IAAI,GAAG;IACpC,IAAI,UACF,SAAS,MAAM,WAAW,GAAG;IAE/B;GACF;GACA,KAAK,mBAAmB;IACtB,MAAM,MAAM,SAAS,GAAG,KAAK;IAC7B,IAAI,WAAW,KAAKA,OAAO,IAAI,GAAG;IAClC,IAAI,CAAC,UAAU;KACb,WAAW;MACT,OAAO;OACL,QAAQ,GAAG,MAAM;OACjB,MAAM,GAAG,MAAM;OACf,UAAU,SAAS,IAAI,GAAG,KAAK;MACjC;MACA,SAAS,CAAC;KACZ;KACA,KAAKA,OAAO,IAAI,KAAK,QAAQ;IAC/B;IAEA,SAAS,QAAQ,GAAG,UAAU,EAAC,IAAI,GAAG,SAAQ;IAC9C;GACF;GACA,KAAK,gBAAgB;IACnB,MAAM,WAAW,KAAKA,OAAO,IAAI,SAAS,GAAG,GAAG,CAAC;IACjD,IAAI,UAAU;KACZ,KAAKA,OAAO,OAAO,SAAS,GAAG,GAAG,CAAC;KACnC,SAAS,MAAM,SAAS,GAAG,IAAI;KAC/B,SAAS,MAAM,OAAO,GAAG,IAAI;KAC7B,KAAKA,OAAO,IAAI,SAAS,GAAG,GAAG,GAAG,QAAQ;IAC5C;IACA;GACF;GACA,KAAK;IACH,KAAKA,OAAO,OAAO,SAAS,GAAG,KAAK,CAAC;IACrC;GACF,KAAK,iBAAiB;IACpB,MAAM,WAAW,KAAKA,OAAO,IAAI,SAAS,GAAG,KAAK,CAAC;IACnD,MAAM,MAAM,UAAU,QAAQ,GAAG;IACjC,IAAI,YAAY,KAAK;KACnB,OAAO,SAAS,QAAQ,GAAG;KAC3B,SAAS,QAAQ,GAAG,OAAO;IAC7B;IACA;GACF;GACA,KAAK,eAAe;IAClB,MAAM,MAAM,SAAS,GAAG,KAAK;IAC7B,MAAM,WAAW,KAAKA,OAAO,IAAI,GAAG;IACpC,IAAI,UAAU;KACZ,OAAO,SAAS,QAAQ,GAAG;KAC3B,IAAI,OAAO,KAAK,SAAS,OAAO,EAAE,WAAW,GAC3C,KAAKA,OAAO,OAAO,GAAG;IAE1B;IACA;GACF;GACA,KAAK,qBAGH,MAAM,IAAI,MAAM,cAAc,GAAG,GAAG,IAAI;GAC1C,SACE,YAAY,EAAE;EAClB;CAEJ;CAEA,SAAS;EAEP,KAAKE,cAAc,MAAM,KAAKF,MAAM;CACtC;CAEA,UAAU;EACR,KAAKE,YAAY;CACnB;CAEA,YAAY;EACV,IAAI,KAAKA,WAAW;GAClB,KAAKF,SAAS,KAAKE;GACnB,KAAKA,YAAY;EACnB;CACF;AACF"}
+{"version":3,"file":"backfill-state.js","names":[],"sources":["../../../../../../zero-cache/src/services/change-streamer/backfill-state.ts"],"sourcesContent":["import {unreachable} from '../../../../shared/src/asserts.ts';\nimport {BigIntJSON} from '../../../../shared/src/bigint-json.ts';\nimport {deepEqual} from '../../../../shared/src/json.ts';\nimport {\n  acceptBackfill,\n  covers,\n} from '../change-source/protocol/backfill-progress.ts';\nimport type {\n  BackfillRequest,\n  ChangeStreamData,\n  Identifier,\n  SchemaChange,\n  TableMetadata,\n} from '../change-source/protocol/current.ts';\nimport {schemaChangeTags} from '../change-source/protocol/current/schema-change-tags.ts';\nimport {cookieOps} from '../replicator/change-log-cookies.ts';\nimport type {ChangeTag, WatermarkedChange} from './change-streamer.ts';\n\nfunction tableKey({schema, name}: Identifier) {\n  return JSON.stringify([schema, name]);\n}\n\ntype Column = BackfillRequest['columns'][string];\n\n/** A mutable BackfillRequest. */\ntype Backfill = {\n  table: {schema: string; name: string; metadata: TableMetadata | null};\n  columns: Record<string, Column>;\n};\n\ntype State = Map<string, Backfill>;\n\nfunction clone(state: State): State {\n  return new Map(\n    Array.from(state, ([key, {table, columns}]) => [\n      key,\n      {table: {...table}, columns: {...columns}},\n    ]),\n  );\n}\n\n/**\n * Tags of messages that affect the backfill state. Other messages need not be\n * parsed.\n */\nconst RELEVANT_TAGS = new Set<ChangeTag>([\n  'backfill',\n  ...schemaChangeTags.filter(t => t !== 'create-index' && t !== 'drop-index'),\n]);\n\n// Serialized changes are typically forwarded (as the same string) to every\n// subscriber, so a single entry cache avoids re-parsing the same message for\n// each subscriber.\nlet lastParsed: {json: string; change: ChangeStreamData} | undefined;\n\nfunction parse(json: string): ChangeStreamData {\n  if (lastParsed?.json !== json) {\n    lastParsed = {json, change: BigIntJSON.parse(json) as ChangeStreamData};\n  }\n  return lastParsed.change;\n}\n\n/**\n * Tracks the state of the pending backfills (and their progress) of a party\n * in the replication stream (e.g. a subscriber, or the change-log itself), by\n * applying the same policies that a subscriber applies to its replica:\n * schema changes are folded with the {@link cookieOps} shared by the\n * cookie jars, and `backfill` / `backfill-completed` messages are accepted\n * or ignored with the continuity policy of {@link acceptBackfill}.\n *\n * Changes are applied transactionally: the changes of a transaction are\n * discarded if it is rolled back, and {@link requests()} only reflects\n * committed transactions.\n */\nexport class BackfillState {\n  #state: State;\n  #beforeTx: State | null = null;\n\n  constructor(requests: readonly BackfillRequest[] = []) {\n    this.#state = new Map(\n      requests\n        .filter(req => Object.keys(req.columns).length > 0)\n        .map(({table, columns}) => [\n          tableKey(table),\n          {table: {...table}, columns: {...columns}},\n        ]),\n    );\n  }\n\n  get #committed(): State {\n    return this.#beforeTx ?? this.#state;\n  }\n\n  /** The (committed) pending backfills. */\n  requests(): BackfillRequest[] {\n    return [...clone(this.#committed).values()];\n  }\n\n  /**\n   * Returns the `requests` with the (committed) progress of this state\n   * applied to the columns that are pending in both, i.e. same table, column,\n   * and backfill ID. Other columns have no progress (i.e. from scratch).\n   */\n  withProgress(requests: readonly BackfillRequest[]): BackfillRequest[] {\n    const state = this.#committed;\n    return requests.map(({table, columns}) => {\n      const tracked = state.get(tableKey(table))?.columns;\n      return {\n        table,\n        columns: Object.fromEntries(\n          Object.entries(columns).map(([name, {id}]) => {\n            const col = tracked?.[name];\n            return [\n              name,\n              col?.progress && deepEqual(col.id, id)\n                ? {id, progress: col.progress}\n                : {id},\n            ];\n          }),\n        ),\n      };\n    });\n  }\n\n  /** Whether there are no (committed) pending backfills. */\n  get empty(): boolean {\n    return this.#committed.size === 0;\n  }\n\n  /**\n   * Returns the columns (as `schema.table.column` strings) with (committed)\n   * pending backfills that are not covered by the (committed) progress of the\n   * `stream`, i.e. columns for which the `stream` will not deliver all of the\n   * necessary data.\n   */\n  uncovered(stream: BackfillState): string[] {\n    const uncovered: string[] = [];\n    const streamState = stream.#committed;\n    for (const [key, {table, columns}] of this.#committed) {\n      const streamColumns = streamState.get(key)?.columns;\n      for (const [name, {id, progress}] of Object.entries(columns)) {\n        const s = streamColumns?.[name];\n        if (!s || !deepEqual(s.id, id) || !covers(s.progress, progress)) {\n          uncovered.push(`${table.schema}.${table.name}.${name}`);\n        }\n      }\n    }\n    return uncovered;\n  }\n\n  /**\n   * Applies a serialized change, parsing it only if it is relevant.\n   *\n   * @returns The columns (as `schema.table.column` strings) of a backfill\n   *          message that were ignored due to the continuity policy.\n   */\n  applySerialized([, tag, json]: WatermarkedChange): string[] {\n    switch (tag) {\n      case 'begin':\n        this.#begin();\n        return [];\n      case 'commit':\n        this.#commit();\n        return [];\n      case 'rollback':\n        this.#rollback();\n        return [];\n    }\n    if (!RELEVANT_TAGS.has(tag)) {\n      return [];\n    }\n    return this.apply(parse(json));\n  }\n\n  /**\n   * Applies a change.\n   *\n   * @returns The columns (as `schema.table.column` strings) of a backfill\n   *          message that were ignored due to the continuity policy.\n   */\n  apply(change: ChangeStreamData): string[] {\n    switch (change[0]) {\n      case 'begin':\n        this.#begin();\n        return [];\n      case 'commit':\n        this.#commit();\n        return [];\n      case 'rollback':\n        this.#rollback();\n        return [];\n    }\n    const msg = change[1];\n    switch (msg.tag) {\n      case 'backfill': {\n        const {relation, columns, progressMarks} = msg;\n        return this.#applyBackfill(relation, columns, (progress, col) => {\n          const result = acceptBackfill(progress, progressMarks);\n          if (result.accept) {\n            col.progress = result.progress;\n            if (col.progress === undefined) {\n              delete col.progress;\n            }\n          }\n          return result.accept;\n        });\n      }\n      case 'backfill-completed': {\n        const {relation, columns, progressMarks} = msg;\n        const backfill = this.#state.get(tableKey(relation));\n        const ignored = this.#applyBackfill(\n          relation,\n          columns,\n          (progress, _, name) => {\n            const {accept} = acceptBackfill(\n              progress,\n              progressMarks && {previous: progressMarks.previous},\n            );\n            if (accept && backfill) {\n              delete backfill.columns[name];\n            }\n            return accept;\n          },\n        );\n        if (backfill && Object.keys(backfill.columns).length === 0) {\n          this.#state.delete(tableKey(relation));\n        }\n        return ignored;\n      }\n      case 'create-table':\n      case 'rename-table':\n      case 'update-table-metadata':\n      case 'add-column':\n      case 'update-column':\n      case 'drop-column':\n      case 'drop-table':\n        this.#applySchemaChange(msg);\n        return [];\n    }\n    return [];\n  }\n\n  #applyBackfill(\n    relation: {schema: string; name: string; rowKey: {columns: string[]}},\n    columns: readonly string[],\n    apply: (progress: Column['progress'], col: Column, name: string) => boolean,\n  ): string[] {\n    const backfill = this.#state.get(tableKey(relation));\n    if (!backfill) {\n      return [];\n    }\n    const ignored: string[] = [];\n    // rowKey columns are excluded from `columns` but are backfilled with them\n    // (e.g. for a new table).\n    for (const name of [...relation.rowKey.columns, ...columns]) {\n      const col = backfill.columns[name];\n      if (!col) {\n        continue;\n      }\n      // Copy-on-write, since the column object may be shared with the\n      // pre-transaction state.\n      const updated = {...col};\n      if (apply(col.progress, updated, name)) {\n        if (name in backfill.columns) {\n          backfill.columns[name] = updated;\n        }\n      } else {\n        ignored.push(`${relation.schema}.${relation.name}.${name}`);\n      }\n    }\n    return ignored;\n  }\n\n  #applySchemaChange(change: SchemaChange) {\n    // Table metadata specified in the same change (e.g. a `create-table` or\n    // `add-column`), for backfills of tables that are not yet tracked.\n    const metadata = new Map<string, TableMetadata>();\n\n    for (const op of cookieOps(change)) {\n      switch (op.op) {\n        case 'upsert-metadata': {\n          const key = tableKey(op.table);\n          metadata.set(key, op.metadata);\n          const backfill = this.#state.get(key);\n          if (backfill) {\n            backfill.table.metadata = op.metadata;\n          }\n          break;\n        }\n        case 'upsert-backfill': {\n          const key = tableKey(op.table);\n          let backfill = this.#state.get(key);\n          if (!backfill) {\n            backfill = {\n              table: {\n                schema: op.table.schema,\n                name: op.table.name,\n                metadata: metadata.get(key) ?? null,\n              },\n              columns: {},\n            };\n            this.#state.set(key, backfill);\n          }\n          // A new backfill always starts from scratch.\n          backfill.columns[op.column] = {id: op.backfill};\n          break;\n        }\n        case 'rename-table': {\n          const backfill = this.#state.get(tableKey(op.old));\n          if (backfill) {\n            this.#state.delete(tableKey(op.old));\n            backfill.table.schema = op.new.schema;\n            backfill.table.name = op.new.name;\n            this.#state.set(tableKey(op.new), backfill);\n          }\n          break;\n        }\n        case 'drop-table':\n          this.#state.delete(tableKey(op.table));\n          break;\n        case 'rename-column': {\n          const backfill = this.#state.get(tableKey(op.table));\n          const col = backfill?.columns[op.old];\n          if (backfill && col) {\n            delete backfill.columns[op.old];\n            backfill.columns[op.new] = col;\n          }\n          break;\n        }\n        case 'drop-column': {\n          const key = tableKey(op.table);\n          const backfill = this.#state.get(key);\n          if (backfill) {\n            delete backfill.columns[op.column];\n            if (Object.keys(backfill.columns).length === 0) {\n              this.#state.delete(key);\n            }\n          }\n          break;\n        }\n        case 'complete-backfill':\n          // Only produced for `backfill-completed`, which is handled\n          // separately (with the continuity policy).\n          throw new Error(`unexpected ${op.op} op`);\n        default:\n          unreachable(op);\n      }\n    }\n  }\n\n  #begin() {\n    // Tolerate a missing commit/rollback, e.g. from an interrupted stream.\n    this.#beforeTx ??= clone(this.#state);\n  }\n\n  #commit() {\n    this.#beforeTx = null;\n  }\n\n  #rollback() {\n    if (this.#beforeTx) {\n      this.#state = this.#beforeTx;\n      this.#beforeTx = null;\n    }\n  }\n}\n"],"mappings":";;;;;;;AAkBA,SAAS,SAAS,EAAC,QAAQ,QAAmB;CAC5C,OAAO,KAAK,UAAU,CAAC,QAAQ,IAAI,CAAC;AACtC;AAYA,SAAS,MAAM,OAAqB;CAClC,OAAO,IAAI,IACT,MAAM,KAAK,QAAQ,CAAC,KAAK,EAAC,OAAO,eAAc,CAC7C,KACA;EAAC,OAAO,EAAC,GAAG,MAAK;EAAG,SAAS,EAAC,GAAG,QAAO;CAAC,CAC3C,CAAC,CACH;AACF;;;;;AAMA,IAAM,gCAAgB,IAAI,IAAe,CACvC,YACA,GAAG,iBAAiB,QAAO,MAAK,MAAM,kBAAkB,MAAM,YAAY,CAC5E,CAAC;AAKD,IAAI;AAEJ,SAAS,MAAM,MAAgC;CAC7C,IAAI,YAAY,SAAS,MACvB,aAAa;EAAC;EAAM,QAAQ,WAAW,MAAM,IAAI;CAAqB;CAExE,OAAO,WAAW;AACpB;;;;;;;;;;;;;AAcA,IAAa,gBAAb,MAA2B;CACzB;CACA,YAA0B;CAE1B,YAAY,WAAuC,CAAC,GAAG;EACrD,KAAK,SAAS,IAAI,IAChB,SACG,QAAO,QAAO,OAAO,KAAK,IAAI,OAAO,CAAC,CAAC,SAAS,CAAC,CAAC,CAClD,KAAK,EAAC,OAAO,cAAa,CACzB,SAAS,KAAK,GACd;GAAC,OAAO,EAAC,GAAG,MAAK;GAAG,SAAS,EAAC,GAAG,QAAO;EAAC,CAC3C,CAAC,CACL;CACF;CAEA,IAAI,aAAoB;EACtB,OAAO,KAAK,aAAa,KAAK;CAChC;;CAGA,WAA8B;EAC5B,OAAO,CAAC,GAAG,MAAM,KAAK,UAAU,CAAC,CAAC,OAAO,CAAC;CAC5C;;;;;;CAOA,aAAa,UAAyD;EACpE,MAAM,QAAQ,KAAK;EACnB,OAAO,SAAS,KAAK,EAAC,OAAO,cAAa;GACxC,MAAM,UAAU,MAAM,IAAI,SAAS,KAAK,CAAC,CAAC,EAAE;GAC5C,OAAO;IACL;IACA,SAAS,OAAO,YACd,OAAO,QAAQ,OAAO,CAAC,CAAC,KAAK,CAAC,MAAM,EAAC,UAAS;KAC5C,MAAM,MAAM,UAAU;KACtB,OAAO,CACL,MACA,KAAK,YAAY,UAAU,IAAI,IAAI,EAAE,IACjC;MAAC;MAAI,UAAU,IAAI;KAAQ,IAC3B,EAAC,GAAE,CACT;IACF,CAAC,CACH;GACF;EACF,CAAC;CACH;;CAGA,IAAI,QAAiB;EACnB,OAAO,KAAK,WAAW,SAAS;CAClC;;;;;;;CAQA,UAAU,QAAiC;EACzC,MAAM,YAAsB,CAAC;EAC7B,MAAM,cAAc,OAAO;EAC3B,KAAK,MAAM,CAAC,KAAK,EAAC,OAAO,cAAa,KAAK,YAAY;GACrD,MAAM,gBAAgB,YAAY,IAAI,GAAG,CAAC,EAAE;GAC5C,KAAK,MAAM,CAAC,MAAM,EAAC,IAAI,eAAc,OAAO,QAAQ,OAAO,GAAG;IAC5D,MAAM,IAAI,gBAAgB;IAC1B,IAAI,CAAC,KAAK,CAAC,UAAU,EAAE,IAAI,EAAE,KAAK,CAAC,OAAO,EAAE,UAAU,QAAQ,GAC5D,UAAU,KAAK,GAAG,MAAM,OAAO,GAAG,MAAM,KAAK,GAAG,MAAM;GAE1D;EACF;EACA,OAAO;CACT;;;;;;;CAQA,gBAAgB,GAAG,KAAK,OAAoC;EAC1D,QAAQ,KAAR;GACE,KAAK;IACH,KAAK,OAAO;IACZ,OAAO,CAAC;GACV,KAAK;IACH,KAAK,QAAQ;IACb,OAAO,CAAC;GACV,KAAK;IACH,KAAK,UAAU;IACf,OAAO,CAAC;EACZ;EACA,IAAI,CAAC,cAAc,IAAI,GAAG,GACxB,OAAO,CAAC;EAEV,OAAO,KAAK,MAAM,MAAM,IAAI,CAAC;CAC/B;;;;;;;CAQA,MAAM,QAAoC;EACxC,QAAQ,OAAO,IAAf;GACE,KAAK;IACH,KAAK,OAAO;IACZ,OAAO,CAAC;GACV,KAAK;IACH,KAAK,QAAQ;IACb,OAAO,CAAC;GACV,KAAK;IACH,KAAK,UAAU;IACf,OAAO,CAAC;EACZ;EACA,MAAM,MAAM,OAAO;EACnB,QAAQ,IAAI,KAAZ;GACE,KAAK,YAAY;IACf,MAAM,EAAC,UAAU,SAAS,kBAAiB;IAC3C,OAAO,KAAK,eAAe,UAAU,UAAU,UAAU,QAAQ;KAC/D,MAAM,SAAS,eAAe,UAAU,aAAa;KACrD,IAAI,OAAO,QAAQ;MACjB,IAAI,WAAW,OAAO;MACtB,IAAI,IAAI,aAAa,KAAA,GACnB,OAAO,IAAI;KAEf;KACA,OAAO,OAAO;IAChB,CAAC;GACH;GACA,KAAK,sBAAsB;IACzB,MAAM,EAAC,UAAU,SAAS,kBAAiB;IAC3C,MAAM,WAAW,KAAK,OAAO,IAAI,SAAS,QAAQ,CAAC;IACnD,MAAM,UAAU,KAAK,eACnB,UACA,UACC,UAAU,GAAG,SAAS;KACrB,MAAM,EAAC,WAAU,eACf,UACA,iBAAiB,EAAC,UAAU,cAAc,SAAQ,CACpD;KACA,IAAI,UAAU,UACZ,OAAO,SAAS,QAAQ;KAE1B,OAAO;IACT,CACF;IACA,IAAI,YAAY,OAAO,KAAK,SAAS,OAAO,CAAC,CAAC,WAAW,GACvD,KAAK,OAAO,OAAO,SAAS,QAAQ,CAAC;IAEvC,OAAO;GACT;GACA,KAAK;GACL,KAAK;GACL,KAAK;GACL,KAAK;GACL,KAAK;GACL,KAAK;GACL,KAAK;IACH,KAAK,mBAAmB,GAAG;IAC3B,OAAO,CAAC;EACZ;EACA,OAAO,CAAC;CACV;CAEA,eACE,UACA,SACA,OACU;EACV,MAAM,WAAW,KAAK,OAAO,IAAI,SAAS,QAAQ,CAAC;EACnD,IAAI,CAAC,UACH,OAAO,CAAC;EAEV,MAAM,UAAoB,CAAC;EAG3B,KAAK,MAAM,QAAQ,CAAC,GAAG,SAAS,OAAO,SAAS,GAAG,OAAO,GAAG;GAC3D,MAAM,MAAM,SAAS,QAAQ;GAC7B,IAAI,CAAC,KACH;GAIF,MAAM,UAAU,EAAC,GAAG,IAAG;GACvB,IAAI,MAAM,IAAI,UAAU,SAAS,IAAI,GAC/B;QAAA,QAAQ,SAAS,SACnB,SAAS,QAAQ,QAAQ;GAAA,OAG3B,QAAQ,KAAK,GAAG,SAAS,OAAO,GAAG,SAAS,KAAK,GAAG,MAAM;EAE9D;EACA,OAAO;CACT;CAEA,mBAAmB,QAAsB;EAGvC,MAAM,2BAAW,IAAI,IAA2B;EAEhD,KAAK,MAAM,MAAM,UAAU,MAAM,GAC/B,QAAQ,GAAG,IAAX;GACE,KAAK,mBAAmB;IACtB,MAAM,MAAM,SAAS,GAAG,KAAK;IAC7B,SAAS,IAAI,KAAK,GAAG,QAAQ;IAC7B,MAAM,WAAW,KAAK,OAAO,IAAI,GAAG;IACpC,IAAI,UACF,SAAS,MAAM,WAAW,GAAG;IAE/B;GACF;GACA,KAAK,mBAAmB;IACtB,MAAM,MAAM,SAAS,GAAG,KAAK;IAC7B,IAAI,WAAW,KAAK,OAAO,IAAI,GAAG;IAClC,IAAI,CAAC,UAAU;KACb,WAAW;MACT,OAAO;OACL,QAAQ,GAAG,MAAM;OACjB,MAAM,GAAG,MAAM;OACf,UAAU,SAAS,IAAI,GAAG,KAAK;MACjC;MACA,SAAS,CAAC;KACZ;KACA,KAAK,OAAO,IAAI,KAAK,QAAQ;IAC/B;IAEA,SAAS,QAAQ,GAAG,UAAU,EAAC,IAAI,GAAG,SAAQ;IAC9C;GACF;GACA,KAAK,gBAAgB;IACnB,MAAM,WAAW,KAAK,OAAO,IAAI,SAAS,GAAG,GAAG,CAAC;IACjD,IAAI,UAAU;KACZ,KAAK,OAAO,OAAO,SAAS,GAAG,GAAG,CAAC;KACnC,SAAS,MAAM,SAAS,GAAG,IAAI;KAC/B,SAAS,MAAM,OAAO,GAAG,IAAI;KAC7B,KAAK,OAAO,IAAI,SAAS,GAAG,GAAG,GAAG,QAAQ;IAC5C;IACA;GACF;GACA,KAAK;IACH,KAAK,OAAO,OAAO,SAAS,GAAG,KAAK,CAAC;IACrC;GACF,KAAK,iBAAiB;IACpB,MAAM,WAAW,KAAK,OAAO,IAAI,SAAS,GAAG,KAAK,CAAC;IACnD,MAAM,MAAM,UAAU,QAAQ,GAAG;IACjC,IAAI,YAAY,KAAK;KACnB,OAAO,SAAS,QAAQ,GAAG;KAC3B,SAAS,QAAQ,GAAG,OAAO;IAC7B;IACA;GACF;GACA,KAAK,eAAe;IAClB,MAAM,MAAM,SAAS,GAAG,KAAK;IAC7B,MAAM,WAAW,KAAK,OAAO,IAAI,GAAG;IACpC,IAAI,UAAU;KACZ,OAAO,SAAS,QAAQ,GAAG;KAC3B,IAAI,OAAO,KAAK,SAAS,OAAO,CAAC,CAAC,WAAW,GAC3C,KAAK,OAAO,OAAO,GAAG;IAE1B;IACA;GACF;GACA,KAAK,qBAGH,MAAM,IAAI,MAAM,cAAc,GAAG,GAAG,IAAI;GAC1C,SACE,YAAY,EAAE;EAClB;CAEJ;CAEA,SAAS;EAEP,KAAK,cAAc,MAAM,KAAK,MAAM;CACtC;CAEA,UAAU;EACR,KAAK,YAAY;CACnB;CAEA,YAAY;EACV,IAAI,KAAK,WAAW;GAClB,KAAK,SAAS,KAAK;GACnB,KAAK,YAAY;EACnB;CACF;AACF"}

package/out/zero-cache/src/services/change-streamer/backup-cleanup-monitor-factory.js.map

@@ -1 +1 @@
-{"version":3,"file":"backup-cleanup-monitor-factory.js","names":[],"sources":["../../../../../../zero-cache/src/services/change-streamer/backup-cleanup-monitor-factory.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport type {NormalizedZeroConfig} from '../../config/normalize.ts';\nimport type {Source} from '../../types/streams.ts';\nimport {\n  backupDestinationURL,\n  getLastBackupTime,\n} from '../litestream/commands.ts';\nimport {type BackedUpWatermark, BackupMonitor} from './backup-monitor.ts';\nimport type {ChangeStreamerService} from './change-streamer.ts';\nimport {LitestreamSyncRequester} from './litestream-sync-requester.ts';\nimport {\n  type BackupStateVerifier,\n  Litestream3PrometheusPoller,\n} from './litestream3-prometheus-poller.ts';\nimport {ReplicaPoller} from './replica-poller.ts';\n\nexport type BackupCleanupMonitorFactoryOptions = {\n  lc: LogContext;\n  config: NormalizedZeroConfig;\n  replicaFile: string;\n  changeStreamer: ChangeStreamerService;\n  verifyBackupState?: BackupStateVerifier | undefined;\n};\n\nexport function createBackupCleanupMonitor({\n  lc,\n  config,\n  replicaFile,\n  changeStreamer,\n  verifyBackupState,\n}: BackupCleanupMonitorFactoryOptions): BackupMonitor {\n  const {litestream, replica} = config;\n  const backupURL = backupDestinationURL(litestream);\n\n  let stream: Source<BackedUpWatermark>;\n\n  if (!backupURL) {\n    stream = new ReplicaPoller(lc, replicaFile).start();\n  } else if (config.litestream.backupUsingV5) {\n    stream = new LitestreamSyncRequester(lc, replicaFile, {\n      intervalMs: litestream.incrementalBackupIntervalSeconds * 1000,\n    }).start();\n  } else {\n    const {port: metricsPort} = litestream;\n    stream = new Litestream3PrometheusPoller(\n      lc,\n      replicaFile,\n      backupURL,\n      `http://localhost:${metricsPort}/metrics`,\n      verifyBackupState ??\n        (() => getLastBackupTime(lc, litestream, replica.file)),\n    ).start();\n  }\n\n  return new BackupMonitor(lc, stream, changeStreamer, replicaFile);\n}\n"],"mappings":";;;;;;AAwBA,SAAgB,2BAA2B,EACzC,IACA,QACA,aACA,gBACA,qBACoD;CACpD,MAAM,EAAC,YAAY,YAAW;CAC9B,MAAM,YAAY,qBAAqB,UAAU;CAEjD,IAAI;CAEJ,IAAI,CAAC,WACH,SAAS,IAAI,cAAc,IAAI,WAAW,EAAE,MAAM;MAC7C,IAAI,OAAO,WAAW,eAC3B,SAAS,IAAI,wBAAwB,IAAI,aAAa,EACpD,YAAY,WAAW,mCAAmC,IAC5D,CAAC,EAAE,MAAM;MACJ;EACL,MAAM,EAAC,MAAM,gBAAe;EAC5B,SAAS,IAAI,4BACX,IACA,aACA,WACA,oBAAoB,YAAY,WAChC,4BACS,kBAAkB,IAAI,YAAY,QAAQ,IAAI,EACzD,EAAE,MAAM;CACV;CAEA,OAAO,IAAI,cAAc,IAAI,QAAQ,gBAAgB,WAAW;AAClE"}
+{"version":3,"file":"backup-cleanup-monitor-factory.js","names":[],"sources":["../../../../../../zero-cache/src/services/change-streamer/backup-cleanup-monitor-factory.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport type {NormalizedZeroConfig} from '../../config/normalize.ts';\nimport type {Source} from '../../types/streams.ts';\nimport {\n  backupDestinationURL,\n  getLastBackupTime,\n} from '../litestream/commands.ts';\nimport {type BackedUpWatermark, BackupMonitor} from './backup-monitor.ts';\nimport type {ChangeStreamerService} from './change-streamer.ts';\nimport {LitestreamSyncRequester} from './litestream-sync-requester.ts';\nimport {\n  type BackupStateVerifier,\n  Litestream3PrometheusPoller,\n} from './litestream3-prometheus-poller.ts';\nimport {ReplicaPoller} from './replica-poller.ts';\n\nexport type BackupCleanupMonitorFactoryOptions = {\n  lc: LogContext;\n  config: NormalizedZeroConfig;\n  replicaFile: string;\n  changeStreamer: ChangeStreamerService;\n  verifyBackupState?: BackupStateVerifier | undefined;\n};\n\nexport function createBackupCleanupMonitor({\n  lc,\n  config,\n  replicaFile,\n  changeStreamer,\n  verifyBackupState,\n}: BackupCleanupMonitorFactoryOptions): BackupMonitor {\n  const {litestream, replica} = config;\n  const backupURL = backupDestinationURL(litestream);\n\n  let stream: Source<BackedUpWatermark>;\n\n  if (!backupURL) {\n    stream = new ReplicaPoller(lc, replicaFile).start();\n  } else if (config.litestream.backupUsingV5) {\n    stream = new LitestreamSyncRequester(lc, replicaFile, {\n      intervalMs: litestream.incrementalBackupIntervalSeconds * 1000,\n    }).start();\n  } else {\n    const {port: metricsPort} = litestream;\n    stream = new Litestream3PrometheusPoller(\n      lc,\n      replicaFile,\n      backupURL,\n      `http://localhost:${metricsPort}/metrics`,\n      verifyBackupState ??\n        (() => getLastBackupTime(lc, litestream, replica.file)),\n    ).start();\n  }\n\n  return new BackupMonitor(lc, stream, changeStreamer, replicaFile);\n}\n"],"mappings":";;;;;;AAwBA,SAAgB,2BAA2B,EACzC,IACA,QACA,aACA,gBACA,qBACoD;CACpD,MAAM,EAAC,YAAY,YAAW;CAC9B,MAAM,YAAY,qBAAqB,UAAU;CAEjD,IAAI;CAEJ,IAAI,CAAC,WACH,SAAS,IAAI,cAAc,IAAI,WAAW,CAAC,CAAC,MAAM;MAC7C,IAAI,OAAO,WAAW,eAC3B,SAAS,IAAI,wBAAwB,IAAI,aAAa,EACpD,YAAY,WAAW,mCAAmC,IAC5D,CAAC,CAAC,CAAC,MAAM;MACJ;EACL,MAAM,EAAC,MAAM,gBAAe;EAC5B,SAAS,IAAI,4BACX,IACA,aACA,WACA,oBAAoB,YAAY,WAChC,4BACS,kBAAkB,IAAI,YAAY,QAAQ,IAAI,EACzD,CAAC,CAAC,MAAM;CACV;CAEA,OAAO,IAAI,cAAc,IAAI,QAAQ,gBAAgB,WAAW;AAClE"}

package/out/zero-cache/src/services/change-streamer/backup-monitor.js.map

@@ -1 +1 @@
-{"version":3,"file":"backup-monitor.js","names":["#lc","#watermarks","#changeStreamer","#replicaFile","#firstBackupReceived","#initBackupLagMetric","#coverageTarget","#readReplicaStateVersion","#latestBackup","#checkFirstBackupCovers","#firstBackupResolved"],"sources":["../../../../../../zero-cache/src/services/change-streamer/backup-monitor.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {resolver} from '@rocicorp/resolver';\nimport {promiseVoid} from '../../../../shared/src/resolved-promises.ts';\nimport {Database} from '../../../../zqlite/src/db.ts';\nimport {getOrCreateGauge} from '../../observability/metrics.ts';\nimport type {Source} from '../../types/streams.ts';\nimport type {SingletonService} from '../service.ts';\nimport type {ChangeStreamerService} from './change-streamer.ts';\n\nexport type BackedUpWatermark = {\n  watermark: string;\n  writeTimeMs?: number | undefined; // optional for debugging; available on v5\n  backupTimeMs: number;\n};\n\nexport class BackupMonitor implements SingletonService {\n  readonly id = 'backup-monitor';\n  readonly #lc: LogContext;\n  readonly #watermarks: Source<BackedUpWatermark>;\n  readonly #changeStreamer: ChangeStreamerService;\n  readonly #replicaFile: string;\n  readonly #firstBackupReceived = resolver();\n\n  #latestBackup: BackedUpWatermark | undefined;\n  #firstBackupResolved = false;\n  /**\n   * The replica's `stateVersion` when the monitor started, i.e. the point the\n   * backup has to reach before it covers what this task is about to serve.\n   * `undefined` when the replica could not be read, which degrades the gate to\n   * \"any watermark\" rather than hanging startup forever.\n   */\n  #coverageTarget: string | undefined;\n\n  constructor(\n    lc: LogContext,\n    watermarks: Source<BackedUpWatermark>,\n    changeStreamer: ChangeStreamerService,\n    replicaFile: string,\n  ) {\n    this.#lc = lc;\n    this.#watermarks = watermarks;\n    this.#changeStreamer = changeStreamer;\n    this.#replicaFile = replicaFile;\n  }\n\n  firstBackupReceived() {\n    return this.#firstBackupReceived.promise;\n  }\n\n  async run() {\n    this.#lc.info?.('starting backup monitor');\n    this.#initBackupLagMetric();\n    this.#coverageTarget = this.#readReplicaStateVersion();\n\n    for await (const backedUp of this.#watermarks) {\n      if (this.#latestBackup) {\n        if (backedUp.watermark < this.#latestBackup.watermark) {\n          this.#lc.warn?.(\n            `ignoring earlier backup watermark ${backedUp.watermark}`,\n            {backedUp},\n          );\n          continue;\n        }\n        if (backedUp.watermark === this.#latestBackup.watermark) {\n          this.#lc.debug?.(\n            `ignoring redundant backup watermark ${backedUp.watermark}`,\n            {backedUp},\n          );\n          continue;\n        }\n      }\n      this.#lc.info?.(`received backup watermark ${backedUp.watermark}`, {\n        backedUp,\n      });\n      this.#latestBackup = backedUp;\n      this.#changeStreamer.trackBackupWatermark(backedUp.watermark);\n      this.#checkFirstBackupCovers(backedUp);\n    }\n    this.#lc.info?.('watermark stream closed. BackupMonitor stopped.');\n  }\n\n  /**\n   * Resolves {@link firstBackupReceived} once the backup actually covers the\n   * replica, rather than on the first watermark of any value.\n   *\n   * A new backup destination starts empty and is filled by re-uploading the\n   * local LTX chain, so the first watermarks read back out of it are real but\n   * far behind the replica. Releasing the readiness gate on one of those lets\n   * the task serve against a backup that does not yet cover it, which is what\n   * demotes a restoring view-syncer to Postgres catchup.\n   *\n   * The target is pinned at startup rather than compared against the replica's\n   * current position, which keeps the gate satisfiable under sustained writes.\n   */\n  #checkFirstBackupCovers(backedUp: BackedUpWatermark) {\n    if (this.#firstBackupResolved) {\n      return;\n    }\n    const target = this.#coverageTarget;\n    if (target !== undefined && backedUp.watermark < target) {\n      this.#lc.info?.(`backup does not yet cover the replica`, {\n        backedUp,\n        coverageTarget: target,\n      });\n      return;\n    }\n    this.#firstBackupResolved = true;\n    this.#firstBackupReceived.resolve();\n  }\n\n  #readReplicaStateVersion(): string | undefined {\n    let db;\n    try {\n      db = new Database(this.#lc, this.#replicaFile, {readonly: true});\n      const {stateVersion} = db\n        .prepare(/*sql*/ `SELECT stateVersion FROM \"_zero.replicationState\"`)\n        .get<{stateVersion: string}>();\n      return stateVersion;\n    } catch (e) {\n      // Without a target the gate degrades to its previous behavior. That is\n      // strictly better than blocking startup on a replica we cannot read.\n      this.#lc.warn?.(\n        `unable to read the replica's stateVersion; ` +\n          `the initial backup gate will accept the first watermark`,\n        e,\n      );\n      return undefined;\n    } finally {\n      db?.close();\n    }\n  }\n\n  stop(): Promise<void> {\n    this.#watermarks.cancel();\n    return promiseVoid;\n  }\n\n  #initBackupLagMetric() {\n    getOrCreateGauge('replica', 'backup_lag', {\n      description:\n        'Latency from when a change is written to the replica ' +\n        'to when it is backed up to litestream. It is expected to create a saw ' +\n        'pattern from 0 to the configured ZERO_LITESTREAM_INCREMENTAL_BACKUP_INTERVAL_MINUTES.',\n      unit: 'millisecond',\n    }).addCallback(o => {\n      const latestBackup = this.#latestBackup;\n      if (!latestBackup) {\n        this.#lc.warn?.(\n          `no backed up watermarks. unable to report replica.backup_lag`,\n        );\n        return;\n      }\n      const db = new Database(this.#lc, this.#replicaFile, {readonly: true});\n      try {\n        const {writeTimeMs} = db\n          .prepare(/*sql*/ `SELECT writeTimeMs FROM \"_zero.replicationState\"`)\n          .get<{writeTimeMs: number}>();\n        const backupLag = Math.max(0, writeTimeMs - latestBackup.backupTimeMs);\n        o.observe(backupLag);\n      } catch (e) {\n        this.#lc.warn?.(`error measuring replica.backup_lag metric`, e);\n      } finally {\n        db.close();\n      }\n    });\n  }\n}\n"],"mappings":";;;;;AAeA,IAAa,gBAAb,MAAuD;CACrD,KAAc;CACd;CACA;CACA;CACA;CACA,uBAAgC,SAAS;CAEzC;CACA,uBAAuB;;;;;;;CAOvB;CAEA,YACE,IACA,YACA,gBACA,aACA;EACA,KAAKA,MAAM;EACX,KAAKC,cAAc;EACnB,KAAKC,kBAAkB;EACvB,KAAKC,eAAe;CACtB;CAEA,sBAAsB;EACpB,OAAO,KAAKC,qBAAqB;CACnC;CAEA,MAAM,MAAM;EACV,KAAKJ,IAAI,OAAO,yBAAyB;EACzC,KAAKK,qBAAqB;EAC1B,KAAKC,kBAAkB,KAAKC,yBAAyB;EAErD,WAAW,MAAM,YAAY,KAAKN,aAAa;GAC7C,IAAI,KAAKO,eAAe;IACtB,IAAI,SAAS,YAAY,KAAKA,cAAc,WAAW;KACrD,KAAKR,IAAI,OACP,qCAAqC,SAAS,aAC9C,EAAC,SAAQ,CACX;KACA;IACF;IACA,IAAI,SAAS,cAAc,KAAKQ,cAAc,WAAW;KACvD,KAAKR,IAAI,QACP,uCAAuC,SAAS,aAChD,EAAC,SAAQ,CACX;KACA;IACF;GACF;GACA,KAAKA,IAAI,OAAO,6BAA6B,SAAS,aAAa,EACjE,SACF,CAAC;GACD,KAAKQ,gBAAgB;GACrB,KAAKN,gBAAgB,qBAAqB,SAAS,SAAS;GAC5D,KAAKO,wBAAwB,QAAQ;EACvC;EACA,KAAKT,IAAI,OAAO,iDAAiD;CACnE;;;;;;;;;;;;;;CAeA,wBAAwB,UAA6B;EACnD,IAAI,KAAKU,sBACP;EAEF,MAAM,SAAS,KAAKJ;EACpB,IAAI,WAAW,KAAA,KAAa,SAAS,YAAY,QAAQ;GACvD,KAAKN,IAAI,OAAO,yCAAyC;IACvD;IACA,gBAAgB;GAClB,CAAC;GACD;EACF;EACA,KAAKU,uBAAuB;EAC5B,KAAKN,qBAAqB,QAAQ;CACpC;CAEA,2BAA+C;EAC7C,IAAI;EACJ,IAAI;GACF,KAAK,IAAI,SAAS,KAAKJ,KAAK,KAAKG,cAAc,EAAC,UAAU,KAAI,CAAC;GAC/D,MAAM,EAAC,iBAAgB,GACpB,QAAgB,mDAAmD,EACnE,IAA4B;GAC/B,OAAO;EACT,SAAS,GAAG;GAGV,KAAKH,IAAI,OACP,sGAEA,CACF;GACA;EACF,UAAU;GACR,IAAI,MAAM;EACZ;CACF;CAEA,OAAsB;EACpB,KAAKC,YAAY,OAAO;EACxB,OAAO;CACT;CAEA,uBAAuB;EACrB,iBAAiB,WAAW,cAAc;GACxC,aACE;GAGF,MAAM;EACR,CAAC,EAAE,aAAY,MAAK;GAClB,MAAM,eAAe,KAAKO;GAC1B,IAAI,CAAC,cAAc;IACjB,KAAKR,IAAI,OACP,8DACF;IACA;GACF;GACA,MAAM,KAAK,IAAI,SAAS,KAAKA,KAAK,KAAKG,cAAc,EAAC,UAAU,KAAI,CAAC;GACrE,IAAI;IACF,MAAM,EAAC,gBAAe,GACnB,QAAgB,kDAAkD,EAClE,IAA2B;IAC9B,MAAM,YAAY,KAAK,IAAI,GAAG,cAAc,aAAa,YAAY;IACrE,EAAE,QAAQ,SAAS;GACrB,SAAS,GAAG;IACV,KAAKH,IAAI,OAAO,6CAA6C,CAAC;GAChE,UAAU;IACR,GAAG,MAAM;GACX;EACF,CAAC;CACH;AACF"}
+{"version":3,"file":"backup-monitor.js","names":[],"sources":["../../../../../../zero-cache/src/services/change-streamer/backup-monitor.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {resolver} from '@rocicorp/resolver';\nimport {promiseVoid} from '../../../../shared/src/resolved-promises.ts';\nimport {Database} from '../../../../zqlite/src/db.ts';\nimport {getOrCreateGauge} from '../../observability/metrics.ts';\nimport type {Source} from '../../types/streams.ts';\nimport type {SingletonService} from '../service.ts';\nimport type {ChangeStreamerService} from './change-streamer.ts';\n\nexport type BackedUpWatermark = {\n  watermark: string;\n  writeTimeMs?: number | undefined; // optional for debugging; available on v5\n  backupTimeMs: number;\n};\n\nexport class BackupMonitor implements SingletonService {\n  readonly id = 'backup-monitor';\n  readonly #lc: LogContext;\n  readonly #watermarks: Source<BackedUpWatermark>;\n  readonly #changeStreamer: ChangeStreamerService;\n  readonly #replicaFile: string;\n  readonly #firstBackupReceived = resolver();\n\n  #latestBackup: BackedUpWatermark | undefined;\n  #firstBackupResolved = false;\n  /**\n   * The replica's `stateVersion` when the monitor started, i.e. the point the\n   * backup has to reach before it covers what this task is about to serve.\n   * `undefined` when the replica could not be read, which degrades the gate to\n   * \"any watermark\" rather than hanging startup forever.\n   */\n  #coverageTarget: string | undefined;\n\n  constructor(\n    lc: LogContext,\n    watermarks: Source<BackedUpWatermark>,\n    changeStreamer: ChangeStreamerService,\n    replicaFile: string,\n  ) {\n    this.#lc = lc;\n    this.#watermarks = watermarks;\n    this.#changeStreamer = changeStreamer;\n    this.#replicaFile = replicaFile;\n  }\n\n  firstBackupReceived() {\n    return this.#firstBackupReceived.promise;\n  }\n\n  async run() {\n    this.#lc.info?.('starting backup monitor');\n    this.#initBackupLagMetric();\n    this.#coverageTarget = this.#readReplicaStateVersion();\n\n    for await (const backedUp of this.#watermarks) {\n      if (this.#latestBackup) {\n        if (backedUp.watermark < this.#latestBackup.watermark) {\n          this.#lc.warn?.(\n            `ignoring earlier backup watermark ${backedUp.watermark}`,\n            {backedUp},\n          );\n          continue;\n        }\n        if (backedUp.watermark === this.#latestBackup.watermark) {\n          this.#lc.debug?.(\n            `ignoring redundant backup watermark ${backedUp.watermark}`,\n            {backedUp},\n          );\n          continue;\n        }\n      }\n      this.#lc.info?.(`received backup watermark ${backedUp.watermark}`, {\n        backedUp,\n      });\n      this.#latestBackup = backedUp;\n      this.#changeStreamer.trackBackupWatermark(backedUp.watermark);\n      this.#checkFirstBackupCovers(backedUp);\n    }\n    this.#lc.info?.('watermark stream closed. BackupMonitor stopped.');\n  }\n\n  /**\n   * Resolves {@link firstBackupReceived} once the backup actually covers the\n   * replica, rather than on the first watermark of any value.\n   *\n   * A new backup destination starts empty and is filled by re-uploading the\n   * local LTX chain, so the first watermarks read back out of it are real but\n   * far behind the replica. Releasing the readiness gate on one of those lets\n   * the task serve against a backup that does not yet cover it, which is what\n   * demotes a restoring view-syncer to Postgres catchup.\n   *\n   * The target is pinned at startup rather than compared against the replica's\n   * current position, which keeps the gate satisfiable under sustained writes.\n   */\n  #checkFirstBackupCovers(backedUp: BackedUpWatermark) {\n    if (this.#firstBackupResolved) {\n      return;\n    }\n    const target = this.#coverageTarget;\n    if (target !== undefined && backedUp.watermark < target) {\n      this.#lc.info?.(`backup does not yet cover the replica`, {\n        backedUp,\n        coverageTarget: target,\n      });\n      return;\n    }\n    this.#firstBackupResolved = true;\n    this.#firstBackupReceived.resolve();\n  }\n\n  #readReplicaStateVersion(): string | undefined {\n    let db;\n    try {\n      db = new Database(this.#lc, this.#replicaFile, {readonly: true});\n      const {stateVersion} = db\n        .prepare(/*sql*/ `SELECT stateVersion FROM \"_zero.replicationState\"`)\n        .get<{stateVersion: string}>();\n      return stateVersion;\n    } catch (e) {\n      // Without a target the gate degrades to its previous behavior. That is\n      // strictly better than blocking startup on a replica we cannot read.\n      this.#lc.warn?.(\n        `unable to read the replica's stateVersion; ` +\n          `the initial backup gate will accept the first watermark`,\n        e,\n      );\n      return undefined;\n    } finally {\n      db?.close();\n    }\n  }\n\n  stop(): Promise<void> {\n    this.#watermarks.cancel();\n    return promiseVoid;\n  }\n\n  #initBackupLagMetric() {\n    getOrCreateGauge('replica', 'backup_lag', {\n      description:\n        'Latency from when a change is written to the replica ' +\n        'to when it is backed up to litestream. It is expected to create a saw ' +\n        'pattern from 0 to the configured ZERO_LITESTREAM_INCREMENTAL_BACKUP_INTERVAL_MINUTES.',\n      unit: 'millisecond',\n    }).addCallback(o => {\n      const latestBackup = this.#latestBackup;\n      if (!latestBackup) {\n        this.#lc.warn?.(\n          `no backed up watermarks. unable to report replica.backup_lag`,\n        );\n        return;\n      }\n      const db = new Database(this.#lc, this.#replicaFile, {readonly: true});\n      try {\n        const {writeTimeMs} = db\n          .prepare(/*sql*/ `SELECT writeTimeMs FROM \"_zero.replicationState\"`)\n          .get<{writeTimeMs: number}>();\n        const backupLag = Math.max(0, writeTimeMs - latestBackup.backupTimeMs);\n        o.observe(backupLag);\n      } catch (e) {\n        this.#lc.warn?.(`error measuring replica.backup_lag metric`, e);\n      } finally {\n        db.close();\n      }\n    });\n  }\n}\n"],"mappings":";;;;;AAeA,IAAa,gBAAb,MAAuD;CACrD,KAAc;CACd;CACA;CACA;CACA;CACA,uBAAgC,SAAS;CAEzC;CACA,uBAAuB;;;;;;;CAOvB;CAEA,YACE,IACA,YACA,gBACA,aACA;EACA,KAAK,MAAM;EACX,KAAK,cAAc;EACnB,KAAK,kBAAkB;EACvB,KAAK,eAAe;CACtB;CAEA,sBAAsB;EACpB,OAAO,KAAK,qBAAqB;CACnC;CAEA,MAAM,MAAM;EACV,KAAK,IAAI,OAAO,yBAAyB;EACzC,KAAK,qBAAqB;EAC1B,KAAK,kBAAkB,KAAK,yBAAyB;EAErD,WAAW,MAAM,YAAY,KAAK,aAAa;GAC7C,IAAI,KAAK,eAAe;IACtB,IAAI,SAAS,YAAY,KAAK,cAAc,WAAW;KACrD,KAAK,IAAI,OACP,qCAAqC,SAAS,aAC9C,EAAC,SAAQ,CACX;KACA;IACF;IACA,IAAI,SAAS,cAAc,KAAK,cAAc,WAAW;KACvD,KAAK,IAAI,QACP,uCAAuC,SAAS,aAChD,EAAC,SAAQ,CACX;KACA;IACF;GACF;GACA,KAAK,IAAI,OAAO,6BAA6B,SAAS,aAAa,EACjE,SACF,CAAC;GACD,KAAK,gBAAgB;GACrB,KAAK,gBAAgB,qBAAqB,SAAS,SAAS;GAC5D,KAAK,wBAAwB,QAAQ;EACvC;EACA,KAAK,IAAI,OAAO,iDAAiD;CACnE;;;;;;;;;;;;;;CAeA,wBAAwB,UAA6B;EACnD,IAAI,KAAK,sBACP;EAEF,MAAM,SAAS,KAAK;EACpB,IAAI,WAAW,KAAA,KAAa,SAAS,YAAY,QAAQ;GACvD,KAAK,IAAI,OAAO,yCAAyC;IACvD;IACA,gBAAgB;GAClB,CAAC;GACD;EACF;EACA,KAAK,uBAAuB;EAC5B,KAAK,qBAAqB,QAAQ;CACpC;CAEA,2BAA+C;EAC7C,IAAI;EACJ,IAAI;GACF,KAAK,IAAI,SAAS,KAAK,KAAK,KAAK,cAAc,EAAC,UAAU,KAAI,CAAC;GAC/D,MAAM,EAAC,iBAAgB,GACpB,QAAgB,mDAAmD,CAAC,CACpE,IAA4B;GAC/B,OAAO;EACT,SAAS,GAAG;GAGV,KAAK,IAAI,OACP,sGAEA,CACF;GACA;EACF,UAAU;GACR,IAAI,MAAM;EACZ;CACF;CAEA,OAAsB;EACpB,KAAK,YAAY,OAAO;EACxB,OAAO;CACT;CAEA,uBAAuB;EACrB,iBAAiB,WAAW,cAAc;GACxC,aACE;GAGF,MAAM;EACR,CAAC,CAAC,CAAC,aAAY,MAAK;GAClB,MAAM,eAAe,KAAK;GAC1B,IAAI,CAAC,cAAc;IACjB,KAAK,IAAI,OACP,8DACF;IACA;GACF;GACA,MAAM,KAAK,IAAI,SAAS,KAAK,KAAK,KAAK,cAAc,EAAC,UAAU,KAAI,CAAC;GACrE,IAAI;IACF,MAAM,EAAC,gBAAe,GACnB,QAAgB,kDAAkD,CAAC,CACnE,IAA2B;IAC9B,MAAM,YAAY,KAAK,IAAI,GAAG,cAAc,aAAa,YAAY;IACrE,EAAE,QAAQ,SAAS;GACrB,SAAS,GAAG;IACV,KAAK,IAAI,OAAO,6CAA6C,CAAC;GAChE,UAAU;IACR,GAAG,MAAM;GACX;EACF,CAAC;CACH;AACF"}

package/out/zero-cache/src/services/change-streamer/broadcast.js

@@ -182,6 +182,6 @@ function toBatch(change) {
 	return Array.isArray(change[0]) ? change : [change];
 }
 //#endregion
-export { Broadcast };
+export { Broadcast, preSerializeBatch };
 
 //# sourceMappingURL=broadcast.js.map

package/out/zero-cache/src/services/change-streamer/broadcast.js.map

@@ -1 +1 @@
-{"version":3,"file":"broadcast.js","names":["#lc","#pending","#completed","#done","#watermark","#majority","#start","#earlyReleaseTimeoutProportion","#setTimeout","#clearTimeout","#needsBackupResponse","#markCompleted","#setDone","#isDone","#earlyReleaseTimer","#logWithState","#releaseMode"],"sources":["../../../../../../zero-cache/src/services/change-streamer/broadcast.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {resolver} from '@rocicorp/resolver';\nimport {must} from '../../../../shared/src/must.ts';\nimport {isPreSerialized, type PreSerialized} from '../../types/streams.ts';\nimport type {WatermarkedChange} from './change-streamer.ts';\nimport type {Subscriber} from './subscriber.ts';\n\nexport type BroadcastReleaseMode = 'all-subscribers' | 'consensus-timeout';\n\nexport type PreSerializedBatch = PreSerialized & {\n  readonly changes: readonly WatermarkedChange[];\n};\n\nexport function isPreSerializedBatch(val: unknown): val is PreSerializedBatch {\n  return isPreSerialized(val) && 'changes' in val && Array.isArray(val.changes);\n}\n\nexport function preSerializeBatch(\n  changes: readonly WatermarkedChange[],\n): PreSerializedBatch {\n  const jsonList = changes.map(c => c[2]);\n  const joined = jsonList.join(',');\n  const payload = Buffer.from(\n    changes.length === 1 ? `,\"msg\":${jsonList[0]}}` : `,\"batch\":[${joined}]}`,\n    'utf8',\n  );\n  return {\n    changes,\n    payload,\n    byteLength: payload.length,\n  };\n}\n\n/**\n * Enables event-driven early release of a {@link Broadcast}. When provided and a\n * majority has acked, the broadcast schedules its own consensus-timeout release\n * proportional to how long the majority took to ack, instead of relying on the\n * Forwarder's periodic progress tick.\n */\nexport type EarlyReleaseOptions = {\n  /** Time to wait, proportional to majority completion, before releasing. */\n  consensusTimeoutProportion: number;\n  /** Injectable timer, for deterministic testing. */\n  setTimeoutFn?: typeof setTimeout;\n  /** Injectable timer cancellation, for deterministic testing. */\n  clearTimeoutFn?: typeof clearTimeout;\n};\n\n/**\n * Initiates and tracks the progress of a change broadcasted to\n * a set of subscribers, implementing **consensus-based timeouts**\n * to effect short-term graceful degradation when a minority of\n * subscribers are behind.\n *\n * ### Background\n *\n * The purpose of flow control is to pull upstream replication changes\n * no faster than the rate they are processed by downstream subscribers\n * in the steady state. In the change-streamer, this is done by occasionally\n * waiting for ACKs from subscribers before continuing; without doing so,\n * I/O buffers fill up and cause the system to spend most of its time in GC.\n *\n * However, the naive algorithm of always waiting for all subscribers (e.g.\n * `Promise.all()`) can behave poorly in scenarios where subscribers\n * are imbalanced:\n * * New subscribers may have a backlog of changes to catch up with.\n *   Having all subscribers wait for the new subscriber to catch up results\n *   in delaying the entire application.\n * * Broken TCP connections similarly require all subscribers to wait until\n *   connection liveness checks kick in and disconnect the subscriber.\n *\n * A simplistic approach is to add a limit to the amount of time waiting for\n * subscribers, i.e. an ack timeout. However, deciding what this timeout\n * should be is non-trivial because of the heterogeneous nature of changes;\n * while most changes operate on single rows and are relatively predictable\n * in terms of running time, some changes are table-wide operations and can\n * legitimately take an arbitrary amount of time. In such scenarios, a\n * timeout that is too short can stop progress on replication altogether.\n *\n * ### Consensus-based Timeout Algorithm\n *\n * To address these shortcomings, a \"consensus-based timeout\" algorithm is\n * used:\n * * Wait for more than half of the subscribers to finish. (In\n *   case of a single node, or the case of one replication-manager\n *   and one view-syncer, this reduces to waiting for all subscribers.)\n * * Once more than half of the subscribers have finished, proceed after\n *   the configured timeout, proportional to the time it took for the\n *   majority of subscribers to ack, even if not all subscribers have\n *   finished.\n *\n * In other words, the subscribers themselves are used to determine the\n * timeout of each batch of changes; the majority determines this when\n * they complete, upon which a timeout is logically started.\n *\n * In the common case, the remaining subscribers finish soon afterward and\n * the timeout never elapses. However, in pathological cases where a minority\n * of subscribers have a disproportionate amount of load, some will still\n * be processing (or otherwise unresponsive). These subscribers are given\n * a bounded amount of time to catch up at each flushed batch, up to the\n * timeout interval. This guarantees eventual catchup because the\n * subscribers with a backlog of changes necessarily have a higher\n * processing rate than the subscribers that finished (and are made to wait).\n */\nexport class Broadcast {\n  /**\n   * Sends the change to the subscribers without the tracking machinery.\n   * This is suitable for fire-and-forget (i.e. pipelined) sends.\n   */\n  static withoutTracking(\n    subscribers: Iterable<Subscriber>,\n    change: WatermarkedChange | readonly WatermarkedChange[],\n  ) {\n    const changes = toBatch(change);\n    const preSerialized = preSerializeBatch(changes);\n    for (const sub of subscribers) {\n      void sub.sendBatch(changes, preSerialized);\n    }\n  }\n\n  readonly #lc: LogContext;\n  readonly #pending: Set<Subscriber>;\n  readonly #completed: Completed[];\n  readonly #done = resolver();\n  #isDone = false;\n  #releaseMode: BroadcastReleaseMode | undefined;\n\n  readonly #watermark: string;\n  readonly #majority: number;\n\n  readonly #start = performance.now();\n\n  // When set, the broadcast schedules its own consensus-timeout release instead\n  // of waiting for the Forwarder's periodic progress tick.\n  readonly #earlyReleaseTimeoutProportion: number | undefined;\n  readonly #setTimeout: typeof setTimeout;\n  readonly #clearTimeout: typeof clearTimeout;\n  #earlyReleaseTimer: ReturnType<typeof setTimeout> | undefined;\n\n  // Tracks whether the backup-replicator responded. It is a required member\n  // of the consensus timeout since it cannot be dropped as a laggard (since\n  // that would shutdown the replication-manager itself).\n  #needsBackupResponse = false;\n\n  /**\n   * Broadcasts the `change` to the `subscribers` and tracks their\n   * completion.\n   */\n  constructor(\n    lc: LogContext,\n    subscribers: Iterable<Subscriber>,\n    change: WatermarkedChange | readonly WatermarkedChange[],\n    earlyRelease?: EarlyReleaseOptions,\n  ) {\n    const changes = toBatch(change);\n    const preSerialized = preSerializeBatch(changes);\n    this.#lc = lc;\n    this.#pending = new Set(subscribers);\n    this.#completed = [];\n    this.#watermark = must(changes.at(-1))[0];\n    this.#majority = Math.floor(this.#pending.size / 2) + 1;\n    this.#earlyReleaseTimeoutProportion =\n      earlyRelease?.consensusTimeoutProportion;\n    this.#setTimeout = earlyRelease?.setTimeoutFn ?? setTimeout;\n    this.#clearTimeout = earlyRelease?.clearTimeoutFn ?? clearTimeout;\n\n    for (const sub of this.#pending) {\n      const totalChanges = sub.numPending + changes.length;\n      if (sub.mode === 'backup') {\n        // Only gate consensus on the backup-replicator after it has finished\n        // its initial catchup.\n        this.#needsBackupResponse ||= !sub.isBacklogged();\n      }\n      void sub\n        .sendBatch(changes, preSerialized)\n        .catch(() => {})\n        .finally(() => this.#markCompleted(sub, totalChanges));\n    }\n\n    // set done if there are no subscribers (mainly for tests)\n    if (this.#pending.size === 0) {\n      this.#setDone('all-subscribers');\n    }\n  }\n\n  #markCompleted(sub: Subscriber, changes: number) {\n    if (this.#isDone) {\n      return;\n    }\n    const elapsed = performance.now() - this.#start;\n    sub.trackResponseResult('on-time');\n    if (sub.mode === 'backup') {\n      this.#needsBackupResponse = false;\n    }\n    this.#completed.push({sub, changes, elapsed});\n    this.#pending.delete(sub);\n    if (this.#pending.size === 0) {\n      this.#setDone('all-subscribers');\n      return;\n    }\n    // Event-driven consensus-timeout: the moment a majority acks, and it\n    // includes the backup-replicator, arm a single release timer\n    // proportional to how long the majority took to ack. The broadcast\n    // resolves when it fires, unless all subscribers ack first.\n    if (\n      this.#earlyReleaseTimeoutProportion !== undefined &&\n      this.#earlyReleaseTimer === undefined &&\n      this.#completed.length >= this.#majority &&\n      !this.#needsBackupResponse\n    ) {\n      const timeoutMs = Math.ceil(\n        elapsed * this.#earlyReleaseTimeoutProportion,\n      );\n      this.#earlyReleaseTimer = this.#setTimeout(\n        () => this.#setDone('consensus-timeout'),\n        timeoutMs,\n      );\n    }\n  }\n\n  #setDone(releaseMode: BroadcastReleaseMode) {\n    if (this.#isDone) {\n      return;\n    }\n    if (releaseMode === 'consensus-timeout') {\n      const elapsed = performance.now() - this.#start;\n      this.#logWithState(\n        `continuing with ${this.#pending.size} subscriber(s) still pending`,\n        elapsed,\n      );\n    }\n    this.#pending.forEach(sub => sub.trackResponseResult('timed-out'));\n    this.#isDone = true;\n    this.#releaseMode = releaseMode;\n    this.#clearTimeout(this.#earlyReleaseTimer);\n    this.#earlyReleaseTimer = undefined;\n    this.#done.resolve();\n  }\n\n  get isDone(): boolean {\n    return this.#isDone;\n  }\n\n  get done(): Promise<void> {\n    return this.#done.promise;\n  }\n\n  get releaseMode(): BroadcastReleaseMode {\n    return this.#releaseMode ?? 'all-subscribers';\n  }\n\n  logSlowRequests(now: number) {\n    if (this.#isDone || this.#pending.size === 0) {\n      return;\n    }\n    const elapsed = now - this.#start;\n    if (this.#completed.length < this.#majority) {\n      if (elapsed >= 1000) {\n        this.#logWithState(\n          `waiting for at least ${this.#majority} subscribers to finish`,\n          elapsed,\n        );\n      }\n    }\n  }\n\n  #logWithState(msg: string, elapsed: number) {\n    this.#lc\n      .withContext('watermark', this.#watermark)\n      .info?.(`${msg} (${elapsed.toFixed(3)} ms)`, {\n        completed: this.#completed.map(d => ({\n          id: d.sub.id,\n          processed: d.changes,\n          elapsed: d.elapsed,\n        })),\n        pending: Array.from(this.#pending, sub => ({\n          id: sub.id,\n          ...sub.getStats(),\n        })),\n      });\n  }\n}\n\n/** Tracks the completed result of a single subscriber. */\ntype Completed = {\n  sub: Subscriber;\n  /** The number of changes processed. */\n  changes: number;\n  /** The elapsed milliseconds. */\n  elapsed: number;\n};\n\nfunction toBatch(\n  change: WatermarkedChange | readonly WatermarkedChange[],\n): readonly WatermarkedChange[] {\n  if (change.length === 0) {\n    return [];\n  }\n  return Array.isArray(change[0])\n    ? (change as readonly WatermarkedChange[])\n    : [change as WatermarkedChange];\n}\n"],"mappings":";;;;AAiBA,SAAgB,kBACd,SACoB;CACpB,MAAM,WAAW,QAAQ,KAAI,MAAK,EAAE,EAAE;CACtC,MAAM,SAAS,SAAS,KAAK,GAAG;CAChC,MAAM,UAAU,OAAO,KACrB,QAAQ,WAAW,IAAI,UAAU,SAAS,GAAG,KAAK,aAAa,OAAO,KACtE,MACF;CACA,OAAO;EACL;EACA;EACA,YAAY,QAAQ;CACtB;AACF;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAyEA,IAAa,YAAb,MAAuB;;;;;CAKrB,OAAO,gBACL,aACA,QACA;EACA,MAAM,UAAU,QAAQ,MAAM;EAC9B,MAAM,gBAAgB,kBAAkB,OAAO;EAC/C,KAAK,MAAM,OAAO,aAChB,IAAS,UAAU,SAAS,aAAa;CAE7C;CAEA;CACA;CACA;CACA,QAAiB,SAAS;CAC1B,UAAU;CACV;CAEA;CACA;CAEA,SAAkB,YAAY,IAAI;CAIlC;CACA;CACA;CACA;CAKA,uBAAuB;;;;;CAMvB,YACE,IACA,aACA,QACA,cACA;EACA,MAAM,UAAU,QAAQ,MAAM;EAC9B,MAAM,gBAAgB,kBAAkB,OAAO;EAC/C,KAAKA,MAAM;EACX,KAAKC,WAAW,IAAI,IAAI,WAAW;EACnC,KAAKC,aAAa,CAAC;EACnB,KAAKE,aAAa,KAAK,QAAQ,GAAG,EAAE,CAAC,EAAE;EACvC,KAAKC,YAAY,KAAK,MAAM,KAAKJ,SAAS,OAAO,CAAC,IAAI;EACtD,KAAKM,iCACH,cAAc;EAChB,KAAKC,cAAc,cAAc,gBAAgB;EACjD,KAAKC,gBAAgB,cAAc,kBAAkB;EAErD,KAAK,MAAM,OAAO,KAAKR,UAAU;GAC/B,MAAM,eAAe,IAAI,aAAa,QAAQ;GAC9C,IAAI,IAAI,SAAS,UAGf,KAAKS,yBAAyB,CAAC,IAAI,aAAa;GAElD,IACG,UAAU,SAAS,aAAa,EAChC,YAAY,CAAC,CAAC,EACd,cAAc,KAAKC,eAAe,KAAK,YAAY,CAAC;EACzD;EAGA,IAAI,KAAKV,SAAS,SAAS,GACzB,KAAKW,SAAS,iBAAiB;CAEnC;CAEA,eAAe,KAAiB,SAAiB;EAC/C,IAAI,KAAKC,SACP;EAEF,MAAM,UAAU,YAAY,IAAI,IAAI,KAAKP;EACzC,IAAI,oBAAoB,SAAS;EACjC,IAAI,IAAI,SAAS,UACf,KAAKI,uBAAuB;EAE9B,KAAKR,WAAW,KAAK;GAAC;GAAK;GAAS;EAAO,CAAC;EAC5C,KAAKD,SAAS,OAAO,GAAG;EACxB,IAAI,KAAKA,SAAS,SAAS,GAAG;GAC5B,KAAKW,SAAS,iBAAiB;GAC/B;EACF;EAKA,IACE,KAAKL,mCAAmC,KAAA,KACxC,KAAKO,uBAAuB,KAAA,KAC5B,KAAKZ,WAAW,UAAU,KAAKG,aAC/B,CAAC,KAAKK,sBACN;GACA,MAAM,YAAY,KAAK,KACrB,UAAU,KAAKH,8BACjB;GACA,KAAKO,qBAAqB,KAAKN,kBACvB,KAAKI,SAAS,mBAAmB,GACvC,SACF;EACF;CACF;CAEA,SAAS,aAAmC;EAC1C,IAAI,KAAKC,SACP;EAEF,IAAI,gBAAgB,qBAAqB;GACvC,MAAM,UAAU,YAAY,IAAI,IAAI,KAAKP;GACzC,KAAKS,cACH,mBAAmB,KAAKd,SAAS,KAAK,+BACtC,OACF;EACF;EACA,KAAKA,SAAS,SAAQ,QAAO,IAAI,oBAAoB,WAAW,CAAC;EACjE,KAAKY,UAAU;EACf,KAAKG,eAAe;EACpB,KAAKP,cAAc,KAAKK,kBAAkB;EAC1C,KAAKA,qBAAqB,KAAA;EAC1B,KAAKX,MAAM,QAAQ;CACrB;CAEA,IAAI,SAAkB;EACpB,OAAO,KAAKU;CACd;CAEA,IAAI,OAAsB;EACxB,OAAO,KAAKV,MAAM;CACpB;CAEA,IAAI,cAAoC;EACtC,OAAO,KAAKa,gBAAgB;CAC9B;CAEA,gBAAgB,KAAa;EAC3B,IAAI,KAAKH,WAAW,KAAKZ,SAAS,SAAS,GACzC;EAEF,MAAM,UAAU,MAAM,KAAKK;EAC3B,IAAI,KAAKJ,WAAW,SAAS,KAAKG;OAC5B,WAAW,KACb,KAAKU,cACH,wBAAwB,KAAKV,UAAU,yBACvC,OACF;EAAA;CAGN;CAEA,cAAc,KAAa,SAAiB;EAC1C,KAAKL,IACF,YAAY,aAAa,KAAKI,UAAU,EACxC,OAAO,GAAG,IAAI,IAAI,QAAQ,QAAQ,CAAC,EAAE,OAAO;GAC3C,WAAW,KAAKF,WAAW,KAAI,OAAM;IACnC,IAAI,EAAE,IAAI;IACV,WAAW,EAAE;IACb,SAAS,EAAE;GACb,EAAE;GACF,SAAS,MAAM,KAAK,KAAKD,WAAU,SAAQ;IACzC,IAAI,IAAI;IACR,GAAG,IAAI,SAAS;GAClB,EAAE;EACJ,CAAC;CACL;AACF;AAWA,SAAS,QACP,QAC8B;CAC9B,IAAI,OAAO,WAAW,GACpB,OAAO,CAAC;CAEV,OAAO,MAAM,QAAQ,OAAO,EAAE,IACzB,SACD,CAAC,MAA2B;AAClC"}
+{"version":3,"file":"broadcast.js","names":[],"sources":["../../../../../../zero-cache/src/services/change-streamer/broadcast.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {resolver} from '@rocicorp/resolver';\nimport {must} from '../../../../shared/src/must.ts';\nimport {isPreSerialized, type PreSerialized} from '../../types/streams.ts';\nimport type {WatermarkedChange} from './change-streamer.ts';\nimport type {Subscriber} from './subscriber.ts';\n\nexport type BroadcastReleaseMode = 'all-subscribers' | 'consensus-timeout';\n\nexport type PreSerializedBatch = PreSerialized & {\n  readonly changes: readonly WatermarkedChange[];\n};\n\nexport function isPreSerializedBatch(val: unknown): val is PreSerializedBatch {\n  return isPreSerialized(val) && 'changes' in val && Array.isArray(val.changes);\n}\n\nexport function preSerializeBatch(\n  changes: readonly WatermarkedChange[],\n): PreSerializedBatch {\n  const jsonList = changes.map(c => c[2]);\n  const joined = jsonList.join(',');\n  const payload = Buffer.from(\n    changes.length === 1 ? `,\"msg\":${jsonList[0]}}` : `,\"batch\":[${joined}]}`,\n    'utf8',\n  );\n  return {\n    changes,\n    payload,\n    byteLength: payload.length,\n  };\n}\n\n/**\n * Enables event-driven early release of a {@link Broadcast}. When provided and a\n * majority has acked, the broadcast schedules its own consensus-timeout release\n * proportional to how long the majority took to ack, instead of relying on the\n * Forwarder's periodic progress tick.\n */\nexport type EarlyReleaseOptions = {\n  /** Time to wait, proportional to majority completion, before releasing. */\n  consensusTimeoutProportion: number;\n  /** Injectable timer, for deterministic testing. */\n  setTimeoutFn?: typeof setTimeout;\n  /** Injectable timer cancellation, for deterministic testing. */\n  clearTimeoutFn?: typeof clearTimeout;\n};\n\n/**\n * Initiates and tracks the progress of a change broadcasted to\n * a set of subscribers, implementing **consensus-based timeouts**\n * to effect short-term graceful degradation when a minority of\n * subscribers are behind.\n *\n * ### Background\n *\n * The purpose of flow control is to pull upstream replication changes\n * no faster than the rate they are processed by downstream subscribers\n * in the steady state. In the change-streamer, this is done by occasionally\n * waiting for ACKs from subscribers before continuing; without doing so,\n * I/O buffers fill up and cause the system to spend most of its time in GC.\n *\n * However, the naive algorithm of always waiting for all subscribers (e.g.\n * `Promise.all()`) can behave poorly in scenarios where subscribers\n * are imbalanced:\n * * New subscribers may have a backlog of changes to catch up with.\n *   Having all subscribers wait for the new subscriber to catch up results\n *   in delaying the entire application.\n * * Broken TCP connections similarly require all subscribers to wait until\n *   connection liveness checks kick in and disconnect the subscriber.\n *\n * A simplistic approach is to add a limit to the amount of time waiting for\n * subscribers, i.e. an ack timeout. However, deciding what this timeout\n * should be is non-trivial because of the heterogeneous nature of changes;\n * while most changes operate on single rows and are relatively predictable\n * in terms of running time, some changes are table-wide operations and can\n * legitimately take an arbitrary amount of time. In such scenarios, a\n * timeout that is too short can stop progress on replication altogether.\n *\n * ### Consensus-based Timeout Algorithm\n *\n * To address these shortcomings, a \"consensus-based timeout\" algorithm is\n * used:\n * * Wait for more than half of the subscribers to finish. (In\n *   case of a single node, or the case of one replication-manager\n *   and one view-syncer, this reduces to waiting for all subscribers.)\n * * Once more than half of the subscribers have finished, proceed after\n *   the configured timeout, proportional to the time it took for the\n *   majority of subscribers to ack, even if not all subscribers have\n *   finished.\n *\n * In other words, the subscribers themselves are used to determine the\n * timeout of each batch of changes; the majority determines this when\n * they complete, upon which a timeout is logically started.\n *\n * In the common case, the remaining subscribers finish soon afterward and\n * the timeout never elapses. However, in pathological cases where a minority\n * of subscribers have a disproportionate amount of load, some will still\n * be processing (or otherwise unresponsive). These subscribers are given\n * a bounded amount of time to catch up at each flushed batch, up to the\n * timeout interval. This guarantees eventual catchup because the\n * subscribers with a backlog of changes necessarily have a higher\n * processing rate than the subscribers that finished (and are made to wait).\n */\nexport class Broadcast {\n  /**\n   * Sends the change to the subscribers without the tracking machinery.\n   * This is suitable for fire-and-forget (i.e. pipelined) sends.\n   */\n  static withoutTracking(\n    subscribers: Iterable<Subscriber>,\n    change: WatermarkedChange | readonly WatermarkedChange[],\n  ) {\n    const changes = toBatch(change);\n    const preSerialized = preSerializeBatch(changes);\n    for (const sub of subscribers) {\n      void sub.sendBatch(changes, preSerialized);\n    }\n  }\n\n  readonly #lc: LogContext;\n  readonly #pending: Set<Subscriber>;\n  readonly #completed: Completed[];\n  readonly #done = resolver();\n  #isDone = false;\n  #releaseMode: BroadcastReleaseMode | undefined;\n\n  readonly #watermark: string;\n  readonly #majority: number;\n\n  readonly #start = performance.now();\n\n  // When set, the broadcast schedules its own consensus-timeout release instead\n  // of waiting for the Forwarder's periodic progress tick.\n  readonly #earlyReleaseTimeoutProportion: number | undefined;\n  readonly #setTimeout: typeof setTimeout;\n  readonly #clearTimeout: typeof clearTimeout;\n  #earlyReleaseTimer: ReturnType<typeof setTimeout> | undefined;\n\n  // Tracks whether the backup-replicator responded. It is a required member\n  // of the consensus timeout since it cannot be dropped as a laggard (since\n  // that would shutdown the replication-manager itself).\n  #needsBackupResponse = false;\n\n  /**\n   * Broadcasts the `change` to the `subscribers` and tracks their\n   * completion.\n   */\n  constructor(\n    lc: LogContext,\n    subscribers: Iterable<Subscriber>,\n    change: WatermarkedChange | readonly WatermarkedChange[],\n    earlyRelease?: EarlyReleaseOptions,\n  ) {\n    const changes = toBatch(change);\n    const preSerialized = preSerializeBatch(changes);\n    this.#lc = lc;\n    this.#pending = new Set(subscribers);\n    this.#completed = [];\n    this.#watermark = must(changes.at(-1))[0];\n    this.#majority = Math.floor(this.#pending.size / 2) + 1;\n    this.#earlyReleaseTimeoutProportion =\n      earlyRelease?.consensusTimeoutProportion;\n    this.#setTimeout = earlyRelease?.setTimeoutFn ?? setTimeout;\n    this.#clearTimeout = earlyRelease?.clearTimeoutFn ?? clearTimeout;\n\n    for (const sub of this.#pending) {\n      const totalChanges = sub.numPending + changes.length;\n      if (sub.mode === 'backup') {\n        // Only gate consensus on the backup-replicator after it has finished\n        // its initial catchup.\n        this.#needsBackupResponse ||= !sub.isBacklogged();\n      }\n      void sub\n        .sendBatch(changes, preSerialized)\n        .catch(() => {})\n        .finally(() => this.#markCompleted(sub, totalChanges));\n    }\n\n    // set done if there are no subscribers (mainly for tests)\n    if (this.#pending.size === 0) {\n      this.#setDone('all-subscribers');\n    }\n  }\n\n  #markCompleted(sub: Subscriber, changes: number) {\n    if (this.#isDone) {\n      return;\n    }\n    const elapsed = performance.now() - this.#start;\n    sub.trackResponseResult('on-time');\n    if (sub.mode === 'backup') {\n      this.#needsBackupResponse = false;\n    }\n    this.#completed.push({sub, changes, elapsed});\n    this.#pending.delete(sub);\n    if (this.#pending.size === 0) {\n      this.#setDone('all-subscribers');\n      return;\n    }\n    // Event-driven consensus-timeout: the moment a majority acks, and it\n    // includes the backup-replicator, arm a single release timer\n    // proportional to how long the majority took to ack. The broadcast\n    // resolves when it fires, unless all subscribers ack first.\n    if (\n      this.#earlyReleaseTimeoutProportion !== undefined &&\n      this.#earlyReleaseTimer === undefined &&\n      this.#completed.length >= this.#majority &&\n      !this.#needsBackupResponse\n    ) {\n      const timeoutMs = Math.ceil(\n        elapsed * this.#earlyReleaseTimeoutProportion,\n      );\n      this.#earlyReleaseTimer = this.#setTimeout(\n        () => this.#setDone('consensus-timeout'),\n        timeoutMs,\n      );\n    }\n  }\n\n  #setDone(releaseMode: BroadcastReleaseMode) {\n    if (this.#isDone) {\n      return;\n    }\n    if (releaseMode === 'consensus-timeout') {\n      const elapsed = performance.now() - this.#start;\n      this.#logWithState(\n        `continuing with ${this.#pending.size} subscriber(s) still pending`,\n        elapsed,\n      );\n    }\n    this.#pending.forEach(sub => sub.trackResponseResult('timed-out'));\n    this.#isDone = true;\n    this.#releaseMode = releaseMode;\n    this.#clearTimeout(this.#earlyReleaseTimer);\n    this.#earlyReleaseTimer = undefined;\n    this.#done.resolve();\n  }\n\n  get isDone(): boolean {\n    return this.#isDone;\n  }\n\n  get done(): Promise<void> {\n    return this.#done.promise;\n  }\n\n  get releaseMode(): BroadcastReleaseMode {\n    return this.#releaseMode ?? 'all-subscribers';\n  }\n\n  logSlowRequests(now: number) {\n    if (this.#isDone || this.#pending.size === 0) {\n      return;\n    }\n    const elapsed = now - this.#start;\n    if (this.#completed.length < this.#majority) {\n      if (elapsed >= 1000) {\n        this.#logWithState(\n          `waiting for at least ${this.#majority} subscribers to finish`,\n          elapsed,\n        );\n      }\n    }\n  }\n\n  #logWithState(msg: string, elapsed: number) {\n    this.#lc\n      .withContext('watermark', this.#watermark)\n      .info?.(`${msg} (${elapsed.toFixed(3)} ms)`, {\n        completed: this.#completed.map(d => ({\n          id: d.sub.id,\n          processed: d.changes,\n          elapsed: d.elapsed,\n        })),\n        pending: Array.from(this.#pending, sub => ({\n          id: sub.id,\n          ...sub.getStats(),\n        })),\n      });\n  }\n}\n\n/** Tracks the completed result of a single subscriber. */\ntype Completed = {\n  sub: Subscriber;\n  /** The number of changes processed. */\n  changes: number;\n  /** The elapsed milliseconds. */\n  elapsed: number;\n};\n\nfunction toBatch(\n  change: WatermarkedChange | readonly WatermarkedChange[],\n): readonly WatermarkedChange[] {\n  if (change.length === 0) {\n    return [];\n  }\n  return Array.isArray(change[0])\n    ? (change as readonly WatermarkedChange[])\n    : [change as WatermarkedChange];\n}\n"],"mappings":";;;;AAiBA,SAAgB,kBACd,SACoB;CACpB,MAAM,WAAW,QAAQ,KAAI,MAAK,EAAE,EAAE;CACtC,MAAM,SAAS,SAAS,KAAK,GAAG;CAChC,MAAM,UAAU,OAAO,KACrB,QAAQ,WAAW,IAAI,UAAU,SAAS,GAAG,KAAK,aAAa,OAAO,KACtE,MACF;CACA,OAAO;EACL;EACA;EACA,YAAY,QAAQ;CACtB;AACF;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAyEA,IAAa,YAAb,MAAuB;;;;;CAKrB,OAAO,gBACL,aACA,QACA;EACA,MAAM,UAAU,QAAQ,MAAM;EAC9B,MAAM,gBAAgB,kBAAkB,OAAO;EAC/C,KAAK,MAAM,OAAO,aAChB,IAAS,UAAU,SAAS,aAAa;CAE7C;CAEA;CACA;CACA;CACA,QAAiB,SAAS;CAC1B,UAAU;CACV;CAEA;CACA;CAEA,SAAkB,YAAY,IAAI;CAIlC;CACA;CACA;CACA;CAKA,uBAAuB;;;;;CAMvB,YACE,IACA,aACA,QACA,cACA;EACA,MAAM,UAAU,QAAQ,MAAM;EAC9B,MAAM,gBAAgB,kBAAkB,OAAO;EAC/C,KAAK,MAAM;EACX,KAAK,WAAW,IAAI,IAAI,WAAW;EACnC,KAAK,aAAa,CAAC;EACnB,KAAK,aAAa,KAAK,QAAQ,GAAG,EAAE,CAAC,CAAC,CAAC;EACvC,KAAK,YAAY,KAAK,MAAM,KAAK,SAAS,OAAO,CAAC,IAAI;EACtD,KAAK,iCACH,cAAc;EAChB,KAAK,cAAc,cAAc,gBAAgB;EACjD,KAAK,gBAAgB,cAAc,kBAAkB;EAErD,KAAK,MAAM,OAAO,KAAK,UAAU;GAC/B,MAAM,eAAe,IAAI,aAAa,QAAQ;GAC9C,IAAI,IAAI,SAAS,UAGf,KAAK,yBAAyB,CAAC,IAAI,aAAa;GAElD,IACG,UAAU,SAAS,aAAa,CAAC,CACjC,YAAY,CAAC,CAAC,CAAC,CACf,cAAc,KAAK,eAAe,KAAK,YAAY,CAAC;EACzD;EAGA,IAAI,KAAK,SAAS,SAAS,GACzB,KAAK,SAAS,iBAAiB;CAEnC;CAEA,eAAe,KAAiB,SAAiB;EAC/C,IAAI,KAAK,SACP;EAEF,MAAM,UAAU,YAAY,IAAI,IAAI,KAAK;EACzC,IAAI,oBAAoB,SAAS;EACjC,IAAI,IAAI,SAAS,UACf,KAAK,uBAAuB;EAE9B,KAAK,WAAW,KAAK;GAAC;GAAK;GAAS;EAAO,CAAC;EAC5C,KAAK,SAAS,OAAO,GAAG;EACxB,IAAI,KAAK,SAAS,SAAS,GAAG;GAC5B,KAAK,SAAS,iBAAiB;GAC/B;EACF;EAKA,IACE,KAAK,mCAAmC,KAAA,KACxC,KAAK,uBAAuB,KAAA,KAC5B,KAAK,WAAW,UAAU,KAAK,aAC/B,CAAC,KAAK,sBACN;GACA,MAAM,YAAY,KAAK,KACrB,UAAU,KAAK,8BACjB;GACA,KAAK,qBAAqB,KAAK,kBACvB,KAAK,SAAS,mBAAmB,GACvC,SACF;EACF;CACF;CAEA,SAAS,aAAmC;EAC1C,IAAI,KAAK,SACP;EAEF,IAAI,gBAAgB,qBAAqB;GACvC,MAAM,UAAU,YAAY,IAAI,IAAI,KAAK;GACzC,KAAK,cACH,mBAAmB,KAAK,SAAS,KAAK,+BACtC,OACF;EACF;EACA,KAAK,SAAS,SAAQ,QAAO,IAAI,oBAAoB,WAAW,CAAC;EACjE,KAAK,UAAU;EACf,KAAK,eAAe;EACpB,KAAK,cAAc,KAAK,kBAAkB;EAC1C,KAAK,qBAAqB,KAAA;EAC1B,KAAK,MAAM,QAAQ;CACrB;CAEA,IAAI,SAAkB;EACpB,OAAO,KAAK;CACd;CAEA,IAAI,OAAsB;EACxB,OAAO,KAAK,MAAM;CACpB;CAEA,IAAI,cAAoC;EACtC,OAAO,KAAK,gBAAgB;CAC9B;CAEA,gBAAgB,KAAa;EAC3B,IAAI,KAAK,WAAW,KAAK,SAAS,SAAS,GACzC;EAEF,MAAM,UAAU,MAAM,KAAK;EAC3B,IAAI,KAAK,WAAW,SAAS,KAAK,WAC5B;OAAA,WAAW,KACb,KAAK,cACH,wBAAwB,KAAK,UAAU,yBACvC,OACF;EAAA;CAGN;CAEA,cAAc,KAAa,SAAiB;EAC1C,KAAK,IACF,YAAY,aAAa,KAAK,UAAU,CAAC,CACzC,OAAO,GAAG,IAAI,IAAI,QAAQ,QAAQ,CAAC,EAAE,OAAO;GAC3C,WAAW,KAAK,WAAW,KAAI,OAAM;IACnC,IAAI,EAAE,IAAI;IACV,WAAW,EAAE;IACb,SAAS,EAAE;GACb,EAAE;GACF,SAAS,MAAM,KAAK,KAAK,WAAU,SAAQ;IACzC,IAAI,IAAI;IACR,GAAG,IAAI,SAAS;GAClB,EAAE;EACJ,CAAC;CACL;AACF;AAWA,SAAS,QACP,QAC8B;CAC9B,IAAI,OAAO,WAAW,GACpB,OAAO,CAAC;CAEV,OAAO,MAAM,QAAQ,OAAO,EAAE,IACzB,SACD,CAAC,MAA2B;AAClC"}

package/out/zero-cache/src/services/change-streamer/change-log-initializer.js

@@ -251,7 +251,7 @@ function readSchemaChanges(db, after, through) {
       SELECT count(*) AS "rows" FROM (
         SELECT 1 FROM "${CHANGE_LOG_STREAM_TABLE}"
           WHERE "watermark" > ? AND "watermark" <= ?
-          LIMIT ${MAX_FOLD_SCAN_ROWS + 1}
+          LIMIT 10001
       )
     `).get(after, through);
 	if (rows > MAX_FOLD_SCAN_ROWS) return;
@@ -308,6 +308,6 @@ function sortKeys(value) {
 	return Object.fromEntries(Object.entries(value).sort(([a], [b]) => a < b ? -1 : a > b ? 1 : 0).map(([k, v]) => [k, sortKeys(v)]));
 }
 //#endregion
-export { ChangeLogInitializer, replicaInitializationSource };
+export { ChangeLogInitializer, readReplicaInitializationParameters, replicaInitializationSource };
 
 //# sourceMappingURL=change-log-initializer.js.map

package/out/zero-cache/src/services/change-streamer/change-log-initializer.js.map

@@ -1 +1 @@
-{"version":3,"file":"change-log-initializer.js","names":["#lc","#initFromPgChangeLog","#initFromReplica","#sources","#resultCounter","#lastComparison","#pgChangeLog","#readReplica","#compare","#restoredReplica","#classify","#report"],"sources":["../../../../../../zero-cache/src/services/change-streamer/change-log-initializer.ts"],"sourcesContent":["/**\n * Where a stream connection's initialization parameters come from.\n *\n * A change-streamer starts a stream from three things read at one position: the\n * watermark to resume at, the {@link BackfillRequest}s to hand the change\n * source, and the cookie set those requests were derived from. Postgres has\n * always supplied all three (`Storer.getStartStreamInitializationParameters()`).\n * The point of this workstream is that a restored replica can supply them\n * instead, which is what lets the `cdc` schema be decommissioned.\n *\n * This is the two-source stage. Both are computed on every stream connection,\n * compared, and Postgres's is used — so the replica-derived path runs in\n * production, at full rate, before anything depends on it. The flip is\n * `initFromPgChangeLog: false`, not a rewrite.\n *\n * The shape is {@link UpstreamAcker}'s, deliberately: two stores, one output, a\n * flag per store, and the Postgres flag carrying the marker that retires it.\n * That class solved the identical problem for upstream ACKs earlier in this\n * workstream.\n *\n * ### Why the comparison is a fold and not an equality\n *\n * The replica is a subscriber downstream of the change-streamer's write loop,\n * so its state version trails the log's head by however long the replicator is\n * behind. A raw difference between the two cookie sets is therefore the normal\n * case and says nothing. What is checked instead is that the replica's set,\n * carried forward over exactly the schema changes the log holds in the interval,\n * equals Postgres's set at Postgres's watermark. The log holds precisely those\n * transactions, schema changes are rare, and the check is exact rather than\n * approximate — so it also happens to be the strongest available test of the\n * premise that three interpreters of one fold agree.\n */\n\nimport type {LogContext} from '@rocicorp/logger';\nimport {assert} from '../../../../shared/src/asserts.ts';\nimport {BigIntJSON} from '../../../../shared/src/bigint-json.ts';\nimport {must} from '../../../../shared/src/must.ts';\nimport {Database} from '../../../../zqlite/src/db.ts';\nimport {StatementRunner} from '../../db/statements.ts';\nimport {getOrCreateCounter} from '../../observability/metrics.ts';\nimport type {SchemaChange} from '../change-source/protocol/current/data.ts';\nimport {schemaChangeTags} from '../change-source/protocol/current/schema-change-tags.ts';\nimport type {BackfillRequest} from '../change-source/protocol/current/upstream.ts';\nimport {\n  backfillRequestsFrom,\n  foldCookies,\n  type CookieSet,\n} from '../replicator/change-log-cookies.ts';\nimport {\n  CHANGE_LOG_STREAM_TABLE,\n  type ChangeLogResumePoint,\n} from '../replicator/change-log-db.ts';\nimport {\n  readBackfillRequests,\n  readReplicaCookies,\n} from '../replicator/schema/backfilling.ts';\nimport {getReplicationState} from '../replicator/schema/replication-state.ts';\nimport {SQLITE_CHANGE_LOG_BOUNDARY_SQL} from './sqlite-change-log-reader.ts';\n\n/** Everything one stream connection starts from, read at one position. */\nexport type InitializationParameters = {\n  readonly lastWatermark: string;\n  readonly backfillRequests: BackfillRequest[];\n  readonly cookies: CookieSet;\n};\n\n/**\n * - `equal` — the two stores were at the same watermark and held the same\n *   cookies. Nothing had to be folded, which is the strongest of the four.\n * - `equal-after-fold` — the replica trailed, and carrying its set forward over\n *   the log's schema changes in the interval reproduced Postgres's exactly.\n *   The expected steady-state result.\n * - `divergent` — the two disagree at the same position. Alert; investigate\n *   individually.\n * - `inconclusive` — the log does not span the interval, so the fold cannot be\n *   performed. Routine on a cold log, and not a finding.\n * - `error` — the comparison itself failed: the replica could not be read, or\n *   the fold threw. Not one of the four classifications, because it is a\n *   statement about this process rather than about the two stores. Kept\n *   separate rather than folded into `inconclusive` so that a change-streamer\n *   that has silently stopped being able to read its replica is visible before\n *   that read becomes the only source.\n */\nexport type InitComparisonResult =\n  | 'equal'\n  | 'equal-after-fold'\n  | 'divergent'\n  | 'inconclusive'\n  | 'error';\n\nexport type ChangeLogInitializerSources = {\n  /** `Storer.getStartStreamInitializationParameters()`. */\n  readonly pgChangeLog: () => Promise<InitializationParameters>;\n  /** Typically {@link replicaInitializationSource}. */\n  readonly replica: () => InitializationParameters;\n  /**\n   * Reconciles the SQLite change log and returns its current resume point.\n   *\n   * When `resumeFrom` is present, it supplies the resume point. Otherwise, a\n   * valid log uses its own head. A new or invalid log uses `seed`.\n   *\n   * Returns `undefined` when the writer is disabled or unavailable.\n   */\n  readonly reconcileChangeLog: (\n    resumeFrom: ChangeLogResumePoint | undefined,\n    seed: () => ChangeLogResumePoint,\n  ) => ChangeLogResumePoint | undefined;\n  /**\n   * The change log, for the fold. `undefined` before the stream loop's first\n   * reconcile has created it, and again once the writer has failed soft and\n   * deleted it — both of which make the comparison `inconclusive` rather than\n   * an error, since neither says anything about the two stores.\n   */\n  readonly changeLog: () => Database | undefined;\n  /**\n   * Re-initializes the Postgres change log from the replica's resume point\n   * (taking ownership of it) if it does not contain all of the replica's\n   * changes, re-checking under lock. Typically\n   * `Storer.reinitializeFromReplica()`.\n   *\n   * Supplied only when the change log's head is behind the position of the\n   * replication slot from which replication resumes, i.e. when it may be\n   * missing transactions that the slot will not stream. Otherwise, a replica\n   * that is ahead of the change log (as changes are forwarded before they are\n   * stored) is caught up by the change stream, which resumes from the change\n   * log's head.\n   */\n  readonly reinitializePgChangeLog?:\n    | ((params: {\n        watermark: string;\n        cookies: CookieSet;\n      }) => Promise<'reinitialized' | 'maintained'>)\n    | undefined;\n};\n\ntype Opts = {\n  /**\n   * When true, Postgres supplies the resume point and the cookies for a new\n   * change log.\n   *\n   * When false, a valid log resumes from its own head. The replica supplies the\n   * resume point for a new or invalid log.\n   */\n  // TODO: set false when retiring PG\n  readonly initFromPgChangeLog: boolean;\n  /**\n   * Derive the parameters from the replica. When Postgres is enabled, these\n   * parameters are used only for comparison.\n   *\n   * When Postgres is disabled, the replica supplies the seed and the fallback\n   * resume point.\n   */\n  readonly initFromReplica: boolean;\n};\n\n/**\n * The most change-log rows one comparison will scan.\n *\n * The interval is `(replicaWatermark, pgWatermark]`, so it is normally a\n * handful of transactions — but a replicator that has been stalled can leave it\n * arbitrarily large, and this runs on the stream loop between reconciliation and\n * `startStream`, where latency delays the resumption of replication. Over the\n * cap the comparison declines rather than pays: a chronically lagging replicator\n * shows up as `inconclusive`, which is charted, and not as a stall.\n */\nconst MAX_FOLD_SCAN_ROWS = 10_000;\n\nexport class ChangeLogInitializer {\n  readonly #lc: LogContext;\n  readonly #initFromPgChangeLog: boolean;\n  readonly #initFromReplica: boolean;\n  readonly #sources: ChangeLogInitializerSources;\n\n  readonly #resultCounter = getOrCreateCounter(\n    'replica',\n    'sqlite_change_log.init_compare_result',\n    'Comparisons of the replica-derived stream initialization parameters ' +\n      'against the Postgres-derived ones, by classification.',\n  );\n\n  #lastComparison: InitComparisonResult | undefined;\n\n  /**\n   * The (restored) replica's resume point that the Postgres change log must\n   * cover, read once. See {@link #pgChangeLog}.\n   */\n  #restoredReplica: InitializationParameters | undefined;\n\n  constructor(\n    lc: LogContext,\n    {initFromPgChangeLog, initFromReplica}: Opts,\n    sources: ChangeLogInitializerSources,\n  ) {\n    assert(\n      initFromPgChangeLog || initFromReplica,\n      `At least one of initFromPgChangeLog or initFromReplica must be true`,\n    );\n    this.#lc = lc.withContext('component', 'change-log-initializer');\n    this.#initFromPgChangeLog = initFromPgChangeLog;\n    this.#initFromReplica = initFromReplica;\n    this.#sources = sources;\n  }\n\n  /**\n   * Returns the comparison result from the most recent {@link initialize}\n   * call. Returns `undefined` when that call used only one source.\n   *\n   * Tests and metrics use this result. It does not affect initialization.\n   */\n  lastComparison(): InitComparisonResult | undefined {\n    return this.#lastComparison;\n  }\n\n  /**\n   * Returns the initialization parameters for one stream connection after it\n   * reconciles the change log.\n   *\n   * The selected source determines how reconciliation works. Keeping both\n   * operations here hides that choice from the caller.\n   */\n  async initialize(): Promise<InitializationParameters> {\n    this.#lastComparison = undefined;\n    const pg = this.#initFromPgChangeLog\n      ? await this.#pgChangeLog()\n      : undefined;\n    // The replica is required when Postgres is disabled. It seeds a new log and\n    // supplies a resume point when the writer is unavailable.\n    const replica = this.#initFromReplica\n      ? this.#readReplica(pg === undefined)\n      : undefined;\n\n    const reconciled = this.#sources.reconcileChangeLog(\n      pg && {resumeWatermark: pg.lastWatermark, cookies: pg.cookies},\n      () => resumePointOf(must(replica, 'no seed for the SQLite change log')),\n    );\n\n    if (pg) {\n      if (replica) {\n        // Compare only after reconciliation. The fold must read the schema\n        // changes from the replica watermark through the Postgres watermark.\n        // Reconciliation makes sure that the log contains that interval.\n        this.#lastComparison = this.#compare(pg, replica);\n      }\n      return pg;\n    }\n\n    // Use the reconciled log position. If the writer is unavailable, use the\n    // replica position.\n    const resumePoint =\n      reconciled ?? resumePointOf(must(replica, 'no initialization source'));\n    return {\n      lastWatermark: resumePoint.resumeWatermark,\n      cookies: resumePoint.cookies,\n      // Derive the requests from these cookies so that the requests and the\n      // watermark describe the same position.\n      backfillRequests: backfillRequestsFrom(resumePoint.cookies),\n    };\n  }\n\n  /**\n   * Reads the Postgres change log's initialization parameters, first\n   * re-initializing it from the replica (taking ownership of it) if it may be\n   * missing transactions that the replication slot will not stream, e.g. if it\n   * is no longer maintained after running in high-availability mode. This is\n   * the Postgres counterpart of seeding a new SQLite change log from the\n   * replica.\n   *\n   * This only applies when the change log's head is behind the slot's position\n   * (i.e. when {@link ChangeLogInitializerSources.reinitializePgChangeLog} is\n   * supplied), and the change log is only re-initialized if its head is also\n   * behind the replica's. Otherwise, it contains every transaction before the\n   * slot's position: one that was acked after being stored would be in the\n   * change log, and one that was acked after being backed up would be in the\n   * replica, which the change log's head covers.\n   *\n   * The replica is read only once, i.e. as it was restored from the backup,\n   * before any stream is started. The live replica is no reference: it is\n   * caught up from the forwarded changes, which can be ahead of what has been\n   * stored, e.g. by the change log's previous owner. Against that fixed\n   * resume point, re-initialization is idempotent: afterwards, the change log's\n   * head (which only advances) covers it, so subsequent stream connections\n   * leave it untouched.\n   */\n  async #pgChangeLog(): Promise<InitializationParameters> {\n    const pg = await this.#sources.pgChangeLog();\n    const reinitialize = this.#sources.reinitializePgChangeLog;\n    if (!reinitialize) {\n      return pg;\n    }\n    this.#restoredReplica ??= this.#sources.replica();\n    const {lastWatermark, cookies} = this.#restoredReplica;\n    if (pg.lastWatermark >= lastWatermark) {\n      return pg;\n    }\n    const result = await reinitialize({watermark: lastWatermark, cookies});\n    this.#lc.info?.(\n      `Postgres change log at ${pg.lastWatermark} is behind the replica at ` +\n        `${lastWatermark}: ${result}`,\n    );\n    // Either way, the change log has changed since it was read.\n    return this.#sources.pgChangeLog();\n  }\n\n  /**\n   * Compares both sources and records the result. Postgres remains\n   * authoritative when both sources are enabled.\n   */\n  #compare(\n    pg: InitializationParameters,\n    replica: InitializationParameters,\n  ): InitComparisonResult {\n    let result: InitComparisonResult;\n    try {\n      result = this.#classify(pg, replica);\n    } catch (e) {\n      this.#lc.warn?.('failed to compare change-log initialization sources', e);\n      result = 'error';\n    }\n    this.#resultCounter.add(1, {result});\n    this.#lc.debug?.('compared change-log initialization sources', {\n      changeLogInitCompare: {\n        result,\n        pgWatermark: pg.lastWatermark,\n        replicaWatermark: replica.lastWatermark,\n      },\n    });\n    return result;\n  }\n\n  #readReplica(required: boolean): InitializationParameters | undefined {\n    try {\n      return this.#sources.replica();\n    } catch (e) {\n      if (required) {\n        throw e;\n      }\n      // Observational until the flip, so a replica that cannot be read must not\n      // stop replication -- but it must not be quiet either, since the flip\n      // makes this read the only source.\n      this.#lc.warn?.(\n        'failed to derive change-log initialization from the replica',\n        e,\n      );\n      this.#resultCounter.add(1, {result: 'error'});\n      return undefined;\n    }\n  }\n\n  #classify(\n    pg: InitializationParameters,\n    replica: InitializationParameters,\n  ): InitComparisonResult {\n    if (replica.lastWatermark > pg.lastWatermark) {\n      // Transactions reach subscribers before the storer's commit, so a replica\n      // read taken while one is in flight can be a transaction ahead. The fold\n      // only runs forward, so there is nothing to be said either way.\n      return 'inconclusive';\n    }\n    if (replica.lastWatermark === pg.lastWatermark) {\n      return this.#report(pg, replica, replica.cookies, 'equal');\n    }\n\n    const db = this.#sources.changeLog();\n    if (db === undefined || !spansInterval(db, replica.lastWatermark)) {\n      return 'inconclusive';\n    }\n    const changes = readSchemaChanges(\n      db,\n      replica.lastWatermark,\n      pg.lastWatermark,\n    );\n    if (changes === undefined) {\n      return 'inconclusive'; // over MAX_FOLD_SCAN_ROWS\n    }\n    return this.#report(\n      pg,\n      replica,\n      foldCookies(replica.cookies, changes),\n      'equal-after-fold',\n      changes.length,\n    );\n  }\n\n  #report(\n    pg: InitializationParameters,\n    replica: InitializationParameters,\n    derived: CookieSet,\n    whenEqual: InitComparisonResult,\n    foldedChanges = 0,\n  ): InitComparisonResult {\n    const expected = canonicalizeCookies(pg.cookies);\n    const actual = canonicalizeCookies(derived);\n    if (expected === actual) {\n      return whenEqual;\n    }\n    // Both sets are normally empty and are at most a handful of rows, so they\n    // are logged whole: a divergence is investigated individually (`P§:10`)\n    // and the two renderings are the whole of what there is to look at.\n    this.#lc.error?.(\n      'the replica-derived change-log cookies do not match Postgres',\n      {\n        changeLogInitCompare: {\n          pgWatermark: pg.lastWatermark,\n          replicaWatermark: replica.lastWatermark,\n          foldedChanges,\n          expected,\n          actual,\n        },\n      },\n    );\n    return 'divergent';\n  }\n}\n\nconst resumePointOf = ({\n  lastWatermark,\n  cookies,\n}: InitializationParameters): ChangeLogResumePoint => ({\n  resumeWatermark: lastWatermark,\n  cookies,\n});\n\n/**\n * Reads the replica's initialization parameters in one snapshot, which is\n * invariant 15 on the replica side: the state version and the cookies are only\n * meaningful together.\n */\nexport function readReplicaInitializationParameters(\n  db: Database,\n): InitializationParameters {\n  const runner = new StatementRunner(db);\n  runner.begin(); // deferred, i.e. one read snapshot for the three statements\n  try {\n    return {\n      lastWatermark: getReplicationState(runner).stateVersion,\n      backfillRequests: readBackfillRequests(db),\n      cookies: readReplicaCookies(db),\n    };\n  } finally {\n    if (db.inTransaction) {\n      runner.rollback();\n    }\n  }\n}\n\n/**\n * The replica source for {@link ChangeLogInitializer}, opening and closing the\n * file per stream connection rather than holding a handle on a database another\n * process owns.\n *\n * The handle is read-only, which is invariant 19 stated in the API: the\n * replica's cookie tables are written by the change-processor and by nothing\n * else. Note that a read-only SQLite connection still needs to attach the\n * WAL's `-shm`, so this can fail if it opens at a moment when no writer holds\n * the replica; that surfaces as the `error` classification, which is the point\n * of running the path under observation before anything depends on it.\n */\nexport function replicaInitializationSource(\n  lc: LogContext,\n  replicaFile: string,\n): () => InitializationParameters {\n  return () => {\n    const db = new Database(\n      lc.withContext('component', 'change-log-initializer'),\n      replicaFile,\n      {readonly: true},\n    );\n    try {\n      return readReplicaInitializationParameters(db);\n    } finally {\n      db.close();\n    }\n  };\n}\n\n/**\n * Whether the log holds the complete interval above `fromWatermark`, using the\n * same predicate catchup uses to decide `too-old`: the log both reaches back\n * that far and contains that transaction's `commit` row, so the next row after\n * it is the first change of the interval rather than the middle of a\n * transaction the log only partly holds.\n */\nfunction spansInterval(db: Database, fromWatermark: string): boolean {\n  if (!streamTableExists(db)) {\n    return false;\n  }\n  const row = db\n    .prepare(/*sql*/ `\n      SELECT\n        (SELECT min(\"watermark\") FROM \"${CHANGE_LOG_STREAM_TABLE}\")\n          AS \"minWatermark\",\n        coalesce((${SQLITE_CHANGE_LOG_BOUNDARY_SQL}), 0) AS \"boundaryExists\"\n    `)\n    .get<{minWatermark: string | null; boundaryExists: number}>({\n      fromWatermark,\n    });\n  return (\n    row.minWatermark !== null &&\n    fromWatermark >= row.minWatermark &&\n    row.boundaryExists === 1\n  );\n}\n\nconst SCHEMA_CHANGE_TAG_LIST = schemaChangeTags\n  .map(tag => `'${tag}'`)\n  .join(', ');\n\n/**\n * The log's schema changes over `(after, through]`, in stream order, or\n * `undefined` if the interval is too large to scan (see\n * {@link MAX_FOLD_SCAN_ROWS}).\n *\n * The tag is stored beside the verbatim change so this scan never parses the\n * payloads of ordinary data changes. The row count is still checked first so a\n * stalled replicator cannot make the range scan unbounded.\n *\n * The count is itself capped, at one row past the cap it is deciding. A bare\n * `count(*)` over the interval is `O(interval)`, so the unbounded interval this\n * exists to decline would still be walked in full before being declined —\n * which is the latency the cap is here to avoid, not a cheaper form of it. The\n * `LIMIT` makes the decision `O(MAX_FOLD_SCAN_ROWS)`, and the comparison is\n * unaffected: the subquery returns the true count whenever it is within the\n * cap, and `cap + 1` whenever it is not.\n */\nfunction readSchemaChanges(\n  db: Database,\n  after: string,\n  through: string,\n): SchemaChange[] | undefined {\n  const {rows} = db\n    .prepare(/*sql*/ `\n      SELECT count(*) AS \"rows\" FROM (\n        SELECT 1 FROM \"${CHANGE_LOG_STREAM_TABLE}\"\n          WHERE \"watermark\" > ? AND \"watermark\" <= ?\n          LIMIT ${MAX_FOLD_SCAN_ROWS + 1}\n      )\n    `)\n    .get<{rows: number}>(after, through);\n  if (rows > MAX_FOLD_SCAN_ROWS) {\n    return undefined;\n  }\n  return db\n    .prepare(/*sql*/ `\n      SELECT \"change\" FROM \"${CHANGE_LOG_STREAM_TABLE}\"\n        WHERE \"watermark\" > ? AND \"watermark\" <= ?\n          AND \"tag\" IN (${SCHEMA_CHANGE_TAG_LIST})\n        ORDER BY \"watermark\", \"pos\"\n    `)\n    .all<{change: string}>(after, through)\n    .map(({change}) => BigIntJSON.parse(change) as SchemaChange);\n}\n\nfunction streamTableExists(db: Database): boolean {\n  return (\n    db\n      .prepare(/*sql*/ `\n        SELECT 1 FROM \"sqlite_master\" WHERE \"type\" = 'table' AND \"name\" = ?\n      `)\n      .get<{1: number} | undefined>(CHANGE_LOG_STREAM_TABLE) !== undefined\n  );\n}\n\n/**\n * A canonical rendering of a whole cookie set: rows sorted by primary key,\n * object keys sorted at every depth.\n *\n * Both the equality check and the `divergent` log line fall out of this one\n * value, and neither can be defeated by a difference that is not a difference.\n * Postgres stores the cookie documents as `jsonb`, which does not preserve key\n * order, and the two readers order their rows in SQL in two engines under two\n * collations -- `storer.ts` pins `COLLATE \"C\"` for exactly that reason. Sorting\n * again here means an ordering or key-order regression can never be reported as\n * a divergence, which is a class of false alert that would cost more than it\n * caught.\n */\nfunction canonicalizeCookies(cookies: CookieSet): string {\n  return BigIntJSON.stringify({\n    tableMetadata: byKey(cookies.tableMetadata, c => [c.schema, c.table]).map(\n      c => [c.schema, c.table, sortKeys(c.metadata)],\n    ),\n    backfilling: byKey(cookies.backfilling, c => [\n      c.schema,\n      c.table,\n      c.column,\n    ]).map(c => [c.schema, c.table, c.column, sortKeys(c.backfill)]),\n  });\n}\n\nfunction byKey<T>(rows: readonly T[], key: (row: T) => string[]): T[] {\n  return rows\n    .map(row => [JSON.stringify(key(row)), row] as const)\n    .sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0))\n    .map(([, row]) => row);\n}\n\nfunction sortKeys(value: unknown): unknown {\n  if (Array.isArray(value)) {\n    return value.map(sortKeys);\n  }\n  if (value === null || typeof value !== 'object') {\n    return value;\n  }\n  return Object.fromEntries(\n    Object.entries(value)\n      .sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0))\n      .map(([k, v]) => [k, sortKeys(v)]),\n  );\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;AAqKA,IAAM,qBAAqB;AAE3B,IAAa,uBAAb,MAAkC;CAChC;CACA;CACA;CACA;CAEA,iBAA0B,mBACxB,WACA,yCACA,2HAEF;CAEA;;;;;CAMA;CAEA,YACE,IACA,EAAC,qBAAqB,mBACtB,SACA;EACA,OACE,uBAAuB,iBACvB,qEACF;EACA,KAAKA,MAAM,GAAG,YAAY,aAAa,wBAAwB;EAC/D,KAAKC,uBAAuB;EAC5B,KAAKC,mBAAmB;EACxB,KAAKC,WAAW;CAClB;;;;;;;CAQA,iBAAmD;EACjD,OAAO,KAAKE;CACd;;;;;;;;CASA,MAAM,aAAgD;EACpD,KAAKA,kBAAkB,KAAA;EACvB,MAAM,KAAK,KAAKJ,uBACZ,MAAM,KAAKK,aAAa,IACxB,KAAA;EAGJ,MAAM,UAAU,KAAKJ,mBACjB,KAAKK,aAAa,OAAO,KAAA,CAAS,IAClC,KAAA;EAEJ,MAAM,aAAa,KAAKJ,SAAS,mBAC/B,MAAM;GAAC,iBAAiB,GAAG;GAAe,SAAS,GAAG;EAAO,SACvD,cAAc,KAAK,SAAS,mCAAmC,CAAC,CACxE;EAEA,IAAI,IAAI;GACN,IAAI,SAIF,KAAKE,kBAAkB,KAAKG,SAAS,IAAI,OAAO;GAElD,OAAO;EACT;EAIA,MAAM,cACJ,cAAc,cAAc,KAAK,SAAS,0BAA0B,CAAC;EACvE,OAAO;GACL,eAAe,YAAY;GAC3B,SAAS,YAAY;GAGrB,kBAAkB,qBAAqB,YAAY,OAAO;EAC5D;CACF;;;;;;;;;;;;;;;;;;;;;;;;;CA0BA,MAAMF,eAAkD;EACtD,MAAM,KAAK,MAAM,KAAKH,SAAS,YAAY;EAC3C,MAAM,eAAe,KAAKA,SAAS;EACnC,IAAI,CAAC,cACH,OAAO;EAET,KAAKM,qBAAqB,KAAKN,SAAS,QAAQ;EAChD,MAAM,EAAC,eAAe,YAAW,KAAKM;EACtC,IAAI,GAAG,iBAAiB,eACtB,OAAO;EAET,MAAM,SAAS,MAAM,aAAa;GAAC,WAAW;GAAe;EAAO,CAAC;EACrE,KAAKT,IAAI,OACP,0BAA0B,GAAG,cAAc,4BACtC,cAAc,IAAI,QACzB;EAEA,OAAO,KAAKG,SAAS,YAAY;CACnC;;;;;CAMA,SACE,IACA,SACsB;EACtB,IAAI;EACJ,IAAI;GACF,SAAS,KAAKO,UAAU,IAAI,OAAO;EACrC,SAAS,GAAG;GACV,KAAKV,IAAI,OAAO,uDAAuD,CAAC;GACxE,SAAS;EACX;EACA,KAAKI,eAAe,IAAI,GAAG,EAAC,OAAM,CAAC;EACnC,KAAKJ,IAAI,QAAQ,8CAA8C,EAC7D,sBAAsB;GACpB;GACA,aAAa,GAAG;GAChB,kBAAkB,QAAQ;EAC5B,EACF,CAAC;EACD,OAAO;CACT;CAEA,aAAa,UAAyD;EACpE,IAAI;GACF,OAAO,KAAKG,SAAS,QAAQ;EAC/B,SAAS,GAAG;GACV,IAAI,UACF,MAAM;GAKR,KAAKH,IAAI,OACP,+DACA,CACF;GACA,KAAKI,eAAe,IAAI,GAAG,EAAC,QAAQ,QAAO,CAAC;GAC5C;EACF;CACF;CAEA,UACE,IACA,SACsB;EACtB,IAAI,QAAQ,gBAAgB,GAAG,eAI7B,OAAO;EAET,IAAI,QAAQ,kBAAkB,GAAG,eAC/B,OAAO,KAAKO,QAAQ,IAAI,SAAS,QAAQ,SAAS,OAAO;EAG3D,MAAM,KAAK,KAAKR,SAAS,UAAU;EACnC,IAAI,OAAO,KAAA,KAAa,CAAC,cAAc,IAAI,QAAQ,aAAa,GAC9D,OAAO;EAET,MAAM,UAAU,kBACd,IACA,QAAQ,eACR,GAAG,aACL;EACA,IAAI,YAAY,KAAA,GACd,OAAO;EAET,OAAO,KAAKQ,QACV,IACA,SACA,YAAY,QAAQ,SAAS,OAAO,GACpC,oBACA,QAAQ,MACV;CACF;CAEA,QACE,IACA,SACA,SACA,WACA,gBAAgB,GACM;EACtB,MAAM,WAAW,oBAAoB,GAAG,OAAO;EAC/C,MAAM,SAAS,oBAAoB,OAAO;EAC1C,IAAI,aAAa,QACf,OAAO;EAKT,KAAKX,IAAI,QACP,gEACA,EACE,sBAAsB;GACpB,aAAa,GAAG;GAChB,kBAAkB,QAAQ;GAC1B;GACA;GACA;EACF,EACF,CACF;EACA,OAAO;CACT;AACF;AAEA,IAAM,iBAAiB,EACrB,eACA,eACqD;CACrD,iBAAiB;CACjB;AACF;;;;;;AAOA,SAAgB,oCACd,IAC0B;CAC1B,MAAM,SAAS,IAAI,gBAAgB,EAAE;CACrC,OAAO,MAAM;CACb,IAAI;EACF,OAAO;GACL,eAAe,oBAAoB,MAAM,EAAE;GAC3C,kBAAkB,qBAAqB,EAAE;GACzC,SAAS,mBAAmB,EAAE;EAChC;CACF,UAAU;EACR,IAAI,GAAG,eACL,OAAO,SAAS;CAEpB;AACF;;;;;;;;;;;;;AAcA,SAAgB,4BACd,IACA,aACgC;CAChC,aAAa;EACX,MAAM,KAAK,IAAI,SACb,GAAG,YAAY,aAAa,wBAAwB,GACpD,aACA,EAAC,UAAU,KAAI,CACjB;EACA,IAAI;GACF,OAAO,oCAAoC,EAAE;EAC/C,UAAU;GACR,GAAG,MAAM;EACX;CACF;AACF;;;;;;;;AASA,SAAS,cAAc,IAAc,eAAgC;CACnE,IAAI,CAAC,kBAAkB,EAAE,GACvB,OAAO;CAET,MAAM,MAAM,GACT,QAAgB;;yCAEoB,wBAAwB;;oBAE7C,+BAA+B;KAC9C,EACA,IAA2D,EAC1D,cACF,CAAC;CACH,OACE,IAAI,iBAAiB,QACrB,iBAAiB,IAAI,gBACrB,IAAI,mBAAmB;AAE3B;AAEA,IAAM,yBAAyB,iBAC5B,KAAI,QAAO,IAAI,IAAI,EAAE,EACrB,KAAK,IAAI;;;;;;;;;;;;;;;;;;AAmBZ,SAAS,kBACP,IACA,OACA,SAC4B;CAC5B,MAAM,EAAC,SAAQ,GACZ,QAAgB;;yBAEI,wBAAwB;;kBAE/B,qBAAqB,EAAE;;KAEpC,EACA,IAAoB,OAAO,OAAO;CACrC,IAAI,OAAO,oBACT;CAEF,OAAO,GACJ,QAAgB;8BACS,wBAAwB;;0BAE5B,uBAAuB;;KAE5C,EACA,IAAsB,OAAO,OAAO,EACpC,KAAK,EAAC,aAAY,WAAW,MAAM,MAAM,CAAiB;AAC/D;AAEA,SAAS,kBAAkB,IAAuB;CAChD,OACE,GACG,QAAgB;;OAEhB,EACA,IAA6B,uBAAuB,MAAM,KAAA;AAEjE;;;;;;;;;;;;;;AAeA,SAAS,oBAAoB,SAA4B;CACvD,OAAO,WAAW,UAAU;EAC1B,eAAe,MAAM,QAAQ,gBAAe,MAAK,CAAC,EAAE,QAAQ,EAAE,KAAK,CAAC,EAAE,KACpE,MAAK;GAAC,EAAE;GAAQ,EAAE;GAAO,SAAS,EAAE,QAAQ;EAAC,CAC/C;EACA,aAAa,MAAM,QAAQ,cAAa,MAAK;GAC3C,EAAE;GACF,EAAE;GACF,EAAE;EACJ,CAAC,EAAE,KAAI,MAAK;GAAC,EAAE;GAAQ,EAAE;GAAO,EAAE;GAAQ,SAAS,EAAE,QAAQ;EAAC,CAAC;CACjE,CAAC;AACH;AAEA,SAAS,MAAS,MAAoB,KAAgC;CACpE,OAAO,KACJ,KAAI,QAAO,CAAC,KAAK,UAAU,IAAI,GAAG,CAAC,GAAG,GAAG,CAAU,EACnD,MAAM,CAAC,IAAI,CAAC,OAAQ,IAAI,IAAI,KAAK,IAAI,IAAI,IAAI,CAAE,EAC/C,KAAK,GAAG,SAAS,GAAG;AACzB;AAEA,SAAS,SAAS,OAAyB;CACzC,IAAI,MAAM,QAAQ,KAAK,GACrB,OAAO,MAAM,IAAI,QAAQ;CAE3B,IAAI,UAAU,QAAQ,OAAO,UAAU,UACrC,OAAO;CAET,OAAO,OAAO,YACZ,OAAO,QAAQ,KAAK,EACjB,MAAM,CAAC,IAAI,CAAC,OAAQ,IAAI,IAAI,KAAK,IAAI,IAAI,IAAI,CAAE,EAC/C,KAAK,CAAC,GAAG,OAAO,CAAC,GAAG,SAAS,CAAC,CAAC,CAAC,CACrC;AACF"}
+{"version":3,"file":"change-log-initializer.js","names":[],"sources":["../../../../../../zero-cache/src/services/change-streamer/change-log-initializer.ts"],"sourcesContent":["/**\n * Where a stream connection's initialization parameters come from.\n *\n * A change-streamer starts a stream from three things read at one position: the\n * watermark to resume at, the {@link BackfillRequest}s to hand the change\n * source, and the cookie set those requests were derived from. Postgres has\n * always supplied all three (`Storer.getStartStreamInitializationParameters()`).\n * The point of this workstream is that a restored replica can supply them\n * instead, which is what lets the `cdc` schema be decommissioned.\n *\n * This is the two-source stage. Both are computed on every stream connection,\n * compared, and Postgres's is used — so the replica-derived path runs in\n * production, at full rate, before anything depends on it. The flip is\n * `initFromPgChangeLog: false`, not a rewrite.\n *\n * The shape is {@link UpstreamAcker}'s, deliberately: two stores, one output, a\n * flag per store, and the Postgres flag carrying the marker that retires it.\n * That class solved the identical problem for upstream ACKs earlier in this\n * workstream.\n *\n * ### Why the comparison is a fold and not an equality\n *\n * The replica is a subscriber downstream of the change-streamer's write loop,\n * so its state version trails the log's head by however long the replicator is\n * behind. A raw difference between the two cookie sets is therefore the normal\n * case and says nothing. What is checked instead is that the replica's set,\n * carried forward over exactly the schema changes the log holds in the interval,\n * equals Postgres's set at Postgres's watermark. The log holds precisely those\n * transactions, schema changes are rare, and the check is exact rather than\n * approximate — so it also happens to be the strongest available test of the\n * premise that three interpreters of one fold agree.\n */\n\nimport type {LogContext} from '@rocicorp/logger';\nimport {assert} from '../../../../shared/src/asserts.ts';\nimport {BigIntJSON} from '../../../../shared/src/bigint-json.ts';\nimport {must} from '../../../../shared/src/must.ts';\nimport {Database} from '../../../../zqlite/src/db.ts';\nimport {StatementRunner} from '../../db/statements.ts';\nimport {getOrCreateCounter} from '../../observability/metrics.ts';\nimport type {SchemaChange} from '../change-source/protocol/current/data.ts';\nimport {schemaChangeTags} from '../change-source/protocol/current/schema-change-tags.ts';\nimport type {BackfillRequest} from '../change-source/protocol/current/upstream.ts';\nimport {\n  backfillRequestsFrom,\n  foldCookies,\n  type CookieSet,\n} from '../replicator/change-log-cookies.ts';\nimport {\n  CHANGE_LOG_STREAM_TABLE,\n  type ChangeLogResumePoint,\n} from '../replicator/change-log-db.ts';\nimport {\n  readBackfillRequests,\n  readReplicaCookies,\n} from '../replicator/schema/backfilling.ts';\nimport {getReplicationState} from '../replicator/schema/replication-state.ts';\nimport {SQLITE_CHANGE_LOG_BOUNDARY_SQL} from './sqlite-change-log-reader.ts';\n\n/** Everything one stream connection starts from, read at one position. */\nexport type InitializationParameters = {\n  readonly lastWatermark: string;\n  readonly backfillRequests: BackfillRequest[];\n  readonly cookies: CookieSet;\n};\n\n/**\n * - `equal` — the two stores were at the same watermark and held the same\n *   cookies. Nothing had to be folded, which is the strongest of the four.\n * - `equal-after-fold` — the replica trailed, and carrying its set forward over\n *   the log's schema changes in the interval reproduced Postgres's exactly.\n *   The expected steady-state result.\n * - `divergent` — the two disagree at the same position. Alert; investigate\n *   individually.\n * - `inconclusive` — the log does not span the interval, so the fold cannot be\n *   performed. Routine on a cold log, and not a finding.\n * - `error` — the comparison itself failed: the replica could not be read, or\n *   the fold threw. Not one of the four classifications, because it is a\n *   statement about this process rather than about the two stores. Kept\n *   separate rather than folded into `inconclusive` so that a change-streamer\n *   that has silently stopped being able to read its replica is visible before\n *   that read becomes the only source.\n */\nexport type InitComparisonResult =\n  | 'equal'\n  | 'equal-after-fold'\n  | 'divergent'\n  | 'inconclusive'\n  | 'error';\n\nexport type ChangeLogInitializerSources = {\n  /** `Storer.getStartStreamInitializationParameters()`. */\n  readonly pgChangeLog: () => Promise<InitializationParameters>;\n  /** Typically {@link replicaInitializationSource}. */\n  readonly replica: () => InitializationParameters;\n  /**\n   * Reconciles the SQLite change log and returns its current resume point.\n   *\n   * When `resumeFrom` is present, it supplies the resume point. Otherwise, a\n   * valid log uses its own head. A new or invalid log uses `seed`.\n   *\n   * Returns `undefined` when the writer is disabled or unavailable.\n   */\n  readonly reconcileChangeLog: (\n    resumeFrom: ChangeLogResumePoint | undefined,\n    seed: () => ChangeLogResumePoint,\n  ) => ChangeLogResumePoint | undefined;\n  /**\n   * The change log, for the fold. `undefined` before the stream loop's first\n   * reconcile has created it, and again once the writer has failed soft and\n   * deleted it — both of which make the comparison `inconclusive` rather than\n   * an error, since neither says anything about the two stores.\n   */\n  readonly changeLog: () => Database | undefined;\n  /**\n   * Re-initializes the Postgres change log from the replica's resume point\n   * (taking ownership of it) if it does not contain all of the replica's\n   * changes, re-checking under lock. Typically\n   * `Storer.reinitializeFromReplica()`.\n   *\n   * Supplied only when the change log's head is behind the position of the\n   * replication slot from which replication resumes, i.e. when it may be\n   * missing transactions that the slot will not stream. Otherwise, a replica\n   * that is ahead of the change log (as changes are forwarded before they are\n   * stored) is caught up by the change stream, which resumes from the change\n   * log's head.\n   */\n  readonly reinitializePgChangeLog?:\n    | ((params: {\n        watermark: string;\n        cookies: CookieSet;\n      }) => Promise<'reinitialized' | 'maintained'>)\n    | undefined;\n};\n\ntype Opts = {\n  /**\n   * When true, Postgres supplies the resume point and the cookies for a new\n   * change log.\n   *\n   * When false, a valid log resumes from its own head. The replica supplies the\n   * resume point for a new or invalid log.\n   */\n  // TODO: set false when retiring PG\n  readonly initFromPgChangeLog: boolean;\n  /**\n   * Derive the parameters from the replica. When Postgres is enabled, these\n   * parameters are used only for comparison.\n   *\n   * When Postgres is disabled, the replica supplies the seed and the fallback\n   * resume point.\n   */\n  readonly initFromReplica: boolean;\n};\n\n/**\n * The most change-log rows one comparison will scan.\n *\n * The interval is `(replicaWatermark, pgWatermark]`, so it is normally a\n * handful of transactions — but a replicator that has been stalled can leave it\n * arbitrarily large, and this runs on the stream loop between reconciliation and\n * `startStream`, where latency delays the resumption of replication. Over the\n * cap the comparison declines rather than pays: a chronically lagging replicator\n * shows up as `inconclusive`, which is charted, and not as a stall.\n */\nconst MAX_FOLD_SCAN_ROWS = 10_000;\n\nexport class ChangeLogInitializer {\n  readonly #lc: LogContext;\n  readonly #initFromPgChangeLog: boolean;\n  readonly #initFromReplica: boolean;\n  readonly #sources: ChangeLogInitializerSources;\n\n  readonly #resultCounter = getOrCreateCounter(\n    'replica',\n    'sqlite_change_log.init_compare_result',\n    'Comparisons of the replica-derived stream initialization parameters ' +\n      'against the Postgres-derived ones, by classification.',\n  );\n\n  #lastComparison: InitComparisonResult | undefined;\n\n  /**\n   * The (restored) replica's resume point that the Postgres change log must\n   * cover, read once. See {@link #pgChangeLog}.\n   */\n  #restoredReplica: InitializationParameters | undefined;\n\n  constructor(\n    lc: LogContext,\n    {initFromPgChangeLog, initFromReplica}: Opts,\n    sources: ChangeLogInitializerSources,\n  ) {\n    assert(\n      initFromPgChangeLog || initFromReplica,\n      `At least one of initFromPgChangeLog or initFromReplica must be true`,\n    );\n    this.#lc = lc.withContext('component', 'change-log-initializer');\n    this.#initFromPgChangeLog = initFromPgChangeLog;\n    this.#initFromReplica = initFromReplica;\n    this.#sources = sources;\n  }\n\n  /**\n   * Returns the comparison result from the most recent {@link initialize}\n   * call. Returns `undefined` when that call used only one source.\n   *\n   * Tests and metrics use this result. It does not affect initialization.\n   */\n  lastComparison(): InitComparisonResult | undefined {\n    return this.#lastComparison;\n  }\n\n  /**\n   * Returns the initialization parameters for one stream connection after it\n   * reconciles the change log.\n   *\n   * The selected source determines how reconciliation works. Keeping both\n   * operations here hides that choice from the caller.\n   */\n  async initialize(): Promise<InitializationParameters> {\n    this.#lastComparison = undefined;\n    const pg = this.#initFromPgChangeLog\n      ? await this.#pgChangeLog()\n      : undefined;\n    // The replica is required when Postgres is disabled. It seeds a new log and\n    // supplies a resume point when the writer is unavailable.\n    const replica = this.#initFromReplica\n      ? this.#readReplica(pg === undefined)\n      : undefined;\n\n    const reconciled = this.#sources.reconcileChangeLog(\n      pg && {resumeWatermark: pg.lastWatermark, cookies: pg.cookies},\n      () => resumePointOf(must(replica, 'no seed for the SQLite change log')),\n    );\n\n    if (pg) {\n      if (replica) {\n        // Compare only after reconciliation. The fold must read the schema\n        // changes from the replica watermark through the Postgres watermark.\n        // Reconciliation makes sure that the log contains that interval.\n        this.#lastComparison = this.#compare(pg, replica);\n      }\n      return pg;\n    }\n\n    // Use the reconciled log position. If the writer is unavailable, use the\n    // replica position.\n    const resumePoint =\n      reconciled ?? resumePointOf(must(replica, 'no initialization source'));\n    return {\n      lastWatermark: resumePoint.resumeWatermark,\n      cookies: resumePoint.cookies,\n      // Derive the requests from these cookies so that the requests and the\n      // watermark describe the same position.\n      backfillRequests: backfillRequestsFrom(resumePoint.cookies),\n    };\n  }\n\n  /**\n   * Reads the Postgres change log's initialization parameters, first\n   * re-initializing it from the replica (taking ownership of it) if it may be\n   * missing transactions that the replication slot will not stream, e.g. if it\n   * is no longer maintained after running in high-availability mode. This is\n   * the Postgres counterpart of seeding a new SQLite change log from the\n   * replica.\n   *\n   * This only applies when the change log's head is behind the slot's position\n   * (i.e. when {@link ChangeLogInitializerSources.reinitializePgChangeLog} is\n   * supplied), and the change log is only re-initialized if its head is also\n   * behind the replica's. Otherwise, it contains every transaction before the\n   * slot's position: one that was acked after being stored would be in the\n   * change log, and one that was acked after being backed up would be in the\n   * replica, which the change log's head covers.\n   *\n   * The replica is read only once, i.e. as it was restored from the backup,\n   * before any stream is started. The live replica is no reference: it is\n   * caught up from the forwarded changes, which can be ahead of what has been\n   * stored, e.g. by the change log's previous owner. Against that fixed\n   * resume point, re-initialization is idempotent: afterwards, the change log's\n   * head (which only advances) covers it, so subsequent stream connections\n   * leave it untouched.\n   */\n  async #pgChangeLog(): Promise<InitializationParameters> {\n    const pg = await this.#sources.pgChangeLog();\n    const reinitialize = this.#sources.reinitializePgChangeLog;\n    if (!reinitialize) {\n      return pg;\n    }\n    this.#restoredReplica ??= this.#sources.replica();\n    const {lastWatermark, cookies} = this.#restoredReplica;\n    if (pg.lastWatermark >= lastWatermark) {\n      return pg;\n    }\n    const result = await reinitialize({watermark: lastWatermark, cookies});\n    this.#lc.info?.(\n      `Postgres change log at ${pg.lastWatermark} is behind the replica at ` +\n        `${lastWatermark}: ${result}`,\n    );\n    // Either way, the change log has changed since it was read.\n    return this.#sources.pgChangeLog();\n  }\n\n  /**\n   * Compares both sources and records the result. Postgres remains\n   * authoritative when both sources are enabled.\n   */\n  #compare(\n    pg: InitializationParameters,\n    replica: InitializationParameters,\n  ): InitComparisonResult {\n    let result: InitComparisonResult;\n    try {\n      result = this.#classify(pg, replica);\n    } catch (e) {\n      this.#lc.warn?.('failed to compare change-log initialization sources', e);\n      result = 'error';\n    }\n    this.#resultCounter.add(1, {result});\n    this.#lc.debug?.('compared change-log initialization sources', {\n      changeLogInitCompare: {\n        result,\n        pgWatermark: pg.lastWatermark,\n        replicaWatermark: replica.lastWatermark,\n      },\n    });\n    return result;\n  }\n\n  #readReplica(required: boolean): InitializationParameters | undefined {\n    try {\n      return this.#sources.replica();\n    } catch (e) {\n      if (required) {\n        throw e;\n      }\n      // Observational until the flip, so a replica that cannot be read must not\n      // stop replication -- but it must not be quiet either, since the flip\n      // makes this read the only source.\n      this.#lc.warn?.(\n        'failed to derive change-log initialization from the replica',\n        e,\n      );\n      this.#resultCounter.add(1, {result: 'error'});\n      return undefined;\n    }\n  }\n\n  #classify(\n    pg: InitializationParameters,\n    replica: InitializationParameters,\n  ): InitComparisonResult {\n    if (replica.lastWatermark > pg.lastWatermark) {\n      // Transactions reach subscribers before the storer's commit, so a replica\n      // read taken while one is in flight can be a transaction ahead. The fold\n      // only runs forward, so there is nothing to be said either way.\n      return 'inconclusive';\n    }\n    if (replica.lastWatermark === pg.lastWatermark) {\n      return this.#report(pg, replica, replica.cookies, 'equal');\n    }\n\n    const db = this.#sources.changeLog();\n    if (db === undefined || !spansInterval(db, replica.lastWatermark)) {\n      return 'inconclusive';\n    }\n    const changes = readSchemaChanges(\n      db,\n      replica.lastWatermark,\n      pg.lastWatermark,\n    );\n    if (changes === undefined) {\n      return 'inconclusive'; // over MAX_FOLD_SCAN_ROWS\n    }\n    return this.#report(\n      pg,\n      replica,\n      foldCookies(replica.cookies, changes),\n      'equal-after-fold',\n      changes.length,\n    );\n  }\n\n  #report(\n    pg: InitializationParameters,\n    replica: InitializationParameters,\n    derived: CookieSet,\n    whenEqual: InitComparisonResult,\n    foldedChanges = 0,\n  ): InitComparisonResult {\n    const expected = canonicalizeCookies(pg.cookies);\n    const actual = canonicalizeCookies(derived);\n    if (expected === actual) {\n      return whenEqual;\n    }\n    // Both sets are normally empty and are at most a handful of rows, so they\n    // are logged whole: a divergence is investigated individually (`P§:10`)\n    // and the two renderings are the whole of what there is to look at.\n    this.#lc.error?.(\n      'the replica-derived change-log cookies do not match Postgres',\n      {\n        changeLogInitCompare: {\n          pgWatermark: pg.lastWatermark,\n          replicaWatermark: replica.lastWatermark,\n          foldedChanges,\n          expected,\n          actual,\n        },\n      },\n    );\n    return 'divergent';\n  }\n}\n\nconst resumePointOf = ({\n  lastWatermark,\n  cookies,\n}: InitializationParameters): ChangeLogResumePoint => ({\n  resumeWatermark: lastWatermark,\n  cookies,\n});\n\n/**\n * Reads the replica's initialization parameters in one snapshot, which is\n * invariant 15 on the replica side: the state version and the cookies are only\n * meaningful together.\n */\nexport function readReplicaInitializationParameters(\n  db: Database,\n): InitializationParameters {\n  const runner = new StatementRunner(db);\n  runner.begin(); // deferred, i.e. one read snapshot for the three statements\n  try {\n    return {\n      lastWatermark: getReplicationState(runner).stateVersion,\n      backfillRequests: readBackfillRequests(db),\n      cookies: readReplicaCookies(db),\n    };\n  } finally {\n    if (db.inTransaction) {\n      runner.rollback();\n    }\n  }\n}\n\n/**\n * The replica source for {@link ChangeLogInitializer}, opening and closing the\n * file per stream connection rather than holding a handle on a database another\n * process owns.\n *\n * The handle is read-only, which is invariant 19 stated in the API: the\n * replica's cookie tables are written by the change-processor and by nothing\n * else. Note that a read-only SQLite connection still needs to attach the\n * WAL's `-shm`, so this can fail if it opens at a moment when no writer holds\n * the replica; that surfaces as the `error` classification, which is the point\n * of running the path under observation before anything depends on it.\n */\nexport function replicaInitializationSource(\n  lc: LogContext,\n  replicaFile: string,\n): () => InitializationParameters {\n  return () => {\n    const db = new Database(\n      lc.withContext('component', 'change-log-initializer'),\n      replicaFile,\n      {readonly: true},\n    );\n    try {\n      return readReplicaInitializationParameters(db);\n    } finally {\n      db.close();\n    }\n  };\n}\n\n/**\n * Whether the log holds the complete interval above `fromWatermark`, using the\n * same predicate catchup uses to decide `too-old`: the log both reaches back\n * that far and contains that transaction's `commit` row, so the next row after\n * it is the first change of the interval rather than the middle of a\n * transaction the log only partly holds.\n */\nfunction spansInterval(db: Database, fromWatermark: string): boolean {\n  if (!streamTableExists(db)) {\n    return false;\n  }\n  const row = db\n    .prepare(/*sql*/ `\n      SELECT\n        (SELECT min(\"watermark\") FROM \"${CHANGE_LOG_STREAM_TABLE}\")\n          AS \"minWatermark\",\n        coalesce((${SQLITE_CHANGE_LOG_BOUNDARY_SQL}), 0) AS \"boundaryExists\"\n    `)\n    .get<{minWatermark: string | null; boundaryExists: number}>({\n      fromWatermark,\n    });\n  return (\n    row.minWatermark !== null &&\n    fromWatermark >= row.minWatermark &&\n    row.boundaryExists === 1\n  );\n}\n\nconst SCHEMA_CHANGE_TAG_LIST = schemaChangeTags\n  .map(tag => `'${tag}'`)\n  .join(', ');\n\n/**\n * The log's schema changes over `(after, through]`, in stream order, or\n * `undefined` if the interval is too large to scan (see\n * {@link MAX_FOLD_SCAN_ROWS}).\n *\n * The tag is stored beside the verbatim change so this scan never parses the\n * payloads of ordinary data changes. The row count is still checked first so a\n * stalled replicator cannot make the range scan unbounded.\n *\n * The count is itself capped, at one row past the cap it is deciding. A bare\n * `count(*)` over the interval is `O(interval)`, so the unbounded interval this\n * exists to decline would still be walked in full before being declined —\n * which is the latency the cap is here to avoid, not a cheaper form of it. The\n * `LIMIT` makes the decision `O(MAX_FOLD_SCAN_ROWS)`, and the comparison is\n * unaffected: the subquery returns the true count whenever it is within the\n * cap, and `cap + 1` whenever it is not.\n */\nfunction readSchemaChanges(\n  db: Database,\n  after: string,\n  through: string,\n): SchemaChange[] | undefined {\n  const {rows} = db\n    .prepare(/*sql*/ `\n      SELECT count(*) AS \"rows\" FROM (\n        SELECT 1 FROM \"${CHANGE_LOG_STREAM_TABLE}\"\n          WHERE \"watermark\" > ? AND \"watermark\" <= ?\n          LIMIT ${MAX_FOLD_SCAN_ROWS + 1}\n      )\n    `)\n    .get<{rows: number}>(after, through);\n  if (rows > MAX_FOLD_SCAN_ROWS) {\n    return undefined;\n  }\n  return db\n    .prepare(/*sql*/ `\n      SELECT \"change\" FROM \"${CHANGE_LOG_STREAM_TABLE}\"\n        WHERE \"watermark\" > ? AND \"watermark\" <= ?\n          AND \"tag\" IN (${SCHEMA_CHANGE_TAG_LIST})\n        ORDER BY \"watermark\", \"pos\"\n    `)\n    .all<{change: string}>(after, through)\n    .map(({change}) => BigIntJSON.parse(change) as SchemaChange);\n}\n\nfunction streamTableExists(db: Database): boolean {\n  return (\n    db\n      .prepare(/*sql*/ `\n        SELECT 1 FROM \"sqlite_master\" WHERE \"type\" = 'table' AND \"name\" = ?\n      `)\n      .get<{1: number} | undefined>(CHANGE_LOG_STREAM_TABLE) !== undefined\n  );\n}\n\n/**\n * A canonical rendering of a whole cookie set: rows sorted by primary key,\n * object keys sorted at every depth.\n *\n * Both the equality check and the `divergent` log line fall out of this one\n * value, and neither can be defeated by a difference that is not a difference.\n * Postgres stores the cookie documents as `jsonb`, which does not preserve key\n * order, and the two readers order their rows in SQL in two engines under two\n * collations -- `storer.ts` pins `COLLATE \"C\"` for exactly that reason. Sorting\n * again here means an ordering or key-order regression can never be reported as\n * a divergence, which is a class of false alert that would cost more than it\n * caught.\n */\nfunction canonicalizeCookies(cookies: CookieSet): string {\n  return BigIntJSON.stringify({\n    tableMetadata: byKey(cookies.tableMetadata, c => [c.schema, c.table]).map(\n      c => [c.schema, c.table, sortKeys(c.metadata)],\n    ),\n    backfilling: byKey(cookies.backfilling, c => [\n      c.schema,\n      c.table,\n      c.column,\n    ]).map(c => [c.schema, c.table, c.column, sortKeys(c.backfill)]),\n  });\n}\n\nfunction byKey<T>(rows: readonly T[], key: (row: T) => string[]): T[] {\n  return rows\n    .map(row => [JSON.stringify(key(row)), row] as const)\n    .sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0))\n    .map(([, row]) => row);\n}\n\nfunction sortKeys(value: unknown): unknown {\n  if (Array.isArray(value)) {\n    return value.map(sortKeys);\n  }\n  if (value === null || typeof value !== 'object') {\n    return value;\n  }\n  return Object.fromEntries(\n    Object.entries(value)\n      .sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0))\n      .map(([k, v]) => [k, sortKeys(v)]),\n  );\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;AAqKA,IAAM,qBAAqB;AAE3B,IAAa,uBAAb,MAAkC;CAChC;CACA;CACA;CACA;CAEA,iBAA0B,mBACxB,WACA,yCACA,2HAEF;CAEA;;;;;CAMA;CAEA,YACE,IACA,EAAC,qBAAqB,mBACtB,SACA;EACA,OACE,uBAAuB,iBACvB,qEACF;EACA,KAAK,MAAM,GAAG,YAAY,aAAa,wBAAwB;EAC/D,KAAK,uBAAuB;EAC5B,KAAK,mBAAmB;EACxB,KAAK,WAAW;CAClB;;;;;;;CAQA,iBAAmD;EACjD,OAAO,KAAK;CACd;;;;;;;;CASA,MAAM,aAAgD;EACpD,KAAK,kBAAkB,KAAA;EACvB,MAAM,KAAK,KAAK,uBACZ,MAAM,KAAK,aAAa,IACxB,KAAA;EAGJ,MAAM,UAAU,KAAK,mBACjB,KAAK,aAAa,OAAO,KAAA,CAAS,IAClC,KAAA;EAEJ,MAAM,aAAa,KAAK,SAAS,mBAC/B,MAAM;GAAC,iBAAiB,GAAG;GAAe,SAAS,GAAG;EAAO,SACvD,cAAc,KAAK,SAAS,mCAAmC,CAAC,CACxE;EAEA,IAAI,IAAI;GACN,IAAI,SAIF,KAAK,kBAAkB,KAAK,SAAS,IAAI,OAAO;GAElD,OAAO;EACT;EAIA,MAAM,cACJ,cAAc,cAAc,KAAK,SAAS,0BAA0B,CAAC;EACvE,OAAO;GACL,eAAe,YAAY;GAC3B,SAAS,YAAY;GAGrB,kBAAkB,qBAAqB,YAAY,OAAO;EAC5D;CACF;;;;;;;;;;;;;;;;;;;;;;;;;CA0BA,MAAM,eAAkD;EACtD,MAAM,KAAK,MAAM,KAAK,SAAS,YAAY;EAC3C,MAAM,eAAe,KAAK,SAAS;EACnC,IAAI,CAAC,cACH,OAAO;EAET,KAAK,qBAAqB,KAAK,SAAS,QAAQ;EAChD,MAAM,EAAC,eAAe,YAAW,KAAK;EACtC,IAAI,GAAG,iBAAiB,eACtB,OAAO;EAET,MAAM,SAAS,MAAM,aAAa;GAAC,WAAW;GAAe;EAAO,CAAC;EACrE,KAAK,IAAI,OACP,0BAA0B,GAAG,cAAc,4BACtC,cAAc,IAAI,QACzB;EAEA,OAAO,KAAK,SAAS,YAAY;CACnC;;;;;CAMA,SACE,IACA,SACsB;EACtB,IAAI;EACJ,IAAI;GACF,SAAS,KAAK,UAAU,IAAI,OAAO;EACrC,SAAS,GAAG;GACV,KAAK,IAAI,OAAO,uDAAuD,CAAC;GACxE,SAAS;EACX;EACA,KAAK,eAAe,IAAI,GAAG,EAAC,OAAM,CAAC;EACnC,KAAK,IAAI,QAAQ,8CAA8C,EAC7D,sBAAsB;GACpB;GACA,aAAa,GAAG;GAChB,kBAAkB,QAAQ;EAC5B,EACF,CAAC;EACD,OAAO;CACT;CAEA,aAAa,UAAyD;EACpE,IAAI;GACF,OAAO,KAAK,SAAS,QAAQ;EAC/B,SAAS,GAAG;GACV,IAAI,UACF,MAAM;GAKR,KAAK,IAAI,OACP,+DACA,CACF;GACA,KAAK,eAAe,IAAI,GAAG,EAAC,QAAQ,QAAO,CAAC;GAC5C;EACF;CACF;CAEA,UACE,IACA,SACsB;EACtB,IAAI,QAAQ,gBAAgB,GAAG,eAI7B,OAAO;EAET,IAAI,QAAQ,kBAAkB,GAAG,eAC/B,OAAO,KAAK,QAAQ,IAAI,SAAS,QAAQ,SAAS,OAAO;EAG3D,MAAM,KAAK,KAAK,SAAS,UAAU;EACnC,IAAI,OAAO,KAAA,KAAa,CAAC,cAAc,IAAI,QAAQ,aAAa,GAC9D,OAAO;EAET,MAAM,UAAU,kBACd,IACA,QAAQ,eACR,GAAG,aACL;EACA,IAAI,YAAY,KAAA,GACd,OAAO;EAET,OAAO,KAAK,QACV,IACA,SACA,YAAY,QAAQ,SAAS,OAAO,GACpC,oBACA,QAAQ,MACV;CACF;CAEA,QACE,IACA,SACA,SACA,WACA,gBAAgB,GACM;EACtB,MAAM,WAAW,oBAAoB,GAAG,OAAO;EAC/C,MAAM,SAAS,oBAAoB,OAAO;EAC1C,IAAI,aAAa,QACf,OAAO;EAKT,KAAK,IAAI,QACP,gEACA,EACE,sBAAsB;GACpB,aAAa,GAAG;GAChB,kBAAkB,QAAQ;GAC1B;GACA;GACA;EACF,EACF,CACF;EACA,OAAO;CACT;AACF;AAEA,IAAM,iBAAiB,EACrB,eACA,eACqD;CACrD,iBAAiB;CACjB;AACF;;;;;;AAOA,SAAgB,oCACd,IAC0B;CAC1B,MAAM,SAAS,IAAI,gBAAgB,EAAE;CACrC,OAAO,MAAM;CACb,IAAI;EACF,OAAO;GACL,eAAe,oBAAoB,MAAM,CAAC,CAAC;GAC3C,kBAAkB,qBAAqB,EAAE;GACzC,SAAS,mBAAmB,EAAE;EAChC;CACF,UAAU;EACR,IAAI,GAAG,eACL,OAAO,SAAS;CAEpB;AACF;;;;;;;;;;;;;AAcA,SAAgB,4BACd,IACA,aACgC;CAChC,aAAa;EACX,MAAM,KAAK,IAAI,SACb,GAAG,YAAY,aAAa,wBAAwB,GACpD,aACA,EAAC,UAAU,KAAI,CACjB;EACA,IAAI;GACF,OAAO,oCAAoC,EAAE;EAC/C,UAAU;GACR,GAAG,MAAM;EACX;CACF;AACF;;;;;;;;AASA,SAAS,cAAc,IAAc,eAAgC;CACnE,IAAI,CAAC,kBAAkB,EAAE,GACvB,OAAO;CAET,MAAM,MAAM,GACT,QAAgB;;yCAEoB,wBAAwB;;oBAE7C,+BAA+B;KAC9C,CAAC,CACD,IAA2D,EAC1D,cACF,CAAC;CACH,OACE,IAAI,iBAAiB,QACrB,iBAAiB,IAAI,gBACrB,IAAI,mBAAmB;AAE3B;AAEA,IAAM,yBAAyB,iBAC5B,KAAI,QAAO,IAAI,IAAI,EAAE,CAAC,CACtB,KAAK,IAAI;;;;;;;;;;;;;;;;;;AAmBZ,SAAS,kBACP,IACA,OACA,SAC4B;CAC5B,MAAM,EAAC,SAAQ,GACZ,QAAgB;;yBAEI,wBAAwB;;;;KAI5C,CAAC,CACD,IAAoB,OAAO,OAAO;CACrC,IAAI,OAAO,oBACT;CAEF,OAAO,GACJ,QAAgB;8BACS,wBAAwB;;0BAE5B,uBAAuB;;KAE5C,CAAC,CACD,IAAsB,OAAO,OAAO,CAAC,CACrC,KAAK,EAAC,aAAY,WAAW,MAAM,MAAM,CAAiB;AAC/D;AAEA,SAAS,kBAAkB,IAAuB;CAChD,OACE,GACG,QAAgB;;OAEhB,CAAC,CACD,IAA6B,uBAAuB,MAAM,KAAA;AAEjE;;;;;;;;;;;;;;AAeA,SAAS,oBAAoB,SAA4B;CACvD,OAAO,WAAW,UAAU;EAC1B,eAAe,MAAM,QAAQ,gBAAe,MAAK,CAAC,EAAE,QAAQ,EAAE,KAAK,CAAC,CAAC,CAAC,KACpE,MAAK;GAAC,EAAE;GAAQ,EAAE;GAAO,SAAS,EAAE,QAAQ;EAAC,CAC/C;EACA,aAAa,MAAM,QAAQ,cAAa,MAAK;GAC3C,EAAE;GACF,EAAE;GACF,EAAE;EACJ,CAAC,CAAC,CAAC,KAAI,MAAK;GAAC,EAAE;GAAQ,EAAE;GAAO,EAAE;GAAQ,SAAS,EAAE,QAAQ;EAAC,CAAC;CACjE,CAAC;AACH;AAEA,SAAS,MAAS,MAAoB,KAAgC;CACpE,OAAO,KACJ,KAAI,QAAO,CAAC,KAAK,UAAU,IAAI,GAAG,CAAC,GAAG,GAAG,CAAU,CAAC,CACpD,MAAM,CAAC,IAAI,CAAC,OAAQ,IAAI,IAAI,KAAK,IAAI,IAAI,IAAI,CAAE,CAAC,CAChD,KAAK,GAAG,SAAS,GAAG;AACzB;AAEA,SAAS,SAAS,OAAyB;CACzC,IAAI,MAAM,QAAQ,KAAK,GACrB,OAAO,MAAM,IAAI,QAAQ;CAE3B,IAAI,UAAU,QAAQ,OAAO,UAAU,UACrC,OAAO;CAET,OAAO,OAAO,YACZ,OAAO,QAAQ,KAAK,CAAC,CAClB,MAAM,CAAC,IAAI,CAAC,OAAQ,IAAI,IAAI,KAAK,IAAI,IAAI,IAAI,CAAE,CAAC,CAChD,KAAK,CAAC,GAAG,OAAO,CAAC,GAAG,SAAS,CAAC,CAAC,CAAC,CACrC;AACF"}

package/out/zero-cache/src/services/change-streamer/change-log-transaction-hash.js.map

@@ -1 +1 @@
-{"version":3,"file":"change-log-transaction-hash.js","names":["#hash"],"sources":["../../../../../../zero-cache/src/services/change-streamer/change-log-transaction-hash.ts"],"sourcesContent":["import {createHash, type Hash} from 'node:crypto';\n\nexport type ChangeLogHashRow = {\n  readonly watermark: string;\n  readonly pos: number;\n  readonly tag: string;\n  readonly change: string;\n  readonly precommit: string | null;\n};\n\n/**\n * Computes a stable digest over the fields shared by the Postgres and SQLite\n * change logs. Values are length-prefixed so different field boundaries cannot\n * produce the same byte stream.\n */\nexport class ChangeLogTransactionHasher {\n  readonly #hash: Hash;\n\n  constructor() {\n    this.#hash = createHash('sha256');\n    writeString(this.#hash, 'zero-change-log-transaction-v1');\n  }\n\n  add(row: ChangeLogHashRow): void {\n    writeString(this.#hash, row.watermark);\n    writeString(this.#hash, String(row.pos));\n    writeString(this.#hash, row.tag);\n    writeString(this.#hash, row.change);\n    if (row.precommit === null) {\n      this.#hash.update(Uint8Array.of(0));\n    } else {\n      this.#hash.update(Uint8Array.of(1));\n      writeString(this.#hash, row.precommit);\n    }\n  }\n\n  digest(): string {\n    return this.#hash.digest('hex');\n  }\n}\n\nfunction writeString(hash: Hash, value: string): void {\n  const bytes = Buffer.from(value);\n  const length = Buffer.allocUnsafe(4);\n  length.writeUInt32BE(bytes.byteLength);\n  hash.update(length);\n  hash.update(bytes);\n}\n"],"mappings":";;;;;;;AAeA,IAAa,6BAAb,MAAwC;CACtC;CAEA,cAAc;EACZ,KAAKA,QAAQ,WAAW,QAAQ;EAChC,YAAY,KAAKA,OAAO,gCAAgC;CAC1D;CAEA,IAAI,KAA6B;EAC/B,YAAY,KAAKA,OAAO,IAAI,SAAS;EACrC,YAAY,KAAKA,OAAO,OAAO,IAAI,GAAG,CAAC;EACvC,YAAY,KAAKA,OAAO,IAAI,GAAG;EAC/B,YAAY,KAAKA,OAAO,IAAI,MAAM;EAClC,IAAI,IAAI,cAAc,MACpB,KAAKA,MAAM,OAAO,WAAW,GAAG,CAAC,CAAC;OAC7B;GACL,KAAKA,MAAM,OAAO,WAAW,GAAG,CAAC,CAAC;GAClC,YAAY,KAAKA,OAAO,IAAI,SAAS;EACvC;CACF;CAEA,SAAiB;EACf,OAAO,KAAKA,MAAM,OAAO,KAAK;CAChC;AACF;AAEA,SAAS,YAAY,MAAY,OAAqB;CACpD,MAAM,QAAQ,OAAO,KAAK,KAAK;CAC/B,MAAM,SAAS,OAAO,YAAY,CAAC;CACnC,OAAO,cAAc,MAAM,UAAU;CACrC,KAAK,OAAO,MAAM;CAClB,KAAK,OAAO,KAAK;AACnB"}
+{"version":3,"file":"change-log-transaction-hash.js","names":[],"sources":["../../../../../../zero-cache/src/services/change-streamer/change-log-transaction-hash.ts"],"sourcesContent":["import {createHash, type Hash} from 'node:crypto';\n\nexport type ChangeLogHashRow = {\n  readonly watermark: string;\n  readonly pos: number;\n  readonly tag: string;\n  readonly change: string;\n  readonly precommit: string | null;\n};\n\n/**\n * Computes a stable digest over the fields shared by the Postgres and SQLite\n * change logs. Values are length-prefixed so different field boundaries cannot\n * produce the same byte stream.\n */\nexport class ChangeLogTransactionHasher {\n  readonly #hash: Hash;\n\n  constructor() {\n    this.#hash = createHash('sha256');\n    writeString(this.#hash, 'zero-change-log-transaction-v1');\n  }\n\n  add(row: ChangeLogHashRow): void {\n    writeString(this.#hash, row.watermark);\n    writeString(this.#hash, String(row.pos));\n    writeString(this.#hash, row.tag);\n    writeString(this.#hash, row.change);\n    if (row.precommit === null) {\n      this.#hash.update(Uint8Array.of(0));\n    } else {\n      this.#hash.update(Uint8Array.of(1));\n      writeString(this.#hash, row.precommit);\n    }\n  }\n\n  digest(): string {\n    return this.#hash.digest('hex');\n  }\n}\n\nfunction writeString(hash: Hash, value: string): void {\n  const bytes = Buffer.from(value);\n  const length = Buffer.allocUnsafe(4);\n  length.writeUInt32BE(bytes.byteLength);\n  hash.update(length);\n  hash.update(bytes);\n}\n"],"mappings":";;;;;;;AAeA,IAAa,6BAAb,MAAwC;CACtC;CAEA,cAAc;EACZ,KAAK,QAAQ,WAAW,QAAQ;EAChC,YAAY,KAAK,OAAO,gCAAgC;CAC1D;CAEA,IAAI,KAA6B;EAC/B,YAAY,KAAK,OAAO,IAAI,SAAS;EACrC,YAAY,KAAK,OAAO,OAAO,IAAI,GAAG,CAAC;EACvC,YAAY,KAAK,OAAO,IAAI,GAAG;EAC/B,YAAY,KAAK,OAAO,IAAI,MAAM;EAClC,IAAI,IAAI,cAAc,MACpB,KAAK,MAAM,OAAO,WAAW,GAAG,CAAC,CAAC;OAC7B;GACL,KAAK,MAAM,OAAO,WAAW,GAAG,CAAC,CAAC;GAClC,YAAY,KAAK,OAAO,IAAI,SAAS;EACvC;CACF;CAEA,SAAiB;EACf,OAAO,KAAK,MAAM,OAAO,KAAK;CAChC;AACF;AAEA,SAAS,YAAY,MAAY,OAAqB;CACpD,MAAM,QAAQ,OAAO,KAAK,KAAK;CAC/B,MAAM,SAAS,OAAO,YAAY,CAAC;CACnC,OAAO,cAAc,MAAM,UAAU;CACrC,KAAK,OAAO,MAAM;CAClB,KAAK,OAAO,KAAK;AACnB"}

package/out/zero-cache/src/services/change-streamer/change-streamer-http.js

@@ -224,7 +224,9 @@ var ChangeStreamerHttpClient = class {
 	* replication-manager is the one that serves the subscription.
 	*/
 	async connect(taskID) {
-		const ws = new WebSocket$1(await this.#resolveChangeStreamer(SUBSCRIBE_PATH) + `?${new URLSearchParams({ taskID }).toString()}`, { handshakeTimeout: CONNECT_TIMEOUT_MS });
+		const uri = await this.#resolveChangeStreamer(SUBSCRIBE_PATH);
+		const params = new URLSearchParams({ taskID });
+		const ws = new WebSocket$1(uri + `?${params.toString()}`, { handshakeTimeout: CONNECT_TIMEOUT_MS });
 		const outbound = Subscription.create();
 		return {
 			instream: await streamInternalWithSize(this.#lc, ws, subscribeDownstreamSchema, outbound),
@@ -267,6 +269,6 @@ function toStartSubscriptionContext(ctx) {
 	};
 }
 //#endregion
-export { ChangeStreamerHttpClient, ChangeStreamerHttpServer };
+export { ChangeStreamerHttpClient, ChangeStreamerHttpServer, getSubscriberContext };
 
 //# sourceMappingURL=change-streamer-http.js.map

package/out/zero-cache/src/services/change-streamer/change-streamer-http.js.map

@@ -1 +1 @@
-{"version":3,"file":"change-streamer-http.js","names":["#lc","#opts","#changeStreamer","#changes","#reserveSnapshot","#subscribe","#receiveWebsocket","#ensureChangeStreamerStarted","#forwardReservation","#changeStreamerStarted","#shardID","#changeDB","#changeStreamerURI","#resolveChangeStreamer"],"sources":["../../../../../../zero-cache/src/services/change-streamer/change-streamer-http.ts"],"sourcesContent":["import type {IncomingMessage} from 'node:http';\nimport websocket from '@fastify/websocket';\nimport type {LogContext} from '@rocicorp/logger';\nimport WebSocket from 'ws';\nimport {BigIntJSON} from '../../../../shared/src/bigint-json.ts';\nimport {promiseVoid} from '../../../../shared/src/resolved-promises.ts';\nimport type {NormalizedZeroConfig} from '../../config/normalize.ts';\nimport type {IncomingMessageSubset} from '../../types/http.ts';\nimport {pgClient, type PostgresDB} from '../../types/pg.ts';\nimport {type Worker} from '../../types/processes.ts';\nimport {type ShardID} from '../../types/shards.ts';\nimport {\n  readHead,\n  streamInternal,\n  streamInternalStringified,\n  streamInternalWithSize,\n  type Sized,\n  type Source,\n} from '../../types/streams.ts';\nimport {Subscription} from '../../types/subscription.ts';\nimport {URLParams} from '../../types/url-params.ts';\nimport {installWebSocketReceiver} from '../../types/websocket-handoff.ts';\nimport {closeWithError, PROTOCOL_ERROR} from '../../types/ws.ts';\nimport {HttpService, type Options as HttpOptions} from '../http-service.ts';\nimport {handleProfzRequest} from '../profz.ts';\nimport type {PreSerializedBatch} from './broadcast.ts';\nimport {\n  PROTOCOL_VERSION,\n  type ChangeStreamer,\n  type ChangeStreamerService,\n  type SubscriberContext,\n} from './change-streamer.ts';\nimport {discoverChangeStreamerAddress} from './schema/tables.ts';\nimport {type SnapshotMessage} from './snapshot-message.ts';\nimport {\n  type LocalReplica,\n  subscribeDownstreamSchema,\n  subscribeUpstreamSchema,\n  type ReservedMessage,\n  type SnapshotStatus,\n  type SubscribeContext,\n  type SubscribeDownstream,\n  type SubscribeUpstream,\n} from './subscribe.ts';\n\nconst MIN_SUPPORTED_PROTOCOL_VERSION = 4;\n\n// The merged `/subscribe` protocol was introduced at v7.\nconst MIN_SUBSCRIBE_PROTOCOL_VERSION = 7;\n\nconst SNAPSHOT_PATH_PATTERN = '/replication/:version/snapshot';\nconst CHANGES_PATH_PATTERN = '/replication/:version/changes';\nconst SUBSCRIBE_PATH_PATTERN = '/replication/:version/subscribe';\nconst PATH_REGEX =\n  /\\/replication\\/v(?<version>\\d+)\\/(changes|snapshot|subscribe)$/;\n\nconst SUBSCRIBE_PATH = `/replication/v${PROTOCOL_VERSION}/subscribe`;\n\n// Max time for a client to establish the /subscribe websocket (TCP connect\n// plus upgrade) before abandoning the attempt.\nconst CONNECT_TIMEOUT_MS = 10_000;\n\ntype Options = HttpOptions & {\n  startupDelayMs: number;\n  config?:\n    | Pick<NormalizedZeroConfig, 'adminPassword' | 'operatorPassword'>\n    | undefined;\n  getProfileWorker?: (() => Promise<Worker>) | undefined;\n};\n\nexport class ChangeStreamerHttpServer extends HttpService {\n  readonly id = 'change-streamer-http-server';\n  readonly #lc: LogContext;\n  readonly #opts: Options;\n  readonly #changeStreamer: ChangeStreamerService;\n\n  constructor(\n    lc: LogContext,\n    opts: Options,\n    parent: Worker,\n    changeStreamer: ChangeStreamerService,\n  ) {\n    super('change-streamer-http-server', lc, opts, async fastify => {\n      await fastify.register(websocket);\n\n      fastify.get(CHANGES_PATH_PATTERN, {websocket: true}, this.#changes);\n      fastify.get(\n        SNAPSHOT_PATH_PATTERN,\n        {websocket: true},\n        this.#reserveSnapshot,\n      );\n      fastify.get(SUBSCRIBE_PATH_PATTERN, {websocket: true}, this.#subscribe);\n\n      fastify.get('/profz', (req, res) =>\n        handleProfzRequest(\n          lc,\n          opts.config ?? {\n            adminPassword: undefined,\n            operatorPassword: undefined,\n          },\n          req,\n          res,\n          opts.getProfileWorker,\n          undefined,\n          'change-streamer',\n        ),\n      );\n\n      installWebSocketReceiver<'snapshot' | 'changes' | 'subscribe'>(\n        lc,\n        fastify.websocketServer,\n        this.#receiveWebsocket,\n        parent,\n      );\n    });\n\n    this.#lc = lc;\n    this.#opts = opts;\n    this.#changeStreamer = changeStreamer;\n  }\n\n  // Called when receiving a web socket via the main dispatcher handoff.\n  readonly #receiveWebsocket = (\n    ws: WebSocket,\n    action: 'changes' | 'snapshot' | 'subscribe',\n    msg: IncomingMessageSubset,\n  ) => {\n    switch (action) {\n      case 'snapshot':\n        return this.#reserveSnapshot(ws, msg);\n      case 'changes':\n        return this.#changes(ws, msg);\n      case 'subscribe':\n        return this.#subscribe(ws, msg);\n      default:\n        closeWithError(\n          this._lc,\n          ws,\n          `invalid action \"${action}\" received in handoff`,\n        );\n        return;\n    }\n  };\n\n  readonly #reserveSnapshot = async (ws: WebSocket, req: RequestHeaders) => {\n    this.#ensureChangeStreamerStarted('incoming snapshot reservation');\n    try {\n      const url = new URL(\n        req.url ?? '',\n        req.headers.origin ?? 'http://localhost',\n      );\n      checkProtocolVersion(url.pathname);\n      const taskID = url.searchParams.get('taskID');\n      if (!taskID) {\n        throw new Error('Missing taskID in snapshot request');\n      }\n      const downstream =\n        await this.#changeStreamer.startSnapshotReservation(taskID);\n      // Send-only for now: the reservation carries no inbound application\n      // messages (only transport acks, demuxed internally).\n      void streamInternal(this._lc, ws, undefined, downstream);\n    } catch (err) {\n      closeWithError(this._lc, ws, err, PROTOCOL_ERROR);\n    }\n  };\n\n  readonly #changes = async (ws: WebSocket, req: RequestHeaders) => {\n    try {\n      const ctx = getSubscriberContext(req);\n      if (ctx.mode === 'serving') {\n        this.#ensureChangeStreamerStarted('incoming subscription');\n      }\n\n      const downstream = await this.#changeStreamer.subscribe(ctx);\n      // Send-only for now: the subscription carries no inbound application\n      // messages (only transport acks, demuxed internally).\n      void streamInternalStringified(this._lc, ws, undefined, downstream, {\n        batched: ctx.wsBatched,\n      });\n    } catch (err) {\n      closeWithError(this._lc, ws, err, PROTOCOL_ERROR);\n    }\n  };\n\n  /**\n   * The merged v7 `/subscribe` handler. A single bidirectional connection\n   * carries, in sequence: an optional reservation phase (`reserve-snapshot` →\n   * `reserved`) and then the subscription phase (`start-subscription` → change\n   * stream). This ensures that the subscriber subscribes to the same task from\n   * which it restored the backup.\n   */\n  readonly #subscribe = async (ws: WebSocket, req: RequestHeaders) => {\n    let reservation: Source<SnapshotMessage> | undefined;\n    const outbound = Subscription.create<string | PreSerializedBatch>();\n\n    try {\n      const url = new URL(\n        req.url ?? '',\n        req.headers.origin ?? 'http://localhost',\n      );\n      const protocolVersion = checkProtocolVersion(url.pathname);\n      if (protocolVersion < MIN_SUBSCRIBE_PROTOCOL_VERSION) {\n        throw new Error(\n          `the /subscribe endpoint requires protocol v${MIN_SUBSCRIBE_PROTOCOL_VERSION} ` +\n            `(client is at v${protocolVersion})`,\n        );\n      }\n\n      const instream = await streamInternalStringified(\n        this._lc,\n        ws,\n        subscribeUpstreamSchema,\n        outbound,\n        {batched: true},\n      );\n\n      // Keep reading upstream control messages for the life of the connection.\n      // Do not manually exit loop, as that would close the socket. The loop\n      // exits naturally when the connection closes.\n      for await (const msg of instream) {\n        if (msg[0] === 'reserve-snapshot') {\n          const {taskID, localReplica} = msg[1];\n          this.#ensureChangeStreamerStarted('incoming snapshot reservation');\n          reservation = await this.#changeStreamer.startSnapshotReservation(\n            taskID,\n            {localReplica},\n          );\n          void this.#forwardReservation(reservation, outbound);\n        } else if (msg[0] === 'start-subscription') {\n          const ctx: SubscriberContext = {\n            ...msg[1],\n            protocolVersion,\n            wsBatched: true,\n          };\n          if (ctx.mode === 'serving') {\n            this.#ensureChangeStreamerStarted('incoming subscription');\n          }\n          // subscribe() registers the subscriber and then releases any\n          // reservation for that task.\n          reservation = undefined;\n          await this.#changeStreamer.subscribe(ctx, outbound);\n        }\n      }\n    } catch (err) {\n      closeWithError(this._lc, ws, err, PROTOCOL_ERROR);\n    } finally {\n      // Covers a reservation-phase disconnect (instream ended before a\n      // subscription started); idempotent with the close handler above.\n      reservation?.cancel();\n    }\n  };\n\n  /**\n   * Forwards a snapshot reservation's single `['status', ...]` message onto\n   * the merged connection's `outbound` sink as a `['reserved', ...]` message.\n   * The loop then blocks on the held-open reservation until it is cancelled,\n   * by `subscribe()` at the subscription handoff, or by the reservation-phase\n   * teardown on disconnect.\n   */\n  async #forwardReservation(\n    reservation: Source<SnapshotMessage>,\n    outbound: Subscription<string | PreSerializedBatch>,\n  ) {\n    try {\n      for await (const [, status] of reservation) {\n        // Translate the /snapshot protocol's ['status', {tag: 'status', ...}]\n        // message to the /subscribe protocol's ['reserved', {tag: 'snapshot', ...}].\n        outbound.push(\n          BigIntJSON.stringify([\n            'reserved',\n            {...status, tag: 'snapshot'},\n          ] satisfies ReservedMessage),\n        );\n      }\n    } catch (e) {\n      this._lc.warn?.(`error forwarding snapshot reservation`, e);\n      outbound.fail(e instanceof Error ? e : new Error(String(e)));\n    }\n  }\n\n  #changeStreamerStarted = false;\n\n  #ensureChangeStreamerStarted(reason?: string) {\n    if (!this.#changeStreamerStarted && this._state.shouldRun()) {\n      this.#lc.info?.(\n        `starting ChangeStreamerService ${reason ? `(${reason})` : ''}`,\n      );\n      void this.#changeStreamer\n        .run()\n        .catch(e =>\n          this.#lc.warn?.(`ChangeStreamerService ended with error`, e),\n        )\n        .finally(() => this.stop());\n\n      this.#changeStreamerStarted = true;\n    }\n  }\n\n  protected override _onStart(): void {\n    const {startupDelayMs, readinessGate = promiseVoid} = this.#opts;\n    if (startupDelayMs === 0) {\n      // In RMv2, there is no need to delay starting the change streamer\n      // because RM startup is non-disruptive.\n      this.#ensureChangeStreamerStarted();\n    } else {\n      // In RMv1, starting the change-streamer forcibly shuts down the\n      // previous change-streamer, causing view-syncers to reconnect.\n      // If this replication-manager has just started, the routing layer may\n      // not have registered it with DNS, as that only happens after it\n      // confirms health checks. To minimize downtime, the takeover is\n      // delayed for the configured startupDelayMs _after_ beginning to\n      // advertise readiness.\n      void readinessGate.then(() => {\n        this.#lc.info?.(\n          `waiting ${startupDelayMs}ms before taking over the change log`,\n        );\n        this._state.setTimeout(\n          () =>\n            this.#ensureChangeStreamerStarted(\n              `startup delay elapsed (${startupDelayMs} ms)`,\n            ),\n          startupDelayMs,\n        );\n      });\n    }\n  }\n\n  /**\n   * Signals (drainable) subscribers to switch over to another\n   * replication-manager, resolving when they have disconnected. Subscribers\n   * continue to be served in the meantime, but new connections (e.g.\n   * switchover attempts, reconnects, and snapshot reservations) are refused,\n   * so that they are routed to another replication-manager.\n   */\n  async drain(): Promise<void> {\n    this.stopAccepting();\n    if (this.#changeStreamerStarted && this.#changeStreamer.drain) {\n      await this.#changeStreamer.drain();\n    }\n  }\n\n  protected override async _onStop(): Promise<void> {\n    if (this.#changeStreamerStarted) {\n      await this.#changeStreamer.stop();\n    }\n  }\n}\n\nexport interface SnapshotReserver {\n  reserveSnapshot(\n    taskID: string,\n    localReplica?: LocalReplica | undefined,\n  ): Promise<{\n    reserved: SnapshotStatus;\n    followup: ReservationFollowup;\n  }>;\n}\n\nexport interface SubscriptionStarter {\n  subscribe(\n    ctx: SubscriberContext,\n  ): Promise<Source<Sized<SubscribeDownstream>>>;\n}\n\nexport interface ReservationFollowup extends SubscriptionStarter {\n  /**\n   * A signal to indicate that the followup is no longer valid (i.e. the\n   * connection to the reserving task has been severed).\n   */\n  readonly signal: AbortSignal;\n\n  /**\n   * Abandons the reservation without subscribing, closing the connection that\n   * was held open to pin the change-log floor. Used when the caller cannot\n   * proceed to `subscribe()` (e.g. process shutdown while a restore is in\n   * progress) and must not leave the reservation's connection open forever.\n   */\n  cancel(reason?: Error): void;\n}\n\nexport class ChangeStreamerHttpClient\n  implements ChangeStreamer, SnapshotReserver, SubscriptionStarter\n{\n  readonly #lc: LogContext;\n  readonly #shardID: ShardID;\n  readonly #changeDB: PostgresDB;\n  readonly #changeStreamerURI: string | undefined;\n\n  constructor(\n    lc: LogContext,\n    shardID: ShardID,\n    changeDB: string,\n    changeStreamerURI: string | undefined,\n  ) {\n    this.#lc = lc;\n    this.#shardID = shardID;\n    // Create a pg client with a single short-lived connection for the purpose\n    // of change-streamer discovery (i.e. ChangeDB as DNS).\n    this.#changeDB = pgClient(lc, changeDB, 'change-streamer-discovery', {\n      max: 1,\n      ['idle_timeout']: 15,\n    });\n    this.#changeStreamerURI = changeStreamerURI;\n  }\n\n  async #resolveChangeStreamer(path: string) {\n    let baseURL = this.#changeStreamerURI;\n    if (!baseURL) {\n      const address = await discoverChangeStreamerAddress(\n        this.#shardID,\n        this.#changeDB,\n      );\n      if (!address) {\n        throw new Error(`no change-streamer is running`);\n      }\n      baseURL = address.includes('://') ? `${address}/` : `ws://${address}/`;\n    }\n    const uri = new URL(path, baseURL);\n    this.#lc.info?.(`connecting to change-streamer@${uri}`);\n    return uri;\n  }\n\n  async reserveSnapshot(\n    taskID: string,\n    localReplica?: LocalReplica | undefined,\n  ): Promise<{\n    reserved: SnapshotStatus;\n    followup: ReservationFollowup;\n  }> {\n    const {instream, outbound} = await this.connect(taskID);\n    outbound.push([\n      'reserve-snapshot',\n      {taskID, ...(localReplica ? {localReplica} : {})},\n    ]);\n\n    const {head, rest} = readHead(instream);\n    const item = await head.next();\n    if (!item.value) {\n      throw new Error('stream closed without snapshot reservation');\n    }\n    const {data: msg} = item.value;\n    if (msg[0] !== 'reserved') {\n      throw new Error(`expected \"reserved\" message but got ${msg[0]}`);\n    }\n    const reserved: SnapshotStatus = msg[1];\n    const followup: ReservationFollowup = {\n      subscribe: ctx => {\n        outbound.push(['start-subscription', toStartSubscriptionContext(ctx)]);\n        return Promise.resolve(rest);\n      },\n      signal: outbound.signal,\n      cancel: reason => outbound.cancel(reason),\n    };\n\n    return {reserved, followup};\n  }\n\n  async subscribe(\n    ctx: SubscriberContext,\n  ): Promise<Source<Sized<SubscribeDownstream>>> {\n    const {instream, outbound} = await this.connect(ctx.taskID);\n    outbound.push(['start-subscription', toStartSubscriptionContext(ctx)]);\n    return instream;\n  }\n\n  /**\n   * Opens a merged v7 `/subscribe` connection (see subscribe.ts). Returns the\n   * inbound change/reservation stream and an `outbound` sink into which the\n   * caller pushes the `reserve-snapshot` / `start-subscription` control\n   * messages that drive the connection's phases. The single connection spans\n   * the reservation, backup restore, and subscription, so the reserving\n   * replication-manager is the one that serves the subscription.\n   */\n  // exported for testing\n  async connect(taskID: string): Promise<{\n    instream: Source<Sized<SubscribeDownstream>>;\n    outbound: Subscription<SubscribeUpstream>;\n  }> {\n    const uri = await this.#resolveChangeStreamer(SUBSCRIBE_PATH);\n\n    // taskID is carried in the query for observability/routing; the server\n    // reads the authoritative taskID from the control messages themselves.\n    const params = new URLSearchParams({taskID});\n    // Bound the TCP connect + HTTP upgrade. Without this, a SYN black-holed\n    // by a stale Service endpoint (e.g. a just-terminated replication-manager\n    // pod) is retransmitted by the kernel for ~2 minutes before ETIMEDOUT.\n    // Failing fast lets the caller's retry open a fresh connection that is\n    // routed to a live endpoint.\n    const ws = new WebSocket(uri + `?${params.toString()}`, {\n      handshakeTimeout: CONNECT_TIMEOUT_MS,\n    });\n\n    const outbound = Subscription.create<SubscribeUpstream>();\n    const instream = await streamInternalWithSize(\n      this.#lc,\n      ws,\n      subscribeDownstreamSchema,\n      outbound,\n    );\n    return {instream, outbound};\n  }\n}\n\ntype RequestHeaders = Pick<IncomingMessage, 'url' | 'headers'>;\n\nexport function getSubscriberContext(req: RequestHeaders): SubscriberContext {\n  const url = new URL(req.url ?? '', req.headers.origin ?? 'http://localhost');\n  const protocolVersion = checkProtocolVersion(url.pathname);\n  const params = new URLParams(url);\n\n  return {\n    protocolVersion,\n    id: params.get('id', true),\n    taskID: params.get('taskID', true),\n    mode: params.get('mode', false) === 'backup' ? 'backup' : 'serving',\n    replicaVersion: params.get('replicaVersion', true),\n    watermark: params.get('watermark', true),\n    // Absent for subscribers that predate the parameter, which is the safe\n    // default: the barrier falls back to polling rather than waiting on an\n    // ACK that would never be attributed to a writer.\n    wsBatched: params.getBoolean('wsBatched'),\n  };\n}\n\nfunction checkProtocolVersion(pathname: string): number {\n  const match = PATH_REGEX.exec(pathname);\n  if (!match) {\n    throw new Error(`invalid path: ${pathname}`);\n  }\n  const v = Number(match.groups?.version);\n  if (\n    Number.isNaN(v) ||\n    v > PROTOCOL_VERSION ||\n    v < MIN_SUPPORTED_PROTOCOL_VERSION\n  ) {\n    throw new Error(\n      `Cannot service client at protocol v${v}. ` +\n        `Supported protocols: [v${MIN_SUPPORTED_PROTOCOL_VERSION} ... v${PROTOCOL_VERSION}]`,\n    );\n  }\n  return v;\n}\n\n// Projects a client-side SubscriberContext onto the start-subscription\n// message payload, dropping the fields that are implicit on the merged\n// connection: protocolVersion (from the request path), wsBatched (always on).\nfunction toStartSubscriptionContext(ctx: SubscriberContext): SubscribeContext {\n  const {\n    taskID,\n    id,\n    mode,\n    replicaVersion,\n    watermark,\n    backfills,\n    drainable,\n    switchover,\n  } = ctx;\n  return {\n    taskID,\n    id,\n    mode,\n    replicaVersion,\n    watermark,\n    backfills,\n    ...(drainable !== undefined ? {drainable} : {}),\n    ...(switchover !== undefined ? {switchover} : {}),\n  };\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;AA6CA,IAAM,iCAAiC;AAGvC,IAAM,iCAAiC;AAEvC,IAAM,wBAAwB;AAC9B,IAAM,uBAAuB;AAC7B,IAAM,yBAAyB;AAC/B,IAAM,aACJ;AAEF,IAAM,iBAAiB;AAIvB,IAAM,qBAAqB;AAU3B,IAAa,2BAAb,cAA8C,YAAY;CACxD,KAAc;CACd;CACA;CACA;CAEA,YACE,IACA,MACA,QACA,gBACA;EACA,MAAM,+BAA+B,IAAI,MAAM,OAAM,YAAW;GAC9D,MAAM,QAAQ,SAAS,SAAS;GAEhC,QAAQ,IAAI,sBAAsB,EAAC,WAAW,KAAI,GAAG,KAAKG,QAAQ;GAClE,QAAQ,IACN,uBACA,EAAC,WAAW,KAAI,GAChB,KAAKC,gBACP;GACA,QAAQ,IAAI,wBAAwB,EAAC,WAAW,KAAI,GAAG,KAAKC,UAAU;GAEtE,QAAQ,IAAI,WAAW,KAAK,QAC1B,mBACE,IACA,KAAK,UAAU;IACb,eAAe,KAAA;IACf,kBAAkB,KAAA;GACpB,GACA,KACA,KACA,KAAK,kBACL,KAAA,GACA,iBACF,CACF;GAEA,yBACE,IACA,QAAQ,iBACR,KAAKC,mBACL,MACF;EACF,CAAC;EAED,KAAKN,MAAM;EACX,KAAKC,QAAQ;EACb,KAAKC,kBAAkB;CACzB;CAGA,qBACE,IACA,QACA,QACG;EACH,QAAQ,QAAR;GACE,KAAK,YACH,OAAO,KAAKE,iBAAiB,IAAI,GAAG;GACtC,KAAK,WACH,OAAO,KAAKD,SAAS,IAAI,GAAG;GAC9B,KAAK,aACH,OAAO,KAAKE,WAAW,IAAI,GAAG;GAChC;IACE,eACE,KAAK,KACL,IACA,mBAAmB,OAAO,sBAC5B;IACA;EACJ;CACF;CAEA,mBAA4B,OAAO,IAAe,QAAwB;EACxE,KAAKE,6BAA6B,+BAA+B;EACjE,IAAI;GACF,MAAM,MAAM,IAAI,IACd,IAAI,OAAO,IACX,IAAI,QAAQ,UAAU,kBACxB;GACA,qBAAqB,IAAI,QAAQ;GACjC,MAAM,SAAS,IAAI,aAAa,IAAI,QAAQ;GAC5C,IAAI,CAAC,QACH,MAAM,IAAI,MAAM,oCAAoC;GAEtD,MAAM,aACJ,MAAM,KAAKL,gBAAgB,yBAAyB,MAAM;GAG5D,eAAoB,KAAK,KAAK,IAAI,KAAA,GAAW,UAAU;EACzD,SAAS,KAAK;GACZ,eAAe,KAAK,KAAK,IAAI,KAAK,cAAc;EAClD;CACF;CAEA,WAAoB,OAAO,IAAe,QAAwB;EAChE,IAAI;GACF,MAAM,MAAM,qBAAqB,GAAG;GACpC,IAAI,IAAI,SAAS,WACf,KAAKK,6BAA6B,uBAAuB;GAG3D,MAAM,aAAa,MAAM,KAAKL,gBAAgB,UAAU,GAAG;GAG3D,0BAA+B,KAAK,KAAK,IAAI,KAAA,GAAW,YAAY,EAClE,SAAS,IAAI,UACf,CAAC;EACH,SAAS,KAAK;GACZ,eAAe,KAAK,KAAK,IAAI,KAAK,cAAc;EAClD;CACF;;;;;;;;CASA,aAAsB,OAAO,IAAe,QAAwB;EAClE,IAAI;EACJ,MAAM,WAAW,aAAa,OAAoC;EAElE,IAAI;GAKF,MAAM,kBAAkB,qBAAqB,IAJ7B,IACd,IAAI,OAAO,IACX,IAAI,QAAQ,UAAU,kBAEqB,EAAI,QAAQ;GACzD,IAAI,kBAAkB,gCACpB,MAAM,IAAI,MACR,8CAA8C,+BAA+B,kBACzD,gBAAgB,EACtC;GAGF,MAAM,WAAW,MAAM,0BACrB,KAAK,KACL,IACA,yBACA,UACA,EAAC,SAAS,KAAI,CAChB;GAKA,WAAW,MAAM,OAAO,UACtB,IAAI,IAAI,OAAO,oBAAoB;IACjC,MAAM,EAAC,QAAQ,iBAAgB,IAAI;IACnC,KAAKK,6BAA6B,+BAA+B;IACjE,cAAc,MAAM,KAAKL,gBAAgB,yBACvC,QACA,EAAC,aAAY,CACf;IACA,KAAUM,oBAAoB,aAAa,QAAQ;GACrD,OAAO,IAAI,IAAI,OAAO,sBAAsB;IAC1C,MAAM,MAAyB;KAC7B,GAAG,IAAI;KACP;KACA,WAAW;IACb;IACA,IAAI,IAAI,SAAS,WACf,KAAKD,6BAA6B,uBAAuB;IAI3D,cAAc,KAAA;IACd,MAAM,KAAKL,gBAAgB,UAAU,KAAK,QAAQ;GACpD;EAEJ,SAAS,KAAK;GACZ,eAAe,KAAK,KAAK,IAAI,KAAK,cAAc;EAClD,UAAU;GAGR,aAAa,OAAO;EACtB;CACF;;;;;;;;CASA,MAAMM,oBACJ,aACA,UACA;EACA,IAAI;GACF,WAAW,MAAM,GAAG,WAAW,aAG7B,SAAS,KACP,WAAW,UAAU,CACnB,YACA;IAAC,GAAG;IAAQ,KAAK;GAAU,CAC7B,CAA2B,CAC7B;EAEJ,SAAS,GAAG;GACV,KAAK,IAAI,OAAO,yCAAyC,CAAC;GAC1D,SAAS,KAAK,aAAa,QAAQ,IAAI,IAAI,MAAM,OAAO,CAAC,CAAC,CAAC;EAC7D;CACF;CAEA,yBAAyB;CAEzB,6BAA6B,QAAiB;EAC5C,IAAI,CAAC,KAAKC,0BAA0B,KAAK,OAAO,UAAU,GAAG;GAC3D,KAAKT,IAAI,OACP,kCAAkC,SAAS,IAAI,OAAO,KAAK,IAC7D;GACA,KAAUE,gBACP,IAAI,EACJ,OAAM,MACL,KAAKF,IAAI,OAAO,0CAA0C,CAAC,CAC7D,EACC,cAAc,KAAK,KAAK,CAAC;GAE5B,KAAKS,yBAAyB;EAChC;CACF;CAEA,WAAoC;EAClC,MAAM,EAAC,gBAAgB,gBAAgB,gBAAe,KAAKR;EAC3D,IAAI,mBAAmB,GAGrB,KAAKM,6BAA6B;OASlC,cAAmB,WAAW;GAC5B,KAAKP,IAAI,OACP,WAAW,eAAe,qCAC5B;GACA,KAAK,OAAO,iBAER,KAAKO,6BACH,0BAA0B,eAAe,KAC3C,GACF,cACF;EACF,CAAC;CAEL;;;;;;;;CASA,MAAM,QAAuB;EAC3B,KAAK,cAAc;EACnB,IAAI,KAAKE,0BAA0B,KAAKP,gBAAgB,OACtD,MAAM,KAAKA,gBAAgB,MAAM;CAErC;CAEA,MAAyB,UAAyB;EAChD,IAAI,KAAKO,wBACP,MAAM,KAAKP,gBAAgB,KAAK;CAEpC;AACF;AAkCA,IAAa,2BAAb,MAEA;CACE;CACA;CACA;CACA;CAEA,YACE,IACA,SACA,UACA,mBACA;EACA,KAAKF,MAAM;EACX,KAAKU,WAAW;EAGhB,KAAKC,YAAY,SAAS,IAAI,UAAU,6BAA6B;GACnE,KAAK;IACJ,iBAAiB;EACpB,CAAC;EACD,KAAKC,qBAAqB;CAC5B;CAEA,MAAMC,uBAAuB,MAAc;EACzC,IAAI,UAAU,KAAKD;EACnB,IAAI,CAAC,SAAS;GACZ,MAAM,UAAU,MAAM,8BACpB,KAAKF,UACL,KAAKC,SACP;GACA,IAAI,CAAC,SACH,MAAM,IAAI,MAAM,+BAA+B;GAEjD,UAAU,QAAQ,SAAS,KAAK,IAAI,GAAG,QAAQ,KAAK,QAAQ,QAAQ;EACtE;EACA,MAAM,MAAM,IAAI,IAAI,MAAM,OAAO;EACjC,KAAKX,IAAI,OAAO,iCAAiC,KAAK;EACtD,OAAO;CACT;CAEA,MAAM,gBACJ,QACA,cAIC;EACD,MAAM,EAAC,UAAU,aAAY,MAAM,KAAK,QAAQ,MAAM;EACtD,SAAS,KAAK,CACZ,oBACA;GAAC;GAAQ,GAAI,eAAe,EAAC,aAAY,IAAI,CAAC;EAAE,CAClD,CAAC;EAED,MAAM,EAAC,MAAM,SAAQ,SAAS,QAAQ;EACtC,MAAM,OAAO,MAAM,KAAK,KAAK;EAC7B,IAAI,CAAC,KAAK,OACR,MAAM,IAAI,MAAM,4CAA4C;EAE9D,MAAM,EAAC,MAAM,QAAO,KAAK;EACzB,IAAI,IAAI,OAAO,YACb,MAAM,IAAI,MAAM,uCAAuC,IAAI,IAAI;EAYjE,OAAO;GAAC,UAVyB,IAAI;GAUnB,UAAA;IARhB,YAAW,QAAO;KAChB,SAAS,KAAK,CAAC,sBAAsB,2BAA2B,GAAG,CAAC,CAAC;KACrE,OAAO,QAAQ,QAAQ,IAAI;IAC7B;IACA,QAAQ,SAAS;IACjB,SAAQ,WAAU,SAAS,OAAO,MAAM;GAGxB;EAAQ;CAC5B;CAEA,MAAM,UACJ,KAC6C;EAC7C,MAAM,EAAC,UAAU,aAAY,MAAM,KAAK,QAAQ,IAAI,MAAM;EAC1D,SAAS,KAAK,CAAC,sBAAsB,2BAA2B,GAAG,CAAC,CAAC;EACrE,OAAO;CACT;;;;;;;;;CAWA,MAAM,QAAQ,QAGX;EAWD,MAAM,KAAK,IAAI,YAAU,MAVP,KAAKa,uBAAuB,cAAc,IAU7B,IAAI,IANhB,gBAAgB,EAAC,OAAM,CAMP,EAAO,SAAS,KAAK,EACtD,kBAAkB,mBACpB,CAAC;EAED,MAAM,WAAW,aAAa,OAA0B;EAOxD,OAAO;GAAC,UAAA,MANe,uBACrB,KAAKb,KACL,IACA,2BACA,QACF;GACkB;EAAQ;CAC5B;AACF;AAIA,SAAgB,qBAAqB,KAAwC;CAC3E,MAAM,MAAM,IAAI,IAAI,IAAI,OAAO,IAAI,IAAI,QAAQ,UAAU,kBAAkB;CAC3E,MAAM,kBAAkB,qBAAqB,IAAI,QAAQ;CACzD,MAAM,SAAS,IAAI,UAAU,GAAG;CAEhC,OAAO;EACL;EACA,IAAI,OAAO,IAAI,MAAM,IAAI;EACzB,QAAQ,OAAO,IAAI,UAAU,IAAI;EACjC,MAAM,OAAO,IAAI,QAAQ,KAAK,MAAM,WAAW,WAAW;EAC1D,gBAAgB,OAAO,IAAI,kBAAkB,IAAI;EACjD,WAAW,OAAO,IAAI,aAAa,IAAI;EAIvC,WAAW,OAAO,WAAW,WAAW;CAC1C;AACF;AAEA,SAAS,qBAAqB,UAA0B;CACtD,MAAM,QAAQ,WAAW,KAAK,QAAQ;CACtC,IAAI,CAAC,OACH,MAAM,IAAI,MAAM,iBAAiB,UAAU;CAE7C,MAAM,IAAI,OAAO,MAAM,QAAQ,OAAO;CACtC,IACE,OAAO,MAAM,CAAC,KACd,IAAA,KACA,IAAI,gCAEJ,MAAM,IAAI,MACR,sCAAsC,EAAE,2BACZ,+BAA+B,SAC7D;CAEF,OAAO;AACT;AAKA,SAAS,2BAA2B,KAA0C;CAC5E,MAAM,EACJ,QACA,IACA,MACA,gBACA,WACA,WACA,WACA,eACE;CACJ,OAAO;EACL;EACA;EACA;EACA;EACA;EACA;EACA,GAAI,cAAc,KAAA,IAAY,EAAC,UAAS,IAAI,CAAC;EAC7C,GAAI,eAAe,KAAA,IAAY,EAAC,WAAU,IAAI,CAAC;CACjD;AACF"}
+{"version":3,"file":"change-streamer-http.js","names":["WebSocket"],"sources":["../../../../../../zero-cache/src/services/change-streamer/change-streamer-http.ts"],"sourcesContent":["import type {IncomingMessage} from 'node:http';\nimport websocket from '@fastify/websocket';\nimport type {LogContext} from '@rocicorp/logger';\nimport WebSocket from 'ws';\nimport {BigIntJSON} from '../../../../shared/src/bigint-json.ts';\nimport {promiseVoid} from '../../../../shared/src/resolved-promises.ts';\nimport type {NormalizedZeroConfig} from '../../config/normalize.ts';\nimport type {IncomingMessageSubset} from '../../types/http.ts';\nimport {pgClient, type PostgresDB} from '../../types/pg.ts';\nimport {type Worker} from '../../types/processes.ts';\nimport {type ShardID} from '../../types/shards.ts';\nimport {\n  readHead,\n  streamInternal,\n  streamInternalStringified,\n  streamInternalWithSize,\n  type Sized,\n  type Source,\n} from '../../types/streams.ts';\nimport {Subscription} from '../../types/subscription.ts';\nimport {URLParams} from '../../types/url-params.ts';\nimport {installWebSocketReceiver} from '../../types/websocket-handoff.ts';\nimport {closeWithError, PROTOCOL_ERROR} from '../../types/ws.ts';\nimport {HttpService, type Options as HttpOptions} from '../http-service.ts';\nimport {handleProfzRequest} from '../profz.ts';\nimport type {PreSerializedBatch} from './broadcast.ts';\nimport {\n  PROTOCOL_VERSION,\n  type ChangeStreamer,\n  type ChangeStreamerService,\n  type SubscriberContext,\n} from './change-streamer.ts';\nimport {discoverChangeStreamerAddress} from './schema/tables.ts';\nimport {type SnapshotMessage} from './snapshot-message.ts';\nimport {\n  type LocalReplica,\n  subscribeDownstreamSchema,\n  subscribeUpstreamSchema,\n  type ReservedMessage,\n  type SnapshotStatus,\n  type SubscribeContext,\n  type SubscribeDownstream,\n  type SubscribeUpstream,\n} from './subscribe.ts';\n\nconst MIN_SUPPORTED_PROTOCOL_VERSION = 4;\n\n// The merged `/subscribe` protocol was introduced at v7.\nconst MIN_SUBSCRIBE_PROTOCOL_VERSION = 7;\n\nconst SNAPSHOT_PATH_PATTERN = '/replication/:version/snapshot';\nconst CHANGES_PATH_PATTERN = '/replication/:version/changes';\nconst SUBSCRIBE_PATH_PATTERN = '/replication/:version/subscribe';\nconst PATH_REGEX =\n  /\\/replication\\/v(?<version>\\d+)\\/(changes|snapshot|subscribe)$/;\n\nconst SUBSCRIBE_PATH = `/replication/v${PROTOCOL_VERSION}/subscribe`;\n\n// Max time for a client to establish the /subscribe websocket (TCP connect\n// plus upgrade) before abandoning the attempt.\nconst CONNECT_TIMEOUT_MS = 10_000;\n\ntype Options = HttpOptions & {\n  startupDelayMs: number;\n  config?:\n    | Pick<NormalizedZeroConfig, 'adminPassword' | 'operatorPassword'>\n    | undefined;\n  getProfileWorker?: (() => Promise<Worker>) | undefined;\n};\n\nexport class ChangeStreamerHttpServer extends HttpService {\n  readonly id = 'change-streamer-http-server';\n  readonly #lc: LogContext;\n  readonly #opts: Options;\n  readonly #changeStreamer: ChangeStreamerService;\n\n  constructor(\n    lc: LogContext,\n    opts: Options,\n    parent: Worker,\n    changeStreamer: ChangeStreamerService,\n  ) {\n    super('change-streamer-http-server', lc, opts, async fastify => {\n      await fastify.register(websocket);\n\n      fastify.get(CHANGES_PATH_PATTERN, {websocket: true}, this.#changes);\n      fastify.get(\n        SNAPSHOT_PATH_PATTERN,\n        {websocket: true},\n        this.#reserveSnapshot,\n      );\n      fastify.get(SUBSCRIBE_PATH_PATTERN, {websocket: true}, this.#subscribe);\n\n      fastify.get('/profz', (req, res) =>\n        handleProfzRequest(\n          lc,\n          opts.config ?? {\n            adminPassword: undefined,\n            operatorPassword: undefined,\n          },\n          req,\n          res,\n          opts.getProfileWorker,\n          undefined,\n          'change-streamer',\n        ),\n      );\n\n      installWebSocketReceiver<'snapshot' | 'changes' | 'subscribe'>(\n        lc,\n        fastify.websocketServer,\n        this.#receiveWebsocket,\n        parent,\n      );\n    });\n\n    this.#lc = lc;\n    this.#opts = opts;\n    this.#changeStreamer = changeStreamer;\n  }\n\n  // Called when receiving a web socket via the main dispatcher handoff.\n  readonly #receiveWebsocket = (\n    ws: WebSocket,\n    action: 'changes' | 'snapshot' | 'subscribe',\n    msg: IncomingMessageSubset,\n  ) => {\n    switch (action) {\n      case 'snapshot':\n        return this.#reserveSnapshot(ws, msg);\n      case 'changes':\n        return this.#changes(ws, msg);\n      case 'subscribe':\n        return this.#subscribe(ws, msg);\n      default:\n        closeWithError(\n          this._lc,\n          ws,\n          `invalid action \"${action}\" received in handoff`,\n        );\n        return;\n    }\n  };\n\n  readonly #reserveSnapshot = async (ws: WebSocket, req: RequestHeaders) => {\n    this.#ensureChangeStreamerStarted('incoming snapshot reservation');\n    try {\n      const url = new URL(\n        req.url ?? '',\n        req.headers.origin ?? 'http://localhost',\n      );\n      checkProtocolVersion(url.pathname);\n      const taskID = url.searchParams.get('taskID');\n      if (!taskID) {\n        throw new Error('Missing taskID in snapshot request');\n      }\n      const downstream =\n        await this.#changeStreamer.startSnapshotReservation(taskID);\n      // Send-only for now: the reservation carries no inbound application\n      // messages (only transport acks, demuxed internally).\n      void streamInternal(this._lc, ws, undefined, downstream);\n    } catch (err) {\n      closeWithError(this._lc, ws, err, PROTOCOL_ERROR);\n    }\n  };\n\n  readonly #changes = async (ws: WebSocket, req: RequestHeaders) => {\n    try {\n      const ctx = getSubscriberContext(req);\n      if (ctx.mode === 'serving') {\n        this.#ensureChangeStreamerStarted('incoming subscription');\n      }\n\n      const downstream = await this.#changeStreamer.subscribe(ctx);\n      // Send-only for now: the subscription carries no inbound application\n      // messages (only transport acks, demuxed internally).\n      void streamInternalStringified(this._lc, ws, undefined, downstream, {\n        batched: ctx.wsBatched,\n      });\n    } catch (err) {\n      closeWithError(this._lc, ws, err, PROTOCOL_ERROR);\n    }\n  };\n\n  /**\n   * The merged v7 `/subscribe` handler. A single bidirectional connection\n   * carries, in sequence: an optional reservation phase (`reserve-snapshot` →\n   * `reserved`) and then the subscription phase (`start-subscription` → change\n   * stream). This ensures that the subscriber subscribes to the same task from\n   * which it restored the backup.\n   */\n  readonly #subscribe = async (ws: WebSocket, req: RequestHeaders) => {\n    let reservation: Source<SnapshotMessage> | undefined;\n    const outbound = Subscription.create<string | PreSerializedBatch>();\n\n    try {\n      const url = new URL(\n        req.url ?? '',\n        req.headers.origin ?? 'http://localhost',\n      );\n      const protocolVersion = checkProtocolVersion(url.pathname);\n      if (protocolVersion < MIN_SUBSCRIBE_PROTOCOL_VERSION) {\n        throw new Error(\n          `the /subscribe endpoint requires protocol v${MIN_SUBSCRIBE_PROTOCOL_VERSION} ` +\n            `(client is at v${protocolVersion})`,\n        );\n      }\n\n      const instream = await streamInternalStringified(\n        this._lc,\n        ws,\n        subscribeUpstreamSchema,\n        outbound,\n        {batched: true},\n      );\n\n      // Keep reading upstream control messages for the life of the connection.\n      // Do not manually exit loop, as that would close the socket. The loop\n      // exits naturally when the connection closes.\n      for await (const msg of instream) {\n        if (msg[0] === 'reserve-snapshot') {\n          const {taskID, localReplica} = msg[1];\n          this.#ensureChangeStreamerStarted('incoming snapshot reservation');\n          reservation = await this.#changeStreamer.startSnapshotReservation(\n            taskID,\n            {localReplica},\n          );\n          void this.#forwardReservation(reservation, outbound);\n        } else if (msg[0] === 'start-subscription') {\n          const ctx: SubscriberContext = {\n            ...msg[1],\n            protocolVersion,\n            wsBatched: true,\n          };\n          if (ctx.mode === 'serving') {\n            this.#ensureChangeStreamerStarted('incoming subscription');\n          }\n          // subscribe() registers the subscriber and then releases any\n          // reservation for that task.\n          reservation = undefined;\n          await this.#changeStreamer.subscribe(ctx, outbound);\n        }\n      }\n    } catch (err) {\n      closeWithError(this._lc, ws, err, PROTOCOL_ERROR);\n    } finally {\n      // Covers a reservation-phase disconnect (instream ended before a\n      // subscription started); idempotent with the close handler above.\n      reservation?.cancel();\n    }\n  };\n\n  /**\n   * Forwards a snapshot reservation's single `['status', ...]` message onto\n   * the merged connection's `outbound` sink as a `['reserved', ...]` message.\n   * The loop then blocks on the held-open reservation until it is cancelled,\n   * by `subscribe()` at the subscription handoff, or by the reservation-phase\n   * teardown on disconnect.\n   */\n  async #forwardReservation(\n    reservation: Source<SnapshotMessage>,\n    outbound: Subscription<string | PreSerializedBatch>,\n  ) {\n    try {\n      for await (const [, status] of reservation) {\n        // Translate the /snapshot protocol's ['status', {tag: 'status', ...}]\n        // message to the /subscribe protocol's ['reserved', {tag: 'snapshot', ...}].\n        outbound.push(\n          BigIntJSON.stringify([\n            'reserved',\n            {...status, tag: 'snapshot'},\n          ] satisfies ReservedMessage),\n        );\n      }\n    } catch (e) {\n      this._lc.warn?.(`error forwarding snapshot reservation`, e);\n      outbound.fail(e instanceof Error ? e : new Error(String(e)));\n    }\n  }\n\n  #changeStreamerStarted = false;\n\n  #ensureChangeStreamerStarted(reason?: string) {\n    if (!this.#changeStreamerStarted && this._state.shouldRun()) {\n      this.#lc.info?.(\n        `starting ChangeStreamerService ${reason ? `(${reason})` : ''}`,\n      );\n      void this.#changeStreamer\n        .run()\n        .catch(e =>\n          this.#lc.warn?.(`ChangeStreamerService ended with error`, e),\n        )\n        .finally(() => this.stop());\n\n      this.#changeStreamerStarted = true;\n    }\n  }\n\n  protected override _onStart(): void {\n    const {startupDelayMs, readinessGate = promiseVoid} = this.#opts;\n    if (startupDelayMs === 0) {\n      // In RMv2, there is no need to delay starting the change streamer\n      // because RM startup is non-disruptive.\n      this.#ensureChangeStreamerStarted();\n    } else {\n      // In RMv1, starting the change-streamer forcibly shuts down the\n      // previous change-streamer, causing view-syncers to reconnect.\n      // If this replication-manager has just started, the routing layer may\n      // not have registered it with DNS, as that only happens after it\n      // confirms health checks. To minimize downtime, the takeover is\n      // delayed for the configured startupDelayMs _after_ beginning to\n      // advertise readiness.\n      void readinessGate.then(() => {\n        this.#lc.info?.(\n          `waiting ${startupDelayMs}ms before taking over the change log`,\n        );\n        this._state.setTimeout(\n          () =>\n            this.#ensureChangeStreamerStarted(\n              `startup delay elapsed (${startupDelayMs} ms)`,\n            ),\n          startupDelayMs,\n        );\n      });\n    }\n  }\n\n  /**\n   * Signals (drainable) subscribers to switch over to another\n   * replication-manager, resolving when they have disconnected. Subscribers\n   * continue to be served in the meantime, but new connections (e.g.\n   * switchover attempts, reconnects, and snapshot reservations) are refused,\n   * so that they are routed to another replication-manager.\n   */\n  async drain(): Promise<void> {\n    this.stopAccepting();\n    if (this.#changeStreamerStarted && this.#changeStreamer.drain) {\n      await this.#changeStreamer.drain();\n    }\n  }\n\n  protected override async _onStop(): Promise<void> {\n    if (this.#changeStreamerStarted) {\n      await this.#changeStreamer.stop();\n    }\n  }\n}\n\nexport interface SnapshotReserver {\n  reserveSnapshot(\n    taskID: string,\n    localReplica?: LocalReplica | undefined,\n  ): Promise<{\n    reserved: SnapshotStatus;\n    followup: ReservationFollowup;\n  }>;\n}\n\nexport interface SubscriptionStarter {\n  subscribe(\n    ctx: SubscriberContext,\n  ): Promise<Source<Sized<SubscribeDownstream>>>;\n}\n\nexport interface ReservationFollowup extends SubscriptionStarter {\n  /**\n   * A signal to indicate that the followup is no longer valid (i.e. the\n   * connection to the reserving task has been severed).\n   */\n  readonly signal: AbortSignal;\n\n  /**\n   * Abandons the reservation without subscribing, closing the connection that\n   * was held open to pin the change-log floor. Used when the caller cannot\n   * proceed to `subscribe()` (e.g. process shutdown while a restore is in\n   * progress) and must not leave the reservation's connection open forever.\n   */\n  cancel(reason?: Error): void;\n}\n\nexport class ChangeStreamerHttpClient\n  implements ChangeStreamer, SnapshotReserver, SubscriptionStarter\n{\n  readonly #lc: LogContext;\n  readonly #shardID: ShardID;\n  readonly #changeDB: PostgresDB;\n  readonly #changeStreamerURI: string | undefined;\n\n  constructor(\n    lc: LogContext,\n    shardID: ShardID,\n    changeDB: string,\n    changeStreamerURI: string | undefined,\n  ) {\n    this.#lc = lc;\n    this.#shardID = shardID;\n    // Create a pg client with a single short-lived connection for the purpose\n    // of change-streamer discovery (i.e. ChangeDB as DNS).\n    this.#changeDB = pgClient(lc, changeDB, 'change-streamer-discovery', {\n      max: 1,\n      ['idle_timeout']: 15,\n    });\n    this.#changeStreamerURI = changeStreamerURI;\n  }\n\n  async #resolveChangeStreamer(path: string) {\n    let baseURL = this.#changeStreamerURI;\n    if (!baseURL) {\n      const address = await discoverChangeStreamerAddress(\n        this.#shardID,\n        this.#changeDB,\n      );\n      if (!address) {\n        throw new Error(`no change-streamer is running`);\n      }\n      baseURL = address.includes('://') ? `${address}/` : `ws://${address}/`;\n    }\n    const uri = new URL(path, baseURL);\n    this.#lc.info?.(`connecting to change-streamer@${uri}`);\n    return uri;\n  }\n\n  async reserveSnapshot(\n    taskID: string,\n    localReplica?: LocalReplica | undefined,\n  ): Promise<{\n    reserved: SnapshotStatus;\n    followup: ReservationFollowup;\n  }> {\n    const {instream, outbound} = await this.connect(taskID);\n    outbound.push([\n      'reserve-snapshot',\n      {taskID, ...(localReplica ? {localReplica} : {})},\n    ]);\n\n    const {head, rest} = readHead(instream);\n    const item = await head.next();\n    if (!item.value) {\n      throw new Error('stream closed without snapshot reservation');\n    }\n    const {data: msg} = item.value;\n    if (msg[0] !== 'reserved') {\n      throw new Error(`expected \"reserved\" message but got ${msg[0]}`);\n    }\n    const reserved: SnapshotStatus = msg[1];\n    const followup: ReservationFollowup = {\n      subscribe: ctx => {\n        outbound.push(['start-subscription', toStartSubscriptionContext(ctx)]);\n        return Promise.resolve(rest);\n      },\n      signal: outbound.signal,\n      cancel: reason => outbound.cancel(reason),\n    };\n\n    return {reserved, followup};\n  }\n\n  async subscribe(\n    ctx: SubscriberContext,\n  ): Promise<Source<Sized<SubscribeDownstream>>> {\n    const {instream, outbound} = await this.connect(ctx.taskID);\n    outbound.push(['start-subscription', toStartSubscriptionContext(ctx)]);\n    return instream;\n  }\n\n  /**\n   * Opens a merged v7 `/subscribe` connection (see subscribe.ts). Returns the\n   * inbound change/reservation stream and an `outbound` sink into which the\n   * caller pushes the `reserve-snapshot` / `start-subscription` control\n   * messages that drive the connection's phases. The single connection spans\n   * the reservation, backup restore, and subscription, so the reserving\n   * replication-manager is the one that serves the subscription.\n   */\n  // exported for testing\n  async connect(taskID: string): Promise<{\n    instream: Source<Sized<SubscribeDownstream>>;\n    outbound: Subscription<SubscribeUpstream>;\n  }> {\n    const uri = await this.#resolveChangeStreamer(SUBSCRIBE_PATH);\n\n    // taskID is carried in the query for observability/routing; the server\n    // reads the authoritative taskID from the control messages themselves.\n    const params = new URLSearchParams({taskID});\n    // Bound the TCP connect + HTTP upgrade. Without this, a SYN black-holed\n    // by a stale Service endpoint (e.g. a just-terminated replication-manager\n    // pod) is retransmitted by the kernel for ~2 minutes before ETIMEDOUT.\n    // Failing fast lets the caller's retry open a fresh connection that is\n    // routed to a live endpoint.\n    const ws = new WebSocket(uri + `?${params.toString()}`, {\n      handshakeTimeout: CONNECT_TIMEOUT_MS,\n    });\n\n    const outbound = Subscription.create<SubscribeUpstream>();\n    const instream = await streamInternalWithSize(\n      this.#lc,\n      ws,\n      subscribeDownstreamSchema,\n      outbound,\n    );\n    return {instream, outbound};\n  }\n}\n\ntype RequestHeaders = Pick<IncomingMessage, 'url' | 'headers'>;\n\nexport function getSubscriberContext(req: RequestHeaders): SubscriberContext {\n  const url = new URL(req.url ?? '', req.headers.origin ?? 'http://localhost');\n  const protocolVersion = checkProtocolVersion(url.pathname);\n  const params = new URLParams(url);\n\n  return {\n    protocolVersion,\n    id: params.get('id', true),\n    taskID: params.get('taskID', true),\n    mode: params.get('mode', false) === 'backup' ? 'backup' : 'serving',\n    replicaVersion: params.get('replicaVersion', true),\n    watermark: params.get('watermark', true),\n    // Absent for subscribers that predate the parameter, which is the safe\n    // default: the barrier falls back to polling rather than waiting on an\n    // ACK that would never be attributed to a writer.\n    wsBatched: params.getBoolean('wsBatched'),\n  };\n}\n\nfunction checkProtocolVersion(pathname: string): number {\n  const match = PATH_REGEX.exec(pathname);\n  if (!match) {\n    throw new Error(`invalid path: ${pathname}`);\n  }\n  const v = Number(match.groups?.version);\n  if (\n    Number.isNaN(v) ||\n    v > PROTOCOL_VERSION ||\n    v < MIN_SUPPORTED_PROTOCOL_VERSION\n  ) {\n    throw new Error(\n      `Cannot service client at protocol v${v}. ` +\n        `Supported protocols: [v${MIN_SUPPORTED_PROTOCOL_VERSION} ... v${PROTOCOL_VERSION}]`,\n    );\n  }\n  return v;\n}\n\n// Projects a client-side SubscriberContext onto the start-subscription\n// message payload, dropping the fields that are implicit on the merged\n// connection: protocolVersion (from the request path), wsBatched (always on).\nfunction toStartSubscriptionContext(ctx: SubscriberContext): SubscribeContext {\n  const {\n    taskID,\n    id,\n    mode,\n    replicaVersion,\n    watermark,\n    backfills,\n    drainable,\n    switchover,\n  } = ctx;\n  return {\n    taskID,\n    id,\n    mode,\n    replicaVersion,\n    watermark,\n    backfills,\n    ...(drainable !== undefined ? {drainable} : {}),\n    ...(switchover !== undefined ? {switchover} : {}),\n  };\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;AA6CA,IAAM,iCAAiC;AAGvC,IAAM,iCAAiC;AAEvC,IAAM,wBAAwB;AAC9B,IAAM,uBAAuB;AAC7B,IAAM,yBAAyB;AAC/B,IAAM,aACJ;AAEF,IAAM,iBAAiB;AAIvB,IAAM,qBAAqB;AAU3B,IAAa,2BAAb,cAA8C,YAAY;CACxD,KAAc;CACd;CACA;CACA;CAEA,YACE,IACA,MACA,QACA,gBACA;EACA,MAAM,+BAA+B,IAAI,MAAM,OAAM,YAAW;GAC9D,MAAM,QAAQ,SAAS,SAAS;GAEhC,QAAQ,IAAI,sBAAsB,EAAC,WAAW,KAAI,GAAG,KAAK,QAAQ;GAClE,QAAQ,IACN,uBACA,EAAC,WAAW,KAAI,GAChB,KAAK,gBACP;GACA,QAAQ,IAAI,wBAAwB,EAAC,WAAW,KAAI,GAAG,KAAK,UAAU;GAEtE,QAAQ,IAAI,WAAW,KAAK,QAC1B,mBACE,IACA,KAAK,UAAU;IACb,eAAe,KAAA;IACf,kBAAkB,KAAA;GACpB,GACA,KACA,KACA,KAAK,kBACL,KAAA,GACA,iBACF,CACF;GAEA,yBACE,IACA,QAAQ,iBACR,KAAK,mBACL,MACF;EACF,CAAC;EAED,KAAK,MAAM;EACX,KAAK,QAAQ;EACb,KAAK,kBAAkB;CACzB;CAGA,qBACE,IACA,QACA,QACG;EACH,QAAQ,QAAR;GACE,KAAK,YACH,OAAO,KAAK,iBAAiB,IAAI,GAAG;GACtC,KAAK,WACH,OAAO,KAAK,SAAS,IAAI,GAAG;GAC9B,KAAK,aACH,OAAO,KAAK,WAAW,IAAI,GAAG;GAChC;IACE,eACE,KAAK,KACL,IACA,mBAAmB,OAAO,sBAC5B;IACA;EACJ;CACF;CAEA,mBAA4B,OAAO,IAAe,QAAwB;EACxE,KAAK,6BAA6B,+BAA+B;EACjE,IAAI;GACF,MAAM,MAAM,IAAI,IACd,IAAI,OAAO,IACX,IAAI,QAAQ,UAAU,kBACxB;GACA,qBAAqB,IAAI,QAAQ;GACjC,MAAM,SAAS,IAAI,aAAa,IAAI,QAAQ;GAC5C,IAAI,CAAC,QACH,MAAM,IAAI,MAAM,oCAAoC;GAEtD,MAAM,aACJ,MAAM,KAAK,gBAAgB,yBAAyB,MAAM;GAG5D,eAAoB,KAAK,KAAK,IAAI,KAAA,GAAW,UAAU;EACzD,SAAS,KAAK;GACZ,eAAe,KAAK,KAAK,IAAI,KAAK,cAAc;EAClD;CACF;CAEA,WAAoB,OAAO,IAAe,QAAwB;EAChE,IAAI;GACF,MAAM,MAAM,qBAAqB,GAAG;GACpC,IAAI,IAAI,SAAS,WACf,KAAK,6BAA6B,uBAAuB;GAG3D,MAAM,aAAa,MAAM,KAAK,gBAAgB,UAAU,GAAG;GAG3D,0BAA+B,KAAK,KAAK,IAAI,KAAA,GAAW,YAAY,EAClE,SAAS,IAAI,UACf,CAAC;EACH,SAAS,KAAK;GACZ,eAAe,KAAK,KAAK,IAAI,KAAK,cAAc;EAClD;CACF;;;;;;;;CASA,aAAsB,OAAO,IAAe,QAAwB;EAClE,IAAI;EACJ,MAAM,WAAW,aAAa,OAAoC;EAElE,IAAI;GAKF,MAAM,kBAAkB,qBAAqB,IAJ7B,IACd,IAAI,OAAO,IACX,IAAI,QAAQ,UAAU,kBAEqB,CAAA,CAAI,QAAQ;GACzD,IAAI,kBAAkB,gCACpB,MAAM,IAAI,MACR,8CAA8C,+BAA+B,kBACzD,gBAAgB,EACtC;GAGF,MAAM,WAAW,MAAM,0BACrB,KAAK,KACL,IACA,yBACA,UACA,EAAC,SAAS,KAAI,CAChB;GAKA,WAAW,MAAM,OAAO,UACtB,IAAI,IAAI,OAAO,oBAAoB;IACjC,MAAM,EAAC,QAAQ,iBAAgB,IAAI;IACnC,KAAK,6BAA6B,+BAA+B;IACjE,cAAc,MAAM,KAAK,gBAAgB,yBACvC,QACA,EAAC,aAAY,CACf;IACA,KAAU,oBAAoB,aAAa,QAAQ;GACrD,OAAO,IAAI,IAAI,OAAO,sBAAsB;IAC1C,MAAM,MAAyB;KAC7B,GAAG,IAAI;KACP;KACA,WAAW;IACb;IACA,IAAI,IAAI,SAAS,WACf,KAAK,6BAA6B,uBAAuB;IAI3D,cAAc,KAAA;IACd,MAAM,KAAK,gBAAgB,UAAU,KAAK,QAAQ;GACpD;EAEJ,SAAS,KAAK;GACZ,eAAe,KAAK,KAAK,IAAI,KAAK,cAAc;EAClD,UAAU;GAGR,aAAa,OAAO;EACtB;CACF;;;;;;;;CASA,MAAM,oBACJ,aACA,UACA;EACA,IAAI;GACF,WAAW,MAAM,GAAG,WAAW,aAG7B,SAAS,KACP,WAAW,UAAU,CACnB,YACA;IAAC,GAAG;IAAQ,KAAK;GAAU,CAC7B,CAA2B,CAC7B;EAEJ,SAAS,GAAG;GACV,KAAK,IAAI,OAAO,yCAAyC,CAAC;GAC1D,SAAS,KAAK,aAAa,QAAQ,IAAI,IAAI,MAAM,OAAO,CAAC,CAAC,CAAC;EAC7D;CACF;CAEA,yBAAyB;CAEzB,6BAA6B,QAAiB;EAC5C,IAAI,CAAC,KAAK,0BAA0B,KAAK,OAAO,UAAU,GAAG;GAC3D,KAAK,IAAI,OACP,kCAAkC,SAAS,IAAI,OAAO,KAAK,IAC7D;GACA,KAAU,gBACP,IAAI,CAAC,CACL,OAAM,MACL,KAAK,IAAI,OAAO,0CAA0C,CAAC,CAC7D,CAAC,CACA,cAAc,KAAK,KAAK,CAAC;GAE5B,KAAK,yBAAyB;EAChC;CACF;CAEA,WAAoC;EAClC,MAAM,EAAC,gBAAgB,gBAAgB,gBAAe,KAAK;EAC3D,IAAI,mBAAmB,GAGrB,KAAK,6BAA6B;OASlC,cAAmB,WAAW;GAC5B,KAAK,IAAI,OACP,WAAW,eAAe,qCAC5B;GACA,KAAK,OAAO,iBAER,KAAK,6BACH,0BAA0B,eAAe,KAC3C,GACF,cACF;EACF,CAAC;CAEL;;;;;;;;CASA,MAAM,QAAuB;EAC3B,KAAK,cAAc;EACnB,IAAI,KAAK,0BAA0B,KAAK,gBAAgB,OACtD,MAAM,KAAK,gBAAgB,MAAM;CAErC;CAEA,MAAyB,UAAyB;EAChD,IAAI,KAAK,wBACP,MAAM,KAAK,gBAAgB,KAAK;CAEpC;AACF;AAkCA,IAAa,2BAAb,MAEA;CACE;CACA;CACA;CACA;CAEA,YACE,IACA,SACA,UACA,mBACA;EACA,KAAK,MAAM;EACX,KAAK,WAAW;EAGhB,KAAK,YAAY,SAAS,IAAI,UAAU,6BAA6B;GACnE,KAAK;IACJ,iBAAiB;EACpB,CAAC;EACD,KAAK,qBAAqB;CAC5B;CAEA,MAAM,uBAAuB,MAAc;EACzC,IAAI,UAAU,KAAK;EACnB,IAAI,CAAC,SAAS;GACZ,MAAM,UAAU,MAAM,8BACpB,KAAK,UACL,KAAK,SACP;GACA,IAAI,CAAC,SACH,MAAM,IAAI,MAAM,+BAA+B;GAEjD,UAAU,QAAQ,SAAS,KAAK,IAAI,GAAG,QAAQ,KAAK,QAAQ,QAAQ;EACtE;EACA,MAAM,MAAM,IAAI,IAAI,MAAM,OAAO;EACjC,KAAK,IAAI,OAAO,iCAAiC,KAAK;EACtD,OAAO;CACT;CAEA,MAAM,gBACJ,QACA,cAIC;EACD,MAAM,EAAC,UAAU,aAAY,MAAM,KAAK,QAAQ,MAAM;EACtD,SAAS,KAAK,CACZ,oBACA;GAAC;GAAQ,GAAI,eAAe,EAAC,aAAY,IAAI,CAAC;EAAE,CAClD,CAAC;EAED,MAAM,EAAC,MAAM,SAAQ,SAAS,QAAQ;EACtC,MAAM,OAAO,MAAM,KAAK,KAAK;EAC7B,IAAI,CAAC,KAAK,OACR,MAAM,IAAI,MAAM,4CAA4C;EAE9D,MAAM,EAAC,MAAM,QAAO,KAAK;EACzB,IAAI,IAAI,OAAO,YACb,MAAM,IAAI,MAAM,uCAAuC,IAAI,IAAI;EAYjE,OAAO;GAAC,UAVyB,IAAI;GAUnB,UAAA;IARhB,YAAW,QAAO;KAChB,SAAS,KAAK,CAAC,sBAAsB,2BAA2B,GAAG,CAAC,CAAC;KACrE,OAAO,QAAQ,QAAQ,IAAI;IAC7B;IACA,QAAQ,SAAS;IACjB,SAAQ,WAAU,SAAS,OAAO,MAAM;GAGxB;EAAQ;CAC5B;CAEA,MAAM,UACJ,KAC6C;EAC7C,MAAM,EAAC,UAAU,aAAY,MAAM,KAAK,QAAQ,IAAI,MAAM;EAC1D,SAAS,KAAK,CAAC,sBAAsB,2BAA2B,GAAG,CAAC,CAAC;EACrE,OAAO;CACT;;;;;;;;;CAWA,MAAM,QAAQ,QAGX;EACD,MAAM,MAAM,MAAM,KAAK,uBAAuB,cAAc;EAI5D,MAAM,SAAS,IAAI,gBAAgB,EAAC,OAAM,CAAC;EAM3C,MAAM,KAAK,IAAIA,YAAU,MAAM,IAAI,OAAO,SAAS,KAAK,EACtD,kBAAkB,mBACpB,CAAC;EAED,MAAM,WAAW,aAAa,OAA0B;EAOxD,OAAO;GAAC,UAAA,MANe,uBACrB,KAAK,KACL,IACA,2BACA,QACF;GACkB;EAAQ;CAC5B;AACF;AAIA,SAAgB,qBAAqB,KAAwC;CAC3E,MAAM,MAAM,IAAI,IAAI,IAAI,OAAO,IAAI,IAAI,QAAQ,UAAU,kBAAkB;CAC3E,MAAM,kBAAkB,qBAAqB,IAAI,QAAQ;CACzD,MAAM,SAAS,IAAI,UAAU,GAAG;CAEhC,OAAO;EACL;EACA,IAAI,OAAO,IAAI,MAAM,IAAI;EACzB,QAAQ,OAAO,IAAI,UAAU,IAAI;EACjC,MAAM,OAAO,IAAI,QAAQ,KAAK,MAAM,WAAW,WAAW;EAC1D,gBAAgB,OAAO,IAAI,kBAAkB,IAAI;EACjD,WAAW,OAAO,IAAI,aAAa,IAAI;EAIvC,WAAW,OAAO,WAAW,WAAW;CAC1C;AACF;AAEA,SAAS,qBAAqB,UAA0B;CACtD,MAAM,QAAQ,WAAW,KAAK,QAAQ;CACtC,IAAI,CAAC,OACH,MAAM,IAAI,MAAM,iBAAiB,UAAU;CAE7C,MAAM,IAAI,OAAO,MAAM,QAAQ,OAAO;CACtC,IACE,OAAO,MAAM,CAAC,KACd,IAAA,KACA,IAAI,gCAEJ,MAAM,IAAI,MACR,sCAAsC,EAAE,2BACZ,+BAA+B,SAC7D;CAEF,OAAO;AACT;AAKA,SAAS,2BAA2B,KAA0C;CAC5E,MAAM,EACJ,QACA,IACA,MACA,gBACA,WACA,WACA,WACA,eACE;CACJ,OAAO;EACL;EACA;EACA;EACA;EACA;EACA;EACA,GAAI,cAAc,KAAA,IAAY,EAAC,UAAS,IAAI,CAAC;EAC7C,GAAI,eAAe,KAAA,IAAY,EAAC,WAAU,IAAI,CAAC;CACjD;AACF"}